<commit_before>package sde

import (
	"encoding/gob"
	"fmt"
	"io"
	"os"
	"reflect"
	"strings"
)

var (
	ErrTypeDoesNotExist = fmt.Errorf("sde: type does not exist")
	ErrSDEIsNil         = fmt.Errorf("sde: SDE struct was nil")
	ErrTypeIsNil        = fmt.Errorf("sde: SDEType struct was nil")
)

// Load loads an encoding/gob encoded SDE object from file
func Load(filename string) (*SDE, error) {
	if f, err := os.OpenFile(filename, os.O_RDONLY, 0777); err != nil {
		return nil, err
	} else {
		s := &SDE{}
		dec := gob.NewDecoder(f)
		if err := dec.Decode(s); err != nil {
			return nil, err
		}
		return s, nil
	}
	return nil, nil
}

// LoadReader returns an SDE pointer given an io.Reader to read from
func LoadReader(r io.Reader) (*SDE, error) {
	s := &SDE{}
	dec := gob.NewDecoder(r)
	if err := dec.Decode(s); err != nil {
		return nil, err
	}
	return s, nil
}

// Save saves a provided SDE object to disk
func Save(filename string, s *SDE) error {
	if f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0777); err != nil {
		return err
	} else {
		enc := gob.NewEncoder(f)
		if err := enc.Encode(s); err != nil {
			return err
		}
	}
	return nil
}

/*
	SDE is a struct that owns every type for a given SDE.
 		@TODO:
		Add more old methods:
			GetTypeByName
			GetTypeByTag
			...
		Add lookups:
			TypeName
			Attrribute["mDiplsayName"]
			Use a map that isn't gobed and generate on load(use goroutine)
*/
type SDE struct {
	Version  string
	Official bool
	Types    map[int]*SDEType
	Cache    *Cache
}

// Cache is a struct that is included within SDE.
//
// Whenever an SDE file is loaded we populate this and whenever an SDE is
// saved we make the pointer nil.  The struct is supposed to provide
// faster lookups for things like TypeName and mDisplayName
type Cache struct {
	TypeNameLookup    map[string]*SDEType
	DisplayNameLookup map[string]*SDEType
}

// GetType returns a pointer to an SDEType or nil and an error
func (s *SDE) GetType(id int) (sdetype *SDEType, err error) {
	if s == nil {
		return nil, ErrSDEIsNil
	}
	if v, ok := s.Types[id]; ok {
		if v == nil {
			return nil, ErrTypeIsNil
		}
		v.Parent = s
		return v, nil
	} else {
		return nil, ErrTypeDoesNotExist
	}

}

// Search checks for the existance of ss in mDisplayName or TypeName in every type and returns
// a slice of pointers to SDETypes
func (s *SDE) Search(ss string) (sdetypes []*SDEType, err error) {
	out := make([]*SDEType, 0)
	for _, v := range s.Types {
		if strings.Contains(strings.ToLower(v.GetName()), strings.ToLower(ss)) || strings.Contains(strings.ToLower(v.TypeName), strings.ToLower(ss)) {
			out = append(out, v)
		}
	}
	return out, nil
}

// VerifySDEPrint prints the entire list of types/typeids to check for DB corruption
func (s *SDE) VerifySDEPrint() {
	for k, v := range s.Types {
		fmt.Printf("  [%v][%v] %v at %p\n", k, v.TypeID, v.GetName(), v)
	}
}

// FindTypeThatReferences returns any time that refers to the given type
//
// Suprising how fast this method runs
//
// @TODO:
//	When our caching system is finished update this to not iterate all ~3400 types lol
func (s *SDE) FindTypesThatReference(t *SDEType) ([]*SDEType, error) {
	out := make([]*SDEType, 0)
	for _, v := range s.Types {
		for _, attr := range v.Attributes {
			switch tid := attr.(type) {
			case int:
				if tid == t.TypeID && !sdeslicecontains(out, tid) {
					out = append(out, v)
				}
			}
		}
	}
	return out, nil
}

// Size estimates the memory usage of the SDE instance.
func (s *SDE) Size() int {
	base := int(reflect.ValueOf(*s).Type().Size())
	for _, v := range s.Types {
		vv := int(reflect.ValueOf(*v).Type().Size())
		for _, a := range v.Attributes {
			switch reflect.TypeOf(a).Kind() {
			case reflect.String:
				vv += len(a.(string))
				fallthrough
			default:
				vv += int(reflect.ValueOf(a).Type().Size())
			}
		}
		base += vv
	}
	return base
}

// Internal methods

// Use whenever possible.  Benchmarks have shown it takes roughly the same
// amount of time to generate the cache as it does to perform one SDEType
// level lookup.  Let alone one that looks into SDEType.Attributes
func (s *SDE) generateCache() {
	s.Cache = &Cache{}
	s.Cache.TypeNameLookup = make(map[string]*SDEType)
	for _, v := range s.Types {
		s.Cache.TypeNameLookup[v.TypeName] = v
	}
}

func (s *SDE) lookupByTypeName(typeName string) (*SDEType, error) {
	if s.Cache != nil { // Fast lookup
		if v, ok := s.Cache.TypeNameLookup[typeName]; ok {
			return v, nil
		} else {
			return nil, ErrTypeDoesNotExist
		}
	}
	// Default to slow lookup if cache is nil

	for _, v := range s.Types {
		if v.TypeName == typeName {
			return v, nil
		}
	}
	return nil, ErrTypeDoesNotExist
}

/*
	SDEType is a struct representing a single individual type in an SDE.
	@TODO:
		Add old methods.
		Make some cleaner way than before of checking for the existance of *.*... atributes:
		Options:
			1) Substruct them out and create a parser for each(yuck)
			2) Map[string]interface{} parser(ehh)
*/
type SDEType struct {
	TypeID     int
	TypeName   string
	Attributes map[string]interface{}
	Parent     *SDE
}

// GetName returns the string value of Attributes["mDisplayName"] if it exists.  Otherwise we return TypeName
func (s *SDEType) GetName() string {
	if v, ok := s.Attributes["mDisplayName"]; ok {
		return v.(string)
	}
	return s.TypeName
}

// GetAttribute checks if the type has the attribute and returns it.  If it doesn't exist we lookup the weapons projectile type
func (s *SDEType) GetAttribute(attr string) interface{} {
	if v, ok := s.Attributes[attr]; ok {
		return v
	} else {
		if v, ok := s.Attributes["mFireMode0.projectileType"]; ok {
			v, _ := v.(int)
			t, _ := s.Parent.GetType(v) // Ditching error because we don't return an error.  I don't want to break SDETool things yet
			if v, ok := t.Attributes[attr]; ok {
				return v
			}
		}
	}
	return nil
}

// CompareTo prints the differences between two types
func (s *SDEType) CompareTo(t *SDEType) {
	// @TODO: Print differences between typenames/typeid
	for key, value := range s.Attributes {
		if v, ok := t.Attributes[key]; ok {
			if value != v {
				fmt.Printf("CHANGE: %v: %v\n", value, v)
			}
		} else {
			fmt.Printf("ADD: %v: %v\n", key, value)
		}
	}
	for key, value := range t.Attributes {
		if _, ok := s.Attributes[key]; ok {
		} else {
			fmt.Printf("REMOVE: %v: %v\n", key, value)
		}
	}
}

/*
	Helpers
*/

func sdeslicecontains(s []*SDEType, tid int) bool {
	for _, v := range s {
		if v.TypeID == tid {
			return true
		}
	}
	return false
}

func init() {
	gob.Register(SDE{})
	gob.Register(SDEType{})
}
<commit_msg>Nil checks to prevent panics<commit_after>package sde

import (
	"encoding/gob"
	"fmt"
	"io"
	"log"
	"os"
	"reflect"
	"strings"
)

var (
	ErrTypeDoesNotExist = fmt.Errorf("sde: type does not exist")
	ErrSDEIsNil         = fmt.Errorf("sde: SDE struct was nil")
	ErrTypeIsNil        = fmt.Errorf("sde: SDEType struct was nil")
)

// Load loads an encoding/gob encoded SDE object from file
func Load(filename string) (*SDE, error) {
	if f, err := os.OpenFile(filename, os.O_RDONLY, 0777); err != nil {
		return nil, err
	} else {
		s := &SDE{}
		dec := gob.NewDecoder(f)
		if err := dec.Decode(s); err != nil {
			return nil, err
		}
		return s, nil
	}
	return nil, nil
}

// LoadReader returns an SDE pointer given an io.Reader to read from
func LoadReader(r io.Reader) (*SDE, error) {
	s := &SDE{}
	dec := gob.NewDecoder(r)
	if err := dec.Decode(s); err != nil {
		return nil, err
	}
	return s, nil
}

// Save saves a provided SDE object to disk
func Save(filename string, s *SDE) error {
	if f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0777); err != nil {
		return err
	} else {
		enc := gob.NewEncoder(f)
		if err := enc.Encode(s); err != nil {
			return err
		}
	}
	return nil
}

/*
	SDE is a struct that owns every type for a given SDE.
 		@TODO:
		Add more old methods:
			GetTypeByName
			GetTypeByTag
			...
		Add lookups:
			TypeName
			Attrribute["mDiplsayName"]
			Use a map that isn't gobed and generate on load(use goroutine)
*/
type SDE struct {
	Version  string
	Official bool
	Types    map[int]*SDEType
	Cache    *Cache
}

// Cache is a struct that is included within SDE.
//
// Whenever an SDE file is loaded we populate this and whenever an SDE is
// saved we make the pointer nil.  The struct is supposed to provide
// faster lookups for things like TypeName and mDisplayName
type Cache struct {
	TypeNameLookup    map[string]*SDEType
	DisplayNameLookup map[string]*SDEType
}

// GetType returns a pointer to an SDEType or nil and an error
func (s *SDE) GetType(id int) (sdetype *SDEType, err error) {
	if s == nil {
		return nil, ErrSDEIsNil
	}
	if v, ok := s.Types[id]; ok {
		if v == nil {
			return nil, ErrTypeIsNil
		}
		v.Parent = s
		return v, nil
	} else {
		return nil, ErrTypeDoesNotExist
	}

}

// Search checks for the existance of ss in mDisplayName or TypeName in every type and returns
// a slice of pointers to SDETypes
func (s *SDE) Search(ss string) (sdetypes []*SDEType, err error) {
	out := make([]*SDEType, 0)
	for _, v := range s.Types {
		if strings.Contains(strings.ToLower(v.GetName()), strings.ToLower(ss)) || strings.Contains(strings.ToLower(v.TypeName), strings.ToLower(ss)) {
			out = append(out, v)
		}
	}
	return out, nil
}

// VerifySDEPrint prints the entire list of types/typeids to check for DB corruption
func (s *SDE) VerifySDEPrint() {
	for k, v := range s.Types {
		fmt.Printf("  [%v][%v] %v at %p\n", k, v.TypeID, v.GetName(), v)
	}
}

// FindTypeThatReferences returns any time that refers to the given type
//
// Suprising how fast this method runs
//
// @TODO:
//	When our caching system is finished update this to not iterate all ~3400 types lol
func (s *SDE) FindTypesThatReference(t *SDEType) ([]*SDEType, error) {
	out := make([]*SDEType, 0)
	for _, v := range s.Types {
		for _, attr := range v.Attributes {
			switch tid := attr.(type) {
			case int:
				if tid == t.TypeID && !sdeslicecontains(out, tid) {
					out = append(out, v)
				}
			}
		}
	}
	return out, nil
}

// Size estimates the memory usage of the SDE instance.
func (s *SDE) Size() int {
	base := int(reflect.ValueOf(*s).Type().Size())
	for _, v := range s.Types {
		vv := int(reflect.ValueOf(*v).Type().Size())
		for _, a := range v.Attributes {
			switch reflect.TypeOf(a).Kind() {
			case reflect.String:
				vv += len(a.(string))
				fallthrough
			default:
				vv += int(reflect.ValueOf(a).Type().Size())
			}
		}
		base += vv
	}
	return base
}

// Internal methods

// Use whenever possible.  Benchmarks have shown it takes roughly the same
// amount of time to generate the cache as it does to perform one SDEType
// level lookup.  Let alone one that looks into SDEType.Attributes
func (s *SDE) generateCache() {
	s.Cache = &Cache{}
	s.Cache.TypeNameLookup = make(map[string]*SDEType)
	for _, v := range s.Types {
		s.Cache.TypeNameLookup[v.TypeName] = v
	}
}

func (s *SDE) lookupByTypeName(typeName string) (*SDEType, error) {
	if s.Cache != nil { // Fast lookup
		if v, ok := s.Cache.TypeNameLookup[typeName]; ok {
			return v, nil
		} else {
			return nil, ErrTypeDoesNotExist
		}
	}
	// Default to slow lookup if cache is nil

	for _, v := range s.Types {
		if v.TypeName == typeName {
			return v, nil
		}
	}
	return nil, ErrTypeDoesNotExist
}

/*
	SDEType is a struct representing a single individual type in an SDE.
	@TODO:
		Add old methods.
		Make some cleaner way than before of checking for the existance of *.*... atributes:
		Options:
			1) Substruct them out and create a parser for each(yuck)
			2) Map[string]interface{} parser(ehh)
*/
type SDEType struct {
	TypeID     int
	TypeName   string
	Attributes map[string]interface{}
	Parent     *SDE
}

// GetName returns the string value of Attributes["mDisplayName"] if it exists.  Otherwise we return TypeName
func (s *SDEType) GetName() string {
	if v, ok := s.Attributes["mDisplayName"]; ok {
		return v.(string)
	}
	return s.TypeName
}

// GetAttribute checks if the type has the attribute and returns it.  If it doesn't exist we lookup the weapons projectile type
func (s *SDEType) GetAttribute(attr string) interface{} {
	if v, ok := s.Attributes[attr]; ok {
		return v
	} else {
		if v, ok := s.Attributes["mFireMode0.projectileType"]; ok {
			v, _ := v.(int) // Ditch ok.  If it fails we get 0 and t will be nil
			if s.Parent == nil {
				log.Printf("GetAttributes can't lookup projectile.  Parent is nil\n")
				return nil
			}
			t, _ := s.Parent.GetType(v) // Ditching error because we don't return an error.  I don't want to break SDETool things yet
			if t == nil {
				log.Printf("Got nil type.  Returning nil\n")
				return nil
			}
			if v, ok := t.Attributes[attr]; ok {
				return v
			}
		}
	}
	return nil
}

// CompareTo prints the differences between two types
func (s *SDEType) CompareTo(t *SDEType) {
	// @TODO: Print differences between typenames/typeid
	for key, value := range s.Attributes {
		if v, ok := t.Attributes[key]; ok {
			if value != v {
				fmt.Printf("CHANGE: %v: %v\n", value, v)
			}
		} else {
			fmt.Printf("ADD: %v: %v\n", key, value)
		}
	}
	for key, value := range t.Attributes {
		if _, ok := s.Attributes[key]; ok {
		} else {
			fmt.Printf("REMOVE: %v: %v\n", key, value)
		}
	}
}

/*
	Helpers
*/

func sdeslicecontains(s []*SDEType, tid int) bool {
	for _, v := range s {
		if v.TypeID == tid {
			return true
		}
	}
	return false
}

func init() {
	gob.Register(SDE{})
	gob.Register(SDEType{})
}
<|endoftext|>


<commit_before>package morningStar

import (
	"../jsonHttp"
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"sync"
	"time"
)

const IDS_URL = `https://elasticsearch.vibioh.fr/funds/morningStarId/_search?size=8000`
const PERFORMANCE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=1&id=`
const VOLATILITE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=2&id=`
const REFRESH_DELAY = 12
const CONCURRENT_FETCHER = 20

var EMPTY_BYTE = []byte(``)
var ZERO_BYTE = []byte(`0`)
var PERIOD_BYTE = []byte(`.`)
var COMMA_BYTE = []byte(`,`)
var PERCENT_BYTE = []byte(`%`)
var AMP_BYTE = []byte(`&`)
var HTML_AMP_BYTE = []byte(`&`)

var LIST_REQUEST = regexp.MustCompile(`^/list$`)
var PERF_REQUEST = regexp.MustCompile(`^/(.+?)$`)

var ID = regexp.MustCompile(`"_id":"(.*?)"`)
var ISIN = regexp.MustCompile(`ISIN.:(\S+)`)
var LABEL = regexp.MustCompile(`<h1[^>]*?>((?:.|\n)*?)</h1>`)
var RATING = regexp.MustCompile(`<span\sclass=".*?stars([0-9]).*?">`)
var CATEGORY = regexp.MustCompile(`<span[^>]*?>Catégorie</span>.*?<span[^>]*?>(.*?)</span>`)
var PERF_ONE_MONTH = regexp.MustCompile(`<td[^>]*?>1 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_THREE_MONTH = regexp.MustCompile(`<td[^>]*?>3 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_SIX_MONTH = regexp.MustCompile(`<td[^>]*?>6 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_ONE_YEAR = regexp.MustCompile(`<td[^>]*?>1 an</td><td[^>]*?>(.*?)</td>`)
var VOL_3_YEAR = regexp.MustCompile(`<td[^>]*?>Ecart-type 3 ans.?</td><td[^>]*?>(.*?)</td>`)

type SyncedMap struct {
	sync.RWMutex
	performances map[string]Performance
}

func (m *SyncedMap) get(key string) (Performance, bool) {
	m.RLock()
	defer m.RUnlock()

	performance, ok := m.performances[key]
	return performance, ok
}

func (m *SyncedMap) push(key string, performance Performance) {
	m.Lock()
	defer m.Unlock()

	m.performances[key] = performance
}

var PERFORMANCE_CACHE = SyncedMap{performances: make(map[string]Performance)}

type Performance struct {
	Id            string    `json:"id"`
	Isin          string    `json:"isin"`
	Label         string    `json:"label"`
	Category      string    `json:"category"`
	Rating        string    `json:"rating"`
	OneMonth      float64   `json:"1m"`
	ThreeMonth    float64   `json:"3m"`
	SixMonth      float64   `json:"6m"`
	OneYear       float64   `json:"1y"`
	VolThreeYears float64   `json:"v3y"`
	Score         float64   `json:"score"`
	Update        time.Time `json:"ts"`
}

type Results struct {
	Results interface{} `json:"results"`
}

func init() {
	go func() {
		ids := fetchIds()
		refreshCache(ids)

		c := time.Tick(REFRESH_DELAY * time.Hour)
		for range c {
			refreshCache(ids)
		}
	}()
}

func refreshCache(ids [][]byte) {
	log.Print(`Cache refresh - start`)
	defer log.Print(`Cache refresh - end`)
	for _, performance := range retrievePerformances(ids, fetchPerformance) {
		PERFORMANCE_CACHE.push(performance.Id, *performance)
	}
}

func readBody(body io.ReadCloser) ([]byte, error) {
	defer body.Close()
	return ioutil.ReadAll(body)
}

func getBody(url string) ([]byte, error) {
	response, err := http.Get(url)
	if err != nil {
		return nil, errors.New(`Error while retrieving data from ` + url + `:` + err)
	}

	if response.StatusCode >= 400 {
		return nil, errors.New(`Got error ` + strconv.Itoa(response.StatusCode) + ` while getting ` + url + `:` + err)
	}

	body, err := readBody(response.Body)
	if err != nil {
		return nil, errors.New(`Error while reading body of ` + url + `:` + err)
	}

	return body, nil
}

func extractLabel(extract *regexp.Regexp, body []byte, defaultValue []byte) []byte {
	match := extract.FindSubmatch(body)
	if match == nil {
		return defaultValue
	}

	return bytes.Replace(match[1], HTML_AMP_BYTE, AMP_BYTE, -1)
}

func extractPerformance(extract *regexp.Regexp, body []byte) float64 {
	dotResult := bytes.Replace(extractLabel(extract, body, EMPTY_BYTE), COMMA_BYTE, PERIOD_BYTE, -1)
	percentageResult := bytes.Replace(dotResult, PERCENT_BYTE, EMPTY_BYTE, -1)
	trimResult := bytes.TrimSpace(percentageResult)

	result, err := strconv.ParseFloat(string(trimResult), 64)
	if err != nil {
		return 0.0
	}
	return result
}

func cleanId(morningStarId []byte) string {
	return string(bytes.ToLower(morningStarId))
}

func fetchPerformance(morningStarId []byte) (*Performance, error) {
	cleanId := cleanId(morningStarId)
	performanceBody, err := getBody(PERFORMANCE_URL + cleanId)
	if err != nil {
		return nil, err
	}

	volatiliteBody, err := getBody(VOLATILITE_URL + cleanId)
	if err != nil {
		return nil, err
	}

	isin := string(extractLabel(ISIN, performanceBody, EMPTY_BYTE))
	label := string(extractLabel(LABEL, performanceBody, EMPTY_BYTE))
	rating := string(extractLabel(RATING, performanceBody, ZERO_BYTE))
	category := string(extractLabel(CATEGORY, performanceBody, EMPTY_BYTE))
	oneMonth := extractPerformance(PERF_ONE_MONTH, performanceBody)
	threeMonths := extractPerformance(PERF_THREE_MONTH, performanceBody)
	sixMonths := extractPerformance(PERF_SIX_MONTH, performanceBody)
	oneYear := extractPerformance(PERF_ONE_YEAR, performanceBody)
	volThreeYears := extractPerformance(VOL_3_YEAR, volatiliteBody)

	score := (0.25 * oneMonth) + (0.3 * threeMonths) + (0.25 * sixMonths) + (0.2 * oneYear) - (0.1 * volThreeYears)
	scoreTruncated := float64(int(score*100)) / 100

	return &Performance{cleanId, isin, label, category, rating, oneMonth, threeMonths, sixMonths, oneYear, volThreeYears, scoreTruncated, time.Now()}, nil
}

func fetchIds() [][]byte {
	if idsBody, err := getBody(IDS_URL); err != nil {
		log.Print(err)
		return nil
	} else {
		idsMatch := ID.FindAllSubmatch(idsBody, -1)

		ids := make([][]byte, 0, len(idsMatch))
		for _, match := range idsMatch {
			ids = append(ids, match[1])
		}

		return ids
	}
}

func retrievePerformance(morningStarId []byte) (*Performance, error) {
	cleanId := cleanId(morningStarId)

	performance, ok := PERFORMANCE_CACHE.get(cleanId)
	if ok && time.Now().Add(time.Hour*-(REFRESH_DELAY-1)).Before(performance.Update) {
		return &performance, nil
	}

	if performance, err := fetchPerformance(morningStarId); err != nil {
		return nil, err
	} else {
		PERFORMANCE_CACHE.push(cleanId, *performance)
		return performance, nil
	}
}

func concurrentRetrievePerformances(ids [][]byte, wg *sync.WaitGroup, performances chan<- *Performance, method func([]byte) (*Performance, error)) {
	tokens := make(chan int, CONCURRENT_FETCHER)

	clearSemaphores := func() {
		wg.Done()
		<-tokens
	}

	for _, id := range ids {
		tokens <- 1

		go func(morningStarId []byte) {
			defer clearSemaphores()
			if performance, err := method(morningStarId); err == nil {
				performances <- performance
			}
		}(id)
	}
}

func retrievePerformances(ids [][]byte, method func([]byte) (*Performance, error)) []*Performance {
	var wg sync.WaitGroup
	wg.Add(len(ids))

	performances := make(chan *Performance, CONCURRENT_FETCHER)
	go concurrentRetrievePerformances(ids, &wg, performances, method)

	go func() {
		wg.Wait()
		close(performances)
	}()

	results := make([]*Performance, 0, len(ids))
	for performance := range performances {
		results = append(results, performance)
	}

	return results
}

func performanceHandler(w http.ResponseWriter, morningStarId []byte) {
	performance, err := retrievePerformance(morningStarId)

	if err != nil {
		http.Error(w, err.Error(), 500)
	} else {
		jsonHttp.ResponseJson(w, *performance)
	}
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	listBody, err := readBody(r.Body)
	if err != nil {
		http.Error(w, `Error while reading body for list`, 500)
		return
	}

	if len(bytes.TrimSpace(listBody)) == 0 {
		jsonHttp.ResponseJson(w, Results{[0]Performance{}})
		return
	}

	jsonHttp.ResponseJson(w, Results{retrievePerformances(bytes.Split(listBody, COMMA_BYTE), retrievePerformance)})
}

type Handler struct {
}

func (handler Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add(`Access-Control-Allow-Origin`, `*`)
	w.Header().Add(`Access-Control-Allow-Headers`, `Content-Type`)
	w.Header().Add(`Access-Control-Allow-Methods`, `GET, POST`)
	w.Header().Add(`X-Content-Type-Options`, `nosniff`)

	urlPath := []byte(r.URL.Path)

	if LIST_REQUEST.Match(urlPath) {
		listHandler(w, r)
	} else if PERF_REQUEST.Match(urlPath) {
		performanceHandler(w, PERF_REQUEST.FindSubmatch(urlPath)[1])
	}
}
<commit_msg>Adding error information<commit_after>package morningStar

import (
	"../jsonHttp"
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"sync"
	"time"
)

const IDS_URL = `https://elasticsearch.vibioh.fr/funds/morningStarId/_search?size=8000`
const PERFORMANCE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=1&id=`
const VOLATILITE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=2&id=`
const REFRESH_DELAY = 12
const CONCURRENT_FETCHER = 20

var EMPTY_BYTE = []byte(``)
var ZERO_BYTE = []byte(`0`)
var PERIOD_BYTE = []byte(`.`)
var COMMA_BYTE = []byte(`,`)
var PERCENT_BYTE = []byte(`%`)
var AMP_BYTE = []byte(`&`)
var HTML_AMP_BYTE = []byte(`&`)

var LIST_REQUEST = regexp.MustCompile(`^/list$`)
var PERF_REQUEST = regexp.MustCompile(`^/(.+?)$`)

var ID = regexp.MustCompile(`"_id":"(.*?)"`)
var ISIN = regexp.MustCompile(`ISIN.:(\S+)`)
var LABEL = regexp.MustCompile(`<h1[^>]*?>((?:.|\n)*?)</h1>`)
var RATING = regexp.MustCompile(`<span\sclass=".*?stars([0-9]).*?">`)
var CATEGORY = regexp.MustCompile(`<span[^>]*?>Catégorie</span>.*?<span[^>]*?>(.*?)</span>`)
var PERF_ONE_MONTH = regexp.MustCompile(`<td[^>]*?>1 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_THREE_MONTH = regexp.MustCompile(`<td[^>]*?>3 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_SIX_MONTH = regexp.MustCompile(`<td[^>]*?>6 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_ONE_YEAR = regexp.MustCompile(`<td[^>]*?>1 an</td><td[^>]*?>(.*?)</td>`)
var VOL_3_YEAR = regexp.MustCompile(`<td[^>]*?>Ecart-type 3 ans.?</td><td[^>]*?>(.*?)</td>`)

type SyncedMap struct {
	sync.RWMutex
	performances map[string]Performance
}

func (m *SyncedMap) get(key string) (Performance, bool) {
	m.RLock()
	defer m.RUnlock()

	performance, ok := m.performances[key]
	return performance, ok
}

func (m *SyncedMap) push(key string, performance Performance) {
	m.Lock()
	defer m.Unlock()

	m.performances[key] = performance
}

var PERFORMANCE_CACHE = SyncedMap{performances: make(map[string]Performance)}

type Performance struct {
	Id            string    `json:"id"`
	Isin          string    `json:"isin"`
	Label         string    `json:"label"`
	Category      string    `json:"category"`
	Rating        string    `json:"rating"`
	OneMonth      float64   `json:"1m"`
	ThreeMonth    float64   `json:"3m"`
	SixMonth      float64   `json:"6m"`
	OneYear       float64   `json:"1y"`
	VolThreeYears float64   `json:"v3y"`
	Score         float64   `json:"score"`
	Update        time.Time `json:"ts"`
}

type Results struct {
	Results interface{} `json:"results"`
}

func init() {
	go func() {
		ids := fetchIds()
		refreshCache(ids)

		c := time.Tick(REFRESH_DELAY * time.Hour)
		for range c {
			refreshCache(ids)
		}
	}()
}

func refreshCache(ids [][]byte) {
	log.Print(`Cache refresh - start`)
	defer log.Print(`Cache refresh - end`)
	for _, performance := range retrievePerformances(ids, fetchPerformance) {
		PERFORMANCE_CACHE.push(performance.Id, *performance)
	}
}

func readBody(body io.ReadCloser) ([]byte, error) {
	defer body.Close()
	return ioutil.ReadAll(body)
}

func getBody(url string) ([]byte, error) {
	response, err := http.Get(url)
	if err != nil {
		return nil, errors.New(`Error while retrieving data from ` + url + `:` + err.Error())
	}

	if response.StatusCode >= 400 {
		return nil, errors.New(`Got error ` + strconv.Itoa(response.StatusCode) + ` while getting ` + url + `:` + err.Error())
	}

	body, err := readBody(response.Body)
	if err != nil {
		return nil, errors.New(`Error while reading body of ` + url + `:` + err.Error())
	}

	return body, nil
}

func extractLabel(extract *regexp.Regexp, body []byte, defaultValue []byte) []byte {
	match := extract.FindSubmatch(body)
	if match == nil {
		return defaultValue
	}

	return bytes.Replace(match[1], HTML_AMP_BYTE, AMP_BYTE, -1)
}

func extractPerformance(extract *regexp.Regexp, body []byte) float64 {
	dotResult := bytes.Replace(extractLabel(extract, body, EMPTY_BYTE), COMMA_BYTE, PERIOD_BYTE, -1)
	percentageResult := bytes.Replace(dotResult, PERCENT_BYTE, EMPTY_BYTE, -1)
	trimResult := bytes.TrimSpace(percentageResult)

	result, err := strconv.ParseFloat(string(trimResult), 64)
	if err != nil {
		return 0.0
	}
	return result
}

func cleanId(morningStarId []byte) string {
	return string(bytes.ToLower(morningStarId))
}

func fetchPerformance(morningStarId []byte) (*Performance, error) {
	cleanId := cleanId(morningStarId)
	performanceBody, err := getBody(PERFORMANCE_URL + cleanId)
	if err != nil {
		return nil, err
	}

	volatiliteBody, err := getBody(VOLATILITE_URL + cleanId)
	if err != nil {
		return nil, err
	}

	isin := string(extractLabel(ISIN, performanceBody, EMPTY_BYTE))
	label := string(extractLabel(LABEL, performanceBody, EMPTY_BYTE))
	rating := string(extractLabel(RATING, performanceBody, ZERO_BYTE))
	category := string(extractLabel(CATEGORY, performanceBody, EMPTY_BYTE))
	oneMonth := extractPerformance(PERF_ONE_MONTH, performanceBody)
	threeMonths := extractPerformance(PERF_THREE_MONTH, performanceBody)
	sixMonths := extractPerformance(PERF_SIX_MONTH, performanceBody)
	oneYear := extractPerformance(PERF_ONE_YEAR, performanceBody)
	volThreeYears := extractPerformance(VOL_3_YEAR, volatiliteBody)

	score := (0.25 * oneMonth) + (0.3 * threeMonths) + (0.25 * sixMonths) + (0.2 * oneYear) - (0.1 * volThreeYears)
	scoreTruncated := float64(int(score*100)) / 100

	return &Performance{cleanId, isin, label, category, rating, oneMonth, threeMonths, sixMonths, oneYear, volThreeYears, scoreTruncated, time.Now()}, nil
}

func fetchIds() [][]byte {
	if idsBody, err := getBody(IDS_URL); err != nil {
		log.Print(err)
		return nil
	} else {
		idsMatch := ID.FindAllSubmatch(idsBody, -1)

		ids := make([][]byte, 0, len(idsMatch))
		for _, match := range idsMatch {
			ids = append(ids, match[1])
		}

		return ids
	}
}

func retrievePerformance(morningStarId []byte) (*Performance, error) {
	cleanId := cleanId(morningStarId)

	performance, ok := PERFORMANCE_CACHE.get(cleanId)
	if ok && time.Now().Add(time.Hour*-(REFRESH_DELAY-1)).Before(performance.Update) {
		return &performance, nil
	}

	if performance, err := fetchPerformance(morningStarId); err != nil {
		return nil, err
	} else {
		PERFORMANCE_CACHE.push(cleanId, *performance)
		return performance, nil
	}
}

func concurrentRetrievePerformances(ids [][]byte, wg *sync.WaitGroup, performances chan<- *Performance, method func([]byte) (*Performance, error)) {
	tokens := make(chan int, CONCURRENT_FETCHER)

	clearSemaphores := func() {
		wg.Done()
		<-tokens
	}

	for _, id := range ids {
		tokens <- 1

		go func(morningStarId []byte) {
			defer clearSemaphores()
			if performance, err := method(morningStarId); err == nil {
				performances <- performance
			}
		}(id)
	}
}

func retrievePerformances(ids [][]byte, method func([]byte) (*Performance, error)) []*Performance {
	var wg sync.WaitGroup
	wg.Add(len(ids))

	performances := make(chan *Performance, CONCURRENT_FETCHER)
	go concurrentRetrievePerformances(ids, &wg, performances, method)

	go func() {
		wg.Wait()
		close(performances)
	}()

	results := make([]*Performance, 0, len(ids))
	for performance := range performances {
		results = append(results, performance)
	}

	return results
}

func performanceHandler(w http.ResponseWriter, morningStarId []byte) {
	performance, err := retrievePerformance(morningStarId)

	if err != nil {
		http.Error(w, err.Error(), 500)
	} else {
		jsonHttp.ResponseJson(w, *performance)
	}
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	listBody, err := readBody(r.Body)
	if err != nil {
		http.Error(w, `Error while reading body for list: `+err.Error(), 500)
		return
	}

	if len(bytes.TrimSpace(listBody)) == 0 {
		jsonHttp.ResponseJson(w, Results{[0]Performance{}})
		return
	}

	jsonHttp.ResponseJson(w, Results{retrievePerformances(bytes.Split(listBody, COMMA_BYTE), retrievePerformance)})
}

type Handler struct {
}

func (handler Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add(`Access-Control-Allow-Origin`, `*`)
	w.Header().Add(`Access-Control-Allow-Headers`, `Content-Type`)
	w.Header().Add(`Access-Control-Allow-Methods`, `GET, POST`)
	w.Header().Add(`X-Content-Type-Options`, `nosniff`)

	urlPath := []byte(r.URL.Path)

	if LIST_REQUEST.Match(urlPath) {
		listHandler(w, r)
	} else if PERF_REQUEST.Match(urlPath) {
		performanceHandler(w, PERF_REQUEST.FindSubmatch(urlPath)[1])
	}
}
<|endoftext|>


<commit_before>package morningStar

import (
	"../jsonHttp"
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"sync"
	"time"
)

const PERFORMANCE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=1&id=`
const VOLATILITE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=2&id=`
const REFRESH_DELAY = 18
const CONCURRENT_FETCHER = 20

var EMPTY_BYTE = []byte(``)
var ZERO_BYTE = []byte(`0`)
var PERIOD_BYTE = []byte(`.`)
var COMMA_BYTE = []byte(`,`)
var PERCENT_BYTE = []byte(`%`)
var AMP_BYTE = []byte(`&`)
var HTML_AMP_BYTE = []byte(`&`)

var LIST_REQUEST = regexp.MustCompile(`^/list$`)
var PERF_REQUEST = regexp.MustCompile(`^/(.+?)$`)

var ISIN = regexp.MustCompile(`ISIN.:(\S+)`)
var LABEL = regexp.MustCompile(`<h1[^>]*?>((?:.|\n)*?)</h1>`)
var RATING = regexp.MustCompile(`<span\sclass=".*?stars([0-9]).*?">`)
var CATEGORY = regexp.MustCompile(`<span[^>]*?>Catégorie</span>.*?<span[^>]*?>(.*?)</span>`)
var PERF_ONE_MONTH = regexp.MustCompile(`<td[^>]*?>1 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_THREE_MONTH = regexp.MustCompile(`<td[^>]*?>3 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_SIX_MONTH = regexp.MustCompile(`<td[^>]*?>6 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_ONE_YEAR = regexp.MustCompile(`<td[^>]*?>1 an</td><td[^>]*?>(.*?)</td>`)
var VOL_3_YEAR = regexp.MustCompile(`<td[^>]*?>Ecart-type 3 ans.?</td><td[^>]*?>(.*?)</td>`)

type SyncedMap struct {
	sync.RWMutex
	performances map[string]Performance
}

func (m *SyncedMap) get(key string) (Performance, bool) {
	m.RLock()
	defer m.RUnlock()

	performance, ok := m.performances[key]
	return performance, ok
}

func (m *SyncedMap) push(key string, performance Performance) {
	m.Lock()
	defer m.Unlock()

	m.performances[key] = performance
}

var PERFORMANCE_CACHE = SyncedMap{performances: make(map[string]Performance)}

type Performance struct {
	Id            string    `json:"id"`
	Isin          string    `json:"isin"`
	Label         string    `json:"label"`
	Category      string    `json:"category"`
	Rating        string    `json:"rating"`
	OneMonth      float64   `json:"1m"`
	ThreeMonth    float64   `json:"3m"`
	SixMonth      float64   `json:"6m"`
	OneYear       float64   `json:"1y"`
	VolThreeYears float64   `json:"v3y"`
	Score         float64   `json:"score"`
	Update        time.Time `json:"ts"`
}

type Results struct {
	Results interface{} `json:"results"`
}

func readBody(body io.ReadCloser) ([]byte, error) {
	defer body.Close()
	return ioutil.ReadAll(body)
}

func getBody(url string) ([]byte, error) {
	response, err := http.Get(url)
	if err != nil {
		return nil, errors.New(`Error while retrieving data from ` + url)
	}

	if response.StatusCode >= 400 {
		return nil, errors.New(`Got error ` + strconv.Itoa(response.StatusCode) + ` while getting ` + url)
	}

	body, err := readBody(response.Body)
	if err != nil {
		return nil, errors.New(`Error while reading body of ` + url)
	}

	return body, nil
}

func extractLabel(extract *regexp.Regexp, body []byte, defaultValue []byte) []byte {
	match := extract.FindSubmatch(body)
	if match == nil {
		return defaultValue
	}

	return bytes.Replace(match[1], HTML_AMP_BYTE, AMP_BYTE, -1)
}

func extractPerformance(extract *regexp.Regexp, body []byte) float64 {
	dotResult := bytes.Replace(extractLabel(extract, body, EMPTY_BYTE), COMMA_BYTE, PERIOD_BYTE, -1)
	percentageResult := bytes.Replace(dotResult, PERCENT_BYTE, EMPTY_BYTE, -1)
	trimResult := bytes.TrimSpace(percentageResult)

	result, err := strconv.ParseFloat(string(trimResult), 64)
	if err != nil {
		return 0.0
	}
	return result
}

func fetchPerformance(morningStarId string) (*Performance, error) {
	performanceBody, err := getBody(PERFORMANCE_URL + morningStarId)
	if err != nil {
		return nil, err
	}

	volatiliteBody, err := getBody(VOLATILITE_URL + morningStarId)
	if err != nil {
		return nil, err
	}

	isin := string(extractLabel(ISIN, performanceBody, EMPTY_BYTE))
	label := string(extractLabel(LABEL, performanceBody, EMPTY_BYTE))
	rating := string(extractLabel(RATING, performanceBody, ZERO_BYTE))
	category := string(extractLabel(CATEGORY, performanceBody, EMPTY_BYTE))
	oneMonth := extractPerformance(PERF_ONE_MONTH, performanceBody)
	threeMonths := extractPerformance(PERF_THREE_MONTH, performanceBody)
	sixMonths := extractPerformance(PERF_SIX_MONTH, performanceBody)
	oneYear := extractPerformance(PERF_ONE_YEAR, performanceBody)
	volThreeYears := extractPerformance(VOL_3_YEAR, volatiliteBody)

	score := (0.25 * oneMonth) + (0.3 * threeMonths) + (0.25 * sixMonths) + (0.2 * oneYear) - (0.1 * volThreeYears)
	scoreTruncated := float64(int(score*100)) / 100

	return &Performance{morningStarId, isin, label, category, rating, oneMonth, threeMonths, sixMonths, oneYear, volThreeYears, scoreTruncated, time.Now()}, nil
}

func SinglePerformance(morningStarId []byte) (*Performance, error) {
	cleanId := string(bytes.ToLower(morningStarId))

	performance, ok := PERFORMANCE_CACHE.get(cleanId)

	if ok && time.Now().Add(time.Hour*-REFRESH_DELAY).Before(performance.Update) {
		return &performance, nil
	}

	if performance, err := fetchPerformance(cleanId); err != nil {
		return nil, err
	} else {
		PERFORMANCE_CACHE.push(cleanId, *performance)
		return performance, nil
	}
}

func singlePerformanceHandler(w http.ResponseWriter, morningStarId []byte) {
	performance, err := SinglePerformance(morningStarId)

	if err != nil {
		http.Error(w, err.Error(), 500)
	} else {
		jsonHttp.ResponseJson(w, *performance)
	}
}

func allPerformances(ids [][]byte, wg *sync.WaitGroup, performances chan<- *Performance) {
	tokens := make(chan int, CONCURRENT_FETCHER)

	clearSemaphores := func() {
		wg.Done()
		<-tokens
	}

	for _, id := range ids {
		tokens <- 1

		go func(morningStarId []byte) {
			defer clearSemaphores()
			if performance, err := SinglePerformance(morningStarId); err == nil {
				performances <- performance
			}
		}(id)
	}
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	listBody, err := readBody(r.Body)
	if err != nil {
		http.Error(w, `Error while reading body for list`, 500)
		return
	}

	if len(bytes.TrimSpace(listBody)) == 0 {
		jsonHttp.ResponseJson(w, Results{[0]Performance{}})
		return
	}

	performances := make(chan *Performance, CONCURRENT_FETCHER)
	ids := bytes.Split(listBody, COMMA_BYTE)

	var wg sync.WaitGroup
	wg.Add(len(ids))
	go allPerformances(ids, &wg, performances)

	go func() {
		wg.Wait()
		close(performances)
	}()

	results := make([]*Performance, 0, len(ids))
	for performance := range performances {
		results = append(results, performance)
	}

	jsonHttp.ResponseJson(w, Results{results})
}

type Handler struct {
}

func (handler Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add(`Access-Control-Allow-Origin`, `*`)
	w.Header().Add(`Access-Control-Allow-Headers`, `Content-Type`)
	w.Header().Add(`Access-Control-Allow-Methods`, `GET, POST`)
	w.Header().Add(`X-Content-Type-Options`, `nosniff`)

	urlPath := []byte(r.URL.Path)

	if LIST_REQUEST.Match(urlPath) {
		listHandler(w, r)
	} else if PERF_REQUEST.Match(urlPath) {
		singlePerformanceHandler(w, PERF_REQUEST.FindSubmatch(urlPath)[1])
	}
}
<commit_msg>Adding initializing during startup and periodic refresh<commit_after>package morningStar

import (
	"../jsonHttp"
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"sync"
	"time"
)

const IDS_URL = `https://elasticsearch.vibioh.fr/funds/morningStarId/_search?size=8000`
const PERFORMANCE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=1&id=`
const VOLATILITE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=2&id=`
const REFRESH_DELAY = 12
const CONCURRENT_FETCHER = 20

var EMPTY_BYTE = []byte(``)
var ZERO_BYTE = []byte(`0`)
var PERIOD_BYTE = []byte(`.`)
var COMMA_BYTE = []byte(`,`)
var PERCENT_BYTE = []byte(`%`)
var AMP_BYTE = []byte(`&`)
var HTML_AMP_BYTE = []byte(`&`)

var LIST_REQUEST = regexp.MustCompile(`^/list$`)
var PERF_REQUEST = regexp.MustCompile(`^/(.+?)$`)

var ID = regexp.MustCompile(`"_id":"(.*?)"`)
var ISIN = regexp.MustCompile(`ISIN.:(\S+)`)
var LABEL = regexp.MustCompile(`<h1[^>]*?>((?:.|\n)*?)</h1>`)
var RATING = regexp.MustCompile(`<span\sclass=".*?stars([0-9]).*?">`)
var CATEGORY = regexp.MustCompile(`<span[^>]*?>Catégorie</span>.*?<span[^>]*?>(.*?)</span>`)
var PERF_ONE_MONTH = regexp.MustCompile(`<td[^>]*?>1 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_THREE_MONTH = regexp.MustCompile(`<td[^>]*?>3 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_SIX_MONTH = regexp.MustCompile(`<td[^>]*?>6 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_ONE_YEAR = regexp.MustCompile(`<td[^>]*?>1 an</td><td[^>]*?>(.*?)</td>`)
var VOL_3_YEAR = regexp.MustCompile(`<td[^>]*?>Ecart-type 3 ans.?</td><td[^>]*?>(.*?)</td>`)

type SyncedMap struct {
	sync.RWMutex
	performances map[string]Performance
}

func (m *SyncedMap) get(key string) (Performance, bool) {
	m.RLock()
	defer m.RUnlock()

	performance, ok := m.performances[key]
	return performance, ok
}

func (m *SyncedMap) push(key string, performance Performance) {
	m.Lock()
	defer m.Unlock()

	m.performances[key] = performance
}

var PERFORMANCE_CACHE = SyncedMap{performances: make(map[string]Performance)}

type Performance struct {
	Id            string    `json:"id"`
	Isin          string    `json:"isin"`
	Label         string    `json:"label"`
	Category      string    `json:"category"`
	Rating        string    `json:"rating"`
	OneMonth      float64   `json:"1m"`
	ThreeMonth    float64   `json:"3m"`
	SixMonth      float64   `json:"6m"`
	OneYear       float64   `json:"1y"`
	VolThreeYears float64   `json:"v3y"`
	Score         float64   `json:"score"`
	Update        time.Time `json:"ts"`
}

type Results struct {
	Results interface{} `json:"results"`
}

func init() {
	go func() {
		ids := fetchIds()
		refreshCache(ids)

		c := time.Tick(REFRESH_DELAY * time.Hour)
		for range c {
			refreshCache(ids)
		}
	}()
}

func refreshCache(ids [][]byte) {
	log.Print(`Cache refresh - start`)
	defer log.Print(`Cache refresh - end`)
	for _, performance := range retrievePerformances(ids, fetchPerformance) {
		PERFORMANCE_CACHE.push(performance.Id, *performance)
	}
}

func readBody(body io.ReadCloser) ([]byte, error) {
	defer body.Close()
	return ioutil.ReadAll(body)
}

func getBody(url string) ([]byte, error) {
	response, err := http.Get(url)
	if err != nil {
		return nil, errors.New(`Error while retrieving data from ` + url)
	}

	if response.StatusCode >= 400 {
		return nil, errors.New(`Got error ` + strconv.Itoa(response.StatusCode) + ` while getting ` + url)
	}

	body, err := readBody(response.Body)
	if err != nil {
		return nil, errors.New(`Error while reading body of ` + url)
	}

	return body, nil
}

func extractLabel(extract *regexp.Regexp, body []byte, defaultValue []byte) []byte {
	match := extract.FindSubmatch(body)
	if match == nil {
		return defaultValue
	}

	return bytes.Replace(match[1], HTML_AMP_BYTE, AMP_BYTE, -1)
}

func extractPerformance(extract *regexp.Regexp, body []byte) float64 {
	dotResult := bytes.Replace(extractLabel(extract, body, EMPTY_BYTE), COMMA_BYTE, PERIOD_BYTE, -1)
	percentageResult := bytes.Replace(dotResult, PERCENT_BYTE, EMPTY_BYTE, -1)
	trimResult := bytes.TrimSpace(percentageResult)

	result, err := strconv.ParseFloat(string(trimResult), 64)
	if err != nil {
		return 0.0
	}
	return result
}

func cleanId(morningStarId []byte) string {
	return string(bytes.ToLower(morningStarId))
}

func fetchPerformance(morningStarId []byte) (*Performance, error) {
	cleanId := cleanId(morningStarId)
	performanceBody, err := getBody(PERFORMANCE_URL + cleanId)
	if err != nil {
		return nil, err
	}

	volatiliteBody, err := getBody(VOLATILITE_URL + cleanId)
	if err != nil {
		return nil, err
	}

	isin := string(extractLabel(ISIN, performanceBody, EMPTY_BYTE))
	label := string(extractLabel(LABEL, performanceBody, EMPTY_BYTE))
	rating := string(extractLabel(RATING, performanceBody, ZERO_BYTE))
	category := string(extractLabel(CATEGORY, performanceBody, EMPTY_BYTE))
	oneMonth := extractPerformance(PERF_ONE_MONTH, performanceBody)
	threeMonths := extractPerformance(PERF_THREE_MONTH, performanceBody)
	sixMonths := extractPerformance(PERF_SIX_MONTH, performanceBody)
	oneYear := extractPerformance(PERF_ONE_YEAR, performanceBody)
	volThreeYears := extractPerformance(VOL_3_YEAR, volatiliteBody)

	score := (0.25 * oneMonth) + (0.3 * threeMonths) + (0.25 * sixMonths) + (0.2 * oneYear) - (0.1 * volThreeYears)
	scoreTruncated := float64(int(score*100)) / 100

	return &Performance{cleanId, isin, label, category, rating, oneMonth, threeMonths, sixMonths, oneYear, volThreeYears, scoreTruncated, time.Now()}, nil
}

func fetchIds() [][]byte {
	if idsBody, err := getBody(IDS_URL); err != nil {
		log.Print(err)
		return nil
	} else {
		idsMatch := ID.FindAllSubmatch(idsBody, -1)

		ids := make([][]byte, 0, len(idsMatch))
		for _, match := range idsMatch {
			ids = append(ids, match[1])
		}

		return ids
	}
}

func retrievePerformance(morningStarId []byte) (*Performance, error) {
	cleanId := cleanId(morningStarId)

	performance, ok := PERFORMANCE_CACHE.get(cleanId)
	if ok && time.Now().Add(time.Hour*-(REFRESH_DELAY-1)).Before(performance.Update) {
		return &performance, nil
	}

	if performance, err := fetchPerformance(morningStarId); err != nil {
		return nil, err
	} else {
		PERFORMANCE_CACHE.push(cleanId, *performance)
		return performance, nil
	}
}

func concurrentRetrievePerformances(ids [][]byte, wg *sync.WaitGroup, performances chan<- *Performance, method func([]byte) (*Performance, error)) {
	tokens := make(chan int, CONCURRENT_FETCHER)

	clearSemaphores := func() {
		wg.Done()
		<-tokens
	}

	for _, id := range ids {
		tokens <- 1

		go func(morningStarId []byte) {
			defer clearSemaphores()
			if performance, err := method(morningStarId); err == nil {
				performances <- performance
			}
		}(id)
	}
}

func retrievePerformances(ids [][]byte, method func([]byte) (*Performance, error)) []*Performance {
	var wg sync.WaitGroup
	wg.Add(len(ids))

	performances := make(chan *Performance, CONCURRENT_FETCHER)
	go concurrentRetrievePerformances(ids, &wg, performances, method)

	go func() {
		wg.Wait()
		close(performances)
	}()

	results := make([]*Performance, 0, len(ids))
	for performance := range performances {
		results = append(results, performance)
	}

	return results
}

func performanceHandler(w http.ResponseWriter, morningStarId []byte) {
	performance, err := retrievePerformance(morningStarId)

	if err != nil {
		http.Error(w, err.Error(), 500)
	} else {
		jsonHttp.ResponseJson(w, *performance)
	}
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	listBody, err := readBody(r.Body)
	if err != nil {
		http.Error(w, `Error while reading body for list`, 500)
		return
	}

	if len(bytes.TrimSpace(listBody)) == 0 {
		jsonHttp.ResponseJson(w, Results{[0]Performance{}})
		return
	}

	jsonHttp.ResponseJson(w, Results{retrievePerformances(bytes.Split(listBody, COMMA_BYTE), retrievePerformance)})
}

type Handler struct {
}

func (handler Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add(`Access-Control-Allow-Origin`, `*`)
	w.Header().Add(`Access-Control-Allow-Headers`, `Content-Type`)
	w.Header().Add(`Access-Control-Allow-Methods`, `GET, POST`)
	w.Header().Add(`X-Content-Type-Options`, `nosniff`)

	urlPath := []byte(r.URL.Path)

	if LIST_REQUEST.Match(urlPath) {
		listHandler(w, r)
	} else if PERF_REQUEST.Match(urlPath) {
		performanceHandler(w, PERF_REQUEST.FindSubmatch(urlPath)[1])
	}
}
<|endoftext|>


<commit_before>package forge

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
)

// Section struct holds a map of values
type Section struct {
	comments []string
	includes []string
	parent   *Section
	values   map[string]Value
}

// NewSection will create and initialize a new Section
func NewSection() *Section {
	return &Section{
		comments: make([]string, 0),
		includes: make([]string, 0),
		values:   make(map[string]Value),
	}
}

func newChildSection(parent *Section) *Section {
	return &Section{
		comments: make([]string, 0),
		includes: make([]string, 0),
		parent:   parent,
		values:   make(map[string]Value),
	}
}

// AddComment will append a new comment into the section
func (section *Section) AddComment(comment string) {
	section.comments = append(section.comments, comment)
}

// AddInclude will append a new filename into the section
func (section *Section) AddInclude(filename string) {
	section.includes = append(section.includes, filename)
}

// GetComments will return all the comments were defined for this Section
func (section *Section) GetComments() []string {
	return section.comments
}

// GetIncludes will return the filenames of all the includes were parsed for this Section
func (section *Section) GetIncludes() []string {
	return section.includes
}

// GetType will respond with the ValueType of this Section (hint, always SECTION)
func (section *Section) GetType() ValueType {
	return SECTION
}

// GetValue retrieves the raw underlying value stored in this Section
func (section *Section) GetValue() interface{} {
	return section.values
}

// UpdateValue updates the raw underlying value stored in this Section
func (section *Section) UpdateValue(value interface{}) error {
	switch value.(type) {
	case map[string]Value:
		section.values = value.(map[string]Value)
		return nil
	}

	msg := fmt.Sprintf("unsupported type, %s must be of type `map[string]Value`", value)
	return errors.New(msg)
}

// AddSection adds a new child section to this Section with the provided name
func (section *Section) AddSection(name string) *Section {
	childSection := newChildSection(section)
	section.values[name] = childSection
	return childSection
}

// Exists returns true when a value stored under the key exists
func (section *Section) Exists(name string) bool {
	_, err := section.Get(name)
	return err == nil
}

// Get the value (Primative or Section) stored under the name
// will respond with an error if the value does not exist
func (section *Section) Get(name string) (Value, error) {
	value, ok := section.values[name]
	var err error
	if ok == false {
		err = errors.New("value does not exist")
	}
	return value, err
}

// GetBoolean will try to get the value stored under name as a bool
// will respond with an error if the value does not exist or cannot be converted to a bool
func (section *Section) GetBoolean(name string) (bool, error) {
	value, err := section.Get(name)
	if err != nil {
		return false, err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsBoolean()
	case *Section:
		return true, nil
	}

	return false, errors.New("could not convert unknown value to boolean")
}

// GetFloat will try to get the value stored under name as a float64
// will respond with an error if the value does not exist or cannot be converted to a float64
func (section *Section) GetFloat(name string) (float64, error) {
	value, err := section.Get(name)
	if err != nil {
		return float64(0), err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsFloat()
	}

	return float64(0), errors.New("could not convert non-primative value to float")
}

// GetInteger will try to get the value stored under name as a int64
// will respond with an error if the value does not exist or cannot be converted to a int64
func (section *Section) GetInteger(name string) (int64, error) {
	value, err := section.Get(name)
	if err != nil {
		return int64(0), err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsInteger()
	}

	return int64(0), errors.New("could not convert non-primative value to integer")
}

// GetSection will try to get the value stored under name as a Section
// will respond with an error if the value does not exist or is not a Section
func (section *Section) GetSection(name string) (*Section, error) {
	value, err := section.Get(name)
	if err != nil {
		return nil, err
	}

	if value.GetType() == SECTION {
		return value.(*Section), nil
	}
	return nil, errors.New("could not fetch value as section")
}

// GetString will try to get the value stored under name as a string
// will respond with an error if the value does not exist or cannot be converted to a string
func (section *Section) GetString(name string) (string, error) {
	value, err := section.Get(name)
	if err != nil {
		return "", err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsString()
	}

	return "", errors.New("could not convert non-primative value to string")
}

// GetParent will get the parent section associated with this Section or nil
// if it does not have one
func (section *Section) GetParent() *Section {
	return section.parent
}

// HasParent will return true if this Section has a parent
func (section *Section) HasParent() bool {
	return section.parent != nil
}

// Keys will return back a list of all setting names in this Section
func (section *Section) Keys() []string {
	keys := make([]string, 0)
	for key, _ := range section.values {
		keys = append(keys, key)
	}

	return keys
}

// Set will set a value (Primative or Section) to the provided name
func (section *Section) Set(name string, value Value) {
	section.values[name] = value
}

// SetBoolean will set the value for name as a bool
func (section *Section) SetBoolean(name string, value bool) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.values[name] = NewBoolean(value)
	}
}

// SetFloat will set the value for name as a float64
func (section *Section) SetFloat(name string, value float64) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.values[name] = NewFloat(value)
	}
}

// SetInteger will set the value for name as a int64
func (section *Section) SetInteger(name string, value int64) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.values[name] = NewInteger(value)
	}
}

// SetNull will set the value for name as nil
func (section *Section) SetNull(name string) {
	current, err := section.Get(name)

	// Already is a Null, nothing to do
	if err == nil && current.GetType() == NULL {
		return
	}
	section.Set(name, NewNull())
}

// SetString will set the value for name as a string
func (section *Section) SetString(name string, value string) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.Set(name, NewString(value))
	}
}

// Resolve will recursively try to fetch the provided value and will respond
// with an error if the name does not exist or tries to be resolved through
// a non-section value
func (section *Section) Resolve(name string) (Value, error) {
	// Used only in error state return value
	var value Value

	parts := strings.Split(name, ".")
	if len(parts) == 0 {
		return value, errors.New("no name provided")
	}

	var current Value
	current = section
	for _, part := range parts {
		if current.GetType() != SECTION {
			return value, errors.New("trying to resolve value from non-section")
		}

		nextCurrent, err := current.(*Section).Get(part)
		if err != nil {
			return value, errors.New("could not find value in section")
		}
		current = nextCurrent
	}
	return current, nil
}

// ToJSON will convert this Section and all it's underlying values and Sections
// into JSON as a []byte
func (section *Section) ToJSON() ([]byte, error) {
	data := section.ToMap()
	return json.Marshal(data)
}

// ToMap will convert this Section and all it's underlying values and Sections into
// a map[string]interface{}
func (section *Section) ToMap() map[string]interface{} {
	output := make(map[string]interface{})

	for key, value := range section.values {
		if value.GetType() == SECTION {
			output[key] = value.(*Section).ToMap()
		} else {
			output[key] = value.GetValue()
		}
	}
	return output
}
<commit_msg>fix linting errors for Section.Keys method<commit_after>package forge

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
)

// Section struct holds a map of values
type Section struct {
	comments []string
	includes []string
	parent   *Section
	values   map[string]Value
}

// NewSection will create and initialize a new Section
func NewSection() *Section {
	return &Section{
		comments: make([]string, 0),
		includes: make([]string, 0),
		values:   make(map[string]Value),
	}
}

func newChildSection(parent *Section) *Section {
	return &Section{
		comments: make([]string, 0),
		includes: make([]string, 0),
		parent:   parent,
		values:   make(map[string]Value),
	}
}

// AddComment will append a new comment into the section
func (section *Section) AddComment(comment string) {
	section.comments = append(section.comments, comment)
}

// AddInclude will append a new filename into the section
func (section *Section) AddInclude(filename string) {
	section.includes = append(section.includes, filename)
}

// GetComments will return all the comments were defined for this Section
func (section *Section) GetComments() []string {
	return section.comments
}

// GetIncludes will return the filenames of all the includes were parsed for this Section
func (section *Section) GetIncludes() []string {
	return section.includes
}

// GetType will respond with the ValueType of this Section (hint, always SECTION)
func (section *Section) GetType() ValueType {
	return SECTION
}

// GetValue retrieves the raw underlying value stored in this Section
func (section *Section) GetValue() interface{} {
	return section.values
}

// UpdateValue updates the raw underlying value stored in this Section
func (section *Section) UpdateValue(value interface{}) error {
	switch value.(type) {
	case map[string]Value:
		section.values = value.(map[string]Value)
		return nil
	}

	msg := fmt.Sprintf("unsupported type, %s must be of type `map[string]Value`", value)
	return errors.New(msg)
}

// AddSection adds a new child section to this Section with the provided name
func (section *Section) AddSection(name string) *Section {
	childSection := newChildSection(section)
	section.values[name] = childSection
	return childSection
}

// Exists returns true when a value stored under the key exists
func (section *Section) Exists(name string) bool {
	_, err := section.Get(name)
	return err == nil
}

// Get the value (Primative or Section) stored under the name
// will respond with an error if the value does not exist
func (section *Section) Get(name string) (Value, error) {
	value, ok := section.values[name]
	var err error
	if ok == false {
		err = errors.New("value does not exist")
	}
	return value, err
}

// GetBoolean will try to get the value stored under name as a bool
// will respond with an error if the value does not exist or cannot be converted to a bool
func (section *Section) GetBoolean(name string) (bool, error) {
	value, err := section.Get(name)
	if err != nil {
		return false, err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsBoolean()
	case *Section:
		return true, nil
	}

	return false, errors.New("could not convert unknown value to boolean")
}

// GetFloat will try to get the value stored under name as a float64
// will respond with an error if the value does not exist or cannot be converted to a float64
func (section *Section) GetFloat(name string) (float64, error) {
	value, err := section.Get(name)
	if err != nil {
		return float64(0), err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsFloat()
	}

	return float64(0), errors.New("could not convert non-primative value to float")
}

// GetInteger will try to get the value stored under name as a int64
// will respond with an error if the value does not exist or cannot be converted to a int64
func (section *Section) GetInteger(name string) (int64, error) {
	value, err := section.Get(name)
	if err != nil {
		return int64(0), err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsInteger()
	}

	return int64(0), errors.New("could not convert non-primative value to integer")
}

// GetSection will try to get the value stored under name as a Section
// will respond with an error if the value does not exist or is not a Section
func (section *Section) GetSection(name string) (*Section, error) {
	value, err := section.Get(name)
	if err != nil {
		return nil, err
	}

	if value.GetType() == SECTION {
		return value.(*Section), nil
	}
	return nil, errors.New("could not fetch value as section")
}

// GetString will try to get the value stored under name as a string
// will respond with an error if the value does not exist or cannot be converted to a string
func (section *Section) GetString(name string) (string, error) {
	value, err := section.Get(name)
	if err != nil {
		return "", err
	}

	switch value.(type) {
	case *Primative:
		return value.(*Primative).AsString()
	}

	return "", errors.New("could not convert non-primative value to string")
}

// GetParent will get the parent section associated with this Section or nil
// if it does not have one
func (section *Section) GetParent() *Section {
	return section.parent
}

// HasParent will return true if this Section has a parent
func (section *Section) HasParent() bool {
	return section.parent != nil
}

// Keys will return back a list of all setting names in this Section
func (section *Section) Keys() []string {
	var keys []string
	for key := range section.values {
		keys = append(keys, key)
	}

	return keys
}

// Set will set a value (Primative or Section) to the provided name
func (section *Section) Set(name string, value Value) {
	section.values[name] = value
}

// SetBoolean will set the value for name as a bool
func (section *Section) SetBoolean(name string, value bool) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.values[name] = NewBoolean(value)
	}
}

// SetFloat will set the value for name as a float64
func (section *Section) SetFloat(name string, value float64) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.values[name] = NewFloat(value)
	}
}

// SetInteger will set the value for name as a int64
func (section *Section) SetInteger(name string, value int64) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.values[name] = NewInteger(value)
	}
}

// SetNull will set the value for name as nil
func (section *Section) SetNull(name string) {
	current, err := section.Get(name)

	// Already is a Null, nothing to do
	if err == nil && current.GetType() == NULL {
		return
	}
	section.Set(name, NewNull())
}

// SetString will set the value for name as a string
func (section *Section) SetString(name string, value string) {
	current, err := section.Get(name)

	// Exists just update the value/type
	if err == nil {
		current.UpdateValue(value)
	} else {
		section.Set(name, NewString(value))
	}
}

// Resolve will recursively try to fetch the provided value and will respond
// with an error if the name does not exist or tries to be resolved through
// a non-section value
func (section *Section) Resolve(name string) (Value, error) {
	// Used only in error state return value
	var value Value

	parts := strings.Split(name, ".")
	if len(parts) == 0 {
		return value, errors.New("no name provided")
	}

	var current Value
	current = section
	for _, part := range parts {
		if current.GetType() != SECTION {
			return value, errors.New("trying to resolve value from non-section")
		}

		nextCurrent, err := current.(*Section).Get(part)
		if err != nil {
			return value, errors.New("could not find value in section")
		}
		current = nextCurrent
	}
	return current, nil
}

// ToJSON will convert this Section and all it's underlying values and Sections
// into JSON as a []byte
func (section *Section) ToJSON() ([]byte, error) {
	data := section.ToMap()
	return json.Marshal(data)
}

// ToMap will convert this Section and all it's underlying values and Sections into
// a map[string]interface{}
func (section *Section) ToMap() map[string]interface{} {
	output := make(map[string]interface{})

	for key, value := range section.values {
		if value.GetType() == SECTION {
			output[key] = value.(*Section).ToMap()
		} else {
			output[key] = value.GetValue()
		}
	}
	return output
}
<|endoftext|>


<commit_before>// Copyright (c) 2012-today José Nieto, https://xiam.io
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"github.com/go-chi/chi"
	"github.com/go-chi/chi/middleware"
	"github.com/malfunkt/hyperfox/pkg/plugins/capture"
	"log"
	"net"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"strconv"
	"strings"
	"upper.io/db.v3"
)

var (
	reUnsafeChars   = regexp.MustCompile(`[^0-9a-zA-Z\s\.]`)
	reUnsafeFile    = regexp.MustCompile(`[^0-9a-zA-Z-_]`)
	reRepeatedDash  = regexp.MustCompile(`-+`)
	reRepeatedBlank = regexp.MustCompile(`\s+`)
)

const (
	serviceBindHost = `0.0.0.0`
)

const (
	defaultPageSize = uint(10)
)

type pullResponse struct {
	Records []capture.RecordMeta `json:"records"`
	Pages   uint                 `json:"pages"`
	Page    uint                 `json:"page"`
}

func replyCode(w http.ResponseWriter, httpCode int) {
	w.WriteHeader(httpCode)
	_, _ = w.Write([]byte(http.StatusText(httpCode)))
}

type writeOption uint8

const (
	writeNone         writeOption = 0
	writeWire                     = 1
	writeEmbed                    = 2
	writeRequestBody              = 4
	writeResponseBody             = 8
)

func replyBinary(w http.ResponseWriter, r *http.Request, record *capture.Record, opts writeOption) {
	var (
		optRequestBody  = opts&writeRequestBody > 0
		optResponseBody = opts&writeResponseBody > 0
		optWire         = opts&writeWire > 0
		optEmbed        = opts&writeEmbed > 0
	)

	if opts == writeNone {
		return
	}

	if optRequestBody && optResponseBody {
		// we should never have both options enabled at the same time.
		replyCode(w, http.StatusInternalServerError)
		return
	}

	u, err := url.Parse(record.URL)
	if err != nil {
		log.Printf("url.Parse: %w", err)
		replyCode(w, http.StatusInternalServerError)
		return
	}

	basename := u.Host + "-" + path.Base(u.Path)
	basename = reUnsafeFile.ReplaceAllString(basename, "-")
	basename = strings.Trim(reRepeatedDash.ReplaceAllString(basename, "-"), "-")
	if path.Ext(basename) == "" {
		basename = basename + ".txt"
	}

	buf := bytes.NewBuffer(nil)

	if optWire {
		var headers http.Header
		if optRequestBody {
			headers = record.RequestHeader.Header
		}
		if optResponseBody {
			headers = record.Header.Header
		}
		for k, vv := range headers {
			for _, v := range vv {
				buf.WriteString(fmt.Sprintf("%s: %s\r\n", k, v))
			}
		}
		buf.WriteString("\r\n")
	}

	if optRequestBody || optResponseBody {

		if optRequestBody {
			buf.Write(record.RequestBody)
		}
		if optResponseBody {
			buf.Write(record.Body)
		}

		if optEmbed {
			embedContentType := "text/plain; charset=utf-8"
			w.Header().Set(
				"Content-Type",
				embedContentType,
			)
			w.Write(buf.Bytes())
		} else {
			w.Header().Set(
				"Content-Disposition",
				fmt.Sprintf(`attachment; filename="%s"`, basename),
			)
			http.ServeContent(w, r, "", record.DateEnd, bytes.NewReader(buf.Bytes()))
		}
	}

}

func replyJSON(w http.ResponseWriter, data interface{}) {
	var buf []byte
	var err error

	if buf, err = json.Marshal(data); err != nil {
		log.Printf("Marshal: %q", err)
		replyCode(w, http.StatusInternalServerError)
		return
	}

	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-Type", "application/json")

	w.WriteHeader(http.StatusOK)
	_, _ = w.Write(buf)
}

func getCaptureRecord(uuid string) (*capture.Record, error) {
	var record capture.Record

	res := storage.Find(
		db.Cond{"uuid": uuid},
	).Select(
		"uuid",
		"origin",
		"method",
		"status",
		"content_type",
		"content_length",
		"host",
		"url",
		"path",
		"scheme",
		"date_start",
		"date_end",
		"time_taken",
		"header",
		"request_header",
		db.Raw("hex(body) AS body"),
		db.Raw("hex(request_body) AS request_body"),
	)

	if err := res.One(&record); err != nil {
		return nil, err
	}

	{
		body, err := hex.DecodeString(string(record.RequestBody))
		if err != nil {
			return nil, err
		}
		record.RequestBody = body
	}

	{
		body, err := hex.DecodeString(string(record.Body))
		if err != nil {
			return nil, err
		}
		record.Body = body
	}

	return &record, nil
}

func recordMetaHandler(w http.ResponseWriter, r *http.Request) {
	uuid := chi.URLParam(r, "uuid")

	record, err := getCaptureRecord(uuid)
	if err != nil {
		log.Printf("getCaptureRecord: %q", err)
		replyCode(w, http.StatusInternalServerError)
		return
	}

	replyJSON(w, record.RecordMeta)
}

func recordHandler(w http.ResponseWriter, r *http.Request, opts writeOption) {
	uuid := chi.URLParam(r, "uuid")

	record, err := getCaptureRecord(uuid)
	if err != nil {
		log.Printf("getCaptureRecord: %q", err)
		replyCode(w, http.StatusInternalServerError)
		return
	}

	replyBinary(w, r, record, opts)
}

func requestContentHandler(w http.ResponseWriter, r *http.Request) {
	recordHandler(w, r, writeRequestBody)
}

func requestWireHandler(w http.ResponseWriter, r *http.Request) {
	recordHandler(w, r, writeRequestBody|writeWire)
}

func requestEmbedHandler(w http.ResponseWriter, r *http.Request) {
	recordHandler(w, r, writeRequestBody|writeEmbed)
}

func responseContentHandler(w http.ResponseWriter, r *http.Request) {
	recordHandler(w, r, writeResponseBody)
}

func responseWireHandler(w http.ResponseWriter, r *http.Request) {
	recordHandler(w, r, writeResponseBody|writeWire)
}

func responseEmbedHandler(w http.ResponseWriter, r *http.Request) {
	recordHandler(w, r, writeResponseBody|writeEmbed)
}

// capturesHandler service serves paginated requests.
func capturesHandler(w http.ResponseWriter, r *http.Request) {
	var err error
	var response pullResponse

	q := chi.URLParam(r, "q")

	q = reUnsafeChars.ReplaceAllString(q, " ")
	q = reRepeatedBlank.ReplaceAllString(q, " ")

	page := uint(1)
	{
		i, err := strconv.ParseUint(r.URL.Query().Get("page"), 10, 64)
		if err == nil {
			page = uint(i)
		}
	}

	pageSize := defaultPageSize
	{
		i, err := strconv.ParseUint(r.URL.Query().Get("page_size"), 10, 64)
		if err == nil {
			pageSize = uint(i)
		}
	}

	// Result set
	res := storage.Find().
		Select(
			"id",
			"uuid",
			"origin",
			"method",
			"status",
			"content_type",
			"content_length",
			"host",
			"url",
			"path",
			"scheme",
			"date_start",
			"date_end",
			"time_taken",
		).
		OrderBy("id")

	if q != "" {
		terms := strings.Split(q, " ")
		conds := db.Or()

		for _, term := range terms {
			conds = conds.Or(
				db.Or(
					db.Cond{"host LIKE": "%" + term + "%"},
					db.Cond{"origin LIKE": "%" + term + "%"},
					db.Cond{"path LIKE": "%" + term + "%"},
					db.Cond{"content_type LIKE": "%" + term + "%"},
					db.Cond{"method": term},
					db.Cond{"scheme": term},
					db.Cond{"status": term},
				),
			)
		}

		res = res.Where(conds)
	}

	res = res.Paginate(pageSize).Page(page)

	// Pulling information page.
	if err = res.All(&response.Records); err != nil {
		log.Printf("res.All: %q", err)
		replyCode(w, http.StatusInternalServerError)
		return
	}

	// Getting total number of pages.
	response.Page = page
	response.Pages, _ = res.TotalPages()

	replyJSON(w, response)
}

// startServices starts an http server that provides websocket and rest
// services.
func startServices() error {

	r := chi.NewRouter()
	r.Use(middleware.Logger)

	r.Route("/records", func(r chi.Router) {
		r.Get("/", capturesHandler)

		r.Route("/{uuid}", func(r chi.Router) {
			r.Get("/", recordMetaHandler)

			r.Route("/request", func(r chi.Router) {
				r.Get("/", requestContentHandler)
				r.Get("/raw", requestWireHandler)
				r.Get("/embed", requestEmbedHandler)
			})

			r.Route("/response", func(r chi.Router) {
				r.Get("/", responseContentHandler)
				r.Get("/raw", responseWireHandler)
				r.Get("/embed", responseEmbedHandler)
			})
		})
	})

	r.HandleFunc("/live", liveHandler)

	log.Printf("Starting (local) API server...")

	// Looking for a port to listen to.
	ln, err := net.Listen("tcp", serviceBindHost+":8899")
	if err != nil {
		log.Fatal("net.Listen: ", err)
	}

	addr := fmt.Sprintf("%s:%d", serviceBindHost, ln.Addr().(*net.TCPAddr).Port)
	log.Printf("Watch live capture at http://live.hyperfox.org/#/?source=%s", addr)

	srv := &http.Server{
		Addr:    addr,
		Handler: r,
	}

	// Serving API.
	go func() {
		if err := srv.Serve(ln); err != nil {
			panic(err.Error())
		}
	}()

	return err
}
<commit_msg>use r.URL.Query() instead of chi.URLParam<commit_after>// Copyright (c) 2012-today José Nieto, https://xiam.io
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package main

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"github.com/go-chi/chi"
	"github.com/go-chi/chi/middleware"
	"github.com/malfunkt/hyperfox/pkg/plugins/capture"
	"log"
	"net"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"strconv"
	"strings"
	"upper.io/db.v3"
)

var (
	reUnsafeChars   = regexp.MustCompile(`[^0-9a-zA-Z\s\.]`)
	reUnsafeFile    = regexp.MustCompile(`[^0-9a-zA-Z-_]`)
	reRepeatedDash  = regexp.MustCompile(`-+`)
	reRepeatedBlank = regexp.MustCompile(`\s+`)
)

const (
	serviceBindHost = `0.0.0.0`
)

const (
	defaultPageSize = uint(10)
)

type pullResponse struct {
	Records []capture.RecordMeta `json:"records"`
	Pages   uint                 `json:"pages"`
	Page    uint                 `json:"page"`
}

func replyCode(w http.ResponseWriter, httpCode int) {
	w.WriteHeader(httpCode)
	_, _ = w.Write([]byte(http.StatusText(httpCode)))
}

type writeOption uint8

const (
	writeNone         writeOption = 0
	writeWire                     = 1
	writeEmbed                    = 2
	writeRequestBody              = 4
	writeResponseBody             = 8
)

func replyBinary(w http.ResponseWriter, r *http.Request, record *capture.Record, opts writeOption) {
	var (
		optRequestBody  = opts&writeRequestBody > 0
		optResponseBody = opts&writeResponseBody > 0
		optWire         = opts&writeWire > 0
		optEmbed        = opts&writeEmbed > 0
	)

	if opts == writeNone {
		return
	}

	if optRequestBody && optResponseBody {
		// we should never have both options enabled at the same time.
		replyCode(w, http.StatusInternalServerError)
		return
	}

	u, err := url.Parse(record.URL)
	if err != nil {
		log.Printf("url.Parse: %w", err)
		replyCode(w, http.StatusInternalServerError)
		return
	}

	basename := u.Host + "-" + path.Base(u.Path)
	basename = reUnsafeFile.ReplaceAllString(basename, "-")
	basename = strings.Trim(reRepeatedDash.ReplaceAllString(basename, "-"), "-")
	if path.Ext(basename) == "" {
		basename = basename + ".txt"
	}

	buf := bytes.NewBuffer(nil)

	if optWire {
		var headers http.Header
		if optRequestBody {
			headers = record.RequestHeader.Header
		}
		if optResponseBody {
			headers = record.Header.Header
		}
		for k, vv := range headers {
			for _, v := range vv {
				buf.WriteString(fmt.Sprintf("%s: %s\r\n", k, v))
			}
		}
		buf.WriteString("\r\n")
	}

	if optRequestBody || optResponseBody {

		if optRequestBody {
			buf.Write(record.RequestBody)
		}
		if optResponseBody {
			buf.Write(record.Body)
		}

		if optEmbed {
			embedContentType := "text/plain; charset=utf-8"
			w.Header().Set(
				"Content-Type",
				embedContentType,
			)
			w.Write(buf.Bytes())
		} else {
			w.Header().Set(
				"Content-Disposition",
				fmt.Sprintf(`attachment; filename="%s"`, basename),
			)
			http.ServeContent(w, r, "", record.DateEnd, bytes.NewReader(buf.Bytes()))
		}
	}

}

func replyJSON(w http.ResponseWriter, data interface{}) {
	var buf []byte
	var err error

	if buf, err = json.Marshal(data); err != nil {
		log.Printf("Marshal: %q", err)
		replyCode(w, http.StatusInternalServerError)
		return
	}

	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-Type", "application/json")

	w.WriteHeader(http.StatusOK)
	_, _ = w.Write(buf)
}

func getCaptureRecord(uuid string) (*capture.Record, error) {
	var record capture.Record

	res := storage.Find(
		db.Cond{"uuid": uuid},
	).Select(
		"uuid",
		"origin",
		"method",
		"status",
		"content_type",
		"content_length",
		"host",
		"url",
		"path",
		"scheme",
		"date_start",
		"date_end",
		"time_taken",
		"header",
		"request_header",
		db.Raw("hex(body) AS body"),
		db.Raw("hex(request_body) AS request_body"),
	)

	if err := res.One(&record); err != nil {
		return nil, err
	}

	{
		body, err := hex.DecodeString(string(record.RequestBody))
		if err != nil {
			return nil, err
		}
		record.RequestBody = body
	}

	{
		body, err := hex.DecodeString(string(record.Body))
		if err != nil {
			return nil, err
		}
		record.Body = body
	}

	return &record, nil
}

func recordMetaHandler(w http.ResponseWriter, r *http.Request) {
	uuid := chi.URLParam(r, "uuid")

	record, err := getCaptureRecord(uuid)
	if err != nil {
		log.Printf("getCaptureRecord: %q", err)
		replyCode(w, http.StatusInternalServerError)
		return
	}

	replyJSON(w, record.RecordMeta)
}

func recordHandler(w http.ResponseWriter, r *http.Request, opts writeOption) {
	uuid := chi.URLParam(r, "uuid")

	record, err := getCaptureRecord(uuid)
	if err != nil {
		log.Printf("getCaptureRecord: %q", err)
		replyCode(w, http.StatusInternalServerError)
		return
	}

	replyBinary(w, r, record, opts)
}

func requestContentHandler(w http.ResponseWriter, r *http.Request) {
	recordHandler(w, r, writeRequestBody)
}

func requestWireHandler(w http.ResponseWriter, r *http.Request) {
	recordHandler(w, r, writeRequestBody|writeWire)
}

func requestEmbedHandler(w http.ResponseWriter, r *http.Request) {
	recordHandler(w, r, writeRequestBody|writeEmbed)
}

func responseContentHandler(w http.ResponseWriter, r *http.Request) {
	recordHandler(w, r, writeResponseBody)
}

func responseWireHandler(w http.ResponseWriter, r *http.Request) {
	recordHandler(w, r, writeResponseBody|writeWire)
}

func responseEmbedHandler(w http.ResponseWriter, r *http.Request) {
	recordHandler(w, r, writeResponseBody|writeEmbed)
}

// capturesHandler service serves paginated requests.
func capturesHandler(w http.ResponseWriter, r *http.Request) {
	var err error
	var response pullResponse

	q := r.URL.Query().Get("q")

	q = reUnsafeChars.ReplaceAllString(q, " ")
	q = reRepeatedBlank.ReplaceAllString(q, " ")

	page := uint(1)
	{
		i, err := strconv.ParseUint(r.URL.Query().Get("page"), 10, 64)
		if err == nil {
			page = uint(i)
		}
	}

	pageSize := defaultPageSize
	{
		i, err := strconv.ParseUint(r.URL.Query().Get("page_size"), 10, 64)
		if err == nil {
			pageSize = uint(i)
		}
	}

	// Result set
	res := storage.Find().
		Select(
			"id",
			"uuid",
			"origin",
			"method",
			"status",
			"content_type",
			"content_length",
			"host",
			"url",
			"path",
			"scheme",
			"date_start",
			"date_end",
			"time_taken",
		).
		OrderBy("id")

	if q != "" {
		terms := strings.Split(q, " ")
		conds := db.Or()

		for _, term := range terms {
			conds = conds.Or(
				db.Or(
					db.Cond{"host LIKE": "%" + term + "%"},
					db.Cond{"origin LIKE": "%" + term + "%"},
					db.Cond{"path LIKE": "%" + term + "%"},
					db.Cond{"content_type LIKE": "%" + term + "%"},
					db.Cond{"method": term},
					db.Cond{"scheme": term},
					db.Cond{"status": term},
				),
			)
		}

		res = res.Where(conds)
	}

	res = res.Paginate(pageSize).Page(page)

	// Pulling information page.
	if err = res.All(&response.Records); err != nil {
		log.Printf("res.All: %q", err)
		replyCode(w, http.StatusInternalServerError)
		return
	}

	// Getting total number of pages.
	response.Page = page
	response.Pages, _ = res.TotalPages()

	replyJSON(w, response)
}

// startServices starts an http server that provides websocket and rest
// services.
func startServices() error {

	r := chi.NewRouter()
	r.Use(middleware.Logger)

	r.Route("/records", func(r chi.Router) {
		r.Get("/", capturesHandler)

		r.Route("/{uuid}", func(r chi.Router) {
			r.Get("/", recordMetaHandler)

			r.Route("/request", func(r chi.Router) {
				r.Get("/", requestContentHandler)
				r.Get("/raw", requestWireHandler)
				r.Get("/embed", requestEmbedHandler)
			})

			r.Route("/response", func(r chi.Router) {
				r.Get("/", responseContentHandler)
				r.Get("/raw", responseWireHandler)
				r.Get("/embed", responseEmbedHandler)
			})
		})
	})

	r.HandleFunc("/live", liveHandler)

	log.Printf("Starting (local) API server...")

	// Looking for a port to listen to.
	ln, err := net.Listen("tcp", serviceBindHost+":8899")
	if err != nil {
		log.Fatal("net.Listen: ", err)
	}

	addr := fmt.Sprintf("%s:%d", serviceBindHost, ln.Addr().(*net.TCPAddr).Port)
	log.Printf("Watch live capture at http://live.hyperfox.org/#/?source=%s", addr)

	srv := &http.Server{
		Addr:    addr,
		Handler: r,
	}

	// Serving API.
	go func() {
		if err := srv.Serve(ln); err != nil {
			panic(err.Error())
		}
	}()

	return err
}
<|endoftext|>