<commit_before>package main

import (
	"fmt"
	"os"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/blevesearch/bleve"
	"github.com/blevesearch/bleve/analysis/analyzers/custom_analyzer"
	"github.com/blevesearch/bleve/analysis/char_filters/html_char_filter"
	"github.com/blevesearch/bleve/analysis/language/fr"
	"github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter"
	"github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter"
	"github.com/blevesearch/bleve/analysis/token_map"
	"github.com/blevesearch/bleve/analysis/tokenizers/exception"
	bleveuni "github.com/blevesearch/bleve/analysis/tokenizers/unicode"
	"github.com/blevesearch/bleve/index/store/boltdb"
	"github.com/blevesearch/bleve/index/upside_down"
	"github.com/pmezard/apec/jstruct"
	"github.com/pquerna/ffjson/ffjson"
)

type offerResult struct {
	Id    string
	Offer *jstruct.JsonOffer
	Err   error
}

func loadOffers(store *Store) ([]*jstruct.JsonOffer, error) {
	ids, err := store.List()
	if err != nil {
		return nil, err
	}
	sort.Strings(ids)
	pending := make(chan string, len(ids))
	for _, id := range ids {
		pending <- id
	}
	close(pending)

	results := make(chan offerResult, len(ids))
	running := &sync.WaitGroup{}
	jobs := 4
	for i := 0; i < jobs; i++ {
		running.Add(1)
		go func() {
			defer running.Done()
			for id := range pending {
				offer, err := getStoreJsonOffer(store, id)
				results <- offerResult{
					Id:    id,
					Offer: offer,
					Err:   err,
				}
			}
		}()
	}
	go func() {
		running.Wait()
		close(results)
	}()

	offers := []*jstruct.JsonOffer{}
	for r := range results {
		if r.Err != nil {
			fmt.Printf("loading error for %s: %s\n", r.Id, r.Err)
			continue
		}
		offers = append(offers, r.Offer)
	}
	return offers, nil
}

type Offer struct {
	Account   string
	Id        string    `json:"id"`
	HTML      string    `json:"html"`
	Title     string    `json:"title"`
	MinSalary int       `json:"min_salary"`
	MaxSalary int       `json:"max_salary"`
	Date      time.Time `json:"date"`
	URL       string
	Location  string `json:"location"`
}

const (
	ApecURL = "https://cadres.apec.fr/home/mes-offres/recherche-des-offres-demploi/" +
		"liste-des-offres-demploi/detail-de-loffre-demploi.html?numIdOffre="
)

func convertOffer(offer *jstruct.JsonOffer) (*Offer, error) {
	r := &Offer{
		Account:  offer.Account,
		Id:       offer.Id,
		HTML:     offer.HTML,
		Title:    offer.Title,
		URL:      ApecURL + offer.Id,
		Location: offer.Location,
	}
	min, max, err := parseSalary(offer.Salary)
	if err != nil {
		return nil, fmt.Errorf("cannot parse salary %q: %s", offer.Salary, err)
	}
	d, err := time.Parse("2006-01-02T15:04:05.000+0000", offer.Date)
	if err != nil {
		return nil, err
	}
	r.Date = d
	r.MinSalary = min
	r.MaxSalary = max
	return r, nil
}

func getStoreJsonOffer(store *Store, id string) (*jstruct.JsonOffer, error) {
	data, err := store.Get(id)
	if err != nil {
		return nil, err
	}
	if data == nil {
		return nil, nil
	}
	js := &jstruct.JsonOffer{}
	err = ffjson.Unmarshal(data, js)
	return js, err
}

func getStoreOffer(store *Store, id string) (*Offer, error) {
	js, err := getStoreJsonOffer(store, id)
	if err != nil || js == nil {
		return nil, err
	}
	return convertOffer(js)
}

func convertOffers(offers []*jstruct.JsonOffer) ([]*Offer, error) {
	result := make([]*Offer, 0, len(offers))
	for _, o := range offers {
		r, err := convertOffer(o)
		if err != nil {
			return nil, err
		}
		result = append(result, r)
	}
	return result, nil
}

var (
	indexExceptions = []string{
		"c++",
		"c#",
	}
	stopWords = []interface{}{
		"h", // The H in H/F
		"f", // The F in H/F
		// Generic garbage
		"mision",
		"post",
		"entrepris",
		"suivi",
	}
)

func NewOfferIndex(dir string) (bleve.Index, error) {
	err := os.RemoveAll(dir)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	parts := []string{}
	for _, exc := range indexExceptions {
		parts = append(parts, regexp.QuoteMeta(exc))
	}
	pattern := strings.Join(parts, "|")
	pattern = "(?i)(?:" + pattern + ")"

	m := bleve.NewIndexMapping()
	apecTokenizer := "apec"
	err = m.AddCustomTokenizer(apecTokenizer, map[string]interface{}{
		"type":       exception.Name,
		"exceptions": []string{pattern},
		"tokenizer":  bleveuni.Name,
	})
	if err != nil {
		return nil, err
	}

	apecTokens := "apec_tokens"
	err = m.AddCustomTokenMap(apecTokens, map[string]interface{}{
		"type":   token_map.Name,
		"tokens": stopWords,
	})
	if err != nil {
		return nil, err
	}

	apecStop := "apec_stop"
	err = m.AddCustomTokenFilter(apecStop, map[string]interface{}{
		"type":           stop_tokens_filter.Name,
		"stop_token_map": apecTokens,
	})
	if err != nil {
		return nil, err
	}

	frTokens := []string{
		lower_case_filter.Name,
		fr.ElisionName,
		fr.StopName,
		fr.LightStemmerName,
		apecStop,
	}
	fr := map[string]interface{}{
		"type":          custom_analyzer.Name,
		"tokenizer":     apecTokenizer,
		"token_filters": frTokens,
	}
	frHtml := map[string]interface{}{
		"type": custom_analyzer.Name,
		"char_filters": []string{
			html_char_filter.Name,
		},
		"tokenizer":     apecTokenizer,
		"token_filters": frTokens,
	}
	err = m.AddCustomAnalyzer("fr", fr)
	if err != nil {
		return nil, fmt.Errorf("failed to register analyzer fr: %s", err)
	}
	err = m.AddCustomAnalyzer("fr_html", frHtml)
	if err != nil {
		return nil, fmt.Errorf("failed to register analyzer fr_html: %s", err)
	}

	htmlFr := bleve.NewTextFieldMapping()
	htmlFr.Store = false
	htmlFr.IncludeInAll = false
	htmlFr.IncludeTermVectors = false
	htmlFr.Analyzer = "fr_html"

	textFr := bleve.NewTextFieldMapping()
	textFr.Store = false
	textFr.IncludeInAll = false
	textFr.IncludeTermVectors = false
	textFr.Analyzer = "fr"

	textAll := bleve.NewTextFieldMapping()
	textAll.Store = false
	textAll.IncludeInAll = true
	textAll.IncludeTermVectors = false

	date := bleve.NewDateTimeFieldMapping()
	date.Index = false
	date.Store = true
	date.IncludeInAll = false
	date.IncludeTermVectors = false

	offer := bleve.NewDocumentStaticMapping()
	offer.Dynamic = false
	offer.AddFieldMappingsAt("html", htmlFr)
	offer.AddFieldMappingsAt("title", textFr)
	offer.AddFieldMappingsAt("date", date)

	m.AddDocumentMapping("offer", offer)
	m.DefaultMapping = offer

	index, err := bleve.NewUsing(dir, m, upside_down.Name, boltdb.Name,
		map[string]interface{}{
			"nosync": true,
		})
	if err != nil {
		return nil, err
	}
	return index, nil
}

func OpenOfferIndex(path string) (bleve.Index, error) {
	return bleve.OpenUsing(path, map[string]interface{}{
		"nosync": false,
	})
}

var (
	indexCmd     = app.Command("index", "index APEC offers")
	indexMaxSize = indexCmd.Flag("max-count", "maximum number of items to index").
			Short('n').Default("0").Int()
	// Work around kingpin messing with boolean flags starting with --no-xxx (#54)
	indexIndex = indexCmd.Flag("index", "enable indexing (use --no-index to disable it)").
			Default("true").Bool()
	indexMinQuota = indexCmd.Flag("min-quota",
		"stop geocoding when call quota moves below supplied value").Default("500").Int()
	indexDocId = indexCmd.Flag("id", "index only specified document").String()
)

func indexOffers(cfg *Config) error {
	store, err := OpenStore(cfg.Store())
	if err != nil {
		return err
	}
	defer store.Close()
	rawOffers, err := loadOffers(store)
	if err != nil {
		return err
	}
	if *indexDocId != "" {
		kept := []*jstruct.JsonOffer{}
		for _, o := range rawOffers {
			if o.Id == *indexDocId {
				kept = append(kept, o)
			}
		}
		rawOffers = kept
	}
	offers, err := convertOffers(rawOffers)
	if err != nil {
		return err
	}
	if *indexMaxSize > 0 && len(offers) > *indexMaxSize {
		offers = offers[:*indexMaxSize]
	}

	rejected := 0
	geocodingKey := cfg.GeocodingKey()
	if geocodingKey != "" {
		geocoder, err := NewGeocoder(geocodingKey, cfg.Geocoder())
		if err != nil {
			return err
		}
		defer geocoder.Close()
		rejected, err = geocodeOffers(store, geocoder, offers, *indexMinQuota)
		if err != nil {
			return err
		}
		fmt.Printf("%d rejected geocoding\n", rejected)
	}
	if *indexIndex {
		index, err := NewOfferIndex(cfg.Index())
		if err != nil {
			return err
		}
		start := time.Now()
		indexed := 0
		for i, offer := range offers {
			if (i+1)%500 == 0 {
				now := time.Now()
				elapsed := float64(now.Sub(start)) / float64(time.Second)
				fmt.Printf("%d indexed, %.1f/s\n", i+1, float64(i+1)/elapsed)
			}
			err = index.Index(offer.Id, offer)
			if err != nil {
				return err
			}
			indexed += 1
		}
		err = index.Close()
		if err != nil {
			return err
		}
		end := time.Now()
		fmt.Printf("%d/%d documents indexed in %.2fs\n", indexed, len(offers),
			float64(end.Sub(start))/float64(time.Second))
	}
	return nil
}
<commit_msg>index: inline function specific struct<commit_after>package main

import (
	"fmt"
	"os"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/blevesearch/bleve"
	"github.com/blevesearch/bleve/analysis/analyzers/custom_analyzer"
	"github.com/blevesearch/bleve/analysis/char_filters/html_char_filter"
	"github.com/blevesearch/bleve/analysis/language/fr"
	"github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter"
	"github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter"
	"github.com/blevesearch/bleve/analysis/token_map"
	"github.com/blevesearch/bleve/analysis/tokenizers/exception"
	bleveuni "github.com/blevesearch/bleve/analysis/tokenizers/unicode"
	"github.com/blevesearch/bleve/index/store/boltdb"
	"github.com/blevesearch/bleve/index/upside_down"
	"github.com/pmezard/apec/jstruct"
	"github.com/pquerna/ffjson/ffjson"
)

func loadOffers(store *Store) ([]*jstruct.JsonOffer, error) {
	type offerResult struct {
		Id    string
		Offer *jstruct.JsonOffer
		Err   error
	}

	ids, err := store.List()
	if err != nil {
		return nil, err
	}
	sort.Strings(ids)
	pending := make(chan string, len(ids))
	for _, id := range ids {
		pending <- id
	}
	close(pending)

	results := make(chan offerResult, len(ids))
	running := &sync.WaitGroup{}
	jobs := 4
	for i := 0; i < jobs; i++ {
		running.Add(1)
		go func() {
			defer running.Done()
			for id := range pending {
				offer, err := getStoreJsonOffer(store, id)
				results <- offerResult{
					Id:    id,
					Offer: offer,
					Err:   err,
				}
			}
		}()
	}
	go func() {
		running.Wait()
		close(results)
	}()

	offers := []*jstruct.JsonOffer{}
	for r := range results {
		if r.Err != nil {
			fmt.Printf("loading error for %s: %s\n", r.Id, r.Err)
			continue
		}
		offers = append(offers, r.Offer)
	}
	return offers, nil
}

type Offer struct {
	Account   string
	Id        string    `json:"id"`
	HTML      string    `json:"html"`
	Title     string    `json:"title"`
	MinSalary int       `json:"min_salary"`
	MaxSalary int       `json:"max_salary"`
	Date      time.Time `json:"date"`
	URL       string
	Location  string `json:"location"`
}

const (
	ApecURL = "https://cadres.apec.fr/home/mes-offres/recherche-des-offres-demploi/" +
		"liste-des-offres-demploi/detail-de-loffre-demploi.html?numIdOffre="
)

func convertOffer(offer *jstruct.JsonOffer) (*Offer, error) {
	r := &Offer{
		Account:  offer.Account,
		Id:       offer.Id,
		HTML:     offer.HTML,
		Title:    offer.Title,
		URL:      ApecURL + offer.Id,
		Location: offer.Location,
	}
	min, max, err := parseSalary(offer.Salary)
	if err != nil {
		return nil, fmt.Errorf("cannot parse salary %q: %s", offer.Salary, err)
	}
	d, err := time.Parse("2006-01-02T15:04:05.000+0000", offer.Date)
	if err != nil {
		return nil, err
	}
	r.Date = d
	r.MinSalary = min
	r.MaxSalary = max
	return r, nil
}

func getStoreJsonOffer(store *Store, id string) (*jstruct.JsonOffer, error) {
	data, err := store.Get(id)
	if err != nil {
		return nil, err
	}
	if data == nil {
		return nil, nil
	}
	js := &jstruct.JsonOffer{}
	err = ffjson.Unmarshal(data, js)
	return js, err
}

func getStoreOffer(store *Store, id string) (*Offer, error) {
	js, err := getStoreJsonOffer(store, id)
	if err != nil || js == nil {
		return nil, err
	}
	return convertOffer(js)
}

func convertOffers(offers []*jstruct.JsonOffer) ([]*Offer, error) {
	result := make([]*Offer, 0, len(offers))
	for _, o := range offers {
		r, err := convertOffer(o)
		if err != nil {
			return nil, err
		}
		result = append(result, r)
	}
	return result, nil
}

var (
	indexExceptions = []string{
		"c++",
		"c#",
	}
	stopWords = []interface{}{
		"h", // The H in H/F
		"f", // The F in H/F
		// Generic garbage
		"mision",
		"post",
		"entrepris",
		"suivi",
	}
)

func NewOfferIndex(dir string) (bleve.Index, error) {
	err := os.RemoveAll(dir)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	parts := []string{}
	for _, exc := range indexExceptions {
		parts = append(parts, regexp.QuoteMeta(exc))
	}
	pattern := strings.Join(parts, "|")
	pattern = "(?i)(?:" + pattern + ")"

	m := bleve.NewIndexMapping()
	apecTokenizer := "apec"
	err = m.AddCustomTokenizer(apecTokenizer, map[string]interface{}{
		"type":       exception.Name,
		"exceptions": []string{pattern},
		"tokenizer":  bleveuni.Name,
	})
	if err != nil {
		return nil, err
	}

	apecTokens := "apec_tokens"
	err = m.AddCustomTokenMap(apecTokens, map[string]interface{}{
		"type":   token_map.Name,
		"tokens": stopWords,
	})
	if err != nil {
		return nil, err
	}

	apecStop := "apec_stop"
	err = m.AddCustomTokenFilter(apecStop, map[string]interface{}{
		"type":           stop_tokens_filter.Name,
		"stop_token_map": apecTokens,
	})
	if err != nil {
		return nil, err
	}

	frTokens := []string{
		lower_case_filter.Name,
		fr.ElisionName,
		fr.StopName,
		fr.LightStemmerName,
		apecStop,
	}
	fr := map[string]interface{}{
		"type":          custom_analyzer.Name,
		"tokenizer":     apecTokenizer,
		"token_filters": frTokens,
	}
	frHtml := map[string]interface{}{
		"type": custom_analyzer.Name,
		"char_filters": []string{
			html_char_filter.Name,
		},
		"tokenizer":     apecTokenizer,
		"token_filters": frTokens,
	}
	err = m.AddCustomAnalyzer("fr", fr)
	if err != nil {
		return nil, fmt.Errorf("failed to register analyzer fr: %s", err)
	}
	err = m.AddCustomAnalyzer("fr_html", frHtml)
	if err != nil {
		return nil, fmt.Errorf("failed to register analyzer fr_html: %s", err)
	}

	htmlFr := bleve.NewTextFieldMapping()
	htmlFr.Store = false
	htmlFr.IncludeInAll = false
	htmlFr.IncludeTermVectors = false
	htmlFr.Analyzer = "fr_html"

	textFr := bleve.NewTextFieldMapping()
	textFr.Store = false
	textFr.IncludeInAll = false
	textFr.IncludeTermVectors = false
	textFr.Analyzer = "fr"

	textAll := bleve.NewTextFieldMapping()
	textAll.Store = false
	textAll.IncludeInAll = true
	textAll.IncludeTermVectors = false

	date := bleve.NewDateTimeFieldMapping()
	date.Index = false
	date.Store = true
	date.IncludeInAll = false
	date.IncludeTermVectors = false

	offer := bleve.NewDocumentStaticMapping()
	offer.Dynamic = false
	offer.AddFieldMappingsAt("html", htmlFr)
	offer.AddFieldMappingsAt("title", textFr)
	offer.AddFieldMappingsAt("date", date)

	m.AddDocumentMapping("offer", offer)
	m.DefaultMapping = offer

	index, err := bleve.NewUsing(dir, m, upside_down.Name, boltdb.Name,
		map[string]interface{}{
			"nosync": true,
		})
	if err != nil {
		return nil, err
	}
	return index, nil
}

func OpenOfferIndex(path string) (bleve.Index, error) {
	return bleve.OpenUsing(path, map[string]interface{}{
		"nosync": false,
	})
}

var (
	indexCmd     = app.Command("index", "index APEC offers")
	indexMaxSize = indexCmd.Flag("max-count", "maximum number of items to index").
			Short('n').Default("0").Int()
	// Work around kingpin messing with boolean flags starting with --no-xxx (#54)
	indexIndex = indexCmd.Flag("index", "enable indexing (use --no-index to disable it)").
			Default("true").Bool()
	indexMinQuota = indexCmd.Flag("min-quota",
		"stop geocoding when call quota moves below supplied value").Default("500").Int()
	indexDocId = indexCmd.Flag("id", "index only specified document").String()
)

func indexOffers(cfg *Config) error {
	store, err := OpenStore(cfg.Store())
	if err != nil {
		return err
	}
	defer store.Close()
	rawOffers, err := loadOffers(store)
	if err != nil {
		return err
	}
	if *indexDocId != "" {
		kept := []*jstruct.JsonOffer{}
		for _, o := range rawOffers {
			if o.Id == *indexDocId {
				kept = append(kept, o)
			}
		}
		rawOffers = kept
	}
	offers, err := convertOffers(rawOffers)
	if err != nil {
		return err
	}
	if *indexMaxSize > 0 && len(offers) > *indexMaxSize {
		offers = offers[:*indexMaxSize]
	}

	rejected := 0
	geocodingKey := cfg.GeocodingKey()
	if geocodingKey != "" {
		geocoder, err := NewGeocoder(geocodingKey, cfg.Geocoder())
		if err != nil {
			return err
		}
		defer geocoder.Close()
		rejected, err = geocodeOffers(store, geocoder, offers, *indexMinQuota)
		if err != nil {
			return err
		}
		fmt.Printf("%d rejected geocoding\n", rejected)
	}
	if *indexIndex {
		index, err := NewOfferIndex(cfg.Index())
		if err != nil {
			return err
		}
		start := time.Now()
		indexed := 0
		for i, offer := range offers {
			if (i+1)%500 == 0 {
				now := time.Now()
				elapsed := float64(now.Sub(start)) / float64(time.Second)
				fmt.Printf("%d indexed, %.1f/s\n", i+1, float64(i+1)/elapsed)
			}
			err = index.Index(offer.Id, offer)
			if err != nil {
				return err
			}
			indexed += 1
		}
		err = index.Close()
		if err != nil {
			return err
		}
		end := time.Now()
		fmt.Printf("%d/%d documents indexed in %.2fs\n", indexed, len(offers),
			float64(end.Sub(start))/float64(time.Second))
	}
	return nil
}
<|endoftext|>


<commit_before>package services

import (
	"log"
	"strings"

	ptypes "github.com/franela/play-with-docker/types"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/client"
	"golang.org/x/net/context"
)

var c *client.Client

func init() {
	var err error
	c, err = client.NewEnvClient()
	if err != nil {
		// this wont happen if daemon is offline, only for some critical errors
		log.Fatal("Cannot initialize docker client")
	}

}

func GetContainerInfo(id string) (types.ContainerJSON, error) {
	return c.ContainerInspect(context.Background(), id)
}

func CreateNetwork(name string) error {
	// TODO: This line appears to give an error when running on localhost:3000
	// when driver is specified a name must be given.
	opts := types.NetworkCreate{Attachable: true, Driver: "overlay"}
	_, err := c.NetworkCreate(context.Background(), name, opts)

	if err != nil {
		log.Printf("Starting session err [%s]\n", err)

		return err
	}

	return nil
}

func DeleteNetwork(id string) error {
	err := c.NetworkRemove(context.Background(), id)

	if err != nil {
		return err
	}

	return nil
}

func CreateExecConnection(id string, ctx context.Context) (string, error) {
	conf := types.ExecConfig{Tty: true, AttachStdin: true, AttachStderr: true, AttachStdout: true, Cmd: []string{"sh"}}
	resp, err := c.ContainerExecCreate(ctx, id, conf)
	if err != nil {
		return "", err
	}

	return resp.ID, nil
}

func AttachExecConnection(execId string, ctx context.Context) (*types.HijackedResponse, error) {
	conf := types.ExecConfig{Tty: true, AttachStdin: true, AttachStderr: true, AttachStdout: true}
	conn, err := c.ContainerExecAttach(ctx, execId, conf)

	if err != nil {
		return nil, err
	}

	return &conn, nil
}

func CreateInstance(net string, dindImage string) (*ptypes.Instance, error) {

	var maximumPidLimit int64
	maximumPidLimit = 150 // Set a ulimit value to prevent misuse
	h := &container.HostConfig{NetworkMode: container.NetworkMode(net), Privileged: true}
	h.Resources.PidsLimit = maximumPidLimit

	conf := &container.Config{Image: dindImage, Tty: true}
	container, err := c.ContainerCreate(context.Background(), conf, h, nil, "")

	if err != nil {
		return nil, err
	}

	err = c.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
	if err != nil {
		return nil, err
	}

	cinfo, err := GetContainerInfo(container.ID)
	if err != nil {
		return nil, err
	}

	return &ptypes.Instance{Name: strings.Replace(cinfo.Name, "/", "", 1), IP: cinfo.NetworkSettings.Networks[net].IPAddress}, nil
}

func DeleteContainer(id string) error {
	return c.ContainerRemove(context.Background(), id, types.ContainerRemoveOptions{Force: true, RemoveVolumes: true})
}
<commit_msg>Set default resize options<commit_after>package services

import (
	"log"
	"strings"

	ptypes "github.com/franela/play-with-docker/types"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/client"
	"golang.org/x/net/context"
)

var c *client.Client

func init() {
	var err error
	c, err = client.NewEnvClient()
	if err != nil {
		// this wont happen if daemon is offline, only for some critical errors
		log.Fatal("Cannot initialize docker client")
	}

}

func GetContainerInfo(id string) (types.ContainerJSON, error) {
	return c.ContainerInspect(context.Background(), id)
}

func CreateNetwork(name string) error {
	// TODO: This line appears to give an error when running on localhost:3000
	// when driver is specified a name must be given.
	opts := types.NetworkCreate{Attachable: true, Driver: "overlay"}
	_, err := c.NetworkCreate(context.Background(), name, opts)

	if err != nil {
		log.Printf("Starting session err [%s]\n", err)

		return err
	}

	return nil
}

func DeleteNetwork(id string) error {
	err := c.NetworkRemove(context.Background(), id)

	if err != nil {
		return err
	}

	return nil
}

func CreateExecConnection(id string, ctx context.Context) (string, error) {
	conf := types.ExecConfig{Tty: true, AttachStdin: true, AttachStderr: true, AttachStdout: true, Cmd: []string{"sh"}}
	resp, err := c.ContainerExecCreate(ctx, id, conf)
	if err != nil {
		return "", err
	}

	return resp.ID, nil
}

func AttachExecConnection(execId string, ctx context.Context) (*types.HijackedResponse, error) {
	conf := types.ExecConfig{Tty: true, AttachStdin: true, AttachStderr: true, AttachStdout: true}
	conn, err := c.ContainerExecAttach(ctx, execId, conf)

	if err != nil {
		return nil, err
	}

	err = c.ContainerExecResize(ctx, execId, types.ResizeOptions{Height: 24, Width: 80})

	if err != nil {
		return nil, err
	}

	return &conn, nil
}

func CreateInstance(net string, dindImage string) (*ptypes.Instance, error) {

	var maximumPidLimit int64
	maximumPidLimit = 150 // Set a ulimit value to prevent misuse
	h := &container.HostConfig{NetworkMode: container.NetworkMode(net), Privileged: true}
	h.Resources.PidsLimit = maximumPidLimit

	conf := &container.Config{Image: dindImage, Tty: true}
	container, err := c.ContainerCreate(context.Background(), conf, h, nil, "")

	if err != nil {
		return nil, err
	}

	err = c.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
	if err != nil {
		return nil, err
	}

	cinfo, err := GetContainerInfo(container.ID)
	if err != nil {
		return nil, err
	}

	return &ptypes.Instance{Name: strings.Replace(cinfo.Name, "/", "", 1), IP: cinfo.NetworkSettings.Networks[net].IPAddress}, nil
}

func DeleteContainer(id string) error {
	return c.ContainerRemove(context.Background(), id, types.ContainerRemoveOptions{Force: true, RemoveVolumes: true})
}
<|endoftext|>


<commit_before>// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package kustfile

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
	"reflect"
	"regexp"
	"strings"

	"sigs.k8s.io/kustomize/api/filesys"
	"sigs.k8s.io/kustomize/api/konfig"
	"sigs.k8s.io/kustomize/api/types"
	"sigs.k8s.io/yaml"
)

var fieldMarshallingOrder = determineFieldOrder()

// determineFieldOrder returns a slice of Kustomization field
// names in the preferred order for serialization to a file.
// The field list is checked against the actual struct type
// to confirm that all fields are present, and no unknown
// fields are specified. Deprecated fields are removed from
// the list, meaning they will drop to the bottom on output
// (if present). The ordering and/or deprecation of fields
// in nested structs is not determined or considered.
func determineFieldOrder() []string {
	m := make(map[string]bool)
	s := reflect.ValueOf(&types.Kustomization{}).Elem()
	typeOfT := s.Type()
	for i := 0; i < s.NumField(); i++ {
		m[typeOfT.Field(i).Name] = false
	}

	ordered := []string{
		"MetaData",
		"Resources",
		"Bases",
		"NamePrefix",
		"NameSuffix",
		"Namespace",
		"Crds",
		"CommonLabels",
		"Labels",
		"CommonAnnotations",
		"PatchesStrategicMerge",
		"PatchesJson6902",
		"Patches",
		"ConfigMapGenerator",
		"SecretGenerator",
		"GeneratorOptions",
		"Vars",
		"Images",
		"Replicas",
		"Configurations",
		"Generators",
		"Transformers",
		"Inventory",
		"Components",
	}

	// Add deprecated fields here.
	deprecated := map[string]bool{}

	// Account for the inlined TypeMeta fields.
	var result []string
	result = append(result, "APIVersion", "Kind")
	m["TypeMeta"] = true

	// Make sure all these fields are recognized.
	for _, n := range ordered {
		if _, ok := m[n]; ok {
			m[n] = true
		} else {
			log.Fatalf("%s is not a recognized field.", n)
		}
		// Keep if not deprecated.
		if _, f := deprecated[n]; !f {
			result = append(result, n)
		}
	}
	return result
}

// commentedField records the comment associated with a kustomization field
// field has to be a recognized kustomization field
// comment can be empty
type commentedField struct {
	field   string
	comment []byte
}

func (cf *commentedField) appendComment(comment []byte) {
	cf.comment = append(cf.comment, comment...)
}

func squash(x [][]byte) []byte {
	return bytes.Join(x, []byte(``))
}

type kustomizationFile struct {
	path           string
	fSys           filesys.FileSystem
	originalFields []*commentedField
}

// NewKustomizationFile returns a new instance.
func NewKustomizationFile(fSys filesys.FileSystem) (*kustomizationFile, error) { // nolint
	mf := &kustomizationFile{fSys: fSys}
	err := mf.validate()
	if err != nil {
		return nil, err
	}
	return mf, nil
}

func (mf *kustomizationFile) GetPath() string {
	if mf == nil {
		return ""
	}
	return mf.path
}

func (mf *kustomizationFile) validate() error {
	match := 0
	var path []string
	for _, kfilename := range konfig.RecognizedKustomizationFileNames() {
		if mf.fSys.Exists(kfilename) {
			match += 1
			path = append(path, kfilename)
		}
	}

	switch match {
	case 0:
		return fmt.Errorf(
			"Missing kustomization file '%s'.\n",
			konfig.DefaultKustomizationFileName())
	case 1:
		mf.path = path[0]
	default:
		return fmt.Errorf("Found multiple kustomization file: %v\n", path)
	}

	if mf.fSys.IsDir(mf.path) {
		return fmt.Errorf("%s should be a file", mf.path)
	}
	return nil
}

func (mf *kustomizationFile) Read() (*types.Kustomization, error) {
	data, err := mf.fSys.ReadFile(mf.path)
	if err != nil {
		return nil, err
	}
	data, err = types.FixKustomizationPreUnmarshalling(data)
	if err != nil {
		return nil, err
	}
	var k types.Kustomization
	err = k.Unmarshal(data)
	if err != nil {
		return nil, err
	}
	k.FixKustomizationPostUnmarshalling()
	err = mf.parseCommentedFields(data)
	if err != nil {
		return nil, err
	}
	return &k, err
}

func (mf *kustomizationFile) Write(kustomization *types.Kustomization) error {
	if kustomization == nil {
		return errors.New("util: kustomization file arg is nil")
	}
	data, err := mf.marshal(kustomization)
	if err != nil {
		return err
	}
	return mf.fSys.WriteFile(mf.path, data)
}

// StringInSlice returns true if the string is in the slice.
func StringInSlice(str string, list []string) bool {
	for _, v := range list {
		if v == str {
			return true
		}
	}
	return false
}

func (mf *kustomizationFile) parseCommentedFields(content []byte) error {
	buffer := bytes.NewBuffer(content)
	var comments [][]byte

	line, err := buffer.ReadBytes('\n')
	for err == nil {
		if isCommentOrBlankLine(line) {
			comments = append(comments, line)
		} else {
			matched, field := findMatchedField(line)
			if matched {
				mf.originalFields = append(mf.originalFields, &commentedField{field: field, comment: squash(comments)})
				comments = [][]byte{}
			} else if len(comments) > 0 {
				mf.originalFields[len(mf.originalFields)-1].appendComment(squash(comments))
				comments = [][]byte{}
			}
		}
		line, err = buffer.ReadBytes('\n')
	}

	if err != io.EOF {
		return err
	}
	return nil
}

// marshal converts a kustomization to a byte stream.
func (mf *kustomizationFile) marshal(kustomization *types.Kustomization) ([]byte, error) {
	var output []byte
	for _, comment := range mf.originalFields {
		output = append(output, comment.comment...)
		content, err := marshalField(comment.field, kustomization)
		if err != nil {
			return content, err
		}
		output = append(output, content...)
	}
	for _, field := range fieldMarshallingOrder {
		if mf.hasField(field) {
			continue
		}
		content, err := marshalField(field, kustomization)
		if err != nil {
			return content, nil
		}
		output = append(output, content...)
	}
	return output, nil
}

func (mf *kustomizationFile) hasField(name string) bool {
	for _, n := range mf.originalFields {
		if n.field == name {
			return true
		}
	}
	return false
}

/*
 isCommentOrBlankLine determines if a line is a comment or blank line
 Return true for following lines
 # This line is a comment
       # This line is also a comment with several leading white spaces

 (The line above is a blank line)
*/
func isCommentOrBlankLine(line []byte) bool {
	s := bytes.TrimRight(bytes.TrimLeft(line, " "), "\n")
	return len(s) == 0 || bytes.HasPrefix(s, []byte(`#`))
}

func findMatchedField(line []byte) (bool, string) {
	for _, field := range fieldMarshallingOrder {
		// (?i) is for case insensitive regexp matching
		r := regexp.MustCompile("^(" + "(?i)" + field + "):")
		if r.Match(line) {
			return true, field
		}
	}
	return false, ""
}

// marshalField marshal a given field of a kustomization object into yaml format.
// If the field wasn't in the original kustomization.yaml file or wasn't added,
// an empty []byte is returned.
func marshalField(field string, kustomization *types.Kustomization) ([]byte, error) {
	r := reflect.ValueOf(*kustomization)
	v := r.FieldByName(strings.Title(field))

	if !v.IsValid() || isEmpty(v) {
		return []byte{}, nil
	}

	k := &types.Kustomization{}
	kr := reflect.ValueOf(k)
	kv := kr.Elem().FieldByName(strings.Title(field))
	kv.Set(v)

	return yaml.Marshal(k)
}

func isEmpty(v reflect.Value) bool {
	// If v is a pointer type
	if v.Type().Kind() == reflect.Ptr {
		return v.IsNil()
	}
	return v.Len() == 0
}
<commit_msg>add new fields to ordered fields<commit_after>// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package kustfile

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"log"
	"reflect"
	"regexp"
	"strings"

	"sigs.k8s.io/kustomize/api/filesys"
	"sigs.k8s.io/kustomize/api/konfig"
	"sigs.k8s.io/kustomize/api/types"
	"sigs.k8s.io/yaml"
)

var fieldMarshallingOrder = determineFieldOrder()

// determineFieldOrder returns a slice of Kustomization field
// names in the preferred order for serialization to a file.
// The field list is checked against the actual struct type
// to confirm that all fields are present, and no unknown
// fields are specified. Deprecated fields are removed from
// the list, meaning they will drop to the bottom on output
// (if present). The ordering and/or deprecation of fields
// in nested structs is not determined or considered.
func determineFieldOrder() []string {
	m := make(map[string]bool)
	s := reflect.ValueOf(&types.Kustomization{}).Elem()
	typeOfT := s.Type()
	for i := 0; i < s.NumField(); i++ {
		m[typeOfT.Field(i).Name] = false
	}

	ordered := []string{
		"MetaData",
		"Resources",
		"Bases",
		"NamePrefix",
		"NameSuffix",
		"Namespace",
		"Crds",
		"CommonLabels",
		"Labels",
		"CommonAnnotations",
		"PatchesStrategicMerge",
		"PatchesJson6902",
		"Patches",
		"ConfigMapGenerator",
		"SecretGenerator",
		"HelmCharts",
		"HelmChartInflationGenerator",
		"GeneratorOptions",
		"Vars",
		"Images",
		"Replicas",
		"Configurations",
		"Generators",
		"Transformers",
		"Inventory",
		"Components",
	}

	// Add deprecated fields here.
	deprecated := map[string]bool{}

	// Account for the inlined TypeMeta fields.
	var result []string
	result = append(result, "APIVersion", "Kind")
	m["TypeMeta"] = true

	// Make sure all these fields are recognized.
	for _, n := range ordered {
		if _, ok := m[n]; ok {
			m[n] = true
		} else {
			log.Fatalf("%s is not a recognized field.", n)
		}
		// Keep if not deprecated.
		if _, f := deprecated[n]; !f {
			result = append(result, n)
		}
	}
	return result
}

// commentedField records the comment associated with a kustomization field
// field has to be a recognized kustomization field
// comment can be empty
type commentedField struct {
	field   string
	comment []byte
}

func (cf *commentedField) appendComment(comment []byte) {
	cf.comment = append(cf.comment, comment...)
}

func squash(x [][]byte) []byte {
	return bytes.Join(x, []byte(``))
}

type kustomizationFile struct {
	path           string
	fSys           filesys.FileSystem
	originalFields []*commentedField
}

// NewKustomizationFile returns a new instance.
func NewKustomizationFile(fSys filesys.FileSystem) (*kustomizationFile, error) { // nolint
	mf := &kustomizationFile{fSys: fSys}
	err := mf.validate()
	if err != nil {
		return nil, err
	}
	return mf, nil
}

func (mf *kustomizationFile) GetPath() string {
	if mf == nil {
		return ""
	}
	return mf.path
}

func (mf *kustomizationFile) validate() error {
	match := 0
	var path []string
	for _, kfilename := range konfig.RecognizedKustomizationFileNames() {
		if mf.fSys.Exists(kfilename) {
			match += 1
			path = append(path, kfilename)
		}
	}

	switch match {
	case 0:
		return fmt.Errorf(
			"Missing kustomization file '%s'.\n",
			konfig.DefaultKustomizationFileName())
	case 1:
		mf.path = path[0]
	default:
		return fmt.Errorf("Found multiple kustomization file: %v\n", path)
	}

	if mf.fSys.IsDir(mf.path) {
		return fmt.Errorf("%s should be a file", mf.path)
	}
	return nil
}

func (mf *kustomizationFile) Read() (*types.Kustomization, error) {
	data, err := mf.fSys.ReadFile(mf.path)
	if err != nil {
		return nil, err
	}
	data, err = types.FixKustomizationPreUnmarshalling(data)
	if err != nil {
		return nil, err
	}
	var k types.Kustomization
	err = k.Unmarshal(data)
	if err != nil {
		return nil, err
	}
	k.FixKustomizationPostUnmarshalling()
	err = mf.parseCommentedFields(data)
	if err != nil {
		return nil, err
	}
	return &k, err
}

func (mf *kustomizationFile) Write(kustomization *types.Kustomization) error {
	if kustomization == nil {
		return errors.New("util: kustomization file arg is nil")
	}
	data, err := mf.marshal(kustomization)
	if err != nil {
		return err
	}
	return mf.fSys.WriteFile(mf.path, data)
}

// StringInSlice returns true if the string is in the slice.
func StringInSlice(str string, list []string) bool {
	for _, v := range list {
		if v == str {
			return true
		}
	}
	return false
}

func (mf *kustomizationFile) parseCommentedFields(content []byte) error {
	buffer := bytes.NewBuffer(content)
	var comments [][]byte

	line, err := buffer.ReadBytes('\n')
	for err == nil {
		if isCommentOrBlankLine(line) {
			comments = append(comments, line)
		} else {
			matched, field := findMatchedField(line)
			if matched {
				mf.originalFields = append(mf.originalFields, &commentedField{field: field, comment: squash(comments)})
				comments = [][]byte{}
			} else if len(comments) > 0 {
				mf.originalFields[len(mf.originalFields)-1].appendComment(squash(comments))
				comments = [][]byte{}
			}
		}
		line, err = buffer.ReadBytes('\n')
	}

	if err != io.EOF {
		return err
	}
	return nil
}

// marshal converts a kustomization to a byte stream.
func (mf *kustomizationFile) marshal(kustomization *types.Kustomization) ([]byte, error) {
	var output []byte
	for _, comment := range mf.originalFields {
		output = append(output, comment.comment...)
		content, err := marshalField(comment.field, kustomization)
		if err != nil {
			return content, err
		}
		output = append(output, content...)
	}
	for _, field := range fieldMarshallingOrder {
		if mf.hasField(field) {
			continue
		}
		content, err := marshalField(field, kustomization)
		if err != nil {
			return content, nil
		}
		output = append(output, content...)
	}
	return output, nil
}

func (mf *kustomizationFile) hasField(name string) bool {
	for _, n := range mf.originalFields {
		if n.field == name {
			return true
		}
	}
	return false
}

/*
 isCommentOrBlankLine determines if a line is a comment or blank line
 Return true for following lines
 # This line is a comment
       # This line is also a comment with several leading white spaces

 (The line above is a blank line)
*/
func isCommentOrBlankLine(line []byte) bool {
	s := bytes.TrimRight(bytes.TrimLeft(line, " "), "\n")
	return len(s) == 0 || bytes.HasPrefix(s, []byte(`#`))
}

func findMatchedField(line []byte) (bool, string) {
	for _, field := range fieldMarshallingOrder {
		// (?i) is for case insensitive regexp matching
		r := regexp.MustCompile("^(" + "(?i)" + field + "):")
		if r.Match(line) {
			return true, field
		}
	}
	return false, ""
}

// marshalField marshal a given field of a kustomization object into yaml format.
// If the field wasn't in the original kustomization.yaml file or wasn't added,
// an empty []byte is returned.
func marshalField(field string, kustomization *types.Kustomization) ([]byte, error) {
	r := reflect.ValueOf(*kustomization)
	v := r.FieldByName(strings.Title(field))

	if !v.IsValid() || isEmpty(v) {
		return []byte{}, nil
	}

	k := &types.Kustomization{}
	kr := reflect.ValueOf(k)
	kv := kr.Elem().FieldByName(strings.Title(field))
	kv.Set(v)

	return yaml.Marshal(k)
}

func isEmpty(v reflect.Value) bool {
	// If v is a pointer type
	if v.Type().Kind() == reflect.Ptr {
		return v.IsNil()
	}
	return v.Len() == 0
}
<|endoftext|>


<commit_before>package session

import (
	"github.com/gin-gonic/gin"
	"github.com/gorilla/sessions"
	"github.com/iiinsomnia/yiigo"
)

const gosessid = "GOSESSID"

var store *sessions.CookieStore

// Start start session
func Start() {
	store = sessions.NewCookieStore([]byte(yiigo.EnvString("session", "secret", "N0awmAuS2OziVFu^9!*0LY7MeCRgQ&z0")))
}

// Get get session key - value
func Get(c *gin.Context, key string, defaultValule ...interface{}) (interface{}, error) {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		yiigo.Err(err.Error())
		return nil, err
	}

	// Get some session values.
	v, ok := session.Values[key]

	if !ok {
		if len(defaultValule) > 0 {
			return defaultValule, nil
		}

		return nil, nil
	}

	return v, nil
}

// Set set session key - value, duration: seconds
func Set(c *gin.Context, key string, data interface{}, duration ...int) error {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	if len(duration) > 0 {
		session.Options = &sessions.Options{
			Path:   "/",
			MaxAge: duration[0],
		}
	}

	// Set some session values.
	session.Values[key] = data
	// Save it before we write to the response/return from the handler.
	err = session.Save(c.Request, c.Writer)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	return nil
}

// Delete delete session key
func Delete(c *gin.Context, key string) error {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	delete(session.Values, key)

	err = session.Save(c.Request, c.Writer)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	return nil
}

// Destroy destroy session
func Destroy(c *gin.Context) error {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	session.Options = &sessions.Options{
		Path:   "/",
		MaxAge: -1,
	}

	err = session.Save(c.Request, c.Writer)

	if err != nil {
		yiigo.Err(err.Error())
		return err
	}

	return nil
}
<commit_msg>update session<commit_after>package session

import (
	"github.com/gin-gonic/gin"
	"github.com/gorilla/sessions"
	"github.com/iiinsomnia/yiigo"
)

const gosessid = "GOSESSID"

var store *sessions.CookieStore

// Start start session
func Start() {
	store = sessions.NewCookieStore([]byte(yiigo.EnvString("session", "secret", "N0awmAuS2OziVFu^9!*0LY7MeCRgQ&z0")))
}

// Get get session key - value
func Get(c *gin.Context, key string, defaultValule ...interface{}) (interface{}, error) {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		return nil, err
	}

	// Get some session values.
	v, ok := session.Values[key]

	if !ok {
		if len(defaultValule) > 0 {
			return defaultValule, nil
		}

		return nil, nil
	}

	return v, nil
}

// Set set session key - value, duration: seconds
func Set(c *gin.Context, key string, data interface{}, duration ...int) error {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		return err
	}

	if len(duration) > 0 {
		session.Options = &sessions.Options{
			Path:   "/",
			MaxAge: duration[0],
		}
	}

	// Set some session values.
	session.Values[key] = data
	// Save it before we write to the response/return from the handler.
	err = session.Save(c.Request, c.Writer)

	return err
}

// Delete delete session key
func Delete(c *gin.Context, key string) error {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		return err
	}

	delete(session.Values, key)

	err = session.Save(c.Request, c.Writer)

	return err
}

// Destroy destroy session
func Destroy(c *gin.Context) error {
	session, err := store.Get(c.Request, gosessid)

	if err != nil {
		return err
	}

	session.Options = &sessions.Options{
		Path:   "/",
		MaxAge: -1,
	}

	err = session.Save(c.Request, c.Writer)

	return err
}
<|endoftext|>


<commit_before>// session.go - mixnet client session
// Copyright (C) 2018  David Stainton.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package session

import (
	"context"
	"encoding/hex"
	"errors"
	"fmt"
	mrand "math/rand"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	"github.com/katzenpost/client/config"
	cConstants "github.com/katzenpost/client/constants"
	"github.com/katzenpost/client/internal/pkiclient"
	"github.com/katzenpost/client/poisson"
	"github.com/katzenpost/client/utils"
	coreConstants "github.com/katzenpost/core/constants"
	"github.com/katzenpost/core/crypto/ecdh"
	"github.com/katzenpost/core/log"
	"github.com/katzenpost/core/pki"
	"github.com/katzenpost/core/sphinx"
	"github.com/katzenpost/core/sphinx/constants"
	sConstants "github.com/katzenpost/core/sphinx/constants"
	cutils "github.com/katzenpost/core/utils"
	"github.com/katzenpost/core/worker"
	"github.com/katzenpost/minclient"
	"gopkg.in/eapache/channels.v1"
	"gopkg.in/op/go-logging.v1"
)

// Session is the struct type that keeps state for a given session.
type Session struct {
	worker.Worker

	cfg       *config.Config
	pkiClient pki.Client
	minclient *minclient.Client
	log       *logging.Logger

	fatalErrCh chan error
	haltedCh   chan interface{}
	haltOnce   sync.Once

	// λP
	pTimer *poisson.Fount
	// λD
	dTimer *poisson.Fount
	// λL
	lTimer *poisson.Fount

	linkKey   *ecdh.PrivateKey
	opCh      chan workerOp
	onlineAt  time.Time
	hasPKIDoc bool

	egressQueue     EgressQueue
	egressQueueLock *sync.Mutex

	eventCh       channels.Channel
	waitSentChans map[[cConstants.MessageIDLength]byte]channels.Channel
	waitChans     map[[cConstants.MessageIDLength]byte]channels.Channel
	surbIDMap     map[[sConstants.SURBIDLength]byte]*Message
	messageIDMap  map[[cConstants.MessageIDLength]byte]*Message
	mapLock       *sync.Mutex

	decoyLoopTally uint64
}

// New establishes a session with provider using key.
// This method will block until session is connected to the Provider.
func New(ctx context.Context, fatalErrCh chan error, logBackend *log.Backend, cfg *config.Config) (*Session, error) {
	var err error

	// create a pkiclient for our own client lookups
	// AND create a pkiclient for minclient's use
	proxyCfg := cfg.UpstreamProxyConfig()
	pkiClient, err := cfg.NewPKIClient(logBackend, proxyCfg)
	if err != nil {
		return nil, err
	}

	// create a pkiclient for minclient's use
	pkiClient2, err := cfg.NewPKIClient(logBackend, proxyCfg)
	if err != nil {
		return nil, err
	}
	pkiCacheClient := pkiclient.New(pkiClient2)

	log := logBackend.GetLogger(fmt.Sprintf("%s@%s_c", cfg.Account.User, cfg.Account.Provider))

	s := &Session{
		cfg:           cfg,
		pkiClient:     pkiClient,
		log:           log,
		fatalErrCh:    fatalErrCh,
		opCh:          make(chan workerOp),
		eventCh:       channels.NewInfiniteChannel(),
		waitChans:     make(map[[sConstants.SURBIDLength]byte]channels.Channel),
		waitSentChans: make(map[[cConstants.MessageIDLength]byte]channels.Channel),
	}

	// XXX todo: replace all this with persistent data store
	s.surbIDMap = make(map[[sConstants.SURBIDLength]byte]*Message)
	s.messageIDMap = make(map[[cConstants.MessageIDLength]byte]*Message)
	s.mapLock = new(sync.Mutex)

	s.egressQueue = new(Queue)
	s.egressQueueLock = new(sync.Mutex)

	id := cfg.Account.User + "@" + cfg.Account.Provider
	basePath := filepath.Join(cfg.Proxy.DataDir, id)
	if err := cutils.MkDataDir(basePath); err != nil {
		return nil, err
	}

	err = s.loadKeys(basePath)
	if err != nil {
		return nil, err
	}

	// Configure and bring up the minclient instance.
	clientCfg := &minclient.ClientConfig{
		User:                cfg.Account.User,
		Provider:            cfg.Account.Provider,
		ProviderKeyPin:      cfg.Account.ProviderKeyPin,
		LinkKey:             s.linkKey,
		LogBackend:          logBackend,
		PKIClient:           pkiCacheClient,
		OnConnFn:            s.onConnection,
		OnMessageFn:         s.onMessage,
		OnACKFn:             s.onACK,
		OnDocumentFn:        s.onDocument,
		DialContextFn:       proxyCfg.ToDialContext("authority"),
		MessagePollInterval: time.Duration(cfg.Debug.PollingInterval) * time.Second,
		EnableTimeSync:      false, // Be explicit about it.
	}

	s.minclient, err = minclient.New(clientCfg)
	if err != nil {
		return nil, err
	}

	// block until we get the first PKI document
	// and then set our timers accordingly
	doc, err := s.awaitFirstPKIDoc(ctx)
	if err != nil {
		return nil, err
	}
	s.setTimers(doc)

	s.Go(s.worker)
	s.Go(s.eventSinkWorker)
	return s, nil
}

func (s *Session) eventSinkWorker() {
	for {
		s.log.Debug("*** now awaiting events on the event channel ***")
		select {
		case <-s.HaltCh():
			return
		case e := <-s.eventCh.Out():
			switch v := e.(type) {
			case MessageReplyEvent:
				s.log.Debug("MESSAGE REPLY EVENT")
				ch, ok := s.waitChans[*v.MessageID]
				if ok {
					ch.In() <- e
				}
			case MessageSentEvent:
				s.log.Debug("MESSAGE SENT EVENT")
				s.waitSentChans[*v.MessageID].In() <- e
			default:
				s.log.Debug("Error, event sink worker received unknown event type.")
			}
		}
	}
}

func (s *Session) awaitFirstPKIDoc(ctx context.Context) (*pki.Document, error) {
	for {
		var qo workerOp
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case <-s.HaltCh():
			s.log.Debugf("Terminating gracefully.")
			return nil, errors.New("Terminating gracefully.")
		case <-time.After(time.Duration(s.cfg.Debug.InitialMaxPKIRetrievalDelay) * time.Second):
			return nil, errors.New("Timeout failure awaiting first PKI document.")
		case qo = <-s.opCh:
		}
		switch op := qo.(type) {
		case opNewDocument:
			// Determine if PKI doc is valid. If not then abort.
			err := s.isDocValid(op.doc)
			if err != nil {
				s.log.Errorf("Aborting, PKI doc is not valid for the Loopix decoy traffic use case: %v", err)
				err := fmt.Errorf("Aborting, PKI doc is not valid for the Loopix decoy traffic use case: %v", err)
				s.fatalErrCh <- err
				return nil, err
			}
			return op.doc, nil
		default:

			continue
		}
	}
}

func (s *Session) loadKeys(basePath string) error {
	// Load link key.
	var err error
	if s.linkKey, err = config.LoadLinkKey(basePath); err != nil {
		s.log.Errorf("Failure to load link keys: %s", err)
		return err
	}
	return nil
}

// GetService returns a randomly selected service
// matching the specified service name
func (s *Session) GetService(serviceName string) (*utils.ServiceDescriptor, error) {
	doc := s.minclient.CurrentDocument()
	if doc == nil {
		return nil, errors.New("pki doc is nil")
	}
	serviceDescriptors := utils.FindServices(serviceName, doc)
	if len(serviceDescriptors) == 0 {
		return nil, errors.New("GetService failure, service not found in pki doc.")
	}
	return &serviceDescriptors[mrand.Intn(len(serviceDescriptors))], nil
}

// OnConnection will be called by the minclient api
// upon connecting to the Provider
func (s *Session) onConnection(err error) {
	if err == nil {
		s.opCh <- opConnStatusChanged{
			isConnected: true,
		}
	}
}

// OnMessage will be called by the minclient api
// upon receiving a message
func (s *Session) onMessage(ciphertextBlock []byte) error {
	s.log.Debugf("OnMessage")
	return nil
}

func (s *Session) incrementDecoyLoopTally() {
	atomic.AddUint64(&s.decoyLoopTally, 1)
}

func (s *Session) decrementDecoyLoopTally() {
	atomic.AddUint64(&s.decoyLoopTally, ^uint64(1-1))
}

// OnACK is called by the minclient api when we receive a SURB reply message.
func (s *Session) onACK(surbID *[constants.SURBIDLength]byte, ciphertext []byte) error {
	idStr := fmt.Sprintf("[%v]", hex.EncodeToString(surbID[:]))
	s.log.Infof("OnACK with SURBID %x", idStr)
	s.mapLock.Lock()
	defer s.mapLock.Unlock()
	msg, ok := s.surbIDMap[*surbID]
	if !ok {
		s.log.Debug("wtf, received reply with unexpected SURBID")
		return nil
	}
	plaintext, err := sphinx.DecryptSURBPayload(ciphertext, msg.Key)
	if err != nil {
		s.log.Infof("SURB Reply decryption failure: %s", err)
		return err
	}
	if len(plaintext) != coreConstants.ForwardPayloadLength {
		s.log.Warningf("Discarding SURB %v: Invalid payload size: %v", idStr, len(plaintext))
		return nil
	}
	if msg.WithSURB && msg.IsDecoy {
		_, ok := s.surbIDMap[*surbID]
		if ok {
			s.decrementDecoyLoopTally()
			delete(s.surbIDMap, *surbID)
		}
		return nil
	}
	switch msg.SURBType {
	case cConstants.SurbTypeKaetzchen, cConstants.SurbTypeInternal:
		s.eventCh.In() <- &MessageReplyEvent{
			MessageID: msg.ID,
			Payload:   plaintext[2:],
			Err:       nil,
		}
	default:
		s.log.Warningf("Discarding SURB %v: Unknown type: 0x%02x", idStr, msg.SURBType)
	}
	return nil
}

func (s *Session) onDocument(doc *pki.Document) {
	s.log.Debugf("onDocument(): Epoch %v", doc.Epoch)
	s.hasPKIDoc = true
	s.opCh <- opNewDocument{
		doc: doc,
	}
}
<commit_msg>set minclient initial polling to 1 second<commit_after>// session.go - mixnet client session
// Copyright (C) 2018  David Stainton.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package session

import (
	"context"
	"encoding/hex"
	"errors"
	"fmt"
	mrand "math/rand"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	"github.com/katzenpost/client/config"
	cConstants "github.com/katzenpost/client/constants"
	"github.com/katzenpost/client/internal/pkiclient"
	"github.com/katzenpost/client/poisson"
	"github.com/katzenpost/client/utils"
	coreConstants "github.com/katzenpost/core/constants"
	"github.com/katzenpost/core/crypto/ecdh"
	"github.com/katzenpost/core/log"
	"github.com/katzenpost/core/pki"
	"github.com/katzenpost/core/sphinx"
	"github.com/katzenpost/core/sphinx/constants"
	sConstants "github.com/katzenpost/core/sphinx/constants"
	cutils "github.com/katzenpost/core/utils"
	"github.com/katzenpost/core/worker"
	"github.com/katzenpost/minclient"
	"gopkg.in/eapache/channels.v1"
	"gopkg.in/op/go-logging.v1"
)

// Session is the struct type that keeps state for a given session.
type Session struct {
	worker.Worker

	cfg       *config.Config
	pkiClient pki.Client
	minclient *minclient.Client
	log       *logging.Logger

	fatalErrCh chan error
	haltedCh   chan interface{}
	haltOnce   sync.Once

	// λP
	pTimer *poisson.Fount
	// λD
	dTimer *poisson.Fount
	// λL
	lTimer *poisson.Fount

	linkKey   *ecdh.PrivateKey
	opCh      chan workerOp
	onlineAt  time.Time
	hasPKIDoc bool

	egressQueue     EgressQueue
	egressQueueLock *sync.Mutex

	eventCh       channels.Channel
	waitSentChans map[[cConstants.MessageIDLength]byte]channels.Channel
	waitChans     map[[cConstants.MessageIDLength]byte]channels.Channel
	surbIDMap     map[[sConstants.SURBIDLength]byte]*Message
	messageIDMap  map[[cConstants.MessageIDLength]byte]*Message
	mapLock       *sync.Mutex

	decoyLoopTally uint64
}

// New establishes a session with provider using key.
// This method will block until session is connected to the Provider.
func New(ctx context.Context, fatalErrCh chan error, logBackend *log.Backend, cfg *config.Config) (*Session, error) {
	var err error

	// create a pkiclient for our own client lookups
	// AND create a pkiclient for minclient's use
	proxyCfg := cfg.UpstreamProxyConfig()
	pkiClient, err := cfg.NewPKIClient(logBackend, proxyCfg)
	if err != nil {
		return nil, err
	}

	// create a pkiclient for minclient's use
	pkiClient2, err := cfg.NewPKIClient(logBackend, proxyCfg)
	if err != nil {
		return nil, err
	}
	pkiCacheClient := pkiclient.New(pkiClient2)

	log := logBackend.GetLogger(fmt.Sprintf("%s@%s_c", cfg.Account.User, cfg.Account.Provider))

	s := &Session{
		cfg:           cfg,
		pkiClient:     pkiClient,
		log:           log,
		fatalErrCh:    fatalErrCh,
		opCh:          make(chan workerOp),
		eventCh:       channels.NewInfiniteChannel(),
		waitChans:     make(map[[sConstants.SURBIDLength]byte]channels.Channel),
		waitSentChans: make(map[[cConstants.MessageIDLength]byte]channels.Channel),
	}

	// XXX todo: replace all this with persistent data store
	s.surbIDMap = make(map[[sConstants.SURBIDLength]byte]*Message)
	s.messageIDMap = make(map[[cConstants.MessageIDLength]byte]*Message)
	s.mapLock = new(sync.Mutex)

	s.egressQueue = new(Queue)
	s.egressQueueLock = new(sync.Mutex)

	id := cfg.Account.User + "@" + cfg.Account.Provider
	basePath := filepath.Join(cfg.Proxy.DataDir, id)
	if err := cutils.MkDataDir(basePath); err != nil {
		return nil, err
	}

	err = s.loadKeys(basePath)
	if err != nil {
		return nil, err
	}

	// Configure and bring up the minclient instance.
	clientCfg := &minclient.ClientConfig{
		User:                cfg.Account.User,
		Provider:            cfg.Account.Provider,
		ProviderKeyPin:      cfg.Account.ProviderKeyPin,
		LinkKey:             s.linkKey,
		LogBackend:          logBackend,
		PKIClient:           pkiCacheClient,
		OnConnFn:            s.onConnection,
		OnMessageFn:         s.onMessage,
		OnACKFn:             s.onACK,
		OnDocumentFn:        s.onDocument,
		DialContextFn:       proxyCfg.ToDialContext("authority"),
		MessagePollInterval: 1 * time.Second,
		EnableTimeSync:      false, // Be explicit about it.
	}

	s.minclient, err = minclient.New(clientCfg)
	if err != nil {
		return nil, err
	}

	// block until we get the first PKI document
	// and then set our timers accordingly
	doc, err := s.awaitFirstPKIDoc(ctx)
	if err != nil {
		return nil, err
	}
	s.setTimers(doc)

	s.Go(s.worker)
	s.Go(s.eventSinkWorker)
	return s, nil
}

func (s *Session) eventSinkWorker() {
	for {
		s.log.Debug("*** now awaiting events on the event channel ***")
		select {
		case <-s.HaltCh():
			return
		case e := <-s.eventCh.Out():
			switch v := e.(type) {
			case MessageReplyEvent:
				s.log.Debug("MESSAGE REPLY EVENT")
				ch, ok := s.waitChans[*v.MessageID]
				if ok {
					ch.In() <- e
				}
			case MessageSentEvent:
				s.log.Debug("MESSAGE SENT EVENT")
				s.waitSentChans[*v.MessageID].In() <- e
			default:
				s.log.Debug("Error, event sink worker received unknown event type.")
			}
		}
	}
}

func (s *Session) awaitFirstPKIDoc(ctx context.Context) (*pki.Document, error) {
	for {
		var qo workerOp
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case <-s.HaltCh():
			s.log.Debugf("Terminating gracefully.")
			return nil, errors.New("Terminating gracefully.")
		case <-time.After(time.Duration(s.cfg.Debug.InitialMaxPKIRetrievalDelay) * time.Second):
			return nil, errors.New("Timeout failure awaiting first PKI document.")
		case qo = <-s.opCh:
		}
		switch op := qo.(type) {
		case opNewDocument:
			// Determine if PKI doc is valid. If not then abort.
			err := s.isDocValid(op.doc)
			if err != nil {
				s.log.Errorf("Aborting, PKI doc is not valid for the Loopix decoy traffic use case: %v", err)
				err := fmt.Errorf("Aborting, PKI doc is not valid for the Loopix decoy traffic use case: %v", err)
				s.fatalErrCh <- err
				return nil, err
			}
			return op.doc, nil
		default:

			continue
		}
	}
}

func (s *Session) loadKeys(basePath string) error {
	// Load link key.
	var err error
	if s.linkKey, err = config.LoadLinkKey(basePath); err != nil {
		s.log.Errorf("Failure to load link keys: %s", err)
		return err
	}
	return nil
}

// GetService returns a randomly selected service
// matching the specified service name
func (s *Session) GetService(serviceName string) (*utils.ServiceDescriptor, error) {
	doc := s.minclient.CurrentDocument()
	if doc == nil {
		return nil, errors.New("pki doc is nil")
	}
	serviceDescriptors := utils.FindServices(serviceName, doc)
	if len(serviceDescriptors) == 0 {
		return nil, errors.New("GetService failure, service not found in pki doc.")
	}
	return &serviceDescriptors[mrand.Intn(len(serviceDescriptors))], nil
}

// OnConnection will be called by the minclient api
// upon connecting to the Provider
func (s *Session) onConnection(err error) {
	if err == nil {
		s.opCh <- opConnStatusChanged{
			isConnected: true,
		}
	}
}

// OnMessage will be called by the minclient api
// upon receiving a message
func (s *Session) onMessage(ciphertextBlock []byte) error {
	s.log.Debugf("OnMessage")
	return nil
}

func (s *Session) incrementDecoyLoopTally() {
	atomic.AddUint64(&s.decoyLoopTally, 1)
}

func (s *Session) decrementDecoyLoopTally() {
	atomic.AddUint64(&s.decoyLoopTally, ^uint64(1-1))
}

// OnACK is called by the minclient api when we receive a SURB reply message.
func (s *Session) onACK(surbID *[constants.SURBIDLength]byte, ciphertext []byte) error {
	idStr := fmt.Sprintf("[%v]", hex.EncodeToString(surbID[:]))
	s.log.Infof("OnACK with SURBID %x", idStr)
	s.mapLock.Lock()
	defer s.mapLock.Unlock()
	msg, ok := s.surbIDMap[*surbID]
	if !ok {
		s.log.Debug("wtf, received reply with unexpected SURBID")
		return nil
	}
	plaintext, err := sphinx.DecryptSURBPayload(ciphertext, msg.Key)
	if err != nil {
		s.log.Infof("SURB Reply decryption failure: %s", err)
		return err
	}
	if len(plaintext) != coreConstants.ForwardPayloadLength {
		s.log.Warningf("Discarding SURB %v: Invalid payload size: %v", idStr, len(plaintext))
		return nil
	}
	if msg.WithSURB && msg.IsDecoy {
		_, ok := s.surbIDMap[*surbID]
		if ok {
			s.decrementDecoyLoopTally()
			delete(s.surbIDMap, *surbID)
		}
		return nil
	}
	switch msg.SURBType {
	case cConstants.SurbTypeKaetzchen, cConstants.SurbTypeInternal:
		s.eventCh.In() <- &MessageReplyEvent{
			MessageID: msg.ID,
			Payload:   plaintext[2:],
			Err:       nil,
		}
	default:
		s.log.Warningf("Discarding SURB %v: Unknown type: 0x%02x", idStr, msg.SURBType)
	}
	return nil
}

func (s *Session) onDocument(doc *pki.Document) {
	s.log.Debugf("onDocument(): Epoch %v", doc.Epoch)
	s.hasPKIDoc = true
	s.opCh <- opNewDocument{
		doc: doc,
	}
}
<|endoftext|>


<commit_before>// Package gzran is a go implementation of zran by Mark Adler:
// https://github.com/madler/zlib/blob/master/examples/zran.c
// Gzran indexes a gzip file with access points about every span bytes into the
// uncompressed output. The compressed file can be read from randomly once an
// index has been built, having to decompress on the average SPAN/2
// uncompressed bytes before getting to the desired block of data.
package gzran

import (
	"bufio"
	"io"
	"os"

	"github.com/coreos/pkg/gzran/flate"
	"github.com/coreos/pkg/gzran/gzip"
)

const (
	span = 1 << 20 // 1M -- min distance between access points in uncompressed output
)

// Index stores access points into compressed file. Span will determine the
// balance between the speed of random access against the memory requirements
// of the index.
type Index []*point

// Point mimics flate.Decompressor for restoration of decoder state needed for
// random access. This probably saves more state then strictly necessary.
type point struct {
	roffset int64
	woffset int64

	// Input bits, in top of b.
	b  uint32
	nb uint
	// Huffman decoders for literal/length, distance.
	h1, h2 flate.HuffmanDecoder

	// Length arrays used to define Huffman codes.
	bits     *[flate.MaxLit + flate.MaxDist]int
	codebits *[flate.NumCodes]int

	// Output history, buffer.
	hist  *[flate.MaxHist]byte
	hp    int  // current output position in buffer
	hw    int  // have written hist[0:hw] already
	hfull bool // buffer has filled at least once

	// Temporary buffer (avoids repeated allocation).
	buf [4]byte

	step     func(*flate.Decompressor)
	final    bool
	err      error
	hl, hd   *flate.HuffmanDecoder
	copyLen  int
	copyDist int
}

func (idx *Index) addPoint(gz *gzip.Reader, n int64) {
	// convert from io.ReadCloser to flate.Decompresser
	r := gz.Decompressor.(*flate.Decompressor)

	// save decompressor state
	pt := &point{
		woffset: r.Woffset,
		roffset: r.Roffset,
		b:       r.B,
		nb:      r.Nb,
		h1: flate.HuffmanDecoder{
			Min:      r.H1.Min,
			Chunks:   r.H1.Chunks,
			LinkMask: r.H1.LinkMask,
			Links:    make([][]uint32, len(r.H1.Links)),
		},
		h2: flate.HuffmanDecoder{
			Min:      r.H2.Min,
			Chunks:   r.H2.Chunks,
			LinkMask: r.H2.LinkMask,
			Links:    make([][]uint32, len(r.H2.Links)),
		},
		bits:     new([flate.MaxLit + flate.MaxDist]int),
		codebits: new([flate.NumCodes]int),
		hist:     new([flate.MaxHist]byte),
		hp:       r.Hp,
		hw:       r.Hw,
		hfull:    r.Hfull,
		buf:      r.Buf,
		step:     r.Step,
		final:    r.Final,
		err:      r.Err,
		copyLen:  r.CopyLen,
		copyDist: r.CopyDist,
	}

	// save hl and hd
	if r.Hl != nil {
		if r.Hl == &r.H1 {
			pt.hl = &pt.h1
		} else {
			pt.hl = &flate.HuffmanDecoder{
				Min:      r.Hl.Min,
				Chunks:   r.Hl.Chunks,
				LinkMask: r.Hl.LinkMask,
				Links:    make([][]uint32, len(r.Hl.Links)),
			}
			for i := range r.Hl.Links {
				pt.hl.Links[i] = append(pt.hl.Links[i], r.Hl.Links[i]...)
			}
		}
	}
	if r.Hd != nil {
		if r.Hd == &r.H2 {
			pt.hd = &pt.h2
		} else {
			pt.hd = &flate.HuffmanDecoder{
				Min:      r.Hd.Min,
				Chunks:   r.Hd.Chunks,
				LinkMask: r.Hd.LinkMask,
				Links:    make([][]uint32, len(r.Hd.Links)),
			}
			for i := range r.Hd.Links {
				pt.hd.Links[i] = append(pt.hd.Links[i], r.Hd.Links[i]...)
			}
		}

	}
	// copy HuffmanDecoders
	if r.H1.Links != nil {
		for i := range r.H1.Links {
			pt.h1.Links[i] = append(pt.h1.Links[i], r.H1.Links[i]...)
		}
	} else {
		pt.h1.Links = nil
	}
	if r.H2.Links != nil {
		for i := range r.H2.Links {
			pt.h2.Links[i] = append(pt.h2.Links[i], r.H2.Links[i]...)
		}
	} else {
		pt.h2.Links = nil
	}
	// copy hist buf, bits, and codebits
	copy(pt.hist[:], r.Hist[:])
	copy(pt.bits[:], r.Bits[:])
	copy(pt.codebits[:], r.Codebits[:])

	// add point to index
	*idx = append(*idx, pt)
}

// replacement (and copy of) of flate.Read(): always allocates a buffer big
// enough to fully return the block of uncompressed data
func readBlock(f *flate.Decompressor) ([]byte, error) {
	for {
		if len(f.ToRead) > 0 {
			//allocate b to be same size as toRead
			b := make([]byte, len(f.ToRead))

			n := copy(b, f.ToRead)
			f.ToRead = f.ToRead[n:]
			return b, nil
		}
		if f.Err != nil {
			return []byte{}, f.Err
		}
		f.Step(f)
	}
}

// replace (and copy) gzip.Read() to use our own readBlock(). multistream concatenated
// gzips are ignored. mostly a copy of gzip.Read()
func gzReadBlock(z *gzip.Reader) ([]byte, error) {
	if z.Err != nil {
		return []byte{}, z.Err
	}

	zr := z.Decompressor.(*flate.Decompressor) // io.ReadCloser -> flate.Decompressor
	b, err := readBlock(zr)

	z.Digest.Write(b)
	z.Size += uint32(len(b))
	if len(b) != 0 || err != io.EOF {
		z.Err = err
		return b, err
	}

	// Finished file; check checksum + size.
	if _, err := io.ReadFull(z.R, z.Buf[0:8]); err != nil {
		z.Err = err
		return []byte{}, err
	}
	crc32, isize := gzip.Get4(z.Buf[0:4]), gzip.Get4(z.Buf[4:8])
	sum := z.Digest.Sum32()
	if sum != crc32 || isize != z.Size {
		z.Err = gzip.ErrChecksum
		return []byte{}, z.Err
	}

	// multistream not supported
	return []byte{}, io.EOF
}

// readSpan reads blocks until span size or greater bytes have been read.
func readSpan(gz *gzip.Reader) ([]byte, error) {
	var err error
	var block, buf []byte

	for len(buf) < span {
		block, err = gzReadBlock(gz)
		if err != nil {
			return block, err
		}
		buf = append(buf, block...)
	}
	return buf, nil
}

// BuildIndex decompresses given file and builds an index that records the
// state of the gzip decompresser every span bytes into the uncompressed data.
// This index can be used to randomly read uncompressed data from the
// compressed file. Data after the end of the first gzip stream in the
// file is ignored and so concatenated gzip files are not supported.
func BuildIndex(file string) (Index, error) {
	in, err := os.Open(file)
	if err != nil {
		return nil, err
	}
	defer in.Close()

	// read Gzip Header and init our flate.Reader as well
	rr, err := gzip.NewReader(in)
	if err != nil {
		return nil, err
	}

	// concatenated gzips not supported,
	rr.Multistream(false)

	// access point before first block
	idx := new(Index)
	idx.addPoint(rr, 0)

	// create access point in index about every span
	var totalRead int64
	var b []byte
	for {
		b = nil
		b, err = readSpan(rr)
		// don't create index after last block
		if err == io.EOF {
			return *idx, nil
		} else if err != nil {
			return nil, err
		}

		totalRead += int64(len(b))
		idx.addPoint(rr, totalRead)
	}
}

// Restores decompressor to equivalent state as when index access point was taken
func inflateResume(r io.Reader, pt *point) io.ReadCloser {
	f := flate.Decompressor{
		Woffset: pt.woffset,
		Roffset: pt.roffset,
		B:       pt.b,
		Nb:      pt.nb,
		H1: flate.HuffmanDecoder{
			Min:      pt.h1.Min,
			Chunks:   pt.h1.Chunks,
			LinkMask: pt.h1.LinkMask,
			Links:    make([][]uint32, len(pt.h1.Links)),
		},
		H2: flate.HuffmanDecoder{
			Min:      pt.h2.Min,
			Chunks:   pt.h2.Chunks,
			LinkMask: pt.h2.LinkMask,
			Links:    make([][]uint32, len(pt.h2.Links)),
		},
		Bits:     new([flate.MaxLit + flate.MaxDist]int),
		Codebits: new([flate.NumCodes]int),
		Hist:     new([flate.MaxHist]byte),
		Hp:       pt.hp,
		Hw:       pt.hw,
		Hfull:    pt.hfull,
		Buf:      pt.buf,
		Step:     pt.step,
		Final:    pt.final,
		Err:      pt.err,
		CopyLen:  pt.copyLen,
		CopyDist: pt.copyDist,
	}
	// restore hl and hd
	if pt.hl != nil {
		if pt.hl == &pt.h1 {
			f.Hl = &f.H1
		} else {
			f.Hl = &flate.HuffmanDecoder{
				Min:      pt.hl.Min,
				Chunks:   pt.hl.Chunks,
				LinkMask: pt.hl.LinkMask,
				Links:    make([][]uint32, len(pt.hl.Links)),
			}
			for i := range pt.hl.Links {
				f.Hl.Links[i] = append(f.Hl.Links[i], pt.hl.Links[i]...)
			}
		}
	}
	if pt.hd != nil {
		if pt.hd == &pt.h2 {
			f.Hd = &f.H2
		} else {
			f.Hd = &flate.HuffmanDecoder{
				Min:      pt.hd.Min,
				Chunks:   pt.hd.Chunks,
				LinkMask: pt.hd.LinkMask,
				Links:    make([][]uint32, len(pt.hd.Links)),
			}
			for i := range pt.hd.Links {
				f.Hd.Links[i] = append(f.Hd.Links[i], pt.hd.Links[i]...)
			}
		}
	}

	// restore HuffmanDecoders
	for i := range pt.h1.Links {
		f.H1.Links[i] = append(f.H1.Links[i], pt.h1.Links[i]...)
	}
	for i := range pt.h2.Links {
		f.H2.Links[i] = append(f.H2.Links[i], pt.h2.Links[i]...)
	}

	// restore hist buf, bits, and codebits
	copy(f.Hist[:], pt.hist[:])
	copy(f.Bits[:], pt.bits[:])
	copy(f.Codebits[:], pt.codebits[:])

	f.R = bufio.NewReader(r)
	return &f
}

func getHeaderLen(gz *gzip.Reader) int {
	var l int = 10 //min valid gzip header length
	if gz.Flg&gzip.FlagName != 0 {
		l += len(gz.Header.Name) + 1 // +1 for null term
	}
	if gz.Flg&gzip.FlagExtra != 0 {
		l += len(gz.Header.Extra) + 2 // +2 for XLEN
	}
	if gz.Flg&gzip.FlagComment != 0 {
		l += len(gz.Header.Comment) + 1 // +1 for null term
	}
	if gz.Flg&gzip.FlagHdrCrc != 0 {
		l += 2
	}
	return l
}

// Extract uses an Index to read length bytes from offset into uncompressed
// data. If data is requested past the end of the uncompressed data, Extract
// will read less bytes then length and return io.EOF. Offset is zero indexed.
func Extract(filename string, idx Index, offset int64, length int64) ([]byte, error) {
	in, err := os.Open(filename)
	if err != nil {
		return []byte{}, err
	}
	defer in.Close()

	if length <= 0 || idx == nil {
		return []byte{}, nil
	}

	// find access point
	var pt *point
	for i := len(idx) - 1; i >= 0; i-- {
		if idx[i].woffset <= offset {
			pt = idx[i]
			break
		}
	}

	// Read gzip Header to find how many bytes are in the header
	gz, err := gzip.NewReader(in)
	if err != nil {
		return []byte{}, err
	}
	gz.Close()

	// get header length
	headerBytes := getHeaderLen(gz)

	// set file to start of block (roffset + header length)
	_, err = in.Seek(pt.roffset+int64(headerBytes), 0)
	if err != nil {
		return []byte{}, err
	}

	// restore decompresser state
	fr := inflateResume(in, pt)

	// inflate until offset - woffset + bytes have been read, or end of file
	b := make([]byte, length+offset-pt.woffset)
	readWin := b
	var totalRead int
	for {
		n, err := fr.Read(readWin)
		totalRead += n
		readWin = readWin[n:]

		// finished file or read enough
		if n == 0 || totalRead == len(b) || err == io.EOF {
			return b[offset-pt.woffset:], err
		}
	}
}
<commit_msg>fix Extract() return condition<commit_after>// Package gzran is a go implementation of zran by Mark Adler:
// https://github.com/madler/zlib/blob/master/examples/zran.c
// Gzran indexes a gzip file with access points about every span bytes into the
// uncompressed output. The compressed file can be read from randomly once an
// index has been built, having to decompress on the average SPAN/2
// uncompressed bytes before getting to the desired block of data.
package gzran

import (
	"bufio"
	"io"
	"os"

	"github.com/coreos/pkg/gzran/flate"
	"github.com/coreos/pkg/gzran/gzip"
)

const (
	span = 1 << 20 // 1M -- min distance between access points in uncompressed output
)

// Index stores access points into compressed file. Span will determine the
// balance between the speed of random access against the memory requirements
// of the index.
type Index []*point

// Point mimics flate.Decompressor for restoration of decoder state needed for
// random access. This probably saves more state then strictly necessary.
type point struct {
	roffset int64
	woffset int64

	// Input bits, in top of b.
	b  uint32
	nb uint
	// Huffman decoders for literal/length, distance.
	h1, h2 flate.HuffmanDecoder

	// Length arrays used to define Huffman codes.
	bits     *[flate.MaxLit + flate.MaxDist]int
	codebits *[flate.NumCodes]int

	// Output history, buffer.
	hist  *[flate.MaxHist]byte
	hp    int  // current output position in buffer
	hw    int  // have written hist[0:hw] already
	hfull bool // buffer has filled at least once

	// Temporary buffer (avoids repeated allocation).
	buf [4]byte

	step     func(*flate.Decompressor)
	final    bool
	err      error
	hl, hd   *flate.HuffmanDecoder
	copyLen  int
	copyDist int
}

func (idx *Index) addPoint(gz *gzip.Reader, n int64) {
	// convert from io.ReadCloser to flate.Decompresser
	r := gz.Decompressor.(*flate.Decompressor)

	// save decompressor state
	pt := &point{
		woffset: r.Woffset,
		roffset: r.Roffset,
		b:       r.B,
		nb:      r.Nb,
		h1: flate.HuffmanDecoder{
			Min:      r.H1.Min,
			Chunks:   r.H1.Chunks,
			LinkMask: r.H1.LinkMask,
			Links:    make([][]uint32, len(r.H1.Links)),
		},
		h2: flate.HuffmanDecoder{
			Min:      r.H2.Min,
			Chunks:   r.H2.Chunks,
			LinkMask: r.H2.LinkMask,
			Links:    make([][]uint32, len(r.H2.Links)),
		},
		bits:     new([flate.MaxLit + flate.MaxDist]int),
		codebits: new([flate.NumCodes]int),
		hist:     new([flate.MaxHist]byte),
		hp:       r.Hp,
		hw:       r.Hw,
		hfull:    r.Hfull,
		buf:      r.Buf,
		step:     r.Step,
		final:    r.Final,
		err:      r.Err,
		copyLen:  r.CopyLen,
		copyDist: r.CopyDist,
	}

	// save hl and hd
	if r.Hl != nil {
		if r.Hl == &r.H1 {
			pt.hl = &pt.h1
		} else {
			pt.hl = &flate.HuffmanDecoder{
				Min:      r.Hl.Min,
				Chunks:   r.Hl.Chunks,
				LinkMask: r.Hl.LinkMask,
				Links:    make([][]uint32, len(r.Hl.Links)),
			}
			for i := range r.Hl.Links {
				pt.hl.Links[i] = append(pt.hl.Links[i], r.Hl.Links[i]...)
			}
		}
	}
	if r.Hd != nil {
		if r.Hd == &r.H2 {
			pt.hd = &pt.h2
		} else {
			pt.hd = &flate.HuffmanDecoder{
				Min:      r.Hd.Min,
				Chunks:   r.Hd.Chunks,
				LinkMask: r.Hd.LinkMask,
				Links:    make([][]uint32, len(r.Hd.Links)),
			}
			for i := range r.Hd.Links {
				pt.hd.Links[i] = append(pt.hd.Links[i], r.Hd.Links[i]...)
			}
		}

	}
	// copy HuffmanDecoders
	if r.H1.Links != nil {
		for i := range r.H1.Links {
			pt.h1.Links[i] = append(pt.h1.Links[i], r.H1.Links[i]...)
		}
	} else {
		pt.h1.Links = nil
	}
	if r.H2.Links != nil {
		for i := range r.H2.Links {
			pt.h2.Links[i] = append(pt.h2.Links[i], r.H2.Links[i]...)
		}
	} else {
		pt.h2.Links = nil
	}
	// copy hist buf, bits, and codebits
	copy(pt.hist[:], r.Hist[:])
	copy(pt.bits[:], r.Bits[:])
	copy(pt.codebits[:], r.Codebits[:])

	// add point to index
	*idx = append(*idx, pt)
}

// replacement (and copy of) of flate.Read(): always allocates a buffer big
// enough to fully return the block of uncompressed data
func readBlock(f *flate.Decompressor) ([]byte, error) {
	for {
		if len(f.ToRead) > 0 {
			//allocate b to be same size as toRead
			b := make([]byte, len(f.ToRead))

			n := copy(b, f.ToRead)
			f.ToRead = f.ToRead[n:]
			return b, nil
		}
		if f.Err != nil {
			return []byte{}, f.Err
		}
		f.Step(f)
	}
}

// replace (and copy) gzip.Read() to use our own readBlock(). multistream concatenated
// gzips are ignored. mostly a copy of gzip.Read()
func gzReadBlock(z *gzip.Reader) ([]byte, error) {
	if z.Err != nil {
		return []byte{}, z.Err
	}

	zr := z.Decompressor.(*flate.Decompressor) // io.ReadCloser -> flate.Decompressor
	b, err := readBlock(zr)

	z.Digest.Write(b)
	z.Size += uint32(len(b))
	if len(b) != 0 || err != io.EOF {
		z.Err = err
		return b, err
	}

	// Finished file; check checksum + size.
	if _, err := io.ReadFull(z.R, z.Buf[0:8]); err != nil {
		z.Err = err
		return []byte{}, err
	}
	crc32, isize := gzip.Get4(z.Buf[0:4]), gzip.Get4(z.Buf[4:8])
	sum := z.Digest.Sum32()
	if sum != crc32 || isize != z.Size {
		z.Err = gzip.ErrChecksum
		return []byte{}, z.Err
	}

	// multistream not supported
	return []byte{}, io.EOF
}

// readSpan reads blocks until span size or greater bytes have been read.
func readSpan(gz *gzip.Reader) ([]byte, error) {
	var err error
	var block, buf []byte

	for len(buf) < span {
		block, err = gzReadBlock(gz)
		if err != nil {
			return block, err
		}
		buf = append(buf, block...)
	}
	return buf, nil
}

// BuildIndex decompresses given file and builds an index that records the
// state of the gzip decompresser every span bytes into the uncompressed data.
// This index can be used to randomly read uncompressed data from the
// compressed file. Data after the end of the first gzip stream in the
// file is ignored and so concatenated gzip files are not supported.
func BuildIndex(file string) (Index, error) {
	in, err := os.Open(file)
	if err != nil {
		return nil, err
	}
	defer in.Close()

	// read Gzip Header and init our flate.Reader as well
	rr, err := gzip.NewReader(in)
	if err != nil {
		return nil, err
	}

	// concatenated gzips not supported,
	rr.Multistream(false)

	// access point before first block
	idx := new(Index)
	idx.addPoint(rr, 0)

	// create access point in index about every span
	var totalRead int64
	var b []byte
	for {
		b = nil
		b, err = readSpan(rr)
		// don't create index after last block
		if err == io.EOF {
			return *idx, nil
		} else if err != nil {
			return nil, err
		}

		totalRead += int64(len(b))
		idx.addPoint(rr, totalRead)
	}
}

// Restores decompressor to equivalent state as when index access point was taken
func inflateResume(r io.Reader, pt *point) io.ReadCloser {
	f := flate.Decompressor{
		Woffset: pt.woffset,
		Roffset: pt.roffset,
		B:       pt.b,
		Nb:      pt.nb,
		H1: flate.HuffmanDecoder{
			Min:      pt.h1.Min,
			Chunks:   pt.h1.Chunks,
			LinkMask: pt.h1.LinkMask,
			Links:    make([][]uint32, len(pt.h1.Links)),
		},
		H2: flate.HuffmanDecoder{
			Min:      pt.h2.Min,
			Chunks:   pt.h2.Chunks,
			LinkMask: pt.h2.LinkMask,
			Links:    make([][]uint32, len(pt.h2.Links)),
		},
		Bits:     new([flate.MaxLit + flate.MaxDist]int),
		Codebits: new([flate.NumCodes]int),
		Hist:     new([flate.MaxHist]byte),
		Hp:       pt.hp,
		Hw:       pt.hw,
		Hfull:    pt.hfull,
		Buf:      pt.buf,
		Step:     pt.step,
		Final:    pt.final,
		Err:      pt.err,
		CopyLen:  pt.copyLen,
		CopyDist: pt.copyDist,
	}
	// restore hl and hd
	if pt.hl != nil {
		if pt.hl == &pt.h1 {
			f.Hl = &f.H1
		} else {
			f.Hl = &flate.HuffmanDecoder{
				Min:      pt.hl.Min,
				Chunks:   pt.hl.Chunks,
				LinkMask: pt.hl.LinkMask,
				Links:    make([][]uint32, len(pt.hl.Links)),
			}
			for i := range pt.hl.Links {
				f.Hl.Links[i] = append(f.Hl.Links[i], pt.hl.Links[i]...)
			}
		}
	}
	if pt.hd != nil {
		if pt.hd == &pt.h2 {
			f.Hd = &f.H2
		} else {
			f.Hd = &flate.HuffmanDecoder{
				Min:      pt.hd.Min,
				Chunks:   pt.hd.Chunks,
				LinkMask: pt.hd.LinkMask,
				Links:    make([][]uint32, len(pt.hd.Links)),
			}
			for i := range pt.hd.Links {
				f.Hd.Links[i] = append(f.Hd.Links[i], pt.hd.Links[i]...)
			}
		}
	}

	// restore HuffmanDecoders
	for i := range pt.h1.Links {
		f.H1.Links[i] = append(f.H1.Links[i], pt.h1.Links[i]...)
	}
	for i := range pt.h2.Links {
		f.H2.Links[i] = append(f.H2.Links[i], pt.h2.Links[i]...)
	}

	// restore hist buf, bits, and codebits
	copy(f.Hist[:], pt.hist[:])
	copy(f.Bits[:], pt.bits[:])
	copy(f.Codebits[:], pt.codebits[:])

	f.R = bufio.NewReader(r)
	return &f
}

func getHeaderLen(gz *gzip.Reader) int {
	var l int = 10 //min valid gzip header length
	if gz.Flg&gzip.FlagName != 0 {
		l += len(gz.Header.Name) + 1 // +1 for null term
	}
	if gz.Flg&gzip.FlagExtra != 0 {
		l += len(gz.Header.Extra) + 2 // +2 for XLEN
	}
	if gz.Flg&gzip.FlagComment != 0 {
		l += len(gz.Header.Comment) + 1 // +1 for null term
	}
	if gz.Flg&gzip.FlagHdrCrc != 0 {
		l += 2
	}
	return l
}

// Extract uses an Index to read length bytes from offset into uncompressed
// data. If data is requested past the end of the uncompressed data, Extract
// will read less bytes then length and return io.EOF. Offset is zero indexed.
func Extract(filename string, idx Index, offset int64, length int64) ([]byte, error) {
	in, err := os.Open(filename)
	if err != nil {
		return []byte{}, err
	}
	defer in.Close()

	if length <= 0 || idx == nil {
		return []byte{}, nil
	}

	// find access point
	var pt *point
	for i := len(idx) - 1; i >= 0; i-- {
		if idx[i].woffset <= offset {
			pt = idx[i]
			break
		}
	}

	// Read gzip Header to find how many bytes are in the header
	gz, err := gzip.NewReader(in)
	if err != nil {
		return []byte{}, err
	}
	gz.Close()

	// get header length
	headerBytes := getHeaderLen(gz)

	// set file to start of block (roffset + header length)
	_, err = in.Seek(pt.roffset+int64(headerBytes), 0)
	if err != nil {
		return []byte{}, err
	}

	// restore decompresser state
	fr := inflateResume(in, pt)

	// inflate until offset - woffset + bytes have been read, or end of file
	b := make([]byte, length+offset-pt.woffset)
	readWin := b
	var totalRead int
	for {
		n, err := fr.Read(readWin)
		totalRead += n
		readWin = readWin[n:]

		// finished file or read enough
		if n == 0 || totalRead == len(b) || err == io.EOF {
			return b[offset-pt.woffset:], err
		} else if err != nil {
			return []byte{}, err
		}
	}
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package api

import (
	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/utils/formatting"
	"github.com/ava-labs/avalanchego/utils/json"
)

// This file contains structs used in arguments and responses in services

// SuccessResponse indicates success of an API call
type SuccessResponse struct {
	Success bool `json:"success"`
}

// JSONTxID contains the ID of a transaction
type JSONTxID struct {
	TxID ids.ID `json:"txID"`
}

// UserPass contains a username and a password
type UserPass struct {
	Username string `json:"username"`
	Password string `json:"password"`
}

// JSONAddress contains an address
type JSONAddress struct {
	Address string `json:"address"`
}

// JSONAddresses contains a list of address
type JSONAddresses struct {
	Addresses []string `json:"addresses"`
}

// JSONChangeAddr is the address change is sent to, if any
type JSONChangeAddr struct {
	ChangeAddr string `json:"changeAddr"`
}

// JSONTxIDChangeAddr is a tx ID and change address
type JSONTxIDChangeAddr struct {
	JSONTxID
	JSONChangeAddr
}

// JSONFromAddrs is a list of addresses to send funds from
type JSONFromAddrs struct {
	From []string `json:"from"`
}

// JSONSpendHeader is 3 arguments to a method that spends (including those with tx fees)
// 1) The username/password
// 2) The addresses used in the method
// 3) The address to send change to
type JSONSpendHeader struct {
	UserPass
	JSONFromAddrs
	JSONChangeAddr
}

// GetTxArgs ...
type GetTxArgs struct {
	TxID     ids.ID              `json:"txID"`
	Encoding formatting.Encoding `json:"encoding"`
}

// JSONTx JSON representation of the transaction object
type JSONTx struct {
	ID     string `json:"id"`
	Status string `json:"status"`
	// Outputs we use []interface{} here since we cannot reference []*avax.UTXO due to circular dependency problem
	Outputs []interface{} `json:"outputs"`
	// Inputs we use []interface{} here since we cannot reference []*avax.UTXOID due to circular dependency problem
	Inputs []interface{} `json:"inputs"`
}

type GetTxReply struct {
	Tx       interface{}         `json:"tx,omitempty"`
	Encoding formatting.Encoding `json:"encoding"`
}

type FormattedTx struct {
	Tx       string              `json:"tx,omitempty"`
	Encoding formatting.Encoding `json:"encoding"`
}

// Index is an address and an associated UTXO.
// Marks a starting or stopping point when fetching UTXOs. Used for pagination.
type Index struct {
	Address string `json:"address"` // The address as a string
	UTXO    string `json:"utxo"`    // The UTXO ID as a string
}

// GetUTXOsArgs are arguments for passing into GetUTXOs.
// Gets the UTXOs that reference at least one address in [Addresses].
// Returns at most [limit] addresses.
// If specified, [SourceChain] is the chain where the atomic UTXOs were exported from. If empty,
// or the Chain ID of this VM is specified, then GetUTXOs fetches the native UTXOs.
// If [limit] == 0 or > [maxUTXOsToFetch], fetches up to [maxUTXOsToFetch].
// [StartIndex] defines where to start fetching UTXOs (for pagination.)
// UTXOs fetched are from addresses equal to or greater than [StartIndex.Address]
// For address [StartIndex.Address], only UTXOs with IDs greater than [StartIndex.UTXO] will be returned.
// If [StartIndex] is omitted, gets all UTXOs.
// If GetUTXOs is called multiple times, with our without [StartIndex], it is not guaranteed
// that returned UTXOs are unique. That is, the same UTXO may appear in the response of multiple calls.
type GetUTXOsArgs struct {
	Addresses   []string            `json:"addresses"`
	SourceChain string              `json:"sourceChain"`
	Limit       json.Uint32         `json:"limit"`
	StartIndex  Index               `json:"startIndex"`
	Encoding    formatting.Encoding `json:"encoding"`
}

// GetUTXOsReply defines the GetUTXOs replies returned from the API
type GetUTXOsReply struct {
	// Number of UTXOs returned
	NumFetched json.Uint64 `json:"numFetched"`
	// The UTXOs
	UTXOs []string `json:"utxos"`
	// The last UTXO that was returned, and the address it corresponds to.
	// Used for pagination. To get the rest of the UTXOs, call GetUTXOs
	// again and set [StartIndex] to this value.
	EndIndex Index `json:"endIndex"`
	// Encoding specifies the encoding format the UTXOs are returned in
	Encoding formatting.Encoding `json:"encoding"`
}
<commit_msg>comments<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package api

import (
	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/utils/formatting"
	"github.com/ava-labs/avalanchego/utils/json"
)

// This file contains structs used in arguments and responses in services

// SuccessResponse indicates success of an API call
type SuccessResponse struct {
	Success bool `json:"success"`
}

// JSONTxID contains the ID of a transaction
type JSONTxID struct {
	TxID ids.ID `json:"txID"`
}

// UserPass contains a username and a password
type UserPass struct {
	Username string `json:"username"`
	Password string `json:"password"`
}

// JSONAddress contains an address
type JSONAddress struct {
	Address string `json:"address"`
}

// JSONAddresses contains a list of address
type JSONAddresses struct {
	Addresses []string `json:"addresses"`
}

// JSONChangeAddr is the address change is sent to, if any
type JSONChangeAddr struct {
	ChangeAddr string `json:"changeAddr"`
}

// JSONTxIDChangeAddr is a tx ID and change address
type JSONTxIDChangeAddr struct {
	JSONTxID
	JSONChangeAddr
}

// JSONFromAddrs is a list of addresses to send funds from
type JSONFromAddrs struct {
	From []string `json:"from"`
}

// JSONSpendHeader is 3 arguments to a method that spends (including those with tx fees)
// 1) The username/password
// 2) The addresses used in the method
// 3) The address to send change to
type JSONSpendHeader struct {
	UserPass
	JSONFromAddrs
	JSONChangeAddr
}

// GetTxArgs ...
type GetTxArgs struct {
	TxID     ids.ID              `json:"txID"`
	Encoding formatting.Encoding `json:"encoding"`
}

// JSONTx JSON representation of the transaction object
type JSONTx struct {
	ID     string `json:"id"`
	Status string `json:"status"`
	// Outputs we use []interface{} here since we cannot reference []*avax.UTXO due to circular dependency problem
	Outputs []interface{} `json:"outputs"`
	// Inputs we use []interface{} here since we cannot reference []*avax.UTXOID due to circular dependency problem
	Inputs []interface{} `json:"inputs"`
}

type GetTxReply struct {
	// Tx is in [string] format when [GetTxArgs.Encoding] is [Encoding.Hex] or [Encoding.CB58]
	// Tx is an JSONTx object when [GetTxArgs.Encoding] is [Encoding.JSON]
	Tx       interface{}         `json:"tx"`
	Encoding formatting.Encoding `json:"encoding"`
}

type FormattedTx struct {
	Tx       string              `json:"tx,omitempty"`
	Encoding formatting.Encoding `json:"encoding"`
}

// Index is an address and an associated UTXO.
// Marks a starting or stopping point when fetching UTXOs. Used for pagination.
type Index struct {
	Address string `json:"address"` // The address as a string
	UTXO    string `json:"utxo"`    // The UTXO ID as a string
}

// GetUTXOsArgs are arguments for passing into GetUTXOs.
// Gets the UTXOs that reference at least one address in [Addresses].
// Returns at most [limit] addresses.
// If specified, [SourceChain] is the chain where the atomic UTXOs were exported from. If empty,
// or the Chain ID of this VM is specified, then GetUTXOs fetches the native UTXOs.
// If [limit] == 0 or > [maxUTXOsToFetch], fetches up to [maxUTXOsToFetch].
// [StartIndex] defines where to start fetching UTXOs (for pagination.)
// UTXOs fetched are from addresses equal to or greater than [StartIndex.Address]
// For address [StartIndex.Address], only UTXOs with IDs greater than [StartIndex.UTXO] will be returned.
// If [StartIndex] is omitted, gets all UTXOs.
// If GetUTXOs is called multiple times, with our without [StartIndex], it is not guaranteed
// that returned UTXOs are unique. That is, the same UTXO may appear in the response of multiple calls.
type GetUTXOsArgs struct {
	Addresses   []string            `json:"addresses"`
	SourceChain string              `json:"sourceChain"`
	Limit       json.Uint32         `json:"limit"`
	StartIndex  Index               `json:"startIndex"`
	Encoding    formatting.Encoding `json:"encoding"`
}

// GetUTXOsReply defines the GetUTXOs replies returned from the API
type GetUTXOsReply struct {
	// Number of UTXOs returned
	NumFetched json.Uint64 `json:"numFetched"`
	// The UTXOs
	UTXOs []string `json:"utxos"`
	// The last UTXO that was returned, and the address it corresponds to.
	// Used for pagination. To get the rest of the UTXOs, call GetUTXOs
	// again and set [StartIndex] to this value.
	EndIndex Index `json:"endIndex"`
	// Encoding specifies the encoding format the UTXOs are returned in
	Encoding formatting.Encoding `json:"encoding"`
}
<|endoftext|>


<commit_before>package margelet

import (
	"github.com/Syfaro/telegram-bot-api"
	"gopkg.in/redis.v3"
)

// Responder - interface for message responders
type Responder interface {
	Response(bot MargeletAPI, message tgbotapi.Message) error
}

// CommandHandler - interface for command handlers
type CommandHandler interface {
	Responder
	HelpMessage() string
}

// SessionHandler - interface for session handlers
type SessionHandler interface {
	HandleResponse(bot MargeletAPI, message tgbotapi.Message, responses []string) (bool, error)
	HelpMessage() string
}

// MargeletAPI - interface, that describes margelet API
type MargeletAPI interface {
	Send(c tgbotapi.Chattable) (tgbotapi.Message, error)
	QuickSend(chatID int, message string) (tgbotapi.Message, error)
	QuickReply(chatID, messageID int, message string) (tgbotapi.Message, error)

	GetFileDirectURL(fileID string) (string, error)
	IsMessageToMe(message tgbotapi.Message) bool
	GetConfigRepository() *ChatConfigRepository
	GetSessionRepository() *SessionRepository
	GetRedis() *redis.Client
	HandleSession(message tgbotapi.Message, handler SessionHandler)
}

// TGBotAPI - interface, thar describe telegram-bot-api API
type TGBotAPI interface {
	Send(c tgbotapi.Chattable) (tgbotapi.Message, error)
	GetFileDirectURL(fileID string) (string, error)
	IsMessageToMe(message tgbotapi.Message) bool
	GetUpdatesChan(config tgbotapi.UpdateConfig) (<-chan tgbotapi.Update, error)
}
<commit_msg>typo fix<commit_after>package margelet

import (
	"github.com/Syfaro/telegram-bot-api"
	"gopkg.in/redis.v3"
)

// Responder - interface for message responders
type Responder interface {
	Response(bot MargeletAPI, message tgbotapi.Message) error
}

// CommandHandler - interface for command handlers
type CommandHandler interface {
	Responder
	HelpMessage() string
}

// SessionHandler - interface for session handlers
type SessionHandler interface {
	HandleResponse(bot MargeletAPI, message tgbotapi.Message, responses []string) (bool, error)
	HelpMessage() string
}

// MargeletAPI - interface, that describes margelet API
type MargeletAPI interface {
	Send(c tgbotapi.Chattable) (tgbotapi.Message, error)
	QuickSend(chatID int, message string) (tgbotapi.Message, error)
	QuickReply(chatID, messageID int, message string) (tgbotapi.Message, error)
	GetFileDirectURL(fileID string) (string, error)
	IsMessageToMe(message tgbotapi.Message) bool
	GetConfigRepository() *ChatConfigRepository
	GetSessionRepository() *SessionRepository
	GetRedis() *redis.Client
	HandleSession(message tgbotapi.Message, handler SessionHandler)
}

// TGBotAPI - interface, thar describe telegram-bot-api API
type TGBotAPI interface {
	Send(c tgbotapi.Chattable) (tgbotapi.Message, error)
	GetFileDirectURL(fileID string) (string, error)
	IsMessageToMe(message tgbotapi.Message) bool
	GetUpdatesChan(config tgbotapi.UpdateConfig) (<-chan tgbotapi.Update, error)
}
<|endoftext|>