<commit_before>package ca

import (
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/hex"
	"fmt"
	"log"
	"math"
	"math/big"
	"net"
	"time"

	"github.com/letsencrypt/pebble/acme"
	"github.com/letsencrypt/pebble/core"
	"github.com/letsencrypt/pebble/db"
)

const (
	rootCAPrefix         = "Pebble Root CA "
	intermediateCAPrefix = "Pebble Intermediate CA "
)

type CAImpl struct {
	log              *log.Logger
	db               *db.MemoryStore
	ocspResponderURL string

	chains []*chain
}

type chain struct {
	root         *issuer
	intermediate *issuer
}

type issuer struct {
	key  crypto.Signer
	cert *core.Certificate
}

func makeSerial() *big.Int {
	serial, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		panic(fmt.Sprintf("unable to create random serial number: %s", err.Error()))
	}
	return serial
}

// makeKey and makeRootCert are adapted from MiniCA:
// https://github.com/jsha/minica/blob/3a621c05b61fa1c24bcb42fbde4b261db504a74f/main.go

// makeKey creates a new 2048 bit RSA private key
func makeKey() (*rsa.PrivateKey, error) {
	key, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, err
	}
	return key, nil
}

func (ca *CAImpl) makeRootCert(
	subjectKey crypto.Signer,
	subjCNPrefix string,
	signer *issuer) (*core.Certificate, error) {

	serial := makeSerial()
	template := &x509.Certificate{
		Subject: pkix.Name{
			CommonName: subjCNPrefix + hex.EncodeToString(serial.Bytes()[:3]),
		},
		SerialNumber: serial,
		NotBefore:    time.Now(),
		NotAfter:     time.Now().AddDate(30, 0, 0),

		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		BasicConstraintsValid: true,
		IsCA:                  true,
	}

	var signerKey crypto.Signer
	var parent *x509.Certificate
	if signer != nil && signer.key != nil && signer.cert != nil && signer.cert.Cert != nil {
		signerKey = signer.key
		parent = signer.cert.Cert
	} else {
		signerKey = subjectKey
		parent = template
	}

	der, err := x509.CreateCertificate(rand.Reader, template, parent, subjectKey.Public(), signerKey)
	if err != nil {
		return nil, err
	}

	cert, err := x509.ParseCertificate(der)
	if err != nil {
		return nil, err
	}

	hexSerial := hex.EncodeToString(cert.SerialNumber.Bytes())
	newCert := &core.Certificate{
		ID:   hexSerial,
		Cert: cert,
		DER:  der,
	}
	if signer != nil && signer.cert != nil {
		newCert.Issuers = make([]*core.Certificate, 1)
		newCert.Issuers[0] = signer.cert
	}
	_, err = ca.db.AddCertificate(newCert)
	if err != nil {
		return nil, err
	}
	return newCert, nil
}

func (ca *CAImpl) newRootIssuer() (*issuer, error) {
	// Make a root private key
	rk, err := makeKey()
	if err != nil {
		return nil, err
	}
	// Make a self-signed root certificate
	rc, err := ca.makeRootCert(rk, rootCAPrefix, nil)
	if err != nil {
		return nil, err
	}

	ca.log.Printf("Generated new root issuer with serial %s\n", rc.ID)
	return &issuer{
		key:  rk,
		cert: rc,
	}, nil
}

func (ca *CAImpl) newIntermediateIssuer(root *issuer, ik crypto.Signer) (*issuer, error) {
	if root == nil {
		return nil, fmt.Errorf("Internal error: root must not be nil")
	}

	// Make an intermediate certificate with the root issuer
	ic, err := ca.makeRootCert(ik, intermediateCAPrefix, root)
	if err != nil {
		return nil, err
	}
	ca.log.Printf("Generated new intermediate issuer with serial %s\n", ic.ID)
	return &issuer{
		key:  ik,
		cert: ic,
	}, nil
}

func (ca *CAImpl) newChain(ik crypto.Signer) *chain {
	root, err := ca.newRootIssuer()
	if err != nil {
		panic(fmt.Sprintf("Error creating new root issuer: %s", err.Error()))
	}
	intermediate, err := ca.newIntermediateIssuer(root, ik)
	if err != nil {
		panic(fmt.Sprintf("Error creating new intermediate issuer: %s", err.Error()))
	}
	return &chain{
		root:         root,
		intermediate: intermediate,
	}
}

func (ca *CAImpl) newCertificate(domains []string, ips []net.IP, key crypto.PublicKey, accountID string) (*core.Certificate, error) {
	var cn string
	if len(domains) > 0 {
		cn = domains[0]
	} else if len(ips) > 0 {
		cn = ips[0].String()
	} else {
		return nil, fmt.Errorf("must specify at least one domain name or IP address")
	}

	issuer := ca.chains[0].intermediate
	if issuer == nil || issuer.cert == nil {
		return nil, fmt.Errorf("cannot sign certificate - nil issuer")
	}

	serial := makeSerial()
	template := &x509.Certificate{
		DNSNames:    domains,
		IPAddresses: ips,
		Subject: pkix.Name{
			CommonName: cn,
		},
		SerialNumber: serial,
		NotBefore:    time.Now(),
		NotAfter:     time.Now().AddDate(5, 0, 0),

		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		BasicConstraintsValid: true,
		IsCA:                  false,
	}

	if ca.ocspResponderURL != "" {
		template.OCSPServer = []string{ca.ocspResponderURL}
	}

	der, err := x509.CreateCertificate(rand.Reader, template, issuer.cert.Cert, key, issuer.key)
	if err != nil {
		return nil, err
	}
	cert, err := x509.ParseCertificate(der)
	if err != nil {
		return nil, err
	}

	issuers := make([]*core.Certificate, len(ca.chains))
	for i := 0; i < len(ca.chains); i++ {
		issuers[i] = ca.chains[i].intermediate.cert
	}

	hexSerial := hex.EncodeToString(cert.SerialNumber.Bytes())
	newCert := &core.Certificate{
		ID:        hexSerial,
		AccountID: accountID,
		Cert:      cert,
		DER:       der,
		Issuers:   issuers,
	}
	_, err = ca.db.AddCertificate(newCert)
	if err != nil {
		return nil, err
	}
	return newCert, nil
}

func New(log *log.Logger, db *db.MemoryStore, ocspResponderURL string, alternateRoots int) *CAImpl {
	ca := &CAImpl{
		log: log,
		db:  db,
	}

	if ocspResponderURL != "" {
		ca.ocspResponderURL = ocspResponderURL
		ca.log.Printf("Setting OCSP responder URL for issued certificates to %q", ca.ocspResponderURL)
	}

	ik, err := makeKey()
	if err != nil {
		panic(fmt.Sprintf("Error creating new intermediate private key: %s", err.Error()))
	}
	ca.chains = make([]*chain, 1+alternateRoots)
	for i := 0; i < len(ca.chains); i++ {
		ca.chains[i] = ca.newChain(ik)
	}
	return ca
}

func (ca *CAImpl) CompleteOrder(order *core.Order) {
	// Lock the order for reading
	order.RLock()
	// If the order isn't set as beganProcessing produce an error and immediately unlock
	if !order.BeganProcessing {
		ca.log.Printf("Error: Asked to complete order %s which had false beganProcessing.",
			order.ID)
		order.RUnlock()
		return
	}
	// Unlock the order again
	order.RUnlock()

	// Check the authorizations - this is done by the VA before calling
	// CompleteOrder but we do it again for robustness sake.
	for _, authz := range order.AuthorizationObjects {
		// Lock the authorization for reading
		authz.RLock()
		if authz.Status != acme.StatusValid {
			return
		}
		authz.RUnlock()
	}

	// issue a certificate for the csr
	csr := order.ParsedCSR
	cert, err := ca.newCertificate(csr.DNSNames, csr.IPAddresses, csr.PublicKey, order.AccountID)
	if err != nil {
		ca.log.Printf("Error: unable to issue order: %s", err.Error())
		return
	}
	ca.log.Printf("Issued certificate serial %s for order %s\n", cert.ID, order.ID)

	// Lock and update the order to store the issued certificate
	order.Lock()
	order.CertificateObject = cert
	order.Unlock()
}

func (ca *CAImpl) GetNumberOfRootCerts() int {
	return len(ca.chains)
}

func (ca *CAImpl) getChain(no int) *chain {
	if 0 <= no && no < len(ca.chains) {
		return ca.chains[no]
	}
	return nil
}

func (ca *CAImpl) GetRootCert(no int) *core.Certificate {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}
	return chain.root.cert
}

func (ca *CAImpl) GetRootKey(no int) *rsa.PrivateKey {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}

	switch key := chain.root.key.(type) {
	case *rsa.PrivateKey:
		return key
	}
	return nil
}

func (ca *CAImpl) GetIntermediateCert(no int) *core.Certificate {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}
	return chain.intermediate.cert
}

func (ca *CAImpl) GetIntermediateKey(no int) *rsa.PrivateKey {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}

	switch key := chain.intermediate.key.(type) {
	case *rsa.PrivateKey:
		return key
	}
	return nil
}
<commit_msg>Make sure all intermediate certificates have the same subject. (#276)<commit_after>package ca

import (
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/hex"
	"fmt"
	"log"
	"math"
	"math/big"
	"net"
	"time"

	"github.com/letsencrypt/pebble/acme"
	"github.com/letsencrypt/pebble/core"
	"github.com/letsencrypt/pebble/db"
)

const (
	rootCAPrefix         = "Pebble Root CA "
	intermediateCAPrefix = "Pebble Intermediate CA "
)

type CAImpl struct {
	log              *log.Logger
	db               *db.MemoryStore
	ocspResponderURL string

	chains []*chain
}

type chain struct {
	root         *issuer
	intermediate *issuer
}

type issuer struct {
	key  crypto.Signer
	cert *core.Certificate
}

func makeSerial() *big.Int {
	serial, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		panic(fmt.Sprintf("unable to create random serial number: %s", err.Error()))
	}
	return serial
}

// makeKey and makeRootCert are adapted from MiniCA:
// https://github.com/jsha/minica/blob/3a621c05b61fa1c24bcb42fbde4b261db504a74f/main.go

// makeKey creates a new 2048 bit RSA private key
func makeKey() (*rsa.PrivateKey, error) {
	key, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, err
	}
	return key, nil
}

func (ca *CAImpl) makeRootCert(
	subjectKey crypto.Signer,
	subject pkix.Name,
	signer *issuer) (*core.Certificate, error) {

	serial := makeSerial()
	template := &x509.Certificate{
		Subject:      subject,
		SerialNumber: serial,
		NotBefore:    time.Now(),
		NotAfter:     time.Now().AddDate(30, 0, 0),

		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		BasicConstraintsValid: true,
		IsCA:                  true,
	}

	var signerKey crypto.Signer
	var parent *x509.Certificate
	if signer != nil && signer.key != nil && signer.cert != nil && signer.cert.Cert != nil {
		signerKey = signer.key
		parent = signer.cert.Cert
	} else {
		signerKey = subjectKey
		parent = template
	}

	der, err := x509.CreateCertificate(rand.Reader, template, parent, subjectKey.Public(), signerKey)
	if err != nil {
		return nil, err
	}

	cert, err := x509.ParseCertificate(der)
	if err != nil {
		return nil, err
	}

	hexSerial := hex.EncodeToString(cert.SerialNumber.Bytes())
	newCert := &core.Certificate{
		ID:   hexSerial,
		Cert: cert,
		DER:  der,
	}
	if signer != nil && signer.cert != nil {
		newCert.Issuers = make([]*core.Certificate, 1)
		newCert.Issuers[0] = signer.cert
	}
	_, err = ca.db.AddCertificate(newCert)
	if err != nil {
		return nil, err
	}
	return newCert, nil
}

func (ca *CAImpl) newRootIssuer() (*issuer, error) {
	// Make a root private key
	rk, err := makeKey()
	if err != nil {
		return nil, err
	}
	// Make a self-signed root certificate
	subject := pkix.Name{
		CommonName: rootCAPrefix + hex.EncodeToString(makeSerial().Bytes()[:3]),
	}
	rc, err := ca.makeRootCert(rk, subject, nil)
	if err != nil {
		return nil, err
	}

	ca.log.Printf("Generated new root issuer with serial %s\n", rc.ID)
	return &issuer{
		key:  rk,
		cert: rc,
	}, nil
}

func (ca *CAImpl) newIntermediateIssuer(root *issuer, intermediateKey crypto.Signer, subject pkix.Name) (*issuer, error) {
	if root == nil {
		return nil, fmt.Errorf("Internal error: root must not be nil")
	}
	// Make an intermediate certificate with the root issuer
	ic, err := ca.makeRootCert(intermediateKey, subject, root)
	if err != nil {
		return nil, err
	}
	ca.log.Printf("Generated new intermediate issuer with serial %s\n", ic.ID)
	return &issuer{
		key:  intermediateKey,
		cert: ic,
	}, nil
}

func (ca *CAImpl) newChain(intermediateKey crypto.Signer, intermediateSubject pkix.Name) *chain {
	root, err := ca.newRootIssuer()
	if err != nil {
		panic(fmt.Sprintf("Error creating new root issuer: %s", err.Error()))
	}
	intermediate, err := ca.newIntermediateIssuer(root, intermediateKey, intermediateSubject)
	if err != nil {
		panic(fmt.Sprintf("Error creating new intermediate issuer: %s", err.Error()))
	}
	return &chain{
		root:         root,
		intermediate: intermediate,
	}
}

func (ca *CAImpl) newCertificate(domains []string, ips []net.IP, key crypto.PublicKey, accountID string) (*core.Certificate, error) {
	var cn string
	if len(domains) > 0 {
		cn = domains[0]
	} else if len(ips) > 0 {
		cn = ips[0].String()
	} else {
		return nil, fmt.Errorf("must specify at least one domain name or IP address")
	}

	issuer := ca.chains[0].intermediate
	if issuer == nil || issuer.cert == nil {
		return nil, fmt.Errorf("cannot sign certificate - nil issuer")
	}

	serial := makeSerial()
	template := &x509.Certificate{
		DNSNames:    domains,
		IPAddresses: ips,
		Subject: pkix.Name{
			CommonName: cn,
		},
		SerialNumber: serial,
		NotBefore:    time.Now(),
		NotAfter:     time.Now().AddDate(5, 0, 0),

		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
		BasicConstraintsValid: true,
		IsCA:                  false,
	}

	if ca.ocspResponderURL != "" {
		template.OCSPServer = []string{ca.ocspResponderURL}
	}

	der, err := x509.CreateCertificate(rand.Reader, template, issuer.cert.Cert, key, issuer.key)
	if err != nil {
		return nil, err
	}
	cert, err := x509.ParseCertificate(der)
	if err != nil {
		return nil, err
	}

	issuers := make([]*core.Certificate, len(ca.chains))
	for i := 0; i < len(ca.chains); i++ {
		issuers[i] = ca.chains[i].intermediate.cert
	}

	hexSerial := hex.EncodeToString(cert.SerialNumber.Bytes())
	newCert := &core.Certificate{
		ID:        hexSerial,
		AccountID: accountID,
		Cert:      cert,
		DER:       der,
		Issuers:   issuers,
	}
	_, err = ca.db.AddCertificate(newCert)
	if err != nil {
		return nil, err
	}
	return newCert, nil
}

func New(log *log.Logger, db *db.MemoryStore, ocspResponderURL string, alternateRoots int) *CAImpl {
	ca := &CAImpl{
		log: log,
		db:  db,
	}

	if ocspResponderURL != "" {
		ca.ocspResponderURL = ocspResponderURL
		ca.log.Printf("Setting OCSP responder URL for issued certificates to %q", ca.ocspResponderURL)
	}

	intermediateSubject := pkix.Name{
		CommonName: intermediateCAPrefix + hex.EncodeToString(makeSerial().Bytes()[:3]),
	}
	intermediateKey, err := makeKey()
	if err != nil {
		panic(fmt.Sprintf("Error creating new intermediate private key: %s", err.Error()))
	}
	ca.chains = make([]*chain, 1+alternateRoots)
	for i := 0; i < len(ca.chains); i++ {
		ca.chains[i] = ca.newChain(intermediateKey, intermediateSubject)
	}
	return ca
}

func (ca *CAImpl) CompleteOrder(order *core.Order) {
	// Lock the order for reading
	order.RLock()
	// If the order isn't set as beganProcessing produce an error and immediately unlock
	if !order.BeganProcessing {
		ca.log.Printf("Error: Asked to complete order %s which had false beganProcessing.",
			order.ID)
		order.RUnlock()
		return
	}
	// Unlock the order again
	order.RUnlock()

	// Check the authorizations - this is done by the VA before calling
	// CompleteOrder but we do it again for robustness sake.
	for _, authz := range order.AuthorizationObjects {
		// Lock the authorization for reading
		authz.RLock()
		if authz.Status != acme.StatusValid {
			return
		}
		authz.RUnlock()
	}

	// issue a certificate for the csr
	csr := order.ParsedCSR
	cert, err := ca.newCertificate(csr.DNSNames, csr.IPAddresses, csr.PublicKey, order.AccountID)
	if err != nil {
		ca.log.Printf("Error: unable to issue order: %s", err.Error())
		return
	}
	ca.log.Printf("Issued certificate serial %s for order %s\n", cert.ID, order.ID)

	// Lock and update the order to store the issued certificate
	order.Lock()
	order.CertificateObject = cert
	order.Unlock()
}

func (ca *CAImpl) GetNumberOfRootCerts() int {
	return len(ca.chains)
}

func (ca *CAImpl) getChain(no int) *chain {
	if 0 <= no && no < len(ca.chains) {
		return ca.chains[no]
	}
	return nil
}

func (ca *CAImpl) GetRootCert(no int) *core.Certificate {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}
	return chain.root.cert
}

func (ca *CAImpl) GetRootKey(no int) *rsa.PrivateKey {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}

	switch key := chain.root.key.(type) {
	case *rsa.PrivateKey:
		return key
	}
	return nil
}

func (ca *CAImpl) GetIntermediateCert(no int) *core.Certificate {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}
	return chain.intermediate.cert
}

func (ca *CAImpl) GetIntermediateKey(no int) *rsa.PrivateKey {
	chain := ca.getChain(no)
	if chain == nil {
		return nil
	}

	switch key := chain.intermediate.key.(type) {
	case *rsa.PrivateKey:
		return key
	}
	return nil
}
<|endoftext|>


<commit_before>// -*- tab-width: 4; -*-

package main

import (
	"bufio"
	"bytes"
	"encoding/gob"
	"fmt"
	"log"
	"net/http"
	"os"
	"sync"
)

type Cached struct {
	Tweets       Tweets
	Lastmodified string
}

// key: url
type Cache map[string]Cached

func (cache Cache) Store(configpath string) {
	b := new(bytes.Buffer)
	enc := gob.NewEncoder(b)
	err := enc.Encode(cache)
	if err != nil {
		panic(err)
	}

	f, err := os.OpenFile(fmt.Sprintf("%s/cache", configpath),
		os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	if _, err = f.Write(b.Bytes()); err != nil {
		panic(err)
	}
}

func LoadCache(configpath string) Cache {
	cache := make(Cache)

	f, err := os.Open(fmt.Sprintf("%s/cache", configpath))
	if err != nil {
		if !os.IsNotExist(err) {
			panic(err)
		}
		return cache
	}
	defer f.Close()

	dec := gob.NewDecoder(f)
	err = dec.Decode(&cache)
	if err != nil {
		panic(err)
	}
	return cache
}

func (cache Cache) FetchTweets(sources map[string]string) {
	var mu sync.RWMutex

	tweetsch := make(chan Tweets)
	var wg sync.WaitGroup
	// max parallel http fetchers
	var fetchers = make(chan struct{}, maxfetchers)

	for nick, url := range sources {
		wg.Add(1)

		fetchers <- struct{}{}

		// anon func takes needed variables as arg, avoiding capture of iterator variables
		go func(nick string, url string) {
			defer wg.Done()

			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				if debug {
					log.Printf("%s: http.NewRequest fail: %s", url, err)
				}
				tweetsch <- nil
				return
			}

			if conf.Nick != "" && conf.Twturl != "" {
				// TODO: version goes here
				req.Header.Set("User-Agent",
					fmt.Sprintf("%s/%s (+%s; @%s)", progname, GetVersion(),
						conf.Twturl, conf.Nick))
			}

			mu.RLock()
			if cached, ok := cache[url]; ok {
				if cached.Lastmodified != "" {
					req.Header.Set("If-Modified-Since", cached.Lastmodified)
				}
			}
			mu.RUnlock()

			client := http.Client{}
			resp, err := client.Do(req)
			if err != nil {
				if debug {
					log.Printf("%s: client.Do fail: %s", url, err)
				}
				tweetsch <- nil
				return
			}
			defer resp.Body.Close()

			actualurl := resp.Request.URL.String()
			if actualurl != url {
				url = actualurl
			}

			var tweets Tweets

			switch resp.StatusCode {
			case http.StatusOK: // 200
				scanner := bufio.NewScanner(resp.Body)
				tweets = ParseFile(scanner, Tweeter{Nick: nick, URL: url})
				lastmodified := resp.Header.Get("Last-Modified")
				mu.Lock()
				cache[url] = Cached{Tweets: tweets, Lastmodified: lastmodified}
				mu.Unlock()
			case http.StatusNotModified: // 304
				mu.RLock()
				tweets = cache[url].Tweets
				mu.RUnlock()
			}

			tweetsch <- tweets

		}(nick, url)

		<-fetchers
	}

	// close tweets channel when all goroutines are done
	go func() {
		wg.Wait()
		close(tweetsch)
	}()

	if debug {
		log.Print("fetching:\n")
	}

	var n = 0
	// loop until channel closed
	for tweets := range tweetsch {
		n++
		if debug {
			log.Printf("%d ", len(sources)+1-n)
		}
		if debug && len(tweets) > 0 {
			log.Printf("%s\n", tweets[0].Tweeter.URL)
		}
	}
	if debug {
		log.Print("\n")
	}
}

func (cache Cache) GetAll() Tweets {
	var alltweets Tweets
	for url, cached := range cache {
		alltweets = append(alltweets, cached.Tweets...)
		if debug {
			log.Printf("%s\n", url)
		}
	}
	return alltweets
}

func (cache Cache) GetByURL(url string) Tweets {
	if cached, ok := cache[url]; ok {
		return cached.Tweets
	}
	return Tweets{}
}
<commit_msg>added check for file:// when loading twtxt.txt files<commit_after>// -*- tab-width: 4; -*-

package main

import (
	"bufio"
	"bytes"
	"encoding/gob"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"sync"
)

type Cached struct {
	Tweets       Tweets
	Lastmodified string
}

// key: url
type Cache map[string]Cached

func (cache Cache) Store(configpath string) {
	b := new(bytes.Buffer)
	enc := gob.NewEncoder(b)
	err := enc.Encode(cache)
	if err != nil {
		panic(err)
	}

	f, err := os.OpenFile(fmt.Sprintf("%s/cache", configpath),
		os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	if _, err = f.Write(b.Bytes()); err != nil {
		panic(err)
	}
}

func LoadCache(configpath string) Cache {
	cache := make(Cache)

	f, err := os.Open(fmt.Sprintf("%s/cache", configpath))
	if err != nil {
		if !os.IsNotExist(err) {
			panic(err)
		}
		return cache
	}
	defer f.Close()

	dec := gob.NewDecoder(f)
	err = dec.Decode(&cache)
	if err != nil {
		panic(err)
	}
	return cache
}

func (cache Cache) FetchTweets(sources map[string]string) {
	var mu sync.RWMutex

	tweetsch := make(chan Tweets)
	var wg sync.WaitGroup
	// max parallel http fetchers
	var fetchers = make(chan struct{}, maxfetchers)

	for nick, url := range sources {
		wg.Add(1)

		fetchers <- struct{}{}

		// anon func takes needed variables as arg, avoiding capture of iterator variables
		go func(nick string, url string) {
			defer wg.Done()

			if strings.HasPrefix(url, "file://") {
				err := ReadLocalFile(url, nick, &tweetsch, &cache, &mu)
				if err != nil {
					if debug {
						log.Printf("%s: Failed to read and cache local file: %s", url, err)
					}
				}
				return
			}

			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				if debug {
					log.Printf("%s: http.NewRequest fail: %s", url, err)
				}
				tweetsch <- nil
				return
			}

			if conf.Nick != "" && conf.Twturl != "" {
				// TODO: version goes here
				req.Header.Set("User-Agent",
					fmt.Sprintf("%s/%s (+%s; @%s)", progname, GetVersion(),
						conf.Twturl, conf.Nick))
			}

			mu.RLock()
			if cached, ok := cache[url]; ok {
				if cached.Lastmodified != "" {
					req.Header.Set("If-Modified-Since", cached.Lastmodified)
				}
			}
			mu.RUnlock()

			client := http.Client{}
			resp, err := client.Do(req)
			if err != nil {
				if debug {
					log.Printf("%s: client.Do fail: %s", url, err)
				}
				tweetsch <- nil
				return
			}
			defer resp.Body.Close()

			actualurl := resp.Request.URL.String()
			if actualurl != url {
				url = actualurl
			}

			var tweets Tweets

			switch resp.StatusCode {
			case http.StatusOK: // 200
				scanner := bufio.NewScanner(resp.Body)
				tweets = ParseFile(scanner, Tweeter{Nick: nick, URL: url})
				lastmodified := resp.Header.Get("Last-Modified")
				mu.Lock()
				cache[url] = Cached{Tweets: tweets, Lastmodified: lastmodified}
				mu.Unlock()
			case http.StatusNotModified: // 304
				mu.RLock()
				tweets = cache[url].Tweets
				mu.RUnlock()
			}

			tweetsch <- tweets

		}(nick, url)

		<-fetchers
	}

	// close tweets channel when all goroutines are done
	go func() {
		wg.Wait()
		close(tweetsch)
	}()

	if debug {
		log.Print("fetching:\n")
	}

	var n = 0
	// loop until channel closed
	for tweets := range tweetsch {
		n++
		if debug {
			log.Printf("%d ", len(sources)+1-n)
		}
		if debug && len(tweets) > 0 {
			log.Printf("%s\n", tweets[0].Tweeter.URL)
		}
	}
	if debug {
		log.Print("\n")
	}
}

func ReadLocalFile(url string, nick string, tweetsch *chan Tweets, cache *Cache, mu *sync.RWMutex) error {
	path := string(url[6:])
	file, err := os.Stat(path)
	if err != nil {
		if debug {
			log.Printf("%s: Can't stat local file: %s", path, err)
		}
		return err
	}
	if cached, ok := (*cache)[url]; ok {
		if cached.Lastmodified == file.ModTime().String() {
			tweets := (*cache)[url].Tweets
			*tweetsch <- tweets
			return nil
		}
	}
	data, err := ioutil.ReadFile(path)
	if err != nil {
		if debug {
			log.Printf("%s: Can't read local file: %s", path, err)
		}
		*tweetsch <- nil
		return err
	}
	scanner := bufio.NewScanner(bytes.NewReader(data))
	tweets := ParseFile(scanner, Tweeter{Nick: nick, URL: url})
	lastmodified := file.ModTime().String()
	mu.Lock()
	(*cache)[url] = Cached{Tweets: tweets, Lastmodified: lastmodified}
	mu.Unlock()
	*tweetsch <- tweets
	return nil
}

func (cache Cache) GetAll() Tweets {
	var alltweets Tweets
	for url, cached := range cache {
		alltweets = append(alltweets, cached.Tweets...)
		if debug {
			log.Printf("%s\n", url)
		}
	}
	return alltweets
}

func (cache Cache) GetByURL(url string) Tweets {
	if cached, ok := cache[url]; ok {
		return cached.Tweets
	}
	return Tweets{}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	log "github.com/sirupsen/logrus"
	"github.com/yuin/gopher-lua"
	"io/ioutil"
	"strings"
	"sync"
)

// fileID identifies a filename
type fileID string

const emptyFileID = fileID("")

// FileCache manages a set of bytes as a cache
type FileCache struct {
	size              uint64            // Total size of the cache
	blob              []byte            // The cache storage
	index             map[fileID]uint64 // Overview of where the data is placed in the cache
	hits              map[fileID]uint64 // Keeping track of data popularity
	offset            uint64            // The current position in the cache storage (end of data)
	rw                *sync.RWMutex     // Used for avoiding data races and other issues
	cacheWarningGiven bool              // Used to only warn once if the cache is full
	compress          bool              // Enable data compression
	maxEntitySize     uint64            // Maximum size per entity in cache
}

var (
	// ErrRemoval is used if a filename that does not exist is attempted to be removed
	ErrRemoval = errors.New("Can't remove a file ID that does not exist")

	// ErrNoData is used if no data is attempted to be stored in the cache
	ErrNoData = errors.New("No data")

	// ErrAlreadyStored is used if a given filename has already been stored in the cache
	ErrAlreadyStored = errors.New("That file ID is already stored")

	// ErrLargerThanCache is used if the given data is larger than the total cache size
	ErrLargerThanCache = errors.New("Data is larger than the the total cache size")

	// ErrEntityTooLarge is used if a maximum size per entity has been set
	ErrEntityTooLarge = errors.New("Data is larger than the allowed size.")
)

// Creates a new FileCache struct.
// cacheSize is the total cache size, in bytes.
// compress is for enabling compression of cache data.
// maxEntitySize is for setting a per-file maximum size.
func newFileCache(cacheSize uint64, compress bool, maxEntitySize uint64) *FileCache {
	var cache FileCache
	cache.size = cacheSize
	cache.blob = make([]byte, cacheSize) // The cache storage
	cache.index = make(map[fileID]uint64)
	cache.hits = make(map[fileID]uint64)
	cache.rw = &sync.RWMutex{}
	cache.compress = compress
	cache.maxEntitySize = maxEntitySize
	return &cache
}

// Normalize the filename
func (cache *FileCache) normalize(filename string) fileID {
	// If the filename begins with "./", remove it
	if strings.HasPrefix(filename, "./") {
		return fileID(filename[2:])
	}
	return fileID(filename)
}

// Remove bytes from the cache blob
// i is the position, n is the number of bytes to remove
func (cache *FileCache) removeBytes(i, n uint64) {
	cache.blob = append(cache.blob[:i], cache.blob[i+n:]...)
	// Extend to the original capacity after removing bytes
	cache.blob = cache.blob[:cap(cache.blob)]
}

// IsEmpty checks if the cache is empty
func (cache *FileCache) IsEmpty() bool {
	return 0 == len(cache.index)
}

// Shuffle all indices that refers to position after removedpos, offset to the left
// Also moves the end of the data offset to the left
func (cache *FileCache) shuffleIndicesLeft(removedpos, offset uint64) {
	for id, pos := range cache.index {
		if pos > removedpos {
			cache.index[id] -= offset
		}
	}
}

// Remove a data index
func (cache *FileCache) removeIndex(id fileID) {
	delete(cache.index, id)
}

// Remove data from the cache and shuffle the rest of the data to the left
// Also adjusts all index pointers to indexes larger than the current position
// Also adjusts the cache.offset
func (cache *FileCache) remove(id fileID) error {
	if !cache.hasFile(id) {
		return ErrRemoval
	}

	// Find the position and size of the given id
	pos := cache.index[id]
	size := cache.dataSize(id)

	cache.removeIndex(id)
	cache.shuffleIndicesLeft(pos, size)
	cache.removeBytes(pos, size)
	cache.offset -= uint64(size)

	return nil
}

// Find the data with the least hits, that is currently in the cache
func (cache *FileCache) leastPopular() (fileID, error) {
	// If there is no data, return an error
	if len(cache.index) == 0 {
		return emptyFileID, ErrNoData
	}

	// Loop through all the data and return the first with no cache hits
	for id := range cache.index {
		// If there are no cache hits, return this id
		foundHit := false
		for hitID := range cache.hits {
			if hitID == id {
				foundHit = true
				break
			}
		}
		if !foundHit {
			return id, nil
		}
	}

	var (
		leastHits   uint64
		leastHitsID fileID
		firstRound  = true
	)

	// Loop through all the data and find the one with the least cache hits
	for id := range cache.index {
		// If there is a cache hit, check if it is the first round
		if firstRound {
			// First candidate
			leastHits = cache.hits[id]
			leastHitsID = id
			firstRound = false
			continue
		}
		// Not the first round, compare with the least popular ID so far
		if cache.hits[id] < leastHits {
			// Found a less popular ID
			leastHits = cache.hits[id]
			leastHitsID = id
		}
	}

	// Return the one with the fewest cache hits.
	return leastHitsID, nil
}

// Store a file in the cache
// Returns the data (it may be compressed) and an error
func (cache *FileCache) storeData(filename string, data []byte) (storedData []byte, err error) {
	// Compress the data, if compression is enabled
	if cache.compress {
		compressed, err := compress(data)
		if err != nil {
			return nil, fmt.Errorf("Compression error: %s", err)
		}
		data = compressed
	}

	fileSize := uint64(len(data))
	id := cache.normalize(filename)

	if cache.hasFile(id) {
		return nil, ErrAlreadyStored
	}

	if fileSize > cache.size {
		return nil, ErrLargerThanCache
	}

	if cache.maxEntitySize != 0 && fileSize > cache.maxEntitySize {
		return nil, ErrEntityTooLarge
	}

	// Warn once that the cache is now full
	if !cache.cacheWarningGiven && fileSize > cache.freeSpace() {
		log.Warn("Cache is full. You may want to increase the cache size.")
		cache.cacheWarningGiven = true
	}

	// While there is not enough space, remove the least popular data
	for fileSize > cache.freeSpace() {

		// Find the least popular data
		removeID, err := cache.leastPopular()
		if err != nil {
			return nil, err
		}

		// Remove it
		if verboseMode {
			log.Info(fmt.Sprintf("Removing the unpopular %v from the cache (%d bytes)", removeID, cache.dataSize(removeID)))
		}

		spaceBefore := cache.freeSpace()
		err = cache.remove(removeID)
		if err != nil {
			return nil, err
		}
		spaceAfter := cache.freeSpace()

		// Panic if there is no more free cache space after removing data
		if spaceBefore == spaceAfter {
			panic(fmt.Sprintf("Removed %v, but the free space is the same! Still %d bytes.", removeID, spaceAfter))
		}
	}

	if verboseMode {
		log.Info(fmt.Sprintf("Storing in cache: %v", id))
	}

	// Register the position in the data index
	cache.index[id] = cache.offset

	// Copy the contents to the cache
	var i uint64
	for i = 0; i < fileSize; i++ {
		cache.blob[cache.offset+i] = data[i]
	}

	// Move the offset to the end of the data (the next free location)
	cache.offset += uint64(fileSize)

	return data, nil
}

// Check if the given filename exists in the cache
func (cache *FileCache) hasFile(id fileID) bool {
	for key := range cache.index {
		if key == id {
			return true
		}
	}
	return false
}

// Find the next start of a data block, given a position
// Returns a position and true if a next position was found
func (cache *FileCache) nextData(startpos uint64) (uint64, bool) {
	// Find the end of the data (larger than startpos, but smallest diff)
	datasize := cache.size - startpos // Largest possible data size
	endpos := startpos                // Initial end of data
	found := false
	for _, pos := range cache.index {
		if pos > startpos && (pos-startpos) < datasize {
			datasize = (pos - startpos)
			endpos = pos
			found = true
		}
	}
	// endpos is the next start of a data block
	return endpos, found
}

// Find the size of a cached data block. id must exist.
func (cache *FileCache) dataSize(id fileID) uint64 {
	startpos := cache.index[id]

	// Find the next data block
	endpos, foundNext := cache.nextData(startpos)

	// Use the end of data as the end position, if no next position is found
	if !foundNext {
		endpos = cache.offset
	}

	return endpos - startpos
}

// Store a file in the cache
// Return true if we got the data from the file, regardless of cache errors.
func (cache *FileCache) storeFile(filename string) ([]byte, bool, error) {
	// Read the file
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, false, err
	}
	// Store in cache, log a warning if the cache has filled up and needs to make space every time
	_, err = cache.storeData(filename, data)
	return data, true, err
}

// Retrieve a file from the cache, or from disk
func (cache *FileCache) fetchAndCache(filename string) ([]byte, error) {
	// RWMutex locks
	cache.rw.Lock()
	defer cache.rw.Unlock()

	id := cache.normalize(filename)

	// Check if the file needs to be read from disk
	fileCached := cache.hasFile(id)

	if !fileCached {
		if verboseMode {
			log.Info("Reading from disk: " + string(id))
			log.Info("Storing in cache: " + string(id))
		}
		data, gotTheData, err := cache.storeFile(string(id))
		// Cache errors are logged as warnings, and not being returned
		if err != nil {
			// Log cache errors as warnings (could be that the file is too large)
			if verboseMode {
				log.Warn(err)
			}
		}
		// Only return an error here if we were not able to read the file from disk
		if !gotTheData {
			return nil, err
		}
		// Return the data, with no errors to report
		return data, nil
	}

	if verboseMode {
		log.Info("Retrieving from cache: " + string(id))
	}

	// Find the start of the data
	startpos := cache.index[id]

	// Find the size of the data
	size := cache.dataSize(id)

	// Copy the data from the cache
	data := make([]byte, size)
	var i uint64
	for i = 0; i < size; i++ {
		data[i] = cache.blob[startpos+i]
	}

	// Mark a cache hit
	cache.hits[id]++

	// Decompress
	if cache.compress {
		decompressed, err := decompress(data)
		if err != nil {
			return nil, err
		}
		return decompressed, nil
	}

	// Return the data
	return data, nil
}

func (cache *FileCache) freeSpace() uint64 {
	return cache.size - cache.offset
}

// Return formatted cache statstics
func (cache *FileCache) stats() string {
	cache.rw.Lock()
	defer cache.rw.Unlock()

	var buf bytes.Buffer
	buf.WriteString("Cache information:\n")
	buf.WriteString(fmt.Sprintf("\tCompression:\t%s\n", enabledStatus(cache.compress)))
	buf.WriteString(fmt.Sprintf("\tTotal cache:\t%d bytes\n", cache.size))
	buf.WriteString(fmt.Sprintf("\tFree cache:\t%d bytes\n", cache.freeSpace()))
	buf.WriteString(fmt.Sprintf("\tEnd of data:\tat %d\n", cache.offset))
	if len(cache.index) > 0 {
		buf.WriteString("\tData in cache:\n")
		for id, pos := range cache.index {
			buf.WriteString(fmt.Sprintf("\t\tid=%v\tpos=%d\tsize=%d\n", id, pos, cache.dataSize(id)))
		}
	}
	var totalHits uint64
	if len(cache.hits) > 0 {
		buf.WriteString("\tCache hits:\n")
		for id, hits := range cache.hits {
			buf.WriteString(fmt.Sprintf("\t\tid=%v\thits=%d\n", id, hits))
			totalHits += hits
		}
		buf.WriteString(fmt.Sprintf("\tTotal cache hits:\t%d", totalHits))
	}
	return buf.String()
}

// Clear the entire cache
func (cache *FileCache) clear() {
	cache.rw.Lock()
	defer cache.rw.Unlock()

	cache = newFileCache(cache.size, cache.compress, cache.maxEntitySize)

	// Allow one warning if the cache should fill up
	cache.cacheWarningGiven = false
}

// Export functions related to the cache. cache can be nil.
func exportCacheFunctions(L *lua.LState, cache *FileCache) {

	const disabledMessage = "Caching is disabled"
	const clearedMessage = "Cache cleared"

	luaCacheStatsFunc := L.NewFunction(func(L *lua.LState) int {
		if cache == nil {
			L.Push(lua.LString(disabledMessage))
			return 1 // number of results
		}
		info := cache.stats()
		// Return the string, but drop the final newline
		L.Push(lua.LString(info[:len(info)-1]))
		return 1 // number of results
	})

	// Return information about the cache use
	L.SetGlobal("CacheInfo", luaCacheStatsFunc)
	L.SetGlobal("CacheStats", luaCacheStatsFunc) // undocumented alias

	// Clear the cache
	L.SetGlobal("ClearCache", L.NewFunction(func(L *lua.LState) int {
		if cache == nil {
			L.Push(lua.LString(disabledMessage))
			return 1 // number of results
		}
		cache.clear()
		L.Push(lua.LString(clearedMessage))
		return 1 // number of results
	}))

}

// For reading files, with optional caching
func (cache *FileCache) read(filename string, cacheEnabled bool) ([]byte, error) {
	if cacheEnabled {
		// Read the file from cache (or disk, if not cached)
		return cache.fetchAndCache(filename)
	}
	// Normalize the filename
	filename = string(cache.normalize(filename))
	if verboseMode {
		log.Info("Reading from disk: " + filename)
	}
	// Read the file
	return ioutil.ReadFile(filename)
}
<commit_msg>Fixed ClearCache<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	log "github.com/sirupsen/logrus"
	"github.com/yuin/gopher-lua"
	"io/ioutil"
	"strings"
	"sync"
)

// fileID identifies a filename
type fileID string

const emptyFileID = fileID("")

// FileCache manages a set of bytes as a cache
type FileCache struct {
	size              uint64            // Total size of the cache
	blob              []byte            // The cache storage
	index             map[fileID]uint64 // Overview of where the data is placed in the cache
	hits              map[fileID]uint64 // Keeping track of data popularity
	offset            uint64            // The current position in the cache storage (end of data)
	rw                *sync.RWMutex     // Used for avoiding data races and other issues
	cacheWarningGiven bool              // Used to only warn once if the cache is full
	compress          bool              // Enable data compression
	maxEntitySize     uint64            // Maximum size per entity in cache
}

var (
	// ErrRemoval is used if a filename that does not exist is attempted to be removed
	ErrRemoval = errors.New("Can't remove a file ID that does not exist")

	// ErrNoData is used if no data is attempted to be stored in the cache
	ErrNoData = errors.New("No data")

	// ErrAlreadyStored is used if a given filename has already been stored in the cache
	ErrAlreadyStored = errors.New("That file ID is already stored")

	// ErrLargerThanCache is used if the given data is larger than the total cache size
	ErrLargerThanCache = errors.New("Data is larger than the the total cache size")

	// ErrEntityTooLarge is used if a maximum size per entity has been set
	ErrEntityTooLarge = errors.New("Data is larger than the allowed size.")
)

// Creates a new FileCache struct.
// cacheSize is the total cache size, in bytes.
// compress is for enabling compression of cache data.
// maxEntitySize is for setting a per-file maximum size.
func newFileCache(cacheSize uint64, compress bool, maxEntitySize uint64) *FileCache {
	var cache FileCache
	cache.size = cacheSize
	cache.blob = make([]byte, cacheSize) // The cache storage
	cache.index = make(map[fileID]uint64)
	cache.hits = make(map[fileID]uint64)
	cache.rw = &sync.RWMutex{}
	cache.compress = compress
	cache.maxEntitySize = maxEntitySize
	return &cache
}

// Normalize the filename
func (cache *FileCache) normalize(filename string) fileID {
	// If the filename begins with "./", remove it
	if strings.HasPrefix(filename, "./") {
		return fileID(filename[2:])
	}
	return fileID(filename)
}

// Remove bytes from the cache blob
// i is the position, n is the number of bytes to remove
func (cache *FileCache) removeBytes(i, n uint64) {
	cache.blob = append(cache.blob[:i], cache.blob[i+n:]...)
	// Extend to the original capacity after removing bytes
	cache.blob = cache.blob[:cap(cache.blob)]
}

// IsEmpty checks if the cache is empty
func (cache *FileCache) IsEmpty() bool {
	return 0 == len(cache.index)
}

// Shuffle all indices that refers to position after removedpos, offset to the left
// Also moves the end of the data offset to the left
func (cache *FileCache) shuffleIndicesLeft(removedpos, offset uint64) {
	for id, pos := range cache.index {
		if pos > removedpos {
			cache.index[id] -= offset
		}
	}
}

// Remove a data index
func (cache *FileCache) removeIndex(id fileID) {
	delete(cache.index, id)
}

// Remove data from the cache and shuffle the rest of the data to the left
// Also adjusts all index pointers to indexes larger than the current position
// Also adjusts the cache.offset
func (cache *FileCache) remove(id fileID) error {
	if !cache.hasFile(id) {
		return ErrRemoval
	}

	// Find the position and size of the given id
	pos := cache.index[id]
	size := cache.dataSize(id)

	cache.removeIndex(id)
	cache.shuffleIndicesLeft(pos, size)
	cache.removeBytes(pos, size)
	cache.offset -= uint64(size)

	return nil
}

// Find the data with the least hits, that is currently in the cache
func (cache *FileCache) leastPopular() (fileID, error) {
	// If there is no data, return an error
	if len(cache.index) == 0 {
		return emptyFileID, ErrNoData
	}

	// Loop through all the data and return the first with no cache hits
	for id := range cache.index {
		// If there are no cache hits, return this id
		foundHit := false
		for hitID := range cache.hits {
			if hitID == id {
				foundHit = true
				break
			}
		}
		if !foundHit {
			return id, nil
		}
	}

	var (
		leastHits   uint64
		leastHitsID fileID
		firstRound  = true
	)

	// Loop through all the data and find the one with the least cache hits
	for id := range cache.index {
		// If there is a cache hit, check if it is the first round
		if firstRound {
			// First candidate
			leastHits = cache.hits[id]
			leastHitsID = id
			firstRound = false
			continue
		}
		// Not the first round, compare with the least popular ID so far
		if cache.hits[id] < leastHits {
			// Found a less popular ID
			leastHits = cache.hits[id]
			leastHitsID = id
		}
	}

	// Return the one with the fewest cache hits.
	return leastHitsID, nil
}

// Store a file in the cache
// Returns the data (it may be compressed) and an error
func (cache *FileCache) storeData(filename string, data []byte) (storedData []byte, err error) {
	// Compress the data, if compression is enabled
	if cache.compress {
		compressed, err := compress(data)
		if err != nil {
			return nil, fmt.Errorf("Compression error: %s", err)
		}
		data = compressed
	}

	fileSize := uint64(len(data))
	id := cache.normalize(filename)

	if cache.hasFile(id) {
		return nil, ErrAlreadyStored
	}

	if fileSize > cache.size {
		return nil, ErrLargerThanCache
	}

	if cache.maxEntitySize != 0 && fileSize > cache.maxEntitySize {
		return nil, ErrEntityTooLarge
	}

	// Warn once that the cache is now full
	if !cache.cacheWarningGiven && fileSize > cache.freeSpace() {
		log.Warn("Cache is full. You may want to increase the cache size.")
		cache.cacheWarningGiven = true
	}

	// While there is not enough space, remove the least popular data
	for fileSize > cache.freeSpace() {

		// Find the least popular data
		removeID, err := cache.leastPopular()
		if err != nil {
			return nil, err
		}

		// Remove it
		if verboseMode {
			log.Info(fmt.Sprintf("Removing the unpopular %v from the cache (%d bytes)", removeID, cache.dataSize(removeID)))
		}

		spaceBefore := cache.freeSpace()
		err = cache.remove(removeID)
		if err != nil {
			return nil, err
		}
		spaceAfter := cache.freeSpace()

		// Panic if there is no more free cache space after removing data
		if spaceBefore == spaceAfter {
			panic(fmt.Sprintf("Removed %v, but the free space is the same! Still %d bytes.", removeID, spaceAfter))
		}
	}

	if verboseMode {
		log.Info(fmt.Sprintf("Storing in cache: %v", id))
	}

	// Register the position in the data index
	cache.index[id] = cache.offset

	// Copy the contents to the cache
	var i uint64
	for i = 0; i < fileSize; i++ {
		cache.blob[cache.offset+i] = data[i]
	}

	// Move the offset to the end of the data (the next free location)
	cache.offset += uint64(fileSize)

	return data, nil
}

// Check if the given filename exists in the cache
func (cache *FileCache) hasFile(id fileID) bool {
	for key := range cache.index {
		if key == id {
			return true
		}
	}
	return false
}

// Find the next start of a data block, given a position
// Returns a position and true if a next position was found
func (cache *FileCache) nextData(startpos uint64) (uint64, bool) {
	// Find the end of the data (larger than startpos, but smallest diff)
	datasize := cache.size - startpos // Largest possible data size
	endpos := startpos                // Initial end of data
	found := false
	for _, pos := range cache.index {
		if pos > startpos && (pos-startpos) < datasize {
			datasize = (pos - startpos)
			endpos = pos
			found = true
		}
	}
	// endpos is the next start of a data block
	return endpos, found
}

// Find the size of a cached data block. id must exist.
func (cache *FileCache) dataSize(id fileID) uint64 {
	startpos := cache.index[id]

	// Find the next data block
	endpos, foundNext := cache.nextData(startpos)

	// Use the end of data as the end position, if no next position is found
	if !foundNext {
		endpos = cache.offset
	}

	return endpos - startpos
}

// Store a file in the cache
// Return true if we got the data from the file, regardless of cache errors.
func (cache *FileCache) storeFile(filename string) ([]byte, bool, error) {
	// Read the file
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, false, err
	}
	// Store in cache, log a warning if the cache has filled up and needs to make space every time
	_, err = cache.storeData(filename, data)
	return data, true, err
}

// Retrieve a file from the cache, or from disk
func (cache *FileCache) fetchAndCache(filename string) ([]byte, error) {
	// RWMutex locks
	cache.rw.Lock()
	defer cache.rw.Unlock()

	id := cache.normalize(filename)

	// Check if the file needs to be read from disk
	fileCached := cache.hasFile(id)

	if !fileCached {
		if verboseMode {
			log.Info("Reading from disk: " + string(id))
			log.Info("Storing in cache: " + string(id))
		}
		data, gotTheData, err := cache.storeFile(string(id))
		// Cache errors are logged as warnings, and not being returned
		if err != nil {
			// Log cache errors as warnings (could be that the file is too large)
			if verboseMode {
				log.Warn(err)
			}
		}
		// Only return an error here if we were not able to read the file from disk
		if !gotTheData {
			return nil, err
		}
		// Return the data, with no errors to report
		return data, nil
	}

	if verboseMode {
		log.Info("Retrieving from cache: " + string(id))
	}

	// Find the start of the data
	startpos := cache.index[id]

	// Find the size of the data
	size := cache.dataSize(id)

	// Copy the data from the cache
	data := make([]byte, size)
	var i uint64
	for i = 0; i < size; i++ {
		data[i] = cache.blob[startpos+i]
	}

	// Mark a cache hit
	cache.hits[id]++

	// Decompress
	if cache.compress {
		decompressed, err := decompress(data)
		if err != nil {
			return nil, err
		}
		return decompressed, nil
	}

	// Return the data
	return data, nil
}

func (cache *FileCache) freeSpace() uint64 {
	return cache.size - cache.offset
}

// Return formatted cache statstics
func (cache *FileCache) stats() string {
	cache.rw.Lock()
	defer cache.rw.Unlock()

	var buf bytes.Buffer
	buf.WriteString("Cache information:\n")
	buf.WriteString(fmt.Sprintf("\tCompression:\t%s\n", enabledStatus(cache.compress)))
	buf.WriteString(fmt.Sprintf("\tTotal cache:\t%d bytes\n", cache.size))
	buf.WriteString(fmt.Sprintf("\tFree cache:\t%d bytes\n", cache.freeSpace()))
	buf.WriteString(fmt.Sprintf("\tEnd of data:\tat %d\n", cache.offset))
	if len(cache.index) > 0 {
		buf.WriteString("\tData in cache:\n")
		for id, pos := range cache.index {
			buf.WriteString(fmt.Sprintf("\t\tid=%v\tpos=%d\tsize=%d\n", id, pos, cache.dataSize(id)))
		}
	}
	var totalHits uint64
	if len(cache.hits) > 0 {
		buf.WriteString("\tCache hits:\n")
		for id, hits := range cache.hits {
			buf.WriteString(fmt.Sprintf("\t\tid=%v\thits=%d\n", id, hits))
			totalHits += hits
		}
		buf.WriteString(fmt.Sprintf("\tTotal cache hits:\t%d", totalHits))
	}
	return buf.String()
}

// Clear the entire cache
func (cache *FileCache) clear() {
	cache.rw.Lock()
	defer cache.rw.Unlock()

	cache.offset = 0
	cache.index = make(map[fileID]uint64)
	cache.hits = make(map[fileID]uint64)

	// No need to clear the actual bytes, unless perhaps if there should be
	// changes to the caching algorithm in the future.
	//cache.blob = make([]byte, cache.size)

	// Allow one warning if the cache should fill up
	cache.cacheWarningGiven = false
}

// Export functions related to the cache. cache can be nil.
func exportCacheFunctions(L *lua.LState, cache *FileCache) {

	const disabledMessage = "Caching is disabled"
	const clearedMessage = "Cache cleared"

	luaCacheStatsFunc := L.NewFunction(func(L *lua.LState) int {
		if cache == nil {
			L.Push(lua.LString(disabledMessage))
			return 1 // number of results
		}
		info := cache.stats()
		// Return the string, but drop the final newline
		L.Push(lua.LString(info[:len(info)-1]))
		return 1 // number of results
	})

	// Return information about the cache use
	L.SetGlobal("CacheInfo", luaCacheStatsFunc)
	L.SetGlobal("CacheStats", luaCacheStatsFunc) // undocumented alias

	// Clear the cache
	L.SetGlobal("ClearCache", L.NewFunction(func(L *lua.LState) int {
		if cache == nil {
			L.Push(lua.LString(disabledMessage))
			return 1 // number of results
		}
		cache.clear()
		L.Push(lua.LString(clearedMessage))
		return 1 // number of results
	}))

}

// For reading files, with optional caching
func (cache *FileCache) read(filename string, cacheEnabled bool) ([]byte, error) {
	if cacheEnabled {
		// Read the file from cache (or disk, if not cached)
		return cache.fetchAndCache(filename)
	}
	// Normalize the filename
	filename = string(cache.normalize(filename))
	if verboseMode {
		log.Info("Reading from disk: " + filename)
	}
	// Read the file
	return ioutil.ReadFile(filename)
}
<|endoftext|>


<commit_before>package anidb

import (
	"bytes"
	"compress/gzip"
	"encoding/gob"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"reflect"
	"regexp"
	"sync"
	"time"
)

var _ log.Logger

type Cacheable interface {
	// Updates the last modified time
	Touch()
	// Returns true if the Cacheable is nil, or if the last modified time is too old.
	IsStale() bool
}

func init() {
	gob.RegisterName("*github.com/Kovensky/go-anidb.invalidKeyCache", &invalidKeyCache{})
}

type invalidKeyCache struct{ time.Time }

func (c *invalidKeyCache) Touch() {
	c.Time = time.Now()
}
func (c *invalidKeyCache) IsStale() bool {
	return time.Now().Sub(c.Time) > InvalidKeyCacheDuration
}

type cacheDir struct {
	*sync.RWMutex

	CacheDir string
}

func init() {
	if err := SetCacheDir(path.Join(os.TempDir(), "anidb", "cache")); err != nil {
		panic(err)
	}
}

var cache cacheDir

// Sets the cache directory to the given path.
//
// go-anidb needs a valid cache directory to function, so, during module
// initialization, it uses os.TempDir() to set a default cache dir.
// go-anidb panics if it's unable to set the default cache dir.
func SetCacheDir(path string) (err error) {
	m := cache.RWMutex
	if m == nil {
		m = &sync.RWMutex{}
		cache.RWMutex = m
	}
	cache.Lock()

	if err = os.MkdirAll(path, 0755|os.ModeDir); err != nil {
		cache.Unlock()
		return err
	}

	cache = cacheDir{
		RWMutex:  m,
		CacheDir: path,
	}

	cache.Unlock()
	RefreshTitles()
	return nil
}

// Returns the current cache dir.
func GetCacheDir() (path string) {
	cache.RLock()
	defer cache.RUnlock()

	return cache.CacheDir
}

type cacheKey interface{}

// All "bad characters" that can't go in Windows paths.
// It's a superset of the "bad characters" on other OSes, so this works.
var badPath = regexp.MustCompile(`[\\/:\*\?\"<>\|]`)

func stringify(stuff ...cacheKey) []string {
	ret := make([]string, len(stuff))
	for i := range stuff {
		s := fmt.Sprint(stuff[i])
		ret[i] = badPath.ReplaceAllLiteralString(s, "_")
	}
	return ret
}

// Each key but the last is treated as a directory.
// The last key is treated as a regular file.
//
// This also means that cache keys that are file-backed
// cannot have subkeys.
func cachePath(keys ...cacheKey) string {
	parts := append([]string{GetCacheDir()}, stringify(keys...)...)
	p := path.Join(parts...)
	return p
}

// Opens the file that backs the specified keys.
func (c *cacheDir) Open(keys ...cacheKey) (fh *os.File, err error) {
	subItem := cachePath(keys...)
	return os.Open(subItem)
}

// Creates a new file to back the specified keys.
func (c *cacheDir) Create(keys ...cacheKey) (fh *os.File, err error) {
	subItem := cachePath(keys...)
	subDir := path.Dir(subItem)

	if err = os.MkdirAll(subDir, 0755|os.ModeDir); err != nil {
		return nil, err
	}
	return os.Create(subItem)
}

// Deletes the file that backs the specified keys.
func (c *cacheDir) Delete(keys ...cacheKey) (err error) {
	return os.Remove(cachePath(keys...))
}

// Deletes the specified key and all subkeys.
func (c *cacheDir) DeleteAll(keys ...cacheKey) (err error) {
	return os.RemoveAll(cachePath(keys...))
}

func (c *cacheDir) Get(v Cacheable, keys ...cacheKey) (err error) {
	defer func() {
		log.Println("Got entry", keys, "(error", err, ")")
	}()
	if flock := lockFile(cachePath(keys...)); flock != nil {
		flock.Lock()
		defer flock.Unlock()
	}

	fh, err := c.Open(keys...)
	if err != nil {
		return err
	}
	defer func() {
		if e := fh.Close(); err == nil {
			err = e
		}
	}()

	val := reflect.ValueOf(v)
	if k := val.Kind(); k == reflect.Ptr || k == reflect.Interface {
		val = val.Elem()
	}
	if !val.CanSet() {
		// panic because this is an internal coding mistake
		panic("(*cacheDir).Get(): given Cacheable is not setable")
	}
	gz, err := gzip.NewReader(fh)
	if err != nil {
		return err
	}
	defer func() {
		if e := gz.Close(); err == nil {
			err = e
		}
	}()

	// defer func() {
	// 	if err == io.EOF {
	// 		err = nil
	// 	}
	// }()

	switch f := gz.Header.Comment; f {
	case "encoding/gob":
		dec := gob.NewDecoder(gz)
		err = dec.Decode(v)
	default:
		return errors.New(fmt.Sprintf("Cached data (format %q) is not in a known format", f))
	}

	return
}

func (c *cacheDir) Set(v Cacheable, keys ...cacheKey) (n int64, err error) {
	if v := reflect.ValueOf(v); !v.IsValid() {
		panic("reflect.ValueOf() returned invaled value")
	} else if k := v.Kind(); k == reflect.Ptr || k == reflect.Interface {
		if v.IsNil() {
			return // no point in saving nil
		}
	}
	defer func() {
		log.Println("Set entry", keys, "(error", err, ")")
	}()

	// First we encode to memory -- we don't want to create/truncate a file and put bad data in it.
	buf := bytes.Buffer{}
	gz, err := gzip.NewWriterLevel(&buf, gzip.BestCompression)
	if err != nil {
		return 0, err
	}
	gz.Header.Comment = "encoding/gob"

	// it doesn't matter if the caller doesn't see this,
	// the important part is that the cache does.
	v.Touch()

	enc := gob.NewEncoder(gz)
	err = enc.Encode(v)

	if e := gz.Close(); err == nil {
		err = e
	}

	if err != nil {
		return 0, err
	}

	// We have good data, time to actually put it in the cache
	if flock := lockFile(cachePath(keys...)); flock != nil {
		flock.Lock()
		defer flock.Unlock()
	}

	fh, err := c.Create(keys...)
	if err != nil {
		return 0, err
	}
	defer func() {
		if e := fh.Close(); err == nil {
			err = e
		}
	}()
	n, err = io.Copy(fh, &buf)
	return
}

// Checks if the given keys are not marked as invalid.
//
// If the key was marked as invalid but is no longer considered
// so, deletes the invalid marker.
func (c *cacheDir) CheckValid(keys ...cacheKey) bool {
	invKeys := append([]cacheKey{"invalid"}, keys...)
	inv := invalidKeyCache{}

	if cache.Get(&inv, invKeys...) == nil {
		if inv.IsStale() {
			cache.Delete(invKeys...)
		} else {
			return false
		}
	}
	return true
}

// Deletes the given keys and marks them as invalid.
//
// They are considered invalid for InvalidKeyCacheDuration.
func (c *cacheDir) MarkInvalid(keys ...cacheKey) error {
	invKeys := append([]cacheKey{"invalid"}, keys...)

	cache.Delete(keys...)
	_, err := cache.Set(&invalidKeyCache{}, invKeys...)
	return err
}
<commit_msg>anidb: (*cacheDir).Get(): reduce the time a file lock is retained<commit_after>package anidb

import (
	"bytes"
	"compress/gzip"
	"encoding/gob"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"reflect"
	"regexp"
	"sync"
	"time"
)

var _ log.Logger

type Cacheable interface {
	// Updates the last modified time
	Touch()
	// Returns true if the Cacheable is nil, or if the last modified time is too old.
	IsStale() bool
}

func init() {
	gob.RegisterName("*github.com/Kovensky/go-anidb.invalidKeyCache", &invalidKeyCache{})
}

type invalidKeyCache struct{ time.Time }

func (c *invalidKeyCache) Touch() {
	c.Time = time.Now()
}
func (c *invalidKeyCache) IsStale() bool {
	return time.Now().Sub(c.Time) > InvalidKeyCacheDuration
}

type cacheDir struct {
	*sync.RWMutex

	CacheDir string
}

func init() {
	if err := SetCacheDir(path.Join(os.TempDir(), "anidb", "cache")); err != nil {
		panic(err)
	}
}

var cache cacheDir

// Sets the cache directory to the given path.
//
// go-anidb needs a valid cache directory to function, so, during module
// initialization, it uses os.TempDir() to set a default cache dir.
// go-anidb panics if it's unable to set the default cache dir.
func SetCacheDir(path string) (err error) {
	m := cache.RWMutex
	if m == nil {
		m = &sync.RWMutex{}
		cache.RWMutex = m
	}
	cache.Lock()

	if err = os.MkdirAll(path, 0755|os.ModeDir); err != nil {
		cache.Unlock()
		return err
	}

	cache = cacheDir{
		RWMutex:  m,
		CacheDir: path,
	}

	cache.Unlock()
	RefreshTitles()
	return nil
}

// Returns the current cache dir.
func GetCacheDir() (path string) {
	cache.RLock()
	defer cache.RUnlock()

	return cache.CacheDir
}

type cacheKey interface{}

// All "bad characters" that can't go in Windows paths.
// It's a superset of the "bad characters" on other OSes, so this works.
var badPath = regexp.MustCompile(`[\\/:\*\?\"<>\|]`)

func stringify(stuff ...cacheKey) []string {
	ret := make([]string, len(stuff))
	for i := range stuff {
		s := fmt.Sprint(stuff[i])
		ret[i] = badPath.ReplaceAllLiteralString(s, "_")
	}
	return ret
}

// Each key but the last is treated as a directory.
// The last key is treated as a regular file.
//
// This also means that cache keys that are file-backed
// cannot have subkeys.
func cachePath(keys ...cacheKey) string {
	parts := append([]string{GetCacheDir()}, stringify(keys...)...)
	p := path.Join(parts...)
	return p
}

// Opens the file that backs the specified keys.
func (c *cacheDir) Open(keys ...cacheKey) (fh *os.File, err error) {
	subItem := cachePath(keys...)
	return os.Open(subItem)
}

// Creates a new file to back the specified keys.
func (c *cacheDir) Create(keys ...cacheKey) (fh *os.File, err error) {
	subItem := cachePath(keys...)
	subDir := path.Dir(subItem)

	if err = os.MkdirAll(subDir, 0755|os.ModeDir); err != nil {
		return nil, err
	}
	return os.Create(subItem)
}

// Deletes the file that backs the specified keys.
func (c *cacheDir) Delete(keys ...cacheKey) (err error) {
	return os.Remove(cachePath(keys...))
}

// Deletes the specified key and all subkeys.
func (c *cacheDir) DeleteAll(keys ...cacheKey) (err error) {
	return os.RemoveAll(cachePath(keys...))
}

func (c *cacheDir) Get(v Cacheable, keys ...cacheKey) (err error) {
	defer func() {
		log.Println("Got entry", keys, "(error", err, ")")
	}()

	val := reflect.ValueOf(v)
	if k := val.Kind(); k == reflect.Ptr || k == reflect.Interface {
		val = val.Elem()
	}
	if !val.CanSet() {
		// panic because this is an internal coding mistake
		panic("(*cacheDir).Get(): given Cacheable is not setable")
	}

	flock := lockFile(cachePath(keys...))
	if flock != nil {
		flock.Lock()
	}
	defer func() {
		if flock != nil {
			flock.Unlock()
		}
	}()

	fh, err := c.Open(keys...)
	if err != nil {
		return err
	}

	buf := bytes.Buffer{}
	if _, err = io.Copy(&buf, fh); err != nil {
		fh.Close()
		return err
	}
	if err = fh.Close(); err != nil {
		return err
	}

	if flock != nil {
		flock.Unlock()
		flock = nil
	}

	gz, err := gzip.NewReader(&buf)
	if err != nil {
		return err
	}
	defer func() {
		if e := gz.Close(); err == nil {
			err = e
		}
	}()

	// defer func() {
	// 	if err == io.EOF {
	// 		err = nil
	// 	}
	// }()

	switch f := gz.Header.Comment; f {
	case "encoding/gob":
		dec := gob.NewDecoder(gz)
		err = dec.Decode(v)
	default:
		return errors.New(fmt.Sprintf("Cached data (format %q) is not in a known format", f))
	}

	return
}

func (c *cacheDir) Set(v Cacheable, keys ...cacheKey) (n int64, err error) {
	if v := reflect.ValueOf(v); !v.IsValid() {
		panic("reflect.ValueOf() returned invaled value")
	} else if k := v.Kind(); k == reflect.Ptr || k == reflect.Interface {
		if v.IsNil() {
			return // no point in saving nil
		}
	}
	defer func() {
		log.Println("Set entry", keys, "(error", err, ")")
	}()

	// First we encode to memory -- we don't want to create/truncate a file and put bad data in it.
	buf := bytes.Buffer{}
	gz, err := gzip.NewWriterLevel(&buf, gzip.BestCompression)
	if err != nil {
		return 0, err
	}
	gz.Header.Comment = "encoding/gob"

	// it doesn't matter if the caller doesn't see this,
	// the important part is that the cache does.
	v.Touch()

	enc := gob.NewEncoder(gz)
	err = enc.Encode(v)

	if e := gz.Close(); err == nil {
		err = e
	}

	if err != nil {
		return 0, err
	}

	// We have good data, time to actually put it in the cache
	if flock := lockFile(cachePath(keys...)); flock != nil {
		flock.Lock()
		defer flock.Unlock()
	}

	fh, err := c.Create(keys...)
	if err != nil {
		return 0, err
	}
	defer func() {
		if e := fh.Close(); err == nil {
			err = e
		}
	}()
	n, err = io.Copy(fh, &buf)
	return
}

// Checks if the given keys are not marked as invalid.
//
// If the key was marked as invalid but is no longer considered
// so, deletes the invalid marker.
func (c *cacheDir) CheckValid(keys ...cacheKey) bool {
	invKeys := append([]cacheKey{"invalid"}, keys...)
	inv := invalidKeyCache{}

	if cache.Get(&inv, invKeys...) == nil {
		if inv.IsStale() {
			cache.Delete(invKeys...)
		} else {
			return false
		}
	}
	return true
}

// Deletes the given keys and marks them as invalid.
//
// They are considered invalid for InvalidKeyCacheDuration.
func (c *cacheDir) MarkInvalid(keys ...cacheKey) error {
	invKeys := append([]cacheKey{"invalid"}, keys...)

	cache.Delete(keys...)
	_, err := cache.Set(&invalidKeyCache{}, invKeys...)
	return err
}
<|endoftext|>


<commit_before>package cache2go

import (
	"container/list"
	"sync"
	"time"
)

// Cache is an LRU cache. It is not safe for concurrent access.
type Cache struct {
	mu sync.RWMutex
	// MaxEntries is the maximum number of cache entries before
	// an item is evicted. Zero means no limit.
	maxEntries int

	// OnEvicted optionally specificies a callback function to be
	// executed when an entry is purged from the cache.
	OnEvicted func(key string, value interface{})

	lruIndex   *list.List
	ttlIndex   []*list.Element
	cache      map[interface{}]*list.Element
	expiration time.Duration
}

type entry struct {
	key       string
	value     interface{}
	timestamp time.Time
}

// New creates a new Cache.
// If maxEntries is zero, the cache has no limit and it's assumed
// that eviction is done by the caller.
func New(maxEntries int, expire time.Duration) *Cache {
	c := &Cache{
		maxEntries: maxEntries,
		expiration: expire,
		lruIndex:   list.New(),
		ttlIndex:   make([]*list.Element, 0),
		cache:      make(map[interface{}]*list.Element),
	}
	if c.expiration > 0 {
		go c.cleanExpired()
	}
	return c
}

// cleans expired entries performing minimal checks
func (c *Cache) cleanExpired() {
	for {
		if len(c.ttlIndex) == 0 {
			time.Sleep(c.expiration)
			continue
		}
		c.mu.RLock()
		e := c.ttlIndex[0]
		c.mu.RUnlock()

		en := e.Value.(*entry)
		if time.Now().After(en.timestamp.Add(c.expiration)) {
			c.mu.Lock()
			c.removeElement(e)
			c.mu.Unlock()
		} else {
			time.Sleep(time.Now().Sub(en.timestamp.Add(c.expiration)))
		}
	}
}

// Add adds a value to the cache
func (c *Cache) Set(key string, value interface{}) {
	c.mu.Lock()
	if c.cache == nil {
		c.cache = make(map[interface{}]*list.Element)
		c.lruIndex = list.New()
		c.ttlIndex = make([]*list.Element, 0)
	}

	if e, ok := c.cache[key]; ok {
		c.lruIndex.MoveToFront(e)

		en := e.Value.(*entry)
		en.value = value
		en.timestamp = time.Now()

		c.mu.Unlock()
		return
	}
	e := c.lruIndex.PushFront(&entry{key: key, value: value, timestamp: time.Now()})
	c.ttlIndex = append(c.ttlIndex, e)
	c.cache[key] = e
	c.mu.Unlock()

	if c.maxEntries != 0 && c.lruIndex.Len() > c.maxEntries {
		c.RemoveOldest()
	}
}

// Get looks up a key's value from the cache.
func (c *Cache) Get(key string) (value interface{}, ok bool) {
	c.mu.RLock()
	defer c.mu.RUnlock()
	if c.cache == nil {
		return
	}
	if e, hit := c.cache[key]; hit {
		c.lruIndex.MoveToFront(e)
		e.Value.(*entry).timestamp = time.Now()
		return e.Value.(*entry).value, true
	}
	return
}

// Remove removes the provided key from the cache.
func (c *Cache) Remove(key string) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.cache == nil {
		return
	}
	if e, hit := c.cache[key]; hit {
		c.removeElement(e)
	}
}

// RemoveOldest removes the oldest item from the cache.
func (c *Cache) RemoveOldest() {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.cache == nil {
		return
	}
	e := c.lruIndex.Back()
	if e != nil {
		c.removeElement(e)
	}
}

func (c *Cache) removeElement(e *list.Element) {
	c.lruIndex.Remove(e)
	if c.expiration > 0 {
		for i, se := range c.ttlIndex {
			if se == e {
				//delete
				copy(c.ttlIndex[i:], c.ttlIndex[i+1:])
				c.ttlIndex[len(c.ttlIndex)-1] = nil
				c.ttlIndex = c.ttlIndex[:len(c.ttlIndex)-1]
				break
			}
		}
	}
	kv := e.Value.(*entry)
	delete(c.cache, kv.key)
	if c.OnEvicted != nil {
		c.OnEvicted(kv.key, kv.value)
	}
}

// Len returns the number of items in the cache.
func (c *Cache) Len() int {
	c.mu.RLock()
	defer c.mu.RUnlock()
	if c.cache == nil {
		return 0
	}
	return c.lruIndex.Len()
}

// empties the whole cache
func (c *Cache) Flush() {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.lruIndex = list.New()
	c.ttlIndex = make([]*list.Element, 0)
	c.cache = make(map[interface{}]*list.Element)
}
<commit_msg>better locking<commit_after>package cache2go

import (
	"container/list"
	"sync"
	"time"
)

// Cache is an LRU cache. It is not safe for concurrent access.
type Cache struct {
	mu sync.RWMutex
	// MaxEntries is the maximum number of cache entries before
	// an item is evicted. Zero means no limit.
	maxEntries int

	// OnEvicted optionally specificies a callback function to be
	// executed when an entry is purged from the cache.
	OnEvicted func(key string, value interface{})

	lruIndex   *list.List
	ttlIndex   []*list.Element
	cache      map[interface{}]*list.Element
	expiration time.Duration
}

type entry struct {
	key       string
	value     interface{}
	timestamp time.Time
}

// New creates a new Cache.
// If maxEntries is zero, the cache has no limit and it's assumed
// that eviction is done by the caller.
func New(maxEntries int, expire time.Duration) *Cache {
	c := &Cache{
		maxEntries: maxEntries,
		expiration: expire,
		lruIndex:   list.New(),
		ttlIndex:   make([]*list.Element, 0),
		cache:      make(map[interface{}]*list.Element),
	}
	if c.expiration > 0 {
		go c.cleanExpired()
	}
	return c
}

// cleans expired entries performing minimal checks
func (c *Cache) cleanExpired() {
	for {
		c.mu.RLock()
		if len(c.ttlIndex) == 0 {
			c.mu.RUnlock()
			time.Sleep(c.expiration)
			continue
		}
		e := c.ttlIndex[0]
		c.mu.RUnlock()

		en := e.Value.(*entry)
		if time.Now().After(en.timestamp.Add(c.expiration)) {
			c.mu.Lock()
			c.removeElement(e)
			c.mu.Unlock()
		} else {
			time.Sleep(time.Now().Sub(en.timestamp.Add(c.expiration)))
		}
	}
}

// Add adds a value to the cache
func (c *Cache) Set(key string, value interface{}) {
	c.mu.Lock()
	if c.cache == nil {
		c.cache = make(map[interface{}]*list.Element)
		c.lruIndex = list.New()
		c.ttlIndex = make([]*list.Element, 0)
	}

	if e, ok := c.cache[key]; ok {
		c.lruIndex.MoveToFront(e)

		en := e.Value.(*entry)
		en.value = value
		en.timestamp = time.Now()

		c.mu.Unlock()
		return
	}
	e := c.lruIndex.PushFront(&entry{key: key, value: value, timestamp: time.Now()})
	c.ttlIndex = append(c.ttlIndex, e)
	c.cache[key] = e
	c.mu.Unlock()

	if c.maxEntries != 0 && c.lruIndex.Len() > c.maxEntries {
		c.RemoveOldest()
	}
}

// Get looks up a key's value from the cache.
func (c *Cache) Get(key string) (value interface{}, ok bool) {
	c.mu.RLock()
	defer c.mu.RUnlock()
	if c.cache == nil {
		return
	}
	if e, hit := c.cache[key]; hit {
		c.lruIndex.MoveToFront(e)
		e.Value.(*entry).timestamp = time.Now()
		return e.Value.(*entry).value, true
	}
	return
}

// Remove removes the provided key from the cache.
func (c *Cache) Remove(key string) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.cache == nil {
		return
	}
	if e, hit := c.cache[key]; hit {
		c.removeElement(e)
	}
}

// RemoveOldest removes the oldest item from the cache.
func (c *Cache) RemoveOldest() {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.cache == nil {
		return
	}
	e := c.lruIndex.Back()
	if e != nil {
		c.removeElement(e)
	}
}

func (c *Cache) removeElement(e *list.Element) {
	c.lruIndex.Remove(e)
	if c.expiration > 0 {
		for i, se := range c.ttlIndex {
			if se == e {
				//delete
				copy(c.ttlIndex[i:], c.ttlIndex[i+1:])
				c.ttlIndex[len(c.ttlIndex)-1] = nil
				c.ttlIndex = c.ttlIndex[:len(c.ttlIndex)-1]
				break
			}
		}
	}
	kv := e.Value.(*entry)
	delete(c.cache, kv.key)
	if c.OnEvicted != nil {
		c.OnEvicted(kv.key, kv.value)
	}
}

// Len returns the number of items in the cache.
func (c *Cache) Len() int {
	c.mu.RLock()
	defer c.mu.RUnlock()
	if c.cache == nil {
		return 0
	}
	return c.lruIndex.Len()
}

// empties the whole cache
func (c *Cache) Flush() {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.lruIndex = list.New()
	c.ttlIndex = make([]*list.Element, 0)
	c.cache = make(map[interface{}]*list.Element)
}
<|endoftext|>


<commit_before><commit_msg>fleet: For global units, return a nicer global status<commit_after><|endoftext|>


<commit_before><commit_msg>Gauss function in 2D<commit_after><|endoftext|>


<commit_before><commit_msg>funcr: Cache framesToCaller, save a few 100 ns<commit_after><|endoftext|>


<commit_before><commit_msg>test: TestGitHubPullRequest_Post_Flash_mock<commit_after><|endoftext|>


<commit_before><commit_msg>Release rss lock - in case of an error<commit_after><|endoftext|>


<commit_before><commit_msg>also skip internal dir<commit_after><|endoftext|>


<commit_before>package group

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/messagebird/go-rest-api/contact"

	messagebird "github.com/messagebird/go-rest-api"
)

// Group gets returned by the API.
type Group struct {
	ID       string
	HRef     string
	Name     string
	Contacts struct {
		TotalCount int
		HRef       string
	}
	CreatedDatetime time.Time
	UpdatedDatetime time.Time
}

type GroupList struct {
	Offset     int
	Limit      int
	Count      int
	TotalCount int
	Links      struct {
		First    string
		Previous string
		Next     string
		Last     string
	}
	Items []Group
}

// ListOptions can be used to set pagination options in List() and ListContacts().
type ListOptions struct {
	Limit, Offset int
}

// Request represents a contact for write operations, e.g. for creating a new
// group or updating an existing one.
type Request struct {
	Name string `json:"name"`
}

const (
	// path represents the path to the Groups resource.
	path = "groups"

	// contactPath represents the path to the Contacts resource within Groups.
	contactPath = "contacts"
)

// DefaultListOptions provides reasonable values for List().
var DefaultListOptions = &ListOptions{
	Limit:  10,
	Offset: 0,
}

func Create(c *messagebird.Client, request *Request) (*Group, error) {
	if err := validateCreate(request); err != nil {
		return nil, err
	}

	group := &Group{}
	if err := c.Request(group, http.MethodPost, path, request); err != nil {
		return nil, err
	}

	return group, nil
}

func validateCreate(request *Request) error {
	if request.Name == "" {
		return errors.New("name is required")
	}

	return nil
}

// Delete attempts deleting the group with the provided ID. If nil is returned,
// the resource was deleted successfully.
func Delete(c *messagebird.Client, id string) error {
	return c.Request(nil, http.MethodDelete, path+"/"+id, nil)
}

// List retrieves a paginated list of groups, based on the options provided.
// It's worth noting DefaultListOptions.
func List(c *messagebird.Client, options *ListOptions) (*GroupList, error) {
	query, err := listQuery(options)
	if err != nil {
		return nil, err
	}

	groupList := &GroupList{}
	if err := c.Request(groupList, http.MethodGet, path+"?"+query, nil); err != nil {
		return nil, err
	}

	return groupList, nil
}

func listQuery(options *ListOptions) (string, error) {
	if options.Limit < 10 {
		return "", fmt.Errorf("minimum limit is 10, got %d", options.Limit)
	}

	if options.Offset < 0 {
		return "", fmt.Errorf("offset can not be negative")
	}

	values := &url.Values{}

	values.Set("limit", strconv.Itoa(options.Limit))
	values.Set("offset", strconv.Itoa(options.Offset))

	return values.Encode(), nil
}

// Read retrieves the information of an existing group.
func Read(c *messagebird.Client, id string) (*Group, error) {
	group := &Group{}
	if err := c.Request(group, http.MethodGet, path+"/"+id, nil); err != nil {
		return nil, err
	}

	return group, nil
}

// Update overrides the group with any values provided in request.
func Update(c *messagebird.Client, id string, request *Request) error {
	if err := validateUpdate(request); err != nil {
		return err
	}

	return c.Request(nil, http.MethodPatch, path+"/"+id, request)
}

func validateUpdate(request *Request) error {
	if request.Name == "" {
		return errors.New("name is required")
	}

	return nil
}

// AddContacts adds a maximum of 50 contacts to the group.
func AddContacts(c *messagebird.Client, groupID string, contactIDS []string) error {
	if err := validateAddContacts(contactIDS); err != nil {
		return err
	}

	query := addContactsQuery(contactIDS)
	formattedPath := fmt.Sprintf("%s/%s/%s?%s", path, groupID, contactPath, query)

	return c.Request(nil, http.MethodGet, formattedPath, nil)
}

func validateAddContacts(contactIDS []string) error {
	count := len(contactIDS)

	// len(nil) == 0: https://golang.org/ref/spec#Length_and_capacity
	if count == 0 {
		return fmt.Errorf("contactIDS is required")
	}

	if count > 50 {
		return fmt.Errorf("can not add more than 50 contacts per request, got %d", count)
	}

	return nil
}

// addContactsQuery gets a query string to add contacts to a group. We're using
// the alternative "/foo?_method=PUT&key=value" syntax to send the contact IDs
// as GET params. Sending these in the request body would require a painful
// workaround, as client.Request sends request bodies as JSON by default. See
// also: https://developers.messagebird.com/docs/alternatives.
func addContactsQuery(contactIDS []string) string {
	// Slice's length is one bigger than len(IDs) for the _method param.
	params := make([]string, 0, len(contactIDS)+1)
	params = append(params, "_method="+http.MethodPut)

	for _, contactID := range contactIDS {
		params = append(params, "ids[]="+contactID)
	}

	return strings.Join(params, "&")
}

// ListContacts lists the contacts that are a member of a group.
func ListContacts(c *messagebird.Client, groupID string, options *ListOptions) (*contact.ContactList, error) {
	query, err := listQuery(options)
	if err != nil {
		return nil, err
	}

	formattedPath := fmt.Sprintf("%s/%s/%s?%s", path, groupID, contactPath, query)

	contacts := &contact.ContactList{}
	if err = c.Request(contacts, http.MethodGet, formattedPath, nil); err != nil {
		return nil, err
	}

	return contacts, nil
}

// RemoveContact removes the contact from a group. If nil is returned, the
// operation was successful.
func RemoveContact(c *messagebird.Client, groupID, contactID string) error {
	formattedPath := fmt.Sprintf("%s/%s/contacts/%s", path, groupID, contactID)

	return c.Request(nil, http.MethodDelete, formattedPath, nil)
}
<commit_msg>Remove unnecessary newlines<commit_after>package group

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	messagebird "github.com/messagebird/go-rest-api"
	"github.com/messagebird/go-rest-api/contact"
)

// Group gets returned by the API.
type Group struct {
	ID       string
	HRef     string
	Name     string
	Contacts struct {
		TotalCount int
		HRef       string
	}
	CreatedDatetime time.Time
	UpdatedDatetime time.Time
}

type GroupList struct {
	Offset     int
	Limit      int
	Count      int
	TotalCount int
	Links      struct {
		First    string
		Previous string
		Next     string
		Last     string
	}
	Items []Group
}

// ListOptions can be used to set pagination options in List() and ListContacts().
type ListOptions struct {
	Limit, Offset int
}

// Request represents a contact for write operations, e.g. for creating a new
// group or updating an existing one.
type Request struct {
	Name string `json:"name"`
}

const (
	// path represents the path to the Groups resource.
	path = "groups"

	// contactPath represents the path to the Contacts resource within Groups.
	contactPath = "contacts"
)

// DefaultListOptions provides reasonable values for List().
var DefaultListOptions = &ListOptions{
	Limit:  10,
	Offset: 0,
}

func Create(c *messagebird.Client, request *Request) (*Group, error) {
	if err := validateCreate(request); err != nil {
		return nil, err
	}

	group := &Group{}
	if err := c.Request(group, http.MethodPost, path, request); err != nil {
		return nil, err
	}

	return group, nil
}

func validateCreate(request *Request) error {
	if request.Name == "" {
		return errors.New("name is required")
	}

	return nil
}

// Delete attempts deleting the group with the provided ID. If nil is returned,
// the resource was deleted successfully.
func Delete(c *messagebird.Client, id string) error {
	return c.Request(nil, http.MethodDelete, path+"/"+id, nil)
}

// List retrieves a paginated list of groups, based on the options provided.
// It's worth noting DefaultListOptions.
func List(c *messagebird.Client, options *ListOptions) (*GroupList, error) {
	query, err := listQuery(options)
	if err != nil {
		return nil, err
	}

	groupList := &GroupList{}
	if err := c.Request(groupList, http.MethodGet, path+"?"+query, nil); err != nil {
		return nil, err
	}

	return groupList, nil
}

func listQuery(options *ListOptions) (string, error) {
	if options.Limit < 10 {
		return "", fmt.Errorf("minimum limit is 10, got %d", options.Limit)
	}

	if options.Offset < 0 {
		return "", fmt.Errorf("offset can not be negative")
	}

	values := &url.Values{}
	values.Set("limit", strconv.Itoa(options.Limit))
	values.Set("offset", strconv.Itoa(options.Offset))

	return values.Encode(), nil
}

// Read retrieves the information of an existing group.
func Read(c *messagebird.Client, id string) (*Group, error) {
	group := &Group{}
	if err := c.Request(group, http.MethodGet, path+"/"+id, nil); err != nil {
		return nil, err
	}

	return group, nil
}

// Update overrides the group with any values provided in request.
func Update(c *messagebird.Client, id string, request *Request) error {
	if err := validateUpdate(request); err != nil {
		return err
	}

	return c.Request(nil, http.MethodPatch, path+"/"+id, request)
}

func validateUpdate(request *Request) error {
	if request.Name == "" {
		return errors.New("name is required")
	}

	return nil
}

// AddContacts adds a maximum of 50 contacts to the group.
func AddContacts(c *messagebird.Client, groupID string, contactIDS []string) error {
	if err := validateAddContacts(contactIDS); err != nil {
		return err
	}

	query := addContactsQuery(contactIDS)
	formattedPath := fmt.Sprintf("%s/%s/%s?%s", path, groupID, contactPath, query)

	return c.Request(nil, http.MethodGet, formattedPath, nil)
}

func validateAddContacts(contactIDS []string) error {
	count := len(contactIDS)

	// len(nil) == 0: https://golang.org/ref/spec#Length_and_capacity
	if count == 0 {
		return fmt.Errorf("contactIDS is required")
	}

	if count > 50 {
		return fmt.Errorf("can not add more than 50 contacts per request, got %d", count)
	}

	return nil
}

// addContactsQuery gets a query string to add contacts to a group. We're using
// the alternative "/foo?_method=PUT&key=value" syntax to send the contact IDs
// as GET params. Sending these in the request body would require a painful
// workaround, as client.Request sends request bodies as JSON by default. See
// also: https://developers.messagebird.com/docs/alternatives.
func addContactsQuery(contactIDS []string) string {
	// Slice's length is one bigger than len(IDs) for the _method param.
	params := make([]string, 0, len(contactIDS)+1)
	params = append(params, "_method="+http.MethodPut)

	for _, contactID := range contactIDS {
		params = append(params, "ids[]="+contactID)
	}

	return strings.Join(params, "&")
}

// ListContacts lists the contacts that are a member of a group.
func ListContacts(c *messagebird.Client, groupID string, options *ListOptions) (*contact.ContactList, error) {
	query, err := listQuery(options)
	if err != nil {
		return nil, err
	}

	formattedPath := fmt.Sprintf("%s/%s/%s?%s", path, groupID, contactPath, query)

	contacts := &contact.ContactList{}
	if err = c.Request(contacts, http.MethodGet, formattedPath, nil); err != nil {
		return nil, err
	}

	return contacts, nil
}

// RemoveContact removes the contact from a group. If nil is returned, the
// operation was successful.
func RemoveContact(c *messagebird.Client, groupID, contactID string) error {
	formattedPath := fmt.Sprintf("%s/%s/contacts/%s", path, groupID, contactID)

	return c.Request(nil, http.MethodDelete, formattedPath, nil)
}
<|endoftext|>


<commit_before>// +build !go1.5

package gb

import (
	"bytes"
	"go/build"
	"path/filepath"
	"strings"
)

// cgo support functions

// cgo returns a slice of post processed source files and an
// ObjTargets representing the result of compilation of the post .c
// output.
func cgo(pkg *Package) ([]ObjTarget, []string) {
	fn := func(t ...ObjTarget) ([]ObjTarget, []string) {
		return t, nil
	}
	if err := runcgo1(pkg); err != nil {
		return fn(ErrTarget{err})
	}

	defun, err := runcc1(pkg)
	if err != nil {
		return fn(ErrTarget{err})
	}

	cgofiles := []string{filepath.Join(pkg.Objdir(), "_cgo_gotypes.go")}
	for _, f := range pkg.CgoFiles {
		cgofiles = append(cgofiles, filepath.Join(pkg.Objdir(), stripext(f)+".cgo1.go"))
	}
	cfiles := []string{
		filepath.Join(pkg.Objdir(), "_cgo_main.c"),
		filepath.Join(pkg.Objdir(), "_cgo_export.c"),
	}
	cfiles = append(cfiles, pkg.CFiles...)

	for _, f := range pkg.CgoFiles {
		cfiles = append(cfiles, filepath.Join(pkg.Objdir(), stripext(f)+".cgo2.c"))
	}

	var ofiles []string
	for _, f := range cfiles {
		ofile := stripext(f) + ".o"
		ofiles = append(ofiles, ofile)
		if err := rungcc1(pkg.Dir, ofile, f); err != nil {
			return fn(ErrTarget{err})
		}
	}

	ofile, err := rungcc2(pkg.Dir, ofiles)
	if err != nil {
		return fn(ErrTarget{err})
	}

	_, err = runcgo2(pkg, ofile)
	if err != nil {
		return fn(ErrTarget{err})
	}
	imports, err := runcc2(pkg) // TODO(dfc) should compile dynout from above
	if err != nil {
		return fn(ErrTarget{err})
	}

	allo, err := rungcc3(pkg.Dir, ofiles[1:]) // skip _cgo_main.o
	if err != nil {
		return fn(ErrTarget{err})
	}

	return []ObjTarget{cgoTarget(defun), cgoTarget(imports), cgoTarget(allo)}, cgofiles
}

type cgoTarget string

func (t cgoTarget) Objfile() string { return string(t) }
func (t cgoTarget) Result() error   { return nil }

// runcgo1 invokes the cgo tool to process pkg.CgoFiles.
func runcgo1(pkg *Package) error {
	cgo := cgotool(pkg.Context)
	objdir := pkg.Objdir()
	if err := mkdir(objdir); err != nil {
		return err
	}

	args := []string{
		"-objdir", objdir,
		"--",
		"-I", pkg.Dir,
	}
	args = append(args, pkg.CgoFiles...)
	return run(pkg.Dir, cgo, args...)
}

// runcgo2 invokes the cgo tool to create _cgo_import.go
func runcgo2(pkg *Package, ofile string) (string, error) {
	cgo := cgotool(pkg.Context)
	objdir := pkg.Objdir()
	dynout := filepath.Join(objdir, "_cgo_import.c")

	args := []string{
		"-objdir", objdir,
		"-dynimport", ofile,
		"-dynout", dynout,
	}
	return dynout, run(pkg.Dir, cgo, args...)
}

func runcc1(pkg *Package) (string, error) {
	archchar, err := build.ArchChar(pkg.GOARCH)
	if err != nil {
		return "", err
	}
	cc := filepath.Join(pkg.GOROOT, "pkg", "tool", pkg.GOOS+"_"+pkg.GOARCH, archchar+"c")
	objdir := pkg.Objdir()
	ofile := filepath.Join(objdir, "_cgo_defun."+archchar)
	args := []string{
		"-F", "-V", "-w",
		"-trimpath", pkg.Workdir(),
		"-I", objdir,
		"-I", filepath.Join(pkg.GOROOT, "pkg", pkg.GOOS+"_"+pkg.GOARCH), // for runtime.h
		"-o", ofile,
		"-D", "GOOS_" + pkg.GOOS,
		"-D", "GOARCH_" + pkg.GOARCH,
		filepath.Join(objdir, "_cgo_defun.c"),
	}
	return ofile, run(pkg.Dir, cc, args...)
}

// /home/dfc/go/pkg/tool/linux_amd64/6c -F -V -w -trimpath $WORK -I $WORK/cgomain/_obj/ -I /home/dfc/go/pkg/linux_amd64 -o $WORK/cgomain/_obj/_cgo_import.6 -D GOOS_linux -D GOARCH_amd64 $WORK/cgomain/_obj/_cgo_import.c
func runcc2(pkg *Package) (string, error) {
	archchar, err := build.ArchChar(pkg.GOARCH)
	if err != nil {
		return "", err
	}
	cc := filepath.Join(pkg.GOROOT, "pkg", "tool", pkg.GOOS+"_"+pkg.GOARCH, archchar+"c")
	objdir := pkg.Objdir()
	ofile := filepath.Join(objdir, "_cgo_import."+archchar)
	args := []string{
		"-F", "-V", "-w",
		"-trimpath", pkg.Workdir(),
		"-I", objdir,
		"-o", ofile,
		"-D", "GOOS_" + pkg.GOOS,
		"-D", "GOARCH_" + pkg.GOARCH,
		filepath.Join(objdir, "_cgo_import.c"),
	}
	return ofile, run(pkg.Dir, cc, args...)
}

// rungcc1 invokes gcc to compile cfile into ofile
func rungcc1(dir, ofile, cfile string) error {
	gcc := "gcc" // TODO(dfc) handle $CC and clang
	args := []string{
		"-fPIC", "-m64", "-pthread", "-fmessage-length=0",
		"-I", dir,
		"-I", filepath.Dir(ofile),
		"-g", "-O2",
		"-o", ofile,
		"-c", cfile,
	}
	return run(dir, gcc, args...)
}

// rungcc2 links the o files from rungcc1 into a single _cgo_.o.
func rungcc2(dir string, ofiles []string) (string, error) {
	gcc := "gcc" // TODO(dfc) handle $CC and clang
	ofile := filepath.Join(filepath.Dir(ofiles[0]), "_cgo_.o")
	args := []string{
		"-fPIC", "-m64", "-pthread", "-fmessage-length=0",
		"-o", ofile,
	}
	args = append(args, ofiles...)
	args = append(args, "-g", "-O2") // this has to go at the end, because reasons!
	return ofile, run(dir, gcc, args...)
}

// rungcc3 links all previous ofiles together with libgcc into a single _all.o.
func rungcc3(dir string, ofiles []string) (string, error) {
	libgcc, err := libgcc()
	if err != nil {
		return "", nil
	}
	gcc := "gcc" // TODO(dfc) handle $CC and clang
	ofile := filepath.Join(filepath.Dir(ofiles[0]), "_all.o")
	args := []string{
		"-fPIC", "-m64", "-pthread", "-fmessage-length=0",
		"-g", "-O2",
		"-o", ofile,
	}
	args = append(args, ofiles...)
	args = append(args, "-Wl,-r", "-nostdlib", libgcc, "-Wl,--build-id=none")
	return ofile, run(dir, gcc, args...)
}

// libgcc returns the value of gcc -print-libgcc-file-name.
func libgcc() (string, error) {
	gcc := "gcc" // TODO(dfc) handle $CC and clang
	args := []string{
		"-fPIC", "-m64", "-pthread", "-fmessage-length=0",
		"-print-libgcc-file-name",
	}
	var buf bytes.Buffer
	err := runOut(&buf, ".", gcc, args...)
	return strings.TrimSpace(buf.String()), err
}

func cgotool(ctx *Context) string {
	return filepath.Join(ctx.GOROOT, "pkg", "tool", ctx.GOOS+"_"+ctx.GOARCH, "cgo")
}
<commit_msg>consolidate runcc{1,2}<commit_after>// +build !go1.5

package gb

import (
	"bytes"
	"go/build"
	"path/filepath"
	"strings"
)

// cgo support functions

// cgo returns a slice of post processed source files and an
// ObjTargets representing the result of compilation of the post .c
// output.
func cgo(pkg *Package) ([]ObjTarget, []string) {
	fn := func(t ...ObjTarget) ([]ObjTarget, []string) {
		return t, nil
	}
	if err := runcgo1(pkg); err != nil {
		return fn(ErrTarget{err})
	}

	defun, err := runcc(pkg, filepath.Join(pkg.Objdir(), "_cgo_defun.c"))
	if err != nil {
		return fn(ErrTarget{err})
	}

	cgofiles := []string{filepath.Join(pkg.Objdir(), "_cgo_gotypes.go")}
	for _, f := range pkg.CgoFiles {
		cgofiles = append(cgofiles, filepath.Join(pkg.Objdir(), stripext(f)+".cgo1.go"))
	}
	cfiles := []string{
		filepath.Join(pkg.Objdir(), "_cgo_main.c"),
		filepath.Join(pkg.Objdir(), "_cgo_export.c"),
	}
	cfiles = append(cfiles, pkg.CFiles...)

	for _, f := range pkg.CgoFiles {
		cfiles = append(cfiles, filepath.Join(pkg.Objdir(), stripext(f)+".cgo2.c"))
	}

	var ofiles []string
	for _, f := range cfiles {
		ofile := stripext(f) + ".o"
		ofiles = append(ofiles, ofile)
		if err := rungcc1(pkg.Dir, ofile, f); err != nil {
			return fn(ErrTarget{err})
		}
	}

	ofile, err := rungcc2(pkg.Dir, ofiles)
	if err != nil {
		return fn(ErrTarget{err})
	}

	dynout, err := runcgo2(pkg, ofile)
	if err != nil {
		return fn(ErrTarget{err})
	}
	imports, err := runcc(pkg, dynout)
	if err != nil {
		return fn(ErrTarget{err})
	}

	allo, err := rungcc3(pkg.Dir, ofiles[1:]) // skip _cgo_main.o
	if err != nil {
		return fn(ErrTarget{err})
	}

	return []ObjTarget{cgoTarget(defun), cgoTarget(imports), cgoTarget(allo)}, cgofiles
}

type cgoTarget string

func (t cgoTarget) Objfile() string { return string(t) }
func (t cgoTarget) Result() error   { return nil }

// runcgo1 invokes the cgo tool to process pkg.CgoFiles.
func runcgo1(pkg *Package) error {
	cgo := cgotool(pkg.Context)
	objdir := pkg.Objdir()
	if err := mkdir(objdir); err != nil {
		return err
	}

	args := []string{
		"-objdir", objdir,
		"--",
		"-I", pkg.Dir,
	}
	args = append(args, pkg.CgoFiles...)
	return run(pkg.Dir, cgo, args...)
}

// runcgo2 invokes the cgo tool to create _cgo_import.go
func runcgo2(pkg *Package, ofile string) (string, error) {
	cgo := cgotool(pkg.Context)
	objdir := pkg.Objdir()
	dynout := filepath.Join(objdir, "_cgo_import.c")

	args := []string{
		"-objdir", objdir,
		"-dynimport", ofile,
		"-dynout", dynout,
	}
	return dynout, run(pkg.Dir, cgo, args...)
}

func runcc(pkg *Package, cfile string) (string, error) {
	archchar, err := build.ArchChar(pkg.GOARCH)
	if err != nil {
		return "", err
	}
	cc := filepath.Join(pkg.GOROOT, "pkg", "tool", pkg.GOOS+"_"+pkg.GOARCH, archchar+"c")
	objdir := pkg.Objdir()
	ofile := filepath.Join(stripext(cfile) + "." + archchar)
	args := []string{
		"-F", "-V", "-w",
		"-trimpath", pkg.Workdir(),
		"-I", objdir,
		"-I", filepath.Join(pkg.GOROOT, "pkg", pkg.GOOS+"_"+pkg.GOARCH), // for runtime.h
		"-o", ofile,
		"-D", "GOOS_" + pkg.GOOS,
		"-D", "GOARCH_" + pkg.GOARCH,
		cfile,
	}
	return ofile, run(pkg.Dir, cc, args...)
}

// rungcc1 invokes gcc to compile cfile into ofile
func rungcc1(dir, ofile, cfile string) error {
	gcc := "gcc" // TODO(dfc) handle $CC and clang
	args := []string{
		"-fPIC", "-m64", "-pthread", "-fmessage-length=0",
		"-I", dir,
		"-I", filepath.Dir(ofile),
		"-g", "-O2",
		"-o", ofile,
		"-c", cfile,
	}
	return run(dir, gcc, args...)
}

// rungcc2 links the o files from rungcc1 into a single _cgo_.o.
func rungcc2(dir string, ofiles []string) (string, error) {
	gcc := "gcc" // TODO(dfc) handle $CC and clang
	ofile := filepath.Join(filepath.Dir(ofiles[0]), "_cgo_.o")
	args := []string{
		"-fPIC", "-m64", "-pthread", "-fmessage-length=0",
		"-o", ofile,
	}
	args = append(args, ofiles...)
	args = append(args, "-g", "-O2") // this has to go at the end, because reasons!
	return ofile, run(dir, gcc, args...)
}

// rungcc3 links all previous ofiles together with libgcc into a single _all.o.
func rungcc3(dir string, ofiles []string) (string, error) {
	libgcc, err := libgcc()
	if err != nil {
		return "", nil
	}
	gcc := "gcc" // TODO(dfc) handle $CC and clang
	ofile := filepath.Join(filepath.Dir(ofiles[0]), "_all.o")
	args := []string{
		"-fPIC", "-m64", "-pthread", "-fmessage-length=0",
		"-g", "-O2",
		"-o", ofile,
	}
	args = append(args, ofiles...)
	args = append(args, "-Wl,-r", "-nostdlib", libgcc, "-Wl,--build-id=none")
	return ofile, run(dir, gcc, args...)
}

// libgcc returns the value of gcc -print-libgcc-file-name.
func libgcc() (string, error) {
	gcc := "gcc" // TODO(dfc) handle $CC and clang
	args := []string{
		"-fPIC", "-m64", "-pthread", "-fmessage-length=0",
		"-print-libgcc-file-name",
	}
	var buf bytes.Buffer
	err := runOut(&buf, ".", gcc, args...)
	return strings.TrimSpace(buf.String()), err
}

func cgotool(ctx *Context) string {
	return filepath.Join(ctx.GOROOT, "pkg", "tool", ctx.GOOS+"_"+ctx.GOARCH, "cgo")
}
<|endoftext|>


<commit_before><commit_msg>Fixed up the names for techniques to be consistent.<commit_after><|endoftext|>