<commit_before>package z

import (
	"crypto/rand"
	"hash/fnv"
	"testing"

	"github.com/dgryski/go-farm"
)

func BenchmarkMemHash(b *testing.B) {
	buf := make([]byte, 64)
	rand.Read(buf)
	for i := 0; i < b.N; i++ {
		MemHash(buf)
	}
}

func BenchmarkSip(b *testing.B) {
	buf := make([]byte, 64)
	rand.Read(buf)
	for i := 0; i < b.N; i++ {
		SipHash(buf)
	}
}

func BenchmarkFarm(b *testing.B) {
	buf := make([]byte, 64)
	rand.Read(buf)
	for i := 0; i < b.N; i++ {
		farm.Fingerprint64(buf)
	}
}

func BenchmarkFnv(b *testing.B) {
	buf := make([]byte, 64)
	rand.Read(buf)
	f := fnv.New64a()
	for i := 0; i < b.N; i++ {
		f.Write(buf)
		f.Sum64()
		f.Reset()
	}
}

func SipHash(p []byte) (l, h uint64) {
	// Initialization.
	v0 := uint64(8317987320269560794) // k0 ^ 0x736f6d6570736575
	v1 := uint64(7237128889637516672) // k1 ^ 0x646f72616e646f6d
	v2 := uint64(7816392314733513934) // k0 ^ 0x6c7967656e657261
	v3 := uint64(8387220255325274014) // k1 ^ 0x7465646279746573
	t := uint64(len(p)) << 56

	// Compression.
	for len(p) >= 8 {
		m := uint64(p[0]) | uint64(p[1])<<8 | uint64(p[2])<<16 | uint64(p[3])<<24 |
			uint64(p[4])<<32 | uint64(p[5])<<40 | uint64(p[6])<<48 | uint64(p[7])<<56

		v3 ^= m

		// Round 1.
		v0 += v1
		v1 = v1<<13 | v1>>51
		v1 ^= v0
		v0 = v0<<32 | v0>>32

		v2 += v3
		v3 = v3<<16 | v3>>48
		v3 ^= v2

		v0 += v3
		v3 = v3<<21 | v3>>43
		v3 ^= v0

		v2 += v1
		v1 = v1<<17 | v1>>47
		v1 ^= v2
		v2 = v2<<32 | v2>>32

		// Round 2.
		v0 += v1
		v1 = v1<<13 | v1>>51
		v1 ^= v0
		v0 = v0<<32 | v0>>32

		v2 += v3
		v3 = v3<<16 | v3>>48
		v3 ^= v2

		v0 += v3
		v3 = v3<<21 | v3>>43
		v3 ^= v0

		v2 += v1
		v1 = v1<<17 | v1>>47
		v1 ^= v2
		v2 = v2<<32 | v2>>32

		v0 ^= m
		p = p[8:]
	}

	// Compress last block.
	switch len(p) {
	case 7:
		t |= uint64(p[6]) << 48
		fallthrough
	case 6:
		t |= uint64(p[5]) << 40
		fallthrough
	case 5:
		t |= uint64(p[4]) << 32
		fallthrough
	case 4:
		t |= uint64(p[3]) << 24
		fallthrough
	case 3:
		t |= uint64(p[2]) << 16
		fallthrough
	case 2:
		t |= uint64(p[1]) << 8
		fallthrough
	case 1:
		t |= uint64(p[0])
	}

	v3 ^= t

	// Round 1.
	v0 += v1
	v1 = v1<<13 | v1>>51
	v1 ^= v0
	v0 = v0<<32 | v0>>32

	v2 += v3
	v3 = v3<<16 | v3>>48
	v3 ^= v2

	v0 += v3
	v3 = v3<<21 | v3>>43
	v3 ^= v0

	v2 += v1
	v1 = v1<<17 | v1>>47
	v1 ^= v2
	v2 = v2<<32 | v2>>32

	// Round 2.
	v0 += v1
	v1 = v1<<13 | v1>>51
	v1 ^= v0
	v0 = v0<<32 | v0>>32

	v2 += v3
	v3 = v3<<16 | v3>>48
	v3 ^= v2

	v0 += v3
	v3 = v3<<21 | v3>>43
	v3 ^= v0

	v2 += v1
	v1 = v1<<17 | v1>>47
	v1 ^= v2
	v2 = v2<<32 | v2>>32

	v0 ^= t

	// Finalization.
	v2 ^= 0xff

	// Round 1.
	v0 += v1
	v1 = v1<<13 | v1>>51
	v1 ^= v0
	v0 = v0<<32 | v0>>32

	v2 += v3
	v3 = v3<<16 | v3>>48
	v3 ^= v2

	v0 += v3
	v3 = v3<<21 | v3>>43
	v3 ^= v0

	v2 += v1
	v1 = v1<<17 | v1>>47
	v1 ^= v2
	v2 = v2<<32 | v2>>32

	// Round 2.
	v0 += v1
	v1 = v1<<13 | v1>>51
	v1 ^= v0
	v0 = v0<<32 | v0>>32

	v2 += v3
	v3 = v3<<16 | v3>>48
	v3 ^= v2

	v0 += v3
	v3 = v3<<21 | v3>>43
	v3 ^= v0

	v2 += v1
	v1 = v1<<17 | v1>>47
	v1 ^= v2
	v2 = v2<<32 | v2>>32

	// Round 3.
	v0 += v1
	v1 = v1<<13 | v1>>51
	v1 ^= v0
	v0 = v0<<32 | v0>>32

	v2 += v3
	v3 = v3<<16 | v3>>48
	v3 ^= v2

	v0 += v3
	v3 = v3<<21 | v3>>43
	v3 ^= v0

	v2 += v1
	v1 = v1<<17 | v1>>47
	v1 ^= v2
	v2 = v2<<32 | v2>>32

	// Round 4.
	v0 += v1
	v1 = v1<<13 | v1>>51
	v1 ^= v0
	v0 = v0<<32 | v0>>32

	v2 += v3
	v3 = v3<<16 | v3>>48
	v3 ^= v2

	v0 += v3
	v3 = v3<<21 | v3>>43
	v3 ^= v0

	v2 += v1
	v1 = v1<<17 | v1>>47
	v1 ^= v2
	v2 = v2<<32 | v2>>32

	// return v0 ^ v1 ^ v2 ^ v3

	hash := v0 ^ v1 ^ v2 ^ v3
	h = hash >> 1
	l = hash << 1 >> 1
	return l, h
}

func BenchmarkNanoTime(b *testing.B) {
	for i := 0; i < b.N; i++ {
		NanoTime()
	}
}

func BenchmarkCPUTicks(b *testing.B) {
	for i := 0; i < b.N; i++ {
		CPUTicks()
	}
}

func BenchmarkFastRand(b *testing.B) {
	for i := 0; i < b.N; i++ {
		FastRand()
	}
}
<commit_msg>Add benchmarks for math.rand and fastrand (#118)<commit_after>package z

import (
	"hash/fnv"
	"math/rand"
	"sync/atomic"
	"testing"
	"time"

	"github.com/dgryski/go-farm"
)

func BenchmarkMemHash(b *testing.B) {
	buf := make([]byte, 64)
	rand.Read(buf)
	for i := 0; i < b.N; i++ {
		MemHash(buf)
	}
}

func BenchmarkSip(b *testing.B) {
	buf := make([]byte, 64)
	rand.Read(buf)
	for i := 0; i < b.N; i++ {
		SipHash(buf)
	}
}

func BenchmarkFarm(b *testing.B) {
	buf := make([]byte, 64)
	rand.Read(buf)
	for i := 0; i < b.N; i++ {
		farm.Fingerprint64(buf)
	}
}

func BenchmarkFnv(b *testing.B) {
	buf := make([]byte, 64)
	rand.Read(buf)
	f := fnv.New64a()
	for i := 0; i < b.N; i++ {
		f.Write(buf)
		f.Sum64()
		f.Reset()
	}
}

func SipHash(p []byte) (l, h uint64) {
	// Initialization.
	v0 := uint64(8317987320269560794) // k0 ^ 0x736f6d6570736575
	v1 := uint64(7237128889637516672) // k1 ^ 0x646f72616e646f6d
	v2 := uint64(7816392314733513934) // k0 ^ 0x6c7967656e657261
	v3 := uint64(8387220255325274014) // k1 ^ 0x7465646279746573
	t := uint64(len(p)) << 56

	// Compression.
	for len(p) >= 8 {
		m := uint64(p[0]) | uint64(p[1])<<8 | uint64(p[2])<<16 | uint64(p[3])<<24 |
			uint64(p[4])<<32 | uint64(p[5])<<40 | uint64(p[6])<<48 | uint64(p[7])<<56

		v3 ^= m

		// Round 1.
		v0 += v1
		v1 = v1<<13 | v1>>51
		v1 ^= v0
		v0 = v0<<32 | v0>>32

		v2 += v3
		v3 = v3<<16 | v3>>48
		v3 ^= v2

		v0 += v3
		v3 = v3<<21 | v3>>43
		v3 ^= v0

		v2 += v1
		v1 = v1<<17 | v1>>47
		v1 ^= v2
		v2 = v2<<32 | v2>>32

		// Round 2.
		v0 += v1
		v1 = v1<<13 | v1>>51
		v1 ^= v0
		v0 = v0<<32 | v0>>32

		v2 += v3
		v3 = v3<<16 | v3>>48
		v3 ^= v2

		v0 += v3
		v3 = v3<<21 | v3>>43
		v3 ^= v0

		v2 += v1
		v1 = v1<<17 | v1>>47
		v1 ^= v2
		v2 = v2<<32 | v2>>32

		v0 ^= m
		p = p[8:]
	}

	// Compress last block.
	switch len(p) {
	case 7:
		t |= uint64(p[6]) << 48
		fallthrough
	case 6:
		t |= uint64(p[5]) << 40
		fallthrough
	case 5:
		t |= uint64(p[4]) << 32
		fallthrough
	case 4:
		t |= uint64(p[3]) << 24
		fallthrough
	case 3:
		t |= uint64(p[2]) << 16
		fallthrough
	case 2:
		t |= uint64(p[1]) << 8
		fallthrough
	case 1:
		t |= uint64(p[0])
	}

	v3 ^= t

	// Round 1.
	v0 += v1
	v1 = v1<<13 | v1>>51
	v1 ^= v0
	v0 = v0<<32 | v0>>32

	v2 += v3
	v3 = v3<<16 | v3>>48
	v3 ^= v2

	v0 += v3
	v3 = v3<<21 | v3>>43
	v3 ^= v0

	v2 += v1
	v1 = v1<<17 | v1>>47
	v1 ^= v2
	v2 = v2<<32 | v2>>32

	// Round 2.
	v0 += v1
	v1 = v1<<13 | v1>>51
	v1 ^= v0
	v0 = v0<<32 | v0>>32

	v2 += v3
	v3 = v3<<16 | v3>>48
	v3 ^= v2

	v0 += v3
	v3 = v3<<21 | v3>>43
	v3 ^= v0

	v2 += v1
	v1 = v1<<17 | v1>>47
	v1 ^= v2
	v2 = v2<<32 | v2>>32

	v0 ^= t

	// Finalization.
	v2 ^= 0xff

	// Round 1.
	v0 += v1
	v1 = v1<<13 | v1>>51
	v1 ^= v0
	v0 = v0<<32 | v0>>32

	v2 += v3
	v3 = v3<<16 | v3>>48
	v3 ^= v2

	v0 += v3
	v3 = v3<<21 | v3>>43
	v3 ^= v0

	v2 += v1
	v1 = v1<<17 | v1>>47
	v1 ^= v2
	v2 = v2<<32 | v2>>32

	// Round 2.
	v0 += v1
	v1 = v1<<13 | v1>>51
	v1 ^= v0
	v0 = v0<<32 | v0>>32

	v2 += v3
	v3 = v3<<16 | v3>>48
	v3 ^= v2

	v0 += v3
	v3 = v3<<21 | v3>>43
	v3 ^= v0

	v2 += v1
	v1 = v1<<17 | v1>>47
	v1 ^= v2
	v2 = v2<<32 | v2>>32

	// Round 3.
	v0 += v1
	v1 = v1<<13 | v1>>51
	v1 ^= v0
	v0 = v0<<32 | v0>>32

	v2 += v3
	v3 = v3<<16 | v3>>48
	v3 ^= v2

	v0 += v3
	v3 = v3<<21 | v3>>43
	v3 ^= v0

	v2 += v1
	v1 = v1<<17 | v1>>47
	v1 ^= v2
	v2 = v2<<32 | v2>>32

	// Round 4.
	v0 += v1
	v1 = v1<<13 | v1>>51
	v1 ^= v0
	v0 = v0<<32 | v0>>32

	v2 += v3
	v3 = v3<<16 | v3>>48
	v3 ^= v2

	v0 += v3
	v3 = v3<<21 | v3>>43
	v3 ^= v0

	v2 += v1
	v1 = v1<<17 | v1>>47
	v1 ^= v2
	v2 = v2<<32 | v2>>32

	// return v0 ^ v1 ^ v2 ^ v3

	hash := v0 ^ v1 ^ v2 ^ v3
	h = hash >> 1
	l = hash << 1 >> 1
	return l, h
}

func BenchmarkNanoTime(b *testing.B) {
	for i := 0; i < b.N; i++ {
		NanoTime()
	}
}

func BenchmarkCPUTicks(b *testing.B) {
	for i := 0; i < b.N; i++ {
		CPUTicks()
	}
}

// goos: linux
// goarch: amd64
// pkg: github.com/dgraph-io/ristretto/z
// BenchmarkFastRand-16      	1000000000	         0.292 ns/op
// BenchmarkRandSource-16    	1000000000	         0.747 ns/op
// BenchmarkRandGlobal-16    	 6822332	       176 ns/op
// BenchmarkRandAtomic-16    	77950322	        15.4 ns/op
// PASS
// ok  	github.com/dgraph-io/ristretto/z	4.808s
func benchmarkRand(b *testing.B, fab func() func() uint32) {
	b.RunParallel(func(pb *testing.PB) {
		gen := fab()
		for pb.Next() {
			gen()
		}
	})
}

func BenchmarkFastRand(b *testing.B) {
	benchmarkRand(b, func() func() uint32 {
		return FastRand
	})
}

func BenchmarkRandSource(b *testing.B) {
	benchmarkRand(b, func() func() uint32 {
		s := rand.New(rand.NewSource(time.Now().Unix()))
		return func() uint32 { return s.Uint32() }
	})
}

func BenchmarkRandGlobal(b *testing.B) {
	benchmarkRand(b, func() func() uint32 {
		return func() uint32 { return rand.Uint32() }
	})
}

func BenchmarkRandAtomic(b *testing.B) {
	var x uint32
	benchmarkRand(b, func() func() uint32 {
		return func() uint32 { return uint32(atomic.AddUint32(&x, 1)) }
	})
}
<|endoftext|>


<commit_before>// -*- tab-width: 4; -*-

package main

import (
	"bufio"
	"fmt"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"
	"unicode"
)

type Tweeter struct {
	Nick string
	URL  string
}
type Tweet struct {
	Tweeter Tweeter
	Created time.Time
	Text    string
}

// typedef to be able to attach sort methods
type Tweets []Tweet

func (tweets Tweets) Len() int {
	return len(tweets)
}
func (tweets Tweets) Less(i, j int) bool {
	return tweets[i].Created.Before(tweets[j].Created)
}
func (tweets Tweets) Swap(i, j int) {
	tweets[i], tweets[j] = tweets[j], tweets[i]
}

const maxfetchers = 50

func get_tweets(cache Cache, sources map[string]string) Tweets {
	var mu sync.RWMutex

	tweetsch := make(chan Tweets)
	var wg sync.WaitGroup
	// max parallel http fetchers
	var fetchers = make(chan struct{}, maxfetchers)

	for nick, url := range sources {
		wg.Add(1)

		fetchers <- struct{}{}

		// anon func takes needed variables as arg, avoiding capture of iterator variables
		go func(nick string, url string) {
			defer wg.Done()

			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s: http.NewRequest fail: %s", url, err)
				tweetsch <- nil
				return
			}

			if conf.Nick != "" && conf.Twturl != "" {
				// TODO: version goes here
				req.Header.Set("User-Agent",
					fmt.Sprintf("%s/0.1 (+%s; @%s)", progname, conf.Twturl, conf.Nick))
			}

			mu.RLock()
			if cached, ok := cache[url]; ok {
				if cached.Lastmodified != "" {
					req.Header.Set("If-Modified-Since", cached.Lastmodified)
				}
			}
			mu.RUnlock()

			client := http.Client{}
			resp, err := client.Do(req)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s: client.Do fail: %s", url, err)
				tweetsch <- nil
				return
			}
			defer resp.Body.Close()

			actualurl := resp.Request.URL.String()
			if actualurl != url {
				url = actualurl
			}

			var tweets Tweets

			switch resp.StatusCode {
			case http.StatusOK: // 200
				scanner := bufio.NewScanner(resp.Body)
				tweets = parse_file(scanner, Tweeter{Nick: nick, URL: url})
				lastmodified := resp.Header.Get("Last-Modified")
				mu.Lock()
				cache[url] = Cached{Tweets: tweets, Lastmodified: lastmodified}
				mu.Unlock()
			case http.StatusNotModified: // 304
				mu.RLock()
				tweets = cache[url].Tweets
				mu.RUnlock()
			}

			tweetsch <- tweets

		}(nick, url)

		<-fetchers
	}

	// close tweets channel when all goroutines are done
	go func() {
		wg.Wait()
		close(tweetsch)
	}()

	fmt.Fprintf(os.Stderr, "fetching: ")
	var alltweets Tweets
	var n = 0
	// loop until channel closed
	for tweets := range tweetsch {
		n++
		fmt.Fprintf(os.Stderr, "%d ", len(sources)+1-n)
		alltweets = append(alltweets, tweets...)
	}
	fmt.Fprintf(os.Stderr, "\n")

	return alltweets
}

func parse_file(scanner *bufio.Scanner, tweeter Tweeter) Tweets {
	var tweets Tweets
	for scanner.Scan() {
		line := strings.TrimRightFunc(scanner.Text(),
			func(r rune) bool {
				return unicode.IsSpace(r)
			})
		if len(line) == 0 {
			continue
		}
		parts := strings.SplitN(line, "\t", 2)
		if len(parts) != 2 {
			fmt.Fprintf(os.Stderr, "could not parse: '%s' (source:%s)\n", line, tweeter.URL)
			continue
		}
		tweets = append(tweets,
			Tweet{
				Tweeter: tweeter,
				Created: parsetime(parts[0]),
				Text:    parts[1],
			})
	}
	return tweets
}

func parsetime(timestr string) time.Time {
	var tm time.Time
	var err error
	// Twtxt clients generally uses basically time.RFC3339Nano, but sometimes
	// there's a colon in the timezone, or no timezone at all.
	for _, layout := range []string{
		"2006-01-02T15:04:05.999999999Z07:00",
		"2006-01-02T15:04:05.999999999Z0700",
		"2006-01-02T15:04:05.999999999",
	} {
		tm, err = time.Parse(layout, timestr)
		if err != nil {
			continue
		} else {
			break
		}
	}
	if err != nil {
		return time.Unix(0, 0)
	}
	return tm.Round(time.Second)
}
<commit_msg>Appropriate to check for errors after Scan()ing (EOF not an error!)<commit_after>// -*- tab-width: 4; -*-

package main

import (
	"bufio"
	"fmt"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"
	"unicode"
)

type Tweeter struct {
	Nick string
	URL  string
}
type Tweet struct {
	Tweeter Tweeter
	Created time.Time
	Text    string
}

// typedef to be able to attach sort methods
type Tweets []Tweet

func (tweets Tweets) Len() int {
	return len(tweets)
}
func (tweets Tweets) Less(i, j int) bool {
	return tweets[i].Created.Before(tweets[j].Created)
}
func (tweets Tweets) Swap(i, j int) {
	tweets[i], tweets[j] = tweets[j], tweets[i]
}

const maxfetchers = 50

func get_tweets(cache Cache, sources map[string]string) Tweets {
	var mu sync.RWMutex

	tweetsch := make(chan Tweets)
	var wg sync.WaitGroup
	// max parallel http fetchers
	var fetchers = make(chan struct{}, maxfetchers)

	for nick, url := range sources {
		wg.Add(1)

		fetchers <- struct{}{}

		// anon func takes needed variables as arg, avoiding capture of iterator variables
		go func(nick string, url string) {
			defer wg.Done()

			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s: http.NewRequest fail: %s", url, err)
				tweetsch <- nil
				return
			}

			if conf.Nick != "" && conf.Twturl != "" {
				// TODO: version goes here
				req.Header.Set("User-Agent",
					fmt.Sprintf("%s/0.1 (+%s; @%s)", progname, conf.Twturl, conf.Nick))
			}

			mu.RLock()
			if cached, ok := cache[url]; ok {
				if cached.Lastmodified != "" {
					req.Header.Set("If-Modified-Since", cached.Lastmodified)
				}
			}
			mu.RUnlock()

			client := http.Client{}
			resp, err := client.Do(req)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s: client.Do fail: %s", url, err)
				tweetsch <- nil
				return
			}
			defer resp.Body.Close()

			actualurl := resp.Request.URL.String()
			if actualurl != url {
				url = actualurl
			}

			var tweets Tweets

			switch resp.StatusCode {
			case http.StatusOK: // 200
				scanner := bufio.NewScanner(resp.Body)
				tweets = parse_file(scanner, Tweeter{Nick: nick, URL: url})
				lastmodified := resp.Header.Get("Last-Modified")
				mu.Lock()
				cache[url] = Cached{Tweets: tweets, Lastmodified: lastmodified}
				mu.Unlock()
			case http.StatusNotModified: // 304
				mu.RLock()
				tweets = cache[url].Tweets
				mu.RUnlock()
			}

			tweetsch <- tweets

		}(nick, url)

		<-fetchers
	}

	// close tweets channel when all goroutines are done
	go func() {
		wg.Wait()
		close(tweetsch)
	}()

	fmt.Fprintf(os.Stderr, "fetching: ")
	var alltweets Tweets
	var n = 0
	// loop until channel closed
	for tweets := range tweetsch {
		n++
		fmt.Fprintf(os.Stderr, "%d ", len(sources)+1-n)
		alltweets = append(alltweets, tweets...)
	}
	fmt.Fprintf(os.Stderr, "\n")

	return alltweets
}

func parse_file(scanner *bufio.Scanner, tweeter Tweeter) Tweets {
	var tweets Tweets
	for scanner.Scan() {
		line := strings.TrimRightFunc(scanner.Text(),
			func(r rune) bool {
				return unicode.IsSpace(r)
			})
		if len(line) == 0 {
			continue
		}
		parts := strings.SplitN(line, "\t", 2)
		if len(parts) != 2 {
			fmt.Fprintf(os.Stderr, "could not parse: '%s' (source:%s)\n", line, tweeter.URL)
			continue
		}
		tweets = append(tweets,
			Tweet{
				Tweeter: tweeter,
				Created: parsetime(parts[0]),
				Text:    parts[1],
			})
	}
	if err := scanner.Err(); err != nil {
		panic(err)
	}
	return tweets
}

func parsetime(timestr string) time.Time {
	var tm time.Time
	var err error
	// Twtxt clients generally uses basically time.RFC3339Nano, but sometimes
	// there's a colon in the timezone, or no timezone at all.
	for _, layout := range []string{
		"2006-01-02T15:04:05.999999999Z07:00",
		"2006-01-02T15:04:05.999999999Z0700",
		"2006-01-02T15:04:05.999999999",
	} {
		tm, err = time.Parse(layout, timestr)
		if err != nil {
			continue
		} else {
			break
		}
	}
	if err != nil {
		return time.Unix(0, 0)
	}
	return tm.Round(time.Second)
}
<|endoftext|>


<commit_before>// Copyright 2012 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"../.." // Use "github.com/kurrik/twodee"
	"flag"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"runtime/pprof"
	"time"
)

func PrintError(err error) {
	fmt.Printf("[error]: %v\n", err)
}

func init() {
	// See https://code.google.com/p/go/issues/detail?id=3527
	runtime.LockOSThread()
}

func main() {
	var (
		system     *twodee.System
		window     *twodee.Window
		font       *twodee.Font
		err        error
		cpuprofile *string
		memprofile *string
		webprofile *bool
	)
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	memprofile = flag.String("memprofile", "", "write memory profile to this file")
	webprofile = flag.Bool("webprofile", false, "profile with web service")
	flag.Parse()

	if *memprofile != "" || *webprofile == true {
		runtime.MemProfileRate = 1
	}
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	if *webprofile == true {
		go func() {
			log.Println(http.ListenAndServe("localhost:6060", nil))
		}()
	}
	if system, err = twodee.Init(); err != nil {
		PrintError(err)
		os.Exit(1)
	}
	defer system.Terminate()

	camera := twodee.NewCamera(0, 0, 16, 16)
	system.SetSizeCallback(func(w, h int) {
		camera.MatchRatio(w, h)
		camera.Top(0)
	})
	window = &twodee.Window{Width: 640, Height: 480, Scale: 2}
	if err = system.Open(window); err != nil {
		PrintError(err)
		os.Exit(1)
	}

	textures := map[string]string{
		"bricks": "examples/basic/texture.png",
	}
	for name, path := range textures {
		if err = system.LoadTexture(name, path, twodee.IntNearest, 8); err != nil {
			PrintError(err)
			os.Exit(1)
		}
	}
	if font, err = twodee.LoadFont("examples/basic/slkscr.ttf", 24); err != nil {
		PrintError(err)
		os.Exit(1)
	}
	scene := &twodee.Scene{Camera: camera, Font: font}
	parent := system.NewSprite("bricks", 0, 0, 1, 1, 4)
	parent.AddChild(system.NewSprite("bricks", 1, 1, 1, 1, 4))
	scene.AddChild(parent)
	parent.SetFrame(1)
	exit := make(chan bool, 1)
	system.SetKeyCallback(func(key int, state int) {
		switch {
		case key == twodee.KeyEsc:
			exit <- true
		default:
			fmt.Printf("Key: %v, State: %v\n", key, state)
		}
	})
	system.SetCloseCallback(func() int {
		exit <- true
		return 0
	})
	system.SetScrollCallback(func(pos int) {
		fmt.Printf("Scroll: %v\n", pos)
		camera.Zoom(float64(pos) / 50.0)
		camera.Top(0)
	})
	v := twodee.Pt(0.1, 0.05)
	go func() {
		ticker := time.Tick(time.Second / 120.0)
		for true {
			<-ticker
			b := parent.GlobalBounds()
			if b.Max.X > 10 || b.Min.X < 0 {
				v = twodee.Pt(-v.X, v.Y)
			} else if b.Max.Y > 10 || b.Min.Y < 0 {
				v = twodee.Pt(v.X, -v.Y)
			}
			parent.Move(v)
		}
	}()
	ticker := time.NewTicker(time.Second / 60)
	run := true
	for run == true {
		system.Paint(scene)
		select {
		case <-exit:
			ticker.Stop()
			run = false
		default:
		}
		<-ticker.C
	}
	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.WriteHeapProfile(f)
		f.Close()
		return
	}
}
<commit_msg>Tweaking always tweaking.<commit_after>// Copyright 2012 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"../.." // Use "github.com/kurrik/twodee"
	"flag"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"runtime"
	"runtime/pprof"
	"time"
)

func PrintError(err error) {
	fmt.Printf("[error]: %v\n", err)
}

func init() {
	// See https://code.google.com/p/go/issues/detail?id=3527
	runtime.LockOSThread()
}

func main() {
	var (
		system     *twodee.System
		window     *twodee.Window
		font       *twodee.Font
		err        error
		cpuprofile *string
		memprofile *string
		webprofile *bool
	)
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	memprofile = flag.String("memprofile", "", "write memory profile to this file")
	webprofile = flag.Bool("webprofile", false, "profile with web service")
	flag.Parse()

	if *memprofile != "" || *webprofile == true {
		runtime.MemProfileRate = 1
	}
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	if *webprofile == true {
		go func() {
			log.Println(http.ListenAndServe("localhost:6060", nil))
		}()
	}
	if system, err = twodee.Init(); err != nil {
		PrintError(err)
		os.Exit(1)
	}
	defer system.Terminate()

	camera := twodee.NewCamera(0, 0, 16, 16)
	system.SetSizeCallback(func(w, h int) {
		camera.MatchRatio(w, h)
		camera.Top(0)
	})
	window = &twodee.Window{Width: 640, Height: 480, Scale: 2}
	if err = system.Open(window); err != nil {
		PrintError(err)
		os.Exit(1)
	}

	textures := map[string]string{
		"bricks": "examples/basic/texture.png",
	}
	for name, path := range textures {
		if err = system.LoadTexture(name, path, twodee.IntNearest, 8); err != nil {
			PrintError(err)
			os.Exit(1)
		}
	}
	if font, err = twodee.LoadFont("examples/basic/slkscr.ttf", 24); err != nil {
		PrintError(err)
		os.Exit(1)
	}
	scene := &twodee.Scene{Camera: camera, Font: font}
	parent := system.NewSprite("bricks", 0, 0, 1, 1, 4)
	parent.AddChild(system.NewSprite("bricks", 1, 1, 1, 1, 4))
	scene.AddChild(parent)
	parent.SetFrame(1)
	exit := make(chan bool, 1)
	system.SetKeyCallback(func(key int, state int) {
		switch {
		case key == twodee.KeyEsc:
			exit <- true
		default:
			fmt.Printf("Key: %v, State: %v\n", key, state)
		}
	})
	system.SetCloseCallback(func() int {
		exit <- true
		return 0
	})
	system.SetScrollCallback(func(pos int) {
		fmt.Printf("Scroll: %v\n", pos)
		camera.Zoom(float64(pos) / 50.0)
		camera.Top(0)
	})
	v := twodee.Pt(0.1, 0.05)
	go func() {
		ticker := time.Tick(time.Second / 120.0)
		for true {
			<-ticker
			b := parent.GlobalBounds()
			if b.Max.X > 10 || b.Min.X < 0 {
				v = twodee.Pt(-v.X, v.Y)
			} else if b.Max.Y > 10 || b.Min.Y < 0 {
				v = twodee.Pt(v.X, -v.Y)
			}
			parent.Move(v)
		}
	}()
	ticker := time.NewTicker(time.Second / 60)
	run := true
	for run == true {
		<-ticker.C
		system.Paint(scene)
		select {
		case <-exit:
			ticker.Stop()
			run = false
		default:
		}
	}
	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.WriteHeapProfile(f)
		f.Close()
		return
	}
}
<|endoftext|>


<commit_before>package gnosis

import (
	"log"
	"strings"
	"io/ioutil"
	"os"
	"time"
	//"crypto/md5"
	"path"
	//"fmt"

	"github.com/JackKnifed/blackfriday"
"github.com/mschoch/blackfriday-text"
	"github.com/blevesearch/bleve"
	"gopkg.in/fsnotify.v1"
)

type gnosisIndex struct {
	TrueIndex bleve.Index
	Config IndexSection
}

type indexedPage struct {
	Name               string    `json:"name"`
	Filepath string `json:"path"`
	Body               string    `json:"body"`
	Topics string `json:"topic"`
	Keywords string `json:"keyword"`
	Modified           time.Time `json:"modified"`
}

func openIndex(config IndexSection) *gnosisIndex {
	index := new(gnosisIndex)
	index.Config = config
	newIndex, err := bleve.Open(path.Clean(index.Config.IndexPath))
	if err == nil {
		log.Printf("Opening existing index...")
		index.TrueIndex = newIndex
	} else if err == bleve.ErrorIndexPathDoesNotExist {
		log.Printf("Creating new index...")
		// create a mapping
		indexMapping := index.buildIndexMapping()
		newIndex, err := bleve.New(path.Clean(index.Config.IndexPath), indexMapping)
		if err != nil {
			log.Fatal(err)
		} else {
			index.TrueIndex = newIndex
		}
	} else {
		log.Fatal(err)
	}
	return index
}

func (index *gnosisIndex) buildIndexMapping() *bleve.IndexMapping {

	// create a text field type
	enTextFieldMapping := bleve.NewTextFieldMapping()
	enTextFieldMapping.Analyzer = index.Config.IndexType

	// create a date field type
	dateTimeMapping := bleve.NewDateTimeFieldMapping()

	// map out the wiki page
	wikiMapping := bleve.NewDocumentMapping()
	wikiMapping.AddFieldMappingsAt("path", enTextFieldMapping)
	wikiMapping.AddFieldMappingsAt("body", enTextFieldMapping)
	wikiMapping.AddFieldMappingsAt("topic", enTextFieldMapping)
	wikiMapping.AddFieldMappingsAt("keyword", enTextFieldMapping)
	wikiMapping.AddFieldMappingsAt("modified", dateTimeMapping)

	// add the wiki page mapping to a new index
	// ##TODO## revisit? move out to config?
	indexMapping := bleve.NewIndexMapping()
	indexMapping.AddDocumentMapping("wiki", wikiMapping)
	indexMapping.DefaultAnalyzer = index.Config.IndexType

	return indexMapping
}

func (index *gnosisIndex) cleanupMarkdown(input []byte) []byte {
	extensions := 0
	renderer := blackfridaytext.TextRenderer()
	output := blackfriday.Markdown(input, renderer, extensions)
	return output
}

func (index *gnosisIndex) relativePath(filePath string) string {
	filePath = strings.TrimPrefix(filePath, index.Config.WatchDir)
	filePath = path.Clean(filePath)
	return filePath
}

func (index *gnosisIndex) generateWikiFromFile(filePath string) (*indexedPage, error) {
	fileBytes, err := ioutil.ReadFile(filePath)
	if err != nil {
		return nil, err
	}

	// ##TODO## I need to look up the actual index that I'm building and hit all of the fields here
	cleanedUpBytes := index.cleanupMarkdown(fileBytes)
	name := path.Base(filePath)
	name = strings.TrimSuffix(name, ".md")
	rv := indexedPage{
		Name: name,
		Body: string(cleanedUpBytes),
	}
	return &rv, nil
}

func (index *gnosisIndex) processUpdate(path string) {
	log.Printf("updated: %s", path)
	rp := index.relativePath(path)
	wiki, err := index.generateWikiFromFile(path)
	if err != nil {
		log.Print(err)
	} else {
		index.TrueIndex.Index(rp, wiki)
	}
}

func (index *gnosisIndex) processDelete(path string) {
	log.Printf("delete: %s", path)
	rp := index.relativePath(path)
	err := index.TrueIndex.Delete(rp)
	if err != nil {
		log.Print(err)
	}
}

func (index *gnosisIndex) walkForIndexing(path string) {
	// ##TODO## reevaulate by finding all files within a path?
	dirEntries, err := ioutil.ReadDir(path)
	if err != nil {
		log.Fatal(err)
	}
	for _, dirEntry := range dirEntries {
		dirEntryPath := path + string(os.PathSeparator) + dirEntry.Name()
		if dirEntry.IsDir() {
			index.walkForIndexing(dirEntryPath)
		} else if strings.HasSuffix(dirEntry.Name(), ".md") {
			index.processUpdate(dirEntryPath)
		}
	}
}

func (index *gnosisIndex) startWatching(filePath string) *fsnotify.Watcher {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	// maybe rework the index so the Watcher is inside the index? idk

	// start a go routine to process events
	go func() {
		idleTimer := time.NewTimer(10 * time.Second)
		queuedEvents := make([]fsnotify.Event, 0)
		for {
			select {
			case ev := <-watcher.Events:
				queuedEvents = append(queuedEvents, ev)
				idleTimer.Reset(10 * time.Second)
			case err := <-watcher.Errors:
				log.Fatal(err)
			case <-idleTimer.C:
				for _, ev := range queuedEvents {
					if strings.HasSuffix(ev.Name, ".md") {
						switch ev.Op {
						case fsnotify.Remove, fsnotify.Rename:
							// delete the filePath
							index.processDelete(ev.Name)
						case fsnotify.Create, fsnotify.Write:
							// update the filePath
							index.processUpdate(ev.Name)
						default:
							// ignore
						}
					}
				}
				queuedEvents = make([]fsnotify.Event, 0)
				idleTimer.Reset(10 * time.Second)
			}
		}
	}()

	// now actually watch the filePath requested
	err = watcher.Add(filePath)
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("watching '%s' for changes...", filePath)

	return watcher
}
<commit_msg>did some package cleanup<commit_after>package gnosis

import (
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"
	"time"

	"github.com/JackKnifed/blackfriday"
	"github.com/blevesearch/bleve"
)

type gnosisIndex struct {
	TrueIndex bleve.Index
	Config    IndexSection
}

type indexedPage struct {
	Name     string    `json:"name"`
	Filepath string    `json:"path"`
	Body     string    `json:"body"`
	Topics   string    `json:"topic"`
	Keywords string    `json:"keyword"`
	Modified time.Time `json:"modified"`
}

func openIndex(config IndexSection) *gnosisIndex {
	index := new(gnosisIndex)
	index.Config = config
	newIndex, err := bleve.Open(path.Clean(index.Config.IndexPath))
	if err == nil {
		log.Printf("Opening existing index...")
		index.TrueIndex = newIndex
	} else if err == bleve.ErrorIndexPathDoesNotExist {
		log.Printf("Creating new index...")
		// create a mapping
		indexMapping := index.buildIndexMapping()
		newIndex, err := bleve.New(path.Clean(index.Config.IndexPath), indexMapping)
		if err != nil {
			log.Fatal(err)
		} else {
			index.TrueIndex = newIndex
		}
	} else {
		log.Fatal(err)
	}
	return index
}

func (index *gnosisIndex) buildIndexMapping() *bleve.IndexMapping {

	// create a text field type
	enTextFieldMapping := bleve.NewTextFieldMapping()
	enTextFieldMapping.Analyzer = index.Config.IndexType

	// create a date field type
	dateTimeMapping := bleve.NewDateTimeFieldMapping()

	// map out the wiki page
	wikiMapping := bleve.NewDocumentMapping()
	wikiMapping.AddFieldMappingsAt("path", enTextFieldMapping)
	wikiMapping.AddFieldMappingsAt("body", enTextFieldMapping)
	wikiMapping.AddFieldMappingsAt("topic", enTextFieldMapping)
	wikiMapping.AddFieldMappingsAt("keyword", enTextFieldMapping)
	wikiMapping.AddFieldMappingsAt("modified", dateTimeMapping)

	// add the wiki page mapping to a new index
	// ##TODO## revisit? move out to config?
	indexMapping := bleve.NewIndexMapping()
	indexMapping.AddDocumentMapping("wiki", wikiMapping)
	indexMapping.DefaultAnalyzer = index.Config.IndexType

	return indexMapping
}

func (index *gnosisIndex) cleanupMarkdown(input []byte) []byte {
	extensions := 0
	renderer := blackfridaytext.TextRenderer()
	output := blackfriday.Markdown(input, renderer, extensions)
	return output
}

func (index *gnosisIndex) relativePath(filePath string) string {
	filePath = strings.TrimPrefix(filePath, index.Config.WatchDir)
	filePath = path.Clean(filePath)
	return filePath
}

func (index *gnosisIndex) generateWikiFromFile(filePath string) (*indexedPage, error) {
	fileBytes, err := ioutil.ReadFile(filePath)
	if err != nil {
		return nil, err
	}

	// ##TODO## I need to look up the actual index that I'm building and hit all of the fields here
	cleanedUpBytes := index.cleanupMarkdown(fileBytes)
	name := path.Base(filePath)
	name = strings.TrimSuffix(name, ".md")
	rv := indexedPage{
		Name: name,
		Body: string(cleanedUpBytes),
	}
	return &rv, nil
}

func (index *gnosisIndex) processUpdate(path string) {
	log.Printf("updated: %s", path)
	rp := index.relativePath(path)
	wiki, err := index.generateWikiFromFile(path)
	if err != nil {
		log.Print(err)
	} else {
		index.TrueIndex.Index(rp, wiki)
	}
}

func (index *gnosisIndex) processDelete(path string) {
	log.Printf("delete: %s", path)
	rp := index.relativePath(path)
	err := index.TrueIndex.Delete(rp)
	if err != nil {
		log.Print(err)
	}
}

func (index *gnosisIndex) walkForIndexing(path string) {
	// ##TODO## reevaulate by finding all files within a path?
	dirEntries, err := ioutil.ReadDir(path)
	if err != nil {
		log.Fatal(err)
	}
	for _, dirEntry := range dirEntries {
		dirEntryPath := path + string(os.PathSeparator) + dirEntry.Name()
		if dirEntry.IsDir() {
			index.walkForIndexing(dirEntryPath)
		} else if strings.HasSuffix(dirEntry.Name(), ".md") {
			index.processUpdate(dirEntryPath)
		}
	}
}

func (index *gnosisIndex) startWatching(filePath string) *fsnotify.Watcher {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	// maybe rework the index so the Watcher is inside the index? idk

	// start a go routine to process events
	go func() {
		idleTimer := time.NewTimer(10 * time.Second)
		queuedEvents := make([]fsnotify.Event, 0)
		for {
			select {
			case ev := <-watcher.Events:
				queuedEvents = append(queuedEvents, ev)
				idleTimer.Reset(10 * time.Second)
			case err := <-watcher.Errors:
				log.Fatal(err)
			case <-idleTimer.C:
				for _, ev := range queuedEvents {
					if strings.HasSuffix(ev.Name, ".md") {
						switch ev.Op {
						case fsnotify.Remove, fsnotify.Rename:
							// delete the filePath
							index.processDelete(ev.Name)
						case fsnotify.Create, fsnotify.Write:
							// update the filePath
							index.processUpdate(ev.Name)
						default:
							// ignore
						}
					}
				}
				queuedEvents = make([]fsnotify.Event, 0)
				idleTimer.Reset(10 * time.Second)
			}
		}
	}()

	// now actually watch the filePath requested
	err = watcher.Add(filePath)
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("watching '%s' for changes...", filePath)

	return watcher
}
<|endoftext|>


<commit_before>// Copyright ©2014 The bíogo.bam Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bam

import (
	"code.google.com/p/biogo.bam/bgzf"

	"errors"
	"sort"
)

var baiMagic = [4]byte{'B', 'A', 'I', 0x1}

const (
	tileWidth     = 0x4000
	statsDummyBin = 0x924a
)

func makeOffset(vOff uint64) bgzf.Offset {
	return bgzf.Offset{
		File:  int64(vOff >> 16),
		Block: uint16(vOff),
	}
}

func isZero(o bgzf.Offset) bool {
	return o == bgzf.Offset{}
}

func vOffset(o bgzf.Offset) int64 {
	return o.File<<16 | int64(o.Block)
}

func isPlaced(r *Record) bool {
	return r.Ref != nil && r.Pos != -1
}

type byBinNumber []Bin

func (b byBinNumber) Len() int           { return len(b) }
func (b byBinNumber) Less(i, j int) bool { return b[i].Bin < b[j].Bin }
func (b byBinNumber) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }

type byBeginOffset []Chunk

func (c byBeginOffset) Len() int           { return len(c) }
func (c byBeginOffset) Less(i, j int) bool { return vOffset(c[i].Begin) < vOffset(c[j].Begin) }
func (c byBeginOffset) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }

type byVirtOffset []bgzf.Offset

func (o byVirtOffset) Len() int           { return len(o) }
func (o byVirtOffset) Less(i, j int) bool { return vOffset(o[i]) < vOffset(o[j]) }
func (o byVirtOffset) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }

type Index struct {
	References []RefIndex
	Unmapped   *uint64
	isSorted   bool
	lastRecord int
}

type RefIndex struct {
	Bins      []Bin
	Stats     *IndexStats
	Intervals []bgzf.Offset
}

type Bin struct {
	Bin    uint32
	Chunks []Chunk
}

type IndexStats struct {
	Chunk    Chunk
	Mapped   uint64
	Unmapped uint64
}

type Chunk struct {
	Begin bgzf.Offset
	End   bgzf.Offset
}

func (i *Index) Sort() {
	if !i.isSorted {
		for _, ref := range i.References {
			sort.Sort(byBinNumber(ref.Bins))
			for _, bin := range ref.Bins {
				sort.Sort(byBeginOffset(bin.Chunks))
			}
			sort.Sort(byVirtOffset(ref.Intervals))
		}
		i.isSorted = true
	}
}

func (i *Index) Add(r *Record, c Chunk) error {
	if i.Unmapped == nil {
		i.Unmapped = new(uint64)
	}
	if !isPlaced(r) {
		*i.Unmapped++
		return nil
	}

	rid := r.Reference().ID()
	if rid < len(i.References)-1 {
		return errors.New("bam: attempt to add record out of reference ID sort order")
	}
	if rid == len(i.References) {
		i.References = append(i.References, RefIndex{})
	} else {
		refs := make([]RefIndex, rid+1)
		copy(refs, i.References)
		i.References = refs
	}
	ref := &i.References[rid]

	// Record bin information.
	b := uint32(r.Bin())
	for i, bin := range ref.Bins {
		if bin.Bin == b {
			for j, chunk := range ref.Bins[i].Chunks {
				if vOffset(chunk.End) > vOffset(c.Begin) {
					ref.Bins[i].Chunks[j].End = c.End
					goto found
				}
			}
			ref.Bins[i].Chunks = append(ref.Bins[i].Chunks, c)
			goto found
		}
	}
	i.isSorted = false // TODO(kortschak) Consider making use of this more effectively for bin search.
	ref.Bins = append(ref.Bins, Bin{
		Bin:    b,
		Chunks: []Chunk{c},
	})
found:

	// Record interval tile information.
	biv := r.Start() / tileWidth
	if r.Start() < i.lastRecord {
		return errors.New("bam: attempt to add record out of position sort order")
	}
	i.lastRecord = r.Start()
	eiv := r.End() / tileWidth
	if eiv == len(ref.Intervals) {
		if eiv > biv {
			panic("bam: unexpected alignment length")
		}
		ref.Intervals = append(ref.Intervals, c.Begin)
	} else if eiv > len(ref.Intervals) {
		intvs := make([]bgzf.Offset, eiv)
		if len(ref.Intervals) > biv {
			biv = len(ref.Intervals)
		}
		for iv, offset := range intvs[biv:eiv] {
			if !isZero(offset) {
				panic("bam: unexpected non-zero offset")
			}
			intvs[iv+biv] = c.Begin
		}
		copy(intvs, ref.Intervals)
		ref.Intervals = intvs
	}

	// Record index stats.
	if ref.Stats == nil {
		ref.Stats = &IndexStats{
			Chunk: c,
		}
	} else {
		ref.Stats.Chunk.End = c.End
	}
	if r.Flags&Unmapped == 0 {
		ref.Stats.Mapped++
	} else {
		ref.Stats.Unmapped++
	}

	return nil
}

func (i *Index) Chunks(rid, beg, end int) []Chunk {
	if rid >= len(i.References) {
		return nil
	}
	i.Sort()
	ref := i.References[rid]

	iv := beg / tileWidth
	if iv >= len(ref.Intervals) {
		return nil
	}

	// Collect candidate chunks according to the scheme described in
	// the SAM spec under section 5 Indexing BAM.
	var chunks []Chunk
	for _, bin := range reg2bins(beg, end) {
		b := uint32(bin)
		c := sort.Search(len(ref.Bins), func(i int) bool { return ref.Bins[i].Bin >= b })
		if c < len(ref.Bins) && ref.Bins[c].Bin == b {
			for _, chunk := range ref.Bins[c].Chunks {
				// Here we check all tiles starting from the left end of the
				// query region until we get a non-zero offset. The spec states
				// that we only need to check tiles that contain beg. That is
				// not correct since we may have no alignments at the left end
				// of the query region.
				for j, tile := range ref.Intervals[iv:] {
					if isZero(tile) {
						continue
					}
					tbeg := (j + iv) * tileWidth
					tend := tbeg + tileWidth
					// We allow adjacent alignment since samtools behaviour here
					// has always irritated me and it is cheap to discard these
					// later if they are not wanted.
					if tend >= beg && tbeg <= end && vOffset(chunk.End) > vOffset(tile) {
						chunks = append(chunks, chunk)
						break
					}
				}
			}
		}
	}

	// Sort and merge overlaps.
	if !sort.IsSorted(byBeginOffset(chunks)) {
		sort.Sort(byBeginOffset(chunks))
	}

	return adjacent(chunks)
}

type Strategy func([]Chunk) []Chunk

var (
	Identity Strategy = identity
	Adjacent Strategy = adjacent
	Squash   Strategy = squash
)

func CompressorStrategy(near int64) Strategy {
	const blockMask = 1<<16 - 1
	return func(chunks []Chunk) []Chunk {
		if len(chunks) == 0 {
			return nil
		}
		for c := 1; c < len(chunks); c++ {
			leftChunk := chunks[c-1]
			leftChunk.End.File += near
			rightChunk := &chunks[c]
			leftEndOffset := vOffset(leftChunk.End)
			if leftEndOffset&^blockMask >= vOffset(rightChunk.Begin)&^blockMask {
				rightChunk.Begin = leftChunk.Begin
				if leftEndOffset > vOffset(rightChunk.End) {
					rightChunk.End = leftChunk.End
				}
				chunks = append(chunks[:c-1], chunks[c:]...)
				c--
			}
		}
		return chunks
	}
}

func identity(chunks []Chunk) []Chunk { return chunks }

func adjacent(chunks []Chunk) []Chunk {
	if len(chunks) == 0 {
		return nil
	}
	for c := 1; c < len(chunks); c++ {
		leftChunk := chunks[c-1]
		rightChunk := &chunks[c]
		leftEndOffset := vOffset(leftChunk.End)
		if leftEndOffset >= vOffset(rightChunk.Begin) {
			rightChunk.Begin = leftChunk.Begin
			if leftEndOffset > vOffset(rightChunk.End) {
				rightChunk.End = leftChunk.End
			}
			chunks = append(chunks[:c-1], chunks[c:]...)
			c--
		}
	}
	return chunks
}

func squash(chunks []Chunk) []Chunk {
	if len(chunks) == 0 {
		return nil
	}
	left := chunks[0].Begin
	right := chunks[0].End
	for _, c := range chunks[1:] {
		if vOffset(c.End) > vOffset(right) {
			right = c.End
		}
	}
	return []Chunk{{Begin: left, End: right}}
}

func (i *Index) MergeChunks(s Strategy) {
	if s == nil {
		return
	}
	for r, ref := range i.References {
		for b, bin := range ref.Bins {
			if !sort.IsSorted(byBeginOffset(bin.Chunks)) {
				sort.Sort(byBeginOffset(bin.Chunks))
			}
			i.References[r].Bins[b].Chunks = s(bin.Chunks)
			if !sort.IsSorted(byBeginOffset(bin.Chunks)) {
				sort.Sort(byBeginOffset(bin.Chunks))
			}
		}
	}
}
<commit_msg>Simplify CompressorStrategy<commit_after>// Copyright ©2014 The bíogo.bam Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bam

import (
	"code.google.com/p/biogo.bam/bgzf"

	"errors"
	"sort"
)

var baiMagic = [4]byte{'B', 'A', 'I', 0x1}

const (
	tileWidth     = 0x4000
	statsDummyBin = 0x924a
)

func makeOffset(vOff uint64) bgzf.Offset {
	return bgzf.Offset{
		File:  int64(vOff >> 16),
		Block: uint16(vOff),
	}
}

func isZero(o bgzf.Offset) bool {
	return o == bgzf.Offset{}
}

func vOffset(o bgzf.Offset) int64 {
	return o.File<<16 | int64(o.Block)
}

func isPlaced(r *Record) bool {
	return r.Ref != nil && r.Pos != -1
}

type byBinNumber []Bin

func (b byBinNumber) Len() int           { return len(b) }
func (b byBinNumber) Less(i, j int) bool { return b[i].Bin < b[j].Bin }
func (b byBinNumber) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }

type byBeginOffset []Chunk

func (c byBeginOffset) Len() int           { return len(c) }
func (c byBeginOffset) Less(i, j int) bool { return vOffset(c[i].Begin) < vOffset(c[j].Begin) }
func (c byBeginOffset) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }

type byVirtOffset []bgzf.Offset

func (o byVirtOffset) Len() int           { return len(o) }
func (o byVirtOffset) Less(i, j int) bool { return vOffset(o[i]) < vOffset(o[j]) }
func (o byVirtOffset) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }

type Index struct {
	References []RefIndex
	Unmapped   *uint64
	isSorted   bool
	lastRecord int
}

type RefIndex struct {
	Bins      []Bin
	Stats     *IndexStats
	Intervals []bgzf.Offset
}

type Bin struct {
	Bin    uint32
	Chunks []Chunk
}

type IndexStats struct {
	Chunk    Chunk
	Mapped   uint64
	Unmapped uint64
}

type Chunk struct {
	Begin bgzf.Offset
	End   bgzf.Offset
}

func (i *Index) Sort() {
	if !i.isSorted {
		for _, ref := range i.References {
			sort.Sort(byBinNumber(ref.Bins))
			for _, bin := range ref.Bins {
				sort.Sort(byBeginOffset(bin.Chunks))
			}
			sort.Sort(byVirtOffset(ref.Intervals))
		}
		i.isSorted = true
	}
}

func (i *Index) Add(r *Record, c Chunk) error {
	if i.Unmapped == nil {
		i.Unmapped = new(uint64)
	}
	if !isPlaced(r) {
		*i.Unmapped++
		return nil
	}

	rid := r.Reference().ID()
	if rid < len(i.References)-1 {
		return errors.New("bam: attempt to add record out of reference ID sort order")
	}
	if rid == len(i.References) {
		i.References = append(i.References, RefIndex{})
	} else {
		refs := make([]RefIndex, rid+1)
		copy(refs, i.References)
		i.References = refs
	}
	ref := &i.References[rid]

	// Record bin information.
	b := uint32(r.Bin())
	for i, bin := range ref.Bins {
		if bin.Bin == b {
			for j, chunk := range ref.Bins[i].Chunks {
				if vOffset(chunk.End) > vOffset(c.Begin) {
					ref.Bins[i].Chunks[j].End = c.End
					goto found
				}
			}
			ref.Bins[i].Chunks = append(ref.Bins[i].Chunks, c)
			goto found
		}
	}
	i.isSorted = false // TODO(kortschak) Consider making use of this more effectively for bin search.
	ref.Bins = append(ref.Bins, Bin{
		Bin:    b,
		Chunks: []Chunk{c},
	})
found:

	// Record interval tile information.
	biv := r.Start() / tileWidth
	if r.Start() < i.lastRecord {
		return errors.New("bam: attempt to add record out of position sort order")
	}
	i.lastRecord = r.Start()
	eiv := r.End() / tileWidth
	if eiv == len(ref.Intervals) {
		if eiv > biv {
			panic("bam: unexpected alignment length")
		}
		ref.Intervals = append(ref.Intervals, c.Begin)
	} else if eiv > len(ref.Intervals) {
		intvs := make([]bgzf.Offset, eiv)
		if len(ref.Intervals) > biv {
			biv = len(ref.Intervals)
		}
		for iv, offset := range intvs[biv:eiv] {
			if !isZero(offset) {
				panic("bam: unexpected non-zero offset")
			}
			intvs[iv+biv] = c.Begin
		}
		copy(intvs, ref.Intervals)
		ref.Intervals = intvs
	}

	// Record index stats.
	if ref.Stats == nil {
		ref.Stats = &IndexStats{
			Chunk: c,
		}
	} else {
		ref.Stats.Chunk.End = c.End
	}
	if r.Flags&Unmapped == 0 {
		ref.Stats.Mapped++
	} else {
		ref.Stats.Unmapped++
	}

	return nil
}

func (i *Index) Chunks(rid, beg, end int) []Chunk {
	if rid >= len(i.References) {
		return nil
	}
	i.Sort()
	ref := i.References[rid]

	iv := beg / tileWidth
	if iv >= len(ref.Intervals) {
		return nil
	}

	// Collect candidate chunks according to the scheme described in
	// the SAM spec under section 5 Indexing BAM.
	var chunks []Chunk
	for _, bin := range reg2bins(beg, end) {
		b := uint32(bin)
		c := sort.Search(len(ref.Bins), func(i int) bool { return ref.Bins[i].Bin >= b })
		if c < len(ref.Bins) && ref.Bins[c].Bin == b {
			for _, chunk := range ref.Bins[c].Chunks {
				// Here we check all tiles starting from the left end of the
				// query region until we get a non-zero offset. The spec states
				// that we only need to check tiles that contain beg. That is
				// not correct since we may have no alignments at the left end
				// of the query region.
				for j, tile := range ref.Intervals[iv:] {
					if isZero(tile) {
						continue
					}
					tbeg := (j + iv) * tileWidth
					tend := tbeg + tileWidth
					// We allow adjacent alignment since samtools behaviour here
					// has always irritated me and it is cheap to discard these
					// later if they are not wanted.
					if tend >= beg && tbeg <= end && vOffset(chunk.End) > vOffset(tile) {
						chunks = append(chunks, chunk)
						break
					}
				}
			}
		}
	}

	// Sort and merge overlaps.
	if !sort.IsSorted(byBeginOffset(chunks)) {
		sort.Sort(byBeginOffset(chunks))
	}

	return adjacent(chunks)
}

type Strategy func([]Chunk) []Chunk

var (
	Identity Strategy = identity
	Adjacent Strategy = adjacent
	Squash   Strategy = squash
)

func CompressorStrategy(near int64) Strategy {
	return func(chunks []Chunk) []Chunk {
		if len(chunks) == 0 {
			return nil
		}
		for c := 1; c < len(chunks); c++ {
			leftChunk := chunks[c-1]
			rightChunk := &chunks[c]
			if leftChunk.End.File+near >= rightChunk.Begin.File {
				rightChunk.Begin = leftChunk.Begin
				if vOffset(leftChunk.End) > vOffset(rightChunk.End) {
					rightChunk.End = leftChunk.End
				}
				chunks = append(chunks[:c-1], chunks[c:]...)
				c--
			}
		}
		return chunks
	}
}

func identity(chunks []Chunk) []Chunk { return chunks }

func adjacent(chunks []Chunk) []Chunk {
	if len(chunks) == 0 {
		return nil
	}
	for c := 1; c < len(chunks); c++ {
		leftChunk := chunks[c-1]
		rightChunk := &chunks[c]
		leftEndOffset := vOffset(leftChunk.End)
		if leftEndOffset >= vOffset(rightChunk.Begin) {
			rightChunk.Begin = leftChunk.Begin
			if leftEndOffset > vOffset(rightChunk.End) {
				rightChunk.End = leftChunk.End
			}
			chunks = append(chunks[:c-1], chunks[c:]...)
			c--
		}
	}
	return chunks
}

func squash(chunks []Chunk) []Chunk {
	if len(chunks) == 0 {
		return nil
	}
	left := chunks[0].Begin
	right := chunks[0].End
	for _, c := range chunks[1:] {
		if vOffset(c.End) > vOffset(right) {
			right = c.End
		}
	}
	return []Chunk{{Begin: left, End: right}}
}

func (i *Index) MergeChunks(s Strategy) {
	if s == nil {
		return
	}
	for r, ref := range i.References {
		for b, bin := range ref.Bins {
			if !sort.IsSorted(byBeginOffset(bin.Chunks)) {
				sort.Sort(byBeginOffset(bin.Chunks))
			}
			i.References[r].Bins[b].Chunks = s(bin.Chunks)
			if !sort.IsSorted(byBeginOffset(bin.Chunks)) {
				sort.Sort(byBeginOffset(bin.Chunks))
			}
		}
	}
}
<|endoftext|>


<commit_before>// Package p11 wraps `miekg/pkcs11` to make it easier to use and more idiomatic
// to Go, as compared with the more straightforward C wrapper that
// `miekg/pkcs11` presents. All types are safe to use concurrently.
//
// To use, first you open a module (a dynamically loaded library) by providing
// its path on your filesystem. This module is typically provided by
// the maker of your HSM, smartcard, or other cryptographic hardware, or
// sometimes by your operating system. Common module filenames are
// opensc-pkcs11.so, libykcs11.so, and libsofthsm2.so (you'll have to find the
// exact location).
//
// Once you've opened a Module, you can list the slots available with that
// module. Each slot may or may not contain a token. For instance, if you have a
// smartcard reader, that's a slot; if there's a smartcard in it, that's the
// token. Using this package, you can iterate through slots and check their
// information, and the information about tokens in them.
//
// Once you've found the slot with the token you want to use, you can open a
// Session with that token using OpenSession. Almost all operations require
// a session. Sessions use a sync.Mutex to ensure only one operation is active on
// them at a given time, as required by PKCS#11. If you want to get full
// performance out of a multi-core HSM, you will need to create multiple
// sessions.
//
// Once you've got a session, you can login to it. This is not necessary if you
// only want to access non-sensitive data, like certificates and public keys.
// However, to use any secret keys on a token, you'll need to login.
//
// Many operations, like FindObjects, return Objects. These represent pieces of
// data that exist on the token, referring to them by a numeric handle. With
// objects representing private keys, you can perform operations like signing
// and decrypting; with public keys and certificates you can extract their
// values.
//
// To summarize, a typical workflow might look like:
//
//   module, err := p11.OpenModule("/path/to/module.so")
//   if err != nil {
//     return err
//   }
//   slots, err := module.Slots()
//   if err != nil {
//     return err
//   }
//   // ... find the appropriate slot, then ...
//   session, err := slots[0].OpenSession()
//   if err != nil {
//     return err
//   }
//   privateKeyObject, err := session.FindObject(...)
//   if err != nil {
//     return err
//   }
//   privateKey := p11.PrivateKey(privateKeyObject)
//   signature, err := privateKey.Sign(..., []byte{"hello"})
//   if err != nil {
//     return err
//   }
package p11

import (
	"fmt"
	"sync"

	"github.com/miekg/pkcs11"
)

var modules = make(map[string]Module)
var modulesMu sync.Mutex

// OpenModule loads a PKCS#11 module (a .so file or dynamically loaded library).
// It's an error to load a PKCS#11 module multiple times, so this package
// will return a previously loaded Module for the same path if possible.
// Note that there is no facility to unload a module ("finalize" in PKCS#11
// parlance). In general, modules will be unloaded at the end of the process.
// The only place where you are likely to need to explicitly unload a module is
// if you fork your process. If you need to fork, you may want to use the
// lower-level `pkcs11` package.
func OpenModule(path string) (Module, error) {
	modulesMu.Lock()
	defer modulesMu.Unlock()
	module, ok := modules[path]
	if ok {
		return module, nil
	}

	newCtx := pkcs11.New(path)
	if newCtx == nil {
		return Module{}, fmt.Errorf("failed to load module %q", path)
	}

	err := newCtx.Initialize()
	if err != nil {
		return Module{}, fmt.Errorf("failed to initialize module: %s", err)
	}

	modules[path] = Module{newCtx}
	return modules[path], nil
}

// Module represents a PKCS#11 module, and can be used to create Sessions.
type Module struct {
	ctx *pkcs11.Ctx
}

// Info returns general information about the module.
func (m Module) Info() (pkcs11.Info, error) {
	return m.ctx.GetInfo()
}

// Slots returns all available Slots that have a token present.
func (m Module) Slots() ([]Slot, error) {
	ids, err := m.ctx.GetSlotList(true)
	if err != nil {
		return nil, err
	}
	result := make([]Slot, len(ids))
	for i, id := range ids {
		result[i] = Slot{
			ctx: m.ctx,
			id:  id,
		}
	}
	return result, nil
}
<commit_msg>Add p11.Module.Destroy() (#137)<commit_after>// Package p11 wraps `miekg/pkcs11` to make it easier to use and more idiomatic
// to Go, as compared with the more straightforward C wrapper that
// `miekg/pkcs11` presents. All types are safe to use concurrently.
//
// To use, first you open a module (a dynamically loaded library) by providing
// its path on your filesystem. This module is typically provided by
// the maker of your HSM, smartcard, or other cryptographic hardware, or
// sometimes by your operating system. Common module filenames are
// opensc-pkcs11.so, libykcs11.so, and libsofthsm2.so (you'll have to find the
// exact location).
//
// Once you've opened a Module, you can list the slots available with that
// module. Each slot may or may not contain a token. For instance, if you have a
// smartcard reader, that's a slot; if there's a smartcard in it, that's the
// token. Using this package, you can iterate through slots and check their
// information, and the information about tokens in them.
//
// Once you've found the slot with the token you want to use, you can open a
// Session with that token using OpenSession. Almost all operations require
// a session. Sessions use a sync.Mutex to ensure only one operation is active on
// them at a given time, as required by PKCS#11. If you want to get full
// performance out of a multi-core HSM, you will need to create multiple
// sessions.
//
// Once you've got a session, you can login to it. This is not necessary if you
// only want to access non-sensitive data, like certificates and public keys.
// However, to use any secret keys on a token, you'll need to login.
//
// Many operations, like FindObjects, return Objects. These represent pieces of
// data that exist on the token, referring to them by a numeric handle. With
// objects representing private keys, you can perform operations like signing
// and decrypting; with public keys and certificates you can extract their
// values.
//
// To summarize, a typical workflow might look like:
//
//   module, err := p11.OpenModule("/path/to/module.so")
//   if err != nil {
//     return err
//   }
//   slots, err := module.Slots()
//   if err != nil {
//     return err
//   }
//   // ... find the appropriate slot, then ...
//   session, err := slots[0].OpenSession()
//   if err != nil {
//     return err
//   }
//   privateKeyObject, err := session.FindObject(...)
//   if err != nil {
//     return err
//   }
//   privateKey := p11.PrivateKey(privateKeyObject)
//   signature, err := privateKey.Sign(..., []byte{"hello"})
//   if err != nil {
//     return err
//   }
package p11

import (
	"fmt"
	"sync"

	"github.com/miekg/pkcs11"
)

var modules = make(map[string]Module)
var modulesMu sync.Mutex

// OpenModule loads a PKCS#11 module (a .so file or dynamically loaded library).
// It's an error to load a PKCS#11 module multiple times, so this package
// will return a previously loaded Module for the same path if possible.
// Note that there is no facility to unload a module ("finalize" in PKCS#11
// parlance). In general, modules will be unloaded at the end of the process.
// The only place where you are likely to need to explicitly unload a module is
// if you fork your process. If you need to fork, you may want to use the
// lower-level `pkcs11` package.
func OpenModule(path string) (Module, error) {
	modulesMu.Lock()
	defer modulesMu.Unlock()
	module, ok := modules[path]
	if ok {
		return module, nil
	}

	newCtx := pkcs11.New(path)
	if newCtx == nil {
		return Module{}, fmt.Errorf("failed to load module %q", path)
	}

	err := newCtx.Initialize()
	if err != nil {
		return Module{}, fmt.Errorf("failed to initialize module: %s", err)
	}

	modules[path] = Module{newCtx}
	return modules[path], nil
}

// Module represents a PKCS#11 module, and can be used to create Sessions.
type Module struct {
	ctx *pkcs11.Ctx
}

// Info returns general information about the module.
func (m Module) Info() (pkcs11.Info, error) {
	return m.ctx.GetInfo()
}

// Slots returns all available Slots that have a token present.
func (m Module) Slots() ([]Slot, error) {
	ids, err := m.ctx.GetSlotList(true)
	if err != nil {
		return nil, err
	}
	result := make([]Slot, len(ids))
	for i, id := range ids {
		result[i] = Slot{
			ctx: m.ctx,
			id:  id,
		}
	}
	return result, nil
}

// Destroy unloads the module/library.
func (m Module) Destroy() {
	m.ctx.Destroy()
}
<|endoftext|>


<commit_before>package dotweb

import (
	"testing"
	"github.com/devfeel/dotweb/test"
	"encoding/json"
	"fmt"
	"net/http"
)

type Animal struct{
	Hair     string
	HasMouth bool
}

//normal write
func TestWrite(t *testing.T) {
	param := &InitContextParam{
		t,
		&Animal{},
		"",
		test.ToDefault,
	}

	//init param
	context := initResponseContext(param)

	exceptedObject:=&Animal{
		"Black",
		true,
	}

	animalJson,err:=json.Marshal(exceptedObject)
	test.Nil(t,err)

	//call function
	status:=http.StatusNotFound
	_,contextErr:=context.Write(status,animalJson)
	test.Nil(t,contextErr)

	//check result

	//header
	contentType:=context.response.header.Get(HeaderContentType)
	//因writer中的header方法调用过http.Header默认设置
	test.Equal(t,CharsetUTF8,contentType)
	test.Equal(t,status,context.response.Status)

	//body
	body:=string(context.response.body)

	test.Equal(t,string(animalJson),body)
}

//normal write string
//该用例是跑不过的- -,出来是一个[]uint8数组而不是string
func TestWriteString(t *testing.T) {
	param := &InitContextParam{
		t,
		&Animal{},
		"",
		test.ToDefault,
	}

	//init param
	context := initResponseContext(param)

	exceptedObject:=&Animal{
		"Black",
		true,
	}

	animalJson,err:=json.Marshal(exceptedObject)
	test.Nil(t,err)

	//call function
	_,contextErr:=context.WriteString(animalJson)
	test.Nil(t,contextErr)

	//header
	contentType:=context.response.header.Get(HeaderContentType)
	//因writer中的header方法调用过http.Header默认设置
	test.Equal(t,CharsetUTF8,contentType)
	test.Equal(t,defaultHttpCode,context.response.Status)

	//body
	body:=string(context.response.body)

	//fmt.Printf("%T",context.response.body)

	test.Equal(t,string(animalJson),body)
}

func TestWriteJson(t *testing.T) {
	param := &InitContextParam{
		t,
		&Animal{},
		"",
		test.ToDefault,
	}

	//init param
	context := initResponseContext(param)

	exceptedObject:=&Animal{
		"Black",
		true,
	}

	animalJson,err:=json.Marshal(exceptedObject)
	test.Nil(t,err)

	//call function
	_,contextErr:=context.WriteJson(exceptedObject)
	test.Nil(t,contextErr)

	//header
	contentType:=context.response.header.Get(HeaderContentType)
	//因writer中的header方法调用过http.Header默认设置
	test.Equal(t,MIMEApplicationJSONCharsetUTF8,contentType)
	test.Equal(t,defaultHttpCode,context.response.Status)

	//body
	body:=string(context.response.body)

	test.Equal(t,string(animalJson),body)
}

//normal jsonp
func TestWriteJsonp(t *testing.T) {
	param := &InitContextParam{
		t,
		&Animal{},
		"",
		test.ToDefault,
	}

	//init param
	context := initResponseContext(param)

	exceptedObject:=&Animal{
		"Black",
		true,
	}

	callback:="jsonCallBack"

	//call function
	_,err:=context.WriteJsonp(callback,exceptedObject)
	test.Nil(t,err)

	//check result

	//header
	contentType:=context.response.header.Get(HeaderContentType)
	test.Equal(t,MIMEApplicationJavaScriptCharsetUTF8,contentType)
	test.Equal(t,defaultHttpCode,context.response.Status)

	//body
	body:=string(context.response.body)

	animalJson,err:=json.Marshal(exceptedObject)
	test.Nil(t,err)

	excepted:=fmt.Sprint(callback,"(",string(animalJson),");")

	test.Equal(t,excepted,body)
}<commit_msg>修改用例.<commit_after>package dotweb

import (
	"testing"
	"github.com/devfeel/dotweb/test"
	"encoding/json"
	"fmt"
	"net/http"
)

type Animal struct{
	Hair     string
	HasMouth bool
}

//normal write
func TestWrite(t *testing.T) {
	param := &InitContextParam{
		t,
		&Animal{},
		"",
		test.ToDefault,
	}

	//init param
	context := initResponseContext(param)

	exceptedObject:=&Animal{
		"Black",
		true,
	}

	animalJson,err:=json.Marshal(exceptedObject)
	test.Nil(t,err)

	//call function
	status:=http.StatusNotFound
	_,contextErr:=context.Write(status,animalJson)
	test.Nil(t,contextErr)

	//check result

	//header
	contentType:=context.response.header.Get(HeaderContentType)
	//因writer中的header方法调用过http.Header默认设置
	test.Equal(t,CharsetUTF8,contentType)
	test.Equal(t,status,context.response.Status)

	//body
	body:=string(context.response.body)

	test.Equal(t,string(animalJson),body)
}

//normal write string
func TestWriteString(t *testing.T) {
	param := &InitContextParam{
		t,
		&Animal{},
		"",
		test.ToDefault,
	}

	//init param
	context := initResponseContext(param)

	exceptedObject:=&Animal{
		"Black",
		true,
	}

	animalJson,err:=json.Marshal(exceptedObject)
	test.Nil(t,err)

	//call function
	//这里是一个interface数组,用例需要小心.
	_,contextErr:=context.WriteString(string(animalJson))
	test.Nil(t,contextErr)

	//header
	contentType:=context.response.header.Get(HeaderContentType)
	//因writer中的header方法调用过http.Header默认设置
	test.Equal(t,CharsetUTF8,contentType)
	test.Equal(t,defaultHttpCode,context.response.Status)

	//body
	body:=string(context.response.body)

	//fmt.Printf("%T",context.response.body)

	test.Equal(t,string(animalJson),body)
}

func TestWriteJson(t *testing.T) {
	param := &InitContextParam{
		t,
		&Animal{},
		"",
		test.ToDefault,
	}

	//init param
	context := initResponseContext(param)

	exceptedObject:=&Animal{
		"Black",
		true,
	}

	animalJson,err:=json.Marshal(exceptedObject)
	test.Nil(t,err)

	//call function
	_,contextErr:=context.WriteJson(exceptedObject)
	test.Nil(t,contextErr)

	//header
	contentType:=context.response.header.Get(HeaderContentType)
	//因writer中的header方法调用过http.Header默认设置
	test.Equal(t,MIMEApplicationJSONCharsetUTF8,contentType)
	test.Equal(t,defaultHttpCode,context.response.Status)

	//body
	body:=string(context.response.body)

	test.Equal(t,string(animalJson),body)
}

//normal jsonp
func TestWriteJsonp(t *testing.T) {
	param := &InitContextParam{
		t,
		&Animal{},
		"",
		test.ToDefault,
	}

	//init param
	context := initResponseContext(param)

	exceptedObject:=&Animal{
		"Black",
		true,
	}

	callback:="jsonCallBack"

	//call function
	_,err:=context.WriteJsonp(callback,exceptedObject)
	test.Nil(t,err)

	//check result

	//header
	contentType:=context.response.header.Get(HeaderContentType)
	test.Equal(t,MIMEApplicationJavaScriptCharsetUTF8,contentType)
	test.Equal(t,defaultHttpCode,context.response.Status)

	//body
	body:=string(context.response.body)

	animalJson,err:=json.Marshal(exceptedObject)
	test.Nil(t,err)

	excepted:=fmt.Sprint(callback,"(",string(animalJson),");")

	test.Equal(t,excepted,body)
}<|endoftext|>


<commit_before>// Copyright (C) 2014 JT Olds
// See LICENSE for copying information

package oauth2

import (
	"fmt"
	"net/http"
	"net/url"
	"strings"

	"github.com/jtolds/webhelp"
	"github.com/spacemonkeygo/errors"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
)

// ProviderGroup is an http.Handler that keeps track of authentication for
// multiple OAuth2 providers.
//
// Assuming OAuth2 providers have been configured for Facebook, Google,
// LinkedIn, and Github, ProviderGroup handles requests to the following paths:
//  * /all/logout
//  * /facebook/login
//  * /facebook/logout
//  * /facebook/_cb
//  * /google/login
//  * /google/logout
//  * /google/_cb
//  * /linkedin/login
//  * /linkedin/logout
//  * /linkedin/_cb
//  * /github/login
//  * /github/logout
//  * /github/_cb
//
// ProviderGroup will also return associated state to you about each OAuth2
// provider's state, in addition to a LoginRequired middleware and a Login
// URL generator.
type ProviderGroup struct {
	handlers       map[string]*ProviderHandler
	mux            webhelp.DirMux
	urls           RedirectURLs
	group_base_url string
}

// NewProviderGroup makes a provider group. Requires a session namespace (will
// be prepended to ":"+provider_name), the base URL of the ProviderGroup's
// http.Handler, a collection of URLs for redirecting, and a list of specific
// configured providers.
func NewProviderGroup(session_namespace string, group_base_url string,
	urls RedirectURLs, providers ...*Provider) (*ProviderGroup, error) {

	group_base_url = strings.TrimRight(group_base_url, "/")

	g := &ProviderGroup{
		handlers:       make(map[string]*ProviderHandler, len(providers)),
		urls:           urls,
		group_base_url: group_base_url}

	g.mux = webhelp.DirMux{
		"all": webhelp.DirMux{"logout": webhelp.Exact(
			http.HandlerFunc(g.logoutAll))},
	}

	for _, provider := range providers {
		if provider.Name == "" {
			return nil, fmt.Errorf("empty provider name")
		}
		_, exists := g.handlers[provider.Name]
		if exists {
			return nil, fmt.Errorf("two providers given with name %#v",
				provider.Name)
		}
		handler := NewProviderHandler(provider,
			fmt.Sprintf("%s-%s", session_namespace, provider.Name),
			fmt.Sprintf("%s/%s", group_base_url, provider.Name), urls)
		g.handlers[provider.Name] = handler
		g.mux[provider.Name] = handler
	}

	return g, nil
}

func (g *ProviderGroup) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	g.mux.ServeHTTP(w, r)
}

// Routes implements webhelp.RouteLister
func (g *ProviderGroup) Routes(
	cb func(method, path string, annotations []string)) {
	webhelp.Routes(g.mux, cb)
}

var _ http.Handler = (*ProviderGroup)(nil)
var _ webhelp.RouteLister = (*ProviderGroup)(nil)

// Handler returns a specific ProviderHandler given the Provider name
func (g *ProviderGroup) Handler(provider_name string) (rv *ProviderHandler,
	exists bool) {
	rv, exists = g.handlers[provider_name]
	return rv, exists
}

// LoginURL returns the login URL for a given provider.
// redirect_to is the URL to navigate to after logging in, and force_prompt
// tells OAuth2 whether or not the login prompt should always be shown
// regardless of if the user is already logged in.
func (g *ProviderGroup) LoginURL(provider_name, redirect_to string,
	force_prompt bool) string {
	return g.handlers[provider_name].LoginURL(redirect_to, force_prompt)
}

// LogoutURL returns the logout URL for a given provider.
// redirect_to is the URL to navigate to after logging out.
func (g *ProviderGroup) LogoutURL(provider_name, redirect_to string) string {
	return g.handlers[provider_name].LogoutURL(redirect_to)
}

// LogoutAllURL returns the logout URL for all providers.
// redirect_to is the URL to navigate to after logging out.
func (g *ProviderGroup) LogoutAllURL(redirect_to string) string {
	return g.group_base_url + "/all/logout?" + url.Values{
		"redirect_to": {redirect_to}}.Encode()
}

// Tokens will return a map of all the currently valid OAuth2 tokens
func (g *ProviderGroup) Tokens(ctx context.Context) (map[string]*oauth2.Token,
	error) {
	rv := make(map[string]*oauth2.Token)
	var errs errors.ErrorGroup
	for name, handler := range g.handlers {
		token, err := handler.Token(ctx)
		errs.Add(err)
		if err == nil && token != nil {
			rv[name] = token
		}
	}
	return rv, errs.Finalize()
}

// Providers will return a map of all the currently known providers.
func (g *ProviderGroup) Providers() map[string]*ProviderHandler {
	copy := make(map[string]*ProviderHandler, len(g.handlers))
	for name, handler := range g.handlers {
		copy[name] = handler
	}
	return copy
}

// LoggedIn returns true if the user is logged in with any provider
func (g *ProviderGroup) LoggedIn(ctx context.Context) (bool, error) {
	t, err := g.Tokens(ctx)
	return len(t) > 0, err
}

// LogoutAll will not return any HTTP response, but will simply prepare a
// response for logging a user out completely from all providers. If a user
// should log out of just a specific OAuth2 provider, use the Logout method
// on the associated ProviderHandler.
func (g *ProviderGroup) LogoutAll(ctx context.Context,
	w http.ResponseWriter) error {
	var errs errors.ErrorGroup
	for _, handler := range g.handlers {
		errs.Add(handler.Logout(ctx, w))
	}
	return errs.Finalize()
}

func (g *ProviderGroup) logoutAll(w http.ResponseWriter, r *http.Request) {
	err := g.LogoutAll(webhelp.Context(r), w)
	if err != nil {
		webhelp.HandleError(w, r, err)
		return
	}
	redirect_to := r.FormValue("redirect_to")
	if redirect_to == "" {
		redirect_to = g.urls.DefaultLogoutURL
	}
	webhelp.Redirect(w, r, redirect_to)
}

// LoginRequired is a middleware for redirecting users to a login page if
// they aren't logged in yet. login_redirect should take the URL to redirect
// to after logging in and return a URL that will actually do the logging in.
// If you already know which provider a user should use, consider using
// (*ProviderHandler).LoginRequired instead, which doesn't require a
// login_redirect URL.
func (g *ProviderGroup) LoginRequired(h http.Handler,
	login_redirect func(redirect_to string) (url string)) http.Handler {
	return webhelp.RouteHandlerFunc(h,
		func(w http.ResponseWriter, r *http.Request) {
			tokens, err := g.Tokens(webhelp.Context(r))
			if err != nil {
				webhelp.HandleError(w, r, err)
				return
			}
			if len(tokens) <= 0 {
				webhelp.Redirect(w, r, login_redirect(r.RequestURI))
				return
			}
			h.ServeHTTP(w, r)
		})
}
<commit_msg>fix to work with new annotations<commit_after>// Copyright (C) 2014 JT Olds
// See LICENSE for copying information

package oauth2

import (
	"fmt"
	"net/http"
	"net/url"
	"strings"

	"github.com/jtolds/webhelp"
	"github.com/spacemonkeygo/errors"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
)

// ProviderGroup is an http.Handler that keeps track of authentication for
// multiple OAuth2 providers.
//
// Assuming OAuth2 providers have been configured for Facebook, Google,
// LinkedIn, and Github, ProviderGroup handles requests to the following paths:
//  * /all/logout
//  * /facebook/login
//  * /facebook/logout
//  * /facebook/_cb
//  * /google/login
//  * /google/logout
//  * /google/_cb
//  * /linkedin/login
//  * /linkedin/logout
//  * /linkedin/_cb
//  * /github/login
//  * /github/logout
//  * /github/_cb
//
// ProviderGroup will also return associated state to you about each OAuth2
// provider's state, in addition to a LoginRequired middleware and a Login
// URL generator.
type ProviderGroup struct {
	handlers       map[string]*ProviderHandler
	mux            webhelp.DirMux
	urls           RedirectURLs
	group_base_url string
}

// NewProviderGroup makes a provider group. Requires a session namespace (will
// be prepended to ":"+provider_name), the base URL of the ProviderGroup's
// http.Handler, a collection of URLs for redirecting, and a list of specific
// configured providers.
func NewProviderGroup(session_namespace string, group_base_url string,
	urls RedirectURLs, providers ...*Provider) (*ProviderGroup, error) {

	group_base_url = strings.TrimRight(group_base_url, "/")

	g := &ProviderGroup{
		handlers:       make(map[string]*ProviderHandler, len(providers)),
		urls:           urls,
		group_base_url: group_base_url}

	g.mux = webhelp.DirMux{
		"all": webhelp.DirMux{"logout": webhelp.Exact(
			http.HandlerFunc(g.logoutAll))},
	}

	for _, provider := range providers {
		if provider.Name == "" {
			return nil, fmt.Errorf("empty provider name")
		}
		_, exists := g.handlers[provider.Name]
		if exists {
			return nil, fmt.Errorf("two providers given with name %#v",
				provider.Name)
		}
		handler := NewProviderHandler(provider,
			fmt.Sprintf("%s-%s", session_namespace, provider.Name),
			fmt.Sprintf("%s/%s", group_base_url, provider.Name), urls)
		g.handlers[provider.Name] = handler
		g.mux[provider.Name] = handler
	}

	return g, nil
}

func (g *ProviderGroup) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	g.mux.ServeHTTP(w, r)
}

// Routes implements webhelp.RouteLister
func (g *ProviderGroup) Routes(
	cb func(method, path string, annotations map[string]string)) {
	webhelp.Routes(g.mux, cb)
}

var _ http.Handler = (*ProviderGroup)(nil)
var _ webhelp.RouteLister = (*ProviderGroup)(nil)

// Handler returns a specific ProviderHandler given the Provider name
func (g *ProviderGroup) Handler(provider_name string) (rv *ProviderHandler,
	exists bool) {
	rv, exists = g.handlers[provider_name]
	return rv, exists
}

// LoginURL returns the login URL for a given provider.
// redirect_to is the URL to navigate to after logging in, and force_prompt
// tells OAuth2 whether or not the login prompt should always be shown
// regardless of if the user is already logged in.
func (g *ProviderGroup) LoginURL(provider_name, redirect_to string,
	force_prompt bool) string {
	return g.handlers[provider_name].LoginURL(redirect_to, force_prompt)
}

// LogoutURL returns the logout URL for a given provider.
// redirect_to is the URL to navigate to after logging out.
func (g *ProviderGroup) LogoutURL(provider_name, redirect_to string) string {
	return g.handlers[provider_name].LogoutURL(redirect_to)
}

// LogoutAllURL returns the logout URL for all providers.
// redirect_to is the URL to navigate to after logging out.
func (g *ProviderGroup) LogoutAllURL(redirect_to string) string {
	return g.group_base_url + "/all/logout?" + url.Values{
		"redirect_to": {redirect_to}}.Encode()
}

// Tokens will return a map of all the currently valid OAuth2 tokens
func (g *ProviderGroup) Tokens(ctx context.Context) (map[string]*oauth2.Token,
	error) {
	rv := make(map[string]*oauth2.Token)
	var errs errors.ErrorGroup
	for name, handler := range g.handlers {
		token, err := handler.Token(ctx)
		errs.Add(err)
		if err == nil && token != nil {
			rv[name] = token
		}
	}
	return rv, errs.Finalize()
}

// Providers will return a map of all the currently known providers.
func (g *ProviderGroup) Providers() map[string]*ProviderHandler {
	copy := make(map[string]*ProviderHandler, len(g.handlers))
	for name, handler := range g.handlers {
		copy[name] = handler
	}
	return copy
}

// LoggedIn returns true if the user is logged in with any provider
func (g *ProviderGroup) LoggedIn(ctx context.Context) (bool, error) {
	t, err := g.Tokens(ctx)
	return len(t) > 0, err
}

// LogoutAll will not return any HTTP response, but will simply prepare a
// response for logging a user out completely from all providers. If a user
// should log out of just a specific OAuth2 provider, use the Logout method
// on the associated ProviderHandler.
func (g *ProviderGroup) LogoutAll(ctx context.Context,
	w http.ResponseWriter) error {
	var errs errors.ErrorGroup
	for _, handler := range g.handlers {
		errs.Add(handler.Logout(ctx, w))
	}
	return errs.Finalize()
}

func (g *ProviderGroup) logoutAll(w http.ResponseWriter, r *http.Request) {
	err := g.LogoutAll(webhelp.Context(r), w)
	if err != nil {
		webhelp.HandleError(w, r, err)
		return
	}
	redirect_to := r.FormValue("redirect_to")
	if redirect_to == "" {
		redirect_to = g.urls.DefaultLogoutURL
	}
	webhelp.Redirect(w, r, redirect_to)
}

// LoginRequired is a middleware for redirecting users to a login page if
// they aren't logged in yet. login_redirect should take the URL to redirect
// to after logging in and return a URL that will actually do the logging in.
// If you already know which provider a user should use, consider using
// (*ProviderHandler).LoginRequired instead, which doesn't require a
// login_redirect URL.
func (g *ProviderGroup) LoginRequired(h http.Handler,
	login_redirect func(redirect_to string) (url string)) http.Handler {
	return webhelp.RouteHandlerFunc(h,
		func(w http.ResponseWriter, r *http.Request) {
			tokens, err := g.Tokens(webhelp.Context(r))
			if err != nil {
				webhelp.HandleError(w, r, err)
				return
			}
			if len(tokens) <= 0 {
				webhelp.Redirect(w, r, login_redirect(r.RequestURI))
				return
			}
			h.ServeHTTP(w, r)
		})
}
<|endoftext|>


<commit_before>package test

// These tests check for meta level items, like trailing whitespace, correct file naming etc.

import (
	"bufio"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"unicode"
)

func TestTrailingWhitespace(t *testing.T) {
	err := filepath.Walk("..", hasTrailingWhitespace)
	if err != nil {
		t.Fatal(err)
	}
}

func hasTrailingWhitespace(path string, info os.FileInfo, _ error) error {
	// Only handle regular files, skip files that are executable and skip file in the
	// root that start with a .
	if !info.Mode().IsRegular() {
		return nil
	}
	if info.Mode().Perm()&0111 != 0 {
		return nil
	}
	if strings.HasPrefix(path, "../.") {
		return nil
	}

	file, err := os.Open(path)
	if err != nil {
		return nil
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		text := scanner.Text()
		trimmed := strings.TrimRightFunc(text, unicode.IsSpace)
		if len(text) != len(trimmed) {
			return fmt.Errorf("file %q has trailing whitespace, text: %q", path, text)
		}
	}

	return scanner.Err()
}

func TestFileNameHyphen(t *testing.T) {
	err := filepath.Walk("..", hasHyphen)
	if err != nil {
		t.Fatal(err)
	}
}

func hasHyphen(path string, info os.FileInfo, _ error) error {
	// only for regular files, not starting with a . and those that are go files.
	if !info.Mode().IsRegular() {
		return nil
	}
	if strings.HasPrefix(path, "../.") {
		return nil
	}
	if filepath.Ext(path) != ".go" {
		return nil
	}

	if strings.Index(path, "-") > 0 {
		return fmt.Errorf("file %q has a hyphen, please use underscores in file names", path)
	}

	return nil
}

// Test if error messages start with an upper case.
func TestLowercaseLog(t *testing.T) {
	err := filepath.Walk("..", hasLowercase)
	if err != nil {
		t.Fatal(err)
	}
}

func hasLowercase(path string, info os.FileInfo, _ error) error {
	// only for regular files, not starting with a . and those that are go files.
	if !info.Mode().IsRegular() {
		return nil
	}
	if strings.HasPrefix(path, "../.") {
		return nil
	}
	if !strings.HasSuffix(path, "_test.go") {
		return nil
	}

	fs := token.NewFileSet()
	f, err := parser.ParseFile(fs, path, nil, parser.AllErrors)
	if err != nil {
		return err
	}
	l := &logfmt{}
	ast.Walk(l, f)
	if l.err != nil {
		return l.err
	}
	return nil
}

type logfmt struct {
	err error
}

func (l logfmt) Visit(n ast.Node) ast.Visitor {
	if n == nil {
		return nil
	}
	ce, ok := n.(*ast.CallExpr)
	if !ok {
		return l
	}
	se, ok := ce.Fun.(*ast.SelectorExpr)
	if !ok {
		return l
	}
	id, ok := se.X.(*ast.Ident)
	if !ok {
		return l
	}
	if id.Name != "t" { //t *testing.T
		return l
	}

	switch se.Sel.Name {
	case "Errorf":
	case "Logf":
	case "Log":
	case "Fatalf":
	case "Fatal":
	default:
		return l
	}
	// Check first arg, that should have basic lit with capital
	if len(ce.Args) < 1 {
		return l
	}
	bl, ok := ce.Args[0].(*ast.BasicLit)
	if !ok {
		return l
	}
	if bl.Kind != token.STRING {
		return l
	}
	if strings.HasPrefix(bl.Value, "\"%s") || strings.HasPrefix(bl.Value, "\"%d") {
		return l
	}
	if strings.HasPrefix(bl.Value, "\"%v") || strings.HasPrefix(bl.Value, "\"%+v") {
		return l
	}
	for i, u := range bl.Value {
		// disregard "
		if i == 1 && !unicode.IsUpper(u) {
			l.err = fmt.Errorf("test error message %s doesn't start with an uppercase", bl.Value)
			return nil
		}
		if i == 1 {
			break
		}
	}
	return l
}

func TestImportTesting(t *testing.T) {
	err := filepath.Walk("..", hasImportTesting)
	if err != nil {
		t.Fatal(err)
	}
}

func hasImportTesting(path string, info os.FileInfo, _ error) error {
	// only for regular files, not starting with a . and those that are go files.
	if !info.Mode().IsRegular() {
		return nil
	}
	if strings.HasPrefix(path, "../.") {
		return nil
	}
	if strings.HasSuffix(path, "_test.go") {
		return nil
	}

	if strings.HasSuffix(path, ".go") {
		fs := token.NewFileSet()
		f, err := parser.ParseFile(fs, path, nil, parser.AllErrors)
		if err != nil {
			return err
		}
		for _, im := range f.Imports {
			if im.Path.Value == `"testing"` {
				return fmt.Errorf("file %q is importing %q", path, "testing")
			}
		}
	}
	return nil
}

func TestImportOrdering(t *testing.T) {
	err := filepath.Walk("..", hasImportOrdering)
	if err != nil {
		t.Fatal(err)
	}
}

func hasImportOrdering(path string, info os.FileInfo, _ error) error {
	if !info.Mode().IsRegular() {
		return nil
	}
	if strings.HasPrefix(path, "../.") {
		return nil
	}
	if filepath.Ext(path) != ".go" {
		return nil
	}

	fs := token.NewFileSet()
	f, err := parser.ParseFile(fs, path, nil, parser.AllErrors)
	if err != nil {
		return err
	}
	if len(f.Imports) == 0 {
		return nil
	}

	// 3 blocks total, if
	// 3 blocks: std + coredns + 3rd party
	// 2 blocks: std + coredns, std + 3rd party, coredns + 3rd party
	// 1 block: std, coredns, 3rd party
	// first entry in a block specifies the type (std, coredns, 3rd party)
	// we want: std, coredns, 3rd party
	// more than 3 blocks as an error
	blocks := [3][]*ast.ImportSpec{}
	prevpos := 0
	bl := 0
	for _, im := range f.Imports {
		line := fs.Position(im.Path.Pos()).Line
		if line-prevpos > 1 && prevpos > 0 {
			bl++
		}
		if bl > 2 {
			return fmt.Errorf("more than %d import blocks in %q", bl, path)
		}
		blocks[bl] = append(blocks[bl], im)
		prevpos = line
	}
	// if it:
	// contains strings github.com/coredns/coredns -> coredns
	// contains dots -> 3rd
	// no dots -> std
	ip := [3]string{} // type per block, just string, either std, coredns, 3rd
	for i := 0; i <= bl; i++ {
		ip[i] = importtype(blocks[i][0].Path.Value)
	}

	// Ok, now that we have the type, let's see if all members adhere to it.
	// After that we check if the are in the right order.
	for i := 0; i < bl; i++ {
		for _, p := range blocks[i] {
			t := importtype(p.Path.Value)
			if t != ip[i] {
				return fmt.Errorf("import path for %s is not of the same type %q in %q", p.Path.Value, ip[i], path)
			}
		}
	}

	// check order
	switch bl {
	case 0:
		// we don't care
	case 1:
		if ip[0] == "std" && ip[1] == "coredns" {
			break // OK
		}
		if ip[0] == "std" && ip[1] == "3rd" {
			break // OK
		}
		if ip[0] == "coredns" && ip[1] == "3rd" {
			break // OK
		}
		return fmt.Errorf("import path in %q are not in the right order (std -> coredns -> 3rd)", path)
	case 2:
		if ip[0] == "std" && ip[1] == "coredns" && ip[2] == "3rd" {
			break // OK
		}
		return fmt.Errorf("import path in %q are not in the right order (std -> coredns -> 3rd)", path)
	}

	return nil
}

func importtype(s string) string {
	if strings.Contains(s, "github.com/coredns/coredns") {
		return "coredns"
	}
	if strings.Contains(s, ".") {
		return "3rd"
	}
	return "std"
}
<commit_msg>presubmit: improve output (#3641)<commit_after>package test

// These tests check for meta level items, like trailing whitespace, correct file naming etc.

import (
	"bufio"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"unicode"
)

func TestTrailingWhitespace(t *testing.T) {
	walker := hasTrailingWhitespaceWalker{}
	err := filepath.Walk("..", walker.walk)

	if err != nil {
		t.Fatal(err)
	}

	if len(walker.Errors) > 0 {
		for _, err = range walker.Errors {
			t.Error(err)
		}
	}
}

type hasTrailingWhitespaceWalker struct {
	Errors []error
}

func (w *hasTrailingWhitespaceWalker) walk(path string, info os.FileInfo, _ error) error {
	// Only handle regular files, skip files that are executable and skip file in the
	// root that start with a .
	if !info.Mode().IsRegular() {
		return nil
	}
	if info.Mode().Perm()&0111 != 0 {
		return nil
	}
	if strings.HasPrefix(path, "../.") {
		return nil
	}

	file, err := os.Open(path)
	if err != nil {
		return nil
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for i := 1; scanner.Scan(); i++ {
		text := scanner.Text()
		trimmed := strings.TrimRightFunc(text, unicode.IsSpace)
		if len(text) != len(trimmed) {
			absPath, _ := filepath.Abs(path)
			w.Errors = append(w.Errors, fmt.Errorf("file %q has trailing whitespace at line %d, text: %q", absPath, i, text))
		}
	}

	err = scanner.Err()

	if err != nil {
		absPath, _ := filepath.Abs(path)
		err = fmt.Errorf("file %q: %v", absPath, err)
	}

	return err
}

func TestFileNameHyphen(t *testing.T) {
	walker := hasHyphenWalker{}
	err := filepath.Walk("..", walker.walk)

	if err != nil {
		t.Fatal(err)
	}

	if len(walker.Errors) > 0 {
		for _, err = range walker.Errors {
			t.Error(err)
		}
	}
}

type hasHyphenWalker struct {
	Errors []error
}

func (w *hasHyphenWalker) walk(path string, info os.FileInfo, _ error) error {
	// only for regular files, not starting with a . and those that are go files.
	if !info.Mode().IsRegular() {
		return nil
	}
	if strings.HasPrefix(path, "../.") {
		return nil
	}
	if filepath.Ext(path) != ".go" {
		return nil
	}

	if strings.Index(path, "-") > 0 {
		absPath, _ := filepath.Abs(path)
		w.Errors = append(w.Errors, fmt.Errorf("file %q has a hyphen, please use underscores in file names", absPath))
	}

	return nil
}

// Test if error messages start with an upper case.
func TestLowercaseLog(t *testing.T) {
	walker := hasLowercaseWalker{}
	err := filepath.Walk("..", walker.walk)

	if err != nil {
		t.Fatal(err)
	}

	if len(walker.Errors) > 0 {
		for _, err = range walker.Errors {
			t.Error(err)
		}
	}
}

type hasLowercaseWalker struct {
	Errors []error
}

func (w *hasLowercaseWalker) walk(path string, info os.FileInfo, _ error) error {
	// only for regular files, not starting with a . and those that are go files.
	if !info.Mode().IsRegular() {
		return nil
	}
	if strings.HasPrefix(path, "../.") {
		return nil
	}
	if !strings.HasSuffix(path, "_test.go") {
		return nil
	}

	fs := token.NewFileSet()
	f, err := parser.ParseFile(fs, path, nil, parser.AllErrors)
	if err != nil {
		return err
	}
	l := &logfmt{}
	ast.Walk(l, f)
	if l.err != nil {
		w.Errors = append(w.Errors, l.err)
	}
	return nil
}

type logfmt struct {
	err error
}

func (l logfmt) Visit(n ast.Node) ast.Visitor {
	if n == nil {
		return nil
	}
	ce, ok := n.(*ast.CallExpr)
	if !ok {
		return l
	}
	se, ok := ce.Fun.(*ast.SelectorExpr)
	if !ok {
		return l
	}
	id, ok := se.X.(*ast.Ident)
	if !ok {
		return l
	}
	if id.Name != "t" { //t *testing.T
		return l
	}

	switch se.Sel.Name {
	case "Errorf":
	case "Logf":
	case "Log":
	case "Fatalf":
	case "Fatal":
	default:
		return l
	}
	// Check first arg, that should have basic lit with capital
	if len(ce.Args) < 1 {
		return l
	}
	bl, ok := ce.Args[0].(*ast.BasicLit)
	if !ok {
		return l
	}
	if bl.Kind != token.STRING {
		return l
	}
	if strings.HasPrefix(bl.Value, "\"%s") || strings.HasPrefix(bl.Value, "\"%d") {
		return l
	}
	if strings.HasPrefix(bl.Value, "\"%v") || strings.HasPrefix(bl.Value, "\"%+v") {
		return l
	}
	for i, u := range bl.Value {
		// disregard "
		if i == 1 && !unicode.IsUpper(u) {
			l.err = fmt.Errorf("test error message %s doesn't start with an uppercase", bl.Value)
			return nil
		}
		if i == 1 {
			break
		}
	}
	return l
}

func TestImportTesting(t *testing.T) {
	walker := hasLowercaseWalker{}
	err := filepath.Walk("..", walker.walk)

	if err != nil {
		t.Fatal(err)
	}

	if len(walker.Errors) > 0 {
		for _, err = range walker.Errors {
			t.Error(err)
		}
	}
}

type hasImportTestingWalker struct {
	Errors []error
}

func (w *hasImportTestingWalker) walk(path string, info os.FileInfo, _ error) error {
	// only for regular files, not starting with a . and those that are go files.
	if !info.Mode().IsRegular() {
		return nil
	}
	if strings.HasPrefix(path, "../.") {
		return nil
	}
	if strings.HasSuffix(path, "_test.go") {
		return nil
	}

	if strings.HasSuffix(path, ".go") {
		fs := token.NewFileSet()
		f, err := parser.ParseFile(fs, path, nil, parser.AllErrors)
		if err != nil {
			return err
		}
		for _, im := range f.Imports {
			if im.Path.Value == `"testing"` {
				absPath, _ := filepath.Abs(path)
				w.Errors = append(w.Errors, fmt.Errorf("file %q is importing %q", absPath, "testing"))
			}
		}
	}
	return nil
}

func TestImportOrdering(t *testing.T) {
	walker := testImportOrderingWalker{}
	err := filepath.Walk("..", walker.walk)

	if err != nil {
		t.Fatal(err)
	}

	if len(walker.Errors) > 0 {
		for _, err = range walker.Errors {
			t.Error(err)
		}
	}
}

type testImportOrderingWalker struct {
	Errors []error
}

func (w *testImportOrderingWalker) walk(path string, info os.FileInfo, _ error) error {
	if !info.Mode().IsRegular() {
		return nil
	}
	if strings.HasPrefix(path, "../.") {
		return nil
	}
	if filepath.Ext(path) != ".go" {
		return nil
	}

	fs := token.NewFileSet()
	f, err := parser.ParseFile(fs, path, nil, parser.AllErrors)
	if err != nil {
		return err
	}
	if len(f.Imports) == 0 {
		return nil
	}

	// 3 blocks total, if
	// 3 blocks: std + coredns + 3rd party
	// 2 blocks: std + coredns, std + 3rd party, coredns + 3rd party
	// 1 block: std, coredns, 3rd party
	// first entry in a block specifies the type (std, coredns, 3rd party)
	// we want: std, coredns, 3rd party
	// more than 3 blocks as an error
	blocks := [3][]*ast.ImportSpec{}
	prevpos := 0
	bl := 0
	for _, im := range f.Imports {
		line := fs.Position(im.Path.Pos()).Line
		if line-prevpos > 1 && prevpos > 0 {
			bl++
		}
		if bl > 2 {
			absPath, _ := filepath.Abs(path)
			w.Errors = append(w.Errors, fmt.Errorf("more than %d import blocks in %q", bl, absPath))
		}
		blocks[bl] = append(blocks[bl], im)
		prevpos = line
	}
	// if it:
	// contains strings github.com/coredns/coredns -> coredns
	// contains dots -> 3rd
	// no dots -> std
	ip := [3]string{} // type per block, just string, either std, coredns, 3rd
	for i := 0; i <= bl; i++ {
		ip[i] = importtype(blocks[i][0].Path.Value)
	}

	// Ok, now that we have the type, let's see if all members adhere to it.
	// After that we check if the are in the right order.
	for i := 0; i < bl; i++ {
		for _, p := range blocks[i] {
			t := importtype(p.Path.Value)
			if t != ip[i] {
				absPath, _ := filepath.Abs(path)
				w.Errors = append(w.Errors, fmt.Errorf("import path for %s is not of the same type %q in %q", p.Path.Value, ip[i], absPath))
			}
		}
	}

	// check order
	switch bl {
	case 0:
		// we don't care
	case 1:
		if ip[0] == "std" && ip[1] == "coredns" {
			break // OK
		}
		if ip[0] == "std" && ip[1] == "3rd" {
			break // OK
		}
		if ip[0] == "coredns" && ip[1] == "3rd" {
			break // OK
		}
		absPath, _ := filepath.Abs(path)
		w.Errors = append(w.Errors, fmt.Errorf("import path in %q are not in the right order (std -> coredns -> 3rd)", absPath))
	case 2:
		if ip[0] == "std" && ip[1] == "coredns" && ip[2] == "3rd" {
			break // OK
		}
		absPath, _ := filepath.Abs(path)
		w.Errors = append(w.Errors, fmt.Errorf("import path in %q are not in the right order (std -> coredns -> 3rd)", absPath))
	}

	return nil
}

func importtype(s string) string {
	if strings.Contains(s, "github.com/coredns/coredns") {
		return "coredns"
	}
	if strings.Contains(s, ".") {
		return "3rd"
	}
	return "std"
}
<|endoftext|>