<commit_before>// Copyright (c) 2010 Go Fightclub Authors

package alsa

import (
  "fmt"
  "unsafe"
  "afp"
  "os"
)

// #include <alsa/asoundlib.h>
import "C"

/////
// Alsa Source
// Listens to a microphone
type AlsaSource struct {
    ctx *afp.Context
    header afp.StreamHeader
    capture *C.snd_pcm_t
    params *C.snd_pcm_hw_params_t
}

func NewAlsaSource() afp.Filter {
    return &AlsaSource{}
}

func (self *AlsaSource) GetType() int {
    return afp.PIPE_SOURCE
}

func (self *AlsaSource) Init(ctx *afp.Context, args []string) os.Error {
    self.ctx = ctx
    return nil
}

func (self *AlsaSource) Start() {

    self.header = afp.StreamHeader {
        Version: 1,
        Channels: 1,
        SampleSize: 32,
        SampleRate: 44100,
        FrameSize: 4096,
    }

    self.ctx.HeaderSink <- self.header

    retval := self.prepare()
    if ( retval != nil) {
        panic(retval)
    }

    for {
		cbuf := make([]float32, int32(self.header.Channels) * self.header.FrameSize)
		buff := make([][]float32, self.header.FrameSize)
        length := len(cbuf)

        //first off, grab some data from alsa
        read := C.snd_pcm_readi(self.capture, unsafe.Pointer(&cbuf[0]), C.snd_pcm_uframes_t(length))
        if read < C.snd_pcm_sframes_t(length) {
            errno := C.snd_pcm_recover(self.capture, C.int(read), 0)
            if errno < 0 {
                panic(fmt.Sprint( "While reading from ALSA device, failed to recover from error: ", errno))
            }
        }

        // snd_pcm_readi gives us a one dimensional array of interleaved data
        // but what we want is a two dimensional array of samples
        chans := int(self.header.Channels)
		for slice, i := 0, 0; i < length; slice, i = slice + 1, i + chans {
			buff[slice] = make([]float32, chans)
			buff[slice] = cbuf[i : i + chans]
        }

        //send it on down the line
        self.ctx.Sink <- buff
    }

}

func (self *AlsaSource) Stop() os.Error {
    C.snd_pcm_close(self.capture)
    close(self.ctx.Sink)
    return nil
}

/////
// Alsa Sink
// Outputs to speakers via ALSA
type AlsaSink struct {
    ctx *afp.Context
    header afp.StreamHeader
    playback *C.snd_pcm_t
    params *C.snd_pcm_hw_params_t

}

func NewAlsaSink() afp.Filter {
    return &AlsaSink{}
}

func (self *AlsaSink) GetType() int {
    return afp.PIPE_SINK
}

func (self *AlsaSink) Init(ctx *afp.Context, args []string) os.Error {
    self.ctx = ctx
    return nil
}

func (self *AlsaSink) Start() {

    const FRAMES int = 256 //the double buffer holds 256 frames
    self.header = <-self.ctx.HeaderSource

    retval := self.prepare()
    if (retval != nil) {
        panic(retval)
    }

    //almost a do..while
    var cbuf []float32 //C buffer
    var written chan C.snd_pcm_sframes_t
    chans := int(self.header.Channels)
    double := make([][]float32, 0, self.header.FrameSize * 1024)
    double = append(double, <-self.ctx.Source...)
    length := len(double)
    oldLength := length
    written <- C.snd_pcm_sframes_t(length)

    for buffer := range self.ctx.Source { //blocking

        select {

        //wait for the previous write to finish
        case error := <-written:

            if int(error) < oldLength {
                //not all the data was written to the device
                panic(fmt.Sprintf("Could not write all data to ALSA device, wrote: ", written))
            }

            //write to the speaker in another thread
            go func() {
                oldLength = length
                written <- C.snd_pcm_writei(self.playback, unsafe.Pointer(&cbuf[0]), C.snd_pcm_uframes_t(length))
            }()

            double = buffer

        default:
            double = append(buffer, <-self.ctx.Source...)
            length = len(double)

            //cbuf WILL be a new pointer, so we can gaurantee that the address space given to snd_pcm_writei
            //will not be modified once it is passed; but on the down side, we rely on the garbage collector
            //to take care of many stale slices created while filling our buffer
            cbuf = make([]float32, int32(self.header.Channels) * self.header.FrameSize * int32(length))

            //interleave the channels
            streamOffset := 0
            for i := 0; i < length; i++ {
                for j := 0; j < chans; j++ {
                    cbuf[streamOffset] = double[i][j]
                    streamOffset++
                }
            }

        }//end select

    }

    return
}

func (self *AlsaSink) Stop() os.Error {
    C.snd_pcm_close(self.playback)
    return nil
}

// Ugly bastardized C code follows
func (self *AlsaSink) prepare() os.Error {

    if errno := C.snd_pcm_open(&self.playback, C.CString("default"), C.SND_PCM_STREAM_PLAYBACK, 0); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not open device. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_malloc(&self.params); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not allocate hardware parameter structure. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_any(self.playback, self.params); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not initialize hardware parameter structure. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_access(self.playback, self.params, C.SND_PCM_ACCESS_RW_INTERLEAVED); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set access type. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_format(self.playback, self.params, C.SND_PCM_FORMAT_FLOAT); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set sample format. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_rate(self.playback, self.params, C.uint(self.header.SampleRate), 0); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set sample rate. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_channels(self.playback, self.params, C.uint(self.header.Channels)); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set channel count. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params(self.playback, self.params); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set hardware parameters. Error %d", errno) )
    }

    C.snd_pcm_hw_params_free(self.params)

    if errno := C.snd_pcm_prepare(self.playback); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not prepare audio device for use. Error %d", errno) )
    }

    return nil
}

//this one is slightly different
//note the change in scope
func (self *AlsaSource) prepare() os.Error {

    if errno := C.snd_pcm_open(&self.capture, C.CString("default"), C.SND_PCM_STREAM_CAPTURE, 0); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not open device. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_malloc(&self.params); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not allocate hardware parameters. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_any(self.capture, self.params); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not initialize hardware parameter structure. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_access(self.capture, self.params, C.SND_PCM_ACCESS_RW_INTERLEAVED); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set access. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_format(self.capture, self.params, C.SND_PCM_FORMAT_FLOAT); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set sample format. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_rate(self.capture, self.params, C.uint(self.header.SampleRate), 0); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set sample rate. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_channels(self.capture, self.params, 1); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set channel count. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params(self.capture, self.params); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set parameters. Error %d", errno) )
    }

    C.snd_pcm_hw_params_free(self.params)

    if errno := C.snd_pcm_prepare(self.capture); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not prepare audio interface for use. Error %d", errno) )
    }

    return nil
}
<commit_msg>Added some cleanup to Stop()<commit_after>// Copyright (c) 2010 Go Fightclub Authors

package alsa

import (
  "fmt"
  "unsafe"
  "afp"
  "os"
)

// #include <alsa/asoundlib.h>
import "C"

/////
// Alsa Source
// Listens to a microphone
type AlsaSource struct {
    ctx *afp.Context
    header afp.StreamHeader
    capture *C.snd_pcm_t
    params *C.snd_pcm_hw_params_t
}

func NewAlsaSource() afp.Filter {
    return &AlsaSource{}
}

func (self *AlsaSource) GetType() int {
    return afp.PIPE_SOURCE
}

func (self *AlsaSource) Init(ctx *afp.Context, args []string) os.Error {
    self.ctx = ctx
    return nil
}

func (self *AlsaSource) Start() {

    self.header = afp.StreamHeader {
        Version: 1,
        Channels: 1,
        SampleSize: 32,
        SampleRate: 44100,
        FrameSize: 4096,
    }

    self.ctx.HeaderSink <- self.header

    retval := self.prepare()
    if ( retval != nil) {
        panic(retval)
    }

    for {
		cbuf := make([]float32, int32(self.header.Channels) * self.header.FrameSize)
		buff := make([][]float32, self.header.FrameSize)
        length := len(cbuf)

        //first off, grab some data from alsa
        read := C.snd_pcm_readi(self.capture, unsafe.Pointer(&cbuf[0]), C.snd_pcm_uframes_t(length))
        if read < C.snd_pcm_sframes_t(length) {
            errno := C.snd_pcm_recover(self.capture, C.int(read), 0)
            if errno < 0 {
                panic(fmt.Sprint( "While reading from ALSA device, failed to recover from error: ", errno))
            }
        }

        // snd_pcm_readi gives us a one dimensional array of interleaved data
        // but what we want is a two dimensional array of samples
        chans := int(self.header.Channels)
		for slice, i := 0, 0; i < length; slice, i = slice + 1, i + chans {
			buff[slice] = make([]float32, chans)
			buff[slice] = cbuf[i : i + chans]
        }

        //send it on down the line
        self.ctx.Sink <- buff
    }

}

func (self *AlsaSource) Stop() os.Error {
    C.snd_pcm_close(self.capture)
    close(self.ctx.Sink)
    return nil
}

/////
// Alsa Sink
// Outputs to speakers via ALSA
type AlsaSink struct {
    ctx *afp.Context
    header afp.StreamHeader
    playback *C.snd_pcm_t
    params *C.snd_pcm_hw_params_t

}

func NewAlsaSink() afp.Filter {
    return &AlsaSink{}
}

func (self *AlsaSink) GetType() int {
    return afp.PIPE_SINK
}

func (self *AlsaSink) Init(ctx *afp.Context, args []string) os.Error {
    self.ctx = ctx
    return nil
}

func (self *AlsaSink) Start() {

    const FRAMES int = 256 //the double buffer holds 256 frames
    self.header = <-self.ctx.HeaderSource

    retval := self.prepare()
    if (retval != nil) {
        panic(retval)
    }

    //almost a do..while
    var cbuf []float32 //C buffer
    var written chan C.snd_pcm_sframes_t
    chans := int(self.header.Channels)
    double := make([][]float32, 0, self.header.FrameSize * 1024)
    double = append(double, <-self.ctx.Source...)
    length := len(double)
    oldLength := length
    written <- C.snd_pcm_sframes_t(length)

    for buffer := range self.ctx.Source { //blocking

        select {

        //wait for the previous write to finish
        case error := <-written:

            if int(error) < oldLength {
                //not all the data was written to the device
                panic(fmt.Sprintf("Could not write all data to ALSA device, wrote: ", written))
            }

            //write to the speaker in another thread
            go func() {
                oldLength = length
                written <- C.snd_pcm_writei(self.playback, unsafe.Pointer(&cbuf[0]), C.snd_pcm_uframes_t(length))
            }()

            double = buffer

        default:
            double = append(buffer, <-self.ctx.Source...)
            length = len(double)

            //cbuf WILL be a new pointer, so we can gaurantee that the address space given to snd_pcm_writei
            //will not be modified once it is passed; but on the down side, we rely on the garbage collector
            //to take care of many stale slices created while filling our buffer
            cbuf = make([]float32, int32(self.header.Channels) * self.header.FrameSize * int32(length))

            //interleave the channels
            streamOffset := 0
            for i := 0; i < length; i++ {
                for j := 0; j < chans; j++ {
                    cbuf[streamOffset] = double[i][j]
                    streamOffset++
                }
            }

        }//end select

    }

    return
}

func (self *AlsaSink) Stop() os.Error {
    close(self.ctx.Source)
    C.snd_pcm_close(self.playback)
    return nil
}

// Ugly bastardized C code follows
func (self *AlsaSink) prepare() os.Error {

    if errno := C.snd_pcm_open(&self.playback, C.CString("default"), C.SND_PCM_STREAM_PLAYBACK, 0); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not open device. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_malloc(&self.params); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not allocate hardware parameter structure. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_any(self.playback, self.params); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not initialize hardware parameter structure. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_access(self.playback, self.params, C.SND_PCM_ACCESS_RW_INTERLEAVED); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set access type. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_format(self.playback, self.params, C.SND_PCM_FORMAT_FLOAT); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set sample format. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_rate(self.playback, self.params, C.uint(self.header.SampleRate), 0); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set sample rate. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_channels(self.playback, self.params, C.uint(self.header.Channels)); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set channel count. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params(self.playback, self.params); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set hardware parameters. Error %d", errno) )
    }

    C.snd_pcm_hw_params_free(self.params)

    if errno := C.snd_pcm_prepare(self.playback); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not prepare audio device for use. Error %d", errno) )
    }

    return nil
}

//this one is slightly different
//note the change in scope
func (self *AlsaSource) prepare() os.Error {

    if errno := C.snd_pcm_open(&self.capture, C.CString("default"), C.SND_PCM_STREAM_CAPTURE, 0); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not open device. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_malloc(&self.params); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not allocate hardware parameters. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_any(self.capture, self.params); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not initialize hardware parameter structure. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_access(self.capture, self.params, C.SND_PCM_ACCESS_RW_INTERLEAVED); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set access. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_format(self.capture, self.params, C.SND_PCM_FORMAT_FLOAT); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set sample format. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_rate(self.capture, self.params, C.uint(self.header.SampleRate), 0); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set sample rate. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params_set_channels(self.capture, self.params, 1); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set channel count. Error %d", errno) )
    }

    if errno := C.snd_pcm_hw_params(self.capture, self.params); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not set parameters. Error %d", errno) )
    }

    C.snd_pcm_hw_params_free(self.params)

    if errno := C.snd_pcm_prepare(self.capture); errno < 0 {
        return os.NewError( fmt.Sprintf("Could not prepare audio interface for use. Error %d", errno) )
    }

    return nil
}
<|endoftext|>


<commit_before>package main

import (
	"polydawn.net/siphon"
	"github.com/kr/pty"
	"bytes"
	"io"
	"os/exec"
	"testing"
	"time"
	"github.com/coocood/assrt"
)

/**
Expect:
 - all of the input to come back out, because terminals default to echo mode.
 - then the grep'd string should come out, because the command recieved it and matched.
 - the grep'd string should come out surrounded by the escape codes for color, since grep's auto mode should detect that we're in a terminal.
*/
func TestPtySanity(t *testing.T) {
	assert := assrt.NewAssert(t)

	c := exec.Command("grep", "--color=auto", "bar")
	f, err := pty.Start(c)
	if err != nil {
		t.Fatal(err)
	}

	go func() {
		f.Write([]byte("foo\nbar\nbaz\n"))
		/*
		All of the input must be written in a single call to prevent this test from occationally failing nondeterministically, because:
		grep operates stream-wise and will start printing output before it has all of its input,
		and 3/4ths of the output lines are actually from the terminal operating in echo mode on the same channel.
		So there's actually a race between the terminal itself (somewhere down in kernel land I suppose?) and the output of grep.
		*/
		f.Write([]byte{4}) // EOT
	}()

	outBuffer := new(bytes.Buffer)
	io.Copy(outBuffer, f)
	out := string(outBuffer.Bytes())

	expected := // I have no idea where the CR characters come from.
		"foo\r\n"+
		"bar\r\n"+
		"baz\r\n"+
		"bar\r\n";

	assert.Equal(
		expected,
		out,
	)
}

func TestGrep(t *testing.T) {
	assert := assrt.NewAssert(t)

	cmd := exec.Command("grep", "--color=auto", "bar")
	host := siphon.NewHost(cmd, siphon.NewInternalAddr())
	host.Start()

	go func() {
		stdin := host.StdinPipe()
		stdin.Write([]byte("foo\nbar\nbaz\n"))
		stdin.Write([]byte{4}) // EOT
	}()

	outBuffer := new(bytes.Buffer)
	io.Copy(outBuffer, host.StdoutPipe())
	out := string(outBuffer.Bytes())

	expected := // I have no idea where the CR characters come from.
		"foo\r\n"+
		"bar\r\n"+
		"baz\r\n"+
		"bar\r\n";

	assert.Equal(
		expected,
		out,
	)
}

/**
Getting any answer from the tty command at all is pretty good news, since `go test` doesn't let the shell's tty come through.
*/
func TestActuallyNewTty(t *testing.T) {
	assert := assrt.NewAssert(t)

	cmd := exec.Command("tty")
	host := siphon.NewHost(cmd, siphon.NewInternalAddr())
	outBuffer := new(bytes.Buffer)
	hostOut := host.StdoutPipe()
	host.Start()
	io.Copy(outBuffer, hostOut)

	innerTty := string(outBuffer.Bytes())

	assert.NotEqual(
		"", // what exactly you get varies on the current state of your machine, but something like /dev/pts/12 is reasonable.
		innerTty,
	)
}

func TestUnixSocket(t *testing.T) {
	assert := assrt.NewAssert(t)

	addr := siphon.NewAddr("test", "unix", "test.sock")

	cmd := exec.Command("cat", "-")
	host := siphon.NewHost(cmd, addr)
	host.Serve(); defer host.UnServe()

	client := siphon.NewClient(addr)
	client.Connect()
	//client.Attach() // you can't do this in a test.  there's no tty.

	//FIXME: there's really no guarantee that the host finished processing the client connect request by now
	host.Start()

	go func() {
		stdin := client.Stdin()
		stdin.Write([]byte("foo\nbar\nbaz\n"))
		stdin.Write([]byte{4}) // EOT
	}()

	outBuffer := new(bytes.Buffer)
	io.Copy(outBuffer, client.Stdout())
	out := string(outBuffer.Bytes())

	expected :=
		"foo\r\n"+
		"bar\r\n"+
		"baz\r\n";
	expected = expected + expected
	assert.Equal(
		expected,
		out,
	)
}

func TestDoubleClient(t *testing.T) {
	assert := assrt.NewAssert(t)

	addr := siphon.NewAddr("test", "unix", "test.sock")

	cmd := exec.Command("cat", "-")
	host := siphon.NewHost(cmd, addr)
	host.Serve(); defer host.UnServe()

	client1 := siphon.NewClient(addr)
	client1.Connect()

	client2 := siphon.NewClient(addr)
	client2.Connect()

	host.Start()

	go func() {
		// timed pauses are a hack required to make sure the host side flushes the echos in an order we can test.
		client1.Stdin().Write([]byte("foo\n"))
		time.Sleep(50*time.Millisecond)
		client2.Stdin().Write([]byte("bar\n"))
		time.Sleep(50*time.Millisecond)
		client1.Stdin().Write([]byte("baz\n"))
		time.Sleep(50*time.Millisecond)
		client2.Stdin().Write([]byte{4}) // EOT
	}()

	outBuffer1 := new(bytes.Buffer)
	io.Copy(outBuffer1, client1.Stdout())
	out1 := string(outBuffer1.Bytes())

	outBuffer2 := new(bytes.Buffer)
	io.Copy(outBuffer2, client2.Stdout())
	out2 := string(outBuffer2.Bytes())

	expected :=
		"foo\r\nfoo\r\n"+
		"bar\r\nbar\r\n"+
		"baz\r\nbaz\r\n";
	assert.Equal(
		expected,
		out1,
	)
	assert.Equal(
		expected,
		out2,
	)
}

func Template(t *testing.T) {
	assert := assrt.NewAssert(t)
	assert.Equal(
		"yes",
		"no",
	)
}
<commit_msg>fix method signatures in tests; test pass again.<commit_after>package main

import (
	"polydawn.net/siphon"
	"github.com/kr/pty"
	"bytes"
	"io"
	"os/exec"
	"testing"
	"time"
	"github.com/coocood/assrt"
)

/**
Expect:
 - all of the input to come back out, because terminals default to echo mode.
 - then the grep'd string should come out, because the command recieved it and matched.
 - the grep'd string should come out surrounded by the escape codes for color, since grep's auto mode should detect that we're in a terminal.
*/
func TestPtySanity(t *testing.T) {
	assert := assrt.NewAssert(t)

	c := exec.Command("grep", "--color=auto", "bar")
	f, err := pty.Start(c)
	if err != nil {
		t.Fatal(err)
	}

	go func() {
		f.Write([]byte("foo\nbar\nbaz\n"))
		/*
		All of the input must be written in a single call to prevent this test from occationally failing nondeterministically, because:
		grep operates stream-wise and will start printing output before it has all of its input,
		and 3/4ths of the output lines are actually from the terminal operating in echo mode on the same channel.
		So there's actually a race between the terminal itself (somewhere down in kernel land I suppose?) and the output of grep.
		*/
		f.Write([]byte{4}) // EOT
	}()

	outBuffer := new(bytes.Buffer)
	io.Copy(outBuffer, f)
	out := string(outBuffer.Bytes())

	expected := // I have no idea where the CR characters come from.
		"foo\r\n"+
		"bar\r\n"+
		"baz\r\n"+
		"bar\r\n";

	assert.Equal(
		expected,
		out,
	)
}

func TestGrep(t *testing.T) {
	assert := assrt.NewAssert(t)

	cmd := exec.Command("grep", "--color=auto", "bar")
	host := siphon.NewHost(cmd, siphon.NewInternalAddr())
	host.Start()

	go func() {
		stdin := host.StdinPipe()
		stdin.Write([]byte("foo\nbar\nbaz\n"))
		stdin.Write([]byte{4}) // EOT
	}()

	outBuffer := new(bytes.Buffer)
	io.Copy(outBuffer, host.StdoutPipe())
	out := string(outBuffer.Bytes())

	expected := // I have no idea where the CR characters come from.
		"foo\r\n"+
		"bar\r\n"+
		"baz\r\n"+
		"bar\r\n";

	assert.Equal(
		expected,
		out,
	)
}

/**
Getting any answer from the tty command at all is pretty good news, since `go test` doesn't let the shell's tty come through.
*/
func TestActuallyNewTty(t *testing.T) {
	assert := assrt.NewAssert(t)

	cmd := exec.Command("tty")
	host := siphon.NewHost(cmd, siphon.NewInternalAddr())
	outBuffer := new(bytes.Buffer)
	hostOut := host.StdoutPipe()
	host.Start()
	io.Copy(outBuffer, hostOut)

	innerTty := string(outBuffer.Bytes())

	assert.NotEqual(
		"", // what exactly you get varies on the current state of your machine, but something like /dev/pts/12 is reasonable.
		innerTty,
	)
}

func TestUnixSocket(t *testing.T) {
	assert := assrt.NewAssert(t)

	addr := siphon.NewAddr("test", "unix", "test.sock")

	cmd := exec.Command("cat", "-")
	host := siphon.NewHost(cmd, addr)
	host.Serve(); defer host.UnServe()

	client := siphon.Connect(addr)
	client.Connect()
	//client.Attach() // you can't do this in a test.  there's no tty.

	//FIXME: there's really no guarantee that the host finished processing the client connect request by now
	host.Start()

	go func() {
		stdin := client.Stdin()
		stdin.Write([]byte("foo\nbar\nbaz\n"))
		stdin.Write([]byte{4}) // EOT
	}()

	outBuffer := new(bytes.Buffer)
	io.Copy(outBuffer, client.Stdout())
	out := string(outBuffer.Bytes())

	expected :=
		"foo\r\n"+
		"bar\r\n"+
		"baz\r\n";
	expected = expected + expected
	assert.Equal(
		expected,
		out,
	)
}

func TestDoubleClient(t *testing.T) {
	assert := assrt.NewAssert(t)

	addr := siphon.NewAddr("test", "unix", "test.sock")

	cmd := exec.Command("cat", "-")
	host := siphon.NewHost(cmd, addr)
	host.Serve(); defer host.UnServe()

	client1 := siphon.Connect(addr)
	client1.Connect()

	client2 := siphon.Connect(addr)
	client2.Connect()

	host.Start()

	go func() {
		// timed pauses are a hack required to make sure the host side flushes the echos in an order we can test.
		client1.Stdin().Write([]byte("foo\n"))
		time.Sleep(50*time.Millisecond)
		client2.Stdin().Write([]byte("bar\n"))
		time.Sleep(50*time.Millisecond)
		client1.Stdin().Write([]byte("baz\n"))
		time.Sleep(50*time.Millisecond)
		client2.Stdin().Write([]byte{4}) // EOT
	}()

	outBuffer1 := new(bytes.Buffer)
	io.Copy(outBuffer1, client1.Stdout())
	out1 := string(outBuffer1.Bytes())

	outBuffer2 := new(bytes.Buffer)
	io.Copy(outBuffer2, client2.Stdout())
	out2 := string(outBuffer2.Bytes())

	expected :=
		"foo\r\nfoo\r\n"+
		"bar\r\nbar\r\n"+
		"baz\r\nbaz\r\n";
	assert.Equal(
		expected,
		out1,
	)
	assert.Equal(
		expected,
		out2,
	)
}

func Template(t *testing.T) {
	assert := assrt.NewAssert(t)
	assert.Equal(
		"yes",
		"no",
	)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"os";
	"syscall";
)

// ForkExec forks the current process and invokes Exec with the file, arguments,
// and environment specified by argv0, argv, and envv.  It returns the process
// id of the forked process and an Error, if any.  The fd array specifies the
// file descriptors to be set up in the new process: fd[0] will be Unix file
// descriptor 0 (standard input), fd[1] descriptor 1, and so on.  A nil entry
// will cause the child to have no open file descriptor with that index.
// If dir is not empty, the child chdirs into the directory before execing the program.
func ForkExec(argv0 string, argv []string, envv []string, dir string, fd []*File)
	(pid int, err Error)
{
	// Create array of integer (system) fds.
	intfd := make([]int, len(fd));
	for i, f := range fd {
		if f == nil {
			intfd[i] = -1;
		} else {
			intfd[i] = f.Fd();
		}
	}

	p, e := syscall.ForkExec(argv0, argv, envv, dir, intfd);
	if e != 0 {
		return 0, &PathError{"fork/exec", argv0, Errno(e)};
	}
	return p, nil;
}

// Exec replaces the current process with an execution of the program
// named by argv0, with arguments argv and environment envv.
// If successful, Exec never returns.  If it fails, it returns an Error.
// ForkExec is almost always a better way to execute a program.
func Exec(argv0 string, argv []string, envv []string) Error {
	if envv == nil {
		envv = Environ();
	}
	e := syscall.Exec(argv0, argv, envv);
	if e != 0 {
		return &PathError{"exec", argv0, Errno(e)};
	}
	return nil;
}

// TODO(rsc): Should os implement its own syscall.WaitStatus
// wrapper with the methods, or is exposing the underlying one enough?
//
// TODO(rsc): Certainly need to have os.Rusage struct,
// since syscall one might have different field types across
// different OS.

// Waitmsg stores the information about an exited process as reported by Wait.
type Waitmsg struct {
	Pid int;	// The process's id.
	syscall.WaitStatus;	// System-dependent status info.
	Rusage *syscall.Rusage;	// System-dependent resource usage info.
}

// Options for Wait.
const (
	WNOHANG = syscall.WNOHANG;	// Don't wait if no process has exited.
	WSTOPPED = syscall.WSTOPPED;	// If set, status of stopped subprocesses is also reported.
	WUNTRACED = WSTOPPED;
	WRUSAGE = 1<<30;	// Record resource usage.
)

// Wait waits for process pid to exit or stop, and then returns a
// Waitmsg describing its status and an Error, if any. The options
// (WNOHANG etc.) affect the behavior of the Wait call.
func Wait(pid int, options int) (w *Waitmsg, err Error) {
	var status syscall.WaitStatus;
	var rusage *syscall.Rusage;
	if options & WRUSAGE != 0 {
		rusage = new(syscall.Rusage);
		options ^= WRUSAGE;
	}
	pid1, e := syscall.Wait4(pid, &status, options, rusage);
	if e != 0 {
		return nil, NewSyscallError("wait", e);
	}
	w = new(Waitmsg);
	w.Pid = pid1;
	w.WaitStatus = status;
	w.Rusage = rusage;
	return w, nil;
}

// Getpid returns the process id of the caller.
func Getpid() int {
	p, r2, e := syscall.Syscall(syscall.SYS_GETPID, 0, 0, 0);
	return int(p)
}

// Getppid returns the process id of the caller's parent.
func Getppid() int {
	p, r2, e := syscall.Syscall(syscall.SYS_GETPPID, 0, 0, 0);
	return int(p)
}
<commit_msg>Avoid clash between os.WRUSAGE and Linux's WALL flag.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"os";
	"syscall";
)

// ForkExec forks the current process and invokes Exec with the file, arguments,
// and environment specified by argv0, argv, and envv.  It returns the process
// id of the forked process and an Error, if any.  The fd array specifies the
// file descriptors to be set up in the new process: fd[0] will be Unix file
// descriptor 0 (standard input), fd[1] descriptor 1, and so on.  A nil entry
// will cause the child to have no open file descriptor with that index.
// If dir is not empty, the child chdirs into the directory before execing the program.
func ForkExec(argv0 string, argv []string, envv []string, dir string, fd []*File)
	(pid int, err Error)
{
	// Create array of integer (system) fds.
	intfd := make([]int, len(fd));
	for i, f := range fd {
		if f == nil {
			intfd[i] = -1;
		} else {
			intfd[i] = f.Fd();
		}
	}

	p, e := syscall.ForkExec(argv0, argv, envv, dir, intfd);
	if e != 0 {
		return 0, &PathError{"fork/exec", argv0, Errno(e)};
	}
	return p, nil;
}

// Exec replaces the current process with an execution of the program
// named by argv0, with arguments argv and environment envv.
// If successful, Exec never returns.  If it fails, it returns an Error.
// ForkExec is almost always a better way to execute a program.
func Exec(argv0 string, argv []string, envv []string) Error {
	if envv == nil {
		envv = Environ();
	}
	e := syscall.Exec(argv0, argv, envv);
	if e != 0 {
		return &PathError{"exec", argv0, Errno(e)};
	}
	return nil;
}

// TODO(rsc): Should os implement its own syscall.WaitStatus
// wrapper with the methods, or is exposing the underlying one enough?
//
// TODO(rsc): Certainly need to have os.Rusage struct,
// since syscall one might have different field types across
// different OS.

// Waitmsg stores the information about an exited process as reported by Wait.
type Waitmsg struct {
	Pid int;	// The process's id.
	syscall.WaitStatus;	// System-dependent status info.
	Rusage *syscall.Rusage;	// System-dependent resource usage info.
}

// Options for Wait.
const (
	WNOHANG = syscall.WNOHANG;	// Don't wait if no process has exited.
	WSTOPPED = syscall.WSTOPPED;	// If set, status of stopped subprocesses is also reported.
	WUNTRACED = WSTOPPED;
	WRUSAGE = 1<<20;	// Record resource usage.
)

// WRUSAGE must not be too high a bit, to avoid clashing with Linux's
// WCLONE, WALL, and WNOTHREAD flags, which sit in the top few bits of
// the options

// Wait waits for process pid to exit or stop, and then returns a
// Waitmsg describing its status and an Error, if any. The options
// (WNOHANG etc.) affect the behavior of the Wait call.
func Wait(pid int, options int) (w *Waitmsg, err Error) {
	var status syscall.WaitStatus;
	var rusage *syscall.Rusage;
	if options & WRUSAGE != 0 {
		rusage = new(syscall.Rusage);
		options ^= WRUSAGE;
	}
	pid1, e := syscall.Wait4(pid, &status, options, rusage);
	if e != 0 {
		return nil, NewSyscallError("wait", e);
	}
	w = new(Waitmsg);
	w.Pid = pid1;
	w.WaitStatus = status;
	w.Rusage = rusage;
	return w, nil;
}

// Getpid returns the process id of the caller.
func Getpid() int {
	p, r2, e := syscall.Syscall(syscall.SYS_GETPID, 0, 0, 0);
	return int(p)
}

// Getppid returns the process id of the caller's parent.
func Getppid() int {
	p, r2, e := syscall.Syscall(syscall.SYS_GETPPID, 0, 0, 0);
	return int(p)
}
<|endoftext|>


<commit_before>package regexplanet

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"runtime"
	"time"
)

func init() {
	http.HandleFunc("/", root_handler)
	http.HandleFunc("/status.json", status_handler)
	http.HandleFunc("/test.json", test_handler)
}

func root_handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, "Hello, RegexPlanet!")
}

type Status struct {
	Success  bool
	Hostname string
	Getwd    string
	TempDir  string
	Environ  []string
	Version  string
	Seconds  int64
}

func status_handler(w http.ResponseWriter, r *http.Request) {
	var err error
	status := Status{}

	status.Getwd, err = os.Getwd()
	if err != nil {
		status.Getwd = "ERROR!"
	}

	status.Hostname, err = os.Hostname()
	if err != nil {
		status.Hostname = "ERROR"
	}

	status.TempDir = os.TempDir()
	status.Environ = os.Environ()
	status.Version = runtime.Version()
	status.Seconds = time.Now().Unix()
	status.Success = true

	var b []byte
	b, err = json.Marshal(status)
	if err != nil {
		return
	}

	if b[2] == 'S' { // HACK: it doesn't get much hackier than this, but json.Marshal doesn't marshal lower-case members.  Is there a way around this?
		b[2] = 's'
	}

	w.Header().Set("Content-Type", "text/plain; charset=utf8")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Methods", "POST, GET")
	w.Header().Set("Access-Control-Max-Age", "604800") // 1 week

	var callback string = r.FormValue("callback")
	if callback > "" {
		w.Write([]byte(callback))
		w.Write([]byte("("))
		w.Write(b)
		w.Write([]byte(");"))
	} else {
		w.Write(b)
	}
}

type TestResult struct {
	Success bool
	Html    string
}

func test_handler(w http.ResponseWriter, r *http.Request) {

	w.Header().Set("Content-Type", "text/plain; charset=utf8")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Methods", "POST, GET")
	w.Header().Set("Access-Control-Max-Age", "604800") // 1 week

	retVal := TestResult{}

	retVal.Success = true
	retVal.Html = "
Actually, it is a lot less than beta: the real code isn't even written yet!
"

	var err error
	var b []byte
	b, err = json.Marshal(retVal)
	if err != nil {
		fmt.Fprint(w, "{\"success\":false,\"html\":\"
json.Marshal failed
\"}")
		return
	}

	if b[2] == 'S' { // HACK: it doesn't get much hackier than this, but json.Marshal doesn't marshal lower-case members.  Is there a way around this?
		b[2] = 's'
	}

	if b[17] == 'H' {
		b[17] = 'h'
	}

	w.Write(b)
}
<commit_msg>fixed json naming hack<commit_after>package regexplanet

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"runtime"
	"time"
)

func init() {
	http.HandleFunc("/", root_handler)
	http.HandleFunc("/status.json", status_handler)
	http.HandleFunc("/test.json", test_handler)
}

func root_handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, "Hello, RegexPlanet!")
}

func write_with_callback(w http.ResponseWriter, callback string, v interface{}) {

	w.Header().Set("Content-Type", "text/plain; charset=utf8")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Methods", "POST, GET")
	w.Header().Set("Access-Control-Max-Age", "604800") // 1 week

	var b []byte
	var err error
	b, err = json.Marshal(v)
	if err != nil {
		b = []byte("{\"success\":false,\"html\":\"
json.Marshal failed
\"}")
	}

	if callback > "" {
		w.Write([]byte(callback))
		w.Write([]byte("("))
		w.Write(b)
		w.Write([]byte(");"))
	} else {
		w.Write(b)
	}
}

type Status struct {
	Success  bool		`json:"success"`
	Hostname string
	Getwd    string
	TempDir  string
	Environ  []string
	Version  string
	Seconds  int64
}

func status_handler(w http.ResponseWriter, r *http.Request) {
	var err error
	status := Status{}

	status.Getwd, err = os.Getwd()
	if err != nil {
		status.Getwd = "ERROR!"
	}

	status.Hostname, err = os.Hostname()
	if err != nil {
		status.Hostname = "ERROR"
	}

	status.TempDir = os.TempDir()
	status.Environ = os.Environ()
	status.Version = runtime.Version()
	status.Seconds = time.Now().Unix()
	status.Success = true

	write_with_callback(w, r.FormValue("callback"), status)
}

type TestResult struct {
	Success bool		`json:"success"`
	Html    string		`json:"html"`
	Message	string		`json:"message,omitempty"`
}

func test_handler(w http.ResponseWriter, r *http.Request) {

	retVal := TestResult{}

	retVal.Success = true
	retVal.Html = "
Actually, it is a lot less than beta: the real code isn't even written yet!
"

	write_with_callback(w, r.FormValue("callback"), retVal)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/yuin/gopher-lua"
	"gopkg.in/readline.v1"
	"log"
	"strings"
)

const (
	PS1 = "\033[1;31m> \033[0m"
	PS2 = "\033[1;31m>> \033[0m"
)

type REPL struct {
	L       *lua.LState // the lua virtual machine
	toolbox *ToolBox
	rl      *readline.Instance
}

func NewREPL() *REPL {
	repl := new(REPL)
	repl.L = lua.NewState()
	repl.toolbox = NewToolBox("/data")
	if rl, err := readline.New(PS1); err == nil {
		repl.rl = rl
	} else {
		log.Println(err)
		return nil
	}
	return repl
}

// read/eval/print/loop
func (repl *REPL) start() {
	defer func() {
		repl.rl.Close()
	}()

	for {
		if str, err := repl.loadline(); err == nil {
			repl.toolbox.exec(str)
		} else {
			log.Println(err)
			break
		}
	}
}

func incomplete(err error) bool {
	if strings.Index(err.Error(), "EOF") != -1 {
		return true
	}
	return false
}

func (repl *REPL) loadline() (string, error) {
	repl.rl.SetPrompt(PS1)
	if line, err := repl.rl.Readline(); err == nil {
		if _, err := repl.L.LoadString("return " + line); err == nil { // try add return <...> then compile
			return line, nil
		} else {
			return repl.multiline(line)
		}
	} else {
		return "", err
	}
}

func (repl *REPL) multiline(ml string) (string, error) {
	for {
		if _, err := repl.L.LoadString(ml); err == nil { // try compile
			return ml, nil
		} else if !incomplete(err) { // syntax error, but not EOF
			return ml, nil
		} else { // read next line
			repl.rl.SetPrompt(PS2)
			if line, err := repl.rl.Readline(); err == nil {
				ml = ml + "\n" + line
			} else {
				return "", err
			}
		}
	}
}

func main() {
	repl := NewREPL()
	repl.start()
}
<commit_msg>clean<commit_after>package main

import (
	"github.com/yuin/gopher-lua"
	"gopkg.in/readline.v1"
	"log"
	"strings"
)

const (
	PS1 = "\033[1;31m> \033[0m"
	PS2 = "\033[1;31m>> \033[0m"
)

type REPL struct {
	L       *lua.LState // the lua virtual machine
	toolbox *ToolBox
	rl      *readline.Instance
}

func NewREPL() *REPL {
	repl := new(REPL)
	repl.L = lua.NewState()
	repl.toolbox = NewToolBox("/data")
	if rl, err := readline.New(PS1); err == nil {
		repl.rl = rl
	} else {
		log.Println(err)
		return nil
	}
	return repl
}

func (repl *REPL) Close() {
	repl.rl.Close()
}

// read/eval/print/loop
func (repl *REPL) Start() {
	for {
		if str, err := repl.loadline(); err == nil {
			repl.toolbox.exec(str)
		} else {
			log.Println(err)
			break
		}
	}
}

func incomplete(err error) bool {
	if strings.Index(err.Error(), "EOF") != -1 {
		return true
	}
	return false
}

func (repl *REPL) loadline() (string, error) {
	repl.rl.SetPrompt(PS1)
	if line, err := repl.rl.Readline(); err == nil {
		if _, err := repl.L.LoadString("return " + line); err == nil { // try add return <...> then compile
			return line, nil
		} else {
			return repl.multiline(line)
		}
	} else {
		return "", err
	}
}

func (repl *REPL) multiline(ml string) (string, error) {
	for {
		if _, err := repl.L.LoadString(ml); err == nil { // try compile
			return ml, nil
		} else if !incomplete(err) { // syntax error, but not EOF
			return ml, nil
		} else { // read next line
			repl.rl.SetPrompt(PS2)
			if line, err := repl.rl.Readline(); err == nil {
				ml = ml + "\n" + line
			} else {
				return "", err
			}
		}
	}
}

func main() {
	repl := NewREPL()
	repl.Start()
	repl.Close()
}
<|endoftext|>


<commit_before>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilosa

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/pilosa/pilosa/internal"
	"github.com/pilosa/pilosa/pql"
)

// View layout modes.
const (
	ViewStandard = "standard"
	ViewInverse  = "inverse"

	ViewFieldPrefix = "field_"
)

// IsValidView returns true if name is valid.
func IsValidView(name string) bool {
	return name == ViewStandard || name == ViewInverse
}

// View represents a container for frame data.
type View struct {
	mu    sync.RWMutex
	path  string
	index string
	frame string
	name  string

	cacheSize uint32

	// Fragments by slice.
	cacheType string // passed in by frame
	fragments map[uint64]*Fragment

	// maxSlice maintains this view's max slice in order to
	// prevent sending multiple `CreateSliceMessage` messages
	maxSlice uint64

	broadcaster Broadcaster
	stats       StatsClient

	RowAttrStore *AttrStore
	LogOutput    io.Writer
}

// NewView returns a new instance of View.
func NewView(path, index, frame, name string, cacheSize uint32) *View {
	return &View{
		path:      path,
		index:     index,
		frame:     frame,
		name:      name,
		cacheSize: cacheSize,

		cacheType: DefaultCacheType,
		fragments: make(map[uint64]*Fragment),

		broadcaster: NopBroadcaster,
		stats:       NopStatsClient,
		LogOutput:   ioutil.Discard,
	}
}

// Name returns the name the view was initialized with.
func (v *View) Name() string { return v.name }

// Index returns the index name the view was initialized with.
func (v *View) Index() string { return v.index }

// Frame returns the frame name the view was initialized with.
func (v *View) Frame() string { return v.frame }

// Path returns the path the view was initialized with.
func (v *View) Path() string { return v.path }

// Open opens and initializes the view.
func (v *View) Open() error {
	if err := func() error {
		// Ensure the view's path exists.
		if err := os.MkdirAll(v.path, 0777); err != nil {
			return err
		} else if err := os.MkdirAll(filepath.Join(v.path, "fragments"), 0777); err != nil {
			return err
		}

		if err := v.openFragments(); err != nil {
			return err
		}

		return nil
	}(); err != nil {
		v.Close()
		return err
	}

	return nil
}

// logger returns a logger instance for the view.
func (v *View) logger() *log.Logger { return log.New(v.LogOutput, "", log.LstdFlags) }

// openFragments opens and initializes the fragments inside the view.
func (v *View) openFragments() error {
	file, err := os.Open(filepath.Join(v.path, "fragments"))
	if os.IsNotExist(err) {
		return nil
	} else if err != nil {
		return err
	}
	defer file.Close()

	fis, err := file.Readdir(0)
	if err != nil {
		return err
	}

	for _, fi := range fis {
		if fi.IsDir() {
			continue
		}

		// Parse filename into integer.
		slice, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
		if err != nil {
			continue
		}

		frag := v.newFragment(v.FragmentPath(slice), slice)
		if err := frag.Open(); err != nil {
			return fmt.Errorf("open fragment: slice=%d, err=%s", frag.Slice(), err)
		}
		frag.RowAttrStore = v.RowAttrStore
		v.fragments[frag.Slice()] = frag
	}

	return nil
}

// Close closes the view and its fragments.
func (v *View) Close() error {
	v.mu.Lock()
	defer v.mu.Unlock()

	// Close all fragments.
	for _, frag := range v.fragments {
		if err := frag.Close(); err != nil {
			return err
		}
	}
	v.fragments = make(map[uint64]*Fragment)

	return nil
}

// MaxSlice returns the max slice in the view.
func (v *View) MaxSlice() uint64 {
	v.mu.RLock()
	defer v.mu.RUnlock()

	var max uint64
	for slice := range v.fragments {
		if slice > max {
			max = slice
		}
	}

	return max
}

// FragmentPath returns the path to a fragment in the view.
func (v *View) FragmentPath(slice uint64) string {
	return filepath.Join(v.path, "fragments", strconv.FormatUint(slice, 10))
}

// Fragment returns a fragment in the view by slice.
func (v *View) Fragment(slice uint64) *Fragment {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return v.fragment(slice)
}

func (v *View) fragment(slice uint64) *Fragment { return v.fragments[slice] }

// Fragments returns a list of all fragments in the view.
func (v *View) Fragments() []*Fragment {
	v.mu.Lock()
	defer v.mu.Unlock()

	other := make([]*Fragment, 0, len(v.fragments))
	for _, fragment := range v.fragments {
		other = append(other, fragment)
	}
	return other
}

// RecalculateCaches recalculates the cache on every fragment in the view.
func (v *View) RecalculateCaches() {
	for _, fragment := range v.Fragments() {
		fragment.RecalculateCache()
	}
}

// CreateFragmentIfNotExists returns a fragment in the view by slice.
func (v *View) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) {
	v.mu.Lock()
	defer v.mu.Unlock()
	return v.createFragmentIfNotExists(slice)
}

func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
	// Find fragment in cache first.
	if frag := v.fragments[slice]; frag != nil {
		return frag, nil
	}

	// Initialize and open fragment.
	frag := v.newFragment(v.FragmentPath(slice), slice)
	if err := frag.Open(); err != nil {
		return nil, err
	}
	frag.RowAttrStore = v.RowAttrStore

	// Broadcast a message that a new max slice was just created.
	if slice > v.maxSlice {
		v.maxSlice = slice

		// Send the create slice message to all nodes.
		err := v.broadcaster.SendAsync(
			&internal.CreateSliceMessage{
				Index:     v.index,
				Slice:     slice,
				IsInverse: IsInverseView(v.name),
			})
		if err != nil {
			return nil, err
		}
	}

	// Save to lookup.
	v.fragments[slice] = frag
	return frag, nil
}

func (v *View) newFragment(path string, slice uint64) *Fragment {
	frag := NewFragment(path, v.index, v.frame, v.name, slice)
	frag.CacheType = v.cacheType
	frag.CacheSize = v.cacheSize
	frag.LogOutput = v.LogOutput
	frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice))
	return frag
}

// DeleteFragment removes the fragment from the view.
func (v *View) DeleteFragment(slice uint64) error {

	fragment := v.fragments[slice]
	if fragment == nil {
		return ErrFragmentNotFound
	}

	v.logger().Printf("delete fragment: %d", slice)

	// Close data files before deletion.
	if err := fragment.Close(); err != nil {
		return err
	}

	// Delete fragment file.
	if err := os.Remove(fragment.Path()); err != nil {
		return err
	}

	// Delete fragment cache file.
	if err := os.Remove(fragment.CachePath()); err != nil {
		v.logger().Printf("no cache file to delete for slice %d", slice)
	}

	delete(v.fragments, slice)

	return nil
}

// SetBit sets a bit within the view.
func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) {
	slice := columnID / SliceWidth
	frag, err := v.CreateFragmentIfNotExists(slice)
	if err != nil {
		return changed, err
	}
	return frag.SetBit(rowID, columnID)
}

// ClearBit clears a bit within the view.
func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) {
	slice := columnID / SliceWidth
	frag, err := v.CreateFragmentIfNotExists(slice)
	if err != nil {
		return changed, err
	}
	return frag.ClearBit(rowID, columnID)
}

// FieldValue uses a column of bits to read a multi-bit value.
func (v *View) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
	slice := columnID / SliceWidth
	frag, err := v.CreateFragmentIfNotExists(slice)
	if err != nil {
		return value, exists, err
	}
	return frag.FieldValue(columnID, bitDepth)
}

// SetFieldValue uses a column of bits to set a multi-bit value.
func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
	slice := columnID / SliceWidth
	frag, err := v.CreateFragmentIfNotExists(slice)
	if err != nil {
		return changed, err
	}
	return frag.SetFieldValue(columnID, bitDepth, value)
}

// FieldSum returns the sum & count of a field.
func (v *View) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err error) {
	for _, f := range v.Fragments() {
		fsum, fcount, err := f.FieldSum(filter, bitDepth)
		if err != nil {
			return sum, count, err
		}
		sum += fsum
		count += fcount
	}
	return sum, count, nil
}

// FieldRange returns bitmaps with a field value encoding matching the predicate.
func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) {
	bm := NewBitmap()
	for _, frag := range v.Fragments() {
		other, err := frag.FieldRange(op, bitDepth, predicate)
		if err != nil {
			return nil, err
		}
		bm = bm.Union(other)
	}
	return bm, nil
}

// FieldRangeBetween returns bitmaps with a field value encoding matching any
// value between predicateMin and predicateMax.
func (v *View) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Bitmap, error) {
	bm := NewBitmap()
	for _, frag := range v.Fragments() {
		other, err := frag.FieldRangeBetween(bitDepth, predicateMin, predicateMax)
		if err != nil {
			return nil, err
		}
		bm = bm.Union(other)
	}
	return bm, nil
}

// IsInverseView returns true if the view is used for storing an inverted representation.
func IsInverseView(name string) bool {
	return strings.HasPrefix(name, ViewInverse)
}

type viewSlice []*View

func (p viewSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p viewSlice) Len() int           { return len(p) }
func (p viewSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }

// ViewInfo represents schema information for a view.
type ViewInfo struct {
	Name string `json:"name"`
}

type viewInfoSlice []*ViewInfo

func (p viewInfoSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p viewInfoSlice) Len() int           { return len(p) }
func (p viewInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
<commit_msg>add index/frame/view info to delete fragment log information<commit_after>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilosa

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/pilosa/pilosa/internal"
	"github.com/pilosa/pilosa/pql"
)

// View layout modes.
const (
	ViewStandard = "standard"
	ViewInverse  = "inverse"

	ViewFieldPrefix = "field_"
)

// IsValidView returns true if name is valid.
func IsValidView(name string) bool {
	return name == ViewStandard || name == ViewInverse
}

// View represents a container for frame data.
type View struct {
	mu    sync.RWMutex
	path  string
	index string
	frame string
	name  string

	cacheSize uint32

	// Fragments by slice.
	cacheType string // passed in by frame
	fragments map[uint64]*Fragment

	// maxSlice maintains this view's max slice in order to
	// prevent sending multiple `CreateSliceMessage` messages
	maxSlice uint64

	broadcaster Broadcaster
	stats       StatsClient

	RowAttrStore *AttrStore
	LogOutput    io.Writer
}

// NewView returns a new instance of View.
func NewView(path, index, frame, name string, cacheSize uint32) *View {
	return &View{
		path:      path,
		index:     index,
		frame:     frame,
		name:      name,
		cacheSize: cacheSize,

		cacheType: DefaultCacheType,
		fragments: make(map[uint64]*Fragment),

		broadcaster: NopBroadcaster,
		stats:       NopStatsClient,
		LogOutput:   ioutil.Discard,
	}
}

// Name returns the name the view was initialized with.
func (v *View) Name() string { return v.name }

// Index returns the index name the view was initialized with.
func (v *View) Index() string { return v.index }

// Frame returns the frame name the view was initialized with.
func (v *View) Frame() string { return v.frame }

// Path returns the path the view was initialized with.
func (v *View) Path() string { return v.path }

// Open opens and initializes the view.
func (v *View) Open() error {
	if err := func() error {
		// Ensure the view's path exists.
		if err := os.MkdirAll(v.path, 0777); err != nil {
			return err
		} else if err := os.MkdirAll(filepath.Join(v.path, "fragments"), 0777); err != nil {
			return err
		}

		if err := v.openFragments(); err != nil {
			return err
		}

		return nil
	}(); err != nil {
		v.Close()
		return err
	}

	return nil
}

// logger returns a logger instance for the view.
func (v *View) logger() *log.Logger { return log.New(v.LogOutput, "", log.LstdFlags) }

// openFragments opens and initializes the fragments inside the view.
func (v *View) openFragments() error {
	file, err := os.Open(filepath.Join(v.path, "fragments"))
	if os.IsNotExist(err) {
		return nil
	} else if err != nil {
		return err
	}
	defer file.Close()

	fis, err := file.Readdir(0)
	if err != nil {
		return err
	}

	for _, fi := range fis {
		if fi.IsDir() {
			continue
		}

		// Parse filename into integer.
		slice, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
		if err != nil {
			continue
		}

		frag := v.newFragment(v.FragmentPath(slice), slice)
		if err := frag.Open(); err != nil {
			return fmt.Errorf("open fragment: slice=%d, err=%s", frag.Slice(), err)
		}
		frag.RowAttrStore = v.RowAttrStore
		v.fragments[frag.Slice()] = frag
	}

	return nil
}

// Close closes the view and its fragments.
func (v *View) Close() error {
	v.mu.Lock()
	defer v.mu.Unlock()

	// Close all fragments.
	for _, frag := range v.fragments {
		if err := frag.Close(); err != nil {
			return err
		}
	}
	v.fragments = make(map[uint64]*Fragment)

	return nil
}

// MaxSlice returns the max slice in the view.
func (v *View) MaxSlice() uint64 {
	v.mu.RLock()
	defer v.mu.RUnlock()

	var max uint64
	for slice := range v.fragments {
		if slice > max {
			max = slice
		}
	}

	return max
}

// FragmentPath returns the path to a fragment in the view.
func (v *View) FragmentPath(slice uint64) string {
	return filepath.Join(v.path, "fragments", strconv.FormatUint(slice, 10))
}

// Fragment returns a fragment in the view by slice.
func (v *View) Fragment(slice uint64) *Fragment {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return v.fragment(slice)
}

func (v *View) fragment(slice uint64) *Fragment { return v.fragments[slice] }

// Fragments returns a list of all fragments in the view.
func (v *View) Fragments() []*Fragment {
	v.mu.Lock()
	defer v.mu.Unlock()

	other := make([]*Fragment, 0, len(v.fragments))
	for _, fragment := range v.fragments {
		other = append(other, fragment)
	}
	return other
}

// RecalculateCaches recalculates the cache on every fragment in the view.
func (v *View) RecalculateCaches() {
	for _, fragment := range v.Fragments() {
		fragment.RecalculateCache()
	}
}

// CreateFragmentIfNotExists returns a fragment in the view by slice.
func (v *View) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) {
	v.mu.Lock()
	defer v.mu.Unlock()
	return v.createFragmentIfNotExists(slice)
}

func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
	// Find fragment in cache first.
	if frag := v.fragments[slice]; frag != nil {
		return frag, nil
	}

	// Initialize and open fragment.
	frag := v.newFragment(v.FragmentPath(slice), slice)
	if err := frag.Open(); err != nil {
		return nil, err
	}
	frag.RowAttrStore = v.RowAttrStore

	// Broadcast a message that a new max slice was just created.
	if slice > v.maxSlice {
		v.maxSlice = slice

		// Send the create slice message to all nodes.
		err := v.broadcaster.SendAsync(
			&internal.CreateSliceMessage{
				Index:     v.index,
				Slice:     slice,
				IsInverse: IsInverseView(v.name),
			})
		if err != nil {
			return nil, err
		}
	}

	// Save to lookup.
	v.fragments[slice] = frag
	return frag, nil
}

func (v *View) newFragment(path string, slice uint64) *Fragment {
	frag := NewFragment(path, v.index, v.frame, v.name, slice)
	frag.CacheType = v.cacheType
	frag.CacheSize = v.cacheSize
	frag.LogOutput = v.LogOutput
	frag.stats = v.stats.WithTags(fmt.Sprintf("slice:%d", slice))
	return frag
}

// DeleteFragment removes the fragment from the view.
func (v *View) DeleteFragment(slice uint64) error {

	fragment := v.fragments[slice]
	if fragment == nil {
		return ErrFragmentNotFound
	}

	v.logger().Printf("delete fragment: (%s/%s/%s) %d", v.index, v.frame, v.name, slice)

	// Close data files before deletion.
	if err := fragment.Close(); err != nil {
		return err
	}

	// Delete fragment file.
	if err := os.Remove(fragment.Path()); err != nil {
		return err
	}

	// Delete fragment cache file.
	if err := os.Remove(fragment.CachePath()); err != nil {
		v.logger().Printf("no cache file to delete for slice %d", slice)
	}

	delete(v.fragments, slice)

	return nil
}

// SetBit sets a bit within the view.
func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) {
	slice := columnID / SliceWidth
	frag, err := v.CreateFragmentIfNotExists(slice)
	if err != nil {
		return changed, err
	}
	return frag.SetBit(rowID, columnID)
}

// ClearBit clears a bit within the view.
func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) {
	slice := columnID / SliceWidth
	frag, err := v.CreateFragmentIfNotExists(slice)
	if err != nil {
		return changed, err
	}
	return frag.ClearBit(rowID, columnID)
}

// FieldValue uses a column of bits to read a multi-bit value.
func (v *View) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
	slice := columnID / SliceWidth
	frag, err := v.CreateFragmentIfNotExists(slice)
	if err != nil {
		return value, exists, err
	}
	return frag.FieldValue(columnID, bitDepth)
}

// SetFieldValue uses a column of bits to set a multi-bit value.
func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
	slice := columnID / SliceWidth
	frag, err := v.CreateFragmentIfNotExists(slice)
	if err != nil {
		return changed, err
	}
	return frag.SetFieldValue(columnID, bitDepth, value)
}

// FieldSum returns the sum & count of a field.
func (v *View) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err error) {
	for _, f := range v.Fragments() {
		fsum, fcount, err := f.FieldSum(filter, bitDepth)
		if err != nil {
			return sum, count, err
		}
		sum += fsum
		count += fcount
	}
	return sum, count, nil
}

// FieldRange returns bitmaps with a field value encoding matching the predicate.
func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) {
	bm := NewBitmap()
	for _, frag := range v.Fragments() {
		other, err := frag.FieldRange(op, bitDepth, predicate)
		if err != nil {
			return nil, err
		}
		bm = bm.Union(other)
	}
	return bm, nil
}

// FieldRangeBetween returns bitmaps with a field value encoding matching any
// value between predicateMin and predicateMax.
func (v *View) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Bitmap, error) {
	bm := NewBitmap()
	for _, frag := range v.Fragments() {
		other, err := frag.FieldRangeBetween(bitDepth, predicateMin, predicateMax)
		if err != nil {
			return nil, err
		}
		bm = bm.Union(other)
	}
	return bm, nil
}

// IsInverseView returns true if the view is used for storing an inverted representation.
func IsInverseView(name string) bool {
	return strings.HasPrefix(name, ViewInverse)
}

type viewSlice []*View

func (p viewSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p viewSlice) Len() int           { return len(p) }
func (p viewSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }

// ViewInfo represents schema information for a view.
type ViewInfo struct {
	Name string `json:"name"`
}

type viewInfoSlice []*ViewInfo

func (p viewInfoSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p viewInfoSlice) Len() int           { return len(p) }
func (p viewInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"flag"
	"fmt"
	"image"
	"image/jpeg"
	"io"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/disintegration/imaging"
	"github.com/rwcarlsen/goexif/exif"
)

var extensions = []string{".jpg", ".jpeg", ".JPG", ".JPEG"}
var metaDir = ".album"

var root = flag.String("root", "", "Album root")
var testMode = flag.Bool("test", false, "Test mode")
var httpAddress = flag.String("http", ":8080", "Default listening http address")

var metaRoot string

type FlipMode int

const (
	FlipVertical FlipMode = 1 << iota
	FlipHorizontal
)

const (
	topLeftSide     = 1
	topRightSide    = 2
	bottomRightSide = 3
	bottomLeftSide  = 4
	leftSideTop     = 5
	rightSideTop    = 6
	rightSideBottom = 7
	leftSideBottom  = 8
)

type PhotoTask struct {
	path     string
	root     string
	metaRoot string
	hash     string
}

type PhotoWalker struct {
	walkFunc func(path string, info os.FileInfo)
}

func (w PhotoWalker) photoWalker() filepath.WalkFunc {
	return func(path string, info os.FileInfo, err error) error {

		if info.IsDir() {

			if path == "." {
				return nil
			}

			if path == ".." {
				return nil
			}

			if strings.HasSuffix(path, ".app") {
				return filepath.SkipDir
			}

			if strings.HasSuffix(path, ".bundle") {
				return filepath.SkipDir
			}

			if strings.HasPrefix(info.Name(), ".") {
				return filepath.SkipDir
			}

			return nil
		}

		for _, ext := range extensions {
			if ext == filepath.Ext(info.Name()) {
				w.walkFunc(path, info)
				return nil
			}
		}

		return nil
	}
}

func HashPath(path string, root string, metaRoot string) string {
	normalizedPath := strings.TrimPrefix(path, root)
	normalizedPath = strings.TrimPrefix(normalizedPath, "/")

	hashPath := filepath.Join(metaRoot, "hash", normalizedPath) + ".sha1"
	return hashPath
}

func ThumbPath(hash string, root string, metaRoot string) string {
	hashPath := filepath.Join(metaRoot, "thumbs", hash) + ".jpg"
	return hashPath
}

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	flag.PrintDefaults()
}

func ExifOrientation(r io.Reader) (int, FlipMode) {
	var (
		angle int
		flip  FlipMode
	)

	ex, err := exif.Decode(r)
	if err != nil {
		log.Println("Could not decode Exif information")
		return 0, 0
	}

	tag, err := ex.Get(exif.Orientation)
	if err != nil {
		log.Println("No EXIF orientation tag: ", err)
		return 0, 0
	}

	orientation, err := tag.Int(0)

	if err != nil {
		log.Printf("EXIF error %v", err)
		return 0, 0
	}

	switch orientation {
	case topLeftSide:
		// no change required
	case topRightSide:
		flip = FlipHorizontal
	case bottomRightSide:
		angle = 180
	case bottomLeftSide:
		angle = 180
		flip = FlipHorizontal
	case leftSideTop:
		angle = -90
		flip = FlipHorizontal
	case rightSideTop:
		angle = -90
	case rightSideBottom:
		angle = 90
		flip = FlipHorizontal
	case leftSideBottom:
		angle = 90
	}

	return angle, flip
}

func main() {
	flag.Parse()

	if *root == "" {
		Usage()
		return
	}

	*root = filepath.Clean(*root)
	metaRoot = filepath.Join(*root, metaDir)

	if _, rootErr := ioutil.ReadDir(*root); rootErr != nil {
		log.Fatal("Root directory could not be read")
	}

	log.Println("Meta dir: " + metaDir)
	log.Println("Root: " + *root)

	workerCount := runtime.NumCPU()

	photos := []string{}
	hashes := make(map[string]string)

	photoWalker := PhotoWalker{}
	photoWalker.walkFunc = func(path string, info os.FileInfo) {
		photos = append(photos, path)
	}

	walkErr := filepath.Walk(*root, photoWalker.photoWalker())
	if walkErr != nil {
		log.Fatal(walkErr.Error())
	}

	fileCount := len(photos)
	counting := make(chan int)

	ticking := make(chan int, workerCount)
	finishing := make(chan bool)

	hashWorker := make(chan PhotoTask, workerCount)

	go func(work chan PhotoTask) {
		for task := range work {
			ticking <- 1

			task := task

			go func() {
				log.Println("Starting work on", task.path)
				file, fileErr := os.Open(task.path)

				if fileErr != nil {
					log.Println("Could not read photo file at", task.path, fileErr)
					<-ticking
					counting <- 1

					return
				}

				sha := sha1.New()
				io.Copy(sha, file)

				file.Close()

				sum := sha.Sum(nil)

				hashPath := HashPath(task.path, task.root, task.metaRoot)

				sumHex := hex.EncodeToString(sum)

				hashes[task.path] = sumHex

				if currentSumHex, hashErr := ioutil.ReadFile(hashPath); hashErr == nil {
					if string(currentSumHex) == sumHex {
						log.Println("Skipping", task.path)

						<-ticking
						counting <- 1

						return
					}
				}

				if !(*testMode) {
					os.MkdirAll(filepath.Dir(hashPath), 0755)
					file, fileErr := os.Create(hashPath)

					if fileErr != nil {
						log.Println("Could not create hash file: ", fileErr)
					} else {
						defer file.Close()

						if _, hashErr := file.WriteString(sumHex); hashErr != nil {
							log.Println("Could not write hash file: ", hashErr)
						}
					}
				}

				<-ticking
				counting <- 1

			}()

		}
	}(hashWorker)

	go func() {
		for _, path := range photos {
			task := PhotoTask{path: path, root: *root, metaRoot: metaRoot}
			hashWorker <- task
		}
	}()

	go func() {
		taskCount := 0

		for count := range counting {
			taskCount += count

			if taskCount == fileCount {
				close(counting)
				finishing <- true
				return
			}

		}
	}()

	<-finishing
	close(ticking)

	counting = make(chan int)

	ticking = make(chan int, workerCount)
	finishing = make(chan bool)

	photoWorker := make(chan PhotoTask, workerCount)

	go func(work chan PhotoTask) {
		for task := range work {
			ticking <- 1

			task := task

			go func() {
				log.Printf("Starting thumbnail work on %s (%s)", task.path, task.hash)
				file, fileErr := os.Open(task.path)

				if fileErr != nil {
					log.Println("Could not read photo file at", task.path, fileErr)
					<-ticking
					counting <- 1

					return
				}

				var exifBuffer bytes.Buffer

				img, imgErr := jpeg.Decode(io.TeeReader(file, &exifBuffer))

				file.Close()

				if imgErr != nil {
					log.Printf("Jpeg decode error: %v", imgErr)
					<-ticking
					counting <- 1
					return
				}

				maxDimension := 800.0
				width := float64(img.Bounds().Max.X - img.Bounds().Min.X)
				height := float64(img.Bounds().Max.Y - img.Bounds().Min.Y)

				var dstWidth float64
				var dstHeight float64

				biggerDimension := math.Max(width, height)

				if width < maxDimension && height < maxDimension {
					dstWidth = width
					dstHeight = height
				} else {
					scaleFactor := biggerDimension / maxDimension

					dstWidth = width / scaleFactor
					dstHeight = height / scaleFactor
				}

				idstWidth := int(math.Ceil(dstWidth))
				idstHeight := int(math.Ceil(dstHeight))

				var m image.Image
				if dstWidth == width && dstHeight == height {
					m = img
				} else {
					m = imaging.Resize(img, idstWidth, idstHeight, imaging.Box)
				}

				angle, flip := ExifOrientation(&exifBuffer)

				switch angle {
				case 90:
					m = imaging.Rotate90(m)
				case -90:
					m = imaging.Rotate270(m)
				case 180:
					m = imaging.Rotate180(m)
				}

				switch flip {
				case FlipHorizontal:
					m = imaging.FlipH(m)
				case FlipVertical:
					m = imaging.FlipV(m)
				}

				thumbPath := ThumbPath(task.hash, task.root, task.metaRoot)

				if !(*testMode) {
					os.MkdirAll(filepath.Dir(thumbPath), 0755)

					thumbFile, thumbErr := os.Create(thumbPath)
					if thumbErr == nil {

						jpeg.Encode(thumbFile, m, &jpeg.Options{Quality: 85})

						thumbFile.Close()
					}
				}

				log.Printf("Processed image, %0.fx%0.f, thumbnail is %0.fx%0.f", width, height, dstWidth, dstHeight)
				<-ticking

				counting <- 1
			}()

		}
	}(photoWorker)

	go func() {
		for _, path := range photos {
			task := PhotoTask{path: path, root: *root, metaRoot: metaRoot, hash: hashes[path]}
			photoWorker <- task
		}
	}()

	go func() {
		taskCount := 0

		for count := range counting {
			taskCount += count

			if taskCount == fileCount {
				close(counting)
				finishing <- true
				return
			}

		}
	}()

	<-finishing
	close(ticking)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "<!doctype html>Hi, you have %d photos", len(photos))
	})

	log.Println("Listening on", *httpAddress)
	log.Fatal(http.ListenAndServe(*httpAddress, nil))
}
<commit_msg>Stayed with the faster resizing lib after all.<commit_after>package main

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"flag"
	"fmt"
	"image"
	"image/jpeg"
	"io"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/disintegration/imaging"
	"github.com/nfnt/resize"
	"github.com/rwcarlsen/goexif/exif"
)

var extensions = []string{".jpg", ".jpeg", ".JPG", ".JPEG"}
var metaDir = ".album"

var root = flag.String("root", "", "Album root")
var testMode = flag.Bool("test", false, "Test mode")
var httpAddress = flag.String("http", ":8080", "Default listening http address")

var metaRoot string

type FlipMode int

const (
	FlipVertical FlipMode = 1 << iota
	FlipHorizontal
)

const (
	topLeftSide     = 1
	topRightSide    = 2
	bottomRightSide = 3
	bottomLeftSide  = 4
	leftSideTop     = 5
	rightSideTop    = 6
	rightSideBottom = 7
	leftSideBottom  = 8
)

type PhotoTask struct {
	path     string
	root     string
	metaRoot string
	hash     string
}

type PhotoWalker struct {
	walkFunc func(path string, info os.FileInfo)
}

func (w PhotoWalker) photoWalker() filepath.WalkFunc {
	return func(path string, info os.FileInfo, err error) error {

		if info.IsDir() {

			if path == "." {
				return nil
			}

			if path == ".." {
				return nil
			}

			if strings.HasSuffix(path, ".app") {
				return filepath.SkipDir
			}

			if strings.HasSuffix(path, ".bundle") {
				return filepath.SkipDir
			}

			if strings.HasPrefix(info.Name(), ".") {
				return filepath.SkipDir
			}

			return nil
		}

		for _, ext := range extensions {
			if ext == filepath.Ext(info.Name()) {
				w.walkFunc(path, info)
				return nil
			}
		}

		return nil
	}
}

func HashPath(path string, root string, metaRoot string) string {
	normalizedPath := strings.TrimPrefix(path, root)
	normalizedPath = strings.TrimPrefix(normalizedPath, "/")

	hashPath := filepath.Join(metaRoot, "hash", normalizedPath) + ".sha1"
	return hashPath
}

func ThumbPath(hash string, root string, metaRoot string) string {
	hashPath := filepath.Join(metaRoot, "thumbs", hash) + ".jpg"
	return hashPath
}

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	flag.PrintDefaults()
}

func ExifOrientation(r io.Reader) (int, FlipMode) {
	var (
		angle int
		flip  FlipMode
	)

	ex, err := exif.Decode(r)
	if err != nil {
		log.Println("Could not decode Exif information")
		return 0, 0
	}

	tag, err := ex.Get(exif.Orientation)
	if err != nil {
		return 0, 0
	}

	orientation, err := tag.Int(0)

	if err != nil {
		return 0, 0
	}

	switch orientation {
	case topLeftSide:
		// no change required
	case topRightSide:
		flip = FlipHorizontal
	case bottomRightSide:
		angle = 180
	case bottomLeftSide:
		angle = 180
		flip = FlipHorizontal
	case leftSideTop:
		angle = -90
		flip = FlipHorizontal
	case rightSideTop:
		angle = -90
	case rightSideBottom:
		angle = 90
		flip = FlipHorizontal
	case leftSideBottom:
		angle = 90
	}

	return angle, flip
}

func main() {
	flag.Parse()

	if *root == "" {
		Usage()
		return
	}

	*root = filepath.Clean(*root)
	metaRoot = filepath.Join(*root, metaDir)

	if _, rootErr := ioutil.ReadDir(*root); rootErr != nil {
		log.Fatal("Root directory could not be read")
	}

	log.Println("Meta dir: " + metaDir)
	log.Println("Root: " + *root)

	workerCount := runtime.NumCPU()

	photos := []string{}
	hashes := make(map[string]string)

	photoWalker := PhotoWalker{}
	photoWalker.walkFunc = func(path string, info os.FileInfo) {
		photos = append(photos, path)
	}

	walkErr := filepath.Walk(*root, photoWalker.photoWalker())
	if walkErr != nil {
		log.Fatal(walkErr.Error())
	}

	fileCount := len(photos)
	counting := make(chan int)

	ticking := make(chan int, workerCount)
	finishing := make(chan bool)

	hashWorker := make(chan PhotoTask, workerCount)

	go func(work chan PhotoTask) {
		for task := range work {
			ticking <- 1

			task := task

			go func() {
				log.Println("Starting work on", task.path)
				file, fileErr := os.Open(task.path)

				if fileErr != nil {
					log.Println("Could not read photo file at", task.path, fileErr)
					<-ticking
					counting <- 1

					return
				}

				sha := sha1.New()
				io.Copy(sha, file)

				file.Close()

				sum := sha.Sum(nil)

				hashPath := HashPath(task.path, task.root, task.metaRoot)

				sumHex := hex.EncodeToString(sum)

				hashes[task.path] = sumHex

				if currentSumHex, hashErr := ioutil.ReadFile(hashPath); hashErr == nil {
					if string(currentSumHex) == sumHex {
						log.Println("Skipping", task.path)

						<-ticking
						counting <- 1

						return
					}
				}

				if !(*testMode) {
					os.MkdirAll(filepath.Dir(hashPath), 0755)
					file, fileErr := os.Create(hashPath)

					if fileErr != nil {
						log.Println("Could not create hash file: ", fileErr)
					} else {
						defer file.Close()

						if _, hashErr := file.WriteString(sumHex); hashErr != nil {
							log.Println("Could not write hash file: ", hashErr)
						}
					}
				}

				<-ticking
				counting <- 1

			}()

		}
	}(hashWorker)

	go func() {
		for _, path := range photos {
			task := PhotoTask{path: path, root: *root, metaRoot: metaRoot}
			hashWorker <- task
		}
	}()

	go func() {
		taskCount := 0

		for count := range counting {
			taskCount += count

			if taskCount == fileCount {
				close(counting)
				finishing <- true
				return
			}

		}
	}()

	<-finishing
	close(ticking)

	counting = make(chan int)

	ticking = make(chan int, workerCount)
	finishing = make(chan bool)

	photoWorker := make(chan PhotoTask, workerCount)

	go func(work chan PhotoTask) {
		for task := range work {
			ticking <- 1

			task := task

			go func() {
				file, fileErr := os.Open(task.path)

				if fileErr != nil {
					log.Println("Could not read photo file at", task.path, fileErr)
					<-ticking
					counting <- 1

					return
				}

				var exifBuffer bytes.Buffer

				img, imgErr := jpeg.Decode(io.TeeReader(file, &exifBuffer))

				file.Close()

				if imgErr != nil {
					log.Printf("Jpeg decode error: %v", imgErr)
					<-ticking
					counting <- 1
					return
				}

				maxDimension := 800.0
				width := float64(img.Bounds().Max.X - img.Bounds().Min.X)
				height := float64(img.Bounds().Max.Y - img.Bounds().Min.Y)

				var dstWidth float64
				var dstHeight float64

				biggerDimension := math.Max(width, height)

				if width < maxDimension && height < maxDimension {
					dstWidth = width
					dstHeight = height
				} else {
					scaleFactor := biggerDimension / maxDimension

					dstWidth = width / scaleFactor
					dstHeight = height / scaleFactor
				}

				udstWidth := uint(math.Ceil(dstWidth))
				udstHeight := uint(math.Ceil(dstHeight))

				var m image.Image
				if dstWidth == width && dstHeight == height {
					m = img
				} else {
					m = resize.Resize(udstWidth, udstHeight, img, resize.Bilinear)
				}

				angle, flip := ExifOrientation(&exifBuffer)

				switch angle {
				case 90:
					m = imaging.Rotate90(m)
				case -90:
					m = imaging.Rotate270(m)
				case 180:
					m = imaging.Rotate180(m)
				}

				switch flip {
				case FlipHorizontal:
					m = imaging.FlipH(m)
				case FlipVertical:
					m = imaging.FlipV(m)
				}

				thumbPath := ThumbPath(task.hash, task.root, task.metaRoot)

				if !(*testMode) {
					os.MkdirAll(filepath.Dir(thumbPath), 0755)

					thumbFile, thumbErr := os.Create(thumbPath)
					if thumbErr == nil {

						jpeg.Encode(thumbFile, m, &jpeg.Options{Quality: 85})

						thumbFile.Close()
					}
				}

				log.Printf("Processed image, %0.fx%0.f, thumbnail is %0.fx%0.f", width, height, dstWidth, dstHeight)
				<-ticking

				counting <- 1
			}()

		}
	}(photoWorker)

	go func() {
		for _, path := range photos {
			task := PhotoTask{path: path, root: *root, metaRoot: metaRoot, hash: hashes[path]}
			photoWorker <- task
		}
	}()

	go func() {
		taskCount := 0

		for count := range counting {
			taskCount += count

			if taskCount == fileCount {
				close(counting)
				finishing <- true
				return
			}

		}
	}()

	<-finishing
	close(ticking)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "<!doctype html>Hi, you have %d photos", len(photos))
	})

	log.Println("Listening on", *httpAddress)
	log.Fatal(http.ListenAndServe(*httpAddress, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"encoding/hex"
	"flag"
	"fmt"
	"image"
	"image/jpeg"
	"io"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/nfnt/resize"
)

var extensions = []string{".jpg", ".jpeg", ".JPG", ".JPEG"}
var metaDir = ".album"

var root = flag.String("root", "", "Album root")
var testMode = flag.Bool("test", false, "Test mode")
var httpAddress = flag.String("http", ":8080", "Default listening http address")

var metaRoot string

type PhotoTask struct {
	path     string
	root     string
	metaRoot string
	hash     string
}

type PhotoWalker struct {
	walkFunc func(path string, info os.FileInfo)
}

func (w PhotoWalker) photoWalker() filepath.WalkFunc {
	return func(path string, info os.FileInfo, err error) error {

		if info.IsDir() {

			if path == "." {
				return nil
			}

			if path == ".." {
				return nil
			}

			if strings.HasSuffix(path, ".app") {
				return filepath.SkipDir
			}

			if strings.HasSuffix(path, ".bundle") {
				return filepath.SkipDir
			}

			if strings.HasPrefix(info.Name(), ".") {
				return filepath.SkipDir
			}

			return nil
		}

		for _, ext := range extensions {
			if ext == filepath.Ext(info.Name()) {
				w.walkFunc(path, info)
				return nil
			}
		}

		return nil
	}
}

func HashPath(path string, root string, metaRoot string) string {
	normalizedPath := strings.TrimPrefix(path, root)
	normalizedPath = strings.TrimPrefix(normalizedPath, "/")

	hashPath := filepath.Join(metaRoot, "hash", normalizedPath) + ".sha1"
	return hashPath
}

func ThumbPath(hash string, root string, metaRoot string) string {
	hashPath := filepath.Join(metaRoot, "thumbs", hash) + ".jpg"
	return hashPath
}

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	flag.Parse()

	if *root == "" {
		Usage()
		return
	}

	*root = filepath.Clean(*root)
	metaRoot = filepath.Join(*root, metaDir)

	if _, rootErr := ioutil.ReadDir(*root); rootErr != nil {
		log.Fatal("Root directory could not be read")
	}

	log.Println("Meta dir: " + metaDir)
	log.Println("Root: " + *root)

	workerCount := runtime.NumCPU()

	photos := []string{}
	hashes := make(map[string]string)

	photoWalker := PhotoWalker{}
	photoWalker.walkFunc = func(path string, info os.FileInfo) {
		photos = append(photos, path)
	}

	walkErr := filepath.Walk(*root, photoWalker.photoWalker())
	if walkErr != nil {
		log.Fatal(walkErr.Error())
	}

	fileCount := len(photos)
	counting := make(chan int)

	ticking := make(chan int, workerCount)
	finishing := make(chan bool)

	hashWorker := make(chan PhotoTask, workerCount)

	go func(work chan PhotoTask) {
		for task := range work {
			ticking <- 1

			task := task

			go func() {
				log.Println("Starting work on", task.path)
				file, fileErr := os.Open(task.path)

				if fileErr != nil {
					log.Println("Could not read photo file at", task.path, fileErr)
					<-ticking
					counting <- 1

					return
				}

				sha := sha1.New()
				io.Copy(sha, file)

				file.Close()

				sum := sha.Sum(nil)

				hashPath := HashPath(task.path, task.root, task.metaRoot)

				sumHex := hex.EncodeToString(sum)

				hashes[task.path] = sumHex

				if currentSumHex, hashErr := ioutil.ReadFile(hashPath); hashErr == nil {
					if string(currentSumHex) == sumHex {
						log.Println("Skipping", task.path)

						<-ticking
						counting <- 1

						return
					}
				}

				if !(*testMode) {
					os.MkdirAll(filepath.Dir(hashPath), 0755)
					file, fileErr := os.Create(hashPath)

					if fileErr != nil {
						log.Println("Could not create hash file: ", fileErr)
					} else {
						defer file.Close()

						if _, hashErr := file.WriteString(sumHex); hashErr != nil {
							log.Println("Could not write hash file: ", hashErr)
						}
					}
				}

				log.Println("Processed", task.path)
				<-ticking
				counting <- 1

			}()

		}
	}(hashWorker)

	go func() {
		for _, path := range photos {
			task := PhotoTask{path: path, root: *root, metaRoot: metaRoot}
			hashWorker <- task
		}
	}()

	go func() {
		taskCount := 0

		for count := range counting {
			taskCount += count

			if taskCount == fileCount {
				close(counting)
				finishing <- true
				return
			}

		}
	}()

	<-finishing
	close(ticking)

	counting = make(chan int)

	ticking = make(chan int, workerCount)
	finishing = make(chan bool)

	photoWorker := make(chan PhotoTask, workerCount)

	go func(work chan PhotoTask) {
		for task := range work {
			ticking <- 1

			task := task

			go func() {
				log.Printf("Starting thumbnail work on %s (%s)", task.path, task.hash)
				file, fileErr := os.Open(task.path)

				if fileErr != nil {
					log.Println("Could not read photo file at", task.path, fileErr)
					<-ticking
					counting <- 1

					return
				}

				img, imgErr := jpeg.Decode(file)

				file.Close()

				if imgErr != nil {
					log.Printf("Jpeg decode error: %v", imgErr)
					<-ticking
					counting <- 1
					return
				}

				maxDimension := 800.0
				width := float64(img.Bounds().Max.X - img.Bounds().Min.X)
				height := float64(img.Bounds().Max.Y - img.Bounds().Min.Y)

				var dstWidth float64
				var dstHeight float64

				biggerDimension := math.Max(width, height)

				if width < maxDimension && height < maxDimension {
					dstWidth = width
					dstHeight = height
				} else {
					scaleFactor := biggerDimension / maxDimension

					dstWidth = width / scaleFactor
					dstHeight = height / scaleFactor
				}

				udstWidth := uint(math.Ceil(dstWidth))
				udstHeight := uint(math.Ceil(dstHeight))

				var m image.Image
				if dstWidth == width && dstHeight == height {
					m = img
				} else {
					m = resize.Resize(udstWidth, udstHeight, img, resize.Bilinear)
				}

				thumbPath := ThumbPath(task.hash, task.root, task.metaRoot)

				if !(*testMode) {
					os.MkdirAll(filepath.Dir(thumbPath), 0755)

					thumbFile, thumbErr := os.Create(thumbPath)
					if thumbErr == nil {

						jpeg.Encode(thumbFile, m, &jpeg.Options{Quality: 85})

						thumbFile.Close()
					}
				}

				log.Printf("processed image, %0.fx%0.f, thumbnail is %0.fx%0.f", width, height, dstWidth, dstHeight)
				<-ticking

				counting <- 1
			}()

		}
	}(photoWorker)

	go func() {
		for _, path := range photos {
			task := PhotoTask{path: path, root: *root, metaRoot: metaRoot, hash: hashes[path]}
			photoWorker <- task
		}
	}()

	go func() {
		taskCount := 0

		for count := range counting {
			taskCount += count

			if taskCount == fileCount {
				close(counting)
				finishing <- true
				return
			}

		}
	}()

	<-finishing
	close(ticking)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "<!doctype html>Hi, you have %d photos", len(photos))
	})

	log.Println("Listening on", *httpAddress)
	log.Fatal(http.ListenAndServe(*httpAddress, nil))
}
<commit_msg>Thumbnails are rotated and flipped according to EXIF Orientation<commit_after>package main

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"flag"
	"fmt"
	"image"
	"image/jpeg"
	"io"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/disintegration/imaging"
	"github.com/rwcarlsen/goexif/exif"
)

var extensions = []string{".jpg", ".jpeg", ".JPG", ".JPEG"}
var metaDir = ".album"

var root = flag.String("root", "", "Album root")
var testMode = flag.Bool("test", false, "Test mode")
var httpAddress = flag.String("http", ":8080", "Default listening http address")

var metaRoot string

type FlipMode int

const (
	FlipVertical FlipMode = 1 << iota
	FlipHorizontal
)

const (
	topLeftSide     = 1
	topRightSide    = 2
	bottomRightSide = 3
	bottomLeftSide  = 4
	leftSideTop     = 5
	rightSideTop    = 6
	rightSideBottom = 7
	leftSideBottom  = 8
)

type PhotoTask struct {
	path     string
	root     string
	metaRoot string
	hash     string
}

type PhotoWalker struct {
	walkFunc func(path string, info os.FileInfo)
}

func (w PhotoWalker) photoWalker() filepath.WalkFunc {
	return func(path string, info os.FileInfo, err error) error {

		if info.IsDir() {

			if path == "." {
				return nil
			}

			if path == ".." {
				return nil
			}

			if strings.HasSuffix(path, ".app") {
				return filepath.SkipDir
			}

			if strings.HasSuffix(path, ".bundle") {
				return filepath.SkipDir
			}

			if strings.HasPrefix(info.Name(), ".") {
				return filepath.SkipDir
			}

			return nil
		}

		for _, ext := range extensions {
			if ext == filepath.Ext(info.Name()) {
				w.walkFunc(path, info)
				return nil
			}
		}

		return nil
	}
}

func HashPath(path string, root string, metaRoot string) string {
	normalizedPath := strings.TrimPrefix(path, root)
	normalizedPath = strings.TrimPrefix(normalizedPath, "/")

	hashPath := filepath.Join(metaRoot, "hash", normalizedPath) + ".sha1"
	return hashPath
}

func ThumbPath(hash string, root string, metaRoot string) string {
	hashPath := filepath.Join(metaRoot, "thumbs", hash) + ".jpg"
	return hashPath
}

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	flag.PrintDefaults()
}

func ExifOrientation(r io.Reader) (int, FlipMode) {
	var (
		angle int
		flip  FlipMode
	)

	ex, err := exif.Decode(r)
	if err != nil {
		log.Println("Could not decode Exif information")
		return 0, 0
	}

	tag, err := ex.Get(exif.Orientation)
	if err != nil {
		log.Println("No EXIF orientation tag: ", err)
		return 0, 0
	}

	orientation, err := tag.Int(0)

	if err != nil {
		log.Printf("EXIF error %v", err)
		return 0, 0
	}

	switch orientation {
	case topLeftSide:
		// no change required
	case topRightSide:
		flip = FlipHorizontal
	case bottomRightSide:
		angle = 180
	case bottomLeftSide:
		angle = 180
		flip = FlipHorizontal
	case leftSideTop:
		angle = -90
		flip = FlipHorizontal
	case rightSideTop:
		angle = -90
	case rightSideBottom:
		angle = 90
		flip = FlipHorizontal
	case leftSideBottom:
		angle = 90
	}

	return angle, flip
}

func main() {
	flag.Parse()

	if *root == "" {
		Usage()
		return
	}

	*root = filepath.Clean(*root)
	metaRoot = filepath.Join(*root, metaDir)

	if _, rootErr := ioutil.ReadDir(*root); rootErr != nil {
		log.Fatal("Root directory could not be read")
	}

	log.Println("Meta dir: " + metaDir)
	log.Println("Root: " + *root)

	workerCount := runtime.NumCPU()

	photos := []string{}
	hashes := make(map[string]string)

	photoWalker := PhotoWalker{}
	photoWalker.walkFunc = func(path string, info os.FileInfo) {
		photos = append(photos, path)
	}

	walkErr := filepath.Walk(*root, photoWalker.photoWalker())
	if walkErr != nil {
		log.Fatal(walkErr.Error())
	}

	fileCount := len(photos)
	counting := make(chan int)

	ticking := make(chan int, workerCount)
	finishing := make(chan bool)

	hashWorker := make(chan PhotoTask, workerCount)

	go func(work chan PhotoTask) {
		for task := range work {
			ticking <- 1

			task := task

			go func() {
				log.Println("Starting work on", task.path)
				file, fileErr := os.Open(task.path)

				if fileErr != nil {
					log.Println("Could not read photo file at", task.path, fileErr)
					<-ticking
					counting <- 1

					return
				}

				sha := sha1.New()
				io.Copy(sha, file)

				file.Close()

				sum := sha.Sum(nil)

				hashPath := HashPath(task.path, task.root, task.metaRoot)

				sumHex := hex.EncodeToString(sum)

				hashes[task.path] = sumHex

				if currentSumHex, hashErr := ioutil.ReadFile(hashPath); hashErr == nil {
					if string(currentSumHex) == sumHex {
						log.Println("Skipping", task.path)

						<-ticking
						counting <- 1

						return
					}
				}

				if !(*testMode) {
					os.MkdirAll(filepath.Dir(hashPath), 0755)
					file, fileErr := os.Create(hashPath)

					if fileErr != nil {
						log.Println("Could not create hash file: ", fileErr)
					} else {
						defer file.Close()

						if _, hashErr := file.WriteString(sumHex); hashErr != nil {
							log.Println("Could not write hash file: ", hashErr)
						}
					}
				}

				<-ticking
				counting <- 1

			}()

		}
	}(hashWorker)

	go func() {
		for _, path := range photos {
			task := PhotoTask{path: path, root: *root, metaRoot: metaRoot}
			hashWorker <- task
		}
	}()

	go func() {
		taskCount := 0

		for count := range counting {
			taskCount += count

			if taskCount == fileCount {
				close(counting)
				finishing <- true
				return
			}

		}
	}()

	<-finishing
	close(ticking)

	counting = make(chan int)

	ticking = make(chan int, workerCount)
	finishing = make(chan bool)

	photoWorker := make(chan PhotoTask, workerCount)

	go func(work chan PhotoTask) {
		for task := range work {
			ticking <- 1

			task := task

			go func() {
				log.Printf("Starting thumbnail work on %s (%s)", task.path, task.hash)
				file, fileErr := os.Open(task.path)

				if fileErr != nil {
					log.Println("Could not read photo file at", task.path, fileErr)
					<-ticking
					counting <- 1

					return
				}

				var exifBuffer bytes.Buffer

				img, imgErr := jpeg.Decode(io.TeeReader(file, &exifBuffer))

				file.Close()

				if imgErr != nil {
					log.Printf("Jpeg decode error: %v", imgErr)
					<-ticking
					counting <- 1
					return
				}

				maxDimension := 800.0
				width := float64(img.Bounds().Max.X - img.Bounds().Min.X)
				height := float64(img.Bounds().Max.Y - img.Bounds().Min.Y)

				var dstWidth float64
				var dstHeight float64

				biggerDimension := math.Max(width, height)

				if width < maxDimension && height < maxDimension {
					dstWidth = width
					dstHeight = height
				} else {
					scaleFactor := biggerDimension / maxDimension

					dstWidth = width / scaleFactor
					dstHeight = height / scaleFactor
				}

				idstWidth := int(math.Ceil(dstWidth))
				idstHeight := int(math.Ceil(dstHeight))

				var m image.Image
				if dstWidth == width && dstHeight == height {
					m = img
				} else {
					m = imaging.Resize(img, idstWidth, idstHeight, imaging.Box)
				}

				angle, flip := ExifOrientation(&exifBuffer)

				switch angle {
				case 90:
					m = imaging.Rotate90(m)
				case -90:
					m = imaging.Rotate270(m)
				case 180:
					m = imaging.Rotate180(m)
				}

				switch flip {
				case FlipHorizontal:
					m = imaging.FlipH(m)
				case FlipVertical:
					m = imaging.FlipV(m)
				}

				thumbPath := ThumbPath(task.hash, task.root, task.metaRoot)

				if !(*testMode) {
					os.MkdirAll(filepath.Dir(thumbPath), 0755)

					thumbFile, thumbErr := os.Create(thumbPath)
					if thumbErr == nil {

						jpeg.Encode(thumbFile, m, &jpeg.Options{Quality: 85})

						thumbFile.Close()
					}
				}

				log.Printf("Processed image, %0.fx%0.f, thumbnail is %0.fx%0.f", width, height, dstWidth, dstHeight)
				<-ticking

				counting <- 1
			}()

		}
	}(photoWorker)

	go func() {
		for _, path := range photos {
			task := PhotoTask{path: path, root: *root, metaRoot: metaRoot, hash: hashes[path]}
			photoWorker <- task
		}
	}()

	go func() {
		taskCount := 0

		for count := range counting {
			taskCount += count

			if taskCount == fileCount {
				close(counting)
				finishing <- true
				return
			}

		}
	}()

	<-finishing
	close(ticking)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "<!doctype html>Hi, you have %d photos", len(photos))
	})

	log.Println("Listening on", *httpAddress)
	log.Fatal(http.ListenAndServe(*httpAddress, nil))
}
<|endoftext|>