<commit_before>package ffmpeg

import (
	"errors"
	"fmt"
	"github.com/golang/glog"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"unsafe"
)

// #cgo pkg-config: libavformat libavfilter libavcodec libavutil libswscale gnutls
// #include <stdlib.h>
// #include "lpms_ffmpeg.h"
import "C"

var ErrTranscoderRes = errors.New("TranscoderInvalidResolution")
var ErrTranscoderHw = errors.New("TranscoderInvalidHardware")
var ErrTranscoderInp = errors.New("TranscoderInvalidInput")
var ErrTranscoderStp = errors.New("TranscoderStopped")
var ErrTranscoderFmt = errors.New("TranscoderUnrecognizedFormat")
var ErrTranscoderPrf = errors.New("TranscoderUnrecognizedProfile")
var ErrTranscoderGOP = errors.New("TranscoderInvalidGOP")

type Acceleration int

const (
	Software Acceleration = iota
	Nvidia
	Amd
)

type ComponentOptions struct {
	Name string
	Opts map[string]string
}

type Transcoder struct {
	handle  *C.struct_transcode_thread
	stopped bool
	started bool
	mu      *sync.Mutex
}

type TranscodeOptionsIn struct {
	Fname  string
	Accel  Acceleration
	Device string
}

type TranscodeOptions struct {
	Oname   string
	Profile VideoProfile
	Accel   Acceleration
	Device  string

	Muxer        ComponentOptions
	VideoEncoder ComponentOptions
	AudioEncoder ComponentOptions
}

type MediaInfo struct {
	Frames int
	Pixels int64
}

type TranscodeResults struct {
	Decoded MediaInfo
	Encoded []MediaInfo
}

func RTMPToHLS(localRTMPUrl string, outM3U8 string, tmpl string, seglen_secs string, seg_start int) error {
	inp := C.CString(localRTMPUrl)
	outp := C.CString(outM3U8)
	ts_tmpl := C.CString(tmpl)
	seglen := C.CString(seglen_secs)
	segstart := C.CString(fmt.Sprintf("%v", seg_start))
	ret := int(C.lpms_rtmp2hls(inp, outp, ts_tmpl, seglen, segstart))
	C.free(unsafe.Pointer(inp))
	C.free(unsafe.Pointer(outp))
	C.free(unsafe.Pointer(ts_tmpl))
	C.free(unsafe.Pointer(seglen))
	C.free(unsafe.Pointer(segstart))
	if 0 != ret {
		glog.Infof("RTMP2HLS Transmux Return : %v\n", Strerror(ret))
		return ErrorMap[ret]
	}
	return nil
}

func Transcode(input string, workDir string, ps []VideoProfile) error {

	opts := make([]TranscodeOptions, len(ps))
	for i, param := range ps {
		oname := path.Join(workDir, fmt.Sprintf("out%v%v", i, filepath.Base(input)))
		opt := TranscodeOptions{
			Oname:   oname,
			Profile: param,
			Accel:   Software,
		}
		opts[i] = opt
	}
	inopts := &TranscodeOptionsIn{
		Fname: input,
		Accel: Software,
	}
	return Transcode2(inopts, opts)
}

func newAVOpts(opts map[string]string) *C.AVDictionary {
	var dict *C.AVDictionary
	for key, value := range opts {
		k := C.CString(key)
		v := C.CString(value)
		defer C.free(unsafe.Pointer(k))
		defer C.free(unsafe.Pointer(v))
		C.av_dict_set(&dict, k, v, 0)
	}
	return dict
}

// return encoding specific options for the given accel
func configAccel(inAcc, outAcc Acceleration, inDev, outDev string) (string, string, error) {
	switch inAcc {
	case Software:
		switch outAcc {
		case Software:
			return "libx264", "scale", nil
		case Nvidia:
			upload := "hwupload_cuda"
			if outDev != "" {
				upload = upload + "=device=" + outDev
			}
			return "h264_nvenc", upload + ",scale_cuda", nil
		}
	case Nvidia:
		switch outAcc {
		case Software:
			return "libx264", "scale_cuda", nil
		case Nvidia:
			// If we encode on a different device from decode then need to transfer
			if outDev != "" && outDev != inDev {
				return "", "", ErrTranscoderInp // XXX not allowed
			}
			return "h264_nvenc", "scale_cuda", nil
		}
	}
	return "", "", ErrTranscoderHw
}
func accelDeviceType(accel Acceleration) (C.enum_AVHWDeviceType, error) {
	switch accel {
	case Software:
		return C.AV_HWDEVICE_TYPE_NONE, nil
	case Nvidia:
		return C.AV_HWDEVICE_TYPE_CUDA, nil

	}
	return C.AV_HWDEVICE_TYPE_NONE, ErrTranscoderHw
}

func Transcode2(input *TranscodeOptionsIn, ps []TranscodeOptions) error {
	_, err := Transcode3(input, ps)
	return err
}

func Transcode3(input *TranscodeOptionsIn, ps []TranscodeOptions) (*TranscodeResults, error) {
	t := NewTranscoder()
	defer t.StopTranscoder()
	return t.Transcode(input, ps)
}

func (t *Transcoder) Transcode(input *TranscodeOptionsIn, ps []TranscodeOptions) (*TranscodeResults, error) {
	t.mu.Lock()
	defer t.mu.Unlock()
	if t.stopped || t.handle == nil {
		return nil, ErrTranscoderStp
	}
	if input == nil {
		return nil, ErrTranscoderInp
	}
	hw_type, err := accelDeviceType(input.Accel)
	if err != nil {
		return nil, err
	}
	if !t.started {
		ret := int(C.lpms_is_bypass_needed(C.CString(input.Fname)))
		if ret != 1 {
			// Stream is either OK or completely broken, let the transcoder handle it
			t.started = true
		} else {
			// Audio-only segment, fail fast right here as we cannot handle them nicely
			return nil, errors.New("No video parameters found while initializing stream")
		}
	}
	fname := C.CString(input.Fname)
	defer C.free(unsafe.Pointer(fname))
	params := make([]C.output_params, len(ps))
	for i, p := range ps {
		oname := C.CString(p.Oname)
		defer C.free(unsafe.Pointer(oname))

		param := p.Profile
		w, h, err := VideoProfileResolution(param)
		if err != nil {
			if "drop" != p.VideoEncoder.Name && "copy" != p.VideoEncoder.Name {
				return nil, err
			}
		}
		br := strings.Replace(param.Bitrate, "k", "000", 1)
		bitrate, err := strconv.Atoi(br)
		if err != nil {
			if "drop" != p.VideoEncoder.Name && "copy" != p.VideoEncoder.Name {
				return nil, err
			}
		}
		encoder, scale_filter := p.VideoEncoder.Name, "scale"
		if encoder == "" {
			encoder, scale_filter, err = configAccel(input.Accel, p.Accel, input.Device, p.Device)
			if err != nil {
				return nil, err
			}
		}
		// preserve aspect ratio along the larger dimension when rescaling
		var filters string
		filters = fmt.Sprintf("%s='w=if(gte(iw,ih),%d,-2):h=if(lt(iw,ih),%d,-2)'", scale_filter, w, h)
		if input.Accel != Software && p.Accel == Software {
			// needed for hw dec -> hw rescale -> sw enc
			filters = filters + ",hwdownload,format=nv12"
		}
		// set FPS denominator to 1 if unset by user
		if param.FramerateDen == 0 {
			param.FramerateDen = 1
		}
		// Add fps filter *after* scale filter because otherwise we could
		// be scaling duplicate frames unnecessarily. This becomes a DoS vector
		// when a user submits two frames that are "far apart" in pts and
		// the fps filter duplicates frames to fill out the difference to maintain
		// a consistent frame rate.
		// Once we allow for alternating segments, this issue should be mitigated
		// and the fps filter can come *before* the scale filter to minimize work
		// when going from high fps to low fps (much more common when transcoding
		// than going from low fps to high fps)
		var fps C.AVRational
		if param.Framerate > 0 {
			filters += fmt.Sprintf(",fps=%d/%d", param.Framerate, param.FramerateDen)
			fps = C.AVRational{num: C.int(param.Framerate), den: C.int(param.FramerateDen)}
		}
		var muxOpts C.component_opts
		var muxName string
		switch p.Profile.Format {
		case FormatNone:
			muxOpts = C.component_opts{
				// don't free this bc of avformat_write_header API
				opts: newAVOpts(p.Muxer.Opts),
			}
			muxName = p.Muxer.Name
		case FormatMPEGTS:
			muxName = "mpegts"
		case FormatMP4:
			muxName = "mp4"
			muxOpts = C.component_opts{
				opts: newAVOpts(map[string]string{"movflags": "faststart"}),
			}
		default:
			return nil, ErrTranscoderFmt
		}
		if muxName != "" {
			muxOpts.name = C.CString(muxName)
			defer C.free(unsafe.Pointer(muxOpts.name))
		}
		// Set video encoder options
		if len(p.VideoEncoder.Name) <= 0 && len(p.VideoEncoder.Opts) <= 0 {
			p.VideoEncoder.Opts = map[string]string{
				"forced-idr": "1",
			}
			switch p.Profile.Profile {
			case ProfileH264Baseline, ProfileH264Main, ProfileH264High:
				p.VideoEncoder.Opts["profile"] = ProfileParameters[p.Profile.Profile]
			case ProfileH264ConstrainedHigh:
				p.VideoEncoder.Opts["profile"] = ProfileParameters[p.Profile.Profile]
				p.VideoEncoder.Opts["bf"] = "0"
			case ProfileNone:
				// Do nothing, the encoder will use default profile
			default:
				return nil, ErrTranscoderPrf
			}
		}
		gopMs := 0
		if param.GOP != 0 {
			if param.GOP <= GOPInvalid {
				return nil, ErrTranscoderGOP
			}
			// Check for intra-only
			if param.GOP == GOPIntraOnly {
				p.VideoEncoder.Opts["g"] = "0"
			} else {
				if param.Framerate > 0 {
					gop := param.GOP.Seconds()
					interval := strconv.Itoa(int(gop * float64(param.Framerate)))
					p.VideoEncoder.Opts["g"] = interval
				} else {
					gopMs = int(param.GOP.Milliseconds())
				}
			}
		}
		vidOpts := C.component_opts{
			name: C.CString(encoder),
			opts: newAVOpts(p.VideoEncoder.Opts),
		}
		audioEncoder := p.AudioEncoder.Name
		if audioEncoder == "" {
			audioEncoder = "aac"
		}
		audioOpts := C.component_opts{
			name: C.CString(audioEncoder),
			opts: newAVOpts(p.AudioEncoder.Opts),
		}
		vfilt := C.CString(filters)
		defer C.free(unsafe.Pointer(vidOpts.name))
		defer C.free(unsafe.Pointer(audioOpts.name))
		defer C.free(unsafe.Pointer(vfilt))
		params[i] = C.output_params{fname: oname, fps: fps,
			w: C.int(w), h: C.int(h), bitrate: C.int(bitrate),
			gop_time: C.int(gopMs),
			muxer:    muxOpts, audio: audioOpts, video: vidOpts, vfilters: vfilt}
		defer func(param *C.output_params) {
			// Work around the ownership rules:
			// ffmpeg normally takes ownership of the following AVDictionary options
			// However, if we don't pass these opts to ffmpeg, then we need to free
			if param.muxer.opts != nil {
				C.av_dict_free(&param.muxer.opts)
			}
			if param.audio.opts != nil {
				C.av_dict_free(&param.audio.opts)
			}
			if param.video.opts != nil {
				C.av_dict_free(&param.video.opts)
			}
		}(&params[i])
	}
	var device *C.char
	if input.Device != "" {
		device = C.CString(input.Device)
		defer C.free(unsafe.Pointer(device))
	}
	inp := &C.input_params{fname: fname, hw_type: hw_type, device: device,
		handle: t.handle}
	results := make([]C.output_results, len(ps))
	decoded := &C.output_results{}
	var (
		paramsPointer  *C.output_params
		resultsPointer *C.output_results
	)
	if len(params) > 0 {
		paramsPointer = (*C.output_params)(&params[0])
		resultsPointer = (*C.output_results)(&results[0])
	}
	ret := int(C.lpms_transcode(inp, paramsPointer, resultsPointer, C.int(len(params)), decoded))
	if 0 != ret {
		glog.Error("Transcoder Return : ", ErrorMap[ret])
		return nil, ErrorMap[ret]
	}
	tr := make([]MediaInfo, len(ps))
	for i, r := range results {
		tr[i] = MediaInfo{
			Frames: int(r.frames),
			Pixels: int64(r.pixels),
		}
	}
	dec := MediaInfo{
		Frames: int(decoded.frames),
		Pixels: int64(decoded.pixels),
	}
	return &TranscodeResults{Encoded: tr, Decoded: dec}, nil
}

func NewTranscoder() *Transcoder {
	return &Transcoder{
		handle: C.lpms_transcode_new(),
		mu:     &sync.Mutex{},
	}
}

func (t *Transcoder) StopTranscoder() {
	t.mu.Lock()
	defer t.mu.Unlock()
	if t.stopped {
		return
	}
	C.lpms_transcode_stop(t.handle)
	t.handle = nil // prevent accidental reuse
	t.stopped = true
}

func InitFFmpeg() {
	C.lpms_init()
}
<commit_msg>ffmpeg: Fix cstring memleak<commit_after>package ffmpeg

import (
	"errors"
	"fmt"
	"github.com/golang/glog"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"unsafe"
)

// #cgo pkg-config: libavformat libavfilter libavcodec libavutil libswscale gnutls
// #include <stdlib.h>
// #include "lpms_ffmpeg.h"
import "C"

var ErrTranscoderRes = errors.New("TranscoderInvalidResolution")
var ErrTranscoderHw = errors.New("TranscoderInvalidHardware")
var ErrTranscoderInp = errors.New("TranscoderInvalidInput")
var ErrTranscoderStp = errors.New("TranscoderStopped")
var ErrTranscoderFmt = errors.New("TranscoderUnrecognizedFormat")
var ErrTranscoderPrf = errors.New("TranscoderUnrecognizedProfile")
var ErrTranscoderGOP = errors.New("TranscoderInvalidGOP")

type Acceleration int

const (
	Software Acceleration = iota
	Nvidia
	Amd
)

type ComponentOptions struct {
	Name string
	Opts map[string]string
}

type Transcoder struct {
	handle  *C.struct_transcode_thread
	stopped bool
	started bool
	mu      *sync.Mutex
}

type TranscodeOptionsIn struct {
	Fname  string
	Accel  Acceleration
	Device string
}

type TranscodeOptions struct {
	Oname   string
	Profile VideoProfile
	Accel   Acceleration
	Device  string

	Muxer        ComponentOptions
	VideoEncoder ComponentOptions
	AudioEncoder ComponentOptions
}

type MediaInfo struct {
	Frames int
	Pixels int64
}

type TranscodeResults struct {
	Decoded MediaInfo
	Encoded []MediaInfo
}

func RTMPToHLS(localRTMPUrl string, outM3U8 string, tmpl string, seglen_secs string, seg_start int) error {
	inp := C.CString(localRTMPUrl)
	outp := C.CString(outM3U8)
	ts_tmpl := C.CString(tmpl)
	seglen := C.CString(seglen_secs)
	segstart := C.CString(fmt.Sprintf("%v", seg_start))
	ret := int(C.lpms_rtmp2hls(inp, outp, ts_tmpl, seglen, segstart))
	C.free(unsafe.Pointer(inp))
	C.free(unsafe.Pointer(outp))
	C.free(unsafe.Pointer(ts_tmpl))
	C.free(unsafe.Pointer(seglen))
	C.free(unsafe.Pointer(segstart))
	if 0 != ret {
		glog.Infof("RTMP2HLS Transmux Return : %v\n", Strerror(ret))
		return ErrorMap[ret]
	}
	return nil
}

func Transcode(input string, workDir string, ps []VideoProfile) error {

	opts := make([]TranscodeOptions, len(ps))
	for i, param := range ps {
		oname := path.Join(workDir, fmt.Sprintf("out%v%v", i, filepath.Base(input)))
		opt := TranscodeOptions{
			Oname:   oname,
			Profile: param,
			Accel:   Software,
		}
		opts[i] = opt
	}
	inopts := &TranscodeOptionsIn{
		Fname: input,
		Accel: Software,
	}
	return Transcode2(inopts, opts)
}

func newAVOpts(opts map[string]string) *C.AVDictionary {
	var dict *C.AVDictionary
	for key, value := range opts {
		k := C.CString(key)
		v := C.CString(value)
		defer C.free(unsafe.Pointer(k))
		defer C.free(unsafe.Pointer(v))
		C.av_dict_set(&dict, k, v, 0)
	}
	return dict
}

// return encoding specific options for the given accel
func configAccel(inAcc, outAcc Acceleration, inDev, outDev string) (string, string, error) {
	switch inAcc {
	case Software:
		switch outAcc {
		case Software:
			return "libx264", "scale", nil
		case Nvidia:
			upload := "hwupload_cuda"
			if outDev != "" {
				upload = upload + "=device=" + outDev
			}
			return "h264_nvenc", upload + ",scale_cuda", nil
		}
	case Nvidia:
		switch outAcc {
		case Software:
			return "libx264", "scale_cuda", nil
		case Nvidia:
			// If we encode on a different device from decode then need to transfer
			if outDev != "" && outDev != inDev {
				return "", "", ErrTranscoderInp // XXX not allowed
			}
			return "h264_nvenc", "scale_cuda", nil
		}
	}
	return "", "", ErrTranscoderHw
}
func accelDeviceType(accel Acceleration) (C.enum_AVHWDeviceType, error) {
	switch accel {
	case Software:
		return C.AV_HWDEVICE_TYPE_NONE, nil
	case Nvidia:
		return C.AV_HWDEVICE_TYPE_CUDA, nil

	}
	return C.AV_HWDEVICE_TYPE_NONE, ErrTranscoderHw
}

func Transcode2(input *TranscodeOptionsIn, ps []TranscodeOptions) error {
	_, err := Transcode3(input, ps)
	return err
}

func Transcode3(input *TranscodeOptionsIn, ps []TranscodeOptions) (*TranscodeResults, error) {
	t := NewTranscoder()
	defer t.StopTranscoder()
	return t.Transcode(input, ps)
}

func (t *Transcoder) Transcode(input *TranscodeOptionsIn, ps []TranscodeOptions) (*TranscodeResults, error) {
	t.mu.Lock()
	defer t.mu.Unlock()
	if t.stopped || t.handle == nil {
		return nil, ErrTranscoderStp
	}
	if input == nil {
		return nil, ErrTranscoderInp
	}
	hw_type, err := accelDeviceType(input.Accel)
	if err != nil {
		return nil, err
	}
	fname := C.CString(input.Fname)
	defer C.free(unsafe.Pointer(fname))
	if !t.started {
		ret := int(C.lpms_is_bypass_needed(fname))
		if ret != 1 {
			// Stream is either OK or completely broken, let the transcoder handle it
			t.started = true
		} else {
			// Audio-only segment, fail fast right here as we cannot handle them nicely
			return nil, errors.New("No video parameters found while initializing stream")
		}
	}
	params := make([]C.output_params, len(ps))
	for i, p := range ps {
		oname := C.CString(p.Oname)
		defer C.free(unsafe.Pointer(oname))

		param := p.Profile
		w, h, err := VideoProfileResolution(param)
		if err != nil {
			if "drop" != p.VideoEncoder.Name && "copy" != p.VideoEncoder.Name {
				return nil, err
			}
		}
		br := strings.Replace(param.Bitrate, "k", "000", 1)
		bitrate, err := strconv.Atoi(br)
		if err != nil {
			if "drop" != p.VideoEncoder.Name && "copy" != p.VideoEncoder.Name {
				return nil, err
			}
		}
		encoder, scale_filter := p.VideoEncoder.Name, "scale"
		if encoder == "" {
			encoder, scale_filter, err = configAccel(input.Accel, p.Accel, input.Device, p.Device)
			if err != nil {
				return nil, err
			}
		}
		// preserve aspect ratio along the larger dimension when rescaling
		var filters string
		filters = fmt.Sprintf("%s='w=if(gte(iw,ih),%d,-2):h=if(lt(iw,ih),%d,-2)'", scale_filter, w, h)
		if input.Accel != Software && p.Accel == Software {
			// needed for hw dec -> hw rescale -> sw enc
			filters = filters + ",hwdownload,format=nv12"
		}
		// set FPS denominator to 1 if unset by user
		if param.FramerateDen == 0 {
			param.FramerateDen = 1
		}
		// Add fps filter *after* scale filter because otherwise we could
		// be scaling duplicate frames unnecessarily. This becomes a DoS vector
		// when a user submits two frames that are "far apart" in pts and
		// the fps filter duplicates frames to fill out the difference to maintain
		// a consistent frame rate.
		// Once we allow for alternating segments, this issue should be mitigated
		// and the fps filter can come *before* the scale filter to minimize work
		// when going from high fps to low fps (much more common when transcoding
		// than going from low fps to high fps)
		var fps C.AVRational
		if param.Framerate > 0 {
			filters += fmt.Sprintf(",fps=%d/%d", param.Framerate, param.FramerateDen)
			fps = C.AVRational{num: C.int(param.Framerate), den: C.int(param.FramerateDen)}
		}
		var muxOpts C.component_opts
		var muxName string
		switch p.Profile.Format {
		case FormatNone:
			muxOpts = C.component_opts{
				// don't free this bc of avformat_write_header API
				opts: newAVOpts(p.Muxer.Opts),
			}
			muxName = p.Muxer.Name
		case FormatMPEGTS:
			muxName = "mpegts"
		case FormatMP4:
			muxName = "mp4"
			muxOpts = C.component_opts{
				opts: newAVOpts(map[string]string{"movflags": "faststart"}),
			}
		default:
			return nil, ErrTranscoderFmt
		}
		if muxName != "" {
			muxOpts.name = C.CString(muxName)
			defer C.free(unsafe.Pointer(muxOpts.name))
		}
		// Set video encoder options
		if len(p.VideoEncoder.Name) <= 0 && len(p.VideoEncoder.Opts) <= 0 {
			p.VideoEncoder.Opts = map[string]string{
				"forced-idr": "1",
			}
			switch p.Profile.Profile {
			case ProfileH264Baseline, ProfileH264Main, ProfileH264High:
				p.VideoEncoder.Opts["profile"] = ProfileParameters[p.Profile.Profile]
			case ProfileH264ConstrainedHigh:
				p.VideoEncoder.Opts["profile"] = ProfileParameters[p.Profile.Profile]
				p.VideoEncoder.Opts["bf"] = "0"
			case ProfileNone:
				// Do nothing, the encoder will use default profile
			default:
				return nil, ErrTranscoderPrf
			}
		}
		gopMs := 0
		if param.GOP != 0 {
			if param.GOP <= GOPInvalid {
				return nil, ErrTranscoderGOP
			}
			// Check for intra-only
			if param.GOP == GOPIntraOnly {
				p.VideoEncoder.Opts["g"] = "0"
			} else {
				if param.Framerate > 0 {
					gop := param.GOP.Seconds()
					interval := strconv.Itoa(int(gop * float64(param.Framerate)))
					p.VideoEncoder.Opts["g"] = interval
				} else {
					gopMs = int(param.GOP.Milliseconds())
				}
			}
		}
		vidOpts := C.component_opts{
			name: C.CString(encoder),
			opts: newAVOpts(p.VideoEncoder.Opts),
		}
		audioEncoder := p.AudioEncoder.Name
		if audioEncoder == "" {
			audioEncoder = "aac"
		}
		audioOpts := C.component_opts{
			name: C.CString(audioEncoder),
			opts: newAVOpts(p.AudioEncoder.Opts),
		}
		vfilt := C.CString(filters)
		defer C.free(unsafe.Pointer(vidOpts.name))
		defer C.free(unsafe.Pointer(audioOpts.name))
		defer C.free(unsafe.Pointer(vfilt))
		params[i] = C.output_params{fname: oname, fps: fps,
			w: C.int(w), h: C.int(h), bitrate: C.int(bitrate),
			gop_time: C.int(gopMs),
			muxer:    muxOpts, audio: audioOpts, video: vidOpts, vfilters: vfilt}
		defer func(param *C.output_params) {
			// Work around the ownership rules:
			// ffmpeg normally takes ownership of the following AVDictionary options
			// However, if we don't pass these opts to ffmpeg, then we need to free
			if param.muxer.opts != nil {
				C.av_dict_free(&param.muxer.opts)
			}
			if param.audio.opts != nil {
				C.av_dict_free(&param.audio.opts)
			}
			if param.video.opts != nil {
				C.av_dict_free(&param.video.opts)
			}
		}(&params[i])
	}
	var device *C.char
	if input.Device != "" {
		device = C.CString(input.Device)
		defer C.free(unsafe.Pointer(device))
	}
	inp := &C.input_params{fname: fname, hw_type: hw_type, device: device,
		handle: t.handle}
	results := make([]C.output_results, len(ps))
	decoded := &C.output_results{}
	var (
		paramsPointer  *C.output_params
		resultsPointer *C.output_results
	)
	if len(params) > 0 {
		paramsPointer = (*C.output_params)(&params[0])
		resultsPointer = (*C.output_results)(&results[0])
	}
	ret := int(C.lpms_transcode(inp, paramsPointer, resultsPointer, C.int(len(params)), decoded))
	if 0 != ret {
		glog.Error("Transcoder Return : ", ErrorMap[ret])
		return nil, ErrorMap[ret]
	}
	tr := make([]MediaInfo, len(ps))
	for i, r := range results {
		tr[i] = MediaInfo{
			Frames: int(r.frames),
			Pixels: int64(r.pixels),
		}
	}
	dec := MediaInfo{
		Frames: int(decoded.frames),
		Pixels: int64(decoded.pixels),
	}
	return &TranscodeResults{Encoded: tr, Decoded: dec}, nil
}

func NewTranscoder() *Transcoder {
	return &Transcoder{
		handle: C.lpms_transcode_new(),
		mu:     &sync.Mutex{},
	}
}

func (t *Transcoder) StopTranscoder() {
	t.mu.Lock()
	defer t.mu.Unlock()
	if t.stopped {
		return
	}
	C.lpms_transcode_stop(t.handle)
	t.handle = nil // prevent accidental reuse
	t.stopped = true
}

func InitFFmpeg() {
	C.lpms_init()
}
<|endoftext|>


<commit_before><commit_msg>Generate replaced text.<commit_after><|endoftext|>


<commit_before>// Written in 2014 by Philipp Stephani <p.stephani2@gmail.com>.
//
// To the extent possible under law, the author has dedicated all copyright and
// related and neighboring rights to this software to the public domain worldwide.
// This software is distributed without any warranty.
//
// You should have received a copy of the CC0 Public Domain Dedication along with
// this software.  If not, see http://creativecommons.org/publicdomain/zero/1.0/.

package filter

import (
	"strings"

	"github.com/golang/glog"

	"github.com/phst/dupremove/dup"
)

// RemovableFiles returns all files from the given duplicate file group that
// can be removed under the condition that at least one file should remain and
// the directories listed in keep should be left untouched.
func RemovableFiles(group dup.Group, keep []string) []dup.FileName {
	glog.V(2).Infof("searching duplicate group with %d files for candidates for removal", len(group))
	keepCandidates := []dup.FileName{}
	removeCandidates := []dup.FileName{}
	for _, file := range group {
		glog.V(3).Infof("testing whether %s can be removed", file)
		remove := true
		for _, dir := range keep {
			if strings.HasPrefix(string(file), dir) {
				glog.V(2).Infof("file %s cannot be removed because it is in precious directory %s", file, dir)
				remove = false
				break
			}
		}
		if remove {
			glog.V(3).Infof("file %s is a candidate for removal", file)
			removeCandidates = append(removeCandidates, file)
		} else {
			glog.V(3).Infof("file %s will be kept because it is precious", file)
			keepCandidates = append(keepCandidates, file)
		}
	}
	if len(removeCandidates) == 0 || len(keepCandidates) > 0 {
		glog.V(2).Infof("all removal candidates can be removed")
		return removeCandidates
	}
	glog.V(2).Infof("file %s will be kept because it would be the only remaining file in current group", removeCandidates[0])
	return removeCandidates[1:]
}
<commit_msg>We don't need to store files to be kept, only record whether there are any<commit_after>// Written in 2014 by Philipp Stephani <p.stephani2@gmail.com>.
//
// To the extent possible under law, the author has dedicated all copyright and
// related and neighboring rights to this software to the public domain worldwide.
// This software is distributed without any warranty.
//
// You should have received a copy of the CC0 Public Domain Dedication along with
// this software.  If not, see http://creativecommons.org/publicdomain/zero/1.0/.

package filter

import (
	"strings"

	"github.com/golang/glog"

	"github.com/phst/dupremove/dup"
)

// RemovableFiles returns all files from the given duplicate file group that
// can be removed under the condition that at least one file should remain and
// the directories listed in keep should be left untouched.
func RemovableFiles(group dup.Group, keep []string) []dup.FileName {
	glog.V(2).Infof("searching duplicate group with %d files for candidates for removal", len(group))
	kept := 0
	candidates := []dup.FileName{}
	for _, file := range group {
		glog.V(3).Infof("testing whether %s can be removed", file)
		remove := true
		for _, dir := range keep {
			if strings.HasPrefix(string(file), dir) {
				glog.V(2).Infof("file %s cannot be removed because it is in precious directory %s", file, dir)
				remove = false
				break
			}
		}
		if remove {
			glog.V(3).Infof("file %s is a candidate for removal", file)
			candidates = append(candidates, file)
		} else {
			glog.V(3).Infof("file %s will be kept because it is precious", file)
			kept++
		}
	}
	if len(candidates) == 0 || kept > 0 {
		glog.V(2).Infof("all removal candidates can be removed")
		return candidates
	}
	glog.V(2).Infof("file %s will be kept because it would be the only remaining file in current group", candidates[0])
	return candidates[1:]
}
<|endoftext|>


<commit_before>//Command to run test version:
//goapp serve app.yaml
//Command to deploy/update application:
//goapp deploy -application golangnode0 -version 0

package main

import (
	"fmt"
	"html/template"
	"net/http"
)

type webPage struct {
	Title string
}

func helloWorld(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hello World!")
}

func startPage(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "Hello, test application started.")
	fmt.Fprintln(w, "\n - /helloworld - show title page")
	fmt.Fprintln(w, "\n - /showinfo - show information about this thing")
}

func showInfo(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "Information page for test project.")
	fmt.Fprintln(w, "Language - Go;")
	fmt.Fprintln(w, "Platform - Google Application Engine;")
}

func pageRender(w http.ResponseWriter, r *http.Request) {
	templatePage, _ := template.ParseFiles("index.html")
	templatePage.Execute(w, &webPage{"simplePage"})
}

/*
func sendEmail(w http.ResponseWriter, r *http.Redirect) {

}
*/

func init() {
	//fs := http.FileServer(http.Dir("/home/greyson_dean/testGoApp/files"))

	http.HandleFunc("/", startPage)
	http.HandleFunc("/helloworld", helloWorld)
	http.HandleFunc("/showinfo", showInfo)
	//http.Handle("/files/", http.StripPrefix("/files", fs))
	//Wrong code for App Enine - server cant understand what it need to show
	//http.ListenAndServe(":80", nil)
}

//this func not needed for deploy on Google App Engine, init() func replace main()
/*
func main() {
	fmt.Println("Hello, test server started on 8080 port.\n - /helloworld - show title page\n - /showinfo - show information about this thing")
	init()
	http.ListenAndServe(":8080", nil)
}
*/
<commit_msg>added html page hamdler<commit_after>//Command to run test version:
//goapp serve app.yaml
//Command to deploy/update application:
//goapp deploy -application golangnode0 -version 0

package main

import (
	"fmt"
	"html/template"
	"net/http"
)

type webPage struct {
	Title string
}

func helloWorld(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hello World!")
}

func startPage(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "Hello, test application started.")
	fmt.Fprintln(w, "\n - /helloworld - show title page")
	fmt.Fprintln(w, "\n - /showinfo - show information about this thing")
}

func showInfo(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "Information page for test project.")
	fmt.Fprintln(w, "Language - Go;")
	fmt.Fprintln(w, "Platform - Google Application Engine;")
}

func pageRender(w http.ResponseWriter, r *http.Request) {
	templatePage, _ := template.ParseFiles("index.html")
	templatePage.Execute(w, &webPage{"simplePage"})
}

/*
func sendEmail(w http.ResponseWriter, r *http.Redirect) {

}
*/

func init() {
	//fs := http.FileServer(http.Dir("/home/greyson_dean/testGoApp/files"))

	http.HandleFunc("/", startPage)
	http.HandleFunc("/helloworld", helloWorld)
	http.HandleFunc("/showinfo", showInfo)
	http.HandleFunc("/webpage", pageRender)
	//http.Handle("/files/", http.StripPrefix("/files", fs))
	//Wrong code for App Enine - server cant understand what it need to show
	//http.ListenAndServe(":80", nil)
}

//this func not needed for deploy on Google App Engine, init() func replace main()
/*
func main() {
	fmt.Println("Hello, test server started on 8080 port.\n - /helloworld - show title page\n - /showinfo - show information about this thing")
	init()
	http.ListenAndServe(":8080", nil)
}
*/
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/icholy/im/jira"
	"github.com/icholy/im/workday"
)

var (
	isPing   bool
	isWeb    bool
	isToday  bool
	isMonth  bool
	isUndo   bool
	isTest   time.Duration
	webAddr  string
	jiraUser string
	jiraPass string
	timeout  time.Duration
)

func init() {
	flag.BoolVar(&isPing, "ping", false, "update workday extent")
	flag.DurationVar(&timeout, "ping.timeout", 10*time.Second, "ping timeout")
	flag.BoolVar(&isWeb, "web", false, "start web server")
	flag.BoolVar(&isToday, "today", false, "show tasks from today")
	flag.BoolVar(&isMonth, "month", false, "show tasks from month")
	flag.BoolVar(&isUndo, "undo", false, "undo last task for today")
	flag.DurationVar(&isTest, "test", 0, "exit with 0 if there are tasks")
	flag.StringVar(&webAddr, "web.addr", ":8081", "web address to listen on")
	flag.StringVar(&jiraUser, "jira.username", "icholy", "jira username")
	flag.StringVar(&jiraPass, "jira.password", "", "jira password")

	home, err := os.UserHomeDir()
	if err != nil {
		log.Fatal(err)
	}
	workday.DataDir = filepath.Join(home, ".im")

	flag.Parse()
}

func ping() error {
	if err := workday.LockDataDir(); err != nil {
		log.Fatal(err)
	}
	defer workday.UnlockDataDir()
	defer workday.Ping()

	issues, err := jira.InProgress(jiraUser, jiraPass, timeout)
	if err != nil {
		return err
	}
	for _, issue := range issues {
		desc := fmt.Sprintf("%s: %s", issue.Name, issue.Summary)
		if err := workday.AddTask(desc); err != nil {
			return err
		}
	}
	return nil
}

func today() error {
	if err := workday.LockDataDir(); err != nil {
		log.Fatal(err)
	}
	defer workday.UnlockDataDir()
	day, err := workday.LoadDay(time.Now())
	if err != nil {
		return err
	}
	fmt.Println(day.String())
	return nil
}

func month() error {
	if err := workday.LockDataDir(); err != nil {
		return err
	}
	defer workday.UnlockDataDir()
	now := time.Now()
	days, err := workday.DaysForMonth(now.Year(), now.Month())
	if err != nil {
		return err
	}
	for _, d := range days {
		fmt.Println(d.String())
	}
	return nil
}

func getDescription() (string, error) {
	// use args as message if there are any
	if flag.NArg() > 0 {
		return strings.Join(flag.Args(), " "), nil
	}
	// create temp file
	f, err := ioutil.TempFile(os.TempDir(), "im.")
	if err != nil {
		return "", err
	}
	defer f.Close()
	defer os.Remove(f.Name())
	// open file in editor
	editor := os.Getenv("EDITOR")
	if editor == "" {
		return "", errors.New("EDITOR environment variable not set")
	}
	cmd := exec.Command(editor, f.Name())
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return "", err
	}
	// read file contents
	data, err := ioutil.ReadFile(f.Name())
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(data)), nil
}

func test() error {
	now := time.Now()
	day, err := workday.LoadDay(now)
	if err != nil {
		return err
	}
	if len(day.Tasks) == 0 {
		fmt.Println("You don't have any tasks recorded for the day")
		return nil
	}
	var (
		latest  = day.Tasks[len(day.Tasks)-1]
		idleFor = now.Sub(latest.Time)
	)
	if idleFor > isTest {
		fmt.Printf(
			"You haven't recorded any tasks for %s\n",
			time.Duration(idleFor.Seconds())*time.Second, // rounded
		)
	}
	return nil
}

func undo() error {
	return workday.Undo()
}

func addTask() error {
	desc, err := getDescription()
	if err != nil {
		return err
	}
	if desc == "" {
		return errors.New("description cannot be blank")
	}
	if err := workday.LockDataDir(); err != nil {
		return err
	}
	defer workday.UnlockDataDir()
	return workday.AddTask(desc)
}

func handleErr(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	switch {
	case isPing:
		handleErr(ping())
	case isToday:
		handleErr(today())
	case isMonth:
		handleErr(month())
	case isTest != 0:
		handleErr(test())
	case isUndo:
		handleErr(undo())
	case isWeb:
		handleErr(web(webAddr))
	default:
		handleErr(addTask())
	}
}
<commit_msg>don't query jira if there's no user/pass provided<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/icholy/im/jira"
	"github.com/icholy/im/workday"
)

var (
	isPing   bool
	isWeb    bool
	isToday  bool
	isMonth  bool
	isUndo   bool
	isTest   time.Duration
	webAddr  string
	jiraUser string
	jiraPass string
	timeout  time.Duration
)

func init() {
	flag.BoolVar(&isPing, "ping", false, "update workday extent")
	flag.DurationVar(&timeout, "ping.timeout", 10*time.Second, "ping timeout")
	flag.BoolVar(&isWeb, "web", false, "start web server")
	flag.BoolVar(&isToday, "today", false, "show tasks from today")
	flag.BoolVar(&isMonth, "month", false, "show tasks from month")
	flag.BoolVar(&isUndo, "undo", false, "undo last task for today")
	flag.DurationVar(&isTest, "test", 0, "exit with 0 if there are tasks")
	flag.StringVar(&webAddr, "web.addr", ":8081", "web address to listen on")
	flag.StringVar(&jiraUser, "jira.username", "icholy", "jira username")
	flag.StringVar(&jiraPass, "jira.password", "", "jira password")

	home, err := os.UserHomeDir()
	if err != nil {
		log.Fatal(err)
	}
	workday.DataDir = filepath.Join(home, ".im")

	flag.Parse()
}

func ping() error {
	if err := workday.LockDataDir(); err != nil {
		log.Fatal(err)
	}
	defer workday.UnlockDataDir()
	defer workday.Ping()

	if jiraUser == "" && jiraPass == "" {
		return nil
	}
	issues, err := jira.InProgress(jiraUser, jiraPass, timeout)
	if err != nil {
		return err
	}
	for _, issue := range issues {
		desc := fmt.Sprintf("%s: %s", issue.Name, issue.Summary)
		if err := workday.AddTask(desc); err != nil {
			return err
		}
	}
	return nil
}

func today() error {
	if err := workday.LockDataDir(); err != nil {
		log.Fatal(err)
	}
	defer workday.UnlockDataDir()
	day, err := workday.LoadDay(time.Now())
	if err != nil {
		return err
	}
	fmt.Println(day.String())
	return nil
}

func month() error {
	if err := workday.LockDataDir(); err != nil {
		return err
	}
	defer workday.UnlockDataDir()
	now := time.Now()
	days, err := workday.DaysForMonth(now.Year(), now.Month())
	if err != nil {
		return err
	}
	for _, d := range days {
		fmt.Println(d.String())
	}
	return nil
}

func getDescription() (string, error) {
	// use args as message if there are any
	if flag.NArg() > 0 {
		return strings.Join(flag.Args(), " "), nil
	}
	// create temp file
	f, err := ioutil.TempFile(os.TempDir(), "im.")
	if err != nil {
		return "", err
	}
	defer f.Close()
	defer os.Remove(f.Name())
	// open file in editor
	editor := os.Getenv("EDITOR")
	if editor == "" {
		return "", errors.New("EDITOR environment variable not set")
	}
	cmd := exec.Command(editor, f.Name())
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return "", err
	}
	// read file contents
	data, err := ioutil.ReadFile(f.Name())
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(data)), nil
}

func test() error {
	now := time.Now()
	day, err := workday.LoadDay(now)
	if err != nil {
		return err
	}
	if len(day.Tasks) == 0 {
		fmt.Println("You don't have any tasks recorded for the day")
		return nil
	}
	var (
		latest  = day.Tasks[len(day.Tasks)-1]
		idleFor = now.Sub(latest.Time)
	)
	if idleFor > isTest {
		fmt.Printf(
			"You haven't recorded any tasks for %s\n",
			time.Duration(idleFor.Seconds())*time.Second, // rounded
		)
	}
	return nil
}

func undo() error {
	return workday.Undo()
}

func addTask() error {
	desc, err := getDescription()
	if err != nil {
		return err
	}
	if desc == "" {
		return errors.New("description cannot be blank")
	}
	if err := workday.LockDataDir(); err != nil {
		return err
	}
	defer workday.UnlockDataDir()
	return workday.AddTask(desc)
}

func handleErr(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	switch {
	case isPing:
		handleErr(ping())
	case isToday:
		handleErr(today())
	case isMonth:
		handleErr(month())
	case isTest != 0:
		handleErr(test())
	case isUndo:
		handleErr(undo())
	case isWeb:
		handleErr(web(webAddr))
	default:
		handleErr(addTask())
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 PaperCut Software International Pty. Ltd.

package osutils

import (
	"fmt"
	"syscall"
	"unsafe"
)

var (
	winhttp                               = syscall.NewLazyDLL("winhttp.dll")
	winHttpGetIEProxyConfigForCurrentUser = winhttp.NewProc("WinHttpGetIEProxyConfigForCurrentUser")
)

type winHttpCurrentUserIEProxyConfig struct {
	fAuthDetect       uint32
	lpszAutoConfigUrl *uint16
	lpszProxy         *uint16
	lpszProxyBypass   *uint16
}

func getHTTPProxy() (string, error) {
	proxyConfig := winHttpCurrentUserIEProxyConfig{}
	if r, _, e1 := winHttpGetIEProxyConfigForCurrentUser.Call(uintptr(unsafe.Pointer(&proxyConfig))); r == 0 {
		fmt.Printf("r %v, e1 %v\n", r, e1)
		var err error
		if e1 != nil {
			err = e1
		} else {
			err = syscall.GetLastError()
		}
		return "", err
	}

	return ptrToString(proxyConfig.lpszProxy, 1024), nil
}

func ptrToString(p *uint16, maxLength int) string {
	if p == nil {
		return ""
	}
	// Hack! Cast to a large array pointer, then "pretend" to slice to maxLength
	slice := (*[0xffff]uint16)(unsafe.Pointer(p))[:maxLength]
	ret := syscall.UTF16ToString(slice)
	return ret
}
<commit_msg>Removed accidently debug printf statement<commit_after>// Copyright (c) 2016 PaperCut Software International Pty. Ltd.

package osutils

import (
	"syscall"
	"unsafe"
)

var (
	winhttp                               = syscall.NewLazyDLL("winhttp.dll")
	winHttpGetIEProxyConfigForCurrentUser = winhttp.NewProc("WinHttpGetIEProxyConfigForCurrentUser")
)

type winHttpCurrentUserIEProxyConfig struct {
	fAuthDetect       uint32
	lpszAutoConfigUrl *uint16
	lpszProxy         *uint16
	lpszProxyBypass   *uint16
}

func getHTTPProxy() (string, error) {
	proxyConfig := winHttpCurrentUserIEProxyConfig{}
	if r, _, e1 := winHttpGetIEProxyConfigForCurrentUser.Call(uintptr(unsafe.Pointer(&proxyConfig))); r == 0 {
		var err error
		if e1 != nil {
			err = e1
		} else {
			err = syscall.GetLastError()
		}
		return "", err
	}

	return ptrToString(proxyConfig.lpszProxy, 1024), nil
}

func ptrToString(p *uint16, maxLength int) string {
	if p == nil {
		return ""
	}
	// Hack! Cast to a large array pointer, then "pretend" to slice to maxLength
	slice := (*[0xffff]uint16)(unsafe.Pointer(p))[:maxLength]
	ret := syscall.UTF16ToString(slice)
	return ret
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package main

import (
	"net/http"

	"github.com/uber-go/zap"
	testBackend "github.com/uber/zanzibar/test/lib/test_backend"
)

// PORT that the bench server listens to
var PORT int32 = 8092

func main() {
	logger := zap.New(zap.NewJSONEncoder())
	backend := testBackend.CreateBackend(PORT)
	err := backend.Bootstrap()
	if err != nil {
		panic(err)
	}

	handleContacts := func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(202)
	}
	backend.HandleFunc("POST", "/foo/contacts", handleContacts)

	logger.Info("Listening on port & serving")

	backend.Wait()
}
<commit_msg>fix a missing arg<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package main

import (
	"net/http"

	"github.com/uber-go/zap"
	testBackend "github.com/uber/zanzibar/test/lib/test_backend"
)

// PORT that the bench server listens to
var PORT int32 = 8092

func main() {
	logger := zap.New(zap.NewJSONEncoder())
	backend := testBackend.CreateBackend(PORT)
	err := backend.Bootstrap()
	if err != nil {
		panic(err)
	}

	handleContacts := func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(202)
	}
	handleGoogleNow := func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(202)
	}
	backend.HandleFunc("POST", "/foo/contacts", handleContacts)
	backend.HandleFunc("POST", "/foo/contacts", handleGoogleNow)

	logger.Info("Listening on port & serving")

	backend.Wait()
}
<|endoftext|>


<commit_before>package flickr

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

type FlickrClient struct {
	ApiKey      string
	ApiSecret   string
	HTTPClient  *http.Client
	EndpointUrl string
	Method      string
	Args        url.Values
}

func (c *FlickrClient) Sign(tokenSecret string) {
	// the "oauth_signature" param should not be included in the signing process
	c.Args.Del("oauth_signature")
	c.Args.Set("oauth_signature", getSignature(c, tokenSecret))
}

type RequestToken struct {
	OauthCallbackConfirmed bool
	OauthToken             string
	OauthTokenSecret       string
}

func (rt *RequestToken) Parse(response string) error {
	val, err := url.ParseQuery(strings.TrimSpace(response))
	if err != nil {
		return err
	}

	confirmed, _ := strconv.ParseBool(val.Get("oauth_callback_confirmed"))
	rt.OauthCallbackConfirmed = confirmed
	rt.OauthToken = val.Get("oauth_token")
	rt.OauthTokenSecret = val.Get("oauth_token_secret")

	return nil
}

func getSigningBaseString(client *FlickrClient) string {
	request_url := url.QueryEscape(client.EndpointUrl)
	query := url.QueryEscape(client.Args.Encode())

	return fmt.Sprintf("%s&%s&%s", client.Method, request_url, query)
}

func getSignature(client *FlickrClient, token_secret string) string {
	key := fmt.Sprintf("%s&%s", url.QueryEscape(client.ApiSecret), url.QueryEscape(token_secret))
	base_string := getSigningBaseString(client)

	mac := hmac.New(sha1.New, []byte(key))
	mac.Write([]byte(base_string))

	ret := base64.StdEncoding.EncodeToString(mac.Sum(nil))

	return ret
}

func generateNonce() string {
	rand.Seed(time.Now().UTC().UnixNano())
	var letters = []rune("123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ")
	b := make([]rune, 8)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

func getDefaultArgs() url.Values {
	args := url.Values{}
	args.Add("oauth_version", "1.0")
	args.Add("oauth_signature_method", "HMAC-SHA1")
	args.Add("oauth_nonce", generateNonce())
	args.Add("oauth_timestamp", fmt.Sprintf("%d", time.Now().Unix()))

	return args
}

func GetRequestToken(client *FlickrClient) (*RequestToken, error) {
	client.EndpointUrl = "https://www.flickr.com/services/oauth/request_token"
	args := getDefaultArgs()
	args.Add("oauth_consumer_key", client.ApiKey)
	args.Add("oauth_callback", "oob")

	// we don't have token secret at this stage, pass an empty string
	client.Sign("")
	api_url := fmt.Sprintf("%s?%s", client.EndpointUrl, client.Args.Encode())

	res, err := client.HTTPClient.Get(api_url)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	token := RequestToken{}
	token.Parse(string(body))

	return &token, nil
}

func GetAuthorizeUrl() {
}
<commit_msg>added FlickrClient constructor<commit_after>package flickr

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

type FlickrClient struct {
	ApiKey      string
	ApiSecret   string
	HTTPClient  *http.Client
	EndpointUrl string
	Method      string
	Args        url.Values
}

func NewFlickrClient(apiKey string, apiSecret string) *FlickrClient {
	return &FlickrClient{
		ApiKey:     apiKey,
		ApiSecret:  apiSecret,
		HTTPClient: &http.Client{},
		Method:     "GET",
		Args:       url.Values{},
	}
}

func (c *FlickrClient) Sign(tokenSecret string) {
	// the "oauth_signature" param should not be included in the signing process
	c.Args.Del("oauth_signature")
	c.Args.Set("oauth_signature", getSignature(c, tokenSecret))
}

type RequestToken struct {
	OauthCallbackConfirmed bool
	OauthToken             string
	OauthTokenSecret       string
}

func (rt *RequestToken) Parse(response string) error {
	val, err := url.ParseQuery(strings.TrimSpace(response))
	if err != nil {
		return err
	}

	confirmed, _ := strconv.ParseBool(val.Get("oauth_callback_confirmed"))
	rt.OauthCallbackConfirmed = confirmed
	rt.OauthToken = val.Get("oauth_token")
	rt.OauthTokenSecret = val.Get("oauth_token_secret")

	return nil
}

func getSigningBaseString(client *FlickrClient) string {
	request_url := url.QueryEscape(client.EndpointUrl)
	query := url.QueryEscape(client.Args.Encode())

	return fmt.Sprintf("%s&%s&%s", client.Method, request_url, query)
}

func getSignature(client *FlickrClient, token_secret string) string {
	key := fmt.Sprintf("%s&%s", url.QueryEscape(client.ApiSecret), url.QueryEscape(token_secret))
	base_string := getSigningBaseString(client)

	mac := hmac.New(sha1.New, []byte(key))
	mac.Write([]byte(base_string))

	ret := base64.StdEncoding.EncodeToString(mac.Sum(nil))

	return ret
}

func generateNonce() string {
	rand.Seed(time.Now().UTC().UnixNano())
	var letters = []rune("123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ")
	b := make([]rune, 8)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

func getDefaultArgs() url.Values {
	args := url.Values{}
	args.Add("oauth_version", "1.0")
	args.Add("oauth_signature_method", "HMAC-SHA1")
	args.Add("oauth_nonce", generateNonce())
	args.Add("oauth_timestamp", fmt.Sprintf("%d", time.Now().Unix()))

	return args
}

func GetRequestToken(client *FlickrClient) (*RequestToken, error) {
	client.EndpointUrl = "https://www.flickr.com/services/oauth/request_token"
	args := getDefaultArgs()
	args.Add("oauth_consumer_key", client.ApiKey)
	args.Add("oauth_callback", "oob")

	// we don't have token secret at this stage, pass an empty string
	client.Sign("")
	api_url := fmt.Sprintf("%s?%s", client.EndpointUrl, client.Args.Encode())

	res, err := client.HTTPClient.Get(api_url)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	token := RequestToken{}
	token.Parse(string(body))

	return &token, nil
}

func GetAuthorizeUrl() {
}
<|endoftext|>


<commit_before>// Local filesystem interface
package local

// Note that all rclone paths should be / separated.  Anything coming
// from the filepath module will have \ separators on windows so
// should be converted using filepath.ToSlash.  Windows is quite happy
// with / separators so there is no need to convert them back.

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"
	"unicode/utf8"

	"github.com/ncw/rclone/fs"
)

// Register with Fs
func init() {
	fs.Register(&fs.FsInfo{
		Name:  "local",
		NewFs: NewFs,
	})
}

// FsLocal represents a local filesystem rooted at root
type FsLocal struct {
	root        string              // The root directory
	precisionOk sync.Once           // Whether we need to read the precision
	precision   time.Duration       // precision of local filesystem
	warned      map[string]struct{} // whether we have warned about this string
}

// FsObjectLocal represents a local filesystem object
type FsObjectLocal struct {
	local  *FsLocal    // The Fs this object is part of
	remote string      // The remote path
	path   string      // The local path
	info   os.FileInfo // Interface for file info (always present)
	md5sum string      // the md5sum of the object or "" if not calculated
}

// ------------------------------------------------------------

// NewFs contstructs an FsLocal from the path
func NewFs(name, root string) (fs.Fs, error) {
	root = filepath.ToSlash(path.Clean(root))
	f := &FsLocal{
		root:   root,
		warned: make(map[string]struct{}),
	}
	// Check to see if this points to a file
	fi, err := os.Lstat(f.root)
	if err == nil && fi.Mode().IsRegular() {
		// It is a file, so use the parent as the root
		remote := path.Base(root)
		f.root = path.Dir(root)
		obj := f.NewFsObject(remote)
		// return a Fs Limited to this object
		return fs.NewLimited(f, obj), nil
	}
	return f, nil
}

// String converts this FsLocal to a string
func (f *FsLocal) String() string {
	return fmt.Sprintf("Local file system at %s", f.root)
}

// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsLocal) newFsObjectWithInfo(remote string, info os.FileInfo) fs.Object {
	remote = filepath.ToSlash(remote)
	path := path.Join(f.root, remote)
	o := &FsObjectLocal{local: f, remote: remote, path: path}
	if info != nil {
		o.info = info
	} else {
		err := o.lstat()
		if err != nil {
			fs.Debug(o, "Failed to stat %s: %s", path, err)
			return nil
		}
	}
	return o
}

// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsLocal) NewFsObject(remote string) fs.Object {
	return f.newFsObjectWithInfo(remote, nil)
}

// List the path returning a channel of FsObjects
//
// Ignores everything which isn't Storable, eg links etc
func (f *FsLocal) List() fs.ObjectsChan {
	out := make(fs.ObjectsChan, fs.Config.Checkers)
	go func() {
		err := filepath.Walk(f.root, func(path string, fi os.FileInfo, err error) error {
			if err != nil {
				fs.Stats.Error()
				fs.Log(f, "Failed to open directory: %s: %s", path, err)
			} else {
				remote, err := filepath.Rel(f.root, path)
				if err != nil {
					fs.Stats.Error()
					fs.Log(f, "Failed to get relative path %s: %s", path, err)
					return nil
				}
				if remote == "." {
					return nil
					// remote = ""
				}
				if fs := f.newFsObjectWithInfo(remote, fi); fs != nil {
					if fs.Storable() {
						out <- fs
					}
				}
			}
			return nil
		})
		if err != nil {
			fs.Stats.Error()
			fs.Log(f, "Failed to open directory: %s: %s", f.root, err)
		}
		close(out)
	}()
	return out
}

// CleanUtf8 makes string a valid UTF-8 string
//
// Any invalid UTF-8 characters will be replaced with utf8.RuneError
func (f *FsLocal) cleanUtf8(name string) string {
	if utf8.ValidString(name) {
		return name
	}
	if _, ok := f.warned[name]; !ok {
		fs.Debug(f, "Replacing invalid UTF-8 characters in %q", name)
		f.warned[name] = struct{}{}
	}
	return string([]rune(name))
}

// Walk the path returning a channel of FsObjects
func (f *FsLocal) ListDir() fs.DirChan {
	out := make(fs.DirChan, fs.Config.Checkers)
	go func() {
		defer close(out)
		items, err := ioutil.ReadDir(f.root)
		if err != nil {
			fs.Stats.Error()
			fs.Log(f, "Couldn't find read directory: %s", err)
		} else {
			for _, item := range items {
				if item.IsDir() {
					dir := &fs.Dir{
						Name:  f.cleanUtf8(item.Name()),
						When:  item.ModTime(),
						Bytes: 0,
						Count: 0,
					}
					// Go down the tree to count the files and directories
					dirpath := path.Join(f.root, item.Name())
					err := filepath.Walk(dirpath, func(path string, fi os.FileInfo, err error) error {
						if err != nil {
							fs.Stats.Error()
							fs.Log(f, "Failed to open directory: %s: %s", path, err)
						} else {
							dir.Count += 1
							dir.Bytes += fi.Size()
						}
						return nil
					})
					if err != nil {
						fs.Stats.Error()
						fs.Log(f, "Failed to open directory: %s: %s", dirpath, err)
					}
					out <- dir
				}
			}
		}
		// err := f.findRoot(false)
	}()
	return out
}

// Puts the FsObject to the local filesystem
func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
	dstPath := path.Join(f.root, remote)
	// Temporary FsObject under construction - info filled in by Update()
	o := &FsObjectLocal{local: f, remote: remote, path: dstPath}
	err := o.Update(in, modTime, size)
	if err != nil {
		return nil, err
	}
	return o, nil
}

// Mkdir creates the directory if it doesn't exist
func (f *FsLocal) Mkdir() error {
	return os.MkdirAll(f.root, 0770)
}

// Rmdir removes the directory
//
// If it isn't empty it will return an error
func (f *FsLocal) Rmdir() error {
	return os.Remove(f.root)
}

// Return the precision
func (f *FsLocal) Precision() (precision time.Duration) {
	f.precisionOk.Do(func() {
		f.precision = f.readPrecision()
	})
	return f.precision
}

// Read the precision
func (f *FsLocal) readPrecision() (precision time.Duration) {
	// Default precision of 1s
	precision = time.Second

	// Create temporary file and test it
	fd, err := ioutil.TempFile("", "rclone")
	if err != nil {
		// If failed return 1s
		// fmt.Println("Failed to create temp file", err)
		return time.Second
	}
	path := fd.Name()
	// fmt.Println("Created temp file", path)
	err = fd.Close()
	if err != nil {
		return time.Second
	}

	// Delete it on return
	defer func() {
		// fmt.Println("Remove temp file")
		_ = os.Remove(path) // ignore error
	}()

	// Find the minimum duration we can detect
	for duration := time.Duration(1); duration < time.Second; duration *= 10 {
		// Current time with delta
		t := time.Unix(time.Now().Unix(), int64(duration))
		err := os.Chtimes(path, t, t)
		if err != nil {
			// fmt.Println("Failed to Chtimes", err)
			break
		}

		// Read the actual time back
		fi, err := os.Stat(path)
		if err != nil {
			// fmt.Println("Failed to Stat", err)
			break
		}

		// If it matches - have found the precision
		// fmt.Println("compare", fi.ModTime(), t)
		if fi.ModTime() == t {
			// fmt.Println("Precision detected as", duration)
			return duration
		}
	}
	return
}

// Purge deletes all the files and directories
//
// Optional interface: Only implement this if you have a way of
// deleting all the files quicker than just running Remove() on the
// result of List()
func (f *FsLocal) Purge() error {
	fi, err := os.Lstat(f.root)
	if err != nil {
		return err
	}
	if !fi.Mode().IsDir() {
		return fmt.Errorf("Can't Purge non directory: %q", f.root)
	}
	return os.RemoveAll(f.root)
}

// ------------------------------------------------------------

// Return the parent Fs
func (o *FsObjectLocal) Fs() fs.Fs {
	return o.local
}

// Return a string version
func (o *FsObjectLocal) String() string {
	if o == nil {
		return "<nil>"
	}
	return o.remote
}

// Return the remote path
func (o *FsObjectLocal) Remote() string {
	return o.local.cleanUtf8(o.remote)
}

// Md5sum calculates the Md5sum of a file returning a lowercase hex string
func (o *FsObjectLocal) Md5sum() (string, error) {
	if o.md5sum != "" {
		return o.md5sum, nil
	}
	in, err := os.Open(o.path)
	if err != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to open: %s", err)
		return "", err
	}
	hash := md5.New()
	_, err = io.Copy(hash, in)
	closeErr := in.Close()
	if err != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to read: %s", err)
		return "", err
	}
	if closeErr != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to close: %s", closeErr)
		return "", closeErr
	}
	o.md5sum = hex.EncodeToString(hash.Sum(nil))
	return o.md5sum, nil
}

// Size returns the size of an object in bytes
func (o *FsObjectLocal) Size() int64 {
	return o.info.Size()
}

// ModTime returns the modification time of the object
func (o *FsObjectLocal) ModTime() time.Time {
	return o.info.ModTime()
}

// Sets the modification time of the local fs object
func (o *FsObjectLocal) SetModTime(modTime time.Time) {
	err := os.Chtimes(o.path, modTime, modTime)
	if err != nil {
		fs.Debug(o, "Failed to set mtime on file: %s", err)
		return
	}
	// Re-read metadata
	err = o.lstat()
	if err != nil {
		fs.Debug(o, "Failed to stat: %s", err)
		return
	}
}

// Is this object storable
func (o *FsObjectLocal) Storable() bool {
	mode := o.info.Mode()
	if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
		fs.Debug(o, "Can't transfer non file/directory")
		return false
	} else if mode&os.ModeDir != 0 {
		// fs.Debug(o, "Skipping directory")
		return false
	}
	return true
}

// localOpenFile wraps an io.ReadCloser and updates the md5sum of the
// object that is read
type localOpenFile struct {
	o    *FsObjectLocal // object that is open
	in   io.ReadCloser  // handle we are wrapping
	hash hash.Hash      // currently accumulating MD5
}

// Read bytes from the object - see io.Reader
func (file *localOpenFile) Read(p []byte) (n int, err error) {
	n, err = file.in.Read(p)
	if n > 0 {
		// Hash routines never return an error
		_, _ = file.hash.Write(p[:n])
	}
	return
}

// Close the object and update the md5sum
func (file *localOpenFile) Close() (err error) {
	err = file.in.Close()
	if err == nil {
		file.o.md5sum = hex.EncodeToString(file.hash.Sum(nil))
	} else {
		file.o.md5sum = ""
	}
	return err
}

// Open an object for read
func (o *FsObjectLocal) Open() (in io.ReadCloser, err error) {
	in, err = os.Open(o.path)
	if err != nil {
		return
	}
	// Update the md5sum as we go along
	in = &localOpenFile{
		o:    o,
		in:   in,
		hash: md5.New(),
	}
	return
}

// Update the object from in with modTime and size
func (o *FsObjectLocal) Update(in io.Reader, modTime time.Time, size int64) error {
	dir := path.Dir(o.path)
	err := os.MkdirAll(dir, 0770)
	if err != nil {
		return err
	}

	out, err := os.Create(o.path)
	if err != nil {
		return err
	}

	// Calculate the md5sum of the object we are reading as we go along
	hash := md5.New()
	in = io.TeeReader(in, hash)

	_, err = io.Copy(out, in)
	outErr := out.Close()
	if err != nil {
		return err
	}
	if outErr != nil {
		return outErr
	}

	// All successful so update the md5sum
	o.md5sum = hex.EncodeToString(hash.Sum(nil))

	// Set the mtime
	o.SetModTime(modTime)

	// ReRead info now that we have finished
	return o.lstat()
}

// Stat a FsObject into info
func (o *FsObjectLocal) lstat() error {
	info, err := os.Lstat(o.path)
	o.info = info
	return err
}

// Remove an object
func (o *FsObjectLocal) Remove() error {
	return os.Remove(o.path)
}

// Check the interfaces are satisfied
var _ fs.Fs = &FsLocal{}
var _ fs.Purger = &FsLocal{}
var _ fs.Object = &FsObjectLocal{}
<commit_msg>Fix created directories not obeying umask<commit_after>// Local filesystem interface
package local

// Note that all rclone paths should be / separated.  Anything coming
// from the filepath module will have \ separators on windows so
// should be converted using filepath.ToSlash.  Windows is quite happy
// with / separators so there is no need to convert them back.

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"
	"unicode/utf8"

	"github.com/ncw/rclone/fs"
)

// Register with Fs
func init() {
	fs.Register(&fs.FsInfo{
		Name:  "local",
		NewFs: NewFs,
	})
}

// FsLocal represents a local filesystem rooted at root
type FsLocal struct {
	root        string              // The root directory
	precisionOk sync.Once           // Whether we need to read the precision
	precision   time.Duration       // precision of local filesystem
	warned      map[string]struct{} // whether we have warned about this string
}

// FsObjectLocal represents a local filesystem object
type FsObjectLocal struct {
	local  *FsLocal    // The Fs this object is part of
	remote string      // The remote path
	path   string      // The local path
	info   os.FileInfo // Interface for file info (always present)
	md5sum string      // the md5sum of the object or "" if not calculated
}

// ------------------------------------------------------------

// NewFs contstructs an FsLocal from the path
func NewFs(name, root string) (fs.Fs, error) {
	root = filepath.ToSlash(path.Clean(root))
	f := &FsLocal{
		root:   root,
		warned: make(map[string]struct{}),
	}
	// Check to see if this points to a file
	fi, err := os.Lstat(f.root)
	if err == nil && fi.Mode().IsRegular() {
		// It is a file, so use the parent as the root
		remote := path.Base(root)
		f.root = path.Dir(root)
		obj := f.NewFsObject(remote)
		// return a Fs Limited to this object
		return fs.NewLimited(f, obj), nil
	}
	return f, nil
}

// String converts this FsLocal to a string
func (f *FsLocal) String() string {
	return fmt.Sprintf("Local file system at %s", f.root)
}

// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsLocal) newFsObjectWithInfo(remote string, info os.FileInfo) fs.Object {
	remote = filepath.ToSlash(remote)
	path := path.Join(f.root, remote)
	o := &FsObjectLocal{local: f, remote: remote, path: path}
	if info != nil {
		o.info = info
	} else {
		err := o.lstat()
		if err != nil {
			fs.Debug(o, "Failed to stat %s: %s", path, err)
			return nil
		}
	}
	return o
}

// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsLocal) NewFsObject(remote string) fs.Object {
	return f.newFsObjectWithInfo(remote, nil)
}

// List the path returning a channel of FsObjects
//
// Ignores everything which isn't Storable, eg links etc
func (f *FsLocal) List() fs.ObjectsChan {
	out := make(fs.ObjectsChan, fs.Config.Checkers)
	go func() {
		err := filepath.Walk(f.root, func(path string, fi os.FileInfo, err error) error {
			if err != nil {
				fs.Stats.Error()
				fs.Log(f, "Failed to open directory: %s: %s", path, err)
			} else {
				remote, err := filepath.Rel(f.root, path)
				if err != nil {
					fs.Stats.Error()
					fs.Log(f, "Failed to get relative path %s: %s", path, err)
					return nil
				}
				if remote == "." {
					return nil
					// remote = ""
				}
				if fs := f.newFsObjectWithInfo(remote, fi); fs != nil {
					if fs.Storable() {
						out <- fs
					}
				}
			}
			return nil
		})
		if err != nil {
			fs.Stats.Error()
			fs.Log(f, "Failed to open directory: %s: %s", f.root, err)
		}
		close(out)
	}()
	return out
}

// CleanUtf8 makes string a valid UTF-8 string
//
// Any invalid UTF-8 characters will be replaced with utf8.RuneError
func (f *FsLocal) cleanUtf8(name string) string {
	if utf8.ValidString(name) {
		return name
	}
	if _, ok := f.warned[name]; !ok {
		fs.Debug(f, "Replacing invalid UTF-8 characters in %q", name)
		f.warned[name] = struct{}{}
	}
	return string([]rune(name))
}

// Walk the path returning a channel of FsObjects
func (f *FsLocal) ListDir() fs.DirChan {
	out := make(fs.DirChan, fs.Config.Checkers)
	go func() {
		defer close(out)
		items, err := ioutil.ReadDir(f.root)
		if err != nil {
			fs.Stats.Error()
			fs.Log(f, "Couldn't find read directory: %s", err)
		} else {
			for _, item := range items {
				if item.IsDir() {
					dir := &fs.Dir{
						Name:  f.cleanUtf8(item.Name()),
						When:  item.ModTime(),
						Bytes: 0,
						Count: 0,
					}
					// Go down the tree to count the files and directories
					dirpath := path.Join(f.root, item.Name())
					err := filepath.Walk(dirpath, func(path string, fi os.FileInfo, err error) error {
						if err != nil {
							fs.Stats.Error()
							fs.Log(f, "Failed to open directory: %s: %s", path, err)
						} else {
							dir.Count += 1
							dir.Bytes += fi.Size()
						}
						return nil
					})
					if err != nil {
						fs.Stats.Error()
						fs.Log(f, "Failed to open directory: %s: %s", dirpath, err)
					}
					out <- dir
				}
			}
		}
		// err := f.findRoot(false)
	}()
	return out
}

// Puts the FsObject to the local filesystem
func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
	dstPath := path.Join(f.root, remote)
	// Temporary FsObject under construction - info filled in by Update()
	o := &FsObjectLocal{local: f, remote: remote, path: dstPath}
	err := o.Update(in, modTime, size)
	if err != nil {
		return nil, err
	}
	return o, nil
}

// Mkdir creates the directory if it doesn't exist
func (f *FsLocal) Mkdir() error {
	return os.MkdirAll(f.root, 0777)
}

// Rmdir removes the directory
//
// If it isn't empty it will return an error
func (f *FsLocal) Rmdir() error {
	return os.Remove(f.root)
}

// Return the precision
func (f *FsLocal) Precision() (precision time.Duration) {
	f.precisionOk.Do(func() {
		f.precision = f.readPrecision()
	})
	return f.precision
}

// Read the precision
func (f *FsLocal) readPrecision() (precision time.Duration) {
	// Default precision of 1s
	precision = time.Second

	// Create temporary file and test it
	fd, err := ioutil.TempFile("", "rclone")
	if err != nil {
		// If failed return 1s
		// fmt.Println("Failed to create temp file", err)
		return time.Second
	}
	path := fd.Name()
	// fmt.Println("Created temp file", path)
	err = fd.Close()
	if err != nil {
		return time.Second
	}

	// Delete it on return
	defer func() {
		// fmt.Println("Remove temp file")
		_ = os.Remove(path) // ignore error
	}()

	// Find the minimum duration we can detect
	for duration := time.Duration(1); duration < time.Second; duration *= 10 {
		// Current time with delta
		t := time.Unix(time.Now().Unix(), int64(duration))
		err := os.Chtimes(path, t, t)
		if err != nil {
			// fmt.Println("Failed to Chtimes", err)
			break
		}

		// Read the actual time back
		fi, err := os.Stat(path)
		if err != nil {
			// fmt.Println("Failed to Stat", err)
			break
		}

		// If it matches - have found the precision
		// fmt.Println("compare", fi.ModTime(), t)
		if fi.ModTime() == t {
			// fmt.Println("Precision detected as", duration)
			return duration
		}
	}
	return
}

// Purge deletes all the files and directories
//
// Optional interface: Only implement this if you have a way of
// deleting all the files quicker than just running Remove() on the
// result of List()
func (f *FsLocal) Purge() error {
	fi, err := os.Lstat(f.root)
	if err != nil {
		return err
	}
	if !fi.Mode().IsDir() {
		return fmt.Errorf("Can't Purge non directory: %q", f.root)
	}
	return os.RemoveAll(f.root)
}

// ------------------------------------------------------------

// Return the parent Fs
func (o *FsObjectLocal) Fs() fs.Fs {
	return o.local
}

// Return a string version
func (o *FsObjectLocal) String() string {
	if o == nil {
		return "<nil>"
	}
	return o.remote
}

// Return the remote path
func (o *FsObjectLocal) Remote() string {
	return o.local.cleanUtf8(o.remote)
}

// Md5sum calculates the Md5sum of a file returning a lowercase hex string
func (o *FsObjectLocal) Md5sum() (string, error) {
	if o.md5sum != "" {
		return o.md5sum, nil
	}
	in, err := os.Open(o.path)
	if err != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to open: %s", err)
		return "", err
	}
	hash := md5.New()
	_, err = io.Copy(hash, in)
	closeErr := in.Close()
	if err != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to read: %s", err)
		return "", err
	}
	if closeErr != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to close: %s", closeErr)
		return "", closeErr
	}
	o.md5sum = hex.EncodeToString(hash.Sum(nil))
	return o.md5sum, nil
}

// Size returns the size of an object in bytes
func (o *FsObjectLocal) Size() int64 {
	return o.info.Size()
}

// ModTime returns the modification time of the object
func (o *FsObjectLocal) ModTime() time.Time {
	return o.info.ModTime()
}

// Sets the modification time of the local fs object
func (o *FsObjectLocal) SetModTime(modTime time.Time) {
	err := os.Chtimes(o.path, modTime, modTime)
	if err != nil {
		fs.Debug(o, "Failed to set mtime on file: %s", err)
		return
	}
	// Re-read metadata
	err = o.lstat()
	if err != nil {
		fs.Debug(o, "Failed to stat: %s", err)
		return
	}
}

// Is this object storable
func (o *FsObjectLocal) Storable() bool {
	mode := o.info.Mode()
	if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
		fs.Debug(o, "Can't transfer non file/directory")
		return false
	} else if mode&os.ModeDir != 0 {
		// fs.Debug(o, "Skipping directory")
		return false
	}
	return true
}

// localOpenFile wraps an io.ReadCloser and updates the md5sum of the
// object that is read
type localOpenFile struct {
	o    *FsObjectLocal // object that is open
	in   io.ReadCloser  // handle we are wrapping
	hash hash.Hash      // currently accumulating MD5
}

// Read bytes from the object - see io.Reader
func (file *localOpenFile) Read(p []byte) (n int, err error) {
	n, err = file.in.Read(p)
	if n > 0 {
		// Hash routines never return an error
		_, _ = file.hash.Write(p[:n])
	}
	return
}

// Close the object and update the md5sum
func (file *localOpenFile) Close() (err error) {
	err = file.in.Close()
	if err == nil {
		file.o.md5sum = hex.EncodeToString(file.hash.Sum(nil))
	} else {
		file.o.md5sum = ""
	}
	return err
}

// Open an object for read
func (o *FsObjectLocal) Open() (in io.ReadCloser, err error) {
	in, err = os.Open(o.path)
	if err != nil {
		return
	}
	// Update the md5sum as we go along
	in = &localOpenFile{
		o:    o,
		in:   in,
		hash: md5.New(),
	}
	return
}

// Update the object from in with modTime and size
func (o *FsObjectLocal) Update(in io.Reader, modTime time.Time, size int64) error {
	dir := path.Dir(o.path)
	err := os.MkdirAll(dir, 0777)
	if err != nil {
		return err
	}

	out, err := os.Create(o.path)
	if err != nil {
		return err
	}

	// Calculate the md5sum of the object we are reading as we go along
	hash := md5.New()
	in = io.TeeReader(in, hash)

	_, err = io.Copy(out, in)
	outErr := out.Close()
	if err != nil {
		return err
	}
	if outErr != nil {
		return outErr
	}

	// All successful so update the md5sum
	o.md5sum = hex.EncodeToString(hash.Sum(nil))

	// Set the mtime
	o.SetModTime(modTime)

	// ReRead info now that we have finished
	return o.lstat()
}

// Stat a FsObject into info
func (o *FsObjectLocal) lstat() error {
	info, err := os.Lstat(o.path)
	o.info = info
	return err
}

// Remove an object
func (o *FsObjectLocal) Remove() error {
	return os.Remove(o.path)
}

// Check the interfaces are satisfied
var _ fs.Fs = &FsLocal{}
var _ fs.Purger = &FsLocal{}
var _ fs.Object = &FsObjectLocal{}
<|endoftext|>


<commit_before>package frank

import (
	"code.google.com/p/go.net/html"
	"code.google.com/p/go.net/html/atom"
	"errors"
	irc "github.com/fluffle/goirc/client"
	"io"
	"log"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// how many URLs can the cache store
const cacheSize = 100

// how many hours an entry should be considered valid
const cacheValidHours = 24

// how many kilo bytes should be considered when looking for the title
// tag.
const httpReadKByte = 100

// abort HTTP requests if it takes longer than X seconds. Not sure, it’s
// definitely magic involved. Must be larger than 5.
const httpGetDeadline = 10

// don’t repost the same title within this period
const noRepostWithinSeconds = 30

// matches all whitespace and zero bytes. Additionally, all Unicode
// characters of class Cf (format chars, e.g. right-to-left) and Cc
// (control chars) are matched.
var whitespaceRegex = regexp.MustCompile(`[\s\0\p{Cf}\p{Cc}]+`)

var ignoreDomainsRegex = regexp.MustCompile(`^http://p\.nnev\.de`)

var twitterDomainRegex = regexp.MustCompile(`(?i)^https?://(?:[a-z0-9]\.)?twitter.com`)
var twitterPicsRegex = regexp.MustCompile(`(?i)(?:\b|^)pic\.twitter\.com/[a-z0-9]+(?:\b|$)`)

var noSpoilerRegex = regexp.MustCompile(`(?i)(don't|no|kein|nicht) spoiler`)

func UriFind(conn *irc.Conn, line *irc.Line) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("MEGA-WTF:pkg: %v", r)
		}
	}()

	msg := line.Args[1]

	if noSpoilerRegex.MatchString(msg) {
		log.Printf("not spoilering this line: %s", msg)
		return
	}

	urls := extract(msg)

	for _, url := range urls {
		if url == "" {
			continue
		}

		if title := cacheGetTitleByUrl(url); title != "" {
			log.Printf("using cache for URL: %s", url)
			postTitle(conn, line, title, "Cache Info")
			continue
		}

		go func(url string) {
			if ignoreDomainsRegex.MatchString(url) {
				log.Printf("ignoring this URL: %s", url)
				return
			}

			log.Printf("testing URL: %s", url)
			title, _, err := TitleGet(url)
			if err != nil {
				//postTitle(conn, line, err.Error(), "Error")
			} else if title != "" {
				postTitle(conn, line, title, "")
				cacheAdd(url, title)
			}
		}(url)
	}
}

// regexing ////////////////////////////////////////////////////////////

func extract(msg string) []string {
	results := make([]string, 0)
	for idx := strings.Index(msg, "http"); idx > -1; idx = strings.Index(msg, "http") {
		url := msg[idx:]
		if !strings.HasPrefix(url, "http://") &&
			!strings.HasPrefix(url, "https://") {
			msg = msg[idx+len("http"):]
			continue
		}

		// End on commas, but only if they are followed by a space.
		// spiegel.de URLs have commas in them, that would be a
		// false positive otherwise.
		if end := strings.Index(url, ", "); end > -1 {
			url = url[:end]
		}

		// End on closing paren, but only if there is an opening
		// paren before the URL (should fix most false-positives).
		if end := strings.Index(url, ")"); idx > 0 && msg[idx-1] == '(' && end > -1 {
			url = url[:end]
		}

		// Whitespace always ends a URL.
		if end := strings.IndexAny(url, " \t"); end > -1 {
			url = url[:end]
		}

		results = append(results, url)
		msg = msg[idx+len(url):]
	}
	return results
}

// http/html stuff /////////////////////////////////////////////////////

func TitleGet(url string) (string, string, error) {
	// via http://www.reddit.com/r/golang/comments/10awvj/timeout_on_httpget/c6bz49s
	c := http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(time.Second * httpGetDeadline)
				c, err := net.DialTimeout(netw, addr, time.Second*(httpGetDeadline-5))
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
		},
	}

	r, err := c.Get(url)
	if err != nil {
		log.Printf("WTF: could not resolve %s: %s\n", url, err)
		return "", url, err
	}
	defer r.Body.Close()

	lastUrl := r.Request.URL.String()

	// TODO: r.Body → utf8?
	title, tweet := titleParseHtml(io.LimitReader(r.Body, 1024*httpReadKByte))

	if r.StatusCode != 200 {
		return "", lastUrl, errors.New("[" + strconv.Itoa(r.StatusCode) + "] " + title)
	}

	if tweet != "" && twitterDomainRegex.MatchString(lastUrl) {
		title = tweet
	}

	log.Printf("Title for URL %s: %s\n", url, title)

	return title, lastUrl, nil
}

// parses the incoming HTML fragment and tries to extract text from
// suitable tags. Currently this is the page’s title tag and tweets
// when the HTML-code is similar enough to twitter.com. Returns
// title and tweet.
func titleParseHtml(r io.Reader) (string, string) {
	doc, err := html.Parse(r)
	if err != nil {
		log.Printf("WTF: html parser blew up: %s\n", err)
		return "", ""
	}

	title := ""
	tweetText := ""
	tweetUser := ""
	tweetPicUrl := ""

	var f func(*html.Node)
	f = func(n *html.Node) {
		if title == "" && n.Type == html.ElementNode && n.DataAtom == atom.Title {
			title = extractText(n)
			return
		}

		if tweetText != "" && hasClass(n, "permalink-tweet") {
			// reset already found tweet text if we encounter a more
			// “prominent” one
			tweetText = ""
			tweetUser = ""
			tweetPicUrl = ""
		}

		if tweetText == "" && hasClass(n, "tweet-text") {
			tweetText = extractText(n)
			return
		}

		if tweetUser == "" && hasClass(n, "js-user-profile-link") {
			tweetUser = extractText(n)
			return
		}

		isMedia := hasClass(n, "media") || hasClass(n, "media-thumbnail")
		if tweetPicUrl == "" && isMedia && !hasClass(n, "profile-picture") {
			attrVal := getAttr(n, "data-url")
			if attrVal != "" {
				tweetPicUrl = attrVal
				return
			}
		}

		// recurse down
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}

	}
	f(doc)

	// cleanup
	tweet := ""
	if tweetText != "" {
		tweetText = twitterPicsRegex.ReplaceAllString(tweetText, "")
		tweetUser = strings.Replace(tweetUser, "@", "(@", 1) + "): "
		tweet = tweetUser + tweetText + " " + tweetPicUrl
		tweet = clean(tweet)
	}

	return strings.TrimSpace(title), strings.TrimSpace(tweet)
}

func extractText(n *html.Node) string {
	text := ""
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if c.Type == html.TextNode {
			text += c.Data
		} else {
			text += extractText(c)
		}
	}
	return clean(text)
}

func hasClass(n *html.Node, class string) bool {
	if n.Type != html.ElementNode {
		return false
	}

	class = " " + strings.TrimSpace(class) + " "
	attr := strings.Replace(getAttr(n, "class"), "\n", " ", -1)
	if strings.Contains(" "+attr+" ", class) {
		return true
	}
	return false
}

func getAttr(n *html.Node, findAttr string) string {
	for _, attr := range n.Attr {
		if attr.Key == findAttr {
			return attr.Val
		}
	}
	return ""
}

// Cache ///////////////////////////////////////////////////////////////
type Cache struct {
	url   string
	title string
	date  time.Time
}

var cache = [cacheSize]Cache{}
var cacheIndex = 0

func cacheAdd(url string, title string) {
	if len(cache) == cacheIndex {
		cacheIndex = 0
	}
	cache[cacheIndex] = Cache{url, title, time.Now()}
	cacheIndex += 1
}

func cacheGetTitleByUrl(url string) string {
	for _, cc := range cache {
		if cc.url == url && time.Since(cc.date).Hours() <= cacheValidHours {
			return cc.title
		}
	}
	return ""
}

func cacheGetSecondsToLastPost(title string) int {
	var secondsAgo = int(^uint(0) >> 1)
	for _, cc := range cache {
		var a = int(time.Since(cc.date).Seconds())
		if cc.title == title && a < secondsAgo {
			secondsAgo = a
		}
	}
	return secondsAgo
}

// util ////////////////////////////////////////////////////////////////

func postTitle(conn *irc.Conn, line *irc.Line, title string, prefix string) {
	tgt := line.Args[0]

	secondsAgo := cacheGetSecondsToLastPost(title)
	if secondsAgo <= noRepostWithinSeconds {
		log.Printf("Skipping, because posted %d seconds ago (“%s”)", secondsAgo, title)
		return
	}

	log.Printf("nick=%s, target=%s, title=%s", line.Nick, tgt, title)
	// if target is our current nick, it was a private message.
	// Answer the users in this case.
	if tgt == conn.Me().Nick {
		tgt = line.Nick
	}
	if prefix == "" {
		prefix = "Link Info"
	} else {
		prefix = clean(prefix)
	}
	title = clean(title)
	// the IRC spec states that notice should be used instead of msg
	// and that bots should not react to notice at all. However, no
	// real world bot adheres to this. Furthermore, people who can’t
	// configure their client to not highlight them on notices will
	// complain.
	conn.Privmsg(tgt, "["+prefix+"] "+title)
}

func clean(text string) string {
	text = whitespaceRegex.ReplaceAllString(text, " ")
	return strings.TrimSpace(text)
}
<commit_msg>fixed “Verified User”-bug<commit_after>package frank

import (
	"code.google.com/p/go.net/html"
	"code.google.com/p/go.net/html/atom"
	"errors"
	irc "github.com/fluffle/goirc/client"
	"io"
	"log"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// how many URLs can the cache store
const cacheSize = 100

// how many hours an entry should be considered valid
const cacheValidHours = 24

// how many kilo bytes should be considered when looking for the title
// tag.
const httpReadKByte = 100

// abort HTTP requests if it takes longer than X seconds. Not sure, it’s
// definitely magic involved. Must be larger than 5.
const httpGetDeadline = 10

// don’t repost the same title within this period
const noRepostWithinSeconds = 30

// matches all whitespace and zero bytes. Additionally, all Unicode
// characters of class Cf (format chars, e.g. right-to-left) and Cc
// (control chars) are matched.
var whitespaceRegex = regexp.MustCompile(`[\s\0\p{Cf}\p{Cc}]+`)

var ignoreDomainsRegex = regexp.MustCompile(`^http://p\.nnev\.de`)

var twitterDomainRegex = regexp.MustCompile(`(?i)^https?://(?:[a-z0-9]\.)?twitter.com`)
var twitterPicsRegex = regexp.MustCompile(`(?i)(?:\b|^)pic\.twitter\.com/[a-z0-9]+(?:\b|$)`)

var noSpoilerRegex = regexp.MustCompile(`(?i)(don't|no|kein|nicht) spoiler`)

func UriFind(conn *irc.Conn, line *irc.Line) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("MEGA-WTF:pkg: %v", r)
		}
	}()

	msg := line.Args[1]

	if noSpoilerRegex.MatchString(msg) {
		log.Printf("not spoilering this line: %s", msg)
		return
	}

	urls := extract(msg)

	for _, url := range urls {
		if url == "" {
			continue
		}

		if title := cacheGetTitleByUrl(url); title != "" {
			log.Printf("using cache for URL: %s", url)
			postTitle(conn, line, title, "Cache Info")
			continue
		}

		go func(url string) {
			if ignoreDomainsRegex.MatchString(url) {
				log.Printf("ignoring this URL: %s", url)
				return
			}

			log.Printf("testing URL: %s", url)
			title, _, err := TitleGet(url)
			if err != nil {
				//postTitle(conn, line, err.Error(), "Error")
			} else if title != "" {
				postTitle(conn, line, title, "")
				cacheAdd(url, title)
			}
		}(url)
	}
}

// regexing ////////////////////////////////////////////////////////////

func extract(msg string) []string {
	results := make([]string, 0)
	for idx := strings.Index(msg, "http"); idx > -1; idx = strings.Index(msg, "http") {
		url := msg[idx:]
		if !strings.HasPrefix(url, "http://") &&
			!strings.HasPrefix(url, "https://") {
			msg = msg[idx+len("http"):]
			continue
		}

		// End on commas, but only if they are followed by a space.
		// spiegel.de URLs have commas in them, that would be a
		// false positive otherwise.
		if end := strings.Index(url, ", "); end > -1 {
			url = url[:end]
		}

		// End on closing paren, but only if there is an opening
		// paren before the URL (should fix most false-positives).
		if end := strings.Index(url, ")"); idx > 0 && msg[idx-1] == '(' && end > -1 {
			url = url[:end]
		}

		// Whitespace always ends a URL.
		if end := strings.IndexAny(url, " \t"); end > -1 {
			url = url[:end]
		}

		results = append(results, url)
		msg = msg[idx+len(url):]
	}
	return results
}

// http/html stuff /////////////////////////////////////////////////////

func TitleGet(url string) (string, string, error) {
	// via http://www.reddit.com/r/golang/comments/10awvj/timeout_on_httpget/c6bz49s
	c := http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(time.Second * httpGetDeadline)
				c, err := net.DialTimeout(netw, addr, time.Second*(httpGetDeadline-5))
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
		},
	}

	r, err := c.Get(url)
	if err != nil {
		log.Printf("WTF: could not resolve %s: %s\n", url, err)
		return "", url, err
	}
	defer r.Body.Close()

	lastUrl := r.Request.URL.String()

	// TODO: r.Body → utf8?
	title, tweet := titleParseHtml(io.LimitReader(r.Body, 1024*httpReadKByte))

	if r.StatusCode != 200 {
		return "", lastUrl, errors.New("[" + strconv.Itoa(r.StatusCode) + "] " + title)
	}

	if tweet != "" && twitterDomainRegex.MatchString(lastUrl) {
		title = tweet
	}

	log.Printf("Title for URL %s: %s\n", url, title)

	return title, lastUrl, nil
}

// parses the incoming HTML fragment and tries to extract text from
// suitable tags. Currently this is the page’s title tag and tweets
// when the HTML-code is similar enough to twitter.com. Returns
// title and tweet.
func titleParseHtml(r io.Reader) (string, string) {
	doc, err := html.Parse(r)
	if err != nil {
		log.Printf("WTF: html parser blew up: %s\n", err)
		return "", ""
	}

	title := ""
	tweetText := ""
	tweetUserName := ""
	tweetUserScreenName := ""
	tweetPicUrl := ""

	var f func(*html.Node)
	f = func(n *html.Node) {
		if title == "" && n.Type == html.ElementNode && n.DataAtom == atom.Title {
			title = extractText(n)
			return
		}

		if hasClass(n, "permalink-tweet") {
			tweetUserName = getAttr(n, "data-name")
			tweetUserScreenName = getAttr(n, "data-screen-name")
			// find next child “tweet-text”
			for c := n.FirstChild; c!=nil; c = c.NextSibling {
				if hasClass(c, "tweet-text") {
					tweetText = extractText(c)
					break
				}
			}
		}

		isMedia := hasClass(n, "media") || hasClass(n, "media-thumbnail")
		if tweetPicUrl == "" && isMedia && !hasClass(n, "profile-picture") {
			attrVal := getAttr(n, "data-url")
			if attrVal != "" {
				tweetPicUrl = attrVal
				return
			}
		}

		// recurse down
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}

	}
	f(doc)

	// cleanup
	tweet := ""
	tweetUser := ""
	if tweetText != "" {
		tweetText = twitterPicsRegex.ReplaceAllString(tweetText, "")
		tweetUser = tweetUserName + " (@" + tweetUserScreenName + "): "
		tweet = tweetUser + tweetText + " " + tweetPicUrl
		tweet = clean(tweet)
	}

	return strings.TrimSpace(title), strings.TrimSpace(tweet)
}

func extractText(n *html.Node) string {
	text := ""
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if c.Type == html.TextNode {
			text += c.Data
		} else {
			text += extractText(c)
		}
	}
	return clean(text)
}

func hasClass(n *html.Node, class string) bool {
	if n.Type != html.ElementNode {
		return false
	}

	class = " " + strings.TrimSpace(class) + " "
	attr := strings.Replace(getAttr(n, "class"), "\n", " ", -1)
	if strings.Contains(" "+attr+" ", class) {
		return true
	}
	return false
}

func getAttr(n *html.Node, findAttr string) string {
	for _, attr := range n.Attr {
		if attr.Key == findAttr {
			return attr.Val
		}
	}
	return ""
}

// Cache ///////////////////////////////////////////////////////////////
type Cache struct {
	url   string
	title string
	date  time.Time
}

var cache = [cacheSize]Cache{}
var cacheIndex = 0

func cacheAdd(url string, title string) {
	if len(cache) == cacheIndex {
		cacheIndex = 0
	}
	cache[cacheIndex] = Cache{url, title, time.Now()}
	cacheIndex += 1
}

func cacheGetTitleByUrl(url string) string {
	for _, cc := range cache {
		if cc.url == url && time.Since(cc.date).Hours() <= cacheValidHours {
			return cc.title
		}
	}
	return ""
}

func cacheGetSecondsToLastPost(title string) int {
	var secondsAgo = int(^uint(0) >> 1)
	for _, cc := range cache {
		var a = int(time.Since(cc.date).Seconds())
		if cc.title == title && a < secondsAgo {
			secondsAgo = a
		}
	}
	return secondsAgo
}

// util ////////////////////////////////////////////////////////////////

func postTitle(conn *irc.Conn, line *irc.Line, title string, prefix string) {
	tgt := line.Args[0]

	secondsAgo := cacheGetSecondsToLastPost(title)
	if secondsAgo <= noRepostWithinSeconds {
		log.Printf("Skipping, because posted %d seconds ago (“%s”)", secondsAgo, title)
		return
	}

	log.Printf("nick=%s, target=%s, title=%s", line.Nick, tgt, title)
	// if target is our current nick, it was a private message.
	// Answer the users in this case.
	if tgt == conn.Me().Nick {
		tgt = line.Nick
	}
	if prefix == "" {
		prefix = "Link Info"
	} else {
		prefix = clean(prefix)
	}
	title = clean(title)
	// the IRC spec states that notice should be used instead of msg
	// and that bots should not react to notice at all. However, no
	// real world bot adheres to this. Furthermore, people who can’t
	// configure their client to not highlight them on notices will
	// complain.
	conn.Privmsg(tgt, "["+prefix+"] "+title)
}

func clean(text string) string {
	text = whitespaceRegex.ReplaceAllString(text, " ")
	return strings.TrimSpace(text)
}
<|endoftext|>