<commit_before>package spectrum

type ula_byte_t struct {
	valid bool
	value uint8
}

type ula_attr_t struct {
	valid  bool
	value  uint8
	tstate uint
}

type ULA struct {
	// Frame number
	frame uint

	borderColor byte

	// Whether to discern between [data read by ULA] and [data in memory at the end of a frame].
	// If the value is 'false', then fields 'bitmap' and 'attr' will contain no information.
	// The default value is 'true'.
	accurateEmulation bool

	// Screen bitmap data read by ULA, if they differ from data in memory at the end of a frame.
	// Spectrum y-coordinate.
	bitmap [BytesPerLine * ScreenHeight]ula_byte_t

	// Screen attributes read by ULA, if they differ from data in memory at the end of a frame.
	// Linear y-coordinate.
	attr [BytesPerLine * ScreenHeight]ula_attr_t

	// Whether the 8x8 rectangular screen area was modified during the current frame
	dirtyScreen [ScreenWidth_Attr * ScreenHeight_Attr]bool

	speccy *Spectrum48k
}

func NewULA() *ULA {
	return &ULA{accurateEmulation: true}
}

func (ula *ULA) init(speccy *Spectrum48k) {
	ula.speccy = speccy
}

func (ula *ULA) reset() {
	ula.frame = 0
}

func (ula *ULA) getBorderColor() byte {
	return ula.borderColor
}

func (ula *ULA) setBorderColor(borderColor byte) {
	ula.borderColor = borderColor
}

func (ula *ULA) setEmulationAccuracy(accurateEmulation bool) {
	ula.accurateEmulation = accurateEmulation
}

// This function is called at the beginning of each frame
func (ula *ULA) frame_begin() {
	ula.frame++
	if ula.frame == 1 {
		// The very first frame --> repaint the whole screen
		for i := 0; i < ScreenWidth_Attr*ScreenHeight_Attr; i++ {
			ula.dirtyScreen[i] = true
		}
	} else {
		for i := 0; i < ScreenWidth_Attr*ScreenHeight_Attr; i++ {
			ula.dirtyScreen[i] = false
		}
	}

	bitmap := &ula.bitmap
	for ofs := uint16(0); ofs < BytesPerLine*ScreenHeight; ofs++ {
		if bitmap[ofs].valid {
			ula.screenBitmapTouch(SCREEN_BASE_ADDR + ofs)
		}

		bitmap[ofs].valid = false
	}

	attr := &ula.attr
	for ofs := uint16(0); ofs < BytesPerLine*ScreenHeight; ofs++ {
		if attr[ofs].valid {
			linearY := (ofs >> BytesPerLine_log2)
			attr_y := (linearY >> 3)
			attr_x := (ofs & 0x001f)
			ula.screenAttrTouch(ATTR_BASE_ADDR + (attr_y << BytesPerLine_log2) + attr_x)
		}

		attr[ofs].valid = false
	}
}

func (ula *ULA) screenBitmapTouch(address uint16) {
	var attr_x, attr_y uint8 = screenAddr_to_attrXY(address)
	ula.dirtyScreen[uint(attr_y)*ScreenWidth_Attr+uint(attr_x)] = true
}

func (ula *ULA) screenAttrTouch(address uint16) {
	ula.dirtyScreen[address-ATTR_BASE_ADDR] = true
}

// Handle a write to an address in range (SCREEN_BASE_ADDR ... SCREEN_BASE_ADDR+0x1800-1)
func (ula *ULA) screenBitmapWrite(address uint16, oldValue byte, newValue byte) {
	if oldValue != newValue {
		ula.screenBitmapTouch(address)

		if ula.accurateEmulation {
			rel_addr := address - SCREEN_BASE_ADDR
			ula_lineStart_tstate := screenline_start_tstates[rel_addr>>BytesPerLine_log2]
			x, _ := screenAddr_to_xy(address)
			ula_tstate := ula_lineStart_tstate + uint(x>>PIXELS_PER_TSTATE_LOG2)
			if ula_tstate <= ula.speccy.Cpu.tstates {
				// Remember the value read by ULA
				ula.bitmap[rel_addr] = ula_byte_t{true, oldValue}
			}
		}
	}
}

// Handle a write to an address in range (ATTR_BASE_ADDR ... ATTR_BASE_ADDR+0x300-1)
func (ula *ULA) screenAttrWrite(address uint16, oldValue byte, newValue byte) {
	if oldValue != newValue {
		ula.screenAttrTouch(address)

		if ula.accurateEmulation {
			speccy := ula.speccy

			attr_x := uint(address & 0x001f)
			attr_y := uint((address - ATTR_BASE_ADDR) >> ScreenWidth_Attr_log2)

			x := 8 * attr_x
			y := 8 * attr_y

			ofs := (y << BytesPerLine_log2) + attr_x
			ula_tstate := FIRST_SCREEN_BYTE + y*TSTATES_PER_LINE + (x >> PIXELS_PER_TSTATE_LOG2)

			for i := 0; i < 8; i++ {
				if ula_tstate <= speccy.Cpu.tstates {
					ula_attr := &ula.attr[ofs]
					if !ula_attr.valid || (ula_tstate > ula_attr.tstate) {
						*ula_attr = ula_attr_t{true, oldValue, speccy.Cpu.tstates}
					}
					ofs += BytesPerLine
					ula_tstate += TSTATES_PER_LINE
				} else {
					break
				}
			}
		}
	}
}

func (ula *ULA) prepare(display *DisplayInfo) *DisplayData {
	sendDiffOnly := false
	if display.lastFrame != nil {
		sendDiffOnly = true
	}

	var screen DisplayData
	{
		flash := (ula.frame & 0x10) != 0
		flash_previous := ((ula.frame - 1) & 0x10) != 0
		flash_diff := (flash != flash_previous)

		// screen.dirty
		if sendDiffOnly {
			screen.dirty = ula.dirtyScreen
		} else {
			for i := 0; i < ScreenWidth_Attr*ScreenHeight_Attr; i++ {
				screen.dirty[i] = true
			}
		}

		// Fill screen.bitmap & screen.attr, but only the dirty regions.
		var memory_data *[0x10000]byte = ula.speccy.Memory.Data()
		ula_bitmap := &ula.bitmap
		ula_attr := &ula.attr
		screen_dirty := &screen.dirty
		screen_bitmap := &screen.bitmap
		screen_attr := &screen.attr
		for attr_y := uint(0); attr_y < ScreenHeight_Attr; attr_y++ {
			attr_y8 := 8 * attr_y

			for attr_x := uint(0); attr_x < ScreenWidth_Attr; attr_x++ {
				attr_ofs := attr_y*ScreenWidth_Attr + attr_x

				// Make sure to send all changed flashing pixels to the DisplayReceiver
				if flash_diff {
					linearY_ofs := (attr_y8 << BytesPerLine_log2) + attr_x

					for y := 0; y < 8; y++ {
						var attr byte
						if !ula_attr[linearY_ofs].valid {
							attr = memory_data[ATTR_BASE_ADDR+attr_ofs]
						} else {
							attr = ula_attr[linearY_ofs].value
						}

						if (attr & 0x80) != 0 {
							screen_dirty[attr_ofs] = true
							break
						}

						linearY_ofs += BytesPerLine
					}
				}

				if !screen_dirty[attr_ofs] {
					continue
				}

				// screen.bitmap
				{
					screen_addr := xy_to_screenAddr(uint8(8*attr_x), uint8(attr_y8))
					linearY_ofs := (attr_y8 << BytesPerLine_log2) + attr_x

					for y := 0; y < 8; y++ {
						if !ula_bitmap[screen_addr-SCREEN_BASE_ADDR].valid {
							screen_bitmap[linearY_ofs] = memory_data[screen_addr]
						} else {
							screen_bitmap[linearY_ofs] = ula_bitmap[screen_addr-SCREEN_BASE_ADDR].value
						}

						screen_addr += 8 * BytesPerLine
						linearY_ofs += BytesPerLine
					}
				}

				// screen.attr
				{
					linearY_ofs := (attr_y8 << BytesPerLine_log2) + attr_x

					for y := 0; y < 8; y++ {
						var attr byte
						if !ula_attr[linearY_ofs].valid {
							attr = memory_data[ATTR_BASE_ADDR+attr_ofs]
						} else {
							attr = ula_attr[linearY_ofs].value
						}

						ink := ((attr & 0x40) >> 3) | (attr & 0x07)
						paper := (attr & 0x78) >> 3

						if flash && ((attr & 0x80) != 0) {
							/* invert flashing attributes */
							ink, paper = paper, ink
						}

						screen_attr[linearY_ofs] = attr_4bit((ink << 4) | paper)

						linearY_ofs += BytesPerLine
					}
				}
			}
		}

		// screen.borderEvents
		screen.borderEvents_orNil = ula.speccy.Ports.getBorderEvents_orNil()
	}

	return &screen
}

func (ula *ULA) sendScreenToDisplay(display *DisplayInfo, completionTime_orNil chan<- int64) {
	displayData := ula.prepare(display)
	displayData.completionTime_orNil = completionTime_orNil

	if display.missedChanges != nil {
		display.missedChanges.add(displayData)
		displayData = display.missedChanges
		display.missedChanges = nil
	}

	displayData.completionTime_orNil = completionTime_orNil
	displayChannel := display.displayReceiver.getDisplayDataChannel()

	select {
	case displayChannel <- displayData:
		if display.lastFrame == nil {
			display.lastFrame = new(uint)
		}
		*(display.lastFrame) = ula.frame
	default:
		// Nothing was sent over the 'displayChannel', because the send would block.
		// Avoiding the blocking allows the CPU emulation to proceed when the next tick arrives,
		// instead of waiting for the display backend to receive the previous frame.
		// The 'display.lastFrame' is NOT updated.
		display.numMissedFrames++
		display.missedChanges = displayData
	}
}

// Adds the change-set 'b' to the change-set 'a'.
// This modifies 'a' only, 'b' is left unchanged.
// This is not a commutative operation, the order is significant.
func (a *DisplayData) add(b *DisplayData) {
	a_dirty := &a.dirty
	a_bitmap := &a.bitmap
	a_attr := &a.attr

	b_dirty := &b.dirty
	b_bitmap := &b.bitmap
	b_attr := &b.attr

	for attr_y := uint(0); attr_y < ScreenHeight_Attr; attr_y++ {
		attr_y8 := 8 * attr_y

		for attr_x := uint(0); attr_x < ScreenWidth_Attr; attr_x++ {
			attr_ofs := attr_y*ScreenWidth_Attr + attr_x

			if b_dirty[attr_ofs] {
				a_dirty[attr_ofs] = true

				ofs := (attr_y8 << BytesPerLine_log2) + attr_x
				for y := 0; y < 8; y++ {
					a_bitmap[ofs] = b_bitmap[ofs]
					a_attr[ofs] = b_attr[ofs]
					ofs += BytesPerLine
				}
			}
		}
	}

	a.borderEvents_orNil = b.borderEvents_orNil
}
<commit_msg>Fix the previous merge<commit_after>package spectrum

type ula_byte_t struct {
	valid bool
	value uint8
}

type ula_attr_t struct {
	valid  bool
	value  uint8
	tstate uint
}

type ULA struct {
	// Frame number
	frame uint

	borderColor byte

	// Whether to discern between [data read by ULA] and [data in memory at the end of a frame].
	// If the value is 'false', then fields 'bitmap' and 'attr' will contain no information.
	// The default value is 'true'.
	accurateEmulation bool

	// Screen bitmap data read by ULA, if they differ from data in memory at the end of a frame.
	// Spectrum y-coordinate.
	bitmap [BytesPerLine * ScreenHeight]ula_byte_t

	// Screen attributes read by ULA, if they differ from data in memory at the end of a frame.
	// Linear y-coordinate.
	attr [BytesPerLine * ScreenHeight]ula_attr_t

	// Whether the 8x8 rectangular screen area was modified during the current frame
	dirtyScreen [ScreenWidth_Attr * ScreenHeight_Attr]bool

	speccy *Spectrum48k
}

func NewULA() *ULA {
	return &ULA{accurateEmulation: true}
}

func (ula *ULA) init(speccy *Spectrum48k) {
	ula.speccy = speccy
}

func (ula *ULA) reset() {
	ula.frame = 0
}

func (ula *ULA) getBorderColor() byte {
	return ula.borderColor
}

func (ula *ULA) setBorderColor(borderColor byte) {
	ula.borderColor = borderColor
}

func (ula *ULA) setEmulationAccuracy(accurateEmulation bool) {
	ula.accurateEmulation = accurateEmulation
}

// This function is called at the beginning of each frame
func (ula *ULA) frame_begin() {
	ula.frame++
	if ula.frame == 1 {
		// The very first frame --> repaint the whole screen
		for i := 0; i < ScreenWidth_Attr*ScreenHeight_Attr; i++ {
			ula.dirtyScreen[i] = true
		}
	} else {
		for i := 0; i < ScreenWidth_Attr*ScreenHeight_Attr; i++ {
			ula.dirtyScreen[i] = false
		}
	}

	bitmap := &ula.bitmap
	for ofs := uint16(0); ofs < BytesPerLine*ScreenHeight; ofs++ {
		if bitmap[ofs].valid {
			ula.screenBitmapTouch(SCREEN_BASE_ADDR + ofs)
		}

		bitmap[ofs].valid = false
	}

	attr := &ula.attr
	for ofs := uint16(0); ofs < BytesPerLine*ScreenHeight; ofs++ {
		if attr[ofs].valid {
			linearY := (ofs >> BytesPerLine_log2)
			attr_y := (linearY >> 3)
			attr_x := (ofs & 0x001f)
			ula.screenAttrTouch(ATTR_BASE_ADDR + (attr_y << BytesPerLine_log2) + attr_x)
		}

		attr[ofs].valid = false
	}
}

func (ula *ULA) screenBitmapTouch(address uint16) {
	var attr_x, attr_y uint8 = screenAddr_to_attrXY(address)
	ula.dirtyScreen[uint(attr_y)*ScreenWidth_Attr+uint(attr_x)] = true
}

func (ula *ULA) screenAttrTouch(address uint16) {
	ula.dirtyScreen[address-ATTR_BASE_ADDR] = true
}

// Handle a write to an address in range (SCREEN_BASE_ADDR ... SCREEN_BASE_ADDR+0x1800-1)
func (ula *ULA) screenBitmapWrite(address uint16, oldValue byte, newValue byte) {
	if oldValue != newValue {
		ula.screenBitmapTouch(address)

		if ula.accurateEmulation {
			rel_addr := address - SCREEN_BASE_ADDR
			ula_lineStart_tstate := screenline_start_tstates[rel_addr>>BytesPerLine_log2]
			x, _ := screenAddr_to_xy(address)
			ula_tstate := ula_lineStart_tstate + uint(x>>PIXELS_PER_TSTATE_LOG2)
			if ula_tstate <= ula.speccy.Cpu.tstates {
				// Remember the value read by ULA
				ula.bitmap[rel_addr] = ula_byte_t{true, oldValue}
			}
		}
	}
}

// Handle a write to an address in range (ATTR_BASE_ADDR ... ATTR_BASE_ADDR+0x300-1)
func (ula *ULA) screenAttrWrite(address uint16, oldValue byte, newValue byte) {
	if oldValue != newValue {
		ula.screenAttrTouch(address)

		if ula.accurateEmulation {
			speccy := ula.speccy

			attr_x := uint(address & 0x001f)
			attr_y := uint((address - ATTR_BASE_ADDR) >> ScreenWidth_Attr_log2)

			x := 8 * attr_x
			y := 8 * attr_y

			ofs := (y << BytesPerLine_log2) + attr_x
			ula_tstate := FIRST_SCREEN_BYTE + y*TSTATES_PER_LINE + (x >> PIXELS_PER_TSTATE_LOG2)

			for i := 0; i < 8; i++ {
				if ula_tstate <= speccy.Cpu.tstates {
					ula_attr := &ula.attr[ofs]
					if !ula_attr.valid || (ula_tstate > ula_attr.tstate) {
						*ula_attr = ula_attr_t{true, oldValue, speccy.Cpu.tstates}
					}
					ofs += BytesPerLine
					ula_tstate += TSTATES_PER_LINE
				} else {
					break
				}
			}
		}
	}
}

func (ula *ULA) prepare(display *DisplayInfo) *DisplayData {
	sendDiffOnly := false
	if display.lastFrame != nil {
		sendDiffOnly = true
	}

	var screen DisplayData
	{
		flash := (ula.frame & 0x10) != 0
		flash_previous := ((ula.frame - 1) & 0x10) != 0
		flash_diff := (flash != flash_previous)

		// screen.dirty
		if sendDiffOnly {
			screen.dirty = ula.dirtyScreen
		} else {
			for i := 0; i < ScreenWidth_Attr*ScreenHeight_Attr; i++ {
				screen.dirty[i] = true
			}
		}

		// Fill screen.bitmap & screen.attr, but only the dirty regions.
		var memory_data *[0x10000]byte = ula.speccy.Memory.Data()
		ula_bitmap := &ula.bitmap
		ula_attr := &ula.attr
		screen_dirty := &screen.dirty
		screen_bitmap := &screen.bitmap
		screen_attr := &screen.attr
		for attr_y := uint(0); attr_y < ScreenHeight_Attr; attr_y++ {
			attr_y8 := 8 * attr_y

			for attr_x := uint(0); attr_x < ScreenWidth_Attr; attr_x++ {
				attr_ofs := attr_y*ScreenWidth_Attr + attr_x

				// Make sure to send all changed flashing pixels to the DisplayReceiver
				if flash_diff {
					linearY_ofs := (attr_y8 << BytesPerLine_log2) + attr_x

					for y := 0; y < 8; y++ {
						var attr byte
						if !ula_attr[linearY_ofs].valid {
							attr = memory_data[ATTR_BASE_ADDR+attr_ofs]
						} else {
							attr = ula_attr[linearY_ofs].value
						}

						if (attr & 0x80) != 0 {
							screen_dirty[attr_ofs] = true
							break
						}

						linearY_ofs += BytesPerLine
					}
				}

				if !screen_dirty[attr_ofs] {
					continue
				}

				// screen.bitmap
				{
					screen_addr := xy_to_screenAddr(uint8(8*attr_x), uint8(attr_y8))
					linearY_ofs := (attr_y8 << BytesPerLine_log2) + attr_x

					for y := 0; y < 8; y++ {
						if !ula_bitmap[screen_addr-SCREEN_BASE_ADDR].valid {
							screen_bitmap[linearY_ofs] = memory_data[screen_addr]
						} else {
							screen_bitmap[linearY_ofs] = ula_bitmap[screen_addr-SCREEN_BASE_ADDR].value
						}

						screen_addr += 8 * BytesPerLine
						linearY_ofs += BytesPerLine
					}
				}

				// screen.attr
				{
					linearY_ofs := (attr_y8 << BytesPerLine_log2) + attr_x

					for y := 0; y < 8; y++ {
						var attr byte
						if !ula_attr[linearY_ofs].valid {
							attr = memory_data[ATTR_BASE_ADDR+attr_ofs]
						} else {
							attr = ula_attr[linearY_ofs].value
						}

						ink := ((attr & 0x40) >> 3) | (attr & 0x07)
						paper := (attr & 0x78) >> 3

						if flash && ((attr & 0x80) != 0) {
							/* invert flashing attributes */
							ink, paper = paper, ink
						}

						screen_attr[linearY_ofs] = attr_4bit((ink << 4) | paper)

						linearY_ofs += BytesPerLine
					}
				}
			}
		}

		// screen.borderEvents
		screen.borderEvents_orNil = ula.speccy.Ports.getBorderEvents_orNil()
	}

	return &screen
}

func (ula *ULA) sendScreenToDisplay(display *DisplayInfo, completionTime_orNil chan<- int64) {
	displayData := ula.prepare(display)
	displayData.completionTime_orNil = completionTime_orNil

	if display.missedChanges != nil {
		display.missedChanges.add(displayData)
		displayData = display.missedChanges
		display.missedChanges = nil
	}

	displayData.completionTime_orNil = completionTime_orNil
	displayChannel := display.displayReceiver.getDisplayDataChannel()

	var nonBlockingSend bool
	select {
	case displayChannel <- displayData:
		nonBlockingSend = true
	default:
		nonBlockingSend = false
	}

	if nonBlockingSend {
		if display.lastFrame == nil {
			display.lastFrame = new(uint)
		}
		*(display.lastFrame) = ula.frame
	} else {
		// Nothing was sent over the 'displayChannel', because the send would block.
		// Avoiding the blocking allows the CPU emulation to proceed when the next tick arrives,
		// instead of waiting for the display backend to receive the previous frame.
		// The 'display.lastFrame' is NOT updated.
		display.numMissedFrames++
		display.missedChanges = displayData
	}
}

// Adds the change-set 'b' to the change-set 'a'.
// This modifies 'a' only, 'b' is left unchanged.
// This is not a commutative operation, the order is significant.
func (a *DisplayData) add(b *DisplayData) {
	a_dirty := &a.dirty
	a_bitmap := &a.bitmap
	a_attr := &a.attr

	b_dirty := &b.dirty
	b_bitmap := &b.bitmap
	b_attr := &b.attr

	for attr_y := uint(0); attr_y < ScreenHeight_Attr; attr_y++ {
		attr_y8 := 8 * attr_y

		for attr_x := uint(0); attr_x < ScreenWidth_Attr; attr_x++ {
			attr_ofs := attr_y*ScreenWidth_Attr + attr_x

			if b_dirty[attr_ofs] {
				a_dirty[attr_ofs] = true

				ofs := (attr_y8 << BytesPerLine_log2) + attr_x
				for y := 0; y < 8; y++ {
					a_bitmap[ofs] = b_bitmap[ofs]
					a_attr[ofs] = b_attr[ofs]
					ofs += BytesPerLine
				}
			}
		}
	}

	a.borderEvents_orNil = b.borderEvents_orNil
}
<|endoftext|>


<commit_before>package struc

import (
	"encoding/binary"
	"errors"
	"fmt"
	"reflect"
	"regexp"
	"strconv"
	"strings"
	"sync"
)

// struc:"int32,big,sizeof=Data"

var tagWordsRe = regexp.MustCompile(`(\[|\b)[^"]+\b+$`)

type strucTag struct {
	Type   string
	Order  binary.ByteOrder
	Sizeof string
}

func parseStrucTag(tag reflect.StructTag) *strucTag {
	t := &strucTag{
		Order: binary.BigEndian,
	}
	tagStr := tag.Get("struc")
	if tagStr == "" {
		// someone's going to typo this (I already did once)
		// sorry if you made a module actually using this tag
		// and you're mad at me now
		tagStr = tag.Get("struct")
	}
	for _, s := range strings.Split(tagStr, ",") {
		if strings.HasPrefix(s, "sizeof=") {
			tmp := strings.SplitN(s, "=", 2)
			t.Sizeof = tmp[1]
		} else if s == "big" {
			t.Order = binary.BigEndian
		} else if s == "little" {
			t.Order = binary.LittleEndian
		} else {
			t.Type = s
		}
	}
	return t
}

var typeLenRe = regexp.MustCompile(`^\[(\d*)\]`)

func parseField(f reflect.StructField) (fd *Field, err error) {
	tag := parseStrucTag(f.Tag)
	var ok bool
	fd = &Field{
		Name:  f.Name,
		Len:   1,
		Order: tag.Order,
		Slice: false,
		kind:  f.Type.Kind(),
	}
	switch fd.kind {
	case reflect.Array:
		fd.Slice = true
		fd.Array = true
		fd.Len = f.Type.Len()
		fd.kind = f.Type.Elem().Kind()
	case reflect.Slice:
		fd.Slice = true
		fd.Len = -1
		fd.kind = f.Type.Elem().Kind()
	case reflect.Ptr:
		fd.Ptr = true
		fd.kind = f.Type.Elem().Kind()
	}
	var defTypeOk bool
	fd.defType, defTypeOk = reflectTypeMap[fd.kind]
	// find a type in the struct tag
	pureType := typeLenRe.ReplaceAllLiteralString(tag.Type, "")
	if fd.Type, ok = typeLookup[pureType]; ok {
		fd.Len = 1
		match := typeLenRe.FindAllStringSubmatch(tag.Type, -1)
		if len(match) > 0 && len(match[0]) > 1 {
			fd.Slice = true
			first := match[0][1]
			// Field.Len = -1 indicates a []slice
			if first == "" {
				fd.Len = -1
			} else {
				fd.Len, err = strconv.Atoi(first)
			}
		}
		return
	}
	// the user didn't specify a type
	if defTypeOk {
		fd.Type = fd.defType
		return
	}
	err = errors.New("struc: Could not find field type.")
	return
}

var fieldCache = make(map[reflect.Type]Fields)
var parseLock sync.Mutex

func parseFieldsLocked(v reflect.Value) (Fields, error) {
	// we need to repeat this logic because parseFields() below can't be recursively called due to locking
	for v.Kind() == reflect.Ptr {
		v = v.Elem()
	}
	t := v.Type()
	if v.NumField() < 1 {
		return nil, errors.New("struc: Struct has no fields.")
	}
	sizeofMap := make(map[string][]int)
	fields := make(Fields, 0, v.NumField())
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		f, err := parseField(field)
		if err != nil {
			return nil, err
		}
		f.CanSet = v.Field(i).CanSet()
		if !f.CanSet {
			continue
		}
		f.Index = i
		tag := parseStrucTag(field.Tag)
		if tag.Sizeof != "" {
			target, ok := t.FieldByName(tag.Sizeof)
			if !ok {
				return nil, fmt.Errorf("struc: `sizeof=%s` field does not exist", tag.Sizeof)
			}
			f.Sizeof = target.Index
			sizeofMap[tag.Sizeof] = field.Index
		}
		if sizefrom, ok := sizeofMap[field.Name]; ok {
			f.Sizefrom = sizefrom
		}
		if f.Len == -1 && f.Sizefrom == nil {
			return nil, fmt.Errorf("struc: field `%s` is a slice with no length or sizeof field", field.Name)
		}
		// recurse into nested structs
		// TODO: handle loops (probably by indirecting the []Field and putting pointer in cache)
		if f.Type == Struct {
			typ := field.Type
			if f.Ptr {
				typ = typ.Elem()
			}
			if f.Slice {
				typ = typ.Elem()
			}
			f.Fields, err = parseFieldsLocked(reflect.New(typ))
			if err != nil {
				return nil, err
			}
		}
		fields = append(fields, f)
	}
	return fields, nil
}

func parseFields(v reflect.Value) (Fields, error) {
	for v.Kind() == reflect.Ptr {
		v = v.Elem()
	}
	t := v.Type()
	if cached, ok := fieldCache[t]; ok {
		return cached, nil
	}
	// take a lock so multiple goroutines can't parse fields at once
	parseLock.Lock()
	defer parseLock.Unlock()
	// check a second time, just in case it was parsed by another goroutine during lock
	if cached, ok := fieldCache[t]; ok {
		return cached, nil
	}
	fields, err := parseFieldsLocked(v)
	if err != nil {
		return nil, err
	}
	fieldCache[t] = fields
	return fields, nil
}
<commit_msg>improve parseFields thread safety<commit_after>package struc

import (
	"encoding/binary"
	"errors"
	"fmt"
	"reflect"
	"regexp"
	"strconv"
	"strings"
	"sync"
)

// struc:"int32,big,sizeof=Data"

var tagWordsRe = regexp.MustCompile(`(\[|\b)[^"]+\b+$`)

type strucTag struct {
	Type   string
	Order  binary.ByteOrder
	Sizeof string
}

func parseStrucTag(tag reflect.StructTag) *strucTag {
	t := &strucTag{
		Order: binary.BigEndian,
	}
	tagStr := tag.Get("struc")
	if tagStr == "" {
		// someone's going to typo this (I already did once)
		// sorry if you made a module actually using this tag
		// and you're mad at me now
		tagStr = tag.Get("struct")
	}
	for _, s := range strings.Split(tagStr, ",") {
		if strings.HasPrefix(s, "sizeof=") {
			tmp := strings.SplitN(s, "=", 2)
			t.Sizeof = tmp[1]
		} else if s == "big" {
			t.Order = binary.BigEndian
		} else if s == "little" {
			t.Order = binary.LittleEndian
		} else {
			t.Type = s
		}
	}
	return t
}

var typeLenRe = regexp.MustCompile(`^\[(\d*)\]`)

func parseField(f reflect.StructField) (fd *Field, err error) {
	tag := parseStrucTag(f.Tag)
	var ok bool
	fd = &Field{
		Name:  f.Name,
		Len:   1,
		Order: tag.Order,
		Slice: false,
		kind:  f.Type.Kind(),
	}
	switch fd.kind {
	case reflect.Array:
		fd.Slice = true
		fd.Array = true
		fd.Len = f.Type.Len()
		fd.kind = f.Type.Elem().Kind()
	case reflect.Slice:
		fd.Slice = true
		fd.Len = -1
		fd.kind = f.Type.Elem().Kind()
	case reflect.Ptr:
		fd.Ptr = true
		fd.kind = f.Type.Elem().Kind()
	}
	var defTypeOk bool
	fd.defType, defTypeOk = reflectTypeMap[fd.kind]
	// find a type in the struct tag
	pureType := typeLenRe.ReplaceAllLiteralString(tag.Type, "")
	if fd.Type, ok = typeLookup[pureType]; ok {
		fd.Len = 1
		match := typeLenRe.FindAllStringSubmatch(tag.Type, -1)
		if len(match) > 0 && len(match[0]) > 1 {
			fd.Slice = true
			first := match[0][1]
			// Field.Len = -1 indicates a []slice
			if first == "" {
				fd.Len = -1
			} else {
				fd.Len, err = strconv.Atoi(first)
			}
		}
		return
	}
	// the user didn't specify a type
	if defTypeOk {
		fd.Type = fd.defType
		return
	}
	err = errors.New("struc: Could not find field type.")
	return
}

func parseFieldsLocked(v reflect.Value) (Fields, error) {
	// we need to repeat this logic because parseFields() below can't be recursively called due to locking
	for v.Kind() == reflect.Ptr {
		v = v.Elem()
	}
	t := v.Type()
	if v.NumField() < 1 {
		return nil, errors.New("struc: Struct has no fields.")
	}
	sizeofMap := make(map[string][]int)
	fields := make(Fields, 0, v.NumField())
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		f, err := parseField(field)
		if err != nil {
			return nil, err
		}
		f.CanSet = v.Field(i).CanSet()
		if !f.CanSet {
			continue
		}
		f.Index = i
		tag := parseStrucTag(field.Tag)
		if tag.Sizeof != "" {
			target, ok := t.FieldByName(tag.Sizeof)
			if !ok {
				return nil, fmt.Errorf("struc: `sizeof=%s` field does not exist", tag.Sizeof)
			}
			f.Sizeof = target.Index
			sizeofMap[tag.Sizeof] = field.Index
		}
		if sizefrom, ok := sizeofMap[field.Name]; ok {
			f.Sizefrom = sizefrom
		}
		if f.Len == -1 && f.Sizefrom == nil {
			return nil, fmt.Errorf("struc: field `%s` is a slice with no length or sizeof field", field.Name)
		}
		// recurse into nested structs
		// TODO: handle loops (probably by indirecting the []Field and putting pointer in cache)
		if f.Type == Struct {
			typ := field.Type
			if f.Ptr {
				typ = typ.Elem()
			}
			if f.Slice {
				typ = typ.Elem()
			}
			f.Fields, err = parseFieldsLocked(reflect.New(typ))
			if err != nil {
				return nil, err
			}
		}
		fields = append(fields, f)
	}
	return fields, nil
}

var fieldCache = make(map[reflect.Type]Fields)
var fieldCacheLock sync.RWMutex
var parseLock sync.Mutex

func parseFields(v reflect.Value) (Fields, error) {
	for v.Kind() == reflect.Ptr {
		v = v.Elem()
	}
	t := v.Type()
	// this read lock will be manually released on a cache hit, or before parsing
	fieldCacheLock.RLock()
	if cached, ok := fieldCache[t]; ok {
		fieldCacheLock.RUnlock()
		return cached, nil
	}
	// take a lock so multiple goroutines can't parse fields at once
	parseLock.Lock()
	defer parseLock.Unlock()
	// check a second time, just in case it was parsed by another goroutine during lock
	if cached, ok := fieldCache[t]; ok {
		fieldCacheLock.RUnlock()
		return cached, nil
	}
	// if we hold read lock during parse, that's an extra ~70,000ns of lock time
	// so it's manually released here instead of deferring
	fieldCacheLock.RUnlock()
	fields, err := parseFieldsLocked(v)
	if err != nil {
		return nil, err
	}
	fieldCacheLock.Lock()
	fieldCache[t] = fields
	fieldCacheLock.Unlock()
	return fields, nil
}
<|endoftext|>


<commit_before>//  The MIT License (MIT) - http://opensource.org/licenses/MIT
//
//  Copyright (c) 2014 slowfei
//
//  Create on 2014-09-10
//  Update on 2014-09-19
//  Email  slowfei#foxmail.com
//  Home   http://www.slowfei.com

//
package gosfdoc

import (
	"bytes"
	"github.com/slowfei/gosfcore/utils/strings"
	"regexp"
	// "fmt"
	"os"
	"strings"
)

const (
	DOC_FILE_SUFFIX = ".doc"     // document file suffix
	NIL_DOC_NAME    = "document" // nilDocParser struct use
)

var (

	//  主要用于去除注释的前缀
	_prefixFilterTags = [][]byte{
		[]byte(" *\t"),
		[]byte(" *  "),
		[]byte(" * "),
		[]byte("//\t"),
		[]byte("//  "),
		[]byte("// "),
		[]byte("//"),
	}

	_tagStar   = []byte("*")  // comments (*)
	_tagDSlash = []byte("//") // double slash
)

/**
 *	nil document parser
 *	specifically for .doc file serve
 */
type nilDocParser struct {
}

/**
 *	see DocParser interface
 */
func (n *nilDocParser) Name() string {
	return NIL_DOC_NAME
}

/**
 *	see DocParser interface
 */
func (n *nilDocParser) CheckFile(path string, info os.FileInfo) bool {
	return strings.HasSuffix(path, DOC_FILE_SUFFIX)
}

/**
 *	see DocParser interface
 */
func (n *nilDocParser) EachIndexFile(filebuf *FileBuf) {

}

/**
 *	see DocParser interface
 */
func (n *nilDocParser) ParsePreview(fileCont *bytes.Buffer) []Preview {
	return nil
}

/**
 *	see DocParser interface
 */
func (n *nilDocParser) ParseCodeblock(fileCont *bytes.Buffer) []CodeBlock {
	return nil
}

/**
 *  parse public document content
 *
 *  @param `fileBuf`
 *  @return document array
 */
func ParseDocument(fileBuf *FileBuf) []Document {
	var resultDocs []Document = nil

	docsBuf := fileBuf.FinaAll(REXDocument)
	docsCount := len(docsBuf)

	if 0 == docsCount {
		return resultDocs
	}

	resultDocs = make([]Document, 0, docsCount)

	for i := 0; i < docsCount; i++ {
		docStruct := Document{}
		buf := docsBuf[i]

		lines := bytes.Split(buf, []byte("\n"))
		linesCount := len(lines)

		//  title and index parse
		indexTitleLine := lines[0]
		indexTitleMatch := REXDocIndexTitle.FindSubmatch(indexTitleLine)
		//  index 0 is source string
		//  index 1 is "///" || "/***"
		//  index 2 is "index-" index string
		//  index 3 is title

		if 4 == len(indexTitleMatch) {
			// extract title and z-index
			docStruct.SortTag = SFStringsUtil.ToInt(string(indexTitleMatch[2]))
			docStruct.Title = string(indexTitleMatch[3])
		}

		//  content parse
		contentBuf := bytes.NewBuffer(nil)
		var prefixTag []byte = nil
		prefixLen := 0

		for i := 1; i < linesCount-1; i++ {
			newLine := lines[i]

			if i == 1 {
				prefixTag = findPrefixFilterTag(newLine)
				prefixLen = len(prefixTag)
			}

			if nil != prefixTag {

				if 0 == bytes.Index(newLine, prefixTag) {
					contentBuf.Write(newLine[prefixLen:])
				} else {
					trimed := bytes.TrimSpace(newLine)
					// 有可能是空行，所需需要判断这行是否只有（ "*" || "//" ），如果不是则添加追加这一行内容
					if !bytes.Equal(trimed, _tagStar) && !bytes.Equal(trimed, _tagDSlash) {
						contentBuf.Write(newLine)
					}
				}

			} else {
				contentBuf.Write(newLine)
			}

			contentBuf.WriteByte('\n')
		}
		docStruct.Content = contentBuf.String()

		if 0 != len(docStruct.Content) {
			resultDocs = append(resultDocs, docStruct)
		}

	}

	return resultDocs
}

/**
 *  commons parse file about content
 *
 *  @param `fileBuf`
 *  @return about content
 */
func ParseAbout(fileBuf *FileBuf) *About {
	return &About{Content: parseAboutAndIntro(fileBuf, REXAbout)}
}

/**
 *  commons parse file introduction content
 *
 *  @param `fileBuf`
 *  @return introduction content
 */
func ParseIntro(fileBuf *FileBuf) *Intro {
	return &Intro{Content: parseAboutAndIntro(fileBuf, REXIntro)}
}

/**
 *  commons about intro parse
 *
 *  @param fileBuf
 *  @param rex
 */
func parseAboutAndIntro(fileBuf *FileBuf, rex *regexp.Regexp) []byte {
	var result []byte = nil
	var prefixTag []byte = nil
	prefixLen := 0

	buf := fileBuf.Find(rex)

	if 0 < len(buf) {
		appendLine := bytes.NewBuffer(nil)

		lines := bytes.Split(buf, []byte("\n"))
		linesCount := len(lines)

		for i := 1; i < linesCount-1; i++ {
			newLine := lines[i]

			if i == 1 {
				//  记录第一个前缀的标识，以第一个为准，后面的根据要求都要是相同的注释前缀。
				/**
				  (*)remove prefix tag
				  (*)
				  (*)
				*/
				prefixTag = findPrefixFilterTag(newLine)
				prefixLen = len(prefixTag)
			}

			if 0 != len(prefixTag) {

				if 0 == bytes.Index(newLine, prefixTag) {
					appendLine.Write(newLine[prefixLen:])
				} else {
					trimed := bytes.TrimSpace(newLine)
					// 有可能是空行，所需需要判断这行是否只有（ "*" || "//" ），如果不是则添加追加这一行内容
					if !bytes.Equal(trimed, _tagStar) && !bytes.Equal(trimed, _tagDSlash) {
						appendLine.Write(newLine)
					}
				}

			} else {
				appendLine.Write(newLine)
			}

			appendLine.WriteByte('\n')
		}

		if 0 < appendLine.Len() {
			result = appendLine.Bytes()
		}
	}

	return result
}

/**
 *  find prefix filter tag index
 *  //
 *  // content ("// ") is prefix tag
 *  //
 *  see var _prefixFilterTags
 */
func findPrefixFilterTag(src []byte) []byte {
	var pftCount = len(_prefixFilterTags)

	for i := 0; i < pftCount; i++ {
		prefix := _prefixFilterTags[i]
		if 0 == bytes.Index(src, prefix) {
			return prefix
		}
	}

	return nil
}
<commit_msg>更改nilDocParser解析器使用.dc后缀<commit_after>//  The MIT License (MIT) - http://opensource.org/licenses/MIT
//
//  Copyright (c) 2014 slowfei
//
//  Create on 2014-09-10
//  Update on 2014-09-23
//  Email  slowfei#foxmail.com
//  Home   http://www.slowfei.com

//
package gosfdoc

import (
	"bytes"
	"github.com/slowfei/gosfcore/utils/strings"
	"regexp"
	// "fmt"
	"os"
	"strings"
)

const (
	DOC_FILE_SUFFIX = ".dc"      // document file suffix(document comments)
	NIL_DOC_NAME    = "document" // nilDocParser struct use
)

var (

	//  主要用于去除注释的前缀
	_prefixFilterTags = [][]byte{
		[]byte(" *\t"),
		[]byte(" *  "),
		[]byte(" * "),
		[]byte("//\t"),
		[]byte("//  "),
		[]byte("// "),
		[]byte("//"),
	}

	_tagStar   = []byte("*")  // comments (*)
	_tagDSlash = []byte("//") // double slash
)

/**
 *	nil document parser
 *	specifically for .doc file serve
 */
type nilDocParser struct {
}

/**
 *	see DocParser interface
 */
func (n *nilDocParser) Name() string {
	return NIL_DOC_NAME
}

/**
 *	see DocParser interface
 */
func (n *nilDocParser) CheckFile(path string, info os.FileInfo) bool {
	return strings.HasSuffix(path, DOC_FILE_SUFFIX)
}

/**
 *	see DocParser interface
 */
func (n *nilDocParser) EachIndexFile(filebuf *FileBuf) {

}

/**
 *	see DocParser interface
 */
func (n *nilDocParser) ParsePreview(fileCont *bytes.Buffer) []Preview {
	return nil
}

/**
 *	see DocParser interface
 */
func (n *nilDocParser) ParseCodeblock(fileCont *bytes.Buffer) []CodeBlock {
	return nil
}

/**
 *  parse public document content
 *
 *  @param `fileBuf`
 *  @return document array
 */
func ParseDocument(fileBuf *FileBuf) []Document {
	var resultDocs []Document = nil

	docsBuf := fileBuf.FinaAll(REXDocument)
	docsCount := len(docsBuf)

	if 0 == docsCount {
		return resultDocs
	}

	resultDocs = make([]Document, 0, docsCount)

	for i := 0; i < docsCount; i++ {
		docStruct := Document{}
		buf := docsBuf[i]

		lines := bytes.Split(buf, []byte("\n"))
		linesCount := len(lines)

		//  title and index parse
		indexTitleLine := lines[0]
		indexTitleMatch := REXDocIndexTitle.FindSubmatch(indexTitleLine)
		//  index 0 is source string
		//  index 1 is "///" || "/***"
		//  index 2 is "index-" index string
		//  index 3 is title

		if 4 == len(indexTitleMatch) {
			// extract title and z-index
			docStruct.SortTag = SFStringsUtil.ToInt(string(indexTitleMatch[2]))
			docStruct.Title = string(indexTitleMatch[3])
		}

		//  content parse
		contentBuf := bytes.NewBuffer(nil)
		var prefixTag []byte = nil
		prefixLen := 0

		for i := 1; i < linesCount-1; i++ {
			newLine := lines[i]

			if i == 1 {
				prefixTag = findPrefixFilterTag(newLine)
				prefixLen = len(prefixTag)
			}

			if nil != prefixTag {

				if 0 == bytes.Index(newLine, prefixTag) {
					contentBuf.Write(newLine[prefixLen:])
				} else {
					trimed := bytes.TrimSpace(newLine)
					// 有可能是空行，所需需要判断这行是否只有（ "*" || "//" ），如果不是则添加追加这一行内容
					if !bytes.Equal(trimed, _tagStar) && !bytes.Equal(trimed, _tagDSlash) {
						contentBuf.Write(newLine)
					}
				}

			} else {
				contentBuf.Write(newLine)
			}

			contentBuf.WriteByte('\n')
		}
		docStruct.Content = contentBuf.String()

		if 0 != len(docStruct.Content) {
			resultDocs = append(resultDocs, docStruct)
		}

	}

	return resultDocs
}

/**
 *  commons parse file about content
 *
 *  @param `fileBuf`
 *  @return about content
 */
func ParseAbout(fileBuf *FileBuf) *About {
	data := parseAboutAndIntro(fileBuf, REXAbout)

	var result *About = nil
	if 0 != len(data) {
		result = &About{Content: data}
	}

	return result
}

/**
 *  commons parse file introduction content
 *
 *  @param `fileBuf`
 *  @return introduction content
 */
func ParseIntro(fileBuf *FileBuf) *Intro {
	data := parseAboutAndIntro(fileBuf, REXIntro)

	var result *Intro = nil
	if 0 != len(data) {
		result = &Intro{Content: data}
	}

	return result
}

/**
 *  commons about intro parse
 *
 *  @param fileBuf
 *  @param rex
 */
func parseAboutAndIntro(fileBuf *FileBuf, rex *regexp.Regexp) []byte {
	var result []byte = nil
	var prefixTag []byte = nil
	prefixLen := 0

	buf := fileBuf.Find(rex)

	if 0 < len(buf) {
		appendLine := bytes.NewBuffer(nil)

		lines := bytes.Split(buf, []byte("\n"))
		linesCount := len(lines)

		for i := 1; i < linesCount-1; i++ {
			newLine := lines[i]

			if i == 1 {
				//  记录第一个前缀的标识，以第一个为准，后面的根据要求都要是相同的注释前缀。
				/**
				  (*)remove prefix tag
				  (*)
				  (*)
				*/
				prefixTag = findPrefixFilterTag(newLine)
				prefixLen = len(prefixTag)
			}

			if 0 != len(prefixTag) {

				if 0 == bytes.Index(newLine, prefixTag) {
					appendLine.Write(newLine[prefixLen:])
				} else {
					trimed := bytes.TrimSpace(newLine)
					// 有可能是空行，所需需要判断这行是否只有（ "*" || "//" ），如果不是则添加追加这一行内容
					if !bytes.Equal(trimed, _tagStar) && !bytes.Equal(trimed, _tagDSlash) {
						appendLine.Write(newLine)
					}
				}

			} else {
				appendLine.Write(newLine)
			}

			appendLine.WriteByte('\n')
		}

		if 0 < appendLine.Len() {
			result = appendLine.Bytes()
		}
	}

	return result
}

/**
 *  find prefix filter tag index
 *  //
 *  // content ("// ") is prefix tag
 *  //
 *  see var _prefixFilterTags
 */
func findPrefixFilterTag(src []byte) []byte {
	var pftCount = len(_prefixFilterTags)

	for i := 0; i < pftCount; i++ {
		prefix := _prefixFilterTags[i]
		if 0 == bytes.Index(src, prefix) {
			return prefix
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package sgr

import (
	"fmt"
	"strconv"
	"strings"
)

func MustParse(format string) string {
	str, err := parse(true, false, format)
	if err != nil {
		panic(err)
	}
	return str
}

func MustParseln(format string) string {
	str, err := parse(true, true, format)
	if err != nil {
		panic(err)
	}
	return str
}

func MustParseWithoutReset(format string) string {
	str, err := parse(false, false, format)
	if err != nil {
		panic(err)
	}
	return str
}

func Parse(format string) (string, error) {
	return parse(true, false, format)
}

func Parseln(format string) (string, error) {
	return parse(true, true, format)
}

func ParseWithoutReset(format string) (string, error) {
	return parse(false, false, format)
}

func parse(reset bool, newline bool, format string) (string, error) {
	// Builder used to build the colored string.
	sb := new(sgrBuilder)

	// position in the parsing process
	pos := 0

	// index of the currenctly processing square bracket start
	idxStart := 0
	idxEnd := 0

	for {
		// Find next square bracket, break loop when none was found.
		relBlockOpen := strings.IndexRune(format[pos:], '[')
		if relBlockOpen == -1 {
			sb.appendString(format[pos:])
			break
		}
		idxStart = pos + relBlockOpen

		// Test for escaped square bracket
		if format[idxStart+1] == '[' {
			sb.appendString(format[pos : idxStart+2])
			pos = idxStart + 2
			continue
		}

		// Add skipped string (if any)
		if idxStart > pos { //idxStart > pos+1 ???
			sb.appendString(format[pos:idxStart])
		}

		// Find square bracket end
		relBlockClose := strings.IndexRune(format[idxStart:], ']')
		if relBlockClose == -1 {
			return "", fmt.Errorf("Opened square bracket never closed at pos %d.", idxStart)
		}
		idxEnd = idxStart + relBlockClose

		// found a block
		block := format[idxStart+1 : idxEnd]
		fields := strings.Fields(block)
		for _, field := range fields {
			isFgColor := strings.HasPrefix(field, "fg-")
			isBgColor := strings.HasPrefix(field, "bg-")
			if isFgColor || isBgColor {
				// Get the color number from the fg- or bg- code
				clr, err := strconv.Atoi(field[3:])
				if err != nil {
					return "", fmt.Errorf("Invalid block code '%s' in block at position %d.", field, idxStart)
				}
				if clr < 0 || clr > 255 {
					return "", fmt.Errorf("Invalid color number %d. Expecting 0-255.", clr)
				}

				// Add actual fg or bg color to sgrBuilder
				if isFgColor {
					sb.appendSgr(FgColor(clr))
				} else {
					sb.appendSgr(BgColor(clr))
				}
				continue // next field
			}

			switch field {
			// Options
			case "reset":
				sb.appendSgr(Reset)
			case "fg-reset":
				sb.appendSgr(ResetForegroundColor)
			case "bg-reset":
				sb.appendSgr(ResetBackgroundColor)
			case "bold":
				sb.appendSgr(Bold)
			case "boldOff":
				sb.appendSgr(BoldOff)
			case "underline":
				sb.appendSgr(Underline)
			case "underlineOff":
				sb.appendSgr(UnderlineOff)
			case "blink":
				sb.appendSgr(Blink)
			case "blinkOff":
				sb.appendSgr(BlinkOff)
			case "imageNegative":
				sb.appendSgr(ImageNegative)
			case "imagePositive":
				sb.appendSgr(ImagePositive)
			case "framed":
				sb.appendSgr(Framed)
			case "encircled":
				sb.appendSgr(Encircled)
			case "framedEncircledOff":
				sb.appendSgr(FramedEncircledOff)
			case "overlined":
				sb.appendSgr(Overlined)
			case "overlinedOff":
				sb.appendSgr(OverlinedOff)

			// Foreground Colors
			case "fg-black":
				sb.appendSgr(FgBlack)
			case "fg-red":
				sb.appendSgr(FgRed)
			case "fg-green":
				sb.appendSgr(FgGreen)
			case "fg-yellow":
				sb.appendSgr(FgYellow)
			case "fg-blue":
				sb.appendSgr(FgBlue)
			case "fg-magenta":
				sb.appendSgr(FgMagenta)
			case "fg-cyan":
				sb.appendSgr(FgCyan)
			case "fg-grey":
				sb.appendSgr(FgGrey)
			case "fg-white":
				sb.appendSgr(FgWhite)

			// Background Colors
			case "bg-black":
				sb.appendSgr(BgBlack)
			case "bg-red":
				sb.appendSgr(BgRed)
			case "bg-green":
				sb.appendSgr(BgGreen)
			case "bg-yellow":
				sb.appendSgr(BgYellow)
			case "bg-blue":
				sb.appendSgr(BgBlue)
			case "bg-magenta":
				sb.appendSgr(BgMagenta)
			case "bg-cyan":
				sb.appendSgr(BgCyan)
			case "bg-grey":
				sb.appendSgr(BgGrey)
			case "bg-white":
				sb.appendSgr(BgWhite)

			// Not Found
			default:
				return "", fmt.Errorf("Invalid block code '%s' in block at position %d.", field, idxStart)
			}
		}

		// Change starting position for next iteration.
		pos = idxEnd + 1
	}

	if reset {
		sb.appendSgr(Reset)
	}

	if newline {
		sb.appendString("\n")
	}

	return sb.string(), nil
}
<commit_msg>Moved the fg-number parsing below the fg-definition parsing.<commit_after>package sgr

import (
	"fmt"
	"strconv"
	"strings"
)

func MustParse(format string) string {
	str, err := parse(true, false, format)
	if err != nil {
		panic(err)
	}
	return str
}

func MustParseln(format string) string {
	str, err := parse(true, true, format)
	if err != nil {
		panic(err)
	}
	return str
}

func MustParseWithoutReset(format string) string {
	str, err := parse(false, false, format)
	if err != nil {
		panic(err)
	}
	return str
}

func Parse(format string) (string, error) {
	return parse(true, false, format)
}

func Parseln(format string) (string, error) {
	return parse(true, true, format)
}

func ParseWithoutReset(format string) (string, error) {
	return parse(false, false, format)
}

func parse(reset bool, newline bool, format string) (string, error) {
	// Builder used to build the colored string.
	sb := new(sgrBuilder)

	// position in the parsing process
	pos := 0

	// index of the currenctly processing square bracket start
	idxStart := 0
	idxEnd := 0

	for {
		// Find next square bracket, break loop when none was found.
		relBlockOpen := strings.IndexRune(format[pos:], '[')
		if relBlockOpen == -1 {
			sb.appendString(format[pos:])
			break
		}
		idxStart = pos + relBlockOpen

		// Test for escaped square bracket
		if format[idxStart+1] == '[' {
			sb.appendString(format[pos : idxStart+2])
			pos = idxStart + 2
			continue
		}

		// Add skipped string (if any)
		if idxStart > pos { //idxStart > pos+1 ???
			sb.appendString(format[pos:idxStart])
		}

		// Find square bracket end
		relBlockClose := strings.IndexRune(format[idxStart:], ']')
		if relBlockClose == -1 {
			return "", fmt.Errorf("Opened square bracket never closed at pos %d.", idxStart)
		}
		idxEnd = idxStart + relBlockClose

		// found a block
		block := format[idxStart+1 : idxEnd]
		fields := strings.Fields(block)
		for _, field := range fields {

			switch field {
			// Options
			case "reset":
				sb.appendSgr(Reset)
			case "fg-reset":
				sb.appendSgr(ResetForegroundColor)
			case "bg-reset":
				sb.appendSgr(ResetBackgroundColor)
			case "bold":
				sb.appendSgr(Bold)
			case "boldOff":
				sb.appendSgr(BoldOff)
			case "underline":
				sb.appendSgr(Underline)
			case "underlineOff":
				sb.appendSgr(UnderlineOff)
			case "blink":
				sb.appendSgr(Blink)
			case "blinkOff":
				sb.appendSgr(BlinkOff)
			case "imageNegative":
				sb.appendSgr(ImageNegative)
			case "imagePositive":
				sb.appendSgr(ImagePositive)
			case "framed":
				sb.appendSgr(Framed)
			case "encircled":
				sb.appendSgr(Encircled)
			case "framedEncircledOff":
				sb.appendSgr(FramedEncircledOff)
			case "overlined":
				sb.appendSgr(Overlined)
			case "overlinedOff":
				sb.appendSgr(OverlinedOff)

			// Foreground Colors
			case "fg-black":
				sb.appendSgr(FgBlack)
			case "fg-red":
				sb.appendSgr(FgRed)
			case "fg-green":
				sb.appendSgr(FgGreen)
			case "fg-yellow":
				sb.appendSgr(FgYellow)
			case "fg-blue":
				sb.appendSgr(FgBlue)
			case "fg-magenta":
				sb.appendSgr(FgMagenta)
			case "fg-cyan":
				sb.appendSgr(FgCyan)
			case "fg-grey":
				sb.appendSgr(FgGrey)
			case "fg-white":
				sb.appendSgr(FgWhite)

			// Background Colors
			case "bg-black":
				sb.appendSgr(BgBlack)
			case "bg-red":
				sb.appendSgr(BgRed)
			case "bg-green":
				sb.appendSgr(BgGreen)
			case "bg-yellow":
				sb.appendSgr(BgYellow)
			case "bg-blue":
				sb.appendSgr(BgBlue)
			case "bg-magenta":
				sb.appendSgr(BgMagenta)
			case "bg-cyan":
				sb.appendSgr(BgCyan)
			case "bg-grey":
				sb.appendSgr(BgGrey)
			case "bg-white":
				sb.appendSgr(BgWhite)

			// Not Found
			default:
				isFgColor := strings.HasPrefix(field, "fg-")
				isBgColor := strings.HasPrefix(field, "bg-")
				if isFgColor || isBgColor {
					// Get the color number from the fg- or bg- code
					clr, err := strconv.Atoi(field[3:])
					if err != nil {
						goto invalidBlockCode
					}
					if clr < 0 || clr > 255 {
						return "", fmt.Errorf("Invalid color code %s. Expecting 0-255 or a defined color.", field[3:])
					}

					// Add actual fg or bg color to sgrBuilder
					if isFgColor {
						sb.appendSgr(FgColor(clr))
					} else {
						sb.appendSgr(BgColor(clr))
					}
					continue // next field
				} else {
					// not valid
					goto invalidBlockCode
				}
			}

		invalidBlockCode:
			return "", fmt.Errorf("Invalid block code '%s' in block at position %d.", field, idxStart)

		}

		// Change starting position for next iteration.
		pos = idxEnd + 1
	}

	if reset {
		sb.appendSgr(Reset)
	}

	if newline {
		sb.appendString("\n")
	}

	return sb.string(), nil
}
<|endoftext|>


<commit_before>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"compress/gzip"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
)

const (
	// Main config options
	chars   = "abcdefghijklmnopqrstuvwxyz0123456789"
	idSize  = 8
	siteUrl = "http://localhost:9090"
	listen  = "localhost:9090"
	dataDir = "data"
	maxSize = 1 << 20

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went wrong. Woop woop woop woop!"

	// POST error messages
	missingForm = "Form with paste could not be found."
)

var validId *regexp.Regexp = regexp.MustCompile("^[a-z0-9]{" + strconv.FormatInt(idSize, 10) + "}$")

func pathId(id string) string {
	return path.Join(id[0:2], id[2:4], id[4:8])
}

func randomId() string {
	s := make([]byte, idSize)
	var offset uint = 0
	for {
		r := rand.Int63()
		for i := 0; i < 8; i++ {
			randbyte := int(r&0xff) % len(chars)
			s[offset] = chars[randbyte]
			offset++
			if offset == idSize {
				return string(s)
			}
			r >>= 8
		}
	}
	return strings.Repeat(chars[0:1], idSize)
}

func handler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		id := r.URL.Path[1:]
		if len(id) == 0 {
			fmt.Fprintf(w, "<form action=\"%s\" method=\"POST\"><textarea cols=80 rows=48 name=\"paste\"></textarea>
<button type=\"submit\">paste</button></form>", siteUrl)
			return
		}
		if !validId.MatchString(id) {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", invalidId)
			return
		}
		filePath := pathId(id)
		file, err := os.Open(filePath)
		if err != nil {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "%s\n", pasteNotFound)
			return
		}
		compReader, err := gzip.NewReader(file)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		io.Copy(w, compReader)
		compReader.Close()
		file.Close()

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, maxSize)
		var id, filePath string
		for {
			id = randomId()
			filePath = pathId(id)
			if _, err := os.Stat(filePath); os.IsNotExist(err) {
				break
			}
		}
		dir, _ := path.Split(filePath)
		err := os.MkdirAll(dir, 0700)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		err = r.ParseMultipartForm(maxSize << 1)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		vs, found := r.Form["paste"]
		if !found {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", missingForm)
			return
		}
		data := vs[0]
		compWriter := gzip.NewWriter(file)
		_, err = io.WriteString(compWriter, data)
		compWriter.Close()
		file.Close()
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		fmt.Fprintf(w, "%s/%s\n", siteUrl, id)
	}
}

func main() {
	os.Mkdir(dataDir, 0700)
	err := os.Chdir(dataDir)
	if err != nil {
		return
	}
	http.HandleFunc("/", handler)
	http.ListenAndServe(listen, nil)
}
<commit_msg>Better explain options<commit_after>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"compress/gzip"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
)

const (

	// Can't be bigger than 256 chars
	chars = "abcdefghijklmnopqrstuvwxyz0123456789"

	// Should be at least 6 since 4 are used for the dirs
	idSize = 8

	// Where to send users to when they post a paste
	siteUrl = "http://localhost:9090"

	// Local listen address
	listen = "localhost:9090"

	// Where to store the compressed pastes in the filesystem
	dataDir = "data"

	// Maximum paste size (before compression) in bytes
	maxSize = 1 << 20

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went wrong. Woop woop woop woop!"

	// POST error messages
	missingForm = "Form with paste could not be found."
)

var validId *regexp.Regexp = regexp.MustCompile("^[a-z0-9]{" + strconv.FormatInt(idSize, 10) + "}$")

func pathId(id string) string {
	return path.Join(id[0:2], id[2:4], id[4:8])
}

func randomId() string {
	s := make([]byte, idSize)
	var offset uint = 0
	for {
		r := rand.Int63()
		for i := 0; i < 8; i++ {
			randbyte := int(r&0xff) % len(chars)
			s[offset] = chars[randbyte]
			offset++
			if offset == idSize {
				return string(s)
			}
			r >>= 8
		}
	}
	return strings.Repeat(chars[0:1], idSize)
}

func handler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		id := r.URL.Path[1:]
		if len(id) == 0 {
			fmt.Fprintf(w, "<form action=\"%s\" method=\"POST\"><textarea cols=80 rows=48 name=\"paste\"></textarea>
<button type=\"submit\">paste</button></form>", siteUrl)
			return
		}
		if !validId.MatchString(id) {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", invalidId)
			return
		}
		filePath := pathId(id)
		file, err := os.Open(filePath)
		if err != nil {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "%s\n", pasteNotFound)
			return
		}
		compReader, err := gzip.NewReader(file)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		io.Copy(w, compReader)
		compReader.Close()
		file.Close()

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, maxSize)
		var id, filePath string
		for {
			id = randomId()
			filePath = pathId(id)
			if _, err := os.Stat(filePath); os.IsNotExist(err) {
				break
			}
		}
		dir, _ := path.Split(filePath)
		err := os.MkdirAll(dir, 0700)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		err = r.ParseMultipartForm(maxSize << 1)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		vs, found := r.Form["paste"]
		if !found {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", missingForm)
			return
		}
		data := vs[0]
		compWriter := gzip.NewWriter(file)
		_, err = io.WriteString(compWriter, data)
		compWriter.Close()
		file.Close()
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		fmt.Fprintf(w, "%s/%s\n", siteUrl, id)
	}
}

func main() {
	os.Mkdir(dataDir, 0700)
	err := os.Chdir(dataDir)
	if err != nil {
		return
	}
	http.HandleFunc("/", handler)
	http.ListenAndServe(listen, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"io"
	"log"
	"os"
	"os/exec"
	"syscall"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"

	"io/ioutil"

	"google.golang.org/cloud"
	"google.golang.org/cloud/datastore"
)

// BuildData is used to represent general data for a single invokation of
// a coduno build
type BuildData struct {
	Challenge string
	User      string
	Commit    string
	Status    string
	StartTime time.Time
	EndTime   time.Time
}

// LogData is used to represent accumulated log data of a single invokation of
// a coduno build
type LogData struct {
	InLog      string `datastore:",noindex"`
	OutLog     string `datastore:",noindex"`
	ExtraLog   string `datastore:",noindex"`
	PrepareLog string `datastore:",noindex"`
	SysUsage   syscall.Rusage
}

var signal = make(chan int)
var ctx context.Context

func init() {
	err := error(nil)
	home := os.Getenv("HOME")
	secret, err := ioutil.ReadFile(home + "/config/secret.json")
	if err != nil {
		log.Panic(err)
	}
	config, err := google.JWTConfigFromJSON(secret, datastore.ScopeDatastore,
		datastore.ScopeUserEmail)
	if err != nil {
		log.Panic(err)
	}

	ctx = cloud.NewContext("coduno", config.Client(oauth2.NoContext))
}

// LogBuildStart sends info to the datastore, informing that a new build
// started
func LogBuildStart(repo string, commit string, user string) (*datastore.Key,
	*BuildData) {
	key := datastore.NewIncompleteKey(ctx, "testrun", nil)
	build := &BuildData{
		Commit:    commit,
		Challenge: repo,
		User:      user,
		StartTime: time.Now(),
		Status:    "Started",
	}
	key, err := datastore.Put(ctx, key, build)
	if err != nil {
		log.Panic(err)
	}
	return key, build
}

// LogRunComplete logs the end of a completed (failed of finished) run of
// a coduno testrun
func LogRunComplete(pKey *datastore.Key, build *BuildData, in string,
	out string, extra string, exit error, prepLog string, stats syscall.Rusage) {
	tx, err := datastore.NewTransaction(ctx)
	if err != nil {
		log.Panic(err)
	}
	build.EndTime = time.Now()
	if exit != nil {
		build.Status = "FAILED"
	} else {
		build.Status = "DONE"
	}
	_, err = tx.Put(pKey, build)
	if err != nil {
		log.Panic(err)
		err = tx.Rollback()
		if err != nil {
			log.Panic(err)
		}
	}
	data := &LogData{
		InLog:      in,
		OutLog:     out,
		ExtraLog:   extra,
		PrepareLog: prepLog,
		SysUsage:   stats,
	}
	k := datastore.NewIncompleteKey(ctx, "testrun", pKey)
	_, err = tx.Put(k, data)
	if err != nil {
		log.Panic(err)
		err = tx.Rollback()
		if err != nil {
			log.Panic(err)
		}
	}
	_, err = tx.Commit()
	if err != nil {
		log.Panic(err)
	}
}

func sendSig() {
	signal <- 1
}

func pipeOutput(out io.ReadCloser, dest io.Writer, logBuf *bytes.Buffer) {
	tempBuf := make([]byte, 1024)
	writeErr := error(nil)
	r, readErr := int(0), error(nil)

	defer out.Close()
	defer sendSig()

	for readErr == nil {
		r, readErr = out.Read(tempBuf)

		if logBuf != nil {
			logBuf.Write(tempBuf[0:r])
		}

		if dest != nil && r != 0 && writeErr == nil {
			_, writeErr := dest.Write(tempBuf[0:r])
			if writeErr != nil {
				log.Print(writeErr)
			}
		}
	}
}

func main() {
	if len(os.Args) != 6 {
		log.Fatal(os.Args)
	}

	username := os.Args[1]
	repo := os.Args[2]
	commit := os.Args[3]
	tmpdir := os.Args[4]
	testdir := os.Args[5]

	key, build := LogBuildStart(repo, commit, username)
	cmdUser := exec.Command(
		"sudo",
		"docker",
		"run",
		"--rm",
		"-v",
		tmpdir+":/run",
		"coduno/base")
	cmdTest := exec.Command(
		"sudo",
		"docker",
		"run",
		"--rm",
		"-v",
		testdir+":/run",
		"coduno/base")
	outUser, err := cmdUser.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	errUser, err := cmdUser.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	inUser, err := cmdUser.StdinPipe()
	if err != nil {
		log.Fatal(err)
	}
	outTest, err := cmdTest.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	errTest, err := cmdTest.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	inTest, err := cmdTest.StdinPipe()
	if err != nil {
		log.Fatal(err)
	}

	var userToTest bytes.Buffer
	var testToUser bytes.Buffer
	var extraBuf bytes.Buffer
	cmdUser.Start()
	cmdTest.Start()
	go pipeOutput(outUser, inTest, &userToTest)
	go pipeOutput(outTest, inUser, &testToUser)
	go pipeOutput(errUser, os.Stderr, nil)
	go pipeOutput(errTest, ioutil.Discard, &extraBuf)

	<-signal
	<-signal
	<-signal
	<-signal
	exitErr := cmdUser.Wait()

	prepLog, err := ioutil.ReadFile(tmpdir + "/prepare.log")
	if err != nil { // This file should always exist, so an error here should never happen
		log.Fatal(err)
	}

	var stats syscall.Rusage
	statsData, err := ioutil.ReadFile(tmpdir + "/stats.log")
	if err != nil {
		log.Print(err)
	} else {
		err = json.Unmarshal(statsData, &stats)
		if err != nil {
			log.Fatal(err)
		}
	}

	LogRunComplete(key, build, testToUser.String(), userToTest.String(), extraBuf.String(), exitErr, string(prepLog), stats)
}
<commit_msg>Little rewrite<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path"
	"sync"
	"syscall"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"

	"google.golang.org/cloud"
	"google.golang.org/cloud/datastore"
)

// BuildData is used to represent general data for a single invocation of
// a Coduno build
type BuildData struct {
	Challenge string
	User      string
	Commit    string
	Status    string
	StartTime time.Time
	EndTime   time.Time
}

// LogData is used to represent accumulated log data of a single invocation of
// a Coduno build
type LogData struct {
	InLog      string `datastore:",noindex"`
	OutLog     string `datastore:",noindex"`
	ExtraLog   string `datastore:",noindex"`
	PrepareLog string `datastore:",noindex"`
	SysUsage   syscall.Rusage
}

// PipeStatus holds the satus of a buffered pipe.
// It tells how many bytes have been read from the source,
// wrote to the destination and were buffered.
type PipeStatus struct {
	Read,
	Wrote,
	Buffered int

	ReadError,
	WriteError,
	BufferError error
}

const buildKind = "builds"

var ctx context.Context

func init() {
	user, err := user.Current()
	if err != nil {
		panic(err)
	}

	fileName := path.Join(user.HomeDir, "config", "secret.json")
	secret, err := ioutil.ReadFile(fileName)
	if err != nil {
		panic(err)
	}

	config, err := google.JWTConfigFromJSON(secret, datastore.ScopeDatastore, datastore.ScopeUserEmail)
	if err != nil {
		panic(err)
	}

	ctx = cloud.NewContext("coduno", config.Client(oauth2.NoContext))
}

// LogBuildStart sends info to the datastore, informing that a new build
// started
func LogBuildStart(repo string, commit string, user string) (*datastore.Key, *BuildData) {
	key := datastore.NewIncompleteKey(ctx, buildKind, nil)
	build := &BuildData{repo, user, commit, "started", time.Now(), time.Unix(0, 0)}

	key, err := datastore.Put(ctx, key, build)
	if err != nil {
		fmt.Fprintf(os.Stderr, "LogBuildStart: %v", err)
	}
	return key, build
}

// LogRunComplete logs the end of a completed (failed of finished) run of
// a coduno testrun
func LogRunComplete(pKey *datastore.Key, build *BuildData, in,
	out, extra string, exit error, prepLog string, stats syscall.Rusage) {
	tx, err := datastore.NewTransaction(ctx)
	if err != nil {
		fmt.Fprintf(os.Stderr, "LogRunComplete: Could not get transaction!")
	}
	build.EndTime = time.Now()
	if exit != nil {
		build.Status = "failed"
	} else {
		build.Status = "good"
	}
	_, err = tx.Put(pKey, build)
	if err != nil {
		fmt.Fprintf(os.Stderr, "LogRunComplete: Putting build failed!")
		tx.Rollback()
		return
	}
	data := &LogData{in, out, extra, prepLog, stats}
	k := datastore.NewIncompleteKey(ctx, buildKind, pKey)
	_, err = tx.Put(k, data)
	if err != nil {
		fmt.Fprintf(os.Stderr, "LogRunComplete: Putting data failed!")
		tx.Rollback()
		return
	}
	tx.Commit()
}

func pipeOutput(wg *sync.WaitGroup, rc io.ReadCloser, w io.Writer, buf *bytes.Buffer) (parent chan<- PipeStatus) {
	defer wg.Done()
	// if we have no rc we cannot do anything, because
	// that's where the data come from
	if rc == nil {
		return
	}
	defer rc.Close()

	s := PipeStatus{}

	tmp := make([]byte, 1024)

	// to count how many bytes we read/write/buffer on
	// every loop
	var cR, cW, cB int

	for s.ReadError == nil && (s.WriteError == nil || s.BufferError == nil) {
		cR, s.ReadError = rc.Read(tmp)
		s.Read += cR

		if cR == 0 {
			continue
		}

		if buf != nil && s.BufferError == nil {
			cB, s.BufferError = buf.Write(tmp[0:cR])
			s.Buffered += cB
		}

		if w != nil && s.WriteError == nil {
			cW, s.WriteError = w.Write(tmp[0:cR])
			s.Wrote += cW
		}
	}
	parent <- s
	return
}

func main() {
	if len(os.Args) != 6 {
		log.Fatal(os.Args)
	}

	username := os.Args[1]
	repo := os.Args[2]
	commit := os.Args[3]
	tmpdir := os.Args[4]
	testdir := os.Args[5]

	key, build := LogBuildStart(repo, commit, username)
	cmdUser := exec.Command(
		"sudo",
		"docker",
		"run",
		"--rm",
		"-v",
		tmpdir+":/run",
		"coduno/base")
	cmdTest := exec.Command(
		"sudo",
		"docker",
		"run",
		"--rm",
		"-v",
		testdir+":/run",
		"coduno/base")

	outUser, err := cmdUser.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	errUser, err := cmdUser.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	inUser, err := cmdUser.StdinPipe()
	if err != nil {
		log.Fatal(err)
	}
	outTest, err := cmdTest.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	errTest, err := cmdTest.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	inTest, err := cmdTest.StdinPipe()
	if err != nil {
		log.Fatal(err)
	}

	var userToTest, testToUser, extraBuf bytes.Buffer
	cmdUser.Start()
	cmdTest.Start()

	var wg sync.WaitGroup
	wg.Add(4)

	go pipeOutput(&wg, outUser, inTest, &userToTest)
	go pipeOutput(&wg, outTest, inUser, &testToUser)
	go pipeOutput(&wg, errUser, os.Stderr, nil)
	go pipeOutput(&wg, errTest, nil, &extraBuf)

	exitErr := cmdUser.Wait()
	wg.Wait()

	prepLog, err := ioutil.ReadFile(tmpdir + "/prepare.log")
	if err != nil {
		log.Fatal(err)
	}

	var stats syscall.Rusage
	statsData, err := ioutil.ReadFile(tmpdir + "/stats.log")
	if err != nil {
		log.Print(err)
	} else {
		err = json.Unmarshal(statsData, &stats)
		if err != nil {
			log.Fatal(err)
		}
	}

	LogRunComplete(key, build, testToUser.String(), userToTest.String(), extraBuf.String(), exitErr, string(prepLog), stats)
}
<|endoftext|>


<commit_before>package piwik

import (
	"log"
	"net/http"
	"net/url"
	"strings"

	"gopkg.in/macaron.v1"
)

// Options configures the piwik middleware
type Options struct {
	// The URL of your piwik installation (with our without /piwik.php)
	PiwikURL string

	// Ignore the Do not Track header that is sent by the browser. This is not recommended
	IgnoreDoNotTrack bool

	// The ID of the website in piwik
	WebsiteID string

	// The piwik API's access token
	Token string
}

func prepareOptions(options []Options) Options {
	var opt Options
	if len(options) > 0 {
		opt = options[0]
	}
	opt.PiwikURL = strings.TrimSuffix(strings.TrimSuffix(opt.PiwikURL, "piwik.php"), "/") + "/piwik.php?"

	return opt
}

// Piwik returns a new macaron.Handler that sends every request to the piwik tracking API
func Piwik(options ...Options) macaron.Handler {
	opt := prepareOptions(options)

	return func(ctx *macaron.Context, logger *log.Logger) {
		h := ctx.Req.Header
		if !opt.IgnoreDoNotTrack && h.Get("DNT") == "1" {
			return
		}

		params := make(url.Values)
		params.Set("idsite", opt.WebsiteID)
		params.Set("rec", "1")

		proto := h.Get("X-Forwarded-Proto")
		if proto == "" {
			if ctx.Req.TLS != nil {
				proto = "https"
			} else {
				proto = "http"
			}
		}
		host := h.Get("X-Forwarded-Host")
		if host == "" {
			host = ctx.Req.Host
		}

		params.Set("url", proto+"://"+host+ctx.Req.URL.String())
		params.Set("apiv", "1")
		params.Set("urlref", h.Get("Referer"))
		params.Set("ua", h.Get("User-Agent"))
		params.Set("lang", h.Get("Accept-Language"))

		ip := ctx.RemoteAddr()
		if strings.Contains(ip, ",") {
			ipv6 := strings.Split(ip, ",")
			ip = strings.TrimPrefix(strings.TrimSpace(ipv6[0]), "::ffff:")
		}
		params.Set("token_auth", opt.Token)
		params.Set("cip", ip)

		// collecting data is finished, go async now
		go func() {
			res, err := http.Get(opt.PiwikURL + params.Encode())
			if err != nil {
				logger.Println("Error contacting piwik:", err)
			}
			if res.StatusCode != http.StatusOK {
				logger.Println("Error contacting piwik:", res.Status)
			}
		}()

		ctx.Next()
	}
}
<commit_msg>added custom options<commit_after>package piwik

import (
	"encoding/json"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"gopkg.in/macaron.v1"
)

// Options configures the piwik middleware
type Options struct {
	// The URL of your piwik installation (with our without /piwik.php)
	PiwikURL string

	// Ignore the Do not Track header that is sent by the browser. This is not recommended
	IgnoreDoNotTrack bool

	// The ID of the website in piwik
	WebsiteID string

	// The piwik API's access token
	Token string
}

// TrackingParams will be injected into the context and can be used to define
// piwik actions, like an internal search
type TrackingParams struct {
	// custom variables that will be assigned to the action (cvar)
	ActionCVar map[string]string

	// custom variables that will be assigned to the visitor (_cvar)
	VisitorCVar map[string]string

	// when this is set to true, no information will be sent to piwik
	Ignore bool

	params url.Values
}

// Search notifies piwik about a search done. Params can be:
// - the number of search results (int)
// - the catecory of the search (string)
func (t *TrackingParams) Search(keyword string, params ...interface{}) {
	t.params.Set("search", keyword)
	for _, x := range params {
		switch x.(type) {
		case int:
			t.params.Set("search_count", strconv.Itoa(x.(int)))
		case string:
			t.params.Set("search_cat", x.(string))
		}
	}
}

// Ignore provides a handler that will hinder all requests from being tracked
func Ignore(tracker *TrackingParams) {
	tracker.Ignore = true
}

func prepareOptions(options []Options) Options {
	var opt Options
	if len(options) > 0 {
		opt = options[0]
	}
	opt.PiwikURL = strings.TrimSuffix(strings.TrimSuffix(opt.PiwikURL, "piwik.php"), "/") + "/piwik.php?"

	return opt
}

// Piwik returns a new macaron.Handler that sends every request to the piwik tracking API
func Piwik(options ...Options) macaron.Handler {
	opt := prepareOptions(options)

	return func(ctx *macaron.Context, logger *log.Logger) {
		h := ctx.Req.Header
		if !opt.IgnoreDoNotTrack && h.Get("DNT") == "1" {
			ctx.Map(&TrackingParams{
				ActionCVar:  make(map[string]string),
				VisitorCVar: make(map[string]string),
				params:      make(url.Values),
				Ignore:      true,
			})
			ctx.Next()
			return
		}

		params := make(url.Values)
		params.Set("idsite", opt.WebsiteID)
		params.Set("rec", "1")

		proto := h.Get("X-Forwarded-Proto")
		if proto == "" {
			if ctx.Req.TLS != nil {
				proto = "https"
			} else {
				proto = "http"
			}
		}
		host := h.Get("X-Forwarded-Host")
		if host == "" {
			host = ctx.Req.Host
		}

		params.Set("url", proto+"://"+host+ctx.Req.URL.String())
		params.Set("apiv", "1")
		params.Set("urlref", h.Get("Referer"))
		params.Set("ua", h.Get("User-Agent"))
		params.Set("lang", h.Get("Accept-Language"))

		ip := ctx.RemoteAddr()
		if strings.Contains(ip, ",") {
			ipv6 := strings.Split(ip, ",")
			ip = strings.TrimPrefix(strings.TrimSpace(ipv6[0]), "::ffff:")
		}
		params.Set("token_auth", opt.Token)
		params.Set("cip", ip)

		p := &TrackingParams{
			ActionCVar:  make(map[string]string),
			VisitorCVar: make(map[string]string),
			params:      params,
		}
		ctx.Map(p)
		ctx.Next()

		if p.Ignore {
			return
		}

		if (len(p.ActionCVar)) > 0 {
			b, err := json.Marshal(p.ActionCVar)
			if err != nil {
				logger.Println("Error marshalling ActionCVar:", err)
			} else {
				params.Set("cvar", string(b))
			}
		}
		if (len(p.VisitorCVar)) > 0 {
			b, err := json.Marshal(p.VisitorCVar)
			if err != nil {
				logger.Println("Error marshalling VisitorCVar:", err)
			} else {
				params.Set("_cvar", string(b))
			}
		}

		// collecting data is finished, go async now
		go func() {
			res, err := http.Get(opt.PiwikURL + params.Encode())
			if err != nil {
				logger.Println("Error contacting piwik:", err)
			}
			if res.StatusCode != http.StatusOK {
				logger.Println("Error contacting piwik:", res.Status)
			}
		}()
	}
}
<|endoftext|>


<commit_before>package pkcs7 // import "fullsailor.com/pkcs7"
import (
	"crypto"
	"crypto/hmac"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/asn1"
	"errors"
	"fmt"
	"math/big"

	_ "crypto/sha1" // for crypto.SHA1
)

// PKCS7 Represents a PKCS7 structure
type PKCS7 struct {
	Content      []byte
	info         contentInfo
	Certificates []*x509.Certificate
	CRLs         []pkix.CertificateList
	Signers      []signerInfo
	raw          signedData
}

type contentInfo struct {
	ContentType asn1.ObjectIdentifier
	Content     asn1.RawValue `asn1:"explicit,optional"`
}

type unsignedData []byte

var (
	oidData                   = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 7, 1}
	oidSignedData             = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 7, 2}
	oidEnvelopedData          = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 7, 3}
	oidSignedAndEnvelopedData = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 7, 4}
	oidDigestedData           = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 7, 5}
	oidEncryptedData          = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 7, 6}
	oidAttributeContentType   = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 9, 3}
	oidAttributeMessageDigest = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 9, 4}
	oidAttributeSigningTime   = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 9, 5}
)

type signedData struct {
	Version                    int                        `asn1:"default:1"`
	DigestAlgorithmIdentifiers []pkix.AlgorithmIdentifier `asn1:"set"`
	ContentInfo                contentInfo
	Certificates               asn1.RawValue          `asn1:"optional"`
	CRLs                       []pkix.CertificateList `asn1:"optional"`
	SignerInfos                []signerInfo           `asn1:"set"`
}

type attribute struct {
	Type  asn1.ObjectIdentifier
	Value asn1.RawValue `asn1:"set"`
}

type issuerAndSerial struct {
	IssuerName   pkix.RDNSequence
	SerialNumber *big.Int
}

type signerInfo struct {
	Version                   int `asn1:"default:1"`
	IssuerAndSerialNumber     issuerAndSerial
	DigestAlgorithm           pkix.AlgorithmIdentifier
	AuthenticatedAttributes   []attribute `asn1:"optional,tag:0"`
	DigestEncryptionAlgorithm pkix.AlgorithmIdentifier
	EncryptedDigest           []byte
	UnauthenticatedAttributes []attribute `asn1:"optional,tag:1"`
}

// Parse decodes a DER encoded PKCS7 package
func Parse(data []byte) (p7 *PKCS7, err error) {
	var info contentInfo
	rest, err := asn1.Unmarshal(data, &info)
	if len(rest) > 0 {
		err = asn1.SyntaxError{Msg: "trailing data"}
		return
	}
	if err != nil {
		return
	}
	var sd signedData
	asn1.Unmarshal(info.Content.Bytes, &sd)
	certs, err := x509.ParseCertificates(sd.Certificates.Bytes)
	if err != nil {
		return
	}
	for _, crl := range sd.CRLs {
		fmt.Printf("CRLs: %v", crl)
	}
	var content unsignedData
	fmt.Printf("--> Signed Data Version %d\n", sd.Version)
	asn1.Unmarshal(sd.ContentInfo.Content.Bytes, &content)
	return &PKCS7{
		info:         info,
		Content:      content,
		Certificates: certs,
		CRLs:         sd.CRLs,
		Signers:      sd.SignerInfos,
		raw:          sd}, nil
}

type digestInfo struct {
	Algorithm pkix.AlgorithmIdentifier
	Digest    []byte
}

// Verify checks the signatures of a PKCS7 object
func (p7 *PKCS7) Verify() (err error) {
	if len(p7.Signers) == 0 {
		return errors.New("pkcs7: Message has no signers")
	}
	for _, signer := range p7.Signers {
		if err := verifySignature(p7, signer); err != nil {
			return err
		}
	}
	return nil
}

func verifySignature(p7 *PKCS7, signer signerInfo) error {
	fmt.Printf("--> Signer Info Version: %d\n", signer.Version)
	if len(signer.AuthenticatedAttributes) > 0 {

		// TODO(fullsailor): First check the content type match
		digest, err := getDigestFromAttributes(signer.AuthenticatedAttributes)
		if err != nil {
			return err
		}
		h := crypto.SHA1.New()
		h.Write(p7.Content)
		computed := h.Sum(nil)
		if !hmac.Equal(digest, computed) {
			return errors.New("pkcs7: Message digest mismatch")
		}
	}
	cert := getCertFromCertsByIssuerAndSerial(p7.Certificates, signer.IssuerAndSerialNumber)
	if cert == nil {
		return errors.New("pkcs7: No certificate for signer")
	}

	encodedAttributes, err := asn1.Marshal(struct {
		A []attribute `asn1:"set"`
	}{A: signer.AuthenticatedAttributes})
	if err != nil {
		return err
	}
	encodedAttributes = encodedAttributes[3:] // Remove the leading sequence octets
	/*
		fmt.Printf("--> asn.1 attributes %x\n", encodedAttributes)
		h.Write(encodedAttributes)
		messageDigest := h.Sum(nil)
		di := digestInfo{
			Algorithm: signer.DigestAlgorithm,
			Digest:    messageDigest,
		}
		fmt.Printf("--> digestInfo %+v\n", di)
		info, err := asn1.Marshal(di)
		if err != nil {
			return err
		}
		fmt.Printf("--> asn.1 digestInfo %x\n---> length:%d\n", info, len(info))
	*/
	algo := x509.SHA1WithRSA
	return cert.CheckSignature(algo, encodedAttributes, signer.EncryptedDigest)
}

var (
	oidDigestAlgorithmSHA1    = asn1.ObjectIdentifier{1, 3, 14, 3, 2, 26}
	oidEncryptionAlgorithmRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 1}
)

func getDigestFromAttributes(attributes []attribute) (digest []byte, err error) {
	for _, attr := range attributes {
		if attr.Type.Equal(oidAttributeMessageDigest) {
			_, err = asn1.Unmarshal(attr.Value.Bytes, &digest)
			return
		}
	}
	return nil, errors.New("pkcs7: Missing messageDigest attribute")
}

func getCertFromCertsByIssuerAndSerial(certs []*x509.Certificate, ias issuerAndSerial) *x509.Certificate {
	for _, cert := range certs {
		if cert.SerialNumber.Cmp(ias.SerialNumber) == 0 {
			// TODO(fullsailor): Compare Issuer Name & Cert Subject
			return cert
		}
	}
	return nil
}
<commit_msg>Handle compound octet string content.<commit_after>package pkcs7 // import "fullsailor.com/pkcs7"
import (
	"crypto"
	"crypto/hmac"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/asn1"
	"errors"
	"fmt"
	"math/big"

	_ "crypto/sha1" // for crypto.SHA1
)

// PKCS7 Represents a PKCS7 structure
type PKCS7 struct {
	Content      []byte
	info         contentInfo
	Certificates []*x509.Certificate
	CRLs         []pkix.CertificateList
	Signers      []signerInfo
	raw          signedData
}

type contentInfo struct {
	ContentType asn1.ObjectIdentifier
	Content     asn1.RawValue `asn1:"explicit,optional"`
}

type unsignedData []byte

var (
	oidData                   = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 7, 1}
	oidSignedData             = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 7, 2}
	oidEnvelopedData          = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 7, 3}
	oidSignedAndEnvelopedData = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 7, 4}
	oidDigestedData           = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 7, 5}
	oidEncryptedData          = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 7, 6}
	oidAttributeContentType   = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 9, 3}
	oidAttributeMessageDigest = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 9, 4}
	oidAttributeSigningTime   = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 9, 5}
)

type signedData struct {
	Version                    int                        `asn1:"default:1"`
	DigestAlgorithmIdentifiers []pkix.AlgorithmIdentifier `asn1:"set"`
	ContentInfo                contentInfo
	Certificates               asn1.RawValue          `asn1:"optional"`
	CRLs                       []pkix.CertificateList `asn1:"optional"`
	SignerInfos                []signerInfo           `asn1:"set"`
}

type attribute struct {
	Type  asn1.ObjectIdentifier
	Value asn1.RawValue `asn1:"set"`
}

type issuerAndSerial struct {
	IssuerName   pkix.RDNSequence
	SerialNumber *big.Int
}

type signerInfo struct {
	Version                   int `asn1:"default:1"`
	IssuerAndSerialNumber     issuerAndSerial
	DigestAlgorithm           pkix.AlgorithmIdentifier
	AuthenticatedAttributes   []attribute `asn1:"optional,tag:0"`
	DigestEncryptionAlgorithm pkix.AlgorithmIdentifier
	EncryptedDigest           []byte
	UnauthenticatedAttributes []attribute `asn1:"optional,tag:1"`
}

// Parse decodes a DER encoded PKCS7 package
func Parse(data []byte) (p7 *PKCS7, err error) {
	var info contentInfo
	rest, err := asn1.Unmarshal(data, &info)
	if len(rest) > 0 {
		err = asn1.SyntaxError{Msg: "trailing data"}
		return
	}
	if err != nil {
		return
	}
	var sd signedData
	asn1.Unmarshal(info.Content.Bytes, &sd)
	certs, err := x509.ParseCertificates(sd.Certificates.Bytes)
	if err != nil {
		return
	}
	for _, crl := range sd.CRLs {
		fmt.Printf("CRLs: %v", crl)
	}
	fmt.Printf("--> Signed Data Version %d\n", sd.Version)

	var contentBytes []byte
	if sd.ContentInfo.Content.IsCompound {
		var compound asn1.RawValue
		asn1.Unmarshal(sd.ContentInfo.Content.Bytes, &compound)
		contentBytes = compound.Bytes
	} else {
		contentBytes = sd.ContentInfo.Content.Bytes
	}
	var content unsignedData
	asn1.Unmarshal(contentBytes, &content)
	return &PKCS7{
		info:         info,
		Content:      content,
		Certificates: certs,
		CRLs:         sd.CRLs,
		Signers:      sd.SignerInfos,
		raw:          sd}, nil
}

type digestInfo struct {
	Algorithm pkix.AlgorithmIdentifier
	Digest    []byte
}

// Verify checks the signatures of a PKCS7 object
func (p7 *PKCS7) Verify() (err error) {
	if len(p7.Signers) == 0 {
		return errors.New("pkcs7: Message has no signers")
	}
	for _, signer := range p7.Signers {
		if err := verifySignature(p7, signer); err != nil {
			return err
		}
	}
	return nil
}

func verifySignature(p7 *PKCS7, signer signerInfo) error {
	fmt.Printf("--> Signer Info Version: %d\n", signer.Version)
	if len(signer.AuthenticatedAttributes) > 0 {

		// TODO(fullsailor): First check the content type match
		digest, err := getDigestFromAttributes(signer.AuthenticatedAttributes)
		if err != nil {
			return err
		}
		h := crypto.SHA1.New()
		h.Write(p7.Content)
		computed := h.Sum(nil)
		if !hmac.Equal(digest, computed) {
			return errors.New("pkcs7: Message digest mismatch")
		}
	}
	cert := getCertFromCertsByIssuerAndSerial(p7.Certificates, signer.IssuerAndSerialNumber)
	if cert == nil {
		return errors.New("pkcs7: No certificate for signer")
	}

	encodedAttributes, err := asn1.Marshal(struct {
		A []attribute `asn1:"set"`
	}{A: signer.AuthenticatedAttributes})
	if err != nil {
		return err
	}
	encodedAttributes = encodedAttributes[3:] // Remove the leading sequence octets
	/*
		fmt.Printf("--> asn.1 attributes %x\n", encodedAttributes)
		h.Write(encodedAttributes)
		messageDigest := h.Sum(nil)
		di := digestInfo{
			Algorithm: signer.DigestAlgorithm,
			Digest:    messageDigest,
		}
		fmt.Printf("--> digestInfo %+v\n", di)
		info, err := asn1.Marshal(di)
		if err != nil {
			return err
		}
		fmt.Printf("--> asn.1 digestInfo %x\n---> length:%d\n", info, len(info))
	*/
	algo := x509.SHA1WithRSA
	return cert.CheckSignature(algo, encodedAttributes, signer.EncryptedDigest)
}

var (
	oidDigestAlgorithmSHA1    = asn1.ObjectIdentifier{1, 3, 14, 3, 2, 26}
	oidEncryptionAlgorithmRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 1}
)

func getDigestFromAttributes(attributes []attribute) (digest []byte, err error) {
	for _, attr := range attributes {
		if attr.Type.Equal(oidAttributeMessageDigest) {
			_, err = asn1.Unmarshal(attr.Value.Bytes, &digest)
			return
		}
	}
	return nil, errors.New("pkcs7: Missing messageDigest attribute")
}

func getCertFromCertsByIssuerAndSerial(certs []*x509.Certificate, ias issuerAndSerial) *x509.Certificate {
	for _, cert := range certs {
		if cert.SerialNumber.Cmp(ias.SerialNumber) == 0 {
			// TODO(fullsailor): Compare Issuer Name & Cert Subject
			return cert
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/aquilax/tripcode"
	"github.com/gosimple/slug"
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
)

func hfTime(t time.Time) string {
	return t.Format("01.02.2006 15:04")
}

func hfSlug(s string) string {
	return slug.Make(s) + ".html"
}

func hfMod(n int, mod int) int {
	return n % mod
}

func getTripcode(s string) string {
	return tripcode.Tripcode(s)
}

func getVote(t string) int {
	if t == "y" {
		return 1
	}
	if t == "n" {
		return -1
	}
	return 0
}

func inHoneypot(t string) bool {
	if len(t) > 0 {
		return true
	}
	return false
}

func renderText(t string) string {
	extensions := 0
	extensions |= blackfriday.EXTENSION_HARD_LINE_BREAK
	extensions |= blackfriday.HTML_USE_XHTML
	extensions |= blackfriday.HTML_USE_SMARTYPANTS
	extensions |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
	extensions |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
	htmlFlags := 0 |
		blackfriday.HTML_USE_XHTML |
		blackfriday.HTML_USE_SMARTYPANTS |
		blackfriday.HTML_SMARTYPANTS_FRACTIONS

	renderer := blackfriday.HtmlRenderer(htmlFlags, "", "")
	unsafe := blackfriday.Markdown([]byte(t), renderer, extensions)
	return string(bluemonday.UGCPolicy().SanitizeBytes(unsafe))
}
<commit_msg>Bring back extensions<commit_after>package main

import (
	"time"

	"github.com/aquilax/tripcode"
	"github.com/gosimple/slug"
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
)

func hfTime(t time.Time) string {
	return t.Format("01.02.2006 15:04")
}

func hfSlug(s string) string {
	return slug.Make(s) + ".html"
}

func hfMod(n int, mod int) int {
	return n % mod
}

func getTripcode(s string) string {
	return tripcode.Tripcode(s)
}

func getVote(t string) int {
	if t == "y" {
		return 1
	}
	if t == "n" {
		return -1
	}
	return 0
}

func inHoneypot(t string) bool {
	if len(t) > 0 {
		return true
	}
	return false
}

func renderText(t string) string {
	extensions := 0
	extensions |= blackfriday.HTML_USE_XHTML
	extensions |= blackfriday.HTML_USE_SMARTYPANTS
	extensions |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
	extensions |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
	extensions |= blackfriday.EXTENSION_AUTOLINK
	extensions |= blackfriday.EXTENSION_HARD_LINE_BREAK
	extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
	extensions |= blackfriday.EXTENSION_TABLES
	extensions |= blackfriday.EXTENSION_FENCED_CODE
	extensions |= blackfriday.EXTENSION_STRIKETHROUGH
	extensions |= blackfriday.EXTENSION_SPACE_HEADERS
	extensions |= blackfriday.EXTENSION_HEADER_IDS

	htmlFlags := 0 |
		blackfriday.HTML_USE_XHTML |
		blackfriday.HTML_USE_SMARTYPANTS |
		blackfriday.HTML_SMARTYPANTS_FRACTIONS

	renderer := blackfriday.HtmlRenderer(htmlFlags, "", "")
	unsafe := blackfriday.Markdown([]byte(t), renderer, extensions)
	return string(bluemonday.UGCPolicy().SanitizeBytes(unsafe))
}
<|endoftext|>


<commit_before>package cerebrum

import (
	"errors"
	"fmt"
	"net"
	"strconv"
	"strings"

	"github.com/hashicorp/serf/serf"
)

// GetFullEventName computes the full name of a event
func GetFullEventName(name string) string {
	return CerebrumEventPrefix + name
}

// IsCerebrumEvent checks if a serf event is a cerebrum event
func IsCerebrumEvent(name string) bool {
	return strings.HasPrefix(name, CerebrumEventPrefix)
}

// GetRawEventName is used to get the raw event name
func GetRawEventName(name string) string {
	return strings.TrimPrefix(name, CerebrumEventPrefix)
}

// ValidateNode determines whether a node is a known server and returns
//  its data center and role.
func ValidateNode(member serf.Member) (ok bool, role, dc string) {
	if _, ok = member.Tags["id"]; !ok {
		return false, "", ""
	}

	// Get role name
	if role, ok = member.Tags["role"]; !ok {
		return false, "", ""
	} else if role != CerebrumRole {
		return false, "", ""
	}

	// Get datacenter name
	if dc, ok = member.Tags["dc"]; ok {
		return true, role, dc
	}
	return false, "", ""
}

// GetNodeDetails should validate all the Serf tags for the given member and return
// NodeDetails or any error that occurred.
func GetNodeDetails(m serf.Member) (n *NodeDetails, err error) {

	// Validate server node
	ok, role, dc := ValidateNode(m)
	if !ok {
		return nil, errors.New("Invalid server node")
	}

	// Get services
	// Services are meant to be encoded in this format: svc-1:port;svc-2:port
	services := make([]NodeService, 0)
	if svcs, ok := m.Tags["services"]; ok {
		for _, svc := range strings.Split(svcs, ";") {
			if strings.Contains(svc, ":") {
				attrs := strings.Split(svc, ":")
				if len(attrs) >= 2 {

					// Convert port to int
					p, e := strconv.Atoi(attrs[1])
					if e != nil {
						err = fmt.Errorf("service port cannot be converted to string: '%s'", p)
						return
					}

					services = append(services, NodeService{
						Name: attrs[0],
						Port: p,
					})
				} else {
					return nil, fmt.Errorf("Invalid service attributes: '%s'", svc)
				}
			} else {
				return nil, fmt.Errorf("Invalid service attributes: '%s'", svc)
			}
		}
	}

	// All nodes which have this tag are bootstrapped
	_, bootstrap := m.Tags["bootstrap"]

	n = &NodeDetails{
		Bootstrap:  bootstrap,
		ID:         m.Tags["id"],
		Name:       m.Name,
		Role:       role,
		DataCenter: dc,
		Addr:       m.Addr,
		Services:   services,
		Status:     m.Status,
		Tags:       m.Tags,
	}
	return
}

// NodeDetails stores details about a single serf.Member
type NodeDetails struct {
	Bootstrap  bool
	ID         string
	Name       string
	Role       string
	DataCenter string
	Addr       net.IP
	Services   []NodeService
	Tags       map[string]string
	Status     serf.MemberStatus
}

func (n NodeDetails) String() (s string) {
	// NodeDetails{Name: "somename", Role: "role", Cluster: "cluster", Addr: "127.0.0.1:9000"}
	if b, err := n.Addr.MarshalText(); err != nil {
		s = fmt.Sprintf("NodeDetails{Name: \"%s\", Role: \"%s\", DataCenter: \"%s\"}", n.Name, n.Role, n.DataCenter)
	} else {
		s = fmt.Sprintf("NodeDetails{Name: \"%s\", Role: \"%s\", DataCenter: \"%s\", Addr: \"%s:%d\"}", n.Name, n.Role, n.DataCenter, string(b))
	}
	return
}

type NodeService struct {
	Name string
	Port int
}
<commit_msg>Add Port to NodeDetails struct and update string method<commit_after>package cerebrum

import (
	"errors"
	"fmt"
	"net"
	"strconv"
	"strings"

	"github.com/hashicorp/serf/serf"
)

// GetFullEventName computes the full name of a event
func GetFullEventName(name string) string {
	return CerebrumEventPrefix + name
}

// IsCerebrumEvent checks if a serf event is a cerebrum event
func IsCerebrumEvent(name string) bool {
	return strings.HasPrefix(name, CerebrumEventPrefix)
}

// GetRawEventName is used to get the raw event name
func GetRawEventName(name string) string {
	return strings.TrimPrefix(name, CerebrumEventPrefix)
}

// ValidateNode determines whether a node is a known server and returns
//  its data center and role.
func ValidateNode(member serf.Member) (ok bool, role, dc string) {
	if _, ok = member.Tags["id"]; !ok {
		return false, "", ""
	}

	// Get role name
	if role, ok = member.Tags["role"]; !ok {
		return false, "", ""
	} else if role != CerebrumRole {
		return false, "", ""
	}

	// Get datacenter name
	if dc, ok = member.Tags["dc"]; ok {
		return true, role, dc
	}
	return false, "", ""
}

// GetNodeDetails should validate all the Serf tags for the given member and return
// NodeDetails or any error that occurred.
func GetNodeDetails(m serf.Member) (n *NodeDetails, err error) {

	// Validate server node
	ok, role, dc := ValidateNode(m)
	if !ok {
		return nil, errors.New("Invalid server node")
	}

	// Get services
	// Services are meant to be encoded in this format: svc-1:port;svc-2:port
	services := make([]NodeService, 0)
	if svcs, ok := m.Tags["services"]; ok {
		for _, svc := range strings.Split(svcs, ";") {
			if strings.Contains(svc, ":") {
				attrs := strings.Split(svc, ":")
				if len(attrs) >= 2 {

					// Convert port to int
					p, e := strconv.Atoi(attrs[1])
					if e != nil {
						err = fmt.Errorf("service port cannot be converted to string: '%s'", p)
						return
					}

					services = append(services, NodeService{
						Name: attrs[0],
						Port: p,
					})
				} else {
					return nil, fmt.Errorf("Invalid service attributes: '%s'", svc)
				}
			} else {
				return nil, fmt.Errorf("Invalid service attributes: '%s'", svc)
			}
		}
	}

	// All nodes which have this tag are bootstrapped
	_, bootstrap := m.Tags["bootstrap"]

	n = &NodeDetails{
		Bootstrap:  bootstrap,
		ID:         m.Tags["id"],
		Name:       m.Name,
		Role:       role,
		DataCenter: dc,
		Addr:       m.Addr,
		Services:   services,
		Status:     m.Status,
		Tags:       m.Tags,
	}
	return
}

// NodeDetails stores details about a single serf.Member
type NodeDetails struct {
	Bootstrap  bool
	ID         string
	Name       string
	Role       string
	DataCenter string
	Addr       net.IP
	Port       int
	Services   []NodeService
	Tags       map[string]string
	Status     serf.MemberStatus
}

func (n NodeDetails) String() (s string) {
	// NodeDetails{Name: "somename", Role: "role", Cluster: "cluster", Addr: "127.0.0.1:9000"}
	if b, err := n.Addr.MarshalText(); err != nil {
		s = fmt.Sprintf("NodeDetails{Name: \"%s\", Role: \"%s\", DataCenter: \"%s\"}", n.Name, n.Role, n.DataCenter)
	} else {
		s = fmt.Sprintf("NodeDetails{Name: \"%s\", Role: \"%s\", DataCenter: \"%s\", Addr: \"%s:%d\"}", n.Name, n.Role, n.DataCenter, string(b), n.Port)
	}
	return
}

type NodeService struct {
	Name string
	Port int
}
<|endoftext|>


<commit_before>package irelate

import (
	"bufio"
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/brentp/bix"
	"github.com/brentp/irelate/interfaces"
	"github.com/brentp/xopen"
)

// OpenScanFile sets up a (possibly gzipped) file for line-wise reading.
func OpenScanFile(fh io.Reader) (*bufio.Scanner, io.Reader) {
	scanner := bufio.NewScanner(fh)
	scanner.Split(bufio.ScanLines)
	return scanner, fh
}

// ScanToRelatable makes is easy to create a chan Relatable from a file of intervals.
func ScanToRelatable(file io.Reader, fn func(line string) (interfaces.Relatable, error)) RelatableChannel {
	scanner, fh := OpenScanFile(file)
	ch := make(chan interfaces.Relatable, 32)
	go func() {
		i := 0
		for scanner.Scan() {
			v, err := fn(scanner.Text())
			if err != nil {
				if i > 0 { // break on the header.
					log.Println(err)
					break
				}
			} else {
				ch <- v
				i += 1
			}
		}
		if c, ok := fh.(io.ReadCloser); ok {
			c.Close()
		}
		close(ch)
	}()
	return ch
}

func Imin(a uint32, b uint32) uint32 {
	if b < a {
		return b
	}
	return a
}

func Imax(a uint32, b uint32) uint32 {
	if b > a {
		return b
	}
	return a
}

func RegionToParts(region string) (string, int, int, error) {
	parts := strings.Split(region, ":")
	se := strings.Split(parts[1], "-")
	if len(se) != 2 {
		return "", 0, 0, errors.New(fmt.Sprintf("unable to parse region: %s", region))
	}
	s, err := strconv.Atoi(se[0])
	if err != nil {
		return "", 0, 0, errors.New(fmt.Sprintf("unable to parse region: %s", region))
	}
	e, err := strconv.Atoi(se[1])
	if err != nil {
		return "", 0, 0, errors.New(fmt.Sprintf("unable to parse region: %s", region))
	}
	return parts[0], s, e, nil
}

func Streamer(f string, region string) (RelatableChannel, error) {
	var stream chan interfaces.Relatable
	var err error

	var rdr io.Reader
	var bx *bix.Bix
	if region != "" {
		bx, err = bix.New(f)
		if err != nil {
			return nil, err
		}
		chrom, start, end, err := RegionToParts(region)
		if err != nil {
			return nil, err
		}
		rdr, err = bx.Query(chrom, start, end, true)
	} else {
		rdr, err = os.Open(f)
	}
	if err != nil {
		return nil, err
	}
	var buf io.Reader
	if !strings.HasSuffix(f, ".bam") {
		bufr := bufio.NewReader(rdr)
		if is, err := xopen.IsGzip(bufr); is {
			buf, err = gzip.NewReader(bufr)
			if err != nil {
				return nil, err
			}
		} else {
			buf = bufr
		}
	} else {
		buf = rdr
	}

	if strings.HasSuffix(f, ".bam") {
		stream, err = BamToRelatable(buf)
	} else if strings.HasSuffix(f, ".gff") {
		stream, err = GFFToRelatable(buf)
	} else if strings.HasSuffix(f, ".vcf") || strings.HasSuffix(f, ".vcf.gz") {
		v := Vopen(buf, nil)
		stream = StreamVCF(v)
	} else {
		stream = ScanToRelatable(buf, IntervalFromBedLine)
	}
	return stream, err
}
<commit_msg>simplify<commit_after>package irelate

import (
	"bufio"
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/brentp/bix"
	"github.com/brentp/irelate/interfaces"
	"github.com/brentp/xopen"
)

// OpenScanFile sets up a (possibly gzipped) file for line-wise reading.
func OpenScanFile(fh io.Reader) (*bufio.Scanner, io.Reader) {
	scanner := bufio.NewScanner(fh)
	scanner.Split(bufio.ScanLines)
	return scanner, fh
}

// ScanToRelatable makes is easy to create a chan Relatable from a file of intervals.
func ScanToRelatable(file io.Reader, fn func(line string) (interfaces.Relatable, error)) RelatableChannel {
	scanner, fh := OpenScanFile(file)
	ch := make(chan interfaces.Relatable, 32)
	go func() {
		i := 0
		for scanner.Scan() {
			v, err := fn(scanner.Text())
			if err != nil {
				if i > 0 { // break on the header.
					log.Println(err)
					break
				}
			} else {
				ch <- v
				i += 1
			}
		}
		if c, ok := fh.(io.ReadCloser); ok {
			c.Close()
		}
		close(ch)
	}()
	return ch
}

func Imin(a uint32, b uint32) uint32 {
	if b < a {
		return b
	}
	return a
}

func Imax(a uint32, b uint32) uint32 {
	if b > a {
		return b
	}
	return a
}

func RegionToParts(region string) (string, int, int, error) {
	parts := strings.Split(region, ":")
	se := strings.Split(parts[1], "-")
	if len(se) != 2 {
		return "", 0, 0, errors.New(fmt.Sprintf("unable to parse region: %s", region))
	}
	s, err := strconv.Atoi(se[0])
	if err != nil {
		return "", 0, 0, errors.New(fmt.Sprintf("unable to parse region: %s", region))
	}
	e, err := strconv.Atoi(se[1])
	if err != nil {
		return "", 0, 0, errors.New(fmt.Sprintf("unable to parse region: %s", region))
	}
	return parts[0], s, e, nil
}

func Streamer(f string, region string) (RelatableChannel, error) {
	var stream chan interfaces.Relatable
	var err error

	var rdr io.Reader
	if region != "" {
		bx, err := bix.New(f)
		if err != nil {
			return nil, err
		}
		chrom, start, end, err := RegionToParts(region)
		if err != nil {
			return nil, err
		}
		rdr, err = bx.Query(chrom, start, end, true)
	} else {
		rdr, err = os.Open(f)
	}
	if err != nil {
		return nil, err
	}
	var buf io.Reader
	if !strings.HasSuffix(f, ".bam") {
		bufr := bufio.NewReader(rdr)
		if is, err := xopen.IsGzip(bufr); is {
			buf, err = gzip.NewReader(bufr)
			if err != nil {
				return nil, err
			}
		} else {
			buf = bufr
		}
	} else {
		buf = rdr
	}

	if strings.HasSuffix(f, ".bam") {
		stream, err = BamToRelatable(buf)
	} else if strings.HasSuffix(f, ".gff") {
		stream, err = GFFToRelatable(buf)
	} else if strings.HasSuffix(f, ".vcf") || strings.HasSuffix(f, ".vcf.gz") {
		v := Vopen(buf, nil)
		stream = StreamVCF(v)
	} else {
		stream = ScanToRelatable(buf, IntervalFromBedLine)
	}
	return stream, err
}
<|endoftext|>