<commit_before>package log

import (
	"encoding"
	"fmt"
	"sort"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

// PlainFormat implements Formatter to generate plain log messages.
//
// A plain log message looks like:
// DATETIME SEVERITY UTSNAME TOPIC MESSAGE [OPTIONAL FIELDS...]
type PlainFormat struct {
	// Utsname can normally be left blank.
	// If not empty, the string is used instead of the hostname.
	// Utsname must match this regexp: ^[a-z][a-z0-9-]*$
	Utsname string
}

// String returns "plain".
func (f PlainFormat) String() string {
	return "plain"
}

// Format implements Formatter.Format.
func (f PlainFormat) Format(buf []byte, l *Logger, t time.Time, severity int,
	msg string, fields map[string]interface{}) ([]byte, error) {
	var err error

	// assume enough capacity for mandatory fields (except for msg).
	buf = t.UTC().AppendFormat(buf, RFC3339Micro)
	buf = append(buf, ' ')
	if len(f.Utsname) > 0 {
		buf = append(buf, f.Utsname...)
	} else {
		buf = append(buf, utsname...)
	}
	buf = append(buf, ' ')
	buf = append(buf, l.Topic()...)
	buf = append(buf, ' ')
	if ss, ok := severityMap[severity]; ok {
		buf = append(buf, ss...)
	} else {
		buf = strconv.AppendInt(buf, int64(severity), 10)
	}
	buf = append(buf, ": "...)
	buf, err = appendPlain(buf, msg)
	if err != nil {
		return nil, err
	}

	if len(fields) > 0 {
		keys := make([]string, 0, len(fields))
		for k := range fields {
			if !IsValidKey(k) {
				return nil, ErrInvalidKey
			}
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for _, k := range keys {
			if cap(buf) < (len(k) + 2) {
				return nil, ErrTooLarge
			}
			buf = append(buf, ' ')
			buf = append(buf, k...)
			buf = append(buf, '=')
			buf, err = appendPlain(buf, fields[k])
			if err != nil {
				return nil, err
			}
		}
	}

	for k, v := range l.Defaults() {
		if _, ok := fields[k]; ok {
			continue
		}
		if cap(buf) < (len(k) + 2) {
			return nil, ErrTooLarge
		}
		buf = append(buf, ' ')
		buf = append(buf, k...)
		buf = append(buf, '=')
		buf, err = appendPlain(buf, v)
		if err != nil {
			return nil, err
		}
	}

	if cap(buf) < 1 {
		return nil, ErrTooLarge
	}
	return append(buf, '\n'), nil
}

func appendPlain(buf []byte, v interface{}) ([]byte, error) {
	switch t := v.(type) {
	case nil:
		if cap(buf) < 4 {
			return nil, ErrTooLarge
		}
		return append(buf, "null"...), nil
	case bool:
		if cap(buf) < 5 { // len("false")
			return nil, ErrTooLarge
		}
		return strconv.AppendBool(buf, t), nil
	case time.Time:
		// len("2006-01-02T15:04:05.000000Z07:00")
		if cap(buf) < 32 {
			return nil, ErrTooLarge
		}
		return t.UTC().AppendFormat(buf, RFC3339Micro), nil
	case int:
		// len("-9223372036854775807")
		if cap(buf) < 20 {
			return nil, ErrTooLarge
		}
		return strconv.AppendInt(buf, int64(t), 10), nil
	case int8:
		// len("-128")
		if cap(buf) < 4 {
			return nil, ErrTooLarge
		}
		return strconv.AppendInt(buf, int64(t), 10), nil
	case int16:
		// len("-32768")
		if cap(buf) < 6 {
			return nil, ErrTooLarge
		}
		return strconv.AppendInt(buf, int64(t), 10), nil
	case int32:
		// len("-2147483648")
		if cap(buf) < 11 {
			return nil, ErrTooLarge
		}
		return strconv.AppendInt(buf, int64(t), 10), nil
	case int64:
		// len("-9223372036854775807")
		if cap(buf) < 20 {
			return nil, ErrTooLarge
		}
		return strconv.AppendInt(buf, t, 10), nil
	case uint:
		// len("18446744073709551615")
		if cap(buf) < 20 {
			return nil, ErrTooLarge
		}
		return strconv.AppendUint(buf, uint64(t), 10), nil
	case uint8:
		// len("255")
		if cap(buf) < 3 {
			return nil, ErrTooLarge
		}
		return strconv.AppendUint(buf, uint64(t), 10), nil
	case uint16:
		// len("65535")
		if cap(buf) < 5 {
			return nil, ErrTooLarge
		}
		return strconv.AppendUint(buf, uint64(t), 10), nil
	case uint32:
		// len("4294967295")
		if cap(buf) < 10 {
			return nil, ErrTooLarge
		}
		return strconv.AppendUint(buf, uint64(t), 10), nil
	case uint64:
		// len("18446744073709551615")
		if cap(buf) < 20 {
			return nil, ErrTooLarge
		}
		return strconv.AppendUint(buf, t, 10), nil
	case float32:
		if cap(buf) < 256 {
			return nil, ErrTooLarge
		}
		return strconv.AppendFloat(buf, float64(t), 'f', -1, 32), nil
	case float64:
		if cap(buf) < 256 {
			return nil, ErrTooLarge
		}
		return strconv.AppendFloat(buf, t, 'f', -1, 64), nil
	case string:
		if !utf8.ValidString(t) {
			// the next line replaces invalid characters.
			t = strings.ToValidUTF8(t, " ")
		}
		// escaped length = 2*len(t) + 2 double quotes
		if cap(buf) < (len(t)*2 + 2) {
			return nil, ErrTooLarge
		}
		return strconv.AppendQuote(buf, t), nil
	case encoding.TextMarshaler:
		// TextMarshaler encodes into UTF-8 string.
		s, err := t.MarshalText()
		if err != nil {
			return nil, err
		}
		if cap(buf) < (len(s)*2 + 2) {
			return nil, ErrTooLarge
		}
		return strconv.AppendQuote(buf, string(s)), nil
	case error:
		s := t.Error()
		if !utf8.ValidString(s) {
			// the next line replaces invalid characters.
			s = strings.ToValidUTF8(s, " ")
		}
		// escaped length = 2*len(s) + 2 double quotes
		if cap(buf) < (len(s)*2 + 2) {
			return nil, ErrTooLarge
		}
		return strconv.AppendQuote(buf, s), nil
	default:
		// other types are just formatted as string with "%v".
		return appendPlain(buf, fmt.Sprintf("%v", t))
	}
}
<commit_msg>replace invalid UTF8 character with utf8.RuneError<commit_after>package log

import (
	"encoding"
	"fmt"
	"sort"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

// PlainFormat implements Formatter to generate plain log messages.
//
// A plain log message looks like:
// DATETIME SEVERITY UTSNAME TOPIC MESSAGE [OPTIONAL FIELDS...]
type PlainFormat struct {
	// Utsname can normally be left blank.
	// If not empty, the string is used instead of the hostname.
	// Utsname must match this regexp: ^[a-z][a-z0-9-]*$
	Utsname string
}

// String returns "plain".
func (f PlainFormat) String() string {
	return "plain"
}

// Format implements Formatter.Format.
func (f PlainFormat) Format(buf []byte, l *Logger, t time.Time, severity int,
	msg string, fields map[string]interface{}) ([]byte, error) {
	var err error

	// assume enough capacity for mandatory fields (except for msg).
	buf = t.UTC().AppendFormat(buf, RFC3339Micro)
	buf = append(buf, ' ')
	if len(f.Utsname) > 0 {
		buf = append(buf, f.Utsname...)
	} else {
		buf = append(buf, utsname...)
	}
	buf = append(buf, ' ')
	buf = append(buf, l.Topic()...)
	buf = append(buf, ' ')
	if ss, ok := severityMap[severity]; ok {
		buf = append(buf, ss...)
	} else {
		buf = strconv.AppendInt(buf, int64(severity), 10)
	}
	buf = append(buf, ": "...)
	buf, err = appendPlain(buf, msg)
	if err != nil {
		return nil, err
	}

	if len(fields) > 0 {
		keys := make([]string, 0, len(fields))
		for k := range fields {
			if !IsValidKey(k) {
				return nil, ErrInvalidKey
			}
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for _, k := range keys {
			if cap(buf) < (len(k) + 2) {
				return nil, ErrTooLarge
			}
			buf = append(buf, ' ')
			buf = append(buf, k...)
			buf = append(buf, '=')
			buf, err = appendPlain(buf, fields[k])
			if err != nil {
				return nil, err
			}
		}
	}

	for k, v := range l.Defaults() {
		if _, ok := fields[k]; ok {
			continue
		}
		if cap(buf) < (len(k) + 2) {
			return nil, ErrTooLarge
		}
		buf = append(buf, ' ')
		buf = append(buf, k...)
		buf = append(buf, '=')
		buf, err = appendPlain(buf, v)
		if err != nil {
			return nil, err
		}
	}

	if cap(buf) < 1 {
		return nil, ErrTooLarge
	}
	return append(buf, '\n'), nil
}

func appendPlain(buf []byte, v interface{}) ([]byte, error) {
	switch t := v.(type) {
	case nil:
		if cap(buf) < 4 {
			return nil, ErrTooLarge
		}
		return append(buf, "null"...), nil
	case bool:
		if cap(buf) < 5 { // len("false")
			return nil, ErrTooLarge
		}
		return strconv.AppendBool(buf, t), nil
	case time.Time:
		// len("2006-01-02T15:04:05.000000Z07:00")
		if cap(buf) < 32 {
			return nil, ErrTooLarge
		}
		return t.UTC().AppendFormat(buf, RFC3339Micro), nil
	case int:
		// len("-9223372036854775807")
		if cap(buf) < 20 {
			return nil, ErrTooLarge
		}
		return strconv.AppendInt(buf, int64(t), 10), nil
	case int8:
		// len("-128")
		if cap(buf) < 4 {
			return nil, ErrTooLarge
		}
		return strconv.AppendInt(buf, int64(t), 10), nil
	case int16:
		// len("-32768")
		if cap(buf) < 6 {
			return nil, ErrTooLarge
		}
		return strconv.AppendInt(buf, int64(t), 10), nil
	case int32:
		// len("-2147483648")
		if cap(buf) < 11 {
			return nil, ErrTooLarge
		}
		return strconv.AppendInt(buf, int64(t), 10), nil
	case int64:
		// len("-9223372036854775807")
		if cap(buf) < 20 {
			return nil, ErrTooLarge
		}
		return strconv.AppendInt(buf, t, 10), nil
	case uint:
		// len("18446744073709551615")
		if cap(buf) < 20 {
			return nil, ErrTooLarge
		}
		return strconv.AppendUint(buf, uint64(t), 10), nil
	case uint8:
		// len("255")
		if cap(buf) < 3 {
			return nil, ErrTooLarge
		}
		return strconv.AppendUint(buf, uint64(t), 10), nil
	case uint16:
		// len("65535")
		if cap(buf) < 5 {
			return nil, ErrTooLarge
		}
		return strconv.AppendUint(buf, uint64(t), 10), nil
	case uint32:
		// len("4294967295")
		if cap(buf) < 10 {
			return nil, ErrTooLarge
		}
		return strconv.AppendUint(buf, uint64(t), 10), nil
	case uint64:
		// len("18446744073709551615")
		if cap(buf) < 20 {
			return nil, ErrTooLarge
		}
		return strconv.AppendUint(buf, t, 10), nil
	case float32:
		if cap(buf) < 256 {
			return nil, ErrTooLarge
		}
		return strconv.AppendFloat(buf, float64(t), 'f', -1, 32), nil
	case float64:
		if cap(buf) < 256 {
			return nil, ErrTooLarge
		}
		return strconv.AppendFloat(buf, t, 'f', -1, 64), nil
	case string:
		if !utf8.ValidString(t) {
			// the next line replaces invalid characters.
			t = strings.ToValidUTF8(t, string(utf8.RuneError))
		}
		// escaped length = 2*len(t) + 2 double quotes
		if cap(buf) < (len(t)*2 + 2) {
			return nil, ErrTooLarge
		}
		return strconv.AppendQuote(buf, t), nil
	case encoding.TextMarshaler:
		// TextMarshaler encodes into UTF-8 string.
		s, err := t.MarshalText()
		if err != nil {
			return nil, err
		}
		if cap(buf) < (len(s)*2 + 2) {
			return nil, ErrTooLarge
		}
		return strconv.AppendQuote(buf, string(s)), nil
	case error:
		s := t.Error()
		if !utf8.ValidString(s) {
			// the next line replaces invalid characters.
			s = strings.ToValidUTF8(s, string(utf8.RuneError))
		}
		// escaped length = 2*len(s) + 2 double quotes
		if cap(buf) < (len(s)*2 + 2) {
			return nil, ErrTooLarge
		}
		return strconv.AppendQuote(buf, s), nil
	default:
		// other types are just formatted as string with "%v".
		return appendPlain(buf, fmt.Sprintf("%v", t))
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"os"
)

var version = "v1.3"

var optionDocs = map[string]string{
	"-d": "Show debug message",
}

var cmds = []string{
	"account",
	"dircache",
	"listbucket",
	"alilistbucket",
	"prefop",
	"fput",
	"rput",
	"qupload",
	"stat",
	"delete",
	"move",
	"copy",
	"chgm",
	"fetch",
	"prefetch",
	"batchdelete",
	"checkqrsync",
	"b64encode",
	"b64decode",
	"urlencode",
	"urldecode",
	"ts2d",
	"tms2d",
	"tns2d",
	"d2ts",
	"ip",
}
var cmdDocs = map[string][]string{
	"account":       []string{"qshell [-d] account [<AccessKey> <SecretKey>]", "Get/Set AccessKey and SecretKey"},
	"dircache":      []string{"qshell [-d] dircache <DirCacheRootPath> <DirCacheResultFile>", "Cache the directory structure of a file path"},
	"listbucket":    []string{"qshell [-d] listbucket <Bucket> [<Prefix>] <ListBucketResultFile>", "List all the file in the bucket by prefix"},
	"alilistbucket": []string{"qshell [-d] alilistbucket <DataCenter> <Bucket> <AccessKeyId> <AccesskeySecret> [Prefix] <ListBucketResultFile>", "List all the file in the bucket of aliyun oss by prefix"},
	"prefop":        []string{"qshell [-d] prefop <PersistentId>", "Query the fop status"},
	"fput":          []string{"qshell [-d] fput <Bucket> <Key> <LocalFile> [MimeType]", "Form upload a local file"},
	"rput":          []string{"qshell [-d] rput <Bucket> <Key> <LocalFile> [MimeType]", "Resumable upload a local file"},
	"qupload":       []string{"qshell [-d] qupload [<PutThresoldInBytes>] <LocalUploadConfig>", "Batch upload files to the qiniu bucket"},
	"qdownload":     []string{"qshell [-d] qdownload [<ThreadCount>] <LocalDownloadConfig>"},
	"stat":          []string{"qshell [-d] stat <Bucket> <Key>", "Get the basic info of a remote file"},
	"delete":        []string{"qshell [-d] delete <Bucket> <Key>", "Delete a remote file in the bucket"},
	"move":          []string{"qshell [-d] move <SrcBucket> <SrcKey> <DestBucket> <DestKey>", "Move/Rename a file and save in bucket"},
	"copy":          []string{"qshell [-d] copy <SrcBucket> <SrcKey> <DestBucket> [<DestKey>]", "Make a copy of a file and save in bucket"},
	"chgm":          []string{"qshell [-d] chgm <Bucket> <Key> <NewMimeType>", "Change the mimeType of a file"},
	"fetch":         []string{"qshell [-d] fetch <RemoteResourceUrl> <Bucket> <Key>", "Fetch a remote resource by url and save in bucket"},
	"prefetch":      []string{"qshell [-d] prefetch <Bucket> <Key>", "Fetch and update the file in bucket using mirror storage"},
	"batchdelete":   []string{"qshell [-d] batchdelete <Bucket> <KeyListFile>", "Batch delete files in bucket"},
	"checkqrsync":   []string{"qshell [-d] checkqrsync <DirCacheResultFile> <ListBucketResultFile> <IgnoreLocalDir> [Prefix]", "Check the qrsync result"},
	"b64encode":     []string{"qshell [-d] b64encode [<UrlSafe>] <DataToEncode>", "Base64 Encode"},
	"b64decode":     []string{"qshell [-d] b64decode [<UrlSafe>] <DataToDecode>", "Base64 Decode"},
	"urlencode":     []string{"qshell [-d] urlencode <DataToEncode>", "Url encode"},
	"urldecode":     []string{"qshell [-d] urldecode <DataToDecode>", "Url decode"},
	"ts2d":          []string{"qshell [-d] ts2d <TimestampInSeconds>", "Convert timestamp in seconds to a date (TZ: Local)"},
	"tms2d":         []string{"qshell [-d] tms2d <TimestampInMilliSeconds>", "Convert timestamp in milli-seconds to a date (TZ: Local)"},
	"tns2d":         []string{"qshell [-d] tns2d <TimestampIn100NanoSeconds>", "Convert timestamp in 100 nano-seconds to a date (TZ: Local)"},
	"d2ts":          []string{"qshell [-d] d2ts <SecondsToNow>", "Create a timestamp in seconds using seconds to now"},
	"ip":            []string{"qshell [-d] ip <Ip1> [<Ip2> [<Ip3> ...]]]", "Query the ip information"},
}

func Help(cmd string, params ...string) {
	if len(params) == 0 {
		fmt.Println(CmdList())
	} else {
		CmdHelps(params...)
	}
}

func CmdList() string {
	helpAll := fmt.Sprintf("QShell %s\r\n\r\n", version)
	helpAll += "Options:\r\n"
	for k, v := range optionDocs {
		helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", k, v)
	}
	helpAll += "\r\n"
	helpAll += "Commands:\r\n"
	for _, cmd := range cmds {
		if help, ok := cmdDocs[cmd]; ok {
			cmdDesc := help[1]
			helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", cmd, cmdDesc)
		}
	}
	return helpAll
}

func CmdHelps(cmds ...string) {
	defer os.Exit(1)
	if len(cmds) == 0 {
		fmt.Println(CmdList())
	} else {
		for _, cmd := range cmds {
			CmdHelp(cmd)
		}
	}
}

func CmdHelp(cmd string) {
	docStr := fmt.Sprintf("Unknow cmd `%s'", cmd)
	if cmdDoc, ok := cmdDocs[cmd]; ok {
		docStr = fmt.Sprintf("Usage: %s\r\n  %s\r\n", cmdDoc[0], cmdDoc[1])
	}
	fmt.Println(docStr)
}
<commit_msg>Fix help bug.<commit_after>package cli

import (
	"fmt"
	"os"
)

var version = "v1.3"

var optionDocs = map[string]string{
	"-d": "Show debug message",
}

var cmds = []string{
	"account",
	"dircache",
	"listbucket",
	"alilistbucket",
	"prefop",
	"fput",
	"rput",
	"qupload",
	"qdownload",
	"stat",
	"delete",
	"move",
	"copy",
	"chgm",
	"fetch",
	"prefetch",
	"batchdelete",
	"checkqrsync",
	"b64encode",
	"b64decode",
	"urlencode",
	"urldecode",
	"ts2d",
	"tms2d",
	"tns2d",
	"d2ts",
	"ip",
}
var cmdDocs = map[string][]string{
	"account":       []string{"qshell [-d] account [<AccessKey> <SecretKey>]", "Get/Set AccessKey and SecretKey"},
	"dircache":      []string{"qshell [-d] dircache <DirCacheRootPath> <DirCacheResultFile>", "Cache the directory structure of a file path"},
	"listbucket":    []string{"qshell [-d] listbucket <Bucket> [<Prefix>] <ListBucketResultFile>", "List all the file in the bucket by prefix"},
	"alilistbucket": []string{"qshell [-d] alilistbucket <DataCenter> <Bucket> <AccessKeyId> <AccesskeySecret> [Prefix] <ListBucketResultFile>", "List all the file in the bucket of aliyun oss by prefix"},
	"prefop":        []string{"qshell [-d] prefop <PersistentId>", "Query the fop status"},
	"fput":          []string{"qshell [-d] fput <Bucket> <Key> <LocalFile> [MimeType]", "Form upload a local file"},
	"rput":          []string{"qshell [-d] rput <Bucket> <Key> <LocalFile> [MimeType]", "Resumable upload a local file"},
	"qupload":       []string{"qshell [-d] qupload [<PutThresoldInBytes>] <LocalUploadConfig>", "Batch upload files to the qiniu bucket"},
	"qdownload":     []string{"qshell [-d] qdownload [<ThreadCount>] <LocalDownloadConfig>", "Batch download files from the qiniu bucket"},
	"stat":          []string{"qshell [-d] stat <Bucket> <Key>", "Get the basic info of a remote file"},
	"delete":        []string{"qshell [-d] delete <Bucket> <Key>", "Delete a remote file in the bucket"},
	"move":          []string{"qshell [-d] move <SrcBucket> <SrcKey> <DestBucket> <DestKey>", "Move/Rename a file and save in bucket"},
	"copy":          []string{"qshell [-d] copy <SrcBucket> <SrcKey> <DestBucket> [<DestKey>]", "Make a copy of a file and save in bucket"},
	"chgm":          []string{"qshell [-d] chgm <Bucket> <Key> <NewMimeType>", "Change the mimeType of a file"},
	"fetch":         []string{"qshell [-d] fetch <RemoteResourceUrl> <Bucket> <Key>", "Fetch a remote resource by url and save in bucket"},
	"prefetch":      []string{"qshell [-d] prefetch <Bucket> <Key>", "Fetch and update the file in bucket using mirror storage"},
	"batchdelete":   []string{"qshell [-d] batchdelete <Bucket> <KeyListFile>", "Batch delete files in bucket"},
	"checkqrsync":   []string{"qshell [-d] checkqrsync <DirCacheResultFile> <ListBucketResultFile> <IgnoreLocalDir> [Prefix]", "Check the qrsync result"},
	"b64encode":     []string{"qshell [-d] b64encode [<UrlSafe>] <DataToEncode>", "Base64 Encode"},
	"b64decode":     []string{"qshell [-d] b64decode [<UrlSafe>] <DataToDecode>", "Base64 Decode"},
	"urlencode":     []string{"qshell [-d] urlencode <DataToEncode>", "Url encode"},
	"urldecode":     []string{"qshell [-d] urldecode <DataToDecode>", "Url decode"},
	"ts2d":          []string{"qshell [-d] ts2d <TimestampInSeconds>", "Convert timestamp in seconds to a date (TZ: Local)"},
	"tms2d":         []string{"qshell [-d] tms2d <TimestampInMilliSeconds>", "Convert timestamp in milli-seconds to a date (TZ: Local)"},
	"tns2d":         []string{"qshell [-d] tns2d <TimestampIn100NanoSeconds>", "Convert timestamp in 100 nano-seconds to a date (TZ: Local)"},
	"d2ts":          []string{"qshell [-d] d2ts <SecondsToNow>", "Create a timestamp in seconds using seconds to now"},
	"ip":            []string{"qshell [-d] ip <Ip1> [<Ip2> [<Ip3> ...]]]", "Query the ip information"},
}

func Help(cmd string, params ...string) {
	if len(params) == 0 {
		fmt.Println(CmdList())
	} else {
		CmdHelps(params...)
	}
}

func CmdList() string {
	helpAll := fmt.Sprintf("QShell %s\r\n\r\n", version)
	helpAll += "Options:\r\n"
	for k, v := range optionDocs {
		helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", k, v)
	}
	helpAll += "\r\n"
	helpAll += "Commands:\r\n"
	for _, cmd := range cmds {
		if help, ok := cmdDocs[cmd]; ok {
			cmdDesc := help[1]
			helpAll += fmt.Sprintf("\t%-20s%-20s\r\n", cmd, cmdDesc)
		}
	}
	return helpAll
}

func CmdHelps(cmds ...string) {
	defer os.Exit(1)
	if len(cmds) == 0 {
		fmt.Println(CmdList())
	} else {
		for _, cmd := range cmds {
			CmdHelp(cmd)
		}
	}
}

func CmdHelp(cmd string) {
	docStr := fmt.Sprintf("Unknow cmd `%s'", cmd)
	if cmdDoc, ok := cmdDocs[cmd]; ok {
		docStr = fmt.Sprintf("Usage: %s\r\n  %s\r\n", cmdDoc[0], cmdDoc[1])
	}
	fmt.Println(docStr)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"testing"
	"time"
)

func TestMain(m *testing.M) {

	setup()

	retCode := m.Run()

	// your func
	teardown()

	// call with result of m.Run()
	os.Exit(retCode)
}

// TestCaptureHeader tests whether request gets new header assigned
func TestCaptureHeader(t *testing.T) {

	server, dbClient := testTools(200, `{'message': 'here'}`)
	defer server.Close()

	req, err := http.NewRequest("GET", "http://example.com", nil)
	expect(t, err, nil)

	response, err := dbClient.captureRequest(req)

	expect(t, response.Header.Get("hoverfly"), "Was-Here")
}

// TestRequestBodyCaptured tests whether request body is recorded
func TestRequestBodyCaptured(t *testing.T) {

	server, dbClient := testTools(200, `{'message': 'here'}`)
	defer server.Close()

	body := ioutil.NopCloser(bytes.NewBuffer([]byte("fizz=buzz")))

	req, err := http.NewRequest("POST", "http://capture_body.com", body)
	expect(t, err, nil)

	_, err = dbClient.captureRequest(req)
	expect(t, err, nil)

	// since capture
	time.Sleep(10 * time.Millisecond)

	fp := getRequestFingerprint(req)

	payloadBts, err := dbClient.cache.Get([]byte(fp))
	expect(t, err, nil)

	payload, err := decodePayload(payloadBts)
	expect(t, err, nil)
	expect(t, payload.Request.Body, "fizz=buzz")

}

// TestRequestFingerprint tests whether we get correct request ID
func TestRequestFingerprint(t *testing.T) {

	req, err := http.NewRequest("GET", "http://example.com", nil)
	expect(t, err, nil)

	fp := getRequestFingerprint(req)

	expect(t, fp, "92a65ed4ca2b7100037a4cba9afd15ea")

}

func TestDeleteAllRecords(t *testing.T) {

	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()

	// inserting some payloads
	for i := 0; i < 5; i++ {
		req, err := http.NewRequest("GET", fmt.Sprintf("http://delete_all_records.com/q=%d", i), nil)
		expect(t, err, nil)
		dbClient.captureRequest(req)
	}
	err := dbClient.cache.DeleteBucket(dbClient.cache.requestsBucket)
	expect(t, err, nil)
}

func TestPayloadEncodeDecode(t *testing.T) {
	resp := response{
		Status: 200,
		Body:   "body here",
	}

	payload := Payload{Response: resp}

	bts, err := payload.encode()
	expect(t, err, nil)

	pl, err := decodePayload(bts)
	expect(t, err, nil)
	expect(t, pl.Response.Body, resp.Body)
	expect(t, pl.Response.Status, resp.Status)

}

func TestPayloadEncodeEmpty(t *testing.T) {
	payload := Payload{}

	bts, err := payload.encode()
	expect(t, err, nil)

	_, err = decodePayload(bts)
	expect(t, err, nil)
}

func TestDecodeRandomBytes(t *testing.T) {
	bts := []byte("some random stuff here")
	_, err := decodePayload(bts)
	refute(t, err, nil)
}
<commit_msg>added test for matching requests based on body<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"testing"
	"time"
)

func TestMain(m *testing.M) {

	setup()

	retCode := m.Run()

	// your func
	teardown()

	// call with result of m.Run()
	os.Exit(retCode)
}

// TestCaptureHeader tests whether request gets new header assigned
func TestCaptureHeader(t *testing.T) {

	server, dbClient := testTools(200, `{'message': 'here'}`)
	defer server.Close()

	req, err := http.NewRequest("GET", "http://example.com", nil)
	expect(t, err, nil)

	response, err := dbClient.captureRequest(req)

	expect(t, response.Header.Get("hoverfly"), "Was-Here")
}

// TestRequestBodyCaptured tests whether request body is recorded
func TestRequestBodyCaptured(t *testing.T) {

	server, dbClient := testTools(200, `{'message': 'here'}`)
	defer server.Close()

	requestBody := []byte("fizz=buzz")

	body := ioutil.NopCloser(bytes.NewBuffer(requestBody))

	req, err := http.NewRequest("POST", "http://capture_body.com", body)
	expect(t, err, nil)

	_, err = dbClient.captureRequest(req)
	expect(t, err, nil)

	// since capture
	time.Sleep(10 * time.Millisecond)

	fp := getRequestFingerprint(req, requestBody)

	payloadBts, err := dbClient.cache.Get([]byte(fp))
	expect(t, err, nil)

	payload, err := decodePayload(payloadBts)
	expect(t, err, nil)
	expect(t, payload.Request.Body, "fizz=buzz")
}

func TestMatchOnRequestBody(t *testing.T) {

	server, dbClient := testTools(200, `{'message': 'here'}`)
	defer server.Close()

	// preparing and saving requests/responses with unique bodies
	for i := 0; i < 5; i++ {
		requestBody := []byte(fmt.Sprintf("fizz=buzz, number=%d", i))
		body := ioutil.NopCloser(bytes.NewBuffer(requestBody))

		request, err := http.NewRequest("POST", "http://capture_body.com", body)
		expect(t, err, nil)

		resp := response{
			Status: 200,
			Body:   fmt.Sprintf("body here, number=%d", i),
		}
		payload := Payload{Response: resp}

		// creating response
		c := NewConstructor(request, payload)
		response := c.reconstructResponse()

		dbClient.save(request, requestBody, response, []byte(resp.Body))
	}

	// now getting responses
	for i := 0; i < 5; i++ {
		requestBody := []byte(fmt.Sprintf("fizz=buzz, number=%d", i))
		body := ioutil.NopCloser(bytes.NewBuffer(requestBody))

		request, _ := http.NewRequest("POST", "http://capture_body.com", body)

		response := dbClient.getResponse(request)

		responseBody, err := ioutil.ReadAll(response.Body)
		response.Body.Close()

		expect(t, err, nil)
		expect(t, string(responseBody), fmt.Sprintf("body here, number=%d", i))

	}

}

// TestRequestFingerprint tests whether we get correct request ID
func TestRequestFingerprint(t *testing.T) {

	req, err := http.NewRequest("GET", "http://example.com", nil)
	expect(t, err, nil)

	fp := getRequestFingerprint(req, []byte(""))

	expect(t, fp, "92a65ed4ca2b7100037a4cba9afd15ea")

}

func TestDeleteAllRecords(t *testing.T) {

	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()

	// inserting some payloads
	for i := 0; i < 5; i++ {
		req, err := http.NewRequest("GET", fmt.Sprintf("http://delete_all_records.com/q=%d", i), nil)
		expect(t, err, nil)
		dbClient.captureRequest(req)
	}
	err := dbClient.cache.DeleteBucket(dbClient.cache.requestsBucket)
	expect(t, err, nil)
}

func TestPayloadEncodeDecode(t *testing.T) {
	resp := response{
		Status: 200,
		Body:   "body here",
	}

	payload := Payload{Response: resp}

	bts, err := payload.encode()
	expect(t, err, nil)

	pl, err := decodePayload(bts)
	expect(t, err, nil)
	expect(t, pl.Response.Body, resp.Body)
	expect(t, pl.Response.Status, resp.Status)

}

func TestPayloadEncodeEmpty(t *testing.T) {
	payload := Payload{}

	bts, err := payload.encode()
	expect(t, err, nil)

	_, err = decodePayload(bts)
	expect(t, err, nil)
}

func TestDecodeRandomBytes(t *testing.T) {
	bts := []byte("some random stuff here")
	_, err := decodePayload(bts)
	refute(t, err, nil)
}
<|endoftext|>


<commit_before>package monico

import (
	"fmt"
	"io/ioutil"
	"os"
	"testing"
	"time"
)

var tempDir string

func TestMain(m *testing.M) {
	path, err := ioutil.TempDir("", "")
	if err != nil {
		fmt.Fprintln(os.Stderr, "monico_test:", err)
	}
	tempDir = path

	e := m.Run()
	defer os.Exit(e)

	os.RemoveAll(tempDir)
}

func TestPath(t *testing.T) {
	m, err := NewMoniter(tempDir)
	if err != nil {
		t.Errorf("NewMoniter returns %q, want nil", err)
	}
	expect := tempDir
	actual := m.Path()
	if actual != expect {
		t.Errorf("got %q, want %q",
			actual, expect)
	}
}

func TestErrorPath(t *testing.T) {
	_, err := NewMoniter("")
	if err == nil {
		t.Errorf("NewMoniter returns nil, want err")
	}
}

func TestDefaultPath(t *testing.T) {
	m, err := NewMoniterWithWD()
	if err != nil {
		t.Errorf("NewMoniter returns %q, want nil", err)
	}
	expect, err := os.Getwd()
	if err != nil {
		t.Errorf("Failed get working directory")
	}
	actual := m.Path()
	if actual != expect {
		t.Errorf("got %q, want %q",
			actual, expect)
	}
}

func TestModified(t *testing.T) {
	m, err := NewMoniter(tempDir)
	if err != nil {
		t.Errorf("NewMoniter returns %q, want nil", err)
	}
	now := time.Now()
	if err = os.Chtimes(tempDir, now, now); err != nil {
		t.Errorf("Failed os.Chtimes(%q, %v, %v)",
			tempDir, now, now)
	}

	expect := true
	actual, err := m.Modified()
	if err != nil {
		t.Errorf("Modified returns %q, want nil", err)
	}
	if actual != expect {
		t.Errorf("got %v, want %v", actual, expect)
	}
}

func TestNotModified(t *testing.T) {
	m, err := NewMoniter(tempDir)
	if err != nil {
		t.Errorf("NewMoniter returns %q, want nil", err)
	}

	expect := false
	actual, err := m.Modified()
	if err != nil {
		t.Errorf("Modified returns %q, want nil", err)
	}
	if actual != expect {
		t.Errorf("got %v, want %v", actual, expect)
	}
}

func TestUpdateModTime(t *testing.T) {
	m, err := NewMoniter(tempDir)
	if err != nil {
		t.Errorf("NewMoniter returns %q, want nil", err)
	}
	now := time.Now()
	if err = os.Chtimes(tempDir, now, now); err != nil {
		t.Errorf("Failed os.Chtimes(%q, %v, %v)",
			tempDir, now, now)
	}
	if err = m.UpdateModTime(); err != nil {
		t.Errorf("UpdateModTime returns %q, want nil", err)
	}

	expect := now
	actual := m.LastModTime()
	if actual != expect {
		t.Errorf("got %v, want %v",
			actual, expect)
	}
}
<commit_msg>Enable run test<commit_after>package monico

import (
	"fmt"
	"io/ioutil"
	"os"
	"testing"
	"time"
)

var tempDir string

func TestMain(m *testing.M) {
	path, err := ioutil.TempDir("", "")
	if err != nil {
		fmt.Fprintln(os.Stderr, "monico_test:", err)
	}
	tempDir = path

	e := m.Run()
	defer os.Exit(e)

	os.RemoveAll(tempDir)
}

func TestPath(t *testing.T) {
	m, err := NewMoniter(tempDir)
	if err != nil {
		t.Errorf("NewMoniter returns %q, want nil", err)
	}
	expect := tempDir
	actual := m.path
	if actual != expect {
		t.Errorf("got %q, want %q",
			actual, expect)
	}
}

func TestErrorPath(t *testing.T) {
	_, err := NewMoniter("")
	if err == nil {
		t.Errorf("NewMoniter returns nil, want err")
	}
}

func TestDefaultPath(t *testing.T) {
	m, err := NewMoniterWithWD()
	if err != nil {
		t.Errorf("NewMoniter returns %q, want nil", err)
	}
	expect, err := os.Getwd()
	if err != nil {
		t.Errorf("Failed get working directory")
	}
	actual := m.path
	if actual != expect {
		t.Errorf("got %q, want %q",
			actual, expect)
	}
}

func TestModified(t *testing.T) {
	m, err := NewMoniter(tempDir)
	if err != nil {
		t.Errorf("NewMoniter returns %q, want nil", err)
	}
	now := time.Now()
	if err = os.Chtimes(tempDir, now, now); err != nil {
		t.Errorf("Failed os.Chtimes(%q, %v, %v)",
			tempDir, now, now)
	}

	expect := true
	actual, err := m.Modified()
	if err != nil {
		t.Errorf("Modified returns %q, want nil", err)
	}
	if actual != expect {
		t.Errorf("got %v, want %v", actual, expect)
	}
}

func TestNotModified(t *testing.T) {
	m, err := NewMoniter(tempDir)
	if err != nil {
		t.Errorf("NewMoniter returns %q, want nil", err)
	}

	expect := false
	actual, err := m.Modified()
	if err != nil {
		t.Errorf("Modified returns %q, want nil", err)
	}
	if actual != expect {
		t.Errorf("got %v, want %v", actual, expect)
	}
}

func TestUpdateModTime(t *testing.T) {
	m, err := NewMoniter(tempDir)
	if err != nil {
		t.Errorf("NewMoniter returns %q, want nil", err)
	}
	now := time.Now()
	if err = os.Chtimes(tempDir, now, now); err != nil {
		t.Errorf("Failed os.Chtimes(%q, %v, %v)",
			tempDir, now, now)
	}
	if err = m.UpdateModTime(); err != nil {
		t.Errorf("UpdateModTime returns %q, want nil", err)
	}

	expect := now
	actual := m.lastModTime
	if actual != expect {
		t.Errorf("got %v, want %v",
			actual, expect)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

type Linker interface {
	Link(string, string) error
}

type FileLinker struct {
	System
	Logger
	Target, BinPath string
}

func (fl FileLinker) Link(name, version string) error {
	// fl.Stdoutf("Linking %s (v: %s) to %s in %s\n", name, version, fl.Target, fl.BinPath)

	targetPath, err := filepath.Rel(fl.BinPath, fl.Target)
	if err != nil {
		return err
	}

	if strings.HasSuffix(targetPath, fl.Target) {
		targetPath = fl.Target
	}

	fl.Debugf("  target path %s", targetPath)

	fullBinPath := filepath.Join(fl.BinPath, name)
	if len(version) > 0 {
		fullBinPath = fmt.Sprintf("%s--%s", fullBinPath, version)
	}
	fl.Debugf("  full bin path %s", fullBinPath)

	fl.Debugf("linking %s to %s", targetPath, fullBinPath)

	err = removeOldLink(fullBinPath)
	if err != nil {
		return err
	}

	// symlink to holen
	err = os.Symlink(fl.Target, fullBinPath)
	if err != nil {
		return err
	}
	return nil
}

type AliasLinker struct {
	System
}

func (al AliasLinker) Link(name, version string) error {
	var versionSuffix, versionFlag string
	if len(version) > 0 {
		versionSuffix = fmt.Sprintf("--%s", version)
		versionFlag = fmt.Sprintf(" --version %s", version)
	}

	al.Stdoutf("alias %s%s=\"holen run%s %s -- \"\n", name, versionSuffix, versionFlag, name)

	return nil
}

type ScriptLinker struct {
	System
	BinPath string
}

func (sl ScriptLinker) Link(name, version string) error {
	// sl.Stdoutf("Creating small script for %s (v: %s) in %s", name, version, sl.BinPath)

	fullBinPath := filepath.Join(sl.BinPath, name)
	if len(version) > 0 {
		fullBinPath = fmt.Sprintf("%s--%s", fullBinPath, version)
	}

	err := removeOldLink(fullBinPath)
	if err != nil {
		return err
	}

	var versionFlag string
	if len(version) > 0 {
		versionFlag = fmt.Sprintf(" --version %s", version)
	}

	scriptTmpl := `#!/bin/sh
exec holen run{{ .VersionFlag }} {{ .Name }} -- "$@"
`

	scriptData := struct {
		VersionFlag, Name string
	}{
		versionFlag, name,
	}

	tmpl := template.Must(template.New("script").Parse(scriptTmpl))
	var scriptBytes bytes.Buffer

	err = tmpl.Execute(&scriptBytes, scriptData)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(fullBinPath, scriptBytes.Bytes(), 0755)
}

func removeOldLink(fullPath string) error {
	fileStat, err := os.Lstat(fullPath)
	if err == nil {
		if fileStat.Mode()&os.ModeSymlink != 0 {
			// TODO: only remove symlinks that point at holen manifests
			os.Remove(fullPath)

			return nil
		} else if fileStat.Mode().IsRegular() {
			// TODO: only remove files that were generated by script link
			os.Remove(fullPath)

			return nil
		}
		return fmt.Errorf("non-holen file found at %s", fullPath)
	}

	return nil
}
<commit_msg>make pre-link cleanup slightly safer<commit_after>package main

import (
	"bytes"
	"fmt"
	"html/template"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

type Linker interface {
	Link(string, string) error
}

type FileLinker struct {
	System
	Logger
	Target, BinPath string
}

func (fl FileLinker) Link(name, version string) error {
	// fl.Stdoutf("Linking %s (v: %s) to %s in %s\n", name, version, fl.Target, fl.BinPath)

	targetPath, err := filepath.Rel(fl.BinPath, fl.Target)
	if err != nil {
		return err
	}

	if strings.HasSuffix(targetPath, fl.Target) {
		targetPath = fl.Target
	}

	fl.Debugf("  target path %s", targetPath)

	fullBinPath := filepath.Join(fl.BinPath, name)
	if len(version) > 0 {
		fullBinPath = fmt.Sprintf("%s--%s", fullBinPath, version)
	}
	fl.Debugf("  full bin path %s", fullBinPath)

	fl.Debugf("linking %s to %s", targetPath, fullBinPath)

	err = removeOldLink(fullBinPath)
	if err != nil {
		return err
	}

	// symlink to holen
	err = os.Symlink(fl.Target, fullBinPath)
	if err != nil {
		return err
	}
	return nil
}

type AliasLinker struct {
	System
}

func (al AliasLinker) Link(name, version string) error {
	var versionSuffix, versionFlag string
	if len(version) > 0 {
		versionSuffix = fmt.Sprintf("--%s", version)
		versionFlag = fmt.Sprintf(" --version %s", version)
	}

	al.Stdoutf("alias %s%s=\"holen run%s %s -- \"\n", name, versionSuffix, versionFlag, name)

	return nil
}

type ScriptLinker struct {
	System
	BinPath string
}

func (sl ScriptLinker) Link(name, version string) error {
	// sl.Stdoutf("Creating small script for %s (v: %s) in %s", name, version, sl.BinPath)

	fullBinPath := filepath.Join(sl.BinPath, name)
	if len(version) > 0 {
		fullBinPath = fmt.Sprintf("%s--%s", fullBinPath, version)
	}

	err := removeOldLink(fullBinPath)
	if err != nil {
		return err
	}

	var versionFlag string
	if len(version) > 0 {
		versionFlag = fmt.Sprintf(" --version %s", version)
	}

	scriptTmpl := `#!/bin/sh
exec holen run{{ .VersionFlag }} {{ .Name }} -- "$@"
`

	scriptData := struct {
		VersionFlag, Name string
	}{
		versionFlag, name,
	}

	tmpl := template.Must(template.New("script").Parse(scriptTmpl))
	var scriptBytes bytes.Buffer

	err = tmpl.Execute(&scriptBytes, scriptData)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(fullBinPath, scriptBytes.Bytes(), 0755)
}

func removeOldLink(fullPath string) error {
	fileStat, err := os.Lstat(fullPath)
	if err == nil {
		if fileStat.Mode()&os.ModeSymlink != 0 {
			target, _ := os.Readlink(fullPath)

			// TODO: check more thoroughly for links that are created by holen
			if strings.HasSuffix(target, "holen") || strings.HasSuffix(target, ".yaml") {
				os.Remove(fullPath)
				return nil
			}
			return fmt.Errorf("non-holen symlink found at %s", fullPath)
		} else if fileStat.Mode().IsRegular() {
			if fileStat.Size() < 500 {
				data, _ := ioutil.ReadFile(fullPath)
				if strings.Contains(string(data), "holen run") {
					os.Remove(fullPath)
					return nil
				}
			}
		}
		return fmt.Errorf("non-holen file found at %s", fullPath)
	}

	return nil
}
<|endoftext|>


<commit_before>// +build linux

/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mount

import (
	"bufio"
	"fmt"
	"hash/adler32"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"syscall"

	"github.com/golang/glog"
	utilExec "k8s.io/kubernetes/pkg/util/exec"
)

const (
	// How many times to retry for a consistent read of /proc/mounts.
	maxListTries = 3
	// Number of fields per line in /proc/mounts as per the fstab man page.
	expectedNumFieldsPerLine = 6
	// Location of the mount file to use
	procMountsPath = "/proc/mounts"
)

const (
	// 'fsck' found errors and corrected them
	fsckErrorsCorrected = 1
	// 'fsck' found errors but exited without correcting them
	fsckErrorsUncorrected = 4
)

// Mounter provides the default implementation of mount.Interface
// for the linux platform.  This implementation assumes that the
// kubelet is running in the host's root mount namespace.
type Mounter struct{}

var _ = Interface(&Mounter{})

// Mount mounts source to target as fstype with given options. 'source' and 'fstype' must
// be an emtpy string in case it's not required, e.g. for remount, or for auto filesystem
// type, where kernel handles fs type for you. The mount 'options' is a list of options,
// currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is
// required, call Mount with an empty string list or nil.
func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
	bind, bindRemountOpts := isBind(options)

	if bind {
		err := doMount(source, target, fstype, []string{"bind"})
		if err != nil {
			return err
		}
		return doMount(source, target, fstype, bindRemountOpts)
	} else {
		return doMount(source, target, fstype, options)
	}
}

// isBind detects whether a bind mount is being requested and makes the remount options to
// use in case of bind mount, due to the fact that bind mount doesn't respect mount options.
// The list equals:
//   options - 'bind' + 'remount' (no duplicate)
func isBind(options []string) (bool, []string) {
	bindRemountOpts := []string{"remount"}
	bind := false

	if len(options) != 0 {
		for _, option := range options {
			switch option {
			case "bind":
				bind = true
				break
			case "remount":
				break
			default:
				bindRemountOpts = append(bindRemountOpts, option)
			}
		}
	}

	return bind, bindRemountOpts
}

// doMount runs the mount command.
func doMount(source string, target string, fstype string, options []string) error {
	glog.V(5).Infof("Mounting %s %s %s %v", source, target, fstype, options)
	mountArgs := makeMountArgs(source, target, fstype, options)
	command := exec.Command("mount", mountArgs...)
	output, err := command.CombinedOutput()
	if err != nil {
		return fmt.Errorf("Mount failed: %v\nMounting arguments: %s %s %s %v\nOutput: %s\n",
			err, source, target, fstype, options, string(output))
	}
	return err
}

// makeMountArgs makes the arguments to the mount(8) command.
func makeMountArgs(source, target, fstype string, options []string) []string {
	// Build mount command as follows:
	//   mount [-t $fstype] [-o $options] [$source] $target
	mountArgs := []string{}
	if len(fstype) > 0 {
		mountArgs = append(mountArgs, "-t", fstype)
	}
	if len(options) > 0 {
		mountArgs = append(mountArgs, "-o", strings.Join(options, ","))
	}
	if len(source) > 0 {
		mountArgs = append(mountArgs, source)
	}
	mountArgs = append(mountArgs, target)

	return mountArgs
}

// Unmount unmounts the target.
func (mounter *Mounter) Unmount(target string) error {
	glog.V(5).Infof("Unmounting %s", target)
	command := exec.Command("umount", target)
	output, err := command.CombinedOutput()
	if err != nil {
		return fmt.Errorf("Unmount failed: %v\nUnmounting arguments: %s\nOutput: %s\n", err, target, string(output))
	}
	return nil
}

// List returns a list of all mounted filesystems.
func (*Mounter) List() ([]MountPoint, error) {
	return listProcMounts(procMountsPath)
}

// IsLikelyNotMountPoint determines if a directory is not a mountpoint.
// It is fast but not necessarily ALWAYS correct. If the path is in fact
// a bind mount from one part of a mount to another it will not be detected.
// mkdir /tmp/a /tmp/b; mount --bin /tmp/a /tmp/b; IsLikelyNotMountPoint("/tmp/b")
// will return true. When in fact /tmp/b is a mount point. If this situation
// if of interest to you, don't use this function...
func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
	stat, err := os.Stat(file)
	if err != nil {
		return true, err
	}
	rootStat, err := os.Lstat(file + "/..")
	if err != nil {
		return true, err
	}
	// If the directory has a different device as parent, then it is a mountpoint.
	if stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev {
		return false, nil
	}

	return true, nil
}

// DeviceOpened checks if block device in use by calling Open with O_EXCL flag.
// Returns true if open returns errno EBUSY, and false if errno is nil.
// Returns an error if errno is any error other than EBUSY.
// Returns with error if pathname is not a device.
func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) {
	return exclusiveOpenFailsOnDevice(pathname)
}

// PathIsDevice uses FileInfo returned from os.Stat to check if path refers
// to a device.
func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) {
	return pathIsDevice(pathname)
}

func exclusiveOpenFailsOnDevice(pathname string) (bool, error) {
	if isDevice, err := pathIsDevice(pathname); !isDevice {
		return false, fmt.Errorf(
			"PathIsDevice failed for path %q: %v",
			pathname,
			err)
	}
	fd, errno := syscall.Open(pathname, syscall.O_RDONLY|syscall.O_EXCL, 0)
	// If the device is in use, open will return an invalid fd.
	// When this happens, it is expected that Close will fail and throw an error.
	defer syscall.Close(fd)
	if errno == nil {
		// device not in use
		return false, nil
	} else if errno == syscall.EBUSY {
		// device is in use
		return true, nil
	}
	// error during call to Open
	return false, errno
}

func pathIsDevice(pathname string) (bool, error) {
	finfo, err := os.Stat(pathname)
	// err in call to os.Stat
	if err != nil {
		return false, err
	}
	// path refers to a device
	if finfo.Mode()&os.ModeDevice != 0 {
		return true, nil
	}
	// path does not refer to device
	return false, nil
}

func listProcMounts(mountFilePath string) ([]MountPoint, error) {
	hash1, err := readProcMounts(mountFilePath, nil)
	if err != nil {
		return nil, err
	}

	for i := 0; i < maxListTries; i++ {
		mps := []MountPoint{}
		hash2, err := readProcMounts(mountFilePath, &mps)
		if err != nil {
			return nil, err
		}
		if hash1 == hash2 {
			// Success
			return mps, nil
		}
		hash1 = hash2
	}
	return nil, fmt.Errorf("failed to get a consistent snapshot of %v after %d tries", mountFilePath, maxListTries)
}

// readProcMounts reads the given mountFilePath (normally /proc/mounts) and produces a hash
// of the contents.  If the out argument is not nil, this fills it with MountPoint structs.
func readProcMounts(mountFilePath string, out *[]MountPoint) (uint32, error) {
	file, err := os.Open(mountFilePath)
	if err != nil {
		return 0, err
	}
	defer file.Close()
	return readProcMountsFrom(file, out)
}

func readProcMountsFrom(file io.Reader, out *[]MountPoint) (uint32, error) {
	hash := adler32.New()
	scanner := bufio.NewReader(file)
	for {
		line, err := scanner.ReadString('\n')
		if err == io.EOF {
			break
		}
		fields := strings.Fields(line)
		if len(fields) != expectedNumFieldsPerLine {
			return 0, fmt.Errorf("wrong number of fields (expected %d, got %d): %s", expectedNumFieldsPerLine, len(fields), line)
		}

		fmt.Fprintf(hash, "%s", line)

		if out != nil {
			mp := MountPoint{
				Device: fields[0],
				Path:   fields[1],
				Type:   fields[2],
				Opts:   strings.Split(fields[3], ","),
			}

			freq, err := strconv.Atoi(fields[4])
			if err != nil {
				return 0, err
			}
			mp.Freq = freq

			pass, err := strconv.Atoi(fields[5])
			if err != nil {
				return 0, err
			}
			mp.Pass = pass

			*out = append(*out, mp)
		}
	}
	return hash.Sum32(), nil
}

// formatAndMount uses unix utils to format and mount the given disk
func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
	options = append(options, "defaults")

	// Run fsck on the disk to fix repairable issues
	args := []string{"-a", source}
	cmd := mounter.Runner.Command("fsck", args...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		ee, isExitError := err.(utilExec.ExitError)
		switch {
		case err == utilExec.ErrExecutableNotFound:
			glog.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.")
		case isExitError && ee.ExitStatus() == fsckErrorsCorrected:
			glog.Infof("Device %s has errors which were corrected by fsck.", source)
		case isExitError && ee.ExitStatus() == fsckErrorsUncorrected:
			return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s.", source, string(out))
		case isExitError && ee.ExitStatus() > fsckErrorsUncorrected:
			glog.Infof("`fsck` error %s", string(out))
		}
	}

	// Try to mount the disk
	err = mounter.Interface.Mount(source, target, fstype, options)
	if err != nil {
		// It is possible that this disk is not formatted. Double check using diskLooksUnformatted
		notFormatted, err := mounter.diskLooksUnformatted(source)
		if err == nil && notFormatted {
			args = []string{source}
			// Disk is unformatted so format it.
			// Use 'ext4' as the default
			if len(fstype) == 0 {
				fstype = "ext4"
			}
			if fstype == "ext4" || fstype == "ext3" {
				args = []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", source}
			}
			cmd := mounter.Runner.Command("mkfs."+fstype, args...)
			_, err := cmd.CombinedOutput()
			if err == nil {
				// the disk has been formatted successfully try to mount it again.
				return mounter.Interface.Mount(source, target, fstype, options)
			}
			return err
		}
	}
	return err
}

// diskLooksUnformatted uses 'lsblk' to see if the given disk is unformated
func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) {
	args := []string{"-nd", "-o", "FSTYPE", disk}
	cmd := mounter.Runner.Command("lsblk", args...)
	dataOut, err := cmd.CombinedOutput()
	output := strings.TrimSpace(string(dataOut))

	// TODO (#13212): check if this disk has partitions and return false, and
	// an error if so.

	if err != nil {
		return false, err
	}

	return output == "", nil
}
<commit_msg>Add volume and mount logging<commit_after>// +build linux

/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mount

import (
	"bufio"
	"fmt"
	"hash/adler32"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"syscall"

	"github.com/golang/glog"
	utilExec "k8s.io/kubernetes/pkg/util/exec"
)

const (
	// How many times to retry for a consistent read of /proc/mounts.
	maxListTries = 3
	// Number of fields per line in /proc/mounts as per the fstab man page.
	expectedNumFieldsPerLine = 6
	// Location of the mount file to use
	procMountsPath = "/proc/mounts"
)

const (
	// 'fsck' found errors and corrected them
	fsckErrorsCorrected = 1
	// 'fsck' found errors but exited without correcting them
	fsckErrorsUncorrected = 4
)

// Mounter provides the default implementation of mount.Interface
// for the linux platform.  This implementation assumes that the
// kubelet is running in the host's root mount namespace.
type Mounter struct{}

var _ = Interface(&Mounter{})

// Mount mounts source to target as fstype with given options. 'source' and 'fstype' must
// be an emtpy string in case it's not required, e.g. for remount, or for auto filesystem
// type, where kernel handles fs type for you. The mount 'options' is a list of options,
// currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is
// required, call Mount with an empty string list or nil.
func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
	bind, bindRemountOpts := isBind(options)

	if bind {
		err := doMount(source, target, fstype, []string{"bind"})
		if err != nil {
			return err
		}
		return doMount(source, target, fstype, bindRemountOpts)
	} else {
		return doMount(source, target, fstype, options)
	}
}

// isBind detects whether a bind mount is being requested and makes the remount options to
// use in case of bind mount, due to the fact that bind mount doesn't respect mount options.
// The list equals:
//   options - 'bind' + 'remount' (no duplicate)
func isBind(options []string) (bool, []string) {
	bindRemountOpts := []string{"remount"}
	bind := false

	if len(options) != 0 {
		for _, option := range options {
			switch option {
			case "bind":
				bind = true
				break
			case "remount":
				break
			default:
				bindRemountOpts = append(bindRemountOpts, option)
			}
		}
	}

	return bind, bindRemountOpts
}

// doMount runs the mount command.
func doMount(source string, target string, fstype string, options []string) error {
	glog.V(5).Infof("Mounting %s %s %s %v", source, target, fstype, options)
	mountArgs := makeMountArgs(source, target, fstype, options)
	command := exec.Command("mount", mountArgs...)
	output, err := command.CombinedOutput()
	if err != nil {
		glog.Errorf("Mount failed: %v\nMounting arguments: %s %s %s %v\nOutput: %s\n", err, source, target, fstype, options, string(output))
		return fmt.Errorf("mount failed: %v\nMounting arguments: %s %s %s %v\nOutput: %s\n",
			err, source, target, fstype, options, string(output))
	}
	return err
}

// makeMountArgs makes the arguments to the mount(8) command.
func makeMountArgs(source, target, fstype string, options []string) []string {
	// Build mount command as follows:
	//   mount [-t $fstype] [-o $options] [$source] $target
	mountArgs := []string{}
	if len(fstype) > 0 {
		mountArgs = append(mountArgs, "-t", fstype)
	}
	if len(options) > 0 {
		mountArgs = append(mountArgs, "-o", strings.Join(options, ","))
	}
	if len(source) > 0 {
		mountArgs = append(mountArgs, source)
	}
	mountArgs = append(mountArgs, target)

	return mountArgs
}

// Unmount unmounts the target.
func (mounter *Mounter) Unmount(target string) error {
	glog.V(5).Infof("Unmounting %s", target)
	command := exec.Command("umount", target)
	output, err := command.CombinedOutput()
	if err != nil {
		return fmt.Errorf("Unmount failed: %v\nUnmounting arguments: %s\nOutput: %s\n", err, target, string(output))
	}
	return nil
}

// List returns a list of all mounted filesystems.
func (*Mounter) List() ([]MountPoint, error) {
	return listProcMounts(procMountsPath)
}

// IsLikelyNotMountPoint determines if a directory is not a mountpoint.
// It is fast but not necessarily ALWAYS correct. If the path is in fact
// a bind mount from one part of a mount to another it will not be detected.
// mkdir /tmp/a /tmp/b; mount --bin /tmp/a /tmp/b; IsLikelyNotMountPoint("/tmp/b")
// will return true. When in fact /tmp/b is a mount point. If this situation
// if of interest to you, don't use this function...
func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
	stat, err := os.Stat(file)
	if err != nil {
		return true, err
	}
	rootStat, err := os.Lstat(file + "/..")
	if err != nil {
		return true, err
	}
	// If the directory has a different device as parent, then it is a mountpoint.
	if stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev {
		return false, nil
	}

	return true, nil
}

// DeviceOpened checks if block device in use by calling Open with O_EXCL flag.
// Returns true if open returns errno EBUSY, and false if errno is nil.
// Returns an error if errno is any error other than EBUSY.
// Returns with error if pathname is not a device.
func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) {
	return exclusiveOpenFailsOnDevice(pathname)
}

// PathIsDevice uses FileInfo returned from os.Stat to check if path refers
// to a device.
func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) {
	return pathIsDevice(pathname)
}

func exclusiveOpenFailsOnDevice(pathname string) (bool, error) {
	if isDevice, err := pathIsDevice(pathname); !isDevice {
		return false, fmt.Errorf(
			"PathIsDevice failed for path %q: %v",
			pathname,
			err)
	}
	fd, errno := syscall.Open(pathname, syscall.O_RDONLY|syscall.O_EXCL, 0)
	// If the device is in use, open will return an invalid fd.
	// When this happens, it is expected that Close will fail and throw an error.
	defer syscall.Close(fd)
	if errno == nil {
		// device not in use
		return false, nil
	} else if errno == syscall.EBUSY {
		// device is in use
		return true, nil
	}
	// error during call to Open
	return false, errno
}

func pathIsDevice(pathname string) (bool, error) {
	finfo, err := os.Stat(pathname)
	// err in call to os.Stat
	if err != nil {
		return false, err
	}
	// path refers to a device
	if finfo.Mode()&os.ModeDevice != 0 {
		return true, nil
	}
	// path does not refer to device
	return false, nil
}

func listProcMounts(mountFilePath string) ([]MountPoint, error) {
	hash1, err := readProcMounts(mountFilePath, nil)
	if err != nil {
		return nil, err
	}

	for i := 0; i < maxListTries; i++ {
		mps := []MountPoint{}
		hash2, err := readProcMounts(mountFilePath, &mps)
		if err != nil {
			return nil, err
		}
		if hash1 == hash2 {
			// Success
			return mps, nil
		}
		hash1 = hash2
	}
	return nil, fmt.Errorf("failed to get a consistent snapshot of %v after %d tries", mountFilePath, maxListTries)
}

// readProcMounts reads the given mountFilePath (normally /proc/mounts) and produces a hash
// of the contents.  If the out argument is not nil, this fills it with MountPoint structs.
func readProcMounts(mountFilePath string, out *[]MountPoint) (uint32, error) {
	file, err := os.Open(mountFilePath)
	if err != nil {
		return 0, err
	}
	defer file.Close()
	return readProcMountsFrom(file, out)
}

func readProcMountsFrom(file io.Reader, out *[]MountPoint) (uint32, error) {
	hash := adler32.New()
	scanner := bufio.NewReader(file)
	for {
		line, err := scanner.ReadString('\n')
		if err == io.EOF {
			break
		}
		fields := strings.Fields(line)
		if len(fields) != expectedNumFieldsPerLine {
			return 0, fmt.Errorf("wrong number of fields (expected %d, got %d): %s", expectedNumFieldsPerLine, len(fields), line)
		}

		fmt.Fprintf(hash, "%s", line)

		if out != nil {
			mp := MountPoint{
				Device: fields[0],
				Path:   fields[1],
				Type:   fields[2],
				Opts:   strings.Split(fields[3], ","),
			}

			freq, err := strconv.Atoi(fields[4])
			if err != nil {
				return 0, err
			}
			mp.Freq = freq

			pass, err := strconv.Atoi(fields[5])
			if err != nil {
				return 0, err
			}
			mp.Pass = pass

			*out = append(*out, mp)
		}
	}
	return hash.Sum32(), nil
}

// formatAndMount uses unix utils to format and mount the given disk
func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
	options = append(options, "defaults")

	// Run fsck on the disk to fix repairable issues
	glog.V(4).Infof("Checking for issues with fsck on disk: %s", source)
	args := []string{"-a", source}
	cmd := mounter.Runner.Command("fsck", args...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		ee, isExitError := err.(utilExec.ExitError)
		switch {
		case err == utilExec.ErrExecutableNotFound:
			glog.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.")
		case isExitError && ee.ExitStatus() == fsckErrorsCorrected:
			glog.Infof("Device %s has errors which were corrected by fsck.", source)
		case isExitError && ee.ExitStatus() == fsckErrorsUncorrected:
			return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s.", source, string(out))
		case isExitError && ee.ExitStatus() > fsckErrorsUncorrected:
			glog.Infof("`fsck` error %s", string(out))
		}
	}

	// Try to mount the disk
	glog.V(4).Infof("Attempting to mount disk: %s %s %s", fstype, source, target)
	err = mounter.Interface.Mount(source, target, fstype, options)
	if err != nil {
		// It is possible that this disk is not formatted. Double check using diskLooksUnformatted
		notFormatted, err := mounter.diskLooksUnformatted(source)
		if err == nil && notFormatted {
			args = []string{source}
			// Disk is unformatted so format it.
			// Use 'ext4' as the default
			if len(fstype) == 0 {
				fstype = "ext4"
			}
			if fstype == "ext4" || fstype == "ext3" {
				args = []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", source}
			}
			glog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args)
			cmd := mounter.Runner.Command("mkfs."+fstype, args...)
			_, err := cmd.CombinedOutput()
			if err == nil {
				// the disk has been formatted successfully try to mount it again.
				glog.Infof("Disk successfully formatted (mkfs): %s - %s %s", fstype, source, target)
				return mounter.Interface.Mount(source, target, fstype, options)
			}
			glog.Errorf("format of disk %q failed: type:(%q) target:(%q) options:(%q)error:(%v)", source, fstype, target, options, err)
			return err
		}
	}
	return err
}

// diskLooksUnformatted uses 'lsblk' to see if the given disk is unformated
func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) {
	args := []string{"-nd", "-o", "FSTYPE", disk}
	cmd := mounter.Runner.Command("lsblk", args...)
	glog.V(4).Infof("Attempting to determine if disk %q is formatted using lsblk with args: (%v)", disk, args)
	dataOut, err := cmd.CombinedOutput()
	output := strings.TrimSpace(string(dataOut))

	// TODO (#13212): check if this disk has partitions and return false, and
	// an error if so.

	if err != nil {
		glog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err)
		return false, err
	}

	return output == "", nil
}
<|endoftext|>


<commit_before>package libvirt

import (
	"encoding/xml"
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/schema"
	libvirt "github.com/libvirt/libvirt-go"
)

func resourceLibvirtVolume() *schema.Resource {
	return &schema.Resource{
		Create: resourceLibvirtVolumeCreate,
		Read:   resourceLibvirtVolumeRead,
		Delete: resourceLibvirtVolumeDelete,
		Exists: resourceLibvirtVolumeExists,
		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"pool": {
				Type:     schema.TypeString,
				Optional: true,
				Default:  "default",
				ForceNew: true,
			},
			"source": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"size": {
				Type:     schema.TypeInt,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"format": {
				Type:     schema.TypeString,
				Computed: true,
				Optional: true,
				ForceNew: true,
			},
			"base_volume_id": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"base_volume_pool": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"base_volume_name": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
		},
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
	}
}

func resourceLibvirtVolumeCreate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*Client)
	if client.libvirt == nil {
		return fmt.Errorf(LibVirtConIsNil)
	}

	poolName := "default"
	if _, ok := d.GetOk("pool"); ok {
		poolName = d.Get("pool").(string)
	}

	client.poolMutexKV.Lock(poolName)
	defer client.poolMutexKV.Unlock(poolName)

	pool, err := client.libvirt.LookupStoragePoolByName(poolName)
	if err != nil {
		return fmt.Errorf("can't find storage pool '%s'", poolName)
	}
	defer pool.Free()

	// Refresh the pool of the volume so that libvirt knows it is
	// not longer in use.
	waitForSuccess("error refreshing pool for volume", func() error {
		return pool.Refresh(0)
	})

	// Check whether the storage volume already exists. Its name needs to be
	// unique.
	if _, err := pool.LookupStorageVolByName(d.Get("name").(string)); err == nil {
		return fmt.Errorf("storage volume '%s' already exists", d.Get("name").(string))
	}

	volumeDef := newDefVolume()
	volumeDef.Name = d.Get("name").(string)

	var (
		img    image
		volume *libvirt.StorageVol
	)

	specifiedFormat, isFormatSpecified := d.GetOk("format")
	if isFormatSpecified {
		volumeDef.Target.Format.Type = specifiedFormat.(string)
	}

	// an source image was given, this mean we can't choose size
	if source, ok := d.GetOk("source"); ok {
		// source and size conflict
		if _, ok := d.GetOk("size"); ok {
			return fmt.Errorf("'size' can't be specified when also 'source' is given (the size will be set to the size of the source image")
		}
		if _, ok := d.GetOk("base_volume_id"); ok {
			return fmt.Errorf("'base_volume_id' can't be specified when also 'source' is given")
		}

		if _, ok := d.GetOk("base_volume_name"); ok {
			return fmt.Errorf("'base_volume_name' can't be specified when also 'source' is given")
		}

		if img, err = newImage(source.(string)); err != nil {
			return err
		}

		// figure out the format of the image
		isQCOW2, err := img.IsQCOW2()
		if err != nil {
			return fmt.Errorf("Error while determining image type for %s: %s", img.String(), err)
		}
		if isQCOW2 {
			volumeDef.Target.Format.Type = "qcow2"
		}

		if isFormatSpecified && isQCOW2 && specifiedFormat != "qcow2" {
			return fmt.Errorf("Format other than QCOW2 explicitly specified for image detected as QCOW2 image: %s", img.String())
		}

		// update the image in the description, even if the file has not changed
		size, err := img.Size()
		if err != nil {
			return err
		}
		log.Printf("Image %s image is: %d bytes", img, size)
		volumeDef.Capacity.Unit = "B"
		volumeDef.Capacity.Value = size
	} else {
		_, noSize := d.GetOk("size")
		_, noBaseVol := d.GetOk("base_volume_id")

		if noSize && noBaseVol {
			return fmt.Errorf("'size' needs to be specified if no 'source' or 'base_volume_id' is given")
		}
		volumeDef.Capacity.Value = uint64(d.Get("size").(int))
	}

	if baseVolumeID, ok := d.GetOk("base_volume_id"); ok {
		if _, ok := d.GetOk("size"); ok {
			return fmt.Errorf("'size' can't be specified when also 'base_volume_id' is given (the size will be set to the size of the backing image")
		}

		if _, ok := d.GetOk("base_volume_name"); ok {
			return fmt.Errorf("'base_volume_name' can't be specified when also 'base_volume_id' is given")
		}

		volume = nil
		baseVolume, err := client.libvirt.LookupStorageVolByKey(baseVolumeID.(string))
		if err != nil {
			return fmt.Errorf("Can't retrieve volume %s: %v", baseVolumeID.(string), err)
		}
		backingStoreDef, err := newDefBackingStoreFromLibvirt(baseVolume)
		if err != nil {
			return fmt.Errorf("Could not retrieve backing store %s", baseVolumeID.(string))
		}
		volumeDef.BackingStore = &backingStoreDef
	}

	if baseVolumeName, ok := d.GetOk("base_volume_name"); ok {
		if _, ok := d.GetOk("size"); ok {
			return fmt.Errorf("'size' can't be specified when also 'base_volume_name' is given (the size will be set to the size of the backing image")
		}

		volume = nil
		baseVolumePool := pool
		if _, ok := d.GetOk("base_volume_pool"); ok {
			baseVolumePoolName := d.Get("base_volume_pool").(string)
			baseVolumePool, err = client.libvirt.LookupStoragePoolByName(baseVolumePoolName)
			if err != nil {
				return fmt.Errorf("can't find storage pool '%s'", baseVolumePoolName)
			}
			defer baseVolumePool.Free()
		}
		baseVolume, err := baseVolumePool.LookupStorageVolByName(baseVolumeName.(string))
		if err != nil {
			return fmt.Errorf("Can't retrieve volume %s: %v", baseVolumeName.(string), err)
		}
		backingStoreDef, err := newDefBackingStoreFromLibvirt(baseVolume)
		if err != nil {
			return fmt.Errorf("Could not retrieve backing store %s", baseVolumeName.(string))
		}
		volumeDef.BackingStore = &backingStoreDef
	}

	if volume == nil {
		volumeDefXML, err := xml.Marshal(volumeDef)
		if err != nil {
			return fmt.Errorf("Error serializing libvirt volume: %s", err)
		}

		// create the volume
		v, err := pool.StorageVolCreateXML(string(volumeDefXML), 0)
		if err != nil {
			return fmt.Errorf("Error creating libvirt volume: %s", err)
		}
		volume = v
		defer volume.Free()
	}

	// we use the key as the id
	key, err := volume.GetKey()
	if err != nil {
		return fmt.Errorf("Error retrieving volume key: %s", err)
	}
	d.SetId(key)

	// make sure we record the id even if the rest of this gets interrupted
	d.Partial(true)
	d.Set("id", key)
	d.SetPartial("id")
	d.Partial(false)

	log.Printf("[INFO] Volume ID: %s", d.Id())

	// upload source if present
	if _, ok := d.GetOk("source"); ok {
		err = img.Import(newCopier(client.libvirt, volume, volumeDef.Capacity.Value), volumeDef)
		if err != nil {
			return fmt.Errorf("Error while uploading source %s: %s", img.String(), err)
		}
	}

	return resourceLibvirtVolumeRead(d, meta)
}

func resourceLibvirtVolumeRead(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*Client)
	virConn := client.libvirt
	if virConn == nil {
		return fmt.Errorf(LibVirtConIsNil)
	}

	volume, err := lookupVolumeReallyHard(client, d.Get("pool").(string), d.Id())
	if err != nil {
		return err
	}

	if volume == nil {
		log.Printf("Volume '%s' may have been deleted outside Terraform", d.Id())
		d.SetId("")
		return nil
	}
	defer volume.Free()

	volName, err := volume.GetName()
	if err != nil {
		return fmt.Errorf("error retrieving volume name: %s", err)
	}

	volPool, err := volume.LookupPoolByVolume()
	if err != nil {
		return fmt.Errorf("error retrieving pool for volume: %s", err)
	}
	defer volPool.Free()

	volPoolName, err := volPool.GetName()
	if err != nil {
		return fmt.Errorf("error retrieving pool name: %s", err)
	}

	d.Set("pool", volPoolName)
	d.Set("name", volName)

	info, err := volume.GetInfo()
	if err != nil {
		return fmt.Errorf("error retrieving volume name: %s", err)
	}
	d.Set("size", info.Capacity)

	volumeDef, err := newDefVolumeFromLibvirt(volume)
	if err != nil {
		return err
	}

	if volumeDef.Target.Format.Type == "" {
		return fmt.Errorf("Volume has no format specified: %s", volName)
	}
	log.Printf("[DEBUG] Volume %s format: %s", volName, volumeDef.Target.Format.Type)
	d.Set("format", volumeDef.Target.Format.Type)

	return nil
}

func resourceLibvirtVolumeDelete(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*Client)
	if client.libvirt == nil {
		return fmt.Errorf(LibVirtConIsNil)
	}

	return removeVolume(client, d.Id())
}

func resourceLibvirtVolumeExists(d *schema.ResourceData, meta interface{}) (bool, error) {
	log.Printf("[DEBUG] Check if resource libvirt_volume exists")
	client := meta.(*Client)

	volPoolName := d.Get("pool").(string)
	volume, err := lookupVolumeReallyHard(client, volPoolName, d.Id())
	if err != nil {
		return false, err
	}

	if volume == nil {
		return false, nil
	}
	defer volume.Free()

	return true, nil
}
<commit_msg>volume: Rename "format specified" to "format given"<commit_after>package libvirt

import (
	"encoding/xml"
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/schema"
	libvirt "github.com/libvirt/libvirt-go"
)

func resourceLibvirtVolume() *schema.Resource {
	return &schema.Resource{
		Create: resourceLibvirtVolumeCreate,
		Read:   resourceLibvirtVolumeRead,
		Delete: resourceLibvirtVolumeDelete,
		Exists: resourceLibvirtVolumeExists,
		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"pool": {
				Type:     schema.TypeString,
				Optional: true,
				Default:  "default",
				ForceNew: true,
			},
			"source": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"size": {
				Type:     schema.TypeInt,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"format": {
				Type:     schema.TypeString,
				Computed: true,
				Optional: true,
				ForceNew: true,
			},
			"base_volume_id": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"base_volume_pool": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"base_volume_name": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
		},
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
	}
}

func resourceLibvirtVolumeCreate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*Client)
	if client.libvirt == nil {
		return fmt.Errorf(LibVirtConIsNil)
	}

	poolName := "default"
	if _, ok := d.GetOk("pool"); ok {
		poolName = d.Get("pool").(string)
	}

	client.poolMutexKV.Lock(poolName)
	defer client.poolMutexKV.Unlock(poolName)

	pool, err := client.libvirt.LookupStoragePoolByName(poolName)
	if err != nil {
		return fmt.Errorf("can't find storage pool '%s'", poolName)
	}
	defer pool.Free()

	// Refresh the pool of the volume so that libvirt knows it is
	// not longer in use.
	waitForSuccess("error refreshing pool for volume", func() error {
		return pool.Refresh(0)
	})

	// Check whether the storage volume already exists. Its name needs to be
	// unique.
	if _, err := pool.LookupStorageVolByName(d.Get("name").(string)); err == nil {
		return fmt.Errorf("storage volume '%s' already exists", d.Get("name").(string))
	}

	volumeDef := newDefVolume()
	volumeDef.Name = d.Get("name").(string)

	var (
		img    image
		volume *libvirt.StorageVol
	)

	givenFormat, isFormatGiven := d.GetOk("format")
	if isFormatGiven {
		volumeDef.Target.Format.Type = givenFormat.(string)
	}

	// an source image was given, this mean we can't choose size
	if source, ok := d.GetOk("source"); ok {
		// source and size conflict
		if _, ok := d.GetOk("size"); ok {
			return fmt.Errorf("'size' can't be specified when also 'source' is given (the size will be set to the size of the source image")
		}
		if _, ok := d.GetOk("base_volume_id"); ok {
			return fmt.Errorf("'base_volume_id' can't be specified when also 'source' is given")
		}

		if _, ok := d.GetOk("base_volume_name"); ok {
			return fmt.Errorf("'base_volume_name' can't be specified when also 'source' is given")
		}

		if img, err = newImage(source.(string)); err != nil {
			return err
		}

		// figure out the format of the image
		isQCOW2, err := img.IsQCOW2()
		if err != nil {
			return fmt.Errorf("Error while determining image type for %s: %s", img.String(), err)
		}
		if isQCOW2 {
			volumeDef.Target.Format.Type = "qcow2"
		}

		if isFormatGiven && isQCOW2 && givenFormat != "qcow2" {
			return fmt.Errorf("Format other than QCOW2 explicitly specified for image detected as QCOW2 image: %s", img.String())
		}

		// update the image in the description, even if the file has not changed
		size, err := img.Size()
		if err != nil {
			return err
		}
		log.Printf("Image %s image is: %d bytes", img, size)
		volumeDef.Capacity.Unit = "B"
		volumeDef.Capacity.Value = size
	} else {
		_, noSize := d.GetOk("size")
		_, noBaseVol := d.GetOk("base_volume_id")

		if noSize && noBaseVol {
			return fmt.Errorf("'size' needs to be specified if no 'source' or 'base_volume_id' is given")
		}
		volumeDef.Capacity.Value = uint64(d.Get("size").(int))
	}

	if baseVolumeID, ok := d.GetOk("base_volume_id"); ok {
		if _, ok := d.GetOk("size"); ok {
			return fmt.Errorf("'size' can't be specified when also 'base_volume_id' is given (the size will be set to the size of the backing image")
		}

		if _, ok := d.GetOk("base_volume_name"); ok {
			return fmt.Errorf("'base_volume_name' can't be specified when also 'base_volume_id' is given")
		}

		volume = nil
		baseVolume, err := client.libvirt.LookupStorageVolByKey(baseVolumeID.(string))
		if err != nil {
			return fmt.Errorf("Can't retrieve volume %s: %v", baseVolumeID.(string), err)
		}
		backingStoreDef, err := newDefBackingStoreFromLibvirt(baseVolume)
		if err != nil {
			return fmt.Errorf("Could not retrieve backing store %s", baseVolumeID.(string))
		}
		volumeDef.BackingStore = &backingStoreDef
	}

	if baseVolumeName, ok := d.GetOk("base_volume_name"); ok {
		if _, ok := d.GetOk("size"); ok {
			return fmt.Errorf("'size' can't be specified when also 'base_volume_name' is given (the size will be set to the size of the backing image")
		}

		volume = nil
		baseVolumePool := pool
		if _, ok := d.GetOk("base_volume_pool"); ok {
			baseVolumePoolName := d.Get("base_volume_pool").(string)
			baseVolumePool, err = client.libvirt.LookupStoragePoolByName(baseVolumePoolName)
			if err != nil {
				return fmt.Errorf("can't find storage pool '%s'", baseVolumePoolName)
			}
			defer baseVolumePool.Free()
		}
		baseVolume, err := baseVolumePool.LookupStorageVolByName(baseVolumeName.(string))
		if err != nil {
			return fmt.Errorf("Can't retrieve volume %s: %v", baseVolumeName.(string), err)
		}
		backingStoreDef, err := newDefBackingStoreFromLibvirt(baseVolume)
		if err != nil {
			return fmt.Errorf("Could not retrieve backing store %s", baseVolumeName.(string))
		}
		volumeDef.BackingStore = &backingStoreDef
	}

	if volume == nil {
		volumeDefXML, err := xml.Marshal(volumeDef)
		if err != nil {
			return fmt.Errorf("Error serializing libvirt volume: %s", err)
		}

		// create the volume
		v, err := pool.StorageVolCreateXML(string(volumeDefXML), 0)
		if err != nil {
			return fmt.Errorf("Error creating libvirt volume: %s", err)
		}
		volume = v
		defer volume.Free()
	}

	// we use the key as the id
	key, err := volume.GetKey()
	if err != nil {
		return fmt.Errorf("Error retrieving volume key: %s", err)
	}
	d.SetId(key)

	// make sure we record the id even if the rest of this gets interrupted
	d.Partial(true)
	d.Set("id", key)
	d.SetPartial("id")
	d.Partial(false)

	log.Printf("[INFO] Volume ID: %s", d.Id())

	// upload source if present
	if _, ok := d.GetOk("source"); ok {
		err = img.Import(newCopier(client.libvirt, volume, volumeDef.Capacity.Value), volumeDef)
		if err != nil {
			return fmt.Errorf("Error while uploading source %s: %s", img.String(), err)
		}
	}

	return resourceLibvirtVolumeRead(d, meta)
}

func resourceLibvirtVolumeRead(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*Client)
	virConn := client.libvirt
	if virConn == nil {
		return fmt.Errorf(LibVirtConIsNil)
	}

	volume, err := lookupVolumeReallyHard(client, d.Get("pool").(string), d.Id())
	if err != nil {
		return err
	}

	if volume == nil {
		log.Printf("Volume '%s' may have been deleted outside Terraform", d.Id())
		d.SetId("")
		return nil
	}
	defer volume.Free()

	volName, err := volume.GetName()
	if err != nil {
		return fmt.Errorf("error retrieving volume name: %s", err)
	}

	volPool, err := volume.LookupPoolByVolume()
	if err != nil {
		return fmt.Errorf("error retrieving pool for volume: %s", err)
	}
	defer volPool.Free()

	volPoolName, err := volPool.GetName()
	if err != nil {
		return fmt.Errorf("error retrieving pool name: %s", err)
	}

	d.Set("pool", volPoolName)
	d.Set("name", volName)

	info, err := volume.GetInfo()
	if err != nil {
		return fmt.Errorf("error retrieving volume name: %s", err)
	}
	d.Set("size", info.Capacity)

	volumeDef, err := newDefVolumeFromLibvirt(volume)
	if err != nil {
		return err
	}

	if volumeDef.Target.Format.Type == "" {
		return fmt.Errorf("Volume has no format specified: %s", volName)
	}
	log.Printf("[DEBUG] Volume %s format: %s", volName, volumeDef.Target.Format.Type)
	d.Set("format", volumeDef.Target.Format.Type)

	return nil
}

func resourceLibvirtVolumeDelete(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*Client)
	if client.libvirt == nil {
		return fmt.Errorf(LibVirtConIsNil)
	}

	return removeVolume(client, d.Id())
}

func resourceLibvirtVolumeExists(d *schema.ResourceData, meta interface{}) (bool, error) {
	log.Printf("[DEBUG] Check if resource libvirt_volume exists")
	client := meta.(*Client)

	volPoolName := d.Get("pool").(string)
	volume, err := lookupVolumeReallyHard(client, volPoolName, d.Id())
	if err != nil {
		return false, err
	}

	if volume == nil {
		return false, nil
	}
	defer volume.Free()

	return true, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"golang.org/x/net/websocket"
	"html"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	p "path"
	"strconv"
	"strings"
	"time"
)

func main() {
	http.HandleFunc("/", handleAny)
	log.Fatal(http.ListenAndServe(":"+httpPort(), nil))
}

func handleAny(res http.ResponseWriter, req *http.Request) {
	log.Printf("method=%s path=%q", req.Method, req.URL.Path)

	path, err := os.Open(req.URL.Path)
	if err != nil {
		if os.IsNotExist(err) {
			handleError(res, req, err, http.StatusNotFound, "File not found")
			return
		}

		handleError(res, req, err, http.StatusBadRequest, "Error reading path")
		return
	}
	defer path.Close()

	pathInfo, err := path.Stat()
	if err != nil {
		handleError(res, req, err, http.StatusInternalServerError, "Error reading path info")
		return
	}

	if strings.HasPrefix(req.Header.Get("Origin"), "ws://") { // TODO: how to detect? scheme is null
		handleWs(res, req, path, pathInfo)
		return
	}

	switch req.Method {
	case "GET":
		handleGet(res, req, path, pathInfo)
	case "POST":
		handlePost(res, req, path, pathInfo)
	default:
		http.Error(res, "Only GET and POST supported", http.StatusMethodNotAllowed)
	}
}

func handleGet(res http.ResponseWriter, req *http.Request, path *os.File, pathInfo os.FileInfo) {
	if pathInfo.Mode().IsDir() {
		fileInfos, err := path.Readdir(0)
		if err != nil {
			handleError(res, req, err, http.StatusInternalServerError, "Error reading directory")
			return
		}

		if strings.Contains(req.Header.Get("Accept"), "html") {
			renderDirHtml(res, path.Name(), fileInfos)
		} else {
			renderDirJson(res, fileInfos)
		}
	} else if pathInfo.Mode().IsRegular() {
		stat, err := path.Stat()
		if err != nil {
			handleError(res, req, err, http.StatusInternalServerError, "Error reading file stat")
			return
		}

		if stat.Size() == 0 {
			// handle reading special 0-byte unix files
			// TODO: better way to determine these from normal 0-byte files?
			res.Header().Set("Transfer-Encoding", "chunked")
		} else {
			res.Header().Set("Content-Length", strconv.FormatInt(stat.Size(), 10))
		}

		io.Copy(res, path)
	} else {
		handleError(res, req, nil,
			http.StatusBadRequest,
			"Invalid file type for GET. Only directories and regular files are supported.")
	}
}

func handlePost(res http.ResponseWriter, req *http.Request, path *os.File, pathInfo os.FileInfo) {
	resFlusherWriter := flushWriterWrapper{res.(flushWriter)}
	execCmd(res, req, path, pathInfo, req.Body, resFlusherWriter, false)
}

func handleWs(res http.ResponseWriter, req *http.Request, path *os.File, pathInfo os.FileInfo) {
	handler := func(ws *websocket.Conn) {
		execCmd(res, req, path, pathInfo, ws, ws, true)
	}

	websocket.Handler(handler).ServeHTTP(res, req)
}

func execCmd(res http.ResponseWriter, req *http.Request, path *os.File, pathInfo os.FileInfo, in io.Reader, out io.Writer, interactive bool) {
	var cmd *exec.Cmd

	if pathInfo.Mode().IsDir() {
		if interactive {
			// TODO: allow interactive session to have a prompt, support heredocs, handle arrow keys, and generally act like a real terminal
			// cmd = exec.Command("bash", "-i") // double echos (ws + bash -i) and displays arrow character

			// pseudo-interactive-bash worksaround `bash -i` echoing problem, but breaks on heredocs and probably other bash special cases
			dir, err := os.Getwd()
			if err != nil {
				log.Fatal(err)
			}
			log.Println(dir)
			cmd = exec.Command(p.Join(dir, "bin", "pseudo-interactive-bash"))
		} else {
			cmd = exec.Command("bash")
		}
		cmd.Dir = path.Name()
	} else if pathInfo.Mode().IsRegular() && pathInfo.Mode()&0110 != 0 /* is executable for user or group */ {
		cmd = exec.Command(path.Name())
		cmd.Dir = path.Name()[0:strings.LastIndex(path.Name(), string(os.PathSeparator))]
	} else {
		handleError(res, req, nil,
			http.StatusBadRequest,
			"Invalid file type for POST. Only directories and regular executable file are supported")
		return
	}

	cmd.Env = cgiEnv(req)
	if interactive {
		cmd.Env = append(cmd.Env, "PS1=\\[\\033[01;34m\\]\\w\\[\\033[00m\\] \\[\\033[01;32m\\]$ \\[\\033[00m\\]")
	}
	cmd.Stdin = in
	cmd.Stdout = out
	cmd.Stderr = out
	if err := cmd.Run(); err != nil {
		// error already sent to client. log only
		log.Printf("method=%s path=%q message=%q", req.Method, req.URL.Path, err)
	}
}

func renderDirHtml(res http.ResponseWriter, pathName string, fileInfos []os.FileInfo) {
	res.Header().Set("Content-Type", "text/html")
	fmt.Fprintf(res, "

")
	for _, fi := range fileInfos {
		if !strings.HasSuffix(pathName, "/") {
			pathName += "/"
		}

		label := fi.Name()
		if fi.IsDir() {
			label = "/" + label
		}

		fmt.Fprintf(res, "
%s
", pathName, fi.Name(), html.EscapeString(label))
	}
	fmt.Fprintf(res, "

")
}

func renderDirJson(res http.ResponseWriter, fileInfos []os.FileInfo) {
	res.Header().Set("Content-Type", "application/json")
	fileResponses := map[string]FileInfoDetails{}
	for _, fi := range fileInfos {
		fileResponses[fi.Name()] = toFileInfoDetails(fi)
	}
	fileResponsesJson, err := json.MarshalIndent(fileResponses, "", "  ")
	if err != nil {
		panic(err)
	}
	fmt.Fprintln(res, string(fileResponsesJson))
}

type FileInfoDetails struct {
	Size    int64     `json:"size"`
	Type    string    `json:"type"`
	Perm    int       `json:"permission"`
	ModTime time.Time `json:"updated_at"`
}

func toFileInfoDetails(fi os.FileInfo) FileInfoDetails {
	return FileInfoDetails{
		Size:    fi.Size(),
		Type:    string(fi.Mode().String()[0]),
		Perm:    int(fi.Mode().Perm()),
		ModTime: fi.ModTime(),
	}
}

type stdError struct {
	Message string `json:"message"`
	Cause   error  `json:"cause"`
}

func handleError(res http.ResponseWriter, req *http.Request, err error, httpStatus int, message string) {
	stdError := stdError{
		Message: message,
		Cause:   err,
	}

	log.Printf("method=%s path=%q message=%q cause=%q", req.Method, req.URL.Path, stdError.Message, stdError.Cause)

	stdErrorJson, err := json.MarshalIndent(stdError, "", "  ")
	if err != nil {
		panic(err)
	}

	res.Header().Set("Content-Type", "application/json")
	http.Error(res, string(stdErrorJson), httpStatus)
}

type flushWriter interface {
	Flush()
	Write(buf []byte) (int, error)
}

type flushWriterWrapper struct {
	fw flushWriter
}

func (fww flushWriterWrapper) Write(p []byte) (n int, err error) {
	n, err = fww.fw.Write(p)
	fww.fw.Flush()
	return
}

func httpPort() string {
	if port := os.Getenv("PORT"); port != "" {
		return port
	}
	return "5000"
}

// copied from net/http/cgi/host.go
func cgiEnv(req *http.Request) []string {
	env := []string{
		"SERVER_SOFTWARE=go",
		"SERVER_NAME=" + req.Host,
		"SERVER_PROTOCOL=HTTP/1.1",
		"HTTP_HOST=" + req.Host,
		"GATEWAY_INTERFACE=CGI/1.1",
		"REQUEST_METHOD=" + req.Method,
		"QUERY_STRING=" + req.URL.RawQuery,
		"REQUEST_URI=" + req.URL.RequestURI(),
		"PATH_INFO=" + req.URL.Path,
		"SCRIPT_NAME=" + req.URL.Path,
		"SCRIPT_FILENAME=" + req.URL.Path,
		"REMOTE_ADDR=" + req.RemoteAddr,
		"REMOTE_HOST=" + req.RemoteAddr,
		"SERVER_PORT=" + httpPort(),
	}

	if req.TLS != nil {
		env = append(env, "HTTPS=on")
	}

	for k, v := range req.Header {
		k = strings.Map(upperCaseAndUnderscore, k)
		joinStr := ", "
		if k == "COOKIE" {
			joinStr = "; "
		}
		env = append(env, "HTTP_"+k+"="+strings.Join(v, joinStr))
	}

	if req.ContentLength > 0 {
		env = append(env, fmt.Sprintf("CONTENT_LENGTH=%d", req.ContentLength))
	}
	if ctype := req.Header.Get("Content-Type"); ctype != "" {
		env = append(env, "CONTENT_TYPE="+ctype)
	}

	return env
}

func upperCaseAndUnderscore(r rune) rune {
	switch {
	case r >= 'a' && r <= 'z':
		return r - ('a' - 'A')
	case r == '-':
		return '_'
	case r == '=':
		// Maybe not part of the CGI 'spec' but would mess up
		// the environment in any case, as Go represents the
		// environment as a slice of "key=value" strings.
		return '_'
	}
	// TODO: other transformations in spec or practice?
	return r
}
<commit_msg>Only set Content-Length > 0<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"golang.org/x/net/websocket"
	"html"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	p "path"
	"strconv"
	"strings"
	"time"
)

func main() {
	http.HandleFunc("/", handleAny)
	log.Fatal(http.ListenAndServe(":"+httpPort(), nil))
}

func handleAny(res http.ResponseWriter, req *http.Request) {
	log.Printf("method=%s path=%q", req.Method, req.URL.Path)

	path, err := os.Open(req.URL.Path)
	if err != nil {
		if os.IsNotExist(err) {
			handleError(res, req, err, http.StatusNotFound, "File not found")
			return
		}

		handleError(res, req, err, http.StatusBadRequest, "Error reading path")
		return
	}
	defer path.Close()

	pathInfo, err := path.Stat()
	if err != nil {
		handleError(res, req, err, http.StatusInternalServerError, "Error reading path info")
		return
	}

	if strings.HasPrefix(req.Header.Get("Origin"), "ws://") { // TODO: how to detect? scheme is null
		handleWs(res, req, path, pathInfo)
		return
	}

	switch req.Method {
	case "GET":
		handleGet(res, req, path, pathInfo)
	case "POST":
		handlePost(res, req, path, pathInfo)
	default:
		http.Error(res, "Only GET and POST supported", http.StatusMethodNotAllowed)
	}
}

func handleGet(res http.ResponseWriter, req *http.Request, path *os.File, pathInfo os.FileInfo) {
	if pathInfo.Mode().IsDir() {
		fileInfos, err := path.Readdir(0)
		if err != nil {
			handleError(res, req, err, http.StatusInternalServerError, "Error reading directory")
			return
		}

		if strings.Contains(req.Header.Get("Accept"), "html") {
			renderDirHtml(res, path.Name(), fileInfos)
		} else {
			renderDirJson(res, fileInfos)
		}
	} else if pathInfo.Mode().IsRegular() {
		stat, err := path.Stat()
		if err != nil {
			handleError(res, req, err, http.StatusInternalServerError, "Error reading file stat")
			return
		}

		// explicitly set Content-Length
		// handle reading special 0-byte unix files
		if stat.Size() > 0 {
			res.Header().Set("Content-Length", strconv.FormatInt(stat.Size(), 10))
		}

		io.Copy(res, path)
	} else {
		handleError(res, req, nil,
			http.StatusBadRequest,
			"Invalid file type for GET. Only directories and regular files are supported.")
	}
}

func handlePost(res http.ResponseWriter, req *http.Request, path *os.File, pathInfo os.FileInfo) {
	resFlusherWriter := flushWriterWrapper{res.(flushWriter)}
	execCmd(res, req, path, pathInfo, req.Body, resFlusherWriter, false)
}

func handleWs(res http.ResponseWriter, req *http.Request, path *os.File, pathInfo os.FileInfo) {
	handler := func(ws *websocket.Conn) {
		execCmd(res, req, path, pathInfo, ws, ws, true)
	}

	websocket.Handler(handler).ServeHTTP(res, req)
}

func execCmd(res http.ResponseWriter, req *http.Request, path *os.File, pathInfo os.FileInfo, in io.Reader, out io.Writer, interactive bool) {
	var cmd *exec.Cmd

	if pathInfo.Mode().IsDir() {
		if interactive {
			// TODO: allow interactive session to have a prompt, support heredocs, handle arrow keys, and generally act like a real terminal
			// cmd = exec.Command("bash", "-i") // double echos (ws + bash -i) and displays arrow character

			// pseudo-interactive-bash worksaround `bash -i` echoing problem, but breaks on heredocs and probably other bash special cases
			dir, err := os.Getwd()
			if err != nil {
				log.Fatal(err)
			}
			log.Println(dir)
			cmd = exec.Command(p.Join(dir, "bin", "pseudo-interactive-bash"))
		} else {
			cmd = exec.Command("bash")
		}
		cmd.Dir = path.Name()
	} else if pathInfo.Mode().IsRegular() && pathInfo.Mode()&0110 != 0 /* is executable for user or group */ {
		cmd = exec.Command(path.Name())
		cmd.Dir = path.Name()[0:strings.LastIndex(path.Name(), string(os.PathSeparator))]
	} else {
		handleError(res, req, nil,
			http.StatusBadRequest,
			"Invalid file type for POST. Only directories and regular executable file are supported")
		return
	}

	cmd.Env = cgiEnv(req)
	if interactive {
		cmd.Env = append(cmd.Env, "PS1=\\[\\033[01;34m\\]\\w\\[\\033[00m\\] \\[\\033[01;32m\\]$ \\[\\033[00m\\]")
	}
	cmd.Stdin = in
	cmd.Stdout = out
	cmd.Stderr = out
	if err := cmd.Run(); err != nil {
		// error already sent to client. log only
		log.Printf("method=%s path=%q message=%q", req.Method, req.URL.Path, err)
	}
}

func renderDirHtml(res http.ResponseWriter, pathName string, fileInfos []os.FileInfo) {
	res.Header().Set("Content-Type", "text/html")
	fmt.Fprintf(res, "

")
	for _, fi := range fileInfos {
		if !strings.HasSuffix(pathName, "/") {
			pathName += "/"
		}

		label := fi.Name()
		if fi.IsDir() {
			label = "/" + label
		}

		fmt.Fprintf(res, "
%s
", pathName, fi.Name(), html.EscapeString(label))
	}
	fmt.Fprintf(res, "

")
}

func renderDirJson(res http.ResponseWriter, fileInfos []os.FileInfo) {
	res.Header().Set("Content-Type", "application/json")
	fileResponses := map[string]FileInfoDetails{}
	for _, fi := range fileInfos {
		fileResponses[fi.Name()] = toFileInfoDetails(fi)
	}
	fileResponsesJson, err := json.MarshalIndent(fileResponses, "", "  ")
	if err != nil {
		panic(err)
	}
	fmt.Fprintln(res, string(fileResponsesJson))
}

type FileInfoDetails struct {
	Size    int64     `json:"size"`
	Type    string    `json:"type"`
	Perm    int       `json:"permission"`
	ModTime time.Time `json:"updated_at"`
}

func toFileInfoDetails(fi os.FileInfo) FileInfoDetails {
	return FileInfoDetails{
		Size:    fi.Size(),
		Type:    string(fi.Mode().String()[0]),
		Perm:    int(fi.Mode().Perm()),
		ModTime: fi.ModTime(),
	}
}

type stdError struct {
	Message string `json:"message"`
	Cause   error  `json:"cause"`
}

func handleError(res http.ResponseWriter, req *http.Request, err error, httpStatus int, message string) {
	stdError := stdError{
		Message: message,
		Cause:   err,
	}

	log.Printf("method=%s path=%q message=%q cause=%q", req.Method, req.URL.Path, stdError.Message, stdError.Cause)

	stdErrorJson, err := json.MarshalIndent(stdError, "", "  ")
	if err != nil {
		panic(err)
	}

	res.Header().Set("Content-Type", "application/json")
	http.Error(res, string(stdErrorJson), httpStatus)
}

type flushWriter interface {
	Flush()
	Write(buf []byte) (int, error)
}

type flushWriterWrapper struct {
	fw flushWriter
}

func (fww flushWriterWrapper) Write(p []byte) (n int, err error) {
	n, err = fww.fw.Write(p)
	fww.fw.Flush()
	return
}

func httpPort() string {
	if port := os.Getenv("PORT"); port != "" {
		return port
	}
	return "5000"
}

// copied from net/http/cgi/host.go
func cgiEnv(req *http.Request) []string {
	env := []string{
		"SERVER_SOFTWARE=go",
		"SERVER_NAME=" + req.Host,
		"SERVER_PROTOCOL=HTTP/1.1",
		"HTTP_HOST=" + req.Host,
		"GATEWAY_INTERFACE=CGI/1.1",
		"REQUEST_METHOD=" + req.Method,
		"QUERY_STRING=" + req.URL.RawQuery,
		"REQUEST_URI=" + req.URL.RequestURI(),
		"PATH_INFO=" + req.URL.Path,
		"SCRIPT_NAME=" + req.URL.Path,
		"SCRIPT_FILENAME=" + req.URL.Path,
		"REMOTE_ADDR=" + req.RemoteAddr,
		"REMOTE_HOST=" + req.RemoteAddr,
		"SERVER_PORT=" + httpPort(),
	}

	if req.TLS != nil {
		env = append(env, "HTTPS=on")
	}

	for k, v := range req.Header {
		k = strings.Map(upperCaseAndUnderscore, k)
		joinStr := ", "
		if k == "COOKIE" {
			joinStr = "; "
		}
		env = append(env, "HTTP_"+k+"="+strings.Join(v, joinStr))
	}

	if req.ContentLength > 0 {
		env = append(env, fmt.Sprintf("CONTENT_LENGTH=%d", req.ContentLength))
	}
	if ctype := req.Header.Get("Content-Type"); ctype != "" {
		env = append(env, "CONTENT_TYPE="+ctype)
	}

	return env
}

func upperCaseAndUnderscore(r rune) rune {
	switch {
	case r >= 'a' && r <= 'z':
		return r - ('a' - 'A')
	case r == '-':
		return '_'
	case r == '=':
		// Maybe not part of the CGI 'spec' but would mess up
		// the environment in any case, as Go represents the
		// environment as a slice of "key=value" strings.
		return '_'
	}
	// TODO: other transformations in spec or practice?
	return r
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
)

var port = flag.String("p", "8888", "Port to listen on")
var storage = flag.String("s", "/var/lib/shelf", "Path to store files")

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage:	%v -p <port> -s <storage-path>\n\n", os.Args[0])
		flag.PrintDefaults()
	}
}

func errorResponse(w http.ResponseWriter, e error) {
	w.WriteHeader(http.StatusInternalServerError)
	w.Write([]byte(e.Error()))
	log.Println("error:", e.Error())
}

func main() {
	flag.Parse()

	var storagepath string
	if flag.NArg() == 1 {
		// deprecated: passing storage path as argument
		storagepath = flag.Arg(0)
	} else {
		storagepath = *storage
	}
	os.MkdirAll(storagepath, 0755)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		filepath := storagepath + r.RequestURI
		switch r.Method {
		case "HEAD", "GET":
			file, err := os.Open(filepath)
			if err != nil {
				errorResponse(w, err)
				return
			}
			defer file.Close()
			fi, err := file.Stat()
			if err != nil {
				errorResponse(w, err)
				return
			}
			log.Println("GET", r.RequestURI)
			http.ServeContent(w, r, filepath, fi.ModTime(), file)
		case "PUT":
			os.MkdirAll(path.Dir(filepath), 0755)
			file, err := os.Create(filepath)
			if err != nil {
				errorResponse(w, err)
				return
			}
			defer file.Close()
			_, err = io.Copy(file, r.Body)
			if err != nil {
				errorResponse(w, err)
				return
			}
			log.Println("PUT", r.RequestURI)
		case "DELETE":
			err := os.RemoveAll(filepath)
			if err != nil {
				errorResponse(w, err)
				return
			}
			log.Println("DELETE", r.RequestURI)
		default:
			w.WriteHeader(http.StatusMethodNotAllowed)
		}
	})
	log.Println("Shelf serving files on " + *port + " from " + storagepath)
	log.Fatal(http.ListenAndServe(":"+*port, nil))
}
<commit_msg>Serve 404 errors if file is missing.<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
)

var port = flag.String("p", "8888", "Port to listen on")
var storage = flag.String("s", "/var/lib/shelf", "Path to store files")

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage:	%v -p <port> -s <storage-path>\n\n", os.Args[0])
		flag.PrintDefaults()
	}
}

func errorResponse(w http.ResponseWriter, e error) {
	code := http.StatusInternalServerError
	if os.IsNotExist(e) {
		code = http.StatusNotFound
	}
	w.WriteHeader(code)
	w.Write([]byte(e.Error()))
	log.Println("error:", e.Error())
}

func main() {
	flag.Parse()

	var storagepath string
	if flag.NArg() == 1 {
		// deprecated: passing storage path as argument
		storagepath = flag.Arg(0)
	} else {
		storagepath = *storage
	}
	os.MkdirAll(storagepath, 0755)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		filepath := storagepath + r.RequestURI
		switch r.Method {
		case "HEAD", "GET":
			file, err := os.Open(filepath)
			if err != nil {
				errorResponse(w, err)
				return
			}
			defer file.Close()
			fi, err := file.Stat()
			if err != nil {
				errorResponse(w, err)
				return
			}
			log.Println("GET", r.RequestURI)
			http.ServeContent(w, r, filepath, fi.ModTime(), file)
		case "PUT":
			os.MkdirAll(path.Dir(filepath), 0755)
			file, err := os.Create(filepath)
			if err != nil {
				errorResponse(w, err)
				return
			}
			defer file.Close()
			_, err = io.Copy(file, r.Body)
			if err != nil {
				errorResponse(w, err)
				return
			}
			log.Println("PUT", r.RequestURI)
		case "DELETE":
			err := os.RemoveAll(filepath)
			if err != nil {
				errorResponse(w, err)
				return
			}
			log.Println("DELETE", r.RequestURI)
		default:
			w.WriteHeader(http.StatusMethodNotAllowed)
		}
	})
	log.Println("Shelf serving files on " + *port + " from " + storagepath)
	log.Fatal(http.ListenAndServe(":"+*port, nil))
}
<|endoftext|>