<commit_before>package main

import (
    "fmt"
    "net/http"
    "os"
    "os/exec"
    "io"
    "encoding/json"
    "log"
    //"time"
    "strings"
    //"math/rand"
    "regexp"
)

var HEADER_CONTENT_TYPES map[string]string = map[string]string{
    "json": "javascript/json;charset=utf-8",
    "text": "text/plain;charset=utf-8",
    "xml": "text/xml;charset=utf-8",
}

var IGNORE_OPTIONS map[string]bool = map[string]bool{
    "-e": true,
    "-c": true,
    "-d": true,
}

const DEFAULT_FORMAT string = "text"

type Config struct {
    Host string
    Port int
    Reg_filter string
    Mystem_path string
    Mystem_options []string
    Mystem_workers int
    Mystem_answer_size int
    Channel_buffer int
    Max_word_length int
    Max_words int
    format string
    is_d bool
}

type Answer struct {
    data string
    err error
}

type Data struct {
    word string
    channel chan Answer
}

var for_process chan Data
var reg_filter *regexp.Regexp
var config Config

func makePanic(msg string) {
    log.Fatal(msg)
    panic(msg)
}

func processMystemOptions(in_opts []string) (out_opts []string, format string, err error) {
    var opts_arr []string
    var approved bool = false
    var exists bool = false
    var n_exists = false
    for _, opt := range in_opts {
        approved = false
        opt = strings.TrimSpace(opt)
        opt = strings.ToLower(opt)
        if strings.Index(opt, " ") == -1 {
            opts_arr = make([]string, 1)
            opts_arr[0] = opt
            if !bool(IGNORE_OPTIONS[opt]) {
                approved = true
            }
            if (!n_exists) && (opt == "-n") {
                n_exists = true
            }
        } else {
            opts_arr = make([]string, 2)
            opts_arr = strings.Split(opt, " ")
            if opts_arr[0] == "--format" {
                _, exists = HEADER_CONTENT_TYPES[opts_arr[1]]
                if exists {
                    format = opts_arr[1]
                } else {
                    format = DEFAULT_FORMAT
                    opts_arr[1] = format
                    log.Printf("Mystem option '%v' was replaced by default value. Because contain error", opt)
                }
                approved = true
            } else if !bool(IGNORE_OPTIONS[opts_arr[0]]) {
                approved = true
            }
        }
        if approved {
            out_opts = append(out_opts, opts_arr...)
        } else {
            log.Printf("Mystem option '%v' was ignored or replaced by default value. Because wrong or unnecessary", opt)
        }
    }
    if !n_exists {
        out_opts = append(out_opts, "-n")
        log.Print("Added mystem option '-n'. It need for normal work.\n")
    }
    if format == "" {
        format = DEFAULT_FORMAT
    }
    return out_opts, format, err
}

func loadConfig() (cfg Config, err error) {
    var f_name string = "http_mystem.json"
    if (len(os.Args) > 1) {
        f_name = os.Args[1]
    }
    file, err := os.Open(f_name)
    defer file.Close()
    if err == nil {
        f_stat, _ := file.Stat()
        raw_json := make([]byte, f_stat.Size())
        _, err = file.Read(raw_json)
        if err == nil {
            err = json.Unmarshal(raw_json, &cfg)
            if err == nil {
                cfg.Mystem_options, cfg.format, err = processMystemOptions(cfg.Mystem_options)
                //cfg.Mystem_answer_size = cfg.Max_word_length * 500
            }
        }
    }
    return cfg, err
}

func writeStringToPipe(for_write string, pipe io.WriteCloser) (n int, err error) {
    defer func() {
        result := recover()
        if result != nil {
            log.Fatalf("Recover say: %v", result)
        }
    }()
    n, err = pipe.Write([]byte(for_write + "\n"))
    if err != nil {
        log.Panicf("Can't send word to mystem: %v", err)
    }
    return n, err
}

func readStringFromPipe(for_read *string, pipe io.ReadCloser) (n int, err error) {
    defer func() {
        result := recover()
        if result != nil {
            log.Fatalf("Recover say: %v", result)
        }
    }()
    buf := make([]byte, config.Mystem_answer_size)
    n, err = pipe.Read(buf)
    if err == nil {
        *for_read = strings.TrimSpace(string(buf[:n]))
    } else {
        log.Panicf("Can't send word to mystem: %v", err)
    }
    return n, err
}

func readXMLTrash(number int, pipe io.ReadCloser) {
    var data string = ""
    var err error
    for i := 0; i < number; i++ {
        for data == "" {
            _, err = readStringFromPipe(&data, pipe)
            if err != nil {
                makePanic(fmt.Sprintf("Can't start: %v", err))
            }
        }
        data = ""
    }
}

func workerMystem(for_process chan Data, mystem_path string) {
    //name := rand.Int()
    //log.Print("Mystem started ", name)
    mystem := exec.Command(mystem_path, config.Mystem_options...)
    mystem_writer, err_w := mystem.StdinPipe()
    mystem_reader, err_r := mystem.StdoutPipe()
    err := mystem.Start()
    if (err_w != nil) || (err_r != nil) {
        makePanic(fmt.Sprintf("Can't start: \n%v\n%v", err_w, err_r))
    }
    if err != nil {
        makePanic(fmt.Sprintf("Can't start: %v", err))
    } else {
        var data Data
        var answer Answer
        if config.format == "xml" {
            readXMLTrash(2, mystem_reader)
            _, err = writeStringToPipe("тест", mystem_writer)
            if err != nil {
                panic("")
            }
            readXMLTrash(2, mystem_reader)
        } else {
            _, err = writeStringToPipe("тест", mystem_writer)
            if err != nil {
                panic("")
            }
            _, err = readStringFromPipe(&answer.data, mystem_reader)
            if err != nil {
                panic("")
            }
        }
        for {
            data = <- for_process
            _, err = writeStringToPipe(data.word, mystem_writer)
            answer.err = err
            if config.format != "xml" {
                _, err = readStringFromPipe(&answer.data, mystem_reader)
                if err != nil {
                    answer.err = err
                    break
                }
            } else {
                for answer.data == "" {
                    _, err = readStringFromPipe(&answer.data, mystem_reader)
                    if err != nil {
                        answer.err = err
                        break
                    }
                }
            }
            data.channel <- answer
            answer.data = ""
            answer.err = nil
        }
    }
}

func checkInput(req *http.Request, words *[]string, ) (code int) {
    var words_count int = len(req.Form["words[]"])
    if words_count == 0 {
        code = http.StatusNotAcceptable
    } else if words_count > int(config.Max_words) {
        code = http.StatusRequestEntityTooLarge
    } else {
        var w string = ""
        for _, word := range req.Form["words[]"] {
            w = strings.TrimSpace(word)
            if len(w) > int(config.Max_word_length) {
                w = word[:int(config.Max_word_length)]
            }
            if w != "" {
                *words = append(*words, w)
            }
        }
    }
    return code
}

func processWords(resp http.ResponseWriter, req *http.Request) {
    var data Data
    var answer Answer
    var answers []string
    var request_answer string = ""
    req.ParseForm()
    var words []string
    var tmp_answer string = ""
    status_code := checkInput(req, &words)
    var words_count int = len(words)
    resp.Header().Set("Content-Type", HEADER_CONTENT_TYPES[config.format])
    //resp.Header().Set("Content-Type", "text/plain;charset=utf-8")
    if words_count > 0 {
        var err error
        var local_channel chan Answer = make(chan Answer, words_count)
        data.channel = local_channel
        var start int = 0
        for _, word := range words {
            err = nil
            if reg_filter.MatchString(word) {
                data.word = word
                for_process <- data
            } else {
                start++
                switch config.format {
                    case "json": {
                        tmp_answer = fmt.Sprintf(`{"analysis":[], "text":"%v"}`, word)
                    }
                    case "xml": {
                        tmp_answer = "<w>" + word + "</w>"
                    }
                    case "text": {
                        tmp_answer = word
                    }
                }
                answers = append(answers, tmp_answer)
            }
        }
        for i := start; i < words_count; i++ {
            answer = <- local_channel
            if answer.err != nil {
                err = answer.err
            }
            answers = append(answers, answer.data)
        }
        if err == nil {
            switch config.format {
                case "json": {
                    request_answer = "[" + strings.Join(answers, ",") + "]"
                }
                case "xml": {
                    request_answer = `<?xml version="1.0" encoding="utf-8"?><html><body><se>` + strings.Join(answers, "\n") + "</se>"
                }
                case "text": {
                    request_answer = strings.Join(answers, "\n")
                }
            }
        } else {
            //request_answer = fmt.Sprintf(`{"status": %v, "reason": "%v"}`, status_code, http.StatusText(http.StatusInternalServerError)
            resp.WriteHeader(http.StatusInternalServerError)
        }
        close(local_channel)
    } else {
        //request_answer = fmt.Sprintf(`{"status": %v, "reason": "%v"}`, status_code, http.StatusText(status_code))
        resp.WriteHeader(status_code)
    }
    resp.Write([]byte(request_answer))
}

func main() {
    var err error = nil
    config, err = loadConfig()
    if err != nil {
        makePanic(fmt.Sprintf("Can't load config: %v", err))
    } else {
        reg_filter, err = regexp.Compile(config.Reg_filter)
        if err != nil {
            makePanic(fmt.Sprintf("Can`t compile regular expression: %v", err))
        }
        _, err = os.Open(config.Mystem_path)
        if  err != nil {
            makePanic(fmt.Sprintf("Can't find mystem: %v", err))
        }
        var i int
        if config.Channel_buffer > 0 {
            for_process = make(chan Data, int(config.Channel_buffer))
        } else {
            for_process = make(chan Data)
        }
        log.Print("Config load successful")
        for i = 0; i < int(config.Mystem_workers); i++ {
            go workerMystem(for_process, config.Mystem_path)
        }
        http.HandleFunc("/", processWords)
        log.Printf("Server start on: %v:%v", config.Host, config.Port)
        err = http.ListenAndServe(fmt.Sprintf("%v:%v", config.Host, config.Port), nil)
        if err != nil {
            makePanic(fmt.Sprintf("Can't start http server: %v", err))
        }
    }
}<commit_msg>rename package; rename function processMystemOptions to ProcessMystemOptions for tests<commit_after>package http_mystem

import (
    "fmt"
    "net/http"
    "os"
    "os/exec"
    "io"
    "encoding/json"
    "log"
    //"time"
    "strings"
    //"math/rand"
    "regexp"
)

var HEADER_CONTENT_TYPES map[string]string = map[string]string{
    "json": "javascript/json;charset=utf-8",
    "text": "text/plain;charset=utf-8",
    "xml": "text/xml;charset=utf-8",
}

var IGNORE_OPTIONS map[string]bool = map[string]bool{
    "-e": true,
    "-c": true,
    "-d": true,
}

const DEFAULT_FORMAT string = "text"

type Config struct {
    Host string
    Port int
    Reg_filter string
    Mystem_path string
    Mystem_options []string
    Mystem_workers int
    Mystem_answer_size int
    Channel_buffer int
    Max_word_length int
    Max_words int
    format string
    is_d bool
}

type Answer struct {
    data string
    err error
}

type Data struct {
    word string
    channel chan Answer
}

var for_process chan Data
var reg_filter *regexp.Regexp
var config Config

func makePanic(msg string) {
    log.Fatal(msg)
    panic(msg)
}

func ProcessMystemOptions(in_opts []string) (out_opts []string, format string, err error) {
    var opts_arr []string
    var approved bool = false
    var exists bool = false
    var n_exists = false
    for _, opt := range in_opts {
        approved = false
        opt = strings.TrimSpace(opt)
        opt = strings.ToLower(opt)
        if strings.Index(opt, " ") == -1 {
            opts_arr = make([]string, 1)
            opts_arr[0] = opt
            if !bool(IGNORE_OPTIONS[opt]) {
                approved = true
            }
            if (!n_exists) && (opt == "-n") {
                n_exists = true
            }
        } else {
            opts_arr = make([]string, 2)
            opts_arr = strings.Split(opt, " ")
            if opts_arr[0] == "--format" {
                _, exists = HEADER_CONTENT_TYPES[opts_arr[1]]
                if exists {
                    format = opts_arr[1]
                } else {
                    format = DEFAULT_FORMAT
                    opts_arr[1] = format
                    log.Printf("Mystem option '%v' was replaced by default value. Because contain error", opt)
                }
                approved = true
            } else if !bool(IGNORE_OPTIONS[opts_arr[0]]) {
                approved = true
            }
        }
        if approved {
            out_opts = append(out_opts, opts_arr...)
        } else {
            log.Printf("Mystem option '%v' was ignored or replaced by default value. Because wrong or unnecessary", opt)
        }
    }
    if !n_exists {
        out_opts = append(out_opts, "-n")
        log.Print("Added mystem option '-n'. It need for normal work.\n")
    }
    if format == "" {
        format = DEFAULT_FORMAT
    }
    return out_opts, format, err
}

func loadConfig() (cfg Config, err error) {
    var f_name string = "http_mystem.json"
    if (len(os.Args) > 1) {
        f_name = os.Args[1]
    }
    file, err := os.Open(f_name)
    defer file.Close()
    if err == nil {
        f_stat, _ := file.Stat()
        raw_json := make([]byte, f_stat.Size())
        _, err = file.Read(raw_json)
        if err == nil {
            err = json.Unmarshal(raw_json, &cfg)
            if err == nil {
                cfg.Mystem_options, cfg.format, err = ProcessMystemOptions(cfg.Mystem_options)
                //cfg.Mystem_answer_size = cfg.Max_word_length * 500
            }
        }
    }
    return cfg, err
}

func writeStringToPipe(for_write string, pipe io.WriteCloser) (n int, err error) {
    defer func() {
        result := recover()
        if result != nil {
            log.Fatalf("Recover say: %v", result)
        }
    }()
    n, err = pipe.Write([]byte(for_write + "\n"))
    if err != nil {
        log.Panicf("Can't send word to mystem: %v", err)
    }
    return n, err
}

func readStringFromPipe(for_read *string, pipe io.ReadCloser) (n int, err error) {
    defer func() {
        result := recover()
        if result != nil {
            log.Fatalf("Recover say: %v", result)
        }
    }()
    buf := make([]byte, config.Mystem_answer_size)
    n, err = pipe.Read(buf)
    if err == nil {
        *for_read = strings.TrimSpace(string(buf[:n]))
    } else {
        log.Panicf("Can't send word to mystem: %v", err)
    }
    return n, err
}

func readXMLTrash(number int, pipe io.ReadCloser) {
    var data string = ""
    var err error
    for i := 0; i < number; i++ {
        for data == "" {
            _, err = readStringFromPipe(&data, pipe)
            if err != nil {
                makePanic(fmt.Sprintf("Can't start: %v", err))
            }
        }
        data = ""
    }
}

func workerMystem(for_process chan Data, mystem_path string) {
    //name := rand.Int()
    //log.Print("Mystem started ", name)
    mystem := exec.Command(mystem_path, config.Mystem_options...)
    mystem_writer, err_w := mystem.StdinPipe()
    mystem_reader, err_r := mystem.StdoutPipe()
    err := mystem.Start()
    if (err_w != nil) || (err_r != nil) {
        makePanic(fmt.Sprintf("Can't start: \n%v\n%v", err_w, err_r))
    }
    if err != nil {
        makePanic(fmt.Sprintf("Can't start: %v", err))
    } else {
        var data Data
        var answer Answer
        if config.format == "xml" {
            readXMLTrash(2, mystem_reader)
            _, err = writeStringToPipe("тест", mystem_writer)
            if err != nil {
                panic("")
            }
            readXMLTrash(2, mystem_reader)
        } else {
            _, err = writeStringToPipe("тест", mystem_writer)
            if err != nil {
                panic("")
            }
            _, err = readStringFromPipe(&answer.data, mystem_reader)
            if err != nil {
                panic("")
            }
        }
        for {
            data = <- for_process
            _, err = writeStringToPipe(data.word, mystem_writer)
            answer.err = err
            if config.format != "xml" {
                _, err = readStringFromPipe(&answer.data, mystem_reader)
                if err != nil {
                    answer.err = err
                    break
                }
            } else {
                for answer.data == "" {
                    _, err = readStringFromPipe(&answer.data, mystem_reader)
                    if err != nil {
                        answer.err = err
                        break
                    }
                }
            }
            data.channel <- answer
            answer.data = ""
            answer.err = nil
        }
    }
}

func checkInput(req *http.Request, words *[]string, ) (code int) {
    var words_count int = len(req.Form["words[]"])
    if words_count == 0 {
        code = http.StatusNotAcceptable
    } else if words_count > int(config.Max_words) {
        code = http.StatusRequestEntityTooLarge
    } else {
        var w string = ""
        for _, word := range req.Form["words[]"] {
            w = strings.TrimSpace(word)
            if len(w) > int(config.Max_word_length) {
                w = word[:int(config.Max_word_length)]
            }
            if w != "" {
                *words = append(*words, w)
            }
        }
    }
    return code
}

func processWords(resp http.ResponseWriter, req *http.Request) {
    var data Data
    var answer Answer
    var answers []string
    var request_answer string = ""
    req.ParseForm()
    var words []string
    var tmp_answer string = ""
    status_code := checkInput(req, &words)
    var words_count int = len(words)
    resp.Header().Set("Content-Type", HEADER_CONTENT_TYPES[config.format])
    //resp.Header().Set("Content-Type", "text/plain;charset=utf-8")
    if words_count > 0 {
        var err error
        var local_channel chan Answer = make(chan Answer, words_count)
        data.channel = local_channel
        var start int = 0
        for _, word := range words {
            err = nil
            if reg_filter.MatchString(word) {
                data.word = word
                for_process <- data
            } else {
                start++
                switch config.format {
                    case "json": {
                        tmp_answer = fmt.Sprintf(`{"analysis":[], "text":"%v"}`, word)
                    }
                    case "xml": {
                        tmp_answer = "<w>" + word + "</w>"
                    }
                    case "text": {
                        tmp_answer = word
                    }
                }
                answers = append(answers, tmp_answer)
            }
        }
        for i := start; i < words_count; i++ {
            answer = <- local_channel
            if answer.err != nil {
                err = answer.err
            }
            answers = append(answers, answer.data)
        }
        if err == nil {
            switch config.format {
                case "json": {
                    request_answer = "[" + strings.Join(answers, ",") + "]"
                }
                case "xml": {
                    request_answer = `<?xml version="1.0" encoding="utf-8"?><html><body><se>` + strings.Join(answers, "\n") + "</se>"
                }
                case "text": {
                    request_answer = strings.Join(answers, "\n")
                }
            }
        } else {
            //request_answer = fmt.Sprintf(`{"status": %v, "reason": "%v"}`, status_code, http.StatusText(http.StatusInternalServerError)
            resp.WriteHeader(http.StatusInternalServerError)
        }
        close(local_channel)
    } else {
        //request_answer = fmt.Sprintf(`{"status": %v, "reason": "%v"}`, status_code, http.StatusText(status_code))
        resp.WriteHeader(status_code)
    }
    resp.Write([]byte(request_answer))
}

func main() {
    var err error = nil
    config, err = loadConfig()
    if err != nil {
        makePanic(fmt.Sprintf("Can't load config: %v", err))
    } else {
        reg_filter, err = regexp.Compile(config.Reg_filter)
        if err != nil {
            makePanic(fmt.Sprintf("Can`t compile regular expression: %v", err))
        }
        _, err = os.Open(config.Mystem_path)
        if  err != nil {
            makePanic(fmt.Sprintf("Can't find mystem: %v", err))
        }
        var i int
        if config.Channel_buffer > 0 {
            for_process = make(chan Data, int(config.Channel_buffer))
        } else {
            for_process = make(chan Data)
        }
        log.Print("Config load successful")
        for i = 0; i < int(config.Mystem_workers); i++ {
            go workerMystem(for_process, config.Mystem_path)
        }
        http.HandleFunc("/", processWords)
        log.Printf("Server start on: %v:%v", config.Host, config.Port)
        err = http.ListenAndServe(fmt.Sprintf("%v:%v", config.Host, config.Port), nil)
        if err != nil {
            makePanic(fmt.Sprintf("Can't start http server: %v", err))
        }
    }
}<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"syscall"
)

func main() {
	if len(os.Args) != 2 {
		fmt.Printf("usage: %s /path", os.Args[0])
		os.Exit(1)
	}

	fd, err := syscall.Open(os.Args[1], syscall.O_RDONLY, 0)
	if err != nil {
		panic(err)
	}

	kq, err := syscall.Kqueue()
	if err != nil {
		fmt.Println(err)
	}

	ev1 := syscall.Kevent_t{
		Ident:  uint64(fd),
		Filter: syscall.EVFILT_VNODE,
		Flags:  syscall.EV_ADD | syscall.EV_ENABLE | syscall.EV_ONESHOT,
		Fflags: syscall.NOTE_DELETE | syscall.NOTE_WRITE | syscall.NOTE_EXTEND | syscall.NOTE_ATTRIB | syscall.NOTE_LINK | syscall.NOTE_RENAME | syscall.NOTE_REVOKE,
		Data:   0,
		Udata:  nil,
	}

	//Loop:
	// wait for events
	for {
		// create kevent
		events := make([]syscall.Kevent_t, 10)
		n, err := syscall.Kevent(kq, []syscall.Kevent_t{ev1}, events, nil)
		if err != nil {
			log.Println("Error creating kevent")
		}
		// check if there was an event
		for i := 0; i < n; i++ {
			log.Printf("Event [%d] -> %+v data: %#v", i, events[i], events[i].Data)

			if events[i].Fflags == syscall.NOTE_RENAME {
				print("renamed")
			}
			// Fflags:17
			if events[i].Fflags == syscall.NOTE_DELETE || events[i].Fflags == 17 {
				print("deleted")
				//				break Loop
			}
			if n > 0 {
				return
			}
		}
	}

	fmt.Println("fin")
}
<commit_msg>	modified:   main.go<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"syscall"
)

func main() {
	if len(os.Args) != 2 {
		fmt.Printf("usage: %s /path", os.Args[0])
		os.Exit(1)
	}

	file, err := os.Open(os.Args[1])
	if err != nil {
		panic(err)
	}

	kq, err := syscall.Kqueue()
	if err != nil {
		fmt.Println(err)
	}

	ev1 := syscall.Kevent_t{
		Ident:  uint64(file.Fd()),
		Filter: syscall.EVFILT_VNODE,
		Flags:  syscall.EV_ADD | syscall.EV_ENABLE | syscall.EV_ONESHOT,
		Fflags: syscall.NOTE_DELETE | syscall.NOTE_WRITE | syscall.NOTE_EXTEND | syscall.NOTE_ATTRIB | syscall.NOTE_LINK | syscall.NOTE_RENAME | syscall.NOTE_REVOKE,
		Data:   0,
		Udata:  nil,
	}

	//Loop:
	// wait for events
	for {
		// create kevent
		events := make([]syscall.Kevent_t, 10)
		n, err := syscall.Kevent(kq, []syscall.Kevent_t{ev1}, events, nil)
		if err != nil {
			log.Println("Error creating kevent")
		}
		// check if there was an event
		for i := 0; i < n; i++ {
			log.Printf("Event [%d] -> %+v data: %#v", i, events[i], events[i].Data)

			if events[i].Fflags == syscall.NOTE_RENAME {
				print("renamed")
			}
			// Fflags:17
			if events[i].Fflags == syscall.NOTE_DELETE || events[i].Fflags == 17 {
				print("deleted")
				//				break Loop
			}
			if n > 0 {
				fmt.Printf("n = %+v\n", n)
				return
			}
		}
	}

	fmt.Println("fin")
}
<|endoftext|>


<commit_before>// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan.
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/

// See page 83.

// The sha256 command computes the SHA256 hash (an array) of a string.
package main

import (
	"crypto/sha256" //!+
	"fmt"
	"math/bits"
)

func main() {
	c1 := sha256.Sum256([]byte("x"))
	c2 := sha256.Sum256([]byte("X"))
	fmt.Printf("%x\n%x\n%t\n%T\n", c1, c2, c1 == c2, c1)
	for i, n := range c1 {
		fmt.Printf("%08b %08b\n", n, c2[i]) // prints 00000000 11111101
	}
	fmt.Println(matchCount(c1, c2))
	// Output:
	// 2d711642b726b04401627ca9fbac32f5c8530fb1903cc4db02258717921a4881
	// 4b68ab3847feda7d6c62c1fbcbeebfa35eab7351ed5e78f4ddadea5df64b8015
	// false
	// [32]uint8
}

func matchCount(a, b [32]byte) int {
	count := 0
	for i, aByte := range a {
		bByte := b[i]
		count += bits.OnesCount8(^(aByte ^ bByte))
	}
	return count
}

//!-
<commit_msg>Add exercises 4.1 and 4.2<commit_after>// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan.
// License: https://creativecommons.org/licenses/by-nc-sa/4.0/

// See page 83.

// The sha256 command computes the SHA256 hash (an array) of a string.
package main

import (
	"crypto/sha256" //!+
	"crypto/sha512"
	"fmt"
	"math/bits"
	"os"
)

const (
	MATCH = "-m"
	HASH  = "-h"
)

func main() {
	if len(os.Args) < 2 {
		fmt.Println("Incorrect arguments")
		return
	}
	function := os.Args[1]

	if function == MATCH {
		fmt.Println(matchCount(os.Args[2], os.Args[3]))
	} else if function == HASH {
		hash(os.Args[2], os.Args[3])
	}
}

func matchCount(x, y string) int {
	a := sha256.Sum256([]byte(x))
	b := sha256.Sum256([]byte(y))
	count := 0
	for i, aByte := range a {
		bByte := b[i]
		count += bits.OnesCount8(^(aByte ^ bByte))
	}
	return count
}

func hash(flag, x string) {
	if flag == "256" {
		fmt.Println(sha256.Sum256([]byte(x)))
	} else if flag == "384" {
		fmt.Println(sha512.Sum384([]byte(x)))
	} else if flag == "512" {
		fmt.Println(sha512.Sum512([]byte(x)))
	}
}

//!-
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/streadway/amqp"
	"koding/tools/log"
	"koding/tools/sockjs"
	"koding/tools/utils"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

func main() {
	utils.Startup("broker", false)
	utils.RunStatusLogger()

	utils.AmqpAutoReconnect("broker", func(consumeConn, publishConn *amqp.Connection) {
		consumeChannel := utils.CreateAmqpChannel(consumeConn)
		defer consumeChannel.Close()

		routeMap := make(map[string]([]chan<- interface{}))
		var routeMapMutex sync.RWMutex

		service := sockjs.NewService("http://localhost/sockjs.js", true, false, 10*time.Minute, 0, func(receiveChan <-chan interface{}, sendChan chan<- interface{}) {
			defer log.RecoverAndLog()

			r := make([]byte, 128)
			rand.Read(r)
			socketId := base64.StdEncoding.EncodeToString(r)

			utils.ChangeNumClients <- 1
			log.Debug("Client connected: " + socketId)
			defer func() {
				utils.ChangeNumClients <- -1
				log.Debug("Client disconnected: " + socketId)
			}()

			addToRouteMap := func(routingKeyPrefix string) {
				routeMapMutex.Lock()
				routeMap[routingKeyPrefix] = append(routeMap[routingKeyPrefix], sendChan)
				routeMapMutex.Unlock()
			}
			removeFromRouteMap := func(routingKeyPrefix string) {
				routeMapMutex.Lock()
				channels := routeMap[routingKeyPrefix]
				for i, channel := range channels {
					if channel == sendChan {
						channels[i] = channels[len(channels)-1]
						routeMap[routingKeyPrefix] = channels[:len(channels)-1]
						break
					}
				}
				routeMapMutex.Unlock()
			}

			subscriptions := make(map[string]bool)

			controlChannel := utils.CreateAmqpChannel(publishConn)
			defer func() { controlChannel.Close() }() // controlChannel is replaced on error

			err := controlChannel.Publish("authEndpoint", "broker.clientConnected", false, false, amqp.Publishing{Body: []byte(socketId)})
			if err != nil {
				panic(err)
			}

			defer func() {
				for routingKeyPrefix := range subscriptions {
					removeFromRouteMap(routingKeyPrefix)
				}
				err := controlChannel.Publish("authEndpoint", "broker.clientDisconnected", false, false, amqp.Publishing{Body: []byte(socketId)})
				if err != nil {
					panic(err)
				}
			}()

			for data := range receiveChan {
				func() {
					defer func() {
						err := recover()
						if err != nil {
							log.LogError(err)
							controlChannel.Close()
							controlChannel = utils.CreateAmqpChannel(publishConn)
						}
					}()

					var message map[string]string
					err := json.Unmarshal([]byte(data.(string)), &message)
					if err != nil {
						panic(err)
					}
					log.Debug(message)

					action := message["action"]
					switch action {
					case "subscribe":
						routingKeyPrefix := message["routingKeyPrefix"]
						addToRouteMap(routingKeyPrefix)
						subscriptions[routingKeyPrefix] = true

						body, err := json.Marshal(map[string]string{"event": "subscribed", "routingKeyPrefix": routingKeyPrefix})
						if err != nil {
							panic(err)
						}
						sendChan <- string(body)

					case "unsubscribe":
						routingKeyPrefix := message["routingKeyPrefix"]
						removeFromRouteMap(routingKeyPrefix)
						delete(subscriptions, routingKeyPrefix)

					case "publish":
						exchange := message["exchange"]
						routingKey := message["routingKey"]
						if strings.HasPrefix(routingKey, "client.") {
							err := controlChannel.Publish(exchange, routingKey, false, false, amqp.Publishing{CorrelationId: socketId, Body: []byte(message["payload"])})
							if err != nil {
								panic(err)
							}
						} else {
							log.Warn(fmt.Sprintf("Invalid routing key: %v", message))
						}

					default:
						log.Warn(fmt.Sprintf("Invalid action: %v", message))

					}
				}()
			}
		})
		defer service.Close()

		server := &http.Server{Handler: &sockjs.Mux{
			Services: map[string]*sockjs.Service{
				"/subscribe": service,
			},
		}}
		listener, err := net.Listen("tcp", ":8008")
		if err != nil {
			panic(err)
		}
		defer listener.Close()

		go func() {
			err = server.Serve(listener)
			if err != nil {
				log.Warn("Server error: " + err.Error())
			}
			consumeConn.Close()
		}()

		stream := utils.DeclareBindConsumeAmqpQueue(consumeChannel, "topic", "broker", "#")
		for message := range stream {
			routingKey := message.RoutingKey
			body, err := json.Marshal(map[string]string{"event": routingKey, "exchange": message.Exchange, "payload": string(message.Body)})
			if err != nil {
				panic(err)
			}
			bodyStr := string(body)

			pos := strings.IndexRune(routingKey, '.') // skip first dot, since we want at least two components to always include the secret
			for {
				index := strings.IndexRune(routingKey[pos+1:], '.')
				if index == -1 {
					break
				}
				pos += 1 + index
				prefix := routingKey[:pos]
				routeMapMutex.RLock()
				channels := routeMap[prefix]
				routeMapMutex.RUnlock()
				for _, channel := range channels {
					select {
					case channel <- bodyStr:
						// successful
					default:
						log.Warn("Dropped message")
					}
				}
			}
		}
	})
}
<commit_msg>Removed one layer of JSON in broker.<commit_after>package main

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/streadway/amqp"
	"koding/tools/log"
	"koding/tools/sockjs"
	"koding/tools/utils"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

func main() {
	utils.Startup("broker", false)
	utils.RunStatusLogger()

	utils.AmqpAutoReconnect("broker", func(consumeConn, publishConn *amqp.Connection) {
		consumeChannel := utils.CreateAmqpChannel(consumeConn)
		defer consumeChannel.Close()

		routeMap := make(map[string]([]chan<- interface{}))
		var routeMapMutex sync.RWMutex

		service := sockjs.NewService("http://localhost/sockjs.js", true, false, 10*time.Minute, 0, func(receiveChan <-chan interface{}, sendChan chan<- interface{}) {
			defer log.RecoverAndLog()

			r := make([]byte, 128)
			rand.Read(r)
			socketId := base64.StdEncoding.EncodeToString(r)

			utils.ChangeNumClients <- 1
			log.Debug("Client connected: " + socketId)
			defer func() {
				utils.ChangeNumClients <- -1
				log.Debug("Client disconnected: " + socketId)
			}()

			addToRouteMap := func(routingKeyPrefix string) {
				routeMapMutex.Lock()
				routeMap[routingKeyPrefix] = append(routeMap[routingKeyPrefix], sendChan)
				routeMapMutex.Unlock()
			}
			removeFromRouteMap := func(routingKeyPrefix string) {
				routeMapMutex.Lock()
				channels := routeMap[routingKeyPrefix]
				for i, channel := range channels {
					if channel == sendChan {
						channels[i] = channels[len(channels)-1]
						routeMap[routingKeyPrefix] = channels[:len(channels)-1]
						break
					}
				}
				routeMapMutex.Unlock()
			}

			subscriptions := make(map[string]bool)

			controlChannel := utils.CreateAmqpChannel(publishConn)
			defer func() { controlChannel.Close() }() // controlChannel is replaced on error

			err := controlChannel.Publish("authEndpoint", "broker.clientConnected", false, false, amqp.Publishing{Body: []byte(socketId)})
			if err != nil {
				panic(err)
			}

			defer func() {
				for routingKeyPrefix := range subscriptions {
					removeFromRouteMap(routingKeyPrefix)
				}
				err := controlChannel.Publish("authEndpoint", "broker.clientDisconnected", false, false, amqp.Publishing{Body: []byte(socketId)})
				if err != nil {
					panic(err)
				}
			}()

			for data := range receiveChan {
				func() {
					defer func() {
						err := recover()
						if err != nil {
							log.LogError(err)
							controlChannel.Close()
							controlChannel = utils.CreateAmqpChannel(publishConn)
						}
					}()

					message := data.(map[string]interface{})
					log.Debug(message)

					action := message["action"]
					switch action {
					case "subscribe":
						routingKeyPrefix := message["routingKeyPrefix"].(string)
						addToRouteMap(routingKeyPrefix)
						subscriptions[routingKeyPrefix] = true

						body, err := json.Marshal(map[string]string{"event": "subscribed", "routingKeyPrefix": routingKeyPrefix})
						if err != nil {
							panic(err)
						}
						sendChan <- string(body)

					case "unsubscribe":
						routingKeyPrefix := message["routingKeyPrefix"].(string)
						removeFromRouteMap(routingKeyPrefix)
						delete(subscriptions, routingKeyPrefix)

					case "publish":
						exchange := message["exchange"].(string)
						routingKey := message["routingKey"].(string)
						if strings.HasPrefix(routingKey, "client.") {
							err := controlChannel.Publish(exchange, routingKey, false, false, amqp.Publishing{CorrelationId: socketId, Body: []byte(message["payload"].(string))})
							if err != nil {
								panic(err)
							}
						} else {
							log.Warn(fmt.Sprintf("Invalid routing key: %v", message))
						}

					default:
						log.Warn(fmt.Sprintf("Invalid action: %v", message))

					}
				}()
			}
		})
		defer service.Close()

		server := &http.Server{Handler: &sockjs.Mux{
			Services: map[string]*sockjs.Service{
				"/subscribe": service,
			},
		}}
		listener, err := net.Listen("tcp", ":8008")
		if err != nil {
			panic(err)
		}
		defer listener.Close()

		go func() {
			err = server.Serve(listener)
			if err != nil {
				log.Warn("Server error: " + err.Error())
			}
			consumeConn.Close()
		}()

		stream := utils.DeclareBindConsumeAmqpQueue(consumeChannel, "topic", "broker", "#")
		for message := range stream {
			routingKey := message.RoutingKey
			body, err := json.Marshal(map[string]string{"event": routingKey, "exchange": message.Exchange, "payload": string(message.Body)})
			if err != nil {
				panic(err)
			}
			bodyStr := string(body)

			pos := strings.IndexRune(routingKey, '.') // skip first dot, since we want at least two components to always include the secret
			for {
				index := strings.IndexRune(routingKey[pos+1:], '.')
				if index == -1 {
					break
				}
				pos += 1 + index
				prefix := routingKey[:pos]
				routeMapMutex.RLock()
				channels := routeMap[prefix]
				routeMapMutex.RUnlock()
				for _, channel := range channels {
					select {
					case channel <- bodyStr:
						// successful
					default:
						log.Warn("Dropped message")
					}
				}
			}
		}
	})
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"github.com/streadway/amqp"
	"koding/tools/amqputil"
	"koding/tools/config"
	"koding/tools/lifecycle"
	"koding/tools/log"
	"koding/tools/sockjs"
	"koding/tools/utils"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"
)

func main() {
	lifecycle.Startup("broker", false)
	changeClientsGauge := lifecycle.CreateClientsGauge()
	changeNewClientsGauge := log.CreateCounterGauge("newClients", true)
	changeWebsocketClientsGauge := log.CreateCounterGauge("websocketClients", false)
	log.RunGaugesLoop()

	publishConn := amqputil.CreateConnection("broker")
	defer publishConn.Close()

	routeMap := make(map[string]([]*sockjs.Session))
	var routeMapMutex sync.Mutex

	service := sockjs.NewService(config.Current.Client.StaticFilesBaseUrl+"/js/sock.js", 10*time.Minute, func(session *sockjs.Session) {
		defer log.RecoverAndLog()

		r := make([]byte, 128/8)
		rand.Read(r)
		socketId := base64.StdEncoding.EncodeToString(r)
		session.Tag = socketId

		log.Debug("Client connected: " + socketId)
		changeClientsGauge(1)
		changeNewClientsGauge(1)
		if session.IsWebsocket {
			changeWebsocketClientsGauge(1)
		}
		defer func() {
			log.Debug("Client disconnected: " + socketId)
			changeClientsGauge(-1)
			if session.IsWebsocket {
				changeWebsocketClientsGauge(-1)
			}
		}()

		addToRouteMap := func(routingKeyPrefix string) {
			routeMapMutex.Lock()
			defer routeMapMutex.Unlock()
			routeMap[routingKeyPrefix] = append(routeMap[routingKeyPrefix], session)
		}
		removeFromRouteMap := func(routingKeyPrefix string) {
			routeMapMutex.Lock()
			defer routeMapMutex.Unlock()
			routeSessions := routeMap[routingKeyPrefix]
			for i, routeSession := range routeSessions {
				if routeSession == session {
					routeSessions[i] = routeSessions[len(routeSessions)-1]
					routeSessions = routeSessions[:len(routeSessions)-1]
					break
				}
			}
			if len(routeSessions) == 0 {
				delete(routeMap, routingKeyPrefix)
				return
			}
			routeMap[routingKeyPrefix] = routeSessions
		}

		subscriptions := make(map[string]bool)

		var controlChannel *amqp.Channel
		var lastPayload string
		resetControlChannel := func() {
			if controlChannel != nil {
				controlChannel.Close()
			}
			var err error
			controlChannel, err = publishConn.Channel()
			if err != nil {
				panic(err)
			}
			go func() {
				defer log.RecoverAndLog()

				for amqpErr := range controlChannel.NotifyClose(make(chan *amqp.Error)) {
					log.Warn("AMQP channel: "+amqpErr.Error(), "Last publish payload:", lastPayload)

					session.Send(map[string]interface{}{"routingKey": "broker.error", "code": amqpErr.Code, "reason": amqpErr.Reason, "server": amqpErr.Server, "recover": amqpErr.Recover})
				}
			}()
		}
		resetControlChannel()
		defer func() { controlChannel.Close() }()

		err := controlChannel.Publish("auth", "broker.clientConnected", false, false, amqp.Publishing{Body: []byte(socketId)})
		if err != nil {
			panic(err)
		}

		defer func() {
			for routingKeyPrefix := range subscriptions {
				removeFromRouteMap(routingKeyPrefix)
			}
			for {
				err := controlChannel.Publish("auth", "broker.clientDisconnected", false, false, amqp.Publishing{Body: []byte(socketId)})
				if err == nil {
					break
				}
				if amqpError, isAmqpError := err.(*amqp.Error); !isAmqpError || amqpError.Code != 504 {
					panic(err)
				}
				resetControlChannel()
			}
		}()

		for data := range session.ReceiveChan {
			func() {
				defer log.RecoverAndLog()

				message := data.(map[string]interface{})
				log.Debug("Received message", message)

				action := message["action"]
				switch action {
				case "subscribe":
					routingKeyPrefix := message["routingKeyPrefix"].(string)
					addToRouteMap(routingKeyPrefix)
					subscriptions[routingKeyPrefix] = true
					session.Send(map[string]string{"routingKey": "broker.subscribed", "payload": routingKeyPrefix})

				case "unsubscribe":
					routingKeyPrefix := message["routingKeyPrefix"].(string)
					removeFromRouteMap(routingKeyPrefix)
					delete(subscriptions, routingKeyPrefix)

				case "publish":
					exchange := message["exchange"].(string)
					routingKey := message["routingKey"].(string)
					if !strings.HasPrefix(routingKey, "client.") {
						log.Warn("Invalid routing key.", message, socketId)
						return
					}
					for {
						lastPayload = ""
						err := controlChannel.Publish(exchange, routingKey, false, false, amqp.Publishing{CorrelationId: socketId, Body: []byte(message["payload"].(string))})
						if err == nil {
							lastPayload = message["payload"].(string)
							break
						}
						if amqpError, isAmqpError := err.(*amqp.Error); !isAmqpError || amqpError.Code != 504 {
							panic(err)
						}
						resetControlChannel()
					}

				default:
					log.Warn("Invalid action.", message, socketId)

				}
			}()
		}
	})
	defer service.Close()
	service.PanicHandler = log.RecoverAndLog

	go func() {
		server := &http.Server{
			Handler: &sockjs.Mux{
				Services: map[string]*sockjs.Service{
					"/subscribe": service,
				},
			},
		}

		var listener net.Listener
		listener, err := net.ListenTCP("tcp", &net.TCPAddr{nil, config.Current.Broker.Port})
		if err != nil {
			log.LogError(err, 0)
			os.Exit(1)
		}

		if config.Current.Broker.CertFile != "" {
			cert, err := tls.LoadX509KeyPair(config.Current.Broker.CertFile, config.Current.Broker.KeyFile)
			if err != nil {
				log.LogError(err, 0)
				os.Exit(1)
			}
			listener = tls.NewListener(listener, &tls.Config{
				NextProtos:   []string{"http/1.1"},
				Certificates: []tls.Certificate{cert},
			})
		}

		go func() {
			sigtermChannel := make(chan os.Signal)
			signal.Notify(sigtermChannel, syscall.SIGTERM)
			<-sigtermChannel
			lifecycle.BeginShutdown()
			listener.Close()
		}()

		lastErrorTime := time.Now()
		for {
			err := server.Serve(listener)
			if lifecycle.ShuttingDown {
				break
			}
			if err != nil {
				log.Warn("Server error: " + err.Error())
				if time.Now().Sub(lastErrorTime) < time.Second {
					os.Exit(1)
				}
				lastErrorTime = time.Now()
			}
		}
	}()

	consumeConn := amqputil.CreateConnection("broker")
	defer consumeConn.Close()

	consumeChannel := amqputil.CreateChannel(consumeConn)
	defer consumeChannel.Close()

	stream := amqputil.DeclareBindConsumeQueue(consumeChannel, "topic", "broker", "#", false)
	if err := consumeChannel.ExchangeDeclare("updateInstances", "fanout", false, false, false, false, nil); err != nil {
		panic(err)
	}
	if err := consumeChannel.ExchangeBind("broker", "", "updateInstances", false, nil); err != nil {
		panic(err)
	}

	for amqpMessage := range stream {
		routingKey := amqpMessage.RoutingKey
		payload := json.RawMessage(utils.FilterInvalidUTF8(amqpMessage.Body))
		jsonMessage := map[string]interface{}{"routingKey": routingKey, "payload": &payload}

		pos := strings.IndexRune(routingKey, '.') // skip first dot, since we want at least two components to always include the secret
		for pos != -1 && pos < len(routingKey) {
			index := strings.IndexRune(routingKey[pos+1:], '.')
			pos += index + 1
			if index == -1 {
				pos = len(routingKey)
			}
			prefix := routingKey[:pos]
			routeMapMutex.Lock()
			routeSessions := routeMap[prefix]
			for _, routeSession := range routeSessions {
				if !routeSession.Send(jsonMessage) {
					routeSession.Close()
					log.Warn("Dropped session because of broker to client buffer overflow.", routeSession.Tag)
				}
			}
			routeMapMutex.Unlock()
		}
	}
}
<commit_msg>broker: ping/pong<commit_after>package main

import (
	"crypto/rand"
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"github.com/streadway/amqp"
	"koding/tools/amqputil"
	"koding/tools/config"
	"koding/tools/lifecycle"
	"koding/tools/log"
	"koding/tools/sockjs"
	"koding/tools/utils"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"
)

func main() {
	lifecycle.Startup("broker", false)
	changeClientsGauge := lifecycle.CreateClientsGauge()
	changeNewClientsGauge := log.CreateCounterGauge("newClients", true)
	changeWebsocketClientsGauge := log.CreateCounterGauge("websocketClients", false)
	log.RunGaugesLoop()

	publishConn := amqputil.CreateConnection("broker")
	defer publishConn.Close()

	routeMap := make(map[string]([]*sockjs.Session))
	var routeMapMutex sync.Mutex

	service := sockjs.NewService(config.Current.Client.StaticFilesBaseUrl+"/js/sock.js", 10*time.Minute, func(session *sockjs.Session) {
		defer log.RecoverAndLog()

		r := make([]byte, 128/8)
		rand.Read(r)
		socketId := base64.StdEncoding.EncodeToString(r)
		session.Tag = socketId

		log.Debug("Client connected: " + socketId)
		changeClientsGauge(1)
		changeNewClientsGauge(1)
		if session.IsWebsocket {
			changeWebsocketClientsGauge(1)
		}
		defer func() {
			log.Debug("Client disconnected: " + socketId)
			changeClientsGauge(-1)
			if session.IsWebsocket {
				changeWebsocketClientsGauge(-1)
			}
		}()

		addToRouteMap := func(routingKeyPrefix string) {
			routeMapMutex.Lock()
			defer routeMapMutex.Unlock()
			routeMap[routingKeyPrefix] = append(routeMap[routingKeyPrefix], session)
		}
		removeFromRouteMap := func(routingKeyPrefix string) {
			routeMapMutex.Lock()
			defer routeMapMutex.Unlock()
			routeSessions := routeMap[routingKeyPrefix]
			for i, routeSession := range routeSessions {
				if routeSession == session {
					routeSessions[i] = routeSessions[len(routeSessions)-1]
					routeSessions = routeSessions[:len(routeSessions)-1]
					break
				}
			}
			if len(routeSessions) == 0 {
				delete(routeMap, routingKeyPrefix)
				return
			}
			routeMap[routingKeyPrefix] = routeSessions
		}

		subscriptions := make(map[string]bool)

		var controlChannel *amqp.Channel
		var lastPayload string
		resetControlChannel := func() {
			if controlChannel != nil {
				controlChannel.Close()
			}
			var err error
			controlChannel, err = publishConn.Channel()
			if err != nil {
				panic(err)
			}
			go func() {
				defer log.RecoverAndLog()

				for amqpErr := range controlChannel.NotifyClose(make(chan *amqp.Error)) {
					log.Warn("AMQP channel: "+amqpErr.Error(), "Last publish payload:", lastPayload)

					session.Send(map[string]interface{}{"routingKey": "broker.error", "code": amqpErr.Code, "reason": amqpErr.Reason, "server": amqpErr.Server, "recover": amqpErr.Recover})
				}
			}()
		}
		resetControlChannel()
		defer func() { controlChannel.Close() }()

		err := controlChannel.Publish("auth", "broker.clientConnected", false, false, amqp.Publishing{Body: []byte(socketId)})
		if err != nil {
			panic(err)
		}

		defer func() {
			for routingKeyPrefix := range subscriptions {
				removeFromRouteMap(routingKeyPrefix)
			}
			for {
				err := controlChannel.Publish("auth", "broker.clientDisconnected", false, false, amqp.Publishing{Body: []byte(socketId)})
				if err == nil {
					break
				}
				if amqpError, isAmqpError := err.(*amqp.Error); !isAmqpError || amqpError.Code != 504 {
					panic(err)
				}
				resetControlChannel()
			}
		}()

		for data := range session.ReceiveChan {
			func() {
				defer log.RecoverAndLog()

				message := data.(map[string]interface{})
				log.Debug("Received message", message)

				action := message["action"]
				switch action {
				case "subscribe":
					routingKeyPrefix := message["routingKeyPrefix"].(string)
					addToRouteMap(routingKeyPrefix)
					subscriptions[routingKeyPrefix] = true
					session.Send(map[string]string{"routingKey": "broker.subscribed", "payload": routingKeyPrefix})

				case "unsubscribe":
					routingKeyPrefix := message["routingKeyPrefix"].(string)
					removeFromRouteMap(routingKeyPrefix)
					delete(subscriptions, routingKeyPrefix)

				case "publish":
					exchange := message["exchange"].(string)
					routingKey := message["routingKey"].(string)
					if !strings.HasPrefix(routingKey, "client.") {
						log.Warn("Invalid routing key.", message, socketId)
						return
					}
					for {
						lastPayload = ""
						err := controlChannel.Publish(exchange, routingKey, false, false, amqp.Publishing{CorrelationId: socketId, Body: []byte(message["payload"].(string))})
						if err == nil {
							lastPayload = message["payload"].(string)
							break
						}
						if amqpError, isAmqpError := err.(*amqp.Error); !isAmqpError || amqpError.Code != 504 {
							panic(err)
						}
						resetControlChannel()
					}

				case "ping":
					session.Send(map[string]string{"routingKey": "broker.pong"})

				default:
					log.Warn("Invalid action.", message, socketId)

				}
			}()
		}
	})
	defer service.Close()
	service.PanicHandler = log.RecoverAndLog

	go func() {
		server := &http.Server{
			Handler: &sockjs.Mux{
				Services: map[string]*sockjs.Service{
					"/subscribe": service,
				},
			},
		}

		var listener net.Listener
		listener, err := net.ListenTCP("tcp", &net.TCPAddr{nil, config.Current.Broker.Port})
		if err != nil {
			log.LogError(err, 0)
			os.Exit(1)
		}

		if config.Current.Broker.CertFile != "" {
			cert, err := tls.LoadX509KeyPair(config.Current.Broker.CertFile, config.Current.Broker.KeyFile)
			if err != nil {
				log.LogError(err, 0)
				os.Exit(1)
			}
			listener = tls.NewListener(listener, &tls.Config{
				NextProtos:   []string{"http/1.1"},
				Certificates: []tls.Certificate{cert},
			})
		}

		go func() {
			sigtermChannel := make(chan os.Signal)
			signal.Notify(sigtermChannel, syscall.SIGTERM)
			<-sigtermChannel
			lifecycle.BeginShutdown()
			listener.Close()
		}()

		lastErrorTime := time.Now()
		for {
			err := server.Serve(listener)
			if lifecycle.ShuttingDown {
				break
			}
			if err != nil {
				log.Warn("Server error: " + err.Error())
				if time.Now().Sub(lastErrorTime) < time.Second {
					os.Exit(1)
				}
				lastErrorTime = time.Now()
			}
		}
	}()

	consumeConn := amqputil.CreateConnection("broker")
	defer consumeConn.Close()

	consumeChannel := amqputil.CreateChannel(consumeConn)
	defer consumeChannel.Close()

	stream := amqputil.DeclareBindConsumeQueue(consumeChannel, "topic", "broker", "#", false)
	if err := consumeChannel.ExchangeDeclare("updateInstances", "fanout", false, false, false, false, nil); err != nil {
		panic(err)
	}
	if err := consumeChannel.ExchangeBind("broker", "", "updateInstances", false, nil); err != nil {
		panic(err)
	}

	for amqpMessage := range stream {
		routingKey := amqpMessage.RoutingKey
		payload := json.RawMessage(utils.FilterInvalidUTF8(amqpMessage.Body))
		jsonMessage := map[string]interface{}{"routingKey": routingKey, "payload": &payload}

		pos := strings.IndexRune(routingKey, '.') // skip first dot, since we want at least two components to always include the secret
		for pos != -1 && pos < len(routingKey) {
			index := strings.IndexRune(routingKey[pos+1:], '.')
			pos += index + 1
			if index == -1 {
				pos = len(routingKey)
			}
			prefix := routingKey[:pos]
			routeMapMutex.Lock()
			routeSessions := routeMap[prefix]
			for _, routeSession := range routeSessions {
				if !routeSession.Send(jsonMessage) {
					routeSession.Close()
					log.Warn("Dropped session because of broker to client buffer overflow.", routeSession.Tag)
				}
			}
			routeMapMutex.Unlock()
		}
	}
}
<|endoftext|>


<commit_before>/* Copyright 2016 The Bazel Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Bare bones Go testing support for Bazel.

package main

import (
	"flag"
	"go/ast"
	"go/parser"
	"go/token"
	"log"
	"os"
	"strings"
	"text/template"
)

// Cases holds template data.
type Cases struct {
	Package        string
	RunDir         string
	TestNames      []string
	BenchmarkNames []string
	HasTestMain    bool
}

func main() {
	pkg := flag.String("package", "", "package from which to import test methods.")
	out := flag.String("output", "", "output file to write. Defaults to stdout.")
	flag.Parse()

	if *pkg == "" {
		log.Fatal("must set --package.")
	}

	outFile := os.Stdout
	if *out != "" {
		var err error
		outFile, err = os.Create(*out)
		if err != nil {
			log.Fatalf("os.Create(%q): %v", *out, err)
		}
		defer outFile.Close()
	}

	cases := Cases{
		Package: *pkg,
		RunDir:  os.Getenv("RUNDIR"),
	}
	testFileSet := token.NewFileSet()
	for _, f := range flag.Args() {
		parse, err := parser.ParseFile(testFileSet, f, nil, parser.ParseComments)
		if err != nil {
			log.Fatalf("ParseFile(%q): %v", f, err)
		}

		for _, d := range parse.Decls {
			fn, ok := d.(*ast.FuncDecl)
			if !ok {
				continue
			}
			if fn.Recv != nil {
				continue
			}
			if fn.Name.Name == "TestMain" {
				// TestMain is not, itself, a test
				cases.HasTestMain = true
				continue
			}
			if strings.HasPrefix(fn.Name.Name, "Test") {
				cases.TestNames = append(cases.TestNames, fn.Name.Name)
			}
			if strings.HasPrefix(fn.Name.Name, "Benchmark") {
				cases.BenchmarkNames = append(cases.BenchmarkNames, fn.Name.Name)
			}

			// Here we check the signature of the Test* function. To
			// be considered a test:

			// 1. The function should have a single argument.
			if len(fn.Type.Params.List) != 1 {
				continue
			}

			// 2. The function should return nothing.
			if fn.Type.Results != nil {
				continue
			}

			// 3. The only parameter should have a type identified as
			//    *<something>.T
			starExpr, ok := fn.Type.Params.List[0].Type.(*ast.StarExpr)
			if !ok {
				continue
			}
			selExpr, ok := starExpr.X.(*ast.SelectorExpr)
			if !ok {
				continue
			}

			// We do not descriminate on the referenced type of the
			// parameter being *testing.T. Instead we assert that it
			// should be *<something>.T. This is because the import
			// could have been aliased as a different identifier.

			if strings.HasPrefix(fn.Name.Name, "Test") {
				if selExpr.Sel.Name != "T" {
					continue
				}
				cases.TestNames = append(cases.TestNames, fn.Name.Name)
			}
			if strings.HasPrefix(fn.Name.Name, "Benchmark") {
				if selExpr.Sel.Name != "B" {
					continue
				}
				cases.BenchmarkNames = append(cases.BenchmarkNames, fn.Name.Name)
			}
		}
	}

	tpl := template.Must(template.New("source").Parse(`
package main
import (
	"os"
	"testing"

{{ if .TestNames }}
        undertest "{{.Package}}"
{{else if .BenchmarkNames }}
        undertest "{{.Package}}"
{{ end }}
)

func everything(pat, str string) (bool, error) {
	return true, nil
}

var tests = []testing.InternalTest{
{{range .TestNames}}
   {"{{.}}", undertest.{{.}} },
{{end}}
}

var benchmarks = []testing.InternalBenchmark{
{{range .BenchmarkNames}}
   {"{{.}}", undertest.{{.}} },
{{end}}
}

func main() {
  os.Chdir("{{.RunDir}}")
  {{if not .HasTestMain}}
  testing.Main(everything, tests, benchmarks, nil)
  {{else}}
  m := testing.MainStart(everything, tests, benchmarks, nil)
  undertest.TestMain(m)
  {{end}}
}
`))
	if err := tpl.Execute(outFile, &cases); err != nil {
		log.Fatalf("template.Execute(%v): %v", cases, err)
	}
}
<commit_msg>Don't add tests and benchmarks twice. (#241)<commit_after>/* Copyright 2016 The Bazel Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Bare bones Go testing support for Bazel.

package main

import (
	"flag"
	"go/ast"
	"go/parser"
	"go/token"
	"log"
	"os"
	"strings"
	"text/template"
)

// Cases holds template data.
type Cases struct {
	Package        string
	RunDir         string
	TestNames      []string
	BenchmarkNames []string
	HasTestMain    bool
}

func main() {
	pkg := flag.String("package", "", "package from which to import test methods.")
	out := flag.String("output", "", "output file to write. Defaults to stdout.")
	flag.Parse()

	if *pkg == "" {
		log.Fatal("must set --package.")
	}

	outFile := os.Stdout
	if *out != "" {
		var err error
		outFile, err = os.Create(*out)
		if err != nil {
			log.Fatalf("os.Create(%q): %v", *out, err)
		}
		defer outFile.Close()
	}

	cases := Cases{
		Package: *pkg,
		RunDir:  os.Getenv("RUNDIR"),
	}
	testFileSet := token.NewFileSet()
	for _, f := range flag.Args() {
		parse, err := parser.ParseFile(testFileSet, f, nil, parser.ParseComments)
		if err != nil {
			log.Fatalf("ParseFile(%q): %v", f, err)
		}

		for _, d := range parse.Decls {
			fn, ok := d.(*ast.FuncDecl)
			if !ok {
				continue
			}
			if fn.Recv != nil {
				continue
			}
			if fn.Name.Name == "TestMain" {
				// TestMain is not, itself, a test
				cases.HasTestMain = true
				continue
			}

			// Here we check the signature of the Test* function. To
			// be considered a test:

			// 1. The function should have a single argument.
			if len(fn.Type.Params.List) != 1 {
				continue
			}

			// 2. The function should return nothing.
			if fn.Type.Results != nil {
				continue
			}

			// 3. The only parameter should have a type identified as
			//    *<something>.T
			starExpr, ok := fn.Type.Params.List[0].Type.(*ast.StarExpr)
			if !ok {
				continue
			}
			selExpr, ok := starExpr.X.(*ast.SelectorExpr)
			if !ok {
				continue
			}

			// We do not descriminate on the referenced type of the
			// parameter being *testing.T. Instead we assert that it
			// should be *<something>.T. This is because the import
			// could have been aliased as a different identifier.

			if strings.HasPrefix(fn.Name.Name, "Test") {
				if selExpr.Sel.Name != "T" {
					continue
				}
				cases.TestNames = append(cases.TestNames, fn.Name.Name)
			}
			if strings.HasPrefix(fn.Name.Name, "Benchmark") {
				if selExpr.Sel.Name != "B" {
					continue
				}
				cases.BenchmarkNames = append(cases.BenchmarkNames, fn.Name.Name)
			}
		}
	}

	tpl := template.Must(template.New("source").Parse(`
package main
import (
	"os"
	"testing"

{{ if .TestNames }}
        undertest "{{.Package}}"
{{else if .BenchmarkNames }}
        undertest "{{.Package}}"
{{ end }}
)

func everything(pat, str string) (bool, error) {
	return true, nil
}

var tests = []testing.InternalTest{
{{range .TestNames}}
   {"{{.}}", undertest.{{.}} },
{{end}}
}

var benchmarks = []testing.InternalBenchmark{
{{range .BenchmarkNames}}
   {"{{.}}", undertest.{{.}} },
{{end}}
}

func main() {
  os.Chdir("{{.RunDir}}")
  {{if not .HasTestMain}}
  testing.Main(everything, tests, benchmarks, nil)
  {{else}}
  m := testing.MainStart(everything, tests, benchmarks, nil)
  undertest.TestMain(m)
  {{end}}
}
`))
	if err := tpl.Execute(outFile, &cases); err != nil {
		log.Fatalf("template.Execute(%v): %v", cases, err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"
)

func main() {
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		fmt.Println(scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintf(os.Stderr, "reading standard input", err)
	}
}

func unxhtmlify_line(s string) string {
	return strings.Replace(s, " /", "", 1)
}
<commit_msg>unxhtmlify.go: Use regex replacement<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"regexp"
)

func main() {
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		fmt.Println(scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintf(os.Stderr, "reading standard input", err)
	}
}

func unxhtmlify_line(s string) string {
	re := regexp.MustCompile(" ?/>")
	return re.ReplaceAllString(s, ">")
}
<|endoftext|>


<commit_before>package filecache

import (
	"errors"
	"os"
	"path"
	"path/filepath"
	"sync"
	"syscall"
	"time"

	"github.com/anacrolix/log"

	"github.com/anacrolix/missinggo/v2/pproffd"
	"github.com/anacrolix/missinggo/v2/resource"
)

const (
	dirPerm  = 0755
	filePerm = 0644
)

type Cache struct {
	root     string
	mu       sync.Mutex
	capacity int64
	filled   int64
	policy   Policy
	items    map[key]itemState
}

type CacheInfo struct {
	Capacity int64
	Filled   int64
	NumItems int
}

type ItemInfo struct {
	Path     key
	Accessed time.Time
	Size     int64
}

// Calls the function for every item known to be in the cache.
func (me *Cache) WalkItems(cb func(ItemInfo)) {
	me.mu.Lock()
	defer me.mu.Unlock()
	for k, ii := range me.items {
		cb(ItemInfo{
			Path:     k,
			Accessed: ii.Accessed,
			Size:     ii.Size,
		})
	}
}

func (me *Cache) Info() (ret CacheInfo) {
	me.mu.Lock()
	defer me.mu.Unlock()
	ret.Capacity = me.capacity
	ret.Filled = me.filled
	ret.NumItems = len(me.items)
	return
}

// Setting a negative capacity means unlimited.
func (me *Cache) SetCapacity(capacity int64) {
	me.mu.Lock()
	defer me.mu.Unlock()
	me.capacity = capacity
}

func NewCache(root string) (ret *Cache, err error) {
	root, err = filepath.Abs(root)
	ret = &Cache{
		root:     root,
		capacity: -1, // unlimited
	}
	ret.mu.Lock()
	go func() {
		defer ret.mu.Unlock()
		ret.rescan()
	}()
	return
}

// An empty return path is an error.
func sanitizePath(p string) (ret key) {
	if p == "" {
		return
	}
	ret = key(path.Clean("/" + p))
	if ret[0] == '/' {
		ret = ret[1:]
	}
	return
}

// Leaf is a descendant of root.
func pruneEmptyDirs(root string, leaf string) (err error) {
	rootInfo, err := os.Stat(root)
	if err != nil {
		return
	}
	for {
		var leafInfo os.FileInfo
		leafInfo, err = os.Stat(leaf)
		if os.IsNotExist(err) {
			goto parent
		}
		if err != nil {
			return
		}
		if !leafInfo.IsDir() {
			return
		}
		if os.SameFile(rootInfo, leafInfo) {
			return
		}
		if os.Remove(leaf) != nil {
			return
		}
	parent:
		leaf = filepath.Dir(leaf)
	}
}

func (me *Cache) Remove(path string) error {
	me.mu.Lock()
	defer me.mu.Unlock()
	return me.remove(sanitizePath(path))
}

var (
	ErrBadPath = errors.New("bad path")
	ErrIsDir   = errors.New("is directory")
)

func (me *Cache) StatFile(path string) (os.FileInfo, error) {
	return os.Stat(me.realpath(sanitizePath(path)))
}

func isMissingDir(err error) bool {
	// I'm not sure why we get EINVAL for missing path components sometimes. It happens on MacOS. I
	// wonder if it would happen the same on other OS.
	return errors.Is(err, syscall.EINVAL) || errors.Is(err, syscall.ENOENT)
}

func (me *Cache) OpenFile(path string, flag int) (ret *File, err error) {
	key := sanitizePath(path)
	if key == "" {
		err = ErrIsDir
		return
	}
	filePath := me.realpath(key)
	f, err := os.OpenFile(filePath, flag, filePerm)
	// Ensure intermediate directories and try again.
	if flag&os.O_CREATE != 0 && isMissingDir(err) {
		dirPath := filepath.Dir(filePath)
		if dirPath == "" {
			panic(key)
		}
		defer func() {
			if err != nil {
				go me.pruneEmptyDirs(key)
			}
		}()
		for {
			err = os.MkdirAll(dirPath, dirPerm)
			if isMissingDir(err) {
				continue
			} else if err != nil {
				return
			}
			f, err = os.OpenFile(filePath, flag, filePerm)
			if isMissingDir(err) {
				continue
			}
			break
		}
	}
	if err != nil {
		return
	}
	ret = &File{
		path: key,
		f:    pproffd.WrapOSFile(f),
		onRead: func(n int) {
			me.mu.Lock()
			defer me.mu.Unlock()
			me.updateItem(key, func(i *itemState, ok bool) bool {
				i.Accessed = time.Now()
				return ok
			})
		},
		afterWrite: func(endOff int64) {
			me.mu.Lock()
			defer me.mu.Unlock()
			me.updateItem(key, func(i *itemState, ok bool) bool {
				i.Accessed = time.Now()
				if endOff > i.Size {
					i.Size = endOff
				}
				return ok
			})
		},
	}
	me.mu.Lock()
	defer me.mu.Unlock()
	me.updateItem(key, func(i *itemState, ok bool) bool {
		if !ok {
			*i, ok = me.statKey(key)
		}
		i.Accessed = time.Now()
		return ok
	})
	return
}

func (me *Cache) rescan() {
	me.filled = 0
	me.policy = new(lru)
	me.items = make(map[key]itemState)
	err := filepath.Walk(me.root, func(path string, info os.FileInfo, err error) error {
		if os.IsNotExist(err) {
			return nil
		}
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		path, err = filepath.Rel(me.root, path)
		if err != nil {
			log.Print(err)
			return nil
		}
		key := sanitizePath(path)
		me.updateItem(key, func(i *itemState, ok bool) bool {
			if ok {
				panic("scanned duplicate items")
			}
			*i, ok = me.statKey(key)
			return ok
		})
		return nil
	})
	if err != nil {
		panic(err)
	}
}

func (me *Cache) statKey(k key) (i itemState, ok bool) {
	fi, err := os.Stat(me.realpath(k))
	if os.IsNotExist(err) {
		return
	}
	if err != nil {
		panic(err)
	}
	i.FromOSFileInfo(fi)
	ok = true
	return
}

func (me *Cache) updateItem(k key, u func(*itemState, bool) bool) {
	ii, ok := me.items[k]
	me.filled -= ii.Size
	if u(&ii, ok) {
		me.filled += ii.Size
		me.policy.Used(k, ii.Accessed)
		me.items[k] = ii
	} else {
		me.policy.Forget(k)
		delete(me.items, k)
	}
	me.trimToCapacity()
}

func (me *Cache) realpath(path key) string {
	return filepath.Join(me.root, filepath.FromSlash(string(path)))
}

func (me *Cache) TrimToCapacity() {
	me.mu.Lock()
	defer me.mu.Unlock()
	me.trimToCapacity()
}

func (me *Cache) pruneEmptyDirs(path key) {
	pruneEmptyDirs(me.root, me.realpath(path))
}

func (me *Cache) remove(path key) error {
	err := os.Remove(me.realpath(path))
	if os.IsNotExist(err) {
		err = nil
	}
	if err != nil {
		return err
	}
	me.pruneEmptyDirs(path)
	me.updateItem(path, func(*itemState, bool) bool {
		return false
	})
	return nil
}

func (me *Cache) trimToCapacity() {
	if me.capacity < 0 {
		return
	}
	for me.filled > me.capacity {
		me.remove(me.policy.Choose().(key))
	}
}

// TODO: Do I need this?
func (me *Cache) pathInfo(p string) itemState {
	return me.items[sanitizePath(p)]
}

func (me *Cache) Rename(from, to string) (err error) {
	_from := sanitizePath(from)
	_to := sanitizePath(to)
	me.mu.Lock()
	defer me.mu.Unlock()
	err = os.MkdirAll(filepath.Dir(me.realpath(_to)), dirPerm)
	if err != nil {
		return
	}
	err = os.Rename(me.realpath(_from), me.realpath(_to))
	if err != nil {
		return
	}
	// We can do a dance here to copy the state from the old item, but lets
	// just stat the new item for now.
	me.updateItem(_from, func(i *itemState, ok bool) bool {
		return false
	})
	me.updateItem(_to, func(i *itemState, ok bool) bool {
		*i, ok = me.statKey(_to)
		return ok
	})
	return
}

func (me *Cache) Stat(path string) (os.FileInfo, error) {
	return os.Stat(me.realpath(sanitizePath(path)))
}

func (me *Cache) AsResourceProvider() resource.Provider {
	return &uniformResourceProvider{me}
}
<commit_msg>fix filecache not making directories on windows (#22)<commit_after>package filecache

import (
	"errors"
	"os"
	"path"
	"path/filepath"
	"sync"
	"syscall"
	"time"
	"io/fs"

	"github.com/anacrolix/log"

	"github.com/anacrolix/missinggo/v2/pproffd"
	"github.com/anacrolix/missinggo/v2/resource"
)

const (
	dirPerm  = 0755
	filePerm = 0644
)

type Cache struct {
	root     string
	mu       sync.Mutex
	capacity int64
	filled   int64
	policy   Policy
	items    map[key]itemState
}

type CacheInfo struct {
	Capacity int64
	Filled   int64
	NumItems int
}

type ItemInfo struct {
	Path     key
	Accessed time.Time
	Size     int64
}

// Calls the function for every item known to be in the cache.
func (me *Cache) WalkItems(cb func(ItemInfo)) {
	me.mu.Lock()
	defer me.mu.Unlock()
	for k, ii := range me.items {
		cb(ItemInfo{
			Path:     k,
			Accessed: ii.Accessed,
			Size:     ii.Size,
		})
	}
}

func (me *Cache) Info() (ret CacheInfo) {
	me.mu.Lock()
	defer me.mu.Unlock()
	ret.Capacity = me.capacity
	ret.Filled = me.filled
	ret.NumItems = len(me.items)
	return
}

// Setting a negative capacity means unlimited.
func (me *Cache) SetCapacity(capacity int64) {
	me.mu.Lock()
	defer me.mu.Unlock()
	me.capacity = capacity
}

func NewCache(root string) (ret *Cache, err error) {
	root, err = filepath.Abs(root)
	ret = &Cache{
		root:     root,
		capacity: -1, // unlimited
	}
	ret.mu.Lock()
	go func() {
		defer ret.mu.Unlock()
		ret.rescan()
	}()
	return
}

// An empty return path is an error.
func sanitizePath(p string) (ret key) {
	if p == "" {
		return
	}
	ret = key(path.Clean("/" + p))
	if ret[0] == '/' {
		ret = ret[1:]
	}
	return
}

// Leaf is a descendant of root.
func pruneEmptyDirs(root string, leaf string) (err error) {
	rootInfo, err := os.Stat(root)
	if err != nil {
		return
	}
	for {
		var leafInfo os.FileInfo
		leafInfo, err = os.Stat(leaf)
		if os.IsNotExist(err) {
			goto parent
		}
		if err != nil {
			return
		}
		if !leafInfo.IsDir() {
			return
		}
		if os.SameFile(rootInfo, leafInfo) {
			return
		}
		if os.Remove(leaf) != nil {
			return
		}
	parent:
		leaf = filepath.Dir(leaf)
	}
}

func (me *Cache) Remove(path string) error {
	me.mu.Lock()
	defer me.mu.Unlock()
	return me.remove(sanitizePath(path))
}

var (
	ErrBadPath = errors.New("bad path")
	ErrIsDir   = errors.New("is directory")
)

func (me *Cache) StatFile(path string) (os.FileInfo, error) {
	return os.Stat(me.realpath(sanitizePath(path)))
}

func isMissingDir(err error) bool {
	// I'm not sure why we get EINVAL for missing path components sometimes. It happens on MacOS. I
	// wonder if it would happen the same on other OS.
	return errors.Is(err, syscall.EINVAL) || errors.Is(err, syscall.ENOENT) || errors.Is(err, fs.ErrNotExist)
}

func (me *Cache) OpenFile(path string, flag int) (ret *File, err error) {
	key := sanitizePath(path)
	if key == "" {
		err = ErrIsDir
		return
	}
	filePath := me.realpath(key)
	f, err := os.OpenFile(filePath, flag, filePerm)
	// Ensure intermediate directories and try again.
	if flag&os.O_CREATE != 0 && isMissingDir(err) {
		dirPath := filepath.Dir(filePath)
		if dirPath == "" {
			panic(key)
		}
		defer func() {
			if err != nil {
				go me.pruneEmptyDirs(key)
			}
		}()
		for {
			err = os.MkdirAll(dirPath, dirPerm)
			if isMissingDir(err) {
				continue
			} else if err != nil {
				return
			}
			f, err = os.OpenFile(filePath, flag, filePerm)
			if isMissingDir(err) {
				continue
			}
			break
		}
	}
	if err != nil {
		return
	}
	ret = &File{
		path: key,
		f:    pproffd.WrapOSFile(f),
		onRead: func(n int) {
			me.mu.Lock()
			defer me.mu.Unlock()
			me.updateItem(key, func(i *itemState, ok bool) bool {
				i.Accessed = time.Now()
				return ok
			})
		},
		afterWrite: func(endOff int64) {
			me.mu.Lock()
			defer me.mu.Unlock()
			me.updateItem(key, func(i *itemState, ok bool) bool {
				i.Accessed = time.Now()
				if endOff > i.Size {
					i.Size = endOff
				}
				return ok
			})
		},
	}
	me.mu.Lock()
	defer me.mu.Unlock()
	me.updateItem(key, func(i *itemState, ok bool) bool {
		if !ok {
			*i, ok = me.statKey(key)
		}
		i.Accessed = time.Now()
		return ok
	})
	return
}

func (me *Cache) rescan() {
	me.filled = 0
	me.policy = new(lru)
	me.items = make(map[key]itemState)
	err := filepath.Walk(me.root, func(path string, info os.FileInfo, err error) error {
		if os.IsNotExist(err) {
			return nil
		}
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		path, err = filepath.Rel(me.root, path)
		if err != nil {
			log.Print(err)
			return nil
		}
		key := sanitizePath(path)
		me.updateItem(key, func(i *itemState, ok bool) bool {
			if ok {
				panic("scanned duplicate items")
			}
			*i, ok = me.statKey(key)
			return ok
		})
		return nil
	})
	if err != nil {
		panic(err)
	}
}

func (me *Cache) statKey(k key) (i itemState, ok bool) {
	fi, err := os.Stat(me.realpath(k))
	if os.IsNotExist(err) {
		return
	}
	if err != nil {
		panic(err)
	}
	i.FromOSFileInfo(fi)
	ok = true
	return
}

func (me *Cache) updateItem(k key, u func(*itemState, bool) bool) {
	ii, ok := me.items[k]
	me.filled -= ii.Size
	if u(&ii, ok) {
		me.filled += ii.Size
		me.policy.Used(k, ii.Accessed)
		me.items[k] = ii
	} else {
		me.policy.Forget(k)
		delete(me.items, k)
	}
	me.trimToCapacity()
}

func (me *Cache) realpath(path key) string {
	return filepath.Join(me.root, filepath.FromSlash(string(path)))
}

func (me *Cache) TrimToCapacity() {
	me.mu.Lock()
	defer me.mu.Unlock()
	me.trimToCapacity()
}

func (me *Cache) pruneEmptyDirs(path key) {
	pruneEmptyDirs(me.root, me.realpath(path))
}

func (me *Cache) remove(path key) error {
	err := os.Remove(me.realpath(path))
	if os.IsNotExist(err) {
		err = nil
	}
	if err != nil {
		return err
	}
	me.pruneEmptyDirs(path)
	me.updateItem(path, func(*itemState, bool) bool {
		return false
	})
	return nil
}

func (me *Cache) trimToCapacity() {
	if me.capacity < 0 {
		return
	}
	for me.filled > me.capacity {
		me.remove(me.policy.Choose().(key))
	}
}

// TODO: Do I need this?
func (me *Cache) pathInfo(p string) itemState {
	return me.items[sanitizePath(p)]
}

func (me *Cache) Rename(from, to string) (err error) {
	_from := sanitizePath(from)
	_to := sanitizePath(to)
	me.mu.Lock()
	defer me.mu.Unlock()
	err = os.MkdirAll(filepath.Dir(me.realpath(_to)), dirPerm)
	if err != nil {
		return
	}
	err = os.Rename(me.realpath(_from), me.realpath(_to))
	if err != nil {
		return
	}
	// We can do a dance here to copy the state from the old item, but lets
	// just stat the new item for now.
	me.updateItem(_from, func(i *itemState, ok bool) bool {
		return false
	})
	me.updateItem(_to, func(i *itemState, ok bool) bool {
		*i, ok = me.statKey(_to)
		return ok
	})
	return
}

func (me *Cache) Stat(path string) (os.FileInfo, error) {
	return os.Stat(me.realpath(sanitizePath(path)))
}

func (me *Cache) AsResourceProvider() resource.Provider {
	return &uniformResourceProvider{me}
}
<|endoftext|>


<commit_before>package captain // import "github.com/harbur/captain/captain"

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/fsouza/go-dockerclient"
)

var endpoint = "unix:///var/run/docker.sock"
var client, _ = docker.NewClient(endpoint)

func buildImage(app App, tag string) error {
	info("Building image %s:%s", app.Image, tag)

	// Nasty issue with CircleCI https://github.com/docker/docker/issues/4897
	if os.Getenv("CIRCLECI") == "true" {
		info("Running at %s environment...", "CIRCLECI")
		execute("docker", "build", "-t", app.Image+":"+tag, filepath.Dir(app.Build))
		return nil
	}

	opts := docker.BuildImageOptions{
		Name:                app.Image + ":" + tag,
		Dockerfile:          filepath.Base(app.Build),
		NoCache:             options.force,
		SuppressOutput:      false,
		RmTmpContainer:      true,
		ForceRmTmpContainer: true,
		OutputStream:        os.Stdout,
		ContextDir:          filepath.Dir(app.Build),
	}
	err := client.BuildImage(opts)
	if err != nil {
		fmt.Printf("%s", err)
	}
	return err
}

func tagImage(app App, origin string, tag string) error {
	if tag != "" {
		info("Tagging image %s:%s as %s:%s", app.Image, origin, app.Image, tag)
		opts := docker.TagImageOptions{Repo: app.Image, Tag: tag, Force: true}
		err := client.TagImage(app.Image + ":" + origin, opts)
		if err != nil {
			fmt.Printf("%s", err)
		}
		return err
	}

	debug("Skipping tag of %s - no git repository", app.Image)

	return nil
}

func imageExist(app App, tag string) bool {
	repo:=app.Image+":"+tag
	image, _ := client.InspectImage(repo)
	if image!=nil {
		return true
	}
	return false
}
<commit_msg>Support boot2docker/mac.<commit_after>package captain // import "github.com/harbur/captain/captain"

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/fsouza/go-dockerclient"
)

const defaultEndpoint = "unix:///var/run/docker.sock"

var client *docker.Client

func init() {
	var err error
	client, err = newDockerClientFromEnv()
	if err != nil {
		panic(err)
	}
}

func buildImage(app App, tag string) error {
	info("Building image %s:%s", app.Image, tag)

	// Nasty issue with CircleCI https://github.com/docker/docker/issues/4897
	if os.Getenv("CIRCLECI") == "true" {
		info("Running at %s environment...", "CIRCLECI")
		execute("docker", "build", "-t", app.Image+":"+tag, filepath.Dir(app.Build))
		return nil
	}

	opts := docker.BuildImageOptions{
		Name:                app.Image + ":" + tag,
		Dockerfile:          filepath.Base(app.Build),
		NoCache:             options.force,
		SuppressOutput:      false,
		RmTmpContainer:      true,
		ForceRmTmpContainer: true,
		OutputStream:        os.Stdout,
		ContextDir:          filepath.Dir(app.Build),
	}
	err := client.BuildImage(opts)
	if err != nil {
		fmt.Printf("%s", err)
	}
	return err
}

func tagImage(app App, origin string, tag string) error {
	if tag != "" {
		info("Tagging image %s:%s as %s:%s", app.Image, origin, app.Image, tag)
		opts := docker.TagImageOptions{Repo: app.Image, Tag: tag, Force: true}
		err := client.TagImage(app.Image+":"+origin, opts)
		if err != nil {
			fmt.Printf("%s", err)
		}
		return err
	}

	debug("Skipping tag of %s - no git repository", app.Image)

	return nil
}

func imageExist(app App, tag string) bool {
	repo := app.Image + ":" + tag
	image, _ := client.InspectImage(repo)
	if image != nil {
		return true
	}
	return false
}

func newDockerClient(socket, certPath string) (*docker.Client, error) {
	if certPath != "" {
		cert := certPath + "/cert.pem"
		key := certPath + "/key.pem"
		ca := certPath + "/ca.pem"
		return docker.NewTLSClient(socket, cert, key, ca)
	}

	if socket == "" {
		socket = defaultEndpoint
	}

	return docker.NewClient(socket)
}

func newDockerClientFromEnv() (*docker.Client, error) {
	return newDockerClient(os.Getenv("DOCKER_HOST"), os.Getenv("DOCKER_CERT_PATH"))
}
<|endoftext|>


<commit_before>package pg_test

import (
	"net"
	"time"

	. "gopkg.in/check.v1"

	"gopkg.in/pg.v4"
)

var _ = Suite(&ListenerTest{})

type ListenerTest struct {
	db *pg.DB
	ln *pg.Listener
}

func (t *ListenerTest) SetUpTest(c *C) {
	opt := pgOptions()
	opt.PoolSize = 2
	opt.PoolTimeout = time.Second
	t.db = pg.Connect(opt)

	ln, err := t.db.Listen("test_channel")
	c.Assert(err, IsNil)
	t.ln = ln
}

func (t *ListenerTest) TearDownTest(c *C) {
	c.Assert(t.db.Close(), IsNil)
}

func (t *ListenerTest) TestListenNotify(c *C) {
	wait := make(chan struct{}, 2)
	go func() {
		wait <- struct{}{}
		channel, payload, err := t.ln.Receive()
		c.Assert(err, IsNil)
		c.Assert(channel, Equals, "test_channel")
		c.Assert(payload, Equals, "")
		wait <- struct{}{}
	}()

	select {
	case <-wait:
		// ok
	case <-time.After(3 * time.Second):
		c.Fatal("timeout")
	}

	_, err := t.db.Exec("NOTIFY test_channel")
	c.Assert(err, IsNil)

	select {
	case <-wait:
		// ok
	case <-time.After(3 * time.Second):
		c.Fatal("timeout")
	}
}

func (t *ListenerTest) TestCloseAbortsListener(c *C) {
	wait := make(chan struct{}, 2)

	go func() {
		wait <- struct{}{}
		_, _, err := t.ln.Receive()
		c.Assert(err, ErrorMatches, `^(.*use of closed network connection|EOF)$`)
		wait <- struct{}{}
	}()

	select {
	case <-wait:
		// ok
	case <-time.After(3 * time.Second):
		c.Fatal("timeout")
	}

	select {
	case <-wait:
		c.Fatal("Receive is not blocked")
	case <-time.After(time.Second):
		// ok
	}

	c.Assert(t.ln.Close(), IsNil)

	select {
	case <-wait:
		// ok
	case <-time.After(3 * time.Second):
		c.Fatal("timeout")
	}
}

func (t *ListenerTest) TestListenTimeout(c *C) {
	channel, payload, err := t.ln.ReceiveTimeout(time.Second)
	c.Assert(err.(net.Error).Timeout(), Equals, true)
	c.Assert(channel, Equals, "")
	c.Assert(payload, Equals, "")
}

func (t *ListenerTest) TestReconnectOnListenError(c *C) {
	cn := t.ln.CurrentConn()
	c.Assert(cn, Not(IsNil))
	cn.NetConn = &badConn{}

	err := t.ln.Listen("test_channel2")
	c.Assert(err, ErrorMatches, `bad connection`)

	err = t.ln.Listen("test_channel2")
	c.Assert(err, IsNil)
}

func (t *ListenerTest) TestReconnectOnReceiveError(c *C) {
	cn := t.ln.CurrentConn()
	c.Assert(cn, Not(IsNil))
	cn.NetConn = &badConn{}

	_, _, err := t.ln.ReceiveTimeout(time.Second)
	c.Assert(err, ErrorMatches, `bad connection`)

	_, _, err = t.ln.ReceiveTimeout(time.Second)
	c.Assert(err.(net.Error).Timeout(), Equals, true)

	wait := make(chan struct{}, 2)
	go func() {
		wait <- struct{}{}
		_, _, err := t.ln.Receive()
		c.Assert(err, IsNil)
		wait <- struct{}{}
	}()

	select {
	case <-wait:
		// ok
	case <-time.After(3 * time.Second):
		c.Fatal("timeout")
	}

	select {
	case <-wait:
		c.Fatal("Receive is not blocked")
	case <-time.After(time.Second):
		// ok
	}

	_, err = t.db.Exec("NOTIFY test_channel")
	c.Assert(err, IsNil)

	select {
	case <-wait:
		// ok
	case <-time.After(3 * time.Second):
		c.Fatal("timeout")
	}
}
<commit_msg>Listener_test: rewrites using ginkgo.<commit_after>package pg_test

import (
	"net"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"gopkg.in/pg.v4"
)

var _ = Context("Listener", func() {
	var db *pg.DB
	var ln *pg.Listener

	BeforeEach(func() {
		opt := pgOptions()
		opt.PoolSize = 2
		opt.PoolTimeout = time.Second

		db = pg.Connect(opt)

		var err error
		ln, err = db.Listen("test_channel")
		Expect(err).NotTo(HaveOccurred())
	})

	var _ = AfterEach(func() {
		err := db.Close()
		Expect(err).NotTo(HaveOccurred())
	})

	It("listens for notifications", func() {
		wait := make(chan struct{}, 2)
		go func() {
			defer GinkgoRecover()

			wait <- struct{}{}
			channel, payload, err := ln.Receive()
			Expect(err).NotTo(HaveOccurred())
			Expect(channel).To(Equal("test_channel"))
			Expect(payload).To(Equal(""))
			wait <- struct{}{}
		}()

		select {
		case <-wait:
			// ok
		case <-time.After(3 * time.Second):
			Fail("timeout")
		}

		_, err := db.Exec("NOTIFY test_channel")
		Expect(err).NotTo(HaveOccurred())

		select {
		case <-wait:
			// ok
		case <-time.After(3 * time.Second):
			Fail("timeout")
		}
	})

	It("is aborted when DB is closed", func() {
		wait := make(chan struct{}, 2)

		go func() {
			defer GinkgoRecover()

			wait <- struct{}{}
			_, _, err := ln.Receive()

			Expect(err.Error()).Should(MatchRegexp(`^(.*use of closed network connection|EOF)$`))
			wait <- struct{}{}
		}()

		select {
		case <-wait:
			// ok
		case <-time.After(3 * time.Second):
			Fail("timeout")
		}

		select {
		case <-wait:
			Fail("Receive is not blocked")
		case <-time.After(time.Second):
			// ok
		}

		Expect(ln.Close()).To(BeNil())

		select {
		case <-wait:
			// ok
		case <-time.After(3 * time.Second):
			Fail("timeout")
		}
	})

	It("returns an error on timeout", func() {
		channel, payload, err := ln.ReceiveTimeout(time.Second)
		Expect(err.(net.Error).Timeout()).To(BeTrue())
		Expect(channel).To(Equal(""))
		Expect(payload).To(Equal(""))
	})

	It("reconnects on listen error", func() {
		cn := ln.CurrentConn()
		Expect(cn).NotTo(BeNil())
		cn.NetConn = &badConn{}

		err := ln.Listen("test_channel2")
		Expect(err).Should(MatchError("bad connection"))

		err = ln.Listen("test_channel2")
		Expect(err).NotTo(HaveOccurred())
	})

	It("reconnects on receive error", func() {
		cn := ln.CurrentConn()
		Expect(cn).NotTo(BeNil())
		cn.NetConn = &badConn{}

		_, _, err := ln.ReceiveTimeout(time.Second)
		Expect(err).Should(MatchError("bad connection"))

		_, _, err = ln.ReceiveTimeout(time.Second)
		Expect(err.(net.Error).Timeout()).To(BeTrue())

		wait := make(chan struct{}, 2)
		go func() {
			defer GinkgoRecover()

			wait <- struct{}{}
			_, _, err := ln.Receive()
			Expect(err).NotTo(HaveOccurred())
			wait <- struct{}{}
		}()

		select {
		case <-wait:
			// ok
		case <-time.After(3 * time.Second):
			Fail("timeout")
		}

		select {
		case <-wait:
			Fail("Receive is not blocked")
		case <-time.After(time.Second):
			// ok
		}

		_, err = db.Exec("NOTIFY test_channel")
		Expect(err).NotTo(HaveOccurred())

		select {
		case <-wait:
			// ok
		case <-time.After(3 * time.Second):
			Fail("timeout")
		}
	})
})
<|endoftext|>


<commit_before>package server

const ReleaseVersion = "0.4.2"
<commit_msg>chore(server): bump to 0.4.2+git<commit_after>package server

const ReleaseVersion = "0.4.2+git"
<|endoftext|>