<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"

	"github.com/googlecloudplatform/gcsfuse/fs"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse"
	"golang.org/x/net/context"
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "  %s [flags] <mount-point>\n", os.Args[0])
	flag.PrintDefaults()
}

var fBucketName = flag.String("bucket", "", "Name of GCS bucket to mount.")

var fImplicitDirs = flag.Bool(
	"implicit_dirs",
	false,
	"Implicitly define directories based on their content. See docs/semantics.md.")

var fSupportNlink = flag.Bool(
	"support_nlink",
	false,
	"Return meaningful values for nlink from fstat(2). See docs/semantics.md.")

func getBucketName() string {
	s := *fBucketName
	if s == "" {
		fmt.Println("You must set --bucket.")
		os.Exit(1)
	}

	return s
}

func registerSIGINTHandler(mountPoint string) {
	// Register for SIGINT.
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)

	// Start a goroutine that will unmount when the signal is received.
	go func() {
		for {
			<-signalChan
			log.Println("Received SIGINT, attempting to unmount...")

			err := fuse.Unmount(mountPoint)
			if err != nil {
				log.Printf("Failed to unmount in response to SIGINT: %v", err)
			} else {
				log.Printf("Successfully unmounted in response to SIGINT.")
				return
			}
		}
	}()
}

func main() {
	// Make logging output better.
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	// Set up flags.
	flag.Usage = usage
	flag.Parse()

	// Grab the mount point.
	if flag.NArg() != 1 {
		usage()
		os.Exit(1)
	}

	mountPoint := flag.Arg(0)

	// Set up a GCS connection.
	log.Println("Initializing GCS connection.")
	conn, err := getConn()
	if err != nil {
		log.Fatal("Couldn't get GCS connection: ", err)
	}

	// Create a file system server.
	serverCfg := &fs.ServerConfig{
		Clock:               timeutil.RealClock(),
		Bucket:              conn.GetBucket(getBucketName()),
		ImplicitDirectories: *fImplicitDirs,
		SupportNlink:        *fSupportNlink,
	}

	server, err := fs.NewServer(serverCfg)
	if err != nil {
		log.Fatal("fs.NewServer:", err)
	}

	// Mount the file system.
	mountedFS, err := fuse.Mount(mountPoint, server, &fuse.MountConfig{})
	if err != nil {
		log.Fatal("Mount:", err)
	}

	log.Println("File system has been successfully mounted.")

	// Let the user unmount with Ctrl-C (SIGINT).
	registerSIGINTHandler(mountedFS.Dir())

	// Wait for it to be unmounted.
	if err := mountedFS.Join(context.Background()); err != nil {
		log.Fatal("MountedFileSystem.Join:", err)
	}

	log.Println("Successfully exiting.")
}
<commit_msg>Refactored bucket wiring code.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"

	"github.com/googlecloudplatform/gcsfuse/fs"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "  %s [flags] <mount-point>\n", os.Args[0])
	flag.PrintDefaults()
}

var fBucketName = flag.String("bucket", "", "Name of GCS bucket to mount.")

var fImplicitDirs = flag.Bool(
	"implicit_dirs",
	false,
	"Implicitly define directories based on their content. See docs/semantics.md.")

var fSupportNlink = flag.Bool(
	"support_nlink",
	false,
	"Return meaningful values for nlink from fstat(2). See docs/semantics.md.")

func getBucketName() string {
	s := *fBucketName
	if s == "" {
		fmt.Println("You must set --bucket.")
		os.Exit(1)
	}

	return s
}

func registerSIGINTHandler(mountPoint string) {
	// Register for SIGINT.
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)

	// Start a goroutine that will unmount when the signal is received.
	go func() {
		for {
			<-signalChan
			log.Println("Received SIGINT, attempting to unmount...")

			err := fuse.Unmount(mountPoint)
			if err != nil {
				log.Printf("Failed to unmount in response to SIGINT: %v", err)
			} else {
				log.Printf("Successfully unmounted in response to SIGINT.")
				return
			}
		}
	}()
}

func getBucket() (b gcs.Bucket) {
	// Set up a GCS connection.
	log.Println("Initializing GCS connection.")
	conn, err := getConn()
	if err != nil {
		log.Fatal("Couldn't get GCS connection: ", err)
	}

	// Extract the appropriate bucket.
	b = conn.GetBucket(getBucketName())

	return
}

func main() {
	// Make logging output better.
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	// Set up flags.
	flag.Usage = usage
	flag.Parse()

	// Grab the mount point.
	if flag.NArg() != 1 {
		usage()
		os.Exit(1)
	}

	mountPoint := flag.Arg(0)

	// Create a file system server.
	serverCfg := &fs.ServerConfig{
		Clock:               timeutil.RealClock(),
		Bucket:              getBucket(),
		ImplicitDirectories: *fImplicitDirs,
		SupportNlink:        *fSupportNlink,
	}

	server, err := fs.NewServer(serverCfg)
	if err != nil {
		log.Fatal("fs.NewServer:", err)
	}

	// Mount the file system.
	mountedFS, err := fuse.Mount(mountPoint, server, &fuse.MountConfig{})
	if err != nil {
		log.Fatal("Mount:", err)
	}

	log.Println("File system has been successfully mounted.")

	// Let the user unmount with Ctrl-C (SIGINT).
	registerSIGINTHandler(mountedFS.Dir())

	// Wait for it to be unmounted.
	if err := mountedFS.Join(context.Background()); err != nil {
		log.Fatal("MountedFileSystem.Join:", err)
	}

	log.Println("Successfully exiting.")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"

	"github.com/mitchellh/cli"
)

func MountCommandFactory() (cli.Command, error) {
	return &MountCommand{}, nil
}

type MountCommand struct{}

func (c *MountCommand) Run(args []string) int {
	// All of the arguments are required currently, so error if anything
	// is missing.
	if len(args) != 3 {
		fmt.Printf(c.Help())
		return 1
	}

	k, err := CreateKlientClient(NewKlientOptions())
	if err != nil {
		log.Fatal(err)
	}

	if err := k.Dial(); err != nil {
		log.Fatal(err)
	}

	mountRequest := struct {
		Name       string `json:"name"`
		LocalPath  string `json:"localPath"`
		RemotePath string `json:"remotePath"`
	}{
		Name:       args[0],
		RemotePath: args[1],
		LocalPath:  args[2],
	}

	// Don't care about the response currently, since there is none.
	if _, err := k.Tell("remote.mountFolder", mountRequest); err != nil {
		return 1
	}

	return 0
}

func (*MountCommand) Help() string {
	helpText := `
Usage: %s mount <machine name> <remote folder> <local folder>

    Mount a remote folder from the given remote machine, to the specified
    local folder.
`
	return fmt.Sprintf(helpText, Name)
}

func (*MountCommand) Synopsis() string {
	return fmt.Sprintf("Mount a remote folder to a local folder")
}
<commit_msg>default to /home/<username> as remote mount path if none is given<commit_after>package main

import (
	"fmt"
	"log"

	"github.com/mitchellh/cli"
)

func MountCommandFactory() (cli.Command, error) {
	return &MountCommand{}, nil
}

type MountCommand struct{}

func (c *MountCommand) Run(args []string) int {
	if len(args) < 2 {
		log.Fatal(c.Help())
	}

	k, err := CreateKlientClient(NewKlientOptions())
	if err != nil {
		log.Fatal(err)
	}

	if err := k.Dial(); err != nil {
		log.Fatal(err)
	}

	mountRequest := struct {
		Name       string `json:"name"`
		LocalPath  string `json:"localPath"`
		RemotePath string `json:"remotePath"`
	}{
		Name:      args[0],
		LocalPath: args[1],
	}

	if len(args) > 2 {
		mountRequest.RemotePath = args[2]
	}

	// Don't care about the response currently, since there is none.
	if _, err := k.Tell("remote.mountFolder", mountRequest); err != nil {
		log.Fatal(err)
	}

	return 0
}

func (*MountCommand) Help() string {
	helpText := `
Usage: %s mount <machine name> <local folder> <remote folder>

    Mount a remote folder from the given remote machine, to the specified
    local folder.
`
	return fmt.Sprintf(helpText, Name)
}

func (*MountCommand) Synopsis() string {
	return fmt.Sprintf("Mount a remote folder to a local folder")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"http"
	"json"
	"log"
	"os"
	"path"
	"template"
	"time"
	"xml"
)

type Configuration struct {
	UpdateInterval int64
	Feeds          []FeedInfo
}

type FeedInfo struct {
	Name string
	URL  string
	Type string
}

type Feed struct {
	Info  FeedInfo
	Items map[string]Item
}

type Item struct {
	Title   string
	GUID    string
	URL     string
	Date    string // TODO this may need to be a struct
	Desc    string
	Content string
	Read    bool
}

type RSSData struct {
	Channel Channel
}

type Channel struct {
	Title       string
	Link        string
	Description string
	Item        []RSSItemData
}

type RSSItemData struct {
	Title       string
	Link        string
	PubDate     string
	GUID        string
	Description string
	Content     string
}

type TemplateData struct {
	Config Configuration
	Feeds  map[string]Feed
}

var (
	Config        Configuration
	Feeds         map[string]Feed
	TmplData      TemplateData

	page_template template.Template
	page_content  string
	client        http.Client
)

func main() {
	flag.Parse()

	InitTemplate()
	ReadConfig()
	InitCache()
	WriteCache()
	go RunHTTPServer()

	ticks := time.Tick(1e9 * Config.UpdateInterval)
	for {
		ReadFeeds()
		<-ticks
	}
}

func InitTemplate() {
	log.Print("Initializing Page Template")
	page_template.Parse(page_template_string)
}

func ReadConfig() {
	log.Print("Reading Config")
	// Read config from ~/.munchrc
	file, err := os.Open(path.Join(os.Getenv("HOME"),".munchrc"))
	if err != nil {
		log.Fatal(err.String())
	}
	decoder := json.NewDecoder(file)
	err = decoder.Decode(&Config)
	if err != nil {
		log.Fatal(err.String())
	}
}

func InitCache() {
	Feeds = make(map[string]Feed)
	// Ensure the cache directory exists
	cachePath := path.Join(os.Getenv("HOME"), ".munch.d", "cache")
	os.MkdirAll(cachePath, 0700)
	// For each feed
	for _, info := range Config.Feeds {
		name := info.Name
		fPath := path.Join(cachePath, name)
		file, _ := os.Open(fPath)
		if file != nil {
		} else {
			log.Print("New Feed: ", name)
			feed := Feed{}
			feed.Info = info
			feed.Items = make(map[string]Item)
			Feeds[name] = feed
		}
	}
}

func WriteCache() {
	// TODO
}

func ReadFeeds() {
	log.Print("Updating feeds")
	for _, feed := range Feeds {
		switch (feed.Info.Type) {
		case "RSS":
			readRSS(feed)
		default:
			log.Print("Ignoring unknown feed of type ", feed.Info.Type)
		}
	}
	log.Print("Done")
}

func readRSS(feed Feed) {
	url := feed.Info.URL
	log.Print(url)
	r, err := client.Get(url)
	if err != nil {
		log.Print("ERROR: ", err.String())
		return
	}
	reader := r.Body
	feedData := RSSData{}
	err = xml.Unmarshal(reader, &feedData)
	if err != nil {
		log.Print("ERROR: ", err.String())
		return
	}
	// now transform the XML into our internal data structure
	changed := false
	for _, itemData := range feedData.Channel.Item {
		guid := itemData.GUID
		_, ok := feed.Items[guid]
		if !ok {
			// GUID not found - add the item
			changed = true
			item := Item {
				Title: itemData.Title,
				GUID: guid,
				URL: itemData.Link,
				Date: itemData.PubDate,
				Desc: itemData.Description,
				Content: itemData.Content,
				Read: false,
			}
			feed.Items[guid] = item
		}
	}
	if (changed) {
		UpdatePage()
		// TODO run some commands from Config?
	}
}

func readAtom(feed Feed) {

}

func readRDF(feed Feed) {

}

func UpdatePage() {
	// TODO create a write on page_content
}

func RunHTTPServer() {
	log.Print("Spawning HTTP Server")
	http.HandleFunc("/", HTTPHandler)
	err := http.ListenAndServe("localhost:8090", nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err.String())
	}
}

func HTTPHandler(w http.ResponseWriter, req *http.Request) {
	TmplData.Feeds = Feeds
	TmplData.Config = Config
	err := page_template.Execute(w, TmplData)
	if err != nil {
		log.Print("ERROR: ", err.String())
	}
}

<commit_msg>Adding an 'updater' goroutine to avoid race conditions<commit_after>package main

import (
	"bytes"
	"flag"
	"http"
	"json"
	"log"
	"os"
	"path"
	"template"
	"time"
	"xml"
)

type Configuration struct {
	UpdateInterval int64
	Feeds          []FeedInfo
}

type FeedInfo struct {
	Name string
	URL  string
	Type string
}

type Feed struct {
	Info  FeedInfo
	Items map[string]Item
}

type Item struct {
	Title   string
	GUID    string
	URL     string
	Date    string // TODO this may need to be a struct
	Desc    string
	Content string
	Read    bool
}

type RSSData struct {
	Channel Channel
}

type Channel struct {
	Title       string
	Link        string
	Description string
	Item        []RSSItemData
}

type RSSItemData struct {
	Title       string
	Link        string
	PubDate     string
	GUID        string
	Description string
	Content     string
}

type TemplateData struct {
	Config *Configuration
	Feeds  map[string]Feed
}

const (
	DoUpdate = iota
	GetContent
)

var (
	Config        Configuration
	Feeds         map[string]Feed

	page_template template.Template
	client        http.Client
	updates       chan int
	content       chan string
)

func main() {
	flag.Parse()

	updates = make(chan int)
	content = make(chan string)

	InitTemplate()
	ReadConfig()
	InitCache()
	go HandleUpdates()
	WriteCache()
	go RunHTTPServer()

	ticks := time.Tick(1e9 * Config.UpdateInterval)
	for {
		go ReadFeeds()
		<-ticks
	}
}

func InitTemplate() {
	log.Print("Initializing Page Template")
	page_template.Parse(page_template_string)
}

func ReadConfig() {
	log.Print("Reading Config")
	// Read config from ~/.munchrc
	file, err := os.Open(path.Join(os.Getenv("HOME"),".munchrc"))
	if err != nil {
		log.Fatal(err.String())
	}
	decoder := json.NewDecoder(file)
	err = decoder.Decode(&Config)
	if err != nil {
		log.Fatal(err.String())
	}
}

func InitCache() {
	Feeds = make(map[string]Feed)
	// Ensure the cache directory exists
	cachePath := path.Join(os.Getenv("HOME"), ".munch.d", "cache")
	os.MkdirAll(cachePath, 0700)
	// For each feed
	for _, info := range Config.Feeds {
		name := info.Name
		fPath := path.Join(cachePath, name)
		file, _ := os.Open(fPath)
		if file != nil {
		} else {
			log.Print("New Feed: ", name)
			feed := Feed{}
			feed.Info = info
			feed.Items = make(map[string]Item)
			Feeds[name] = feed
		}
	}
}

func HandleUpdates() {
	pageBuffer := new(bytes.Buffer)
	tmplData := TemplateData{}
	tmplData.Feeds = Feeds
	tmplData.Config = &Config
	err := page_template.Execute(pageBuffer, tmplData)
	if err != nil {
		log.Print("ERROR: ", err.String())
	}
	for u := range updates {
		switch u {
		case DoUpdate:
			pageBuffer = new(bytes.Buffer)
			err = page_template.Execute(pageBuffer, tmplData)
			if err != nil {
				log.Print("ERROR: ", err.String())
			}
		case GetContent:
			content <- pageBuffer.String()
		default:
			panic("Undefined request to updater")
		}
	}
}

func WriteCache() {
	// TODO
}

func ReadFeeds() {
	log.Print("Updating feeds")
	for _, feed := range Feeds {
		switch (feed.Info.Type) {
		case "RSS":
			readRSS(feed)
		default:
			log.Print("Ignoring unknown feed of type ", feed.Info.Type)
		}
	}
	log.Print("Done")
}

func readRSS(feed Feed) {
	url := feed.Info.URL
	log.Print(url)
	r, err := client.Get(url)
	if err != nil {
		log.Print("ERROR: ", err.String())
		return
	}
	reader := r.Body
	feedData := RSSData{}
	err = xml.Unmarshal(reader, &feedData)
	if err != nil {
		log.Print("ERROR: ", err.String())
		return
	}
	// now transform the XML into our internal data structure
	changed := false
	for _, itemData := range feedData.Channel.Item {
		guid := itemData.GUID
		_, ok := feed.Items[guid]
		if !ok {
			// GUID not found - add the item
			changed = true
			item := Item {
				Title: itemData.Title,
				GUID: guid,
				URL: itemData.Link,
				Date: itemData.PubDate,
				Desc: itemData.Description,
				Content: itemData.Content,
				Read: false,
			}
			feed.Items[guid] = item
		}
	}
	if (changed) {
		UpdatePage()
		// TODO run some commands from Config?
	}
}

func readAtom(feed Feed) {

}

func readRDF(feed Feed) {

}

func UpdatePage() {
	updates <- DoUpdate
}

func RunHTTPServer() {
	log.Print("Spawning HTTP Server")
	http.HandleFunc("/", HTTPHandler)
	err := http.ListenAndServe("localhost:8090", nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err.String())
	}
}

func HTTPHandler(w http.ResponseWriter, req *http.Request) {
	updates <- GetContent
	w.Write([]byte(<-content))
}

<|endoftext|>


<commit_before>package sched

import (
    "log"
    "net"
    "time"
    "sync"
    "strings"
    "container/list"
    "huabot-sched/db"
)

type Sched struct {
    TotalWorkerCount int
    timer            *time.Timer
    grabQueue        *list.List
    jobQueue         *list.List
    entryPoint       string
    JobLocker        *sync.Mutex
    Funcs            map[string]*FuncStat
}

type FuncStat struct {
    TotalWorker uint `json:"worker_count"`
    TotalJob    uint `json:"job_count"`
    ProcJob     uint `json:"processing"`
}

func (stat *FuncStat) IncrWorker() uint {
    stat.TotalWorker += 1
    return stat.TotalWorker
}

func (stat *FuncStat) DecrWorker() uint {
    stat.TotalWorker -= 1
    return stat.TotalWorker
}

func (stat *FuncStat) IncrJob() uint {
    stat.TotalJob += 1
    return stat.TotalJob
}

func (stat *FuncStat) DecrJob() uint {
    stat.TotalJob -= 1
    return stat.TotalJob
}

func (stat *FuncStat) IncrProc() uint {
    stat.ProcJob += 1
    return stat.ProcJob
}

func (stat *FuncStat) DecrProc() uint {
    stat.ProcJob -= 1
    return stat.ProcJob
}

func NewSched(entryPoint string) *Sched {
    sched := new(Sched)
    sched.TotalWorkerCount = 0
    sched.timer = time.NewTimer(1 * time.Hour)
    sched.grabQueue = list.New()
    sched.jobQueue = list.New()
    sched.entryPoint = entryPoint
    sched.JobLocker = new(sync.Mutex)
    sched.Funcs = make(map[string]*FuncStat)
    return sched
}

func (sched *Sched) Serve() {
    parts := strings.SplitN(sched.entryPoint, "://", 2)
    if parts[0] == "unix" {
        sockCheck(parts[1])
    }
    sched.checkJobQueue()
    go sched.handle()
    listen, err := net.Listen(parts[0], parts[1])
    if err != nil {
        log.Fatal(err)
    }
    defer listen.Close()
    log.Printf("huabot-sched started on %s\n", sched.entryPoint)
    for {
        conn, err := listen.Accept()
        if err != nil {
            log.Fatal(err)
        }
        sched.HandleConnection(conn)
    }
}

func (sched *Sched) Notify() {
    sched.timer.Reset(time.Millisecond)
}

func (sched *Sched) DieWorker(worker *Worker) {
    defer sched.Notify()
    sched.TotalWorkerCount -= 1
    log.Printf("Total worker: %d\n", sched.TotalWorkerCount)
    sched.removeGrabQueue(worker)
    worker.Close()
}

func (sched *Sched) HandleConnection(conn net.Conn) {
    c := Conn{Conn: conn}
    payload, err := c.Receive()
    if err != nil {
        return
    }
    switch payload[0] {
    case TYPE_CLIENT:
        client := NewClient(sched, c)
        go client.Handle()
        break
    case TYPE_WORKER:
        worker := NewWorker(sched, c)
        sched.TotalWorkerCount += 1
        log.Printf("Total worker: %d\n", sched.TotalWorkerCount)
        go worker.Handle()
        break
    default:
        log.Printf("Unsupport client %d\n", payload[0])
        c.Close()
        break
    }
}

func (sched *Sched) Done(jobId int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, err := db.GetJob(jobId)
    if err == nil {
        job.Delete()
        sched.DecrStatJob(job)
        sched.DecrStatProc(job)
    }
    return
}

func (sched *Sched) isDoJob(job db.Job) bool {
    now := time.Now()
    current := int64(now.Unix())
    ret := false
    for e := sched.jobQueue.Front(); e != nil; e = e.Next() {
        chk := e.Value.(db.Job)
        runAt := chk.RunAt
        if runAt < chk.SchedAt {
            runAt = chk.SchedAt
        }
        if chk.Timeout > 0 && runAt + chk.Timeout < current {
            newJob, _ := db.GetJob(chk.Id)
            if newJob.Status == db.JOB_STATUS_PROC {
                sched.DecrStatProc(newJob)
                newJob.Status = db.JOB_STATUS_READY
                newJob.Save()
            }
            sched.jobQueue.Remove(e)
            continue
        }
        if chk.Id == job.Id {
            old := e.Value.(db.Job)
            runAt := old.RunAt
            if runAt < old.SchedAt {
                runAt = old.SchedAt
            }
            if old.Timeout > 0 && runAt + old.Timeout < current {
                ret = false
            } else {
                ret = true
            }
        }
    }
    return ret
}

func (sched *Sched) SubmitJob(worker *Worker, job db.Job) {
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if job.Name == "" {
        job.Delete()
        return
    }
    if sched.isDoJob(job) {
        return
    }
    if !worker.alive {
        return
    }
    if err := worker.HandleDo(job); err != nil {
        worker.alive = false
        go sched.DieWorker(worker)
        return
    }
    now := time.Now()
    current := int64(now.Unix())
    job.Status = db.JOB_STATUS_PROC
    job.RunAt = current
    job.Save()
    sched.IncrStatProc(job)
    sched.jobQueue.PushBack(job)
    sched.removeGrabQueue(worker)
}

func (sched *Sched) handle() {
    var current time.Time
    var timestamp int64
    var schedJob db.Job
    var isFirst bool
    for {
        if sched.grabQueue.Len() == 0 {
            sched.timer.Reset(time.Minute)
            current =<-sched.timer.C
            continue
        }

        isFirst = true
        for Func, stat := range sched.Funcs {
            if stat.TotalWorker == 0 || (stat.TotalJob > 0 && stat.ProcJob == stat.TotalJob) {
                continue
            }
            jobs, err := db.RangeSchedJob(Func, db.JOB_STATUS_READY, 0, 0)
            if err != nil || len(jobs) == 0 {
                stat.TotalJob = stat.ProcJob
                continue
            }

            if isFirst {
                schedJob = jobs[0]
                isFirst = false
                continue
            }

            if schedJob.SchedAt > jobs[0].SchedAt {
                schedJob = jobs[0]
            }
        }
        if isFirst {
            sched.timer.Reset(time.Minute)
            current =<-sched.timer.C
            continue
        }

        timestamp = int64(time.Now().Unix())

        if schedJob.SchedAt > timestamp {
            sched.timer.Reset(time.Second * time.Duration(schedJob.SchedAt - timestamp))
            current =<-sched.timer.C
            timestamp = int64(current.Unix())
            if schedJob.SchedAt > timestamp {
                continue
            }
        }

        isSubmited := false
        for e := sched.grabQueue.Front(); e != nil; e = e.Next() {
            worker := e.Value.(*Worker)
            for _, Func := range worker.Funcs {
                if schedJob.Func == Func {
                    sched.SubmitJob(worker, schedJob)
                    isSubmited = true
                    break
                }
            }
            if isSubmited {
                break
            }
        }

        if !isSubmited {
            sched.DecrStatFunc(schedJob.Func)
        }
    }
}

func (sched *Sched) Fail(jobId int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, _ := db.GetJob(jobId)
    sched.DecrStatProc(job)
    job.Status = db.JOB_STATUS_READY
    job.Save()
    return
}

func (sched *Sched) IncrStatFunc(Func string) {
    stat, ok := sched.Funcs[Func]
    if !ok {
        stat = new(FuncStat)
        sched.Funcs[Func] = stat
    }
    stat.IncrWorker()
}

func (sched *Sched) DecrStatFunc(Func string) {
    stat, ok := sched.Funcs[Func]
    if ok {
        stat.DecrWorker()
    }
}

func (sched *Sched) IncrStatJob(job db.Job) {
    stat, ok := sched.Funcs[job.Func]
    if !ok {
        stat = new(FuncStat)
        sched.Funcs[job.Func] = stat
    }
    stat.IncrJob()
}

func (sched *Sched) DecrStatJob(job db.Job) {
    stat, ok := sched.Funcs[job.Func]
    if ok {
        stat.DecrJob()
    }
}

func (sched *Sched) IncrStatProc(job db.Job) {
    stat, ok := sched.Funcs[job.Func]
    if !ok {
        stat = new(FuncStat)
        sched.Funcs[job.Func] = stat
    }
    if job.Status == db.JOB_STATUS_PROC {
        stat.IncrProc()
    }
}

func (sched *Sched) DecrStatProc(job db.Job) {
    stat, ok := sched.Funcs[job.Func]
    if ok && job.Status == db.JOB_STATUS_PROC {
        stat.DecrProc()
    }
}

func (sched *Sched) SchedLater(jobId int64, delay int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, _ := db.GetJob(jobId)
    sched.DecrStatProc(job)
    job.Status = db.JOB_STATUS_READY
    var now = time.Now()
    job.SchedAt = int64(now.Unix()) + delay
    job.Save()
    return
}

func (sched *Sched) removeGrabQueue(worker *Worker) {
    for e := sched.grabQueue.Front(); e != nil; e = e.Next() {
        if e.Value.(*Worker) == worker {
            sched.grabQueue.Remove(e)
        }
    }
}

func (sched *Sched) checkJobQueue() {
    start := 0
    limit := 20
    total, _ := db.CountJob()
    updateQueue := make([]db.Job, 0)
    removeQueue := make([]db.Job, 0)
    var now = time.Now()
    current := int64(now.Unix())

    for start = 0; start < int(total); start += limit {
        jobs, _ := db.RangeJob(start, start + limit - 1)
        for _, job := range jobs {
            if job.Name == "" {
                removeQueue = append(removeQueue, job)
                continue
            }
            sched.IncrStatJob(job)
            if job.Status != db.JOB_STATUS_PROC {
                continue
            }
            runAt := job.RunAt
            if runAt < job.SchedAt {
                runAt = job.SchedAt
            }
            if runAt + job.Timeout < current {
                updateQueue = append(updateQueue, job)
            } else {
                sched.jobQueue.PushBack(job)
                sched.IncrStatProc(job)
            }
        }
    }

    for _, job := range updateQueue {
        job.Status = db.JOB_STATUS_READY
        job.Save()
    }

    for _, job := range removeQueue {
        job.Delete()
    }
}

func (sched *Sched) Close() {
}
<commit_msg>update Counter<commit_after>package sched

import (
    "log"
    "net"
    "time"
    "sync"
    "strings"
    "container/list"
    "huabot-sched/db"
)

type Sched struct {
    TotalWorkerCount int
    timer            *time.Timer
    grabQueue        *list.List
    jobQueue         *list.List
    entryPoint       string
    JobLocker        *sync.Mutex
    Funcs            map[string]*FuncStat
}

type Counter uint

func (c *Counter) Incr() {
    *c = *c + 1
}

func (c *Counter) Decr() {
    *c = *c - 1
}

type FuncStat struct {
    TotalWorker Counter `json:"worker_count"`
    TotalJob    Counter `json:"job_count"`
    ProcJob     Counter `json:"processing"`
}

func NewSched(entryPoint string) *Sched {
    sched := new(Sched)
    sched.TotalWorkerCount = 0
    sched.timer = time.NewTimer(1 * time.Hour)
    sched.grabQueue = list.New()
    sched.jobQueue = list.New()
    sched.entryPoint = entryPoint
    sched.JobLocker = new(sync.Mutex)
    sched.Funcs = make(map[string]*FuncStat)
    return sched
}

func (sched *Sched) Serve() {
    parts := strings.SplitN(sched.entryPoint, "://", 2)
    if parts[0] == "unix" {
        sockCheck(parts[1])
    }
    sched.checkJobQueue()
    go sched.handle()
    listen, err := net.Listen(parts[0], parts[1])
    if err != nil {
        log.Fatal(err)
    }
    defer listen.Close()
    log.Printf("huabot-sched started on %s\n", sched.entryPoint)
    for {
        conn, err := listen.Accept()
        if err != nil {
            log.Fatal(err)
        }
        sched.HandleConnection(conn)
    }
}

func (sched *Sched) Notify() {
    sched.timer.Reset(time.Millisecond)
}

func (sched *Sched) DieWorker(worker *Worker) {
    defer sched.Notify()
    sched.TotalWorkerCount -= 1
    log.Printf("Total worker: %d\n", sched.TotalWorkerCount)
    sched.removeGrabQueue(worker)
    worker.Close()
}

func (sched *Sched) HandleConnection(conn net.Conn) {
    c := Conn{Conn: conn}
    payload, err := c.Receive()
    if err != nil {
        return
    }
    switch payload[0] {
    case TYPE_CLIENT:
        client := NewClient(sched, c)
        go client.Handle()
        break
    case TYPE_WORKER:
        worker := NewWorker(sched, c)
        sched.TotalWorkerCount += 1
        log.Printf("Total worker: %d\n", sched.TotalWorkerCount)
        go worker.Handle()
        break
    default:
        log.Printf("Unsupport client %d\n", payload[0])
        c.Close()
        break
    }
}

func (sched *Sched) Done(jobId int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, err := db.GetJob(jobId)
    if err == nil {
        job.Delete()
        sched.DecrStatJob(job)
        sched.DecrStatProc(job)
    }
    return
}

func (sched *Sched) isDoJob(job db.Job) bool {
    now := time.Now()
    current := int64(now.Unix())
    ret := false
    for e := sched.jobQueue.Front(); e != nil; e = e.Next() {
        chk := e.Value.(db.Job)
        runAt := chk.RunAt
        if runAt < chk.SchedAt {
            runAt = chk.SchedAt
        }
        if chk.Timeout > 0 && runAt + chk.Timeout < current {
            newJob, _ := db.GetJob(chk.Id)
            if newJob.Status == db.JOB_STATUS_PROC {
                sched.DecrStatProc(newJob)
                newJob.Status = db.JOB_STATUS_READY
                newJob.Save()
            }
            sched.jobQueue.Remove(e)
            continue
        }
        if chk.Id == job.Id {
            old := e.Value.(db.Job)
            runAt := old.RunAt
            if runAt < old.SchedAt {
                runAt = old.SchedAt
            }
            if old.Timeout > 0 && runAt + old.Timeout < current {
                ret = false
            } else {
                ret = true
            }
        }
    }
    return ret
}

func (sched *Sched) SubmitJob(worker *Worker, job db.Job) {
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    if job.Name == "" {
        job.Delete()
        return
    }
    if sched.isDoJob(job) {
        return
    }
    if !worker.alive {
        return
    }
    if err := worker.HandleDo(job); err != nil {
        worker.alive = false
        go sched.DieWorker(worker)
        return
    }
    now := time.Now()
    current := int64(now.Unix())
    job.Status = db.JOB_STATUS_PROC
    job.RunAt = current
    job.Save()
    sched.IncrStatProc(job)
    sched.jobQueue.PushBack(job)
    sched.removeGrabQueue(worker)
}

func (sched *Sched) handle() {
    var current time.Time
    var timestamp int64
    var schedJob db.Job
    var isFirst bool
    for {
        if sched.grabQueue.Len() == 0 {
            sched.timer.Reset(time.Minute)
            current =<-sched.timer.C
            continue
        }

        isFirst = true
        for Func, stat := range sched.Funcs {
            if stat.TotalWorker == 0 || (stat.TotalJob > 0 && stat.ProcJob == stat.TotalJob) {
                continue
            }
            jobs, err := db.RangeSchedJob(Func, db.JOB_STATUS_READY, 0, 0)
            if err != nil || len(jobs) == 0 {
                stat.TotalJob = stat.ProcJob
                continue
            }

            if isFirst {
                schedJob = jobs[0]
                isFirst = false
                continue
            }

            if schedJob.SchedAt > jobs[0].SchedAt {
                schedJob = jobs[0]
            }
        }
        if isFirst {
            sched.timer.Reset(time.Minute)
            current =<-sched.timer.C
            continue
        }

        timestamp = int64(time.Now().Unix())

        if schedJob.SchedAt > timestamp {
            sched.timer.Reset(time.Second * time.Duration(schedJob.SchedAt - timestamp))
            current =<-sched.timer.C
            timestamp = int64(current.Unix())
            if schedJob.SchedAt > timestamp {
                continue
            }
        }

        isSubmited := false
        for e := sched.grabQueue.Front(); e != nil; e = e.Next() {
            worker := e.Value.(*Worker)
            for _, Func := range worker.Funcs {
                if schedJob.Func == Func {
                    sched.SubmitJob(worker, schedJob)
                    isSubmited = true
                    break
                }
            }
            if isSubmited {
                break
            }
        }

        if !isSubmited {
            sched.DecrStatFunc(schedJob.Func)
        }
    }
}

func (sched *Sched) Fail(jobId int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, _ := db.GetJob(jobId)
    sched.DecrStatProc(job)
    job.Status = db.JOB_STATUS_READY
    job.Save()
    return
}

func (sched *Sched) IncrStatFunc(Func string) {
    stat, ok := sched.Funcs[Func]
    if !ok {
        stat = new(FuncStat)
        sched.Funcs[Func] = stat
    }
    stat.TotalWorker.Incr()
}

func (sched *Sched) DecrStatFunc(Func string) {
    stat, ok := sched.Funcs[Func]
    if ok {
        stat.TotalWorker.Decr()
    }
}

func (sched *Sched) IncrStatJob(job db.Job) {
    stat, ok := sched.Funcs[job.Func]
    if !ok {
        stat = new(FuncStat)
        sched.Funcs[job.Func] = stat
    }
    stat.TotalJob.Incr()
}

func (sched *Sched) DecrStatJob(job db.Job) {
    stat, ok := sched.Funcs[job.Func]
    if ok {
        stat.TotalJob.Decr()
    }
}

func (sched *Sched) IncrStatProc(job db.Job) {
    stat, ok := sched.Funcs[job.Func]
    if !ok {
        stat = new(FuncStat)
        sched.Funcs[job.Func] = stat
    }
    if job.Status == db.JOB_STATUS_PROC {
        stat.ProcJob.Incr()
    }
}

func (sched *Sched) DecrStatProc(job db.Job) {
    stat, ok := sched.Funcs[job.Func]
    if ok && job.Status == db.JOB_STATUS_PROC {
        stat.ProcJob.Decr()
    }
}

func (sched *Sched) SchedLater(jobId int64, delay int64) {
    defer sched.Notify()
    defer sched.JobLocker.Unlock()
    sched.JobLocker.Lock()
    removeListJob(sched.jobQueue, jobId)
    job, _ := db.GetJob(jobId)
    sched.DecrStatProc(job)
    job.Status = db.JOB_STATUS_READY
    var now = time.Now()
    job.SchedAt = int64(now.Unix()) + delay
    job.Save()
    return
}

func (sched *Sched) removeGrabQueue(worker *Worker) {
    for e := sched.grabQueue.Front(); e != nil; e = e.Next() {
        if e.Value.(*Worker) == worker {
            sched.grabQueue.Remove(e)
        }
    }
}

func (sched *Sched) checkJobQueue() {
    start := 0
    limit := 20
    total, _ := db.CountJob()
    updateQueue := make([]db.Job, 0)
    removeQueue := make([]db.Job, 0)
    var now = time.Now()
    current := int64(now.Unix())

    for start = 0; start < int(total); start += limit {
        jobs, _ := db.RangeJob(start, start + limit - 1)
        for _, job := range jobs {
            if job.Name == "" {
                removeQueue = append(removeQueue, job)
                continue
            }
            sched.IncrStatJob(job)
            if job.Status != db.JOB_STATUS_PROC {
                continue
            }
            runAt := job.RunAt
            if runAt < job.SchedAt {
                runAt = job.SchedAt
            }
            if runAt + job.Timeout < current {
                updateQueue = append(updateQueue, job)
            } else {
                sched.jobQueue.PushBack(job)
                sched.IncrStatProc(job)
            }
        }
    }

    for _, job := range updateQueue {
        job.Status = db.JOB_STATUS_READY
        job.Save()
    }

    for _, job := range removeQueue {
        job.Delete()
    }
}

func (sched *Sched) Close() {
}
<|endoftext|>


<commit_before>package sdl

/*
#include "sdl_wrapper.h"
*/
import "C"
import "unsafe"
import "reflect"

const (
	SWSURFACE = C.SDL_SWSURFACE
	PREALLOC  = C.SDL_PREALLOC
	RLEACCEL  = C.SDL_RLEACCEL
	DONTFREE  = C.SDL_DONTFREE
)

// Surface (https://wiki.libsdl.org/SDL_Surface)
type Surface struct {
	Flags    uint32
	Format   *PixelFormat
	W        int32
	H        int32
	Pitch    int32
	pixels   unsafe.Pointer // use Pixels() for access
	UserData unsafe.Pointer
	Locked   int32
	LockData unsafe.Pointer
	ClipRect Rect
	_        unsafe.Pointer // private c field 'map'
	RefCount int32
}
type cSurface C.SDL_Surface

func (surface *Surface) cptr() *C.SDL_Surface {
	return (*C.SDL_Surface)(unsafe.Pointer(surface))
}

// Surface (https://wiki.libsdl.org/SDL_MustLock)
func (surface *Surface) MustLock() bool {
	return (surface.Flags & RLEACCEL) != 0
}

// CreateRGBSurface (https://wiki.libsdl.org/SDL_CreateRGBSurface)
func CreateRGBSurface(flags uint32, width, height, depth int32, Rmask, Gmask, Bmask, Amask uint32) *Surface {
	return (*Surface)(unsafe.Pointer(C.SDL_CreateRGBSurface(C.Uint32(flags),
		C.int(width),
		C.int(height),
		C.int(depth),
		C.Uint32(Rmask),
		C.Uint32(Gmask),
		C.Uint32(Bmask),
		C.Uint32(Amask))))
}

// CreateRGBSurfaceFrom (https://wiki.libsdl.org/SDL_CreateRGBSurfaceFrom)
func CreateRGBSurfaceFrom(pixels unsafe.Pointer, width, height, depth, pitch int, Rmask, Gmask, Bmask, Amask uint32) *Surface {
	return (*Surface)(unsafe.Pointer(C.SDL_CreateRGBSurfaceFrom(pixels,
		C.int(width),
		C.int(height),
		C.int(depth),
		C.int(pitch),
		C.Uint32(Rmask),
		C.Uint32(Gmask),
		C.Uint32(Bmask),
		C.Uint32(Amask))))
}

// Surface (https://wiki.libsdl.org/SDL_FreeSurface)
func (surface *Surface) Free() {
	C.SDL_FreeSurface(surface.cptr())
}

// Surface (https://wiki.libsdl.org/SDL_SetSurfacePalette)
func (surface *Surface) SetPalette(palette *Palette) error {
	if C.SDL_SetSurfacePalette(surface.cptr(), palette.cptr()) != 0 {
		return GetError()
	}
	return nil
}

// Surface (https://wiki.libsdl.org/SDL_LockSurface)
func (surface *Surface) Lock() {
	C.SDL_LockSurface(surface.cptr())
}

// Surface (https://wiki.libsdl.org/SDL_UnlockSurface)
func (surface *Surface) Unlock() {
	C.SDL_UnlockSurface(surface.cptr())
}

// LoadBMP_RW (https://wiki.libsdl.org/SDL_LoadBMP_RW)
func LoadBMP_RW(src *RWops, freeSrc int) *Surface {
	_surface := C.SDL_LoadBMP_RW(src.cptr(), C.int(freeSrc))
	return (*Surface)(unsafe.Pointer(_surface))
}

// LoadBMP (https://wiki.libsdl.org/SDL_LoadBMP)
func LoadBMP(file string) *Surface {
	return (*Surface)(LoadBMP_RW(RWFromFile(file, "rb"), 1))
}

// Surface (https://wiki.libsdl.org/SDL_SaveBMP_RW)
func (surface *Surface) SaveBMP_RW(dst *RWops, freeDst int) int {
	return int(C.SDL_SaveBMP_RW(surface.cptr(), dst.cptr(), C.int(freeDst)))
}

// Surface (https://wiki.libsdl.org/SDL_SaveBMP)
func (surface *Surface) SaveBMP(file string) int {
	return int(surface.SaveBMP_RW(RWFromFile(file, "wb"), 1))
}

// Surface (https://wiki.libsdl.org/SDL_SetSurfaceRLE)
func (surface *Surface) SetRLE(flag int) int {
	return int(C.SDL_SetSurfaceRLE(surface.cptr(), C.int(flag)))
}

// Surface (https://wiki.libsdl.org/SDL_SetColorKey)
func (surface *Surface) SetColorKey(flag int, key uint32) int {
	return int(C.SDL_SetColorKey(surface.cptr(), C.int(flag), C.Uint32(key)))
}

// Surface (https://wiki.libsdl.org/SDL_GetColorKey)
func (surface *Surface) GetColorKey() (key uint32, status int) {
	_key := (*C.Uint32)(unsafe.Pointer(&key))
	status = int(C.SDL_GetColorKey(surface.cptr(), _key))
	return key, status
}

// Surface (https://wiki.libsdl.org/SDL_SetSurfaceColorMod)
func (surface *Surface) SetColorMod(r, g, b uint8) int {
	return int(C.SDL_SetSurfaceColorMod(surface.cptr(), C.Uint8(r), C.Uint8(g), C.Uint8(b)))
}

// Surface (https://wiki.libsdl.org/SDL_GetSurfaceColorMod)
func (surface *Surface) GetColorMod() (r, g, b uint8, status int) {
	_r := (*C.Uint8)(unsafe.Pointer(&r))
	_g := (*C.Uint8)(unsafe.Pointer(&g))
	_b := (*C.Uint8)(unsafe.Pointer(&b))
	status = int(C.SDL_GetSurfaceColorMod(surface.cptr(), _r, _g, _b))
	return r, g, b, status
}

// Surface (https://wiki.libsdl.org/SDL_SetSurfaceAlphaMod)
func (surface *Surface) SetAlphaMod(alpha uint8) int {
	return int(C.SDL_SetSurfaceAlphaMod(surface.cptr(), C.Uint8(alpha)))
}

// Surface (https://wiki.libsdl.org/SDL_GetSurfaceAlphaMod)
func (surface *Surface) GetAlphaMod() (alpha uint8, status int) {
	_alpha := (*C.Uint8)(unsafe.Pointer(&alpha))
	status = int(C.SDL_GetSurfaceAlphaMod(surface.cptr(), _alpha))
	return alpha, status
}

// Surface (https://wiki.libsdl.org/SDL_SetSurfaceBlendMode)
func (surface *Surface) SetBlendMode(bm BlendMode) int {
	return int(C.SDL_SetSurfaceBlendMode(surface.cptr(), bm.c()))
}

// Surface (https://wiki.libsdl.org/SDL_GetSurfaceBlendMode)
func (surface *Surface) GetBlendMode() (bm BlendMode, status int) {
	status = int(C.SDL_GetSurfaceBlendMode(surface.cptr(), bm.cptr()))
	return bm, status
}

// Surface (https://wiki.libsdl.org/SDL_SetClipRect)
func (surface *Surface) SetClipRect(rect *Rect) bool {
	return C.SDL_SetClipRect(surface.cptr(), rect.cptr()) > 0
}

// Surface (https://wiki.libsdl.org/SDL_GetClipRect)
func (surface *Surface) GetClipRect(rect *Rect) {
	C.SDL_GetClipRect(surface.cptr(), rect.cptr())
}

// Surface (https://wiki.libsdl.org/SDL_ConvertSurface)
func (surface *Surface) Convert(fmt *PixelFormat, flags uint32) *Surface {
	_surface := C.SDL_ConvertSurface(surface.cptr(), fmt.cptr(), C.Uint32(flags))
	return (*Surface)(unsafe.Pointer(_surface))
}

// Surface (https://wiki.libsdl.org/SDL_ConvertSurfaceFormat)
func (surface *Surface) ConvertFormat(pixelFormat uint32, flags uint32) *Surface {
	return (*Surface)(unsafe.Pointer(C.SDL_ConvertSurfaceFormat(surface.cptr(), C.Uint32(pixelFormat), C.Uint32(flags))))
}

// ConvertPixels (https://wiki.libsdl.org/SDL_ConvertPixels)
func ConvertPixels(width, height int, srcFormat uint32, src unsafe.Pointer, srcPitch int,
	dstFormat uint32, dst unsafe.Pointer, dstPitch int) int {
	return int(C.SDL_ConvertPixels(C.int(width), C.int(height), C.Uint32(srcFormat), src, C.int(srcPitch), C.Uint32(dstFormat), dst, C.int(dstPitch)))
}

// Surface (https://wiki.libsdl.org/SDL_FillRect)
func (surface *Surface) FillRect(rect *Rect, color uint32) error {
	if C.SDL_FillRect(surface.cptr(), rect.cptr(), C.Uint32(color)) != 0 {
		return GetError()
	}
	return nil
}

// Surface (https://wiki.libsdl.org/SDL_FillRects)
func (surface *Surface) FillRects(rects []Rect, color uint32) error {
	if C.SDL_FillRects(surface.cptr(), rects[0].cptr(), C.int(len(rects)), C.Uint32(color)) != 0 {
		return GetError()
	}
	return nil
}

// Surface (https://wiki.libsdl.org/SDL_BlitSurface)
func (src *Surface) Blit(srcRect *Rect, dst *Surface, dstRect *Rect) int {
	return int(C.SDL_BlitSurface(src.cptr(), srcRect.cptr(), dst.cptr(), dstRect.cptr()))
}

// Surface (https://wiki.libsdl.org/SDL_BlitScaled)
func (src *Surface) BlitScaled(srcRect *Rect, dst *Surface, dstRect *Rect) int {
	return int(C.SDL_BlitScaled(src.cptr(), srcRect.cptr(), dst.cptr(), dstRect.cptr()))
}

// Surface (https://wiki.libsdl.org/SDL_UpperBlit)
func (src *Surface) UpperBlit(srcRect *Rect, dst *Surface, dstRect *Rect) int {
	return int(C.SDL_UpperBlit(src.cptr(), srcRect.cptr(), dst.cptr(), dstRect.cptr()))
}

// Surface (https://wiki.libsdl.org/SDL_LowerBlit)
func (src *Surface) LowerBlit(srcRect *Rect, dst *Surface, dstRect *Rect) int {
	return int(C.SDL_LowerBlit(src.cptr(), srcRect.cptr(), dst.cptr(), dstRect.cptr()))
}

// Surface (https://wiki.libsdl.org/SDL_SoftStretch)
func (src *Surface) SoftStretch(srcRect *Rect, dst *Surface, dstRect *Rect) int {
	return int(C.SDL_SoftStretch(src.cptr(), srcRect.cptr(), dst.cptr(), dstRect.cptr()))
}

// Surface (https://wiki.libsdl.org/SDL_UpperBlitScaled)
func (src *Surface) UpperBlitScaled(srcRect *Rect, dst *Surface, dstRect *Rect) int {
	return int(C.SDL_UpperBlitScaled(src.cptr(), srcRect.cptr(), dst.cptr(), dstRect.cptr()))
}

// Surface (https://wiki.libsdl.org/SDL_LowerBlitScaled)
func (src *Surface) LowerBlitScaled(srcRect *Rect, dst *Surface, dstRect *Rect) int {
	return int(C.SDL_LowerBlitScaled(src.cptr(), srcRect.cptr(), dst.cptr(), dstRect.cptr()))
}

func (surface *Surface) PixelNum() int {
	return int(surface.W * surface.H)
}

func (surface *Surface) BytesPerPixel() int {
	return int(surface.Format.BytesPerPixel)
}

func (surface *Surface) Pixels() []byte {
	var b []byte
	length := int(surface.W*surface.H) * int(surface.Format.BytesPerPixel)
	sliceHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b))
	sliceHeader.Cap = int(length)
	sliceHeader.Len = int(length)
	sliceHeader.Data = uintptr(surface.pixels)
	return b
}

func (surface *Surface) Data() unsafe.Pointer {
	return surface.pixels
}
<commit_msg>Make SaveBMP and SaveBMP_RW idiomatic with respect to errors<commit_after>package sdl

/*
#include "sdl_wrapper.h"
*/
import "C"
import "unsafe"
import "reflect"

const (
	SWSURFACE = C.SDL_SWSURFACE
	PREALLOC  = C.SDL_PREALLOC
	RLEACCEL  = C.SDL_RLEACCEL
	DONTFREE  = C.SDL_DONTFREE
)

// Surface (https://wiki.libsdl.org/SDL_Surface)
type Surface struct {
	Flags    uint32
	Format   *PixelFormat
	W        int32
	H        int32
	Pitch    int32
	pixels   unsafe.Pointer // use Pixels() for access
	UserData unsafe.Pointer
	Locked   int32
	LockData unsafe.Pointer
	ClipRect Rect
	_        unsafe.Pointer // private c field 'map'
	RefCount int32
}
type cSurface C.SDL_Surface

func (surface *Surface) cptr() *C.SDL_Surface {
	return (*C.SDL_Surface)(unsafe.Pointer(surface))
}

// Surface (https://wiki.libsdl.org/SDL_MustLock)
func (surface *Surface) MustLock() bool {
	return (surface.Flags & RLEACCEL) != 0
}

// CreateRGBSurface (https://wiki.libsdl.org/SDL_CreateRGBSurface)
func CreateRGBSurface(flags uint32, width, height, depth int32, Rmask, Gmask, Bmask, Amask uint32) *Surface {
	return (*Surface)(unsafe.Pointer(C.SDL_CreateRGBSurface(C.Uint32(flags),
		C.int(width),
		C.int(height),
		C.int(depth),
		C.Uint32(Rmask),
		C.Uint32(Gmask),
		C.Uint32(Bmask),
		C.Uint32(Amask))))
}

// CreateRGBSurfaceFrom (https://wiki.libsdl.org/SDL_CreateRGBSurfaceFrom)
func CreateRGBSurfaceFrom(pixels unsafe.Pointer, width, height, depth, pitch int, Rmask, Gmask, Bmask, Amask uint32) *Surface {
	return (*Surface)(unsafe.Pointer(C.SDL_CreateRGBSurfaceFrom(pixels,
		C.int(width),
		C.int(height),
		C.int(depth),
		C.int(pitch),
		C.Uint32(Rmask),
		C.Uint32(Gmask),
		C.Uint32(Bmask),
		C.Uint32(Amask))))
}

// Surface (https://wiki.libsdl.org/SDL_FreeSurface)
func (surface *Surface) Free() {
	C.SDL_FreeSurface(surface.cptr())
}

// Surface (https://wiki.libsdl.org/SDL_SetSurfacePalette)
func (surface *Surface) SetPalette(palette *Palette) error {
	if C.SDL_SetSurfacePalette(surface.cptr(), palette.cptr()) != 0 {
		return GetError()
	}
	return nil
}

// Surface (https://wiki.libsdl.org/SDL_LockSurface)
func (surface *Surface) Lock() {
	C.SDL_LockSurface(surface.cptr())
}

// Surface (https://wiki.libsdl.org/SDL_UnlockSurface)
func (surface *Surface) Unlock() {
	C.SDL_UnlockSurface(surface.cptr())
}

// LoadBMP_RW (https://wiki.libsdl.org/SDL_LoadBMP_RW)
func LoadBMP_RW(src *RWops, freeSrc int) *Surface {
	_surface := C.SDL_LoadBMP_RW(src.cptr(), C.int(freeSrc))
	return (*Surface)(unsafe.Pointer(_surface))
}

// LoadBMP (https://wiki.libsdl.org/SDL_LoadBMP)
func LoadBMP(file string) *Surface {
	return (*Surface)(LoadBMP_RW(RWFromFile(file, "rb"), 1))
}

// Surface (https://wiki.libsdl.org/SDL_SaveBMP_RW)
func (surface *Surface) SaveBMP_RW(dst *RWops, freeDst int) error {
	if C.SDL_SaveBMP_RW(surface.cptr(), dst.cptr(), C.int(freeDst)) != 0 {
		return GetError()
	}
	return nil
}

// Surface (https://wiki.libsdl.org/SDL_SaveBMP)
func (surface *Surface) SaveBMP(file string) error {
	return surface.SaveBMP_RW(RWFromFile(file, "wb"), 1)
}

// Surface (https://wiki.libsdl.org/SDL_SetSurfaceRLE)
func (surface *Surface) SetRLE(flag int) int {
	return int(C.SDL_SetSurfaceRLE(surface.cptr(), C.int(flag)))
}

// Surface (https://wiki.libsdl.org/SDL_SetColorKey)
func (surface *Surface) SetColorKey(flag int, key uint32) int {
	return int(C.SDL_SetColorKey(surface.cptr(), C.int(flag), C.Uint32(key)))
}

// Surface (https://wiki.libsdl.org/SDL_GetColorKey)
func (surface *Surface) GetColorKey() (key uint32, status int) {
	_key := (*C.Uint32)(unsafe.Pointer(&key))
	status = int(C.SDL_GetColorKey(surface.cptr(), _key))
	return key, status
}

// Surface (https://wiki.libsdl.org/SDL_SetSurfaceColorMod)
func (surface *Surface) SetColorMod(r, g, b uint8) int {
	return int(C.SDL_SetSurfaceColorMod(surface.cptr(), C.Uint8(r), C.Uint8(g), C.Uint8(b)))
}

// Surface (https://wiki.libsdl.org/SDL_GetSurfaceColorMod)
func (surface *Surface) GetColorMod() (r, g, b uint8, status int) {
	_r := (*C.Uint8)(unsafe.Pointer(&r))
	_g := (*C.Uint8)(unsafe.Pointer(&g))
	_b := (*C.Uint8)(unsafe.Pointer(&b))
	status = int(C.SDL_GetSurfaceColorMod(surface.cptr(), _r, _g, _b))
	return r, g, b, status
}

// Surface (https://wiki.libsdl.org/SDL_SetSurfaceAlphaMod)
func (surface *Surface) SetAlphaMod(alpha uint8) int {
	return int(C.SDL_SetSurfaceAlphaMod(surface.cptr(), C.Uint8(alpha)))
}

// Surface (https://wiki.libsdl.org/SDL_GetSurfaceAlphaMod)
func (surface *Surface) GetAlphaMod() (alpha uint8, status int) {
	_alpha := (*C.Uint8)(unsafe.Pointer(&alpha))
	status = int(C.SDL_GetSurfaceAlphaMod(surface.cptr(), _alpha))
	return alpha, status
}

// Surface (https://wiki.libsdl.org/SDL_SetSurfaceBlendMode)
func (surface *Surface) SetBlendMode(bm BlendMode) int {
	return int(C.SDL_SetSurfaceBlendMode(surface.cptr(), bm.c()))
}

// Surface (https://wiki.libsdl.org/SDL_GetSurfaceBlendMode)
func (surface *Surface) GetBlendMode() (bm BlendMode, status int) {
	status = int(C.SDL_GetSurfaceBlendMode(surface.cptr(), bm.cptr()))
	return bm, status
}

// Surface (https://wiki.libsdl.org/SDL_SetClipRect)
func (surface *Surface) SetClipRect(rect *Rect) bool {
	return C.SDL_SetClipRect(surface.cptr(), rect.cptr()) > 0
}

// Surface (https://wiki.libsdl.org/SDL_GetClipRect)
func (surface *Surface) GetClipRect(rect *Rect) {
	C.SDL_GetClipRect(surface.cptr(), rect.cptr())
}

// Surface (https://wiki.libsdl.org/SDL_ConvertSurface)
func (surface *Surface) Convert(fmt *PixelFormat, flags uint32) *Surface {
	_surface := C.SDL_ConvertSurface(surface.cptr(), fmt.cptr(), C.Uint32(flags))
	return (*Surface)(unsafe.Pointer(_surface))
}

// Surface (https://wiki.libsdl.org/SDL_ConvertSurfaceFormat)
func (surface *Surface) ConvertFormat(pixelFormat uint32, flags uint32) *Surface {
	return (*Surface)(unsafe.Pointer(C.SDL_ConvertSurfaceFormat(surface.cptr(), C.Uint32(pixelFormat), C.Uint32(flags))))
}

// ConvertPixels (https://wiki.libsdl.org/SDL_ConvertPixels)
func ConvertPixels(width, height int, srcFormat uint32, src unsafe.Pointer, srcPitch int,
	dstFormat uint32, dst unsafe.Pointer, dstPitch int) int {
	return int(C.SDL_ConvertPixels(C.int(width), C.int(height), C.Uint32(srcFormat), src, C.int(srcPitch), C.Uint32(dstFormat), dst, C.int(dstPitch)))
}

// Surface (https://wiki.libsdl.org/SDL_FillRect)
func (surface *Surface) FillRect(rect *Rect, color uint32) error {
	if C.SDL_FillRect(surface.cptr(), rect.cptr(), C.Uint32(color)) != 0 {
		return GetError()
	}
	return nil
}

// Surface (https://wiki.libsdl.org/SDL_FillRects)
func (surface *Surface) FillRects(rects []Rect, color uint32) error {
	if C.SDL_FillRects(surface.cptr(), rects[0].cptr(), C.int(len(rects)), C.Uint32(color)) != 0 {
		return GetError()
	}
	return nil
}

// Surface (https://wiki.libsdl.org/SDL_BlitSurface)
func (src *Surface) Blit(srcRect *Rect, dst *Surface, dstRect *Rect) int {
	return int(C.SDL_BlitSurface(src.cptr(), srcRect.cptr(), dst.cptr(), dstRect.cptr()))
}

// Surface (https://wiki.libsdl.org/SDL_BlitScaled)
func (src *Surface) BlitScaled(srcRect *Rect, dst *Surface, dstRect *Rect) int {
	return int(C.SDL_BlitScaled(src.cptr(), srcRect.cptr(), dst.cptr(), dstRect.cptr()))
}

// Surface (https://wiki.libsdl.org/SDL_UpperBlit)
func (src *Surface) UpperBlit(srcRect *Rect, dst *Surface, dstRect *Rect) int {
	return int(C.SDL_UpperBlit(src.cptr(), srcRect.cptr(), dst.cptr(), dstRect.cptr()))
}

// Surface (https://wiki.libsdl.org/SDL_LowerBlit)
func (src *Surface) LowerBlit(srcRect *Rect, dst *Surface, dstRect *Rect) int {
	return int(C.SDL_LowerBlit(src.cptr(), srcRect.cptr(), dst.cptr(), dstRect.cptr()))
}

// Surface (https://wiki.libsdl.org/SDL_SoftStretch)
func (src *Surface) SoftStretch(srcRect *Rect, dst *Surface, dstRect *Rect) int {
	return int(C.SDL_SoftStretch(src.cptr(), srcRect.cptr(), dst.cptr(), dstRect.cptr()))
}

// Surface (https://wiki.libsdl.org/SDL_UpperBlitScaled)
func (src *Surface) UpperBlitScaled(srcRect *Rect, dst *Surface, dstRect *Rect) int {
	return int(C.SDL_UpperBlitScaled(src.cptr(), srcRect.cptr(), dst.cptr(), dstRect.cptr()))
}

// Surface (https://wiki.libsdl.org/SDL_LowerBlitScaled)
func (src *Surface) LowerBlitScaled(srcRect *Rect, dst *Surface, dstRect *Rect) int {
	return int(C.SDL_LowerBlitScaled(src.cptr(), srcRect.cptr(), dst.cptr(), dstRect.cptr()))
}

func (surface *Surface) PixelNum() int {
	return int(surface.W * surface.H)
}

func (surface *Surface) BytesPerPixel() int {
	return int(surface.Format.BytesPerPixel)
}

func (surface *Surface) Pixels() []byte {
	var b []byte
	length := int(surface.W*surface.H) * int(surface.Format.BytesPerPixel)
	sliceHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b))
	sliceHeader.Cap = int(length)
	sliceHeader.Len = int(length)
	sliceHeader.Data = uintptr(surface.pixels)
	return b
}

func (surface *Surface) Data() unsafe.Pointer {
	return surface.pixels
}
<|endoftext|>


<commit_before>package bot

import (
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"runtime"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/demisto/alfred/conf"
	"github.com/demisto/alfred/domain"
	"github.com/demisto/alfred/queue"
	"github.com/demisto/alfred/repo"
	"github.com/demisto/goxforce"
	"github.com/slavikm/govt"
)

const (
	numOfPositivesToConvict = 3
	xfeScoreToConvict       = 3
)

// Worker reads messages from the queue and does the actual work
type Worker struct {
	q    queue.Queue
	c    chan *domain.WorkRequest
	r    repo.Repo
	xfe  *goxforce.Client
	vt   *govt.Client
	clam *clamEngine
}

// NewWorker that loads work messages from the queue
func NewWorker(r repo.Repo, q queue.Queue) (*Worker, error) {
	xfe, err := goxforce.New(goxforce.SetErrorLog(log.New(conf.LogWriter, "XFE:", log.Lshortfile)))
	if err != nil {
		return nil, err
	}
	vt, err := govt.New(govt.SetApikey(conf.Options.VT), govt.SetErrorLog(log.New(os.Stderr, "VT:", log.Lshortfile)))
	if err != nil {
		return nil, err
	}
	clam, err := newClamEngine()
	if err != nil {
		return nil, err
	}
	return &Worker{
		r:    r,
		q:    q,
		c:    make(chan *domain.WorkRequest, runtime.NumCPU()),
		xfe:  xfe,
		vt:   vt,
		clam: clam,
	}, nil
}

func (w *Worker) handle() {
	for msg := range w.c {
		if msg == nil {
			w.clam.close()
			return
		}
		if msg.ReplyQueue == "" {
			logrus.Warnf("Got message without a reply queue destination %+v\n", msg)
			continue
		}
		reply := &domain.WorkReply{Context: msg.Context, MessageID: msg.MessageID}
		switch msg.Type {
		case "message":
			if strings.Contains(msg.Text, "<http") {
				w.handleURL(msg.Text, reply)
			}
			if ip := ipReg.FindString(msg.Text); ip != "" {
				w.handleIP(ip, reply)
			}
			if hash := md5Reg.FindString(msg.Text); hash != "" {
				w.handleMD5(hash, reply)
			}
		case "file":
			w.handleFile(msg, reply)
		}
		if err := w.q.PushWorkReply(msg.ReplyQueue, reply); err != nil {
			logrus.Warnf("Error pushing message to reply queue %+v - %v\n", msg, err)
		}
	}
}

// Start the dedup process. To stop, just close the queue.
func (w *Worker) Start() {
	// Right now, just use the numebr of CPUs
	for i := 0; i < runtime.NumCPU(); i++ {
		go w.handle()
	}
	for {
		msg, err := w.q.PopWork(0)
		if err != nil || msg == nil {
			logrus.Info("Stoping WorkManager process")
			close(w.c)
			return
		}
		w.c <- msg
	}
}

func contextFromMap(c map[string]interface{}) *domain.Context {
	return &domain.Context{
		Team:         c["team"].(string),
		User:         c["user"].(string),
		OriginalUser: c["original_user"].(string),
		Channel:      c["channel"].(string),
	}
}

// GetContext from a message based on actual type
func GetContext(context interface{}) (*domain.Context, error) {
	switch c := context.(type) {
	case *domain.Context:
		// Hack to duplicate the context so if we are using channels not to override it
		cx := *c
		return &cx, nil
	case map[string]interface{}:
		return contextFromMap(c), nil
	default:
		return nil, errors.New("Unknown context")
	}
}

func (w *Worker) handleURL(text string, reply *domain.WorkReply) {
	start := strings.Index(text, "<http")
	end := strings.Index(text[start:], ">")
	if end > 0 {
		end = end + start
		filter := strings.Index(text[start:end], "|")
		if filter > 0 {
			end = start + filter
		}
		url := text[start+1 : end]
		logrus.Debugf("URL found - %s\n", url)
		reply.URL.Details = url
		reply.Type |= domain.ReplyTypeURL
		// Do the network commands in parallel
		c := make(chan int, 2)
		go func() {
			urlResp, err := w.xfe.URL(url)
			if err != nil {
				// Small hack - see if the URL was not found
				if strings.Contains(err.Error(), "404") {
					reply.URL.XFE.NotFound = true
				} else {
					reply.URL.XFE.Error = err.Error()
				}
			} else {
				reply.URL.XFE.URLDetails = urlResp.Result
			}
			resolve, err := w.xfe.Resolve(url)
			if err == nil {
				reply.URL.XFE.Resolve = *resolve
			}
			c <- 1
		}()
		go func() {
			vtResp, err := w.vt.GetUrlReport(url)
			if err != nil {
				reply.URL.VT.Error = err.Error()
			} else {
				reply.URL.VT.URLReport = *vtResp
			}
			c <- 1
		}()
		for i := 0; i < 2; i++ {
			<-c
		}
		if reply.URL.XFE.URLDetails.Score > xfeScoreToConvict || reply.URL.VT.URLReport.Positives > numOfPositivesToConvict {
			// This is known bad scenario
			reply.URL.Result = domain.ResultDirty
		} else if !reply.MD5.XFE.NotFound || reply.MD5.VT.FileReport.ResponseCode == 1 {
			// At least one of reputation services found this to be known good
			// Keep the default
			reply.URL.Result = domain.ResultClean
		}
	}
}

func (w *Worker) handleIP(ip string, reply *domain.WorkReply) {
	reply.IP.Details = ip
	reply.Type |= domain.ReplyTypeIP
	// First, let's check if IP is globally unicast addressable and is public
	ipData := net.ParseIP(ip)
	ipv4 := ipData.To4()
	if ipv4 == nil {
		// If not IPv4 then return - by default it will be marked clean
		return
	}
	if !ipv4.IsGlobalUnicast() {
		// If not global unicast ignore
		return
	}
	// Private networks
	if ipv4[0] == 10 || ipv4[0] == 172 && ipv4[1] >= 16 && ipv4[1] <= 31 || ipv4[0] == 192 && ipv4[1] == 168 {
		return
	}
	c := make(chan int, 2)
	go func() {
		ipResp, err := w.xfe.IPR(ip)
		if err != nil {
			// Small hack - see if the URL was not found
			if strings.Contains(err.Error(), "404") {
				reply.IP.XFE.NotFound = true
			} else {
				reply.IP.XFE.Error = err.Error()
			}
		} else {
			reply.IP.XFE.IPReputation = *ipResp
		}
		c <- 1
	}()
	go func() {
		vtResp, err := w.vt.GetIpReport(ip)
		if err != nil {
			reply.IP.VT.Error = err.Error()
		} else {
			reply.IP.VT.IPReport = *vtResp
		}
		c <- 1
	}()
	for i := 0; i < 2; i++ {
		<-c
	}
	var vtPositives uint16
	for i := range reply.IP.VT.IPReport.DetectedUrls {
		if reply.IP.VT.IPReport.DetectedUrls[i].Positives > vtPositives {
			vtPositives = reply.IP.VT.IPReport.DetectedUrls[i].Positives
		}
	}
	reply.IP.Result = domain.ResultUnknown
	if reply.IP.XFE.IPReputation.Score > xfeScoreToConvict || vtPositives > numOfPositivesToConvict {
		// This is known bad scenario
		reply.IP.Result = domain.ResultDirty
	} else if !reply.MD5.XFE.NotFound || reply.MD5.VT.FileReport.ResponseCode == 1 {
		// At least one of reputation services found this to be known good
		// Keep the default
		reply.IP.Result = domain.ResultClean
	}
}

func (w *Worker) handleMD5(md5 string, reply *domain.WorkReply) {
	reply.Type |= domain.ReplyTypeMD5
	reply.MD5.Details = md5
	c := make(chan int, 2)
	go func() {
		md5Resp, err := w.xfe.MalwareDetails(md5)
		if err != nil {
			// Small hack - see if the file was not found
			if strings.Contains(err.Error(), "404") {
				reply.MD5.XFE.NotFound = true
			} else {
				reply.MD5.XFE.Error = err.Error()
			}
		} else {
			reply.MD5.XFE.Malware = md5Resp.Malware
		}
		c <- 1
	}()
	go func() {
		vtResp, err := w.vt.GetFileReport(md5)
		if err != nil {
			reply.MD5.VT.Error = err.Error()
		} else {
			reply.MD5.VT.FileReport = *vtResp
		}
		c <- 1
	}()
	for i := 0; i < 2; i++ {
		<-c
	}
	reply.MD5.Result = domain.ResultUnknown
	if len(reply.MD5.XFE.Malware.Family) > 0 || reply.MD5.VT.FileReport.Positives > numOfPositivesToConvict {
		// This is known bad scenario
		reply.MD5.Result = domain.ResultDirty
	} else if !reply.MD5.XFE.NotFound || reply.MD5.VT.FileReport.ResponseCode == 1 {
		// At least one of reputation services found this to be known good
		// Keep the default
		reply.MD5.Result = domain.ResultClean
	}
}

func (w *Worker) handleFile(request *domain.WorkRequest, reply *domain.WorkReply) {
	reply.Type |= domain.ReplyTypeFile
	reply.File.Details = request.File
	if request.File.Size > 30*1024*1024 {
		logrus.Infof("File %s is bigger than 30M, skipping\n", request.File.Name)
		reply.File.FileTooLarge = true
		return
	}
	hash := md5.New()
	resp, err := http.Get(request.File.URL)
	if err != nil {
		logrus.Errorf("Unable to download file - %v\n", err)
		return
	}
	defer resp.Body.Close()
	buf := &bytes.Buffer{}
	io.Copy(buf, resp.Body)
	io.Copy(hash, bytes.NewReader(buf.Bytes()))
	h := fmt.Sprintf("%x", hash.Sum(nil))
	logrus.Debugf("MD5 for file %s is %s\n", request.File.Name, h)
	// Do the network commands in parallel
	c := make(chan int, 1)
	go func() {
		virus, err := w.clam.scan(request.File.Name, buf.Bytes())
		if (err == nil || err.Error() == "Virus(es) detected") && virus != "" {
			reply.File.Virus = virus
		} else if err != nil {
			reply.File.Error = err.Error()
		}
		c <- 1
	}()
	w.handleMD5(h, reply)
	<-c
	reply.File.Result = domain.ResultUnknown
	if reply.File.Virus != "" || len(reply.MD5.XFE.Malware.Family) > 0 || reply.MD5.VT.FileReport.Positives > numOfPositivesToConvict {
		// This is known bad scenario
		reply.File.Result = domain.ResultDirty
	} else if reply.File.Virus == "" && (!reply.MD5.XFE.NotFound || reply.MD5.VT.FileReport.ResponseCode == 1) {
		// At least one of reputation services found this to be known good
		// Keep the default
		reply.File.Result = domain.ResultClean
	}
}
<commit_msg>Fix so that if ClamAV says it's clean then mark clean<commit_after>package bot

import (
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"runtime"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/demisto/alfred/conf"
	"github.com/demisto/alfred/domain"
	"github.com/demisto/alfred/queue"
	"github.com/demisto/alfred/repo"
	"github.com/demisto/goxforce"
	"github.com/slavikm/govt"
)

const (
	numOfPositivesToConvict = 3
	xfeScoreToConvict       = 3
)

// Worker reads messages from the queue and does the actual work
type Worker struct {
	q    queue.Queue
	c    chan *domain.WorkRequest
	r    repo.Repo
	xfe  *goxforce.Client
	vt   *govt.Client
	clam *clamEngine
}

// NewWorker that loads work messages from the queue
func NewWorker(r repo.Repo, q queue.Queue) (*Worker, error) {
	xfe, err := goxforce.New(goxforce.SetErrorLog(log.New(conf.LogWriter, "XFE:", log.Lshortfile)))
	if err != nil {
		return nil, err
	}
	vt, err := govt.New(govt.SetApikey(conf.Options.VT), govt.SetErrorLog(log.New(os.Stderr, "VT:", log.Lshortfile)))
	if err != nil {
		return nil, err
	}
	clam, err := newClamEngine()
	if err != nil {
		return nil, err
	}
	return &Worker{
		r:    r,
		q:    q,
		c:    make(chan *domain.WorkRequest, runtime.NumCPU()),
		xfe:  xfe,
		vt:   vt,
		clam: clam,
	}, nil
}

func (w *Worker) handle() {
	for msg := range w.c {
		if msg == nil {
			w.clam.close()
			return
		}
		if msg.ReplyQueue == "" {
			logrus.Warnf("Got message without a reply queue destination %+v\n", msg)
			continue
		}
		reply := &domain.WorkReply{Context: msg.Context, MessageID: msg.MessageID}
		switch msg.Type {
		case "message":
			if strings.Contains(msg.Text, "<http") {
				w.handleURL(msg.Text, reply)
			}
			if ip := ipReg.FindString(msg.Text); ip != "" {
				w.handleIP(ip, reply)
			}
			if hash := md5Reg.FindString(msg.Text); hash != "" {
				w.handleMD5(hash, reply)
			}
		case "file":
			w.handleFile(msg, reply)
		}
		if err := w.q.PushWorkReply(msg.ReplyQueue, reply); err != nil {
			logrus.Warnf("Error pushing message to reply queue %+v - %v\n", msg, err)
		}
	}
}

// Start the dedup process. To stop, just close the queue.
func (w *Worker) Start() {
	// Right now, just use the numebr of CPUs
	for i := 0; i < runtime.NumCPU(); i++ {
		go w.handle()
	}
	for {
		msg, err := w.q.PopWork(0)
		if err != nil || msg == nil {
			logrus.Info("Stoping WorkManager process")
			close(w.c)
			return
		}
		w.c <- msg
	}
}

func contextFromMap(c map[string]interface{}) *domain.Context {
	return &domain.Context{
		Team:         c["team"].(string),
		User:         c["user"].(string),
		OriginalUser: c["original_user"].(string),
		Channel:      c["channel"].(string),
	}
}

// GetContext from a message based on actual type
func GetContext(context interface{}) (*domain.Context, error) {
	switch c := context.(type) {
	case *domain.Context:
		// Hack to duplicate the context so if we are using channels not to override it
		cx := *c
		return &cx, nil
	case map[string]interface{}:
		return contextFromMap(c), nil
	default:
		return nil, errors.New("Unknown context")
	}
}

func (w *Worker) handleURL(text string, reply *domain.WorkReply) {
	start := strings.Index(text, "<http")
	end := strings.Index(text[start:], ">")
	if end > 0 {
		end = end + start
		filter := strings.Index(text[start:end], "|")
		if filter > 0 {
			end = start + filter
		}
		url := text[start+1 : end]
		logrus.Debugf("URL found - %s\n", url)
		reply.URL.Details = url
		reply.Type |= domain.ReplyTypeURL
		// Do the network commands in parallel
		c := make(chan int, 2)
		go func() {
			urlResp, err := w.xfe.URL(url)
			if err != nil {
				// Small hack - see if the URL was not found
				if strings.Contains(err.Error(), "404") {
					reply.URL.XFE.NotFound = true
				} else {
					reply.URL.XFE.Error = err.Error()
				}
			} else {
				reply.URL.XFE.URLDetails = urlResp.Result
			}
			resolve, err := w.xfe.Resolve(url)
			if err == nil {
				reply.URL.XFE.Resolve = *resolve
			}
			c <- 1
		}()
		go func() {
			vtResp, err := w.vt.GetUrlReport(url)
			if err != nil {
				reply.URL.VT.Error = err.Error()
			} else {
				reply.URL.VT.URLReport = *vtResp
			}
			c <- 1
		}()
		for i := 0; i < 2; i++ {
			<-c
		}
		if reply.URL.XFE.URLDetails.Score > xfeScoreToConvict || reply.URL.VT.URLReport.Positives > numOfPositivesToConvict {
			// This is known bad scenario
			reply.URL.Result = domain.ResultDirty
		} else if !reply.MD5.XFE.NotFound || reply.MD5.VT.FileReport.ResponseCode == 1 {
			// At least one of reputation services found this to be known good
			// Keep the default
			reply.URL.Result = domain.ResultClean
		}
	}
}

func (w *Worker) handleIP(ip string, reply *domain.WorkReply) {
	reply.IP.Details = ip
	reply.Type |= domain.ReplyTypeIP
	// First, let's check if IP is globally unicast addressable and is public
	ipData := net.ParseIP(ip)
	ipv4 := ipData.To4()
	if ipv4 == nil {
		// If not IPv4 then return - by default it will be marked clean
		return
	}
	if !ipv4.IsGlobalUnicast() {
		// If not global unicast ignore
		return
	}
	// Private networks
	if ipv4[0] == 10 || ipv4[0] == 172 && ipv4[1] >= 16 && ipv4[1] <= 31 || ipv4[0] == 192 && ipv4[1] == 168 {
		return
	}
	c := make(chan int, 2)
	go func() {
		ipResp, err := w.xfe.IPR(ip)
		if err != nil {
			// Small hack - see if the URL was not found
			if strings.Contains(err.Error(), "404") {
				reply.IP.XFE.NotFound = true
			} else {
				reply.IP.XFE.Error = err.Error()
			}
		} else {
			reply.IP.XFE.IPReputation = *ipResp
		}
		c <- 1
	}()
	go func() {
		vtResp, err := w.vt.GetIpReport(ip)
		if err != nil {
			reply.IP.VT.Error = err.Error()
		} else {
			reply.IP.VT.IPReport = *vtResp
		}
		c <- 1
	}()
	for i := 0; i < 2; i++ {
		<-c
	}
	var vtPositives uint16
	for i := range reply.IP.VT.IPReport.DetectedUrls {
		if reply.IP.VT.IPReport.DetectedUrls[i].Positives > vtPositives {
			vtPositives = reply.IP.VT.IPReport.DetectedUrls[i].Positives
		}
	}
	reply.IP.Result = domain.ResultUnknown
	if reply.IP.XFE.IPReputation.Score > xfeScoreToConvict || vtPositives > numOfPositivesToConvict {
		// This is known bad scenario
		reply.IP.Result = domain.ResultDirty
	} else if !reply.MD5.XFE.NotFound || reply.MD5.VT.FileReport.ResponseCode == 1 {
		// At least one of reputation services found this to be known good
		// Keep the default
		reply.IP.Result = domain.ResultClean
	}
}

func (w *Worker) handleMD5(md5 string, reply *domain.WorkReply) {
	reply.Type |= domain.ReplyTypeMD5
	reply.MD5.Details = md5
	c := make(chan int, 2)
	go func() {
		md5Resp, err := w.xfe.MalwareDetails(md5)
		if err != nil {
			// Small hack - see if the file was not found
			if strings.Contains(err.Error(), "404") {
				reply.MD5.XFE.NotFound = true
			} else {
				reply.MD5.XFE.Error = err.Error()
			}
		} else {
			reply.MD5.XFE.Malware = md5Resp.Malware
		}
		c <- 1
	}()
	go func() {
		vtResp, err := w.vt.GetFileReport(md5)
		if err != nil {
			reply.MD5.VT.Error = err.Error()
		} else {
			reply.MD5.VT.FileReport = *vtResp
		}
		c <- 1
	}()
	for i := 0; i < 2; i++ {
		<-c
	}
	reply.MD5.Result = domain.ResultUnknown
	if len(reply.MD5.XFE.Malware.Family) > 0 || reply.MD5.VT.FileReport.Positives > numOfPositivesToConvict {
		// This is known bad scenario
		reply.MD5.Result = domain.ResultDirty
	} else if !reply.MD5.XFE.NotFound || reply.MD5.VT.FileReport.ResponseCode == 1 {
		// At least one of reputation services found this to be known good
		// Keep the default
		reply.MD5.Result = domain.ResultClean
	}
}

func (w *Worker) handleFile(request *domain.WorkRequest, reply *domain.WorkReply) {
	reply.Type |= domain.ReplyTypeFile
	reply.File.Details = request.File
	if request.File.Size > 30*1024*1024 {
		logrus.Infof("File %s is bigger than 30M, skipping\n", request.File.Name)
		reply.File.FileTooLarge = true
		return
	}
	hash := md5.New()
	resp, err := http.Get(request.File.URL)
	if err != nil {
		logrus.Errorf("Unable to download file - %v\n", err)
		return
	}
	defer resp.Body.Close()
	buf := &bytes.Buffer{}
	io.Copy(buf, resp.Body)
	io.Copy(hash, bytes.NewReader(buf.Bytes()))
	h := fmt.Sprintf("%x", hash.Sum(nil))
	logrus.Debugf("MD5 for file %s is %s\n", request.File.Name, h)
	// Do the network commands in parallel
	c := make(chan int, 1)
	go func() {
		virus, err := w.clam.scan(request.File.Name, buf.Bytes())
		if (err == nil || err.Error() == "Virus(es) detected") && virus != "" {
			reply.File.Virus = virus
		} else if err != nil {
			reply.File.Error = err.Error()
		}
		c <- 1
	}()
	w.handleMD5(h, reply)
	<-c
	reply.File.Result = domain.ResultUnknown
	if reply.File.Virus != "" || len(reply.MD5.XFE.Malware.Family) > 0 || reply.MD5.VT.FileReport.Positives > numOfPositivesToConvict {
		// This is known bad scenario
		reply.File.Result = domain.ResultDirty
	} else if reply.File.Virus == "" || !reply.MD5.XFE.NotFound || reply.MD5.VT.FileReport.ResponseCode == 1 {
		// At least one of reputation services found this to be known good
		// Keep the default
		reply.File.Result = domain.ResultClean
	}
}
<|endoftext|>


<commit_before>package thus

import (
	"flag"
	"io"
	"log"
	"net/http"
	"os"
	"strconv"
)

// generate handler function to serve form
func generateFormHandler(host string, port int) func(http.ResponseWriter, *http.Request) {
	if 0 == len(host) {
		host = "localhost"
	}
	// Serve the HTML form to upload a file
	return func(w http.ResponseWriter, req *http.Request) {
		if "GET" != req.Method {
			log.Fatal("unsupported request method")
		}
		io.WriteString(w, "\n")
		io.WriteString(w, "<form action=\"http://"+host+":"+strconv.Itoa(port)+"/receive\" method=\"post\" enctype=\"multipart/form-data\">")
		io.WriteString(w, "filename:")
		io.WriteString(w, "<input type=\"file\" name=\"file\" id=\"file\">")
		io.WriteString(w, "<input type=\"submit\" name=\"submit\" value=\"submit\">")
		io.WriteString(w, "</form>\n")
	}
}

// Handle the file upload
func uploadHandler(w http.ResponseWriter, req *http.Request) {
	if "POST" != req.Method {
		log.Fatal("unsupported request method for file upload")
	}

	// get file by ID from POST request
	file, header, err := req.FormFile("file")
	if err != nil {
		log.Fatal("error getting file from form data: ", err)
	}
	defer file.Close()

	// try to create file handler
	out, err := os.Create("/tmp/uploaded_file")
	if err != nil {
		log.Fatal("Could not create file. Maybe wrong privileges?")
	}
	defer out.Close()

	// write content of file to disk
	_, err = io.Copy(out, file)
	if err != nil {
		log.Fatal(err)
	}

	// indicate success
	io.WriteString(w, "sucessfully uploaded ")
	io.WriteString(w, header.Filename)
	io.WriteString(w, "")
}

func main() {
	var port = flag.Int("port", 8080, "port to listen on (default: 8080)")
	var ip = flag.String("ip", "", "IP to bind to")
	flag.Parse()
	log.Println("listening on " + *ip + ":" + strconv.Itoa(*port))
	http.HandleFunc("/", generateFormHandler(*ip, *port))
	http.HandleFunc("/receive", uploadHandler)
	err := http.ListenAndServe(*ip+":"+strconv.Itoa(*port), nil)
	if err != nil {
		log.Fatal("server failed: ", err)
	}
}
<commit_msg>fmt fix<commit_after>package thus

import (
	"flag"
	"io"
	"log"
	"net/http"
	"os"
	"strconv"
)

// generate handler function to serve form
func generateFormHandler(host string, port int) func(http.ResponseWriter, *http.Request) {
	if 0 == len(host) {
		host = "localhost"
	}
	// Serve the HTML form to upload a file
	return func(w http.ResponseWriter, req *http.Request) {
		if "GET" != req.Method {
			log.Fatal("unsupported request method")
		}
		io.WriteString(w, "\n")
		io.WriteString(w, "<form action=\"http://"+host+":"+strconv.Itoa(port)+"/receive\" method=\"post\" enctype=\"multipart/form-data\">")
		io.WriteString(w, "filename:")
		io.WriteString(w, "<input type=\"file\" name=\"file\" id=\"file\">")
		io.WriteString(w, "<input type=\"submit\" name=\"submit\" value=\"submit\">")
		io.WriteString(w, "</form>\n")
	}
}

// Handle the file upload
func uploadHandler(w http.ResponseWriter, req *http.Request) {
	if "POST" != req.Method {
		log.Fatal("unsupported request method for file upload")
	}

	// get file by ID from POST request
	file, header, err := req.FormFile("file")
	if err != nil {
		log.Fatal("error getting file from form data: ", err)
	}
	defer file.Close()

	// try to create file handler
	out, err := os.Create("/tmp/uploaded_file")
	if err != nil {
		log.Fatal("Could not create file. Maybe wrong privileges?")
	}
	defer out.Close()

	// write content of file to disk
	_, err = io.Copy(out, file)
	if err != nil {
		log.Fatal(err)
	}

	// indicate success
	io.WriteString(w, "sucessfully uploaded ")
	io.WriteString(w, header.Filename)
	io.WriteString(w, "")
}

func main() {
	var port = flag.Int("port", 8080, "port to listen on (default: 8080)")
	var ip = flag.String("ip", "", "IP to bind to")
	flag.Parse()
	log.Println("listening on "+*ip+":"+strconv.Itoa(*port))
	http.HandleFunc("/", generateFormHandler(*ip, *port))
	http.HandleFunc("/receive", uploadHandler)
	err := http.ListenAndServe(*ip+":"+strconv.Itoa(*port), nil)
	if err != nil {
		log.Fatal("server failed: ", err)
	}
}
<|endoftext|>


<commit_before>package cgolmnl

/*
#cgo LDFLAGS: -lmnl
#include <libmnl/libmnl.h>
#include <stdio.h>
*/
import "C"

import (
	"unsafe"
	"os"
)

/**
 * mnl_nlmsg_size - calculate the size of Netlink message (without alignment)
 *
 * size_t mnl_nlmsg_size(size_t len)
 */
func NlmsgSize(size Size_t) Size_t {
	return Size_t(C.mnl_nlmsg_size(C.size_t(size)))
}

/**
 * mnl_nlmsg_get_payload_len - get the length of the Netlink payload
 *
 * size_t mnl_nlmsg_get_payload_len(const struct nlmsghdr *nlh)
 */
func NlmsgGetPayloadLen(nlh *Nlmsghdr) Size_t {
	return Size_t(C.mnl_nlmsg_get_payload_len((*C.struct_nlmsghdr)(unsafe.Pointer(nlh))))
}

/**
 * mnl_nlmsg_put_header - reserve and prepare room for Netlink header
 *
 * struct nlmsghdr *mnl_nlmsg_put_header(void *buf)
 */
func NlmsgPutHeader(buf unsafe.Pointer) *Nlmsghdr {
	return (*Nlmsghdr)(unsafe.Pointer(C.mnl_nlmsg_put_header(buf)))
}
func NlmsgPutHeaderBytes(buf []byte) *Nlmsghdr {
	return NlmsgPutHeader(unsafe.Pointer(&buf[0]))
}

/**
 * mnl_nlmsg_put_extra_header - reserve and prepare room for an extra header
 *
 * void *
 * mnl_nlmsg_put_extra_header(struct nlmsghdr *nlh, size_t size)
 */
func NlmsgPutExtraHeader(nlh *Nlmsghdr, size Size_t) unsafe.Pointer {
	return C.mnl_nlmsg_put_extra_header((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)), C.size_t(size))
}

/**
 * mnl_nlmsg_get_payload - get a pointer to the payload of the netlink message
 *
 * void *mnl_nlmsg_get_payload(const struct nlmsghdr *nlh)
 */
func NlmsgGetPayload(nlh *Nlmsghdr) unsafe.Pointer {
	return C.mnl_nlmsg_get_payload((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)))
}
func NlmsgGetPayloadBytes(nlh *Nlmsghdr) []byte {
	return SharedBytes(NlmsgGetPayload(nlh), int(NlmsgGetPayloadLen(nlh)))
}

/**
 * mnl_nlmsg_get_payload_offset - get a pointer to the payload of the message
 *
 * void *
 * mnl_nlmsg_get_payload_offset(const struct nlmsghdr *nlh, size_t offset)
 */
func NlmsgGetPayloadOffset(nlh *Nlmsghdr, offset Size_t) unsafe.Pointer {
	return C.mnl_nlmsg_get_payload_offset((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)), C.size_t(offset))
}
func NlmsgGetPayloadOffsetBytes(nlh *Nlmsghdr, offset Size_t) []byte {
	return SharedBytes(NlmsgGetPayloadOffset(nlh, offset), int(NlmsgGetPayloadLen(nlh) - Size_t(MnlAlign(uint32(offset)))))
}

/**
 * mnl_nlmsg_ok - check a there is room for netlink message
 *
 * bool mnl_nlmsg_ok(const struct nlmsghdr *nlh, int len)
 */
func NlmsgOk(nlh *Nlmsghdr, size int) bool {
	// test fails
	//   unexpected fault address 0x--------
	//   fatal error: fault
	// sometimes without below
	if size < SizeofNlmsghdr { return false }
	return bool(C.mnl_nlmsg_ok((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)), C.int(size)))
}

/**
 * mnl_nlmsg_next - get the next netlink message in a multipart message
 *
 * struct nlmsghdr *
 * mnl_nlmsg_next(const struct nlmsghdr *nlh, int *len)
 */
func NlmsgNext(nlh *Nlmsghdr, size int) (*Nlmsghdr, int) {
	c_size := C.int(size)
	h := (*Nlmsghdr)(unsafe.Pointer(C.mnl_nlmsg_next((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)), &c_size)))
	return h, int(c_size)
}

/**
 * mnl_nlmsg_get_payload_tail - get the ending of the netlink message
 *
 * void *mnl_nlmsg_get_payload_tail(const struct nlmsghdr *nlh)
 */
func NlmsgGetPayloadTail(nlh *Nlmsghdr) unsafe.Pointer {
	return C.mnl_nlmsg_get_payload_tail((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)))
}

/**
 * mnl_nlmsg_seq_ok - perform sequence tracking
 *
 * bool
 * mnl_nlmsg_seq_ok(const struct nlmsghdr *nlh, uint32_t seq)
 */
func NlmsgSeqOk(nlh *Nlmsghdr, seq uint32) bool {
	return bool(C.mnl_nlmsg_seq_ok((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)), C.uint(seq)))
}

/**
 * mnl_nlmsg_portid_ok - perform portID origin check
 *
 * bool
 * mnl_nlmsg_portid_ok(const struct nlmsghdr *nlh, uint32_t portid)
 */
func NlmsgPortidOk(nlh *Nlmsghdr, portid uint32) bool {
	return bool(C.mnl_nlmsg_portid_ok((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)), C.uint(portid)))
}

/**
 * mnl_nlmsg_fprintf - print netlink message to file
 *
 * void
 * mnl_nlmsg_fprintf(FILE *fd, const void *data, size_t datalen,
 *		     size_t extra_header_size)
 */
func NlmsgFprint(fd *os.File, data []byte, extra_header_size Size_t) {
	C.mnl_nlmsg_fprintf(C.fdopen(C.int(fd.Fd()), C.CString("w")),
		unsafe.Pointer(&data[0]), C.size_t(len(data)), C.size_t(extra_header_size))
}

/*
 * Netlink message batch helpers
 */

// struct mnl_nlmsg_batch
type NlmsgBatchDescriptor [0]byte // unsafe.Pointer

/**
 * mnl_nlmsg_batch_start - initialize a batch
 *
 * struct mnl_nlmsg_batch *mnl_nlmsg_batch_start(void *buf, size_t limit)
 */
func NlmsgBatchStart(buf []byte, limit Size_t) (*NlmsgBatchDescriptor, error) {
	ret, err := C.mnl_nlmsg_batch_start(unsafe.Pointer(&buf[0]), C.size_t(limit))
	return  (*NlmsgBatchDescriptor)(ret), err
}

/**
 * mnl_nlmsg_batch_stop - release a batch
 *
 * void mnl_nlmsg_batch_stop(struct mnl_nlmsg_batch *b)
 */
func NlmsgBatchStop(b *NlmsgBatchDescriptor) {
	C.mnl_nlmsg_batch_stop((*[0]byte)(b))
}

/**
 * mnl_nlmsg_batch_next - get room for the next message in the batch
 *
 * bool mnl_nlmsg_batch_next(struct mnl_nlmsg_batch *b)
 */
func NlmsgBatchNext(b *NlmsgBatchDescriptor) bool {
	return bool(C.mnl_nlmsg_batch_next((*[0]byte)(b)))
}

/**
 * mnl_nlmsg_batch_reset - reset the batch
 *
 * void mnl_nlmsg_batch_reset(struct mnl_nlmsg_batch *b)
 */
func NlmsgBatchReset(b *NlmsgBatchDescriptor) {
	C.mnl_nlmsg_batch_reset((*[0]byte)(b))
}

/**
 * mnl_nlmsg_batch_size - get current size of the batch
 *
 * size_t mnl_nlmsg_batch_size(struct mnl_nlmsg_batch *b)
 */
func NlmsgBatchSize(b *NlmsgBatchDescriptor) Size_t {
	return Size_t(C.mnl_nlmsg_batch_size((*[0]byte)(b)))
}

/**
 * mnl_nlmsg_batch_head - get head of this batch
 *
 * void *mnl_nlmsg_batch_head(struct mnl_nlmsg_batch *b)
 */
func NlmsgBatchHead(b *NlmsgBatchDescriptor) unsafe.Pointer {
	return C.mnl_nlmsg_batch_head((*[0]byte)(b))
}
func NlmsgBatchHeadBytes(b *NlmsgBatchDescriptor) []byte {
	return SharedBytes(NlmsgBatchHead(b), int(NlmsgBatchSize(b)))
}

/**
 * mnl_nlmsg_batch_current - returns current position in the batch
 *
 * void *mnl_nlmsg_batch_current(struct mnl_nlmsg_batch *b)
 */
func NlmsgBatchCurrent(b *NlmsgBatchDescriptor) unsafe.Pointer {
	return C.mnl_nlmsg_batch_current((*[0]byte)(b))
}

/**
 * mnl_nlmsg_batch_is_empty - check if there is any message in the batch
 *
 * bool mnl_nlmsg_batch_is_empty(struct mnl_nlmsg_batch *b)
 */
func NlmsgBatchIsEmpty(b *NlmsgBatchDescriptor) bool {
	return bool(C.mnl_nlmsg_batch_is_empty((*[0]byte)(b)))
}

/**
 * mnl_nlmsg_batch_reset_buffer - reset the new buffer
 *
 * void mnl_nlmsg_batch_reset_buffer(struct mnl_nlmsg_batch *b, void *buf, size_t limit)
func NlmsgBatchResetBuffer(b NlmsgBatchDescriptor, buf []byte) {
	C.mnl_nlmsg_batch_reset_buffer((*[0]byte)(b), (unsafe.Pointer)(&buf[0]), C.size_t(len(buf)))
}
 */
<commit_msg>nlmsg: fix mem leak and add NlmsgPutHeaderBytes<commit_after>package cgolmnl

/*
#cgo LDFLAGS: -lmnl
#include <stdlib.h>
#include <stdio.h>
#include <libmnl/libmnl.h>
*/
import "C"

import (
	"unsafe"
	"os"
	"syscall"
)

/**
 * mnl_nlmsg_size - calculate the size of Netlink message (without alignment)
 *
 * size_t mnl_nlmsg_size(size_t len)
 */
func NlmsgSize(size Size_t) Size_t {
	return Size_t(C.mnl_nlmsg_size(C.size_t(size)))
}

/**
 * mnl_nlmsg_get_payload_len - get the length of the Netlink payload
 *
 * size_t mnl_nlmsg_get_payload_len(const struct nlmsghdr *nlh)
 */
func NlmsgGetPayloadLen(nlh *Nlmsghdr) Size_t {
	return Size_t(C.mnl_nlmsg_get_payload_len((*C.struct_nlmsghdr)(unsafe.Pointer(nlh))))
}

/**
 * mnl_nlmsg_put_header - reserve and prepare room for Netlink header
 *
 * struct nlmsghdr *mnl_nlmsg_put_header(void *buf)
 */
func NlmsgPutHeader(buf unsafe.Pointer) *Nlmsghdr {
	return (*Nlmsghdr)(unsafe.Pointer(C.mnl_nlmsg_put_header(buf)))
}
func NlmsgPutHeaderBytes(buf []byte) (*Nlmsghdr, error) {
	if len(buf) < int(MNL_NLMSG_HDRLEN) { return nil, syscall.EINVAL }
	return NlmsgPutHeader(unsafe.Pointer(&buf[0])), nil
}

/**
 * mnl_nlmsg_put_extra_header - reserve and prepare room for an extra header
 *
 * void *
 * mnl_nlmsg_put_extra_header(struct nlmsghdr *nlh, size_t size)
 */
func NlmsgPutExtraHeader(nlh *Nlmsghdr, size Size_t) unsafe.Pointer {
	return C.mnl_nlmsg_put_extra_header((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)), C.size_t(size))
}

/**
 * mnl_nlmsg_get_payload - get a pointer to the payload of the netlink message
 *
 * void *mnl_nlmsg_get_payload(const struct nlmsghdr *nlh)
 */
func NlmsgGetPayload(nlh *Nlmsghdr) unsafe.Pointer {
	return C.mnl_nlmsg_get_payload((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)))
}
func NlmsgGetPayloadBytes(nlh *Nlmsghdr) []byte {
	return SharedBytes(NlmsgGetPayload(nlh), int(NlmsgGetPayloadLen(nlh)))
}

/**
 * mnl_nlmsg_get_payload_offset - get a pointer to the payload of the message
 *
 * void *
 * mnl_nlmsg_get_payload_offset(const struct nlmsghdr *nlh, size_t offset)
 */
func NlmsgGetPayloadOffset(nlh *Nlmsghdr, offset Size_t) unsafe.Pointer {
	return C.mnl_nlmsg_get_payload_offset((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)), C.size_t(offset))
}
func NlmsgGetPayloadOffsetBytes(nlh *Nlmsghdr, offset Size_t) []byte {
	return SharedBytes(NlmsgGetPayloadOffset(nlh, offset), int(NlmsgGetPayloadLen(nlh) - Size_t(MnlAlign(uint32(offset)))))
}

/**
 * mnl_nlmsg_ok - check a there is room for netlink message
 *
 * bool mnl_nlmsg_ok(const struct nlmsghdr *nlh, int len)
 */
func NlmsgOk(nlh *Nlmsghdr, size int) bool {
	// test fails
	//   unexpected fault address 0x--------
	//   fatal error: fault
	// sometimes without below
	if size < SizeofNlmsghdr { return false }
	return bool(C.mnl_nlmsg_ok((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)), C.int(size)))
}

/**
 * mnl_nlmsg_next - get the next netlink message in a multipart message
 *
 * struct nlmsghdr *
 * mnl_nlmsg_next(const struct nlmsghdr *nlh, int *len)
 */
func NlmsgNext(nlh *Nlmsghdr, size int) (*Nlmsghdr, int) {
	c_size := C.int(size)
	h := (*Nlmsghdr)(unsafe.Pointer(C.mnl_nlmsg_next((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)), &c_size)))
	return h, int(c_size)
}

/**
 * mnl_nlmsg_get_payload_tail - get the ending of the netlink message
 *
 * void *mnl_nlmsg_get_payload_tail(const struct nlmsghdr *nlh)
 */
func NlmsgGetPayloadTail(nlh *Nlmsghdr) unsafe.Pointer {
	return C.mnl_nlmsg_get_payload_tail((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)))
}

/**
 * mnl_nlmsg_seq_ok - perform sequence tracking
 *
 * bool
 * mnl_nlmsg_seq_ok(const struct nlmsghdr *nlh, uint32_t seq)
 */
func NlmsgSeqOk(nlh *Nlmsghdr, seq uint32) bool {
	return bool(C.mnl_nlmsg_seq_ok((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)), C.uint(seq)))
}

/**
 * mnl_nlmsg_portid_ok - perform portID origin check
 *
 * bool
 * mnl_nlmsg_portid_ok(const struct nlmsghdr *nlh, uint32_t portid)
 */
func NlmsgPortidOk(nlh *Nlmsghdr, portid uint32) bool {
	return bool(C.mnl_nlmsg_portid_ok((*C.struct_nlmsghdr)(unsafe.Pointer(nlh)), C.uint(portid)))
}

/**
 * mnl_nlmsg_fprintf - print netlink message to file
 *
 * void
 * mnl_nlmsg_fprintf(FILE *fd, const void *data, size_t datalen,
 *		     size_t extra_header_size)
 */
func NlmsgFprint(fd *os.File, data []byte, extra_header_size Size_t) {
	mode := C.CString("w")
	defer C.free(unsafe.Pointer(mode))
	C.mnl_nlmsg_fprintf(C.fdopen(C.int(fd.Fd()), mode),
		unsafe.Pointer(&data[0]), C.size_t(len(data)), C.size_t(extra_header_size))
}

/*
 * Netlink message batch helpers
 */

// struct mnl_nlmsg_batch
type NlmsgBatchDescriptor [0]byte // unsafe.Pointer

/**
 * mnl_nlmsg_batch_start - initialize a batch
 *
 * struct mnl_nlmsg_batch *mnl_nlmsg_batch_start(void *buf, size_t limit)
 */
func NlmsgBatchStart(buf []byte, limit Size_t) (*NlmsgBatchDescriptor, error) {
	ret, err := C.mnl_nlmsg_batch_start(unsafe.Pointer(&buf[0]), C.size_t(limit))
	return  (*NlmsgBatchDescriptor)(ret), err
}

/**
 * mnl_nlmsg_batch_stop - release a batch
 *
 * void mnl_nlmsg_batch_stop(struct mnl_nlmsg_batch *b)
 */
func NlmsgBatchStop(b *NlmsgBatchDescriptor) {
	C.mnl_nlmsg_batch_stop((*[0]byte)(b))
}

/**
 * mnl_nlmsg_batch_next - get room for the next message in the batch
 *
 * bool mnl_nlmsg_batch_next(struct mnl_nlmsg_batch *b)
 */
func NlmsgBatchNext(b *NlmsgBatchDescriptor) bool {
	return bool(C.mnl_nlmsg_batch_next((*[0]byte)(b)))
}

/**
 * mnl_nlmsg_batch_reset - reset the batch
 *
 * void mnl_nlmsg_batch_reset(struct mnl_nlmsg_batch *b)
 */
func NlmsgBatchReset(b *NlmsgBatchDescriptor) {
	C.mnl_nlmsg_batch_reset((*[0]byte)(b))
}

/**
 * mnl_nlmsg_batch_size - get current size of the batch
 *
 * size_t mnl_nlmsg_batch_size(struct mnl_nlmsg_batch *b)
 */
func NlmsgBatchSize(b *NlmsgBatchDescriptor) Size_t {
	return Size_t(C.mnl_nlmsg_batch_size((*[0]byte)(b)))
}

/**
 * mnl_nlmsg_batch_head - get head of this batch
 *
 * void *mnl_nlmsg_batch_head(struct mnl_nlmsg_batch *b)
 */
func NlmsgBatchHead(b *NlmsgBatchDescriptor) unsafe.Pointer {
	return C.mnl_nlmsg_batch_head((*[0]byte)(b))
}
func NlmsgBatchHeadBytes(b *NlmsgBatchDescriptor) []byte {
	return SharedBytes(NlmsgBatchHead(b), int(NlmsgBatchSize(b)))
}

/**
 * mnl_nlmsg_batch_current - returns current position in the batch
 *
 * void *mnl_nlmsg_batch_current(struct mnl_nlmsg_batch *b)
 */
func NlmsgBatchCurrent(b *NlmsgBatchDescriptor) unsafe.Pointer {
	return C.mnl_nlmsg_batch_current((*[0]byte)(b))
}

/**
 * mnl_nlmsg_batch_is_empty - check if there is any message in the batch
 *
 * bool mnl_nlmsg_batch_is_empty(struct mnl_nlmsg_batch *b)
 */
func NlmsgBatchIsEmpty(b *NlmsgBatchDescriptor) bool {
	return bool(C.mnl_nlmsg_batch_is_empty((*[0]byte)(b)))
}

/**
 * mnl_nlmsg_batch_reset_buffer - reset the new buffer
 *
 * void mnl_nlmsg_batch_reset_buffer(struct mnl_nlmsg_batch *b, void *buf, size_t limit)
func NlmsgBatchResetBuffer(b NlmsgBatchDescriptor, buf []byte) {
	C.mnl_nlmsg_batch_reset_buffer((*[0]byte)(b), (unsafe.Pointer)(&buf[0]), C.size_t(len(buf)))
}
 */
<|endoftext|>


<commit_before>package mergo

import (
	"testing"
	"time"
)

type structWithoutTimePointer struct {
	Created time.Time
}

func TestIssue38Merge(t *testing.T) {
	dst := structWithoutTimePointer{
		time.Now(),
	}

	expected := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
	src := structWithoutTimePointer{
		expected,
	}
	if err := Merge(&dst, src); err != nil {
		t.Errorf("Error while merging %s", err)
	}
	if dst.Created != src.Created {
		t.Fatalf("Created not merged in properly: dst.Created(%v) != src.Created(%v)", dst.Created, src.Created)
	}
}

func TestIssue38MergeWithOverwrite(t *testing.T) {
	dst := structWithoutTimePointer{
		time.Now(),
	}

	expected := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
	src := structWithoutTimePointer{
		expected,
	}
	if err := MergeWithOverwrite(&dst, src); err != nil {
		t.Errorf("Error while merging %s", err)
	}
	if dst.Created != src.Created {
		t.Fatalf("Created not merged in properly: dst.Created(%v) != src.Created(%v)", dst.Created, src.Created)
	}
}
<commit_msg>Fixing test<commit_after>package mergo

import (
	"testing"
	"time"
)

type structWithoutTimePointer struct {
	Created time.Time
}

func TestIssue38Merge(t *testing.T) {
	dst := structWithoutTimePointer{
		time.Now(),
	}

	expected := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
	src := structWithoutTimePointer{
		expected,
	}
	if err := Merge(&dst, src); err != nil {
		t.Errorf("Error while merging %s", err)
	}
	if dst.Created == src.Created {
		t.Fatalf("Created merged unexpectedly: dst.Created(%v) == src.Created(%v)", dst.Created, src.Created)
	}
}

func TestIssue38MergeEmptyStruct(t *testing.T) {
	dst := structWithoutTimePointer{
	}

	expected := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
	src := structWithoutTimePointer{
		expected,
	}
	if err := Merge(&dst, src); err != nil {
		t.Errorf("Error while merging %s", err)
	}
	if dst.Created == src.Created {
		t.Fatalf("Created merged unexpectedly: dst.Created(%v) == src.Created(%v)", dst.Created, src.Created)
	}
}

func TestIssue38MergeWithOverwrite(t *testing.T) {
	dst := structWithoutTimePointer{
		time.Now(),
	}

	expected := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
	src := structWithoutTimePointer{
		expected,
	}
	if err := MergeWithOverwrite(&dst, src); err != nil {
		t.Errorf("Error while merging %s", err)
	}
	if dst.Created != src.Created {
		t.Fatalf("Created not merged in properly: dst.Created(%v) != src.Created(%v)", dst.Created, src.Created)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"html/template"
	"log"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/gorilla/mux"
)

const (
	PORT   = ":8080"
	LENGTH = 12
)

var templates = template.Must(template.ParseFiles("static/index.html", "static/login.html", "static/register.html", "static/todo.html", "static/edit.html", "static/add.html"))

func checkErr(err error) {
	if err != nil {
		log.Println(err)
	}
}

func genName() string {
	name := uniuri.NewLen(LENGTH)

	return name
}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	err := templates.ExecuteTemplate(w, "index.html", "")
	checkErr(err)

}

func todoHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	todo := vars["id"]

}

func addHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		err := templates.ExecuteTemplate(w, "add.html", "")
		checkErr(err)

	case "POST":
		title := r.FormValue("title")
		task := r.FormValue("task")
		duedate := r.FormValue("duedate")

	}

}

func editHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	todo := vars["id"]

}

func delHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	todo := vars["id"]

}

func finishHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	todo := vars["id"]

}

func userHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id := vars["id"]

}

func userDelHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id := vars["id"]

}

func loginHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		err := templates.ExecuteTemplate(w, "login.html", "")
		checkErr(err)
	case "POST":
		email := r.FormValue("email")
		pass := r.FormValue("pass")
	}

}

func registerHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		err := templates.ExecuteTemplate(w, "login.html", "")
		checkErr(err)
	case "POST":
		email := r.FormValue("email")
		pass := r.FormValue("pass")
	}

}

func logoutHandler(w http.ResponseWriter, r *http.Request) {
	cookie := &http.Cookie{
		Name:   "session",
		Value:  "",
		Path:   "/",
		MaxAge: -1,
	}
	http.SetCookie(w, cookie)
	http.Redirect(w, r, "/", 301)

}

func resetHandler(w http.ResponseWriter, r *http.Request) {

}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/", rootHandler)

	router.HandleFunc("/todo", todoHandler)
	router.HandleFunc("/todo/{id}", todoHandler)
	router.HandleFunc("/todo/add", addHandler)
	router.HandleFunc("/todo/edit/{id}", editHandler)
	router.HandleFunc("/todo/del/{id}", delHandler)

	router.HandleFunc("/finish/{id}", finishHandler)

	router.HandleFunc("/user", userHandler)
	router.HandleFunc("/user/{id}", userHandler)
	router.HandleFunc("/user/del/{id}", userDelHandler)

	router.HandleFunc("/register", registerHandler)
	router.HandleFunc("/login", loginHandler)
	router.HandleFunc("/logout", logoutHandler)
	router.HandleFunc("/resetpass", resetHandler)
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<commit_msg>Add template to todohandler<commit_after>package main

import (
	"html/template"
	"log"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/gorilla/mux"
)

const (
	PORT   = ":8080"
	LENGTH = 12
)

var templates = template.Must(template.ParseFiles("static/index.html", "static/login.html", "static/register.html", "static/todo.html", "static/edit.html", "static/add.html"))

func checkErr(err error) {
	if err != nil {
		log.Println(err)
	}
}

func genName() string {
	name := uniuri.NewLen(LENGTH)

	return name
}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	err := templates.ExecuteTemplate(w, "index.html", "")
	checkErr(err)

}

func todoHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	todo := vars["id"]
	err := templates.ExecuteTemplate(w, "todo.html", "")
	checkErr(err)

}

func addHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		err := templates.ExecuteTemplate(w, "add.html", "")
		checkErr(err)

	case "POST":
		title := r.FormValue("title")
		task := r.FormValue("task")
		duedate := r.FormValue("duedate")

	}

}

func editHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	todo := vars["id"]

}

func delHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	todo := vars["id"]

}

func finishHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	todo := vars["id"]

}

func userHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id := vars["id"]

}

func userDelHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	id := vars["id"]

}

func loginHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		err := templates.ExecuteTemplate(w, "login.html", "")
		checkErr(err)
	case "POST":
		email := r.FormValue("email")
		pass := r.FormValue("pass")
	}

}

func registerHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		err := templates.ExecuteTemplate(w, "login.html", "")
		checkErr(err)
	case "POST":
		email := r.FormValue("email")
		pass := r.FormValue("pass")
	}

}

func logoutHandler(w http.ResponseWriter, r *http.Request) {
	cookie := &http.Cookie{
		Name:   "session",
		Value:  "",
		Path:   "/",
		MaxAge: -1,
	}
	http.SetCookie(w, cookie)
	http.Redirect(w, r, "/", 301)

}

func resetHandler(w http.ResponseWriter, r *http.Request) {

}

func main() {
	router := mux.NewRouter()
	router.HandleFunc("/", rootHandler)

	router.HandleFunc("/todo", todoHandler)
	router.HandleFunc("/todo/{id}", todoHandler)
	router.HandleFunc("/todo/add", addHandler)
	router.HandleFunc("/todo/edit/{id}", editHandler)
	router.HandleFunc("/todo/del/{id}", delHandler)

	router.HandleFunc("/finish/{id}", finishHandler)

	router.HandleFunc("/user", userHandler)
	router.HandleFunc("/user/{id}", userHandler)
	router.HandleFunc("/user/del/{id}", userDelHandler)

	router.HandleFunc("/register", registerHandler)
	router.HandleFunc("/login", loginHandler)
	router.HandleFunc("/logout", logoutHandler)
	router.HandleFunc("/resetpass", resetHandler)
	err := http.ListenAndServe(PORT, router)
	if err != nil {
		log.Fatal(err)
	}

}
<|endoftext|>