<commit_before>package notifier

import (
	"bytes"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/oinume/lekcije/server/config"
	"github.com/oinume/lekcije/server/emailer"
	"github.com/oinume/lekcije/server/errors"
	"github.com/oinume/lekcije/server/fetcher"
	"github.com/oinume/lekcije/server/logger"
	"github.com/oinume/lekcije/server/model"
	"github.com/oinume/lekcije/server/stopwatch"
	"github.com/oinume/lekcije/server/util"
	"go.uber.org/zap"
)

type Notifier struct {
	db              *gorm.DB
	fetcher         *fetcher.LessonFetcher
	dryRun          bool
	lessonService   *model.LessonService
	teachers        map[uint32]*model.Teacher
	fetchedLessons  map[uint32][]*model.Lesson
	sender          emailer.Sender
	senderWaitGroup *sync.WaitGroup
	stopwatch       stopwatch.Stopwatch
	sync.Mutex
}

type teachersAndLessons struct {
	data         map[uint32]*model.TeacherLessons
	lessonsCount int
	teacherIDs   []uint32
}

func (tal *teachersAndLessons) CountLessons() int {
	count := 0
	for _, l := range tal.data {
		count += len(l.Lessons)
	}
	return count
}

// Filter out by NotificationTimeSpanList.
// If a lesson is within NotificationTimeSpanList, it'll be included in returned value.
func (tal *teachersAndLessons) FilterBy(list model.NotificationTimeSpanList) *teachersAndLessons {
	if len(list) == 0 {
		return tal
	}
	ret := NewTeachersAndLessons(len(tal.data))
	for teacherID, tl := range tal.data {
		lessons := make([]*model.Lesson, 0, len(tl.Lessons))
		for _, lesson := range tl.Lessons {
			dt := lesson.Datetime
			t, _ := time.Parse("15:04", fmt.Sprintf("%02d:%02d", dt.Hour(), dt.Minute()))
			if list.Within(t) {
				lessons = append(lessons, lesson)
			}
		}
		ret.data[teacherID] = model.NewTeacherLessons(tl.Teacher, lessons)
	}
	return ret
}

func (tal *teachersAndLessons) String() string {
	b := new(bytes.Buffer)
	for _, tl := range tal.data {
		fmt.Fprintf(b, "Teacher: %+v", tl.Teacher)
		fmt.Fprint(b, ", Lessons:")
		for _, l := range tl.Lessons {
			fmt.Fprintf(b, " {%+v}", l)
		}
	}
	return b.String()
}

func NewTeachersAndLessons(length int) *teachersAndLessons {
	return &teachersAndLessons{
		data:         make(map[uint32]*model.TeacherLessons, length),
		lessonsCount: -1,
		teacherIDs:   make([]uint32, 0, length),
	}
}

func NewNotifier(db *gorm.DB, fetcher *fetcher.LessonFetcher, dryRun bool, sender emailer.Sender) *Notifier {
	return &Notifier{
		db:              db,
		fetcher:         fetcher,
		dryRun:          dryRun,
		teachers:        make(map[uint32]*model.Teacher, 1000),
		fetchedLessons:  make(map[uint32][]*model.Lesson, 1000),
		sender:          sender,
		senderWaitGroup: &sync.WaitGroup{},
		stopwatch:       stopwatch.NewSync().Start(),
	}
}

func (n *Notifier) SendNotification(user *model.User) error {
	followingTeacherService := model.NewFollowingTeacherService(n.db)
	n.lessonService = model.NewLessonService(n.db)
	const maxFetchErrorCount = 5
	teacherIDs, err := followingTeacherService.FindTeacherIDsByUserID(user.ID, maxFetchErrorCount)
	if err != nil {
		return errors.Wrapperf(err, "Failed to FindTeacherIDsByUserID(): userID=%v", user.ID)
	}
	n.stopwatch.Mark(fmt.Sprintf("FindTeacherIDsByUserID:%d", user.ID))

	if len(teacherIDs) == 0 {
		return nil
	}

	// Comment out due to papertrail limit
	//logger.App.Info("n", zap.Uint("userID", uint(user.ID)), zap.Int("teachers", len(teacherIDs)))

	//availableTeachersAndLessons := make(map[uint32][]*model.Lesson, 1000)
	availableTeachersAndLessons := NewTeachersAndLessons(1000)
	wg := &sync.WaitGroup{}
	for _, teacherID := range teacherIDs {
		wg.Add(1)
		go func(teacherID uint32) {
			defer n.stopwatch.Mark(fmt.Sprintf("fetchAndExtractNewAvailableLessons:%d", teacherID))
			defer wg.Done()
			fetched, newAvailable, err := n.fetchAndExtractNewAvailableLessons(teacherID)
			if err != nil {
				switch err.(type) {
				case *errors.NotFound:
					if err := model.NewTeacherService(n.db).IncrementFetchErrorCount(teacherID, 1); err != nil {
						logger.App.Error(
							"IncrementFetchErrorCount failed",
							zap.Uint("teacherID", uint(teacherID)), zap.Error(err),
						)
					}
					logger.App.Warn("Cannot find teacher", zap.Uint("teacherID", uint(teacherID)))
				// TODO: Handle a case eikaiwa.dmm.com is down
				default:
					logger.App.Error("Cannot fetch teacher", zap.Uint("teacherID", uint(teacherID)), zap.Error(err))
				}
				return
			}

			n.Lock()
			defer n.Unlock()
			n.teachers[teacherID] = fetched.Teacher
			if _, ok := n.fetchedLessons[teacherID]; !ok {
				n.fetchedLessons[teacherID] = make([]*model.Lesson, 0, 5000)
			}
			n.fetchedLessons[teacherID] = append(n.fetchedLessons[teacherID], fetched.Lessons...)
			if len(newAvailable.Lessons) > 0 {
				availableTeachersAndLessons.data[teacherID] = newAvailable
			}
			//fmt.Printf("go routine finished: user=%v\n", user.ID)
		}(teacherID)

		if err != nil {
			return err
		}
	}
	wg.Wait()

	notificationTimeSpanService := model.NewNotificationTimeSpanService(n.db)
	timeSpans, err := notificationTimeSpanService.FindByUserID(user.ID)
	if err != nil {
		return err
	}
	filteredAvailable := availableTeachersAndLessons.FilterBy(model.NotificationTimeSpanList(timeSpans))
	if err := n.sendNotificationToUser(user, filteredAvailable); err != nil {
		return err
	}

	time.Sleep(150 * time.Millisecond)
	n.stopwatch.Mark("sleep")

	return nil
}

// Returns teacher, fetchedLessons, newAvailableLessons, error
func (n *Notifier) fetchAndExtractNewAvailableLessons(teacherID uint32) (
	*model.TeacherLessons, *model.TeacherLessons, error,
) {
	teacher, fetchedLessons, err := n.fetcher.Fetch(teacherID)
	if err != nil {
		return nil, nil, err
	}
	logger.App.Debug(
		"fetcher.Fetch",
		zap.Uint("teacherID", uint(teacher.ID)),
		zap.Int("lessons", len(fetchedLessons)),
	)

	//fmt.Printf("fetchedLessons ---\n")
	//for _, l := range fetchedLessons {
	//	fmt.Printf("teacherID=%v, datetime=%v, status=%v\n", l.TeacherId, l.Datetime, l.Status)
	//}

	now := time.Now().In(config.LocalTimezone())
	fromDate := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, config.LocalTimezone())
	toDate := fromDate.Add(24 * 6 * time.Hour)
	lastFetchedLessons, err := n.lessonService.FindLessons(teacher.ID, fromDate, toDate)
	if err != nil {
		return nil, nil, err
	}
	//fmt.Printf("lastFetchedLessons ---\n")
	//for _, l := range lastFetchedLessons {
	//	fmt.Printf("teacherID=%v, datetime=%v, status=%v\n", l.TeacherId, l.Datetime, l.Status)
	//}

	newAvailableLessons := n.lessonService.GetNewAvailableLessons(lastFetchedLessons, fetchedLessons)
	//fmt.Printf("newAvailableLessons ---\n")
	//for _, l := range newAvailableLessons {
	//	fmt.Printf("teacherID=%v, datetime=%v, status=%v\n", l.TeacherId, l.Datetime, l.Status)
	//}
	return model.NewTeacherLessons(teacher, fetchedLessons),
		model.NewTeacherLessons(teacher, newAvailableLessons),
		nil
	//return teacher, fetchedLessons, newAvailableLessons, nil
}

func (n *Notifier) sendNotificationToUser(
	user *model.User, lessonsPerTeacher *teachersAndLessons,
) error {
	lessonsCount := 0
	var teacherIDs []int
	for teacherID, l := range lessonsPerTeacher.data {
		teacherIDs = append(teacherIDs, int(teacherID))
		lessonsCount += len(l.Lessons)
	}
	if lessonsPerTeacher.CountLessons() == 0 {
		// Don't send notification
		return nil
	}

	sort.Ints(teacherIDs)
	var teacherIDs2 []uint32
	var teacherNames []string
	for _, id := range teacherIDs {
		teacherIDs2 = append(teacherIDs2, uint32(id))
		teacherNames = append(teacherNames, n.teachers[uint32(id)].Name)
	}

	// TODO: getEmailTemplate as a static file
	t := emailer.NewTemplate("notifier", getEmailTemplateJP())
	data := struct {
		To                string
		TeacherNames      string
		TeacherIDs        []uint32
		Teachers          map[uint32]*model.Teacher
		LessonsPerTeacher map[uint32]*model.TeacherLessons
		WebURL            string
	}{
		To:                user.Email,
		TeacherNames:      strings.Join(teacherNames, ", "),
		TeacherIDs:        teacherIDs2,
		Teachers:          n.teachers,
		LessonsPerTeacher: lessonsPerTeacher.data,
		WebURL:            config.WebURL(),
	}
	email, err := emailer.NewEmailFromTemplate(t, data)
	if err != nil {
		return errors.InternalWrapf(err, "Failed to create emailer.Email from template: to=%v", user.Email)
	}
	email.SetCustomArg("email_type", model.EmailTypeNewLessonNotifier)
	email.SetCustomArg("user_id", fmt.Sprint(user.ID))
	email.SetCustomArg("teacher_ids", strings.Join(util.Uint32ToStringSlice(teacherIDs2...), ","))
	//fmt.Printf("--- mail ---\n%s", email.BodyString())
	n.stopwatch.Mark("emailer.NewEmailFromTemplate")

	logger.App.Info("sendNotificationToUser", zap.String("email", user.Email))

	n.senderWaitGroup.Add(1)
	go func(email *emailer.Email) {
		defer n.stopwatch.Mark(fmt.Sprintf("sender.Send:%d", user.ID))
		defer n.senderWaitGroup.Done()
		if err := n.sender.Send(email); err != nil {
			logger.App.Error(
				"Failed to sendNotificationToUser",
				zap.String("email", user.Email), zap.Error(err),
			)
		}
	}(email)

	return nil
}

func getEmailTemplateJP() string {
	return strings.TrimSpace(`
From: lekcije <lekcije@lekcije.com>
To: {{ .To }}
Subject: {{ .TeacherNames }}の空きレッスンがあります
Body: text/html
{{ range $teacherID := .TeacherIDs }}
{{- $teacher := index $.Teachers $teacherID -}}
--- {{ $teacher.Name }} ---
  {{- $tal := index $.LessonsPerTeacher $teacherID }}
  {{- range $lesson := $tal.Lessons }}
{{ $lesson.Datetime.Format "2006-01-02 15:04" }}
  {{- end }}

レッスンの予約はこちらから:
PC
Mobile

{{ end }}
PR ────────────────
全世界で３万人が受講！「 英語速読・記憶講座」が￥2400！
PR ────────────────

空きレッスンの通知の解除はこちら

お問い合わせ
	`)
}

func (n *Notifier) Close() {
	n.senderWaitGroup.Wait()
	defer n.fetcher.Close()
	defer func() {
		if n.dryRun {
			return
		}
		for teacherID, lessons := range n.fetchedLessons {
			if _, err := n.lessonService.UpdateLessons(lessons); err != nil {
				logger.App.Error(
					"An error ocurred in Notifier.Close",
					zap.Error(err), zap.Uint("teacherID", uint(teacherID)),
				)
			}
		}
	}()
	defer func() {
		n.stopwatch.Stop()
		//logger.App.Info("Stopwatch report", zap.String("report", watch.Report()))
		//fmt.Println("--- stopwatch ---")
		//fmt.Println(n.stopwatch.Report())
	}()
}
<commit_msg>Change PR link<commit_after>package notifier

import (
	"bytes"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/oinume/lekcije/server/config"
	"github.com/oinume/lekcije/server/emailer"
	"github.com/oinume/lekcije/server/errors"
	"github.com/oinume/lekcije/server/fetcher"
	"github.com/oinume/lekcije/server/logger"
	"github.com/oinume/lekcije/server/model"
	"github.com/oinume/lekcije/server/stopwatch"
	"github.com/oinume/lekcije/server/util"
	"go.uber.org/zap"
)

type Notifier struct {
	db              *gorm.DB
	fetcher         *fetcher.LessonFetcher
	dryRun          bool
	lessonService   *model.LessonService
	teachers        map[uint32]*model.Teacher
	fetchedLessons  map[uint32][]*model.Lesson
	sender          emailer.Sender
	senderWaitGroup *sync.WaitGroup
	stopwatch       stopwatch.Stopwatch
	sync.Mutex
}

type teachersAndLessons struct {
	data         map[uint32]*model.TeacherLessons
	lessonsCount int
	teacherIDs   []uint32
}

func (tal *teachersAndLessons) CountLessons() int {
	count := 0
	for _, l := range tal.data {
		count += len(l.Lessons)
	}
	return count
}

// Filter out by NotificationTimeSpanList.
// If a lesson is within NotificationTimeSpanList, it'll be included in returned value.
func (tal *teachersAndLessons) FilterBy(list model.NotificationTimeSpanList) *teachersAndLessons {
	if len(list) == 0 {
		return tal
	}
	ret := NewTeachersAndLessons(len(tal.data))
	for teacherID, tl := range tal.data {
		lessons := make([]*model.Lesson, 0, len(tl.Lessons))
		for _, lesson := range tl.Lessons {
			dt := lesson.Datetime
			t, _ := time.Parse("15:04", fmt.Sprintf("%02d:%02d", dt.Hour(), dt.Minute()))
			if list.Within(t) {
				lessons = append(lessons, lesson)
			}
		}
		ret.data[teacherID] = model.NewTeacherLessons(tl.Teacher, lessons)
	}
	return ret
}

func (tal *teachersAndLessons) String() string {
	b := new(bytes.Buffer)
	for _, tl := range tal.data {
		fmt.Fprintf(b, "Teacher: %+v", tl.Teacher)
		fmt.Fprint(b, ", Lessons:")
		for _, l := range tl.Lessons {
			fmt.Fprintf(b, " {%+v}", l)
		}
	}
	return b.String()
}

func NewTeachersAndLessons(length int) *teachersAndLessons {
	return &teachersAndLessons{
		data:         make(map[uint32]*model.TeacherLessons, length),
		lessonsCount: -1,
		teacherIDs:   make([]uint32, 0, length),
	}
}

func NewNotifier(db *gorm.DB, fetcher *fetcher.LessonFetcher, dryRun bool, sender emailer.Sender) *Notifier {
	return &Notifier{
		db:              db,
		fetcher:         fetcher,
		dryRun:          dryRun,
		teachers:        make(map[uint32]*model.Teacher, 1000),
		fetchedLessons:  make(map[uint32][]*model.Lesson, 1000),
		sender:          sender,
		senderWaitGroup: &sync.WaitGroup{},
		stopwatch:       stopwatch.NewSync().Start(),
	}
}

func (n *Notifier) SendNotification(user *model.User) error {
	followingTeacherService := model.NewFollowingTeacherService(n.db)
	n.lessonService = model.NewLessonService(n.db)
	const maxFetchErrorCount = 5
	teacherIDs, err := followingTeacherService.FindTeacherIDsByUserID(user.ID, maxFetchErrorCount)
	if err != nil {
		return errors.Wrapperf(err, "Failed to FindTeacherIDsByUserID(): userID=%v", user.ID)
	}
	n.stopwatch.Mark(fmt.Sprintf("FindTeacherIDsByUserID:%d", user.ID))

	if len(teacherIDs) == 0 {
		return nil
	}

	// Comment out due to papertrail limit
	//logger.App.Info("n", zap.Uint("userID", uint(user.ID)), zap.Int("teachers", len(teacherIDs)))

	//availableTeachersAndLessons := make(map[uint32][]*model.Lesson, 1000)
	availableTeachersAndLessons := NewTeachersAndLessons(1000)
	wg := &sync.WaitGroup{}
	for _, teacherID := range teacherIDs {
		wg.Add(1)
		go func(teacherID uint32) {
			defer n.stopwatch.Mark(fmt.Sprintf("fetchAndExtractNewAvailableLessons:%d", teacherID))
			defer wg.Done()
			fetched, newAvailable, err := n.fetchAndExtractNewAvailableLessons(teacherID)
			if err != nil {
				switch err.(type) {
				case *errors.NotFound:
					if err := model.NewTeacherService(n.db).IncrementFetchErrorCount(teacherID, 1); err != nil {
						logger.App.Error(
							"IncrementFetchErrorCount failed",
							zap.Uint("teacherID", uint(teacherID)), zap.Error(err),
						)
					}
					logger.App.Warn("Cannot find teacher", zap.Uint("teacherID", uint(teacherID)))
				// TODO: Handle a case eikaiwa.dmm.com is down
				default:
					logger.App.Error("Cannot fetch teacher", zap.Uint("teacherID", uint(teacherID)), zap.Error(err))
				}
				return
			}

			n.Lock()
			defer n.Unlock()
			n.teachers[teacherID] = fetched.Teacher
			if _, ok := n.fetchedLessons[teacherID]; !ok {
				n.fetchedLessons[teacherID] = make([]*model.Lesson, 0, 5000)
			}
			n.fetchedLessons[teacherID] = append(n.fetchedLessons[teacherID], fetched.Lessons...)
			if len(newAvailable.Lessons) > 0 {
				availableTeachersAndLessons.data[teacherID] = newAvailable
			}
			//fmt.Printf("go routine finished: user=%v\n", user.ID)
		}(teacherID)

		if err != nil {
			return err
		}
	}
	wg.Wait()

	notificationTimeSpanService := model.NewNotificationTimeSpanService(n.db)
	timeSpans, err := notificationTimeSpanService.FindByUserID(user.ID)
	if err != nil {
		return err
	}
	filteredAvailable := availableTeachersAndLessons.FilterBy(model.NotificationTimeSpanList(timeSpans))
	if err := n.sendNotificationToUser(user, filteredAvailable); err != nil {
		return err
	}

	time.Sleep(150 * time.Millisecond)
	n.stopwatch.Mark("sleep")

	return nil
}

// Returns teacher, fetchedLessons, newAvailableLessons, error
func (n *Notifier) fetchAndExtractNewAvailableLessons(teacherID uint32) (
	*model.TeacherLessons, *model.TeacherLessons, error,
) {
	teacher, fetchedLessons, err := n.fetcher.Fetch(teacherID)
	if err != nil {
		return nil, nil, err
	}
	logger.App.Debug(
		"fetcher.Fetch",
		zap.Uint("teacherID", uint(teacher.ID)),
		zap.Int("lessons", len(fetchedLessons)),
	)

	//fmt.Printf("fetchedLessons ---\n")
	//for _, l := range fetchedLessons {
	//	fmt.Printf("teacherID=%v, datetime=%v, status=%v\n", l.TeacherId, l.Datetime, l.Status)
	//}

	now := time.Now().In(config.LocalTimezone())
	fromDate := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, config.LocalTimezone())
	toDate := fromDate.Add(24 * 6 * time.Hour)
	lastFetchedLessons, err := n.lessonService.FindLessons(teacher.ID, fromDate, toDate)
	if err != nil {
		return nil, nil, err
	}
	//fmt.Printf("lastFetchedLessons ---\n")
	//for _, l := range lastFetchedLessons {
	//	fmt.Printf("teacherID=%v, datetime=%v, status=%v\n", l.TeacherId, l.Datetime, l.Status)
	//}

	newAvailableLessons := n.lessonService.GetNewAvailableLessons(lastFetchedLessons, fetchedLessons)
	//fmt.Printf("newAvailableLessons ---\n")
	//for _, l := range newAvailableLessons {
	//	fmt.Printf("teacherID=%v, datetime=%v, status=%v\n", l.TeacherId, l.Datetime, l.Status)
	//}
	return model.NewTeacherLessons(teacher, fetchedLessons),
		model.NewTeacherLessons(teacher, newAvailableLessons),
		nil
	//return teacher, fetchedLessons, newAvailableLessons, nil
}

func (n *Notifier) sendNotificationToUser(
	user *model.User, lessonsPerTeacher *teachersAndLessons,
) error {
	lessonsCount := 0
	var teacherIDs []int
	for teacherID, l := range lessonsPerTeacher.data {
		teacherIDs = append(teacherIDs, int(teacherID))
		lessonsCount += len(l.Lessons)
	}
	if lessonsPerTeacher.CountLessons() == 0 {
		// Don't send notification
		return nil
	}

	sort.Ints(teacherIDs)
	var teacherIDs2 []uint32
	var teacherNames []string
	for _, id := range teacherIDs {
		teacherIDs2 = append(teacherIDs2, uint32(id))
		teacherNames = append(teacherNames, n.teachers[uint32(id)].Name)
	}

	// TODO: getEmailTemplate as a static file
	t := emailer.NewTemplate("notifier", getEmailTemplateJP())
	data := struct {
		To                string
		TeacherNames      string
		TeacherIDs        []uint32
		Teachers          map[uint32]*model.Teacher
		LessonsPerTeacher map[uint32]*model.TeacherLessons
		WebURL            string
	}{
		To:                user.Email,
		TeacherNames:      strings.Join(teacherNames, ", "),
		TeacherIDs:        teacherIDs2,
		Teachers:          n.teachers,
		LessonsPerTeacher: lessonsPerTeacher.data,
		WebURL:            config.WebURL(),
	}
	email, err := emailer.NewEmailFromTemplate(t, data)
	if err != nil {
		return errors.InternalWrapf(err, "Failed to create emailer.Email from template: to=%v", user.Email)
	}
	email.SetCustomArg("email_type", model.EmailTypeNewLessonNotifier)
	email.SetCustomArg("user_id", fmt.Sprint(user.ID))
	email.SetCustomArg("teacher_ids", strings.Join(util.Uint32ToStringSlice(teacherIDs2...), ","))
	//fmt.Printf("--- mail ---\n%s", email.BodyString())
	n.stopwatch.Mark("emailer.NewEmailFromTemplate")

	logger.App.Info("sendNotificationToUser", zap.String("email", user.Email))

	n.senderWaitGroup.Add(1)
	go func(email *emailer.Email) {
		defer n.stopwatch.Mark(fmt.Sprintf("sender.Send:%d", user.ID))
		defer n.senderWaitGroup.Done()
		if err := n.sender.Send(email); err != nil {
			logger.App.Error(
				"Failed to sendNotificationToUser",
				zap.String("email", user.Email), zap.Error(err),
			)
		}
	}(email)

	return nil
}

func getEmailTemplateJP() string {
	return strings.TrimSpace(`
From: lekcije <lekcije@lekcije.com>
To: {{ .To }}
Subject: {{ .TeacherNames }}の空きレッスンがあります
Body: text/html
{{ range $teacherID := .TeacherIDs }}
{{- $teacher := index $.Teachers $teacherID -}}
--- {{ $teacher.Name }} ---
  {{- $tal := index $.LessonsPerTeacher $teacherID }}
  {{- range $lesson := $tal.Lessons }}
{{ $lesson.Datetime.Format "2006-01-02 15:04" }}
  {{- end }}

レッスンの予約はこちらから:
PC
Mobile

{{ end }}
PR ────────────────
設定画面からレッスン希望時間帯が設定できるようになりました
PR ────────────────

空きレッスンの通知の解除はこちら

お問い合わせ
	`)
}

func (n *Notifier) Close() {
	n.senderWaitGroup.Wait()
	defer n.fetcher.Close()
	defer func() {
		if n.dryRun {
			return
		}
		for teacherID, lessons := range n.fetchedLessons {
			if _, err := n.lessonService.UpdateLessons(lessons); err != nil {
				logger.App.Error(
					"An error ocurred in Notifier.Close",
					zap.Error(err), zap.Uint("teacherID", uint(teacherID)),
				)
			}
		}
	}()
	defer func() {
		n.stopwatch.Stop()
		//logger.App.Info("Stopwatch report", zap.String("report", watch.Report()))
		//fmt.Println("--- stopwatch ---")
		//fmt.Println(n.stopwatch.Report())
	}()
}
<|endoftext|>


<commit_before>package bwhatsapp

import (
	"fmt"
	"mime"
	"strings"
	"time"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/Rhymen/go-whatsapp"
	"github.com/jpillora/backoff"
)

/*
Implement handling messages coming from WhatsApp
Check:
- https://github.com/Rhymen/go-whatsapp#add-message-handlers
- https://github.com/Rhymen/go-whatsapp/blob/master/handler.go
- https://github.com/tulir/mautrix-whatsapp/tree/master/whatsapp-ext for more advanced command handling
*/

// HandleError received from WhatsApp
func (b *Bwhatsapp) HandleError(err error) {
	// ignore received invalid data errors. https://github.com/42wim/matterbridge/issues/843
	// ignore tag 174 errors. https://github.com/42wim/matterbridge/issues/1094
	if strings.Contains(err.Error(), "error processing data: received invalid data") ||
		strings.Contains(err.Error(), "invalid string with tag 174") {
		return
	}

	switch err.(type) {
	case *whatsapp.ErrConnectionClosed, *whatsapp.ErrConnectionFailed:
		b.reconnect(err)
	default:
		switch err {
		case whatsapp.ErrConnectionTimeout:
			b.reconnect(err)
		default:
			b.Log.Errorf("%v", err)
		}
	}
}

func (b *Bwhatsapp) reconnect(err error) {
	bf := &backoff.Backoff{
		Min:    time.Second,
		Max:    5 * time.Minute,
		Jitter: true,
	}

	for {
		d := bf.Duration()

		b.Log.Errorf("Connection failed, underlying error: %v", err)
		b.Log.Infof("Waiting %s...", d)

		time.Sleep(d)

		b.Log.Info("Reconnecting...")

		err := b.conn.Restore()
		if err == nil {
			bf.Reset()
			b.startedAt = uint64(time.Now().Unix())

			return
		}
	}
}

// HandleTextMessage sent from WhatsApp, relay it to the brige
func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
	if message.Info.FromMe {
		return
	}
	// whatsapp sends last messages to show context , cut them
	if message.Info.Timestamp < b.startedAt {
		return
	}

	messageTime := time.Unix(int64(message.Info.Timestamp), 0) // TODO check how behaves between timezones
	groupJID := message.Info.RemoteJid

	senderJID := message.Info.SenderJid
	if len(senderJID) == 0 {
		// TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
		if message.Info.Source != nil && message.Info.Source.Participant != nil {
			senderJID = *message.Info.Source.Participant
		}
	}

	// translate sender's JID to the nicest username we can get
	senderName := b.getSenderName(senderJID)
	if senderName == "" {
		senderName = "Someone" // don't expose telephone number
	}

	extText := message.Info.Source.Message.ExtendedTextMessage
	if extText != nil && extText.ContextInfo != nil && extText.ContextInfo.MentionedJid != nil {
		// handle user mentions
		for _, mentionedJID := range extText.ContextInfo.MentionedJid {
			numberAndSuffix := strings.SplitN(mentionedJID, "@", 2)

			// mentions comes as telephone numbers and we don't want to expose it to other bridges
			// replace it with something more meaninful to others
			mention := b.getSenderNotify(numberAndSuffix[0] + "@s.whatsapp.net")
			if mention == "" {
				mention = "someone"
			}

			message.Text = strings.Replace(message.Text, "@"+numberAndSuffix[0], "@"+mention, 1)
		}
	}

	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)

	rmsg := config.Message{
		UserID:    senderJID,
		Username:  senderName,
		Text:      message.Text,
		Timestamp: messageTime,
		Channel:   groupJID,
		Account:   b.Account,
		Protocol:  b.Protocol,
		Extra:     make(map[string][]interface{}),
		//	ParentID: TODO, // TODO handle thread replies  // map from Info.QuotedMessageID string
		ID: message.Info.Id,
	}

	if avatarURL, exists := b.userAvatars[senderJID]; exists {
		rmsg.Avatar = avatarURL
	}

	b.Log.Debugf("<= Message is %#v", rmsg)

	b.Remote <- rmsg
}

// HandleImageMessage sent from WhatsApp, relay it to the brige
func (b *Bwhatsapp) HandleImageMessage(message whatsapp.ImageMessage) {
	if message.Info.FromMe { // || !strings.Contains(strings.ToLower(message.Text), "@echo") {
		return
	}

	// whatsapp sends last messages to show context , cut them
	if message.Info.Timestamp < b.startedAt {
		return
	}

	messageTime := time.Unix(int64(message.Info.Timestamp), 0) // TODO check how behaves between timezones
	groupJID := message.Info.RemoteJid

	senderJID := message.Info.SenderJid
	if len(senderJID) == 0 {
		// TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
		if message.Info.Source != nil && message.Info.Source.Participant != nil {
			senderJID = *message.Info.Source.Participant
		}
	}

	// translate sender's Jid to the nicest username we can get
	senderName := b.getSenderName(senderJID)
	if senderName == "" {
		senderName = "Someone" // don't expose telephone number
	}

	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)

	rmsg := config.Message{
		UserID:    senderJID,
		Username:  senderName,
		Timestamp: messageTime,
		Channel:   groupJID,
		Account:   b.Account,
		Protocol:  b.Protocol,
		Extra:     make(map[string][]interface{}),
		//  ParentID: TODO,      // TODO handle thread replies  // map from Info.QuotedMessageID string
		ID: message.Info.Id,
	}

	if avatarURL, exists := b.userAvatars[senderJID]; exists {
		rmsg.Avatar = avatarURL
	}

	// Download and unencrypt content
	data, err := message.Download()
	if err != nil {
		b.Log.Errorf("%v", err)
		return
	}

	// Get file extension by mimetype
	fileExt, err := mime.ExtensionsByType(message.Type)
	if err != nil {
		b.Log.Errorf("%v", err)
		return
	}

	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])

	b.Log.Debugf("<= Image downloaded and unencrypted")

	// Move file to bridge storage
	helper.HandleDownloadData(b.Log, &rmsg, filename, message.Caption, "", &data, b.General)

	b.Log.Debugf("<= Image Message is %#v", rmsg)

	b.Remote <- rmsg
}

// HandleVideoMessage downloads video messages
func (b *Bwhatsapp) HandleVideoMessage(message whatsapp.VideoMessage) {
	if message.Info.FromMe || message.Info.Timestamp < b.startedAt {
		return
	}

	senderJID := message.Info.SenderJid
	if len(message.Info.SenderJid) == 0 && message.Info.Source != nil && message.Info.Source.Participant != nil {
		senderJID = *message.Info.Source.Participant
	}

	senderName := b.getSenderName(message.Info.SenderJid)
	if senderName == "" {
		senderName = "Someone" // don't expose telephone number
	}

	rmsg := config.Message{
		UserID:   senderJID,
		Username: senderName,
		Channel:  message.Info.RemoteJid,
		Account:  b.Account,
		Protocol: b.Protocol,
		Extra:    make(map[string][]interface{}),
		ID:       message.Info.Id,
	}

	if avatarURL, exists := b.userAvatars[senderJID]; exists {
		rmsg.Avatar = avatarURL
	}

	fileExt, err := mime.ExtensionsByType(message.Type)
	if err != nil {
		b.Log.Errorf("Mimetype detection error: %s", err)

		return
	}

	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])

	b.Log.Debugf("Trying to download %s with size %#v and type %s", filename, message.Length, message.Type)

	data, err := message.Download()
	if err != nil {
		b.Log.Errorf("Download video failed: %s", err)

		return
	}

	// Move file to bridge storage
	helper.HandleDownloadData(b.Log, &rmsg, filename, message.Caption, "", &data, b.General)

	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
	b.Log.Debugf("<= Message is %#v", rmsg)

	b.Remote <- rmsg
}

//
//func (b *Bwhatsapp) HandleJsonMessage(message string) {
//	fmt.Println(message) // TODO implement
//}
// TODO HandleRawMessage
// TODO HandleAudioMessage
<commit_msg>Handle audio downloads (whatsapp)<commit_after>package bwhatsapp

import (
	"fmt"
	"mime"
	"strings"
	"time"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/Rhymen/go-whatsapp"
	"github.com/jpillora/backoff"
)

/*
Implement handling messages coming from WhatsApp
Check:
- https://github.com/Rhymen/go-whatsapp#add-message-handlers
- https://github.com/Rhymen/go-whatsapp/blob/master/handler.go
- https://github.com/tulir/mautrix-whatsapp/tree/master/whatsapp-ext for more advanced command handling
*/

// HandleError received from WhatsApp
func (b *Bwhatsapp) HandleError(err error) {
	// ignore received invalid data errors. https://github.com/42wim/matterbridge/issues/843
	// ignore tag 174 errors. https://github.com/42wim/matterbridge/issues/1094
	if strings.Contains(err.Error(), "error processing data: received invalid data") ||
		strings.Contains(err.Error(), "invalid string with tag 174") {
		return
	}

	switch err.(type) {
	case *whatsapp.ErrConnectionClosed, *whatsapp.ErrConnectionFailed:
		b.reconnect(err)
	default:
		switch err {
		case whatsapp.ErrConnectionTimeout:
			b.reconnect(err)
		default:
			b.Log.Errorf("%v", err)
		}
	}
}

func (b *Bwhatsapp) reconnect(err error) {
	bf := &backoff.Backoff{
		Min:    time.Second,
		Max:    5 * time.Minute,
		Jitter: true,
	}

	for {
		d := bf.Duration()

		b.Log.Errorf("Connection failed, underlying error: %v", err)
		b.Log.Infof("Waiting %s...", d)

		time.Sleep(d)

		b.Log.Info("Reconnecting...")

		err := b.conn.Restore()
		if err == nil {
			bf.Reset()
			b.startedAt = uint64(time.Now().Unix())

			return
		}
	}
}

// HandleTextMessage sent from WhatsApp, relay it to the brige
func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
	if message.Info.FromMe {
		return
	}
	// whatsapp sends last messages to show context , cut them
	if message.Info.Timestamp < b.startedAt {
		return
	}

	messageTime := time.Unix(int64(message.Info.Timestamp), 0) // TODO check how behaves between timezones
	groupJID := message.Info.RemoteJid

	senderJID := message.Info.SenderJid
	if len(senderJID) == 0 {
		// TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
		if message.Info.Source != nil && message.Info.Source.Participant != nil {
			senderJID = *message.Info.Source.Participant
		}
	}

	// translate sender's JID to the nicest username we can get
	senderName := b.getSenderName(senderJID)
	if senderName == "" {
		senderName = "Someone" // don't expose telephone number
	}

	extText := message.Info.Source.Message.ExtendedTextMessage
	if extText != nil && extText.ContextInfo != nil && extText.ContextInfo.MentionedJid != nil {
		// handle user mentions
		for _, mentionedJID := range extText.ContextInfo.MentionedJid {
			numberAndSuffix := strings.SplitN(mentionedJID, "@", 2)

			// mentions comes as telephone numbers and we don't want to expose it to other bridges
			// replace it with something more meaninful to others
			mention := b.getSenderNotify(numberAndSuffix[0] + "@s.whatsapp.net")
			if mention == "" {
				mention = "someone"
			}

			message.Text = strings.Replace(message.Text, "@"+numberAndSuffix[0], "@"+mention, 1)
		}
	}

	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)

	rmsg := config.Message{
		UserID:    senderJID,
		Username:  senderName,
		Text:      message.Text,
		Timestamp: messageTime,
		Channel:   groupJID,
		Account:   b.Account,
		Protocol:  b.Protocol,
		Extra:     make(map[string][]interface{}),
		//	ParentID: TODO, // TODO handle thread replies  // map from Info.QuotedMessageID string
		ID: message.Info.Id,
	}

	if avatarURL, exists := b.userAvatars[senderJID]; exists {
		rmsg.Avatar = avatarURL
	}

	b.Log.Debugf("<= Message is %#v", rmsg)

	b.Remote <- rmsg
}

// HandleImageMessage sent from WhatsApp, relay it to the brige
func (b *Bwhatsapp) HandleImageMessage(message whatsapp.ImageMessage) {
	if message.Info.FromMe { // || !strings.Contains(strings.ToLower(message.Text), "@echo") {
		return
	}

	// whatsapp sends last messages to show context , cut them
	if message.Info.Timestamp < b.startedAt {
		return
	}

	messageTime := time.Unix(int64(message.Info.Timestamp), 0) // TODO check how behaves between timezones
	groupJID := message.Info.RemoteJid

	senderJID := message.Info.SenderJid
	if len(senderJID) == 0 {
		// TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
		if message.Info.Source != nil && message.Info.Source.Participant != nil {
			senderJID = *message.Info.Source.Participant
		}
	}

	// translate sender's Jid to the nicest username we can get
	senderName := b.getSenderName(senderJID)
	if senderName == "" {
		senderName = "Someone" // don't expose telephone number
	}

	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)

	rmsg := config.Message{
		UserID:    senderJID,
		Username:  senderName,
		Timestamp: messageTime,
		Channel:   groupJID,
		Account:   b.Account,
		Protocol:  b.Protocol,
		Extra:     make(map[string][]interface{}),
		//  ParentID: TODO,      // TODO handle thread replies  // map from Info.QuotedMessageID string
		ID: message.Info.Id,
	}

	if avatarURL, exists := b.userAvatars[senderJID]; exists {
		rmsg.Avatar = avatarURL
	}

	// Download and unencrypt content
	data, err := message.Download()
	if err != nil {
		b.Log.Errorf("%v", err)
		return
	}

	// Get file extension by mimetype
	fileExt, err := mime.ExtensionsByType(message.Type)
	if err != nil {
		b.Log.Errorf("%v", err)
		return
	}

	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])

	b.Log.Debugf("<= Image downloaded and unencrypted")

	// Move file to bridge storage
	helper.HandleDownloadData(b.Log, &rmsg, filename, message.Caption, "", &data, b.General)

	b.Log.Debugf("<= Image Message is %#v", rmsg)

	b.Remote <- rmsg
}

// HandleVideoMessage downloads video messages
func (b *Bwhatsapp) HandleVideoMessage(message whatsapp.VideoMessage) {
	if message.Info.FromMe || message.Info.Timestamp < b.startedAt {
		return
	}

	senderJID := message.Info.SenderJid
	if len(message.Info.SenderJid) == 0 && message.Info.Source != nil && message.Info.Source.Participant != nil {
		senderJID = *message.Info.Source.Participant
	}

	senderName := b.getSenderName(message.Info.SenderJid)
	if senderName == "" {
		senderName = "Someone" // don't expose telephone number
	}

	rmsg := config.Message{
		UserID:   senderJID,
		Username: senderName,
		Channel:  message.Info.RemoteJid,
		Account:  b.Account,
		Protocol: b.Protocol,
		Extra:    make(map[string][]interface{}),
		ID:       message.Info.Id,
	}

	if avatarURL, exists := b.userAvatars[senderJID]; exists {
		rmsg.Avatar = avatarURL
	}

	fileExt, err := mime.ExtensionsByType(message.Type)
	if err != nil {
		b.Log.Errorf("Mimetype detection error: %s", err)

		return
	}

	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])

	b.Log.Debugf("Trying to download %s with size %#v and type %s", filename, message.Length, message.Type)

	data, err := message.Download()
	if err != nil {
		b.Log.Errorf("Download video failed: %s", err)

		return
	}

	// Move file to bridge storage
	helper.HandleDownloadData(b.Log, &rmsg, filename, message.Caption, "", &data, b.General)

	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
	b.Log.Debugf("<= Message is %#v", rmsg)

	b.Remote <- rmsg
}

// HandleAudioMessage downloads audio messages
func (b *Bwhatsapp) HandleAudioMessage(message whatsapp.AudioMessage) {
	if message.Info.FromMe || message.Info.Timestamp < b.startedAt {
		return
	}

	senderJID := message.Info.SenderJid
	if len(message.Info.SenderJid) == 0 && message.Info.Source != nil && message.Info.Source.Participant != nil {
		senderJID = *message.Info.Source.Participant
	}

	senderName := b.getSenderName(message.Info.SenderJid)
	if senderName == "" {
		senderName = "Someone" // don't expose telephone number
	}

	rmsg := config.Message{
		UserID:   senderJID,
		Username: senderName,
		Channel:  message.Info.RemoteJid,
		Account:  b.Account,
		Protocol: b.Protocol,
		Extra:    make(map[string][]interface{}),
		ID:       message.Info.Id,
	}

	if avatarURL, exists := b.userAvatars[senderJID]; exists {
		rmsg.Avatar = avatarURL
	}

	fileExt, err := mime.ExtensionsByType(message.Type)
	if err != nil {
		b.Log.Errorf("Mimetype detection error: %s", err)

		return
	}

	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])

	b.Log.Debugf("Trying to download %s with size %#v and type %s", filename, message.Length, message.Type)

	data, err := message.Download()
	if err != nil {
		b.Log.Errorf("Download audio failed: %s", err)

		return
	}

	// Move file to bridge storage
	helper.HandleDownloadData(b.Log, &rmsg, filename, "audio message", "", &data, b.General)

	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
	b.Log.Debugf("<= Message is %#v", rmsg)

	b.Remote <- rmsg
}

//
//func (b *Bwhatsapp) HandleJsonMessage(message string) {
//	fmt.Println(message) // TODO implement
//}
// TODO HandleRawMessage
// TODO HandleAudioMessage
<|endoftext|>


<commit_before>package fake

import (
	"net/http"

	"github.com/johannesboyne/gofakes3"
	"github.com/johannesboyne/gofakes3/backend/s3mem"
	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/bucket"
	"github.com/sirupsen/logrus"
)

var _ bucket.BucketDriver = &Fake{}

type Fake struct {
	backend    *s3mem.Backend
	httpServer *http.Server
}

func New() *Fake {
	backend := s3mem.New()
	return &Fake{
		backend: backend,
		httpServer: &http.Server{
			Addr:    ":8085",
			Handler: gofakes3.New(backend).Server(),
		},
	}
}

// String name representation of driver
func (f *Fake) String() string {
	return "fake"
}

// Start starts a new fake object storage server
func (f *Fake) Start() error {
	logrus.Infof("Starting fake object storage driver on %s", f.httpServer.Addr)
	return f.httpServer.ListenAndServe()
}

// Stop closes the http server for the fake driver
func (f *Fake) Stop() error {
	return f.httpServer.Close()
}

// CreateBucket provisions a new in-memory bucket
func (f *Fake) CreateBucket(name string, region string, anonymousBucketAccessMode api.AnonymousBucketAccessMode) (string, error) {
	logrus.Info("bucket_driver.Fake create bucket received")
	return name, f.backend.CreateBucket(name)
}

// DeleteBucket deprovisions an in-memory bucket
func (f *Fake) DeleteBucket(name string, region string, clearBucket bool) error {
	logrus.Info("bucket_driver.Fake delete bucket received")
	return f.backend.DeleteBucket(name)
}

// AccessBucket grants access to the in-memory bucket
// Dummy impplementation
// Actual implementation to be done once we have more clarity on the downstream API
func (f *Fake) GrantBucketAccess(id string, accountName string, accessPolicy string) (string, *bucket.BucketAccessCredentials, error) {
	logrus.Info("bucket_driver.Fake access bucket received")
	return accountName,
		&bucket.BucketAccessCredentials{
			AccessKeyId:     "YOUR-ACCESSKEYID",
			SecretAccessKey: "YOUR-SECRETACCESSKEY",
		}, nil
}

// DeleteBucket deprovisions an in-memory bucket
// Dummy impplementation
// Actual implementation to be done once we have more clarity on the downstream API
func (f *Fake) RevokeBucketAccess(id string, accountId string) error {
	logrus.Info("bucket_driver.Fake revoke bucket received")
	return nil
}
<commit_msg>Make fake bucket driver idempotent (#2080)<commit_after>package fake

import (
	"net/http"

	"github.com/johannesboyne/gofakes3"
	"github.com/johannesboyne/gofakes3/backend/s3mem"
	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/bucket"
	"github.com/sirupsen/logrus"
)

var _ bucket.BucketDriver = &Fake{}

type Fake struct {
	backend    *s3mem.Backend
	httpServer *http.Server
}

func New() *Fake {
	backend := s3mem.New()
	return &Fake{
		backend: backend,
		httpServer: &http.Server{
			Addr:    ":8085",
			Handler: gofakes3.New(backend).Server(),
		},
	}
}

// String name representation of driver
func (f *Fake) String() string {
	return "fake"
}

// Start starts a new fake object storage server
func (f *Fake) Start() error {
	logrus.Infof("Starting fake object storage driver on %s", f.httpServer.Addr)
	return f.httpServer.ListenAndServe()
}

// Stop closes the http server for the fake driver
func (f *Fake) Stop() error {
	return f.httpServer.Close()
}

// CreateBucket provisions a new in-memory bucket
func (f *Fake) CreateBucket(name string, region string, anonymousBucketAccessMode api.AnonymousBucketAccessMode) (string, error) {
	logrus.Info("bucket_driver.Fake create bucket received")
	err := f.backend.CreateBucket(name)
	if err == gofakes3.ErrBucketAlreadyExists {
		return name, nil
	}

	return name, err
}

// DeleteBucket deprovisions an in-memory bucket
func (f *Fake) DeleteBucket(name string, region string, clearBucket bool) error {
	logrus.Info("bucket_driver.Fake delete bucket received")
	err := f.backend.DeleteBucket(name)
	if err == gofakes3.ErrNoSuchBucket {
		return nil
	}

	return err
}

// AccessBucket grants access to the in-memory bucket
// Dummy impplementation
// Actual implementation to be done once we have more clarity on the downstream API
func (f *Fake) GrantBucketAccess(id string, accountName string, accessPolicy string) (string, *bucket.BucketAccessCredentials, error) {
	logrus.Info("bucket_driver.Fake access bucket received")
	return accountName,
		&bucket.BucketAccessCredentials{
			AccessKeyId:     "YOUR-ACCESSKEYID",
			SecretAccessKey: "YOUR-SECRETACCESSKEY",
		}, nil
}

// DeleteBucket deprovisions an in-memory bucket
// Dummy impplementation
// Actual implementation to be done once we have more clarity on the downstream API
func (f *Fake) RevokeBucketAccess(id string, accountId string) error {
	logrus.Info("bucket_driver.Fake revoke bucket received")
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/freeformz/shh/mm"
	"github.com/freeformz/shh/pollers"
	"time"
)

func writeOut(measurements chan *mm.Measurement) {
	for measurement := range measurements {
		fmt.Println(measurement)
	}
}

func main() {
	measurements := make(chan *mm.Measurement, 100)
	duration, _ := time.ParseDuration("5s")
	ticks := time.Tick(duration)
	go writeOut(measurements)

	mp := pollers.NewMultiPoller()
	mp.RegisterPoller(pollers.Load{})
	mp.RegisterPoller(pollers.Cpu{})

	for now := range ticks {
		measurements <- &mm.Measurement{now, "tick", []byte("true")}
		go mp.Poll(now, measurements)
	}
}
<commit_msg>support SHH_INTERVAL and default to 10s<commit_after>package main

import (
	"fmt"
	"github.com/freeformz/shh/mm"
	"github.com/freeformz/shh/pollers"
	"log"
	"os"
	"time"
)

const (
	DefaultInterval = "10s"
)

func writeOut(measurements chan *mm.Measurement) {
	for measurement := range measurements {
		fmt.Println(measurement)
	}
}

func getDuration() time.Duration {
	interval := os.Getenv("SSH_INTERVAL")

	if interval == "" {
		interval = DefaultInterval
	}

	duration, err := time.ParseDuration(interval)

	if err != nil {
		log.Fatal("unable to parse SHH_INTERVAL: " + interval)
	}

	return duration
}

func main() {
	measurements := make(chan *mm.Measurement, 100)

	duration := getDuration()
	fmt.Sprintf("duration=%s", duration)
	ticks := time.Tick(duration)

	go writeOut(measurements)

	mp := pollers.NewMultiPoller()
	mp.RegisterPoller(pollers.Load{})
	mp.RegisterPoller(pollers.Cpu{})

	for now := range ticks {
		measurements <- &mm.Measurement{now, "tick", []byte("true")}
		go mp.Poll(now, measurements)
	}
}
<|endoftext|>


<commit_before>package stream

import (
	"testing"
	"time"

	"github.com/Jeffail/benthos/v3/lib/input"
	"github.com/Jeffail/benthos/v3/lib/log"
	"github.com/Jeffail/benthos/v3/lib/metrics"
	"github.com/Jeffail/benthos/v3/lib/output"
	"github.com/Jeffail/benthos/v3/lib/processor"
	"github.com/Jeffail/benthos/v3/lib/types"
)

func TestTypeConstruction(t *testing.T) {
	conf := NewConfig()
	conf.Input.Type = input.TypeNanomsg
	conf.Input.Nanomsg.PollTimeout = "100ms"
	conf.Output.Type = output.TypeNanomsg

	strm, err := New(conf) // nanomsg => nanomsg
	if err != nil {
		t.Fatal(err)
	}

	if strm.logger == nil {
		t.Error("nil logger")
	}

	if strm.stats == nil {
		t.Error("nil stats")
	}

	if strm.manager == nil {
		t.Error("nil manager")
	}

	if err = strm.Stop(time.Second * 10); err != nil {
		t.Error(err)
	}

	newStats := metrics.DudType{
		ID: 1,
	}
	newLogger := log.Noop()
	newMgr := types.DudMgr{
		ID: 2,
	}

	strm, err = New(conf, OptSetLogger(newLogger), OptSetStats(newStats), OptSetManager(newMgr))
	if err != nil {
		t.Fatal(err)
	}

	if strm.logger != newLogger {
		t.Error("wrong logger")
	}

	if strm.stats != newStats {
		t.Error("wrong stats")
	}

	if strm.manager != newMgr {
		t.Error("wrong manager")
	}
}

func TestTypeCloseGracefully(t *testing.T) {
	conf := NewConfig()
	conf.Input.Type = input.TypeHTTPServer
	conf.Output.Type = output.TypeHTTPServer

	strm, err := New(conf)
	if err != nil {
		t.Fatal(err)
	}

	if err = strm.stopGracefully(time.Second * 5); err != nil {
		t.Error(err)
	}

	conf.Buffer.Type = "memory"

	strm, err = New(conf)
	if err != nil {
		t.Fatal(err)
	}

	if err = strm.stopGracefully(time.Second * 5); err != nil {
		t.Error(err)
	}

	conf.Pipeline.Processors = []processor.Config{
		processor.NewConfig(),
	}

	strm, err = New(conf)
	if err != nil {
		t.Fatal(err)
	}

	if err = strm.stopGracefully(time.Second * 5); err != nil {
		t.Error(err)
	}
}

func TestTypeCloseOrdered(t *testing.T) {
	conf := NewConfig()
	conf.Input.Type = input.TypeNanomsg
	conf.Output.Type = output.TypeNanomsg

	strm, err := New(conf)
	if err != nil {
		t.Fatal(err)
	}

	if err = strm.stopOrdered(time.Second * 5); err != nil {
		t.Error(err)
	}

	conf.Buffer.Type = "memory"

	strm, err = New(conf)
	if err != nil {
		t.Fatal(err)
	}

	if err = strm.stopOrdered(time.Second * 5); err != nil {
		t.Error(err)
	}

	conf.Pipeline.Processors = []processor.Config{
		processor.NewConfig(),
	}

	strm, err = New(conf)
	if err != nil {
		t.Fatal(err)
	}

	if err = strm.stopOrdered(time.Second * 5); err != nil {
		t.Error(err)
	}
}

func TestTypeCloseUnordered(t *testing.T) {
	conf := NewConfig()
	conf.Input.Type = input.TypeNanomsg
	conf.Output.Type = input.TypeNanomsg

	strm, err := New(conf)
	if err != nil {
		t.Fatal(err)
	}

	if err = strm.stopUnordered(time.Second * 5); err != nil {
		t.Error(err)
	}

	conf.Buffer.Type = "memory"

	strm, err = New(conf)
	if err != nil {
		t.Fatal(err)
	}

	if err = strm.stopUnordered(time.Second * 5); err != nil {
		t.Error(err)
	}

	conf.Pipeline.Processors = []processor.Config{
		processor.NewConfig(),
	}

	strm, err = New(conf)
	if err != nil {
		t.Fatal(err)
	}

	if err = strm.stopUnordered(time.Second * 5); err != nil {
		t.Error(err)
	}
}
<commit_msg>Refactor some tests<commit_after>package stream

import (
	"testing"
	"time"

	"github.com/Jeffail/benthos/v3/lib/input"
	"github.com/Jeffail/benthos/v3/lib/log"
	"github.com/Jeffail/benthos/v3/lib/metrics"
	"github.com/Jeffail/benthos/v3/lib/output"
	"github.com/Jeffail/benthos/v3/lib/processor"
	"github.com/Jeffail/benthos/v3/lib/types"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestTypeConstruction(t *testing.T) {
	conf := NewConfig()
	conf.Input.Type = input.TypeNanomsg
	conf.Input.Nanomsg.PollTimeout = "100ms"
	conf.Output.Type = output.TypeNanomsg

	strm, err := New(conf) // nanomsg => nanomsg
	require.NoError(t, err)

	assert.NotNil(t, strm.logger)
	assert.NotNil(t, strm.stats)
	assert.NotNil(t, strm.manager)

	assert.NoError(t, strm.Stop(time.Minute))

	newStats := metrics.DudType{
		ID: 1,
	}
	newLogger := log.Noop()
	newMgr := types.DudMgr{
		ID: 2,
	}

	strm, err = New(conf, OptSetLogger(newLogger), OptSetStats(newStats), OptSetManager(newMgr))
	require.NoError(t, err)

	assert.Equal(t, newLogger, strm.logger)
	assert.Equal(t, newStats, strm.stats)
	assert.Equal(t, newMgr, strm.manager)
}

func TestTypeCloseGracefully(t *testing.T) {
	conf := NewConfig()
	conf.Input.Type = input.TypeHTTPServer
	conf.Output.Type = output.TypeHTTPServer

	strm, err := New(conf)
	require.NoError(t, err)
	assert.NoError(t, strm.stopGracefully(time.Minute))

	conf.Buffer.Type = "memory"

	strm, err = New(conf)
	require.NoError(t, err)
	assert.NoError(t, strm.stopGracefully(time.Minute))

	conf.Pipeline.Processors = []processor.Config{
		processor.NewConfig(),
	}

	strm, err = New(conf)
	require.NoError(t, err)
	assert.NoError(t, strm.stopGracefully(time.Minute))
}

func TestTypeCloseOrdered(t *testing.T) {
	conf := NewConfig()
	conf.Input.Type = input.TypeNanomsg
	conf.Output.Type = output.TypeNanomsg

	strm, err := New(conf)
	require.NoError(t, err)
	assert.NoError(t, strm.stopOrdered(time.Minute))

	conf.Buffer.Type = "memory"

	strm, err = New(conf)
	require.NoError(t, err)
	assert.NoError(t, strm.stopOrdered(time.Minute))

	conf.Pipeline.Processors = []processor.Config{
		processor.NewConfig(),
	}

	strm, err = New(conf)
	require.NoError(t, err)
	assert.NoError(t, strm.stopOrdered(time.Minute))
}

func TestTypeCloseUnordered(t *testing.T) {
	conf := NewConfig()
	conf.Input.Type = input.TypeNanomsg
	conf.Output.Type = input.TypeNanomsg

	strm, err := New(conf)
	require.NoError(t, err)
	assert.NoError(t, strm.stopUnordered(time.Minute))

	conf.Buffer.Type = "memory"

	strm, err = New(conf)
	require.NoError(t, err)
	assert.NoError(t, strm.stopUnordered(time.Minute))

	conf.Pipeline.Processors = []processor.Config{
		processor.NewConfig(),
	}

	strm, err = New(conf)
	require.NoError(t, err)
	assert.NoError(t, strm.stopUnordered(time.Minute))
}
<|endoftext|>


<commit_before>package apollostats

import (
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/GeertJohan/go.rice"
	"github.com/gin-gonic/gin"
)

type Instance struct {
	Debug bool
	DB    *DB

	addr   string
	router *gin.Engine
}

func (i *Instance) Init() {
	if i.Debug == false {
		gin.SetMode(gin.ReleaseMode)
	}

	// TODO: replace Default with New and use custom logger and stuff
	i.router = gin.Default()

	// Load templates
	funcmap := template.FuncMap{
		"pretty_time": func(t time.Time) string {
			return t.Format("2006-01-02 15:04 MST")
		},
		"year": func() int {
			return time.Now().Year()
		},
	}

	//tmpl := template.Must(template.New("ServerTemplates").Funcs(funcmap).ParseGlob("templates/*"))
	templatebox := rice.MustFindBox("templates")
	templates := template.New("ServerTemplates").Funcs(funcmap)
	templatebox.Walk("", func(p string, i os.FileInfo, e error) error {
		if i.IsDir() {
			return nil
		}
		s, e := templatebox.String(p)
		if e != nil {
			log.Fatalf("Failed to load template: %s\n%s\n", p, e)
		}
		template.Must(templates.New(p).Parse(s))
		return nil
	})
	i.router.SetHTMLTemplate(templates)

	// Setup all URLS
	//i.router.Static("/static", "./static")
	static := rice.MustFindBox("static")
	i.router.StaticFS("/static/", static.HTTPBox())

	i.router.GET("/", i.index)
	i.router.GET("/bans", i.bans)
	i.router.GET("/account_items", i.account_items)
	i.router.GET("/rounds", i.rounds)
	i.router.GET("/round/:round_id", i.round_detail)
}

func (i *Instance) Serve(addr string) error {
	i.addr = addr
	return i.router.Run(i.addr)
}

func (i *Instance) index(c *gin.Context) {
	c.HTML(http.StatusOK, "index.html", gin.H{
		"pagetitle": "Index",
	})
}

func (i *Instance) bans(c *gin.Context) {
	c.HTML(http.StatusOK, "bans.html", gin.H{
		"pagetitle": "Bans",
		"Bans":      i.DB.AllBans(),
	})
}

func (i *Instance) account_items(c *gin.Context) {
	c.HTML(http.StatusOK, "account_items.html", gin.H{
		"pagetitle":    "Account Items",
		"AccountItems": i.DB.AllAccountItems(),
	})
}

func (i *Instance) rounds(c *gin.Context) {
	c.HTML(http.StatusOK, "rounds.html", gin.H{
		"pagetitle": "Rounds",
		"Rounds":    i.DB.AllRounds(),
	})
}

func (i *Instance) round_detail(c *gin.Context) {
	id, e := strconv.ParseInt(c.Param("round_id"), 10, 0)
	if e != nil {
		// TODO: error page
		//c.AbortWithError(http.StatusNotFound, fmt.Errorf("Round not found"))
		c.String(http.StatusNotFound, "Round not found")
		return
	}

	round, e := i.DB.GetRound(id)
	if e != nil {
		//c.AbortWithError(http.StatusNotFound, e)
		c.String(http.StatusNotFound, "Round not found")
		return
	}

	antags := i.DB.GetAntags(id)
	ai_laws := i.DB.GetAILaws(id)
	deaths := i.DB.GetDeaths(id)

	c.HTML(http.StatusOK, "round_detail.html", gin.H{
		"pagetitle": fmt.Sprintf("Round #%d", round.ID),
		"Round":     round,
		"Antags":    antags,
		"AILaws":    ai_laws,
		"Deaths":    deaths,
	})
}
<commit_msg>Updates the comments for web.go.<commit_after>package apollostats

import (
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/GeertJohan/go.rice"
	"github.com/gin-gonic/gin"
)

type Instance struct {
	Debug bool
	DB    *DB

	addr   string
	router *gin.Engine
}

func (i *Instance) Init() {
	if i.Debug == false {
		gin.SetMode(gin.ReleaseMode)
	}

	// TODO: replace Default with New and use custom logger and stuff
	i.router = gin.Default()

	// Custom functions for the templates
	funcmap := template.FuncMap{
		"pretty_time": func(t time.Time) string {
			return t.Format("2006-01-02 15:04 MST")
		},
		"year": func() int {
			return time.Now().Year()
		},
	}

	// Load templates
	templatebox := rice.MustFindBox("templates")
	templates := template.New("ServerTemplates").Funcs(funcmap)
	// Iterate over all templates and mash them together
	templatebox.Walk("", func(p string, i os.FileInfo, e error) error {
		if i.IsDir() {
			return nil
		}
		s, e := templatebox.String(p)
		if e != nil {
			log.Fatalf("Failed to load template: %s\n%s\n", p, e)
		}
		template.Must(templates.New(p).Parse(s))
		return nil
	})
	i.router.SetHTMLTemplate(templates)

	// And static files
	static := rice.MustFindBox("static")
	i.router.StaticFS("/static/", static.HTTPBox())

	// Setup all views
	i.router.GET("/", i.index)
	i.router.GET("/bans", i.bans)
	i.router.GET("/account_items", i.account_items)
	i.router.GET("/rounds", i.rounds)
	i.router.GET("/round/:round_id", i.round_detail)
}

func (i *Instance) Serve(addr string) error {
	i.addr = addr
	return i.router.Run(i.addr)
}

func (i *Instance) index(c *gin.Context) {
	c.HTML(http.StatusOK, "index.html", gin.H{
		"pagetitle": "Index",
	})
}

func (i *Instance) bans(c *gin.Context) {
	c.HTML(http.StatusOK, "bans.html", gin.H{
		"pagetitle": "Bans",
		"Bans":      i.DB.AllBans(),
	})
}

func (i *Instance) account_items(c *gin.Context) {
	c.HTML(http.StatusOK, "account_items.html", gin.H{
		"pagetitle":    "Account Items",
		"AccountItems": i.DB.AllAccountItems(),
	})
}

func (i *Instance) rounds(c *gin.Context) {
	c.HTML(http.StatusOK, "rounds.html", gin.H{
		"pagetitle": "Rounds",
		"Rounds":    i.DB.AllRounds(),
	})
}

func (i *Instance) round_detail(c *gin.Context) {
	id, e := strconv.ParseInt(c.Param("round_id"), 10, 0)
	if e != nil {
		// TODO: error page
		//c.AbortWithError(http.StatusNotFound, fmt.Errorf("Round not found"))
		c.String(http.StatusNotFound, "Round not found")
		return
	}

	round, e := i.DB.GetRound(id)
	if e != nil {
		//c.AbortWithError(http.StatusNotFound, e)
		c.String(http.StatusNotFound, "Round not found")
		return
	}

	antags := i.DB.GetAntags(id)
	ai_laws := i.DB.GetAILaws(id)
	deaths := i.DB.GetDeaths(id)

	c.HTML(http.StatusOK, "round_detail.html", gin.H{
		"pagetitle": fmt.Sprintf("Round #%d", round.ID),
		"Round":     round,
		"Antags":    antags,
		"AILaws":    ai_laws,
		"Deaths":    deaths,
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"syscall"
	"time"

	"github.com/jmoiron/sqlx"
	"github.com/lib/pq"
)

func sqlInterface(
	rootAccount *Account,
	transactions []*Transaction,
) {

	dbDir := filepath.Join(os.TempDir(), fmt.Sprintf("keep-%d", rand.Int63()))
	if out, err := exec.Command("initdb", "-D", dbDir).CombinedOutput(); err != nil {
		fail("%v: %s", err, out)
	}
	pt("db dir: %s\n", dbDir)
	defer exec.Command("rm", "-rf", dbDir).Run()

	port := 10000 + rand.Intn(50000)
	conf := `
		port = ` + fmt.Sprintf("%d", port) + `
	`
	if err := ioutil.WriteFile(filepath.Join(dbDir, "postgresql.conf"), []byte(conf), 0644); err != nil {
		fail("%v", err)
	}
	c := exec.Command("postgres", "-D", dbDir)
	c.SysProcAttr = &syscall.SysProcAttr{
		Setpgid: true,
	}
	if err := c.Start(); err != nil {
		fail("%v", err)
	}
	defer syscall.Kill(-c.Process.Pid, syscall.SIGKILL)
	time.Sleep(time.Second)
	pt("db started\n")

	db, err := sqlx.Open("postgres", fmt.Sprintf("postgres://localhost:%d/postgres?sslmode=disable", port))
	if err != nil {
		fail("%v", err)
	}
	defer db.Close()
	tx := db.MustBegin()
	if _, err := tx.Exec(`
		CREATE TABLE entries (
			id bigserial primary key,
			transaction bigint,
			transaction_description text,
			time timestamp with time zone,
			account text[],
			currency text,
			amount numeric,
			description text
		)
		`,
	); err != nil {
		fail("%v", err)
	}
	for _, view := range views {
		if _, err := tx.Exec(view); err != nil {
			fail("%v", err)
		}
	}
	for tid, transaction := range transactions {
		for _, entry := range transaction.Entries {
			if _, err := tx.Exec(`
				INSERT INTO entries
				(
					transaction, transaction_description,
					time, account, currency, amount, description
				)
				VALUES (
					$1, $2,
					$3, $4, $5, $6::numeric, $7
				)
				`,
				tid,
				transaction.Description,
				entry.Time,
				func() (ret pq.StringArray) {
					acc := entry.Account
					for acc != rootAccount {
						ret = append(ret, acc.Name)
						acc = acc.Parent
					}
					for i := len(ret)/2 - 1; i >= 0; i-- {
						j := len(ret) - 1 - i
						ret[i], ret[j] = ret[j], ret[i]
					}
					return
				}(),
				entry.Currency,
				entry.Amount.FloatString(3),
				entry.Description,
			); err != nil {
				fail("%v", err)
			}
		}
	}
	if err := tx.Commit(); err != nil {
		fail("%v", err)
	}
	pt("data loaded\n")

	sigs := make(chan os.Signal)
	go func() {
		for {
			<-sigs
		}
	}()
	signal.Notify(sigs, os.Interrupt)

	psql := exec.Command("psql", fmt.Sprintf("postgres://localhost:%d/postgres", port))
	psql.Stdout = os.Stdout
	psql.Stdin = os.Stdin
	psql.Stderr = os.Stderr
	if err := psql.Run(); err != nil {
		fail("%v", err)
	}
}

var views = []string{
	// props
	`
	create view props as
	select distinct on (time, transaction)
	time, transaction_description
	from entries
	where account[1] = '支出'
	and account[2] in (
		'数码',
		'物品',
		'衣物服饰',
		'消耗品',
		'保健品',
		'书籍',
		'药物',
		'性用品'
	)
	order by time desc, transaction
	`,

	// monthly
	`
	create view monthly as
	select

	to_char(date_trunc('month', time), 'YYYY-MM') as month,

	COALESCE(
		'支出' || currency || (sum(amount) filter (where account[1] = '支出'))::numeric(20,2)::text
		|| E'：\n'
		|| (
			select string_agg(
				currency || amount::numeric(20,2)::text || ' ' || account,
				E'\n'
				order by amount desc
			) from (
				select account[2] as account, currency, sum(amount) as amount
				from unnest(
					array_agg(id) filter (where account[1] = '支出')
				) as id
				join entries e2 using (id)
				group by account[2], currency
			) t0
		),
		'-'
	) || E'\n' as expenses,

	COALESCE(
		'收入' || currency || (sum(amount) filter (where account[1] = '收入'))::numeric(20,2)::text
		|| E'：\n'
		|| (
			select string_agg(
				currency || amount::numeric(20, 2)::text || ' ' || account,
				E'\n'
				order by amount asc
			) from (
				select account[2] as account, currency, sum(amount) as amount
				from unnest(
					array_agg(id) filter (where account[1] = '收入')
				) as id
				join entries e2 using (id)
				group by account[2], currency
			) t0
		),
		'-'
	) || E'\n' as income,

	COALESCE(
		'净资产' || currency || (
			-sum(amount) filter (where account[1] = '收入')
			-
			sum(amount) filter (where account[1] = '支出')
		)::numeric(20,2)::text,
		'-'
	) || E'\n' as net_income,

	COALESCE(
		E'资产：\n' || (
			select string_agg(
				currency
				|| amount::numeric(20, 2)::text || ' ' || account
				|| E'\n'
				|| '= 增' || pos_amount::numeric(20, 2)::text
				|| ' 减' || neg_amount::numeric(20, 2)::text,
				E'\n'
				order by amount desc
			) from (
				select account[2] as account, currency, sum(amount) as amount,
				COALESCE(sum(amount) filter (where amount >= 0), 0) as pos_amount,
				COALESCE(-sum(amount) filter (where amount < 0), 0) as neg_amount
				from unnest(
					array_agg(id) filter (where account[1] = '资产')
				) as id
				join entries e2 using (id)
				group by account[2], currency
			) t0
		),
		'-'
	) || E'\n' as equity,

	COALESCE(
		E'负债：\n' || (
			select string_agg(
				currency
				|| amount::numeric(20, 2)::text || ' ' || account
				|| E'\n'
				|| '= 还' || pos_amount::numeric(20, 2)::text
				|| ' 借' || neg_amount::numeric(20, 2)::text,
				E'\n'
				order by amount asc
			) from (
				select account[2] as account, currency, sum(amount) as amount,
				COALESCE(sum(amount) filter (where amount >= 0), 0) as pos_amount,
				COALESCE(-sum(amount) filter (where amount < 0), 0) as neg_amount
				from unnest(
					array_agg(id) filter (where account[1] = '负债')
				) as id
				join entries e2 using (id)
				group by account[2], currency
			) t0
		),
		'-'
	) || E'\n' as liability

	from
	entries

	group by date_trunc('month', time), currency
	order by month desc, currency asc
	`,

	// this year expenses
	`
	create view this_year_expenses as
	select
	currency, sum(amount), account[2],
	jsonb_pretty(jsonb_agg(
			to_char(time, 'YYYY-MM-DD')
			|| ' '
			|| currency
			|| amount::numeric(10,2)
			|| ' '
			|| transaction_description
			order by amount desc, time desc
	))
	from entries
	where extract(year from time) = extract(year from now())
	and account[1] = '支出'
	group by account[2],currency
	order by sum desc
	`,

	//
}
<commit_msg>update sql<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"syscall"
	"time"

	"github.com/jmoiron/sqlx"
	"github.com/lib/pq"
)

func sqlInterface(
	rootAccount *Account,
	transactions []*Transaction,
) {

	dbDir := filepath.Join(os.TempDir(), fmt.Sprintf("keep-%d", rand.Int63()))
	if out, err := exec.Command("initdb", "-D", dbDir).CombinedOutput(); err != nil {
		fail("%v: %s", err, out)
	}
	pt("db dir: %s\n", dbDir)
	defer exec.Command("rm", "-rf", dbDir).Run()

	port := 10000 + rand.Intn(50000)
	conf := `
		port = ` + fmt.Sprintf("%d", port) + `
	`
	if err := ioutil.WriteFile(filepath.Join(dbDir, "postgresql.conf"), []byte(conf), 0644); err != nil {
		fail("%v", err)
	}
	c := exec.Command("postgres", "-D", dbDir)
	c.SysProcAttr = &syscall.SysProcAttr{
		Setpgid: true,
	}
	if err := c.Start(); err != nil {
		fail("%v", err)
	}
	defer syscall.Kill(-c.Process.Pid, syscall.SIGKILL)
	time.Sleep(time.Second)
	pt("db started\n")

	db, err := sqlx.Open("postgres", fmt.Sprintf("postgres://localhost:%d/postgres?sslmode=disable", port))
	if err != nil {
		fail("%v", err)
	}
	defer db.Close()
	tx := db.MustBegin()
	if _, err := tx.Exec(`
		CREATE TABLE entries (
			id bigserial primary key,
			transaction bigint,
			transaction_description text,
			date timestamp with time zone,
			account text[],
			currency text,
			amount numeric,
			description text
		)
		`,
	); err != nil {
		fail("%v", err)
	}
	for _, view := range views {
		if _, err := tx.Exec(view); err != nil {
			fail("%v", err)
		}
	}
	for tid, transaction := range transactions {
		for _, entry := range transaction.Entries {
			if _, err := tx.Exec(`
				INSERT INTO entries
				(
					transaction, transaction_description,
					date, account, currency, amount, description
				)
				VALUES (
					$1, $2,
					$3, $4, $5, $6::numeric, $7
				)
				`,
				tid,
				transaction.Description,
				entry.Time,
				func() (ret pq.StringArray) {
					acc := entry.Account
					for acc != rootAccount {
						ret = append(ret, acc.Name)
						acc = acc.Parent
					}
					for i := len(ret)/2 - 1; i >= 0; i-- {
						j := len(ret) - 1 - i
						ret[i], ret[j] = ret[j], ret[i]
					}
					return
				}(),
				entry.Currency,
				entry.Amount.FloatString(3),
				entry.Description,
			); err != nil {
				fail("%v", err)
			}
		}
	}
	if err := tx.Commit(); err != nil {
		fail("%v", err)
	}
	pt("data loaded\n")

	sigs := make(chan os.Signal)
	go func() {
		for {
			<-sigs
		}
	}()
	signal.Notify(sigs, os.Interrupt)

	psql := exec.Command("psql", fmt.Sprintf("postgres://localhost:%d/postgres", port))
	psql.Stdout = os.Stdout
	psql.Stdin = os.Stdin
	psql.Stderr = os.Stderr
	if err := psql.Run(); err != nil {
		fail("%v", err)
	}
}

var views = []string{
	// props
	`
	create view props as
	select distinct on (date, transaction)
	date, transaction_description
	from entries
	where account[1] = '支出'
	and account[2] in (
		'数码',
		'物品',
		'衣物服饰',
		'消耗品',
		'保健品',
		'书籍',
		'药物',
		'性用品'
	)
	order by date desc, transaction
	`,

	// consumables
	`
	create view consumables as
	select distinct on (date, transaction)
	date, transaction_description
	from entries
	where account[1] = '支出'
	and account[2] in (
		'饮食',
		'消耗品',
		'药物',
		'保健品'
	)
	order by date desc, transaction
	`,

	// monthly
	`
	create view monthly as
	select

	to_char(date_trunc('month', date), 'YYYY-MM') as month,

	COALESCE(
		'支出' || currency || (sum(amount) filter (where account[1] = '支出'))::numeric(20,2)::text
		|| E'：\n'
		|| (
			select string_agg(
				currency || amount::numeric(20,2)::text || ' ' || account,
				E'\n'
				order by amount desc
			) from (
				select account[2] as account, currency, sum(amount) as amount
				from unnest(
					array_agg(id) filter (where account[1] = '支出')
				) as id
				join entries e2 using (id)
				group by account[2], currency
			) t0
		),
		'-'
	) || E'\n' as expenses,

	COALESCE(
		'收入' || currency || (sum(amount) filter (where account[1] = '收入'))::numeric(20,2)::text
		|| E'：\n'
		|| (
			select string_agg(
				currency || amount::numeric(20, 2)::text || ' ' || account,
				E'\n'
				order by amount asc
			) from (
				select account[2] as account, currency, sum(amount) as amount
				from unnest(
					array_agg(id) filter (where account[1] = '收入')
				) as id
				join entries e2 using (id)
				group by account[2], currency
			) t0
		),
		'-'
	) || E'\n' as income,

	COALESCE(
		'净资产' || currency || (
			-sum(amount) filter (where account[1] = '收入')
			-
			sum(amount) filter (where account[1] = '支出')
		)::numeric(20,2)::text,
		'-'
	) || E'\n' as net_income,

	COALESCE(
		E'资产：\n' || (
			select string_agg(
				currency
				|| amount::numeric(20, 2)::text || ' ' || account
				|| E'\n'
				|| '= 增' || pos_amount::numeric(20, 2)::text
				|| ' 减' || neg_amount::numeric(20, 2)::text,
				E'\n'
				order by amount desc
			) from (
				select account[2] as account, currency, sum(amount) as amount,
				COALESCE(sum(amount) filter (where amount >= 0), 0) as pos_amount,
				COALESCE(-sum(amount) filter (where amount < 0), 0) as neg_amount
				from unnest(
					array_agg(id) filter (where account[1] = '资产')
				) as id
				join entries e2 using (id)
				group by account[2], currency
			) t0
		),
		'-'
	) || E'\n' as equity,

	COALESCE(
		E'负债：\n' || (
			select string_agg(
				currency
				|| amount::numeric(20, 2)::text || ' ' || account
				|| E'\n'
				|| '= 还' || pos_amount::numeric(20, 2)::text
				|| ' 借' || neg_amount::numeric(20, 2)::text,
				E'\n'
				order by amount asc
			) from (
				select account[2] as account, currency, sum(amount) as amount,
				COALESCE(sum(amount) filter (where amount >= 0), 0) as pos_amount,
				COALESCE(-sum(amount) filter (where amount < 0), 0) as neg_amount
				from unnest(
					array_agg(id) filter (where account[1] = '负债')
				) as id
				join entries e2 using (id)
				group by account[2], currency
			) t0
		),
		'-'
	) || E'\n' as liability

	from
	entries

	group by date_trunc('month', date), currency
	order by month desc, currency asc
	`,

	// this year expenses
	`
	create view this_year_expenses as
	select
	currency, sum(amount), account[2],
	jsonb_pretty(jsonb_agg(
			to_char(date, 'YYYY-MM-DD')
			|| ' '
			|| currency
			|| amount::numeric(10,2)
			|| ' '
			|| transaction_description
			order by amount desc, date desc
	))
	from entries
	where extract(year from date) = extract(year from now())
	and account[1] = '支出'
	group by account[2],currency
	order by sum desc
	`,

	//
}
<|endoftext|>


<commit_before>package parser

import (
	"testing"

	"github.com/flowdev/gflowparser/data"
)

func TestParsePort(t *testing.T) {
	p, err := NewParsePort()
	if err != nil {
		t.Fatalf("Unexpected error: %s", err)
	}
	runTests(t, p.In, []parseTestData{
		{
			givenName:        "empty",
			givenContent:     ``,
			expectedValue:    nil,
			expectedErrCount: 1,
		}, {
			givenName:        "no match 1",
			givenContent:     `1:a`,
			expectedValue:    nil,
			expectedErrCount: 1,
		}, {
			givenName:        "no match 2",
			givenContent:     `_a`,
			expectedValue:    nil,
			expectedErrCount: 1,
		}, {
			givenName:        "simple 1",
			givenContent:     `aB`,
			expectedValue:    data.Port{Name: "aB"},
			expectedErrCount: 0,
		}, {
			givenName:        "simple 2",
			givenContent:     `a0`,
			expectedValue:    data.Port{Name: "a0"},
			expectedErrCount: 0,
		}, {
			givenName:        "simple 3",
			givenContent:     `aB_cd`,
			expectedValue:    data.Port{Name: "aB"},
			expectedErrCount: 0,
		}, {
			givenName:        "simple 4",
			givenContent:     `abcDef`,
			expectedValue:    data.Port{Name: "abcDef"},
			expectedErrCount: 0,
		}, {
			givenName:        "complex 1",
			givenContent:     `ab1Cd:1`,
			expectedValue:    data.Port{Name: "ab1Cd", HasIndex: true, Index: 1},
			expectedErrCount: 0,
		}, {
			givenName:        "complex 2",
			givenContent:     `a1Bc:003`,
			expectedValue:    data.Port{Name: "a1Bc", HasIndex: true, Index: 3},
			expectedErrCount: 0,
		},
	})
}

func TestParseArrow(t *testing.T) {
	p, err := NewParseArrow()
	if err != nil {
		t.Fatalf("Unexpected error: %s", err)
	}
	runTests(t, p.In, []parseTestData{
		{
			givenName:        "empty",
			givenContent:     ``,
			expectedValue:    nil,
			expectedErrCount: 1,
		}, {
			givenName:        "no match 1",
			givenContent:     `>`,
			expectedValue:    nil,
			expectedErrCount: 1,
		}, {
			givenName:        "no match 2",
			givenContent:     `aPort (pack.Data)- bPort`,
			expectedValue:    nil,
			expectedErrCount: 1,
		}, {
			givenName:        "simple 1",
			givenContent:     `->`,
			expectedValue:    data.Arrow{},
			expectedErrCount: 0,
		}, {
			givenName:    "simple 2",
			givenContent: `aPort->bPort`,
			expectedValue: data.Arrow{
				FromPort: &data.Port{Name: "aPort"},
				ToPort:   &data.Port{Name: "bPort"},
			},
			expectedErrCount: 0,
		}, {
			givenName:    "simple 3",
			givenContent: `(Data)->`,
			expectedValue: data.Arrow{
				Data: []data.Type{data.Type{LocalType: "Data"}},
			},
			expectedErrCount: 0,
		}, {
			givenName:    "complex",
			givenContent: "aPort // comment1\n ( \t Data \t ) \t -> // comment2\n bPort",
			expectedValue: data.Arrow{
				FromPort: &data.Port{Name: "aPort"},
				Data:     []data.Type{data.Type{LocalType: "Data"}},
				ToPort:   &data.Port{Name: "bPort"},
			},
			expectedErrCount: 0,
		},
	})
}
<commit_msg>Fix remaining tests<commit_after>package parser

import (
	"testing"

	"github.com/flowdev/gflowparser/data"
)

func TestParsePort(t *testing.T) {
	p, err := NewParsePort()
	if err != nil {
		t.Fatalf("Unexpected error: %s", err)
	}
	runTests(t, p.In, []parseTestData{
		{
			givenName:        "empty",
			givenContent:     ``,
			expectedValue:    nil,
			expectedErrCount: 1,
		}, {
			givenName:        "no match 1",
			givenContent:     `1:a`,
			expectedValue:    nil,
			expectedErrCount: 1,
		}, {
			givenName:        "no match 2",
			givenContent:     `_a`,
			expectedValue:    nil,
			expectedErrCount: 1,
		}, {
			givenName:        "simple 1",
			givenContent:     `aB`,
			expectedValue:    data.Port{Name: "aB"},
			expectedErrCount: 0,
		}, {
			givenName:        "simple 2",
			givenContent:     `a0`,
			expectedValue:    data.Port{Name: "a0"},
			expectedErrCount: 0,
		}, {
			givenName:        "simple 3",
			givenContent:     `aB_cd`,
			expectedValue:    data.Port{Name: "aB"},
			expectedErrCount: 0,
		}, {
			givenName:        "simple 4",
			givenContent:     `abcDef`,
			expectedValue:    data.Port{Name: "abcDef"},
			expectedErrCount: 0,
		}, {
			givenName:        "complex 1",
			givenContent:     `ab1Cd:1`,
			expectedValue:    data.Port{Name: "ab1Cd", HasIndex: true, Index: 1},
			expectedErrCount: 0,
		}, {
			givenName:        "complex 2",
			givenContent:     `a1Bc:003`,
			expectedValue:    data.Port{Name: "a1Bc", HasIndex: true, Index: 3},
			expectedErrCount: 0,
		},
	})
}

func TestParseArrow(t *testing.T) {
	p, err := NewParseArrow()
	if err != nil {
		t.Fatalf("Unexpected error: %s", err)
	}
	runTests(t, p.In, []parseTestData{
		{
			givenName:        "empty",
			givenContent:     ``,
			expectedValue:    nil,
			expectedErrCount: 1,
		}, {
			givenName:        "no match 1",
			givenContent:     `>`,
			expectedValue:    nil,
			expectedErrCount: 1,
		}, {
			givenName:        "no match 2",
			givenContent:     `aPort (pack.Data)- bPort`,
			expectedValue:    nil,
			expectedErrCount: 1,
		}, {
			givenName:        "simple 1",
			givenContent:     `->`,
			expectedValue:    data.Arrow{},
			expectedErrCount: 0,
		}, {
			givenName:    "simple 2",
			givenContent: `aPort->bPort`,
			expectedValue: data.Arrow{
				FromPort: &data.Port{Name: "aPort"},
				ToPort:   &data.Port{Name: "bPort", SrcPos: 7},
			},
			expectedErrCount: 0,
		}, {
			givenName:    "simple 3",
			givenContent: `(Data)->`,
			expectedValue: data.Arrow{
				Data: []data.Type{data.Type{LocalType: "Data", SrcPos: 1}},
			},
			expectedErrCount: 0,
		}, {
			givenName:    "complex",
			givenContent: "aPort // comment1\n ( \t Data \t ) \t -> // comment2\n bPort",
			expectedValue: data.Arrow{
				FromPort: &data.Port{Name: "aPort"},
				Data:     []data.Type{data.Type{LocalType: "Data", SrcPos: 23}},
				ToPort:   &data.Port{Name: "bPort", SrcPos: 50},
			},
			expectedErrCount: 0,
		},
	})
}
<|endoftext|>


<commit_before>package ghg

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"github.com/mholt/archiver"
	"github.com/mitchellh/ioprogress"
	"github.com/octokit/go-octokit/octokit"
	"github.com/pkg/errors"
)

func getOctCli(token string) *octokit.Client {
	var auth octokit.AuthMethod
	if token != "" {
		auth = octokit.TokenAuth{AccessToken: token}
	}
	return octokit.NewClient(auth)
}

type ghg struct {
	ghgHome string
	target  string
	client  *octokit.Client
	upgrade bool
}

func (gh *ghg) getGhgHome() string {
	if gh.ghgHome != "" {
		return gh.ghgHome
	}
	return "."
}

func (gh *ghg) getBinDir() string {
	return filepath.Join(gh.getGhgHome(), "bin")
}

var releaseByTagURL = octokit.Hyperlink("repos/{owner}/{repo}/releases/tags/{tag}")
var (
	archiveReg = regexp.MustCompile(`\.(?:zip|tgz|tar\.gz)$`)
	anyExtReg  = regexp.MustCompile(`\.[a-zA-Z0-9]+$`)
)

func (gh *ghg) get() error {
	owner, repo, tag, err := getOwnerRepoAndTag(gh.target)
	if err != nil {
		return errors.Wrap(err, "failed to resolve target")
	}
	log.Printf("fetch the GitHub release for %s\n", gh.target)
	var url *url.URL
	if tag == "" {
		url, err = octokit.ReleasesLatestURL.Expand(octokit.M{"owner": owner, "repo": repo})
	} else {
		url, err = releaseByTagURL.Expand(octokit.M{"owner": owner, "repo": repo, "tag": tag})
	}
	if err != nil {
		return errors.Wrap(err, "failed to build GitHub URL")
	}
	release, r := gh.client.Releases(url).Latest()
	if r.HasError() {
		return errors.Wrap(r.Err, "failed to fetch a release")
	}
	tag = release.TagName
	goarch := runtime.GOARCH
	goos := runtime.GOOS
	var urls []string
	for _, asset := range release.Assets {
		name := asset.Name
		if strings.Contains(name, goarch) && strings.Contains(name, goos) &&
			(archiveReg.MatchString(name) || !anyExtReg.MatchString(name)) {
			urls = append(urls, fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", owner, repo, tag, name))
		}
	}
	if len(urls) < 1 {
		return fmt.Errorf("no assets available")
	}
	err = os.MkdirAll(filepath.Join(gh.getGhgHome(), "tmp"), 0755)
	if err != nil {
		return errors.Wrap(err, "failed to create tmpdir")
	}
	log.Printf("install %s/%s version: %s", owner, repo, tag)
	for _, url := range urls {
		err := gh.install(url)
		if err != nil {
			return err
		}
	}
	return nil
}

func (gh *ghg) install(url string) error {
	log.Printf("download %s\n", url)
	archivePath, err := gh.download(url)
	if err != nil {
		return errors.Wrap(err, "failed to download")
	}
	tmpdir := filepath.Dir(archivePath)
	defer os.RemoveAll(tmpdir)

	if !archiveReg.MatchString(url) {
		_, repo, _, _ := getOwnerRepoAndTag(gh.target)
		name := lcs(repo, filepath.Base(archivePath))
		name = strings.Trim(name, "-_")
		if name == "" {
			name = repo
		}
		if runtime.GOOS == "windows" {
			name += ".exe"
		}
		return gh.place(archivePath, filepath.Join(gh.getBinDir(), name))
	}
	workDir := filepath.Join(tmpdir, "work")
	os.MkdirAll(workDir, 0755)

	log.Printf("extract %s\n", path.Base(url))
	err = extract(archivePath, workDir)
	if err != nil {
		return errors.Wrap(err, "failed to extract")
	}

	bin := gh.getBinDir()
	os.MkdirAll(bin, 0755)

	err = gh.pickupExecutable(workDir)
	if err != nil {
		return errors.Wrap(err, "failed to pickup")
	}
	return nil
}

func (gh *ghg) place(src, dest string) error {
	if exists(dest) {
		if !gh.upgrade {
			log.Printf("%s already exists. skip installing. You can use -u flag for overwrite it", dest)
			return nil
		}
		log.Printf("%s exists. overwrite it", dest)
	}
	log.Printf("install %s\n", filepath.Base(dest))
	return os.Rename(src, dest)
}

func (gh *ghg) download(url string) (fpath string, err error) {
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		err = errors.Wrap(err, "failed to create request")
		return
	}
	req.Header.Set("User-Agent", fmt.Sprintf("ghg/%s", version))
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		err = errors.Wrap(err, "failed to create request")
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		err = fmt.Errorf("http response not OK. code: %d, url: %s", resp.StatusCode, url)
		return
	}
	archiveBase := path.Base(url)
	tempdir, err := ioutil.TempDir(filepath.Join(gh.getGhgHome(), "tmp"), "")
	if err != nil {
		err = errors.Wrap(err, "failed to create tempdir")
		return
	}
	defer func() {
		if err != nil {
			os.RemoveAll(tempdir)
		}
	}()
	fpath = filepath.Join(tempdir, archiveBase)
	f, err := os.OpenFile(filepath.Join(tempdir, archiveBase), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
	if err != nil {
		err = errors.Wrap(err, "failed to open file")
		return
	}
	defer f.Close()
	progressR := progbar(resp.Body, resp.ContentLength)
	_, err = io.Copy(f, progressR)
	if err != nil {
		err = errors.Wrap(err, "failed to read response")
		return
	}
	return fpath, nil
}

func progbar(r io.Reader, size int64) io.Reader {
	bar := ioprogress.DrawTextFormatBar(40)
	f := func(progress, total int64) string {
		return fmt.Sprintf(
			"%s %s",
			bar(progress, total),
			ioprogress.DrawTextFormatBytes(progress, total))
	}
	return &ioprogress.Reader{
		Reader:   r,
		Size:     size,
		DrawFunc: ioprogress.DrawTerminalf(os.Stderr, f),
	}
}

func extract(src, dest string) error {
	base := filepath.Base(src)
	if strings.HasSuffix(base, ".zip") {
		return archiver.Zip.Open(src, dest)
	}
	if strings.HasSuffix(base, ".tar.gz") || strings.HasSuffix(base, ".tgz") {
		return archiver.TarGz.Open(src, dest)
	}
	return fmt.Errorf("failed to extract file: %s", src)
}

var targetReg = regexp.MustCompile(`^(?:([^/]+)/)?([^@]+)(?:@(.+))?$`)

func getOwnerRepoAndTag(target string) (owner, repo, tag string, err error) {
	matches := targetReg.FindStringSubmatch(target)
	if len(matches) != 4 {
		err = fmt.Errorf("failed to get owner, repo and tag")
		return
	}
	owner = matches[1]
	repo = matches[2]
	tag = matches[3]
	if owner == "" {
		owner = repo
	}
	return
}

var executableReg = func() *regexp.Regexp {
	s := `^[a-z][-_a-zA-Z0-9]+`
	if runtime.GOOS == "windows" {
		s += `\.exe`
	}
	return regexp.MustCompile(s + `$`)
}()

func (gh *ghg) pickupExecutable(src string) error {
	bindir := gh.getBinDir()
	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
		if err != nil || info.IsDir() {
			return err
		}
		if name := info.Name(); (info.Mode()&0111) != 0 && executableReg.MatchString(name) {
			return gh.place(path, filepath.Join(bindir, name))
		}
		return nil
	})
}

func exists(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil
}

func lcs(a, b string) string {
	arunes := []rune(a)
	brunes := []rune(b)
	aLen := len(arunes)
	bLen := len(brunes)
	lengths := make([][]int, aLen+1)
	for i := 0; i <= aLen; i++ {
		lengths[i] = make([]int, bLen+1)
	}
	// row 0 and column 0 are initialized to 0 already

	for i := 0; i < aLen; i++ {
		for j := 0; j < bLen; j++ {
			if arunes[i] == brunes[j] {
				lengths[i+1][j+1] = lengths[i][j] + 1
			} else if lengths[i+1][j] > lengths[i][j+1] {
				lengths[i+1][j+1] = lengths[i+1][j]
			} else {
				lengths[i+1][j+1] = lengths[i][j+1]
			}
		}
	}

	// read the substring out from the matrix
	s := make([]rune, 0, lengths[aLen][bLen])
	for x, y := aLen, bLen; x != 0 && y != 0; {
		if lengths[x][y] == lengths[x-1][y] {
			x--
		} else if lengths[x][y] == lengths[x][y-1] {
			y--
		} else {
			s = append(s, arunes[x-1])
			x--
			y--
		}
	}
	// reverse string
	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
		s[i], s[j] = s[j], s[i]
	}
	return string(s)
}
<commit_msg>Fix rename error that occurs if the target is a naked binary<commit_after>package ghg

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"github.com/mholt/archiver"
	"github.com/mitchellh/ioprogress"
	"github.com/octokit/go-octokit/octokit"
	"github.com/pkg/errors"
)

func getOctCli(token string) *octokit.Client {
	var auth octokit.AuthMethod
	if token != "" {
		auth = octokit.TokenAuth{AccessToken: token}
	}
	return octokit.NewClient(auth)
}

type ghg struct {
	ghgHome string
	target  string
	client  *octokit.Client
	upgrade bool
}

func (gh *ghg) getGhgHome() string {
	if gh.ghgHome != "" {
		return gh.ghgHome
	}
	return "."
}

func (gh *ghg) getBinDir() string {
	return filepath.Join(gh.getGhgHome(), "bin")
}

var releaseByTagURL = octokit.Hyperlink("repos/{owner}/{repo}/releases/tags/{tag}")
var (
	archiveReg = regexp.MustCompile(`\.(?:zip|tgz|tar\.gz)$`)
	anyExtReg  = regexp.MustCompile(`\.[a-zA-Z0-9]+$`)
)

func (gh *ghg) get() error {
	owner, repo, tag, err := getOwnerRepoAndTag(gh.target)
	if err != nil {
		return errors.Wrap(err, "failed to resolve target")
	}
	log.Printf("fetch the GitHub release for %s\n", gh.target)
	var url *url.URL
	if tag == "" {
		url, err = octokit.ReleasesLatestURL.Expand(octokit.M{"owner": owner, "repo": repo})
	} else {
		url, err = releaseByTagURL.Expand(octokit.M{"owner": owner, "repo": repo, "tag": tag})
	}
	if err != nil {
		return errors.Wrap(err, "failed to build GitHub URL")
	}
	release, r := gh.client.Releases(url).Latest()
	if r.HasError() {
		return errors.Wrap(r.Err, "failed to fetch a release")
	}
	tag = release.TagName
	goarch := runtime.GOARCH
	goos := runtime.GOOS
	var urls []string
	for _, asset := range release.Assets {
		name := asset.Name
		if strings.Contains(name, goarch) && strings.Contains(name, goos) &&
			(archiveReg.MatchString(name) || !anyExtReg.MatchString(name)) {
			urls = append(urls, fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", owner, repo, tag, name))
		}
	}
	if len(urls) < 1 {
		return fmt.Errorf("no assets available")
	}
	err = os.MkdirAll(filepath.Join(gh.getGhgHome(), "tmp"), 0755)
	if err != nil {
		return errors.Wrap(err, "failed to create tmpdir")
	}
	log.Printf("install %s/%s version: %s", owner, repo, tag)
	for _, url := range urls {
		err := gh.install(url)
		if err != nil {
			return err
		}
	}
	return nil
}

func (gh *ghg) install(url string) error {
	log.Printf("download %s\n", url)
	archivePath, err := gh.download(url)
	if err != nil {
		return errors.Wrap(err, "failed to download")
	}
	tmpdir := filepath.Dir(archivePath)
	defer os.RemoveAll(tmpdir)

	bin := gh.getBinDir()
	os.MkdirAll(bin, 0755)

	if !archiveReg.MatchString(url) {
		_, repo, _, _ := getOwnerRepoAndTag(gh.target)
		name := lcs(repo, filepath.Base(archivePath))
		name = strings.Trim(name, "-_")
		if name == "" {
			name = repo
		}
		if runtime.GOOS == "windows" {
			name += ".exe"
		}
		return gh.place(archivePath, filepath.Join(gh.getBinDir(), name))
	}
	workDir := filepath.Join(tmpdir, "work")
	os.MkdirAll(workDir, 0755)

	log.Printf("extract %s\n", path.Base(url))
	err = extract(archivePath, workDir)
	if err != nil {
		return errors.Wrap(err, "failed to extract")
	}

	err = gh.pickupExecutable(workDir)
	if err != nil {
		return errors.Wrap(err, "failed to pickup")
	}
	return nil
}

func (gh *ghg) place(src, dest string) error {
	if exists(dest) {
		if !gh.upgrade {
			log.Printf("%s already exists. skip installing. You can use -u flag for overwrite it", dest)
			return nil
		}
		log.Printf("%s exists. overwrite it", dest)
	}
	log.Printf("install %s\n", filepath.Base(dest))
	return os.Rename(src, dest)
}

func (gh *ghg) download(url string) (fpath string, err error) {
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		err = errors.Wrap(err, "failed to create request")
		return
	}
	req.Header.Set("User-Agent", fmt.Sprintf("ghg/%s", version))
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		err = errors.Wrap(err, "failed to create request")
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		err = fmt.Errorf("http response not OK. code: %d, url: %s", resp.StatusCode, url)
		return
	}
	archiveBase := path.Base(url)
	tempdir, err := ioutil.TempDir(filepath.Join(gh.getGhgHome(), "tmp"), "")
	if err != nil {
		err = errors.Wrap(err, "failed to create tempdir")
		return
	}
	defer func() {
		if err != nil {
			os.RemoveAll(tempdir)
		}
	}()
	fpath = filepath.Join(tempdir, archiveBase)
	f, err := os.OpenFile(filepath.Join(tempdir, archiveBase), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
	if err != nil {
		err = errors.Wrap(err, "failed to open file")
		return
	}
	defer f.Close()
	progressR := progbar(resp.Body, resp.ContentLength)
	_, err = io.Copy(f, progressR)
	if err != nil {
		err = errors.Wrap(err, "failed to read response")
		return
	}
	return fpath, nil
}

func progbar(r io.Reader, size int64) io.Reader {
	bar := ioprogress.DrawTextFormatBar(40)
	f := func(progress, total int64) string {
		return fmt.Sprintf(
			"%s %s",
			bar(progress, total),
			ioprogress.DrawTextFormatBytes(progress, total))
	}
	return &ioprogress.Reader{
		Reader:   r,
		Size:     size,
		DrawFunc: ioprogress.DrawTerminalf(os.Stderr, f),
	}
}

func extract(src, dest string) error {
	base := filepath.Base(src)
	if strings.HasSuffix(base, ".zip") {
		return archiver.Zip.Open(src, dest)
	}
	if strings.HasSuffix(base, ".tar.gz") || strings.HasSuffix(base, ".tgz") {
		return archiver.TarGz.Open(src, dest)
	}
	return fmt.Errorf("failed to extract file: %s", src)
}

var targetReg = regexp.MustCompile(`^(?:([^/]+)/)?([^@]+)(?:@(.+))?$`)

func getOwnerRepoAndTag(target string) (owner, repo, tag string, err error) {
	matches := targetReg.FindStringSubmatch(target)
	if len(matches) != 4 {
		err = fmt.Errorf("failed to get owner, repo and tag")
		return
	}
	owner = matches[1]
	repo = matches[2]
	tag = matches[3]
	if owner == "" {
		owner = repo
	}
	return
}

var executableReg = func() *regexp.Regexp {
	s := `^[a-z][-_a-zA-Z0-9]+`
	if runtime.GOOS == "windows" {
		s += `\.exe`
	}
	return regexp.MustCompile(s + `$`)
}()

func (gh *ghg) pickupExecutable(src string) error {
	bindir := gh.getBinDir()
	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
		if err != nil || info.IsDir() {
			return err
		}
		if name := info.Name(); (info.Mode()&0111) != 0 && executableReg.MatchString(name) {
			return gh.place(path, filepath.Join(bindir, name))
		}
		return nil
	})
}

func exists(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil
}

func lcs(a, b string) string {
	arunes := []rune(a)
	brunes := []rune(b)
	aLen := len(arunes)
	bLen := len(brunes)
	lengths := make([][]int, aLen+1)
	for i := 0; i <= aLen; i++ {
		lengths[i] = make([]int, bLen+1)
	}
	// row 0 and column 0 are initialized to 0 already

	for i := 0; i < aLen; i++ {
		for j := 0; j < bLen; j++ {
			if arunes[i] == brunes[j] {
				lengths[i+1][j+1] = lengths[i][j] + 1
			} else if lengths[i+1][j] > lengths[i][j+1] {
				lengths[i+1][j+1] = lengths[i+1][j]
			} else {
				lengths[i+1][j+1] = lengths[i][j+1]
			}
		}
	}

	// read the substring out from the matrix
	s := make([]rune, 0, lengths[aLen][bLen])
	for x, y := aLen, bLen; x != 0 && y != 0; {
		if lengths[x][y] == lengths[x-1][y] {
			x--
		} else if lengths[x][y] == lengths[x][y-1] {
			y--
		} else {
			s = append(s, arunes[x-1])
			x--
			y--
		}
	}
	// reverse string
	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
		s[i], s[j] = s[j], s[i]
	}
	return string(s)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"io"
	"path/filepath"
	"time"

	"github.com/google/go-github/github"
	"github.com/pkg/errors"
	"golang.org/x/sync/errgroup"
)

type GHR struct {
	GitHub GitHub

	outStream io.Writer
}

func (g *GHR) CreateRelease(ctx context.Context, req *github.RepositoryRelease, recreate bool) (*github.RepositoryRelease, error) {

	// When draft release creation is requested,
	// create it witout any check (it can).
	if *req.Draft {
		fmt.Fprintln(g.outStream, "==> Create a draft release")
		return g.GitHub.CreateRelease(ctx, req)
	}

	// Check release is exist or not.
	// If release is not found, then create a new release.
	release, err := g.GitHub.GetRelease(ctx, *req.TagName)
	if err != nil {
		if err != RelaseNotFound {
			return nil, errors.Wrap(err, "failed to get release")
		}
		Debugf("Release (with tag %s) is not found: create a new one",
			*req.TagName)

		if recreate {
			fmt.Fprintf(g.outStream,
				"WARNING: '-recreate' is specified but release (%s) is not found",
				*req.TagName)
		}

		fmt.Fprintln(g.outStream, "==> Create a new release")
		return g.GitHub.CreateRelease(ctx, req)
	}

	// recreate is not true. Then use that existing release.
	if !recreate {
		Debugf("Release (with tag %s) exists: use existing one",
			*req.TagName)

		fmt.Fprintf(g.outStream, "WARNING: found release (%s). Use existing one.",
			*req.TagName)
		return release, nil
	}

	// When recreate is requested, delete existing release
	// and create a new release.
	fmt.Fprintln(g.outStream, "==> Recreate a release")
	if err := g.DeleteRelease(ctx, *release.ID, *req.TagName); err != nil {
		return nil, err
	}

	return g.GitHub.CreateRelease(ctx, req)
}

func (g *GHR) DeleteRelease(ctx context.Context, ID int, tag string) error {

	err := g.GitHub.DeleteRelease(ctx, ID)
	if err != nil {
		return err
	}

	err = g.GitHub.DeleteTag(ctx, tag)
	if err != nil {
		return err
	}

	// This is because sometimes process of creating release on GitHub is more
	// fast than deleting tag.
	time.Sleep(5 * time.Second)

	return nil
}

func (g *GHR) UploadAssets(ctx context.Context, releaseID int, localAssets []string, parallel int) error {
	start := time.Now()
	defer func() {
		Debugf("UploadAssets: time: %d ms", int(time.Since(start).Seconds()*1000))
	}()

	eg, ctx := errgroup.WithContext(ctx)
	semaphore := make(chan struct{}, parallel)
	for _, localAsset := range localAssets {
		localAsset := localAsset
		eg.Go(func() error {
			semaphore <- struct{}{}
			defer func() {
				<-semaphore
			}()

			fmt.Fprintf(g.outStream, "--> Uploading: %15s\n", filepath.Base(localAsset))
			_, err := g.GitHub.UploadAsset(ctx, releaseID, localAsset)
			if err != nil {
				return errors.Wrapf(err,
					"failed to upload asset: %s", localAsset)
			}
			return nil
		})
	}

	if err := eg.Wait(); err != nil {
		return errors.Wrap(err, "one of goroutines is failed")
	}

	return nil
}

func (g *GHR) DeleteAssets(ctx context.Context, releaseID int, localAssets []string, parallel int) error {
	start := time.Now()
	defer func() {
		Debugf("DeleteAssets: time: %d ms", int(time.Since(start).Seconds()*1000))
	}()

	eg, ctx := errgroup.WithContext(ctx)

	assets, err := g.GitHub.ListAssets(ctx, releaseID)
	if err != nil {
		return errors.Wrap(err, "failed to list assets")
	}

	semaphore := make(chan struct{}, parallel)
	for _, localAsset := range localAssets {
		for _, asset := range assets {
			// https://golang.org/doc/faq#closures_and_goroutines
			localAsset, asset := localAsset, asset

			// Uploaded asset name is same as basename of local file
			if *asset.Name == filepath.Base(localAsset) {
				eg.Go(func() error {
					semaphore <- struct{}{}
					defer func() {
						<-semaphore
					}()

					fmt.Fprintf(g.outStream, "--> Deleting: %15s\n", *asset.Name)
					if err := g.GitHub.DeleteAsset(ctx, *asset.ID); err != nil {
						return errors.Wrapf(err,
							"failed to delete asset: %s", *asset.Name)
					}
					return nil
				})
			}
		}
	}

	if err := eg.Wait(); err != nil {
		return errors.Wrap(err, "one of goroutines is failed")
	}

	return nil
}
<commit_msg>chore(ghr): fix typo<commit_after>package main

import (
	"context"
	"fmt"
	"io"
	"path/filepath"
	"time"

	"github.com/google/go-github/github"
	"github.com/pkg/errors"
	"golang.org/x/sync/errgroup"
)

type GHR struct {
	GitHub GitHub

	outStream io.Writer
}

func (g *GHR) CreateRelease(ctx context.Context, req *github.RepositoryRelease, recreate bool) (*github.RepositoryRelease, error) {

	// When draft release creation is requested,
	// create it without any check (it can).
	if *req.Draft {
		fmt.Fprintln(g.outStream, "==> Create a draft release")
		return g.GitHub.CreateRelease(ctx, req)
	}

	// Check release is exist or not.
	// If release is not found, then create a new release.
	release, err := g.GitHub.GetRelease(ctx, *req.TagName)
	if err != nil {
		if err != RelaseNotFound {
			return nil, errors.Wrap(err, "failed to get release")
		}
		Debugf("Release (with tag %s) is not found: create a new one",
			*req.TagName)

		if recreate {
			fmt.Fprintf(g.outStream,
				"WARNING: '-recreate' is specified but release (%s) is not found",
				*req.TagName)
		}

		fmt.Fprintln(g.outStream, "==> Create a new release")
		return g.GitHub.CreateRelease(ctx, req)
	}

	// recreate is not true. Then use that existing release.
	if !recreate {
		Debugf("Release (with tag %s) exists: use existing one",
			*req.TagName)

		fmt.Fprintf(g.outStream, "WARNING: found release (%s). Use existing one.",
			*req.TagName)
		return release, nil
	}

	// When recreate is requested, delete existing release
	// and create a new release.
	fmt.Fprintln(g.outStream, "==> Recreate a release")
	if err := g.DeleteRelease(ctx, *release.ID, *req.TagName); err != nil {
		return nil, err
	}

	return g.GitHub.CreateRelease(ctx, req)
}

func (g *GHR) DeleteRelease(ctx context.Context, ID int, tag string) error {

	err := g.GitHub.DeleteRelease(ctx, ID)
	if err != nil {
		return err
	}

	err = g.GitHub.DeleteTag(ctx, tag)
	if err != nil {
		return err
	}

	// This is because sometimes process of creating release on GitHub is more
	// fast than deleting tag.
	time.Sleep(5 * time.Second)

	return nil
}

func (g *GHR) UploadAssets(ctx context.Context, releaseID int, localAssets []string, parallel int) error {
	start := time.Now()
	defer func() {
		Debugf("UploadAssets: time: %d ms", int(time.Since(start).Seconds()*1000))
	}()

	eg, ctx := errgroup.WithContext(ctx)
	semaphore := make(chan struct{}, parallel)
	for _, localAsset := range localAssets {
		localAsset := localAsset
		eg.Go(func() error {
			semaphore <- struct{}{}
			defer func() {
				<-semaphore
			}()

			fmt.Fprintf(g.outStream, "--> Uploading: %15s\n", filepath.Base(localAsset))
			_, err := g.GitHub.UploadAsset(ctx, releaseID, localAsset)
			if err != nil {
				return errors.Wrapf(err,
					"failed to upload asset: %s", localAsset)
			}
			return nil
		})
	}

	if err := eg.Wait(); err != nil {
		return errors.Wrap(err, "one of goroutines is failed")
	}

	return nil
}

func (g *GHR) DeleteAssets(ctx context.Context, releaseID int, localAssets []string, parallel int) error {
	start := time.Now()
	defer func() {
		Debugf("DeleteAssets: time: %d ms", int(time.Since(start).Seconds()*1000))
	}()

	eg, ctx := errgroup.WithContext(ctx)

	assets, err := g.GitHub.ListAssets(ctx, releaseID)
	if err != nil {
		return errors.Wrap(err, "failed to list assets")
	}

	semaphore := make(chan struct{}, parallel)
	for _, localAsset := range localAssets {
		for _, asset := range assets {
			// https://golang.org/doc/faq#closures_and_goroutines
			localAsset, asset := localAsset, asset

			// Uploaded asset name is same as basename of local file
			if *asset.Name == filepath.Base(localAsset) {
				eg.Go(func() error {
					semaphore <- struct{}{}
					defer func() {
						<-semaphore
					}()

					fmt.Fprintf(g.outStream, "--> Deleting: %15s\n", *asset.Name)
					if err := g.GitHub.DeleteAsset(ctx, *asset.ID); err != nil {
						return errors.Wrapf(err,
							"failed to delete asset: %s", *asset.Name)
					}
					return nil
				})
			}
		}
	}

	if err := eg.Wait(); err != nil {
		return errors.Wrap(err, "one of goroutines is failed")
	}

	return nil
}
<|endoftext|>