<commit_before>package iapi

import (
	"strings"
	"testing"
)

func TestGetValidNotification(t *testing.T) {

	name := "valid-notification"
	_, err := Icinga2_Server.GetNotification(name)

	if err != nil {
		t.Error(err)
	}
}

func TestGetInvalidNotification(t *testing.T) {

	name := "invalid-notification"

	_, err := Icinga2_Server.GetNotification(name)

	if err != nil {
		t.Error(err)
	}
}

func TestCreateNotificationCommandDNE(t *testing.T) {

	hostname := "host"
	notificationname := hostname+"!"+hostname
	command := "invalid-command"
	servicename := ""
	interval := 1800

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, nil, nil, nil)

	if !strings.Contains(err.Error(), "type 'NotificationCommand' does not exist.") {
		t.Error(err)
	}
}

// func TestCreateNotificationHostDNE
// Try and create a notification for a host that does not exist.
// Should fail with an error about the host not existing.
func TestCreateNotificationHostDNE(t *testing.T) {

	hostname := "host-dne"
	notificationname := hostname+"!"+hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, nil, nil, nil)

	if !strings.Contains(err.Error(), "type 'Host' does not exist.") {
		t.Error(err)
	}
}

func TestCreateNotificationInvalidName(t *testing.T) {

	hostname := "host-dne"
	notificationname := "invalid-name"
	command := "mail-host-notification"
	servicename := ""
	interval := 1800

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, nil, nil, nil)

	if !strings.Contains(err.Error(), "Invalid Notification name.") {
		t.Error(err)
	}
}

func TestCreateNotificationUserDNE(t *testing.T) {

	hostname := "host"
	group := []string{"linux-servers"}
	notificationname := hostname+"!"+hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800
	users := []string{"user-dne"}

    _, _ = Icinga2_Server.CreateHost(hostname, "127.0.0.1", "hostalive", nil, nil, group)
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, users, nil, nil)

	if !strings.Contains(err.Error(), "type 'User' does not exist.") {
		t.Error(err)
	}
}

func TestCreateHostNotification(t *testing.T) {

	hostname := "host"
	groups := []string{"linux-servers"}
	notificationname := hostname+"!"+hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800
	username := "user"

    _, _ = Icinga2_Server.CreateHost(hostname, "127.0.0.1", "hostalive", nil, nil, groups)
    _, _ = Icinga2_Server.CreateUser(username, "user@example.com")
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if err != nil {
		t.Error(err)
	}
}

func TestCreateHostNotificationAlreadyExists(t *testing.T) {

	hostname := "host"
	notificationname := hostname+"!"+hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800
	username := "user"

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if !strings.HasSuffix(err.Error(), " already exists.") {
		t.Error(err)
	}
}

func TestCreateServiceNotification(t *testing.T) {

	hostname := "host"
	servicename := "test"
	check_command := "random"
	notificationname := hostname+"!"+servicename+"!"+hostname+"-"+servicename
	command := "mail-service-notification"
	interval := 1800
	username := "user"

    _, _ = Icinga2_Server.CreateUser(username, "user@example.com")
    _, _ = Icinga2_Server.CreateService(servicename, hostname, check_command)
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if err != nil {
		t.Error(err)
	}
}

func TestCreateServiceNotificationAlreadyExists(t *testing.T) {

	hostname := "host"
	servicename := "test"
	check_command := "random"
	notificationname := hostname+"!"+servicename+"!"+hostname+"-"+servicename
	command := "mail-service-notification"
	interval := 1800
	username := "user"

    _, _ = Icinga2_Server.CreateUser(username, "user@example.com")
    _, _ = Icinga2_Server.CreateService(servicename, hostname, check_command)
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if !strings.HasSuffix(err.Error(), " already exists.") {
		t.Error(err)
	}
}

func TestDeleteServiceNotification(t *testing.T) {

	hostname := "host"
	servicename := "test"
	notificationname := hostname+"!"+servicename+"!"+hostname+"-"+servicename

	err := Icinga2_Server.DeleteNotification(notificationname)

	if err != nil {
		t.Error(err)
	}
}

func TestDeleteServiceNotificationDNE(t *testing.T) {

	hostname := "host"
	servicename := "test"
	notificationname := hostname+"!"+servicename+"!"+hostname+"-"+servicename

	err := Icinga2_Server.DeleteNotification(notificationname)

	if err.Error() != "No objects found." {
		t.Error(err)
	}
}

func TestDeleteHostNotification(t *testing.T) {

	hostname := "host"
	notificationname := hostname+"!"+hostname
	username := "user"

	err := Icinga2_Server.DeleteNotification(notificationname)
    _ = Icinga2_Server.DeleteHost(hostname)
    _ = Icinga2_Server.DeleteUser(username)

	if err != nil {
		t.Error(err)
	}
}

func TestDeleteHostNotificationDNE(t *testing.T) {

	hostname := "host"
	notificationname := hostname+"!"+hostname

	err := Icinga2_Server.DeleteNotification(notificationname)

	if err.Error() != "No objects found." {
		t.Error(err)
	}
}

// func TestDeleteNotificationNonAPI
// Notifications that were not created via the API, cannot be deleted via the API
// Should get an error about not being created via the API
func TestDeleteNotificationNonAPI(t *testing.T) {

	notificationname := "mail-icingaadmin"

	err := Icinga2_Server.DeleteNotification(notificationname)
	if err.Error() != "No objects found." {
		t.Error(err)
	}
}<commit_msg>Updated notifications_test.go account for new parameter in CreateService<commit_after>package iapi

import (
	"strings"
	"testing"
)

func TestGetValidNotification(t *testing.T) {

	name := "valid-notification"
	_, err := Icinga2_Server.GetNotification(name)

	if err != nil {
		t.Error(err)
	}
}

func TestGetInvalidNotification(t *testing.T) {

	name := "invalid-notification"

	_, err := Icinga2_Server.GetNotification(name)

	if err != nil {
		t.Error(err)
	}
}

func TestCreateNotificationCommandDNE(t *testing.T) {

	hostname := "host"
	notificationname := hostname+"!"+hostname
	command := "invalid-command"
	servicename := ""
	interval := 1800

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, nil, nil, nil)

	if !strings.Contains(err.Error(), "type 'NotificationCommand' does not exist.") {
		t.Error(err)
	}
}

// func TestCreateNotificationHostDNE
// Try and create a notification for a host that does not exist.
// Should fail with an error about the host not existing.
func TestCreateNotificationHostDNE(t *testing.T) {

	hostname := "host-dne"
	notificationname := hostname+"!"+hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, nil, nil, nil)

	if !strings.Contains(err.Error(), "type 'Host' does not exist.") {
		t.Error(err)
	}
}

func TestCreateNotificationInvalidName(t *testing.T) {

	hostname := "host-dne"
	notificationname := "invalid-name"
	command := "mail-host-notification"
	servicename := ""
	interval := 1800

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, nil, nil, nil)

	if !strings.Contains(err.Error(), "Invalid Notification name.") {
		t.Error(err)
	}
}

func TestCreateNotificationUserDNE(t *testing.T) {

	hostname := "host"
	group := []string{"linux-servers"}
	notificationname := hostname+"!"+hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800
	users := []string{"user-dne"}

    _, _ = Icinga2_Server.CreateHost(hostname, "127.0.0.1", "hostalive", nil, nil, group)
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, users, nil, nil)

	if !strings.Contains(err.Error(), "type 'User' does not exist.") {
		t.Error(err)
	}
}

func TestCreateHostNotification(t *testing.T) {

	hostname := "host"
	groups := []string{"linux-servers"}
	notificationname := hostname+"!"+hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800
	username := "user"

    _, _ = Icinga2_Server.CreateHost(hostname, "127.0.0.1", "hostalive", nil, nil, groups)
    _, _ = Icinga2_Server.CreateUser(username, "user@example.com")
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if err != nil {
		t.Error(err)
	}
}

func TestCreateHostNotificationAlreadyExists(t *testing.T) {

	hostname := "host"
	notificationname := hostname+"!"+hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800
	username := "user"

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if !strings.HasSuffix(err.Error(), " already exists.") {
		t.Error(err)
	}
}

func TestCreateServiceNotification(t *testing.T) {

	hostname := "host"
	servicename := "test"
	check_command := "random"
	notificationname := hostname+"!"+servicename+"!"+hostname+"-"+servicename
	command := "mail-service-notification"
	interval := 1800
	username := "user"

    _, _ = Icinga2_Server.CreateUser(username, "user@example.com")
    _, _ = Icinga2_Server.CreateService(servicename, hostname, check_command,nil)
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if err != nil {
		t.Error(err)
	}
}

func TestCreateServiceNotificationAlreadyExists(t *testing.T) {

	hostname := "host"
	servicename := "test"
	check_command := "random"
	notificationname := hostname+"!"+servicename+"!"+hostname+"-"+servicename
	command := "mail-service-notification"
	interval := 1800
	username := "user"

    _, _ = Icinga2_Server.CreateUser(username, "user@example.com")
    _, _ = Icinga2_Server.CreateService(servicename, hostname, check_command, nil)
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if !strings.HasSuffix(err.Error(), " already exists.") {
		t.Error(err)
	}
}

func TestDeleteServiceNotification(t *testing.T) {

	hostname := "host"
	servicename := "test"
	notificationname := hostname+"!"+servicename+"!"+hostname+"-"+servicename

	err := Icinga2_Server.DeleteNotification(notificationname)

	if err != nil {
		t.Error(err)
	}
}

func TestDeleteServiceNotificationDNE(t *testing.T) {

	hostname := "host"
	servicename := "test"
	notificationname := hostname+"!"+servicename+"!"+hostname+"-"+servicename

	err := Icinga2_Server.DeleteNotification(notificationname)

	if err.Error() != "No objects found." {
		t.Error(err)
	}
}

func TestDeleteHostNotification(t *testing.T) {

	hostname := "host"
	notificationname := hostname+"!"+hostname
	username := "user"

	err := Icinga2_Server.DeleteNotification(notificationname)
    _ = Icinga2_Server.DeleteHost(hostname)
    _ = Icinga2_Server.DeleteUser(username)

	if err != nil {
		t.Error(err)
	}
}

func TestDeleteHostNotificationDNE(t *testing.T) {

	hostname := "host"
	notificationname := hostname+"!"+hostname

	err := Icinga2_Server.DeleteNotification(notificationname)

	if err.Error() != "No objects found." {
		t.Error(err)
	}
}

// func TestDeleteNotificationNonAPI
// Notifications that were not created via the API, cannot be deleted via the API
// Should get an error about not being created via the API
func TestDeleteNotificationNonAPI(t *testing.T) {

	notificationname := "mail-icingaadmin"

	err := Icinga2_Server.DeleteNotification(notificationname)
	if err.Error() != "No objects found." {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package iapi

import (
	"strings"
	"testing"
)

func TestGetValidNotification(t *testing.T) {

	name := "valid-notification"
	_, err := Icinga2_Server.GetNotification(name)

	if err != nil {
		t.Error(err)
	}
}

func TestGetInvalidNotification(t *testing.T) {

	name := "invalid-notification"

	_, err := Icinga2_Server.GetNotification(name)

	if err != nil {
		t.Error(err)
	}
}

func TestCreateNotificationCommandDNE(t *testing.T) {

	hostname := "host"
	notificationname := hostname + "!" + hostname
	command := "invalid-command"
	servicename := ""
	interval := 1800

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, nil, nil, nil)

	if !strings.Contains(err.Error(), "500 Object could not be created") {
		t.Error(err)
	}
}

// func TestCreateNotificationHostDNE
// Try and create a notification for a host that does not exist.
// Should fail with an error about the host not existing.
func TestCreateNotificationHostDNE(t *testing.T) {

	hostname := "host-dne"
	notificationname := hostname + "!" + hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, nil, nil, nil)

	if !strings.Contains(err.Error(), "500 Object could not be created") {
		t.Error(err)
	}
}

func TestCreateNotificationInvalidName(t *testing.T) {

	hostname := "host-dne"
	notificationname := "invalid-name"
	command := "mail-host-notification"
	servicename := ""
	interval := 1800

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, nil, nil, nil)

	if !strings.Contains(err.Error(), "500 Object could not be created") {
		t.Error(err)
	}
}

func TestCreateNotificationUserDNE(t *testing.T) {

	hostname := "host"
	group := []string{"linux-servers"}
	notificationname := hostname + "!" + hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800
	users := []string{"user-dne"}

	_, _ = Icinga2_Server.CreateHost(hostname, "127.0.0.1", "hostalive", nil, nil, group)
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, users, nil, nil)

	if !strings.Contains(err.Error(), "500 Object could not be created") {
		t.Error(err)
	}
}

func TestCreateHostNotification(t *testing.T) {

	hostname := "host"
	groups := []string{"linux-servers"}
	notificationname := hostname + "!" + hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800
	username := "user"

	_, _ = Icinga2_Server.CreateHost(hostname, "127.0.0.1", "hostalive", nil, nil, groups)
	_, _ = Icinga2_Server.CreateUser(username, "user@example.com")
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if err != nil {
		t.Error(err)
	}
}

func TestCreateHostNotificationAlreadyExists(t *testing.T) {

	hostname := "host"
	notificationname := hostname + "!" + hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800
	username := "user"

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if !strings.HasSuffix(err.Error(), "500 Object could not be created") {
		t.Error(err)
	}
}

func TestCreateServiceNotification(t *testing.T) {

	hostname := "host"
	servicename := "test"
	check_command := "random"
	notificationname := hostname + "!" + servicename + "!" + hostname + "-" + servicename
	command := "mail-service-notification"
	interval := 1800
	username := "user"

	_, _ = Icinga2_Server.CreateUser(username, "user@example.com")
	_, _ = Icinga2_Server.CreateService(servicename, hostname, check_command, nil)
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if err != nil {
		t.Error(err)
	}
}

func TestCreateServiceNotificationAlreadyExists(t *testing.T) {

	hostname := "host"
	servicename := "test"
	check_command := "random"
	notificationname := hostname + "!" + servicename + "!" + hostname + "-" + servicename
	command := "mail-service-notification"
	interval := 1800
	username := "user"

	_, _ = Icinga2_Server.CreateUser(username, "user@example.com")
	_, _ = Icinga2_Server.CreateService(servicename, hostname, check_command, nil)
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if !strings.HasSuffix(err.Error(), "500 Object could not be created") {
		t.Error(err)
	}
}

func TestDeleteServiceNotification(t *testing.T) {

	hostname := "host"
	servicename := "test"
	notificationname := hostname + "!" + servicename + "!" + hostname + "-" + servicename

	err := Icinga2_Server.DeleteNotification(notificationname)

	if err != nil {
		t.Error(err)
	}
}

func TestDeleteServiceNotificationDNE(t *testing.T) {

	hostname := "host"
	servicename := "test"
	notificationname := hostname + "!" + servicename + "!" + hostname + "-" + servicename

	err := Icinga2_Server.DeleteNotification(notificationname)

	if err.Error() != "No objects found." {
		t.Error(err)
	}
}

func TestDeleteHostNotification(t *testing.T) {

	hostname := "host"
	notificationname := hostname + "!" + hostname
	username := "user"

	err := Icinga2_Server.DeleteNotification(notificationname)
	_ = Icinga2_Server.DeleteHost(hostname)
	_ = Icinga2_Server.DeleteUser(username)

	if err != nil {
		t.Error(err)
	}
}

func TestDeleteHostNotificationDNE(t *testing.T) {

	hostname := "host"
	notificationname := hostname + "!" + hostname

	err := Icinga2_Server.DeleteNotification(notificationname)

	if err.Error() != "No objects found." {
		t.Error(err)
	}
}

// func TestDeleteNotificationNonAPI
// Notifications that were not created via the API, cannot be deleted via the API
// Should get an error about not being created via the API
func TestDeleteNotificationNonAPI(t *testing.T) {

	notificationname := "mail-icingaadmin"

	err := Icinga2_Server.DeleteNotification(notificationname)
	if err.Error() != "No objects found." {
		t.Error(err)
	}
}
<commit_msg>notifications: fix CreateService func signature<commit_after>package iapi

import (
	"strings"
	"testing"
)

func TestGetValidNotification(t *testing.T) {

	name := "valid-notification"
	_, err := Icinga2_Server.GetNotification(name)

	if err != nil {
		t.Error(err)
	}
}

func TestGetInvalidNotification(t *testing.T) {

	name := "invalid-notification"

	_, err := Icinga2_Server.GetNotification(name)

	if err != nil {
		t.Error(err)
	}
}

func TestCreateNotificationCommandDNE(t *testing.T) {

	hostname := "host"
	notificationname := hostname + "!" + hostname
	command := "invalid-command"
	servicename := ""
	interval := 1800

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, nil, nil, nil)

	if !strings.Contains(err.Error(), "500 Object could not be created") {
		t.Error(err)
	}
}

// func TestCreateNotificationHostDNE
// Try and create a notification for a host that does not exist.
// Should fail with an error about the host not existing.
func TestCreateNotificationHostDNE(t *testing.T) {

	hostname := "host-dne"
	notificationname := hostname + "!" + hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, nil, nil, nil)

	if !strings.Contains(err.Error(), "500 Object could not be created") {
		t.Error(err)
	}
}

func TestCreateNotificationInvalidName(t *testing.T) {

	hostname := "host-dne"
	notificationname := "invalid-name"
	command := "mail-host-notification"
	servicename := ""
	interval := 1800

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, nil, nil, nil)

	if !strings.Contains(err.Error(), "500 Object could not be created") {
		t.Error(err)
	}
}

func TestCreateNotificationUserDNE(t *testing.T) {

	hostname := "host"
	group := []string{"linux-servers"}
	notificationname := hostname + "!" + hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800
	users := []string{"user-dne"}

	_, _ = Icinga2_Server.CreateHost(hostname, "127.0.0.1", "hostalive", nil, nil, group)
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, users, nil, nil)

	if !strings.Contains(err.Error(), "500 Object could not be created") {
		t.Error(err)
	}
}

func TestCreateHostNotification(t *testing.T) {

	hostname := "host"
	groups := []string{"linux-servers"}
	notificationname := hostname + "!" + hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800
	username := "user"

	_, _ = Icinga2_Server.CreateHost(hostname, "127.0.0.1", "hostalive", nil, nil, groups)
	_, _ = Icinga2_Server.CreateUser(username, "user@example.com")
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if err != nil {
		t.Error(err)
	}
}

func TestCreateHostNotificationAlreadyExists(t *testing.T) {

	hostname := "host"
	notificationname := hostname + "!" + hostname
	command := "mail-host-notification"
	servicename := ""
	interval := 1800
	username := "user"

	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if !strings.HasSuffix(err.Error(), "500 Object could not be created") {
		t.Error(err)
	}
}

func TestCreateServiceNotification(t *testing.T) {

	hostname := "host"
	servicename := "test"
	check_command := "random"
	notificationname := hostname + "!" + servicename + "!" + hostname + "-" + servicename
	command := "mail-service-notification"
	interval := 1800
	username := "user"

	_, _ = Icinga2_Server.CreateUser(username, "user@example.com")
	_, _ = Icinga2_Server.CreateService(servicename, hostname, check_command, nil, nil)
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if err != nil {
		t.Error(err)
	}
}

func TestCreateServiceNotificationAlreadyExists(t *testing.T) {

	hostname := "host"
	servicename := "test"
	check_command := "random"
	notificationname := hostname + "!" + servicename + "!" + hostname + "-" + servicename
	command := "mail-service-notification"
	interval := 1800
	username := "user"

	_, _ = Icinga2_Server.CreateUser(username, "user@example.com")
	_, _ = Icinga2_Server.CreateService(servicename, hostname, check_command, nil, nil)
	_, err := Icinga2_Server.CreateNotification(notificationname, hostname, command, servicename, interval, []string{username}, nil, nil)

	if !strings.HasSuffix(err.Error(), "500 Object could not be created") {
		t.Error(err)
	}
}

func TestDeleteServiceNotification(t *testing.T) {

	hostname := "host"
	servicename := "test"
	notificationname := hostname + "!" + servicename + "!" + hostname + "-" + servicename

	err := Icinga2_Server.DeleteNotification(notificationname)

	if err != nil {
		t.Error(err)
	}
}

func TestDeleteServiceNotificationDNE(t *testing.T) {

	hostname := "host"
	servicename := "test"
	notificationname := hostname + "!" + servicename + "!" + hostname + "-" + servicename

	err := Icinga2_Server.DeleteNotification(notificationname)

	if err.Error() != "No objects found." {
		t.Error(err)
	}
}

func TestDeleteHostNotification(t *testing.T) {

	hostname := "host"
	notificationname := hostname + "!" + hostname
	username := "user"

	err := Icinga2_Server.DeleteNotification(notificationname)
	_ = Icinga2_Server.DeleteHost(hostname)
	_ = Icinga2_Server.DeleteUser(username)

	if err != nil {
		t.Error(err)
	}
}

func TestDeleteHostNotificationDNE(t *testing.T) {

	hostname := "host"
	notificationname := hostname + "!" + hostname

	err := Icinga2_Server.DeleteNotification(notificationname)

	if err.Error() != "No objects found." {
		t.Error(err)
	}
}

// func TestDeleteNotificationNonAPI
// Notifications that were not created via the API, cannot be deleted via the API
// Should get an error about not being created via the API
func TestDeleteNotificationNonAPI(t *testing.T) {

	notificationname := "mail-icingaadmin"

	err := Icinga2_Server.DeleteNotification(notificationname)
	if err.Error() != "No objects found." {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package symlink

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type SL struct {
	dst  string
	path string
}

var osMkdirAll = os.MkdirAll

func New(link, dst string) (*SL, error) {
	var err error
	if dst, err = dirAbsPath(dst); err != nil {
		return nil, err
	}
	exist, _, err := dirExists(dst)
	msgerr := ""
	if err != nil {
		msgerr = fmt.Sprintf("\rError: '%+v'", err)
	}
	if !exist {
		return nil, fmt.Errorf("Unknown destination '%s'%s", dst, msgerr)
	}

	if link, err = dirAbsPath(link); err != nil {
		return nil, err
	}
	linkdir := filepath.Dir(filepath.Dir(link)) + string(filepath.Separator)
	fmt.Printf("link='%+v\nlinkdir=%+v\ndst=%+v\n", link, linkdir, dst)
	var hasLinkDir, hasLink bool
	var linkDirTarget, linkTarget string
	if hasLinkDir, linkDirTarget, err = dirExists(linkdir); err != nil {
		if strings.Contains(err.Error(), "The system cannot find the") == false {
			return nil, fmt.Errorf("Impossible to check/access link parent folder '%s':\n'%+v'", linkdir, err)
		}
	}
	if linkDirTarget != "" {
		// move folder to x.1 (or error)
		if err = moveToDotX(linkdir); err != nil {
			return nil, err
		}
		hasLinkDir = false
	}
	if !hasLinkDir {
		if err = osMkdirAll(linkdir, os.ModeDir); err != nil {
			return nil, fmt.Errorf("Impossible to create link parent folder '%s':\n'%+v'", linkdir, err)
		}
	}
	// fmt.Printf("==== link '%s'\n", link)
	if hasLink, linkTarget, err = dirExists(link); err != nil {
		if strings.Contains(err.Error(), "The system cannot find the") == false {
			return nil, fmt.Errorf("Impossible to check/access link'%s':\n'%+v'", link, err)
		}
	}
	if linkTarget != "" {
		// check if points already to dst. If not move or error
	} else if hasLink {
		// move folder to xx.1
	}
	if _, err = execcmd("mklink", fmt.Sprintf("/J %s %s", link, dst), linkdir); err != nil {
		return nil, fmt.Errorf("Impossible to create junction between '%s' and '%s':\n'%+v'", link, dst, err)
	}
	res := &SL{path: link, dst: dst}
	return res, nil
}

var osRename = os.Rename

func moveToDotX(path string) error {
	path = filepath.Dir(path)
	i := 1
	newpath := fmt.Sprintf("%s.%d", path, i)
	for {

		exist, _, err := dirExists(newpath)
		if err != nil {
			if strings.Contains(err.Error(), "The system cannot find the") == false {
				return err
			}
		}
		if !exist {
			break
		}
		i = i + 1
		newpath = fmt.Sprintf("%s.%d", path, i)
	}
	fmt.Printf("Move '%s' to '%s'\n", path, newpath)
	if err := osRename(path, newpath); err != nil {
		return err
	}
	return nil
}

// a/b => c:\path\to\a\b\
func dirAbsPath(path string) (string, error) {
	path = filepath.FromSlash(path)
	var err error
	path, err = filepath.Abs(path)
	if err != nil {
		return "", err
	}
	sep := string(filepath.Separator)
	if strings.HasSuffix(path, sep) == false {
		path = path + sep
	}
	return path, nil
}

var osStat = os.Stat

func dirExists(path string) (bool, string, error) {
	fi, err := osStat(path)
	// fmt.Printf("==== fi='%+v', err='%+v'\n", fi, err)
	if fi == nil {
		return false, "", err
	}
	// sys := fi.Sys().(*syscall.Win32FileAttributeData)
	if err == nil {
		return true, "", nil
	}
	if strings.HasPrefix(err.Error(), "readlink ") == false {
		return false, "", err
	}
	// This is a symlink (JUNCTION on Windows)
	dir := filepath.Dir(path)
	base := filepath.Base(dir)
	dir = filepath.Dir(dir)
	sdir := ""
	if sdir, err = execcmd("dir", ".", dir); err != nil {
		return false, "", err
	}
	r := regexp.MustCompile(fmt.Sprintf(`(?m)<J[UO]NCTION>\s+%s\s+\[([^\]]+)\]\s*$`, base))
	n := r.FindAllStringSubmatch(sdir, -1)
	// fmt.Printf("n='%+v'\nr='%+v'\n", n, r)
	if len(n) == 1 {
		return true, n[0][1], nil
	}
	return false, "", fmt.Errorf("Unable to find junction symlink in parent dir '%s' for '%s'", dir, base)
}

func cmdRun(cmd *exec.Cmd) error {
	return cmd.Run()
}

var execRun = cmdRun

func execcmd(exe, cmd string, dir string) (string, error) {
	args := strings.Split(cmd, " ")
	args = append([]string{"/c", exe}, args...)
	c := exec.Command("cmd", args...)
	c.Dir = dir
	var bout bytes.Buffer
	c.Stdout = &bout
	var berr bytes.Buffer
	c.Stderr = &berr
	err := execRun(c)
	if err != nil {
		return bout.String(), fmt.Errorf("Unable to run '%s %s' in '%s': err '%s'\n'%s'", exe, cmd, dir, err.Error(), berr.String())
	} else if berr.String() != "" {
		return bout.String(), fmt.Errorf("Warning on run '%s %s' in '%s': '%s'", exe, cmd, dir, berr.String())
	}
	return bout.String(), nil
}
<commit_msg>symlink.go: implement existing link cases<commit_after>package symlink

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type SL struct {
	dst  string
	path string
}

var osMkdirAll = os.MkdirAll

func New(link, dst string) (*SL, error) {
	var err error
	if dst, err = dirAbsPath(dst); err != nil {
		return nil, err
	}
	exist, _, err := dirExists(dst)
	msgerr := ""
	if err != nil {
		msgerr = fmt.Sprintf("\rError: '%+v'", err)
	}
	if !exist {
		return nil, fmt.Errorf("Unknown destination '%s'%s", dst, msgerr)
	}

	if link, err = dirAbsPath(link); err != nil {
		return nil, err
	}
	linkdir := filepath.Dir(filepath.Dir(link)) + string(filepath.Separator)
	fmt.Printf("link='%+v\nlinkdir=%+v\ndst=%+v\n", link, linkdir, dst)
	var hasLinkDir, hasLink bool
	var linkDirTarget, linkTarget string
	if hasLinkDir, linkDirTarget, err = dirExists(linkdir); err != nil {
		if strings.Contains(err.Error(), "The system cannot find the") == false {
			return nil, fmt.Errorf("Impossible to check/access link parent folder '%s':\n'%+v'", linkdir, err)
		}
	}
	if linkDirTarget != "" {
		// move folder to x.1 (or error)
		if err = moveToDotX(linkdir); err != nil {
			return nil, err
		}
		hasLinkDir = false
	}
	if !hasLinkDir {
		if err = osMkdirAll(linkdir, os.ModeDir); err != nil {
			return nil, fmt.Errorf("Impossible to create link parent folder '%s':\n'%+v'", linkdir, err)
		}
	}
	// fmt.Printf("==== link '%s'\n", link)
	if hasLink, linkTarget, err = dirExists(link); err != nil {
		if strings.Contains(err.Error(), "The system cannot find the") == false {
			return nil, fmt.Errorf("Impossible to check/access link'%s':\n'%+v'", link, err)
		}
	}
	if linkTarget == dst {
		return &SL{path: link, dst: dst}, nil
	}
	if hasLink {
		if err = moveToDotX(link); err != nil {
			return nil, err
		}
	}
	if _, err = execcmd("mklink", fmt.Sprintf("/J %s %s", link, dst), linkdir); err != nil {
		return nil, fmt.Errorf("Impossible to create junction between '%s' and '%s':\n'%+v'", link, dst, err)
	}
	res := &SL{path: link, dst: dst}
	return res, nil
}

var osRename = os.Rename

func moveToDotX(path string) error {
	path = filepath.Dir(path)
	i := 1
	newpath := fmt.Sprintf("%s.%d", path, i)
	for {

		exist, _, err := dirExists(newpath)
		if err != nil {
			if strings.Contains(err.Error(), "The system cannot find the") == false {
				return err
			}
		}
		if !exist {
			break
		}
		i = i + 1
		newpath = fmt.Sprintf("%s.%d", path, i)
	}
	fmt.Printf("Move '%s' to '%s'\n", path, newpath)
	if err := osRename(path, newpath); err != nil {
		return err
	}
	return nil
}

// a/b => c:\path\to\a\b\
func dirAbsPath(path string) (string, error) {
	path = filepath.FromSlash(path)
	var err error
	path, err = filepath.Abs(path)
	if err != nil {
		return "", err
	}
	sep := string(filepath.Separator)
	if strings.HasSuffix(path, sep) == false {
		path = path + sep
	}
	return path, nil
}

var osStat = os.Stat

func dirExists(path string) (bool, string, error) {
	fi, err := osStat(path)
	// fmt.Printf("==== fi='%+v', err='%+v'\n", fi, err)
	if fi == nil {
		return false, "", err
	}
	// sys := fi.Sys().(*syscall.Win32FileAttributeData)
	if err == nil {
		return true, "", nil
	}
	if strings.HasPrefix(err.Error(), "readlink ") == false {
		return false, "", err
	}
	// This is a symlink (JUNCTION on Windows)
	dir := filepath.Dir(path)
	base := filepath.Base(dir)
	dir = filepath.Dir(dir)
	sdir := ""
	if sdir, err = execcmd("dir", ".", dir); err != nil {
		return false, "", err
	}
	r := regexp.MustCompile(fmt.Sprintf(`(?m)<J[UO]NCTION>\s+%s\s+\[([^\]]+)\]\s*$`, base))
	n := r.FindAllStringSubmatch(sdir, -1)
	// fmt.Printf("n='%+v'\nr='%+v'\n", n, r)
	if len(n) == 1 {
		return true, n[0][1], nil
	}
	return false, "", fmt.Errorf("Unable to find junction symlink in parent dir '%s' for '%s'", dir, base)
}

func cmdRun(cmd *exec.Cmd) error {
	return cmd.Run()
}

var execRun = cmdRun

func execcmd(exe, cmd string, dir string) (string, error) {
	args := strings.Split(cmd, " ")
	args = append([]string{"/c", exe}, args...)
	c := exec.Command("cmd", args...)
	c.Dir = dir
	var bout bytes.Buffer
	c.Stdout = &bout
	var berr bytes.Buffer
	c.Stderr = &berr
	err := execRun(c)
	if err != nil {
		return bout.String(), fmt.Errorf("Unable to run '%s %s' in '%s': err '%s'\n'%s'", exe, cmd, dir, err.Error(), berr.String())
	} else if berr.String() != "" {
		return bout.String(), fmt.Errorf("Warning on run '%s %s' in '%s': '%s'", exe, cmd, dir, berr.String())
	}
	return bout.String(), nil
}
<|endoftext|>


<commit_before>package raft_logstore

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"

	"github.com/hashicorp/raft"
)

// trivial log store, writing one entry into one file each.
// fulfills the raft.LogStore interface.
type RobustLogStore struct {
	l         sync.RWMutex
	lowIndex  uint64
	highIndex uint64
	dir       string
}

type uint64Slice []uint64

func (p uint64Slice) Len() int           { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
func (p uint64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func NewRobustLogStore(dir string) (*RobustLogStore, error) {
	if err := os.MkdirAll(filepath.Join(dir, "robustlogs"), 0700); err != nil {
		return nil, err
	}

	return &RobustLogStore{
		dir: dir,
	}, nil
}

// GetAll returns all indexes that are currently present in the log store. This
// is NOT part of the raft.LogStore interface — we use it when snapshotting.
func (s *RobustLogStore) GetAll() ([]uint64, error) {
	var indexes []uint64
	dir, err := os.Open(filepath.Join(s.dir, "robustlogs"))
	if err != nil {
		return indexes, err
	}
	defer dir.Close()

	names, err := dir.Readdirnames(-1)
	if err != nil {
		return indexes, err
	}

	for _, name := range names {
		if !strings.HasPrefix(name, "entry.") {
			continue
		}

		dot := strings.LastIndex(name, ".")
		if dot == -1 {
			continue
		}

		index, err := strconv.ParseInt(name[dot+1:], 0, 64)
		if err != nil {
			return indexes, fmt.Errorf("Unexpected filename, does not confirm to entry.%%d: %q. Parse error: %v", name, err)
		}

		indexes = append(indexes, uint64(index))
	}

	sort.Sort(uint64Slice(indexes))

	return indexes, nil
}

func (s *RobustLogStore) FirstIndex() (uint64, error) {
	s.l.RLock()
	defer s.l.RUnlock()
	return s.lowIndex, nil
}

func (s *RobustLogStore) LastIndex() (uint64, error) {
	s.l.RLock()
	defer s.l.RUnlock()
	return s.highIndex, nil
}

func (s *RobustLogStore) GetLog(index uint64, rlog *raft.Log) error {
	s.l.Lock()
	defer s.l.Unlock()
	f, err := os.Open(filepath.Join(s.dir, fmt.Sprintf("robustlogs/entry.%d", index)))
	if err != nil {
		if os.IsNotExist(err) {
			return raft.ErrLogNotFound
		}
		return err
	}
	defer f.Close()

	var elog raft.Log
	if err := json.NewDecoder(f).Decode(&elog); err != nil {
		return err
	}
	*rlog = elog
	return nil
}

func (s *RobustLogStore) StoreLog(log *raft.Log) error {
	return s.StoreLogs([]*raft.Log{log})
}

func (s *RobustLogStore) StoreLogs(logs []*raft.Log) error {
	s.l.Lock()
	defer s.l.Unlock()

	for _, entry := range logs {
		log.Printf("writing index %d to file\n", entry.Index)
		suffix := strconv.FormatUint(entry.Index, 10)
		f, err := os.Create(filepath.Join(s.dir, "robustlogs/incomplete."+suffix))
		if err != nil {
			return err
		}
		if err := json.NewEncoder(f).Encode(entry); err != nil {
			f.Close()
			return err
		}
		if entry.Index < s.lowIndex || s.lowIndex == 0 {
			s.lowIndex = entry.Index
		}
		if entry.Index > s.highIndex {
			s.highIndex = entry.Index
		}
		f.Close()
		if err := os.Rename(
			filepath.Join(s.dir, "robustlogs/incomplete."+suffix),
			filepath.Join(s.dir, "robustlogs/entry."+suffix)); err != nil {
			return err
		}
	}

	return nil
}

func (s *RobustLogStore) DeleteRange(min, max uint64) error {
	s.l.Lock()
	defer s.l.Unlock()
	for i := min; i <= max; i++ {
		log.Printf("deleting index %d\n", i)
		if err := os.Remove(filepath.Join(s.dir, fmt.Sprintf("robustlogs/entry.%d", i))); err != nil {
			return err
		}
	}
	s.lowIndex = max + 1
	return nil
}

func (s *RobustLogStore) DeleteAll() error {
	s.l.Lock()
	defer s.l.Unlock()
	if err := os.RemoveAll(filepath.Join(s.dir, "robustlogs")); err != nil {
		return err
	}
	if err := os.MkdirAll(filepath.Join(s.dir, "robustlogs"), 0700); err != nil {
		return err
	}
	s.lowIndex = 0
	s.highIndex = 0
	return nil
}
<commit_msg>simplify raft_logstore: directly decode into output variable<commit_after>package raft_logstore

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"

	"github.com/hashicorp/raft"
)

// trivial log store, writing one entry into one file each.
// fulfills the raft.LogStore interface.
type RobustLogStore struct {
	l         sync.RWMutex
	lowIndex  uint64
	highIndex uint64
	dir       string
}

type uint64Slice []uint64

func (p uint64Slice) Len() int           { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
func (p uint64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func NewRobustLogStore(dir string) (*RobustLogStore, error) {
	if err := os.MkdirAll(filepath.Join(dir, "robustlogs"), 0700); err != nil {
		return nil, err
	}

	return &RobustLogStore{
		dir: dir,
	}, nil
}

// GetAll returns all indexes that are currently present in the log store. This
// is NOT part of the raft.LogStore interface — we use it when snapshotting.
func (s *RobustLogStore) GetAll() ([]uint64, error) {
	var indexes []uint64
	dir, err := os.Open(filepath.Join(s.dir, "robustlogs"))
	if err != nil {
		return indexes, err
	}
	defer dir.Close()

	names, err := dir.Readdirnames(-1)
	if err != nil {
		return indexes, err
	}

	for _, name := range names {
		if !strings.HasPrefix(name, "entry.") {
			continue
		}

		dot := strings.LastIndex(name, ".")
		if dot == -1 {
			continue
		}

		index, err := strconv.ParseInt(name[dot+1:], 0, 64)
		if err != nil {
			return indexes, fmt.Errorf("Unexpected filename, does not confirm to entry.%%d: %q. Parse error: %v", name, err)
		}

		indexes = append(indexes, uint64(index))
	}

	sort.Sort(uint64Slice(indexes))

	return indexes, nil
}

func (s *RobustLogStore) FirstIndex() (uint64, error) {
	s.l.RLock()
	defer s.l.RUnlock()
	return s.lowIndex, nil
}

func (s *RobustLogStore) LastIndex() (uint64, error) {
	s.l.RLock()
	defer s.l.RUnlock()
	return s.highIndex, nil
}

func (s *RobustLogStore) GetLog(index uint64, rlog *raft.Log) error {
	s.l.Lock()
	defer s.l.Unlock()
	f, err := os.Open(filepath.Join(s.dir, fmt.Sprintf("robustlogs/entry.%d", index)))
	if err != nil {
		if os.IsNotExist(err) {
			return raft.ErrLogNotFound
		}
		return err
	}
	defer f.Close()
	return json.NewDecoder(f).Decode(rlog)
}

func (s *RobustLogStore) StoreLog(log *raft.Log) error {
	return s.StoreLogs([]*raft.Log{log})
}

func (s *RobustLogStore) StoreLogs(logs []*raft.Log) error {
	s.l.Lock()
	defer s.l.Unlock()

	for _, entry := range logs {
		log.Printf("writing index %d to file\n", entry.Index)
		suffix := strconv.FormatUint(entry.Index, 10)
		f, err := os.Create(filepath.Join(s.dir, "robustlogs/incomplete."+suffix))
		if err != nil {
			return err
		}
		if err := json.NewEncoder(f).Encode(entry); err != nil {
			f.Close()
			return err
		}
		if entry.Index < s.lowIndex || s.lowIndex == 0 {
			s.lowIndex = entry.Index
		}
		if entry.Index > s.highIndex {
			s.highIndex = entry.Index
		}
		f.Close()
		if err := os.Rename(
			filepath.Join(s.dir, "robustlogs/incomplete."+suffix),
			filepath.Join(s.dir, "robustlogs/entry."+suffix)); err != nil {
			return err
		}
	}

	return nil
}

func (s *RobustLogStore) DeleteRange(min, max uint64) error {
	s.l.Lock()
	defer s.l.Unlock()
	for i := min; i <= max; i++ {
		log.Printf("deleting index %d\n", i)
		if err := os.Remove(filepath.Join(s.dir, fmt.Sprintf("robustlogs/entry.%d", i))); err != nil {
			return err
		}
	}
	s.lowIndex = max + 1
	return nil
}

func (s *RobustLogStore) DeleteAll() error {
	s.l.Lock()
	defer s.l.Unlock()
	if err := os.RemoveAll(filepath.Join(s.dir, "robustlogs")); err != nil {
		return err
	}
	if err := os.MkdirAll(filepath.Join(s.dir, "robustlogs"), 0700); err != nil {
		return err
	}
	s.lowIndex = 0
	s.highIndex = 0
	return nil
}
<|endoftext|>


<commit_before>package es

import (
	"fmt"
	"log"
	"os"
	"time"
)

var timer = time.NewTimer(1 * time.Second)

type IndexerStats struct {
	Runs        int64
	IndexedDocs int64
	TotalTime   time.Duration
	started     time.Time
}

func (i *IndexerStats) String() string {
	diff := time.Since(i.started)
	perSecond := float64(i.IndexedDocs) / diff.Seconds()
	return fmt.Sprintf("docs=%d %.01f/second", i.IndexedDocs, perSecond)
}

func (stats *IndexerStats) Add(count int, dur time.Duration) {
	if stats.started.IsZero() {
		stats.started = time.Now()
	}
	stats.Runs++
	stats.IndexedDocs += int64(count)
	stats.TotalTime += dur
}

type Indexer struct {
	Index *Index

	IndexEvery time.Duration // triggers a new index run after that duration, will be reset when BatchSize reached
	BatchSize  int           // triggers a new index run when the batch reaches that size

	docsBatch   []*Doc
	docsChannel chan *Doc
	timer       *time.Timer
	Stats       *IndexerStats
}

func (indexer *Indexer) Finish() error {
	return indexer.indexBatch()
}

func (indexer *Indexer) resetBatch() {
	indexer.docsBatch = make([]*Doc, 0, indexer.BatchSize)
}

func (indexer *Indexer) resetTimer() bool {
	return indexer.timer.Reset(indexer.IndexEvery)
}

var logger = log.New(os.Stderr, "", 0)

// TODO change handling to e.g. Add command with implizit opening of channel
// to not expose the channel to the public
func (indexer *Indexer) Start() chan *Doc {
	if indexer.BatchSize == 0 {
		indexer.BatchSize = 100
	}
	if indexer.IndexEvery == 0 {
		indexer.IndexEvery = 1 * time.Hour
	}
	indexer.timer = time.NewTimer(indexer.IndexEvery)
	indexer.docsChannel = make(chan *Doc, indexer.BatchSize)
	indexer.resetBatch()
	go func(indexer *Indexer) {
		for {
			select {
			case <-indexer.timer.C:
				e := indexer.indexBatch()
				if e != nil {
					logger.Printf("ERROR=%q", e)
				}
				indexer.resetTimer()
			case doc, ok := <-indexer.docsChannel:
				if !ok {
					indexer.timer.Stop()
					e := indexer.indexBatch()
					if e != nil {
						logger.Printf("ERROR=%q", e)
					}
					return
				}
				if doc.Index == "" {
					doc.Index = indexer.Index.Index
				}
				if doc.Type == "" {
					doc.Type = indexer.Index.Type
				}
				indexer.docsBatch = append(indexer.docsBatch, doc)
				if len(indexer.docsBatch) >= indexer.BatchSize {
					indexer.timer.Stop()
					e := indexer.indexBatch()
					if e != nil {
						logger.Printf("ERROR: %s", e)
					}
					indexer.resetTimer()
				}
			}
		}
	}(indexer)
	return indexer.docsChannel
}

func (indexer *Indexer) Close() error {
	if indexer.docsChannel != nil {
		close(indexer.docsChannel)
	}
	return indexer.indexBatch()
}

func (indexer *Indexer) indexBatch() error {
	if len(indexer.docsBatch) < 1 {
		return nil
	}
	started := time.Now()
	e := indexer.Index.IndexDocs(indexer.docsBatch)
	if e != nil {
		return e
	}
	indexer.Stats.Add(len(indexer.docsBatch), time.Now().Sub(started))
	indexer.resetBatch()
	return nil
}
<commit_msg>fix Stats line<commit_after>package es

import (
	"fmt"
	"log"
	"os"
	"time"
)

var timer = time.NewTimer(1 * time.Second)

type IndexerStats struct {
	Runs        int64
	IndexedDocs int64
	TotalTime   time.Duration
	started     time.Time
}

func (i *IndexerStats) String() string {
	diff := time.Since(i.started)
	perSecond := float64(i.IndexedDocs) / diff.Seconds()
	return fmt.Sprintf("docs=%d %.01f/second", i.IndexedDocs, perSecond)
}

func (stats *IndexerStats) Add(count int, dur time.Duration) {
	if stats == nil {
		panic("stats can not be nil")
	}
	if stats.started.IsZero() {
		stats.started = time.Now()
	}
	stats.Runs++
	stats.IndexedDocs += int64(count)
	stats.TotalTime += dur
}

type Indexer struct {
	Index *Index

	IndexEvery time.Duration // triggers a new index run after that duration, will be reset when BatchSize reached
	BatchSize  int           // triggers a new index run when the batch reaches that size

	docsBatch   []*Doc
	docsChannel chan *Doc
	timer       *time.Timer
	Stats       *IndexerStats
}

func (indexer *Indexer) Finish() error {
	return indexer.indexBatch()
}

func (indexer *Indexer) resetBatch() {
	indexer.docsBatch = make([]*Doc, 0, indexer.BatchSize)
}

func (indexer *Indexer) resetTimer() bool {
	return indexer.timer.Reset(indexer.IndexEvery)
}

var logger = log.New(os.Stderr, "", 0)

// TODO change handling to e.g. Add command with implizit opening of channel
// to not expose the channel to the public
func (indexer *Indexer) Start() chan *Doc {
	if indexer.BatchSize == 0 {
		indexer.BatchSize = 100
	}
	if indexer.IndexEvery == 0 {
		indexer.IndexEvery = 1 * time.Hour
	}
	indexer.timer = time.NewTimer(indexer.IndexEvery)
	indexer.docsChannel = make(chan *Doc, indexer.BatchSize)
	indexer.resetBatch()
	go func(indexer *Indexer) {
		for {
			select {
			case <-indexer.timer.C:
				e := indexer.indexBatch()
				if e != nil {
					logger.Printf("ERROR=%q", e)
				}
				indexer.resetTimer()
			case doc, ok := <-indexer.docsChannel:
				if !ok {
					indexer.timer.Stop()
					e := indexer.indexBatch()
					if e != nil {
						logger.Printf("ERROR=%q", e)
					}
					return
				}
				if doc.Index == "" {
					doc.Index = indexer.Index.Index
				}
				if doc.Type == "" {
					doc.Type = indexer.Index.Type
				}
				indexer.docsBatch = append(indexer.docsBatch, doc)
				if len(indexer.docsBatch) >= indexer.BatchSize {
					indexer.timer.Stop()
					e := indexer.indexBatch()
					if e != nil {
						logger.Printf("ERROR: %s", e)
					}
					indexer.resetTimer()
				}
			}
		}
	}(indexer)
	return indexer.docsChannel
}

func (indexer *Indexer) Close() error {
	if indexer.docsChannel != nil {
		close(indexer.docsChannel)
	}
	return indexer.indexBatch()
}

func (indexer *Indexer) indexBatch() error {
	if len(indexer.docsBatch) < 1 {
		return nil
	}
	started := time.Now()
	e := indexer.Index.IndexDocs(indexer.docsBatch)
	if e != nil {
		return e
	}
	if indexer.Stats == nil {
		indexer.Stats = &IndexerStats{}
	}
	indexer.Stats.Add(len(indexer.docsBatch), time.Now().Sub(started))
	indexer.resetBatch()
	return nil
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

// Package archive provides a Docker and OCI compatible importer
package archive

import (
	"archive/tar"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"path"

	"github.com/containerd/containerd/archive/compression"
	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/log"
	digest "github.com/opencontainers/go-digest"
	specs "github.com/opencontainers/image-spec/specs-go"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

type importOpts struct {
	compress bool
}

// ImportOpt is an option for importing an OCI index
type ImportOpt func(*importOpts) error

// WithImportCompression compresses uncompressed layers on import.
// This is used for import formats which do not include the manifest.
func WithImportCompression() ImportOpt {
	return func(io *importOpts) error {
		io.compress = true
		return nil
	}
}

// ImportIndex imports an index from a tar archive image bundle
// - implements Docker v1.1, v1.2 and OCI v1.
// - prefers OCI v1 when provided
// - creates OCI index for Docker formats
// - normalizes Docker references and adds as OCI ref name
//      e.g. alpine:latest -> docker.io/library/alpine:latest
// - existing OCI reference names are untouched
func ImportIndex(ctx context.Context, store content.Store, reader io.Reader, opts ...ImportOpt) (ocispec.Descriptor, error) {
	var (
		tr = tar.NewReader(reader)

		ociLayout ocispec.ImageLayout
		mfsts     []struct {
			Config   string
			RepoTags []string
			Layers   []string
		}
		symlinks = make(map[string]string)
		blobs    = make(map[string]ocispec.Descriptor)
		iopts    importOpts
	)

	for _, o := range opts {
		if err := o(&iopts); err != nil {
			return ocispec.Descriptor{}, err
		}
	}

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return ocispec.Descriptor{}, err
		}
		if hdr.Typeflag == tar.TypeSymlink {
			symlinks[hdr.Name] = path.Join(path.Dir(hdr.Name), hdr.Linkname)
		}

		if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
			if hdr.Typeflag != tar.TypeDir {
				log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored")
			}
			continue
		}

		hdrName := path.Clean(hdr.Name)
		if hdrName == ocispec.ImageLayoutFile {
			if err = onUntarJSON(tr, &ociLayout); err != nil {
				return ocispec.Descriptor{}, errors.Wrapf(err, "untar oci layout %q", hdr.Name)
			}
		} else if hdrName == "manifest.json" {
			if err = onUntarJSON(tr, &mfsts); err != nil {
				return ocispec.Descriptor{}, errors.Wrapf(err, "untar manifest %q", hdr.Name)
			}
		} else {
			dgst, err := onUntarBlob(ctx, tr, store, hdr.Size, "tar-"+hdrName)
			if err != nil {
				return ocispec.Descriptor{}, errors.Wrapf(err, "failed to ingest %q", hdr.Name)
			}

			blobs[hdrName] = ocispec.Descriptor{
				Digest: dgst,
				Size:   hdr.Size,
			}
		}
	}

	// If OCI layout was given, interpret the tar as an OCI layout.
	// When not provided, the layout of the tar will be interpreted
	// as Docker v1.1 or v1.2.
	if ociLayout.Version != "" {
		if ociLayout.Version != ocispec.ImageLayoutVersion {
			return ocispec.Descriptor{}, errors.Errorf("unsupported OCI version %s", ociLayout.Version)
		}

		idx, ok := blobs["index.json"]
		if !ok {
			return ocispec.Descriptor{}, errors.Errorf("missing index.json in OCI layout %s", ocispec.ImageLayoutVersion)
		}

		idx.MediaType = ocispec.MediaTypeImageIndex
		return idx, nil
	}

	if mfsts == nil {
		return ocispec.Descriptor{}, errors.Errorf("unrecognized image format")
	}

	for name, linkname := range symlinks {
		desc, ok := blobs[linkname]
		if !ok {
			return ocispec.Descriptor{}, errors.Errorf("no target for symlink layer from %q to %q", name, linkname)
		}
		blobs[name] = desc
	}

	idx := ocispec.Index{
		Versioned: specs.Versioned{
			SchemaVersion: 2,
		},
	}
	for _, mfst := range mfsts {
		config, ok := blobs[mfst.Config]
		if !ok {
			return ocispec.Descriptor{}, errors.Errorf("image config %q not found", mfst.Config)
		}
		config.MediaType = images.MediaTypeDockerSchema2Config

		layers, err := resolveLayers(ctx, store, mfst.Layers, blobs, iopts.compress)
		if err != nil {
			return ocispec.Descriptor{}, errors.Wrap(err, "failed to resolve layers")
		}

		manifest := struct {
			SchemaVersion int                  `json:"schemaVersion"`
			MediaType     string               `json:"mediaType"`
			Config        ocispec.Descriptor   `json:"config"`
			Layers        []ocispec.Descriptor `json:"layers"`
		}{
			SchemaVersion: 2,
			MediaType:     images.MediaTypeDockerSchema2Manifest,
			Config:        config,
			Layers:        layers,
		}

		desc, err := writeManifest(ctx, store, manifest, manifest.MediaType)
		if err != nil {
			return ocispec.Descriptor{}, errors.Wrap(err, "write docker manifest")
		}

		platforms, err := images.Platforms(ctx, store, desc)
		if err != nil {
			return ocispec.Descriptor{}, errors.Wrap(err, "unable to resolve platform")
		}
		if len(platforms) > 0 {
			// Only one platform can be resolved from non-index manifest,
			// The platform can only come from the config included above,
			// if the config has no platform it can be safely omitted.
			desc.Platform = &platforms[0]
		}

		if len(mfst.RepoTags) == 0 {
			idx.Manifests = append(idx.Manifests, desc)
		} else {
			// Add descriptor per tag
			for _, ref := range mfst.RepoTags {
				mfstdesc := desc

				normalized, err := normalizeReference(ref)
				if err != nil {
					return ocispec.Descriptor{}, err
				}

				mfstdesc.Annotations = map[string]string{
					images.AnnotationImageName: normalized,
					ocispec.AnnotationRefName:  ociReferenceName(normalized),
				}

				idx.Manifests = append(idx.Manifests, mfstdesc)
			}
		}
	}

	return writeManifest(ctx, store, idx, ocispec.MediaTypeImageIndex)
}

func onUntarJSON(r io.Reader, j interface{}) error {
	b, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}
	return json.Unmarshal(b, j)
}

func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, size int64, ref string) (digest.Digest, error) {
	dgstr := digest.Canonical.Digester()

	if err := content.WriteBlob(ctx, store, ref, io.TeeReader(r, dgstr.Hash()), ocispec.Descriptor{Size: size}); err != nil {
		return "", err
	}

	return dgstr.Digest(), nil
}

func resolveLayers(ctx context.Context, store content.Store, layerFiles []string, blobs map[string]ocispec.Descriptor, compress bool) ([]ocispec.Descriptor, error) {
	layers := make([]ocispec.Descriptor, len(layerFiles))
	descs := map[digest.Digest]*ocispec.Descriptor{}
	filters := []string{}
	for i, f := range layerFiles {
		desc, ok := blobs[f]
		if !ok {
			return nil, errors.Errorf("layer %q not found", f)
		}
		layers[i] = desc
		descs[desc.Digest] = &layers[i]
		filters = append(filters, "labels.\"containerd.io/uncompressed\"=="+desc.Digest.String())
	}

	err := store.Walk(ctx, func(info content.Info) error {
		dgst, ok := info.Labels["containerd.io/uncompressed"]
		if ok {
			desc := descs[digest.Digest(dgst)]
			if desc != nil {
				desc.MediaType = images.MediaTypeDockerSchema2LayerGzip
				desc.Digest = info.Digest
				desc.Size = info.Size
			}
		}
		return nil
	}, filters...)
	if err != nil {
		return nil, errors.Wrap(err, "failure checking for compressed blobs")
	}

	for i, desc := range layers {
		if desc.MediaType != "" {
			continue
		}
		// Open blob, resolve media type
		ra, err := store.ReaderAt(ctx, desc)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to open %q (%s)", layerFiles[i], desc.Digest)
		}
		s, err := compression.DecompressStream(content.NewReader(ra))
		if err != nil {
			ra.Close()
			return nil, errors.Wrapf(err, "failed to detect compression for %q", layerFiles[i])
		}
		if s.GetCompression() == compression.Uncompressed {
			if compress {
				ref := fmt.Sprintf("compress-blob-%s-%s", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
				labels := map[string]string{
					"containerd.io/uncompressed": desc.Digest.String(),
				}
				layers[i], err = compressBlob(ctx, store, s, ref, content.WithLabels(labels))
				if err != nil {
					s.Close()
					ra.Close()
					return nil, err
				}
				layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
			} else {
				layers[i].MediaType = images.MediaTypeDockerSchema2Layer
			}
		} else {
			layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
		}
		s.Close()
		ra.Close()
	}
	return layers, nil
}

func compressBlob(ctx context.Context, cs content.Store, r io.Reader, ref string, opts ...content.Opt) (desc ocispec.Descriptor, err error) {
	w, err := content.OpenWriter(ctx, cs, content.WithRef(ref))
	if err != nil {
		return ocispec.Descriptor{}, errors.Wrap(err, "failed to open writer")
	}

	defer func() {
		w.Close()
		if err != nil {
			cs.Abort(ctx, ref)
		}
	}()
	if err := w.Truncate(0); err != nil {
		return ocispec.Descriptor{}, errors.Wrap(err, "failed to truncate writer")
	}

	cw, err := compression.CompressStream(w, compression.Gzip)
	if err != nil {
		return ocispec.Descriptor{}, err
	}

	if _, err := io.Copy(cw, r); err != nil {
		return ocispec.Descriptor{}, err
	}
	if err := cw.Close(); err != nil {
		return ocispec.Descriptor{}, err
	}

	cst, err := w.Status()
	if err != nil {
		return ocispec.Descriptor{}, errors.Wrap(err, "failed to get writer status")
	}

	desc.Digest = w.Digest()
	desc.Size = cst.Offset

	if err := w.Commit(ctx, desc.Size, desc.Digest, opts...); err != nil {
		if !errdefs.IsAlreadyExists(err) {
			return ocispec.Descriptor{}, errors.Wrap(err, "failed to commit")
		}
	}

	return desc, nil
}

func writeManifest(ctx context.Context, cs content.Ingester, manifest interface{}, mediaType string) (ocispec.Descriptor, error) {
	manifestBytes, err := json.Marshal(manifest)
	if err != nil {
		return ocispec.Descriptor{}, err
	}

	desc := ocispec.Descriptor{
		MediaType: mediaType,
		Digest:    digest.FromBytes(manifestBytes),
		Size:      int64(len(manifestBytes)),
	}
	if err := content.WriteBlob(ctx, cs, "manifest-"+desc.Digest.String(), bytes.NewReader(manifestBytes), desc); err != nil {
		return ocispec.Descriptor{}, err
	}

	return desc, nil
}
<commit_msg>ctr: Fixes Windows image import<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

// Package archive provides a Docker and OCI compatible importer
package archive

import (
	"archive/tar"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"path"

	"github.com/containerd/containerd/archive/compression"
	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/platforms"
	digest "github.com/opencontainers/go-digest"
	specs "github.com/opencontainers/image-spec/specs-go"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

type importOpts struct {
	compress bool
}

// ImportOpt is an option for importing an OCI index
type ImportOpt func(*importOpts) error

// WithImportCompression compresses uncompressed layers on import.
// This is used for import formats which do not include the manifest.
func WithImportCompression() ImportOpt {
	return func(io *importOpts) error {
		io.compress = true
		return nil
	}
}

// ImportIndex imports an index from a tar archive image bundle
// - implements Docker v1.1, v1.2 and OCI v1.
// - prefers OCI v1 when provided
// - creates OCI index for Docker formats
// - normalizes Docker references and adds as OCI ref name
//      e.g. alpine:latest -> docker.io/library/alpine:latest
// - existing OCI reference names are untouched
func ImportIndex(ctx context.Context, store content.Store, reader io.Reader, opts ...ImportOpt) (ocispec.Descriptor, error) {
	var (
		tr = tar.NewReader(reader)

		ociLayout ocispec.ImageLayout
		mfsts     []struct {
			Config   string
			RepoTags []string
			Layers   []string
		}
		symlinks = make(map[string]string)
		blobs    = make(map[string]ocispec.Descriptor)
		iopts    importOpts
	)

	for _, o := range opts {
		if err := o(&iopts); err != nil {
			return ocispec.Descriptor{}, err
		}
	}

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return ocispec.Descriptor{}, err
		}
		if hdr.Typeflag == tar.TypeSymlink {
			symlinks[hdr.Name] = path.Join(path.Dir(hdr.Name), hdr.Linkname)
		}

		if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
			if hdr.Typeflag != tar.TypeDir {
				log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored")
			}
			continue
		}

		hdrName := path.Clean(hdr.Name)
		if hdrName == ocispec.ImageLayoutFile {
			if err = onUntarJSON(tr, &ociLayout); err != nil {
				return ocispec.Descriptor{}, errors.Wrapf(err, "untar oci layout %q", hdr.Name)
			}
		} else if hdrName == "manifest.json" {
			if err = onUntarJSON(tr, &mfsts); err != nil {
				return ocispec.Descriptor{}, errors.Wrapf(err, "untar manifest %q", hdr.Name)
			}
		} else {
			dgst, err := onUntarBlob(ctx, tr, store, hdr.Size, "tar-"+hdrName)
			if err != nil {
				return ocispec.Descriptor{}, errors.Wrapf(err, "failed to ingest %q", hdr.Name)
			}

			blobs[hdrName] = ocispec.Descriptor{
				Digest: dgst,
				Size:   hdr.Size,
			}
		}
	}

	// If OCI layout was given, interpret the tar as an OCI layout.
	// When not provided, the layout of the tar will be interpreted
	// as Docker v1.1 or v1.2.
	if ociLayout.Version != "" {
		if ociLayout.Version != ocispec.ImageLayoutVersion {
			return ocispec.Descriptor{}, errors.Errorf("unsupported OCI version %s", ociLayout.Version)
		}

		idx, ok := blobs["index.json"]
		if !ok {
			return ocispec.Descriptor{}, errors.Errorf("missing index.json in OCI layout %s", ocispec.ImageLayoutVersion)
		}

		idx.MediaType = ocispec.MediaTypeImageIndex
		return idx, nil
	}

	if mfsts == nil {
		return ocispec.Descriptor{}, errors.Errorf("unrecognized image format")
	}

	for name, linkname := range symlinks {
		desc, ok := blobs[linkname]
		if !ok {
			return ocispec.Descriptor{}, errors.Errorf("no target for symlink layer from %q to %q", name, linkname)
		}
		blobs[name] = desc
	}

	idx := ocispec.Index{
		Versioned: specs.Versioned{
			SchemaVersion: 2,
		},
	}
	for _, mfst := range mfsts {
		config, ok := blobs[mfst.Config]
		if !ok {
			return ocispec.Descriptor{}, errors.Errorf("image config %q not found", mfst.Config)
		}
		config.MediaType = images.MediaTypeDockerSchema2Config

		layers, err := resolveLayers(ctx, store, mfst.Layers, blobs, iopts.compress)
		if err != nil {
			return ocispec.Descriptor{}, errors.Wrap(err, "failed to resolve layers")
		}

		manifest := struct {
			SchemaVersion int                  `json:"schemaVersion"`
			MediaType     string               `json:"mediaType"`
			Config        ocispec.Descriptor   `json:"config"`
			Layers        []ocispec.Descriptor `json:"layers"`
		}{
			SchemaVersion: 2,
			MediaType:     images.MediaTypeDockerSchema2Manifest,
			Config:        config,
			Layers:        layers,
		}

		desc, err := writeManifest(ctx, store, manifest, manifest.MediaType)
		if err != nil {
			return ocispec.Descriptor{}, errors.Wrap(err, "write docker manifest")
		}

		imgPlatforms, err := images.Platforms(ctx, store, desc)
		if err != nil {
			return ocispec.Descriptor{}, errors.Wrap(err, "unable to resolve platform")
		}
		if len(imgPlatforms) > 0 {
			// Only one platform can be resolved from non-index manifest,
			// The platform can only come from the config included above,
			// if the config has no platform it can be safely omitted.
			desc.Platform = &imgPlatforms[0]

			// If the image we've just imported is a Windows image without the OSVersion set,
			// we could just assume it matches this host's OS Version. Without this, the
			// children labels might not be set on the image content, leading to it being
			// garbage collected, breaking the image.
			// See: https://github.com/containerd/containerd/issues/5690
			if desc.Platform.OS == "windows" && desc.Platform.OSVersion == "" {
				platform := platforms.DefaultSpec()
				desc.Platform.OSVersion = platform.OSVersion
			}
		}

		if len(mfst.RepoTags) == 0 {
			idx.Manifests = append(idx.Manifests, desc)
		} else {
			// Add descriptor per tag
			for _, ref := range mfst.RepoTags {
				mfstdesc := desc

				normalized, err := normalizeReference(ref)
				if err != nil {
					return ocispec.Descriptor{}, err
				}

				mfstdesc.Annotations = map[string]string{
					images.AnnotationImageName: normalized,
					ocispec.AnnotationRefName:  ociReferenceName(normalized),
				}

				idx.Manifests = append(idx.Manifests, mfstdesc)
			}
		}
	}

	return writeManifest(ctx, store, idx, ocispec.MediaTypeImageIndex)
}

func onUntarJSON(r io.Reader, j interface{}) error {
	b, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}
	return json.Unmarshal(b, j)
}

func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, size int64, ref string) (digest.Digest, error) {
	dgstr := digest.Canonical.Digester()

	if err := content.WriteBlob(ctx, store, ref, io.TeeReader(r, dgstr.Hash()), ocispec.Descriptor{Size: size}); err != nil {
		return "", err
	}

	return dgstr.Digest(), nil
}

func resolveLayers(ctx context.Context, store content.Store, layerFiles []string, blobs map[string]ocispec.Descriptor, compress bool) ([]ocispec.Descriptor, error) {
	layers := make([]ocispec.Descriptor, len(layerFiles))
	descs := map[digest.Digest]*ocispec.Descriptor{}
	filters := []string{}
	for i, f := range layerFiles {
		desc, ok := blobs[f]
		if !ok {
			return nil, errors.Errorf("layer %q not found", f)
		}
		layers[i] = desc
		descs[desc.Digest] = &layers[i]
		filters = append(filters, "labels.\"containerd.io/uncompressed\"=="+desc.Digest.String())
	}

	err := store.Walk(ctx, func(info content.Info) error {
		dgst, ok := info.Labels["containerd.io/uncompressed"]
		if ok {
			desc := descs[digest.Digest(dgst)]
			if desc != nil {
				desc.MediaType = images.MediaTypeDockerSchema2LayerGzip
				desc.Digest = info.Digest
				desc.Size = info.Size
			}
		}
		return nil
	}, filters...)
	if err != nil {
		return nil, errors.Wrap(err, "failure checking for compressed blobs")
	}

	for i, desc := range layers {
		if desc.MediaType != "" {
			continue
		}
		// Open blob, resolve media type
		ra, err := store.ReaderAt(ctx, desc)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to open %q (%s)", layerFiles[i], desc.Digest)
		}
		s, err := compression.DecompressStream(content.NewReader(ra))
		if err != nil {
			ra.Close()
			return nil, errors.Wrapf(err, "failed to detect compression for %q", layerFiles[i])
		}
		if s.GetCompression() == compression.Uncompressed {
			if compress {
				ref := fmt.Sprintf("compress-blob-%s-%s", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
				labels := map[string]string{
					"containerd.io/uncompressed": desc.Digest.String(),
				}
				layers[i], err = compressBlob(ctx, store, s, ref, content.WithLabels(labels))
				if err != nil {
					s.Close()
					ra.Close()
					return nil, err
				}
				layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
			} else {
				layers[i].MediaType = images.MediaTypeDockerSchema2Layer
			}
		} else {
			layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
		}
		s.Close()
		ra.Close()
	}
	return layers, nil
}

func compressBlob(ctx context.Context, cs content.Store, r io.Reader, ref string, opts ...content.Opt) (desc ocispec.Descriptor, err error) {
	w, err := content.OpenWriter(ctx, cs, content.WithRef(ref))
	if err != nil {
		return ocispec.Descriptor{}, errors.Wrap(err, "failed to open writer")
	}

	defer func() {
		w.Close()
		if err != nil {
			cs.Abort(ctx, ref)
		}
	}()
	if err := w.Truncate(0); err != nil {
		return ocispec.Descriptor{}, errors.Wrap(err, "failed to truncate writer")
	}

	cw, err := compression.CompressStream(w, compression.Gzip)
	if err != nil {
		return ocispec.Descriptor{}, err
	}

	if _, err := io.Copy(cw, r); err != nil {
		return ocispec.Descriptor{}, err
	}
	if err := cw.Close(); err != nil {
		return ocispec.Descriptor{}, err
	}

	cst, err := w.Status()
	if err != nil {
		return ocispec.Descriptor{}, errors.Wrap(err, "failed to get writer status")
	}

	desc.Digest = w.Digest()
	desc.Size = cst.Offset

	if err := w.Commit(ctx, desc.Size, desc.Digest, opts...); err != nil {
		if !errdefs.IsAlreadyExists(err) {
			return ocispec.Descriptor{}, errors.Wrap(err, "failed to commit")
		}
	}

	return desc, nil
}

func writeManifest(ctx context.Context, cs content.Ingester, manifest interface{}, mediaType string) (ocispec.Descriptor, error) {
	manifestBytes, err := json.Marshal(manifest)
	if err != nil {
		return ocispec.Descriptor{}, err
	}

	desc := ocispec.Descriptor{
		MediaType: mediaType,
		Digest:    digest.FromBytes(manifestBytes),
		Size:      int64(len(manifestBytes)),
	}
	if err := content.WriteBlob(ctx, cs, "manifest-"+desc.Digest.String(), bytes.NewReader(manifestBytes), desc); err != nil {
		return ocispec.Descriptor{}, err
	}

	return desc, nil
}
<|endoftext|>


<commit_before>package scanner

type ImageDataBase struct {
}

func LoadImageDataBase(baseDir string) (*ImageDataBase, error) {
	return loadImageDataBase(baseDir)
}
<commit_msg>Check in embyronic Filter, Directory, Image and Hash types.<commit_after>package scanner

type FilterEntry string

type Filter []FilterEntry

type Directory struct {
}

type Image struct {
	filter       Filter
	topDirectory *Directory
}

type Hash [64]byte

type ImageDataBase struct {
	imageMap map[string]*Image
}

func LoadImageDataBase(baseDir string) (*ImageDataBase, error) {
	return loadImageDataBase(baseDir)
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2021 The GoPlus Authors (goplus.org). All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package gopmod

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/goplus/gop/env"
)

// -----------------------------------------------------------------------------

type Fingerp struct {
	Hash    [20]byte
	ModTime time.Time
}

type Source interface {
	Fingerp() (*Fingerp, error) // source code fingerprint
	GenGo(outFile, modFile string) error
}

type Project struct {
	Source
	AutoGenFile   string // autogen file of output
	FriendlyFname string // friendly fname of source
	BuildArgs     []string
	ExecArgs      []string
	UseDefaultCtx bool
	ForceToGen    bool
	FlagNRINC     bool // do not run if not changed
	FlagRTOE      bool // remove tempfile on error
}

type Context struct {
	modfile string
	dir     string
	defctx  bool
}

func New(dir string) *Context {
	modfile, err := env.GOPMOD(dir)
	if err != nil {
		return NewDefault(dir)
	}
	return &Context{modfile: modfile, dir: dir}
}

func NewDefault(dir string) *Context {
	modfile := env.HOME() + "/.gop/run/go.mod"
	if _, err := os.Stat(modfile); os.IsNotExist(err) {
		genDefaultGopMod(modfile)
	}
	return &Context{modfile: modfile, dir: dir, defctx: true}
}

func (p *Context) GoCommand(op string, src *Project) GoCmd {
	if src.UseDefaultCtx {
		p = NewDefault(p.dir)
	}
	fp, err := src.Fingerp()
	if err != nil {
		log.Panicln(err)
	}
	out := p.out(src, fp.Hash[:])
	if src.ForceToGen || fileIsDirty(fp.ModTime, out.goFile) {
		if p.defctx {
			dir, _ := filepath.Split(out.goFile)
			os.Mkdir(dir, 0755)
		}
		if err := src.GenGo(out.goFile, p.modfile); err != nil {
			log.Panicln(err)
		}
	} else if src.FlagNRINC { // do not run if not changed
		return GoCmd{}
	}
	return goCommand(p.dir, op, &out)
}

func fileIsDirty(srcMod time.Time, destFile string) bool {
	fiDest, err := os.Stat(destFile)
	if err != nil {
		return true
	}
	return srcMod.After(fiDest.ModTime())
}

type goTarget struct {
	goFile  string
	outFile string
	proj    *Project
	defctx  bool
}

func (p *Context) out(src *Project, hash []byte) (ret goTarget) {
	fname := src.FriendlyFname
	if !strings.HasSuffix(fname, ".go") {
		fname += ".go"
	}
	dir, _ := filepath.Split(p.modfile)
	ret.outFile = dir + "g" + base64.RawURLEncoding.EncodeToString(hash)
	ret.proj = src
	ret.defctx = p.defctx
	if ret.defctx || src.AutoGenFile == "" {
		ret.goFile = ret.outFile + fname
	} else {
		ret.goFile = src.AutoGenFile
	}
	if runtime.GOOS == "windows" {
		ret.outFile += ".exe"
	}
	return
}

// -----------------------------------------------------------------------------

const (
	dummyGoFile = `package dummy

import (
	_ "github.com/goplus/gop"
)
`
	gomodFormat = `module goplus.org/userapp

go 1.16

require (
	github.com/goplus/gop %s
)

replace (
	github.com/goplus/gop => %s
)
`
)

func genGomodFile(modfile string) {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, gomodFormat, GOPVERSION, GOPROOT)
	err := os.WriteFile(modfile, buf.Bytes(), 0666)
	if err != nil {
		log.Panicln(err)
	}
}

func genDummyProject(dir string) {
	err := os.WriteFile(dir+"/dummy.go", []byte(dummyGoFile), 0666)
	if err != nil {
		log.Panicln(err)
	}
}

func execCommand(dir, command string, args ...string) {
	err := runCommand(dir, command, args...)
	if err != nil {
		switch e := err.(type) {
		case *exec.ExitError:
			os.Exit(e.ExitCode())
		default:
			log.Fatalln(err)
		}
	}
}

func runCommand(dir, command string, args ...string) error {
	cmd := exec.Command(command, args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Env = os.Environ()
	cmd.Dir = dir
	return cmd.Run()
}

func genDefaultGopMod(modfile string) {
	dir, _ := filepath.Split(modfile)
	dummy := dir + "dummy"
	os.MkdirAll(dummy, 0755)
	genGomodFile(modfile)
	genDummyProject(dummy)
	execCommand(dir, "go", "mod", "tidy")
}

// -----------------------------------------------------------------------------
<commit_msg>genGomodFile: use relpath in windows<commit_after>/*
 * Copyright (c) 2021 The GoPlus Authors (goplus.org). All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package gopmod

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/goplus/gop/env"
)

// -----------------------------------------------------------------------------

type Fingerp struct {
	Hash    [20]byte
	ModTime time.Time
}

type Source interface {
	Fingerp() (*Fingerp, error) // source code fingerprint
	GenGo(outFile, modFile string) error
}

type Project struct {
	Source
	AutoGenFile   string // autogen file of output
	FriendlyFname string // friendly fname of source
	BuildArgs     []string
	ExecArgs      []string
	UseDefaultCtx bool
	ForceToGen    bool
	FlagNRINC     bool // do not run if not changed
	FlagRTOE      bool // remove tempfile on error
}

type Context struct {
	modfile string
	dir     string
	defctx  bool
}

func New(dir string) *Context {
	modfile, err := env.GOPMOD(dir)
	if err != nil {
		return NewDefault(dir)
	}
	return &Context{modfile: modfile, dir: dir}
}

func NewDefault(dir string) *Context {
	modfile := env.HOME() + "/.gop/run/go.mod"
	if _, err := os.Stat(modfile); os.IsNotExist(err) {
		genDefaultGopMod(modfile)
	}
	return &Context{modfile: modfile, dir: dir, defctx: true}
}

func (p *Context) GoCommand(op string, src *Project) GoCmd {
	if src.UseDefaultCtx {
		p = NewDefault(p.dir)
	}
	fp, err := src.Fingerp()
	if err != nil {
		log.Panicln(err)
	}
	out := p.out(src, fp.Hash[:])
	if src.ForceToGen || fileIsDirty(fp.ModTime, out.goFile) {
		if p.defctx {
			dir, _ := filepath.Split(out.goFile)
			os.Mkdir(dir, 0755)
		}
		if err := src.GenGo(out.goFile, p.modfile); err != nil {
			log.Panicln(err)
		}
	} else if src.FlagNRINC { // do not run if not changed
		return GoCmd{}
	}
	return goCommand(p.dir, op, &out)
}

func fileIsDirty(srcMod time.Time, destFile string) bool {
	fiDest, err := os.Stat(destFile)
	if err != nil {
		return true
	}
	return srcMod.After(fiDest.ModTime())
}

type goTarget struct {
	goFile  string
	outFile string
	proj    *Project
	defctx  bool
}

func (p *Context) out(src *Project, hash []byte) (ret goTarget) {
	fname := src.FriendlyFname
	if !strings.HasSuffix(fname, ".go") {
		fname += ".go"
	}
	dir, _ := filepath.Split(p.modfile)
	ret.outFile = dir + "g" + base64.RawURLEncoding.EncodeToString(hash)
	ret.proj = src
	ret.defctx = p.defctx
	if ret.defctx || src.AutoGenFile == "" {
		ret.goFile = ret.outFile + fname
	} else {
		ret.goFile = src.AutoGenFile
	}
	if inWindows {
		ret.outFile += ".exe"
	}
	return
}

const (
	inWindows = (runtime.GOOS == "windows")
)

// -----------------------------------------------------------------------------

const (
	dummyGoFile = `package dummy

import (
	_ "github.com/goplus/gop"
)
`
	gomodFormat = `module goplus.org/userapp

go 1.16

require (
	github.com/goplus/gop %s
)

replace (
	github.com/goplus/gop => %s
)
`
)

func genGomodFile(modfile string) {
	var buf bytes.Buffer
	var err error
	var gopRoot = GOPROOT
	if inWindows {
		if gopRoot, err = filepath.Rel(filepath.Dir(modfile), gopRoot); err != nil {
			log.Panicln(err)
		}
		gopRoot = filepath.ToSlash(gopRoot)
	}
	fmt.Fprintf(&buf, gomodFormat, GOPVERSION, gopRoot)
	err = os.WriteFile(modfile, buf.Bytes(), 0666)
	if err != nil {
		log.Panicln(err)
	}
}

func genDummyProject(dir string) {
	err := os.WriteFile(dir+"/dummy.go", []byte(dummyGoFile), 0666)
	if err != nil {
		log.Panicln(err)
	}
}

func execCommand(dir, command string, args ...string) {
	err := runCommand(dir, command, args...)
	if err != nil {
		switch e := err.(type) {
		case *exec.ExitError:
			os.Exit(e.ExitCode())
		default:
			log.Fatalln(err)
		}
	}
}

func runCommand(dir, command string, args ...string) error {
	cmd := exec.Command(command, args...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Env = os.Environ()
	cmd.Dir = dir
	return cmd.Run()
}

func genDefaultGopMod(modfile string) {
	dir, _ := filepath.Split(modfile)
	dummy := dir + "dummy"
	os.MkdirAll(dummy, 0755)
	genGomodFile(modfile)
	genDummyProject(dummy)
	execCommand(dir, "go", "mod", "tidy")
}

// -----------------------------------------------------------------------------
<|endoftext|>