<commit_before>package usbtmc

import (
	"github.com/truveris/gousb/usb"
	"log"
)

type UsbtmcContext struct {
	ctx *usb.Context
}

func NewContext() *UsbtmcContext {
	c := &UsbtmcContext{
		ctx: usb.NewContext(),
	}
	return c
}

func (c *UsbtmcContext) Debug(level int) {
	c.ctx.Debug(level)
}

func (c *UsbtmcContext) Close() error {
	return c.ctx.Close()
}

type Instrument struct {
	Device              *usb.Device
	BulkInEndpoint      usb.Endpoint
	BulkOutEndpoint     usb.Endpoint
	InterruptInEndpoint usb.Endpoint
}

func (c *UsbtmcContext) NewInstrument(visaResourceName string) *Instrument {
	var usbtmcConfig uint8
	var usbtmcInterface uint8
	var usbtmcSetup uint8
	var bulkOutEndpointAddress uint8
	var bulkInEndpointAddress uint8
	var interruptInEndpointAddress uint8
	// TODO(mdr) Need to handle the error potentially return by ListDevices
	// FIXME(mdr) Need to handle error in case given a bad visaResource
	devices, _ := c.ctx.ListDevices(FindUsbtmcFromResourceString(visaResourceName))
	device := devices[0]
	for _, config := range device.Descriptor.Configs {
		for _, iface := range config.Interfaces {
			for _, setup := range iface.Setups {
				if setup.IfClass == 0xfe && setup.IfSubClass == 0x03 {
					usbtmcConfig = uint8(config.Config)
					usbtmcInterface = uint8(iface.Number)
					usbtmcSetup = uint8(setup.Number)
					for _, endpoint := range setup.Endpoints {
						endpointAttributes := endpoint.Attributes
						endpointDirection := endpoint.Address & uint8(usb.ENDPOINT_DIR_MASK)
						endpointType := endpointAttributes & uint8(usb.TRANSFER_TYPE_MASK)
						if endpointType == uint8(usb.TRANSFER_TYPE_BULK) {
							if endpointDirection == uint8(usb.ENDPOINT_DIR_IN) {
								bulkInEndpointAddress = endpoint.Address | uint8(usb.ENDPOINT_DIR_IN)
							} else if endpointDirection == uint8(usb.ENDPOINT_DIR_OUT) {
								bulkOutEndpointAddress = endpoint.Address | uint8(usb.ENDPOINT_DIR_OUT)
							}
						} else if endpointType == uint8(usb.TRANSFER_TYPE_INTERRUPT) {
							if endpointDirection == uint8(usb.ENDPOINT_DIR_IN) {
								interruptInEndpointAddress = endpoint.Address | uint8(usb.ENDPOINT_DIR_IN)
							}
						}
					}
				}
			}
		}
	}

	bulkInEndpoint, err := device.OpenEndpoint(
		usbtmcConfig, usbtmcInterface, usbtmcSetup, bulkInEndpointAddress)
	if err != nil {
		log.Fatal("Error opening bulkInEndpoint")
	}

	bulkOutEndpoint, err := device.OpenEndpoint(
		usbtmcConfig, usbtmcInterface, usbtmcSetup, bulkOutEndpointAddress)
	if err != nil {
		log.Fatal("Error opening bulkOutEndpoint")
	}

	// TODO(mdr): Need to make the interruptInEndpoint optional
	interruptInEndpoint, err := device.OpenEndpoint(
		usbtmcConfig, usbtmcInterface, usbtmcSetup, interruptInEndpointAddress)
	if err != nil {
		log.Fatal("Error opening interruptInEndpoint")
	}

	return &Instrument{
		Device:              device,
		BulkInEndpoint:      bulkInEndpoint,
		BulkOutEndpoint:     bulkOutEndpoint,
		InterruptInEndpoint: interruptInEndpoint,
	}
}

func (i *Instrument) Close() error {
	return i.Device.Close()
}

func FindVisaResourceName(visaResourceName string, c *usb.Context) (*usb.Device, error) {
	devices, err := c.ListDevices(FindUsbtmcFromResourceString(visaResourceName))
	return devices[0], err
}

func FindUsbtmcFromResourceString(resourceString string) func(desc *usb.Descriptor) bool {
	visaResource, err := NewVisaResource(resourceString)
	if err != nil {
		log.Fatal("Invalid visaResource")
	}

	if visaResource.interfaceType != "USB" {
		log.Fatal("Non-usb resource provided")
	}

	return func(desc *usb.Descriptor) bool {
		hasUsbtmcInterface := false
		switch {
		case uint16(desc.Vendor) == visaResource.manufacturerId &&
			uint16(desc.Product) == visaResource.modelCode &&
			desc.Class == 0x00 && desc.SubClass == 0x00:
			for _, config := range desc.Configs {
				for _, iface := range config.Interfaces {
					for _, setup := range iface.Setups {
						switch {
						case setup.IfClass == 0xfe && setup.IfSubClass == 0x03 && setup.IfProtocol == 00:
							hasUsbtmcInterface = true
							log.Printf(
								"USBTMC interface found on S/N %s, Vendor %d, Product %d",
								desc.SerialNumber,
								desc.Vendor,
								desc.Product,
							)
							log.Printf("--> %s, %s", config, setup)
						case setup.IfClass == 0xfe && setup.IfSubClass == 0x03 && setup.IfProtocol == 01:
							hasUsbtmcInterface = true
							log.Printf(
								"USB488 interface found on S/N %s, Vendor %d, Product %d",
								desc.SerialNumber,
								desc.Vendor,
								desc.Product,
							)
							log.Printf("--> %s, %s", config, setup)
						}
					}
				}
			}
		default:
			return false
		}
		return hasUsbtmcInterface
	}
}

func FindAllUsbtmcInterfaces(desc *usb.Descriptor) bool {
	hasUsbtmcInterface := false
	switch {
	case desc.Class == 0x00 && desc.SubClass == 0x00:
		for _, config := range desc.Configs {
			for _, iface := range config.Interfaces {
				for _, setup := range iface.Setups {
					switch {
					case setup.IfClass == 0xfe && setup.IfSubClass == 0x03 && setup.IfProtocol == 00:
						hasUsbtmcInterface = true
						log.Printf(
							"USBTMC interface found on S/N %s, Vendor %d, Product %d",
							desc.SerialNumber,
							desc.Vendor,
							desc.Product,
						)
						log.Printf("--> %s, %s", config, setup)
					case setup.IfClass == 0xfe && setup.IfSubClass == 0x03 && setup.IfProtocol == 01:
						hasUsbtmcInterface = true
						log.Printf(
							"USB488 interface found on S/N %s, Vendor %d, Product %d",
							desc.SerialNumber,
							desc.Vendor,
							desc.Product,
						)
						log.Printf("--> %s, %s", config, setup)
					}
				}
			}
		}
	default:
		return false
	}
	return hasUsbtmcInterface
}
<commit_msg>Add func to createBulkOutHeaderPrefix<commit_after>package usbtmc

import (
	"github.com/truveris/gousb/usb"
	"log"
)

type UsbtmcContext struct {
	ctx *usb.Context
}

func NewContext() *UsbtmcContext {
	c := &UsbtmcContext{
		ctx: usb.NewContext(),
	}
	return c
}

func (c *UsbtmcContext) Debug(level int) {
	c.ctx.Debug(level)
}

func (c *UsbtmcContext) Close() error {
	return c.ctx.Close()
}

type Instrument struct {
	Device              *usb.Device
	BulkInEndpoint      usb.Endpoint
	BulkOutEndpoint     usb.Endpoint
	InterruptInEndpoint usb.Endpoint
	bTag                byte
}

func (c *UsbtmcContext) NewInstrument(visaResourceName string) *Instrument {
	var usbtmcConfig uint8
	var usbtmcInterface uint8
	var usbtmcSetup uint8
	var bulkOutEndpointAddress uint8
	var bulkInEndpointAddress uint8
	var interruptInEndpointAddress uint8
	// TODO(mdr) Need to handle the error potentially return by ListDevices
	// FIXME(mdr) Need to handle error in case given a bad visaResource
	devices, _ := c.ctx.ListDevices(FindUsbtmcFromResourceString(visaResourceName))
	device := devices[0]
	for _, config := range device.Descriptor.Configs {
		for _, iface := range config.Interfaces {
			for _, setup := range iface.Setups {
				if setup.IfClass == 0xfe && setup.IfSubClass == 0x03 {
					usbtmcConfig = uint8(config.Config)
					usbtmcInterface = uint8(iface.Number)
					usbtmcSetup = uint8(setup.Number)
					for _, endpoint := range setup.Endpoints {
						endpointAttributes := endpoint.Attributes
						endpointDirection := endpoint.Address & uint8(usb.ENDPOINT_DIR_MASK)
						endpointType := endpointAttributes & uint8(usb.TRANSFER_TYPE_MASK)
						if endpointType == uint8(usb.TRANSFER_TYPE_BULK) {
							if endpointDirection == uint8(usb.ENDPOINT_DIR_IN) {
								bulkInEndpointAddress = endpoint.Address | uint8(usb.ENDPOINT_DIR_IN)
							} else if endpointDirection == uint8(usb.ENDPOINT_DIR_OUT) {
								bulkOutEndpointAddress = endpoint.Address | uint8(usb.ENDPOINT_DIR_OUT)
							}
						} else if endpointType == uint8(usb.TRANSFER_TYPE_INTERRUPT) {
							if endpointDirection == uint8(usb.ENDPOINT_DIR_IN) {
								interruptInEndpointAddress = endpoint.Address | uint8(usb.ENDPOINT_DIR_IN)
							}
						}
					}
				}
			}
		}
	}

	bulkInEndpoint, err := device.OpenEndpoint(
		usbtmcConfig, usbtmcInterface, usbtmcSetup, bulkInEndpointAddress)
	if err != nil {
		log.Fatal("Error opening bulkInEndpoint")
	}

	bulkOutEndpoint, err := device.OpenEndpoint(
		usbtmcConfig, usbtmcInterface, usbtmcSetup, bulkOutEndpointAddress)
	if err != nil {
		log.Fatal("Error opening bulkOutEndpoint")
	}

	// TODO(mdr): Need to make the interruptInEndpoint optional
	interruptInEndpoint, err := device.OpenEndpoint(
		usbtmcConfig, usbtmcInterface, usbtmcSetup, interruptInEndpointAddress)
	if err != nil {
		log.Fatal("Error opening interruptInEndpoint")
	}

	return &Instrument{
		Device:              device,
		BulkInEndpoint:      bulkInEndpoint,
		BulkOutEndpoint:     bulkOutEndpoint,
		InterruptInEndpoint: interruptInEndpoint,
	}
}

func (i *Instrument) Close() error {
	return i.Device.Close()
}

func FindVisaResourceName(visaResourceName string, c *usb.Context) (*usb.Device, error) {
	devices, err := c.ListDevices(FindUsbtmcFromResourceString(visaResourceName))
	return devices[0], err
}

func FindUsbtmcFromResourceString(resourceString string) func(desc *usb.Descriptor) bool {
	visaResource, err := NewVisaResource(resourceString)
	if err != nil {
		log.Fatal("Invalid visaResource")
	}

	if visaResource.interfaceType != "USB" {
		log.Fatal("Non-usb resource provided")
	}

	return func(desc *usb.Descriptor) bool {
		hasUsbtmcInterface := false
		switch {
		case uint16(desc.Vendor) == visaResource.manufacturerId &&
			uint16(desc.Product) == visaResource.modelCode &&
			desc.Class == 0x00 && desc.SubClass == 0x00:
			for _, config := range desc.Configs {
				for _, iface := range config.Interfaces {
					for _, setup := range iface.Setups {
						switch {
						case setup.IfClass == 0xfe && setup.IfSubClass == 0x03 && setup.IfProtocol == 00:
							hasUsbtmcInterface = true
							log.Printf(
								"USBTMC interface found on S/N %s, Vendor %d, Product %d",
								desc.SerialNumber,
								desc.Vendor,
								desc.Product,
							)
							log.Printf("--> %s, %s", config, setup)
						case setup.IfClass == 0xfe && setup.IfSubClass == 0x03 && setup.IfProtocol == 01:
							hasUsbtmcInterface = true
							log.Printf(
								"USB488 interface found on S/N %s, Vendor %d, Product %d",
								desc.SerialNumber,
								desc.Vendor,
								desc.Product,
							)
							log.Printf("--> %s, %s", config, setup)
						}
					}
				}
			}
		default:
			return false
		}
		return hasUsbtmcInterface
	}
}

func FindAllUsbtmcInterfaces(desc *usb.Descriptor) bool {
	hasUsbtmcInterface := false
	switch {
	case desc.Class == 0x00 && desc.SubClass == 0x00:
		for _, config := range desc.Configs {
			for _, iface := range config.Interfaces {
				for _, setup := range iface.Setups {
					switch {
					case setup.IfClass == 0xfe && setup.IfSubClass == 0x03 && setup.IfProtocol == 00:
						hasUsbtmcInterface = true
						log.Printf(
							"USBTMC interface found on S/N %s, Vendor %d, Product %d",
							desc.SerialNumber,
							desc.Vendor,
							desc.Product,
						)
						log.Printf("--> %s, %s", config, setup)
					case setup.IfClass == 0xfe && setup.IfSubClass == 0x03 && setup.IfProtocol == 01:
						hasUsbtmcInterface = true
						log.Printf(
							"USB488 interface found on S/N %s, Vendor %d, Product %d",
							desc.SerialNumber,
							desc.Vendor,
							desc.Product,
						)
						log.Printf("--> %s, %s", config, setup)
					}
				}
			}
		}
	default:
		return false
	}
	return hasUsbtmcInterface
}

func (instrument *Instrument) createBulkOutHeaderPrefix(msgId MsgId) [4]byte {
	return [4]byte{
		byte(msgId),
		instrument.bTag,
		invertbTag(instrument.bTag),
		0x00,
	}
}

func invertbTag(bTag byte) byte {
	return bTag ^ 0xff
}
<|endoftext|>


<commit_before>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"fmt"
	"net"
	"strings"
	"testing"
	"time"

	"github.com/docker/machine/libmachine/state"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/test/integration/util"
)

func TestStartStop(t *testing.T) {
	tests := []struct {
		name string
		args []string
	}{
		{"nocache_oldest", []string{
			"--cache-images=false",
			fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion),
			"--kvm-network=default",
		}},
		{"feature_gates_newest_cni", []string{
			"--feature-gates",
			"ServerSideApply=true",
			"--network-plugin=cni",
			"--extra-config=kubelet.network-plugin=cni",
			"--extra-config=kubeadm.pod-network-cidr=192.168.111.111/16",
			fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion),
		}},
		{"containerd_and_non_default_apiserver_port", []string{
			"--container-runtime=containerd",
			"--docker-opt containerd=/var/run/containerd/containerd.sock",
			"--apiserver-port=8444",
		}},
		{"crio_ignore_preflights", []string{
			"--container-runtime=crio",
			"--extra-config",
			"kubeadm.ignore-preflight-errors=SystemVerification",
		}},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			r := NewMinikubeRunner(t)
			if !strings.Contains(test.name, "docker") && usingNoneDriver(r) {
				t.Skipf("skipping %s - incompatible with none driver", test.name)
			}

			r.RunCommand("config set WantReportErrorPrompt false", true)
			r.RunCommand("delete", false)
			r.CheckStatus(state.None.String())
			r.Start(test.args...)
			r.CheckStatus(state.Running.String())

			ip := r.RunCommand("ip", true)
			ip = strings.TrimRight(ip, "\n")
			if net.ParseIP(ip) == nil {
				t.Fatalf("IP command returned an invalid address: %s", ip)
			}

			// check for the current-context before and after the stop
			kubectlRunner := util.NewKubectlRunner(t)
			currentContext, err := kubectlRunner.RunCommand([]string{"config", "current-context"})
			if err != nil {
				t.Fatalf("Failed to fetch current-context")
			}
			if strings.TrimRight(string(currentContext), "\n") != "minikube" {
				t.Fatalf("got current-context - %q, want  current-context %q", string(currentContext), "minikube")
			}

			checkStop := func() error {
				r.RunCommand("stop", true)
				return r.CheckStatusNoFail(state.Stopped.String())
			}

			if err := util.Retry(t, checkStop, 5*time.Second, 6); err != nil {
				t.Fatalf("timed out while checking stopped status: %v", err)
			}

			// running this command results in error when the current-context is not set
			if err := r.Run("config current-context"); err != nil {
				t.Logf("current-context is not set to minikube")
			}

			r.Start(test.args...)
			r.CheckStatus(state.Running.String())

			r.RunCommand("delete", true)
			r.CheckStatus(state.None.String())
		})
	}
}
<commit_msg>Improve --kvm-network integration test<commit_after>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"fmt"
	"net"
	"strings"
	"testing"
	"time"

	"github.com/docker/machine/libmachine/state"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/test/integration/util"
)

func TestStartStop(t *testing.T) {
	tests := []struct {
		name string
		args []string
	}{
		{"nocache_oldest", []string{
			"--cache-images=false",
			fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion),
			"--kvm-network=test-net",
		}},
		{"feature_gates_newest_cni", []string{
			"--feature-gates",
			"ServerSideApply=true",
			"--network-plugin=cni",
			"--extra-config=kubelet.network-plugin=cni",
			"--extra-config=kubeadm.pod-network-cidr=192.168.111.111/16",
			fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion),
		}},
		{"containerd_and_non_default_apiserver_port", []string{
			"--container-runtime=containerd",
			"--docker-opt containerd=/var/run/containerd/containerd.sock",
			"--apiserver-port=8444",
		}},
		{"crio_ignore_preflights", []string{
			"--container-runtime=crio",
			"--extra-config",
			"kubeadm.ignore-preflight-errors=SystemVerification",
		}},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			r := NewMinikubeRunner(t)
			if !strings.Contains(test.name, "docker") && usingNoneDriver(r) {
				t.Skipf("skipping %s - incompatible with none driver", test.name)
			}

			r.RunCommand("config set WantReportErrorPrompt false", true)
			r.RunCommand("delete", false)
			r.CheckStatus(state.None.String())
			r.Start(test.args...)
			r.CheckStatus(state.Running.String())

			ip := r.RunCommand("ip", true)
			ip = strings.TrimRight(ip, "\n")
			if net.ParseIP(ip) == nil {
				t.Fatalf("IP command returned an invalid address: %s", ip)
			}

			// check for the current-context before and after the stop
			kubectlRunner := util.NewKubectlRunner(t)
			currentContext, err := kubectlRunner.RunCommand([]string{"config", "current-context"})
			if err != nil {
				t.Fatalf("Failed to fetch current-context")
			}
			if strings.TrimRight(string(currentContext), "\n") != "minikube" {
				t.Fatalf("got current-context - %q, want  current-context %q", string(currentContext), "minikube")
			}

			checkStop := func() error {
				r.RunCommand("stop", true)
				return r.CheckStatusNoFail(state.Stopped.String())
			}

			if err := util.Retry(t, checkStop, 5*time.Second, 6); err != nil {
				t.Fatalf("timed out while checking stopped status: %v", err)
			}

			// running this command results in error when the current-context is not set
			if err := r.Run("config current-context"); err != nil {
				t.Logf("current-context is not set to minikube")
			}

			r.Start(test.args...)
			r.CheckStatus(state.Running.String())

			r.RunCommand("delete", true)
			r.CheckStatus(state.None.String())
		})
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright ©1998-2022 by Richard A. Wilkes. All rights reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, version 2.0. If a copy of the MPL was not distributed with
 * this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * This Source Code Form is "Incompatible With Secondary Licenses", as
 * defined by the Mozilla Public License, version 2.0.
 */

package ui

import (
	"bytes"
	_ "embed"
	"fmt"
	"image"
	"os"
	"path/filepath"
	"syscall"

	"github.com/richardwilkes/gcs/v5/model/library"
	"github.com/richardwilkes/gcs/v5/ui/svglayer"
	"github.com/richardwilkes/toolbox/cmdline"
	"github.com/richardwilkes/toolbox/errs"
	"github.com/richardwilkes/toolbox/formats/icon"
	"github.com/richardwilkes/toolbox/formats/icon/ico"
	"github.com/richardwilkes/toolbox/log/jot"
	"github.com/richardwilkes/toolbox/xio/fs/paths"
	"golang.org/x/sys/windows/registry"
)

const (
	shcneAssocChanged = 0x08000000
	shcfnIDlist       = 0
)

//go:embed doc-256.png
var docIconBytes []byte

var (
	shell32            = syscall.NewLazyDLL("shell32.dll")
	shChangeNotifyProc = shell32.NewProc("SHChangeNotify")
	softwareClasses    = `Software\Classes\`
)

func performPlatformStartup() {
	if err := configureRegistry(); err != nil {
		jot.Error(err)
	}
}

func configureRegistry() error {
	exePath, err := os.Executable()
	if err != nil {
		return errs.Wrap(err)
	}
	if exePath, err = filepath.Abs(exePath); err != nil {
		return errs.Wrap(err)
	}
	var docBaseIcon image.Image
	if docBaseIcon, _, err = image.Decode(bytes.NewBuffer(docIconBytes)); err != nil {
		return errs.Wrap(err)
	}
	for i := range library.KnownFileTypes {
		if fi := &library.KnownFileTypes[i]; fi.IsGCSData {
			// Create the doc icon
			var overlay image.Image
			if overlay, err = svglayer.CreateImageFromSVG(fi, 128); err != nil {
				return err
			}
			docPath := filepath.Join(paths.AppDataDir(), fi.Extensions[0][1:]+".ico")
			if err = writeIco(icon.Stack(docBaseIcon, overlay), docPath); err != nil {
				return err
			}

			// Create the entry that points to the app's information for the extension
			appExtKey := cmdline.AppIdentifier + fi.Extensions[0]
			if err = setKey(softwareClasses+fi.Extensions[0], "", appExtKey); err != nil {
				return err
			}

			// Create the entry for the extension
			path := softwareClasses + appExtKey
			if err = setKey(path, "", fi.Name); err != nil {
				return err
			}
			if err = setKey(path+`\DefaultIcon`, "", docPath); err != nil {
				return err
			}
			if err = setKey(path+`\Shell`, "", ""); err != nil {
				return err
			}
			if err = setKey(path+`\Shell\Open`, "", ""); err != nil {
				return err
			}
			if err = setKey(path+`\Shell\Open\Command`, "", fmt.Sprintf(`"%s" "%%1"`, exePath)); err != nil {
				return err
			}
		}
	}
	shChangeNotifyProc.Call(shcneAssocChanged, shcfnIDlist, 0, 0)
	return nil
}

func writeIco(img image.Image, path string) (err error) {
	var f *os.File
	if f, err = os.Create(path); err != nil {
		return errs.Wrap(err)
	}
	defer func() {
		if cerr := f.Close(); cerr != nil && err == nil {
			err = errs.Wrap(cerr)
		}
	}()
	err = errs.Wrap(ico.Encode(f, img))
	return
}

func setKey(path, name, value string) error {
	k, _, err := registry.CreateKey(registry.CURRENT_USER, path, registry.READ|registry.WRITE)
	if err != nil {
		return errs.Wrap(err)
	}
	if err = k.SetStringValue(name, value); err != nil {
		return errs.Wrap(err)
	}
	if err = k.Close(); err != nil {
		return errs.Wrap(err)
	}
	return nil
}

/*
void RegSet( HKEY hkeyHive, const char* pszVar, const char* pszVa

lue ) {

  HKEY hkey;

  char szValueCurrent[1000];
  DWORD dwType;
  DWORD dwSize = sizeof( szValueCurrent );

  int iRC = RegGetValue( hkeyHive, pszVar, NULL, RRF_RT_ANY, &dwType, szValueCurrent, &dwSize );

  bool bDidntExist = iRC == ERROR_FILE_NOT_FOUND;

  if ( iRC != ERROR_SUCCESS && !bDidntExist )
      AKS( AKSFatal, "RegGetValue( %s ): %s", pszVar, strerror( iRC ) );

  DWORD dwDisposition;
  iRC = RegCreateKeyEx( hkeyHive, pszVar, 0, 0, 0, KEY_ALL_ACCESS, NULL, &hkey, &dwDisposition );
  if ( iRC != ERROR_SUCCESS )
      AKS( AKSFatal, "RegCreateKeyEx( %s ): %s", pszVar, strerror( iRC ) );

  iRC = RegSetValueEx( hkey, "", 0, REG_SZ, (BYTE*) pszValue, strlen( pszValue ) + 1 );
  if ( iRC != ERROR_SUCCESS )
      AKS( AKSFatal, "RegSetValueEx( %s ): %s", pszVar, strerror( iRC ) );

  RegCloseKey(hkey);
}
*/
<commit_msg>Ensure app data dir exists before trying to write .ico's into it<commit_after>/*
 * Copyright ©1998-2022 by Richard A. Wilkes. All rights reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, version 2.0. If a copy of the MPL was not distributed with
 * this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * This Source Code Form is "Incompatible With Secondary Licenses", as
 * defined by the Mozilla Public License, version 2.0.
 */

package ui

import (
	"bytes"
	_ "embed"
	"fmt"
	"image"
	"os"
	"path/filepath"
	"syscall"

	"github.com/richardwilkes/gcs/v5/model/library"
	"github.com/richardwilkes/gcs/v5/ui/svglayer"
	"github.com/richardwilkes/toolbox/cmdline"
	"github.com/richardwilkes/toolbox/errs"
	"github.com/richardwilkes/toolbox/formats/icon"
	"github.com/richardwilkes/toolbox/formats/icon/ico"
	"github.com/richardwilkes/toolbox/log/jot"
	"github.com/richardwilkes/toolbox/xio/fs/paths"
	"golang.org/x/sys/windows/registry"
)

const (
	shcneAssocChanged = 0x08000000
	shcfnIDlist       = 0
)

//go:embed doc-256.png
var docIconBytes []byte

var (
	shell32            = syscall.NewLazyDLL("shell32.dll")
	shChangeNotifyProc = shell32.NewProc("SHChangeNotify")
	softwareClasses    = `Software\Classes\`
)

func performPlatformStartup() {
	if err := configureRegistry(); err != nil {
		jot.Error(err)
	}
}

func configureRegistry() error {
	exePath, err := os.Executable()
	if err != nil {
		return errs.Wrap(err)
	}
	if exePath, err = filepath.Abs(exePath); err != nil {
		return errs.Wrap(err)
	}
	var docBaseIcon image.Image
	if docBaseIcon, _, err = image.Decode(bytes.NewBuffer(docIconBytes)); err != nil {
		return errs.Wrap(err)
	}
	appDataDir := paths.AppDataDir()
	if err = os.MkdirAll(appDataDir, 0o755); err != nil {
		return errs.Wrap(err)
	}
	for i := range library.KnownFileTypes {
		if fi := &library.KnownFileTypes[i]; fi.IsGCSData {
			// Create the doc icon
			var overlay image.Image
			if overlay, err = svglayer.CreateImageFromSVG(fi, 128); err != nil {
				return err
			}
			docPath := filepath.Join(appDataDir, fi.Extensions[0][1:]+".ico")
			if err = writeIco(icon.Stack(docBaseIcon, overlay), docPath); err != nil {
				return err
			}

			// Create the entry that points to the app's information for the extension
			appExtKey := cmdline.AppIdentifier + fi.Extensions[0]
			if err = setKey(softwareClasses+fi.Extensions[0], "", appExtKey); err != nil {
				return err
			}

			// Create the entry for the extension
			path := softwareClasses + appExtKey
			if err = setKey(path, "", fi.Name); err != nil {
				return err
			}
			if err = setKey(path+`\DefaultIcon`, "", docPath); err != nil {
				return err
			}
			if err = setKey(path+`\Shell`, "", ""); err != nil {
				return err
			}
			if err = setKey(path+`\Shell\Open`, "", ""); err != nil {
				return err
			}
			if err = setKey(path+`\Shell\Open\Command`, "", fmt.Sprintf(`"%s" "%%1"`, exePath)); err != nil {
				return err
			}
		}
	}
	shChangeNotifyProc.Call(shcneAssocChanged, shcfnIDlist, 0, 0)
	return nil
}

func writeIco(img image.Image, path string) (err error) {
	var f *os.File
	if f, err = os.Create(path); err != nil {
		return errs.Wrap(err)
	}
	defer func() {
		if cerr := f.Close(); cerr != nil && err == nil {
			err = errs.Wrap(cerr)
		}
	}()
	err = errs.Wrap(ico.Encode(f, img))
	return
}

func setKey(path, name, value string) error {
	k, _, err := registry.CreateKey(registry.CURRENT_USER, path, registry.READ|registry.WRITE)
	if err != nil {
		return errs.Wrap(err)
	}
	if err = k.SetStringValue(name, value); err != nil {
		return errs.Wrap(err)
	}
	if err = k.Close(); err != nil {
		return errs.Wrap(err)
	}
	return nil
}

/*
void RegSet( HKEY hkeyHive, const char* pszVar, const char* pszVa

lue ) {

  HKEY hkey;

  char szValueCurrent[1000];
  DWORD dwType;
  DWORD dwSize = sizeof( szValueCurrent );

  int iRC = RegGetValue( hkeyHive, pszVar, NULL, RRF_RT_ANY, &dwType, szValueCurrent, &dwSize );

  bool bDidntExist = iRC == ERROR_FILE_NOT_FOUND;

  if ( iRC != ERROR_SUCCESS && !bDidntExist )
      AKS( AKSFatal, "RegGetValue( %s ): %s", pszVar, strerror( iRC ) );

  DWORD dwDisposition;
  iRC = RegCreateKeyEx( hkeyHive, pszVar, 0, 0, 0, KEY_ALL_ACCESS, NULL, &hkey, &dwDisposition );
  if ( iRC != ERROR_SUCCESS )
      AKS( AKSFatal, "RegCreateKeyEx( %s ): %s", pszVar, strerror( iRC ) );

  iRC = RegSetValueEx( hkey, "", 0, REG_SZ, (BYTE*) pszValue, strlen( pszValue ) + 1 );
  if ( iRC != ERROR_SUCCESS )
      AKS( AKSFatal, "RegSetValueEx( %s ): %s", pszVar, strerror( iRC ) );

  RegCloseKey(hkey);
}
*/
<|endoftext|>


<commit_before>package io

import (
	"bytes"
	"errors"
	"io"
	"os"

	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
	mdag "github.com/jbenet/go-ipfs/merkledag"
	ft "github.com/jbenet/go-ipfs/unixfs"
	ftpb "github.com/jbenet/go-ipfs/unixfs/pb"
)

var ErrIsDir = errors.New("this dag node is a directory")

// DagReader provides a way to easily read the data contained in a dag.
type DagReader struct {
	serv mdag.DAGService

	// the node being read
	node *mdag.Node

	// cached protobuf structure from node.Data
	pbdata *ftpb.Data

	// the current data buffer to be read from
	// will either be a bytes.Reader or a child DagReader
	buf ReadSeekCloser

	// NodeGetters for each of 'nodes' child links
	promises []mdag.NodeGetter

	// the index of the child link currently being read from
	linkPosition int

	// current offset for the read head within the 'file'
	offset int64

	// Our context
	ctx context.Context

	// context cancel for children
	cancel func()
}

type ReadSeekCloser interface {
	io.Reader
	io.Seeker
	io.Closer
	io.WriterTo
}

// NewDagReader creates a new reader object that reads the data represented by the given
// node, using the passed in DAGService for data retreival
func NewDagReader(ctx context.Context, n *mdag.Node, serv mdag.DAGService) (*DagReader, error) {
	pb := new(ftpb.Data)
	err := proto.Unmarshal(n.Data, pb)
	if err != nil {
		return nil, err
	}

	switch pb.GetType() {
	case ftpb.Data_Directory:
		// Dont allow reading directories
		return nil, ErrIsDir
	case ftpb.Data_Raw:
		fallthrough
	case ftpb.Data_File:
		return newDataFileReader(ctx, n, pb, serv), nil
	case ftpb.Data_Metadata:
		if len(n.Links) == 0 {
			return nil, errors.New("incorrectly formatted metadata object")
		}
		child, err := n.Links[0].GetNode(serv)
		if err != nil {
			return nil, err
		}
		return NewDagReader(ctx, child, serv)
	default:
		return nil, ft.ErrUnrecognizedType
	}
}

func newDataFileReader(ctx context.Context, n *mdag.Node, pb *ftpb.Data, serv mdag.DAGService) *DagReader {
	fctx, cancel := context.WithCancel(ctx)
	promises := serv.GetDAG(fctx, n)
	return &DagReader{
		node:     n,
		serv:     serv,
		buf:      NewRSNCFromBytes(pb.GetData()),
		promises: promises,
		ctx:      fctx,
		cancel:   cancel,
		pbdata:   pb,
	}
}

// precalcNextBuf follows the next link in line and loads it from the DAGService,
// setting the next buffer to read from
func (dr *DagReader) precalcNextBuf() error {
	dr.buf.Close() // Just to make sure
	if dr.linkPosition >= len(dr.promises) {
		return io.EOF
	}
	nxt, err := dr.promises[dr.linkPosition].Get()
	if err != nil {
		return err
	}
	dr.linkPosition++

	pb := new(ftpb.Data)
	err = proto.Unmarshal(nxt.Data, pb)
	if err != nil {
		return err
	}

	switch pb.GetType() {
	case ftpb.Data_Directory:
		// A directory should not exist within a file
		return ft.ErrInvalidDirLocation
	case ftpb.Data_File:
		dr.buf = newDataFileReader(dr.ctx, nxt, pb, dr.serv)
		return nil
	case ftpb.Data_Raw:
		dr.buf = NewRSNCFromBytes(pb.GetData())
		return nil
	case ftpb.Data_Metadata:
		return errors.New("Shouldnt have had metadata object inside file")
	default:
		return ft.ErrUnrecognizedType
	}
}

// Size return the total length of the data from the DAG structured file.
func (dr *DagReader) Size() int64 {
	return int64(dr.pbdata.GetFilesize())
}

// Read reads data from the DAG structured file
func (dr *DagReader) Read(b []byte) (int, error) {
	// If no cached buffer, load one
	total := 0
	for {
		// Attempt to fill bytes from cached buffer
		n, err := dr.buf.Read(b[total:])
		total += n
		dr.offset += int64(n)
		if err != nil {
			// EOF is expected
			if err != io.EOF {
				return total, err
			}
		}

		// If weve read enough bytes, return
		if total == len(b) {
			return total, nil
		}

		// Otherwise, load up the next block
		err = dr.precalcNextBuf()
		if err != nil {
			return total, err
		}
	}
}

func (dr *DagReader) WriteTo(w io.Writer) (int64, error) {
	// If no cached buffer, load one
	total := int64(0)
	for {
		// Attempt to write bytes from cached buffer
		n, err := dr.buf.WriteTo(w)
		total += n
		dr.offset += n
		if err != nil {
			if err != io.EOF {
				return total, err
			}
		}

		// Otherwise, load up the next block
		err = dr.precalcNextBuf()
		if err != nil {
			if err == io.EOF {
				return total, nil
			}
			return total, err
		}
	}
}

func (dr *DagReader) Close() error {
	dr.cancel()
	return nil
}

// Seek implements io.Seeker, and will seek to a given offset in the file
// interface matches standard unix seek
// TODO: check if we can do relative seeks, to reduce the amount of dagreader
// recreations that need to happen.
func (dr *DagReader) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	case os.SEEK_SET:
		if offset < 0 {
			return -1, errors.New("Invalid offset")
		}

		// Grab cached protobuf object (solely to make code look cleaner)
		pb := dr.pbdata

		// left represents the number of bytes remaining to seek to (from beginning)
		left := offset
		if int64(len(pb.Data)) >= offset {
			// Close current buf to close potential child dagreader
			dr.buf.Close()
			dr.buf = NewRSNCFromBytes(pb.GetData()[offset:])

			// start reading links from the beginning
			dr.linkPosition = 0
			dr.offset = offset
			return offset, nil
		} else {
			// skip past root block data
			left -= int64(len(pb.Data))
		}

		// iterate through links and find where we need to be
		for i := 0; i < len(pb.Blocksizes); i++ {
			if pb.Blocksizes[i] > uint64(left) {
				dr.linkPosition = i
				break
			} else {
				left -= int64(pb.Blocksizes[i])
			}
		}

		// start sub-block request
		err := dr.precalcNextBuf()
		if err != nil {
			return 0, err
		}

		// set proper offset within child readseeker
		n, err := dr.buf.Seek(left, os.SEEK_SET)
		if err != nil {
			return -1, err
		}

		// sanity
		left -= n
		if left != 0 {
			return -1, errors.New("failed to seek properly")
		}
		dr.offset = offset
		return offset, nil
	case os.SEEK_CUR:
		// TODO: be smarter here
		noffset := dr.offset + offset
		return dr.Seek(noffset, os.SEEK_SET)
	case os.SEEK_END:
		noffset := int64(dr.pbdata.GetFilesize()) - offset
		return dr.Seek(noffset, os.SEEK_SET)
	default:
		return 0, errors.New("invalid whence")
	}
	return 0, nil
}

// readSeekNopCloser wraps a bytes.Reader to implement ReadSeekCloser
type readSeekNopCloser struct {
	*bytes.Reader
}

func NewRSNCFromBytes(b []byte) ReadSeekCloser {
	return &readSeekNopCloser{bytes.NewReader(b)}
}

func (r *readSeekNopCloser) Close() error { return nil }
<commit_msg>better error message from dagreader with bad protofbuf<commit_after>package io

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"

	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
	mdag "github.com/jbenet/go-ipfs/merkledag"
	ft "github.com/jbenet/go-ipfs/unixfs"
	ftpb "github.com/jbenet/go-ipfs/unixfs/pb"
)

var ErrIsDir = errors.New("this dag node is a directory")

// DagReader provides a way to easily read the data contained in a dag.
type DagReader struct {
	serv mdag.DAGService

	// the node being read
	node *mdag.Node

	// cached protobuf structure from node.Data
	pbdata *ftpb.Data

	// the current data buffer to be read from
	// will either be a bytes.Reader or a child DagReader
	buf ReadSeekCloser

	// NodeGetters for each of 'nodes' child links
	promises []mdag.NodeGetter

	// the index of the child link currently being read from
	linkPosition int

	// current offset for the read head within the 'file'
	offset int64

	// Our context
	ctx context.Context

	// context cancel for children
	cancel func()
}

type ReadSeekCloser interface {
	io.Reader
	io.Seeker
	io.Closer
	io.WriterTo
}

// NewDagReader creates a new reader object that reads the data represented by the given
// node, using the passed in DAGService for data retreival
func NewDagReader(ctx context.Context, n *mdag.Node, serv mdag.DAGService) (*DagReader, error) {
	pb := new(ftpb.Data)
	err := proto.Unmarshal(n.Data, pb)
	if err != nil {
		return nil, err
	}

	switch pb.GetType() {
	case ftpb.Data_Directory:
		// Dont allow reading directories
		return nil, ErrIsDir
	case ftpb.Data_Raw:
		fallthrough
	case ftpb.Data_File:
		return newDataFileReader(ctx, n, pb, serv), nil
	case ftpb.Data_Metadata:
		if len(n.Links) == 0 {
			return nil, errors.New("incorrectly formatted metadata object")
		}
		child, err := n.Links[0].GetNode(serv)
		if err != nil {
			return nil, err
		}
		return NewDagReader(ctx, child, serv)
	default:
		return nil, ft.ErrUnrecognizedType
	}
}

func newDataFileReader(ctx context.Context, n *mdag.Node, pb *ftpb.Data, serv mdag.DAGService) *DagReader {
	fctx, cancel := context.WithCancel(ctx)
	promises := serv.GetDAG(fctx, n)
	return &DagReader{
		node:     n,
		serv:     serv,
		buf:      NewRSNCFromBytes(pb.GetData()),
		promises: promises,
		ctx:      fctx,
		cancel:   cancel,
		pbdata:   pb,
	}
}

// precalcNextBuf follows the next link in line and loads it from the DAGService,
// setting the next buffer to read from
func (dr *DagReader) precalcNextBuf() error {
	dr.buf.Close() // Just to make sure
	if dr.linkPosition >= len(dr.promises) {
		return io.EOF
	}
	nxt, err := dr.promises[dr.linkPosition].Get()
	if err != nil {
		return err
	}
	dr.linkPosition++

	pb := new(ftpb.Data)
	err = proto.Unmarshal(nxt.Data, pb)
	if err != nil {
		return fmt.Errorf("incorrectly formatted protobuf: %s", err)
	}

	switch pb.GetType() {
	case ftpb.Data_Directory:
		// A directory should not exist within a file
		return ft.ErrInvalidDirLocation
	case ftpb.Data_File:
		dr.buf = newDataFileReader(dr.ctx, nxt, pb, dr.serv)
		return nil
	case ftpb.Data_Raw:
		dr.buf = NewRSNCFromBytes(pb.GetData())
		return nil
	case ftpb.Data_Metadata:
		return errors.New("Shouldnt have had metadata object inside file")
	default:
		return ft.ErrUnrecognizedType
	}
}

// Size return the total length of the data from the DAG structured file.
func (dr *DagReader) Size() int64 {
	return int64(dr.pbdata.GetFilesize())
}

// Read reads data from the DAG structured file
func (dr *DagReader) Read(b []byte) (int, error) {
	// If no cached buffer, load one
	total := 0
	for {
		// Attempt to fill bytes from cached buffer
		n, err := dr.buf.Read(b[total:])
		total += n
		dr.offset += int64(n)
		if err != nil {
			// EOF is expected
			if err != io.EOF {
				return total, err
			}
		}

		// If weve read enough bytes, return
		if total == len(b) {
			return total, nil
		}

		// Otherwise, load up the next block
		err = dr.precalcNextBuf()
		if err != nil {
			return total, err
		}
	}
}

func (dr *DagReader) WriteTo(w io.Writer) (int64, error) {
	// If no cached buffer, load one
	total := int64(0)
	for {
		// Attempt to write bytes from cached buffer
		n, err := dr.buf.WriteTo(w)
		total += n
		dr.offset += n
		if err != nil {
			if err != io.EOF {
				return total, err
			}
		}

		// Otherwise, load up the next block
		err = dr.precalcNextBuf()
		if err != nil {
			if err == io.EOF {
				return total, nil
			}
			return total, err
		}
	}
}

func (dr *DagReader) Close() error {
	dr.cancel()
	return nil
}

// Seek implements io.Seeker, and will seek to a given offset in the file
// interface matches standard unix seek
// TODO: check if we can do relative seeks, to reduce the amount of dagreader
// recreations that need to happen.
func (dr *DagReader) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	case os.SEEK_SET:
		if offset < 0 {
			return -1, errors.New("Invalid offset")
		}

		// Grab cached protobuf object (solely to make code look cleaner)
		pb := dr.pbdata

		// left represents the number of bytes remaining to seek to (from beginning)
		left := offset
		if int64(len(pb.Data)) >= offset {
			// Close current buf to close potential child dagreader
			dr.buf.Close()
			dr.buf = NewRSNCFromBytes(pb.GetData()[offset:])

			// start reading links from the beginning
			dr.linkPosition = 0
			dr.offset = offset
			return offset, nil
		} else {
			// skip past root block data
			left -= int64(len(pb.Data))
		}

		// iterate through links and find where we need to be
		for i := 0; i < len(pb.Blocksizes); i++ {
			if pb.Blocksizes[i] > uint64(left) {
				dr.linkPosition = i
				break
			} else {
				left -= int64(pb.Blocksizes[i])
			}
		}

		// start sub-block request
		err := dr.precalcNextBuf()
		if err != nil {
			return 0, err
		}

		// set proper offset within child readseeker
		n, err := dr.buf.Seek(left, os.SEEK_SET)
		if err != nil {
			return -1, err
		}

		// sanity
		left -= n
		if left != 0 {
			return -1, errors.New("failed to seek properly")
		}
		dr.offset = offset
		return offset, nil
	case os.SEEK_CUR:
		// TODO: be smarter here
		noffset := dr.offset + offset
		return dr.Seek(noffset, os.SEEK_SET)
	case os.SEEK_END:
		noffset := int64(dr.pbdata.GetFilesize()) - offset
		return dr.Seek(noffset, os.SEEK_SET)
	default:
		return 0, errors.New("invalid whence")
	}
	return 0, nil
}

// readSeekNopCloser wraps a bytes.Reader to implement ReadSeekCloser
type readSeekNopCloser struct {
	*bytes.Reader
}

func NewRSNCFromBytes(b []byte) ReadSeekCloser {
	return &readSeekNopCloser{bytes.NewReader(b)}
}

func (r *readSeekNopCloser) Close() error { return nil }
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"go/format"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

var (
	delimiterLeft  = *flag.String("dl", "{{", "Left delimiter")
	delimiterRight = *flag.String("dr", "}}", "Right delimiter")
	suffix         = *flag.String("suffix", "gohtml", "The GoHTML templates file suffix")
)

func main() {
	flag.Parse()
	// 如果命令行参数为空，那么输出帮助
	if flag.Arg(0) == "" {
		fmt.Println(`
 ██████╗         ██╗  ██╗████████╗███╗   ███╗██╗
██╔════╝  █████╗ ██║  ██║╚══██╔══╝████╗ ████║██║
██║  ███╗██╔══██╗███████║   ██║   ██╔████╔██║██║
██║   ██║██║  ██║██╔══██║   ██║   ██║╚██╔╝██║██║
╚██████╔╝╚█████╔╝██║  ██║   ██║   ██║ ╚═╝ ██║███████╗
 ╚═════╝  ╚════╝ ╚═╝  ╚═╝   ╚═╝   ╚═╝     ╚═╝╚══════╝
=====================================================
GoHTML使用帮助:
===============

命令结构:
  命令 [参数] <模板文件夹路径>

参数：
  -dl <字符串>     | 默认：{{     | 左分隔符样式
  -dr <字符串>     | 默认：}}     | 右分隔符样式
  -suffix <字符串> | 默认：gohtml | GoHTML模板文件后缀

举例：
  gohtml -dl <{ -dr }> -suffix temp /home/bluek404/gocode/web/view
   |则会将
   |/home/bluek404/gocode/web/view
   |里面所有temp为后缀的文件转换为go文件后放到同一文件夹内
   |并设置左分隔符为“<{” ，右分割符为“}>”

备注：
  方括号[]为选填项目，尖括号<>为必填项目`)
		return
	}
	// 获取文件夹位置参数
	folder := flag.Arg(0)
	info, err := os.Lstat(folder)
	if err != nil {
		log.Println(err)
		return
	}
	if !info.IsDir() {
		log.Println("所输入路径不是文件夹")
		return
	}

	err = filepath.Walk(folder, walk)
	if err != nil {
		log.Println(err)
		return
	}
	fmt.Println("-==处理完成==-")
}

func walk(path string, info os.FileInfo, err error) error {
	if info == nil {
		return err
	}
	if info.IsDir() {
		return nil
	}
	goHTML := regexp.MustCompile("." + suffix + "$")
	// 检查是否为gohtml文件
	if !goHTML.MatchString(info.Name()) {
		return nil
	}
	buf, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	buf, err = format.Source([]byte(generate(string(buf))))
	if err != nil {
		return fmt.Errorf("Format error: %v %v", path, err))
	}
	// 将文件后缀变为.go
	outPath := path[:len(path)-len(suffix)] + "go"
	// 输出文件
	ioutil.WriteFile(outPath, buf, 0)
	fmt.Println(path, "==>", outPath)
	return nil
}

func generate(in string) string {
	var _buffer bytes.Buffer

	isHTML := regexp.MustCompile(`^\s*<.*>\s*$`)
	delimiterLeftLen := len(delimiterLeft)
	delimiterRightLen := len(delimiterRight)
	// 分隔符
	delimiter := regexp.MustCompile(delimiterLeft + ".*" + delimiterRight)

	a := regexp.MustCompile(`^\s*`)
	z := regexp.MustCompile(`\s*$`)

	var htmlBUF string

	r := new(readLine)
	for {
		// 按行读取处理
		buf, ok := r.read(in)

		// 检查本行是否为HTML
		if isHTML.MatchString(buf) {
			// 去首尾空
			buf = a.ReplaceAllLiteralString(buf, "")
			buf = z.ReplaceAllLiteralString(buf, "")
			// 转义双引号
			buf = strings.Replace(buf, `"`, `\"`, -1)
			// 检查是否有分隔符需要替换
			if delimiter.MatchString(buf) {
				// 替换分隔符
				// 找到本行全部分隔符
				delimiterBUF := delimiter.FindAllString(buf, -1)
				for _, v := range delimiterBUF {
					// 去掉两边分隔符
					vBUF := v[delimiterLeftLen : len(v)-delimiterRightLen]
					// 组合插入
					vBUF = "\")\n_buffer.WriteString(" + vBUF + ")\n_buffer.WriteString(\""
					// 替换
					buf = strings.Replace(buf, v, vBUF, -1)
				}
			}
			// 将本行添加到缓存中，输出为一行
			// 就是将两行
			// _buffer.WriteString('xxx')
			// _buffer.WriteString('xxx')
			// 简化为一行
			// _buffer.WriteString('xxxxxx')
			htmlBUF += buf
		} else {
			// 本行为Golang
			// 检查是否有html需要输出
			if htmlBUF != "" {
				// 输出html
				_buffer.WriteString("_buffer.WriteString(\"" + htmlBUF + "\")\n")
				// 清空缓存
				htmlBUF = ""
			}
			_buffer.WriteString(buf + "\n")
		}
		// 检查是否已经读取完毕
		if ok {
			break
		}
	}
	return _buffer.String()
}

type readLine struct {
	// 用于记录每行的起始位置
	start int
	end   int
}

// 按行输出字符串
func (r *readLine) read(str string) (string, bool) {
	// 找到换行符位置
	n := strings.Index(str[r.start:], "\n")
	if n == -1 {
		// 无法找到换行符，已经到达最后一行
		// 返回ok
		line := string(str[r.start:])
		r.start = 0
		r.end = 0
		return line, true
	}
	// 行的结束位置
	r.end = r.start + n
	// 提取一行
	line := string(str[r.start:r.end])
	// 下一行的开始位置
	r.start = r.end + 1

	return line, false
}
<commit_msg>删除多余内容<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/format"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

var (
	delimiterLeft  = *flag.String("dl", "{{", "Left delimiter")
	delimiterRight = *flag.String("dr", "}}", "Right delimiter")
	suffix         = *flag.String("suffix", "gohtml", "The GoHTML templates file suffix")
)

func main() {
	flag.Parse()
	// 如果命令行参数为空，那么输出帮助
	if flag.Arg(0) == "" {
		fmt.Println(`
 ██████╗         ██╗  ██╗████████╗███╗   ███╗██╗
██╔════╝  █████╗ ██║  ██║╚══██╔══╝████╗ ████║██║
██║  ███╗██╔══██╗███████║   ██║   ██╔████╔██║██║
██║   ██║██║  ██║██╔══██║   ██║   ██║╚██╔╝██║██║
╚██████╔╝╚█████╔╝██║  ██║   ██║   ██║ ╚═╝ ██║███████╗
 ╚═════╝  ╚════╝ ╚═╝  ╚═╝   ╚═╝   ╚═╝     ╚═╝╚══════╝
=====================================================
GoHTML使用帮助:
===============

命令结构:
  命令 [参数] <模板文件夹路径>

参数：
  -dl <字符串>     | 默认：{{     | 左分隔符样式
  -dr <字符串>     | 默认：}}     | 右分隔符样式
  -suffix <字符串> | 默认：gohtml | GoHTML模板文件后缀

举例：
  gohtml -dl <{ -dr }> -suffix temp /home/bluek404/gocode/web/view
   |则会将
   |/home/bluek404/gocode/web/view
   |里面所有temp为后缀的文件转换为go文件后放到同一文件夹内
   |并设置左分隔符为“<{” ，右分割符为“}>”

备注：
  方括号[]为选填项目，尖括号<>为必填项目`)
		return
	}
	// 获取文件夹位置参数
	folder := flag.Arg(0)
	info, err := os.Lstat(folder)
	if err != nil {
		log.Println(err)
		return
	}
	if !info.IsDir() {
		log.Println("所输入路径不是文件夹")
		return
	}

	err = filepath.Walk(folder, walk)
	if err != nil {
		log.Println(err)
		return
	}
	fmt.Println("-==处理完成==-")
}

func walk(path string, info os.FileInfo, err error) error {
	if info == nil {
		return err
	}
	if info.IsDir() {
		return nil
	}
	goHTML := regexp.MustCompile("." + suffix + "$")
	// 检查是否为gohtml文件
	if !goHTML.MatchString(info.Name()) {
		return nil
	}
	buf, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	buf, err = format.Source([]byte(generate(string(buf))))
	if err != nil {
		return fmt.Errorf("Format error: %v %v", path, err)
	}
	// 将文件后缀变为.go
	outPath := path[:len(path)-len(suffix)] + "go"
	// 输出文件
	ioutil.WriteFile(outPath, buf, 0)
	fmt.Println(path, "==>", outPath)
	return nil
}

func generate(in string) string {
	var _buffer bytes.Buffer

	isHTML := regexp.MustCompile(`^\s*<.*>\s*$`)
	delimiterLeftLen := len(delimiterLeft)
	delimiterRightLen := len(delimiterRight)
	// 分隔符
	delimiter := regexp.MustCompile(delimiterLeft + ".*" + delimiterRight)

	a := regexp.MustCompile(`^\s*`)
	z := regexp.MustCompile(`\s*$`)

	var htmlBUF string

	r := new(readLine)
	for {
		// 按行读取处理
		buf, ok := r.read(in)

		// 检查本行是否为HTML
		if isHTML.MatchString(buf) {
			// 去首尾空
			buf = a.ReplaceAllLiteralString(buf, "")
			buf = z.ReplaceAllLiteralString(buf, "")
			// 转义双引号
			buf = strings.Replace(buf, `"`, `\"`, -1)
			// 检查是否有分隔符需要替换
			if delimiter.MatchString(buf) {
				// 替换分隔符
				// 找到本行全部分隔符
				delimiterBUF := delimiter.FindAllString(buf, -1)
				for _, v := range delimiterBUF {
					// 去掉两边分隔符
					vBUF := v[delimiterLeftLen : len(v)-delimiterRightLen]
					// 组合插入
					vBUF = "\")\n_buffer.WriteString(" + vBUF + ")\n_buffer.WriteString(\""
					// 替换
					buf = strings.Replace(buf, v, vBUF, -1)
				}
			}
			// 将本行添加到缓存中，输出为一行
			// 就是将两行
			// _buffer.WriteString('xxx')
			// _buffer.WriteString('xxx')
			// 简化为一行
			// _buffer.WriteString('xxxxxx')
			htmlBUF += buf
		} else {
			// 本行为Golang
			// 检查是否有html需要输出
			if htmlBUF != "" {
				// 输出html
				_buffer.WriteString("_buffer.WriteString(\"" + htmlBUF + "\")\n")
				// 清空缓存
				htmlBUF = ""
			}
			_buffer.WriteString(buf + "\n")
		}
		// 检查是否已经读取完毕
		if ok {
			break
		}
	}
	return _buffer.String()
}

type readLine struct {
	// 用于记录每行的起始位置
	start int
	end   int
}

// 按行输出字符串
func (r *readLine) read(str string) (string, bool) {
	// 找到换行符位置
	n := strings.Index(str[r.start:], "\n")
	if n == -1 {
		// 无法找到换行符，已经到达最后一行
		// 返回ok
		line := string(str[r.start:])
		r.start = 0
		r.end = 0
		return line, true
	}
	// 行的结束位置
	r.end = r.start + n
	// 提取一行
	line := string(str[r.start:r.end])
	// 下一行的开始位置
	r.start = r.end + 1

	return line, false
}
<|endoftext|>


<commit_before>package goapns

import (
	"time"
)

const PriorityHigh = 10
const PriorityLow = 5

type Header struct {
	APNSID     string
	Expiration time.Time
	Priority   int
	Topic      string
}

func NewHeader() Header {
	h := Header{"", time.Time{}, PriorityHigh, ""}
	return h
}
<commit_msg>Documented Header.go<commit_after>package goapns

import (
	"time"
)

//PriorityHigh sepcifies a high priority for the notification.
//Default priotity is PriorityHigh so if you don't specify any priority,
//PriorityHigh is assumed.
//
//It sends the push message immediately.
//Notifications with this priority must trigger an alert, sound, or badge on the target device.
//It is an error to use this priority for a push notification that contains only the content-available key
const PriorityHigh = 10

//PriorityLow sepcifies a low priority for the notification.
//Default priotity is PriorityHigh so if you don't specify any priority,
//PriorityHigh is assumed.
//
//Send the push message at a time that takes into account power considerations for the device.
//Notifications with this priority might be grouped and delivered in bursts.
//They are throttled, and in some cases are not delivered.
const PriorityLow = 5

//Header collects the header fields for the notification.
type Header struct {

	//APNSID is a canonical UUID that identifies the notification.
	//If there is an error sending the notification, APNs uses this value to identify the notification to your server.
	//The canonical form is 32 lowercase hexadecimal digits, displayed in five groups separated by hyphens in the form 8-4-4-4-12.
	//An example UUID is as follows:
	//123e4567-e89b-12d3-a456-42665544000
	//If you omit this header, a new UUID is created by APNs and returned in the response.
	APNSID string

	//Expiration is a UNIX epoch date expressed in seconds (UTC).
	//This header identifies the date when the notification is no longer valid and can be discarded.
	//If this value is nonzero, APNs stores the notification and tries to deliver it at least once,
	//repeating the attempt as needed if it is unable to deliver the notification the first time.
	//If the value is 0, APNs treats the notification as if it expires immediately and does not store the notification or attempt to redeliver it.
	Expiration time.Time

	//Priority is the priority of the notification.
	//Set it to PriorityHigh or PriorityLow but do not
	//use custom numbers in any case.
	Priority int

	//Topic of the remote notification, which is typically the bundle ID for your app.
	//The certificate you create in Member Center must include the capability for this topic.
	//If your certificate includes multiple topics, you must specify a value for this header.
	//If you omit this header and your APNs certificate does not specify multiple topics,
	//the APNs server uses the certificate’s Subject as the default topic.
	Topic string
}

//NewHeader creates a new Header with high priority.
func NewHeader() Header {
	h := Header{"", time.Time{}, PriorityHigh, ""}
	return h
}
<|endoftext|>


<commit_before><commit_msg>Always try to get current volume free space<commit_after><|endoftext|>


<commit_before>package javascript

import (
	"encoding/json"
	"fmt"
	"net/url"
	"os"
	"path/filepath"

	"strings"

	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/container"
	"sourcegraph.com/sourcegraph/srcgraph/dep2"
	"sourcegraph.com/sourcegraph/srcgraph/scan"
	"sourcegraph.com/sourcegraph/srcgraph/unit"
)

func init() {
	scan.Register("npm", scan.DockerScanner{defaultNPM})
	dep2.RegisterLister(&CommonJSPackage{}, defaultNPM)
	dep2.RegisterResolver(npmDependencyTargetType, dep2.DockerResolver{defaultNPM})
}

const nodeStdlibRepoURL = "git://github.com/joyent/node.git"

type nodeVersion struct{}

type npmVersion struct{ nodeVersion }

var (
	defaultNode = nodeVersion{}
	defaultNPM  = &npmVersion{defaultNode}
)

func (_ *nodeVersion) baseDockerfile() ([]byte, error) {
	return []byte(baseNPMDockerfile), nil
}

const baseNPMDockerfile = `FROM ubuntu:14.04
RUN apt-get update -qq
RUN apt-get install -qqy nodejs npm git`

// containerDir returns the directory in the Docker container to use for the
// local directory dir.
func containerDir(dir string) string {
	return filepath.Join("/tmp/sg", filepath.Base(dir))
}

func (v *npmVersion) BuildScanner(dir string, c *config.Repository) (*container.Command, error) {
	dockerfile, err := v.baseDockerfile()
	if err != nil {
		return nil, err
	}

	const (
		findpkgsNPM = "commonjs-findpkgs@0.0.3"
		findpkgsGit = "git://github.com/sourcegraph/commonjs-findpkgs.git"
		findpkgsSrc = findpkgsNPM
	)
	dockerfile = append(dockerfile, []byte("\n\nRUN npm install --quiet -g "+findpkgsSrc+"\n")...)

	containerDir := containerDir(dir)
	cont := container.Container{
		Dockerfile: dockerfile,
		RunOptions: []string{"-v", dir + ":" + containerDir},
		Cmd:        []string{"commonjs-findpkgs"},
		Dir:        containerDir,
	}
	cmd := container.Command{
		Container: cont,
		Transform: func(orig []byte) ([]byte, error) {
			var pkgs []*CommonJSPackage
			err := json.Unmarshal(orig, &pkgs)
			if err != nil {
				return nil, err
			}

			return json.Marshal(pkgs)
		},
	}
	return &cmd, nil
}

func (v *npmVersion) UnmarshalSourceUnits(data []byte) ([]unit.SourceUnit, error) {
	if len(data) == 0 {
		return nil, nil
	}

	var npmPackages []*CommonJSPackage
	err := json.Unmarshal(data, &npmPackages)
	if err != nil {
		return nil, err
	}

	units := make([]unit.SourceUnit, len(npmPackages))
	for i, p := range npmPackages {
		units[i] = p
	}

	return units, nil
}

// npmDependency is a name/version pair that represents an NPM dependency. This
// pair corresponds to the object property/value pairs in package.json
// "dependency" objects.
type npmDependency struct {
	// Name is the package name of the dependency.
	Name string

	// Spec is the specifier of the version, which can be an NPM version number,
	// a tarball URL, a git/hg clone URL, etc.
	Spec string
}

const npmDependencyTargetType = "npm-dep"

func (v *npmVersion) BuildResolver(dep *dep2.RawDependency, c *config.Repository) (*container.Command, error) {
	var npmDep npmDependency
	j, _ := json.Marshal(dep.Target)
	json.Unmarshal(j, &npmDep)

	dockerfile, err := v.baseDockerfile()
	if err != nil {
		return nil, err
	}
	dockerfile = append(dockerfile, []byte("\n\nRUN npm install --quiet -g deptool@~0.0.2\n")...)

	cmd := container.Command{
		Container: container.Container{
			Dockerfile: dockerfile,
			Cmd:        []string{"nodejs", "/usr/local/bin/npm-deptool", npmDep.Name + "@" + npmDep.Spec},
		},
		Transform: func(orig []byte) ([]byte, error) {
			// resolvedDep is output from npm-deptool.
			type npmDeptoolOutput struct {
				ResolvedURL string `json:"_resolved"`
				ID          string `json:"_id"`
				Repository  struct {
					Type string
					URL  string
				}
			}
			var resolvedDeps map[string]npmDeptoolOutput
			err := json.Unmarshal(orig, &resolvedDeps)
			if err != nil {
				return nil, err
			}

			if len(resolvedDeps) == 0 {
				return nil, fmt.Errorf("npm-deptool did not output anything for raw dependency %+v", dep)
			}
			if len(resolvedDeps) != 1 {
				return nil, fmt.Errorf("npm-deptool unexpectedly returned %d deps for raw dependency %+v", len(resolvedDeps), dep)
			}

			var resolvedDep npmDeptoolOutput
			for _, v := range resolvedDeps {
				resolvedDep = v
			}

			var toRepoCloneURL, toRevSpec string
			if strings.HasPrefix(resolvedDep.ResolvedURL, "https://registry.npmjs.org/") {
				// known npm package, so the repository refers to it
				toRepoCloneURL = resolvedDep.Repository.URL
			} else {
				// external tarball, git repo url, etc., so the repository might
				// refer to the source repo (if this is a fork) or not be
				// present at all
				u, err := url.Parse(resolvedDep.ResolvedURL)
				if err != nil {
					return nil, err
				}
				toRevSpec = u.Fragment

				u.Fragment = ""
				toRepoCloneURL = u.String()
			}

			return json.Marshal(&dep2.ResolvedTarget{
				ToRepoCloneURL:  toRepoCloneURL,
				ToUnitType:      unit.Type((&CommonJSPackage{})),
				ToUnit:          ".",
				ToVersionString: resolvedDep.ID,
				ToRevSpec:       toRevSpec,
			})
		},
	}
	return &cmd, nil
}

// List reads the "dependencies" key in the NPM package's package.json file and
// outputs the properties as raw dependencies.
func (v *npmVersion) List(dir string, unit unit.SourceUnit, c *config.Repository) ([]*dep2.RawDependency, error) {
	pkg := unit.(*CommonJSPackage)

	if pkg.PackageJSONFile == "" {
		// No package.json file, so we won't be able to find any dependencies anyway.
		return nil, nil
	}

	pkgFile := filepath.Join(dir, pkg.PackageJSONFile)

	f, err := os.Open(pkgFile)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	var pkgjson struct {
		Dependencies    map[string]string `json:"dependencies"`
		DevDependencies map[string]string `json:"devDependencies"`
	}
	err = json.NewDecoder(f).Decode(&pkgjson)
	if err != nil {
		return nil, err
	}

	rawDeps := make([]*dep2.RawDependency, len(pkgjson.Dependencies)+len(pkgjson.DevDependencies))
	i := 0
	addDeps := func(deps map[string]string) {
		for name, spec := range deps {
			rawDeps[i] = &dep2.RawDependency{
				FromFile:   pkg.PackageJSONFile,
				TargetType: npmDependencyTargetType,
				Target:     npmDependency{Name: name, Spec: spec},
			}
			i++
		}
	}
	addDeps(pkgjson.Dependencies)
	addDeps(pkgjson.DevDependencies)

	return rawDeps, nil
}
<commit_msg>fix gyp and "node" binary issues in nodejs toolchain<commit_after>package javascript

import (
	"encoding/json"
	"fmt"
	"net/url"
	"os"
	"path/filepath"

	"strings"

	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/container"
	"sourcegraph.com/sourcegraph/srcgraph/dep2"
	"sourcegraph.com/sourcegraph/srcgraph/scan"
	"sourcegraph.com/sourcegraph/srcgraph/unit"
)

func init() {
	scan.Register("npm", scan.DockerScanner{defaultNPM})
	dep2.RegisterLister(&CommonJSPackage{}, defaultNPM)
	dep2.RegisterResolver(npmDependencyTargetType, dep2.DockerResolver{defaultNPM})
}

const nodeStdlibRepoURL = "git://github.com/joyent/node.git"

type nodeVersion struct{}

type npmVersion struct{ nodeVersion }

var (
	defaultNode = nodeVersion{}
	defaultNPM  = &npmVersion{defaultNode}
)

func (_ *nodeVersion) baseDockerfile() ([]byte, error) {
	return []byte(baseNPMDockerfile), nil
}

const baseNPMDockerfile = `FROM ubuntu:14.04
RUN apt-get update -qq
RUN apt-get install -qqy nodejs node-gyp npm git

# Some NPM modules expect the node.js interpreter to be "node", not "nodejs" (as
# it is on Ubuntu).
RUN ln -s /usr/bin/nodejs /usr/bin/node
`

// containerDir returns the directory in the Docker container to use for the
// local directory dir.
func containerDir(dir string) string {
	return filepath.Join("/tmp/sg", filepath.Base(dir))
}

func (v *npmVersion) BuildScanner(dir string, c *config.Repository) (*container.Command, error) {
	dockerfile, err := v.baseDockerfile()
	if err != nil {
		return nil, err
	}

	const (
		findpkgsNPM = "commonjs-findpkgs@0.0.3"
		findpkgsGit = "git://github.com/sourcegraph/commonjs-findpkgs.git"
		findpkgsSrc = findpkgsNPM
	)
	dockerfile = append(dockerfile, []byte("\n\nRUN npm install --quiet -g "+findpkgsSrc+"\n")...)

	containerDir := containerDir(dir)
	cont := container.Container{
		Dockerfile: dockerfile,
		RunOptions: []string{"-v", dir + ":" + containerDir},
		Cmd:        []string{"commonjs-findpkgs"},
		Dir:        containerDir,
	}
	cmd := container.Command{
		Container: cont,
		Transform: func(orig []byte) ([]byte, error) {
			var pkgs []*CommonJSPackage
			err := json.Unmarshal(orig, &pkgs)
			if err != nil {
				return nil, err
			}

			return json.Marshal(pkgs)
		},
	}
	return &cmd, nil
}

func (v *npmVersion) UnmarshalSourceUnits(data []byte) ([]unit.SourceUnit, error) {
	if len(data) == 0 {
		return nil, nil
	}

	var npmPackages []*CommonJSPackage
	err := json.Unmarshal(data, &npmPackages)
	if err != nil {
		return nil, err
	}

	units := make([]unit.SourceUnit, len(npmPackages))
	for i, p := range npmPackages {
		units[i] = p
	}

	return units, nil
}

// npmDependency is a name/version pair that represents an NPM dependency. This
// pair corresponds to the object property/value pairs in package.json
// "dependency" objects.
type npmDependency struct {
	// Name is the package name of the dependency.
	Name string

	// Spec is the specifier of the version, which can be an NPM version number,
	// a tarball URL, a git/hg clone URL, etc.
	Spec string
}

const npmDependencyTargetType = "npm-dep"

func (v *npmVersion) BuildResolver(dep *dep2.RawDependency, c *config.Repository) (*container.Command, error) {
	var npmDep npmDependency
	j, _ := json.Marshal(dep.Target)
	json.Unmarshal(j, &npmDep)

	dockerfile, err := v.baseDockerfile()
	if err != nil {
		return nil, err
	}
	dockerfile = append(dockerfile, []byte("\n\nRUN npm install --quiet -g deptool@~0.0.2\n")...)

	cmd := container.Command{
		Container: container.Container{
			Dockerfile: dockerfile,
			Cmd:        []string{"nodejs", "/usr/local/bin/npm-deptool", npmDep.Name + "@" + npmDep.Spec},
		},
		Transform: func(orig []byte) ([]byte, error) {
			// resolvedDep is output from npm-deptool.
			type npmDeptoolOutput struct {
				ResolvedURL string `json:"_resolved"`
				ID          string `json:"_id"`
				Repository  struct {
					Type string
					URL  string
				}
			}
			var resolvedDeps map[string]npmDeptoolOutput
			err := json.Unmarshal(orig, &resolvedDeps)
			if err != nil {
				return nil, err
			}

			if len(resolvedDeps) == 0 {
				return nil, fmt.Errorf("npm-deptool did not output anything for raw dependency %+v", dep)
			}
			if len(resolvedDeps) != 1 {
				return nil, fmt.Errorf("npm-deptool unexpectedly returned %d deps for raw dependency %+v", len(resolvedDeps), dep)
			}

			var resolvedDep npmDeptoolOutput
			for _, v := range resolvedDeps {
				resolvedDep = v
			}

			var toRepoCloneURL, toRevSpec string
			if strings.HasPrefix(resolvedDep.ResolvedURL, "https://registry.npmjs.org/") {
				// known npm package, so the repository refers to it
				toRepoCloneURL = resolvedDep.Repository.URL
			} else {
				// external tarball, git repo url, etc., so the repository might
				// refer to the source repo (if this is a fork) or not be
				// present at all
				u, err := url.Parse(resolvedDep.ResolvedURL)
				if err != nil {
					return nil, err
				}
				toRevSpec = u.Fragment

				u.Fragment = ""
				toRepoCloneURL = u.String()
			}

			return json.Marshal(&dep2.ResolvedTarget{
				ToRepoCloneURL:  toRepoCloneURL,
				ToUnitType:      unit.Type((&CommonJSPackage{})),
				ToUnit:          ".",
				ToVersionString: resolvedDep.ID,
				ToRevSpec:       toRevSpec,
			})
		},
	}
	return &cmd, nil
}

// List reads the "dependencies" key in the NPM package's package.json file and
// outputs the properties as raw dependencies.
func (v *npmVersion) List(dir string, unit unit.SourceUnit, c *config.Repository) ([]*dep2.RawDependency, error) {
	pkg := unit.(*CommonJSPackage)

	if pkg.PackageJSONFile == "" {
		// No package.json file, so we won't be able to find any dependencies anyway.
		return nil, nil
	}

	pkgFile := filepath.Join(dir, pkg.PackageJSONFile)

	f, err := os.Open(pkgFile)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	var pkgjson struct {
		Dependencies    map[string]string `json:"dependencies"`
		DevDependencies map[string]string `json:"devDependencies"`
	}
	err = json.NewDecoder(f).Decode(&pkgjson)
	if err != nil {
		return nil, err
	}

	rawDeps := make([]*dep2.RawDependency, len(pkgjson.Dependencies)+len(pkgjson.DevDependencies))
	i := 0
	addDeps := func(deps map[string]string) {
		for name, spec := range deps {
			rawDeps[i] = &dep2.RawDependency{
				FromFile:   pkg.PackageJSONFile,
				TargetType: npmDependencyTargetType,
				Target:     npmDependency{Name: name, Spec: spec},
			}
			i++
		}
	}
	addDeps(pkgjson.Dependencies)
	addDeps(pkgjson.DevDependencies)

	return rawDeps, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package events

import (
	"fmt"
	"time"

	v1 "k8s.io/api/core/v1"
	eventsv1 "k8s.io/api/events/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/clock"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/tools/record/util"
	"k8s.io/client-go/tools/reference"
	"k8s.io/klog/v2"
)

type recorderImpl struct {
	scheme              *runtime.Scheme
	reportingController string
	reportingInstance   string
	*watch.Broadcaster
	clock clock.Clock
}

func (recorder *recorderImpl) Eventf(regarding runtime.Object, related runtime.Object, eventtype, reason, action, note string, args ...interface{}) {
	timestamp := metav1.MicroTime{time.Now()}
	message := fmt.Sprintf(note, args...)
	refRegarding, err := reference.GetReference(recorder.scheme, regarding)
	if err != nil {
		klog.Errorf("Could not construct reference to: '%#v' due to: '%v'. Will not report event: '%v' '%v' '%v'", regarding, err, eventtype, reason, message)
		return
	}
	refRelated, err := reference.GetReference(recorder.scheme, related)
	if err != nil {
		klog.V(9).Infof("Could not construct reference to: '%#v' due to: '%v'.", related, err)
	}
	if !util.ValidateEventType(eventtype) {
		klog.Errorf("Unsupported event type: '%v'", eventtype)
		return
	}
	event := recorder.makeEvent(refRegarding, refRelated, timestamp, eventtype, reason, message, recorder.reportingController, recorder.reportingInstance, action)
	go func() {
		defer utilruntime.HandleCrash()
		recorder.Action(watch.Added, event)
	}()
}

func (recorder *recorderImpl) makeEvent(refRegarding *v1.ObjectReference, refRelated *v1.ObjectReference, timestamp metav1.MicroTime, eventtype, reason, message string, reportingController string, reportingInstance string, action string) *eventsv1.Event {
	t := metav1.Time{Time: recorder.clock.Now()}
	namespace := refRegarding.Namespace
	if namespace == "" {
		namespace = metav1.NamespaceDefault
	}
	return &eventsv1.Event{
		ObjectMeta: metav1.ObjectMeta{
			Name:      fmt.Sprintf("%v.%x", refRegarding.Name, t.UnixNano()),
			Namespace: namespace,
		},
		EventTime:           timestamp,
		Series:              nil,
		ReportingController: reportingController,
		ReportingInstance:   reportingInstance,
		Action:              action,
		Reason:              reason,
		Regarding:           *refRegarding,
		Related:             refRelated,
		Note:                message,
		Type:                eventtype,
		// TODO: remove this when we change conversion to convert eventSource
		// to reportingController
		DeprecatedSource: v1.EventSource{Component: reportingController},
	}
}
<commit_msg>Remove DeprecatedSource assignment to avoid validation failure<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package events

import (
	"fmt"
	"time"

	v1 "k8s.io/api/core/v1"
	eventsv1 "k8s.io/api/events/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/clock"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/tools/record/util"
	"k8s.io/client-go/tools/reference"
	"k8s.io/klog/v2"
)

type recorderImpl struct {
	scheme              *runtime.Scheme
	reportingController string
	reportingInstance   string
	*watch.Broadcaster
	clock clock.Clock
}

func (recorder *recorderImpl) Eventf(regarding runtime.Object, related runtime.Object, eventtype, reason, action, note string, args ...interface{}) {
	timestamp := metav1.MicroTime{time.Now()}
	message := fmt.Sprintf(note, args...)
	refRegarding, err := reference.GetReference(recorder.scheme, regarding)
	if err != nil {
		klog.Errorf("Could not construct reference to: '%#v' due to: '%v'. Will not report event: '%v' '%v' '%v'", regarding, err, eventtype, reason, message)
		return
	}
	refRelated, err := reference.GetReference(recorder.scheme, related)
	if err != nil {
		klog.V(9).Infof("Could not construct reference to: '%#v' due to: '%v'.", related, err)
	}
	if !util.ValidateEventType(eventtype) {
		klog.Errorf("Unsupported event type: '%v'", eventtype)
		return
	}
	event := recorder.makeEvent(refRegarding, refRelated, timestamp, eventtype, reason, message, recorder.reportingController, recorder.reportingInstance, action)
	go func() {
		defer utilruntime.HandleCrash()
		recorder.Action(watch.Added, event)
	}()
}

func (recorder *recorderImpl) makeEvent(refRegarding *v1.ObjectReference, refRelated *v1.ObjectReference, timestamp metav1.MicroTime, eventtype, reason, message string, reportingController string, reportingInstance string, action string) *eventsv1.Event {
	t := metav1.Time{Time: recorder.clock.Now()}
	namespace := refRegarding.Namespace
	if namespace == "" {
		namespace = metav1.NamespaceDefault
	}
	return &eventsv1.Event{
		ObjectMeta: metav1.ObjectMeta{
			Name:      fmt.Sprintf("%v.%x", refRegarding.Name, t.UnixNano()),
			Namespace: namespace,
		},
		EventTime:           timestamp,
		Series:              nil,
		ReportingController: reportingController,
		ReportingInstance:   reportingInstance,
		Action:              action,
		Reason:              reason,
		Regarding:           *refRegarding,
		Related:             refRelated,
		Note:                message,
		Type:                eventtype,
	}
}
<|endoftext|>


<commit_before>package webapp

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/zdebeer99/mux"
)

const (
	KeySessionId      = "SessionId"
	KeyDatabaseObject = "DatabaseObject"
	KeyUser           = "User"
)

type Handler interface {
	ServeHTTP(c *Context)
}

type HandlerFunc func(c *Context)

func (h HandlerFunc) ServeHTTP(c *Context) {
	h(c)
}

// Wrap converts a http.Handler into a negroni.Handler so it can be used as a Negroni
// middleware. The next http.HandlerFunc is automatically called after the Handler
// is executed.
func Wrap(handler Handler) MiddlewareHandler {
	return MiddlewareHandlerFunc(func(c *Context, next HandlerFunc) {
		handler.ServeHTTP(c)
		next(c)
	})
}

//WebappHandler Wrap a mux handler and calls a webapp handler
func WebappHandlerFunc(f func(*Context)) func(interface{}) {
	return func(mx interface{}) {
		c := mx.(*Context)
		f(c)
	}
}

// Negroni is a stack of Middleware Handlers that can be invoked as an http.Handler.
// Negroni middleware is evaluated in the order that they are added to the stack using
// the Use and UseHandler methods.
type Webapp struct {
	parent       *Webapp
	children     []*Webapp
	middleware   middleware
	handlers     []MiddlewareHandler
	router       *RouterContext
	RenderEngine Renderer
}

// New returns a new Negroni instance with no middleware preconfigured.
func New(handlers ...MiddlewareHandler) *Webapp {
	web := &Webapp{
		handlers:   handlers,
		middleware: build(handlers),
	}
	web.router = NewRouter()
	web.router.SetContextFactory(web.contextFactory)
	web.RenderEngine = NewJadeRender("./views")
	return web
}

func (this *Webapp) NewRoute(f func(*Context)) *mux.Route {
	return this.router.NewRoute().HandlerFunc(WebappHandlerFunc(f))
}

func (this *Webapp) Handle(path string, handler Handler) *mux.Route {
	return this.router.Handle(path, NewMuxHandlerAdapter(handler))
}

func (this *Webapp) HandleFunc(path string, f func(*Context)) *mux.Route {
	return this.router.HandleFunc(path, WebappHandlerFunc(f))
}

func (this *Webapp) Get(path string, f func(*Context)) *mux.Route {
	return this.router.HandleFunc(path, WebappHandlerFunc(f)).Methods("GET")
}

func (this *Webapp) Post(path string, f func(*Context)) *mux.Route {
	return this.router.HandleFunc(path, WebappHandlerFunc(f)).Methods("POST")
}

func (this *Webapp) FileServer(path_prefix string, file_path string) {
	this.router.FileServer(path_prefix, file_path)
}

// Classic returns a new Negroni instance with the default middleware already
// in the stack.
//
// Recovery - Panic Recovery Middleware
// Logger - Request/Response Logging
func Classic() *Webapp {
	return New(NewRecovery(), NewLogger())
}

func (n *Webapp) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	c := NewContext(n, rw, r)
	n.ServeHTTPContext(c)
}

func (n *Webapp) ServeHTTPContext(c *Context) {
	n.middleware.ServeHTTP(c)
}

// Use adds a Handler onto the middleware stack. Handlers are invoked in the order they are added to a Negroni.
func (n *Webapp) Use(handler MiddlewareHandler) {
	n.handlers = append(n.handlers, handler)
	n.middleware = build(n.handlers)
}

// UseFunc adds a Negroni-style handler function onto the middleware stack.
func (n *Webapp) UseFunc(handlerFunc func(c *Context, next HandlerFunc)) {
	n.Use(MiddlewareHandlerFunc(handlerFunc))
}

// UseHandler adds a http.Handler onto the middleware stack. Handlers are invoked in the order they are added to a Negroni.
func (n *Webapp) UseHandler(handler Handler) {
	n.Use(Wrap(handler))
}

// UseHandler adds a http.HandlerFunc-style handler function onto the middleware stack.
func (n *Webapp) UseHandlerFunc(handlerFunc func(c *Context)) {
	n.UseHandler(HandlerFunc(handlerFunc))
}

// Run is a convenience function that runs the negroni stack as an HTTP
// server. The addr string takes the same format as http.ListenAndServe.
func (this *Webapp) Run(addr string) {
	l := log.New(os.Stdout, "[webapp] ", 0)
	for _, child := range this.children {
		child.Run("")
	}
	if this.parent != nil {
		this.UseHandler(this.router)
		l.Printf("listening for subrouter")
	} else {
		this.UseHandler(this.router)
		l.Printf("listening on %s", addr)
		l.Fatal(http.ListenAndServe(addr, this))
	}
}

// Returns a list of all the handlers in the current Negroni middleware chain.
func (n *Webapp) Handlers() []MiddlewareHandler {
	return n.handlers
}

func (this *Webapp) SubRoute(path string) *Webapp {
	web := this.newChild()
	web.router = NewRouterBase(mux.NewRouter().PathPrefix(path).Subrouter())
	this.NewRoute(func(c *Context) {
		fmt.Println("HELLO", c.Route.GetName())
		web.ServeHTTPContext(c)
	}).PathPrefix(path)
	return web
}

func (this *Webapp) newChild() *Webapp {
	web := New()
	web.parent = this
	this.children = append(this.children, web)
	return web
}

func build(handlers []MiddlewareHandler) middleware {
	var next middleware

	if len(handlers) == 0 {
		return voidMiddleware()
	} else if len(handlers) > 1 {
		next = build(handlers[1:])
	} else {
		next = voidMiddleware()
	}

	return middleware{handlers[0], &next}
}

func voidMiddleware() middleware {
	return middleware{
		MiddlewareHandlerFunc(func(c *Context, next HandlerFunc) {}),
		&middleware{},
	}
}

func (this *Webapp) contextFactory(w http.ResponseWriter, req *http.Request) interface{} {
	c := NewContext(this, w, req)
	c.app = this
	return c
}

//RouterContext Wrap mux.Router to support ServeHTTP(*Context)
type RouterContext struct {
	*mux.Router
}

//NewRouter Create a new mux router adapter
func NewRouter() *RouterContext {
	return &RouterContext{mux.NewRouter()}
}

//NewRouter Create a new mux router adapter
func NewRouterBase(router *mux.Router) *RouterContext {
	return &RouterContext{router}
}

//Wrapped ServeHttp
func (this *RouterContext) ServeHTTP(c *Context) {
	this.Router.ServeHTTPContext(c)
}
<commit_msg>Removed HELLO print.<commit_after>package webapp

import (
	"log"
	"net/http"
	"os"

	"github.com/zdebeer99/mux"
)

const (
	KeySessionId      = "SessionId"
	KeyDatabaseObject = "DatabaseObject"
	KeyUser           = "User"
)

type Handler interface {
	ServeHTTP(c *Context)
}

type HandlerFunc func(c *Context)

func (h HandlerFunc) ServeHTTP(c *Context) {
	h(c)
}

// Wrap converts a http.Handler into a negroni.Handler so it can be used as a Negroni
// middleware. The next http.HandlerFunc is automatically called after the Handler
// is executed.
func Wrap(handler Handler) MiddlewareHandler {
	return MiddlewareHandlerFunc(func(c *Context, next HandlerFunc) {
		handler.ServeHTTP(c)
		next(c)
	})
}

//WebappHandler Wrap a mux handler and calls a webapp handler
func WebappHandlerFunc(f func(*Context)) func(interface{}) {
	return func(mx interface{}) {
		c := mx.(*Context)
		f(c)
	}
}

// Negroni is a stack of Middleware Handlers that can be invoked as an http.Handler.
// Negroni middleware is evaluated in the order that they are added to the stack using
// the Use and UseHandler methods.
type Webapp struct {
	parent       *Webapp
	children     []*Webapp
	middleware   middleware
	handlers     []MiddlewareHandler
	router       *RouterContext
	RenderEngine Renderer
}

// New returns a new Negroni instance with no middleware preconfigured.
func New(handlers ...MiddlewareHandler) *Webapp {
	web := &Webapp{
		handlers:   handlers,
		middleware: build(handlers),
	}
	web.router = NewRouter()
	web.router.SetContextFactory(web.contextFactory)
	web.RenderEngine = NewJadeRender("./views")
	return web
}

func (this *Webapp) NewRoute(f func(*Context)) *mux.Route {
	return this.router.NewRoute().HandlerFunc(WebappHandlerFunc(f))
}

func (this *Webapp) Handle(path string, handler Handler) *mux.Route {
	return this.router.Handle(path, NewMuxHandlerAdapter(handler))
}

func (this *Webapp) HandleFunc(path string, f func(*Context)) *mux.Route {
	return this.router.HandleFunc(path, WebappHandlerFunc(f))
}

func (this *Webapp) Get(path string, f func(*Context)) *mux.Route {
	return this.router.HandleFunc(path, WebappHandlerFunc(f)).Methods("GET")
}

func (this *Webapp) Post(path string, f func(*Context)) *mux.Route {
	return this.router.HandleFunc(path, WebappHandlerFunc(f)).Methods("POST")
}

func (this *Webapp) FileServer(path_prefix string, file_path string) {
	this.router.FileServer(path_prefix, file_path)
}

// Classic returns a new Negroni instance with the default middleware already
// in the stack.
//
// Recovery - Panic Recovery Middleware
// Logger - Request/Response Logging
func Classic() *Webapp {
	return New(NewRecovery(), NewLogger())
}

func (n *Webapp) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	c := NewContext(n, rw, r)
	n.ServeHTTPContext(c)
}

func (n *Webapp) ServeHTTPContext(c *Context) {
	n.middleware.ServeHTTP(c)
}

// Use adds a Handler onto the middleware stack. Handlers are invoked in the order they are added to a Negroni.
func (n *Webapp) Use(handler MiddlewareHandler) {
	n.handlers = append(n.handlers, handler)
	n.middleware = build(n.handlers)
}

// UseFunc adds a Negroni-style handler function onto the middleware stack.
func (n *Webapp) UseFunc(handlerFunc func(c *Context, next HandlerFunc)) {
	n.Use(MiddlewareHandlerFunc(handlerFunc))
}

// UseHandler adds a http.Handler onto the middleware stack. Handlers are invoked in the order they are added to a Negroni.
func (n *Webapp) UseHandler(handler Handler) {
	n.Use(Wrap(handler))
}

// UseHandler adds a http.HandlerFunc-style handler function onto the middleware stack.
func (n *Webapp) UseHandlerFunc(handlerFunc func(c *Context)) {
	n.UseHandler(HandlerFunc(handlerFunc))
}

// Run is a convenience function that runs the negroni stack as an HTTP
// server. The addr string takes the same format as http.ListenAndServe.
func (this *Webapp) Run(addr string) {
	l := log.New(os.Stdout, "[webapp] ", 0)
	for _, child := range this.children {
		child.Run("")
	}
	if this.parent != nil {
		this.UseHandler(this.router)
		l.Printf("listening for subrouter")
	} else {
		this.UseHandler(this.router)
		l.Printf("listening on %s", addr)
		l.Fatal(http.ListenAndServe(addr, this))
	}
}

// Returns a list of all the handlers in the current Negroni middleware chain.
func (n *Webapp) Handlers() []MiddlewareHandler {
	return n.handlers
}

func (this *Webapp) SubRoute(path string) *Webapp {
	web := this.newChild()
	web.router = NewRouterBase(mux.NewRouter().PathPrefix(path).Subrouter())
	this.NewRoute(func(c *Context) {
		web.ServeHTTPContext(c)
	}).PathPrefix(path)
	return web
}

func (this *Webapp) newChild() *Webapp {
	web := New()
	web.parent = this
	this.children = append(this.children, web)
	return web
}

func build(handlers []MiddlewareHandler) middleware {
	var next middleware

	if len(handlers) == 0 {
		return voidMiddleware()
	} else if len(handlers) > 1 {
		next = build(handlers[1:])
	} else {
		next = voidMiddleware()
	}

	return middleware{handlers[0], &next}
}

func voidMiddleware() middleware {
	return middleware{
		MiddlewareHandlerFunc(func(c *Context, next HandlerFunc) {}),
		&middleware{},
	}
}

func (this *Webapp) contextFactory(w http.ResponseWriter, req *http.Request) interface{} {
	c := NewContext(this, w, req)
	c.app = this
	return c
}

//RouterContext Wrap mux.Router to support ServeHTTP(*Context)
type RouterContext struct {
	*mux.Router
}

//NewRouter Create a new mux router adapter
func NewRouter() *RouterContext {
	return &RouterContext{mux.NewRouter()}
}

//NewRouter Create a new mux router adapter
func NewRouterBase(router *mux.Router) *RouterContext {
	return &RouterContext{router}
}

//Wrapped ServeHttp
func (this *RouterContext) ServeHTTP(c *Context) {
	this.Router.ServeHTTPContext(c)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strings"
	"time"

	"github.com/crask/kafka/consumergroup"
	"github.com/golang/glog"
	"gopkg.in/Shopify/sarama.v1"
)

type Worker struct {
	Callback  *WorkerCallback
	Topics    []string
	Zookeeper []string
	ZkPath    string
	Consumer  *consumergroup.ConsumerGroup
}

type WorkerCallback struct {
	Url          string
	RetryTimes   int
	Timeout      time.Duration
	BypassFailed bool
	FailedSleep  time.Duration
}

func NewWorker() *Worker {
	return &Worker{}
}

func (this *Worker) Init(config *CallbackItemConfig) error {
	this.Callback = &WorkerCallback{
		Url:          config.Url,
		RetryTimes:   config.RetryTimes,
		Timeout:      config.Timeout,
		BypassFailed: config.BypassFailed,
		FailedSleep:  config.FailedSleep,
	}
	this.Topics = config.Topics
	this.Zookeeper = config.Zookeepers
	this.ZkPath = config.ZkPath
	this.Consumer = nil

	cgConfig := consumergroup.NewConfig()
	cgConfig.Offsets.ProcessingTimeout = 10 * time.Second
	cgConfig.Offsets.CommitInterval = time.Duration(commitInterval) * time.Second
	cgConfig.Offsets.Initial = sarama.OffsetNewest

	// Random Sleeping to avoid burst commit onto ZK.
	r := rand.Intn(commitInterval)
	time.Sleep(time.Duration(r))

	if len(this.ZkPath) > 0 {
		cgConfig.Zookeeper.Chroot = this.ZkPath
	}

	cgName := this.getGroupName()
	consumer, err := consumergroup.JoinConsumerGroup(cgName, this.Topics, this.Zookeeper, cgConfig)
	if err != nil {
		glog.Errorf("Failed to join consumer group for url[%v], %v", this.Callback.Url, err.Error())
		return err
	} else {
		glog.V(1).Infof("Join consumer group for url[%s] with UUID[%s]", this.Callback.Url, cgName)
	}

	glog.Infoln(consumer)

	this.Consumer = consumer
	return nil
}

func (this *Worker) getGroupName() string {
	m := md5.New()
	m.Write([]byte(this.Callback.Url))
	s := hex.EncodeToString(m.Sum(nil))
	return s
}

func (this *Worker) Work() {

	consumer := this.Consumer

	go func() {
		for err := range consumer.Errors() {
			glog.Errorln("Error working consumers", err)
		}
	}()

	eventCount := 0
	offsets := make(map[string]map[int32]int64)

	for message := range consumer.Messages() {
		if offsets[message.Topic] == nil {
			offsets[message.Topic] = make(map[int32]int64)
		}

		eventCount += 1
		if offsets[message.Topic][message.Partition] != 0 && offsets[message.Topic][message.Partition] != message.Offset-1 {
			glog.Errorf("Unexpected offset on %s:%d. Expected %d, found %d, diff %d.\n", message.Topic, message.Partition, offsets[message.Topic][message.Partition]+1, message.Offset, message.Offset-offsets[message.Topic][message.Partition]+1)
		}

		msg := CreateMsg(message)
		glog.V(2).Infof("received message,[topic:%s][partition:%d][offset:%d]", msg.Topic, msg.Partition, msg.Offset)

		deliverySuccessed := false
		retry_times := 0
		for {
			for !deliverySuccessed && retry_times < this.Callback.RetryTimes {
				deliverySuccessed, _ = this.delivery(msg, retry_times)
				if !deliverySuccessed {
					retry_times++
				} else {
					break
				}
			}

			if deliverySuccessed {
				break
			}

			if this.Callback.BypassFailed {
				glog.Errorf("tried to delivery message [url:%s][topic:%s][partition:%d][offset:%d] for %d times and all failed. BypassFailed is :%t ,will not retry", this.Callback.Url, msg.Topic, msg.Partition, msg.Offset, retry_times, this.Callback.BypassFailed)
				break
			} else {
				glog.Errorf("tried to delivery message [url:%s][topic:%s][partition:%d][offset:%d] for %d times and all failed. BypassFailed is :%t ,sleep %s to retry", this.Callback.Url, msg.Topic, msg.Partition, msg.Offset, retry_times, this.Callback.BypassFailed, this.Callback.FailedSleep)
				time.Sleep(this.Callback.FailedSleep)
				retry_times = 0
			}
		}

		offsets[message.Topic][message.Partition] = message.Offset
		consumer.CommitUpto(message)
		glog.Infof("commited message,[topic:%s][partition:%d][offset:%d][url:%s]", msg.Topic, msg.Partition, msg.Offset, this.Callback.Url)

	}

}

func (this *Worker) delivery(msg *Msg, retry_times int) (success bool, err error) {
	glog.V(2).Infof("delivery message,[url:%s][retry_times:%d][topic:%s][partition:%d][offset:%d]", this.Callback.Url, retry_times, msg.Topic, msg.Partition, msg.Offset)

	client := &http.Client{}
	client.Timeout = this.Callback.Timeout

	type RMSG struct {
		Topic        string `json:"Topic"`
		PartitionKey string `json:"PartitionKey"`
		TimeStamp    int    `json:"TimeStamp"`
		Data         string `json:"Data"`
		LogId        string `json:"LogId"`
		ContentType  string `json:"ContentType"`
	}
	var rmsg RMSG
	json.Unmarshal(msg.Value, &rmsg)
	req, _ := http.NewRequest("POST", this.Callback.Url, ioutil.NopCloser(strings.NewReader(rmsg.Data)))
	if rmsg.ContentType == "" {
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
	} else {
		req.Header.Set("Content-Type", rmsg.ContentType)
	}
	req.Header.Set("User-Agent", "Taiji pusher consumer(go)/v"+VERSION)
	req.Header.Set("X-Retry-Times", fmt.Sprintf("%d", retry_times))
	req.Header.Set("X-Kmq-Topic", msg.Topic)
	req.Header.Set("X-Kmq-Partition", fmt.Sprintf("%d", msg.Partition))
	req.Header.Set("X-Kmq-Partition-Key", rmsg.PartitionKey)
	req.Header.Set("X-Kmq-Offset", fmt.Sprintf("%d", msg.Offset))
	req.Header.Set("X-Kmq-Logid", fmt.Sprintf("%s", rmsg.LogId))
	req.Header.Set("Meilishuo", "uid:0;ip:0.0.0.0;v:0;master:0")
	req.Header.Set("X-Kmq-Timestamp", fmt.Sprintf("%d", rmsg.TimeStamp))
	resp, err := client.Do(req)
	suc := true
	if nil == err {
		defer resp.Body.Close()
		suc = (resp.StatusCode == 200)
		if suc == false {
			rbody, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				rbody = []byte{}
			}
			glog.Errorf("delivery failed,[retry_times:%d][topic:%s][partition:%d][offset:%d][msg:%s][url:%s][http_code:%d][response_body:%s]",
				retry_times, msg.Topic, msg.Partition, msg.Offset, rmsg.Data, this.Callback.Url, resp.StatusCode, rbody)
		}
	} else {
		glog.Errorf("delivery failed,[retry_times:%d][topic:%s][partition:%d][offset:%d][msg:%s][url:%s][error:%s]",
			retry_times, msg.Topic, msg.Partition, msg.Offset, rmsg.Data, this.Callback.Url, err.Error())
		suc = false
	}

	return suc, err
}

func (this *Worker) Closed() bool {
	return this.Consumer.Closed()
}

func (this *Worker) Close() {
	if err := this.Consumer.Close(); err != nil {
		glog.Errorln("Error closing consumers", err)
	}
}
<commit_msg>add rpclog time cost<commit_after>package main

import (
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strings"
	"time"

	"github.com/crask/kafka/consumergroup"
	"github.com/golang/glog"
	"gopkg.in/Shopify/sarama.v1"
)

type Worker struct {
	Callback  *WorkerCallback
	Topics    []string
	Zookeeper []string
	ZkPath    string
	Consumer  *consumergroup.ConsumerGroup
}

type WorkerCallback struct {
	Url          string
	RetryTimes   int
	Timeout      time.Duration
	BypassFailed bool
	FailedSleep  time.Duration
}

func NewWorker() *Worker {
	return &Worker{}
}

func (this *Worker) Init(config *CallbackItemConfig) error {
	this.Callback = &WorkerCallback{
		Url:          config.Url,
		RetryTimes:   config.RetryTimes,
		Timeout:      config.Timeout,
		BypassFailed: config.BypassFailed,
		FailedSleep:  config.FailedSleep,
	}
	this.Topics = config.Topics
	this.Zookeeper = config.Zookeepers
	this.ZkPath = config.ZkPath
	this.Consumer = nil

	cgConfig := consumergroup.NewConfig()
	cgConfig.Offsets.ProcessingTimeout = 10 * time.Second
	cgConfig.Offsets.CommitInterval = time.Duration(commitInterval) * time.Second
	cgConfig.Offsets.Initial = sarama.OffsetNewest

	// Random Sleeping to avoid burst commit onto ZK.
	r := rand.Intn(commitInterval)
	time.Sleep(time.Duration(r))

	if len(this.ZkPath) > 0 {
		cgConfig.Zookeeper.Chroot = this.ZkPath
	}

	cgName := this.getGroupName()
	consumer, err := consumergroup.JoinConsumerGroup(cgName, this.Topics, this.Zookeeper, cgConfig)
	if err != nil {
		glog.Errorf("Failed to join consumer group for url[%v], %v", this.Callback.Url, err.Error())
		return err
	} else {
		glog.V(1).Infof("Join consumer group for url[%s] with UUID[%s]", this.Callback.Url, cgName)
	}

	glog.Infoln(consumer)

	this.Consumer = consumer
	return nil
}

func (this *Worker) getGroupName() string {
	m := md5.New()
	m.Write([]byte(this.Callback.Url))
	s := hex.EncodeToString(m.Sum(nil))
	return s
}

func (this *Worker) Work() {

	consumer := this.Consumer

	go func() {
		for err := range consumer.Errors() {
			glog.Errorln("Error working consumers", err)
		}
	}()

	eventCount := 0
	offsets := make(map[string]map[int32]int64)

	for message := range consumer.Messages() {
		if offsets[message.Topic] == nil {
			offsets[message.Topic] = make(map[int32]int64)
		}

		eventCount += 1
		if offsets[message.Topic][message.Partition] != 0 && offsets[message.Topic][message.Partition] != message.Offset-1 {
			glog.Errorf("Unexpected offset on %s:%d. Expected %d, found %d, diff %d.\n", message.Topic, message.Partition, offsets[message.Topic][message.Partition]+1, message.Offset, message.Offset-offsets[message.Topic][message.Partition]+1)
		}

		msg := CreateMsg(message)
		glog.V(2).Infof("received message,[topic:%s][partition:%d][offset:%d]", msg.Topic, msg.Partition, msg.Offset)

		deliverySuccessed := false
		retry_times := 0
		for {
			for !deliverySuccessed && retry_times < this.Callback.RetryTimes {
				deliverySuccessed, _ = this.delivery(msg, retry_times)
				if !deliverySuccessed {
					retry_times++
				} else {
					break
				}
			}

			if deliverySuccessed {
				break
			}

			if this.Callback.BypassFailed {
				glog.Errorf("tried to delivery message [url:%s][topic:%s][partition:%d][offset:%d] for %d times and all failed. BypassFailed is :%t ,will not retry", this.Callback.Url, msg.Topic, msg.Partition, msg.Offset, retry_times, this.Callback.BypassFailed)
				break
			} else {
				glog.Errorf("tried to delivery message [url:%s][topic:%s][partition:%d][offset:%d] for %d times and all failed. BypassFailed is :%t ,sleep %s to retry", this.Callback.Url, msg.Topic, msg.Partition, msg.Offset, retry_times, this.Callback.BypassFailed, this.Callback.FailedSleep)
				time.Sleep(this.Callback.FailedSleep)
				retry_times = 0
			}
		}

		offsets[message.Topic][message.Partition] = message.Offset
		consumer.CommitUpto(message)
		glog.Infof("commited message,[topic:%s][partition:%d][offset:%d][url:%s]", msg.Topic, msg.Partition, msg.Offset, this.Callback.Url)

	}

}

func (this *Worker) delivery(msg *Msg, retry_times int) (success bool, err error) {
	var tsrpc, terpc time.Time

	client := &http.Client{}
	client.Timeout = this.Callback.Timeout

	type RMSG struct {
		Topic        string `json:"Topic"`
		PartitionKey string `json:"PartitionKey"`
		TimeStamp    int    `json:"TimeStamp"`
		Data         string `json:"Data"`
		LogId        string `json:"LogId"`
		ContentType  string `json:"ContentType"`
	}
	var rmsg RMSG
	json.Unmarshal(msg.Value, &rmsg)
	req, _ := http.NewRequest("POST", this.Callback.Url, ioutil.NopCloser(strings.NewReader(rmsg.Data)))
	if rmsg.ContentType == "" {
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
	} else {
		req.Header.Set("Content-Type", rmsg.ContentType)
	}
	req.Header.Set("User-Agent", "Taiji pusher consumer(go)/v"+VERSION)
	req.Header.Set("X-Retry-Times", fmt.Sprintf("%d", retry_times))
	req.Header.Set("X-Kmq-Topic", msg.Topic)
	req.Header.Set("X-Kmq-Partition", fmt.Sprintf("%d", msg.Partition))
	req.Header.Set("X-Kmq-Partition-Key", rmsg.PartitionKey)
	req.Header.Set("X-Kmq-Offset", fmt.Sprintf("%d", msg.Offset))
	req.Header.Set("X-Kmq-Logid", fmt.Sprintf("%s", rmsg.LogId))
	req.Header.Set("Meilishuo", "uid:0;ip:0.0.0.0;v:0;master:0")
	req.Header.Set("X-Kmq-Timestamp", fmt.Sprintf("%d", rmsg.TimeStamp))
	tsrpc = time.Now()
	resp, err := client.Do(req)
	terpc = time.Now()
	suc := true
	if nil == err {
		defer resp.Body.Close()
		suc = (resp.StatusCode == 200)
		if suc == false {
			rbody, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				rbody = []byte{}
			}
			glog.Errorf("delivery failed,[retry_times:%d][topic:%s][partition:%d][offset:%d][msg:%s][url:%s][http_code:%d][cost:%v][response_body:%s]",
				retry_times, msg.Topic, msg.Partition, msg.Offset, rmsg.Data, this.Callback.Url, resp.StatusCode, terpc.Sub(tsrpc), rbody)
		}
	} else {
		glog.Errorf("delivery failed,[retry_times:%d][topic:%s][partition:%d][offset:%d][msg:%s][url:%s][error:%s][cost:%v]",
			retry_times, msg.Topic, msg.Partition, msg.Offset, rmsg.Data, this.Callback.Url, err.Error(), terpc.Sub(tsrpc))
		suc = false
	}
	glog.V(2).Infof("delivery message,[url:%s][retry_times:%d][topic:%s][partition:%d][offset:%d][cost:%v]", this.Callback.Url, retry_times, msg.Topic, msg.Partition, msg.Offset, terpc.Sub(tsrpc))

	return suc, err
}

func (this *Worker) Closed() bool {
	return this.Consumer.Closed()
}

func (this *Worker) Close() {
	if err := this.Consumer.Close(); err != nil {
		glog.Errorln("Error closing consumers", err)
	}
}
<|endoftext|>