<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package servicecenter

import (
	"net/url"
	"strconv"
	"strings"

	"github.com/apache/servicecomb-service-center/pkg/log"
	"github.com/apache/servicecomb-service-center/server/admin/model"
	scpb "github.com/apache/servicecomb-service-center/server/core/proto"
	pb "github.com/apache/servicecomb-service-center/syncer/proto"
	"github.com/gogo/protobuf/proto"
)

const (
	expansionDatasource = "datasource"
	expansionSchema     = "schema"
)

// toSyncData transform service-center service cache to SyncData
func toSyncData(cache *model.Cache, schemas []*scpb.Schema) (data *pb.SyncData) {
	data = &pb.SyncData{
		Services:  make([]*pb.SyncService, 0, len(cache.Microservices)),
		Instances: make([]*pb.SyncInstance, 0, len(cache.Instances)),
	}

	for _, service := range cache.Microservices {
		domainProject := getDomainProjectFromServiceKey(service.Key)
		if domainProject == "" {
			continue
		}
		syncService := toSyncService(service.Value)
		syncService.DomainProject = domainProject
		syncService.Expansions = append(syncService.Expansions, schemaExpansions(service.Value, schemas)...)

		syncInstances := toSyncInstances(syncService.ServiceId, cache.Instances)
		if len(syncInstances) == 0 {
			continue
		}

		data.Services = append(data.Services, syncService)
		data.Instances = append(data.Instances, syncInstances...)
	}
	return
}

// toSyncService transform service-center service to SyncService
func toSyncService(service *scpb.MicroService) (syncService *pb.SyncService) {
	syncService = &pb.SyncService{
		ServiceId:   service.ServiceId,
		App:         service.AppId,
		Name:        service.ServiceName,
		Version:     service.Version,
		Environment: service.Environment,
		PluginName:  PluginName,
	}
	switch service.Status {
	case scpb.MS_UP:
		syncService.Status = pb.SyncService_UP
	case scpb.MS_DOWN:
		syncService.Status = pb.SyncService_DOWN
	default:
		syncService.Status = pb.SyncService_UNKNOWN
	}

	content, err := proto.Marshal(service)
	if err != nil {
		log.Errorf(err, "transform sc service to syncer service failed: %s", err)
		return
	}

	syncService.Expansions = []*pb.Expansion{{
		Kind:   expansionDatasource,
		Bytes:  content,
		Labels: map[string]string{},
	}}
	return
}

//  toSyncInstances transform service-center instances to SyncInstances
func toSyncInstances(serviceID string, instances []*model.Instance) (syncInstances []*pb.SyncInstance) {
	for _, inst := range instances {
		if inst.Value.Status != scpb.MSI_UP {
			continue
		}

		if inst.Value.ServiceId == serviceID {
			syncInstances = append(syncInstances, toSyncInstance(serviceID, inst.Value))
		}
	}
	return
}

// toSyncInstance transform service-center instance to SyncInstance
func toSyncInstance(serviceID string, instance *scpb.MicroServiceInstance) (syncInstance *pb.SyncInstance) {
	syncInstance = &pb.SyncInstance{
		InstanceId: instance.InstanceId,
		ServiceId:  serviceID,
		Endpoints:  make([]string, 0, len(instance.Endpoints)),
		HostName:   instance.HostName,
		Version:    instance.Version,
		PluginName: PluginName,
	}
	switch instance.Status {
	case scpb.MSI_UP:
		syncInstance.Status = pb.SyncInstance_UP
	case scpb.MSI_DOWN:
		syncInstance.Status = pb.SyncInstance_DOWN
	case scpb.MSI_STARTING:
		syncInstance.Status = pb.SyncInstance_STARTING
	case scpb.MSI_OUTOFSERVICE:
		syncInstance.Status = pb.SyncInstance_OUTOFSERVICE
	default:
		syncInstance.Status = pb.SyncInstance_UNKNOWN
	}

	for _, ep := range instance.Endpoints {
		prefix := "http://"
		addr, err := url.Parse(ep)
		if err != nil {
			log.Errorf(err, "parse sc instance endpoint failed: %s", err)
			continue
		}
		if addr.Scheme == "rest" {
			b, _ := strconv.ParseBool(addr.Query().Get("sslEnabled"))
			if b {
				prefix = "https://"
			}
		}
		syncInstance.Endpoints = append(syncInstance.Endpoints, strings.Replace(ep, addr.Scheme+"://", prefix, 1))
	}

	if instance.HealthCheck != nil {
		syncInstance.HealthCheck = &pb.HealthCheck{
			Port:     instance.HealthCheck.Port,
			Interval: instance.HealthCheck.Interval,
			Times:    instance.HealthCheck.Times,
			Url:      instance.HealthCheck.Url,
		}
	}

	content, err := proto.Marshal(instance)
	if err != nil {
		log.Errorf(err, "transform sc instance to syncer instance failed: %s", err)
		return
	}

	syncInstance.Expansions = []*pb.Expansion{{
		Kind:   expansionDatasource,
		Bytes:  content,
		Labels: map[string]string{},
	}}
	return
}

func schemaExpansions(service *scpb.MicroService, schemas []*scpb.Schema) (expansions []*pb.Expansion) {
	for _, val := range schemas {
		if !inSlice(service.Schemas, val.SchemaId) {
			continue
		}

		content, err := proto.Marshal(val)
		if err != nil {
			log.Errorf(err, "proto marshal schemas failed, app = %s, service = %s, version = %s datasource = %s",
				service.AppId, service.ServiceName, service.Version, expansionSchema)
			continue
		}
		expansions = append(expansions, &pb.Expansion{
			Kind:   expansionSchema,
			Bytes:  content,
			Labels: map[string]string{},
		})
	}
	return
}

// toService transform SyncService to service-center service
func toService(syncService *pb.SyncService) (service *scpb.MicroService) {
	service = &scpb.MicroService{}
	if syncService.PluginName == PluginName && len(syncService.Expansions) > 0 {
		matches := pb.Expansions(syncService.Expansions).Find(expansionDatasource, map[string]string{})
		if len(matches) > 0 {
			err := proto.Unmarshal(matches[0].Bytes, service)
			if err != nil {
				log.Errorf(err, "proto unmarshal %s service, serviceID = %s, kind = %v, content = %v failed",
					PluginName, service.ServiceId, matches[0].Kind, matches[0].Bytes)
			}
		}
	}
	service.AppId = syncService.App
	service.ServiceId = syncService.ServiceId
	service.ServiceName = syncService.Name
	service.Version = syncService.Version
	service.Status = pb.SyncService_Status_name[int32(syncService.Status)]
	service.Environment = syncService.Environment
	return
}

// toInstance transform SyncInstance to service-center instance
func toInstance(syncInstance *pb.SyncInstance) (instance *scpb.MicroServiceInstance) {
	instance = &scpb.MicroServiceInstance{}
	if syncInstance.PluginName == PluginName && len(syncInstance.Expansions) > 0 {
		matches := pb.Expansions(syncInstance.Expansions).Find(expansionDatasource, map[string]string{})
		if len(matches) > 0 {
			err := proto.Unmarshal(matches[0].Bytes, instance)
			if err != nil {
				log.Errorf(err, "proto unmarshal %s instance, instanceID = %s, kind = %v, content = %v failed",
					PluginName, instance.InstanceId, matches[0].Kind, matches[0].Bytes)
			}
		}
	}
	instance.InstanceId = syncInstance.InstanceId
	instance.ServiceId = syncInstance.ServiceId
	instance.Endpoints = make([]string, 0, len(syncInstance.Endpoints))
	instance.HostName = syncInstance.HostName
	instance.Version = syncInstance.Version
	instance.Status = pb.SyncInstance_Status_name[int32(syncInstance.Status)]

	for _, ep := range syncInstance.Endpoints {
		addr, err := url.Parse(ep)
		if err != nil {
			log.Errorf(err, "parse sc instance endpoint failed: %s", err)
			continue
		}
		endpoint := ""
		switch addr.Scheme {
		case "http":
			endpoint = strings.Replace(ep, "http://", "rest://", 1)
		case "https":
			endpoint = strings.Replace(ep, "https://", "rest://", 1) + "?sslEnabled=true"
		case "rest":
			endpoint = ep

		}
		instance.Endpoints = append(instance.Endpoints, endpoint)
	}

	if syncInstance.HealthCheck != nil && syncInstance.HealthCheck.Mode != pb.HealthCheck_UNKNOWN {
		instance.HealthCheck = &scpb.HealthCheck{
			Mode:     pb.HealthCheck_Modes_name[int32(syncInstance.HealthCheck.Mode)],
			Port:     syncInstance.HealthCheck.Port,
			Interval: syncInstance.HealthCheck.Interval,
			Times:    syncInstance.HealthCheck.Times,
			Url:      syncInstance.HealthCheck.Url,
		}
	}
	return
}

func getDomainProjectFromServiceKey(serviceKey string) string {
	tenant := strings.Split(serviceKey, "/")
	if len(tenant) < 6 {
		return ""
	}
	return strings.Join(tenant[4:6], "/")
}

func inSlice(slice []string, val string) bool {
	for _, item := range slice {
		if item == val {
			return true
		}
	}
	return false
}
<commit_msg>syncer supports highway protocol<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package servicecenter

import (
	"net/url"
	"strconv"
	"strings"

	"github.com/apache/servicecomb-service-center/pkg/log"
	"github.com/apache/servicecomb-service-center/server/admin/model"
	scpb "github.com/apache/servicecomb-service-center/server/core/proto"
	pb "github.com/apache/servicecomb-service-center/syncer/proto"
	"github.com/gogo/protobuf/proto"
)

const (
	expansionDatasource = "datasource"
	expansionSchema     = "schema"
)

// toSyncData transform service-center service cache to SyncData
func toSyncData(cache *model.Cache, schemas []*scpb.Schema) (data *pb.SyncData) {
	data = &pb.SyncData{
		Services:  make([]*pb.SyncService, 0, len(cache.Microservices)),
		Instances: make([]*pb.SyncInstance, 0, len(cache.Instances)),
	}

	for _, service := range cache.Microservices {
		domainProject := getDomainProjectFromServiceKey(service.Key)
		if domainProject == "" {
			continue
		}
		syncService := toSyncService(service.Value)
		syncService.DomainProject = domainProject
		syncService.Expansions = append(syncService.Expansions, schemaExpansions(service.Value, schemas)...)

		syncInstances := toSyncInstances(syncService.ServiceId, cache.Instances)
		if len(syncInstances) == 0 {
			continue
		}

		data.Services = append(data.Services, syncService)
		data.Instances = append(data.Instances, syncInstances...)
	}
	return
}

// toSyncService transform service-center service to SyncService
func toSyncService(service *scpb.MicroService) (syncService *pb.SyncService) {
	syncService = &pb.SyncService{
		ServiceId:   service.ServiceId,
		App:         service.AppId,
		Name:        service.ServiceName,
		Version:     service.Version,
		Environment: service.Environment,
		PluginName:  PluginName,
	}
	switch service.Status {
	case scpb.MS_UP:
		syncService.Status = pb.SyncService_UP
	case scpb.MS_DOWN:
		syncService.Status = pb.SyncService_DOWN
	default:
		syncService.Status = pb.SyncService_UNKNOWN
	}

	content, err := proto.Marshal(service)
	if err != nil {
		log.Errorf(err, "transform sc service to syncer service failed: %s", err)
		return
	}

	syncService.Expansions = []*pb.Expansion{{
		Kind:   expansionDatasource,
		Bytes:  content,
		Labels: map[string]string{},
	}}
	return
}

//  toSyncInstances transform service-center instances to SyncInstances
func toSyncInstances(serviceID string, instances []*model.Instance) (syncInstances []*pb.SyncInstance) {
	for _, inst := range instances {
		if inst.Value.Status != scpb.MSI_UP {
			continue
		}

		if inst.Value.ServiceId == serviceID {
			syncInstances = append(syncInstances, toSyncInstance(serviceID, inst.Value))
		}
	}
	return
}

// toSyncInstance transform service-center instance to SyncInstance
func toSyncInstance(serviceID string, instance *scpb.MicroServiceInstance) (syncInstance *pb.SyncInstance) {
	syncInstance = &pb.SyncInstance{
		InstanceId: instance.InstanceId,
		ServiceId:  serviceID,
		Endpoints:  make([]string, 0, len(instance.Endpoints)),
		HostName:   instance.HostName,
		Version:    instance.Version,
		PluginName: PluginName,
	}
	switch instance.Status {
	case scpb.MSI_UP:
		syncInstance.Status = pb.SyncInstance_UP
	case scpb.MSI_DOWN:
		syncInstance.Status = pb.SyncInstance_DOWN
	case scpb.MSI_STARTING:
		syncInstance.Status = pb.SyncInstance_STARTING
	case scpb.MSI_OUTOFSERVICE:
		syncInstance.Status = pb.SyncInstance_OUTOFSERVICE
	default:
		syncInstance.Status = pb.SyncInstance_UNKNOWN
	}

	for _, ep := range instance.Endpoints {
		endpoint := ep
		addr, err := url.Parse(ep)
		if err != nil {
			log.Errorf(err, "parse sc instance endpoint failed: %s", err)
			continue
		}
		if addr.Scheme == "rest" {
			prefix := "http://"
			b, _ := strconv.ParseBool(addr.Query().Get("sslEnabled"))
			if b {
				prefix = "https://"
			}
			endpoint = strings.Replace(ep, addr.Scheme+"://", prefix, 1)
		}
		syncInstance.Endpoints = append(syncInstance.Endpoints, endpoint)
	}

	if instance.HealthCheck != nil {
		syncInstance.HealthCheck = &pb.HealthCheck{
			Port:     instance.HealthCheck.Port,
			Interval: instance.HealthCheck.Interval,
			Times:    instance.HealthCheck.Times,
			Url:      instance.HealthCheck.Url,
		}
	}

	content, err := proto.Marshal(instance)
	if err != nil {
		log.Errorf(err, "transform sc instance to syncer instance failed: %s", err)
		return
	}

	syncInstance.Expansions = []*pb.Expansion{{
		Kind:   expansionDatasource,
		Bytes:  content,
		Labels: map[string]string{},
	}}
	return
}

func schemaExpansions(service *scpb.MicroService, schemas []*scpb.Schema) (expansions []*pb.Expansion) {
	for _, val := range schemas {
		if !inSlice(service.Schemas, val.SchemaId) {
			continue
		}

		content, err := proto.Marshal(val)
		if err != nil {
			log.Errorf(err, "proto marshal schemas failed, app = %s, service = %s, version = %s datasource = %s",
				service.AppId, service.ServiceName, service.Version, expansionSchema)
			continue
		}
		expansions = append(expansions, &pb.Expansion{
			Kind:   expansionSchema,
			Bytes:  content,
			Labels: map[string]string{},
		})
	}
	return
}

// toService transform SyncService to service-center service
func toService(syncService *pb.SyncService) (service *scpb.MicroService) {
	service = &scpb.MicroService{}
	var err error
	if syncService.PluginName == PluginName && len(syncService.Expansions) > 0 {
		matches := pb.Expansions(syncService.Expansions).Find(expansionDatasource, map[string]string{})
		if len(matches) > 0 {
			err = proto.Unmarshal(matches[0].Bytes, service)
			if err == nil {
				service.ServiceId = syncService.ServiceId
				return
			}
			log.Errorf(err, "proto unmarshal %s service, serviceID = %s, kind = %v, content = %v failed",
				PluginName, service.ServiceId, matches[0].Kind, matches[0].Bytes)
		}
	}
	service.AppId = syncService.App
	service.ServiceId = syncService.ServiceId
	service.ServiceName = syncService.Name
	service.Version = syncService.Version
	service.Status = pb.SyncService_Status_name[int32(syncService.Status)]
	service.Environment = syncService.Environment
	return
}

// toInstance transform SyncInstance to service-center instance
func toInstance(syncInstance *pb.SyncInstance) (instance *scpb.MicroServiceInstance) {
	instance = &scpb.MicroServiceInstance{}
	if syncInstance.PluginName == PluginName && len(syncInstance.Expansions) > 0 {
		matches := pb.Expansions(syncInstance.Expansions).Find(expansionDatasource, map[string]string{})
		if len(matches) > 0 {
			err := proto.Unmarshal(matches[0].Bytes, instance)
			if err == nil {
				instance.InstanceId = syncInstance.InstanceId
				instance.ServiceId = syncInstance.ServiceId
				return
			}
			log.Errorf(err, "proto unmarshal %s instance, instanceID = %s, kind = %v, content = %v failed",
				PluginName, instance.InstanceId, matches[0].Kind, matches[0].Bytes)

		}
	}
	instance.InstanceId = syncInstance.InstanceId
	instance.ServiceId = syncInstance.ServiceId
	instance.Endpoints = make([]string, 0, len(syncInstance.Endpoints))
	instance.HostName = syncInstance.HostName
	instance.Version = syncInstance.Version
	instance.Status = pb.SyncInstance_Status_name[int32(syncInstance.Status)]

	for _, ep := range syncInstance.Endpoints {
		addr, err := url.Parse(ep)
		if err != nil {
			log.Errorf(err, "parse sc instance endpoint failed: %s", err)
			continue
		}
		endpoint := ""
		switch addr.Scheme {
		case "http":
			endpoint = strings.Replace(ep, "http://", "rest://", 1)
		case "https":
			endpoint = strings.Replace(ep, "https://", "rest://", 1) + "?sslEnabled=true"
		case "rest", "highway":
			endpoint = ep
		}
		instance.Endpoints = append(instance.Endpoints, endpoint)
	}

	if syncInstance.HealthCheck != nil && syncInstance.HealthCheck.Mode != pb.HealthCheck_UNKNOWN {
		instance.HealthCheck = &scpb.HealthCheck{
			Mode:     pb.HealthCheck_Modes_name[int32(syncInstance.HealthCheck.Mode)],
			Port:     syncInstance.HealthCheck.Port,
			Interval: syncInstance.HealthCheck.Interval,
			Times:    syncInstance.HealthCheck.Times,
			Url:      syncInstance.HealthCheck.Url,
		}
	}
	return
}

func getDomainProjectFromServiceKey(serviceKey string) string {
	tenant := strings.Split(serviceKey, "/")
	if len(tenant) < 6 {
		return ""
	}
	return strings.Join(tenant[4:6], "/")
}

func inSlice(slice []string, val string) bool {
	for _, item := range slice {
		if item == val {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. 2017 All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package etcdraft

import (
	"testing"

	cb "github.com/hyperledger/fabric-protos-go/common"
	"github.com/hyperledger/fabric/common/flogging"
	"github.com/hyperledger/fabric/protoutil"
	"github.com/stretchr/testify/assert"
	"go.uber.org/zap"
)

func getSeedBlock() *cb.Block {
	seedBlock := protoutil.NewBlock(0, []byte("firsthash"))
	seedBlock.Data.Data = [][]byte{[]byte("somebytes")}
	return seedBlock
}

func TestCreateNextBlock(t *testing.T) {
	first := protoutil.NewBlock(0, []byte("firsthash"))
	bc := &blockCreator{
		hash:   protoutil.BlockHeaderHash(first.Header),
		number: first.Header.Number,
		logger: flogging.NewFabricLogger(zap.NewNop()),
	}

	second := bc.createNextBlock([]*cb.Envelope{{Payload: []byte("some other bytes")}})
	assert.Equal(t, first.Header.Number+1, second.Header.Number)
	assert.Equal(t, protoutil.BlockDataHash(second.Data), second.Header.DataHash)
	assert.Equal(t, protoutil.BlockHeaderHash(first.Header), second.Header.PreviousHash)

	third := bc.createNextBlock([]*cb.Envelope{{Payload: []byte("some other bytes")}})
	assert.Equal(t, second.Header.Number+1, third.Header.Number)
	assert.Equal(t, protoutil.BlockDataHash(third.Data), third.Header.DataHash)
	assert.Equal(t, protoutil.BlockHeaderHash(second.Header), third.Header.PreviousHash)
}
<commit_msg>Remove dead code from etcdraft/blockcreator_test<commit_after>/*
Copyright IBM Corp. 2017 All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package etcdraft

import (
	"testing"

	cb "github.com/hyperledger/fabric-protos-go/common"
	"github.com/hyperledger/fabric/common/flogging"
	"github.com/hyperledger/fabric/protoutil"
	"github.com/stretchr/testify/assert"
	"go.uber.org/zap"
)

func TestCreateNextBlock(t *testing.T) {
	first := protoutil.NewBlock(0, []byte("firsthash"))
	bc := &blockCreator{
		hash:   protoutil.BlockHeaderHash(first.Header),
		number: first.Header.Number,
		logger: flogging.NewFabricLogger(zap.NewNop()),
	}

	second := bc.createNextBlock([]*cb.Envelope{{Payload: []byte("some other bytes")}})
	assert.Equal(t, first.Header.Number+1, second.Header.Number)
	assert.Equal(t, protoutil.BlockDataHash(second.Data), second.Header.DataHash)
	assert.Equal(t, protoutil.BlockHeaderHash(first.Header), second.Header.PreviousHash)

	third := bc.createNextBlock([]*cb.Envelope{{Payload: []byte("some other bytes")}})
	assert.Equal(t, second.Header.Number+1, third.Header.Number)
	assert.Equal(t, protoutil.BlockDataHash(third.Data), third.Header.DataHash)
	assert.Equal(t, protoutil.BlockHeaderHash(second.Header), third.Header.PreviousHash)
}
<|endoftext|>


<commit_before><commit_msg>add secp256k1_test.go<commit_after><|endoftext|>


<commit_before>package serial

import (
	"os"
	"io"
	//"fmt"
	"unsafe"
	"syscall"
)

type serialPort struct {
	f *os.File
}

const EV_RXCHAR = 0x0001

type structDCB struct {
	DCBlength, BaudRate uint32
	flags [4]byte
	wReserved, XonLim, XoffLim uint16
	ByteSize, Parity, StopBits byte
	XonChar, XoffChar, ErrorChar, EofChar, EvtChar byte
	wReserved1 uint16
}

type structTimeouts struct {
	ReadIntervalTimeout uint32
	ReadTotalTimeoutMultiplier uint32
	ReadTotalTimeoutConstant uint32
	WriteTotalTimeoutMultiplier uint32
	WriteTotalTimeoutConstant uint32
}

var setCommState uint32
var setCommTimeouts uint32
var setCommMask uint32
var waitCommEvent uint32
var getOverlappedResult uint32
var createEvent uint32

func loadDll(lib uint32, name string) uint32 {
	addr, err := syscall.GetProcAddress(lib, name)
	if err != 0 {
		panic(name + " " + syscall.Errstr(err))
	}
	return addr
}

func init() {
	k32, err := syscall.LoadLibrary("kernel32.dll")
	if err != 0 {
		panic("LoadLibrary "+ syscall.Errstr(err))
	}
	defer syscall.FreeLibrary(k32)

	setCommState        = loadDll(k32, "SetCommState")
	setCommTimeouts     = loadDll(k32, "SetCommTimeouts")
	setCommMask         = loadDll(k32, "SetCommMask")
	waitCommEvent       = loadDll(k32, "WaitCommEvent")
	getOverlappedResult = loadDll(k32, "GetOverlappedResult")
	createEvent         = loadDll(k32, "CreateEvent")
}

const FILE_FLAGS_OVERLAPPED = 0x40000000

func OpenPort(name string, baud int) (io.ReadWriteCloser, os.Error) {
	//h, e := CreateFile(StringToUTF16Ptr(path), access, sharemode, sa, createmode, FILE_ATTRIBUTE_NORMAL, 0)

	h, e := syscall.CreateFile(syscall.StringToUTF16Ptr(name),
		syscall.GENERIC_READ|syscall.GENERIC_WRITE,
		uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE),
		nil,
		syscall.OPEN_EXISTING,
		syscall.FILE_ATTRIBUTE_NORMAL|FILE_FLAGS_OVERLAPPED,
		0)

	if e != 0 {
		return nil, &os.PathError{"open", name, os.Errno(e)}
	}

	f := os.NewFile(int(h), name)

	var params structDCB
	params.DCBlength = uint32(unsafe.Sizeof(params))

	params.BaudRate = uint32(baud)
	params.ByteSize = 8

	_, _, e2 := syscall.Syscall(uintptr(setCommState), uintptr(h), uintptr(unsafe.Pointer(&params)), 0, 0)
	if e2 != 0 {
		f.Close()
		return nil, os.Errno(e)
	}

	// Turn off buffers
	/*
	if ok, err := C.SetupComm(*handle, 16, 16); ok == C.FALSE {
		f.Close()
		return nil, err
	}
	 */

	var timeouts structTimeouts
	timeouts.ReadIntervalTimeout = 1<<32 - 1
	timeouts.ReadTotalTimeoutConstant = 0
	_, _, e2 = syscall.Syscall(uintptr(setCommTimeouts), uintptr(h), uintptr(unsafe.Pointer(&timeouts)), 0, 0)
	if e2 != 0 {
		f.Close()
		return nil, os.Errno(e)
	}

	_, _, e2 = syscall.Syscall(uintptr(setCommMask), uintptr(h), EV_RXCHAR, 0, 0)
	if e2 != 0 {
		f.Close()
		return nil, os.Errno(e)
	}

	port := serialPort{f}

	return &port, nil
}

func (p *serialPort) Close() os.Error {
	return p.f.Close()
}

func (p *serialPort) Write(buf []byte) (int, os.Error) {
	var events uint32
	var overlapped syscall.Overlapped
	var n int

	fd := p.f.Fd()

	r, _, e := syscall.Syscall(uintptr(createEvent), 0, 1, 0, 0)
	if e != 0 {
		return 0, os.Errno(e)
	}
	overlapped.HEvent = (*uint8)(unsafe.Pointer(r))

	e1 := syscall.WriteFile(int32(fd), buf, &events, &overlapped)
	if e1 != 0 {
		return 0, os.Errno(e)
	}

	_, _, e = syscall.Syscall(uintptr(getOverlappedResult), uintptr(fd), uintptr(unsafe.Pointer(&overlapped)), uintptr(unsafe.Pointer(&n)), 1)
	if e != 0 {
		return 0, os.Errno(e)
	}

	return p.f.Write(buf)
}

func (p *serialPort) Read(buf []byte) (int, os.Error) {
	var events uint32
	var overlapped syscall.Overlapped
	var n int

	fd := p.f.Fd()

	r, _, e := syscall.Syscall(uintptr(createEvent), 0, 1, 0, 0)
	if e != 0 {
		return 0, os.Errno(e)
	}
	overlapped.HEvent = (*uint8)(unsafe.Pointer(r))

loop:
	_, _, e = syscall.Syscall(uintptr(waitCommEvent), uintptr(unsafe.Pointer(&events)), uintptr(unsafe.Pointer(&overlapped)), 0, 0)
	if e != 0 {
		return 0, os.Errno(e)
	}

	_, _, e = syscall.Syscall(uintptr(getOverlappedResult), uintptr(fd), uintptr(unsafe.Pointer(&overlapped)), uintptr(unsafe.Pointer(&n)), 1)
	if e != 0 {
		return 0, os.Errno(e)
	}

	// There is a small race window here between returning from WaitCommEvent and reading from the file.
	// If we receive data in this window, we will read that data, but the RXFLAG will still be set
	// and next time the WaitCommEvent() will return but we will have already read the data.  That's
	// why we have the stupid goto loop on EOF.
	e1 := syscall.ReadFile(int32(fd), buf, &events, &overlapped)
	if e1 != 0 {
		return 0, os.Errno(e)
	}

	_, _, e = syscall.Syscall(uintptr(getOverlappedResult), uintptr(fd), uintptr(unsafe.Pointer(&overlapped)), uintptr(unsafe.Pointer(&n)), 1)
	if e != 0 {
		return 0, os.Errno(e)
	}
	if n == 0 {
		goto loop
	}

	return n, nil
}
<commit_msg>Clean up windows serial port syscalls<commit_after>package serial

import (
	"os"
	"io"
	//"fmt"
	"unsafe"
	"syscall"
)

type serialPort struct {
	f *os.File
}

const EV_RXCHAR = 0x0001

type structDCB struct {
	DCBlength, BaudRate uint32
	flags [4]byte
	wReserved, XonLim, XoffLim uint16
	ByteSize, Parity, StopBits byte
	XonChar, XoffChar, ErrorChar, EofChar, EvtChar byte
	wReserved1 uint16
}

type structTimeouts struct {
	ReadIntervalTimeout uint32
	ReadTotalTimeoutMultiplier uint32
	ReadTotalTimeoutConstant uint32
	WriteTotalTimeoutMultiplier uint32
	WriteTotalTimeoutConstant uint32
}

var nSetCommState uint32
var nSetCommTimeouts uint32
var nSetCommMask uint32
var nWaitCommEvent uint32
var nGetOverlappedResult uint32
var nCreateEvent uint32

func loadDll(lib uint32, name string) uint32 {
	addr, err := syscall.GetProcAddress(lib, name)
	if err != 0 {
		panic(name + " " + syscall.Errstr(err))
	}
	return addr
}

func init() {
	k32, err := syscall.LoadLibrary("kernel32.dll")
	if err != 0 {
		panic("LoadLibrary "+ syscall.Errstr(err))
	}
	defer syscall.FreeLibrary(k32)

	nSetCommState        = loadDll(k32, "SetCommState")
	nSetCommTimeouts     = loadDll(k32, "SetCommTimeouts")
	nSetCommMask         = loadDll(k32, "SetCommMask")
	nWaitCommEvent       = loadDll(k32, "WaitCommEvent")
	nGetOverlappedResult = loadDll(k32, "GetOverlappedResult")
	nCreateEvent         = loadDll(k32, "CreateEvent")
}

func setCommState(h int32, baud int) os.Error {
	var params structDCB
	params.DCBlength = uint32(unsafe.Sizeof(params))

	params.BaudRate = uint32(baud)
	params.ByteSize = 8

	_, _, e := syscall.Syscall(uintptr(nSetCommState), uintptr(h), uintptr(unsafe.Pointer(&params)), 0, 0)
	if e != 0 {
		return os.Errno(e)
	}
	return nil
}

func setCommTimeouts(h int32) os.Error {
	var timeouts structTimeouts
	timeouts.ReadIntervalTimeout = 1<<32 - 1
	timeouts.ReadTotalTimeoutConstant = 0
	_, _, e := syscall.Syscall(uintptr(nSetCommTimeouts), uintptr(h), uintptr(unsafe.Pointer(&timeouts)), 0, 0)
	if e != 0 {
		return os.Errno(e)
	}
	return nil
}

func setCommMask(h int32) os.Error {
	_, _, e := syscall.Syscall(uintptr(nSetCommMask), uintptr(h), EV_RXCHAR, 0, 0)
	if e != 0 {
		return os.Errno(e)
	}
	return nil
}

const FILE_FLAGS_OVERLAPPED = 0x40000000

func OpenPort(name string, baud int) (io.ReadWriteCloser, os.Error) {
	//h, e := CreateFile(StringToUTF16Ptr(path), access, sharemode, sa, createmode, FILE_ATTRIBUTE_NORMAL, 0)

	h, e := syscall.CreateFile(syscall.StringToUTF16Ptr(name),
		syscall.GENERIC_READ|syscall.GENERIC_WRITE,
		uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE),
		nil,
		syscall.OPEN_EXISTING,
		syscall.FILE_ATTRIBUTE_NORMAL|FILE_FLAGS_OVERLAPPED,
		0)

	if e != 0 {
		return nil, &os.PathError{"open", name, os.Errno(e)}
	}

	f := os.NewFile(int(h), name)

	if err := setCommState(h, baud); err != nil {
		f.Close()
		return nil, err
	}

	// Turn off buffers
	/*
	if ok, err := C.SetupComm(*handle, 16, 16); ok == C.FALSE {
		f.Close()
		return nil, err
	}
	 */

	if err := setCommTimeouts(h); err != nil {
		f.Close()
		return nil, err
	}
	if err := setCommMask(h); err != nil {
		f.Close()
		return nil, err
	}

	port := serialPort{f}

	return &port, nil
}

func (p *serialPort) Close() os.Error {
	return p.f.Close()
}

func newOverlapped() (*syscall.Overlapped, os.Error) {
	var overlapped syscall.Overlapped
	r, _, e := syscall.Syscall(uintptr(nCreateEvent), 0, 1, 0, 0)
	if e != 0 {
		return nil, os.Errno(e)
	}
	overlapped.HEvent = (*uint8)(unsafe.Pointer(r))
	return &overlapped, nil
}

func getOverlappedResults(h int, overlapped *syscall.Overlapped) (int, os.Error) {
	var n int
	_, _, e := syscall.Syscall(uintptr(nGetOverlappedResult), uintptr(h),
		uintptr(unsafe.Pointer(overlapped)),
		uintptr(unsafe.Pointer(&n)), 1)
	if e != 0 {
		return 0, os.Errno(e)
	}

	return n, nil
}

func (p *serialPort) Write(buf []byte) (int, os.Error) {
	var events uint32

	fd := p.f.Fd()

	overlapped, err := newOverlapped()
	if err != nil {
		return 0, err
	}

	e := syscall.WriteFile(int32(fd), buf, &events, overlapped)
	if e != 0 {
		return 0, os.Errno(e)
	}

	n, err := getOverlappedResults(fd, overlapped)

	return n, err
}

func waitCommEvent(overlapped *syscall.Overlapped) os.Error {
	var events uint32
	_, _, e := syscall.Syscall(uintptr(nWaitCommEvent), uintptr(unsafe.Pointer(&events)), uintptr(unsafe.Pointer(overlapped)), 0, 0)
	if e != 0 {
		return os.Errno(e)
	}
	return nil
}

func (p *serialPort) Read(buf []byte) (int, os.Error) {
	var events uint32

	fd := p.f.Fd()

	overlapped, err := newOverlapped()
	if err != nil {
		return 0, err
	}

loop:
	if err = waitCommEvent(overlapped); err != nil {
		return 0, nil
	}

	_, err = getOverlappedResults(fd, overlapped)
	if err != nil {
		return 0, err
	}

	// There is a small race window here between returning from WaitCommEvent and reading from the file.
	// If we receive data in this window, we will read that data, but the RXFLAG will still be set
	// and next time the WaitCommEvent() will return but we will have already read the data.  That's
	// why we have the stupid goto loop on EOF.
	e := syscall.ReadFile(int32(fd), buf, &events, overlapped)
	if e != 0 {
		return 0, os.Errno(e)
	}

	n, err := getOverlappedResults(fd, overlapped)
	if err != nil {
		return 0, err
	}
	if n == 0 {
		goto loop
	}

	return n, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package coordinator

import (
	"bytes"
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/proto/google"
	logdog "go.chromium.org/luci/logdog/api/endpoints/coordinator/services/v1"
)

// ErrStreamArchived is returned by ArchivalParams' PublishTask if the supplied
// LogStream is already archived.
var ErrStreamArchived = errors.New("stream is already archived")

// ArchivalParams is the archival configuration.
type ArchivalParams struct {
	// RequestID is the unique request ID to use as a random base for the
	// archival key.
	RequestID string

	// PreviousKey (rescheduled tasks only) is the archive key of the previous archive task.
	PreviousKey []byte

	// SettleDelay is the amount of settle delay to attach to this request.
	SettleDelay time.Duration

	// CompletePeriod is the amount of time after the initial archival task is
	// executed when the task should fail if the stream is incomplete. After this
	// period has expired, the archival may complete successfully even if the
	// stream is missing log entries.
	CompletePeriod time.Duration
}

// PublishTask creates and dispatches a task queue task for the supplied
// LogStream. PublishTask is goroutine-safe.
//
// This should be run within a transaction on lst. On success, lst's state will
// be updated to reflect the archival tasking. This will NOT update lst's
// datastore entity; the caller must make sure to call Put within the same
// transaction for transactional safety.
//
// If the task is created successfully, this will return nil. If the LogStream
// already had a task dispatched, it will return ErrStreamArchived.
func (p *ArchivalParams) PublishTask(c context.Context, ap ArchivalPublisher, lst *LogStreamState) error {
	if lst.ArchivalState().Archived() {
		// An archival task has already been dispatched for this log stream.
		return ErrStreamArchived
	}

	id := lst.ID()
	if len(lst.ArchivalKey) > 0 {
		// This is a rescheduled task. Check to see if the key matches.
		if !bytes.Equal(p.PreviousKey, lst.ArchivalKey) {
			logging.Warningf(c, "Key does not match, this is probably a duplicate request, discarding.")
			return nil
		}
		lst.ArchiveRetryCount++
	}
	msg := logdog.ArchiveTask{
		Project:      string(Project(c)),
		Id:           string(id),
		Key:          p.createArchivalKey(id, ap.NewPublishIndex()),
		DispatchedAt: google.NewTimestamp(clock.Now(c)),
	}
	if p.SettleDelay > 0 {
		msg.SettleDelay = google.NewDuration(p.SettleDelay)
	}
	if p.CompletePeriod > 0 {
		msg.CompletePeriod = google.NewDuration(p.CompletePeriod)
	}

	// Publish an archival request.
	if err := ap.Publish(c, &msg); err != nil {
		return err
	}

	// Update our LogStream's ArchiveState to reflect that an archival task has
	// been dispatched.
	lst.ArchivalKey = msg.Key
	return nil
}

// createArchivalKey returns a unique archival request key.
//
// The uniqueness is ensured by folding several components into a hash:
//	- The request ID, which is unique per HTTP request.
//	- The stream path.
//	- An atomically-incrementing key index, which is unique per ArchivalParams
//	  instance.
//
// The first two should be sufficient for a unique value, since a given request
// will only be handed to a single instance, and the atomic value is unique
// within the instance.
func (p *ArchivalParams) createArchivalKey(id HashID, pidx uint64) []byte {
	hash := sha256.New()
	if _, err := fmt.Fprintf(hash, "%s\x00%s\x00%d", p.RequestID, id, pidx); err != nil {
		panic(err)
	}
	return hash.Sum(nil)
}
<commit_msg>[logdog] Remove duplicate request check in tumble<commit_after>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package coordinator

import (
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/proto/google"
	logdog "go.chromium.org/luci/logdog/api/endpoints/coordinator/services/v1"
)

// ErrStreamArchived is returned by ArchivalParams' PublishTask if the supplied
// LogStream is already archived.
var ErrStreamArchived = errors.New("stream is already archived")

// ArchivalParams is the archival configuration.
type ArchivalParams struct {
	// RequestID is the unique request ID to use as a random base for the
	// archival key.
	RequestID string

	// PreviousKey (rescheduled tasks only) is the archive key of the previous archive task.
	PreviousKey []byte

	// SettleDelay is the amount of settle delay to attach to this request.
	SettleDelay time.Duration

	// CompletePeriod is the amount of time after the initial archival task is
	// executed when the task should fail if the stream is incomplete. After this
	// period has expired, the archival may complete successfully even if the
	// stream is missing log entries.
	CompletePeriod time.Duration
}

// PublishTask creates and dispatches a task queue task for the supplied
// LogStream. PublishTask is goroutine-safe.
//
// This should be run within a transaction on lst. On success, lst's state will
// be updated to reflect the archival tasking. This will NOT update lst's
// datastore entity; the caller must make sure to call Put within the same
// transaction for transactional safety.
//
// If the task is created successfully, this will return nil. If the LogStream
// already had a task dispatched, it will return ErrStreamArchived.
func (p *ArchivalParams) PublishTask(c context.Context, ap ArchivalPublisher, lst *LogStreamState) error {
	if lst.ArchivalState().Archived() {
		// An archival task has already been dispatched for this log stream.
		return ErrStreamArchived
	}

	id := lst.ID()
	msg := logdog.ArchiveTask{
		Project:      string(Project(c)),
		Id:           string(id),
		Key:          p.createArchivalKey(id, ap.NewPublishIndex()),
		DispatchedAt: google.NewTimestamp(clock.Now(c)),
	}
	if p.SettleDelay > 0 {
		msg.SettleDelay = google.NewDuration(p.SettleDelay)
	}
	if p.CompletePeriod > 0 {
		msg.CompletePeriod = google.NewDuration(p.CompletePeriod)
	}

	// Publish an archival request.
	if err := ap.Publish(c, &msg); err != nil {
		return err
	}

	// Update our LogStream's ArchiveState to reflect that an archival task has
	// been dispatched.
	lst.ArchivalKey = msg.Key
	return nil
}

// createArchivalKey returns a unique archival request key.
//
// The uniqueness is ensured by folding several components into a hash:
//	- The request ID, which is unique per HTTP request.
//	- The stream path.
//	- An atomically-incrementing key index, which is unique per ArchivalParams
//	  instance.
//
// The first two should be sufficient for a unique value, since a given request
// will only be handed to a single instance, and the atomic value is unique
// within the instance.
func (p *ArchivalParams) createArchivalKey(id HashID, pidx uint64) []byte {
	hash := sha256.New()
	if _, err := fmt.Fprintf(hash, "%s\x00%s\x00%d", p.RequestID, id, pidx); err != nil {
		panic(err)
	}
	return hash.Sum(nil)
}
<|endoftext|>


<commit_before>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package queue

import (
	"crypto/ed25519"
	"encoding/json"
	"fmt"
	"sync"
	"time"

	"github.com/matrix-org/gomatrixserverlib"
	"github.com/prometheus/client_golang/prometheus"
	log "github.com/sirupsen/logrus"
	"github.com/tidwall/gjson"

	fedapi "github.com/matrix-org/dendrite/federationapi/api"
	"github.com/matrix-org/dendrite/federationapi/statistics"
	"github.com/matrix-org/dendrite/federationapi/storage"
	"github.com/matrix-org/dendrite/federationapi/storage/shared"
	"github.com/matrix-org/dendrite/roomserver/api"
	"github.com/matrix-org/dendrite/setup/process"
)

// OutgoingQueues is a collection of queues for sending transactions to other
// matrix servers
type OutgoingQueues struct {
	db          storage.Database
	process     *process.ProcessContext
	disabled    bool
	rsAPI       api.FederationRoomserverAPI
	origin      gomatrixserverlib.ServerName
	client      fedapi.FederationClient
	statistics  *statistics.Statistics
	signing     *SigningInfo
	queuesMutex sync.Mutex // protects the below
	queues      map[gomatrixserverlib.ServerName]*destinationQueue
}

func init() {
	prometheus.MustRegister(
		destinationQueueTotal, destinationQueueRunning,
		destinationQueueBackingOff,
	)
}

var destinationQueueTotal = prometheus.NewGauge(
	prometheus.GaugeOpts{
		Namespace: "dendrite",
		Subsystem: "federationapi",
		Name:      "destination_queues_total",
	},
)

var destinationQueueRunning = prometheus.NewGauge(
	prometheus.GaugeOpts{
		Namespace: "dendrite",
		Subsystem: "federationapi",
		Name:      "destination_queues_running",
	},
)

var destinationQueueBackingOff = prometheus.NewGauge(
	prometheus.GaugeOpts{
		Namespace: "dendrite",
		Subsystem: "federationapi",
		Name:      "destination_queues_backing_off",
	},
)

// NewOutgoingQueues makes a new OutgoingQueues
func NewOutgoingQueues(
	db storage.Database,
	process *process.ProcessContext,
	disabled bool,
	origin gomatrixserverlib.ServerName,
	client fedapi.FederationClient,
	rsAPI api.FederationRoomserverAPI,
	statistics *statistics.Statistics,
	signing *SigningInfo,
) *OutgoingQueues {
	queues := &OutgoingQueues{
		disabled:   disabled,
		process:    process,
		db:         db,
		rsAPI:      rsAPI,
		origin:     origin,
		client:     client,
		statistics: statistics,
		signing:    signing,
		queues:     map[gomatrixserverlib.ServerName]*destinationQueue{},
	}
	// Look up which servers we have pending items for and then rehydrate those queues.
	if !disabled {
		serverNames := map[gomatrixserverlib.ServerName]struct{}{}
		if names, err := db.GetPendingPDUServerNames(process.Context()); err == nil {
			for _, serverName := range names {
				serverNames[serverName] = struct{}{}
			}
		} else {
			log.WithError(err).Error("Failed to get PDU server names for destination queue hydration")
		}
		if names, err := db.GetPendingEDUServerNames(process.Context()); err == nil {
			for _, serverName := range names {
				serverNames[serverName] = struct{}{}
			}
		} else {
			log.WithError(err).Error("Failed to get EDU server names for destination queue hydration")
		}
		offset, step := time.Second*5, time.Second
		if max := len(serverNames); max > 120 {
			step = (time.Second * 120) / time.Duration(max)
		}
		for serverName := range serverNames {
			if queue := queues.getQueue(serverName); queue != nil {
				time.AfterFunc(offset, queue.wakeQueueIfNeeded)
				offset += step
			}
		}
	}
	return queues
}

// TODO: Move this somewhere useful for other components as we often need to ferry these 3 variables
// around together
type SigningInfo struct {
	ServerName gomatrixserverlib.ServerName
	KeyID      gomatrixserverlib.KeyID
	PrivateKey ed25519.PrivateKey
}

type queuedPDU struct {
	receipt *shared.Receipt
	pdu     *gomatrixserverlib.HeaderedEvent
}

type queuedEDU struct {
	receipt *shared.Receipt
	edu     *gomatrixserverlib.EDU
}

func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *destinationQueue {
	if oqs.statistics.ForServer(destination).Blacklisted() {
		return nil
	}
	oqs.queuesMutex.Lock()
	defer oqs.queuesMutex.Unlock()
	oq, ok := oqs.queues[destination]
	if !ok || oq != nil {
		destinationQueueTotal.Inc()
		oq = &destinationQueue{
			queues:           oqs,
			db:               oqs.db,
			process:          oqs.process,
			rsAPI:            oqs.rsAPI,
			origin:           oqs.origin,
			destination:      destination,
			client:           oqs.client,
			statistics:       oqs.statistics.ForServer(destination),
			notify:           make(chan struct{}, 1),
			interruptBackoff: make(chan bool),
			signing:          oqs.signing,
		}
		oqs.queues[destination] = oq
	}
	return oq
}

func (oqs *OutgoingQueues) clearQueue(oq *destinationQueue) {
	oqs.queuesMutex.Lock()
	defer oqs.queuesMutex.Unlock()

	delete(oqs.queues, oq.destination)
	destinationQueueTotal.Dec()
}

// SendEvent sends an event to the destinations
func (oqs *OutgoingQueues) SendEvent(
	ev *gomatrixserverlib.HeaderedEvent, origin gomatrixserverlib.ServerName,
	destinations []gomatrixserverlib.ServerName,
) error {
	if oqs.disabled {
		log.Trace("Federation is disabled, not sending event")
		return nil
	}
	if origin != oqs.origin {
		// TODO: Support virtual hosting; gh issue #577.
		return fmt.Errorf(
			"sendevent: unexpected server to send as: got %q expected %q",
			origin, oqs.origin,
		)
	}

	// Deduplicate destinations and remove the origin from the list of
	// destinations just to be sure.
	destmap := map[gomatrixserverlib.ServerName]struct{}{}
	for _, d := range destinations {
		destmap[d] = struct{}{}
	}
	delete(destmap, oqs.origin)
	delete(destmap, oqs.signing.ServerName)

	// Check if any of the destinations are prohibited by server ACLs.
	for destination := range destmap {
		if api.IsServerBannedFromRoom(
			oqs.process.Context(),
			oqs.rsAPI,
			ev.RoomID(),
			destination,
		) {
			delete(destmap, destination)
		}
	}

	// If there are no remaining destinations then give up.
	if len(destmap) == 0 {
		return nil
	}

	log.WithFields(log.Fields{
		"destinations": len(destmap), "event": ev.EventID(),
	}).Infof("Sending event")

	headeredJSON, err := json.Marshal(ev)
	if err != nil {
		return fmt.Errorf("json.Marshal: %w", err)
	}

	nid, err := oqs.db.StoreJSON(oqs.process.Context(), string(headeredJSON))
	if err != nil {
		return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err)
	}

	for destination := range destmap {
		if queue := oqs.getQueue(destination); queue != nil {
			queue.sendEvent(ev, nid)
		}
	}

	return nil
}

// SendEDU sends an EDU event to the destinations.
func (oqs *OutgoingQueues) SendEDU(
	e *gomatrixserverlib.EDU, origin gomatrixserverlib.ServerName,
	destinations []gomatrixserverlib.ServerName,
) error {
	if oqs.disabled {
		log.Trace("Federation is disabled, not sending EDU")
		return nil
	}
	if origin != oqs.origin {
		// TODO: Support virtual hosting; gh issue #577.
		return fmt.Errorf(
			"sendevent: unexpected server to send as: got %q expected %q",
			origin, oqs.origin,
		)
	}

	// Deduplicate destinations and remove the origin from the list of
	// destinations just to be sure.
	destmap := map[gomatrixserverlib.ServerName]struct{}{}
	for _, d := range destinations {
		destmap[d] = struct{}{}
	}
	delete(destmap, oqs.origin)
	delete(destmap, oqs.signing.ServerName)

	// There is absolutely no guarantee that the EDU will have a room_id
	// field, as it is not required by the spec. However, if it *does*
	// (e.g. typing notifications) then we should try to make sure we don't
	// bother sending them to servers that are prohibited by the server
	// ACLs.
	if result := gjson.GetBytes(e.Content, "room_id"); result.Exists() {
		for destination := range destmap {
			if api.IsServerBannedFromRoom(
				oqs.process.Context(),
				oqs.rsAPI,
				result.Str,
				destination,
			) {
				delete(destmap, destination)
			}
		}
	}

	// If there are no remaining destinations then give up.
	if len(destmap) == 0 {
		return nil
	}

	log.WithFields(log.Fields{
		"destinations": len(destmap), "edu_type": e.Type,
	}).Info("Sending EDU event")

	ephemeralJSON, err := json.Marshal(e)
	if err != nil {
		return fmt.Errorf("json.Marshal: %w", err)
	}

	nid, err := oqs.db.StoreJSON(oqs.process.Context(), string(ephemeralJSON))
	if err != nil {
		return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err)
	}

	for destination := range destmap {
		if queue := oqs.getQueue(destination); queue != nil {
			queue.sendEDU(e, nid)
		}
	}

	return nil
}

// RetryServer attempts to resend events to the given server if we had given up.
func (oqs *OutgoingQueues) RetryServer(srv gomatrixserverlib.ServerName) {
	if oqs.disabled {
		return
	}
	if queue := oqs.getQueue(srv); queue != nil {
		queue.wakeQueueIfNeeded()
	}
}
<commit_msg>Only create a new destinationQueue if we don't have one (#2620)<commit_after>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package queue

import (
	"crypto/ed25519"
	"encoding/json"
	"fmt"
	"sync"
	"time"

	"github.com/matrix-org/gomatrixserverlib"
	"github.com/prometheus/client_golang/prometheus"
	log "github.com/sirupsen/logrus"
	"github.com/tidwall/gjson"

	fedapi "github.com/matrix-org/dendrite/federationapi/api"
	"github.com/matrix-org/dendrite/federationapi/statistics"
	"github.com/matrix-org/dendrite/federationapi/storage"
	"github.com/matrix-org/dendrite/federationapi/storage/shared"
	"github.com/matrix-org/dendrite/roomserver/api"
	"github.com/matrix-org/dendrite/setup/process"
)

// OutgoingQueues is a collection of queues for sending transactions to other
// matrix servers
type OutgoingQueues struct {
	db          storage.Database
	process     *process.ProcessContext
	disabled    bool
	rsAPI       api.FederationRoomserverAPI
	origin      gomatrixserverlib.ServerName
	client      fedapi.FederationClient
	statistics  *statistics.Statistics
	signing     *SigningInfo
	queuesMutex sync.Mutex // protects the below
	queues      map[gomatrixserverlib.ServerName]*destinationQueue
}

func init() {
	prometheus.MustRegister(
		destinationQueueTotal, destinationQueueRunning,
		destinationQueueBackingOff,
	)
}

var destinationQueueTotal = prometheus.NewGauge(
	prometheus.GaugeOpts{
		Namespace: "dendrite",
		Subsystem: "federationapi",
		Name:      "destination_queues_total",
	},
)

var destinationQueueRunning = prometheus.NewGauge(
	prometheus.GaugeOpts{
		Namespace: "dendrite",
		Subsystem: "federationapi",
		Name:      "destination_queues_running",
	},
)

var destinationQueueBackingOff = prometheus.NewGauge(
	prometheus.GaugeOpts{
		Namespace: "dendrite",
		Subsystem: "federationapi",
		Name:      "destination_queues_backing_off",
	},
)

// NewOutgoingQueues makes a new OutgoingQueues
func NewOutgoingQueues(
	db storage.Database,
	process *process.ProcessContext,
	disabled bool,
	origin gomatrixserverlib.ServerName,
	client fedapi.FederationClient,
	rsAPI api.FederationRoomserverAPI,
	statistics *statistics.Statistics,
	signing *SigningInfo,
) *OutgoingQueues {
	queues := &OutgoingQueues{
		disabled:   disabled,
		process:    process,
		db:         db,
		rsAPI:      rsAPI,
		origin:     origin,
		client:     client,
		statistics: statistics,
		signing:    signing,
		queues:     map[gomatrixserverlib.ServerName]*destinationQueue{},
	}
	// Look up which servers we have pending items for and then rehydrate those queues.
	if !disabled {
		serverNames := map[gomatrixserverlib.ServerName]struct{}{}
		if names, err := db.GetPendingPDUServerNames(process.Context()); err == nil {
			for _, serverName := range names {
				serverNames[serverName] = struct{}{}
			}
		} else {
			log.WithError(err).Error("Failed to get PDU server names for destination queue hydration")
		}
		if names, err := db.GetPendingEDUServerNames(process.Context()); err == nil {
			for _, serverName := range names {
				serverNames[serverName] = struct{}{}
			}
		} else {
			log.WithError(err).Error("Failed to get EDU server names for destination queue hydration")
		}
		offset, step := time.Second*5, time.Second
		if max := len(serverNames); max > 120 {
			step = (time.Second * 120) / time.Duration(max)
		}
		for serverName := range serverNames {
			if queue := queues.getQueue(serverName); queue != nil {
				time.AfterFunc(offset, queue.wakeQueueIfNeeded)
				offset += step
			}
		}
	}
	return queues
}

// TODO: Move this somewhere useful for other components as we often need to ferry these 3 variables
// around together
type SigningInfo struct {
	ServerName gomatrixserverlib.ServerName
	KeyID      gomatrixserverlib.KeyID
	PrivateKey ed25519.PrivateKey
}

type queuedPDU struct {
	receipt *shared.Receipt
	pdu     *gomatrixserverlib.HeaderedEvent
}

type queuedEDU struct {
	receipt *shared.Receipt
	edu     *gomatrixserverlib.EDU
}

func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *destinationQueue {
	if oqs.statistics.ForServer(destination).Blacklisted() {
		return nil
	}
	oqs.queuesMutex.Lock()
	defer oqs.queuesMutex.Unlock()
	oq, ok := oqs.queues[destination]
	if !ok || oq == nil {
		destinationQueueTotal.Inc()
		oq = &destinationQueue{
			queues:           oqs,
			db:               oqs.db,
			process:          oqs.process,
			rsAPI:            oqs.rsAPI,
			origin:           oqs.origin,
			destination:      destination,
			client:           oqs.client,
			statistics:       oqs.statistics.ForServer(destination),
			notify:           make(chan struct{}, 1),
			interruptBackoff: make(chan bool),
			signing:          oqs.signing,
		}
		oqs.queues[destination] = oq
	}
	return oq
}

func (oqs *OutgoingQueues) clearQueue(oq *destinationQueue) {
	oqs.queuesMutex.Lock()
	defer oqs.queuesMutex.Unlock()

	delete(oqs.queues, oq.destination)
	destinationQueueTotal.Dec()
}

// SendEvent sends an event to the destinations
func (oqs *OutgoingQueues) SendEvent(
	ev *gomatrixserverlib.HeaderedEvent, origin gomatrixserverlib.ServerName,
	destinations []gomatrixserverlib.ServerName,
) error {
	if oqs.disabled {
		log.Trace("Federation is disabled, not sending event")
		return nil
	}
	if origin != oqs.origin {
		// TODO: Support virtual hosting; gh issue #577.
		return fmt.Errorf(
			"sendevent: unexpected server to send as: got %q expected %q",
			origin, oqs.origin,
		)
	}

	// Deduplicate destinations and remove the origin from the list of
	// destinations just to be sure.
	destmap := map[gomatrixserverlib.ServerName]struct{}{}
	for _, d := range destinations {
		destmap[d] = struct{}{}
	}
	delete(destmap, oqs.origin)
	delete(destmap, oqs.signing.ServerName)

	// Check if any of the destinations are prohibited by server ACLs.
	for destination := range destmap {
		if api.IsServerBannedFromRoom(
			oqs.process.Context(),
			oqs.rsAPI,
			ev.RoomID(),
			destination,
		) {
			delete(destmap, destination)
		}
	}

	// If there are no remaining destinations then give up.
	if len(destmap) == 0 {
		return nil
	}

	log.WithFields(log.Fields{
		"destinations": len(destmap), "event": ev.EventID(),
	}).Infof("Sending event")

	headeredJSON, err := json.Marshal(ev)
	if err != nil {
		return fmt.Errorf("json.Marshal: %w", err)
	}

	nid, err := oqs.db.StoreJSON(oqs.process.Context(), string(headeredJSON))
	if err != nil {
		return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err)
	}

	for destination := range destmap {
		if queue := oqs.getQueue(destination); queue != nil {
			queue.sendEvent(ev, nid)
		}
	}

	return nil
}

// SendEDU sends an EDU event to the destinations.
func (oqs *OutgoingQueues) SendEDU(
	e *gomatrixserverlib.EDU, origin gomatrixserverlib.ServerName,
	destinations []gomatrixserverlib.ServerName,
) error {
	if oqs.disabled {
		log.Trace("Federation is disabled, not sending EDU")
		return nil
	}
	if origin != oqs.origin {
		// TODO: Support virtual hosting; gh issue #577.
		return fmt.Errorf(
			"sendevent: unexpected server to send as: got %q expected %q",
			origin, oqs.origin,
		)
	}

	// Deduplicate destinations and remove the origin from the list of
	// destinations just to be sure.
	destmap := map[gomatrixserverlib.ServerName]struct{}{}
	for _, d := range destinations {
		destmap[d] = struct{}{}
	}
	delete(destmap, oqs.origin)
	delete(destmap, oqs.signing.ServerName)

	// There is absolutely no guarantee that the EDU will have a room_id
	// field, as it is not required by the spec. However, if it *does*
	// (e.g. typing notifications) then we should try to make sure we don't
	// bother sending them to servers that are prohibited by the server
	// ACLs.
	if result := gjson.GetBytes(e.Content, "room_id"); result.Exists() {
		for destination := range destmap {
			if api.IsServerBannedFromRoom(
				oqs.process.Context(),
				oqs.rsAPI,
				result.Str,
				destination,
			) {
				delete(destmap, destination)
			}
		}
	}

	// If there are no remaining destinations then give up.
	if len(destmap) == 0 {
		return nil
	}

	log.WithFields(log.Fields{
		"destinations": len(destmap), "edu_type": e.Type,
	}).Info("Sending EDU event")

	ephemeralJSON, err := json.Marshal(e)
	if err != nil {
		return fmt.Errorf("json.Marshal: %w", err)
	}

	nid, err := oqs.db.StoreJSON(oqs.process.Context(), string(ephemeralJSON))
	if err != nil {
		return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err)
	}

	for destination := range destmap {
		if queue := oqs.getQueue(destination); queue != nil {
			queue.sendEDU(e, nid)
		}
	}

	return nil
}

// RetryServer attempts to resend events to the given server if we had given up.
func (oqs *OutgoingQueues) RetryServer(srv gomatrixserverlib.ServerName) {
	if oqs.disabled {
		return
	}
	if queue := oqs.getQueue(srv); queue != nil {
		queue.wakeQueueIfNeeded()
	}
}
<|endoftext|>


<commit_before>package main

import (
  "bufio"
  "path/filepath"
  "fmt"
  "os"
  "crypto/rand"
  "math/big"
  "unicode"
  "unicode/utf8"
  "flag"
)

func main() {
  var dict        string = "/usr/share/dict/"
  var words       []string
  var err         error
  var dictLength  *big.Int

  lang, pplen, wordMaxLen := setFlags()
  langPath := dict + *lang

  if len(*lang) == 0 {
    usage(dict)
    os.Exit(1)
  }

  _, err = os.Stat(langPath)
  if os.IsNotExist(err) {
    fmt.Printf("Language \"%s\" is not installed\n\n", *lang)
    usage(dict)
    os.Exit(1)
  }

  words = readLines(langPath, *wordMaxLen)
  fmt.Printf("  - %d\n", *pplen)
  fmt.Printf("  - %d\n", len(words))
  fmt.Printf("  - %d\n", *wordMaxLen)

  dictLength = big.NewInt(int64(len(words)))
  for i := 0; i < *pplen; i++ {
    var index *big.Int
    index, err = rand.Int(rand.Reader, dictLength)
    word := []byte(words[index.Uint64()])
    fmt.Printf("%s", upperFirst(toUtf8(word)))
  }
  fmt.Println("")
}

func setFlags() (*string, *int, *int){
  lang        := flag.String("l", "", "Language to use for generating the passphrase")
  numWords    := flag.Int("n", 5, "Number of words to use in the passphrase")
  wordMaxLen  := flag.Int("m", 10, "Max length of words to use in passphrase")
  flag.Parse()

  return lang, numWords, wordMaxLen
}

func toUtf8(iso8859_1_buf []byte) string {
    buf := make([]rune, len(iso8859_1_buf))
    for i, b := range iso8859_1_buf {
        buf[i] = rune(b)
    }
    return string(buf)
}

func upperFirst(s string) string {
	if s == "" {
		return ""
	}
	r, n := utf8.DecodeRuneInString(s)
	return string(unicode.ToUpper(r)) + s[n:]
}

func usage(dictPath string) {
  fmt.Println("Usage:")
  flag.PrintDefaults()
  fmt.Println("")
  fmt.Println("Available languages")
  fmt.Println("===================")
  filepath.Walk(
    dictPath,
    func(path string, f os.FileInfo, err error) error {
      if !f.IsDir() && f.Mode()&os.ModeSymlink == 0 && f.Name() != "README.select-wordlist"{
        fmt.Printf("  - %s\n", f.Name())
      }
      return nil
    },
  )
}

// readLines reads a whole file into memory
// and returns a slice of its lines.
func readLines(path string, wordMaxLen int) ([]string) {
  file, err := os.Open(path)
  if err != nil {
    return nil
  }
  defer file.Close()

  var lines []string
  scanner := bufio.NewScanner(file)
  for scanner.Scan() {
    word := scanner.Text()
    if (len(word) <= wordMaxLen) {
      lines = append(lines, word)
    }
  }
  return lines
}
<commit_msg>Cleaning it up a bit<commit_after>package main

import (
  "bufio"
  "path/filepath"
  "fmt"
  "os"
  "crypto/rand"
  "math/big"
  "unicode"
  "unicode/utf8"
  "flag"
)

func main() {
  var dicts       string = "/usr/share/dict/"
  var err         error

  lang, pplen, wordMaxLen := setFlags()
  langPath := dicts + *lang

  if len(*lang) == 0 {
    usage(dicts)
    os.Exit(1)
  }

  _, err = os.Stat(langPath)
  if os.IsNotExist(err) {
    fmt.Printf("Language \"%s\" is not installed\n\n", *lang)
    usage(dicts)
    os.Exit(1)
  }

  words := readLines(langPath, *wordMaxLen)
  fmt.Printf("  - %d\n", *pplen)
  fmt.Printf("  - %d\n", len(words))
  fmt.Printf("  - %d\n", *wordMaxLen)

  dictLength := big.NewInt(int64(len(words)))
  for i := 0; i < *pplen; i++ {
    var index *big.Int
    index, err = rand.Int(rand.Reader, dictLength)
    word := []byte(words[index.Uint64()])
    fmt.Printf("%s", upperFirst(toUtf8(word)))
  }
  fmt.Println("")
}

func setFlags() (*string, *int, *int){
  lang        := flag.String("l", "", "Language to use for generating the passphrase")
  numWords    := flag.Int("n", 5, "Number of words to use in the passphrase")
  wordMaxLen  := flag.Int("m", 10, "Max length of words to use in passphrase")
  flag.Parse()

  return lang, numWords, wordMaxLen
}

func toUtf8(iso8859_1_buf []byte) string {
    buf := make([]rune, len(iso8859_1_buf))
    for i, b := range iso8859_1_buf {
        buf[i] = rune(b)
    }
    return string(buf)
}

func upperFirst(s string) string {
	if s == "" {
		return ""
	}
	r, n := utf8.DecodeRuneInString(s)
	return string(unicode.ToUpper(r)) + s[n:]
}

func usage(dictPath string) {
  fmt.Println("Usage:")
  flag.PrintDefaults()
  fmt.Println("")
  fmt.Println("Available languages")
  fmt.Println("===================")
  filepath.Walk(
    dictPath,
    func(path string, f os.FileInfo, err error) error {
      if !f.IsDir() && f.Mode()&os.ModeSymlink == 0 && f.Name() != "README.select-wordlist"{
        fmt.Printf("  - %s\n", f.Name())
      }
      return nil
    },
  )
}

// readLines reads a whole file into memory
// and returns a slice of its lines.
func readLines(path string, wordMaxLen int) ([]string) {
  file, err := os.Open(path)
  if err != nil {
    return nil
  }
  defer file.Close()

  var lines []string
  scanner := bufio.NewScanner(file)
  for scanner.Scan() {
    word := scanner.Text()
    if (len(word) <= wordMaxLen) {
      lines = append(lines, word)
    }
  }
  return lines
}
<|endoftext|>


<commit_before>// Package wsdlgen generates Go source code from wsdl documents.
//
// The wsdlgen package generates Go source for calling the various
// methods defined in a WSDL (Web Service Definition Language) document.
// The generated Go source is self-contained, with no dependencies on
// non-standard packages.
//
// Code generation for the wsdlgen package can be configured by using
// the provided Option functions.
package wsdlgen

import (
	"encoding/xml"
	"errors"
	"fmt"
	"go/ast"
	"io/ioutil"
	"strings"

	"aqwari.net/xml/internal/gen"
	"aqwari.net/xml/wsdl"
	"aqwari.net/xml/xsdgen"
)

// Types conforming to the Logger interface can receive information about
// the code generation process.
type Logger interface {
	Printf(format string, v ...interface{})
}

type printer struct {
	*Config
	code *xsdgen.Code
	wsdl *wsdl.Definition
	file *ast.File
}

// Provides aspects about an RPC call to the template for the function
// bodies.
type opArgs struct {
	// formatted with appropriate variable names
	input, output []string

	// URL to send request to
	Address string

	// POST or GET
	Method string

	// Name of the method to call
	MsgName xml.Name

	// if we're returning individual values, these slices
	// are in an order matching the input/output slices.
	InputName, OutputName xml.Name
	InputFields           []field
	OutputFields          []field

	// If not "", inputs come in a wrapper struct
	InputType string

	// If not "", we return values in a wrapper struct
	ReturnType   string
	ReturnFields []field
}

// struct members. Need to export the fields for our template
type field struct {
	Name, Type string
	XMLName    xml.Name

	// If this is a wrapper struct for >InputThreshold arguments,
	// PublicType holds the type that we want to expose to the
	// user. For example, if the web service expects an xsdDate
	// to be sent to it, PublicType will be time.Time and a conversion
	// will take place before sending the request to the server.
	PublicType string

	// This refers to the name of the value to assign to this field
	// in the argument list. Empty for return values.
	InputArg string
}

// GenAST creates a Go source file containing type and method declarations
// that can be used to access the service described in the provided set of wsdl
// files.
func (cfg *Config) GenAST(files ...string) (*ast.File, error) {
	if len(files) == 0 {
		return nil, errors.New("must provide at least one file name")
	}
	if cfg.pkgName == "" {
		cfg.pkgName = "ws"
	}
	if cfg.pkgHeader == "" {
		cfg.pkgHeader = fmt.Sprintf("Package %s", cfg.pkgName)
	}
	docs := make([][]byte, 0, len(files))
	for _, filename := range files {
		if data, err := ioutil.ReadFile(filename); err != nil {
			return nil, err
		} else {
			cfg.debugf("read %s", filename)
			docs = append(docs, data)
		}
	}

	cfg.debugf("parsing WSDL file %s", files[0])
	def, err := wsdl.Parse(docs[0])
	if err != nil {
		return nil, err
	}
	cfg.verbosef("building xsd type whitelist from WSDL")
	cfg.registerXSDTypes(def)

	cfg.verbosef("generating type declarations from xml schema")
	code, err := cfg.xsdgen.GenCode(docs...)
	if err != nil {
		return nil, err
	}

	cfg.verbosef("generating function definitions from WSDL")
	return cfg.genAST(def, code)
}

func (cfg *Config) genAST(def *wsdl.Definition, code *xsdgen.Code) (*ast.File, error) {
	file, err := code.GenAST()
	if err != nil {
		return nil, err
	}
	file.Name = ast.NewIdent(cfg.pkgName)
	file = gen.PackageDoc(file, cfg.pkgHeader, "\n", def.Doc)
	p := &printer{
		Config: cfg,
		wsdl:   def,
		file:   file,
		code:   code,
	}
	return p.genAST()
}

func (p *printer) genAST() (*ast.File, error) {
	p.addHelpers()
	for _, port := range p.wsdl.Ports {
		if err := p.port(port); err != nil {
			return nil, err
		}
	}
	return p.file, nil
}

func (p *printer) port(port wsdl.Port) error {
	for _, operation := range port.Operations {
		if err := p.operation(port, operation); err != nil {
			return err
		}
	}
	return nil
}

func (p *printer) operation(port wsdl.Port, op wsdl.Operation) error {
	input, ok := p.wsdl.Message[op.Input]
	if !ok {
		return fmt.Errorf("unknown input message type %s", op.Input.Local)
	}
	output, ok := p.wsdl.Message[op.Output]
	if !ok {
		return fmt.Errorf("unknown output message type %s", op.Output.Local)
	}
	params, err := p.opArgs(port.Address, port.Method, input, output)
	if err != nil {
		return err
	}

	if params.InputType != "" {
		decls, err := gen.Snippets(params, `
			type {{.InputType}} struct {
			{{ range .InputFields -}}
				{{.Name}} {{.PublicType}}
			{{ end -}}
			}`,
		)
		if err != nil {
			return err
		}
		p.file.Decls = append(p.file.Decls, decls...)
	}
	if params.ReturnType != "" {
		decls, err := gen.Snippets(params, `
			type {{.ReturnType}} struct {
			{{ range .ReturnFields -}}
				{{.Name}} {{.Type}}
			{{ end -}}
			}`,
		)
		if err != nil {
			return err
		}
		p.file.Decls = append(p.file.Decls, decls...)
	}
	fn := gen.Func(p.xsdgen.NameOf(op.Name)).
		Comment(op.Doc).
		Receiver("c *Client").
		Args(params.input...).
		BodyTmpl(`
			var input struct {
				XMLName struct{} `+"`"+`xml:"{{.InputName.Space}} {{.InputName.Local}}"`+"`"+`
				{{ range .InputFields -}}
				{{.Name}} {{.Type}} `+"`"+`xml:"{{.XMLName.Space}} {{.XMLName.Local}}"`+"`"+`
				{{ end -}}
			}

			{{- range .InputFields }}
			input.{{.Name}} = {{.Type}}({{.InputArg}})
			{{ end }}

			var output struct {
				XMLName struct{} `+"`"+`xml:"{{.OutputName.Space}} {{.OutputName.Local}}"`+"`"+`
				{{ range .OutputFields -}}
				{{.Name}} {{.Type}} `+"`"+`xml:"{{.XMLName.Space}} {{.XMLName.Local}}"`+"`"+`
				{{ end -}}
			}

			err := c.do({{.Method|printf "%q"}}, {{.Address|printf "%q"}}, &input, &output)

			{{ if .OutputFields -}}
			return {{ range .OutputFields }}{{.Type}}(output.{{.Name}}), {{ end }} err
			{{- else if .ReturnType -}}
			var result {{ .ReturnType }}
			{{ range .ReturnFields -}}
			result.{{.Name}} = {{.Type}}(output.{{.InputArg}})
			{{ end -}}
			return result, err
			{{- else -}}
			return err
			{{- end -}}
		`, params).
		Returns(params.output...)
	if decl, err := fn.Decl(); err != nil {
		return err
	} else {
		p.file.Decls = append(p.file.Decls, decl)
	}
	return nil
}

// The xsdgen package generates private types for some builtin
// types. These types should be hidden from the user and converted
// on the fly.
func exposeType(typ string) string {
	switch typ {
	case "xsdDate", "xsdTime", "xsdDateTime", "gDay",
		"gMonth", "gMonthDay", "gYear", "gYearMonth":
		return "time.Time"
	case "hexBinary", "base64Binary":
		return "[]byte"
	case "idrefs", "nmtokens", "notation", "entities":
		return "[]string"
	}
	return typ
}

func (p *printer) opArgs(addr, method string, input, output wsdl.Message) (opArgs, error) {
	var args opArgs
	args.Address = addr
	args.Method = method
	args.InputName = input.Name
	for _, part := range input.Parts {
		typ := p.code.NameOf(part.Type)
		inputType := exposeType(typ)
		vname := gen.Sanitize(part.Name)
		args.input = append(args.input, vname+" "+inputType)
		args.InputFields = append(args.InputFields, field{
			Name:       strings.Title(part.Name),
			Type:       typ,
			PublicType: exposeType(typ),
			XMLName:    xml.Name{p.wsdl.TargetNS, part.Name},
			InputArg:   vname,
		})
	}
	if len(args.input) > p.maxArgs {
		args.InputType = strings.Title(args.InputName.Local)
		args.input = []string{"v " + args.InputName.Local}
		for i, v := range input.Parts {
			args.InputFields[i].InputArg = "v." + strings.Title(v.Name)
		}
	}
	args.OutputName = output.Name
	for _, part := range output.Parts {
		typ := p.code.NameOf(part.Type)
		outputType := exposeType(typ)
		args.output = append(args.output, outputType)
		args.OutputFields = append(args.OutputFields, field{
			Name:    strings.Title(part.Name),
			Type:    typ,
			XMLName: xml.Name{p.wsdl.TargetNS, part.Name},
		})
	}
	if len(args.output) > p.maxReturns {
		args.ReturnType = strings.Title(args.OutputName.Local)
		args.ReturnFields = make([]field, len(args.OutputFields))
		for i, v := range args.OutputFields {
			args.ReturnFields[i] = field{
				Name:     v.Name,
				Type:     exposeType(v.Type),
				InputArg: v.Name,
			}
		}
		args.output = []string{args.ReturnType}
	}
	// NOTE(droyo) if we decide to name our return values,
	// we have to change this too.
	args.output = append(args.output, "error")

	return args, nil
}

// To keep our output small (as possible), we only generate type
// declarations for the types that are named in the WSDL definition.
func (cfg *Config) registerXSDTypes(def *wsdl.Definition) {
	xmlns := make(map[string]struct{})
	// Some schema may list messages that are not used by any
	// ports, so we have to be thorough.
	for _, port := range def.Ports {
		for _, op := range port.Operations {
			for _, name := range []xml.Name{op.Input, op.Output} {
				if msg, ok := def.Message[name]; !ok {
					cfg.logf("ERROR: No message def found for %s", name.Local)
				} else {
					for _, part := range msg.Parts {
						xmlns[part.Type.Space] = struct{}{}
						cfg.xsdgen.Option(xsdgen.AllowType(part.Type))
					}
				}
			}
		}
	}
	namespaces := make([]string, 0, len(xmlns))
	for ns := range xmlns {
		namespaces = append(namespaces, ns)
	}
	cfg.xsdgen.Option(xsdgen.Namespaces(namespaces...))
}
<commit_msg>Avoid type name shadowing when arg is named after its type<commit_after>// Package wsdlgen generates Go source code from wsdl documents.
//
// The wsdlgen package generates Go source for calling the various
// methods defined in a WSDL (Web Service Definition Language) document.
// The generated Go source is self-contained, with no dependencies on
// non-standard packages.
//
// Code generation for the wsdlgen package can be configured by using
// the provided Option functions.
package wsdlgen

import (
	"encoding/xml"
	"errors"
	"fmt"
	"go/ast"
	"io/ioutil"
	"strings"

	"aqwari.net/xml/internal/gen"
	"aqwari.net/xml/wsdl"
	"aqwari.net/xml/xsdgen"
)

// Types conforming to the Logger interface can receive information about
// the code generation process.
type Logger interface {
	Printf(format string, v ...interface{})
}

type printer struct {
	*Config
	code *xsdgen.Code
	wsdl *wsdl.Definition
	file *ast.File
}

// Provides aspects about an RPC call to the template for the function
// bodies.
type opArgs struct {
	// formatted with appropriate variable names
	input, output []string

	// URL to send request to
	Address string

	// POST or GET
	Method string

	// Name of the method to call
	MsgName xml.Name

	// if we're returning individual values, these slices
	// are in an order matching the input/output slices.
	InputName, OutputName xml.Name
	InputFields           []field
	OutputFields          []field

	// If not "", inputs come in a wrapper struct
	InputType string

	// If not "", we return values in a wrapper struct
	ReturnType   string
	ReturnFields []field
}

// struct members. Need to export the fields for our template
type field struct {
	Name, Type string
	XMLName    xml.Name

	// If this is a wrapper struct for >InputThreshold arguments,
	// PublicType holds the type that we want to expose to the
	// user. For example, if the web service expects an xsdDate
	// to be sent to it, PublicType will be time.Time and a conversion
	// will take place before sending the request to the server.
	PublicType string

	// This refers to the name of the value to assign to this field
	// in the argument list. Empty for return values.
	InputArg string
}

// GenAST creates a Go source file containing type and method declarations
// that can be used to access the service described in the provided set of wsdl
// files.
func (cfg *Config) GenAST(files ...string) (*ast.File, error) {
	if len(files) == 0 {
		return nil, errors.New("must provide at least one file name")
	}
	if cfg.pkgName == "" {
		cfg.pkgName = "ws"
	}
	if cfg.pkgHeader == "" {
		cfg.pkgHeader = fmt.Sprintf("Package %s", cfg.pkgName)
	}
	docs := make([][]byte, 0, len(files))
	for _, filename := range files {
		if data, err := ioutil.ReadFile(filename); err != nil {
			return nil, err
		} else {
			cfg.debugf("read %s", filename)
			docs = append(docs, data)
		}
	}

	cfg.debugf("parsing WSDL file %s", files[0])
	def, err := wsdl.Parse(docs[0])
	if err != nil {
		return nil, err
	}
	cfg.verbosef("building xsd type whitelist from WSDL")
	cfg.registerXSDTypes(def)

	cfg.verbosef("generating type declarations from xml schema")
	code, err := cfg.xsdgen.GenCode(docs...)
	if err != nil {
		return nil, err
	}

	cfg.verbosef("generating function definitions from WSDL")
	return cfg.genAST(def, code)
}

func (cfg *Config) genAST(def *wsdl.Definition, code *xsdgen.Code) (*ast.File, error) {
	file, err := code.GenAST()
	if err != nil {
		return nil, err
	}
	file.Name = ast.NewIdent(cfg.pkgName)
	file = gen.PackageDoc(file, cfg.pkgHeader, "\n", def.Doc)
	p := &printer{
		Config: cfg,
		wsdl:   def,
		file:   file,
		code:   code,
	}
	return p.genAST()
}

func (p *printer) genAST() (*ast.File, error) {
	p.addHelpers()
	for _, port := range p.wsdl.Ports {
		if err := p.port(port); err != nil {
			return nil, err
		}
	}
	return p.file, nil
}

func (p *printer) port(port wsdl.Port) error {
	for _, operation := range port.Operations {
		if err := p.operation(port, operation); err != nil {
			return err
		}
	}
	return nil
}

func (p *printer) operation(port wsdl.Port, op wsdl.Operation) error {
	input, ok := p.wsdl.Message[op.Input]
	if !ok {
		return fmt.Errorf("unknown input message type %s", op.Input.Local)
	}
	output, ok := p.wsdl.Message[op.Output]
	if !ok {
		return fmt.Errorf("unknown output message type %s", op.Output.Local)
	}
	params, err := p.opArgs(port.Address, port.Method, input, output)
	if err != nil {
		return err
	}

	if params.InputType != "" {
		decls, err := gen.Snippets(params, `
			type {{.InputType}} struct {
			{{ range .InputFields -}}
				{{.Name}} {{.PublicType}}
			{{ end -}}
			}`,
		)
		if err != nil {
			return err
		}
		p.file.Decls = append(p.file.Decls, decls...)
	}
	if params.ReturnType != "" {
		decls, err := gen.Snippets(params, `
			type {{.ReturnType}} struct {
			{{ range .ReturnFields -}}
				{{.Name}} {{.Type}}
			{{ end -}}
			}`,
		)
		if err != nil {
			return err
		}
		p.file.Decls = append(p.file.Decls, decls...)
	}
	fn := gen.Func(p.xsdgen.NameOf(op.Name)).
		Comment(op.Doc).
		Receiver("c *Client").
		Args(params.input...).
		BodyTmpl(`
			var input struct {
				XMLName struct{} `+"`"+`xml:"{{.InputName.Space}} {{.InputName.Local}}"`+"`"+`
				{{ range .InputFields -}}
				{{.Name}} {{.Type}} `+"`"+`xml:"{{.XMLName.Space}} {{.XMLName.Local}}"`+"`"+`
				{{ end -}}
			}

			{{- range .InputFields }}
			input.{{.Name}} = {{.Type}}({{.InputArg}})
			{{ end }}

			var output struct {
				XMLName struct{} `+"`"+`xml:"{{.OutputName.Space}} {{.OutputName.Local}}"`+"`"+`
				{{ range .OutputFields -}}
				{{.Name}} {{.Type}} `+"`"+`xml:"{{.XMLName.Space}} {{.XMLName.Local}}"`+"`"+`
				{{ end -}}
			}

			err := c.do({{.Method|printf "%q"}}, {{.Address|printf "%q"}}, &input, &output)

			{{ if .OutputFields -}}
			return {{ range .OutputFields }}{{.Type}}(output.{{.Name}}), {{ end }} err
			{{- else if .ReturnType -}}
			var result {{ .ReturnType }}
			{{ range .ReturnFields -}}
			result.{{.Name}} = {{.Type}}(output.{{.InputArg}})
			{{ end -}}
			return result, err
			{{- else -}}
			return err
			{{- end -}}
		`, params).
		Returns(params.output...)
	if decl, err := fn.Decl(); err != nil {
		return err
	} else {
		p.file.Decls = append(p.file.Decls, decl)
	}
	return nil
}

// The xsdgen package generates private types for some builtin
// types. These types should be hidden from the user and converted
// on the fly.
func exposeType(typ string) string {
	switch typ {
	case "xsdDate", "xsdTime", "xsdDateTime", "gDay",
		"gMonth", "gMonthDay", "gYear", "gYearMonth":
		return "time.Time"
	case "hexBinary", "base64Binary":
		return "[]byte"
	case "idrefs", "nmtokens", "notation", "entities":
		return "[]string"
	}
	return typ
}

func (p *printer) opArgs(addr, method string, input, output wsdl.Message) (opArgs, error) {
	var args opArgs
	args.Address = addr
	args.Method = method
	args.InputName = input.Name
	for _, part := range input.Parts {
		typ := p.code.NameOf(part.Type)
		inputType := exposeType(typ)
		vname := gen.Sanitize(part.Name)
		if vname == typ {
			vname += "_"
		}
		args.input = append(args.input, vname+" "+inputType)
		args.InputFields = append(args.InputFields, field{
			Name:       strings.Title(part.Name),
			Type:       typ,
			PublicType: exposeType(typ),
			XMLName:    xml.Name{p.wsdl.TargetNS, part.Name},
			InputArg:   vname,
		})
	}
	if len(args.input) > p.maxArgs {
		args.InputType = strings.Title(args.InputName.Local)
		args.input = []string{"v " + args.InputName.Local}
		for i, v := range input.Parts {
			args.InputFields[i].InputArg = "v." + strings.Title(v.Name)
		}
	}
	args.OutputName = output.Name
	for _, part := range output.Parts {
		typ := p.code.NameOf(part.Type)
		outputType := exposeType(typ)
		args.output = append(args.output, outputType)
		args.OutputFields = append(args.OutputFields, field{
			Name:    strings.Title(part.Name),
			Type:    typ,
			XMLName: xml.Name{p.wsdl.TargetNS, part.Name},
		})
	}
	if len(args.output) > p.maxReturns {
		args.ReturnType = strings.Title(args.OutputName.Local)
		args.ReturnFields = make([]field, len(args.OutputFields))
		for i, v := range args.OutputFields {
			args.ReturnFields[i] = field{
				Name:     v.Name,
				Type:     exposeType(v.Type),
				InputArg: v.Name,
			}
		}
		args.output = []string{args.ReturnType}
	}
	// NOTE(droyo) if we decide to name our return values,
	// we have to change this too.
	args.output = append(args.output, "error")

	return args, nil
}

// To keep our output small (as possible), we only generate type
// declarations for the types that are named in the WSDL definition.
func (cfg *Config) registerXSDTypes(def *wsdl.Definition) {
	xmlns := make(map[string]struct{})
	// Some schema may list messages that are not used by any
	// ports, so we have to be thorough.
	for _, port := range def.Ports {
		for _, op := range port.Operations {
			for _, name := range []xml.Name{op.Input, op.Output} {
				if msg, ok := def.Message[name]; !ok {
					cfg.logf("ERROR: No message def found for %s", name.Local)
				} else {
					for _, part := range msg.Parts {
						xmlns[part.Type.Space] = struct{}{}
						cfg.xsdgen.Option(xsdgen.AllowType(part.Type))
					}
				}
			}
		}
	}
	namespaces := make([]string, 0, len(xmlns))
	for ns := range xmlns {
		namespaces = append(namespaces, ns)
	}
	cfg.xsdgen.Option(xsdgen.Namespaces(namespaces...))
}
<|endoftext|>


<commit_before>package middleware

import (
	"github.com/gin-gonic/gin"
	"strings"

	u "github.com/techjanitor/pram-get/utils"
)

// REDIS KEY STRUCTURE
//
// HASH:
// "thread:1:45"
// "post:1:45"
// "tag:1:4"
// "index:1"
// "image:1"
//
// REGULAR:
// "directory:1"
// "tags:1"
// "tagtypes"
// "pram"
// "taginfo:21"

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {
		var result []byte
		var err error

		// Get request path
		path := c.Request.URL.Path

		// bool for analytics middleware
		c.Set("cached", false)

		// Trim leading / from path and split
		params := strings.Split(strings.Trim(path, "/"), "/")

		// Make key from path
		key := redisKey{}
		key.expireKey(params[0])
		key.generateKey(params...)

		// Initialize cache handle
		cache := u.RedisCache

		if key.Hash {
			// Check to see if there is already a key we can serve
			result, err = cache.HGet(key.Key, key.Field)
			if err == u.ErrCacheMiss {

				c.Next()

				// Check if there was an error from the controller
				controllerError, _ := c.Get("controllerError")
				if controllerError != nil {
					c.Abort()
					return
				}

				// Get data from controller
				data := c.MustGet("data").([]byte)

				// Set output to cache
				err = cache.HMSet(key.Key, key.Field, data)
				if err != nil {
					c.Error(err)
					c.Abort()
					return
				}

			} else if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

		}

		if !key.Hash {
			// Check to see if there is already a key we can serve
			result, err = cache.Get(key.Key)
			if err == u.ErrCacheMiss {

				c.Next()

				// Check if there was an error from the controller
				controllerError, _ := c.Get("controllerError")
				if controllerError != nil {
					c.Abort()
					return
				}

				// Get data from controller
				data := c.MustGet("data").([]byte)

				if key.Expire {

					// Set output to cache
					err = cache.SetEx(key.Key, 60, data)
					if err != nil {
						c.Error(err)
						c.Abort()
						return
					}

				} else {

					// Set output to cache
					err = cache.Set(key.Key, data)
					if err != nil {
						c.Error(err)
						c.Abort()
						return
					}

				}

			} else if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

		}

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()

	}

}

type redisKey struct {
	Key    string
	Field  string
	Hash   bool
	Expire bool
}

// Will take the params from the request and turn them into a key
func (r *redisKey) generateKey(params ...string) {

	var keys []string

	switch {
	// keys like pram, directory, and tags
	case len(params) <= 2:
		r.Key = strings.Join(params, ":")
	// index and image
	case len(params) == 3:
		keys = append(keys, params[0], params[1])
		r.Field = params[2]
		r.Hash = true
		r.Key = strings.Join(keys, ":")
	// thread, post, and tag
	case len(params) == 4:
		keys = append(keys, params[0], params[1], params[2])
		r.Field = params[3]
		r.Hash = true
		r.Key = strings.Join(keys, ":")
	}

	return

}

// Check if key should be expired
func (r *redisKey) expireKey(key string) {

	keyList := map[string]bool{
		"pram": true,
	}

	if keyList[strings.ToLower(key)] {
		r.Expire = true
	}

	return

}
<commit_msg>fix cached issue in analytics<commit_after>package middleware

import (
	"github.com/gin-gonic/gin"
	"strings"

	u "github.com/techjanitor/pram-get/utils"
)

// REDIS KEY STRUCTURE
//
// HASH:
// "thread:1:45"
// "post:1:45"
// "tag:1:4"
// "index:1"
// "image:1"
//
// REGULAR:
// "directory:1"
// "tags:1"
// "tagtypes"
// "pram"
// "taginfo:21"

// Cache will check for the key in Redis and serve it. If not found, it will
// take the marshalled JSON from the controller and set it in Redis
func Cache() gin.HandlerFunc {
	return func(c *gin.Context) {
		var result []byte
		var err error

		// Get request path
		path := c.Request.URL.Path

		// bool for analytics middleware
		c.Set("cached", false)

		// Trim leading / from path and split
		params := strings.Split(strings.Trim(path, "/"), "/")

		// Make key from path
		key := redisKey{}
		key.expireKey(params[0])
		key.generateKey(params...)

		// Initialize cache handle
		cache := u.RedisCache

		if key.Hash {
			// Check to see if there is already a key we can serve
			result, err = cache.HGet(key.Key, key.Field)
			if err == u.ErrCacheMiss {

				c.Next()

				// Check if there was an error from the controller
				controllerError, _ := c.Get("controllerError")
				if controllerError != nil {
					c.Abort()
					return
				}

				// Get data from controller
				data := c.MustGet("data").([]byte)

				// Set output to cache
				err = cache.HMSet(key.Key, key.Field, data)
				if err != nil {
					c.Error(err)
					c.Abort()
					return
				}

				return

			} else if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

		}

		if !key.Hash {
			// Check to see if there is already a key we can serve
			result, err = cache.Get(key.Key)
			if err == u.ErrCacheMiss {

				c.Next()

				// Check if there was an error from the controller
				controllerError, _ := c.Get("controllerError")
				if controllerError != nil {
					c.Abort()
					return
				}

				// Get data from controller
				data := c.MustGet("data").([]byte)

				if key.Expire {

					// Set output to cache
					err = cache.SetEx(key.Key, 60, data)
					if err != nil {
						c.Error(err)
						c.Abort()
						return
					}

				} else {

					// Set output to cache
					err = cache.Set(key.Key, data)
					if err != nil {
						c.Error(err)
						c.Abort()
						return
					}

				}

				return

			} else if err != nil {
				c.Error(err)
				c.Abort()
				return
			}

		}

		// if we made it this far then the page was cached
		c.Set("cached", true)

		c.Writer.Header().Set("Content-Type", "application/json")
		c.Writer.Write(result)
		c.Abort()

	}

}

type redisKey struct {
	Key    string
	Field  string
	Hash   bool
	Expire bool
}

// Will take the params from the request and turn them into a key
func (r *redisKey) generateKey(params ...string) {

	var keys []string

	switch {
	// keys like pram, directory, and tags
	case len(params) <= 2:
		r.Key = strings.Join(params, ":")
	// index and image
	case len(params) == 3:
		keys = append(keys, params[0], params[1])
		r.Field = params[2]
		r.Hash = true
		r.Key = strings.Join(keys, ":")
	// thread, post, and tag
	case len(params) == 4:
		keys = append(keys, params[0], params[1], params[2])
		r.Field = params[3]
		r.Hash = true
		r.Key = strings.Join(keys, ":")
	}

	return

}

// Check if key should be expired
func (r *redisKey) expireKey(key string) {

	keyList := map[string]bool{
		"pram": true,
	}

	if keyList[strings.ToLower(key)] {
		r.Expire = true
	}

	return

}
<|endoftext|>