<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigquery

import (
	"fmt"
	"net/http"
	"reflect"
	"testing"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/api/googleapi"
	"google.golang.org/cloud/internal/testutil"
)

func TestIntegration(t *testing.T) {
	if testing.Short() {
		t.Skip("Integration tests skipped in short mode")
	}

	ctx := context.Background()
	ts := testutil.TokenSource(ctx, Scope)
	if ts == nil {
		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
	}

	projID := testutil.ProjID()
	c, err := NewClient(ctx, projID)
	if err != nil {
		t.Fatal(err)
	}
	ds := c.Dataset("bigquery_integration_test")
	if err := ds.Create(ctx); err != nil && !hasStatusCode(err, http.StatusConflict) { // AlreadyExists is 409
		t.Fatal(err)
	}
	schema := Schema([]*FieldSchema{
		{Name: "name", Type: StringFieldType},
		{Name: "num", Type: IntegerFieldType},
	})
	table := c.OpenTable(projID, ds.id, "t1")
	// Delete the table in case it already exists. (Ignore errors.)
	table.Delete(ctx)
	// Create the table.
	table, err = c.CreateTable(ctx, projID, ds.id, "t1", schema,
		TableExpiration(time.Now().Add(5*time.Minute)))
	if err != nil {
		t.Fatal(err)
	}
	// Check table metadata.
	md, err := table.Metadata(ctx)
	if err != nil {
		t.Fatal(err)
	}
	// TODO(jba): check md more thorougly.
	if got, want := md.ID, fmt.Sprintf("%s:%s.%s", projID, ds.id, table.TableID); got != want {
		t.Errorf("metadata.ID: got %q, want %q", got, want)
	}
	if got, want := md.Type, RegularTable; got != want {
		t.Errorf("metadata.Type: got %v, want %v", got, want)
	}

	// List tables in the dataset.
	tables, err := ds.ListTables(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if got, want := len(tables), 1; got != want {
		t.Fatalf("ListTables: got %d, want %d", got, want)
	}
	want := *table
	if got := tables[0]; !reflect.DeepEqual(got, &want) {
		t.Errorf("ListTables: got %v, want %v", got, &want)
	}

	// Populate the table.
	upl := table.NewUploader()
	var rows []*ValuesSaver
	for i, name := range []string{"a", "b", "c"} {
		rows = append(rows, &ValuesSaver{
			Schema:   schema,
			InsertID: name,
			Row:      []Value{name, i},
		})
	}
	if err := upl.Put(ctx, rows); err != nil {
		t.Fatal(err)
	}

	checkRead := func(src ReadSource) {
		it, err := c.Read(ctx, src)
		if err != nil {
			t.Fatal(err)
		}
		for i := 0; it.Next(ctx); i++ {
			var vals ValueList
			if err := it.Get(&vals); err != nil {
				t.Fatal(err)
			}
			if got, want := vals, rows[i].Row; !reflect.DeepEqual([]Value(got), want) {
				t.Errorf("got %v, want %v", got, want)
			}
		}
	}
	// Read the table.
	checkRead(table)

	// Query the table.
	q := &Query{
		Q:                "select name, num from t1",
		DefaultProjectID: projID,
		DefaultDatasetID: ds.id,
	}
	checkRead(q)

	// Query the long way.
	dest := &Table{}
	job1, err := c.Copy(ctx, dest, q, WriteTruncate)
	if err != nil {
		t.Fatal(err)
	}
	job2, err := c.JobFromID(ctx, job1.ID())
	if err != nil {
		t.Fatal(err)
	}
	// TODO(jba): poll status until job is done
	_, err = job2.Status(ctx)
	if err != nil {
		t.Fatal(err)
	}

	checkRead(job2)

	// TODO(jba): patch the table
}

func hasStatusCode(err error, code int) bool {
	if e, ok := err.(*googleapi.Error); ok && e.Code == code {
		return true
	}
	return false
}
<commit_msg>bigquery: use the token source in integration test<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigquery

import (
	"fmt"
	"net/http"
	"reflect"
	"testing"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/api/googleapi"
	"google.golang.org/cloud"
	"google.golang.org/cloud/internal/testutil"
)

func TestIntegration(t *testing.T) {
	if testing.Short() {
		t.Skip("Integration tests skipped in short mode")
	}

	ctx := context.Background()
	ts := testutil.TokenSource(ctx, Scope)
	if ts == nil {
		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
	}

	projID := testutil.ProjID()
	c, err := NewClient(ctx, projID, cloud.WithTokenSource(ts))
	if err != nil {
		t.Fatal(err)
	}
	ds := c.Dataset("bigquery_integration_test")
	if err := ds.Create(ctx); err != nil && !hasStatusCode(err, http.StatusConflict) { // AlreadyExists is 409
		t.Fatal(err)
	}
	schema := Schema([]*FieldSchema{
		{Name: "name", Type: StringFieldType},
		{Name: "num", Type: IntegerFieldType},
	})
	table := c.OpenTable(projID, ds.id, "t1")
	// Delete the table in case it already exists. (Ignore errors.)
	table.Delete(ctx)
	// Create the table.
	table, err = c.CreateTable(ctx, projID, ds.id, "t1", schema,
		TableExpiration(time.Now().Add(5*time.Minute)))
	if err != nil {
		t.Fatal(err)
	}
	// Check table metadata.
	md, err := table.Metadata(ctx)
	if err != nil {
		t.Fatal(err)
	}
	// TODO(jba): check md more thorougly.
	if got, want := md.ID, fmt.Sprintf("%s:%s.%s", projID, ds.id, table.TableID); got != want {
		t.Errorf("metadata.ID: got %q, want %q", got, want)
	}
	if got, want := md.Type, RegularTable; got != want {
		t.Errorf("metadata.Type: got %v, want %v", got, want)
	}

	// List tables in the dataset.
	tables, err := ds.ListTables(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if got, want := len(tables), 1; got != want {
		t.Fatalf("ListTables: got %d, want %d", got, want)
	}
	want := *table
	if got := tables[0]; !reflect.DeepEqual(got, &want) {
		t.Errorf("ListTables: got %v, want %v", got, &want)
	}

	// Populate the table.
	upl := table.NewUploader()
	var rows []*ValuesSaver
	for i, name := range []string{"a", "b", "c"} {
		rows = append(rows, &ValuesSaver{
			Schema:   schema,
			InsertID: name,
			Row:      []Value{name, i},
		})
	}
	if err := upl.Put(ctx, rows); err != nil {
		t.Fatal(err)
	}

	checkRead := func(src ReadSource) {
		it, err := c.Read(ctx, src)
		if err != nil {
			t.Fatal(err)
		}
		for i := 0; it.Next(ctx); i++ {
			var vals ValueList
			if err := it.Get(&vals); err != nil {
				t.Fatal(err)
			}
			if got, want := vals, rows[i].Row; !reflect.DeepEqual([]Value(got), want) {
				t.Errorf("got %v, want %v", got, want)
			}
		}
	}
	// Read the table.
	checkRead(table)

	// Query the table.
	q := &Query{
		Q:                "select name, num from t1",
		DefaultProjectID: projID,
		DefaultDatasetID: ds.id,
	}
	checkRead(q)

	// Query the long way.
	dest := &Table{}
	job1, err := c.Copy(ctx, dest, q, WriteTruncate)
	if err != nil {
		t.Fatal(err)
	}
	job2, err := c.JobFromID(ctx, job1.ID())
	if err != nil {
		t.Fatal(err)
	}
	// TODO(jba): poll status until job is done
	_, err = job2.Status(ctx)
	if err != nil {
		t.Fatal(err)
	}

	checkRead(job2)

	// TODO(jba): patch the table
}

func hasStatusCode(err error, code int) bool {
	if e, ok := err.(*googleapi.Error); ok && e.Code == code {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>/* Copyright 2015 LinkedIn Corp. Licensed under the Apache License, Version
 * 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */

package main

import (
	"bytes"
	"encoding/binary"
	"errors"
	"github.com/Shopify/sarama"
	log "github.com/cihub/seelog"
	"sync"
	"time"
)

type KafkaClient struct {
	app                *ApplicationContext
	cluster            string
	client             sarama.Client
	masterConsumer     sarama.Consumer
	partitionConsumers []sarama.PartitionConsumer
	requestChannel     chan *BrokerTopicRequest
	messageChannel     chan *sarama.ConsumerMessage
	errorChannel       chan *sarama.ConsumerError
	wgFanIn            sync.WaitGroup
	wgProcessor        sync.WaitGroup
	topicMap           map[string]int
	topicMapLock       sync.RWMutex
	brokerOffsetTicker *time.Ticker
}

type BrokerTopicRequest struct {
	Result chan int
	Topic  string
}

func NewKafkaClient(app *ApplicationContext, cluster string) (*KafkaClient, error) {
	// Set up sarama client
	clientConfig := sarama.NewConfig()
	clientConfig.ClientID = app.Config.General.ClientID
	sclient, err := sarama.NewClient(app.Config.Kafka[cluster].Brokers, clientConfig)
	if err != nil {
		return nil, err
	}

	// Create sarama master consumer
	master, err := sarama.NewConsumerFromClient(sclient)
	if err != nil {
		sclient.Close()
		return nil, err
	}

	client := &KafkaClient{
		app:            app,
		cluster:        cluster,
		client:         sclient,
		masterConsumer: master,
		requestChannel: make(chan *BrokerTopicRequest),
		messageChannel: make(chan *sarama.ConsumerMessage),
		errorChannel:   make(chan *sarama.ConsumerError),
		wgFanIn:        sync.WaitGroup{},
		wgProcessor:    sync.WaitGroup{},
		topicMap:       make(map[string]int),
		topicMapLock:   sync.RWMutex{},
	}

	// Start the main processor goroutines for __consumer_offset messages
	client.wgProcessor.Add(2)
	go func() {
		defer client.wgProcessor.Done()
		for msg := range client.messageChannel {
			go client.processConsumerOffsetsMessage(msg)
		}
	}()
	go func() {
		defer client.wgProcessor.Done()
		for err := range client.errorChannel {
			log.Errorf("Consume error on %s:%v: %v", err.Topic, err.Partition, err.Err)
		}
	}()

	// Start goroutine to handle topic metadata requests. Do this first because the getOffsets call needs this working
	client.RefreshTopicMap()
	go func() {
		for r := range client.requestChannel {
			client.getPartitionCount(r)
		}
	}()

	// Now get the first set of offsets and start a goroutine to continually check them
	client.getOffsets()
	client.brokerOffsetTicker = time.NewTicker(time.Duration(client.app.Config.Tickers.BrokerOffsets) * time.Second)
	go func() {
		for _ = range client.brokerOffsetTicker.C {
			client.getOffsets()
		}
	}()

	// Get a partition count for the consumption topic
	partitions, err := client.client.Partitions(client.app.Config.Kafka[client.cluster].OffsetsTopic)
	if err != nil {
		return nil, err
	}

	// Start consumers for each partition with fan in
	client.partitionConsumers = make([]sarama.PartitionConsumer, len(partitions))
	log.Infof("Starting consumers for %v partitions of %s in cluster %s", len(partitions), client.app.Config.Kafka[client.cluster].OffsetsTopic, client.cluster)
	for i, partition := range partitions {
		pconsumer, err := client.masterConsumer.ConsumePartition(client.app.Config.Kafka[client.cluster].OffsetsTopic, partition, sarama.OffsetNewest)
		if err != nil {
			return nil, err
		}
		client.partitionConsumers[i] = pconsumer
		client.wgFanIn.Add(2)
		go func() {
			defer client.wgFanIn.Done()
			for msg := range pconsumer.Messages() {
				client.messageChannel <- msg
			}
		}()
		go func() {
			defer client.wgFanIn.Done()
			for err := range pconsumer.Errors() {
				client.errorChannel <- err
			}
		}()
	}

	return client, nil
}

func (client *KafkaClient) Stop() {
	// We don't really need to do a safe stop, because we're not maintaining offsets. But we'll do it anyways
	for _, pconsumer := range client.partitionConsumers {
		pconsumer.AsyncClose()
	}

	// Wait for the Messages and Errors channel to be fully drained.
	client.wgFanIn.Wait()
	close(client.errorChannel)
	close(client.messageChannel)
	client.wgProcessor.Wait()

	// Stop the offset checker and the topic metdata refresh and request channel
	client.brokerOffsetTicker.Stop()
	close(client.requestChannel)
}

// Send the offset on the specified channel, but wait no more than maxTime seconds to do so
func timeoutSendOffset(offsetChannel chan *PartitionOffset, offset *PartitionOffset, maxTime int) {
	timeout := time.After(time.Duration(maxTime) * time.Second)
	select {
	case offsetChannel <- offset:
	case <-timeout:
	}
}

// This function performs massively parallel OffsetRequests, which is better than Sarama's internal implementation,
// which does one at a time. Several orders of magnitude faster.
func (client *KafkaClient) getOffsets() error {
	// Start with refreshing the topic list
	client.RefreshTopicMap()

	requests := make(map[int32]*sarama.OffsetRequest)
	brokers := make(map[int32]*sarama.Broker)

	client.topicMapLock.RLock()

	// Generate an OffsetRequest for each topic:partition and bucket it to the leader broker
	for topic, partitions := range client.topicMap {
		for i := 0; i < partitions; i++ {
			broker, err := client.client.Leader(topic, int32(i))
			if err != nil {
				client.topicMapLock.RUnlock()
				return err
			}
			if _, ok := requests[broker.ID()]; !ok {
				requests[broker.ID()] = &sarama.OffsetRequest{}
			}
			brokers[broker.ID()] = broker
			requests[broker.ID()].AddBlock(topic, int32(i), sarama.OffsetNewest, 1)
		}
	}

	// Send out the OffsetRequest to each broker for all the partitions it is leader for
	// The results go to the offset storage module
	var wg sync.WaitGroup

	getBrokerOffsets := func(brokerID int32, request *sarama.OffsetRequest) {
		defer wg.Done()
		response, err := brokers[brokerID].GetAvailableOffsets(request)
		if err != nil {
			log.Errorf("Cannot fetch offsets from broker %v: %v", brokerID, err)
			return
		}
		ts := time.Now().Unix() * 1000
		for topic, partitions := range response.Blocks {
			for partition, offsetResponse := range partitions {
				if offsetResponse.Err != sarama.ErrNoError {
					log.Warnf("Error in OffsetResponse for %s:%v from broker %v: %s", topic, partition, brokerID, offsetResponse.Err.Error())
					continue
				}
				offset := &PartitionOffset{
					Cluster:             client.cluster,
					Topic:               topic,
					Partition:           partition,
					Offset:              offsetResponse.Offsets[0],
					Timestamp:           ts,
					TopicPartitionCount: client.topicMap[topic],
				}
				timeoutSendOffset(client.app.Storage.offsetChannel, offset, 1)
			}
		}
	}

	for brokerID, request := range requests {
		wg.Add(1)
		go getBrokerOffsets(brokerID, request)
	}

	wg.Wait()
	client.topicMapLock.RUnlock()
	return nil
}

func (client *KafkaClient) RefreshTopicMap() {
	client.topicMapLock.Lock()
	topics, _ := client.client.Topics()
	for _, topic := range topics {
		partitions, _ := client.client.Partitions(topic)
		client.topicMap[topic] = len(partitions)
	}
	client.topicMapLock.Unlock()
}

func (client *KafkaClient) getPartitionCount(r *BrokerTopicRequest) {
	client.topicMapLock.RLock()
	if partitions, ok := client.topicMap[r.Topic]; ok {
		r.Result <- partitions
	} else {
		r.Result <- -1
	}
	client.topicMapLock.RUnlock()
}

func readString(buf *bytes.Buffer) (string, error) {
	var strlen uint16
	err := binary.Read(buf, binary.BigEndian, &strlen)
	if err != nil {
		return "", err
	}
	strbytes := make([]byte, strlen)
	n, err := buf.Read(strbytes)
	if (err != nil) || (n != int(strlen)) {
		return "", errors.New("string underflow")
	}
	return string(strbytes), nil
}

func (client *KafkaClient) processConsumerOffsetsMessage(msg *sarama.ConsumerMessage) {
	var keyver, valver uint16
	var partition uint32
	var offset, timestamp uint64

	buf := bytes.NewBuffer(msg.Key)
	err := binary.Read(buf, binary.BigEndian, &keyver)
	if (err != nil) || ((keyver != 0) && (keyver != 1)) {
		log.Warnf("Failed to decode %s:%v offset %v: keyver", msg.Topic, msg.Partition, msg.Offset)
		return
	}
	group, err := readString(buf)
	if err != nil {
		log.Warnf("Failed to decode %s:%v offset %v: group", msg.Topic, msg.Partition, msg.Offset)
		return
	}
	topic, err := readString(buf)
	if err != nil {
		log.Warnf("Failed to decode %s:%v offset %v: topic", msg.Topic, msg.Partition, msg.Offset)
		return
	}
	err = binary.Read(buf, binary.BigEndian, &partition)
	if err != nil {
		log.Warnf("Failed to decode %s:%v offset %v: partition", msg.Topic, msg.Partition, msg.Offset)
		return
	}

	buf = bytes.NewBuffer(msg.Value)
	err = binary.Read(buf, binary.BigEndian, &valver)
	if (err != nil) || ((valver != 0) && (valver != 1)) {
		log.Warnf("Failed to decode %s:%v offset %v: valver", msg.Topic, msg.Partition, msg.Offset)
		return
	}
	err = binary.Read(buf, binary.BigEndian, &offset)
	if err != nil {
		log.Warnf("Failed to decode %s:%v offset %v: offset", msg.Topic, msg.Partition, msg.Offset)
		return
	}
	_, err = readString(buf)
	if err != nil {
		log.Warnf("Failed to decode %s:%v offset %v: metadata", msg.Topic, msg.Partition, msg.Offset)
		return
	}
	err = binary.Read(buf, binary.BigEndian, &timestamp)
	if err != nil {
		log.Warnf("Failed to decode %s:%v offset %v: timestamp", msg.Topic, msg.Partition, msg.Offset)
		return
	}

	// fmt.Printf("[%s,%s,%v]::OffsetAndMetadata[%v,%s,%v]\n", group, topic, partition, offset, metadata, timestamp)
	partitionOffset := &PartitionOffset{
		Cluster:   client.cluster,
		Topic:     topic,
		Partition: int32(partition),
		Group:     group,
		Timestamp: int64(timestamp),
		Offset:    int64(offset),
	}
	timeoutSendOffset(client.app.Storage.offsetChannel, partitionOffset, 1)
	return
}
<commit_msg>Ensure blacklisted topic not in client topicMap when refresh kafka topic map<commit_after>/* Copyright 2015 LinkedIn Corp. Licensed under the Apache License, Version
 * 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */

package main

import (
	"bytes"
	"encoding/binary"
	"errors"
	"github.com/Shopify/sarama"
	log "github.com/cihub/seelog"
	"sync"
	"time"
)

type KafkaClient struct {
	app                *ApplicationContext
	cluster            string
	client             sarama.Client
	masterConsumer     sarama.Consumer
	partitionConsumers []sarama.PartitionConsumer
	requestChannel     chan *BrokerTopicRequest
	messageChannel     chan *sarama.ConsumerMessage
	errorChannel       chan *sarama.ConsumerError
	wgFanIn            sync.WaitGroup
	wgProcessor        sync.WaitGroup
	topicMap           map[string]int
	topicMapLock       sync.RWMutex
	brokerOffsetTicker *time.Ticker
}

type BrokerTopicRequest struct {
	Result chan int
	Topic  string
}

func NewKafkaClient(app *ApplicationContext, cluster string) (*KafkaClient, error) {
	// Set up sarama client
	clientConfig := sarama.NewConfig()
	clientConfig.ClientID = app.Config.General.ClientID
	sclient, err := sarama.NewClient(app.Config.Kafka[cluster].Brokers, clientConfig)
	if err != nil {
		return nil, err
	}

	// Create sarama master consumer
	master, err := sarama.NewConsumerFromClient(sclient)
	if err != nil {
		sclient.Close()
		return nil, err
	}

	client := &KafkaClient{
		app:            app,
		cluster:        cluster,
		client:         sclient,
		masterConsumer: master,
		requestChannel: make(chan *BrokerTopicRequest),
		messageChannel: make(chan *sarama.ConsumerMessage),
		errorChannel:   make(chan *sarama.ConsumerError),
		wgFanIn:        sync.WaitGroup{},
		wgProcessor:    sync.WaitGroup{},
		topicMap:       make(map[string]int),
		topicMapLock:   sync.RWMutex{},
	}

	// Start the main processor goroutines for __consumer_offset messages
	client.wgProcessor.Add(2)
	go func() {
		defer client.wgProcessor.Done()
		for msg := range client.messageChannel {
			go client.processConsumerOffsetsMessage(msg)
		}
	}()
	go func() {
		defer client.wgProcessor.Done()
		for err := range client.errorChannel {
			log.Errorf("Consume error on %s:%v: %v", err.Topic, err.Partition, err.Err)
		}
	}()

	// Start goroutine to handle topic metadata requests. Do this first because the getOffsets call needs this working
	client.RefreshTopicMap()
	go func() {
		for r := range client.requestChannel {
			client.getPartitionCount(r)
		}
	}()

	// Now get the first set of offsets and start a goroutine to continually check them
	client.getOffsets()
	client.brokerOffsetTicker = time.NewTicker(time.Duration(client.app.Config.Tickers.BrokerOffsets) * time.Second)
	go func() {
		for _ = range client.brokerOffsetTicker.C {
			client.getOffsets()
		}
	}()

	// Get a partition count for the consumption topic
	partitions, err := client.client.Partitions(client.app.Config.Kafka[client.cluster].OffsetsTopic)
	if err != nil {
		return nil, err
	}

	// Start consumers for each partition with fan in
	client.partitionConsumers = make([]sarama.PartitionConsumer, len(partitions))
	log.Infof("Starting consumers for %v partitions of %s in cluster %s", len(partitions), client.app.Config.Kafka[client.cluster].OffsetsTopic, client.cluster)
	for i, partition := range partitions {
		pconsumer, err := client.masterConsumer.ConsumePartition(client.app.Config.Kafka[client.cluster].OffsetsTopic, partition, sarama.OffsetNewest)
		if err != nil {
			return nil, err
		}
		client.partitionConsumers[i] = pconsumer
		client.wgFanIn.Add(2)
		go func() {
			defer client.wgFanIn.Done()
			for msg := range pconsumer.Messages() {
				client.messageChannel <- msg
			}
		}()
		go func() {
			defer client.wgFanIn.Done()
			for err := range pconsumer.Errors() {
				client.errorChannel <- err
			}
		}()
	}

	return client, nil
}

func (client *KafkaClient) Stop() {
	// We don't really need to do a safe stop, because we're not maintaining offsets. But we'll do it anyways
	for _, pconsumer := range client.partitionConsumers {
		pconsumer.AsyncClose()
	}

	// Wait for the Messages and Errors channel to be fully drained.
	client.wgFanIn.Wait()
	close(client.errorChannel)
	close(client.messageChannel)
	client.wgProcessor.Wait()

	// Stop the offset checker and the topic metdata refresh and request channel
	client.brokerOffsetTicker.Stop()
	close(client.requestChannel)
}

// Send the offset on the specified channel, but wait no more than maxTime seconds to do so
func timeoutSendOffset(offsetChannel chan *PartitionOffset, offset *PartitionOffset, maxTime int) {
	timeout := time.After(time.Duration(maxTime) * time.Second)
	select {
	case offsetChannel <- offset:
	case <-timeout:
	}
}

// This function performs massively parallel OffsetRequests, which is better than Sarama's internal implementation,
// which does one at a time. Several orders of magnitude faster.
func (client *KafkaClient) getOffsets() error {
	// Start with refreshing the topic list
	client.RefreshTopicMap()

	requests := make(map[int32]*sarama.OffsetRequest)
	brokers := make(map[int32]*sarama.Broker)

	client.topicMapLock.RLock()

	// Generate an OffsetRequest for each topic:partition and bucket it to the leader broker
	for topic, partitions := range client.topicMap {
		for i := 0; i < partitions; i++ {
			broker, err := client.client.Leader(topic, int32(i))
			if err != nil {
				client.topicMapLock.RUnlock()
				return err
			}
			if _, ok := requests[broker.ID()]; !ok {
				requests[broker.ID()] = &sarama.OffsetRequest{}
			}
			brokers[broker.ID()] = broker
			requests[broker.ID()].AddBlock(topic, int32(i), sarama.OffsetNewest, 1)
		}
	}

	// Send out the OffsetRequest to each broker for all the partitions it is leader for
	// The results go to the offset storage module
	var wg sync.WaitGroup

	getBrokerOffsets := func(brokerID int32, request *sarama.OffsetRequest) {
		defer wg.Done()
		response, err := brokers[brokerID].GetAvailableOffsets(request)
		if err != nil {
			log.Errorf("Cannot fetch offsets from broker %v: %v", brokerID, err)
			return
		}
		ts := time.Now().Unix() * 1000
		for topic, partitions := range response.Blocks {
			for partition, offsetResponse := range partitions {
				if offsetResponse.Err != sarama.ErrNoError {
					log.Warnf("Error in OffsetResponse for %s:%v from broker %v: %s", topic, partition, brokerID, offsetResponse.Err.Error())
					continue
				}
				offset := &PartitionOffset{
					Cluster:             client.cluster,
					Topic:               topic,
					Partition:           partition,
					Offset:              offsetResponse.Offsets[0],
					Timestamp:           ts,
					TopicPartitionCount: client.topicMap[topic],
				}
				timeoutSendOffset(client.app.Storage.offsetChannel, offset, 1)
			}
		}
	}

	for brokerID, request := range requests {
		wg.Add(1)
		go getBrokerOffsets(brokerID, request)
	}

	wg.Wait()
	client.topicMapLock.RUnlock()
	return nil
}

func (client *KafkaClient) RefreshTopicMap() {
	client.topicMapLock.Lock()
	topics, _ := client.client.Topics()
	for _, topic := range topics {
		if (client.app.Storage.topicBlacklist != nil) && client.app.Storage.topicBlacklist.MatchString(topic) {
			continue
		}
		partitions, _ := client.client.Partitions(topic)
		client.topicMap[topic] = len(partitions)
	}
	client.topicMapLock.Unlock()
}

func (client *KafkaClient) getPartitionCount(r *BrokerTopicRequest) {
	client.topicMapLock.RLock()
	if partitions, ok := client.topicMap[r.Topic]; ok {
		r.Result <- partitions
	} else {
		r.Result <- -1
	}
	client.topicMapLock.RUnlock()
}

func readString(buf *bytes.Buffer) (string, error) {
	var strlen uint16
	err := binary.Read(buf, binary.BigEndian, &strlen)
	if err != nil {
		return "", err
	}
	strbytes := make([]byte, strlen)
	n, err := buf.Read(strbytes)
	if (err != nil) || (n != int(strlen)) {
		return "", errors.New("string underflow")
	}
	return string(strbytes), nil
}

func (client *KafkaClient) processConsumerOffsetsMessage(msg *sarama.ConsumerMessage) {
	var keyver, valver uint16
	var partition uint32
	var offset, timestamp uint64

	buf := bytes.NewBuffer(msg.Key)
	err := binary.Read(buf, binary.BigEndian, &keyver)
	if (err != nil) || ((keyver != 0) && (keyver != 1)) {
		log.Warnf("Failed to decode %s:%v offset %v: keyver", msg.Topic, msg.Partition, msg.Offset)
		return
	}
	group, err := readString(buf)
	if err != nil {
		log.Warnf("Failed to decode %s:%v offset %v: group", msg.Topic, msg.Partition, msg.Offset)
		return
	}
	topic, err := readString(buf)
	if err != nil {
		log.Warnf("Failed to decode %s:%v offset %v: topic", msg.Topic, msg.Partition, msg.Offset)
		return
	}
	err = binary.Read(buf, binary.BigEndian, &partition)
	if err != nil {
		log.Warnf("Failed to decode %s:%v offset %v: partition", msg.Topic, msg.Partition, msg.Offset)
		return
	}

	buf = bytes.NewBuffer(msg.Value)
	err = binary.Read(buf, binary.BigEndian, &valver)
	if (err != nil) || ((valver != 0) && (valver != 1)) {
		log.Warnf("Failed to decode %s:%v offset %v: valver", msg.Topic, msg.Partition, msg.Offset)
		return
	}
	err = binary.Read(buf, binary.BigEndian, &offset)
	if err != nil {
		log.Warnf("Failed to decode %s:%v offset %v: offset", msg.Topic, msg.Partition, msg.Offset)
		return
	}
	_, err = readString(buf)
	if err != nil {
		log.Warnf("Failed to decode %s:%v offset %v: metadata", msg.Topic, msg.Partition, msg.Offset)
		return
	}
	err = binary.Read(buf, binary.BigEndian, &timestamp)
	if err != nil {
		log.Warnf("Failed to decode %s:%v offset %v: timestamp", msg.Topic, msg.Partition, msg.Offset)
		return
	}

	// fmt.Printf("[%s,%s,%v]::OffsetAndMetadata[%v,%s,%v]\n", group, topic, partition, offset, metadata, timestamp)
	partitionOffset := &PartitionOffset{
		Cluster:   client.cluster,
		Topic:     topic,
		Partition: int32(partition),
		Group:     group,
		Timestamp: int64(timestamp),
		Offset:    int64(offset),
	}
	timeoutSendOffset(client.app.Storage.offsetChannel, partitionOffset, 1)
	return
}
<|endoftext|>


<commit_before>// Copyright © 2014-2015 Daniele Tricoli <eriol@mornie.org>.
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package db // import "eriol.xyz/perpetua/db"

import (
	"database/sql"
	"fmt"
	"log"
	"os"
	"strings"

	_ "github.com/mattn/go-sqlite3"
)

type Store struct {
	db *sql.DB
}

// Open SQLite3 database specified in database path.
func (s *Store) Open(database string) (err error) {
	db, err := sql.Open("sqlite3", database)
	if err != nil {
		return err
	}
	s.db = db

	if _, err := os.Stat(database); err != nil {
		if os.IsNotExist(err) {
			s.createDatabase()
		}
	}

	return nil
}

func (s *Store) Close() {
	s.db.Close()
}

func (s *Store) createDatabase() {

	// quotes_acl table connects a quote with a specific channel so
	// that the quote will be quoted only in that specific channel.
	// Quotes not present in this table are public (quoted in every
	// channel where the bot joined)
	tables := `
	CREATE TABLE people (
	    id INTEGER NOT NULL PRIMARY KEY autoincrement,
	    name TEXT COLLATE NOCASE
	);

	CREATE TABLE quotes (
	    id INTEGER NOT NULL PRIMARY KEY autoincrement,
	    person_id INTEGER NOT NULL,
	    quote TEXT COLLATE NOCASE,
	    FOREIGN KEY(person_id) REFERENCES people(id)
	);

	CREATE TABLE alias (
	    person_id INTEGER NOT NULL,
	    alias TEXT COLLATE NOCASE,
	    FOREIGN KEY(person_id) REFERENCES people(id)
    );

	CREATE TABLE channels (
	    id INTEGER NOT NULL PRIMARY KEY autoincrement,
	    name TEXT COLLATE NOCASE
	);

	CREATE TABLE quotes_acl (
	    quote_id INTEGER NOT NULL,
	    channel_id INTEGER NOT NULL,
	    FOREIGN KEY(quote_id) REFERENCES quotes(id),
	    FOREIGN KEY(channel_id) REFERENCES channels(id)
	);`

	_, err := s.db.Exec(tables)

	if err != nil {
		log.Fatal(err)
	}
}

func (s *Store) getPerson(name string) (id int) {
	query_people := `SELECT id FROM people WHERE name = ?`
	query_alias := `SELECT person_id FROM alias WHERE alias = ?`

	s.db.QueryRow(query_people, name).Scan(&id)
	if id == 0 {
		s.db.QueryRow(query_alias, name).Scan(&id)
	}

	return
}

func (s *Store) getChannel(name string) (id int) {
	query := `SELECT id FROM channels WHERE name = ?`

	s.db.QueryRow(query, name).Scan(&id)

	return
}

func (s *Store) GetQuote(person, channel string) (quote string) {
	query := `SELECT quote FROM quotes WHERE person_id = ?
		AND id NOT IN (
			SELECT quote_id FROM quotes_acl
			EXCEPT
			SELECT quote_id FROM quotes_acl WHERE channel_id = ?)
		ORDER BY RANDOM() LIMIT 1;`

	s.db.QueryRow(
		query,
		s.getPerson(person),
		s.getChannel(channel)).Scan(&quote)

	return
}

func (s *Store) GetQuoteAbout(person, argument, channel string) (quote string) {
	// A double quote can't be present in argument because of the
	// regex used but removing anyway
	argument = strings.Replace(argument, "\"", "", -1)

	query := "SELECT quote FROM quotes WHERE person_id = ? " +
		"AND quote LIKE \"%%%s%%\" " +
		"AND id NOT IN ( " +
		"SELECT quote_id FROM quotes_acl " +
		"EXCEPT " +
		"SELECT quote_id FROM quotes_acl WHERE channel_id = ?) " +
		"ORDER BY RANDOM() LIMIT 1;"
	query = fmt.Sprintf(query, argument)

	s.db.QueryRow(
		query,
		s.getPerson(person),
		s.getChannel(channel)).Scan(&quote)

	return
}
<commit_msg>Add docstring and return<commit_after>// Copyright © 2014-2015 Daniele Tricoli <eriol@mornie.org>.
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package db // import "eriol.xyz/perpetua/db"

import (
	"database/sql"
	"fmt"
	"log"
	"os"
	"strings"

	_ "github.com/mattn/go-sqlite3"
)

type Store struct {
	db *sql.DB
}

// Open SQLite3 database specified in database path.
func (s *Store) Open(database string) (err error) {
	db, err := sql.Open("sqlite3", database)
	if err != nil {
		return err
	}
	s.db = db

	if _, err := os.Stat(database); err != nil {
		if os.IsNotExist(err) {
			s.createDatabase()
		}
	}

	return nil
}

// Close SQLite3 database used by Store.
func (s *Store) Close() error {
	return s.db.Close()
}

func (s *Store) createDatabase() {

	// quotes_acl table connects a quote with a specific channel so
	// that the quote will be quoted only in that specific channel.
	// Quotes not present in this table are public (quoted in every
	// channel where the bot joined)
	tables := `
	CREATE TABLE people (
	    id INTEGER NOT NULL PRIMARY KEY autoincrement,
	    name TEXT COLLATE NOCASE
	);

	CREATE TABLE quotes (
	    id INTEGER NOT NULL PRIMARY KEY autoincrement,
	    person_id INTEGER NOT NULL,
	    quote TEXT COLLATE NOCASE,
	    FOREIGN KEY(person_id) REFERENCES people(id)
	);

	CREATE TABLE alias (
	    person_id INTEGER NOT NULL,
	    alias TEXT COLLATE NOCASE,
	    FOREIGN KEY(person_id) REFERENCES people(id)
    );

	CREATE TABLE channels (
	    id INTEGER NOT NULL PRIMARY KEY autoincrement,
	    name TEXT COLLATE NOCASE
	);

	CREATE TABLE quotes_acl (
	    quote_id INTEGER NOT NULL,
	    channel_id INTEGER NOT NULL,
	    FOREIGN KEY(quote_id) REFERENCES quotes(id),
	    FOREIGN KEY(channel_id) REFERENCES channels(id)
	);`

	_, err := s.db.Exec(tables)

	if err != nil {
		log.Fatal(err)
	}
}

func (s *Store) getPerson(name string) (id int) {
	query_people := `SELECT id FROM people WHERE name = ?`
	query_alias := `SELECT person_id FROM alias WHERE alias = ?`

	s.db.QueryRow(query_people, name).Scan(&id)
	if id == 0 {
		s.db.QueryRow(query_alias, name).Scan(&id)
	}

	return
}

func (s *Store) getChannel(name string) (id int) {
	query := `SELECT id FROM channels WHERE name = ?`

	s.db.QueryRow(query, name).Scan(&id)

	return
}

func (s *Store) GetQuote(person, channel string) (quote string) {
	query := `SELECT quote FROM quotes WHERE person_id = ?
		AND id NOT IN (
			SELECT quote_id FROM quotes_acl
			EXCEPT
			SELECT quote_id FROM quotes_acl WHERE channel_id = ?)
		ORDER BY RANDOM() LIMIT 1;`

	s.db.QueryRow(
		query,
		s.getPerson(person),
		s.getChannel(channel)).Scan(&quote)

	return
}

func (s *Store) GetQuoteAbout(person, argument, channel string) (quote string) {
	// A double quote can't be present in argument because of the
	// regex used but removing anyway
	argument = strings.Replace(argument, "\"", "", -1)

	query := "SELECT quote FROM quotes WHERE person_id = ? " +
		"AND quote LIKE \"%%%s%%\" " +
		"AND id NOT IN ( " +
		"SELECT quote_id FROM quotes_acl " +
		"EXCEPT " +
		"SELECT quote_id FROM quotes_acl WHERE channel_id = ?) " +
		"ORDER BY RANDOM() LIMIT 1;"
	query = fmt.Sprintf(query, argument)

	s.db.QueryRow(
		query,
		s.getPerson(person),
		s.getChannel(channel)).Scan(&quote)

	return
}
<|endoftext|>


<commit_before>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package analysis

import (
	"fmt"
	"reflect"

	"github.com/go-openapi/spec"
)

// Mixin modifies the primary swagger spec by adding the paths and
// definitions from the mixin specs. Top level parameters and
// responses from the mixins are also carried over. Operation id
// collisions are avoided by appending "Mixin<N>" but only if
// needed.
//
// The following parts of primary are subject to merge, filling empty details
//   - Info
//   - BasePath
//   - Host
//   - ExternalDocs
//
// Consider calling FixEmptyResponseDescriptions() on the modified primary
// if you read them from storage and they are valid to start with.
//
// Entries in "paths", "definitions", "parameters" and "responses" are
// added to the primary in the order of the given mixins. If the entry
// already exists in primary it is skipped with a warning message.
//
// The count of skipped entries (from collisions) is returned so any
// deviation from the number expected can flag a warning in your build
// scripts. Carefully review the collisions before accepting them;
// consider renaming things if possible.
//
// No key normalization takes place (paths, type defs,
// etc). Ensure they are canonical if your downstream tools do
// key normalization of any form.
//
// Merging schemes (http, https), and consumers/producers do not account for
// collisions.
func Mixin(primary *spec.Swagger, mixins ...*spec.Swagger) []string {
	skipped := make([]string, 0, len(mixins))
	opIds := getOpIds(primary)
	initPrimary(primary)

	for i, m := range mixins {
		skipped = append(skipped, mergeSwaggerProps(primary, m)...)

		skipped = append(skipped, mergeConsumes(primary, m)...)

		skipped = append(skipped, mergeProduces(primary, m)...)

		skipped = append(skipped, mergeTags(primary, m)...)

		skipped = append(skipped, mergeSchemes(primary, m)...)

		skipped = append(skipped, mergeSecurityDefinitions(primary, m)...)

		skipped = append(skipped, mergeSecurityRequirements(primary, m)...)

		skipped = append(skipped, mergeDefinitions(primary, m)...)

		// merging paths requires a map of operationIDs to work with
		skipped = append(skipped, mergePaths(primary, m, opIds, i)...)

		skipped = append(skipped, mergeParameters(primary, m)...)

		skipped = append(skipped, mergeResponses(primary, m)...)
	}
	return skipped
}

// getOpIds extracts all the paths.<path>.operationIds from the given
// spec and returns them as the keys in a map with 'true' values.
func getOpIds(s *spec.Swagger) map[string]bool {
	rv := make(map[string]bool)
	if s.Paths == nil {
		return rv
	}
	for _, v := range s.Paths.Paths {
		piops := pathItemOps(v)
		for _, op := range piops {
			rv[op.ID] = true
		}
	}
	return rv
}

func pathItemOps(p spec.PathItem) []*spec.Operation {
	var rv []*spec.Operation
	rv = appendOp(rv, p.Get)
	rv = appendOp(rv, p.Put)
	rv = appendOp(rv, p.Post)
	rv = appendOp(rv, p.Delete)
	rv = appendOp(rv, p.Head)
	rv = appendOp(rv, p.Patch)
	return rv
}

func appendOp(ops []*spec.Operation, op *spec.Operation) []*spec.Operation {
	if op == nil {
		return ops
	}
	return append(ops, op)
}

func mergeSecurityDefinitions(primary *spec.Swagger, m *spec.Swagger) (skipped []string) {
	for k, v := range m.SecurityDefinitions {
		if _, exists := primary.SecurityDefinitions[k]; exists {
			warn := fmt.Sprintf(
				"SecurityDefinitions entry '%v' already exists in primary or higher priority mixin, skipping\n", k)
			skipped = append(skipped, warn)
			continue
		}
		primary.SecurityDefinitions[k] = v
	}
	return
}

func mergeSecurityRequirements(primary *spec.Swagger, m *spec.Swagger) (skipped []string) {
	for _, v := range m.Security {
		found := false
		for _, vv := range primary.Security {
			if reflect.DeepEqual(v, vv) {
				found = true
				break
			}
		}
		if found {
			warn := fmt.Sprintf(
				"Security requirement: '%v' already exists in primary or higher priority mixin, skipping\n", v)
			skipped = append(skipped, warn)
			continue
		}
		primary.Security = append(primary.Security, v)
	}
	return
}

func mergeDefinitions(primary *spec.Swagger, m *spec.Swagger) (skipped []string) {
	for k, v := range m.Definitions {
		// assume name collisions represent IDENTICAL type. careful.
		if _, exists := primary.Definitions[k]; exists {
			warn := fmt.Sprintf(
				"definitions entry '%v' already exists in primary or higher priority mixin, skipping\n", k)
			skipped = append(skipped, warn)
			continue
		}
		primary.Definitions[k] = v
	}
	return
}

func mergePaths(primary *spec.Swagger, m *spec.Swagger, opIds map[string]bool, mixIndex int) (skipped []string) {
	if m.Paths != nil {
		for k, v := range m.Paths.Paths {
			if _, exists := primary.Paths.Paths[k]; exists {
				warn := fmt.Sprintf(
					"paths entry '%v' already exists in primary or higher priority mixin, skipping\n", k)
				skipped = append(skipped, warn)
				continue
			}

			// Swagger requires that operationIds be
			// unique within a spec. If we find a
			// collision we append "Mixin0" to the
			// operatoinId we are adding, where 0 is mixin
			// index.  We assume that operationIds with
			// all the proivded specs are already unique.
			piops := pathItemOps(v)
			for _, piop := range piops {
				if opIds[piop.ID] {
					piop.ID = fmt.Sprintf("%v%v%v", piop.ID, "Mixin", mixIndex)
				}
				opIds[piop.ID] = true
			}
			primary.Paths.Paths[k] = v
		}
	}
	return
}

func mergeParameters(primary *spec.Swagger, m *spec.Swagger) (skipped []string) {
	for k, v := range m.Parameters {
		// could try to rename on conflict but would
		// have to fix $refs in the mixin. Complain
		// for now
		if _, exists := primary.Parameters[k]; exists {
			warn := fmt.Sprintf(
				"top level parameters entry '%v' already exists in primary or higher priority mixin, skipping\n", k)
			skipped = append(skipped, warn)
			continue
		}
		primary.Parameters[k] = v
	}
	return
}

func mergeResponses(primary *spec.Swagger, m *spec.Swagger) (skipped []string) {
	for k, v := range m.Responses {
		// could try to rename on conflict but would
		// have to fix $refs in the mixin. Complain
		// for now
		if _, exists := primary.Responses[k]; exists {
			warn := fmt.Sprintf(
				"top level responses entry '%v' already exists in primary or higher priority mixin, skipping\n", k)
			skipped = append(skipped, warn)
			continue
		}
		primary.Responses[k] = v
	}
	return skipped
}

func mergeConsumes(primary *spec.Swagger, m *spec.Swagger) []string {
	for _, v := range m.Consumes {
		found := false
		for _, vv := range primary.Consumes {
			if v == vv {
				found = true
				break
			}
		}
		if found {
			// no warning here: we just skip it
			continue
		}
		primary.Consumes = append(primary.Consumes, v)
	}
	return []string{}
}

func mergeProduces(primary *spec.Swagger, m *spec.Swagger) []string {
	for _, v := range m.Produces {
		found := false
		for _, vv := range primary.Produces {
			if v == vv {
				found = true
				break
			}
		}
		if found {
			// no warning here: we just skip it
			continue
		}
		primary.Produces = append(primary.Produces, v)
	}
	return []string{}
}

func mergeTags(primary *spec.Swagger, m *spec.Swagger) (skipped []string) {
	for _, v := range m.Tags {
		found := false
		for _, vv := range primary.Tags {
			if v.Name == vv.Name {
				found = true
				break
			}
		}
		if found {
			warn := fmt.Sprintf(
				"top level tags entry with name '%v' already exists in primary or higher priority mixin, skipping\n", v.Name)
			skipped = append(skipped, warn)
			continue
		}
		primary.Tags = append(primary.Tags, v)
	}
	return
}

func mergeSchemes(primary *spec.Swagger, m *spec.Swagger) []string {
	for _, v := range m.Schemes {
		found := false
		for _, vv := range primary.Schemes {
			if v == vv {
				found = true
				break
			}
		}
		if found {
			// no warning here: we just skip it
			continue
		}
		primary.Schemes = append(primary.Schemes, v)
	}
	return []string{}
}

func mergeSwaggerProps(primary *spec.Swagger, m *spec.Swagger) []string {
	var skipped []string
	primary.Extensions, skipped = mergeExtensions(primary.Extensions, m.Extensions)

	// merging details in swagger top properties
	if primary.Host == "" {
		primary.Host = m.Host
	}
	if primary.BasePath == "" {
		primary.BasePath = m.BasePath
	}
	if primary.Info == nil {
		primary.Info = m.Info
	} else if m.Info != nil {
		var sk []string
		primary.Info.Extensions, sk = mergeExtensions(primary.Info.Extensions, m.Info.Extensions)
		skipped = append(skipped, sk...)
		if primary.Info.Description == "" {
			primary.Info.Description = m.Info.Description
		}
		if primary.Info.Title == "" {
			primary.Info.Description = m.Info.Description
		}
		if primary.Info.TermsOfService == "" {
			primary.Info.TermsOfService = m.Info.TermsOfService
		}
		if primary.Info.Version == "" {
			primary.Info.Version = m.Info.Version
		}

		if primary.Info.Contact == nil {
			primary.Info.Contact = m.Info.Contact
		} else if m.Info.Contact != nil {
			var csk []string
			primary.Info.Contact.Extensions, csk = mergeExtensions(primary.Info.Contact.Extensions, m.Info.Contact.Extensions)
			skipped = append(skipped,csk...)
			if primary.Info.Contact.Name == "" {
				primary.Info.Contact.Name = m.Info.Contact.Name
			}
			if primary.Info.Contact.URL == "" {
				primary.Info.Contact.URL = m.Info.Contact.URL
			}
			if primary.Info.Contact.Email == "" {
				primary.Info.Contact.Email = m.Info.Contact.Email
			}
		}

		if primary.Info.License == nil {
			primary.Info.License = m.Info.License
		} else if m.Info.License != nil {
			var lsk []string
			primary.Info.License.Extensions, lsk = mergeExtensions(primary.Info.License.Extensions, m.Info.License.Extensions)
			skipped = append(skipped,lsk...)
			if primary.Info.License.Name == "" {
				primary.Info.License.Name = m.Info.License.Name
			}
			if primary.Info.License.URL == "" {
				primary.Info.License.URL = m.Info.License.URL
			}
		}

	}
	if primary.ExternalDocs == nil {
		primary.ExternalDocs = m.ExternalDocs
	} else if m.ExternalDocs != nil {
		if primary.ExternalDocs.Description == "" {
			primary.ExternalDocs.Description = m.ExternalDocs.Description
		}
		if primary.ExternalDocs.URL == "" {
			primary.ExternalDocs.URL = m.ExternalDocs.URL
		}
	}
	return skipped
}

func mergeExtensions(primary spec.Extensions, m spec.Extensions) (result spec.Extensions, skipped []string) {
	if primary == nil {
		result = m
		return
	}
	if m == nil {
		result = primary
		return
	}
	result = primary
	for k, v := range m {
		if _, found := primary[k]; found {
			skipped = append(skipped, k)
			continue
		}
		primary[k] = v
	}
	return
}

func initPrimary(primary *spec.Swagger) {
	if primary.SecurityDefinitions == nil {
		primary.SecurityDefinitions = make(map[string]*spec.SecurityScheme)
	}
	if primary.Security == nil {
		primary.Security = make([]map[string][]string, 0, 10)
	}
	if primary.Produces == nil {
		primary.Produces = make([]string, 0, 10)
	}
	if primary.Consumes == nil {
		primary.Consumes = make([]string, 0, 10)
	}
	if primary.Tags == nil {
		primary.Tags = make([]spec.Tag, 0, 10)
	}
	if primary.Schemes == nil {
		primary.Schemes = make([]string, 0, 10)
	}
	if primary.Paths == nil {
		primary.Paths = &spec.Paths{Paths: make(map[string]spec.PathItem)}
	}
	if primary.Paths.Paths == nil {
		primary.Paths.Paths = make(map[string]spec.PathItem)
	}
	if primary.Definitions == nil {
		primary.Definitions = make(spec.Definitions)
	}
	if primary.Parameters == nil {
		primary.Parameters = make(map[string]spec.Parameter)
	}
	if primary.Responses == nil {
		primary.Responses = make(map[string]spec.Response)
	}
}
<commit_msg>Update mixin.go<commit_after>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package analysis

import (
	"fmt"
	"reflect"

	"github.com/go-openapi/spec"
)

// Mixin modifies the primary swagger spec by adding the paths and
// definitions from the mixin specs. Top level parameters and
// responses from the mixins are also carried over. Operation id
// collisions are avoided by appending "Mixin<N>" but only if
// needed.
//
// The following parts of primary are subject to merge, filling empty details
//   - Info
//   - BasePath
//   - Host
//   - ExternalDocs
//
// Consider calling FixEmptyResponseDescriptions() on the modified primary
// if you read them from storage and they are valid to start with.
//
// Entries in "paths", "definitions", "parameters" and "responses" are
// added to the primary in the order of the given mixins. If the entry
// already exists in primary it is skipped with a warning message.
//
// The count of skipped entries (from collisions) is returned so any
// deviation from the number expected can flag a warning in your build
// scripts. Carefully review the collisions before accepting them;
// consider renaming things if possible.
//
// No key normalization takes place (paths, type defs,
// etc). Ensure they are canonical if your downstream tools do
// key normalization of any form.
//
// Merging schemes (http, https), and consumers/producers do not account for
// collisions.
func Mixin(primary *spec.Swagger, mixins ...*spec.Swagger) []string {
	skipped := make([]string, 0, len(mixins))
	opIds := getOpIds(primary)
	initPrimary(primary)

	for i, m := range mixins {
		skipped = append(skipped, mergeSwaggerProps(primary, m)...)

		skipped = append(skipped, mergeConsumes(primary, m)...)

		skipped = append(skipped, mergeProduces(primary, m)...)

		skipped = append(skipped, mergeTags(primary, m)...)

		skipped = append(skipped, mergeSchemes(primary, m)...)

		skipped = append(skipped, mergeSecurityDefinitions(primary, m)...)

		skipped = append(skipped, mergeSecurityRequirements(primary, m)...)

		skipped = append(skipped, mergeDefinitions(primary, m)...)

		// merging paths requires a map of operationIDs to work with
		skipped = append(skipped, mergePaths(primary, m, opIds, i)...)

		skipped = append(skipped, mergeParameters(primary, m)...)

		skipped = append(skipped, mergeResponses(primary, m)...)
	}
	return skipped
}

// getOpIds extracts all the paths.<path>.operationIds from the given
// spec and returns them as the keys in a map with 'true' values.
func getOpIds(s *spec.Swagger) map[string]bool {
	rv := make(map[string]bool)
	if s.Paths == nil {
		return rv
	}
	for _, v := range s.Paths.Paths {
		piops := pathItemOps(v)
		for _, op := range piops {
			rv[op.ID] = true
		}
	}
	return rv
}

func pathItemOps(p spec.PathItem) []*spec.Operation {
	var rv []*spec.Operation
	rv = appendOp(rv, p.Get)
	rv = appendOp(rv, p.Put)
	rv = appendOp(rv, p.Post)
	rv = appendOp(rv, p.Delete)
	rv = appendOp(rv, p.Head)
	rv = appendOp(rv, p.Patch)
	return rv
}

func appendOp(ops []*spec.Operation, op *spec.Operation) []*spec.Operation {
	if op == nil {
		return ops
	}
	return append(ops, op)
}

func mergeSecurityDefinitions(primary *spec.Swagger, m *spec.Swagger) (skipped []string) {
	for k, v := range m.SecurityDefinitions {
		if _, exists := primary.SecurityDefinitions[k]; exists {
			warn := fmt.Sprintf(
				"SecurityDefinitions entry '%v' already exists in primary or higher priority mixin, skipping\n", k)
			skipped = append(skipped, warn)
			continue
		}
		primary.SecurityDefinitions[k] = v
	}
	return
}

func mergeSecurityRequirements(primary *spec.Swagger, m *spec.Swagger) (skipped []string) {
	for _, v := range m.Security {
		found := false
		for _, vv := range primary.Security {
			if reflect.DeepEqual(v, vv) {
				found = true
				break
			}
		}
		if found {
			warn := fmt.Sprintf(
				"Security requirement: '%v' already exists in primary or higher priority mixin, skipping\n", v)
			skipped = append(skipped, warn)
			continue
		}
		primary.Security = append(primary.Security, v)
	}
	return
}

func mergeDefinitions(primary *spec.Swagger, m *spec.Swagger) (skipped []string) {
	for k, v := range m.Definitions {
		// assume name collisions represent IDENTICAL type. careful.
		if _, exists := primary.Definitions[k]; exists {
			warn := fmt.Sprintf(
				"definitions entry '%v' already exists in primary or higher priority mixin, skipping\n", k)
			skipped = append(skipped, warn)
			continue
		}
		primary.Definitions[k] = v
	}
	return
}

func mergePaths(primary *spec.Swagger, m *spec.Swagger, opIds map[string]bool, mixIndex int) (skipped []string) {
	if m.Paths != nil {
		for k, v := range m.Paths.Paths {
			if _, exists := primary.Paths.Paths[k]; exists {
				warn := fmt.Sprintf(
					"paths entry '%v' already exists in primary or higher priority mixin, skipping\n", k)
				skipped = append(skipped, warn)
				continue
			}

			// Swagger requires that operationIds be
			// unique within a spec. If we find a
			// collision we append "Mixin0" to the
			// operatoinId we are adding, where 0 is mixin
			// index.  We assume that operationIds with
			// all the proivded specs are already unique.
			piops := pathItemOps(v)
			for _, piop := range piops {
				if opIds[piop.ID] {
					piop.ID = fmt.Sprintf("%v%v%v", piop.ID, "Mixin", mixIndex)
				}
				opIds[piop.ID] = true
			}
			primary.Paths.Paths[k] = v
		}
	}
	return
}

func mergeParameters(primary *spec.Swagger, m *spec.Swagger) (skipped []string) {
	for k, v := range m.Parameters {
		// could try to rename on conflict but would
		// have to fix $refs in the mixin. Complain
		// for now
		if _, exists := primary.Parameters[k]; exists {
			warn := fmt.Sprintf(
				"top level parameters entry '%v' already exists in primary or higher priority mixin, skipping\n", k)
			skipped = append(skipped, warn)
			continue
		}
		primary.Parameters[k] = v
	}
	return
}

func mergeResponses(primary *spec.Swagger, m *spec.Swagger) (skipped []string) {
	for k, v := range m.Responses {
		// could try to rename on conflict but would
		// have to fix $refs in the mixin. Complain
		// for now
		if _, exists := primary.Responses[k]; exists {
			warn := fmt.Sprintf(
				"top level responses entry '%v' already exists in primary or higher priority mixin, skipping\n", k)
			skipped = append(skipped, warn)
			continue
		}
		primary.Responses[k] = v
	}
	return skipped
}

func mergeConsumes(primary *spec.Swagger, m *spec.Swagger) []string {
	for _, v := range m.Consumes {
		found := false
		for _, vv := range primary.Consumes {
			if v == vv {
				found = true
				break
			}
		}
		if found {
			// no warning here: we just skip it
			continue
		}
		primary.Consumes = append(primary.Consumes, v)
	}
	return []string{}
}

func mergeProduces(primary *spec.Swagger, m *spec.Swagger) []string {
	for _, v := range m.Produces {
		found := false
		for _, vv := range primary.Produces {
			if v == vv {
				found = true
				break
			}
		}
		if found {
			// no warning here: we just skip it
			continue
		}
		primary.Produces = append(primary.Produces, v)
	}
	return []string{}
}

func mergeTags(primary *spec.Swagger, m *spec.Swagger) (skipped []string) {
	for _, v := range m.Tags {
		found := false
		for _, vv := range primary.Tags {
			if v.Name == vv.Name {
				found = true
				break
			}
		}
		if found {
			warn := fmt.Sprintf(
				"top level tags entry with name '%v' already exists in primary or higher priority mixin, skipping\n", v.Name)
			skipped = append(skipped, warn)
			continue
		}
		primary.Tags = append(primary.Tags, v)
	}
	return
}

func mergeSchemes(primary *spec.Swagger, m *spec.Swagger) []string {
	for _, v := range m.Schemes {
		found := false
		for _, vv := range primary.Schemes {
			if v == vv {
				found = true
				break
			}
		}
		if found {
			// no warning here: we just skip it
			continue
		}
		primary.Schemes = append(primary.Schemes, v)
	}
	return []string{}
}

func mergeSwaggerProps(primary *spec.Swagger, m *spec.Swagger) []string {
	var skipped []string
	primary.Extensions, skipped = mergeExtensions(primary.Extensions, m.Extensions)

	// merging details in swagger top properties
	if primary.Host == "" {
		primary.Host = m.Host
	}
	if primary.BasePath == "" {
		primary.BasePath = m.BasePath
	}
	if primary.Info == nil {
		primary.Info = m.Info
	} else if m.Info != nil {
		var sk []string
		primary.Info.Extensions, sk = mergeExtensions(primary.Info.Extensions, m.Info.Extensions)
		skipped = append(skipped, sk...)
		if primary.Info.Description == "" {
			primary.Info.Description = m.Info.Description
		}
		if primary.Info.Title == "" {
			primary.Info.Description = m.Info.Description
		}
		if primary.Info.TermsOfService == "" {
			primary.Info.TermsOfService = m.Info.TermsOfService
		}
		if primary.Info.Version == "" {
			primary.Info.Version = m.Info.Version
		}

		if primary.Info.Contact == nil {
			primary.Info.Contact = m.Info.Contact
		} else if m.Info.Contact != nil {
			var csk []string
			primary.Info.Contact.Extensions, csk = mergeExtensions(primary.Info.Contact.Extensions, m.Info.Contact.Extensions)
			skipped = append(skipped, csk...)
			if primary.Info.Contact.Name == "" {
				primary.Info.Contact.Name = m.Info.Contact.Name
			}
			if primary.Info.Contact.URL == "" {
				primary.Info.Contact.URL = m.Info.Contact.URL
			}
			if primary.Info.Contact.Email == "" {
				primary.Info.Contact.Email = m.Info.Contact.Email
			}
		}

		if primary.Info.License == nil {
			primary.Info.License = m.Info.License
		} else if m.Info.License != nil {
			var lsk []string
			primary.Info.License.Extensions, lsk = mergeExtensions(primary.Info.License.Extensions, m.Info.License.Extensions)
			skipped = append(skipped,lsk...)
			if primary.Info.License.Name == "" {
				primary.Info.License.Name = m.Info.License.Name
			}
			if primary.Info.License.URL == "" {
				primary.Info.License.URL = m.Info.License.URL
			}
		}

	}
	if primary.ExternalDocs == nil {
		primary.ExternalDocs = m.ExternalDocs
	} else if m.ExternalDocs != nil {
		if primary.ExternalDocs.Description == "" {
			primary.ExternalDocs.Description = m.ExternalDocs.Description
		}
		if primary.ExternalDocs.URL == "" {
			primary.ExternalDocs.URL = m.ExternalDocs.URL
		}
	}
	return skipped
}

func mergeExtensions(primary spec.Extensions, m spec.Extensions) (result spec.Extensions, skipped []string) {
	if primary == nil {
		result = m
		return
	}
	if m == nil {
		result = primary
		return
	}
	result = primary
	for k, v := range m {
		if _, found := primary[k]; found {
			skipped = append(skipped, k)
			continue
		}
		primary[k] = v
	}
	return
}

func initPrimary(primary *spec.Swagger) {
	if primary.SecurityDefinitions == nil {
		primary.SecurityDefinitions = make(map[string]*spec.SecurityScheme)
	}
	if primary.Security == nil {
		primary.Security = make([]map[string][]string, 0, 10)
	}
	if primary.Produces == nil {
		primary.Produces = make([]string, 0, 10)
	}
	if primary.Consumes == nil {
		primary.Consumes = make([]string, 0, 10)
	}
	if primary.Tags == nil {
		primary.Tags = make([]spec.Tag, 0, 10)
	}
	if primary.Schemes == nil {
		primary.Schemes = make([]string, 0, 10)
	}
	if primary.Paths == nil {
		primary.Paths = &spec.Paths{Paths: make(map[string]spec.PathItem)}
	}
	if primary.Paths.Paths == nil {
		primary.Paths.Paths = make(map[string]spec.PathItem)
	}
	if primary.Definitions == nil {
		primary.Definitions = make(spec.Definitions)
	}
	if primary.Parameters == nil {
		primary.Parameters = make(map[string]spec.Parameter)
	}
	if primary.Responses == nil {
		primary.Responses = make(map[string]spec.Response)
	}
}
<|endoftext|>


<commit_before>package scummatlas

import (
	_ "bufio"
	"fmt"
	goimage "image"
	"image/color"
	"io/ioutil"
	"log"
	"os"
	b "scummatlas/binaryutils"
	"scummatlas/image"
)

type BoxMatrix bool

type Room struct {
	data     []byte
	offset   int
	Width    int
	Height   int
	ObjCount int
	//ColorCycle ColorCycle
	TranspIndex  uint8
	Palette      color.Palette
	Image        *goimage.RGBA
	Objects      map[int]Object
	ExitScript   Script
	EntryScript  Script
	Boxes        []Box
	LocalScripts map[int]Script
	BoxMatrix    BoxMatrix
}

func NewRoom(data []byte) *Room {
	room := new(Room)
	room.data = data
	room.offset = 0
	room.Objects = make(map[int]Object)
	room.LocalScripts = make(map[int]Script)

	blockName := room.getBlockName()
	if blockName != "ROOM" {
		panic("Can't find ROOM")
	}

	room.offset = 8
	fmt.Printf("Block Name\tBlock Size\n=============\n")
	for room.offset < len(data) {
		blockName := room.getBlockName()
		fmt.Printf("%v\t%v bytes\n", blockName, room.getBlockSize())

		switch blockName {
		case "RMHD":
			room.parseRMHD()
		case "BOXD":
			room.parseBOXD()
		case "EXCD":
			//room.parseEXCD()
		case "ENCD":
			//room.parseENCD()
		case "EPAL":
			room.parseEPAL()
		case "CLUT":
			log.SetOutput(ioutil.Discard)
			room.parseCLUT()
		case "LSCR":
			//room.parseLSCR()
		case "OBCD":
			//room.parseOBCD()
		case "OBIM":
			room.parseOBIM()
		case "RMIM":
			room.parseRMIM()
		case "TRNS":
			room.parseTRNS()
		}
		log.SetOutput(os.Stdout)
		room.nextBlock()
	}

	fmt.Println("New ROOM\n")
	room.Print()
	return room
}

func (r *Room) parseLSCR() {
	scriptId := int(r.data[r.offset+8])
	scriptBlock := r.data[r.offset+9 : r.offset+r.getBlockSize()]
	script := parseScriptBlock(scriptBlock)
	r.LocalScripts[scriptId] = script
	if len(script) == 0 {
		fmt.Printf("DUMP from %x\n", r.offset+9)
		fmt.Printf("%x", scriptBlock)
	}
	fmt.Printf("\nLocal ScriptID 0x%02x, size %d, script %v\n", scriptId, r.getBlockSize(), script)
}

func (r *Room) parseTRNS() {
	r.TranspIndex = r.data[r.offset+8]
	fmt.Println("Transparent index", r.TranspIndex)
}

func (r *Room) parseOBIM() {
	blockSize := b.BE32(r.data, r.offset+4)
	objImg, id := NewObjectImageFromOBIM(r.data[r.offset:r.offset+blockSize], r)
	fmt.Printf("======================\nObject with id 0x%02X\n%+v\n", id, objImg)

	existing, ok := r.Objects[id]
	if !ok {
		existing = Object{Id: id}
	}
	existing.Image = objImg
	r.Objects[id] = existing
}

func (r *Room) parseOBCD() {
	blockSize := b.BE32(r.data, r.offset+4)
	object := NewObjectFromOBCD(r.data[r.offset : r.offset+blockSize])

	existingObject, ok := r.Objects[object.Id]
	if ok {
		object.Image = existingObject.Image
	}
	r.Objects[object.Id] = object
}

func (r *Room) parseENCD() {
	r.EntryScript = parseScriptBlock(r.data[r.offset+8 : r.offset+r.getBlockSize()])
}

func (r *Room) parseEPAL() {
	fmt.Println("EGA palette, not used")
	/*
		paletteData := r.data[r.offset+8 : r.offset+r.getBlockSize()]
		fmt.Println("Palette data size ", len(paletteData))

		r.Palette = image.ParsePalette(r.data[r.offset+8 : r.offset+8+3*256])
		fmt.Println("Palette length", len(r.Palette))
		fmt.Println(r.Palette)
	*/

}

func (r *Room) parseCLUT() {
	paletteData := r.data[r.offset+8 : r.offset+r.getBlockSize()]
	log.Println("Palette data size ", len(paletteData))

	r.Palette = image.ParsePalette(r.data[r.offset+8 : r.offset+8+3*256])
	log.Println("Palette length", len(r.Palette))

	for _, color := range r.Palette {
		r, g, b, _ := color.RGBA()
		r8, g8, b8 := uint8(r), uint8(g), uint8(b)
		log.Printf(" %x%x%x", r8, g8, b8)
	}
	log.Println()
}

func (r *Room) parseEXCD() {
	r.ExitScript = parseScriptBlock(r.data[r.offset+8 : r.offset+r.getBlockSize()])
}

func (r *Room) parseRMIM() {
	if string(r.data[r.offset+8:r.offset+12]) != "RMIH" {
		panic("Not room image header")
	}
	headerSize := b.BE32(r.data, r.offset+12)
	zBuffers := b.LE16(r.data, r.offset+16)
	fmt.Println("headerSize", headerSize)
	fmt.Println("zBuffers", zBuffers)

	if b.FourCharString(r.data, r.offset+18) != "IM00" {
		panic("Not room image found")
	}
	imageOffset := r.offset + 18
	imageSize := b.BE32(r.data, imageOffset+4)
	fmt.Println(b.FourCharString(r.data, imageOffset), imageSize)

	r.Image = image.ParseImage(
		r.data[imageOffset:imageOffset+4+imageSize],
		zBuffers,
		r.Width,
		r.Height,
		r.Palette,
		r.TranspIndex,
		false)
}

func (r *Room) parseBOXD() {
	boxCount := b.LE16(r.data, r.offset+8)
	var boxOffset int
	fmt.Println("BOXCOUNT", boxCount)
	for i := 0; i < boxCount; i++ {
		boxOffset = r.offset + 10 + i*20
		box := NewBox(r.data[boxOffset : boxOffset+20])
		r.Boxes = append(r.Boxes, box)
	}
}

func (r *Room) parseRMHD() {
	fmt.Println("RMHD offset", r.offset)
	r.Width = b.LE16(r.data, r.offset+8)
	r.Height = b.LE16(r.data, r.offset+10)
	r.ObjCount = b.LE16(r.data, r.offset+12)
	fmt.Printf("Room size %vx%v\n", r.Width, r.Height)
}

func (r Room) Print() {
	fmt.Println("Size: ", r.Width, r.Height)
	fmt.Println("Object count: ", r.ObjCount)
	fmt.Println("Boxes: ", len(r.Boxes))
}

func (r Room) getBlockName() string {
	return string(r.data[r.offset : r.offset+4])
}

func (r Room) getBlockSize() int {
	return b.BE32(r.data, r.offset+4)
}

func (r *Room) nextBlock() {
	r.offset += r.getBlockSize()
}

func NewBox(data []byte) Box {
	box := new(Box)

	box.ulx = b.LE16(data, 0)
	box.uly = b.LE16(data, 2)
	box.urx = b.LE16(data, 4)
	box.ury = b.LE16(data, 6)
	box.lrx = b.LE16(data, 8)
	box.lry = b.LE16(data, 10)
	box.llx = b.LE16(data, 12)
	box.lly = b.LE16(data, 14)
	box.mask = data[16]
	box.flags = data[17]
	box.scale = b.LE16(data, 18)

	return *box
}
<commit_msg>cleanup<commit_after>package scummatlas

import (
	_ "bufio"
	"fmt"
	goimage "image"
	"image/color"
	"io/ioutil"
	"log"
	"os"
	b "scummatlas/binaryutils"
	"scummatlas/image"
)

type BoxMatrix bool

type Room struct {
	data         []byte
	offset       int
	Width        int
	Height       int
	ObjCount     int
	TranspIndex  uint8
	Palette      color.Palette
	Image        *goimage.RGBA
	Boxes        []Box
	BoxMatrix    BoxMatrix
	ExitScript   Script
	EntryScript  Script
	LocalScripts map[int]Script
	Objects      map[int]Object
	//ColorCycle ColorCycle
}

func NewRoom(data []byte) *Room {
	blockName := room.getBlockName()
	if blockName != "ROOM" {
		panic("Can't find ROOM")
	}

	room := new(Room)
	room.data = data
	room.offset = 8
	room.Objects = make(map[int]Object)
	room.LocalScripts = make(map[int]Script)

	fmt.Printf("Block Name\tBlock Size\n=============\n")
	for room.offset < len(data) {
		blockName := room.getBlockName()
		fmt.Printf("%v\t%v bytes\n", blockName, room.getBlockSize())

		switch blockName {
		case "RMHD":
			room.parseRMHD()
		case "BOXD":
			room.parseBOXD()
		case "EXCD":
			//room.parseEXCD()
		case "ENCD":
			//room.parseENCD()
		case "EPAL":
			room.parseEPAL()
		case "CLUT":
			log.SetOutput(ioutil.Discard)
			room.parseCLUT()
		case "LSCR":
			//room.parseLSCR()
		case "OBCD":
			//room.parseOBCD()
		case "OBIM":
			room.parseOBIM()
		case "RMIM":
			room.parseRMIM()
		case "TRNS":
			room.parseTRNS()
		}
		log.SetOutput(os.Stdout)
		room.nextBlock()
	}

	return room
}

func (r *Room) parseLSCR() {
	scriptId := int(r.data[r.offset+8])
	scriptBlock := r.data[r.offset+9 : r.offset+r.getBlockSize()]
	script := parseScriptBlock(scriptBlock)
	r.LocalScripts[scriptId] = script
	if len(script) == 0 {
		fmt.Printf("DUMP from %x\n", r.offset+9)
		fmt.Printf("%x", scriptBlock)
	}
	fmt.Printf("\nLocal ScriptID 0x%02x, size %d, script %v\n", scriptId, r.getBlockSize(), script)
}

func (r *Room) parseTRNS() {
	r.TranspIndex = r.data[r.offset+8]
	fmt.Println("Transparent index", r.TranspIndex)
}

func (r *Room) parseOBIM() {
	blockSize := b.BE32(r.data, r.offset+4)
	objImg, id := NewObjectImageFromOBIM(r.data[r.offset:r.offset+blockSize], r)
	fmt.Printf("======================\nObject with id 0x%02X\n%+v\n", id, objImg)

	existing, ok := r.Objects[id]
	if !ok {
		existing = Object{Id: id}
	}
	existing.Image = objImg
	r.Objects[id] = existing
}

func (r *Room) parseOBCD() {
	blockSize := b.BE32(r.data, r.offset+4)
	object := NewObjectFromOBCD(r.data[r.offset : r.offset+blockSize])

	existingObject, ok := r.Objects[object.Id]
	if ok {
		object.Image = existingObject.Image
	}
	r.Objects[object.Id] = object
}

func (r *Room) parseENCD() {
	r.EntryScript = parseScriptBlock(r.data[r.offset+8 : r.offset+r.getBlockSize()])
}

func (r *Room) parseEPAL() {
	fmt.Println("EGA palette, not used")
	/*
		paletteData := r.data[r.offset+8 : r.offset+r.getBlockSize()]
		fmt.Println("Palette data size ", len(paletteData))

		r.Palette = image.ParsePalette(r.data[r.offset+8 : r.offset+8+3*256])
		fmt.Println("Palette length", len(r.Palette))
		fmt.Println(r.Palette)
	*/

}

func (r *Room) parseCLUT() {
	paletteData := r.data[r.offset+8 : r.offset+r.getBlockSize()]
	log.Println("Palette data size ", len(paletteData))

	r.Palette = image.ParsePalette(r.data[r.offset+8 : r.offset+8+3*256])
	log.Println("Palette length", len(r.Palette))

	for _, color := range r.Palette {
		r, g, b, _ := color.RGBA()
		r8, g8, b8 := uint8(r), uint8(g), uint8(b)
		log.Printf(" %x%x%x", r8, g8, b8)
	}
	log.Println()
}

func (r *Room) parseEXCD() {
	r.ExitScript = parseScriptBlock(r.data[r.offset+8 : r.offset+r.getBlockSize()])
}

func (r *Room) parseRMIM() {
	if string(r.data[r.offset+8:r.offset+12]) != "RMIH" {
		panic("Not room image header")
	}
	headerSize := b.BE32(r.data, r.offset+12)
	zBuffers := b.LE16(r.data, r.offset+16)
	fmt.Println("headerSize", headerSize)
	fmt.Println("zBuffers", zBuffers)

	if b.FourCharString(r.data, r.offset+18) != "IM00" {
		panic("Not room image found")
	}
	imageOffset := r.offset + 18
	imageSize := b.BE32(r.data, imageOffset+4)
	fmt.Println(b.FourCharString(r.data, imageOffset), imageSize)

	r.Image = image.ParseImage(
		r.data[imageOffset:imageOffset+4+imageSize],
		zBuffers,
		r.Width,
		r.Height,
		r.Palette,
		r.TranspIndex,
		false)
}

func (r *Room) parseBOXD() {
	boxCount := b.LE16(r.data, r.offset+8)
	var boxOffset int
	fmt.Println("BOXCOUNT", boxCount)
	for i := 0; i < boxCount; i++ {
		boxOffset = r.offset + 10 + i*20
		box := NewBox(r.data[boxOffset : boxOffset+20])
		r.Boxes = append(r.Boxes, box)
	}
}

func (r *Room) parseRMHD() {
	fmt.Println("RMHD offset", r.offset)
	r.Width = b.LE16(r.data, r.offset+8)
	r.Height = b.LE16(r.data, r.offset+10)
	r.ObjCount = b.LE16(r.data, r.offset+12)
	fmt.Printf("Room size %vx%v\n", r.Width, r.Height)
}

func (r Room) Print() {
	fmt.Println("Size: ", r.Width, r.Height)
	fmt.Println("Object count: ", r.ObjCount)
	fmt.Println("Boxes: ", len(r.Boxes))
}

func (r Room) getBlockName() string {
	return string(r.data[r.offset : r.offset+4])
}

func (r Room) getBlockSize() int {
	return b.BE32(r.data, r.offset+4)
}

func (r *Room) nextBlock() {
	r.offset += r.getBlockSize()
}

func NewBox(data []byte) Box {
	box := new(Box)

	box.ulx = b.LE16(data, 0)
	box.uly = b.LE16(data, 2)
	box.urx = b.LE16(data, 4)
	box.ury = b.LE16(data, 6)
	box.lrx = b.LE16(data, 8)
	box.lry = b.LE16(data, 10)
	box.llx = b.LE16(data, 12)
	box.lly = b.LE16(data, 14)
	box.mask = data[16]
	box.flags = data[17]
	box.scale = b.LE16(data, 18)

	return *box
}
<|endoftext|>


<commit_before>package yext

import "fmt"

type CustomFieldType string

const (
	CUSTOMFIELDTYPE_YESNO          = "BOOLEAN"
	CUSTOMFIELDTYPE_SINGLELINETEXT = "TEXT"
	CUSTOMFIELDTYPE_MULTILINETEXT  = "MULTILINE_TEXT"
	CUSTOMFIELDTYPE_SINGLEOPTION   = "SINGLE_OPTION"
	CUSTOMFIELDTYPE_URL            = "URL"
	CUSTOMFIELDTYPE_DATE           = "DATE"
	CUSTOMFIELDTYPE_NUMBER         = "NUMBER"
	CUSTOMFIELDTYPE_MULTIOPTION    = "MULTI_OPTION"
	CUSTOMFIELDTYPE_TEXTLIST       = "TEXT_LIST"
	CUSTOMFIELDTYPE_PHOTO          = "PHOTO"
	CUSTOMFIELDTYPE_GALLERY        = "GALLERY"
	CUSTOMFIELDTYPE_VIDEO          = "VIDEO"
	CUSTOMFIELDTYPE_HOURS          = "HOURS"
	CUSTOMFIELDTYPE_DAILYTIMES     = "DAILY_TIMES"
	CUSTOMFIELDTYPE_LOCATIONLIST   = "LOCATION_LIST"
	// not sure what to do with "DAILYTIMES", omitting
)

type CustomFieldOption struct {
	Key   string `json:"key,omitempty"`
	Value string `json:"value"`
}

// CustomField is the representation of a Custom Field definition in Yext Location Manager.
// For details see https://www.yext.com/support/platform-api/#Administration_API/Custom_Fields.htm
type CustomField struct {
	Id                         string              `json:"id"`
	Type                       string              `json:"type"`
	Name                       string              `json:"name"`
	Options                    []CustomFieldOption `json:"options"` // Only present for multi-option custom fields
	Group                      string              `json:"group"`
	Description                string              `json:"description"`
	AlternateLanguageBehaviour string              `json:"alternateLanguageBehavior"`
}

type CustomFieldValue interface {
	CustomFieldTag() string
}

type YesNo bool

func (y YesNo) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_YESNO
}

type SingleLineText string

func (s SingleLineText) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_SINGLELINETEXT
}

type MultiLineText string

func (m MultiLineText) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_MULTILINETEXT
}

type Url string

func (u Url) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_URL
}

type Date string

func (d Date) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_DATE
}

type Number string

func (n Number) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_NUMBER
}

type OptionField interface {
	CustomFieldValue
	SetOptionId(id string)
	UnsetOptionId(id string)
	IsOptionIdSet(id string) bool
}

type SingleOption string

func (s SingleOption) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_SINGLEOPTION
}

func (s *SingleOption) SetOptionId(id string) {
	*s = SingleOption(id)
}

func (s *SingleOption) UnsetOptionId(id string) {
	if string(*s) == id {
		*s = SingleOption("")
	}
}

func (s *SingleOption) IsOptionIdSet(id string) bool {
	return *s == SingleOption(id)
}

type MultiOption UnorderedStrings

func (m MultiOption) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_MULTIOPTION
}

func (m MultiOption) Equal(c Comparable) bool {
	var n MultiOption
	switch v := c.(type) {
	case MultiOption:
		n = v
	case *MultiOption:
		n = *v
	default:
		panic(fmt.Errorf("%v is not a MultiOption is %T", c, c))
	}
	if len(m) != len(n) {
		return false
	}
	a := UnorderedStrings(m)
	b := UnorderedStrings(n)
	return (&a).Equal(&b)

}

func (m *MultiOption) SetOptionId(id string) {
	if !m.IsOptionIdSet(id) {
		*m = append(*m, id)
	}
}

func (m *MultiOption) UnsetOptionId(id string) {
	if m.IsOptionIdSet(id) {
		t := []string(*m)
		indexOfTarget := -1
		for i := 0; i < len(*m); i++ {
			if t[i] == id {
				indexOfTarget = i
			}
		}
		if indexOfTarget >= 0 {
			*m = append(t[:indexOfTarget], t[indexOfTarget+1:]...)
		}
	}
}

func (m *MultiOption) IsOptionIdSet(id string) bool {
	for _, option := range *m {
		if option == id {
			return true
		}
	}
	return false
}

type TextList []string

func (t TextList) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_TEXTLIST
}

type LocationList UnorderedStrings

func (l LocationList) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_LOCATIONLIST
}

type Photo struct {
	Url             string `json:"url,omitempty"`
	Description     string `json:"description,omitempty"`
	Details         string `json:"details,omitempty"`
	ClickThroughURL string `json:"clickthroughUrl,omitempty"`
}

func (p *Photo) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_PHOTO
}

type Gallery []*Photo

func (g *Gallery) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_GALLERY
}

type Video struct {
	Description string `json:"description"`
	Url         string `json:"url"`
}

type VideoGallery []Video

func (v *VideoGallery) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_VIDEO
}

type Hours struct {
	AdditionalText string         `json:"additionalHoursText,omitempty"`
	Hours          string         `json:"hours,omitempty"`
	HolidayHours   []HolidayHours `json:"holidayHours,omitempty"`
}

func (h *Hours) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_HOURS
}

type DailyTimes struct {
	DailyTimes string `json:"dailyTimes,omitempty"`
}

func (d DailyTimes) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_DAILYTIMES
}
<commit_msg>Add comparison for LocationList type<commit_after>package yext

import "fmt"

type CustomFieldType string

const (
	CUSTOMFIELDTYPE_YESNO          = "BOOLEAN"
	CUSTOMFIELDTYPE_SINGLELINETEXT = "TEXT"
	CUSTOMFIELDTYPE_MULTILINETEXT  = "MULTILINE_TEXT"
	CUSTOMFIELDTYPE_SINGLEOPTION   = "SINGLE_OPTION"
	CUSTOMFIELDTYPE_URL            = "URL"
	CUSTOMFIELDTYPE_DATE           = "DATE"
	CUSTOMFIELDTYPE_NUMBER         = "NUMBER"
	CUSTOMFIELDTYPE_MULTIOPTION    = "MULTI_OPTION"
	CUSTOMFIELDTYPE_TEXTLIST       = "TEXT_LIST"
	CUSTOMFIELDTYPE_PHOTO          = "PHOTO"
	CUSTOMFIELDTYPE_GALLERY        = "GALLERY"
	CUSTOMFIELDTYPE_VIDEO          = "VIDEO"
	CUSTOMFIELDTYPE_HOURS          = "HOURS"
	CUSTOMFIELDTYPE_DAILYTIMES     = "DAILY_TIMES"
	CUSTOMFIELDTYPE_LOCATIONLIST   = "LOCATION_LIST"
	// not sure what to do with "DAILYTIMES", omitting
)

type CustomFieldOption struct {
	Key   string `json:"key,omitempty"`
	Value string `json:"value"`
}

// CustomField is the representation of a Custom Field definition in Yext Location Manager.
// For details see https://www.yext.com/support/platform-api/#Administration_API/Custom_Fields.htm
type CustomField struct {
	Id                         string              `json:"id"`
	Type                       string              `json:"type"`
	Name                       string              `json:"name"`
	Options                    []CustomFieldOption `json:"options"` // Only present for multi-option custom fields
	Group                      string              `json:"group"`
	Description                string              `json:"description"`
	AlternateLanguageBehaviour string              `json:"alternateLanguageBehavior"`
}

type CustomFieldValue interface {
	CustomFieldTag() string
}

type YesNo bool

func (y YesNo) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_YESNO
}

type SingleLineText string

func (s SingleLineText) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_SINGLELINETEXT
}

type MultiLineText string

func (m MultiLineText) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_MULTILINETEXT
}

type Url string

func (u Url) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_URL
}

type Date string

func (d Date) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_DATE
}

type Number string

func (n Number) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_NUMBER
}

type OptionField interface {
	CustomFieldValue
	SetOptionId(id string)
	UnsetOptionId(id string)
	IsOptionIdSet(id string) bool
}

type SingleOption string

func (s SingleOption) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_SINGLEOPTION
}

func (s *SingleOption) SetOptionId(id string) {
	*s = SingleOption(id)
}

func (s *SingleOption) UnsetOptionId(id string) {
	if string(*s) == id {
		*s = SingleOption("")
	}
}

func (s *SingleOption) IsOptionIdSet(id string) bool {
	return *s == SingleOption(id)
}

type MultiOption UnorderedStrings

func (m MultiOption) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_MULTIOPTION
}

func (m MultiOption) Equal(c Comparable) bool {
	var n MultiOption
	switch v := c.(type) {
	case MultiOption:
		n = v
	case *MultiOption:
		n = *v
	default:
		panic(fmt.Errorf("%v is not a MultiOption is %T", c, c))
	}
	if len(m) != len(n) {
		return false
	}
	a := UnorderedStrings(m)
	b := UnorderedStrings(n)
	return (&a).Equal(&b)

}

func (m *MultiOption) SetOptionId(id string) {
	if !m.IsOptionIdSet(id) {
		*m = append(*m, id)
	}
}

func (m *MultiOption) UnsetOptionId(id string) {
	if m.IsOptionIdSet(id) {
		t := []string(*m)
		indexOfTarget := -1
		for i := 0; i < len(*m); i++ {
			if t[i] == id {
				indexOfTarget = i
			}
		}
		if indexOfTarget >= 0 {
			*m = append(t[:indexOfTarget], t[indexOfTarget+1:]...)
		}
	}
}

func (m *MultiOption) IsOptionIdSet(id string) bool {
	for _, option := range *m {
		if option == id {
			return true
		}
	}
	return false
}

type TextList []string

func (t TextList) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_TEXTLIST
}

type LocationList UnorderedStrings

func (l LocationList) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_LOCATIONLIST
}

func (m LocationList) Equal(c Comparable) bool {
	var n LocationList
	switch v := c.(type) {
	case LocationList:
		n = v
	case *LocationList:
		n = *v
	default:
		panic(fmt.Errorf("%v is not a LocationList is %T", c, c))
	}
	if len(m) != len(n) {
		return false
	}
	a := UnorderedStrings(m)
	b := UnorderedStrings(n)
	return (&a).Equal(&b)
}

type Photo struct {
	Url             string `json:"url,omitempty"`
	Description     string `json:"description,omitempty"`
	Details         string `json:"details,omitempty"`
	ClickThroughURL string `json:"clickthroughUrl,omitempty"`
}

func (p *Photo) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_PHOTO
}

type Gallery []*Photo

func (g *Gallery) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_GALLERY
}

type Video struct {
	Description string `json:"description"`
	Url         string `json:"url"`
}

type VideoGallery []Video

func (v *VideoGallery) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_VIDEO
}

type Hours struct {
	AdditionalText string         `json:"additionalHoursText,omitempty"`
	Hours          string         `json:"hours,omitempty"`
	HolidayHours   []HolidayHours `json:"holidayHours,omitempty"`
}

func (h *Hours) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_HOURS
}

type DailyTimes struct {
	DailyTimes string `json:"dailyTimes,omitempty"`
}

func (d DailyTimes) CustomFieldTag() string {
	return CUSTOMFIELDTYPE_DAILYTIMES
}
<|endoftext|>


<commit_before>package build

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"fmt"
	"go/build"
	"io"
	"os"
	"strings"
	"time"

	"github.com/blakesmith/ar"
)

const controlFile = `Package: %s
Version: %s
Architecture: %s
Maintainer: Koding Developers <hello@koding.com>
Installed-Size: %d
Section: devel
Priority: extra
Description: %s Kite
`

func (b *Build) Linux() error {
	debName := b.Output + ".deb"
	tarFile := b.Output + ".tar.gz"

	deb, err := os.Create(debName + ".inprogress")
	if err != nil {
		return fmt.Errorf("cannot create deb: %v", err)
	}
	defer deb.Close()

	// create first tar file
	fmt.Println("creating tarfile", tarFile)
	if err := b.TarGzFile(); err != nil {
		return err
	}

	fmt.Println("openening tarfile", tarFile)
	tf, err := os.Open(tarFile)
	if err != nil {
		return err
	}
	defer tf.Close()

	fmt.Println("creating debfile from tarfile")
	if err := b.createDeb(tf, deb); err != nil {
		return err
	}

	if err := os.Rename(debName+".inprogress", debName); err != nil {
		return err
	}

	fmt.Println("package", debName, "ready")
	return nil
}

func debArch() string {
	arch := build.Default.GOARCH
	if arch == "386" {
		return "i386"
	}
	return arch
}

func (b *Build) createDeb(tarball io.Reader, deb io.Writer) error {
	now := time.Now()
	dataTarGz, md5sums, instSize, err := b.translateTarball(now, tarball)
	if err != nil {
		return err
	}

	controlTarGz, err := b.createControl(now, instSize, md5sums)
	if err != nil {
		return err
	}
	w := ar.NewWriter(deb)
	if err := w.WriteGlobalHeader(); err != nil {
		return fmt.Errorf("cannot write ar header to deb file: %v", err)
	}

	if err := addArFile(now, w, "debian-binary", []byte("2.0\n")); err != nil {
		return fmt.Errorf("cannot pack debian-binary: %v", err)
	}
	if err := addArFile(now, w, "control.tar.gz", controlTarGz); err != nil {
		return fmt.Errorf("cannot add control.tar.gz to deb: %v", err)
	}
	if err := addArFile(now, w, "data.tar.gz", dataTarGz); err != nil {
		return fmt.Errorf("cannot add data.tar.gz to deb: %v", err)
	}
	return nil
}

func addArFile(now time.Time, w *ar.Writer, name string, body []byte) error {
	hdr := ar.Header{
		Name:    name,
		Size:    int64(len(body)),
		Mode:    0644,
		ModTime: now,
	}
	if err := w.WriteHeader(&hdr); err != nil {
		return fmt.Errorf("cannot write file header: %v", err)
	}
	_, err := w.Write(body)
	return err
}

func (b *Build) translateTarball(now time.Time, tarball io.Reader) (dataTarGz, md5sums []byte, instSize int64, err error) {
	buf := &bytes.Buffer{}
	compress := gzip.NewWriter(buf)
	out := tar.NewWriter(compress)

	md5buf := &bytes.Buffer{}
	md5tmp := make([]byte, 0, md5.Size)

	uncompress, err := gzip.NewReader(tarball)
	if err != nil {
		return nil, nil, 0, fmt.Errorf("cannot uncompress tarball: %v", err)
	}

	in := tar.NewReader(uncompress)
	first := true
	for {
		h, err := in.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, nil, 0, fmt.Errorf("cannot read tarball: %v", err)
		}

		instSize += h.Size
		h.Name = strings.TrimLeft(h.Name, "./")
		if first && h.Name != b.AppName && h.Name != b.AppName+"/" {
			h := tar.Header{
				Name:     "./opt/koding/" + b.AppName,
				Mode:     0755,
				ModTime:  h.ModTime,
				Typeflag: tar.TypeDir,
			}

			if err := out.WriteHeader(&h); err != nil {
				return nil, nil, 0, fmt.Errorf("cannot write header of %s to data.tar.gz: %v", h.Name, err)
			}
		}

		const prefix = "./opt/koding/"
		h.Name = prefix + h.Name
		if h.Typeflag == tar.TypeDir && !strings.HasSuffix(h.Name, "/") {
			h.Name += "/"
		}

		if err := out.WriteHeader(h); err != nil {
			return nil, nil, 0, fmt.Errorf("cannot write header of %s to data.tar.gz: %v", h.Name, err)
		}

		fmt.Println("tar: packing", h.Name[len(prefix):])

		if h.Typeflag == tar.TypeDir {
			continue
		}

		digest := md5.New()
		if _, err := io.Copy(out, io.TeeReader(in, digest)); err != nil {
			return nil, nil, 0, err
		}

		fmt.Fprintf(md5buf, "%x  %s\n", digest.Sum(md5tmp), h.Name[2:])
	}

	if err := out.Close(); err != nil {
		return nil, nil, 0, err
	}

	if err := compress.Close(); err != nil {
		return nil, nil, 0, err
	}

	return buf.Bytes(), md5buf.Bytes(), instSize, nil
}

func addTarSymlink(now time.Time, out *tar.Writer, name, target string) error {
	h := tar.Header{
		Name:     name,
		Linkname: target,
		Mode:     0777,
		ModTime:  now,
		Typeflag: tar.TypeSymlink,
	}
	if err := out.WriteHeader(&h); err != nil {
		return fmt.Errorf("cannot write header of %s to data.tar.gz: %v", h.Name, err)
	}
	return nil
}

func (b *Build) createControl(now time.Time, instSize int64, md5sums []byte) (controlTarGz []byte, err error) {
	buf := &bytes.Buffer{}
	compress := gzip.NewWriter(buf)
	tarball := tar.NewWriter(compress)

	body := []byte(fmt.Sprintf(
		controlFile,
		b.AppName,     // Package
		b.Version,     // Version
		debArch(),     // Architecture
		instSize/1024, // Installed-Size
		b.AppName,     // Description
	))

	hdr := tar.Header{
		Name:     "control",
		Size:     int64(len(body)),
		Mode:     0644,
		ModTime:  now,
		Typeflag: tar.TypeReg,
	}
	if err := tarball.WriteHeader(&hdr); err != nil {
		return nil, fmt.Errorf("cannot write header of control file to control.tar.gz: %v", err)
	}
	if _, err := tarball.Write(body); err != nil {
		return nil, fmt.Errorf("cannot write control file to control.tar.gz: %v", err)
	}

	hdr = tar.Header{
		Name:     "md5sums",
		Size:     int64(len(md5sums)),
		Mode:     0644,
		ModTime:  now,
		Typeflag: tar.TypeReg,
	}
	if err := tarball.WriteHeader(&hdr); err != nil {
		return nil, fmt.Errorf("cannot write header of md5sums file to control.tar.gz: %v", err)
	}
	if _, err := tarball.Write(md5sums); err != nil {
		return nil, fmt.Errorf("cannot write md5sums file to control.tar.gz: %v", err)
	}

	if err := tarball.Close(); err != nil {
		return nil, fmt.Errorf("closing control.tar.gz: %v", err)
	}
	if err := compress.Close(); err != nil {
		return nil, fmt.Errorf("closing control.tar.gz: %v", err)
	}
	return buf.Bytes(), nil
}
<commit_msg>build/deb: working version of a deb file<commit_after>package build

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"fmt"
	"go/build"
	"io"
	"os"
	"strings"
	"time"

	"github.com/blakesmith/ar"
)

const (
	installPrefix = "./opt/kite/"

	controlFile = `Package: %s
Version: %s
Architecture: %s
Maintainer: Koding Developers <hello@koding.com>
Installed-Size: %d
Section: devel
Priority: extra
Description: %s Kite
`
)

func (b *Build) Linux() error {
	debFile := b.Output + ".deb"
	tarFile := b.Output + ".tar.gz"

	deb, err := os.Create(debFile + ".inprogress")
	if err != nil {
		return fmt.Errorf("cannot create deb: %v", err)
	}
	defer deb.Close()

	// create first tar file
	if err := b.TarGzFile(); err != nil {
		return err
	}

	tf, err := os.Open(tarFile)
	if err != nil {
		return err
	}
	defer tf.Close()

	if err := b.createDeb(tf, deb); err != nil {
		return err
	}

	if err := os.Rename(debFile+".inprogress", debFile); err != nil {
		return err
	}

	fmt.Println("package", debFile, "ready")
	fmt.Println("tarfile", tarFile, "ready")
	return nil
}

func debArch() string {
	arch := build.Default.GOARCH
	if arch == "386" {
		return "i386"
	}
	return arch
}

func (b *Build) createDeb(tarball io.Reader, deb io.Writer) error {
	now := time.Now()
	dataTarGz, md5sums, instSize, err := b.translateTarball(now, tarball)
	if err != nil {
		return err
	}

	controlTarGz, err := b.createControl(now, instSize, md5sums)
	if err != nil {
		return err
	}
	w := ar.NewWriter(deb)
	if err := w.WriteGlobalHeader(); err != nil {
		return fmt.Errorf("cannot write ar header to deb file: %v", err)
	}

	if err := addArFile(now, w, "debian-binary", []byte("2.0\n")); err != nil {
		return fmt.Errorf("cannot pack debian-binary: %v", err)
	}
	if err := addArFile(now, w, "control.tar.gz", controlTarGz); err != nil {
		return fmt.Errorf("cannot add control.tar.gz to deb: %v", err)
	}
	if err := addArFile(now, w, "data.tar.gz", dataTarGz); err != nil {
		return fmt.Errorf("cannot add data.tar.gz to deb: %v", err)
	}
	return nil
}

func addArFile(now time.Time, w *ar.Writer, name string, body []byte) error {
	hdr := ar.Header{
		Name:    name,
		Size:    int64(len(body)),
		Mode:    0644,
		ModTime: now,
	}
	if err := w.WriteHeader(&hdr); err != nil {
		return fmt.Errorf("cannot write file header: %v", err)
	}
	_, err := w.Write(body)
	return err
}

func (b *Build) translateTarball(now time.Time, tarball io.Reader) (dataTarGz, md5sums []byte, instSize int64, err error) {
	buf := &bytes.Buffer{}
	compress := gzip.NewWriter(buf)
	out := tar.NewWriter(compress)

	md5buf := &bytes.Buffer{}
	md5tmp := make([]byte, 0, md5.Size)

	uncompress, err := gzip.NewReader(tarball)
	if err != nil {
		return nil, nil, 0, fmt.Errorf("cannot uncompress tarball: %v", err)
	}

	in := tar.NewReader(uncompress)
	for {
		h, err := in.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, nil, 0, fmt.Errorf("cannot read tarball: %v", err)
		}

		instSize += h.Size
		h.Name = strings.TrimLeft(h.Name, "./")

		ha := tar.Header{
			Name:     installPrefix,
			Mode:     0755,
			ModTime:  h.ModTime,
			Typeflag: tar.TypeDir,
		}

		if err := out.WriteHeader(&ha); err != nil {
			return nil, nil, 0, fmt.Errorf("cannot write header of %s to data.tar.gz: %v", h.Name, err)
		}

		h.Name = installPrefix + h.Name
		if h.Typeflag == tar.TypeDir && !strings.HasSuffix(h.Name, "/") {
			h.Name += "/"
		}

		if err := out.WriteHeader(h); err != nil {
			return nil, nil, 0, fmt.Errorf("cannot write header of %s to data.tar.gz: %v", h.Name, err)
		}

		// fmt.Println("tar: packing", h.Name[len(installPrefix):])
		if h.Typeflag == tar.TypeDir {
			continue
		}

		digest := md5.New()
		if _, err := io.Copy(out, io.TeeReader(in, digest)); err != nil {
			return nil, nil, 0, err
		}

		fmt.Fprintf(md5buf, "%x  %s\n", digest.Sum(md5tmp), h.Name[2:])
	}

	if err := out.Close(); err != nil {
		return nil, nil, 0, err
	}

	if err := compress.Close(); err != nil {
		return nil, nil, 0, err
	}

	return buf.Bytes(), md5buf.Bytes(), instSize, nil
}

func addTarSymlink(now time.Time, out *tar.Writer, name, target string) error {
	h := tar.Header{
		Name:     name,
		Linkname: target,
		Mode:     0777,
		ModTime:  now,
		Typeflag: tar.TypeSymlink,
	}
	if err := out.WriteHeader(&h); err != nil {
		return fmt.Errorf("cannot write header of %s to data.tar.gz: %v", h.Name, err)
	}
	return nil
}

func (b *Build) createControl(now time.Time, instSize int64, md5sums []byte) (controlTarGz []byte, err error) {
	buf := &bytes.Buffer{}
	compress := gzip.NewWriter(buf)
	tarball := tar.NewWriter(compress)

	body := []byte(fmt.Sprintf(
		controlFile,
		b.AppName,     // Package
		b.Version,     // Version
		debArch(),     // Architecture
		instSize/1024, // Installed-Size
		b.AppName,     // Description
	))

	hdr := tar.Header{
		Name:     "control",
		Size:     int64(len(body)),
		Mode:     0644,
		ModTime:  now,
		Typeflag: tar.TypeReg,
	}
	if err := tarball.WriteHeader(&hdr); err != nil {
		return nil, fmt.Errorf("cannot write header of control file to control.tar.gz: %v", err)
	}
	if _, err := tarball.Write(body); err != nil {
		return nil, fmt.Errorf("cannot write control file to control.tar.gz: %v", err)
	}

	hdr = tar.Header{
		Name:     "md5sums",
		Size:     int64(len(md5sums)),
		Mode:     0644,
		ModTime:  now,
		Typeflag: tar.TypeReg,
	}
	if err := tarball.WriteHeader(&hdr); err != nil {
		return nil, fmt.Errorf("cannot write header of md5sums file to control.tar.gz: %v", err)
	}
	if _, err := tarball.Write(md5sums); err != nil {
		return nil, fmt.Errorf("cannot write md5sums file to control.tar.gz: %v", err)
	}

	if err := tarball.Close(); err != nil {
		return nil, fmt.Errorf("closing control.tar.gz: %v", err)
	}
	if err := compress.Close(); err != nil {
		return nil, fmt.Errorf("closing control.tar.gz: %v", err)
	}
	return buf.Bytes(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"path/filepath"
	"strings"

	"github.com/mitchellh/cli"
)

func MountCommandFactory() (cli.Command, error) {
	return &MountCommand{}, nil
}

type MountCommand struct{}

func (c *MountCommand) Run(args []string) int {
	if len(args) < 2 {
		log.Fatal(c.Help())
	}

	k, err := CreateKlientClient(NewKlientOptions())
	if err != nil {
		log.Fatal(err)
	}

	if err := k.Dial(); err != nil {
		log.Fatal(err)
	}

	var localPath = args[1]

	// use absolute path unless empty
	if strings.TrimSpace(args[1]) != "" {
		absoluteLocalPath, err := filepath.Abs(args[1])
		if err == nil {
			localPath = absoluteLocalPath
		}
	}

	mountRequest := struct {
		Name       string `json:"name"`
		LocalPath  string `json:"localPath"`
		RemotePath string `json:"remotePath"`
	}{
		Name:      args[0],
		LocalPath: localPath,
	}

	if len(args) > 2 {
		mountRequest.RemotePath = args[2]
	}

	resp, err := k.Tell("remote.mountFolder", mountRequest)
	if err != nil {
		log.Fatal(err)
	}

	var warning string
	if err := resp.Unmarshal(&warning); err != nil {
		return 0
	}

	if len(warning) > 0 {
		fmt.Printf("Warning: %s", warning)
	}

	return 0
}

func (*MountCommand) Help() string {
	helpText := `
Usage: %s mount <machine name> </path/to/local/folder> </path/to/remote/folder>

    Mount a remote folder from the given remote machine, to the specified
    local folder.
`
	return fmt.Sprintf(helpText, Name)
}

func (*MountCommand) Synopsis() string {
	return fmt.Sprintf("Mount a remote folder to a local folder")
}
<commit_msg>klientctl: be explict with return responses<commit_after>package main

import (
	"fmt"
	"log"
	"path/filepath"
	"strings"

	"github.com/mitchellh/cli"
)

func MountCommandFactory() (cli.Command, error) {
	return &MountCommand{}, nil
}

type MountCommand struct{}

func (c *MountCommand) Run(args []string) int {
	if len(args) < 2 {
		log.Fatal(c.Help())
	}

	k, err := CreateKlientClient(NewKlientOptions())
	if err != nil {
		log.Fatal(err)
	}

	if err := k.Dial(); err != nil {
		log.Fatal(err)
	}

	var localPath = args[1]

	// use absolute path unless empty
	if strings.TrimSpace(args[1]) != "" {
		absoluteLocalPath, err := filepath.Abs(args[1])
		if err == nil {
			localPath = absoluteLocalPath
		}
	}

	mountRequest := struct {
		Name       string `json:"name"`
		LocalPath  string `json:"localPath"`
		RemotePath string `json:"remotePath"`
	}{
		Name:      args[0],
		LocalPath: localPath,
	}

	if len(args) > 2 {
		mountRequest.RemotePath = args[2]
	}

	resp, err := k.Tell("remote.mountFolder", mountRequest)
	if err != nil {
		log.Fatal(err)
	}

	if resp == nil {
		return 0
	}

	var warning string
	if err := resp.Unmarshal(&warning); err != nil {
		return 0
	}

	if len(warning) > 0 {
		fmt.Printf("Warning: %s", warning)
	}

	return 0
}

func (*MountCommand) Help() string {
	helpText := `
Usage: %s mount <machine name> </path/to/local/folder> </path/to/remote/folder>

    Mount a remote folder from the given remote machine, to the specified
    local folder.
`
	return fmt.Sprintf(helpText, Name)
}

func (*MountCommand) Synopsis() string {
	return fmt.Sprintf("Mount a remote folder to a local folder")
}
<|endoftext|>