<commit_before>package models

import (
	kodingmodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"math/rand"
	"strconv"
	"time"

	"github.com/koding/bongo"
	. "github.com/smartystreets/goconvey/convey"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

func CreateChannelWithParticipants() (*Channel, []*Account) {
	account1 := CreateAccountWithTest()
	account2 := CreateAccountWithTest()
	account3 := CreateAccountWithTest()
	accounts := []*Account{account1, account2, account3}

	channel := CreateChannelWithTest(account1.Id)
	AddParticipantsWithTest(channel.Id, account1.Id, account2.Id, account3.Id)

	return channel, accounts
}

func CreateChannelWithTest(accountId int64) *Channel {
	// create and account instance
	channel := NewChannel()
	channel.CreatorId = accountId

	err := channel.Create()
	So(err, ShouldBeNil)

	return channel
}

// CreateTypedChannelWithTest creates a channel specific to a group with the
// given type constant
func CreateTypedChannelWithTest(accountId int64, typeConstant string) *Channel {
	// create and account instance
	channel := NewChannel()
	channel.Name = RandomName()
	// there is a check for group channels for ensuring that there will be only
	// one group channel at a time, override that
	channel.GroupName = RandomName()
	channel.TypeConstant = typeConstant
	channel.CreatorId = accountId

	err := channel.Create()
	So(err, ShouldBeNil)

	return channel
}

func CreateTypedGroupedChannelWithTest(accountId int64, typeConstant, groupName string) *Channel {
	// create and account instance
	channel := NewChannel()
	channel.Name = RandomName()
	// there is a check for group channels for unsuring that there will be only
	// one group channel at a time, override that
	channel.GroupName = groupName
	channel.TypeConstant = typeConstant
	channel.CreatorId = accountId

	err := channel.Create()
	So(err, ShouldBeNil)

	return channel
}

func CreateTypedPublicChannelWithTest(accountId int64, typeConstant string) *Channel {
	// create and account instance
	channel := NewChannel()
	channel.Name = RandomName()
	// there is a check for group channels for unsuring that there will be only
	// one group channel at a time, override that
	channel.GroupName = RandomName()
	channel.TypeConstant = typeConstant
	channel.CreatorId = accountId
	channel.PrivacyConstant = Channel_PRIVACY_PUBLIC

	err := channel.Create()
	So(err, ShouldBeNil)

	return channel
}

func CreateMessage(channelId, accountId int64, typeConstant string) *ChannelMessage {
	return CreateMessageWithBody(channelId, accountId, typeConstant, "testing message")
}

func CreateMessageWithBody(channelId, accountId int64, typeConstant, body string) *ChannelMessage {
	cm := NewChannelMessage()

	cm.AccountId = accountId
	// set channel id
	cm.InitialChannelId = channelId
	cm.TypeConstant = typeConstant
	// set body
	cm.Body = body

	err := cm.Create()
	So(err, ShouldBeNil)

	c := NewChannel()
	c.Id = channelId
	_, err = c.EnsureMessage(cm, false)
	So(err, ShouldBeNil)

	return cm
}

func CreateTrollMessage(channelId, accountId int64, typeConstant string) *ChannelMessage {
	cm := NewChannelMessage()

	cm.AccountId = accountId
	// set channel id
	cm.InitialChannelId = channelId
	cm.TypeConstant = typeConstant
	// set body
	cm.Body = "testing message"
	cm.MetaBits = Troll

	err := cm.Create()
	So(err, ShouldBeNil)

	return cm
}

func AddParticipantsWithTest(channelId int64, accountIds ...int64) {

	for _, accountId := range accountIds {
		participant := NewChannelParticipant()
		participant.ChannelId = channelId
		participant.AccountId = accountId

		err := participant.Create()
		So(err, ShouldBeNil)
	}
}

func createAccount() (*Account, error) {
	// create and account instance
	author := NewAccount()

	// create a fake mongo id
	oldId := bson.NewObjectId()
	// assign it to our test user
	author.OldId = oldId.Hex()

	// seed the random data generator
	rand.Seed(time.Now().UnixNano())

	author.Nick = "malitest" + strconv.Itoa(rand.Intn(10e9))

	if err := author.Create(); err != nil {
		return nil, err
	}

	return author, nil
}

func CreateAccountWithTest() *Account {
	account, err := createAccount()
	So(err, ShouldBeNil)
	So(account, ShouldNotBeNil)
	So(account.Id, ShouldNotEqual, 0)
	return account
}

func CreateMessageWithTest() *ChannelMessage {
	account := CreateAccountWithTest()
	channel := CreateChannelWithTest(account.Id)

	cm := NewChannelMessage()
	cm.AccountId = account.Id
	cm.InitialChannelId = channel.Id
	cm.Body = "5five"
	return cm
}

func CreateAccountInBothDbs() (*Account, error) {
	return CreateAccountInBothDbsWithNick(bson.NewObjectId().Hex())
}

func CreateAccountInBothDbsWithCheck() *Account {
	acc, err := CreateAccountInBothDbsWithNick(bson.NewObjectId().Hex())
	So(err, ShouldBeNil)
	So(acc, ShouldNotBeNil)
	return acc
}

func CreateRandomGroupDataWithChecks() (*Account, *Channel, string) {
	groupName := RandomGroupName()

	account := CreateAccountInBothDbsWithCheck()

	groupChannel := CreateTypedGroupedChannelWithTest(
		account.Id,
		Channel_TYPE_GROUP,
		groupName,
	)

	_, err := groupChannel.AddParticipant(account.Id)
	So(err, ShouldBeNil)

	_, err = CreateGroupInMongo(groupName, groupChannel.Id)
	So(err, ShouldBeNil)

	return account, groupChannel, groupName
}

func CreateGroupInMongo(groupName string, socialapiId int64) (*kodingmodels.Group, error) {
	g := &kodingmodels.Group{
		Id:                             bson.NewObjectId(),
		Body:                           groupName,
		Title:                          groupName,
		Slug:                           groupName,
		Privacy:                        "private",
		Visibility:                     "hidden",
		SocialApiChannelId:             strconv.FormatInt(socialapiId, 10),
		SocialApiAnnouncementChannelId: strconv.FormatInt(socialapiId, 10),
	}

	return g, modelhelper.CreateGroup(g)
}

func CreateAccountInBothDbsWithNick(nick string) (*Account, error) {
	accId := bson.NewObjectId()
	accHex := nick

	oldAcc, err := modelhelper.GetAccount(nick)
	if err == mgo.ErrNotFound {

		oldAcc = &kodingmodels.Account{
			Id: accId,
			Profile: struct {
				Nickname  string `bson:"nickname" json:"nickname"`
				FirstName string `bson:"firstName" json:"firstName"`
				LastName  string `bson:"lastName" json:"lastName"`
				Hash      string `bson:"hash" json:"hash"`
			}{
				Nickname: nick,
			},
		}

		err := modelhelper.CreateAccount(oldAcc)
		if err != nil {
			return nil, err
		}
	}

	_, err = modelhelper.GetUser(nick)
	if err == mgo.ErrNotFound {
		oldUser := &kodingmodels.User{
			ObjectId:       bson.NewObjectId(),
			Password:       accHex,
			Salt:           accHex,
			Name:           nick,
			Email:          accHex + "@koding.com",
			Status:         "confirmed",
			EmailFrequency: &kodingmodels.EmailFrequency{},
		}

		err = modelhelper.CreateUser(oldUser)
		if err != nil {
			return nil, err
		}
	}

	a := NewAccount()
	a.Nick = nick
	a.OldId = accId.Hex()

	if err := a.ByNick(nick); err == bongo.RecordNotFound {
		if err := a.Create(); err != nil {
			return nil, err
		}
	}

	if oldAcc.SocialApiId != strconv.FormatInt(a.Id, 10) {
		s := modelhelper.Selector{"_id": oldAcc.Id}
		o := modelhelper.Selector{"$set": modelhelper.Selector{
			"socialApiId": strconv.FormatInt(a.Id, 10),
		}}

		if err := modelhelper.UpdateAccount(s, o); err != nil {
			return nil, err
		}
	}

	return a, nil
}

func UpdateUsernameInBothDbs(currentUsername, newUsername string) error {
	err := modelhelper.UpdateUser(
		bson.M{"username": currentUsername},
		bson.M{"username": newUsername},
	)

	if err != nil {
		return err
	}

	err = modelhelper.UpdateAccount(
		modelhelper.Selector{"profile.nickname": currentUsername},
		modelhelper.Selector{"$set": modelhelper.Selector{"profile.nickname": newUsername}},
	)

	if err != nil {
		return err
	}

	acc := NewAccount()
	if err := acc.ByNick(currentUsername); err != nil {
		return err
	}

	acc.Nick = newUsername

	return acc.Update()
}
<commit_msg>socialapi/models: fix user creation for multi test runs<commit_after>package models

import (
	kodingmodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"math/rand"
	"strconv"
	"time"

	"github.com/koding/bongo"
	. "github.com/smartystreets/goconvey/convey"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

func CreateChannelWithParticipants() (*Channel, []*Account) {
	account1 := CreateAccountWithTest()
	account2 := CreateAccountWithTest()
	account3 := CreateAccountWithTest()
	accounts := []*Account{account1, account2, account3}

	channel := CreateChannelWithTest(account1.Id)
	AddParticipantsWithTest(channel.Id, account1.Id, account2.Id, account3.Id)

	return channel, accounts
}

func CreateChannelWithTest(accountId int64) *Channel {
	// create and account instance
	channel := NewChannel()
	channel.CreatorId = accountId

	err := channel.Create()
	So(err, ShouldBeNil)

	return channel
}

// CreateTypedChannelWithTest creates a channel specific to a group with the
// given type constant
func CreateTypedChannelWithTest(accountId int64, typeConstant string) *Channel {
	// create and account instance
	channel := NewChannel()
	channel.Name = RandomName()
	// there is a check for group channels for ensuring that there will be only
	// one group channel at a time, override that
	channel.GroupName = RandomName()
	channel.TypeConstant = typeConstant
	channel.CreatorId = accountId

	err := channel.Create()
	So(err, ShouldBeNil)

	return channel
}

func CreateTypedGroupedChannelWithTest(accountId int64, typeConstant, groupName string) *Channel {
	// create and account instance
	channel := NewChannel()
	channel.Name = RandomName()
	// there is a check for group channels for unsuring that there will be only
	// one group channel at a time, override that
	channel.GroupName = groupName
	channel.TypeConstant = typeConstant
	channel.CreatorId = accountId

	err := channel.Create()
	So(err, ShouldBeNil)

	return channel
}

func CreateTypedPublicChannelWithTest(accountId int64, typeConstant string) *Channel {
	// create and account instance
	channel := NewChannel()
	channel.Name = RandomName()
	// there is a check for group channels for unsuring that there will be only
	// one group channel at a time, override that
	channel.GroupName = RandomName()
	channel.TypeConstant = typeConstant
	channel.CreatorId = accountId
	channel.PrivacyConstant = Channel_PRIVACY_PUBLIC

	err := channel.Create()
	So(err, ShouldBeNil)

	return channel
}

func CreateMessage(channelId, accountId int64, typeConstant string) *ChannelMessage {
	return CreateMessageWithBody(channelId, accountId, typeConstant, "testing message")
}

func CreateMessageWithBody(channelId, accountId int64, typeConstant, body string) *ChannelMessage {
	cm := NewChannelMessage()

	cm.AccountId = accountId
	// set channel id
	cm.InitialChannelId = channelId
	cm.TypeConstant = typeConstant
	// set body
	cm.Body = body

	err := cm.Create()
	So(err, ShouldBeNil)

	c := NewChannel()
	c.Id = channelId
	_, err = c.EnsureMessage(cm, false)
	So(err, ShouldBeNil)

	return cm
}

func CreateTrollMessage(channelId, accountId int64, typeConstant string) *ChannelMessage {
	cm := NewChannelMessage()

	cm.AccountId = accountId
	// set channel id
	cm.InitialChannelId = channelId
	cm.TypeConstant = typeConstant
	// set body
	cm.Body = "testing message"
	cm.MetaBits = Troll

	err := cm.Create()
	So(err, ShouldBeNil)

	return cm
}

func AddParticipantsWithTest(channelId int64, accountIds ...int64) {

	for _, accountId := range accountIds {
		participant := NewChannelParticipant()
		participant.ChannelId = channelId
		participant.AccountId = accountId

		err := participant.Create()
		So(err, ShouldBeNil)
	}
}

func createAccount() (*Account, error) {
	// create and account instance
	author := NewAccount()

	// create a fake mongo id
	oldId := bson.NewObjectId()
	// assign it to our test user
	author.OldId = oldId.Hex()

	// seed the random data generator
	rand.Seed(time.Now().UnixNano())

	author.Nick = "malitest" + strconv.Itoa(rand.Intn(10e9))

	if err := author.Create(); err != nil {
		return nil, err
	}

	return author, nil
}

func CreateAccountWithTest() *Account {
	account, err := createAccount()
	So(err, ShouldBeNil)
	So(account, ShouldNotBeNil)
	So(account.Id, ShouldNotEqual, 0)
	return account
}

func CreateMessageWithTest() *ChannelMessage {
	account := CreateAccountWithTest()
	channel := CreateChannelWithTest(account.Id)

	cm := NewChannelMessage()
	cm.AccountId = account.Id
	cm.InitialChannelId = channel.Id
	cm.Body = "5five"
	return cm
}

func CreateAccountInBothDbs() (*Account, error) {
	return CreateAccountInBothDbsWithNick(bson.NewObjectId().Hex())
}

func CreateAccountInBothDbsWithCheck() *Account {
	acc, err := CreateAccountInBothDbsWithNick(bson.NewObjectId().Hex())
	So(err, ShouldBeNil)
	So(acc, ShouldNotBeNil)
	return acc
}

func CreateRandomGroupDataWithChecks() (*Account, *Channel, string) {
	groupName := RandomGroupName()

	account := CreateAccountInBothDbsWithCheck()

	groupChannel := CreateTypedGroupedChannelWithTest(
		account.Id,
		Channel_TYPE_GROUP,
		groupName,
	)

	_, err := groupChannel.AddParticipant(account.Id)
	So(err, ShouldBeNil)

	_, err = CreateGroupInMongo(groupName, groupChannel.Id)
	So(err, ShouldBeNil)

	return account, groupChannel, groupName
}

func CreateGroupInMongo(groupName string, socialapiId int64) (*kodingmodels.Group, error) {
	g := &kodingmodels.Group{
		Id:                             bson.NewObjectId(),
		Body:                           groupName,
		Title:                          groupName,
		Slug:                           groupName,
		Privacy:                        "private",
		Visibility:                     "hidden",
		SocialApiChannelId:             strconv.FormatInt(socialapiId, 10),
		SocialApiAnnouncementChannelId: strconv.FormatInt(socialapiId, 10),
	}

	return g, modelhelper.CreateGroup(g)
}

func CreateAccountInBothDbsWithNick(nick string) (*Account, error) {
	accId := bson.NewObjectId()
	accHex := nick

	oldAcc, err := modelhelper.GetAccount(nick)
	if err == mgo.ErrNotFound {

		oldAcc = &kodingmodels.Account{
			Id: accId,
			Profile: struct {
				Nickname  string `bson:"nickname" json:"nickname"`
				FirstName string `bson:"firstName" json:"firstName"`
				LastName  string `bson:"lastName" json:"lastName"`
				Hash      string `bson:"hash" json:"hash"`
			}{
				Nickname: nick,
			},
		}

		err := modelhelper.CreateAccount(oldAcc)
		if err != nil {
			return nil, err
		}
	}

	_, err = modelhelper.GetUser(nick)
	if err == mgo.ErrNotFound {
		oldUser := &kodingmodels.User{
			ObjectId:       bson.NewObjectId(),
			Password:       accHex,
			Salt:           accHex,
			Name:           nick,
			Email:          accHex + "@koding.com",
			SanitizedEmail: accHex + "@koding.com",
			Status:         "confirmed",
			EmailFrequency: &kodingmodels.EmailFrequency{},
		}

		err = modelhelper.CreateUser(oldUser)
		if err != nil {
			return nil, err
		}
	}

	a := NewAccount()
	a.Nick = nick
	a.OldId = accId.Hex()

	if err := a.ByNick(nick); err == bongo.RecordNotFound {
		if err := a.Create(); err != nil {
			return nil, err
		}
	}

	if oldAcc.SocialApiId != strconv.FormatInt(a.Id, 10) {
		s := modelhelper.Selector{"_id": oldAcc.Id}
		o := modelhelper.Selector{"$set": modelhelper.Selector{
			"socialApiId": strconv.FormatInt(a.Id, 10),
		}}

		if err := modelhelper.UpdateAccount(s, o); err != nil {
			return nil, err
		}
	}

	return a, nil
}

func UpdateUsernameInBothDbs(currentUsername, newUsername string) error {
	err := modelhelper.UpdateUser(
		bson.M{"username": currentUsername},
		bson.M{"username": newUsername},
	)

	if err != nil {
		return err
	}

	err = modelhelper.UpdateAccount(
		modelhelper.Selector{"profile.nickname": currentUsername},
		modelhelper.Selector{"$set": modelhelper.Selector{"profile.nickname": newUsername}},
	)

	if err != nil {
		return err
	}

	acc := NewAccount()
	if err := acc.ByNick(currentUsername); err != nil {
		return err
	}

	acc.Nick = newUsername

	return acc.Update()
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mysqlctl

import (
	"fmt"
	"sync"

	log "github.com/golang/glog"
	"golang.org/x/net/context"

	"github.com/youtube/vitess/go/mysql"
	"github.com/youtube/vitess/go/pools"
	"github.com/youtube/vitess/go/vt/dbconfigs"
)

var (
	// ErrBinlogUnavailable is returned by this library when we
	// cannot find a suitable binlog to satisfy the request.
	ErrBinlogUnavailable = fmt.Errorf("cannot find relevant binlogs on this server")
)

// SlaveConnection represents a connection to mysqld that pretends to be a slave
// connecting for replication. Each such connection must identify itself to
// mysqld with a server ID that is unique both among other SlaveConnections and
// among actual slaves in the topology.
type SlaveConnection struct {
	*mysql.Conn
	mysqld  *Mysqld
	slaveID uint32
	cancel  context.CancelFunc
	wg      sync.WaitGroup
}

// NewSlaveConnection creates a new slave connection to the mysqld instance.
// It uses a pools.IDPool to ensure that the server IDs used to connect are
// unique within this process. This is done with the assumptions that:
//
// 1) No other processes are making fake slave connections to our mysqld.
// 2) No real slave servers will have IDs in the range 1-N where N is the peak
//    number of concurrent fake slave connections we will ever make.
func (mysqld *Mysqld) NewSlaveConnection() (*SlaveConnection, error) {
	conn, err := mysqld.connectForReplication()
	if err != nil {
		return nil, err
	}

	sc := &SlaveConnection{
		Conn:    conn,
		mysqld:  mysqld,
		slaveID: slaveIDPool.Get(),
	}
	log.Infof("new slave connection: slaveID=%d", sc.slaveID)
	return sc, nil
}

// connectForReplication create a MySQL connection ready to use for replication.
func (mysqld *Mysqld) connectForReplication() (*mysql.Conn, error) {
	params, err := dbconfigs.WithCredentials(&mysqld.dbcfgs.Dba)
	if err != nil {
		return nil, err
	}

	ctx := context.Background()
	conn, err := mysql.Connect(ctx, &params)
	if err != nil {
		return nil, err
	}

	// Tell the server that we understand the format of events
	// that will be used if binlog_checksum is enabled on the server.
	if _, err := conn.ExecuteFetch("SET @master_binlog_checksum=@@global.binlog_checksum", 0, false); err != nil {
		return nil, fmt.Errorf("failed to set @master_binlog_checksum=@@global.binlog_checksum: %v", err)
	}

	return conn, nil
}

// slaveIDPool is the IDPool for server IDs used to connect as a slave.
var slaveIDPool = pools.NewIDPool()

// StartBinlogDumpFromCurrent requests a replication binlog dump from
// the current position.
func (sc *SlaveConnection) StartBinlogDumpFromCurrent(ctx context.Context) (mysql.Position, <-chan mysql.BinlogEvent, error) {
	ctx, sc.cancel = context.WithCancel(ctx)

	masterPosition, err := sc.mysqld.MasterPosition()
	if err != nil {
		return mysql.Position{}, nil, fmt.Errorf("failed to get master position: %v", err)
	}

	c, err := sc.StartBinlogDumpFromPosition(ctx, masterPosition)
	return masterPosition, c, err
}

// StartBinlogDumpFromPosition requests a replication binlog dump from
// the master mysqld at the given Position and then sends binlog
// events to the provided channel.
// The stream will continue in the background, waiting for new events if
// necessary, until the connection is closed, either by the master or
// by canceling the context.
//
// Note the context is valid and used until eventChan is closed.
func (sc *SlaveConnection) StartBinlogDumpFromPosition(ctx context.Context, startPos mysql.Position) (<-chan mysql.BinlogEvent, error) {
	ctx, sc.cancel = context.WithCancel(ctx)

	log.Infof("sending binlog dump command: startPos=%v, slaveID=%v", startPos, sc.slaveID)
	if err := sc.SendBinlogDumpCommand(sc.slaveID, startPos); err != nil {
		log.Errorf("couldn't send binlog dump command: %v", err)
		return nil, err
	}

	// Read the first packet to see if it's an error response to our dump command.
	buf, err := sc.Conn.ReadPacket()
	if err != nil {
		log.Errorf("couldn't start binlog dump: %v", err)
		return nil, err
	}

	// FIXME(alainjobart) I think we can use a buffered channel for better performance.
	eventChan := make(chan mysql.BinlogEvent)

	// Start reading events.
	sc.wg.Add(1)
	go func() {
		defer func() {
			close(eventChan)
			sc.wg.Done()
		}()
		for {
			// Handle EOF and error case.
			switch buf[0] {
			case mysql.EOFPacket:
				log.Infof("received EOF packet in binlog dump: %#v", buf)
				return
			case mysql.ErrPacket:
				err := mysql.ParseErrorPacket(buf)
				log.Infof("received error packet in binlog dump: %v", err)
				return
			}

			select {
			// Skip the first byte because it's only used for signaling EOF / error.
			case eventChan <- sc.Conn.MakeBinlogEvent(buf[1:]):
			case <-ctx.Done():
				return
			}

			buf, err = sc.Conn.ReadPacket()
			if err != nil {
				if sqlErr, ok := err.(*mysql.SQLError); ok && sqlErr.Number() == mysql.CRServerLost {
					// CRServerLost = Lost connection to MySQL server during query
					// This is not necessarily an error. It could just be that we closed
					// the connection from outside.
					log.Infof("connection closed during binlog stream (possibly intentional): %v", err)
					return
				}
				log.Errorf("read error while streaming binlog events: %v", err)
				return
			}
		}
	}()

	return eventChan, nil
}

// StartBinlogDumpFromBinlogBeforeTimestamp requests a replication
// binlog dump from the master mysqld starting with a file that has
// timestamps smaller than the provided timestamp, and then sends
// binlog events to the provided channel.
//
// The startup phase will list all the binary logs, and find the one
// that has events starting strictly before the provided timestamp. It
// will then start from there, and stream all events. It is the
// responsability of the calling site to filter the events more.
//
// MySQL 5.6+ note: we need to do it that way because of the way the
// GTIDSet works. In the previous two streaming functions, we pass in
// the full GTIDSet (that has the list of all transactions seen in
// the replication stream). In this case, we don't know it, all we
// have is the binlog file names. We depend on parsing the first
// PREVIOUS_GTIDS_EVENT event in the logs to get it. So we need the
// caller to parse that event, and it can't be skipped because its
// timestamp is lower. Then, for each subsequent event, the caller
// also needs to add the event GTID to its GTIDSet. Otherwise it won't
// be correct ever. So the caller really needs to build up its GTIDSet
// along the entire file, not just for events whose timestamp is in a
// given range.
//
// The stream will continue in the background, waiting for new events if
// necessary, until the connection is closed, either by the master or
// by canceling the context.
//
// Note the context is valid and used until eventChan is closed.
func (sc *SlaveConnection) StartBinlogDumpFromBinlogBeforeTimestamp(ctx context.Context, timestamp int64) (<-chan mysql.BinlogEvent, error) {
	ctx, sc.cancel = context.WithCancel(ctx)

	// List the binlogs.
	binlogs, err := sc.Conn.ExecuteFetch("SHOW BINARY LOGS", 1000, false)
	if err != nil {
		return nil, fmt.Errorf("failed to SHOW BINARY LOGS: %v", err)
	}

	// Start with the most recent binlog file until we find the right event.
	var binlogIndex int
	var event mysql.BinlogEvent
	for binlogIndex = len(binlogs.Rows) - 1; binlogIndex >= 0; binlogIndex-- {
		// Exit the loop early if context is canceled.
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
		}

		// Start dumping the logs. The position is '4' to skip the
		// Binlog File Header. See this page for more info:
		// https://dev.mysql.com/doc/internals/en/binlog-file.html
		binlog := binlogs.Rows[binlogIndex][0].ToString()
		if err := sc.Conn.WriteComBinlogDump(sc.slaveID, binlog, 4, 0); err != nil {
			return nil, fmt.Errorf("failed to send the ComBinlogDump command: %v", err)
		}

		// Get the first event to get its timestamp. We skip
		// events that don't have timestamps (although it seems
		// most do anyway).
		for {
			buf, err := sc.Conn.ReadPacket()
			if err != nil {
				return nil, fmt.Errorf("couldn't start binlog dump of binlog %v: %v", binlog, err)
			}

			// Handle EOF and error case.
			switch buf[0] {
			case mysql.EOFPacket:
				return nil, fmt.Errorf("received EOF packet for first packet of binlog %v", binlog)
			case mysql.ErrPacket:
				err := mysql.ParseErrorPacket(buf)
				return nil, fmt.Errorf("received error packet for first packet of binlog %v", err)
			}

			// Parse the full event.
			event = sc.Conn.MakeBinlogEvent(buf[1:])
			if !event.IsValid() {
				return nil, fmt.Errorf("first event from binlog %v is not valid", binlog)
			}
			if event.Timestamp() > 0 {
				// We found the first event with a
				// valid timestamp.
				break
			}
		}
		if int64(event.Timestamp()) < timestamp {
			// The first event in this binlog has a smaller
			// timestamp than what we need, we found a good
			// starting point.
			break
		}

		// The timestamp is higher, we need to try the older files.
		// Close and re-open our connection.
		sc.Conn.Close()
		conn, err := sc.mysqld.connectForReplication()
		if err != nil {
			return nil, err
		}
		sc.Conn = conn
	}
	if binlogIndex == -1 {
		// We haven't found a suitable binlog
		log.Errorf("couldn't find an old enough binlog to match timestamp >= %v (looked at %v files)", timestamp, len(binlogs.Rows))
		return nil, ErrBinlogUnavailable
	}

	// Now just loop sending and reading events.
	// FIXME(alainjobart) I think we can use a buffered channel for better performance.
	eventChan := make(chan mysql.BinlogEvent)

	// Start reading events.
	sc.wg.Add(1)
	go func() {
		defer func() {
			close(eventChan)
			sc.wg.Done()
		}()

		for {
			select {
			case eventChan <- event:
			case <-ctx.Done():
				return
			}

			buf, err := sc.Conn.ReadPacket()
			if err != nil {
				if sqlErr, ok := err.(*mysql.SQLError); ok && sqlErr.Number() == mysql.CRServerLost {
					// CRServerLost = Lost connection to MySQL server during query
					// This is not necessarily an error. It could just be that we closed
					// the connection from outside.
					log.Infof("connection closed during binlog stream (possibly intentional): %v", err)
					return
				}
				log.Errorf("read error while streaming binlog events: %v", err)
				return
			}

			// Handle EOF and error case.
			switch buf[0] {
			case mysql.EOFPacket:
				// The master is telling us to stop.
				log.Infof("received EOF packet in binlog dump: %#v", buf)
				return
			case mysql.ErrPacket:
				err := mysql.ParseErrorPacket(buf)
				log.Infof("received error packet in binlog dump: %v", err)
			}

			// Skip the first byte because it's only used
			// for signaling EOF / error.
			event = sc.Conn.MakeBinlogEvent(buf[1:])
		}
	}()

	return eventChan, nil
}

// Close closes the slave connection, which also signals an ongoing dump
// started with StartBinlogDump() to stop and close its BinlogEvent channel.
// The ID for the slave connection is recycled back into the pool.
func (sc *SlaveConnection) Close() {
	if sc.Conn != nil {
		log.Infof("closing slave socket to unblock reads")
		sc.Conn.Close()

		log.Infof("waiting for slave dump thread to end")
		sc.cancel()
		sc.wg.Wait()

		log.Infof("closing slave MySQL client, recycling slaveID %v", sc.slaveID)
		sc.Conn = nil
		slaveIDPool.Put(sc.slaveID)
	}
}
<commit_msg>Fixing binlog server panic.<commit_after>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mysqlctl

import (
	"fmt"
	"sync"

	log "github.com/golang/glog"
	"golang.org/x/net/context"

	"github.com/youtube/vitess/go/mysql"
	"github.com/youtube/vitess/go/pools"
	"github.com/youtube/vitess/go/vt/dbconfigs"
)

var (
	// ErrBinlogUnavailable is returned by this library when we
	// cannot find a suitable binlog to satisfy the request.
	ErrBinlogUnavailable = fmt.Errorf("cannot find relevant binlogs on this server")
)

// SlaveConnection represents a connection to mysqld that pretends to be a slave
// connecting for replication. Each such connection must identify itself to
// mysqld with a server ID that is unique both among other SlaveConnections and
// among actual slaves in the topology.
type SlaveConnection struct {
	*mysql.Conn
	mysqld  *Mysqld
	slaveID uint32
	cancel  context.CancelFunc
	wg      sync.WaitGroup
}

// NewSlaveConnection creates a new slave connection to the mysqld instance.
// It uses a pools.IDPool to ensure that the server IDs used to connect are
// unique within this process. This is done with the assumptions that:
//
// 1) No other processes are making fake slave connections to our mysqld.
// 2) No real slave servers will have IDs in the range 1-N where N is the peak
//    number of concurrent fake slave connections we will ever make.
func (mysqld *Mysqld) NewSlaveConnection() (*SlaveConnection, error) {
	conn, err := mysqld.connectForReplication()
	if err != nil {
		return nil, err
	}

	sc := &SlaveConnection{
		Conn:    conn,
		mysqld:  mysqld,
		slaveID: slaveIDPool.Get(),
	}
	log.Infof("new slave connection: slaveID=%d", sc.slaveID)
	return sc, nil
}

// connectForReplication create a MySQL connection ready to use for replication.
func (mysqld *Mysqld) connectForReplication() (*mysql.Conn, error) {
	params, err := dbconfigs.WithCredentials(&mysqld.dbcfgs.Dba)
	if err != nil {
		return nil, err
	}

	ctx := context.Background()
	conn, err := mysql.Connect(ctx, &params)
	if err != nil {
		return nil, err
	}

	// Tell the server that we understand the format of events
	// that will be used if binlog_checksum is enabled on the server.
	if _, err := conn.ExecuteFetch("SET @master_binlog_checksum=@@global.binlog_checksum", 0, false); err != nil {
		return nil, fmt.Errorf("failed to set @master_binlog_checksum=@@global.binlog_checksum: %v", err)
	}

	return conn, nil
}

// slaveIDPool is the IDPool for server IDs used to connect as a slave.
var slaveIDPool = pools.NewIDPool()

// StartBinlogDumpFromCurrent requests a replication binlog dump from
// the current position.
func (sc *SlaveConnection) StartBinlogDumpFromCurrent(ctx context.Context) (mysql.Position, <-chan mysql.BinlogEvent, error) {
	ctx, sc.cancel = context.WithCancel(ctx)

	masterPosition, err := sc.mysqld.MasterPosition()
	if err != nil {
		return mysql.Position{}, nil, fmt.Errorf("failed to get master position: %v", err)
	}

	c, err := sc.StartBinlogDumpFromPosition(ctx, masterPosition)
	return masterPosition, c, err
}

// StartBinlogDumpFromPosition requests a replication binlog dump from
// the master mysqld at the given Position and then sends binlog
// events to the provided channel.
// The stream will continue in the background, waiting for new events if
// necessary, until the connection is closed, either by the master or
// by canceling the context.
//
// Note the context is valid and used until eventChan is closed.
func (sc *SlaveConnection) StartBinlogDumpFromPosition(ctx context.Context, startPos mysql.Position) (<-chan mysql.BinlogEvent, error) {
	ctx, sc.cancel = context.WithCancel(ctx)

	log.Infof("sending binlog dump command: startPos=%v, slaveID=%v", startPos, sc.slaveID)
	if err := sc.SendBinlogDumpCommand(sc.slaveID, startPos); err != nil {
		log.Errorf("couldn't send binlog dump command: %v", err)
		return nil, err
	}

	// Read the first packet to see if it's an error response to our dump command.
	buf, err := sc.Conn.ReadPacket()
	if err != nil {
		log.Errorf("couldn't start binlog dump: %v", err)
		return nil, err
	}

	// FIXME(alainjobart) I think we can use a buffered channel for better performance.
	eventChan := make(chan mysql.BinlogEvent)

	// Start reading events.
	sc.wg.Add(1)
	go func() {
		defer func() {
			close(eventChan)
			sc.wg.Done()
		}()
		for {
			// Handle EOF and error case.
			switch buf[0] {
			case mysql.EOFPacket:
				log.Infof("received EOF packet in binlog dump: %#v", buf)
				return
			case mysql.ErrPacket:
				err := mysql.ParseErrorPacket(buf)
				log.Infof("received error packet in binlog dump: %v", err)
				return
			}

			select {
			// Skip the first byte because it's only used for signaling EOF / error.
			case eventChan <- sc.Conn.MakeBinlogEvent(buf[1:]):
			case <-ctx.Done():
				return
			}

			buf, err = sc.Conn.ReadPacket()
			if err != nil {
				if sqlErr, ok := err.(*mysql.SQLError); ok && sqlErr.Number() == mysql.CRServerLost {
					// CRServerLost = Lost connection to MySQL server during query
					// This is not necessarily an error. It could just be that we closed
					// the connection from outside.
					log.Infof("connection closed during binlog stream (possibly intentional): %v", err)
					return
				}
				log.Errorf("read error while streaming binlog events: %v", err)
				return
			}
		}
	}()

	return eventChan, nil
}

// StartBinlogDumpFromBinlogBeforeTimestamp requests a replication
// binlog dump from the master mysqld starting with a file that has
// timestamps smaller than the provided timestamp, and then sends
// binlog events to the provided channel.
//
// The startup phase will list all the binary logs, and find the one
// that has events starting strictly before the provided timestamp. It
// will then start from there, and stream all events. It is the
// responsability of the calling site to filter the events more.
//
// MySQL 5.6+ note: we need to do it that way because of the way the
// GTIDSet works. In the previous two streaming functions, we pass in
// the full GTIDSet (that has the list of all transactions seen in
// the replication stream). In this case, we don't know it, all we
// have is the binlog file names. We depend on parsing the first
// PREVIOUS_GTIDS_EVENT event in the logs to get it. So we need the
// caller to parse that event, and it can't be skipped because its
// timestamp is lower. Then, for each subsequent event, the caller
// also needs to add the event GTID to its GTIDSet. Otherwise it won't
// be correct ever. So the caller really needs to build up its GTIDSet
// along the entire file, not just for events whose timestamp is in a
// given range.
//
// The stream will continue in the background, waiting for new events if
// necessary, until the connection is closed, either by the master or
// by canceling the context.
//
// Note the context is valid and used until eventChan is closed.
func (sc *SlaveConnection) StartBinlogDumpFromBinlogBeforeTimestamp(ctx context.Context, timestamp int64) (<-chan mysql.BinlogEvent, error) {
	ctx, sc.cancel = context.WithCancel(ctx)

	// List the binlogs.
	binlogs, err := sc.Conn.ExecuteFetch("SHOW BINARY LOGS", 1000, false)
	if err != nil {
		return nil, fmt.Errorf("failed to SHOW BINARY LOGS: %v", err)
	}

	// Start with the most recent binlog file until we find the right event.
	var binlogIndex int
	var event mysql.BinlogEvent
	for binlogIndex = len(binlogs.Rows) - 1; binlogIndex >= 0; binlogIndex-- {
		// Exit the loop early if context is canceled.
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
		}

		// Start dumping the logs. The position is '4' to skip the
		// Binlog File Header. See this page for more info:
		// https://dev.mysql.com/doc/internals/en/binlog-file.html
		binlog := binlogs.Rows[binlogIndex][0].ToString()
		if err := sc.Conn.WriteComBinlogDump(sc.slaveID, binlog, 4, 0); err != nil {
			return nil, fmt.Errorf("failed to send the ComBinlogDump command: %v", err)
		}

		// Get the first event to get its timestamp. We skip
		// events that don't have timestamps (although it seems
		// most do anyway).
		for {
			buf, err := sc.Conn.ReadPacket()
			if err != nil {
				return nil, fmt.Errorf("couldn't start binlog dump of binlog %v: %v", binlog, err)
			}

			// Handle EOF and error case.
			switch buf[0] {
			case mysql.EOFPacket:
				return nil, fmt.Errorf("received EOF packet for first packet of binlog %v", binlog)
			case mysql.ErrPacket:
				err := mysql.ParseErrorPacket(buf)
				return nil, fmt.Errorf("received error packet for first packet of binlog %v", err)
			}

			// Parse the full event.
			event = sc.Conn.MakeBinlogEvent(buf[1:])
			if !event.IsValid() {
				return nil, fmt.Errorf("first event from binlog %v is not valid", binlog)
			}
			if event.Timestamp() > 0 {
				// We found the first event with a
				// valid timestamp.
				break
			}
		}
		if int64(event.Timestamp()) < timestamp {
			// The first event in this binlog has a smaller
			// timestamp than what we need, we found a good
			// starting point.
			break
		}

		// The timestamp is higher, we need to try the older files.
		// Close and re-open our connection.
		sc.Conn.Close()
		conn, err := sc.mysqld.connectForReplication()
		if err != nil {
			return nil, err
		}
		sc.Conn = conn
	}
	if binlogIndex == -1 {
		// We haven't found a suitable binlog
		log.Errorf("couldn't find an old enough binlog to match timestamp >= %v (looked at %v files)", timestamp, len(binlogs.Rows))
		return nil, ErrBinlogUnavailable
	}

	// Now just loop sending and reading events.
	// FIXME(alainjobart) I think we can use a buffered channel for better performance.
	eventChan := make(chan mysql.BinlogEvent)

	// Start reading events.
	sc.wg.Add(1)
	go func() {
		defer func() {
			close(eventChan)
			sc.wg.Done()
		}()

		for {
			select {
			case eventChan <- event:
			case <-ctx.Done():
				return
			}

			buf, err := sc.Conn.ReadPacket()
			if err != nil {
				if sqlErr, ok := err.(*mysql.SQLError); ok && sqlErr.Number() == mysql.CRServerLost {
					// CRServerLost = Lost connection to MySQL server during query
					// This is not necessarily an error. It could just be that we closed
					// the connection from outside.
					log.Infof("connection closed during binlog stream (possibly intentional): %v", err)
					return
				}
				log.Errorf("read error while streaming binlog events: %v", err)
				return
			}

			// Handle EOF and error case.
			switch buf[0] {
			case mysql.EOFPacket:
				// The master is telling us to stop.
				log.Infof("received EOF packet in binlog dump: %#v", buf)
				return
			case mysql.ErrPacket:
				err := mysql.ParseErrorPacket(buf)
				log.Infof("received error packet in binlog dump: %v", err)
			}

			// Skip the first byte because it's only used
			// for signaling EOF / error.
			event = sc.Conn.MakeBinlogEvent(buf[1:])
		}
	}()

	return eventChan, nil
}

// Close closes the slave connection, which also signals an ongoing dump
// started with StartBinlogDump() to stop and close its BinlogEvent channel.
// The ID for the slave connection is recycled back into the pool.
func (sc *SlaveConnection) Close() {
	if sc.Conn != nil {
		log.Infof("closing slave socket to unblock reads")
		sc.Conn.Close()

		// sc.cancel is set at the beginning of the StartBinlogDump*
		// methods. If we error out before then, it's nil.
		// Note we also may error out before adding 1 to sc.wg,
		// but then the Wait() still works.
		if sc.cancel != nil {
			log.Infof("waiting for slave dump thread to end")
			sc.cancel()
			sc.wg.Wait()
			sc.cancel = nil
		}

		log.Infof("closing slave MySQL client, recycling slaveID %v", sc.slaveID)
		sc.Conn = nil
		slaveIDPool.Put(sc.slaveID)
	}
}
<|endoftext|>


<commit_before>package twitch

import (
	"fmt"
	"strconv"
	"strings"
	"time"
)

// MessageType different message types possible to receive via IRC
type MessageType int

const (
	// UNSET is the default message type, for whenever a new message type is added by twitch that we don't parse yet
	UNSET MessageType = -1
	// WHISPER private messages
	WHISPER MessageType = 0
	// PRIVMSG standard chat message
	PRIVMSG MessageType = 1
	// CLEARCHAT timeout messages
	CLEARCHAT MessageType = 2
	// ROOMSTATE changes like sub mode
	ROOMSTATE MessageType = 3
	// USERNOTICE messages like subs, resubs, raids, etc
	USERNOTICE MessageType = 4
	// USERSTATE messages
	USERSTATE MessageType = 5
	// NOTICE messages like sub mode, host on
	NOTICE MessageType = 6
)

type message struct {
	Type        MessageType
	Time        time.Time
	Channel     string
	ChannelID   string
	UserID      string
	Username    string
	DisplayName string
	UserType    string
	Color       string
	Action      bool
	Badges      map[string]int
	Emotes      []*Emote
	Tags        map[string]string
	Text        string
	Raw         string
}

// Emote twitch emotes
type Emote struct {
	Name  string
	ID    string
	Count int
}

func parseMessage(line string) *message {
	if !strings.HasPrefix(line, "@") {
		return &message{
			Text: line,
			Raw:  line,
			Type: UNSET,
		}
	}
	spl := strings.SplitN(line, " :", 3)
	if len(spl) < 3 {
		return parseOtherMessage(line)
	}
	action := false
	tags, middle, text := spl[0], spl[1], spl[2]
	if strings.HasPrefix(text, "\u0001ACTION ") && strings.HasSuffix(text, "\u0001") {
		action = true
		text = text[8 : len(text)-1]
	}
	msg := &message{
		Text:   text,
		Tags:   map[string]string{},
		Action: action,
		Type:   UNSET,
	}
	msg.Username, msg.Type, msg.Channel = parseMiddle(middle)
	parseTags(msg, tags[1:])
	if msg.Type == CLEARCHAT {
		targetUser := msg.Text
		msg.Username = targetUser

		msg.Text = fmt.Sprintf("%s was timed out for %s: %s", targetUser, msg.Tags["ban-duration"], msg.Tags["ban-reason"])
	}
	msg.Raw = line
	return msg
}

func parseOtherMessage(line string) *message {
	msg := &message{
		Type: UNSET,
	}
	split := strings.Split(line, " ")
	msg.Raw = line

	msg.Type = parseMessageType(split[2])
	msg.Tags = make(map[string]string)

	// Parse out channel if it exists in this line
	if len(split) >= 4 && len(split[3]) > 1 && split[3][0] == '#' {
		// Remove # from channel
		msg.Channel = split[3][1:]
	}

	tagsString := strings.Fields(strings.TrimPrefix(split[0], "@"))
	tags := strings.Split(tagsString[0], ";")
	for _, tag := range tags {
		tagSplit := strings.Split(tag, "=")

		value := ""
		if len(tagSplit) > 1 {
			value = tagSplit[1]
		}

		msg.Tags[tagSplit[0]] = value
	}

	if msg.Type == CLEARCHAT {
		msg.Text = "Chat has been cleared by a moderator"
	}
	return msg
}

func parseMessageType(messageType string) MessageType {
	switch messageType {
	case "PRIVMSG":
		return PRIVMSG
	case "WHISPER":
		return WHISPER
	case "CLEARCHAT":
		return CLEARCHAT
	case "NOTICE":
		return NOTICE
	case "ROOMSTATE":
		return ROOMSTATE
	case "USERSTATE":
		return USERSTATE
	case "USERNOTICE":
		return USERNOTICE
	default:
		return UNSET
	}
}

func parseMiddle(middle string) (string, MessageType, string) {
	var username string
	var msgType MessageType
	var channel string

	for i, c := range middle {
		if c == '!' {
			username = middle[:i]
			middle = middle[i:]
		}
	}
	start := -1
	for i, c := range middle {
		if c == ' ' {
			if start == -1 {
				start = i + 1
			} else {
				typ := middle[start:i]
				msgType = parseMessageType(typ)
				middle = middle[i:]
			}
		}
	}
	for i, c := range middle {
		if c == '#' {
			channel = middle[i+1:]
		}
	}

	return username, msgType, channel
}

func parseTags(msg *message, tagsRaw string) {
	tags := strings.Split(tagsRaw, ";")
	for _, tag := range tags {
		spl := strings.SplitN(tag, "=", 2)
		value := strings.Replace(spl[1], "\\:", ";", -1)
		value = strings.Replace(value, "\\s", " ", -1)
		value = strings.Replace(value, "\\\\", "\\", -1)
		switch spl[0] {
		case "badges":
			msg.Badges = parseBadges(value)
		case "color":
			msg.Color = value
		case "display-name":
			msg.DisplayName = value
		case "emotes":
			msg.Emotes = parseTwitchEmotes(value, msg.Text)
		case "user-type":
			msg.UserType = value
		case "tmi-sent-ts":
			i, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				msg.Time = time.Unix(0, int64(i*1e6))
			}
		case "room-id":
			msg.ChannelID = value
		case "target-user-id":
			msg.UserID = value
		case "user-id":
			msg.UserID = value
		}
		msg.Tags[spl[0]] = value
	}
}

func parseBadges(badges string) map[string]int {
	m := map[string]int{}
	spl := strings.Split(badges, ",")
	for _, badge := range spl {
		s := strings.SplitN(badge, "/", 2)
		if len(s) < 2 {
			continue
		}
		n, _ := strconv.Atoi(s[1])
		m[s[0]] = n
	}
	return m
}

func parseTwitchEmotes(emoteTag, text string) []*Emote {
	emotes := []*Emote{}

	if emoteTag == "" {
		return emotes
	}

	runes := []rune(text)

	emoteSlice := strings.Split(emoteTag, "/")
	for i := range emoteSlice {
		spl := strings.Split(emoteSlice[i], ":")
		pos := strings.Split(spl[1], ",")
		sp := strings.Split(pos[0], "-")
		start, _ := strconv.Atoi(sp[0])
		end, _ := strconv.Atoi(sp[1])
		id := spl[0]
		e := &Emote{
			ID:    id,
			Count: strings.Count(emoteSlice[i], "-"),
			Name:  string(runes[start : end+1]),
		}

		emotes = append(emotes, e)
	}
	return emotes
}

func parseJoinPart(text string) (string, string) {
	username := strings.Split(text, "!")
	channel := strings.Split(username[1], "#")
	return strings.Trim(channel[1], " "), strings.Trim(username[0], " :")
}

func parseNames(text string) (string, []string) {
	lines := strings.Split(text, ":")
	channelDirty := strings.Split(lines[1], "#")
	channel := strings.Trim(channelDirty[1], " ")
	users := strings.Split(lines[2], " ")

	return channel, users
}
<commit_msg>fix parseNames behavior when joining Room fixes https://github.com/gempir/go-twitch-irc/issues/94<commit_after>package twitch

import (
	"fmt"
	"strconv"
	"strings"
	"time"
)

// MessageType different message types possible to receive via IRC
type MessageType int

const (
	// UNSET is the default message type, for whenever a new message type is added by twitch that we don't parse yet
	UNSET MessageType = -1
	// WHISPER private messages
	WHISPER MessageType = 0
	// PRIVMSG standard chat message
	PRIVMSG MessageType = 1
	// CLEARCHAT timeout messages
	CLEARCHAT MessageType = 2
	// ROOMSTATE changes like sub mode
	ROOMSTATE MessageType = 3
	// USERNOTICE messages like subs, resubs, raids, etc
	USERNOTICE MessageType = 4
	// USERSTATE messages
	USERSTATE MessageType = 5
	// NOTICE messages like sub mode, host on
	NOTICE MessageType = 6
)

type message struct {
	Type        MessageType
	Time        time.Time
	Channel     string
	ChannelID   string
	UserID      string
	Username    string
	DisplayName string
	UserType    string
	Color       string
	Action      bool
	Badges      map[string]int
	Emotes      []*Emote
	Tags        map[string]string
	Text        string
	Raw         string
}

// Emote twitch emotes
type Emote struct {
	Name  string
	ID    string
	Count int
}

func parseMessage(line string) *message {
	if !strings.HasPrefix(line, "@") {
		return &message{
			Text: line,
			Raw:  line,
			Type: UNSET,
		}
	}
	spl := strings.SplitN(line, " :", 3)
	if len(spl) < 3 {
		return parseOtherMessage(line)
	}
	action := false
	tags, middle, text := spl[0], spl[1], spl[2]
	if strings.HasPrefix(text, "\u0001ACTION ") && strings.HasSuffix(text, "\u0001") {
		action = true
		text = text[8 : len(text)-1]
	}
	msg := &message{
		Text:   text,
		Tags:   map[string]string{},
		Action: action,
		Type:   UNSET,
	}
	msg.Username, msg.Type, msg.Channel = parseMiddle(middle)
	parseTags(msg, tags[1:])
	if msg.Type == CLEARCHAT {
		targetUser := msg.Text
		msg.Username = targetUser

		msg.Text = fmt.Sprintf("%s was timed out for %s: %s", targetUser, msg.Tags["ban-duration"], msg.Tags["ban-reason"])
	}
	msg.Raw = line
	return msg
}

func parseOtherMessage(line string) *message {
	msg := &message{
		Type: UNSET,
	}
	split := strings.Split(line, " ")
	msg.Raw = line

	msg.Type = parseMessageType(split[2])
	msg.Tags = make(map[string]string)

	// Parse out channel if it exists in this line
	if len(split) >= 4 && len(split[3]) > 1 && split[3][0] == '#' {
		// Remove # from channel
		msg.Channel = split[3][1:]
	}

	tagsString := strings.Fields(strings.TrimPrefix(split[0], "@"))
	tags := strings.Split(tagsString[0], ";")
	for _, tag := range tags {
		tagSplit := strings.Split(tag, "=")

		value := ""
		if len(tagSplit) > 1 {
			value = tagSplit[1]
		}

		msg.Tags[tagSplit[0]] = value
	}

	if msg.Type == CLEARCHAT {
		msg.Text = "Chat has been cleared by a moderator"
	}
	return msg
}

func parseMessageType(messageType string) MessageType {
	switch messageType {
	case "PRIVMSG":
		return PRIVMSG
	case "WHISPER":
		return WHISPER
	case "CLEARCHAT":
		return CLEARCHAT
	case "NOTICE":
		return NOTICE
	case "ROOMSTATE":
		return ROOMSTATE
	case "USERSTATE":
		return USERSTATE
	case "USERNOTICE":
		return USERNOTICE
	default:
		return UNSET
	}
}

func parseMiddle(middle string) (string, MessageType, string) {
	var username string
	var msgType MessageType
	var channel string

	for i, c := range middle {
		if c == '!' {
			username = middle[:i]
			middle = middle[i:]
		}
	}
	start := -1
	for i, c := range middle {
		if c == ' ' {
			if start == -1 {
				start = i + 1
			} else {
				typ := middle[start:i]
				msgType = parseMessageType(typ)
				middle = middle[i:]
			}
		}
	}
	for i, c := range middle {
		if c == '#' {
			channel = middle[i+1:]
		}
	}

	return username, msgType, channel
}

func parseTags(msg *message, tagsRaw string) {
	tags := strings.Split(tagsRaw, ";")
	for _, tag := range tags {
		spl := strings.SplitN(tag, "=", 2)
		value := strings.Replace(spl[1], "\\:", ";", -1)
		value = strings.Replace(value, "\\s", " ", -1)
		value = strings.Replace(value, "\\\\", "\\", -1)
		switch spl[0] {
		case "badges":
			msg.Badges = parseBadges(value)
		case "color":
			msg.Color = value
		case "display-name":
			msg.DisplayName = value
		case "emotes":
			msg.Emotes = parseTwitchEmotes(value, msg.Text)
		case "user-type":
			msg.UserType = value
		case "tmi-sent-ts":
			i, err := strconv.ParseInt(value, 10, 64)
			if err == nil {
				msg.Time = time.Unix(0, int64(i*1e6))
			}
		case "room-id":
			msg.ChannelID = value
		case "target-user-id":
			msg.UserID = value
		case "user-id":
			msg.UserID = value
		}
		msg.Tags[spl[0]] = value
	}
}

func parseBadges(badges string) map[string]int {
	m := map[string]int{}
	spl := strings.Split(badges, ",")
	for _, badge := range spl {
		s := strings.SplitN(badge, "/", 2)
		if len(s) < 2 {
			continue
		}
		n, _ := strconv.Atoi(s[1])
		m[s[0]] = n
	}
	return m
}

func parseTwitchEmotes(emoteTag, text string) []*Emote {
	emotes := []*Emote{}

	if emoteTag == "" {
		return emotes
	}

	runes := []rune(text)

	emoteSlice := strings.Split(emoteTag, "/")
	for i := range emoteSlice {
		spl := strings.Split(emoteSlice[i], ":")
		pos := strings.Split(spl[1], ",")
		sp := strings.Split(pos[0], "-")
		start, _ := strconv.Atoi(sp[0])
		end, _ := strconv.Atoi(sp[1])
		id := spl[0]
		e := &Emote{
			ID:    id,
			Count: strings.Count(emoteSlice[i], "-"),
			Name:  string(runes[start : end+1]),
		}

		emotes = append(emotes, e)
	}
	return emotes
}

func parseJoinPart(text string) (string, string) {
	username := strings.Split(text, "!")
	channel := strings.Split(username[1], "#")
	return strings.Trim(channel[1], " "), strings.Trim(username[0], " :")
}

func parseNames(text string) (string, []string) {
	lines := strings.Split(text, " :")
	channelDirty := strings.Split(lines[0], "#")
	channel := strings.Trim(channelDirty[1], " ")
	users := strings.Split(lines[1], " ")

	return channel, users
}
<|endoftext|>


<commit_before>package gocd

import (
	"fmt"
)

// Equal is true if the two materials are logically equivalent. Not neccesarily literally equal.
func (m Material) Equal(a *Material) bool {
	if m.Type != a.Type {
		return false
	}
	switch m.Type {
	case "git":
		return m.Attributes.equalGit(&a.Attributes)
	default:
		panic(fmt.Errorf("Material comparison not implemented for '%s'", m.Type))
	}
}

func (a1 MaterialAttributes) equalGit(a2 *MaterialAttributes) bool {
	if a1.URL == a2.URL {
		if a1.Branch == a2.Branch {
			// Check if branches are equal
			return true
		} else if a1.Branch == "" && a2.Branch == "master" || a1.Branch == "master" && a2.Branch == "" {
			// Check if branches are master equal
			return true
		}
	}
	return false
}
<commit_msg>Simplified material comparison<commit_after>package gocd

import (
	"fmt"
)

// Equal is true if the two materials are logically equivalent. Not neccesarily literally equal.
func (m Material) Equal(a *Material) bool {
	if m.Type != a.Type {
		return false
	}
	switch m.Type {
	case "git":
		return m.Attributes.equalGit(&a.Attributes)
	default:
		panic(fmt.Errorf("Material comparison not implemented for '%s'", m.Type))
	}
}

func (a1 MaterialAttributes) equalGit(a2 *MaterialAttributes) bool {
	urlsEqual := a1.URL == a2.URL
	branchesEqual := a1.Branch == a2.Branch ||
		a1.Branch == "" && a2.Branch == "master" ||
		a1.Branch == "master" && a2.Branch == ""

	if !urlsEqual {
		return false
	}
	return branchesEqual
}
<|endoftext|>


<commit_before>package google

import (
	"fmt"

	"regexp"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"
	computeBeta "google.golang.org/api/compute/v0.beta"
	"google.golang.org/api/compute/v1"
)

const (
	addressTypeExternal = "EXTERNAL"
	addressTypeInternal = "INTERNAL"
)

var (
	computeAddressIdTemplate = "projects/%s/regions/%s/addresses/%s"
	computeAddressLinkRegex  = regexp.MustCompile("projects/(.+)/regions/(.+)/addresses/(.+)$")
	AddressBaseApiVersion    = v1
	AddressVersionedFeatures = []Feature{
		{Version: v0beta, Item: "address_type", DefaultValue: addressTypeExternal},
		{Version: v0beta, Item: "subnetwork"},
	}
)

func resourceComputeAddress() *schema.Resource {
	return &schema.Resource{
		Create: resourceComputeAddressCreate,
		Read:   resourceComputeAddressRead,
		Delete: resourceComputeAddressDelete,
		Importer: &schema.ResourceImporter{
			State: resourceComputeAddressImportState,
		},

		SchemaVersion: 1,
		MigrateState:  resourceComputeAddressMigrateState,

		// These fields mostly correlate to the fields in the beta Address
		// resource. See https://cloud.google.com/compute/docs/reference/beta/addresses#resource
		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"address_type": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Default:  addressTypeExternal,
				ValidateFunc: validation.StringInSlice(
					[]string{addressTypeInternal, addressTypeExternal}, false),
			},

			"subnetwork": &schema.Schema{
				Type:             schema.TypeString,
				Optional:         true,
				ForceNew:         true,
				Computed:         true,
				DiffSuppressFunc: linkDiffSuppress,
			},

			// address will be computed unless it is specified explicitly.
			// address may only be specified for the INTERNAL address_type.
			"address": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			"project": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},

			"region": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},

			"self_link": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceComputeAddressCreate(d *schema.ResourceData, meta interface{}) error {
	computeApiVersion := getComputeApiVersion(d, AddressBaseApiVersion, AddressVersionedFeatures)
	config := meta.(*Config)

	region, err := getRegion(d, config)
	if err != nil {
		return err
	}

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	// Build the address parameter
	v0BetaAddress := &computeBeta.Address{
		Name:        d.Get("name").(string),
		AddressType: d.Get("address_type").(string),
		Subnetwork:  d.Get("subnetwork").(string),
	}

	if desired, ok := d.GetOk("address"); ok {
		v0BetaAddress.Address = desired.(string)
	}

	var op interface{}
	switch computeApiVersion {
	case v1:
		v1Address := &compute.Address{}
		err = Convert(v0BetaAddress, v1Address)
		if err != nil {
			return err
		}
		op, err = config.clientCompute.Addresses.Insert(
			project, region, v1Address).Do()
	case v0beta:
		op, err = config.clientComputeBeta.Addresses.Insert(
			project, region, v0BetaAddress).Do()
	}
	if err != nil {
		return fmt.Errorf("Error creating address: %s", err)
	}

	// It probably maybe worked, so store the ID now
	d.SetId(computeAddressId{
		Project: project,
		Region:  region,
		Name:    v0BetaAddress.Name,
	}.canonicalId())

	err = computeSharedOperationWait(config.clientCompute, op, project, "Creating Address")
	if err != nil {
		return err
	}

	return resourceComputeAddressRead(d, meta)
}

func resourceComputeAddressRead(d *schema.ResourceData, meta interface{}) error {
	computeApiVersion := getComputeApiVersion(d, AddressBaseApiVersion, AddressVersionedFeatures)
	config := meta.(*Config)

	addressId, err := parseComputeAddressId(d.Id(), config)
	if err != nil {
		return err
	}

	addr := &computeBeta.Address{}
	switch computeApiVersion {
	case v1:
		v1Address, err := config.clientCompute.Addresses.Get(
			addressId.Project, addressId.Region, addressId.Name).Do()
		if err != nil {
			return handleNotFoundError(err, d, fmt.Sprintf("Address %q", d.Get("name").(string)))
		}

		err = Convert(v1Address, addr)
		if err != nil {
			return err
		}

	case v0beta:
		var err error
		addr, err = config.clientComputeBeta.Addresses.Get(
			addressId.Project, addressId.Region, addressId.Name).Do()
		if err != nil {
			return handleNotFoundError(err, d, fmt.Sprintf("Address %q", d.Get("name").(string)))
		}

	}

	// The v1 API does not include an AddressType field, as it only supports
	// external addresses. An empty AddressType implies a v1 address that has
	// been converted to v0beta, and thus an EXTERNAL address.
	d.Set("address_type", addr.AddressType)
	if addr.AddressType == "" {
		d.Set("address_type", addressTypeExternal)
	}
	d.Set("subnetwork", ConvertSelfLinkToV1(addr.Subnetwork))
	d.Set("address", addr.Address)
	d.Set("self_link", ConvertSelfLinkToV1(addr.SelfLink))
	d.Set("name", addr.Name)
	d.Set("project", addressId.Project)
	d.Set("region", GetResourceNameFromSelfLink(addr.Region))

	return nil
}

func resourceComputeAddressDelete(d *schema.ResourceData, meta interface{}) error {
	computeApiVersion := getComputeApiVersion(d, AddressBaseApiVersion, AddressVersionedFeatures)
	config := meta.(*Config)

	addressId, err := parseComputeAddressId(d.Id(), config)
	if err != nil {
		return err
	}

	// Delete the address
	var op interface{}
	switch computeApiVersion {
	case v1:
		op, err = config.clientCompute.Addresses.Delete(
			addressId.Project, addressId.Region, addressId.Name).Do()
		if err != nil {
			return fmt.Errorf("Error deleting address: %s", err)
		}
	case v0beta:
		op, err = config.clientComputeBeta.Addresses.Delete(
			addressId.Project, addressId.Region, addressId.Name).Do()
		if err != nil {
			return fmt.Errorf("Error deleting address: %s", err)
		}
	}

	err = computeSharedOperationWait(config.clientCompute, op, addressId.Project, "Deleting Address")
	if err != nil {
		return err
	}

	d.SetId("")
	return nil
}

func resourceComputeAddressImportState(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	config := meta.(*Config)

	addressId, err := parseComputeAddressId(d.Id(), config)
	if err != nil {
		return nil, err
	}

	d.SetId(addressId.canonicalId())

	return []*schema.ResourceData{d}, nil
}
<commit_msg>Use v1 API for google_compute_address (#1384)<commit_after>package google

import (
	"fmt"

	"regexp"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"
	"google.golang.org/api/compute/v1"
)

const (
	addressTypeExternal = "EXTERNAL"
	addressTypeInternal = "INTERNAL"
)

var (
	computeAddressIdTemplate = "projects/%s/regions/%s/addresses/%s"
	computeAddressLinkRegex  = regexp.MustCompile("projects/(.+)/regions/(.+)/addresses/(.+)$")
)

func resourceComputeAddress() *schema.Resource {
	return &schema.Resource{
		Create: resourceComputeAddressCreate,
		Read:   resourceComputeAddressRead,
		Delete: resourceComputeAddressDelete,
		Importer: &schema.ResourceImporter{
			State: resourceComputeAddressImportState,
		},

		SchemaVersion: 1,
		MigrateState:  resourceComputeAddressMigrateState,

		// These fields mostly correlate to the fields in the beta Address
		// resource. See https://cloud.google.com/compute/docs/reference/beta/addresses#resource
		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"address_type": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Default:  addressTypeExternal,
				ValidateFunc: validation.StringInSlice(
					[]string{addressTypeInternal, addressTypeExternal}, false),
			},

			"subnetwork": &schema.Schema{
				Type:             schema.TypeString,
				Optional:         true,
				ForceNew:         true,
				Computed:         true,
				DiffSuppressFunc: linkDiffSuppress,
			},

			// address will be computed unless it is specified explicitly.
			// address may only be specified for the INTERNAL address_type.
			"address": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			"project": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},

			"region": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},

			"self_link": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceComputeAddressCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	region, err := getRegion(d, config)
	if err != nil {
		return err
	}

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	// Build the address parameter
	address := &compute.Address{
		Name:        d.Get("name").(string),
		AddressType: d.Get("address_type").(string),
		Subnetwork:  d.Get("subnetwork").(string),
		Address:     d.Get("address").(string),
	}

	op, err := config.clientCompute.Addresses.Insert(project, region, address).Do()
	if err != nil {
		return fmt.Errorf("Error creating address: %s", err)
	}

	// It probably maybe worked, so store the ID now
	d.SetId(computeAddressId{
		Project: project,
		Region:  region,
		Name:    address.Name,
	}.canonicalId())

	err = computeSharedOperationWait(config.clientCompute, op, project, "Creating Address")
	if err != nil {
		return err
	}

	return resourceComputeAddressRead(d, meta)
}

func resourceComputeAddressRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	addressId, err := parseComputeAddressId(d.Id(), config)
	if err != nil {
		return err
	}

	addr, err := config.clientCompute.Addresses.Get(
		addressId.Project, addressId.Region, addressId.Name).Do()
	if err != nil {
		return handleNotFoundError(err, d, fmt.Sprintf("Address %q", d.Get("name").(string)))
	}

	// The v1 API does not include an AddressType field, as it only supports
	// external addresses. An empty AddressType implies a v1 address that has
	// been converted to v0beta, and thus an EXTERNAL address.
	d.Set("address_type", addr.AddressType)
	if addr.AddressType == "" {
		d.Set("address_type", addressTypeExternal)
	}
	d.Set("subnetwork", ConvertSelfLinkToV1(addr.Subnetwork))
	d.Set("address", addr.Address)
	d.Set("self_link", ConvertSelfLinkToV1(addr.SelfLink))
	d.Set("name", addr.Name)
	d.Set("project", addressId.Project)
	d.Set("region", GetResourceNameFromSelfLink(addr.Region))

	return nil
}

func resourceComputeAddressDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	addressId, err := parseComputeAddressId(d.Id(), config)
	if err != nil {
		return err
	}

	// Delete the address
	op, err := config.clientCompute.Addresses.Delete(
		addressId.Project, addressId.Region, addressId.Name).Do()
	if err != nil {
		return fmt.Errorf("Error deleting address: %s", err)
	}

	err = computeSharedOperationWait(config.clientCompute, op, addressId.Project, "Deleting Address")
	if err != nil {
		return err
	}

	d.SetId("")
	return nil
}

func resourceComputeAddressImportState(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	config := meta.(*Config)

	addressId, err := parseComputeAddressId(d.Id(), config)
	if err != nil {
		return nil, err
	}

	d.SetId(addressId.canonicalId())

	return []*schema.ResourceData{d}, nil
}
<|endoftext|>


<commit_before>package google

import (
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/schema"
	"google.golang.org/api/compute/v1"
	"regexp"
	"strings"
)

var (
	computeAddressIdTemplate = "projects/%s/regions/%s/addresses/%s"
	computeAddressLinkRegex  = regexp.MustCompile("projects/(.+)/regions/(.+)/addresses/(.+)$")
)

func resourceComputeAddress() *schema.Resource {
	return &schema.Resource{
		Create: resourceComputeAddressCreate,
		Read:   resourceComputeAddressRead,
		Delete: resourceComputeAddressDelete,
		Importer: &schema.ResourceImporter{
			State: resourceComputeAddressImportState,
		},

		SchemaVersion: 1,
		MigrateState:  resourceComputeAddressMigrateState,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"address": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"project": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"region": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},

			"self_link": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceComputeAddressCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	region, err := getRegion(d, config)
	if err != nil {
		return err
	}

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	// Build the address parameter
	addr := &compute.Address{Name: d.Get("name").(string)}
	op, err := config.clientCompute.Addresses.Insert(
		project, region, addr).Do()
	if err != nil {
		return fmt.Errorf("Error creating address: %s", err)
	}

	// It probably maybe worked, so store the ID now
	d.SetId(computeAddressId{
		Project: project,
		Region:  region,
		Name:    addr.Name,
	}.canonicalId())

	err = computeOperationWait(config, op, project, "Creating Address")
	if err != nil {
		return err
	}

	return resourceComputeAddressRead(d, meta)
}

func resourceComputeAddressRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	addressId, err := parseComputeAddressId(d.Id(), config)
	if err != nil {
		return err
	}

	addr, err := config.clientCompute.Addresses.Get(
		addressId.Project, addressId.Region, addressId.Name).Do()
	if err != nil {
		return handleNotFoundError(err, d, fmt.Sprintf("Address %q", d.Get("name").(string)))
	}

	d.Set("address", addr.Address)
	d.Set("self_link", addr.SelfLink)
	d.Set("name", addr.Name)
	d.Set("region", addr.Region)

	return nil
}

func resourceComputeAddressDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	addressId, err := parseComputeAddressId(d.Id(), config)
	if err != nil {
		return err
	}

	// Delete the address
	log.Printf("[DEBUG] address delete request")
	op, err := config.clientCompute.Addresses.Delete(
		addressId.Project, addressId.Region, addressId.Name).Do()
	if err != nil {
		return fmt.Errorf("Error deleting address: %s", err)
	}

	err = computeOperationWait(config, op, addressId.Project, "Deleting Address")
	if err != nil {
		return err
	}

	d.SetId("")
	return nil
}

func resourceComputeAddressImportState(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	config := meta.(*Config)

	addressId, err := parseComputeAddressId(d.Id(), config)
	if err != nil {
		return nil, err
	}

	d.SetId(addressId.canonicalId())

	return []*schema.ResourceData{d}, nil
}

type computeAddressId struct {
	Project string
	Region  string
	Name    string
}

func (s computeAddressId) canonicalId() string {
	return fmt.Sprintf(computeAddressIdTemplate, s.Project, s.Region, s.Name)
}

func parseComputeAddressId(id string, config *Config) (*computeAddressId, error) {
	var parts []string
	if computeAddressLinkRegex.MatchString(id) {
		parts = computeAddressLinkRegex.FindStringSubmatch(id)

		return &computeAddressId{
			Project: parts[1],
			Region:  parts[2],
			Name:    parts[3],
		}, nil
	} else {
		parts = strings.Split(id, "/")
	}

	if len(parts) == 3 {
		return &computeAddressId{
			Project: parts[0],
			Region:  parts[1],
			Name:    parts[2],
		}, nil
	} else if len(parts) == 2 {
		// Project is optional.
		if config.Project == "" {
			return nil, fmt.Errorf("The default project for the provider must be set when using the `{region}/{name}` id format.")
		}

		return &computeAddressId{
			Project: config.Project,
			Region:  parts[0],
			Name:    parts[1],
		}, nil
	} else if len(parts) == 1 {
		// Project and region is optional
		if config.Project == "" {
			return nil, fmt.Errorf("The default project for the provider must be set when using the `{name}` id format.")
		}
		if config.Region == "" {
			return nil, fmt.Errorf("The default region for the provider must be set when using the `{name}` id format.")
		}

		return &computeAddressId{
			Project: config.Project,
			Region:  config.Region,
			Name:    parts[0],
		}, nil
	}

	return nil, fmt.Errorf("Invalid compute address id. Expecting resource link, `{project}/{region}/{name}`, `{region}/{name}` or `{name}` format.")
}
<commit_msg>Save region name-only instead of the self-link in compute_address (#422)<commit_after>package google

import (
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/schema"
	"google.golang.org/api/compute/v1"
	"regexp"
	"strings"
)

var (
	computeAddressIdTemplate = "projects/%s/regions/%s/addresses/%s"
	computeAddressLinkRegex  = regexp.MustCompile("projects/(.+)/regions/(.+)/addresses/(.+)$")
)

func resourceComputeAddress() *schema.Resource {
	return &schema.Resource{
		Create: resourceComputeAddressCreate,
		Read:   resourceComputeAddressRead,
		Delete: resourceComputeAddressDelete,
		Importer: &schema.ResourceImporter{
			State: resourceComputeAddressImportState,
		},

		SchemaVersion: 1,
		MigrateState:  resourceComputeAddressMigrateState,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"address": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"project": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"region": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},

			"self_link": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceComputeAddressCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	region, err := getRegion(d, config)
	if err != nil {
		return err
	}

	project, err := getProject(d, config)
	if err != nil {
		return err
	}

	// Build the address parameter
	addr := &compute.Address{Name: d.Get("name").(string)}
	op, err := config.clientCompute.Addresses.Insert(
		project, region, addr).Do()
	if err != nil {
		return fmt.Errorf("Error creating address: %s", err)
	}

	// It probably maybe worked, so store the ID now
	d.SetId(computeAddressId{
		Project: project,
		Region:  region,
		Name:    addr.Name,
	}.canonicalId())

	err = computeOperationWait(config, op, project, "Creating Address")
	if err != nil {
		return err
	}

	return resourceComputeAddressRead(d, meta)
}

func resourceComputeAddressRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	addressId, err := parseComputeAddressId(d.Id(), config)
	if err != nil {
		return err
	}

	addr, err := config.clientCompute.Addresses.Get(
		addressId.Project, addressId.Region, addressId.Name).Do()
	if err != nil {
		return handleNotFoundError(err, d, fmt.Sprintf("Address %q", d.Get("name").(string)))
	}

	d.Set("address", addr.Address)
	d.Set("self_link", addr.SelfLink)
	d.Set("name", addr.Name)
	d.Set("region", GetResourceNameFromSelfLink(addr.Region))

	return nil
}

func resourceComputeAddressDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	addressId, err := parseComputeAddressId(d.Id(), config)
	if err != nil {
		return err
	}

	// Delete the address
	log.Printf("[DEBUG] address delete request")
	op, err := config.clientCompute.Addresses.Delete(
		addressId.Project, addressId.Region, addressId.Name).Do()
	if err != nil {
		return fmt.Errorf("Error deleting address: %s", err)
	}

	err = computeOperationWait(config, op, addressId.Project, "Deleting Address")
	if err != nil {
		return err
	}

	d.SetId("")
	return nil
}

func resourceComputeAddressImportState(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	config := meta.(*Config)

	addressId, err := parseComputeAddressId(d.Id(), config)
	if err != nil {
		return nil, err
	}

	d.SetId(addressId.canonicalId())

	return []*schema.ResourceData{d}, nil
}

type computeAddressId struct {
	Project string
	Region  string
	Name    string
}

func (s computeAddressId) canonicalId() string {
	return fmt.Sprintf(computeAddressIdTemplate, s.Project, s.Region, s.Name)
}

func parseComputeAddressId(id string, config *Config) (*computeAddressId, error) {
	var parts []string
	if computeAddressLinkRegex.MatchString(id) {
		parts = computeAddressLinkRegex.FindStringSubmatch(id)

		return &computeAddressId{
			Project: parts[1],
			Region:  parts[2],
			Name:    parts[3],
		}, nil
	} else {
		parts = strings.Split(id, "/")
	}

	if len(parts) == 3 {
		return &computeAddressId{
			Project: parts[0],
			Region:  parts[1],
			Name:    parts[2],
		}, nil
	} else if len(parts) == 2 {
		// Project is optional.
		if config.Project == "" {
			return nil, fmt.Errorf("The default project for the provider must be set when using the `{region}/{name}` id format.")
		}

		return &computeAddressId{
			Project: config.Project,
			Region:  parts[0],
			Name:    parts[1],
		}, nil
	} else if len(parts) == 1 {
		// Project and region is optional
		if config.Project == "" {
			return nil, fmt.Errorf("The default project for the provider must be set when using the `{name}` id format.")
		}
		if config.Region == "" {
			return nil, fmt.Errorf("The default region for the provider must be set when using the `{name}` id format.")
		}

		return &computeAddressId{
			Project: config.Project,
			Region:  config.Region,
			Name:    parts[0],
		}, nil
	}

	return nil, fmt.Errorf("Invalid compute address id. Expecting resource link, `{project}/{region}/{name}`, `{region}/{name}` or `{name}` format.")
}
<|endoftext|>


<commit_before>/*
Package lint implements Vale's syntax-aware linting functionality.

The package is split into core linting logic (this file), source code
(code.go), and markup (markup.go). The general flow is as follows:

    Lint (files and directories)     LintString (stdin)
                \                   /
                 lintFiles         /
                         \        /
                          +      +
    +-------------------+ lintFile ------+|lintMarkdown|lintADoc|lintRST
    |                    /    |    \       |            |       /
    |                   /     |     \      |           /       /
    |                  /      |      \     |          +--------
    |                 /       |       \    |         /
    |                +        +        +   +        +
    |               lintCode  lintLines  lintHTML
    |               |         |              |
    |               |         |              +
    |                \        |         lintProse
    |                 \       |        /
    |                  +      +       +
    |                      lintText
    |   <= add Alerts{}       |
    +-------------------------+
*/
package lint

import (
	"errors"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	"github.com/errata-ai/vale/check"
	"github.com/errata-ai/vale/config"
	"github.com/errata-ai/vale/core"
	"github.com/remeh/sizedwaitgroup"
)

// A Linter lints a File.
type Linter struct {
	Manager *check.Manager

	seen      map[string]bool
	glob      *core.Glob
	nonGlobal bool
}

type lintResult struct {
	file *core.File
	err  error
}

// NewLinter initializes a Linter.
func NewLinter(cfg *config.Config) (*Linter, error) {
	mgr, err := check.NewManager(cfg)

	globalStyles := len(cfg.GBaseStyles)
	globalChecks := len(cfg.GChecks)

	return &Linter{
		Manager: mgr,

		seen:      make(map[string]bool),
		nonGlobal: globalStyles+globalChecks == 0}, err
}

// LintString src according to its format.
func (l *Linter) LintString(src string) ([]*core.File, error) {
	linted := l.lintFile(src)
	return []*core.File{linted.file}, linted.err
}

// Lint src according to its format.
func (l *Linter) Lint(input []string, pat string) ([]*core.File, error) {
	var linted []*core.File

	done := make(chan core.File)
	defer close(done)

	if err := l.setupContent(); err != nil {
		return linted, err
	}

	gp, err := core.NewGlob(pat)
	if err != nil {
		return linted, err
	}

	l.glob = &gp
	for _, src := range input {
		filesChan, errChan := l.lintFiles(done, src)

		for result := range filesChan {
			if result.err != nil {
				return linted, result.err
			} else if l.Manager.Config.Normalize {
				result.file.Path = filepath.ToSlash(result.file.Path)
			}
			linted = append(linted, result.file)
		}

		if err := <-errChan; err != nil {
			return linted, err
		}
	}

	return linted, nil
}

// lintFiles walks the `root` directory, creating a new goroutine to lint any
// file that matches the given glob pattern.
func (l *Linter) lintFiles(done <-chan core.File, root string) (<-chan lintResult, <-chan error) {
	filesChan := make(chan lintResult)
	errChan := make(chan error, 1)

	go func() {
		wg := sizedwaitgroup.New(5)

		err := filepath.Walk(root, func(fp string, fi os.FileInfo, err error) error {

			if fi.IsDir() && core.ShouldIgnoreDirectory(fi.Name()) {
				return filepath.SkipDir
			}

			if err != nil || fi.IsDir() {
				return nil
			} else if l.skip(fp) {
				return nil
			}

			wg.Add()
			go func(fp string) {
				select {
				case filesChan <- l.lintFile(fp):
				case <-done:
				}
				wg.Done()
			}(fp)

			// Abort the walk if done is closed.
			select {
			case <-done:
				return errors.New("walk canceled")
			default:
				return nil
			}
		})

		// Walk has returned, so all calls to wg.Add are done.  Start a
		// goroutine to close c once all the sends are done.
		go func() {
			wg.Wait()
			close(filesChan)
		}()
		errChan <- err
	}()

	return filesChan, errChan
}

// lintFile creates a new `File` from the path `src` and selects a linter based
// on its format.
func (l *Linter) lintFile(src string) lintResult {
	var err error

	file, err := core.NewFile(src, l.Manager.Config)
	if err != nil {
		return lintResult{err: err}
	} else if len(file.Checks) == 0 && len(file.BaseStyles) == 0 {
		if len(l.Manager.Config.GBaseStyles) == 0 && len(l.Manager.Config.GChecks) == 0 {
			// There's nothing to do; bail early.
			return lintResult{file: file}
		}
	}

	if file.Format == "markup" && !l.Manager.Config.Simple {
		switch file.NormedExt {
		case ".adoc":
			err = l.lintADoc(file)
		case ".md":
			err = l.lintMarkdown(file)
		case ".rst":
			err = l.lintRST(file)
		case ".xml":
			err = l.lintXML(file)
		case ".dita":
			err = l.lintDITA(file)
		case ".html":
			l.lintHTML(file)
		}
	} else if file.Format == "code" && !l.Manager.Config.Simple {
		l.lintCode(file)
	} else {
		l.lintLines(file)
	}

	return lintResult{file, err}
}

func (l *Linter) lintProse(f *core.File, ctx, txt, raw string, lnTotal, lnLength int) {
	var b core.Block

	text := core.PrepText(txt)
	rawText := core.PrepText(raw)

	if l.Manager.HasScope("paragraph") || l.Manager.HasScope("sentence") {
		senScope := "sentence" + f.RealExt
		hasCtx := ctx != ""
		for _, p := range strings.SplitAfter(text, "\n\n") {
			for _, s := range core.SentenceTokenizer.Tokenize(p) {
				sent := strings.TrimSpace(s)
				if hasCtx {
					b = core.NewBlock(ctx, sent, "", senScope)
				} else {
					b = core.NewBlock(p, sent, "", senScope)
				}
				l.lintText(f, b, lnTotal, lnLength)
			}
			l.lintText(
				f,
				core.NewBlock(ctx, p, "", "paragraph"+f.RealExt),
				lnTotal,
				lnLength)
		}
	}

	l.lintText(
		f,
		core.NewBlock(ctx, text, rawText, "text"+f.RealExt),
		lnTotal,
		lnLength)
}

func (l *Linter) lintLines(f *core.File) {
	var line string
	lines := 1
	for f.Scanner.Scan() {
		line = core.PrepText(f.Scanner.Text() + "\n")
		l.lintText(f, core.NewBlock("", line, "", "text"+f.RealExt), lines+1, 0)
		lines++
	}
}

func (l *Linter) lintText(f *core.File, blk core.Block, lines int, pad int) {
	var txt string
	var wg sync.WaitGroup

	f.ChkToCtx = make(map[string]string)
	hasCode := core.StringInSlice(f.NormedExt, []string{".md", ".adoc", ".rst"})

	results := make(chan core.Alert)
	for name, chk := range l.Manager.Rules() {
		if chk.Fields().Code && hasCode && !l.Manager.Config.Simple {
			txt = blk.Raw
		} else {
			txt = blk.Text
		}

		if !l.shouldRun(name, f, chk, blk) {
			continue
		}

		wg.Add(1)
		go func(txt, name string, f *core.File, chk check.Rule) {
			info := chk.Fields()
			for _, a := range chk.Run(txt, f) {
				core.FormatAlert(&a, info.Limit, info.Level, name)
				results <- a
			}
			wg.Done()
		}(txt, name, f, chk)
	}

	go func() {
		wg.Wait()
		close(results)
	}()

	for a := range results {
		f.AddAlert(a, blk, lines, pad)
	}
}

func (l *Linter) shouldRun(name string, f *core.File, chk check.Rule, blk core.Block) bool {
	min := l.Manager.Config.MinAlertLevel
	run := false

	details := chk.Fields()
	if strings.Count(name, ".") > 1 {
		// NOTE: This fixes the loading issue with consistency checks.
		//
		// See #129.
		list := strings.Split(name, ".")
		name = strings.Join([]string{list[0], list[1]}, ".")
	}

	// It has been disabled via an in-text comment.
	if f.QueryComments(name) {
		return false
	} else if core.LevelToInt[details.Level] < min {
		return false
	} else if !blk.Scope.ContainsString(details.Scope) {
		return false
	}

	// Has the check been disabled for this extension?
	if val, ok := f.Checks[name]; ok && !run {
		if !val {
			return false
		}
		run = true
	}

	// Has the check been disabled for all extensions?
	if val, ok := l.Manager.Config.GChecks[name]; ok && !run {
		if !val {
			return false
		}
		run = true
	}

	style := strings.Split(name, ".")[0]
	if !run && !core.StringInSlice(style, f.BaseStyles) {
		return false
	}

	return true
}

// setupContent handles any necessary building, compiling, or pre-processing.
func (l *Linter) setupContent() error {
	if l.Manager.Config.SphinxAuto != "" {
		parts := strings.Split(l.Manager.Config.SphinxAuto, " ")
		return exec.Command(parts[0], parts[1:]...).Run()
	}
	return nil
}

func (l *Linter) match(s string) bool {
	if l.glob == nil {
		return true
	}
	return l.glob.Match(s)
}

func (l *Linter) skip(fp string) bool {
	var ext string

	old := filepath.Ext(fp)
	if normed, found := l.Manager.Config.Formats[strings.Trim(old, ".")]; found {
		ext = "." + normed
		fp = fp[0:len(fp)-len(old)] + ext
	} else {
		ext = old
	}

	if status, found := l.seen[ext]; found && status {
		return true
	} else if !l.match(fp) {
		l.seen[ext] = true
		return true
	} else if l.nonGlobal {
		found := false
		for _, pat := range l.Manager.Config.SecToPat {
			if pat.Match(fp) {
				found = true
			}
		}
		if !found {
			l.seen[ext] = true
			return true
		}
	}

	return false
}
<commit_msg>perf: don't split on lines with `--ignore-syntax`<commit_after>/*
Package lint implements Vale's syntax-aware linting functionality.

The package is split into core linting logic (this file), source code
(code.go), and markup (markup.go). The general flow is as follows:

    Lint (files and directories)     LintString (stdin)
                \                   /
                 lintFiles         /
                         \        /
                          +      +
    +-------------------+ lintFile ------+|lintMarkdown|lintADoc|lintRST
    |                    /    |    \       |            |       /
    |                   /     |     \      |           /       /
    |                  /      |      \     |          +--------
    |                 /       |       \    |         /
    |                +        +        +   +        +
    |               lintCode  lintLines  lintHTML
    |               |         |              |
    |               |         |              +
    |                \        |         lintProse
    |                 \       |        /
    |                  +      +       +
    |                      lintText
    |   <= add Alerts{}       |
    +-------------------------+
*/
package lint

import (
	"errors"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	"github.com/errata-ai/vale/check"
	"github.com/errata-ai/vale/config"
	"github.com/errata-ai/vale/core"
	"github.com/remeh/sizedwaitgroup"
)

// A Linter lints a File.
type Linter struct {
	Manager *check.Manager

	seen      map[string]bool
	glob      *core.Glob
	nonGlobal bool
}

type lintResult struct {
	file *core.File
	err  error
}

// NewLinter initializes a Linter.
func NewLinter(cfg *config.Config) (*Linter, error) {
	mgr, err := check.NewManager(cfg)

	globalStyles := len(cfg.GBaseStyles)
	globalChecks := len(cfg.GChecks)

	return &Linter{
		Manager: mgr,

		seen:      make(map[string]bool),
		nonGlobal: globalStyles+globalChecks == 0}, err
}

// LintString src according to its format.
func (l *Linter) LintString(src string) ([]*core.File, error) {
	linted := l.lintFile(src)
	return []*core.File{linted.file}, linted.err
}

// Lint src according to its format.
func (l *Linter) Lint(input []string, pat string) ([]*core.File, error) {
	var linted []*core.File

	done := make(chan core.File)
	defer close(done)

	if err := l.setupContent(); err != nil {
		return linted, err
	}

	gp, err := core.NewGlob(pat)
	if err != nil {
		return linted, err
	}

	l.glob = &gp
	for _, src := range input {
		filesChan, errChan := l.lintFiles(done, src)

		for result := range filesChan {
			if result.err != nil {
				return linted, result.err
			} else if l.Manager.Config.Normalize {
				result.file.Path = filepath.ToSlash(result.file.Path)
			}
			linted = append(linted, result.file)
		}

		if err := <-errChan; err != nil {
			return linted, err
		}
	}

	return linted, nil
}

// lintFiles walks the `root` directory, creating a new goroutine to lint any
// file that matches the given glob pattern.
func (l *Linter) lintFiles(done <-chan core.File, root string) (<-chan lintResult, <-chan error) {
	filesChan := make(chan lintResult)
	errChan := make(chan error, 1)

	go func() {
		wg := sizedwaitgroup.New(5)

		err := filepath.Walk(root, func(fp string, fi os.FileInfo, err error) error {

			if fi.IsDir() && core.ShouldIgnoreDirectory(fi.Name()) {
				return filepath.SkipDir
			}

			if err != nil || fi.IsDir() {
				return nil
			} else if l.skip(fp) {
				return nil
			}

			wg.Add()
			go func(fp string) {
				select {
				case filesChan <- l.lintFile(fp):
				case <-done:
				}
				wg.Done()
			}(fp)

			// Abort the walk if done is closed.
			select {
			case <-done:
				return errors.New("walk canceled")
			default:
				return nil
			}
		})

		// Walk has returned, so all calls to wg.Add are done.  Start a
		// goroutine to close c once all the sends are done.
		go func() {
			wg.Wait()
			close(filesChan)
		}()
		errChan <- err
	}()

	return filesChan, errChan
}

// lintFile creates a new `File` from the path `src` and selects a linter based
// on its format.
func (l *Linter) lintFile(src string) lintResult {
	var err error

	file, err := core.NewFile(src, l.Manager.Config)
	if err != nil {
		return lintResult{err: err}
	} else if len(file.Checks) == 0 && len(file.BaseStyles) == 0 {
		if len(l.Manager.Config.GBaseStyles) == 0 && len(l.Manager.Config.GChecks) == 0 {
			// There's nothing to do; bail early.
			return lintResult{file: file}
		}
	}

	if file.Format == "markup" && !l.Manager.Config.Simple {
		switch file.NormedExt {
		case ".adoc":
			err = l.lintADoc(file)
		case ".md":
			err = l.lintMarkdown(file)
		case ".rst":
			err = l.lintRST(file)
		case ".xml":
			err = l.lintXML(file)
		case ".dita":
			err = l.lintDITA(file)
		case ".html":
			l.lintHTML(file)
		}
	} else if file.Format == "code" && !l.Manager.Config.Simple {
		l.lintCode(file)
	} else {
		l.lintLines(file)
	}

	return lintResult{file, err}
}

func (l *Linter) lintProse(f *core.File, ctx, txt, raw string, lnTotal, lnLength int) {
	var b core.Block

	text := core.PrepText(txt)
	rawText := core.PrepText(raw)

	if l.Manager.HasScope("paragraph") || l.Manager.HasScope("sentence") {
		senScope := "sentence" + f.RealExt
		hasCtx := ctx != ""
		for _, p := range strings.SplitAfter(text, "\n\n") {
			for _, s := range core.SentenceTokenizer.Tokenize(p) {
				sent := strings.TrimSpace(s)
				if hasCtx {
					b = core.NewBlock(ctx, sent, "", senScope)
				} else {
					b = core.NewBlock(p, sent, "", senScope)
				}
				l.lintText(f, b, lnTotal, lnLength)
			}
			l.lintText(
				f,
				core.NewBlock(ctx, p, "", "paragraph"+f.RealExt),
				lnTotal,
				lnLength)
		}
	}

	l.lintText(
		f,
		core.NewBlock(ctx, text, rawText, "text"+f.RealExt),
		lnTotal,
		lnLength)
}

func (l *Linter) lintLines(f *core.File) {
	block := core.NewBlock("", f.Content, "", "text"+f.RealExt)
	l.lintText(f, block, len(f.Lines), 0)
}

func (l *Linter) lintText(f *core.File, blk core.Block, lines int, pad int) {
	var txt string
	var wg sync.WaitGroup

	f.ChkToCtx = make(map[string]string)
	hasCode := core.StringInSlice(f.NormedExt, []string{".md", ".adoc", ".rst"})

	results := make(chan core.Alert)
	for name, chk := range l.Manager.Rules() {
		if chk.Fields().Code && hasCode && !l.Manager.Config.Simple {
			txt = blk.Raw
		} else {
			txt = blk.Text
		}

		if !l.shouldRun(name, f, chk, blk) {
			continue
		}

		wg.Add(1)
		go func(txt, name string, f *core.File, chk check.Rule) {
			info := chk.Fields()
			for _, a := range chk.Run(txt, f) {
				core.FormatAlert(&a, info.Limit, info.Level, name)
				results <- a
			}
			wg.Done()
		}(txt, name, f, chk)
	}

	go func() {
		wg.Wait()
		close(results)
	}()

	for a := range results {
		f.AddAlert(a, blk, lines, pad)
	}
}

func (l *Linter) shouldRun(name string, f *core.File, chk check.Rule, blk core.Block) bool {
	min := l.Manager.Config.MinAlertLevel
	run := false

	details := chk.Fields()
	if strings.Count(name, ".") > 1 {
		// NOTE: This fixes the loading issue with consistency checks.
		//
		// See #129.
		list := strings.Split(name, ".")
		name = strings.Join([]string{list[0], list[1]}, ".")
	}

	// It has been disabled via an in-text comment.
	if f.QueryComments(name) {
		return false
	} else if core.LevelToInt[details.Level] < min {
		return false
	} else if !blk.Scope.ContainsString(details.Scope) {
		return false
	}

	// Has the check been disabled for this extension?
	if val, ok := f.Checks[name]; ok && !run {
		if !val {
			return false
		}
		run = true
	}

	// Has the check been disabled for all extensions?
	if val, ok := l.Manager.Config.GChecks[name]; ok && !run {
		if !val {
			return false
		}
		run = true
	}

	style := strings.Split(name, ".")[0]
	if !run && !core.StringInSlice(style, f.BaseStyles) {
		return false
	}

	return true
}

// setupContent handles any necessary building, compiling, or pre-processing.
func (l *Linter) setupContent() error {
	if l.Manager.Config.SphinxAuto != "" {
		parts := strings.Split(l.Manager.Config.SphinxAuto, " ")
		return exec.Command(parts[0], parts[1:]...).Run()
	}
	return nil
}

func (l *Linter) match(s string) bool {
	if l.glob == nil {
		return true
	}
	return l.glob.Match(s)
}

func (l *Linter) skip(fp string) bool {
	var ext string

	old := filepath.Ext(fp)
	if normed, found := l.Manager.Config.Formats[strings.Trim(old, ".")]; found {
		ext = "." + normed
		fp = fp[0:len(fp)-len(old)] + ext
	} else {
		ext = old
	}

	if status, found := l.seen[ext]; found && status {
		return true
	} else if !l.match(fp) {
		l.seen[ext] = true
		return true
	} else if l.nonGlobal {
		found := false
		for _, pat := range l.Manager.Config.SecToPat {
			if pat.Match(fp) {
				found = true
			}
		}
		if !found {
			l.seen[ext] = true
			return true
		}
	}

	return false
}
<|endoftext|>