<commit_before>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mysqlctl

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"

	"code.google.com/p/vitess/go/ioutil2"
	"code.google.com/p/vitess/go/relog"
)

// These methods deal with cloning a running instance of mysql.

const (
	maxLagSeconds = 5
)

const (
	replicaSourceFile = "replica_source.json"
)

// Validate that this instance is a reasonable source of data.
// FIXME(msolomon) Provide a hook to call out to a program to decide.
// Is this an option to vtaction? Or look for $VTROOT/bin/vthook_validate_clone_source?
// What environment variables do we have to provide? dbname, host, socket?
func (mysqld *Mysqld) ValidateCloneSource() error {
	slaveStatus, err := mysqld.slaveStatus()
	if err != nil {
		if err != ERR_NOT_SLAVE {
			return fmt.Errorf("mysqlctl: ValidateCloneSource failed, %v", err)
		}
	} else {
		lagSeconds, _ := strconv.Atoi(slaveStatus["seconds_behind_master"])
		if lagSeconds > maxLagSeconds {
			return fmt.Errorf("mysqlctl: ValidateCloneSource failed, lag_seconds exceed maximum tolerance (%v)", lagSeconds)
		}
	}
	// FIXME(msolomon) check free space based on an estimate of the current
	// size of the db files.
	// Also, check that we aren't already cloning/compressing or acting as a
	// source. Mysqld being down isn't enough, presumably that will be
	// restarted as soon as the snapshot is taken.
	return nil
}

func (mysqld *Mysqld) ValidateCloneTarget() error {
	rows, err := mysqld.fetchSuperQuery("SHOW DATABASES")
	if err != nil {
		return fmt.Errorf("mysqlctl: ValidateCloneTarget failed, %v", err)
	}

	for _, row := range rows {
		if strings.HasPrefix(row[0].(string), "vt_") {
			dbName := row[0].(string)
			tableRows, err := mysqld.fetchSuperQuery("SHOW TABLES FROM " + dbName)
			if err != nil {
				return fmt.Errorf("mysqlctl: ValidateCloneTarget failed, %v", err)
			} else if len(tableRows) == 0 {
				// no tables == empty db, all is well
				continue
			}
			return fmt.Errorf("mysqlctl: ValidateCloneTarget failed, found active db %v", dbName)
		}
	}

	return nil
}

func (mysqld *Mysqld) ValidateSplitReplicaTarget() error {
	return errors.New("unimplemented")
	rows, err := mysqld.fetchSuperQuery("SHOW PROCESSLIST")
	if err != nil {
		return err
	}
	if len(rows) > 4 {
		return errors.New("too many active db processes")
	}

	rows, err = mysqld.fetchSuperQuery("SHOW DATABASES")
	if err != nil {
		return err
	}

	// NOTE: we expect that database was already created during tablet
	// assignment.
	return nil
}

func compressFiles(srcDir, dstDir string) ([]DataFile, error) {
	dataFiles := make([]DataFile, 0, 128)
	fiList, err := ioutil.ReadDir(srcDir)
	if err != nil {
		return nil, err
	}
	for _, fi := range fiList {
		if !fi.IsDir() {
			srcPath := path.Join(srcDir, fi.Name())
			dstPath := path.Join(dstDir, fi.Name()+".gz")
			if err := compressFile(srcPath, dstPath); err != nil {
				return nil, err
			}
			hash, err := md5File(dstPath)
			if err != nil {
				return nil, err
			}
			dataFiles = append(dataFiles, DataFile{dstPath, hash})
			relog.Info("clone data ready %v:%v", dstPath, hash)
		}
	}
	return dataFiles, nil
}

func (mysqld *Mysqld) FindVtDatabases() ([]string, error) {
	fiList, err := ioutil.ReadDir(mysqld.config.DataDir)
	if err != nil {
		return nil, err
	}

	dbNames := make([]string, 0, 16)
	for _, fi := range fiList {
		if strings.HasSuffix(fi.Name(), "vt_") {
			dbNames = append(dbNames, fi.Name())
		}
	}
	return dbNames, nil
}

func (mysqld *Mysqld) createSnapshot(dbName, snapshotPath string) ([]DataFile, error) {
	// wrapErr := func(err error) error {
	// 	return fmt.Errorf("mysqlctl: createSnapshot failed: %v", err)
	// }

	// FIXME(msolomon) Must match patterns in mycnf - probably belongs
	// in there as derived paths.
	snapshotDataSrcPath := path.Join(snapshotPath, dataDir, dbName)
	snapshotInnodbDataSrcPath := path.Join(snapshotPath, innodbDataSubdir)
	snapshotInnodbLogSrcPath := path.Join(snapshotPath, innodbLogSubdir)
	// clean out and start fresh
	for _, _path := range []string{snapshotDataSrcPath, snapshotInnodbDataSrcPath, snapshotInnodbLogSrcPath} {
		if err := os.RemoveAll(_path); err != nil {
			return nil, err
		}
		if err := os.MkdirAll(_path, 0775); err != nil {
			return nil, err
		}
	}

	allDataFiles := make([]DataFile, 0, 128)

	dbDataDir := path.Join(mysqld.config.DataDir, dbName)
	dataFiles, err := compressFiles(dbDataDir, snapshotDataSrcPath)
	if err != nil {
		return nil, err
	}
	allDataFiles = append(allDataFiles, dataFiles...)

	dataFiles, err = compressFiles(mysqld.config.InnodbDataHomeDir, snapshotInnodbDataSrcPath)
	if err != nil {
		return nil, err
	}
	allDataFiles = append(allDataFiles, dataFiles...)

	dataFiles, err = compressFiles(mysqld.config.InnodbLogGroupHomeDir, snapshotInnodbLogSrcPath)
	if err != nil {
		return nil, err
	}
	allDataFiles = append(allDataFiles, dataFiles...)

	return allDataFiles, nil
}

// This function runs on the machine acting as the source for the clone.
//
// Check master/slave status
// Shutdown mysql
// Check paths for storing data
// Compress /vt/vt_[0-9a-f]+/data/vt_.+
// Compute md5() sums
// Place in /vt/clone_src where they will be served by http server (not rpc)
// Restart mysql
func (mysqld *Mysqld) CreateSnapshot(dbName, sourceAddr string, allowHierarchicalReplication bool) (replicaSource *ReplicaSource, err error) {
	if dbName == "" {
		return nil, errors.New("CreateSnapshot failed: no database name provided")
	}

	if err = mysqld.ValidateCloneSource(); err != nil {
		return nil, err
	}

	// If the source is a slave use the master replication position,
	// unless we are allowing hierachical replicas.
	masterAddr := ""
	replicationPosition, statusErr := mysqld.SlaveStatus()
	if statusErr != nil {
		if statusErr != ERR_NOT_SLAVE {
			// this is a real error
			return nil, statusErr
		}
		// we are really a master, so we need that position
		replicationPosition, statusErr = mysqld.MasterStatus()
		if statusErr != nil {
			return nil, statusErr
		}
		masterAddr = mysqld.Addr()
	} else {
		// we are a slave, check our replication strategy
		if allowHierarchicalReplication {
			masterAddr = mysqld.Addr()
		} else {
			masterAddr, err = mysqld.GetMasterAddr()
			if err != nil {
				return nil, err
			}
		}
	}

	// save initial state so we can restore on Start()
	slaveStartRequired := false
	if slaveStatus, slaveErr := mysqld.slaveStatus(); slaveErr == nil {
		slaveStartRequired = (slaveStatus["Slave_IO_Running"] == "Yes" && slaveStatus["Slave_SQL_Running"] == "Yes")
	}
	readOnly, err := mysqld.IsReadOnly()
	if err != nil {
		return nil, err
	}
	if err = Shutdown(mysqld, true); err != nil {
		return nil, err
	}

	var rs *ReplicaSource
	dataFiles, snapshotErr := mysqld.createSnapshot(dbName, mysqld.config.SnapshotDir)
	if snapshotErr != nil {
		relog.Error("CreateSnapshot failed: %v", snapshotErr)
	} else {
		rs = NewReplicaSource(sourceAddr, masterAddr, mysqld.replParams.Uname, mysqld.replParams.Pass,
			dbName, dataFiles, replicationPosition)
		rsFile := path.Join(mysqld.config.SnapshotDir, replicaSourceFile)
		if snapshotErr = writeJson(rsFile, rs); snapshotErr != nil {
			relog.Error("CreateSnapshot failed: %v", snapshotErr)
		}
	}

	// Try to restart mysqld regardless of snapshot success..
	if err = Start(mysqld); err != nil {
		return nil, err
	}

	// restore original mysqld state that we saved above
	if slaveStartRequired {
		if err = mysqld.StartSlave(); err != nil {
			return nil, err
		}
		// this should be quick, but we might as well just wait
		if err = mysqld.WaitForSlaveStart(SlaveStartDeadline); err != nil {
			return nil, err
		}
	}

	if err = mysqld.SetReadOnly(readOnly); err != nil {
		return nil, err
	}

	if snapshotErr != nil {
		return nil, snapshotErr
	}

	return rs, nil
}

func writeJson(filename string, x interface{}) error {
	data, err := json.MarshalIndent(x, "  ", "  ")
	if err != nil {
		return err
	}
	return ioutil2.WriteFileAtomic(filename, data, 0660)
}

func ReadReplicaSource(filename string) (*ReplicaSource, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	rs := new(ReplicaSource)
	if err = json.Unmarshal(data, rs); err != nil {
		return nil, fmt.Errorf("ReadReplicaSource failed: %v %v", filename, err)
	}
	return rs, nil
}

// This piece runs on the presumably empty machine acting as the target in the
// create replica action.
//
// validate target (self)
// shutdown_mysql()
// create temp data directory /vt/target/vt_<keyspace>
// copy compressed data files via HTTP
// verify md5sum of compressed files
// uncompress into /vt/vt_<target-uid>/data/vt_<keyspace>
// start_mysql()
// clean up compressed files
func (mysqld *Mysqld) RestoreFromSnapshot(replicaSource *ReplicaSource) error {
	if replicaSource == nil {
		return errors.New("RestoreFromSnapshot: nil replicaSource")
	}

	relog.Debug("ValidateCloneTarget")
	if err := mysqld.ValidateCloneTarget(); err != nil {
		return err
	}

	relog.Debug("Shutdown mysqld")
	if err := Shutdown(mysqld, true); err != nil {
		return err
	}

	relog.Debug("Fetch snapshot")
	if err := mysqld.fetchSnapshot(replicaSource); err != nil {
		return err
	}

	relog.Debug("Restart mysqld")
	if err := Start(mysqld); err != nil {
		return err
	}

	cmdList := StartReplicationCommands(replicaSource.ReplicationState)
	relog.Info("StartReplicationCommands %#v", cmdList)
	if err := mysqld.executeSuperQueryList(cmdList); err != nil {
		return err
	}

	return mysqld.WaitForSlaveStart(SlaveStartDeadline)
}

func (mysqld *Mysqld) fetchSnapshot(replicaSource *ReplicaSource) error {
	replicaDbPath := path.Join(mysqld.config.DataDir, replicaSource.DbName)

	cleanDirs := []string{mysqld.config.SnapshotDir, replicaDbPath,
		mysqld.config.InnodbDataHomeDir, mysqld.config.InnodbLogGroupHomeDir}

	// clean out and start fresh
	// FIXME(msolomon) this might be changed to allow partial recovery, but at that point
	// we are starting to reimplement rsync.
	for _, dir := range cleanDirs {
		if err := os.RemoveAll(dir); err != nil {
			return err
		}
		if err := os.MkdirAll(dir, 0775); err != nil {
			return err
		}
	}

	// FIXME(msolomon) parallelize
	// FIXME(msolomon) automatically retry a file transfer at least once
	// FIXME(msolomon) deadlines?
	for _, fi := range replicaSource.Files {
		relativePath := strings.SplitN(fi.Path, "/", 5)[4]
		gzFilename := path.Join(mysqld.config.SnapshotDir, relativePath)
		filename := path.Join(mysqld.config.TabletDir, relativePath)
		// trim .gz
		filename = filename[:len(filename)-3]

		// Ensure directory for final destination.
		dir, _ := path.Split(gzFilename)
		if err := os.MkdirAll(dir, 0775); err != nil {
			return err
		}

		furl := "http://" + replicaSource.Addr + fi.Path
		if err := fetchSnapshotUrl(furl, fi.Hash, gzFilename, filename); err != nil {
			return err
		}

		relog.Info("fetched snapshot file: %v", filename)
	}
	return nil
}

func fetchSnapshotUrl(srcUrl, srcHash, tmpFilename, dstFilename string) error {
	resp, err := http.Get(srcUrl)
	if resp.StatusCode != 200 {
		return errors.New("failed fetching " + srcUrl + ": " + resp.Status)
	}
	defer resp.Body.Close()

	// FIXME(msolomon) buffer output?
	file, err := os.OpenFile(tmpFilename, os.O_CREATE|os.O_WRONLY, 0660)
	if err != nil {
		return err
	}
	defer file.Close()

	if _, err = io.Copy(file, resp.Body); err != nil {
		return err
	}

	file.Close()
	hash, err := md5File(tmpFilename)
	if err != nil {
		return err
	}

	if srcHash != hash {
		return errors.New("hash mismatch for " + tmpFilename + ", " + srcHash + " != " + hash)
	}

	if err := uncompressFile(tmpFilename, dstFilename); err != nil {
		return err
	}

	if err := os.Remove(tmpFilename); err != nil {
		// don't stop the process for this error
		relog.Warning("failed to remove temp file: %v", err)
	}

	return nil
}
<commit_msg>fix snapshot so the data is consistent even when data is flowing into mysql<commit_after>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mysqlctl

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"

	"code.google.com/p/vitess/go/ioutil2"
	"code.google.com/p/vitess/go/relog"
)

// These methods deal with cloning a running instance of mysql.

const (
	maxLagSeconds = 5
)

const (
	replicaSourceFile = "replica_source.json"
)

// Validate that this instance is a reasonable source of data.
// FIXME(msolomon) Provide a hook to call out to a program to decide.
// Is this an option to vtaction? Or look for $VTROOT/bin/vthook_validate_clone_source?
// What environment variables do we have to provide? dbname, host, socket?
func (mysqld *Mysqld) ValidateCloneSource() error {
	slaveStatus, err := mysqld.slaveStatus()
	if err != nil {
		if err != ERR_NOT_SLAVE {
			return fmt.Errorf("mysqlctl: ValidateCloneSource failed, %v", err)
		}
	} else {
		lagSeconds, _ := strconv.Atoi(slaveStatus["seconds_behind_master"])
		if lagSeconds > maxLagSeconds {
			return fmt.Errorf("mysqlctl: ValidateCloneSource failed, lag_seconds exceed maximum tolerance (%v)", lagSeconds)
		}
	}
	// FIXME(msolomon) check free space based on an estimate of the current
	// size of the db files.
	// Also, check that we aren't already cloning/compressing or acting as a
	// source. Mysqld being down isn't enough, presumably that will be
	// restarted as soon as the snapshot is taken.
	return nil
}

func (mysqld *Mysqld) ValidateCloneTarget() error {
	rows, err := mysqld.fetchSuperQuery("SHOW DATABASES")
	if err != nil {
		return fmt.Errorf("mysqlctl: ValidateCloneTarget failed, %v", err)
	}

	for _, row := range rows {
		if strings.HasPrefix(row[0].(string), "vt_") {
			dbName := row[0].(string)
			tableRows, err := mysqld.fetchSuperQuery("SHOW TABLES FROM " + dbName)
			if err != nil {
				return fmt.Errorf("mysqlctl: ValidateCloneTarget failed, %v", err)
			} else if len(tableRows) == 0 {
				// no tables == empty db, all is well
				continue
			}
			return fmt.Errorf("mysqlctl: ValidateCloneTarget failed, found active db %v", dbName)
		}
	}

	return nil
}

func (mysqld *Mysqld) ValidateSplitReplicaTarget() error {
	return errors.New("unimplemented")
	rows, err := mysqld.fetchSuperQuery("SHOW PROCESSLIST")
	if err != nil {
		return err
	}
	if len(rows) > 4 {
		return errors.New("too many active db processes")
	}

	rows, err = mysqld.fetchSuperQuery("SHOW DATABASES")
	if err != nil {
		return err
	}

	// NOTE: we expect that database was already created during tablet
	// assignment.
	return nil
}

func compressFiles(srcDir, dstDir string) ([]DataFile, error) {
	dataFiles := make([]DataFile, 0, 128)
	fiList, err := ioutil.ReadDir(srcDir)
	if err != nil {
		return nil, err
	}
	for _, fi := range fiList {
		if !fi.IsDir() {
			srcPath := path.Join(srcDir, fi.Name())
			dstPath := path.Join(dstDir, fi.Name()+".gz")
			if err := compressFile(srcPath, dstPath); err != nil {
				return nil, err
			}
			hash, err := md5File(dstPath)
			if err != nil {
				return nil, err
			}
			dataFiles = append(dataFiles, DataFile{dstPath, hash})
			relog.Info("clone data ready %v:%v", dstPath, hash)
		}
	}
	return dataFiles, nil
}

func (mysqld *Mysqld) FindVtDatabases() ([]string, error) {
	fiList, err := ioutil.ReadDir(mysqld.config.DataDir)
	if err != nil {
		return nil, err
	}

	dbNames := make([]string, 0, 16)
	for _, fi := range fiList {
		if strings.HasSuffix(fi.Name(), "vt_") {
			dbNames = append(dbNames, fi.Name())
		}
	}
	return dbNames, nil
}

func (mysqld *Mysqld) createSnapshot(dbName, snapshotPath string) ([]DataFile, error) {
	// wrapErr := func(err error) error {
	// 	return fmt.Errorf("mysqlctl: createSnapshot failed: %v", err)
	// }

	// FIXME(msolomon) Must match patterns in mycnf - probably belongs
	// in there as derived paths.
	snapshotDataSrcPath := path.Join(snapshotPath, dataDir, dbName)
	snapshotInnodbDataSrcPath := path.Join(snapshotPath, innodbDataSubdir)
	snapshotInnodbLogSrcPath := path.Join(snapshotPath, innodbLogSubdir)
	// clean out and start fresh
	for _, _path := range []string{snapshotDataSrcPath, snapshotInnodbDataSrcPath, snapshotInnodbLogSrcPath} {
		if err := os.RemoveAll(_path); err != nil {
			return nil, err
		}
		if err := os.MkdirAll(_path, 0775); err != nil {
			return nil, err
		}
	}

	allDataFiles := make([]DataFile, 0, 128)

	dbDataDir := path.Join(mysqld.config.DataDir, dbName)
	dataFiles, err := compressFiles(dbDataDir, snapshotDataSrcPath)
	if err != nil {
		return nil, err
	}
	allDataFiles = append(allDataFiles, dataFiles...)

	dataFiles, err = compressFiles(mysqld.config.InnodbDataHomeDir, snapshotInnodbDataSrcPath)
	if err != nil {
		return nil, err
	}
	allDataFiles = append(allDataFiles, dataFiles...)

	dataFiles, err = compressFiles(mysqld.config.InnodbLogGroupHomeDir, snapshotInnodbLogSrcPath)
	if err != nil {
		return nil, err
	}
	allDataFiles = append(allDataFiles, dataFiles...)

	return allDataFiles, nil
}

// This function runs on the machine acting as the source for the clone.
//
// Check master/slave status and determine restore needs.
// If this instance is a slave, stop replication, otherwise place in read-only mode.
// Record replication position.
// Shutdown mysql
// Check paths for storing data
// Compress /vt/vt_[0-9a-f]+/data/vt_.+
// Compute md5() sums
// Place in /vt/clone_src where they will be served by http server (not rpc)
// Restart mysql
func (mysqld *Mysqld) CreateSnapshot(dbName, sourceAddr string, allowHierarchicalReplication bool) (replicaSource *ReplicaSource, err error) {
	if dbName == "" {
		return nil, errors.New("CreateSnapshot failed: no database name provided")
	}

	if err = mysqld.ValidateCloneSource(); err != nil {
		return nil, err
	}

	// save initial state so we can restore on Start()
	slaveStartRequired := false
	sourceIsMaster := false
	readOnly := true

	slaveStatus, slaveErr := mysqld.slaveStatus()
	if slaveErr == nil {
		slaveStartRequired = (slaveStatus["Slave_IO_Running"] == "Yes" && slaveStatus["Slave_SQL_Running"] == "Yes")
	} else if slaveErr == ERR_NOT_SLAVE {
		sourceIsMaster = true
	} else {
		// If we can't get any data, just fail.
		return nil, err
	}

	readOnly, err = mysqld.IsReadOnly()
	if err != nil {
		return nil, err
	}

	// Stop sources of writes so we can get a consistent replication position.
	// If the source is a slave use the master replication position
	// unless we are allowing hierachical replicas.
	masterAddr := ""
	var replicationPosition *ReplicationPosition
	if sourceIsMaster {
		if err = mysqld.SetReadOnly(true); err != nil {
			return nil, err
		}
		replicationPosition, err = mysqld.MasterStatus()
		if err != nil {
			return nil, err
		}
		masterAddr = mysqld.Addr()
	} else {
		if err = mysqld.StopSlave(); err != nil {
			return nil, err
		}
		replicationPosition, err = mysqld.SlaveStatus()
		if err != nil {
			return nil, err
		}
		// We are a slave, check our replication strategy before choosing
		// the master address.
		if allowHierarchicalReplication {
			masterAddr = mysqld.Addr()
		} else {
			masterAddr, err = mysqld.GetMasterAddr()
			if err != nil {
				return nil, err
			}
		}
	}

	if err = Shutdown(mysqld, true); err != nil {
		return nil, err
	}

	var rs *ReplicaSource
	dataFiles, snapshotErr := mysqld.createSnapshot(dbName, mysqld.config.SnapshotDir)
	if snapshotErr != nil {
		relog.Error("CreateSnapshot failed: %v", snapshotErr)
	} else {
		rs = NewReplicaSource(sourceAddr, masterAddr, mysqld.replParams.Uname, mysqld.replParams.Pass,
			dbName, dataFiles, replicationPosition)
		rsFile := path.Join(mysqld.config.SnapshotDir, replicaSourceFile)
		if snapshotErr = writeJson(rsFile, rs); snapshotErr != nil {
			relog.Error("CreateSnapshot failed: %v", snapshotErr)
		}
	}

	// Try to restart mysqld regardless of snapshot success.
	if err = Start(mysqld); err != nil {
		return nil, err
	}

	// Restore original mysqld state that we saved above.
	if slaveStartRequired {
		if err = mysqld.StartSlave(); err != nil {
			return nil, err
		}
		// this should be quick, but we might as well just wait
		if err = mysqld.WaitForSlaveStart(SlaveStartDeadline); err != nil {
			return nil, err
		}
	}

	if err = mysqld.SetReadOnly(readOnly); err != nil {
		return nil, err
	}

	if snapshotErr != nil {
		return nil, snapshotErr
	}

	return rs, nil
}

func writeJson(filename string, x interface{}) error {
	data, err := json.MarshalIndent(x, "  ", "  ")
	if err != nil {
		return err
	}
	return ioutil2.WriteFileAtomic(filename, data, 0660)
}

func ReadReplicaSource(filename string) (*ReplicaSource, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	rs := new(ReplicaSource)
	if err = json.Unmarshal(data, rs); err != nil {
		return nil, fmt.Errorf("ReadReplicaSource failed: %v %v", filename, err)
	}
	return rs, nil
}

// This piece runs on the presumably empty machine acting as the target in the
// create replica action.
//
// validate target (self)
// shutdown_mysql()
// create temp data directory /vt/target/vt_<keyspace>
// copy compressed data files via HTTP
// verify md5sum of compressed files
// uncompress into /vt/vt_<target-uid>/data/vt_<keyspace>
// start_mysql()
// clean up compressed files
func (mysqld *Mysqld) RestoreFromSnapshot(replicaSource *ReplicaSource) error {
	if replicaSource == nil {
		return errors.New("RestoreFromSnapshot: nil replicaSource")
	}

	relog.Debug("ValidateCloneTarget")
	if err := mysqld.ValidateCloneTarget(); err != nil {
		return err
	}

	relog.Debug("Shutdown mysqld")
	if err := Shutdown(mysqld, true); err != nil {
		return err
	}

	relog.Debug("Fetch snapshot")
	if err := mysqld.fetchSnapshot(replicaSource); err != nil {
		return err
	}

	relog.Debug("Restart mysqld")
	if err := Start(mysqld); err != nil {
		return err
	}

	cmdList := StartReplicationCommands(replicaSource.ReplicationState)
	relog.Info("StartReplicationCommands %#v", cmdList)
	if err := mysqld.executeSuperQueryList(cmdList); err != nil {
		return err
	}

	return mysqld.WaitForSlaveStart(SlaveStartDeadline)
}

func (mysqld *Mysqld) fetchSnapshot(replicaSource *ReplicaSource) error {
	replicaDbPath := path.Join(mysqld.config.DataDir, replicaSource.DbName)

	cleanDirs := []string{mysqld.config.SnapshotDir, replicaDbPath,
		mysqld.config.InnodbDataHomeDir, mysqld.config.InnodbLogGroupHomeDir}

	// clean out and start fresh
	// FIXME(msolomon) this might be changed to allow partial recovery, but at that point
	// we are starting to reimplement rsync.
	for _, dir := range cleanDirs {
		if err := os.RemoveAll(dir); err != nil {
			return err
		}
		if err := os.MkdirAll(dir, 0775); err != nil {
			return err
		}
	}

	// FIXME(msolomon) parallelize
	// FIXME(msolomon) automatically retry a file transfer at least once
	// FIXME(msolomon) deadlines?
	for _, fi := range replicaSource.Files {
		relativePath := strings.SplitN(fi.Path, "/", 5)[4]
		gzFilename := path.Join(mysqld.config.SnapshotDir, relativePath)
		filename := path.Join(mysqld.config.TabletDir, relativePath)
		// trim .gz
		filename = filename[:len(filename)-3]

		// Ensure directory for final destination.
		dir, _ := path.Split(gzFilename)
		if err := os.MkdirAll(dir, 0775); err != nil {
			return err
		}

		furl := "http://" + replicaSource.Addr + fi.Path
		if err := fetchSnapshotUrl(furl, fi.Hash, gzFilename, filename); err != nil {
			return err
		}

		relog.Info("fetched snapshot file: %v", filename)
	}
	return nil
}

func fetchSnapshotUrl(srcUrl, srcHash, tmpFilename, dstFilename string) error {
	resp, err := http.Get(srcUrl)
	if resp.StatusCode != 200 {
		return errors.New("failed fetching " + srcUrl + ": " + resp.Status)
	}
	defer resp.Body.Close()

	// FIXME(msolomon) buffer output?
	file, err := os.OpenFile(tmpFilename, os.O_CREATE|os.O_WRONLY, 0660)
	if err != nil {
		return err
	}
	defer file.Close()

	if _, err = io.Copy(file, resp.Body); err != nil {
		return err
	}

	file.Close()
	hash, err := md5File(tmpFilename)
	if err != nil {
		return err
	}

	if srcHash != hash {
		return errors.New("hash mismatch for " + tmpFilename + ", " + srcHash + " != " + hash)
	}

	if err := uncompressFile(tmpFilename, dstFilename); err != nil {
		return err
	}

	if err := os.Remove(tmpFilename); err != nil {
		// don't stop the process for this error
		relog.Warning("failed to remove temp file: %v", err)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package datastore

import (
	"fmt"
	"reflect"
	"strings"
	"sync"
	"unicode"
)

// Entities with more than this many indexed properties will not be saved.
const maxIndexedProperties = 5000

// []byte fields more than 1 megabyte long will not be loaded or saved.
const maxBlobLen = 1 << 20

// Property is a name/value pair plus some metadata. A datastore entity's
// contents are loaded and saved as a sequence of Properties. Each property
// name must be unique within an entity.
type Property struct {
	// Name is the property name.
	Name string
	// Value is the property value. The valid types are:
	//	- int64
	//	- bool
	//	- string
	//	- float64
	//	- *Key
	//	- time.Time
	//	- GeoPoint
	//	- []byte (up to 1 megabyte in length)
	// Value can also be:
	//	- []interface{} where each element is one of the above types
	// This set is smaller than the set of valid struct field types that the
	// datastore can load and save. A Value's type must be explicitly on
	// the list above; it is not sufficient for the underlying type to be
	// on that list. For example, a Value of "type myInt64 int64" is
	// invalid. Smaller-width integers and floats are also invalid. Again,
	// this is more restrictive than the set of valid struct field types.
	//
	// A Value will have an opaque type when loading entities from an index,
	// such as via a projection query. Load entities into a struct instead
	// of a PropertyLoadSaver when using a projection query.
	//
	// A Value may also be the nil interface value; this is equivalent to
	// Python's None but not directly representable by a Go struct. Loading
	// a nil-valued property into a struct will set that field to the zero
	// value.
	Value interface{}
	// NoIndex is whether the datastore cannot index this property.
	// If NoIndex is set to false, []byte and string values are limited to
	// 1500 bytes.
	NoIndex bool
}

// PropertyLoadSaver can be converted from and to a slice of Properties.
type PropertyLoadSaver interface {
	Load([]Property) error
	Save() ([]Property, error)
}

// PropertyList converts a []Property to implement PropertyLoadSaver.
type PropertyList []Property

var (
	typeOfPropertyLoadSaver = reflect.TypeOf((*PropertyLoadSaver)(nil)).Elem()
	typeOfPropertyList      = reflect.TypeOf(PropertyList(nil))
)

// Load loads all of the provided properties into l.
// It does not first reset *l to an empty slice.
func (l *PropertyList) Load(p []Property) error {
	*l = append(*l, p...)
	return nil
}

// Save saves all of l's properties as a slice of Properties.
func (l *PropertyList) Save() ([]Property, error) {
	return *l, nil
}

// validPropertyName returns whether name consists of one or more valid Go
// identifiers joined by ".".
func validPropertyName(name string) bool {
	if name == "" {
		return false
	}
	for _, s := range strings.Split(name, ".") {
		if s == "" {
			return false
		}
		first := true
		for _, c := range s {
			if first {
				first = false
				if c != '_' && !unicode.IsLetter(c) {
					return false
				}
			} else {
				if c != '_' && !unicode.IsLetter(c) && !unicode.IsDigit(c) {
					return false
				}
			}
		}
	}
	return true
}

// structTag is the parsed `datastore:"name,options"` tag of a struct field.
// If a field has no tag, or the tag has an empty name, then the structTag's
// name is just the field name. A "-" name means that the datastore ignores
// that field.
type structTag struct {
	name    string
	noIndex bool
}

// structCodec describes how to convert a struct to and from a sequence of
// properties.
type structCodec struct {
	// byIndex gives the structTag for the i'th field.
	byIndex []structTag
	// byName gives the field codec for the structTag with the given name.
	byName map[string]fieldCodec
	// hasSlice is whether a struct or any of its nested or embedded structs
	// has a slice-typed field (other than []byte).
	hasSlice bool
	// complete is whether the structCodec is complete. An incomplete
	// structCodec may be encountered when walking a recursive struct.
	complete bool
}

// fieldCodec is a struct field's index and, if that struct field's type is
// itself a struct, that substruct's structCodec.
type fieldCodec struct {
	index          int
	substructCodec *structCodec
}

// structCodecs collects the structCodecs that have already been calculated.
var (
	structCodecsMutex sync.Mutex
	structCodecs      = make(map[reflect.Type]*structCodec)
)

// getStructCodec returns the structCodec for the given struct type.
func getStructCodec(t reflect.Type) (*structCodec, error) {
	structCodecsMutex.Lock()
	defer structCodecsMutex.Unlock()
	return getStructCodecLocked(t)
}

// getStructCodecLocked implements getStructCodec. The structCodecsMutex must
// be held when calling this function.
func getStructCodecLocked(t reflect.Type) (ret *structCodec, retErr error) {
	c, ok := structCodecs[t]
	if ok {
		return c, nil
	}
	c = &structCodec{
		byIndex: make([]structTag, t.NumField()),
		byName:  make(map[string]fieldCodec),
	}

	// Add c to the structCodecs map before we are sure it is good. If t is
	// a recursive type, it needs to find the incomplete entry for itself in
	// the map.
	structCodecs[t] = c
	defer func() {
		if retErr != nil {
			delete(structCodecs, t)
		}
	}()

	for i := range c.byIndex {
		f := t.Field(i)
		name, opts := f.Tag.Get("datastore"), ""
		if i := strings.Index(name, ","); i != -1 {
			name, opts = name[:i], name[i+1:]
		}
		if name == "" {
			if !f.Anonymous {
				name = f.Name
			}
		} else if name == "-" {
			c.byIndex[i] = structTag{name: name}
			continue
		} else if !validPropertyName(name) {
			return nil, fmt.Errorf("datastore: struct tag has invalid property name: %q", name)
		}

		substructType, fIsSlice := reflect.Type(nil), false
		switch f.Type.Kind() {
		case reflect.Struct:
			substructType = f.Type
		case reflect.Slice:
			if f.Type.Elem().Kind() == reflect.Struct {
				substructType = f.Type.Elem()
			}
			fIsSlice = f.Type != typeOfByteSlice
			c.hasSlice = c.hasSlice || fIsSlice
		}

		if substructType != nil && substructType != typeOfTime && substructType != typeOfGeoPoint {
			if name != "" {
				name = name + "."
			}
			sub, err := getStructCodecLocked(substructType)
			if err != nil {
				return nil, err
			}
			if !sub.complete {
				return nil, fmt.Errorf("datastore: recursive struct: field %q", f.Name)
			}
			if fIsSlice && sub.hasSlice {
				return nil, fmt.Errorf(
					"datastore: flattening nested structs leads to a slice of slices: field %q", f.Name)
			}
			c.hasSlice = c.hasSlice || sub.hasSlice
			for relName := range sub.byName {
				absName := name + relName
				if _, ok := c.byName[absName]; ok {
					return nil, fmt.Errorf("datastore: struct tag has repeated property name: %q", absName)
				}
				c.byName[absName] = fieldCodec{index: i, substructCodec: sub}
			}
		} else {
			if _, ok := c.byName[name]; ok {
				return nil, fmt.Errorf("datastore: struct tag has repeated property name: %q", name)
			}
			c.byName[name] = fieldCodec{index: i}
		}

		c.byIndex[i] = structTag{
			name:    name,
			noIndex: opts == "noindex",
		}
	}
	c.complete = true
	return c, nil
}

// structPLS adapts a struct to be a PropertyLoadSaver.
type structPLS struct {
	v     reflect.Value
	codec *structCodec
}

// newStructPLS returns a PropertyLoadSaver for the struct pointer p.
func newStructPLS(p interface{}) (PropertyLoadSaver, error) {
	v := reflect.ValueOf(p)
	if v.Kind() != reflect.Ptr || v.Elem().Kind() != reflect.Struct {
		return nil, ErrInvalidEntityType
	}
	v = v.Elem()
	codec, err := getStructCodec(v.Type())
	if err != nil {
		return nil, err
	}
	return structPLS{v, codec}, nil
}

// LoadStruct loads the properties from p to dst.
// dst must be a struct pointer.
func LoadStruct(dst interface{}, p []Property) error {
	x, err := newStructPLS(dst)
	if err != nil {
		return err
	}
	return x.Load(p)
}

// SaveStruct returns the properties from src as a slice of Properties.
// src must be a struct pointer.
func SaveStruct(src interface{}) ([]Property, error) {
	x, err := newStructPLS(src)
	if err != nil {
		return nil, err
	}
	return x.Save()
}
<commit_msg>datastore: update maxIndexedProperties from 5000 to 20000<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package datastore

import (
	"fmt"
	"reflect"
	"strings"
	"sync"
	"unicode"
)

// Entities with more than this many indexed properties will not be saved.
const maxIndexedProperties = 20000

// []byte fields more than 1 megabyte long will not be loaded or saved.
const maxBlobLen = 1 << 20

// Property is a name/value pair plus some metadata. A datastore entity's
// contents are loaded and saved as a sequence of Properties. Each property
// name must be unique within an entity.
type Property struct {
	// Name is the property name.
	Name string
	// Value is the property value. The valid types are:
	//	- int64
	//	- bool
	//	- string
	//	- float64
	//	- *Key
	//	- time.Time
	//	- GeoPoint
	//	- []byte (up to 1 megabyte in length)
	// Value can also be:
	//	- []interface{} where each element is one of the above types
	// This set is smaller than the set of valid struct field types that the
	// datastore can load and save. A Value's type must be explicitly on
	// the list above; it is not sufficient for the underlying type to be
	// on that list. For example, a Value of "type myInt64 int64" is
	// invalid. Smaller-width integers and floats are also invalid. Again,
	// this is more restrictive than the set of valid struct field types.
	//
	// A Value will have an opaque type when loading entities from an index,
	// such as via a projection query. Load entities into a struct instead
	// of a PropertyLoadSaver when using a projection query.
	//
	// A Value may also be the nil interface value; this is equivalent to
	// Python's None but not directly representable by a Go struct. Loading
	// a nil-valued property into a struct will set that field to the zero
	// value.
	Value interface{}
	// NoIndex is whether the datastore cannot index this property.
	// If NoIndex is set to false, []byte and string values are limited to
	// 1500 bytes.
	NoIndex bool
}

// PropertyLoadSaver can be converted from and to a slice of Properties.
type PropertyLoadSaver interface {
	Load([]Property) error
	Save() ([]Property, error)
}

// PropertyList converts a []Property to implement PropertyLoadSaver.
type PropertyList []Property

var (
	typeOfPropertyLoadSaver = reflect.TypeOf((*PropertyLoadSaver)(nil)).Elem()
	typeOfPropertyList      = reflect.TypeOf(PropertyList(nil))
)

// Load loads all of the provided properties into l.
// It does not first reset *l to an empty slice.
func (l *PropertyList) Load(p []Property) error {
	*l = append(*l, p...)
	return nil
}

// Save saves all of l's properties as a slice of Properties.
func (l *PropertyList) Save() ([]Property, error) {
	return *l, nil
}

// validPropertyName returns whether name consists of one or more valid Go
// identifiers joined by ".".
func validPropertyName(name string) bool {
	if name == "" {
		return false
	}
	for _, s := range strings.Split(name, ".") {
		if s == "" {
			return false
		}
		first := true
		for _, c := range s {
			if first {
				first = false
				if c != '_' && !unicode.IsLetter(c) {
					return false
				}
			} else {
				if c != '_' && !unicode.IsLetter(c) && !unicode.IsDigit(c) {
					return false
				}
			}
		}
	}
	return true
}

// structTag is the parsed `datastore:"name,options"` tag of a struct field.
// If a field has no tag, or the tag has an empty name, then the structTag's
// name is just the field name. A "-" name means that the datastore ignores
// that field.
type structTag struct {
	name    string
	noIndex bool
}

// structCodec describes how to convert a struct to and from a sequence of
// properties.
type structCodec struct {
	// byIndex gives the structTag for the i'th field.
	byIndex []structTag
	// byName gives the field codec for the structTag with the given name.
	byName map[string]fieldCodec
	// hasSlice is whether a struct or any of its nested or embedded structs
	// has a slice-typed field (other than []byte).
	hasSlice bool
	// complete is whether the structCodec is complete. An incomplete
	// structCodec may be encountered when walking a recursive struct.
	complete bool
}

// fieldCodec is a struct field's index and, if that struct field's type is
// itself a struct, that substruct's structCodec.
type fieldCodec struct {
	index          int
	substructCodec *structCodec
}

// structCodecs collects the structCodecs that have already been calculated.
var (
	structCodecsMutex sync.Mutex
	structCodecs      = make(map[reflect.Type]*structCodec)
)

// getStructCodec returns the structCodec for the given struct type.
func getStructCodec(t reflect.Type) (*structCodec, error) {
	structCodecsMutex.Lock()
	defer structCodecsMutex.Unlock()
	return getStructCodecLocked(t)
}

// getStructCodecLocked implements getStructCodec. The structCodecsMutex must
// be held when calling this function.
func getStructCodecLocked(t reflect.Type) (ret *structCodec, retErr error) {
	c, ok := structCodecs[t]
	if ok {
		return c, nil
	}
	c = &structCodec{
		byIndex: make([]structTag, t.NumField()),
		byName:  make(map[string]fieldCodec),
	}

	// Add c to the structCodecs map before we are sure it is good. If t is
	// a recursive type, it needs to find the incomplete entry for itself in
	// the map.
	structCodecs[t] = c
	defer func() {
		if retErr != nil {
			delete(structCodecs, t)
		}
	}()

	for i := range c.byIndex {
		f := t.Field(i)
		name, opts := f.Tag.Get("datastore"), ""
		if i := strings.Index(name, ","); i != -1 {
			name, opts = name[:i], name[i+1:]
		}
		if name == "" {
			if !f.Anonymous {
				name = f.Name
			}
		} else if name == "-" {
			c.byIndex[i] = structTag{name: name}
			continue
		} else if !validPropertyName(name) {
			return nil, fmt.Errorf("datastore: struct tag has invalid property name: %q", name)
		}

		substructType, fIsSlice := reflect.Type(nil), false
		switch f.Type.Kind() {
		case reflect.Struct:
			substructType = f.Type
		case reflect.Slice:
			if f.Type.Elem().Kind() == reflect.Struct {
				substructType = f.Type.Elem()
			}
			fIsSlice = f.Type != typeOfByteSlice
			c.hasSlice = c.hasSlice || fIsSlice
		}

		if substructType != nil && substructType != typeOfTime && substructType != typeOfGeoPoint {
			if name != "" {
				name = name + "."
			}
			sub, err := getStructCodecLocked(substructType)
			if err != nil {
				return nil, err
			}
			if !sub.complete {
				return nil, fmt.Errorf("datastore: recursive struct: field %q", f.Name)
			}
			if fIsSlice && sub.hasSlice {
				return nil, fmt.Errorf(
					"datastore: flattening nested structs leads to a slice of slices: field %q", f.Name)
			}
			c.hasSlice = c.hasSlice || sub.hasSlice
			for relName := range sub.byName {
				absName := name + relName
				if _, ok := c.byName[absName]; ok {
					return nil, fmt.Errorf("datastore: struct tag has repeated property name: %q", absName)
				}
				c.byName[absName] = fieldCodec{index: i, substructCodec: sub}
			}
		} else {
			if _, ok := c.byName[name]; ok {
				return nil, fmt.Errorf("datastore: struct tag has repeated property name: %q", name)
			}
			c.byName[name] = fieldCodec{index: i}
		}

		c.byIndex[i] = structTag{
			name:    name,
			noIndex: opts == "noindex",
		}
	}
	c.complete = true
	return c, nil
}

// structPLS adapts a struct to be a PropertyLoadSaver.
type structPLS struct {
	v     reflect.Value
	codec *structCodec
}

// newStructPLS returns a PropertyLoadSaver for the struct pointer p.
func newStructPLS(p interface{}) (PropertyLoadSaver, error) {
	v := reflect.ValueOf(p)
	if v.Kind() != reflect.Ptr || v.Elem().Kind() != reflect.Struct {
		return nil, ErrInvalidEntityType
	}
	v = v.Elem()
	codec, err := getStructCodec(v.Type())
	if err != nil {
		return nil, err
	}
	return structPLS{v, codec}, nil
}

// LoadStruct loads the properties from p to dst.
// dst must be a struct pointer.
func LoadStruct(dst interface{}, p []Property) error {
	x, err := newStructPLS(dst)
	if err != nil {
		return err
	}
	return x.Load(p)
}

// SaveStruct returns the properties from src as a slice of Properties.
// src must be a struct pointer.
func SaveStruct(src interface{}) ([]Property, error) {
	x, err := newStructPLS(src)
	if err != nil {
		return nil, err
	}
	return x.Save()
}
<|endoftext|>


<commit_before>package azurerm

import (
	"fmt"
	"log"

	"net/http"

	"github.com/Azure/azure-sdk-for-go/arm/eventhub"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceArmEventHubConsumerGroup() *schema.Resource {
	return &schema.Resource{
		Create: resourceArmEventHubConsumerGroupCreateUpdate,
		Read:   resourceArmEventHubConsumerGroupRead,
		Update: resourceArmEventHubConsumerGroupCreateUpdate,
		Delete: resourceArmEventHubConsumerGroupDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"namespace_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"eventhub_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"resource_group_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"location": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"user_metadata": {
				Type:     schema.TypeString,
				Optional: true,
			},
		},
	}
}

func resourceArmEventHubConsumerGroupCreateUpdate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*ArmClient)
	eventhubClient := client.eventHubConsumerGroupClient
	log.Printf("[INFO] preparing arguments for Azure ARM EventHub Consumer Group creation.")

	name := d.Get("name").(string)
	namespaceName := d.Get("namespace_name").(string)
	eventHubName := d.Get("eventhub_name").(string)
	location := d.Get("location").(string)
	resGroup := d.Get("resource_group_name").(string)
	userMetaData := d.Get("user_metadata").(string)

	parameters := eventhub.ConsumerGroupCreateOrUpdateParameters{
		Name:     &name,
		Location: &location,
		ConsumerGroupProperties: &eventhub.ConsumerGroupProperties{
			UserMetadata: &userMetaData,
		},
	}

	_, err := eventhubClient.CreateOrUpdate(resGroup, namespaceName, eventHubName, name, parameters)
	if err != nil {
		return err
	}

	read, err := eventhubClient.Get(resGroup, namespaceName, eventHubName, name)

	if err != nil {
		return err
	}

	if read.ID == nil {
		return fmt.Errorf("Cannot read EventHub Consumer Group %s (resource group %s) ID", name, resGroup)
	}

	d.SetId(*read.ID)

	return resourceArmEventHubConsumerGroupRead(d, meta)
}

func resourceArmEventHubConsumerGroupRead(d *schema.ResourceData, meta interface{}) error {
	eventhubClient := meta.(*ArmClient).eventHubConsumerGroupClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	resGroup := id.ResourceGroup
	namespaceName := id.Path["namespaces"]
	eventHubName := id.Path["eventhubs"]
	name := id.Path["consumergroups"]

	resp, err := eventhubClient.Get(resGroup, namespaceName, eventHubName, name)
	if err != nil {
		return fmt.Errorf("Error making Read request on Azure EventHub Consumer Group %s: %s", name, err)
	}
	if resp.StatusCode == http.StatusNotFound {
		d.SetId("")
		return nil
	}

	d.Set("name", name)
	d.Set("eventhub_name", eventHubName)
	d.Set("namespace_name", namespaceName)
	d.Set("resource_group_name", resGroup)
	d.Set("location", azureRMNormalizeLocation(*resp.Location))
	d.Set("user_metadata", resp.ConsumerGroupProperties.UserMetadata)

	return nil
}

func resourceArmEventHubConsumerGroupDelete(d *schema.ResourceData, meta interface{}) error {
	eventhubClient := meta.(*ArmClient).eventHubConsumerGroupClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	resGroup := id.ResourceGroup
	namespaceName := id.Path["namespaces"]
	eventHubName := id.Path["eventhubs"]
	name := id.Path["consumergroups"]

	resp, err := eventhubClient.Delete(resGroup, namespaceName, eventHubName, name)

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("Error issuing Azure ARM delete request of EventHub Consumer Group '%s': %s", name, err)
	}

	return nil
}
<commit_msg>Switching to use the locationSchema for Event Hub Consumer Groups<commit_after>package azurerm

import (
	"fmt"
	"log"

	"net/http"

	"github.com/Azure/azure-sdk-for-go/arm/eventhub"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceArmEventHubConsumerGroup() *schema.Resource {
	return &schema.Resource{
		Create: resourceArmEventHubConsumerGroupCreateUpdate,
		Read:   resourceArmEventHubConsumerGroupRead,
		Update: resourceArmEventHubConsumerGroupCreateUpdate,
		Delete: resourceArmEventHubConsumerGroupDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"namespace_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"eventhub_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"resource_group_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"location": locationSchema(),

			"user_metadata": {
				Type:     schema.TypeString,
				Optional: true,
			},
		},
	}
}

func resourceArmEventHubConsumerGroupCreateUpdate(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*ArmClient)
	eventhubClient := client.eventHubConsumerGroupClient
	log.Printf("[INFO] preparing arguments for Azure ARM EventHub Consumer Group creation.")

	name := d.Get("name").(string)
	namespaceName := d.Get("namespace_name").(string)
	eventHubName := d.Get("eventhub_name").(string)
	location := d.Get("location").(string)
	resGroup := d.Get("resource_group_name").(string)
	userMetaData := d.Get("user_metadata").(string)

	parameters := eventhub.ConsumerGroupCreateOrUpdateParameters{
		Name:     &name,
		Location: &location,
		ConsumerGroupProperties: &eventhub.ConsumerGroupProperties{
			UserMetadata: &userMetaData,
		},
	}

	_, err := eventhubClient.CreateOrUpdate(resGroup, namespaceName, eventHubName, name, parameters)
	if err != nil {
		return err
	}

	read, err := eventhubClient.Get(resGroup, namespaceName, eventHubName, name)

	if err != nil {
		return err
	}

	if read.ID == nil {
		return fmt.Errorf("Cannot read EventHub Consumer Group %s (resource group %s) ID", name, resGroup)
	}

	d.SetId(*read.ID)

	return resourceArmEventHubConsumerGroupRead(d, meta)
}

func resourceArmEventHubConsumerGroupRead(d *schema.ResourceData, meta interface{}) error {
	eventhubClient := meta.(*ArmClient).eventHubConsumerGroupClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	resGroup := id.ResourceGroup
	namespaceName := id.Path["namespaces"]
	eventHubName := id.Path["eventhubs"]
	name := id.Path["consumergroups"]

	resp, err := eventhubClient.Get(resGroup, namespaceName, eventHubName, name)
	if err != nil {
		return fmt.Errorf("Error making Read request on Azure EventHub Consumer Group %s: %s", name, err)
	}
	if resp.StatusCode == http.StatusNotFound {
		d.SetId("")
		return nil
	}

	d.Set("name", name)
	d.Set("eventhub_name", eventHubName)
	d.Set("namespace_name", namespaceName)
	d.Set("resource_group_name", resGroup)
	d.Set("location", azureRMNormalizeLocation(*resp.Location))
	d.Set("user_metadata", resp.ConsumerGroupProperties.UserMetadata)

	return nil
}

func resourceArmEventHubConsumerGroupDelete(d *schema.ResourceData, meta interface{}) error {
	eventhubClient := meta.(*ArmClient).eventHubConsumerGroupClient

	id, err := parseAzureResourceID(d.Id())
	if err != nil {
		return err
	}
	resGroup := id.ResourceGroup
	namespaceName := id.Path["namespaces"]
	eventHubName := id.Path["eventhubs"]
	name := id.Path["consumergroups"]

	resp, err := eventhubClient.Delete(resGroup, namespaceName, eventHubName, name)

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("Error issuing Azure ARM delete request of EventHub Consumer Group '%s': %s", name, err)
	}

	return nil
}
<|endoftext|>


<commit_before>//
// +build unit

package perfevents

import (
	"testing"

	"github.com/intelsdi-x/pulse/control/plugin"
	"github.com/intelsdi-x/pulse/control/plugin/cpolicy"
	. "github.com/smartystreets/goconvey/convey"
)

func TestPSUtilPlugin(t *testing.T) {
	Convey("Meta should return metadata for the plugin", t, func() {
		meta := Meta()
		So(meta.Name, ShouldResemble, name)
		So(meta.Version, ShouldResemble, version)
		So(meta.Type, ShouldResemble, plugin.CollectorPluginType)
	})

	Convey("Create PerfEvents Collector", t, func() {
		pfCol := NewPerfeventsCollector()
		Convey("So pfCol should not be nil", func() {
			So(pfCol, ShouldNotBeNil)
		})
		Convey("So pfCol should be of Psutil type", func() {
			So(pfCol, ShouldHaveSameTypeAs, &Perfevents{})
		})
		configPolicy, err := pfCol.GetConfigPolicy()
		Convey("pfCol.GetConfigPolicy() should return a config policy", func() {
			Convey("So config policy should not be nil", func() {
				So(configPolicy, ShouldNotBeNil)
			})
			Convey("So getting config policy should not return an error", func() {
				So(err, ShouldBeNil)
			})
			Convey("So config policy should be a cpolicy.ConfigPolicy", func() {
				So(configPolicy, ShouldHaveSameTypeAs, &cpolicy.ConfigPolicy{})
			})
		})
	})
}
<commit_msg>Update test name<commit_after>//
// +build unit

package perfevents

import (
	"testing"

	"github.com/intelsdi-x/pulse/control/plugin"
	"github.com/intelsdi-x/pulse/control/plugin/cpolicy"
	. "github.com/smartystreets/goconvey/convey"
)

func TestPerfeventsPlugin(t *testing.T) {
	Convey("Meta should return metadata for the plugin", t, func() {
		meta := Meta()
		So(meta.Name, ShouldResemble, name)
		So(meta.Version, ShouldResemble, version)
		So(meta.Type, ShouldResemble, plugin.CollectorPluginType)
	})

	Convey("Create PerfEvents Collector", t, func() {
		pfCol := NewPerfeventsCollector()
		Convey("So pfCol should not be nil", func() {
			So(pfCol, ShouldNotBeNil)
		})
		Convey("So pfCol should be of Psutil type", func() {
			So(pfCol, ShouldHaveSameTypeAs, &Perfevents{})
		})
		configPolicy, err := pfCol.GetConfigPolicy()
		Convey("pfCol.GetConfigPolicy() should return a config policy", func() {
			Convey("So config policy should not be nil", func() {
				So(configPolicy, ShouldNotBeNil)
			})
			Convey("So getting config policy should not return an error", func() {
				So(err, ShouldBeNil)
			})
			Convey("So config policy should be a cpolicy.ConfigPolicy", func() {
				So(configPolicy, ShouldHaveSameTypeAs, &cpolicy.ConfigPolicy{})
			})
		})
	})
}
<|endoftext|>


<commit_before>// The MIT License (MIT)

// Copyright (c) 2015 pangweiwei(sineysan@163.com)

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package main

import (
"os"
"bufio"
"net"
"fmt"
"reflect"
"strings"
"encoding/binary"
"bytes"
"os/signal"
"flag"
)

var prompt string = ""

type Client struct {
	input chan string
	netin chan string
	console *bufio.Reader
	reader *bufio.Reader
	writer *bufio.Writer
}

func (c *Client) ReadConsole() {
	for {
		l,_:=c.console.ReadString('\n')
		c.input<-l
	}
}

func (c *Client) ReadNet() {
	for {
		data:=make([]byte,4)
		l,_:=c.reader.Read(data)
		if l==0 {
			c.netin<-"quit"
			return
		}
		var len int32
		buf:=bytes.NewBuffer(data)
		binary.Read(buf,binary.LittleEndian,&len);

		data=make([]byte,len)
		l,_=c.reader.Read(data)

		if int32(l)!=len {
			c.netin<-"quit"
		} else {
			c.netin<-string(data)
		}
	}
}

func (c *Client) send(data string) {
	buf:=new(bytes.Buffer)
	binary.Write(buf,binary.LittleEndian,int32(len(data)))
	binary.Write(buf,binary.LittleEndian,[]byte(data))
	c.writer.Write(buf.Bytes())
	c.writer.Flush()
}

func (c *Client) onInput(data string) {
	c.send(data)
}

func invoke(c *Client, name string, args... interface{}) {
	inputs := make([]reflect.Value, len(args))
	for i, _ := range args {
		inputs[i] = reflect.ValueOf(args[i])
	}
	v:=reflect.ValueOf(c)
	m:=v.MethodByName(name)
	if m.IsValid() {
		m.Call(inputs)
	} else {
		fmt.Printf("Invalid cmd %s\n",name)
	}
}

func (c *Client) Cmd_print(tail string) {
	fmt.Println(tail)
}

func (c *Client) Cmd_ret(tail string) {
	fmt.Print(prompt)
}

func (c *Client) Cmd_quit(tail string) {
	os.Exit(0)
}

func (c *Client) Cmd_break(tail string) {
	prompt="ldb> "
	args:=strings.Split(tail,",")
	fmt.Printf("Break at %s:%s\n",args[0],args[1])
	fmt.Print(prompt)
}

func (c *Client) Cmd_resume(tail string) {
	fmt.Println("")
	prompt="slua> "
	fmt.Print(prompt)
}

func (c *Client) onCommand(data string) {
	cmd:=data
	tail:=""
	n:=strings.Index(data," ")
	if n>0 {
		cmd=data[0:n]
		tail=data[n+1:]
	}

	invoke(c,"Cmd_"+cmd,tail)
}

func (c *Client) close() {
	c.send("quit")
	os.Exit(0)
}

func main() {

	hostptr:=flag.String("host","localhost","remote host ip to connect")
	portptr:=flag.Int("port",10240,"remote host port to connect")
	flag.Parse()

	fmt.Printf("Connect to %s:%d\n",*hostptr,*portptr)

	prompt="slua> "

	conn,err:=net.Dial("tcp","localhost:10240")
	if err!=nil {
		fmt.Println(err)
		return
	}

	defer conn.Close()

	fmt.Println("Host connected")
	fmt.Println("Type 'start' to continue game")

	console:=bufio.NewReader(os.Stdin)
	nw:=bufio.NewWriter(conn)
	nr:=bufio.NewReader(conn)

	client:=&Client{
		input:make(chan string),
		netin:make(chan string),
		console:console,
		reader:nr,
		writer:nw,
	}

	fmt.Print(prompt)

	go client.ReadConsole()
	go client.ReadNet()

	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc,os.Interrupt,os.Kill)

	// main loop
	for {

		select {

			case data:=<-client.input:
				client.onInput(data)
			case data:=<-client.netin:
				client.onCommand(data)
			case _=<-sigc:
				client.close()
		}

	}
}<commit_msg> fix connect addr<commit_after>// The MIT License (MIT)

// Copyright (c) 2015 pangweiwei(sineysan@163.com)

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package main

import (
"os"
"bufio"
"net"
"fmt"
"reflect"
"strings"
"encoding/binary"
"bytes"
"os/signal"
"flag"
)

var prompt string = ""

type Client struct {
	input chan string
	netin chan string
	console *bufio.Reader
	reader *bufio.Reader
	writer *bufio.Writer
}

func (c *Client) ReadConsole() {
	for {
		l,_:=c.console.ReadString('\n')
		c.input<-l
	}
}

func (c *Client) ReadNet() {
	for {
		data:=make([]byte,4)
		l,_:=c.reader.Read(data)
		if l==0 {
			c.netin<-"quit"
			return
		}
		var len int32
		buf:=bytes.NewBuffer(data)
		binary.Read(buf,binary.LittleEndian,&len);

		data=make([]byte,len)
		l,_=c.reader.Read(data)

		if int32(l)!=len {
			c.netin<-"quit"
		} else {
			c.netin<-string(data)
		}
	}
}

func (c *Client) send(data string) {
	buf:=new(bytes.Buffer)
	binary.Write(buf,binary.LittleEndian,int32(len(data)))
	binary.Write(buf,binary.LittleEndian,[]byte(data))
	c.writer.Write(buf.Bytes())
	c.writer.Flush()
}

func (c *Client) onInput(data string) {
	c.send(data)
}

func invoke(c *Client, name string, args... interface{}) {
	inputs := make([]reflect.Value, len(args))
	for i, _ := range args {
		inputs[i] = reflect.ValueOf(args[i])
	}
	v:=reflect.ValueOf(c)
	m:=v.MethodByName(name)
	if m.IsValid() {
		m.Call(inputs)
	} else {
		fmt.Printf("Invalid cmd %s\n",name)
	}
}

func (c *Client) Cmd_print(tail string) {
	fmt.Println(tail)
}

func (c *Client) Cmd_ret(tail string) {
	fmt.Print(prompt)
}

func (c *Client) Cmd_quit(tail string) {
	os.Exit(0)
}

func (c *Client) Cmd_break(tail string) {
	prompt="ldb> "
	args:=strings.Split(tail,",")
	fmt.Printf("Break at %s:%s\n",args[0],args[1])
	fmt.Print(prompt)
}

func (c *Client) Cmd_resume(tail string) {
	fmt.Println("")
	prompt="slua> "
	fmt.Print(prompt)
}

func (c *Client) onCommand(data string) {
	cmd:=data
	tail:=""
	n:=strings.Index(data," ")
	if n>0 {
		cmd=data[0:n]
		tail=data[n+1:]
	}

	invoke(c,"Cmd_"+cmd,tail)
}

func (c *Client) close() {
	c.send("quit")
	os.Exit(0)
}

func main() {

	hostptr:=flag.String("host","localhost","remote host ip to connect")
	portptr:=flag.Int("port",10240,"remote host port to connect")
	flag.Parse()

	fmt.Printf("Connect to %s:%d\n",*hostptr,*portptr)

	prompt="slua> "

	conn,err:=net.Dial("tcp",fmt.Sprintf("%s:%d",*hostptr,*portptr))
	if err!=nil {
		fmt.Println(err)
		return
	}

	defer conn.Close()

	fmt.Println("Host connected")
	fmt.Println("Type 'start' to continue game")

	console:=bufio.NewReader(os.Stdin)
	nw:=bufio.NewWriter(conn)
	nr:=bufio.NewReader(conn)

	client:=&Client{
		input:make(chan string),
		netin:make(chan string),
		console:console,
		reader:nr,
		writer:nw,
	}

	fmt.Print(prompt)

	go client.ReadConsole()
	go client.ReadNet()

	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc,os.Interrupt,os.Kill)

	// main loop
	for {

		select {

			case data:=<-client.input:
				client.onInput(data)
			case data:=<-client.netin:
				client.onCommand(data)
			case _=<-sigc:
				client.close()
		}

	}
}<|endoftext|>


<commit_before>// Copyright (c) 2014 Alex Kalyvitis
// Portions Copyright (c) 2011 The Go Authors

package mustache

import (
	"fmt"

	"strings"
	"unicode"
	"unicode/utf8"
)

// token represents a token or text string returned from the scanner.
type token struct {
	typ  tokenType
	val  string
	line int
	col  int
}

// String satisfies the fmt.Stringer interface making it easier to print tokens.
func (i token) String() string {
	return fmt.Sprintf("%s:%q", i.typ, i.val)
}

// tokenType identifies the type of lex tokens.
type tokenType int

const (
	tokenError tokenType = iota // error occurred; value is text of error
	tokenEOF
	tokenIdentifier     // alphanumeric identifier
	tokenLeftDelim      // {{ left action delimiter
	tokenRightDelim     // }} right action delimiter
	tokenText           // plain text
	tokenLine           // new line
	tokenComment        // {{! this is a comment and is ignored}}
	tokenSectionStart   // {{#foo}} denotes a section start
	tokenSectionInverse // {{^foo}} denotes an inverse section start
	tokenSectionEnd     // {{/foo}} denotes the closing of a section
	tokenRawStart       // { denotes the beginning of an unencoded identifier
	tokenRawEnd         // } denotes the end of an unencoded identifier
	tokenRawAlt         // {{&foo}} is an alternative way to define raw tags
	tokenPartial        // {{>foo}} denotes a partial
	tokenSetDelim       // {{={% %}=}} sets delimiters to {% and %}
	tokenSetLeftDelim   // denotes a custom left delimiter
	tokenSetRightDelim  // denotes a custom right delimiter
)

// Make the types prettyprint.
var tokenName = map[tokenType]string{
	tokenError:          "t_error",
	tokenEOF:            "t_eof",
	tokenIdentifier:     "t_ident",
	tokenLeftDelim:      "t_left_delim",
	tokenRightDelim:     "t_right_delim",
	tokenText:           "t_text",
	tokenLine:           "t_line",
	tokenComment:        "t_comment",
	tokenSectionStart:   "t_section_start",
	tokenSectionInverse: "t_section_inverse",
	tokenSectionEnd:     "t_section_end",
	tokenRawStart:       "t_raw_start",
	tokenRawEnd:         "t_raw_end",
	tokenRawAlt:         "t_raw_alt",
	tokenPartial:        "t_partial",
	tokenSetDelim:       "t_set_delim",
	tokenSetLeftDelim:   "t_set_left_delim",
	tokenSetRightDelim:  "t_set_right_delim",
}

// String satisfies the fmt.Stringer interface making it easier to print tokens.
func (i tokenType) String() string {
	s := tokenName[i]
	if s == "" {
		return fmt.Sprintf("t_unknown_%d", int(i))
	}
	return s
}

const eof = -1

// stateFn represents the state of the scanner as a function that returns the
// next state.
type stateFn func(*lexer) stateFn

// lexer holds the state of the scanner.
type lexer struct {
	name       string     // the name of the input; used only for error reports.
	input      string     // the string being scanned.
	leftDelim  string     // start of action.
	rightDelim string     // end of action.
	state      stateFn    // the next lexing function to enter.
	pos        int        // current position in the input.
	start      int        // start position of this token.
	width      int        // width of last rune read from input.
	tokens     chan token // channel of scanned tokens.
}

// next returns the next rune in the input.
func (l *lexer) next() (r rune) {
	if l.pos >= len(l.input) {
		l.width = 0
		return eof
	}
	r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
	l.pos += l.width
	return r
}

// peek returns but does not consume the next rune in the input.
func (l *lexer) peek() rune {
	r := l.next()
	l.backup()
	return r
}

// backup steps back one rune. Can only be called once per call of next.
func (l *lexer) backup() {
	l.pos -= l.width
}

// emit passes an token back to the client.
func (l *lexer) emit(t tokenType) {
	l.tokens <- token{
		t,
		l.input[l.start:l.pos],
		l.lineNum(),
		l.columnNum(),
	}
	l.start = l.pos
}

// ignore skips over the pending input before this point.
func (l *lexer) ignore() {
	l.start = l.pos
}

// lineNum reports which line we're on. Doing it this way
// means we don't have to worry about peek double counting.
func (l *lexer) lineNum() int {
	return 1 + strings.Count(l.input[:l.pos], "\n")
}

// columnNum reports the character of the current line we're on.
func (l *lexer) columnNum() int {
	if lf := strings.LastIndex(l.input[:l.pos], "\n"); lf != -1 {
		return len(l.input[lf+1 : l.pos])
	}
	return len(l.input[:l.pos])
}

// error returns an error token and terminates the scan by passing
// back a nil pointer that will be the next state, terminating l.token.
func (l *lexer) errorf(format string, args ...interface{}) stateFn {
	l.tokens <- token{
		tokenError,
		fmt.Sprintf(format, args...),
		l.lineNum(),
		l.columnNum(),
	}
	return nil
}

// token returns the next token from the input.
func (l *lexer) token() token {
	for {
		select {
		case token := <-l.tokens:
			return token
		default:
			l.state = l.state(l)
		}
	}
	panic("not reached")
}

// state functions.

// stateText scans until an opening action delimiter, "{{".
func stateText(l *lexer) stateFn {
	for {
		// Lookahead for {{ which should switch to lexing an open tag instead of
		// regular text tokens.
		if strings.HasPrefix(l.input[l.pos:], l.leftDelim) {
			if l.pos > l.start {
				l.emit(tokenText)
			}
			return stateLeftDelim
		}
		// When a new line character is encountered it is emited as a separate
		// token. This helps the rendering engine decide whether or not to
		// render certain ast nodes when standalone tags are encountered.
		if strings.HasPrefix(l.input[l.pos:], "\n") {
			if l.pos > l.start {
				l.emit(tokenText)
			}
			return stateLine
		}
		// Produce a token and exit the loop if we have reached the end of file.
		if l.next() == eof {
			break
		}
	}
	// Emit whatever we gathered so far as text.
	if l.pos > l.start {
		l.emit(tokenText)
	}
	// Always end with EOF token. The parser will keep asking for tokens until
	// an tokenEOF or tokenError token are encountered.
	l.emit(tokenEOF)
	// The text state doesn't have a default next state.
	return nil
}

// stateLeftDelim scans the left delimiter, which is known to be present.
func stateLeftDelim(l *lexer) stateFn {
	l.pos += len(l.leftDelim)
	if l.peek() == '=' {
		// When the lexer encounters "{{=" it proceeds to the set delimiter
		// state which alters the left and right delimiters. This operation is
		// hidden from the parser and no tokens are emited.
		l.next()
		return stateSetDelim
	}
	l.emit(tokenLeftDelim)
	return stateTag
}

// stateLine scans a new line character and emits a token for it.
func stateLine(l *lexer) stateFn {
	l.next()
	l.emit(tokenLine)
	return stateText
}

// stateRightDelim scans the right delimiter, which is known to be present.
func stateRightDelim(l *lexer) stateFn {
	l.pos += len(l.rightDelim)
	l.emit(tokenRightDelim)
	return stateText
}

// stateTag scans the elements inside action delimiters.
func stateTag(l *lexer) stateFn {
	if strings.HasPrefix(l.input[l.pos:], "}"+l.rightDelim) {
		l.pos++
		l.emit(tokenRawEnd)
		return stateRightDelim
	}
	if strings.HasPrefix(l.input[l.pos:], l.rightDelim) {
		return stateRightDelim
	}
	switch r := l.next(); {
	case r == eof || r == '\n':
		return l.errorf("unclosed action")
	case whitespace(r):
		l.ignore()
	case r == '!':
		l.emit(tokenComment)
		return stateComment
	case r == '#':
		l.emit(tokenSectionStart)
	case r == '^':
		l.emit(tokenSectionInverse)
	case r == '/':
		l.emit(tokenSectionEnd)
	case r == '&':
		l.emit(tokenRawAlt)
	case r == '>':
		l.emit(tokenPartial)
	case r == '{':
		l.emit(tokenRawStart)
	case alphanum(r):
		l.backup()
		return stateIdent
	default:
		return l.errorf("unrecognized character in action: %#U", r)
	}
	return stateTag
}

// stateIdent scans an alphanumeric or field.
func stateIdent(l *lexer) stateFn {
Loop:
	for {
		switch r := l.next(); {
		case alphanum(r):
			// absorb.
		default:
			l.backup()
			l.emit(tokenIdentifier)
			break Loop
		}
	}
	return stateTag
}

// stateComment scans a comment. The left comment marker is known to be present.
func stateComment(l *lexer) stateFn {
	i := strings.Index(l.input[l.pos:], l.rightDelim)
	if i < 0 {
		return l.errorf("unclosed tag")
	}
	l.pos += i
	l.emit(tokenText)
	l.pos += len(l.rightDelim)
	l.emit(tokenRightDelim)
	return stateText
}

// stateSetDelim scans a set of set delimiter tags and replaces the lexers left
// and right delimiters to new values.
func stateSetDelim(l *lexer) stateFn {
	end := "=" + l.rightDelim
	i := strings.Index(l.input[l.pos:], end)
	if i < 0 {
		return l.errorf("unclosed tag")
	}
	delims := strings.Split(l.input[l.pos:l.pos+i], " ") // " | | "
	if len(delims) < 2 {
		l.errorf("set delimiters should be separated by a space")
	}
	delimFn := leftFn
	for _, delim := range delims {
		if delim != "" {
			if delimFn != nil {
				delimFn = delimFn(l, delim)
			}
		}
	}
	l.pos += i + len(end)
	l.ignore()
	return stateText
}

// delimFn is a self referencing function which helps with setting the right
// delimiter in the right order.
type delimFn func(l *lexer, s string) delimFn

// leftFn sets the left delimiter to s and returns a rightFn.
func leftFn(l *lexer, s string) delimFn {
	l.leftDelim = s
	return rightFn
}

// rightFn sets the right delimiter to s.
func rightFn(l *lexer, s string) delimFn {
	l.rightDelim = s
	return nil
}

// newLexer creates a new scanner for the input string.
func newLexer(input, left, right string) *lexer {
	l := &lexer{
		input:      input,
		leftDelim:  left,
		rightDelim: right,
		tokens:     make(chan token, 2),
	}
	l.state = stateText // initial state
	return l
}

// whitespace reports whether r is a space character.
func whitespace(r rune) bool {
	switch r {
	case ' ', '\t', '\n', '\r':
		return true
	}
	return false
}

// alphanum reports whether r is an alphabetic, digit, or underscore.
func alphanum(r rune) bool {
	return r == '_' || r == '.' || unicode.IsLetter(r) || unicode.IsDigit(r)
}
<commit_msg>added String method to lexer<commit_after>// Copyright (c) 2014 Alex Kalyvitis
// Portions Copyright (c) 2011 The Go Authors

package mustache

import (
	"bytes"
	"fmt"

	"strings"
	"unicode"
	"unicode/utf8"
)

// token represents a token or text string returned from the scanner.
type token struct {
	typ  tokenType
	val  string
	line int
	col  int
}

// String satisfies the fmt.Stringer interface making it easier to print tokens.
func (i token) String() string {
	return fmt.Sprintf("%s:%q", i.typ, i.val)
}

// tokenType identifies the type of lex tokens.
type tokenType int

const (
	tokenError tokenType = iota // error occurred; value is text of error
	tokenEOF
	tokenIdentifier     // alphanumeric identifier
	tokenLeftDelim      // {{ left action delimiter
	tokenRightDelim     // }} right action delimiter
	tokenText           // plain text
	tokenLine           // new line
	tokenComment        // {{! this is a comment and is ignored}}
	tokenSectionStart   // {{#foo}} denotes a section start
	tokenSectionInverse // {{^foo}} denotes an inverse section start
	tokenSectionEnd     // {{/foo}} denotes the closing of a section
	tokenRawStart       // { denotes the beginning of an unencoded identifier
	tokenRawEnd         // } denotes the end of an unencoded identifier
	tokenRawAlt         // {{&foo}} is an alternative way to define raw tags
	tokenPartial        // {{>foo}} denotes a partial
	tokenSetDelim       // {{={% %}=}} sets delimiters to {% and %}
	tokenSetLeftDelim   // denotes a custom left delimiter
	tokenSetRightDelim  // denotes a custom right delimiter
)

// Make the types prettyprint.
var tokenName = map[tokenType]string{
	tokenError:          "t_error",
	tokenEOF:            "t_eof",
	tokenIdentifier:     "t_ident",
	tokenLeftDelim:      "t_left_delim",
	tokenRightDelim:     "t_right_delim",
	tokenText:           "t_text",
	tokenLine:           "t_line",
	tokenComment:        "t_comment",
	tokenSectionStart:   "t_section_start",
	tokenSectionInverse: "t_section_inverse",
	tokenSectionEnd:     "t_section_end",
	tokenRawStart:       "t_raw_start",
	tokenRawEnd:         "t_raw_end",
	tokenRawAlt:         "t_raw_alt",
	tokenPartial:        "t_partial",
	tokenSetDelim:       "t_set_delim",
	tokenSetLeftDelim:   "t_set_left_delim",
	tokenSetRightDelim:  "t_set_right_delim",
}

// String satisfies the fmt.Stringer interface making it easier to print tokens.
func (i tokenType) String() string {
	s := tokenName[i]
	if s == "" {
		return fmt.Sprintf("t_unknown_%d", int(i))
	}
	return s
}

const eof = -1

// stateFn represents the state of the scanner as a function that returns the
// next state.
type stateFn func(*lexer) stateFn

// lexer holds the state of the scanner.
type lexer struct {
	name       string     // the name of the input; used only for error reports.
	input      string     // the string being scanned.
	leftDelim  string     // start of action.
	rightDelim string     // end of action.
	state      stateFn    // the next lexing function to enter.
	pos        int        // current position in the input.
	start      int        // start position of this token.
	width      int        // width of last rune read from input.
	tokens     chan token // channel of scanned tokens.
}

// next returns the next rune in the input.
func (l *lexer) next() (r rune) {
	if l.pos >= len(l.input) {
		l.width = 0
		return eof
	}
	r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
	l.pos += l.width
	return r
}

// peek returns but does not consume the next rune in the input.
func (l *lexer) peek() rune {
	r := l.next()
	l.backup()
	return r
}

// backup steps back one rune. Can only be called once per call of next.
func (l *lexer) backup() {
	l.pos -= l.width
}

// emit passes an token back to the client.
func (l *lexer) emit(t tokenType) {
	l.tokens <- token{
		t,
		l.input[l.start:l.pos],
		l.lineNum(),
		l.columnNum(),
	}
	l.start = l.pos
}

// ignore skips over the pending input before this point.
func (l *lexer) ignore() {
	l.start = l.pos
}

// lineNum reports which line we're on. Doing it this way
// means we don't have to worry about peek double counting.
func (l *lexer) lineNum() int {
	return 1 + strings.Count(l.input[:l.pos], "\n")
}

// columnNum reports the character of the current line we're on.
func (l *lexer) columnNum() int {
	if lf := strings.LastIndex(l.input[:l.pos], "\n"); lf != -1 {
		return len(l.input[lf+1 : l.pos])
	}
	return len(l.input[:l.pos])
}

// error returns an error token and terminates the scan by passing
// back a nil pointer that will be the next state, terminating l.token.
func (l *lexer) errorf(format string, args ...interface{}) stateFn {
	l.tokens <- token{
		tokenError,
		fmt.Sprintf(format, args...),
		l.lineNum(),
		l.columnNum(),
	}
	return nil
}

// token returns the next token from the input.
func (l *lexer) token() token {
	for {
		select {
		case token := <-l.tokens:
			return token
		default:
			l.state = l.state(l)
		}
	}
	panic("not reached")
}

func (l *lexer) String() string {
	w := bytes.NewBuffer(nil)
	fmt.Fprintf(w, "Template: %q\n", l.input)
	fmt.Fprintf(w, "Index   : %q\n", l.pos)
	fmt.Fprintf(w, "Current : %q\n", l.input[l.pos])
	fmt.Fprintf(w, "Buffer  : %q\n", l.input[l.start:l.pos])
	return w.String()
}

// newLexer creates a new scanner for the input string.
func newLexer(input, left, right string) *lexer {
	l := &lexer{
		input:      input,
		leftDelim:  left,
		rightDelim: right,
		tokens:     make(chan token, 2),
	}
	l.state = stateText // initial state
	return l
}

// state functions.

// stateText scans until an opening action delimiter, "{{".
func stateText(l *lexer) stateFn {
	for {
		// Lookahead for {{ which should switch to lexing an open tag instead of
		// regular text tokens.
		if strings.HasPrefix(l.input[l.pos:], l.leftDelim) {
			if l.pos > l.start {
				l.emit(tokenText)
			}
			return stateLeftDelim
		}
		// When a new line character is encountered it is emited as a separate
		// token. This helps the rendering engine decide whether or not to
		// render certain ast nodes when standalone tags are encountered.
		if strings.HasPrefix(l.input[l.pos:], "\n") {
			if l.pos > l.start {
				l.emit(tokenText)
			}
			return stateLine
		}
		// Produce a token and exit the loop if we have reached the end of file.
		if l.next() == eof {
			break
		}
	}
	// Emit whatever we gathered so far as text.
	if l.pos > l.start {
		l.emit(tokenText)
	}
	// Always end with EOF token. The parser will keep asking for tokens until
	// an tokenEOF or tokenError token are encountered.
	l.emit(tokenEOF)
	// The text state doesn't have a default next state.
	return nil
}

// stateLeftDelim scans the left delimiter, which is known to be present.
func stateLeftDelim(l *lexer) stateFn {
	l.pos += len(l.leftDelim)
	if l.peek() == '=' {
		// When the lexer encounters "{{=" it proceeds to the set delimiter
		// state which alters the left and right delimiters. This operation is
		// hidden from the parser and no tokens are emited.
		l.next()
		return stateSetDelim
	}
	l.emit(tokenLeftDelim)
	return stateTag
}

// stateLine scans a new line character and emits a token for it.
func stateLine(l *lexer) stateFn {
	l.next()
	l.emit(tokenLine)
	return stateText
}

// stateRightDelim scans the right delimiter, which is known to be present.
func stateRightDelim(l *lexer) stateFn {
	l.pos += len(l.rightDelim)
	l.emit(tokenRightDelim)
	return stateText
}

// stateTag scans the elements inside action delimiters.
func stateTag(l *lexer) stateFn {
	if strings.HasPrefix(l.input[l.pos:], "}"+l.rightDelim) {
		l.pos++
		l.emit(tokenRawEnd)
		return stateRightDelim
	}
	if strings.HasPrefix(l.input[l.pos:], l.rightDelim) {
		return stateRightDelim
	}
	switch r := l.next(); {
	case r == eof || r == '\n':
		return l.errorf("unclosed action")
	case whitespace(r):
		l.ignore()
	case r == '!':
		l.emit(tokenComment)
		return stateComment
	case r == '#':
		l.emit(tokenSectionStart)
	case r == '^':
		l.emit(tokenSectionInverse)
	case r == '/':
		l.emit(tokenSectionEnd)
	case r == '&':
		l.emit(tokenRawAlt)
	case r == '>':
		l.emit(tokenPartial)
	case r == '{':
		l.emit(tokenRawStart)
	case alphanum(r):
		l.backup()
		return stateIdent
	default:
		return l.errorf("unrecognized character in action: %#U", r)
	}
	return stateTag
}

// stateIdent scans an alphanumeric or field.
func stateIdent(l *lexer) stateFn {
Loop:
	for {
		switch r := l.next(); {
		case alphanum(r):
			// absorb.
		default:
			l.backup()
			l.emit(tokenIdentifier)
			break Loop
		}
	}
	return stateTag
}

// stateComment scans a comment. The left comment marker is known to be present.
func stateComment(l *lexer) stateFn {
	i := strings.Index(l.input[l.pos:], l.rightDelim)
	if i < 0 {
		return l.errorf("unclosed tag")
	}
	l.pos += i
	l.emit(tokenText)
	return stateRightDelim
}

// stateSetDelim scans a set of set delimiter tags and replaces the lexers left
// and right delimiters to new values.
func stateSetDelim(l *lexer) stateFn {
	end := "=" + l.rightDelim
	i := strings.Index(l.input[l.pos:], end)
	if i < 0 {
		return l.errorf("unclosed tag")
	}
	delims := strings.Split(l.input[l.pos:l.pos+i], " ") // " | | "
	if len(delims) < 2 {
		l.errorf("set delimiters should be separated by a space")
	}
	delimFn := leftFn
	for _, delim := range delims {
		if delim != "" {
			if delimFn != nil {
				delimFn = delimFn(l, delim)
			}
		}
	}
	l.pos += i + len(end)
	l.ignore()
	return stateText
}

// delimFn is a self referencing function which helps with setting the right
// delimiter in the right order.
type delimFn func(l *lexer, s string) delimFn

// leftFn sets the left delimiter to s and returns a rightFn.
func leftFn(l *lexer, s string) delimFn {
	l.leftDelim = s
	return rightFn
}

// rightFn sets the right delimiter to s.
func rightFn(l *lexer, s string) delimFn {
	l.rightDelim = s
	return nil
}

// whitespace reports whether r is a space character.
func whitespace(r rune) bool {
	switch r {
	case ' ', '\t', '\n', '\r':
		return true
	}
	return false
}

// alphanum reports whether r is an alphabetic, digit, or underscore.
func alphanum(r rune) bool {
	return r == '_' || r == '.' || unicode.IsLetter(r) || unicode.IsDigit(r)
}
<|endoftext|>


<commit_before>// +build arm64,linux

package archutils

// #include <sys/epoll.h>
/*
int EpollCreate1(int flag) {
        return epoll_create1(0);
}

int EpollCtl(int efd, int op,int sfd, int Events, int Fd) {
        struct epoll_event event;
        event.events = Events;
        event.data.fd = Fd;

        return epoll_ctl(efd,op,sfd,&event);
}

typedef struct Event{
        uint32_t events;
        int fd;
};

struct epoll_event events[128];
int run_epoll_wait(int fd, struct Event *event) {
        int n, i;
        n = epoll_wait(fd, events, 128, -1);
        for (i = 0; i < n; i++) {
                event[i].events = events[i].events;
                event[i].fd = events[i].data.fd;
        }
        return n;
}
*/
import "C"

import (
	"fmt"
	"syscall"
	"unsafe"
)

func EpollCreate1(flag int) (int, error) {
	fd := int(C.EpollCreate1(0))
	if fd < 0 {
		return fd, fmt.Errorf("failed to create epoll, errno is %d", fd)
	}
	return fd, nil
}

func EpollCtl(epfd int, op int, fd int, event *syscall.EpollEvent) error {
	errno := C.EpollCtl(C.int(epfd), C.int(syscall.EPOLL_CTL_ADD), C.int(fd), C.int(event.Events), C.int(event.Fd))
	if errno < 0 {
		return fmt.Errorf("Failed to ctl epoll")
	}
	return nil
}

func EpollWait(epfd int, events []syscall.EpollEvent, msec int) (int, error) {
	var c_events [128]C.struct_Event
	n := int(C.run_epoll_wait(C.int(epfd), (*C.struct_Event)(unsafe.Pointer(&c_events))))
	if n < 0 {
		return int(n), fmt.Errorf("Failed to wait epoll")
	}
	for i := 0; i < n; i++ {
		events[i].Fd = int32(c_events[i].fd)
		events[i].Events = uint32(c_events[i].events)
	}
	return int(n), nil
}
<commit_msg>Use flag for aarch64 EpollCreate1<commit_after>// +build arm64,linux

package archutils

// #include <sys/epoll.h>
/*
int EpollCreate1(int flag) {
        return epoll_create1(flag);
}

int EpollCtl(int efd, int op, int sfd, int Events, int Fd) {
        struct epoll_event event;
        event.events = Events;
        event.data.fd = Fd;

        return epoll_ctl(efd, op, sfd, &event);
}

typedef struct Event{
        uint32_t events;
        int fd;
};

struct epoll_event events[128];
int run_epoll_wait(int fd, struct Event *event) {
        int n, i;
        n = epoll_wait(fd, events, 128, -1);
        for (i = 0; i < n; i++) {
                event[i].events = events[i].events;
                event[i].fd = events[i].data.fd;
        }
        return n;
}
*/
import "C"

import (
	"fmt"
	"syscall"
	"unsafe"
)

func EpollCreate1(flag int) (int, error) {
	fd := int(C.EpollCreate1(C.int(flag)))
	if fd < 0 {
		return fd, fmt.Errorf("failed to create epoll, errno is %d", fd)
	}
	return fd, nil
}

func EpollCtl(epfd int, op int, fd int, event *syscall.EpollEvent) error {
	errno := C.EpollCtl(C.int(epfd), C.int(syscall.EPOLL_CTL_ADD), C.int(fd), C.int(event.Events), C.int(event.Fd))
	if errno < 0 {
		return fmt.Errorf("Failed to ctl epoll")
	}
	return nil
}

func EpollWait(epfd int, events []syscall.EpollEvent, msec int) (int, error) {
	var c_events [128]C.struct_Event
	n := int(C.run_epoll_wait(C.int(epfd), (*C.struct_Event)(unsafe.Pointer(&c_events))))
	if n < 0 {
		return int(n), fmt.Errorf("Failed to wait epoll")
	}
	for i := 0; i < n; i++ {
		events[i].Fd = int32(c_events[i].fd)
		events[i].Events = uint32(c_events[i].events)
	}
	return int(n), nil
}
<|endoftext|>