<commit_before>// Copyright (c) 2013-2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package blockchain

import (
	"fmt"
	"math"
	"runtime"

	"github.com/btcsuite/btcd/txscript"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
)

// txValidateItem holds a transaction along with which input to validate.
type txValidateItem struct {
	txInIndex int
	txIn      *wire.TxIn
	tx        *btcutil.Tx
}

// txValidator provides a type which asynchronously validates transaction
// inputs.  It provides several channels for communication and a processing
// function that is intended to be in run multiple goroutines.
type txValidator struct {
	validateChan chan *txValidateItem
	quitChan     chan struct{}
	resultChan   chan error
	utxoView     *UtxoViewpoint
	flags        txscript.ScriptFlags
	sigCache     *txscript.SigCache
	hashCache    *txscript.HashCache
}

// sendResult sends the result of a script pair validation on the internal
// result channel while respecting the quit channel.  The allows orderly
// shutdown when the validation process is aborted early due to a validation
// error in one of the other goroutines.
func (v *txValidator) sendResult(result error) {
	select {
	case v.resultChan <- result:
	case <-v.quitChan:
	}
}

// validateHandler consumes items to validate from the internal validate channel
// and returns the result of the validation on the internal result channel. It
// must be run as a goroutine.
func (v *txValidator) validateHandler() {
out:
	for {
		select {
		case txVI := <-v.validateChan:
			// Ensure the referenced input transaction is available.
			txIn := txVI.txIn
			originTxHash := &txIn.PreviousOutPoint.Hash
			originTxIndex := txIn.PreviousOutPoint.Index
			txEntry := v.utxoView.LookupEntry(originTxHash)
			if txEntry == nil {
				str := fmt.Sprintf("unable to find input "+
					"transaction %v referenced from "+
					"transaction %v", originTxHash,
					txVI.tx.Sha())
				err := ruleError(ErrMissingTx, str)
				v.sendResult(err)
				break out
			}

			// Ensure the referenced input transaction public key
			// script is available.
			pkScript := txEntry.PkScriptByIndex(originTxIndex)
			if pkScript == nil {
				str := fmt.Sprintf("unable to find unspent "+
					"output %v script referenced from "+
					"transaction %s:%d",
					txIn.PreviousOutPoint, txVI.tx.Sha(),
					txVI.txInIndex)
				err := ruleError(ErrBadTxInput, str)
				v.sendResult(err)
				break out
			}

			// Create a new script engine for the script pair.
			sigScript := txIn.SignatureScript
			inputAmount := txEntry.AmountByIndex(originTxIndex)
			vm, err := txscript.NewEngine(pkScript, txVI.tx.MsgTx(),
				txVI.txInIndex, v.flags, v.sigCache, v.hashCache,
				inputAmount)
			if err != nil {
				str := fmt.Sprintf("failed to parse input "+
					"%s:%d which references output %s:%d - "+
					"%v (input script bytes %x, prev output "+
					"script bytes %x)", txVI.tx.Sha(),
					txVI.txInIndex, originTxHash,
					originTxIndex, err, sigScript, pkScript)
				err := ruleError(ErrScriptMalformed, str)
				v.sendResult(err)
				break out
			}

			// Execute the script pair.
			if err := vm.Execute(); err != nil {
				str := fmt.Sprintf("failed to validate input "+
					"%s:%d which references output %s:%d - "+
					"%v (input script bytes %x, prev output "+
					"script bytes %x)", txVI.tx.Sha(),
					txVI.txInIndex, originTxHash,
					originTxIndex, err, sigScript, pkScript)
				err := ruleError(ErrScriptValidation, str)
				v.sendResult(err)
				break out
			}

			// Validation succeeded.
			v.sendResult(nil)

		case <-v.quitChan:
			break out
		}
	}
}

// Validate validates the scripts for all of the passed transaction inputs using
// multiple goroutines.
func (v *txValidator) Validate(items []*txValidateItem) error {
	if len(items) == 0 {
		return nil
	}

	// Limit the number of goroutines to do script validation based on the
	// number of processor cores.  This help ensure the system stays
	// reasonably responsive under heavy load.
	maxGoRoutines := runtime.NumCPU() * 3
	if maxGoRoutines <= 0 {
		maxGoRoutines = 1
	}
	if maxGoRoutines > len(items) {
		maxGoRoutines = len(items)
	}

	// Start up validation handlers that are used to asynchronously
	// validate each transaction input.
	for i := 0; i < maxGoRoutines; i++ {
		go v.validateHandler()
	}

	// Validate each of the inputs.  The quit channel is closed when any
	// errors occur so all processing goroutines exit regardless of which
	// input had the validation error.
	numInputs := len(items)
	currentItem := 0
	processedItems := 0
	for processedItems < numInputs {
		// Only send items while there are still items that need to
		// be processed.  The select statement will never select a nil
		// channel.
		var validateChan chan *txValidateItem
		var item *txValidateItem
		if currentItem < numInputs {
			validateChan = v.validateChan
			item = items[currentItem]
		}

		select {
		case validateChan <- item:
			currentItem++

		case err := <-v.resultChan:
			processedItems++
			if err != nil {
				close(v.quitChan)
				return err
			}
		}
	}

	close(v.quitChan)
	return nil
}

// newTxValidator returns a new instance of txValidator to be used for
// validating transaction scripts asynchronously.
func newTxValidator(utxoView *UtxoViewpoint, flags txscript.ScriptFlags,
	sigCache *txscript.SigCache, hashCache *txscript.HashCache) *txValidator {
	return &txValidator{
		validateChan: make(chan *txValidateItem),
		quitChan:     make(chan struct{}),
		resultChan:   make(chan error),
		utxoView:     utxoView,
		sigCache:     sigCache,
		hashCache:    hashCache,
		flags:        flags,
	}
}

// ValidateTransactionScripts validates the scripts for the passed transaction
// using multiple goroutines.
func ValidateTransactionScripts(tx *btcutil.Tx, utxoView *UtxoViewpoint,
	flags txscript.ScriptFlags, sigCache *txscript.SigCache,
	hashCache *txscript.HashCache) error {

	msgTx := tx.MsgTx()
	if !hashCache.ContainsHashes(msgTx) {
		hashCache.AddSigHashes(msgTx)
	}

	// Collect all of the transaction inputs and required information for
	// validation.
	txIns := tx.MsgTx().TxIn
	txValItems := make([]*txValidateItem, 0, len(txIns))
	for txInIdx, txIn := range txIns {
		// Skip coinbases.
		if txIn.PreviousOutPoint.Index == math.MaxUint32 {
			continue
		}

		txVI := &txValidateItem{
			txInIndex: txInIdx,
			txIn:      txIn,
			tx:        tx,
		}
		txValItems = append(txValItems, txVI)
	}

	// Validate all of the inputs.
	validator := newTxValidator(utxoView, flags, sigCache, hashCache)
	if err := validator.Validate(txValItems); err != nil {
		return err
	}

	return nil
}

// checkBlockScripts executes and validates the scripts for all transactions in
// the passed block using multiple goroutines.
func checkBlockScripts(block *btcutil.Block, utxoView *UtxoViewpoint,
	scriptFlags txscript.ScriptFlags, sigCache *txscript.SigCache,
	hashCache *txscript.HashCache) error {

	// Collect all of the transaction inputs and required information for
	// validation for all transactions in the block into a single slice.
	numInputs := 0
	for _, tx := range block.Transactions() {
		numInputs += len(tx.MsgTx().TxIn)
	}
	txValItems := make([]*txValidateItem, 0, numInputs)
	for _, tx := range block.Transactions() {
		msgTx := tx.MsgTx()

		// If the HashCache is present, and it doesn't yet contain the
		// partial sighashes for this transaction, then we add the
		// sighashes for the transaction. This allows us to take
		// advantage of the potential speed savings due to the new
		// digest algorithm (BIP0143).
		// TODO(roasbeef): guard with segwit script flag
		if hashCache != nil && !hashCache.ContainsHashes(msgTx) {
			hashCache.AddSigHashes(msgTx)
		}

		for txInIdx, txIn := range msgTx.TxIn {
			// Skip coinbases.
			if txIn.PreviousOutPoint.Index == math.MaxUint32 {
				continue
			}

			txVI := &txValidateItem{
				txInIndex: txInIdx,
				txIn:      txIn,
				tx:        tx,
			}
			txValItems = append(txValItems, txVI)
		}
	}

	// Validate all of the inputs.
	validator := newTxValidator(utxoView, scriptFlags, sigCache, hashCache)
	if err := validator.Validate(txValItems); err != nil {
		return err
	}

	// If the HashCache is present, once we have validated the block, we no
	// longer need the cached hashes for these transactions, so we purge
	// them from the cache.
	if hashCache != nil {
		for _, tx := range block.Transactions() {
			hashCache.PurgeSigHashes(tx.MsgTx())
		}
	}

	return nil
}
<commit_msg>blockchain: temporary logging<commit_after>// Copyright (c) 2013-2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package blockchain

import (
	"fmt"
	"math"
	"runtime"
	"time"

	"github.com/btcsuite/btcd/txscript"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
)

// txValidateItem holds a transaction along with which input to validate.
type txValidateItem struct {
	txInIndex int
	txIn      *wire.TxIn
	tx        *btcutil.Tx
}

// txValidator provides a type which asynchronously validates transaction
// inputs.  It provides several channels for communication and a processing
// function that is intended to be in run multiple goroutines.
type txValidator struct {
	validateChan chan *txValidateItem
	quitChan     chan struct{}
	resultChan   chan error
	utxoView     *UtxoViewpoint
	flags        txscript.ScriptFlags
	sigCache     *txscript.SigCache
	hashCache    *txscript.HashCache
}

// sendResult sends the result of a script pair validation on the internal
// result channel while respecting the quit channel.  The allows orderly
// shutdown when the validation process is aborted early due to a validation
// error in one of the other goroutines.
func (v *txValidator) sendResult(result error) {
	select {
	case v.resultChan <- result:
	case <-v.quitChan:
	}
}

// validateHandler consumes items to validate from the internal validate channel
// and returns the result of the validation on the internal result channel. It
// must be run as a goroutine.
func (v *txValidator) validateHandler() {
out:
	for {
		select {
		case txVI := <-v.validateChan:
			// Ensure the referenced input transaction is available.
			txIn := txVI.txIn
			originTxHash := &txIn.PreviousOutPoint.Hash
			originTxIndex := txIn.PreviousOutPoint.Index
			txEntry := v.utxoView.LookupEntry(originTxHash)
			if txEntry == nil {
				str := fmt.Sprintf("unable to find input "+
					"transaction %v referenced from "+
					"transaction %v", originTxHash,
					txVI.tx.Sha())
				err := ruleError(ErrMissingTx, str)
				v.sendResult(err)
				break out
			}

			// Ensure the referenced input transaction public key
			// script is available.
			pkScript := txEntry.PkScriptByIndex(originTxIndex)
			if pkScript == nil {
				str := fmt.Sprintf("unable to find unspent "+
					"output %v script referenced from "+
					"transaction %s:%d",
					txIn.PreviousOutPoint, txVI.tx.Sha(),
					txVI.txInIndex)
				err := ruleError(ErrBadTxInput, str)
				v.sendResult(err)
				break out
			}

			// Create a new script engine for the script pair.
			sigScript := txIn.SignatureScript
			inputAmount := txEntry.AmountByIndex(originTxIndex)
			vm, err := txscript.NewEngine(pkScript, txVI.tx.MsgTx(),
				txVI.txInIndex, v.flags, v.sigCache, v.hashCache,
				inputAmount)
			if err != nil {
				str := fmt.Sprintf("failed to parse input "+
					"%s:%d which references output %s:%d - "+
					"%v (input script bytes %x, prev output "+
					"script bytes %x)", txVI.tx.Sha(),
					txVI.txInIndex, originTxHash,
					originTxIndex, err, sigScript, pkScript)
				err := ruleError(ErrScriptMalformed, str)
				v.sendResult(err)
				break out
			}

			// Execute the script pair.
			if err := vm.Execute(); err != nil {
				str := fmt.Sprintf("failed to validate input "+
					"%s:%d which references output %s:%d - "+
					"%v (input script bytes %x, prev output "+
					"script bytes %x)", txVI.tx.Sha(),
					txVI.txInIndex, originTxHash,
					originTxIndex, err, sigScript, pkScript)
				err := ruleError(ErrScriptValidation, str)
				v.sendResult(err)
				break out
			}

			// Validation succeeded.
			v.sendResult(nil)

		case <-v.quitChan:
			break out
		}
	}
}

// Validate validates the scripts for all of the passed transaction inputs using
// multiple goroutines.
func (v *txValidator) Validate(items []*txValidateItem) error {
	if len(items) == 0 {
		return nil
	}

	// Limit the number of goroutines to do script validation based on the
	// number of processor cores.  This help ensure the system stays
	// reasonably responsive under heavy load.
	maxGoRoutines := runtime.NumCPU() * 3
	if maxGoRoutines <= 0 {
		maxGoRoutines = 1
	}
	if maxGoRoutines > len(items) {
		maxGoRoutines = len(items)
	}

	// Start up validation handlers that are used to asynchronously
	// validate each transaction input.
	for i := 0; i < maxGoRoutines; i++ {
		go v.validateHandler()
	}

	// Validate each of the inputs.  The quit channel is closed when any
	// errors occur so all processing goroutines exit regardless of which
	// input had the validation error.
	numInputs := len(items)
	currentItem := 0
	processedItems := 0
	for processedItems < numInputs {
		// Only send items while there are still items that need to
		// be processed.  The select statement will never select a nil
		// channel.
		var validateChan chan *txValidateItem
		var item *txValidateItem
		if currentItem < numInputs {
			validateChan = v.validateChan
			item = items[currentItem]
		}

		select {
		case validateChan <- item:
			currentItem++

		case err := <-v.resultChan:
			processedItems++
			if err != nil {
				close(v.quitChan)
				return err
			}
		}
	}

	close(v.quitChan)
	return nil
}

// newTxValidator returns a new instance of txValidator to be used for
// validating transaction scripts asynchronously.
func newTxValidator(utxoView *UtxoViewpoint, flags txscript.ScriptFlags,
	sigCache *txscript.SigCache, hashCache *txscript.HashCache) *txValidator {
	return &txValidator{
		validateChan: make(chan *txValidateItem),
		quitChan:     make(chan struct{}),
		resultChan:   make(chan error),
		utxoView:     utxoView,
		sigCache:     sigCache,
		hashCache:    hashCache,
		flags:        flags,
	}
}

// ValidateTransactionScripts validates the scripts for the passed transaction
// using multiple goroutines.
func ValidateTransactionScripts(tx *btcutil.Tx, utxoView *UtxoViewpoint,
	flags txscript.ScriptFlags, sigCache *txscript.SigCache,
	hashCache *txscript.HashCache) error {

	msgTx := tx.MsgTx()
	if !hashCache.ContainsHashes(msgTx) {
		hashCache.AddSigHashes(msgTx)
	}

	// Collect all of the transaction inputs and required information for
	// validation.
	txIns := tx.MsgTx().TxIn
	txValItems := make([]*txValidateItem, 0, len(txIns))
	for txInIdx, txIn := range txIns {
		// Skip coinbases.
		if txIn.PreviousOutPoint.Index == math.MaxUint32 {
			continue
		}

		txVI := &txValidateItem{
			txInIndex: txInIdx,
			txIn:      txIn,
			tx:        tx,
		}
		txValItems = append(txValItems, txVI)
	}

	// Validate all of the inputs.
	validator := newTxValidator(utxoView, flags, sigCache, hashCache)
	if err := validator.Validate(txValItems); err != nil {
		return err
	}

	return nil
}

// checkBlockScripts executes and validates the scripts for all transactions in
// the passed block using multiple goroutines.
func checkBlockScripts(block *btcutil.Block, utxoView *UtxoViewpoint,
	scriptFlags txscript.ScriptFlags, sigCache *txscript.SigCache,
	hashCache *txscript.HashCache) error {

	// Collect all of the transaction inputs and required information for
	// validation for all transactions in the block into a single slice.
	numInputs := 0
	for _, tx := range block.Transactions() {
		numInputs += len(tx.MsgTx().TxIn)
	}
	txValItems := make([]*txValidateItem, 0, numInputs)
	for _, tx := range block.Transactions() {
		msgTx := tx.MsgTx()

		// If the HashCache is present, and it doesn't yet contain the
		// partial sighashes for this transaction, then we add the
		// sighashes for the transaction. This allows us to take
		// advantage of the potential speed savings due to the new
		// digest algorithm (BIP0143).
		// TODO(roasbeef): guard with segwit script flag
		if hashCache != nil && !hashCache.ContainsHashes(msgTx) {
			hashCache.AddSigHashes(msgTx)
		}

		for txInIdx, txIn := range msgTx.TxIn {
			// Skip coinbases.
			if txIn.PreviousOutPoint.Index == math.MaxUint32 {
				continue
			}

			txVI := &txValidateItem{
				txInIndex: txInIdx,
				txIn:      txIn,
				tx:        tx,
			}
			txValItems = append(txValItems, txVI)
		}
	}

	// Validate all of the inputs.
	start := time.Now()
	validator := newTxValidator(utxoView, scriptFlags, sigCache, hashCache)
	if err := validator.Validate(txValItems); err != nil {
		return err
	}
	elapsed := time.Since(start)
	log.Infof("block %v took %v to verify", block.Sha(), elapsed)

	// If the HashCache is present, once we have validated the block, we no
	// longer need the cached hashes for these transactions, so we purge
	// them from the cache.
	if hashCache != nil {
		for _, tx := range block.Transactions() {
			hashCache.PurgeSigHashes(tx.MsgTx())
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package command

import (
	"net/http"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/gorilla/mux"
)

func init() {
	cmdMaster.Run = runMaster // break init cycle
}

var cmdMaster = &Command{
	UsageLine: "master -port=9333",
	Short:     "start a master server",
	Long: `start a master server to provide volume=>location mapping service
  and sequence number of file ids

  `,
}

var (
	mport                   = cmdMaster.Flag.Int("port", 9333, "http listen port")
	masterIp                = cmdMaster.Flag.String("ip", "localhost", "master <ip>|<server> address")
	masterBindIp            = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	metaFolder              = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
	masterPeers             = cmdMaster.Flag.String("peers", "", "other master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094")
	volumeSizeLimitMB       = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
	mpulse                  = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	confFile                = cmdMaster.Flag.String("conf", "/etc/weedfs/weedfs.conf", "Deprecating! xml configuration file")
	defaultReplicaPlacement = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
	mTimeout                = cmdMaster.Flag.Int("idleTimeout", 10, "connection idle seconds")
	mMaxCpu                 = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
	garbageThreshold        = cmdMaster.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces")
	masterWhiteListOption   = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	masterSecureKey         = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
	masterCpuProfile        = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")

	masterWhiteList []string
)

func runMaster(cmd *Command, args []string) bool {
	if *mMaxCpu < 1 {
		*mMaxCpu = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(*mMaxCpu)
	if *masterCpuProfile != "" {
		f, err := os.Create(*masterCpuProfile)
		if err != nil {
			glog.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	if err := util.TestFolderWritable(*metaFolder); err != nil {
		glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *metaFolder, err)
	}
	if *masterWhiteListOption != "" {
		masterWhiteList = strings.Split(*masterWhiteListOption, ",")
	}

	r := mux.NewRouter()
	ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
		*volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold,
		masterWhiteList, *masterSecureKey,
	)

	listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport)

	glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress)

	listener, e := util.NewListener(listeningAddress, time.Duration(*mTimeout)*time.Second)
	if e != nil {
		glog.Fatalf("Master startup error: %v", e)
	}

	go func() {
		time.Sleep(100 * time.Millisecond)
		myMasterAddress := *masterIp + ":" + strconv.Itoa(*mport)
		var peers []string
		if *masterPeers != "" {
			peers = strings.Split(*masterPeers, ",")
		}
		raftServer := weed_server.NewRaftServer(r, peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
		ms.SetRaftServer(raftServer)
	}()

	if e := http.Serve(listener, r); e != nil {
		glog.Fatalf("Fail to serve: %v", e)
	}
	return true
}
<commit_msg>collect proof when stopping<commit_after>package command

import (
	"net/http"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/server"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/gorilla/mux"
)

func init() {
	cmdMaster.Run = runMaster // break init cycle
}

var cmdMaster = &Command{
	UsageLine: "master -port=9333",
	Short:     "start a master server",
	Long: `start a master server to provide volume=>location mapping service
  and sequence number of file ids

  `,
}

var (
	mport                   = cmdMaster.Flag.Int("port", 9333, "http listen port")
	masterIp                = cmdMaster.Flag.String("ip", "localhost", "master <ip>|<server> address")
	masterBindIp            = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	metaFolder              = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
	masterPeers             = cmdMaster.Flag.String("peers", "", "other master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094")
	volumeSizeLimitMB       = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
	mpulse                  = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	confFile                = cmdMaster.Flag.String("conf", "/etc/weedfs/weedfs.conf", "Deprecating! xml configuration file")
	defaultReplicaPlacement = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
	mTimeout                = cmdMaster.Flag.Int("idleTimeout", 10, "connection idle seconds")
	mMaxCpu                 = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
	garbageThreshold        = cmdMaster.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces")
	masterWhiteListOption   = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	masterSecureKey         = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
	masterCpuProfile        = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")

	masterWhiteList []string
)

func runMaster(cmd *Command, args []string) bool {
	if *mMaxCpu < 1 {
		*mMaxCpu = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(*mMaxCpu)
	if *masterCpuProfile != "" {
		f, err := os.Create(*masterCpuProfile)
		if err != nil {
			glog.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
		OnInterrupt(func() {
			pprof.StopCPUProfile()
		})
	}
	if err := util.TestFolderWritable(*metaFolder); err != nil {
		glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *metaFolder, err)
	}
	if *masterWhiteListOption != "" {
		masterWhiteList = strings.Split(*masterWhiteListOption, ",")
	}

	r := mux.NewRouter()
	ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
		*volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold,
		masterWhiteList, *masterSecureKey,
	)

	listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport)

	glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress)

	listener, e := util.NewListener(listeningAddress, time.Duration(*mTimeout)*time.Second)
	if e != nil {
		glog.Fatalf("Master startup error: %v", e)
	}

	go func() {
		time.Sleep(100 * time.Millisecond)
		myMasterAddress := *masterIp + ":" + strconv.Itoa(*mport)
		var peers []string
		if *masterPeers != "" {
			peers = strings.Split(*masterPeers, ",")
		}
		raftServer := weed_server.NewRaftServer(r, peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
		ms.SetRaftServer(raftServer)
	}()

	if e := http.Serve(listener, r); e != nil {
		glog.Fatalf("Fail to serve: %v", e)
	}
	return true
}
<|endoftext|>


<commit_before>package util

const (
	VERSION = "0.94"
)
<commit_msg>prepare for 0.95<commit_after>package util

const (
	VERSION = "0.95"
)
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"path/filepath"

	log "github.com/sirupsen/logrus"

	"github.com/lxc/lxd/lxd/ucred"
	"github.com/lxc/lxd/shared"
)

func tlsConfig(uid uint32) (*tls.Config, error) {
	// Load the client certificate.
	content, err := ioutil.ReadFile(filepath.Join("users", fmt.Sprintf("%d", uid), "client.crt"))
	if err != nil {
		return nil, fmt.Errorf("Unable to open client certificate: %w", err)
	}

	tlsClientCert := string(content)

	// Load the client key.
	content, err = ioutil.ReadFile(filepath.Join("users", fmt.Sprintf("%d", uid), "client.key"))
	if err != nil {
		return nil, fmt.Errorf("Unable to open client key: %w", err)
	}

	tlsClientKey := string(content)

	// Load the server certificate.
	content, err = ioutil.ReadFile(shared.VarPath("server.crt"))
	if err != nil {
		return nil, fmt.Errorf("Unable to open server certificate: %w", err)
	}

	tlsServerCert := string(content)

	return shared.GetTLSConfigMem(tlsClientCert, tlsClientKey, "", tlsServerCert, false)
}

func proxyConnection(conn *net.UnixConn) {
	defer func() {
		_ = conn.Close()

		mu.Lock()
		connections -= 1
		mu.Unlock()
	}()

	// Increase counters.
	mu.Lock()
	transactions += 1
	connections += 1
	mu.Unlock()

	// Get credentials.
	creds, err := ucred.GetCred(conn)
	if err != nil {
		log.Errorf("Unable to get user credentials: %s", err)
		return
	}

	// Setup logging context.
	logger := log.WithFields(log.Fields{
		"uid": creds.Uid,
		"gid": creds.Gid,
		"pid": creds.Pid,
	})

	logger.Debug("Connected")
	defer logger.Debug("Disconnected")

	// Check if the user was setup.
	if !shared.PathExists(filepath.Join("users", fmt.Sprintf("%d", creds.Uid))) {
		log.Infof("Setting up LXD for uid %d", creds.Uid)
		err := lxdSetupUser(creds.Uid)
		if err != nil {
			log.Errorf("Failed to setup new user: %v", err)
			return
		}
	}

	// Connect to LXD.
	unixAddr, err := net.ResolveUnixAddr("unix", shared.VarPath("unix.socket"))
	if err != nil {
		log.Errorf("Unable to resolve the target server: %v", err)
		return
	}

	client, err := net.DialUnix("unix", nil, unixAddr)
	if err != nil {
		log.Errorf("Unable to connect to target server: %v", err)
		return
	}

	defer func() { _ = client.Close() }()

	// Get the TLS configuration
	tlsConfig, err := tlsConfig(creds.Uid)
	if err != nil {
		log.Errorf("Failed to load TLS connection settings: %v", err)
		return
	}

	// Setup TLS.
	_, err = client.Write([]byte("STARTTLS\n"))
	if err != nil {
		log.Errorf("Failed to setup TLS connection to target server: %v", err)
		return
	}

	tlsClient := tls.Client(client, tlsConfig)

	// Establish the TLS handshake.
	err = tlsClient.Handshake()
	if err != nil {
		_ = conn.Close()
		log.Errorf("Failed TLS handshake with target server: %v", err)
		return
	}

	// Start proxying.
	go func() { _, _ = io.Copy(conn, tlsClient) }()
	_, _ = io.Copy(tlsClient, conn)
}
<commit_msg>lxd-user: Fix use with clusters<commit_after>package main

import (
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"path/filepath"

	log "github.com/sirupsen/logrus"

	"github.com/lxc/lxd/lxd/ucred"
	"github.com/lxc/lxd/shared"
)

func tlsConfig(uid uint32) (*tls.Config, error) {
	// Load the client certificate.
	content, err := ioutil.ReadFile(filepath.Join("users", fmt.Sprintf("%d", uid), "client.crt"))
	if err != nil {
		return nil, fmt.Errorf("Unable to open client certificate: %w", err)
	}

	tlsClientCert := string(content)

	// Load the client key.
	content, err = ioutil.ReadFile(filepath.Join("users", fmt.Sprintf("%d", uid), "client.key"))
	if err != nil {
		return nil, fmt.Errorf("Unable to open client key: %w", err)
	}

	tlsClientKey := string(content)

	// Load the server certificate.
	certPath := shared.VarPath("cluster.crt")
	if !shared.PathExists(certPath) {
		certPath = shared.VarPath("server.crt")
	}

	content, err = ioutil.ReadFile(certPath)
	if err != nil {
		return nil, fmt.Errorf("Unable to open server certificate: %w", err)
	}

	tlsServerCert := string(content)

	return shared.GetTLSConfigMem(tlsClientCert, tlsClientKey, "", tlsServerCert, false)
}

func proxyConnection(conn *net.UnixConn) {
	defer func() {
		_ = conn.Close()

		mu.Lock()
		connections -= 1
		mu.Unlock()
	}()

	// Increase counters.
	mu.Lock()
	transactions += 1
	connections += 1
	mu.Unlock()

	// Get credentials.
	creds, err := ucred.GetCred(conn)
	if err != nil {
		log.Errorf("Unable to get user credentials: %s", err)
		return
	}

	// Setup logging context.
	logger := log.WithFields(log.Fields{
		"uid": creds.Uid,
		"gid": creds.Gid,
		"pid": creds.Pid,
	})

	logger.Debug("Connected")
	defer logger.Debug("Disconnected")

	// Check if the user was setup.
	if !shared.PathExists(filepath.Join("users", fmt.Sprintf("%d", creds.Uid))) {
		log.Infof("Setting up LXD for uid %d", creds.Uid)
		err := lxdSetupUser(creds.Uid)
		if err != nil {
			log.Errorf("Failed to setup new user: %v", err)
			return
		}
	}

	// Connect to LXD.
	unixAddr, err := net.ResolveUnixAddr("unix", shared.VarPath("unix.socket"))
	if err != nil {
		log.Errorf("Unable to resolve the target server: %v", err)
		return
	}

	client, err := net.DialUnix("unix", nil, unixAddr)
	if err != nil {
		log.Errorf("Unable to connect to target server: %v", err)
		return
	}

	defer func() { _ = client.Close() }()

	// Get the TLS configuration
	tlsConfig, err := tlsConfig(creds.Uid)
	if err != nil {
		log.Errorf("Failed to load TLS connection settings: %v", err)
		return
	}

	// Setup TLS.
	_, err = client.Write([]byte("STARTTLS\n"))
	if err != nil {
		log.Errorf("Failed to setup TLS connection to target server: %v", err)
		return
	}

	tlsClient := tls.Client(client, tlsConfig)

	// Establish the TLS handshake.
	err = tlsClient.Handshake()
	if err != nil {
		_ = conn.Close()
		log.Errorf("Failed TLS handshake with target server: %v", err)
		return
	}

	// Start proxying.
	go func() { _, _ = io.Copy(conn, tlsClient) }()
	_, _ = io.Copy(tlsClient, conn)
}
<|endoftext|>


<commit_before>package autorelay_test

import (
	"context"
	"net"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/libp2p/go-libp2p"
	"github.com/libp2p/go-libp2p/p2p/host/autorelay"
	relayv1 "github.com/libp2p/go-libp2p/p2p/protocol/circuitv1/relay"
	relayv2 "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"

	"github.com/libp2p/go-libp2p-core/event"
	"github.com/libp2p/go-libp2p-core/host"
	"github.com/libp2p/go-libp2p-core/network"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/routing"

	discovery "github.com/libp2p/go-libp2p-discovery"

	"github.com/ipfs/go-cid"
	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr/net"
)

// test specific parameters
func init() {
	autorelay.BootDelay = 1 * time.Second
	autorelay.AdvertiseBootDelay = 100 * time.Millisecond
}

// mock routing
type mockRoutingTable struct {
	mx        sync.Mutex
	providers map[string]map[peer.ID]peer.AddrInfo
	peers     map[peer.ID]peer.AddrInfo
}

type mockRouting struct {
	h   host.Host
	tab *mockRoutingTable
}

func newMockRoutingTable() *mockRoutingTable {
	return &mockRoutingTable{providers: make(map[string]map[peer.ID]peer.AddrInfo)}
}

func newMockRouting(h host.Host, tab *mockRoutingTable) *mockRouting {
	return &mockRouting{h: h, tab: tab}
}

func (m *mockRouting) FindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, error) {
	m.tab.mx.Lock()
	defer m.tab.mx.Unlock()
	pi, ok := m.tab.peers[p]
	if !ok {
		return peer.AddrInfo{}, routing.ErrNotFound
	}
	return pi, nil
}

func (m *mockRouting) Provide(ctx context.Context, cid cid.Cid, bcast bool) error {
	m.tab.mx.Lock()
	defer m.tab.mx.Unlock()

	pmap, ok := m.tab.providers[cid.String()]
	if !ok {
		pmap = make(map[peer.ID]peer.AddrInfo)
		m.tab.providers[cid.String()] = pmap
	}

	pi := peer.AddrInfo{ID: m.h.ID(), Addrs: m.h.Addrs()}
	pmap[m.h.ID()] = pi
	if m.tab.peers == nil {
		m.tab.peers = make(map[peer.ID]peer.AddrInfo)
	}
	m.tab.peers[m.h.ID()] = pi

	return nil
}

func (m *mockRouting) FindProvidersAsync(ctx context.Context, cid cid.Cid, limit int) <-chan peer.AddrInfo {
	ch := make(chan peer.AddrInfo)
	go func() {
		defer close(ch)
		m.tab.mx.Lock()
		defer m.tab.mx.Unlock()

		pmap, ok := m.tab.providers[cid.String()]
		if !ok {
			return
		}

		for _, pi := range pmap {
			select {
			case ch <- pi:
			case <-ctx.Done():
				return
			}
		}
	}()

	return ch
}

// connector
func connect(t *testing.T, a, b host.Host) {
	pinfo := peer.AddrInfo{ID: a.ID(), Addrs: a.Addrs()}
	err := b.Connect(context.Background(), pinfo)
	if err != nil {
		t.Fatal(err)
	}
}

// and the actual test!
func TestAutoRelay(t *testing.T) {
	manet.Private4 = []*net.IPNet{}

	t.Run("with a circuitv1 relay", func(t *testing.T) {
		testAutoRelay(t, false)
	})
	t.Run("testing autorelay with circuitv2 relay", func(t *testing.T) {
		testAutoRelay(t, true)
	})
}

func isRelayAddr(addr ma.Multiaddr) bool {
	_, err := addr.ValueForProtocol(ma.P_CIRCUIT)
	return err == nil
}

func testAutoRelay(t *testing.T, useRelayv2 bool) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	mtab := newMockRoutingTable()
	makeRouting := func(h host.Host) (*mockRouting, error) {
		mr := newMockRouting(h, mtab)
		return mr, nil
	}
	makePeerRouting := func(h host.Host) (routing.PeerRouting, error) {
		return makeRouting(h)
	}

	// this is the relay host
	// announce dns addrs because filter out private addresses from relays,
	// and we consider dns addresses "public".
	relayHost, err := libp2p.New(
		libp2p.DisableRelay(),
		libp2p.AddrsFactory(func(addrs []ma.Multiaddr) []ma.Multiaddr {
			for i, addr := range addrs {
				saddr := addr.String()
				if strings.HasPrefix(saddr, "/ip4/127.0.0.1/") {
					addrNoIP := strings.TrimPrefix(saddr, "/ip4/127.0.0.1")
					addrs[i] = ma.StringCast("/dns4/localhost" + addrNoIP)
				}
			}
			return addrs
		}))
	if err != nil {
		t.Fatal(err)
	}

	// instantiate the relay
	if useRelayv2 {
		r, err := relayv2.New(relayHost)
		if err != nil {
			t.Fatal(err)
		}
		defer r.Close()
	} else {
		r, err := relayv1.NewRelay(relayHost)
		if err != nil {
			t.Fatal(err)
		}
		defer r.Close()
	}

	// advertise the relay
	relayRouting, err := makeRouting(relayHost)
	if err != nil {
		t.Fatal(err)
	}
	relayDiscovery := discovery.NewRoutingDiscovery(relayRouting)
	autorelay.Advertise(ctx, relayDiscovery)

	// the client hosts
	h1, err := libp2p.New(libp2p.EnableRelay())
	if err != nil {
		t.Fatal(err)
	}

	h2, err := libp2p.New(libp2p.EnableRelay(), libp2p.EnableAutoRelay(), libp2p.Routing(makePeerRouting))
	if err != nil {
		t.Fatal(err)
	}
	h3, err := libp2p.New(libp2p.EnableRelay())
	if err != nil {
		t.Fatal(err)
	}

	// verify that we don't advertise relay addrs initially
	for _, addr := range h2.Addrs() {
		if isRelayAddr(addr) {
			t.Fatal("relay addr advertised before auto detection")
		}
	}

	// connect to AutoNAT, have it resolve to private.
	connect(t, h1, h2)
	time.Sleep(300 * time.Millisecond)
	privEmitter, _ := h2.EventBus().Emitter(new(event.EvtLocalReachabilityChanged))
	privEmitter.Emit(event.EvtLocalReachabilityChanged{Reachability: network.ReachabilityPrivate})
	// Wait for detection to do its magic
	time.Sleep(3000 * time.Millisecond)

	// verify that we now advertise relay addrs (but not unspecific relay addrs)
	unspecificRelay := ma.StringCast("/p2p-circuit")
	var haveRelay bool
	for _, addr := range h2.Addrs() {
		if addr.Equal(unspecificRelay) {
			t.Fatal("unspecific relay addr advertised")
		}
		if isRelayAddr(addr) {
			haveRelay = true
		}
	}
	if !haveRelay {
		t.Fatal("No relay addrs advertised")
	}

	// verify that we can connect through the relay
	var raddrs []ma.Multiaddr
	for _, addr := range h2.Addrs() {
		if isRelayAddr(addr) {
			raddrs = append(raddrs, addr)
		}
	}

	err = h3.Connect(ctx, peer.AddrInfo{ID: h2.ID(), Addrs: raddrs})
	if err != nil {
		t.Fatal(err)
	}

	// verify that we have pushed relay addrs to connected peers
	haveRelay = false
	for _, addr := range h1.Peerstore().Addrs(h2.ID()) {
		if addr.Equal(unspecificRelay) {
			t.Fatal("unspecific relay addr advertised")
		}

		_, err := addr.ValueForProtocol(ma.P_CIRCUIT)
		if err == nil {
			haveRelay = true
		}
	}

	if !haveRelay {
		t.Fatal("No relay addrs pushed")
	}
}
<commit_msg>fix missing private address reset after auto relay test<commit_after>package autorelay_test

import (
	"context"
	"net"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/libp2p/go-libp2p"
	"github.com/libp2p/go-libp2p/p2p/host/autorelay"
	relayv1 "github.com/libp2p/go-libp2p/p2p/protocol/circuitv1/relay"
	relayv2 "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"

	"github.com/libp2p/go-libp2p-core/event"
	"github.com/libp2p/go-libp2p-core/host"
	"github.com/libp2p/go-libp2p-core/network"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/routing"

	discovery "github.com/libp2p/go-libp2p-discovery"

	"github.com/ipfs/go-cid"
	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr/net"
)

// test specific parameters
func init() {
	autorelay.BootDelay = 1 * time.Second
	autorelay.AdvertiseBootDelay = 100 * time.Millisecond
}

// mock routing
type mockRoutingTable struct {
	mx        sync.Mutex
	providers map[string]map[peer.ID]peer.AddrInfo
	peers     map[peer.ID]peer.AddrInfo
}

type mockRouting struct {
	h   host.Host
	tab *mockRoutingTable
}

func newMockRoutingTable() *mockRoutingTable {
	return &mockRoutingTable{providers: make(map[string]map[peer.ID]peer.AddrInfo)}
}

func newMockRouting(h host.Host, tab *mockRoutingTable) *mockRouting {
	return &mockRouting{h: h, tab: tab}
}

func (m *mockRouting) FindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, error) {
	m.tab.mx.Lock()
	defer m.tab.mx.Unlock()
	pi, ok := m.tab.peers[p]
	if !ok {
		return peer.AddrInfo{}, routing.ErrNotFound
	}
	return pi, nil
}

func (m *mockRouting) Provide(ctx context.Context, cid cid.Cid, bcast bool) error {
	m.tab.mx.Lock()
	defer m.tab.mx.Unlock()

	pmap, ok := m.tab.providers[cid.String()]
	if !ok {
		pmap = make(map[peer.ID]peer.AddrInfo)
		m.tab.providers[cid.String()] = pmap
	}

	pi := peer.AddrInfo{ID: m.h.ID(), Addrs: m.h.Addrs()}
	pmap[m.h.ID()] = pi
	if m.tab.peers == nil {
		m.tab.peers = make(map[peer.ID]peer.AddrInfo)
	}
	m.tab.peers[m.h.ID()] = pi

	return nil
}

func (m *mockRouting) FindProvidersAsync(ctx context.Context, cid cid.Cid, limit int) <-chan peer.AddrInfo {
	ch := make(chan peer.AddrInfo)
	go func() {
		defer close(ch)
		m.tab.mx.Lock()
		defer m.tab.mx.Unlock()

		pmap, ok := m.tab.providers[cid.String()]
		if !ok {
			return
		}

		for _, pi := range pmap {
			select {
			case ch <- pi:
			case <-ctx.Done():
				return
			}
		}
	}()

	return ch
}

// connector
func connect(t *testing.T, a, b host.Host) {
	pinfo := peer.AddrInfo{ID: a.ID(), Addrs: a.Addrs()}
	err := b.Connect(context.Background(), pinfo)
	if err != nil {
		t.Fatal(err)
	}
}

// and the actual test!
func TestAutoRelay(t *testing.T) {
	private4 := manet.Private4
	t.Cleanup(func() { manet.Private4 = private4 })
	manet.Private4 = []*net.IPNet{}

	t.Run("with a circuitv1 relay", func(t *testing.T) {
		testAutoRelay(t, false)
	})
	t.Run("testing autorelay with circuitv2 relay", func(t *testing.T) {
		testAutoRelay(t, true)
	})
}

func isRelayAddr(addr ma.Multiaddr) bool {
	_, err := addr.ValueForProtocol(ma.P_CIRCUIT)
	return err == nil
}

func testAutoRelay(t *testing.T, useRelayv2 bool) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	mtab := newMockRoutingTable()
	makeRouting := func(h host.Host) (*mockRouting, error) {
		mr := newMockRouting(h, mtab)
		return mr, nil
	}
	makePeerRouting := func(h host.Host) (routing.PeerRouting, error) {
		return makeRouting(h)
	}

	// this is the relay host
	// announce dns addrs because filter out private addresses from relays,
	// and we consider dns addresses "public".
	relayHost, err := libp2p.New(
		libp2p.DisableRelay(),
		libp2p.AddrsFactory(func(addrs []ma.Multiaddr) []ma.Multiaddr {
			for i, addr := range addrs {
				saddr := addr.String()
				if strings.HasPrefix(saddr, "/ip4/127.0.0.1/") {
					addrNoIP := strings.TrimPrefix(saddr, "/ip4/127.0.0.1")
					addrs[i] = ma.StringCast("/dns4/localhost" + addrNoIP)
				}
			}
			return addrs
		}))
	if err != nil {
		t.Fatal(err)
	}
	defer relayHost.Close()

	// instantiate the relay
	if useRelayv2 {
		r, err := relayv2.New(relayHost)
		if err != nil {
			t.Fatal(err)
		}
		defer r.Close()
	} else {
		r, err := relayv1.NewRelay(relayHost)
		if err != nil {
			t.Fatal(err)
		}
		defer r.Close()
	}

	// advertise the relay
	relayRouting, err := makeRouting(relayHost)
	if err != nil {
		t.Fatal(err)
	}
	relayDiscovery := discovery.NewRoutingDiscovery(relayRouting)
	autorelay.Advertise(ctx, relayDiscovery)

	// the client hosts
	h1, err := libp2p.New(libp2p.EnableRelay())
	if err != nil {
		t.Fatal(err)
	}
	defer h1.Close()

	h2, err := libp2p.New(libp2p.EnableRelay(), libp2p.EnableAutoRelay(), libp2p.Routing(makePeerRouting))
	if err != nil {
		t.Fatal(err)
	}
	defer h2.Close()
	h3, err := libp2p.New(libp2p.EnableRelay())
	if err != nil {
		t.Fatal(err)
	}
	defer h3.Close()

	// verify that we don't advertise relay addrs initially
	for _, addr := range h2.Addrs() {
		if isRelayAddr(addr) {
			t.Fatal("relay addr advertised before auto detection")
		}
	}

	// connect to AutoNAT, have it resolve to private.
	connect(t, h1, h2)
	time.Sleep(300 * time.Millisecond)
	privEmitter, _ := h2.EventBus().Emitter(new(event.EvtLocalReachabilityChanged))
	privEmitter.Emit(event.EvtLocalReachabilityChanged{Reachability: network.ReachabilityPrivate})
	// Wait for detection to do its magic
	time.Sleep(3000 * time.Millisecond)

	// verify that we now advertise relay addrs (but not unspecific relay addrs)
	unspecificRelay := ma.StringCast("/p2p-circuit")
	var haveRelay bool
	for _, addr := range h2.Addrs() {
		if addr.Equal(unspecificRelay) {
			t.Fatal("unspecific relay addr advertised")
		}
		if isRelayAddr(addr) {
			haveRelay = true
		}
	}
	if !haveRelay {
		t.Fatal("No relay addrs advertised")
	}

	// verify that we can connect through the relay
	var raddrs []ma.Multiaddr
	for _, addr := range h2.Addrs() {
		if isRelayAddr(addr) {
			raddrs = append(raddrs, addr)
		}
	}

	err = h3.Connect(ctx, peer.AddrInfo{ID: h2.ID(), Addrs: raddrs})
	if err != nil {
		t.Fatal(err)
	}

	// verify that we have pushed relay addrs to connected peers
	haveRelay = false
	for _, addr := range h1.Peerstore().Addrs(h2.ID()) {
		if addr.Equal(unspecificRelay) {
			t.Fatal("unspecific relay addr advertised")
		}

		_, err := addr.ValueForProtocol(ma.P_CIRCUIT)
		if err == nil {
			haveRelay = true
		}
	}

	if !haveRelay {
		t.Fatal("No relay addrs pushed")
	}
}
<|endoftext|>


<commit_before>package noise

import (
	"bytes"
	"context"
	"errors"
	"io"
	"math/rand"
	"net"
	"testing"
	"time"

	crypto "github.com/libp2p/go-libp2p-core/crypto"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/sec"

	"github.com/stretchr/testify/require"
)

func newTestTransport(t *testing.T, typ, bits int) *Transport {
	priv, pub, err := crypto.GenerateKeyPair(typ, bits)
	if err != nil {
		t.Fatal(err)
	}
	id, err := peer.IDFromPublicKey(pub)
	if err != nil {
		t.Fatal(err)
	}
	return &Transport{
		localID:    id,
		privateKey: priv,
	}
}

// Create a new pair of connected TCP sockets.
func newConnPair(t *testing.T) (net.Conn, net.Conn) {
	lstnr, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		t.Fatalf("Failed to listen: %v", err)
		return nil, nil
	}

	var clientErr error
	var client net.Conn
	addr := lstnr.Addr()
	done := make(chan struct{})

	go func() {
		defer close(done)
		client, clientErr = net.Dial(addr.Network(), addr.String())
	}()

	server, err := lstnr.Accept()
	<-done

	lstnr.Close()

	if err != nil {
		t.Fatalf("Failed to accept: %v", err)
	}

	if clientErr != nil {
		t.Fatalf("Failed to connect: %v", clientErr)
	}

	return client, server
}

func connect(t *testing.T, initTransport, respTransport *Transport) (*secureSession, *secureSession) {
	init, resp := newConnPair(t)

	var initConn sec.SecureConn
	var initErr error
	done := make(chan struct{})
	go func() {
		defer close(done)
		initConn, initErr = initTransport.SecureOutbound(context.TODO(), init, respTransport.localID)
	}()

	respConn, respErr := respTransport.SecureInbound(context.TODO(), resp)
	<-done

	if initErr != nil {
		t.Fatal(initErr)
	}

	if respErr != nil {
		t.Fatal(respErr)
	}

	return initConn.(*secureSession), respConn.(*secureSession)
}

func TestDeadlines(t *testing.T) {
	initTransport := newTestTransport(t, crypto.Ed25519, 2048)
	respTransport := newTestTransport(t, crypto.Ed25519, 2048)

	init, resp := newConnPair(t)
	defer init.Close()
	defer resp.Close()

	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
	defer cancel()

	_, err := initTransport.SecureOutbound(ctx, init, respTransport.localID)
	if err == nil {
		t.Fatalf("expected i/o timeout err; got: %s", err)
	}

	var neterr net.Error
	if ok := errors.As(err, &neterr); !ok || !neterr.Timeout() {
		t.Fatalf("expected i/o timeout err; got: %s", err)
	}
}

func TestIDs(t *testing.T) {
	initTransport := newTestTransport(t, crypto.Ed25519, 2048)
	respTransport := newTestTransport(t, crypto.Ed25519, 2048)

	initConn, respConn := connect(t, initTransport, respTransport)
	defer initConn.Close()
	defer respConn.Close()

	if initConn.LocalPeer() != initTransport.localID {
		t.Fatal("Initiator Local Peer ID mismatch.")
	}

	if respConn.RemotePeer() != initTransport.localID {
		t.Fatal("Responder Remote Peer ID mismatch.")
	}

	if initConn.LocalPeer() != respConn.RemotePeer() {
		t.Fatal("Responder Local Peer ID mismatch.")
	}

	// TODO: check after stage 0 of handshake if updated
	if initConn.RemotePeer() != respTransport.localID {
		t.Errorf("Initiator Remote Peer ID mismatch. expected %x got %x", respTransport.localID, initConn.RemotePeer())
	}
}

func TestKeys(t *testing.T) {
	initTransport := newTestTransport(t, crypto.Ed25519, 2048)
	respTransport := newTestTransport(t, crypto.Ed25519, 2048)

	initConn, respConn := connect(t, initTransport, respTransport)
	defer initConn.Close()
	defer respConn.Close()

	sk := respConn.LocalPrivateKey()
	pk := sk.GetPublic()

	if !sk.Equals(respTransport.privateKey) {
		t.Error("Private key Mismatch.")
	}

	if !pk.Equals(initConn.RemotePublicKey()) {
		t.Errorf("Public key mismatch. expected %x got %x", pk, initConn.RemotePublicKey())
	}
}

func TestPeerIDMismatchFailsHandshake(t *testing.T) {
	initTransport := newTestTransport(t, crypto.Ed25519, 2048)
	respTransport := newTestTransport(t, crypto.Ed25519, 2048)
	init, resp := newConnPair(t)

	var initErr error
	done := make(chan struct{})
	go func() {
		defer close(done)
		_, initErr = initTransport.SecureOutbound(context.TODO(), init, "a-random-peer-id")
	}()

	_, _ = respTransport.SecureInbound(context.TODO(), resp)
	<-done

	if initErr == nil {
		t.Fatal("expected initiator to fail with peer ID mismatch error")
	}
}

func makeLargePlaintext(size int) []byte {
	buf := make([]byte, size)
	rand.Read(buf)
	return buf
}

func TestLargePayloads(t *testing.T) {
	initTransport := newTestTransport(t, crypto.Ed25519, 2048)
	respTransport := newTestTransport(t, crypto.Ed25519, 2048)

	initConn, respConn := connect(t, initTransport, respTransport)
	defer initConn.Close()
	defer respConn.Close()

	// enough to require a couple Noise messages, with a size that
	// isn't a neat multiple of Noise message size, just in case
	size := 100000

	before := makeLargePlaintext(size)
	_, err := initConn.Write(before)
	if err != nil {
		t.Fatal(err)
	}

	after := make([]byte, len(before))
	afterLen, err := io.ReadFull(respConn, after)
	if err != nil {
		t.Fatal(err)
	}

	if len(before) != afterLen {
		t.Errorf("expected to read same amount of data as written. written=%d read=%d", len(before), afterLen)
	}
	if !bytes.Equal(before, after) {
		t.Error("Message mismatch.")
	}
}

// Tests XX handshake
func TestHandshakeXX(t *testing.T) {
	initTransport := newTestTransport(t, crypto.Ed25519, 2048)
	respTransport := newTestTransport(t, crypto.Ed25519, 2048)

	initConn, respConn := connect(t, initTransport, respTransport)
	defer initConn.Close()
	defer respConn.Close()

	before := []byte("hello world")
	_, err := initConn.Write(before)
	if err != nil {
		t.Fatal(err)
	}

	after := make([]byte, len(before))
	_, err = respConn.Read(after)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(before, after) {
		t.Errorf("Message mismatch. %v != %v", before, after)
	}
}

func TestLargerBufferThanPayload(t *testing.T) {
	initTransport := newTestTransport(t, crypto.Ed25519, 2048)
	respTransport := newTestTransport(t, crypto.Ed25519, 2048)

	initConn, respConn := connect(t, initTransport, respTransport)
	defer initConn.Close()
	defer respConn.Close()

	before := []byte("hello world")
	_, err := initConn.Write(before)
	require.NoError(t, err)

	after := make([]byte, len(before)+2)
	afterLen, err := respConn.Read(after)
	require.NoError(t, err)

	require.Equal(t, len(before), afterLen)
	require.Equal(t, before, after[:len(before)])
	// all bytes after the payload should be empty
	require.Equal(t, []byte{0, 0}, after[len(before):])
}
<commit_msg>more unit tests<commit_after>package noise

import (
	"bytes"
	"context"
	"errors"
	"golang.org/x/crypto/poly1305"
	"io"
	"math/rand"
	"net"
	"testing"
	"time"

	crypto "github.com/libp2p/go-libp2p-core/crypto"
	"github.com/libp2p/go-libp2p-core/peer"
	"github.com/libp2p/go-libp2p-core/sec"

	"github.com/stretchr/testify/require"
)

func newTestTransport(t *testing.T, typ, bits int) *Transport {
	priv, pub, err := crypto.GenerateKeyPair(typ, bits)
	if err != nil {
		t.Fatal(err)
	}
	id, err := peer.IDFromPublicKey(pub)
	if err != nil {
		t.Fatal(err)
	}
	return &Transport{
		localID:    id,
		privateKey: priv,
	}
}

// Create a new pair of connected TCP sockets.
func newConnPair(t *testing.T) (net.Conn, net.Conn) {
	lstnr, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		t.Fatalf("Failed to listen: %v", err)
		return nil, nil
	}

	var clientErr error
	var client net.Conn
	addr := lstnr.Addr()
	done := make(chan struct{})

	go func() {
		defer close(done)
		client, clientErr = net.Dial(addr.Network(), addr.String())
	}()

	server, err := lstnr.Accept()
	<-done

	lstnr.Close()

	if err != nil {
		t.Fatalf("Failed to accept: %v", err)
	}

	if clientErr != nil {
		t.Fatalf("Failed to connect: %v", clientErr)
	}

	return client, server
}

func connect(t *testing.T, initTransport, respTransport *Transport) (*secureSession, *secureSession) {
	init, resp := newConnPair(t)

	var initConn sec.SecureConn
	var initErr error
	done := make(chan struct{})
	go func() {
		defer close(done)
		initConn, initErr = initTransport.SecureOutbound(context.TODO(), init, respTransport.localID)
	}()

	respConn, respErr := respTransport.SecureInbound(context.TODO(), resp)
	<-done

	if initErr != nil {
		t.Fatal(initErr)
	}

	if respErr != nil {
		t.Fatal(respErr)
	}

	return initConn.(*secureSession), respConn.(*secureSession)
}

func TestDeadlines(t *testing.T) {
	initTransport := newTestTransport(t, crypto.Ed25519, 2048)
	respTransport := newTestTransport(t, crypto.Ed25519, 2048)

	init, resp := newConnPair(t)
	defer init.Close()
	defer resp.Close()

	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
	defer cancel()

	_, err := initTransport.SecureOutbound(ctx, init, respTransport.localID)
	if err == nil {
		t.Fatalf("expected i/o timeout err; got: %s", err)
	}

	var neterr net.Error
	if ok := errors.As(err, &neterr); !ok || !neterr.Timeout() {
		t.Fatalf("expected i/o timeout err; got: %s", err)
	}
}

func TestIDs(t *testing.T) {
	initTransport := newTestTransport(t, crypto.Ed25519, 2048)
	respTransport := newTestTransport(t, crypto.Ed25519, 2048)

	initConn, respConn := connect(t, initTransport, respTransport)
	defer initConn.Close()
	defer respConn.Close()

	if initConn.LocalPeer() != initTransport.localID {
		t.Fatal("Initiator Local Peer ID mismatch.")
	}

	if respConn.RemotePeer() != initTransport.localID {
		t.Fatal("Responder Remote Peer ID mismatch.")
	}

	if initConn.LocalPeer() != respConn.RemotePeer() {
		t.Fatal("Responder Local Peer ID mismatch.")
	}

	// TODO: check after stage 0 of handshake if updated
	if initConn.RemotePeer() != respTransport.localID {
		t.Errorf("Initiator Remote Peer ID mismatch. expected %x got %x", respTransport.localID, initConn.RemotePeer())
	}
}

func TestKeys(t *testing.T) {
	initTransport := newTestTransport(t, crypto.Ed25519, 2048)
	respTransport := newTestTransport(t, crypto.Ed25519, 2048)

	initConn, respConn := connect(t, initTransport, respTransport)
	defer initConn.Close()
	defer respConn.Close()

	sk := respConn.LocalPrivateKey()
	pk := sk.GetPublic()

	if !sk.Equals(respTransport.privateKey) {
		t.Error("Private key Mismatch.")
	}

	if !pk.Equals(initConn.RemotePublicKey()) {
		t.Errorf("Public key mismatch. expected %x got %x", pk, initConn.RemotePublicKey())
	}
}

func TestPeerIDMismatchFailsHandshake(t *testing.T) {
	initTransport := newTestTransport(t, crypto.Ed25519, 2048)
	respTransport := newTestTransport(t, crypto.Ed25519, 2048)
	init, resp := newConnPair(t)

	var initErr error
	done := make(chan struct{})
	go func() {
		defer close(done)
		_, initErr = initTransport.SecureOutbound(context.TODO(), init, "a-random-peer-id")
	}()

	_, _ = respTransport.SecureInbound(context.TODO(), resp)
	<-done

	if initErr == nil {
		t.Fatal("expected initiator to fail with peer ID mismatch error")
	}
}

func makeLargePlaintext(size int) []byte {
	buf := make([]byte, size)
	rand.Read(buf)
	return buf
}

func TestLargePayloads(t *testing.T) {
	initTransport := newTestTransport(t, crypto.Ed25519, 2048)
	respTransport := newTestTransport(t, crypto.Ed25519, 2048)

	initConn, respConn := connect(t, initTransport, respTransport)
	defer initConn.Close()
	defer respConn.Close()

	// enough to require a couple Noise messages, with a size that
	// isn't a neat multiple of Noise message size, just in case
	size := 100000

	before := makeLargePlaintext(size)
	_, err := initConn.Write(before)
	if err != nil {
		t.Fatal(err)
	}

	after := make([]byte, len(before))
	afterLen, err := io.ReadFull(respConn, after)
	if err != nil {
		t.Fatal(err)
	}

	if len(before) != afterLen {
		t.Errorf("expected to read same amount of data as written. written=%d read=%d", len(before), afterLen)
	}
	if !bytes.Equal(before, after) {
		t.Error("Message mismatch.")
	}
}

// Tests XX handshake
func TestHandshakeXX(t *testing.T) {
	initTransport := newTestTransport(t, crypto.Ed25519, 2048)
	respTransport := newTestTransport(t, crypto.Ed25519, 2048)

	initConn, respConn := connect(t, initTransport, respTransport)
	defer initConn.Close()
	defer respConn.Close()

	before := []byte("hello world")
	_, err := initConn.Write(before)
	if err != nil {
		t.Fatal(err)
	}

	after := make([]byte, len(before))
	_, err = respConn.Read(after)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(before, after) {
		t.Errorf("Message mismatch. %v != %v", before, after)
	}
}

func TestBufferEqEncPayload(t *testing.T) {
	initTransport := newTestTransport(t, crypto.Ed25519, 2048)
	respTransport := newTestTransport(t, crypto.Ed25519, 2048)

	initConn, respConn := connect(t, initTransport, respTransport)
	defer initConn.Close()
	defer respConn.Close()

	before := []byte("hello world")
	_, err := initConn.Write(before)
	require.NoError(t, err)

	after := make([]byte, len(before)+poly1305.TagSize)
	afterLen, err := respConn.Read(after)
	require.NoError(t, err)

	require.Equal(t, len(before), afterLen)
	require.Equal(t, before, after[:len(before)])
}

func TestBufferEqDecryptedPayload(t *testing.T) {
	initTransport := newTestTransport(t, crypto.Ed25519, 2048)
	respTransport := newTestTransport(t, crypto.Ed25519, 2048)

	initConn, respConn := connect(t, initTransport, respTransport)
	defer initConn.Close()
	defer respConn.Close()

	before := []byte("hello world")
	_, err := initConn.Write(before)
	require.NoError(t, err)

	after := make([]byte, len(before)+1)
	afterLen, err := respConn.Read(after)
	require.NoError(t, err)

	require.Equal(t, len(before), afterLen)
	require.Equal(t, before, after[:len(before)])
}
<|endoftext|>


<commit_before>package os

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"syscall"

	. "github.com/candid82/joker/core"
)

func env() Object {
	res := EmptyArrayMap()
	for _, v := range os.Environ() {
		parts := strings.Split(v, "=")
		res.Add(String{S: parts[0]}, String{S: parts[1]})
	}
	return res
}

func setEnv(key string, value string) Object {
	err := os.Setenv(key, value)
	PanicOnErr(err)
	return NIL
}

func commandArgs() Object {
	res := EmptyVector()
	for _, arg := range os.Args {
		res = res.Conjoin(String{S: arg})
	}
	return res
}

const defaultFailedCode = 127 // seen from 'sh no-such-file' on OS X and Ubuntu

func execute(name string, opts Map) Object {
	var dir string
	var args []string
	var stdin io.Reader
	ok, dirObj := opts.Get(MakeKeyword("dir"))
	if ok {
		dir = AssertString(dirObj, "dir must be a string").S
	}
	ok, argsObj := opts.Get(MakeKeyword("args"))
	if ok {
		s := AssertSeqable(argsObj, "args must be Seqable").Seq()
		for !s.IsEmpty() {
			args = append(args, AssertString(s.First(), "args must be strings").S)
			s = s.Rest()
		}
	}
	ok, stdinObj := opts.Get(MakeKeyword("stdin"))
	if ok {
		if stdinObj.Equals(MakeKeyword("pipe")) {
			stdin = Stdin
		} else {
			stdin = strings.NewReader(AssertString(stdinObj, "stdin must be either :pipe keyword or a string").S)
		}
	}
	return sh(dir, stdin, name, args)
}

func sh(dir string, stdin io.Reader, name string, args []string) Object {
	cmd := exec.Command(name, args...)
	cmd.Dir = dir
	stdoutReader, err := cmd.StdoutPipe()
	PanicOnErr(err)
	stderrReader, err := cmd.StderrPipe()
	PanicOnErr(err)
	var stdinWriter io.WriteCloser
	if stdin == Stdin {
		cmd.Stdin = Stdin
	} else {
		writer, err := cmd.StdinPipe()
		PanicOnErr(err)
		stdinWriter = writer
	}

	bufOut := new(bytes.Buffer)
	bufErr := new(bytes.Buffer)

	go io.Copy(bufOut, stdoutReader)
	go io.Copy(bufErr, stderrReader)

	if err = cmd.Start(); err != nil {
		panic(RT.NewError(err.Error()))
	}

	if stdin != nil && stdinWriter != nil {
		go func() {
			io.Copy(stdinWriter, stdin)
			stdinWriter.Close()
		}()
	}

	err = cmd.Wait()
	stdoutString := bufOut.String()
	stderrString := bufErr.String()
	res := EmptyArrayMap()
	res.Add(MakeKeyword("success"), Boolean{B: err == nil})

	var exitCode int
	if err != nil {
		res.Add(MakeKeyword("err-msg"), String{S: err.Error()})
		if exiterr, ok := err.(*exec.ExitError); ok {
			ws := exiterr.Sys().(syscall.WaitStatus)
			exitCode = ws.ExitStatus()
		} else {
			exitCode = defaultFailedCode
			if stderrString == "" {
				stderrString = err.Error()
			}
		}
	} else {
		ws := cmd.ProcessState.Sys().(syscall.WaitStatus)
		exitCode = ws.ExitStatus()
	}
	res.Add(MakeKeyword("exit"), Int{I: exitCode})
	res.Add(MakeKeyword("out"), String{S: stdoutString})
	res.Add(MakeKeyword("err"), String{S: stderrString})
	return res
}

func mkdir(name string, perm int) Object {
	err := os.Mkdir(name, os.FileMode(perm))
	PanicOnErr(err)
	return NIL
}

func readDir(dirname string) Object {
	files, err := ioutil.ReadDir(dirname)
	PanicOnErr(err)
	res := EmptyVector()
	name := MakeKeyword("name")
	size := MakeKeyword("size")
	mode := MakeKeyword("mode")
	isDir := MakeKeyword("dir?")
	modTime := MakeKeyword("modtime")
	for _, f := range files {
		m := EmptyArrayMap()
		m.Add(name, MakeString(f.Name()))
		m.Add(size, MakeInt(int(f.Size())))
		m.Add(mode, MakeInt(int(f.Mode())))
		m.Add(isDir, MakeBoolean(f.IsDir()))
		m.Add(modTime, MakeInt(int(f.ModTime().Unix())))
		res = res.Conjoin(m)
	}
	return res
}

func getwd() string {
	res, err := os.Getwd()
	PanicOnErr(err)
	return res
}

func chdir(dirname string) Object {
	err := os.Chdir(dirname)
	PanicOnErr(err)
	return NIL
}

func stat(filename string) Object {
	info, err := os.Stat(filename)
	PanicOnErr(err)
	return FileInfoMap(info.Name(), info)
}
<commit_msg>Simplify capturing of output so cmd.Wait() waits for it<commit_after>package os

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"syscall"

	. "github.com/candid82/joker/core"
)

func env() Object {
	res := EmptyArrayMap()
	for _, v := range os.Environ() {
		parts := strings.Split(v, "=")
		res.Add(String{S: parts[0]}, String{S: parts[1]})
	}
	return res
}

func setEnv(key string, value string) Object {
	err := os.Setenv(key, value)
	PanicOnErr(err)
	return NIL
}

func commandArgs() Object {
	res := EmptyVector()
	for _, arg := range os.Args {
		res = res.Conjoin(String{S: arg})
	}
	return res
}

const defaultFailedCode = 127 // seen from 'sh no-such-file' on OS X and Ubuntu

func execute(name string, opts Map) Object {
	var dir string
	var args []string
	var stdin io.Reader
	ok, dirObj := opts.Get(MakeKeyword("dir"))
	if ok {
		dir = AssertString(dirObj, "dir must be a string").S
	}
	ok, argsObj := opts.Get(MakeKeyword("args"))
	if ok {
		s := AssertSeqable(argsObj, "args must be Seqable").Seq()
		for !s.IsEmpty() {
			args = append(args, AssertString(s.First(), "args must be strings").S)
			s = s.Rest()
		}
	}
	ok, stdinObj := opts.Get(MakeKeyword("stdin"))
	if ok {
		if stdinObj.Equals(MakeKeyword("pipe")) {
			stdin = Stdin
		} else {
			stdin = strings.NewReader(AssertString(stdinObj, "stdin must be either :pipe keyword or a string").S)
		}
	}
	return sh(dir, stdin, name, args)
}

func sh(dir string, stdin io.Reader, name string, args []string) Object {
	cmd := exec.Command(name, args...)
	cmd.Dir = dir

	var stdout, stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	var stdinWriter io.WriteCloser
	if stdin == Stdin {
		cmd.Stdin = Stdin
	} else {
		writer, err := cmd.StdinPipe()
		PanicOnErr(err)
		stdinWriter = writer
	}

	err := cmd.Start()
	PanicOnErr(err)

	if stdin != nil && stdinWriter != nil {
		go func() {
			io.Copy(stdinWriter, stdin)
			stdinWriter.Close()
		}()
	}

	err = cmd.Wait()

	stdoutString := string(stdout.Bytes())
	stderrString := string(stderr.Bytes())

	res := EmptyArrayMap()
	res.Add(MakeKeyword("success"), Boolean{B: err == nil})

	var exitCode int
	if err != nil {
		res.Add(MakeKeyword("err-msg"), String{S: err.Error()})
		if exiterr, ok := err.(*exec.ExitError); ok {
			ws := exiterr.Sys().(syscall.WaitStatus)
			exitCode = ws.ExitStatus()
		} else {
			exitCode = defaultFailedCode
			if stderrString == "" {
				stderrString = err.Error()
			}
		}
	} else {
		ws := cmd.ProcessState.Sys().(syscall.WaitStatus)
		exitCode = ws.ExitStatus()
	}
	res.Add(MakeKeyword("exit"), Int{I: exitCode})
	res.Add(MakeKeyword("out"), String{S: stdoutString})
	res.Add(MakeKeyword("err"), String{S: stderrString})
	return res
}

func mkdir(name string, perm int) Object {
	err := os.Mkdir(name, os.FileMode(perm))
	PanicOnErr(err)
	return NIL
}

func readDir(dirname string) Object {
	files, err := ioutil.ReadDir(dirname)
	PanicOnErr(err)
	res := EmptyVector()
	name := MakeKeyword("name")
	size := MakeKeyword("size")
	mode := MakeKeyword("mode")
	isDir := MakeKeyword("dir?")
	modTime := MakeKeyword("modtime")
	for _, f := range files {
		m := EmptyArrayMap()
		m.Add(name, MakeString(f.Name()))
		m.Add(size, MakeInt(int(f.Size())))
		m.Add(mode, MakeInt(int(f.Mode())))
		m.Add(isDir, MakeBoolean(f.IsDir()))
		m.Add(modTime, MakeInt(int(f.ModTime().Unix())))
		res = res.Conjoin(m)
	}
	return res
}

func getwd() string {
	res, err := os.Getwd()
	PanicOnErr(err)
	return res
}

func chdir(dirname string) Object {
	err := os.Chdir(dirname)
	PanicOnErr(err)
	return NIL
}

func stat(filename string) Object {
	info, err := os.Stat(filename)
	PanicOnErr(err)
	return FileInfoMap(info.Name(), info)
}
<|endoftext|>


<commit_before>package ethchain

import (
	"fmt"
	"github.com/ethereum/eth-go/ethutil"
	"math/big"
)

/*
 * The State transitioning model
 *
 * A state transition is a change made when a transaction is applied to the current world state
 * The state transitioning model does all all the necessary work to work out a valid new state root.
 * 1) Nonce handling
 * 2) Pre pay / buy gas of the coinbase (miner)
 * 3) Create a new state object if the recipient is \0*32
 * 4) Value transfer
 * == If contract creation ==
 * 4a) Attempt to run transaction data
 * 4b) If valid, use result as code for the new state object
 * == end ==
 * 5) Run Script section
 * 6) Derive new state root
 */
type StateTransition struct {
	coinbase []byte
	tx       *Transaction
	gas      *big.Int
	state    *State
	block    *Block

	cb, rec, sen *StateObject
}

func NewStateTransition(coinbase []byte, tx *Transaction, state *State, block *Block) *StateTransition {
	return &StateTransition{coinbase, tx, new(big.Int), state, block, nil, nil, nil}
}

func (self *StateTransition) Coinbase() *StateObject {
	if self.cb != nil {
		return self.cb
	}

	self.cb = self.state.GetAccount(self.coinbase)
	return self.cb
}
func (self *StateTransition) Sender() *StateObject {
	if self.sen != nil {
		return self.sen
	}

	self.sen = self.state.GetAccount(self.tx.Sender())
	return self.sen
}
func (self *StateTransition) Receiver() *StateObject {
	if self.tx.CreatesContract() {
		return nil
	}

	if self.rec != nil {
		return self.rec
	}

	self.rec = self.state.GetAccount(self.tx.Recipient)
	return self.rec
}

func (self *StateTransition) MakeStateObject(state *State, tx *Transaction) *StateObject {
	contract := MakeContract(tx, state)
	if contract != nil {
		state.states[string(tx.CreationAddress())] = contract.state

		return contract
	}

	return nil
}

func (self *StateTransition) UseGas(amount *big.Int) error {
	if self.gas.Cmp(amount) < 0 {
		return OutOfGasError()
	}
	self.gas.Sub(self.gas, amount)

	return nil
}

func (self *StateTransition) AddGas(amount *big.Int) {
	self.gas.Add(self.gas, amount)
}

func (self *StateTransition) BuyGas() error {
	var err error

	sender := self.Sender()
	if sender.Amount.Cmp(self.tx.GasValue()) < 0 {
		return fmt.Errorf("Insufficient funds to pre-pay gas. Req %v, has %v", self.tx.GasValue(), self.tx.Value)
	}

	coinbase := self.Coinbase()
	err = coinbase.BuyGas(self.tx.Gas, self.tx.GasPrice)
	if err != nil {
		return err
	}
	self.state.UpdateStateObject(coinbase)

	self.AddGas(self.tx.Gas)
	sender.SubAmount(self.tx.GasValue())

	return nil
}

func (self *StateTransition) TransitionState() (err error) {
	//snapshot := st.state.Snapshot()

	defer func() {
		if r := recover(); r != nil {
			ethutil.Config.Log.Infoln(r)
			err = fmt.Errorf("state transition err %v", r)
		}
	}()

	var (
		tx       = self.tx
		sender   = self.Sender()
		receiver *StateObject
	)

	// Make sure this transaction's nonce is correct
	if sender.Nonce != tx.Nonce {
		return NonceError(tx.Nonce, sender.Nonce)
	}

	// Pre-pay gas / Buy gas of the coinbase account
	if err = self.BuyGas(); err != nil {
		return err
	}

	// XXX Transactions after this point are considered valid.

	defer func() {
		self.state.UpdateStateObject(sender)
		self.state.UpdateStateObject(receiver)
	}()

	// Increment the nonce for the next transaction
	sender.Nonce += 1

	// Get the receiver (TODO fix this, if coinbase is the receiver we need to save/retrieve)
	receiver = self.Receiver()

	// Transaction gas
	if err = self.UseGas(GasTx); err != nil {
		return err
	}

	// Pay data gas
	dataPrice := big.NewInt(int64(len(tx.Data)))
	dataPrice.Mul(dataPrice, GasData)
	if err = self.UseGas(dataPrice); err != nil {
		return err
	}

	// If the receiver is nil it's a contract (\0*32).
	if receiver == nil {
		// Create a new state object for the contract
		receiver = self.MakeStateObject(self.state, tx)
		if receiver == nil {
			return fmt.Errorf("ERR. Unable to create contract with transaction %v", tx)
		}
	}

	// Transfer value from sender to receiver
	if err = self.transferValue(sender, receiver); err != nil {
		return err
	}

	// Process the init code and create 'valid' contract
	if tx.CreatesContract() {
		//fmt.Println(Disassemble(receiver.Init()))
		// Evaluate the initialization script
		// and use the return value as the
		// script section for the state object.
		//script, gas, err = sm.Eval(state, contract.Init(), contract, tx, block)
		code, err := self.Eval(receiver.Init(), receiver)
		if err != nil {
			return fmt.Errorf("Error during init script run %v", err)
		}

		receiver.script = code
	} else {
		if len(receiver.Script()) > 0 {
			_, err := self.Eval(receiver.Script(), receiver)
			if err != nil {
				return fmt.Errorf("Error during code execution %v", err)
			}
		}
	}

	// Return remaining gas
	remaining := new(big.Int).Mul(self.gas, tx.GasPrice)
	sender.AddAmount(remaining)

	return nil
}

func (self *StateTransition) transferValue(sender, receiver *StateObject) error {
	if sender.Amount.Cmp(self.tx.Value) < 0 {
		return fmt.Errorf("Insufficient funds to transfer value. Req %v, has %v", self.tx.Value, sender.Amount)
	}

	if self.tx.Value.Cmp(ethutil.Big0) > 0 {
		// Subtract the amount from the senders account
		sender.SubAmount(self.tx.Value)
		// Add the amount to receivers account which should conclude this transaction
		receiver.AddAmount(self.tx.Value)

		ethutil.Config.Log.Debugf("%x => %x (%v) %x\n", sender.Address()[:4], receiver.Address()[:4], self.tx.Value, self.tx.Hash())
	}

	return nil
}

func (self *StateTransition) Eval(script []byte, context *StateObject) (ret []byte, err error) {
	var (
		tx        = self.tx
		block     = self.block
		initiator = self.Sender()
		state     = self.state
	)

	closure := NewClosure(initiator, context, script, state, self.gas, tx.GasPrice)
	vm := NewVm(state, nil, RuntimeVars{
		Origin:      initiator.Address(),
		BlockNumber: block.BlockInfo().Number,
		PrevHash:    block.PrevHash,
		Coinbase:    block.Coinbase,
		Time:        block.Time,
		Diff:        block.Difficulty,
		Value:       tx.Value,
	})
	ret, _, err = closure.Call(vm, tx.Data, nil)

	return
}
<commit_msg>Removed some comments<commit_after>package ethchain

import (
	"fmt"
	"github.com/ethereum/eth-go/ethutil"
	"math/big"
)

/*
 * The State transitioning model
 *
 * A state transition is a change made when a transaction is applied to the current world state
 * The state transitioning model does all all the necessary work to work out a valid new state root.
 * 1) Nonce handling
 * 2) Pre pay / buy gas of the coinbase (miner)
 * 3) Create a new state object if the recipient is \0*32
 * 4) Value transfer
 * == If contract creation ==
 * 4a) Attempt to run transaction data
 * 4b) If valid, use result as code for the new state object
 * == end ==
 * 5) Run Script section
 * 6) Derive new state root
 */
type StateTransition struct {
	coinbase []byte
	tx       *Transaction
	gas      *big.Int
	state    *State
	block    *Block

	cb, rec, sen *StateObject
}

func NewStateTransition(coinbase []byte, tx *Transaction, state *State, block *Block) *StateTransition {
	return &StateTransition{coinbase, tx, new(big.Int), state, block, nil, nil, nil}
}

func (self *StateTransition) Coinbase() *StateObject {
	if self.cb != nil {
		return self.cb
	}

	self.cb = self.state.GetAccount(self.coinbase)
	return self.cb
}
func (self *StateTransition) Sender() *StateObject {
	if self.sen != nil {
		return self.sen
	}

	self.sen = self.state.GetAccount(self.tx.Sender())
	return self.sen
}
func (self *StateTransition) Receiver() *StateObject {
	if self.tx.CreatesContract() {
		return nil
	}

	if self.rec != nil {
		return self.rec
	}

	self.rec = self.state.GetAccount(self.tx.Recipient)
	return self.rec
}

func (self *StateTransition) MakeStateObject(state *State, tx *Transaction) *StateObject {
	contract := MakeContract(tx, state)
	if contract != nil {
		state.states[string(tx.CreationAddress())] = contract.state

		return contract
	}

	return nil
}

func (self *StateTransition) UseGas(amount *big.Int) error {
	if self.gas.Cmp(amount) < 0 {
		return OutOfGasError()
	}
	self.gas.Sub(self.gas, amount)

	return nil
}

func (self *StateTransition) AddGas(amount *big.Int) {
	self.gas.Add(self.gas, amount)
}

func (self *StateTransition) BuyGas() error {
	var err error

	sender := self.Sender()
	if sender.Amount.Cmp(self.tx.GasValue()) < 0 {
		return fmt.Errorf("Insufficient funds to pre-pay gas. Req %v, has %v", self.tx.GasValue(), self.tx.Value)
	}

	coinbase := self.Coinbase()
	err = coinbase.BuyGas(self.tx.Gas, self.tx.GasPrice)
	if err != nil {
		return err
	}
	//self.state.UpdateStateObject(coinbase)

	self.AddGas(self.tx.Gas)
	sender.SubAmount(self.tx.GasValue())

	return nil
}

func (self *StateTransition) TransitionState() (err error) {
	//snapshot := st.state.Snapshot()

	defer func() {
		if r := recover(); r != nil {
			ethutil.Config.Log.Infoln(r)
			err = fmt.Errorf("state transition err %v", r)
		}
	}()

	var (
		tx       = self.tx
		sender   = self.Sender()
		receiver *StateObject
	)

	// Make sure this transaction's nonce is correct
	if sender.Nonce != tx.Nonce {
		return NonceError(tx.Nonce, sender.Nonce)
	}

	// Pre-pay gas / Buy gas of the coinbase account
	if err = self.BuyGas(); err != nil {
		return err
	}

	// XXX Transactions after this point are considered valid.

	defer func() {
		self.state.UpdateStateObject(sender)
		self.state.UpdateStateObject(receiver)
	}()

	// Increment the nonce for the next transaction
	sender.Nonce += 1

	// Get the receiver (TODO fix this, if coinbase is the receiver we need to save/retrieve)
	receiver = self.Receiver()

	// Transaction gas
	if err = self.UseGas(GasTx); err != nil {
		return err
	}

	// Pay data gas
	dataPrice := big.NewInt(int64(len(tx.Data)))
	dataPrice.Mul(dataPrice, GasData)
	if err = self.UseGas(dataPrice); err != nil {
		return err
	}

	// If the receiver is nil it's a contract (\0*32).
	if receiver == nil {
		// Create a new state object for the contract
		receiver = self.MakeStateObject(self.state, tx)
		if receiver == nil {
			return fmt.Errorf("ERR. Unable to create contract with transaction %v", tx)
		}
	}

	// Transfer value from sender to receiver
	if err = self.transferValue(sender, receiver); err != nil {
		return err
	}

	// Process the init code and create 'valid' contract
	if tx.CreatesContract() {
		// Evaluate the initialization script
		// and use the return value as the
		// script section for the state object.
		//script, gas, err = sm.Eval(state, contract.Init(), contract, tx, block)
		code, err := self.Eval(receiver.Init(), receiver)
		if err != nil {
			return fmt.Errorf("Error during init script run %v", err)
		}

		receiver.script = code
	} else {
		if len(receiver.Script()) > 0 {
			_, err := self.Eval(receiver.Script(), receiver)
			if err != nil {
				return fmt.Errorf("Error during code execution %v", err)
			}
		}
	}

	// Return remaining gas
	remaining := new(big.Int).Mul(self.gas, tx.GasPrice)
	sender.AddAmount(remaining)

	return nil
}

func (self *StateTransition) transferValue(sender, receiver *StateObject) error {
	if sender.Amount.Cmp(self.tx.Value) < 0 {
		return fmt.Errorf("Insufficient funds to transfer value. Req %v, has %v", self.tx.Value, sender.Amount)
	}

	if self.tx.Value.Cmp(ethutil.Big0) > 0 {
		// Subtract the amount from the senders account
		sender.SubAmount(self.tx.Value)
		// Add the amount to receivers account which should conclude this transaction
		receiver.AddAmount(self.tx.Value)

		ethutil.Config.Log.Debugf("%x => %x (%v) %x\n", sender.Address()[:4], receiver.Address()[:4], self.tx.Value, self.tx.Hash())
	}

	return nil
}

func (self *StateTransition) Eval(script []byte, context *StateObject) (ret []byte, err error) {
	var (
		tx        = self.tx
		block     = self.block
		initiator = self.Sender()
		state     = self.state
	)

	closure := NewClosure(initiator, context, script, state, self.gas, tx.GasPrice)
	vm := NewVm(state, nil, RuntimeVars{
		Origin:      initiator.Address(),
		BlockNumber: block.BlockInfo().Number,
		PrevHash:    block.PrevHash,
		Coinbase:    block.Coinbase,
		Time:        block.Time,
		Diff:        block.Difficulty,
		Value:       tx.Value,
	})
	ret, _, err = closure.Call(vm, tx.Data, nil)

	return
}
<|endoftext|>


<commit_before>// Copyright 2018 the Service Broker Project Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"math/rand"
	"sort"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/gcp-service-broker/pkg/broker"
	"github.com/pivotal-cf/brokerapi"
)

// RunExamplesForService runs all the examples for a given service name against
// the service broker pointed to by client. All examples in the registry get run
// if serviceName is blank. If exampleName is non-blank then only the example
// with the given name is run.
func RunExamplesForService(registry broker.BrokerRegistry, client *Client, serviceName, exampleName string) error {
	rand.Seed(time.Now().UTC().UnixNano())

	allExamples, err := getAllCompleteServiceExamples(registry)
	if err != nil {
		return err
	}

	for _, completeServiceExample := range filterMatchingServiceExamples(allExamples, serviceName, exampleName) {
		if err := RunExample(client, completeServiceExample); err != nil {
			return err
		}
	}

	return nil

}

type CompleteServiceExample struct {
	Example        broker.ServiceExample   `json: "service_example, inline"`
	ServiceName    string                  `json: "service_name, inline"`
	ServiceId      string                  `json: "service_id, inline"`
	ExpectedOutput []broker.BrokerVariable `json: "broker_variables, inline"`
}

func getAllCompleteServiceExamples(registry broker.BrokerRegistry) ([]CompleteServiceExample, error) {
	var allExamples []CompleteServiceExample

	services := registry.GetAllServices()

	for _, service := range services {
		for _, example := range service.Examples {
			serviceCatalogEntry, err := service.CatalogEntry()
			if err != nil {
				return nil, err
			}

			var completeServiceExample = CompleteServiceExample{
				Example:        example,
				ServiceId:      serviceCatalogEntry.ID,
				ServiceName:    service.Name,
				ExpectedOutput: service.BindOutputVariables,
			}

			allExamples = append(allExamples, completeServiceExample)
		}
	}

	// Sort by ServiceName and ExampleName so there's a consistent order in the UI and tests.
	sort.Slice(allExamples, func(i int, j int) bool {
		if strings.Compare(allExamples[i].ServiceName, allExamples[j].ServiceName) != 0 {
			return allExamples[i].ServiceName < allExamples[j].ServiceName
		} else {
			return allExamples[i].Example.Name < allExamples[j].Example.Name
		}
	})

	return allExamples, nil
}

// Do not run example if:
// 1. The service name is specified and does not match the current example's ServiceName
// 2. The service name is specified and matches the current example's ServiceName, and the example name is specified and does not match the current example's ExampleName
func filterMatchingServiceExamples(allExamples []CompleteServiceExample, serviceName, exampleName string) []CompleteServiceExample {
	var matchingExamples []CompleteServiceExample

	for _, completeServiceExample := range allExamples {

		if (serviceName != "" && serviceName != completeServiceExample.ServiceName) || (serviceName != "" && exampleName != "" && exampleName != completeServiceExample.Example.Name) {
			continue
		}

		matchingExamples = append(matchingExamples, completeServiceExample)
	}

	return matchingExamples
}

// RunExample runs a single example against the given service on the broker
// pointed to by client.
func RunExample(client *Client, serviceExample CompleteServiceExample) error {
	executor, err := newExampleExecutor(client, serviceExample)
	if err != nil {
		return err
	}

	executor.LogTestInfo()

	// Cleanup the test if it fails partway through
	defer func() {
		log.Println("Cleaning up the environment")
		executor.Unbind()
		executor.Deprovision()
	}()

	if err := executor.Provision(); err != nil {
		return err
	}

	bindResponse, err := executor.Bind()
	if err != nil {
		return err
	}

	if err := executor.Unbind(); err != nil {
		return err
	}

	if err := executor.Deprovision(); err != nil {
		return err
	}

	// Check that the binding response has the same fields as expected
	var binding brokerapi.Binding
	err = json.Unmarshal(bindResponse, &binding)
	if err != nil {
		return err
	}

	credentialsEntry := binding.Credentials.(map[string]interface{})

	if err := broker.ValidateVariables(credentialsEntry, serviceExample.ExpectedOutput); err != nil {

		log.Printf("Error: results don't match JSON Schema: %v", err)
		return err
	}

	return nil
}

func retry(timeout, period time.Duration, function func() (tryAgain bool, err error)) error {
	to := time.After(timeout)
	tick := time.Tick(period)

	if tryAgain, err := function(); !tryAgain {
		return err
	}

	// Keep trying until we're timed out or got a result or got an error
	for {
		select {
		case <-to:
			return errors.New("Timeout while waiting for result")
		case <-tick:
			tryAgain, err := function()

			if !tryAgain {
				return err
			}
		}
	}
}

func pollUntilFinished(client *Client, instanceId string) error {
	return retry(15*time.Minute, 15*time.Second, func() (bool, error) {
		log.Println("Polling for async job")

		resp := client.LastOperation(instanceId)
		if resp.InError() {
			return false, resp.Error
		}

		if resp.StatusCode != 200 {
			log.Printf("Bad status code %d, needed 200", resp.StatusCode)
			return true, nil
		}

		var responseBody map[string]string
		err := json.Unmarshal(resp.ResponseBody, &responseBody)
		if err != nil {
			return false, err
		}

		state := responseBody["state"]
		eq := state == string(brokerapi.Succeeded)
		log.Printf("Last operation for %q was %q\n", instanceId, state)

		return !eq, nil

	})
}

func newExampleExecutor(client *Client, serviceExample CompleteServiceExample) (*exampleExecutor, error) {
	provisionParams, err := json.Marshal(serviceExample.Example.ProvisionParams)
	if err != nil {
		return nil, err
	}

	bindParams, err := json.Marshal(serviceExample.Example.BindParams)
	if err != nil {
		return nil, err
	}

	testid := rand.Uint32()

	return &exampleExecutor{
		Name:       fmt.Sprintf("%s/%s", serviceExample.ServiceName, serviceExample.Example.Name),
		ServiceId:  serviceExample.ServiceId,
		PlanId:     serviceExample.Example.PlanId,
		InstanceId: fmt.Sprintf("ex%d", testid),
		BindingId:  fmt.Sprintf("ex%d", testid),

		ProvisionParams: provisionParams,
		BindParams:      bindParams,

		client: client,
	}, nil
}

type exampleExecutor struct {
	Name string

	ServiceId  string
	PlanId     string
	InstanceId string
	BindingId  string

	ProvisionParams json.RawMessage
	BindParams      json.RawMessage

	client *Client
}

// Provision attempts to create a service instance from the example.
// Multiple calls to provision will attempt to create a resource with the same
// ServiceId and details.
// If the response is an async result, Provision will attempt to wait until
// the Provision is complete.
func (ee *exampleExecutor) Provision() error {
	log.Printf("Provisioning %s\n", ee.Name)

	resp := ee.client.Provision(ee.InstanceId, ee.ServiceId, ee.PlanId, ee.ProvisionParams)

	log.Println(resp.String())
	if resp.InError() {
		return resp.Error
	}

	switch resp.StatusCode {
	case 201:
		return nil
	case 202:
		return ee.pollUntilFinished()
	default:
		return fmt.Errorf("Unexpected response code %d", resp.StatusCode)
	}
}

func (ee *exampleExecutor) pollUntilFinished() error {
	return pollUntilFinished(ee.client, ee.InstanceId)
}

// Deprovision destroys the instance created by a call to Provision.
func (ee *exampleExecutor) Deprovision() error {
	log.Printf("Deprovisioning %s\n", ee.Name)
	resp := ee.client.Deprovision(ee.InstanceId, ee.ServiceId, ee.PlanId)

	log.Println(resp.String())
	if resp.InError() {
		return resp.Error
	}

	switch resp.StatusCode {
	case 200:
		return nil
	case 202:
		return ee.pollUntilFinished()
	default:
		return fmt.Errorf("Unexpected response code %d", resp.StatusCode)
	}
}

// Unbind unbinds the exact binding created by a call to Bind.
func (ee *exampleExecutor) Unbind() error {
	// XXX(josephlewis42) Due to some unknown reason, binding Postgres and MySQL
	// don't wait for all operations to finish before returning even though it
	// looks like they do so we can get 500 errors back the first few times we try
	// to unbind. Issue #222 was opened to address this. In the meantime this
	// is a hack to get around it that will still fail if the 500 errors truly
	// occur because of a real, unrecoverable, server error.
	return retry(15*time.Minute, 15*time.Second, func() (bool, error) {
		log.Printf("Unbinding %s\n", ee.Name)
		resp := ee.client.Unbind(ee.InstanceId, ee.BindingId, ee.ServiceId, ee.PlanId)

		log.Println(resp.String())
		if resp.InError() {
			return false, resp.Error
		}

		if resp.StatusCode == 200 {
			return false, nil
		}

		if resp.StatusCode == 500 {
			return true, nil
		}

		return false, fmt.Errorf("Unexpected response code %d", resp.StatusCode)
	})
}

// Bind executes the bind portion of the create, this can only be called
// once successfully as subsequent binds will attempt to create bindings with
// the same ID.
func (ee *exampleExecutor) Bind() (json.RawMessage, error) {
	log.Printf("Binding %s\n", ee.Name)
	resp := ee.client.Bind(ee.InstanceId, ee.BindingId, ee.ServiceId, ee.PlanId, ee.BindParams)

	log.Println(resp.String())
	if resp.InError() {
		return nil, resp.Error
	}

	if resp.StatusCode == 201 {
		return resp.ResponseBody, nil
	}

	return nil, fmt.Errorf("Unexpected response code %d", resp.StatusCode)
}

// LogTestInfo writes information about the running example and a manual backout
// strategy if the test dies part of the way through.
func (ee *exampleExecutor) LogTestInfo() {
	log.Printf("Running Example: %s\n", ee.Name)

	ips := fmt.Sprintf("--instanceid %q --planid %q --serviceid %q", ee.InstanceId, ee.PlanId, ee.ServiceId)
	log.Printf("gcp-service-broker client provision %s --params %q\n", ips, ee.ProvisionParams)
	log.Printf("gcp-service-broker client bind %s --bindingid %q --params %q\n", ips, ee.BindingId, ee.BindParams)
	log.Printf("gcp-service-broker client unbind %s --bindingid %q\n", ips, ee.BindingId)
	log.Printf("gcp-service-broker client deprovision %s\n", ips)
}
<commit_msg>Separate functions for isolated unit testing<commit_after>// Copyright 2018 the Service Broker Project Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"math/rand"
	"sort"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/gcp-service-broker/pkg/broker"
	"github.com/pivotal-cf/brokerapi"
)

// RunExamplesForService runs all the examples for a given service name against
// the service broker pointed to by client. All examples in the registry get run
// if serviceName is blank. If exampleName is non-blank then only the example
// with the given name is run.
func RunExamplesForService(registry broker.BrokerRegistry, client *Client, serviceName, exampleName string) error {

	rand.Seed(time.Now().UTC().UnixNano())

	if exampleName != "" && serviceName == "" {
		fmt.Errorf("If an example name is specified, you must provide an accompanying service name.")
	}

	allExamples, err := GetAllCompleteServiceExamples(registry)
	if err != nil {
		return err
	}

	for _, completeServiceExample := range FilterMatchingServiceExamples(allExamples, serviceName, exampleName) {
		if err := RunExample(client, completeServiceExample); err != nil {
			return err
		}
	}

	return nil

}

type CompleteServiceExample struct {
	broker.ServiceExample `json: ",inline"`
	ServiceName           string                  `json: "service_name"`
	ServiceId             string                  `json: "service_id"`
	ExpectedOutput        []broker.BrokerVariable `json: "expected_output"`
}

func GetAllCompleteServiceExamples(registry broker.BrokerRegistry) ([]CompleteServiceExample, error) {
	var allExamples []CompleteServiceExample

	services := registry.GetAllServices()

	for _, service := range services {

		serviceExamples, err := GetExamplesForAService(service)

		if err != nil {
			return nil, err
		}

		allExamples = append(allExamples, serviceExamples...)
	}

	// Sort by ServiceName and ExampleName so there's a consistent order in the UI and tests.
	sort.Slice(allExamples, func(i int, j int) bool {
		if strings.Compare(allExamples[i].ServiceName, allExamples[j].ServiceName) != 0 {
			return allExamples[i].ServiceName < allExamples[j].ServiceName
		} else {
			return allExamples[i].ServiceExample.Name < allExamples[j].ServiceExample.Name
		}
	})

	return allExamples, nil
}

func GetExamplesForAService(service *broker.ServiceDefinition) ([]CompleteServiceExample, error) {

	var examples []CompleteServiceExample

	for _, example := range service.Examples {
		serviceCatalogEntry, err := service.CatalogEntry()

		if err != nil {
			return nil, err
		}

		var completeServiceExample = CompleteServiceExample{
			ServiceExample: example,
			ServiceId:      serviceCatalogEntry.ID,
			ServiceName:    service.Name,
			ExpectedOutput: service.BindOutputVariables,
		}

		examples = append(examples, completeServiceExample)
	}

	return examples, nil
}

// Do not run example if:
// 1. The service name is specified and does not match the current example's ServiceName
// 2. The service name is specified and matches the current example's ServiceName, and the example name is specified and does not match the current example's ExampleName
func FilterMatchingServiceExamples(allExamples []CompleteServiceExample, serviceName, exampleName string) []CompleteServiceExample {
	var matchingExamples []CompleteServiceExample

	for _, completeServiceExample := range allExamples {

		if (serviceName != "" && serviceName != completeServiceExample.ServiceName) || (exampleName != "" && exampleName != completeServiceExample.ServiceExample.Name) {
			continue
		}

		matchingExamples = append(matchingExamples, completeServiceExample)
	}

	return matchingExamples
}

// RunExample runs a single example against the given service on the broker
// pointed to by client.
func RunExample(client *Client, serviceExample CompleteServiceExample) error {
	executor, err := newExampleExecutor(client, serviceExample)
	if err != nil {
		return err
	}

	executor.LogTestInfo()

	// Cleanup the test if it fails partway through
	defer func() {
		log.Println("Cleaning up the environment")
		executor.Unbind()
		executor.Deprovision()
	}()

	if err := executor.Provision(); err != nil {
		return err
	}

	bindResponse, err := executor.Bind()
	if err != nil {
		return err
	}

	if err := executor.Unbind(); err != nil {
		return err
	}

	if err := executor.Deprovision(); err != nil {
		return err
	}

	// Check that the binding response has the same fields as expected
	var binding brokerapi.Binding
	err = json.Unmarshal(bindResponse, &binding)
	if err != nil {
		return err
	}

	credentialsEntry := binding.Credentials.(map[string]interface{})

	if err := broker.ValidateVariables(credentialsEntry, serviceExample.ExpectedOutput); err != nil {

		log.Printf("Error: results don't match JSON Schema: %v", err)
		return err
	}

	return nil
}

func retry(timeout, period time.Duration, function func() (tryAgain bool, err error)) error {
	to := time.After(timeout)
	tick := time.Tick(period)

	if tryAgain, err := function(); !tryAgain {
		return err
	}

	// Keep trying until we're timed out or got a result or got an error
	for {
		select {
		case <-to:
			return errors.New("Timeout while waiting for result")
		case <-tick:
			tryAgain, err := function()

			if !tryAgain {
				return err
			}
		}
	}
}

func pollUntilFinished(client *Client, instanceId string) error {
	return retry(15*time.Minute, 15*time.Second, func() (bool, error) {
		log.Println("Polling for async job")

		resp := client.LastOperation(instanceId)
		if resp.InError() {
			return false, resp.Error
		}

		if resp.StatusCode != 200 {
			log.Printf("Bad status code %d, needed 200", resp.StatusCode)
			return true, nil
		}

		var responseBody map[string]string
		err := json.Unmarshal(resp.ResponseBody, &responseBody)
		if err != nil {
			return false, err
		}

		state := responseBody["state"]
		eq := state == string(brokerapi.Succeeded)
		log.Printf("Last operation for %q was %q\n", instanceId, state)

		return !eq, nil

	})
}

func newExampleExecutor(client *Client, serviceExample CompleteServiceExample) (*exampleExecutor, error) {
	provisionParams, err := json.Marshal(serviceExample.ServiceExample.ProvisionParams)
	if err != nil {
		return nil, err
	}

	bindParams, err := json.Marshal(serviceExample.ServiceExample.BindParams)
	if err != nil {
		return nil, err
	}

	testid := rand.Uint32()

	return &exampleExecutor{
		Name:       fmt.Sprintf("%s/%s", serviceExample.ServiceName, serviceExample.ServiceExample.Name),
		ServiceId:  serviceExample.ServiceId,
		PlanId:     serviceExample.ServiceExample.PlanId,
		InstanceId: fmt.Sprintf("ex%d", testid),
		BindingId:  fmt.Sprintf("ex%d", testid),

		ProvisionParams: provisionParams,
		BindParams:      bindParams,

		client: client,
	}, nil
}

type exampleExecutor struct {
	Name string

	ServiceId  string
	PlanId     string
	InstanceId string
	BindingId  string

	ProvisionParams json.RawMessage
	BindParams      json.RawMessage

	client *Client
}

// Provision attempts to create a service instance from the example.
// Multiple calls to provision will attempt to create a resource with the same
// ServiceId and details.
// If the response is an async result, Provision will attempt to wait until
// the Provision is complete.
func (ee *exampleExecutor) Provision() error {
	log.Printf("Provisioning %s\n", ee.Name)

	resp := ee.client.Provision(ee.InstanceId, ee.ServiceId, ee.PlanId, ee.ProvisionParams)

	log.Println(resp.String())
	if resp.InError() {
		return resp.Error
	}

	switch resp.StatusCode {
	case 201:
		return nil
	case 202:
		return ee.pollUntilFinished()
	default:
		return fmt.Errorf("Unexpected response code %d", resp.StatusCode)
	}
}

func (ee *exampleExecutor) pollUntilFinished() error {
	return pollUntilFinished(ee.client, ee.InstanceId)
}

// Deprovision destroys the instance created by a call to Provision.
func (ee *exampleExecutor) Deprovision() error {
	log.Printf("Deprovisioning %s\n", ee.Name)
	resp := ee.client.Deprovision(ee.InstanceId, ee.ServiceId, ee.PlanId)

	log.Println(resp.String())
	if resp.InError() {
		return resp.Error
	}

	switch resp.StatusCode {
	case 200:
		return nil
	case 202:
		return ee.pollUntilFinished()
	default:
		return fmt.Errorf("Unexpected response code %d", resp.StatusCode)
	}
}

// Unbind unbinds the exact binding created by a call to Bind.
func (ee *exampleExecutor) Unbind() error {
	// XXX(josephlewis42) Due to some unknown reason, binding Postgres and MySQL
	// don't wait for all operations to finish before returning even though it
	// looks like they do so we can get 500 errors back the first few times we try
	// to unbind. Issue #222 was opened to address this. In the meantime this
	// is a hack to get around it that will still fail if the 500 errors truly
	// occur because of a real, unrecoverable, server error.
	return retry(15*time.Minute, 15*time.Second, func() (bool, error) {
		log.Printf("Unbinding %s\n", ee.Name)
		resp := ee.client.Unbind(ee.InstanceId, ee.BindingId, ee.ServiceId, ee.PlanId)

		log.Println(resp.String())
		if resp.InError() {
			return false, resp.Error
		}

		if resp.StatusCode == 200 {
			return false, nil
		}

		if resp.StatusCode == 500 {
			return true, nil
		}

		return false, fmt.Errorf("Unexpected response code %d", resp.StatusCode)
	})
}

// Bind executes the bind portion of the create, this can only be called
// once successfully as subsequent binds will attempt to create bindings with
// the same ID.
func (ee *exampleExecutor) Bind() (json.RawMessage, error) {
	log.Printf("Binding %s\n", ee.Name)
	resp := ee.client.Bind(ee.InstanceId, ee.BindingId, ee.ServiceId, ee.PlanId, ee.BindParams)

	log.Println(resp.String())
	if resp.InError() {
		return nil, resp.Error
	}

	if resp.StatusCode == 201 {
		return resp.ResponseBody, nil
	}

	return nil, fmt.Errorf("Unexpected response code %d", resp.StatusCode)
}

// LogTestInfo writes information about the running example and a manual backout
// strategy if the test dies part of the way through.
func (ee *exampleExecutor) LogTestInfo() {
	log.Printf("Running Example: %s\n", ee.Name)

	ips := fmt.Sprintf("--instanceid %q --planid %q --serviceid %q", ee.InstanceId, ee.PlanId, ee.ServiceId)
	log.Printf("gcp-service-broker client provision %s --params %q\n", ips, ee.ProvisionParams)
	log.Printf("gcp-service-broker client bind %s --bindingid %q --params %q\n", ips, ee.BindingId, ee.BindParams)
	log.Printf("gcp-service-broker client unbind %s --bindingid %q\n", ips, ee.BindingId)
	log.Printf("gcp-service-broker client deprovision %s\n", ips)
}
<|endoftext|>