<commit_before>package login

import (
	"context"
	"crypto/rand"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os/exec"
	"runtime"
	"strings"
	"sync"

	"github.com/bwplotka/oidc"
)

const (
	codeParam  = "code"
	stateParam = "state"

	errParam     = "error"
	errDescParam = "error_description"
)

func rand128Bits() string {
	buff := make([]byte, 16) // 128 bit random ID.
	if _, err := io.ReadFull(rand.Reader, buff); err != nil {
		panic(err)
	}
	return strings.TrimRight(base64.URLEncoding.EncodeToString(buff), "=")
}

// open opens the specified URL in the default browser of the user.
func openBrowser(url string) error {
	var cmd string
	var args []string

	switch runtime.GOOS {
	case "windows":
		cmd = "cmd"
		args = []string{"/c", "start"}
	case "darwin":
		cmd = "open"
	default: // "linux", "freebsd", "openbsd", "netbsd"
		cmd = "xdg-open"
	}
	args = append(args, url)
	return exec.Command(cmd, args...).Start()
}

// callbackResponse contains return message from callback server including token or error.
type callbackResponse struct {
	token *oidc.Token
	err   error
}

// callbackRequest specifies values that are needed for expected callback handling.
type callbackRequest struct {
	ctx           context.Context
	expectedState string

	cfg    oidc.Config
	client *oidc.Client
}

// CallbackServer carries a callback handler for OIDC auth code flow.
// NOTE: This is not thread-safe in terms of multiple logins in the same time.
type CallbackServer struct {
	redirectURL string
	callbackCh  chan *callbackResponse

	// CallbackReq is written in separate thread so guard that.
	callbackReqMu sync.Mutex
	// If empty, nothing is expected, so callback should immediately return err.
	callbackReq *callbackRequest
}

// NewServer creates HTTP server with OIDC callback on the bindAddress an argument. BindAddress is the ultimately a redirectURL that all clients MUST register
// first on the OIDC server. It can (and is recommended) to point to localhost. Bind Address must include port. You can specify 0 if your
// OIDC provider support wildcard on port (almost all server does NOT).
func NewServer(bindAddress string) (srv *CallbackServer, closeSrv func(), err error) {
	bindURL, err := url.Parse(bindAddress)
	if err != nil {
		return nil, nil, fmt.Errorf("BindAddress is not in a form of URL. Err: %v", err)
	}

	listener, err := net.Listen("tcp", bindURL.Host)
	if err != nil {
		return nil, nil, fmt.Errorf("Failed to Listen for tcp on: %s. Err: %v", bindURL.Host, err)
	}

	s := &CallbackServer{
		redirectURL: fmt.Sprintf("http://%s%s", listener.Addr().String(), bindURL.Path),
		callbackCh:  make(chan *callbackResponse),
	}
	mux := http.NewServeMux()
	mux.HandleFunc(bindURL.Path, s.callbackHandler)

	go func() {
		http.Serve(listener, mux)
	}()

	return s, func() {
		listener.Close()
		close(s.callbackCh)
	}, nil
}

// NewReuseServer creates HTTP server with OIDC callback registered on given HTTP mux. Server constructed in such way
// is not responsible for serving the callback. This is responsibility of the caller.
func NewReuseServer(pattern string, listenAddress string, mux *http.ServeMux) *CallbackServer {
	s := &CallbackServer{
		redirectURL: fmt.Sprintf("http://%s%s", listenAddress, pattern),
		callbackCh:  make(chan *callbackResponse),
	}
	mux.HandleFunc(pattern, s.callbackHandler)
	return s
}

// callbackHandler handles redirect from OIDC provider with either code or error parameters.
// If none callback is expected it will return error.
// In case of valid code with corresponded state it will perform token exchange with OIDC provider.
// Any message is propagated via Go channel if the callback was expected.
// NOTE: This is not thread-safe in terms of multiple logins in the same time.
func (s *CallbackServer) callbackHandler(w http.ResponseWriter, r *http.Request) {
	s.callbackReqMu.Lock()
	if s.callbackReq == nil {
		w.WriteHeader(http.StatusPreconditionFailed)
		w.Write([]byte("Did not expect OIDC callback"))
		return
	}
	defer func() {
		s.callbackReq = nil
		s.callbackReqMu.Unlock()
	}()

	err := r.ParseForm()
	if err != nil {
		err := fmt.Errorf("Failed to parse request form. Err: %v", err)
		s.errRespond(w, r, err)
		return
	}

	code, state, err := parseCallbackRequest(r.Form)
	if err != nil {
		s.errRespond(w, r, err)
		return
	}

	if state != s.callbackReq.expectedState {
		err := fmt.Errorf("Invalid state parameter. Got %s, expected: %s", state, s.callbackReq.expectedState)
		s.errRespond(w, r, err)
		return
	}

	ctx := mergeContexts(r.Context(), s.callbackReq.ctx)
	oidcToken, err := s.callbackReq.client.Exchange(ctx, s.callbackReq.cfg, code)
	if err != nil {
		s.errRespond(w, r, err)
		return
	}

	callbackResponse := &callbackResponse{
		token: oidcToken,
	}
	OKCallbackResponse(w, r)
	select {
	case <-s.callbackReq.ctx.Done():
	case s.callbackCh <- callbackResponse:
	}
	return
}

func parseCallbackRequest(form url.Values) (code string, state string, err error) {
	state = form.Get(stateParam)
	if state == "" {
		return "", "", errors.New("User session error. No state parameter.")
	}

	if errorCode := form.Get(errParam); errorCode != "" {
		// Got error from provider. Passing through.
		return "", "", fmt.Errorf("Got error from provider: %s Desc: %s", errorCode, form.Get(errDescParam))
	}

	code = form.Get(codeParam)
	if code == "" {
		return "", "", errors.New("Missing code token.")
	}

	return code, state, nil
}

// OKCallbackResponse is package wide function variable that returns HTTP response on successful OIDC `code` flow.
var OKCallbackResponse = func(w http.ResponseWriter, _ *http.Request) {
	w.WriteHeader(http.StatusOK)
	w.Write([]byte("OIDC authentication flow is completed. You can close browser tab."))
}

// ErrCallbackResponse is package wide function variable that returns HTTP response on failed OIDC `code` flow.
// Note that, by default we don't want user to see anything wrong on browser side. All errors are propagated to command.
// If it is required otherwise, override this function.
var ErrCallbackResponse = func(w http.ResponseWriter, _ *http.Request, _ error) {
	w.WriteHeader(http.StatusOK)
	w.Write([]byte("OIDC authentication flow is completed. You can close browser tab."))
}

func (s *CallbackServer) errRespond(w http.ResponseWriter, r *http.Request, err error) {
	callbackResponse := &callbackResponse{
		err: err,
	}
	ErrCallbackResponse(w, r, err)

	select {
	case <-s.callbackReq.ctx.Done():
	case s.callbackCh <- callbackResponse:
	}
	return
}

func mergeContexts(originalCtx context.Context, oidcCtx context.Context) context.Context {
	if customClient := originalCtx.Value(oidc.HTTPClientCtxKey); customClient != nil {
		return originalCtx
	}
	return context.WithValue(originalCtx, oidc.HTTPClientCtxKey, oidcCtx.Value(oidc.HTTPClientCtxKey))
}

func (s *CallbackServer) ExpectCallback(callbackReq *callbackRequest) {
	s.callbackReqMu.Lock()
	defer s.callbackReqMu.Unlock()
	s.callbackReq = callbackReq
}

func (s *CallbackServer) Callback() <-chan *callbackResponse {
	return s.callbackCh
}

func (s *CallbackServer) RedirectURL() string {
	return s.redirectURL
}
<commit_msg>Escape ampersand when opening a new browser on windows. (#28)<commit_after>package login

import (
	"context"
	"crypto/rand"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os/exec"
	"runtime"
	"strings"
	"sync"

	"github.com/bwplotka/oidc"
)

const (
	codeParam  = "code"
	stateParam = "state"

	errParam     = "error"
	errDescParam = "error_description"
)

func rand128Bits() string {
	buff := make([]byte, 16) // 128 bit random ID.
	if _, err := io.ReadFull(rand.Reader, buff); err != nil {
		panic(err)
	}
	return strings.TrimRight(base64.URLEncoding.EncodeToString(buff), "=")
}

// open opens the specified URL in the default browser of the user.
func openBrowser(url string) error {
	var cmd string
	var args []string

	switch runtime.GOOS {
	case "windows":
		cmd = "cmd"
		args = []string{"/c", "start"}
		// If we don't escape &, cmd will ignore everything after the first &.
		url = strings.Replace(url, "&", "^&", -1)
	case "darwin":
		cmd = "open"
	default: // "linux", "freebsd", "openbsd", "netbsd"
		cmd = "xdg-open"
	}
	args = append(args, url)
	return exec.Command(cmd, args...).Start()
}

// callbackResponse contains return message from callback server including token or error.
type callbackResponse struct {
	token *oidc.Token
	err   error
}

// callbackRequest specifies values that are needed for expected callback handling.
type callbackRequest struct {
	ctx           context.Context
	expectedState string

	cfg    oidc.Config
	client *oidc.Client
}

// CallbackServer carries a callback handler for OIDC auth code flow.
// NOTE: This is not thread-safe in terms of multiple logins in the same time.
type CallbackServer struct {
	redirectURL string
	callbackCh  chan *callbackResponse

	// CallbackReq is written in separate thread so guard that.
	callbackReqMu sync.Mutex
	// If empty, nothing is expected, so callback should immediately return err.
	callbackReq *callbackRequest
}

// NewServer creates HTTP server with OIDC callback on the bindAddress an argument. BindAddress is the ultimately a redirectURL that all clients MUST register
// first on the OIDC server. It can (and is recommended) to point to localhost. Bind Address must include port. You can specify 0 if your
// OIDC provider support wildcard on port (almost all server does NOT).
func NewServer(bindAddress string) (srv *CallbackServer, closeSrv func(), err error) {
	bindURL, err := url.Parse(bindAddress)
	if err != nil {
		return nil, nil, fmt.Errorf("BindAddress is not in a form of URL. Err: %v", err)
	}

	listener, err := net.Listen("tcp", bindURL.Host)
	if err != nil {
		return nil, nil, fmt.Errorf("Failed to Listen for tcp on: %s. Err: %v", bindURL.Host, err)
	}

	s := &CallbackServer{
		redirectURL: fmt.Sprintf("http://%s%s", listener.Addr().String(), bindURL.Path),
		callbackCh:  make(chan *callbackResponse),
	}
	mux := http.NewServeMux()
	mux.HandleFunc(bindURL.Path, s.callbackHandler)

	go func() {
		http.Serve(listener, mux)
	}()

	return s, func() {
		listener.Close()
		close(s.callbackCh)
	}, nil
}

// NewReuseServer creates HTTP server with OIDC callback registered on given HTTP mux. Server constructed in such way
// is not responsible for serving the callback. This is responsibility of the caller.
func NewReuseServer(pattern string, listenAddress string, mux *http.ServeMux) *CallbackServer {
	s := &CallbackServer{
		redirectURL: fmt.Sprintf("http://%s%s", listenAddress, pattern),
		callbackCh:  make(chan *callbackResponse),
	}
	mux.HandleFunc(pattern, s.callbackHandler)
	return s
}

// callbackHandler handles redirect from OIDC provider with either code or error parameters.
// If none callback is expected it will return error.
// In case of valid code with corresponded state it will perform token exchange with OIDC provider.
// Any message is propagated via Go channel if the callback was expected.
// NOTE: This is not thread-safe in terms of multiple logins in the same time.
func (s *CallbackServer) callbackHandler(w http.ResponseWriter, r *http.Request) {
	s.callbackReqMu.Lock()
	if s.callbackReq == nil {
		w.WriteHeader(http.StatusPreconditionFailed)
		w.Write([]byte("Did not expect OIDC callback"))
		return
	}
	defer func() {
		s.callbackReq = nil
		s.callbackReqMu.Unlock()
	}()

	err := r.ParseForm()
	if err != nil {
		err := fmt.Errorf("Failed to parse request form. Err: %v", err)
		s.errRespond(w, r, err)
		return
	}

	code, state, err := parseCallbackRequest(r.Form)
	if err != nil {
		s.errRespond(w, r, err)
		return
	}

	if state != s.callbackReq.expectedState {
		err := fmt.Errorf("Invalid state parameter. Got %s, expected: %s", state, s.callbackReq.expectedState)
		s.errRespond(w, r, err)
		return
	}

	ctx := mergeContexts(r.Context(), s.callbackReq.ctx)
	oidcToken, err := s.callbackReq.client.Exchange(ctx, s.callbackReq.cfg, code)
	if err != nil {
		s.errRespond(w, r, err)
		return
	}

	callbackResponse := &callbackResponse{
		token: oidcToken,
	}
	OKCallbackResponse(w, r)
	select {
	case <-s.callbackReq.ctx.Done():
	case s.callbackCh <- callbackResponse:
	}
	return
}

func parseCallbackRequest(form url.Values) (code string, state string, err error) {
	state = form.Get(stateParam)
	if state == "" {
		return "", "", errors.New("User session error. No state parameter.")
	}

	if errorCode := form.Get(errParam); errorCode != "" {
		// Got error from provider. Passing through.
		return "", "", fmt.Errorf("Got error from provider: %s Desc: %s", errorCode, form.Get(errDescParam))
	}

	code = form.Get(codeParam)
	if code == "" {
		return "", "", errors.New("Missing code token.")
	}

	return code, state, nil
}

// OKCallbackResponse is package wide function variable that returns HTTP response on successful OIDC `code` flow.
var OKCallbackResponse = func(w http.ResponseWriter, _ *http.Request) {
	w.WriteHeader(http.StatusOK)
	w.Write([]byte("OIDC authentication flow is completed. You can close browser tab."))
}

// ErrCallbackResponse is package wide function variable that returns HTTP response on failed OIDC `code` flow.
// Note that, by default we don't want user to see anything wrong on browser side. All errors are propagated to command.
// If it is required otherwise, override this function.
var ErrCallbackResponse = func(w http.ResponseWriter, _ *http.Request, _ error) {
	w.WriteHeader(http.StatusOK)
	w.Write([]byte("OIDC authentication flow is completed. You can close browser tab."))
}

func (s *CallbackServer) errRespond(w http.ResponseWriter, r *http.Request, err error) {
	callbackResponse := &callbackResponse{
		err: err,
	}
	ErrCallbackResponse(w, r, err)

	select {
	case <-s.callbackReq.ctx.Done():
	case s.callbackCh <- callbackResponse:
	}
	return
}

func mergeContexts(originalCtx context.Context, oidcCtx context.Context) context.Context {
	if customClient := originalCtx.Value(oidc.HTTPClientCtxKey); customClient != nil {
		return originalCtx
	}
	return context.WithValue(originalCtx, oidc.HTTPClientCtxKey, oidcCtx.Value(oidc.HTTPClientCtxKey))
}

func (s *CallbackServer) ExpectCallback(callbackReq *callbackRequest) {
	s.callbackReqMu.Lock()
	defer s.callbackReqMu.Unlock()
	s.callbackReq = callbackReq
}

func (s *CallbackServer) Callback() <-chan *callbackResponse {
	return s.callbackCh
}

func (s *CallbackServer) RedirectURL() string {
	return s.redirectURL
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package mining

import (
	"sort"
	"time"

	log "github.com/sirupsen/logrus"

	"github.com/bytom/account"
	"github.com/bytom/blockchain/txbuilder"
	"github.com/bytom/consensus"
	"github.com/bytom/errors"
	"github.com/bytom/protocol"
	"github.com/bytom/protocol/bc"
	"github.com/bytom/protocol/bc/types"
	"github.com/bytom/protocol/state"
	"github.com/bytom/protocol/validation"
	"github.com/bytom/protocol/vm/vmutil"
)

// createCoinbaseTx returns a coinbase transaction paying an appropriate subsidy
// based on the passed block height to the provided address.  When the address
// is nil, the coinbase transaction will instead be redeemable by anyone.
func createCoinbaseTx(accountManager *account.Manager, amount uint64, blockHeight uint64) (tx *types.Tx, err error) {
	amount += consensus.BlockSubsidy(blockHeight)

	var script []byte
	if accountManager == nil {
		script, err = vmutil.DefaultCoinbaseProgram()
	} else {
		script, err = accountManager.GetCoinbaseControlProgram()
	}
	if err != nil {
		return
	}

	builder := txbuilder.NewBuilder(time.Now())
	if err = builder.AddInput(types.NewCoinbaseInput([]byte(string(blockHeight))), &txbuilder.SigningInstruction{}); err != nil {
		return
	}
	if err = builder.AddOutput(types.NewTxOutput(*consensus.BTMAssetID, amount, script)); err != nil {
		return
	}
	_, txData, err := builder.Build()
	if err != nil {
		return
	}

	byteData, err := txData.MarshalText()
	if err != nil {
		return
	}
	txData.SerializedSize = uint64(len(byteData))

	tx = &types.Tx{
		TxData: *txData,
		Tx:     types.MapTx(txData),
	}
	return
}

// NewBlockTemplate returns a new block template that is ready to be solved
func NewBlockTemplate(c *protocol.Chain, txPool *protocol.TxPool, accountManager *account.Manager) (b *types.Block, err error) {
	view := state.NewUtxoViewpoint()
	txStatus := bc.NewTransactionStatus()
	txStatus.SetStatus(0, false)
	txEntries := []*bc.Tx{nil}
	gasUsed := uint64(0)
	txFee := uint64(0)

	// get preblock info for generate next block
	preBlockHeader := c.BestBlockHeader()
	preBlockHash := preBlockHeader.Hash()
	nextBlockHeight := preBlockHeader.Height + 1
	nextBits, err := c.CalcNextBits(&preBlockHash)
	if err != nil {
		return nil, err
	}

	b = &types.Block{
		BlockHeader: types.BlockHeader{
			Version:           1,
			Height:            nextBlockHeight,
			PreviousBlockHash: preBlockHash,
			Timestamp:         uint64(time.Now().Unix()),
			BlockCommitment:   types.BlockCommitment{},
			Bits:              nextBits,
		},
	}
	bcBlock := &bc.Block{BlockHeader: &bc.BlockHeader{Height: nextBlockHeight}}
	b.Transactions = []*types.Tx{nil}

	txs := txPool.GetTransactions()
	sort.Sort(ByTime(txs))
	for _, txDesc := range txs {
		tx := txDesc.Tx.Tx
		gasOnlyTx := false

		if err := c.GetTransactionsUtxo(view, []*bc.Tx{tx}); err != nil {
			log.WithField("error", err).Error("mining block generate skip tx due to")
			txPool.RemoveTransaction(&tx.ID)
			continue
		}

		gasStatus, err := validation.ValidateTx(tx, bcBlock)
		if err != nil {
			if !gasStatus.GasValid {
				log.WithField("error", err).Error("mining block generate skip tx due to")
				txPool.RemoveTransaction(&tx.ID)
				continue
			}
			gasOnlyTx = true
		}

		if gasUsed+uint64(gasStatus.GasUsed) > consensus.MaxBlockGas {
			break
		}

		if err := view.ApplyTransaction(bcBlock, tx, gasOnlyTx); err != nil {
			log.WithField("error", err).Error("mining block generate skip tx due to")
			txPool.RemoveTransaction(&tx.ID)
			continue
		}

		txStatus.SetStatus(len(b.Transactions), gasOnlyTx)
		b.Transactions = append(b.Transactions, txDesc.Tx)
		txEntries = append(txEntries, tx)
		gasUsed += uint64(gasStatus.GasUsed)
		txFee += txDesc.Fee

		if gasUsed == consensus.MaxBlockGas {
			break
		}
	}

	// creater coinbase transaction
	b.Transactions[0], err = createCoinbaseTx(accountManager, txFee, nextBlockHeight)
	if err != nil {
		return nil, errors.Wrap(err, "fail on createCoinbaseTx")
	}
	txEntries[0] = b.Transactions[0].Tx

	b.BlockHeader.BlockCommitment.TransactionsMerkleRoot, err = bc.TxMerkleRoot(txEntries)
	if err != nil {
		return nil, err
	}

	b.BlockHeader.BlockCommitment.TransactionStatusHash, err = bc.TxStatusMerkleRoot(txStatus.VerifyStatus)
	return b, err
}
<commit_msg>fix the coinbase generate bug<commit_after>// Copyright (c) 2014-2016 The btcsuite developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package mining

import (
	"sort"
	"strconv"
	"time"

	log "github.com/sirupsen/logrus"

	"github.com/bytom/account"
	"github.com/bytom/blockchain/txbuilder"
	"github.com/bytom/consensus"
	"github.com/bytom/errors"
	"github.com/bytom/protocol"
	"github.com/bytom/protocol/bc"
	"github.com/bytom/protocol/bc/types"
	"github.com/bytom/protocol/state"
	"github.com/bytom/protocol/validation"
	"github.com/bytom/protocol/vm/vmutil"
)

// createCoinbaseTx returns a coinbase transaction paying an appropriate subsidy
// based on the passed block height to the provided address.  When the address
// is nil, the coinbase transaction will instead be redeemable by anyone.
func createCoinbaseTx(accountManager *account.Manager, amount uint64, blockHeight uint64) (tx *types.Tx, err error) {
	amount += consensus.BlockSubsidy(blockHeight)

	var script []byte
	if accountManager == nil {
		script, err = vmutil.DefaultCoinbaseProgram()
	} else {
		script, err = accountManager.GetCoinbaseControlProgram()
	}
	if err != nil {
		return
	}

	builder := txbuilder.NewBuilder(time.Now())
	if err = builder.AddInput(types.NewCoinbaseInput(
		append([]byte{0x00}, []byte(strconv.FormatUint(blockHeight, 10))...),
	), &txbuilder.SigningInstruction{}); err != nil {
		return
	}
	if err = builder.AddOutput(types.NewTxOutput(*consensus.BTMAssetID, amount, script)); err != nil {
		return
	}
	_, txData, err := builder.Build()
	if err != nil {
		return
	}

	byteData, err := txData.MarshalText()
	if err != nil {
		return
	}
	txData.SerializedSize = uint64(len(byteData))

	tx = &types.Tx{
		TxData: *txData,
		Tx:     types.MapTx(txData),
	}
	return
}

// NewBlockTemplate returns a new block template that is ready to be solved
func NewBlockTemplate(c *protocol.Chain, txPool *protocol.TxPool, accountManager *account.Manager) (b *types.Block, err error) {
	view := state.NewUtxoViewpoint()
	txStatus := bc.NewTransactionStatus()
	txStatus.SetStatus(0, false)
	txEntries := []*bc.Tx{nil}
	gasUsed := uint64(0)
	txFee := uint64(0)

	// get preblock info for generate next block
	preBlockHeader := c.BestBlockHeader()
	preBlockHash := preBlockHeader.Hash()
	nextBlockHeight := preBlockHeader.Height + 1
	nextBits, err := c.CalcNextBits(&preBlockHash)
	if err != nil {
		return nil, err
	}

	b = &types.Block{
		BlockHeader: types.BlockHeader{
			Version:           1,
			Height:            nextBlockHeight,
			PreviousBlockHash: preBlockHash,
			Timestamp:         uint64(time.Now().Unix()),
			BlockCommitment:   types.BlockCommitment{},
			Bits:              nextBits,
		},
	}
	bcBlock := &bc.Block{BlockHeader: &bc.BlockHeader{Height: nextBlockHeight}}
	b.Transactions = []*types.Tx{nil}

	txs := txPool.GetTransactions()
	sort.Sort(ByTime(txs))
	for _, txDesc := range txs {
		tx := txDesc.Tx.Tx
		gasOnlyTx := false

		if err := c.GetTransactionsUtxo(view, []*bc.Tx{tx}); err != nil {
			log.WithField("error", err).Error("mining block generate skip tx due to")
			txPool.RemoveTransaction(&tx.ID)
			continue
		}

		gasStatus, err := validation.ValidateTx(tx, bcBlock)
		if err != nil {
			if !gasStatus.GasValid {
				log.WithField("error", err).Error("mining block generate skip tx due to")
				txPool.RemoveTransaction(&tx.ID)
				continue
			}
			gasOnlyTx = true
		}

		if gasUsed+uint64(gasStatus.GasUsed) > consensus.MaxBlockGas {
			break
		}

		if err := view.ApplyTransaction(bcBlock, tx, gasOnlyTx); err != nil {
			log.WithField("error", err).Error("mining block generate skip tx due to")
			txPool.RemoveTransaction(&tx.ID)
			continue
		}

		txStatus.SetStatus(len(b.Transactions), gasOnlyTx)
		b.Transactions = append(b.Transactions, txDesc.Tx)
		txEntries = append(txEntries, tx)
		gasUsed += uint64(gasStatus.GasUsed)
		txFee += txDesc.Fee

		if gasUsed == consensus.MaxBlockGas {
			break
		}
	}

	// creater coinbase transaction
	b.Transactions[0], err = createCoinbaseTx(accountManager, txFee, nextBlockHeight)
	if err != nil {
		return nil, errors.Wrap(err, "fail on createCoinbaseTx")
	}
	txEntries[0] = b.Transactions[0].Tx

	b.BlockHeader.BlockCommitment.TransactionsMerkleRoot, err = bc.TxMerkleRoot(txEntries)
	if err != nil {
		return nil, err
	}

	b.BlockHeader.BlockCommitment.TransactionStatusHash, err = bc.TxStatusMerkleRoot(txStatus.VerifyStatus)
	return b, err
}
<|endoftext|>


<commit_before>package orderedtask

import (
	"bytes"
	"fmt"
	"math/rand"
	"sync"
	"testing"
	"time"
)

type Msg struct {
	text   string
	offset uint64
}

func TestSimpleExample(test *testing.T) {
	//runtime.GOMAXPROCS(2)
	const MsgCnt = 5000
	const PoolSize = 6

	//Create the pool with PoolSize workers
	//  Note: if you lower the pool size, the total runtime gets longer due to the
	//       lack of parallelization.
	//
	pool := NewPool(PoolSize, func(workerlocal map[string]interface{}, t *Task) {
		var buf bytes.Buffer
		if b, ok := workerlocal["buf"]; !ok {
			//worker local is not shared between go routines, so it's a good place to place a store a reusable items like buffers
			buf = bytes.Buffer{}
			workerlocal["buf"] = buf
		} else {
			buf = b.(bytes.Buffer)
		}
		buf.Reset()

		msg := t.Input.(*Msg)
		amt := time.Duration(rand.Intn(5))
		time.Sleep(time.Millisecond * amt) // long running operation
		t.Output = msg
	})

	wg := &sync.WaitGroup{}

	//Consume messages from the pool
	//  Note: they should be in order by the offset
	wg.Add(1)
	go func() {
		defer wg.Done()
		i := 0
		expectedoffset := uint64(0)
		for t := range pool.Results() {
			msg := t.Output.(*Msg)
			//fmt.Printf("msg off:%v text:%v \n", msg.offset, msg.text)

			i++
			if i == MsgCnt-1 {
				return
			} else if msg.offset != expectedoffset {
				test.Fatalf("the offsets weren't in order: got:%d expected:%d", msg.offset, expectedoffset)
			}
			expectedoffset++
		}
	}()

	//Produce messages into the pool
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < MsgCnt; i++ {
			m := &Msg{fmt.Sprintf("{'foo':'%d'}", i), uint64(i)}
			pool.Enqueue() <- &Task{Index: m.offset, Input: m}
		}
	}()

	wg.Wait()
}

func TestSlowConsumers(test *testing.T) {

	const MsgCnt = 200
	const PoolSize = 2
	const ConsumerSleep = 5

	msgchan := make(chan *Task, 10)

	pool := NewPool(PoolSize, func(workerlocal map[string]interface{}, t *Task) {
		msg := t.Input.(*Msg)
		amt := time.Duration(rand.Intn(2))
		time.Sleep(time.Millisecond * amt)
		t.Output = msg
	})

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < MsgCnt; i++ {
			m := &Msg{fmt.Sprintf("{'foo':'%d'}", i), uint64(i)}
			msgchan <- &Task{Index: m.offset, Input: m}
		}
		close(msgchan)
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		i := 0
		expectedoffset := uint64(0)

		//ticketbox := pool.GetTicketBox()

		for {
			select {
			case pool.Enqueue() <- <-msgchan:
			case t := <-pool.Results():
				//ticketbox.ReturnTicket()

				msg := t.Output.(*Msg)
				i++
				if i == MsgCnt-1 {
					return
				} else if msg.offset != expectedoffset {
					test.Fatalf("out of order: got:%d expected:%d", msg.offset, expectedoffset)
				}
				expectedoffset++
				amt := time.Duration(ConsumerSleep)
				time.Sleep(time.Millisecond * amt)
			}
		}
	}()

	wg.Wait()
}

func TestSlowProducers(test *testing.T) {

	const MsgCnt = 10000
	const PoolSize = 2

	pool := NewPool(PoolSize, func(workerlocal map[string]interface{}, t *Task) {
		msg := t.Input.(*Msg)
		amt := time.Duration(rand.Intn(10))
		time.Sleep(time.Millisecond * amt)
		t.Output = msg
	})

	producechan := make(chan *Task, 200)

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < MsgCnt; i++ {
			m := &Msg{fmt.Sprintf("{'foo':'%d'}", i), uint64(i)}
			producechan <- &Task{Index: m.offset, Input: m}
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		i := 0
		expectedoffset := uint64(0)
		getData := func() *Task {
			task := <-producechan
			return task
		}

		for {
			select {
			case pool.Enqueue() <- getData():
				pool.Enqueue()
				amt := time.Duration(rand.Intn(15))
				time.Sleep(time.Millisecond * amt)
			case t := <-pool.Results():
				msg := t.Output.(*Msg)

				i++
				if i == MsgCnt-1 {
					return
				} else if msg.offset != expectedoffset {
					test.Fatalf("the offsets weren't in order: got:%d expected:%d", msg.offset, expectedoffset)
				}
				expectedoffset++
			}
		}
	}()

	wg.Wait()
}

func TestFastWorkers(test *testing.T) {

	const MsgCnt = 50000
	const PoolSize = 2

	pool := NewPool(PoolSize, func(workerlocal map[string]interface{}, t *Task) {
		msg := t.Input.(*Msg)
		t.Output = msg
	})

	wg := &sync.WaitGroup{}

	//Produce messages into the pool
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < MsgCnt; i++ {
			m := &Msg{fmt.Sprintf("{'foo':'%d'}", i), uint64(i)}
			pool.Enqueue() <- &Task{Index: m.offset, Input: m}
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		i := 0
		expectedoffset := uint64(0)
		for t := range pool.Results() {
			msg := t.Output.(*Msg)

			i++
			if i == MsgCnt-1 {
				return
			} else if msg.offset != expectedoffset {
				test.Fatalf("the offsets weren't in order: got:%d expected:%d", msg.offset, expectedoffset)
			}
			expectedoffset++
		}
	}()

	wg.Wait()
}
<commit_msg>put in the ticker box changes.<commit_after>package orderedtask

import (
	"bytes"
	"fmt"
	"math/rand"
	"sync"
	"testing"
	"time"
)

type Msg struct {
	text   string
	offset uint64
}

func TestSimpleExample(test *testing.T) {
	//runtime.GOMAXPROCS(2)
	const MsgCnt = 5000
	const PoolSize = 6

	//Create the pool with PoolSize workers
	//  Note: if you lower the pool size, the total runtime gets longer due to the
	//       lack of parallelization.
	//
	pool := NewPool(PoolSize, func(workerlocal map[string]interface{}, t *Task) {
		var buf bytes.Buffer
		if b, ok := workerlocal["buf"]; !ok {
			//worker local is not shared between go routines, so it's a good place to place a store a reusable items like buffers
			buf = bytes.Buffer{}
			workerlocal["buf"] = buf
		} else {
			buf = b.(bytes.Buffer)
		}
		buf.Reset()

		msg := t.Input.(*Msg)
		amt := time.Duration(rand.Intn(5))
		time.Sleep(time.Millisecond * amt) // long running operation
		t.Output = msg
	})

	wg := &sync.WaitGroup{}

	//Consume messages from the pool
	//  Note: they should be in order by the offset
	wg.Add(1)
	go func() {
		defer wg.Done()
		i := 0
		expectedoffset := uint64(0)
		for t := range pool.Results() {
			msg := t.Output.(*Msg)
			//fmt.Printf("msg off:%v text:%v \n", msg.offset, msg.text)

			i++
			if i == MsgCnt-1 {
				return
			} else if msg.offset != expectedoffset {
				test.Fatalf("the offsets weren't in order: got:%d expected:%d", msg.offset, expectedoffset)
			}
			expectedoffset++
		}
	}()

	//Produce messages into the pool
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < MsgCnt; i++ {
			m := &Msg{fmt.Sprintf("{'foo':'%d'}", i), uint64(i)}
			pool.Enqueue() <- &Task{Index: m.offset, Input: m}
		}
	}()

	wg.Wait()
}

func TestSlowConsumers(test *testing.T) {

	const MsgCnt = 200
	const PoolSize = 2
	const ConsumerSleep = 5

	msgchan := make(chan *Task, 10)

	pool := NewPool(PoolSize, func(workerlocal map[string]interface{}, t *Task) {
		msg := t.Input.(*Msg)
		amt := time.Duration(rand.Intn(2))
		time.Sleep(time.Millisecond * amt)
		t.Output = msg
	})

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < MsgCnt; i++ {
			m := &Msg{fmt.Sprintf("{'foo':'%d'}", i), uint64(i)}
			msgchan <- &Task{Index: m.offset, Input: m}
		}
		close(msgchan)
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		i := 0
		expectedoffset := uint64(0)

		ticketbox := pool.GetTicketBox()

		for {
			select {
			case <-ticketbox.Tickets():
				t, ok := <-msgchan
				if ok {
					pool.Enqueue() <- t
				}
			case t := <-pool.Results():
				ticketbox.ReturnTicket()

				msg := t.Output.(*Msg)
				i++
				if i == MsgCnt-1 {
					return
				} else if msg.offset != expectedoffset {
					test.Fatalf("out of order: got:%d expected:%d", msg.offset, expectedoffset)
				}
				expectedoffset++
				amt := time.Duration(ConsumerSleep)
				time.Sleep(time.Millisecond * amt)
			}
		}
	}()

	wg.Wait()
}

func TestSlowProducers(test *testing.T) {

	const MsgCnt = 10000
	const PoolSize = 2

	pool := NewPool(PoolSize, func(workerlocal map[string]interface{}, t *Task) {
		msg := t.Input.(*Msg)
		amt := time.Duration(rand.Intn(10))
		time.Sleep(time.Millisecond * amt)
		t.Output = msg
	})

	producechan := make(chan *Task, 200)

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < MsgCnt; i++ {
			m := &Msg{fmt.Sprintf("{'foo':'%d'}", i), uint64(i)}
			producechan <- &Task{Index: m.offset, Input: m}
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		i := 0
		expectedoffset := uint64(0)
		getData := func() *Task {
			task := <-producechan
			return task
		}

		for {
			select {
			case pool.Enqueue() <- getData():
				pool.Enqueue()
				amt := time.Duration(rand.Intn(15))
				time.Sleep(time.Millisecond * amt)
			case t := <-pool.Results():
				msg := t.Output.(*Msg)

				i++
				if i == MsgCnt-1 {
					return
				} else if msg.offset != expectedoffset {
					test.Fatalf("the offsets weren't in order: got:%d expected:%d", msg.offset, expectedoffset)
				}
				expectedoffset++
			}
		}
	}()

	wg.Wait()
}

func TestFastWorkers(test *testing.T) {

	const MsgCnt = 50000
	const PoolSize = 2

	pool := NewPool(PoolSize, func(workerlocal map[string]interface{}, t *Task) {
		msg := t.Input.(*Msg)
		t.Output = msg
	})

	wg := &sync.WaitGroup{}

	//Produce messages into the pool
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < MsgCnt; i++ {
			m := &Msg{fmt.Sprintf("{'foo':'%d'}", i), uint64(i)}
			pool.Enqueue() <- &Task{Index: m.offset, Input: m}
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		i := 0
		expectedoffset := uint64(0)
		for t := range pool.Results() {
			msg := t.Output.(*Msg)

			i++
			if i == MsgCnt-1 {
				return
			} else if msg.offset != expectedoffset {
				test.Fatalf("the offsets weren't in order: got:%d expected:%d", msg.offset, expectedoffset)
			}
			expectedoffset++
		}
	}()

	wg.Wait()
}
<|endoftext|>


<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tree

import (
	"reflect"
	"testing"

	"upspin.io/context"
	"upspin.io/errors"
	"upspin.io/factotum"
	"upspin.io/key/inprocess"
	"upspin.io/upspin"

	_ "upspin.io/pack/ee"
	_ "upspin.io/store/inprocess"
)

const (
	userName   = "user@domain.com"
	serverName = "tree@server.com"
)

// This test checks the tree for log consistency by exercising the life-cycle of a tree,
// from creating a new tree from scratch, adding new nodes, flushing it to Store then
// adding more nodes to a new tree and having to load it from the Store.
func TestPutNodes(t *testing.T) {
	cfg := newConfigForTesting(t)
	tree := New(userName, cfg)

	dir1 := upspin.DirEntry{
		Name:    userName + "/",
		Attr:    upspin.AttrDirectory,
		Packing: cfg.Context.Packing(),
		Writer:  serverName,
	}
	err := tree.Put(&dir1)
	if err != nil {
		t.Fatal(err)
	}
	dir2 := upspin.DirEntry{
		Name:    userName + "/dir",
		Attr:    upspin.AttrDirectory,
		Packing: cfg.Context.Packing(),
		Writer:  serverName,
	}
	err = tree.Put(&dir2)
	if err != nil {
		t.Fatal(err)
	}
	dir3 := upspin.DirEntry{
		Name:    userName + "/dir/doc.pdf",
		Attr:    upspin.AttrNone,
		Packing: cfg.Context.Packing(),
		Writer:  serverName,
	}
	err = tree.Put(&dir3)
	if err != nil {
		t.Fatal(err)
	}

	// Verify three log entries were written.
	if got, want := cfg.Log.LastIndex(), 2; got != want {
		t.Fatalf("LastIndex = %d, want %d", got, want)
	}
	entries, err := cfg.Log.Read(0, 3)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(entries[0], dir1) {
		t.Errorf("dir1 = %v, want %v", entries[0], dir1)
	}
	if !reflect.DeepEqual(entries[1], dir2) {
		t.Errorf("dir2 = %v, want %v", entries[1], dir2)
	}
	if !reflect.DeepEqual(entries[2], dir3) {
		t.Errorf("dir3 = %v, want %v", entries[2], dir3)
	}

	// Lookup path.
	de, dirty, err := tree.Lookup(userName + "/dir/doc.pdf")
	if err != nil {
		t.Fatal(err)
	}
	if !dirty {
		t.Errorf("dirty = %v, want %v", dirty, true)
	}
	if !reflect.DeepEqual(*de, dir3) {
		t.Errorf("de = %v, want %v", de, dir3)
	}

	// Flush to later build a new tree and verify new is equivalent to old.
	err = tree.Flush()
	if err != nil {
		t.Fatal(err)
	}

	// New log index shows we're now at the end of the log.
	got, err := cfg.LogIndex.ReadLastIndex()
	if err != nil {
		t.Fatal(err)
	}
	if want := cfg.Log.LastIndex(); got != want {
		t.Fatalf("cfg.Log.LastIndex() = %d, want %d", got, want)
	}

	// Lookup now returns !dirty.
	de, dirty, err = tree.Lookup(userName + "/dir/doc.pdf")
	if err != nil {
		t.Fatal(err)
	}
	if dirty {
		t.Errorf("dirty = %v, want %v", dirty, false)
	}
	if !reflect.DeepEqual(*de, dir3) {
		t.Errorf("de = %v, want %v", de, dir3)
	}

	// Now start a new tree from scratch and confirm it is loaded from the Store.
	tree2 := New(userName, cfg)

	dir4 := &upspin.DirEntry{
		Name:    userName + "/dir/img.jpg",
		Attr:    upspin.AttrNone,
		Packing: cfg.Context.Packing(),
		Writer:  userName, // This was written by the user, the server is just packing it in a dir block.
	}
	err = tree2.Put(dir4)
	if err != nil {
		t.Fatal(err)
	}
	if got, want := cfg.Log.LastIndex(), 3; got != want {
		t.Fatalf("cfg.Log.LastIndex() = %d, want %d", cfg.Log.LastIndex(), want)
	}

	// Delete dir4.
	err = tree2.Delete(userName + "/dir/img.jpg")
	if err != nil {
		t.Fatal(err)
	}
	// Lookup won't return it.
	_, _, err = tree.Lookup(userName + "/dir/img.jpg")
	expectedErr := errors.E("Delete", errors.NotExist, upspin.PathName(userName+"/dir/img.jpg"))
	if errors.Match(expectedErr, err) {
		t.Fatalf("err = %s, want = %s", err, expectedErr)
	}
	// One new entry was written to the log (an updated dir2).
	if got, want := cfg.Log.LastIndex(), 4; got != want {
		t.Fatalf("cfg.Log.LastIndex() = %d, want %d", cfg.Log.LastIndex(), want)
	}
	// Verify logged entry is a new dir2
	entries, err = cfg.Log.Read(4, 1)
	if err != nil {
		t.Fatal(err)
	}
	if got, want := entries[0].Name, upspin.PathName(userName+"/dir"); got != want {
		t.Errorf("entries[0].Name = %s, want = %s", got, want)
	}
}

// Test that an empty root can be saved and retrieved.
// Roots are handled differently than other directory entries.
func TestPutEmptyRoot(t *testing.T) {
	cfg := newConfigForTesting(t)
	tree := New(userName, cfg)

	dir1 := &upspin.DirEntry{
		Name:    userName + "/",
		Attr:    upspin.AttrDirectory,
		Packing: cfg.Context.Packing(),
		Writer:  serverName,
	}
	err := tree.Put(dir1)
	if err != nil {
		t.Fatal(err)
	}

	err = tree.Flush()
	if err != nil {
		t.Fatal(err)
	}

	// Now start a new tree from scratch and confirm it is loaded from the Store just the same.
	tree2 := New(userName, cfg)

	dir2 := &upspin.DirEntry{
		Name:    userName + "/dir",
		Attr:    upspin.AttrDirectory,
		Packing: cfg.Context.Packing(),
		Writer:  serverName,
	}
	err = tree2.Put(dir2)
	if err != nil {
		t.Fatal(err)
	}

	// Try to put a file under an non-existent dir
	dir3 := &upspin.DirEntry{
		Name:    userName + "/invaliddir/myfile.txt",
		Attr:    upspin.AttrNone,
		Packing: cfg.Context.Packing(),
		Writer:  serverName,
	}
	err = tree2.Put(dir3)
	if err == nil {
		t.Fatal("Expected error, got none")
	}
	expectedErr := errors.E(errors.NotExist)
	if !errors.Match(expectedErr, err) {
		t.Errorf("err = %s, want %s", err, expectedErr)
	}
}

// TODO: TestPutLargeNode: test that a huge DirEntry (>blockSize) gets split into multiple ones.
// TODO: Run all tests in loop using Plain and Debug packs as well.
// TODO: test more error cases.
// TODO: Implement and test starting the tree from a non-empty log and a log index not at the end of the log.

// newConfigForTesting creates a config with mocks, fakes, inprocess and otherwise testing
// versions of the Tree's dependencies.
func newConfigForTesting(t *testing.T) *Config {
	pubKey := upspin.PublicKey("p256\n104278369061367353805983276707664349405797936579880352274235000127123465616334\n26941412685198548642075210264642864401950753555952207894712845271039438170192\n")
	// TODO: rename factotum.DeprecatedNew to NewWithKeys or NewForTesting.
	factotum, err := factotum.DeprecatedNew(
		pubKey,
		"82201047360680847258309465671292633303992565667422607675215625927005262185934")
	if err != nil {
		t.Fatal(err)
	}
	endpointInProcess := upspin.Endpoint{
		Transport: upspin.InProcess,
		NetAddr:   "",
	}
	context := context.New().
		SetFactotum(factotum).
		SetUserName(serverName).
		SetStoreEndpoint(endpointInProcess).
		SetKeyEndpoint(endpointInProcess).
		SetPacking(upspin.EEPack)
	key := context.KeyServer()
	testKey, ok := key.(*inprocess.Service)
	if !ok {
		t.Fatal(err)
	}
	// Set the public key for the tree, since it must do Auth against the Store.
	testKey.SetPublicKeys(serverName, []upspin.PublicKey{pubKey})

	// Set the public key for the user, since EE Pack requires the dir owner to have a wrapped key.
	// TODO: re-think this for directories, but probably correct as-is because if the dir server goes
	// rogue or fails, the user can always run a dir server locally as himself and retrieve dir blocks.
	testKey.SetPublicKeys(userName, []upspin.PublicKey{pubKey})

	return &Config{
		Context: context,
		Log: &fakeLog{
			user: userName,
		},
		LogIndex: &fakeLogIndex{
			user: userName,
		},
	}
}

// fakeLog implements a simple, in-memory Log for testing.
type fakeLog struct {
	user       upspin.UserName
	dirEntries []upspin.DirEntry
}

var _ Log = (*fakeLog)(nil)

// fakeLog implements a simple, in-memory LogIndex for testing.
type fakeLogIndex struct {
	user      upspin.UserName
	root      *upspin.DirEntry
	lastIndex int
}

var _ LogIndex = (*fakeLogIndex)(nil)

// User returns the user name for whom this log logs.
func (l *fakeLog) User() upspin.UserName {
	return l.user
}

// Append appends a DirEntry at the end of the log.
func (l *fakeLog) Append(de *upspin.DirEntry) error {
	l.dirEntries = append(l.dirEntries, *de)
	return nil
}

// Read reads at most n entries from the log starting at index.
func (l *fakeLog) Read(index, n int) ([]upspin.DirEntry, error) {
	return l.dirEntries[index : index+n], nil // No error checking.
}

// LastIndex returns the index of the most-recently-appended entry.
func (l *fakeLog) LastIndex() int {
	return len(l.dirEntries) - 1
}

// Root returns the location of the user's root.
func (l *fakeLogIndex) Root() (*upspin.DirEntry, error) {
	if l.root != nil {
		return l.root, nil
	}
	return nil, errors.E(errors.NotExist)
}

// SaveRoot saves the user's root.
func (l *fakeLogIndex) SaveRoot(r *upspin.DirEntry) error {
	l.root = r
	return nil
}

// User returns the user name who owns the root of the tree that this
// log index represents.
func (l *fakeLogIndex) User() upspin.UserName {
	return l.user
}

// ReadLastIndex reads from stable storage the index saved by SaveLastIndex.
func (l *fakeLogIndex) ReadLastIndex() (int, error) {
	return l.lastIndex, nil
}

// SaveLastIndex saves to stable storage the last index processed.
func (l *fakeLogIndex) SaveLastIndex(idx int) error {
	l.lastIndex = idx
	return nil
}
<commit_msg>dir/server/tree: fix tests to work with new Factotum<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tree

import (
	"os"
	"path/filepath"
	"reflect"
	"testing"

	"upspin.io/context"
	"upspin.io/errors"
	"upspin.io/factotum"
	"upspin.io/key/inprocess"
	"upspin.io/log"
	"upspin.io/upspin"

	_ "upspin.io/pack/ee"
	_ "upspin.io/store/inprocess"
)

const (
	userName   = "user@domain.com"
	serverName = "tree@server.com"
)

// This test checks the tree for log consistency by exercising the life-cycle of a tree,
// from creating a new tree from scratch, adding new nodes, flushing it to Store then
// adding more nodes to a new tree and having to load it from the Store.
func TestPutNodes(t *testing.T) {
	cfg := newConfigForTesting(t)
	tree := New(userName, cfg)

	dir1 := upspin.DirEntry{
		Name:    userName + "/",
		Attr:    upspin.AttrDirectory,
		Packing: cfg.Context.Packing(),
		Writer:  serverName,
	}
	err := tree.Put(&dir1)
	if err != nil {
		t.Fatal(err)
	}
	dir2 := upspin.DirEntry{
		Name:    userName + "/dir",
		Attr:    upspin.AttrDirectory,
		Packing: cfg.Context.Packing(),
		Writer:  serverName,
	}
	err = tree.Put(&dir2)
	if err != nil {
		t.Fatal(err)
	}
	dir3 := upspin.DirEntry{
		Name:    userName + "/dir/doc.pdf",
		Attr:    upspin.AttrNone,
		Packing: cfg.Context.Packing(),
		Writer:  serverName,
	}
	err = tree.Put(&dir3)
	if err != nil {
		t.Fatal(err)
	}

	// Verify three log entries were written.
	if got, want := cfg.Log.LastIndex(), 2; got != want {
		t.Fatalf("LastIndex = %d, want %d", got, want)
	}
	entries, err := cfg.Log.Read(0, 3)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(entries[0], dir1) {
		t.Errorf("dir1 = %v, want %v", entries[0], dir1)
	}
	if !reflect.DeepEqual(entries[1], dir2) {
		t.Errorf("dir2 = %v, want %v", entries[1], dir2)
	}
	if !reflect.DeepEqual(entries[2], dir3) {
		t.Errorf("dir3 = %v, want %v", entries[2], dir3)
	}

	// Lookup path.
	de, dirty, err := tree.Lookup(userName + "/dir/doc.pdf")
	if err != nil {
		t.Fatal(err)
	}
	if !dirty {
		t.Errorf("dirty = %v, want %v", dirty, true)
	}
	if !reflect.DeepEqual(*de, dir3) {
		t.Errorf("de = %v, want %v", de, dir3)
	}

	// Flush to later build a new tree and verify new is equivalent to old.
	err = tree.Flush()
	if err != nil {
		t.Fatal(err)
	}

	// New log index shows we're now at the end of the log.
	got, err := cfg.LogIndex.ReadLastIndex()
	if err != nil {
		t.Fatal(err)
	}
	if want := cfg.Log.LastIndex(); got != want {
		t.Fatalf("cfg.Log.LastIndex() = %d, want %d", got, want)
	}

	// Lookup now returns !dirty.
	de, dirty, err = tree.Lookup(userName + "/dir/doc.pdf")
	if err != nil {
		t.Fatal(err)
	}
	if dirty {
		t.Errorf("dirty = %v, want %v", dirty, false)
	}
	if !reflect.DeepEqual(*de, dir3) {
		t.Errorf("de = %v, want %v", de, dir3)
	}

	// Now start a new tree from scratch and confirm it is loaded from the Store.
	tree2 := New(userName, cfg)

	dir4 := &upspin.DirEntry{
		Name:    userName + "/dir/img.jpg",
		Attr:    upspin.AttrNone,
		Packing: cfg.Context.Packing(),
		Writer:  userName, // This was written by the user, the server is just packing it in a dir block.
	}
	err = tree2.Put(dir4)
	if err != nil {
		t.Fatal(err)
	}
	if got, want := cfg.Log.LastIndex(), 3; got != want {
		t.Fatalf("cfg.Log.LastIndex() = %d, want %d", cfg.Log.LastIndex(), want)
	}

	// Delete dir4.
	err = tree2.Delete(userName + "/dir/img.jpg")
	if err != nil {
		t.Fatal(err)
	}
	// Lookup won't return it.
	_, _, err = tree.Lookup(userName + "/dir/img.jpg")
	expectedErr := errors.E("Delete", errors.NotExist, upspin.PathName(userName+"/dir/img.jpg"))
	if errors.Match(expectedErr, err) {
		t.Fatalf("err = %s, want = %s", err, expectedErr)
	}
	// One new entry was written to the log (an updated dir2).
	if got, want := cfg.Log.LastIndex(), 4; got != want {
		t.Fatalf("cfg.Log.LastIndex() = %d, want %d", cfg.Log.LastIndex(), want)
	}
	// Verify logged entry is a new dir2
	entries, err = cfg.Log.Read(4, 1)
	if err != nil {
		t.Fatal(err)
	}
	if got, want := entries[0].Name, upspin.PathName(userName+"/dir"); got != want {
		t.Errorf("entries[0].Name = %s, want = %s", got, want)
	}
}

// Test that an empty root can be saved and retrieved.
// Roots are handled differently than other directory entries.
func TestPutEmptyRoot(t *testing.T) {
	cfg := newConfigForTesting(t)
	tree := New(userName, cfg)

	dir1 := &upspin.DirEntry{
		Name:    userName + "/",
		Attr:    upspin.AttrDirectory,
		Packing: cfg.Context.Packing(),
		Writer:  serverName,
	}
	err := tree.Put(dir1)
	if err != nil {
		t.Fatal(err)
	}

	err = tree.Flush()
	if err != nil {
		t.Fatal(err)
	}

	// Now start a new tree from scratch and confirm it is loaded from the Store just the same.
	tree2 := New(userName, cfg)

	dir2 := &upspin.DirEntry{
		Name:    userName + "/dir",
		Attr:    upspin.AttrDirectory,
		Packing: cfg.Context.Packing(),
		Writer:  serverName,
	}
	err = tree2.Put(dir2)
	if err != nil {
		t.Fatal(err)
	}

	// Try to put a file under an non-existent dir
	dir3 := &upspin.DirEntry{
		Name:    userName + "/invaliddir/myfile.txt",
		Attr:    upspin.AttrNone,
		Packing: cfg.Context.Packing(),
		Writer:  serverName,
	}
	err = tree2.Put(dir3)
	if err == nil {
		t.Fatal("Expected error, got none")
	}
	expectedErr := errors.E(errors.NotExist)
	if !errors.Match(expectedErr, err) {
		t.Errorf("err = %s, want %s", err, expectedErr)
	}
}

// TODO: TestPutLargeNode: test that a huge DirEntry (>blockSize) gets split into multiple ones.
// TODO: Run all tests in loop using Plain and Debug packs as well.
// TODO: test more error cases.
// TODO: Implement and test starting the tree from a non-empty log and a log index not at the end of the log.

// newConfigForTesting creates a config with mocks, fakes, inprocess and otherwise testing
// versions of the Tree's dependencies.
func newConfigForTesting(t *testing.T) *Config {
	factotum, err := factotum.New(repo("key/testdata/upspin-test"))
	if err != nil {
		t.Fatal(err)
	}
	endpointInProcess := upspin.Endpoint{
		Transport: upspin.InProcess,
		NetAddr:   "",
	}
	context := context.New().
		SetFactotum(factotum).
		SetUserName(serverName).
		SetStoreEndpoint(endpointInProcess).
		SetKeyEndpoint(endpointInProcess).
		SetPacking(upspin.EEPack)
	key := context.KeyServer()
	testKey, ok := key.(*inprocess.Service)
	if !ok {
		t.Fatal(err)
	}
	// Set the public key for the tree, since it must do Auth against the Store.
	testKey.SetPublicKeys(serverName, []upspin.PublicKey{factotum.PublicKey()})

	// Set the public key for the user, since EE Pack requires the dir owner to have a wrapped key.
	// TODO: re-think this for directories, but probably correct as-is because if the dir server goes
	// rogue or fails, the user can always run a dir server locally as himself and retrieve dir blocks.
	testKey.SetPublicKeys(userName, []upspin.PublicKey{factotum.PublicKey()})

	return &Config{
		Context: context,
		Log: &fakeLog{
			user: userName,
		},
		LogIndex: &fakeLogIndex{
			user: userName,
		},
	}
}

// fakeLog implements a simple, in-memory Log for testing.
type fakeLog struct {
	user       upspin.UserName
	dirEntries []upspin.DirEntry
}

var _ Log = (*fakeLog)(nil)

// fakeLog implements a simple, in-memory LogIndex for testing.
type fakeLogIndex struct {
	user      upspin.UserName
	root      *upspin.DirEntry
	lastIndex int
}

var _ LogIndex = (*fakeLogIndex)(nil)

// User returns the user name for whom this log logs.
func (l *fakeLog) User() upspin.UserName {
	return l.user
}

// Append appends a DirEntry at the end of the log.
func (l *fakeLog) Append(de *upspin.DirEntry) error {
	l.dirEntries = append(l.dirEntries, *de)
	return nil
}

// Read reads at most n entries from the log starting at index.
func (l *fakeLog) Read(index, n int) ([]upspin.DirEntry, error) {
	return l.dirEntries[index : index+n], nil // No error checking.
}

// LastIndex returns the index of the most-recently-appended entry.
func (l *fakeLog) LastIndex() int {
	return len(l.dirEntries) - 1
}

// Root returns the location of the user's root.
func (l *fakeLogIndex) Root() (*upspin.DirEntry, error) {
	if l.root != nil {
		return l.root, nil
	}
	return nil, errors.E(errors.NotExist)
}

// SaveRoot saves the user's root.
func (l *fakeLogIndex) SaveRoot(r *upspin.DirEntry) error {
	l.root = r
	return nil
}

// User returns the user name who owns the root of the tree that this
// log index represents.
func (l *fakeLogIndex) User() upspin.UserName {
	return l.user
}

// ReadLastIndex reads from stable storage the index saved by SaveLastIndex.
func (l *fakeLogIndex) ReadLastIndex() (int, error) {
	return l.lastIndex, nil
}

// SaveLastIndex saves to stable storage the last index processed.
func (l *fakeLogIndex) SaveLastIndex(idx int) error {
	l.lastIndex = idx
	return nil
}

// repo returns the local pathname of a file in the upspin repository.
func repo(dir string) string {
	gopath := os.Getenv("GOPATH")
	if len(gopath) == 0 {
		log.Fatal("no GOPATH")
	}
	return filepath.Join(gopath, "src/upspin.io/"+dir)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/docker-library/go-dockerlibrary/manifest"
	"github.com/docker-library/go-dockerlibrary/pkg/execpipe"

	goGit "gopkg.in/src-d/go-git.v4"
	goGitConfig "gopkg.in/src-d/go-git.v4/config"
	goGitPlumbing "gopkg.in/src-d/go-git.v4/plumbing"
)

func gitCache() string {
	return filepath.Join(defaultCache, "git")
}

func gitCommand(args ...string) *exec.Cmd {
	if debugFlag {
		fmt.Printf("$ git %q\n", args)
	}
	cmd := exec.Command("git", args...)
	cmd.Dir = gitCache()
	return cmd
}

func git(args ...string) ([]byte, error) {
	out, err := gitCommand(args...).Output()
	if err != nil {
		if ee, ok := err.(*exec.ExitError); ok {
			return nil, fmt.Errorf("%v\ncommand: git %q\n%s", ee, args, string(ee.Stderr))
		}
	}
	return out, err
}

var gitRepo *goGit.Repository

func ensureGitInit() error {
	if gitRepo != nil {
		return nil
	}

	gitCacheDir := gitCache()
	err := os.MkdirAll(gitCacheDir, os.ModePerm)
	if err != nil {
		return err
	}

	gitRepo, err = goGit.PlainInit(gitCacheDir, true)
	if err == goGit.ErrRepositoryAlreadyExists {
		gitRepo, err = goGit.PlainOpen(gitCacheDir)
	}
	if err != nil {
		return err
	}

	// ensure garbage collection is disabled so we keep dangling commits
	config, err := gitRepo.Config()
	if err != nil {
		return err
	}
	config.Raw = config.Raw.SetOption("gc", "", "auto", "0")
	gitRepo.Storer.SetConfig(config)

	return nil
}

var fullGitCommitRegex = regexp.MustCompile(`^[0-9a-f]{40}$|^[0-9a-f]{64}$`)

func getGitCommit(commit string) (string, error) {
	if fullGitCommitRegex.MatchString(commit) {
		_, err := gitRepo.CommitObject(goGitPlumbing.NewHash(commit))
		if err != nil {
			return "", err
		}
		return commit, nil
	}

	h, err := gitRepo.ResolveRevision(goGitPlumbing.Revision(commit + "^{commit}"))
	if err != nil {
		return "", err
	}
	return h.String(), nil
}

func gitStream(args ...string) (io.ReadCloser, error) {
	return execpipe.Run(gitCommand(args...))
}

func gitArchive(commit string, dir string) (io.ReadCloser, error) {
	if dir == "." {
		dir = ""
	} else {
		dir += "/"
	}
	return gitStream("archive", "--format=tar", commit+":"+dir)
}

func gitShow(commit string, file string) (string, error) {
	gitCommit, err := gitRepo.CommitObject(goGitPlumbing.NewHash(commit))
	if err != nil {
		return "", err
	}

	gitFile, err := gitCommit.File(file)
	if err != nil {
		return "", err
	}

	contents, err := gitFile.Contents()
	if err != nil {
		return "", err
	}

	return contents, nil
}

// for gitNormalizeForTagUsage()
// see http://stackoverflow.com/a/26382358/433558
var (
	gitBadTagChars = regexp.MustCompile(`(?:` + strings.Join([]string{
		`[^0-9a-zA-Z/._-]+`,

		// They can include slash `/` for hierarchical (directory) grouping, but no slash-separated component can begin with a dot `.` or end with the sequence `.lock`.
		`/[.]+`,
		`[.]lock(?:/|$)`,

		// They cannot have two consecutive dots `..` anywhere.
		`[.][.]+`,

		// They cannot end with a dot `.`
		// They cannot begin or end with a slash `/`
		`[/.]+$`,
		`^[/.]+`,
	}, `|`) + `)`)

	gitMultipleSlashes = regexp.MustCompile(`(?://+)`)
)

// strip/replace "bad" characters from text for use as a Git tag
func gitNormalizeForTagUsage(text string) string {
	return gitMultipleSlashes.ReplaceAllString(gitBadTagChars.ReplaceAllString(text, "-"), "/")
}

var gitRepoCache = map[string]string{}

func (r Repo) fetchGitRepo(arch string, entry *manifest.Manifest2822Entry) (string, error) {
	cacheKey := strings.Join([]string{
		entry.ArchGitRepo(arch),
		entry.ArchGitFetch(arch),
		entry.ArchGitCommit(arch),
	}, "\n")
	if commit, ok := gitRepoCache[cacheKey]; ok {
		entry.SetGitCommit(arch, commit)
		return commit, nil
	}

	err := ensureGitInit()
	if err != nil {
		return "", err
	}

	if manifest.GitCommitRegex.MatchString(entry.ArchGitCommit(arch)) {
		commit, err := getGitCommit(entry.ArchGitCommit(arch))
		if err == nil {
			gitRepoCache[cacheKey] = commit
			entry.SetGitCommit(arch, commit)
			return commit, nil
		}
	}

	fetchString := entry.ArchGitFetch(arch) + ":"
	if entry.ArchGitCommit(arch) == "FETCH_HEAD" {
		// fetch remote tag references to a local tag ref so that we can cache them and not re-fetch every time
		localRef := "refs/tags/" + gitNormalizeForTagUsage(cacheKey)
		commit, err := getGitCommit(localRef)
		if err == nil {
			gitRepoCache[cacheKey] = commit
			entry.SetGitCommit(arch, commit)
			return commit, nil
		}
		fetchString += localRef
	} else {
		// we create a temporary remote dir so that we can clean it up completely afterwards
		refBase := "refs/remotes"
		refBaseDir := filepath.Join(gitCache(), refBase)

		err := os.MkdirAll(refBaseDir, os.ModePerm)
		if err != nil {
			return "", err
		}

		tempRefDir, err := ioutil.TempDir(refBaseDir, "temp")
		if err != nil {
			return "", err
		}
		defer os.RemoveAll(tempRefDir)

		tempRef := path.Join(refBase, filepath.Base(tempRefDir))
		if entry.ArchGitFetch(arch) == manifest.DefaultLineBasedFetch {
			// backwards compat (see manifest/line-based.go in go-dockerlibrary)
			fetchString += tempRef + "/*"
		} else {
			fetchString += tempRef + "/temp"
		}
	}

	if strings.HasPrefix(entry.ArchGitRepo(arch), "git://github.com/") {
		fmt.Fprintf(os.Stderr, "warning: insecure protocol git:// detected: %s\n", entry.ArchGitRepo(arch))
		entry.SetGitRepo(arch, strings.Replace(entry.ArchGitRepo(arch), "git://", "https://", 1))
	}

	gitRemote, err := gitRepo.CreateRemoteAnonymous(&goGitConfig.RemoteConfig{
		Name: "anonymous",
		URLs: []string{entry.ArchGitRepo(arch)},
	})
	if err != nil {
		return "", err
	}

	err = gitRemote.Fetch(&goGit.FetchOptions{
		RefSpecs: []goGitConfig.RefSpec{goGitConfig.RefSpec(fetchString)},
		Tags:     goGit.NoTags,

		//Progress: os.Stdout,
	})
	if err != nil {
		return "", err
	}

	commit, err := getGitCommit(entry.ArchGitCommit(arch))
	if err != nil {
		return "", err
	}

	_, err = gitRepo.CreateTag(arch+"/"+r.RepoName+"/"+entry.Tags[0], goGitPlumbing.NewHash(commit), nil)
	if err != nil {
		return "", err
	}

	gitRepoCache[cacheKey] = commit
	entry.SetGitCommit(arch, commit)
	return commit, nil
}
<commit_msg>Add "DeleteTag" before "CreateTag" to avoid "ErrTagExists"<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/docker-library/go-dockerlibrary/manifest"
	"github.com/docker-library/go-dockerlibrary/pkg/execpipe"

	goGit "gopkg.in/src-d/go-git.v4"
	goGitConfig "gopkg.in/src-d/go-git.v4/config"
	goGitPlumbing "gopkg.in/src-d/go-git.v4/plumbing"
)

func gitCache() string {
	return filepath.Join(defaultCache, "git")
}

func gitCommand(args ...string) *exec.Cmd {
	if debugFlag {
		fmt.Printf("$ git %q\n", args)
	}
	cmd := exec.Command("git", args...)
	cmd.Dir = gitCache()
	return cmd
}

func git(args ...string) ([]byte, error) {
	out, err := gitCommand(args...).Output()
	if err != nil {
		if ee, ok := err.(*exec.ExitError); ok {
			return nil, fmt.Errorf("%v\ncommand: git %q\n%s", ee, args, string(ee.Stderr))
		}
	}
	return out, err
}

var gitRepo *goGit.Repository

func ensureGitInit() error {
	if gitRepo != nil {
		return nil
	}

	gitCacheDir := gitCache()
	err := os.MkdirAll(gitCacheDir, os.ModePerm)
	if err != nil {
		return err
	}

	gitRepo, err = goGit.PlainInit(gitCacheDir, true)
	if err == goGit.ErrRepositoryAlreadyExists {
		gitRepo, err = goGit.PlainOpen(gitCacheDir)
	}
	if err != nil {
		return err
	}

	// ensure garbage collection is disabled so we keep dangling commits
	config, err := gitRepo.Config()
	if err != nil {
		return err
	}
	config.Raw = config.Raw.SetOption("gc", "", "auto", "0")
	gitRepo.Storer.SetConfig(config)

	return nil
}

var fullGitCommitRegex = regexp.MustCompile(`^[0-9a-f]{40}$|^[0-9a-f]{64}$`)

func getGitCommit(commit string) (string, error) {
	if fullGitCommitRegex.MatchString(commit) {
		_, err := gitRepo.CommitObject(goGitPlumbing.NewHash(commit))
		if err != nil {
			return "", err
		}
		return commit, nil
	}

	h, err := gitRepo.ResolveRevision(goGitPlumbing.Revision(commit + "^{commit}"))
	if err != nil {
		return "", err
	}
	return h.String(), nil
}

func gitStream(args ...string) (io.ReadCloser, error) {
	return execpipe.Run(gitCommand(args...))
}

func gitArchive(commit string, dir string) (io.ReadCloser, error) {
	if dir == "." {
		dir = ""
	} else {
		dir += "/"
	}
	return gitStream("archive", "--format=tar", commit+":"+dir)
}

func gitShow(commit string, file string) (string, error) {
	gitCommit, err := gitRepo.CommitObject(goGitPlumbing.NewHash(commit))
	if err != nil {
		return "", err
	}

	gitFile, err := gitCommit.File(file)
	if err != nil {
		return "", err
	}

	contents, err := gitFile.Contents()
	if err != nil {
		return "", err
	}

	return contents, nil
}

// for gitNormalizeForTagUsage()
// see http://stackoverflow.com/a/26382358/433558
var (
	gitBadTagChars = regexp.MustCompile(`(?:` + strings.Join([]string{
		`[^0-9a-zA-Z/._-]+`,

		// They can include slash `/` for hierarchical (directory) grouping, but no slash-separated component can begin with a dot `.` or end with the sequence `.lock`.
		`/[.]+`,
		`[.]lock(?:/|$)`,

		// They cannot have two consecutive dots `..` anywhere.
		`[.][.]+`,

		// They cannot end with a dot `.`
		// They cannot begin or end with a slash `/`
		`[/.]+$`,
		`^[/.]+`,
	}, `|`) + `)`)

	gitMultipleSlashes = regexp.MustCompile(`(?://+)`)
)

// strip/replace "bad" characters from text for use as a Git tag
func gitNormalizeForTagUsage(text string) string {
	return gitMultipleSlashes.ReplaceAllString(gitBadTagChars.ReplaceAllString(text, "-"), "/")
}

var gitRepoCache = map[string]string{}

func (r Repo) fetchGitRepo(arch string, entry *manifest.Manifest2822Entry) (string, error) {
	cacheKey := strings.Join([]string{
		entry.ArchGitRepo(arch),
		entry.ArchGitFetch(arch),
		entry.ArchGitCommit(arch),
	}, "\n")
	if commit, ok := gitRepoCache[cacheKey]; ok {
		entry.SetGitCommit(arch, commit)
		return commit, nil
	}

	err := ensureGitInit()
	if err != nil {
		return "", err
	}

	if manifest.GitCommitRegex.MatchString(entry.ArchGitCommit(arch)) {
		commit, err := getGitCommit(entry.ArchGitCommit(arch))
		if err == nil {
			gitRepoCache[cacheKey] = commit
			entry.SetGitCommit(arch, commit)
			return commit, nil
		}
	}

	fetchString := entry.ArchGitFetch(arch) + ":"
	if entry.ArchGitCommit(arch) == "FETCH_HEAD" {
		// fetch remote tag references to a local tag ref so that we can cache them and not re-fetch every time
		localRef := "refs/tags/" + gitNormalizeForTagUsage(cacheKey)
		commit, err := getGitCommit(localRef)
		if err == nil {
			gitRepoCache[cacheKey] = commit
			entry.SetGitCommit(arch, commit)
			return commit, nil
		}
		fetchString += localRef
	} else {
		// we create a temporary remote dir so that we can clean it up completely afterwards
		refBase := "refs/remotes"
		refBaseDir := filepath.Join(gitCache(), refBase)

		err := os.MkdirAll(refBaseDir, os.ModePerm)
		if err != nil {
			return "", err
		}

		tempRefDir, err := ioutil.TempDir(refBaseDir, "temp")
		if err != nil {
			return "", err
		}
		defer os.RemoveAll(tempRefDir)

		tempRef := path.Join(refBase, filepath.Base(tempRefDir))
		if entry.ArchGitFetch(arch) == manifest.DefaultLineBasedFetch {
			// backwards compat (see manifest/line-based.go in go-dockerlibrary)
			fetchString += tempRef + "/*"
		} else {
			fetchString += tempRef + "/temp"
		}
	}

	if strings.HasPrefix(entry.ArchGitRepo(arch), "git://github.com/") {
		fmt.Fprintf(os.Stderr, "warning: insecure protocol git:// detected: %s\n", entry.ArchGitRepo(arch))
		entry.SetGitRepo(arch, strings.Replace(entry.ArchGitRepo(arch), "git://", "https://", 1))
	}

	gitRemote, err := gitRepo.CreateRemoteAnonymous(&goGitConfig.RemoteConfig{
		Name: "anonymous",
		URLs: []string{entry.ArchGitRepo(arch)},
	})
	if err != nil {
		return "", err
	}

	err = gitRemote.Fetch(&goGit.FetchOptions{
		RefSpecs: []goGitConfig.RefSpec{goGitConfig.RefSpec(fetchString)},
		Tags:     goGit.NoTags,

		//Progress: os.Stdout,
	})
	if err != nil {
		return "", err
	}

	commit, err := getGitCommit(entry.ArchGitCommit(arch))
	if err != nil {
		return "", err
	}

	gitTag := arch+"/"+r.RepoName+"/"+entry.Tags[0]
	gitRepo.DeleteTag(gitTag) // avoid "ErrTagExists"
	_, err = gitRepo.CreateTag(gitTag, goGitPlumbing.NewHash(commit), nil)
	if err != nil {
		return "", err
	}

	gitRepoCache[cacheKey] = commit
	entry.SetGitCommit(arch, commit)
	return commit, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/shared"
)

var fileCmd = APIEndpoint{
	Name: "file",
	Path: "files",

	Get:    APIEndpointAction{Handler: fileHandler},
	Post:   APIEndpointAction{Handler: fileHandler},
	Delete: APIEndpointAction{Handler: fileHandler},
}

func fileHandler(d *Daemon, r *http.Request) response.Response {
	path := r.FormValue("path")
	if path == "" {
		return response.BadRequest(fmt.Errorf("missing path argument"))
	}

	switch r.Method {
	case "GET":
		return fileGet(path, r)
	case "POST":
		return filePost(path, r)
	case "DELETE":
		return fileDelete(path, r)
	default:
		return response.NotFound(fmt.Errorf("Method '%s' not found", r.Method))
	}
}

func fileGet(path string, r *http.Request) response.Response {
	uid, gid, mode, fType, dirEnts, err := getFileInfo(path)
	if err != nil {
		return response.SmartError(err)
	}

	headers := map[string]string{
		"X-LXD-uid":  fmt.Sprintf("%d", uid),
		"X-LXD-gid":  fmt.Sprintf("%d", gid),
		"X-LXD-mode": fmt.Sprintf("%04o", mode),
		"X-LXD-type": fType,
	}

	if fType == "file" || fType == "symlink" {
		// Make a file response struct
		files := make([]response.FileResponseEntry, 1)
		files[0].Identifier = filepath.Base(path)

		f, err := ioutil.TempFile(filepath.Dir(path), "lxd_getfile_")
		if err != nil {
			return response.SmartError(err)
		}
		defer f.Close()

		if fType == "file" {
			src, err := os.Open(path)
			if err != nil {
				return response.SmartError(err)
			}
			defer src.Close()

			_, err = io.Copy(f, src)
			if err != nil {
				return response.SmartError(err)
			}
		} else {
			target, err := os.Readlink(path)
			if err != nil {
				return response.SmartError(err)
			}

			_, err = f.WriteString(target + "\n")
			if err != nil {
				return response.SmartError(err)
			}
		}

		files[0].Path = f.Name()
		files[0].Filename = filepath.Base(path)

		return response.FileResponse(r, files, headers, true)
	} else if fType == "directory" {
		return response.SyncResponseHeaders(true, dirEnts, headers)
	}

	return response.InternalError(fmt.Errorf("bad file type %s", fType))
}

func filePost(path string, r *http.Request) response.Response {
	// Extract file ownership and mode from headers
	uid, gid, mode, fType, write := shared.ParseLXDFileHeaders(r.Header)

	if !shared.StringInSlice(write, []string{"overwrite", "append"}) {
		return response.BadRequest(fmt.Errorf("Bad file write mode: %s", write))
	}

	if fType == "file" {
		// Write file content to a tempfile
		temp, err := ioutil.TempFile("", "lxd_forkputfile_")
		if err != nil {
			return response.InternalError(err)
		}
		defer func() {
			temp.Close()
			os.Remove(temp.Name())
		}()

		_, err = io.Copy(temp, r.Body)
		if err != nil {
			return response.InternalError(err)
		}

		// Transfer the file into the container
		err = filePush("file", temp.Name(), path, uid, gid, mode, write)
		if err != nil {
			return response.InternalError(err)
		}

		return response.EmptySyncResponse
	} else if fType == "symlink" {
		target, err := ioutil.ReadAll(r.Body)
		if err != nil {
			return response.InternalError(err)
		}

		err = filePush("symlink", string(target), path, uid, gid, mode, write)
		if err != nil {
			return response.InternalError(err)
		}
		return response.EmptySyncResponse
	} else if fType == "directory" {
		err := filePush("directory", "", path, uid, gid, mode, write)
		if err != nil {
			return response.InternalError(err)
		}
		return response.EmptySyncResponse
	}

	return response.BadRequest(fmt.Errorf("Bad file type: %s", fType))
}

func fileDelete(path string, r *http.Request) response.Response {
	err := os.Remove(path)
	if err != nil {
		return response.SmartError(err)
	}

	return response.EmptySyncResponse
}

func getFileInfo(path string) (int64, int64, os.FileMode, string, []string, error) {
	var stat unix.Stat_t

	err := os.Chdir("/")
	if err != nil {
		return -1, -1, 0, "", nil, err
	}

	fi, err := os.Lstat(path)
	if err != nil {
		return -1, -1, 0, "", nil, err
	}

	err = unix.Lstat(path, &stat)
	if err != nil {
		return -1, -1, 0, "", nil, err
	}

	var fType string
	var dirEnts []string

	if fi.Mode().IsDir() {
		fType = "directory"

		f, err := os.Open(path)
		if err != nil {
			return -1, -1, 0, "", nil, err
		}

		dirEnts, err = f.Readdirnames(0)
		if err != nil {
			return -1, -1, 0, "", nil, err
		}

	} else {
		if fi.Mode()&os.ModeSymlink != 0 {
			fType = "symlink"
		} else {
			fType = "file"
		}
	}

	// 0xFFF = 0b7777
	return int64(stat.Uid), int64(stat.Gid), fi.Mode() & 0xFFF, fType, dirEnts, nil
}

func filePush(fType string, srcpath string, dstpath string, uid int64, gid int64, mode int, write string) error {
	switch fType {
	case "file":
		if !shared.PathExists(dstpath) {
			if uid == -1 {
				uid = 0
			}

			if gid == -1 {
				gid = 0
			}

			if mode == -1 {
				mode = 0
			}
		}

		flags := os.O_CREATE | os.O_WRONLY

		if write == "overwrite" {
			flags |= os.O_TRUNC
		} else if write == "append" {
			flags |= os.O_APPEND
		}

		dst, err := os.OpenFile(dstpath, flags, os.FileMode(mode))
		if err != nil {
			return err
		}
		defer dst.Close()

		src, err := os.Open(srcpath)
		if err != nil {
			return err
		}
		defer src.Close()

		_, err = io.Copy(dst, src)
		if err != nil {
			return err
		}

		err = os.Chown(dst.Name(), int(uid), int(gid))
		if err != nil {
			return err
		}

		return nil
	case "symlink":
		if uid == -1 {
			uid = 0
		}

		if gid == -1 {
			gid = 0
		}

		err := os.Symlink(srcpath, dstpath)
		if err != nil {
			return err
		}

		err = os.Lchown(dstpath, int(uid), int(gid))
		if err != nil {
			return err
		}

		return nil
	case "directory":
		if uid == -1 {
			uid = 0
		}

		if gid == -1 {
			gid = 0
		}

		if mode == -1 {
			mode = 0
		}

		err := os.MkdirAll(dstpath, os.FileMode(mode))
		if err != nil {
			return err
		}

		err = os.Chown(dstpath, int(uid), int(gid))
		if err != nil {
			return err
		}

		return nil
	}

	return fmt.Errorf("Bad file type: %s", fType)
}
<commit_msg>lxd-agent: Update for FileResponse changes<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/shared"
)

var fileCmd = APIEndpoint{
	Name: "file",
	Path: "files",

	Get:    APIEndpointAction{Handler: fileHandler},
	Post:   APIEndpointAction{Handler: fileHandler},
	Delete: APIEndpointAction{Handler: fileHandler},
}

func fileHandler(d *Daemon, r *http.Request) response.Response {
	path := r.FormValue("path")
	if path == "" {
		return response.BadRequest(fmt.Errorf("missing path argument"))
	}

	switch r.Method {
	case "GET":
		return fileGet(path, r)
	case "POST":
		return filePost(path, r)
	case "DELETE":
		return fileDelete(path, r)
	default:
		return response.NotFound(fmt.Errorf("Method '%s' not found", r.Method))
	}
}

func fileGet(path string, r *http.Request) response.Response {
	uid, gid, mode, fType, dirEnts, err := getFileInfo(path)
	if err != nil {
		return response.SmartError(err)
	}

	headers := map[string]string{
		"X-LXD-uid":  fmt.Sprintf("%d", uid),
		"X-LXD-gid":  fmt.Sprintf("%d", gid),
		"X-LXD-mode": fmt.Sprintf("%04o", mode),
		"X-LXD-type": fType,
	}

	if fType == "file" || fType == "symlink" {
		// Make a file response struct
		files := make([]response.FileResponseEntry, 1)
		files[0].Identifier = filepath.Base(path)

		f, err := ioutil.TempFile(filepath.Dir(path), "lxd_getfile_")
		if err != nil {
			return response.SmartError(err)
		}
		defer f.Close()

		if fType == "file" {
			src, err := os.Open(path)
			if err != nil {
				return response.SmartError(err)
			}
			defer src.Close()

			_, err = io.Copy(f, src)
			if err != nil {
				return response.SmartError(err)
			}
		} else {
			target, err := os.Readlink(path)
			if err != nil {
				return response.SmartError(err)
			}

			_, err = f.WriteString(target + "\n")
			if err != nil {
				return response.SmartError(err)
			}
		}

		files[0].Path = f.Name()
		files[0].Filename = filepath.Base(path)
		files[0].Cleanup = func() { os.Remove(f.Name()) }

		return response.FileResponse(r, files, headers)
	} else if fType == "directory" {
		return response.SyncResponseHeaders(true, dirEnts, headers)
	}

	return response.InternalError(fmt.Errorf("bad file type %s", fType))
}

func filePost(path string, r *http.Request) response.Response {
	// Extract file ownership and mode from headers
	uid, gid, mode, fType, write := shared.ParseLXDFileHeaders(r.Header)

	if !shared.StringInSlice(write, []string{"overwrite", "append"}) {
		return response.BadRequest(fmt.Errorf("Bad file write mode: %s", write))
	}

	if fType == "file" {
		// Write file content to a tempfile
		temp, err := ioutil.TempFile("", "lxd_forkputfile_")
		if err != nil {
			return response.InternalError(err)
		}
		defer func() {
			temp.Close()
			os.Remove(temp.Name())
		}()

		_, err = io.Copy(temp, r.Body)
		if err != nil {
			return response.InternalError(err)
		}

		// Transfer the file into the container
		err = filePush("file", temp.Name(), path, uid, gid, mode, write)
		if err != nil {
			return response.InternalError(err)
		}

		return response.EmptySyncResponse
	} else if fType == "symlink" {
		target, err := ioutil.ReadAll(r.Body)
		if err != nil {
			return response.InternalError(err)
		}

		err = filePush("symlink", string(target), path, uid, gid, mode, write)
		if err != nil {
			return response.InternalError(err)
		}
		return response.EmptySyncResponse
	} else if fType == "directory" {
		err := filePush("directory", "", path, uid, gid, mode, write)
		if err != nil {
			return response.InternalError(err)
		}
		return response.EmptySyncResponse
	}

	return response.BadRequest(fmt.Errorf("Bad file type: %s", fType))
}

func fileDelete(path string, r *http.Request) response.Response {
	err := os.Remove(path)
	if err != nil {
		return response.SmartError(err)
	}

	return response.EmptySyncResponse
}

func getFileInfo(path string) (int64, int64, os.FileMode, string, []string, error) {
	var stat unix.Stat_t

	err := os.Chdir("/")
	if err != nil {
		return -1, -1, 0, "", nil, err
	}

	fi, err := os.Lstat(path)
	if err != nil {
		return -1, -1, 0, "", nil, err
	}

	err = unix.Lstat(path, &stat)
	if err != nil {
		return -1, -1, 0, "", nil, err
	}

	var fType string
	var dirEnts []string

	if fi.Mode().IsDir() {
		fType = "directory"

		f, err := os.Open(path)
		if err != nil {
			return -1, -1, 0, "", nil, err
		}

		dirEnts, err = f.Readdirnames(0)
		if err != nil {
			return -1, -1, 0, "", nil, err
		}

	} else {
		if fi.Mode()&os.ModeSymlink != 0 {
			fType = "symlink"
		} else {
			fType = "file"
		}
	}

	// 0xFFF = 0b7777
	return int64(stat.Uid), int64(stat.Gid), fi.Mode() & 0xFFF, fType, dirEnts, nil
}

func filePush(fType string, srcpath string, dstpath string, uid int64, gid int64, mode int, write string) error {
	switch fType {
	case "file":
		if !shared.PathExists(dstpath) {
			if uid == -1 {
				uid = 0
			}

			if gid == -1 {
				gid = 0
			}

			if mode == -1 {
				mode = 0
			}
		}

		flags := os.O_CREATE | os.O_WRONLY

		if write == "overwrite" {
			flags |= os.O_TRUNC
		} else if write == "append" {
			flags |= os.O_APPEND
		}

		dst, err := os.OpenFile(dstpath, flags, os.FileMode(mode))
		if err != nil {
			return err
		}
		defer dst.Close()

		src, err := os.Open(srcpath)
		if err != nil {
			return err
		}
		defer src.Close()

		_, err = io.Copy(dst, src)
		if err != nil {
			return err
		}

		err = os.Chown(dst.Name(), int(uid), int(gid))
		if err != nil {
			return err
		}

		return nil
	case "symlink":
		if uid == -1 {
			uid = 0
		}

		if gid == -1 {
			gid = 0
		}

		err := os.Symlink(srcpath, dstpath)
		if err != nil {
			return err
		}

		err = os.Lchown(dstpath, int(uid), int(gid))
		if err != nil {
			return err
		}

		return nil
	case "directory":
		if uid == -1 {
			uid = 0
		}

		if gid == -1 {
			gid = 0
		}

		if mode == -1 {
			mode = 0
		}

		err := os.MkdirAll(dstpath, os.FileMode(mode))
		if err != nil {
			return err
		}

		err = os.Chown(dstpath, int(uid), int(gid))
		if err != nil {
			return err
		}

		return nil
	}

	return fmt.Errorf("Bad file type: %s", fType)
}
<|endoftext|>


<commit_before>package volman_test

import (
	"encoding/json"
	"os"
	"testing"

	"fmt"
	"path"
	"path/filepath"

	"code.cloudfoundry.org/inigo/gardenrunner"
	"code.cloudfoundry.org/inigo/helpers"
	"code.cloudfoundry.org/inigo/world"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/ginkgoreporter"
	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/localip"
	"code.cloudfoundry.org/voldriver"
	"code.cloudfoundry.org/volman"
	"github.com/cloudfoundry-incubator/garden"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

var (
	componentMaker world.ComponentMaker

	gardenProcess ifrit.Process
	gardenClient  garden.Client

	volmanClient        volman.Manager
	driverSyncer        ifrit.Runner
	driverSyncerProcess ifrit.Process
	localDriverProcess  ifrit.Process

	logger lager.Logger

	driverPluginsPath string
)

var _ = SynchronizedBeforeSuite(func() []byte {
	payload, err := json.Marshal(world.BuiltArtifacts{
		Executables: CompileTestedExecutables(),
	})
	Expect(err).NotTo(HaveOccurred())

	return payload
}, func(encodedBuiltArtifacts []byte) {
	var builtArtifacts world.BuiltArtifacts

	err := json.Unmarshal(encodedBuiltArtifacts, &builtArtifacts)
	Expect(err).NotTo(HaveOccurred())

	localIP, err := localip.LocalIP()
	Expect(err).NotTo(HaveOccurred())

	componentMaker = helpers.MakeComponentMaker(builtArtifacts, localIP)
})

var _ = BeforeEach(func() {
	logger = lagertest.NewTestLogger("volman-inigo-suite")

	gardenProcess = ginkgomon.Invoke(componentMaker.Garden())
	gardenClient = componentMaker.GardenClient()

	localDriverProcess = ginkgomon.Invoke(componentMaker.VolmanDriver(logger))

	// make a dummy spec file not corresponding to a running driver just to make sure volman ignores it
	driverPluginsPath = path.Join(componentMaker.VolmanDriverConfigDir, fmt.Sprintf("node-%d", config.GinkgoConfig.ParallelNode))
	voldriver.WriteDriverSpec(logger, driverPluginsPath, "deaddriver", "json", []byte(`{"Name":"deaddriver","Addr":"https://127.0.0.1:1111"}`))

	volmanClient, driverSyncer = componentMaker.VolmanClient(logger)
	driverSyncerProcess = ginkgomon.Invoke(driverSyncer)

})

var _ = AfterEach(func() {
	destroyContainerErrors := helpers.CleanupGarden(gardenClient)

	helpers.StopProcesses(gardenProcess, driverSyncerProcess, localDriverProcess)

	Expect(destroyContainerErrors).To(
		BeEmpty(),
		"%d containers failed to be destroyed!",
		len(destroyContainerErrors),
	)

	os.Remove(filepath.Join(driverPluginsPath, "deaddriver.json"))
})

func TestVolman(t *testing.T) {
	helpers.RegisterDefaultTimeouts()

	RegisterFailHandler(Fail)

	RunSpecsWithDefaultAndCustomReporters(t, "Volman Integration Suite", []Reporter{
		ginkgoreporter.New(GinkgoWriter),
	})
}

func CompileTestedExecutables() world.BuiltExecutables {
	var err error

	builtExecutables := world.BuiltExecutables{}

	builtExecutables["garden"], err = gexec.BuildIn(os.Getenv("GARDEN_GOPATH"), gardenrunner.GardenServerPackageName(), "-race", "-a", "-tags", "daemon")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["local-driver"], err = gexec.Build("github.com/cloudfoundry-incubator/localdriver/cmd/localdriver", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["auctioneer"], err = gexec.BuildIn(os.Getenv("AUCTIONEER_GOPATH"), "code.cloudfoundry.org/auctioneer/cmd/auctioneer", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["rep"], err = gexec.BuildIn(os.Getenv("REP_GOPATH"), "code.cloudfoundry.org/rep/cmd/rep", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["bbs"], err = gexec.BuildIn(os.Getenv("BBS_GOPATH"), "code.cloudfoundry.org/bbs/cmd/bbs", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["file-server"], err = gexec.BuildIn(os.Getenv("FILE_SERVER_GOPATH"), "code.cloudfoundry.org/fileserver/cmd/file-server", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["route-emitter"], err = gexec.BuildIn(os.Getenv("ROUTE_EMITTER_GOPATH"), "code.cloudfoundry.org/route-emitter/cmd/route-emitter", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["router"], err = gexec.BuildIn(os.Getenv("ROUTER_GOPATH"), "github.com/cloudfoundry/gorouter", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["ssh-proxy"], err = gexec.Build("code.cloudfoundry.org/diego-ssh/cmd/ssh-proxy", "-race")
	Expect(err).NotTo(HaveOccurred())

	return builtExecutables
}
<commit_msg>localdriver -> code.cloudfoundry.org [#127201811](https://www.pivotaltracker.com/story/show/127201811)<commit_after>package volman_test

import (
	"encoding/json"
	"os"
	"testing"

	"fmt"
	"path"
	"path/filepath"

	"code.cloudfoundry.org/inigo/gardenrunner"
	"code.cloudfoundry.org/inigo/helpers"
	"code.cloudfoundry.org/inigo/world"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/ginkgoreporter"
	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/localip"
	"code.cloudfoundry.org/voldriver"
	"code.cloudfoundry.org/volman"
	"github.com/cloudfoundry-incubator/garden"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

var (
	componentMaker world.ComponentMaker

	gardenProcess ifrit.Process
	gardenClient  garden.Client

	volmanClient        volman.Manager
	driverSyncer        ifrit.Runner
	driverSyncerProcess ifrit.Process
	localDriverProcess  ifrit.Process

	logger lager.Logger

	driverPluginsPath string
)

var _ = SynchronizedBeforeSuite(func() []byte {
	payload, err := json.Marshal(world.BuiltArtifacts{
		Executables: CompileTestedExecutables(),
	})
	Expect(err).NotTo(HaveOccurred())

	return payload
}, func(encodedBuiltArtifacts []byte) {
	var builtArtifacts world.BuiltArtifacts

	err := json.Unmarshal(encodedBuiltArtifacts, &builtArtifacts)
	Expect(err).NotTo(HaveOccurred())

	localIP, err := localip.LocalIP()
	Expect(err).NotTo(HaveOccurred())

	componentMaker = helpers.MakeComponentMaker(builtArtifacts, localIP)
})

var _ = BeforeEach(func() {
	logger = lagertest.NewTestLogger("volman-inigo-suite")

	gardenProcess = ginkgomon.Invoke(componentMaker.Garden())
	gardenClient = componentMaker.GardenClient()

	localDriverProcess = ginkgomon.Invoke(componentMaker.VolmanDriver(logger))

	// make a dummy spec file not corresponding to a running driver just to make sure volman ignores it
	driverPluginsPath = path.Join(componentMaker.VolmanDriverConfigDir, fmt.Sprintf("node-%d", config.GinkgoConfig.ParallelNode))
	voldriver.WriteDriverSpec(logger, driverPluginsPath, "deaddriver", "json", []byte(`{"Name":"deaddriver","Addr":"https://127.0.0.1:1111"}`))

	volmanClient, driverSyncer = componentMaker.VolmanClient(logger)
	driverSyncerProcess = ginkgomon.Invoke(driverSyncer)

})

var _ = AfterEach(func() {
	destroyContainerErrors := helpers.CleanupGarden(gardenClient)

	helpers.StopProcesses(gardenProcess, driverSyncerProcess, localDriverProcess)

	Expect(destroyContainerErrors).To(
		BeEmpty(),
		"%d containers failed to be destroyed!",
		len(destroyContainerErrors),
	)

	os.Remove(filepath.Join(driverPluginsPath, "deaddriver.json"))
})

func TestVolman(t *testing.T) {
	helpers.RegisterDefaultTimeouts()

	RegisterFailHandler(Fail)

	RunSpecsWithDefaultAndCustomReporters(t, "Volman Integration Suite", []Reporter{
		ginkgoreporter.New(GinkgoWriter),
	})
}

func CompileTestedExecutables() world.BuiltExecutables {
	var err error

	builtExecutables := world.BuiltExecutables{}

	builtExecutables["garden"], err = gexec.BuildIn(os.Getenv("GARDEN_GOPATH"), gardenrunner.GardenServerPackageName(), "-race", "-a", "-tags", "daemon")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["local-driver"], err = gexec.Build("code.cloudfoundry.org/localdriver/cmd/localdriver", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["auctioneer"], err = gexec.BuildIn(os.Getenv("AUCTIONEER_GOPATH"), "code.cloudfoundry.org/auctioneer/cmd/auctioneer", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["rep"], err = gexec.BuildIn(os.Getenv("REP_GOPATH"), "code.cloudfoundry.org/rep/cmd/rep", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["bbs"], err = gexec.BuildIn(os.Getenv("BBS_GOPATH"), "code.cloudfoundry.org/bbs/cmd/bbs", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["file-server"], err = gexec.BuildIn(os.Getenv("FILE_SERVER_GOPATH"), "code.cloudfoundry.org/fileserver/cmd/file-server", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["route-emitter"], err = gexec.BuildIn(os.Getenv("ROUTE_EMITTER_GOPATH"), "code.cloudfoundry.org/route-emitter/cmd/route-emitter", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["router"], err = gexec.BuildIn(os.Getenv("ROUTER_GOPATH"), "github.com/cloudfoundry/gorouter", "-race")
	Expect(err).NotTo(HaveOccurred())

	builtExecutables["ssh-proxy"], err = gexec.Build("code.cloudfoundry.org/diego-ssh/cmd/ssh-proxy", "-race")
	Expect(err).NotTo(HaveOccurred())

	return builtExecutables
}
<|endoftext|>