<commit_before>// Copyright (c) 2014-2018 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package peer

import (
	"bytes"
	"container/list"
	"encoding/hex"
	"github.com/bitmark-inc/bitmarkd/block"
	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/bitmarkd/genesis"
	"github.com/bitmark-inc/bitmarkd/messagebus"
	"github.com/bitmark-inc/bitmarkd/mode"
	"github.com/bitmark-inc/bitmarkd/peer/upstream"
	"github.com/bitmark-inc/bitmarkd/util"
	"github.com/bitmark-inc/logger"
	"time"
)

// various timeouts
const (
	cycleInterval         = 15 * time.Second // pause to limit bandwidth
	connectorTimeout      = 60 * time.Second // time out for connections
	samplelingLimit       = 10               // number of cycles to be 1 block out of sync before resync
	fetchBlocksPerCycle   = 200              // number of blocks to fetch in one set
	forkProtection        = 60               // fail to fork if height difference is greater than this
	minimumClients        = 3                // do not proceed unless this many clients are connected
	maximumDynamicClients = 10               // total number of dynamic clients
)

// a state type for the thread
type connectorState int

// state of the connector process
const (
	cStateConnecting   connectorState = iota // register to nodes and make outgoing connections
	cStateHighestBlock connectorState = iota // locate node(s) with highest block number
	cStateForkDetect   connectorState = iota // read block hashes to check for possible fork
	cStateFetchBlocks  connectorState = iota // fetch blocks from current or fork point
	cStateRebuild      connectorState = iota // rebuild database from fork point (config setting to force total rebuild)
	cStateSampling     connectorState = iota // signal resync complete and sample nodes to see if out of sync occurs
)

type connector struct {
	log *logger.L

	staticClients []*upstream.Upstream

	dynamicClients list.List

	state connectorState

	theClient        *upstream.Upstream // client used for fetching blocks
	startBlockNumber uint64             // block number where local chain forks
	height           uint64             // block number on best node
	samples          int                // counter to detect missed block broadcast
}

// initialise the connector
func (conn *connector) initialise(privateKey []byte, publicKey []byte, connect []Connection, dynamicEnabled bool) error {

	log := logger.New("connector")
	conn.log = log

	log.Info("initialising…")

	// allocate all sockets
	staticCount := len(connect) // can be zero
	if 0 == staticCount && !dynamicEnabled {
		log.Error("zero static connections and dynamic is disabled")
		return fault.ErrNoConnectionsAvailable
	}
	conn.staticClients = make([]*upstream.Upstream, staticCount)

	// error code for goto fail
	errX := error(nil)

	// initially connect all static sockets
	for i, c := range connect {
		address, err := util.NewConnection(c.Address)
		if nil != err {
			log.Errorf("client[%d]=address: %q  error: %s", i, c.Address, err)
			errX = err
			goto fail
		}
		serverPublicKey, err := hex.DecodeString(c.PublicKey)
		if nil != err {
			log.Errorf("client[%d]=public: %q  error: %s", i, c.PublicKey, err)
			errX = err
			goto fail
		}

		// prevent connection to self
		if bytes.Equal(publicKey, serverPublicKey) {
			errX = fault.ErrConnectingToSelfForbidden
			log.Errorf("client[%d]=public: %q  error: %s", i, c.PublicKey, errX)
			goto fail
		}

		client, err := upstream.New(privateKey, publicKey, connectorTimeout)
		if nil != err {
			log.Errorf("client[%d]=%q  error: %s", i, address, err)
			errX = err
			goto fail
		}

		conn.staticClients[i] = client
		globalData.connectorClients = append(globalData.connectorClients, client)

		err = client.Connect(address, serverPublicKey)
		if nil != err {
			log.Errorf("connect[%d]=%q  error: %s", i, address, err)
			errX = err
			goto fail
		}
		log.Infof("public key: %x  at: %q", serverPublicKey, c.Address)
	}

	// just create sockets for dynamic clients
	for i := 0; i < maximumDynamicClients; i += 1 {
		client, err := upstream.New(privateKey, publicKey, connectorTimeout)
		if nil != err {
			log.Errorf("client[%d]  error: %s", i, err)
			errX = err
			goto fail
		}

		// create list of all dynamic clients
		conn.dynamicClients.PushBack(client)

		globalData.connectorClients = append(globalData.connectorClients, client)
	}

	// start state machine
	conn.state = cStateConnecting

	return nil

	// error handling
fail:
	conn.destroy()

	return errX
}

func (conn *connector) allClients(f func(client *upstream.Upstream, e *list.Element)) {
	for _, client := range conn.staticClients {
		f(client, nil)
	}
	for e := conn.dynamicClients.Front(); nil != e; e = e.Next() {
		f(e.Value.(*upstream.Upstream), e)
	}
}

func (conn *connector) searchClients(f func(client *upstream.Upstream, e *list.Element) bool) {
	for _, client := range conn.staticClients {
		if f(client, nil) {
			return
		}
	}
	for e := conn.dynamicClients.Front(); nil != e; e = e.Next() {
		if f(e.Value.(*upstream.Upstream), e) {
			return
		}
	}
}

func (conn *connector) destroy() {
	conn.allClients(func(client *upstream.Upstream, e *list.Element) {
		client.Destroy()
	})
}

// various RPC calls to upstream connections
func (conn *connector) Run(args interface{}, shutdown <-chan struct{}) {

	log := conn.log

	log.Info("starting…")

	queue := messagebus.Bus.Connector.Chan()

loop:
	for {
		// wait for shutdown
		log.Info("waiting…")

		select {
		case <-shutdown:
			break loop
		case item := <-queue:
			conn.log.Infof("received control: %s  public key: %x  connect: %x", item.Command, item.Parameters[0], item.Parameters[1])
			//connectToUpstream(conn.log, conn.clients, conn.dynamicStart, item.Command, item.Parameters[0], item.Parameters[1])
			conn.connectUpstream(item.Command, item.Parameters[0], item.Parameters[1])

		case <-time.After(cycleInterval):
			conn.process()
		}
	}
	log.Info("shutting down…")
	conn.destroy()
	log.Info("stopped")
}

// process the connect and return response
func (conn *connector) process() {
	// run the machine until it pauses
	for conn.runStateMachine() {
	}
}

// run state machine
// return:
//   true  if want more cycles
//   false to pase for I/O
func (conn *connector) runStateMachine() bool {
	log := conn.log

	log.Infof("current state: %s", conn.state)

	continueLooping := true

	switch conn.state {
	case cStateConnecting:
		mode.Set(mode.Resynchronise)
		clientCount := 0

		conn.allClients(func(client *upstream.Upstream, e *list.Element) {
			if client.IsOK() {

				clientCount += 1
			}
		})

		log.Infof("connections: %d", clientCount)
		globalData.clientCount = clientCount
		if clientCount >= minimumClients {
			conn.state += 1
		} else {
			log.Warnf("can not reach the minimum client counts")
			messagebus.Bus.Announce.Send("reconnect")
		}
		continueLooping = false

	case cStateHighestBlock:
		conn.height, conn.theClient = getHeight(conn)
		if conn.height > 0 && nil != conn.theClient {
			conn.state += 1
		} else {
			continueLooping = false
		}
		log.Infof("highest block number: %d", conn.height)

	case cStateForkDetect:
		height := block.GetHeight()
		if conn.height <= height {
			conn.state = cStateRebuild
		} else {
			// first block number
			conn.startBlockNumber = genesis.BlockNumber + 1
			conn.state += 1 // assume success
			log.Infof("block number: %d", height)

			// check digests of descending blocks (to detect a fork)
		check_digests:
			for h := height; h > genesis.BlockNumber; h -= 1 {
				digest, err := block.DigestForBlock(h)
				if nil != err {
					log.Infof("block number: %d  local digest error: %s", h, err)
					conn.state = cStateHighestBlock // retry
					break check_digests
				}
				d, err := conn.theClient.GetBlockDigest(h)
				if nil != err {
					log.Infof("block number: %d  fetch digest error: %s", h, err)
					conn.state = cStateHighestBlock // retry
					break check_digests
				} else if d == digest {
					if height-h >= forkProtection {
						log.Errorf("fork protection at: %d - %d >= %d", height, h, forkProtection)
						conn.state = cStateHighestBlock
						break check_digests
					}
					conn.startBlockNumber = h + 1
					log.Infof("fork from block number: %d", conn.startBlockNumber)

					// remove old blocks
					err := block.DeleteDownToBlock(conn.startBlockNumber)
					if nil != err {
						log.Errorf("delete down to block number: %d  error: %s", conn.startBlockNumber, err)
						conn.state = cStateHighestBlock // retry
					}
					break check_digests
				}
			}
		}

	case cStateFetchBlocks:

		continueLooping = false

	fetch_blocks:
		for n := 0; n < fetchBlocksPerCycle; n += 1 {

			if conn.startBlockNumber > conn.height {
				conn.state = cStateHighestBlock // just in case block height has changed
				continueLooping = true
				break fetch_blocks
			}

			log.Infof("fetch block number: %d", conn.startBlockNumber)
			packedBlock, err := conn.theClient.GetBlockData(conn.startBlockNumber)
			if nil != err {
				log.Errorf("fetch block number: %d  error: %s", conn.startBlockNumber, err)
				conn.state = cStateHighestBlock // retry
				break fetch_blocks
			}
			log.Debugf("store block number: %d", conn.startBlockNumber)
			err = block.StoreIncoming(packedBlock)
			if nil != err {
				log.Errorf("store block number: %d  error: %s", conn.startBlockNumber, err)
				conn.state = cStateHighestBlock // retry
				break fetch_blocks
			}

			// next block
			conn.startBlockNumber += 1

		}

	case cStateRebuild:
		// return to normal operations
		conn.state += 1  // next state
		conn.samples = 0 // zero out the counter
		mode.Set(mode.Normal)
		continueLooping = false

	case cStateSampling:
		// check peers
		conn.height, conn.theClient = getHeight(conn)
		height := block.GetHeight()

		log.Infof("height  remote: %d  local: %d", conn.height, height)

		continueLooping = false

		if conn.height > height {
			if conn.height-height >= 2 {
				conn.state = cStateForkDetect
				continueLooping = true
			} else {
				conn.samples += 1
				if conn.samples > samplelingLimit {
					conn.state = cStateForkDetect
					continueLooping = true
				}
			}
		}
	}
	return continueLooping
}

func getHeight(conn *connector) (height uint64, theClient *upstream.Upstream) {
	theClient = nil
	height = 0

	conn.allClients(func(client *upstream.Upstream, e *list.Element) {
		h := client.GetHeight()
		if h > height {
			height = h
			theClient = client
		}
	})

	globalData.blockHeight = height
	return height, theClient
}

func (state connectorState) String() string {
	switch state {
	case cStateConnecting:
		return "Connecting"
	case cStateHighestBlock:
		return "HighestBlock"
	case cStateForkDetect:
		return "ForkDetect"
	case cStateFetchBlocks:
		return "FetchBlocks"
	case cStateRebuild:
		return "Rebuild"
	case cStateSampling:
		return "Sampling"
	default:
		return "*Unknown*"
	}
}

func (conn *connector) connectUpstream(priority string, serverPublicKey []byte, addresses []byte) error {

	log := conn.log

	log.Infof("connect: %s to: %x @ %x", priority, serverPublicKey, addresses)

	// extract the first valid address
	var address *util.Connection

extract_addresses:
	for {
		conn, n := util.PackedConnection(addresses).Unpack()
		addresses = addresses[n:]

		// ***** FIX THIS: could select for IPv4 or IPv6 here
		// ***** FIX THIS: need to get preference e.g. if have IPv6 the prefer IPv6
		if nil != conn {
			address = conn
			break extract_addresses
		}
		if n <= 0 {
			break extract_addresses
		}
		log.Errorf("reconnect: %x (conn: %x)  error: address is nil", serverPublicKey, conn)
	}

	if nil == address {
		log.Errorf("reconnect: %s  error: no addresses found", serverPublicKey)
		return fault.ErrAddressIsNil
	}

	// see if already connected to this node
	alreadyConnected := false
	conn.searchClients(func(client *upstream.Upstream, e *list.Element) bool {
		if client.IsConnectedTo(serverPublicKey) {
			if nil == e {
				log.Debugf("already have static connection to: %x @ %s", serverPublicKey, *address)
			} else {
				log.Debugf("ignore change to: %x @ %s", serverPublicKey, *address)
				conn.dynamicClients.MoveToBack(e)
			}
			alreadyConnected = true
			return true
		}
		return false
	})

	if alreadyConnected {
		return nil
	}

	// reconnect the oldest entry to new node
	log.Infof("reconnect: %x @ %s", serverPublicKey, *address)
	client := conn.dynamicClients.Front().Value.(*upstream.Upstream)
	err := client.Connect(address, serverPublicKey)
	if nil != err {
		log.Errorf("ConnectTo: %x @ %s  error: %s", serverPublicKey, *address, err)
	} else {
		conn.dynamicClients.MoveToBack(conn.dynamicClients.Front())
	}

	return err
}
<commit_msg>[peer] fix incorrect string format<commit_after>// Copyright (c) 2014-2018 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package peer

import (
	"bytes"
	"container/list"
	"encoding/hex"
	"github.com/bitmark-inc/bitmarkd/block"
	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/bitmarkd/genesis"
	"github.com/bitmark-inc/bitmarkd/messagebus"
	"github.com/bitmark-inc/bitmarkd/mode"
	"github.com/bitmark-inc/bitmarkd/peer/upstream"
	"github.com/bitmark-inc/bitmarkd/util"
	"github.com/bitmark-inc/logger"
	"time"
)

// various timeouts
const (
	cycleInterval         = 15 * time.Second // pause to limit bandwidth
	connectorTimeout      = 60 * time.Second // time out for connections
	samplelingLimit       = 10               // number of cycles to be 1 block out of sync before resync
	fetchBlocksPerCycle   = 200              // number of blocks to fetch in one set
	forkProtection        = 60               // fail to fork if height difference is greater than this
	minimumClients        = 3                // do not proceed unless this many clients are connected
	maximumDynamicClients = 10               // total number of dynamic clients
)

// a state type for the thread
type connectorState int

// state of the connector process
const (
	cStateConnecting   connectorState = iota // register to nodes and make outgoing connections
	cStateHighestBlock connectorState = iota // locate node(s) with highest block number
	cStateForkDetect   connectorState = iota // read block hashes to check for possible fork
	cStateFetchBlocks  connectorState = iota // fetch blocks from current or fork point
	cStateRebuild      connectorState = iota // rebuild database from fork point (config setting to force total rebuild)
	cStateSampling     connectorState = iota // signal resync complete and sample nodes to see if out of sync occurs
)

type connector struct {
	log *logger.L

	staticClients []*upstream.Upstream

	dynamicClients list.List

	state connectorState

	theClient        *upstream.Upstream // client used for fetching blocks
	startBlockNumber uint64             // block number where local chain forks
	height           uint64             // block number on best node
	samples          int                // counter to detect missed block broadcast
}

// initialise the connector
func (conn *connector) initialise(privateKey []byte, publicKey []byte, connect []Connection, dynamicEnabled bool) error {

	log := logger.New("connector")
	conn.log = log

	log.Info("initialising…")

	// allocate all sockets
	staticCount := len(connect) // can be zero
	if 0 == staticCount && !dynamicEnabled {
		log.Error("zero static connections and dynamic is disabled")
		return fault.ErrNoConnectionsAvailable
	}
	conn.staticClients = make([]*upstream.Upstream, staticCount)

	// error code for goto fail
	errX := error(nil)

	// initially connect all static sockets
	for i, c := range connect {
		address, err := util.NewConnection(c.Address)
		if nil != err {
			log.Errorf("client[%d]=address: %q  error: %s", i, c.Address, err)
			errX = err
			goto fail
		}
		serverPublicKey, err := hex.DecodeString(c.PublicKey)
		if nil != err {
			log.Errorf("client[%d]=public: %q  error: %s", i, c.PublicKey, err)
			errX = err
			goto fail
		}

		// prevent connection to self
		if bytes.Equal(publicKey, serverPublicKey) {
			errX = fault.ErrConnectingToSelfForbidden
			log.Errorf("client[%d]=public: %q  error: %s", i, c.PublicKey, errX)
			goto fail
		}

		client, err := upstream.New(privateKey, publicKey, connectorTimeout)
		if nil != err {
			log.Errorf("client[%d]=%q  error: %s", i, address, err)
			errX = err
			goto fail
		}

		conn.staticClients[i] = client
		globalData.connectorClients = append(globalData.connectorClients, client)

		err = client.Connect(address, serverPublicKey)
		if nil != err {
			log.Errorf("connect[%d]=%q  error: %s", i, address, err)
			errX = err
			goto fail
		}
		log.Infof("public key: %x  at: %q", serverPublicKey, c.Address)
	}

	// just create sockets for dynamic clients
	for i := 0; i < maximumDynamicClients; i += 1 {
		client, err := upstream.New(privateKey, publicKey, connectorTimeout)
		if nil != err {
			log.Errorf("client[%d]  error: %s", i, err)
			errX = err
			goto fail
		}

		// create list of all dynamic clients
		conn.dynamicClients.PushBack(client)

		globalData.connectorClients = append(globalData.connectorClients, client)
	}

	// start state machine
	conn.state = cStateConnecting

	return nil

	// error handling
fail:
	conn.destroy()

	return errX
}

func (conn *connector) allClients(f func(client *upstream.Upstream, e *list.Element)) {
	for _, client := range conn.staticClients {
		f(client, nil)
	}
	for e := conn.dynamicClients.Front(); nil != e; e = e.Next() {
		f(e.Value.(*upstream.Upstream), e)
	}
}

func (conn *connector) searchClients(f func(client *upstream.Upstream, e *list.Element) bool) {
	for _, client := range conn.staticClients {
		if f(client, nil) {
			return
		}
	}
	for e := conn.dynamicClients.Front(); nil != e; e = e.Next() {
		if f(e.Value.(*upstream.Upstream), e) {
			return
		}
	}
}

func (conn *connector) destroy() {
	conn.allClients(func(client *upstream.Upstream, e *list.Element) {
		client.Destroy()
	})
}

// various RPC calls to upstream connections
func (conn *connector) Run(args interface{}, shutdown <-chan struct{}) {

	log := conn.log

	log.Info("starting…")

	queue := messagebus.Bus.Connector.Chan()

loop:
	for {
		// wait for shutdown
		log.Info("waiting…")

		select {
		case <-shutdown:
			break loop
		case item := <-queue:
			conn.log.Infof("received control: %s  public key: %x  connect: %x", item.Command, item.Parameters[0], item.Parameters[1])
			//connectToUpstream(conn.log, conn.clients, conn.dynamicStart, item.Command, item.Parameters[0], item.Parameters[1])
			conn.connectUpstream(item.Command, item.Parameters[0], item.Parameters[1])

		case <-time.After(cycleInterval):
			conn.process()
		}
	}
	log.Info("shutting down…")
	conn.destroy()
	log.Info("stopped")
}

// process the connect and return response
func (conn *connector) process() {
	// run the machine until it pauses
	for conn.runStateMachine() {
	}
}

// run state machine
// return:
//   true  if want more cycles
//   false to pase for I/O
func (conn *connector) runStateMachine() bool {
	log := conn.log

	log.Infof("current state: %s", conn.state)

	continueLooping := true

	switch conn.state {
	case cStateConnecting:
		mode.Set(mode.Resynchronise)
		clientCount := 0

		conn.allClients(func(client *upstream.Upstream, e *list.Element) {
			if client.IsOK() {

				clientCount += 1
			}
		})

		log.Infof("connections: %d", clientCount)
		globalData.clientCount = clientCount
		if clientCount >= minimumClients {
			conn.state += 1
		} else {
			log.Warnf("can not reach the minimum client counts")
			messagebus.Bus.Announce.Send("reconnect")
		}
		continueLooping = false

	case cStateHighestBlock:
		conn.height, conn.theClient = getHeight(conn)
		if conn.height > 0 && nil != conn.theClient {
			conn.state += 1
		} else {
			continueLooping = false
		}
		log.Infof("highest block number: %d", conn.height)

	case cStateForkDetect:
		height := block.GetHeight()
		if conn.height <= height {
			conn.state = cStateRebuild
		} else {
			// first block number
			conn.startBlockNumber = genesis.BlockNumber + 1
			conn.state += 1 // assume success
			log.Infof("block number: %d", height)

			// check digests of descending blocks (to detect a fork)
		check_digests:
			for h := height; h > genesis.BlockNumber; h -= 1 {
				digest, err := block.DigestForBlock(h)
				if nil != err {
					log.Infof("block number: %d  local digest error: %s", h, err)
					conn.state = cStateHighestBlock // retry
					break check_digests
				}
				d, err := conn.theClient.GetBlockDigest(h)
				if nil != err {
					log.Infof("block number: %d  fetch digest error: %s", h, err)
					conn.state = cStateHighestBlock // retry
					break check_digests
				} else if d == digest {
					if height-h >= forkProtection {
						log.Errorf("fork protection at: %d - %d >= %d", height, h, forkProtection)
						conn.state = cStateHighestBlock
						break check_digests
					}
					conn.startBlockNumber = h + 1
					log.Infof("fork from block number: %d", conn.startBlockNumber)

					// remove old blocks
					err := block.DeleteDownToBlock(conn.startBlockNumber)
					if nil != err {
						log.Errorf("delete down to block number: %d  error: %s", conn.startBlockNumber, err)
						conn.state = cStateHighestBlock // retry
					}
					break check_digests
				}
			}
		}

	case cStateFetchBlocks:

		continueLooping = false

	fetch_blocks:
		for n := 0; n < fetchBlocksPerCycle; n += 1 {

			if conn.startBlockNumber > conn.height {
				conn.state = cStateHighestBlock // just in case block height has changed
				continueLooping = true
				break fetch_blocks
			}

			log.Infof("fetch block number: %d", conn.startBlockNumber)
			packedBlock, err := conn.theClient.GetBlockData(conn.startBlockNumber)
			if nil != err {
				log.Errorf("fetch block number: %d  error: %s", conn.startBlockNumber, err)
				conn.state = cStateHighestBlock // retry
				break fetch_blocks
			}
			log.Debugf("store block number: %d", conn.startBlockNumber)
			err = block.StoreIncoming(packedBlock)
			if nil != err {
				log.Errorf("store block number: %d  error: %s", conn.startBlockNumber, err)
				conn.state = cStateHighestBlock // retry
				break fetch_blocks
			}

			// next block
			conn.startBlockNumber += 1

		}

	case cStateRebuild:
		// return to normal operations
		conn.state += 1  // next state
		conn.samples = 0 // zero out the counter
		mode.Set(mode.Normal)
		continueLooping = false

	case cStateSampling:
		// check peers
		conn.height, conn.theClient = getHeight(conn)
		height := block.GetHeight()

		log.Infof("height  remote: %d  local: %d", conn.height, height)

		continueLooping = false

		if conn.height > height {
			if conn.height-height >= 2 {
				conn.state = cStateForkDetect
				continueLooping = true
			} else {
				conn.samples += 1
				if conn.samples > samplelingLimit {
					conn.state = cStateForkDetect
					continueLooping = true
				}
			}
		}
	}
	return continueLooping
}

func getHeight(conn *connector) (height uint64, theClient *upstream.Upstream) {
	theClient = nil
	height = 0

	conn.allClients(func(client *upstream.Upstream, e *list.Element) {
		h := client.GetHeight()
		if h > height {
			height = h
			theClient = client
		}
	})

	globalData.blockHeight = height
	return height, theClient
}

func (state connectorState) String() string {
	switch state {
	case cStateConnecting:
		return "Connecting"
	case cStateHighestBlock:
		return "HighestBlock"
	case cStateForkDetect:
		return "ForkDetect"
	case cStateFetchBlocks:
		return "FetchBlocks"
	case cStateRebuild:
		return "Rebuild"
	case cStateSampling:
		return "Sampling"
	default:
		return "*Unknown*"
	}
}

func (conn *connector) connectUpstream(priority string, serverPublicKey []byte, addresses []byte) error {

	log := conn.log

	log.Infof("connect: %s to: %x @ %x", priority, serverPublicKey, addresses)

	// extract the first valid address
	var address *util.Connection

extract_addresses:
	for {
		conn, n := util.PackedConnection(addresses).Unpack()
		addresses = addresses[n:]

		// ***** FIX THIS: could select for IPv4 or IPv6 here
		// ***** FIX THIS: need to get preference e.g. if have IPv6 the prefer IPv6
		if nil != conn {
			address = conn
			break extract_addresses
		}
		if n <= 0 {
			break extract_addresses
		}
		log.Errorf("reconnect: %x (conn: %x)  error: address is nil", serverPublicKey, conn)
	}

	if nil == address {
		log.Errorf("reconnect: %x  error: no addresses found", serverPublicKey)
		return fault.ErrAddressIsNil
	}

	// see if already connected to this node
	alreadyConnected := false
	conn.searchClients(func(client *upstream.Upstream, e *list.Element) bool {
		if client.IsConnectedTo(serverPublicKey) {
			if nil == e {
				log.Debugf("already have static connection to: %x @ %s", serverPublicKey, *address)
			} else {
				log.Debugf("ignore change to: %x @ %s", serverPublicKey, *address)
				conn.dynamicClients.MoveToBack(e)
			}
			alreadyConnected = true
			return true
		}
		return false
	})

	if alreadyConnected {
		return nil
	}

	// reconnect the oldest entry to new node
	log.Infof("reconnect: %x @ %s", serverPublicKey, *address)
	client := conn.dynamicClients.Front().Value.(*upstream.Upstream)
	err := client.Connect(address, serverPublicKey)
	if nil != err {
		log.Errorf("ConnectTo: %x @ %s  error: %s", serverPublicKey, *address, err)
	} else {
		conn.dynamicClients.MoveToBack(conn.dynamicClients.Front())
	}

	return err
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/spf13/cobra"
	"github.com/voidint/gbb/config"
	"github.com/voidint/gbb/tool"
	"github.com/voidint/gbb/util"
)

const (
	// DefaultConfFile default configuration file path
	DefaultConfFile = "gbb.json"
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use: "gbb",
	Long: `Compile assistant.
Copyright (c) 2016, 2018, voidint. All rights reserved.`,

	Run: func(cmd *cobra.Command, args []string) {
		if gopts.ConfigFile == DefaultConfFile {
			gopts.ConfigFile = filepath.Join(wd, "gbb.json")
		}

		if !util.FileExist(gopts.ConfigFile) {
			genConfigFile(gopts.ConfigFile)
			return
		}
		conf, err := config.Load(gopts.ConfigFile)
		if err != nil {
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(-1)
			return
		}
		conf.Debug = gopts.Debug
		conf.All = gopts.All

		if conf.Version != Version {
			gt, err := util.VersionGreaterThan(Version, conf.Version)
			if err != nil {
				fmt.Fprintln(os.Stderr, err.Error())
				os.Exit(-1)
				return
			}

			if gt { // 程序版本大于配置文件版本，重新生成配置文件。
				fmt.Printf("Warning: The gbb.json file needs to be upgraded.\n\n")
				genConfigFile(gopts.ConfigFile)
			} else {
				// 配置文件版本大于程序版本，提醒用户升级程序。
				fmt.Printf("Warning: This program needs to be upgraded by `go get -u -v github.com/voidint/gbb`\n\n")
			}
			return
		}

		if err := tool.Build(conf, wd); err != nil {
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(-1)
			return
		}
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
		return
	}
}

// GlobalOptions global options
type GlobalOptions struct {
	All        bool
	Debug      bool
	ConfigFile string
}

var (
	wd    string // current work directory
	gopts GlobalOptions
)

func init() {
	cobra.OnInitialize(initConfig)
	RootCmd.PersistentFlags().BoolVarP(&gopts.All, "all", "a", false, "build all packages")
	RootCmd.PersistentFlags().BoolVarP(&gopts.Debug, "debug", "D", false, "enable debug mode")
	RootCmd.PersistentFlags().StringVarP(&gopts.ConfigFile, "config", "c", DefaultConfFile, "configuration file")
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	var err error
	wd, err = os.Getwd()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
		return
	}
}
<commit_msg>Modify help information.<commit_after>package cmd

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/spf13/cobra"
	"github.com/voidint/gbb/config"
	"github.com/voidint/gbb/tool"
	"github.com/voidint/gbb/util"
)

const (
	// DefaultConfFile default configuration file path
	DefaultConfFile = "gbb.json"
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use: "gbb",
	Long: `Go project compilation assistant.
Copyright (c) 2016, 2018, voidint. All rights reserved.`,

	Run: func(cmd *cobra.Command, args []string) {
		if gopts.ConfigFile == DefaultConfFile {
			gopts.ConfigFile = filepath.Join(wd, "gbb.json")
		}

		if !util.FileExist(gopts.ConfigFile) {
			genConfigFile(gopts.ConfigFile)
			return
		}
		conf, err := config.Load(gopts.ConfigFile)
		if err != nil {
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(-1)
			return
		}
		conf.Debug = gopts.Debug
		conf.All = gopts.All

		if conf.Version != Version {
			gt, err := util.VersionGreaterThan(Version, conf.Version)
			if err != nil {
				fmt.Fprintln(os.Stderr, err.Error())
				os.Exit(-1)
				return
			}

			if gt { // 程序版本大于配置文件版本，重新生成配置文件。
				fmt.Printf("Warning: The gbb.json file needs to be upgraded.\n\n")
				genConfigFile(gopts.ConfigFile)
			} else {
				// 配置文件版本大于程序版本，提醒用户升级程序。
				fmt.Printf("Warning: This program needs to be upgraded by `go get -u -v github.com/voidint/gbb`\n\n")
			}
			return
		}

		if err := tool.Build(conf, wd); err != nil {
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(-1)
			return
		}
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
		return
	}
}

// GlobalOptions global options
type GlobalOptions struct {
	All        bool
	Debug      bool
	ConfigFile string
}

var (
	wd    string // current work directory
	gopts GlobalOptions
)

func init() {
	cobra.OnInitialize(initConfig)
	RootCmd.PersistentFlags().BoolVarP(&gopts.All, "all", "a", false, "build all packages")
	RootCmd.PersistentFlags().BoolVarP(&gopts.Debug, "debug", "D", false, "enable debug mode")
	RootCmd.PersistentFlags().StringVarP(&gopts.ConfigFile, "config", "c", DefaultConfFile, "configuration file")
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	var err error
	wd, err = os.Getwd()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
		return
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"runtime"
	"sync"
	"syscall"

	"golang.org/x/net/context"

	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/internal/buffer"
	"github.com/jacobsa/fuse/internal/fusekernel"
)

// Ask the Linux kernel for larger read requests.
//
// As of 2015-03-26, the behavior in the kernel is:
//
//  *  (http://goo.gl/bQ1f1i, http://goo.gl/HwBrR6) Set the local variable
//     ra_pages to be init_response->max_readahead divided by the page size.
//
//  *  (http://goo.gl/gcIsSh, http://goo.gl/LKV2vA) Set
//     backing_dev_info::ra_pages to the min of that value and what was sent
//     in the request's max_readahead field.
//
//  *  (http://goo.gl/u2SqzH) Use backing_dev_info::ra_pages when deciding
//     how much to read ahead.
//
//  *  (http://goo.gl/JnhbdL) Don't read ahead at all if that field is zero.
//
// Reading a page at a time is a drag. Ask for a larger size.
const maxReadahead = 1 << 20

// A connection to the fuse kernel process.
type Connection struct {
	debugLogger *log.Logger
	errorLogger *log.Logger

	// The device through which we're talking to the kernel, and the protocol
	// version that we're using to talk to it.
	dev      *os.File
	protocol fusekernel.Protocol

	// The context from which all op contexts inherit.
	parentCtx context.Context

	// For logging purposes only.
	nextOpID uint32

	mu sync.Mutex

	// A map from fuse "unique" request ID (*not* the op ID for logging used
	// above) to a function that cancel's its associated context.
	//
	// GUARDED_BY(mu)
	cancelFuncs map[uint64]func()
}

// Create a connection wrapping the supplied file descriptor connected to the
// kernel. You must eventually call c.close().
//
// The loggers may be nil.
func newConnection(
	parentCtx context.Context,
	debugLogger *log.Logger,
	errorLogger *log.Logger,
	dev *os.File) (c *Connection, err error) {
	c = &Connection{
		debugLogger: debugLogger,
		errorLogger: errorLogger,
		dev:         dev,
		parentCtx:   parentCtx,
		cancelFuncs: make(map[uint64]func()),
	}

	// Initialize.
	err = c.Init()
	if err != nil {
		c.close()
		err = fmt.Errorf("Init: %v", err)
		return
	}

	return
}

// Do the work necessary to cause the mount process to complete.
func (c *Connection) Init() (err error) {
	// Read the init op.
	op, err := c.ReadOp()
	if err != nil {
		err = fmt.Errorf("Reading init op: %v", err)
		return
	}

	initOp, ok := op.(*fuseops.InternalInitOp)
	if !ok {
		err = fmt.Errorf("Expected *fuseops.InternalInitOp, got %T", op)
		return
	}

	// Make sure the protocol version spoken by the kernel is new enough.
	min := fusekernel.Protocol{
		fusekernel.ProtoVersionMinMajor,
		fusekernel.ProtoVersionMinMinor,
	}

	if initOp.Kernel.LT(min) {
		initOp.Respond(syscall.EPROTO)
		err = fmt.Errorf("Version too old: %v", initOp.Kernel)
		return
	}

	// Downgrade our protocol if necessary.
	c.protocol = fusekernel.Protocol{
		fusekernel.ProtoVersionMaxMajor,
		fusekernel.ProtoVersionMaxMinor,
	}

	if initOp.Kernel.LT(c.protocol) {
		c.protocol = initOp.Kernel
	}

	// Respond to the init op.
	initOp.Library = c.protocol
	initOp.MaxReadahead = maxReadahead
	initOp.MaxWrite = buffer.MaxWriteSize
	initOp.Flags = fusekernel.InitBigWrites
	initOp.Respond(nil)

	return
}

// Log information for an operation with the given ID. calldepth is the depth
// to use when recovering file:line information with runtime.Caller.
func (c *Connection) debugLog(
	opID uint32,
	calldepth int,
	format string,
	v ...interface{}) {
	if c.debugLogger == nil {
		return
	}

	// Get file:line info.
	var file string
	var line int
	var ok bool

	_, file, line, ok = runtime.Caller(calldepth)
	if !ok {
		file = "???"
	}

	fileLine := fmt.Sprintf("%v:%v", path.Base(file), line)

	// Format the actual message to be printed.
	msg := fmt.Sprintf(
		"Op 0x%08x %24s] %v",
		opID,
		fileLine,
		fmt.Sprintf(format, v...))

	// Print it.
	c.debugLogger.Println(msg)
}

// LOCKS_EXCLUDED(c.mu)
func (c *Connection) recordCancelFunc(
	fuseID uint64,
	f func()) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if _, ok := c.cancelFuncs[fuseID]; ok {
		panic(fmt.Sprintf("Already have cancel func for request %v", fuseID))
	}

	c.cancelFuncs[fuseID] = f
}

// Set up state for an op that is about to be returned to the user, given its
// underlying fuse opcode and request ID.
//
// Return a context that should be used for the op.
//
// LOCKS_EXCLUDED(c.mu)
func (c *Connection) beginOp(
	opCode uint32,
	fuseID uint64) (ctx context.Context) {
	// Start with the parent context.
	ctx = c.parentCtx

	// Set up a cancellation function.
	//
	// Special case: On Darwin, osxfuse aggressively reuses "unique" request IDs.
	// This matters for Forget requests, which have no reply associated and
	// therefore have IDs that are immediately eligible for reuse. For these, we
	// should not record any state keyed on their ID.
	//
	// Cf. https://github.com/osxfuse/osxfuse/issues/208
	if opCode != fusekernel.OpForget {
		var cancel func()
		ctx, cancel = context.WithCancel(ctx)
		c.recordCancelFunc(fuseID, cancel)
	}

	return
}

// Clean up all state associated with an op to which the user has responded,
// given its underlying fuse opcode and request ID. This must be called before
// a response is sent to the kernel, to avoid a race where the request's ID
// might be reused by osxfuse.
//
// LOCKS_EXCLUDED(c.mu)
func (c *Connection) finishOp(
	opCode uint32,
	fuseID uint64) {
	c.mu.Lock()
	defer c.mu.Unlock()

	// Even though the op is finished, context.WithCancel requires us to arrange
	// for the cancellation function to be invoked. We also must remove it from
	// our map.
	//
	// Special case: we don't do this for Forget requests. See the note in
	// beginOp above.
	if opCode != fusekernel.OpForget {
		cancel, ok := c.cancelFuncs[fuseID]
		if !ok {
			panic(fmt.Sprintf("Unknown request ID in finishOp: %v", fuseID))
		}

		cancel()
		delete(c.cancelFuncs, fuseID)
	}
}

// LOCKS_EXCLUDED(c.mu)
func (c *Connection) handleInterrupt(fuseID uint64) {
	c.mu.Lock()
	defer c.mu.Unlock()

	// NOTE(jacobsa): fuse.txt in the Linux kernel documentation
	// (https://goo.gl/H55Dnr) defines the kernel <-> userspace protocol for
	// interrupts.
	//
	// In particular, my reading of it is that an interrupt request cannot be
	// delivered to userspace before the original request. The part about the
	// race and EAGAIN appears to be aimed at userspace programs that
	// concurrently process requests (cf. http://goo.gl/BES2rs).
	//
	// So in this method if we can't find the ID to be interrupted, it means that
	// the request has already been replied to.
	//
	// Cf. https://github.com/osxfuse/osxfuse/issues/208
	// Cf. http://comments.gmane.org/gmane.comp.file-systems.fuse.devel/14675
	cancel, ok := c.cancelFuncs[fuseID]
	if !ok {
		return
	}

	cancel()
}

func (c *Connection) allocateInMessage() (m *buffer.InMessage) {
	// TODO(jacobsa): Use a freelist.
	m = new(buffer.InMessage)
	return
}

func (c *Connection) destroyInMessage(m *buffer.InMessage) {
	// TODO(jacobsa): Use a freelist.
}

// Read the next message from the kernel. The message must later be destroyed
// using destroyInMessage.
func (c *Connection) readMessage() (m *buffer.InMessage, err error) {
	// Allocate a message.
	m = c.allocateInMessage()

	// Loop past transient errors.
	for {
		// Attempt a reaed.
		err = m.Init(c.dev)

		// Special cases:
		//
		//  *  ENODEV means fuse has hung up.
		//
		//  *  EINTR means we should try again. (This seems to happen often on
		//     OS X, cf. http://golang.org/issue/11180)
		//
		if pe, ok := err.(*os.PathError); ok {
			switch pe.Err {
			case syscall.ENODEV:
				err = io.EOF

			case syscall.EINTR:
				err = nil
				continue
			}
		}

		if err != nil {
			c.destroyInMessage(m)
			m = nil
			return
		}

		return
	}
}

// Write the supplied message to the kernel.
func (c *Connection) writeMessage(msg []byte) (err error) {
	// Avoid the retry loop in os.File.Write.
	n, err := syscall.Write(int(c.dev.Fd()), msg)
	if err != nil {
		return
	}

	if n != len(msg) {
		err = fmt.Errorf("Wrote %d bytes; expected %d", n, len(msg))
		return
	}

	return
}

// Read the next op from the kernel process. Return io.EOF if the kernel has
// closed the connection.
//
// This function delivers ops in exactly the order they are received from
// /dev/fuse. It must not be called multiple times concurrently.
//
// LOCKS_EXCLUDED(c.mu)
func (c *Connection) ReadOp() (op fuseops.Op, err error) {
	// Keep going until we find a request we know how to convert.
	for {
		// Read the next message from the kernel.
		var m *buffer.InMessage
		m, err = c.readMessage()
		if err != nil {
			return
		}

		// Choose an ID for this operation for the purposes of logging.
		opID := c.nextOpID
		c.nextOpID++

		// Set up op dependencies.
		opCtx := c.beginOp(m.Header().Opcode, m.Header().Unique)

		var debugLogForOp func(int, string, ...interface{})
		if c.debugLogger != nil {
			debugLogForOp = func(calldepth int, format string, v ...interface{}) {
				c.debugLog(opID, calldepth+1, format, v...)
			}
		}

		sendReply := func(
			op fuseops.Op,
			fuseID uint64,
			replyMsg []byte,
			opErr error) (err error) {
			// Make sure we destroy the message, as required by readMessage.
			defer c.destroyInMessage(m)

			// Clean up state for this op.
			c.finishOp(m.Header().Opcode, m.Header().Unique)

			// Debug logging
			if c.debugLogger != nil {
				if opErr == nil {
					op.Logf("-> OK: %s", op.DebugString())
				} else {
					op.Logf("-> error: %v", opErr)
				}
			}

			// Error logging
			if opErr != nil && c.errorLogger != nil {
				c.errorLogger.Printf("(%s) error: %v", op.ShortDesc(), opErr)
			}

			// Send the reply to the kernel.
			err = c.writeMessage(replyMsg)
			if err != nil {
				err = fmt.Errorf("writeMessage: %v", err)
				return
			}

			return
		}

		// Convert the message to an Op.
		op, err = fuseops.Convert(
			opCtx,
			m,
			c.protocol,
			debugLogForOp,
			c.errorLogger,
			sendReply)

		if err != nil {
			err = fmt.Errorf("fuseops.Convert: %v", err)
			return
		}

		// Log the receipt of the operation.
		c.debugLog(opID, 1, "<- %v", op.ShortDesc())

		// Special case: responding to statfs is required to make mounting work on
		// OS X. We don't currently expose the capability for the file system to
		// intercept this.
		if _, ok := op.(*fuseops.InternalStatFSOp); ok {
			op.Respond(nil)
			continue
		}

		// Special case: handle interrupt requests.
		if interruptOp, ok := op.(*fuseops.InternalInterruptOp); ok {
			c.handleInterrupt(interruptOp.FuseID)
			continue
		}

		return
	}
}

// Close the connection. Must not be called until operations that were read
// from the connection have been responded to.
func (c *Connection) close() (err error) {
	// Posix doesn't say that close can be called concurrently with read or
	// write, but luckily we exclude the possibility of a race by requiring the
	// user to respond to all ops first.
	err = c.dev.Close()
	return
}
<commit_msg>Use a freelist for buffer.InMessage structs.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"runtime"
	"sync"
	"syscall"

	"golang.org/x/net/context"

	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/internal/buffer"
	"github.com/jacobsa/fuse/internal/fusekernel"
)

// Ask the Linux kernel for larger read requests.
//
// As of 2015-03-26, the behavior in the kernel is:
//
//  *  (http://goo.gl/bQ1f1i, http://goo.gl/HwBrR6) Set the local variable
//     ra_pages to be init_response->max_readahead divided by the page size.
//
//  *  (http://goo.gl/gcIsSh, http://goo.gl/LKV2vA) Set
//     backing_dev_info::ra_pages to the min of that value and what was sent
//     in the request's max_readahead field.
//
//  *  (http://goo.gl/u2SqzH) Use backing_dev_info::ra_pages when deciding
//     how much to read ahead.
//
//  *  (http://goo.gl/JnhbdL) Don't read ahead at all if that field is zero.
//
// Reading a page at a time is a drag. Ask for a larger size.
const maxReadahead = 1 << 20

// A connection to the fuse kernel process.
type Connection struct {
	debugLogger *log.Logger
	errorLogger *log.Logger

	// The device through which we're talking to the kernel, and the protocol
	// version that we're using to talk to it.
	dev      *os.File
	protocol fusekernel.Protocol

	// The context from which all op contexts inherit.
	parentCtx context.Context

	// For logging purposes only.
	nextOpID uint32

	mu sync.Mutex

	// A freelist of InMessage structs, the allocation of which can be a hot spot
	// for CPU usage. Each element is in an undefined state, and must be
	// re-initialized.
	//
	// GUARDED_BY(mu)
	messageFreelist []*buffer.InMessage

	// A map from fuse "unique" request ID (*not* the op ID for logging used
	// above) to a function that cancel's its associated context.
	//
	// GUARDED_BY(mu)
	cancelFuncs map[uint64]func()
}

// Create a connection wrapping the supplied file descriptor connected to the
// kernel. You must eventually call c.close().
//
// The loggers may be nil.
func newConnection(
	parentCtx context.Context,
	debugLogger *log.Logger,
	errorLogger *log.Logger,
	dev *os.File) (c *Connection, err error) {
	c = &Connection{
		debugLogger: debugLogger,
		errorLogger: errorLogger,
		dev:         dev,
		parentCtx:   parentCtx,
		cancelFuncs: make(map[uint64]func()),
	}

	// Initialize.
	err = c.Init()
	if err != nil {
		c.close()
		err = fmt.Errorf("Init: %v", err)
		return
	}

	return
}

// Do the work necessary to cause the mount process to complete.
func (c *Connection) Init() (err error) {
	// Read the init op.
	op, err := c.ReadOp()
	if err != nil {
		err = fmt.Errorf("Reading init op: %v", err)
		return
	}

	initOp, ok := op.(*fuseops.InternalInitOp)
	if !ok {
		err = fmt.Errorf("Expected *fuseops.InternalInitOp, got %T", op)
		return
	}

	// Make sure the protocol version spoken by the kernel is new enough.
	min := fusekernel.Protocol{
		fusekernel.ProtoVersionMinMajor,
		fusekernel.ProtoVersionMinMinor,
	}

	if initOp.Kernel.LT(min) {
		initOp.Respond(syscall.EPROTO)
		err = fmt.Errorf("Version too old: %v", initOp.Kernel)
		return
	}

	// Downgrade our protocol if necessary.
	c.protocol = fusekernel.Protocol{
		fusekernel.ProtoVersionMaxMajor,
		fusekernel.ProtoVersionMaxMinor,
	}

	if initOp.Kernel.LT(c.protocol) {
		c.protocol = initOp.Kernel
	}

	// Respond to the init op.
	initOp.Library = c.protocol
	initOp.MaxReadahead = maxReadahead
	initOp.MaxWrite = buffer.MaxWriteSize
	initOp.Flags = fusekernel.InitBigWrites
	initOp.Respond(nil)

	return
}

// Log information for an operation with the given ID. calldepth is the depth
// to use when recovering file:line information with runtime.Caller.
func (c *Connection) debugLog(
	opID uint32,
	calldepth int,
	format string,
	v ...interface{}) {
	if c.debugLogger == nil {
		return
	}

	// Get file:line info.
	var file string
	var line int
	var ok bool

	_, file, line, ok = runtime.Caller(calldepth)
	if !ok {
		file = "???"
	}

	fileLine := fmt.Sprintf("%v:%v", path.Base(file), line)

	// Format the actual message to be printed.
	msg := fmt.Sprintf(
		"Op 0x%08x %24s] %v",
		opID,
		fileLine,
		fmt.Sprintf(format, v...))

	// Print it.
	c.debugLogger.Println(msg)
}

// LOCKS_EXCLUDED(c.mu)
func (c *Connection) recordCancelFunc(
	fuseID uint64,
	f func()) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if _, ok := c.cancelFuncs[fuseID]; ok {
		panic(fmt.Sprintf("Already have cancel func for request %v", fuseID))
	}

	c.cancelFuncs[fuseID] = f
}

// Set up state for an op that is about to be returned to the user, given its
// underlying fuse opcode and request ID.
//
// Return a context that should be used for the op.
//
// LOCKS_EXCLUDED(c.mu)
func (c *Connection) beginOp(
	opCode uint32,
	fuseID uint64) (ctx context.Context) {
	// Start with the parent context.
	ctx = c.parentCtx

	// Set up a cancellation function.
	//
	// Special case: On Darwin, osxfuse aggressively reuses "unique" request IDs.
	// This matters for Forget requests, which have no reply associated and
	// therefore have IDs that are immediately eligible for reuse. For these, we
	// should not record any state keyed on their ID.
	//
	// Cf. https://github.com/osxfuse/osxfuse/issues/208
	if opCode != fusekernel.OpForget {
		var cancel func()
		ctx, cancel = context.WithCancel(ctx)
		c.recordCancelFunc(fuseID, cancel)
	}

	return
}

// Clean up all state associated with an op to which the user has responded,
// given its underlying fuse opcode and request ID. This must be called before
// a response is sent to the kernel, to avoid a race where the request's ID
// might be reused by osxfuse.
//
// LOCKS_EXCLUDED(c.mu)
func (c *Connection) finishOp(
	opCode uint32,
	fuseID uint64) {
	c.mu.Lock()
	defer c.mu.Unlock()

	// Even though the op is finished, context.WithCancel requires us to arrange
	// for the cancellation function to be invoked. We also must remove it from
	// our map.
	//
	// Special case: we don't do this for Forget requests. See the note in
	// beginOp above.
	if opCode != fusekernel.OpForget {
		cancel, ok := c.cancelFuncs[fuseID]
		if !ok {
			panic(fmt.Sprintf("Unknown request ID in finishOp: %v", fuseID))
		}

		cancel()
		delete(c.cancelFuncs, fuseID)
	}
}

// LOCKS_EXCLUDED(c.mu)
func (c *Connection) handleInterrupt(fuseID uint64) {
	c.mu.Lock()
	defer c.mu.Unlock()

	// NOTE(jacobsa): fuse.txt in the Linux kernel documentation
	// (https://goo.gl/H55Dnr) defines the kernel <-> userspace protocol for
	// interrupts.
	//
	// In particular, my reading of it is that an interrupt request cannot be
	// delivered to userspace before the original request. The part about the
	// race and EAGAIN appears to be aimed at userspace programs that
	// concurrently process requests (cf. http://goo.gl/BES2rs).
	//
	// So in this method if we can't find the ID to be interrupted, it means that
	// the request has already been replied to.
	//
	// Cf. https://github.com/osxfuse/osxfuse/issues/208
	// Cf. http://comments.gmane.org/gmane.comp.file-systems.fuse.devel/14675
	cancel, ok := c.cancelFuncs[fuseID]
	if !ok {
		return
	}

	cancel()
}

// m.Init must be called.
func (c *Connection) allocateInMessage() (m *buffer.InMessage) {
	c.mu.Lock()
	defer c.mu.Unlock()

	// Can we pull from the freelist?
	l := len(c.messageFreelist)
	if l != 0 {
		m = c.messageFreelist[l-1]
		c.messageFreelist = c.messageFreelist[:l-1]
		return
	}

	// Otherwise, allocate a new one.
	m = new(buffer.InMessage)

	return
}

func (c *Connection) destroyInMessage(m *buffer.InMessage) {
	c.mu.Lock()
	defer c.mu.Unlock()

	// Stick it on the freelist.
	c.messageFreelist = append(c.messageFreelist, m)
}

// Read the next message from the kernel. The message must later be destroyed
// using destroyInMessage.
func (c *Connection) readMessage() (m *buffer.InMessage, err error) {
	// Allocate a message.
	m = c.allocateInMessage()

	// Loop past transient errors.
	for {
		// Attempt a reaed.
		err = m.Init(c.dev)

		// Special cases:
		//
		//  *  ENODEV means fuse has hung up.
		//
		//  *  EINTR means we should try again. (This seems to happen often on
		//     OS X, cf. http://golang.org/issue/11180)
		//
		if pe, ok := err.(*os.PathError); ok {
			switch pe.Err {
			case syscall.ENODEV:
				err = io.EOF

			case syscall.EINTR:
				err = nil
				continue
			}
		}

		if err != nil {
			c.destroyInMessage(m)
			m = nil
			return
		}

		return
	}
}

// Write the supplied message to the kernel.
func (c *Connection) writeMessage(msg []byte) (err error) {
	// Avoid the retry loop in os.File.Write.
	n, err := syscall.Write(int(c.dev.Fd()), msg)
	if err != nil {
		return
	}

	if n != len(msg) {
		err = fmt.Errorf("Wrote %d bytes; expected %d", n, len(msg))
		return
	}

	return
}

// Read the next op from the kernel process. Return io.EOF if the kernel has
// closed the connection.
//
// This function delivers ops in exactly the order they are received from
// /dev/fuse. It must not be called multiple times concurrently.
//
// LOCKS_EXCLUDED(c.mu)
func (c *Connection) ReadOp() (op fuseops.Op, err error) {
	// Keep going until we find a request we know how to convert.
	for {
		// Read the next message from the kernel.
		var m *buffer.InMessage
		m, err = c.readMessage()
		if err != nil {
			return
		}

		// Choose an ID for this operation for the purposes of logging.
		opID := c.nextOpID
		c.nextOpID++

		// Set up op dependencies.
		opCtx := c.beginOp(m.Header().Opcode, m.Header().Unique)

		var debugLogForOp func(int, string, ...interface{})
		if c.debugLogger != nil {
			debugLogForOp = func(calldepth int, format string, v ...interface{}) {
				c.debugLog(opID, calldepth+1, format, v...)
			}
		}

		sendReply := func(
			op fuseops.Op,
			fuseID uint64,
			replyMsg []byte,
			opErr error) (err error) {
			// Make sure we destroy the message, as required by readMessage.
			defer c.destroyInMessage(m)

			// Clean up state for this op.
			c.finishOp(m.Header().Opcode, m.Header().Unique)

			// Debug logging
			if c.debugLogger != nil {
				if opErr == nil {
					op.Logf("-> OK: %s", op.DebugString())
				} else {
					op.Logf("-> error: %v", opErr)
				}
			}

			// Error logging
			if opErr != nil && c.errorLogger != nil {
				c.errorLogger.Printf("(%s) error: %v", op.ShortDesc(), opErr)
			}

			// Send the reply to the kernel.
			err = c.writeMessage(replyMsg)
			if err != nil {
				err = fmt.Errorf("writeMessage: %v", err)
				return
			}

			return
		}

		// Convert the message to an Op.
		op, err = fuseops.Convert(
			opCtx,
			m,
			c.protocol,
			debugLogForOp,
			c.errorLogger,
			sendReply)

		if err != nil {
			err = fmt.Errorf("fuseops.Convert: %v", err)
			return
		}

		// Log the receipt of the operation.
		c.debugLog(opID, 1, "<- %v", op.ShortDesc())

		// Special case: responding to statfs is required to make mounting work on
		// OS X. We don't currently expose the capability for the file system to
		// intercept this.
		if _, ok := op.(*fuseops.InternalStatFSOp); ok {
			op.Respond(nil)
			continue
		}

		// Special case: handle interrupt requests.
		if interruptOp, ok := op.(*fuseops.InternalInterruptOp); ok {
			c.handleInterrupt(interruptOp.FuseID)
			continue
		}

		return
	}
}

// Close the connection. Must not be called until operations that were read
// from the connection have been responded to.
func (c *Connection) close() (err error) {
	// Posix doesn't say that close can be called concurrently with read or
	// write, but luckily we exclude the possibility of a race by requiring the
	// user to respond to all ops first.
	err = c.dev.Close()
	return
}
<|endoftext|>


<commit_before>package rev

import (
	"database/sql"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"runtime/debug"
	"strings"
)

type Controller struct {
	Name       string
	Type       *ControllerType
	MethodType *MethodType

	Request  *Request
	Response *Response

	Flash      Flash                  // User cookie, cleared after each request.
	Session    Session                // Session, stored in cookie, signed.
	Params     *Params                // Parameters from URL and form (including multipart).
	Args       map[string]interface{} // Per-request scratch space.
	RenderArgs map[string]interface{} // Args passed to the template.
	Validation *Validation            // Data validation helpers
	Txn        *sql.Tx                // Nil by default, but may be used by the app / plugins
}

func NewController(req *Request, resp *Response, ct *ControllerType) *Controller {
	return &Controller{
		Name:     ct.Type.Name(),
		Type:     ct,
		Request:  req,
		Response: resp,
		Params:   ParseParams(req),
		RenderArgs: map[string]interface{}{
			"RunMode": RunMode,
		},
	}
}

func (c *Controller) FlashParams() {
	for key, vals := range c.Params.Values {
		c.Flash.Out[key] = vals[0]
	}
}

func (c *Controller) SetCookie(cookie *http.Cookie) {
	http.SetCookie(c.Response.Out, cookie)
}

// Invoke the given method, save headers/cookies to the response, and apply the
// result.  (e.g. render a template to the response)
func (c *Controller) Invoke(appControllerPtr reflect.Value, method reflect.Value, methodArgs []reflect.Value) {

	// Handle panics.
	defer func() {
		if err := recover(); err != nil {
			handleInvocationPanic(c, err)
		}
	}()

	// Clean up from the request.
	defer func() {
		// Delete temp files.
		if c.Request.MultipartForm != nil {
			err := c.Request.MultipartForm.RemoveAll()
			if err != nil {
				WARN.Println("Error removing temporary files:", err)
			}
		}

		for _, tmpFile := range c.Params.tmpFiles {
			err := os.Remove(tmpFile.Name())
			if err != nil {
				WARN.Println("Could not remove upload temp file:", err)
			}
		}
	}()

	// Run the plugins.
	plugins.BeforeRequest(c)

	// Calculate the Result by running the interceptors and the action.
	resultValue := func() reflect.Value {
		// Call the BEFORE interceptors
		result := c.invokeInterceptors(BEFORE, appControllerPtr)
		if result != nil {
			return reflect.ValueOf(result)
		}

		// Invoke the action.
		resultValue := method.Call(methodArgs)[0]

		// Call the AFTER interceptors
		result = c.invokeInterceptors(AFTER, appControllerPtr)
		if result != nil {
			return reflect.ValueOf(result)
		}
		return resultValue
	}()

	plugins.AfterRequest(c)

	if resultValue.IsNil() {
		return
	}
	result := resultValue.Interface().(Result)

	// Apply the result, which generally results in the ResponseWriter getting written.
	result.Apply(c.Request, c.Response)
}

// This function handles a panic in an action invocation.
// It cleans up the stack trace, logs it, and displays an error page.
func handleInvocationPanic(c *Controller, err interface{}) {
	plugins.OnException(c, err)
	stack := string(debug.Stack())
	ERROR.Println(err, "\n", stack)

	error := NewErrorFromPanic(err)
	if error == nil {
		c.Response.Out.WriteHeader(500)
		c.Response.Out.Write([]byte(stack))
		return
	}

	c.RenderError(error).Apply(c.Request, c.Response)
}

func (c *Controller) invokeInterceptors(when InterceptTime, appControllerPtr reflect.Value) Result {
	var result Result
	for _, intc := range getInterceptors(when, appControllerPtr) {
		resultValue := intc.Invoke(appControllerPtr)
		if !resultValue.IsNil() {
			result = resultValue.Interface().(Result)
		}
		if when == BEFORE && result != nil {
			return result
		}
	}
	return result
}

func (c *Controller) RenderError(err error) Result {
	return ErrorResult{c.RenderArgs, err}
}

// Render a template corresponding to the calling Controller method.
// Arguments will be added to c.RenderArgs prior to rendering the template.
// They are keyed on their local identifier.
//
// For example:
//
//     func (c Users) ShowUser(id int) rev.Result {
//     	 user := loadUser(id)
//     	 return c.Render(user)
//     }
//
// This action will render views/Users/ShowUser.html, passing in an extra
// key-value "user": (User).
func (c *Controller) Render(extraRenderArgs ...interface{}) Result {
	// Get the calling function name.
	pc, _, line, ok := runtime.Caller(1)
	if !ok {
		ERROR.Println("Failed to get Caller information")
		return nil
	}
	// e.g. sample/app/controllers.(*Application).Index
	var fqViewName string = runtime.FuncForPC(pc).Name()
	var viewName string = fqViewName[strings.LastIndex(fqViewName, ".")+1 : len(fqViewName)]

	// Determine what method we are in.
	// (e.g. the invoked controller method might have delegated to another method)
	methodType := c.MethodType
	if methodType.Name != viewName {
		methodType = c.Type.Method(viewName)
		if methodType == nil {
			return c.RenderError(fmt.Errorf(
				"No Method %s in Controller %s when loading the view."+
					" (delegating Render is only supported within the same controller)",
				viewName, c.Name))
		}
	}

	// Get the extra RenderArgs passed in.
	if renderArgNames, ok := methodType.RenderArgNames[line]; ok {
		if len(renderArgNames) == len(extraRenderArgs) {
			for i, extraRenderArg := range extraRenderArgs {
				c.RenderArgs[renderArgNames[i]] = extraRenderArg
			}
		} else {
			ERROR.Println(len(renderArgNames), "RenderArg names found for",
				len(extraRenderArgs), "extra RenderArgs")
		}
	} else {
		ERROR.Println("No RenderArg names found for Render call on line", line,
			"(Method", methodType, ", ViewName", viewName, ")")
	}

	return c.RenderTemplate(c.Name + "/" + viewName + ".html")
}

// A less magical way to render a template.
// Renders the given template, using the current RenderArgs.
func (c *Controller) RenderTemplate(templatePath string) Result {

	// Get the Template.
	template, err := MainTemplateLoader.Template(templatePath)
	if err != nil {
		return c.RenderError(err)
	}

	return &RenderTemplateResult{
		Template:   template,
		RenderArgs: c.RenderArgs,
	}
}

// Uses encoding/json.Marshal to return JSON to the client.
func (c *Controller) RenderJson(o interface{}) Result {
	return RenderJsonResult{o}
}

// Uses encoding/xml.Marshal to return XML to the client.
func (c *Controller) RenderXml(o interface{}) Result {
	return RenderXmlResult{o}
}

// Render plaintext in response, printf style.
func (c *Controller) RenderText(text string, objs ...interface{}) Result {
	finalText := text
	if len(objs) > 0 {
		finalText = fmt.Sprintf(text, objs)
	}
	return &RenderTextResult{finalText}
}

// Render a "todo" indicating that the action isn't done yet.
func (c *Controller) Todo() Result {
	c.Response.Status = http.StatusNotImplemented
	return c.RenderError(&Error{
		Title:       "TODO",
		Description: "This action is not implemented",
	})
}

func (c *Controller) NotFound(msg string) Result {
	c.Response.Status = http.StatusNotFound
	return c.RenderError(&Error{
		Title:       "Not Found",
		Description: msg,
	})
}

// Return a file, either displayed inline or downloaded as an attachment.
// The name and size are taken from the file info.
func (c *Controller) RenderFile(file *os.File, delivery ContentDisposition) Result {
	var length int64 = -1
	fileInfo, err := file.Stat()
	if err != nil {
		WARN.Println("RenderFile error:", err)
	}
	if fileInfo != nil {
		length = fileInfo.Size()
	}
	return &BinaryResult{
		Reader:   file,
		Name:     filepath.Base(file.Name()),
		Length:   length,
		Delivery: delivery,
	}
}

// Redirect to an action or to a URL.
//   c.Redirect(Controller.Action)
//   c.Redirect("/controller/action")
//   c.Redirect("/controller/%d/action", id)
func (c *Controller) Redirect(val interface{}, args ...interface{}) Result {
	if url, ok := val.(string); ok {
		if len(args) == 0 {
			return &RedirectToUrlResult{url}
		}
		return &RedirectToUrlResult{fmt.Sprintf(url, args...)}
	}
	return &RedirectToActionResult{val}
}
<commit_msg>Initialize Args with an empty map.<commit_after>package rev

import (
	"database/sql"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"runtime/debug"
	"strings"
)

type Controller struct {
	Name       string
	Type       *ControllerType
	MethodType *MethodType

	Request  *Request
	Response *Response

	Flash      Flash                  // User cookie, cleared after each request.
	Session    Session                // Session, stored in cookie, signed.
	Params     *Params                // Parameters from URL and form (including multipart).
	Args       map[string]interface{} // Per-request scratch space.
	RenderArgs map[string]interface{} // Args passed to the template.
	Validation *Validation            // Data validation helpers
	Txn        *sql.Tx                // Nil by default, but may be used by the app / plugins
}

func NewController(req *Request, resp *Response, ct *ControllerType) *Controller {
	return &Controller{
		Name:     ct.Type.Name(),
		Type:     ct,
		Request:  req,
		Response: resp,
		Params:   ParseParams(req),
		Args:     map[string]interface{}{},
		RenderArgs: map[string]interface{}{
			"RunMode": RunMode,
		},
	}
}

func (c *Controller) FlashParams() {
	for key, vals := range c.Params.Values {
		c.Flash.Out[key] = vals[0]
	}
}

func (c *Controller) SetCookie(cookie *http.Cookie) {
	http.SetCookie(c.Response.Out, cookie)
}

// Invoke the given method, save headers/cookies to the response, and apply the
// result.  (e.g. render a template to the response)
func (c *Controller) Invoke(appControllerPtr reflect.Value, method reflect.Value, methodArgs []reflect.Value) {

	// Handle panics.
	defer func() {
		if err := recover(); err != nil {
			handleInvocationPanic(c, err)
		}
	}()

	// Clean up from the request.
	defer func() {
		// Delete temp files.
		if c.Request.MultipartForm != nil {
			err := c.Request.MultipartForm.RemoveAll()
			if err != nil {
				WARN.Println("Error removing temporary files:", err)
			}
		}

		for _, tmpFile := range c.Params.tmpFiles {
			err := os.Remove(tmpFile.Name())
			if err != nil {
				WARN.Println("Could not remove upload temp file:", err)
			}
		}
	}()

	// Run the plugins.
	plugins.BeforeRequest(c)

	// Calculate the Result by running the interceptors and the action.
	resultValue := func() reflect.Value {
		// Call the BEFORE interceptors
		result := c.invokeInterceptors(BEFORE, appControllerPtr)
		if result != nil {
			return reflect.ValueOf(result)
		}

		// Invoke the action.
		resultValue := method.Call(methodArgs)[0]

		// Call the AFTER interceptors
		result = c.invokeInterceptors(AFTER, appControllerPtr)
		if result != nil {
			return reflect.ValueOf(result)
		}
		return resultValue
	}()

	plugins.AfterRequest(c)

	if resultValue.IsNil() {
		return
	}
	result := resultValue.Interface().(Result)

	// Apply the result, which generally results in the ResponseWriter getting written.
	result.Apply(c.Request, c.Response)
}

// This function handles a panic in an action invocation.
// It cleans up the stack trace, logs it, and displays an error page.
func handleInvocationPanic(c *Controller, err interface{}) {
	plugins.OnException(c, err)
	stack := string(debug.Stack())
	ERROR.Println(err, "\n", stack)

	error := NewErrorFromPanic(err)
	if error == nil {
		c.Response.Out.WriteHeader(500)
		c.Response.Out.Write([]byte(stack))
		return
	}

	c.RenderError(error).Apply(c.Request, c.Response)
}

func (c *Controller) invokeInterceptors(when InterceptTime, appControllerPtr reflect.Value) Result {
	var result Result
	for _, intc := range getInterceptors(when, appControllerPtr) {
		resultValue := intc.Invoke(appControllerPtr)
		if !resultValue.IsNil() {
			result = resultValue.Interface().(Result)
		}
		if when == BEFORE && result != nil {
			return result
		}
	}
	return result
}

func (c *Controller) RenderError(err error) Result {
	return ErrorResult{c.RenderArgs, err}
}

// Render a template corresponding to the calling Controller method.
// Arguments will be added to c.RenderArgs prior to rendering the template.
// They are keyed on their local identifier.
//
// For example:
//
//     func (c Users) ShowUser(id int) rev.Result {
//     	 user := loadUser(id)
//     	 return c.Render(user)
//     }
//
// This action will render views/Users/ShowUser.html, passing in an extra
// key-value "user": (User).
func (c *Controller) Render(extraRenderArgs ...interface{}) Result {
	// Get the calling function name.
	pc, _, line, ok := runtime.Caller(1)
	if !ok {
		ERROR.Println("Failed to get Caller information")
		return nil
	}
	// e.g. sample/app/controllers.(*Application).Index
	var fqViewName string = runtime.FuncForPC(pc).Name()
	var viewName string = fqViewName[strings.LastIndex(fqViewName, ".")+1 : len(fqViewName)]

	// Determine what method we are in.
	// (e.g. the invoked controller method might have delegated to another method)
	methodType := c.MethodType
	if methodType.Name != viewName {
		methodType = c.Type.Method(viewName)
		if methodType == nil {
			return c.RenderError(fmt.Errorf(
				"No Method %s in Controller %s when loading the view."+
					" (delegating Render is only supported within the same controller)",
				viewName, c.Name))
		}
	}

	// Get the extra RenderArgs passed in.
	if renderArgNames, ok := methodType.RenderArgNames[line]; ok {
		if len(renderArgNames) == len(extraRenderArgs) {
			for i, extraRenderArg := range extraRenderArgs {
				c.RenderArgs[renderArgNames[i]] = extraRenderArg
			}
		} else {
			ERROR.Println(len(renderArgNames), "RenderArg names found for",
				len(extraRenderArgs), "extra RenderArgs")
		}
	} else {
		ERROR.Println("No RenderArg names found for Render call on line", line,
			"(Method", methodType, ", ViewName", viewName, ")")
	}

	return c.RenderTemplate(c.Name + "/" + viewName + ".html")
}

// A less magical way to render a template.
// Renders the given template, using the current RenderArgs.
func (c *Controller) RenderTemplate(templatePath string) Result {

	// Get the Template.
	template, err := MainTemplateLoader.Template(templatePath)
	if err != nil {
		return c.RenderError(err)
	}

	return &RenderTemplateResult{
		Template:   template,
		RenderArgs: c.RenderArgs,
	}
}

// Uses encoding/json.Marshal to return JSON to the client.
func (c *Controller) RenderJson(o interface{}) Result {
	return RenderJsonResult{o}
}

// Uses encoding/xml.Marshal to return XML to the client.
func (c *Controller) RenderXml(o interface{}) Result {
	return RenderXmlResult{o}
}

// Render plaintext in response, printf style.
func (c *Controller) RenderText(text string, objs ...interface{}) Result {
	finalText := text
	if len(objs) > 0 {
		finalText = fmt.Sprintf(text, objs)
	}
	return &RenderTextResult{finalText}
}

// Render a "todo" indicating that the action isn't done yet.
func (c *Controller) Todo() Result {
	c.Response.Status = http.StatusNotImplemented
	return c.RenderError(&Error{
		Title:       "TODO",
		Description: "This action is not implemented",
	})
}

func (c *Controller) NotFound(msg string) Result {
	c.Response.Status = http.StatusNotFound
	return c.RenderError(&Error{
		Title:       "Not Found",
		Description: msg,
	})
}

// Return a file, either displayed inline or downloaded as an attachment.
// The name and size are taken from the file info.
func (c *Controller) RenderFile(file *os.File, delivery ContentDisposition) Result {
	var length int64 = -1
	fileInfo, err := file.Stat()
	if err != nil {
		WARN.Println("RenderFile error:", err)
	}
	if fileInfo != nil {
		length = fileInfo.Size()
	}
	return &BinaryResult{
		Reader:   file,
		Name:     filepath.Base(file.Name()),
		Length:   length,
		Delivery: delivery,
	}
}

// Redirect to an action or to a URL.
//   c.Redirect(Controller.Action)
//   c.Redirect("/controller/action")
//   c.Redirect("/controller/%d/action", id)
func (c *Controller) Redirect(val interface{}, args ...interface{}) Result {
	if url, ok := val.(string); ok {
		if len(args) == 0 {
			return &RedirectToUrlResult{url}
		}
		return &RedirectToUrlResult{fmt.Sprintf(url, args...)}
	}
	return &RedirectToActionResult{val}
}
<|endoftext|>


<commit_before>// Copyright © 2017 Farhad Farahi <farhad.farahi@gmail.com>
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	homedir "github.com/mitchellh/go-homedir"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"log"
	"mongobench/bench"
	"os"
	_ "strconv"
	"strings"
)

var (
	cfgFile       string
	threads       int
	batch         int
	queryFilePath string
	host          string
	database      string
	collection    string
	timeout       int
	username      string
	password      string
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "mongobench",
	Short: "A small Benchmark tool for mongo deployment",
	Long:  `A small Benchmark tool for mongo deployment`,
	// Uncomment the following line if your bare application
	// has an action associated with it:
	Run: rootCmd,
}

func rootCmd(cmd *cobra.Command, args []string) {
	if versionFlag := getFlagBoolPtr(cmd, "version"); versionFlag != nil {
		fmt.Println("MongoBench v1.0.0")
	} else {
		bench.Bench(threads, batch, queryFilePath, host, database, collection, timeout, username, password)
	}
}

func getFlagBoolPtr(cmd *cobra.Command, flag string) *bool {
	f := cmd.Flags().Lookup(flag)
	if f == nil {
		log.Printf("Flag accessed but not defined for command %s: %s", cmd.Name(), flag)
	}
	// Check if flag was not set at all.
	if !f.Changed && f.DefValue == f.Value.String() {
		return nil
	}
	var ret bool
	// Caseless compare.
	if strings.ToLower(f.Value.String()) == "true" {
		ret = true
	} else {
		ret = false
	}
	return &ret
}

/*
func getFlagInt(cmd *cobra.Command, flag string) int {
	f := cmd.Flags().Lookup(flag)
	if f == nil {
		log.Printf("Flag accessed but not defined for command %s: %s", cmd.Name(), flag)
	}
	v, err := strconv.Atoi(f.Value.String())
	// This is likely not a sufficiently friendly error message, but cobra
	// should prevent non-integer values from reaching here.
	if err != nil {
		log.Println(err)
	}
	return v
}
*/

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	// Here you will define your flags and configuration settings.
	// Cobra supports persistent flags, which, if defined here,
	// will be global for your application.
	//RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.mongobench.yaml)")

	// Cobra also supports local flags, which will only run
	// when this action is called directly.
	RootCmd.Flags().BoolP("version", "v", false, "Prints version")
	RootCmd.Flags().IntVarP(&threads, "threads", "t", 100, "Total number of threads to use. Equal to number of queries against mongodb")
	RootCmd.Flags().IntVarP(&batch, "batch", "b", 100, "Number of threads per batch.")
	RootCmd.Flags().StringVarP(&queryFilePath, "queryFile", "q", "/tmp/query", `Path to the query file, one query per line. Only the query string, example: {"branchCode":230}"`)
	RootCmd.Flags().StringVarP(&host, "host", "H", "localhost:27017", "IP addresses or Hostnames and ports of the mongo hosts to connect to separated by commas, example: mongo1:27017, mongo2:27017")
	RootCmd.Flags().StringVarP(&database, "database", "d", "journaldb", "Database to run queries against")
	RootCmd.Flags().StringVarP(&collection, "collection", "c", "journal", "Collection to run queries against")
	RootCmd.Flags().IntVarP(&timeout, "timeout", "T", 15, "db query timeout in seconds")
	RootCmd.Flags().StringVarP(&username, "username", "u", "", "Username for DB Authentication, Do not use this if you DB doesnt have authentication enabled")
	RootCmd.Flags().StringVarP(&password, "password", "p", "", "Password for DB Authentication, Do not use this if you DB doesnt have authentication enabled")
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	if cfgFile != "" {
		// Use config file from the flag.
		viper.SetConfigFile(cfgFile)
	} else {
		// Find home directory.
		home, err := homedir.Dir()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		// Search config in home directory with name ".mongobench" (without extension).
		viper.AddConfigPath(home)
		viper.SetConfigName(".mongobench")
	}

	viper.AutomaticEnv() // read in environment variables that match

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		fmt.Println("Using config file:", viper.ConfigFileUsed())
	}
}
<commit_msg>handled where threads are smaller than batches<commit_after>// Copyright © 2017 Farhad Farahi <farhad.farahi@gmail.com>
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	homedir "github.com/mitchellh/go-homedir"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"log"
	"mongobench/bench"
	"os"
	_ "strconv"
	"strings"
)

var (
	cfgFile       string
	threads       int
	batch         int
	queryFilePath string
	host          string
	database      string
	collection    string
	timeout       int
	username      string
	password      string
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "mongobench",
	Short: "A small Benchmark tool for mongo deployment",
	Long:  `A small Benchmark tool for mongo deployment`,
	// Uncomment the following line if your bare application
	// has an action associated with it:
	Run: rootCmd,
}

func rootCmd(cmd *cobra.Command, args []string) {
	if versionFlag := getFlagBoolPtr(cmd, "version"); versionFlag != nil {
		fmt.Println("MongoBench v1.0.1")
	} else {
		if batch >= threads {
			batch = threads
		}
		bench.Bench(threads, batch, queryFilePath, host, database, collection, timeout, username, password)
	}
}

func getFlagBoolPtr(cmd *cobra.Command, flag string) *bool {
	f := cmd.Flags().Lookup(flag)
	if f == nil {
		log.Printf("Flag accessed but not defined for command %s: %s", cmd.Name(), flag)
	}
	// Check if flag was not set at all.
	if !f.Changed && f.DefValue == f.Value.String() {
		return nil
	}
	var ret bool
	// Caseless compare.
	if strings.ToLower(f.Value.String()) == "true" {
		ret = true
	} else {
		ret = false
	}
	return &ret
}

/*
func getFlagInt(cmd *cobra.Command, flag string) int {
	f := cmd.Flags().Lookup(flag)
	if f == nil {
		log.Printf("Flag accessed but not defined for command %s: %s", cmd.Name(), flag)
	}
	v, err := strconv.Atoi(f.Value.String())
	// This is likely not a sufficiently friendly error message, but cobra
	// should prevent non-integer values from reaching here.
	if err != nil {
		log.Println(err)
	}
	return v
}
*/

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	// Here you will define your flags and configuration settings.
	// Cobra supports persistent flags, which, if defined here,
	// will be global for your application.
	//RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.mongobench.yaml)")

	// Cobra also supports local flags, which will only run
	// when this action is called directly.
	RootCmd.Flags().BoolP("version", "v", false, "Prints version")
	RootCmd.Flags().IntVarP(&threads, "threads", "t", 100, "Total number of threads to use. Equal to number of queries against mongodb")
	RootCmd.Flags().IntVarP(&batch, "batch", "b", 100, "Number of threads per batch.")
	RootCmd.Flags().StringVarP(&queryFilePath, "queryFile", "q", "/tmp/query", `Path to the query file, one query per line. Only the query string, example: {"branchCode":230}"`)
	RootCmd.Flags().StringVarP(&host, "host", "H", "localhost:27017", "IP addresses or Hostnames and ports of the mongo hosts to connect to separated by commas, example: mongo1:27017, mongo2:27017")
	RootCmd.Flags().StringVarP(&database, "database", "d", "journaldb", "Database to run queries against")
	RootCmd.Flags().StringVarP(&collection, "collection", "c", "journal", "Collection to run queries against")
	RootCmd.Flags().IntVarP(&timeout, "timeout", "T", 15, "db query timeout in seconds")
	RootCmd.Flags().StringVarP(&username, "username", "u", "", "Username for DB Authentication, Do not use this if you DB doesnt have authentication enabled")
	RootCmd.Flags().StringVarP(&password, "password", "p", "", "Password for DB Authentication, Do not use this if you DB doesnt have authentication enabled")
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	if cfgFile != "" {
		// Use config file from the flag.
		viper.SetConfigFile(cfgFile)
	} else {
		// Find home directory.
		home, err := homedir.Dir()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		// Search config in home directory with name ".mongobench" (without extension).
		viper.AddConfigPath(home)
		viper.SetConfigName(".mongobench")
	}

	viper.AutomaticEnv() // read in environment variables that match

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		fmt.Println("Using config file:", viper.ConfigFileUsed())
	}
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package madmin

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
)

// BackendType - represents different backend types.
type BackendType int

// Enum for different backend types.
const (
	Unknown BackendType = iota
	// Filesystem backend.
	FS
	// Multi disk Erasure (single, distributed) backend.
	Erasure

	// Add your own backend.
)

// StorageInfo - represents total capacity of underlying storage.
type StorageInfo struct {
	// Total disk space.
	Total int64
	// Free available disk space.
	Free int64
	// Backend type.
	Backend struct {
		// Represents various backend types, currently on FS and Erasure.
		Type BackendType

		// Following fields are only meaningful if BackendType is Erasure.
		OnlineDisks  int // Online disks during server startup.
		OfflineDisks int // Offline disks during server startup.
		ReadQuorum   int // Minimum disks required for successful read operations.
		WriteQuorum  int // Minimum disks required for successful write operations.
	}
}

// ServerProperties holds some of the server's information such as uptime,
// version, region, ..
type ServerProperties struct {
	Uptime   time.Duration `json:"uptime"`
	Version  string        `json:"version"`
	CommitID string        `json:"commitID"`
	Region   string        `json:"region"`
	SQSARN   []string      `json:"sqsARN"`
}

// ServerConnStats holds network information
type ServerConnStats struct {
	TotalInputBytes  uint64 `json:"transferred"`
	TotalOutputBytes uint64 `json:"received"`
}

// ServerInfoData holds storage, connections and other
// information of a given server
type ServerInfoData struct {
	StorageInfo StorageInfo      `json:"storage"`
	ConnStats   ServerConnStats  `json:"network"`
	Properties  ServerProperties `json:"server"`
}

// ServerInfo holds server information result of one node
type ServerInfo struct {
	Error string          `json:"error"`
	Addr  string          `json:"addr"`
	Data  *ServerInfoData `json:"data"`
}

// ServerInfo - Connect to a minio server and call Server Info Management API
// to fetch server's information represented by ServerInfo structure
func (adm *AdminClient) ServerInfo() ([]ServerInfo, error) {
	// Prepare web service request
	reqData := requestData{}
	reqData.queryValues = make(url.Values)
	reqData.queryValues.Set("info", "")
	reqData.customHeaders = make(http.Header)

	resp, err := adm.executeMethod("GET", reqData)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	// Check response http status code
	if resp.StatusCode != http.StatusOK {
		return nil, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	var serversInfo []ServerInfo

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(respBytes, &serversInfo)
	if err != nil {
		return nil, err
	}

	return serversInfo, nil
}
<commit_msg>add HTTPStats to madmin (#5299)<commit_after>/*
 * Minio Cloud Storage, (C) 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package madmin

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
)

// BackendType - represents different backend types.
type BackendType int

// Enum for different backend types.
const (
	Unknown BackendType = iota
	// Filesystem backend.
	FS
	// Multi disk Erasure (single, distributed) backend.
	Erasure

	// Add your own backend.
)

// StorageInfo - represents total capacity of underlying storage.
type StorageInfo struct {
	// Total disk space.
	Total int64
	// Free available disk space.
	Free int64
	// Backend type.
	Backend struct {
		// Represents various backend types, currently on FS and Erasure.
		Type BackendType

		// Following fields are only meaningful if BackendType is Erasure.
		OnlineDisks  int // Online disks during server startup.
		OfflineDisks int // Offline disks during server startup.
		ReadQuorum   int // Minimum disks required for successful read operations.
		WriteQuorum  int // Minimum disks required for successful write operations.
	}
}

// ServerProperties holds some of the server's information such as uptime,
// version, region, ..
type ServerProperties struct {
	Uptime   time.Duration `json:"uptime"`
	Version  string        `json:"version"`
	CommitID string        `json:"commitID"`
	Region   string        `json:"region"`
	SQSARN   []string      `json:"sqsARN"`
}

// ServerConnStats holds network information
type ServerConnStats struct {
	TotalInputBytes  uint64 `json:"transferred"`
	TotalOutputBytes uint64 `json:"received"`
}

// ServerHTTPMethodStats holds total number of HTTP operations from/to the server,
// including the average duration the call was spent.
type ServerHTTPMethodStats struct {
	Count       uint64 `json:"count"`
	AvgDuration string `json:"avgDuration"`
}

// ServerHTTPStats holds all type of http operations performed to/from the server
// including their average execution time.
type ServerHTTPStats struct {
	TotalHEADStats     ServerHTTPMethodStats `json:"totalHEADs"`
	SuccessHEADStats   ServerHTTPMethodStats `json:"successHEADs"`
	TotalGETStats      ServerHTTPMethodStats `json:"totalGETs"`
	SuccessGETStats    ServerHTTPMethodStats `json:"successGETs"`
	TotalPUTStats      ServerHTTPMethodStats `json:"totalPUTs"`
	SuccessPUTStats    ServerHTTPMethodStats `json:"successPUTs"`
	TotalPOSTStats     ServerHTTPMethodStats `json:"totalPOSTs"`
	SuccessPOSTStats   ServerHTTPMethodStats `json:"successPOSTs"`
	TotalDELETEStats   ServerHTTPMethodStats `json:"totalDELETEs"`
	SuccessDELETEStats ServerHTTPMethodStats `json:"successDELETEs"`
}

// ServerInfoData holds storage, connections and other
// information of a given server
type ServerInfoData struct {
	StorageInfo StorageInfo      `json:"storage"`
	ConnStats   ServerConnStats  `json:"network"`
	HTTPStats   ServerHTTPStats  `json:"http"`
	Properties  ServerProperties `json:"server"`
}

// ServerInfo holds server information result of one node
type ServerInfo struct {
	Error string          `json:"error"`
	Addr  string          `json:"addr"`
	Data  *ServerInfoData `json:"data"`
}

// ServerInfo - Connect to a minio server and call Server Info Management API
// to fetch server's information represented by ServerInfo structure
func (adm *AdminClient) ServerInfo() ([]ServerInfo, error) {
	// Prepare web service request
	reqData := requestData{}
	reqData.queryValues = make(url.Values)
	reqData.queryValues.Set("info", "")
	reqData.customHeaders = make(http.Header)

	resp, err := adm.executeMethod("GET", reqData)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	// Check response http status code
	if resp.StatusCode != http.StatusOK {
		return nil, httpRespToErrorResponse(resp)
	}

	// Unmarshal the server's json response
	var serversInfo []ServerInfo

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(respBytes, &serversInfo)
	if err != nil {
		return nil, err
	}

	return serversInfo, nil
}
<|endoftext|>