<commit_before>// Copyright (c) 2013-2015 The btcsuite developers
// Copyright (c) 2015-2016 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package chain

import (
	"bytes"
	"context"
	"sync"
	"time"

	"github.com/decred/dcrd/chaincfg"
	"github.com/decred/dcrd/chaincfg/chainhash"
	dcrrpcclient "github.com/decred/dcrd/rpcclient"
	"github.com/decred/dcrd/wire"
	"github.com/decred/dcrwallet/errors"
)

var requiredChainServerAPI = semver{major: 4, minor: 0, patch: 0}

// RPCClient represents a persistent client connection to a decred RPC server
// for information regarding the current best block chain.
type RPCClient struct {
	*dcrrpcclient.Client
	connConfig  *dcrrpcclient.ConnConfig // Work around unexported field
	chainParams *chaincfg.Params

	enqueueNotification       chan interface{}
	dequeueNotification       chan interface{}
	enqueueVotingNotification chan interface{}
	dequeueVotingNotification chan interface{}

	quit    chan struct{}
	wg      sync.WaitGroup
	started bool
	quitMtx sync.Mutex
}

// NewRPCClient creates a client connection to the server described by the
// connect string.  If disableTLS is false, the remote RPC certificate must be
// provided in the certs slice.  The connection is not established immediately,
// but must be done using the Start method.  If the remote server does not
// operate on the same decred network as described by the passed chain
// parameters, the connection will be disconnected.
func NewRPCClient(chainParams *chaincfg.Params, connect, user, pass string, certs []byte,
	disableTLS bool) (*RPCClient, error) {

	client := &RPCClient{
		connConfig: &dcrrpcclient.ConnConfig{
			Host:                 connect,
			Endpoint:             "ws",
			User:                 user,
			Pass:                 pass,
			Certificates:         certs,
			DisableAutoReconnect: true,
			DisableConnectOnNew:  true,
			DisableTLS:           disableTLS,
		},
		chainParams:               chainParams,
		enqueueNotification:       make(chan interface{}),
		dequeueNotification:       make(chan interface{}),
		enqueueVotingNotification: make(chan interface{}),
		dequeueVotingNotification: make(chan interface{}),
		quit:                      make(chan struct{}),
	}
	ntfnCallbacks := &dcrrpcclient.NotificationHandlers{
		OnBlockConnected:        client.onBlockConnected,
		OnBlockDisconnected:     client.onBlockDisconnected,
		OnRelevantTxAccepted:    client.onRelevantTxAccepted,
		OnReorganization:        client.onReorganization,
		OnWinningTickets:        client.onWinningTickets,
		OnSpentAndMissedTickets: client.onSpentAndMissedTickets,
		OnStakeDifficulty:       client.onStakeDifficulty,
	}
	rpcClient, err := dcrrpcclient.New(client.connConfig, ntfnCallbacks)
	if err != nil {
		return nil, err
	}
	client.Client = rpcClient
	return client, nil
}

// Start attempts to establish a client connection with the remote server.
// If successful, handler goroutines are started to process notifications
// sent by the server.  After a limited number of connection attempts, this
// function gives up, and therefore will not block forever waiting for the
// connection to be established to a server that may not exist.
func (c *RPCClient) Start(ctx context.Context, retry bool) (err error) {
	const op errors.Op = "rpcclient.Start"

	err = c.Client.Connect(ctx, retry)
	if err != nil {
		return errors.E(op, errors.IO, err)
	}

	defer func() {
		if err != nil {
			c.Disconnect()
		}
	}()

	// Verify that the server is running on the expected network.
	net, err := c.Client.GetCurrentNet()
	if err != nil {
		return errors.E(op, errors.E(errors.Op("dcrd.jsonrpc.getcurrentnet"), err))
	}
	if net != c.chainParams.Net {
		return errors.E(op, "mismatched networks")
	}

	// Ensure the RPC server has a compatible API version.
	var serverAPI semver
	versions, err := c.Client.Version()
	if err == nil {
		versionResult := versions["dcrdjsonrpcapi"]
		serverAPI = semver{
			major: versionResult.Major,
			minor: versionResult.Minor,
			patch: versionResult.Patch,
		}
	}
	if !semverCompatible(requiredChainServerAPI, serverAPI) {
		return errors.E(op, errors.Errorf("advertised API version %v incompatible "+
			"with required version %v", serverAPI, requiredChainServerAPI))
	}

	c.quitMtx.Lock()
	c.started = true
	c.quitMtx.Unlock()

	c.wg.Add(2)
	go c.handler()
	go c.handlerVoting()
	return nil
}

// Stop disconnects the client and signals the shutdown of all goroutines
// started by Start.
func (c *RPCClient) Stop() {
	c.quitMtx.Lock()
	select {
	case <-c.quit:
	default:
		close(c.quit)
		c.Client.Shutdown()

		if !c.started {
			close(c.dequeueNotification)
			close(c.dequeueVotingNotification)
		}
	}
	c.quitMtx.Unlock()
}

// WaitForShutdown blocks until both the client has finished disconnecting
// and all handlers have exited.
func (c *RPCClient) WaitForShutdown() {
	c.Client.WaitForShutdown()
	c.wg.Wait()
}

// Notification types.  These are defined here and processed from from reading
// a notificationChan to avoid handling these notifications directly in
// dcrrpcclient callbacks, which isn't very Go-like and doesn't allow
// blocking client calls.
type (
	// blockConnected is a notification for a newly-attached block to the
	// best chain.
	blockConnected struct {
		blockHeader  []byte
		transactions [][]byte
	}

	// blockDisconnected is a notifcation that the block described by the header
	// was reorganized out of the best chain.
	blockDisconnected struct {
		blockHeader []byte
	}

	// relevantTxAccepted is a notification that a transaction accepted by
	// mempool passed the client's transaction filter.
	relevantTxAccepted struct {
		transaction *wire.MsgTx
	}

	// reorganization is a notification that a reorg has happen with the new
	// old and new tip included.
	reorganization struct {
		oldHash   *chainhash.Hash
		oldHeight int64
		newHash   *chainhash.Hash
		newHeight int64
	}

	// winningTickets is a notification with the winning tickets (and the
	// block they are in.
	winningTickets struct {
		blockHash   *chainhash.Hash
		blockHeight int64
		tickets     []*chainhash.Hash
	}

	// missedTickets is a notifcation for tickets that have been missed.
	missedTickets struct {
		blockHash   *chainhash.Hash
		blockHeight int64
		tickets     []*chainhash.Hash
	}

	// stakeDifficulty is a notification for the current stake difficulty.
	stakeDifficulty struct {
		blockHash   *chainhash.Hash
		blockHeight int64
		stakeDiff   int64
	}
)

// notifications returns a channel of parsed notifications sent by the remote
// decred RPC server.  This channel must be continually read or the process
// may abort for running out memory, as unread notifications are queued for
// later reads.
func (c *RPCClient) notifications() <-chan interface{} {
	return c.dequeueNotification
}

// notificationsVoting returns a channel of parsed voting notifications sent
// by the remote RPC server.  This channel must be continually read or the
// process may abort for running out memory, as unread notifications are
// queued for later reads.
func (c *RPCClient) notificationsVoting() <-chan interface{} {
	return c.dequeueVotingNotification
}

func (c *RPCClient) onBlockConnected(header []byte, transactions [][]byte) {
	select {
	case c.enqueueNotification <- blockConnected{
		blockHeader:  header,
		transactions: transactions,
	}:
	case <-c.quit:
	}
}

func (c *RPCClient) onBlockDisconnected(header []byte) {
	select {
	case c.enqueueNotification <- blockDisconnected{
		blockHeader: header,
	}:
	case <-c.quit:
	}
}

func (c *RPCClient) onRelevantTxAccepted(transaction []byte) {
	msgTx := new(wire.MsgTx)
	err := msgTx.Deserialize(bytes.NewReader(transaction))
	if err != nil {
		log.Errorf("Failed to deserialize announced transaction: %v", err)
		return
	}
	select {
	case c.enqueueNotification <- relevantTxAccepted{
		transaction: msgTx,
	}:
	case <-c.quit:
	}
}

// onReorganization handles reorganization notifications and passes them
// downstream to the notifications queue.
func (c *RPCClient) onReorganization(oldHash *chainhash.Hash, oldHeight int32,
	newHash *chainhash.Hash, newHeight int32) {
	select {
	case c.enqueueNotification <- reorganization{
		oldHash,
		int64(oldHeight),
		newHash,
		int64(newHeight),
	}:
	case <-c.quit:
	}
}

// onWinningTickets handles winning tickets notifications data and passes it
// downstream to the notifications queue.
func (c *RPCClient) onWinningTickets(hash *chainhash.Hash, height int64, tickets []*chainhash.Hash) {
	select {
	case c.enqueueVotingNotification <- winningTickets{
		blockHash:   hash,
		blockHeight: height,
		tickets:     tickets,
	}:
	case <-c.quit:
	}
}

// onSpentAndMissedTickets handles missed tickets notifications data and passes
// it downstream to the notifications queue.
func (c *RPCClient) onSpentAndMissedTickets(blockHash *chainhash.Hash, height int64, sdiff int64, tickets map[chainhash.Hash]bool) {
	var missed []*chainhash.Hash

	// Copy the missed ticket hashes to a slice.
	for ticketHash, spent := range tickets {
		if !spent {
			ticketHash := ticketHash
			missed = append(missed, &ticketHash)
		}
	}

	if len(missed) == 0 {
		return
	}

	select {
	case c.enqueueNotification <- missedTickets{
		blockHash:   blockHash,
		blockHeight: height,
		tickets:     missed,
	}:
	case <-c.quit:
	}
}

// onStakeDifficulty handles stake difficulty notifications data and passes it
// downstream to the notification queue.
func (c *RPCClient) onStakeDifficulty(hash *chainhash.Hash,
	height int64,
	stakeDiff int64) {

	select {
	case c.enqueueNotification <- stakeDifficulty{
		hash,
		height,
		stakeDiff,
	}:
	case <-c.quit:
	}
}

// handler maintains a queue of notifications and the current state (best
// block) of the chain.
func (c *RPCClient) handler() {
	// TODO: Rather than leaving this as an unbounded queue for all types of
	// notifications, try dropping ones where a later enqueued notification
	// can fully invalidate one waiting to be processed.  For example,
	// blockconnected notifications for greater block heights can remove the
	// need to process earlier blockconnected notifications still waiting
	// here.

	var notifications []interface{}
	enqueue := c.enqueueNotification
	var dequeue chan interface{}
	var next interface{}
	pingChan := time.After(time.Minute)
	pingChanReset := make(chan (<-chan time.Time))
out:
	for {
		select {
		case n, ok := <-enqueue:
			if !ok {
				// If no notifications are queued for handling,
				// the queue is finished.
				if len(notifications) == 0 {
					break out
				}
				// nil channel so no more reads can occur.
				enqueue = nil
				continue
			}
			if len(notifications) == 0 {
				next = n
				dequeue = c.dequeueNotification
			}
			notifications = append(notifications, n)
			pingChan = time.After(time.Minute)

		case dequeue <- next:
			notifications[0] = nil
			notifications = notifications[1:]
			if len(notifications) != 0 {
				next = notifications[0]
			} else {
				// If no more notifications can be enqueued, the
				// queue is finished.
				if enqueue == nil {
					break out
				}
				dequeue = nil
			}

		case <-pingChan:
			// No notifications were received in the last 60s. Ensure the
			// connection is still active by making a new request to the server.
			//
			// This MUST wait for the response in a new goroutine so as to not
			// block channel sends enqueueing more notifications.  Doing so
			// would cause a deadlock and after the timeout expires, the client
			// would be shut down.
			//
			// TODO: A minute timeout is used to prevent the handler loop from
			// blocking here forever, but this is much larger than it needs to
			// be due to dcrd processing websocket requests synchronously (see
			// https://github.com/btcsuite/btcd/issues/504).  Decrease this to
			// something saner like 3s when the above issue is fixed.
			type sessionResult struct {
				err error
			}
			sessionResponse := make(chan sessionResult, 1)
			go func() {
				_, err := c.Session()
				sessionResponse <- sessionResult{err}
			}()
			go func() {
				select {
				case resp := <-sessionResponse:
					if resp.err != nil {
						log.Errorf("Failed to receive session "+
							"result: %v", resp.err)
						c.Stop()
					}
					pingChanReset <- time.After(time.Minute)

				case <-time.After(time.Minute):
					log.Errorf("Timeout waiting for session RPC")
					c.Stop()
				}
			}()

		case ch := <-pingChanReset:
			pingChan = ch

		case <-c.quit:
			break out
		}
	}

	c.Stop()
	close(c.dequeueNotification)
	c.wg.Done()
}

// handler maintains a queue of notifications and the current state (best
// block) of the chain.
func (c *RPCClient) handlerVoting() {
	var notifications []interface{}
	enqueue := c.enqueueVotingNotification
	var dequeue chan interface{}
	var next interface{}
out:
	for {
		select {
		case n, ok := <-enqueue:
			if !ok {
				// If no notifications are queued for handling,
				// the queue is finished.
				if len(notifications) == 0 {
					break out
				}
				// nil channel so no more reads can occur.
				enqueue = nil
				continue
			}
			if len(notifications) == 0 {
				next = n
				dequeue = c.dequeueVotingNotification
			}
			notifications = append(notifications, n)

		case dequeue <- next:
			notifications[0] = nil
			notifications = notifications[1:]
			if len(notifications) != 0 {
				next = notifications[0]
			} else {
				// If no more notifications can be enqueued, the
				// queue is finished.
				if enqueue == nil {
					break out
				}
				dequeue = nil
			}

		case <-c.quit:
			break out
		}
	}
	close(c.dequeueVotingNotification)
	c.wg.Done()
}

// POSTClient creates the equivalent HTTP POST dcrrpcclient.Client.
func (c *RPCClient) POSTClient() (*dcrrpcclient.Client, error) {
	configCopy := *c.connConfig
	configCopy.HTTPPostMode = true
	client, err := dcrrpcclient.New(&configCopy, nil)
	if err != nil {
		return nil, errors.E(errors.Op("chain.POSTClient"), err)
	}
	return client, nil
}
<commit_msg>Bump required dcrd JSON-RPC API version<commit_after>// Copyright (c) 2013-2015 The btcsuite developers
// Copyright (c) 2015-2016 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package chain

import (
	"bytes"
	"context"
	"sync"
	"time"

	"github.com/decred/dcrd/chaincfg"
	"github.com/decred/dcrd/chaincfg/chainhash"
	dcrrpcclient "github.com/decred/dcrd/rpcclient"
	"github.com/decred/dcrd/wire"
	"github.com/decred/dcrwallet/errors"
)

var requiredChainServerAPI = semver{major: 5, minor: 0, patch: 0}

// RPCClient represents a persistent client connection to a decred RPC server
// for information regarding the current best block chain.
type RPCClient struct {
	*dcrrpcclient.Client
	connConfig  *dcrrpcclient.ConnConfig // Work around unexported field
	chainParams *chaincfg.Params

	enqueueNotification       chan interface{}
	dequeueNotification       chan interface{}
	enqueueVotingNotification chan interface{}
	dequeueVotingNotification chan interface{}

	quit    chan struct{}
	wg      sync.WaitGroup
	started bool
	quitMtx sync.Mutex
}

// NewRPCClient creates a client connection to the server described by the
// connect string.  If disableTLS is false, the remote RPC certificate must be
// provided in the certs slice.  The connection is not established immediately,
// but must be done using the Start method.  If the remote server does not
// operate on the same decred network as described by the passed chain
// parameters, the connection will be disconnected.
func NewRPCClient(chainParams *chaincfg.Params, connect, user, pass string, certs []byte,
	disableTLS bool) (*RPCClient, error) {

	client := &RPCClient{
		connConfig: &dcrrpcclient.ConnConfig{
			Host:                 connect,
			Endpoint:             "ws",
			User:                 user,
			Pass:                 pass,
			Certificates:         certs,
			DisableAutoReconnect: true,
			DisableConnectOnNew:  true,
			DisableTLS:           disableTLS,
		},
		chainParams:               chainParams,
		enqueueNotification:       make(chan interface{}),
		dequeueNotification:       make(chan interface{}),
		enqueueVotingNotification: make(chan interface{}),
		dequeueVotingNotification: make(chan interface{}),
		quit:                      make(chan struct{}),
	}
	ntfnCallbacks := &dcrrpcclient.NotificationHandlers{
		OnBlockConnected:        client.onBlockConnected,
		OnBlockDisconnected:     client.onBlockDisconnected,
		OnRelevantTxAccepted:    client.onRelevantTxAccepted,
		OnReorganization:        client.onReorganization,
		OnWinningTickets:        client.onWinningTickets,
		OnSpentAndMissedTickets: client.onSpentAndMissedTickets,
		OnStakeDifficulty:       client.onStakeDifficulty,
	}
	rpcClient, err := dcrrpcclient.New(client.connConfig, ntfnCallbacks)
	if err != nil {
		return nil, err
	}
	client.Client = rpcClient
	return client, nil
}

// Start attempts to establish a client connection with the remote server.
// If successful, handler goroutines are started to process notifications
// sent by the server.  After a limited number of connection attempts, this
// function gives up, and therefore will not block forever waiting for the
// connection to be established to a server that may not exist.
func (c *RPCClient) Start(ctx context.Context, retry bool) (err error) {
	const op errors.Op = "rpcclient.Start"

	err = c.Client.Connect(ctx, retry)
	if err != nil {
		return errors.E(op, errors.IO, err)
	}

	defer func() {
		if err != nil {
			c.Disconnect()
		}
	}()

	// Verify that the server is running on the expected network.
	net, err := c.Client.GetCurrentNet()
	if err != nil {
		return errors.E(op, errors.E(errors.Op("dcrd.jsonrpc.getcurrentnet"), err))
	}
	if net != c.chainParams.Net {
		return errors.E(op, "mismatched networks")
	}

	// Ensure the RPC server has a compatible API version.
	var serverAPI semver
	versions, err := c.Client.Version()
	if err == nil {
		versionResult := versions["dcrdjsonrpcapi"]
		serverAPI = semver{
			major: versionResult.Major,
			minor: versionResult.Minor,
			patch: versionResult.Patch,
		}
	}
	if !semverCompatible(requiredChainServerAPI, serverAPI) {
		return errors.E(op, errors.Errorf("advertised API version %v incompatible "+
			"with required version %v", serverAPI, requiredChainServerAPI))
	}

	c.quitMtx.Lock()
	c.started = true
	c.quitMtx.Unlock()

	c.wg.Add(2)
	go c.handler()
	go c.handlerVoting()
	return nil
}

// Stop disconnects the client and signals the shutdown of all goroutines
// started by Start.
func (c *RPCClient) Stop() {
	c.quitMtx.Lock()
	select {
	case <-c.quit:
	default:
		close(c.quit)
		c.Client.Shutdown()

		if !c.started {
			close(c.dequeueNotification)
			close(c.dequeueVotingNotification)
		}
	}
	c.quitMtx.Unlock()
}

// WaitForShutdown blocks until both the client has finished disconnecting
// and all handlers have exited.
func (c *RPCClient) WaitForShutdown() {
	c.Client.WaitForShutdown()
	c.wg.Wait()
}

// Notification types.  These are defined here and processed from from reading
// a notificationChan to avoid handling these notifications directly in
// dcrrpcclient callbacks, which isn't very Go-like and doesn't allow
// blocking client calls.
type (
	// blockConnected is a notification for a newly-attached block to the
	// best chain.
	blockConnected struct {
		blockHeader  []byte
		transactions [][]byte
	}

	// blockDisconnected is a notifcation that the block described by the header
	// was reorganized out of the best chain.
	blockDisconnected struct {
		blockHeader []byte
	}

	// relevantTxAccepted is a notification that a transaction accepted by
	// mempool passed the client's transaction filter.
	relevantTxAccepted struct {
		transaction *wire.MsgTx
	}

	// reorganization is a notification that a reorg has happen with the new
	// old and new tip included.
	reorganization struct {
		oldHash   *chainhash.Hash
		oldHeight int64
		newHash   *chainhash.Hash
		newHeight int64
	}

	// winningTickets is a notification with the winning tickets (and the
	// block they are in.
	winningTickets struct {
		blockHash   *chainhash.Hash
		blockHeight int64
		tickets     []*chainhash.Hash
	}

	// missedTickets is a notifcation for tickets that have been missed.
	missedTickets struct {
		blockHash   *chainhash.Hash
		blockHeight int64
		tickets     []*chainhash.Hash
	}

	// stakeDifficulty is a notification for the current stake difficulty.
	stakeDifficulty struct {
		blockHash   *chainhash.Hash
		blockHeight int64
		stakeDiff   int64
	}
)

// notifications returns a channel of parsed notifications sent by the remote
// decred RPC server.  This channel must be continually read or the process
// may abort for running out memory, as unread notifications are queued for
// later reads.
func (c *RPCClient) notifications() <-chan interface{} {
	return c.dequeueNotification
}

// notificationsVoting returns a channel of parsed voting notifications sent
// by the remote RPC server.  This channel must be continually read or the
// process may abort for running out memory, as unread notifications are
// queued for later reads.
func (c *RPCClient) notificationsVoting() <-chan interface{} {
	return c.dequeueVotingNotification
}

func (c *RPCClient) onBlockConnected(header []byte, transactions [][]byte) {
	select {
	case c.enqueueNotification <- blockConnected{
		blockHeader:  header,
		transactions: transactions,
	}:
	case <-c.quit:
	}
}

func (c *RPCClient) onBlockDisconnected(header []byte) {
	select {
	case c.enqueueNotification <- blockDisconnected{
		blockHeader: header,
	}:
	case <-c.quit:
	}
}

func (c *RPCClient) onRelevantTxAccepted(transaction []byte) {
	msgTx := new(wire.MsgTx)
	err := msgTx.Deserialize(bytes.NewReader(transaction))
	if err != nil {
		log.Errorf("Failed to deserialize announced transaction: %v", err)
		return
	}
	select {
	case c.enqueueNotification <- relevantTxAccepted{
		transaction: msgTx,
	}:
	case <-c.quit:
	}
}

// onReorganization handles reorganization notifications and passes them
// downstream to the notifications queue.
func (c *RPCClient) onReorganization(oldHash *chainhash.Hash, oldHeight int32,
	newHash *chainhash.Hash, newHeight int32) {
	select {
	case c.enqueueNotification <- reorganization{
		oldHash,
		int64(oldHeight),
		newHash,
		int64(newHeight),
	}:
	case <-c.quit:
	}
}

// onWinningTickets handles winning tickets notifications data and passes it
// downstream to the notifications queue.
func (c *RPCClient) onWinningTickets(hash *chainhash.Hash, height int64, tickets []*chainhash.Hash) {
	select {
	case c.enqueueVotingNotification <- winningTickets{
		blockHash:   hash,
		blockHeight: height,
		tickets:     tickets,
	}:
	case <-c.quit:
	}
}

// onSpentAndMissedTickets handles missed tickets notifications data and passes
// it downstream to the notifications queue.
func (c *RPCClient) onSpentAndMissedTickets(blockHash *chainhash.Hash, height int64, sdiff int64, tickets map[chainhash.Hash]bool) {
	var missed []*chainhash.Hash

	// Copy the missed ticket hashes to a slice.
	for ticketHash, spent := range tickets {
		if !spent {
			ticketHash := ticketHash
			missed = append(missed, &ticketHash)
		}
	}

	if len(missed) == 0 {
		return
	}

	select {
	case c.enqueueNotification <- missedTickets{
		blockHash:   blockHash,
		blockHeight: height,
		tickets:     missed,
	}:
	case <-c.quit:
	}
}

// onStakeDifficulty handles stake difficulty notifications data and passes it
// downstream to the notification queue.
func (c *RPCClient) onStakeDifficulty(hash *chainhash.Hash,
	height int64,
	stakeDiff int64) {

	select {
	case c.enqueueNotification <- stakeDifficulty{
		hash,
		height,
		stakeDiff,
	}:
	case <-c.quit:
	}
}

// handler maintains a queue of notifications and the current state (best
// block) of the chain.
func (c *RPCClient) handler() {
	// TODO: Rather than leaving this as an unbounded queue for all types of
	// notifications, try dropping ones where a later enqueued notification
	// can fully invalidate one waiting to be processed.  For example,
	// blockconnected notifications for greater block heights can remove the
	// need to process earlier blockconnected notifications still waiting
	// here.

	var notifications []interface{}
	enqueue := c.enqueueNotification
	var dequeue chan interface{}
	var next interface{}
	pingChan := time.After(time.Minute)
	pingChanReset := make(chan (<-chan time.Time))
out:
	for {
		select {
		case n, ok := <-enqueue:
			if !ok {
				// If no notifications are queued for handling,
				// the queue is finished.
				if len(notifications) == 0 {
					break out
				}
				// nil channel so no more reads can occur.
				enqueue = nil
				continue
			}
			if len(notifications) == 0 {
				next = n
				dequeue = c.dequeueNotification
			}
			notifications = append(notifications, n)
			pingChan = time.After(time.Minute)

		case dequeue <- next:
			notifications[0] = nil
			notifications = notifications[1:]
			if len(notifications) != 0 {
				next = notifications[0]
			} else {
				// If no more notifications can be enqueued, the
				// queue is finished.
				if enqueue == nil {
					break out
				}
				dequeue = nil
			}

		case <-pingChan:
			// No notifications were received in the last 60s. Ensure the
			// connection is still active by making a new request to the server.
			//
			// This MUST wait for the response in a new goroutine so as to not
			// block channel sends enqueueing more notifications.  Doing so
			// would cause a deadlock and after the timeout expires, the client
			// would be shut down.
			//
			// TODO: A minute timeout is used to prevent the handler loop from
			// blocking here forever, but this is much larger than it needs to
			// be due to dcrd processing websocket requests synchronously (see
			// https://github.com/btcsuite/btcd/issues/504).  Decrease this to
			// something saner like 3s when the above issue is fixed.
			type sessionResult struct {
				err error
			}
			sessionResponse := make(chan sessionResult, 1)
			go func() {
				_, err := c.Session()
				sessionResponse <- sessionResult{err}
			}()
			go func() {
				select {
				case resp := <-sessionResponse:
					if resp.err != nil {
						log.Errorf("Failed to receive session "+
							"result: %v", resp.err)
						c.Stop()
					}
					pingChanReset <- time.After(time.Minute)

				case <-time.After(time.Minute):
					log.Errorf("Timeout waiting for session RPC")
					c.Stop()
				}
			}()

		case ch := <-pingChanReset:
			pingChan = ch

		case <-c.quit:
			break out
		}
	}

	c.Stop()
	close(c.dequeueNotification)
	c.wg.Done()
}

// handler maintains a queue of notifications and the current state (best
// block) of the chain.
func (c *RPCClient) handlerVoting() {
	var notifications []interface{}
	enqueue := c.enqueueVotingNotification
	var dequeue chan interface{}
	var next interface{}
out:
	for {
		select {
		case n, ok := <-enqueue:
			if !ok {
				// If no notifications are queued for handling,
				// the queue is finished.
				if len(notifications) == 0 {
					break out
				}
				// nil channel so no more reads can occur.
				enqueue = nil
				continue
			}
			if len(notifications) == 0 {
				next = n
				dequeue = c.dequeueVotingNotification
			}
			notifications = append(notifications, n)

		case dequeue <- next:
			notifications[0] = nil
			notifications = notifications[1:]
			if len(notifications) != 0 {
				next = notifications[0]
			} else {
				// If no more notifications can be enqueued, the
				// queue is finished.
				if enqueue == nil {
					break out
				}
				dequeue = nil
			}

		case <-c.quit:
			break out
		}
	}
	close(c.dequeueVotingNotification)
	c.wg.Done()
}

// POSTClient creates the equivalent HTTP POST dcrrpcclient.Client.
func (c *RPCClient) POSTClient() (*dcrrpcclient.Client, error) {
	configCopy := *c.connConfig
	configCopy.HTTPPostMode = true
	client, err := dcrrpcclient.New(&configCopy, nil)
	if err != nil {
		return nil, errors.E(errors.Op("chain.POSTClient"), err)
	}
	return client, nil
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package tests_test

import (
	"fmt"
	"strings"
	"time"

	expect "github.com/google/goexpect"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/client-go/kubecli"
	"kubevirt.io/kubevirt/tests"
	cd "kubevirt.io/kubevirt/tests/containerdisk"
)

var _ = Describe("[rfe_id:588][crit:medium][vendor:cnv-qe@redhat.com][level:component]ContainerDisk", func() {

	var virtClient kubecli.KubevirtClient
	var err error

	BeforeEach(func() {
		virtClient, err = kubecli.GetKubevirtClient()
		tests.PanicOnError(err)

		tests.BeforeTestCleanup()
	})

	LaunchVMI := func(vmi *v1.VirtualMachineInstance) runtime.Object {
		By("Starting a VirtualMachineInstance")
		obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
		Expect(err).To(BeNil())
		return obj
	}

	VerifyContainerDiskVMI := func(vmi *v1.VirtualMachineInstance, obj runtime.Object, ignoreWarnings bool) {
		_, ok := obj.(*v1.VirtualMachineInstance)
		Expect(ok).To(BeTrue(), "Object is not of type *v1.VirtualMachineInstance")
		if ignoreWarnings == true {
			tests.WaitForSuccessfulVMIStartIgnoreWarnings(obj)
		} else {
			tests.WaitForSuccessfulVMIStart(obj)
		}

		// Verify Registry Disks are Online
		pods, err := virtClient.CoreV1().Pods(tests.NamespaceTestDefault).List(tests.UnfinishedVMIPodSelector(vmi))
		Expect(err).To(BeNil())

		By("Checking the number of VirtualMachineInstance disks")
		disksFound := 0
		for _, pod := range pods.Items {
			if pod.ObjectMeta.DeletionTimestamp != nil {
				continue
			}
			for _, containerStatus := range pod.Status.ContainerStatuses {
				if strings.HasPrefix(containerStatus.Name, "volume") == false {
					// only check readiness of disk containers
					continue
				}
				disksFound++
			}
			break
		}
		Expect(disksFound).To(Equal(1))
	}

	table.DescribeTable("should", func(image string, policy k8sv1.PullPolicy, expectedPolicy k8sv1.PullPolicy) {
		vmi := tests.NewRandomVMIWithEphemeralDisk(image)
		vmi.Spec.Volumes[0].ContainerDisk.ImagePullPolicy = policy
		vmi = tests.RunVMIAndExpectScheduling(vmi, 60)
		Expect(vmi.Spec.Volumes[0].ContainerDisk.ImagePullPolicy).To(Equal(expectedPolicy))
		pod := tests.GetPodByVirtualMachineInstance(vmi, vmi.Namespace)
		container := tests.GetContainerDiskContainerOfPod(pod, vmi.Spec.Volumes[0].Name)
		Expect(container.ImagePullPolicy).To(Equal(expectedPolicy))
	},
		table.Entry("[test_id:3246]generate and set Always pull policy", "test", k8sv1.PullPolicy(""), k8sv1.PullAlways),
		table.Entry("[test_id:3247]generate and set Always pull policy", "test:latest", k8sv1.PullPolicy(""), k8sv1.PullAlways),
		table.Entry("[test_id:3248]generate and set IfNotPresent pull policy", "test@sha256:9c2b78e11c25b3fd0b24b0ed684a112052dff03eee4ca4bdcc4f3168f9a14396", k8sv1.PullPolicy(""), k8sv1.PullIfNotPresent),
		table.Entry("[test_id:3249]pass through Never pull policy to the pod", "test@sha256:9c2b78e11c25b3fd0b24b0ed684a112052dff03eee4ca4bdcc4f3168f9a14396", k8sv1.PullNever, k8sv1.PullNever),
		table.Entry("[test_id:3250]pass through IfNotPresent pull policy to the pod", "test:latest", k8sv1.PullIfNotPresent, k8sv1.PullIfNotPresent),
	)

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting and stopping the same VirtualMachineInstance", func() {
		Context("with ephemeral registry disk", func() {
			It("[test_id:1463]should success multiple times", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(cd.ContainerDiskFor(cd.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
				num := 2
				for i := 0; i < num; i++ {
					By("Starting the VirtualMachineInstance")
					obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
					Expect(err).To(BeNil())
					tests.WaitForSuccessfulVMIStart(obj)

					By("Stopping the VirtualMachineInstance")
					_, err = virtClient.RestClient().Delete().Resource("virtualmachineinstances").Namespace(vmi.GetObjectMeta().GetNamespace()).Name(vmi.GetObjectMeta().GetName()).Do().Get()
					Expect(err).To(BeNil())
					By("Waiting until the VirtualMachineInstance is gone")
					tests.WaitForVirtualMachineToDisappearWithTimeout(vmi, 120)
				}
			})
		})
	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting a VirtualMachineInstance", func() {
		Context("with ephemeral registry disk", func() {
			It("[test_id:1464]should not modify the spec on status update", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(cd.ContainerDiskFor(cd.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
				v1.SetObjectDefaults_VirtualMachineInstance(vmi)

				By("Starting the VirtualMachineInstance")
				vmi, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
				Expect(err).To(BeNil())
				tests.WaitForSuccessfulVMIStart(vmi)
				startedVMI, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.ObjectMeta.Name, &metav1.GetOptions{})
				Expect(err).To(BeNil())
				By("Checking that the VirtualMachineInstance spec did not change")
				Expect(startedVMI.Spec).To(Equal(vmi.Spec))
			})
		})
	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting multiple VMIs", func() {
		Context("with ephemeral registry disk", func() {
			It("[test_id:1465]should success", func() {
				num := 5
				vmis := make([]*v1.VirtualMachineInstance, 0, num)
				objs := make([]runtime.Object, 0, num)
				for i := 0; i < num; i++ {
					vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(cd.ContainerDiskFor(cd.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
					// FIXME if we give too much ram, the vmis really boot and eat all our memory (cache?)
					vmi.Spec.Domain.Resources.Requests[k8sv1.ResourceMemory] = resource.MustParse("1M")
					obj := LaunchVMI(vmi)
					vmis = append(vmis, vmi)
					objs = append(objs, obj)
				}

				for idx, vmi := range vmis {
					// TODO once networking is implemented properly set ignoreWarnings == false here.
					// We have to ignore warnings because VMIs started in parallel
					// may cause libvirt to fail to create the macvtap device in
					// the host network.
					// The new network implementation we're working on should resolve this.
					// NOTE the VirtualMachineInstance still starts successfully regardless of this warning.
					// It just requires virt-handler to retry the Start command at the moment.
					VerifyContainerDiskVMI(vmi, objs[idx], true)
				}
			}) // Timeout is long because this test involves multiple parallel VirtualMachineInstance launches.
		})
	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting from custom image location", func() {
		Context("with disk at /custom-disk/downloaded", func() {
			It("[test_id:1466]should boot normally", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(cd.ContainerDiskFor(cd.ContainerDiskCirrosCustomLocation), "#!/bin/bash\necho 'hello'\n")
				for ind, volume := range vmi.Spec.Volumes {
					if volume.ContainerDisk != nil {
						vmi.Spec.Volumes[ind].ContainerDisk.Path = "/custom-disk/downloaded"
					}
				}
				By("Starting the VirtualMachineInstance")
				obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
				Expect(err).To(BeNil())
				tests.WaitForSuccessfulVMIStart(obj)
			})
		})

	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting with virtio-win", func() {
		Context("with virtio-win as secondary disk", func() {
			It("[test_id:1467]should boot and have the virtio as sata CDROM", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(cd.ContainerDiskFor(cd.ContainerDiskAlpine))
				tests.AddEphemeralCdrom(vmi, "disk4", "sata", cd.ContainerDiskFor(cd.ContainerDiskVirtio))

				By("Starting the VirtualMachineInstance")
				obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
				Expect(err).To(BeNil(), "expected vmi to start with no problem")
				tests.WaitForSuccessfulVMIStart(obj)

				By("Checking whether the second disk really contains virtio drivers")
				expecter, err := tests.LoggedInAlpineExpecter(vmi)
				Expect(err).ToNot(HaveOccurred(), "expected alpine to login properly")
				defer expecter.Close()

				_, err = expecter.ExpectBatch([]expect.Batcher{
					// mount virtio cdrom and check files are there
					&expect.BSnd{S: "mount -t iso9600 /dev/cdrom\n"},
					&expect.BSnd{S: "echo $?\n"},
					&expect.BExp{R: tests.RetValue("0")},
					&expect.BSnd{S: "cd /media/cdrom\n"},
					&expect.BSnd{S: "ls virtio-win_license.txt guest-agent\n"},
					&expect.BSnd{S: "echo $?\n"},
					&expect.BExp{R: tests.RetValue("0")},
				}, 200*time.Second)
				Expect(err).ToNot(HaveOccurred(), "expected virtio files to be mounted properly")
			})
		})
	})

	Describe("[rfe_id:4052][crit:high][vendor:cnv-qe@redhat.com][level:component]VMI disk permissions", func() {
		Context("with ephemeral registry disk", func() {
			It("[test_id:4299]should not have world write permissions", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(cd.ContainerDiskFor(cd.ContainerDiskAlpine))

				By("Starting a New VMI")
				vmi, err = virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
				Expect(err).ToNot(HaveOccurred())
				tests.WaitForSuccessfulVMIStart(vmi)

				By("Ensuring VMI is running by logging in")
				tests.WaitUntilVMIReady(vmi, tests.LoggedInAlpineExpecter)

				By("Fetching virt-launcher Pod")
				pod := tests.GetPodByVirtualMachineInstance(vmi, tests.NamespaceTestDefault)

				writableImagePath := fmt.Sprintf("/var/run/kubevirt-ephemeral-disks/disk-data/%v/disk.qcow2", vmi.Spec.Domain.Devices.Disks[0].Name)

				writableImageOctalMode, err := tests.ExecuteCommandOnPod(
					virtClient,
					pod,
					"compute",
					[]string{"/usr/bin/bash", "-c", fmt.Sprintf("stat -c %%a %s", writableImagePath)},
				)
				Expect(err).ToNot(HaveOccurred())

				By("Checking the writable Image Octal mode")
				Expect(strings.Trim(writableImageOctalMode, "\n")).To(Equal("640"), "Octal Mode of writable Image should be 640")

				readonlyImageOctalMode, err := tests.ExecuteCommandOnPod(
					virtClient,
					pod,
					"compute",
					[]string{"/usr/bin/bash", "-c", "stat -c %a /var/run/kubevirt/container-disks/disk_0.img"},
				)
				Expect(err).ToNot(HaveOccurred())

				By("Checking the read-only Image Octal mode")
				Expect(strings.Trim(readonlyImageOctalMode, "\n")).To(Equal("444"), "Octal Mode of read-only Image should be 444")
			})
		})
	})
})
<commit_msg>Add the first conformance test<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package tests_test

import (
	"fmt"
	"strings"
	"time"

	expect "github.com/google/goexpect"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/client-go/kubecli"
	"kubevirt.io/kubevirt/tests"
	cd "kubevirt.io/kubevirt/tests/containerdisk"
)

var _ = Describe("[rfe_id:588][crit:medium][vendor:cnv-qe@redhat.com][level:component]ContainerDisk", func() {

	var virtClient kubecli.KubevirtClient
	var err error

	BeforeEach(func() {
		virtClient, err = kubecli.GetKubevirtClient()
		tests.PanicOnError(err)

		tests.BeforeTestCleanup()
	})

	LaunchVMI := func(vmi *v1.VirtualMachineInstance) runtime.Object {
		By("Starting a VirtualMachineInstance")
		obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
		Expect(err).To(BeNil())
		return obj
	}

	VerifyContainerDiskVMI := func(vmi *v1.VirtualMachineInstance, obj runtime.Object, ignoreWarnings bool) {
		_, ok := obj.(*v1.VirtualMachineInstance)
		Expect(ok).To(BeTrue(), "Object is not of type *v1.VirtualMachineInstance")
		if ignoreWarnings == true {
			tests.WaitForSuccessfulVMIStartIgnoreWarnings(obj)
		} else {
			tests.WaitForSuccessfulVMIStart(obj)
		}

		// Verify Registry Disks are Online
		pods, err := virtClient.CoreV1().Pods(tests.NamespaceTestDefault).List(tests.UnfinishedVMIPodSelector(vmi))
		Expect(err).To(BeNil())

		By("Checking the number of VirtualMachineInstance disks")
		disksFound := 0
		for _, pod := range pods.Items {
			if pod.ObjectMeta.DeletionTimestamp != nil {
				continue
			}
			for _, containerStatus := range pod.Status.ContainerStatuses {
				if strings.HasPrefix(containerStatus.Name, "volume") == false {
					// only check readiness of disk containers
					continue
				}
				disksFound++
			}
			break
		}
		Expect(disksFound).To(Equal(1))
	}

	table.DescribeTable("should", func(image string, policy k8sv1.PullPolicy, expectedPolicy k8sv1.PullPolicy) {
		vmi := tests.NewRandomVMIWithEphemeralDisk(image)
		vmi.Spec.Volumes[0].ContainerDisk.ImagePullPolicy = policy
		vmi = tests.RunVMIAndExpectScheduling(vmi, 60)
		Expect(vmi.Spec.Volumes[0].ContainerDisk.ImagePullPolicy).To(Equal(expectedPolicy))
		pod := tests.GetPodByVirtualMachineInstance(vmi, vmi.Namespace)
		container := tests.GetContainerDiskContainerOfPod(pod, vmi.Spec.Volumes[0].Name)
		Expect(container.ImagePullPolicy).To(Equal(expectedPolicy))
	},
		table.Entry("[test_id:3246]generate and set Always pull policy", "test", k8sv1.PullPolicy(""), k8sv1.PullAlways),
		table.Entry("[test_id:3247]generate and set Always pull policy", "test:latest", k8sv1.PullPolicy(""), k8sv1.PullAlways),
		table.Entry("[test_id:3248]generate and set IfNotPresent pull policy", "test@sha256:9c2b78e11c25b3fd0b24b0ed684a112052dff03eee4ca4bdcc4f3168f9a14396", k8sv1.PullPolicy(""), k8sv1.PullIfNotPresent),
		table.Entry("[test_id:3249]pass through Never pull policy to the pod", "test@sha256:9c2b78e11c25b3fd0b24b0ed684a112052dff03eee4ca4bdcc4f3168f9a14396", k8sv1.PullNever, k8sv1.PullNever),
		table.Entry("[test_id:3250]pass through IfNotPresent pull policy to the pod", "test:latest", k8sv1.PullIfNotPresent, k8sv1.PullIfNotPresent),
	)

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting and stopping the same VirtualMachineInstance", func() {
		Context("with ephemeral registry disk", func() {
			It("[test_id:1463][Conformance] should success multiple times", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(cd.ContainerDiskFor(cd.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
				num := 2
				for i := 0; i < num; i++ {
					By("Starting the VirtualMachineInstance")
					obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
					Expect(err).To(BeNil())
					tests.WaitForSuccessfulVMIStart(obj)

					By("Stopping the VirtualMachineInstance")
					_, err = virtClient.RestClient().Delete().Resource("virtualmachineinstances").Namespace(vmi.GetObjectMeta().GetNamespace()).Name(vmi.GetObjectMeta().GetName()).Do().Get()
					Expect(err).To(BeNil())
					By("Waiting until the VirtualMachineInstance is gone")
					tests.WaitForVirtualMachineToDisappearWithTimeout(vmi, 120)
				}
			})
		})
	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting a VirtualMachineInstance", func() {
		Context("with ephemeral registry disk", func() {
			It("[test_id:1464]should not modify the spec on status update", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(cd.ContainerDiskFor(cd.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
				v1.SetObjectDefaults_VirtualMachineInstance(vmi)

				By("Starting the VirtualMachineInstance")
				vmi, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
				Expect(err).To(BeNil())
				tests.WaitForSuccessfulVMIStart(vmi)
				startedVMI, err := virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Get(vmi.ObjectMeta.Name, &metav1.GetOptions{})
				Expect(err).To(BeNil())
				By("Checking that the VirtualMachineInstance spec did not change")
				Expect(startedVMI.Spec).To(Equal(vmi.Spec))
			})
		})
	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting multiple VMIs", func() {
		Context("with ephemeral registry disk", func() {
			It("[test_id:1465]should success", func() {
				num := 5
				vmis := make([]*v1.VirtualMachineInstance, 0, num)
				objs := make([]runtime.Object, 0, num)
				for i := 0; i < num; i++ {
					vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(cd.ContainerDiskFor(cd.ContainerDiskCirros), "#!/bin/bash\necho 'hello'\n")
					// FIXME if we give too much ram, the vmis really boot and eat all our memory (cache?)
					vmi.Spec.Domain.Resources.Requests[k8sv1.ResourceMemory] = resource.MustParse("1M")
					obj := LaunchVMI(vmi)
					vmis = append(vmis, vmi)
					objs = append(objs, obj)
				}

				for idx, vmi := range vmis {
					// TODO once networking is implemented properly set ignoreWarnings == false here.
					// We have to ignore warnings because VMIs started in parallel
					// may cause libvirt to fail to create the macvtap device in
					// the host network.
					// The new network implementation we're working on should resolve this.
					// NOTE the VirtualMachineInstance still starts successfully regardless of this warning.
					// It just requires virt-handler to retry the Start command at the moment.
					VerifyContainerDiskVMI(vmi, objs[idx], true)
				}
			}) // Timeout is long because this test involves multiple parallel VirtualMachineInstance launches.
		})
	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting from custom image location", func() {
		Context("with disk at /custom-disk/downloaded", func() {
			It("[test_id:1466]should boot normally", func() {
				vmi := tests.NewRandomVMIWithEphemeralDiskAndUserdata(cd.ContainerDiskFor(cd.ContainerDiskCirrosCustomLocation), "#!/bin/bash\necho 'hello'\n")
				for ind, volume := range vmi.Spec.Volumes {
					if volume.ContainerDisk != nil {
						vmi.Spec.Volumes[ind].ContainerDisk.Path = "/custom-disk/downloaded"
					}
				}
				By("Starting the VirtualMachineInstance")
				obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
				Expect(err).To(BeNil())
				tests.WaitForSuccessfulVMIStart(obj)
			})
		})

	})

	Describe("[rfe_id:273][crit:medium][vendor:cnv-qe@redhat.com][level:component]Starting with virtio-win", func() {
		Context("with virtio-win as secondary disk", func() {
			It("[test_id:1467]should boot and have the virtio as sata CDROM", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(cd.ContainerDiskFor(cd.ContainerDiskAlpine))
				tests.AddEphemeralCdrom(vmi, "disk4", "sata", cd.ContainerDiskFor(cd.ContainerDiskVirtio))

				By("Starting the VirtualMachineInstance")
				obj, err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Get()
				Expect(err).To(BeNil(), "expected vmi to start with no problem")
				tests.WaitForSuccessfulVMIStart(obj)

				By("Checking whether the second disk really contains virtio drivers")
				expecter, err := tests.LoggedInAlpineExpecter(vmi)
				Expect(err).ToNot(HaveOccurred(), "expected alpine to login properly")
				defer expecter.Close()

				_, err = expecter.ExpectBatch([]expect.Batcher{
					// mount virtio cdrom and check files are there
					&expect.BSnd{S: "mount -t iso9600 /dev/cdrom\n"},
					&expect.BSnd{S: "echo $?\n"},
					&expect.BExp{R: tests.RetValue("0")},
					&expect.BSnd{S: "cd /media/cdrom\n"},
					&expect.BSnd{S: "ls virtio-win_license.txt guest-agent\n"},
					&expect.BSnd{S: "echo $?\n"},
					&expect.BExp{R: tests.RetValue("0")},
				}, 200*time.Second)
				Expect(err).ToNot(HaveOccurred(), "expected virtio files to be mounted properly")
			})
		})
	})

	Describe("[rfe_id:4052][crit:high][vendor:cnv-qe@redhat.com][level:component]VMI disk permissions", func() {
		Context("with ephemeral registry disk", func() {
			It("[test_id:4299]should not have world write permissions", func() {
				vmi := tests.NewRandomVMIWithEphemeralDisk(cd.ContainerDiskFor(cd.ContainerDiskAlpine))

				By("Starting a New VMI")
				vmi, err = virtClient.VirtualMachineInstance(tests.NamespaceTestDefault).Create(vmi)
				Expect(err).ToNot(HaveOccurred())
				tests.WaitForSuccessfulVMIStart(vmi)

				By("Ensuring VMI is running by logging in")
				tests.WaitUntilVMIReady(vmi, tests.LoggedInAlpineExpecter)

				By("Fetching virt-launcher Pod")
				pod := tests.GetPodByVirtualMachineInstance(vmi, tests.NamespaceTestDefault)

				writableImagePath := fmt.Sprintf("/var/run/kubevirt-ephemeral-disks/disk-data/%v/disk.qcow2", vmi.Spec.Domain.Devices.Disks[0].Name)

				writableImageOctalMode, err := tests.ExecuteCommandOnPod(
					virtClient,
					pod,
					"compute",
					[]string{"/usr/bin/bash", "-c", fmt.Sprintf("stat -c %%a %s", writableImagePath)},
				)
				Expect(err).ToNot(HaveOccurred())

				By("Checking the writable Image Octal mode")
				Expect(strings.Trim(writableImageOctalMode, "\n")).To(Equal("640"), "Octal Mode of writable Image should be 640")

				readonlyImageOctalMode, err := tests.ExecuteCommandOnPod(
					virtClient,
					pod,
					"compute",
					[]string{"/usr/bin/bash", "-c", "stat -c %a /var/run/kubevirt/container-disks/disk_0.img"},
				)
				Expect(err).ToNot(HaveOccurred())

				By("Checking the read-only Image Octal mode")
				Expect(strings.Trim(readonlyImageOctalMode, "\n")).To(Equal("444"), "Octal Mode of read-only Image should be 444")
			})
		})
	})
})
<|endoftext|>


<commit_before>// httpsignatures is a golang implementation of the http-signatures spec
// found at https://tools.ietf.org/html/draft-cavage-http-signatures
package httpsignatures

import (
	"crypto/hmac"
	"encoding/base64"
	"errors"
	"fmt"
	"net/http"
	"regexp"
	"strings"
)

const (
	headerSignature     = "Signature"
	headerAuthorization = "Authorization"

	RequestTarget = "(request-target)"

	authScheme = "Signature "
)

var (
	ErrorNoSignatureHeader = errors.New("No Signature header found in request")

	signatureRegex = regexp.MustCompile(`(\w+)="([^"]*)"`)
)

// Signature is the hashed key + headers, either from a request or a signer
type Signature struct {
	KeyID     string
	Algorithm *Algorithm
	Headers   HeaderList
	Signature string
}

// FromRequest creates a new Signature from the Request
// both Signature and Authorization http headers are supported.
func FromRequest(r *http.Request) (*Signature, error) {
	if s, ok := r.Header[headerSignature]; ok {
		return FromString(s[0])
	}
	if a, ok := r.Header[headerAuthorization]; ok {
		return FromString(strings.TrimPrefix(a[0], authScheme))
	}
	return nil, ErrorNoSignatureHeader
}

// FromString creates a new Signature from its encoded form,
// eg `keyId="a",algorithm="b",headers="c",signature="d"`
func FromString(in string) (*Signature, error) {
	var res Signature = Signature{}
	var key string
	var value string

	for _, m := range signatureRegex.FindAllStringSubmatch(in, -1) {
		key = m[1]
		value = m[2]

		if key == "keyId" {
			res.KeyID = value
		} else if key == "algorithm" {
			alg, err := algorithmFromString(value)
			if err != nil {
				return nil, err
			}
			res.Algorithm = alg
		} else if key == "headers" {
			res.Headers = headerListFromString(value)
		} else if key == "signature" {
			res.Signature = value
		} else {
			return nil, errors.New(fmt.Sprintf("Unexpected key in signature '%s'", key))
		}
	}

	if len(res.Signature) == 0 {
		return nil, errors.New("Missing signature")
	}

	if len(res.KeyID) == 0 {
		return nil, errors.New("Missing keyId")
	}

	if res.Algorithm == nil {
		return nil, errors.New("Missing algorithm")
	}

	return &res, nil
}

// String returns the encoded form of the Signature
func (s Signature) String() string {
	str := fmt.Sprintf(
		`keyId="%s",algorithm="%s",signature="%s"`,
		s.KeyID,
		s.Algorithm.name,
		s.Signature,
	)

	if len(s.Headers) > 0 {
		str += fmt.Sprintf(`,headers="%s"`, s.Headers.String())
	}

	return str
}

func (s Signature) calculateSignature(key string, r *http.Request) (string, error) {
	hash := hmac.New(s.Algorithm.hash, []byte(key))

	signingString, err := s.Headers.signingString(r)
	if err != nil {
		return "", err
	}

	hash.Write([]byte(signingString))

	return base64.StdEncoding.EncodeToString(hash.Sum(nil)), nil
}

// Sign this signature using the given key
func (s *Signature) sign(key string, r *http.Request) error {
	sig, err := s.calculateSignature(key, r)
	if err != nil {
		return err
	}

	s.Signature = sig
	return nil
}

// IsValid validates this signature for the given key
func (s Signature) IsValid(key string, r *http.Request) bool {
	if !s.Headers.hasDate() {
		return false
	}

	sig, err := s.calculateSignature(key, r)
	if err != nil {
		return false
	}
	return s.Signature == sig
}

type HeaderList []string

func headerListFromString(list string) HeaderList {
	return strings.Split(strings.ToLower(string(list)), " ")
}

func (h HeaderList) String() string {
	return strings.ToLower(strings.Join(h, " "))
}

func (h HeaderList) hasDate() bool {
	for _, header := range h {
		if header == "date" {
			return true
		}
	}

	return false
}

func (h HeaderList) signingString(req *http.Request) (string, error) {
	lines := []string{}

	for _, header := range h {
		if header == RequestTarget {
			lines = append(lines, requestTargetLine(req))
		} else {
			line, err := headerLine(req, header)
			if err != nil {
				return "", err
			}
			lines = append(lines, line)
		}
	}

	return strings.Join(lines, "\n"), nil
}

func requestTargetLine(req *http.Request) string {
	var url string = ""
	if req.URL != nil {
		url = req.URL.RequestURI()
	}

	return fmt.Sprintf("%s: %s %s", RequestTarget, strings.ToLower(req.Method), url)
}

func headerLine(req *http.Request, header string) (string, error) {

	if value := req.Header.Get(header); value != "" {
		return fmt.Sprintf("%s: %s", header, value), nil
	}

	return "", errors.New(fmt.Sprintf("Missing required header '%s'", header))
}
<commit_msg>Use constant time comparison (#9)<commit_after>// httpsignatures is a golang implementation of the http-signatures spec
// found at https://tools.ietf.org/html/draft-cavage-http-signatures
package httpsignatures

import (
	"crypto/hmac"
	"crypto/subtle"
	"encoding/base64"
	"errors"
	"fmt"
	"net/http"
	"regexp"
	"strings"
)

const (
	headerSignature     = "Signature"
	headerAuthorization = "Authorization"

	RequestTarget = "(request-target)"

	authScheme = "Signature "
)

var (
	ErrorNoSignatureHeader = errors.New("No Signature header found in request")

	signatureRegex = regexp.MustCompile(`(\w+)="([^"]*)"`)
)

// Signature is the hashed key + headers, either from a request or a signer
type Signature struct {
	KeyID     string
	Algorithm *Algorithm
	Headers   HeaderList
	Signature string
}

// FromRequest creates a new Signature from the Request
// both Signature and Authorization http headers are supported.
func FromRequest(r *http.Request) (*Signature, error) {
	if s, ok := r.Header[headerSignature]; ok {
		return FromString(s[0])
	}
	if a, ok := r.Header[headerAuthorization]; ok {
		return FromString(strings.TrimPrefix(a[0], authScheme))
	}
	return nil, ErrorNoSignatureHeader
}

// FromString creates a new Signature from its encoded form,
// eg `keyId="a",algorithm="b",headers="c",signature="d"`
func FromString(in string) (*Signature, error) {
	var res Signature = Signature{}
	var key string
	var value string

	for _, m := range signatureRegex.FindAllStringSubmatch(in, -1) {
		key = m[1]
		value = m[2]

		if key == "keyId" {
			res.KeyID = value
		} else if key == "algorithm" {
			alg, err := algorithmFromString(value)
			if err != nil {
				return nil, err
			}
			res.Algorithm = alg
		} else if key == "headers" {
			res.Headers = headerListFromString(value)
		} else if key == "signature" {
			res.Signature = value
		} else {
			return nil, errors.New(fmt.Sprintf("Unexpected key in signature '%s'", key))
		}
	}

	if len(res.Signature) == 0 {
		return nil, errors.New("Missing signature")
	}

	if len(res.KeyID) == 0 {
		return nil, errors.New("Missing keyId")
	}

	if res.Algorithm == nil {
		return nil, errors.New("Missing algorithm")
	}

	return &res, nil
}

// String returns the encoded form of the Signature
func (s Signature) String() string {
	str := fmt.Sprintf(
		`keyId="%s",algorithm="%s",signature="%s"`,
		s.KeyID,
		s.Algorithm.name,
		s.Signature,
	)

	if len(s.Headers) > 0 {
		str += fmt.Sprintf(`,headers="%s"`, s.Headers.String())
	}

	return str
}

func (s Signature) calculateSignature(key string, r *http.Request) (string, error) {
	hash := hmac.New(s.Algorithm.hash, []byte(key))

	signingString, err := s.Headers.signingString(r)
	if err != nil {
		return "", err
	}

	hash.Write([]byte(signingString))

	return base64.StdEncoding.EncodeToString(hash.Sum(nil)), nil
}

// Sign this signature using the given key
func (s *Signature) sign(key string, r *http.Request) error {
	sig, err := s.calculateSignature(key, r)
	if err != nil {
		return err
	}

	s.Signature = sig
	return nil
}

// IsValid validates this signature for the given key
func (s Signature) IsValid(key string, r *http.Request) bool {
	if !s.Headers.hasDate() {
		return false
	}

	sig, err := s.calculateSignature(key, r)
	if err != nil {
		return false
	}

	return subtle.ConstantTimeCompare([]byte(s.Signature), []byte(sig)) == 1
}

type HeaderList []string

func headerListFromString(list string) HeaderList {
	return strings.Split(strings.ToLower(string(list)), " ")
}

func (h HeaderList) String() string {
	return strings.ToLower(strings.Join(h, " "))
}

func (h HeaderList) hasDate() bool {
	for _, header := range h {
		if header == "date" {
			return true
		}
	}

	return false
}

func (h HeaderList) signingString(req *http.Request) (string, error) {
	lines := []string{}

	for _, header := range h {
		if header == RequestTarget {
			lines = append(lines, requestTargetLine(req))
		} else {
			line, err := headerLine(req, header)
			if err != nil {
				return "", err
			}
			lines = append(lines, line)
		}
	}

	return strings.Join(lines, "\n"), nil
}

func requestTargetLine(req *http.Request) string {
	var url string = ""
	if req.URL != nil {
		url = req.URL.RequestURI()
	}

	return fmt.Sprintf("%s: %s %s", RequestTarget, strings.ToLower(req.Method), url)
}

func headerLine(req *http.Request, header string) (string, error) {

	if value := req.Header.Get(header); value != "" {
		return fmt.Sprintf("%s: %s", header, value), nil
	}

	return "", errors.New(fmt.Sprintf("Missing required header '%s'", header))
}
<|endoftext|>


<commit_before>// +build darwin

package cpu

import (
	"fmt"
	"io"
	"os/exec"
	"strconv"
	"strings"

	common "github.com/shirou/gopsutil/common"
)

const HELPER_PATH = "/tmp/gopsutil_cpu_helper"

var ClocksPerSec = float64(100)

func init() {
	out, err := exec.Command("/usr/bin/getconf", "CLK_TCK").Output()
	// ignore errors
	if err == nil {
		i, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
		if err == nil {
			ClocksPerSec = float64(i)
		}
	}

	// adhoc compile on the host. Errors will be ignored.
	// gcc is required to compile.
	if !common.PathExists(HELPER_PATH) {
		cmd := exec.Command("gcc", "-o", HELPER_PATH, "-x", "c", "-")
		stdin, err := cmd.StdinPipe()
		if err != nil {
			return
		}
		io.WriteString(stdin, cpu_helper_src)
		stdin.Close()
		cmd.Output()
	}
}

func CPUTimes(percpu bool) ([]CPUTimesStat, error) {
	var ret []CPUTimesStat
	if !common.PathExists(HELPER_PATH) {
		return nil, fmt.Errorf("gopsutil helper(%s) does not exists. gcc required.", HELPER_PATH)
	}

	out, err := exec.Command(HELPER_PATH).Output()
	if err != nil {
		return ret, err
	}

	for _, line := range strings.Split(string(out), "\n") {
		f := strings.Split(string(line), ",")
		if len(f) != 5 {
			continue
		}
		cpu, err := strconv.ParseFloat(f[0], 64)
		if err != nil {
			return ret, err
		}
		// cpu:99 means total, so just ignore if percpu
		if (percpu && cpu == 99) || (!percpu && cpu != 99) {
			continue
		}
		user, err := strconv.ParseFloat(f[1], 64)
		if err != nil {
			return ret, err
		}
		sys, err := strconv.ParseFloat(f[2], 64)
		if err != nil {
			return ret, err
		}
		idle, err := strconv.ParseFloat(f[3], 64)
		if err != nil {
			return ret, err
		}
		nice, err := strconv.ParseFloat(f[4], 64)
		if err != nil {
			return ret, err
		}
		c := CPUTimesStat{
			User:   float64(user / ClocksPerSec),
			Nice:   float64(nice / ClocksPerSec),
			System: float64(sys / ClocksPerSec),
			Idle:   float64(idle / ClocksPerSec),
		}
		if !percpu {
			c.CPU = "cpu-total"
		} else {
			c.CPU = fmt.Sprintf("cpu%d", uint16(cpu))
		}
		ret = append(ret, c)
	}
	return ret, nil
}

// Returns only one CPUInfoStat on FreeBSD
func CPUInfo() ([]CPUInfoStat, error) {
	var ret []CPUInfoStat

	out, err := exec.Command("/usr/sbin/sysctl", "machdep.cpu").Output()
	if err != nil {
		return ret, err
	}

	c := CPUInfoStat{}
	for _, line := range strings.Split(string(out), "\n") {
		values := strings.Fields(line)
		if len(values) < 1 {
			continue
		}

		t, err := strconv.ParseInt(values[1], 10, 64)
		// err is not checked here because some value is string.
		if strings.HasPrefix(line, "machdep.cpu.brand_string") {
			c.ModelName = strings.Join(values[1:], " ")
		} else if strings.HasPrefix(line, "machdep.cpu.family") {
			c.Family = values[1]
		} else if strings.HasPrefix(line, "machdep.cpu.model") {
			c.Model = values[1]
		} else if strings.HasPrefix(line, "machdep.cpu.stepping") {
			if err != nil {
				return ret, err
			}
			c.Stepping = int32(t)
		} else if strings.HasPrefix(line, "machdep.cpu.features") {
			for _, v := range values[1:] {
				c.Flags = append(c.Flags, strings.ToLower(v))
			}
		} else if strings.HasPrefix(line, "machdep.cpu.leaf7_features") {
			for _, v := range values[1:] {
				c.Flags = append(c.Flags, strings.ToLower(v))
			}
		} else if strings.HasPrefix(line, "machdep.cpu.extfeatures") {
			for _, v := range values[1:] {
				c.Flags = append(c.Flags, strings.ToLower(v))
			}
		} else if strings.HasPrefix(line, "machdep.cpu.core_count") {
			if err != nil {
				return ret, err
			}
			c.Cores = int32(t)
		} else if strings.HasPrefix(line, "machdep.cpu.cache.size") {
			if err != nil {
				return ret, err
			}
			c.CacheSize = int32(t)
		} else if strings.HasPrefix(line, "machdep.cpu.vendor") {
			c.VendorID = values[1]
		}

		// TODO:
		// c.Mhz = mustParseFloat64(values[1])
	}

	return append(ret, c), nil
}

const cpu_helper_src = `
#include <mach/mach.h>
#include <mach/mach_error.h>
#include <stdio.h>

int main() {
  	natural_t cpuCount;
	processor_info_array_t ia;
	mach_msg_type_number_t ic;

	kern_return_t error = host_processor_info(mach_host_self(),
		PROCESSOR_CPU_LOAD_INFO, &cpuCount, &ia, &ic);
	if (error) {
		return error;
	}

	processor_cpu_load_info_data_t* cpuLoadInfo =
		(processor_cpu_load_info_data_t*) ia;

	unsigned int all[4];
	// cpu_no,user,system,idle,nice
	for (int cpu=0; cpu<cpuCount; cpu++){
	  printf("%d,", cpu);
	  for (int i=0; i<4; i++){
		printf("%d", cpuLoadInfo[cpu].cpu_ticks[i]);
		all[i] = all[i] + cpuLoadInfo[cpu].cpu_ticks[i];
		if (i != 3){
		  printf(",");
		}else{
		  printf("\n");
		}
	  }
	}
	printf("99,");
	for (int i=0; i<4; i++){
	  printf("%d", all[i]);
	  if (i != 3){
		printf(",");
	  }else{
		printf("\n");
	  }
	}

	vm_deallocate(mach_task_self(), (vm_address_t)ia, ic);
	return error;
}
`
<commit_msg>cpu[darwin]: enable cpu helper only the environemnetal variable set.<commit_after>// +build darwin

package cpu

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"

	common "github.com/shirou/gopsutil/common"
)

const HELPER_PATH = "/tmp/gopsutil_cpu_helper"

// enable cpu helper. It may become security problem.
// This env valiable approach will be changed.
const HELPER_ENABLE_ENV = "ALLLOW_INSECURE_CPU_HELPER"

var ClocksPerSec = float64(100)

func init() {
	out, err := exec.Command("/usr/bin/getconf", "CLK_TCK").Output()
	// ignore errors
	if err == nil {
		i, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
		if err == nil {
			ClocksPerSec = float64(i)
		}
	}

	// adhoc compile on the host. Errors will be ignored.
	// gcc is required to compile.
	if !common.PathExists(HELPER_PATH) && os.Getenv(HELPER_ENABLE_ENV) == "yes" {
		cmd := exec.Command("gcc", "-o", HELPER_PATH, "-x", "c", "-")
		stdin, err := cmd.StdinPipe()
		if err != nil {
			return
		}
		io.WriteString(stdin, cpu_helper_src)
		stdin.Close()
		cmd.Output()
	}
}

func CPUTimes(percpu bool) ([]CPUTimesStat, error) {
	var ret []CPUTimesStat
	if !common.PathExists(HELPER_PATH) {
		return nil, fmt.Errorf("could not get cpu time")
	}

	out, err := exec.Command(HELPER_PATH).Output()
	if err != nil {
		return ret, err
	}

	for _, line := range strings.Split(string(out), "\n") {
		f := strings.Split(string(line), ",")
		if len(f) != 5 {
			continue
		}
		cpu, err := strconv.ParseFloat(f[0], 64)
		if err != nil {
			return ret, err
		}
		// cpu:99 means total, so just ignore if percpu
		if (percpu && cpu == 99) || (!percpu && cpu != 99) {
			continue
		}
		user, err := strconv.ParseFloat(f[1], 64)
		if err != nil {
			return ret, err
		}
		sys, err := strconv.ParseFloat(f[2], 64)
		if err != nil {
			return ret, err
		}
		idle, err := strconv.ParseFloat(f[3], 64)
		if err != nil {
			return ret, err
		}
		nice, err := strconv.ParseFloat(f[4], 64)
		if err != nil {
			return ret, err
		}
		c := CPUTimesStat{
			User:   float64(user / ClocksPerSec),
			Nice:   float64(nice / ClocksPerSec),
			System: float64(sys / ClocksPerSec),
			Idle:   float64(idle / ClocksPerSec),
		}
		if !percpu {
			c.CPU = "cpu-total"
		} else {
			c.CPU = fmt.Sprintf("cpu%d", uint16(cpu))
		}
		ret = append(ret, c)
	}
	return ret, nil
}

// Returns only one CPUInfoStat on FreeBSD
func CPUInfo() ([]CPUInfoStat, error) {
	var ret []CPUInfoStat

	out, err := exec.Command("/usr/sbin/sysctl", "machdep.cpu").Output()
	if err != nil {
		return ret, err
	}

	c := CPUInfoStat{}
	for _, line := range strings.Split(string(out), "\n") {
		values := strings.Fields(line)
		if len(values) < 1 {
			continue
		}

		t, err := strconv.ParseInt(values[1], 10, 64)
		// err is not checked here because some value is string.
		if strings.HasPrefix(line, "machdep.cpu.brand_string") {
			c.ModelName = strings.Join(values[1:], " ")
		} else if strings.HasPrefix(line, "machdep.cpu.family") {
			c.Family = values[1]
		} else if strings.HasPrefix(line, "machdep.cpu.model") {
			c.Model = values[1]
		} else if strings.HasPrefix(line, "machdep.cpu.stepping") {
			if err != nil {
				return ret, err
			}
			c.Stepping = int32(t)
		} else if strings.HasPrefix(line, "machdep.cpu.features") {
			for _, v := range values[1:] {
				c.Flags = append(c.Flags, strings.ToLower(v))
			}
		} else if strings.HasPrefix(line, "machdep.cpu.leaf7_features") {
			for _, v := range values[1:] {
				c.Flags = append(c.Flags, strings.ToLower(v))
			}
		} else if strings.HasPrefix(line, "machdep.cpu.extfeatures") {
			for _, v := range values[1:] {
				c.Flags = append(c.Flags, strings.ToLower(v))
			}
		} else if strings.HasPrefix(line, "machdep.cpu.core_count") {
			if err != nil {
				return ret, err
			}
			c.Cores = int32(t)
		} else if strings.HasPrefix(line, "machdep.cpu.cache.size") {
			if err != nil {
				return ret, err
			}
			c.CacheSize = int32(t)
		} else if strings.HasPrefix(line, "machdep.cpu.vendor") {
			c.VendorID = values[1]
		}

		// TODO:
		// c.Mhz = mustParseFloat64(values[1])
	}

	return append(ret, c), nil
}

const cpu_helper_src = `
#include <mach/mach.h>
#include <mach/mach_error.h>
#include <stdio.h>

int main() {
  	natural_t cpuCount;
	processor_info_array_t ia;
	mach_msg_type_number_t ic;

	kern_return_t error = host_processor_info(mach_host_self(),
		PROCESSOR_CPU_LOAD_INFO, &cpuCount, &ia, &ic);
	if (error) {
		return error;
	}

	processor_cpu_load_info_data_t* cpuLoadInfo =
		(processor_cpu_load_info_data_t*) ia;

	unsigned int all[4];
	// cpu_no,user,system,idle,nice
	for (int cpu=0; cpu<cpuCount; cpu++){
	  printf("%d,", cpu);
	  for (int i=0; i<4; i++){
		printf("%d", cpuLoadInfo[cpu].cpu_ticks[i]);
		all[i] = all[i] + cpuLoadInfo[cpu].cpu_ticks[i];
		if (i != 3){
		  printf(",");
		}else{
		  printf("\n");
		}
	  }
	}
	printf("99,");
	for (int i=0; i<4; i++){
	  printf("%d", all[i]);
	  if (i != 3){
		printf(",");
	  }else{
		printf("\n");
	  }
	}

	vm_deallocate(mach_task_self(), (vm_address_t)ia, ic);
	return error;
}
`
<|endoftext|>


<commit_before>package mmm

import (
	"testing"
	"unsafe"
)

// -----------------------------------------------------------------------------

func TestBytes_BytesOf_bool(t *testing.T) {
	var v bool = true
	size := unsafe.Sizeof(v)
	bytes := make([]byte, size)

	if err := BytesOf(v, bytes); err != nil {
		t.Error(err)
	}

	rv := *((*bool)(unsafe.Pointer(&(bytes[0]))))
	if v != rv {
		t.Error("invalid bytes for bool")
	}
}

func TestBytes_BytesOf_int(t *testing.T) {
	var v int = 42
	size := unsafe.Sizeof(v)
	bytes := make([]byte, size)

	if err := BytesOf(v, bytes); err != nil {
		t.Error(err)
	}

	rv := *((*int)(unsafe.Pointer(&(bytes[0]))))
	if v != rv {
		t.Error("invalid bytes for bool")
	}
}
<commit_msg>added TestBytes_BytesOf_int8<commit_after>package mmm

import (
	"testing"
	"unsafe"
)

// -----------------------------------------------------------------------------

func TestBytes_BytesOf_bool(t *testing.T) {
	var v bool = true
	size := unsafe.Sizeof(v)
	bytes := make([]byte, size)

	if err := BytesOf(v, bytes); err != nil {
		t.Error(err)
	}

	rv := *((*bool)(unsafe.Pointer(&(bytes[0]))))
	if v != rv {
		t.Error("invalid bytes for bool")
	}
}

func TestBytes_BytesOf_int(t *testing.T) {
	var v int = 42
	size := unsafe.Sizeof(v)
	bytes := make([]byte, size)

	if err := BytesOf(v, bytes); err != nil {
		t.Error(err)
	}

	rv := *((*int)(unsafe.Pointer(&(bytes[0]))))
	if v != rv {
		t.Error("invalid bytes for int")
	}
}

func TestBytes_BytesOf_int8(t *testing.T) {
	var v int8 = 42
	size := unsafe.Sizeof(v)
	bytes := make([]byte, size)

	if err := BytesOf(v, bytes); err != nil {
		t.Error(err)
	}

	rv := *((*int8)(unsafe.Pointer(&(bytes[0]))))
	if v != rv {
		t.Error("invalid bytes for int8")
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub

import (
	"fmt"
	"reflect"
	"testing"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/api/iterator"
)

func TestReturnsDoneOnStop(t *testing.T) {
	type testCase struct {
		abort func(*MessageIterator, context.CancelFunc)
		want  error
	}

	for _, tc := range []testCase{
		{
			abort: func(it *MessageIterator, cancel context.CancelFunc) {
				it.Stop()
			},
			want: iterator.Done,
		},
		{
			abort: func(it *MessageIterator, cancel context.CancelFunc) {
				cancel()
			},
			want: context.Canceled,
		},
		{
			abort: func(it *MessageIterator, cancel context.CancelFunc) {
				it.Stop()
				cancel()
			},
			want: iterator.Done,
		},
		{
			abort: func(it *MessageIterator, cancel context.CancelFunc) {
				cancel()
				it.Stop()
			},
			want: iterator.Done,
		},
	} {
		s := &blockingFetch{}
		ctx, cancel := context.WithCancel(context.Background())
		it := newMessageIterator(ctx, s, "subname", &pullOptions{ackDeadline: time.Second * 10, maxExtension: time.Hour})
		defer it.Stop()
		tc.abort(it, cancel)

		_, err := it.Next()
		if err != tc.want {
			t.Errorf("iterator Next error after abort: got:\n%v\nwant:\n%v", err, tc.want)
		}
	}
}

// blockingFetch implements message fetching by not returning until its context is cancelled.
type blockingFetch struct {
	service
}

func (s *blockingFetch) fetchMessages(ctx context.Context, subName string, maxMessages int32) ([]*Message, error) {
	<-ctx.Done()
	return nil, ctx.Err()
}

// justInTimeFetch simulates the situation where the iterator is aborted just after the fetch RPC
// succeeds, so the rest of puller.Next will continue to execute and return sucessfully.
type justInTimeFetch struct {
	service
}

func (s *justInTimeFetch) fetchMessages(ctx context.Context, subName string, maxMessages int32) ([]*Message, error) {
	<-ctx.Done()
	// The context was cancelled, but let's pretend that this happend just after our RPC returned.

	var result []*Message
	for i := 0; i < int(maxMessages); i++ {
		val := fmt.Sprintf("msg%v", i)
		result = append(result, &Message{Data: []byte(val), ackID: val})
	}
	return result, nil
}

func (s *justInTimeFetch) splitAckIDs(ids []string) ([]string, []string) {
	return nil, nil
}

func (s *justInTimeFetch) modifyAckDeadline(ctx context.Context, subName string, deadline time.Duration, ackIDs []string) error {
	return nil
}

func TestAfterAbortReturnsNoMoreThanOneMessage(t *testing.T) {
	// Each test case is excercised by making two concurrent blocking calls on a
	// MessageIterator, and then aborting the iterator.
	// The result should be one call to Next returning a message, and the other returning an error.
	type testCase struct {
		abort func(*MessageIterator, context.CancelFunc)
		// want is the error that should be returned from one Next invocation.
		want error
	}
	for n := 1; n < 3; n++ {
		for _, tc := range []testCase{
			{
				abort: func(it *MessageIterator, cancel context.CancelFunc) {
					it.Stop()
				},
				want: iterator.Done,
			},
			{
				abort: func(it *MessageIterator, cancel context.CancelFunc) {
					cancel()
				},
				want: context.Canceled,
			},
			{
				abort: func(it *MessageIterator, cancel context.CancelFunc) {
					it.Stop()
					cancel()
				},
				want: iterator.Done,
			},
			{
				abort: func(it *MessageIterator, cancel context.CancelFunc) {
					cancel()
					it.Stop()
				},
				want: iterator.Done,
			},
		} {
			s := &justInTimeFetch{}
			ctx, cancel := context.WithCancel(context.Background())

			// if maxPrefetch == 1, there will be no messages in the puller buffer when Next is invoked the second time.
			// if maxPrefetch == 2, there will be 1 message in the puller buffer when Next is invoked the second time.
			po := &pullOptions{
				ackDeadline:  time.Second * 10,
				maxExtension: time.Hour,
				maxPrefetch:  int32(n),
			}
			it := newMessageIterator(ctx, s, "subname", po)
			defer it.Stop()

			type result struct {
				m   *Message
				err error
			}
			results := make(chan *result, 2)

			for i := 0; i < 2; i++ {
				go func() {
					m, err := it.Next()
					results <- &result{m, err}
					if err == nil {
						m.Done(false)
					}
				}()
			}
			// Wait for goroutines to block on it.Next().
			time.Sleep(time.Millisecond)
			tc.abort(it, cancel)

			result1 := <-results
			result2 := <-results

			// There should be one error result, and one non-error result.
			// Make result1 be the non-error result.
			if result1.err != nil {
				result1, result2 = result2, result1
			}

			if string(result1.m.Data) != "msg0" {
				t.Errorf("After abort, got message: %v, want %v", result1.m.Data, "msg0")
			}
			if result1.err != nil {
				t.Errorf("After abort, got : %v, want nil", result1.err)
			}
			if result2.m != nil {
				t.Errorf("After abort, got message: %v, want nil", result2.m)
			}
			if result2.err != tc.want {
				t.Errorf("After abort, got err: %v, want %v", result2.err, tc.want)
			}
		}
	}
}

type fetcherServiceWithModifyAckDeadline struct {
	fetcherService
	events chan string
}

func (f *fetcherServiceWithModifyAckDeadline) modifyAckDeadline(_ context.Context, _ string, d time.Duration, ids []string) error {
	f.events <- fmt.Sprintf("modAck(%v, %s)", ids, d)
	return nil
}

func (f *fetcherServiceWithModifyAckDeadline) splitAckIDs(ackIDs []string) ([]string, []string) {
	return ackIDs, nil
}

func TestMultipleStopCallsBlockUntilMessageDone(t *testing.T) {
	events := make(chan string, 3)
	s := &fetcherServiceWithModifyAckDeadline{
		fetcherService{
			results: []fetchResult{
				{
					msgs: []*Message{{ackID: "a"}, {ackID: "b"}},
				},
			},
		},
		events,
	}

	ctx := context.Background()
	it := newMessageIterator(ctx, s, "subname", &pullOptions{ackDeadline: time.Second * 10, maxExtension: 0})

	m, err := it.Next()
	if err != nil {
		t.Errorf("error calling Next: %v", err)
	}

	go func() {
		it.Stop()
		events <- "stopped"
	}()
	go func() {
		it.Stop()
		events <- "stopped"
	}()

	time.Sleep(10 * time.Millisecond)
	m.Done(false)

	got := []string{<-events, <-events, <-events}
	want := []string{"modAck([a], 0s)", "stopped", "stopped"}
	if !reflect.DeepEqual(got, want) {
		t.Errorf("stopping iterator, got: %v ; want: %v", got, want)
	}

	// The iterator is stopped, so should not return another message.
	m, err = it.Next()
	if m != nil {
		t.Errorf("message got: %v ; want: nil", m)
	}
	if err != iterator.Done {
		t.Errorf("err got: %v ; want: %v", err, iterator.Done)
	}
}

func TestFastNack(t *testing.T) {
	events := make(chan string, 3)
	s := &fetcherServiceWithModifyAckDeadline{
		fetcherService{
			results: []fetchResult{
				{
					msgs: []*Message{{ackID: "a"}, {ackID: "b"}},
				},
			},
		},
		events,
	}

	ctx := context.Background()
	it := newMessageIterator(ctx, s, "subname", &pullOptions{
		ackDeadline:  time.Second * 6,
		maxExtension: time.Second * 10,
	})
	// Get both messages.
	_, err := it.Next()
	if err != nil {
		t.Errorf("error calling Next: %v", err)
	}
	m2, err := it.Next()
	if err != nil {
		t.Errorf("error calling Next: %v", err)
	}
	// Ignore the first, nack the second.
	m2.Done(false)

	got := []string{<-events, <-events}
	// The nack should happen before the deadline extension.
	want := []string{"modAck([b], 0s)", "modAck([a], 6s)"}
	if !reflect.DeepEqual(got, want) {
		t.Errorf("got: %v ; want: %v", got, want)
	}
}
<commit_msg>pubsub: fix test breakage due to go 1.6 time.Duration.String<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub

import (
	"fmt"
	"reflect"
	"testing"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/api/iterator"
)

func TestReturnsDoneOnStop(t *testing.T) {
	type testCase struct {
		abort func(*MessageIterator, context.CancelFunc)
		want  error
	}

	for _, tc := range []testCase{
		{
			abort: func(it *MessageIterator, cancel context.CancelFunc) {
				it.Stop()
			},
			want: iterator.Done,
		},
		{
			abort: func(it *MessageIterator, cancel context.CancelFunc) {
				cancel()
			},
			want: context.Canceled,
		},
		{
			abort: func(it *MessageIterator, cancel context.CancelFunc) {
				it.Stop()
				cancel()
			},
			want: iterator.Done,
		},
		{
			abort: func(it *MessageIterator, cancel context.CancelFunc) {
				cancel()
				it.Stop()
			},
			want: iterator.Done,
		},
	} {
		s := &blockingFetch{}
		ctx, cancel := context.WithCancel(context.Background())
		it := newMessageIterator(ctx, s, "subname", &pullOptions{ackDeadline: time.Second * 10, maxExtension: time.Hour})
		defer it.Stop()
		tc.abort(it, cancel)

		_, err := it.Next()
		if err != tc.want {
			t.Errorf("iterator Next error after abort: got:\n%v\nwant:\n%v", err, tc.want)
		}
	}
}

// blockingFetch implements message fetching by not returning until its context is cancelled.
type blockingFetch struct {
	service
}

func (s *blockingFetch) fetchMessages(ctx context.Context, subName string, maxMessages int32) ([]*Message, error) {
	<-ctx.Done()
	return nil, ctx.Err()
}

// justInTimeFetch simulates the situation where the iterator is aborted just after the fetch RPC
// succeeds, so the rest of puller.Next will continue to execute and return sucessfully.
type justInTimeFetch struct {
	service
}

func (s *justInTimeFetch) fetchMessages(ctx context.Context, subName string, maxMessages int32) ([]*Message, error) {
	<-ctx.Done()
	// The context was cancelled, but let's pretend that this happend just after our RPC returned.

	var result []*Message
	for i := 0; i < int(maxMessages); i++ {
		val := fmt.Sprintf("msg%v", i)
		result = append(result, &Message{Data: []byte(val), ackID: val})
	}
	return result, nil
}

func (s *justInTimeFetch) splitAckIDs(ids []string) ([]string, []string) {
	return nil, nil
}

func (s *justInTimeFetch) modifyAckDeadline(ctx context.Context, subName string, deadline time.Duration, ackIDs []string) error {
	return nil
}

func TestAfterAbortReturnsNoMoreThanOneMessage(t *testing.T) {
	// Each test case is excercised by making two concurrent blocking calls on a
	// MessageIterator, and then aborting the iterator.
	// The result should be one call to Next returning a message, and the other returning an error.
	type testCase struct {
		abort func(*MessageIterator, context.CancelFunc)
		// want is the error that should be returned from one Next invocation.
		want error
	}
	for n := 1; n < 3; n++ {
		for _, tc := range []testCase{
			{
				abort: func(it *MessageIterator, cancel context.CancelFunc) {
					it.Stop()
				},
				want: iterator.Done,
			},
			{
				abort: func(it *MessageIterator, cancel context.CancelFunc) {
					cancel()
				},
				want: context.Canceled,
			},
			{
				abort: func(it *MessageIterator, cancel context.CancelFunc) {
					it.Stop()
					cancel()
				},
				want: iterator.Done,
			},
			{
				abort: func(it *MessageIterator, cancel context.CancelFunc) {
					cancel()
					it.Stop()
				},
				want: iterator.Done,
			},
		} {
			s := &justInTimeFetch{}
			ctx, cancel := context.WithCancel(context.Background())

			// if maxPrefetch == 1, there will be no messages in the puller buffer when Next is invoked the second time.
			// if maxPrefetch == 2, there will be 1 message in the puller buffer when Next is invoked the second time.
			po := &pullOptions{
				ackDeadline:  time.Second * 10,
				maxExtension: time.Hour,
				maxPrefetch:  int32(n),
			}
			it := newMessageIterator(ctx, s, "subname", po)
			defer it.Stop()

			type result struct {
				m   *Message
				err error
			}
			results := make(chan *result, 2)

			for i := 0; i < 2; i++ {
				go func() {
					m, err := it.Next()
					results <- &result{m, err}
					if err == nil {
						m.Done(false)
					}
				}()
			}
			// Wait for goroutines to block on it.Next().
			time.Sleep(time.Millisecond)
			tc.abort(it, cancel)

			result1 := <-results
			result2 := <-results

			// There should be one error result, and one non-error result.
			// Make result1 be the non-error result.
			if result1.err != nil {
				result1, result2 = result2, result1
			}

			if string(result1.m.Data) != "msg0" {
				t.Errorf("After abort, got message: %v, want %v", result1.m.Data, "msg0")
			}
			if result1.err != nil {
				t.Errorf("After abort, got : %v, want nil", result1.err)
			}
			if result2.m != nil {
				t.Errorf("After abort, got message: %v, want nil", result2.m)
			}
			if result2.err != tc.want {
				t.Errorf("After abort, got err: %v, want %v", result2.err, tc.want)
			}
		}
	}
}

type fetcherServiceWithModifyAckDeadline struct {
	fetcherService
	events chan string
}

func (f *fetcherServiceWithModifyAckDeadline) modifyAckDeadline(_ context.Context, _ string, d time.Duration, ids []string) error {
	// Different versions of Go use different representations for time.Duration(0).
	var ds string
	if d == 0 {
		ds = "0s"
	} else {
		ds = d.String()
	}
	f.events <- fmt.Sprintf("modAck(%v, %s)", ids, ds)
	return nil
}

func (f *fetcherServiceWithModifyAckDeadline) splitAckIDs(ackIDs []string) ([]string, []string) {
	return ackIDs, nil
}

func TestMultipleStopCallsBlockUntilMessageDone(t *testing.T) {
	events := make(chan string, 3)
	s := &fetcherServiceWithModifyAckDeadline{
		fetcherService{
			results: []fetchResult{
				{
					msgs: []*Message{{ackID: "a"}, {ackID: "b"}},
				},
			},
		},
		events,
	}

	ctx := context.Background()
	it := newMessageIterator(ctx, s, "subname", &pullOptions{ackDeadline: time.Second * 10, maxExtension: 0})

	m, err := it.Next()
	if err != nil {
		t.Errorf("error calling Next: %v", err)
	}

	go func() {
		it.Stop()
		events <- "stopped"
	}()
	go func() {
		it.Stop()
		events <- "stopped"
	}()

	time.Sleep(10 * time.Millisecond)
	m.Done(false)

	got := []string{<-events, <-events, <-events}
	want := []string{"modAck([a], 0s)", "stopped", "stopped"}
	if !reflect.DeepEqual(got, want) {
		t.Errorf("stopping iterator, got: %v ; want: %v", got, want)
	}

	// The iterator is stopped, so should not return another message.
	m, err = it.Next()
	if m != nil {
		t.Errorf("message got: %v ; want: nil", m)
	}
	if err != iterator.Done {
		t.Errorf("err got: %v ; want: %v", err, iterator.Done)
	}
}

func TestFastNack(t *testing.T) {
	events := make(chan string, 3)
	s := &fetcherServiceWithModifyAckDeadline{
		fetcherService{
			results: []fetchResult{
				{
					msgs: []*Message{{ackID: "a"}, {ackID: "b"}},
				},
			},
		},
		events,
	}

	ctx := context.Background()
	it := newMessageIterator(ctx, s, "subname", &pullOptions{
		ackDeadline:  time.Second * 6,
		maxExtension: time.Second * 10,
	})
	// Get both messages.
	_, err := it.Next()
	if err != nil {
		t.Errorf("error calling Next: %v", err)
	}
	m2, err := it.Next()
	if err != nil {
		t.Errorf("error calling Next: %v", err)
	}
	// Ignore the first, nack the second.
	m2.Done(false)

	got := []string{<-events, <-events}
	// The nack should happen before the deadline extension.
	want := []string{"modAck([b], 0s)", "modAck([a], 6s)"}
	if !reflect.DeepEqual(got, want) {
		t.Errorf("got: %v ; want: %v", got, want)
	}
}
<|endoftext|>


<commit_before>package crawler

import (
	"time"
)

// Config contains configuration for a Crawler.
type Config struct {
	DirEntryBufferSize uint          // Size of buffer for processing directory entry channels.
	MinUpdateAge       time.Duration // The minimum age for items to be updated.
	StatTimeout        time.Duration // Timeout for Stat() calls.
	DirEntryTimeout    time.Duration // Timeout *between* directory entries.
}

// DefaultConfig generates a default configuration for a Crawler.
func DefaultConfig() *Config {
	return &Config{
		DirEntryBufferSize: 256,
		MinUpdateAge:       time.Hour,
		StatTimeout:        60 * time.Second,
		DirEntryTimeout:    60 * time.Second,
	}
}
<commit_msg>Increase directory listing buffer size.<commit_after>package crawler

import (
	"time"
)

// Config contains configuration for a Crawler.
type Config struct {
	DirEntryBufferSize uint          // Size of buffer for processing directory entry channels.
	MinUpdateAge       time.Duration // The minimum age for items to be updated.
	StatTimeout        time.Duration // Timeout for Stat() calls.
	DirEntryTimeout    time.Duration // Timeout *between* directory entries.
}

// DefaultConfig generates a default configuration for a Crawler.
func DefaultConfig() *Config {
	return &Config{
		DirEntryBufferSize: 2048,
		MinUpdateAge:       time.Hour,
		StatTimeout:        60 * time.Second,
		DirEntryTimeout:    60 * time.Second,
	}
}
<|endoftext|>