<commit_before>// Package host is an implementation of the host module, and is responsible for
// participating in the storage ecosystem, turning available disk space an
// internet bandwidth into profit for the user.
package host

// TODO: what happens if the renter submits the revision early, before the
// final revision. Will the host mark the contract as complete?

// TODO: Host and renter are reporting errors where the renter is not adding
// enough fees to the file contract.

// TODO: Test the safety of the builder, it should be okay to have multiple
// builders open for up to 600 seconds, which means multiple blocks could be
// received in that time period. Should also check what happens if a parent
// gets confirmed on the blockchain before the builder is finished.

// TODO: Double check that any network connection has a finite deadline -
// handling action items properly requires that the locks held on the
// obligations eventually be released. There's also some more advanced
// implementation that needs to happen with the storage obligation locks to
// make sure that someone who wants a lock is able to get it eventually.

// TODO: Add contract compensation from form contract to the storage obligation
// financial metrics, and to the host's tracking.

// TODO: merge the network interfaces stuff, don't forget to include the
// 'announced' variable as one of the outputs.

// TODO: 'announced' doesn't tell you if the announcement made it to the
// blockchain.

// TODO: Need to make sure that the revision exchange for the renter and the
// host is being handled correctly. For the host, it's not so difficult. The
// host need only send the most recent revision every time. But, the host
// should not sign a revision unless the renter has explicitly signed such that
// the 'WholeTransaction' fields cover only the revision and that the
// signatures for the revision don't depend on anything else. The renter needs
// to verify the same when checking on a file contract revision from the host.
// If the host has submitted a file contract revision where the signatures have
// signed the whole file contract, there is an issue.

// TODO: there is a mistake in the file contract revision rpc, the host, if it
// does not have the right file contract id, should be returning an error there
// to the renter (and not just to it's calling function without informing the
// renter what's up).

// TODO: Need to make sure that the correct height is being used when adding
// sectors to the storage manager - in some places right now WindowStart is
// being used but really it's WindowEnd that should be in use.

// TODO: The host needs some way to blacklist file contracts that are being
// abusive by repeatedly getting free download batches.

// TODO: clean up all of the magic numbers in the host.

// TODO: revamp the finances for the storage obligations.

// TODO: host_test.go has commented out tests.

// TODO: network_test.go has commented out tests.

// TODO: persist_test.go has commented out tests.

// TODO: update_test.go has commented out tests.

import (
	"errors"
	"fmt"
	"net"
	"path/filepath"
	"sync"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/host/contractmanager"
	"github.com/NebulousLabs/Sia/persist"
	siasync "github.com/NebulousLabs/Sia/sync"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// Names of the various persistent files in the host.
	dbFilename   = modules.HostDir + ".db"
	logFile      = modules.HostDir + ".log"
	settingsFile = modules.HostDir + ".json"
)

var (
	// dbMetadata is a header that gets put into the database to identify a
	// version and indicate that the database holds host information.
	dbMetadata = persist.Metadata{
		Header:  "Sia Host DB",
		Version: "0.5.2",
	}

	// persistMetadata is the header that gets written to the persist file, and is
	// used to recognize other persist files.
	persistMetadata = persist.Metadata{
		Header:  "Sia Host",
		Version: "1.2.0",
	}

	// errHostClosed gets returned when a call is rejected due to the host
	// having been closed.
	errHostClosed = errors.New("call is disabled because the host is closed")

	// Nil dependency errors.
	errNilCS     = errors.New("host cannot use a nil state")
	errNilTpool  = errors.New("host cannot use a nil transaction pool")
	errNilWallet = errors.New("host cannot use a nil wallet")
)

// A Host contains all the fields necessary for storing files for clients and
// performing the storage proofs on the received files.
type Host struct {
	// RPC Metrics - atomic variables need to be placed at the top to preserve
	// compatibility with 32bit systems. These values are not persistent.
	atomicDownloadCalls       uint64
	atomicErroredCalls        uint64
	atomicFormContractCalls   uint64
	atomicRenewCalls          uint64
	atomicReviseCalls         uint64
	atomicRecentRevisionCalls uint64
	atomicSettingsCalls       uint64
	atomicUnrecognizedCalls   uint64

	// Error management. There are a few different types of errors returned by
	// the host. These errors intentionally not persistent, so that the logging
	// limits of each error type will be reset each time the host is reset.
	// These values are not persistent.
	atomicCommunicationErrors uint64
	atomicConnectionErrors    uint64
	atomicConsensusErrors     uint64
	atomicInternalErrors      uint64
	atomicNormalErrors        uint64

	// Dependencies.
	cs     modules.ConsensusSet
	tpool  modules.TransactionPool
	wallet modules.Wallet
	dependencies
	modules.StorageManager

	// Host ACID fields - these fields need to be updated in serial, ACID
	// transactions.
	announced         bool
	announceConfirmed bool
	blockHeight       types.BlockHeight
	publicKey         types.SiaPublicKey
	secretKey         crypto.SecretKey
	recentChange      modules.ConsensusChangeID
	unlockHash        types.UnlockHash // A wallet address that can receive coins.

	// Host transient fields - these fields are either determined at startup or
	// otherwise are not critical to always be correct.
	autoAddress          modules.NetAddress // Determined using automatic tooling in network.go
	financialMetrics     modules.HostFinancialMetrics
	settings             modules.HostInternalSettings
	revisionNumber       uint64
	workingStatus        modules.HostWorkingStatus
	connectabilityStatus modules.HostConnectabilityStatus

	// A map of storage obligations that are currently being modified. Locks on
	// storage obligations can be long-running, and each storage obligation can
	// be locked separately.
	lockedStorageObligations map[types.FileContractID]*siasync.TryMutex

	// Utilities.
	db         *persist.BoltDatabase
	listener   net.Listener
	log        *persist.Logger
	mu         sync.RWMutex
	persistDir string
	port       string
	tg         siasync.ThreadGroup
}

// checkUnlockHash will check that the host has an unlock hash. If the host
// does not have an unlock hash, an attempt will be made to get an unlock hash
// from the wallet. That may fail due to the wallet being locked, in which case
// an error is returned.
func (h *Host) checkUnlockHash() error {
	if h.unlockHash == (types.UnlockHash{}) {
		uc, err := h.wallet.NextAddress()
		if err != nil {
			return err
		}

		// Set the unlock hash and save the host. Saving is important, because
		// the host will be using this unlock hash to establish identity, and
		// losing it will mean silently losing part of the host identity.
		h.unlockHash = uc.UnlockHash()
		err = h.saveSync()
		if err != nil {
			return err
		}
	}
	return nil
}

// newHost returns an initialized Host, taking a set of dependencies as input.
// By making the dependencies an argument of the 'new' call, the host can be
// mocked such that the dependencies can return unexpected errors or unique
// behaviors during testing, enabling easier testing of the failure modes of
// the Host.
func newHost(dependencies dependencies, cs modules.ConsensusSet, tpool modules.TransactionPool, wallet modules.Wallet, listenerAddress string, persistDir string) (*Host, error) {
	// Check that all the dependencies were provided.
	if cs == nil {
		return nil, errNilCS
	}
	if tpool == nil {
		return nil, errNilTpool
	}
	if wallet == nil {
		return nil, errNilWallet
	}

	// Create the host object.
	h := &Host{
		cs:           cs,
		tpool:        tpool,
		wallet:       wallet,
		dependencies: dependencies,

		lockedStorageObligations: make(map[types.FileContractID]*siasync.TryMutex),

		persistDir: persistDir,
	}

	// Call stop in the event of a partial startup.
	var err error
	defer func() {
		if err != nil {
			err = composeErrors(h.tg.Stop(), err)
		}
	}()

	// Create the perist directory if it does not yet exist.
	err = dependencies.mkdirAll(h.persistDir, 0700)
	if err != nil {
		return nil, err
	}

	// Initialize the logger, and set up the stop call that will close the
	// logger.
	h.log, err = dependencies.newLogger(filepath.Join(h.persistDir, logFile))
	if err != nil {
		return nil, err
	}
	h.tg.AfterStop(func() {
		err = h.log.Close()
		if err != nil {
			// State of the logger is uncertain, a Println will have to
			// suffice.
			fmt.Println("Error when closing the logger:", err)
		}
	})

	// Add the storage manager to the host, and set up the stop call that will
	// close the storage manager.
	h.StorageManager, err = contractmanager.New(filepath.Join(persistDir, "contractmanager"))
	if err != nil {
		h.log.Println("Could not open the storage manager:", err)
		return nil, err
	}
	h.tg.AfterStop(func() {
		err = h.StorageManager.Close()
		if err != nil {
			h.log.Println("Could not close storage manager:", err)
		}
	})

	// Load the prior persistence structures, and configure the host to save
	// before shutting down.
	err = h.load()
	if err != nil {
		return nil, err
	}
	h.tg.AfterStop(func() {
		err = h.saveSync()
		if err != nil {
			h.log.Println("Could not save host upon shutdown:", err)
		}
	})

	// Initialize the networking.
	err = h.initNetworking(listenerAddress)
	if err != nil {
		h.log.Println("Could not initialize host networking:", err)
		return nil, err
	}
	return h, nil
}

// New returns an initialized Host.
func New(cs modules.ConsensusSet, tpool modules.TransactionPool, wallet modules.Wallet, address string, persistDir string) (*Host, error) {
	return newHost(productionDependencies{}, cs, tpool, wallet, address, persistDir)
}

// Close shuts down the host.
func (h *Host) Close() error {
	return h.tg.Stop()
}

// ExternalSettings returns the hosts external settings. These values cannot be
// set by the user (host is configured through InternalSettings), and are the
// values that get displayed to other hosts on the network.
func (h *Host) ExternalSettings() modules.HostExternalSettings {
	h.mu.RLock()
	defer h.mu.RUnlock()
	err := h.tg.Add()
	if err != nil {
		build.Critical("Call to ExternalSettings after close")
	}
	defer h.tg.Done()
	return h.externalSettings()
}

// WorkingStatus returns the working state of the host, where working is
// defined as having received more than workingStatusThreshold settings calls
// over the period of workingStatusFrequency.
func (h *Host) WorkingStatus() modules.HostWorkingStatus {
	h.mu.RLock()
	defer h.mu.RUnlock()
	return h.workingStatus
}

// ConnectabilityStatus returns the connectability state of the host, whether
// the host can connect to itself on its configured netaddress.
func (h *Host) ConnectabilityStatus() modules.HostConnectabilityStatus {
	h.mu.RLock()
	defer h.mu.RUnlock()
	return h.connectabilityStatus
}

// FinancialMetrics returns information about the financial commitments,
// rewards, and activities of the host.
func (h *Host) FinancialMetrics() modules.HostFinancialMetrics {
	h.mu.RLock()
	defer h.mu.RUnlock()
	err := h.tg.Add()
	if err != nil {
		build.Critical("Call to FinancialMetrics after close")
	}
	defer h.tg.Done()
	return h.financialMetrics
}

// PublicKey returns the public key of the host that is used to facilitate
// relationships between the host and renter.
func (h *Host) PublicKey() types.SiaPublicKey {
	h.mu.RLock()
	defer h.mu.RUnlock()
	return h.publicKey
}

// SetInternalSettings updates the host's internal HostInternalSettings object.
func (h *Host) SetInternalSettings(settings modules.HostInternalSettings) error {
	h.mu.Lock()
	defer h.mu.Unlock()
	err := h.tg.Add()
	if err != nil {
		return err
	}
	defer h.tg.Done()

	// The host should not be accepting file contracts if it does not have an
	// unlock hash.
	if settings.AcceptingContracts {
		err := h.checkUnlockHash()
		if err != nil {
			return errors.New("internal settings not updated, no unlock hash: " + err.Error())
		}
	}

	if settings.NetAddress != "" {
		err := settings.NetAddress.IsValid()
		if err != nil {
			return errors.New("internal settings not updated, invalid NetAddress: " + err.Error())
		}
	}

	// Check if the net address for the host has changed. If it has, and it's
	// not equal to the auto address, then the host is going to need to make
	// another blockchain announcement.
	if h.settings.NetAddress != settings.NetAddress && settings.NetAddress != h.autoAddress {
		h.announced = false
	}

	h.settings = settings
	h.revisionNumber++

	err = h.saveSync()
	if err != nil {
		return errors.New("internal settings updated, but failed saving to disk: " + err.Error())
	}
	return nil
}

// InternalSettings returns the settings of a host.
func (h *Host) InternalSettings() modules.HostInternalSettings {
	h.mu.RLock()
	defer h.mu.RUnlock()
	err := h.tg.Add()
	if err != nil {
		return modules.HostInternalSettings{}
	}
	defer h.tg.Done()
	return h.settings
}
<commit_msg>remove unused Host.atomicRecentRevisionCalls<commit_after>// Package host is an implementation of the host module, and is responsible for
// participating in the storage ecosystem, turning available disk space an
// internet bandwidth into profit for the user.
package host

// TODO: what happens if the renter submits the revision early, before the
// final revision. Will the host mark the contract as complete?

// TODO: Host and renter are reporting errors where the renter is not adding
// enough fees to the file contract.

// TODO: Test the safety of the builder, it should be okay to have multiple
// builders open for up to 600 seconds, which means multiple blocks could be
// received in that time period. Should also check what happens if a parent
// gets confirmed on the blockchain before the builder is finished.

// TODO: Double check that any network connection has a finite deadline -
// handling action items properly requires that the locks held on the
// obligations eventually be released. There's also some more advanced
// implementation that needs to happen with the storage obligation locks to
// make sure that someone who wants a lock is able to get it eventually.

// TODO: Add contract compensation from form contract to the storage obligation
// financial metrics, and to the host's tracking.

// TODO: merge the network interfaces stuff, don't forget to include the
// 'announced' variable as one of the outputs.

// TODO: 'announced' doesn't tell you if the announcement made it to the
// blockchain.

// TODO: Need to make sure that the revision exchange for the renter and the
// host is being handled correctly. For the host, it's not so difficult. The
// host need only send the most recent revision every time. But, the host
// should not sign a revision unless the renter has explicitly signed such that
// the 'WholeTransaction' fields cover only the revision and that the
// signatures for the revision don't depend on anything else. The renter needs
// to verify the same when checking on a file contract revision from the host.
// If the host has submitted a file contract revision where the signatures have
// signed the whole file contract, there is an issue.

// TODO: there is a mistake in the file contract revision rpc, the host, if it
// does not have the right file contract id, should be returning an error there
// to the renter (and not just to it's calling function without informing the
// renter what's up).

// TODO: Need to make sure that the correct height is being used when adding
// sectors to the storage manager - in some places right now WindowStart is
// being used but really it's WindowEnd that should be in use.

// TODO: The host needs some way to blacklist file contracts that are being
// abusive by repeatedly getting free download batches.

// TODO: clean up all of the magic numbers in the host.

// TODO: revamp the finances for the storage obligations.

// TODO: host_test.go has commented out tests.

// TODO: network_test.go has commented out tests.

// TODO: persist_test.go has commented out tests.

// TODO: update_test.go has commented out tests.

import (
	"errors"
	"fmt"
	"net"
	"path/filepath"
	"sync"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/host/contractmanager"
	"github.com/NebulousLabs/Sia/persist"
	siasync "github.com/NebulousLabs/Sia/sync"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// Names of the various persistent files in the host.
	dbFilename   = modules.HostDir + ".db"
	logFile      = modules.HostDir + ".log"
	settingsFile = modules.HostDir + ".json"
)

var (
	// dbMetadata is a header that gets put into the database to identify a
	// version and indicate that the database holds host information.
	dbMetadata = persist.Metadata{
		Header:  "Sia Host DB",
		Version: "0.5.2",
	}

	// persistMetadata is the header that gets written to the persist file, and is
	// used to recognize other persist files.
	persistMetadata = persist.Metadata{
		Header:  "Sia Host",
		Version: "1.2.0",
	}

	// errHostClosed gets returned when a call is rejected due to the host
	// having been closed.
	errHostClosed = errors.New("call is disabled because the host is closed")

	// Nil dependency errors.
	errNilCS     = errors.New("host cannot use a nil state")
	errNilTpool  = errors.New("host cannot use a nil transaction pool")
	errNilWallet = errors.New("host cannot use a nil wallet")
)

// A Host contains all the fields necessary for storing files for clients and
// performing the storage proofs on the received files.
type Host struct {
	// RPC Metrics - atomic variables need to be placed at the top to preserve
	// compatibility with 32bit systems. These values are not persistent.
	atomicDownloadCalls     uint64
	atomicErroredCalls      uint64
	atomicFormContractCalls uint64
	atomicRenewCalls        uint64
	atomicReviseCalls       uint64
	atomicSettingsCalls     uint64
	atomicUnrecognizedCalls uint64

	// Error management. There are a few different types of errors returned by
	// the host. These errors intentionally not persistent, so that the logging
	// limits of each error type will be reset each time the host is reset.
	// These values are not persistent.
	atomicCommunicationErrors uint64
	atomicConnectionErrors    uint64
	atomicConsensusErrors     uint64
	atomicInternalErrors      uint64
	atomicNormalErrors        uint64

	// Dependencies.
	cs     modules.ConsensusSet
	tpool  modules.TransactionPool
	wallet modules.Wallet
	dependencies
	modules.StorageManager

	// Host ACID fields - these fields need to be updated in serial, ACID
	// transactions.
	announced         bool
	announceConfirmed bool
	blockHeight       types.BlockHeight
	publicKey         types.SiaPublicKey
	secretKey         crypto.SecretKey
	recentChange      modules.ConsensusChangeID
	unlockHash        types.UnlockHash // A wallet address that can receive coins.

	// Host transient fields - these fields are either determined at startup or
	// otherwise are not critical to always be correct.
	autoAddress          modules.NetAddress // Determined using automatic tooling in network.go
	financialMetrics     modules.HostFinancialMetrics
	settings             modules.HostInternalSettings
	revisionNumber       uint64
	workingStatus        modules.HostWorkingStatus
	connectabilityStatus modules.HostConnectabilityStatus

	// A map of storage obligations that are currently being modified. Locks on
	// storage obligations can be long-running, and each storage obligation can
	// be locked separately.
	lockedStorageObligations map[types.FileContractID]*siasync.TryMutex

	// Utilities.
	db         *persist.BoltDatabase
	listener   net.Listener
	log        *persist.Logger
	mu         sync.RWMutex
	persistDir string
	port       string
	tg         siasync.ThreadGroup
}

// checkUnlockHash will check that the host has an unlock hash. If the host
// does not have an unlock hash, an attempt will be made to get an unlock hash
// from the wallet. That may fail due to the wallet being locked, in which case
// an error is returned.
func (h *Host) checkUnlockHash() error {
	if h.unlockHash == (types.UnlockHash{}) {
		uc, err := h.wallet.NextAddress()
		if err != nil {
			return err
		}

		// Set the unlock hash and save the host. Saving is important, because
		// the host will be using this unlock hash to establish identity, and
		// losing it will mean silently losing part of the host identity.
		h.unlockHash = uc.UnlockHash()
		err = h.saveSync()
		if err != nil {
			return err
		}
	}
	return nil
}

// newHost returns an initialized Host, taking a set of dependencies as input.
// By making the dependencies an argument of the 'new' call, the host can be
// mocked such that the dependencies can return unexpected errors or unique
// behaviors during testing, enabling easier testing of the failure modes of
// the Host.
func newHost(dependencies dependencies, cs modules.ConsensusSet, tpool modules.TransactionPool, wallet modules.Wallet, listenerAddress string, persistDir string) (*Host, error) {
	// Check that all the dependencies were provided.
	if cs == nil {
		return nil, errNilCS
	}
	if tpool == nil {
		return nil, errNilTpool
	}
	if wallet == nil {
		return nil, errNilWallet
	}

	// Create the host object.
	h := &Host{
		cs:           cs,
		tpool:        tpool,
		wallet:       wallet,
		dependencies: dependencies,

		lockedStorageObligations: make(map[types.FileContractID]*siasync.TryMutex),

		persistDir: persistDir,
	}

	// Call stop in the event of a partial startup.
	var err error
	defer func() {
		if err != nil {
			err = composeErrors(h.tg.Stop(), err)
		}
	}()

	// Create the perist directory if it does not yet exist.
	err = dependencies.mkdirAll(h.persistDir, 0700)
	if err != nil {
		return nil, err
	}

	// Initialize the logger, and set up the stop call that will close the
	// logger.
	h.log, err = dependencies.newLogger(filepath.Join(h.persistDir, logFile))
	if err != nil {
		return nil, err
	}
	h.tg.AfterStop(func() {
		err = h.log.Close()
		if err != nil {
			// State of the logger is uncertain, a Println will have to
			// suffice.
			fmt.Println("Error when closing the logger:", err)
		}
	})

	// Add the storage manager to the host, and set up the stop call that will
	// close the storage manager.
	h.StorageManager, err = contractmanager.New(filepath.Join(persistDir, "contractmanager"))
	if err != nil {
		h.log.Println("Could not open the storage manager:", err)
		return nil, err
	}
	h.tg.AfterStop(func() {
		err = h.StorageManager.Close()
		if err != nil {
			h.log.Println("Could not close storage manager:", err)
		}
	})

	// Load the prior persistence structures, and configure the host to save
	// before shutting down.
	err = h.load()
	if err != nil {
		return nil, err
	}
	h.tg.AfterStop(func() {
		err = h.saveSync()
		if err != nil {
			h.log.Println("Could not save host upon shutdown:", err)
		}
	})

	// Initialize the networking.
	err = h.initNetworking(listenerAddress)
	if err != nil {
		h.log.Println("Could not initialize host networking:", err)
		return nil, err
	}
	return h, nil
}

// New returns an initialized Host.
func New(cs modules.ConsensusSet, tpool modules.TransactionPool, wallet modules.Wallet, address string, persistDir string) (*Host, error) {
	return newHost(productionDependencies{}, cs, tpool, wallet, address, persistDir)
}

// Close shuts down the host.
func (h *Host) Close() error {
	return h.tg.Stop()
}

// ExternalSettings returns the hosts external settings. These values cannot be
// set by the user (host is configured through InternalSettings), and are the
// values that get displayed to other hosts on the network.
func (h *Host) ExternalSettings() modules.HostExternalSettings {
	h.mu.RLock()
	defer h.mu.RUnlock()
	err := h.tg.Add()
	if err != nil {
		build.Critical("Call to ExternalSettings after close")
	}
	defer h.tg.Done()
	return h.externalSettings()
}

// WorkingStatus returns the working state of the host, where working is
// defined as having received more than workingStatusThreshold settings calls
// over the period of workingStatusFrequency.
func (h *Host) WorkingStatus() modules.HostWorkingStatus {
	h.mu.RLock()
	defer h.mu.RUnlock()
	return h.workingStatus
}

// ConnectabilityStatus returns the connectability state of the host, whether
// the host can connect to itself on its configured netaddress.
func (h *Host) ConnectabilityStatus() modules.HostConnectabilityStatus {
	h.mu.RLock()
	defer h.mu.RUnlock()
	return h.connectabilityStatus
}

// FinancialMetrics returns information about the financial commitments,
// rewards, and activities of the host.
func (h *Host) FinancialMetrics() modules.HostFinancialMetrics {
	h.mu.RLock()
	defer h.mu.RUnlock()
	err := h.tg.Add()
	if err != nil {
		build.Critical("Call to FinancialMetrics after close")
	}
	defer h.tg.Done()
	return h.financialMetrics
}

// PublicKey returns the public key of the host that is used to facilitate
// relationships between the host and renter.
func (h *Host) PublicKey() types.SiaPublicKey {
	h.mu.RLock()
	defer h.mu.RUnlock()
	return h.publicKey
}

// SetInternalSettings updates the host's internal HostInternalSettings object.
func (h *Host) SetInternalSettings(settings modules.HostInternalSettings) error {
	h.mu.Lock()
	defer h.mu.Unlock()
	err := h.tg.Add()
	if err != nil {
		return err
	}
	defer h.tg.Done()

	// The host should not be accepting file contracts if it does not have an
	// unlock hash.
	if settings.AcceptingContracts {
		err := h.checkUnlockHash()
		if err != nil {
			return errors.New("internal settings not updated, no unlock hash: " + err.Error())
		}
	}

	if settings.NetAddress != "" {
		err := settings.NetAddress.IsValid()
		if err != nil {
			return errors.New("internal settings not updated, invalid NetAddress: " + err.Error())
		}
	}

	// Check if the net address for the host has changed. If it has, and it's
	// not equal to the auto address, then the host is going to need to make
	// another blockchain announcement.
	if h.settings.NetAddress != settings.NetAddress && settings.NetAddress != h.autoAddress {
		h.announced = false
	}

	h.settings = settings
	h.revisionNumber++

	err = h.saveSync()
	if err != nil {
		return errors.New("internal settings updated, but failed saving to disk: " + err.Error())
	}
	return nil
}

// InternalSettings returns the settings of a host.
func (h *Host) InternalSettings() modules.HostInternalSettings {
	h.mu.RLock()
	defer h.mu.RUnlock()
	err := h.tg.Add()
	if err != nil {
		return modules.HostInternalSettings{}
	}
	defer h.tg.Done()
	return h.settings
}
<|endoftext|>


<commit_before>//go:build !windows && !darwin
// +build !windows,!darwin

package volume

import (
	"errors"
	"regexp"
	"strconv"
	"strings"
)

var useAmixer bool

func init() {
	if _, err := execCmd([]string{"pactl", "info"}); err != nil {
		useAmixer = true
	}
}

func cmdEnv() []string {
	return []string{"LANG=C", "LC_ALL=C"}
}

func getVolumeCmd() []string {
	if useAmixer {
		return []string{"amixer", "get", "Master"}
	}
	return []string{"pactl", "list", "sinks"}
}

func getPADefaultSink() (string, error) {
	out, err := execCmd([]string{"pactl", "info"})
	if err != nil {
		return "", err
	}

	lines := strings.Split(string(out), "\n")

	defaultSinkStr := "Default Sink: "
	for _, line := range lines {
		s := strings.TrimLeft(line, " \t")
		if strings.HasPrefix(s, defaultSinkStr) {
			return strings.TrimSpace(strings.Replace(s, defaultSinkStr, "", 1)), nil
		}
	}
	return "", errors.New("could not find PulseAudio Default Sink")
}

var volumePattern = regexp.MustCompile(`\d+%`)

func parseVolume(out string) (int, error) {
	sinkName, sinkNameErr := getPADefaultSink()

	paDefaultSinkHooked := false

	if sinkNameErr != nil {
		paDefaultSinkHooked = true
	}

	lines := strings.Split(out, "\n")

	for _, line := range lines {
		s := strings.TrimLeft(line, " \t")

		if !useAmixer && strings.Contains(s, "Name: "+string(sinkName)) {
			paDefaultSinkHooked = true
		}

		if useAmixer && strings.Contains(s, "Playback") && strings.Contains(s, "%") ||
			!useAmixer && paDefaultSinkHooked && strings.HasPrefix(s, "Volume:") {
			volumeStr := volumePattern.FindString(s)
			return strconv.Atoi(volumeStr[:len(volumeStr)-1])
		}
	}
	return 0, errors.New("no volume found")
}

func setVolumeCmd(volume int) []string {
	if useAmixer {
		return []string{"amixer", "set", "Master", strconv.Itoa(volume) + "%"}
	}
	return []string{"pactl", "set-sink-volume", "@DEFAULT_SINK@", strconv.Itoa(volume) + "%"}
}

func increaseVolumeCmd(diff int) []string {
	var sign string
	if diff >= 0 {
		sign = "+"
	} else if useAmixer {
		diff = -diff
		sign = "-"
	}
	if useAmixer {
		return []string{"amixer", "set", "Master", strconv.Itoa(diff) + "%" + sign}
	}
	return []string{"pactl", "--", "set-sink-volume", "@DEFAULT_SINK@", sign + strconv.Itoa(diff) + "%"}
}

func getMutedCmd() []string {
	if useAmixer {
		return []string{"amixer", "get", "Master"}
	}
	return []string{"pactl", "list", "sinks"}
}

func parseMuted(out string) (bool, error) {
	sinkName, sinkNameErr := getPADefaultSink()

	paDefaultSinkHooked := false

	if sinkNameErr != nil {
		paDefaultSinkHooked = true
	}

	lines := strings.Split(out, "\n")
	for _, line := range lines {
		s := strings.TrimLeft(line, " \t")

		if !useAmixer && strings.Contains(s, "Name: "+string(sinkName)) {
			paDefaultSinkHooked = true
		}

		if useAmixer && strings.Contains(s, "Playback") && strings.Contains(s, "%") ||
			!useAmixer && paDefaultSinkHooked && strings.HasPrefix(s, "Mute: ") {
			if strings.Contains(s, "[off]") || strings.Contains(s, "yes") {
				return true, nil
			} else if strings.Contains(s, "[on]") || strings.Contains(s, "no") {
				return false, nil
			}
		}
	}
	return false, errors.New("no muted information found")
}

func muteCmd() []string {
	if useAmixer {
		return []string{"amixer", "-D", "pulse", "set", "Master", "mute"}
	}
	return []string{"pactl", "set-sink-mute", "@DEFAULT_SINK@", "1"}
}

func unmuteCmd() []string {
	if useAmixer {
		return []string{"amixer", "-D", "pulse", "set", "Master", "unmute"}
	}
	return []string{"pactl", "set-sink-mute", "@DEFAULT_SINK@", "0"}
}
<commit_msg>refactor default sink detection<commit_after>//go:build !windows && !darwin
// +build !windows,!darwin

package volume

import (
	"errors"
	"regexp"
	"strconv"
	"strings"
)

var useAmixer bool

func init() {
	if _, err := execCmd([]string{"pactl", "info"}); err != nil {
		useAmixer = true
	}
}

func cmdEnv() []string {
	return []string{"LANG=C", "LC_ALL=C"}
}

func getVolumeCmd() []string {
	if useAmixer {
		return []string{"amixer", "get", "Master"}
	}
	return []string{"pactl", "list", "sinks"}
}

func getPulseAudioDefaultSink() (string, error) {
	out, err := execCmd([]string{"pactl", "info"})
	if err != nil {
		return "", err
	}

	lines := strings.Split(string(out), "\n")

	defaultSinkStr := "Default Sink: "
	for _, line := range lines {
		s := strings.TrimLeft(line, " \t")
		if strings.HasPrefix(s, defaultSinkStr) {
			return strings.TrimSpace(strings.Replace(s, defaultSinkStr, "", 1)), nil
		}
	}
	return "", errors.New("could not find PulseAudio Default Sink")
}

var volumePattern = regexp.MustCompile(`\d+%`)

func parseVolume(out string) (int, error) {
	sinkName, sinkNameErr := getPulseAudioDefaultSink()
	isDefaultSink := sinkNameErr != nil

	lines := strings.Split(out, "\n")

	for _, line := range lines {
		s := strings.TrimLeft(line, " \t")

		if !useAmixer && !isDefaultSink && strings.Contains(s, "Name: "+sinkName) {
			isDefaultSink = true
		}

		if useAmixer && strings.Contains(s, "Playback") && strings.Contains(s, "%") ||
			!useAmixer && isDefaultSink && strings.HasPrefix(s, "Volume:") {
			volumeStr := volumePattern.FindString(s)
			return strconv.Atoi(volumeStr[:len(volumeStr)-1])
		}
	}
	return 0, errors.New("no volume found")
}

func setVolumeCmd(volume int) []string {
	if useAmixer {
		return []string{"amixer", "set", "Master", strconv.Itoa(volume) + "%"}
	}
	return []string{"pactl", "set-sink-volume", "@DEFAULT_SINK@", strconv.Itoa(volume) + "%"}
}

func increaseVolumeCmd(diff int) []string {
	var sign string
	if diff >= 0 {
		sign = "+"
	} else if useAmixer {
		diff = -diff
		sign = "-"
	}
	if useAmixer {
		return []string{"amixer", "set", "Master", strconv.Itoa(diff) + "%" + sign}
	}
	return []string{"pactl", "--", "set-sink-volume", "@DEFAULT_SINK@", sign + strconv.Itoa(diff) + "%"}
}

func getMutedCmd() []string {
	if useAmixer {
		return []string{"amixer", "get", "Master"}
	}
	return []string{"pactl", "list", "sinks"}
}

func parseMuted(out string) (bool, error) {
	sinkName, sinkNameErr := getPulseAudioDefaultSink()
	isDefaultSink := sinkNameErr != nil

	lines := strings.Split(out, "\n")
	for _, line := range lines {
		s := strings.TrimLeft(line, " \t")

		if !useAmixer && !isDefaultSink && strings.Contains(s, "Name: "+sinkName) {
			isDefaultSink = true
		}

		if useAmixer && strings.Contains(s, "Playback") && strings.Contains(s, "%") ||
			!useAmixer && isDefaultSink && strings.HasPrefix(s, "Mute: ") {
			if strings.Contains(s, "[off]") || strings.Contains(s, "yes") {
				return true, nil
			} else if strings.Contains(s, "[on]") || strings.Contains(s, "no") {
				return false, nil
			}
		}
	}
	return false, errors.New("no muted information found")
}

func muteCmd() []string {
	if useAmixer {
		return []string{"amixer", "-D", "pulse", "set", "Master", "mute"}
	}
	return []string{"pactl", "set-sink-mute", "@DEFAULT_SINK@", "1"}
}

func unmuteCmd() []string {
	if useAmixer {
		return []string{"amixer", "-D", "pulse", "set", "Master", "unmute"}
	}
	return []string{"pactl", "set-sink-mute", "@DEFAULT_SINK@", "0"}
}
<|endoftext|>


<commit_before>package waddell

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"os/exec"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/getlantern/testify/assert"
)

const (
	Hello         = "Hello"
	HelloYourself = "Hello %s!"

	NumPeers = 100

	TestTopic = TopicId(1)
)

// TestPeerIdRoundTrip makes sure that we can write and read a PeerId to/from a
// byte array.
func TestPeerIdRoundTrip(t *testing.T) {
	b := make([]byte, PeerIdLength)
	orig := randomPeerId()
	orig.write(b)
	read, err := readPeerId(b)
	if err != nil {
		t.Errorf("Unable to read peer id: %s", err)
	} else {
		assert.Equal(t, orig, read)
	}
}

func TestPeerIdStringRoundTrip(t *testing.T) {
	orig := randomPeerId()
	read, err := PeerIdFromString(orig.String())
	if err != nil {
		t.Errorf("Unable to read peer id: %s", err)
	} else {
		assert.Equal(t, orig, read)
	}
}

func TestTopicIdRoundTrip(t *testing.T) {
	orig := TopicId(5)
	read, err := readTopicId(orig.toBytes())
	if err != nil {
		t.Errorf("Error reading topic id: %s", err)
	} else {
		assert.Equal(t, orig, read)
	}
}

func TestTopicIdTruncation(t *testing.T) {
	orig := TopicId(5)
	_, err := readTopicId(orig.toBytes()[:1])
	assert.Error(t, err)
}

func TestBadDialerWithNoReconnect(t *testing.T) {
	cfg := &ClientConfig{
		ReconnectAttempts: 0,
		Dial: func() (net.Conn, error) {
			return nil, fmt.Errorf("I won't dial, no way!")
		},
	}
	client, err := NewClient(cfg)
	assert.Error(t, err, "Connecting with no reconnect attempts should have failed")
	if err == nil {
		client.Close()
	}
}

func TestBadDialerWithMultipleReconnect(t *testing.T) {
	cfg := &ClientConfig{
		ReconnectAttempts: 2,
		Dial: func() (net.Conn, error) {
			return nil, fmt.Errorf("I won't dial, no way!")
		},
	}
	start := time.Now()
	client, err := NewClient(cfg)
	defer client.Close()
	delta := time.Now().Sub(start)
	assert.Error(t, err, "Connecting with 2 reconnect attempts should have failed")
	expectedDelta := reconnectDelayInterval * 3
	assert.True(t, delta >= expectedDelta, fmt.Sprintf("Reconnecting didn't wait long enough. Should have waited %s, only waited %s", expectedDelta, delta))
}

func TestPeersPlainText(t *testing.T) {
	doTestPeers(t, false)
}

func TestPeersTLS(t *testing.T) {
	doTestPeers(t, true)
}

func doTestPeers(t *testing.T, useTLS bool) {
	socketsAtStart := countTCPFiles()
	closeActions := make([]func(), 0)
	peers := make([]*Client, 0, NumPeers)
	defer func() {
		for _, action := range closeActions {
			action()
		}
		// Wait a short time to let sockets finish closing
		time.Sleep(250 * time.Millisecond)
		socketsAtEnd := countTCPFiles()
		assert.Equal(t, socketsAtStart, socketsAtEnd, "All file descriptors should have been closed")

		// Make sure we can't do stuff with closed client
		client := peers[0]

		err := client.SendKeepAlive()
		assert.Error(t, err, "Attempting to SendKeepAlive on closed client should fail")

		// Make sure we can't obtain in or out topics after closing clients
		defer func() {
			defer func() {
				if r := recover(); r != nil {
					log.Tracef("Recovered: %v", r)
				}
			}()
			client.In(TestTopic)
			t.Error("Attempting to get in topic on closed client should have panicked")
		}()

		defer func() {
			defer func() {
				if r := recover(); r != nil {
					log.Tracef("Recovered: %v", r)
				}
			}()
			client.Out(TestTopic)
			t.Error("Attempting to get out topic on closed client should have panicked")
		}()
	}()

	pkfile := ""
	certfile := ""
	cert := ""
	if useTLS {
		pkfile = "waddell_test_pk.pem"
		certfile = "waddell_test_cert.pem"
		certBytes, err := ioutil.ReadFile(certfile)
		if err != nil {
			log.Fatalf("Unable to read cert from file: %s", err)
		}
		cert = string(certBytes)
	}

	listener, err := Listen("localhost:0", pkfile, certfile)
	if err != nil {
		log.Fatalf("Unable to listen: %s", err)
	}

	go func() {
		server := &Server{}
		err = server.Serve(listener)
	}()

	serverAddr := listener.Addr().String()

	succeedOnDial := int32(0)
	dial := func() (net.Conn, error) {
		if atomic.CompareAndSwapInt32(&succeedOnDial, 0, 1) {
			return nil, fmt.Errorf("Deliberately failing the first time around")
		}
		return net.Dial("tcp", serverAddr)
	}

	idCallbackTriggered := int32(0)
	connect := func() *Client {
		cfg := &ClientConfig{
			Dial:              dial,
			ServerCert:        cert,
			ReconnectAttempts: 1,
			OnId: func(id PeerId) {
				atomic.AddInt32(&idCallbackTriggered, 1)
			},
		}
		client, err := NewClient(cfg)
		if err != nil {
			log.Fatalf("Unable to connect client: %s", err)
		}
		return client
	}

	peersCh := make(chan *Client, NumPeers)
	// Connect clients
	for i := 0; i < NumPeers; i++ {
		go func() {
			peer := connect()
			assert.NoError(t, err, "Unable to connect peer")
			peersCh <- peer
		}()
	}

	for i := 0; i < NumPeers; i++ {
		peer := <-peersCh
		peers = append(peers, peer)
		closeActions = append(closeActions, func() {
			err := peer.Close()
			assert.NoError(t, err, "Closing client shouldn't result in error")
		})
	}

	var wg sync.WaitGroup
	wg.Add(NumPeers)

	// Send some large data to a peer that doesn't read, just to make sure we
	// handle blocked readers okay

	// We include ClientMgr here to test it, not because it's convenient
	var cbAddr string
	var cbId PeerId
	var cbMutex sync.Mutex
	clientMgr := &ClientMgr{
		Dial: func(addr string) (net.Conn, error) {
			return net.Dial("tcp", addr)
		},
		ServerCert:        cert,
		ReconnectAttempts: 1,
		OnId: func(addr string, id PeerId) {
			cbMutex.Lock()
			defer cbMutex.Unlock()
			cbAddr = addr
			cbId = id
		},
	}
	closeActions = append(closeActions, func() {
		errs := clientMgr.Close()
		assert.Nil(t, errs, "Closing clientMgr shouldn't result in errors")
	})

	badPeer, err := clientMgr.ClientTo(serverAddr)
	if err != nil {
		log.Fatalf("Unable to connect bad peer: %s", err)
	}
	cbMutex.Lock()
	badPeerId := badPeer.CurrentId()
	assert.Equal(t, serverAddr, cbAddr, "IdCallback should have recorded the server's addr")
	assert.Equal(t, badPeerId, cbId, "IdCallback should have recorded the correct id")
	cbMutex.Unlock()
	ld := largeData()
	for i := 0; i < 10; i++ {
		if err != nil {
			log.Fatalf("Unable to get peer id: %s", err)
		}
		badPeer.Out(TestTopic) <- Message(badPeerId, ld)
	}

	// Simulate readers and writers
	for i := 0; i < NumPeers; i++ {
		peer := peers[i]
		isWriter := i%2 == 1

		if isWriter {
			go func() {
				// Simulate writer
				defer wg.Done()

				// Write to each reader
				for j := 0; j < NumPeers; j += 2 {
					recip := peers[j]
					peer.Out(TestTopic) <- Message(recip.CurrentId(), []byte(Hello[:2]), []byte(Hello[2:]))
					if err != nil {
						log.Fatalf("Unable to write hello: %s", err)
					} else {
						resp := <-peer.In(TestTopic)
						assert.Equal(t, fmt.Sprintf(HelloYourself, peer.CurrentId()), string(resp.Body), "Response should match expected.")
						assert.Equal(t, recip.CurrentId(), resp.From, "Peer on response should match expected")
					}
				}
			}()
		} else {
			go func() {
				// Simulate reader
				defer wg.Done()

				// Read from all readers
				for j := 1; j < NumPeers; j += 2 {
					err := peer.SendKeepAlive()
					if err != nil {
						log.Fatalf("Unable to send KeepAlive: %s", err)
					}
					msg := <-peer.In(TestTopic)
					assert.Equal(t, Hello, string(msg.Body), "Hello message should match expected")
					peer.Out(TestTopic) <- Message(msg.From, []byte(fmt.Sprintf(HelloYourself, msg.From)))
					if err != nil {
						log.Fatalf("Unable to write response to HELLO message: %s", err)
					}
				}
			}()
		}
	}

	wg.Wait()

	// Close server last
	closeActions = append(closeActions, func() {
		err := listener.Close()
		assert.NoError(t, err, "Closing listener shouldn't fail")
	})
	assert.Equal(t, NumPeers, idCallbackTriggered, "IdCallback should have been called once for each connected peer")
}

func largeData() []byte {
	b := make([]byte, 60000)
	for i := 0; i < len(b); i++ {
		b[i] = byte(rand.Int())
	}
	return b
}

// see https://groups.google.com/forum/#!topic/golang-nuts/c0AnWXjzNIA
func countTCPFiles() int {
	out, err := exec.Command("lsof", "-p", fmt.Sprintf("%v", os.Getpid())).Output()
	if err != nil {
		log.Fatal(err)
	}
	log.Tracef("lsof result: %s", string(out))
	return bytes.Count(out, []byte("TCP"))
}
<commit_msg>getlantern/lantern#2117 Switched to utility for counting file descriptors<commit_after>package waddell

import (
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/getlantern/fdcount"
	"github.com/getlantern/testify/assert"
)

const (
	Hello         = "Hello"
	HelloYourself = "Hello %s!"

	NumPeers = 100

	TestTopic = TopicId(1)
)

// TestPeerIdRoundTrip makes sure that we can write and read a PeerId to/from a
// byte array.
func TestPeerIdRoundTrip(t *testing.T) {
	b := make([]byte, PeerIdLength)
	orig := randomPeerId()
	orig.write(b)
	read, err := readPeerId(b)
	if err != nil {
		t.Errorf("Unable to read peer id: %s", err)
	} else {
		assert.Equal(t, orig, read)
	}
}

func TestPeerIdStringRoundTrip(t *testing.T) {
	orig := randomPeerId()
	read, err := PeerIdFromString(orig.String())
	if err != nil {
		t.Errorf("Unable to read peer id: %s", err)
	} else {
		assert.Equal(t, orig, read)
	}
}

func TestTopicIdRoundTrip(t *testing.T) {
	orig := TopicId(5)
	read, err := readTopicId(orig.toBytes())
	if err != nil {
		t.Errorf("Error reading topic id: %s", err)
	} else {
		assert.Equal(t, orig, read)
	}
}

func TestTopicIdTruncation(t *testing.T) {
	orig := TopicId(5)
	_, err := readTopicId(orig.toBytes()[:1])
	assert.Error(t, err)
}

func TestBadDialerWithNoReconnect(t *testing.T) {
	cfg := &ClientConfig{
		ReconnectAttempts: 0,
		Dial: func() (net.Conn, error) {
			return nil, fmt.Errorf("I won't dial, no way!")
		},
	}
	client, err := NewClient(cfg)
	assert.Error(t, err, "Connecting with no reconnect attempts should have failed")
	if err == nil {
		client.Close()
	}
}

func TestBadDialerWithMultipleReconnect(t *testing.T) {
	cfg := &ClientConfig{
		ReconnectAttempts: 2,
		Dial: func() (net.Conn, error) {
			return nil, fmt.Errorf("I won't dial, no way!")
		},
	}
	start := time.Now()
	client, err := NewClient(cfg)
	defer client.Close()
	delta := time.Now().Sub(start)
	assert.Error(t, err, "Connecting with 2 reconnect attempts should have failed")
	expectedDelta := reconnectDelayInterval * 3
	assert.True(t, delta >= expectedDelta, fmt.Sprintf("Reconnecting didn't wait long enough. Should have waited %s, only waited %s", expectedDelta, delta))
}

func TestPeersPlainText(t *testing.T) {
	doTestPeers(t, false)
}

func TestPeersTLS(t *testing.T) {
	doTestPeers(t, true)
}

func doTestPeers(t *testing.T, useTLS bool) {
	_, fdc, err := fdcount.Matching("TCP")
	if err != nil {
		t.Fatal(err)
	}

	closeActions := make([]func(), 0)
	peers := make([]*Client, 0, NumPeers)
	defer func() {
		for _, action := range closeActions {
			action()
		}

		// Wait a short time to let sockets finish closing
		time.Sleep(250 * time.Millisecond)
		assert.NoError(t, fdc.AssertDelta(0), "All file descriptors should have been closed")

		// Make sure we can't do stuff with closed client
		client := peers[0]

		err := client.SendKeepAlive()
		assert.Error(t, err, "Attempting to SendKeepAlive on closed client should fail")

		// Make sure we can't obtain in or out topics after closing clients
		defer func() {
			defer func() {
				if r := recover(); r != nil {
					log.Tracef("Recovered: %v", r)
				}
			}()
			client.In(TestTopic)
			t.Error("Attempting to get in topic on closed client should have panicked")
		}()

		defer func() {
			defer func() {
				if r := recover(); r != nil {
					log.Tracef("Recovered: %v", r)
				}
			}()
			client.Out(TestTopic)
			t.Error("Attempting to get out topic on closed client should have panicked")
		}()
	}()

	pkfile := ""
	certfile := ""
	cert := ""
	if useTLS {
		pkfile = "waddell_test_pk.pem"
		certfile = "waddell_test_cert.pem"
		certBytes, err := ioutil.ReadFile(certfile)
		if err != nil {
			log.Fatalf("Unable to read cert from file: %s", err)
		}
		cert = string(certBytes)
	}

	listener, err := Listen("localhost:0", pkfile, certfile)
	if err != nil {
		log.Fatalf("Unable to listen: %s", err)
	}

	go func() {
		server := &Server{}
		err = server.Serve(listener)
	}()

	serverAddr := listener.Addr().String()

	succeedOnDial := int32(0)
	dial := func() (net.Conn, error) {
		if atomic.CompareAndSwapInt32(&succeedOnDial, 0, 1) {
			return nil, fmt.Errorf("Deliberately failing the first time around")
		}
		return net.Dial("tcp", serverAddr)
	}

	idCallbackTriggered := int32(0)
	connect := func() *Client {
		cfg := &ClientConfig{
			Dial:              dial,
			ServerCert:        cert,
			ReconnectAttempts: 1,
			OnId: func(id PeerId) {
				atomic.AddInt32(&idCallbackTriggered, 1)
			},
		}
		client, err := NewClient(cfg)
		if err != nil {
			log.Fatalf("Unable to connect client: %s", err)
		}
		return client
	}

	peersCh := make(chan *Client, NumPeers)
	// Connect clients
	for i := 0; i < NumPeers; i++ {
		go func() {
			peer := connect()
			assert.NoError(t, err, "Unable to connect peer")
			peersCh <- peer
		}()
	}

	for i := 0; i < NumPeers; i++ {
		peer := <-peersCh
		peers = append(peers, peer)
		closeActions = append(closeActions, func() {
			err := peer.Close()
			assert.NoError(t, err, "Closing client shouldn't result in error")
		})
	}

	var wg sync.WaitGroup
	wg.Add(NumPeers)

	// Send some large data to a peer that doesn't read, just to make sure we
	// handle blocked readers okay

	// We include ClientMgr here to test it, not because it's convenient
	var cbAddr string
	var cbId PeerId
	var cbMutex sync.Mutex
	clientMgr := &ClientMgr{
		Dial: func(addr string) (net.Conn, error) {
			return net.Dial("tcp", addr)
		},
		ServerCert:        cert,
		ReconnectAttempts: 1,
		OnId: func(addr string, id PeerId) {
			cbMutex.Lock()
			defer cbMutex.Unlock()
			cbAddr = addr
			cbId = id
		},
	}
	closeActions = append(closeActions, func() {
		errs := clientMgr.Close()
		assert.Nil(t, errs, "Closing clientMgr shouldn't result in errors")
	})

	badPeer, err := clientMgr.ClientTo(serverAddr)
	if err != nil {
		log.Fatalf("Unable to connect bad peer: %s", err)
	}
	cbMutex.Lock()
	badPeerId := badPeer.CurrentId()
	assert.Equal(t, serverAddr, cbAddr, "IdCallback should have recorded the server's addr")
	assert.Equal(t, badPeerId, cbId, "IdCallback should have recorded the correct id")
	cbMutex.Unlock()
	ld := largeData()
	for i := 0; i < 10; i++ {
		if err != nil {
			log.Fatalf("Unable to get peer id: %s", err)
		}
		badPeer.Out(TestTopic) <- Message(badPeerId, ld)
	}

	// Simulate readers and writers
	for i := 0; i < NumPeers; i++ {
		peer := peers[i]
		isWriter := i%2 == 1

		if isWriter {
			go func() {
				// Simulate writer
				defer wg.Done()

				// Write to each reader
				for j := 0; j < NumPeers; j += 2 {
					recip := peers[j]
					peer.Out(TestTopic) <- Message(recip.CurrentId(), []byte(Hello[:2]), []byte(Hello[2:]))
					if err != nil {
						log.Fatalf("Unable to write hello: %s", err)
					} else {
						resp := <-peer.In(TestTopic)
						assert.Equal(t, fmt.Sprintf(HelloYourself, peer.CurrentId()), string(resp.Body), "Response should match expected.")
						assert.Equal(t, recip.CurrentId(), resp.From, "Peer on response should match expected")
					}
				}
			}()
		} else {
			go func() {
				// Simulate reader
				defer wg.Done()

				// Read from all readers
				for j := 1; j < NumPeers; j += 2 {
					err := peer.SendKeepAlive()
					if err != nil {
						log.Fatalf("Unable to send KeepAlive: %s", err)
					}
					msg := <-peer.In(TestTopic)
					assert.Equal(t, Hello, string(msg.Body), "Hello message should match expected")
					peer.Out(TestTopic) <- Message(msg.From, []byte(fmt.Sprintf(HelloYourself, msg.From)))
					if err != nil {
						log.Fatalf("Unable to write response to HELLO message: %s", err)
					}
				}
			}()
		}
	}

	wg.Wait()

	// Close server last
	closeActions = append(closeActions, func() {
		err := listener.Close()
		assert.NoError(t, err, "Closing listener shouldn't fail")
	})
	assert.Equal(t, NumPeers, idCallbackTriggered, "IdCallback should have been called once for each connected peer")
}

func largeData() []byte {
	b := make([]byte, 60000)
	for i := 0; i < len(b); i++ {
		b[i] = byte(rand.Int())
	}
	return b
}
<|endoftext|>


<commit_before>package audit

import (
	"encoding/json"
	"io"
	"time"

	"github.com/hashicorp/vault/logical"
)

// FormatJSON is a Formatter implementation that structures data into
// a JSON format.
type FormatJSON struct{}

func (f *FormatJSON) FormatRequest(
	w io.Writer,
	auth *logical.Auth,
	req *logical.Request,
	err error) error {

	// If auth is nil, make an empty one
	if auth == nil {
		auth = new(logical.Auth)
	}
	var errString string
	if err != nil {
		errString = err.Error()
	}

	// Encode!
	enc := json.NewEncoder(w)
	return enc.Encode(&JSONRequestEntry{
		Time:  time.Now().UTC().Format(time.RFC3339),
		Type:  "request",
		Error: errString,

		Auth: JSONAuth{
			DisplayName: auth.DisplayName,
			Policies:    auth.Policies,
			Metadata:    auth.Metadata,
		},

		Request: JSONRequest{
			ClientToken: req.ClientToken,
			Operation:   req.Operation,
			Path:        req.Path,
			Data:        req.Data,
			RemoteAddr:  getRemoteAddr(req),
		},
	})
}

func (f *FormatJSON) FormatResponse(
	w io.Writer,
	auth *logical.Auth,
	req *logical.Request,
	resp *logical.Response,
	err error) error {
	// If things are nil, make empty to avoid panics
	if auth == nil {
		auth = new(logical.Auth)
	}
	if resp == nil {
		resp = new(logical.Response)
	}
	var errString string
	if err != nil {
		errString = err.Error()
	}

	var respAuth *JSONAuth
	if resp.Auth != nil {
		respAuth = &JSONAuth{
			ClientToken: resp.Auth.ClientToken,
			DisplayName: resp.Auth.DisplayName,
			Policies:    resp.Auth.Policies,
			Metadata:    resp.Auth.Metadata,
		}
	}

	var respSecret *JSONSecret
	if resp.Secret != nil {
		respSecret = &JSONSecret{
			LeaseID: resp.Secret.LeaseID,
		}
	}

	// Encode!
	enc := json.NewEncoder(w)
	return enc.Encode(&JSONResponseEntry{
		Time:  time.Now().UTC().Format(time.RFC3339),
		Type:  "response",
		Error: errString,

		Auth: JSONAuth{
			Policies: auth.Policies,
			Metadata: auth.Metadata,
		},

		Request: JSONRequest{
			Operation:  req.Operation,
			Path:       req.Path,
			Data:       req.Data,
			RemoteAddr: getRemoteAddr(req),
		},

		Response: JSONResponse{
			Auth:     respAuth,
			Secret:   respSecret,
			Data:     resp.Data,
			Redirect: resp.Redirect,
		},
	})
}

// JSONRequest is the structure of a request audit log entry in JSON.
type JSONRequestEntry struct {
	Time    string      `json:"time"`
	Type    string      `json:"type"`
	Auth    JSONAuth    `json:"auth"`
	Request JSONRequest `json:"request"`
	Error   string      `json:"error"`
}

// JSONResponseEntry is the structure of a response audit log entry in JSON.
type JSONResponseEntry struct {
	Time     string       `json:"time"`
	Type     string       `json:"type"`
	Error    string       `json:"error"`
	Auth     JSONAuth     `json:"auth"`
	Request  JSONRequest  `json:"request"`
	Response JSONResponse `json:"response"`
}

type JSONRequest struct {
	Operation   logical.Operation      `json:"operation"`
	ClientToken string                 `json:"client_token"`
	Path        string                 `json:"path"`
	Data        map[string]interface{} `json:"data"`
	RemoteAddr  string                 `json:"remote_address"`
}

type JSONResponse struct {
	Auth     *JSONAuth              `json:"auth,omitempty"`
	Secret   *JSONSecret            `json:"secret,emitempty"`
	Data     map[string]interface{} `json:"data"`
	Redirect string                 `json:"redirect"`
}

type JSONAuth struct {
	ClientToken string            `json:"client_token,omitempty"`
	DisplayName string            `json:"display_name"`
	Policies    []string          `json:"policies"`
	Metadata    map[string]string `json:"metadata"`
}

type JSONSecret struct {
	LeaseID string `json:"lease_id"`
}

// getRemoteAddr safely gets the remote address avoiding a nil pointer
func getRemoteAddr(req *logical.Request) string {
	if req != nil && req.Connection != nil {
		return req.Connection.RemoteAddr
	}
	return ""
}
<commit_msg>Enable printing of accessor in audit logs<commit_after>package audit

import (
	"encoding/json"
	"io"
	"time"

	"github.com/hashicorp/vault/logical"
)

// FormatJSON is a Formatter implementation that structures data into
// a JSON format.
type FormatJSON struct{}

func (f *FormatJSON) FormatRequest(
	w io.Writer,
	auth *logical.Auth,
	req *logical.Request,
	err error) error {

	// If auth is nil, make an empty one
	if auth == nil {
		auth = new(logical.Auth)
	}
	var errString string
	if err != nil {
		errString = err.Error()
	}

	// Encode!
	enc := json.NewEncoder(w)
	return enc.Encode(&JSONRequestEntry{
		Time:  time.Now().UTC().Format(time.RFC3339),
		Type:  "request",
		Error: errString,

		Auth: JSONAuth{
			DisplayName: auth.DisplayName,
			Policies:    auth.Policies,
			Metadata:    auth.Metadata,
		},

		Request: JSONRequest{
			ClientToken: req.ClientToken,
			Operation:   req.Operation,
			Path:        req.Path,
			Data:        req.Data,
			RemoteAddr:  getRemoteAddr(req),
		},
	})
}

func (f *FormatJSON) FormatResponse(
	w io.Writer,
	auth *logical.Auth,
	req *logical.Request,
	resp *logical.Response,
	err error) error {
	// If things are nil, make empty to avoid panics
	if auth == nil {
		auth = new(logical.Auth)
	}
	if resp == nil {
		resp = new(logical.Response)
	}
	var errString string
	if err != nil {
		errString = err.Error()
	}

	var respAuth *JSONAuth
	if resp.Auth != nil {
		respAuth = &JSONAuth{
			ClientToken: resp.Auth.ClientToken,
			Accessor:    resp.Auth.Accessor,
			DisplayName: resp.Auth.DisplayName,
			Policies:    resp.Auth.Policies,
			Metadata:    resp.Auth.Metadata,
		}
	}

	var respSecret *JSONSecret
	if resp.Secret != nil {
		respSecret = &JSONSecret{
			LeaseID: resp.Secret.LeaseID,
		}
	}

	// Encode!
	enc := json.NewEncoder(w)
	return enc.Encode(&JSONResponseEntry{
		Time:  time.Now().UTC().Format(time.RFC3339),
		Type:  "response",
		Error: errString,

		Auth: JSONAuth{
			Policies: auth.Policies,
			Metadata: auth.Metadata,
		},

		Request: JSONRequest{
			Operation:  req.Operation,
			Path:       req.Path,
			Data:       req.Data,
			RemoteAddr: getRemoteAddr(req),
		},

		Response: JSONResponse{
			Auth:     respAuth,
			Secret:   respSecret,
			Data:     resp.Data,
			Redirect: resp.Redirect,
		},
	})
}

// JSONRequest is the structure of a request audit log entry in JSON.
type JSONRequestEntry struct {
	Time    string      `json:"time"`
	Type    string      `json:"type"`
	Auth    JSONAuth    `json:"auth"`
	Request JSONRequest `json:"request"`
	Error   string      `json:"error"`
}

// JSONResponseEntry is the structure of a response audit log entry in JSON.
type JSONResponseEntry struct {
	Time     string       `json:"time"`
	Type     string       `json:"type"`
	Error    string       `json:"error"`
	Auth     JSONAuth     `json:"auth"`
	Request  JSONRequest  `json:"request"`
	Response JSONResponse `json:"response"`
}

type JSONRequest struct {
	Operation   logical.Operation      `json:"operation"`
	ClientToken string                 `json:"client_token"`
	Path        string                 `json:"path"`
	Data        map[string]interface{} `json:"data"`
	RemoteAddr  string                 `json:"remote_address"`
}

type JSONResponse struct {
	Auth     *JSONAuth              `json:"auth,omitempty"`
	Secret   *JSONSecret            `json:"secret,emitempty"`
	Data     map[string]interface{} `json:"data"`
	Redirect string                 `json:"redirect"`
}

type JSONAuth struct {
	ClientToken string            `json:"client_token,omitempty"`
	Accessor    string            `json:"accessor,omitempty"`
	DisplayName string            `json:"display_name"`
	Policies    []string          `json:"policies"`
	Metadata    map[string]string `json:"metadata"`
}

type JSONSecret struct {
	LeaseID string `json:"lease_id"`
}

// getRemoteAddr safely gets the remote address avoiding a nil pointer
func getRemoteAddr(req *logical.Request) string {
	if req != nil && req.Connection != nil {
		return req.Connection.RemoteAddr
	}
	return ""
}
<|endoftext|>


<commit_before>package jwt

import (
	"testing"

	"github.com/paulormart/assert"
)

func TestToken(t *testing.T) {

	pk, err := LoadPrivateKey("")
	if err != nil {
		t.Fatal(err)
	}

	token, err := NewToken("identifier", "bearer", "users jobs", 3650, pk)
	if err != nil {
		t.Fatal(err)
	}

	assert.Equal(t, true, len(token) > 0)

	err = Verify(token, &pk.PublicKey)
	if err != nil {
		t.Fatal(err)
	}

	pubk, err := LoadPublicKey("")
	if err != nil {
		t.Fatal(err)
	}

	err = Verify(token, pubk)
	if err != nil {
		t.Fatal(err)
	}

	i := GetIdentifier(token)
	if i != "bearer" {
		t.Fatal("invalid identifier")
	}

	a := GetAudience(token)
	if a != "bearer" {
		t.Fatal("invalid audience")
	}

	s := GetScope(token)
	if s != "users jobs" {
		t.Fatal("invalid scope")
	}
}
<commit_msg>fix error - test get identifier<commit_after>package jwt

import (
	"testing"

	"github.com/paulormart/assert"
)

func TestToken(t *testing.T) {

	pk, err := LoadPrivateKey("")
	if err != nil {
		t.Fatal(err)
	}

	token, err := NewToken("identifier", "bearer", "users jobs", 3650, pk)
	if err != nil {
		t.Fatal(err)
	}

	assert.Equal(t, true, len(token) > 0)

	err = Verify(token, &pk.PublicKey)
	if err != nil {
		t.Fatal(err)
	}

	pubk, err := LoadPublicKey("")
	if err != nil {
		t.Fatal(err)
	}

	err = Verify(token, pubk)
	if err != nil {
		t.Fatal(err)
	}

	i := GetIdentifier(token)
	if i != "identifier" {
		t.Fatal("invalid identifier")
	}

	a := GetAudience(token)
	if a != "bearer" {
		t.Fatal("invalid audience")
	}

	s := GetScope(token)
	if s != "users jobs" {
		t.Fatal("invalid scope")
	}
}
<|endoftext|>


<commit_before>package client

import (
	"sync"

	"github.com/streadway/amqp"
)

type dispatcher struct {
	sync.RWMutex
	routes map[string]chan amqp.Delivery
}

func newDispatcher() *dispatcher {
	return &dispatcher{
		routes: make(map[string]chan amqp.Delivery),
	}
}

func (d *dispatcher) add(routingKey string) chan amqp.Delivery {
	d.Lock()
	defer d.Unlock()
	ch := make(chan amqp.Delivery, 1)
	d.routes[routingKey] = ch
	return ch
}

func (d *dispatcher) pop(routingKey string) chan amqp.Delivery {
	d.Lock()
	defer d.Unlock()
	if channel, ok := d.routes[routingKey]; ok {
		delete(d.routes, routingKey)
		return channel
	}
	return nil
}
<commit_msg>Make dispatcher a little more generic - its requests we're tracking<commit_after>package client

import (
	"sync"

	"github.com/streadway/amqp"
)

type dispatcher struct {
	sync.RWMutex
	requests map[string]chan amqp.Delivery
}

func newDispatcher() *dispatcher {
	return &dispatcher{
		requests: make(map[string]chan amqp.Delivery),
	}
}

func (d *dispatcher) add(requestId string) chan amqp.Delivery {
	d.Lock()
	defer d.Unlock()
	ch := make(chan amqp.Delivery, 1)
	d.requests[requestId] = ch
	return ch
}

func (d *dispatcher) pop(requestId string) chan amqp.Delivery {
	d.Lock()
	defer d.Unlock()
	if channel, ok := d.requests[requestId]; ok {
		delete(d.requests, requestId)
		return channel
	}
	return nil
}
<|endoftext|>


<commit_before>package hod

import (
	"bytes"
	"encoding/json"
	"net/http"
	"time"

	turtle "github.com/gtfierro/hod/goraptor"
	"github.com/pkg/error"
)

type ResultMap map[string]turtle.URI
type Result struct {
	Rows    []ResultMap
	Count   int
	Elapsed time.Duration
}

// Placeholder for now
type Options struct {
	ValueOnly bool
}

func DefaultOptions() *Options {
	return &Options{
		ValueOnly: false,
	}
}

type HodClientHTTP struct {
	url string
}

func NewHTTPClient(url string) *HodClientHTTP {
	c := &HodClientHTTP{
		url: url,
	}

	return c
}

func (c *HodClientHTTP) DoQuery(query string, options *Options) (res Result, err error) {
	b := new(bytes.Buffer)
	b.WriteString(query)

	resp, err := http.Post(c.url, "application/json", b)
	defer resp.Body.Close()
	if err != nil {
		err = errors.Wrap(err, "Problem posting to HodDB")
		return
	}

	if err = json.NewDecoder(resp.Body).Decode(&res); err != nil {
		err = errors.Wrap(err, "Problem decoding response body")
		return
	}

	return
}
<commit_msg>Add HodDB clients: HTTP + bw2<commit_after>// This package provide a couple simple HodDB clients
// HTTP Client:
//
//
//    package main
//
//    import (
//    	"fmt"
//    	"github.com/gtfierro/hod/clients/go"
//    )
//
//    func main() {
//    	c := hod.NewHTTPClient("http://ciee.cal-sdb.org/api/query")
//
//    	query := `SELECT ?x ?r WHERE {
//        ?x rdf:type/rdfs:subClassOf* brick:Temperature_Sensor .
//        ?x bf:isLocatedIn ?r .
//      };`
//
//    	res, err := c.DoQuery(query, nil)
//    	fmt.Println(err)
//    	fmt.Printf("%+v", res)
//
//    }
//
//
// BOSSWAVE client
//
//    package main
//
//    import (
//    	"fmt"
//    	"github.com/gtfierro/hod/clients/go"
//	    bw2 "gopkg.in/immesys/bw2bind.v5"
//    )
//
//    func main() {
//
//	    client := bw2.ConnectOrExit("")
//	    client.OverrideAutoChainTo(true)
//	    client.SetEntityFromEnvironOrExit()
//	    bc, err := hod.NewBW2Client(client, "ciee/hod")
//	    if err != nil {
//          panic(err)
//      }
//
//    	query := `SELECT ?x ?r WHERE {
//        ?x rdf:type/rdfs:subClassOf* brick:Temperature_Sensor .
//        ?x bf:isLocatedIn ?r .
//      };`
//
//	    res, err := bc.DoQuery(query, nil)
//	    fmt.Println(err)
//	    fmt.Printf("%+v", res)
//
//    }
package hod

import (
	"bytes"
	"encoding/json"
	"fmt"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	turtle "github.com/gtfierro/hod/goraptor"
	"github.com/pkg/errors"
	bw2 "gopkg.in/immesys/bw2bind.v5"
)

var ErrNoResponse = errors.New("No response from archiver")

const RESULT_PONUM = `2.0.10.2/32`

var QUERY_PONUM = bw2.FromDotForm(`2.0.10.1`)

func init() {
	rand.Seed(time.Now().UnixNano())
}

type ResultMap map[string]turtle.URI
type Result struct {
	Rows    []ResultMap
	Count   int
	Elapsed time.Duration
}

type Options struct {
	Timeout time.Duration
}

func DefaultOptions() *Options {
	return &Options{
		30 * time.Second,
	}
}

type HodClientHTTP struct {
	url string
}

func NewHTTPClient(url string) *HodClientHTTP {
	c := &HodClientHTTP{
		url: url,
	}

	return c
}

func (c *HodClientHTTP) DoQuery(query string, options *Options) (res Result, err error) {
	b := new(bytes.Buffer)
	b.WriteString(query)

	resp, err := http.Post(c.url, "application/json", b)
	defer resp.Body.Close()
	if err != nil {
		err = errors.Wrap(err, "Problem posting to HodDB")
		return
	}

	if err = json.NewDecoder(resp.Body).Decode(&res); err != nil {
		err = errors.Wrap(err, "Problem decoding response body")
		return
	}

	return
}

type HodClientBW2 struct {
	client  *bw2.BW2Client
	uri     string
	waiting map[string]chan BW2Result
	sync.RWMutex
}

type BW2Result struct {
	Nonce   string        `msgpack:"Nonce"`
	Error   error         `msgpack:-`
	err     string        `msgpack:"Error"`
	Rows    []ResultMap   `msgpack:"Rows"`
	Count   int           `msgpack:"Count"`
	Elapsed time.Duration `msgpack:-`
	elapsed int64         `msgpack:"Elapsed"`
}

type querymsg struct {
	Nonce string
	Query string
}

func NewBW2Client(client *bw2.BW2Client, uri string) (*HodClientBW2, error) {
	c := &HodClientBW2{
		client:  client,
		uri:     strings.TrimSuffix(uri, "/") + "/s.hod/_/i.hod",
		waiting: make(map[string]chan BW2Result),
	}

	sub, err := c.client.Subscribe(&bw2.SubscribeParams{
		URI: c.uri + "/signal/result",
	})
	if err != nil {
		return c, errors.Wrapf(err, "Could not subscribe to HodDB (%s)", uri)
	}

	go func() {
		for msg := range sub {
			po := msg.GetOnePODF(RESULT_PONUM)
			if po == nil {
				continue
			}

			var res BW2Result

			if err := po.(bw2.MsgPackPayloadObject).ValueInto(&res); err != nil {
				res.Error = err
			}
			fmt.Printf("%+v", res)

			c.Lock()
			if replyChan, found := c.waiting[res.Nonce]; found {
				select {
				case replyChan <- res:
					delete(c.waiting, res.Nonce)
				default:
				}
			}
			c.Unlock()
		}
	}()
	return c, nil
}

func (c *HodClientBW2) markWaitFor(nonce string, replyChan chan BW2Result) {
	c.Lock()
	c.waiting[nonce] = replyChan
	c.Unlock()
}

func (bc *HodClientBW2) DoQuery(query string, opts *Options) (res BW2Result, err error) {
	nonce := strconv.FormatUint(rand.Uint64(), 10)
	msg := querymsg{nonce, query}
	po, _ := bw2.CreateMsgPackPayloadObject(QUERY_PONUM, msg)

	if err = bc.client.Publish(&bw2.PublishParams{
		URI:            bc.uri + "/slot/query",
		PayloadObjects: []bw2.PayloadObject{po},
	}); err != nil {
		return
	}

	if opts == nil {
		opts = DefaultOptions()
	}

	timeoutChan := time.After(opts.Timeout)

	replyChan := make(chan BW2Result, 1)
	bc.markWaitFor(nonce, replyChan)

	for {
		select {
		case <-timeoutChan:
			err = ErrNoResponse
			return
		case res = <-replyChan:
			if res.Error != nil {
				err = res.Error
				return
			}
		}
	}

	return
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package collectors

import (
	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
	"golang.org/x/net/context"
	"k8s.io/client-go/kubernetes"
	v1batch "k8s.io/client-go/pkg/apis/batch/v1"
	"k8s.io/client-go/tools/cache"
)

var (
	descJobInfo = prometheus.NewDesc(
		"kube_job_info",
		"Information about job.",
		[]string{"namespace", "job"}, nil,
	)
	descJobCreated = prometheus.NewDesc(
		"kube_job_created",
		"Unix creation timestamp",
		[]string{"namespace", "job"}, nil,
	)
	descJobSpecParallelism = prometheus.NewDesc(
		"kube_job_spec_parallelism",
		"The maximum desired number of pods the job should run at any given time.",
		[]string{"namespace", "job"}, nil,
	)
	descJobSpecCompletions = prometheus.NewDesc(
		"kube_job_spec_completions",
		"The desired number of successfully finished pods the job should be run with.",
		[]string{"namespace", "job"}, nil,
	)
	descJobSpecActiveDeadlineSeconds = prometheus.NewDesc(
		"kube_job_spec_active_deadline_seconds",
		"The duration in seconds relative to the startTime that the job may be active before the system tries to terminate it.",
		[]string{"namespace", "job"}, nil,
	)
	descJobStatusSucceeded = prometheus.NewDesc(
		"kube_job_status_succeeded",
		"The number of pods which reached Phase Succeeded.",
		[]string{"namespace", "job"}, nil,
	)
	descJobStatusFailed = prometheus.NewDesc(
		"kube_job_status_failed",
		"The number of pods which reached Phase Failed.",
		[]string{"namespace", "job"}, nil,
	)
	descJobStatusActive = prometheus.NewDesc(
		"kube_job_status_active",
		"The number of actively running pods.",
		[]string{"namespace", "job"}, nil,
	)
	descJobConditionComplete = prometheus.NewDesc(
		"kube_job_complete",
		"The job has completed its execution.",
		[]string{"namespace", "job", "condition"}, nil,
	)
	descJobConditionFailed = prometheus.NewDesc(
		"kube_job_failed",
		"The job has failed its execution.",
		[]string{"namespace", "job", "condition"}, nil,
	)
	descJobStatusStartTime = prometheus.NewDesc(
		"kube_job_status_start_time",
		"StartTime represents time when the job was acknowledged by the Job Manager.",
		[]string{"namespace", "job"}, nil,
	)
	descJobStatusCompletionTime = prometheus.NewDesc(
		"kube_job_status_completion_time",
		"CompletionTime represents time when the job was completed.",
		[]string{"namespace", "job"}, nil,
	)
)

type JobLister func() ([]v1batch.Job, error)

func (l JobLister) List() ([]v1batch.Job, error) {
	return l()
}

func RegisterJobCollector(registry prometheus.Registerer, kubeClient kubernetes.Interface, namespace string) {
	client := kubeClient.BatchV1().RESTClient()
	jlw := cache.NewListWatchFromClient(client, "jobs", namespace, nil)
	jinf := cache.NewSharedInformer(jlw, &v1batch.Job{}, resyncPeriod)

	jobLister := JobLister(func() (jobs []v1batch.Job, err error) {
		for _, c := range jinf.GetStore().List() {
			jobs = append(jobs, *(c.(*v1batch.Job)))
		}
		return jobs, nil
	})

	registry.MustRegister(&jobCollector{store: jobLister})
	go jinf.Run(context.Background().Done())
}

type jobStore interface {
	List() (jobs []v1batch.Job, err error)
}

// jobCollector collects metrics about all jobs in the cluster.
type jobCollector struct {
	store jobStore
}

// Describe implements the prometheus.Collector interface.
func (dc *jobCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- descJobInfo
	ch <- descJobCreated
	ch <- descJobSpecParallelism
	ch <- descJobSpecCompletions
	ch <- descJobSpecActiveDeadlineSeconds
	ch <- descJobStatusSucceeded
	ch <- descJobStatusFailed
	ch <- descJobStatusActive
	ch <- descJobConditionComplete
	ch <- descJobConditionFailed
	ch <- descJobStatusStartTime
	ch <- descJobStatusCompletionTime
}

// Collect implements the prometheus.Collector interface.
func (jc *jobCollector) Collect(ch chan<- prometheus.Metric) {
	jobs, err := jc.store.List()
	if err != nil {
		glog.Errorf("listing jobs failed: %s", err)
		return
	}
	for _, j := range jobs {
		jc.collectJob(ch, j)
	}
}

func (jc *jobCollector) collectJob(ch chan<- prometheus.Metric, j v1batch.Job) {
	addGauge := func(desc *prometheus.Desc, v float64, lv ...string) {
		lv = append([]string{j.Namespace, j.Name}, lv...)
		ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, v, lv...)
	}
	addCounter := func(desc *prometheus.Desc, v float64, lv ...string) {
		lv = append([]string{j.Namespace, j.Name}, lv...)
		ch <- prometheus.MustNewConstMetric(desc, prometheus.CounterValue, v, lv...)
	}

	addGauge(descJobInfo, 1)

	if j.Spec.Parallelism != nil {
		addGauge(descJobSpecParallelism, float64(*j.Spec.Parallelism))
	}

	if j.Spec.Completions != nil {
		addGauge(descJobSpecCompletions, float64(*j.Spec.Completions))
	}
	if !j.CreationTimestamp.IsZero() {
		addGauge(descJobCreated, float64(j.CreationTimestamp.Unix()))
	}

	if j.Spec.ActiveDeadlineSeconds != nil {
		addGauge(descJobSpecActiveDeadlineSeconds, float64(*j.Spec.ActiveDeadlineSeconds))
	}

	addGauge(descJobStatusSucceeded, float64(j.Status.Succeeded))
	addGauge(descJobStatusFailed, float64(j.Status.Failed))
	addGauge(descJobStatusActive, float64(j.Status.Active))

	addCounter(descJobStatusStartTime, float64(j.Status.StartTime.Unix()))

	if j.Status.CompletionTime != nil {
		addCounter(descJobStatusCompletionTime, float64(j.Status.CompletionTime.Unix()))
	}

	for _, c := range j.Status.Conditions {
		switch c.Type {
		case v1batch.JobComplete:
			addConditionMetrics(ch, descJobConditionComplete, c.Status, j.Namespace, j.Name)
		case v1batch.JobFailed:
			addConditionMetrics(ch, descJobConditionFailed, c.Status, j.Namespace, j.Name)
		}
	}
}
<commit_msg>Job start time can be nil<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package collectors

import (
	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
	"golang.org/x/net/context"
	"k8s.io/client-go/kubernetes"
	v1batch "k8s.io/client-go/pkg/apis/batch/v1"
	"k8s.io/client-go/tools/cache"
)

var (
	descJobInfo = prometheus.NewDesc(
		"kube_job_info",
		"Information about job.",
		[]string{"namespace", "job"}, nil,
	)
	descJobCreated = prometheus.NewDesc(
		"kube_job_created",
		"Unix creation timestamp",
		[]string{"namespace", "job"}, nil,
	)
	descJobSpecParallelism = prometheus.NewDesc(
		"kube_job_spec_parallelism",
		"The maximum desired number of pods the job should run at any given time.",
		[]string{"namespace", "job"}, nil,
	)
	descJobSpecCompletions = prometheus.NewDesc(
		"kube_job_spec_completions",
		"The desired number of successfully finished pods the job should be run with.",
		[]string{"namespace", "job"}, nil,
	)
	descJobSpecActiveDeadlineSeconds = prometheus.NewDesc(
		"kube_job_spec_active_deadline_seconds",
		"The duration in seconds relative to the startTime that the job may be active before the system tries to terminate it.",
		[]string{"namespace", "job"}, nil,
	)
	descJobStatusSucceeded = prometheus.NewDesc(
		"kube_job_status_succeeded",
		"The number of pods which reached Phase Succeeded.",
		[]string{"namespace", "job"}, nil,
	)
	descJobStatusFailed = prometheus.NewDesc(
		"kube_job_status_failed",
		"The number of pods which reached Phase Failed.",
		[]string{"namespace", "job"}, nil,
	)
	descJobStatusActive = prometheus.NewDesc(
		"kube_job_status_active",
		"The number of actively running pods.",
		[]string{"namespace", "job"}, nil,
	)
	descJobConditionComplete = prometheus.NewDesc(
		"kube_job_complete",
		"The job has completed its execution.",
		[]string{"namespace", "job", "condition"}, nil,
	)
	descJobConditionFailed = prometheus.NewDesc(
		"kube_job_failed",
		"The job has failed its execution.",
		[]string{"namespace", "job", "condition"}, nil,
	)
	descJobStatusStartTime = prometheus.NewDesc(
		"kube_job_status_start_time",
		"StartTime represents time when the job was acknowledged by the Job Manager.",
		[]string{"namespace", "job"}, nil,
	)
	descJobStatusCompletionTime = prometheus.NewDesc(
		"kube_job_status_completion_time",
		"CompletionTime represents time when the job was completed.",
		[]string{"namespace", "job"}, nil,
	)
)

type JobLister func() ([]v1batch.Job, error)

func (l JobLister) List() ([]v1batch.Job, error) {
	return l()
}

func RegisterJobCollector(registry prometheus.Registerer, kubeClient kubernetes.Interface, namespace string) {
	client := kubeClient.BatchV1().RESTClient()
	jlw := cache.NewListWatchFromClient(client, "jobs", namespace, nil)
	jinf := cache.NewSharedInformer(jlw, &v1batch.Job{}, resyncPeriod)

	jobLister := JobLister(func() (jobs []v1batch.Job, err error) {
		for _, c := range jinf.GetStore().List() {
			jobs = append(jobs, *(c.(*v1batch.Job)))
		}
		return jobs, nil
	})

	registry.MustRegister(&jobCollector{store: jobLister})
	go jinf.Run(context.Background().Done())
}

type jobStore interface {
	List() (jobs []v1batch.Job, err error)
}

// jobCollector collects metrics about all jobs in the cluster.
type jobCollector struct {
	store jobStore
}

// Describe implements the prometheus.Collector interface.
func (dc *jobCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- descJobInfo
	ch <- descJobCreated
	ch <- descJobSpecParallelism
	ch <- descJobSpecCompletions
	ch <- descJobSpecActiveDeadlineSeconds
	ch <- descJobStatusSucceeded
	ch <- descJobStatusFailed
	ch <- descJobStatusActive
	ch <- descJobConditionComplete
	ch <- descJobConditionFailed
	ch <- descJobStatusStartTime
	ch <- descJobStatusCompletionTime
}

// Collect implements the prometheus.Collector interface.
func (jc *jobCollector) Collect(ch chan<- prometheus.Metric) {
	jobs, err := jc.store.List()
	if err != nil {
		glog.Errorf("listing jobs failed: %s", err)
		return
	}
	for _, j := range jobs {
		jc.collectJob(ch, j)
	}
}

func (jc *jobCollector) collectJob(ch chan<- prometheus.Metric, j v1batch.Job) {
	addGauge := func(desc *prometheus.Desc, v float64, lv ...string) {
		lv = append([]string{j.Namespace, j.Name}, lv...)
		ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, v, lv...)
	}
	addCounter := func(desc *prometheus.Desc, v float64, lv ...string) {
		lv = append([]string{j.Namespace, j.Name}, lv...)
		ch <- prometheus.MustNewConstMetric(desc, prometheus.CounterValue, v, lv...)
	}

	addGauge(descJobInfo, 1)

	if j.Spec.Parallelism != nil {
		addGauge(descJobSpecParallelism, float64(*j.Spec.Parallelism))
	}

	if j.Spec.Completions != nil {
		addGauge(descJobSpecCompletions, float64(*j.Spec.Completions))
	}
	if !j.CreationTimestamp.IsZero() {
		addGauge(descJobCreated, float64(j.CreationTimestamp.Unix()))
	}

	if j.Spec.ActiveDeadlineSeconds != nil {
		addGauge(descJobSpecActiveDeadlineSeconds, float64(*j.Spec.ActiveDeadlineSeconds))
	}

	addGauge(descJobStatusSucceeded, float64(j.Status.Succeeded))
	addGauge(descJobStatusFailed, float64(j.Status.Failed))
	addGauge(descJobStatusActive, float64(j.Status.Active))

	if j.Status.StartTime != nil {
		addCounter(descJobStatusStartTime, float64(j.Status.StartTime.Unix()))
	}

	if j.Status.CompletionTime != nil {
		addCounter(descJobStatusCompletionTime, float64(j.Status.CompletionTime.Unix()))
	}

	for _, c := range j.Status.Conditions {
		switch c.Type {
		case v1batch.JobComplete:
			addConditionMetrics(ch, descJobConditionComplete, c.Status, j.Namespace, j.Name)
		case v1batch.JobFailed:
			addConditionMetrics(ch, descJobConditionFailed, c.Status, j.Namespace, j.Name)
		}
	}
}
<|endoftext|>


<commit_before>package command

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"time"
)

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
}

const imageSearchURL = "https://ajax.googleapis.com/ajax/services/search/images"

// https://developers.google.com/image-search/v1/jsondevguide
// imgsz=small|medium|large|xlarge restricts results to medium-sized images
const imageSize = "large"

var httpclient = http.Client{Timeout: 10 * time.Second}
var validImageFormats = []string{"png", "jpg"}

// searchImage retrives an image URL for given terms.
func searchImage(terms ...string) (string, error) {
	if len(terms) == 0 {
		return "", fmt.Errorf("no search term given")
	}

	keyword := strings.Join(terms, "+")

	u, _ := url.Parse(imageSearchURL)
	v := u.Query()
	v.Set("q", keyword)
	v.Set("v", "1.0")
	v.Set("imgsz", imageSize)
	u.RawQuery = v.Encode()

	resp, err := http.Get(u.String())
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	// datastructure of image search response
	var response struct {
		ResponseData struct {
			Results []struct {
				UnescapedURL string `json:"unescapedURL"`
			} `json:"results"`
		} `json:"responseData"`
	}

	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return "", err
	}

	results := response.ResponseData.Results
	if len(results) == 0 {
		return "", fmt.Errorf("no results for the given criteria: %v\n", keyword)
	}

	for _, v := range results {
		if validImage(v.UnescapedURL) {
			return v.UnescapedURL, nil
		}
	}

	return "", fmt.Errorf("no valid image format found for the given criteria: %v\n", keyword)
}

// randChoice randomly choice an element from given elems.
func randChoice(elems []string) string {
	return elems[rand.Intn(len(elems))]
}

// validImage reports whether the given url string has a valid image format.
func validImage(s string) bool {
	// extract extension
	i := strings.LastIndex(s, ".")
	if i < 0 {
		return false
	}
	ext := strings.ToLower(s[i+1:])
	for _, format := range validImageFormats {
		if ext == format {
			return true
		}
	}
	return false
}
<commit_msg>Google image search api is deprecated and has reached EOL as of today.<commit_after>package command

import (
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"strings"
	"time"

	customsearch "google.golang.org/api/customsearch/v1"
	"google.golang.org/api/googleapi/transport"
)

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
}

var (
	imageApiKey       = os.Getenv("ILBER_IMAGESEARCH_APIKEY")
	searchEngineID    = os.Getenv("ILBER_SEARCHENGINE_ID")
	imageclient       = &http.Client{Transport: &transport.APIKey{Key: imageApiKey}}
	validImageFormats = []string{"png", "jpg"}

	httpclient = &http.Client{Timeout: 10 * time.Second}
)

// searchImage retrives an image URL for given terms.
func searchImage(terms ...string) (string, error) {
	if len(terms) == 0 {
		return "", fmt.Errorf("no search term given")
	}

	keyword := strings.Join(terms, "+")

	service, err := customsearch.New(imageclient)
	if err != nil {
		return "", fmt.Errorf("Error creating customsearch client: %v", err)
	}
	cse := customsearch.NewCseService(service)
	call := cse.List(keyword).Cx(searchEngineID).SearchType("image").Num(1)
	resp, err := call.Do()
	if err != nil {
		return "", fmt.Errorf("Error making image search API call: %v", err)
	}
	if len(resp.Items) == 0 {
		return "", fmt.Errorf("Could not find any image based on the given criteria: %v", keyword)
	}

	imageurl := resp.Items[0].Link
	return imageurl, nil
}

// randChoice randomly choice an element from given elems.
func randChoice(elems []string) string {
	return elems[rand.Intn(len(elems))]
}
<|endoftext|>


<commit_before>package command

import (
	"encoding/json"
	"fmt"
	"os"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/coreos/go-etcd/etcd"
)

type handlerFunc func(*cli.Context, *etcd.Client) (*etcd.Response, error)
type printFunc func(*etcd.Response, string)

// dumpCURL blindly dumps all curl output to os.Stderr
func dumpCURL(client *etcd.Client) {
	client.OpenCURL()
	for {
		fmt.Fprintf(os.Stderr, "Curl-Example: %s\n", client.RecvCURL())
	}
}

// rawhandle wraps the command function handlers and sets up the
// environment but performs no output formatting.
func rawhandle(c *cli.Context, fn handlerFunc) (*etcd.Response, error) {
	peers := c.GlobalString("C")
	client := etcd.NewClient(trimsplit(peers, ","))

	if c.GlobalBool("debug") {
		go dumpCURL(client)
	}

	// Sync cluster.
	ok := client.SyncCluster()
	if c.GlobalBool("debug") {
		fmt.Fprintf(os.Stderr, "Cluster-Peers: %s\n",
			strings.Join(client.GetCluster(), " "))
	}

	if !ok {
		fmt.Println("Cannot sync with the cluster")
		os.Exit(FailedToConnectToHost)
	}

	// Execute handler function.
	return fn(c, client)
}

// handlePrint wraps the command function handlers to parse global flags
// into a client and to properly format the response objects.
func handlePrint(c *cli.Context, fn handlerFunc, pFn printFunc) {
	resp, err := rawhandle(c, fn)

	// Print error and exit, if necessary.
	if err != nil {
		fmt.Println("Error:", err)
		os.Exit(ErrorFromEtcd)
	}

	if resp != nil {
		pFn(resp, c.GlobalString("output"))
	}
}

// handleKey handles a request that wants to do operations on a single key.
func handleKey(c *cli.Context, fn handlerFunc) {
	handlePrint(c, fn, printKey)
}

// printKey writes the etcd response to STDOUT in the given format.
func printKey(resp *etcd.Response, format string) {
	// printKey is only for keys, error on directories
	if resp.Node.Dir == true {
		fmt.Fprintln(os.Stderr, fmt.Sprintf("%s: Is a directory", resp.Node.Key))
		os.Exit(1)
	}

	// Format the result.
	switch format {
	case "simple":
		fmt.Println(resp.Node.Value)
	case "extended":
		// Extended prints in a rfc2822 style format
		fmt.Println("Key:", resp.Node.Key)
		fmt.Println("Modified-Index:", resp.Node.ModifiedIndex)
		fmt.Println("TTL:", resp.Node.TTL)
		fmt.Println("")
		fmt.Println(resp.Node.Value)
	case "json":
		b, err := json.Marshal(resp)
		if err != nil {
			panic(err)
		}
		fmt.Println(string(b))
	default:
		fmt.Fprintln(os.Stderr, "Unsupported output format:", format)
	}
}
<commit_msg>chore(command): use the long form to lookup peers arg<commit_after>package command

import (
	"encoding/json"
	"fmt"
	"os"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/coreos/go-etcd/etcd"
)

type handlerFunc func(*cli.Context, *etcd.Client) (*etcd.Response, error)
type printFunc func(*etcd.Response, string)

// dumpCURL blindly dumps all curl output to os.Stderr
func dumpCURL(client *etcd.Client) {
	client.OpenCURL()
	for {
		fmt.Fprintf(os.Stderr, "Curl-Example: %s\n", client.RecvCURL())
	}
}

// rawhandle wraps the command function handlers and sets up the
// environment but performs no output formatting.
func rawhandle(c *cli.Context, fn handlerFunc) (*etcd.Response, error) {
	peers := c.GlobalString("peers")
	client := etcd.NewClient(trimsplit(peers, ","))

	if c.GlobalBool("debug") {
		go dumpCURL(client)
	}

	// Sync cluster.
	ok := client.SyncCluster()
	if c.GlobalBool("debug") {
		fmt.Fprintf(os.Stderr, "Cluster-Peers: %s\n",
			strings.Join(client.GetCluster(), " "))
	}

	if !ok {
		fmt.Println("Cannot sync with the cluster")
		os.Exit(FailedToConnectToHost)
	}

	// Execute handler function.
	return fn(c, client)
}

// handlePrint wraps the command function handlers to parse global flags
// into a client and to properly format the response objects.
func handlePrint(c *cli.Context, fn handlerFunc, pFn printFunc) {
	resp, err := rawhandle(c, fn)

	// Print error and exit, if necessary.
	if err != nil {
		fmt.Println("Error:", err)
		os.Exit(ErrorFromEtcd)
	}

	if resp != nil {
		pFn(resp, c.GlobalString("output"))
	}
}

// handleKey handles a request that wants to do operations on a single key.
func handleKey(c *cli.Context, fn handlerFunc) {
	handlePrint(c, fn, printKey)
}

// printKey writes the etcd response to STDOUT in the given format.
func printKey(resp *etcd.Response, format string) {
	// printKey is only for keys, error on directories
	if resp.Node.Dir == true {
		fmt.Fprintln(os.Stderr, fmt.Sprintf("%s: Is a directory", resp.Node.Key))
		os.Exit(1)
	}

	// Format the result.
	switch format {
	case "simple":
		fmt.Println(resp.Node.Value)
	case "extended":
		// Extended prints in a rfc2822 style format
		fmt.Println("Key:", resp.Node.Key)
		fmt.Println("Modified-Index:", resp.Node.ModifiedIndex)
		fmt.Println("TTL:", resp.Node.TTL)
		fmt.Println("")
		fmt.Println(resp.Node.Value)
	case "json":
		b, err := json.Marshal(resp)
		if err != nil {
			panic(err)
		}
		fmt.Println(string(b))
	default:
		fmt.Fprintln(os.Stderr, "Unsupported output format:", format)
	}
}
<|endoftext|>