<commit_before>package core

import (
	"errors"
	"fmt"
	"io"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58"
	ctxgroup "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup"
	datastore "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"

	bstore "github.com/jbenet/go-ipfs/blocks/blockstore"
	bserv "github.com/jbenet/go-ipfs/blockservice"
	config "github.com/jbenet/go-ipfs/config"
	diag "github.com/jbenet/go-ipfs/diagnostics"
	exchange "github.com/jbenet/go-ipfs/exchange"
	bitswap "github.com/jbenet/go-ipfs/exchange/bitswap"
	bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network"
	"github.com/jbenet/go-ipfs/exchange/offline"
	mount "github.com/jbenet/go-ipfs/fuse/mount"
	importer "github.com/jbenet/go-ipfs/importer"
	chunk "github.com/jbenet/go-ipfs/importer/chunk"
	merkledag "github.com/jbenet/go-ipfs/merkledag"
	namesys "github.com/jbenet/go-ipfs/namesys"
	ic "github.com/jbenet/go-ipfs/p2p/crypto"
	p2phost "github.com/jbenet/go-ipfs/p2p/host"
	p2pbhost "github.com/jbenet/go-ipfs/p2p/host/basic"
	swarm "github.com/jbenet/go-ipfs/p2p/net/swarm"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	path "github.com/jbenet/go-ipfs/path"
	pin "github.com/jbenet/go-ipfs/pin"
	routing "github.com/jbenet/go-ipfs/routing"
	dht "github.com/jbenet/go-ipfs/routing/dht"
	uio "github.com/jbenet/go-ipfs/unixfs/io"
	u "github.com/jbenet/go-ipfs/util"
	ds2 "github.com/jbenet/go-ipfs/util/datastore2"
	debugerror "github.com/jbenet/go-ipfs/util/debugerror"
	eventlog "github.com/jbenet/go-ipfs/util/eventlog"
	lgbl "github.com/jbenet/go-ipfs/util/eventlog/loggables"
)

const IpnsValidatorTag = "ipns"
const kSizeBlockstoreWriteCache = 100

var log = eventlog.Logger("core")

// IpfsNode is IPFS Core module. It represents an IPFS instance.
type IpfsNode struct {

	// Self
	Config     *config.Config // the node's configuration
	Identity   peer.ID        // the local node's identity
	PrivateKey ic.PrivKey     // the local node's private Key
	onlineMode bool           // alternatively, offline

	// Local node
	Datastore ds2.ThreadSafeDatastoreCloser // the local datastore
	Pinning   pin.Pinner                    // the pinning manager
	Mounts    Mounts                        // current mount state, if any.

	// Services
	Peerstore   peer.Peerstore       // storage for other Peer instances
	PeerHost    p2phost.Host         // the network host (server+client)
	Routing     routing.IpfsRouting  // the routing system. recommend ipfs-dht
	Exchange    exchange.Interface   // the block exchange + strategy (bitswap)
	Blockstore  bstore.Blockstore    // the block store (lower level)
	Blocks      *bserv.BlockService  // the block service, get/add blocks.
	DAG         merkledag.DAGService // the merkle dag service, get/add objects.
	Resolver    *path.Resolver       // the path resolution system
	Namesys     namesys.NameSystem   // the name system, resolves paths to hashes
	Diagnostics *diag.Diagnostics    // the diagnostics service

	ctxgroup.ContextGroup

	// dht allows node to Bootstrap when dht is present
	// TODO privatize before merging. This is here temporarily during the
	// migration of the TestNet constructor
	DHT *dht.IpfsDHT
}

// Mounts defines what the node's mount state is. This should
// perhaps be moved to the daemon or mount. It's here because
// it needs to be accessible across daemon requests.
type Mounts struct {
	Ipfs mount.Mount
	Ipns mount.Mount
}

var errTODO = errors.New("TODO")

type ConfigOption func(ctx context.Context) (*IpfsNode, error)

func NewIPFSNode(ctx context.Context, option ConfigOption) (*IpfsNode, error) {
	node, err := option(ctx)
	if err != nil {
		return nil, err
	}

	// Need to make sure it's perfectly clear 1) which variables are expected
	// to be initialized at this point, and 2) which variables will be
	// initialized after this point.

	node.Blocks, err = bserv.New(node.Blockstore, node.Exchange)
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	if node.Peerstore == nil {
		node.Peerstore = peer.NewPeerstore()
	}
	node.DAG = merkledag.NewDAGService(node.Blocks)
	node.Pinning, err = pin.LoadPinner(node.Datastore, node.DAG)
	if err != nil {
		node.Pinning = pin.NewPinner(node.Datastore, node.DAG)
	}
	node.Resolver = &path.Resolver{DAG: node.DAG}
	return node, nil
}

func Offline(cfg *config.Config) ConfigOption {
	return Standard(cfg, false)
}

func Online(cfg *config.Config) ConfigOption {
	return Standard(cfg, true)
}

// DEPRECATED: use Online, Offline functions
func Standard(cfg *config.Config, online bool) ConfigOption {
	return func(ctx context.Context) (n *IpfsNode, err error) {

		success := false // flip to true after all sub-system inits succeed
		defer func() {
			if !success && n != nil {
				n.Close()
			}
		}()

		if cfg == nil {
			return nil, debugerror.Errorf("configuration required")
		}
		n = &IpfsNode{
			onlineMode: online,
			Config:     cfg,
		}

		n.ContextGroup = ctxgroup.WithContextAndTeardown(ctx, n.teardown)
		ctx = n.ContextGroup.Context()

		// setup Peerstore
		n.Peerstore = peer.NewPeerstore()

		// setup datastore.
		if n.Datastore, err = makeDatastore(cfg.Datastore); err != nil {
			return nil, debugerror.Wrap(err)
		}

		// setup local peer ID (private key is loaded in online setup)
		if err := n.loadID(); err != nil {
			return nil, err
		}

		n.Blockstore, err = bstore.WriteCached(bstore.NewBlockstore(n.Datastore), kSizeBlockstoreWriteCache)
		if err != nil {
			return nil, debugerror.Wrap(err)
		}

		// setup online services
		if online {
			if err := n.StartOnlineServices(); err != nil {
				return nil, err // debugerror.Wraps.
			}
		} else {
			n.Exchange = offline.Exchange(n.Blockstore)
		}

		success = true
		return n, nil
	}
}

func (n *IpfsNode) StartOnlineServices() error {
	ctx := n.Context()

	if n.PeerHost != nil { // already online.
		return debugerror.New("node already online")
	}

	// load private key
	if err := n.loadPrivateKey(); err != nil {
		return err
	}

	peerhost, err := constructPeerHost(ctx, n.ContextGroup, n.Config, n.Identity, n.Peerstore)
	if err != nil {
		return debugerror.Wrap(err)
	}
	n.PeerHost = peerhost

	// setup diagnostics service
	n.Diagnostics = diag.NewDiagnostics(n.Identity, n.PeerHost)

	// setup routing service
	dhtRouting, err := constructDHTRouting(ctx, n.ContextGroup, n.PeerHost, n.Datastore)
	if err != nil {
		return debugerror.Wrap(err)
	}
	n.DHT = dhtRouting
	n.Routing = dhtRouting

	// setup exchange service
	const alwaysSendToPeer = true // use YesManStrategy
	bitswapNetwork := bsnet.NewFromIpfsHost(n.PeerHost, n.Routing)
	n.Exchange = bitswap.New(ctx, n.Identity, bitswapNetwork, n.Blockstore, alwaysSendToPeer)

	// setup name system
	// TODO implement an offline namesys that serves only local names.
	n.Namesys = namesys.NewNameSystem(n.Routing)

	// TODO consider moving connection supervision into the Network. We've
	// discussed improvements to this Node constructor. One improvement
	// would be to make the node configurable, allowing clients to inject
	// an Exchange, Network, or Routing component and have the constructor
	// manage the wiring. In that scenario, this dangling function is a bit
	// awkward.
	var bootstrapPeers []peer.PeerInfo
	for _, bootstrap := range n.Config.Bootstrap {
		p, err := toPeer(bootstrap)
		if err != nil {
			log.Event(ctx, "bootstrapError", n.Identity, lgbl.Error(err))
			log.Errorf("%s bootstrap error: %s", n.Identity, err)
			return err
		}
		bootstrapPeers = append(bootstrapPeers, p)
	}
	go superviseConnections(ctx, n.PeerHost, n.DHT, n.Peerstore, bootstrapPeers)
	return nil
}

func (n *IpfsNode) teardown() error {
	if err := n.Datastore.Close(); err != nil {
		return err
	}
	return nil
}

func (n *IpfsNode) OnlineMode() bool {
	return n.onlineMode
}

func (n *IpfsNode) Bootstrap(ctx context.Context, peers []peer.PeerInfo) error {
	if n.DHT != nil {
		for _, p := range peers {
			// TODO bootstrap(ctx, n.PeerHost, n.DHT, n.Peerstore, peers)
			if err := n.DHT.Connect(ctx, p.ID); err != nil {
				return err
			}
		}
	}
	return nil
}

// TODO we may not want to add these methods to the core. Maybe they should be
// defined as free functions in another package that use public fields on the
// node.
//
// e.g. reader, err := unix.Cat(node)

func (n *IpfsNode) Cat(k u.Key) (io.Reader, error) {
	catterdag := n.DAG
	nodeCatted, err := (&path.Resolver{catterdag}).ResolvePath(k.String())
	if err != nil {
		return nil, err
	}
	return uio.NewDagReader(nodeCatted, catterdag)
}

func (n *IpfsNode) Add(r io.Reader) (u.Key, error) {
	nodeAdded, err := importer.BuildDagFromReader(
		r,
		n.DAG,
		nil,
		chunk.DefaultSplitter,
	)
	if err != nil {
		return "", err
	}
	return nodeAdded.Key()
}

func (n *IpfsNode) loadID() error {
	if n.Identity != "" {
		return debugerror.New("identity already loaded")
	}

	cid := n.Config.Identity.PeerID
	if cid == "" {
		return debugerror.New("Identity was not set in config (was ipfs init run?)")
	}
	if len(cid) == 0 {
		return debugerror.New("No peer ID in config! (was ipfs init run?)")
	}

	n.Identity = peer.ID(b58.Decode(cid))
	return nil
}

func (n *IpfsNode) loadPrivateKey() error {
	if n.Identity == "" || n.Peerstore == nil {
		return debugerror.New("loaded private key out of order.")
	}

	if n.PrivateKey != nil {
		return debugerror.New("private key already loaded")
	}

	sk, err := loadPrivateKey(&n.Config.Identity, n.Identity)
	if err != nil {
		return err
	}

	n.PrivateKey = sk
	n.Peerstore.AddPrivKey(n.Identity, n.PrivateKey)
	return nil
}

func loadPrivateKey(cfg *config.Identity, id peer.ID) (ic.PrivKey, error) {
	sk, err := cfg.DecodePrivateKey("passphrase todo!")
	if err != nil {
		return nil, err
	}

	id2, err := peer.IDFromPrivateKey(sk)
	if err != nil {
		return nil, err
	}

	if id2 != id {
		return nil, fmt.Errorf("private key in config does not match id: %s != %s", id, id2)
	}

	return sk, nil
}

func listenAddresses(cfg *config.Config) ([]ma.Multiaddr, error) {

	var err error
	listen := make([]ma.Multiaddr, len(cfg.Addresses.Swarm))
	for i, addr := range cfg.Addresses.Swarm {

		listen[i], err = ma.NewMultiaddr(addr)
		if err != nil {
			return nil, fmt.Errorf("Failure to parse config.Addresses.Swarm[%d]: %s", i, cfg.Addresses.Swarm)
		}
	}

	return listen, nil
}

// isolates the complex initialization steps
func constructPeerHost(ctx context.Context, ctxg ctxgroup.ContextGroup, cfg *config.Config, id peer.ID, ps peer.Peerstore) (p2phost.Host, error) {
	listenAddrs, err := listenAddresses(cfg)
	// make sure we dont error out if our config includes some addresses we cant use.
	filteredAddrs := swarm.FilterAddrs(listenAddrs)
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	network, err := swarm.NewNetwork(ctx, filteredAddrs, id, ps)
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	ctxg.AddChildGroup(network.CtxGroup())

	peerhost := p2pbhost.New(network)
	// explicitly set these as our listen addrs.
	// (why not do it inside inet.NewNetwork? because this way we can
	// listen on addresses without necessarily advertising those publicly.)
	addrs, err := peerhost.Network().InterfaceListenAddresses()
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	ps.AddAddresses(id, addrs)
	return peerhost, nil
}

func constructDHTRouting(ctx context.Context, ctxg ctxgroup.ContextGroup, host p2phost.Host, ds datastore.ThreadSafeDatastore) (*dht.IpfsDHT, error) {
	dhtRouting := dht.NewDHT(ctx, host, ds)
	dhtRouting.Validators[IpnsValidatorTag] = namesys.ValidateIpnsRecord
	ctxg.AddChildGroup(dhtRouting)
	return dhtRouting, nil
}
<commit_msg>refactor(core): distinguish Online services<commit_after>package core

import (
	"errors"
	"fmt"
	"io"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58"
	ctxgroup "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup"
	datastore "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"

	bstore "github.com/jbenet/go-ipfs/blocks/blockstore"
	bserv "github.com/jbenet/go-ipfs/blockservice"
	config "github.com/jbenet/go-ipfs/config"
	diag "github.com/jbenet/go-ipfs/diagnostics"
	exchange "github.com/jbenet/go-ipfs/exchange"
	bitswap "github.com/jbenet/go-ipfs/exchange/bitswap"
	bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network"
	"github.com/jbenet/go-ipfs/exchange/offline"
	mount "github.com/jbenet/go-ipfs/fuse/mount"
	importer "github.com/jbenet/go-ipfs/importer"
	chunk "github.com/jbenet/go-ipfs/importer/chunk"
	merkledag "github.com/jbenet/go-ipfs/merkledag"
	namesys "github.com/jbenet/go-ipfs/namesys"
	ic "github.com/jbenet/go-ipfs/p2p/crypto"
	p2phost "github.com/jbenet/go-ipfs/p2p/host"
	p2pbhost "github.com/jbenet/go-ipfs/p2p/host/basic"
	swarm "github.com/jbenet/go-ipfs/p2p/net/swarm"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	path "github.com/jbenet/go-ipfs/path"
	pin "github.com/jbenet/go-ipfs/pin"
	routing "github.com/jbenet/go-ipfs/routing"
	dht "github.com/jbenet/go-ipfs/routing/dht"
	uio "github.com/jbenet/go-ipfs/unixfs/io"
	u "github.com/jbenet/go-ipfs/util"
	ds2 "github.com/jbenet/go-ipfs/util/datastore2"
	debugerror "github.com/jbenet/go-ipfs/util/debugerror"
	eventlog "github.com/jbenet/go-ipfs/util/eventlog"
	lgbl "github.com/jbenet/go-ipfs/util/eventlog/loggables"
)

const IpnsValidatorTag = "ipns"
const kSizeBlockstoreWriteCache = 100

var log = eventlog.Logger("core")

// IpfsNode is IPFS Core module. It represents an IPFS instance.
type IpfsNode struct {

	// Self
	Config     *config.Config // the node's configuration
	Identity   peer.ID        // the local node's identity
	onlineMode bool           // alternatively, offline

	// Local node
	Datastore ds2.ThreadSafeDatastoreCloser // the local datastore
	Pinning   pin.Pinner                    // the pinning manager
	Mounts    Mounts                        // current mount state, if any.

	// Services
	Peerstore  peer.Peerstore       // storage for other Peer instances
	Blockstore bstore.Blockstore    // the block store (lower level)
	Blocks     *bserv.BlockService  // the block service, get/add blocks.
	DAG        merkledag.DAGService // the merkle dag service, get/add objects.
	Resolver   *path.Resolver       // the path resolution system

	// Online
	PrivateKey  ic.PrivKey          // the local node's private Key
	PeerHost    p2phost.Host        // the network host (server+client)
	Routing     routing.IpfsRouting // the routing system. recommend ipfs-dht
	Exchange    exchange.Interface  // the block exchange + strategy (bitswap)
	Namesys     namesys.NameSystem  // the name system, resolves paths to hashes
	Diagnostics *diag.Diagnostics   // the diagnostics service

	ctxgroup.ContextGroup

	// dht allows node to Bootstrap when dht is present
	// TODO privatize before merging. This is here temporarily during the
	// migration of the TestNet constructor
	DHT *dht.IpfsDHT
}

// Mounts defines what the node's mount state is. This should
// perhaps be moved to the daemon or mount. It's here because
// it needs to be accessible across daemon requests.
type Mounts struct {
	Ipfs mount.Mount
	Ipns mount.Mount
}

var errTODO = errors.New("TODO")

type ConfigOption func(ctx context.Context) (*IpfsNode, error)

func NewIPFSNode(ctx context.Context, option ConfigOption) (*IpfsNode, error) {
	node, err := option(ctx)
	if err != nil {
		return nil, err
	}

	// Need to make sure it's perfectly clear 1) which variables are expected
	// to be initialized at this point, and 2) which variables will be
	// initialized after this point.

	node.Blocks, err = bserv.New(node.Blockstore, node.Exchange)
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	if node.Peerstore == nil {
		node.Peerstore = peer.NewPeerstore()
	}
	node.DAG = merkledag.NewDAGService(node.Blocks)
	node.Pinning, err = pin.LoadPinner(node.Datastore, node.DAG)
	if err != nil {
		node.Pinning = pin.NewPinner(node.Datastore, node.DAG)
	}
	node.Resolver = &path.Resolver{DAG: node.DAG}
	return node, nil
}

func Offline(cfg *config.Config) ConfigOption {
	return Standard(cfg, false)
}

func Online(cfg *config.Config) ConfigOption {
	return Standard(cfg, true)
}

// DEPRECATED: use Online, Offline functions
func Standard(cfg *config.Config, online bool) ConfigOption {
	return func(ctx context.Context) (n *IpfsNode, err error) {

		success := false // flip to true after all sub-system inits succeed
		defer func() {
			if !success && n != nil {
				n.Close()
			}
		}()

		if cfg == nil {
			return nil, debugerror.Errorf("configuration required")
		}
		n = &IpfsNode{
			onlineMode: online,
			Config:     cfg,
		}

		n.ContextGroup = ctxgroup.WithContextAndTeardown(ctx, n.teardown)
		ctx = n.ContextGroup.Context()

		// setup Peerstore
		n.Peerstore = peer.NewPeerstore()

		// setup datastore.
		if n.Datastore, err = makeDatastore(cfg.Datastore); err != nil {
			return nil, debugerror.Wrap(err)
		}

		// setup local peer ID (private key is loaded in online setup)
		if err := n.loadID(); err != nil {
			return nil, err
		}

		n.Blockstore, err = bstore.WriteCached(bstore.NewBlockstore(n.Datastore), kSizeBlockstoreWriteCache)
		if err != nil {
			return nil, debugerror.Wrap(err)
		}

		// setup online services
		if online {
			if err := n.StartOnlineServices(); err != nil {
				return nil, err // debugerror.Wraps.
			}
		} else {
			n.Exchange = offline.Exchange(n.Blockstore)
		}

		success = true
		return n, nil
	}
}

func (n *IpfsNode) StartOnlineServices() error {
	ctx := n.Context()

	if n.PeerHost != nil { // already online.
		return debugerror.New("node already online")
	}

	// load private key
	if err := n.loadPrivateKey(); err != nil {
		return err
	}

	peerhost, err := constructPeerHost(ctx, n.ContextGroup, n.Config, n.Identity, n.Peerstore)
	if err != nil {
		return debugerror.Wrap(err)
	}
	n.PeerHost = peerhost

	// setup diagnostics service
	n.Diagnostics = diag.NewDiagnostics(n.Identity, n.PeerHost)

	// setup routing service
	dhtRouting, err := constructDHTRouting(ctx, n.ContextGroup, n.PeerHost, n.Datastore)
	if err != nil {
		return debugerror.Wrap(err)
	}
	n.DHT = dhtRouting
	n.Routing = dhtRouting

	// setup exchange service
	const alwaysSendToPeer = true // use YesManStrategy
	bitswapNetwork := bsnet.NewFromIpfsHost(n.PeerHost, n.Routing)
	n.Exchange = bitswap.New(ctx, n.Identity, bitswapNetwork, n.Blockstore, alwaysSendToPeer)

	// setup name system
	// TODO implement an offline namesys that serves only local names.
	n.Namesys = namesys.NewNameSystem(n.Routing)

	// TODO consider moving connection supervision into the Network. We've
	// discussed improvements to this Node constructor. One improvement
	// would be to make the node configurable, allowing clients to inject
	// an Exchange, Network, or Routing component and have the constructor
	// manage the wiring. In that scenario, this dangling function is a bit
	// awkward.
	var bootstrapPeers []peer.PeerInfo
	for _, bootstrap := range n.Config.Bootstrap {
		p, err := toPeer(bootstrap)
		if err != nil {
			log.Event(ctx, "bootstrapError", n.Identity, lgbl.Error(err))
			log.Errorf("%s bootstrap error: %s", n.Identity, err)
			return err
		}
		bootstrapPeers = append(bootstrapPeers, p)
	}
	go superviseConnections(ctx, n.PeerHost, n.DHT, n.Peerstore, bootstrapPeers)
	return nil
}

func (n *IpfsNode) teardown() error {
	if err := n.Datastore.Close(); err != nil {
		return err
	}
	return nil
}

func (n *IpfsNode) OnlineMode() bool {
	return n.onlineMode
}

func (n *IpfsNode) Bootstrap(ctx context.Context, peers []peer.PeerInfo) error {
	if n.DHT != nil {
		for _, p := range peers {
			// TODO bootstrap(ctx, n.PeerHost, n.DHT, n.Peerstore, peers)
			if err := n.DHT.Connect(ctx, p.ID); err != nil {
				return err
			}
		}
	}
	return nil
}

// TODO we may not want to add these methods to the core. Maybe they should be
// defined as free functions in another package that use public fields on the
// node.
//
// e.g. reader, err := unix.Cat(node)

func (n *IpfsNode) Cat(k u.Key) (io.Reader, error) {
	catterdag := n.DAG
	nodeCatted, err := (&path.Resolver{catterdag}).ResolvePath(k.String())
	if err != nil {
		return nil, err
	}
	return uio.NewDagReader(nodeCatted, catterdag)
}

func (n *IpfsNode) Add(r io.Reader) (u.Key, error) {
	nodeAdded, err := importer.BuildDagFromReader(
		r,
		n.DAG,
		nil,
		chunk.DefaultSplitter,
	)
	if err != nil {
		return "", err
	}
	return nodeAdded.Key()
}

func (n *IpfsNode) loadID() error {
	if n.Identity != "" {
		return debugerror.New("identity already loaded")
	}

	cid := n.Config.Identity.PeerID
	if cid == "" {
		return debugerror.New("Identity was not set in config (was ipfs init run?)")
	}
	if len(cid) == 0 {
		return debugerror.New("No peer ID in config! (was ipfs init run?)")
	}

	n.Identity = peer.ID(b58.Decode(cid))
	return nil
}

func (n *IpfsNode) loadPrivateKey() error {
	if n.Identity == "" || n.Peerstore == nil {
		return debugerror.New("loaded private key out of order.")
	}

	if n.PrivateKey != nil {
		return debugerror.New("private key already loaded")
	}

	sk, err := loadPrivateKey(&n.Config.Identity, n.Identity)
	if err != nil {
		return err
	}

	n.PrivateKey = sk
	n.Peerstore.AddPrivKey(n.Identity, n.PrivateKey)
	return nil
}

func loadPrivateKey(cfg *config.Identity, id peer.ID) (ic.PrivKey, error) {
	sk, err := cfg.DecodePrivateKey("passphrase todo!")
	if err != nil {
		return nil, err
	}

	id2, err := peer.IDFromPrivateKey(sk)
	if err != nil {
		return nil, err
	}

	if id2 != id {
		return nil, fmt.Errorf("private key in config does not match id: %s != %s", id, id2)
	}

	return sk, nil
}

func listenAddresses(cfg *config.Config) ([]ma.Multiaddr, error) {

	var err error
	listen := make([]ma.Multiaddr, len(cfg.Addresses.Swarm))
	for i, addr := range cfg.Addresses.Swarm {

		listen[i], err = ma.NewMultiaddr(addr)
		if err != nil {
			return nil, fmt.Errorf("Failure to parse config.Addresses.Swarm[%d]: %s", i, cfg.Addresses.Swarm)
		}
	}

	return listen, nil
}

// isolates the complex initialization steps
func constructPeerHost(ctx context.Context, ctxg ctxgroup.ContextGroup, cfg *config.Config, id peer.ID, ps peer.Peerstore) (p2phost.Host, error) {
	listenAddrs, err := listenAddresses(cfg)
	// make sure we dont error out if our config includes some addresses we cant use.
	filteredAddrs := swarm.FilterAddrs(listenAddrs)
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	network, err := swarm.NewNetwork(ctx, filteredAddrs, id, ps)
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	ctxg.AddChildGroup(network.CtxGroup())

	peerhost := p2pbhost.New(network)
	// explicitly set these as our listen addrs.
	// (why not do it inside inet.NewNetwork? because this way we can
	// listen on addresses without necessarily advertising those publicly.)
	addrs, err := peerhost.Network().InterfaceListenAddresses()
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	ps.AddAddresses(id, addrs)
	return peerhost, nil
}

func constructDHTRouting(ctx context.Context, ctxg ctxgroup.ContextGroup, host p2phost.Host, ds datastore.ThreadSafeDatastore) (*dht.IpfsDHT, error) {
	dhtRouting := dht.NewDHT(ctx, host, ds)
	dhtRouting.Validators[IpnsValidatorTag] = namesys.ValidateIpnsRecord
	ctxg.AddChildGroup(dhtRouting)
	return dhtRouting, nil
}
<|endoftext|>


<commit_before>package core

import (
	"fmt"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58"
	ctxgroup "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup"
	datastore "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"

	bstore "github.com/jbenet/go-ipfs/blocks/blockstore"
	bserv "github.com/jbenet/go-ipfs/blockservice"
	config "github.com/jbenet/go-ipfs/config"
	diag "github.com/jbenet/go-ipfs/diagnostics"
	exchange "github.com/jbenet/go-ipfs/exchange"
	bitswap "github.com/jbenet/go-ipfs/exchange/bitswap"
	bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network"
	offline "github.com/jbenet/go-ipfs/exchange/offline"
	mount "github.com/jbenet/go-ipfs/fuse/mount"
	merkledag "github.com/jbenet/go-ipfs/merkledag"
	namesys "github.com/jbenet/go-ipfs/namesys"
	ic "github.com/jbenet/go-ipfs/p2p/crypto"
	p2phost "github.com/jbenet/go-ipfs/p2p/host"
	p2pbhost "github.com/jbenet/go-ipfs/p2p/host/basic"
	swarm "github.com/jbenet/go-ipfs/p2p/net/swarm"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	path "github.com/jbenet/go-ipfs/path"
	pin "github.com/jbenet/go-ipfs/pin"
	routing "github.com/jbenet/go-ipfs/routing"
	dht "github.com/jbenet/go-ipfs/routing/dht"
	util "github.com/jbenet/go-ipfs/util"
	ds2 "github.com/jbenet/go-ipfs/util/datastore2"
	debugerror "github.com/jbenet/go-ipfs/util/debugerror"
	eventlog "github.com/jbenet/go-ipfs/util/eventlog"
	lgbl "github.com/jbenet/go-ipfs/util/eventlog/loggables"
)

const IpnsValidatorTag = "ipns"
const kSizeBlockstoreWriteCache = 100

var log = eventlog.Logger("core")

type mode int

const (
	// zero value is not a valid mode, must be explicitly set
	invalidMode mode = iota
	offlineMode
	onlineMode
)

// IpfsNode is IPFS Core module. It represents an IPFS instance.
type IpfsNode struct {

	// Self
	Identity peer.ID // the local node's identity

	// TODO abstract as repo.Repo
	Config    *config.Config                // the node's configuration
	Datastore ds2.ThreadSafeDatastoreCloser // the local datastore

	// Local node
	Pinning pin.Pinner // the pinning manager
	Mounts  Mounts     // current mount state, if any.

	// Services
	Peerstore  peer.Peerstore       // storage for other Peer instances
	Blockstore bstore.Blockstore    // the block store (lower level)
	Blocks     *bserv.BlockService  // the block service, get/add blocks.
	DAG        merkledag.DAGService // the merkle dag service, get/add objects.
	Resolver   *path.Resolver       // the path resolution system

	// Online
	PrivateKey  ic.PrivKey          // the local node's private Key
	PeerHost    p2phost.Host        // the network host (server+client)
	Routing     routing.IpfsRouting // the routing system. recommend ipfs-dht
	Exchange    exchange.Interface  // the block exchange + strategy (bitswap)
	Namesys     namesys.NameSystem  // the name system, resolves paths to hashes
	Diagnostics *diag.Diagnostics   // the diagnostics service

	ctxgroup.ContextGroup

	// dht allows node to Bootstrap when dht is present
	// TODO privatize before merging. This is here temporarily during the
	// migration of the TestNet constructor
	DHT  *dht.IpfsDHT
	mode mode
}

// Mounts defines what the node's mount state is. This should
// perhaps be moved to the daemon or mount. It's here because
// it needs to be accessible across daemon requests.
type Mounts struct {
	Ipfs mount.Mount
	Ipns mount.Mount
}

type ConfigOption func(ctx context.Context) (*IpfsNode, error)

func NewIPFSNode(ctx context.Context, option ConfigOption) (*IpfsNode, error) {
	node, err := option(ctx)
	if err != nil {
		return nil, err
	}

	// Need to make sure it's perfectly clear 1) which variables are expected
	// to be initialized at this point, and 2) which variables will be
	// initialized after this point.

	node.Blocks, err = bserv.New(node.Blockstore, node.Exchange)
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	if node.Peerstore == nil {
		node.Peerstore = peer.NewPeerstore()
	}
	node.DAG = merkledag.NewDAGService(node.Blocks)
	node.Pinning, err = pin.LoadPinner(node.Datastore, node.DAG)
	if err != nil {
		node.Pinning = pin.NewPinner(node.Datastore, node.DAG)
	}
	node.Resolver = &path.Resolver{DAG: node.DAG}
	return node, nil
}

func Offline(cfg *config.Config) ConfigOption {
	return Standard(cfg, false)
}

func Online(cfg *config.Config) ConfigOption {
	return Standard(cfg, true)
}

// DEPRECATED: use Online, Offline functions
func Standard(cfg *config.Config, online bool) ConfigOption {
	return func(ctx context.Context) (n *IpfsNode, err error) {

		success := false // flip to true after all sub-system inits succeed
		defer func() {
			if !success && n != nil {
				n.Close()
			}
		}()

		if cfg == nil {
			return nil, debugerror.Errorf("configuration required")
		}
		n = &IpfsNode{
			mode: func() mode {
				if online {
					return onlineMode
				}
				return offlineMode
			}(),
			Config: cfg,
		}

		n.ContextGroup = ctxgroup.WithContextAndTeardown(ctx, n.teardown)
		ctx = n.ContextGroup.Context()

		// setup Peerstore
		n.Peerstore = peer.NewPeerstore()

		// setup datastore.
		if n.Datastore, err = makeDatastore(cfg.Datastore); err != nil {
			return nil, debugerror.Wrap(err)
		}

		// setup local peer ID (private key is loaded in online setup)
		if err := n.loadID(); err != nil {
			return nil, err
		}

		n.Blockstore, err = bstore.WriteCached(bstore.NewBlockstore(n.Datastore), kSizeBlockstoreWriteCache)
		if err != nil {
			return nil, debugerror.Wrap(err)
		}

		// setup online services
		if online {
			if err := n.StartOnlineServices(); err != nil {
				return nil, err // debugerror.Wraps.
			}
		} else {
			n.Exchange = offline.Exchange(n.Blockstore)
		}

		success = true
		return n, nil
	}
}

func (n *IpfsNode) StartOnlineServices() error {
	ctx := n.Context()

	if n.PeerHost != nil { // already online.
		return debugerror.New("node already online")
	}

	// load private key
	if err := n.loadPrivateKey(); err != nil {
		return err
	}

	peerhost, err := constructPeerHost(ctx, n.ContextGroup, n.Config, n.Identity, n.Peerstore)
	if err != nil {
		return debugerror.Wrap(err)
	}
	n.PeerHost = peerhost

	// setup diagnostics service
	n.Diagnostics = diag.NewDiagnostics(n.Identity, n.PeerHost)

	// setup routing service
	dhtRouting, err := constructDHTRouting(ctx, n.ContextGroup, n.PeerHost, n.Datastore)
	if err != nil {
		return debugerror.Wrap(err)
	}
	n.DHT = dhtRouting
	n.Routing = dhtRouting

	// setup exchange service
	const alwaysSendToPeer = true // use YesManStrategy
	bitswapNetwork := bsnet.NewFromIpfsHost(n.PeerHost, n.Routing)
	n.Exchange = bitswap.New(ctx, n.Identity, bitswapNetwork, n.Blockstore, alwaysSendToPeer)

	// setup name system
	// TODO implement an offline namesys that serves only local names.
	n.Namesys = namesys.NewNameSystem(n.Routing)

	// TODO consider moving connection supervision into the Network. We've
	// discussed improvements to this Node constructor. One improvement
	// would be to make the node configurable, allowing clients to inject
	// an Exchange, Network, or Routing component and have the constructor
	// manage the wiring. In that scenario, this dangling function is a bit
	// awkward.
	var bootstrapPeers []peer.PeerInfo
	for _, bootstrap := range n.Config.Bootstrap {
		p, err := toPeer(bootstrap)
		if err != nil {
			log.Event(ctx, "bootstrapError", n.Identity, lgbl.Error(err))
			log.Errorf("%s bootstrap error: %s", n.Identity, err)
			return err
		}
		bootstrapPeers = append(bootstrapPeers, p)
	}
	go superviseConnections(ctx, n.PeerHost, n.DHT, n.Peerstore, bootstrapPeers)
	return nil
}

func (n *IpfsNode) teardown() error {
	if err := n.Datastore.Close(); err != nil {
		return err
	}
	return nil
}

func (n *IpfsNode) OnlineMode() bool {
	switch n.mode {
	case onlineMode:
		return true
	default:
		return false
	}
}

func (n *IpfsNode) Resolve(k util.Key) (*merkledag.Node, error) {
	return (&path.Resolver{n.DAG}).ResolvePath(k.String())
}

func (n *IpfsNode) Bootstrap(ctx context.Context, peers []peer.PeerInfo) error {
	if n.DHT != nil {
		for _, p := range peers {
			// TODO bootstrap(ctx, n.PeerHost, n.DHT, n.Peerstore, peers)
			if err := n.DHT.Connect(ctx, p.ID); err != nil {
				return err
			}
		}
	}
	return nil
}

func (n *IpfsNode) loadID() error {
	if n.Identity != "" {
		return debugerror.New("identity already loaded")
	}

	cid := n.Config.Identity.PeerID
	if cid == "" {
		return debugerror.New("Identity was not set in config (was ipfs init run?)")
	}
	if len(cid) == 0 {
		return debugerror.New("No peer ID in config! (was ipfs init run?)")
	}

	n.Identity = peer.ID(b58.Decode(cid))
	return nil
}

func (n *IpfsNode) loadPrivateKey() error {
	if n.Identity == "" || n.Peerstore == nil {
		return debugerror.New("loaded private key out of order.")
	}

	if n.PrivateKey != nil {
		return debugerror.New("private key already loaded")
	}

	sk, err := loadPrivateKey(&n.Config.Identity, n.Identity)
	if err != nil {
		return err
	}

	n.PrivateKey = sk
	n.Peerstore.AddPrivKey(n.Identity, n.PrivateKey)
	return nil
}

func loadPrivateKey(cfg *config.Identity, id peer.ID) (ic.PrivKey, error) {
	sk, err := cfg.DecodePrivateKey("passphrase todo!")
	if err != nil {
		return nil, err
	}

	id2, err := peer.IDFromPrivateKey(sk)
	if err != nil {
		return nil, err
	}

	if id2 != id {
		return nil, fmt.Errorf("private key in config does not match id: %s != %s", id, id2)
	}

	return sk, nil
}

func listenAddresses(cfg *config.Config) ([]ma.Multiaddr, error) {

	var err error
	listen := make([]ma.Multiaddr, len(cfg.Addresses.Swarm))
	for i, addr := range cfg.Addresses.Swarm {

		listen[i], err = ma.NewMultiaddr(addr)
		if err != nil {
			return nil, fmt.Errorf("Failure to parse config.Addresses.Swarm[%d]: %s", i, cfg.Addresses.Swarm)
		}
	}

	return listen, nil
}

// isolates the complex initialization steps
func constructPeerHost(ctx context.Context, ctxg ctxgroup.ContextGroup, cfg *config.Config, id peer.ID, ps peer.Peerstore) (p2phost.Host, error) {
	listenAddrs, err := listenAddresses(cfg)
	// make sure we dont error out if our config includes some addresses we cant use.
	filteredAddrs := swarm.FilterAddrs(listenAddrs)
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	network, err := swarm.NewNetwork(ctx, filteredAddrs, id, ps)
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	ctxg.AddChildGroup(network.CtxGroup())

	peerhost := p2pbhost.New(network)
	// explicitly set these as our listen addrs.
	// (why not do it inside inet.NewNetwork? because this way we can
	// listen on addresses without necessarily advertising those publicly.)
	addrs, err := peerhost.Network().InterfaceListenAddresses()
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	ps.AddAddresses(id, addrs)
	return peerhost, nil
}

func constructDHTRouting(ctx context.Context, ctxg ctxgroup.ContextGroup, host p2phost.Host, ds datastore.ThreadSafeDatastore) (*dht.IpfsDHT, error) {
	dhtRouting := dht.NewDHT(ctx, host, ds)
	dhtRouting.Validators[IpnsValidatorTag] = namesys.ValidateIpnsRecord
	ctxg.AddChildGroup(dhtRouting)
	return dhtRouting, nil
}
<commit_msg>fix(core): perform sophisticated boostrap operation<commit_after>package core

import (
	"fmt"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58"
	ctxgroup "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup"
	datastore "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"

	bstore "github.com/jbenet/go-ipfs/blocks/blockstore"
	bserv "github.com/jbenet/go-ipfs/blockservice"
	config "github.com/jbenet/go-ipfs/config"
	diag "github.com/jbenet/go-ipfs/diagnostics"
	exchange "github.com/jbenet/go-ipfs/exchange"
	bitswap "github.com/jbenet/go-ipfs/exchange/bitswap"
	bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network"
	offline "github.com/jbenet/go-ipfs/exchange/offline"
	mount "github.com/jbenet/go-ipfs/fuse/mount"
	merkledag "github.com/jbenet/go-ipfs/merkledag"
	namesys "github.com/jbenet/go-ipfs/namesys"
	ic "github.com/jbenet/go-ipfs/p2p/crypto"
	p2phost "github.com/jbenet/go-ipfs/p2p/host"
	p2pbhost "github.com/jbenet/go-ipfs/p2p/host/basic"
	swarm "github.com/jbenet/go-ipfs/p2p/net/swarm"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	path "github.com/jbenet/go-ipfs/path"
	pin "github.com/jbenet/go-ipfs/pin"
	routing "github.com/jbenet/go-ipfs/routing"
	dht "github.com/jbenet/go-ipfs/routing/dht"
	util "github.com/jbenet/go-ipfs/util"
	ds2 "github.com/jbenet/go-ipfs/util/datastore2"
	debugerror "github.com/jbenet/go-ipfs/util/debugerror"
	eventlog "github.com/jbenet/go-ipfs/util/eventlog"
	lgbl "github.com/jbenet/go-ipfs/util/eventlog/loggables"
)

const IpnsValidatorTag = "ipns"
const kSizeBlockstoreWriteCache = 100

var log = eventlog.Logger("core")

type mode int

const (
	// zero value is not a valid mode, must be explicitly set
	invalidMode mode = iota
	offlineMode
	onlineMode
)

// IpfsNode is IPFS Core module. It represents an IPFS instance.
type IpfsNode struct {

	// Self
	Identity peer.ID // the local node's identity

	// TODO abstract as repo.Repo
	Config    *config.Config                // the node's configuration
	Datastore ds2.ThreadSafeDatastoreCloser // the local datastore

	// Local node
	Pinning pin.Pinner // the pinning manager
	Mounts  Mounts     // current mount state, if any.

	// Services
	Peerstore  peer.Peerstore       // storage for other Peer instances
	Blockstore bstore.Blockstore    // the block store (lower level)
	Blocks     *bserv.BlockService  // the block service, get/add blocks.
	DAG        merkledag.DAGService // the merkle dag service, get/add objects.
	Resolver   *path.Resolver       // the path resolution system

	// Online
	PrivateKey  ic.PrivKey          // the local node's private Key
	PeerHost    p2phost.Host        // the network host (server+client)
	Routing     routing.IpfsRouting // the routing system. recommend ipfs-dht
	Exchange    exchange.Interface  // the block exchange + strategy (bitswap)
	Namesys     namesys.NameSystem  // the name system, resolves paths to hashes
	Diagnostics *diag.Diagnostics   // the diagnostics service

	ctxgroup.ContextGroup

	// dht allows node to Bootstrap when dht is present
	// TODO privatize before merging. This is here temporarily during the
	// migration of the TestNet constructor
	DHT  *dht.IpfsDHT
	mode mode
}

// Mounts defines what the node's mount state is. This should
// perhaps be moved to the daemon or mount. It's here because
// it needs to be accessible across daemon requests.
type Mounts struct {
	Ipfs mount.Mount
	Ipns mount.Mount
}

type ConfigOption func(ctx context.Context) (*IpfsNode, error)

func NewIPFSNode(ctx context.Context, option ConfigOption) (*IpfsNode, error) {
	node, err := option(ctx)
	if err != nil {
		return nil, err
	}

	// Need to make sure it's perfectly clear 1) which variables are expected
	// to be initialized at this point, and 2) which variables will be
	// initialized after this point.

	node.Blocks, err = bserv.New(node.Blockstore, node.Exchange)
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	if node.Peerstore == nil {
		node.Peerstore = peer.NewPeerstore()
	}
	node.DAG = merkledag.NewDAGService(node.Blocks)
	node.Pinning, err = pin.LoadPinner(node.Datastore, node.DAG)
	if err != nil {
		node.Pinning = pin.NewPinner(node.Datastore, node.DAG)
	}
	node.Resolver = &path.Resolver{DAG: node.DAG}
	return node, nil
}

func Offline(cfg *config.Config) ConfigOption {
	return Standard(cfg, false)
}

func Online(cfg *config.Config) ConfigOption {
	return Standard(cfg, true)
}

// DEPRECATED: use Online, Offline functions
func Standard(cfg *config.Config, online bool) ConfigOption {
	return func(ctx context.Context) (n *IpfsNode, err error) {

		success := false // flip to true after all sub-system inits succeed
		defer func() {
			if !success && n != nil {
				n.Close()
			}
		}()

		if cfg == nil {
			return nil, debugerror.Errorf("configuration required")
		}
		n = &IpfsNode{
			mode: func() mode {
				if online {
					return onlineMode
				}
				return offlineMode
			}(),
			Config: cfg,
		}

		n.ContextGroup = ctxgroup.WithContextAndTeardown(ctx, n.teardown)
		ctx = n.ContextGroup.Context()

		// setup Peerstore
		n.Peerstore = peer.NewPeerstore()

		// setup datastore.
		if n.Datastore, err = makeDatastore(cfg.Datastore); err != nil {
			return nil, debugerror.Wrap(err)
		}

		// setup local peer ID (private key is loaded in online setup)
		if err := n.loadID(); err != nil {
			return nil, err
		}

		n.Blockstore, err = bstore.WriteCached(bstore.NewBlockstore(n.Datastore), kSizeBlockstoreWriteCache)
		if err != nil {
			return nil, debugerror.Wrap(err)
		}

		// setup online services
		if online {
			if err := n.StartOnlineServices(); err != nil {
				return nil, err // debugerror.Wraps.
			}
		} else {
			n.Exchange = offline.Exchange(n.Blockstore)
		}

		success = true
		return n, nil
	}
}

func (n *IpfsNode) StartOnlineServices() error {
	ctx := n.Context()

	if n.PeerHost != nil { // already online.
		return debugerror.New("node already online")
	}

	// load private key
	if err := n.loadPrivateKey(); err != nil {
		return err
	}

	peerhost, err := constructPeerHost(ctx, n.ContextGroup, n.Config, n.Identity, n.Peerstore)
	if err != nil {
		return debugerror.Wrap(err)
	}
	n.PeerHost = peerhost

	// setup diagnostics service
	n.Diagnostics = diag.NewDiagnostics(n.Identity, n.PeerHost)

	// setup routing service
	dhtRouting, err := constructDHTRouting(ctx, n.ContextGroup, n.PeerHost, n.Datastore)
	if err != nil {
		return debugerror.Wrap(err)
	}
	n.DHT = dhtRouting
	n.Routing = dhtRouting

	// setup exchange service
	const alwaysSendToPeer = true // use YesManStrategy
	bitswapNetwork := bsnet.NewFromIpfsHost(n.PeerHost, n.Routing)
	n.Exchange = bitswap.New(ctx, n.Identity, bitswapNetwork, n.Blockstore, alwaysSendToPeer)

	// setup name system
	// TODO implement an offline namesys that serves only local names.
	n.Namesys = namesys.NewNameSystem(n.Routing)

	// TODO consider moving connection supervision into the Network. We've
	// discussed improvements to this Node constructor. One improvement
	// would be to make the node configurable, allowing clients to inject
	// an Exchange, Network, or Routing component and have the constructor
	// manage the wiring. In that scenario, this dangling function is a bit
	// awkward.
	var bootstrapPeers []peer.PeerInfo
	for _, bootstrap := range n.Config.Bootstrap {
		p, err := toPeer(bootstrap)
		if err != nil {
			log.Event(ctx, "bootstrapError", n.Identity, lgbl.Error(err))
			log.Errorf("%s bootstrap error: %s", n.Identity, err)
			return err
		}
		bootstrapPeers = append(bootstrapPeers, p)
	}
	go superviseConnections(ctx, n.PeerHost, n.DHT, n.Peerstore, bootstrapPeers)
	return nil
}

func (n *IpfsNode) teardown() error {
	if err := n.Datastore.Close(); err != nil {
		return err
	}
	return nil
}

func (n *IpfsNode) OnlineMode() bool {
	switch n.mode {
	case onlineMode:
		return true
	default:
		return false
	}
}

func (n *IpfsNode) Resolve(k util.Key) (*merkledag.Node, error) {
	return (&path.Resolver{n.DAG}).ResolvePath(k.String())
}

// Bootstrap is undefined when node is not in OnlineMode
func (n *IpfsNode) Bootstrap(ctx context.Context, peers []peer.PeerInfo) error {

	// TODO what should return value be when in offlineMode?

	if n.DHT != nil {
		return bootstrap(ctx, n.PeerHost, n.DHT, n.Peerstore, peers)
	}
	return nil
}

func (n *IpfsNode) loadID() error {
	if n.Identity != "" {
		return debugerror.New("identity already loaded")
	}

	cid := n.Config.Identity.PeerID
	if cid == "" {
		return debugerror.New("Identity was not set in config (was ipfs init run?)")
	}
	if len(cid) == 0 {
		return debugerror.New("No peer ID in config! (was ipfs init run?)")
	}

	n.Identity = peer.ID(b58.Decode(cid))
	return nil
}

func (n *IpfsNode) loadPrivateKey() error {
	if n.Identity == "" || n.Peerstore == nil {
		return debugerror.New("loaded private key out of order.")
	}

	if n.PrivateKey != nil {
		return debugerror.New("private key already loaded")
	}

	sk, err := loadPrivateKey(&n.Config.Identity, n.Identity)
	if err != nil {
		return err
	}

	n.PrivateKey = sk
	n.Peerstore.AddPrivKey(n.Identity, n.PrivateKey)
	return nil
}

func loadPrivateKey(cfg *config.Identity, id peer.ID) (ic.PrivKey, error) {
	sk, err := cfg.DecodePrivateKey("passphrase todo!")
	if err != nil {
		return nil, err
	}

	id2, err := peer.IDFromPrivateKey(sk)
	if err != nil {
		return nil, err
	}

	if id2 != id {
		return nil, fmt.Errorf("private key in config does not match id: %s != %s", id, id2)
	}

	return sk, nil
}

func listenAddresses(cfg *config.Config) ([]ma.Multiaddr, error) {

	var err error
	listen := make([]ma.Multiaddr, len(cfg.Addresses.Swarm))
	for i, addr := range cfg.Addresses.Swarm {

		listen[i], err = ma.NewMultiaddr(addr)
		if err != nil {
			return nil, fmt.Errorf("Failure to parse config.Addresses.Swarm[%d]: %s", i, cfg.Addresses.Swarm)
		}
	}

	return listen, nil
}

// isolates the complex initialization steps
func constructPeerHost(ctx context.Context, ctxg ctxgroup.ContextGroup, cfg *config.Config, id peer.ID, ps peer.Peerstore) (p2phost.Host, error) {
	listenAddrs, err := listenAddresses(cfg)
	// make sure we dont error out if our config includes some addresses we cant use.
	filteredAddrs := swarm.FilterAddrs(listenAddrs)
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	network, err := swarm.NewNetwork(ctx, filteredAddrs, id, ps)
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	ctxg.AddChildGroup(network.CtxGroup())

	peerhost := p2pbhost.New(network)
	// explicitly set these as our listen addrs.
	// (why not do it inside inet.NewNetwork? because this way we can
	// listen on addresses without necessarily advertising those publicly.)
	addrs, err := peerhost.Network().InterfaceListenAddresses()
	if err != nil {
		return nil, debugerror.Wrap(err)
	}
	ps.AddAddresses(id, addrs)
	return peerhost, nil
}

func constructDHTRouting(ctx context.Context, ctxg ctxgroup.ContextGroup, host p2phost.Host, ds datastore.ThreadSafeDatastore) (*dht.IpfsDHT, error) {
	dhtRouting := dht.NewDHT(ctx, host, ds)
	dhtRouting.Validators[IpnsValidatorTag] = namesys.ValidateIpnsRecord
	ctxg.AddChildGroup(dhtRouting)
	return dhtRouting, nil
}
<|endoftext|>


<commit_before>package fake_gordon

import (
	"code.google.com/p/gogoprotobuf/proto"
	"github.com/nu7hatch/gouuid"
	"github.com/vito/gordon/warden"
	"io/ioutil"
	"os"
	"sync"
)

type FakeGordon struct {
	Connected    bool
	ConnectError error

	createdHandles []string
	CreateError    error

	stoppedHandles []string
	StopError      error

	destroyedHandles []string
	DestroyError     error

	SpawnError error

	LinkError error

	NetInError error

	LimitMemoryError error

	GetMemoryLimitError error

	LimitDiskError error

	GetDiskLimitError error

	ListError error

	InfoError error

	AttachError error

	scriptsThatRan              []*RunningScript
	runCallbacks                map[*RunningScript]RunCallback
	runReturnProcessID          uint32
	runReturnProcessPayloadChan <-chan *warden.ProcessPayload
	runReturnError              error

	copiedIn    []*CopiedIn
	copyInError error

	copiedOut                     []*CopiedOut
	fileContentToProvideOnCopyOut []byte
	copyOutError                  error

	lock *sync.Mutex
}

type RunCallback func() (uint32, <-chan *warden.ProcessPayload, error)

type RunningScript struct {
	Handle string
	Script string
}

type CopiedIn struct {
	Handle string
	Src    string
	Dst    string
}

type CopiedOut struct {
	Handle string
	Src    string
	Dst    string
	Owner  string
}

func New() *FakeGordon {
	f := &FakeGordon{}
	f.Reset()
	return f
}

func (f *FakeGordon) Reset() {
	f.lock = &sync.Mutex{}
	f.Connected = false
	f.ConnectError = nil

	f.createdHandles = []string{}
	f.CreateError = nil

	f.stoppedHandles = []string{}
	f.StopError = nil

	f.destroyedHandles = []string{}
	f.DestroyError = nil

	f.SpawnError = nil
	f.LinkError = nil
	f.NetInError = nil
	f.LimitMemoryError = nil
	f.GetMemoryLimitError = nil
	f.LimitDiskError = nil
	f.GetDiskLimitError = nil
	f.ListError = nil
	f.InfoError = nil
	f.AttachError = nil

	f.scriptsThatRan = make([]*RunningScript, 0)
	f.runCallbacks = make(map[*RunningScript]RunCallback)
	f.runReturnProcessID = 0
	f.runReturnError = nil

	f.copyInError = nil
	f.copyOutError = nil
	f.copiedIn = []*CopiedIn{}
	f.copiedOut = []*CopiedOut{}
	f.fileContentToProvideOnCopyOut = []byte{}
}

func (f *FakeGordon) Connect() error {
	f.lock.Lock()
	defer f.lock.Unlock()
	f.Connected = true
	return f.ConnectError
}

func (f *FakeGordon) Create() (*warden.CreateResponse, error) {
	f.lock.Lock()
	defer f.lock.Unlock()
	if f.CreateError != nil {
		return nil, f.CreateError
	}

	handleUuid, _ := uuid.NewV4()
	handle := handleUuid.String()[:11]

	f.createdHandles = append(f.createdHandles, handle)

	return &warden.CreateResponse{
		Handle: proto.String(handle),
	}, nil
}

func (f *FakeGordon) CreatedHandles() []string {
	f.lock.Lock()
	defer f.lock.Unlock()

	return f.createdHandles
}

func (f *FakeGordon) Stop(handle string, background, kill bool) (*warden.StopResponse, error) {
	f.lock.Lock()
	defer f.lock.Unlock()
	if f.StopError != nil {
		return nil, f.StopError
	}

	f.stoppedHandles = append(f.stoppedHandles, handle)

	return &warden.StopResponse{}, nil
}

func (f *FakeGordon) StoppedHandles() []string {
	f.lock.Lock()
	defer f.lock.Unlock()

	return f.stoppedHandles
}

func (f *FakeGordon) Destroy(handle string) (*warden.DestroyResponse, error) {
	f.lock.Lock()
	defer f.lock.Unlock()
	if f.DestroyError != nil {
		return nil, f.DestroyError
	}

	f.destroyedHandles = append(f.destroyedHandles, handle)

	return &warden.DestroyResponse{}, nil
}

func (f *FakeGordon) DestroyedHandles() []string {
	f.lock.Lock()
	defer f.lock.Unlock()

	return f.destroyedHandles
}

func (f *FakeGordon) NetIn(handle string) (*warden.NetInResponse, error) {
	panic("NOOP!")
	return nil, f.NetInError
}

func (f *FakeGordon) LimitMemory(handle string, limit uint64) (*warden.LimitMemoryResponse, error) {
	panic("NOOP!")
	return nil, f.LimitMemoryError
}

func (f *FakeGordon) GetMemoryLimit(handle string) (uint64, error) {
	panic("NOOP!")
	return 0, f.GetMemoryLimitError
}

func (f *FakeGordon) LimitDisk(handle string, limit uint64) (*warden.LimitDiskResponse, error) {
	panic("NOOP!")
	return nil, f.LimitDiskError
}

func (f *FakeGordon) GetDiskLimit(handle string) (uint64, error) {
	panic("NOOP!")
	return 0, f.GetDiskLimitError
}

func (f *FakeGordon) List() (*warden.ListResponse, error) {
	panic("NOOP!")
	return nil, f.ListError
}

func (f *FakeGordon) Info(handle string) (*warden.InfoResponse, error) {
	panic("NOOP!")
	return nil, f.InfoError
}

func (f *FakeGordon) CopyIn(handle, src, dst string) (*warden.CopyInResponse, error) {
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.copyInError != nil {
		return nil, f.copyInError
	}

	f.copiedIn = append(f.copiedIn, &CopiedIn{
		Handle: handle,
		Src:    src,
		Dst:    dst,
	})

	return &warden.CopyInResponse{}, nil
}

func (f *FakeGordon) ThingsCopiedIn() []*CopiedIn {
	f.lock.Lock()
	defer f.lock.Unlock()

	return f.copiedIn
}

func (f *FakeGordon) SetCopyInErr(err error) {
	f.lock.Lock()
	defer f.lock.Unlock()

	f.copyInError = err
}

func (f *FakeGordon) CopyOut(handle, src, dst, owner string) (*warden.CopyOutResponse, error) {
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.copyOutError != nil {
		return nil, f.copyOutError
	}

	f.copiedOut = append(f.copiedOut, &CopiedOut{
		Handle: handle,
		Src:    src,
		Dst:    dst,
		Owner:  owner,
	})

	if len(f.fileContentToProvideOnCopyOut) > 0 {
		ioutil.WriteFile(dst, f.fileContentToProvideOnCopyOut, os.ModePerm)
	}

	return &warden.CopyOutResponse{}, nil
}

func (f *FakeGordon) ThingsCopiedOut() []*CopiedOut {
	f.lock.Lock()
	defer f.lock.Unlock()

	return f.copiedOut
}

func (f *FakeGordon) SetCopyOutErr(err error) {
	f.lock.Lock()
	defer f.lock.Unlock()

	f.copyOutError = err
}

func (f *FakeGordon) SetCopyOutFileContent(data []byte) {
	f.lock.Lock()
	defer f.lock.Unlock()

	f.fileContentToProvideOnCopyOut = data
}

func (f *FakeGordon) Attach(handle string, jobID uint32) (<-chan *warden.ProcessPayload, error) {
	panic("NOOP!")
	return nil, f.AttachError
}

func (f *FakeGordon) ScriptsThatRan() []*RunningScript {
	f.lock.Lock()
	defer f.lock.Unlock()
	return f.scriptsThatRan
}

func (f *FakeGordon) SetRunReturnValues(processID uint32, processPayloadChan <-chan *warden.ProcessPayload, err error) {
	f.lock.Lock()
	defer f.lock.Unlock()
	f.runReturnProcessID = processID
	f.runReturnProcessPayloadChan = processPayloadChan
	f.runReturnError = err
}

func (f *FakeGordon) WhenRunning(handle string, script string, callback RunCallback) {
	f.lock.Lock()
	defer f.lock.Unlock()

	f.runCallbacks[&RunningScript{handle, script}] = callback
}

func (f *FakeGordon) Run(handle string, script string) (uint32, <-chan *warden.ProcessPayload, error) {
	f.lock.Lock()
	defer f.lock.Unlock()

	f.scriptsThatRan = append(f.scriptsThatRan, &RunningScript{
		Handle: handle,
		Script: script,
	})

	for ro, cb := range f.runCallbacks {
		if ro.Handle == handle && ro.Script == script {
			return cb()
		}
	}

	return f.runReturnProcessID, f.runReturnProcessPayloadChan, f.runReturnError
}
<commit_msg>fake WhenRunning: always match if no handle given<commit_after>package fake_gordon

import (
	"code.google.com/p/gogoprotobuf/proto"
	"github.com/nu7hatch/gouuid"
	"github.com/vito/gordon/warden"
	"io/ioutil"
	"os"
	"sync"
)

type FakeGordon struct {
	Connected    bool
	ConnectError error

	createdHandles []string
	CreateError    error

	stoppedHandles []string
	StopError      error

	destroyedHandles []string
	DestroyError     error

	SpawnError error

	LinkError error

	NetInError error

	LimitMemoryError error

	GetMemoryLimitError error

	LimitDiskError error

	GetDiskLimitError error

	ListError error

	InfoError error

	AttachError error

	scriptsThatRan              []*RunningScript
	runCallbacks                map[*RunningScript]RunCallback
	runReturnProcessID          uint32
	runReturnProcessPayloadChan <-chan *warden.ProcessPayload
	runReturnError              error

	copiedIn    []*CopiedIn
	copyInError error

	copiedOut                     []*CopiedOut
	fileContentToProvideOnCopyOut []byte
	copyOutError                  error

	lock *sync.Mutex
}

type RunCallback func() (uint32, <-chan *warden.ProcessPayload, error)

type RunningScript struct {
	Handle string
	Script string
}

type CopiedIn struct {
	Handle string
	Src    string
	Dst    string
}

type CopiedOut struct {
	Handle string
	Src    string
	Dst    string
	Owner  string
}

func New() *FakeGordon {
	f := &FakeGordon{}
	f.Reset()
	return f
}

func (f *FakeGordon) Reset() {
	f.lock = &sync.Mutex{}
	f.Connected = false
	f.ConnectError = nil

	f.createdHandles = []string{}
	f.CreateError = nil

	f.stoppedHandles = []string{}
	f.StopError = nil

	f.destroyedHandles = []string{}
	f.DestroyError = nil

	f.SpawnError = nil
	f.LinkError = nil
	f.NetInError = nil
	f.LimitMemoryError = nil
	f.GetMemoryLimitError = nil
	f.LimitDiskError = nil
	f.GetDiskLimitError = nil
	f.ListError = nil
	f.InfoError = nil
	f.AttachError = nil

	f.scriptsThatRan = make([]*RunningScript, 0)
	f.runCallbacks = make(map[*RunningScript]RunCallback)
	f.runReturnProcessID = 0
	f.runReturnError = nil

	f.copyInError = nil
	f.copyOutError = nil
	f.copiedIn = []*CopiedIn{}
	f.copiedOut = []*CopiedOut{}
	f.fileContentToProvideOnCopyOut = []byte{}
}

func (f *FakeGordon) Connect() error {
	f.lock.Lock()
	defer f.lock.Unlock()
	f.Connected = true
	return f.ConnectError
}

func (f *FakeGordon) Create() (*warden.CreateResponse, error) {
	f.lock.Lock()
	defer f.lock.Unlock()
	if f.CreateError != nil {
		return nil, f.CreateError
	}

	handleUuid, _ := uuid.NewV4()
	handle := handleUuid.String()[:11]

	f.createdHandles = append(f.createdHandles, handle)

	return &warden.CreateResponse{
		Handle: proto.String(handle),
	}, nil
}

func (f *FakeGordon) CreatedHandles() []string {
	f.lock.Lock()
	defer f.lock.Unlock()

	return f.createdHandles
}

func (f *FakeGordon) Stop(handle string, background, kill bool) (*warden.StopResponse, error) {
	f.lock.Lock()
	defer f.lock.Unlock()
	if f.StopError != nil {
		return nil, f.StopError
	}

	f.stoppedHandles = append(f.stoppedHandles, handle)

	return &warden.StopResponse{}, nil
}

func (f *FakeGordon) StoppedHandles() []string {
	f.lock.Lock()
	defer f.lock.Unlock()

	return f.stoppedHandles
}

func (f *FakeGordon) Destroy(handle string) (*warden.DestroyResponse, error) {
	f.lock.Lock()
	defer f.lock.Unlock()
	if f.DestroyError != nil {
		return nil, f.DestroyError
	}

	f.destroyedHandles = append(f.destroyedHandles, handle)

	return &warden.DestroyResponse{}, nil
}

func (f *FakeGordon) DestroyedHandles() []string {
	f.lock.Lock()
	defer f.lock.Unlock()

	return f.destroyedHandles
}

func (f *FakeGordon) NetIn(handle string) (*warden.NetInResponse, error) {
	panic("NOOP!")
	return nil, f.NetInError
}

func (f *FakeGordon) LimitMemory(handle string, limit uint64) (*warden.LimitMemoryResponse, error) {
	panic("NOOP!")
	return nil, f.LimitMemoryError
}

func (f *FakeGordon) GetMemoryLimit(handle string) (uint64, error) {
	panic("NOOP!")
	return 0, f.GetMemoryLimitError
}

func (f *FakeGordon) LimitDisk(handle string, limit uint64) (*warden.LimitDiskResponse, error) {
	panic("NOOP!")
	return nil, f.LimitDiskError
}

func (f *FakeGordon) GetDiskLimit(handle string) (uint64, error) {
	panic("NOOP!")
	return 0, f.GetDiskLimitError
}

func (f *FakeGordon) List() (*warden.ListResponse, error) {
	panic("NOOP!")
	return nil, f.ListError
}

func (f *FakeGordon) Info(handle string) (*warden.InfoResponse, error) {
	panic("NOOP!")
	return nil, f.InfoError
}

func (f *FakeGordon) CopyIn(handle, src, dst string) (*warden.CopyInResponse, error) {
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.copyInError != nil {
		return nil, f.copyInError
	}

	f.copiedIn = append(f.copiedIn, &CopiedIn{
		Handle: handle,
		Src:    src,
		Dst:    dst,
	})

	return &warden.CopyInResponse{}, nil
}

func (f *FakeGordon) ThingsCopiedIn() []*CopiedIn {
	f.lock.Lock()
	defer f.lock.Unlock()

	return f.copiedIn
}

func (f *FakeGordon) SetCopyInErr(err error) {
	f.lock.Lock()
	defer f.lock.Unlock()

	f.copyInError = err
}

func (f *FakeGordon) CopyOut(handle, src, dst, owner string) (*warden.CopyOutResponse, error) {
	f.lock.Lock()
	defer f.lock.Unlock()

	if f.copyOutError != nil {
		return nil, f.copyOutError
	}

	f.copiedOut = append(f.copiedOut, &CopiedOut{
		Handle: handle,
		Src:    src,
		Dst:    dst,
		Owner:  owner,
	})

	if len(f.fileContentToProvideOnCopyOut) > 0 {
		ioutil.WriteFile(dst, f.fileContentToProvideOnCopyOut, os.ModePerm)
	}

	return &warden.CopyOutResponse{}, nil
}

func (f *FakeGordon) ThingsCopiedOut() []*CopiedOut {
	f.lock.Lock()
	defer f.lock.Unlock()

	return f.copiedOut
}

func (f *FakeGordon) SetCopyOutErr(err error) {
	f.lock.Lock()
	defer f.lock.Unlock()

	f.copyOutError = err
}

func (f *FakeGordon) SetCopyOutFileContent(data []byte) {
	f.lock.Lock()
	defer f.lock.Unlock()

	f.fileContentToProvideOnCopyOut = data
}

func (f *FakeGordon) Attach(handle string, jobID uint32) (<-chan *warden.ProcessPayload, error) {
	panic("NOOP!")
	return nil, f.AttachError
}

func (f *FakeGordon) ScriptsThatRan() []*RunningScript {
	f.lock.Lock()
	defer f.lock.Unlock()
	return f.scriptsThatRan
}

func (f *FakeGordon) SetRunReturnValues(processID uint32, processPayloadChan <-chan *warden.ProcessPayload, err error) {
	f.lock.Lock()
	defer f.lock.Unlock()
	f.runReturnProcessID = processID
	f.runReturnProcessPayloadChan = processPayloadChan
	f.runReturnError = err
}

func (f *FakeGordon) WhenRunning(handle string, script string, callback RunCallback) {
	f.lock.Lock()
	defer f.lock.Unlock()

	f.runCallbacks[&RunningScript{handle, script}] = callback
}

func (f *FakeGordon) Run(handle string, script string) (uint32, <-chan *warden.ProcessPayload, error) {
	f.lock.Lock()

	f.scriptsThatRan = append(f.scriptsThatRan, &RunningScript{
		Handle: handle,
		Script: script,
	})

	f.lock.Unlock()

	for ro, cb := range f.runCallbacks {
		if (ro.Handle == "" || ro.Handle == handle) && ro.Script == script {
			return cb()
		}
	}

	return f.runReturnProcessID, f.runReturnProcessPayloadChan, f.runReturnError
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package webrisk

import (
	"context"
	"os"
	"testing"
	"time"

	pb "github.com/google/webrisk/internal/webrisk_proto"
)

// The system tests below are non-deterministic and operate by performing
// network requests against the Web Risk API servers. Thus, in order to
// operate they need the user's API key. This can be specified using the -apikey
// command-line flag when running the tests.
var apiKey = os.Getenv("WEBRISK_APIKEY")

func TestNetworkAPIUpdate(t *testing.T) {
	if apiKey == "" {
		t.Skip()
	}

	nm, err := newNetAPI(DefaultServerURL, apiKey, "")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}

	req := &pb.ComputeThreatListDiffRequest{
		ThreatType: pb.ThreatType_MALWARE,
	}

	dat, err := nm.ListUpdate(context.Background(), req.ThreatType,
		req.VersionToken, req.Constraints.SupportedCompressions)
	if err != nil {
		t.Fatal(err)
	}

	hashes, err := decodeHashes(dat.GetAdditions())
	if err != nil {
		t.Fatal(err)
	}
	for i := 0; i < len(hashes) && i < 10; i++ {
		hash := hashes[i]
		fullHashReq := &pb.SearchHashesRequest{
			ThreatTypes: []pb.ThreatType{pb.ThreatType_MALWARE},
			HashPrefix:  []byte(hash),
		}
		fullHashResp, err := nm.HashLookup(context.Background(),
			fullHashReq.HashPrefix, fullHashReq.ThreatTypes)
		if err != nil {
			t.Fatal(err)
		}
		if got := len(fullHashResp.GetThreats()); got < 1 {
			t.Fatalf("len(r.GetMatches()), got: %v, want: > 0", got)
		}
	}
}

func TestNetworkAPILookup(t *testing.T) {
	if apiKey == "" {
		t.Skip()
	}

	nm, err := newNetAPI(DefaultServerURL, apiKey, "")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}

	var c = pb.ComputeThreatListDiffRequest{
		ThreatType: pb.ThreatType_MALWARE,
	}
	url := "http://testwebrisk | w.appspot.com/apiv4/ANY_PLATFORM/MALWARE/URL/"
	hash := hashFromPattern(url)
	req := &pb.SearchHashesRequest{
		ThreatTypes: []pb.ThreatType{c.ThreatType},
		HashPrefix:  []byte(hash[:minHashPrefixLength]),
	}
	resp, err := nm.HashLookup(context.Background(), req.HashPrefix,
		req.ThreatTypes)
	if err != nil {
		t.Fatalf("Lookup failed: %v", err)
	}
	if len(resp.GetThreats()) < 1 {
		t.Fatalf("No matches returned. Resp %v. Url %v.", resp.String(), url)
	}
}

func TestWebriskClient(t *testing.T) {
	if apiKey == "" {
		t.Skip()
	}

	sb, err := NewWebriskClient(Config{
		APIKey:       apiKey,
		ID:           "GoWebriskClientSystemTest",
		DBPath:       "/tmp/webriskClient.db",
		UpdatePeriod: 10 * time.Second,
		ThreatLists:  []ThreatType{ThreatTypeMalware},
	})
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	if err := sb.WaitUntilReady(ctx); err != nil {
		t.Fatal(err)
	}
	cancel()

	url := "http://testwebrisk | w.appspot.com/apiv4/ANY_PLATFORM/MALWARE/URL/"

	urls := []string{url, url + "?q=test"}
	threats, e := sb.LookupURLs(urls)
	if e != nil {
		t.Fatal(e)
	}
	if len(threats[0]) != 1 || len(threats[1]) != 1 {
		t.Errorf("lookupURL failed")
	}

	if err := sb.Close(); err != nil {
		t.Fatal(err)
	}
	ctx, cancel = context.WithTimeout(context.Background(), time.Millisecond)
	if err := sb.WaitUntilReady(ctx); err != errClosed {
		t.Errorf("sb.WaitUntilReady() = %v on closed WebriskClient, want %v", err, errClosed)
	}
	cancel()

	for _, hs := range sb.db.tfl {
		if hs.Len() == 0 {
			t.Errorf("Database length: got %d,, want >0", hs.Len())
		}
	}
	if len(sb.c.pttls) != 1 {
		t.Errorf("Cache length: got %d, want 1", len(sb.c.pttls))
	}
}
<commit_msg>Fixes the client system test's url.<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package webrisk

import (
	"context"
	"os"
	"testing"
	"time"

	pb "github.com/google/webrisk/internal/webrisk_proto"
)

// The system tests below are non-deterministic and operate by performing
// network requests against the Web Risk API servers. Thus, in order to
// operate they need the user's API key. This can be specified using the -apikey
// command-line flag when running the tests.
var apiKey = os.Getenv("APIKEY")

func TestNetworkAPIUpdate(t *testing.T) {
	if apiKey == "" {
		t.Skip()
	}

	nm, err := newNetAPI(DefaultServerURL, apiKey, "")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}

	req := &pb.ComputeThreatListDiffRequest{
		ThreatType: pb.ThreatType_MALWARE,
	}

	dat, err := nm.ListUpdate(context.Background(), req.ThreatType,
		req.VersionToken, req.Constraints.SupportedCompressions)
	if err != nil {
		t.Fatal(err)
	}

	hashes, err := decodeHashes(dat.GetAdditions())
	if err != nil {
		t.Fatal(err)
	}
	for i := 0; i < len(hashes) && i < 10; i++ {
		hash := hashes[i]
		fullHashReq := &pb.SearchHashesRequest{
			ThreatTypes: []pb.ThreatType{pb.ThreatType_MALWARE},
			HashPrefix:  []byte(hash),
		}
		fullHashResp, err := nm.HashLookup(context.Background(),
			fullHashReq.HashPrefix, fullHashReq.ThreatTypes)
		if err != nil {
			t.Fatal(err)
		}
		if got := len(fullHashResp.GetThreats()); got < 1 {
			t.Fatalf("len(r.GetMatches()), got: %v, want: > 0", got)
		}
	}
}

func TestNetworkAPILookup(t *testing.T) {
	if apiKey == "" {
		t.Skip()
	}

	nm, err := newNetAPI(DefaultServerURL, apiKey, "")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}

	var c = pb.ComputeThreatListDiffRequest{
		ThreatType: pb.ThreatType_MALWARE,
	}
	url := "http://testwebrisk | w.appspot.com/apiv4/ANY_PLATFORM/MALWARE/URL/"
	hash := hashFromPattern(url)
	req := &pb.SearchHashesRequest{
		ThreatTypes: []pb.ThreatType{c.ThreatType},
		HashPrefix:  []byte(hash[:minHashPrefixLength]),
	}
	resp, err := nm.HashLookup(context.Background(), req.HashPrefix,
		req.ThreatTypes)
	if err != nil {
		t.Fatalf("Lookup failed: %v", err)
	}
	if len(resp.GetThreats()) < 1 {
		t.Fatalf("No matches returned. Resp %v. Url %v.", resp.String(), url)
	}
}

func TestWebriskClient(t *testing.T) {
	if apiKey == "" {
		t.Skip()
	}

	sb, err := NewWebriskClient(Config{
		APIKey:       apiKey,
		ID:           "GoWebriskClientSystemTest",
		DBPath:       "/tmp/webriskClient.db",
		UpdatePeriod: 10 * time.Second,
		ThreatLists:  []ThreatType{ThreatTypeMalware},
	})
	if err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	if err := sb.WaitUntilReady(ctx); err != nil {
		t.Fatal(err)
	}
	cancel()

	url := "http://testsafebrowsing.appspot.com/apiv4/ANY_PLATFORM/MALWARE/URL/"

	urls := []string{url, url + "?q=test"}
	threats, e := sb.LookupURLs(urls)
	if e != nil {
		t.Fatal(e)
	}
	if len(threats[0]) != 1 || len(threats[1]) != 1 {
		t.Errorf("lookupURL failed")
	}

	if err := sb.Close(); err != nil {
		t.Fatal(err)
	}
	ctx, cancel = context.WithTimeout(context.Background(), time.Millisecond)
	if err := sb.WaitUntilReady(ctx); err != errClosed {
		t.Errorf("sb.WaitUntilReady() = %v on closed WebriskClient, want %v", err, errClosed)
	}
	cancel()

	for _, hs := range sb.db.tfl {
		if hs.Len() == 0 {
			t.Errorf("Database length: got %d,, want >0", hs.Len())
		}
	}
	if len(sb.c.pttls) != 1 {
		t.Errorf("Cache length: got %d, want 1", len(sb.c.pttls))
	}
}
<|endoftext|>


<commit_before>package mount

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
	"github.com/hanwen/go-fuse/v2/fuse"
	"math"
	"sync"
)

type DirectoryHandleId uint64

const (
	directoryStreamBaseOffset = 2 // . & ..
)

type DirectoryHandle struct {
	isFinished        bool
	entryStream       []*filer.Entry
	entryStreamOffset uint64
}

func (dh *DirectoryHandle) reset() {
	*dh = DirectoryHandle{
		isFinished:        false,
		entryStream:       []*filer.Entry{},
		entryStreamOffset: directoryStreamBaseOffset,
	}
}

type DirectoryHandleToInode struct {
	// shares the file handle id sequencer with FileHandleToInode{nextFh}
	sync.Mutex
	dir2inode map[DirectoryHandleId]*DirectoryHandle
}

func NewDirectoryHandleToInode() *DirectoryHandleToInode {
	return &DirectoryHandleToInode{
		dir2inode: make(map[DirectoryHandleId]*DirectoryHandle),
	}
}

func (wfs *WFS) AcquireDirectoryHandle() (DirectoryHandleId, *DirectoryHandle) {
	wfs.fhmap.Lock()
	fh := wfs.fhmap.nextFh
	wfs.fhmap.nextFh++
	wfs.fhmap.Unlock()

	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	dh := new(DirectoryHandle)
	dh.reset()
	wfs.dhmap.dir2inode[DirectoryHandleId(fh)] = dh
	return DirectoryHandleId(fh), dh
}

func (wfs *WFS) GetDirectoryHandle(dhid DirectoryHandleId) *DirectoryHandle {
	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	if dh, found := wfs.dhmap.dir2inode[dhid]; found {
		return dh
	}
	dh := new(DirectoryHandle)
	dh.reset()
	wfs.dhmap.dir2inode[dhid] = dh
	return dh
}

func (wfs *WFS) ReleaseDirectoryHandle(dhid DirectoryHandleId) {
	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	delete(wfs.dhmap.dir2inode, dhid)
}

// Directory handling

/** Open directory
 *
 * Unless the 'default_permissions' mount option is given,
 * this method should check if opendir is permitted for this
 * directory. Optionally opendir may also return an arbitrary
 * filehandle in the fuse_file_info structure, which will be
 * passed to readdir, releasedir and fsyncdir.
 */
func (wfs *WFS) OpenDir(cancel <-chan struct{}, input *fuse.OpenIn, out *fuse.OpenOut) (code fuse.Status) {
	if !wfs.inodeToPath.HasInode(input.NodeId) {
		return fuse.ENOENT
	}
	dhid, _ := wfs.AcquireDirectoryHandle()
	out.Fh = uint64(dhid)
	return fuse.OK
}

/** Release directory
 *
 * If the directory has been removed after the call to opendir, the
 * path parameter will be NULL.
 */
func (wfs *WFS) ReleaseDir(input *fuse.ReleaseIn) {
	wfs.ReleaseDirectoryHandle(DirectoryHandleId(input.Fh))
}

/** Synchronize directory contents
 *
 * If the directory has been removed after the call to opendir, the
 * path parameter will be NULL.
 *
 * If the datasync parameter is non-zero, then only the user data
 * should be flushed, not the meta data
 */
func (wfs *WFS) FsyncDir(cancel <-chan struct{}, input *fuse.FsyncIn) (code fuse.Status) {
	return fuse.OK
}

/** Read directory
 *
 * The filesystem may choose between two modes of operation:
 *
 * 1) The readdir implementation ignores the offset parameter, and
 * passes zero to the filler function's offset.  The filler
 * function will not return '1' (unless an error happens), so the
 * whole directory is read in a single readdir operation.
 *
 * 2) The readdir implementation keeps track of the offsets of the
 * directory entries.  It uses the offset parameter and always
 * passes non-zero offset to the filler function.  When the buffer
 * is full (or an error happens) the filler function will return
 * '1'.
 */
func (wfs *WFS) ReadDir(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
	return wfs.doReadDirectory(input, out, false)
}

func (wfs *WFS) ReadDirPlus(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
	return wfs.doReadDirectory(input, out, true)
}

func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPlusMode bool) fuse.Status {
	dh := wfs.GetDirectoryHandle(DirectoryHandleId(input.Fh))
	if input.Offset == 0 {
		dh.reset()
	} else if dh.isFinished && input.Offset >= dh.entryStreamOffset {
		entryCurrentIndex := input.Offset - dh.entryStreamOffset
		if uint64(len(dh.entryStream)) <= entryCurrentIndex {
			return fuse.OK
		}
	}

	isEarlyTerminated := false
	dirPath, code := wfs.inodeToPath.GetPath(input.NodeId)
	if code != fuse.OK {
		return code
	}

	var dirEntry fuse.DirEntry
	processEachEntryFn := func(entry *filer.Entry) bool {
		dirEntry.Name = entry.Name()
		dirEntry.Mode = toSyscallMode(entry.Mode)
		inode := wfs.inodeToPath.Lookup(dirPath.Child(dirEntry.Name), entry.Crtime.Unix(), entry.IsDirectory(), len(entry.HardLinkId) > 0, entry.Inode, isPlusMode)
		dirEntry.Ino = inode
		if !isPlusMode {
			if !out.AddDirEntry(dirEntry) {
				isEarlyTerminated = true
				return false
			}
		} else {
			entryOut := out.AddDirLookupEntry(dirEntry)
			if entryOut == nil {
				isEarlyTerminated = true
				return false
			}
			if fh, found := wfs.fhmap.FindFileHandle(inode); found {
				glog.V(4).Infof("readdir opened file %s", dirPath.Child(dirEntry.Name))
				entry = filer.FromPbEntry(string(dirPath), fh.entry)
			}
			wfs.outputFilerEntry(entryOut, inode, entry)
		}
		return true
	}

	if input.Offset < directoryStreamBaseOffset {
		if !isPlusMode {
			if input.Offset == 0 {
				out.AddDirEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: "."})
			}
			out.AddDirEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: ".."})
		} else {
			if input.Offset == 0 {
				out.AddDirLookupEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: "."})
			}
			out.AddDirLookupEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: ".."})
		}
		input.Offset = directoryStreamBaseOffset
	}

	var lastEntryName string
	if input.Offset >= dh.entryStreamOffset {
		if input.Offset > dh.entryStreamOffset {
			entryPreviousIndex := (input.Offset - dh.entryStreamOffset) - 1
			if uint64(len(dh.entryStream)) > entryPreviousIndex {
				lastEntryName = dh.entryStream[entryPreviousIndex].Name()
				dh.entryStream = dh.entryStream[entryPreviousIndex:]
				dh.entryStreamOffset = input.Offset - 1
			}
		}
		entryCurrentIndex := input.Offset - dh.entryStreamOffset
		for uint64(len(dh.entryStream)) > entryCurrentIndex {
			entry := dh.entryStream[entryCurrentIndex]
			if processEachEntryFn(entry) {
				lastEntryName = entry.Name()
				entryCurrentIndex++
			} else {
				// early terminated
				return fuse.OK
			}
		}
	}

	var err error
	if err = meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath, nil); err != nil {
		glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
	}
	listErr := wfs.metaCache.ListDirectoryEntries(context.Background(), dirPath, lastEntryName, false, int64(math.MaxInt32), func(entry *filer.Entry) bool {
		dh.entryStream = append(dh.entryStream, entry)
		return processEachEntryFn(entry)
	})
	if listErr != nil {
		glog.Errorf("list meta cache: %v", listErr)
		return fuse.EIO
	}

	if !isEarlyTerminated {
		dh.isFinished = true
	}

	return fuse.OK
}
<commit_msg>mount: ReadDir return EIO on EnsureVisited err<commit_after>package mount

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
	"github.com/hanwen/go-fuse/v2/fuse"
	"math"
	"sync"
)

type DirectoryHandleId uint64

const (
	directoryStreamBaseOffset = 2 // . & ..
)

type DirectoryHandle struct {
	isFinished        bool
	entryStream       []*filer.Entry
	entryStreamOffset uint64
}

func (dh *DirectoryHandle) reset() {
	*dh = DirectoryHandle{
		isFinished:        false,
		entryStream:       []*filer.Entry{},
		entryStreamOffset: directoryStreamBaseOffset,
	}
}

type DirectoryHandleToInode struct {
	// shares the file handle id sequencer with FileHandleToInode{nextFh}
	sync.Mutex
	dir2inode map[DirectoryHandleId]*DirectoryHandle
}

func NewDirectoryHandleToInode() *DirectoryHandleToInode {
	return &DirectoryHandleToInode{
		dir2inode: make(map[DirectoryHandleId]*DirectoryHandle),
	}
}

func (wfs *WFS) AcquireDirectoryHandle() (DirectoryHandleId, *DirectoryHandle) {
	wfs.fhmap.Lock()
	fh := wfs.fhmap.nextFh
	wfs.fhmap.nextFh++
	wfs.fhmap.Unlock()

	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	dh := new(DirectoryHandle)
	dh.reset()
	wfs.dhmap.dir2inode[DirectoryHandleId(fh)] = dh
	return DirectoryHandleId(fh), dh
}

func (wfs *WFS) GetDirectoryHandle(dhid DirectoryHandleId) *DirectoryHandle {
	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	if dh, found := wfs.dhmap.dir2inode[dhid]; found {
		return dh
	}
	dh := new(DirectoryHandle)
	dh.reset()
	wfs.dhmap.dir2inode[dhid] = dh
	return dh
}

func (wfs *WFS) ReleaseDirectoryHandle(dhid DirectoryHandleId) {
	wfs.dhmap.Lock()
	defer wfs.dhmap.Unlock()
	delete(wfs.dhmap.dir2inode, dhid)
}

// Directory handling

/** Open directory
 *
 * Unless the 'default_permissions' mount option is given,
 * this method should check if opendir is permitted for this
 * directory. Optionally opendir may also return an arbitrary
 * filehandle in the fuse_file_info structure, which will be
 * passed to readdir, releasedir and fsyncdir.
 */
func (wfs *WFS) OpenDir(cancel <-chan struct{}, input *fuse.OpenIn, out *fuse.OpenOut) (code fuse.Status) {
	if !wfs.inodeToPath.HasInode(input.NodeId) {
		return fuse.ENOENT
	}
	dhid, _ := wfs.AcquireDirectoryHandle()
	out.Fh = uint64(dhid)
	return fuse.OK
}

/** Release directory
 *
 * If the directory has been removed after the call to opendir, the
 * path parameter will be NULL.
 */
func (wfs *WFS) ReleaseDir(input *fuse.ReleaseIn) {
	wfs.ReleaseDirectoryHandle(DirectoryHandleId(input.Fh))
}

/** Synchronize directory contents
 *
 * If the directory has been removed after the call to opendir, the
 * path parameter will be NULL.
 *
 * If the datasync parameter is non-zero, then only the user data
 * should be flushed, not the meta data
 */
func (wfs *WFS) FsyncDir(cancel <-chan struct{}, input *fuse.FsyncIn) (code fuse.Status) {
	return fuse.OK
}

/** Read directory
 *
 * The filesystem may choose between two modes of operation:
 *
 * 1) The readdir implementation ignores the offset parameter, and
 * passes zero to the filler function's offset.  The filler
 * function will not return '1' (unless an error happens), so the
 * whole directory is read in a single readdir operation.
 *
 * 2) The readdir implementation keeps track of the offsets of the
 * directory entries.  It uses the offset parameter and always
 * passes non-zero offset to the filler function.  When the buffer
 * is full (or an error happens) the filler function will return
 * '1'.
 */
func (wfs *WFS) ReadDir(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
	return wfs.doReadDirectory(input, out, false)
}

func (wfs *WFS) ReadDirPlus(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
	return wfs.doReadDirectory(input, out, true)
}

func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPlusMode bool) fuse.Status {
	dh := wfs.GetDirectoryHandle(DirectoryHandleId(input.Fh))
	if input.Offset == 0 {
		dh.reset()
	} else if dh.isFinished && input.Offset >= dh.entryStreamOffset {
		entryCurrentIndex := input.Offset - dh.entryStreamOffset
		if uint64(len(dh.entryStream)) <= entryCurrentIndex {
			return fuse.OK
		}
	}

	isEarlyTerminated := false
	dirPath, code := wfs.inodeToPath.GetPath(input.NodeId)
	if code != fuse.OK {
		return code
	}

	var dirEntry fuse.DirEntry
	processEachEntryFn := func(entry *filer.Entry) bool {
		dirEntry.Name = entry.Name()
		dirEntry.Mode = toSyscallMode(entry.Mode)
		inode := wfs.inodeToPath.Lookup(dirPath.Child(dirEntry.Name), entry.Crtime.Unix(), entry.IsDirectory(), len(entry.HardLinkId) > 0, entry.Inode, isPlusMode)
		dirEntry.Ino = inode
		if !isPlusMode {
			if !out.AddDirEntry(dirEntry) {
				isEarlyTerminated = true
				return false
			}
		} else {
			entryOut := out.AddDirLookupEntry(dirEntry)
			if entryOut == nil {
				isEarlyTerminated = true
				return false
			}
			if fh, found := wfs.fhmap.FindFileHandle(inode); found {
				glog.V(4).Infof("readdir opened file %s", dirPath.Child(dirEntry.Name))
				entry = filer.FromPbEntry(string(dirPath), fh.entry)
			}
			wfs.outputFilerEntry(entryOut, inode, entry)
		}
		return true
	}

	if input.Offset < directoryStreamBaseOffset {
		if !isPlusMode {
			if input.Offset == 0 {
				out.AddDirEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: "."})
			}
			out.AddDirEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: ".."})
		} else {
			if input.Offset == 0 {
				out.AddDirLookupEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: "."})
			}
			out.AddDirLookupEntry(fuse.DirEntry{Mode: fuse.S_IFDIR, Name: ".."})
		}
		input.Offset = directoryStreamBaseOffset
	}

	var lastEntryName string
	if input.Offset >= dh.entryStreamOffset {
		if input.Offset > dh.entryStreamOffset {
			entryPreviousIndex := (input.Offset - dh.entryStreamOffset) - 1
			if uint64(len(dh.entryStream)) > entryPreviousIndex {
				lastEntryName = dh.entryStream[entryPreviousIndex].Name()
				dh.entryStream = dh.entryStream[entryPreviousIndex:]
				dh.entryStreamOffset = input.Offset - 1
			}
		}
		entryCurrentIndex := input.Offset - dh.entryStreamOffset
		for uint64(len(dh.entryStream)) > entryCurrentIndex {
			entry := dh.entryStream[entryCurrentIndex]
			if processEachEntryFn(entry) {
				lastEntryName = entry.Name()
				entryCurrentIndex++
			} else {
				// early terminated
				return fuse.OK
			}
		}
	}

	var err error
	if err = meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath, nil); err != nil {
		glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
		return fuse.EIO
	}
	listErr := wfs.metaCache.ListDirectoryEntries(context.Background(), dirPath, lastEntryName, false, int64(math.MaxInt32), func(entry *filer.Entry) bool {
		dh.entryStream = append(dh.entryStream, entry)
		return processEachEntryFn(entry)
	})
	if listErr != nil {
		glog.Errorf("list meta cache: %v", listErr)
		return fuse.EIO
	}

	if !isEarlyTerminated {
		dh.isFinished = true
	}

	return fuse.OK
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package featuretests

import (
	"bufio"
	"io/ioutil"
	"time"

	"github.com/juju/loggo"
	"github.com/juju/names"
	jujutesting "github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"
	"gopkg.in/mgo.v2/bson"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/api"
	agentcmd "github.com/juju/juju/cmd/jujud/agent"
	agenttesting "github.com/juju/juju/cmd/jujud/agent/testing"
	"github.com/juju/juju/feature"
	"github.com/juju/juju/lease"
	"github.com/juju/juju/state"
	coretesting "github.com/juju/juju/testing"
	"github.com/juju/juju/testing/factory"
	"github.com/juju/juju/version"
	"github.com/juju/juju/worker/logsender"
	"github.com/juju/juju/worker/peergrouper"
)

// dblogSuite tests that logs flow correctly from the machine and unit
// agents over the API into MongoDB. These are very much integration
// tests with more detailed testing of the individual components
// being done in unit tests.
type dblogSuite struct {
	agenttesting.AgentSuite
}

func (s *dblogSuite) SetUpTest(c *gc.C) {
	s.SetInitialFeatureFlags("db-log")
	s.AgentSuite.SetUpTest(c)

	// Change the path to "juju-run", so that the
	// tests don't try to write to /usr/local/bin.
	file, _ := ioutil.TempFile("", "juju-run")
	defer file.Close()
	s.PatchValue(&agentcmd.JujuRun, file.Name())
}

func (s *dblogSuite) TestMachineAgentLogsGoToDB(c *gc.C) {
	s.SetFeatureFlags("db-log")
	foundLogs := s.runMachineAgentTest(c)
	c.Assert(foundLogs, jc.IsTrue)
}

func (s *dblogSuite) TestMachineAgentLogsGoToDBWithJES(c *gc.C) {
	s.SetFeatureFlags(feature.JES)
	foundLogs := s.runMachineAgentTest(c)
	c.Assert(foundLogs, jc.IsTrue)
}

func (s *dblogSuite) TestMachineAgentWithoutFeatureFlag(c *gc.C) {
	s.SetFeatureFlags()
	foundLogs := s.runMachineAgentTest(c)
	c.Assert(foundLogs, jc.IsFalse)
}

func (s *dblogSuite) TestUnitAgentLogsGoToDB(c *gc.C) {
	s.SetFeatureFlags("db-log")
	foundLogs := s.runUnitAgentTest(c)
	c.Assert(foundLogs, jc.IsTrue)
}

func (s *dblogSuite) TestUnitAgentLogsGoToDBWithJES(c *gc.C) {
	s.SetFeatureFlags(feature.JES)
	foundLogs := s.runUnitAgentTest(c)
	c.Assert(foundLogs, jc.IsTrue)
}

func (s *dblogSuite) TestUnitAgentWithoutFeatureFlag(c *gc.C) {
	s.SetFeatureFlags()
	foundLogs := s.runUnitAgentTest(c)
	c.Assert(foundLogs, jc.IsFalse)
}

func (s *dblogSuite) runMachineAgentTest(c *gc.C) bool {
	// Create a machine and an agent for it.
	m, password := s.Factory.MakeMachineReturningPassword(c, &factory.MachineParams{
		Nonce: agent.BootstrapNonce,
	})

	s.PrimeAgent(c, m.Tag(), password, version.Current)
	agentConf := agentcmd.NewAgentConf(s.DataDir())
	agentConf.ReadConfig(m.Tag().String())
	logsCh, err := logsender.InstallBufferedLogWriter(1000)
	c.Assert(err, jc.ErrorIsNil)
	machineAgentFactory := agentcmd.MachineAgentFactoryFn(agentConf, agentConf, logsCh)
	a := machineAgentFactory(m.Id())

	// Ensure there's no logs to begin with.
	c.Assert(s.getLogCount(c, m.Tag()), gc.Equals, 0)

	// Start the agent.
	go func() { c.Check(a.Run(nil), jc.ErrorIsNil) }()
	defer a.Stop()

	return s.waitForLogs(c, m.Tag())
}

func (s *dblogSuite) runUnitAgentTest(c *gc.C) bool {
	// Lease setup stuff, only needed when running a uniter.
	m, err := lease.NewLeaseManager(s.State)
	c.Assert(err, jc.ErrorIsNil)
	s.AddCleanup(func(c *gc.C) {
		m.Kill()
		c.Assert(m.Wait(), jc.ErrorIsNil)
	})
	// Create a unit and an agent for it.
	u, password := s.Factory.MakeUnitReturningPassword(c, nil)
	s.PrimeAgent(c, u.Tag(), password, version.Current)
	logsCh, err := logsender.InstallBufferedLogWriter(1000)
	c.Assert(err, jc.ErrorIsNil)
	a := agentcmd.NewUnitAgent(nil, logsCh)
	s.InitAgent(c, a, "--unit-name", u.Name(), "--log-to-stderr=true")

	// Ensure there's no logs to begin with.
	c.Assert(s.getLogCount(c, u.Tag()), gc.Equals, 0)

	// Start the agent.
	go func() { c.Assert(a.Run(nil), jc.ErrorIsNil) }()
	defer a.Stop()

	return s.waitForLogs(c, u.Tag())
}

func (s *dblogSuite) getLogCount(c *gc.C, entity names.Tag) int {
	// TODO(mjs) - replace this with State's functionality for reading
	// logs from the DB, once it gets this. This will happen before
	// the DB logging feature branch is merged.
	logs := s.Session.DB("logs").C("logs")
	count, err := logs.Find(bson.M{"n": entity.String()}).Count()
	c.Assert(err, jc.ErrorIsNil)
	return count
}

func (s *dblogSuite) waitForLogs(c *gc.C, entityTag names.Tag) bool {
	for a := coretesting.LongAttempt.Start(); a.Next(); {
		if s.getLogCount(c, entityTag) > 0 {
			return true
		}
	}
	return false
}

// debugLogDbSuite tests that the debuglog API works when logs are
// being read from the database.
type debugLogDbSuite struct {
	agenttesting.AgentSuite
}

var _ = gc.Suite(&debugLogDbSuite{})

func (s *debugLogDbSuite) SetUpSuite(c *gc.C) {
	s.SetInitialFeatureFlags("db-log")

	// Restart mongod with a the replicaset enabled.
	mongod := jujutesting.MgoServer
	mongod.Params = []string{"--replSet", "juju"}
	mongod.Restart()

	// Initiate the replicaset.
	info := mongod.DialInfo()
	args := peergrouper.InitiateMongoParams{
		DialInfo:       info,
		MemberHostPort: mongod.Addr(),
	}
	err := peergrouper.MaybeInitiateMongoServer(args)
	c.Assert(err, jc.ErrorIsNil)

	s.AgentSuite.SetUpSuite(c)
}

func (s *debugLogDbSuite) TearDownSuite(c *gc.C) {
	// Restart mongod without the replicaset enabled so as not to
	// affect other test that reply on this mongod instance in this
	// package.
	mongod := jujutesting.MgoServer
	mongod.Params = []string{}
	mongod.Restart()

	s.AgentSuite.TearDownSuite(c)
}

func (s *debugLogDbSuite) TestLogsAPI(c *gc.C) {
	dbLogger := state.NewDbLogger(s.State, names.NewMachineTag("99"))
	defer dbLogger.Close()

	t := time.Date(2015, 6, 23, 13, 8, 49, 0, time.UTC)
	dbLogger.Log(t, "juju.foo", "code.go:42", loggo.INFO, "all is well")
	dbLogger.Log(t.Add(time.Second), "juju.bar", "go.go:99", loggo.ERROR, "no it isn't")

	lines := make(chan string)
	go func(numLines int) {
		client := s.APIState.Client()
		reader, err := client.WatchDebugLog(api.DebugLogParams{})
		c.Assert(err, jc.ErrorIsNil)
		defer reader.Close()

		bufReader := bufio.NewReader(reader)
		for n := 0; n < numLines; n++ {
			line, err := bufReader.ReadString('\n')
			c.Assert(err, jc.ErrorIsNil)
			lines <- line
		}
	}(3)

	assertLine := func(expected string) {
		select {
		case actual := <-lines:
			c.Assert(actual, gc.Equals, expected)
		case <-time.After(coretesting.LongWait):
			c.Fatal("timed out waiting for log line")
		}
	}

	// Read the 2 lines that are in the logs collection.
	assertLine("machine-99: 2015-06-23 13:08:49 INFO juju.foo code.go:42 all is well\n")
	assertLine("machine-99: 2015-06-23 13:08:50 ERROR juju.bar go.go:99 no it isn't\n")

	// Now write and observe another log. This should be read from the oplog.
	dbLogger.Log(t.Add(2*time.Second), "ju.jitsu", "no.go:3", loggo.WARNING, "beep beep")
	assertLine("machine-99: 2015-06-23 13:08:51 WARNING ju.jitsu no.go:3 beep beep\n")
}
<commit_msg>Skip changing password during test<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package featuretests

import (
	"bufio"
	"io/ioutil"
	"time"

	"github.com/juju/loggo"
	"github.com/juju/names"
	jujutesting "github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"
	"gopkg.in/mgo.v2/bson"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/api"
	agentcmd "github.com/juju/juju/cmd/jujud/agent"
	agenttesting "github.com/juju/juju/cmd/jujud/agent/testing"
	"github.com/juju/juju/cmd/jujud/util/password"
	"github.com/juju/juju/feature"
	"github.com/juju/juju/lease"
	"github.com/juju/juju/state"
	coretesting "github.com/juju/juju/testing"
	"github.com/juju/juju/testing/factory"
	"github.com/juju/juju/version"
	"github.com/juju/juju/worker/logsender"
	"github.com/juju/juju/worker/peergrouper"
)

// dblogSuite tests that logs flow correctly from the machine and unit
// agents over the API into MongoDB. These are very much integration
// tests with more detailed testing of the individual components
// being done in unit tests.
type dblogSuite struct {
	agenttesting.AgentSuite
}

func (s *dblogSuite) SetUpTest(c *gc.C) {
	s.SetInitialFeatureFlags("db-log")
	s.AgentSuite.SetUpTest(c)
	s.PatchValue(&password.EnsureJujudPassword, func() error { return nil })

	// Change the path to "juju-run", so that the
	// tests don't try to write to /usr/local/bin.
	file, _ := ioutil.TempFile("", "juju-run")
	defer file.Close()
	s.PatchValue(&agentcmd.JujuRun, file.Name())
}

func (s *dblogSuite) TestMachineAgentLogsGoToDB(c *gc.C) {
	s.SetFeatureFlags("db-log")
	foundLogs := s.runMachineAgentTest(c)
	c.Assert(foundLogs, jc.IsTrue)
}

func (s *dblogSuite) TestMachineAgentLogsGoToDBWithJES(c *gc.C) {
	s.SetFeatureFlags(feature.JES)
	foundLogs := s.runMachineAgentTest(c)
	c.Assert(foundLogs, jc.IsTrue)
}

func (s *dblogSuite) TestMachineAgentWithoutFeatureFlag(c *gc.C) {
	s.SetFeatureFlags()
	foundLogs := s.runMachineAgentTest(c)
	c.Assert(foundLogs, jc.IsFalse)
}

func (s *dblogSuite) TestUnitAgentLogsGoToDB(c *gc.C) {
	s.SetFeatureFlags("db-log")
	foundLogs := s.runUnitAgentTest(c)
	c.Assert(foundLogs, jc.IsTrue)
}

func (s *dblogSuite) TestUnitAgentLogsGoToDBWithJES(c *gc.C) {
	s.SetFeatureFlags(feature.JES)
	foundLogs := s.runUnitAgentTest(c)
	c.Assert(foundLogs, jc.IsTrue)
}

func (s *dblogSuite) TestUnitAgentWithoutFeatureFlag(c *gc.C) {
	s.SetFeatureFlags()
	foundLogs := s.runUnitAgentTest(c)
	c.Assert(foundLogs, jc.IsFalse)
}

func (s *dblogSuite) runMachineAgentTest(c *gc.C) bool {
	// Create a machine and an agent for it.
	m, password := s.Factory.MakeMachineReturningPassword(c, &factory.MachineParams{
		Nonce: agent.BootstrapNonce,
	})

	s.PrimeAgent(c, m.Tag(), password, version.Current)
	agentConf := agentcmd.NewAgentConf(s.DataDir())
	agentConf.ReadConfig(m.Tag().String())
	logsCh, err := logsender.InstallBufferedLogWriter(1000)
	c.Assert(err, jc.ErrorIsNil)
	machineAgentFactory := agentcmd.MachineAgentFactoryFn(agentConf, agentConf, logsCh)
	a := machineAgentFactory(m.Id())

	// Ensure there's no logs to begin with.
	c.Assert(s.getLogCount(c, m.Tag()), gc.Equals, 0)

	// Start the agent.
	go func() { c.Check(a.Run(nil), jc.ErrorIsNil) }()
	defer a.Stop()

	return s.waitForLogs(c, m.Tag())
}

func (s *dblogSuite) runUnitAgentTest(c *gc.C) bool {
	// Lease setup stuff, only needed when running a uniter.
	m, err := lease.NewLeaseManager(s.State)
	c.Assert(err, jc.ErrorIsNil)
	s.AddCleanup(func(c *gc.C) {
		m.Kill()
		c.Assert(m.Wait(), jc.ErrorIsNil)
	})
	// Create a unit and an agent for it.
	u, password := s.Factory.MakeUnitReturningPassword(c, nil)
	s.PrimeAgent(c, u.Tag(), password, version.Current)
	logsCh, err := logsender.InstallBufferedLogWriter(1000)
	c.Assert(err, jc.ErrorIsNil)
	a := agentcmd.NewUnitAgent(nil, logsCh)
	s.InitAgent(c, a, "--unit-name", u.Name(), "--log-to-stderr=true")

	// Ensure there's no logs to begin with.
	c.Assert(s.getLogCount(c, u.Tag()), gc.Equals, 0)

	// Start the agent.
	go func() { c.Assert(a.Run(nil), jc.ErrorIsNil) }()
	defer a.Stop()

	return s.waitForLogs(c, u.Tag())
}

func (s *dblogSuite) getLogCount(c *gc.C, entity names.Tag) int {
	// TODO(mjs) - replace this with State's functionality for reading
	// logs from the DB, once it gets this. This will happen before
	// the DB logging feature branch is merged.
	logs := s.Session.DB("logs").C("logs")
	count, err := logs.Find(bson.M{"n": entity.String()}).Count()
	c.Assert(err, jc.ErrorIsNil)
	return count
}

func (s *dblogSuite) waitForLogs(c *gc.C, entityTag names.Tag) bool {
	for a := coretesting.LongAttempt.Start(); a.Next(); {
		if s.getLogCount(c, entityTag) > 0 {
			return true
		}
	}
	return false
}

// debugLogDbSuite tests that the debuglog API works when logs are
// being read from the database.
type debugLogDbSuite struct {
	agenttesting.AgentSuite
}

var _ = gc.Suite(&debugLogDbSuite{})

func (s *debugLogDbSuite) SetUpSuite(c *gc.C) {
	s.SetInitialFeatureFlags("db-log")

	// Restart mongod with a the replicaset enabled.
	mongod := jujutesting.MgoServer
	mongod.Params = []string{"--replSet", "juju"}
	mongod.Restart()

	// Initiate the replicaset.
	info := mongod.DialInfo()
	args := peergrouper.InitiateMongoParams{
		DialInfo:       info,
		MemberHostPort: mongod.Addr(),
	}
	err := peergrouper.MaybeInitiateMongoServer(args)
	c.Assert(err, jc.ErrorIsNil)

	s.AgentSuite.SetUpSuite(c)
}

func (s *debugLogDbSuite) TearDownSuite(c *gc.C) {
	// Restart mongod without the replicaset enabled so as not to
	// affect other test that reply on this mongod instance in this
	// package.
	mongod := jujutesting.MgoServer
	mongod.Params = []string{}
	mongod.Restart()

	s.AgentSuite.TearDownSuite(c)
}

func (s *debugLogDbSuite) TestLogsAPI(c *gc.C) {
	dbLogger := state.NewDbLogger(s.State, names.NewMachineTag("99"))
	defer dbLogger.Close()

	t := time.Date(2015, 6, 23, 13, 8, 49, 0, time.UTC)
	dbLogger.Log(t, "juju.foo", "code.go:42", loggo.INFO, "all is well")
	dbLogger.Log(t.Add(time.Second), "juju.bar", "go.go:99", loggo.ERROR, "no it isn't")

	lines := make(chan string)
	go func(numLines int) {
		client := s.APIState.Client()
		reader, err := client.WatchDebugLog(api.DebugLogParams{})
		c.Assert(err, jc.ErrorIsNil)
		defer reader.Close()

		bufReader := bufio.NewReader(reader)
		for n := 0; n < numLines; n++ {
			line, err := bufReader.ReadString('\n')
			c.Assert(err, jc.ErrorIsNil)
			lines <- line
		}
	}(3)

	assertLine := func(expected string) {
		select {
		case actual := <-lines:
			c.Assert(actual, gc.Equals, expected)
		case <-time.After(coretesting.LongWait):
			c.Fatal("timed out waiting for log line")
		}
	}

	// Read the 2 lines that are in the logs collection.
	assertLine("machine-99: 2015-06-23 13:08:49 INFO juju.foo code.go:42 all is well\n")
	assertLine("machine-99: 2015-06-23 13:08:50 ERROR juju.bar go.go:99 no it isn't\n")

	// Now write and observe another log. This should be read from the oplog.
	dbLogger.Log(t.Add(2*time.Second), "ju.jitsu", "no.go:3", loggo.WARNING, "beep beep")
	assertLine("machine-99: 2015-06-23 13:08:51 WARNING ju.jitsu no.go:3 beep beep\n")
}
<|endoftext|>