<commit_before>package datastore

import (
	"errors"
	"time"

	query "github.com/ipfs/go-datastore/query"
)

/*
Datastore represents storage for any key-value pair.

Datastores are general enough to be backed by all kinds of different storage:
in-memory caches, databases, a remote datastore, flat files on disk, etc.

The general idea is to wrap a more complicated storage facility in a simple,
uniform interface, keeping the freedom of using the right tools for the job.
In particular, a Datastore can aggregate other datastores in interesting ways,
like sharded (to distribute load) or tiered access (caches before databases).

While Datastores should be written general enough to accept all sorts of
values, some implementations will undoubtedly have to be specific (e.g. SQL
databases where fields should be decomposed into columns), particularly to
support queries efficiently. Moreover, certain datastores may enforce certain
types of values (e.g. requiring an io.Reader, a specific struct, etc) or
serialization formats (JSON, Protobufs, etc).

IMPORTANT: No Datastore should ever Panic! This is a cross-module interface,
and thus it should behave predictably and handle exceptional conditions with
proper error reporting. Thus, all Datastore calls may return errors, which
should be checked by callers.
*/
type Datastore interface {
	// Put stores the object `value` named by `key`.
	//
	// The generalized Datastore interface does not impose a value type,
	// allowing various datastore middleware implementations (which do not
	// handle the values directly) to be composed together.
	//
	// Ultimately, the lowest-level datastore will need to do some value checking
	// or risk getting incorrect values. It may also be useful to expose a more
	// type-safe interface to your application, and do the checking up-front.
	Put(key Key, value []byte) error

	// Get retrieves the object `value` named by `key`.
	// Get will return ErrNotFound if the key is not mapped to a value.
	Get(key Key) (value []byte, err error)

	// Has returns whether the `key` is mapped to a `value`.
	// In some contexts, it may be much cheaper only to check for existence of
	// a value, rather than retrieving the value itself. (e.g. HTTP HEAD).
	// The default implementation is found in `GetBackedHas`.
	Has(key Key) (exists bool, err error)

	// Delete removes the value for given `key`.
	Delete(key Key) error

	// Query searches the datastore and returns a query result. This function
	// may return before the query actually runs. To wait for the query:
	//
	//   result, _ := ds.Query(q)
	//
	//   // use the channel interface; result may come in at different times
	//   for entry := range result.Next() { ... }
	//
	//   // or wait for the query to be completely done
	//   entries, _ := result.Rest()
	//   for entry := range entries { ... }
	//
	Query(q query.Query) (query.Results, error)
}

type Batching interface {
	Datastore

	Batch() (Batch, error)
}

var ErrBatchUnsupported = errors.New("this datastore does not support batching")

// ThreadSafeDatastore is an interface that all threadsafe datastore should
// implement to leverage type safety checks.
type ThreadSafeDatastore interface {
	Datastore

	IsThreadSafe()
}

// CheckedDatastore is an interface that should be implemented by datastores
// which may need checking on-disk data integrity.
type CheckedDatastore interface {
	Datastore

	Check() error
}

// CheckedDatastore is an interface that should be implemented by datastores
// which want to provide a mechanism to check data integrity and/or
// error correction.
type ScrubbedDatastore interface {
	Datastore

	Scrub() error
}

// GCDatastore is an interface that should be implemented by datastores which
// don't free disk space by just removing data from them.
type GCDatastore interface {
	Datastore

	CollectGarbage() error
}

// PersistentDatastore is an interface that should be implemented by datastores
// which can report disk usage.
type PersistentDatastore interface {
	Datastore

	// DiskUsage returns the space used by a datastore, in bytes.
	DiskUsage() (uint64, error)
}

// DiskUsage checks if a Datastore is a
// PersistentDatastore and returns its DiskUsage(),
// otherwise returns 0.
func DiskUsage(d Datastore) (uint64, error) {
	persDs, ok := d.(PersistentDatastore)
	if !ok {
		return 0, nil
	}
	return persDs.DiskUsage()
}

// TTLDatastore is an interface that should be implemented by datastores that
// support expiring entries.
type TTLDatastore interface {
	Datastore

	PutWithTTL(key Key, value interface{}, ttl time.Duration) error
	SetTTL(key Key, ttl time.Duration) error
}

// Txn is an interface for transactions that can be committed or discarded.
type Txn interface {
	// Txn has the same capabilities as Datastore. Txns allow users to batch
	// queries and mutations to the Datastore into atomic groups, or transactions.
	// Actions performed on a transaction will not take hold until a successful
	// call to Commit has been made. Likewise, transactions can be aborted by
	// calling Discard before a successful Commit has been made.
	Put(key Key, value interface{}) error
	Get(key Key) (value interface{}, err error)
	Has(key Key) (exists bool, err error)
	Delete(key Key) error
	Query(q query.Query) (query.Results, error)

	// Commit finalizes a transaction, attempting to commit it to the Datastore.
	// May return an error if the transaction has gone stale. The presence of an
	// error is an indication that the data was not committed to the Datastore.
	Commit() error
	// Discard throws away changes recorded in a transaction without committing
	// them to the underlying Datastore. Any calls made to Discard after Commit
	// has been successfully called will have no effect on the transaction and
	// state of the Datastore, making it safe to defer.
	Discard()
}

// TxnDatastore is an interface that should be implemented by datastores that
// support transactions.
type TxnDatastore interface {
	Datastore

	NewTransaction(readOnly bool) Txn
}

// Errors

// ErrNotFound is returned by Get, Has, and Delete when a datastore does not
// map the given key to a value.
var ErrNotFound = errors.New("datastore: key not found")

// ErrInvalidType is returned by Put when a given value is incopatible with
// the type the datastore supports. This means a conversion (or serialization)
// is needed beforehand.
var ErrInvalidType = errors.New("datastore: invalid type error")

// GetBackedHas provides a default Datastore.Has implementation.
// It exists so Datastore.Has implementations can use it, like so:
//
// func (*d SomeDatastore) Has(key Key) (exists bool, err error) {
//   return GetBackedHas(d, key)
// }
func GetBackedHas(ds Datastore, key Key) (bool, error) {
	_, err := ds.Get(key)
	switch err {
	case nil:
		return true, nil
	case ErrNotFound:
		return false, nil
	default:
		return false, err
	}
}

type Batch interface {
	Put(key Key, val []byte) error

	Delete(key Key) error

	Commit() error
}
<commit_msg>Revert "Copy Datastore -> Txn for clarity of origin"<commit_after>package datastore

import (
	"errors"
	"time"

	query "github.com/ipfs/go-datastore/query"
)

/*
Datastore represents storage for any key-value pair.

Datastores are general enough to be backed by all kinds of different storage:
in-memory caches, databases, a remote datastore, flat files on disk, etc.

The general idea is to wrap a more complicated storage facility in a simple,
uniform interface, keeping the freedom of using the right tools for the job.
In particular, a Datastore can aggregate other datastores in interesting ways,
like sharded (to distribute load) or tiered access (caches before databases).

While Datastores should be written general enough to accept all sorts of
values, some implementations will undoubtedly have to be specific (e.g. SQL
databases where fields should be decomposed into columns), particularly to
support queries efficiently. Moreover, certain datastores may enforce certain
types of values (e.g. requiring an io.Reader, a specific struct, etc) or
serialization formats (JSON, Protobufs, etc).

IMPORTANT: No Datastore should ever Panic! This is a cross-module interface,
and thus it should behave predictably and handle exceptional conditions with
proper error reporting. Thus, all Datastore calls may return errors, which
should be checked by callers.
*/
type Datastore interface {
	// Put stores the object `value` named by `key`.
	//
	// The generalized Datastore interface does not impose a value type,
	// allowing various datastore middleware implementations (which do not
	// handle the values directly) to be composed together.
	//
	// Ultimately, the lowest-level datastore will need to do some value checking
	// or risk getting incorrect values. It may also be useful to expose a more
	// type-safe interface to your application, and do the checking up-front.
	Put(key Key, value []byte) error

	// Get retrieves the object `value` named by `key`.
	// Get will return ErrNotFound if the key is not mapped to a value.
	Get(key Key) (value []byte, err error)

	// Has returns whether the `key` is mapped to a `value`.
	// In some contexts, it may be much cheaper only to check for existence of
	// a value, rather than retrieving the value itself. (e.g. HTTP HEAD).
	// The default implementation is found in `GetBackedHas`.
	Has(key Key) (exists bool, err error)

	// Delete removes the value for given `key`.
	Delete(key Key) error

	// Query searches the datastore and returns a query result. This function
	// may return before the query actually runs. To wait for the query:
	//
	//   result, _ := ds.Query(q)
	//
	//   // use the channel interface; result may come in at different times
	//   for entry := range result.Next() { ... }
	//
	//   // or wait for the query to be completely done
	//   entries, _ := result.Rest()
	//   for entry := range entries { ... }
	//
	Query(q query.Query) (query.Results, error)
}

type Batching interface {
	Datastore

	Batch() (Batch, error)
}

var ErrBatchUnsupported = errors.New("this datastore does not support batching")

// ThreadSafeDatastore is an interface that all threadsafe datastore should
// implement to leverage type safety checks.
type ThreadSafeDatastore interface {
	Datastore

	IsThreadSafe()
}

// CheckedDatastore is an interface that should be implemented by datastores
// which may need checking on-disk data integrity.
type CheckedDatastore interface {
	Datastore

	Check() error
}

// CheckedDatastore is an interface that should be implemented by datastores
// which want to provide a mechanism to check data integrity and/or
// error correction.
type ScrubbedDatastore interface {
	Datastore

	Scrub() error
}

// GCDatastore is an interface that should be implemented by datastores which
// don't free disk space by just removing data from them.
type GCDatastore interface {
	Datastore

	CollectGarbage() error
}

// PersistentDatastore is an interface that should be implemented by datastores
// which can report disk usage.
type PersistentDatastore interface {
	Datastore

	// DiskUsage returns the space used by a datastore, in bytes.
	DiskUsage() (uint64, error)
}

// DiskUsage checks if a Datastore is a
// PersistentDatastore and returns its DiskUsage(),
// otherwise returns 0.
func DiskUsage(d Datastore) (uint64, error) {
	persDs, ok := d.(PersistentDatastore)
	if !ok {
		return 0, nil
	}
	return persDs.DiskUsage()
}

// TTLDatastore is an interface that should be implemented by datastores that
// support expiring entries.
type TTLDatastore interface {
	Datastore

	PutWithTTL(key Key, value interface{}, ttl time.Duration) error
	SetTTL(key Key, ttl time.Duration) error
}

// Txn is an interface for transactions that can be committed or discarded.
type Txn interface {
	// Txn extends the Datastore type. Txns allow users to batch queries and
	// mutations to the Datastore into atomic groups, or transactions. Actions
	// performed on a transaction will not take hold until a successful call to
	// Commit has been made. Likewise, transactions can be aborted by calling
	// Discard before a successful Commit has been made.
	Datastore

	// Commit finalizes a transaction, attempting to commit it to the Datastore.
	// May return an error if the transaction has gone stale. The presence of an
	// error is an indication that the data was not committed to the Datastore.
	Commit() error
	// Discard throws away changes recorded in a transaction without committing
	// them to the underlying Datastore. Any calls made to Discard after Commit
	// has been successfully called will have no effect on the transaction and
	// state of the Datastore, making it safe to defer.
	Discard()
}

// TxDatastore is an interface that should be implemented by datastores that
// support transactions.
type TxnDatastore interface {
	Datastore

	NewTransaction(readOnly bool) Txn
}

// Errors

// ErrNotFound is returned by Get, Has, and Delete when a datastore does not
// map the given key to a value.
var ErrNotFound = errors.New("datastore: key not found")

// ErrInvalidType is returned by Put when a given value is incopatible with
// the type the datastore supports. This means a conversion (or serialization)
// is needed beforehand.
var ErrInvalidType = errors.New("datastore: invalid type error")

// GetBackedHas provides a default Datastore.Has implementation.
// It exists so Datastore.Has implementations can use it, like so:
//
// func (*d SomeDatastore) Has(key Key) (exists bool, err error) {
//   return GetBackedHas(d, key)
// }
func GetBackedHas(ds Datastore, key Key) (bool, error) {
	_, err := ds.Get(key)
	switch err {
	case nil:
		return true, nil
	case ErrNotFound:
		return false, nil
	default:
		return false, err
	}
}

type Batch interface {
	Put(key Key, val []byte) error

	Delete(key Key) error

	Commit() error
}
<|endoftext|>


<commit_before>// Package gateway connects a Sia node to the Sia flood network. The flood
// network is used to propagate blocks and transactions. The gateway is the
// primary avenue that a node uses to hear about transactions and blocks, and
// is the primary avenue used to tell the network about blocks that you have
// mined or about transactions that you have created.
package gateway

import (
	"errors"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"sync"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/persist"
	siasync "github.com/NebulousLabs/Sia/sync"
)

var (
	errNoPeers     = errors.New("no peers")
	errUnreachable = errors.New("peer did not respond to ping")
)

// Gateway implements the modules.Gateway interface.
type Gateway struct {
	listener net.Listener
	myAddr   modules.NetAddress
	port     string

	// handlers are the RPCs that the Gateway can handle.
	//
	// initRPCs are the RPCs that the Gateway calls upon connecting to a peer.
	handlers map[rpcID]modules.RPCFunc
	initRPCs map[string]modules.RPCFunc

	// nodes is the set of all known nodes (i.e. potential peers).
	//
	// peers are the nodes that the gateway is currently connected to.
	//
	// peerTG is a special thread group for tracking peer connections, and will
	// block shutdown until all peer connections have been closed out. The peer
	// connections are put in a separate TG because of their unique
	// requirements - they have the potential to live for the lifetime of the
	// program, but also the potential to close early. Calling threads.OnStop
	// for each peer could create a huge backlog of functions that do nothing
	// (because most of the peers disconnected prior to shutdown). And they
	// can't call threads.Add because they are potentially very long running
	// and would block any threads.Flush() calls. So a second threadgroup is
	// added which handles clean-shutdown for the peers, without blocking
	// threads.Flush() calls.
	nodes  map[modules.NetAddress]struct{}
	peers  map[modules.NetAddress]*peer
	peerTG siasync.ThreadGroup

	// Utilities.
	log        *persist.Logger
	mu         sync.RWMutex
	persistDir string
	threads    siasync.ThreadGroup
}

// Address returns the NetAddress of the Gateway.
func (g *Gateway) Address() modules.NetAddress {
	g.mu.RLock()
	defer g.mu.RUnlock()
	return g.myAddr
}

// Close saves the state of the Gateway and stops its listener process.
func (g *Gateway) Close() error {
	if err := g.threads.Stop(); err != nil {
		return err
	}
	g.mu.RLock()
	defer g.mu.RUnlock()
	return g.saveSync()
}

// New returns an initialized Gateway.
func New(addr string, bootstrap bool, persistDir string) (*Gateway, error) {
	// Create the directory if it doesn't exist.
	err := os.MkdirAll(persistDir, 0700)
	if err != nil {
		return nil, err
	}

	g := &Gateway{
		handlers: make(map[rpcID]modules.RPCFunc),
		initRPCs: make(map[string]modules.RPCFunc),

		peers: make(map[modules.NetAddress]*peer),
		nodes: make(map[modules.NetAddress]struct{}),

		persistDir: persistDir,
	}

	// Create the logger.
	g.log, err = persist.NewFileLogger(filepath.Join(g.persistDir, logFile))
	if err != nil {
		return nil, err
	}
	// Establish the closing of the logger.
	g.threads.AfterStop(func() {
		if err := g.log.Close(); err != nil {
			// The logger may or may not be working here, so use a println
			// instead.
			fmt.Println("Failed to close the gateway logger:", err)
		}
	})

	// Establish that the peerTG must complete shutdown before the primary
	// thread group completes shutdown.
	g.threads.OnStop(func() {
		err = g.peerTG.Stop()
		if err != nil {
			g.log.Println("ERROR: peerTG experienced errors while shutting down:", err)
		}
	})

	// Register RPCs.
	g.RegisterRPC("ShareNodes", g.shareNodes)
	g.RegisterConnectCall("ShareNodes", g.requestNodes)
	// Establish the de-registration of the RPCs.
	g.threads.OnStop(func() {
		g.UnregisterRPC("ShareNodes")
		g.UnregisterConnectCall("ShareNodes")
	})

	// Load the old node list. If it doesn't exist, no problem, but if it does,
	// we want to know about any errors preventing us from loading it.
	if loadErr := g.load(); loadErr != nil && !os.IsNotExist(loadErr) {
		return nil, loadErr
	}

	// Add the bootstrap peers to the node list.
	if bootstrap {
		for _, addr := range modules.BootstrapPeers {
			err := g.addNode(addr)
			if err != nil && err != errNodeExists {
				g.log.Printf("WARN: failed to add the bootstrap node '%v': %v", addr, err)
			}
		}
	}

	// Create the listener which will listen for new connections from peers.
	permanentListenClosedChan := make(chan struct{})
	g.listener, err = net.Listen("tcp", addr)
	if err != nil {
		return nil, err
	}
	// Automatically close the listener when g.threads.Stop() is called.
	g.threads.OnStop(func() {
		err := g.listener.Close()
		if err != nil {
			g.log.Println("WARN: closing the listener failed:", err)
		}
		<-permanentListenClosedChan
	})
	// Set the address and port of the gateway.
	_, g.port, err = net.SplitHostPort(g.listener.Addr().String())
	if err != nil {
		return nil, err
	}
	// Set myAddr equal to the address returned by the listener. It will be
	// overwritten by threadedLearnHostname later on.
	g.myAddr = modules.NetAddress(g.listener.Addr().String())

	// Spawn the peer connection listener.
	go g.permanentListen(permanentListenClosedChan)

	// Spawn the peer manager and provide tools for ensuring clean shutdown.
	peerManagerClosedChan := make(chan struct{})
	g.threads.OnStop(func() {
		<-peerManagerClosedChan
	})
	go g.permanentPeerManager(peerManagerClosedChan)

	// Spawn the node manager and provide tools for ensuring clean shudown.
	nodeManagerClosedChan := make(chan struct{})
	g.threads.OnStop(func() {
		<-nodeManagerClosedChan
	})
	go g.permanentNodeManager(nodeManagerClosedChan)

	// Spawn the node purger and provide tools for ensuring clean shutdown.
	nodePurgerClosedChan := make(chan struct{})
	g.threads.OnStop(func() {
		<-nodePurgerClosedChan
	})
	go g.permanentNodePurger(nodePurgerClosedChan)

	// Spawn threads to take care of port forwarding and hostname discovery.
	go g.threadedForwardPort(g.port)
	go g.threadedLearnHostname()

	g.log.Println("INFO: gateway created, started logging")
	return g, nil
}

// enforce that Gateway satisfies the modules.Gateway interface
var _ modules.Gateway = (*Gateway)(nil)
<commit_msg>document the gateway and create a TODO list<commit_after>// Package gateway connects a Sia node to the Sia flood network. The flood
// network is used to propagate blocks and transactions. The gateway is the
// primary avenue that a node uses to hear about transactions and blocks, and
// is the primary avenue used to tell the network about blocks that you have
// mined or about transactions that you have created.
package gateway

// For the user to be securely connected to the network, the user must be
// connected to at least one node which will send them all of the blocks. An
// attacker can trick the user into thinking that a different blockchain is the
// full blockchain if the user is not connected to any nodes who are seeing +
// broadcasting the real chain (and instead is connected only to attacker nodes
// or to nodes that are not broadcasting). This situation is called an eclipse
// attack.
//
// Connecting to a large number of nodes increases the resiliancy of the
// network, but also puts a networking burden on the nodes and can slow down
// block propagation or increase orphan rates. The gateway's job is to keep the
// network efficient while also protecting the user against attacks.
//
// The gateway keeps a list of nodes that it knows about. It uses this list to
// form connections with other nodes, and then uses those connections to
// participate in the flood network. The primary vector for an attacker to
// achieve an eclipse attack is node list domination. If a gateway's nodelist
// is heavily dominated by attacking nodes, then when the gateway chooses to
// make random connections the gateway is at risk of selecting only attacker
// nodes.
//
// The gateway defends itself from these attacks by minimizing the amount of
// control that an attacker has over the node list and peer list. The first
// major defense is that the gateway maintais 8 'outbound' relationships, which
// means that the gateway created those relationships instead of an attacker.
// If a node forms a connection to you, that node is called 'inbound', and
// because it may be an attacker node, it is not trusted. Outbound nodes can
// also be attacker nodes, but they are less likely to be attacker nodes
// because you chose them, instead of them choosing you.
//
// If the gateway forms too many connections, the gateway will allow incoming
// connections by kicking an existing peer. But, to limit the amount of control
// that an attacker may have, only inbound peers are selected to be kicked.
// Furthermore, to increase the difficulty of attack, if a new inbound
// connection shares the same IP address as an existing connection, the shared
// connection is the connection that gets dropped (unless that connection is an
// inbound connection).
//
// Nodes are added to a peerlist in two methods. The first method is that a
// gateway will ask its outbound peers for a list of nodes. If the node list is
// below a certain size (see consts.go), the gateway will repeatedly ask
// outbound peers to expand the list. Nodes are also added to the nodelist
// after they successfully form a connection with the gateway. To limit the
// attacker's ability to add nodes to the nodelist, connections are
// ratelimited. An attacker with lots of IP addresses still has the ability to
// fill up the nodelist, however getting 90% dominance of the nodelist requries
// forming thousands of connections, which will take hours or days. By that
// time, the attacked node should already have its set of outbound peers,
// limiting the amount of damage that the attacker can do.
//
// To limit DNS-based tomfoolry, nodes are only added to the nodelist if their
// connection information takes the form of an IP address.
//
// Some research has been done on Bitcoin's flood networks. The more relevant
// research has been listed below. The papers listed first are more relevant.
//     Eclipse Attacks on Bitcoin's Peer-to-Peer Network (Heilman, Kendler, Zohar, Goldberg)
//     Stubborn Mining: Generalizing Selfish Mining and Combining with an Eclipse Attack (Nayak, Kumar, Miller, Shi)
//     An Overview of BGP Hijacking (https://www.bishopfox.com/blog/2015/08/an-overview-of-bgp-hijacking/)

// TODO: Currently the gateway does not do much in terms of bucketing. The
// gateway should make sure that it has outbound peers from a wide range of IP
// addresses, and when kicking inbound peers it shouldn't just favor kicking
// peers of the same IP address, it should favor kicking peers of the same ip
// address range.
//
// TODO: Currently the gateway does not save a list of its outbound
// connections. When it restarts, it will have a full nodelist (which may be
// primarily attacker nodes) and it will be connecting primarily to nodes in
// the nodelist. Instead, it should start by trying to connect to peers that
// have previously been outbound peers, as it is less likely that those have
// been manipulated.
//
// TODO: When peers connect to eachother, and when they add nodes to the node
// list, there is no verification that the peers are running on the same Sia
// network, something that will be problematic if we set up a large testnet.
// It's already problematic, and currently only solved by using a different set
// of bootstrap nodes. If there is any cross-polination (which an attacker
// could do pretty easily), the gateways will not clean up over time, which
// will degrade the quality of the flood network as the two networks will
// continously flood eachother with irrelevant information. Additionally, there
// is no public key exhcange, so communications cannot be effectively encrypted
// or authenticated. The nodes must have some way to share keys.
//
// TODO: There is currently no way for a user to choose the bootstrap nodes
// that the gateway uses, short of modifying the source code. A method for
// manipulating the bootstrap node list (before startup) is desired.
//
// TODO: Gateway hostname discovery currently has significant centralization,
// namely the fallback is a single third-party website that can easily form any
// response it wants. Instead, multiple TLS-protected third party websites
// should be used, and the plurality answer should be accepted as the true
// hostname.
//
// TODO: The gateway currently does hostname discovery in a non-blocking way,
// which means that the first few peers that it connects to may not get the
// correct hostname. This means that you may give the remote peer the wrong
// hostname, which means they will not be able to dial you back, which means
// they will not add you to their node list.
//
// TODO: The gateway should encrypt and authenticate all communications. Though
// the gateway participates in a flood network, practical attacks have been
// demonstrated which have been able to confuse nodes by manipulating messages
// from their peers. Encryption + authentication would have made the attack
// more difficult.

import (
	"errors"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"sync"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/persist"
	siasync "github.com/NebulousLabs/Sia/sync"
)

var (
	errNoPeers     = errors.New("no peers")
	errUnreachable = errors.New("peer did not respond to ping")
)

// Gateway implements the modules.Gateway interface.
type Gateway struct {
	listener net.Listener
	myAddr   modules.NetAddress
	port     string

	// handlers are the RPCs that the Gateway can handle.
	//
	// initRPCs are the RPCs that the Gateway calls upon connecting to a peer.
	handlers map[rpcID]modules.RPCFunc
	initRPCs map[string]modules.RPCFunc

	// nodes is the set of all known nodes (i.e. potential peers).
	//
	// peers are the nodes that the gateway is currently connected to.
	//
	// peerTG is a special thread group for tracking peer connections, and will
	// block shutdown until all peer connections have been closed out. The peer
	// connections are put in a separate TG because of their unique
	// requirements - they have the potential to live for the lifetime of the
	// program, but also the potential to close early. Calling threads.OnStop
	// for each peer could create a huge backlog of functions that do nothing
	// (because most of the peers disconnected prior to shutdown). And they
	// can't call threads.Add because they are potentially very long running
	// and would block any threads.Flush() calls. So a second threadgroup is
	// added which handles clean-shutdown for the peers, without blocking
	// threads.Flush() calls.
	nodes  map[modules.NetAddress]struct{}
	peers  map[modules.NetAddress]*peer
	peerTG siasync.ThreadGroup

	// Utilities.
	log        *persist.Logger
	mu         sync.RWMutex
	persistDir string
	threads    siasync.ThreadGroup
}

// Address returns the NetAddress of the Gateway.
func (g *Gateway) Address() modules.NetAddress {
	g.mu.RLock()
	defer g.mu.RUnlock()
	return g.myAddr
}

// Close saves the state of the Gateway and stops its listener process.
func (g *Gateway) Close() error {
	if err := g.threads.Stop(); err != nil {
		return err
	}
	g.mu.RLock()
	defer g.mu.RUnlock()
	return g.saveSync()
}

// New returns an initialized Gateway.
func New(addr string, bootstrap bool, persistDir string) (*Gateway, error) {
	// Create the directory if it doesn't exist.
	err := os.MkdirAll(persistDir, 0700)
	if err != nil {
		return nil, err
	}

	g := &Gateway{
		handlers: make(map[rpcID]modules.RPCFunc),
		initRPCs: make(map[string]modules.RPCFunc),

		peers: make(map[modules.NetAddress]*peer),
		nodes: make(map[modules.NetAddress]struct{}),

		persistDir: persistDir,
	}

	// Create the logger.
	g.log, err = persist.NewFileLogger(filepath.Join(g.persistDir, logFile))
	if err != nil {
		return nil, err
	}
	// Establish the closing of the logger.
	g.threads.AfterStop(func() {
		if err := g.log.Close(); err != nil {
			// The logger may or may not be working here, so use a println
			// instead.
			fmt.Println("Failed to close the gateway logger:", err)
		}
	})

	// Establish that the peerTG must complete shutdown before the primary
	// thread group completes shutdown.
	g.threads.OnStop(func() {
		err = g.peerTG.Stop()
		if err != nil {
			g.log.Println("ERROR: peerTG experienced errors while shutting down:", err)
		}
	})

	// Register RPCs.
	g.RegisterRPC("ShareNodes", g.shareNodes)
	g.RegisterConnectCall("ShareNodes", g.requestNodes)
	// Establish the de-registration of the RPCs.
	g.threads.OnStop(func() {
		g.UnregisterRPC("ShareNodes")
		g.UnregisterConnectCall("ShareNodes")
	})

	// Load the old node list. If it doesn't exist, no problem, but if it does,
	// we want to know about any errors preventing us from loading it.
	if loadErr := g.load(); loadErr != nil && !os.IsNotExist(loadErr) {
		return nil, loadErr
	}

	// Add the bootstrap peers to the node list.
	if bootstrap {
		for _, addr := range modules.BootstrapPeers {
			err := g.addNode(addr)
			if err != nil && err != errNodeExists {
				g.log.Printf("WARN: failed to add the bootstrap node '%v': %v", addr, err)
			}
		}
	}

	// Create the listener which will listen for new connections from peers.
	permanentListenClosedChan := make(chan struct{})
	g.listener, err = net.Listen("tcp", addr)
	if err != nil {
		return nil, err
	}
	// Automatically close the listener when g.threads.Stop() is called.
	g.threads.OnStop(func() {
		err := g.listener.Close()
		if err != nil {
			g.log.Println("WARN: closing the listener failed:", err)
		}
		<-permanentListenClosedChan
	})
	// Set the address and port of the gateway.
	_, g.port, err = net.SplitHostPort(g.listener.Addr().String())
	if err != nil {
		return nil, err
	}
	// Set myAddr equal to the address returned by the listener. It will be
	// overwritten by threadedLearnHostname later on.
	g.myAddr = modules.NetAddress(g.listener.Addr().String())

	// Spawn the peer connection listener.
	go g.permanentListen(permanentListenClosedChan)

	// Spawn the peer manager and provide tools for ensuring clean shutdown.
	peerManagerClosedChan := make(chan struct{})
	g.threads.OnStop(func() {
		<-peerManagerClosedChan
	})
	go g.permanentPeerManager(peerManagerClosedChan)

	// Spawn the node manager and provide tools for ensuring clean shudown.
	nodeManagerClosedChan := make(chan struct{})
	g.threads.OnStop(func() {
		<-nodeManagerClosedChan
	})
	go g.permanentNodeManager(nodeManagerClosedChan)

	// Spawn the node purger and provide tools for ensuring clean shutdown.
	nodePurgerClosedChan := make(chan struct{})
	g.threads.OnStop(func() {
		<-nodePurgerClosedChan
	})
	go g.permanentNodePurger(nodePurgerClosedChan)

	// Spawn threads to take care of port forwarding and hostname discovery.
	go g.threadedForwardPort(g.port)
	go g.threadedLearnHostname()

	g.log.Println("INFO: gateway created, started logging")
	return g, nil
}

// enforce that Gateway satisfies the modules.Gateway interface
var _ modules.Gateway = (*Gateway)(nil)
<|endoftext|>


<commit_before>package db

import (
	"regexp"
	"strconv"
	"time"
)

var cTypeDigit = regexp.MustCompile("^[0-9]+$")

type Member struct {
	ID      int    `sql:"bigint(20) NOT NULL AUTO_INCREMENT"`
	Slack   string `gorm:"type:varchar(191);not null;default:'';unique_index"`
	Xbl     string `gorm:"type:varchar(191);not null;default:'';index"`
	Psn     string `gorm:"type:varchar(191);not null;default:''"`
	Destiny string `gorm:"type:varchar(191);not null;default:''"`
	Seen    int    `gorm:"type:bigint;not null;index;default:0"`
	Name    string `gorm:"type:varchar(191);not null;default:''"`
	TZ      string `gorm:"type:varchar(191);not null;default:''"`
	db      *DB    `gorm:"-"`
}

func (m *Member) Save() error {
	rval := m.db.Save(m)
	return rval.Error
}

func (d *DB) MemberByAny(some string) (*Member, error) {
	if cTypeDigit.MatchString(some) {
		i, err := strconv.Atoi(some)
		if err != nil {
			return nil, err
		}
		return d.MemberByID(i)
	}
	if member, _ := d.MemberBySlackID(some); member != nil {
		return member, nil
	}
	return d.MemberByName(some)
}

func (d *DB) MemberByID(id int) (*Member, error) {
	m := new(Member)
	err := d.First(&m, id).Error
	m.db = d
	return m, err
}

func (d *DB) MemberByName(name string) (*Member, error) {
	m := new(Member)
	err := d.DB.Where("name = ?", name).First(&m).Error
	m.db = d
	return m, err
}

func (d *DB) MemberBySlackID(id string) (*Member, error) {
	m := new(Member)
	err := d.DB.Where("slack = ?", id).First(&m).Error
	m.db = d
	return m, err
}

func (d *DB) MembersActive(since time.Time) ([]*Member, error) {
	m := []*Member{}
	err := d.DB.Where("seen >= ?", since.Unix()).Find(&m).Error
	i.db = d
	return m, err
}

func (d *DB) Members() ([]*Member, error) {
	m := []*Member{}
	err := d.Find(&m).Error
	for _, i := range m {
		i.db = d
	}
	return m, err
}
<commit_msg>whoops<commit_after>package db

import (
	"regexp"
	"strconv"
	"time"
)

var cTypeDigit = regexp.MustCompile("^[0-9]+$")

type Member struct {
	ID      int    `sql:"bigint(20) NOT NULL AUTO_INCREMENT"`
	Slack   string `gorm:"type:varchar(191);not null;default:'';unique_index"`
	Xbl     string `gorm:"type:varchar(191);not null;default:'';index"`
	Psn     string `gorm:"type:varchar(191);not null;default:''"`
	Destiny string `gorm:"type:varchar(191);not null;default:''"`
	Seen    int    `gorm:"type:bigint;not null;index;default:0"`
	Name    string `gorm:"type:varchar(191);not null;default:''"`
	TZ      string `gorm:"type:varchar(191);not null;default:''"`
	db      *DB    `gorm:"-"`
}

func (m *Member) Save() error {
	rval := m.db.Save(m)
	return rval.Error
}

func (d *DB) MemberByAny(some string) (*Member, error) {
	if cTypeDigit.MatchString(some) {
		i, err := strconv.Atoi(some)
		if err != nil {
			return nil, err
		}
		return d.MemberByID(i)
	}
	if member, _ := d.MemberBySlackID(some); member != nil {
		return member, nil
	}
	return d.MemberByName(some)
}

func (d *DB) MemberByID(id int) (*Member, error) {
	m := new(Member)
	err := d.First(&m, id).Error
	m.db = d
	return m, err
}

func (d *DB) MemberByName(name string) (*Member, error) {
	m := new(Member)
	err := d.DB.Where("name = ?", name).First(&m).Error
	m.db = d
	return m, err
}

func (d *DB) MemberBySlackID(id string) (*Member, error) {
	m := new(Member)
	err := d.DB.Where("slack = ?", id).First(&m).Error
	m.db = d
	return m, err
}

func (d *DB) MembersActive(since time.Time) ([]*Member, error) {
	m := []*Member{}
	err := d.DB.Where("seen >= ?", since.Unix()).Find(&m).Error
	for _, i := range m {
		i.db = d
	}
	return m, err
}

func (d *DB) Members() ([]*Member, error) {
	m := []*Member{}
	err := d.Find(&m).Error
	for _, i := range m {
		i.db = d
	}
	return m, err
}
<|endoftext|>


<commit_before>package db

import (
	"errors"
	"github.com/snickers/snickers/types"
	"sync"
)

// Database struct that persists configurations
type memoryDatabase struct {
	mtx sync.RWMutex

	presets map[string]types.Preset
	jobs    map[string]types.Job
}

var databaseInit sync.Once
var memoryInstance *memoryDatabase

// GetDatabase returns database singleton
func getMemoryDatabase() (Storage, error) {
	databaseInit.Do(func() {
		memoryInstance = &memoryDatabase{}
		memoryInstance.presets = map[string]types.Preset{}
		memoryInstance.jobs = map[string]types.Job{}
	})

	return memoryInstance, nil
}

// ClearDatabase clears the database
func (r *memoryDatabase) ClearDatabase() error {
	r.mtx.Lock()
	defer r.mtx.Unlock()

	memoryInstance.presets = map[string]types.Preset{}
	memoryInstance.jobs = map[string]types.Job{}
	return nil
}

// StorePreset stores preset information
func (r *memoryDatabase) StorePreset(preset types.Preset) (types.Preset, error) {
	r.mtx.Lock()
	defer r.mtx.Unlock()

	if _, ok := r.presets[preset.Name]; ok {
		return types.Preset{}, errors.New("Error 409: Preset already exists, please update instead.")
	}

	r.presets[preset.Name] = preset
	return preset, nil
}

// RetrievePreset retrieves one preset from the database
func (r *memoryDatabase) RetrievePreset(presetName string) (types.Preset, error) {
	r.mtx.RLock()
	defer r.mtx.RUnlock()

	if val, ok := r.presets[presetName]; ok {
		return val, nil
	}
	return types.Preset{}, errors.New("preset not found")
}

// UpdatePreset updates a preset
func (r *memoryDatabase) UpdatePreset(presetName string, newPreset types.Preset) (types.Preset, error) {
	r.mtx.Lock()
	defer r.mtx.Unlock()

	r.presets[presetName] = newPreset
	return newPreset, nil
}

// GetPresets retrieves all presets of the database
func (r *memoryDatabase) GetPresets() ([]types.Preset, error) {
	r.mtx.RLock()
	defer r.mtx.RUnlock()

	res := make([]types.Preset, 0, len(r.presets))
	for _, value := range r.presets {
		res = append(res, value)
	}
	return res, nil
}

// DeletePreset deletes a preset from the database
func (r *memoryDatabase) DeletePreset(presetName string) (types.Preset, error) {
	r.mtx.Lock()
	defer r.mtx.Unlock()

	if val, ok := r.presets[presetName]; ok {
		delete(r.presets, presetName)
		return val, nil
	}
	return types.Preset{}, errors.New("preset not found")
}

// StoreJob stores job information
func (r *memoryDatabase) StoreJob(job types.Job) (types.Job, error) {
	r.mtx.Lock()
	defer r.mtx.Unlock()

	r.jobs[job.ID] = job
	return job, nil
}

// RetrieveJob retrieves one job from the database
func (r *memoryDatabase) RetrieveJob(jobID string) (types.Job, error) {
	r.mtx.RLock()
	defer r.mtx.RUnlock()

	if val, ok := r.jobs[jobID]; ok {
		return val, nil
	}
	return types.Job{}, errors.New("job not found")
}

// UpdateJob updates a job
func (r *memoryDatabase) UpdateJob(jobID string, newJob types.Job) (types.Job, error) {
	r.mtx.Lock()
	defer r.mtx.Unlock()

	r.jobs[jobID] = newJob
	return newJob, nil
}

//GetJobs retrieves all jobs of the database
func (r *memoryDatabase) GetJobs() ([]types.Job, error) {
	r.mtx.RLock()
	defer r.mtx.RUnlock()

	res := make([]types.Job, 0, len(r.jobs))
	for _, value := range r.jobs {
		res = append(res, value)
	}
	return res, nil
}
<commit_msg>Now using RetrievePreset method, moved locks so it will work<commit_after>package db

import (
	"errors"
	"github.com/snickers/snickers/types"
	"sync"
)

// Database struct that persists configurations
type memoryDatabase struct {
	mtx sync.RWMutex

	presets map[string]types.Preset
	jobs    map[string]types.Job
}

var databaseInit sync.Once
var memoryInstance *memoryDatabase

// GetDatabase returns database singleton
func getMemoryDatabase() (Storage, error) {
	databaseInit.Do(func() {
		memoryInstance = &memoryDatabase{}
		memoryInstance.presets = map[string]types.Preset{}
		memoryInstance.jobs = map[string]types.Job{}
	})

	return memoryInstance, nil
}

// ClearDatabase clears the database
func (r *memoryDatabase) ClearDatabase() error {
	r.mtx.Lock()
	defer r.mtx.Unlock()

	memoryInstance.presets = map[string]types.Preset{}
	memoryInstance.jobs = map[string]types.Job{}
	return nil
}

// StorePreset stores preset information
func (r *memoryDatabase) StorePreset(preset types.Preset) (types.Preset, error) {

	//prevent replacing existing preset
	if _, err := r.RetrievePreset(preset.Name); err == nil {
		return types.Preset{}, errors.New("Error 409: Preset already exists, please update instead.")
	}

	r.mtx.Lock()
	defer r.mtx.Unlock()

	r.presets[preset.Name] = preset
	return preset, nil
}

// RetrievePreset retrieves one preset from the database
func (r *memoryDatabase) RetrievePreset(presetName string) (types.Preset, error) {
	r.mtx.RLock()
	defer r.mtx.RUnlock()

	if val, ok := r.presets[presetName]; ok {
		return val, nil
	}
	return types.Preset{}, errors.New("preset not found")
}

// UpdatePreset updates a preset
func (r *memoryDatabase) UpdatePreset(presetName string, newPreset types.Preset) (types.Preset, error) {
	r.mtx.Lock()
	defer r.mtx.Unlock()

	r.presets[presetName] = newPreset
	return newPreset, nil
}

// GetPresets retrieves all presets of the database
func (r *memoryDatabase) GetPresets() ([]types.Preset, error) {
	r.mtx.RLock()
	defer r.mtx.RUnlock()

	res := make([]types.Preset, 0, len(r.presets))
	for _, value := range r.presets {
		res = append(res, value)
	}
	return res, nil
}

// DeletePreset deletes a preset from the database
func (r *memoryDatabase) DeletePreset(presetName string) (types.Preset, error) {
	r.mtx.Lock()
	defer r.mtx.Unlock()

	if val, ok := r.presets[presetName]; ok {
		delete(r.presets, presetName)
		return val, nil
	}
	return types.Preset{}, errors.New("preset not found")
}

// StoreJob stores job information
func (r *memoryDatabase) StoreJob(job types.Job) (types.Job, error) {
	r.mtx.Lock()
	defer r.mtx.Unlock()

	r.jobs[job.ID] = job
	return job, nil
}

// RetrieveJob retrieves one job from the database
func (r *memoryDatabase) RetrieveJob(jobID string) (types.Job, error) {
	r.mtx.RLock()
	defer r.mtx.RUnlock()

	if val, ok := r.jobs[jobID]; ok {
		return val, nil
	}
	return types.Job{}, errors.New("job not found")
}

// UpdateJob updates a job
func (r *memoryDatabase) UpdateJob(jobID string, newJob types.Job) (types.Job, error) {
	r.mtx.Lock()
	defer r.mtx.Unlock()

	r.jobs[jobID] = newJob
	return newJob, nil
}

//GetJobs retrieves all jobs of the database
func (r *memoryDatabase) GetJobs() ([]types.Job, error) {
	r.mtx.RLock()
	defer r.mtx.RUnlock()

	res := make([]types.Job, 0, len(r.jobs))
	for _, value := range r.jobs {
		res = append(res, value)
	}
	return res, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package context_test

import (
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/process"
	"github.com/juju/juju/process/context"
	"github.com/juju/juju/worker/uniter/runner/jujuc"
)

type registerSuite struct {
	commandSuite

	registerCmd *context.RegisterCommand
}

var _ = gc.Suite(&registerSuite{})

func (s *registerSuite) SetUpTest(c *gc.C) {
	s.commandSuite.SetUpTest(c)

	registerCmd := context.NewRegisterCommand(s.Ctx.Context)
	s.cmdName = "register" + jujuc.CmdSuffix
	s.cmd = registerCmd
	s.registerCmd = registerCmd
}

func (s *registerSuite) init(c *gc.C, name, id, status string) {
	s.registerCmd.Init([]string{s.proc.Name, "abc123"})
	//s.registerCmd.Details = ...
	//s.registerCmd.Space = ...
	//s.registerCmd.Env = ...
}

func (s *registerSuite) TestCommandRegistered(c *gc.C) {
	s.checkCommandRegistered(c)
}

func (s *registerSuite) TestHelp(c *gc.C) {
	s.checkHelp(c, `
usage: register [options] <name> <id> [<details>]
purpose: register a workload process

options:
--env  (= )
    environment variables
--space (= "")
    network space

"register" is used while a hook is running to let Juju know that
a workload process has been manually started. The information used
to start the process must be provided when "register" is run.

The process name must correspond to one of the processes defined in
the charm's metadata.yaml.
`[1:])
}

func (s *registerSuite) TestInitAllArgs(c *gc.C) {
	err := s.registerCmd.Init([]string{
		s.proc.Name,
		"abc123",
		`{"id":"abc123", "status":"okay"}`,
	})
	c.Assert(err, jc.ErrorIsNil)

	c.Check(s.registerCmd.Name, gc.Equals, s.proc.Name)
	c.Check(s.registerCmd.Id, gc.Equals, "abc123")
	c.Check(s.registerCmd.Details, jc.DeepEquals, process.LaunchDetails{
		UniqueID: "abc123",
		Status:   "okay",
	})
	c.Check(s.registerCmd.Space, gc.Equals, "")
	c.Check(s.registerCmd.Env, gc.HasLen, 0)
}

func (s *registerSuite) TestInitMinArgs(c *gc.C) {
	err := s.registerCmd.Init([]string{
		s.proc.Name,
		"abc123",
	})
	c.Assert(err, jc.ErrorIsNil)

	c.Check(s.registerCmd.Name, gc.Equals, s.proc.Name)
	c.Check(s.registerCmd.Id, gc.Equals, "abc123")
	c.Check(s.registerCmd.Details, jc.DeepEquals, process.LaunchDetails{})
	c.Check(s.registerCmd.Space, gc.Equals, "")
	c.Check(s.registerCmd.Env, gc.HasLen, 0)
}

func (s *registerSuite) TestInitTooFewArgs(c *gc.C) {
	err := s.registerCmd.Init([]string{})
	c.Check(err, gc.ErrorMatches, "expected at least 2 args, got: .*")

	err = s.registerCmd.Init([]string{s.proc.Name})
	c.Check(err, gc.ErrorMatches, "expected at least 2 args, got: .*")
}

func (s *registerSuite) TestInitTooManyArgs(c *gc.C) {
	err := s.registerCmd.Init([]string{
		s.proc.Name,
		"abc123",
		`{"id":"abc123", "status":"okay"}`,
		"other",
	})

	c.Check(err, gc.ErrorMatches, "expected at most 3 args, got: .*")
}

func (s *registerSuite) TestInitEmptyName(c *gc.C) {
	err := s.registerCmd.Init([]string{
		"",
		"abc123",
	})

	c.Check(err, gc.ErrorMatches, "got empty name")
}

func (s *registerSuite) TestInitEmptyId(c *gc.C) {
	err := s.registerCmd.Init([]string{
		s.proc.Name,
		"",
	})

	c.Check(err, gc.ErrorMatches, "got empty id")
}

func (s *registerSuite) TestInitMissingDetailsId(c *gc.C) {
	err := s.registerCmd.Init([]string{
		s.proc.Name,
		"abc123",
		`{"status":"okay"}`,
	})

	c.Check(err, gc.ErrorMatches, "UniqueID must be set")
}

func (s *registerSuite) TestInitMissingDetailsStatus(c *gc.C) {
	err := s.registerCmd.Init([]string{
		s.proc.Name,
		"abc123",
		`{"id":"abc123"}`,
	})

	c.Check(err, gc.ErrorMatches, "Status must be set")
}

func (s *registerSuite) TestInitBadJSON(c *gc.C) {
	err := s.registerCmd.Init([]string{
		s.proc.Name,
		"abc123",
		`{"id":"abc123", "status":"okay"`,
	})

	c.Check(err, gc.ErrorMatches, "unexpected end of JSON input")
}

func (s *registerSuite) TestRun(c *gc.C) {
	s.init(c, s.proc.Name, "abc123", "running")

	s.checkRun(c, "", "")
	// TODO(ericsnow) Check the underlying calls here (s.Stub.CheckCallNames).
}
<commit_msg>Check the side effect calls in TestRun.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package context_test

import (
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/process"
	"github.com/juju/juju/process/context"
	"github.com/juju/juju/worker/uniter/runner/jujuc"
)

type registerSuite struct {
	commandSuite

	registerCmd *context.RegisterCommand
}

var _ = gc.Suite(&registerSuite{})

func (s *registerSuite) SetUpTest(c *gc.C) {
	s.commandSuite.SetUpTest(c)

	registerCmd := context.NewRegisterCommand(s.Ctx.Context)
	s.Stub.CheckCallNames(c, "Component")
	s.Stub.Calls = nil

	s.cmdName = "register" + jujuc.CmdSuffix
	s.cmd = registerCmd
	s.registerCmd = registerCmd
}

func (s *registerSuite) init(c *gc.C, name, id, status string) {
	s.registerCmd.Init([]string{s.proc.Name, "abc123"})
	//s.registerCmd.Details = ...
	//s.registerCmd.Space = ...
	//s.registerCmd.Env = ...
}

func (s *registerSuite) TestCommandRegistered(c *gc.C) {
	s.checkCommandRegistered(c)
}

func (s *registerSuite) TestHelp(c *gc.C) {
	s.checkHelp(c, `
usage: register [options] <name> <id> [<details>]
purpose: register a workload process

options:
--env  (= )
    environment variables
--space (= "")
    network space

"register" is used while a hook is running to let Juju know that
a workload process has been manually started. The information used
to start the process must be provided when "register" is run.

The process name must correspond to one of the processes defined in
the charm's metadata.yaml.
`[1:])
}

func (s *registerSuite) TestInitAllArgs(c *gc.C) {
	err := s.registerCmd.Init([]string{
		s.proc.Name,
		"abc123",
		`{"id":"abc123", "status":"okay"}`,
	})
	c.Assert(err, jc.ErrorIsNil)

	c.Check(s.registerCmd.Name, gc.Equals, s.proc.Name)
	c.Check(s.registerCmd.Id, gc.Equals, "abc123")
	c.Check(s.registerCmd.Details, jc.DeepEquals, process.LaunchDetails{
		UniqueID: "abc123",
		Status:   "okay",
	})
	c.Check(s.registerCmd.Space, gc.Equals, "")
	c.Check(s.registerCmd.Env, gc.HasLen, 0)
}

func (s *registerSuite) TestInitMinArgs(c *gc.C) {
	err := s.registerCmd.Init([]string{
		s.proc.Name,
		"abc123",
	})
	c.Assert(err, jc.ErrorIsNil)

	c.Check(s.registerCmd.Name, gc.Equals, s.proc.Name)
	c.Check(s.registerCmd.Id, gc.Equals, "abc123")
	c.Check(s.registerCmd.Details, jc.DeepEquals, process.LaunchDetails{})
	c.Check(s.registerCmd.Space, gc.Equals, "")
	c.Check(s.registerCmd.Env, gc.HasLen, 0)
}

func (s *registerSuite) TestInitTooFewArgs(c *gc.C) {
	err := s.registerCmd.Init([]string{})
	c.Check(err, gc.ErrorMatches, "expected at least 2 args, got: .*")

	err = s.registerCmd.Init([]string{s.proc.Name})
	c.Check(err, gc.ErrorMatches, "expected at least 2 args, got: .*")
}

func (s *registerSuite) TestInitTooManyArgs(c *gc.C) {
	err := s.registerCmd.Init([]string{
		s.proc.Name,
		"abc123",
		`{"id":"abc123", "status":"okay"}`,
		"other",
	})

	c.Check(err, gc.ErrorMatches, "expected at most 3 args, got: .*")
}

func (s *registerSuite) TestInitEmptyName(c *gc.C) {
	err := s.registerCmd.Init([]string{
		"",
		"abc123",
	})

	c.Check(err, gc.ErrorMatches, "got empty name")
}

func (s *registerSuite) TestInitEmptyId(c *gc.C) {
	err := s.registerCmd.Init([]string{
		s.proc.Name,
		"",
	})

	c.Check(err, gc.ErrorMatches, "got empty id")
}

func (s *registerSuite) TestInitMissingDetailsId(c *gc.C) {
	err := s.registerCmd.Init([]string{
		s.proc.Name,
		"abc123",
		`{"status":"okay"}`,
	})

	c.Check(err, gc.ErrorMatches, "UniqueID must be set")
}

func (s *registerSuite) TestInitMissingDetailsStatus(c *gc.C) {
	err := s.registerCmd.Init([]string{
		s.proc.Name,
		"abc123",
		`{"id":"abc123"}`,
	})

	c.Check(err, gc.ErrorMatches, "Status must be set")
}

func (s *registerSuite) TestInitBadJSON(c *gc.C) {
	err := s.registerCmd.Init([]string{
		s.proc.Name,
		"abc123",
		`{"id":"abc123", "status":"okay"`,
	})

	c.Check(err, gc.ErrorMatches, "unexpected end of JSON input")
}

func (s *registerSuite) TestRun(c *gc.C) {
	s.init(c, s.proc.Name, "abc123", "running")

	s.checkRun(c, "", "")
	s.Stub.CheckCallNames(c, "Set")
}
<|endoftext|>


<commit_before>package model_loader

import (
	"encoding/json"
	"fmt"
	valid "github.com/asaskevich/govalidator"
	"github.com/centurylinkcloud/clc-go-cli/parser"
	"reflect"
	"time"
)

const (
	timeFormat = "2006-01-02 15:04:05"
)

func LoadModel(parsedArgs map[string]interface{}, inputModel interface{}) error {
	metaModel := reflect.ValueOf(inputModel)
	if metaModel.Kind() != reflect.Ptr {
		return fmt.Errorf("Input model must be passed by pointer.")
	}
	for k, v := range parsedArgs {
		field, err := getFieldByName(metaModel, k)
		if err != nil {
			return err
		}
		err = loadValue(k, v, field)
		if err != nil {
			return err
		}
	}
	return nil
}

func loadValue(key string, arg interface{}, field reflect.Value) error {
	if arg == nil {
		return nil
	}
	switch field.Interface().(type) {
	case int64:
		if !valid.IsInt(arg.(string)) {
			return fmt.Errorf("Type mismatch: %s value must be integer.", key)
		} else {
			argInt, _ := valid.ToInt(arg.(string))
			field.SetInt(argInt)
			return nil
		}
	case float64:
		if !valid.IsFloat(arg.(string)) {
			return fmt.Errorf("Type mismatch: %s value must be float.", key)
		} else {
			argFloat, _ := valid.ToFloat(arg.(string))
			field.SetFloat(argFloat)
			return nil
		}
	case time.Time:
		if argTime, err := time.Parse(timeFormat, arg.(string)); err != nil {
			return fmt.Errorf("Type mismatch: %s value must be datetime in `YYYY-MM-DD hh:mm:ss` format.", key)
		} else {
			field.Set(reflect.ValueOf(argTime))
			return nil
		}
	case bool:
		if arg == "true" {
			field.SetBool(true)
		} else if arg == "false" {
			field.SetBool(false)
		} else {
			return fmt.Errorf("Type mismatch: %s value must be either true or false.", key)
		}
		return nil
	case string:
		field.SetString(arg.(string))
		return nil
	}
	if isStruct(field) {
		argStruct, err := parseStruct(arg)
		if err != nil {
			return err
		}
		for k, v := range argStruct {
			nestedField, err := getFieldByName(field.Addr(), k)
			if err != nil {
				return err
			}
			err = loadValue(k, v, nestedField)
			if err != nil {
				return err
			}
		}
		return nil
	} else if isSlice(field) {
		argSlice, err := parseSlice(arg)
		if err != nil {
			return err
		}
		for _, v := range argSlice {
			elementPtr := getEmplySliceType(field)
			err = loadValue(key, v, elementPtr.Elem())
			if err != nil {
				return err
			}
			field.Set(reflect.Append(field, elementPtr.Elem()))
		}
		return nil
	}
	return fmt.Errorf("Unsupported field type %s", field.Kind())
}

func getFieldByName(model reflect.Value, name string) (reflect.Value, error) {
	field := model.Elem().FieldByName(name)
	if !field.IsValid() {
		return reflect.ValueOf(nil), fmt.Errorf("Field `%s` does not exist.", name)
	}
	return field, nil
}

func isStruct(model reflect.Value) bool {
	return model.Kind() == reflect.Struct
}

func isSlice(model reflect.Value) bool {
	return model.Kind() == reflect.Slice
}

// Parses an object of type map[string]interface{} either from JSON or from a=b,c=d,.. notation.
// Also, calls NormalizeKeys with the parsed object.
// If arg is already of type map[string]interface{} returns it as is.
func parseStruct(arg interface{}) (map[string]interface{}, error) {
	if argMap, isMap := arg.(map[string]interface{}); isMap {
		return argMap, nil
	}
	parsed := make(map[string]interface{}, 0)
	if err := json.Unmarshal([]byte(arg.(string)), &parsed); err == nil {
		parser.NormalizeKeys(parsed)
		return parsed, nil
	}
	if parsed, err := parser.ParseObject(arg.(string)); err == nil {
		return parsed, nil
	}
	return nil, fmt.Errorf("`%s` is neither in JSON nor in key=value,.. format.", arg.(string))
}

// Parses an object of type []interface{} either from JSON.
// Also, calls NormalizeKeys with the parsed object.
// If arg is already of type []interface{} returns it as is.
func parseSlice(arg interface{}) ([]interface{}, error) {
	if argSlice, isSlice := arg.([]interface{}); isSlice {
		return argSlice, nil
	}
	parsed := make([]interface{}, 0)
	if err := json.Unmarshal([]byte(arg.(string)), &parsed); err == nil {
		parser.NormalizeKeys(parsed)
		return parsed, nil
	}
	return nil, fmt.Errorf("`%s` is neither in JSON nor in key=value,.. format.", arg.(string))
}

func getEmplySliceType(slice reflect.Value) reflect.Value {
	return reflect.New(slice.Type().Elem())
}
<commit_msg>fail gracefully when slices are specified for the simple type fields<commit_after>package model_loader

import (
	"encoding/json"
	"fmt"
	valid "github.com/asaskevich/govalidator"
	"github.com/centurylinkcloud/clc-go-cli/parser"
	"reflect"
	"time"
)

const (
	timeFormat = "2006-01-02 15:04:05"
)

func LoadModel(parsedArgs map[string]interface{}, inputModel interface{}) error {
	metaModel := reflect.ValueOf(inputModel)
	if metaModel.Kind() != reflect.Ptr {
		return fmt.Errorf("Input model must be passed by pointer.")
	}
	for k, v := range parsedArgs {
		field, err := getFieldByName(metaModel, k)
		if err != nil {
			return err
		}
		err = loadValue(k, v, field)
		if err != nil {
			return err
		}
	}
	return nil
}

func loadValue(key string, arg interface{}, field reflect.Value) error {
	if arg == nil {
		return nil
	}
	isArgSlice := reflect.ValueOf(arg).Kind() == reflect.Slice
	switch field.Interface().(type) {
	case int64:
		if isArgSlice || !valid.IsInt(arg.(string)) {
			return fmt.Errorf("Type mismatch: %s value must be integer.", key)
		} else {
			argInt, _ := valid.ToInt(arg.(string))
			field.SetInt(argInt)
			return nil
		}
	case float64:
		if isArgSlice || !valid.IsFloat(arg.(string)) {
			return fmt.Errorf("Type mismatch: %s value must be float.", key)
		} else {
			argFloat, _ := valid.ToFloat(arg.(string))
			field.SetFloat(argFloat)
			return nil
		}
	case time.Time:
		mismatch := isArgSlice
		var argTime time.Time
		var err error
		if !mismatch {
			if argTime, err = time.Parse(timeFormat, arg.(string)); err != nil {
				mismatch = true
			}
		}
		if mismatch {
			return fmt.Errorf("Type mismatch: %s value must be datetime in `YYYY-MM-DD hh:mm:ss` format.", key)
		} else {
			field.Set(reflect.ValueOf(argTime))
			return nil
		}
	case bool:
		if arg == "true" {
			field.SetBool(true)
		} else if arg == "false" {
			field.SetBool(false)
		} else {
			return fmt.Errorf("Type mismatch: %s value must be either true or false.", key)
		}
		return nil
	case string:
		if isArgSlice {
			return fmt.Errorf("Type mismatch: %s value must be string.", key)
		}
		field.SetString(arg.(string))
		return nil
	}
	if isStruct(field) {
		argStruct, err := parseStruct(arg)
		if err != nil {
			return err
		}
		for k, v := range argStruct {
			nestedField, err := getFieldByName(field.Addr(), k)
			if err != nil {
				return err
			}
			err = loadValue(k, v, nestedField)
			if err != nil {
				return err
			}
		}
		return nil
	} else if isSlice(field) {
		argSlice, err := parseSlice(arg)
		if err != nil {
			return err
		}
		for _, v := range argSlice {
			elementPtr := getEmplySliceType(field)
			err = loadValue(key, v, elementPtr.Elem())
			if err != nil {
				return err
			}
			field.Set(reflect.Append(field, elementPtr.Elem()))
		}
		return nil
	}
	return fmt.Errorf("Unsupported field type %s", field.Kind())
}

func getFieldByName(model reflect.Value, name string) (reflect.Value, error) {
	field := model.Elem().FieldByName(name)
	if !field.IsValid() {
		return reflect.ValueOf(nil), fmt.Errorf("Field `%s` does not exist.", name)
	}
	return field, nil
}

func isStruct(model reflect.Value) bool {
	return model.Kind() == reflect.Struct
}

func isSlice(model reflect.Value) bool {
	return model.Kind() == reflect.Slice
}

// Parses an object of type map[string]interface{} either from JSON or from a=b,c=d,.. notation.
// Also, calls NormalizeKeys with the parsed object.
// If arg is already of type map[string]interface{} returns it as is.
func parseStruct(arg interface{}) (map[string]interface{}, error) {
	if argMap, isMap := arg.(map[string]interface{}); isMap {
		return argMap, nil
	}
	parsed := make(map[string]interface{}, 0)
	if err := json.Unmarshal([]byte(arg.(string)), &parsed); err == nil {
		parser.NormalizeKeys(parsed)
		return parsed, nil
	}
	if parsed, err := parser.ParseObject(arg.(string)); err == nil {
		return parsed, nil
	}
	return nil, fmt.Errorf("`%s` is neither in JSON nor in key=value,.. format.", arg.(string))
}

// Parses an object of type []interface{} either from JSON.
// Also, calls NormalizeKeys with the parsed object.
// If arg is already of type []interface{} returns it as is.
func parseSlice(arg interface{}) ([]interface{}, error) {
	if argSlice, isSlice := arg.([]interface{}); isSlice {
		return argSlice, nil
	}
	parsed := make([]interface{}, 0)
	if err := json.Unmarshal([]byte(arg.(string)), &parsed); err == nil {
		parser.NormalizeKeys(parsed)
		return parsed, nil
	}
	return nil, fmt.Errorf("`%s` is neither in JSON nor in key=value,.. format.", arg.(string))
}

func getEmplySliceType(slice reflect.Value) reflect.Value {
	return reflect.New(slice.Type().Elem())
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/gin-gonic/gin"
	"golang.org/x/oauth2"
	"gopkg.in/yaml.v2"
)

type Consent struct {
	Challenge string `form:"challenge" binding:"required"`
}

func handleConsent(c *gin.Context) {
	var data Consent
	if c.Bind(&data) == nil {
	}

	c.JSON(200, gin.H{"challenge": data.Challenge})
}

func main() {
	r := gin.Default()
	r.GET("/", handleConsent)

	r.Run(":3000")
}
<commit_msg>Added handling OAuth2 credentials<commit_after>package main

import (
	_ "fmt"
	"github.com/gin-gonic/gin"
	"golang.org/x/net/context"
	_ "golang.org/x/oauth2"
	"golang.org/x/oauth2/clientcredentials"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
)

// Structure of the .hydra.yml file shared with Hydra
type Config struct {
	Port         int    `yaml:"port"`
	Issuer       string `yaml:"issuer"`
	ConsentURL   string `yaml:"consent_url"`
	ClusterURL   string `yaml:"cluster_url"`
	ClientID     string `yaml:"client_id"`
	ClientSecret string `yaml:"client_secret"`
}

// Form to parse when granting a consent
type Consent struct {
	Challenge string `form:"challenge" binding:"required"`
}

func handleConsent(c *gin.Context) {
	var data Consent
	if c.Bind(&data) == nil {
	}

	c.JSON(200, gin.H{"challenge": data.Challenge})
}

func main() {

	// IdP is has credentials preconfigured by hydra.
	// Let's parse the yaml file with that information
	var config Config
	source, err := ioutil.ReadFile("/root/.hydra.yml")
	if err != nil {
		panic(err)
	}
	err = yaml.Unmarshal(source, &config)
	if err != nil {
		panic(err)
	}

	// Use the credentials to login to hydra
	credentials := clientcredentials.Config{
		ClientID:     config.ClientID,
		ClientSecret: config.ClientSecret,
		TokenURL:     os.Getenv("HYDRA_URL") + "/oauth2/token",
		Scopes:       []string{"core", "hydra.keys.get"},
	}

	// client := credentials.Client(context.Background())
	_, err = credentials.Token(context.Background())
	if err != nil {
		panic(err)
	}

	r := gin.Default()
	r.GET("/", handleConsent)

	r.Run(":3000")
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "html/template"
    "io"
    "log"
    "net"
    "net/http"
    "regexp"
    "strings"
)

type Client struct {
    IP net.IP
    Port string
    Header http.Header
}

func isCli(userAgent string) bool {
    match, _ := regexp.MatchString("^(?i)(curl|wget|fetch\\slibfetch)\\/.*$",
        userAgent)
    return match
}

func parseRealIP(req *http.Request) (net.IP, string) {
    var host string
    var port string
    realIP := req.Header.Get("X-Real-IP")
    if realIP != "" {
        host = realIP
    } else {
        host, port, _ = net.SplitHostPort(req.RemoteAddr)
    }
    return net.ParseIP(host), port
}

func pathToKey(path string) string {
    re := regexp.MustCompile("^\\/|\\.json$")
    return re.ReplaceAllLiteralString(strings.ToLower(path), "")
}

func isJson(req *http.Request) bool {
    return strings.HasSuffix(req.URL.Path, ".json") ||
        strings.Contains(req.Header.Get("Accept"), "application/json")
}

func handler(w http.ResponseWriter, req *http.Request) {
    if req.Method != "GET" {
        http.Error(w, "Invalid request method", 405)
        return
    }

    ip, port := parseRealIP(req)
    header := pathToKey(req.URL.Path)

    if isCli(req.UserAgent()) {
        if header == "" || header == "ip" {
            io.WriteString(w, fmt.Sprintf("%s\n", ip))
        } else if header == "port" {
            io.WriteString(w, fmt.Sprintf("%s\n", port))
        } else {
            value := req.Header.Get(header)
            io.WriteString(w, fmt.Sprintf("%s\n", value))
        }
    } else {
        funcMap := template.FuncMap {
            "ToLower": strings.ToLower,
        }
        t, _ := template.
            New("index.html").
            Funcs(funcMap).
            ParseFiles("index.html")
        client := &Client{IP: ip, Port: port, Header: req.Header}
        t.Execute(w, client)
    }
}

func main() {
    http.Handle("/assets/", http.StripPrefix("/assets/",
        http.FileServer(http.Dir("assets/"))))
    http.HandleFunc("/", handler)
    err := http.ListenAndServe(":8080", nil)
    if err != nil {
        log.Fatal("ListenAndServe: ", err)
    }
}
<commit_msg>Formatting<commit_after>package main

import (
    "fmt"
    "html/template"
    "io"
    "log"
    "net"
    "net/http"
    "regexp"
    "strings"
)

type Client struct {
    IP     net.IP
    Port   string
    Header http.Header
}

func isCli(userAgent string) bool {
    match, _ := regexp.MatchString("^(?i)(curl|wget|fetch\\slibfetch)\\/.*$",
        userAgent)
    return match
}

func parseRealIP(req *http.Request) (net.IP, string) {
    var host string
    var port string
    realIP := req.Header.Get("X-Real-IP")
    if realIP != "" {
        host = realIP
    } else {
        host, port, _ = net.SplitHostPort(req.RemoteAddr)
    }
    return net.ParseIP(host), port
}

func pathToKey(path string) string {
    re := regexp.MustCompile("^\\/|\\.json$")
    return re.ReplaceAllLiteralString(strings.ToLower(path), "")
}

func isJson(req *http.Request) bool {
    return strings.HasSuffix(req.URL.Path, ".json") ||
        strings.Contains(req.Header.Get("Accept"), "application/json")
}

func handler(w http.ResponseWriter, req *http.Request) {
    if req.Method != "GET" {
        http.Error(w, "Invalid request method", 405)
        return
    }

    ip, port := parseRealIP(req)
    header := pathToKey(req.URL.Path)

    if isCli(req.UserAgent()) {
        if header == "" || header == "ip" {
            io.WriteString(w, fmt.Sprintf("%s\n", ip))
        } else if header == "port" {
            io.WriteString(w, fmt.Sprintf("%s\n", port))
        } else {
            value := req.Header.Get(header)
            io.WriteString(w, fmt.Sprintf("%s\n", value))
        }
    } else {
        funcMap := template.FuncMap{
            "ToLower": strings.ToLower,
        }
        t, _ := template.
            New("index.html").
            Funcs(funcMap).
            ParseFiles("index.html")
        client := &Client{IP: ip, Port: port, Header: req.Header}
        t.Execute(w, client)
    }
}

func main() {
    http.Handle("/assets/", http.StripPrefix("/assets/",
        http.FileServer(http.Dir("assets/"))))
    http.HandleFunc("/", handler)
    err := http.ListenAndServe(":8080", nil)
    if err != nil {
        log.Fatal("ListenAndServe: ", err)
    }
}
<|endoftext|>


<commit_before>// All of the methods used to communicate with the digital_ocean API
// are here. Their API is on a path to V2, so just plain JSON is used
// in place of a proper client library for now.

package digitalocean

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/mitchellh/mapstructure"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const DIGITALOCEAN_API_URL = "https://api.digitalocean.com"

type Image struct {
	Id           uint
	Name         string
	Distribution string
}

type ImagesResp struct {
	Images []Image
}

type DigitalOceanClient struct {
	// The http client for communicating
	client *http.Client

	// The base URL of the API
	BaseURL string

	// Credentials
	ClientID string
	APIKey   string
}

// Creates a new client for communicating with DO
func (d DigitalOceanClient) New(client string, key string) *DigitalOceanClient {
	c := &DigitalOceanClient{
		client: &http.Client{
			Transport: &http.Transport{
				Proxy: http.ProxyFromEnvironment,
			},
		},
		BaseURL:  DIGITALOCEAN_API_URL,
		ClientID: client,
		APIKey:   key,
	}
	return c
}

// Creates an SSH Key and returns it's id
func (d DigitalOceanClient) CreateKey(name string, pub string) (uint, error) {
	params := url.Values{}
	params.Set("name", name)
	params.Set("ssh_pub_key", pub)

	body, err := NewRequest(d, "ssh_keys/new", params)
	if err != nil {
		return 0, err
	}

	// Read the SSH key's ID we just created
	key := body["ssh_key"].(map[string]interface{})
	keyId := key["id"].(float64)
	return uint(keyId), nil
}

// Destroys an SSH key
func (d DigitalOceanClient) DestroyKey(id uint) error {
	path := fmt.Sprintf("ssh_keys/%v/destroy", id)
	_, err := NewRequest(d, path, url.Values{})
	return err
}

// Creates a droplet and returns it's id
func (d DigitalOceanClient) CreateDroplet(name string, size uint, image uint, region uint, keyId uint) (uint, error) {
	params := url.Values{}
	params.Set("name", name)
	params.Set("size_id", fmt.Sprintf("%v", size))
	params.Set("image_id", fmt.Sprintf("%v", image))
	params.Set("region_id", fmt.Sprintf("%v", region))
	params.Set("ssh_key_ids", fmt.Sprintf("%v", keyId))

	body, err := NewRequest(d, "droplets/new", params)
	if err != nil {
		return 0, err
	}

	// Read the Droplets ID
	droplet := body["droplet"].(map[string]interface{})
	dropletId := droplet["id"].(float64)
	return uint(dropletId), err
}

// Destroys a droplet
func (d DigitalOceanClient) DestroyDroplet(id uint) error {
	path := fmt.Sprintf("droplets/%v/destroy", id)
	_, err := NewRequest(d, path, url.Values{})
	return err
}

// Powers off a droplet
func (d DigitalOceanClient) PowerOffDroplet(id uint) error {
	path := fmt.Sprintf("droplets/%v/power_off", id)

	_, err := NewRequest(d, path, url.Values{})

	return err
}

// Shutsdown a droplet. This is a "soft" shutdown.
func (d DigitalOceanClient) ShutdownDroplet(id uint) error {
	path := fmt.Sprintf("droplets/%v/shutdown", id)

	_, err := NewRequest(d, path, url.Values{})

	return err
}

// Creates a snaphot of a droplet by it's ID
func (d DigitalOceanClient) CreateSnapshot(id uint, name string) error {
	path := fmt.Sprintf("droplets/%v/snapshot", id)

	params := url.Values{}
	params.Set("name", name)

	_, err := NewRequest(d, path, params)

	return err
}

// Returns all available images.
func (d DigitalOceanClient) Images() ([]Image, error) {
	resp, err := NewRequest(d, "images", url.Values{})
	if err != nil {
		return nil, err
	}

	var result ImagesResp
	if err := mapstructure.Decode(resp, &result); err != nil {
		return nil, err
	}

	return result.Images, nil
}

// Destroys an image by its ID.
func (d DigitalOceanClient) DestroyImage(id uint) error {
	path := fmt.Sprintf("images/%d/destroy", id)
	_, err := NewRequest(d, path, url.Values{})
	return err
}

// Returns DO's string representation of status "off" "new" "active" etc.
func (d DigitalOceanClient) DropletStatus(id uint) (string, string, error) {
	path := fmt.Sprintf("droplets/%v", id)

	body, err := NewRequest(d, path, url.Values{})
	if err != nil {
		return "", "", err
	}

	var ip string

	// Read the droplet's "status"
	droplet := body["droplet"].(map[string]interface{})
	status := droplet["status"].(string)

	if droplet["ip_address"] != nil {
		ip = droplet["ip_address"].(string)
	}

	return ip, status, err
}

// Sends an api request and returns a generic map[string]interface of
// the response.
func NewRequest(d DigitalOceanClient, path string, params url.Values) (map[string]interface{}, error) {
	client := d.client

	// Add the authentication parameters
	params.Set("client_id", d.ClientID)
	params.Set("api_key", d.APIKey)

	url := fmt.Sprintf("%s/%s?%s", DIGITALOCEAN_API_URL, path, params.Encode())

	// Do some basic scrubbing so sensitive information doesn't appear in logs
	scrubbedUrl := strings.Replace(url, d.ClientID, "CLIENT_ID", -1)
	scrubbedUrl = strings.Replace(scrubbedUrl, d.APIKey, "API_KEY", -1)
	log.Printf("sending new request to digitalocean: %s", scrubbedUrl)

	var lastErr error
	for attempts := 1; attempts < 5; attempts++ {
		resp, err := client.Get(url)
		if err != nil {
			return nil, err
		}

		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			return nil, err
		}

		log.Printf("response from digitalocean: %s", body)

		var decodedResponse map[string]interface{}
		err = json.Unmarshal(body, &decodedResponse)
		if err != nil {
			err = errors.New(fmt.Sprintf("Failed to decode JSON response (HTTP %v) from DigitalOcean: %s",
				resp.StatusCode, body))
			return decodedResponse, err
		}

		// Check for errors sent by digitalocean
		status := decodedResponse["status"].(string)
		if status == "OK" {
			return decodedResponse, nil
		}

		if status == "ERROR" {
			status = decodedResponse["error_message"].(string)
		}

		lastErr = errors.New(fmt.Sprintf("Received error from DigitalOcean (%d): %s",
			resp.StatusCode, status))
		log.Println(lastErr)
		if strings.Contains(status, "a pending event") {
			// Retry, DigitalOcean sends these dumb "pending event"
			// errors all the time.
			time.Sleep(5 * time.Second)
			continue
		}

		// Some other kind of error. Just return.
		return decodedResponse, lastErr
	}

	return nil, lastErr
}
<commit_msg>builder/digitalocean: retry pending events a lot more<commit_after>// All of the methods used to communicate with the digital_ocean API
// are here. Their API is on a path to V2, so just plain JSON is used
// in place of a proper client library for now.

package digitalocean

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/mitchellh/mapstructure"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const DIGITALOCEAN_API_URL = "https://api.digitalocean.com"

type Image struct {
	Id           uint
	Name         string
	Distribution string
}

type ImagesResp struct {
	Images []Image
}

type DigitalOceanClient struct {
	// The http client for communicating
	client *http.Client

	// The base URL of the API
	BaseURL string

	// Credentials
	ClientID string
	APIKey   string
}

// Creates a new client for communicating with DO
func (d DigitalOceanClient) New(client string, key string) *DigitalOceanClient {
	c := &DigitalOceanClient{
		client: &http.Client{
			Transport: &http.Transport{
				Proxy: http.ProxyFromEnvironment,
			},
		},
		BaseURL:  DIGITALOCEAN_API_URL,
		ClientID: client,
		APIKey:   key,
	}
	return c
}

// Creates an SSH Key and returns it's id
func (d DigitalOceanClient) CreateKey(name string, pub string) (uint, error) {
	params := url.Values{}
	params.Set("name", name)
	params.Set("ssh_pub_key", pub)

	body, err := NewRequest(d, "ssh_keys/new", params)
	if err != nil {
		return 0, err
	}

	// Read the SSH key's ID we just created
	key := body["ssh_key"].(map[string]interface{})
	keyId := key["id"].(float64)
	return uint(keyId), nil
}

// Destroys an SSH key
func (d DigitalOceanClient) DestroyKey(id uint) error {
	path := fmt.Sprintf("ssh_keys/%v/destroy", id)
	_, err := NewRequest(d, path, url.Values{})
	return err
}

// Creates a droplet and returns it's id
func (d DigitalOceanClient) CreateDroplet(name string, size uint, image uint, region uint, keyId uint) (uint, error) {
	params := url.Values{}
	params.Set("name", name)
	params.Set("size_id", fmt.Sprintf("%v", size))
	params.Set("image_id", fmt.Sprintf("%v", image))
	params.Set("region_id", fmt.Sprintf("%v", region))
	params.Set("ssh_key_ids", fmt.Sprintf("%v", keyId))

	body, err := NewRequest(d, "droplets/new", params)
	if err != nil {
		return 0, err
	}

	// Read the Droplets ID
	droplet := body["droplet"].(map[string]interface{})
	dropletId := droplet["id"].(float64)
	return uint(dropletId), err
}

// Destroys a droplet
func (d DigitalOceanClient) DestroyDroplet(id uint) error {
	path := fmt.Sprintf("droplets/%v/destroy", id)
	_, err := NewRequest(d, path, url.Values{})
	return err
}

// Powers off a droplet
func (d DigitalOceanClient) PowerOffDroplet(id uint) error {
	path := fmt.Sprintf("droplets/%v/power_off", id)

	_, err := NewRequest(d, path, url.Values{})

	return err
}

// Shutsdown a droplet. This is a "soft" shutdown.
func (d DigitalOceanClient) ShutdownDroplet(id uint) error {
	path := fmt.Sprintf("droplets/%v/shutdown", id)

	_, err := NewRequest(d, path, url.Values{})

	return err
}

// Creates a snaphot of a droplet by it's ID
func (d DigitalOceanClient) CreateSnapshot(id uint, name string) error {
	path := fmt.Sprintf("droplets/%v/snapshot", id)

	params := url.Values{}
	params.Set("name", name)

	_, err := NewRequest(d, path, params)

	return err
}

// Returns all available images.
func (d DigitalOceanClient) Images() ([]Image, error) {
	resp, err := NewRequest(d, "images", url.Values{})
	if err != nil {
		return nil, err
	}

	var result ImagesResp
	if err := mapstructure.Decode(resp, &result); err != nil {
		return nil, err
	}

	return result.Images, nil
}

// Destroys an image by its ID.
func (d DigitalOceanClient) DestroyImage(id uint) error {
	path := fmt.Sprintf("images/%d/destroy", id)
	_, err := NewRequest(d, path, url.Values{})
	return err
}

// Returns DO's string representation of status "off" "new" "active" etc.
func (d DigitalOceanClient) DropletStatus(id uint) (string, string, error) {
	path := fmt.Sprintf("droplets/%v", id)

	body, err := NewRequest(d, path, url.Values{})
	if err != nil {
		return "", "", err
	}

	var ip string

	// Read the droplet's "status"
	droplet := body["droplet"].(map[string]interface{})
	status := droplet["status"].(string)

	if droplet["ip_address"] != nil {
		ip = droplet["ip_address"].(string)
	}

	return ip, status, err
}

// Sends an api request and returns a generic map[string]interface of
// the response.
func NewRequest(d DigitalOceanClient, path string, params url.Values) (map[string]interface{}, error) {
	client := d.client

	// Add the authentication parameters
	params.Set("client_id", d.ClientID)
	params.Set("api_key", d.APIKey)

	url := fmt.Sprintf("%s/%s?%s", DIGITALOCEAN_API_URL, path, params.Encode())

	// Do some basic scrubbing so sensitive information doesn't appear in logs
	scrubbedUrl := strings.Replace(url, d.ClientID, "CLIENT_ID", -1)
	scrubbedUrl = strings.Replace(scrubbedUrl, d.APIKey, "API_KEY", -1)
	log.Printf("sending new request to digitalocean: %s", scrubbedUrl)

	var lastErr error
	for attempts := 1; attempts < 10; attempts++ {
		resp, err := client.Get(url)
		if err != nil {
			return nil, err
		}

		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			return nil, err
		}

		log.Printf("response from digitalocean: %s", body)

		var decodedResponse map[string]interface{}
		err = json.Unmarshal(body, &decodedResponse)
		if err != nil {
			err = errors.New(fmt.Sprintf("Failed to decode JSON response (HTTP %v) from DigitalOcean: %s",
				resp.StatusCode, body))
			return decodedResponse, err
		}

		// Check for errors sent by digitalocean
		status := decodedResponse["status"].(string)
		if status == "OK" {
			return decodedResponse, nil
		}

		if status == "ERROR" {
			status = decodedResponse["error_message"].(string)
		}

		lastErr = errors.New(fmt.Sprintf("Received error from DigitalOcean (%d): %s",
			resp.StatusCode, status))
		log.Println(lastErr)
		if strings.Contains(status, "a pending event") {
			// Retry, DigitalOcean sends these dumb "pending event"
			// errors all the time.
			time.Sleep(5 * time.Second)
			continue
		}

		// Some other kind of error. Just return.
		return decodedResponse, lastErr
	}

	return nil, lastErr
}
<|endoftext|>


<commit_before>package scheduler

import (
	"context"
	"errors"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/db/lock"
	"github.com/concourse/concourse/atc/metric"
	"github.com/concourse/concourse/atc/scheduler/algorithm"
)

//go:generate counterfeiter . BuildScheduler

type BuildScheduler interface {
	Schedule(
		logger lager.Logger,
		pipeline db.Pipeline,
		job db.Job,
		resources db.Resources,
		relatedJobIDs algorithm.NameToIDMap,
	) error
}

var errPipelineRemoved = errors.New("pipeline removed")

type schedulerRunner struct {
	logger             lager.Logger
	pipelineFactory    db.PipelineFactory
	scheduler          BuildScheduler
	guardJobScheduling chan struct{}
}

func NewRunner(logger lager.Logger, pipelineFactory db.PipelineFactory, scheduler BuildScheduler, maxJobs uint64) Runner {
	newGuardJobScheduling := make(chan struct{}, maxJobs)
	return &schedulerRunner{
		logger:             logger,
		pipelineFactory:    pipelineFactory,
		scheduler:          scheduler,
		guardJobScheduling: newGuardJobScheduling,
	}
}

func (s *schedulerRunner) Run(ctx context.Context) error {
	s.logger.Info("start")
	defer s.logger.Info("end")

	pipelines, err := s.pipelineFactory.PipelinesToSchedule()
	if err != nil {
		s.logger.Error("failed-to-get-pipelines-to-schedule", err)
		return err
	}

	for _, pipeline := range pipelines {
		// Grabs out the requested time that triggered off the pipeline schedule in
		// order to set the last scheduled to the exact time of this triggering
		// request
		requestedTime := pipeline.RequestedTime()

		go func(pipeline db.Pipeline, requestedTime time.Time) {
			err = s.schedulePipeline(pipeline)
			if err != nil {
				s.logger.Error("failed-to-schedule-pipeline", err, lager.Data{"pipeline": pipeline.Name()})
				return
			}

			err = pipeline.UpdateLastScheduled(requestedTime)
			if err != nil {
				s.logger.Error("failed-to-update-last-scheduled", err, lager.Data{"pipeline": pipeline.Name()})
			}
		}(pipeline, requestedTime)
	}

	return nil
}

func (s *schedulerRunner) schedulePipeline(pipeline db.Pipeline) error {
	logger := s.logger.Session("pipeline", lager.Data{"pipeline": pipeline.Name()})

	jobs, err := pipeline.Jobs()
	if err != nil {
		logger.Error("failed-to-get-jobs", err)
		return err
	}

	resources, err := pipeline.Resources()
	if err != nil {
		logger.Error("failed-to-get-resources", err)
		return err
	}

	jobsMap := map[string]int{}

	for _, job := range jobs {
		jobsMap[job.Name()] = job.ID()
	}

	for _, job := range jobs {
		schedulingLock, acquired, err := job.AcquireSchedulingLock(logger)
		if err != nil {
			logger.Error("failed-to-acquire-scheduling-lock", err)
			return nil
		}

		if !acquired {
			continue
		}

		s.guardJobScheduling <- struct{}{}
		go func(job db.Job) {
			err = s.scheduleJob(logger, schedulingLock, pipeline, job, resources, jobsMap)
			if err != nil {
				err = pipeline.RequestSchedule()
				if err != nil {
					// XXX if requesting schedule fails because of a connection failure,
					// we have no way of retrying the scheduler
					logger.Error("failed-to-request-schedule-on-job", err, lager.Data{"pipeline": job.PipelineName(), "job": job.Name()})
				}
			}
			<-s.guardJobScheduling
		}(job)
	}

	return nil
}

func (s *schedulerRunner) scheduleJob(logger lager.Logger, schedulingLock lock.Lock, pipeline db.Pipeline, currentJob db.Job, resources db.Resources, jobs algorithm.NameToIDMap) error {
	logger = logger.Session("job", lager.Data{"job": currentJob.Name()})

	defer schedulingLock.Release()

	found, err := currentJob.Reload()
	if err != nil {
		logger.Error("failed-to-update-job-config", err)
		return err
	}

	if !found {
		logger.Error("job-not-found", err)
		return err
	}

	sLog := logger.Session("scheduling")
	jStart := time.Now()

	err = s.scheduler.Schedule(
		sLog,
		pipeline,
		currentJob,
		resources,
		jobs,
	)
	if err != nil {
		logger.Error("failed-to-schedule", err)
		return err
	}

	metric.SchedulingJobDuration{
		PipelineName: currentJob.PipelineName(),
		JobName:      currentJob.Name(),
		JobID:        currentJob.ID(),
		Duration:     time.Since(jStart),
	}.Emit(sLog)

	return nil
}
<commit_msg>atc: fix data race<commit_after>package scheduler

import (
	"context"
	"errors"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/concourse/atc/db"
	"github.com/concourse/concourse/atc/db/lock"
	"github.com/concourse/concourse/atc/metric"
	"github.com/concourse/concourse/atc/scheduler/algorithm"
)

//go:generate counterfeiter . BuildScheduler

type BuildScheduler interface {
	Schedule(
		logger lager.Logger,
		pipeline db.Pipeline,
		job db.Job,
		resources db.Resources,
		relatedJobIDs algorithm.NameToIDMap,
	) error
}

var errPipelineRemoved = errors.New("pipeline removed")

type schedulerRunner struct {
	logger             lager.Logger
	pipelineFactory    db.PipelineFactory
	scheduler          BuildScheduler
	guardJobScheduling chan struct{}
}

func NewRunner(logger lager.Logger, pipelineFactory db.PipelineFactory, scheduler BuildScheduler, maxJobs uint64) Runner {
	newGuardJobScheduling := make(chan struct{}, maxJobs)
	return &schedulerRunner{
		logger:             logger,
		pipelineFactory:    pipelineFactory,
		scheduler:          scheduler,
		guardJobScheduling: newGuardJobScheduling,
	}
}

func (s *schedulerRunner) Run(ctx context.Context) error {
	s.logger.Info("start")
	defer s.logger.Info("end")

	pipelines, err := s.pipelineFactory.PipelinesToSchedule()
	if err != nil {
		s.logger.Error("failed-to-get-pipelines-to-schedule", err)
		return err
	}

	for _, pipeline := range pipelines {
		// Grabs out the requested time that triggered off the pipeline schedule in
		// order to set the last scheduled to the exact time of this triggering
		// request
		requestedTime := pipeline.RequestedTime()

		go func(pipeline db.Pipeline, requestedTime time.Time) {
			err := s.schedulePipeline(pipeline)
			if err != nil {
				s.logger.Error("failed-to-schedule-pipeline", err, lager.Data{"pipeline": pipeline.Name()})
				return
			}

			err = pipeline.UpdateLastScheduled(requestedTime)
			if err != nil {
				s.logger.Error("failed-to-update-last-scheduled", err, lager.Data{"pipeline": pipeline.Name()})
			}
		}(pipeline, requestedTime)
	}

	return nil
}

func (s *schedulerRunner) schedulePipeline(pipeline db.Pipeline) error {
	logger := s.logger.Session("pipeline", lager.Data{"pipeline": pipeline.Name()})

	jobs, err := pipeline.Jobs()
	if err != nil {
		logger.Error("failed-to-get-jobs", err)
		return err
	}

	resources, err := pipeline.Resources()
	if err != nil {
		logger.Error("failed-to-get-resources", err)
		return err
	}

	jobsMap := map[string]int{}

	for _, job := range jobs {
		jobsMap[job.Name()] = job.ID()
	}

	for _, job := range jobs {
		schedulingLock, acquired, err := job.AcquireSchedulingLock(logger)
		if err != nil {
			logger.Error("failed-to-acquire-scheduling-lock", err)
			return nil
		}

		if !acquired {
			continue
		}

		s.guardJobScheduling <- struct{}{}
		go func(job db.Job) {
			err := s.scheduleJob(logger, schedulingLock, pipeline, job, resources, jobsMap)
			if err != nil {
				err = pipeline.RequestSchedule()
				if err != nil {
					// XXX if requesting schedule fails because of a connection failure,
					// we have no way of retrying the scheduler
					logger.Error("failed-to-request-schedule-on-job", err, lager.Data{"pipeline": job.PipelineName(), "job": job.Name()})
				}
			}
			<-s.guardJobScheduling
		}(job)
	}

	return nil
}

func (s *schedulerRunner) scheduleJob(logger lager.Logger, schedulingLock lock.Lock, pipeline db.Pipeline, currentJob db.Job, resources db.Resources, jobs algorithm.NameToIDMap) error {
	logger = logger.Session("job", lager.Data{"job": currentJob.Name()})

	defer schedulingLock.Release()

	found, err := currentJob.Reload()
	if err != nil {
		logger.Error("failed-to-update-job-config", err)
		return err
	}

	if !found {
		logger.Error("job-not-found", err)
		return err
	}

	sLog := logger.Session("scheduling")
	jStart := time.Now()

	err = s.scheduler.Schedule(
		sLog,
		pipeline,
		currentJob,
		resources,
		jobs,
	)
	if err != nil {
		logger.Error("failed-to-schedule", err)
		return err
	}

	metric.SchedulingJobDuration{
		PipelineName: currentJob.PipelineName(),
		JobName:      currentJob.Name(),
		JobID:        currentJob.ID(),
		Duration:     time.Since(jStart),
	}.Emit(sLog)

	return nil
}
<|endoftext|>


<commit_before>package code2docx

import (
	"bufio"
	"log"
	"os"
	"strings"

	"baliance.com/gooxml/document"
	"baliance.com/gooxml/schema/soo/wml"
)

type Outer struct {
	Config  *Config
	Base    string
	Files   []string
	Docfile string
	doc     *document.Document
}

func (o *Outer) WriteDoc() error {
	o.init()
	for _, file := range o.Files {

		filename := strings.Replace(file, o.Base, "", 1)

		o.writeToDoc(filename, file)

	}
	o.doc.SaveToFile(o.Docfile)
	return nil
}

func (o *Outer) init() {
	doc := document.New()
	o.doc = doc
	hdr := doc.AddHeader()
	para := hdr.AddParagraph()
	setParagraphAlignCenter(para)
	para.AddRun().AddText(o.Config.Header)
	doc.BodySection().SetHeader(hdr, wml.ST_HdrFtrDefault)

	para = doc.AddParagraph()
	para.SetStyle("Title")
	setParagraphAlignCenter(para)
	para.AddRun().AddText(o.Config.Title)

	para = doc.AddParagraph()
	para.AddRun().AddText("")
}

func ensurePPr(p *wml.CT_P) {
	if p.PPr == nil {
		p.PPr = wml.NewCT_PPr()
	}
}
func setParagraphAlignCenter(para document.Paragraph) {
	paraAlign := wml.NewCT_Jc()
	paraAlign.ValAttr = wml.ST_JcCenter
	x := para.X()
	ensurePPr(x)
	x.PPr.Jc = paraAlign
}

func (o *Outer) writeToDoc(filename, filepath string) {
	doc := o.doc
	para := doc.AddParagraph()
	run := para.AddRun()
	run.AddText(filename + ":")
	run.AddBreak()

	fio, err := os.Open(filepath)
	if err != nil {
		log.Fatal("open file error: ", err)
	}
	defer fio.Close()
	fbscaner := bufio.NewScanner(fio)
	for fbscaner.Scan() {
		para := doc.AddParagraph()
		run := para.AddRun()
		run.AddText(fbscaner.Text())
	}
	if err := fbscaner.Err(); err != nil {
		log.Fatal("read file error: ", err)
	}

	para = doc.AddParagraph()
	run = para.AddRun()
	run.AddBreak()
}
<commit_msg>refactor<commit_after>package code2docx

import (
	"bufio"
	"log"
	"os"
	"strings"

	"baliance.com/gooxml/document"
	"baliance.com/gooxml/schema/soo/wml"
)

type Outer struct {
	Config  *Config
	Base    string
	Files   []string
	Docfile string
	doc     *document.Document
}

func (o *Outer) WriteDoc() error {
	o.init()
	for _, file := range o.Files {

		filename := strings.Replace(file, o.Base, "", 1)

		o.writeToDoc(filename, file)

	}
	o.doc.SaveToFile(o.Docfile)
	return nil
}

func (o *Outer) init() {
	doc := document.New()
	o.doc = doc
	hdr := doc.AddHeader()
	para := hdr.AddParagraph()
	setParagraphAlignCenter(para)
	para.AddRun().AddText(o.Config.Header)
	doc.BodySection().SetHeader(hdr, wml.ST_HdrFtrDefault)

	para = doc.AddParagraph()
	para.SetStyle("Title")
	setParagraphAlignCenter(para)
	para.AddRun().AddText(o.Config.Title)

	para = doc.AddParagraph()
	para.AddRun().AddText("")
}

func ensurePPr(para document.Paragraph) {
	if para.X().PPr == nil {
		para.X().PPr = wml.NewCT_PPr()
	}
}
func setParagraphAlignCenter(para document.Paragraph) {
	ensurePPr(para)
	paraAlign := wml.NewCT_Jc()
	paraAlign.ValAttr = wml.ST_JcCenter
	para.X().PPr.Jc = paraAlign
}

func (o *Outer) writeToDoc(filename, filepath string) {
	doc := o.doc
	para := doc.AddParagraph()
	run := para.AddRun()
	run.AddText(filename + ":")
	run.AddBreak()

	fio, err := os.Open(filepath)
	if err != nil {
		log.Fatal("open file error: ", err)
	}
	defer fio.Close()
	fbscaner := bufio.NewScanner(fio)
	for fbscaner.Scan() {
		para := doc.AddParagraph()
		run := para.AddRun()
		run.AddText(fbscaner.Text())
	}
	if err := fbscaner.Err(); err != nil {
		log.Fatal("read file error: ", err)
	}

	para = doc.AddParagraph()
	run = para.AddRun()
	run.AddBreak()
}
<|endoftext|>