<commit_before>package rtorrent

import (
	"fmt"
	"net/http"

	"github.com/mrobinsn/go-rtorrent/xmlrpc"
	"github.com/pkg/errors"
)

// RTorrent is used to communicate with a remote rTorrent instance
type RTorrent struct {
	addr         string
	xmlrpcClient *xmlrpc.Client
}

// Torrent represents a torrent in rTorrent
type Torrent struct {
	Hash      string
	Name      string
	Path      string
	Size      int
	Label     string
	Completed bool
	Ratio     float64
}

// Status represents the status of a torrent
type Status struct {
	Completed      bool
	CompletedBytes int
	DownRate       int
	UpRate         int
	Ratio          float64
	Size           int
}

// File represents a file in rTorrent
type File struct {
	Path string
	Size int
}

// View represents a "view" within RTorrent
type View string

const (
	// ViewMain represents the "main" view, containing all torrents
	ViewMain View = "main"
	// ViewStarted represents the "started" view, containing only torrents that have been started
	ViewStarted View = "started"
	// ViewStopped represents the "stopped" view, containing only torrents that have been stopped
	ViewStopped View = "stopped"
	// ViewHashing represents the "hashing" view, containing only torrents that are currently hashing
	ViewHashing View = "hashing"
	// ViewSeeding represents the "seeding" view, containing only torrents that are currently seeding
	ViewSeeding View = "seeding"
)

// Pretty returns a formatted string representing this Torrent
func (t *Torrent) Pretty() string {
	return fmt.Sprintf("Torrent:\n\tHash: %v\n\tName: %v\n\tPath: %v\n\tLabel: %v\n\tSize: %v bytes\n\tCompleted: %v\n\tRatio: %v\n", t.Hash, t.Name, t.Path, t.Label, t.Size, t.Completed, t.Ratio)
}

// Pretty returns a formatted string representing this File
func (f *File) Pretty() string {
	return fmt.Sprintf("File:\n\tPath: %v\n\tSize: %v bytes\n", f.Path, f.Size)
}

// New returns a new instance of `RTorrent`
// Pass in a true value for `insecure` to turn off certificate verification
func New(addr string, insecure bool) *RTorrent {
	return &RTorrent{
		addr:         addr,
		xmlrpcClient: xmlrpc.NewClient(addr, insecure),
	}
}

// WithHTTPClient allows you to a provide a custom http.Client.
func (r *RTorrent) WithHTTPClient(client *http.Client) *RTorrent {
	r.xmlrpcClient = xmlrpc.NewClientWithHTTPClient(r.addr, client)
	return r
}

// Add adds a new torrent by URL
func (r *RTorrent) Add(url string) error {
	_, err := r.xmlrpcClient.Call("load.start", "", url)
	if err != nil {
		return errors.Wrap(err, "load.start XMLRPC call failed")
	}
	return nil
}

// AddTorrent adds a new torrent by the torrent files data
func (r *RTorrent) AddTorrent(data []byte) error {
	_, err := r.xmlrpcClient.Call("load.raw_start", "", data)
	if err != nil {
		return errors.Wrap(err, "load.raw_start XMLRPC call failed")
	}
	return nil
}

// IP returns the IP reported by this RTorrent instance
func (r *RTorrent) IP() (string, error) {
	result, err := r.xmlrpcClient.Call("network.bind_address")
	if err != nil {
		return "", errors.Wrap(err, "network.bind_address XMLRPC call failed")
	}
	if ips, ok := result.([]interface{}); ok {
		result = ips[0]
	}
	if ip, ok := result.(string); ok {
		return ip, nil
	}
	return "", errors.Errorf("result isn't string: %v", result)
}

// Name returns the name reported by this RTorrent instance
func (r *RTorrent) Name() (string, error) {
	result, err := r.xmlrpcClient.Call("system.hostname")
	if err != nil {
		return "", errors.Wrap(err, "system.hostname XMLRPC call failed")
	}
	if names, ok := result.([]interface{}); ok {
		result = names[0]
	}
	if name, ok := result.(string); ok {
		return name, nil
	}
	return "", errors.Errorf("result isn't string: %v", result)
}

// DownTotal returns the total downloaded metric reported by this RTorrent instance (bytes)
func (r *RTorrent) DownTotal() (int, error) {
	result, err := r.xmlrpcClient.Call("throttle.global_down.total")
	if err != nil {
		return 0, errors.Wrap(err, "throttle.global_down.total XMLRPC call failed")
	}
	if totals, ok := result.([]interface{}); ok {
		result = totals[0]
	}
	if total, ok := result.(int); ok {
		return total, nil
	}
	return 0, errors.Errorf("result isn't int: %v", result)
}

// DownRate returns the current download rate reported by this RTorrent instance (bytes/s)
func (r *RTorrent) DownRate() (int, error) {
	result, err := r.xmlrpcClient.Call("throttle.global_down.rate")
	if err != nil {
		return 0, errors.Wrap(err, "throttle.global_down.rate XMLRPC call failed")
	}
	if totals, ok := result.([]interface{}); ok {
		result = totals[0]
	}
	if total, ok := result.(int); ok {
		return total, nil
	}
	return 0, errors.Errorf("result isn't int: %v", result)
}

// UpTotal returns the total uploaded metric reported by this RTorrent instance (bytes)
func (r *RTorrent) UpTotal() (int, error) {
	result, err := r.xmlrpcClient.Call("throttle.global_up.total")
	if err != nil {
		return 0, errors.Wrap(err, "throttle.global_up.total XMLRPC call failed")
	}
	if totals, ok := result.([]interface{}); ok {
		result = totals[0]
	}
	if total, ok := result.(int); ok {
		return total, nil
	}
	return 0, errors.Errorf("result isn't int: %v", result)
}

// UpRate returns the current upload rate reported by this RTorrent instance (bytes/s)
func (r *RTorrent) UpRate() (int, error) {
	result, err := r.xmlrpcClient.Call("throttle.global_up.rate")
	if err != nil {
		return 0, errors.Wrap(err, "throttle.global_up.rate XMLRPC call failed")
	}
	if totals, ok := result.([]interface{}); ok {
		result = totals[0]
	}
	if total, ok := result.(int); ok {
		return total, nil
	}
	return 0, errors.Errorf("result isn't int: %v", result)
}

// GetTorrents returns all of the torrents reported by this RTorrent instance
func (r *RTorrent) GetTorrents(view View) ([]Torrent, error) {
	args := []interface{}{"", string(view), "d.name=", "d.size_bytes=", "d.hash=", "d.custom1=", "d.base_path=", "d.is_active=", "d.complete=", "d.ratio="}
	results, err := r.xmlrpcClient.Call("d.multicall2", args...)
	var torrents []Torrent
	if err != nil {
		return torrents, errors.Wrap(err, "d.multicall2 XMLRPC call failed")
	}
	for _, outerResult := range results.([]interface{}) {
		for _, innerResult := range outerResult.([]interface{}) {
			torrentData := innerResult.([]interface{})
			torrents = append(torrents, Torrent{
				Hash:      torrentData[2].(string),
				Name:      torrentData[0].(string),
				Path:      torrentData[4].(string),
				Size:      torrentData[1].(int),
				Label:     torrentData[3].(string),
				Completed: torrentData[6].(int) > 0,
				Ratio:     float64(torrentData[7].(int)) / float64(1000),
			})
		}
	}
	return torrents, nil
}

// GetTorrent returns the torrent identified by the given hash
func (r *RTorrent) GetTorrent(hash string) (Torrent, error) {
	var t Torrent
	t.Hash = hash
	// Name
	results, err := r.xmlrpcClient.Call("d.name", t.Hash)
	if err != nil {
		return t, errors.Wrap(err, "d.name XMLRPC call failed")
	}
	t.Name = results.([]interface{})[0].(string)
	// Size
	results, err = r.xmlrpcClient.Call("d.size_bytes", t.Hash)
	if err != nil {
		return t, errors.Wrap(err, "d.size_bytes XMLRPC call failed")
	}
	t.Size = results.([]interface{})[0].(int)
	// Label
	results, err = r.xmlrpcClient.Call("d.custom1", t.Hash)
	if err != nil {
		return t, errors.Wrap(err, "d.custom1 XMLRPC call failed")
	}
	t.Label = results.([]interface{})[0].(string)
	// Path
	results, err = r.xmlrpcClient.Call("d.base_path", t.Hash)
	if err != nil {
		return t, errors.Wrap(err, "d.base_path XMLRPC call failed")
	}
	t.Path = results.([]interface{})[0].(string)
	// Completed
	results, err = r.xmlrpcClient.Call("d.complete", t.Hash)
	if err != nil {
		return t, errors.Wrap(err, "d.complete XMLRPC call failed")
	}
	t.Completed = results.([]interface{})[0].(int) > 0
	// Ratio
	results, err = r.xmlrpcClient.Call("d.ratio", t.Hash)
	if err != nil {
		return t, errors.Wrap(err, "d.ratio XMLRPC call failed")
	}
	t.Ratio = float64(results.([]interface{})[0].(int)) / float64(1000)
	return t, nil
}

// Delete removes the torrent
func (r *RTorrent) Delete(t Torrent) error {
	_, err := r.xmlrpcClient.Call("d.erase", t.Hash)
	if err != nil {
		return errors.Wrap(err, "d.erase XMLRPC call failed")
	}
	return nil
}

// GetFiles returns all of the files for a given `Torrent`
func (r *RTorrent) GetFiles(t Torrent) ([]File, error) {
	args := []interface{}{t.Hash, 0, "f.path=", "f.size_bytes="}
	results, err := r.xmlrpcClient.Call("f.multicall", args...)
	var files []File
	if err != nil {
		return files, errors.Wrap(err, "f.multicall XMLRPC call failed")
	}
	for _, outerResult := range results.([]interface{}) {
		for _, innerResult := range outerResult.([]interface{}) {
			fileData := innerResult.([]interface{})
			files = append(files, File{
				Path: fileData[0].(string),
				Size: fileData[1].(int),
			})
		}
	}
	return files, nil
}

// SetLabel sets the label on the given Torrent
func (r *RTorrent) SetLabel(t Torrent, newLabel string) error {
	t.Label = newLabel
	args := []interface{}{t.Hash, newLabel}
	if _, err := r.xmlrpcClient.Call("d.custom1.set", args...); err != nil {
		return errors.Wrap(err, "d.custom1.set XMLRPC call failed")
	}
	return nil
}

// GetStatus returns the Status for a given Torrent
func (r *RTorrent) GetStatus(t Torrent) (Status, error) {
	var s Status
	// Completed
	results, err := r.xmlrpcClient.Call("d.complete", t.Hash)
	if err != nil {
		return s, errors.Wrap(err, "d.complete XMLRPC call failed")
	}
	s.Completed = results.([]interface{})[0].(int) > 0
	// CompletedBytes
	results, err = r.xmlrpcClient.Call("d.completed_bytes", t.Hash)
	if err != nil {
		return s, errors.Wrap(err, "d.completed_bytes XMLRPC call failed")
	}
	s.CompletedBytes = results.([]interface{})[0].(int)
	// DownRate
	results, err = r.xmlrpcClient.Call("d.down.rate", t.Hash)
	if err != nil {
		return s, errors.Wrap(err, "d.down.rate XMLRPC call failed")
	}
	s.DownRate = results.([]interface{})[0].(int)
	// UpRate
	results, err = r.xmlrpcClient.Call("d.up.rate", t.Hash)
	if err != nil {
		return s, errors.Wrap(err, "d.up.rate XMLRPC call failed")
	}
	s.UpRate = results.([]interface{})[0].(int)
	// Ratio
	results, err = r.xmlrpcClient.Call("d.ratio", t.Hash)
	if err != nil {
		return s, errors.Wrap(err, "d.ratio XMLRPC call failed")
	}
	s.Ratio = float64(results.([]interface{})[0].(int)) / float64(1000)
	// Size
	results, err = r.xmlrpcClient.Call("d.size_bytes", t.Hash)
	if err != nil {
		return s, errors.Wrap(err, "d.size_bytes XMLRPC call failed")
	}
	s.Size = results.([]interface{})[0].(int)
	return s, nil
}
<commit_msg>add methods to add torrents in stopped state<commit_after>package rtorrent

import (
	"fmt"
	"net/http"

	"github.com/mrobinsn/go-rtorrent/xmlrpc"
	"github.com/pkg/errors"
)

// RTorrent is used to communicate with a remote rTorrent instance
type RTorrent struct {
	addr         string
	xmlrpcClient *xmlrpc.Client
}

// Torrent represents a torrent in rTorrent
type Torrent struct {
	Hash      string
	Name      string
	Path      string
	Size      int
	Label     string
	Completed bool
	Ratio     float64
}

// Status represents the status of a torrent
type Status struct {
	Completed      bool
	CompletedBytes int
	DownRate       int
	UpRate         int
	Ratio          float64
	Size           int
}

// File represents a file in rTorrent
type File struct {
	Path string
	Size int
}

// View represents a "view" within RTorrent
type View string

const (
	// ViewMain represents the "main" view, containing all torrents
	ViewMain View = "main"
	// ViewStarted represents the "started" view, containing only torrents that have been started
	ViewStarted View = "started"
	// ViewStopped represents the "stopped" view, containing only torrents that have been stopped
	ViewStopped View = "stopped"
	// ViewHashing represents the "hashing" view, containing only torrents that are currently hashing
	ViewHashing View = "hashing"
	// ViewSeeding represents the "seeding" view, containing only torrents that are currently seeding
	ViewSeeding View = "seeding"
)

// Pretty returns a formatted string representing this Torrent
func (t *Torrent) Pretty() string {
	return fmt.Sprintf("Torrent:\n\tHash: %v\n\tName: %v\n\tPath: %v\n\tLabel: %v\n\tSize: %v bytes\n\tCompleted: %v\n\tRatio: %v\n", t.Hash, t.Name, t.Path, t.Label, t.Size, t.Completed, t.Ratio)
}

// Pretty returns a formatted string representing this File
func (f *File) Pretty() string {
	return fmt.Sprintf("File:\n\tPath: %v\n\tSize: %v bytes\n", f.Path, f.Size)
}

// New returns a new instance of `RTorrent`
// Pass in a true value for `insecure` to turn off certificate verification
func New(addr string, insecure bool) *RTorrent {
	return &RTorrent{
		addr:         addr,
		xmlrpcClient: xmlrpc.NewClient(addr, insecure),
	}
}

// WithHTTPClient allows you to a provide a custom http.Client.
func (r *RTorrent) WithHTTPClient(client *http.Client) *RTorrent {
	r.xmlrpcClient = xmlrpc.NewClientWithHTTPClient(r.addr, client)
	return r
}

// AddStopped adds a new torrent by URL in a stopped state
func (r *RTorrent) AddStopped(url string) error {
	_, err := r.xmlrpcClient.Call("load.normal", "", url)
	if err != nil {
		return errors.Wrap(err, "load.normal XMLRPC call failed")
	}
	return nil
}

// Add adds a new torrent by URL and starts the torrent
func (r *RTorrent) Add(url string) error {
	_, err := r.xmlrpcClient.Call("load.start", "", url)
	if err != nil {
		return errors.Wrap(err, "load.start XMLRPC call failed")
	}
	return nil
}

// AddTorrentStopped adds a new torrent by the torrent files data but does not start the torrent
func (r *RTorrent) AddTorrentStopped(data []byte) error {
	_, err := r.xmlrpcClient.Call("load.raw", "", data)
	if err != nil {
		return errors.Wrap(err, "load.raw XMLRPC call failed")
	}
	return nil
}

// AddTorrent adds a new torrent by the torrent files data and starts the torrent
func (r *RTorrent) AddTorrent(data []byte) error {
	_, err := r.xmlrpcClient.Call("load.raw_start", "", data)
	if err != nil {
		return errors.Wrap(err, "load.raw_start XMLRPC call failed")
	}
	return nil
}

// IP returns the IP reported by this RTorrent instance
func (r *RTorrent) IP() (string, error) {
	result, err := r.xmlrpcClient.Call("network.bind_address")
	if err != nil {
		return "", errors.Wrap(err, "network.bind_address XMLRPC call failed")
	}
	if ips, ok := result.([]interface{}); ok {
		result = ips[0]
	}
	if ip, ok := result.(string); ok {
		return ip, nil
	}
	return "", errors.Errorf("result isn't string: %v", result)
}

// Name returns the name reported by this RTorrent instance
func (r *RTorrent) Name() (string, error) {
	result, err := r.xmlrpcClient.Call("system.hostname")
	if err != nil {
		return "", errors.Wrap(err, "system.hostname XMLRPC call failed")
	}
	if names, ok := result.([]interface{}); ok {
		result = names[0]
	}
	if name, ok := result.(string); ok {
		return name, nil
	}
	return "", errors.Errorf("result isn't string: %v", result)
}

// DownTotal returns the total downloaded metric reported by this RTorrent instance (bytes)
func (r *RTorrent) DownTotal() (int, error) {
	result, err := r.xmlrpcClient.Call("throttle.global_down.total")
	if err != nil {
		return 0, errors.Wrap(err, "throttle.global_down.total XMLRPC call failed")
	}
	if totals, ok := result.([]interface{}); ok {
		result = totals[0]
	}
	if total, ok := result.(int); ok {
		return total, nil
	}
	return 0, errors.Errorf("result isn't int: %v", result)
}

// DownRate returns the current download rate reported by this RTorrent instance (bytes/s)
func (r *RTorrent) DownRate() (int, error) {
	result, err := r.xmlrpcClient.Call("throttle.global_down.rate")
	if err != nil {
		return 0, errors.Wrap(err, "throttle.global_down.rate XMLRPC call failed")
	}
	if totals, ok := result.([]interface{}); ok {
		result = totals[0]
	}
	if total, ok := result.(int); ok {
		return total, nil
	}
	return 0, errors.Errorf("result isn't int: %v", result)
}

// UpTotal returns the total uploaded metric reported by this RTorrent instance (bytes)
func (r *RTorrent) UpTotal() (int, error) {
	result, err := r.xmlrpcClient.Call("throttle.global_up.total")
	if err != nil {
		return 0, errors.Wrap(err, "throttle.global_up.total XMLRPC call failed")
	}
	if totals, ok := result.([]interface{}); ok {
		result = totals[0]
	}
	if total, ok := result.(int); ok {
		return total, nil
	}
	return 0, errors.Errorf("result isn't int: %v", result)
}

// UpRate returns the current upload rate reported by this RTorrent instance (bytes/s)
func (r *RTorrent) UpRate() (int, error) {
	result, err := r.xmlrpcClient.Call("throttle.global_up.rate")
	if err != nil {
		return 0, errors.Wrap(err, "throttle.global_up.rate XMLRPC call failed")
	}
	if totals, ok := result.([]interface{}); ok {
		result = totals[0]
	}
	if total, ok := result.(int); ok {
		return total, nil
	}
	return 0, errors.Errorf("result isn't int: %v", result)
}

// GetTorrents returns all of the torrents reported by this RTorrent instance
func (r *RTorrent) GetTorrents(view View) ([]Torrent, error) {
	args := []interface{}{"", string(view), "d.name=", "d.size_bytes=", "d.hash=", "d.custom1=", "d.base_path=", "d.is_active=", "d.complete=", "d.ratio="}
	results, err := r.xmlrpcClient.Call("d.multicall2", args...)
	var torrents []Torrent
	if err != nil {
		return torrents, errors.Wrap(err, "d.multicall2 XMLRPC call failed")
	}
	for _, outerResult := range results.([]interface{}) {
		for _, innerResult := range outerResult.([]interface{}) {
			torrentData := innerResult.([]interface{})
			torrents = append(torrents, Torrent{
				Hash:      torrentData[2].(string),
				Name:      torrentData[0].(string),
				Path:      torrentData[4].(string),
				Size:      torrentData[1].(int),
				Label:     torrentData[3].(string),
				Completed: torrentData[6].(int) > 0,
				Ratio:     float64(torrentData[7].(int)) / float64(1000),
			})
		}
	}
	return torrents, nil
}

// GetTorrent returns the torrent identified by the given hash
func (r *RTorrent) GetTorrent(hash string) (Torrent, error) {
	var t Torrent
	t.Hash = hash
	// Name
	results, err := r.xmlrpcClient.Call("d.name", t.Hash)
	if err != nil {
		return t, errors.Wrap(err, "d.name XMLRPC call failed")
	}
	t.Name = results.([]interface{})[0].(string)
	// Size
	results, err = r.xmlrpcClient.Call("d.size_bytes", t.Hash)
	if err != nil {
		return t, errors.Wrap(err, "d.size_bytes XMLRPC call failed")
	}
	t.Size = results.([]interface{})[0].(int)
	// Label
	results, err = r.xmlrpcClient.Call("d.custom1", t.Hash)
	if err != nil {
		return t, errors.Wrap(err, "d.custom1 XMLRPC call failed")
	}
	t.Label = results.([]interface{})[0].(string)
	// Path
	results, err = r.xmlrpcClient.Call("d.base_path", t.Hash)
	if err != nil {
		return t, errors.Wrap(err, "d.base_path XMLRPC call failed")
	}
	t.Path = results.([]interface{})[0].(string)
	// Completed
	results, err = r.xmlrpcClient.Call("d.complete", t.Hash)
	if err != nil {
		return t, errors.Wrap(err, "d.complete XMLRPC call failed")
	}
	t.Completed = results.([]interface{})[0].(int) > 0
	// Ratio
	results, err = r.xmlrpcClient.Call("d.ratio", t.Hash)
	if err != nil {
		return t, errors.Wrap(err, "d.ratio XMLRPC call failed")
	}
	t.Ratio = float64(results.([]interface{})[0].(int)) / float64(1000)
	return t, nil
}

// Delete removes the torrent
func (r *RTorrent) Delete(t Torrent) error {
	_, err := r.xmlrpcClient.Call("d.erase", t.Hash)
	if err != nil {
		return errors.Wrap(err, "d.erase XMLRPC call failed")
	}
	return nil
}

// GetFiles returns all of the files for a given `Torrent`
func (r *RTorrent) GetFiles(t Torrent) ([]File, error) {
	args := []interface{}{t.Hash, 0, "f.path=", "f.size_bytes="}
	results, err := r.xmlrpcClient.Call("f.multicall", args...)
	var files []File
	if err != nil {
		return files, errors.Wrap(err, "f.multicall XMLRPC call failed")
	}
	for _, outerResult := range results.([]interface{}) {
		for _, innerResult := range outerResult.([]interface{}) {
			fileData := innerResult.([]interface{})
			files = append(files, File{
				Path: fileData[0].(string),
				Size: fileData[1].(int),
			})
		}
	}
	return files, nil
}

// SetLabel sets the label on the given Torrent
func (r *RTorrent) SetLabel(t Torrent, newLabel string) error {
	t.Label = newLabel
	args := []interface{}{t.Hash, newLabel}
	if _, err := r.xmlrpcClient.Call("d.custom1.set", args...); err != nil {
		return errors.Wrap(err, "d.custom1.set XMLRPC call failed")
	}
	return nil
}

// GetStatus returns the Status for a given Torrent
func (r *RTorrent) GetStatus(t Torrent) (Status, error) {
	var s Status
	// Completed
	results, err := r.xmlrpcClient.Call("d.complete", t.Hash)
	if err != nil {
		return s, errors.Wrap(err, "d.complete XMLRPC call failed")
	}
	s.Completed = results.([]interface{})[0].(int) > 0
	// CompletedBytes
	results, err = r.xmlrpcClient.Call("d.completed_bytes", t.Hash)
	if err != nil {
		return s, errors.Wrap(err, "d.completed_bytes XMLRPC call failed")
	}
	s.CompletedBytes = results.([]interface{})[0].(int)
	// DownRate
	results, err = r.xmlrpcClient.Call("d.down.rate", t.Hash)
	if err != nil {
		return s, errors.Wrap(err, "d.down.rate XMLRPC call failed")
	}
	s.DownRate = results.([]interface{})[0].(int)
	// UpRate
	results, err = r.xmlrpcClient.Call("d.up.rate", t.Hash)
	if err != nil {
		return s, errors.Wrap(err, "d.up.rate XMLRPC call failed")
	}
	s.UpRate = results.([]interface{})[0].(int)
	// Ratio
	results, err = r.xmlrpcClient.Call("d.ratio", t.Hash)
	if err != nil {
		return s, errors.Wrap(err, "d.ratio XMLRPC call failed")
	}
	s.Ratio = float64(results.([]interface{})[0].(int)) / float64(1000)
	// Size
	results, err = r.xmlrpcClient.Call("d.size_bytes", t.Hash)
	if err != nil {
		return s, errors.Wrap(err, "d.size_bytes XMLRPC call failed")
	}
	s.Size = results.([]interface{})[0].(int)
	return s, nil
}
<|endoftext|>


<commit_before>//+build go1.12,integration,linux

package netlink_test

import (
	"net"
	"testing"
	"time"

	"github.com/mdlayher/netlink"

	"golang.org/x/sys/unix"
)

func TestLinuxConnIntegrationTimeout(t *testing.T) {
	conn, err := netlink.Dial(unix.NETLINK_GENERIC, nil)
	if err != nil {
		t.Fatalf("failed to dial: %v", err)
	}
	defer conn.Close()

	timeout := 1 * time.Millisecond
	if err := conn.SetReadDeadline(time.Now().Add(timeout)); err != nil {
		t.Fatalf("failed to set deadline: %v", err)
	}

	errC := make(chan error)
	go func() {
		_, err := conn.Receive()
		errC <- err
	}()

	select {
	case err := <-errC:
		mustBeTimeoutNetError(t, err)
	case <-time.After(timeout + 1*time.Millisecond):
		t.Fatalf("timeout did not fire")
	}
}

func TestLinuxConnIntegrationExecuteAfterReadDeadline(t *testing.T) {
	conn, err := netlink.Dial(unix.NETLINK_GENERIC, nil)
	if err != nil {
		t.Fatalf("failed to dial: %v", err)
	}
	defer conn.Close()

	timeout := 1 * time.Millisecond
	if err := conn.SetReadDeadline(time.Now().Add(timeout)); err != nil {
		t.Fatalf("failed to set deadline: %v", err)
	}
	time.Sleep(2 * timeout)

	req := netlink.Message{
		Header: netlink.Header{
			Flags:    netlink.Request | netlink.Acknowledge,
			Sequence: 1,
		},
	}
	got, err := conn.Execute(req)
	if err == nil {
		t.Fatalf("Execute succeeded: got %v", got)
	}
	mustBeTimeoutNetError(t, err)
}

func mustBeTimeoutNetError(t *testing.T, err error) {
	t.Helper()
	ne, ok := err.(net.Error)
	if !ok {
		t.Fatalf("didn't get a net.Error: got a %T instead", err)
	}
	if !ne.Timeout() {
		t.Fatalf("didn't get a timeout")
	}
}
<commit_msg>netlink: fix names of some integration tests<commit_after>//+build go1.12,integration,linux

package netlink_test

import (
	"net"
	"testing"
	"time"

	"github.com/mdlayher/netlink"

	"golang.org/x/sys/unix"
)

func TestIntegrationConnTimeout(t *testing.T) {
	conn, err := netlink.Dial(unix.NETLINK_GENERIC, nil)
	if err != nil {
		t.Fatalf("failed to dial: %v", err)
	}
	defer conn.Close()

	timeout := 1 * time.Millisecond
	if err := conn.SetReadDeadline(time.Now().Add(timeout)); err != nil {
		t.Fatalf("failed to set deadline: %v", err)
	}

	errC := make(chan error)
	go func() {
		_, err := conn.Receive()
		errC <- err
	}()

	select {
	case err := <-errC:
		mustBeTimeoutNetError(t, err)
	case <-time.After(timeout + 1*time.Millisecond):
		t.Fatalf("timeout did not fire")
	}
}

func TestIntegrationConnExecuteAfterReadDeadline(t *testing.T) {
	conn, err := netlink.Dial(unix.NETLINK_GENERIC, nil)
	if err != nil {
		t.Fatalf("failed to dial: %v", err)
	}
	defer conn.Close()

	timeout := 1 * time.Millisecond
	if err := conn.SetReadDeadline(time.Now().Add(timeout)); err != nil {
		t.Fatalf("failed to set deadline: %v", err)
	}
	time.Sleep(2 * timeout)

	req := netlink.Message{
		Header: netlink.Header{
			Flags:    netlink.Request | netlink.Acknowledge,
			Sequence: 1,
		},
	}
	got, err := conn.Execute(req)
	if err == nil {
		t.Fatalf("Execute succeeded: got %v", got)
	}
	mustBeTimeoutNetError(t, err)
}

func mustBeTimeoutNetError(t *testing.T, err error) {
	t.Helper()
	ne, ok := err.(net.Error)
	if !ok {
		t.Fatalf("didn't get a net.Error: got a %T instead", err)
	}
	if !ne.Timeout() {
		t.Fatalf("didn't get a timeout")
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigquery

import (
	"time"

	"cloud.google.com/go/internal/optional"

	"golang.org/x/net/context"
	"google.golang.org/api/iterator"
)

// Dataset is a reference to a BigQuery dataset.
type Dataset struct {
	ProjectID string
	DatasetID string
	c         *Client
}

type DatasetMetadata struct {
	CreationTime           time.Time
	LastModifiedTime       time.Time     // When the dataset or any of its tables were modified.
	DefaultTableExpiration time.Duration // The default expiration time for new tables.
	Description            string        // The user-friendly description of this dataset.
	Name                   string        // The user-friendly name for this dataset.
	ID                     string
	Location               string            // The geo location of the dataset.
	Labels                 map[string]string // User-provided labels.

	// ETag is the ETag obtained when reading metadata. Pass it to Dataset.Update to
	// ensure that the metadata hasn't changed since it was read.
	ETag string
	// TODO(jba): access rules
}

type DatasetMetadataToUpdate struct {
	Description optional.String // The user-friendly description of this table.
	Name        optional.String // The user-friendly name for this dataset.
	// DefaultTableExpiration is the the default expiration time for new tables.
	// If set to time.Duration(0), new tables never expire.
	DefaultTableExpiration optional.Duration
}

// Dataset creates a handle to a BigQuery dataset in the client's project.
func (c *Client) Dataset(id string) *Dataset {
	return c.DatasetInProject(c.projectID, id)
}

// DatasetInProject creates a handle to a BigQuery dataset in the specified project.
func (c *Client) DatasetInProject(projectID, datasetID string) *Dataset {
	return &Dataset{
		ProjectID: projectID,
		DatasetID: datasetID,
		c:         c,
	}
}

// Create creates a dataset in the BigQuery service. An error will be returned
// if the dataset already exists.
func (d *Dataset) Create(ctx context.Context) error {
	return d.c.service.insertDataset(ctx, d.DatasetID, d.ProjectID)
}

// Delete deletes the dataset.
func (d *Dataset) Delete(ctx context.Context) error {
	return d.c.service.deleteDataset(ctx, d.DatasetID, d.ProjectID)
}

// Metadata fetches the metadata for the dataset.
func (d *Dataset) Metadata(ctx context.Context) (*DatasetMetadata, error) {
	return d.c.service.getDatasetMetadata(ctx, d.ProjectID, d.DatasetID)
}

// Update modifies specific Dataset metadata fields.
// To perform a read-modify-write that protects against intervening reads,
// set the etag argument to the DatasetMetadata.ETag field from the read.
// Pass the empty string for etag for a "blind write" that will always succeed.
func (d *Dataset) Update(ctx context.Context, dm DatasetMetadataToUpdate, etag string) (*DatasetMetadata, error) {
	return d.c.service.patchDataset(ctx, d.ProjectID, d.DatasetID, &dm, etag)
}

// Table creates a handle to a BigQuery table in the dataset.
// To determine if a table exists, call Table.Metadata.
// If the table does not already exist, use Table.Create to create it.
func (d *Dataset) Table(tableID string) *Table {
	return &Table{ProjectID: d.ProjectID, DatasetID: d.DatasetID, TableID: tableID, c: d.c}
}

// Tables returns an iterator over the tables in the Dataset.
func (d *Dataset) Tables(ctx context.Context) *TableIterator {
	it := &TableIterator{
		ctx:     ctx,
		dataset: d,
	}
	it.pageInfo, it.nextFunc = iterator.NewPageInfo(
		it.fetch,
		func() int { return len(it.tables) },
		func() interface{} { b := it.tables; it.tables = nil; return b })
	return it
}

// A TableIterator is an iterator over Tables.
type TableIterator struct {
	ctx      context.Context
	dataset  *Dataset
	tables   []*Table
	pageInfo *iterator.PageInfo
	nextFunc func() error
}

// Next returns the next result. Its second return value is Done if there are
// no more results. Once Next returns Done, all subsequent calls will return
// Done.
func (it *TableIterator) Next() (*Table, error) {
	if err := it.nextFunc(); err != nil {
		return nil, err
	}
	t := it.tables[0]
	it.tables = it.tables[1:]
	return t, nil
}

// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
func (it *TableIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }

func (it *TableIterator) fetch(pageSize int, pageToken string) (string, error) {
	tables, tok, err := it.dataset.c.service.listTables(it.ctx, it.dataset.ProjectID, it.dataset.DatasetID, pageSize, pageToken)
	if err != nil {
		return "", err
	}
	for _, t := range tables {
		t.c = it.dataset.c
		it.tables = append(it.tables, t)
	}
	return tok, nil
}

// Datasets returns an iterator over the datasets in the Client's project.
func (c *Client) Datasets(ctx context.Context) *DatasetIterator {
	return c.DatasetsInProject(ctx, c.projectID)
}

// DatasetsInProject returns an iterator over the datasets in the provided project.
func (c *Client) DatasetsInProject(ctx context.Context, projectID string) *DatasetIterator {
	it := &DatasetIterator{
		ctx:       ctx,
		c:         c,
		projectID: projectID,
	}
	it.pageInfo, it.nextFunc = iterator.NewPageInfo(
		it.fetch,
		func() int { return len(it.items) },
		func() interface{} { b := it.items; it.items = nil; return b })
	return it
}

// DatasetIterator iterates over the datasets in a project.
type DatasetIterator struct {
	// ListHidden causes hidden datasets to be listed when set to true.
	ListHidden bool

	// Filter restricts the datasets returned by label. The filter syntax is described in
	// https://cloud.google.com/bigquery/docs/labeling-datasets#filtering_datasets_using_labels
	Filter string

	ctx       context.Context
	projectID string
	c         *Client
	pageInfo  *iterator.PageInfo
	nextFunc  func() error
	items     []*Dataset
}

// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
func (it *DatasetIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }

func (it *DatasetIterator) Next() (*Dataset, error) {
	if err := it.nextFunc(); err != nil {
		return nil, err
	}
	item := it.items[0]
	it.items = it.items[1:]
	return item, nil
}

func (it *DatasetIterator) fetch(pageSize int, pageToken string) (string, error) {
	datasets, nextPageToken, err := it.c.service.listDatasets(it.ctx, it.projectID,
		pageSize, pageToken, it.ListHidden, it.Filter)
	if err != nil {
		return "", err
	}
	for _, d := range datasets {
		d.c = it.c
		it.items = append(it.items, d)
	}
	return nextPageToken, nil
}
<commit_msg>bigquery: export ProjectID on DatasetIterator<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bigquery

import (
	"time"

	"cloud.google.com/go/internal/optional"

	"golang.org/x/net/context"
	"google.golang.org/api/iterator"
)

// Dataset is a reference to a BigQuery dataset.
type Dataset struct {
	ProjectID string
	DatasetID string
	c         *Client
}

type DatasetMetadata struct {
	CreationTime           time.Time
	LastModifiedTime       time.Time     // When the dataset or any of its tables were modified.
	DefaultTableExpiration time.Duration // The default expiration time for new tables.
	Description            string        // The user-friendly description of this dataset.
	Name                   string        // The user-friendly name for this dataset.
	ID                     string
	Location               string            // The geo location of the dataset.
	Labels                 map[string]string // User-provided labels.

	// ETag is the ETag obtained when reading metadata. Pass it to Dataset.Update to
	// ensure that the metadata hasn't changed since it was read.
	ETag string
	// TODO(jba): access rules
}

type DatasetMetadataToUpdate struct {
	Description optional.String // The user-friendly description of this table.
	Name        optional.String // The user-friendly name for this dataset.
	// DefaultTableExpiration is the the default expiration time for new tables.
	// If set to time.Duration(0), new tables never expire.
	DefaultTableExpiration optional.Duration
}

// Dataset creates a handle to a BigQuery dataset in the client's project.
func (c *Client) Dataset(id string) *Dataset {
	return c.DatasetInProject(c.projectID, id)
}

// DatasetInProject creates a handle to a BigQuery dataset in the specified project.
func (c *Client) DatasetInProject(projectID, datasetID string) *Dataset {
	return &Dataset{
		ProjectID: projectID,
		DatasetID: datasetID,
		c:         c,
	}
}

// Create creates a dataset in the BigQuery service. An error will be returned
// if the dataset already exists.
func (d *Dataset) Create(ctx context.Context) error {
	return d.c.service.insertDataset(ctx, d.DatasetID, d.ProjectID)
}

// Delete deletes the dataset.
func (d *Dataset) Delete(ctx context.Context) error {
	return d.c.service.deleteDataset(ctx, d.DatasetID, d.ProjectID)
}

// Metadata fetches the metadata for the dataset.
func (d *Dataset) Metadata(ctx context.Context) (*DatasetMetadata, error) {
	return d.c.service.getDatasetMetadata(ctx, d.ProjectID, d.DatasetID)
}

// Update modifies specific Dataset metadata fields.
// To perform a read-modify-write that protects against intervening reads,
// set the etag argument to the DatasetMetadata.ETag field from the read.
// Pass the empty string for etag for a "blind write" that will always succeed.
func (d *Dataset) Update(ctx context.Context, dm DatasetMetadataToUpdate, etag string) (*DatasetMetadata, error) {
	return d.c.service.patchDataset(ctx, d.ProjectID, d.DatasetID, &dm, etag)
}

// Table creates a handle to a BigQuery table in the dataset.
// To determine if a table exists, call Table.Metadata.
// If the table does not already exist, use Table.Create to create it.
func (d *Dataset) Table(tableID string) *Table {
	return &Table{ProjectID: d.ProjectID, DatasetID: d.DatasetID, TableID: tableID, c: d.c}
}

// Tables returns an iterator over the tables in the Dataset.
func (d *Dataset) Tables(ctx context.Context) *TableIterator {
	it := &TableIterator{
		ctx:     ctx,
		dataset: d,
	}
	it.pageInfo, it.nextFunc = iterator.NewPageInfo(
		it.fetch,
		func() int { return len(it.tables) },
		func() interface{} { b := it.tables; it.tables = nil; return b })
	return it
}

// A TableIterator is an iterator over Tables.
type TableIterator struct {
	ctx      context.Context
	dataset  *Dataset
	tables   []*Table
	pageInfo *iterator.PageInfo
	nextFunc func() error
}

// Next returns the next result. Its second return value is Done if there are
// no more results. Once Next returns Done, all subsequent calls will return
// Done.
func (it *TableIterator) Next() (*Table, error) {
	if err := it.nextFunc(); err != nil {
		return nil, err
	}
	t := it.tables[0]
	it.tables = it.tables[1:]
	return t, nil
}

// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
func (it *TableIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }

func (it *TableIterator) fetch(pageSize int, pageToken string) (string, error) {
	tables, tok, err := it.dataset.c.service.listTables(it.ctx, it.dataset.ProjectID, it.dataset.DatasetID, pageSize, pageToken)
	if err != nil {
		return "", err
	}
	for _, t := range tables {
		t.c = it.dataset.c
		it.tables = append(it.tables, t)
	}
	return tok, nil
}

// Datasets returns an iterator over the datasets in a project.
// The Client's project is used by default, but that can be
// changed by setting ProjectID on the returned iterator before calling Next.
func (c *Client) Datasets(ctx context.Context) *DatasetIterator {
	return c.DatasetsInProject(ctx, c.projectID)
}

// DatasetsInProject returns an iterator over the datasets in the provided project.
//
// Deprecated: call Client.Datasets, then set ProjectID on the returned iterator.
func (c *Client) DatasetsInProject(ctx context.Context, projectID string) *DatasetIterator {
	it := &DatasetIterator{
		ctx:       ctx,
		c:         c,
		ProjectID: projectID,
	}
	it.pageInfo, it.nextFunc = iterator.NewPageInfo(
		it.fetch,
		func() int { return len(it.items) },
		func() interface{} { b := it.items; it.items = nil; return b })
	return it
}

// DatasetIterator iterates over the datasets in a project.
type DatasetIterator struct {
	// ListHidden causes hidden datasets to be listed when set to true.
	// Set before the first call to Next.
	ListHidden bool

	// Filter restricts the datasets returned by label. The filter syntax is described in
	// https://cloud.google.com/bigquery/docs/labeling-datasets#filtering_datasets_using_labels
	// Set before the first call to Next.
	Filter string

	// The project ID of the listed datasets.
	// Set before the first call to Next.
	ProjectID string

	ctx      context.Context
	c        *Client
	pageInfo *iterator.PageInfo
	nextFunc func() error
	items    []*Dataset
}

// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
func (it *DatasetIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }

func (it *DatasetIterator) Next() (*Dataset, error) {
	if err := it.nextFunc(); err != nil {
		return nil, err
	}
	item := it.items[0]
	it.items = it.items[1:]
	return item, nil
}

func (it *DatasetIterator) fetch(pageSize int, pageToken string) (string, error) {
	datasets, nextPageToken, err := it.c.service.listDatasets(it.ctx, it.ProjectID,
		pageSize, pageToken, it.ListHidden, it.Filter)
	if err != nil {
		return "", err
	}
	for _, d := range datasets {
		d.c = it.c
		it.items = append(it.items, d)
	}
	return nextPageToken, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"koding/db/mongodb/modelhelper"
	"koding/kites/common"
	"koding/kites/kloud/api/amazon"
	"koding/kites/kloud/api/sl"
	"koding/kites/kloud/contexthelper/publickeys"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/dnsstorage"
	"koding/kites/kloud/keycreator"
	"koding/kites/kloud/klient"
	"koding/kites/kloud/kloud"
	"koding/kites/kloud/pkg/dnsclient"
	"koding/kites/kloud/plans"
	"koding/kites/kloud/provider/aws"
	"koding/kites/kloud/provider/koding"
	"koding/kites/kloud/provider/softlayer"
	"koding/kites/kloud/userdata"
	"koding/kites/terraformer"
	"log"
	"net/url"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/net/context"

	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/koding/kite"
	"github.com/koding/kite/config"
	"github.com/koding/kite/kontrol"
	"github.com/koding/kite/testkeys"
	"github.com/koding/kite/testutil"
	"github.com/koding/klient/app"
	"github.com/koding/logging"

	// Klient uses vendored Kite, so we need to export it explicitly to make use of it
	klKite "github.com/koding/klient/Godeps/_workspace/src/github.com/koding/kite"
)

var (
	kloudKite      *kite.Kite
	klientKite     *klKite.Kite
	kld            *kloud.Kloud
	conf           *config.Config
	kodingProvider *koding.Provider
	awsProvider    *awsprovider.Provider
	slProvider     *softlayer.Provider
)

func main() {
	// This is somehow overriden by one of the packages here. I've searched for
	// hours with no luck. So I'm restoring it again back.
	logging.DefaultHandler = logging.NewWriterHandler(os.Stderr)

	if err := realMain(); err != nil {
		log.Fatalln(err)
	}
}

func realMain() error {
	if err := startInstances(); err != nil {
		return err
	}

	// now create a test kite which calls kloud or klient, respectively

	queryString := klientKite.Kite().String()

	userKite := kite.New("user", "0.0.1")
	username := "fatih"
	c := conf.Copy()
	c.KiteKey = testutil.NewKiteKeyUsername(username).Raw
	c.Username = username
	userKite.Config = c

	userKite.Log.Info("Searching for klient: %s", queryString)
	klientRef, err := klient.NewWithTimeout(userKite, queryString, time.Minute*5)
	if err != nil {
		return err
	}
	defer klientRef.Close()

	userKite.Log.Debug("Sending a ping message")
	if err := klientRef.Ping(); err != nil {
		return err
	}

	return nil
}

func startInstances() error {
	repoPath, err := currentRepoPath()
	if err != nil {
		return err
	}

	conf = config.New()
	conf.Username = "koding"

	conf.KontrolURL = os.Getenv("KLOUD_KONTROL_URL")
	if conf.KontrolURL == "" {
		conf.KontrolURL = "http://localhost:4099/kite"
	}

	conf.KontrolKey = testkeys.Public
	conf.KontrolUser = "koding"
	conf.KiteKey = testutil.NewKiteKey().Raw
	conf.Transport = config.XHRPolling

	// Power up our own kontrol kite for self-contained tests
	log.Println("Starting Kontrol Test Instance")
	kontrol.DefaultPort = 4099
	kntrl := kontrol.New(conf.Copy(), "0.1.0")
	p := kontrol.NewPostgres(nil, kntrl.Kite.Log)
	kntrl.SetKeyPairStorage(p)
	kntrl.SetStorage(p)
	kntrl.AddKeyPair("", testkeys.Public, testkeys.Private)

	go kntrl.Run()
	<-kntrl.Kite.ServerReadyNotify()

	// Power up kloud kite
	log.Println("Starting Kloud Test Instance")
	kloudKite = kite.New("kloud", "0.0.1")
	kloudKite.Config = conf.Copy()
	kloudKite.Config.Port = 4002
	kiteURL := &url.URL{Scheme: "http", Host: "localhost:4002", Path: "/kite"}
	_, err = kloudKite.Register(kiteURL)
	if err != nil {
		return fmt.Errorf("kloud: %s", err)
	}

	kodingProvider, awsProvider, slProvider = providers()

	// Add Kloud handlers
	kld := kloudWithProviders(kodingProvider, awsProvider, slProvider)
	kloudKite.HandleFunc("plan", kld.Plan)
	kloudKite.HandleFunc("apply", kld.Apply)
	kloudKite.HandleFunc("bootstrap", kld.Bootstrap)
	kloudKite.HandleFunc("authenticate", kld.Authenticate)

	kloudKite.HandleFunc("build", kld.Build)
	kloudKite.HandleFunc("destroy", kld.Destroy)
	kloudKite.HandleFunc("stop", kld.Stop)
	kloudKite.HandleFunc("start", kld.Start)
	kloudKite.HandleFunc("reinit", kld.Reinit)
	kloudKite.HandleFunc("resize", kld.Resize)
	kloudKite.HandleFunc("restart", kld.Restart)
	kloudKite.HandleFunc("event", kld.Event)
	kloudKite.HandleFunc("createSnapshot", kld.CreateSnapshot)
	kloudKite.HandleFunc("deleteSnapshot", kld.DeleteSnapshot)

	go kloudKite.Run()
	<-kloudKite.ServerReadyNotify()

	// Power up our terraformer kite
	log.Println("Starting Terraform Test Instance")
	tConf := &terraformer.Config{
		Port:        2300,
		Region:      "dev",
		Environment: "dev",
		AWS: terraformer.AWS{
			Key:    os.Getenv("TERRAFORMER_KEY"),
			Secret: os.Getenv("TERRAFORMER_SECRET"),
			Bucket: "koding-terraformer-state-dev",
		},
		LocalStorePath: filepath.Join(repoPath, filepath.FromSlash("go/data/terraformer")),
	}

	t, err := terraformer.New(tConf, common.NewLogger("terraformer", true))
	if err != nil {
		return fmt.Errorf("terraformer: %s", err)
	}

	terraformerKite, err := terraformer.NewKite(t, tConf)
	if err != nil {
		log.Fatal("terraformer ", err.Error())
	}

	// no need to register to kontrol, kloud talks directly via a secret key
	terraformerKite.Config = conf.Copy()
	terraformerKite.Config.Port = 2300

	go terraformerKite.Run()
	<-terraformerKite.ServerReadyNotify()

	log.Println("=== Test instances are up and ready!")

	// hashicorp.terraform outputs many logs, discard them
	log.SetOutput(ioutil.Discard)

	log.Println("=== Starting Klient now!!!")
	startKlient()
	return nil
}

func startKlient() {
	dbPath := ""
	u, err := user.Current()
	if err == nil {
		dbPath = filepath.Join(u.HomeDir, "/.config/koding/klient.bolt")
	}

	klientConf := &app.KlientConfig{
		Name:        "klient",
		Environment: "localhost",
		Region:      "localhost",
		Version:     "0.0.1",
		DBPath:      dbPath,
		IP:          "",
		Port:        56789,
		RegisterURL: "localhost:56789",
		KontrolURL:  conf.KontrolURL,
		// Debug:       true,
	}

	a := app.NewKlient(klientConf)
	klientKite = a.Kite()

	// Run Forrest, Run!
	go a.Run()
	<-klientKite.ServerReadyNotify()
}

func providers() (*koding.Provider, *awsprovider.Provider, *softlayer.Provider) {
	c := credentials.NewStaticCredentials(os.Getenv("KLOUD_ACCESSKEY"), os.Getenv("KLOUD_SECRETKEY"), "")

	mongoURL := os.Getenv("KLOUD_MONGODB_URL")
	if mongoURL == "" {
		panic("KLOUD_MONGODB_URL is not set")
	}

	modelhelper.Initialize(mongoURL)
	db := modelhelper.Mongo

	dnsInstance := dnsclient.NewRoute53Client(c, "dev.koding.io")
	dnsStorage := dnsstorage.NewMongodbStorage(db)
	usd := &userdata.Userdata{
		Keycreator: &keycreator.Key{
			KontrolURL:        conf.KontrolURL,
			KontrolPrivateKey: testkeys.Private,
			KontrolPublicKey:  testkeys.Public,
		},
		Bucket: userdata.NewBucket("koding-klient", "development/latest", c),
	}
	opts := &amazon.ClientOptions{
		Credentials: c,
		Regions:     amazon.ProductionRegions,
		Log:         common.NewLogger("koding", true),
	}

	ec2clients, err := amazon.NewClients(opts)
	if err != nil {
		panic(err)
	}

	slclient, err := sl.NewSoftlayer(
		os.Getenv("KLOUD_TESTACCOUNT_SLUSERNAME"),
		os.Getenv("KLOUD_TESTACCOUNT_SLAPIKEY"),
	)
	if err != nil {
		panic(err)
	}

	kdp := &koding.Provider{
		DB:             db,
		Log:            opts.Log,
		DNSClient:      dnsInstance,
		DNSStorage:     dnsStorage,
		Kite:           kloudKite,
		EC2Clients:     ec2clients,
		Userdata:       usd,
		PaymentFetcher: NewTestFetcher("hobbyist"),
		CheckerFetcher: NewTestChecker(),
	}

	awsp := &awsprovider.Provider{
		DB:         db,
		Log:        common.NewLogger("kloud-aws", true),
		DNSClient:  dnsInstance,
		DNSStorage: dnsStorage,
		Kite:       kloudKite,
		Userdata:   usd,
	}

	slp := &softlayer.Provider{
		DB:         db,
		Log:        common.NewLogger("kloud-provider", true),
		DNSClient:  dnsInstance,
		DNSStorage: dnsStorage,
		SLClient:   slclient,
		Kite:       kloudKite,
		Userdata:   usd,
	}

	return kdp, awsp, slp
}

func kloudWithProviders(p *koding.Provider, a *awsprovider.Provider, s *softlayer.Provider) *kloud.Kloud {
	debugEnabled := true
	kloudLogger := common.NewLogger("kloud", debugEnabled)
	sess := &session.Session{
		DB:         p.DB,
		Kite:       p.Kite,
		DNSClient:  p.DNSClient,
		DNSStorage: p.DNSStorage,
		AWSClients: p.EC2Clients,
		Userdata:   p.Userdata,
		Log:        kloudLogger,
	}

	kld := kloud.New()
	kld.ContextCreator = func(ctx context.Context) context.Context {
		return session.NewContext(ctx, sess)
	}

	userPrivateKey, userPublicKey := userMachinesKeys(
		os.Getenv("KLOUD_USER_PUBLICKEY"),
		os.Getenv("KLOUD_USER_PRIVATEKEY"),
	)

	kld.PublicKeys = &publickeys.Keys{
		KeyName:    publickeys.DeployKeyName,
		PrivateKey: userPrivateKey,
		PublicKey:  userPublicKey,
	}
	kld.Log = kloudLogger
	kld.DomainStorage = p.DNSStorage
	kld.Domainer = p.DNSClient
	kld.Locker = p
	kld.AddProvider("koding", p)
	kld.AddProvider("aws", a)
	kld.AddProvider("softlayer", s)
	return kld
}

// TestFetcher satisfies the fetcher interface
type TestFetcher struct {
	Plan string
}

func NewTestFetcher(plan string) *TestFetcher {
	return &TestFetcher{
		Plan: plan,
	}
}

func (t *TestFetcher) Fetch(ctx context.Context, username string) (*plans.PaymentResponse, error) {
	return &plans.PaymentResponse{
		Plan:  t.Plan,
		State: "active",
	}, nil
}

// TestFetcher satisfies the fetcher interface
type TestChecker struct{}

func NewTestChecker() *TestChecker {
	return &TestChecker{}
}

func (t *TestChecker) Fetch(ctx context.Context, plan string) (plans.Checker, error) {
	return &TestPlan{}, nil
}

type TestPlan struct{}

func (t *TestPlan) Total(username string) error {
	return nil
}

func (t *TestPlan) AlwaysOn(username string) error {
	return nil
}

func (t *TestPlan) SnapshotTotal(machineId, username string) error {
	return nil
}

func (t *TestPlan) Storage(wantStorage int, username string) error {
	return nil
}

func (t *TestPlan) AllowedInstances(wantInstance plans.InstanceType) error {
	return nil
}

func (t *TestPlan) NetworkUsage(username string) error {
	return nil
}

func userMachinesKeys(publicPath, privatePath string) (string, string) {
	pubKey, err := ioutil.ReadFile(publicPath)
	if err != nil {
		log.Fatalln(err)
	}
	publicKey := string(pubKey)

	privKey, err := ioutil.ReadFile(privatePath)
	if err != nil {
		log.Fatalln(err)
	}
	privateKey := string(privKey)

	return strings.TrimSpace(privateKey), strings.TrimSpace(publicKey)
}

// currentRepoPath returns the root path of current koding git repository
func currentRepoPath() (string, error) {
	p, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
	if err != nil {
		return "", err
	}
	return string(bytes.TrimSpace(p)), nil
}
<commit_msg>local: fix annoying logging, copy config to klient<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"koding/db/mongodb/modelhelper"
	"koding/kites/common"
	"koding/kites/kloud/api/amazon"
	"koding/kites/kloud/api/sl"
	"koding/kites/kloud/contexthelper/publickeys"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/dnsstorage"
	"koding/kites/kloud/keycreator"
	"koding/kites/kloud/klient"
	"koding/kites/kloud/kloud"
	"koding/kites/kloud/pkg/dnsclient"
	"koding/kites/kloud/plans"
	"koding/kites/kloud/provider/aws"
	"koding/kites/kloud/provider/koding"
	"koding/kites/kloud/provider/softlayer"
	"koding/kites/kloud/userdata"
	"koding/kites/terraformer"
	"log"
	"net/url"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/net/context"

	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/koding/kite"
	"github.com/koding/kite/config"
	"github.com/koding/kite/kontrol"
	"github.com/koding/kite/testkeys"
	"github.com/koding/kite/testutil"
	"github.com/koding/klient/app"
	"github.com/koding/logging"

	// Klient uses vendored Kite, so we need to export it explicitly to make use of it
	klKite "github.com/koding/klient/Godeps/_workspace/src/github.com/koding/kite"
	klKiteConf "github.com/koding/klient/Godeps/_workspace/src/github.com/koding/kite/config"
)

var (
	kloudKite      *kite.Kite
	klientKite     *klKite.Kite
	kld            *kloud.Kloud
	conf           *config.Config
	kodingProvider *koding.Provider
	awsProvider    *awsprovider.Provider
	slProvider     *softlayer.Provider
)

func main() {
	// This is somehow overriden by one of the packages here. I've searched for
	// hours with no luck. So I'm restoring it again back.
	handler := logging.NewWriterHandler(os.Stderr)
	handler.Colorize = true
	logging.DefaultHandler = handler

	if err := realMain(); err != nil {
		log.Fatalln(err)
	}
}

func realMain() error {
	if err := startInstances(); err != nil {
		return err
	}

	// now create a test kite which calls kloud or klient, respectively
	queryString := klientKite.Kite().String()

	userKite := kite.New("user", "0.0.1")
	username := "fatih"
	c := conf.Copy()
	c.KiteKey = testutil.NewKiteKeyUsername(username).Raw
	c.Username = username
	userKite.Config = c

	userKite.Log.Info("Searching for klient: %s", queryString)
	klientRef, err := klient.NewWithTimeout(userKite, queryString, time.Minute*1)
	if err != nil {
		return err
	}
	defer klientRef.Close()

	userKite.Log.Info("Sending a ping message")
	if err := klientRef.Ping(); err != nil {
		return err
	}

	return nil
}

func startInstances() error {
	repoPath, err := currentRepoPath()
	if err != nil {
		return err
	}

	conf = testutil.NewConfig() // username is "testuser"
	conf.Environment = "localhost-env"
	conf.Region = "localhost-region"
	conf.KontrolURL = os.Getenv("KLOUD_KONTROL_URL")
	if conf.KontrolURL == "" {
		conf.KontrolURL = "http://localhost:4099/kite"
	}
	conf.Transport = config.XHRPolling

	// Power up our own kontrol kite for self-contained tests
	log.Println("Starting Kontrol Test Instance")
	kontrol.DefaultPort = 4099
	kntrl := kontrol.New(conf.Copy(), "0.1.0")
	p := kontrol.NewPostgres(nil, kntrl.Kite.Log)
	kntrl.SetKeyPairStorage(p)
	kntrl.SetStorage(p)
	kntrl.AddKeyPair("", testkeys.Public, testkeys.Private)

	go kntrl.Run()
	<-kntrl.Kite.ServerReadyNotify()

	// Power up kloud kite
	log.Println("Starting Kloud Test Instance")
	kloudKite = kite.New("kloud", "0.0.1")
	kloudKite.Config = conf.Copy()
	kloudKite.Config.Port = 4002
	kiteURL := &url.URL{Scheme: "http", Host: "localhost:4002", Path: "/kite"}
	_, err = kloudKite.Register(kiteURL)
	if err != nil {
		return fmt.Errorf("kloud: %s", err)
	}

	kodingProvider, awsProvider, slProvider = providers()

	// Add Kloud handlers
	kld := kloudWithProviders(kodingProvider, awsProvider, slProvider)
	kloudKite.HandleFunc("plan", kld.Plan)
	kloudKite.HandleFunc("apply", kld.Apply)
	kloudKite.HandleFunc("bootstrap", kld.Bootstrap)
	kloudKite.HandleFunc("authenticate", kld.Authenticate)

	kloudKite.HandleFunc("build", kld.Build)
	kloudKite.HandleFunc("destroy", kld.Destroy)
	kloudKite.HandleFunc("stop", kld.Stop)
	kloudKite.HandleFunc("start", kld.Start)
	kloudKite.HandleFunc("reinit", kld.Reinit)
	kloudKite.HandleFunc("resize", kld.Resize)
	kloudKite.HandleFunc("restart", kld.Restart)
	kloudKite.HandleFunc("event", kld.Event)
	kloudKite.HandleFunc("createSnapshot", kld.CreateSnapshot)
	kloudKite.HandleFunc("deleteSnapshot", kld.DeleteSnapshot)

	go kloudKite.Run()
	<-kloudKite.ServerReadyNotify()

	// Power up our terraformer kite
	log.Println("Starting Terraform Test Instance")
	tConf := &terraformer.Config{
		Port:        2300,
		Region:      "dev",
		Environment: "dev",
		AWS: terraformer.AWS{
			Key:    os.Getenv("TERRAFORMER_KEY"),
			Secret: os.Getenv("TERRAFORMER_SECRET"),
			Bucket: "koding-terraformer-state-dev",
		},
		LocalStorePath: filepath.Join(repoPath, filepath.FromSlash("go/data/terraformer")),
	}

	t, err := terraformer.New(tConf, common.NewLogger("terraformer", true))
	if err != nil {
		return fmt.Errorf("terraformer: %s", err)
	}

	terraformerKite, err := terraformer.NewKite(t, tConf)
	if err != nil {
		log.Fatal("terraformer ", err.Error())
	}

	// no need to register to kontrol, kloud talks directly via a secret key
	terraformerKite.Config = conf.Copy()
	terraformerKite.Config.Port = 2300

	go terraformerKite.Run()
	<-terraformerKite.ServerReadyNotify()

	log.Println("=== Test instances are up and ready!")

	// hashicorp.terraform outputs many logs, discard them
	log.SetOutput(ioutil.Discard)

	log.Println("=== Starting Klient now!!!")
	startKlient()
	return nil
}

func startKlient() {
	dbPath := ""
	u, err := user.Current()
	if err == nil {
		dbPath = filepath.Join(u.HomeDir, "/.config/koding/klient.bolt")
	}

	klientConf := &app.KlientConfig{
		Name:        "klient",
		Environment: "localhost",
		Region:      "localhost",
		Version:     "0.0.1",
		DBPath:      dbPath,
		IP:          "",
		Port:        56789,
		RegisterURL: "http://localhost:56789/kite",
		KontrolURL:  conf.KontrolURL,
		// Debug:       true,
	}

	a := app.NewKlient(klientConf)
	klientKite = a.Kite()
	klientKite.Config = &klKiteConf.Config{
		Username:    conf.Username,
		Environment: conf.Environment,
		Region:      conf.Region,
		Id:          conf.Id,
		KiteKey:     conf.KiteKey,
		Transport:   klKiteConf.Transports[conf.Transport.String()],
		KontrolURL:  conf.KontrolURL,
		KontrolKey:  conf.KontrolKey,
		KontrolUser: conf.KontrolUser,
	}

	// Run Forrest, Run!
	go a.Run()
	<-klientKite.ServerReadyNotify()
}

func providers() (*koding.Provider, *awsprovider.Provider, *softlayer.Provider) {
	c := credentials.NewStaticCredentials(os.Getenv("KLOUD_ACCESSKEY"), os.Getenv("KLOUD_SECRETKEY"), "")

	mongoURL := os.Getenv("KLOUD_MONGODB_URL")
	if mongoURL == "" {
		panic("KLOUD_MONGODB_URL is not set")
	}

	modelhelper.Initialize(mongoURL)
	db := modelhelper.Mongo

	dnsInstance := dnsclient.NewRoute53Client(c, "dev.koding.io")
	dnsStorage := dnsstorage.NewMongodbStorage(db)
	usd := &userdata.Userdata{
		Keycreator: &keycreator.Key{
			KontrolURL:        conf.KontrolURL,
			KontrolPrivateKey: testkeys.Private,
			KontrolPublicKey:  testkeys.Public,
		},
		Bucket: userdata.NewBucket("koding-klient", "development/latest", c),
	}
	opts := &amazon.ClientOptions{
		Credentials: c,
		Regions:     amazon.ProductionRegions,
		Log:         common.NewLogger("koding", true),
	}

	ec2clients, err := amazon.NewClients(opts)
	if err != nil {
		panic(err)
	}

	slclient, err := sl.NewSoftlayer(
		os.Getenv("KLOUD_TESTACCOUNT_SLUSERNAME"),
		os.Getenv("KLOUD_TESTACCOUNT_SLAPIKEY"),
	)
	if err != nil {
		panic(err)
	}

	kdp := &koding.Provider{
		DB:             db,
		Log:            opts.Log,
		DNSClient:      dnsInstance,
		DNSStorage:     dnsStorage,
		Kite:           kloudKite,
		EC2Clients:     ec2clients,
		Userdata:       usd,
		PaymentFetcher: NewTestFetcher("hobbyist"),
		CheckerFetcher: NewTestChecker(),
	}

	awsp := &awsprovider.Provider{
		DB:         db,
		Log:        common.NewLogger("kloud-aws", true),
		DNSClient:  dnsInstance,
		DNSStorage: dnsStorage,
		Kite:       kloudKite,
		Userdata:   usd,
	}

	slp := &softlayer.Provider{
		DB:         db,
		Log:        common.NewLogger("kloud-provider", true),
		DNSClient:  dnsInstance,
		DNSStorage: dnsStorage,
		SLClient:   slclient,
		Kite:       kloudKite,
		Userdata:   usd,
	}

	return kdp, awsp, slp
}

func kloudWithProviders(p *koding.Provider, a *awsprovider.Provider, s *softlayer.Provider) *kloud.Kloud {
	debugEnabled := true
	kloudLogger := common.NewLogger("kloud", debugEnabled)
	sess := &session.Session{
		DB:         p.DB,
		Kite:       p.Kite,
		DNSClient:  p.DNSClient,
		DNSStorage: p.DNSStorage,
		AWSClients: p.EC2Clients,
		Userdata:   p.Userdata,
		Log:        kloudLogger,
	}

	kld := kloud.New()
	kld.ContextCreator = func(ctx context.Context) context.Context {
		return session.NewContext(ctx, sess)
	}

	userPrivateKey, userPublicKey := userMachinesKeys(
		os.Getenv("KLOUD_USER_PUBLICKEY"),
		os.Getenv("KLOUD_USER_PRIVATEKEY"),
	)

	kld.PublicKeys = &publickeys.Keys{
		KeyName:    publickeys.DeployKeyName,
		PrivateKey: userPrivateKey,
		PublicKey:  userPublicKey,
	}
	kld.Log = kloudLogger
	kld.DomainStorage = p.DNSStorage
	kld.Domainer = p.DNSClient
	kld.Locker = p
	kld.AddProvider("koding", p)
	kld.AddProvider("aws", a)
	kld.AddProvider("softlayer", s)
	return kld
}

// TestFetcher satisfies the fetcher interface
type TestFetcher struct {
	Plan string
}

func NewTestFetcher(plan string) *TestFetcher {
	return &TestFetcher{
		Plan: plan,
	}
}

func (t *TestFetcher) Fetch(ctx context.Context, username string) (*plans.PaymentResponse, error) {
	return &plans.PaymentResponse{
		Plan:  t.Plan,
		State: "active",
	}, nil
}

// TestFetcher satisfies the fetcher interface
type TestChecker struct{}

func NewTestChecker() *TestChecker {
	return &TestChecker{}
}

func (t *TestChecker) Fetch(ctx context.Context, plan string) (plans.Checker, error) {
	return &TestPlan{}, nil
}

type TestPlan struct{}

func (t *TestPlan) Total(username string) error {
	return nil
}

func (t *TestPlan) AlwaysOn(username string) error {
	return nil
}

func (t *TestPlan) SnapshotTotal(machineId, username string) error {
	return nil
}

func (t *TestPlan) Storage(wantStorage int, username string) error {
	return nil
}

func (t *TestPlan) AllowedInstances(wantInstance plans.InstanceType) error {
	return nil
}

func (t *TestPlan) NetworkUsage(username string) error {
	return nil
}

func userMachinesKeys(publicPath, privatePath string) (string, string) {
	pubKey, err := ioutil.ReadFile(publicPath)
	if err != nil {
		log.Fatalln(err)
	}
	publicKey := string(pubKey)

	privKey, err := ioutil.ReadFile(privatePath)
	if err != nil {
		log.Fatalln(err)
	}
	privateKey := string(privKey)

	return strings.TrimSpace(privateKey), strings.TrimSpace(publicKey)
}

// currentRepoPath returns the root path of current koding git repository
func currentRepoPath() (string, error) {
	p, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
	if err != nil {
		return "", err
	}
	return string(bytes.TrimSpace(p)), nil
}
<|endoftext|>


<commit_before>package machinegroup

import (
	"errors"
	"fmt"
	"path/filepath"

	"koding/klient/machine"
	"koding/klient/machine/mount"
	"koding/klient/machine/mount/sync"
)

// MountRequest defines machine group mount request.
type MountRequest struct {
	// ID is a unique identifier for the remote machine.
	ID machine.ID `json:"id"`

	// Mount describes the mount to be headed.
	Mount mount.Mount `json:"mount"`
}

// HeadMountRequest defines machine group head mount request.
type HeadMountRequest struct {
	MountRequest
}

// HeadMountResponse defines machine group head mount response.
type HeadMountResponse struct {
	// ExistMountID is not empty when mount to a given remote folder already exists.
	ExistMountID mount.ID `json:"existMountID,omitempty"`

	// AbsRemotePath stores absolute representation of remote path.
	AbsRemotePath string `json:"absRemotePath"`

	// AllCount stores the number of all files handled by mount.
	AllCount int `json:"allCount"`

	// AllDiskSize stores the size of all files handled by mount.
	AllDiskSize int64 `json:"allDiskSize"`
}

// HeadMount retrieves information about existing mount or prepares remote
// machine for mounting. It can tell in advance if remote directory exists and
// if it is possible to mount it. This function does not create any mount data.
func (g *Group) HeadMount(req *HeadMountRequest) (*HeadMountResponse, error) {
	if req == nil {
		return nil, errors.New("invalid nil request")
	}

	// Check if local path is not already mounted.
	switch mountID, err := g.mount.Path(req.Mount.Path); err {
	case nil:
		return nil, fmt.Errorf("path %q is already used by mount: %s", req.Mount.Path, mountID)
	case mount.ErrMountNotFound: // valid.
	default:
		return nil, err
	}

	c, err := g.client.Client(req.ID)
	if err != nil {
		return nil, err
	}

	absRemotePath, count, diskSize, err := c.MountHeadIndex(req.Mount.RemotePath)
	if err != nil {
		return nil, err
	}

	res := &HeadMountResponse{
		ExistMountID:  "",
		AbsRemotePath: absRemotePath,
		AllCount:      count,
		AllDiskSize:   diskSize,
	}

	// Check if remote folder of provided machine is already mounted.
	mountIDs, err := g.mount.RemotePath(absRemotePath)
	if err != nil {
		if err != mount.ErrMountNotFound {
			g.log.Warning("Cannot obtain list of mounts to %s remote directory: %s", absRemotePath, err)
		}
		return res, nil
	}

	for _, mountID := range mountIDs {
		id, err := g.mount.MachineID(mountID)
		if err != nil {
			continue
		}

		if id == req.ID {
			res.ExistMountID = mountID
			g.log.Warning("Remote machine %s mount to %s already exist: %s", id, absRemotePath, mountID)
			break
		}
	}

	return res, nil
}

// AddMountRequest defines machine group add mount request.
type AddMountRequest struct {
	MountRequest
}

// AddMountResponse defines machine group add mount response.
type AddMountResponse struct {
	// MountID is a unique identifier of created mount.
	MountID mount.ID `json:"mountID"`
}

// AddMount fetches remote index, prepares mount cache, and runs mount sync in
// the background. If this function returns nil, one have to call Unmount method
// in order to remove created mount.
func (g *Group) AddMount(req *AddMountRequest) (res *AddMountResponse, err error) {
	if req == nil {
		return nil, errors.New("invalid nil request")
	}

	// Immediately add mount to group, this prevents subtle data races when
	// mounts are added concurrently.
	mountID := mount.MakeID()
	if err = g.mount.Add(req.ID, mountID, req.Mount); err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			if e := g.mount.Remove(mountID); e != nil {
				g.log.Error("Cannot clean up mount data for %s", mountID)
			}
		}
	}()

	// Start mount synchronization.
	if err = g.sync.Add(mountID, req.Mount); err != nil {
		g.log.Error("Synchronization of %s mount failed: %s", mountID, err)
		return nil, err
	}

	g.log.Info("Successfully created mount %s for %s", mountID, req.Mount)

	return &AddMountResponse{
		MountID: mountID,
	}, nil
}

// ListMountRequest defines machine group mount list request.
type ListMountRequest struct {
	// ID is an optional identifier for the remote machine. If set, only
	// mounts related to this machine will be returned.
	ID machine.ID `json:"id,omitempty"`

	// MountID is an optional identifier of a mount which is meant to be listed.
	MountID mount.ID `json:"mountID,omitempty"`
}

// ListMountResponse defines machine group mount list response.
type ListMountResponse struct {
	// Mounts is a map that contains machine aliases as keys and created mount
	// infos which store the current synchronization status of mount.
	Mounts map[string][]sync.Info `json:"mounts"`
}

// ListMount checks the status of mounts and returns their infos. This function
// guarantees that if returned error is nil, response Mounts field is always
// non-nil. It doesn't fail if provided filters contain incorrect values.
func (g *Group) ListMount(req *ListMountRequest) (*ListMountResponse, error) {
	mms, err := g.getMounts(req)
	if err != nil {
		return nil, err
	}

	res := &ListMountResponse{
		Mounts: make(map[string][]sync.Info),
	}

	// Get machine aliases.
	var id2alias = make(map[machine.ID]string)
	for _, mm := range mms {
		if _, ok := id2alias[mm.id]; ok {
			continue
		}

		// Create does not generate new alias if it already exists.
		alias, err := g.alias.Create(mm.id)
		if err != nil {
			// Instead of failing here, use machine ID directly.
			id2alias[mm.id] = string(mm.id)
		} else {
			id2alias[mm.id] = alias
		}
	}

	// Get mount infos.
	for mountID, mm := range mms {
		info, err := g.sync.Info(mountID)
		alias := id2alias[mm.id]
		if err != nil {
			// Add mount to the list but log not synchronized mount.
			res.Mounts[alias] = append(res.Mounts[alias], sync.Info{
				ID:    mountID,
				Mount: mm.m,
			})
			g.log.Warning("Mount %s for %s is not synchronized: %s", mountID, mm.m, err)
			continue
		}

		res.Mounts[alias] = append(res.Mounts[alias], *info)
	}

	return res, nil
}

// mountsMachine is a helper type that contains mount paths and the machine
// which serves its remote path.
type mountsMachine struct {
	m  mount.Mount
	id machine.ID
}

// getRequestedMounts uses filters defined in list mount request to obtain
// list of requested mounts.
func (g *Group) getMounts(req *ListMountRequest) (map[mount.ID]mountsMachine, error) {
	if req == nil {
		return nil, errors.New("invalid nil request")
	}

	// Get list of requested mounts with machine ID filter.
	var mms = make(map[mount.ID]mountsMachine)
	if req.ID != "" {
		mounts, err := g.mount.All(req.ID)
		if err != nil {
			return mms, nil
		}

		for mountID, m := range mounts {
			mms[mountID] = mountsMachine{m: m, id: req.ID}
		}
	}

	// Get requested mount with mount ID filter.
	if req.MountID != "" {
		mms = map[mount.ID]mountsMachine{} // Clear map.

		id, err := g.mount.MachineID(req.MountID)
		if err != nil || (req.ID != "" && id != req.ID) {
			return mms, nil
		}

		mounts, err := g.mount.All(id)
		if err != nil {
			return mms, nil
		}

		mms[req.MountID] = mountsMachine{m: mounts[req.MountID], id: id}
	}

	// Get all mounts when filters are not set.
	if req.ID == "" && req.MountID == "" {
		for _, id := range g.mount.Registered() {
			mounts, err := g.mount.All(id)
			if err != nil {
				continue
			}

			for mountID, m := range mounts {
				mms[mountID] = mountsMachine{m: m, id: id}
			}
		}
	}

	return mms, nil
}

// UmountRequest defines machine group umount request.
type UmountRequest struct {
	// Identifier is a string that identifiers requested mount. It can be either
	// mount ID or local path which is going to be unmounted.
	Identifier string `json:"identifier"`
}

// UmountResponse defines machine group umount response.
type UmountResponse struct {
	// MountID is a unique identifier of removed mount.
	MountID mount.ID `json:"mountID"`

	// Mount defines the mount which was removed.
	Mount mount.Mount `json:"mount"`
}

// Umount removes existing mount. It cleans both bolt and mount cache.
func (g *Group) Umount(req *UmountRequest) (res *UmountResponse, err error) {
	if req == nil {
		return nil, errors.New("invalid nil request")
	}

	// Get mount ID from identifier.
	mountID, err := mount.IDFromString(req.Identifier)
	if err != nil {
		absPath, e := filepath.Abs(req.Identifier)
		if mountID, err = g.mount.Path(absPath); e != nil || err != nil {
			g.log.Error("Cannot found mount with identifier: %s", req.Identifier)
			return nil, fmt.Errorf("unknown mount: %q", req.Identifier)
		}
	}

	// Stop mount synchronization routine.
	if err := g.sync.Drop(mountID); err != nil {
		g.log.Error("Cannot remove synced mount %s: %s", mountID, err)
		return nil, err
	}

	// Get mount machine.
	id, err := g.mount.MachineID(mountID)
	if err != nil {
		g.log.Error("Could not find machine ID for mount %s: %s", mountID, err)
	}

	var m mount.Mount
	// Get mount object. Ignore errors since mount is only for logging purposes.
	if mounts, err := g.mount.All(id); err == nil {
		m = mounts[mountID]
	}

	// Remove mount from cache. If this operation fails, sync process does not
	// to be restarted.
	if err := g.mount.Remove(mountID); err != nil {
		g.log.Error("Cannot clear mount cache for %s: %s", mountID, err)
	}

	g.log.Info("Successfully removed mount %s for %s", mountID, m)

	return &UmountResponse{
		MountID: mountID,
		Mount:   m,
	}, nil
}

// UmountRequest defines machine group umount request.
type UmountRequest struct {
	// Identifier is a string that identifiers requested mount. It can be either
	// mount ID or local path which is going to be unmounted.
	Identifier string `json:"identifier"`
}

// UmountResponse defines machine group umount response.
type UmountResponse struct {
	// MountID is a unique identifier of removed mount.
	MountID mount.ID `json:"mountID"`

	// Mount defines the mount which was removed.
	Mount mount.Mount `json:"mount"`
}

// Umount removes existing mount. It cleans both bolt and mount cache.
func (g *Group) Umount(req *UmountRequest) (res *UmountResponse, err error) {
	if req == nil {
		return nil, errors.New("invalid nil request")
	}

	// Get mount ID from identifier.
	mountID, err := mount.IDFromString(req.Identifier)
	if err != nil {
		absPath, e := filepath.Abs(req.Identifier)
		if mountID, err = g.mount.Path(absPath); e != nil || err != nil {
			return nil, fmt.Errorf("unknown mount: %q", req.Identifier)
		}
	}

	// Stop mount synchronization routine.
	if err := g.sync.Drop(mountID); err != nil {
		g.log.Error("Cannot remove synced mount %s: %s", mountID, err)
		return nil, err
	}

	// Remove mount from cache. If this operation fails, sync process does not
	// to be restarted.
	if err := g.mount.Remove(mountID); err != nil {
		g.log.Error("Cannot clear mount cache for %s: %s", mountID, err)
	}

	// Get mount machine.
	id, err := g.mount.MachineID(mountID)
	if err != nil {
		g.log.Error("Could not find machine ID for mount %s: %s", mountID, err)
	}

	var m mount.Mount
	// Get mount object. Ignore errors since mount is only for logging purposes.
	if mounts, err := g.mount.All(id); err == nil {
		m = mounts[mountID]
	}

	g.log.Info("Successfully removed mount %s for %s", mountID, m)

	return &UmountResponse{
		MountID: mountID,
		Mount:   m,
	}, nil
}
<commit_msg>klient/machine: remove duplicated umount method<commit_after>package machinegroup

import (
	"errors"
	"fmt"
	"path/filepath"

	"koding/klient/machine"
	"koding/klient/machine/mount"
	"koding/klient/machine/mount/sync"
)

// MountRequest defines machine group mount request.
type MountRequest struct {
	// ID is a unique identifier for the remote machine.
	ID machine.ID `json:"id"`

	// Mount describes the mount to be headed.
	Mount mount.Mount `json:"mount"`
}

// HeadMountRequest defines machine group head mount request.
type HeadMountRequest struct {
	MountRequest
}

// HeadMountResponse defines machine group head mount response.
type HeadMountResponse struct {
	// ExistMountID is not empty when mount to a given remote folder already exists.
	ExistMountID mount.ID `json:"existMountID,omitempty"`

	// AbsRemotePath stores absolute representation of remote path.
	AbsRemotePath string `json:"absRemotePath"`

	// AllCount stores the number of all files handled by mount.
	AllCount int `json:"allCount"`

	// AllDiskSize stores the size of all files handled by mount.
	AllDiskSize int64 `json:"allDiskSize"`
}

// HeadMount retrieves information about existing mount or prepares remote
// machine for mounting. It can tell in advance if remote directory exists and
// if it is possible to mount it. This function does not create any mount data.
func (g *Group) HeadMount(req *HeadMountRequest) (*HeadMountResponse, error) {
	if req == nil {
		return nil, errors.New("invalid nil request")
	}

	// Check if local path is not already mounted.
	switch mountID, err := g.mount.Path(req.Mount.Path); err {
	case nil:
		return nil, fmt.Errorf("path %q is already used by mount: %s", req.Mount.Path, mountID)
	case mount.ErrMountNotFound: // valid.
	default:
		return nil, err
	}

	c, err := g.client.Client(req.ID)
	if err != nil {
		return nil, err
	}

	absRemotePath, count, diskSize, err := c.MountHeadIndex(req.Mount.RemotePath)
	if err != nil {
		return nil, err
	}

	res := &HeadMountResponse{
		ExistMountID:  "",
		AbsRemotePath: absRemotePath,
		AllCount:      count,
		AllDiskSize:   diskSize,
	}

	// Check if remote folder of provided machine is already mounted.
	mountIDs, err := g.mount.RemotePath(absRemotePath)
	if err != nil {
		if err != mount.ErrMountNotFound {
			g.log.Warning("Cannot obtain list of mounts to %s remote directory: %s", absRemotePath, err)
		}
		return res, nil
	}

	for _, mountID := range mountIDs {
		id, err := g.mount.MachineID(mountID)
		if err != nil {
			continue
		}

		if id == req.ID {
			res.ExistMountID = mountID
			g.log.Warning("Remote machine %s mount to %s already exist: %s", id, absRemotePath, mountID)
			break
		}
	}

	return res, nil
}

// AddMountRequest defines machine group add mount request.
type AddMountRequest struct {
	MountRequest
}

// AddMountResponse defines machine group add mount response.
type AddMountResponse struct {
	// MountID is a unique identifier of created mount.
	MountID mount.ID `json:"mountID"`
}

// AddMount fetches remote index, prepares mount cache, and runs mount sync in
// the background. If this function returns nil, one have to call Unmount method
// in order to remove created mount.
func (g *Group) AddMount(req *AddMountRequest) (res *AddMountResponse, err error) {
	if req == nil {
		return nil, errors.New("invalid nil request")
	}

	// Immediately add mount to group, this prevents subtle data races when
	// mounts are added concurrently.
	mountID := mount.MakeID()
	if err = g.mount.Add(req.ID, mountID, req.Mount); err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			if e := g.mount.Remove(mountID); e != nil {
				g.log.Error("Cannot clean up mount data for %s", mountID)
			}
		}
	}()

	// Start mount synchronization.
	if err = g.sync.Add(mountID, req.Mount); err != nil {
		g.log.Error("Synchronization of %s mount failed: %s", mountID, err)
		return nil, err
	}

	g.log.Info("Successfully created mount %s for %s", mountID, req.Mount)

	return &AddMountResponse{
		MountID: mountID,
	}, nil
}

// ListMountRequest defines machine group mount list request.
type ListMountRequest struct {
	// ID is an optional identifier for the remote machine. If set, only
	// mounts related to this machine will be returned.
	ID machine.ID `json:"id,omitempty"`

	// MountID is an optional identifier of a mount which is meant to be listed.
	MountID mount.ID `json:"mountID,omitempty"`
}

// ListMountResponse defines machine group mount list response.
type ListMountResponse struct {
	// Mounts is a map that contains machine aliases as keys and created mount
	// infos which store the current synchronization status of mount.
	Mounts map[string][]sync.Info `json:"mounts"`
}

// ListMount checks the status of mounts and returns their infos. This function
// guarantees that if returned error is nil, response Mounts field is always
// non-nil. It doesn't fail if provided filters contain incorrect values.
func (g *Group) ListMount(req *ListMountRequest) (*ListMountResponse, error) {
	mms, err := g.getMounts(req)
	if err != nil {
		return nil, err
	}

	res := &ListMountResponse{
		Mounts: make(map[string][]sync.Info),
	}

	// Get machine aliases.
	var id2alias = make(map[machine.ID]string)
	for _, mm := range mms {
		if _, ok := id2alias[mm.id]; ok {
			continue
		}

		// Create does not generate new alias if it already exists.
		alias, err := g.alias.Create(mm.id)
		if err != nil {
			// Instead of failing here, use machine ID directly.
			id2alias[mm.id] = string(mm.id)
		} else {
			id2alias[mm.id] = alias
		}
	}

	// Get mount infos.
	for mountID, mm := range mms {
		info, err := g.sync.Info(mountID)
		alias := id2alias[mm.id]
		if err != nil {
			// Add mount to the list but log not synchronized mount.
			res.Mounts[alias] = append(res.Mounts[alias], sync.Info{
				ID:    mountID,
				Mount: mm.m,
			})
			g.log.Warning("Mount %s for %s is not synchronized: %s", mountID, mm.m, err)
			continue
		}

		res.Mounts[alias] = append(res.Mounts[alias], *info)
	}

	return res, nil
}

// mountsMachine is a helper type that contains mount paths and the machine
// which serves its remote path.
type mountsMachine struct {
	m  mount.Mount
	id machine.ID
}

// getRequestedMounts uses filters defined in list mount request to obtain
// list of requested mounts.
func (g *Group) getMounts(req *ListMountRequest) (map[mount.ID]mountsMachine, error) {
	if req == nil {
		return nil, errors.New("invalid nil request")
	}

	// Get list of requested mounts with machine ID filter.
	var mms = make(map[mount.ID]mountsMachine)
	if req.ID != "" {
		mounts, err := g.mount.All(req.ID)
		if err != nil {
			return mms, nil
		}

		for mountID, m := range mounts {
			mms[mountID] = mountsMachine{m: m, id: req.ID}
		}
	}

	// Get requested mount with mount ID filter.
	if req.MountID != "" {
		mms = map[mount.ID]mountsMachine{} // Clear map.

		id, err := g.mount.MachineID(req.MountID)
		if err != nil || (req.ID != "" && id != req.ID) {
			return mms, nil
		}

		mounts, err := g.mount.All(id)
		if err != nil {
			return mms, nil
		}

		mms[req.MountID] = mountsMachine{m: mounts[req.MountID], id: id}
	}

	// Get all mounts when filters are not set.
	if req.ID == "" && req.MountID == "" {
		for _, id := range g.mount.Registered() {
			mounts, err := g.mount.All(id)
			if err != nil {
				continue
			}

			for mountID, m := range mounts {
				mms[mountID] = mountsMachine{m: m, id: id}
			}
		}
	}

	return mms, nil
}

// UmountRequest defines machine group umount request.
type UmountRequest struct {
	// Identifier is a string that identifiers requested mount. It can be either
	// mount ID or local path which is going to be unmounted.
	Identifier string `json:"identifier"`
}

// UmountResponse defines machine group umount response.
type UmountResponse struct {
	// MountID is a unique identifier of removed mount.
	MountID mount.ID `json:"mountID"`

	// Mount defines the mount which was removed.
	Mount mount.Mount `json:"mount"`
}

// Umount removes existing mount. It cleans both bolt and mount cache.
func (g *Group) Umount(req *UmountRequest) (res *UmountResponse, err error) {
	if req == nil {
		return nil, errors.New("invalid nil request")
	}

	// Get mount ID from identifier.
	mountID, err := mount.IDFromString(req.Identifier)
	if err != nil {
		absPath, e := filepath.Abs(req.Identifier)
		if mountID, err = g.mount.Path(absPath); e != nil || err != nil {
			g.log.Error("Cannot found mount with identifier: %s", req.Identifier)
			return nil, fmt.Errorf("unknown mount: %q", req.Identifier)
		}
	}

	// Stop mount synchronization routine.
	if err := g.sync.Drop(mountID); err != nil {
		g.log.Error("Cannot remove synced mount %s: %s", mountID, err)
		return nil, err
	}

	// Get mount machine.
	id, err := g.mount.MachineID(mountID)
	if err != nil {
		g.log.Error("Could not find machine ID for mount %s: %s", mountID, err)
	}

	var m mount.Mount
	// Get mount object. Ignore errors since mount is only for logging purposes.
	if mounts, err := g.mount.All(id); err == nil {
		m = mounts[mountID]
	}

	// Remove mount from cache. If this operation fails, sync process does not
	// to be restarted.
	if err := g.mount.Remove(mountID); err != nil {
		g.log.Error("Cannot clear mount cache for %s: %s", mountID, err)
	}

	g.log.Info("Successfully removed mount %s for %s", mountID, m)

	return &UmountResponse{
		MountID: mountID,
		Mount:   m,
	}, nil
}
<|endoftext|>