<commit_before>package agent

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"runtime"
	// "strconv"
	"strings"
	"sync"
	"time"

	"github.com/codegangsta/cli"

	"github.com/subutai-io/base/agent/agent/alert"
	"github.com/subutai-io/base/agent/agent/connect"
	"github.com/subutai-io/base/agent/agent/container"
	"github.com/subutai-io/base/agent/agent/executer"
	"github.com/subutai-io/base/agent/agent/utils"
	"github.com/subutai-io/base/agent/cli"
	"github.com/subutai-io/base/agent/config"
	cont "github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/lib/gpg"
	"github.com/subutai-io/base/agent/log"
)

type Response struct {
	Beat Heartbeat `json:"response"`
}

type Heartbeat struct {
	Type       string                `json:"type"`
	Hostname   string                `json:"hostname"`
	Id         string                `json:"id"`
	Arch       string                `json:"arch"`
	Instance   string                `json:"instance"`
	Interfaces []utils.Iface         `json:"interfaces,omitempty"`
	Containers []container.Container `json:"containers"`
	Alert      []alert.Load          `json:"alert, omitempty"`
}

var (
	lastHeartbeat []byte
	mutex         sync.Mutex
)

func initAgent() {
	// move .gnupg dir to app home
	os.Setenv("GNUPGHOME", config.Agent.DataPrefix+".gnupg")

	if cont.State("management") == "STOPPED" {
		cont.Start("management")
	}
	container.PoolInstance()
	Instance()
}

func Start(c *cli.Context) {
	http.HandleFunc("/trigger", trigger)
	http.HandleFunc("/ping", ping)
	http.HandleFunc("/heartbeat", heartbeatCall)
	go http.ListenAndServe(":7070", nil)

	go container.ContainersRestoreState()
	go lib.Collect()
	initAgent()

	for {
		// log.Debug("NumGoroutine " + strconv.Itoa(runtime.NumGoroutine()))
		Instance()
		if heartbeat() {
			time.Sleep(30 * time.Second)
		} else {
			time.Sleep(5 * time.Second)
		}
	}
}

func heartbeat() bool {
	mutex.Lock()
	defer mutex.Unlock()

	client := tlsConfig()
	hostname, _ := os.Hostname()
	instancetype := utils.InstanceType()
	fingerprint := gpg.GetFingerprint(hostname + "@subutai.io")

	list := container.GetActiveContainers(false)

	beat := Heartbeat{
		Type:       "HEARTBEAT",
		Hostname:   hostname,
		Id:         fingerprint,
		Arch:       strings.ToUpper(runtime.GOARCH),
		Instance:   instancetype,
		Containers: list,
		Interfaces: utils.GetInterfaces(),
		Alert:      alert.Alert(list),
	}
	res := Response{Beat: beat}
	jbeat, _ := json.Marshal(&res)

	if string(jbeat) == string(lastHeartbeat) {
		return true
	}
	lastHeartbeat = jbeat

	message, err := json.Marshal(map[string]string{
		"hostId":   fingerprint,
		"response": gpg.EncryptWrapper(config.Agent.GpgUser, config.Management.GpgUser, string(jbeat)),
	})
	log.Check(log.WarnLevel, "Marshal response json", err)

	resp, err := client.PostForm("https://"+config.Management.Host+":8444/rest/v1/agent/heartbeat", url.Values{"heartbeat": {string(message)}})
	if !log.Check(log.WarnLevel, "Sending heartbeat: "+string(jbeat), err) {
		resp.Body.Close()
	} else {
		connect.Connect(config.Management.Host, config.Management.Port, config.Agent.GpgUser, config.Management.Secret)
		return false
	}
	return true
}

func execute(rsp executer.EncRequest) {
	var req executer.Request
	var md, contName, pub, keyring, payload string
	var flag bool
	var err error

	hostname, _ := os.Hostname()
	fingerprint := gpg.GetFingerprint(hostname + "@subutai.io")

	if rsp.HostId == fingerprint {
		flag = true
		md = gpg.DecryptWrapper(rsp.Request)
	} else {
		flag = false
		contName, err = container.PoolInstance().GetTargetHostName(rsp.HostId)
		counter := 0
		for err != nil {
			contName, err = container.PoolInstance().GetTargetHostName(rsp.HostId)
			time.Sleep(time.Second * 3)
			if counter = counter + 1; counter == 100 {
				log.Warn("Container wait timeout: " + contName)
				return
			}
		}

		pub = config.Agent.LxcPrefix + contName + "/public.pub"
		keyring = config.Agent.LxcPrefix + contName + "/secret.sec"
		log.Info("Getting public keyring", "keyring", keyring)
		md = gpg.DecryptNoDefaultKeyring(rsp.Request, keyring, pub)
	}

	i := strings.Index(md, "{")
	j := strings.LastIndex(md, "}") + 1
	if i > j && i > 0 {
		log.Warn("Error getting JSON request")
		return
	}
	request := md[i:j]

	err = json.Unmarshal([]byte(request), &req.Request)
	log.Check(log.WarnLevel, "Decrypting request", err)

	//create channels for stdout and stderr
	sOut := make(chan executer.ResponseOptions)
	req.Execute(flag, sOut)

	for sOut != nil {
		select {
		case elem, ok := <-sOut:
			if ok {
				resp := executer.Response{ResponseOpts: elem}
				jsonR, err := json.Marshal(resp)
				log.Check(log.WarnLevel, "Marshal response", err)
				if rsp.HostId == fingerprint {
					payload = gpg.EncryptWrapper(config.Agent.GpgUser, config.Management.GpgUser, string(jsonR))
				} else {
					payload = gpg.EncryptWrapperNoDefaultKeyring(contName, config.Management.GpgUser, string(jsonR), pub, keyring)
				}
				message, err := json.Marshal(map[string]string{
					"hostId":   elem.Id,
					"response": payload,
				})
				log.Check(log.WarnLevel, "Marshal response json "+elem.CommandId, err)
				response(message)
			} else {
				sOut = nil
			}
		}
	}
	heartbeat()
}

func tlsConfig() *http.Client {
	tlsconfig := newTLSConfig()
	for tlsconfig == nil || len(tlsconfig.Certificates[0].Certificate) == 0 {
		time.Sleep(time.Second * 2)
		for utils.PublicCert() == "" {
			x509generate()
		}
		tlsconfig = newTLSConfig()
	}

	transport := &http.Transport{TLSClientConfig: tlsconfig}
	client := &http.Client{Transport: transport}
	return client
}

func response(msg []byte) {
	client := tlsConfig()
	resp, err := client.PostForm("https://"+config.Management.Host+":8444/rest/v1/agent/response", url.Values{"response": {string(msg)}})
	if !log.Check(log.WarnLevel, "Sending response "+string(msg), err) {
		resp.Body.Close()
	}
}

func command() {
	var rsp []executer.EncRequest
	client := tlsConfig()
	hostname, _ := os.Hostname()
	fingerprint := gpg.GetFingerprint(hostname + "@subutai.io")

	resp, err := client.Get("https://" + config.Management.Host + ":8444/rest/v1/agent/requests/" + fingerprint)
	if log.Check(log.WarnLevel, "Getting requests", err) {
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNoContent {
		return
	}

	data, err := ioutil.ReadAll(resp.Body)
	if !log.Check(log.WarnLevel, "Reading body", err) {
		log.Check(log.WarnLevel, "Unmarshal payload", json.Unmarshal(data, &rsp))
		for _, request := range rsp {
			execute(request)
		}
	}
}

func ping(rw http.ResponseWriter, request *http.Request) {
	if request.Method == http.MethodGet && strings.Split(request.RemoteAddr, ":")[0] == config.Management.Host {
		rw.WriteHeader(http.StatusOK)
		heartbeat()
	} else {
		rw.WriteHeader(http.StatusForbidden)
	}
}

func trigger(rw http.ResponseWriter, request *http.Request) {
	if request.Method == http.MethodPost && strings.Split(request.RemoteAddr, ":")[0] == config.Management.Host {
		rw.WriteHeader(http.StatusAccepted)
		command()
	} else {
		rw.WriteHeader(http.StatusForbidden)
	}
}

func heartbeatCall(rw http.ResponseWriter, request *http.Request) {
	if request.Method == http.MethodGet && strings.Split(request.RemoteAddr, ":")[0] == config.Management.Host {
		rw.WriteHeader(http.StatusOK)
		lastHeartbeat = []byte{}
		heartbeat()
	} else {
		rw.WriteHeader(http.StatusForbidden)
	}
}
<commit_msg>Removed Agent heartbeat call from /ping<commit_after>package agent

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"runtime"
	// "strconv"
	"strings"
	"sync"
	"time"

	"github.com/codegangsta/cli"

	"github.com/subutai-io/base/agent/agent/alert"
	"github.com/subutai-io/base/agent/agent/connect"
	"github.com/subutai-io/base/agent/agent/container"
	"github.com/subutai-io/base/agent/agent/executer"
	"github.com/subutai-io/base/agent/agent/utils"
	"github.com/subutai-io/base/agent/cli"
	"github.com/subutai-io/base/agent/config"
	cont "github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/lib/gpg"
	"github.com/subutai-io/base/agent/log"
)

type Response struct {
	Beat Heartbeat `json:"response"`
}

type Heartbeat struct {
	Type       string                `json:"type"`
	Hostname   string                `json:"hostname"`
	Id         string                `json:"id"`
	Arch       string                `json:"arch"`
	Instance   string                `json:"instance"`
	Interfaces []utils.Iface         `json:"interfaces,omitempty"`
	Containers []container.Container `json:"containers"`
	Alert      []alert.Load          `json:"alert, omitempty"`
}

var (
	lastHeartbeat []byte
	mutex         sync.Mutex
)

func initAgent() {
	// move .gnupg dir to app home
	os.Setenv("GNUPGHOME", config.Agent.DataPrefix+".gnupg")

	if cont.State("management") == "STOPPED" {
		cont.Start("management")
	}
	container.PoolInstance()
	Instance()
}

func Start(c *cli.Context) {
	http.HandleFunc("/trigger", trigger)
	http.HandleFunc("/ping", ping)
	http.HandleFunc("/heartbeat", heartbeatCall)
	go http.ListenAndServe(":7070", nil)

	go container.ContainersRestoreState()
	go lib.Collect()
	initAgent()

	for {
		// log.Debug("NumGoroutine " + strconv.Itoa(runtime.NumGoroutine()))
		Instance()
		if heartbeat() {
			time.Sleep(30 * time.Second)
		} else {
			time.Sleep(5 * time.Second)
		}
	}
}

func heartbeat() bool {
	mutex.Lock()
	defer mutex.Unlock()

	client := tlsConfig()
	hostname, _ := os.Hostname()
	instancetype := utils.InstanceType()
	fingerprint := gpg.GetFingerprint(hostname + "@subutai.io")

	list := container.GetActiveContainers(false)

	beat := Heartbeat{
		Type:       "HEARTBEAT",
		Hostname:   hostname,
		Id:         fingerprint,
		Arch:       strings.ToUpper(runtime.GOARCH),
		Instance:   instancetype,
		Containers: list,
		Interfaces: utils.GetInterfaces(),
		Alert:      alert.Alert(list),
	}
	res := Response{Beat: beat}
	jbeat, _ := json.Marshal(&res)

	if string(jbeat) == string(lastHeartbeat) {
		return true
	}
	lastHeartbeat = jbeat

	message, err := json.Marshal(map[string]string{
		"hostId":   fingerprint,
		"response": gpg.EncryptWrapper(config.Agent.GpgUser, config.Management.GpgUser, string(jbeat)),
	})
	log.Check(log.WarnLevel, "Marshal response json", err)

	resp, err := client.PostForm("https://"+config.Management.Host+":8444/rest/v1/agent/heartbeat", url.Values{"heartbeat": {string(message)}})
	if !log.Check(log.WarnLevel, "Sending heartbeat: "+string(jbeat), err) {
		resp.Body.Close()
	} else {
		connect.Connect(config.Management.Host, config.Management.Port, config.Agent.GpgUser, config.Management.Secret)
		return false
	}
	return true
}

func execute(rsp executer.EncRequest) {
	var req executer.Request
	var md, contName, pub, keyring, payload string
	var flag bool
	var err error

	hostname, _ := os.Hostname()
	fingerprint := gpg.GetFingerprint(hostname + "@subutai.io")

	if rsp.HostId == fingerprint {
		flag = true
		md = gpg.DecryptWrapper(rsp.Request)
	} else {
		flag = false
		contName, err = container.PoolInstance().GetTargetHostName(rsp.HostId)
		counter := 0
		for err != nil {
			contName, err = container.PoolInstance().GetTargetHostName(rsp.HostId)
			time.Sleep(time.Second * 3)
			if counter = counter + 1; counter == 100 {
				log.Warn("Container wait timeout: " + contName)
				return
			}
		}

		pub = config.Agent.LxcPrefix + contName + "/public.pub"
		keyring = config.Agent.LxcPrefix + contName + "/secret.sec"
		log.Info("Getting public keyring", "keyring", keyring)
		md = gpg.DecryptNoDefaultKeyring(rsp.Request, keyring, pub)
	}

	i := strings.Index(md, "{")
	j := strings.LastIndex(md, "}") + 1
	if i > j && i > 0 {
		log.Warn("Error getting JSON request")
		return
	}
	request := md[i:j]

	err = json.Unmarshal([]byte(request), &req.Request)
	log.Check(log.WarnLevel, "Decrypting request", err)

	//create channels for stdout and stderr
	sOut := make(chan executer.ResponseOptions)
	req.Execute(flag, sOut)

	for sOut != nil {
		select {
		case elem, ok := <-sOut:
			if ok {
				resp := executer.Response{ResponseOpts: elem}
				jsonR, err := json.Marshal(resp)
				log.Check(log.WarnLevel, "Marshal response", err)
				if rsp.HostId == fingerprint {
					payload = gpg.EncryptWrapper(config.Agent.GpgUser, config.Management.GpgUser, string(jsonR))
				} else {
					payload = gpg.EncryptWrapperNoDefaultKeyring(contName, config.Management.GpgUser, string(jsonR), pub, keyring)
				}
				message, err := json.Marshal(map[string]string{
					"hostId":   elem.Id,
					"response": payload,
				})
				log.Check(log.WarnLevel, "Marshal response json "+elem.CommandId, err)
				response(message)
			} else {
				sOut = nil
			}
		}
	}
	heartbeat()
}

func tlsConfig() *http.Client {
	tlsconfig := newTLSConfig()
	for tlsconfig == nil || len(tlsconfig.Certificates[0].Certificate) == 0 {
		time.Sleep(time.Second * 2)
		for utils.PublicCert() == "" {
			x509generate()
		}
		tlsconfig = newTLSConfig()
	}

	transport := &http.Transport{TLSClientConfig: tlsconfig}
	client := &http.Client{Transport: transport}
	return client
}

func response(msg []byte) {
	client := tlsConfig()
	resp, err := client.PostForm("https://"+config.Management.Host+":8444/rest/v1/agent/response", url.Values{"response": {string(msg)}})
	if !log.Check(log.WarnLevel, "Sending response "+string(msg), err) {
		resp.Body.Close()
	}
}

func command() {
	var rsp []executer.EncRequest
	client := tlsConfig()
	hostname, _ := os.Hostname()
	fingerprint := gpg.GetFingerprint(hostname + "@subutai.io")

	resp, err := client.Get("https://" + config.Management.Host + ":8444/rest/v1/agent/requests/" + fingerprint)
	if log.Check(log.WarnLevel, "Getting requests", err) {
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNoContent {
		return
	}

	data, err := ioutil.ReadAll(resp.Body)
	if !log.Check(log.WarnLevel, "Reading body", err) {
		log.Check(log.WarnLevel, "Unmarshal payload", json.Unmarshal(data, &rsp))
		for _, request := range rsp {
			execute(request)
		}
	}
}

func ping(rw http.ResponseWriter, request *http.Request) {
	if request.Method == http.MethodGet && strings.Split(request.RemoteAddr, ":")[0] == config.Management.Host {
		rw.WriteHeader(http.StatusOK)
	} else {
		rw.WriteHeader(http.StatusForbidden)
	}
}

func trigger(rw http.ResponseWriter, request *http.Request) {
	if request.Method == http.MethodPost && strings.Split(request.RemoteAddr, ":")[0] == config.Management.Host {
		rw.WriteHeader(http.StatusAccepted)
		command()
	} else {
		rw.WriteHeader(http.StatusForbidden)
	}
}

func heartbeatCall(rw http.ResponseWriter, request *http.Request) {
	if request.Method == http.MethodGet && strings.Split(request.RemoteAddr, ":")[0] == config.Management.Host {
		rw.WriteHeader(http.StatusOK)
		lastHeartbeat = []byte{}
		heartbeat()
	} else {
		rw.WriteHeader(http.StatusForbidden)
	}
}
<|endoftext|>


<commit_before>package agent

import (
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/hashicorp/nomad/client"
	"github.com/hashicorp/nomad/nomad"
	"github.com/hashicorp/nomad/nomad/structs"
)

// Agent is a long running daemon that is used to run both
// clients and servers. Servers are responsible for managing
// state and making scheduling decisions. Clients can be
// scheduled to, and are responsible for interfacing with
// servers to run allocations.
type Agent struct {
	config    *Config
	logger    *log.Logger
	logOutput io.Writer

	server *nomad.Server
	client *client.Client

	shutdown     bool
	shutdownCh   chan struct{}
	shutdownLock sync.Mutex
}

// NewAgent is used to create a new agent with the given configuration
func NewAgent(config *Config, logOutput io.Writer) (*Agent, error) {
	// Ensure we have a log sink
	if logOutput == nil {
		logOutput = os.Stderr
	}

	a := &Agent{
		config:     config,
		logger:     log.New(logOutput, "", log.LstdFlags),
		logOutput:  logOutput,
		shutdownCh: make(chan struct{}),
	}

	if err := a.setupServer(); err != nil {
		return nil, err
	}
	if err := a.setupClient(); err != nil {
		return nil, err
	}
	if a.client == nil && a.server == nil {
		return nil, fmt.Errorf("must have at least client or server mode enabled")
	}
	return a, nil
}

// serverConfig is used to generate a new server configuration struct
// for initializing a nomad server.
func (a *Agent) serverConfig() (*nomad.Config, error) {
	conf := a.config.NomadConfig
	if conf == nil {
		conf = nomad.DefaultConfig()
	}
	conf.LogOutput = a.logOutput
	conf.DevMode = a.config.DevMode
	conf.Build = fmt.Sprintf("%s%s", a.config.Version, a.config.VersionPrerelease)
	if a.config.Region != "" {
		conf.Region = a.config.Region
	}
	if a.config.Datacenter != "" {
		conf.Datacenter = a.config.Datacenter
	}
	if a.config.NodeName != "" {
		conf.NodeName = a.config.NodeName
	}
	if a.config.Server.BootstrapExpect > 0 {
		if a.config.Server.BootstrapExpect == 1 {
			conf.Bootstrap = true
		} else {
			conf.BootstrapExpect = a.config.Server.BootstrapExpect
		}
	}
	if a.config.DataDir != "" {
		conf.DataDir = filepath.Join(a.config.DataDir, "server")
	}
	if a.config.Server.DataDir != "" {
		conf.DataDir = a.config.Server.DataDir
	}
	if a.config.Server.ProtocolVersion != 0 {
		conf.ProtocolVersion = uint8(a.config.Server.ProtocolVersion)
	}
	if a.config.Server.NumSchedulers != 0 {
		conf.NumSchedulers = a.config.Server.NumSchedulers
	}
	if len(a.config.Server.EnabledSchedulers) != 0 {
		conf.EnabledSchedulers = a.config.Server.EnabledSchedulers
	}

	// Set up the advertise addrs
	if addr := a.config.AdvertiseAddrs.Serf; addr != "" {
		serfAddr, err := net.ResolveTCPAddr("tcp", addr)
		if err != nil {
			return nil, fmt.Errorf("error resolving serf advertise address: %s", err)
		}
		conf.SerfConfig.MemberlistConfig.AdvertiseAddr = serfAddr.IP.String()
		conf.SerfConfig.MemberlistConfig.AdvertisePort = serfAddr.Port
	}
	if addr := a.config.AdvertiseAddrs.RPC; addr != "" {
		rpcAddr, err := net.ResolveTCPAddr("tcp", addr)
		if err != nil {
			return nil, fmt.Errorf("error resolving rpc advertise address: %s", err)
		}
		conf.RPCAdvertise = rpcAddr
	}

	// Set up the bind addresses
	if addr := a.config.BindAddr; addr != "" {
		conf.RPCAddr.IP = net.ParseIP(addr)
		conf.SerfConfig.MemberlistConfig.BindAddr = addr
	}
	if addr := a.config.Addresses.RPC; addr != "" {
		conf.RPCAddr.IP = net.ParseIP(addr)
	}
	if addr := a.config.Addresses.Serf; addr != "" {
		conf.SerfConfig.MemberlistConfig.BindAddr = addr
	}

	// Set up the ports
	if port := a.config.Ports.RPC; port != 0 {
		conf.RPCAddr.Port = port
	}
	if port := a.config.Ports.Serf; port != 0 {
		conf.SerfConfig.MemberlistConfig.BindPort = port
	}

	if gcThreshold := a.config.Server.NodeGCThreshold; gcThreshold != "" {
		dur, err := time.ParseDuration(gcThreshold)
		if err != nil {
			return nil, err
		}
		conf.NodeGCThreshold = dur
	}

	return conf, nil
}

// setupServer is used to setup the server if enabled
func (a *Agent) setupServer() error {
	if !a.config.Server.Enabled {
		return nil
	}

	// Setup the configuration
	conf, err := a.serverConfig()
	if err != nil {
		return fmt.Errorf("server config setup failed: %s", err)
	}

	// Create the server
	server, err := nomad.NewServer(conf)
	if err != nil {
		return fmt.Errorf("server setup failed: %v", err)
	}

	a.server = server
	return nil
}

// setupClient is used to setup the client if enabled
func (a *Agent) setupClient() error {
	if !a.config.Client.Enabled {
		return nil
	}

	// Setup the configuration
	conf := a.config.ClientConfig
	if conf == nil {
		conf = client.DefaultConfig()
	}
	if a.server != nil {
		conf.RPCHandler = a.server
	}
	conf.LogOutput = a.logOutput
	conf.DevMode = a.config.DevMode
	if a.config.Region != "" {
		conf.Region = a.config.Region
	}
	if a.config.DataDir != "" {
		conf.StateDir = filepath.Join(a.config.DataDir, "client")
		conf.AllocDir = filepath.Join(a.config.DataDir, "alloc")
	}
	if a.config.Client.StateDir != "" {
		conf.StateDir = a.config.Client.StateDir
	}
	if a.config.Client.AllocDir != "" {
		conf.AllocDir = a.config.Client.AllocDir
	}
	conf.Servers = a.config.Client.Servers
	if a.config.Client.NetworkInterface != "" {
		conf.NetworkInterface = a.config.Client.NetworkInterface
	}
	conf.Options = a.config.Client.Options
	if a.config.Client.NetworkSpeed != 0 {
		conf.NetworkSpeed = a.config.Client.NetworkSpeed
	}
	if a.config.Client.MaxKillTimeout != "" {
		dur, err := time.ParseDuration(a.config.Client.MaxKillTimeout)
		if err != nil {
			return fmt.Errorf("Error parsing retry interval: %s", err)
		}
		conf.MaxKillTimeout = dur
	}

	// Setup the node
	conf.Node = new(structs.Node)
	conf.Node.Datacenter = a.config.Datacenter
	conf.Node.Name = a.config.NodeName
	conf.Node.Meta = a.config.Client.Meta
	conf.Node.NodeClass = a.config.Client.NodeClass
	conf.Node.HTTPAddr = fmt.Sprintf("%s:%d", a.config.BindAddr, a.config.Ports.HTTP)

	// Create the client
	client, err := client.NewClient(conf)
	if err != nil {
		return fmt.Errorf("client setup failed: %v", err)
	}
	a.client = client
	return nil
}

// Leave is used gracefully exit. Clients will inform servers
// of their departure so that allocations can be rescheduled.
func (a *Agent) Leave() error {
	if a.client != nil {
		if err := a.client.Leave(); err != nil {
			a.logger.Printf("[ERR] agent: client leave failed: %v", err)
		}
	}
	if a.server != nil {
		if err := a.server.Leave(); err != nil {
			a.logger.Printf("[ERR] agent: server leave failed: %v", err)
		}
	}
	return nil
}

// Shutdown is used to terminate the agent.
func (a *Agent) Shutdown() error {
	a.shutdownLock.Lock()
	defer a.shutdownLock.Unlock()

	if a.shutdown {
		return nil
	}

	a.logger.Println("[INFO] agent: requesting shutdown")
	if a.client != nil {
		if err := a.client.Shutdown(); err != nil {
			a.logger.Printf("[ERR] agent: client shutdown failed: %v", err)
		}
	}
	if a.server != nil {
		if err := a.server.Shutdown(); err != nil {
			a.logger.Printf("[ERR] agent: server shutdown failed: %v", err)
		}
	}

	a.logger.Println("[INFO] agent: shutdown complete")
	a.shutdown = true
	close(a.shutdownCh)
	return nil
}

// RPC is used to make an RPC call to the Nomad servers
func (a *Agent) RPC(method string, args interface{}, reply interface{}) error {
	if a.server != nil {
		return a.server.RPC(method, args, reply)
	}
	return a.client.RPC(method, args, reply)
}

// Client returns the configured client or nil
func (a *Agent) Client() *client.Client {
	return a.client
}

// Server returns the configured server or nil
func (a *Agent) Server() *nomad.Server {
	return a.server
}

// Stats is used to return statistics for debugging and insight
// for various sub-systems
func (a *Agent) Stats() map[string]map[string]string {
	stats := make(map[string]map[string]string)
	if a.server != nil {
		subStat := a.server.Stats()
		for k, v := range subStat {
			stats[k] = v
		}
	}
	if a.client != nil {
		subStat := a.client.Stats()
		for k, v := range subStat {
			stats[k] = v
		}
	}
	return stats
}
<commit_msg>Using the bindAddr only when user hasn't overrident it<commit_after>package agent

import (
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/hashicorp/nomad/client"
	"github.com/hashicorp/nomad/nomad"
	"github.com/hashicorp/nomad/nomad/structs"
)

// Agent is a long running daemon that is used to run both
// clients and servers. Servers are responsible for managing
// state and making scheduling decisions. Clients can be
// scheduled to, and are responsible for interfacing with
// servers to run allocations.
type Agent struct {
	config    *Config
	logger    *log.Logger
	logOutput io.Writer

	server *nomad.Server
	client *client.Client

	shutdown     bool
	shutdownCh   chan struct{}
	shutdownLock sync.Mutex
}

// NewAgent is used to create a new agent with the given configuration
func NewAgent(config *Config, logOutput io.Writer) (*Agent, error) {
	// Ensure we have a log sink
	if logOutput == nil {
		logOutput = os.Stderr
	}

	a := &Agent{
		config:     config,
		logger:     log.New(logOutput, "", log.LstdFlags),
		logOutput:  logOutput,
		shutdownCh: make(chan struct{}),
	}

	if err := a.setupServer(); err != nil {
		return nil, err
	}
	if err := a.setupClient(); err != nil {
		return nil, err
	}
	if a.client == nil && a.server == nil {
		return nil, fmt.Errorf("must have at least client or server mode enabled")
	}
	return a, nil
}

// serverConfig is used to generate a new server configuration struct
// for initializing a nomad server.
func (a *Agent) serverConfig() (*nomad.Config, error) {
	conf := a.config.NomadConfig
	if conf == nil {
		conf = nomad.DefaultConfig()
	}
	conf.LogOutput = a.logOutput
	conf.DevMode = a.config.DevMode
	conf.Build = fmt.Sprintf("%s%s", a.config.Version, a.config.VersionPrerelease)
	if a.config.Region != "" {
		conf.Region = a.config.Region
	}
	if a.config.Datacenter != "" {
		conf.Datacenter = a.config.Datacenter
	}
	if a.config.NodeName != "" {
		conf.NodeName = a.config.NodeName
	}
	if a.config.Server.BootstrapExpect > 0 {
		if a.config.Server.BootstrapExpect == 1 {
			conf.Bootstrap = true
		} else {
			conf.BootstrapExpect = a.config.Server.BootstrapExpect
		}
	}
	if a.config.DataDir != "" {
		conf.DataDir = filepath.Join(a.config.DataDir, "server")
	}
	if a.config.Server.DataDir != "" {
		conf.DataDir = a.config.Server.DataDir
	}
	if a.config.Server.ProtocolVersion != 0 {
		conf.ProtocolVersion = uint8(a.config.Server.ProtocolVersion)
	}
	if a.config.Server.NumSchedulers != 0 {
		conf.NumSchedulers = a.config.Server.NumSchedulers
	}
	if len(a.config.Server.EnabledSchedulers) != 0 {
		conf.EnabledSchedulers = a.config.Server.EnabledSchedulers
	}

	// Set up the advertise addrs
	if addr := a.config.AdvertiseAddrs.Serf; addr != "" {
		serfAddr, err := net.ResolveTCPAddr("tcp", addr)
		if err != nil {
			return nil, fmt.Errorf("error resolving serf advertise address: %s", err)
		}
		conf.SerfConfig.MemberlistConfig.AdvertiseAddr = serfAddr.IP.String()
		conf.SerfConfig.MemberlistConfig.AdvertisePort = serfAddr.Port
	}
	if addr := a.config.AdvertiseAddrs.RPC; addr != "" {
		rpcAddr, err := net.ResolveTCPAddr("tcp", addr)
		if err != nil {
			return nil, fmt.Errorf("error resolving rpc advertise address: %s", err)
		}
		conf.RPCAdvertise = rpcAddr
	}

	// Set up the bind addresses
	if addr := a.config.BindAddr; addr != "" {
		conf.RPCAddr.IP = net.ParseIP(addr)
		conf.SerfConfig.MemberlistConfig.BindAddr = addr
	}
	if addr := a.config.Addresses.RPC; addr != "" {
		conf.RPCAddr.IP = net.ParseIP(addr)
	}
	if addr := a.config.Addresses.Serf; addr != "" {
		conf.SerfConfig.MemberlistConfig.BindAddr = addr
	}

	// Set up the ports
	if port := a.config.Ports.RPC; port != 0 {
		conf.RPCAddr.Port = port
	}
	if port := a.config.Ports.Serf; port != 0 {
		conf.SerfConfig.MemberlistConfig.BindPort = port
	}

	if gcThreshold := a.config.Server.NodeGCThreshold; gcThreshold != "" {
		dur, err := time.ParseDuration(gcThreshold)
		if err != nil {
			return nil, err
		}
		conf.NodeGCThreshold = dur
	}

	return conf, nil
}

// setupServer is used to setup the server if enabled
func (a *Agent) setupServer() error {
	if !a.config.Server.Enabled {
		return nil
	}

	// Setup the configuration
	conf, err := a.serverConfig()
	if err != nil {
		return fmt.Errorf("server config setup failed: %s", err)
	}

	// Create the server
	server, err := nomad.NewServer(conf)
	if err != nil {
		return fmt.Errorf("server setup failed: %v", err)
	}

	a.server = server
	return nil
}

// setupClient is used to setup the client if enabled
func (a *Agent) setupClient() error {
	if !a.config.Client.Enabled {
		return nil
	}

	// Setup the configuration
	conf := a.config.ClientConfig
	if conf == nil {
		conf = client.DefaultConfig()
	}
	if a.server != nil {
		conf.RPCHandler = a.server
	}
	conf.LogOutput = a.logOutput
	conf.DevMode = a.config.DevMode
	if a.config.Region != "" {
		conf.Region = a.config.Region
	}
	if a.config.DataDir != "" {
		conf.StateDir = filepath.Join(a.config.DataDir, "client")
		conf.AllocDir = filepath.Join(a.config.DataDir, "alloc")
	}
	if a.config.Client.StateDir != "" {
		conf.StateDir = a.config.Client.StateDir
	}
	if a.config.Client.AllocDir != "" {
		conf.AllocDir = a.config.Client.AllocDir
	}
	conf.Servers = a.config.Client.Servers
	if a.config.Client.NetworkInterface != "" {
		conf.NetworkInterface = a.config.Client.NetworkInterface
	}
	conf.Options = a.config.Client.Options
	if a.config.Client.NetworkSpeed != 0 {
		conf.NetworkSpeed = a.config.Client.NetworkSpeed
	}
	if a.config.Client.MaxKillTimeout != "" {
		dur, err := time.ParseDuration(a.config.Client.MaxKillTimeout)
		if err != nil {
			return fmt.Errorf("Error parsing retry interval: %s", err)
		}
		conf.MaxKillTimeout = dur
	}

	// Setup the node
	conf.Node = new(structs.Node)
	conf.Node.Datacenter = a.config.Datacenter
	conf.Node.Name = a.config.NodeName
	conf.Node.Meta = a.config.Client.Meta
	conf.Node.NodeClass = a.config.Client.NodeClass
	httpAddr := a.config.BindAddr
	if a.config.Addresses.HTTP != "" {
		httpAddr = a.config.Addresses.HTTP
	}
	conf.Node.HTTPAddr = fmt.Sprintf("%s:%d", httpAddr, a.config.Ports.HTTP)

	// Create the client
	client, err := client.NewClient(conf)
	if err != nil {
		return fmt.Errorf("client setup failed: %v", err)
	}
	a.client = client
	return nil
}

// Leave is used gracefully exit. Clients will inform servers
// of their departure so that allocations can be rescheduled.
func (a *Agent) Leave() error {
	if a.client != nil {
		if err := a.client.Leave(); err != nil {
			a.logger.Printf("[ERR] agent: client leave failed: %v", err)
		}
	}
	if a.server != nil {
		if err := a.server.Leave(); err != nil {
			a.logger.Printf("[ERR] agent: server leave failed: %v", err)
		}
	}
	return nil
}

// Shutdown is used to terminate the agent.
func (a *Agent) Shutdown() error {
	a.shutdownLock.Lock()
	defer a.shutdownLock.Unlock()

	if a.shutdown {
		return nil
	}

	a.logger.Println("[INFO] agent: requesting shutdown")
	if a.client != nil {
		if err := a.client.Shutdown(); err != nil {
			a.logger.Printf("[ERR] agent: client shutdown failed: %v", err)
		}
	}
	if a.server != nil {
		if err := a.server.Shutdown(); err != nil {
			a.logger.Printf("[ERR] agent: server shutdown failed: %v", err)
		}
	}

	a.logger.Println("[INFO] agent: shutdown complete")
	a.shutdown = true
	close(a.shutdownCh)
	return nil
}

// RPC is used to make an RPC call to the Nomad servers
func (a *Agent) RPC(method string, args interface{}, reply interface{}) error {
	if a.server != nil {
		return a.server.RPC(method, args, reply)
	}
	return a.client.RPC(method, args, reply)
}

// Client returns the configured client or nil
func (a *Agent) Client() *client.Client {
	return a.client
}

// Server returns the configured server or nil
func (a *Agent) Server() *nomad.Server {
	return a.server
}

// Stats is used to return statistics for debugging and insight
// for various sub-systems
func (a *Agent) Stats() map[string]map[string]string {
	stats := make(map[string]map[string]string)
	if a.server != nil {
		subStat := a.server.Stats()
		for k, v := range subStat {
			stats[k] = v
		}
	}
	if a.client != nil {
		subStat := a.client.Stats()
		for k, v := range subStat {
			stats[k] = v
		}
	}
	return stats
}
<|endoftext|>


<commit_before>package sgload

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"strings"
	"time"

	sgreplicate "github.com/couchbaselabs/sg-replicate"
	"github.com/peterbourgon/g2s"
)

var (
	// The sample rate -- set this to 0.1 if you only want
	// 10% of the samples to be pushed to statds, or .01 if you only
	// want 1% of the samples pushed to statsd.  Useful for
	// not overwhelming stats if you have too many samples.
	statsdSampleRate float32 = 1.0
)

type SGDataStore struct {
	SyncGatewayUrl       string
	SyncGatewayAdminPort int
	UserCreds            UserCred
	StatsdClient         *g2s.Statsd
	CompressionEnabled   bool
}

func NewSGDataStore(sgUrl string, sgAdminPort int, statsdClient *g2s.Statsd, compressionEnabled bool) *SGDataStore {
	return &SGDataStore{
		SyncGatewayUrl:       sgUrl,
		SyncGatewayAdminPort: sgAdminPort,
		StatsdClient:         statsdClient,
		CompressionEnabled:   compressionEnabled,
	}
}

func (s *SGDataStore) SetUserCreds(u UserCred) {
	s.UserCreds = u
}

func (s SGDataStore) CreateUser(u UserCred, channelNames []string) error {

	adminUrl, err := s.sgAdminURL()
	if err != nil {
		return err
	}

	adminUrlUserEndpoint, err := addEndpointToUrl(adminUrl, "_user")
	if err != nil {
		return err
	}

	adminUrlUserEndpoint = addTrailingSlash(adminUrlUserEndpoint)

	userDoc := map[string]interface{}{}
	userDoc["name"] = u.Username
	userDoc["password"] = u.Password
	userDoc["admin_channels"] = channelNames

	docBytes, err := json.Marshal(userDoc)
	if err != nil {
		return err
	}
	buf := bytes.NewBuffer(docBytes)

	req, err := http.NewRequest("POST", adminUrlUserEndpoint, buf)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	s.pushTimingStat("create_user", time.Since(startTime))

	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	defer resp.Body.Close()
	io.Copy(ioutil.Discard, resp.Body)

	return nil
}

func (s SGDataStore) sgAdminURL() (string, error) {

	sgAdminURL := s.SyncGatewayUrl

	parsedSgUrl, err := url.Parse(s.SyncGatewayUrl)
	if err != nil {
		log.Printf("Error parsing url: %v", err)
		return sgAdminURL, err
	}

	// find the port from the url
	host, port, err := splitHostPortWrapper(parsedSgUrl.Host)
	if err != nil {
		return sgAdminURL, err
	}

	if port != "" {
		// is there a port?
		// do a regex replace on :port on the original url
		r := regexp.MustCompile(port)
		sgAdminURL = r.ReplaceAllString(
			sgAdminURL,
			fmt.Sprintf("%d", s.SyncGatewayAdminPort),
		)

	} else {
		// is there no port?
		// do a regex replace of host with host:port
		r := regexp.MustCompile(host)
		hostWithAdminPort := fmt.Sprintf("%v:%v", host, s.SyncGatewayAdminPort)
		sgAdminURL = r.ReplaceAllString(
			sgAdminURL,
			hostWithAdminPort,
		)

	}

	// return it
	return sgAdminURL, nil

}

func (s SGDataStore) Changes(sinceVal Sincer, limit int) (changes sgreplicate.Changes, newSinceVal Sincer, err error) {

	changesFeedEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_changes")
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}

	changesFeedParams := NewChangesFeedParams(sinceVal, limit)

	changesFeedUrl := fmt.Sprintf(
		"%s?%s",
		changesFeedEndpoint,
		changesFeedParams,
	)

	req, err := http.NewRequest("GET", changesFeedUrl, nil)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}
	s.pushTimingStat("changes_feed", time.Since(startTime))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return sgreplicate.Changes{}, sinceVal, fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	defer resp.Body.Close()
	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(&changes)
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}
	lastSequenceStr, ok := changes.LastSequence.(string)
	if !ok {
		return sgreplicate.Changes{}, sinceVal, fmt.Errorf("Could not convert changes.LastSequence to string")
	}
	lastSequenceSincer := StringSincer{
		Since: lastSequenceStr,
	}

	return changes, lastSequenceSincer, nil
}

// Create a single document in Sync Gateway
func (s SGDataStore) CreateDocument(d Document) (sgreplicate.DocumentRevisionPair, error) {

	docRevisionPairs, err := s.BulkCreateDocuments([]Document{d})
	if err != nil {
		return sgreplicate.DocumentRevisionPair{}, err
	}
	if len(docRevisionPairs) == 0 {
		return sgreplicate.DocumentRevisionPair{}, fmt.Errorf("Unexpected response")
	}
	return docRevisionPairs[0], nil

}

// Bulk create a set of documents in Sync Gateway
func (s SGDataStore) BulkCreateDocuments(docs []Document) ([]sgreplicate.DocumentRevisionPair, error) {

	defer s.pushCounter("create_document_counter", len(docs))

	// Set the "created_at" timestamp which is used to calculate the
	// gateload roundtrip time
	updateCreatedAtTimestamp(docs)

	bulkDocsResponse := []sgreplicate.DocumentRevisionPair{}

	bulkDocsEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_bulk_docs")
	if err != nil {
		return bulkDocsResponse, err
	}

	bulkDocs := BulkDocs{
		Documents: docs,
		NewEdits:  true,
	}
	docBytes, err := json.Marshal(bulkDocs)
	if err != nil {
		return bulkDocsResponse, err
	}
	var buf *bytes.Buffer
	if s.CompressionEnabled {
		buf = &bytes.Buffer{}
		gzipWriter := gzip.NewWriter(buf)
		if _, err := gzipWriter.Write(docBytes); err != nil {
			return bulkDocsResponse, err
		}
		if err = gzipWriter.Close(); err != nil {
			return bulkDocsResponse, err
		}
	} else {
		buf = bytes.NewBuffer(docBytes)
	}

	req, err := http.NewRequest("POST", bulkDocsEndpoint, buf)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")
	if s.CompressionEnabled {
		req.Header.Set("Content-Encoding", "gzip")
	}

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return bulkDocsResponse, err
	}
	s.pushTimingStat("create_document", timeDeltaPerDocument(len(docs), time.Since(startTime)))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return bulkDocsResponse, fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	defer resp.Body.Close()

	decoder := json.NewDecoder(resp.Body)
	if err = decoder.Decode(&bulkDocsResponse); err != nil {
		return bulkDocsResponse, err
	}

	// If any of the bulk docs had errors, return an error
	for _, docRevisionPair := range bulkDocsResponse {
		if docRevisionPair.Error != "" {
			return bulkDocsResponse, fmt.Errorf("%v", docRevisionPair.Error)
		}
	}

	return bulkDocsResponse, nil

}

func (s SGDataStore) BulkGetDocuments(r sgreplicate.BulkGetRequest) ([]sgreplicate.Document, error) {

	defer s.pushCounter("get_document_counter", len(r.Docs))

	bulkGetEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_bulk_get")
	if err != nil {
		return nil, err
	}

	bulkGetBytes, err := json.Marshal(r)
	if err != nil {
		return nil, fmt.Errorf("BulkGetDocuemnts failed to marshal request: %v", err)
	}

	buf := bytes.NewBuffer(bulkGetBytes)

	req, err := http.NewRequest("POST", bulkGetEndpoint, buf)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	s.pushTimingStat("get_document", timeDeltaPerDocument(len(r.Docs), time.Since(startTime)))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return nil, fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	// Parse the response and make sure that we got all the docs we requested
	defer resp.Body.Close()

	// Logger function that ignores any logging from the ReadBulkGetResponse method
	loggerFunc := func(key string, format string, args ...interface{}) {}
	documents, err := sgreplicate.ReadBulkGetResponse(resp, loggerFunc)
	if len(documents) != len(r.Docs) {
		return nil, fmt.Errorf("Expected %d docs, got %d docs", len(r.Docs), len(documents))
	}

	for _, doc := range documents {
		createAtRFC3339NanoIface, ok := doc.Body["created_at"]
		if !ok {
			logger.Warn("Document missing created_at field", "doc.Body", doc.Body)
			continue
		}
		createAtRFC3339NanoStr, ok := createAtRFC3339NanoIface.(string)
		if !ok {
			logger.Warn("Document created_at not a string", "doc.Body", doc.Body)
			continue
		}
		createAtRFC3339Nano, err := time.Parse(
			time.RFC3339Nano,
			createAtRFC3339NanoStr,
		)
		if err != nil {
			logger.Warn("Could not parse doc.created_at field into time", "createAtRFC3339Nano", createAtRFC3339Nano)
			continue
		}
		delta := time.Since(createAtRFC3339Nano)
		s.pushTimingStat("gateload_roundtrip", delta)

		possiblyLogVerboseWarning(delta, doc)

	}

	return documents, nil

}

// If the round trip time is over a certain threshold, log a verbose
// warning.  Trying to debug https://github.com/couchbaselabs/sgload/issues/12
func possiblyLogVerboseWarning(delta time.Duration, doc sgreplicate.Document) {

	maxSecondsExpected := time.Duration(60)
	if delta > time.Duration(time.Second*maxSecondsExpected) {

		createAtRFC3339NanoIface, ok := doc.Body["created_at"]
		if !ok {
			logger.Warn("Document missing created_at field", "doc.Body", doc.Body)
		}
		createAtRFC3339NanoStr, ok := createAtRFC3339NanoIface.(string)
		if !ok {
			logger.Warn("Document created_at not a string", "doc.Body", doc.Body)
		}
		createAtRFC3339Nano, err := time.Parse(
			time.RFC3339Nano,
			createAtRFC3339NanoStr,
		)
		if err != nil {
			logger.Warn("Could not parse doc.created_at field into time", "createAtRFC3339Nano", createAtRFC3339Nano)
		}

		logger.Warn(
			"Gateload roundtrip time exceeeded expected time",
			"maxSecondsExpected",
			maxSecondsExpected,
			"delta",
			delta,
			"createAtRFC3339NanoStr",
			createAtRFC3339NanoStr,
			"createAtRFC3339Nano",
			createAtRFC3339Nano,
			"time.Since(createAtRFC3339Nano)",
			time.Since(createAtRFC3339Nano),
			"now",
			time.Now(),
			"now RFC3339Nano",
			time.Now().Format(time.RFC3339Nano),
		)

	}

}

// add BasicAuth header for user if needed
func (s SGDataStore) addAuthIfNeeded(req *http.Request) {
	if !s.UserCreds.Empty() {
		req.SetBasicAuth(s.UserCreds.Username, s.UserCreds.Password)
	}
}

func (s SGDataStore) pushTimingStat(key string, delta time.Duration) {
	if s.StatsdClient == nil {
		return
	}
	s.StatsdClient.Timing(
		statsdSampleRate,
		key,
		delta,
	)
}

func (s SGDataStore) pushCounter(key string, n int) {
	if s.StatsdClient == nil {
		return
	}
	s.StatsdClient.Counter(
		statsdSampleRate,
		key,
		n,
	)
}

func splitHostPortWrapper(host string) (string, string, error) {
	if !strings.Contains(host, ":") {
		return host, "", nil
	}

	return net.SplitHostPort(host)
}

func addEndpointToUrl(urlStr, endpoint string) (string, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return "", err
	}
	u.Path = path.Join(u.Path, endpoint)
	return u.String(), nil
}

func addTrailingSlash(urlStr string) string {
	return fmt.Sprintf("%v/", urlStr)
}

func timeDeltaPerDocument(numDocs int, timeDeltaAllDocs time.Duration) time.Duration {
	if numDocs == 0 {
		return timeDeltaAllDocs
	}
	return time.Duration(int64(timeDeltaAllDocs) / int64(numDocs))
}
<commit_msg>Improve misleading error<commit_after>package sgload

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"path"
	"regexp"
	"strings"
	"time"

	sgreplicate "github.com/couchbaselabs/sg-replicate"
	"github.com/peterbourgon/g2s"
)

var (
	// The sample rate -- set this to 0.1 if you only want
	// 10% of the samples to be pushed to statds, or .01 if you only
	// want 1% of the samples pushed to statsd.  Useful for
	// not overwhelming stats if you have too many samples.
	statsdSampleRate float32 = 1.0
)

type SGDataStore struct {
	SyncGatewayUrl       string
	SyncGatewayAdminPort int
	UserCreds            UserCred
	StatsdClient         *g2s.Statsd
	CompressionEnabled   bool
}

func NewSGDataStore(sgUrl string, sgAdminPort int, statsdClient *g2s.Statsd, compressionEnabled bool) *SGDataStore {
	return &SGDataStore{
		SyncGatewayUrl:       sgUrl,
		SyncGatewayAdminPort: sgAdminPort,
		StatsdClient:         statsdClient,
		CompressionEnabled:   compressionEnabled,
	}
}

func (s *SGDataStore) SetUserCreds(u UserCred) {
	s.UserCreds = u
}

func (s SGDataStore) CreateUser(u UserCred, channelNames []string) error {

	adminUrl, err := s.sgAdminURL()
	if err != nil {
		return err
	}

	adminUrlUserEndpoint, err := addEndpointToUrl(adminUrl, "_user")
	if err != nil {
		return err
	}

	adminUrlUserEndpoint = addTrailingSlash(adminUrlUserEndpoint)

	userDoc := map[string]interface{}{}
	userDoc["name"] = u.Username
	userDoc["password"] = u.Password
	userDoc["admin_channels"] = channelNames

	docBytes, err := json.Marshal(userDoc)
	if err != nil {
		return err
	}
	buf := bytes.NewBuffer(docBytes)

	req, err := http.NewRequest("POST", adminUrlUserEndpoint, buf)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	s.pushTimingStat("create_user", time.Since(startTime))

	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	defer resp.Body.Close()
	io.Copy(ioutil.Discard, resp.Body)

	return nil
}

func (s SGDataStore) sgAdminURL() (string, error) {

	sgAdminURL := s.SyncGatewayUrl

	parsedSgUrl, err := url.Parse(s.SyncGatewayUrl)
	if err != nil {
		log.Printf("Error parsing url: %v", err)
		return sgAdminURL, err
	}

	// find the port from the url
	host, port, err := splitHostPortWrapper(parsedSgUrl.Host)
	if err != nil {
		return sgAdminURL, err
	}

	if port != "" {
		// is there a port?
		// do a regex replace on :port on the original url
		r := regexp.MustCompile(port)
		sgAdminURL = r.ReplaceAllString(
			sgAdminURL,
			fmt.Sprintf("%d", s.SyncGatewayAdminPort),
		)

	} else {
		// is there no port?
		// do a regex replace of host with host:port
		r := regexp.MustCompile(host)
		hostWithAdminPort := fmt.Sprintf("%v:%v", host, s.SyncGatewayAdminPort)
		sgAdminURL = r.ReplaceAllString(
			sgAdminURL,
			hostWithAdminPort,
		)

	}

	// return it
	return sgAdminURL, nil

}

func (s SGDataStore) Changes(sinceVal Sincer, limit int) (changes sgreplicate.Changes, newSinceVal Sincer, err error) {

	changesFeedEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_changes")
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}

	changesFeedParams := NewChangesFeedParams(sinceVal, limit)

	changesFeedUrl := fmt.Sprintf(
		"%s?%s",
		changesFeedEndpoint,
		changesFeedParams,
	)

	req, err := http.NewRequest("GET", changesFeedUrl, nil)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}
	s.pushTimingStat("changes_feed", time.Since(startTime))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return sgreplicate.Changes{}, sinceVal, fmt.Errorf("Unexpected response status for changes_feed GET request: %d", resp.StatusCode)
	}

	defer resp.Body.Close()
	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(&changes)
	if err != nil {
		return sgreplicate.Changes{}, sinceVal, err
	}
	lastSequenceStr, ok := changes.LastSequence.(string)
	if !ok {
		return sgreplicate.Changes{}, sinceVal, fmt.Errorf("Could not convert changes.LastSequence to string")
	}
	lastSequenceSincer := StringSincer{
		Since: lastSequenceStr,
	}

	return changes, lastSequenceSincer, nil
}

// Create a single document in Sync Gateway
func (s SGDataStore) CreateDocument(d Document) (sgreplicate.DocumentRevisionPair, error) {

	docRevisionPairs, err := s.BulkCreateDocuments([]Document{d})
	if err != nil {
		return sgreplicate.DocumentRevisionPair{}, err
	}
	if len(docRevisionPairs) == 0 {
		return sgreplicate.DocumentRevisionPair{}, fmt.Errorf("Unexpected response")
	}
	return docRevisionPairs[0], nil

}

// Bulk create a set of documents in Sync Gateway
func (s SGDataStore) BulkCreateDocuments(docs []Document) ([]sgreplicate.DocumentRevisionPair, error) {

	defer s.pushCounter("create_document_counter", len(docs))

	// Set the "created_at" timestamp which is used to calculate the
	// gateload roundtrip time
	updateCreatedAtTimestamp(docs)

	bulkDocsResponse := []sgreplicate.DocumentRevisionPair{}

	bulkDocsEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_bulk_docs")
	if err != nil {
		return bulkDocsResponse, err
	}

	bulkDocs := BulkDocs{
		Documents: docs,
		NewEdits:  true,
	}
	docBytes, err := json.Marshal(bulkDocs)
	if err != nil {
		return bulkDocsResponse, err
	}
	var buf *bytes.Buffer
	if s.CompressionEnabled {
		buf = &bytes.Buffer{}
		gzipWriter := gzip.NewWriter(buf)
		if _, err := gzipWriter.Write(docBytes); err != nil {
			return bulkDocsResponse, err
		}
		if err = gzipWriter.Close(); err != nil {
			return bulkDocsResponse, err
		}
	} else {
		buf = bytes.NewBuffer(docBytes)
	}

	req, err := http.NewRequest("POST", bulkDocsEndpoint, buf)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")
	if s.CompressionEnabled {
		req.Header.Set("Content-Encoding", "gzip")
	}

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return bulkDocsResponse, err
	}
	s.pushTimingStat("create_document", timeDeltaPerDocument(len(docs), time.Since(startTime)))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return bulkDocsResponse, fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	defer resp.Body.Close()

	decoder := json.NewDecoder(resp.Body)
	if err = decoder.Decode(&bulkDocsResponse); err != nil {
		return bulkDocsResponse, err
	}

	// If any of the bulk docs had errors, return an error
	for _, docRevisionPair := range bulkDocsResponse {
		if docRevisionPair.Error != "" {
			return bulkDocsResponse, fmt.Errorf("%v", docRevisionPair.Error)
		}
	}

	return bulkDocsResponse, nil

}

func (s SGDataStore) BulkGetDocuments(r sgreplicate.BulkGetRequest) ([]sgreplicate.Document, error) {

	defer s.pushCounter("get_document_counter", len(r.Docs))

	bulkGetEndpoint, err := addEndpointToUrl(s.SyncGatewayUrl, "_bulk_get")
	if err != nil {
		return nil, err
	}

	bulkGetBytes, err := json.Marshal(r)
	if err != nil {
		return nil, fmt.Errorf("BulkGetDocuemnts failed to marshal request: %v", err)
	}

	buf := bytes.NewBuffer(bulkGetBytes)

	req, err := http.NewRequest("POST", bulkGetEndpoint, buf)
	s.addAuthIfNeeded(req)

	req.Header.Set("Content-Type", "application/json")

	client := http.DefaultClient

	startTime := time.Now()
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	s.pushTimingStat("get_document", timeDeltaPerDocument(len(r.Docs), time.Since(startTime)))
	if resp.StatusCode < 200 || resp.StatusCode > 201 {
		return nil, fmt.Errorf("Unexpected response status for POST request: %d", resp.StatusCode)
	}

	// Parse the response and make sure that we got all the docs we requested
	defer resp.Body.Close()

	// Logger function that ignores any logging from the ReadBulkGetResponse method
	loggerFunc := func(key string, format string, args ...interface{}) {}
	documents, err := sgreplicate.ReadBulkGetResponse(resp, loggerFunc)
	if len(documents) != len(r.Docs) {
		return nil, fmt.Errorf("Expected %d docs, got %d docs", len(r.Docs), len(documents))
	}

	for _, doc := range documents {
		createAtRFC3339NanoIface, ok := doc.Body["created_at"]
		if !ok {
			logger.Warn("Document missing created_at field", "doc.Body", doc.Body)
			continue
		}
		createAtRFC3339NanoStr, ok := createAtRFC3339NanoIface.(string)
		if !ok {
			logger.Warn("Document created_at not a string", "doc.Body", doc.Body)
			continue
		}
		createAtRFC3339Nano, err := time.Parse(
			time.RFC3339Nano,
			createAtRFC3339NanoStr,
		)
		if err != nil {
			logger.Warn("Could not parse doc.created_at field into time", "createAtRFC3339Nano", createAtRFC3339Nano)
			continue
		}
		delta := time.Since(createAtRFC3339Nano)
		s.pushTimingStat("gateload_roundtrip", delta)

		possiblyLogVerboseWarning(delta, doc)

	}

	return documents, nil

}

// If the round trip time is over a certain threshold, log a verbose
// warning.  Trying to debug https://github.com/couchbaselabs/sgload/issues/12
func possiblyLogVerboseWarning(delta time.Duration, doc sgreplicate.Document) {

	maxSecondsExpected := time.Duration(60)
	if delta > time.Duration(time.Second*maxSecondsExpected) {

		createAtRFC3339NanoIface, ok := doc.Body["created_at"]
		if !ok {
			logger.Warn("Document missing created_at field", "doc.Body", doc.Body)
		}
		createAtRFC3339NanoStr, ok := createAtRFC3339NanoIface.(string)
		if !ok {
			logger.Warn("Document created_at not a string", "doc.Body", doc.Body)
		}
		createAtRFC3339Nano, err := time.Parse(
			time.RFC3339Nano,
			createAtRFC3339NanoStr,
		)
		if err != nil {
			logger.Warn("Could not parse doc.created_at field into time", "createAtRFC3339Nano", createAtRFC3339Nano)
		}

		logger.Warn(
			"Gateload roundtrip time exceeeded expected time",
			"maxSecondsExpected",
			maxSecondsExpected,
			"delta",
			delta,
			"createAtRFC3339NanoStr",
			createAtRFC3339NanoStr,
			"createAtRFC3339Nano",
			createAtRFC3339Nano,
			"time.Since(createAtRFC3339Nano)",
			time.Since(createAtRFC3339Nano),
			"now",
			time.Now(),
			"now RFC3339Nano",
			time.Now().Format(time.RFC3339Nano),
		)

	}

}

// add BasicAuth header for user if needed
func (s SGDataStore) addAuthIfNeeded(req *http.Request) {
	if !s.UserCreds.Empty() {
		req.SetBasicAuth(s.UserCreds.Username, s.UserCreds.Password)
	}
}

func (s SGDataStore) pushTimingStat(key string, delta time.Duration) {
	if s.StatsdClient == nil {
		return
	}
	s.StatsdClient.Timing(
		statsdSampleRate,
		key,
		delta,
	)
}

func (s SGDataStore) pushCounter(key string, n int) {
	if s.StatsdClient == nil {
		return
	}
	s.StatsdClient.Counter(
		statsdSampleRate,
		key,
		n,
	)
}

func splitHostPortWrapper(host string) (string, string, error) {
	if !strings.Contains(host, ":") {
		return host, "", nil
	}

	return net.SplitHostPort(host)
}

func addEndpointToUrl(urlStr, endpoint string) (string, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return "", err
	}
	u.Path = path.Join(u.Path, endpoint)
	return u.String(), nil
}

func addTrailingSlash(urlStr string) string {
	return fmt.Sprintf("%v/", urlStr)
}

func timeDeltaPerDocument(numDocs int, timeDeltaAllDocs time.Duration) time.Duration {
	if numDocs == 0 {
		return timeDeltaAllDocs
	}
	return time.Duration(int64(timeDeltaAllDocs) / int64(numDocs))
}
<|endoftext|>


<commit_before>//
// command/docker_test.go
//
// Copyright (c) 2016-2017 Junpei Kawamoto
//
// This software is released under the MIT License.
//
// http://opensource.org/licenses/mit-license.php
//

package command

import (
	"fmt"
	"strings"
	"testing"
)

func TestDockerfile(t *testing.T) {

	travis := Travis{}
	opt := DockerfileOpt{
		BaseImage:  "testimage",
		AptProxy:   "http://apt.proxy.test/",
		PypiProxy:  "http://pypi.proxy.test/",
		HTTPProxy:  "http://proxy.test/",
		HTTPSProxy: "https://proxy.test/",
		NoProxy:    "localhost",
	}

	res, err := Dockerfile(&travis, &opt, SourceArchive)
	if err != nil {
		t.Error("Dockerfile returns an error:", err.Error())
	}
	dockerfile := string(res)

	if !strings.Contains(dockerfile, fmt.Sprintln("FROM", opt.BaseImage)) {
		t.Error("The base image of the Dockerfile isn't correct:", dockerfile)
	}

	if !strings.Contains(dockerfile, fmt.Sprintln("ENV HTTP_PROXY", opt.HTTPProxy)) {
		t.Error("Dockerfile doesn't set the correct http proxy:", dockerfile)
	}

	if !strings.Contains(dockerfile, fmt.Sprintln("ENV HTTPS_PROXY", opt.HTTPSProxy)) {
		t.Error("Dockerfile doesn't set the correct https proxy:", dockerfile)
	}

	if !strings.Contains(dockerfile, fmt.Sprintln("ENV NO_PROXY", opt.NoProxy)) {
		t.Error("Dockerfile doesn't set the correct no proxy env:", dockerfile)
	}

	if !strings.Contains(dockerfile, fmt.Sprintf("Acquire::http { Proxy \\\"%s\\\"; };", opt.AptProxy)) {
		t.Error("Dockerfile doesn't specify the correct apt proxy:", dockerfile)
	}

	if !strings.Contains(dockerfile, fmt.Sprintf("RUN PYPI_PROXY=%s", opt.PypiProxy)) {
		t.Error("Dockerfile doesn't specify the correct pypi proxy:", dockerfile)
	}

}

func TestDockerfileWithoutOptions(t *testing.T) {

	travis := Travis{}
	opt := DockerfileOpt{
		BaseImage: "testimage",
	}

	res, err := Dockerfile(&travis, &opt, SourceArchive)
	if err != nil {
		t.Error("Dockerfile returns an error:", err.Error())
	}
	dockerfile := string(res)

	if strings.Contains(dockerfile, "ENV HTTP_PROXY") {
		t.Error("Dockerfile set a http proxy:", dockerfile)
	}

	if strings.Contains(dockerfile, "ENV HTTPS_PROXY") {
		t.Error("Dockerfile set a https proxy:", dockerfile)
	}

	if strings.Contains(dockerfile, "ENV NO_PROXY") {
		t.Error("Dockerfile set no_proxy env:", dockerfile)
	}

	if strings.Contains(dockerfile, "Acquire::http") {
		t.Error("Dockerfile set an apt proxy:", dockerfile)
	}

	if strings.Contains(dockerfile, "RUN PYPI_PROXY=") {
		t.Error("Dockerfile set a pypi proxy:", dockerfile)
	}

}

func TestDockerfilePython(t *testing.T) {

	var travis Travis
	travis.Language = "python"
	travis.Addons.Apt.Packages = []string{"package1", "package2"}
	travis.BeforeInstall = []string{"abc", "def"}

	opt := DockerfileOpt{
		BaseImage: "ubuntu:latest",
	}
	archive := "source.tar.gz"

	res, err := Dockerfile(&travis, &opt, archive)
	if err != nil {
		t.Error("Dockerfile returns an error:", err.Error())
	}
	dockerfile := string(res)

	if !strings.Contains(dockerfile, "RUN pip install --upgrade pip") {
		t.Error("Dockerfile doesn't install pip packages:", dockerfile)
	}

	if !strings.Contains(dockerfile, "RUN apt-get install -y package1") {
		t.Error("Dockerfile doesn't install required packages:", dockerfile)
	}
	if !strings.Contains(dockerfile, "RUN apt-get install -y package2") {
		t.Error("Dockerfile doesn't install required packages:", dockerfile)
	}

	if !strings.Contains(dockerfile, "ADD source.tar.gz /data") {
		t.Error("Dockerfile doesn't add correct source files:", dockerfile)
	}

	if !strings.Contains(dockerfile, "RUN abc") || !strings.Contains(dockerfile, "RUN def") {
		t.Error("Dockerfile doesn't execute commands in before install:", dockerfile)
	}

}
<commit_msg>Add tests to create a dockerfile for go project.<commit_after>//
// command/docker_test.go
//
// Copyright (c) 2016-2017 Junpei Kawamoto
//
// This software is released under the MIT License.
//
// http://opensource.org/licenses/mit-license.php
//

package command

import (
	"fmt"
	"strings"
	"testing"
)

func TestDockerfile(t *testing.T) {

	travis := Travis{}
	opt := DockerfileOpt{
		BaseImage:  "testimage",
		AptProxy:   "http://apt.proxy.test/",
		PypiProxy:  "http://pypi.proxy.test/",
		HTTPProxy:  "http://proxy.test/",
		HTTPSProxy: "https://proxy.test/",
		NoProxy:    "localhost",
	}

	res, err := Dockerfile(&travis, &opt, SourceArchive)
	if err != nil {
		t.Error("Dockerfile returns an error:", err.Error())
	}
	dockerfile := string(res)

	if !strings.Contains(dockerfile, fmt.Sprintln("FROM", opt.BaseImage)) {
		t.Error("The base image of the Dockerfile isn't correct:", dockerfile)
	}

	if !strings.Contains(dockerfile, fmt.Sprintln("ENV HTTP_PROXY", opt.HTTPProxy)) {
		t.Error("Dockerfile doesn't set the correct http proxy:", dockerfile)
	}

	if !strings.Contains(dockerfile, fmt.Sprintln("ENV HTTPS_PROXY", opt.HTTPSProxy)) {
		t.Error("Dockerfile doesn't set the correct https proxy:", dockerfile)
	}

	if !strings.Contains(dockerfile, fmt.Sprintln("ENV NO_PROXY", opt.NoProxy)) {
		t.Error("Dockerfile doesn't set the correct no proxy env:", dockerfile)
	}

	if !strings.Contains(dockerfile, fmt.Sprintf("Acquire::http { Proxy \\\"%s\\\"; };", opt.AptProxy)) {
		t.Error("Dockerfile doesn't specify the correct apt proxy:", dockerfile)
	}

	if !strings.Contains(dockerfile, fmt.Sprintf("RUN PYPI_PROXY=%s", opt.PypiProxy)) {
		t.Error("Dockerfile doesn't specify the correct pypi proxy:", dockerfile)
	}

}

func TestDockerfileWithoutOptions(t *testing.T) {

	travis := Travis{}
	opt := DockerfileOpt{
		BaseImage: "testimage",
	}

	res, err := Dockerfile(&travis, &opt, SourceArchive)
	if err != nil {
		t.Error("Dockerfile returns an error:", err.Error())
	}
	dockerfile := string(res)

	if strings.Contains(dockerfile, "ENV HTTP_PROXY") {
		t.Error("Dockerfile set a http proxy:", dockerfile)
	}

	if strings.Contains(dockerfile, "ENV HTTPS_PROXY") {
		t.Error("Dockerfile set a https proxy:", dockerfile)
	}

	if strings.Contains(dockerfile, "ENV NO_PROXY") {
		t.Error("Dockerfile set no_proxy env:", dockerfile)
	}

	if strings.Contains(dockerfile, "Acquire::http") {
		t.Error("Dockerfile set an apt proxy:", dockerfile)
	}

	if strings.Contains(dockerfile, "RUN PYPI_PROXY=") {
		t.Error("Dockerfile set a pypi proxy:", dockerfile)
	}

}

func TestDockerfilePython(t *testing.T) {

	var travis Travis
	travis.Language = "python"
	travis.Addons.Apt.Packages = []string{"package1", "package2"}
	travis.BeforeInstall = []string{"abc", "def"}

	opt := DockerfileOpt{
		BaseImage: "ubuntu:latest",
	}
	archive := "source.tar.gz"

	res, err := Dockerfile(&travis, &opt, archive)
	if err != nil {
		t.Error("Dockerfile returns an error:", err.Error())
	}
	dockerfile := string(res)

	if !strings.Contains(dockerfile, "RUN pip install --upgrade pip") {
		t.Error("Dockerfile doesn't install pip packages:", dockerfile)
	}

	if !strings.Contains(dockerfile, "RUN apt-get install -y package1") {
		t.Error("Dockerfile doesn't install required packages:", dockerfile)
	}
	if !strings.Contains(dockerfile, "RUN apt-get install -y package2") {
		t.Error("Dockerfile doesn't install required packages:", dockerfile)
	}

	if !strings.Contains(dockerfile, "ADD source.tar.gz /data") {
		t.Error("Dockerfile doesn't add correct source files:", dockerfile)
	}

	if !strings.Contains(dockerfile, "RUN abc") || !strings.Contains(dockerfile, "RUN def") {
		t.Error("Dockerfile doesn't execute commands in before install:", dockerfile)
	}

}

func TestDockerfileGo(t *testing.T) {

	var travis Travis
	travis.Language = "go"
	travis.Addons.Apt.Packages = []string{"package1", "package2"}
	travis.BeforeInstall = []string{"abc", "def"}

	opt := DockerfileOpt{
		BaseImage:  "ubuntu:latest",
		Repository: "path/to/repo",
	}
	archive := "source.tar.gz"

	res, err := Dockerfile(&travis, &opt, archive)
	if err != nil {
		t.Error("Dockerfile returns an error:", err.Error())
	}
	dockerfile := string(res)

	if !strings.Contains(dockerfile, "wget  package1  package2") {
		t.Error("Dockerfile doesn't install required packages:", dockerfile)
	}

	if !strings.Contains(dockerfile, fmt.Sprintf("ADD %s $GOPATH/src/%s", archive, opt.Repository)) {
		t.Error("Dockerfile doesn't add correct source files:", dockerfile)
	}

	if !strings.Contains(dockerfile, "RUN abc") || !strings.Contains(dockerfile, "RUN def") {
		t.Error("Dockerfile doesn't execute commands in before install:", dockerfile)
	}

}
<|endoftext|>


<commit_before>// package network implements micro network node
package network

import (
	"os"
	"strings"
	"time"

	"github.com/micro/cli"
	"github.com/micro/go-micro"
	"github.com/micro/go-micro/network"
	"github.com/micro/go-micro/network/resolver"
	"github.com/micro/go-micro/network/resolver/dns"
	"github.com/micro/go-micro/network/resolver/http"
	"github.com/micro/go-micro/network/resolver/registry"
	"github.com/micro/go-micro/router"
	"github.com/micro/go-micro/tunnel"
	"github.com/micro/go-micro/util/log"
)

var (
	// Name of the network service
	Name = "go.micro.network"
	// Address is the tunnel address
	Address = ":8084"
	// Tunnel is the name of the tunnel
	Tunnel = "tun:0"
	// Resolver is network resolver
	Resolver = "dns"
)

// run runs the micro server
func run(ctx *cli.Context, srvOpts ...micro.Option) {
	// Init plugins
	for _, p := range Plugins() {
		p.Init(ctx)
	}

	if len(ctx.GlobalString("server_name")) > 0 {
		Name = ctx.GlobalString("server_name")
	}
	if len(ctx.String("address")) > 0 {
		Address = ctx.String("address")
	}
	if len(ctx.String("tunnel_id")) > 0 {
		Tunnel = ctx.String("tunnel_id")
		// We need host:port for the Endpoint value in the proxy
		parts := strings.Split(Tunnel, ":")
		if len(parts) == 1 {
			Tunnel = Tunnel + ":0"
		}
	}
	var nodes []string
	if len(ctx.String("server")) > 0 {
		nodes = strings.Split(ctx.String("server"), ",")
	}

	if len(ctx.String("resolver")) > 0 {
		Resolver = ctx.String("resolver")
	}
	var res resolver.Resolver
	switch Resolver {
	case "dns":
		res = &dns.Resolver{}
	case "http":
		res = &http.Resolver{}
	case "registry":
		res = &registry.Resolver{}
	}

	// create a tunnel
	tun := tunnel.NewTunnel(
		tunnel.Address(Address),
		tunnel.Nodes(nodes...),
	)

	// local tunnel router
	rtr := router.NewRouter(
		router.Network(Name),
	)

	// creaate new network
	net := network.NewNetwork(
		network.Name(Name),
		network.Address(Address),
		network.Tunnel(tun),
		network.Router(rtr),
		network.Resolver(res),
	)

	if err := net.Connect(); err != nil {
		log.Logf("Network failed to connect: %v", err)
		os.Exit(1)
	}

	// Initialise service
	service := micro.NewService(
		micro.Name(Name),
		micro.RegisterTTL(time.Duration(ctx.GlobalInt("register_ttl"))*time.Second),
		micro.RegisterInterval(time.Duration(ctx.GlobalInt("register_interval"))*time.Second),
		micro.Server(net.Server()),
	)

	// initialize router
	rtr.Init(
		router.Id(service.Server().Options().Id),
		router.Registry(service.Client().Options().Registry),
	)

	if err := service.Run(); err != nil {
		log.Log("Network %s failed: %v", Name, err)
	}
}

func Commands(options ...micro.Option) []cli.Command {
	command := cli.Command{
		Name:  "network",
		Usage: "Run the micro network node",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:   "address",
				Usage:  "Set the micro network address :8084",
				EnvVar: "MICRO_NETWORK_ADDRESS",
			},
			cli.StringFlag{
				Name:   "tunnel_id",
				Usage:  "Id of the tunnel used as the internal dial/listen address.",
				EnvVar: "MICRO_TUNNEL_ID",
			},
			cli.StringFlag{
				Name:   "server",
				Usage:  "Set the micro network server address. This can be a comma separated list.",
				EnvVar: "MICRO_NETWORK_SERVER",
			},
			cli.StringFlag{
				Name:   "resolver",
				Usage:  "Set the micro network resolver. This can be a comma separated list.",
				EnvVar: "MICRO_NETWORK_RESOLVER",
			},
		},
		Action: func(ctx *cli.Context) {
			run(ctx, options...)
		},
	}

	for _, p := range Plugins() {
		if cmds := p.Commands(); len(cmds) > 0 {
			command.Subcommands = append(command.Subcommands, cmds...)
		}

		if flags := p.Flags(); len(flags) > 0 {
			command.Flags = append(command.Flags, flags...)
		}
	}

	return []cli.Command{command}
}
<commit_msg>Change default resolver to registry<commit_after>// package network implements micro network node
package network

import (
	"os"
	"strings"
	"time"

	"github.com/micro/cli"
	"github.com/micro/go-micro"
	"github.com/micro/go-micro/network"
	"github.com/micro/go-micro/network/resolver"
	"github.com/micro/go-micro/network/resolver/dns"
	"github.com/micro/go-micro/network/resolver/http"
	"github.com/micro/go-micro/network/resolver/registry"
	"github.com/micro/go-micro/router"
	"github.com/micro/go-micro/tunnel"
	"github.com/micro/go-micro/util/log"
)

var (
	// Name of the network service
	Name = "go.micro.network"
	// Address is the network address
	Address = ":8084"
	// Tunnel is the name of the network tunnel
	Tunnel = "tun:0"
	// Resolver is the network resolver
	Resolver = "registry"
)

// run runs the micro server
func run(ctx *cli.Context, srvOpts ...micro.Option) {
	// Init plugins
	for _, p := range Plugins() {
		p.Init(ctx)
	}

	if len(ctx.GlobalString("server_name")) > 0 {
		Name = ctx.GlobalString("server_name")
	}
	if len(ctx.String("address")) > 0 {
		Address = ctx.String("address")
	}
	if len(ctx.String("tunnel_id")) > 0 {
		Tunnel = ctx.String("tunnel_id")
		// We need host:port for the Endpoint value in the proxy
		parts := strings.Split(Tunnel, ":")
		if len(parts) == 1 {
			Tunnel = Tunnel + ":0"
		}
	}
	var nodes []string
	if len(ctx.String("server")) > 0 {
		nodes = strings.Split(ctx.String("server"), ",")
	}

	if len(ctx.String("resolver")) > 0 {
		Resolver = ctx.String("resolver")
	}
	var res resolver.Resolver
	switch Resolver {
	case "dns":
		res = &dns.Resolver{}
	case "http":
		res = &http.Resolver{}
	case "registry":
		res = &registry.Resolver{}
	}

	// create a tunnel
	tun := tunnel.NewTunnel(
		tunnel.Address(Address),
		tunnel.Nodes(nodes...),
	)

	// local tunnel router
	rtr := router.NewRouter(
		router.Network(Name),
	)

	// creaate new network
	net := network.NewNetwork(
		network.Name(Name),
		network.Address(Address),
		network.Tunnel(tun),
		network.Router(rtr),
		network.Resolver(res),
	)

	if err := net.Connect(); err != nil {
		log.Logf("Network failed to connect: %v", err)
		os.Exit(1)
	}

	// Initialise service
	service := micro.NewService(
		micro.Name(Name),
		micro.RegisterTTL(time.Duration(ctx.GlobalInt("register_ttl"))*time.Second),
		micro.RegisterInterval(time.Duration(ctx.GlobalInt("register_interval"))*time.Second),
		micro.Server(net.Server()),
	)

	// initialize router
	rtr.Init(
		router.Id(service.Server().Options().Id),
		router.Registry(service.Client().Options().Registry),
	)

	if err := service.Run(); err != nil {
		log.Logf("Network %s failed: %v", Name, err)
		os.Exit(1)
	}
}

func Commands(options ...micro.Option) []cli.Command {
	command := cli.Command{
		Name:  "network",
		Usage: "Run the micro network node",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:   "address",
				Usage:  "Set the micro network address :8084",
				EnvVar: "MICRO_NETWORK_ADDRESS",
			},
			cli.StringFlag{
				Name:   "tunnel_id",
				Usage:  "Id of the tunnel used as the internal dial/listen address.",
				EnvVar: "MICRO_TUNNEL_ID",
			},
			cli.StringFlag{
				Name:   "server",
				Usage:  "Set the micro network server address. This can be a comma separated list.",
				EnvVar: "MICRO_NETWORK_SERVER",
			},
			cli.StringFlag{
				Name:   "resolver",
				Usage:  "Set the micro network resolver. This can be a comma separated list.",
				EnvVar: "MICRO_NETWORK_RESOLVER",
			},
		},
		Action: func(ctx *cli.Context) {
			run(ctx, options...)
		},
	}

	for _, p := range Plugins() {
		if cmds := p.Commands(); len(cmds) > 0 {
			command.Subcommands = append(command.Subcommands, cmds...)
		}

		if flags := p.Flags(); len(flags) > 0 {
			command.Flags = append(command.Flags, flags...)
		}
	}

	return []cli.Command{command}
}
<|endoftext|>


<commit_before>// Copyright © 2016 Nick Klauer <klauer@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package trackello

import (
	"sort"
	"sync"

	"fmt"
	"github.com/VojtechVitek/go-trello"
	"github.com/pkg/errors"
)

// Board is a super-type for a Trello board.  Board also contains a mutex and map of a List ID to a List.
type Board struct {
	id      string
	board   *trello.Board
	listMux *sync.RWMutex
	lists   map[string]List
}

// NewBoard will create a new Board type, using a trello.Board as a starting point.
func NewBoard(b *trello.Board) *Board {
	return &Board{
		id:      b.Id,
		board:   b,
		listMux: &sync.RWMutex{},
		lists:   make(map[string]List),
	}
}

// PopulateLists will Populate the board's lists with cards and missing data.
func (b *Board) PopulateLists() error {
	lists, err := b.board.Lists()
	if err != nil {
		return errors.Wrapf(err, "Unable to get Lists for Board %s", b.board.Name)
	}
	wg := sync.WaitGroup{}
	for _, list := range lists {
		list := list
		wg.Add(1)
		go func(list trello.List) {
			defer wg.Done()
			// 1. calculate the actions for the list
			trackList := NewList(&list)
			if err = trackList.MapCards(); err != nil {
				return
			}
			// 2. return the list actions to return to the board
			b.listMux.Lock()
			b.lists[trackList.list.Id] = *trackList
			b.listMux.Unlock()
		}(list)
	}
	wg.Wait()

	return nil
}

// MapActions queries Trello's API for all of the recent actions performed on a Board, and maps that to the
// board itself, into a list and card.
func (b *Board) MapActions() error {
	wg := sync.WaitGroup{}
	for _, list := range b.lists {
		wg.Add(1)
		go func(l List) {
			defer wg.Done()
			b.listMux.Lock()
			if err := l.MapActions(); err != nil {
				panic(err)
			}
			b.listMux.Unlock()
		}(list)
	}
	wg.Wait()
	return nil
}

// PrintActions will print the board actions out.
func (b *Board) PrintActions() {
	lists := make([]List, len(b.lists))
	for _, list := range b.lists {
		lists = append(lists, list)
	}
	b.listMux.Lock()
	sort.Sort(ByListName(lists))
	b.listMux.Unlock()
	for _, list := range lists {
		b.listMux.RLock()
		fmt.Printf("%s", list.PrintNonZeroActions())
		b.listMux.RUnlock()
	}
}
<commit_msg>fix data race with error variable in goroutine<commit_after>// Copyright © 2016 Nick Klauer <klauer@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package trackello

import (
	"sort"
	"sync"

	"fmt"
	"github.com/VojtechVitek/go-trello"
	"github.com/pkg/errors"
)

// Board is a super-type for a Trello board.  Board also contains a mutex and map of a List ID to a List.
type Board struct {
	id      string
	board   *trello.Board
	listMux *sync.RWMutex
	lists   map[string]List
}

// NewBoard will create a new Board type, using a trello.Board as a starting point.
func NewBoard(b *trello.Board) *Board {
	return &Board{
		id:      b.Id,
		board:   b,
		listMux: &sync.RWMutex{},
		lists:   make(map[string]List),
	}
}

// PopulateLists will Populate the board's lists with cards and missing data.
func (b *Board) PopulateLists() error {
	lists, err := b.board.Lists()
	if err != nil {
		return errors.Wrapf(err, "Unable to get Lists for Board %s", b.board.Name)
	}
	wg := sync.WaitGroup{}
	for _, list := range lists {
		list := list
		wg.Add(1)
		go func(list trello.List) {
			defer wg.Done()
			// 1. calculate the actions for the list
			trackList := NewList(&list)
			if e := trackList.MapCards(); e != nil {
				return
			}
			// 2. return the list actions to return to the board
			b.listMux.Lock()
			b.lists[trackList.list.Id] = *trackList
			b.listMux.Unlock()
		}(list)
	}
	wg.Wait()

	return nil
}

// MapActions queries Trello's API for all of the recent actions performed on a Board, and maps that to the
// board itself, into a list and card.
func (b *Board) MapActions() error {
	wg := sync.WaitGroup{}
	for _, list := range b.lists {
		wg.Add(1)
		go func(l List) {
			defer wg.Done()
			b.listMux.Lock()
			if err := l.MapActions(); err != nil {
				panic(err)
			}
			b.listMux.Unlock()
		}(list)
	}
	wg.Wait()
	return nil
}

// PrintActions will print the board actions out.
func (b *Board) PrintActions() {
	lists := make([]List, len(b.lists))
	for _, list := range b.lists {
		lists = append(lists, list)
	}
	b.listMux.Lock()
	sort.Sort(ByListName(lists))
	b.listMux.Unlock()
	for _, list := range lists {
		b.listMux.RLock()
		fmt.Printf("%s", list.PrintNonZeroActions())
		b.listMux.RUnlock()
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 go-trello authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package trello

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

type Board struct {
	client         *Client
	Id             string   `json:"id"`
	Name           string   `json:"name"`
	Desc           string   `json:"desc"`
	DescData       string   `json:"descData"`
	Closed         bool     `json:"closed"`
	IdOrganization string   `json:"idOrganization"`
	Pinned         []string `json:"pinned"`
	Url            string   `json:"url"`
	ShortUrl       string   `json:"shortUrl`
	Prefs          struct {
		PermissionLevel       string `json:"permissionLevel"`
		Voting                string `json:"voting"`
		Comments              string `json:"comments"`
		Invitations           string `json:"invitations"`
		SelfJoin              bool   `json:"selfjoin"`
		CardCovers            bool   `json:"cardCovers"`
		CardAging             string `json:"cardAging"`
		CalendarFeedEnabled   bool   `json:"calendarFeedEnabled"`
		Background            string `json:"background"`
		BackgroundColor       string `json:"backgroundColor"`
		BackgroundImage       string `json:"backgroundImage"`
		BackgroundImageScaled string `json:"backgroundImageScaled"`
		BackgroundTile        bool   `json:"backgroundTile"`
		BackgroundBrightness  string `json:"backgroundBrightness"`
		CanBePublic           bool   `json:"canBePublic"`
		CanBeOrg              bool   `json:"canBeOrg"`
		CanBePrivate          bool   `json:"canBePrivate"`
		CanInvite             bool   `json:"canInvite"`
	} `json:"prefs"`
	LabelNames struct {
		Red    string `json:"red"`
		Orange string `json:"orange"`
		Yellow string `json:"yellow"`
		Green  string `json:"green"`
		Blue   string `json:"blue"`
		Purple string `json:"purple"`
	} `json:"labelNames"`
}

func (c *Client) Board(boardId string) (board *Board, err error) {
	req, err := http.NewRequest("GET", c.endpoint+"/boards/"+boardId, nil)
	if err != nil {
		return
	}

	resp, err := c.client.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	} else if resp.StatusCode != 200 {
		err = fmt.Errorf("Received unexpected status %d while trying to retrieve the server data", resp.StatusCode)
		return
	}

	err = json.Unmarshal(body, &board)
	board.client = c
	return
}

func (b *Board) Lists() (lists []List, err error) {
	req, err := http.NewRequest("GET", b.client.endpoint+"/boards/"+b.Id+"/lists", nil)
	if err != nil {
		return
	}

	resp, err := b.client.client.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	} else if resp.StatusCode != 200 {
		err = fmt.Errorf("Received unexpected status %d while trying to retrieve the server data", resp.StatusCode)
		return
	}

	err = json.Unmarshal(body, &lists)
	for i, _ := range lists {
		lists[i].client = b.client
	}
	return
}

func (b *Board) Members() (members []Member, err error) {
	req, err := http.NewRequest("GET", b.client.endpoint+"/boards/"+b.Id+"/members", nil)
	if err != nil {
		return
	}

	resp, err := b.client.client.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	} else if resp.StatusCode != 200 {
		err = fmt.Errorf("Received unexpected status %d while trying to retrieve the server data", resp.StatusCode)
		return
	}

	err = json.Unmarshal(body, &members)
	return
}

func (b *Board) Cards() (cards []Card, err error) {
	req, err := http.NewRequest("GET", b.client.endpoint+"/boards/"+b.Id+"/cards", nil)
	if err != nil {
		return
	}

	resp, err := b.client.client.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	} else if resp.StatusCode != 200 {
		err = fmt.Errorf("Received unexpected status %d while trying to retrieve the server data", resp.StatusCode)
		return
	}

	err = json.Unmarshal(body, &cards)
	for i, _ := range cards {
		cards[i].client = b.client
	}
	return
}

func (b *Board) Card(IdCard string) (card *Card, err error) {
	req, err := http.NewRequest("GET", b.client.endpoint+"/boards/"+b.Id+"/cards/"+IdCard, nil)
	if err != nil {
		return
	}

	resp, err := b.client.client.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	} else if resp.StatusCode != 200 {
		err = fmt.Errorf("Received unexpected status %d while trying to retrieve the server data", resp.StatusCode)
		return
	}

	err = json.Unmarshal(body, &card)
	card.client = b.client
	return
}
<commit_msg>Add Board.Checklists() method<commit_after>/*
Copyright 2014 go-trello authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package trello

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

type Board struct {
	client         *Client
	Id             string   `json:"id"`
	Name           string   `json:"name"`
	Desc           string   `json:"desc"`
	DescData       string   `json:"descData"`
	Closed         bool     `json:"closed"`
	IdOrganization string   `json:"idOrganization"`
	Pinned         []string `json:"pinned"`
	Url            string   `json:"url"`
	ShortUrl       string   `json:"shortUrl`
	Prefs          struct {
		PermissionLevel       string `json:"permissionLevel"`
		Voting                string `json:"voting"`
		Comments              string `json:"comments"`
		Invitations           string `json:"invitations"`
		SelfJoin              bool   `json:"selfjoin"`
		CardCovers            bool   `json:"cardCovers"`
		CardAging             string `json:"cardAging"`
		CalendarFeedEnabled   bool   `json:"calendarFeedEnabled"`
		Background            string `json:"background"`
		BackgroundColor       string `json:"backgroundColor"`
		BackgroundImage       string `json:"backgroundImage"`
		BackgroundImageScaled string `json:"backgroundImageScaled"`
		BackgroundTile        bool   `json:"backgroundTile"`
		BackgroundBrightness  string `json:"backgroundBrightness"`
		CanBePublic           bool   `json:"canBePublic"`
		CanBeOrg              bool   `json:"canBeOrg"`
		CanBePrivate          bool   `json:"canBePrivate"`
		CanInvite             bool   `json:"canInvite"`
	} `json:"prefs"`
	LabelNames struct {
		Red    string `json:"red"`
		Orange string `json:"orange"`
		Yellow string `json:"yellow"`
		Green  string `json:"green"`
		Blue   string `json:"blue"`
		Purple string `json:"purple"`
	} `json:"labelNames"`
}

func (c *Client) Board(boardId string) (board *Board, err error) {
	req, err := http.NewRequest("GET", c.endpoint+"/boards/"+boardId, nil)
	if err != nil {
		return
	}

	resp, err := c.client.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	} else if resp.StatusCode != 200 {
		err = fmt.Errorf("Received unexpected status %d while trying to retrieve the server data", resp.StatusCode)
		return
	}

	err = json.Unmarshal(body, &board)
	board.client = c
	return
}

func (b *Board) Lists() (lists []List, err error) {
	req, err := http.NewRequest("GET", b.client.endpoint+"/boards/"+b.Id+"/lists", nil)
	if err != nil {
		return
	}

	resp, err := b.client.client.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	} else if resp.StatusCode != 200 {
		err = fmt.Errorf("Received unexpected status %d while trying to retrieve the server data", resp.StatusCode)
		return
	}

	err = json.Unmarshal(body, &lists)
	for i, _ := range lists {
		lists[i].client = b.client
	}
	return
}

func (b *Board) Members() (members []Member, err error) {
	req, err := http.NewRequest("GET", b.client.endpoint+"/boards/"+b.Id+"/members", nil)
	if err != nil {
		return
	}

	resp, err := b.client.client.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	} else if resp.StatusCode != 200 {
		err = fmt.Errorf("Received unexpected status %d while trying to retrieve the server data", resp.StatusCode)
		return
	}

	err = json.Unmarshal(body, &members)
	return
}

func (b *Board) Cards() (cards []Card, err error) {
	req, err := http.NewRequest("GET", b.client.endpoint+"/boards/"+b.Id+"/cards", nil)
	if err != nil {
		return
	}

	resp, err := b.client.client.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	} else if resp.StatusCode != 200 {
		err = fmt.Errorf("Received unexpected status %d while trying to retrieve the server data", resp.StatusCode)
		return
	}

	err = json.Unmarshal(body, &cards)
	for i, _ := range cards {
		cards[i].client = b.client
	}
	return
}

func (b *Board) Card(IdCard string) (card *Card, err error) {
	req, err := http.NewRequest("GET", b.client.endpoint+"/boards/"+b.Id+"/cards/"+IdCard, nil)
	if err != nil {
		return
	}

	resp, err := b.client.client.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	} else if resp.StatusCode != 200 {
		err = fmt.Errorf("Received unexpected status %d while trying to retrieve the server data", resp.StatusCode)
		return
	}

	err = json.Unmarshal(body, &card)
	card.client = b.client
	return
}

func (b *Board) Checklists() (checklists []Checklist, err error) {
	req, err := http.NewRequest("GET", b.client.endpoint+"/boards/"+b.Id+"/checklists", nil)
	if err != nil {
		return
	}

	resp, err := b.client.client.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	} else if resp.StatusCode != 200 {
		err = fmt.Errorf("Received unexpected status %d while trying to retrieve the server data", resp.StatusCode)
		return
	}

	err = json.Unmarshal(body, &checklists)
	for i, _ := range checklists {
		checklists[i].client = b.client
	}
	return
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/lambda"
)

type awsLambda struct {
	name     string
	payload  []byte
	response string
}

func (a *awsLambda) Invoke(sess *session.Session) error {
	svc := lambda.New(sess)

	params := &lambda.InvokeInput{
		FunctionName: aws.String(a.name),
	}

	if a.payload != nil {
		params.Payload = a.payload
	}

	log.Debug(fmt.Sprintln("Calling [Invoke] with parameters:", params))
	resp, err := svc.Invoke(params)

	if err != nil {
		return err
	}

	if resp.FunctionError != nil {
		return fmt.Errorf(*resp.FunctionError)
	}

	a.response = string(resp.Payload)

	log.Debug(fmt.Sprintln("Lambda response:", a.response))
	return nil
}
<commit_msg>moved invoke command to aws_lambda<commit_after>package commands

import (
	"fmt"
	"qaz/utils"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/lambda"
	"github.com/spf13/cobra"
)

type awsLambda struct {
	name     string
	payload  []byte
	response string
}

func (a *awsLambda) Invoke(sess *session.Session) error {
	svc := lambda.New(sess)

	params := &lambda.InvokeInput{
		FunctionName: aws.String(a.name),
	}

	if a.payload != nil {
		params.Payload = a.payload
	}

	log.Debug(fmt.Sprintln("Calling [Invoke] with parameters:", params))
	resp, err := svc.Invoke(params)

	if err != nil {
		return err
	}

	if resp.FunctionError != nil {
		return fmt.Errorf(*resp.FunctionError)
	}

	a.response = string(resp.Payload)

	log.Debug(fmt.Sprintln("Lambda response:", a.response))
	return nil
}

// invoke command
var invokeCmd = &cobra.Command{
	Use:    "invoke",
	Short:  "Invoke AWS Lambda Functions",
	PreRun: initialise,
	Run: func(cmd *cobra.Command, args []string) {

		if len(args) < 1 {
			fmt.Println("No Lambda Function specified")
			return
		}

		sess, err := manager.GetSess(run.profile)
		utils.HandleError(err)

		f := awsLambda{name: args[0]}

		if run.funcEvent != "" {
			f.payload = []byte(run.funcEvent)
		}

		if err := f.Invoke(sess); err != nil {
			if strings.Contains(err.Error(), "Unhandled") {
				log.Error(fmt.Sprintf("Unhandled Exception: Potential Issue with Lambda Function Logic: %s...\n", f.name))
			}
			utils.HandleError(err)
		}

		fmt.Println(f.response)

	},
}
<|endoftext|>


<commit_before>package etcdrunner

import (
	"bufio"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"sync"
	"time"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd"
	"github.com/flynn/flynn/pkg/attempt"
)

var Attempts = attempt.Strategy{
	Min:   5,
	Total: 5 * time.Second,
	Delay: 200 * time.Millisecond,
}

type TestingT interface {
	Fatal(...interface{})
	Fatalf(string, ...interface{})
	Log(...interface{})
}

func RunEtcdServer(t TestingT) (string, func()) {
	killCh := make(chan struct{})
	doneCh := make(chan struct{})
	dataDir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatal("tempdir failed: ", err)
	}
	port, err := RandomPort()
	if err != nil {
		t.Fatal("error getting random etcd port: ", err)
	}
	clusterPort, err := RandomPort()
	if err != nil {
		t.Fatal("error getting random cluster port: ", err)
	}
	go func() {
		cmd := exec.Command("etcd",
			"--data-dir", dataDir,
			"--listen-client-urls", "http://127.0.0.1:"+port,
			"--listen-peer-urls", "http://127.0.0.1:"+clusterPort,
			"--initial-advertise-peer-urls", "http://127.0.0.1:"+clusterPort,
			"--initial-cluster", "default=http://127.0.0.1:"+clusterPort,
		)
		var stderr, stdout io.Reader
		if os.Getenv("DEBUG") != "" {
			stderr, _ = cmd.StderrPipe()
			stdout, _ = cmd.StdoutPipe()
		}
		if err := cmd.Start(); err != nil {
			t.Fatal("etcd start failed: ", err)
			return
		}
		cmdDone := make(chan error)
		go func() {
			if stdout != nil {
				LogOutput("etcd", stdout, stderr)
			}
			cmdDone <- cmd.Wait()
		}()
		defer close(doneCh)
		select {
		case <-killCh:
			if err := cmd.Process.Kill(); err != nil {
				t.Fatal("failed to kill etcd: ", err)
				return
			}
			err := <-cmdDone
			t.Log("etcd process exited: ", err)
		case err := <-cmdDone:
			t.Log("etcd process exited unexpectedly: ", err)
			return
		}
		if err := os.RemoveAll(dataDir); err != nil {
			t.Log("etcd data removal failed: ", err)
		}
	}()
	addr := "http://127.0.0.1:" + port

	// wait for etcd to come up
	client := etcd.NewClient([]string{addr})
	err = Attempts.Run(func() (err error) {
		_, err = client.Get("/", false, false)
		if e, ok := err.(*etcd.EtcdError); ok && e.ErrorCode == 100 {
			// Valid 404 from etcd (> v0.5)
			err = nil
		}
		return
	})
	if err != nil {
		t.Fatal("Failed to connect to etcd: ", err)
	}

	return addr, func() {
		close(killCh)
		<-doneCh
	}
}

func LogOutput(name string, rs ...io.Reader) {
	var wg sync.WaitGroup
	wg.Add(len(rs))
	for _, r := range rs {
		go func(r io.Reader) {
			scanner := bufio.NewScanner(r)
			for scanner.Scan() {
				log.Println(name+":", scanner.Text())
			}
			wg.Done()
		}(r)
	}
	wg.Wait()
}

func RandomPort() (string, error) {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return "", err
	}
	_, port, _ := net.SplitHostPort(l.Addr().String())
	l.Close()
	return port, err
}
<commit_msg>discoverd/testutil: Add etcd client advertise flag<commit_after>package etcdrunner

import (
	"bufio"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"sync"
	"time"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd"
	"github.com/flynn/flynn/pkg/attempt"
)

var Attempts = attempt.Strategy{
	Min:   5,
	Total: 5 * time.Second,
	Delay: 200 * time.Millisecond,
}

type TestingT interface {
	Fatal(...interface{})
	Fatalf(string, ...interface{})
	Log(...interface{})
}

func RunEtcdServer(t TestingT) (string, func()) {
	killCh := make(chan struct{})
	doneCh := make(chan struct{})
	dataDir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatal("tempdir failed: ", err)
	}
	port, err := RandomPort()
	if err != nil {
		t.Fatal("error getting random etcd port: ", err)
	}
	clusterPort, err := RandomPort()
	if err != nil {
		t.Fatal("error getting random cluster port: ", err)
	}
	go func() {
		cmd := exec.Command("etcd",
			"--data-dir", dataDir,
			"--listen-client-urls", "http://127.0.0.1:"+port,
			"--advertise-client-urls", "http://127.0.0.1:"+port,
			"--listen-peer-urls", "http://127.0.0.1:"+clusterPort,
			"--initial-advertise-peer-urls", "http://127.0.0.1:"+clusterPort,
			"--initial-cluster", "default=http://127.0.0.1:"+clusterPort,
		)
		var stderr, stdout io.Reader
		if os.Getenv("DEBUG") != "" {
			stderr, _ = cmd.StderrPipe()
			stdout, _ = cmd.StdoutPipe()
		}
		if err := cmd.Start(); err != nil {
			t.Fatal("etcd start failed: ", err)
			return
		}
		cmdDone := make(chan error)
		go func() {
			if stdout != nil {
				LogOutput("etcd", stdout, stderr)
			}
			cmdDone <- cmd.Wait()
		}()
		defer close(doneCh)
		select {
		case <-killCh:
			if err := cmd.Process.Kill(); err != nil {
				t.Fatal("failed to kill etcd: ", err)
				return
			}
			err := <-cmdDone
			t.Log("etcd process exited: ", err)
		case err := <-cmdDone:
			t.Log("etcd process exited unexpectedly: ", err)
			return
		}
		if err := os.RemoveAll(dataDir); err != nil {
			t.Log("etcd data removal failed: ", err)
		}
	}()
	addr := "http://127.0.0.1:" + port

	// wait for etcd to come up
	client := etcd.NewClient([]string{addr})
	err = Attempts.Run(func() (err error) {
		_, err = client.Get("/", false, false)
		if e, ok := err.(*etcd.EtcdError); ok && e.ErrorCode == 100 {
			// Valid 404 from etcd (> v0.5)
			err = nil
		}
		return
	})
	if err != nil {
		t.Fatal("Failed to connect to etcd: ", err)
	}

	return addr, func() {
		close(killCh)
		<-doneCh
	}
}

func LogOutput(name string, rs ...io.Reader) {
	var wg sync.WaitGroup
	wg.Add(len(rs))
	for _, r := range rs {
		go func(r io.Reader) {
			scanner := bufio.NewScanner(r)
			for scanner.Scan() {
				log.Println(name+":", scanner.Text())
			}
			wg.Done()
		}(r)
	}
	wg.Wait()
}

func RandomPort() (string, error) {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return "", err
	}
	_, port, _ := net.SplitHostPort(l.Addr().String())
	l.Close()
	return port, err
}
<|endoftext|>