<commit_before>package minion

import (
	"bytes"
	"encoding/json"
	"log"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/dnaeon/gru/classifier"
	"github.com/dnaeon/gru/task"
	"github.com/dnaeon/gru/utils"

	"github.com/pborman/uuid"
	etcdclient "github.com/coreos/etcd/client"
	"golang.org/x/net/context"
)

// Minions keyspace in etcd
const EtcdMinionSpace = "/gru/minion"

// Etcd Minion
type etcdMinion struct {
	// Name of the minion
	name string

	// Minion root directory in etcd
	rootDir string

	// Minion queue directory in etcd
	queueDir string

	// Log directory of previously executed tasks
	logDir string

	// Classifier directory in etcd
	classifierDir string

	// Minion unique identifier
	id uuid.UUID

	// KeysAPI client to etcd
	kapi etcdclient.KeysAPI

	// Channel over which tasks are sent for processing
	taskQueue chan *task.Task
}

// Creates a new etcd minion
func NewEtcdMinion(name string, cfg etcdclient.Config) Minion {
	c, err := etcdclient.New(cfg)
	if err != nil {
		log.Fatal(err)
	}

	kapi := etcdclient.NewKeysAPI(c)
	id := utils.GenerateUUID(name)
	rootDir := filepath.Join(EtcdMinionSpace, id.String())
	queueDir := filepath.Join(rootDir, "queue")
	classifierDir := filepath.Join(rootDir, "classifier")
	logDir := filepath.Join(rootDir, "log")
	taskQueue := make(chan *task.Task)

	m := &etcdMinion{
		name:          name,
		rootDir:       rootDir,
		queueDir:      queueDir,
		classifierDir: classifierDir,
		logDir:        logDir,
		id:            id,
		kapi:          kapi,
		taskQueue:     taskQueue,
	}

	return m
}

// Checks for any pending tasks and sends them
// for processing if there are any
func (m *etcdMinion) checkQueue() error {
	opts := &etcdclient.GetOptions{
		Recursive: true,
		Sort:      true,
	}

	// Get backlog tasks if any
	// If the directory key in etcd is missing that is okay, since
	// it means there are no pending tasks for processing
	resp, err := m.kapi.Get(context.Background(), m.queueDir, opts)
	if err != nil {
		if eerr, ok := err.(etcdclient.Error); !ok || eerr.Code == etcdclient.ErrorCodeKeyNotFound {
			return err
		}
	}

	backlog := resp.Node.Nodes
	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d pending tasks in queue", len(backlog))
	for _, node := range backlog {
		t, err := EtcdUnmarshalTask(node)
		m.kapi.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		m.taskQueue <- t
	}

	return nil
}

// Runs periodic jobs such as refreshing classifiers and
// updating the lastseen time every five minutes
func (m *etcdMinion) periodicRunner() {
	schedule := time.Minute * 5
	ticker := time.NewTicker(schedule)
	log.Printf("Periodic scheduler set to run every %s\n", schedule)

	for now := range ticker.C {
		// Run any periodic jobs
		m.classify()
		m.SetLastseen(now.Unix())
	}
}

// Processes new tasks
func (m *etcdMinion) processTask(t *task.Task) error {
	var buf bytes.Buffer

	// Update state of task to indicate that we are now processing it
	t.State = task.TaskStateProcessing
	m.SaveTaskResult(t)

	cmd := exec.Command(t.Command, t.Args...)
	cmd.Stdout = &buf
	cmd.Stderr = &buf

	log.Printf("Processing task %s\n", t.TaskID)

	cmdError := cmd.Run()
	t.TimeProcessed = time.Now().Unix()
	t.Result = buf.String()

	if cmdError != nil {
		log.Printf("Failed to process task %s\n", t.TaskID)
		t.Error = cmdError.Error()
		t.State = task.TaskStateFailed
	} else {
		log.Printf("Finished processing task %s\n", t.TaskID)
		t.State = task.TaskStateSuccess
	}

	m.SaveTaskResult(t)

	return cmdError
}

// Classifies the minion
func (m *etcdMinion) classify() error {
	for key, _ := range classifier.Registry {
		klassifier, err := classifier.Get(key)

		if err != nil {
			log.Printf("Failed to get classifier %s: %s\n", key, err)
			continue
		}

		m.SetClassifier(klassifier)
	}

	return nil
}

// Unmarshals task from etcd
func EtcdUnmarshalTask(node *etcdclient.Node) (*task.Task, error) {
	task := new(task.Task)
	err := json.Unmarshal([]byte(node.Value), &task)

	return task, err
}

// Returns the minion unique identifier
func (m *etcdMinion) ID() uuid.UUID {
	return m.id
}

// Set the human-readable name of the minion in etcd
func (m *etcdMinion) SetName(name string) error {
	nameKey := filepath.Join(m.rootDir, "name")
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), nameKey, m.name, opts)
	if err != nil {
		log.Printf("Failed to set name of minion: %s\n", err)
	}

	return err
}

// Set the time the minion was last seen in seconds since the Epoch
func (m *etcdMinion) SetLastseen(s int64) error {
	lastseenKey := filepath.Join(m.rootDir, "lastseen")
	lastseenValue := strconv.FormatInt(s, 10)
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), lastseenKey, lastseenValue, opts)
	if err != nil {
		log.Printf("Failed to set lastseen time: %s\n", err)
	}

	return err
}

// Classifies the minion
func (m *etcdMinion) SetClassifier(c *classifier.Classifier) error {
	// Classifiers in etcd expire after an hour
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
		TTL:       time.Hour,
	}

	// Serialize classifier to JSON and save it in etcd
	data, err := json.Marshal(c)
	if err != nil {
		log.Printf("Failed to serialize classifier %s: %s\n", c.Key, err)
		return err
	}

	// Classifier key in etcd
	klassifierKey := filepath.Join(m.classifierDir, c.Key)
	_, err = m.kapi.Set(context.Background(), klassifierKey, string(data), opts)

	if err != nil {
		log.Printf("Failed to set classifier %s: %s\n", c.Key, err)
		return err
	}

	return nil
}

// Monitors etcd for new tasks
func (m *etcdMinion) TaskListener(c chan<- *task.Task) error {
	log.Printf("Task listener is watching %s\n", m.queueDir)

	watcherOpts := &etcdclient.WatcherOptions{
		Recursive: true,
	}
	watcher := m.kapi.Watcher(m.queueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			log.Printf("Failed to receive task: %s\n", err)
			continue
		}

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		// Unmarshal and remove task from the queue
		t, err := EtcdUnmarshalTask(resp.Node)
		m.kapi.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			log.Printf("Received invalid task %s: %s\n", resp.Node.Key, err)
			continue
		}

		log.Printf("Received task %s\n", t.TaskID)
		c <- t
	}

	return nil
}

// Processes new tasks
func (m *etcdMinion) TaskRunner(c <-chan *task.Task) error {
	log.Println("Starting task runner")

	for t := range c {
		t.State = task.TaskStateQueued
		t.TimeReceived = time.Now().Unix()
		m.SaveTaskResult(t)

		if t.IsConcurrent {
			go m.processTask(t)
		} else {
			m.processTask(t)
		}
	}

	return nil
}

// Saves a task in etcd
func (m *etcdMinion) SaveTaskResult(t *task.Task) error {
	taskKey := filepath.Join(m.logDir, t.TaskID.String())

	data, err := json.Marshal(t)
	if err != nil {
		log.Printf("Failed to serialize task %s: %s\n", t.TaskID, err)
		return err
	}

	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err = m.kapi.Set(context.Background(), taskKey, string(data), opts)
	if err != nil {
		log.Printf("Failed to save task %s: %s\n", t.TaskID, err)
	}

	return err
}

// Main entry point of the minion
func (m *etcdMinion) Serve() error {
	err := m.SetName(m.name)
	if err != nil {
		return err
	}

	now := time.Now().Unix()
	err = m.SetLastseen(now)
	if err != nil {
		return err
	}

	// Start minion services
	go m.classify()
	go m.periodicRunner()
	go m.checkQueue()
	go m.TaskRunner(m.taskQueue)
	go m.TaskListener(m.taskQueue)

	log.Printf("Minion %s is ready to serve", m.ID())

	return nil
}

// Stops the minion and performs any cleanup tasks
func (m *etcdMinion) Stop() error {
	log.Println("Minion is shutting down")

	close(m.taskQueue)

	return nil
}
<commit_msg>SetName() sets the minion name in etcd to the provided argument<commit_after>package minion

import (
	"bytes"
	"encoding/json"
	"log"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/dnaeon/gru/classifier"
	"github.com/dnaeon/gru/task"
	"github.com/dnaeon/gru/utils"

	etcdclient "github.com/coreos/etcd/client"
	"github.com/pborman/uuid"
	"golang.org/x/net/context"
)

// Minions keyspace in etcd
const EtcdMinionSpace = "/gru/minion"

// Etcd Minion
type etcdMinion struct {
	// Name of the minion
	name string

	// Minion root directory in etcd
	rootDir string

	// Minion queue directory in etcd
	queueDir string

	// Log directory of previously executed tasks
	logDir string

	// Classifier directory in etcd
	classifierDir string

	// Minion unique identifier
	id uuid.UUID

	// KeysAPI client to etcd
	kapi etcdclient.KeysAPI

	// Channel over which tasks are sent for processing
	taskQueue chan *task.Task
}

// Creates a new etcd minion
func NewEtcdMinion(name string, cfg etcdclient.Config) Minion {
	c, err := etcdclient.New(cfg)
	if err != nil {
		log.Fatal(err)
	}

	kapi := etcdclient.NewKeysAPI(c)
	id := utils.GenerateUUID(name)
	rootDir := filepath.Join(EtcdMinionSpace, id.String())
	queueDir := filepath.Join(rootDir, "queue")
	classifierDir := filepath.Join(rootDir, "classifier")
	logDir := filepath.Join(rootDir, "log")
	taskQueue := make(chan *task.Task)

	m := &etcdMinion{
		name:          name,
		rootDir:       rootDir,
		queueDir:      queueDir,
		classifierDir: classifierDir,
		logDir:        logDir,
		id:            id,
		kapi:          kapi,
		taskQueue:     taskQueue,
	}

	return m
}

// Checks for any pending tasks and sends them
// for processing if there are any
func (m *etcdMinion) checkQueue() error {
	opts := &etcdclient.GetOptions{
		Recursive: true,
		Sort:      true,
	}

	// Get backlog tasks if any
	// If the directory key in etcd is missing that is okay, since
	// it means there are no pending tasks for processing
	resp, err := m.kapi.Get(context.Background(), m.queueDir, opts)
	if err != nil {
		if eerr, ok := err.(etcdclient.Error); !ok || eerr.Code == etcdclient.ErrorCodeKeyNotFound {
			return err
		}
	}

	backlog := resp.Node.Nodes
	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d pending tasks in queue", len(backlog))
	for _, node := range backlog {
		t, err := EtcdUnmarshalTask(node)
		m.kapi.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		m.taskQueue <- t
	}

	return nil
}

// Runs periodic jobs such as refreshing classifiers and
// updating the lastseen time every five minutes
func (m *etcdMinion) periodicRunner() {
	schedule := time.Minute * 5
	ticker := time.NewTicker(schedule)
	log.Printf("Periodic scheduler set to run every %s\n", schedule)

	for now := range ticker.C {
		// Run any periodic jobs
		m.classify()
		m.SetLastseen(now.Unix())
	}
}

// Processes new tasks
func (m *etcdMinion) processTask(t *task.Task) error {
	var buf bytes.Buffer

	// Update state of task to indicate that we are now processing it
	t.State = task.TaskStateProcessing
	m.SaveTaskResult(t)

	cmd := exec.Command(t.Command, t.Args...)
	cmd.Stdout = &buf
	cmd.Stderr = &buf

	log.Printf("Processing task %s\n", t.TaskID)

	cmdError := cmd.Run()
	t.TimeProcessed = time.Now().Unix()
	t.Result = buf.String()

	if cmdError != nil {
		log.Printf("Failed to process task %s\n", t.TaskID)
		t.Error = cmdError.Error()
		t.State = task.TaskStateFailed
	} else {
		log.Printf("Finished processing task %s\n", t.TaskID)
		t.State = task.TaskStateSuccess
	}

	m.SaveTaskResult(t)

	return cmdError
}

// Classifies the minion
func (m *etcdMinion) classify() error {
	for key, _ := range classifier.Registry {
		klassifier, err := classifier.Get(key)

		if err != nil {
			log.Printf("Failed to get classifier %s: %s\n", key, err)
			continue
		}

		m.SetClassifier(klassifier)
	}

	return nil
}

// Unmarshals task from etcd
func EtcdUnmarshalTask(node *etcdclient.Node) (*task.Task, error) {
	task := new(task.Task)
	err := json.Unmarshal([]byte(node.Value), &task)

	return task, err
}

// Returns the minion unique identifier
func (m *etcdMinion) ID() uuid.UUID {
	return m.id
}

// Set the human-readable name of the minion in etcd
func (m *etcdMinion) SetName(name string) error {
	nameKey := filepath.Join(m.rootDir, "name")
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), nameKey, name, opts)
	if err != nil {
		log.Printf("Failed to set name of minion: %s\n", err)
	}

	return err
}

// Set the time the minion was last seen in seconds since the Epoch
func (m *etcdMinion) SetLastseen(s int64) error {
	lastseenKey := filepath.Join(m.rootDir, "lastseen")
	lastseenValue := strconv.FormatInt(s, 10)
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err := m.kapi.Set(context.Background(), lastseenKey, lastseenValue, opts)
	if err != nil {
		log.Printf("Failed to set lastseen time: %s\n", err)
	}

	return err
}

// Classifies the minion
func (m *etcdMinion) SetClassifier(c *classifier.Classifier) error {
	// Classifiers in etcd expire after an hour
	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
		TTL:       time.Hour,
	}

	// Serialize classifier to JSON and save it in etcd
	data, err := json.Marshal(c)
	if err != nil {
		log.Printf("Failed to serialize classifier %s: %s\n", c.Key, err)
		return err
	}

	// Classifier key in etcd
	klassifierKey := filepath.Join(m.classifierDir, c.Key)
	_, err = m.kapi.Set(context.Background(), klassifierKey, string(data), opts)

	if err != nil {
		log.Printf("Failed to set classifier %s: %s\n", c.Key, err)
		return err
	}

	return nil
}

// Monitors etcd for new tasks
func (m *etcdMinion) TaskListener(c chan<- *task.Task) error {
	log.Printf("Task listener is watching %s\n", m.queueDir)

	watcherOpts := &etcdclient.WatcherOptions{
		Recursive: true,
	}
	watcher := m.kapi.Watcher(m.queueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			log.Printf("Failed to receive task: %s\n", err)
			continue
		}

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		// Unmarshal and remove task from the queue
		t, err := EtcdUnmarshalTask(resp.Node)
		m.kapi.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			log.Printf("Received invalid task %s: %s\n", resp.Node.Key, err)
			continue
		}

		log.Printf("Received task %s\n", t.TaskID)
		c <- t
	}

	return nil
}

// Processes new tasks
func (m *etcdMinion) TaskRunner(c <-chan *task.Task) error {
	log.Println("Starting task runner")

	for t := range c {
		t.State = task.TaskStateQueued
		t.TimeReceived = time.Now().Unix()
		m.SaveTaskResult(t)

		if t.IsConcurrent {
			go m.processTask(t)
		} else {
			m.processTask(t)
		}
	}

	return nil
}

// Saves a task in etcd
func (m *etcdMinion) SaveTaskResult(t *task.Task) error {
	taskKey := filepath.Join(m.logDir, t.TaskID.String())

	data, err := json.Marshal(t)
	if err != nil {
		log.Printf("Failed to serialize task %s: %s\n", t.TaskID, err)
		return err
	}

	opts := &etcdclient.SetOptions{
		PrevExist: etcdclient.PrevIgnore,
	}

	_, err = m.kapi.Set(context.Background(), taskKey, string(data), opts)
	if err != nil {
		log.Printf("Failed to save task %s: %s\n", t.TaskID, err)
	}

	return err
}

// Main entry point of the minion
func (m *etcdMinion) Serve() error {
	err := m.SetName(m.name)
	if err != nil {
		return err
	}

	now := time.Now().Unix()
	err = m.SetLastseen(now)
	if err != nil {
		return err
	}

	// Start minion services
	go m.classify()
	go m.periodicRunner()
	go m.checkQueue()
	go m.TaskRunner(m.taskQueue)
	go m.TaskListener(m.taskQueue)

	log.Printf("Minion %s is ready to serve", m.ID())

	return nil
}

// Stops the minion and performs any cleanup tasks
func (m *etcdMinion) Stop() error {
	log.Println("Minion is shutting down")

	close(m.taskQueue)

	return nil
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build integration

package common

import (
	"testing"
)

func TestIntegration(t *testing.T) {
	t.Log("just for exercising integration coverage, remove in next PR")
}
<commit_msg>Delete integration_test.go (#880)<commit_after><|endoftext|>


<commit_before>package js

import (
	"context"
	"errors"
	// log "github.com/Sirupsen/logrus"
	"github.com/loadimpact/speedboat/lib"
	"github.com/loadimpact/speedboat/stats"
	"github.com/robertkrimen/otto"
	"sync"
	"sync/atomic"
)

var ErrDefaultExport = errors.New("you must export a 'default' function")

const entrypoint = "__$$entrypoint$$__"

type Runner struct {
	Runtime      *Runtime
	DefaultGroup *lib.Group
	Groups       []*lib.Group

	groupIDCounter int64
	testIDCounter  int64
	groupsMutex    sync.Mutex
}

func NewRunner(runtime *Runtime, exports otto.Value) (*Runner, error) {
	expObj := exports.Object()
	if expObj == nil {
		return nil, ErrDefaultExport
	}

	// Values "remember" which VM they belong to, so to get a callable that works across VM copies,
	// we have to stick it in the global scope, then retrieve it again from the new instance.
	callable, err := expObj.Get("default")
	if err != nil {
		return nil, err
	}
	if !callable.IsFunction() {
		return nil, ErrDefaultExport
	}
	if err := runtime.VM.Set(entrypoint, callable); err != nil {
		return nil, err
	}

	r := &Runner{Runtime: runtime}
	r.DefaultGroup = lib.NewGroup("", nil, nil)
	r.Groups = []*lib.Group{r.DefaultGroup}

	return r, nil
}

func (r *Runner) NewVU() (lib.VU, error) {
	u := &VU{
		runner: r,
		vm:     r.Runtime.VM.Copy(),
		group:  r.DefaultGroup,
	}

	callable, err := u.vm.Get(entrypoint)
	if err != nil {
		return nil, err
	}
	u.callable = callable

	if err := u.vm.Set("__vu_impl__", u); err != nil {
		return nil, err
	}

	return u, nil
}

func (r *Runner) GetGroups() []*lib.Group {
	return r.Groups
}

type VU struct {
	ID int64

	runner   *Runner
	vm       *otto.Otto
	callable otto.Value

	ctx   context.Context
	group *lib.Group
}

func (u *VU) RunOnce(ctx context.Context) ([]stats.Sample, error) {
	u.ctx = ctx
	if _, err := u.callable.Call(otto.UndefinedValue()); err != nil {
		u.ctx = nil
		return nil, err
	}
	u.ctx = nil
	return nil, nil
}

func (u *VU) Reconfigure(id int64) error {
	u.ID = id
	return nil
}

func (u *VU) DoGroup(call otto.FunctionCall) otto.Value {
	name := call.Argument(0).String()
	group, ok := u.group.Group(name, &(u.runner.groupIDCounter))
	if !ok {
		u.runner.groupsMutex.Lock()
		u.runner.Groups = append(u.runner.Groups, group)
		u.runner.groupsMutex.Unlock()
	}
	u.group = group
	defer func() { u.group = group.Parent }()

	fn := call.Argument(1)
	if !fn.IsFunction() {
		panic(call.Otto.MakeSyntaxError("fn must be a function"))
	}

	val, err := fn.Call(call.This)
	if err != nil {
		panic(err)
	}
	return val
}

func (u *VU) DoTest(call otto.FunctionCall) otto.Value {
	if len(call.ArgumentList) < 2 {
		return otto.UndefinedValue()
	}

	arg0 := call.Argument(0)
	for _, v := range call.ArgumentList[1:] {
		obj := v.Object()
		if obj == nil {
			panic(call.Otto.MakeTypeError("tests must be objects"))
		}
		for _, name := range obj.Keys() {
			val, err := obj.Get(name)
			if err != nil {
				panic(err)
			}

			var result bool

		typeSwitchLoop:
			for {
				switch {
				case val.IsFunction():
					val, err = val.Call(otto.UndefinedValue(), arg0)
					if err != nil {
						panic(err)
					}
					continue typeSwitchLoop
				case val.IsUndefined() || val.IsNull():
					result = false
				case val.IsBoolean():
					b, err := val.ToBoolean()
					if err != nil {
						panic(err)
					}
					result = b
				case val.IsNumber():
					f, err := val.ToFloat()
					if err != nil {
						panic(err)
					}
					result = (f != 0)
				case val.IsString():
					s, err := val.ToString()
					if err != nil {
						panic(err)
					}
					result = (s != "")
				}
				break
			}

			test, _ := u.group.Test(name, &(u.runner.testIDCounter))
			if result {
				atomic.AddInt64(&(test.Passes), 1)
			} else {
				atomic.AddInt64(&(test.Fails), 1)
			}
		}
	}
	return otto.UndefinedValue()
}
<commit_msg>[refactor] Cleaned up test implementation<commit_after>package js

import (
	"context"
	"errors"
	// log "github.com/Sirupsen/logrus"
	"github.com/loadimpact/speedboat/lib"
	"github.com/loadimpact/speedboat/stats"
	"github.com/robertkrimen/otto"
	"sync"
	"sync/atomic"
)

var ErrDefaultExport = errors.New("you must export a 'default' function")

const entrypoint = "__$$entrypoint$$__"

type Runner struct {
	Runtime      *Runtime
	DefaultGroup *lib.Group
	Groups       []*lib.Group

	groupIDCounter int64
	testIDCounter  int64
	groupsMutex    sync.Mutex
}

func NewRunner(runtime *Runtime, exports otto.Value) (*Runner, error) {
	expObj := exports.Object()
	if expObj == nil {
		return nil, ErrDefaultExport
	}

	// Values "remember" which VM they belong to, so to get a callable that works across VM copies,
	// we have to stick it in the global scope, then retrieve it again from the new instance.
	callable, err := expObj.Get("default")
	if err != nil {
		return nil, err
	}
	if !callable.IsFunction() {
		return nil, ErrDefaultExport
	}
	if err := runtime.VM.Set(entrypoint, callable); err != nil {
		return nil, err
	}

	r := &Runner{Runtime: runtime}
	r.DefaultGroup = lib.NewGroup("", nil, nil)
	r.Groups = []*lib.Group{r.DefaultGroup}

	return r, nil
}

func (r *Runner) NewVU() (lib.VU, error) {
	u := &VU{
		runner: r,
		vm:     r.Runtime.VM.Copy(),
		group:  r.DefaultGroup,
	}

	callable, err := u.vm.Get(entrypoint)
	if err != nil {
		return nil, err
	}
	u.callable = callable

	if err := u.vm.Set("__vu_impl__", u); err != nil {
		return nil, err
	}

	return u, nil
}

func (r *Runner) GetGroups() []*lib.Group {
	return r.Groups
}

type VU struct {
	ID int64

	runner   *Runner
	vm       *otto.Otto
	callable otto.Value

	ctx   context.Context
	group *lib.Group
}

func (u *VU) RunOnce(ctx context.Context) ([]stats.Sample, error) {
	u.ctx = ctx
	if _, err := u.callable.Call(otto.UndefinedValue()); err != nil {
		u.ctx = nil
		return nil, err
	}
	u.ctx = nil
	return nil, nil
}

func (u *VU) Reconfigure(id int64) error {
	u.ID = id
	return nil
}

func (u *VU) DoGroup(call otto.FunctionCall) otto.Value {
	name := call.Argument(0).String()
	group, ok := u.group.Group(name, &(u.runner.groupIDCounter))
	if !ok {
		u.runner.groupsMutex.Lock()
		u.runner.Groups = append(u.runner.Groups, group)
		u.runner.groupsMutex.Unlock()
	}
	u.group = group
	defer func() { u.group = group.Parent }()

	fn := call.Argument(1)
	if !fn.IsFunction() {
		panic(call.Otto.MakeSyntaxError("fn must be a function"))
	}

	val, err := fn.Call(call.This)
	if err != nil {
		panic(err)
	}
	return val
}

func (u *VU) DoTest(call otto.FunctionCall) otto.Value {
	if len(call.ArgumentList) < 2 {
		return otto.UndefinedValue()
	}

	arg0 := call.Argument(0)
	for _, v := range call.ArgumentList[1:] {
		obj := v.Object()
		if obj == nil {
			panic(call.Otto.MakeTypeError("tests must be objects"))
		}
		for _, name := range obj.Keys() {
			val, err := obj.Get(name)
			if err != nil {
				panic(err)
			}

			result, err := Test(val, arg0)
			if err != nil {
				panic(err)
			}

			test, _ := u.group.Test(name, &(u.runner.testIDCounter))
			if result {
				atomic.AddInt64(&(test.Passes), 1)
			} else {
				atomic.AddInt64(&(test.Fails), 1)
			}
		}
	}
	return otto.UndefinedValue()
}

func Test(val, arg0 otto.Value) (bool, error) {
	switch {
	case val.IsFunction():
		val, err := val.Call(otto.UndefinedValue(), arg0)
		if err != nil {
			return false, err
		}
		return Test(val, arg0)
	case val.IsBoolean():
		b, err := val.ToBoolean()
		if err != nil {
			return false, err
		}
		return b, nil
	case val.IsNumber():
		f, err := val.ToFloat()
		if err != nil {
			return false, err
		}
		return f != 0, nil
	case val.IsString():
		s, err := val.ToString()
		if err != nil {
			return false, err
		}
		return s != "", nil
	default:
		return false, nil
	}
}
<|endoftext|>


<commit_before>package deploy

import (
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/cenk/backoff"
	"github.com/fatih/color"
	marathon "github.com/gambol99/go-marathon"

	"github.com/servehub/serve/manifest"
	"github.com/servehub/utils"
)

func init() {
	manifest.PluginRegestry.Add("deploy.marathon", DeployMarathon{})
}

type DeployMarathon struct{}

func (p DeployMarathon) Run(data manifest.Manifest) error {
	if data.GetBool("purge") {
		return p.Uninstall(data)
	} else {
		return p.Install(data)
	}
}

func (p DeployMarathon) Install(data manifest.Manifest) error {
	marathonApi, err := MarathonClient(data.GetString("marathon-address"))
	if err != nil {
		return err
	}

	fullName := data.GetString("app-name")

	backoffSeconds := data.GetFloat("backoff-seconds")
	backoffFactor := data.GetFloat("backoff-factor")
	maxLaunchDelaySeconds := data.GetFloat("max-launch-delay-seconds")
	taskKillGracePeriodSeconds := data.GetFloat("task-kill-grace-period-seconds")
	minHealthCapacity := data.GetFloat("min-health-capacity")
	maxOverCapacity := data.GetFloat("max-over-capacity")

	app := &marathon.Application{
		User:                       data.GetString("user"),
		BackoffSeconds:             &backoffSeconds,
		BackoffFactor:              &backoffFactor,
		MaxLaunchDelaySeconds:      &maxLaunchDelaySeconds,
		TaskKillGracePeriodSeconds: &taskKillGracePeriodSeconds,
		UpgradeStrategy: &marathon.UpgradeStrategy{
			MinimumHealthCapacity: &minHealthCapacity,
			MaximumOverCapacity:   &maxOverCapacity,
		},
	}

	healthPort := ""
	if len(data.GetArray("ports")) > 0 {
	  healthPort = data.GetString("listen-port")
	}

	portArgs := ""
	if healthPort != "" {
		portArgs = "--port " + healthPort
	}

	app.Name(fullName)
	app.Command(fmt.Sprintf("exec serve-tools consul supervisor --service '%s' %s start %s", fullName, portArgs, data.GetString("cmd")))
	app.Count(data.GetInt("instances"))
	app.Memory(float64(data.GetInt("mem")))

	if cpu, err := strconv.ParseFloat(data.GetString("cpu"), 64); err == nil {
		app.CPU(cpu)
	}

	if cluster := data.GetString("cluster"); cluster != "" {
		cs := strings.SplitN(cluster, ":", 2)
		if len(cs) < 2 {
			cs = append(cs, "true")
		}
		app.AddConstraint(cs[0], "CLUSTER", cs[1])
		app.AddLabel(cs[0], cs[1])
	}

	for _, cons := range data.GetArray("constraints") {
		if consArr, ok := cons.Unwrap().([]interface{}); ok {
			consStrings := make([]string, len(consArr))
			for i, c := range consArr {
				consStrings[i] = fmt.Sprintf("%s", c)
			}
			app.AddConstraint(consStrings...)
		}
	}

	for _, port := range data.GetArray("ports") {
		app.AddPortDefinition(marathon.PortDefinition{Name: port.GetStringOr("name", "")}.SetPort(port.GetIntOr("port", 0)))
	}

	app.AddEnv("SERVICE_DEPLOY_TIME", time.Now().Format(time.RFC3339)) // force redeploy app

	for k, v := range data.GetMap("envs") {
		app.AddEnv(k, strings.TrimSpace(fmt.Sprintf("%v", v.Unwrap())))
	}

	for k, v := range data.GetMap("environment") {
		app.AddEnv(k, strings.TrimSpace(fmt.Sprintf("%v", v.Unwrap())))
	}

	for _, uri := range data.GetArrayForce("package-uri") {
		app.AddUris(fmt.Sprintf("%v", uri))
	}

	if data.GetBool("docker.enabled") {
		app.Cmd = nil
		app.EmptyUris()
		app.EmptyPortDefinitions()

		for _, arg := range data.GetArrayForce("docker.args") {
			app.AddArgs(fmt.Sprintf("%v", arg))
		}

		doc := marathon.NewDockerContainer()
		doc.Docker.Image = data.GetString("docker.image")
		doc.Docker.Network = strings.ToUpper(data.GetString("docker.network"))
		doc.Docker.SetForcePullImage(true)
		doc.EmptyVolumes()

    ports := data.GetArray("docker.ports")
    if len(ports) == 0 {
      healthPort = ""
    }

		for _, port := range ports {
			doc.Docker.ExposePort(marathon.PortMapping{
				ContainerPort: port.GetIntOr("containerPort", 0),
				HostPort:      port.GetIntOr("hostPort", 0),
				Name:          port.GetStringOr("name", ""),
				Protocol:      "tcp",
			})

			// set service name for docker-registrator: one name for all ports
			if port.GetIntOr("containerPort", 0) != 0 {
				app.AddEnv(fmt.Sprintf("SERVICE_%d_NAME", port.GetInt("containerPort")), fullName)
			}

			// if exists only default port definition — disable healthcheck
			if len(ports) == 1 && port.GetIntOr("containerPort", 0) == 0 && port.GetIntOr("hostPort", 0) == 0 && port.GetStringOr("name", "") == "" {
			  healthPort = ""
			}
		}

		for _, vol := range data.GetArray("docker.volumes") {
			doc.Volume(vol.GetString("hostPath"), vol.GetString("containerPath"), vol.GetString("mode"))
		}

		for k, v := range data.GetMap("docker.parameters") {
			doc.Docker.AddParameter(k, fmt.Sprintf("%v", v.Unwrap()))
		}

		app.Container = doc
	}

	if healthPort != "" {
		health := marathon.NewDefaultHealthCheck()
		health.Protocol = "TCP"
		app.AddHealthCheck(*health)
	} else {
	  delete(*app.Env, "SERVICE_CHECK_TCP")
	}

	if _, err := marathonApi.UpdateApplication(app, false); err != nil {
		color.Yellow("marathon <- %s", app)
		return err
	}

	color.Green("marathon <- %s", app)

	consulApi, err := utils.ConsulClient(data.GetString("consul-address"))
	if err != nil {
		return err
	}

	if err := utils.RegisterPluginData("deploy.marathon", fullName, data.String(), data.GetString("consul-address")); err != nil {
		return err
	}

	bc := backoff.NewExponentialBackOff()
	if maxBc, err := time.ParseDuration(data.GetString("backoff-max-elapsed-time")); err == nil {
		bc.MaxElapsedTime = maxBc
	} else {
		log.Println(color.YellowString("Error on parse `backoff-max-elapsed-time` duration `%s`: %v", data.GetString("backoff-max-elapsed-time"), err))
	}

	if err := backoff.Retry(func() error {
		services, _, err := consulApi.Health().Service(fullName, "", true, nil)

		if err != nil {
			log.Println(color.RedString("Error in check health in consul: %v", err))
			return err
		}

		if len(services) == 0 {
			log.Printf("Service `%s` not started yet! Retry...", fullName)
			return fmt.Errorf("Service `%s` not started!", fullName)
		}

		log.Println(color.GreenString("Service `%s` successfully started!", fullName))
		return nil
	}, bc); err != nil {
		log.Println(color.RedString("Error on deploy `%s`: %v. Cleanup...", fullName, err))

		if err := utils.MarkAsOutdated(consulApi, fullName, 0); err != nil {
			return err
		}

		return err
	}

	return nil
}

func (p DeployMarathon) Uninstall(data manifest.Manifest) error {
	marathonApi, err := MarathonClient(data.GetString("marathon-address"))
	if err != nil {
		return err
	}

	name := data.GetString("app-name")

	if _, err := marathonApi.Application(name); err == nil {
		if _, err := marathonApi.DeleteApplication(name, false); err != nil {
			return err
		}
	} else {
		log.Println(color.YellowString("App `%s` doesnt exists in marathon!", name))
	}

	return utils.DeletePluginData("deploy.marathon", name, data.GetString("consul-address"))
}

func MarathonClient(marathonAddress string) (marathon.Marathon, error) {
	conf := marathon.NewDefaultConfig()
	conf.URL = fmt.Sprintf("http://%s", marathonAddress)
	conf.LogOutput = os.Stderr
	return marathon.NewClient(conf)
}
<commit_msg>= marathon: health.GracePeriodSeconds = 300<commit_after>package deploy

import (
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/cenk/backoff"
	"github.com/fatih/color"
	marathon "github.com/gambol99/go-marathon"

	"github.com/servehub/serve/manifest"
	"github.com/servehub/utils"
)

func init() {
	manifest.PluginRegestry.Add("deploy.marathon", DeployMarathon{})
}

type DeployMarathon struct{}

func (p DeployMarathon) Run(data manifest.Manifest) error {
	if data.GetBool("purge") {
		return p.Uninstall(data)
	} else {
		return p.Install(data)
	}
}

func (p DeployMarathon) Install(data manifest.Manifest) error {
	marathonApi, err := MarathonClient(data.GetString("marathon-address"))
	if err != nil {
		return err
	}

	fullName := data.GetString("app-name")

	backoffSeconds := data.GetFloat("backoff-seconds")
	backoffFactor := data.GetFloat("backoff-factor")
	maxLaunchDelaySeconds := data.GetFloat("max-launch-delay-seconds")
	taskKillGracePeriodSeconds := data.GetFloat("task-kill-grace-period-seconds")
	minHealthCapacity := data.GetFloat("min-health-capacity")
	maxOverCapacity := data.GetFloat("max-over-capacity")

	app := &marathon.Application{
		User:                       data.GetString("user"),
		BackoffSeconds:             &backoffSeconds,
		BackoffFactor:              &backoffFactor,
		MaxLaunchDelaySeconds:      &maxLaunchDelaySeconds,
		TaskKillGracePeriodSeconds: &taskKillGracePeriodSeconds,
		UpgradeStrategy: &marathon.UpgradeStrategy{
			MinimumHealthCapacity: &minHealthCapacity,
			MaximumOverCapacity:   &maxOverCapacity,
		},
	}

	healthPort := ""
	if len(data.GetArray("ports")) > 0 {
	  healthPort = data.GetString("listen-port")
	}

	portArgs := ""
	if healthPort != "" {
		portArgs = "--port " + healthPort
	}

	app.Name(fullName)
	app.Command(fmt.Sprintf("exec serve-tools consul supervisor --service '%s' %s start %s", fullName, portArgs, data.GetString("cmd")))
	app.Count(data.GetInt("instances"))
	app.Memory(float64(data.GetInt("mem")))

	if cpu, err := strconv.ParseFloat(data.GetString("cpu"), 64); err == nil {
		app.CPU(cpu)
	}

	if cluster := data.GetString("cluster"); cluster != "" {
		cs := strings.SplitN(cluster, ":", 2)
		if len(cs) < 2 {
			cs = append(cs, "true")
		}
		app.AddConstraint(cs[0], "CLUSTER", cs[1])
		app.AddLabel(cs[0], cs[1])
	}

	for _, cons := range data.GetArray("constraints") {
		if consArr, ok := cons.Unwrap().([]interface{}); ok {
			consStrings := make([]string, len(consArr))
			for i, c := range consArr {
				consStrings[i] = fmt.Sprintf("%s", c)
			}
			app.AddConstraint(consStrings...)
		}
	}

	for _, port := range data.GetArray("ports") {
		app.AddPortDefinition(marathon.PortDefinition{Name: port.GetStringOr("name", "")}.SetPort(port.GetIntOr("port", 0)))
	}

	app.AddEnv("SERVICE_DEPLOY_TIME", time.Now().Format(time.RFC3339)) // force redeploy app

	for k, v := range data.GetMap("envs") {
		app.AddEnv(k, strings.TrimSpace(fmt.Sprintf("%v", v.Unwrap())))
	}

	for k, v := range data.GetMap("environment") {
		app.AddEnv(k, strings.TrimSpace(fmt.Sprintf("%v", v.Unwrap())))
	}

	for _, uri := range data.GetArrayForce("package-uri") {
		app.AddUris(fmt.Sprintf("%v", uri))
	}

	if data.GetBool("docker.enabled") {
		app.Cmd = nil
		app.EmptyUris()
		app.EmptyPortDefinitions()

		for _, arg := range data.GetArrayForce("docker.args") {
			app.AddArgs(fmt.Sprintf("%v", arg))
		}

		doc := marathon.NewDockerContainer()
		doc.Docker.Image = data.GetString("docker.image")
		doc.Docker.Network = strings.ToUpper(data.GetString("docker.network"))
		doc.Docker.SetForcePullImage(true)
		doc.EmptyVolumes()

    ports := data.GetArray("docker.ports")
    if len(ports) == 0 {
      healthPort = ""
    }

		for _, port := range ports {
			doc.Docker.ExposePort(marathon.PortMapping{
				ContainerPort: port.GetIntOr("containerPort", 0),
				HostPort:      port.GetIntOr("hostPort", 0),
				Name:          port.GetStringOr("name", ""),
				Protocol:      "tcp",
			})

			// set service name for docker-registrator: one name for all ports
			if port.GetIntOr("containerPort", 0) != 0 {
				app.AddEnv(fmt.Sprintf("SERVICE_%d_NAME", port.GetInt("containerPort")), fullName)
			}

			// if exists only default port definition — disable healthcheck
			if len(ports) == 1 && port.GetIntOr("containerPort", 0) == 0 && port.GetIntOr("hostPort", 0) == 0 && port.GetStringOr("name", "") == "" {
			  healthPort = ""
			}
		}

		for _, vol := range data.GetArray("docker.volumes") {
			doc.Volume(vol.GetString("hostPath"), vol.GetString("containerPath"), vol.GetString("mode"))
		}

		for k, v := range data.GetMap("docker.parameters") {
			doc.Docker.AddParameter(k, fmt.Sprintf("%v", v.Unwrap()))
		}

		app.Container = doc
	}

	if healthPort != "" {
		health := marathon.NewDefaultHealthCheck()
		health.Protocol = "TCP"
		health.GracePeriodSeconds = 300
		app.AddHealthCheck(*health)
	} else {
	  delete(*app.Env, "SERVICE_CHECK_TCP")
	}

	if _, err := marathonApi.UpdateApplication(app, false); err != nil {
		color.Yellow("marathon <- %s", app)
		return err
	}

	color.Green("marathon <- %s", app)

	consulApi, err := utils.ConsulClient(data.GetString("consul-address"))
	if err != nil {
		return err
	}

	if err := utils.RegisterPluginData("deploy.marathon", fullName, data.String(), data.GetString("consul-address")); err != nil {
		return err
	}

	bc := backoff.NewExponentialBackOff()
	if maxBc, err := time.ParseDuration(data.GetString("backoff-max-elapsed-time")); err == nil {
		bc.MaxElapsedTime = maxBc
	} else {
		log.Println(color.YellowString("Error on parse `backoff-max-elapsed-time` duration `%s`: %v", data.GetString("backoff-max-elapsed-time"), err))
	}

	if err := backoff.Retry(func() error {
		services, _, err := consulApi.Health().Service(fullName, "", true, nil)

		if err != nil {
			log.Println(color.RedString("Error in check health in consul: %v", err))
			return err
		}

		if len(services) == 0 {
			log.Printf("Service `%s` not started yet! Retry...", fullName)
			return fmt.Errorf("Service `%s` not started!", fullName)
		}

		log.Println(color.GreenString("Service `%s` successfully started!", fullName))
		return nil
	}, bc); err != nil {
		log.Println(color.RedString("Error on deploy `%s`: %v. Cleanup...", fullName, err))

		if err := utils.MarkAsOutdated(consulApi, fullName, 0); err != nil {
			return err
		}

		return err
	}

	return nil
}

func (p DeployMarathon) Uninstall(data manifest.Manifest) error {
	marathonApi, err := MarathonClient(data.GetString("marathon-address"))
	if err != nil {
		return err
	}

	name := data.GetString("app-name")

	if _, err := marathonApi.Application(name); err == nil {
		if _, err := marathonApi.DeleteApplication(name, false); err != nil {
			return err
		}
	} else {
		log.Println(color.YellowString("App `%s` doesnt exists in marathon!", name))
	}

	return utils.DeletePluginData("deploy.marathon", name, data.GetString("consul-address"))
}

func MarathonClient(marathonAddress string) (marathon.Marathon, error) {
	conf := marathon.NewDefaultConfig()
	conf.URL = fmt.Sprintf("http://%s", marathonAddress)
	conf.LogOutput = os.Stderr
	return marathon.NewClient(conf)
}
<|endoftext|>


<commit_before><commit_msg>The package doc gets a pointer to the README. Part of #370.<commit_after><|endoftext|>


<commit_before>/*
 * Minio Client, (C) 2015, 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"
	"syscall"

	"github.com/fatih/color"
	"github.com/minio/cli"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/minio/pkg/probe"
	"github.com/minio/pb"
)

// mirror specific flags.
var (
	mirrorFlags = []cli.Flag{
		cli.BoolFlag{
			Name:  "help, h",
			Usage: "Help of mirror.",
		},
		cli.BoolFlag{
			Name:  "force",
			Usage: "Force overwrite of an existing target(s).",
		},
		cli.BoolFlag{
			Name:  "fake",
			Usage: "Perform a fake mirror operation.",
		},
	}
)

//  Mirror folders recursively from a single source to many destinations
var mirrorCmd = cli.Command{
	Name:   "mirror",
	Usage:  "Mirror folders recursively from a single source to single destination.",
	Action: mainMirror,
	Flags:  append(mirrorFlags, globalFlags...),
	CustomHelpTemplate: `NAME:
   mc {{.Name}} - {{.Usage}}

USAGE:
   mc {{.Name}} [FLAGS] SOURCE TARGET

FLAGS:
  {{range .Flags}}{{.}}
  {{end}}
EXAMPLES:
   1. Mirror a bucket recursively from Minio cloud storage to a bucket on Amazon S3 cloud storage.
      $ mc {{.Name}} play/photos/2014 s3/backup-photos

   2. Mirror a local folder recursively to Amazon S3 cloud storage.
      $ mc {{.Name}} backup/ s3/archive

   3. Mirror a bucket from aliased Amazon S3 cloud storage to a folder on Windows.
      $ mc {{.Name}} s3\documents\2014\ C:\backup\2014

   4. Mirror a bucket from aliased Amazon S3 cloud storage to a local folder use '--force' to overwrite destination.
      $ mc {{.Name}} --force s3/miniocloud miniocloud-backup

   5. Fake mirror a bucket from Minio cloud storage to a bucket on Amazon S3 cloud storage.
      $ mc {{.Name}} --fake play/photos/2014 s3/backup-photos/2014

`,
}

// mirrorMessage container for file mirror messages
type mirrorMessage struct {
	Status string `json:"status"`
	Source string `json:"source"`
	Target string `json:"target"`
}

// String colorized mirror message
func (m mirrorMessage) String() string {
	return console.Colorize("Mirror", fmt.Sprintf("‘%s’ -> ‘%s’", m.Source, m.Target))
}

// JSON jsonified mirror message
func (m mirrorMessage) JSON() string {
	m.Status = "success"
	mirrorMessageBytes, e := json.Marshal(m)
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(mirrorMessageBytes)
}

// mirrorStatMessage container for mirror accounting message
type mirrorStatMessage struct {
	Total       int64
	Transferred int64
	Speed       float64
}

// mirrorStatMessage mirror accounting message
func (c mirrorStatMessage) String() string {
	speedBox := pb.Format(int64(c.Speed)).To(pb.U_BYTES).String()
	if speedBox == "" {
		speedBox = "0 MB"
	} else {
		speedBox = speedBox + "/s"
	}
	message := fmt.Sprintf("Total: %s, Transferred: %s, Speed: %s", pb.Format(c.Total).To(pb.U_BYTES),
		pb.Format(c.Transferred).To(pb.U_BYTES), speedBox)
	return message
}

// doMirror - Mirror an object to multiple destination. mirrorURLs status contains a copy of sURLs and error if any.
func doMirror(sURLs mirrorURLs, progressReader *progressBar, accountingReader *accounter) mirrorURLs {
	if sURLs.Error != nil { // Errorneous sURLs passed.
		sURLs.Error = sURLs.Error.Trace()
		return sURLs
	}

	sourceAlias := sURLs.SourceAlias
	sourceURL := sURLs.SourceContent.URL
	targetAlias := sURLs.TargetAlias
	targetURL := sURLs.TargetContent.URL
	length := sURLs.SourceContent.Size

	if !globalQuiet && !globalJSON {
		progressReader = progressReader.SetCaption(sourceURL.String() + ": ")
	}

	reader, err := getSourceStreamFromAlias(sourceAlias, sourceURL.String())
	if err != nil {
		sURLs.Error = err.Trace(sourceURL.String())
		return sURLs
	}

	var progress io.Reader
	if globalQuiet || globalJSON {
		sourcePath := filepath.ToSlash(filepath.Join(sourceAlias, sourceURL.Path))
		targetPath := filepath.ToSlash(filepath.Join(targetAlias, targetURL.Path))
		printMsg(mirrorMessage{
			Source: sourcePath,
			Target: targetPath,
		})
		if globalQuiet || globalJSON {
			progress = accountingReader
		}
	} else {
		// Set up progress bar.
		progress = progressReader.ProgressBar
	}
	_, err = putTargetStreamFromAlias(targetAlias, targetURL.String(), reader, length, progress)
	if err != nil {
		sURLs.Error = err.Trace(targetURL.String())
		return sURLs
	}

	sURLs.Error = nil // just for safety
	return sURLs
}

// doMirrorFake - Perform a fake mirror to update the progress bar appropriately.
func doMirrorFake(sURLs mirrorURLs, progressReader *progressBar) mirrorURLs {
	if !globalDebug && !globalJSON {
		progressReader.ProgressBar.Add64(sURLs.SourceContent.Size)
	}
	return sURLs
}

// doPrepareMirrorURLs scans the source URL and prepares a list of objects for mirroring.
func doPrepareMirrorURLs(session *sessionV6, isForce bool, isFake bool, trapCh <-chan bool) {
	sourceURL := session.Header.CommandArgs[0] // first one is source.
	targetURL := session.Header.CommandArgs[1]
	var totalBytes int64
	var totalObjects int

	// Create a session data file to store the processed URLs.
	dataFP := session.NewDataWriter()

	var scanBar scanBarFunc
	if !globalQuiet && !globalJSON { // set up progress bar
		scanBar = scanBarFactory()
	}

	URLsCh := prepareMirrorURLs(sourceURL, targetURL, isForce, isFake)
	done := false
	for done == false {
		select {
		case sURLs, ok := <-URLsCh:
			if !ok { // Done with URL prepration
				done = true
				break
			}
			if sURLs.Error != nil {
				// Print in new line and adjust to top so that we don't print over the ongoing scan bar
				if !globalQuiet && !globalJSON {
					console.Eraseline()
				}
				errorIf(sURLs.Error.Trace(), "Unable to prepare URLs for mirroring.")
				break
			}
			if sURLs.isEmpty() {
				break
			}
			jsonData, e := json.Marshal(sURLs)
			if e != nil {
				session.Delete()
				fatalIf(probe.NewError(e), "Unable to marshal URLs into JSON.")
			}
			fmt.Fprintln(dataFP, string(jsonData))
			if !globalQuiet && !globalJSON {
				scanBar(sURLs.SourceContent.URL.String())
			}

			totalBytes += sURLs.SourceContent.Size
			totalObjects++
		case <-trapCh:
			// Print in new line and adjust to top so that we don't print over the ongoing scan bar
			if !globalQuiet && !globalJSON {
				console.Eraseline()
			}
			session.Delete() // If we are interrupted during the URL scanning, we drop the session.
			os.Exit(0)
		}
	}
	session.Header.TotalBytes = totalBytes
	session.Header.TotalObjects = totalObjects
	session.Save()
}

// Session'fied mirror command.
func doMirrorSession(session *sessionV6) {
	isForce := session.Header.CommandBoolFlags["force"]
	isFake := session.Header.CommandBoolFlags["fake"]

	// Initialize signal trap.
	trapCh := signalTrap(os.Interrupt, syscall.SIGTERM)

	if !session.HasData() {
		doPrepareMirrorURLs(session, isForce, isFake, trapCh)
	}

	// Enable accounting reader by default.
	accntReader := newAccounter(session.Header.TotalBytes)

	// Set up progress bar.
	var progressReader *progressBar
	if !globalQuiet && !globalJSON {
		progressReader = newProgressBar(session.Header.TotalBytes)
	}

	// Prepare URL scanner from session data file.
	urlScanner := bufio.NewScanner(session.NewDataReader())

	// isCopied returns true if an object has been already copied
	// or not. This is useful when we resume from a session.
	isCopied := isCopiedFactory(session.Header.LastCopied)

	// Wait on status of doMirror() operation.
	var statusCh = make(chan mirrorURLs)

	// Add a wait group for the below go-routine.
	var wg = new(sync.WaitGroup)
	wg.Add(1)

	// Go routine to monitor signal traps if any.
	go func() {
		defer wg.Done()
		for {
			select {
			case <-trapCh:
				// Receive interrupt notification.
				if !globalQuiet && !globalJSON {
					console.Eraseline()
				}
				session.CloseAndDie()
			case sURLs, ok := <-statusCh:
				// Status channel is closed, we should return.
				if !ok {
					return
				}
				if sURLs.Error == nil {
					session.Header.LastCopied = sURLs.SourceContent.URL.String()
					session.Save()
				} else {
					// Print in new line and adjust to top so that we
					// don't print over the ongoing progress bar.
					if !globalQuiet && !globalJSON {
						console.Eraseline()
					}
					errorIf(sURLs.Error.Trace(sURLs.SourceContent.URL.String()),
						fmt.Sprintf("Failed to copy ‘%s’.", sURLs.SourceContent.URL.String()))
					// For all non critical errors we can continue for the
					// remaining files.
					switch sURLs.Error.ToGoError().(type) {
					// Handle this specifically for filesystem related errors.
					case BrokenSymlink, TooManyLevelsSymlink, PathNotFound, PathInsufficientPermission:
						continue
					case BucketNameEmpty, ObjectMissing, ObjectAlreadyExists, BucketDoesNotExist, BucketInvalid:
						continue
					}
					// For critical errors we should exit. Session
					// can be resumed after the user figures out
					// the problem.
					session.CloseAndDie()
				}
			}
		}
	}()

	// Loop through all urls and mirror.
	for urlScanner.Scan() {
		var sURLs mirrorURLs
		// Unmarshal copyURLs from each line.
		json.Unmarshal([]byte(urlScanner.Text()), &sURLs)
		// Verify if previously copied, notify progress bar.
		if isCopied(sURLs.SourceContent.URL.String()) {
			statusCh <- doMirrorFake(sURLs, progressReader)
		} else {
			// Mirror is initiated if its not a fake run.
			if !isFake {
				statusCh <- doMirror(sURLs, progressReader, accntReader)
			} else {
				// fake Mirror is initiated if its a fake run.
				statusCh <- doMirrorFake(sURLs, progressReader)
			}
		}
	}

	// Close the goroutine.
	close(statusCh)

	// Wait for the goroutines to finish.
	wg.Wait()

	if !globalQuiet && !globalJSON {
		if progressReader.ProgressBar.Get() > 0 {
			progressReader.ProgressBar.Finish()
		}
	} else {
		accntStat := accntReader.Stat()
		mrStatMessage := mirrorStatMessage{
			Total:       accntStat.Total,
			Transferred: accntStat.Transferred,
			Speed:       accntStat.Speed,
		}
		console.Println(console.Colorize("Mirror", mrStatMessage.String()))
	}
}

// Main entry point for mirror command.
func mainMirror(ctx *cli.Context) {
	// Set global flags from context.
	setGlobalsFromContext(ctx)

	// check 'mirror' cli arguments.
	checkMirrorSyntax(ctx)

	// Additional command speific theme customization.
	console.SetColor("Mirror", color.New(color.FgGreen, color.Bold))

	var e error
	session := newSessionV6()
	session.Header.CommandType = "mirror"
	session.Header.RootPath, e = os.Getwd()
	if e != nil {
		session.Delete()
		fatalIf(probe.NewError(e), "Unable to get current working folder.")
	}

	// Set command flags from context.
	isForce := ctx.Bool("force")
	isFake := ctx.Bool("fake")
	session.Header.CommandBoolFlags["force"] = isForce
	session.Header.CommandBoolFlags["fake"] = isFake

	// extract URLs.
	session.Header.CommandArgs = ctx.Args()
	doMirrorSession(session)
	session.Delete()
}
<commit_msg>Fix a crash when --fake is coupled with --quiet in mirror cmd<commit_after>/*
 * Minio Client, (C) 2015, 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"
	"syscall"

	"github.com/fatih/color"
	"github.com/minio/cli"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/minio/pkg/probe"
	"github.com/minio/pb"
)

// mirror specific flags.
var (
	mirrorFlags = []cli.Flag{
		cli.BoolFlag{
			Name:  "help, h",
			Usage: "Help of mirror.",
		},
		cli.BoolFlag{
			Name:  "force",
			Usage: "Force overwrite of an existing target(s).",
		},
		cli.BoolFlag{
			Name:  "fake",
			Usage: "Perform a fake mirror operation.",
		},
	}
)

//  Mirror folders recursively from a single source to many destinations
var mirrorCmd = cli.Command{
	Name:   "mirror",
	Usage:  "Mirror folders recursively from a single source to single destination.",
	Action: mainMirror,
	Flags:  append(mirrorFlags, globalFlags...),
	CustomHelpTemplate: `NAME:
   mc {{.Name}} - {{.Usage}}

USAGE:
   mc {{.Name}} [FLAGS] SOURCE TARGET

FLAGS:
  {{range .Flags}}{{.}}
  {{end}}
EXAMPLES:
   1. Mirror a bucket recursively from Minio cloud storage to a bucket on Amazon S3 cloud storage.
      $ mc {{.Name}} play/photos/2014 s3/backup-photos

   2. Mirror a local folder recursively to Amazon S3 cloud storage.
      $ mc {{.Name}} backup/ s3/archive

   3. Mirror a bucket from aliased Amazon S3 cloud storage to a folder on Windows.
      $ mc {{.Name}} s3\documents\2014\ C:\backup\2014

   4. Mirror a bucket from aliased Amazon S3 cloud storage to a local folder use '--force' to overwrite destination.
      $ mc {{.Name}} --force s3/miniocloud miniocloud-backup

   5. Fake mirror a bucket from Minio cloud storage to a bucket on Amazon S3 cloud storage.
      $ mc {{.Name}} --fake play/photos/2014 s3/backup-photos/2014

`,
}

// mirrorMessage container for file mirror messages
type mirrorMessage struct {
	Status string `json:"status"`
	Source string `json:"source"`
	Target string `json:"target"`
}

// String colorized mirror message
func (m mirrorMessage) String() string {
	return console.Colorize("Mirror", fmt.Sprintf("‘%s’ -> ‘%s’", m.Source, m.Target))
}

// JSON jsonified mirror message
func (m mirrorMessage) JSON() string {
	m.Status = "success"
	mirrorMessageBytes, e := json.Marshal(m)
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(mirrorMessageBytes)
}

// mirrorStatMessage container for mirror accounting message
type mirrorStatMessage struct {
	Total       int64
	Transferred int64
	Speed       float64
}

// mirrorStatMessage mirror accounting message
func (c mirrorStatMessage) String() string {
	speedBox := pb.Format(int64(c.Speed)).To(pb.U_BYTES).String()
	if speedBox == "" {
		speedBox = "0 MB"
	} else {
		speedBox = speedBox + "/s"
	}
	message := fmt.Sprintf("Total: %s, Transferred: %s, Speed: %s", pb.Format(c.Total).To(pb.U_BYTES),
		pb.Format(c.Transferred).To(pb.U_BYTES), speedBox)
	return message
}

// doMirror - Mirror an object to multiple destination. mirrorURLs status contains a copy of sURLs and error if any.
func doMirror(sURLs mirrorURLs, progressReader *progressBar, accountingReader *accounter) mirrorURLs {
	if sURLs.Error != nil { // Errorneous sURLs passed.
		sURLs.Error = sURLs.Error.Trace()
		return sURLs
	}

	sourceAlias := sURLs.SourceAlias
	sourceURL := sURLs.SourceContent.URL
	targetAlias := sURLs.TargetAlias
	targetURL := sURLs.TargetContent.URL
	length := sURLs.SourceContent.Size

	if !globalQuiet && !globalJSON {
		progressReader = progressReader.SetCaption(sourceURL.String() + ": ")
	}

	reader, err := getSourceStreamFromAlias(sourceAlias, sourceURL.String())
	if err != nil {
		sURLs.Error = err.Trace(sourceURL.String())
		return sURLs
	}

	var progress io.Reader
	if globalQuiet || globalJSON {
		sourcePath := filepath.ToSlash(filepath.Join(sourceAlias, sourceURL.Path))
		targetPath := filepath.ToSlash(filepath.Join(targetAlias, targetURL.Path))
		printMsg(mirrorMessage{
			Source: sourcePath,
			Target: targetPath,
		})
		if globalQuiet || globalJSON {
			progress = accountingReader
		}
	} else {
		// Set up progress bar.
		progress = progressReader.ProgressBar
	}
	_, err = putTargetStreamFromAlias(targetAlias, targetURL.String(), reader, length, progress)
	if err != nil {
		sURLs.Error = err.Trace(targetURL.String())
		return sURLs
	}

	sURLs.Error = nil // just for safety
	return sURLs
}

// doMirrorFake - Perform a fake mirror to update the progress bar appropriately.
func doMirrorFake(sURLs mirrorURLs, progressReader *progressBar) mirrorURLs {
	if !globalDebug && !globalJSON && !globalQuiet {
		progressReader.ProgressBar.Add64(sURLs.SourceContent.Size)
	}
	return sURLs
}

// doPrepareMirrorURLs scans the source URL and prepares a list of objects for mirroring.
func doPrepareMirrorURLs(session *sessionV6, isForce bool, isFake bool, trapCh <-chan bool) {
	sourceURL := session.Header.CommandArgs[0] // first one is source.
	targetURL := session.Header.CommandArgs[1]
	var totalBytes int64
	var totalObjects int

	// Create a session data file to store the processed URLs.
	dataFP := session.NewDataWriter()

	var scanBar scanBarFunc
	if !globalQuiet && !globalJSON { // set up progress bar
		scanBar = scanBarFactory()
	}

	URLsCh := prepareMirrorURLs(sourceURL, targetURL, isForce, isFake)
	done := false
	for done == false {
		select {
		case sURLs, ok := <-URLsCh:
			if !ok { // Done with URL prepration
				done = true
				break
			}
			if sURLs.Error != nil {
				// Print in new line and adjust to top so that we don't print over the ongoing scan bar
				if !globalQuiet && !globalJSON {
					console.Eraseline()
				}
				errorIf(sURLs.Error.Trace(), "Unable to prepare URLs for mirroring.")
				break
			}
			if sURLs.isEmpty() {
				break
			}
			jsonData, e := json.Marshal(sURLs)
			if e != nil {
				session.Delete()
				fatalIf(probe.NewError(e), "Unable to marshal URLs into JSON.")
			}
			fmt.Fprintln(dataFP, string(jsonData))
			if !globalQuiet && !globalJSON {
				scanBar(sURLs.SourceContent.URL.String())
			}

			totalBytes += sURLs.SourceContent.Size
			totalObjects++
		case <-trapCh:
			// Print in new line and adjust to top so that we don't print over the ongoing scan bar
			if !globalQuiet && !globalJSON {
				console.Eraseline()
			}
			session.Delete() // If we are interrupted during the URL scanning, we drop the session.
			os.Exit(0)
		}
	}
	session.Header.TotalBytes = totalBytes
	session.Header.TotalObjects = totalObjects
	session.Save()
}

// Session'fied mirror command.
func doMirrorSession(session *sessionV6) {
	isForce := session.Header.CommandBoolFlags["force"]
	isFake := session.Header.CommandBoolFlags["fake"]

	// Initialize signal trap.
	trapCh := signalTrap(os.Interrupt, syscall.SIGTERM)

	if !session.HasData() {
		doPrepareMirrorURLs(session, isForce, isFake, trapCh)
	}

	// Enable accounting reader by default.
	accntReader := newAccounter(session.Header.TotalBytes)

	// Set up progress bar.
	var progressReader *progressBar
	if !globalQuiet && !globalJSON {
		progressReader = newProgressBar(session.Header.TotalBytes)
	}

	// Prepare URL scanner from session data file.
	urlScanner := bufio.NewScanner(session.NewDataReader())

	// isCopied returns true if an object has been already copied
	// or not. This is useful when we resume from a session.
	isCopied := isCopiedFactory(session.Header.LastCopied)

	// Wait on status of doMirror() operation.
	var statusCh = make(chan mirrorURLs)

	// Add a wait group for the below go-routine.
	var wg = new(sync.WaitGroup)
	wg.Add(1)

	// Go routine to monitor signal traps if any.
	go func() {
		defer wg.Done()
		for {
			select {
			case <-trapCh:
				// Receive interrupt notification.
				if !globalQuiet && !globalJSON {
					console.Eraseline()
				}
				session.CloseAndDie()
			case sURLs, ok := <-statusCh:
				// Status channel is closed, we should return.
				if !ok {
					return
				}
				if sURLs.Error == nil {
					session.Header.LastCopied = sURLs.SourceContent.URL.String()
					session.Save()
				} else {
					// Print in new line and adjust to top so that we
					// don't print over the ongoing progress bar.
					if !globalQuiet && !globalJSON {
						console.Eraseline()
					}
					errorIf(sURLs.Error.Trace(sURLs.SourceContent.URL.String()),
						fmt.Sprintf("Failed to copy ‘%s’.", sURLs.SourceContent.URL.String()))
					// For all non critical errors we can continue for the
					// remaining files.
					switch sURLs.Error.ToGoError().(type) {
					// Handle this specifically for filesystem related errors.
					case BrokenSymlink, TooManyLevelsSymlink, PathNotFound, PathInsufficientPermission:
						continue
					case BucketNameEmpty, ObjectMissing, ObjectAlreadyExists, BucketDoesNotExist, BucketInvalid:
						continue
					}
					// For critical errors we should exit. Session
					// can be resumed after the user figures out
					// the problem.
					session.CloseAndDie()
				}
			}
		}
	}()

	// Loop through all urls and mirror.
	for urlScanner.Scan() {
		var sURLs mirrorURLs
		// Unmarshal copyURLs from each line.
		json.Unmarshal([]byte(urlScanner.Text()), &sURLs)
		// Verify if previously copied, notify progress bar.
		if isCopied(sURLs.SourceContent.URL.String()) {
			statusCh <- doMirrorFake(sURLs, progressReader)
		} else {
			// Mirror is initiated if its not a fake run.
			if !isFake {
				statusCh <- doMirror(sURLs, progressReader, accntReader)
			} else {
				// fake Mirror is initiated if its a fake run.
				statusCh <- doMirrorFake(sURLs, progressReader)
			}
		}
	}

	// Close the goroutine.
	close(statusCh)

	// Wait for the goroutines to finish.
	wg.Wait()

	if !globalQuiet && !globalJSON {
		if progressReader.ProgressBar.Get() > 0 {
			progressReader.ProgressBar.Finish()
		}
	} else {
		accntStat := accntReader.Stat()
		mrStatMessage := mirrorStatMessage{
			Total:       accntStat.Total,
			Transferred: accntStat.Transferred,
			Speed:       accntStat.Speed,
		}
		console.Println(console.Colorize("Mirror", mrStatMessage.String()))
	}
}

// Main entry point for mirror command.
func mainMirror(ctx *cli.Context) {
	// Set global flags from context.
	setGlobalsFromContext(ctx)

	// check 'mirror' cli arguments.
	checkMirrorSyntax(ctx)

	// Additional command speific theme customization.
	console.SetColor("Mirror", color.New(color.FgGreen, color.Bold))

	var e error
	session := newSessionV6()
	session.Header.CommandType = "mirror"
	session.Header.RootPath, e = os.Getwd()
	if e != nil {
		session.Delete()
		fatalIf(probe.NewError(e), "Unable to get current working folder.")
	}

	// Set command flags from context.
	isForce := ctx.Bool("force")
	isFake := ctx.Bool("fake")
	session.Header.CommandBoolFlags["force"] = isForce
	session.Header.CommandBoolFlags["fake"] = isFake

	// extract URLs.
	session.Header.CommandArgs = ctx.Args()
	doMirrorSession(session)
	session.Delete()
}
<|endoftext|>


<commit_before>// Copyright 2016 Apcera Inc. All rights reserved.

package server

import (
	"fmt"
	natsd "github.com/nats-io/gnatsd/server"
	"io/ioutil"
	"os"
	"testing"
)

func TestConfigureLogger(t *testing.T) {

	defer RemoveLogger()

	checkDebugTraceOff := func() {
		if debug != 0 || trace != 0 {
			t.Fatalf("Expected debug/trace to be disabled.")
		}
	}

	// Test nil options
	ConfigureLogger(nil, nil)
	checkDebugTraceOff()

	sOpts := GetDefaultOptions()
	nOpts := natsd.Options{}

	// Neither enabled (defaults are off)
	ConfigureLogger(sOpts, &nOpts)
	checkDebugTraceOff()

	// NATS debug options enabled, stan off
	nOpts.Debug = true
	nOpts.Trace = true
	ConfigureLogger(sOpts, &nOpts)
	checkDebugTraceOff()

	// STAN debug options enabled, nats off
	nOpts.Debug = false
	nOpts.Trace = false
	sOpts.Debug = true
	sOpts.Trace = true
	ConfigureLogger(sOpts, &nOpts)
	if debug == 0 || trace == 0 {
		t.Fatalf("Expected debug/trace to be enabled.")
	}

	// All enabled... (coverage)
	nOpts.Debug = true
	nOpts.Trace = true
	sOpts.Debug = true
	sOpts.Trace = true
	ConfigureLogger(sOpts, &nOpts)
	if debug == 0 || trace == 0 {
		t.Fatalf("Expected debug/trace to be enabled.")
	}

	// turn off logging we've enabled
	RemoveLogger()
}

func TestLogging(t *testing.T) {

	defer RemoveLogger()

	// test without a logger
	Noticef("noop")

	sOpts := GetDefaultOptions()
	nOpts := &natsd.Options{}

	// test stdout
	sOpts.Debug = true
	sOpts.Trace = true
	ConfigureLogger(sOpts, nOpts)

	// test syslog
	nOpts = &natsd.Options{}
	nOpts.Syslog = true
	ConfigureLogger(sOpts, nOpts)

	// test remote syslog
	nOpts = &natsd.Options{}
	nOpts.RemoteSyslog = "udp://localhost:514"
	ConfigureLogger(sOpts, nOpts)

	// test file
	tmpDir, err := ioutil.TempDir("", "_stan_server")
	if err != nil {
		t.Fatal("Could not create tmp dir")
	}
	defer os.RemoveAll(tmpDir)

	file, err := ioutil.TempFile(tmpDir, "stan_server:log_")

	nOpts = &natsd.Options{}
	nOpts.LogFile = file.Name()
	ConfigureLogger(sOpts, nOpts)
}

type dummyLogger struct {
	msg string
}

func (d *dummyLogger) Noticef(format string, args ...interface{}) {
	d.msg = fmt.Sprintf(format, args...)
}

func (d *dummyLogger) Debugf(format string, args ...interface{}) {
	d.msg = fmt.Sprintf(format, args...)
}

func (d *dummyLogger) Tracef(format string, args ...interface{}) {
	d.msg = fmt.Sprintf(format, args...)
}

func (d *dummyLogger) Errorf(format string, args ...interface{}) {
	d.msg = fmt.Sprintf(format, args...)
}

func (d *dummyLogger) Fatalf(format string, args ...interface{}) {
	d.msg = fmt.Sprintf(format, args...)
}

func (d *dummyLogger) Reset() {
	d.msg = ""
}

func TestLogOutput(t *testing.T) {
	defer RemoveLogger()

	// dummy to override the configured logger.
	d := &dummyLogger{}

	checkLogger := func(output string) {
		if d.msg != output {
			t.Fatalf("Unexpected logger message: %v", d.msg)
		}
		d.Reset()
	}

	sOpts := GetDefaultOptions()
	ConfigureLogger(sOpts, nil)

	// override the default logger.
	stanLog.Lock()
	stanLog.logger = d
	stanLog.Unlock()

	// write to our logger and check values
	Noticef("foo")
	checkLogger("foo")

	Errorf("foo")
	checkLogger("foo")

	Fatalf("foo")
	checkLogger("foo")

	// debug is NOT set, value should be empty.
	Debugf("foo")
	checkLogger("")

	// trace is NOT set, value should be empty.
	Tracef("foo")
	checkLogger("")

	// enable debug and trace
	sOpts.Debug = true
	sOpts.Trace = true

	// reconfigure with debug/trace enabled
	ConfigureLogger(sOpts, nil)

	// override the default logger.
	stanLog.Lock()
	stanLog.logger = d
	stanLog.Unlock()

	// Debug is set so we should have the value
	Debugf("foo")
	checkLogger("foo")

	// Trace is set so we should have the value
	Tracef("foo")
	checkLogger("foo")
}
<commit_msg>Comment out syslog tests for Travis.<commit_after>// Copyright 2016 Apcera Inc. All rights reserved.

package server

import (
	"fmt"
	natsd "github.com/nats-io/gnatsd/server"
	"io/ioutil"
	"os"
	"testing"
)

func TestConfigureLogger(t *testing.T) {

	defer RemoveLogger()

	checkDebugTraceOff := func() {
		if debug != 0 || trace != 0 {
			t.Fatalf("Expected debug/trace to be disabled.")
		}
	}

	// Test nil options
	ConfigureLogger(nil, nil)
	checkDebugTraceOff()

	sOpts := GetDefaultOptions()
	nOpts := natsd.Options{}

	// Neither enabled (defaults are off)
	ConfigureLogger(sOpts, &nOpts)
	checkDebugTraceOff()

	// NATS debug options enabled, stan off
	nOpts.Debug = true
	nOpts.Trace = true
	ConfigureLogger(sOpts, &nOpts)
	checkDebugTraceOff()

	// STAN debug options enabled, nats off
	nOpts.Debug = false
	nOpts.Trace = false
	sOpts.Debug = true
	sOpts.Trace = true
	ConfigureLogger(sOpts, &nOpts)
	if debug == 0 || trace == 0 {
		t.Fatalf("Expected debug/trace to be enabled.")
	}

	// All enabled... (coverage)
	nOpts.Debug = true
	nOpts.Trace = true
	sOpts.Debug = true
	sOpts.Trace = true
	ConfigureLogger(sOpts, &nOpts)
	if debug == 0 || trace == 0 {
		t.Fatalf("Expected debug/trace to be enabled.")
	}

	// turn off logging we've enabled
	RemoveLogger()
}

func TestLogging(t *testing.T) {

	defer RemoveLogger()

	// test without a logger
	Noticef("noop")

	sOpts := GetDefaultOptions()
	nOpts := &natsd.Options{}

	// test stdout
	sOpts.Debug = true
	sOpts.Trace = true
	ConfigureLogger(sOpts, nOpts)

	// skip syslog until there is support in Travis
	//nOpts = &natsd.Options{}
	//nOpts.Syslog = true
	//ConfigureLogger(sOpts, nOpts)

	//nOpts = &natsd.Options{}
	//nOpts.RemoteSyslog = "udp://localhost:514"
	//ConfigureLogger(sOpts, nOpts)

	// test file
	tmpDir, err := ioutil.TempDir("", "_stan_server")
	if err != nil {
		t.Fatal("Could not create tmp dir")
	}
	defer os.RemoveAll(tmpDir)

	file, err := ioutil.TempFile(tmpDir, "stan_server:log_")

	nOpts = &natsd.Options{}
	nOpts.LogFile = file.Name()
	ConfigureLogger(sOpts, nOpts)
}

type dummyLogger struct {
	msg string
}

func (d *dummyLogger) Noticef(format string, args ...interface{}) {
	d.msg = fmt.Sprintf(format, args...)
}

func (d *dummyLogger) Debugf(format string, args ...interface{}) {
	d.msg = fmt.Sprintf(format, args...)
}

func (d *dummyLogger) Tracef(format string, args ...interface{}) {
	d.msg = fmt.Sprintf(format, args...)
}

func (d *dummyLogger) Errorf(format string, args ...interface{}) {
	d.msg = fmt.Sprintf(format, args...)
}

func (d *dummyLogger) Fatalf(format string, args ...interface{}) {
	d.msg = fmt.Sprintf(format, args...)
}

func (d *dummyLogger) Reset() {
	d.msg = ""
}

func TestLogOutput(t *testing.T) {
	defer RemoveLogger()

	// dummy to override the configured logger.
	d := &dummyLogger{}

	checkLogger := func(output string) {
		if d.msg != output {
			t.Fatalf("Unexpected logger message: %v", d.msg)
		}
		d.Reset()
	}

	sOpts := GetDefaultOptions()
	ConfigureLogger(sOpts, nil)

	// override the default logger.
	stanLog.Lock()
	stanLog.logger = d
	stanLog.Unlock()

	// write to our logger and check values
	Noticef("foo")
	checkLogger("foo")

	Errorf("foo")
	checkLogger("foo")

	Fatalf("foo")
	checkLogger("foo")

	// debug is NOT set, value should be empty.
	Debugf("foo")
	checkLogger("")

	// trace is NOT set, value should be empty.
	Tracef("foo")
	checkLogger("")

	// enable debug and trace
	sOpts.Debug = true
	sOpts.Trace = true

	// reconfigure with debug/trace enabled
	ConfigureLogger(sOpts, nil)

	// override the default logger.
	stanLog.Lock()
	stanLog.logger = d
	stanLog.Unlock()

	// Debug is set so we should have the value
	Debugf("foo")
	checkLogger("foo")

	// Trace is set so we should have the value
	Tracef("foo")
	checkLogger("foo")
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"golang.org/x/crypto/ssh"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/service/ec2"
)

var script string = `
sudo su --login root -c '
mkdir -p ~/go/src/github.com/cloudfoundry-incubator &&
	cd ~/go/src/github.com/cloudfoundry-incubator &&
	git clone git://github.com/cloudfoundry-incubator/garden-linux.git &&
	cd garden-linux &&
	git reset --hard %s &&
	scripts/drone-test
'
`

func main() {
	commit := flag.String("commit", "", "Commit SHA to run against")
	imageId := flag.String("imageID", "", "The ami imageID")
	user := flag.String("user", "ubuntu", "ssh user")
	instanceType := flag.String("instanceType", "m3.large", "The aws instance type")
	keyName := flag.String("keyName", "ci_aws_key", "The aws key name")
	region := flag.String("region", "us-east-1", "The aws region")

	flag.Parse()

	if *commit == "" {
		log.Fatal("The commit SHA is missing.")
	}

	if *imageId == "" {
		log.Fatal("The imageID is missing.")
	}

	creds, err := aws.EnvCreds()
	if err != nil {
		log.Fatal("Please ensure that the AWS_SECRET_KEY and AWS_ACCESS_KEY variables are set")
	}
	ec2Config := &aws.Config{
		Region:      *region,
		Credentials: creds,
	}

	ec2Client := ec2.New(ec2Config)

	maxCount := new(int64)
	*maxCount = 1
	minCount := new(int64)
	*minCount = 1
	instanceSpec := &ec2.RunInstancesInput{
		ImageID:      imageId,
		InstanceType: instanceType,
		KeyName:      keyName,
		MaxCount:     maxCount,
		MinCount:     minCount,
	}
	instance := EC2Instance{
		ec2Client:    ec2Client,
		instanceSpec: instanceSpec,
	}

	signer, err := ssh.ParsePrivateKey([]byte(os.Getenv("SSH_PRIVATE_KEY")))
	if err != nil {
		log.Fatal(err)
	}

	auths := []ssh.AuthMethod{ssh.PublicKeys(signer)}

	sshConfig := &ssh.ClientConfig{
		User: *user,
		Auth: auths,
	}

	sshClient := SSHClient{
		sshConfig: sshConfig,
		protocol:  "tcp",
		stdin:     os.Stdin,
		stdout:    os.Stdout,
		stderr:    os.Stderr,
	}

	exitStatus, err := runCommand(sshClient, instance, fmt.Sprintf(script, *commit))
	if err != nil {
		log.Fatal(err)
	}

	if exitStatus != 0 {
		log.Printf("Remote command exited non zero: %d\n", exitStatus)
	}

	os.Exit(exitStatus)
}

func runCommand(sshClient SSHClient, instance EC2Instance, command string) (int, error) {
	err := instance.Start()
	if err != nil {
		return 1, err
	}
	fmt.Printf("Instance Started: %s\n", instance.InstanceId())
	defer terminate(instance)

	address, err := instance.PublicDNS()
	if err != nil {
		return 1, err
	}
	fmt.Printf("Instance available at: %s\n", address)

	sshClient.host = address
	sshClient.port = 22

	fmt.Println("Running Command")
	exitStatus, err := sshClient.Run(command)
	if err != nil {
		return 1, err
	}

	return exitStatus, nil
}

func terminate(instance EC2Instance) {
	fmt.Printf("Terminating instance: %s\n", instance.InstanceId())
	err := instance.Terminate()
	if err != nil {
		panic(err)
	}
}

type EC2Instance struct {
	ec2Client    *ec2.EC2
	instanceSpec *ec2.RunInstancesInput
	instanceInfo *ec2.Instance
}

func (inst *EC2Instance) InstanceId() string {
	if inst.instanceInfo != nil {
		return *inst.instanceInfo.InstanceID
	}
	return ""
}

func (inst *EC2Instance) Start() error {
	startResp, err := inst.ec2Client.RunInstances(inst.instanceSpec)
	if err != nil {
		return err
	}

	instance := startResp.Instances[0]
	inst.instanceInfo = instance

	key := "Name"
	value := "garden-ci-test-instance"
	_, err = inst.ec2Client.CreateTags(&ec2.CreateTagsInput{
		Resources: []*string{instance.InstanceID},
		Tags: []*ec2.Tag{
			&ec2.Tag{Key: &key, Value: &value},
		},
	})

	return err
}

func (inst *EC2Instance) PublicDNS() (string, error) {
	if name := *inst.instanceInfo.PublicDNSName; name != "" {
		return name, nil
	}

	for attempt := 1; attempt <= 100; attempt++ {
		time.Sleep(500 * time.Millisecond)

		r, err := inst.ec2Client.DescribeInstances(&ec2.DescribeInstancesInput{
			InstanceIDs: []*string{inst.instanceInfo.InstanceID},
		})
		if err != nil {
			return "", err
		}

		if len(r.Reservations) == 0 || len(r.Reservations[0].Instances) == 0 {
			return "", fmt.Errorf("instance not found: %s", inst.instanceInfo.InstanceID)
		}

		rinstance := r.Reservations[0].Instances[0]
		if host := *rinstance.PublicDNSName; host != "" {
			inst.instanceInfo = rinstance
			return host, nil
		}
	}

	return "", errors.New("couldn't determine IP address for instance")
}

func (inst *EC2Instance) Terminate() error {
	_, err := inst.ec2Client.TerminateInstances(&ec2.TerminateInstancesInput{
		InstanceIDs: []*string{inst.instanceInfo.InstanceID},
	})
	return err
}

type SSHClient struct {
	sshConfig *ssh.ClientConfig
	protocol  string
	host      string
	port      int
	stdin     io.Reader
	stdout    io.Writer
	stderr    io.Writer
}

func (cl *SSHClient) Run(command string) (int, error) {
	address := fmt.Sprintf("%s:%d", cl.host, cl.port)

	var client *ssh.Client
	var err error
	for attempt := 1; attempt <= 100; attempt++ {
		time.Sleep(500 * time.Millisecond)
		client, err = ssh.Dial(cl.protocol, address, cl.sshConfig)
		if err == nil {
			break
		}
	}
	if err != nil {
		return 0, fmt.Errorf("Failed to dial: %s", err)
	}

	session, err := client.NewSession()
	if err != nil {
		return 0, fmt.Errorf("Failed to create session: %s", err)
	}
	defer session.Close()

	modes := ssh.TerminalModes{
		ssh.ECHO:          0,     // disable echoing
		ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}

	if err := session.RequestPty("xterm", 80, 40, modes); err != nil {
		return 0, fmt.Errorf("request for pseudo terminal failed: %s", err)
	}

	stdin, err := session.StdinPipe()
	if err != nil {
		return 0, fmt.Errorf("Unable to setup stdin for session: %v\n", err)
	}

	stdout, err := session.StdoutPipe()
	if err != nil {
		return 0, fmt.Errorf("Unable to setup stdout for session: %v\n", err)
	}

	stderr, err := session.StderrPipe()
	if err != nil {
		return 0, fmt.Errorf("Unable to setup stderr for session: %v\n", err)
	}

	go io.Copy(stdin, cl.stdin)
	go io.Copy(cl.stdout, stdout)
	go io.Copy(cl.stderr, stderr)

	err = session.Run(command)
	if err != nil {
		if exitErr, ok := err.(*ssh.ExitError); ok {
			return exitErr.ExitStatus(), nil
		} else {
			return 0, err
		}
	}

	return 0, nil
}
<commit_msg>Use large instance with faster io to get rid of test flakes<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"golang.org/x/crypto/ssh"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/service/ec2"
)

var script string = `
sudo su --login root -c '
mkdir -p ~/go/src/github.com/cloudfoundry-incubator &&
	cd ~/go/src/github.com/cloudfoundry-incubator &&
	git clone git://github.com/cloudfoundry-incubator/garden-linux.git &&
	cd garden-linux &&
	git reset --hard %s &&
	scripts/drone-test
'
`

func main() {
	commit := flag.String("commit", "", "Commit SHA to run against")
	imageId := flag.String("imageID", "", "The ami imageID")
	user := flag.String("user", "ubuntu", "ssh user")
	instanceType := flag.String("instanceType", "m3.xlarge", "The aws instance type")
	keyName := flag.String("keyName", "ci_aws_key", "The aws key name")
	region := flag.String("region", "us-east-1", "The aws region")

	flag.Parse()

	if *commit == "" {
		log.Fatal("The commit SHA is missing.")
	}

	if *imageId == "" {
		log.Fatal("The imageID is missing.")
	}

	creds, err := aws.EnvCreds()
	if err != nil {
		log.Fatal("Please ensure that the AWS_SECRET_KEY and AWS_ACCESS_KEY variables are set")
	}
	ec2Config := &aws.Config{
		Region:      *region,
		Credentials: creds,
	}

	ec2Client := ec2.New(ec2Config)

	maxCount := new(int64)
	*maxCount = 1
	minCount := new(int64)
	*minCount = 1
	ebsOptimized := new(bool)
	*ebsOptimized = true
	instanceSpec := &ec2.RunInstancesInput{
		ImageID:      imageId,
		InstanceType: instanceType,
		KeyName:      keyName,
		MaxCount:     maxCount,
		MinCount:     minCount,
		EBSOptimized: ebsOptimized,
	}
	instance := EC2Instance{
		ec2Client:    ec2Client,
		instanceSpec: instanceSpec,
	}

	signer, err := ssh.ParsePrivateKey([]byte(os.Getenv("SSH_PRIVATE_KEY")))
	if err != nil {
		log.Fatal(err)
	}

	auths := []ssh.AuthMethod{ssh.PublicKeys(signer)}

	sshConfig := &ssh.ClientConfig{
		User: *user,
		Auth: auths,
	}

	sshClient := SSHClient{
		sshConfig: sshConfig,
		protocol:  "tcp",
		stdin:     os.Stdin,
		stdout:    os.Stdout,
		stderr:    os.Stderr,
	}

	exitStatus, err := runCommand(sshClient, instance, fmt.Sprintf(script, *commit))
	if err != nil {
		log.Fatal(err)
	}

	if exitStatus != 0 {
		log.Printf("Remote command exited non zero: %d\n", exitStatus)
	}

	os.Exit(exitStatus)
}

func runCommand(sshClient SSHClient, instance EC2Instance, command string) (int, error) {
	err := instance.Start()
	if err != nil {
		return 1, err
	}
	fmt.Printf("Instance Started: %s\n", instance.InstanceId())
	defer terminate(instance)

	address, err := instance.PublicDNS()
	if err != nil {
		return 1, err
	}
	fmt.Printf("Instance available at: %s\n", address)

	sshClient.host = address
	sshClient.port = 22

	fmt.Println("Running Command")
	exitStatus, err := sshClient.Run(command)
	if err != nil {
		return 1, err
	}

	return exitStatus, nil
}

func terminate(instance EC2Instance) {
	fmt.Printf("Terminating instance: %s\n", instance.InstanceId())
	err := instance.Terminate()
	if err != nil {
		panic(err)
	}
}

type EC2Instance struct {
	ec2Client    *ec2.EC2
	instanceSpec *ec2.RunInstancesInput
	instanceInfo *ec2.Instance
}

func (inst *EC2Instance) InstanceId() string {
	if inst.instanceInfo != nil {
		return *inst.instanceInfo.InstanceID
	}
	return ""
}

func (inst *EC2Instance) Start() error {
	startResp, err := inst.ec2Client.RunInstances(inst.instanceSpec)
	if err != nil {
		return err
	}

	instance := startResp.Instances[0]
	inst.instanceInfo = instance

	key := "Name"
	value := "garden-ci-test-instance"
	_, err = inst.ec2Client.CreateTags(&ec2.CreateTagsInput{
		Resources: []*string{instance.InstanceID},
		Tags: []*ec2.Tag{
			&ec2.Tag{Key: &key, Value: &value},
		},
	})

	return err
}

func (inst *EC2Instance) PublicDNS() (string, error) {
	if name := *inst.instanceInfo.PublicDNSName; name != "" {
		return name, nil
	}

	for attempt := 1; attempt <= 100; attempt++ {
		time.Sleep(500 * time.Millisecond)

		r, err := inst.ec2Client.DescribeInstances(&ec2.DescribeInstancesInput{
			InstanceIDs: []*string{inst.instanceInfo.InstanceID},
		})
		if err != nil {
			return "", err
		}

		if len(r.Reservations) == 0 || len(r.Reservations[0].Instances) == 0 {
			return "", fmt.Errorf("instance not found: %s", inst.instanceInfo.InstanceID)
		}

		rinstance := r.Reservations[0].Instances[0]
		if host := *rinstance.PublicDNSName; host != "" {
			inst.instanceInfo = rinstance
			return host, nil
		}
	}

	return "", errors.New("couldn't determine IP address for instance")
}

func (inst *EC2Instance) Terminate() error {
	_, err := inst.ec2Client.TerminateInstances(&ec2.TerminateInstancesInput{
		InstanceIDs: []*string{inst.instanceInfo.InstanceID},
	})
	return err
}

type SSHClient struct {
	sshConfig *ssh.ClientConfig
	protocol  string
	host      string
	port      int
	stdin     io.Reader
	stdout    io.Writer
	stderr    io.Writer
}

func (cl *SSHClient) Run(command string) (int, error) {
	address := fmt.Sprintf("%s:%d", cl.host, cl.port)

	var client *ssh.Client
	var err error
	for attempt := 1; attempt <= 100; attempt++ {
		time.Sleep(500 * time.Millisecond)
		client, err = ssh.Dial(cl.protocol, address, cl.sshConfig)
		if err == nil {
			break
		}
	}
	if err != nil {
		return 0, fmt.Errorf("Failed to dial: %s", err)
	}

	session, err := client.NewSession()
	if err != nil {
		return 0, fmt.Errorf("Failed to create session: %s", err)
	}
	defer session.Close()

	modes := ssh.TerminalModes{
		ssh.ECHO:          0,     // disable echoing
		ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}

	if err := session.RequestPty("xterm", 80, 40, modes); err != nil {
		return 0, fmt.Errorf("request for pseudo terminal failed: %s", err)
	}

	stdin, err := session.StdinPipe()
	if err != nil {
		return 0, fmt.Errorf("Unable to setup stdin for session: %v\n", err)
	}

	stdout, err := session.StdoutPipe()
	if err != nil {
		return 0, fmt.Errorf("Unable to setup stdout for session: %v\n", err)
	}

	stderr, err := session.StderrPipe()
	if err != nil {
		return 0, fmt.Errorf("Unable to setup stderr for session: %v\n", err)
	}

	go io.Copy(stdin, cl.stdin)
	go io.Copy(cl.stdout, stdout)
	go io.Copy(cl.stderr, stderr)

	err = session.Run(command)
	if err != nil {
		if exitErr, ok := err.(*ssh.ExitError); ok {
			return exitErr.ExitStatus(), nil
		} else {
			return 0, err
		}
	}

	return 0, nil
}
<|endoftext|>