<commit_before>package scheduler

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	log "github.com/golang/glog"
	mesos "github.com/mesos/mesos-go/mesosproto"
	mutil "github.com/mesos/mesos-go/mesosutil"
	bindings "github.com/mesos/mesos-go/scheduler"
	"github.com/mesosphere/kubernetes-mesos/pkg/executor/messages"
	"github.com/mesosphere/kubernetes-mesos/pkg/offers"
	"github.com/mesosphere/kubernetes-mesos/pkg/scheduler/meta"
	"github.com/mesosphere/kubernetes-mesos/pkg/scheduler/podtask"
)

const (
	defaultOfferTTL       = 5    // seconds that an offer is viable, prior to being expired
	defaultOfferLingerTTL = 120  // seconds that an expired offer lingers in history
	defaultListenerDelay  = 1    // number of seconds between offer listener notifications
	defaultUpdatesBacklog = 2048 // size of the pod updates channel
)

type Slave struct {
	HostName string
	Offers   map[string]empty
}

func newSlave(hostName string) *Slave {
	return &Slave{
		HostName: hostName,
		Offers:   make(map[string]empty),
	}
}

type PluginInterface interface {
	// the apiserver may have a different state for the pod than we do
	// so reconcile our records, but only for this one pod
	reconcilePod(api.Pod)

	// execute the Scheduling plugin, should start a go routine and return immediately
	Run()
}

// KubernetesScheduler implements:
// 1: A mesos scheduler.
// 2: A kubernetes scheduler plugin.
// 3: A kubernetes pod.Registry.
type KubernetesScheduler struct {
	// We use a lock here to avoid races
	// between invoking the mesos callback
	// and the invoking the pod registry interfaces.
	// In particular, changes to podtask.T objects are currently guarded by this lock.
	*sync.RWMutex

	// Mesos context.
	executor    *mesos.ExecutorInfo
	driver      bindings.SchedulerDriver
	frameworkId *mesos.FrameworkID
	masterInfo  *mesos.MasterInfo
	registered  bool

	offers       offers.Registry
	slaves       map[string]*Slave // SlaveID => slave.
	slaveIDs     map[string]string // Slave's hostname => slaveID
	taskRegistry podtask.Registry

	scheduleFunc PodScheduleFunc // The function that does scheduling.

	client     *client.Client
	plugin     PluginInterface
	etcdClient tools.EtcdClient
}

// New create a new KubernetesScheduler
func New(executor *mesos.ExecutorInfo, scheduleFunc PodScheduleFunc, client *client.Client, etcdClient tools.EtcdClient) *KubernetesScheduler {
	var k *KubernetesScheduler
	k = &KubernetesScheduler{
		RWMutex:  new(sync.RWMutex),
		executor: executor,
		offers: offers.CreateRegistry(offers.RegistryConfig{
			DeclineOffer: func(id string) error {
				offerId := mutil.NewOfferID(id)
				filters := &mesos.Filters{}
				_, err := k.driver.DeclineOffer(offerId, filters)
				return err
			},
			TTL:           defaultOfferTTL * time.Second,
			LingerTTL:     defaultOfferLingerTTL * time.Second, // remember expired offers so that we can tell if a previously scheduler offer relies on one
			ListenerDelay: defaultListenerDelay * time.Second,
		}),
		slaves:       make(map[string]*Slave),
		slaveIDs:     make(map[string]string),
		taskRegistry: podtask.NewInMemoryRegistry(),
		scheduleFunc: scheduleFunc,
		client:       client,
		etcdClient:   etcdClient,
	}
	return k
}

func (k *KubernetesScheduler) Init(d bindings.SchedulerDriver, pl PluginInterface) {
	k.driver = d
	k.plugin = pl
	k.offers.Init()
}

// Registered is called when the scheduler registered with the master successfully.
func (k *KubernetesScheduler) Registered(driver bindings.SchedulerDriver,
	frameworkId *mesos.FrameworkID, masterInfo *mesos.MasterInfo) {
	k.frameworkId = frameworkId
	k.masterInfo = masterInfo
	k.registered = true
	go k.storeFrameworkId() //TODO(jdef) only do this if we're checkpointing?
	log.Infof("Scheduler registered with the master: %v with frameworkId: %v\n", masterInfo, frameworkId)

	//TODO(jdef) partial reconciliation started... needs work
	r := &Reconciler{Action: k.ReconcileRunningTasks}
	go util.Forever(func() { r.Run(driver) }, 5*time.Minute) // TODO(jdef) parameterize reconciliation interval
}

func (k *KubernetesScheduler) storeFrameworkId() {
	_, err := k.etcdClient.Set(meta.FrameworkIDKey, k.frameworkId.GetValue(), 0)
	if err != nil {
		log.Error(err)
	}
}

// Reregistered is called when the scheduler re-registered with the master successfully.
// This happends when the master fails over.
func (k *KubernetesScheduler) Reregistered(driver bindings.SchedulerDriver, masterInfo *mesos.MasterInfo) {
	log.Infof("Scheduler reregistered with the master: %v\n", masterInfo)
	k.registered = true

	//TODO(jdef) partial reconciliation started... needs work
	r := &Reconciler{Action: k.ReconcileRunningTasks}
	go util.Forever(func() { r.Run(driver) }, 5*time.Minute) // TODO(jdef) parameterize reconciliation interval
}

// Disconnected is called when the scheduler loses connection to the master.
func (k *KubernetesScheduler) Disconnected(driver bindings.SchedulerDriver) {
	log.Infof("Master disconnected!\n")
	k.registered = false

	k.Lock()
	defer k.Unlock()

	// discard all cached offers to avoid unnecessary TASK_LOST updates
	k.offers.Invalidate("")
}

// ResourceOffers is called when the scheduler receives some offers from the master.
func (k *KubernetesScheduler) ResourceOffers(driver bindings.SchedulerDriver, offers []*mesos.Offer) {
	log.V(2).Infof("Received offers %+v", offers)

	k.Lock()
	defer k.Unlock()

	// Record the offers in the global offer map as well as each slave's offer map.
	k.offers.Add(offers)
	for _, offer := range offers {
		offerId := offer.GetId().GetValue()
		slaveId := offer.GetSlaveId().GetValue()

		slave, exists := k.slaves[slaveId]
		if !exists {
			k.slaves[slaveId] = newSlave(offer.GetHostname())
			slave = k.slaves[slaveId]
		}
		slave.Offers[offerId] = empty{}
		k.slaveIDs[slave.HostName] = slaveId
	}
}

// OfferRescinded is called when the resources are recinded from the scheduler.
func (k *KubernetesScheduler) OfferRescinded(driver bindings.SchedulerDriver, offerId *mesos.OfferID) {
	log.Infof("Offer rescinded %v\n", offerId)

	oid := offerId.GetValue()
	if offer, ok := k.offers.Get(oid); ok {
		k.offers.Delete(oid)
		if details := offer.Details(); details != nil {
			k.Lock()
			defer k.Unlock()

			slaveId := details.GetSlaveId().GetValue()

			if slave, found := k.slaves[slaveId]; !found {
				log.Infof("No slave for id %s associated with offer id %s", slaveId, oid)
			} else {
				delete(slave.Offers, oid)
			}
		} // else, offer already expired / lingering
	}
}

// StatusUpdate is called when a status update message is sent to the scheduler.
func (k *KubernetesScheduler) StatusUpdate(driver bindings.SchedulerDriver, taskStatus *mesos.TaskStatus) {
	//TODO(jdef) we're going to make changes to podtask.T objects in here and since the current TaskRegistry
	//implementation is in-memory we need a critical section for this.
	k.Lock()
	defer k.Unlock()

	log.Infof("Received status update %v\n", taskStatus)

	switch taskStatus.GetState() {
	case mesos.TaskState_TASK_STAGING:
		log.Errorf("Not implemented: task staging")
	case mesos.TaskState_TASK_RUNNING, mesos.TaskState_TASK_FINISHED, mesos.TaskState_TASK_STARTING:
		if !func() (exists bool) {
			slaveId := taskStatus.GetSlaveId().GetValue()
			_, exists = k.slaves[slaveId]
			return
		}() {
			log.Warningf("Ignore status %+v because the slave does not exist", taskStatus)
			return
		}
		fallthrough
	case mesos.TaskState_TASK_KILLED:
		k.taskRegistry.UpdateStatus(taskStatus)
	case mesos.TaskState_TASK_FAILED:
		if task, _ := k.taskRegistry.UpdateStatus(taskStatus); task != nil {
			if task.Has(podtask.Launched) && messages.CreateBindingFailure == taskStatus.GetMessage() {
				go k.plugin.reconcilePod(*task.Pod)
			}
		}
	case mesos.TaskState_TASK_LOST:
		task, state := k.taskRegistry.UpdateStatus(taskStatus)
		if state == podtask.StateRunning && taskStatus.ExecutorId != nil && taskStatus.SlaveId != nil {
			//TODO(jdef) this may not be meaningful once we have proper checkpointing and master detection
			//If we're reconciling and receive this then the executor may be
			//running a task that we need it to kill. It's possible that the framework
			//is unrecognized by the master at this point, so KillTask is not guaranteed
			//to do anything. The underlying driver transport may be able to send a
			//FrameworkMessage directly to the slave to terminate the task.
			log.V(2).Info("forwarding TASK_LOST message to executor %v on slave %v", taskStatus.ExecutorId, taskStatus.SlaveId)
			data := fmt.Sprintf("task-lost:%s", task.ID) //TODO(jdef) use a real message type
			if _, err := driver.SendFrameworkMessage(taskStatus.ExecutorId, taskStatus.SlaveId, data); err != nil {
				log.Error(err)
			}
		}
	}
}

// FrameworkMessage is called when the scheduler receives a message from the executor.
func (k *KubernetesScheduler) FrameworkMessage(driver bindings.SchedulerDriver,
	executorId *mesos.ExecutorID, slaveId *mesos.SlaveID, message string) {
	log.Infof("Received messages from executor %v of slave %v, %v\n", executorId, slaveId, message)
}

// SlaveLost is called when some slave is lost.
func (k *KubernetesScheduler) SlaveLost(driver bindings.SchedulerDriver, slaveId *mesos.SlaveID) {
	log.Infof("Slave %v is lost\n", slaveId)

	k.Lock()
	defer k.Unlock()

	// invalidate all offers mapped to that slave
	if slave, ok := k.slaves[slaveId.GetValue()]; ok {
		for offerId := range slave.Offers {
			k.offers.Invalidate(offerId)
		}
	}

	// TODO(jdef): delete slave from our internal list?

	// unfinished tasks/pods will be dropped. use a replication controller if you want pods to
	// be restarted when slaves die.
}

// ExecutorLost is called when some executor is lost.
func (k *KubernetesScheduler) ExecutorLost(driver bindings.SchedulerDriver,
	executorId *mesos.ExecutorID, slaveId *mesos.SlaveID, status int) {
	log.Infof("Executor %v of slave %v is lost, status: %v\n", executorId, slaveId, status)
	// TODO(yifan): Restart any unfinished tasks of the executor.
}

// Error is called when there is an unrecoverable error in the scheduler or scheduler driver.
// The driver should have been aborted before this is invoked.
func (k *KubernetesScheduler) Error(driver bindings.SchedulerDriver, message string) {
	log.Fatalf("fatal scheduler error: %v\n", message)
}

// intended to be invoked as a Reconciler.Action by Reconciler.Run
func (k *KubernetesScheduler) ReconcileRunningTasks(driver bindings.SchedulerDriver, canceled <-chan struct{}) error {
	log.Info("reconcile running tasks")
	remaining := util.NewStringSet()
	statusList := []*mesos.TaskStatus{}

	filter := podtask.StateRunning
	remaining.Insert(k.taskRegistry.List(&filter)...)
	if _, err := driver.ReconcileTasks(statusList); err != nil {
		return err
	}
	start := time.Now()
	const maxBackoff = 120 * time.Second
	first := true
	for backoff := 1 * time.Second; first || remaining.Len() > 0; backoff = backoff * 2 {
		first = false
		//TODO(jdef) reconcileTasks(remaining, canceled)
		if backoff > maxBackoff {
			backoff = maxBackoff
		}
		select {
		case <-canceled:
			return nil //TODO(jdef) should probably return a cancelation error
		case <-time.After(backoff):
			func() {
				k.RLock()
				defer k.RUnlock()
				for taskId := range remaining {
					if task, state := k.taskRegistry.Get(taskId); state == podtask.StateRunning && task.UpdatedTime.Before(start) {
						// keep this task in remaining list
						continue
					}
					remaining.Delete(taskId)
				}
			}()
		}
	}
	return nil
}

type Reconciler struct {
	Action  func(driver bindings.SchedulerDriver, canceled <-chan struct{}) error
	running int32 // 1 when Action is running, 0 otherwise
}

// execute task reconciliation, returns a cancelation channel or nil if reconciliation is already running.
// a client may signal that reconciliation should be canceled by closing the cancelation channel.
// no objects are ever read from, or written to, the cancelation channel.
func (r *Reconciler) Run(driver bindings.SchedulerDriver) <-chan struct{} {
	if atomic.CompareAndSwapInt32(&r.running, 0, 1) {
		canceled := make(chan struct{})
		go func() {
			defer atomic.StoreInt32(&r.running, 0)
			err := r.Action(driver, canceled)
			if err != nil {
				log.Error(err)
			}
		}()
		return canceled
	}
	return nil
}
<commit_msg>better logging for reconciliation errors<commit_after>package scheduler

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	log "github.com/golang/glog"
	mesos "github.com/mesos/mesos-go/mesosproto"
	mutil "github.com/mesos/mesos-go/mesosutil"
	bindings "github.com/mesos/mesos-go/scheduler"
	"github.com/mesosphere/kubernetes-mesos/pkg/executor/messages"
	"github.com/mesosphere/kubernetes-mesos/pkg/offers"
	"github.com/mesosphere/kubernetes-mesos/pkg/scheduler/meta"
	"github.com/mesosphere/kubernetes-mesos/pkg/scheduler/podtask"
)

const (
	defaultOfferTTL       = 5    // seconds that an offer is viable, prior to being expired
	defaultOfferLingerTTL = 120  // seconds that an expired offer lingers in history
	defaultListenerDelay  = 1    // number of seconds between offer listener notifications
	defaultUpdatesBacklog = 2048 // size of the pod updates channel
)

type Slave struct {
	HostName string
	Offers   map[string]empty
}

func newSlave(hostName string) *Slave {
	return &Slave{
		HostName: hostName,
		Offers:   make(map[string]empty),
	}
}

type PluginInterface interface {
	// the apiserver may have a different state for the pod than we do
	// so reconcile our records, but only for this one pod
	reconcilePod(api.Pod)

	// execute the Scheduling plugin, should start a go routine and return immediately
	Run()
}

// KubernetesScheduler implements:
// 1: A mesos scheduler.
// 2: A kubernetes scheduler plugin.
// 3: A kubernetes pod.Registry.
type KubernetesScheduler struct {
	// We use a lock here to avoid races
	// between invoking the mesos callback
	// and the invoking the pod registry interfaces.
	// In particular, changes to podtask.T objects are currently guarded by this lock.
	*sync.RWMutex

	// Mesos context.
	executor    *mesos.ExecutorInfo
	driver      bindings.SchedulerDriver
	frameworkId *mesos.FrameworkID
	masterInfo  *mesos.MasterInfo
	registered  bool

	offers       offers.Registry
	slaves       map[string]*Slave // SlaveID => slave.
	slaveIDs     map[string]string // Slave's hostname => slaveID
	taskRegistry podtask.Registry

	scheduleFunc PodScheduleFunc // The function that does scheduling.

	client     *client.Client
	plugin     PluginInterface
	etcdClient tools.EtcdClient
}

// New create a new KubernetesScheduler
func New(executor *mesos.ExecutorInfo, scheduleFunc PodScheduleFunc, client *client.Client, etcdClient tools.EtcdClient) *KubernetesScheduler {
	var k *KubernetesScheduler
	k = &KubernetesScheduler{
		RWMutex:  new(sync.RWMutex),
		executor: executor,
		offers: offers.CreateRegistry(offers.RegistryConfig{
			DeclineOffer: func(id string) error {
				offerId := mutil.NewOfferID(id)
				filters := &mesos.Filters{}
				_, err := k.driver.DeclineOffer(offerId, filters)
				return err
			},
			TTL:           defaultOfferTTL * time.Second,
			LingerTTL:     defaultOfferLingerTTL * time.Second, // remember expired offers so that we can tell if a previously scheduler offer relies on one
			ListenerDelay: defaultListenerDelay * time.Second,
		}),
		slaves:       make(map[string]*Slave),
		slaveIDs:     make(map[string]string),
		taskRegistry: podtask.NewInMemoryRegistry(),
		scheduleFunc: scheduleFunc,
		client:       client,
		etcdClient:   etcdClient,
	}
	return k
}

func (k *KubernetesScheduler) Init(d bindings.SchedulerDriver, pl PluginInterface) {
	k.driver = d
	k.plugin = pl
	k.offers.Init()
}

// Registered is called when the scheduler registered with the master successfully.
func (k *KubernetesScheduler) Registered(driver bindings.SchedulerDriver,
	frameworkId *mesos.FrameworkID, masterInfo *mesos.MasterInfo) {
	k.frameworkId = frameworkId
	k.masterInfo = masterInfo
	k.registered = true
	go k.storeFrameworkId() //TODO(jdef) only do this if we're checkpointing?
	log.Infof("Scheduler registered with the master: %v with frameworkId: %v\n", masterInfo, frameworkId)

	//TODO(jdef) partial reconciliation started... needs work
	r := &Reconciler{Action: k.ReconcileRunningTasks}
	go util.Forever(func() { r.Run(driver) }, 5*time.Minute) // TODO(jdef) parameterize reconciliation interval
}

func (k *KubernetesScheduler) storeFrameworkId() {
	_, err := k.etcdClient.Set(meta.FrameworkIDKey, k.frameworkId.GetValue(), 0)
	if err != nil {
		log.Error(err)
	}
}

// Reregistered is called when the scheduler re-registered with the master successfully.
// This happends when the master fails over.
func (k *KubernetesScheduler) Reregistered(driver bindings.SchedulerDriver, masterInfo *mesos.MasterInfo) {
	log.Infof("Scheduler reregistered with the master: %v\n", masterInfo)
	k.registered = true

	//TODO(jdef) partial reconciliation started... needs work
	r := &Reconciler{Action: k.ReconcileRunningTasks}
	go util.Forever(func() { r.Run(driver) }, 5*time.Minute) // TODO(jdef) parameterize reconciliation interval
}

// Disconnected is called when the scheduler loses connection to the master.
func (k *KubernetesScheduler) Disconnected(driver bindings.SchedulerDriver) {
	log.Infof("Master disconnected!\n")
	k.registered = false

	k.Lock()
	defer k.Unlock()

	// discard all cached offers to avoid unnecessary TASK_LOST updates
	k.offers.Invalidate("")
}

// ResourceOffers is called when the scheduler receives some offers from the master.
func (k *KubernetesScheduler) ResourceOffers(driver bindings.SchedulerDriver, offers []*mesos.Offer) {
	log.V(2).Infof("Received offers %+v", offers)

	k.Lock()
	defer k.Unlock()

	// Record the offers in the global offer map as well as each slave's offer map.
	k.offers.Add(offers)
	for _, offer := range offers {
		offerId := offer.GetId().GetValue()
		slaveId := offer.GetSlaveId().GetValue()

		slave, exists := k.slaves[slaveId]
		if !exists {
			k.slaves[slaveId] = newSlave(offer.GetHostname())
			slave = k.slaves[slaveId]
		}
		slave.Offers[offerId] = empty{}
		k.slaveIDs[slave.HostName] = slaveId
	}
}

// OfferRescinded is called when the resources are recinded from the scheduler.
func (k *KubernetesScheduler) OfferRescinded(driver bindings.SchedulerDriver, offerId *mesos.OfferID) {
	log.Infof("Offer rescinded %v\n", offerId)

	oid := offerId.GetValue()
	if offer, ok := k.offers.Get(oid); ok {
		k.offers.Delete(oid)
		if details := offer.Details(); details != nil {
			k.Lock()
			defer k.Unlock()

			slaveId := details.GetSlaveId().GetValue()

			if slave, found := k.slaves[slaveId]; !found {
				log.Infof("No slave for id %s associated with offer id %s", slaveId, oid)
			} else {
				delete(slave.Offers, oid)
			}
		} // else, offer already expired / lingering
	}
}

// StatusUpdate is called when a status update message is sent to the scheduler.
func (k *KubernetesScheduler) StatusUpdate(driver bindings.SchedulerDriver, taskStatus *mesos.TaskStatus) {
	//TODO(jdef) we're going to make changes to podtask.T objects in here and since the current TaskRegistry
	//implementation is in-memory we need a critical section for this.
	k.Lock()
	defer k.Unlock()

	log.Infof("Received status update %v\n", taskStatus)

	switch taskStatus.GetState() {
	case mesos.TaskState_TASK_STAGING:
		log.Errorf("Not implemented: task staging")
	case mesos.TaskState_TASK_RUNNING, mesos.TaskState_TASK_FINISHED, mesos.TaskState_TASK_STARTING:
		if !func() (exists bool) {
			slaveId := taskStatus.GetSlaveId().GetValue()
			_, exists = k.slaves[slaveId]
			return
		}() {
			log.Warningf("Ignore status %+v because the slave does not exist", taskStatus)
			return
		}
		fallthrough
	case mesos.TaskState_TASK_KILLED:
		k.taskRegistry.UpdateStatus(taskStatus)
	case mesos.TaskState_TASK_FAILED:
		if task, _ := k.taskRegistry.UpdateStatus(taskStatus); task != nil {
			if task.Has(podtask.Launched) && messages.CreateBindingFailure == taskStatus.GetMessage() {
				go k.plugin.reconcilePod(*task.Pod)
			}
		}
	case mesos.TaskState_TASK_LOST:
		task, state := k.taskRegistry.UpdateStatus(taskStatus)
		if state == podtask.StateRunning && taskStatus.ExecutorId != nil && taskStatus.SlaveId != nil {
			//TODO(jdef) this may not be meaningful once we have proper checkpointing and master detection
			//If we're reconciling and receive this then the executor may be
			//running a task that we need it to kill. It's possible that the framework
			//is unrecognized by the master at this point, so KillTask is not guaranteed
			//to do anything. The underlying driver transport may be able to send a
			//FrameworkMessage directly to the slave to terminate the task.
			log.V(2).Info("forwarding TASK_LOST message to executor %v on slave %v", taskStatus.ExecutorId, taskStatus.SlaveId)
			data := fmt.Sprintf("task-lost:%s", task.ID) //TODO(jdef) use a real message type
			if _, err := driver.SendFrameworkMessage(taskStatus.ExecutorId, taskStatus.SlaveId, data); err != nil {
				log.Error(err)
			}
		}
	}
}

// FrameworkMessage is called when the scheduler receives a message from the executor.
func (k *KubernetesScheduler) FrameworkMessage(driver bindings.SchedulerDriver,
	executorId *mesos.ExecutorID, slaveId *mesos.SlaveID, message string) {
	log.Infof("Received messages from executor %v of slave %v, %v\n", executorId, slaveId, message)
}

// SlaveLost is called when some slave is lost.
func (k *KubernetesScheduler) SlaveLost(driver bindings.SchedulerDriver, slaveId *mesos.SlaveID) {
	log.Infof("Slave %v is lost\n", slaveId)

	k.Lock()
	defer k.Unlock()

	// invalidate all offers mapped to that slave
	if slave, ok := k.slaves[slaveId.GetValue()]; ok {
		for offerId := range slave.Offers {
			k.offers.Invalidate(offerId)
		}
	}

	// TODO(jdef): delete slave from our internal list?

	// unfinished tasks/pods will be dropped. use a replication controller if you want pods to
	// be restarted when slaves die.
}

// ExecutorLost is called when some executor is lost.
func (k *KubernetesScheduler) ExecutorLost(driver bindings.SchedulerDriver,
	executorId *mesos.ExecutorID, slaveId *mesos.SlaveID, status int) {
	log.Infof("Executor %v of slave %v is lost, status: %v\n", executorId, slaveId, status)
	// TODO(yifan): Restart any unfinished tasks of the executor.
}

// Error is called when there is an unrecoverable error in the scheduler or scheduler driver.
// The driver should have been aborted before this is invoked.
func (k *KubernetesScheduler) Error(driver bindings.SchedulerDriver, message string) {
	log.Fatalf("fatal scheduler error: %v\n", message)
}

// intended to be invoked as a Reconciler.Action by Reconciler.Run
func (k *KubernetesScheduler) ReconcileRunningTasks(driver bindings.SchedulerDriver, canceled <-chan struct{}) error {
	log.Info("reconcile running tasks")
	remaining := util.NewStringSet()
	statusList := []*mesos.TaskStatus{}

	filter := podtask.StateRunning
	remaining.Insert(k.taskRegistry.List(&filter)...)
	if _, err := driver.ReconcileTasks(statusList); err != nil {
		return err
	}
	start := time.Now()
	const maxBackoff = 120 * time.Second
	first := true
	for backoff := 1 * time.Second; first || remaining.Len() > 0; backoff = backoff * 2 {
		first = false
		//TODO(jdef) reconcileTasks(remaining, canceled)
		if backoff > maxBackoff {
			backoff = maxBackoff
		}
		select {
		case <-canceled:
			return nil //TODO(jdef) should probably return a cancelation error
		case <-time.After(backoff):
			func() {
				k.RLock()
				defer k.RUnlock()
				for taskId := range remaining {
					if task, state := k.taskRegistry.Get(taskId); state == podtask.StateRunning && task.UpdatedTime.Before(start) {
						// keep this task in remaining list
						continue
					}
					remaining.Delete(taskId)
				}
			}()
		}
	}
	return nil
}

type Reconciler struct {
	Action  func(driver bindings.SchedulerDriver, canceled <-chan struct{}) error
	running int32 // 1 when Action is running, 0 otherwise
}

// execute task reconciliation, returns a cancelation channel or nil if reconciliation is already running.
// a client may signal that reconciliation should be canceled by closing the cancelation channel.
// no objects are ever read from, or written to, the cancelation channel.
func (r *Reconciler) Run(driver bindings.SchedulerDriver) <-chan struct{} {
	if atomic.CompareAndSwapInt32(&r.running, 0, 1) {
		canceled := make(chan struct{})
		go func() {
			defer atomic.StoreInt32(&r.running, 0)
			err := r.Action(driver, canceled)
			if err != nil {
				log.Errorf("reconciler action failed: %v", err)
			}
		}()
		return canceled
	}
	return nil
}
<|endoftext|>


<commit_before>/*
 * Copyright 2019, EnMasse authors.
 * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
 */

package images

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"gopkg.in/yaml.v2"
	corev1 "k8s.io/api/core/v1"
)

var (
	imageMap          map[string]string
	defaultPullPolicy corev1.PullPolicy
)

func loadImageMapFrom(fileName string) (map[string]string, error) {

	var data = make(map[string]string)

	file, err := os.Open(fileName)
	if err != nil {
		return nil, err
	}

	defer func() { _ = file.Close() }()

	byteValue, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}

	if err := yaml.Unmarshal(byteValue, data); err != nil {
		return nil, err
	}

	return data, nil

}

func setIfEnvPresent(data map[string]string, key string, envVar string) {
	value, present := os.LookupEnv(envVar)
	if present {
		data[key] = value
	}
}

func overrideImageMapFromEnv(data map[string]string) {
	setIfEnvPresent(data, "iot-auth-service", "IOT_AUTH_SERVICE_IMAGE")
	setIfEnvPresent(data, "iot-device-registry-file", "IOT_DEVICE_REGISTRY_FILE_IMAGE")
	setIfEnvPresent(data, "iot-gc", "IOT_GC_IMAGE")
	setIfEnvPresent(data, "iot-http-adapter", "IOT_HTTP_ADAPTER_IMAGE")
	setIfEnvPresent(data, "iot-mqtt-adapter", "IOT_MQTT_ADAPTER_IMAGE")
	setIfEnvPresent(data, "iot-tenant-service", "IOT_TENANT_SERVICE_IMAGE_IMAGE")
	setIfEnvPresent(data, "iot-proxy-configurator", "IOT_PROXY_CONFIGURATOR_IMAGE")
	setIfEnvPresent(data, "qdrouterd-base", "QDROUTERD_BASE_IMAGE")
	setIfEnvPresent(data, "none-authservice", "NONE_AUTHSERVICE_IMAGE")
	setIfEnvPresent(data, "keycloak", "KEYCLOAK_IMAGE")
	setIfEnvPresent(data, "keycloak-plugin", "KEYCLOAK_PLUGIN_IMAGE")
}

const defaultImageMapFileName = "operatorImageMap.yaml"

func init() {
	defaultPullPolicy = corev1.PullPolicy(os.Getenv("IMAGE_PULL_POLICY"))
}

func lazyLoadImageMap() {
	if imageMap != nil {
		return
	}

	fileName := os.Getenv("ENMASSE_IMAGE_MAP_FILE")
	if fileName == "" {
		fileName = defaultImageMapFileName
	}

	var err error
	imageMap, err = loadImageMapFrom(fileName)
	if err != nil {
		panic(err)
	}

	overrideImageMapFromEnv(imageMap)
}

func GetImage(name string) (string, error) {
	lazyLoadImageMap()

	if imageMap == nil {
		return "", fmt.Errorf("'%s' not found or 'ENMASSE_IMAGE_MAP_FILE' not set to a readable file, unable to lookup image names", defaultImageMapFileName)
	}

	value, ok := imageMap[name]
	if !ok {
		return "", fmt.Errorf("image '%s' not found in the mapping table", name)
	}

	return value, nil
}

func PullPolicyFromImageName(imageName string) corev1.PullPolicy {

	if defaultPullPolicy != "" {
		return defaultPullPolicy
	}

	if strings.HasSuffix(imageName, "-SNAPSHOT") || strings.HasSuffix(imageName, ":latest") {
		return corev1.PullAlways
	} else {
		return corev1.PullIfNotPresent
	}

}
<commit_msg>Replace console images from env<commit_after>/*
 * Copyright 2019, EnMasse authors.
 * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
 */

package images

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"gopkg.in/yaml.v2"
	corev1 "k8s.io/api/core/v1"
)

var (
	imageMap          map[string]string
	defaultPullPolicy corev1.PullPolicy
)

func loadImageMapFrom(fileName string) (map[string]string, error) {

	var data = make(map[string]string)

	file, err := os.Open(fileName)
	if err != nil {
		return nil, err
	}

	defer func() { _ = file.Close() }()

	byteValue, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}

	if err := yaml.Unmarshal(byteValue, data); err != nil {
		return nil, err
	}

	return data, nil

}

func setIfEnvPresent(data map[string]string, key string, envVar string) {
	value, present := os.LookupEnv(envVar)
	if present {
		data[key] = value
	}
}

func overrideImageMapFromEnv(data map[string]string) {
	setIfEnvPresent(data, "iot-auth-service", "IOT_AUTH_SERVICE_IMAGE")
	setIfEnvPresent(data, "iot-device-registry-file", "IOT_DEVICE_REGISTRY_FILE_IMAGE")
	setIfEnvPresent(data, "iot-gc", "IOT_GC_IMAGE")
	setIfEnvPresent(data, "iot-http-adapter", "IOT_HTTP_ADAPTER_IMAGE")
	setIfEnvPresent(data, "iot-mqtt-adapter", "IOT_MQTT_ADAPTER_IMAGE")
	setIfEnvPresent(data, "iot-tenant-service", "IOT_TENANT_SERVICE_IMAGE_IMAGE")
	setIfEnvPresent(data, "iot-proxy-configurator", "IOT_PROXY_CONFIGURATOR_IMAGE")
	setIfEnvPresent(data, "qdrouterd-base", "QDROUTERD_BASE_IMAGE")
	setIfEnvPresent(data, "none-authservice", "NONE_AUTHSERVICE_IMAGE")
	setIfEnvPresent(data, "keycloak", "KEYCLOAK_IMAGE")
	setIfEnvPresent(data, "keycloak-plugin", "KEYCLOAK_PLUGIN_IMAGE")
	setIfEnvPresent(data, "console-init", "CONSOLE_INIT_IMAGE")
	setIfEnvPresent(data, "console-proxy-openshift", "CONSOLE_PROXY_OPENSHIFT_IMAGE")
	setIfEnvPresent(data, "console-proxy-kubernetes", "CONSOLE_PROXY_KUBERNETES_IMAGE")
	setIfEnvPresent(data, "console-httpd", "CONSOLE_HTTPD_IMAGE")
}

const defaultImageMapFileName = "operatorImageMap.yaml"

func init() {
	defaultPullPolicy = corev1.PullPolicy(os.Getenv("IMAGE_PULL_POLICY"))
}

func lazyLoadImageMap() {
	if imageMap != nil {
		return
	}

	fileName := os.Getenv("ENMASSE_IMAGE_MAP_FILE")
	if fileName == "" {
		fileName = defaultImageMapFileName
	}

	var err error
	imageMap, err = loadImageMapFrom(fileName)
	if err != nil {
		panic(err)
	}

	overrideImageMapFromEnv(imageMap)
}

func GetImage(name string) (string, error) {
	lazyLoadImageMap()

	if imageMap == nil {
		return "", fmt.Errorf("'%s' not found or 'ENMASSE_IMAGE_MAP_FILE' not set to a readable file, unable to lookup image names", defaultImageMapFileName)
	}

	value, ok := imageMap[name]
	if !ok {
		return "", fmt.Errorf("image '%s' not found in the mapping table", name)
	}

	return value, nil
}

func PullPolicyFromImageName(imageName string) corev1.PullPolicy {

	if defaultPullPolicy != "" {
		return defaultPullPolicy
	}

	if strings.HasSuffix(imageName, "-SNAPSHOT") || strings.HasSuffix(imageName, ":latest") {
		return corev1.PullAlways
	} else {
		return corev1.PullIfNotPresent
	}

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	humanize "github.com/dustin/go-humanize"
	drive "google.golang.org/api/drive/v2"
	"google.golang.org/api/googleapi"

	"appengine"
	"appengine/delay"
	"appengine/mail"
	"appengine/urlfetch"
)

const (
	clientID        = "1045967131934-0gdt52c0bp0e1g9dquib9atfvqjmjkjl.apps.googleusercontent.com"
	clientSecret    = "Dna3ItrA-0yhbpvjj_8516oG"
	redirectURLPath = "/oauth"
	fields          = googleapi.Field("items(id,title,fileSize,md5Checksum),nextPageToken")
)

var scopes = strings.Join([]string{
	drive.DriveReadonlyScope,
	"email",
}, " ")

var emailTmpl = template.Must(template.New("email").Parse(`

You have {{.ReapableBytesStr}} of reapable files

The Drive Deduplifier has scanned {{.TotalFiles}} files and found {{.ReapableFiles}} that are duplicates.

`))

func init() {
	http.HandleFunc("/start", startHandler)
	http.HandleFunc(redirectURLPath, oauthHandler)
}

func startHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	redirectURL := fmt.Sprintf("https://%s.appspot.com", appengine.AppID(ctx)) + redirectURLPath
	url := "https://accounts.google.com/o/oauth2/auth?" + url.Values{
		"response_type":   {"code"},
		"approval_prompt": {"force"},
		"client_id":       {clientID},
		"redirect_uri":    {redirectURL},
		"scope":           {scopes},
	}.Encode()
	http.Redirect(w, r, url, http.StatusSeeOther)
}

func oauthHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	code := r.FormValue("code")
	tok, err := getAccessToken(ctx, code)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	generateReport.Call(ctx, tok)
	http.Redirect(w, r, "/started", http.StatusSeeOther)
}

func getAccessToken(ctx appengine.Context, code string) (string, error) {
	client := urlfetch.Client(ctx)

	req, err := http.NewRequest("POST", "https://accounts.google.com/o/oauth2/token", strings.NewReader(url.Values{
		"client_id":     {clientID},
		"client_secret": {clientSecret},
		"code":          {code},
		"scope":         {scopes},
		"redirect_uri":  {fmt.Sprintf("https://%s.appspot.com", appengine.AppID(ctx)) + redirectURLPath},
		"grant_type":    {"authorization_code"},
	}.Encode()))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	resp, err := client.Do(req)
	if err != nil {
		ctx.Errorf("exchanging code: %v", err)
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		all, _ := ioutil.ReadAll(resp.Body)
		ctx.Errorf(string(all))
		return "", errors.New("couldn't get token")
	}

	var b struct {
		AccessToken string `json:"access_token"`
	}
	if err := json.NewDecoder(resp.Body).Decode(&b); err != nil {
		ctx.Errorf("decoding json: %v", err)
		return "", err
	}
	return b.AccessToken, nil
}

func getEmail(ctx appengine.Context, tok string) (string, error) {
	client := &http.Client{Transport: authTransport{tok, urlfetch.Client(ctx)}}
	req, _ := http.NewRequest("GET", "https://www.googleapis.com/oauth2/v2/userinfo", nil)
	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}
	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("http error %d", resp.StatusCode)
	}
	var info struct {
		Email string `json:"email"`
	}
	defer resp.Body.Close()
	if err := json.NewDecoder(resp.Body).Decode(&info); err != nil {
		return "", err
	}
	return info.Email, nil
}

var generateReport = delay.Func("generate", func(ctx appengine.Context, tok string) error {
	email, err := getEmail(ctx, tok)
	if err != nil {
		return err
	}

	client := &http.Client{Transport: authTransport{tok, urlfetch.Client(ctx)}}
	svc, err := drive.New(client)
	if err != nil {
		return err
	}
	pageToken := ""

	type file struct {
		md5, title string
		size       int64
	}

	var report report
	md5s := map[file][]string{}
	for {
		fs, err := svc.Files.List().
			MaxResults(1000).
			PageToken(pageToken).
			Fields(fields).
			Do()
		if err != nil {
			return err
		}
		report.TotalFiles += len(fs.Items)
		for _, f := range fs.Items {
			if f.Md5Checksum != "" {
				k := file{f.Md5Checksum, f.Title, f.FileSize}
				md5s[k] = append(md5s[k], f.Id)
			}
		}
		pageToken = fs.NextPageToken
		if pageToken == "" {
			break
		}
	}

	for k, v := range md5s {
		if len(v) > 1 {
			//ctx.Infof("===", k.title, "(md5:", k.md5, ") ===")
			//ctx.Infof("- reapable IDs:", strings.Join(v[1:], ", "))
			report.ReapableFiles += len(v) - 1
			report.ReapableBytes += k.size * int64(len(v)-1)
		}
	}

	ctx.Infof("sending report to %q", email)

	var body bytes.Buffer
	if err := emailTmpl.Execute(&body, map[string]interface{}{
		"TotalFiles":       report.ReapableFiles,
		"ReapableFiles":    report.ReapableFiles,
		"ReapableBytesStr": humanize.Bytes(uint64(report.ReapableBytes)),
	}); err != nil {
		return err
	}
	return mail.Send(ctx, &mail.Message{
		Sender:   "imjasonh@gmail.com",
		To:       []string{email},
		Subject:  "Drive Deduplifier Report",
		HTMLBody: body.String(),
	})
})

type report struct {
	TotalFiles, ReapableFiles int
	ReapableBytes             int64
}

type authTransport struct {
	tok    string
	client *http.Client
}

func (t authTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	r.Header.Set("Authorization", "Bearer "+t.tok)
	return t.client.Do(r)
}
<commit_msg>Let users know how much of their quota they're using, for context<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	humanize "github.com/dustin/go-humanize"
	drive "google.golang.org/api/drive/v2"
	"google.golang.org/api/googleapi"

	"appengine"
	"appengine/delay"
	"appengine/mail"
	"appengine/urlfetch"
)

const (
	clientID        = "1045967131934-0gdt52c0bp0e1g9dquib9atfvqjmjkjl.apps.googleusercontent.com"
	clientSecret    = "Dna3ItrA-0yhbpvjj_8516oG"
	redirectURLPath = "/oauth"
	fields          = googleapi.Field("items(id,title,fileSize,md5Checksum),nextPageToken")
)

var scopes = strings.Join([]string{
	drive.DriveMetadataReadonlyScope,
	"email",
}, " ")

var emailTmpl = template.Must(template.New("email").Parse(`

You have {{.ReapableBytes}} of duplicate files

The Drive Deduplifier has scanned {{.TotalFiles}} files and found {{.ReapableFiles}} that are duplicates.

You are using {{.UsedQuota}} of your total {{.TotalQuota}} quota.

`))

func init() {
	http.HandleFunc("/start", startHandler)
	http.HandleFunc(redirectURLPath, oauthHandler)
}

func startHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	redirectURL := fmt.Sprintf("https://%s.appspot.com", appengine.AppID(ctx)) + redirectURLPath
	url := "https://accounts.google.com/o/oauth2/auth?" + url.Values{
		"response_type":   {"code"},
		"approval_prompt": {"force"},
		"client_id":       {clientID},
		"redirect_uri":    {redirectURL},
		"scope":           {scopes},
	}.Encode()
	http.Redirect(w, r, url, http.StatusSeeOther)
}

func oauthHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	code := r.FormValue("code")
	tok, err := getAccessToken(ctx, code)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	generateReport.Call(ctx, tok)
	http.Redirect(w, r, "/started", http.StatusSeeOther)
}

func getAccessToken(ctx appengine.Context, code string) (string, error) {
	client := urlfetch.Client(ctx)

	req, err := http.NewRequest("POST", "https://accounts.google.com/o/oauth2/token", strings.NewReader(url.Values{
		"client_id":     {clientID},
		"client_secret": {clientSecret},
		"code":          {code},
		"scope":         {scopes},
		"redirect_uri":  {fmt.Sprintf("https://%s.appspot.com", appengine.AppID(ctx)) + redirectURLPath},
		"grant_type":    {"authorization_code"},
	}.Encode()))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	resp, err := client.Do(req)
	if err != nil {
		ctx.Errorf("exchanging code: %v", err)
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		all, _ := ioutil.ReadAll(resp.Body)
		ctx.Errorf(string(all))
		return "", errors.New("couldn't get token")
	}

	var b struct {
		AccessToken string `json:"access_token"`
	}
	if err := json.NewDecoder(resp.Body).Decode(&b); err != nil {
		ctx.Errorf("decoding json: %v", err)
		return "", err
	}
	return b.AccessToken, nil
}

func getEmail(ctx appengine.Context, tok string) (string, error) {
	client := &http.Client{Transport: authTransport{tok, urlfetch.Client(ctx)}}
	req, _ := http.NewRequest("GET", "https://www.googleapis.com/oauth2/v2/userinfo", nil)
	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}
	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("http error %d", resp.StatusCode)
	}
	var info struct {
		Email string `json:"email"`
	}
	defer resp.Body.Close()
	if err := json.NewDecoder(resp.Body).Decode(&info); err != nil {
		return "", err
	}
	return info.Email, nil
}

var generateReport = delay.Func("generate", func(ctx appengine.Context, tok string) error {
	email, err := getEmail(ctx, tok)
	if err != nil {
		return err
	}

	client := &http.Client{Transport: authTransport{tok, urlfetch.Client(ctx)}}
	svc, err := drive.New(client)
	if err != nil {
		return err
	}
	pageToken := ""

	type file struct {
		md5, title string
		size       int64
	}

	var report report
	md5s := map[file][]string{}
	for {
		fs, err := svc.Files.List().
			MaxResults(1000).
			PageToken(pageToken).
			Fields(fields).
			Do()
		if err != nil {
			return err
		}
		report.TotalFiles += len(fs.Items)
		for _, f := range fs.Items {
			if f.Md5Checksum != "" {
				k := file{f.Md5Checksum, f.Title, f.FileSize}
				md5s[k] = append(md5s[k], f.Id)
			}
		}
		pageToken = fs.NextPageToken
		if pageToken == "" {
			break
		}
	}

	for k, v := range md5s {
		if len(v) > 1 {
			//ctx.Infof("===", k.title, "(md5:", k.md5, ") ===")
			//ctx.Infof("- reapable IDs:", strings.Join(v[1:], ", "))
			report.ReapableFiles += len(v) - 1
			report.ReapableBytes += k.size * int64(len(v)-1)
		}
	}

	about, err := svc.About.Get().
		Fields("quotaBytesUsed,quotaBytesTotal").
		Do()
	if err != nil {
		return err
	}
	report.TotalQuota = about.QuotaBytesTotal
	report.UsedQuota = about.QuotaBytesUsed

	ctx.Infof("sending report to %q", email)

	var body bytes.Buffer
	if err := emailTmpl.Execute(&body, map[string]interface{}{
		"TotalFiles":    report.TotalFiles,
		"ReapableFiles": report.ReapableFiles,
		"ReapableBytes": humanize.Bytes(uint64(report.ReapableBytes)),
		"TotalQuota":    humanize.Bytes(uint64(report.TotalQuota)),
		"UsedQuota":     humanize.Bytes(uint64(report.UsedQuota)),
	}); err != nil {
		return err
	}
	return mail.Send(ctx, &mail.Message{
		Sender:   "imjasonh@gmail.com",
		To:       []string{email},
		Subject:  "Drive Deduplifier Report",
		HTMLBody: body.String(),
	})
})

type report struct {
	TotalFiles, ReapableFiles int
	ReapableBytes             int64
	TotalQuota, UsedQuota     int64
}

type authTransport struct {
	tok    string
	client *http.Client
}

func (t authTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	r.Header.Set("Authorization", "Bearer "+t.tok)
	return t.client.Do(r)
}
<|endoftext|>


<commit_before>/*
Copyright 2011 The Perkeep Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package webserver implements a superset wrapper of http.Server.
//
// Among other things, it can throttle its connections, inherit its
// listening socket from a file descriptor in the environment, and
// log all activity.
package webserver // import "perkeep.org/pkg/webserver"

import (
	"crypto/rand"
	"crypto/tls"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"perkeep.org/pkg/webserver/listen"

	"go4.org/net/throttle"
	"go4.org/wkfs"
	"golang.org/x/net/http2"
)

const alpnProto = "acme-tls/1" // from golang.org/x/crypto/acme.ALPNProto

type Server struct {
	mux      *http.ServeMux
	listener net.Listener
	verbose  bool // log HTTP requests and response codes

	Logger *log.Logger // or nil.

	// H2Server is the HTTP/2 server config.
	H2Server http2.Server

	// enableTLS sets the Server up for listening to HTTPS connections.
	enableTLS bool
	// tlsCertFile (tlsKeyFile) is the path to the HTTPS certificate (key) file.
	tlsCertFile, tlsKeyFile string
	// certManager is set as GetCertificate in the tls.Config of the listener. But tlsCertFile takes precedence.
	certManager func(*tls.ClientHelloInfo) (*tls.Certificate, error)

	mu   sync.Mutex
	reqs int64
}

func New() *Server {
	verbose, _ := strconv.ParseBool(os.Getenv("CAMLI_HTTP_DEBUG"))
	return &Server{
		mux:     http.NewServeMux(),
		verbose: verbose,
	}
}

func (s *Server) printf(format string, v ...interface{}) {
	if s.Logger != nil {
		s.Logger.Printf(format, v...)
		return
	}
	log.Printf(format, v...)
}

func (s *Server) fatalf(format string, v ...interface{}) {
	if s.Logger != nil {
		s.Logger.Fatalf(format, v...)
		return
	}
	log.Fatalf(format, v...)
}

// TLSSetup specifies how the server gets its TLS certificate.
type TLSSetup struct {
	// Certfile is the path to the TLS certificate file. It takes precedence over CertManager.
	CertFile string
	// KeyFile is the path to the TLS key file.
	KeyFile string
	// CertManager is the tls.GetCertificate of the tls Config. But CertFile takes precedence.
	CertManager func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error)
}

func (s *Server) SetTLS(setup TLSSetup) {
	s.enableTLS = true
	s.certManager = setup.CertManager
	s.tlsCertFile = setup.CertFile
	s.tlsKeyFile = setup.KeyFile
}

func (s *Server) ListenURL() string {
	if s.listener == nil {
		return ""
	}
	taddr, ok := s.listener.Addr().(*net.TCPAddr)
	if !ok {
		return ""
	}
	scheme := "http"
	if s.enableTLS {
		scheme = "https"
	}
	if taddr.IP.IsUnspecified() {
		return fmt.Sprintf("%s://localhost:%d", scheme, taddr.Port)
	}
	return fmt.Sprintf("%s://%s", scheme, s.listener.Addr())
}

func (s *Server) HandleFunc(pattern string, fn func(http.ResponseWriter, *http.Request)) {
	s.mux.HandleFunc(pattern, fn)
}

func (s *Server) Handle(pattern string, handler http.Handler) {
	s.mux.Handle(pattern, handler)
}

func (s *Server) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	var n int64
	if s.verbose {
		s.mu.Lock()
		s.reqs++
		n = s.reqs
		s.mu.Unlock()
		s.printf("Request #%d: %s %s (from %s) ...", n, req.Method, req.RequestURI, req.RemoteAddr)
		rw = &trackResponseWriter{ResponseWriter: rw}
	}
	s.mux.ServeHTTP(rw, req)
	if s.verbose {
		tw := rw.(*trackResponseWriter)
		s.printf("Request #%d: %s %s = code %d, %d bytes", n, req.Method, req.RequestURI, tw.code, tw.resSize)
	}
}

type trackResponseWriter struct {
	http.ResponseWriter
	code    int
	resSize int64
}

func (tw *trackResponseWriter) WriteHeader(code int) {
	tw.code = code
	tw.ResponseWriter.WriteHeader(code)
}

func (tw *trackResponseWriter) Write(p []byte) (int, error) {
	if tw.code == 0 {
		tw.code = 200
	}
	tw.resSize += int64(len(p))
	return tw.ResponseWriter.Write(p)
}

// Listen starts listening on the given host:port addr.
func (s *Server) Listen(addr string) error {
	if s.listener != nil {
		return nil
	}

	if addr == "" {
		return fmt.Errorf("<host>:<port> needs to be provided to start listening")
	}

	var err error
	s.listener, err = listen.Listen(addr)
	if err != nil {
		return fmt.Errorf("Failed to listen on %s: %v", addr, err)
	}
	base := s.ListenURL()
	s.printf("Starting to listen on %s\n", base)

	doEnableTLS := func() error {
		config := &tls.Config{
			Rand:       rand.Reader,
			Time:       time.Now,
			NextProtos: []string{http2.NextProtoTLS, "http/1.1"},
			MinVersion: tls.VersionTLS12,
		}
		if s.tlsCertFile == "" && s.certManager != nil {
			config.GetCertificate = s.certManager
			// TODO(mpl): see if we can instead use
			// https://godoc.org/golang.org/x/crypto/acme/autocert#Manager.TLSConfig
			config.NextProtos = append(config.NextProtos, alpnProto)
			s.listener = tls.NewListener(s.listener, config)
			return nil
		}

		config.Certificates = make([]tls.Certificate, 1)
		config.Certificates[0], err = loadX509KeyPair(s.tlsCertFile, s.tlsKeyFile)
		if err != nil {
			return fmt.Errorf("Failed to load TLS cert: %v", err)
		}
		s.listener = tls.NewListener(s.listener, config)
		return nil
	}
	if s.enableTLS {
		if err := doEnableTLS(); err != nil {
			return err
		}
	}

	if strings.HasSuffix(base, ":0") {
		s.printf("Now listening on %s\n", s.ListenURL())
	}

	return nil
}

func (s *Server) throttleListener() net.Listener {
	kBps, _ := strconv.Atoi(os.Getenv("DEV_THROTTLE_KBPS"))
	ms, _ := strconv.Atoi(os.Getenv("DEV_THROTTLE_LATENCY_MS"))
	if kBps == 0 && ms == 0 {
		return s.listener
	}
	rate := throttle.Rate{
		KBps:    kBps,
		Latency: time.Duration(ms) * time.Millisecond,
	}
	return &throttle.Listener{
		Listener: s.listener,
		Down:     rate,
		Up:       rate, // TODO: separate rates?
	}
}

func (s *Server) Serve() {
	if err := s.Listen(""); err != nil {
		s.fatalf("Listen error: %v", err)
	}
	go runTestHarnessIntegration(s.listener)

	srv := &http.Server{
		Handler: s,
	}
	// TODO: allow configuring src.ErrorLog (and plumb through to
	// Google Cloud Logging when run on GCE, eventually)

	// Setup the NPN NextProto map for HTTP/2 support:
	http2.ConfigureServer(srv, &s.H2Server)

	err := srv.Serve(s.throttleListener())
	if err != nil {
		s.printf("Error in http server: %v\n", err)
		os.Exit(1)
	}
}

// Signals the test harness that we've started listening.
// TODO: write back the port number that we randomly selected?
// For now just writes back a single byte.
func runTestHarnessIntegration(listener net.Listener) {
	addr := os.Getenv("CAMLI_SET_BASE_URL_AND_SEND_ADDR_TO")
	c, err := net.Dial("tcp", addr)
	if err == nil {
		fmt.Fprintf(c, "%s\n", listener.Addr())
		c.Close()
	}
}

// loadX509KeyPair is a copy of tls.LoadX509KeyPair but using wkfs.
func loadX509KeyPair(certFile, keyFile string) (cert tls.Certificate, err error) {
	certPEMBlock, err := wkfs.ReadFile(certFile)
	if err != nil {
		return
	}
	keyPEMBlock, err := wkfs.ReadFile(keyFile)
	if err != nil {
		return
	}
	return tls.X509KeyPair(certPEMBlock, keyPEMBlock)
}
<commit_msg>runTestHarnessIntegration: Remove TODO, already implemented (#1403)<commit_after>/*
Copyright 2011 The Perkeep Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package webserver implements a superset wrapper of http.Server.
//
// Among other things, it can throttle its connections, inherit its
// listening socket from a file descriptor in the environment, and
// log all activity.
package webserver // import "perkeep.org/pkg/webserver"

import (
	"crypto/rand"
	"crypto/tls"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"perkeep.org/pkg/webserver/listen"

	"go4.org/net/throttle"
	"go4.org/wkfs"
	"golang.org/x/net/http2"
)

const alpnProto = "acme-tls/1" // from golang.org/x/crypto/acme.ALPNProto

type Server struct {
	mux      *http.ServeMux
	listener net.Listener
	verbose  bool // log HTTP requests and response codes

	Logger *log.Logger // or nil.

	// H2Server is the HTTP/2 server config.
	H2Server http2.Server

	// enableTLS sets the Server up for listening to HTTPS connections.
	enableTLS bool
	// tlsCertFile (tlsKeyFile) is the path to the HTTPS certificate (key) file.
	tlsCertFile, tlsKeyFile string
	// certManager is set as GetCertificate in the tls.Config of the listener. But tlsCertFile takes precedence.
	certManager func(*tls.ClientHelloInfo) (*tls.Certificate, error)

	mu   sync.Mutex
	reqs int64
}

func New() *Server {
	verbose, _ := strconv.ParseBool(os.Getenv("CAMLI_HTTP_DEBUG"))
	return &Server{
		mux:     http.NewServeMux(),
		verbose: verbose,
	}
}

func (s *Server) printf(format string, v ...interface{}) {
	if s.Logger != nil {
		s.Logger.Printf(format, v...)
		return
	}
	log.Printf(format, v...)
}

func (s *Server) fatalf(format string, v ...interface{}) {
	if s.Logger != nil {
		s.Logger.Fatalf(format, v...)
		return
	}
	log.Fatalf(format, v...)
}

// TLSSetup specifies how the server gets its TLS certificate.
type TLSSetup struct {
	// Certfile is the path to the TLS certificate file. It takes precedence over CertManager.
	CertFile string
	// KeyFile is the path to the TLS key file.
	KeyFile string
	// CertManager is the tls.GetCertificate of the tls Config. But CertFile takes precedence.
	CertManager func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error)
}

func (s *Server) SetTLS(setup TLSSetup) {
	s.enableTLS = true
	s.certManager = setup.CertManager
	s.tlsCertFile = setup.CertFile
	s.tlsKeyFile = setup.KeyFile
}

func (s *Server) ListenURL() string {
	if s.listener == nil {
		return ""
	}
	taddr, ok := s.listener.Addr().(*net.TCPAddr)
	if !ok {
		return ""
	}
	scheme := "http"
	if s.enableTLS {
		scheme = "https"
	}
	if taddr.IP.IsUnspecified() {
		return fmt.Sprintf("%s://localhost:%d", scheme, taddr.Port)
	}
	return fmt.Sprintf("%s://%s", scheme, s.listener.Addr())
}

func (s *Server) HandleFunc(pattern string, fn func(http.ResponseWriter, *http.Request)) {
	s.mux.HandleFunc(pattern, fn)
}

func (s *Server) Handle(pattern string, handler http.Handler) {
	s.mux.Handle(pattern, handler)
}

func (s *Server) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	var n int64
	if s.verbose {
		s.mu.Lock()
		s.reqs++
		n = s.reqs
		s.mu.Unlock()
		s.printf("Request #%d: %s %s (from %s) ...", n, req.Method, req.RequestURI, req.RemoteAddr)
		rw = &trackResponseWriter{ResponseWriter: rw}
	}
	s.mux.ServeHTTP(rw, req)
	if s.verbose {
		tw := rw.(*trackResponseWriter)
		s.printf("Request #%d: %s %s = code %d, %d bytes", n, req.Method, req.RequestURI, tw.code, tw.resSize)
	}
}

type trackResponseWriter struct {
	http.ResponseWriter
	code    int
	resSize int64
}

func (tw *trackResponseWriter) WriteHeader(code int) {
	tw.code = code
	tw.ResponseWriter.WriteHeader(code)
}

func (tw *trackResponseWriter) Write(p []byte) (int, error) {
	if tw.code == 0 {
		tw.code = 200
	}
	tw.resSize += int64(len(p))
	return tw.ResponseWriter.Write(p)
}

// Listen starts listening on the given host:port addr.
func (s *Server) Listen(addr string) error {
	if s.listener != nil {
		return nil
	}

	if addr == "" {
		return fmt.Errorf("<host>:<port> needs to be provided to start listening")
	}

	var err error
	s.listener, err = listen.Listen(addr)
	if err != nil {
		return fmt.Errorf("Failed to listen on %s: %v", addr, err)
	}
	base := s.ListenURL()
	s.printf("Starting to listen on %s\n", base)

	doEnableTLS := func() error {
		config := &tls.Config{
			Rand:       rand.Reader,
			Time:       time.Now,
			NextProtos: []string{http2.NextProtoTLS, "http/1.1"},
			MinVersion: tls.VersionTLS12,
		}
		if s.tlsCertFile == "" && s.certManager != nil {
			config.GetCertificate = s.certManager
			// TODO(mpl): see if we can instead use
			// https://godoc.org/golang.org/x/crypto/acme/autocert#Manager.TLSConfig
			config.NextProtos = append(config.NextProtos, alpnProto)
			s.listener = tls.NewListener(s.listener, config)
			return nil
		}

		config.Certificates = make([]tls.Certificate, 1)
		config.Certificates[0], err = loadX509KeyPair(s.tlsCertFile, s.tlsKeyFile)
		if err != nil {
			return fmt.Errorf("Failed to load TLS cert: %v", err)
		}
		s.listener = tls.NewListener(s.listener, config)
		return nil
	}
	if s.enableTLS {
		if err := doEnableTLS(); err != nil {
			return err
		}
	}

	if strings.HasSuffix(base, ":0") {
		s.printf("Now listening on %s\n", s.ListenURL())
	}

	return nil
}

func (s *Server) throttleListener() net.Listener {
	kBps, _ := strconv.Atoi(os.Getenv("DEV_THROTTLE_KBPS"))
	ms, _ := strconv.Atoi(os.Getenv("DEV_THROTTLE_LATENCY_MS"))
	if kBps == 0 && ms == 0 {
		return s.listener
	}
	rate := throttle.Rate{
		KBps:    kBps,
		Latency: time.Duration(ms) * time.Millisecond,
	}
	return &throttle.Listener{
		Listener: s.listener,
		Down:     rate,
		Up:       rate, // TODO: separate rates?
	}
}

func (s *Server) Serve() {
	if err := s.Listen(""); err != nil {
		s.fatalf("Listen error: %v", err)
	}
	go runTestHarnessIntegration(s.listener)

	srv := &http.Server{
		Handler: s,
	}
	// TODO: allow configuring src.ErrorLog (and plumb through to
	// Google Cloud Logging when run on GCE, eventually)

	// Setup the NPN NextProto map for HTTP/2 support:
	http2.ConfigureServer(srv, &s.H2Server)

	err := srv.Serve(s.throttleListener())
	if err != nil {
		s.printf("Error in http server: %v\n", err)
		os.Exit(1)
	}
}

// Signals the test harness that we've started listening.
// Writes back the address that we randomly selected.
func runTestHarnessIntegration(listener net.Listener) {
	addr := os.Getenv("CAMLI_SET_BASE_URL_AND_SEND_ADDR_TO")
	if addr == "" {
		return
	}
	c, err := net.Dial("tcp", addr)
	if err == nil {
		fmt.Fprintf(c, "%s\n", listener.Addr())
		c.Close()
	}
}

// loadX509KeyPair is a copy of tls.LoadX509KeyPair but using wkfs.
func loadX509KeyPair(certFile, keyFile string) (cert tls.Certificate, err error) {
	certPEMBlock, err := wkfs.ReadFile(certFile)
	if err != nil {
		return
	}
	keyPEMBlock, err := wkfs.ReadFile(keyFile)
	if err != nil {
		return
	}
	return tls.X509KeyPair(certPEMBlock, keyPEMBlock)
}
<|endoftext|>


<commit_before>//go:build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"testing"
)

// stderrAllow are regular expressions acceptable to find in normal stderr
var stderrAllow = []string{
	// kubectl out of date warning
	`kubectl`,
	// slow docker warning
	`slow|long time|Restarting the docker service may improve`,
	// don't care if we can't push images to other profiles
	`cache_images.go:.*error getting status`,
	// don't care if we can't push images to other profiles which are deleted.
	`cache_images.go:.*Failed to load profile`,
	// "! 'docker' driver reported a issue that could affect the performance."
	`docker.*issue.*performance`,
	// "* Suggestion: enable overlayfs kernel module on your Linux"
	`Suggestion.*overlayfs`,
	// "! docker is currently using the btrfs storage driver, consider switching to overlay2 for better performance"
	`docker.*btrfs storage driver`,
	// jenkins VMs (debian 9) cgoups don't allow setting memory
	`Your cgroup does not allow setting memory.`,
	// progress bar output
	`    > .*`,
	// Warning of issues with specific Kubernetes versions
	`Kubernetes .* has a known `,
}

// stderrAllowRe combines rootCauses into a single regex
var stderrAllowRe = regexp.MustCompile(strings.Join(stderrAllow, "|"))

// TestErrorSpam asserts that there are no unexpected errors displayed in minikube command outputs.
func TestErrorSpam(t *testing.T) {
	if NoneDriver() {
		t.Skip("none driver always shows a warning")
	}

	profile := UniqueProfileName("nospam")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(25))
	defer CleanupWithLogs(t, profile, cancel)

	logDir := filepath.Join(os.TempDir(), profile)
	if err := os.MkdirAll(logDir, 0755); err != nil {
		t.Fatalf("Unable to make logDir %s: %v", logDir, err)
	}
	defer os.RemoveAll(logDir)

	t.Run("setup", func(t *testing.T) {
		// This should likely use multi-node once it's ready
		// use `--log_dir` flag to run isolated and avoid race condition - ie, failing to clean up (locked) log files created by other concurently-run tests, or counting them in results
		args := append([]string{"start", "-p", profile, "-n=1", "--memory=2250", "--wait=false", fmt.Sprintf("--log_dir=%s", logDir)}, StartArgs()...)

		rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
		if err != nil {
			t.Errorf("%q failed: %v", rr.Command(), err)
		}

		stdout := rr.Stdout.String()
		stderr := rr.Stderr.String()

		for _, line := range strings.Split(stderr, "\n") {
			if stderrAllowRe.MatchString(line) {
				t.Logf("acceptable stderr: %q", line)
				continue
			}

			if len(strings.TrimSpace(line)) > 0 {
				t.Errorf("unexpected stderr: %q", line)
			}
		}

		for _, line := range strings.Split(stdout, "\n") {
			keywords := []string{"error", "fail", "warning", "conflict"}
			for _, keyword := range keywords {
				if strings.Contains(line, keyword) {
					t.Errorf("unexpected %q in stdout: %q", keyword, line)
				}
			}
		}

		if t.Failed() {
			t.Logf("minikube stdout:\n%s", stdout)
			t.Logf("minikube stderr:\n%s", stderr)
		}

		steps := []string{
			"Generating certificates and keys ...",
			"Booting up control plane ...",
			"Configuring RBAC rules ...",
		}
		for _, step := range steps {
			if !strings.Contains(stdout, step) {
				t.Errorf("missing kubeadm init sub-step %q", step)
			}
		}
	})

	logTests := []struct {
		command string
		args    []string
	}{
		{
			command: "start",
			args:    []string{"--dry-run"},
		},
		{
			command: "status",
		}, {
			command: "pause",
		}, {
			command: "unpause",
		}, {
			command: "stop",
		},
	}

	for _, test := range logTests {
		t.Run(test.command, func(t *testing.T) {
			// before starting the test, ensure no other logs from the current command are written
			logFiles, err := getLogFiles(logDir, test.command)
			if err != nil {
				t.Fatalf("failed to get old log files for command %s : %v", test.command, err)
			}
			cleanupLogFiles(t, logFiles)

			args := []string{"-p", profile, "--log_dir", logDir, test.command}
			args = append(args, test.args...)

			// run command twice
			for i := 0; i < 2; i++ {
				rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
				if err != nil {
					t.Logf("%q failed: %v", rr.Command(), err)
				}
			}

			// check if one log file exists
			if err := checkLogFileCount(test.command, logDir, 1); err != nil {
				t.Fatal(err)
			}

			// get log file generated above
			logFiles, err = getLogFiles(logDir, test.command)
			if err != nil {
				t.Fatalf("failed to get new log files for command %s : %v", test.command, err)
			}

			// make file at least 1024 KB in size
			if err := os.Truncate(logFiles[0], 2e7); err != nil {
				t.Fatalf("failed to increase file size to 1024KB: %v", err)
			}

			// run command again
			rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
			if err != nil {
				t.Logf("%q failed: %v", rr.Command(), err)
			}

			// check if two log files exist now
			if err := checkLogFileCount(test.command, logDir, 2); err != nil {
				t.Fatal(err)
			}
		})

	}
}

func getLogFiles(logDir string, command string) ([]string, error) {
	return filepath.Glob(filepath.Join(logDir, fmt.Sprintf("minikube_%s*", command)))
}

func checkLogFileCount(command string, logDir string, expectedNumberOfLogFiles int) error {
	// get log files generated above
	logFiles, err := getLogFiles(logDir, command)
	if err != nil {
		return fmt.Errorf("failed to get new log files for command %s : %v", command, err)
	}

	if len(logFiles) != expectedNumberOfLogFiles {
		return fmt.Errorf("Running cmd %q resulted in %d log file(s); expected: %d", command, len(logFiles), expectedNumberOfLogFiles)
	}

	return nil
}

// cleanupLogFiles removes logfiles generated during testing
func cleanupLogFiles(t *testing.T, logFiles []string) {
	t.Logf("Cleaning up %d logfile(s) ...", len(logFiles))
	for _, logFile := range logFiles {
		if err := os.Remove(logFile); err != nil {
			t.Logf("failed to cleanup log file: %s : %v", logFile, err)
		}
	}
}
<commit_msg>add another line to TestErrorSpam<commit_after>//go:build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"testing"
)

// stderrAllow are regular expressions acceptable to find in normal stderr
var stderrAllow = []string{
	// kubectl out of date warning
	`kubectl`,
	// slow docker warning
	`slow|long time|Restarting the docker service may improve`,
	// don't care if we can't push images to other profiles
	`cache_images.go:.*error getting status`,
	// don't care if we can't push images to other profiles which are deleted.
	`cache_images.go:.*Failed to load profile`,
	// "! 'docker' driver reported a issue that could affect the performance."
	`docker.*issue.*performance`,
	// "* Suggestion: enable overlayfs kernel module on your Linux"
	`Suggestion.*overlayfs`,
	// "! docker is currently using the btrfs storage driver, consider switching to overlay2 for better performance"
	`docker.*btrfs storage driver`,
	// jenkins VMs (debian 9) cgoups don't allow setting memory
	`Your cgroup does not allow setting memory.`,
	// progress bar output
	`    > .*`,
	// Warning of issues with specific Kubernetes versions
	`Kubernetes .* has a known `,
	`For more information, see`,
}

// stderrAllowRe combines rootCauses into a single regex
var stderrAllowRe = regexp.MustCompile(strings.Join(stderrAllow, "|"))

// TestErrorSpam asserts that there are no unexpected errors displayed in minikube command outputs.
func TestErrorSpam(t *testing.T) {
	if NoneDriver() {
		t.Skip("none driver always shows a warning")
	}

	profile := UniqueProfileName("nospam")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(25))
	defer CleanupWithLogs(t, profile, cancel)

	logDir := filepath.Join(os.TempDir(), profile)
	if err := os.MkdirAll(logDir, 0755); err != nil {
		t.Fatalf("Unable to make logDir %s: %v", logDir, err)
	}
	defer os.RemoveAll(logDir)

	t.Run("setup", func(t *testing.T) {
		// This should likely use multi-node once it's ready
		// use `--log_dir` flag to run isolated and avoid race condition - ie, failing to clean up (locked) log files created by other concurently-run tests, or counting them in results
		args := append([]string{"start", "-p", profile, "-n=1", "--memory=2250", "--wait=false", fmt.Sprintf("--log_dir=%s", logDir)}, StartArgs()...)

		rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
		if err != nil {
			t.Errorf("%q failed: %v", rr.Command(), err)
		}

		stdout := rr.Stdout.String()
		stderr := rr.Stderr.String()

		for _, line := range strings.Split(stderr, "\n") {
			if stderrAllowRe.MatchString(line) {
				t.Logf("acceptable stderr: %q", line)
				continue
			}

			if len(strings.TrimSpace(line)) > 0 {
				t.Errorf("unexpected stderr: %q", line)
			}
		}

		for _, line := range strings.Split(stdout, "\n") {
			keywords := []string{"error", "fail", "warning", "conflict"}
			for _, keyword := range keywords {
				if strings.Contains(line, keyword) {
					t.Errorf("unexpected %q in stdout: %q", keyword, line)
				}
			}
		}

		if t.Failed() {
			t.Logf("minikube stdout:\n%s", stdout)
			t.Logf("minikube stderr:\n%s", stderr)
		}

		steps := []string{
			"Generating certificates and keys ...",
			"Booting up control plane ...",
			"Configuring RBAC rules ...",
		}
		for _, step := range steps {
			if !strings.Contains(stdout, step) {
				t.Errorf("missing kubeadm init sub-step %q", step)
			}
		}
	})

	logTests := []struct {
		command string
		args    []string
	}{
		{
			command: "start",
			args:    []string{"--dry-run"},
		},
		{
			command: "status",
		}, {
			command: "pause",
		}, {
			command: "unpause",
		}, {
			command: "stop",
		},
	}

	for _, test := range logTests {
		t.Run(test.command, func(t *testing.T) {
			// before starting the test, ensure no other logs from the current command are written
			logFiles, err := getLogFiles(logDir, test.command)
			if err != nil {
				t.Fatalf("failed to get old log files for command %s : %v", test.command, err)
			}
			cleanupLogFiles(t, logFiles)

			args := []string{"-p", profile, "--log_dir", logDir, test.command}
			args = append(args, test.args...)

			// run command twice
			for i := 0; i < 2; i++ {
				rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
				if err != nil {
					t.Logf("%q failed: %v", rr.Command(), err)
				}
			}

			// check if one log file exists
			if err := checkLogFileCount(test.command, logDir, 1); err != nil {
				t.Fatal(err)
			}

			// get log file generated above
			logFiles, err = getLogFiles(logDir, test.command)
			if err != nil {
				t.Fatalf("failed to get new log files for command %s : %v", test.command, err)
			}

			// make file at least 1024 KB in size
			if err := os.Truncate(logFiles[0], 2e7); err != nil {
				t.Fatalf("failed to increase file size to 1024KB: %v", err)
			}

			// run command again
			rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
			if err != nil {
				t.Logf("%q failed: %v", rr.Command(), err)
			}

			// check if two log files exist now
			if err := checkLogFileCount(test.command, logDir, 2); err != nil {
				t.Fatal(err)
			}
		})

	}
}

func getLogFiles(logDir string, command string) ([]string, error) {
	return filepath.Glob(filepath.Join(logDir, fmt.Sprintf("minikube_%s*", command)))
}

func checkLogFileCount(command string, logDir string, expectedNumberOfLogFiles int) error {
	// get log files generated above
	logFiles, err := getLogFiles(logDir, command)
	if err != nil {
		return fmt.Errorf("failed to get new log files for command %s : %v", command, err)
	}

	if len(logFiles) != expectedNumberOfLogFiles {
		return fmt.Errorf("Running cmd %q resulted in %d log file(s); expected: %d", command, len(logFiles), expectedNumberOfLogFiles)
	}

	return nil
}

// cleanupLogFiles removes logfiles generated during testing
func cleanupLogFiles(t *testing.T, logFiles []string) {
	t.Logf("Cleaning up %d logfile(s) ...", len(logFiles))
	for _, logFile := range logFiles {
		if err := os.Remove(logFile); err != nil {
			t.Logf("failed to cleanup log file: %s : %v", logFile, err)
		}
	}
}
<|endoftext|>


<commit_before>spackage scaleway

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/log"
	"github.com/docker/machine/libmachine/mcnflag"
	"github.com/docker/machine/libmachine/ssh"
	"github.com/docker/machine/libmachine/state"
	"github.com/moul/anonuuid"
	"github.com/scaleway/scaleway-cli/pkg/api"
	"github.com/scaleway/scaleway-cli/pkg/config"
)

const (
	// VERSION represents the semver version of the package
	VERSION           = "v1.2.0+dev"
	defaultImage      = "ubuntu-xenial"
	defaultBootscript = "docker"
)

var scwAPI *api.ScalewayAPI

// Driver represents the docker driver interface
type Driver struct {
	*drivers.BaseDriver
	ServerID       string
	Organization   string
	IPID           string
	Token          string
	CommercialType string
	name           string
	image          string
	ip             string
	volumes        string
	stopping       bool
	created        bool
	// userDataFile string
	// ipv6         bool
}

// DriverName returns the name of the driver
func (d *Driver) DriverName() string {
	if d.CommercialType == "" {
		return "scaleway"
	}
	return fmt.Sprintf("scaleway(%v)", d.CommercialType)
}

func (d *Driver) getClient() (cl *api.ScalewayAPI, err error) {
	if scwAPI == nil {
		scwAPI, err = api.NewScalewayAPI(d.Organization, d.Token, "docker-machine-driver-scaleway/%v"+VERSION)
	}
	cl = scwAPI
	return
}

// SetConfigFromFlags sets the flags
func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) (err error) {
	if flags.Bool("scaleway-debug") {
		logrus.SetOutput(os.Stderr)
		logrus.SetLevel(logrus.DebugLevel)
	}

	d.Token, d.Organization = flags.String("scaleway-token"), flags.String("scaleway-organization")
	if d.Token == "" || d.Organization == "" {
		config, cfgErr := config.GetConfig()
		if cfgErr == nil {
			if d.Token == "" {
				d.Token = config.Token
			}
			if d.Organization == "" {
				d.Organization = config.Organization
			}
		} else {
			return fmt.Errorf("You must provide organization and token")
		}
	}
	d.CommercialType = flags.String("scaleway-commercial-type")
	d.name = flags.String("scaleway-name")
	d.image = flags.String("scaleway-image")
	d.ip = flags.String("scaleway-ip")
	d.volumes = flags.String("scaleway-volumes")
	return
}

// NewDriver returns a new driver
func NewDriver(hostName, storePath string) *Driver {
	return &Driver{
		BaseDriver: &drivers.BaseDriver{},
	}
}

// GetCreateFlags registers the flags
func (d *Driver) GetCreateFlags() []mcnflag.Flag {
	return []mcnflag.Flag{
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_TOKEN",
			Name:   "scaleway-token",
			Usage:  "Scaleway token",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_ORGANIZATION",
			Name:   "scaleway-organization",
			Usage:  "Scaleway organization",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_NAME",
			Name:   "scaleway-name",
			Usage:  "Assign a name",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_COMMERCIAL_TYPE",
			Name:   "scaleway-commercial-type",
			Usage:  "Specifies the commercial type",
			Value:  "VC1S",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_IMAGE",
			Name:   "scaleway-image",
			Usage:  "Specifies the image",
			Value:  defaultImage,
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_IP",
			Name:   "scaleway-ip",
			Usage:  "Specifies the IP address",
			Value:  "",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_VOLUMES",
			Name:   "scaleway-volumes",
			Usage:  "Attach additional volume (e.g., 50G)",
			Value:  "",
		},
		mcnflag.BoolFlag{
			EnvVar: "SCALEWAY_DEBUG",
			Name:   "scaleway-debug",
			Usage:  "Enables Scaleway client debugging",
		},
		// mcnflag.StringFlag{
		//     EnvVar: "SCALEWAY_USERDATA",
		//     Name:   "scaleway-userdata",
		//     Usage:  "Path to file with user-data",
		// },
		// mcnflag.BoolFlag{
		// 	EnvVar: "SCALEWAY_IPV6",
		// 	Name:   "scaleway-ipv6",
		// 	Usage:  "Enable ipv6",
		// },
	}
}

// Create configures and starts a scaleway server
func (d *Driver) Create() (err error) {
	var publicKey []byte
	var cl *api.ScalewayAPI

	log.Infof("Creating SSH key...")
	if err = ssh.GenerateSSHKey(d.GetSSHKeyPath()); err != nil {
		return err
	}
	publicKey, err = ioutil.ReadFile(d.GetSSHKeyPath() + ".pub")
	if err != nil {
		return
	}
	log.Infof("Creating server...")
	cl, err = d.getClient()
	if err != nil {
		return
	}
	if d.ip != "" {
		var ips *api.ScalewayGetIPS

		ips, err = cl.GetIPS()
		if err != nil {
			return
		}
		if anonuuid.IsUUID(d.ip) == nil {
			d.IPID = d.ip
			for _, ip := range ips.IPS {
				if ip.ID == d.ip {
					d.IPAddress = ip.Address
					break
				}
			}
			if d.IPAddress == "" {
				err = fmt.Errorf("IP UUID %v not found", d.IPID)
				return
			}
		} else {
			d.IPAddress = d.ip
			for _, ip := range ips.IPS {
				if ip.Address == d.ip {
					d.IPID = ip.ID
					break
				}
			}
			if d.IPID == "" {
				err = fmt.Errorf("IP address %v not found", d.ip)
				return
			}
		}
	} else {
		var ip *api.ScalewayGetIP

		ip, err = cl.NewIP()
		if err != nil {
			return
		}
		d.IPAddress = ip.IP.Address
		d.IPID = ip.IP.ID
	}
	d.ServerID, err = api.CreateServer(cl, &api.ConfigCreateServer{
		ImageName:         d.image,
		CommercialType:    d.CommercialType,
		Name:              d.name,
		Bootscript:        defaultBootscript,
		AdditionalVolumes: d.volumes,
		IP:                d.IPID,
		Env: strings.Join([]string{"AUTHORIZED_KEY",
			strings.Replace(string(publicKey[:len(publicKey)-1]), " ", "_", -1)}, "="),
	})
	if err != nil {
		return
	}
	log.Infof("Starting server...")
	err = api.StartServer(cl, d.ServerID, false)
	d.created = true
	return
}

// GetSSHHostname returns the IP of the server
func (d *Driver) GetSSHHostname() (string, error) {
	return d.IPAddress, nil
}

// GetState returns the state of the server
func (d *Driver) GetState() (st state.State, err error) {
	var server *api.ScalewayServer
	var cl *api.ScalewayAPI

	st = state.Error
	cl, err = d.getClient()
	if err != nil {
		return
	}
	server, err = cl.GetServer(d.ServerID)
	if err != nil {
		return
	}
	st = state.None
	switch server.State {
	case "starting":
		st = state.Starting
	case "running":
		if d.created {
			time.Sleep(60 * time.Second)
			d.created = false
		}
		st = state.Running
	case "stopping":
		st = state.Stopping
	case "stopped":
		st = state.Stopped
	}
	if d.stopping {
		time.Sleep(5 * time.Second)
	}
	return
}

// GetURL returns IP + docker port
func (d *Driver) GetURL() (string, error) {
	if err := drivers.MustBeRunning(d); err != nil {
		return "", err
	}
	return fmt.Sprintf("tcp://%s", net.JoinHostPort(d.IPAddress, "2376")), nil
}

func (d *Driver) postAction(action string) (err error) {
	var cl *api.ScalewayAPI

	cl, err = d.getClient()
	if err != nil {
		return
	}
	err = cl.PostServerAction(d.ServerID, action)
	return
}

// Kill does nothing
func (d *Driver) Kill() error {
	return errors.New("scaleway driver does not support kill")
}

// Remove shutdowns the server and removes the IP
func (d *Driver) Remove() (err error) {
	var cl *api.ScalewayAPI

	cl, err = d.getClient()
	if err != nil {
		return
	}
	err = cl.PostServerAction(d.ServerID, "terminate")
	if err != nil {
		return
	}
	for {
		_, err = cl.GetServer(d.ServerID)
		if err != nil {
			break
		}
	}
	err = cl.DeleteIP(d.IPID)
	return
}

// Restart reboots the server
func (d *Driver) Restart() error {
	return d.postAction("reboot")
}

// Start starts the server
func (d *Driver) Start() error {
	return d.postAction("poweron")
}

// Stop stops the server
func (d *Driver) Stop() error {
	d.stopping = true
	return d.postAction("poweroff")
}
<commit_msg>Remove typo (fix #40)<commit_after>package scaleway

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/log"
	"github.com/docker/machine/libmachine/mcnflag"
	"github.com/docker/machine/libmachine/ssh"
	"github.com/docker/machine/libmachine/state"
	"github.com/moul/anonuuid"
	"github.com/scaleway/scaleway-cli/pkg/api"
	"github.com/scaleway/scaleway-cli/pkg/config"
)

const (
	// VERSION represents the semver version of the package
	VERSION           = "v1.2.0+dev"
	defaultImage      = "ubuntu-xenial"
	defaultBootscript = "docker"
)

var scwAPI *api.ScalewayAPI

// Driver represents the docker driver interface
type Driver struct {
	*drivers.BaseDriver
	ServerID       string
	Organization   string
	IPID           string
	Token          string
	CommercialType string
	name           string
	image          string
	ip             string
	volumes        string
	stopping       bool
	created        bool
	// userDataFile string
	// ipv6         bool
}

// DriverName returns the name of the driver
func (d *Driver) DriverName() string {
	if d.CommercialType == "" {
		return "scaleway"
	}
	return fmt.Sprintf("scaleway(%v)", d.CommercialType)
}

func (d *Driver) getClient() (cl *api.ScalewayAPI, err error) {
	if scwAPI == nil {
		scwAPI, err = api.NewScalewayAPI(d.Organization, d.Token, "docker-machine-driver-scaleway/%v"+VERSION)
	}
	cl = scwAPI
	return
}

// SetConfigFromFlags sets the flags
func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) (err error) {
	if flags.Bool("scaleway-debug") {
		logrus.SetOutput(os.Stderr)
		logrus.SetLevel(logrus.DebugLevel)
	}

	d.Token, d.Organization = flags.String("scaleway-token"), flags.String("scaleway-organization")
	if d.Token == "" || d.Organization == "" {
		config, cfgErr := config.GetConfig()
		if cfgErr == nil {
			if d.Token == "" {
				d.Token = config.Token
			}
			if d.Organization == "" {
				d.Organization = config.Organization
			}
		} else {
			return fmt.Errorf("You must provide organization and token")
		}
	}
	d.CommercialType = flags.String("scaleway-commercial-type")
	d.name = flags.String("scaleway-name")
	d.image = flags.String("scaleway-image")
	d.ip = flags.String("scaleway-ip")
	d.volumes = flags.String("scaleway-volumes")
	return
}

// NewDriver returns a new driver
func NewDriver(hostName, storePath string) *Driver {
	return &Driver{
		BaseDriver: &drivers.BaseDriver{},
	}
}

// GetCreateFlags registers the flags
func (d *Driver) GetCreateFlags() []mcnflag.Flag {
	return []mcnflag.Flag{
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_TOKEN",
			Name:   "scaleway-token",
			Usage:  "Scaleway token",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_ORGANIZATION",
			Name:   "scaleway-organization",
			Usage:  "Scaleway organization",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_NAME",
			Name:   "scaleway-name",
			Usage:  "Assign a name",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_COMMERCIAL_TYPE",
			Name:   "scaleway-commercial-type",
			Usage:  "Specifies the commercial type",
			Value:  "VC1S",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_IMAGE",
			Name:   "scaleway-image",
			Usage:  "Specifies the image",
			Value:  defaultImage,
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_IP",
			Name:   "scaleway-ip",
			Usage:  "Specifies the IP address",
			Value:  "",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_VOLUMES",
			Name:   "scaleway-volumes",
			Usage:  "Attach additional volume (e.g., 50G)",
			Value:  "",
		},
		mcnflag.BoolFlag{
			EnvVar: "SCALEWAY_DEBUG",
			Name:   "scaleway-debug",
			Usage:  "Enables Scaleway client debugging",
		},
		// mcnflag.StringFlag{
		//     EnvVar: "SCALEWAY_USERDATA",
		//     Name:   "scaleway-userdata",
		//     Usage:  "Path to file with user-data",
		// },
		// mcnflag.BoolFlag{
		// 	EnvVar: "SCALEWAY_IPV6",
		// 	Name:   "scaleway-ipv6",
		// 	Usage:  "Enable ipv6",
		// },
	}
}

// Create configures and starts a scaleway server
func (d *Driver) Create() (err error) {
	var publicKey []byte
	var cl *api.ScalewayAPI

	log.Infof("Creating SSH key...")
	if err = ssh.GenerateSSHKey(d.GetSSHKeyPath()); err != nil {
		return err
	}
	publicKey, err = ioutil.ReadFile(d.GetSSHKeyPath() + ".pub")
	if err != nil {
		return
	}
	log.Infof("Creating server...")
	cl, err = d.getClient()
	if err != nil {
		return
	}
	if d.ip != "" {
		var ips *api.ScalewayGetIPS

		ips, err = cl.GetIPS()
		if err != nil {
			return
		}
		if anonuuid.IsUUID(d.ip) == nil {
			d.IPID = d.ip
			for _, ip := range ips.IPS {
				if ip.ID == d.ip {
					d.IPAddress = ip.Address
					break
				}
			}
			if d.IPAddress == "" {
				err = fmt.Errorf("IP UUID %v not found", d.IPID)
				return
			}
		} else {
			d.IPAddress = d.ip
			for _, ip := range ips.IPS {
				if ip.Address == d.ip {
					d.IPID = ip.ID
					break
				}
			}
			if d.IPID == "" {
				err = fmt.Errorf("IP address %v not found", d.ip)
				return
			}
		}
	} else {
		var ip *api.ScalewayGetIP

		ip, err = cl.NewIP()
		if err != nil {
			return
		}
		d.IPAddress = ip.IP.Address
		d.IPID = ip.IP.ID
	}
	d.ServerID, err = api.CreateServer(cl, &api.ConfigCreateServer{
		ImageName:         d.image,
		CommercialType:    d.CommercialType,
		Name:              d.name,
		Bootscript:        defaultBootscript,
		AdditionalVolumes: d.volumes,
		IP:                d.IPID,
		Env: strings.Join([]string{"AUTHORIZED_KEY",
			strings.Replace(string(publicKey[:len(publicKey)-1]), " ", "_", -1)}, "="),
	})
	if err != nil {
		return
	}
	log.Infof("Starting server...")
	err = api.StartServer(cl, d.ServerID, false)
	d.created = true
	return
}

// GetSSHHostname returns the IP of the server
func (d *Driver) GetSSHHostname() (string, error) {
	return d.IPAddress, nil
}

// GetState returns the state of the server
func (d *Driver) GetState() (st state.State, err error) {
	var server *api.ScalewayServer
	var cl *api.ScalewayAPI

	st = state.Error
	cl, err = d.getClient()
	if err != nil {
		return
	}
	server, err = cl.GetServer(d.ServerID)
	if err != nil {
		return
	}
	st = state.None
	switch server.State {
	case "starting":
		st = state.Starting
	case "running":
		if d.created {
			time.Sleep(60 * time.Second)
			d.created = false
		}
		st = state.Running
	case "stopping":
		st = state.Stopping
	case "stopped":
		st = state.Stopped
	}
	if d.stopping {
		time.Sleep(5 * time.Second)
	}
	return
}

// GetURL returns IP + docker port
func (d *Driver) GetURL() (string, error) {
	if err := drivers.MustBeRunning(d); err != nil {
		return "", err
	}
	return fmt.Sprintf("tcp://%s", net.JoinHostPort(d.IPAddress, "2376")), nil
}

func (d *Driver) postAction(action string) (err error) {
	var cl *api.ScalewayAPI

	cl, err = d.getClient()
	if err != nil {
		return
	}
	err = cl.PostServerAction(d.ServerID, action)
	return
}

// Kill does nothing
func (d *Driver) Kill() error {
	return errors.New("scaleway driver does not support kill")
}

// Remove shutdowns the server and removes the IP
func (d *Driver) Remove() (err error) {
	var cl *api.ScalewayAPI

	cl, err = d.getClient()
	if err != nil {
		return
	}
	err = cl.PostServerAction(d.ServerID, "terminate")
	if err != nil {
		return
	}
	for {
		_, err = cl.GetServer(d.ServerID)
		if err != nil {
			break
		}
	}
	err = cl.DeleteIP(d.IPID)
	return
}

// Restart reboots the server
func (d *Driver) Restart() error {
	return d.postAction("reboot")
}

// Start starts the server
func (d *Driver) Start() error {
	return d.postAction("poweron")
}

// Stop stops the server
func (d *Driver) Stop() error {
	d.stopping = true
	return d.postAction("poweroff")
}
<|endoftext|>


<commit_before>package scaleway

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/log"
	"github.com/docker/machine/libmachine/mcnflag"
	"github.com/docker/machine/libmachine/ssh"
	"github.com/docker/machine/libmachine/state"
	"github.com/scaleway/scaleway-cli/pkg/api"
	"github.com/scaleway/scaleway-cli/pkg/config"
)

const (
	// VERSION represents the semver version of the package
	VERSION           = "v1.0.2+dev"
	defaultImage      = "ubuntu-xenial"
	defaultBootscript = "docker"
)

var scwAPI *api.ScalewayAPI

// Driver represents the docker driver interface
type Driver struct {
	*drivers.BaseDriver
	ServerID       string
	Organization   string
	IPID           string
	Token          string
	CommercialType string
	name           string
	image          string
	stopping       bool
	created        bool
	// size         string
	// userDataFile string
	// ipv6         bool
}

// DriverName returns the name of the driver
func (d *Driver) DriverName() string {
	if d.CommercialType == "" {
		return "scaleway"
	}
	return fmt.Sprintf("scaleway (%v)", d.CommercialType)
}

func (d *Driver) getClient() (cl *api.ScalewayAPI, err error) {
	if scwAPI == nil {
		scwAPI, err = api.NewScalewayAPI(d.Organization, d.Token, "docker-machine-driver-scaleway/%v"+VERSION)
	}
	cl = scwAPI
	return
}

// SetConfigFromFlags sets the flags
func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) (err error) {
	if flags.Bool("scaleway-debug") {
		logrus.SetOutput(os.Stderr)
		logrus.SetLevel(logrus.DebugLevel)
	}

	d.Token, d.Organization = flags.String("scaleway-token"), flags.String("scaleway-organization")
	if d.Token == "" || d.Organization == "" {
		config, cfgErr := config.GetConfig()
		if cfgErr == nil {
			if d.Token == "" {
				d.Token = config.Token
			}
			if d.Organization == "" {
				d.Organization = config.Organization
			}
		} else {
			return fmt.Errorf("You must provide organization and token")
		}
	}
	d.CommercialType = flags.String("scaleway-commercial-type")
	d.name = flags.String("scaleway-name")
	d.image = flags.String("scaleway-image")
	return
}

// NewDriver returns a new driver
func NewDriver(hostName, storePath string) *Driver {
	return &Driver{
		BaseDriver: &drivers.BaseDriver{},
	}
}

// GetCreateFlags registers the flags
func (d *Driver) GetCreateFlags() []mcnflag.Flag {
	return []mcnflag.Flag{
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_TOKEN",
			Name:   "scaleway-token",
			Usage:  "Scaleway token",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_ORGANIZATION",
			Name:   "scaleway-organization",
			Usage:  "Scaleway organization",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_NAME",
			Name:   "scaleway-name",
			Usage:  "Assign a name",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_COMMERCIAL_TYPE",
			Name:   "scaleway-commercial-type",
			Usage:  "Specifies the commercial type",
			Value:  "VC1S",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_IMAGE",
			Name:   "scaleway-image",
			Usage:  "Specifies the image",
			Value:  defaultImage,
		},
		mcnflag.BoolFlag{
			EnvVar: "SCALEWAY_DEBUG",
			Name:   "scaleway-debug",
			Usage:  "Enables Scaleway client debugging",
		},
		// mcnflag.StringFlag{
		//     EnvVar: "SCALEWAY_USERDATA",
		//     Name:   "scaleway-userdata",
		//     Usage:  "Path to file with user-data",
		// },
		// mcnflag.BoolFlag{
		// 	EnvVar: "SCALEWAY_IPV6",
		// 	Name:   "scaleway-ipv6",
		// 	Usage:  "Enable ipv6",
		// },
	}
}

// Create configures and starts a scaleway server
func (d *Driver) Create() (err error) {
	var publicKey []byte
	var cl *api.ScalewayAPI
	var ip *api.ScalewayGetIP

	log.Infof("Creating SSH key...")
	if err = ssh.GenerateSSHKey(d.GetSSHKeyPath()); err != nil {
		return err
	}
	publicKey, err = ioutil.ReadFile(d.GetSSHKeyPath() + ".pub")
	if err != nil {
		return
	}
	log.Infof("Creating server...")
	cl, err = d.getClient()
	if err != nil {
		return
	}
	ip, err = cl.NewIP()
	if err != nil {
		return
	}
	d.IPAddress = ip.IP.Address
	d.IPID = ip.IP.ID
	d.ServerID, err = api.CreateServer(cl, &api.ConfigCreateServer{
		ImageName:      d.image,
		CommercialType: d.CommercialType,
		Name:           d.name,
		Bootscript:     defaultBootscript,
		IP:             ip.IP.ID,
		Env: strings.Join([]string{"AUTHORIZED_KEY",
			strings.Replace(string(publicKey[:len(publicKey)-1]), " ", "_", -1)}, "="),
	})
	if err != nil {
		return
	}
	log.Infof("Starting server...")
	err = api.StartServer(cl, d.ServerID, false)
	d.created = true
	return
}

// GetSSHHostname returns the IP of the server
func (d *Driver) GetSSHHostname() (string, error) {
	return d.IPAddress, nil
}

// GetState returns the state of the server
func (d *Driver) GetState() (st state.State, err error) {
	var server *api.ScalewayServer
	var cl *api.ScalewayAPI

	st = state.Error
	cl, err = d.getClient()
	if err != nil {
		return
	}
	server, err = cl.GetServer(d.ServerID)
	if err != nil {
		return
	}
	st = state.None
	switch server.State {
	case "starting":
		st = state.Starting
	case "running":
		if d.created {
			time.Sleep(60 * time.Second)
			d.created = false
		}
		st = state.Running
	case "stopping":
		st = state.Stopping
	case "stopped":
		st = state.Stopped
	}
	if d.stopping {
		time.Sleep(5 * time.Second)
	}
	return
}

// GetURL returns IP + docker port
func (d *Driver) GetURL() (string, error) {
	if err := drivers.MustBeRunning(d); err != nil {
		return "", err
	}
	return fmt.Sprintf("tcp://%s", net.JoinHostPort(d.IPAddress, "2376")), nil
}

func (d *Driver) postAction(action string) (err error) {
	var cl *api.ScalewayAPI

	cl, err = d.getClient()
	if err != nil {
		return
	}
	err = cl.PostServerAction(d.ServerID, action)
	return
}

// Kill does nothing
func (d *Driver) Kill() error {
	return errors.New("scaleway driver does not support kill")
}

// Remove shutdowns the server and removes the IP
func (d *Driver) Remove() (err error) {
	var cl *api.ScalewayAPI

	cl, err = d.getClient()
	if err != nil {
		return
	}
	err = cl.PostServerAction(d.ServerID, "terminate")
	if err != nil {
		return
	}
	for {
		_, err = cl.GetServer(d.ServerID)
		if err != nil {
			break
		}
	}
	err = cl.DeleteIP(d.IPID)
	return
}

// Restart reboots the server
func (d *Driver) Restart() error {
	return d.postAction("reboot")
}

// Start starts the server
func (d *Driver) Start() error {
	return d.postAction("poweron")
}

// Stop stops the server
func (d *Driver) Stop() error {
	d.stopping = true
	return d.postAction("poweroff")
}
<commit_msg>Fix netstat loop<commit_after>package scaleway

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/docker/machine/libmachine/log"
	"github.com/docker/machine/libmachine/mcnflag"
	"github.com/docker/machine/libmachine/ssh"
	"github.com/docker/machine/libmachine/state"
	"github.com/scaleway/scaleway-cli/pkg/api"
	"github.com/scaleway/scaleway-cli/pkg/config"
)

const (
	// VERSION represents the semver version of the package
	VERSION           = "v1.0.2+dev"
	defaultImage      = "ubuntu-xenial"
	defaultBootscript = "docker"
)

var scwAPI *api.ScalewayAPI

// Driver represents the docker driver interface
type Driver struct {
	*drivers.BaseDriver
	ServerID       string
	Organization   string
	IPID           string
	Token          string
	CommercialType string
	name           string
	image          string
	stopping       bool
	created        bool
	// size         string
	// userDataFile string
	// ipv6         bool
}

// DriverName returns the name of the driver
func (d *Driver) DriverName() string {
	if d.CommercialType == "" {
		return "scaleway"
	}
	return fmt.Sprintf("scaleway(%v)", d.CommercialType)
}

func (d *Driver) getClient() (cl *api.ScalewayAPI, err error) {
	if scwAPI == nil {
		scwAPI, err = api.NewScalewayAPI(d.Organization, d.Token, "docker-machine-driver-scaleway/%v"+VERSION)
	}
	cl = scwAPI
	return
}

// SetConfigFromFlags sets the flags
func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) (err error) {
	if flags.Bool("scaleway-debug") {
		logrus.SetOutput(os.Stderr)
		logrus.SetLevel(logrus.DebugLevel)
	}

	d.Token, d.Organization = flags.String("scaleway-token"), flags.String("scaleway-organization")
	if d.Token == "" || d.Organization == "" {
		config, cfgErr := config.GetConfig()
		if cfgErr == nil {
			if d.Token == "" {
				d.Token = config.Token
			}
			if d.Organization == "" {
				d.Organization = config.Organization
			}
		} else {
			return fmt.Errorf("You must provide organization and token")
		}
	}
	d.CommercialType = flags.String("scaleway-commercial-type")
	d.name = flags.String("scaleway-name")
	d.image = flags.String("scaleway-image")
	return
}

// NewDriver returns a new driver
func NewDriver(hostName, storePath string) *Driver {
	return &Driver{
		BaseDriver: &drivers.BaseDriver{},
	}
}

// GetCreateFlags registers the flags
func (d *Driver) GetCreateFlags() []mcnflag.Flag {
	return []mcnflag.Flag{
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_TOKEN",
			Name:   "scaleway-token",
			Usage:  "Scaleway token",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_ORGANIZATION",
			Name:   "scaleway-organization",
			Usage:  "Scaleway organization",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_NAME",
			Name:   "scaleway-name",
			Usage:  "Assign a name",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_COMMERCIAL_TYPE",
			Name:   "scaleway-commercial-type",
			Usage:  "Specifies the commercial type",
			Value:  "VC1S",
		},
		mcnflag.StringFlag{
			EnvVar: "SCALEWAY_IMAGE",
			Name:   "scaleway-image",
			Usage:  "Specifies the image",
			Value:  defaultImage,
		},
		mcnflag.BoolFlag{
			EnvVar: "SCALEWAY_DEBUG",
			Name:   "scaleway-debug",
			Usage:  "Enables Scaleway client debugging",
		},
		// mcnflag.StringFlag{
		//     EnvVar: "SCALEWAY_USERDATA",
		//     Name:   "scaleway-userdata",
		//     Usage:  "Path to file with user-data",
		// },
		// mcnflag.BoolFlag{
		// 	EnvVar: "SCALEWAY_IPV6",
		// 	Name:   "scaleway-ipv6",
		// 	Usage:  "Enable ipv6",
		// },
	}
}

// Create configures and starts a scaleway server
func (d *Driver) Create() (err error) {
	var publicKey []byte
	var cl *api.ScalewayAPI
	var ip *api.ScalewayGetIP

	log.Infof("Creating SSH key...")
	if err = ssh.GenerateSSHKey(d.GetSSHKeyPath()); err != nil {
		return err
	}
	publicKey, err = ioutil.ReadFile(d.GetSSHKeyPath() + ".pub")
	if err != nil {
		return
	}
	log.Infof("Creating server...")
	cl, err = d.getClient()
	if err != nil {
		return
	}
	ip, err = cl.NewIP()
	if err != nil {
		return
	}
	d.IPAddress = ip.IP.Address
	d.IPID = ip.IP.ID
	d.ServerID, err = api.CreateServer(cl, &api.ConfigCreateServer{
		ImageName:      d.image,
		CommercialType: d.CommercialType,
		Name:           d.name,
		Bootscript:     defaultBootscript,
		IP:             ip.IP.ID,
		Env: strings.Join([]string{"AUTHORIZED_KEY",
			strings.Replace(string(publicKey[:len(publicKey)-1]), " ", "_", -1)}, "="),
	})
	if err != nil {
		return
	}
	log.Infof("Starting server...")
	err = api.StartServer(cl, d.ServerID, false)
	d.created = true
	return
}

// GetSSHHostname returns the IP of the server
func (d *Driver) GetSSHHostname() (string, error) {
	return d.IPAddress, nil
}

// GetState returns the state of the server
func (d *Driver) GetState() (st state.State, err error) {
	var server *api.ScalewayServer
	var cl *api.ScalewayAPI

	st = state.Error
	cl, err = d.getClient()
	if err != nil {
		return
	}
	server, err = cl.GetServer(d.ServerID)
	if err != nil {
		return
	}
	st = state.None
	switch server.State {
	case "starting":
		st = state.Starting
	case "running":
		if d.created {
			time.Sleep(60 * time.Second)
			d.created = false
		}
		st = state.Running
	case "stopping":
		st = state.Stopping
	case "stopped":
		st = state.Stopped
	}
	if d.stopping {
		time.Sleep(5 * time.Second)
	}
	return
}

// GetURL returns IP + docker port
func (d *Driver) GetURL() (string, error) {
	if err := drivers.MustBeRunning(d); err != nil {
		return "", err
	}
	return fmt.Sprintf("tcp://%s", net.JoinHostPort(d.IPAddress, "2376")), nil
}

func (d *Driver) postAction(action string) (err error) {
	var cl *api.ScalewayAPI

	cl, err = d.getClient()
	if err != nil {
		return
	}
	err = cl.PostServerAction(d.ServerID, action)
	return
}

// Kill does nothing
func (d *Driver) Kill() error {
	return errors.New("scaleway driver does not support kill")
}

// Remove shutdowns the server and removes the IP
func (d *Driver) Remove() (err error) {
	var cl *api.ScalewayAPI

	cl, err = d.getClient()
	if err != nil {
		return
	}
	err = cl.PostServerAction(d.ServerID, "terminate")
	if err != nil {
		return
	}
	for {
		_, err = cl.GetServer(d.ServerID)
		if err != nil {
			break
		}
	}
	err = cl.DeleteIP(d.IPID)
	return
}

// Restart reboots the server
func (d *Driver) Restart() error {
	return d.postAction("reboot")
}

// Start starts the server
func (d *Driver) Start() error {
	return d.postAction("poweron")
}

// Stop stops the server
func (d *Driver) Stop() error {
	d.stopping = true
	return d.postAction("poweroff")
}
<|endoftext|>