<commit_before>/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	exec "github.com/mesos/mesos-go/executor"
	mesos "github.com/mesos/mesos-go/mesosproto"
	log "github.com/sirupsen/logrus"

	"github.com/paypal/dce-go/config"
	"github.com/paypal/dce-go/dce/monitor"
	"github.com/paypal/dce-go/plugin"
	_ "github.com/paypal/dce-go/plugin/example"
	_ "github.com/paypal/dce-go/plugin/general"
	"github.com/paypal/dce-go/types"
	"github.com/paypal/dce-go/utils"
	fileUtils "github.com/paypal/dce-go/utils/file"
	"github.com/paypal/dce-go/utils/pod"
	"github.com/paypal/dce-go/utils/wait"
)

var logger *log.Entry
var extpoints []plugin.ComposePlugin

type dockerComposeExecutor struct {
	tasksLaunched int
}

func newDockerComposeExecutor() *dockerComposeExecutor {
	return &dockerComposeExecutor{tasksLaunched: 0}
}

func (exec *dockerComposeExecutor) Registered(driver exec.ExecutorDriver, execInfo *mesos.ExecutorInfo, fwinfo *mesos.FrameworkInfo, slaveInfo *mesos.SlaveInfo) {
	fmt.Println("====================Mesos Registered====================")
	fmt.Println("Mesos Register : Registered Executor on slave ", slaveInfo.GetHostname())
	pod.ComposeExcutorDriver = driver
}

func (exec *dockerComposeExecutor) Reregistered(driver exec.ExecutorDriver, slaveInfo *mesos.SlaveInfo) {
	fmt.Println("====================Mesos Reregistered====================")
	fmt.Println("Mesos Re-registered Re-registered : Executor on slave ", slaveInfo.GetHostname())
}

func (exec *dockerComposeExecutor) Disconnected(exec.ExecutorDriver) {
	fmt.Println("====================Mesos Disconnected====================")
	fmt.Println("Mesos Disconnected : Mesos Executordisconnected.")
}

func (exec *dockerComposeExecutor) LaunchTask(driver exec.ExecutorDriver, taskInfo *mesos.TaskInfo) {
	fmt.Println("====================Mesos LaunchTask====================")
	pod.ComposeExcutorDriver = driver
	log.Infof("Task DataByte: %v", taskInfo.Executor.Data)
	log.Infof("Task DataByte2: %v", taskInfo.Data)
	var taskData map[string]interface{}
	if err := json.Unmarshal(taskInfo.Executor.Data,&taskData); err != nil{
		log.Infof("Task Data: Can not read task data")
	}else{
		log.Infof("Task Data: %v", taskData)
	}
	task, err := json.Marshal(taskInfo)
	if err != nil {
		log.Println("Error marshalling taskInfo", err.Error())
	}
	buf := new(bytes.Buffer)
	json.Indent(buf, task, "", " ")
	fmt.Println("taskInfo : ", buf)

	isService := pod.IsService(taskInfo)
	fmt.Printf("task is service: %v\n", isService)
	config.GetConfig().Set(types.IS_SERVICE, isService)

	logger = log.WithFields(log.Fields{
		"requuid":   pod.GetLabel("requuid", taskInfo),
		"tenant":    pod.GetLabel("tenant", taskInfo),
		"namespace": pod.GetLabel("namespace", taskInfo),
		"pool":      pod.GetLabel("pool", taskInfo),
	})

	pod.ComposeTaskInfo = taskInfo
	executorId := taskInfo.GetExecutor().GetExecutorId().GetValue()

	// Update pod status to STARTING
	pod.SetPodStatus(types.POD_STARTING)

	// Update mesos state TO STARTING
	pod.SendMesosStatus(driver, taskInfo.GetTaskId(), mesos.TaskState_TASK_STARTING.Enum())

	// Get required compose file list
	pod.ComposeFiles, _ = fileUtils.GetFiles(taskInfo)

	// Generate app folder to keep temp files
	err = fileUtils.GenerateAppFolder()
	if err != nil {
		logger.Errorln("Error creating app folder")
	}

	// Create context with timeout
	// Wait for pod launching until timeout
	var ctx context.Context
	var cancel context.CancelFunc
	ctx = context.Background()
	ctx, cancel = context.WithTimeout(ctx, config.GetLaunchTimeout()*time.Millisecond)
	go pod.WaitOnPod(&ctx)

	// Get order of plugins from config or mesos labels
	pluginOrder, err := fileUtils.GetPluginOrder(taskInfo)
	if err != nil {
		logger.Println("Plugin order missing in mesos label, trying to get it from config")
		pluginOrder = strings.Split(config.GetConfigSection("plugins")[types.PLUGIN_ORDER], ",")
	}
	pod.PluginOrder = pluginOrder
	logger.Println("PluginOrder : ", pluginOrder)

	// Select plugin extension points from plugin pools
	extpoints = plugin.GetOrderedExtpoints(pluginOrder)

	// Executing PreLaunchTask in order
	_, err = utils.PluginPanicHandler(utils.ConditionFunc(func() (string, error) {
		for i, ext := range extpoints {
			if ext == nil {
				logger.Errorln("Error getting plugins from plugin registration pools")
				return "", errors.New("plugin is nil")
			}
			err = ext.PreLaunchTask(&ctx, &pod.ComposeFiles, executorId, taskInfo)
			if err != nil {
				logger.Errorf("Error executing PreLaunchTask of plugin : %v\n", err)
				return "", err
			}
			if config.EnableComposeTrace() {
				fileUtils.DumpPluginModifiedComposeFiles(ctx, pluginOrder[i], i)
			}
		}
		return "", err
	}))
	if err != nil {
		pod.SetPodStatus(types.POD_FAILED)
		cancel()
		pod.SendMesosStatus(driver, taskInfo.GetTaskId(), mesos.TaskState_TASK_FAILED.Enum())
		return
	}

	// Service list from all compose files
	podServices := getServices(ctx)
	logger.Printf("pod service list: %v", podServices)

	// Write updated compose files into pod folder
	err = fileUtils.WriteChangeToFiles(ctx)
	if err != nil {
		logger.Errorf("Failure writing updated compose files : %v", err)
		pod.SetPodStatus(types.POD_FAILED)
		cancel()
		pod.SendMesosStatus(driver, taskInfo.GetTaskId(), mesos.TaskState_TASK_FAILED.Enum())
	}

	// Pulling image and launch pod
	replyPodStatus := pullAndLaunchPod()

	logger.Printf("Pod status returned by pullAndLaunchPod : %v", replyPodStatus)

	// Take an action depends on different status
	switch replyPodStatus {
	case types.POD_FAILED:
		cancel()
		pod.SendPodStatus(types.POD_FAILED)

	case types.POD_PULL_FAILED:
		cancel()
		pod.SendPodStatus(types.POD_PULL_FAILED)

	case types.POD_STARTING:
		// Initial health check
		res, err := initHealthCheck(podServices)
		if err != nil || res == types.POD_FAILED {
			cancel()
			pod.SendPodStatus(types.POD_FAILED)
		}

		// Temp status keeps the pod status returned by PostLaunchTask
		tempStatus, err := utils.PluginPanicHandler(utils.ConditionFunc(func() (string, error) {
			var tempStatus string
			for _, ext := range extpoints {
				logger.Println("Executing post launch task plugin")

				tempStatus, err = ext.PostLaunchTask(&ctx, pod.ComposeFiles, taskInfo)
				if err != nil {
					logger.Errorf("Error executing PostLaunchTask : %v", err)
				}
				logger.Printf("Get pod status : %s returned by PostLaunchTask", tempStatus)

				if tempStatus == types.POD_FAILED {
					return tempStatus, nil
				}
			}
			return tempStatus, nil
		}))
		if err != nil {
			logger.Errorf("Error executing PostLaunchTask : %v", err)
		}
		if tempStatus == types.POD_FAILED {
			cancel()
			pod.SendPodStatus(types.POD_FAILED)
		}

		if res == types.POD_RUNNING {
			cancel()
			if pod.GetPodStatus() != types.POD_RUNNING {
				pod.SendPodStatus(types.POD_RUNNING)
				go monitor.MonitorPoller()
			}
		}

		//For adhoc job, send finished to mesos if job already finished during init health check
		if res == types.POD_FINISHED {
			cancel()
			pod.SendPodStatus(types.POD_FINISHED)
		}

	default:
		logger.Printf("default: Unknown status -- %s from pullAndLaunchPod ", replyPodStatus)

	}

	logger.Println("====================Mesos LaunchTask Returned====================")
}

func (exec *dockerComposeExecutor) KillTask(driver exec.ExecutorDriver, taskId *mesos.TaskID) {
	log.Println("====================Mesos KillTask====================")

	logKill := log.WithFields(log.Fields{
		"taskId": taskId,
	})

	status := pod.GetPodStatus()
	switch status {
	case types.POD_FAILED:
		logKill.Printf("Mesos Kill Task : Current task status is %s , ignore killTask", status)

	case types.POD_RUNNING:
		logKill.Printf("Mesos Kill Task : Current task status is %s , continue killTask", status)
		pod.SetPodStatus(types.POD_KILLED)

		err := pod.StopPod(pod.ComposeFiles)
		if err != nil {
			logKill.Errorf("Error cleaning up pod : %v", err.Error())
		}

		err = pod.SendMesosStatus(driver, taskId, mesos.TaskState_TASK_KILLED.Enum())
		if err != nil {
			logKill.Errorf("Error during kill Task : %v", err.Error())
		}

		log.Println("====================Stop ExecutorDriver====================")
		time.Sleep(200 * time.Millisecond)
		driver.Stop()

	}

	logKill.Println("====================Mesos KillTask Stopped====================")
}

func (exec *dockerComposeExecutor) FrameworkMessage(driver exec.ExecutorDriver, msg string) {
	fmt.Printf("Got framework message: %v\n", msg)
}

func (exec *dockerComposeExecutor) Shutdown(driver exec.ExecutorDriver) {
	// Execute shutdown plugin extensions in order
	for _, ext := range extpoints {
		ext.Shutdown(pod.ComposeExcutorDriver)
	}
	fmt.Println("====================Stop ExecutorDriver====================")
	driver.Stop()
}

func (exec *dockerComposeExecutor) Error(driver exec.ExecutorDriver, err string) {
	fmt.Printf("Got error message : %v\n", err)
}

func pullAndLaunchPod() string {
	logger.Println("====================Pod Pull And Launch====================")

	if !config.SkipPullImages() {
		err := wait.PollRetry(config.GetPullRetryCount(), time.Duration(config.GetPollInterval())*time.Millisecond, wait.ConditionFunc(func() (string, error) {
			return "", pod.PullImage(pod.ComposeFiles)
		}))

		if err != nil {
			logger.Printf("POD_IMAGE_PULL_FAILED -- %v", err)
			return types.POD_PULL_FAILED
		}
	}

	return pod.LaunchPod(pod.ComposeFiles)
}

func initHealthCheck(podServices map[string]bool) (string, error) {
	res, err := wait.WaitUntil(config.GetLaunchTimeout()*time.Millisecond, wait.ConditionCHFunc(func(healthCheckReply chan string) {
		pod.HealthCheck(pod.ComposeFiles, podServices, healthCheckReply)
	}))

	if err != nil {
		log.Printf("POD_INIT_HEALTH_CHECK_TIMEOUT -- %v", err)
		return types.POD_FAILED, err
	}
	return res, err
}

func getServices(ctx context.Context) map[string]bool {
	podService := make(map[string]bool)
	filesMap := ctx.Value(types.SERVICE_DETAIL).(types.ServiceDetail)

	for _, file := range pod.ComposeFiles {
		servMap := filesMap[file][types.SERVICES].(map[interface{}]interface{})

		for serviceName := range servMap {
			podService[serviceName.(string)] = true
		}
	}
	return podService
}

func init() {
	flag.Parse()
	log.SetOutput(os.Stdout)

	// Set log to debug level when trace mode is turned on
	if config.EnableDebugMode() {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.InfoLevel)
	}
}

func main() {
	fmt.Println("====================Genesis Executor (Go)====================")

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGUSR1)
	go func() {
		for {
			sig := <-sig
			fmt.Printf("Received signal %s\n", sig.String())
			if sig == syscall.SIGUSR1 {
				switchDebugMode()
			}
		}
	}()

	dConfig := exec.DriverConfig{
		Executor: newDockerComposeExecutor(),
	}

	driver, err := exec.NewMesosExecutorDriver(dConfig)
	if err != nil {
		fmt.Errorf("Unable to create a ExecutorDriver : %v\n", err.Error())
	}

	_, err = driver.Start()
	if err != nil {
		fmt.Errorf("Got error: %v\n", err.Error())
		return
	}

	fmt.Println("Executor : Executor process has started and running.")
	driver.Join()

}

func switchDebugMode() {
	if config.EnableDebugMode() {
		config.GetConfig().Set(config.DEBUG_MODE, false)
		log.Println("###Turn off debug mode###")
		log.SetLevel(log.InfoLevel)
	} else {
		config.GetConfig().Set(config.DEBUG_MODE, true)
		log.Println("###Turn on debug mode###")
		log.SetLevel(log.DebugLevel)
	}
}
<commit_msg>task data logs<commit_after>/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	exec "github.com/mesos/mesos-go/executor"
	mesos "github.com/mesos/mesos-go/mesosproto"
	log "github.com/sirupsen/logrus"

	"github.com/paypal/dce-go/config"
	"github.com/paypal/dce-go/dce/monitor"
	"github.com/paypal/dce-go/plugin"
	_ "github.com/paypal/dce-go/plugin/example"
	_ "github.com/paypal/dce-go/plugin/general"
	"github.com/paypal/dce-go/types"
	"github.com/paypal/dce-go/utils"
	fileUtils "github.com/paypal/dce-go/utils/file"
	"github.com/paypal/dce-go/utils/pod"
	"github.com/paypal/dce-go/utils/wait"
)

var logger *log.Entry
var extpoints []plugin.ComposePlugin

type dockerComposeExecutor struct {
	tasksLaunched int
}

func newDockerComposeExecutor() *dockerComposeExecutor {
	return &dockerComposeExecutor{tasksLaunched: 0}
}

func (exec *dockerComposeExecutor) Registered(driver exec.ExecutorDriver, execInfo *mesos.ExecutorInfo, fwinfo *mesos.FrameworkInfo, slaveInfo *mesos.SlaveInfo) {
	fmt.Println("====================Mesos Registered====================")
	fmt.Println("Mesos Register : Registered Executor on slave ", slaveInfo.GetHostname())
	pod.ComposeExcutorDriver = driver
}

func (exec *dockerComposeExecutor) Reregistered(driver exec.ExecutorDriver, slaveInfo *mesos.SlaveInfo) {
	fmt.Println("====================Mesos Reregistered====================")
	fmt.Println("Mesos Re-registered Re-registered : Executor on slave ", slaveInfo.GetHostname())
}

func (exec *dockerComposeExecutor) Disconnected(exec.ExecutorDriver) {
	fmt.Println("====================Mesos Disconnected====================")
	fmt.Println("Mesos Disconnected : Mesos Executordisconnected.")
}

func (exec *dockerComposeExecutor) LaunchTask(driver exec.ExecutorDriver, taskInfo *mesos.TaskInfo) {
	fmt.Println("====================Mesos LaunchTask====================")
	pod.ComposeExcutorDriver = driver

	log.Infof("Task DataByte: %v", taskInfo.Data)
	n := len(taskInfo.Data)
	log.Infof("Task Data: %v",string(taskInfo.Data[:n]))
	var taskData map[string]interface{}
	if err := json.Unmarshal(taskInfo.Data,&taskData); err != nil{
		log.Infof("Task Data: Can not read task data %v",err)
	}else{
		log.Infof("Task Data: %v", taskData)
	}
	task, err := json.Marshal(taskInfo)
	if err != nil {
		log.Println("Error marshalling taskInfo", err.Error())
	}
	buf := new(bytes.Buffer)
	json.Indent(buf, task, "", " ")
	fmt.Println("taskInfo : ", buf)

	isService := pod.IsService(taskInfo)
	fmt.Printf("task is service: %v\n", isService)
	config.GetConfig().Set(types.IS_SERVICE, isService)

	logger = log.WithFields(log.Fields{
		"requuid":   pod.GetLabel("requuid", taskInfo),
		"tenant":    pod.GetLabel("tenant", taskInfo),
		"namespace": pod.GetLabel("namespace", taskInfo),
		"pool":      pod.GetLabel("pool", taskInfo),
	})

	pod.ComposeTaskInfo = taskInfo
	executorId := taskInfo.GetExecutor().GetExecutorId().GetValue()

	// Update pod status to STARTING
	pod.SetPodStatus(types.POD_STARTING)

	// Update mesos state TO STARTING
	pod.SendMesosStatus(driver, taskInfo.GetTaskId(), mesos.TaskState_TASK_STARTING.Enum())

	// Get required compose file list
	pod.ComposeFiles, _ = fileUtils.GetFiles(taskInfo)

	// Generate app folder to keep temp files
	err = fileUtils.GenerateAppFolder()
	if err != nil {
		logger.Errorln("Error creating app folder")
	}

	// Create context with timeout
	// Wait for pod launching until timeout
	var ctx context.Context
	var cancel context.CancelFunc
	ctx = context.Background()
	ctx, cancel = context.WithTimeout(ctx, config.GetLaunchTimeout()*time.Millisecond)
	go pod.WaitOnPod(&ctx)

	// Get order of plugins from config or mesos labels
	pluginOrder, err := fileUtils.GetPluginOrder(taskInfo)
	if err != nil {
		logger.Println("Plugin order missing in mesos label, trying to get it from config")
		pluginOrder = strings.Split(config.GetConfigSection("plugins")[types.PLUGIN_ORDER], ",")
	}
	pod.PluginOrder = pluginOrder
	logger.Println("PluginOrder : ", pluginOrder)

	// Select plugin extension points from plugin pools
	extpoints = plugin.GetOrderedExtpoints(pluginOrder)

	// Executing PreLaunchTask in order
	_, err = utils.PluginPanicHandler(utils.ConditionFunc(func() (string, error) {
		for i, ext := range extpoints {
			if ext == nil {
				logger.Errorln("Error getting plugins from plugin registration pools")
				return "", errors.New("plugin is nil")
			}
			err = ext.PreLaunchTask(&ctx, &pod.ComposeFiles, executorId, taskInfo)
			if err != nil {
				logger.Errorf("Error executing PreLaunchTask of plugin : %v\n", err)
				return "", err
			}
			if config.EnableComposeTrace() {
				fileUtils.DumpPluginModifiedComposeFiles(ctx, pluginOrder[i], i)
			}
		}
		return "", err
	}))
	if err != nil {
		pod.SetPodStatus(types.POD_FAILED)
		cancel()
		pod.SendMesosStatus(driver, taskInfo.GetTaskId(), mesos.TaskState_TASK_FAILED.Enum())
		return
	}

	// Service list from all compose files
	podServices := getServices(ctx)
	logger.Printf("pod service list: %v", podServices)

	// Write updated compose files into pod folder
	err = fileUtils.WriteChangeToFiles(ctx)
	if err != nil {
		logger.Errorf("Failure writing updated compose files : %v", err)
		pod.SetPodStatus(types.POD_FAILED)
		cancel()
		pod.SendMesosStatus(driver, taskInfo.GetTaskId(), mesos.TaskState_TASK_FAILED.Enum())
	}

	// Pulling image and launch pod
	replyPodStatus := pullAndLaunchPod()

	logger.Printf("Pod status returned by pullAndLaunchPod : %v", replyPodStatus)

	// Take an action depends on different status
	switch replyPodStatus {
	case types.POD_FAILED:
		cancel()
		pod.SendPodStatus(types.POD_FAILED)

	case types.POD_PULL_FAILED:
		cancel()
		pod.SendPodStatus(types.POD_PULL_FAILED)

	case types.POD_STARTING:
		// Initial health check
		res, err := initHealthCheck(podServices)
		if err != nil || res == types.POD_FAILED {
			cancel()
			pod.SendPodStatus(types.POD_FAILED)
		}

		// Temp status keeps the pod status returned by PostLaunchTask
		tempStatus, err := utils.PluginPanicHandler(utils.ConditionFunc(func() (string, error) {
			var tempStatus string
			for _, ext := range extpoints {
				logger.Println("Executing post launch task plugin")

				tempStatus, err = ext.PostLaunchTask(&ctx, pod.ComposeFiles, taskInfo)
				if err != nil {
					logger.Errorf("Error executing PostLaunchTask : %v", err)
				}
				logger.Printf("Get pod status : %s returned by PostLaunchTask", tempStatus)

				if tempStatus == types.POD_FAILED {
					return tempStatus, nil
				}
			}
			return tempStatus, nil
		}))
		if err != nil {
			logger.Errorf("Error executing PostLaunchTask : %v", err)
		}
		if tempStatus == types.POD_FAILED {
			cancel()
			pod.SendPodStatus(types.POD_FAILED)
		}

		if res == types.POD_RUNNING {
			cancel()
			if pod.GetPodStatus() != types.POD_RUNNING {
				pod.SendPodStatus(types.POD_RUNNING)
				go monitor.MonitorPoller()
			}
		}

		//For adhoc job, send finished to mesos if job already finished during init health check
		if res == types.POD_FINISHED {
			cancel()
			pod.SendPodStatus(types.POD_FINISHED)
		}

	default:
		logger.Printf("default: Unknown status -- %s from pullAndLaunchPod ", replyPodStatus)

	}

	logger.Println("====================Mesos LaunchTask Returned====================")
}

func (exec *dockerComposeExecutor) KillTask(driver exec.ExecutorDriver, taskId *mesos.TaskID) {
	log.Println("====================Mesos KillTask====================")

	logKill := log.WithFields(log.Fields{
		"taskId": taskId,
	})

	status := pod.GetPodStatus()
	switch status {
	case types.POD_FAILED:
		logKill.Printf("Mesos Kill Task : Current task status is %s , ignore killTask", status)

	case types.POD_RUNNING:
		logKill.Printf("Mesos Kill Task : Current task status is %s , continue killTask", status)
		pod.SetPodStatus(types.POD_KILLED)

		err := pod.StopPod(pod.ComposeFiles)
		if err != nil {
			logKill.Errorf("Error cleaning up pod : %v", err.Error())
		}

		err = pod.SendMesosStatus(driver, taskId, mesos.TaskState_TASK_KILLED.Enum())
		if err != nil {
			logKill.Errorf("Error during kill Task : %v", err.Error())
		}

		log.Println("====================Stop ExecutorDriver====================")
		time.Sleep(200 * time.Millisecond)
		driver.Stop()

	}

	logKill.Println("====================Mesos KillTask Stopped====================")
}

func (exec *dockerComposeExecutor) FrameworkMessage(driver exec.ExecutorDriver, msg string) {
	fmt.Printf("Got framework message: %v\n", msg)
}

func (exec *dockerComposeExecutor) Shutdown(driver exec.ExecutorDriver) {
	// Execute shutdown plugin extensions in order
	for _, ext := range extpoints {
		ext.Shutdown(pod.ComposeExcutorDriver)
	}
	fmt.Println("====================Stop ExecutorDriver====================")
	driver.Stop()
}

func (exec *dockerComposeExecutor) Error(driver exec.ExecutorDriver, err string) {
	fmt.Printf("Got error message : %v\n", err)
}

func pullAndLaunchPod() string {
	logger.Println("====================Pod Pull And Launch====================")

	if !config.SkipPullImages() {
		err := wait.PollRetry(config.GetPullRetryCount(), time.Duration(config.GetPollInterval())*time.Millisecond, wait.ConditionFunc(func() (string, error) {
			return "", pod.PullImage(pod.ComposeFiles)
		}))

		if err != nil {
			logger.Printf("POD_IMAGE_PULL_FAILED -- %v", err)
			return types.POD_PULL_FAILED
		}
	}

	return pod.LaunchPod(pod.ComposeFiles)
}

func initHealthCheck(podServices map[string]bool) (string, error) {
	res, err := wait.WaitUntil(config.GetLaunchTimeout()*time.Millisecond, wait.ConditionCHFunc(func(healthCheckReply chan string) {
		pod.HealthCheck(pod.ComposeFiles, podServices, healthCheckReply)
	}))

	if err != nil {
		log.Printf("POD_INIT_HEALTH_CHECK_TIMEOUT -- %v", err)
		return types.POD_FAILED, err
	}
	return res, err
}

func getServices(ctx context.Context) map[string]bool {
	podService := make(map[string]bool)
	filesMap := ctx.Value(types.SERVICE_DETAIL).(types.ServiceDetail)

	for _, file := range pod.ComposeFiles {
		servMap := filesMap[file][types.SERVICES].(map[interface{}]interface{})

		for serviceName := range servMap {
			podService[serviceName.(string)] = true
		}
	}
	return podService
}

func init() {
	flag.Parse()
	log.SetOutput(os.Stdout)

	// Set log to debug level when trace mode is turned on
	if config.EnableDebugMode() {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.InfoLevel)
	}
}

func main() {
	fmt.Println("====================Genesis Executor (Go)====================")

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGUSR1)
	go func() {
		for {
			sig := <-sig
			fmt.Printf("Received signal %s\n", sig.String())
			if sig == syscall.SIGUSR1 {
				switchDebugMode()
			}
		}
	}()

	dConfig := exec.DriverConfig{
		Executor: newDockerComposeExecutor(),
	}

	driver, err := exec.NewMesosExecutorDriver(dConfig)
	if err != nil {
		fmt.Errorf("Unable to create a ExecutorDriver : %v\n", err.Error())
	}

	_, err = driver.Start()
	if err != nil {
		fmt.Errorf("Got error: %v\n", err.Error())
		return
	}

	fmt.Println("Executor : Executor process has started and running.")
	driver.Join()

}

func switchDebugMode() {
	if config.EnableDebugMode() {
		config.GetConfig().Set(config.DEBUG_MODE, false)
		log.Println("###Turn off debug mode###")
		log.SetLevel(log.InfoLevel)
	} else {
		config.GetConfig().Set(config.DEBUG_MODE, true)
		log.Println("###Turn on debug mode###")
		log.SetLevel(log.DebugLevel)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math"
)

func sqrt(x float64) float64 {
	if x == 0 {
		return 0
	}
	z := 1.0
	for i := 0; i < int(x); i++ {
		z = z - ((z*z - x) / (2 * z))
	}
	return z
}

func main() {
	fmt.Println(sqrt(2))
	fmt.Println(math.Sqrt(2))
}
<commit_msg>Last edit NewtonSqrt<commit_after>package main

import (
	"fmt"
	"math"
)

func newtonSqrt(x float64) float64 {
	// Square root of 0 is 0
	if x == 0 {
		return 0
	}

	// Starting point
	z := 1.0

	for i := 0; i < int(x); i++ {
		z = z - ((z*z - x) / (2 * z))
	}
	return z
} // newtonSqrt

func main() {
	// Debug
	// fmt.Println(sqrt(2))
	// fmt.Println(math.Sqrt(2))

	numbers := 20
	for i := 0; i < numbers; i++ {
		mathSqrt := math.Sqrt(float64(i))
		newton := newtonSqrt(float64(i))
		fmt.Println(i, "square root:")
		fmt.Println("  math.Sqrt:", mathSqrt)
		fmt.Println("  Newton's method:", newton)
		fmt.Println("  Difference:", math.Abs(mathSqrt-newton))
	}
} // main
<|endoftext|>


<commit_before>package driver

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
)

type OverlayDriver struct {
	OverlaysDir string
}

func (driver *OverlayDriver) CreateVolume(path string) error {
	layerDir := driver.layerDir(path)

	err := os.MkdirAll(layerDir, 0755)
	if err != nil {
		return err
	}

	err = os.Mkdir(path, 0755)
	if err != nil {
		return err
	}

	return syscall.Mount(path, layerDir, "", syscall.MS_BIND, "")
}

func (driver *OverlayDriver) DestroyVolume(path string) error {
	err := syscall.Unmount(path, 0)
	if err != nil {
		return err
	}

	err = os.RemoveAll(driver.workDir(path))
	if err != nil {
		return err
	}

	err = os.RemoveAll(driver.layerDir(path))
	if err != nil {
		return err
	}

	return os.RemoveAll(path)
}

func (driver *OverlayDriver) CreateCopyOnWriteLayer(path string, parent string) error {
	ancestry, err := driver.ancestry(parent)
	if err != nil {
		return err
	}

	childDir := driver.layerDir(path)
	workDir := driver.workDir(path)

	err = os.MkdirAll(childDir, 0755)
	if err != nil {
		return err
	}

	err = os.MkdirAll(workDir, 0755)
	if err != nil {
		return err
	}

	err = os.MkdirAll(path, 0755)
	if err != nil {
		return err
	}

	opts := fmt.Sprintf(
		"lowerdir=%s,upperdir=%s,workdir=%s",
		strings.Join(ancestry, ":"),
		childDir,
		workDir,
	)

	return syscall.Mount("overlay", path, "overlay", 0, opts)
}

func (driver *OverlayDriver) GetVolumeSizeInBytes(path string) (int64, error) {
	stdout := &bytes.Buffer{}
	cmd := exec.Command("du", driver.layerDir(path))
	cmd.Stdout = stdout

	err := cmd.Run()
	if err != nil {
		return 0, err
	}

	var size int64
	_, err = fmt.Sscanf(stdout.String(), "%d", &size)
	if err != nil {
		return 0, err
	}

	return size, nil
}

func (driver *OverlayDriver) layerDir(path string) string {
	return filepath.Join(driver.OverlaysDir, driver.pathId(path))
}

func (driver *OverlayDriver) workDir(path string) string {
	return filepath.Join(driver.OverlaysDir, "work", driver.pathId(path))
}

func (driver *OverlayDriver) ancestry(path string) ([]string, error) {
	ancestry := []string{}

	currentPath := path
	for {
		ancestry = append(ancestry, driver.layerDir(currentPath))

		parentVolume, err := os.Readlink(filepath.Join(filepath.Dir(currentPath), "parent"))
		if err != nil {
			if _, ok := err.(*os.PathError); ok {
				break
			}

			return nil, err
		}

		currentPath = filepath.Join(parentVolume, "volume")
	}

	return ancestry, nil
}

func (driver *OverlayDriver) pathId(path string) string {
	return filepath.Base(filepath.Dir(path))
}
<commit_msg>bind-mount empty volumes the other way around<commit_after>package driver

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
)

type OverlayDriver struct {
	OverlaysDir string
}

func (driver *OverlayDriver) CreateVolume(path string) error {
	layerDir := driver.layerDir(path)

	err := os.MkdirAll(layerDir, 0755)
	if err != nil {
		return err
	}

	err = os.Mkdir(path, 0755)
	if err != nil {
		return err
	}

	return syscall.Mount(layerDir, path, "", syscall.MS_BIND, "")
}

func (driver *OverlayDriver) DestroyVolume(path string) error {
	err := syscall.Unmount(path, 0)
	if err != nil {
		return err
	}

	err = os.RemoveAll(driver.workDir(path))
	if err != nil {
		return err
	}

	err = os.RemoveAll(driver.layerDir(path))
	if err != nil {
		return err
	}

	return os.RemoveAll(path)
}

func (driver *OverlayDriver) CreateCopyOnWriteLayer(path string, parent string) error {
	ancestry, err := driver.ancestry(parent)
	if err != nil {
		return err
	}

	childDir := driver.layerDir(path)
	workDir := driver.workDir(path)

	err = os.MkdirAll(childDir, 0755)
	if err != nil {
		return err
	}

	err = os.MkdirAll(workDir, 0755)
	if err != nil {
		return err
	}

	err = os.MkdirAll(path, 0755)
	if err != nil {
		return err
	}

	opts := fmt.Sprintf(
		"lowerdir=%s,upperdir=%s,workdir=%s",
		strings.Join(ancestry, ":"),
		childDir,
		workDir,
	)

	return syscall.Mount("overlay", path, "overlay", 0, opts)
}

func (driver *OverlayDriver) GetVolumeSizeInBytes(path string) (int64, error) {
	stdout := &bytes.Buffer{}
	cmd := exec.Command("du", driver.layerDir(path))
	cmd.Stdout = stdout

	err := cmd.Run()
	if err != nil {
		return 0, err
	}

	var size int64
	_, err = fmt.Sscanf(stdout.String(), "%d", &size)
	if err != nil {
		return 0, err
	}

	return size, nil
}

func (driver *OverlayDriver) layerDir(path string) string {
	return filepath.Join(driver.OverlaysDir, driver.pathId(path))
}

func (driver *OverlayDriver) workDir(path string) string {
	return filepath.Join(driver.OverlaysDir, "work", driver.pathId(path))
}

func (driver *OverlayDriver) ancestry(path string) ([]string, error) {
	ancestry := []string{}

	currentPath := path
	for {
		ancestry = append(ancestry, driver.layerDir(currentPath))

		parentVolume, err := os.Readlink(filepath.Join(filepath.Dir(currentPath), "parent"))
		if err != nil {
			if _, ok := err.(*os.PathError); ok {
				break
			}

			return nil, err
		}

		currentPath = filepath.Join(parentVolume, "volume")
	}

	return ancestry, nil
}

func (driver *OverlayDriver) pathId(path string) string {
	return filepath.Base(filepath.Dir(path))
}
<|endoftext|>


<commit_before>package brain

import (
	"encoding/json"
	"io"

	"github.com/BytemarkHosting/bytemark-client/lib/output"
	"github.com/BytemarkHosting/bytemark-client/lib/output/prettyprint"
)

// MigrationJobQueue is a list of disc IDs that are still to be migrated as
// part of a migration job.
type MigrationJobQueue struct {
	Discs []int `json:"discs,omitempty"`
}

// MigrationJobLocations represents source or target locations for a migration
// job. Discs, pools and tails maybe represented by ID number, label or UUID.
type MigrationJobLocations struct {
	Discs []json.Number `json:"discs,omitempty"`
	Pools []json.Number `json:"pools,omitempty"`
	Tails []json.Number `json:"tails,omitempty"`
}

// MigrationJobDestinations represents available desintations for a migration
// job. Unlike MigrationJobLocations, these are represented using ID number
// only.
type MigrationJobDestinations struct {
	Pools []int `json:"pools,omitempty"`
}

// MigrationJobOptions represents options on a migration job.
type MigrationJobOptions struct {
	Priority int `json:"priority,omitempty"`
}

// MigrationJobDiscStatus represents the current status of a migration job.
// Each entry is a list of disc IDs indicating the fate of discs that
// have been removed from the queue.
type MigrationJobDiscStatus struct {
	Done      []int `json:"done,omitempty"`
	Errored   []int `json:"errored,omitempty"`
	Cancelled []int `json:"cancelled,omitempty"`
	Skipped   []int `json:"skipped,omitempty"`
}

// MigrationJobStatus captures the status of a migration job, currently only
// discs.
type MigrationJobStatus struct {
	Discs MigrationJobDiscStatus `json:"discs,omitempty"`
}

// MigrationJobSpec is a specification of a migration job to be created
type MigrationJobSpec struct {
	Options      MigrationJobOptions   `json:"options,omitempty"`
	Sources      MigrationJobLocations `json:"sources,omitempty"`
	Destinations MigrationJobLocations `json:"destinations,omitempty"`
}

// MigrationJob is a representation of a migration job.
type MigrationJob struct {
	ID           int                      `json:"id,omitempty"`
	Args         MigrationJobSpec         `json:"args,omitempty"`
	Queue        MigrationJobQueue        `json:"queue,omitempty"`
	Destinations MigrationJobDestinations `json:"destinations,omitempty"`
	Status       MigrationJobStatus       `json:"status,omitempty"`
	Priority     int                      `json:"priority,omitempty"`
	StartedAt    string                   `json:"started_at,omitempty"`
	FinishedAt   string                   `json:"finished_at,omitempty"`
	CreatedAt    string                   `json:"created_at,omitempty"`
	UpdatedAt    string                   `json:"updated_at,omitempty"`
}

// DefaultFields returns the list of default fields to feed to github.com/BytemarkHosting/row.From for this type.
func (mj MigrationJob) DefaultFields(f output.Format) string {
	return "ID, Queue, Destinations, Status, Priority, StartedAt, FinishedAt, CreatedAt, UpdatedAt"
}

// PrettyPrint formats a MigrationJobQueue for display
func (mjq MigrationJobQueue) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
	const template = `{{ define "migration_job_queue_full" }}
{{- with .Discs }}
     discs:
{{- range . }}
       • {{.}}
{{- end }}
{{- end -}}
{{- end -}}{{- define "migration_job_queue_sgl" -}}{{ range .Discs }} {{.}}{{ end }}{{- end -}}`
	return prettyprint.Run(wr, template, "migration_job_queue"+string(detail), mjq)
}

// PrettyPrint formats a MigrationJobStatus for display
func (mjs MigrationJobStatus) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
	const template = `{{- define "migration_job_status_full" -}}
{{- with .Discs -}}
{{- with .Done }}
     done:
    {{- range . }}
       • {{.}}
    {{- end }}
{{- end -}}
{{- with .Errored }}
     errored:
    {{- range . }}
       • {{.}}
    {{- end }}
{{- end -}}
{{- with .Cancelled }}
     cancelled:
    {{- range . }}
       • {{.}}
    {{- end }}
{{- end -}}
{{- with .Skipped }}
     skipped:
    {{- range . }}
       • {{.}}
    {{- end }}
{{- end -}}
{{- end -}}
{{- end -}}`
	return prettyprint.Run(wr, template, "migration_job_status"+string(detail), mjs)
}

// PrettyPrint outputs a nice human-readable overview of the migration
func (mj MigrationJob) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
	const template = `{{ define "migration_job_full" }} ▸ {{ .ID }}
{{ with .Queue }}   queue: {{ prettysprint . "_full" }}
{{ end -}}{{- with .Status }}   status: {{ prettysprint . "_full" }}
{{ end -}}{{- with .Priority }}   priority: {{ . }}
{{ end -}}{{- with .StartedAt }}   started_at: {{ . }}
{{ end -}}{{- with .FinishedAt }}   finished_at: {{ . }}
{{ end -}}{{- with .CreatedAt }}   created_at: {{ . }}
{{ end -}}{{- with .UpdatedAt }}   updated_at: {{ . }}
{{ end -}}{{- end -}}{{- define "migration_job_medium" }} ▸ {{ .ID }} queue:{{ prettysprint .Queue "_sgl" }}
{{- end -}}{{- define "migration_job_sgl" -}}{{ template "migration_job_medium" . }}{{- end -}}`
	return prettyprint.Run(wr, template, "migration_job"+string(detail), mj)
}

// DefaultFields appeases quality tests
func (x MigrationJobDestinations) DefaultFields(f output.Format) string {
    return ""
}

// PrettyPrint appeases quality tests
func (x MigrationJobDestinations) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
    return nil
}

// DefaultFields appeases quality tests
func (x MigrationJobDiscStatus) DefaultFields(f output.Format) string {
    return ""
}

// PrettyPrint appeases quality tests
func (x MigrationJobDiscStatus) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
    return nil
}

// DefaultFields appeases quality tests
func (x MigrationJobLocations) DefaultFields(f output.Format) string {
    return ""
}

// PrettyPrint appeases quality tests
func (x MigrationJobLocations) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
    return nil
}

// DefaultFields appeases quality tests
func (x MigrationJobOptions) DefaultFields(f output.Format) string {
    return ""
}

// PrettyPrint appeases quality tests
func (x MigrationJobOptions) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
    return nil
}

// DefaultFields appeases quality tests
func (mjq MigrationJobQueue) DefaultFields(f output.Format) string {
    return ""
}

// DefaultFields appeases quality tests
func (x MigrationJobSpec) DefaultFields(f output.Format) string {
    return ""
}

// PrettyPrint appeases quality tests
func (x MigrationJobSpec) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
    return nil
}

// DefaultFields appeases quality tests
func (mjs MigrationJobStatus) DefaultFields(f output.Format) string {
    return ""
}
<commit_msg>use util.NumberOrString in MigrationJobLocations<commit_after>package brain

import (
	"io"

	"github.com/BytemarkHosting/bytemark-client/lib/output"
	"github.com/BytemarkHosting/bytemark-client/lib/output/prettyprint"
	"github.com/BytemarkHosting/bytemark-client/lib/util"
)

// MigrationJobQueue is a list of disc IDs that are still to be migrated as
// part of a migration job.
type MigrationJobQueue struct {
	Discs []int `json:"discs,omitempty"`
}

// MigrationJobLocations represents source or target locations for a migration
// job. Discs, pools and tails maybe represented by ID number, label or UUID.
type MigrationJobLocations struct {
	Discs []util.NumberOrString `json:"discs,omitempty"`
	Pools []util.NumberOrString `json:"pools,omitempty"`
	Tails []util.NumberOrString `json:"tails,omitempty"`
}

// MigrationJobDestinations represents available desintations for a migration
// job. Unlike MigrationJobLocations, these are represented using ID number
// only.
type MigrationJobDestinations struct {
	Pools []int `json:"pools,omitempty"`
}

// MigrationJobOptions represents options on a migration job.
type MigrationJobOptions struct {
	Priority int `json:"priority,omitempty"`
}

// MigrationJobDiscStatus represents the current status of a migration job.
// Each entry is a list of disc IDs indicating the fate of discs that
// have been removed from the queue.
type MigrationJobDiscStatus struct {
	Done      []int `json:"done,omitempty"`
	Errored   []int `json:"errored,omitempty"`
	Cancelled []int `json:"cancelled,omitempty"`
	Skipped   []int `json:"skipped,omitempty"`
}

// MigrationJobStatus captures the status of a migration job, currently only
// discs.
type MigrationJobStatus struct {
	Discs MigrationJobDiscStatus `json:"discs,omitempty"`
}

// MigrationJobSpec is a specification of a migration job to be created
type MigrationJobSpec struct {
	Options      MigrationJobOptions   `json:"options,omitempty"`
	Sources      MigrationJobLocations `json:"sources,omitempty"`
	Destinations MigrationJobLocations `json:"destinations,omitempty"`
}

// MigrationJob is a representation of a migration job.
type MigrationJob struct {
	ID           int                      `json:"id,omitempty"`
	Args         MigrationJobSpec         `json:"args,omitempty"`
	Queue        MigrationJobQueue        `json:"queue,omitempty"`
	Destinations MigrationJobDestinations `json:"destinations,omitempty"`
	Status       MigrationJobStatus       `json:"status,omitempty"`
	Priority     int                      `json:"priority,omitempty"`
	StartedAt    string                   `json:"started_at,omitempty"`
	FinishedAt   string                   `json:"finished_at,omitempty"`
	CreatedAt    string                   `json:"created_at,omitempty"`
	UpdatedAt    string                   `json:"updated_at,omitempty"`
}

// DefaultFields returns the list of default fields to feed to github.com/BytemarkHosting/row.From for this type.
func (mj MigrationJob) DefaultFields(f output.Format) string {
	return "ID, Queue, Destinations, Status, Priority, StartedAt, FinishedAt, CreatedAt, UpdatedAt"
}

// PrettyPrint formats a MigrationJobQueue for display
func (mjq MigrationJobQueue) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
	const template = `{{ define "migration_job_queue_full" }}
{{- with .Discs }}
     discs:
{{- range . }}
       • {{.}}
{{- end }}
{{- end -}}
{{- end -}}{{- define "migration_job_queue_sgl" -}}{{ range .Discs }} {{.}}{{ end }}{{- end -}}`
	return prettyprint.Run(wr, template, "migration_job_queue"+string(detail), mjq)
}

// PrettyPrint formats a MigrationJobStatus for display
func (mjs MigrationJobStatus) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
	const template = `{{- define "migration_job_status_full" -}}
{{- with .Discs -}}
{{- with .Done }}
     done:
    {{- range . }}
       • {{.}}
    {{- end }}
{{- end -}}
{{- with .Errored }}
     errored:
    {{- range . }}
       • {{.}}
    {{- end }}
{{- end -}}
{{- with .Cancelled }}
     cancelled:
    {{- range . }}
       • {{.}}
    {{- end }}
{{- end -}}
{{- with .Skipped }}
     skipped:
    {{- range . }}
       • {{.}}
    {{- end }}
{{- end -}}
{{- end -}}
{{- end -}}`
	return prettyprint.Run(wr, template, "migration_job_status"+string(detail), mjs)
}

// PrettyPrint outputs a nice human-readable overview of the migration
func (mj MigrationJob) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
	const template = `{{ define "migration_job_full" }} ▸ {{ .ID }}
{{ with .Queue }}   queue: {{ prettysprint . "_full" }}
{{ end -}}{{- with .Status }}   status: {{ prettysprint . "_full" }}
{{ end -}}{{- with .Priority }}   priority: {{ . }}
{{ end -}}{{- with .StartedAt }}   started_at: {{ . }}
{{ end -}}{{- with .FinishedAt }}   finished_at: {{ . }}
{{ end -}}{{- with .CreatedAt }}   created_at: {{ . }}
{{ end -}}{{- with .UpdatedAt }}   updated_at: {{ . }}
{{ end -}}{{- end -}}{{- define "migration_job_medium" }} ▸ {{ .ID }} queue:{{ prettysprint .Queue "_sgl" }}
{{- end -}}{{- define "migration_job_sgl" -}}{{ template "migration_job_medium" . }}{{- end -}}`
	return prettyprint.Run(wr, template, "migration_job"+string(detail), mj)
}

// DefaultFields appeases quality tests
func (x MigrationJobDestinations) DefaultFields(f output.Format) string {
	return ""
}

// PrettyPrint appeases quality tests
func (x MigrationJobDestinations) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
	return nil
}

// DefaultFields appeases quality tests
func (x MigrationJobDiscStatus) DefaultFields(f output.Format) string {
	return ""
}

// PrettyPrint appeases quality tests
func (x MigrationJobDiscStatus) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
	return nil
}

// DefaultFields appeases quality tests
func (x MigrationJobLocations) DefaultFields(f output.Format) string {
	return ""
}

// PrettyPrint appeases quality tests
func (x MigrationJobLocations) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
	return nil
}

// DefaultFields appeases quality tests
func (x MigrationJobOptions) DefaultFields(f output.Format) string {
	return ""
}

// PrettyPrint appeases quality tests
func (x MigrationJobOptions) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
	return nil
}

// DefaultFields appeases quality tests
func (mjq MigrationJobQueue) DefaultFields(f output.Format) string {
	return ""
}

// DefaultFields appeases quality tests
func (x MigrationJobSpec) DefaultFields(f output.Format) string {
	return ""
}

// PrettyPrint appeases quality tests
func (x MigrationJobSpec) PrettyPrint(wr io.Writer, detail prettyprint.DetailLevel) error {
	return nil
}

// DefaultFields appeases quality tests
func (mjs MigrationJobStatus) DefaultFields(f output.Format) string {
	return ""
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
	Package fmt implements formatted I/O with functions analogous
	to C's printf and scanf.  The format 'verbs' are derived from C's but
	are simpler.

	Printing

	The verbs:

	General:
		%v	the value in a default format.
			when printing structs, the plus flag (%+v) adds field names
		%#v	a Go-syntax representation of the value
		%T	a Go-syntax representation of the type of the value
		%%	a literal percent sign; consumes no value

	Boolean:
		%t	the word true or false
	Integer:
		%b	base 2
		%c	the character represented by the corresponding Unicode code point
		%d	base 10
		%o	base 8
		%q	a single-quoted character literal safely escaped with Go syntax.
		%x	base 16, with lower-case letters for a-f
		%X	base 16, with upper-case letters for A-F
		%U	Unicode format: U+1234; same as "U+%04X"
	Floating-point and complex constituents:
		%b	decimalless scientific notation with exponent a power of two,
			in the manner of strconv.FormatFloat with the 'b' format,
			e.g. -123456p-78
		%e	scientific notation, e.g. -1234.456e+78
		%E	scientific notation, e.g. -1234.456E+78
		%f	decimal point but no exponent, e.g. 123.456
		%F	synonym for %f
		%g	whichever of %e or %f produces more compact output
		%G	whichever of %E or %f produces more compact output
	String and slice of bytes:
		%s	the uninterpreted bytes of the string or slice
		%q	a double-quoted string safely escaped with Go syntax
		%x	base 16, lower-case, two characters per byte
		%X	base 16, upper-case, two characters per byte
	Pointer:
		%p	base 16 notation, with leading 0x

	There is no 'u' flag.  Integers are printed unsigned if they have unsigned type.
	Similarly, there is no need to specify the size of the operand (int8, int64).

	Width is specified by an optional decimal number immediately following the verb.
	If absent, the width is whatever is necessary to represent the value.
	Precision is specified after the (optional) width by a period followed by a
	decimal number. If no period is present, a default precision is used.
	A period with no following number specifies a precision of zero.
	Examples:
		%f:    default width, default precision
		%9f    width 9, default precision
		%.2f   default width, precision 2
		%9.2f  width 9, precision 2
		%9.f   width 9, precision 0

	Width and precision are measured in units of Unicode code points.
	(This differs from C's printf where the units are numbers
	of bytes.) Either or both of the flags may be replaced with the
	character '*', causing their values to be obtained from the next
	operand, which must be of type int.

	For most values, width is the minimum number of characters to output,
	padding the formatted form with spaces if necessary.
	For strings, precision is the maximum number of characters to output,
	truncating if necessary.

	For floating-point values, width sets the minimum width of the field and
	precision sets the number of places after the decimal, if appropriate,
	except that for %g/%G it sets the total number of digits. For example,
	given 123.45 the format %6.2f prints 123.45 while %.4g prints 123.5.
	The default precision for %e and %f is 6; for %g it is the smallest
	number of digits necessary to identify the value uniquely.

	For complex numbers, the width and precision apply to the two
	components independently and the result is parenthsized, so %f applied
	to 1.2+3.4i produces (1.200000+3.400000i).

	Other flags:
		+	always print a sign for numeric values;
			guarantee ASCII-only output for %q (%+q)
		-	pad with spaces on the right rather than the left (left-justify the field)
		#	alternate format: add leading 0 for octal (%#o), 0x for hex (%#x);
			0X for hex (%#X); suppress 0x for %p (%#p);
			for %q, print a raw (backquoted) string if strconv.CanBackquote
			returns true;
			write e.g. U+0078 'x' if the character is printable for %U (%#U).
		' '	(space) leave a space for elided sign in numbers (% d);
			put spaces between bytes printing strings or slices in hex (% x, % X)
		0	pad with leading zeros rather than spaces;
			for numbers, this moves the padding after the sign

	Flags are ignored by verbs that do not expect them.
	For example there is no alternate decimal format, so %#d and %d
	behave identically.

	For each Printf-like function, there is also a Print function
	that takes no format and is equivalent to saying %v for every
	operand.  Another variant Println inserts blanks between
	operands and appends a newline.

	Regardless of the verb, if an operand is an interface value,
	the internal concrete value is used, not the interface itself.
	Thus:
		var i interface{} = 23
		fmt.Printf("%v\n", i)
	will print 23.

	Except when printed using the the verbs %T and %p, special
	formatting considerations apply for operands that implement
	certain interfaces. In order of application:

	1. If an operand implements the Formatter interface, it will
	be invoked. Formatter provides fine control of formatting.

	2. If the %v verb is used with the # flag (%#v) and the operand
	implements the GoStringer interface, that will be invoked.

	If the format (which is implicitly %v for Println etc.) is valid
	for a string (%s %q %v %x %X), the following two rules apply:

	3. If an operand implements the error interface, the Error method
	will be invoked to convert the object to a string, which will then
	be formatted as required by the verb (if any).

	4. If an operand implements method String() string, that method
	will be invoked to convert the object to a string, which will then
	be formatted as required by the verb (if any).

	For compound operands such as slices and structs, the format
	applies to the elements of each operand, recursively, not to the
	operand as a whole. Thus %q will quote each element of a slice
	of strings, and %6.2f will control formatting for each element
	of a floating-point array.

	To avoid recursion in cases such as
		type X string
		func (x X) String() string { return Sprintf("<%s>", x) }
	convert the value before recurring:
		func (x X) String() string { return Sprintf("<%s>", string(x)) }

	Explicit argument indexes:

	In Printf, Sprintf, and Fprintf, the default behavior is for each
	formatting verb to format successive arguments passed in the call.
	However, the notation [n] immediately before the verb indicates that the
	nth one-indexed argument is to be formatted instead. The same notation
	before a '*' for a width or precision selects the argument index holding
	the value. After processing a bracketed expression [n], arguments n+1,
	n+2, etc. will be processed unless otherwise directed.

	For example,
		fmt.Sprintf("%[2]d %[1]d\n", 11, 22)
	will yield "22, 11", while
		fmt.Sprintf("%[3]*.[2]*[1]f", 12.0, 2, 6),
	equivalent to
		fmt.Sprintf("%6.2f", 12.0),
	will yield " 12.00". Because an explicit index affects subsequent verbs,
	this notation can be used to print the same values multiple times
	by resetting the index for the first argument to be repeated:
		fmt.Sprintf("%d %d %#[1]x %#x", 16, 17)
	will yield "16 17 0x10 0x11".

	Format errors:

	If an invalid argument is given for a verb, such as providing
	a string to %d, the generated string will contain a
	description of the problem, as in these examples:

		Wrong type or unknown verb: %!verb(type=value)
			Printf("%d", hi):          %!d(string=hi)
		Too many arguments: %!(EXTRA type=value)
			Printf("hi", "guys"):      hi%!(EXTRA string=guys)
		Too few arguments: %!verb(MISSING)
			Printf("hi%d"):            hi %!d(MISSING)
		Non-int for width or precision: %!(BADWIDTH) or %!(BADPREC)
			Printf("%*s", 4.5, "hi"):  %!(BADWIDTH)hi
			Printf("%.*s", 4.5, "hi"): %!(BADPREC)hi
		Invalid or invalid use of argument index: %!(BADINDEX)
			Printf("%*[2]d", 7):       %!d(BADINDEX)
			Printf("%.[2]d", 7):       %!d(BADINDEX)

	All errors begin with the string "%!" followed sometimes
	by a single character (the verb) and end with a parenthesized
	description.

	If an Error or String method triggers a panic when called by a
	print routine, the fmt package reformats the error message
	from the panic, decorating it with an indication that it came
	through the fmt package.  For example, if a String method
	calls panic("bad"), the resulting formatted message will look
	like
		%!s(PANIC=bad)

	The %!s just shows the print verb in use when the failure
	occurred.

	Scanning

	An analogous set of functions scans formatted text to yield
	values.  Scan, Scanf and Scanln read from os.Stdin; Fscan,
	Fscanf and Fscanln read from a specified io.Reader; Sscan,
	Sscanf and Sscanln read from an argument string.  Scanln,
	Fscanln and Sscanln stop scanning at a newline and require that
	the items be followed by one; Scanf, Fscanf and Sscanf require
	newlines in the input to match newlines in the format; the other
	routines treat newlines as spaces.

	Scanf, Fscanf, and Sscanf parse the arguments according to a
	format string, analogous to that of Printf.  For example, %x
	will scan an integer as a hexadecimal number, and %v will scan
	the default representation format for the value.

	The formats behave analogously to those of Printf with the
	following exceptions:

		%p is not implemented
		%T is not implemented
		%e %E %f %F %g %G are all equivalent and scan any floating point or complex value
		%s and %v on strings scan a space-delimited token
		Flags # and + are not implemented.

	The familiar base-setting prefixes 0 (octal) and 0x
	(hexadecimal) are accepted when scanning integers without a
	format or with the %v verb.

	Width is interpreted in the input text (%5s means at most
	five runes of input will be read to scan a string) but there
	is no syntax for scanning with a precision (no %5.2f, just
	%5f).

	When scanning with a format, all non-empty runs of space
	characters (except newline) are equivalent to a single
	space in both the format and the input.  With that proviso,
	text in the format string must match the input text; scanning
	stops if it does not, with the return value of the function
	indicating the number of arguments scanned.

	In all the scanning functions, a carriage return followed
	immediately by a newline is treated as a plain newline
	(\r\n means the same as \n).

	In all the scanning functions, if an operand implements method
	Scan (that is, it implements the Scanner interface) that
	method will be used to scan the text for that operand.  Also,
	if the number of arguments scanned is less than the number of
	arguments provided, an error is returned.

	All arguments to be scanned must be either pointers to basic
	types or implementations of the Scanner interface.

	Note: Fscan etc. can read one character (rune) past the input
	they return, which means that a loop calling a scan routine
	may skip some of the input.  This is usually a problem only
	when there is no space between input values.  If the reader
	provided to Fscan implements ReadRune, that method will be used
	to read characters.  If the reader also implements UnreadRune,
	that method will be used to save the character and successive
	calls will not lose data.  To attach ReadRune and UnreadRune
	methods to a reader without that capability, use
	bufio.NewReader.
*/
package fmt
<commit_msg>fmt: fix typo in help doc<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
	Package fmt implements formatted I/O with functions analogous
	to C's printf and scanf.  The format 'verbs' are derived from C's but
	are simpler.

	Printing

	The verbs:

	General:
		%v	the value in a default format.
			when printing structs, the plus flag (%+v) adds field names
		%#v	a Go-syntax representation of the value
		%T	a Go-syntax representation of the type of the value
		%%	a literal percent sign; consumes no value

	Boolean:
		%t	the word true or false
	Integer:
		%b	base 2
		%c	the character represented by the corresponding Unicode code point
		%d	base 10
		%o	base 8
		%q	a single-quoted character literal safely escaped with Go syntax.
		%x	base 16, with lower-case letters for a-f
		%X	base 16, with upper-case letters for A-F
		%U	Unicode format: U+1234; same as "U+%04X"
	Floating-point and complex constituents:
		%b	decimalless scientific notation with exponent a power of two,
			in the manner of strconv.FormatFloat with the 'b' format,
			e.g. -123456p-78
		%e	scientific notation, e.g. -1234.456e+78
		%E	scientific notation, e.g. -1234.456E+78
		%f	decimal point but no exponent, e.g. 123.456
		%F	synonym for %f
		%g	whichever of %e or %f produces more compact output
		%G	whichever of %E or %f produces more compact output
	String and slice of bytes:
		%s	the uninterpreted bytes of the string or slice
		%q	a double-quoted string safely escaped with Go syntax
		%x	base 16, lower-case, two characters per byte
		%X	base 16, upper-case, two characters per byte
	Pointer:
		%p	base 16 notation, with leading 0x

	There is no 'u' flag.  Integers are printed unsigned if they have unsigned type.
	Similarly, there is no need to specify the size of the operand (int8, int64).

	Width is specified by an optional decimal number immediately following the verb.
	If absent, the width is whatever is necessary to represent the value.
	Precision is specified after the (optional) width by a period followed by a
	decimal number. If no period is present, a default precision is used.
	A period with no following number specifies a precision of zero.
	Examples:
		%f:    default width, default precision
		%9f    width 9, default precision
		%.2f   default width, precision 2
		%9.2f  width 9, precision 2
		%9.f   width 9, precision 0

	Width and precision are measured in units of Unicode code points.
	(This differs from C's printf where the units are numbers
	of bytes.) Either or both of the flags may be replaced with the
	character '*', causing their values to be obtained from the next
	operand, which must be of type int.

	For most values, width is the minimum number of characters to output,
	padding the formatted form with spaces if necessary.
	For strings, precision is the maximum number of characters to output,
	truncating if necessary.

	For floating-point values, width sets the minimum width of the field and
	precision sets the number of places after the decimal, if appropriate,
	except that for %g/%G it sets the total number of digits. For example,
	given 123.45 the format %6.2f prints 123.45 while %.4g prints 123.5.
	The default precision for %e and %f is 6; for %g it is the smallest
	number of digits necessary to identify the value uniquely.

	For complex numbers, the width and precision apply to the two
	components independently and the result is parenthsized, so %f applied
	to 1.2+3.4i produces (1.200000+3.400000i).

	Other flags:
		+	always print a sign for numeric values;
			guarantee ASCII-only output for %q (%+q)
		-	pad with spaces on the right rather than the left (left-justify the field)
		#	alternate format: add leading 0 for octal (%#o), 0x for hex (%#x);
			0X for hex (%#X); suppress 0x for %p (%#p);
			for %q, print a raw (backquoted) string if strconv.CanBackquote
			returns true;
			write e.g. U+0078 'x' if the character is printable for %U (%#U).
		' '	(space) leave a space for elided sign in numbers (% d);
			put spaces between bytes printing strings or slices in hex (% x, % X)
		0	pad with leading zeros rather than spaces;
			for numbers, this moves the padding after the sign

	Flags are ignored by verbs that do not expect them.
	For example there is no alternate decimal format, so %#d and %d
	behave identically.

	For each Printf-like function, there is also a Print function
	that takes no format and is equivalent to saying %v for every
	operand.  Another variant Println inserts blanks between
	operands and appends a newline.

	Regardless of the verb, if an operand is an interface value,
	the internal concrete value is used, not the interface itself.
	Thus:
		var i interface{} = 23
		fmt.Printf("%v\n", i)
	will print 23.

	Except when printed using the verbs %T and %p, special
	formatting considerations apply for operands that implement
	certain interfaces. In order of application:

	1. If an operand implements the Formatter interface, it will
	be invoked. Formatter provides fine control of formatting.

	2. If the %v verb is used with the # flag (%#v) and the operand
	implements the GoStringer interface, that will be invoked.

	If the format (which is implicitly %v for Println etc.) is valid
	for a string (%s %q %v %x %X), the following two rules apply:

	3. If an operand implements the error interface, the Error method
	will be invoked to convert the object to a string, which will then
	be formatted as required by the verb (if any).

	4. If an operand implements method String() string, that method
	will be invoked to convert the object to a string, which will then
	be formatted as required by the verb (if any).

	For compound operands such as slices and structs, the format
	applies to the elements of each operand, recursively, not to the
	operand as a whole. Thus %q will quote each element of a slice
	of strings, and %6.2f will control formatting for each element
	of a floating-point array.

	To avoid recursion in cases such as
		type X string
		func (x X) String() string { return Sprintf("<%s>", x) }
	convert the value before recurring:
		func (x X) String() string { return Sprintf("<%s>", string(x)) }

	Explicit argument indexes:

	In Printf, Sprintf, and Fprintf, the default behavior is for each
	formatting verb to format successive arguments passed in the call.
	However, the notation [n] immediately before the verb indicates that the
	nth one-indexed argument is to be formatted instead. The same notation
	before a '*' for a width or precision selects the argument index holding
	the value. After processing a bracketed expression [n], arguments n+1,
	n+2, etc. will be processed unless otherwise directed.

	For example,
		fmt.Sprintf("%[2]d %[1]d\n", 11, 22)
	will yield "22, 11", while
		fmt.Sprintf("%[3]*.[2]*[1]f", 12.0, 2, 6),
	equivalent to
		fmt.Sprintf("%6.2f", 12.0),
	will yield " 12.00". Because an explicit index affects subsequent verbs,
	this notation can be used to print the same values multiple times
	by resetting the index for the first argument to be repeated:
		fmt.Sprintf("%d %d %#[1]x %#x", 16, 17)
	will yield "16 17 0x10 0x11".

	Format errors:

	If an invalid argument is given for a verb, such as providing
	a string to %d, the generated string will contain a
	description of the problem, as in these examples:

		Wrong type or unknown verb: %!verb(type=value)
			Printf("%d", hi):          %!d(string=hi)
		Too many arguments: %!(EXTRA type=value)
			Printf("hi", "guys"):      hi%!(EXTRA string=guys)
		Too few arguments: %!verb(MISSING)
			Printf("hi%d"):            hi %!d(MISSING)
		Non-int for width or precision: %!(BADWIDTH) or %!(BADPREC)
			Printf("%*s", 4.5, "hi"):  %!(BADWIDTH)hi
			Printf("%.*s", 4.5, "hi"): %!(BADPREC)hi
		Invalid or invalid use of argument index: %!(BADINDEX)
			Printf("%*[2]d", 7):       %!d(BADINDEX)
			Printf("%.[2]d", 7):       %!d(BADINDEX)

	All errors begin with the string "%!" followed sometimes
	by a single character (the verb) and end with a parenthesized
	description.

	If an Error or String method triggers a panic when called by a
	print routine, the fmt package reformats the error message
	from the panic, decorating it with an indication that it came
	through the fmt package.  For example, if a String method
	calls panic("bad"), the resulting formatted message will look
	like
		%!s(PANIC=bad)

	The %!s just shows the print verb in use when the failure
	occurred.

	Scanning

	An analogous set of functions scans formatted text to yield
	values.  Scan, Scanf and Scanln read from os.Stdin; Fscan,
	Fscanf and Fscanln read from a specified io.Reader; Sscan,
	Sscanf and Sscanln read from an argument string.  Scanln,
	Fscanln and Sscanln stop scanning at a newline and require that
	the items be followed by one; Scanf, Fscanf and Sscanf require
	newlines in the input to match newlines in the format; the other
	routines treat newlines as spaces.

	Scanf, Fscanf, and Sscanf parse the arguments according to a
	format string, analogous to that of Printf.  For example, %x
	will scan an integer as a hexadecimal number, and %v will scan
	the default representation format for the value.

	The formats behave analogously to those of Printf with the
	following exceptions:

		%p is not implemented
		%T is not implemented
		%e %E %f %F %g %G are all equivalent and scan any floating point or complex value
		%s and %v on strings scan a space-delimited token
		Flags # and + are not implemented.

	The familiar base-setting prefixes 0 (octal) and 0x
	(hexadecimal) are accepted when scanning integers without a
	format or with the %v verb.

	Width is interpreted in the input text (%5s means at most
	five runes of input will be read to scan a string) but there
	is no syntax for scanning with a precision (no %5.2f, just
	%5f).

	When scanning with a format, all non-empty runs of space
	characters (except newline) are equivalent to a single
	space in both the format and the input.  With that proviso,
	text in the format string must match the input text; scanning
	stops if it does not, with the return value of the function
	indicating the number of arguments scanned.

	In all the scanning functions, a carriage return followed
	immediately by a newline is treated as a plain newline
	(\r\n means the same as \n).

	In all the scanning functions, if an operand implements method
	Scan (that is, it implements the Scanner interface) that
	method will be used to scan the text for that operand.  Also,
	if the number of arguments scanned is less than the number of
	arguments provided, an error is returned.

	All arguments to be scanned must be either pointers to basic
	types or implementations of the Scanner interface.

	Note: Fscan etc. can read one character (rune) past the input
	they return, which means that a loop calling a scan routine
	may skip some of the input.  This is usually a problem only
	when there is no space between input values.  If the reader
	provided to Fscan implements ReadRune, that method will be used
	to read characters.  If the reader also implements UnreadRune,
	that method will be used to save the character and successive
	calls will not lose data.  To attach ReadRune and UnreadRune
	methods to a reader without that capability, use
	bufio.NewReader.
*/
package fmt
<|endoftext|>


<commit_before>package fzf

import (
	"bytes"
	"regexp"
	"strconv"
	"strings"
	"unicode/utf8"

	"github.com/junegunn/fzf/src/tui"
)

type ansiOffset struct {
	offset [2]int32
	color  ansiState
}

type ansiState struct {
	fg   tui.Color
	bg   tui.Color
	attr tui.Attr
	lbg  tui.Color
}

func (s *ansiState) colored() bool {
	return s.fg != -1 || s.bg != -1 || s.attr > 0 || s.lbg >= 0
}

func (s *ansiState) equals(t *ansiState) bool {
	if t == nil {
		return !s.colored()
	}
	return s.fg == t.fg && s.bg == t.bg && s.attr == t.attr && s.lbg == t.lbg
}

func (s *ansiState) ToString() string {
	if !s.colored() {
		return ""
	}

	ret := ""
	if s.attr&tui.Bold > 0 {
		ret += "1;"
	}
	if s.attr&tui.Dim > 0 {
		ret += "2;"
	}
	if s.attr&tui.Italic > 0 {
		ret += "3;"
	}
	if s.attr&tui.Underline > 0 {
		ret += "4;"
	}
	if s.attr&tui.Blink > 0 {
		ret += "5;"
	}
	if s.attr&tui.Reverse > 0 {
		ret += "7;"
	}
	ret += toAnsiString(s.fg, 30) + toAnsiString(s.bg, 40)

	return "\x1b[" + strings.TrimSuffix(ret, ";") + "m"
}

func toAnsiString(color tui.Color, offset int) string {
	col := int(color)
	ret := ""
	if col == -1 {
		ret += strconv.Itoa(offset + 9)
	} else if col < 8 {
		ret += strconv.Itoa(offset + col)
	} else if col < 16 {
		ret += strconv.Itoa(offset - 30 + 90 + col - 8)
	} else if col < 256 {
		ret += strconv.Itoa(offset+8) + ";5;" + strconv.Itoa(col)
	} else if col >= (1 << 24) {
		r := strconv.Itoa((col >> 16) & 0xff)
		g := strconv.Itoa((col >> 8) & 0xff)
		b := strconv.Itoa(col & 0xff)
		ret += strconv.Itoa(offset+8) + ";2;" + r + ";" + g + ";" + b
	}
	return ret + ";"
}

var ansiRegex *regexp.Regexp

func init() {
	/*
		References:
		- https://github.com/gnachman/iTerm2
		- http://ascii-table.com/ansi-escape-sequences.php
		- http://ascii-table.com/ansi-escape-sequences-vt-100.php
		- http://tldp.org/HOWTO/Bash-Prompt-HOWTO/x405.html
		- https://invisible-island.net/xterm/ctlseqs/ctlseqs.html
	*/
	// The following regular expression will include not all but most of the
	// frequently used ANSI sequences
	ansiRegex = regexp.MustCompile("(?:\x1b[\\[()][0-9;]*[a-zA-Z@]|\x1b][0-9];[[:print:]]+(?:\x1b\\\\|\x07)|\x1b.|[\x0e\x0f]|.\x08)")
}

func findAnsiStart(str string) int {
	idx := 0
	for ; idx < len(str); idx++ {
		b := str[idx]
		if b == 0x1b || b == 0x0e || b == 0x0f {
			return idx
		}
		if b == 0x08 && idx > 0 {
			return idx - 1
		}
	}
	return idx
}

func extractColor(str string, state *ansiState, proc func(string, *ansiState) bool) (string, *[]ansiOffset, *ansiState) {
	var offsets []ansiOffset
	var output bytes.Buffer

	if state != nil {
		offsets = append(offsets, ansiOffset{[2]int32{0, 0}, *state})
	}

	prevIdx := 0
	runeCount := 0
	for idx := 0; idx < len(str); {
		idx += findAnsiStart(str[idx:])
		if idx == len(str) {
			break
		}

		// Make sure that we found an ANSI code
		offset := ansiRegex.FindStringIndex(str[idx:])
		if len(offset) < 2 {
			idx++
			continue
		}
		offset[0] += idx
		offset[1] += idx
		idx = offset[1]

		// Check if we should continue
		prev := str[prevIdx:offset[0]]
		if proc != nil && !proc(prev, state) {
			return "", nil, nil
		}

		prevIdx = offset[1]
		runeCount += utf8.RuneCountInString(prev)
		output.WriteString(prev)

		newState := interpretCode(str[offset[0]:offset[1]], state)
		if !newState.equals(state) {
			if state != nil {
				// Update last offset
				(&offsets[len(offsets)-1]).offset[1] = int32(runeCount)
			}

			if newState.colored() {
				// Append new offset
				state = newState
				offsets = append(offsets, ansiOffset{[2]int32{int32(runeCount), int32(runeCount)}, *state})
			} else {
				// Discard state
				state = nil
			}
		}
	}

	var rest string
	var trimmed string

	if prevIdx == 0 {
		// No ANSI code found
		rest = str
		trimmed = str
	} else {
		rest = str[prevIdx:]
		output.WriteString(rest)
		trimmed = output.String()
	}
	if len(rest) > 0 && state != nil {
		// Update last offset
		runeCount += utf8.RuneCountInString(rest)
		(&offsets[len(offsets)-1]).offset[1] = int32(runeCount)
	}
	if proc != nil {
		proc(rest, state)
	}
	if len(offsets) == 0 {
		return trimmed, nil, state
	}
	return trimmed, &offsets, state
}

func interpretCode(ansiCode string, prevState *ansiState) *ansiState {
	// State
	var state *ansiState
	if prevState == nil {
		state = &ansiState{-1, -1, 0, -1}
	} else {
		state = &ansiState{prevState.fg, prevState.bg, prevState.attr, prevState.lbg}
	}
	if ansiCode[0] != '\x1b' || ansiCode[1] != '[' || ansiCode[len(ansiCode)-1] != 'm' {
		if strings.HasSuffix(ansiCode, "0K") {
			state.lbg = prevState.bg
		}
		return state
	}

	ptr := &state.fg
	state256 := 0

	init := func() {
		state.fg = -1
		state.bg = -1
		state.attr = 0
		state256 = 0
	}

	ansiCode = ansiCode[2 : len(ansiCode)-1]
	if len(ansiCode) == 0 {
		init()
	}
	for _, code := range strings.Split(ansiCode, ";") {
		if num, err := strconv.Atoi(code); err == nil {
			switch state256 {
			case 0:
				switch num {
				case 38:
					ptr = &state.fg
					state256++
				case 48:
					ptr = &state.bg
					state256++
				case 39:
					state.fg = -1
				case 49:
					state.bg = -1
				case 1:
					state.attr = state.attr | tui.Bold
				case 2:
					state.attr = state.attr | tui.Dim
				case 3:
					state.attr = state.attr | tui.Italic
				case 4:
					state.attr = state.attr | tui.Underline
				case 5:
					state.attr = state.attr | tui.Blink
				case 7:
					state.attr = state.attr | tui.Reverse
				case 23: // tput rmso
					state.attr = state.attr &^ tui.Italic
				case 24: // tput rmul
					state.attr = state.attr &^ tui.Underline
				case 0:
					init()
				default:
					if num >= 30 && num <= 37 {
						state.fg = tui.Color(num - 30)
					} else if num >= 40 && num <= 47 {
						state.bg = tui.Color(num - 40)
					} else if num >= 90 && num <= 97 {
						state.fg = tui.Color(num - 90 + 8)
					} else if num >= 100 && num <= 107 {
						state.bg = tui.Color(num - 100 + 8)
					}
				}
			case 1:
				switch num {
				case 2:
					state256 = 10 // MAGIC
				case 5:
					state256++
				default:
					state256 = 0
				}
			case 2:
				*ptr = tui.Color(num)
				state256 = 0
			case 10:
				*ptr = tui.Color(1<<24) | tui.Color(num<<16)
				state256++
			case 11:
				*ptr = *ptr | tui.Color(num<<8)
				state256++
			case 12:
				*ptr = *ptr | tui.Color(num)
				state256 = 0
			}
		}
	}
	if state256 > 0 {
		*ptr = -1
	}
	return state
}
<commit_msg>Fix segmentation fault on \x1b[0K<commit_after>package fzf

import (
	"bytes"
	"regexp"
	"strconv"
	"strings"
	"unicode/utf8"

	"github.com/junegunn/fzf/src/tui"
)

type ansiOffset struct {
	offset [2]int32
	color  ansiState
}

type ansiState struct {
	fg   tui.Color
	bg   tui.Color
	attr tui.Attr
	lbg  tui.Color
}

func (s *ansiState) colored() bool {
	return s.fg != -1 || s.bg != -1 || s.attr > 0 || s.lbg >= 0
}

func (s *ansiState) equals(t *ansiState) bool {
	if t == nil {
		return !s.colored()
	}
	return s.fg == t.fg && s.bg == t.bg && s.attr == t.attr && s.lbg == t.lbg
}

func (s *ansiState) ToString() string {
	if !s.colored() {
		return ""
	}

	ret := ""
	if s.attr&tui.Bold > 0 {
		ret += "1;"
	}
	if s.attr&tui.Dim > 0 {
		ret += "2;"
	}
	if s.attr&tui.Italic > 0 {
		ret += "3;"
	}
	if s.attr&tui.Underline > 0 {
		ret += "4;"
	}
	if s.attr&tui.Blink > 0 {
		ret += "5;"
	}
	if s.attr&tui.Reverse > 0 {
		ret += "7;"
	}
	ret += toAnsiString(s.fg, 30) + toAnsiString(s.bg, 40)

	return "\x1b[" + strings.TrimSuffix(ret, ";") + "m"
}

func toAnsiString(color tui.Color, offset int) string {
	col := int(color)
	ret := ""
	if col == -1 {
		ret += strconv.Itoa(offset + 9)
	} else if col < 8 {
		ret += strconv.Itoa(offset + col)
	} else if col < 16 {
		ret += strconv.Itoa(offset - 30 + 90 + col - 8)
	} else if col < 256 {
		ret += strconv.Itoa(offset+8) + ";5;" + strconv.Itoa(col)
	} else if col >= (1 << 24) {
		r := strconv.Itoa((col >> 16) & 0xff)
		g := strconv.Itoa((col >> 8) & 0xff)
		b := strconv.Itoa(col & 0xff)
		ret += strconv.Itoa(offset+8) + ";2;" + r + ";" + g + ";" + b
	}
	return ret + ";"
}

var ansiRegex *regexp.Regexp

func init() {
	/*
		References:
		- https://github.com/gnachman/iTerm2
		- http://ascii-table.com/ansi-escape-sequences.php
		- http://ascii-table.com/ansi-escape-sequences-vt-100.php
		- http://tldp.org/HOWTO/Bash-Prompt-HOWTO/x405.html
		- https://invisible-island.net/xterm/ctlseqs/ctlseqs.html
	*/
	// The following regular expression will include not all but most of the
	// frequently used ANSI sequences
	ansiRegex = regexp.MustCompile("(?:\x1b[\\[()][0-9;]*[a-zA-Z@]|\x1b][0-9];[[:print:]]+(?:\x1b\\\\|\x07)|\x1b.|[\x0e\x0f]|.\x08)")
}

func findAnsiStart(str string) int {
	idx := 0
	for ; idx < len(str); idx++ {
		b := str[idx]
		if b == 0x1b || b == 0x0e || b == 0x0f {
			return idx
		}
		if b == 0x08 && idx > 0 {
			return idx - 1
		}
	}
	return idx
}

func extractColor(str string, state *ansiState, proc func(string, *ansiState) bool) (string, *[]ansiOffset, *ansiState) {
	var offsets []ansiOffset
	var output bytes.Buffer

	if state != nil {
		offsets = append(offsets, ansiOffset{[2]int32{0, 0}, *state})
	}

	prevIdx := 0
	runeCount := 0
	for idx := 0; idx < len(str); {
		idx += findAnsiStart(str[idx:])
		if idx == len(str) {
			break
		}

		// Make sure that we found an ANSI code
		offset := ansiRegex.FindStringIndex(str[idx:])
		if len(offset) < 2 {
			idx++
			continue
		}
		offset[0] += idx
		offset[1] += idx
		idx = offset[1]

		// Check if we should continue
		prev := str[prevIdx:offset[0]]
		if proc != nil && !proc(prev, state) {
			return "", nil, nil
		}

		prevIdx = offset[1]
		runeCount += utf8.RuneCountInString(prev)
		output.WriteString(prev)

		newState := interpretCode(str[offset[0]:offset[1]], state)
		if !newState.equals(state) {
			if state != nil {
				// Update last offset
				(&offsets[len(offsets)-1]).offset[1] = int32(runeCount)
			}

			if newState.colored() {
				// Append new offset
				state = newState
				offsets = append(offsets, ansiOffset{[2]int32{int32(runeCount), int32(runeCount)}, *state})
			} else {
				// Discard state
				state = nil
			}
		}
	}

	var rest string
	var trimmed string

	if prevIdx == 0 {
		// No ANSI code found
		rest = str
		trimmed = str
	} else {
		rest = str[prevIdx:]
		output.WriteString(rest)
		trimmed = output.String()
	}
	if len(rest) > 0 && state != nil {
		// Update last offset
		runeCount += utf8.RuneCountInString(rest)
		(&offsets[len(offsets)-1]).offset[1] = int32(runeCount)
	}
	if proc != nil {
		proc(rest, state)
	}
	if len(offsets) == 0 {
		return trimmed, nil, state
	}
	return trimmed, &offsets, state
}

func interpretCode(ansiCode string, prevState *ansiState) *ansiState {
	// State
	var state *ansiState
	if prevState == nil {
		state = &ansiState{-1, -1, 0, -1}
	} else {
		state = &ansiState{prevState.fg, prevState.bg, prevState.attr, prevState.lbg}
	}
	if ansiCode[0] != '\x1b' || ansiCode[1] != '[' || ansiCode[len(ansiCode)-1] != 'm' {
		if strings.HasSuffix(ansiCode, "0K") && prevState != nil {
			state.lbg = prevState.bg
		}
		return state
	}

	ptr := &state.fg
	state256 := 0

	init := func() {
		state.fg = -1
		state.bg = -1
		state.attr = 0
		state256 = 0
	}

	ansiCode = ansiCode[2 : len(ansiCode)-1]
	if len(ansiCode) == 0 {
		init()
	}
	for _, code := range strings.Split(ansiCode, ";") {
		if num, err := strconv.Atoi(code); err == nil {
			switch state256 {
			case 0:
				switch num {
				case 38:
					ptr = &state.fg
					state256++
				case 48:
					ptr = &state.bg
					state256++
				case 39:
					state.fg = -1
				case 49:
					state.bg = -1
				case 1:
					state.attr = state.attr | tui.Bold
				case 2:
					state.attr = state.attr | tui.Dim
				case 3:
					state.attr = state.attr | tui.Italic
				case 4:
					state.attr = state.attr | tui.Underline
				case 5:
					state.attr = state.attr | tui.Blink
				case 7:
					state.attr = state.attr | tui.Reverse
				case 23: // tput rmso
					state.attr = state.attr &^ tui.Italic
				case 24: // tput rmul
					state.attr = state.attr &^ tui.Underline
				case 0:
					init()
				default:
					if num >= 30 && num <= 37 {
						state.fg = tui.Color(num - 30)
					} else if num >= 40 && num <= 47 {
						state.bg = tui.Color(num - 40)
					} else if num >= 90 && num <= 97 {
						state.fg = tui.Color(num - 90 + 8)
					} else if num >= 100 && num <= 107 {
						state.bg = tui.Color(num - 100 + 8)
					}
				}
			case 1:
				switch num {
				case 2:
					state256 = 10 // MAGIC
				case 5:
					state256++
				default:
					state256 = 0
				}
			case 2:
				*ptr = tui.Color(num)
				state256 = 0
			case 10:
				*ptr = tui.Color(1<<24) | tui.Color(num<<16)
				state256++
			case 11:
				*ptr = *ptr | tui.Color(num<<8)
				state256++
			case 12:
				*ptr = *ptr | tui.Color(num)
				state256 = 0
			}
		}
	}
	if state256 > 0 {
		*ptr = -1
	}
	return state
}
<|endoftext|>


<commit_before>package blocktoattr

import (
	"github.com/hashicorp/hcl2/ext/dynblock"
	"github.com/hashicorp/hcl2/hcl"
	"github.com/hashicorp/hcl2/hcldec"
	"github.com/hashicorp/terraform/configs/configschema"
)

// ExpandedVariables finds all of the global variables referenced in the
// given body with the given schema while taking into account the possibilities
// both of "dynamic" blocks being expanded and the possibility of certain
// attributes being written instead as nested blocks as allowed by the
// FixUpBlockAttrs function.
//
// This function exists to allow variables to be analyzed prior to dynamic
// block expansion while also dealing with the fact that dynamic block expansion
// might in turn produce nested blocks that are subject to FixUpBlockAttrs.
//
// This is intended as a drop-in replacement for dynblock.VariablesHCLDec,
// which is itself a drop-in replacement for hcldec.Variables.
func ExpandedVariables(body hcl.Body, schema *configschema.Block) []hcl.Traversal {
	rootNode := dynblock.WalkVariables(body)
	return walkVariables(rootNode, body, schema)
}

func walkVariables(node dynblock.WalkVariablesNode, body hcl.Body, schema *configschema.Block) []hcl.Traversal {
	givenRawSchema := hcldec.ImpliedSchema(schema.DecoderSpec())
	ambiguousNames := ambiguousNames(schema)
	effectiveRawSchema := effectiveSchema(givenRawSchema, body, ambiguousNames, false)
	vars, children := node.Visit(effectiveRawSchema)

	for _, child := range children {
		if blockS, exists := schema.BlockTypes[child.BlockTypeName]; exists {
			vars = append(vars, walkVariables(child.Node, child.Body(), &blockS.Block)...)
		} else if attrS, exists := schema.Attributes[child.BlockTypeName]; exists && attrS.Type.ElementType().IsObjectType() {
			synthSchema := SchemaForCtyElementType(attrS.Type.ElementType())
			vars = append(vars, walkVariables(child.Node, child.Body(), synthSchema)...)
		}
	}

	return vars
}
<commit_msg>Fixes issue where attribute in nested is not of list type/is invalid and would panic<commit_after>package blocktoattr

import (
	"github.com/hashicorp/hcl2/ext/dynblock"
	"github.com/hashicorp/hcl2/hcl"
	"github.com/hashicorp/hcl2/hcldec"
	"github.com/hashicorp/terraform/configs/configschema"
)

// ExpandedVariables finds all of the global variables referenced in the
// given body with the given schema while taking into account the possibilities
// both of "dynamic" blocks being expanded and the possibility of certain
// attributes being written instead as nested blocks as allowed by the
// FixUpBlockAttrs function.
//
// This function exists to allow variables to be analyzed prior to dynamic
// block expansion while also dealing with the fact that dynamic block expansion
// might in turn produce nested blocks that are subject to FixUpBlockAttrs.
//
// This is intended as a drop-in replacement for dynblock.VariablesHCLDec,
// which is itself a drop-in replacement for hcldec.Variables.
func ExpandedVariables(body hcl.Body, schema *configschema.Block) []hcl.Traversal {
	rootNode := dynblock.WalkVariables(body)
	return walkVariables(rootNode, body, schema)
}

func walkVariables(node dynblock.WalkVariablesNode, body hcl.Body, schema *configschema.Block) []hcl.Traversal {
	givenRawSchema := hcldec.ImpliedSchema(schema.DecoderSpec())
	ambiguousNames := ambiguousNames(schema)
	effectiveRawSchema := effectiveSchema(givenRawSchema, body, ambiguousNames, false)
	vars, children := node.Visit(effectiveRawSchema)

	for _, child := range children {
		if blockS, exists := schema.BlockTypes[child.BlockTypeName]; exists {
			vars = append(vars, walkVariables(child.Node, child.Body(), &blockS.Block)...)
		} else if attrS, exists := schema.Attributes[child.BlockTypeName]; exists && attrS.Type.IsCollectionType() && attrS.Type.ElementType().IsObjectType() {
			synthSchema := SchemaForCtyElementType(attrS.Type.ElementType())
			vars = append(vars, walkVariables(child.Node, child.Body(), synthSchema)...)
		}
	}

	return vars
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2016 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package probes

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/lebauce/dockerclient"

	"github.com/redhat-cip/skydive/config"
	"github.com/redhat-cip/skydive/logging"
	"github.com/redhat-cip/skydive/topology/graph"
)

type DockerProbe struct {
	NetNSProbe
	url     string
	client  *dockerclient.DockerClient
	nsProbe *NetNSProbe
	running atomic.Value
	quit    chan bool
	wg      sync.WaitGroup
}

type DockerContainerAttributes struct {
	ContainerID string
}

func (probe *DockerProbe) containerNamespace(info dockerclient.ContainerInfo) string {
	return fmt.Sprintf("/proc/%d/ns/net", info.State.Pid)
}

func (probe *DockerProbe) registerContainer(info dockerclient.ContainerInfo) {
	namespace := probe.containerNamespace(info)
	logging.GetLogger().Debugf("Register docker container %s and PID %d", info.Id, info.State.Pid)
	metadata := &graph.Metadata{
		"Docker.ContainerID":   info.Id,
		"Docker.ContainerName": info.Name,
	}
	probe.nsProbe.Register(namespace, metadata)
}

func (probe *DockerProbe) unregisterContainer(info dockerclient.ContainerInfo) {
	namespace := probe.containerNamespace(info)
	logging.GetLogger().Debugf("Stop listening for namespace %s with PID %d", namespace, info.State.Pid)
	probe.nsProbe.Unregister(namespace)
}

func (probe *DockerProbe) handleDockerEvent(event *dockerclient.Event) {
	info, err := probe.client.InspectContainer(event.ID)
	if err != nil {
		return
	}

	if event.Status == "start" {
		probe.registerContainer(*info)
	} else if event.Status == "die" {
		probe.unregisterContainer(*info)
	}
}

func (probe *DockerProbe) connect() {
	var err error

	logging.GetLogger().Debugf("Connecting to Docker daemon: %s", probe.url)
	probe.client, err = dockerclient.NewDockerClient(probe.url, nil)
	if err != nil {
		logging.GetLogger().Errorf("Failed to connect to Docker daemon: %s", err.Error())
		return
	}

	eventsOptions := &dockerclient.MonitorEventsOptions{
		Filters: &dockerclient.MonitorEventsFilters{
			Events: []string{"start", "die"},
		},
	}

	probe.quit = make(chan bool)
	eventErrChan, err := probe.client.MonitorEvents(eventsOptions, nil)
	if err != nil {
		logging.GetLogger().Errorf("Unable to monitor Docker events: %s", err.Error())
		return
	}

	containers, err := probe.client.ListContainers(false, false, "")
	if err != nil {
		logging.GetLogger().Errorf("Failed to list containers: %s", err.Error())
		return
	}

	probe.wg.Add(2)

	go func() {
		defer probe.wg.Done()

		for _, c := range containers {
			if probe.running.Load() == false {
				break
			}

			info, err := probe.client.InspectContainer(c.Id)
			if err != nil {
				logging.GetLogger().Errorf("Failed to inspect container %s: %s", c.Id, err.Error())
				continue
			}

			probe.registerContainer(*info)
		}
	}()

	defer probe.wg.Done()
	for {
		select {
		case <-probe.quit:
			return
		case e := <-eventErrChan:
			if e.Error != nil {
				logging.GetLogger().Errorf("Got error while waiting for Docker event: %s", e.Error.Error())
				return
			}
			probe.handleDockerEvent(&e.Event)
		}
	}
}

func (probe *DockerProbe) Start() {
	probe.running.Store(true)

	go func() {
		for probe.running.Load() == true {
			probe.connect()

			time.Sleep(1 * time.Second)
		}
	}()
}

func (probe *DockerProbe) Stop() {
	probe.running.Store(false)
	close(probe.quit)
	probe.wg.Wait()
}

func NewDockerProbe(g *graph.Graph, n *graph.Node, dockerURL string) *DockerProbe {
	return &DockerProbe{
		NetNSProbe: *NewNetNSProbe(g, n),
		url:        dockerURL,
	}
}

func NewDockerProbeFromConfig(g *graph.Graph, n *graph.Node) *DockerProbe {
	dockerURL := config.GetConfig().GetString("docker.url")
	return NewDockerProbe(g, n, dockerURL)
}
<commit_msg>Remove unused namespace probe attribute in Docker probe<commit_after>/*
 * Copyright (C) 2016 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package probes

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/lebauce/dockerclient"

	"github.com/redhat-cip/skydive/config"
	"github.com/redhat-cip/skydive/logging"
	"github.com/redhat-cip/skydive/topology/graph"
)

type DockerProbe struct {
	NetNSProbe
	url     string
	client  *dockerclient.DockerClient
	running atomic.Value
	quit    chan bool
	wg      sync.WaitGroup
}

type DockerContainerAttributes struct {
	ContainerID string
}

func (probe *DockerProbe) containerNamespace(info dockerclient.ContainerInfo) string {
	return fmt.Sprintf("/proc/%d/ns/net", info.State.Pid)
}

func (probe *DockerProbe) registerContainer(info dockerclient.ContainerInfo) {
	namespace := probe.containerNamespace(info)
	logging.GetLogger().Debugf("Register docker container %s and PID %d", info.Id, info.State.Pid)
	metadata := &graph.Metadata{
		"Docker.ContainerID":   info.Id,
		"Docker.ContainerName": info.Name,
	}
	probe.Register(namespace, metadata)
}

func (probe *DockerProbe) unregisterContainer(info dockerclient.ContainerInfo) {
	namespace := probe.containerNamespace(info)
	logging.GetLogger().Debugf("Stop listening for namespace %s with PID %d", namespace, info.State.Pid)
	probe.Unregister(namespace)
}

func (probe *DockerProbe) handleDockerEvent(event *dockerclient.Event) {
	info, err := probe.client.InspectContainer(event.ID)
	if err != nil {
		return
	}

	if event.Status == "start" {
		probe.registerContainer(*info)
	} else if event.Status == "die" {
		probe.unregisterContainer(*info)
	}
}

func (probe *DockerProbe) connect() {
	var err error

	logging.GetLogger().Debugf("Connecting to Docker daemon: %s", probe.url)
	probe.client, err = dockerclient.NewDockerClient(probe.url, nil)
	if err != nil {
		logging.GetLogger().Errorf("Failed to connect to Docker daemon: %s", err.Error())
		return
	}

	eventsOptions := &dockerclient.MonitorEventsOptions{
		Filters: &dockerclient.MonitorEventsFilters{
			Events: []string{"start", "die"},
		},
	}

	probe.quit = make(chan bool)
	eventErrChan, err := probe.client.MonitorEvents(eventsOptions, nil)
	if err != nil {
		logging.GetLogger().Errorf("Unable to monitor Docker events: %s", err.Error())
		return
	}

	containers, err := probe.client.ListContainers(false, false, "")
	if err != nil {
		logging.GetLogger().Errorf("Failed to list containers: %s", err.Error())
		return
	}

	probe.wg.Add(2)

	go func() {
		defer probe.wg.Done()

		for _, c := range containers {
			if probe.running.Load() == false {
				break
			}

			info, err := probe.client.InspectContainer(c.Id)
			if err != nil {
				logging.GetLogger().Errorf("Failed to inspect container %s: %s", c.Id, err.Error())
				continue
			}

			probe.registerContainer(*info)
		}
	}()

	defer probe.wg.Done()
	for {
		select {
		case <-probe.quit:
			return
		case e := <-eventErrChan:
			if e.Error != nil {
				logging.GetLogger().Errorf("Got error while waiting for Docker event: %s", e.Error.Error())
				return
			}
			probe.handleDockerEvent(&e.Event)
		}
	}
}

func (probe *DockerProbe) Start() {
	probe.running.Store(true)

	go func() {
		for probe.running.Load() == true {
			probe.connect()

			time.Sleep(1 * time.Second)
		}
	}()
}

func (probe *DockerProbe) Stop() {
	probe.running.Store(false)
	close(probe.quit)
	probe.wg.Wait()
}

func NewDockerProbe(g *graph.Graph, n *graph.Node, dockerURL string) *DockerProbe {
	return &DockerProbe{
		NetNSProbe: *NewNetNSProbe(g, n),
		url:        dockerURL,
	}
}

func NewDockerProbeFromConfig(g *graph.Graph, n *graph.Node) *DockerProbe {
	dockerURL := config.GetConfig().GetString("docker.url")
	return NewDockerProbe(g, n, dockerURL)
}
<|endoftext|>


<commit_before>package main

import (
	log "github.com/GameGophers/nsq-logger"
	"github.com/fzzy/radix/redis"
	"golang.org/x/net/context"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"os"
	pb "proto"
	"strconv"
	"strings"
	"time"
)

const (
	SERVICE             = "[BGSAVE]"
	DEFAULT_SAVE_DELAY  = 100 * time.Millisecond
	DEFAULT_REDIS_HOST  = "127.0.0.1:6379"
	DEFAULT_MONGODB_URL = "mongodb://127.0.0.1/mydb"
	ENV_REDIS_HOST      = "REDIS_HOST"
	ENV_MONGODB_URL     = "MONGODB_URL"
	BUFSIZ              = 4096
	BATCH_SIZE          = 1024 // data save batch size
)

type server struct {
	wait         chan string
	redis_client *redis.Client
	db           *mgo.Database
}

func (s *server) init() {
	// read redis host
	redis_host := DEFAULT_REDIS_HOST
	if env := os.Getenv(ENV_REDIS_HOST); env != "" {
		redis_host = env
	}
	// start connection to redis
	client, err := redis.Dial("tcp", redis_host)
	if err != nil {
		log.Critical(err)
		os.Exit(-1)
	}
	s.redis_client = client

	// read mongodb host
	mongodb_url := DEFAULT_MONGODB_URL
	if env := os.Getenv(ENV_MONGODB_URL); env != "" {
		mongodb_url = env
	}

	// start connection to mongodb
	sess, err := mgo.Dial(mongodb_url)
	if err != nil {
		log.Critical(err)
		os.Exit(-1)
	}
	// database is provided in url
	s.db = sess.DB("")

	// wait chan
	s.wait = make(chan string, BUFSIZ)
	go s.loader_task()
}

func (s *server) MarkDirty(ctx context.Context, in *pb.BgSave_Key) (*pb.BgSave_NullResult, error) {
	s.wait <- in.Name
	return &pb.BgSave_NullResult{}, nil
}

func (s *server) MarkDirties(ctx context.Context, in *pb.BgSave_Keys) (*pb.BgSave_NullResult, error) {
	for k := range in.Names {
		s.wait <- in.Names[k]
	}
	return &pb.BgSave_NullResult{}, nil
}

// background loader, copy chan into map, execute dump every DEFAULT_SAVE_DELAY
func (s *server) loader_task() {
	for {
		dirty := make(map[string]bool)
		timer := time.After(DEFAULT_SAVE_DELAY)
		select {
		case key := <-s.wait:
			dirty[key] = true
		case <-timer:
			if len(dirty) > 0 {
				s.dump(dirty)
				dirty = make(map[string]bool)
			}
			timer = time.After(DEFAULT_SAVE_DELAY)
		}
	}
}

// dump all dirty data into backend database
func (s *server) dump(dirty map[string]bool) {
	// copy dirty map into array
	dirty_list := make([]interface{}, 0, len(dirty))
	for k := range dirty {
		dirty_list = append(dirty_list, k)
	}

	// write data in batch
	var sublist []interface{}
	for i := 0; i < len(dirty_list); i += BATCH_SIZE {
		if (i+1)*BATCH_SIZE > len(dirty_list) { // reach end
			sublist = dirty_list[i*BATCH_SIZE:]
		} else {
			sublist = dirty_list[i*BATCH_SIZE : (i+1)*BATCH_SIZE]
		}

		// mget data from redis
		records, err := s.redis_client.Cmd("mget", sublist...).ListBytes()
		if err != nil {
			log.Critical(err)
			return
		}

		// save to mongodb
		var tmp map[string]interface{}
		for k, v := range sublist {
			err := bson.Unmarshal(records[k], &tmp)
			if err != nil {
				log.Critical(err)
				continue
			}

			// split key into TABLE NAME and RECORD ID
			strs := strings.Split(v.(string), ":")
			if len(strs) != 2 { // log the wrong key
				log.Critical("cannot split key", v)
				continue
			}
			tblname, id_str := strs[0], strs[1]
			// save data to mongodb
			id, err := strconv.Atoi(id_str)
			if err != nil {
				log.Critical(err)
				continue
			}

			_, err = s.db.C(tblname).Upsert(bson.M{"Id": id}, tmp)
			if err != nil {
				log.Critical(err)
				continue
			}
		}
	}
}
<commit_msg>change interface{} to string . for it's always string.<commit_after>package main

import (
	log "github.com/GameGophers/nsq-logger"
	"github.com/fzzy/radix/redis"
	"golang.org/x/net/context"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"os"
	pb "proto"
	"strconv"
	"strings"
	"time"
)

const (
	SERVICE             = "[BGSAVE]"
	DEFAULT_SAVE_DELAY  = 100 * time.Millisecond
	DEFAULT_REDIS_HOST  = "127.0.0.1:6379"
	DEFAULT_MONGODB_URL = "mongodb://127.0.0.1/mydb"
	ENV_REDIS_HOST      = "REDIS_HOST"
	ENV_MONGODB_URL     = "MONGODB_URL"
	BUFSIZ              = 4096
	BATCH_SIZE          = 1024 // data save batch size
)

type server struct {
	wait         chan string
	redis_client *redis.Client
	db           *mgo.Database
}

func (s *server) init() {
	// read redis host
	redis_host := DEFAULT_REDIS_HOST
	if env := os.Getenv(ENV_REDIS_HOST); env != "" {
		redis_host = env
	}
	// start connection to redis
	client, err := redis.Dial("tcp", redis_host)
	if err != nil {
		log.Critical(err)
		os.Exit(-1)
	}
	s.redis_client = client

	// read mongodb host
	mongodb_url := DEFAULT_MONGODB_URL
	if env := os.Getenv(ENV_MONGODB_URL); env != "" {
		mongodb_url = env
	}

	// start connection to mongodb
	sess, err := mgo.Dial(mongodb_url)
	if err != nil {
		log.Critical(err)
		os.Exit(-1)
	}
	// database is provided in url
	s.db = sess.DB("")

	// wait chan
	s.wait = make(chan string, BUFSIZ)
	go s.loader_task()
}

func (s *server) MarkDirty(ctx context.Context, in *pb.BgSave_Key) (*pb.BgSave_NullResult, error) {
	s.wait <- in.Name
	return &pb.BgSave_NullResult{}, nil
}

func (s *server) MarkDirties(ctx context.Context, in *pb.BgSave_Keys) (*pb.BgSave_NullResult, error) {
	for k := range in.Names {
		s.wait <- in.Names[k]
	}
	return &pb.BgSave_NullResult{}, nil
}

// background loader, copy chan into map, execute dump every DEFAULT_SAVE_DELAY
func (s *server) loader_task() {
	for {
		dirty := make(map[string]bool)
		timer := time.After(DEFAULT_SAVE_DELAY)
		select {
		case key := <-s.wait:
			dirty[key] = true
		case <-timer:
			if len(dirty) > 0 {
				s.dump(dirty)
				dirty = make(map[string]bool)
			}
			timer = time.After(DEFAULT_SAVE_DELAY)
		}
	}
}

// dump all dirty data into backend database
func (s *server) dump(dirty map[string]bool) {
	// copy dirty map into array
	dirty_list := make([]string, 0, len(dirty))
	for k := range dirty {
		dirty_list = append(dirty_list, k)
	}

	// write data in batch
	var sublist []string
	for i := 0; i < len(dirty_list); i += BATCH_SIZE {
		if (i+1)*BATCH_SIZE > len(dirty_list) { // reach end
			sublist = dirty_list[i*BATCH_SIZE:]
		} else {
			sublist = dirty_list[i*BATCH_SIZE : (i+1)*BATCH_SIZE]
		}

		// mget data from redis
		records, err := s.redis_client.Cmd("mget", sublist...).ListBytes()
		if err != nil {
			log.Critical(err)
			return
		}

		// save to mongodb
		var tmp map[string]interface{}
		for k, v := range sublist {
			err := bson.Unmarshal(records[k], &tmp)
			if err != nil {
				log.Critical(err)
				continue
			}

			// split key into TABLE NAME and RECORD ID
			strs := strings.Split(v, ":")
			if len(strs) != 2 { // log the wrong key
				log.Critical("cannot split key", v)
				continue
			}
			tblname, id_str := strs[0], strs[1]
			// save data to mongodb
			id, err := strconv.Atoi(id_str)
			if err != nil {
				log.Critical(err)
				continue
			}

			_, err = s.db.C(tblname).Upsert(bson.M{"Id": id}, tmp)
			if err != nil {
				log.Critical(err)
				continue
			}
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>Add cattle-fleet-local-system to system namespaces list<commit_after><|endoftext|>


<commit_before><commit_msg>Add code coverage for utils.go. (#1571)<commit_after><|endoftext|>