<commit_before>/*
 * Copyright 2014 Xuyuan Pang <xuyuanp@gmail.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package glogger

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"runtime"
	"time"
)

// gLogger is the default Logger
type gLogger struct {
	GroupFilter
	handlerGroup
	name  string
	level LogLevel
}

// NewLogger return a new Logger with debug level as default.
func NewLogger() Logger {
	l := &gLogger{
		level: DebugLevel,
	}
	return l
}

// Debug see details in Logger interface
func (l *gLogger) Debug(f string, v ...interface{}) {
	l.log(DebugLevel, fmt.Sprintf(f, v...))
}

// Info see details in Logger interface
func (l *gLogger) Info(f string, v ...interface{}) {
	l.log(InfoLevel, fmt.Sprintf(f, v...))
}

// Warning see details in Logger interface
func (l *gLogger) Warning(f string, v ...interface{}) {
	l.log(WarnLevel, fmt.Sprintf(f, v...))
}

// Error see details in Logger interface
func (l *gLogger) Error(f string, v ...interface{}) {
	l.log(ErrorLevel, fmt.Sprintf(f, v...))
}

// Critical see details in Logger interface
func (l *gLogger) Critical(f string, v ...interface{}) {
	l.log(CriticalLevel, fmt.Sprintf(f, v...))
}

func (l *gLogger) log(level LogLevel, msg string) {
	if level < l.level {
		return
	}
	now := time.Now()
	pc, file, line, ok := runtime.Caller(2)
	var funcname string
	if !ok {
		file = "???"
		line = 0
		funcname = "???"
	} else {
		funcname = runtime.FuncForPC(pc).Name()
	}
	rec := NewRecord(l.name, now, level, file, funcname, line, msg)
	if !l.Filter(rec) {
		return
	}
	l.Handle(rec)
}

func (l *gLogger) Level() LogLevel {
	return l.level
}

func (l *gLogger) SetLevel(level LogLevel) {
	l.level = level
}

func (l *gLogger) LoadConfig(config []byte) {
	var m map[string]interface{}
	if err := json.Unmarshal(config, &m); err == nil {
		l.LoadConfigFromMap(m)
	} else {
		panic(err)
	}
}

func (l *gLogger) LoadConfigFromMap(config map[string]interface{}) {
	if level, ok := config["level"]; ok {
		l.level = StringToLevel[level.(string)]
	} else {
		panic("'level' field is required")
	}
	if handlers, ok := config["handlers"]; ok {
		if len(handlers.([]interface{})) == 0 {
			panic("handler name is required")
		}
		for _, hname := range handlers.([]interface{}) {
			if handler := GetHandler(hname.(string)); handler != nil {
				l.AddHandler(handler)
			} else {
				panic("unknown handler name: " + hname.(string))
			}
		}
	} else {
		panic("'handlers' field is required")
	}
}

func (l *gLogger) LoadConfigFromFile(fileName string) {
	if file, err := os.Open(fileName); err == nil {
		defer file.Close()
		if code, err := ioutil.ReadAll(file); err == nil {
			l.LoadConfig(code)
		} else {
			panic(err)
		}
	} else {
		panic(err)
	}
}
<commit_msg>handle message in goroutine<commit_after>/*
 * Copyright 2014 Xuyuan Pang <xuyuanp@gmail.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package glogger

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"runtime"
	"time"
)

// gLogger is the default Logger
type gLogger struct {
	GroupFilter
	handlerGroup
	name  string
	level LogLevel
	ch    chan *Record
}

// NewLogger return a new Logger with debug level as default.
func NewLogger() Logger {
	l := &gLogger{
		level: DebugLevel,
		ch:    make(chan *Record, 100000),
	}
	go l.run()
	return l
}

// Debug see details in Logger interface
func (l *gLogger) Debug(f string, v ...interface{}) {
	l.log(DebugLevel, fmt.Sprintf(f, v...))
}

// Info see details in Logger interface
func (l *gLogger) Info(f string, v ...interface{}) {
	l.log(InfoLevel, fmt.Sprintf(f, v...))
}

// Warning see details in Logger interface
func (l *gLogger) Warning(f string, v ...interface{}) {
	l.log(WarnLevel, fmt.Sprintf(f, v...))
}

// Error see details in Logger interface
func (l *gLogger) Error(f string, v ...interface{}) {
	l.log(ErrorLevel, fmt.Sprintf(f, v...))
}

// Critical see details in Logger interface
func (l *gLogger) Critical(f string, v ...interface{}) {
	l.log(CriticalLevel, fmt.Sprintf(f, v...))
}

func (l *gLogger) log(level LogLevel, msg string) {
	if level < l.level {
		return
	}
	now := time.Now()
	pc, file, line, ok := runtime.Caller(2)
	var funcname string
	if !ok {
		file = "???"
		line = 0
		funcname = "???"
	} else {
		funcname = runtime.FuncForPC(pc).Name()
	}
	rec := NewRecord(l.name, now, level, file, funcname, line, msg)
	if !l.Filter(rec) {
		return
	}
	l.ch <- rec
}

func (l *gLogger) run() {
	for {
		rec := <-l.ch
		l.Handle(rec)
	}
}

func (l *gLogger) Level() LogLevel {
	return l.level
}

func (l *gLogger) SetLevel(level LogLevel) {
	l.level = level
}

func (l *gLogger) LoadConfig(config []byte) {
	var m map[string]interface{}
	if err := json.Unmarshal(config, &m); err == nil {
		l.LoadConfigFromMap(m)
	} else {
		panic(err)
	}
}

func (l *gLogger) LoadConfigFromMap(config map[string]interface{}) {
	if level, ok := config["level"]; ok {
		l.level = StringToLevel[level.(string)]
	} else {
		panic("'level' field is required")
	}
	if handlers, ok := config["handlers"]; ok {
		if len(handlers.([]interface{})) == 0 {
			panic("handler name is required")
		}
		for _, hname := range handlers.([]interface{}) {
			if handler := GetHandler(hname.(string)); handler != nil {
				l.AddHandler(handler)
			} else {
				panic("unknown handler name: " + hname.(string))
			}
		}
	} else {
		panic("'handlers' field is required")
	}
}

func (l *gLogger) LoadConfigFromFile(fileName string) {
	if file, err := os.Open(fileName); err == nil {
		defer file.Close()
		if code, err := ioutil.ReadAll(file); err == nil {
			l.LoadConfig(code)
		} else {
			panic(err)
		}
	} else {
		panic(err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/url"
	"os"
	"time"

	"k8s.io/contrib/cluster-autoscaler/config"
	"k8s.io/contrib/cluster-autoscaler/simulator"
	"k8s.io/contrib/cluster-autoscaler/utils/gce"
	kube_api "k8s.io/kubernetes/pkg/api"
	kube_record "k8s.io/kubernetes/pkg/client/record"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"

	"github.com/golang/glog"
)

var (
	migConfigFlag           config.MigConfigFlag
	kubernetes              = flag.String("kubernetes", "", "Kuberentes master location. Leave blank for default")
	cloudConfig             = flag.String("cloud-config", "", "The path to the cloud provider configuration file.  Empty string for no configuration file.")
	verifyUnschedulablePods = flag.Bool("verify-unschedulable-pods", true,
		"If enabled CA will ensure that each pod marked by Scheduler as unschedulable actually can't be scheduled on any node."+
			"This prevents from adding unnecessary nodes in situation when CA and Scheduler have different configuration.")
	scaleDownEnabled = flag.Bool("experimental-scale-down-enabled", false, "Should CA scale down the cluster")
	scaleDownDelay   = flag.Duration("scale-down-delay", 10*time.Minute,
		"Duration from the last scale up to the time when CA starts to check scale down options")
	scaleDownUnderutilizedTime = flag.Duration("scale-down-underutilized-time", 10*time.Minute,
		"How long the node should be underutilized before it is eligible for scale down")
	scaleDownUtilizationThreshold = flag.Float64("scale-down-utilization-threshold", 0.5,
		"Node reservation level below which a node can be considered for scale down")
	scaleDownTrialFrequency = flag.Duration("scale-down-trial-frequency", 10*time.Minute,
		"How often scale down possiblity is check")
)

func main() {
	flag.Var(&migConfigFlag, "nodes", "sets min,max size and url of a MIG to be controlled by Cluster Autoscaler. "+
		"Can be used multiple times. Format: <min>:<max>:<migurl>")
	flag.Parse()

	url, err := url.Parse(*kubernetes)
	if err != nil {
		glog.Fatalf("Failed to parse Kuberentes url: %v", err)
	}

	// Configuration
	kubeConfig, err := config.GetKubeClientConfig(url)
	if err != nil {
		glog.Fatalf("Failed to build Kuberentes client configuration: %v", err)
	}
	migConfigs := make([]*config.MigConfig, 0, len(migConfigFlag))
	for i := range migConfigFlag {
		migConfigs = append(migConfigs, &migConfigFlag[i])
	}

	// GCE Manager
	var gceManager *gce.GceManager
	if *cloudConfig != "" {
		config, err := os.Open(*cloudConfig)
		if err != nil {
			glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", *cloudConfig, err)
		}
		defer config.Close()
		gceManager, err = gce.CreateGceManager(migConfigs, config)
	} else {
		gceManager, err = gce.CreateGceManager(migConfigs, nil)
	}
	if err != nil {
		glog.Fatalf("Failed to create GCE Manager: %v", err)
	}

	kubeClient := kube_client.NewOrDie(kubeConfig)

	predicateChecker, err := simulator.NewPredicateChecker(kubeClient)
	if err != nil {
		glog.Fatalf("Failed to create predicate checker: %v", err)
	}
	unschedulablePodLister := NewUnschedulablePodLister(kubeClient)
	scheduledPodLister := NewScheduledPodLister(kubeClient)
	nodeLister := NewNodeLister(kubeClient)

	lastScaleUpTime := time.Now()
	lastScaleDownFailedTrial := time.Now()
	underutilizedNodes := make(map[string]time.Time)

	eventBroadcaster := kube_record.NewBroadcaster()
	eventBroadcaster.StartLogging(glog.Infof)
	eventBroadcaster.StartRecordingToSink(kubeClient.Events(""))
	recorder := eventBroadcaster.NewRecorder(kube_api.EventSource{Component: "cluster-autoscaler"})

	for {
		select {
		case <-time.After(time.Minute):
			{
				nodes, err := nodeLister.List()
				if err != nil {
					glog.Errorf("Failed to list nodes: %v", err)
					continue
				}
				if len(nodes) == 0 {
					glog.Errorf("No nodes in the cluster")
					continue
				}

				if err := CheckMigsAndNodes(nodes, gceManager); err != nil {
					glog.Warningf("Cluster is not ready for autoscaling: %v", err)
					continue
				}

				allUnschedulablePods, err := unschedulablePodLister.List()
				if err != nil {
					glog.Errorf("Failed to list unscheduled pods: %v", err)
					continue
				}

				allScheduled, err := scheduledPodLister.List()
				if err != nil {
					glog.Errorf("Failed to list scheduled pods: %v", err)
					continue
				}

				// We need to reset all pods that have been marked as unschedulable not after
				// the newest node became available for the scheduler.
				allNodesAvailableTime := GetAllNodesAvailableTime(nodes)
				podsToReset, unschedulablePodsToHelp := SlicePodsByPodScheduledTime(allUnschedulablePods, allNodesAvailableTime)
				ResetPodScheduledCondition(kubeClient, podsToReset)

				// We need to check whether pods marked as unschedulable are actually unschedulable.
				// This should prevent from adding unnecessary nodes. Example of such situation:
				// - CA and Scheduler has slightly different configuration
				// - Scheduler can't schedule a pod and marks it as unschedulable
				// - CA added a node which should help the pod
				// - Scheduler doesn't schedule the pod on the new node
				//   because according to it logic it doesn't fit there
				// - CA see the pod is still unschedulable, so it adds another node to help it
				//
				// With the check enabled the last point won't happen because CA will ignore a pod
				// which is supposed to schedule on an existing node.
				//
				// Without below check cluster might be unnecessary scaled up to the max allowed size
				// in the describe situation.
				schedulablePodsPresent := false
				if *verifyUnschedulablePods {
					newUnschedulablePodsToHelp := FilterOutSchedulable(unschedulablePodsToHelp, nodes, allScheduled, predicateChecker)

					if len(newUnschedulablePodsToHelp) != len(unschedulablePodsToHelp) {
						schedulablePodsPresent = true
					}
					unschedulablePodsToHelp = newUnschedulablePodsToHelp
				}

				if len(unschedulablePodsToHelp) == 0 {
					glog.V(1).Info("No unschedulable pods")
				} else {
					scaledUp, err := ScaleUp(unschedulablePodsToHelp, nodes, migConfigs, gceManager, kubeClient, predicateChecker, recorder)
					if err != nil {
						glog.Errorf("Failed to scale up: %v", err)
						continue
					} else {
						if scaledUp {
							lastScaleUpTime = time.Now()
							// No scale down in this iteration.
							continue
						}
					}
				}

				if *scaleDownEnabled {
					// In dry run only utilization is updated
					calculateUtilizationOnly := lastScaleUpTime.Add(*scaleDownDelay).After(time.Now()) ||
						lastScaleDownFailedTrial.Add(*scaleDownTrialFrequency).After(time.Now()) ||
						schedulablePodsPresent

					underutilizedNodes = CalculateUnderutilizedNodes(
						nodes,
						underutilizedNodes,
						*scaleDownUtilizationThreshold,
						allScheduled,
						predicateChecker)

					if !calculateUtilizationOnly {
						result, err := ScaleDown(
							nodes,
							underutilizedNodes,
							*scaleDownUnderutilizedTime,
							allScheduled,
							gceManager, kubeClient, predicateChecker)
						if err != nil {
							glog.Errorf("Failed to scale down: %v", err)
						} else {
							if result == ScaleDownNodeDeleted {
								// Clean the utilization map to be super sure that the simulated
								// deletions are made in the new context.
								underutilizedNodes = make(map[string]time.Time, len(underutilizedNodes))
							} else {
								lastScaleDownFailedTrial = time.Now()
							}
						}
					}
				}
			}
		}
	}
}
<commit_msg>Cluster-autoscaler: enable scale down by default<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/url"
	"os"
	"time"

	"k8s.io/contrib/cluster-autoscaler/config"
	"k8s.io/contrib/cluster-autoscaler/simulator"
	"k8s.io/contrib/cluster-autoscaler/utils/gce"
	kube_api "k8s.io/kubernetes/pkg/api"
	kube_record "k8s.io/kubernetes/pkg/client/record"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"

	"github.com/golang/glog"
)

var (
	migConfigFlag           config.MigConfigFlag
	kubernetes              = flag.String("kubernetes", "", "Kuberentes master location. Leave blank for default")
	cloudConfig             = flag.String("cloud-config", "", "The path to the cloud provider configuration file.  Empty string for no configuration file.")
	verifyUnschedulablePods = flag.Bool("verify-unschedulable-pods", true,
		"If enabled CA will ensure that each pod marked by Scheduler as unschedulable actually can't be scheduled on any node."+
			"This prevents from adding unnecessary nodes in situation when CA and Scheduler have different configuration.")
	scaleDownEnabled = flag.Bool("scale-down-enabled", true, "Should CA scale down the cluster")
	scaleDownDelay   = flag.Duration("scale-down-delay", 10*time.Minute,
		"Duration from the last scale up to the time when CA starts to check scale down options")
	scaleDownUnderutilizedTime = flag.Duration("scale-down-underutilized-time", 10*time.Minute,
		"How long the node should be underutilized before it is eligible for scale down")
	scaleDownUtilizationThreshold = flag.Float64("scale-down-utilization-threshold", 0.5,
		"Node reservation level below which a node can be considered for scale down")
	scaleDownTrialFrequency = flag.Duration("scale-down-trial-frequency", 10*time.Minute,
		"How often scale down possiblity is check")
)

func main() {
	flag.Var(&migConfigFlag, "nodes", "sets min,max size and url of a MIG to be controlled by Cluster Autoscaler. "+
		"Can be used multiple times. Format: <min>:<max>:<migurl>")
	flag.Parse()

	url, err := url.Parse(*kubernetes)
	if err != nil {
		glog.Fatalf("Failed to parse Kuberentes url: %v", err)
	}

	// Configuration
	kubeConfig, err := config.GetKubeClientConfig(url)
	if err != nil {
		glog.Fatalf("Failed to build Kuberentes client configuration: %v", err)
	}
	migConfigs := make([]*config.MigConfig, 0, len(migConfigFlag))
	for i := range migConfigFlag {
		migConfigs = append(migConfigs, &migConfigFlag[i])
	}

	// GCE Manager
	var gceManager *gce.GceManager
	if *cloudConfig != "" {
		config, err := os.Open(*cloudConfig)
		if err != nil {
			glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", *cloudConfig, err)
		}
		defer config.Close()
		gceManager, err = gce.CreateGceManager(migConfigs, config)
	} else {
		gceManager, err = gce.CreateGceManager(migConfigs, nil)
	}
	if err != nil {
		glog.Fatalf("Failed to create GCE Manager: %v", err)
	}

	kubeClient := kube_client.NewOrDie(kubeConfig)

	predicateChecker, err := simulator.NewPredicateChecker(kubeClient)
	if err != nil {
		glog.Fatalf("Failed to create predicate checker: %v", err)
	}
	unschedulablePodLister := NewUnschedulablePodLister(kubeClient)
	scheduledPodLister := NewScheduledPodLister(kubeClient)
	nodeLister := NewNodeLister(kubeClient)

	lastScaleUpTime := time.Now()
	lastScaleDownFailedTrial := time.Now()
	underutilizedNodes := make(map[string]time.Time)

	eventBroadcaster := kube_record.NewBroadcaster()
	eventBroadcaster.StartLogging(glog.Infof)
	eventBroadcaster.StartRecordingToSink(kubeClient.Events(""))
	recorder := eventBroadcaster.NewRecorder(kube_api.EventSource{Component: "cluster-autoscaler"})

	for {
		select {
		case <-time.After(time.Minute):
			{
				nodes, err := nodeLister.List()
				if err != nil {
					glog.Errorf("Failed to list nodes: %v", err)
					continue
				}
				if len(nodes) == 0 {
					glog.Errorf("No nodes in the cluster")
					continue
				}

				if err := CheckMigsAndNodes(nodes, gceManager); err != nil {
					glog.Warningf("Cluster is not ready for autoscaling: %v", err)
					continue
				}

				allUnschedulablePods, err := unschedulablePodLister.List()
				if err != nil {
					glog.Errorf("Failed to list unscheduled pods: %v", err)
					continue
				}

				allScheduled, err := scheduledPodLister.List()
				if err != nil {
					glog.Errorf("Failed to list scheduled pods: %v", err)
					continue
				}

				// We need to reset all pods that have been marked as unschedulable not after
				// the newest node became available for the scheduler.
				allNodesAvailableTime := GetAllNodesAvailableTime(nodes)
				podsToReset, unschedulablePodsToHelp := SlicePodsByPodScheduledTime(allUnschedulablePods, allNodesAvailableTime)
				ResetPodScheduledCondition(kubeClient, podsToReset)

				// We need to check whether pods marked as unschedulable are actually unschedulable.
				// This should prevent from adding unnecessary nodes. Example of such situation:
				// - CA and Scheduler has slightly different configuration
				// - Scheduler can't schedule a pod and marks it as unschedulable
				// - CA added a node which should help the pod
				// - Scheduler doesn't schedule the pod on the new node
				//   because according to it logic it doesn't fit there
				// - CA see the pod is still unschedulable, so it adds another node to help it
				//
				// With the check enabled the last point won't happen because CA will ignore a pod
				// which is supposed to schedule on an existing node.
				//
				// Without below check cluster might be unnecessary scaled up to the max allowed size
				// in the describe situation.
				schedulablePodsPresent := false
				if *verifyUnschedulablePods {
					newUnschedulablePodsToHelp := FilterOutSchedulable(unschedulablePodsToHelp, nodes, allScheduled, predicateChecker)

					if len(newUnschedulablePodsToHelp) != len(unschedulablePodsToHelp) {
						schedulablePodsPresent = true
					}
					unschedulablePodsToHelp = newUnschedulablePodsToHelp
				}

				if len(unschedulablePodsToHelp) == 0 {
					glog.V(1).Info("No unschedulable pods")
				} else {
					scaledUp, err := ScaleUp(unschedulablePodsToHelp, nodes, migConfigs, gceManager, kubeClient, predicateChecker, recorder)
					if err != nil {
						glog.Errorf("Failed to scale up: %v", err)
						continue
					} else {
						if scaledUp {
							lastScaleUpTime = time.Now()
							// No scale down in this iteration.
							continue
						}
					}
				}

				if *scaleDownEnabled {
					// In dry run only utilization is updated
					calculateUtilizationOnly := lastScaleUpTime.Add(*scaleDownDelay).After(time.Now()) ||
						lastScaleDownFailedTrial.Add(*scaleDownTrialFrequency).After(time.Now()) ||
						schedulablePodsPresent

					underutilizedNodes = CalculateUnderutilizedNodes(
						nodes,
						underutilizedNodes,
						*scaleDownUtilizationThreshold,
						allScheduled,
						predicateChecker)

					if !calculateUtilizationOnly {
						result, err := ScaleDown(
							nodes,
							underutilizedNodes,
							*scaleDownUnderutilizedTime,
							allScheduled,
							gceManager, kubeClient, predicateChecker)
						if err != nil {
							glog.Errorf("Failed to scale down: %v", err)
						} else {
							if result == ScaleDownNodeDeleted {
								// Clean the utilization map to be super sure that the simulated
								// deletions are made in the new context.
								underutilizedNodes = make(map[string]time.Time, len(underutilizedNodes))
							} else {
								lastScaleDownFailedTrial = time.Now()
							}
						}
					}
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"strings"
	"time"

	alks "github.com/Cox-Automotive/alks-go"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/terraform"
)

func resourceAlksIamRole() *schema.Resource {
	return &schema.Resource{
		Create: resourceAlksIamRoleCreate,
		Read:   resourceAlksIamRoleRead,
		Update: resourceAlksIamRoleUpdate,
		Exists: resourceAlksIamRoleExists,
		Delete: resourceAlksIamRoleDelete,

		SchemaVersion: 1,
		MigrateState:  migrateState,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"type": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"include_default_policies": &schema.Schema{
				Type:     schema.TypeBool,
				Required: true,
				ForceNew: true,
			},
			"role_added_to_ip": &schema.Schema{
				Type:     schema.TypeBool,
				Computed: true,
			},
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"ip_arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"enable_alks_access": &schema.Schema{
				Type:     schema.TypeBool,
				Default:  false,
				Optional: true,
			},
		},
	}
}

func resourceAlksIamTrustRole() *schema.Resource {
	return &schema.Resource{
		Create: resourceAlksIamTrustRoleCreate,
		Read:   resourceAlksIamRoleRead,
		Update: resourceAlksIamRoleUpdate,
		Exists: resourceAlksIamRoleExists,
		Delete: resourceAlksIamRoleDelete,

		SchemaVersion: 1,
		MigrateState:  migrateState,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"type": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"trust_arn": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"role_added_to_ip": &schema.Schema{
				Type:     schema.TypeBool,
				Computed: true,
			},
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"ip_arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"enable_alks_access": &schema.Schema{
				Type:     schema.TypeBool,
				Default:  false,
				Optional: true,
			},
		},
	}
}

func resourceAlksIamRoleCreate(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] ALKS IAM Role Create")

	var roleName = d.Get("name").(string)
	var roleType = d.Get("type").(string)
	var incDefPol = d.Get("include_default_policies").(bool)
	var enableAlksAccess = d.Get("enable_alks_access").(bool)

	client := meta.(*alks.Client)
	resp, err := client.CreateIamRole(roleName, roleType, incDefPol, enableAlksAccess)

	if err != nil {
		return err
	}

	d.SetId(resp.RoleName)
	d.Set("arn", resp.RoleArn)
	d.Set("ip_arn", resp.RoleIPArn)
	d.Set("role_added_to_ip", resp.RoleAddedToIP)

	log.Printf("[INFO] alks_iamrole.id: %v", d.Id())

	return nil
}

func resourceAlksIamTrustRoleCreate(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] ALKS IAM Trust Role Create")

	var roleName = d.Get("name").(string)
	var roleType = d.Get("type").(string)
	var trustArn = d.Get("trust_arn").(string)
	var enableAlksAccess = d.Get("enable_alks_access").(bool)

	client := meta.(*alks.Client)

	var resp *alks.IamRoleResponse
	err := resource.Retry(2*time.Minute, func() *resource.RetryError {
		var err error
		resp, err = client.CreateIamTrustRole(roleName, roleType, trustArn, enableAlksAccess)
		if err != nil {
			if strings.Contains(err.Error(), "Role already exists") || strings.Contains(err.Error(), "Instance profile exists") {
				return resource.NonRetryableError(err)
			}

			return resource.RetryableError(err)
		}
		return resource.NonRetryableError(err)
	})

	if err != nil {
		return err
	}

	response := *resp

	d.SetId(response.RoleName)
	d.Set("arn", response.RoleArn)
	d.Set("ip_arn", response.RoleIPArn)
	d.Set("role_added_to_ip", response.RoleAddedToIP)

	log.Printf("[INFO] alks_iamtrustrole.id: %v", d.Id())

	return nil
}

func resourceAlksIamRoleDelete(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] ALKS IAM Role Delete")

	client := meta.(*alks.Client)
	err := client.DeleteIamRole(d.Id())

	if err != nil {
		return err
	}

	return nil
}

func resourceAlksIamRoleExists(d *schema.ResourceData, meta interface{}) (b bool, e error) {
	log.Printf("[INFO] ALKS IAM Role Exists")

	client := meta.(*alks.Client)

	foundrole, err := client.GetIamRole(d.Id())

	if err != nil {
		// TODO: Clean-up this logic, likely by improving the error responses from `alks-go`
		if strings.Contains(err.Error(), "Role not found") {
			return false, nil
		}

		return false, err
	}

	if foundrole == nil {
		return false, nil
	}

	return true, nil
}

func resourceAlksIamRoleRead(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] ALKS IAM Role Read")

	client := meta.(*alks.Client)

	foundrole, err := client.GetIamRole(d.Id())

	if err != nil {
		return err
	}

	return populateResourceDataFromRole(foundrole, d)
}

func resourceAlksIamRoleUpdate(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] ALKS IAM Role Update")

	// enable partial state mode
	d.Partial(true)

	if d.HasChange("enable_alks_access") {
		// try updating enable_alks_access
		if err := updateAlksAccess(d, meta); err != nil {
			return err
		}

		d.SetPartial("enable_alks_access")
	}

	d.Partial(false)

	return nil
}

func updateAlksAccess(d *schema.ResourceData, meta interface{}) error {
	var alksAccess = d.Get("enable_alks_access").(bool)
	var roleArn = d.Get("arn").(string)
	client := meta.(*alks.Client)
	// create the machine identity
	if alksAccess {
		_, err := client.AddRoleMachineIdentity(roleArn)
		if err != nil {
			return err
		}
	} else {
		// delete the machine identity
		_, err := client.DeleteRoleMachineIdentity(roleArn)
		if err != nil {
			return err
		}
	}
	return nil
}

func populateResourceDataFromRole(role *alks.GetIamRoleResponse, d *schema.ResourceData) error {
	d.SetId(role.RoleName)
	d.Set("arn", role.RoleArn)
	d.Set("ip_arn", role.RoleIPArn)
	d.Set("enable_alks_access", role.AlksAccess)

	// role type isnt returned by alks api so this will always false report on a remote state change
	// for more info see issue #125 on ALKS repo
	// d.Set("type", role.RoleType)

	return nil
}

func populateResourceDataFromMI(mi *alks.MachineIdentityResponse, d *schema.ResourceData) error {
	d.SetId(mi.MachineIdentityArn)
	d.Set("machine_identity_arn", mi.MachineIdentityArn)

	return nil
}

func migrateState(version int, state *terraform.InstanceState, meta interface{}) (*terraform.InstanceState, error) {
	switch version {
	case 0:
		log.Println("[INFO] Found Instance State v0, migrating to v1")
		return migrateV0toV1(state)
	default:
		return state, fmt.Errorf("Unrecognized version '%d' in schema for instance of ALKS IAM role '%s'", version, state.Attributes["name"])
	}
}

func migrateV0toV1(state *terraform.InstanceState) (*terraform.InstanceState, error) {
	if state.Empty() {
		log.Println("[DEBUG] Empty InstanceState, nothing to migrate")
		return state, nil
	}

	if _, ok := state.Attributes["enable_alks_access"]; !ok {
		log.Printf("[DEBUG] Attributes before migration: %#v", state.Attributes)
		state.Attributes["enable_alks_access"] = "false"
		log.Printf("[DEBUG] Attributes after migration: %#v", state.Attributes)
	}

	return state, nil
}
<commit_msg>Adds sleeps to retryable operations to slow down API request rate<commit_after>package main

import (
	"fmt"
	"log"
	"strings"
	"time"

	alks "github.com/Cox-Automotive/alks-go"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/terraform"
)

func resourceAlksIamRole() *schema.Resource {
	return &schema.Resource{
		Create: resourceAlksIamRoleCreate,
		Read:   resourceAlksIamRoleRead,
		Update: resourceAlksIamRoleUpdate,
		Exists: resourceAlksIamRoleExists,
		Delete: resourceAlksIamRoleDelete,

		SchemaVersion: 1,
		MigrateState:  migrateState,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"type": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"include_default_policies": &schema.Schema{
				Type:     schema.TypeBool,
				Required: true,
				ForceNew: true,
			},
			"role_added_to_ip": &schema.Schema{
				Type:     schema.TypeBool,
				Computed: true,
			},
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"ip_arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"enable_alks_access": &schema.Schema{
				Type:     schema.TypeBool,
				Default:  false,
				Optional: true,
			},
		},
	}
}

func resourceAlksIamTrustRole() *schema.Resource {
	return &schema.Resource{
		Create: resourceAlksIamTrustRoleCreate,
		Read:   resourceAlksIamRoleRead,
		Update: resourceAlksIamRoleUpdate,
		Exists: resourceAlksIamRoleExists,
		Delete: resourceAlksIamRoleDelete,

		SchemaVersion: 1,
		MigrateState:  migrateState,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"type": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"trust_arn": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"role_added_to_ip": &schema.Schema{
				Type:     schema.TypeBool,
				Computed: true,
			},
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"ip_arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"enable_alks_access": &schema.Schema{
				Type:     schema.TypeBool,
				Default:  false,
				Optional: true,
			},
		},
	}
}

func resourceAlksIamRoleCreate(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] ALKS IAM Role Create")

	var roleName = d.Get("name").(string)
	var roleType = d.Get("type").(string)
	var incDefPol = d.Get("include_default_policies").(bool)
	var enableAlksAccess = d.Get("enable_alks_access").(bool)

	client := meta.(*alks.Client)
	resp, err := client.CreateIamRole(roleName, roleType, incDefPol, enableAlksAccess)

	if err != nil {
		return err
	}

	d.SetId(resp.RoleName)
	d.Set("arn", resp.RoleArn)
	d.Set("ip_arn", resp.RoleIPArn)
	d.Set("role_added_to_ip", resp.RoleAddedToIP)

	log.Printf("[INFO] alks_iamrole.id: %v", d.Id())

	return nil
}

func resourceAlksIamTrustRoleCreate(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] ALKS IAM Trust Role Create")

	var roleName = d.Get("name").(string)
	var roleType = d.Get("type").(string)
	var trustArn = d.Get("trust_arn").(string)
	var enableAlksAccess = d.Get("enable_alks_access").(bool)

	client := meta.(*alks.Client)

	var resp *alks.IamRoleResponse
	err := resource.Retry(2*time.Minute, func() *resource.RetryError {
		var err error
		resp, err = client.CreateIamTrustRole(roleName, roleType, trustArn, enableAlksAccess)
		if err != nil {
			if strings.Contains(err.Error(), "Role already exists") || strings.Contains(err.Error(), "Instance profile exists") {
				return resource.NonRetryableError(err)
			}

			// Due to IAM eventual consistency, you might've just created a role that you need to link to a trust
			// We'll keep checking every 15 seconds for up to 2 minutes to see if the role appears
			time.Sleep(15 * time.Second)
			return resource.RetryableError(err)
		}
		return resource.NonRetryableError(err)
	})

	if err != nil {
		return err
	}

	response := *resp

	d.SetId(response.RoleName)
	d.Set("arn", response.RoleArn)
	d.Set("ip_arn", response.RoleIPArn)
	d.Set("role_added_to_ip", response.RoleAddedToIP)

	log.Printf("[INFO] alks_iamtrustrole.id: %v", d.Id())

	return nil
}

func resourceAlksIamRoleDelete(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] ALKS IAM Role Delete")

	client := meta.(*alks.Client)
	err := client.DeleteIamRole(d.Id())

	if err != nil {
		return err
	}

	return nil
}

func resourceAlksIamRoleExists(d *schema.ResourceData, meta interface{}) (b bool, e error) {
	log.Printf("[INFO] ALKS IAM Role Exists")

	client := meta.(*alks.Client)

	foundrole, err := client.GetIamRole(d.Id())

	if err != nil {
		// TODO: Clean-up this logic, likely by improving the error responses from `alks-go`
		if strings.Contains(err.Error(), "Role not found") {
			return false, nil
		}

		return false, err
	}

	if foundrole == nil {
		return false, nil
	}

	return true, nil
}

func resourceAlksIamRoleRead(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] ALKS IAM Role Read")

	client := meta.(*alks.Client)

	foundrole, err := client.GetIamRole(d.Id())

	if err != nil {
		return err
	}

	return populateResourceDataFromRole(foundrole, d)
}

func resourceAlksIamRoleUpdate(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] ALKS IAM Role Update")

	// enable partial state mode
	d.Partial(true)

	if d.HasChange("enable_alks_access") {
		// try updating enable_alks_access
		if err := updateAlksAccess(d, meta); err != nil {
			return err
		}

		d.SetPartial("enable_alks_access")
	}

	d.Partial(false)

	return nil
}

func updateAlksAccess(d *schema.ResourceData, meta interface{}) error {
	var alksAccess = d.Get("enable_alks_access").(bool)
	var roleArn = d.Get("arn").(string)
	client := meta.(*alks.Client)
	// create the machine identity
	if alksAccess {
		_, err := client.AddRoleMachineIdentity(roleArn)
		if err != nil {
			return err
		}
	} else {
		// delete the machine identity
		_, err := client.DeleteRoleMachineIdentity(roleArn)
		if err != nil {
			return err
		}
	}
	return nil
}

func populateResourceDataFromRole(role *alks.GetIamRoleResponse, d *schema.ResourceData) error {
	d.SetId(role.RoleName)
	d.Set("arn", role.RoleArn)
	d.Set("ip_arn", role.RoleIPArn)
	d.Set("enable_alks_access", role.AlksAccess)

	// role type isnt returned by alks api so this will always false report on a remote state change
	// for more info see issue #125 on ALKS repo
	// d.Set("type", role.RoleType)

	return nil
}

func populateResourceDataFromMI(mi *alks.MachineIdentityResponse, d *schema.ResourceData) error {
	d.SetId(mi.MachineIdentityArn)
	d.Set("machine_identity_arn", mi.MachineIdentityArn)

	return nil
}

func migrateState(version int, state *terraform.InstanceState, meta interface{}) (*terraform.InstanceState, error) {
	switch version {
	case 0:
		log.Println("[INFO] Found Instance State v0, migrating to v1")
		return migrateV0toV1(state)
	default:
		return state, fmt.Errorf("Unrecognized version '%d' in schema for instance of ALKS IAM role '%s'", version, state.Attributes["name"])
	}
}

func migrateV0toV1(state *terraform.InstanceState) (*terraform.InstanceState, error) {
	if state.Empty() {
		log.Println("[DEBUG] Empty InstanceState, nothing to migrate")
		return state, nil
	}

	if _, ok := state.Attributes["enable_alks_access"]; !ok {
		log.Printf("[DEBUG] Attributes before migration: %#v", state.Attributes)
		state.Attributes["enable_alks_access"] = "false"
		log.Printf("[DEBUG] Attributes after migration: %#v", state.Attributes)
	}

	return state, nil
}
<|endoftext|>


<commit_before>package sqlite3

import (
	"database/sql"
	"io/ioutil"
	"os"
	"path"
	"testing"
)

func TestFailures(t *testing.T) {
	dirName, err := ioutil.TempDir("", "sqlite3")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dirName)

	dbFileName := path.Join(dirName, "test.db")
	f, err := os.Create(dbFileName)
	if err != nil {
		t.Error(err)
	}
	f.Write([]byte{1, 2, 3, 4, 5})
	f.Close()

	db, err := sql.Open("sqlite3", dbFileName)
	if err == nil {
		_, err = db.Exec("drop table foo")
	}
	if err.Code != ErrNotADB {
		t.Error("wrong error code for corrupted DB")
	}
	if err.Error() == "" {
		t.Error("wrong error string for corrupted DB")
	}
	db.Close()
}
<commit_msg>Update test expectations follow change of concrete error type<commit_after>package sqlite3

import (
	"database/sql"
	"io/ioutil"
	"os"
	"path"
	"testing"
)

func TestFailures(t *testing.T) {
	dirName, err := ioutil.TempDir("", "sqlite3")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dirName)

	dbFileName := path.Join(dirName, "test.db")
	f, err := os.Create(dbFileName)
	if err != nil {
		t.Error(err)
	}
	f.Write([]byte{1, 2, 3, 4, 5})
	f.Close()

	db, err := sql.Open("sqlite3", dbFileName)
	if err == nil {
		_, err = db.Exec("drop table foo")
	}

	sqliteErr := err.(Error)
	if sqliteErr.Code != ErrNotADB {
		t.Error("wrong error code for corrupted DB")
	}
	if err.Error() == "" {
		t.Error("wrong error string for corrupted DB")
	}
	db.Close()
}
<|endoftext|>


<commit_before>package braintree

import "testing"
import "net/http"
import "bytes"
import "io/ioutil"

func TestGatewayError(t *testing.T) {
	t.Parallel()

	err := &GatewayError{Message: "Johnny"}
	got := err.Error()
	want := "Johnny"
	if got != want {
		t.Errorf("got %v, want %v", got, want)
	}
}

func TestValidationError(t *testing.T) {
	t.Parallel()

	err := &ValidationError{Code: 5, Message: "Johnny"}
	got := err.Error()
	want := "Code 5: Johnny"
	if got != want {
		t.Errorf("got %v, want %v", got, want)
	}
}

func TestProcessError(t *testing.T) {
	t.Parallel()

	err := &ProcessorError{Code: 5, Message: "Johnny"}
	got := err.Error()
	want := "Code 5: Johnny"
	if got != want {
		t.Errorf("got %v, want %v", got, want)
	}
}

func TestParseError(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name      string
		resp      *http.Response
		wantError string
	}{
		{
			name: "unexpectedError",
			resp: &http.Response{
				Body:       ioutil.NopCloser(bytes.NewBufferString("<api-error-response></api-error-response>")),
				StatusCode: http.StatusUnprocessableEntity,
				Status:     "422 Unprocessable Entity",
			},
			wantError: "422 Unprocessable Entity",
		},
		{
			name: "emptyBody",
			resp: &http.Response{
				Body:       ioutil.NopCloser(bytes.NewBufferString("")),
				StatusCode: http.StatusUnprocessableEntity,
			},
			wantError: "EOF",
		},
	}

	for _, test := range tests {
		test := test
		t.Run(test.name, func(t *testing.T) {
			t.Parallel()

			if got := parseError(test.resp); got == nil || got.Error() != test.wantError {
				t.Errorf("want %v, got %v", test.wantError, got)
			}
		})
	}

}
<commit_msg>improve TestParseError<commit_after>package braintree

import "testing"
import "net/http"
import "bytes"
import "io/ioutil"

func TestGatewayError(t *testing.T) {
	t.Parallel()

	err := &GatewayError{Message: "Johnny"}
	got := err.Error()
	want := "Johnny"
	if got != want {
		t.Errorf("got %v, want %v", got, want)
	}
}

func TestValidationError(t *testing.T) {
	t.Parallel()

	err := &ValidationError{Code: 5, Message: "Johnny"}
	got := err.Error()
	want := "Code 5: Johnny"
	if got != want {
		t.Errorf("got %v, want %v", got, want)
	}
}

func TestProcessError(t *testing.T) {
	t.Parallel()

	err := &ProcessorError{Code: 5, Message: "Johnny"}
	got := err.Error()
	want := "Code 5: Johnny"
	if got != want {
		t.Errorf("got %v, want %v", got, want)
	}
}

func TestParseError(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name      string
		resp      *http.Response
		wantError string
	}{
		{
			name: "unexpectedError",
			resp: &http.Response{
				Body:       ioutil.NopCloser(bytes.NewBufferString("<api-error-response></api-error-response>")),
				StatusCode: http.StatusUnprocessableEntity,
				Status:     "422 Unprocessable Entity",
			},
			wantError: "422 Unprocessable Entity",
		},
		{
			name: "emptyBody",
			resp: &http.Response{
				Body:       ioutil.NopCloser(bytes.NewBufferString("")),
				StatusCode: http.StatusUnprocessableEntity,
			},
			wantError: "EOF",
		},
		{
			name: "invalidProcessorResponse",
			resp: &http.Response{
				Body:       ioutil.NopCloser(bytes.NewBufferString("<api-error-response><transaction><processor-response-code>text</processor-response-code></transaction></api-error-response>")),
				StatusCode: http.StatusUnprocessableEntity,
			},
			wantError: "strconv.Atoi: parsing \"text\": invalid syntax",
		},
	}

	for _, test := range tests {
		test := test
		t.Run(test.name, func(t *testing.T) {
			t.Parallel()

			if got := parseError(test.resp); got == nil || got.Error() != test.wantError {
				t.Errorf("want %v, got %v", test.wantError, got)
			}
		})
	}

}
<|endoftext|>


<commit_before>package messagebird

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestError(t *testing.T) {
	t.Run("Single", func(t *testing.T) {
		errRes := ErrorResponse{
			Errors: []Error{
				{
					Code:        42,
					Description: "something bad",
					Parameter:   "foo",
				},
			},
		}
		assert.Error(t, errRes)
	})

	t.Run("Multiple", func(t *testing.T) {
		errRes := ErrorResponse{
			Errors: []Error{
				{
					Code:        42,
					Description: "something bad",
					Parameter:   "foo",
				},
				{
					Code:        42,
					Description: "something else",
					Parameter:   "foo",
				},
			},
		}
		assert.Error(t, errRes)
	})
}
<commit_msg>improve error_test<commit_after>package messagebird

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestError(t *testing.T) {
	t.Run("Single", func(t *testing.T) {
		errRes := ErrorResponse{
			Errors: []Error{
				{
					Code:        42,
					Description: "something bad",
					Parameter:   "foo",
				},
			},
		}
		assert.Error(t, errRes)
	})

	t.Run("Multiple", func(t *testing.T) {
		errRes := ErrorResponse{
			Errors: []Error{
				{
					Code:        42,
					Description: "something bad",
					Parameter:   "foo",
				},
				{
					Code:        42,
					Description: "something else",
					Parameter:   "foo",
				},
			},
		}
		assert.Error(t, errRes)
	})
}

func TestErrorWithEmptyErrors(t *testing.T) {
	errRes := ErrorResponse{}
	assert.Error(t, errRes)
	assert.Equal(t, "API errors: ", errRes.Error())
}

func TestErrorWithOneEmptyError(t *testing.T) {
	errRes := ErrorResponse{Errors: []Error{
		{
			Code:        11,
			Description: "",
			Parameter:   "",
		},
	}}
	assert.Error(t, errRes)
	assert.Equal(t, "API errors: ", errRes.Error())
}
<|endoftext|>


<commit_before>package collectors

import (
	"fmt"
	"strings"
	"time"

	"bosun.org/_third_party/github.com/StackExchange/wmi"
)

var (
	epochUTCTime = time.Unix(0, 0).UTC()
)

func queryWmi(query string, dst interface{}) error {
	return queryWmiNamespace(query, dst, "")
}

func queryWmiNamespace(query string, dst interface{}, namespace string) error {
	return wmi.QueryNamespace(query, dst, namespace)
}

func wmiInit(c *IntervalCollector, dst func() interface{}, where string, query *string) func() {
	return func() {
		*query = wmi.CreateQuery(dst(), where)
		c.Enable = func() bool {
			return queryWmi(*query, dst()) == nil
		}
	}
}

func wmiInitNamespace(c *IntervalCollector, dst func() interface{}, where string, query *string, namespace string) func() {
	return func() {
		*query = wmi.CreateQuery(dst(), where)
		c.Enable = func() bool {
			return queryWmiNamespace(*query, dst(), namespace) == nil
		}
	}
}

// wmiParseDatetime converts a string from the CIM_DATETIME format into UTC time.
// Example: "20150616101948.494497-360" = 2015-06-16 04:19:48.494497 +0000 UTC.
func wmiParseCIMDatetime(cimdatetime string) (time.Time, error) {
	i := strings.IndexAny(cimdatetime, "+-")
	if i < 0 {
		return epochUTCTime, fmt.Errorf("Invalid CIM_DATETIME format, cannot find UTC offset.")
	}
	t, err := time.Parse("20060102150405", cimdatetime[0:i])
	if err != nil {
		return epochUTCTime, err
	}
	offset, err := time.ParseDuration(fmt.Sprintf("%vm", cimdatetime[i:]))
	if err != nil {
		return epochUTCTime, err
	}
	return t.Add(offset), nil
}
<commit_msg>cmd/scollector: don't use time var just for an error return<commit_after>package collectors

import (
	"fmt"
	"strings"
	"time"

	"bosun.org/_third_party/github.com/StackExchange/wmi"
)

func queryWmi(query string, dst interface{}) error {
	return queryWmiNamespace(query, dst, "")
}

func queryWmiNamespace(query string, dst interface{}, namespace string) error {
	return wmi.QueryNamespace(query, dst, namespace)
}

func wmiInit(c *IntervalCollector, dst func() interface{}, where string, query *string) func() {
	return func() {
		*query = wmi.CreateQuery(dst(), where)
		c.Enable = func() bool {
			return queryWmi(*query, dst()) == nil
		}
	}
}

func wmiInitNamespace(c *IntervalCollector, dst func() interface{}, where string, query *string, namespace string) func() {
	return func() {
		*query = wmi.CreateQuery(dst(), where)
		c.Enable = func() bool {
			return queryWmiNamespace(*query, dst(), namespace) == nil
		}
	}
}

// wmiParseDatetime converts a string from the CIM_DATETIME format into UTC time.
// Example: "20150616101948.494497-360" = 2015-06-16 04:19:48.494497 +0000 UTC.
func wmiParseCIMDatetime(cimdatetime string) (time.Time, error) {
	i := strings.IndexAny(cimdatetime, "+-")
	if i < 0 {
		return time.Time{}, fmt.Errorf("Invalid CIM_DATETIME format, cannot find UTC offset.")
	}
	t, err := time.Parse("20060102150405", cimdatetime[:i])
	if err != nil {
		return time.Time{}, err
	}
	offset, err := time.ParseDuration(fmt.Sprintf("%vm", cimdatetime[i:]))
	if err != nil {
		return time.Time{}, err
	}
	return t.Add(offset), nil
}
<|endoftext|>


<commit_before>package runc

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/inconshreveable/log15"
	"github.com/polydawn/gosh"
	"github.com/spacemonkeygo/errors"
	"github.com/spacemonkeygo/errors/try"

	"polydawn.net/repeatr/def"
	"polydawn.net/repeatr/executor"
	"polydawn.net/repeatr/executor/basicjob"
	"polydawn.net/repeatr/executor/util"
	"polydawn.net/repeatr/io"
	"polydawn.net/repeatr/io/assets"
	"polydawn.net/repeatr/lib/flak"
	"polydawn.net/repeatr/lib/streamer"
)

// interface assertion
var _ executor.Executor = &Executor{}

type Executor struct {
	workspacePath string
}

func (e *Executor) Configure(workspacePath string) {
	e.workspacePath = workspacePath
}

func (e *Executor) Start(f def.Formula, id def.JobID, stdin io.Reader, journal io.Writer) def.Job {
	// TODO this function sig and its interface are long overdue for an aggressive refactor.
	// - `journal` is Rong.  The streams mux should be accessible after this function's scope!
	//   - either that or it's time to get cracking on saving the stream mux as an output
	// - `journal` should still be a thing, but it should be a logger.
	// - All these other values should move along in a `Job` struct
	//   - `BasicJob` sorta started, but is drunk:
	//      - if we're gonna have that, it's incomplete on the inputs
	//      - for some reason it mixes in responsibility for waiting for some of the ouputs
	//      - that use of channels and public fields is stupidly indefensive
	//   - The current `Job` interface is in the wrong package
	// - almost all of the scopes in these functions is wrong
	//   - they should be realigned until they actually assist the defers and cleanups
	//     - e.g. withErrorCapture, withJobWorkPath, withFilesystems, etc

	// Prepare the forumla for execution on this host
	def.ValidateAll(&f)

	job := basicjob.New(id)
	jobReady := make(chan struct{})

	go func() {
		// Run the formula in a temporary directory
		flak.WithDir(func(dir string) {

			// spool our output to a muxed stream
			var strm streamer.Mux
			strm = streamer.CborFileMux(filepath.Join(dir, "log"))
			outS := strm.Appender(1)
			errS := strm.Appender(2)
			job.Streams = strm
			defer func() {
				// Regardless of how the job ends (or even if it fails the remaining setup), output streams must be terminated.
				outS.Close()
				errS.Close()
			}()

			// Job is ready to stream process output
			close(jobReady)

			// Set up a logger.  Tag all messages with this jobid.
			logger := log15.New(log15.Ctx{"JobID": id})
			logger.SetHandler(log15.StreamHandler(journal, log15.TerminalFormat()))

			job.Result = e.Run(f, job, dir, stdin, outS, errS, logger)
		}, e.workspacePath, "job", string(job.Id()))

		// Directory is clean; job complete
		close(job.WaitChan)
	}()

	<-jobReady
	return job
}

// Executes a job, catching any panics.
func (e *Executor) Run(f def.Formula, j def.Job, d string, stdin io.Reader, outS, errS io.WriteCloser, journal log15.Logger) def.JobResult {
	r := def.JobResult{
		ID:       j.Id(),
		ExitCode: -1,
	}

	try.Do(func() {
		e.Execute(f, j, d, &r, outS, errS, journal)
	}).Catch(executor.Error, func(err *errors.Error) {
		r.Error = err
	}).Catch(integrity.Error, func(err *errors.Error) {
		r.Error = err
	}).CatchAll(func(err error) {
		r.Error = executor.UnknownError.Wrap(err).(*errors.Error)
	}).Done()

	return r
}

// Execute a formula in a specified directory. MAY PANIC.
func (e *Executor) Execute(formula def.Formula, job def.Job, jobPath string, result *def.JobResult, stdout, stderr io.WriteCloser, journal log15.Logger) {
	rootfsPath := filepath.Join(jobPath, "rootfs")

	// Prepare inputs
	transmat := util.DefaultTransmat()
	inputArenas := util.ProvisionInputs(transmat, formula.Inputs, journal)
	util.ProvisionOutputs(formula.Outputs, rootfsPath, journal)

	// Assemble filesystem
	assembly := util.AssembleFilesystem(
		util.BestAssembler(),
		rootfsPath,
		formula.Inputs,
		inputArenas,
		formula.Action.Escapes.Mounts,
		journal,
	)
	defer assembly.Teardown()

	// Emit configs for runc.
	runcConfigJsonPath := filepath.Join(jobPath, "config.json")
	cfg := EmitRuncConfigStruct(formula, rootfsPath)
	buf, err := json.Marshal(cfg)
	if err != nil {
		panic(executor.UnknownError.Wrap(err))
	}
	ioutil.WriteFile(runcConfigJsonPath, buf, 0600)
	runcRuntimeJsonPath := filepath.Join(jobPath, "runtime.json")
	cfg = EmitRuncRuntimeStruct(formula)
	buf, err = json.Marshal(cfg)
	if err != nil {
		panic(executor.UnknownError.Wrap(err))
	}
	ioutil.WriteFile(runcRuntimeJsonPath, buf, 0600)

	// Routing logs through a fifo appears to work, but we're going to use a file as a buffer anyway:
	//  in the event of nasty breakdowns, it's preferable that the runc log remain readable even if repeatr was the process to end first.
	logPath := filepath.Join(jobPath, "runc-debug.log")

	// Get handle to invokable runc plugin.
	runcPath := filepath.Join(assets.Get("runc"), "bin/runc")

	// Prepare command to exec
	args := []string{
		"--root", filepath.Join(e.workspacePath, "shared"), // a tmpfs would be appropriate
		"--log", logPath,
		"--log-format", "json",
		"start",
		"--config-file", runcConfigJsonPath,
		"--runtime-file", runcRuntimeJsonPath,
	}
	cmd := exec.Command(runcPath, args...)
	cmd.Stdin = nil
	cmd.Stdout = stdout
	cmd.Stderr = stderr

	// launch execution.
	// transform gosh's typed errors to repeatr's hierarchical errors.
	// this is... not untroubled code: since we're invoking a helper that's then
	//  proxying the exec even further, most errors are fatal (the mapping here is
	//   very different than in e.g. chroot executor, and provides much less meaning).
	var proc gosh.Proc
	try.Do(func() {
		proc = gosh.ExecProcCmd(cmd)
	}).CatchAll(func(err error) {
		switch err.(type) {
		case gosh.NoSuchCommandError:
			panic(executor.ConfigError.New("runc binary is missing"))
		case gosh.NoArgumentsError:
			panic(executor.UnknownError.Wrap(err))
		case gosh.NoSuchCwdError:
			panic(executor.UnknownError.Wrap(err))
		case gosh.ProcMonitorError:
			panic(executor.TaskExecError.Wrap(err))
		default:
			panic(executor.UnknownError.Wrap(err))
		}
	}).Done()

	var runcLog io.ReadCloser
	runcLog, err = os.OpenFile(logPath, os.O_CREATE|os.O_RDONLY, 0644)
	// note this open races child; doesn't matter.
	if err != nil {
		panic(executor.TaskExecError.New("failed to tail runc log: %s", err))
	}
	// swaddle the file in userland-interruptable reader;
	//  obviously we don't want to stop watching the logs when we hit the end of the still-growing file.
	runcLog = streamer.NewTailReader(runcLog)
	// close the reader when we return (which means after waiting for the exit code, which overall DTRT).
	defer runcLog.Close()

	// Proxy runc's logs out in realtime; also, detect errors and exit statuses from the stream.
	var realError error
	var someError bool // see the "NOTE WELL" section below -.-
	go func() {
		dec := json.NewDecoder(runcLog)
		for {
			var logMsg map[string]string
			err := dec.Decode(&logMsg)
			if err != nil {
				if err == io.EOF {
					return
				}
				panic(executor.TaskExecError.New("unparsable log from runc: %s", err))
			}
			// remap
			if _, ok := logMsg["msg"]; !ok {
				logMsg["msg"] = ""
			}
			ctx := log15.Ctx{}
			for k, v := range logMsg {
				if k == "msg" {
					continue
				}
				ctx["runc-"+k] = v
			}
			// with runc, everything we hear is at least a warning.
			journal.Warn(logMsg["msg"], ctx)
			// actually filtering the interesting structures and raising issues
			// note that we don't need to capture the "exit status" message, because that
			//  code *does* come out correctly... but we do need to sometimes override it again.
			// NOTE WELL: we cannot guarantee to capture all semantic runc failure modes.
			//  Errors may slip through with exit status 1: there are still many fail states
			//  which runc does not log with sufficient consistency or a sufficiently separate
			//  control channel for us to be able to reliably disambiguate them from stderr
			//  output of a successfully executing job!
			// We have whitelisted what we can; the following oddities remain:
			//   - runc will log an "exit status ${n}" message for other failures of its internal forking
			//     - this one is at least on a clear control channel, so we raise it as a panic, even if we don't know what it is
			//   - lots of system initialization paths in runc will error directly stderr with no clear sigils or separation from usermode stderr.
			//     - and these mean we're just screwed, and require additional upstream patches to address.
			switch logMsg["msg"] {
			case "Container start failed: [8] System error: no such file or directory":
				realError = executor.NoSuchCwdError.New("cannot set cwd to %q: no such file or directory", formula.Action.Cwd)
			case "Container start failed: [8] System error: not a directory":
				realError = executor.NoSuchCwdError.New("cannot set cwd to %q: not a directory", formula.Action.Cwd)
			default:
				// broader patterns required for some of these so we can ignore the vagaries of how the command name was quoted
				if strings.HasPrefix(logMsg["msg"], "Container start failed: [8] System error: exec: ") {
					if strings.HasSuffix(logMsg["msg"], ": executable file not found in $PATH") {
						realError = executor.NoSuchCommandError.New("command %q not found", formula.Action.Entrypoint[0])
					} else if strings.HasSuffix(logMsg["msg"], ": no such file or directory") {
						realError = executor.NoSuchCommandError.New("command %q not found", formula.Action.Entrypoint[0])
					}
				} else if strings.HasPrefix(logMsg["msg"], "exit status ") {
					// by itself this doesn't mean much, but if we can't figure out
					//  what happened in particular by the proc exit, be quite alarmed.
					// and no, this does *not* mean the real job process exit code, as you might expect.
					someError = true
				}
			}
		}
	}()

	// Wait for the job to complete
	result.ExitCode = proc.GetExitCode()
	//runcLog.Close() // this could/should happen before PreserveOutputs.  see todo about fixing scopes.

	// If we had a CnC error (rather than the real subprocess exit code):
	//  - reset code to -1 because the runc exit code wasn't really from the job command
	//  - zero the output buffers because runc (again) doesn't understand what control channels are
	//  - finally, raise the error
	if someError && realError == nil {
		realError = executor.UnknownError.New("runc errored in an unrecognized fashion")
	}
	if realError != nil {
		result.ExitCode = -1
		// TODO just overwrite the streamer, i guess?
		panic(realError)
	}

	// Save outputs
	result.Outputs = util.PreserveOutputs(transmat, formula.Outputs, rootfsPath, journal)
}
<commit_msg>Handle all log from runc and collect goroutines.<commit_after>package runc

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	"github.com/inconshreveable/log15"
	"github.com/polydawn/gosh"
	"github.com/spacemonkeygo/errors"
	"github.com/spacemonkeygo/errors/try"

	"polydawn.net/repeatr/def"
	"polydawn.net/repeatr/executor"
	"polydawn.net/repeatr/executor/basicjob"
	"polydawn.net/repeatr/executor/util"
	"polydawn.net/repeatr/io"
	"polydawn.net/repeatr/io/assets"
	"polydawn.net/repeatr/lib/flak"
	"polydawn.net/repeatr/lib/streamer"
)

// interface assertion
var _ executor.Executor = &Executor{}

type Executor struct {
	workspacePath string
}

func (e *Executor) Configure(workspacePath string) {
	e.workspacePath = workspacePath
}

func (e *Executor) Start(f def.Formula, id def.JobID, stdin io.Reader, journal io.Writer) def.Job {
	// TODO this function sig and its interface are long overdue for an aggressive refactor.
	// - `journal` is Rong.  The streams mux should be accessible after this function's scope!
	//   - either that or it's time to get cracking on saving the stream mux as an output
	// - `journal` should still be a thing, but it should be a logger.
	// - All these other values should move along in a `Job` struct
	//   - `BasicJob` sorta started, but is drunk:
	//      - if we're gonna have that, it's incomplete on the inputs
	//      - for some reason it mixes in responsibility for waiting for some of the ouputs
	//      - that use of channels and public fields is stupidly indefensive
	//   - The current `Job` interface is in the wrong package
	// - almost all of the scopes in these functions is wrong
	//   - they should be realigned until they actually assist the defers and cleanups
	//     - e.g. withErrorCapture, withJobWorkPath, withFilesystems, etc

	// Prepare the forumla for execution on this host
	def.ValidateAll(&f)

	job := basicjob.New(id)
	jobReady := make(chan struct{})

	go func() {
		// Run the formula in a temporary directory
		flak.WithDir(func(dir string) {

			// spool our output to a muxed stream
			var strm streamer.Mux
			strm = streamer.CborFileMux(filepath.Join(dir, "log"))
			outS := strm.Appender(1)
			errS := strm.Appender(2)
			job.Streams = strm
			defer func() {
				// Regardless of how the job ends (or even if it fails the remaining setup), output streams must be terminated.
				outS.Close()
				errS.Close()
			}()

			// Job is ready to stream process output
			close(jobReady)

			// Set up a logger.  Tag all messages with this jobid.
			logger := log15.New(log15.Ctx{"JobID": id})
			logger.SetHandler(log15.StreamHandler(journal, log15.TerminalFormat()))

			job.Result = e.Run(f, job, dir, stdin, outS, errS, logger)
		}, e.workspacePath, "job", string(job.Id()))

		// Directory is clean; job complete
		close(job.WaitChan)
	}()

	<-jobReady
	return job
}

// Executes a job, catching any panics.
func (e *Executor) Run(f def.Formula, j def.Job, d string, stdin io.Reader, outS, errS io.WriteCloser, journal log15.Logger) def.JobResult {
	r := def.JobResult{
		ID:       j.Id(),
		ExitCode: -1,
	}

	try.Do(func() {
		e.Execute(f, j, d, &r, outS, errS, journal)
	}).Catch(executor.Error, func(err *errors.Error) {
		r.Error = err
	}).Catch(integrity.Error, func(err *errors.Error) {
		r.Error = err
	}).CatchAll(func(err error) {
		r.Error = executor.UnknownError.Wrap(err).(*errors.Error)
	}).Done()

	return r
}

// Execute a formula in a specified directory. MAY PANIC.
func (e *Executor) Execute(formula def.Formula, job def.Job, jobPath string, result *def.JobResult, stdout, stderr io.WriteCloser, journal log15.Logger) {
	rootfsPath := filepath.Join(jobPath, "rootfs")

	// Prepare inputs
	transmat := util.DefaultTransmat()
	inputArenas := util.ProvisionInputs(transmat, formula.Inputs, journal)
	util.ProvisionOutputs(formula.Outputs, rootfsPath, journal)

	// Assemble filesystem
	assembly := util.AssembleFilesystem(
		util.BestAssembler(),
		rootfsPath,
		formula.Inputs,
		inputArenas,
		formula.Action.Escapes.Mounts,
		journal,
	)
	defer assembly.Teardown()

	// Emit configs for runc.
	runcConfigJsonPath := filepath.Join(jobPath, "config.json")
	cfg := EmitRuncConfigStruct(formula, rootfsPath)
	buf, err := json.Marshal(cfg)
	if err != nil {
		panic(executor.UnknownError.Wrap(err))
	}
	ioutil.WriteFile(runcConfigJsonPath, buf, 0600)
	runcRuntimeJsonPath := filepath.Join(jobPath, "runtime.json")
	cfg = EmitRuncRuntimeStruct(formula)
	buf, err = json.Marshal(cfg)
	if err != nil {
		panic(executor.UnknownError.Wrap(err))
	}
	ioutil.WriteFile(runcRuntimeJsonPath, buf, 0600)

	// Routing logs through a fifo appears to work, but we're going to use a file as a buffer anyway:
	//  in the event of nasty breakdowns, it's preferable that the runc log remain readable even if repeatr was the process to end first.
	logPath := filepath.Join(jobPath, "runc-debug.log")

	// Get handle to invokable runc plugin.
	runcPath := filepath.Join(assets.Get("runc"), "bin/runc")

	// Prepare command to exec
	args := []string{
		"--root", filepath.Join(e.workspacePath, "shared"), // a tmpfs would be appropriate
		"--log", logPath,
		"--log-format", "json",
		"start",
		"--config-file", runcConfigJsonPath,
		"--runtime-file", runcRuntimeJsonPath,
	}
	cmd := exec.Command(runcPath, args...)
	cmd.Stdin = nil
	cmd.Stdout = stdout
	cmd.Stderr = stderr

	// launch execution.
	// transform gosh's typed errors to repeatr's hierarchical errors.
	// this is... not untroubled code: since we're invoking a helper that's then
	//  proxying the exec even further, most errors are fatal (the mapping here is
	//   very different than in e.g. chroot executor, and provides much less meaning).
	var proc gosh.Proc
	try.Do(func() {
		proc = gosh.ExecProcCmd(cmd)
	}).CatchAll(func(err error) {
		switch err.(type) {
		case gosh.NoSuchCommandError:
			panic(executor.ConfigError.New("runc binary is missing"))
		case gosh.NoArgumentsError:
			panic(executor.UnknownError.Wrap(err))
		case gosh.NoSuchCwdError:
			panic(executor.UnknownError.Wrap(err))
		case gosh.ProcMonitorError:
			panic(executor.TaskExecError.Wrap(err))
		default:
			panic(executor.UnknownError.Wrap(err))
		}
	}).Done()

	var runcLog io.ReadCloser
	runcLog, err = os.OpenFile(logPath, os.O_CREATE|os.O_RDONLY, 0644)
	// note this open races child; doesn't matter.
	if err != nil {
		panic(executor.TaskExecError.New("failed to tail runc log: %s", err))
	}
	// swaddle the file in userland-interruptable reader;
	//  obviously we don't want to stop watching the logs when we hit the end of the still-growing file.
	runcLog = streamer.NewTailReader(runcLog)

	// Proxy runc's logs out in realtime; also, detect errors and exit statuses from the stream.
	var realError error
	var someError bool // see the "NOTE WELL" section below -.-
	var tailerDone sync.WaitGroup
	tailerDone.Add(1)
	go func() {
		defer tailerDone.Done()
		dec := json.NewDecoder(runcLog)
		for {
			var logMsg map[string]string
			err := dec.Decode(&logMsg)
			if err != nil {
				if err == io.EOF {
					return
				}
				panic(executor.TaskExecError.New("unparsable log from runc: %s", err))
			}
			// remap
			if _, ok := logMsg["msg"]; !ok {
				logMsg["msg"] = ""
			}
			ctx := log15.Ctx{}
			for k, v := range logMsg {
				if k == "msg" {
					continue
				}
				ctx["runc-"+k] = v
			}
			// with runc, everything we hear is at least a warning.
			journal.Warn(logMsg["msg"], ctx)
			// actually filtering the interesting structures and raising issues
			// note that we don't need to capture the "exit status" message, because that
			//  code *does* come out correctly... but we do need to sometimes override it again.
			// NOTE WELL: we cannot guarantee to capture all semantic runc failure modes.
			//  Errors may slip through with exit status 1: there are still many fail states
			//  which runc does not log with sufficient consistency or a sufficiently separate
			//  control channel for us to be able to reliably disambiguate them from stderr
			//  output of a successfully executing job!
			// We have whitelisted what we can; the following oddities remain:
			//   - runc will log an "exit status ${n}" message for other failures of its internal forking
			//     - this one is at least on a clear control channel, so we raise it as a panic, even if we don't know what it is
			//   - lots of system initialization paths in runc will error directly stderr with no clear sigils or separation from usermode stderr.
			//     - and these mean we're just screwed, and require additional upstream patches to address.
			switch logMsg["msg"] {
			case "Container start failed: [8] System error: no such file or directory":
				realError = executor.NoSuchCwdError.New("cannot set cwd to %q: no such file or directory", formula.Action.Cwd)
			case "Container start failed: [8] System error: not a directory":
				realError = executor.NoSuchCwdError.New("cannot set cwd to %q: not a directory", formula.Action.Cwd)
			default:
				// broader patterns required for some of these so we can ignore the vagaries of how the command name was quoted
				if strings.HasPrefix(logMsg["msg"], "Container start failed: [8] System error: exec: ") {
					if strings.HasSuffix(logMsg["msg"], ": executable file not found in $PATH") {
						realError = executor.NoSuchCommandError.New("command %q not found", formula.Action.Entrypoint[0])
					} else if strings.HasSuffix(logMsg["msg"], ": no such file or directory") {
						realError = executor.NoSuchCommandError.New("command %q not found", formula.Action.Entrypoint[0])
					}
				} else if strings.HasPrefix(logMsg["msg"], "exit status ") {
					// by itself this doesn't mean much, but if we can't figure out
					//  what happened in particular by the proc exit, be quite alarmed.
					// and no, this does *not* mean the real job process exit code, as you might expect.
					someError = true
				}
			}
		}
	}()

	// Wait for the job to complete.
	// Tell the log tailer to drain as soon as the proc exits.
	result.ExitCode = proc.GetExitCode()
	runcLog.Close()

	// Wait for the tailer routine to drain & exit (this sync guards the err vars).
	tailerDone.Wait()

	// If we had a CnC error (rather than the real subprocess exit code):
	//  - reset code to -1 because the runc exit code wasn't really from the job command
	//  - finally, raise the error
	// FIXME we WISH we could zero the output buffers because runc pushes duplicate error messages
	//  down a channel that's indistinguishable from the application stderr... but that's tricky for several reasons:
	//  - we support streaming them out, right?
	//  - that means we'd have to have been blocking them already; we can't zero retroactively.
	//  - there's no "all clear" signal available from runc that would let us know we're clear to start flushing the stream if we blocked it.
	//  - So, we're unable to pass the executor compat tests until patches to runc clean up this behavior.
	if someError && realError == nil {
		realError = executor.UnknownError.New("runc errored in an unrecognized fashion")
	}
	if realError != nil {
		result.ExitCode = -1
		panic(realError)
	}

	// Save outputs
	result.Outputs = util.PreserveOutputs(transmat, formula.Outputs, rootfsPath, journal)
}
<|endoftext|>


<commit_before>package jsonapi

import (
	"encoding/json"
	"fmt"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type PublishStatus int

const (
	StatusUnpublished PublishStatus = iota
	StatusPublished
)

var publishStatusValues = []string{
	StatusUnpublished: "unpublished",
	StatusPublished:   "published",
}

func (s PublishStatus) String() string {
	if s < 0 || int(s) >= len(publishStatusValues) {
		panic("value out of range")
	}
	return publishStatusValues[s]
}

// MarshalText implements the TextMarshaler interface.
func (s PublishStatus) MarshalText() (text []byte, err error) {
	return []byte(s.String()), nil
}

// UnmarshalText implements the TextUnmarshaler interface.
func (s *PublishStatus) UnmarshalText(text []byte) error {
	var label string
	json.Unmarshal(text, &label)

	for key, v := range publishStatusValues {
		if v == label {
			*s = PublishStatus(key)
			return nil
		}
	}

	return fmt.Errorf("invalid value `%s`", label)
}

func (s *PublishStatus) UnmarshalJSON(data []byte) error {
	return s.UnmarshalText(data)
}

type EnumPost struct {
	ID     string `json:"-"`
	Title  string
	Status PublishStatus
}

func (e EnumPost) GetID() string {
	return e.ID
}

func (e *EnumPost) SetID(ID string) error {
	e.ID = ID

	return nil
}

var _ = Describe("Unmarshal", func() {
	Context("When unmarshaling objects including enums", func() {
		status := StatusPublished
		statusValue := "\"published\""
		singleJSON := []byte(`{"data":{"id": "1", "type": "enumPosts", "attributes": {"title":"First Post","status":"published"}}}`)
		firstPost := EnumPost{ID: "1", Title: "First Post", Status: StatusPublished}
		singlePostMap := map[string]interface{}{
			"data": map[string]interface{}{
				"id":   "1",
				"type": "enumPosts",
				"attributes": map[string]interface{}{
					"title":  firstPost.Title,
					"status": StatusPublished,
				},
			},
		}

		It("unmarshals status string values to int enum type", func() {
			var result PublishStatus
			result.UnmarshalText([]byte(statusValue))
			Expect(result).To(Equal(status))
		})

		It("unmarshals single objects into a struct", func() {
			var post EnumPost
			err := Unmarshal(singlePostMap, &post)
			Expect(err).To(BeNil())
			Expect(post).To(Equal(firstPost))
		})

		It("unmarshals JSON", func() {
			var posts []EnumPost
			err := UnmarshalFromJSON(singleJSON, &posts)
			Expect(err).To(BeNil())
			Expect(posts).To(Equal([]EnumPost{firstPost}))
		})
	})
})
<commit_msg>add a marshal test for the enum bug too<commit_after>package jsonapi

import (
	"encoding/json"
	"fmt"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type PublishStatus int

const (
	StatusUnpublished PublishStatus = iota
	StatusPublished
)

var publishStatusValues = []string{
	StatusUnpublished: "unpublished",
	StatusPublished:   "published",
}

func (s PublishStatus) String() string {
	if s < 0 || int(s) >= len(publishStatusValues) {
		panic("value out of range")
	}
	return publishStatusValues[s]
}

// MarshalText implements the TextMarshaler interface.
func (s PublishStatus) MarshalText() (text []byte, err error) {
	return []byte(s.String()), nil
}

// UnmarshalText implements the TextUnmarshaler interface.
func (s *PublishStatus) UnmarshalText(text []byte) error {
	var label string
	json.Unmarshal(text, &label)

	for key, v := range publishStatusValues {
		if v == label {
			*s = PublishStatus(key)
			return nil
		}
	}

	return fmt.Errorf("invalid value `%s`", label)
}

func (s *PublishStatus) UnmarshalJSON(data []byte) error {
	return s.UnmarshalText(data)
}

type EnumPost struct {
	ID     string `json:"-"`
	Title  string
	Status PublishStatus
}

func (e EnumPost) GetID() string {
	return e.ID
}

func (e *EnumPost) SetID(ID string) error {
	e.ID = ID

	return nil
}

var _ = Describe("Custom enum types", func() {
	status := StatusPublished
	statusValue := "\"published\""
	singleJSON := []byte(`{"data":{"id": "1", "type": "enumPosts", "attributes": {"title":"First Post","status":"published"}}}`)
	firstPost := EnumPost{ID: "1", Title: "First Post", Status: StatusPublished}
	singlePostMap := map[string]interface{}{
		"data": map[string]interface{}{
			"id":   "1",
			"type": "enumPosts",
			"attributes": map[string]interface{}{
				"title":  firstPost.Title,
				"status": StatusPublished,
			},
		},
	}

	Context("When marshaling objects including enumes", func() {
		singlePost := EnumPost{
			ID:     "1",
			Title:  "First Post",
			Status: StatusPublished,
		}

		It("marshals JSON", func() {
			result, err := MarshalToJSON(singlePost)
			Expect(err).ToNot(HaveOccurred())
			Expect(result).To(MatchJSON(singleJSON))
		})
	})

	Context("When unmarshaling objects including enums", func() {

		It("unmarshals status string values to int enum type", func() {
			var result PublishStatus
			result.UnmarshalText([]byte(statusValue))
			Expect(result).To(Equal(status))
		})

		It("unmarshals single objects into a struct", func() {
			var post EnumPost
			err := Unmarshal(singlePostMap, &post)
			Expect(err).ToNot(HaveOccurred())
			Expect(post).To(Equal(firstPost))
		})

		It("unmarshals JSON", func() {
			var posts []EnumPost
			err := UnmarshalFromJSON(singleJSON, &posts)
			Expect(err).ToNot(HaveOccurred())
			Expect(posts).To(Equal([]EnumPost{firstPost}))
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/vanng822/r2router"
	"github.com/vanng822/uploader"
	"github.com/vanng822/uploader/storage/mongodb"
	"github.com/vanng822/uploader/storage/s3"
	"net/http"
	"os"
)

const (
	STORAGE_TYPE_FILE    = "file"
	STORAGE_TYPE_MONGODB = "mongodb"
	STORAGE_TYPE_S3      = "s3"
)

type StorageConfig struct {
	Type           string
	Configurations map[string]string
}

var storage uploader.ImageStorage
	switch config.Type {
	case STORAGE_TYPE_FILE:
		storage = uploader.NewImageStorageFile(config.Configurations)
	case STORAGE_TYPE_MONGODB:
		storage = mongodb.New(config.Configurations)
	case STORAGE_TYPE_S3:
		storage = s3.New(config.Configurations)
	default:
		panic(fmt.Sprintf("Unsupported storage type %s", config.Type))
	}
	return storage

type EndpointConfig struct {
	Endpoint  string
	FileField string
	Storage   *StorageConfig
}

type Config struct {
	Host      string
	Port      int
	Endpoints []*EndpointConfig
}

func LoadConfig(filename string) *Config {
	file, err := os.Open(filename)
	if err != nil {
		panic(err)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	conf := Config{}
	err = decoder.Decode(&conf)
	if err != nil {
		panic(err)
	}
	return &conf
}

func main() {

	var (
		config string
		host   string
		port   int
	)

	flag.StringVar(&host, "h", "", "Host to bind to")
	flag.IntVar(&port, "p", 0, "Port number to listen on")
	flag.StringVar(&config, "c", "./config/app.json", "Path to configurations")
	flag.Parse()

	conf := LoadConfig(config)

	if host != "" {
		conf.Host = host
	}

	if port != 0 {
		conf.Port = port
	}

	if len(conf.Endpoints) == 0 {
		panic("There is no endpoint configured")
	}

	app := r2router.NewSeeforRouter()

	for _, endpoint := range conf.Endpoints {

		go func(endpoint *EndpointConfig) {
			u := uploader.NewUploader(GetStorage(endpoint.Storage))
			handler := uploader.NewHandler(u)

			app.Group(endpoint.Endpoint, func(r *r2router.GroupRouter) {
				r.Get("/:filename", func(res http.ResponseWriter, req *http.Request, params r2router.Params) {
					handler.HandleGet(res, params.Get("filename"))
				})
				r.Post("/", func(res http.ResponseWriter, req *http.Request, params r2router.Params) {
					file, _, err := req.FormFile(endpoint.FileField)
					if err != nil {
						res.WriteHeader(http.StatusBadRequest)
						return
					}
					handler.HandlePost(res, file)
				})
				r.Put("/:filename", func(res http.ResponseWriter, req *http.Request, params r2router.Params) {
					file, _, err := req.FormFile(endpoint.FileField)
					if err != nil {
						res.WriteHeader(http.StatusBadRequest)
						return
					}
					handler.HandlePut(res, file, params.Get("filename"))
				})
				r.Delete("/:filename", func(res http.ResponseWriter, req *http.Request, params r2router.Params) {
					handler.HandleDelete(res, params.Get("filename"))
				})
			})
		}(endpoint)
	}

	http.ListenAndServe(fmt.Sprintf("%s:%d", conf.Host, conf.Port), app)
}<commit_msg>Router is not multi goroutines safe. Register each endpoint at the time<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/vanng822/r2router"
	"github.com/vanng822/uploader"
	"github.com/vanng822/uploader/storage/mongodb"
	"github.com/vanng822/uploader/storage/s3"
	"net/http"
	"os"
)

const (
	STORAGE_TYPE_FILE    = "file"
	STORAGE_TYPE_MONGODB = "mongodb"
	STORAGE_TYPE_S3      = "s3"
)

type StorageConfig struct {
	Type           string
	Configurations map[string]string
}

var storage uploader.ImageStorage
	switch config.Type {
	case STORAGE_TYPE_FILE:
		storage = uploader.NewImageStorageFile(config.Configurations)
	case STORAGE_TYPE_MONGODB:
		storage = mongodb.New(config.Configurations)
	case STORAGE_TYPE_S3:
		storage = s3.New(config.Configurations)
	default:
		panic(fmt.Sprintf("Unsupported storage type %s", config.Type))
	}
	return storage

type EndpointConfig struct {
	Endpoint  string
	FileField string
	Storage   *StorageConfig
}

type Config struct {
	Host      string
	Port      int
	Endpoints []*EndpointConfig
}

func LoadConfig(filename string) *Config {
	file, err := os.Open(filename)
	if err != nil {
		panic(err)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	conf := Config{}
	err = decoder.Decode(&conf)
	if err != nil {
		panic(err)
	}
	return &conf
}

func main() {

	var (
		config string
		host   string
		port   int
	)

	flag.StringVar(&host, "h", "", "Host to bind to")
	flag.IntVar(&port, "p", 0, "Port number to listen on")
	flag.StringVar(&config, "c", "./config/app.json", "Path to configurations")
	flag.Parse()

	conf := LoadConfig(config)

	if host != "" {
		conf.Host = host
	}

	if port != 0 {
		conf.Port = port
	}

	if len(conf.Endpoints) == 0 {
		panic("There is no endpoint configured")
	}

	app := r2router.NewSeeforRouter()

	for _, endpoint := range conf.Endpoints {
		func(endpoint *EndpointConfig) {
			u := uploader.NewUploader(GetStorage(endpoint.Storage))
			handler := uploader.NewHandler(u)

			app.Group(endpoint.Endpoint, func(r *r2router.GroupRouter) {
				r.Get("/:filename", func(res http.ResponseWriter, req *http.Request, params r2router.Params) {
					handler.HandleGet(res, params.Get("filename"))
				})
				r.Post("/", func(res http.ResponseWriter, req *http.Request, params r2router.Params) {
					file, _, err := req.FormFile(endpoint.FileField)
					if err != nil {
						res.WriteHeader(http.StatusBadRequest)
						return
					}
					handler.HandlePost(res, file)
				})
				r.Put("/:filename", func(res http.ResponseWriter, req *http.Request, params r2router.Params) {
					file, _, err := req.FormFile(endpoint.FileField)
					if err != nil {
						res.WriteHeader(http.StatusBadRequest)
						return
					}
					handler.HandlePut(res, file, params.Get("filename"))
				})
				r.Delete("/:filename", func(res http.ResponseWriter, req *http.Request, params r2router.Params) {
					handler.HandleDelete(res, params.Get("filename"))
				})
			})
		}(endpoint)
	}

	http.ListenAndServe(fmt.Sprintf("%s:%d", conf.Host, conf.Port), app)
}<|endoftext|>