<commit_before>package deploylog

import (
	"fmt"
	"sort"
	"time"

	"github.com/golang/glog"
	kapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/rest"
	"k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/controller"
	kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
	"k8s.io/kubernetes/pkg/labels"
	genericrest "k8s.io/kubernetes/pkg/registry/generic/rest"
	"k8s.io/kubernetes/pkg/registry/pod"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/wait"

	"github.com/openshift/origin/pkg/client"
	deployapi "github.com/openshift/origin/pkg/deploy/api"
	"github.com/openshift/origin/pkg/deploy/api/validation"
	"github.com/openshift/origin/pkg/deploy/registry"
	deployutil "github.com/openshift/origin/pkg/deploy/util"
)

const (
	// defaultTimeout is the default time to wait for the logs of a deployment.
	defaultTimeout time.Duration = 20 * time.Second
	// defaultInterval is the default interval for polling a not found deployment.
	defaultInterval time.Duration = 1 * time.Second
)

// podGetter implements the ResourceGetter interface. Used by LogLocation to
// retrieve the deployer pod
type podGetter struct {
	pn unversioned.PodsNamespacer
}

// Get is responsible for retrieving the deployer pod
func (g *podGetter) Get(ctx kapi.Context, name string) (runtime.Object, error) {
	namespace, ok := kapi.NamespaceFrom(ctx)
	if !ok {
		return nil, errors.NewBadRequest("namespace parameter required.")
	}
	return g.pn.Pods(namespace).Get(name)
}

// REST is an implementation of RESTStorage for the api server.
type REST struct {
	dn       client.DeploymentConfigsNamespacer
	rn       unversioned.ReplicationControllersNamespacer
	pn       unversioned.PodsNamespacer
	connInfo kubeletclient.ConnectionInfoGetter
	timeout  time.Duration
	interval time.Duration
}

// REST implements GetterWithOptions
var _ = rest.GetterWithOptions(&REST{})

// NewREST creates a new REST for DeploymentLogs. It uses three clients: one for configs,
// one for deployments (replication controllers) and one for pods to get the necessary
// attributes to assemble the URL to which the request shall be redirected in order to
// get the deployment logs.
func NewREST(dn client.DeploymentConfigsNamespacer, rn unversioned.ReplicationControllersNamespacer, pn unversioned.PodsNamespacer, connectionInfo kubeletclient.ConnectionInfoGetter) *REST {
	return &REST{
		dn:       dn,
		rn:       rn,
		pn:       pn,
		connInfo: connectionInfo,
		timeout:  defaultTimeout,
		interval: defaultInterval,
	}
}

// NewGetOptions returns a new options object for deployment logs
func (r *REST) NewGetOptions() (runtime.Object, bool, string) {
	return &deployapi.DeploymentLogOptions{}, false, ""
}

// New creates an empty DeploymentLog resource
func (r *REST) New() runtime.Object {
	return &deployapi.DeploymentLog{}
}

// Get returns a streamer resource with the contents of the deployment log
func (r *REST) Get(ctx kapi.Context, name string, opts runtime.Object) (runtime.Object, error) {
	// Ensure we have a namespace in the context
	namespace, ok := kapi.NamespaceFrom(ctx)
	if !ok {
		return nil, errors.NewBadRequest("namespace parameter required.")
	}

	// Validate DeploymentLogOptions
	deployLogOpts, ok := opts.(*deployapi.DeploymentLogOptions)
	if !ok {
		return nil, errors.NewBadRequest("did not get an expected options.")
	}
	if errs := validation.ValidateDeploymentLogOptions(deployLogOpts); len(errs) > 0 {
		return nil, errors.NewInvalid(deployapi.Kind("DeploymentLogOptions"), "", errs)
	}

	// Fetch deploymentConfig and check latest version; if 0, there are no deployments
	// for this config
	config, err := r.dn.DeploymentConfigs(namespace).Get(name)
	if err != nil {
		return nil, errors.NewNotFound(deployapi.Resource("deploymentconfig"), name)
	}
	desiredVersion := config.Status.LatestVersion
	if desiredVersion == 0 {
		return nil, errors.NewBadRequest(fmt.Sprintf("no deployment exists for deploymentConfig %q", config.Name))
	}

	// Support retrieving logs for older deployments
	switch {
	case deployLogOpts.Version == nil:
		// Latest or previous
		if deployLogOpts.Previous {
			desiredVersion--
			if desiredVersion < 1 {
				return nil, errors.NewBadRequest(fmt.Sprintf("no previous deployment exists for deploymentConfig %q", config.Name))
			}
		}
	case *deployLogOpts.Version <= 0 || *deployLogOpts.Version > config.Status.LatestVersion:
		// Invalid version
		return nil, errors.NewBadRequest(fmt.Sprintf("invalid version for deploymentConfig %q: %d", config.Name, *deployLogOpts.Version))
	default:
		desiredVersion = *deployLogOpts.Version
	}

	// Get desired deployment
	targetName := deployutil.DeploymentNameForConfigVersion(config.Name, desiredVersion)
	target, err := r.waitForExistingDeployment(namespace, targetName)
	if err != nil {
		return nil, err
	}
	podName := deployutil.DeployerPodNameForDeployment(target.Name)

	// Check for deployment status; if it is new or pending, we will wait for it. If it is complete,
	// the deployment completed successfully and the deployer pod will be deleted so we will return a
	// success message. If it is running or failed, retrieve the log from the deployer pod.
	status := deployutil.DeploymentStatusFor(target)
	switch status {
	case deployapi.DeploymentStatusNew, deployapi.DeploymentStatusPending:
		if deployLogOpts.NoWait {
			glog.V(4).Infof("Deployment %s is in %s state. No logs to retrieve yet.", deployutil.LabelForDeployment(target), status)
			return &genericrest.LocationStreamer{}, nil
		}
		glog.V(4).Infof("Deployment %s is in %s state, waiting for it to start...", deployutil.LabelForDeployment(target), status)

		if err := deployutil.WaitForRunningDeployerPod(r.pn, target, r.timeout); err != nil {
			return nil, errors.NewBadRequest(fmt.Sprintf("failed to run deployer pod %s: %v", podName, err))
		}

		latest, ok, err := registry.WaitForRunningDeployment(r.rn, target, r.timeout)
		if err != nil {
			return nil, errors.NewBadRequest(fmt.Sprintf("unable to wait for deployment %s to run: %v", deployutil.LabelForDeployment(target), err))
		}
		if !ok {
			return nil, errors.NewServerTimeout(kapi.Resource("ReplicationController"), "get", 2)
		}
		if deployutil.IsCompleteDeployment(latest) {
			podName, err = r.returnApplicationPodName(target)
			if err != nil {
				return nil, err
			}
		}
	case deployapi.DeploymentStatusComplete:
		podName, err = r.returnApplicationPodName(target)
		if err != nil {
			return nil, err
		}
	}

	logOpts := deployapi.DeploymentToPodLogOptions(deployLogOpts)
	location, transport, err := pod.LogLocation(&podGetter{r.pn}, r.connInfo, ctx, podName, logOpts)
	if err != nil {
		return nil, errors.NewBadRequest(err.Error())
	}

	return &genericrest.LocationStreamer{
		Location:        location,
		Transport:       transport,
		ContentType:     "text/plain",
		Flush:           deployLogOpts.Follow,
		ResponseChecker: genericrest.NewGenericHttpResponseChecker(kapi.Resource("pod"), podName),
	}, nil
}

// waitForExistingDeployment will use the timeout to wait for a deployment to appear.
func (r *REST) waitForExistingDeployment(namespace, name string) (*kapi.ReplicationController, error) {
	var (
		target *kapi.ReplicationController
		err    error
	)

	condition := func() (bool, error) {
		target, err = r.rn.ReplicationControllers(namespace).Get(name)
		switch {
		case errors.IsNotFound(err):
			return false, nil
		case err != nil:
			return false, err
		}
		return true, nil
	}

	err = wait.PollImmediate(r.interval, r.timeout, condition)
	if err == wait.ErrWaitTimeout {
		err = errors.NewNotFound(kapi.Resource("replicationcontrollers"), name)
	}
	return target, err
}

// returnApplicationPodName returns the best candidate pod for the target deployment in order to
// view its logs.
func (r *REST) returnApplicationPodName(target *kapi.ReplicationController) (string, error) {
	selector := labels.Set(target.Spec.Selector).AsSelector()
	sortBy := func(pods []*kapi.Pod) sort.Interface { return controller.ByLogging(pods) }

	pod, _, err := kcmdutil.GetFirstPod(r.pn, target.Namespace, selector, r.timeout, sortBy)
	if err != nil {
		return "", errors.NewInternalError(err)
	}
	return pod.Name, nil
}
<commit_msg>Bump the timeout for deployment logs<commit_after>package deploylog

import (
	"fmt"
	"sort"
	"time"

	"github.com/golang/glog"
	kapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/rest"
	"k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/controller"
	kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
	"k8s.io/kubernetes/pkg/labels"
	genericrest "k8s.io/kubernetes/pkg/registry/generic/rest"
	"k8s.io/kubernetes/pkg/registry/pod"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/wait"

	"github.com/openshift/origin/pkg/client"
	deployapi "github.com/openshift/origin/pkg/deploy/api"
	"github.com/openshift/origin/pkg/deploy/api/validation"
	"github.com/openshift/origin/pkg/deploy/registry"
	deployutil "github.com/openshift/origin/pkg/deploy/util"
)

const (
	// defaultTimeout is the default time to wait for the logs of a deployment.
	defaultTimeout time.Duration = 60 * time.Second
	// defaultInterval is the default interval for polling a not found deployment.
	defaultInterval time.Duration = 1 * time.Second
)

// podGetter implements the ResourceGetter interface. Used by LogLocation to
// retrieve the deployer pod
type podGetter struct {
	pn unversioned.PodsNamespacer
}

// Get is responsible for retrieving the deployer pod
func (g *podGetter) Get(ctx kapi.Context, name string) (runtime.Object, error) {
	namespace, ok := kapi.NamespaceFrom(ctx)
	if !ok {
		return nil, errors.NewBadRequest("namespace parameter required.")
	}
	return g.pn.Pods(namespace).Get(name)
}

// REST is an implementation of RESTStorage for the api server.
type REST struct {
	dn       client.DeploymentConfigsNamespacer
	rn       unversioned.ReplicationControllersNamespacer
	pn       unversioned.PodsNamespacer
	connInfo kubeletclient.ConnectionInfoGetter
	timeout  time.Duration
	interval time.Duration
}

// REST implements GetterWithOptions
var _ = rest.GetterWithOptions(&REST{})

// NewREST creates a new REST for DeploymentLogs. It uses three clients: one for configs,
// one for deployments (replication controllers) and one for pods to get the necessary
// attributes to assemble the URL to which the request shall be redirected in order to
// get the deployment logs.
func NewREST(dn client.DeploymentConfigsNamespacer, rn unversioned.ReplicationControllersNamespacer, pn unversioned.PodsNamespacer, connectionInfo kubeletclient.ConnectionInfoGetter) *REST {
	return &REST{
		dn:       dn,
		rn:       rn,
		pn:       pn,
		connInfo: connectionInfo,
		timeout:  defaultTimeout,
		interval: defaultInterval,
	}
}

// NewGetOptions returns a new options object for deployment logs
func (r *REST) NewGetOptions() (runtime.Object, bool, string) {
	return &deployapi.DeploymentLogOptions{}, false, ""
}

// New creates an empty DeploymentLog resource
func (r *REST) New() runtime.Object {
	return &deployapi.DeploymentLog{}
}

// Get returns a streamer resource with the contents of the deployment log
func (r *REST) Get(ctx kapi.Context, name string, opts runtime.Object) (runtime.Object, error) {
	// Ensure we have a namespace in the context
	namespace, ok := kapi.NamespaceFrom(ctx)
	if !ok {
		return nil, errors.NewBadRequest("namespace parameter required.")
	}

	// Validate DeploymentLogOptions
	deployLogOpts, ok := opts.(*deployapi.DeploymentLogOptions)
	if !ok {
		return nil, errors.NewBadRequest("did not get an expected options.")
	}
	if errs := validation.ValidateDeploymentLogOptions(deployLogOpts); len(errs) > 0 {
		return nil, errors.NewInvalid(deployapi.Kind("DeploymentLogOptions"), "", errs)
	}

	// Fetch deploymentConfig and check latest version; if 0, there are no deployments
	// for this config
	config, err := r.dn.DeploymentConfigs(namespace).Get(name)
	if err != nil {
		return nil, errors.NewNotFound(deployapi.Resource("deploymentconfig"), name)
	}
	desiredVersion := config.Status.LatestVersion
	if desiredVersion == 0 {
		return nil, errors.NewBadRequest(fmt.Sprintf("no deployment exists for deploymentConfig %q", config.Name))
	}

	// Support retrieving logs for older deployments
	switch {
	case deployLogOpts.Version == nil:
		// Latest or previous
		if deployLogOpts.Previous {
			desiredVersion--
			if desiredVersion < 1 {
				return nil, errors.NewBadRequest(fmt.Sprintf("no previous deployment exists for deploymentConfig %q", config.Name))
			}
		}
	case *deployLogOpts.Version <= 0 || *deployLogOpts.Version > config.Status.LatestVersion:
		// Invalid version
		return nil, errors.NewBadRequest(fmt.Sprintf("invalid version for deploymentConfig %q: %d", config.Name, *deployLogOpts.Version))
	default:
		desiredVersion = *deployLogOpts.Version
	}

	// Get desired deployment
	targetName := deployutil.DeploymentNameForConfigVersion(config.Name, desiredVersion)
	target, err := r.waitForExistingDeployment(namespace, targetName)
	if err != nil {
		return nil, err
	}
	podName := deployutil.DeployerPodNameForDeployment(target.Name)

	// Check for deployment status; if it is new or pending, we will wait for it. If it is complete,
	// the deployment completed successfully and the deployer pod will be deleted so we will return a
	// success message. If it is running or failed, retrieve the log from the deployer pod.
	status := deployutil.DeploymentStatusFor(target)
	switch status {
	case deployapi.DeploymentStatusNew, deployapi.DeploymentStatusPending:
		if deployLogOpts.NoWait {
			glog.V(4).Infof("Deployment %s is in %s state. No logs to retrieve yet.", deployutil.LabelForDeployment(target), status)
			return &genericrest.LocationStreamer{}, nil
		}
		glog.V(4).Infof("Deployment %s is in %s state, waiting for it to start...", deployutil.LabelForDeployment(target), status)

		if err := deployutil.WaitForRunningDeployerPod(r.pn, target, r.timeout); err != nil {
			return nil, errors.NewBadRequest(fmt.Sprintf("failed to run deployer pod %s: %v", podName, err))
		}

		latest, ok, err := registry.WaitForRunningDeployment(r.rn, target, r.timeout)
		if err != nil {
			return nil, errors.NewBadRequest(fmt.Sprintf("unable to wait for deployment %s to run: %v", deployutil.LabelForDeployment(target), err))
		}
		if !ok {
			return nil, errors.NewServerTimeout(kapi.Resource("ReplicationController"), "get", 2)
		}
		if deployutil.IsCompleteDeployment(latest) {
			podName, err = r.returnApplicationPodName(target)
			if err != nil {
				return nil, err
			}
		}
	case deployapi.DeploymentStatusComplete:
		podName, err = r.returnApplicationPodName(target)
		if err != nil {
			return nil, err
		}
	}

	logOpts := deployapi.DeploymentToPodLogOptions(deployLogOpts)
	location, transport, err := pod.LogLocation(&podGetter{r.pn}, r.connInfo, ctx, podName, logOpts)
	if err != nil {
		return nil, errors.NewBadRequest(err.Error())
	}

	return &genericrest.LocationStreamer{
		Location:        location,
		Transport:       transport,
		ContentType:     "text/plain",
		Flush:           deployLogOpts.Follow,
		ResponseChecker: genericrest.NewGenericHttpResponseChecker(kapi.Resource("pod"), podName),
	}, nil
}

// waitForExistingDeployment will use the timeout to wait for a deployment to appear.
func (r *REST) waitForExistingDeployment(namespace, name string) (*kapi.ReplicationController, error) {
	var (
		target *kapi.ReplicationController
		err    error
	)

	condition := func() (bool, error) {
		target, err = r.rn.ReplicationControllers(namespace).Get(name)
		switch {
		case errors.IsNotFound(err):
			return false, nil
		case err != nil:
			return false, err
		}
		return true, nil
	}

	err = wait.PollImmediate(r.interval, r.timeout, condition)
	if err == wait.ErrWaitTimeout {
		err = errors.NewNotFound(kapi.Resource("replicationcontrollers"), name)
	}
	return target, err
}

// returnApplicationPodName returns the best candidate pod for the target deployment in order to
// view its logs.
func (r *REST) returnApplicationPodName(target *kapi.ReplicationController) (string, error) {
	selector := labels.Set(target.Spec.Selector).AsSelector()
	sortBy := func(pods []*kapi.Pod) sort.Interface { return controller.ByLogging(pods) }

	pod, _, err := kcmdutil.GetFirstPod(r.pn, target.Namespace, selector, r.timeout, sortBy)
	if err != nil {
		return "", errors.NewInternalError(err)
	}
	return pod.Name, nil
}
<|endoftext|>


<commit_before>// +build linux,cgo

package devicemapper // import "github.com/docker/docker/pkg/devicemapper"

/*
#define _GNU_SOURCE
#include <libdevmapper.h>
#include <linux/fs.h>   // FIXME: present only for BLKGETSIZE64, maybe we can remove it?

// FIXME: Can't we find a way to do the logging in pure Go?
extern void DevmapperLogCallback(int level, char *file, int line, int dm_errno_or_class, char *str);

static void	log_cb(int level, const char *file, int line, int dm_errno_or_class, const char *f, ...)
{
	char *buffer = NULL;
	va_list ap;
	int ret;

	va_start(ap, f);
	ret = vasprintf(&buffer, f, ap);
	va_end(ap);
	if (ret < 0) {
		// memory allocation failed -- should never happen?
		return;
	}

	DevmapperLogCallback(level, (char *)file, line, dm_errno_or_class, buffer);
	free(buffer);
}

static void	log_with_errno_init()
{
	dm_log_with_errno_init(log_cb);
}
*/
import "C"

import (
	"reflect"
	"unsafe"
)

type (
	cdmTask C.struct_dm_task
)

// IOCTL consts
const (
	BlkGetSize64 = C.BLKGETSIZE64
	BlkDiscard   = C.BLKDISCARD
)

// Devicemapper cookie flags.
const (
	DmUdevDisableSubsystemRulesFlag = C.DM_UDEV_DISABLE_SUBSYSTEM_RULES_FLAG
	DmUdevDisableDiskRulesFlag      = C.DM_UDEV_DISABLE_DISK_RULES_FLAG
	DmUdevDisableOtherRulesFlag     = C.DM_UDEV_DISABLE_OTHER_RULES_FLAG
	DmUdevDisableLibraryFallback    = C.DM_UDEV_DISABLE_LIBRARY_FALLBACK
)

// DeviceMapper mapped functions.
var (
	DmGetLibraryVersion       = dmGetLibraryVersionFct
	DmGetNextTarget           = dmGetNextTargetFct
	DmSetDevDir               = dmSetDevDirFct
	DmTaskAddTarget           = dmTaskAddTargetFct
	DmTaskCreate              = dmTaskCreateFct
	DmTaskDestroy             = dmTaskDestroyFct
	DmTaskGetDeps             = dmTaskGetDepsFct
	DmTaskGetInfo             = dmTaskGetInfoFct
	DmTaskGetDriverVersion    = dmTaskGetDriverVersionFct
	DmTaskRun                 = dmTaskRunFct
	DmTaskSetAddNode          = dmTaskSetAddNodeFct
	DmTaskSetCookie           = dmTaskSetCookieFct
	DmTaskSetMessage          = dmTaskSetMessageFct
	DmTaskSetName             = dmTaskSetNameFct
	DmTaskSetSector           = dmTaskSetSectorFct
	DmUdevWait                = dmUdevWaitFct
	DmUdevSetSyncSupport      = dmUdevSetSyncSupportFct
	DmUdevGetSyncSupport      = dmUdevGetSyncSupportFct
	DmCookieSupported         = dmCookieSupportedFct
	LogWithErrnoInit          = logWithErrnoInitFct
	DmTaskDeferredRemove      = dmTaskDeferredRemoveFct
	DmTaskGetInfoWithDeferred = dmTaskGetInfoWithDeferredFct
)

func free(p *C.char) {
	C.free(unsafe.Pointer(p))
}

func dmTaskDestroyFct(task *cdmTask) {
	C.dm_task_destroy((*C.struct_dm_task)(task))
}

func dmTaskCreateFct(taskType int) *cdmTask {
	return (*cdmTask)(C.dm_task_create(C.int(taskType)))
}

func dmTaskRunFct(task *cdmTask) int {
	ret, _ := C.dm_task_run((*C.struct_dm_task)(task))
	return int(ret)
}

func dmTaskSetNameFct(task *cdmTask, name string) int {
	Cname := C.CString(name)
	defer free(Cname)

	return int(C.dm_task_set_name((*C.struct_dm_task)(task), Cname))
}

func dmTaskSetMessageFct(task *cdmTask, message string) int {
	Cmessage := C.CString(message)
	defer free(Cmessage)

	return int(C.dm_task_set_message((*C.struct_dm_task)(task), Cmessage))
}

func dmTaskSetSectorFct(task *cdmTask, sector uint64) int {
	return int(C.dm_task_set_sector((*C.struct_dm_task)(task), C.uint64_t(sector)))
}

func dmTaskSetCookieFct(task *cdmTask, cookie *uint, flags uint16) int {
	cCookie := C.uint32_t(*cookie)
	defer func() {
		*cookie = uint(cCookie)
	}()
	return int(C.dm_task_set_cookie((*C.struct_dm_task)(task), &cCookie, C.uint16_t(flags)))
}

func dmTaskSetAddNodeFct(task *cdmTask, addNode AddNodeType) int {
	return int(C.dm_task_set_add_node((*C.struct_dm_task)(task), C.dm_add_node_t(addNode)))
}

func dmTaskAddTargetFct(task *cdmTask,
	start, size uint64, ttype, params string) int {

	Cttype := C.CString(ttype)
	defer free(Cttype)

	Cparams := C.CString(params)
	defer free(Cparams)

	return int(C.dm_task_add_target((*C.struct_dm_task)(task), C.uint64_t(start), C.uint64_t(size), Cttype, Cparams))
}

func dmTaskGetDepsFct(task *cdmTask) *Deps {
	Cdeps := C.dm_task_get_deps((*C.struct_dm_task)(task))
	if Cdeps == nil {
		return nil
	}

	// golang issue: https://github.com/golang/go/issues/11925
	hdr := reflect.SliceHeader{
		Data: uintptr(unsafe.Pointer(uintptr(unsafe.Pointer(Cdeps)) + unsafe.Sizeof(*Cdeps))),
		Len:  int(Cdeps.count),
		Cap:  int(Cdeps.count),
	}
	devices := *(*[]C.uint64_t)(unsafe.Pointer(&hdr))

	deps := &Deps{
		Count:  uint32(Cdeps.count),
		Filler: uint32(Cdeps.filler),
	}
	for _, device := range devices {
		deps.Device = append(deps.Device, uint64(device))
	}
	return deps
}

func dmTaskGetInfoFct(task *cdmTask, info *Info) int {
	Cinfo := C.struct_dm_info{}
	defer func() {
		info.Exists = int(Cinfo.exists)
		info.Suspended = int(Cinfo.suspended)
		info.LiveTable = int(Cinfo.live_table)
		info.InactiveTable = int(Cinfo.inactive_table)
		info.OpenCount = int32(Cinfo.open_count)
		info.EventNr = uint32(Cinfo.event_nr)
		info.Major = uint32(Cinfo.major)
		info.Minor = uint32(Cinfo.minor)
		info.ReadOnly = int(Cinfo.read_only)
		info.TargetCount = int32(Cinfo.target_count)
	}()
	return int(C.dm_task_get_info((*C.struct_dm_task)(task), &Cinfo))
}

func dmTaskGetDriverVersionFct(task *cdmTask) string {
	buffer := C.malloc(128)
	defer C.free(buffer)
	res := C.dm_task_get_driver_version((*C.struct_dm_task)(task), (*C.char)(buffer), 128)
	if res == 0 {
		return ""
	}
	return C.GoString((*C.char)(buffer))
}

func dmGetNextTargetFct(task *cdmTask, next unsafe.Pointer, start, length *uint64, target, params *string) unsafe.Pointer {
	var (
		Cstart, Clength      C.uint64_t
		CtargetType, Cparams *C.char
	)
	defer func() {
		*start = uint64(Cstart)
		*length = uint64(Clength)
		*target = C.GoString(CtargetType)
		*params = C.GoString(Cparams)
	}()

	nextp := C.dm_get_next_target((*C.struct_dm_task)(task), next, &Cstart, &Clength, &CtargetType, &Cparams)
	return nextp
}

func dmUdevSetSyncSupportFct(syncWithUdev int) {
	C.dm_udev_set_sync_support(C.int(syncWithUdev))
}

func dmUdevGetSyncSupportFct() int {
	return int(C.dm_udev_get_sync_support())
}

func dmUdevWaitFct(cookie uint) int {
	return int(C.dm_udev_wait(C.uint32_t(cookie)))
}

func dmCookieSupportedFct() int {
	return int(C.dm_cookie_supported())
}

func logWithErrnoInitFct() {
	C.log_with_errno_init()
}

func dmSetDevDirFct(dir string) int {
	Cdir := C.CString(dir)
	defer free(Cdir)

	return int(C.dm_set_dev_dir(Cdir))
}

func dmGetLibraryVersionFct(version *string) int {
	buffer := C.CString(string(make([]byte, 128)))
	defer free(buffer)
	defer func() {
		*version = C.GoString(buffer)
	}()
	return int(C.dm_get_library_version(buffer, 128))
}
<commit_msg>pkg/devicemapper ignore SA4000 false positive (staticcheck)<commit_after>// +build linux,cgo

package devicemapper // import "github.com/docker/docker/pkg/devicemapper"

/*
#define _GNU_SOURCE
#include <libdevmapper.h>
#include <linux/fs.h>   // FIXME: present only for BLKGETSIZE64, maybe we can remove it?

// FIXME: Can't we find a way to do the logging in pure Go?
extern void DevmapperLogCallback(int level, char *file, int line, int dm_errno_or_class, char *str);

static void	log_cb(int level, const char *file, int line, int dm_errno_or_class, const char *f, ...)
{
	char *buffer = NULL;
	va_list ap;
	int ret;

	va_start(ap, f);
	ret = vasprintf(&buffer, f, ap);
	va_end(ap);
	if (ret < 0) {
		// memory allocation failed -- should never happen?
		return;
	}

	DevmapperLogCallback(level, (char *)file, line, dm_errno_or_class, buffer);
	free(buffer);
}

static void	log_with_errno_init()
{
	dm_log_with_errno_init(log_cb);
}
*/
import "C"

import (
	"reflect"
	"unsafe"
)

type (
	cdmTask C.struct_dm_task
)

// IOCTL consts
const (
	BlkGetSize64 = C.BLKGETSIZE64
	BlkDiscard   = C.BLKDISCARD
)

// Devicemapper cookie flags.
const (
	DmUdevDisableSubsystemRulesFlag = C.DM_UDEV_DISABLE_SUBSYSTEM_RULES_FLAG
	DmUdevDisableDiskRulesFlag      = C.DM_UDEV_DISABLE_DISK_RULES_FLAG
	DmUdevDisableOtherRulesFlag     = C.DM_UDEV_DISABLE_OTHER_RULES_FLAG
	DmUdevDisableLibraryFallback    = C.DM_UDEV_DISABLE_LIBRARY_FALLBACK
)

// DeviceMapper mapped functions.
var (
	DmGetLibraryVersion       = dmGetLibraryVersionFct
	DmGetNextTarget           = dmGetNextTargetFct
	DmSetDevDir               = dmSetDevDirFct
	DmTaskAddTarget           = dmTaskAddTargetFct
	DmTaskCreate              = dmTaskCreateFct
	DmTaskDestroy             = dmTaskDestroyFct
	DmTaskGetDeps             = dmTaskGetDepsFct
	DmTaskGetInfo             = dmTaskGetInfoFct
	DmTaskGetDriverVersion    = dmTaskGetDriverVersionFct
	DmTaskRun                 = dmTaskRunFct
	DmTaskSetAddNode          = dmTaskSetAddNodeFct
	DmTaskSetCookie           = dmTaskSetCookieFct
	DmTaskSetMessage          = dmTaskSetMessageFct
	DmTaskSetName             = dmTaskSetNameFct
	DmTaskSetSector           = dmTaskSetSectorFct
	DmUdevWait                = dmUdevWaitFct
	DmUdevSetSyncSupport      = dmUdevSetSyncSupportFct
	DmUdevGetSyncSupport      = dmUdevGetSyncSupportFct
	DmCookieSupported         = dmCookieSupportedFct
	LogWithErrnoInit          = logWithErrnoInitFct
	DmTaskDeferredRemove      = dmTaskDeferredRemoveFct
	DmTaskGetInfoWithDeferred = dmTaskGetInfoWithDeferredFct
)

func free(p *C.char) {
	C.free(unsafe.Pointer(p))
}

func dmTaskDestroyFct(task *cdmTask) {
	C.dm_task_destroy((*C.struct_dm_task)(task))
}

func dmTaskCreateFct(taskType int) *cdmTask {
	return (*cdmTask)(C.dm_task_create(C.int(taskType)))
}

func dmTaskRunFct(task *cdmTask) int {
	ret, _ := C.dm_task_run((*C.struct_dm_task)(task))
	return int(ret)
}

func dmTaskSetNameFct(task *cdmTask, name string) int {
	Cname := C.CString(name)
	defer free(Cname)

	return int(C.dm_task_set_name((*C.struct_dm_task)(task), Cname))
}

func dmTaskSetMessageFct(task *cdmTask, message string) int {
	Cmessage := C.CString(message)
	defer free(Cmessage)

	return int(C.dm_task_set_message((*C.struct_dm_task)(task), Cmessage))
}

func dmTaskSetSectorFct(task *cdmTask, sector uint64) int {
	return int(C.dm_task_set_sector((*C.struct_dm_task)(task), C.uint64_t(sector)))
}

func dmTaskSetCookieFct(task *cdmTask, cookie *uint, flags uint16) int {
	cCookie := C.uint32_t(*cookie)
	defer func() {
		*cookie = uint(cCookie)
	}()
	return int(C.dm_task_set_cookie((*C.struct_dm_task)(task), &cCookie, C.uint16_t(flags)))
}

func dmTaskSetAddNodeFct(task *cdmTask, addNode AddNodeType) int {
	return int(C.dm_task_set_add_node((*C.struct_dm_task)(task), C.dm_add_node_t(addNode)))
}

func dmTaskAddTargetFct(task *cdmTask,
	start, size uint64, ttype, params string) int {

	Cttype := C.CString(ttype)
	defer free(Cttype)

	Cparams := C.CString(params)
	defer free(Cparams)

	return int(C.dm_task_add_target((*C.struct_dm_task)(task), C.uint64_t(start), C.uint64_t(size), Cttype, Cparams))
}

func dmTaskGetDepsFct(task *cdmTask) *Deps {
	Cdeps := C.dm_task_get_deps((*C.struct_dm_task)(task))
	if Cdeps == nil {
		return nil
	}

	// golang issue: https://github.com/golang/go/issues/11925
	hdr := reflect.SliceHeader{
		Data: uintptr(unsafe.Pointer(uintptr(unsafe.Pointer(Cdeps)) + unsafe.Sizeof(*Cdeps))),
		Len:  int(Cdeps.count),
		Cap:  int(Cdeps.count),
	}
	devices := *(*[]C.uint64_t)(unsafe.Pointer(&hdr))

	deps := &Deps{
		Count:  uint32(Cdeps.count),
		Filler: uint32(Cdeps.filler),
	}
	for _, device := range devices {
		deps.Device = append(deps.Device, uint64(device))
	}
	return deps
}

func dmTaskGetInfoFct(task *cdmTask, info *Info) int {
	Cinfo := C.struct_dm_info{}
	defer func() {
		info.Exists = int(Cinfo.exists)
		info.Suspended = int(Cinfo.suspended)
		info.LiveTable = int(Cinfo.live_table)
		info.InactiveTable = int(Cinfo.inactive_table)
		info.OpenCount = int32(Cinfo.open_count)
		info.EventNr = uint32(Cinfo.event_nr)
		info.Major = uint32(Cinfo.major)
		info.Minor = uint32(Cinfo.minor)
		info.ReadOnly = int(Cinfo.read_only)
		info.TargetCount = int32(Cinfo.target_count)
	}()
	return int(C.dm_task_get_info((*C.struct_dm_task)(task), &Cinfo))
}

func dmTaskGetDriverVersionFct(task *cdmTask) string {
	buffer := C.malloc(128)
	defer C.free(buffer)
	res := C.dm_task_get_driver_version((*C.struct_dm_task)(task), (*C.char)(buffer), 128)
	if res == 0 {
		return ""
	}
	return C.GoString((*C.char)(buffer))
}

func dmGetNextTargetFct(task *cdmTask, next unsafe.Pointer, start, length *uint64, target, params *string) unsafe.Pointer {
	var (
		Cstart, Clength      C.uint64_t
		CtargetType, Cparams *C.char
	)
	defer func() {
		*start = uint64(Cstart)
		*length = uint64(Clength)
		*target = C.GoString(CtargetType)
		*params = C.GoString(Cparams)
	}()

	//lint:ignore SA4000 false positive on (identical expressions on the left and right side of the '==' operator) (staticcheck)
	nextp := C.dm_get_next_target((*C.struct_dm_task)(task), next, &Cstart, &Clength, &CtargetType, &Cparams)
	return nextp
}

func dmUdevSetSyncSupportFct(syncWithUdev int) {
	C.dm_udev_set_sync_support(C.int(syncWithUdev))
}

func dmUdevGetSyncSupportFct() int {
	return int(C.dm_udev_get_sync_support())
}

func dmUdevWaitFct(cookie uint) int {
	return int(C.dm_udev_wait(C.uint32_t(cookie)))
}

func dmCookieSupportedFct() int {
	return int(C.dm_cookie_supported())
}

func logWithErrnoInitFct() {
	C.log_with_errno_init()
}

func dmSetDevDirFct(dir string) int {
	Cdir := C.CString(dir)
	defer free(Cdir)

	return int(C.dm_set_dev_dir(Cdir))
}

func dmGetLibraryVersionFct(version *string) int {
	buffer := C.CString(string(make([]byte, 128)))
	defer free(buffer)
	defer func() {
		*version = C.GoString(buffer)
	}()
	return int(C.dm_get_library_version(buffer, 128))
}
<|endoftext|>


<commit_before>// Copyright 2014 Brave New Software

//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at

//        http://www.apache.org/licenses/LICENSE-2.0

//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//

package statshub

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"math"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"code.google.com/p/go.net/websocket"

	"github.com/getlantern/statshub/bigquery"
)

const (
	ANY                = "*"
	STREAMING_INTERVAL = 30 * time.Second

	INTERVAL_QUERY_TEMPL = `
SELECT
    INTEGER(TIMESTAMP_TO_SEC(_ts) / %d) AS period,
	_dim,
	MAX(%s.%s) AS value
FROM [%s]
WHERE
	_ts <= DATE_ADD(CURRENT_TIMESTAMP(), -%d, "DAY")
    AND _ts > DATE_ADD(CURRENT_TIMESTAMP(), -%d, "DAY")
    %s
GROUP BY period, _dim
ORDER BY period`

	DIM_WHERE_TEMPL = "AND _dim = '%s'"

	ONE_MINUTE = 60
	ONE_HOUR   = 60 * ONE_MINUTE
	ONE_DAY    = 24 * ONE_HOUR
	ONE_WEEK   = 7 * ONE_DAY
	ONE_MONTH  = 30 * ONE_DAY  // Approximation
	ONE_YEAR   = 365 * ONE_DAY // Approximation
)

var (
	nextStreamingClientId = 0
	streamingClients      = make(map[int]*streamingClient)
	newStreamingClient    = make(chan *streamingClient)
	closedStreamingClient = make(chan int)
)

type streamingClient struct {
	ws       *websocket.Conn
	updates  chan *streamingUpdate
	id       chan int
	dimName  string // the name of the dimension that this client is querying (e.g. "fallback")
	dimKey   string // the key of the dimension that this client is querying (e.g. "instance_fp-afisk-at-getlantern-dot-org-50e8-4-2014-2-24" or "total")
	statType string // the type of stat being queried (e.g. "counter" or "gauge")
	statName string // the name of the stat being queried (e.g. "bytesGiven")
}

type streamingUpdate struct {
	asOf time.Time
	dims map[string]map[string]*Stats
}

// ClientQueryResponse is a Response to a StatsQuery
type StreamingQueryResponse struct {
	Response
	Intervals []StreamingQueryResponseInterval `json:"intervals"`
}

type StreamingQueryResponseInterval struct {
	AsOfSeconds int64            `json:"asOfSeconds"`
	Values      map[string]int64 `json:"values"`
}

func init() {
	http.Handle("/stream/", websocket.Handler(streamStats))
	go handleStreamingClients()
}

// handleStreamingClients handles streaming updates to subscribed streaming clients
func handleStreamingClients() {
	for {
		nextInterval := time.Now().Truncate(STREAMING_INTERVAL).Add(STREAMING_INTERVAL)
		waitTime := nextInterval.Sub(time.Now())
		select {
		case client := <-newStreamingClient:
			// Add new client to map
			nextStreamingClientId++
			streamingClients[nextStreamingClientId] = client
			client.id <- nextStreamingClientId
		case closedId := <-closedStreamingClient:
			// Remove disconnected client from map
			delete(streamingClients, closedId)
		case <-time.After(waitTime):
			// Query fallback and country dims
			// TODO: only query for the stuff that clients have asked for
			dims, err := QueryDims([]string{"fallback", "country"})
			if err != nil {
				log.Printf("Unable to query dims: %s", err)
			} else {
				// Publish update to clients
				update := &streamingUpdate{asOf: nextInterval, dims: dims}
				for _, client := range streamingClients {
					client.updates <- update
				}
			}
		}
	}
}

// streamStats streams stats over a websocket
func streamStats(ws *websocket.Conn) {
	singleSlashPath := strings.Replace(ws.Request().URL.Path, "//", "/", -1)
	pathParts := strings.Split(singleSlashPath, "/")

	if len(pathParts) < 6 {
		data, err := json.Marshal(&Response{Succeeded: false, Error: fmt.Sprintf("Wrong path: %s. Expected something like: %s", singleSlashPath, "/stream/country/*/counter/bytesGiven")})
		if err == nil {
			ws.Write(data)
		}
		return
	}

	client := &streamingClient{
		ws:       ws,
		updates:  make(chan *streamingUpdate, 100),
		id:       make(chan int),
		dimName:  pathParts[2],
		dimKey:   pathParts[3],
		statType: pathParts[4],
		statName: pathParts[5],
	}

	client.loadHistory()

	go client.writeUpdates()

	var wg sync.WaitGroup
	wg.Add(1)

	// Read from the client (we don't expect to get anything, but this allows us
	// to check for closed connections)
	go func() {
		id := <-client.id
		msg := make([]byte, 1)
		for {
			_, err := ws.Read(msg)
			if err == io.EOF {
				closedStreamingClient <- id
				ws.Close()
				wg.Done()
				return
			}
		}
	}()

	newStreamingClient <- client
	wg.Wait()
}

// loadHistory loads historical data based on the specified query and sends it
// to the client
func (client *streamingClient) loadHistory() {
	intervals := []StreamingQueryResponseInterval{}
	// Monthly figures for  1 month back to 1 year back
	intervals = client.loadHistoryForRange(intervals, ONE_MONTH, ONE_MONTH, ONE_YEAR)
	// Weekly figures for 1 week back to 1 month back
	intervals = client.loadHistoryForRange(intervals, ONE_WEEK, ONE_WEEK, ONE_MONTH)
	// Daily figures for 1 day back to 1 week back
	intervals = client.loadHistoryForRange(intervals, ONE_DAY, ONE_DAY, ONE_WEEK)
	// Hourly figures for the last 1 day
	intervals = client.loadHistoryForRange(intervals, ONE_HOUR, 0, ONE_DAY)

	resp := &StreamingQueryResponse{
		Response:  Response{Succeeded: true},
		Intervals: intervals,
	}
	client.writeResponse(resp)
}

// loadHistoryForRange loads history for a date range
func (client *streamingClient) loadHistoryForRange(
	intervals []StreamingQueryResponseInterval,
	intervalInSeconds int,
	startOffsetInDays int,
	endOffsetInDays int) []StreamingQueryResponseInterval {

	additionalWhereClause := ""
	if client.dimKey != ANY {
		additionalWhereClause = fmt.Sprintf(DIM_WHERE_TEMPL, client.dimKey)
	}

	// Yup, this allows SQL injection, but the BigQuery database doesn't allow
	// any updates of the database, only queries, so we don't worry about it
	queryString := fmt.Sprintf(
		INTERVAL_QUERY_TEMPL,
		intervalInSeconds,
		client.statType,
		client.statName,
		client.dimName,
		startOffsetInDays,
		endOffsetInDays,
		additionalWhereClause)
	rows, err := bigquery.Query(queryString, math.MaxInt32)
	if err != nil {
		log.Printf("Unable to run query: %s\n%s\n\n", err, queryString)
	}

	if len(rows) > 0 {
		lastCutoff := int64(0) // will cause first row to be seen as a new cutoff
		var interval StreamingQueryResponseInterval
		for _, row := range rows {
			cutoff, err := strconv.ParseInt(row[0].(string), 10, 64)
			if err != nil {
				log.Printf("Unable to read cutoff %s: %s", row[0], err)
				return intervals
			}
			if cutoff != lastCutoff {
				// Start a new interval
				asOf := cutoff * int64(intervalInSeconds)
				interval = StreamingQueryResponseInterval{asOf, make(map[string]int64)}
				intervals = append(intervals, interval)
			}
			lastCutoff = cutoff

			dim := row[1].(string)

			value := int64(0)
			valueIf := row[2]
			if valueIf != nil {
				value, err = strconv.ParseInt(valueIf.(string), 10, 64)
				if err != nil {
					log.Printf("Unable to read value %s: %s", row[2], err)
					return intervals
				}
			}
			interval.Values[dim] = value
		}
	}

	return intervals
}

// writeUpdates grabs streaming updates and sends them to the client
func (client *streamingClient) writeUpdates() {
	for {
		// This gets data for all dims
		update := <-client.updates
		values := make(map[string]int64)
		dim := update.dims[client.dimName]
		queryingSpecificDimKey := client.dimKey != ANY
		if dim != nil {
			for dimKey, stats := range dim {
				if !queryingSpecificDimKey || dimKey == client.dimKey {
					switch client.statType {
					case "counter":
						values[dimKey] = stats.Counters[client.statName]
					case "gauge":
						values[dimKey] = stats.Gauges[client.statName]
					default:
						log.Printf("Client has unknown statType: %s", client.statType)
					}
				}
			}
		}
		resp := &StreamingQueryResponse{
			Response: Response{Succeeded: true},
			Intervals: []StreamingQueryResponseInterval{
				StreamingQueryResponseInterval{update.asOf.Unix(), values},
			},
		}
		client.writeResponse(resp)
	}
}

func (client *streamingClient) writeResponse(resp *StreamingQueryResponse) {
	encoded, err := json.Marshal(resp)
	if err != nil {
		log.Printf("Unable to marshal json: %s", err)
	} else {
		client.ws.Write(encoded)
	}
}
<commit_msg>Corrected reporting cutoffs<commit_after>// Copyright 2014 Brave New Software

//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at

//        http://www.apache.org/licenses/LICENSE-2.0

//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//

package statshub

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"math"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"code.google.com/p/go.net/websocket"

	"github.com/getlantern/statshub/bigquery"
)

const (
	ANY                = "*"
	STREAMING_INTERVAL = 30 * time.Second

	INTERVAL_QUERY_TEMPL = `
SELECT
    INTEGER(TIMESTAMP_TO_SEC(_ts) / %d) AS period,
	_dim,
	MAX(%s.%s) AS value
FROM [%s]
WHERE
	_ts <= DATE_ADD(CURRENT_TIMESTAMP(), -%d, "DAY")
    AND _ts > DATE_ADD(CURRENT_TIMESTAMP(), -%d, "DAY")
    %s
GROUP BY period, _dim
ORDER BY period`

	DIM_WHERE_TEMPL = "AND _dim = '%s'"

	ONE_MINUTE     = 60
	ONE_HOUR       = 60 * ONE_MINUTE
	ONE_DAY        = 24 * ONE_HOUR
	ONE_WEEK_DAYS  = 7
	ONE_WEEK       = ONE_WEEK_DAYS * ONE_DAY
	ONE_MONTH_DAYS = 30 // Approximation
	ONE_MONTH      = ONE_MONTH_DAYS * ONE_DAY
	ONE_YEAR_DAYS  = 365
)

var (
	nextStreamingClientId = 0
	streamingClients      = make(map[int]*streamingClient)
	newStreamingClient    = make(chan *streamingClient)
	closedStreamingClient = make(chan int)
)

type streamingClient struct {
	ws       *websocket.Conn
	updates  chan *streamingUpdate
	id       chan int
	dimName  string // the name of the dimension that this client is querying (e.g. "fallback")
	dimKey   string // the key of the dimension that this client is querying (e.g. "instance_fp-afisk-at-getlantern-dot-org-50e8-4-2014-2-24" or "total")
	statType string // the type of stat being queried (e.g. "counter" or "gauge")
	statName string // the name of the stat being queried (e.g. "bytesGiven")
}

type streamingUpdate struct {
	asOf time.Time
	dims map[string]map[string]*Stats
}

// ClientQueryResponse is a Response to a StatsQuery
type StreamingQueryResponse struct {
	Response
	Intervals []StreamingQueryResponseInterval `json:"intervals"`
}

type StreamingQueryResponseInterval struct {
	AsOfSeconds int64            `json:"asOfSeconds"`
	Values      map[string]int64 `json:"values"`
}

func init() {
	http.Handle("/stream/", websocket.Handler(streamStats))
	go handleStreamingClients()
}

// handleStreamingClients handles streaming updates to subscribed streaming clients
func handleStreamingClients() {
	for {
		nextInterval := time.Now().Truncate(STREAMING_INTERVAL).Add(STREAMING_INTERVAL)
		waitTime := nextInterval.Sub(time.Now())
		select {
		case client := <-newStreamingClient:
			// Add new client to map
			nextStreamingClientId++
			streamingClients[nextStreamingClientId] = client
			client.id <- nextStreamingClientId
		case closedId := <-closedStreamingClient:
			// Remove disconnected client from map
			delete(streamingClients, closedId)
		case <-time.After(waitTime):
			// Query fallback and country dims
			// TODO: only query for the stuff that clients have asked for
			dims, err := QueryDims([]string{"fallback", "country"})
			if err != nil {
				log.Printf("Unable to query dims: %s", err)
			} else {
				// Publish update to clients
				update := &streamingUpdate{asOf: nextInterval, dims: dims}
				for _, client := range streamingClients {
					client.updates <- update
				}
			}
		}
	}
}

// streamStats streams stats over a websocket
func streamStats(ws *websocket.Conn) {
	singleSlashPath := strings.Replace(ws.Request().URL.Path, "//", "/", -1)
	pathParts := strings.Split(singleSlashPath, "/")

	if len(pathParts) < 6 {
		data, err := json.Marshal(&Response{Succeeded: false, Error: fmt.Sprintf("Wrong path: %s. Expected something like: %s", singleSlashPath, "/stream/country/*/counter/bytesGiven")})
		if err == nil {
			ws.Write(data)
		}
		return
	}

	client := &streamingClient{
		ws:       ws,
		updates:  make(chan *streamingUpdate, 100),
		id:       make(chan int),
		dimName:  pathParts[2],
		dimKey:   pathParts[3],
		statType: pathParts[4],
		statName: pathParts[5],
	}

	client.loadHistory()

	go client.writeUpdates()

	var wg sync.WaitGroup
	wg.Add(1)

	// Read from the client (we don't expect to get anything, but this allows us
	// to check for closed connections)
	go func() {
		id := <-client.id
		msg := make([]byte, 1)
		for {
			_, err := ws.Read(msg)
			if err == io.EOF {
				closedStreamingClient <- id
				ws.Close()
				wg.Done()
				return
			}
		}
	}()

	newStreamingClient <- client
	wg.Wait()
}

// loadHistory loads historical data based on the specified query and sends it
// to the client
func (client *streamingClient) loadHistory() {
	intervals := []StreamingQueryResponseInterval{}
	// Monthly figures for  1 month back to 1 year back
	intervals = client.loadHistoryForRange(intervals, ONE_MONTH, ONE_MONTH, ONE_YEAR_DAYS)
	// Weekly figures for 1 week back to 1 month back
	intervals = client.loadHistoryForRange(intervals, ONE_WEEK, ONE_WEEK, ONE_MONTH_DAYS)
	// Daily figures for 1 day back to 1 week back
	intervals = client.loadHistoryForRange(intervals, ONE_DAY, ONE_DAY, ONE_WEEK_DAYS)
	// Hourly figures for the last 1 day
	intervals = client.loadHistoryForRange(intervals, ONE_HOUR, 0, ONE_DAY)

	resp := &StreamingQueryResponse{
		Response:  Response{Succeeded: true},
		Intervals: intervals,
	}
	client.writeResponse(resp)
}

// loadHistoryForRange loads history for a date range
func (client *streamingClient) loadHistoryForRange(
	intervals []StreamingQueryResponseInterval,
	intervalInSeconds int,
	startOffsetInDays int,
	endOffsetInDays int) []StreamingQueryResponseInterval {

	additionalWhereClause := ""
	if client.dimKey != ANY {
		additionalWhereClause = fmt.Sprintf(DIM_WHERE_TEMPL, client.dimKey)
	}

	// Yup, this allows SQL injection, but the BigQuery database doesn't allow
	// any updates of the database, only queries, so we don't worry about it
	queryString := fmt.Sprintf(
		INTERVAL_QUERY_TEMPL,
		intervalInSeconds,
		client.statType,
		client.statName,
		client.dimName,
		startOffsetInDays,
		endOffsetInDays,
		additionalWhereClause)
	rows, err := bigquery.Query(queryString, math.MaxInt32)
	if err != nil {
		log.Printf("Unable to run query: %s\n%s\n\n", err, queryString)
	}

	if len(rows) > 0 {
		lastCutoff := int64(0) // will cause first row to be seen as a new cutoff
		var interval StreamingQueryResponseInterval
		for _, row := range rows {
			cutoff, err := strconv.ParseInt(row[0].(string), 10, 64)
			if err != nil {
				log.Printf("Unable to read cutoff %s: %s", row[0], err)
				return intervals
			}
			if cutoff != lastCutoff {
				// Start a new interval
				asOf := cutoff * int64(intervalInSeconds)
				interval = StreamingQueryResponseInterval{asOf, make(map[string]int64)}
				intervals = append(intervals, interval)
			}
			lastCutoff = cutoff

			dim := row[1].(string)

			value := int64(0)
			valueIf := row[2]
			if valueIf != nil {
				value, err = strconv.ParseInt(valueIf.(string), 10, 64)
				if err != nil {
					log.Printf("Unable to read value %s: %s", row[2], err)
					return intervals
				}
			}
			interval.Values[dim] = value
		}
	}

	return intervals
}

// writeUpdates grabs streaming updates and sends them to the client
func (client *streamingClient) writeUpdates() {
	for {
		// This gets data for all dims
		update := <-client.updates
		values := make(map[string]int64)
		dim := update.dims[client.dimName]
		queryingSpecificDimKey := client.dimKey != ANY
		if dim != nil {
			for dimKey, stats := range dim {
				if !queryingSpecificDimKey || dimKey == client.dimKey {
					switch client.statType {
					case "counter":
						values[dimKey] = stats.Counters[client.statName]
					case "gauge":
						values[dimKey] = stats.Gauges[client.statName]
					default:
						log.Printf("Client has unknown statType: %s", client.statType)
					}
				}
			}
		}
		resp := &StreamingQueryResponse{
			Response: Response{Succeeded: true},
			Intervals: []StreamingQueryResponseInterval{
				StreamingQueryResponseInterval{update.asOf.Unix(), values},
			},
		}
		client.writeResponse(resp)
	}
}

func (client *streamingClient) writeResponse(resp *StreamingQueryResponse) {
	encoded, err := json.Marshal(resp)
	if err != nil {
		log.Printf("Unable to marshal json: %s", err)
	} else {
		client.ws.Write(encoded)
	}
}
<|endoftext|>


<commit_before>// +build !linux

/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package procfs

import (
	"fmt"
)

// GetFullContainerName gets the container name given the root process id of the container.
func GetFullContainerName(pid int) (string, error) {
	return "", fmt.Errorf("GetFullContainerName is unsupported in this build")
}
<commit_msg>Fixes #30886<commit_after>// +build !linux

/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package procfs

import (
	"fmt"
	"syscall"
)

type ProcFS struct{}

func NewProcFS() ProcFSInterface {
	return &ProcFS{}
}

// GetFullContainerName gets the container name given the root process id of the container.
func (pfs *ProcFS) GetFullContainerName(pid int) (string, error) {
	return "", fmt.Errorf("GetFullContainerName is unsupported in this build")
}

// Find process(es) using a regular expression and send a specified
// signal to each process
func PKill(name string, sig syscall.Signal) error {
	return fmt.Errorf("PKill is unsupported in this build")
}

// Find process(es) with a specified name (exact match)
// and return their pid(s)
func PidOf(name string) ([]int, error) {
	return []int{}, fmt.Errorf("PidOf is unsupported in this build")
}
<|endoftext|>


<commit_before>package nhc

import (
	"bufio"
	"encoding/json"
	"fmt"
	"net"
	"os"
	"reflect"
	"testing"
	"time"

	"github.com/mch1307/gomotics/config"
)

const (
	connectHost  = "localhost"
	connectPort  = "8000"
	connectProto = "tcp"
)

var (
	actions = `{"cmd":"listactions","data":[{"id":0,"name":"light","type":1,"location":1,"value1":0},{"id":1,"name":"power switch","type":1,"location":2,"value1":0}]}
	`
	locations = `{"cmd":"listlocations","data":[{"id":0,"name":""},{"id":1,"name":"Living Room"},{"id":2,"name":"Kitchen"}]}
	`
	actionEvent = `{"event":"listactions","data":[{"id":1,"value1":100}]}
	`
	testConf = config.NhcConf{Host: "localhost", Port: 8000}
	command  = Event{ID: 1, Value: 100}
	myCmd    NhcSimpleCmd
)

type Sessions []*Session

var Clients Sessions

type Session struct {
	sType      string
	connection net.Conn
	reader     *bufio.Reader
	writer     *bufio.Writer
}

func NewSession(conn net.Conn) *Session {
	reader := bufio.NewReader(conn)
	writer := bufio.NewWriter(conn)
	session := &Session{
		connection: conn,
		reader:     reader,
		writer:     writer,
	}
	Clients = append(Clients, session)
	return session
}

func (session *Session) Handle() {
	for {
		//fmt.Println("mock msg: ", nhcMessage.Cmd, nhcMessage.Event, nhcMessage.Data)
		message, _ := bufio.NewReader(session.connection).ReadBytes('\n')
		if len(message) > 0 {
			if err := json.Unmarshal(message, &nhcMessage); err != nil {
				fmt.Println("error reading input ", err)
			}
			if nhcMessage.Cmd == "startevents" {
				fmt.Println("Listener session")
				session.sType = "listener"
			} else if nhcMessage.Cmd == "listactions" {
				fmt.Println("Actions: ", nhcMessage.Cmd, nhcMessage.Event, session.sType)
				session.connection.Write([]byte(actions))
				nhcMessage.Cmd = "dropme"
			} else if nhcMessage.Cmd == "listlocations" {
				fmt.Println("Location: ", nhcMessage.Cmd, nhcMessage.Event, session.sType)
				session.connection.Write([]byte(locations))
				nhcMessage.Cmd = "dropme"
			} else if nhcMessage.Cmd == "executeactions" {
				fmt.Println("Event: ", nhcMessage.Cmd, nhcMessage.Event, session.sType)
				for _, cli := range Clients {
					if cli.sType == "listener" {
						cli.connection.Write([]byte(actionEvent))
					}
				}
			}
		}
	}
}

func init() {
	config.Conf.NhcConfig.Host = "localhost"
	config.Conf.NhcConfig.Port = 8000
	go MockNHC()

	go Listener()
	time.Sleep(500 * time.Millisecond)
	Init(&testConf)
	myCmd.Cmd = "executeactions"
	myCmd.ID = 1
	myCmd.Value = 100
	//SendCommand(myCmd.Stringify())
}

// MockNHC simulates a NHC controller on localhost:8000
func MockNHC() {
	l, err := net.Listen(connectProto, connectHost+":"+connectPort)
	if err != nil {
		fmt.Println("Error listening:", err.Error())
		os.Exit(1)
	}
	// Close the listener when the application closes.
	defer l.Close()
	fmt.Println("Listening on " + connectHost + ":" + connectPort)
	for {
		// Listen for an incoming connection.
		conn, err := l.Accept()

		if err != nil {
			fmt.Println("Error accepting: ", err.Error())
			os.Exit(1)
		}
		// populate the list of Clients
		client := NewSession(conn)
		// handle connection in goroutine
		go client.Handle()
	}
}

func handleConnection(conn net.Conn) {
	var nhcMessage Message

	for {
		//fmt.Println("mock msg: ", nhcMessage.Cmd, nhcMessage.Event, nhcMessage.Data)
		message, _ := bufio.NewReader(conn).ReadBytes('\n')
		if len(message) > 0 {
			if err := json.Unmarshal(message, &nhcMessage); err != nil {
				fmt.Println("error reading input ", err)
			}
			if nhcMessage.Cmd == "listactions" {
				//fmt.Println("Actions: ", nhcMessage.Cmd, nhcMessage.Event, nhcMessage.Data)
				conn.Write([]byte(actions))
				nhcMessage.Cmd = "dropme"
			} else if nhcMessage.Cmd == "listlocations" {
				//fmt.Println("Location: ", nhcMessage.Cmd, nhcMessage.Event, nhcMessage.Data)
				conn.Write([]byte(locations))
				nhcMessage.Cmd = "dropme"
			} else if nhcMessage.Cmd == "executeactions" {
				//fmt.Println("Event: ", nhcMessage.Cmd, nhcMessage.Event, nhcMessage.Data)
				conn.Write([]byte(actionEvent))
			}
		}
	}
}
func Test_getLocation(t *testing.T) {
	//popFakeData()
	//Init(&testConf)
	id := 1
	expect := "Living Room"
	t.Run("location", func(t *testing.T) {
		if got := GetLocation(id); !reflect.DeepEqual(got.Name, expect) {
			t.Errorf("getLocation() = %v, expected %v", got.Name, expect)
		}
	})

}

func TestGetAction(t *testing.T) {

	type args struct {
		id int
	}
	tests := []struct {
		name       string
		arg        int
		exName     string
		exLocation string
	}{
		{"action0", 0, "light", "Living Room"},
		{"action1", 1, "power switch", "Kitchen"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := GetAction(tt.arg)
			if !reflect.DeepEqual(got.Name, tt.exName) {
				t.Errorf("GetAction() name = %v, want %v", got.Name, tt.exName)
			} else if !reflect.DeepEqual(got.Name, tt.exName) {
				t.Errorf("GetAction() location = %v, expect %v", got.Location, tt.exLocation)
			}
		})
	}
}

func TestGetItems(t *testing.T) {
	tests := []struct {
		name  string
		id    int
		exVal int
	}{
		{"fakeSwitch", 1, 100},
	}
	SendCommand(myCmd.Stringify())
	time.Sleep(100 * time.Millisecond)
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := GetItems()
			for _, item := range got {
				if item.ID == tt.id {
					if item.State != 100 {
						t.Errorf("GetItems() check item has proper status. Expected: %v, got: %v", tt.exVal, item.State)
					}
				}
			}
		})
	}
}
<commit_msg>Improve test coverage<commit_after>package nhc

import (
	"bufio"
	"encoding/json"
	"fmt"
	"net"
	"os"
	"reflect"
	"testing"
	"time"

	"github.com/mch1307/gomotics/config"
)

const (
	connectHost  = "localhost"
	connectPort  = "8000"
	connectProto = "tcp"
)

var (
	actions = `{"cmd":"listactions","data":[{"id":0,"name":"light","type":1,"location":1,"value1":0},{"id":1,"name":"power switch","type":1,"location":2,"value1":0}]}
	`
	locations = `{"cmd":"listlocations","data":[{"id":0,"name":""},{"id":1,"name":"Living Room"},{"id":2,"name":"Kitchen"}]}
	`
	actionEvent = `{"event":"listactions","data":[{"id":1,"value1":100}]}
	`
	testConf = config.NhcConf{Host: "localhost", Port: 8000}
	command  = Event{ID: 1, Value: 100}
	myCmd    NhcSimpleCmd
)

type Sessions []*Session

var Clients Sessions

type Session struct {
	sType      string
	connection net.Conn
	reader     *bufio.Reader
	writer     *bufio.Writer
}

func NewSession(conn net.Conn) *Session {
	reader := bufio.NewReader(conn)
	writer := bufio.NewWriter(conn)
	session := &Session{
		connection: conn,
		reader:     reader,
		writer:     writer,
	}
	Clients = append(Clients, session)
	return session
}

func (session *Session) Handle() {
	for {
		//fmt.Println("mock msg: ", nhcMessage.Cmd, nhcMessage.Event, nhcMessage.Data)
		message, _ := bufio.NewReader(session.connection).ReadBytes('\n')
		if len(message) > 0 {
			if err := json.Unmarshal(message, &nhcMessage); err != nil {
				fmt.Println("error reading input ", err)
			}
			if nhcMessage.Cmd == "startevents" {
				fmt.Println("Listener session")
				session.sType = "listener"
			} else if nhcMessage.Cmd == "listactions" {
				fmt.Println("Actions: ", nhcMessage.Cmd, nhcMessage.Event, session.sType)
				session.connection.Write([]byte(actions))
				nhcMessage.Cmd = "dropme"
			} else if nhcMessage.Cmd == "listlocations" {
				fmt.Println("Location: ", nhcMessage.Cmd, nhcMessage.Event, session.sType)
				session.connection.Write([]byte(locations))
				nhcMessage.Cmd = "dropme"
			} else if nhcMessage.Cmd == "executeactions" {
				fmt.Println("Event: ", nhcMessage.Cmd, nhcMessage.Event, session.sType)
				for _, cli := range Clients {
					if cli.sType == "listener" {
						cli.connection.Write([]byte(actionEvent))
					}
				}
			}
		}
	}
}

func init() {
	config.Conf.NhcConfig.Host = "localhost"
	config.Conf.NhcConfig.Port = 8000
	go MockNHC()

	go Listener()
	time.Sleep(500 * time.Millisecond)
	Init(&testConf)
	// call twice to test update items in persit.go
	Init(&testConf)
	myCmd.Cmd = "executeactions"
	myCmd.ID = 1
	myCmd.Value = 100
	//SendCommand(myCmd.Stringify())
}

// MockNHC simulates a NHC controller on localhost:8000
func MockNHC() {
	l, err := net.Listen(connectProto, connectHost+":"+connectPort)
	if err != nil {
		fmt.Println("Error listening:", err.Error())
		os.Exit(1)
	}
	// Close the listener when the application closes.
	defer l.Close()
	fmt.Println("Listening on " + connectHost + ":" + connectPort)
	for {
		// Listen for an incoming connection.
		conn, err := l.Accept()

		if err != nil {
			fmt.Println("Error accepting: ", err.Error())
			os.Exit(1)
		}
		// populate the list of Clients
		client := NewSession(conn)
		// handle connection in goroutine
		go client.Handle()
	}
}

func handleConnection(conn net.Conn) {
	var nhcMessage Message

	for {
		//fmt.Println("mock msg: ", nhcMessage.Cmd, nhcMessage.Event, nhcMessage.Data)
		message, _ := bufio.NewReader(conn).ReadBytes('\n')
		if len(message) > 0 {
			if err := json.Unmarshal(message, &nhcMessage); err != nil {
				fmt.Println("error reading input ", err)
			}
			if nhcMessage.Cmd == "listactions" {
				//fmt.Println("Actions: ", nhcMessage.Cmd, nhcMessage.Event, nhcMessage.Data)
				conn.Write([]byte(actions))
				nhcMessage.Cmd = "dropme"
			} else if nhcMessage.Cmd == "listlocations" {
				//fmt.Println("Location: ", nhcMessage.Cmd, nhcMessage.Event, nhcMessage.Data)
				conn.Write([]byte(locations))
				nhcMessage.Cmd = "dropme"
			} else if nhcMessage.Cmd == "executeactions" {
				//fmt.Println("Event: ", nhcMessage.Cmd, nhcMessage.Event, nhcMessage.Data)
				conn.Write([]byte(actionEvent))
			}
		}
	}
}
func Test_getLocation(t *testing.T) {
	//popFakeData()
	//Init(&testConf)
	id := 1
	expect := "Living Room"
	t.Run("location", func(t *testing.T) {
		if got := GetLocation(id); !reflect.DeepEqual(got.Name, expect) {
			t.Errorf("getLocation() = %v, expected %v", got.Name, expect)
		}
	})

}

func TestGetAction(t *testing.T) {

	type args struct {
		id int
	}
	tests := []struct {
		name       string
		arg        int
		exName     string
		exLocation string
	}{
		{"action0", 0, "light", "Living Room"},
		{"action1", 1, "power switch", "Kitchen"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := GetAction(tt.arg)
			if !reflect.DeepEqual(got.Name, tt.exName) {
				t.Errorf("GetAction() name = %v, want %v", got.Name, tt.exName)
			} else if !reflect.DeepEqual(got.Name, tt.exName) {
				t.Errorf("GetAction() location = %v, expect %v", got.Location, tt.exLocation)
			}
		})
	}
}

func TestGetItems(t *testing.T) {
	tests := []struct {
		name  string
		id    int
		exVal int
	}{
		{"fakeSwitch", 1, 100},
	}
	SendCommand(myCmd.Stringify())
	time.Sleep(100 * time.Millisecond)
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := GetItems()
			for _, item := range got {
				if item.ID == tt.id {
					if item.State != 100 {
						t.Errorf("GetItems() check item has proper status. Expected: %v, got: %v", tt.exVal, item.State)
					}
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>package integration_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("In", func() {
	var inDestination string
	var gitRepo string

	var output inResponse

	BeforeEach(func() {
		var err error
		inDestination, err = ioutil.TempDir("", "in-destination")
		gitRepo, err = ioutil.TempDir("", "git-repo")
		Ω(err).ShouldNot(HaveOccurred())
	})

	AfterEach(func() {
		err := os.RemoveAll(inDestination)
		Ω(err).ShouldNot(HaveOccurred())

		err = os.RemoveAll(gitRepo)
		Ω(err).ShouldNot(HaveOccurred())
	})

	Context("when the config is incomplete", func() {
		var session *gexec.Session

		BeforeEach(func() {
			session = runIn("{}", inDestination, 1)
		})

		It("returns all config errors", func() {
			errorMessages := string(session.Err.Contents())

			Ω(errorMessages).Should(ContainSubstring("invalid payload (missing uri)"))
			Ω(errorMessages).Should(ContainSubstring("invalid payload (missing branch)"))
			Ω(errorMessages).Should(ContainSubstring("invalid payload (missing pool)"))
		})
	})

	Context("when given version removed a lock", func() {
		BeforeEach(func() {
			var err error

			setupGitRepo(gitRepo)

			claimLock := exec.Command("bash", "-e", "-c", `
				git mv lock-pool/unclaimed/some-lock lock-pool/claimed/some-lock
				git commit -m 'claiming some-lock'

				git rm lock-pool/claimed/some-lock
				git commit -m 'removing some-lock'
			`)
			claimLock.Dir = gitRepo

			err = claimLock.Run()
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("return succesfully", func() {
			gitVersion := exec.Command("git", "rev-parse", "HEAD")
			gitVersion.Dir = gitRepo
			sha, err := gitVersion.Output()
			Ω(err).ShouldNot(HaveOccurred())
			shaStr := strings.TrimSpace(string(sha))

			jsonIn := fmt.Sprintf(`
				{
					"source": {
						"uri": "%s",
						"branch": "master",
						"pool": "lock-pool"
					},
					"version": {
						"ref": "%s"
					}
				}`, gitRepo, shaStr)

			session := runIn(jsonIn, inDestination, 0)

			err = json.Unmarshal(session.Out.Contents(), &output)
			Ω(err).ShouldNot(HaveOccurred())
			Ω(output).Should(Equal(inResponse{
				Version: version{
					Ref: shaStr,
				},
				Metadata: []metadataPair{
					{Name: "lock_name", Value: "some-lock"},
					{Name: "pool_name", Value: "lock-pool"},
				},
			}))
		})
	})

	Context("when a previous version is given", func() {
		BeforeEach(func() {
			var err error

			setupGitRepo(gitRepo)

			claimLock := exec.Command("bash", "-e", "-c", `
				git mv lock-pool/unclaimed/some-lock lock-pool/claimed/some-lock
				git commit -m 'claiming some-lock'
			`)
			claimLock.Dir = gitRepo

			err = claimLock.Run()
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("outputs the metadata for the environment", func() {
			gitVersion := exec.Command("git", "rev-parse", "HEAD")
			gitVersion.Dir = gitRepo
			sha, err := gitVersion.Output()
			Ω(err).ShouldNot(HaveOccurred())
			shaStr := strings.TrimSpace(string(sha))

			jsonIn := fmt.Sprintf(`
				{
					"source": {
						"uri": "%s",
						"branch": "master",
						"pool": "lock-pool"
					},
					"version": {
						"ref": "%s"
					}
				}`, gitRepo, shaStr)

			session := runIn(jsonIn, inDestination, 0)

			err = json.Unmarshal(session.Out.Contents(), &output)
			Ω(err).ShouldNot(HaveOccurred())

			metaDataFile := filepath.Join(inDestination, "metadata")
			Ω(metaDataFile).Should(BeARegularFile())

			fileContents, err := ioutil.ReadFile(metaDataFile)
			Ω(err).ShouldNot(HaveOccurred())

			Ω(fileContents).Should(MatchJSON(`{"some":"json"}`))

			lockNameFile := filepath.Join(inDestination, "name")
			Ω(lockNameFile).Should(BeARegularFile())

			fileContents, err = ioutil.ReadFile(lockNameFile)
			Ω(err).ShouldNot(HaveOccurred())

			Ω(strings.TrimSpace(string(fileContents))).Should(Equal("some-lock"))

			Ω(output).Should(Equal(inResponse{
				Version: version{
					Ref: shaStr,
				},
				Metadata: []metadataPair{
					{Name: "lock_name", Value: "some-lock"},
					{Name: "pool_name", Value: "lock-pool"},
				},
			}))
		})

		Context("when the lock from the previous version has been released and we are trying to run it again", func() {
			var shaStr string

			Context("when the given commit claimed the lock but the lock was unclaimed afterwards", func() {
				BeforeEach(func() {
					var err error
					gitVersion := exec.Command("git", "rev-parse", "HEAD")
					gitVersion.Dir = gitRepo
					sha, err := gitVersion.Output()
					Ω(err).ShouldNot(HaveOccurred())
					shaStr = strings.TrimSpace(string(sha))

					unclaimLock := exec.Command("bash", "-e", "-c", `
						git mv lock-pool/claimed/some-lock lock-pool/unclaimed/some-lock
						git commit -m 'unclaiming: some-lock'
					`)
					unclaimLock.Dir = gitRepo

					err = unclaimLock.Run()
					Ω(err).ShouldNot(HaveOccurred())
				})

				It("fails with a useful error message because the lock is no longer safe to use", func() {
					jsonIn := fmt.Sprintf(`
						{
							"source": {
								"uri": "%s",
								"branch": "master",
								"pool": "lock-pool"
							},
							"version": {
								"ref": "%s"
							}
						}`, gitRepo, shaStr)

					session := runIn(jsonIn, inDestination, 1)

					Ω(session.Err).Should(gbytes.Say("error: lock instance is no longer acquired"))
				})

				Context("when the lock is acquired again but by another pipeline run and is run", func() {
					BeforeEach(func() {
						claimLock := exec.Command("bash", "-e", "-c", `
							git mv lock-pool/unclaimed/some-lock lock-pool/claimed/some-lock
							git commit -m 'claiming some-lock'
						`)
						claimLock.Dir = gitRepo

						err := claimLock.Run()
						Ω(err).ShouldNot(HaveOccurred())
					})

					It("fails with a useful error message because the lock has been acquired by another pipeline run", func() {
						jsonIn := fmt.Sprintf(`
						{
							"source": {
								"uri": "%s",
								"branch": "master",
								"pool": "lock-pool"
							},
							"version": {
								"ref": "%s"
							}
						}`, gitRepo, shaStr)

						session := runIn(jsonIn, inDestination, 1)

						Ω(session.Err).Should(gbytes.Say("error: lock instance is no longer acquired"))
					})
				})
			})

			Context("when the commit itself unclaimed the lock", func() {
				var shaStr string
				BeforeEach(func() {
					var err error

					unclaimLock := exec.Command("bash", "-e", "-c", `
						git mv lock-pool/claimed/some-lock lock-pool/unclaimed/some-lock
						git commit -m 'unclaiming: some-lock'
					`)
					unclaimLock.Dir = gitRepo

					err = unclaimLock.Run()
					Ω(err).ShouldNot(HaveOccurred())

					gitVersion := exec.Command("git", "rev-parse", "HEAD")
					gitVersion.Dir = gitRepo

					sha, err := gitVersion.Output()
					Ω(err).ShouldNot(HaveOccurred())
					shaStr = strings.TrimSpace(string(sha))

					someOtherCommit := exec.Command("bash", "-e", "-c", `
						git mv lock-pool/unclaimed/some-lock lock-pool/claimed/some-lock
						git commit -m 'doing something some-lock'
					`)
					someOtherCommit.Dir = gitRepo

					err = someOtherCommit.Run()
					Ω(err).ShouldNot(HaveOccurred())
				})

				It("outputs the metadata for the environment", func() {
					jsonIn := fmt.Sprintf(`
							{
								"source": {
									"uri": "%s",
									"branch": "master",
									"pool": "lock-pool"
								},
								"version": {
									"ref": "%s"
								}
							}`, gitRepo, shaStr)

					session := runIn(jsonIn, inDestination, 0)

					Consistently(session.Err).ShouldNot(gbytes.Say("error: lock instance is no longer acquired"))
					err := json.Unmarshal(session.Out.Contents(), &output)
					Ω(err).ShouldNot(HaveOccurred())

					metaDataFile := filepath.Join(inDestination, "metadata")
					Ω(metaDataFile).Should(BeARegularFile())

					fileContents, err := ioutil.ReadFile(metaDataFile)
					Ω(err).ShouldNot(HaveOccurred())

					Ω(fileContents).Should(MatchJSON(`{"some":"json"}`))

					lockNameFile := filepath.Join(inDestination, "name")
					Ω(lockNameFile).Should(BeARegularFile())

					fileContents, err = ioutil.ReadFile(lockNameFile)
					Ω(err).ShouldNot(HaveOccurred())

					Ω(strings.TrimSpace(string(fileContents))).Should(Equal("some-lock"))

					Ω(output).Should(Equal(inResponse{
						Version: version{
							Ref: shaStr,
						},
						Metadata: []metadataPair{
							{Name: "lock_name", Value: "some-lock"},
							{Name: "pool_name", Value: "lock-pool"},
						},
					}))
				})
			})
		})
	})
})
<commit_msg>Add integration test for modified file without move<commit_after>package integration_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("In", func() {
	var inDestination string
	var gitRepo string

	var output inResponse

	BeforeEach(func() {
		var err error
		inDestination, err = ioutil.TempDir("", "in-destination")
		gitRepo, err = ioutil.TempDir("", "git-repo")
		Ω(err).ShouldNot(HaveOccurred())
	})

	AfterEach(func() {
		err := os.RemoveAll(inDestination)
		Ω(err).ShouldNot(HaveOccurred())

		err = os.RemoveAll(gitRepo)
		Ω(err).ShouldNot(HaveOccurred())
	})

	Context("when the config is incomplete", func() {
		var session *gexec.Session

		BeforeEach(func() {
			session = runIn("{}", inDestination, 1)
		})

		It("returns all config errors", func() {
			errorMessages := string(session.Err.Contents())

			Ω(errorMessages).Should(ContainSubstring("invalid payload (missing uri)"))
			Ω(errorMessages).Should(ContainSubstring("invalid payload (missing branch)"))
			Ω(errorMessages).Should(ContainSubstring("invalid payload (missing pool)"))
		})
	})

	Context("when given version removed a lock", func() {
		BeforeEach(func() {
			var err error

			setupGitRepo(gitRepo)

			claimLock := exec.Command("bash", "-e", "-c", `
				git mv lock-pool/unclaimed/some-lock lock-pool/claimed/some-lock
				git commit -m 'claiming some-lock'

				git rm lock-pool/claimed/some-lock
				git commit -m 'removing some-lock'
			`)
			claimLock.Dir = gitRepo

			err = claimLock.Run()
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("return succesfully", func() {
			gitVersion := exec.Command("git", "rev-parse", "HEAD")
			gitVersion.Dir = gitRepo
			sha, err := gitVersion.Output()
			Ω(err).ShouldNot(HaveOccurred())
			shaStr := strings.TrimSpace(string(sha))

			jsonIn := fmt.Sprintf(`
				{
					"source": {
						"uri": "%s",
						"branch": "master",
						"pool": "lock-pool"
					},
					"version": {
						"ref": "%s"
					}
				}`, gitRepo, shaStr)

			session := runIn(jsonIn, inDestination, 0)

			err = json.Unmarshal(session.Out.Contents(), &output)
			Ω(err).ShouldNot(HaveOccurred())
			Ω(output).Should(Equal(inResponse{
				Version: version{
					Ref: shaStr,
				},
				Metadata: []metadataPair{
					{Name: "lock_name", Value: "some-lock"},
					{Name: "pool_name", Value: "lock-pool"},
				},
			}))
		})
	})

	Context("when a previous version is given", func() {
		BeforeEach(func() {
			var err error

			setupGitRepo(gitRepo)

			claimLock := exec.Command("bash", "-e", "-c", `
				git mv lock-pool/unclaimed/some-lock lock-pool/claimed/some-lock
				git commit -m 'claiming some-lock'
			`)
			claimLock.Dir = gitRepo

			err = claimLock.Run()
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("outputs the metadata for the environment", func() {
			gitVersion := exec.Command("git", "rev-parse", "HEAD")
			gitVersion.Dir = gitRepo
			sha, err := gitVersion.Output()
			Ω(err).ShouldNot(HaveOccurred())
			shaStr := strings.TrimSpace(string(sha))

			jsonIn := fmt.Sprintf(`
				{
					"source": {
						"uri": "%s",
						"branch": "master",
						"pool": "lock-pool"
					},
					"version": {
						"ref": "%s"
					}
				}`, gitRepo, shaStr)

			session := runIn(jsonIn, inDestination, 0)

			err = json.Unmarshal(session.Out.Contents(), &output)
			Ω(err).ShouldNot(HaveOccurred())

			metaDataFile := filepath.Join(inDestination, "metadata")
			Ω(metaDataFile).Should(BeARegularFile())

			fileContents, err := ioutil.ReadFile(metaDataFile)
			Ω(err).ShouldNot(HaveOccurred())

			Ω(fileContents).Should(MatchJSON(`{"some":"json"}`))

			lockNameFile := filepath.Join(inDestination, "name")
			Ω(lockNameFile).Should(BeARegularFile())

			fileContents, err = ioutil.ReadFile(lockNameFile)
			Ω(err).ShouldNot(HaveOccurred())

			Ω(strings.TrimSpace(string(fileContents))).Should(Equal("some-lock"))

			Ω(output).Should(Equal(inResponse{
				Version: version{
					Ref: shaStr,
				},
				Metadata: []metadataPair{
					{Name: "lock_name", Value: "some-lock"},
					{Name: "pool_name", Value: "lock-pool"},
				},
			}))
		})

		Context("when the lock from the previous version has been released and we are trying to run it again", func() {
			var shaStr string

			Context("when the given commit claimed the lock but the lock was unclaimed afterwards", func() {
				BeforeEach(func() {
					var err error
					gitVersion := exec.Command("git", "rev-parse", "HEAD")
					gitVersion.Dir = gitRepo
					sha, err := gitVersion.Output()
					Ω(err).ShouldNot(HaveOccurred())
					shaStr = strings.TrimSpace(string(sha))

					unclaimLock := exec.Command("bash", "-e", "-c", `
						git mv lock-pool/claimed/some-lock lock-pool/unclaimed/some-lock
						git commit -m 'unclaiming: some-lock'
					`)
					unclaimLock.Dir = gitRepo

					err = unclaimLock.Run()
					Ω(err).ShouldNot(HaveOccurred())
				})

				It("fails with a useful error message because the lock is no longer safe to use", func() {
					jsonIn := fmt.Sprintf(`
						{
							"source": {
								"uri": "%s",
								"branch": "master",
								"pool": "lock-pool"
							},
							"version": {
								"ref": "%s"
							}
						}`, gitRepo, shaStr)

					session := runIn(jsonIn, inDestination, 1)

					Ω(session.Err).Should(gbytes.Say("error: lock instance is no longer acquired"))
				})

				Context("when the lock is acquired again but by another pipeline run and is run", func() {
					BeforeEach(func() {
						claimLock := exec.Command("bash", "-e", "-c", `
							git mv lock-pool/unclaimed/some-lock lock-pool/claimed/some-lock
							git commit -m 'claiming some-lock'
						`)
						claimLock.Dir = gitRepo

						err := claimLock.Run()
						Ω(err).ShouldNot(HaveOccurred())
					})

					It("fails with a useful error message because the lock has been acquired by another pipeline run", func() {
						jsonIn := fmt.Sprintf(`
						{
							"source": {
								"uri": "%s",
								"branch": "master",
								"pool": "lock-pool"
							},
							"version": {
								"ref": "%s"
							}
						}`, gitRepo, shaStr)

						session := runIn(jsonIn, inDestination, 1)

						Ω(session.Err).Should(gbytes.Say("error: lock instance is no longer acquired"))
					})
				})
			})
			Context("when the given commit claimed the lock and the file was modified in place afterwards", func() {
				BeforeEach(func() {
					var err error
					gitVersion := exec.Command("git", "rev-parse", "HEAD")
					gitVersion.Dir = gitRepo
					sha, err := gitVersion.Output()
					Ω(err).ShouldNot(HaveOccurred())
					shaStr = strings.TrimSpace(string(sha))

					changeFile := exec.Command("bash", "-e", "-c", `
					echo "additional_key: additional_value" >> lock-pool/claimed/some-lock
					  git add lock-pool/claimed/some-lock
					  git commit -m 'changing: some-lock'
					`)
					changeFile.Dir = gitRepo

					err = changeFile.Run()
					Ω(err).ShouldNot(HaveOccurred())
				})

				It("returns successfully", func() {
					jsonIn := fmt.Sprintf(`
						{
							"source": {
								"uri": "%s",
								"branch": "master",
								"pool": "lock-pool"
							},
							"version": {
								"ref": "%s"
							}
						}`, gitRepo, shaStr)

					session := runIn(jsonIn, inDestination, 0)

					err := json.Unmarshal(session.Out.Contents(), &output)
					Ω(err).ShouldNot(HaveOccurred())
					Ω(output).Should(Equal(inResponse{
						Version: version{
							Ref: shaStr,
						},
						Metadata: []metadataPair{
							{Name: "lock_name", Value: "some-lock"},
							{Name: "pool_name", Value: "lock-pool"},
						},
					}))
				})
			})
			Context("when the commit itself unclaimed the lock", func() {
				var shaStr string
				BeforeEach(func() {
					var err error

					unclaimLock := exec.Command("bash", "-e", "-c", `
						git mv lock-pool/claimed/some-lock lock-pool/unclaimed/some-lock
						git commit -m 'unclaiming: some-lock'
					`)
					unclaimLock.Dir = gitRepo

					err = unclaimLock.Run()
					Ω(err).ShouldNot(HaveOccurred())

					gitVersion := exec.Command("git", "rev-parse", "HEAD")
					gitVersion.Dir = gitRepo

					sha, err := gitVersion.Output()
					Ω(err).ShouldNot(HaveOccurred())
					shaStr = strings.TrimSpace(string(sha))

					someOtherCommit := exec.Command("bash", "-e", "-c", `
						git mv lock-pool/unclaimed/some-lock lock-pool/claimed/some-lock
						git commit -m 'doing something some-lock'
					`)
					someOtherCommit.Dir = gitRepo

					err = someOtherCommit.Run()
					Ω(err).ShouldNot(HaveOccurred())
				})

				It("outputs the metadata for the environment", func() {
					jsonIn := fmt.Sprintf(`
							{
								"source": {
									"uri": "%s",
									"branch": "master",
									"pool": "lock-pool"
								},
								"version": {
									"ref": "%s"
								}
							}`, gitRepo, shaStr)

					session := runIn(jsonIn, inDestination, 0)

					Consistently(session.Err).ShouldNot(gbytes.Say("error: lock instance is no longer acquired"))
					err := json.Unmarshal(session.Out.Contents(), &output)
					Ω(err).ShouldNot(HaveOccurred())

					metaDataFile := filepath.Join(inDestination, "metadata")
					Ω(metaDataFile).Should(BeARegularFile())

					fileContents, err := ioutil.ReadFile(metaDataFile)
					Ω(err).ShouldNot(HaveOccurred())

					Ω(fileContents).Should(MatchJSON(`{"some":"json"}`))

					lockNameFile := filepath.Join(inDestination, "name")
					Ω(lockNameFile).Should(BeARegularFile())

					fileContents, err = ioutil.ReadFile(lockNameFile)
					Ω(err).ShouldNot(HaveOccurred())

					Ω(strings.TrimSpace(string(fileContents))).Should(Equal("some-lock"))

					Ω(output).Should(Equal(inResponse{
						Version: version{
							Ref: shaStr,
						},
						Metadata: []metadataPair{
							{Name: "lock_name", Value: "some-lock"},
							{Name: "pool_name", Value: "lock-pool"},
						},
					}))
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>package integration

import (
	"bufio"
	"crypto/tls"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/apprenda/kismatic/integration/retry"
	homedir "github.com/mitchellh/go-homedir"
	. "github.com/onsi/ginkgo"
)

func leaveIt() bool {
	return os.Getenv("LEAVE_ARTIFACTS") != ""
}
func bailBeforeAnsible() bool {
	return os.Getenv("BAIL_BEFORE_ANSIBLE") != ""
}

func GetSSHKeyFile() (string, error) {
	dir, err := homedir.Dir()
	if err != nil {
		return "", err
	}
	return filepath.Join(dir, ".ssh", "kismatic-integration-testing.pem"), nil
}

type installOptions struct {
	allowPackageInstallation    bool
	autoConfigureDockerRegistry bool
	dockerRegistryIP            string
	dockerRegistryPort          int
	dockerRegistryCAPath        string
}

func installKismaticMini(node NodeDeets, sshKey string) error {
	By("Building a template")
	template, err := template.New("planAWSOverlay").Parse(planAWSOverlay)
	FailIfError(err, "Couldn't parse template")

	By("Building a plan to set up an overlay network cluster on this hardware")
	sshUser := node.SSHUser
	plan := PlanAWS{
		Etcd:                     []NodeDeets{node},
		Master:                   []NodeDeets{node},
		Worker:                   []NodeDeets{node},
		Ingress:                  []NodeDeets{node},
		MasterNodeFQDN:           node.Hostname,
		MasterNodeShortName:      node.Hostname,
		SSHKeyFile:               sshKey,
		SSHUser:                  sshUser,
		AllowPackageInstallation: true,
	}

	By("Writing plan file out to disk")
	f, err := os.Create("kismatic-testing.yaml")
	FailIfError(err, "Error waiting for nodes")
	defer f.Close()
	w := bufio.NewWriter(f)
	err = template.Execute(w, &plan)
	FailIfError(err, "Error filling in plan template")
	w.Flush()

	By("Validing our plan")
	cmd := exec.Command("./kismatic", "install", "validate", "-f", f.Name())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	FailIfError(err, "Error validating plan")

	By("Punch it Chewie!")
	cmd = exec.Command("./kismatic", "install", "apply", "-f", f.Name())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func installKismatic(nodes provisionedNodes, installOpts installOptions, sshKey string) error {
	By("Building a template")
	template, err := template.New("planAWSOverlay").Parse(planAWSOverlay)
	FailIfError(err, "Couldn't parse template")

	By("Building a plan to set up an overlay network cluster on this hardware")
	sshUser := nodes.master[0].SSHUser

	masterDNS := nodes.master[0].Hostname
	if nodes.dnsRecord != nil && nodes.dnsRecord.Name != "" {
		masterDNS = nodes.dnsRecord.Name
	}
	plan := PlanAWS{
		AllowPackageInstallation: installOpts.allowPackageInstallation,
		Etcd:                nodes.etcd,
		Master:              nodes.master,
		Worker:              nodes.worker,
		Ingress:             nodes.ingress,
		MasterNodeFQDN:      masterDNS,
		MasterNodeShortName: masterDNS,
		SSHKeyFile:          sshKey,
		SSHUser:             sshUser,
		AutoConfiguredDockerRegistry: installOpts.autoConfigureDockerRegistry,
		DockerRegistryCAPath:         installOpts.dockerRegistryCAPath,
		DockerRegistryIP:             installOpts.dockerRegistryIP,
		DockerRegistryPort:           installOpts.dockerRegistryPort,
	}

	f, err := os.Create("kismatic-testing.yaml")
	FailIfError(err, "Error creating plan")
	defer f.Close()
	w := bufio.NewWriter(f)
	err = template.Execute(w, &plan)
	FailIfError(err, "Error filling in plan template")
	w.Flush()

	By("Punch it Chewie!")
	cmd := exec.Command("./kismatic", "install", "apply", "-f", f.Name())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()

}

func verifyMasterNodeFailure(nodes provisionedNodes, provisioner infrastructureProvisioner, sshKey string) error {
	By("Removing a Kubernetes master node")
	if err := provisioner.TerminateNode(nodes.master[0]); err != nil {
		return fmt.Errorf("Could not remove node: %v", err)
	}

	By("Rerunning Kuberang")
	if err := runViaSSH([]string{"sudo kuberang"}, []NodeDeets{nodes.master[1]}, sshKey, 5*time.Minute); err != nil {
		return fmt.Errorf("Failed to run kuberang: %v", err)
	}

	return nil
}

func verifyIngressNodes(nodes provisionedNodes, sshKey string) error {
	By("Adding a service and an ingress resource")
	addIngressResource(nodes.master[0], sshKey)

	By("Verifying the service is accessible via the ingress point(s)")
	for _, ingNode := range nodes.ingress {
		if err := verifyIngressPoint(ingNode); err != nil {
			return err
		}
	}

	return nil
}

func verifyIngressNode(node NodeDeets, sshKey string) error {
	By("Adding a service and an ingress resource")
	addIngressResource(node, sshKey)

	By("Verifying the service is accessible via the ingress point(s)")
	return verifyIngressPoint(node)
}

func addIngressResource(node NodeDeets, sshKey string) {
	err := copyFileToRemote("test-resources/ingress.yaml", "/tmp/ingress.yaml", node, sshKey, 1*time.Minute)
	FailIfError(err, "Error copying ingress test file")

	err = runViaSSH([]string{"sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /tmp/tls.key -out /tmp/tls.crt -subj \"/CN=kismaticintegration.com\""}, []NodeDeets{node}, sshKey, 1*time.Minute)
	FailIfError(err, "Error creating certificates for HTTPs")

	err = runViaSSH([]string{"sudo kubectl create secret tls kismaticintegration-tls --cert=/tmp/tls.crt --key=/tmp/tls.key"}, []NodeDeets{node}, sshKey, 1*time.Minute)
	FailIfError(err, "Error creating tls secret")

	err = runViaSSH([]string{"sudo kubectl apply -f /tmp/ingress.yaml"}, []NodeDeets{node}, sshKey, 1*time.Minute)
	FailIfError(err, "Error creating ingress resources")
}

func verifyIngressPoint(node NodeDeets) error {
	// HTTP ingress

	url := "http://" + node.PublicIP + "/echo"
	if err := retry.WithBackoff(func() error { return ingressRequest(url) }, 10); err != nil {
		return err
	}
	// HTTPs ingress
	url = "https://" + node.PublicIP + "/echo-tls"
	if err := retry.WithBackoff(func() error { return ingressRequest(url) }, 7); err != nil {
		return err
	}
	return nil
}

func ingressRequest(url string) error {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := http.Client{
		Timeout:   1000 * time.Millisecond,
		Transport: tr,
	}
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		fmt.Printf("Could not create request for ingress via %s, %v", url, err)
		return fmt.Errorf("Could not create request for ingress via %s, %v", url, err)
	}
	// Set the host header since this is not a real domain, curl $IP/echo -H 'Host: kismaticintegration.com'
	req.Host = "kismaticintegration.com"
	resp, err := client.Do(req)
	if err != nil {
		fmt.Printf("Could not reach ingress via %s, %v", url, err)
		return fmt.Errorf("Could not reach ingress via %s, %v", url, err)
	}
	if resp.StatusCode != 200 {
		fmt.Printf("Ingress status code is not 200, got %d vi %s", resp.StatusCode, url)
		return fmt.Errorf("Ingress status code is not 200, got %d vi %s", resp.StatusCode, url)
	}

	fmt.Println("GOT TO THE END")
	return nil
}

func installKismaticWithABadNode() {
	By("Building a template")
	template, err := template.New("planAWSOverlay").Parse(planAWSOverlay)
	FailIfError(err, "Couldn't parse template")

	By("Faking infrastructure")
	fakeNode := NodeDeets{
		id:       "FakeId",
		PublicIP: "10.0.0.0",
		Hostname: "FakeHostname",
	}

	By("Building a plan to set up an overlay network cluster on this hardware")
	sshKey, err := GetSSHKeyFile()
	FailIfError(err, "Error getting SSH Key file")
	plan := PlanAWS{
		Etcd:                []NodeDeets{fakeNode},
		Master:              []NodeDeets{fakeNode},
		Worker:              []NodeDeets{fakeNode},
		Ingress:             []NodeDeets{fakeNode},
		MasterNodeFQDN:      "yep.nope",
		MasterNodeShortName: "yep",
		SSHUser:             "Billy Rubin",
		SSHKeyFile:          sshKey,
	}
	By("Writing plan file out to disk")
	f, err := os.Create("kismatic-testing.yaml")
	FailIfError(err, "Error waiting for nodes")
	defer f.Close()
	w := bufio.NewWriter(f)
	err = template.Execute(w, &plan)
	FailIfError(err, "Error filling in plan template")
	w.Flush()
	f.Close()

	By("Validing our plan")
	cmd := exec.Command("./kismatic", "install", "validate", "-f", f.Name())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err == nil {
		Fail("Validation succeeeded even though it shouldn't have")
	}

	By("Well, try it anyway")
	cmd = exec.Command("./kismatic", "install", "apply", "-f", f.Name())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err == nil {
		Fail("Application succeeeded even though it shouldn't have")
	}
}

func completesInTime(dothis func(), howLong time.Duration) bool {
	c1 := make(chan string, 1)
	go func() {
		dothis()
		c1 <- "completed"
	}()

	select {
	case <-c1:
		return true
	case <-time.After(howLong):
		return false
	}
}

func FailIfError(err error, message ...string) {
	if err != nil {
		log.Printf(message[0]+": %v\n%v", err, message[1:])
		Fail(message[0])
	}
}

func FailIfSuccess(err error, message ...string) {
	if err == nil {
		Fail("Expected failure")
	}
}

func FileExists(path string) bool {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return false
	}
	return true
}
<commit_msg>Removed debug messages<commit_after>package integration

import (
	"bufio"
	"crypto/tls"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/apprenda/kismatic/integration/retry"
	homedir "github.com/mitchellh/go-homedir"
	. "github.com/onsi/ginkgo"
)

func leaveIt() bool {
	return os.Getenv("LEAVE_ARTIFACTS") != ""
}
func bailBeforeAnsible() bool {
	return os.Getenv("BAIL_BEFORE_ANSIBLE") != ""
}

func GetSSHKeyFile() (string, error) {
	dir, err := homedir.Dir()
	if err != nil {
		return "", err
	}
	return filepath.Join(dir, ".ssh", "kismatic-integration-testing.pem"), nil
}

type installOptions struct {
	allowPackageInstallation    bool
	autoConfigureDockerRegistry bool
	dockerRegistryIP            string
	dockerRegistryPort          int
	dockerRegistryCAPath        string
}

func installKismaticMini(node NodeDeets, sshKey string) error {
	By("Building a template")
	template, err := template.New("planAWSOverlay").Parse(planAWSOverlay)
	FailIfError(err, "Couldn't parse template")

	By("Building a plan to set up an overlay network cluster on this hardware")
	sshUser := node.SSHUser
	plan := PlanAWS{
		Etcd:                     []NodeDeets{node},
		Master:                   []NodeDeets{node},
		Worker:                   []NodeDeets{node},
		Ingress:                  []NodeDeets{node},
		MasterNodeFQDN:           node.Hostname,
		MasterNodeShortName:      node.Hostname,
		SSHKeyFile:               sshKey,
		SSHUser:                  sshUser,
		AllowPackageInstallation: true,
	}

	By("Writing plan file out to disk")
	f, err := os.Create("kismatic-testing.yaml")
	FailIfError(err, "Error waiting for nodes")
	defer f.Close()
	w := bufio.NewWriter(f)
	err = template.Execute(w, &plan)
	FailIfError(err, "Error filling in plan template")
	w.Flush()

	By("Validing our plan")
	cmd := exec.Command("./kismatic", "install", "validate", "-f", f.Name())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	FailIfError(err, "Error validating plan")

	By("Punch it Chewie!")
	cmd = exec.Command("./kismatic", "install", "apply", "-f", f.Name())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func installKismatic(nodes provisionedNodes, installOpts installOptions, sshKey string) error {
	By("Building a template")
	template, err := template.New("planAWSOverlay").Parse(planAWSOverlay)
	FailIfError(err, "Couldn't parse template")

	By("Building a plan to set up an overlay network cluster on this hardware")
	sshUser := nodes.master[0].SSHUser

	masterDNS := nodes.master[0].Hostname
	if nodes.dnsRecord != nil && nodes.dnsRecord.Name != "" {
		masterDNS = nodes.dnsRecord.Name
	}
	plan := PlanAWS{
		AllowPackageInstallation: installOpts.allowPackageInstallation,
		Etcd:                nodes.etcd,
		Master:              nodes.master,
		Worker:              nodes.worker,
		Ingress:             nodes.ingress,
		MasterNodeFQDN:      masterDNS,
		MasterNodeShortName: masterDNS,
		SSHKeyFile:          sshKey,
		SSHUser:             sshUser,
		AutoConfiguredDockerRegistry: installOpts.autoConfigureDockerRegistry,
		DockerRegistryCAPath:         installOpts.dockerRegistryCAPath,
		DockerRegistryIP:             installOpts.dockerRegistryIP,
		DockerRegistryPort:           installOpts.dockerRegistryPort,
	}

	f, err := os.Create("kismatic-testing.yaml")
	FailIfError(err, "Error creating plan")
	defer f.Close()
	w := bufio.NewWriter(f)
	err = template.Execute(w, &plan)
	FailIfError(err, "Error filling in plan template")
	w.Flush()

	By("Punch it Chewie!")
	cmd := exec.Command("./kismatic", "install", "apply", "-f", f.Name())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()

}

func verifyMasterNodeFailure(nodes provisionedNodes, provisioner infrastructureProvisioner, sshKey string) error {
	By("Removing a Kubernetes master node")
	if err := provisioner.TerminateNode(nodes.master[0]); err != nil {
		return fmt.Errorf("Could not remove node: %v", err)
	}

	By("Rerunning Kuberang")
	if err := runViaSSH([]string{"sudo kuberang"}, []NodeDeets{nodes.master[1]}, sshKey, 5*time.Minute); err != nil {
		return fmt.Errorf("Failed to run kuberang: %v", err)
	}

	return nil
}

func verifyIngressNodes(nodes provisionedNodes, sshKey string) error {
	By("Adding a service and an ingress resource")
	addIngressResource(nodes.master[0], sshKey)

	By("Verifying the service is accessible via the ingress point(s)")
	for _, ingNode := range nodes.ingress {
		if err := verifyIngressPoint(ingNode); err != nil {
			return err
		}
	}

	return nil
}

func verifyIngressNode(node NodeDeets, sshKey string) error {
	By("Adding a service and an ingress resource")
	addIngressResource(node, sshKey)

	By("Verifying the service is accessible via the ingress point(s)")
	return verifyIngressPoint(node)
}

func addIngressResource(node NodeDeets, sshKey string) {
	err := copyFileToRemote("test-resources/ingress.yaml", "/tmp/ingress.yaml", node, sshKey, 1*time.Minute)
	FailIfError(err, "Error copying ingress test file")

	err = runViaSSH([]string{"sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /tmp/tls.key -out /tmp/tls.crt -subj \"/CN=kismaticintegration.com\""}, []NodeDeets{node}, sshKey, 1*time.Minute)
	FailIfError(err, "Error creating certificates for HTTPs")

	err = runViaSSH([]string{"sudo kubectl create secret tls kismaticintegration-tls --cert=/tmp/tls.crt --key=/tmp/tls.key"}, []NodeDeets{node}, sshKey, 1*time.Minute)
	FailIfError(err, "Error creating tls secret")

	err = runViaSSH([]string{"sudo kubectl apply -f /tmp/ingress.yaml"}, []NodeDeets{node}, sshKey, 1*time.Minute)
	FailIfError(err, "Error creating ingress resources")
}

func verifyIngressPoint(node NodeDeets) error {
	// HTTP ingress

	url := "http://" + node.PublicIP + "/echo"
	if err := retry.WithBackoff(func() error { return ingressRequest(url) }, 10); err != nil {
		return err
	}
	// HTTPs ingress
	url = "https://" + node.PublicIP + "/echo-tls"
	if err := retry.WithBackoff(func() error { return ingressRequest(url) }, 7); err != nil {
		return err
	}
	return nil
}

func ingressRequest(url string) error {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := http.Client{
		Timeout:   1000 * time.Millisecond,
		Transport: tr,
	}
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return fmt.Errorf("Could not create request for ingress via %s, %v", url, err)
	}
	// Set the host header since this is not a real domain, curl $IP/echo -H 'Host: kismaticintegration.com'
	req.Host = "kismaticintegration.com"
	resp, err := client.Do(req)
	if err != nil {
		return fmt.Errorf("Could not reach ingress via %s, %v", url, err)
	}
	if resp.StatusCode != 200 {
		return fmt.Errorf("Ingress status code is not 200, got %d vi %s", resp.StatusCode, url)
	}

	return nil
}

func installKismaticWithABadNode() {
	By("Building a template")
	template, err := template.New("planAWSOverlay").Parse(planAWSOverlay)
	FailIfError(err, "Couldn't parse template")

	By("Faking infrastructure")
	fakeNode := NodeDeets{
		id:       "FakeId",
		PublicIP: "10.0.0.0",
		Hostname: "FakeHostname",
	}

	By("Building a plan to set up an overlay network cluster on this hardware")
	sshKey, err := GetSSHKeyFile()
	FailIfError(err, "Error getting SSH Key file")
	plan := PlanAWS{
		Etcd:                []NodeDeets{fakeNode},
		Master:              []NodeDeets{fakeNode},
		Worker:              []NodeDeets{fakeNode},
		Ingress:             []NodeDeets{fakeNode},
		MasterNodeFQDN:      "yep.nope",
		MasterNodeShortName: "yep",
		SSHUser:             "Billy Rubin",
		SSHKeyFile:          sshKey,
	}
	By("Writing plan file out to disk")
	f, err := os.Create("kismatic-testing.yaml")
	FailIfError(err, "Error waiting for nodes")
	defer f.Close()
	w := bufio.NewWriter(f)
	err = template.Execute(w, &plan)
	FailIfError(err, "Error filling in plan template")
	w.Flush()
	f.Close()

	By("Validing our plan")
	cmd := exec.Command("./kismatic", "install", "validate", "-f", f.Name())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err == nil {
		Fail("Validation succeeeded even though it shouldn't have")
	}

	By("Well, try it anyway")
	cmd = exec.Command("./kismatic", "install", "apply", "-f", f.Name())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err == nil {
		Fail("Application succeeeded even though it shouldn't have")
	}
}

func completesInTime(dothis func(), howLong time.Duration) bool {
	c1 := make(chan string, 1)
	go func() {
		dothis()
		c1 <- "completed"
	}()

	select {
	case <-c1:
		return true
	case <-time.After(howLong):
		return false
	}
}

func FailIfError(err error, message ...string) {
	if err != nil {
		log.Printf(message[0]+": %v\n%v", err, message[1:])
		Fail(message[0])
	}
}

func FailIfSuccess(err error, message ...string) {
	if err == nil {
		Fail("Expected failure")
	}
}

func FileExists(path string) bool {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return false
	}
	return true
}
<|endoftext|>