<commit_before>package cloudformation

import (
	"context"
	"fmt"
	"strconv"

	"github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/giantswarm/microerror"
	"github.com/giantswarm/operatorkit/controller/context/finalizerskeptcontext"
	"github.com/giantswarm/operatorkit/controller/context/resourcecanceledcontext"

	cloudformationservice "github.com/giantswarm/aws-operator/service/controller/v18/cloudformation"
	"github.com/giantswarm/aws-operator/service/controller/v18/controllercontext"
	"github.com/giantswarm/aws-operator/service/controller/v18/key"
)

func (r *Resource) GetCurrentState(ctx context.Context, obj interface{}) (interface{}, error) {
	customObject, err := key.ToCustomObject(obj)
	if err != nil {
		return StackState{}, microerror.Mask(err)
	}

	stackName := key.MainGuestStackName(customObject)

	sc, err := controllercontext.FromContext(ctx)
	if err != nil {
		return StackState{}, microerror.Mask(err)
	}

	// In order to compute the current state of the guest cluster's cloud
	// formation stack we have to describe the CF stacks and lookup the right
	// stack. We dispatch our custom StackState structure and enrich it with all
	// information necessary to reconcile the cloudformation resource.
	var stackOutputs []*cloudformation.Output
	{
		r.logger.LogCtx(ctx, "level", "debug", "message", "finding the guest cluster main stack outputs in the AWS API")

		var stackStatus string
		stackOutputs, stackStatus, err = sc.CloudFormation.DescribeOutputsAndStatus(stackName)
		if cloudformationservice.IsStackNotFound(err) {
			r.logger.LogCtx(ctx, "level", "debug", "message", "did not find the guest cluster main stack outputs in the AWS API")
			r.logger.LogCtx(ctx, "level", "debug", "message", "the guest cluster main stack does not exist")
			return StackState{}, nil

		} else if cloudformationservice.IsOutputsNotAccessible(err) {
			r.logger.LogCtx(ctx, "level", "debug", "message", "did not find the guest cluster main stack outputs in the AWS API")
			r.logger.LogCtx(ctx, "level", "debug", "message", fmt.Sprintf("the guest cluster main stack has status '%s'", stackStatus))
			if key.IsDeleted(customObject) {
				// Keep finalizers to as long as we don't
				// encounter IsStackNotFound.
				r.logger.LogCtx(ctx, "level", "debug", "message", "keeping finalizers")
				finalizerskeptcontext.SetKept(ctx)
			}
			r.logger.LogCtx(ctx, "level", "debug", "message", "canceling resource")
			resourcecanceledcontext.SetCanceled(ctx)

			return StackState{}, nil

		} else if err != nil {
			return StackState{}, microerror.Mask(err)
		}

		r.logger.LogCtx(ctx, "level", "debug", "message", "found the guest cluster main stack outputs in the AWS API")
	}

	var currentState StackState
	{
		var hostedZoneNameServers string
		if r.route53Enabled {
			hostedZoneNameServers, err = sc.CloudFormation.GetOutputValue(stackOutputs, key.HostedZoneNameServers)
			// TODO introduced: aws-operator@v14; remove with: aws-operator@v13
			// This output was introduced in v14 so it isn't accessible from CF
			// stacks created by earlier versions. We need to handle that.
			//
			// Final version of the code:
			//
			//	if err != nil {
			//		return StackState{}, microerror.Mask(err)
			//	}
			//
			if cloudformationservice.IsOutputNotFound(err) {
				// Fall trough. Empty string is handled in host post stack creation.
			} else if err != nil {
				return StackState{}, microerror.Mask(err)
			}
			// TODO end
		}
		dockerVolumeResourceName, err := sc.CloudFormation.GetOutputValue(stackOutputs, key.DockerVolumeResourceNameKey)
		if cloudformationservice.IsOutputNotFound(err) {
			// Since we are transitioning between versions we will have situations in
			// which old clusters are updated to new versions and miss the docker
			// volume resource name in the CF stack outputs. We ignore this problem
			// for now and move on regardless. On the next resync period the output
			// value will be there, once the cluster got updated.
			//
			// TODO remove this condition as soon as all guest clusters in existence
			// obtain a docker volume resource.
			dockerVolumeResourceName = ""
		} else if err != nil {
			return StackState{}, microerror.Mask(err)
		}

		var workerDockerVolumeSizeGB int
		{
			v, err := sc.CloudFormation.GetOutputValue(stackOutputs, key.WorkerDockerVolumeSizeKey)
			if cloudformationservice.IsOutputNotFound(err) {
				// Since we are transitioning between versions we will have situations in
				// which old clusters are updated to new versions and miss the docker
				// volume resource name in the CF stack outputs. We ignore this problem
				// for now and move on regardless. On the next resync period the output
				// value will be there, once the cluster got updated.
				//
				// TODO remove this condition as soon as all guest clusters in existence
				// obtain a docker volume size. Tracked here: https://github.com/giantswarm/giantswarm/issues/4139.
				v = "100"
			} else if err != nil {
				return StackState{}, microerror.Mask(err)
			}

			sz, err := strconv.ParseUint(v, 10, 32)
			if err != nil {
				return StackState{}, microerror.Mask(err)
			}

			workerDockerVolumeSizeGB = int(sz)
		}

		masterImageID, err := sc.CloudFormation.GetOutputValue(stackOutputs, key.MasterImageIDKey)
		if err != nil {
			return StackState{}, microerror.Mask(err)
		}
		masterInstanceResourceName, err := sc.CloudFormation.GetOutputValue(stackOutputs, key.MasterInstanceResourceNameKey)
		if cloudformationservice.IsOutputNotFound(err) {
			// Since we are transitioning between versions we will have situations in
			// which old clusters are updated to new versions and miss the master
			// instance resource name in the CF stack outputs. We ignore this problem
			// for now and move on regardless. On the next resync period the output
			// value will be there, once the cluster got updated.
			//
			// TODO remove this condition as soon as all guest clusters in existence
			// obtain a master instance resource.
			masterInstanceResourceName = ""
		} else if err != nil {
			return StackState{}, microerror.Mask(err)
		}
		masterInstanceType, err := sc.CloudFormation.GetOutputValue(stackOutputs, key.MasterInstanceTypeKey)
		if err != nil {
			return StackState{}, microerror.Mask(err)
		}
		masterCloudConfigVersion, err := sc.CloudFormation.GetOutputValue(stackOutputs, key.MasterCloudConfigVersionKey)
		if err != nil {
			return StackState{}, microerror.Mask(err)
		}

		workerCount, err := sc.CloudFormation.GetOutputValue(stackOutputs, key.WorkerCountKey)
		if err != nil {
			return StackState{}, microerror.Mask(err)
		}
		workerImageID, err := sc.CloudFormation.GetOutputValue(stackOutputs, key.WorkerImageIDKey)
		if err != nil {
			return StackState{}, microerror.Mask(err)
		}
		workerInstanceType, err := sc.CloudFormation.GetOutputValue(stackOutputs, key.WorkerInstanceTypeKey)
		if err != nil {
			return StackState{}, microerror.Mask(err)
		}
		workerCloudConfigVersion, err := sc.CloudFormation.GetOutputValue(stackOutputs, key.WorkerCloudConfigVersionKey)
		if err != nil {
			return StackState{}, microerror.Mask(err)
		}

		versionBundleVersion, err := sc.CloudFormation.GetOutputValue(stackOutputs, key.VersionBundleVersionKey)
		if cloudformationservice.IsOutputNotFound(err) {
			// Since we are transitioning between versions we will have situations in
			// which old clusters are updated to new versions and miss the version
			// bundle version in the CF stack outputs. We ignore this problem for now
			// and move on regardless. The reconciliation will detect the guest cluster
			// needs to be updated and once this is done, we should be fine again.
			//
			// TODO remove this condition as soon as all guest clusters in existence
			// obtain a version bundle version.
			versionBundleVersion = ""
		} else if err != nil {
			return StackState{}, microerror.Mask(err)
		}

		currentState = StackState{
			Name: stackName,

			HostedZoneNameServers: hostedZoneNameServers,

			DockerVolumeResourceName:   dockerVolumeResourceName,
			MasterImageID:              masterImageID,
			MasterInstanceResourceName: masterInstanceResourceName,
			MasterInstanceType:         masterInstanceType,
			MasterCloudConfigVersion:   masterCloudConfigVersion,

			WorkerCount:              workerCount,
			WorkerDockerVolumeSizeGB: workerDockerVolumeSizeGB,
			WorkerImageID:            workerImageID,
			WorkerInstanceType:       workerInstanceType,
			WorkerCloudConfigVersion: workerCloudConfigVersion,

			VersionBundleVersion: versionBundleVersion,
		}
	}

	return currentState, nil
}
<commit_msg>v18/resource/cloudformation: keep finalizers when stacks still exist (#1237)<commit_after>package cloudformation

import (
	"context"
	"fmt"
	"strconv"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	cloudformationservice "github.com/giantswarm/aws-operator/service/controller/v18/cloudformation"
	"github.com/giantswarm/aws-operator/service/controller/v18/controllercontext"
	"github.com/giantswarm/aws-operator/service/controller/v18/key"
	"github.com/giantswarm/microerror"
	"github.com/giantswarm/operatorkit/controller/context/finalizerskeptcontext"
	"github.com/giantswarm/operatorkit/controller/context/resourcecanceledcontext"
)

func (r *Resource) GetCurrentState(ctx context.Context, obj interface{}) (interface{}, error) {
	customObject, err := key.ToCustomObject(obj)
	if err != nil {
		return StackState{}, microerror.Mask(err)
	}

	stackName := key.MainGuestStackName(customObject)

	ctlCtx, err := controllercontext.FromContext(ctx)
	if err != nil {
		return StackState{}, microerror.Mask(err)
	}

	if key.IsDeleted(customObject) {
		stackNames := []string{
			key.MainGuestStackName(customObject),
			key.MainHostPreStackName(customObject),
			key.MainHostPostStackName(customObject),
		}

		for _, stackName := range stackNames {
			r.logger.LogCtx(ctx, "level", "debug", "message", fmt.Sprintf("finding stack %#q in the AWS API", stackName))

			in := &cloudformation.DescribeStacksInput{
				StackName: aws.String(stackName),
			}

			_, err := ctlCtx.AWSClient.CloudFormation.DescribeStacks(in)
			if cloudformationservice.IsStackNotFound(err) {
				// This handling is far from perfect. We use different
				// packages here. This is all going to be addressed in
				// scope of
				// https://github.com/giantswarm/giantswarm/issues/3783.

				r.logger.LogCtx(ctx, "level", "debug", "message", fmt.Sprintf("did not find stack %#q in the AWS API", stackName))
			} else if err != nil {
				return nil, microerror.Mask(err)
			} else {
				r.logger.LogCtx(ctx, "level", "debug", "message", fmt.Sprintf("found stack %#q in the AWS API", stackName))
				r.logger.LogCtx(ctx, "level", "debug", "message", "keeping finalizer")
				finalizerskeptcontext.SetKept(ctx)
			}
		}
	}

	// In order to compute the current state of the guest cluster's cloud
	// formation stack we have to describe the CF stacks and lookup the right
	// stack. We dispatch our custom StackState structure and enrich it with all
	// information necessary to reconcile the cloudformation resource.
	var stackOutputs []*cloudformation.Output
	{
		r.logger.LogCtx(ctx, "level", "debug", "message", "finding the guest cluster main stack outputs in the AWS API")

		var stackStatus string
		stackOutputs, stackStatus, err = ctlCtx.CloudFormation.DescribeOutputsAndStatus(stackName)
		if cloudformationservice.IsStackNotFound(err) {
			r.logger.LogCtx(ctx, "level", "debug", "message", "did not find the guest cluster main stack outputs in the AWS API")
			r.logger.LogCtx(ctx, "level", "debug", "message", "the guest cluster main stack does not exist")
			return StackState{}, nil

		} else if cloudformationservice.IsOutputsNotAccessible(err) {
			r.logger.LogCtx(ctx, "level", "debug", "message", "did not find the guest cluster main stack outputs in the AWS API")
			r.logger.LogCtx(ctx, "level", "debug", "message", fmt.Sprintf("the guest cluster main stack has status '%s'", stackStatus))
			if key.IsDeleted(customObject) {
				// Keep finalizers to as long as we don't
				// encounter IsStackNotFound.
				r.logger.LogCtx(ctx, "level", "debug", "message", "keeping finalizers")
				finalizerskeptcontext.SetKept(ctx)
			}
			r.logger.LogCtx(ctx, "level", "debug", "message", "canceling resource")
			resourcecanceledcontext.SetCanceled(ctx)

			return StackState{}, nil

		} else if err != nil {
			return StackState{}, microerror.Mask(err)
		}

		r.logger.LogCtx(ctx, "level", "debug", "message", "found the guest cluster main stack outputs in the AWS API")
	}

	var currentState StackState
	{
		var hostedZoneNameServers string
		if r.route53Enabled {
			hostedZoneNameServers, err = ctlCtx.CloudFormation.GetOutputValue(stackOutputs, key.HostedZoneNameServers)
			// TODO introduced: aws-operator@v14; remove with: aws-operator@v13
			// This output was introduced in v14 so it isn't accessible from CF
			// stacks created by earlier versions. We need to handle that.
			//
			// Final version of the code:
			//
			//	if err != nil {
			//		return StackState{}, microerror.Mask(err)
			//	}
			//
			if cloudformationservice.IsOutputNotFound(err) {
				// Fall trough. Empty string is handled in host post stack creation.
			} else if err != nil {
				return StackState{}, microerror.Mask(err)
			}
			// TODO end
		}
		dockerVolumeResourceName, err := ctlCtx.CloudFormation.GetOutputValue(stackOutputs, key.DockerVolumeResourceNameKey)
		if cloudformationservice.IsOutputNotFound(err) {
			// Since we are transitioning between versions we will have situations in
			// which old clusters are updated to new versions and miss the docker
			// volume resource name in the CF stack outputs. We ignore this problem
			// for now and move on regardless. On the next resync period the output
			// value will be there, once the cluster got updated.
			//
			// TODO remove this condition as soon as all guest clusters in existence
			// obtain a docker volume resource.
			dockerVolumeResourceName = ""
		} else if err != nil {
			return StackState{}, microerror.Mask(err)
		}

		var workerDockerVolumeSizeGB int
		{
			v, err := ctlCtx.CloudFormation.GetOutputValue(stackOutputs, key.WorkerDockerVolumeSizeKey)
			if cloudformationservice.IsOutputNotFound(err) {
				// Since we are transitioning between versions we will have situations in
				// which old clusters are updated to new versions and miss the docker
				// volume resource name in the CF stack outputs. We ignore this problem
				// for now and move on regardless. On the next resync period the output
				// value will be there, once the cluster got updated.
				//
				// TODO remove this condition as soon as all guest clusters in existence
				// obtain a docker volume size. Tracked here: https://github.com/giantswarm/giantswarm/issues/4139.
				v = "100"
			} else if err != nil {
				return StackState{}, microerror.Mask(err)
			}

			sz, err := strconv.ParseUint(v, 10, 32)
			if err != nil {
				return StackState{}, microerror.Mask(err)
			}

			workerDockerVolumeSizeGB = int(sz)
		}

		masterImageID, err := ctlCtx.CloudFormation.GetOutputValue(stackOutputs, key.MasterImageIDKey)
		if err != nil {
			return StackState{}, microerror.Mask(err)
		}
		masterInstanceResourceName, err := ctlCtx.CloudFormation.GetOutputValue(stackOutputs, key.MasterInstanceResourceNameKey)
		if cloudformationservice.IsOutputNotFound(err) {
			// Since we are transitioning between versions we will have situations in
			// which old clusters are updated to new versions and miss the master
			// instance resource name in the CF stack outputs. We ignore this problem
			// for now and move on regardless. On the next resync period the output
			// value will be there, once the cluster got updated.
			//
			// TODO remove this condition as soon as all guest clusters in existence
			// obtain a master instance resource.
			masterInstanceResourceName = ""
		} else if err != nil {
			return StackState{}, microerror.Mask(err)
		}
		masterInstanceType, err := ctlCtx.CloudFormation.GetOutputValue(stackOutputs, key.MasterInstanceTypeKey)
		if err != nil {
			return StackState{}, microerror.Mask(err)
		}
		masterCloudConfigVersion, err := ctlCtx.CloudFormation.GetOutputValue(stackOutputs, key.MasterCloudConfigVersionKey)
		if err != nil {
			return StackState{}, microerror.Mask(err)
		}

		workerCount, err := ctlCtx.CloudFormation.GetOutputValue(stackOutputs, key.WorkerCountKey)
		if err != nil {
			return StackState{}, microerror.Mask(err)
		}
		workerImageID, err := ctlCtx.CloudFormation.GetOutputValue(stackOutputs, key.WorkerImageIDKey)
		if err != nil {
			return StackState{}, microerror.Mask(err)
		}
		workerInstanceType, err := ctlCtx.CloudFormation.GetOutputValue(stackOutputs, key.WorkerInstanceTypeKey)
		if err != nil {
			return StackState{}, microerror.Mask(err)
		}
		workerCloudConfigVersion, err := ctlCtx.CloudFormation.GetOutputValue(stackOutputs, key.WorkerCloudConfigVersionKey)
		if err != nil {
			return StackState{}, microerror.Mask(err)
		}

		versionBundleVersion, err := ctlCtx.CloudFormation.GetOutputValue(stackOutputs, key.VersionBundleVersionKey)
		if cloudformationservice.IsOutputNotFound(err) {
			// Since we are transitioning between versions we will have situations in
			// which old clusters are updated to new versions and miss the version
			// bundle version in the CF stack outputs. We ignore this problem for now
			// and move on regardless. The reconciliation will detect the guest cluster
			// needs to be updated and once this is done, we should be fine again.
			//
			// TODO remove this condition as soon as all guest clusters in existence
			// obtain a version bundle version.
			versionBundleVersion = ""
		} else if err != nil {
			return StackState{}, microerror.Mask(err)
		}

		currentState = StackState{
			Name: stackName,

			HostedZoneNameServers: hostedZoneNameServers,

			DockerVolumeResourceName:   dockerVolumeResourceName,
			MasterImageID:              masterImageID,
			MasterInstanceResourceName: masterInstanceResourceName,
			MasterInstanceType:         masterInstanceType,
			MasterCloudConfigVersion:   masterCloudConfigVersion,

			WorkerCount:              workerCount,
			WorkerDockerVolumeSizeGB: workerDockerVolumeSizeGB,
			WorkerImageID:            workerImageID,
			WorkerInstanceType:       workerInstanceType,
			WorkerCloudConfigVersion: workerCloudConfigVersion,

			VersionBundleVersion: versionBundleVersion,
		}
	}

	return currentState, nil
}
<|endoftext|>


<commit_before>package factory

import (
	"errors"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/Sirupsen/logrus"
)

type RKT struct {
	name string
	ID   string
}

func (this *RKT) init() {
	this.name = ""
	this.ID = ""
}

func (this *RKT) SetRT(runtime string) {
	this.name = "rkt"
}

func (this *RKT) GetRT() string {
	return "rkt"
}

func (this *RKT) GetRTID() string {
	return this.ID
}

func (this *RKT) Convert(appName string, workingDir string) (string, error) {
	var cmd *exec.Cmd
	aciName := appName + ".aci"
	//set appName to rkt appname, set rkt aciName to image name
	cmd = exec.Command("../plugins/oci2aci", "--debug", "--name", appName, appName, aciName)
	cmd.Dir = workingDir
	cmd.Stdin = os.Stdin

	out, err := cmd.CombinedOutput()

	logrus.Debugf("Command done")
	if err != nil {
		return string(out), errors.New(string(out) + err.Error())
	}

	return string(out), nil
}

func (this *RKT) StartRT(specDir string) (string, error) {
	logrus.Debugf("Launcing runtime")

	appName := filepath.Base(specDir)
	aciName := appName + ".aci"
	aciPath := filepath.Dir(specDir)

	if retStr, err := this.Convert(appName, aciPath); err != nil {
		return retStr, err
	}

	cmd := exec.Command("rkt", "run", aciName, "--interactive", "--insecure-skip-verify", "--mds"+
		"-register=false", "--volume", "proc,kind=host,source=/bin", "--volume", "dev,kind=host,"+
		"source=/bin", "--volume", "devpts,kind=host,source=/bin", "--volume", "shm,kind=host,"+
		"source=/bin", "--volume", "mqueue,kind=host,source=/bin", "--volume", "sysfs,kind=host,"+
		"source=/bin", "--volume", "cgroup,kind=host,source=/bin", "--net=host")
	cmd.Dir = aciPath
	cmd.Stdin = os.Stdin
	out, err := cmd.CombinedOutput()
	if err != nil {
		return string(out), err
	}
	logrus.Debugf("Command done")

	id, bv, ev := checkResult(appName)
	this.ID = id
	if ev != nil {
		return "", ev
	} else if !bv {
		return string(out), errors.New(string(out))
	}
	return string(out), nil
}

func checkResult(appName string) (string, bool, error) {

	//use rkt list to get uuid of rkt contianer
	cmd := exec.Command("rkt", "list")
	cmd.Stdin = os.Stdin
	listOut, err := cmd.CombinedOutput()
	if err != nil {
		logrus.Fatalf("rkt list err %v\n", err)
	}

	uuid, err := getUuid(string(listOut), appName)
	if err != nil {
		return "", false, errors.New("can not get uuid of rkt app" + appName)
	}
	logrus.Debugf("uuid: %v\n", uuid)

	//use rkt status to get status of app running in rkt container
	cmd = exec.Command("rkt", "status", uuid)
	statusOut, err := cmd.CombinedOutput()
	if err != nil {
		logrus.Fatalf("rkt status err %v\n", err)
	}
	logrus.Debugf("rkt stauts %v\n,%v\n", uuid, string(statusOut))

	s, err := getAppStatus(string(statusOut), appName)
	if s != 0 || err != nil {
		return uuid, false, err
	}

	return uuid, true, nil
}

func getAppStatus(Out string, appName string) (int64, error) {
	line, err := getLine(Out, appName)
	if err != nil {
		logrus.Debugln(err)
		return 1, err
	}
	a := strings.SplitAfter(line, "=")

	res, err := strconv.ParseInt(a[1], 10, 32)
	if err != nil {
		logrus.Debugln(err)
		return 1, err
	}
	return res, nil
}

func getUuid(listOut string, appName string) (string, error) {

	line, err := getLine(listOut, appName)
	if err != nil {
		logrus.Debugln(err)
		return "", err
	}

	return splitUuid(line), nil
}

func splitUuid(line string) string {

	a := strings.Fields(line)
	return strings.TrimSpace(a[0])
}

func getLine(Out string, objName string) (string, error) {

	outArray := strings.Split(Out, "\n")
	flag := false
	var wantLine string
	for _, o := range outArray {
		if strings.Contains(o, objName) {
			wantLine = o
			flag = true
			break
		}
	}

	if !flag {
		return wantLine, errors.New("no line containers " + objName)
	}
	return wantLine, nil
}

func (this *RKT) StopRT(id string) error {

	cmd := exec.Command("rkt", "rm", id)
	cmd.Stdin = os.Stdin
	_, _ = cmd.CombinedOutput()

	return nil
}
<commit_msg>rktFactory: tiny tfix<commit_after>package factory

import (
	"errors"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/Sirupsen/logrus"
)

type RKT struct {
	name string
	ID   string
}

func (this *RKT) init() {
	this.name = ""
	this.ID = ""
}

func (this *RKT) SetRT(runtime string) {
	this.name = "rkt"
}

func (this *RKT) GetRT() string {
	return "rkt"
}

func (this *RKT) GetRTID() string {
	return this.ID
}

func (this *RKT) Convert(appName string, workingDir string) (string, error) {
	var cmd *exec.Cmd
	aciName := appName + ".aci"
	//set appName to rkt appname, set rkt aciName to image name
	cmd = exec.Command("../plugins/oci2aci", "--debug", "--name", appName, appName, aciName)
	cmd.Dir = workingDir
	cmd.Stdin = os.Stdin

	out, err := cmd.CombinedOutput()

	logrus.Debugf("Command done")
	if err != nil {
		return string(out), errors.New(string(out) + err.Error())
	}

	return string(out), nil
}

func (this *RKT) StartRT(specDir string) (string, error) {
	logrus.Debugf("Launcing runtime")

	appName := filepath.Base(specDir)
	aciName := appName + ".aci"
	aciPath := filepath.Dir(specDir)

	if retStr, err := this.Convert(appName, aciPath); err != nil {
		return retStr, err
	}

	cmd := exec.Command("rkt", "run", aciName, "--interactive", "--insecure-skip-verify", "--mds"+
		"-register=false", "--volume", "proc,kind=host,source=/bin", "--volume", "dev,kind=host,"+
		"source=/bin", "--volume", "devpts,kind=host,source=/bin", "--volume", "shm,kind=host,"+
		"source=/bin", "--volume", "mqueue,kind=host,source=/bin", "--volume", "sysfs,kind=host,"+
		"source=/bin", "--volume", "cgroup,kind=host,source=/bin", "--net=host")
	cmd.Dir = aciPath
	cmd.Stdin = os.Stdin
	out, err := cmd.CombinedOutput()
	if err != nil {
		return string(out), err
	}
	logrus.Debugf("Command done")

	id, bv, ev := checkResult(appName)
	this.ID = id
	if ev != nil {
		return "", ev
	} else if !bv {
		return string(out), errors.New(string(out))
	}
	return string(out), nil
}

func checkResult(appName string) (string, bool, error) {

	//use rkt list to get uuid of rkt container
	cmd := exec.Command("rkt", "list")
	cmd.Stdin = os.Stdin
	listOut, err := cmd.CombinedOutput()
	if err != nil {
		logrus.Fatalf("rkt list err %v\n", err)
	}

	uuid, err := getUuid(string(listOut), appName)
	if err != nil {
		return "", false, errors.New("can not get uuid of rkt app" + appName)
	}
	logrus.Debugf("uuid: %v\n", uuid)

	//use rkt status to get status of app running in rkt container
	cmd = exec.Command("rkt", "status", uuid)
	statusOut, err := cmd.CombinedOutput()
	if err != nil {
		logrus.Fatalf("rkt status err %v\n", err)
	}
	logrus.Debugf("rkt stauts %v\n,%v\n", uuid, string(statusOut))

	s, err := getAppStatus(string(statusOut), appName)
	if s != 0 || err != nil {
		return uuid, false, err
	}

	return uuid, true, nil
}

func getAppStatus(Out string, appName string) (int64, error) {
	line, err := getLine(Out, appName)
	if err != nil {
		logrus.Debugln(err)
		return 1, err
	}
	a := strings.SplitAfter(line, "=")

	res, err := strconv.ParseInt(a[1], 10, 32)
	if err != nil {
		logrus.Debugln(err)
		return 1, err
	}
	return res, nil
}

func getUuid(listOut string, appName string) (string, error) {

	line, err := getLine(listOut, appName)
	if err != nil {
		logrus.Debugln(err)
		return "", err
	}

	return splitUuid(line), nil
}

func splitUuid(line string) string {

	a := strings.Fields(line)
	return strings.TrimSpace(a[0])
}

func getLine(Out string, objName string) (string, error) {

	outArray := strings.Split(Out, "\n")
	flag := false
	var wantLine string
	for _, o := range outArray {
		if strings.Contains(o, objName) {
			wantLine = o
			flag = true
			break
		}
	}

	if !flag {
		return wantLine, errors.New("no line containers " + objName)
	}
	return wantLine, nil
}

func (this *RKT) StopRT(id string) error {

	cmd := exec.Command("rkt", "rm", id)
	cmd.Stdin = os.Stdin
	_, _ = cmd.CombinedOutput()

	return nil
}
<|endoftext|>


<commit_before>package yandex_cloud_monitoring

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/outputs"
	"github.com/influxdata/telegraf/selfstat"
)

// YandexCloudMonitoring allows publishing of metrics to the Yandex Cloud Monitoring custom metrics
// service
type YandexCloudMonitoring struct {
	Timeout     internal.Duration `toml:"timeout"`
	EndpointURL string            `toml:"endpoint_url"`
	Service     string            `toml:"service"`

	Log telegraf.Logger

	MetadataTokenURL       string
	MetadataFolderURL      string
	FolderID               string
	IAMToken               string
	IamTokenExpirationTime time.Time

	client *http.Client

	timeFunc func() time.Time

	MetricOutsideWindow selfstat.Stat
}

type yandexCloudMonitoringMessage struct {
	TS      string                        `json:"ts,omitempty"`
	Labels  map[string]string             `json:"labels,omitempty"`
	Metrics []yandexCloudMonitoringMetric `json:"metrics"`
}

type yandexCloudMonitoringMetric struct {
	Name       string            `json:"name"`
	Labels     map[string]string `json:"labels"`
	MetricType string            `json:"type,omitempty"` // DGAUGE|IGAUGE|COUNTER|RATE. Default: DGAUGE
	TS         string            `json:"ts,omitempty"`
	Value      float64           `json:"value"`
}

type MetadataIamToken struct {
	AccessToken string `json:"access_token"`
	ExpiresIn   int64  `json:"expires_in"`
	TokenType   string `json:"token_type"`
}

const (
	defaultRequestTimeout    = time.Second * 20
	defaultEndpointURL       = "https://monitoring.api.cloud.yandex.net/monitoring/v2/data/write"
	defaultMetadataTokenURL  = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token"
	defaultMetadataFolderURL = "http://169.254.169.254/computeMetadata/v1/instance/attributes/folder-id"
)

var sampleConfig = `
  ## Timeout for HTTP writes.
  # timeout = "20s"

  ## Yandex.Cloud monitoring API endpoint. Normally should not be changed
  # endpoint_url = "https://monitoring.api.cloud.yandex.net/monitoring/v2/data/write"

  ## All user metrics should be sent with "custom" service specified. Normally should not be changed
  # service = "custom"
`

// Description provides a description of the plugin
func (a *YandexCloudMonitoring) Description() string {
	return "Send aggregated metrics to Yandex.Cloud Monitoring"
}

// SampleConfig provides a sample configuration for the plugin
func (a *YandexCloudMonitoring) SampleConfig() string {
	return sampleConfig
}

// Connect initializes the plugin and validates connectivity
func (a *YandexCloudMonitoring) Connect() error {
	if a.Timeout.Duration <= 0 {
		a.Timeout.Duration = defaultRequestTimeout
	}
	if a.EndpointURL == "" {
		a.EndpointURL = defaultEndpointURL
	}
	if a.Service == "" {
		a.Service = "custom"
	}
	if a.MetadataTokenURL == "" {
		a.MetadataTokenURL = defaultMetadataTokenURL
	}
	if a.MetadataFolderURL == "" {
		a.MetadataFolderURL = defaultMetadataFolderURL
	}

	a.client = &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
		},
		Timeout: a.Timeout.Duration,
	}

	var err error
	a.FolderID, err = a.getFolderIDFromMetadata()
	if err != nil {
		return err
	}

	a.Log.Infof("Writing to Yandex.Cloud Monitoring URL: %s", a.EndpointURL)

	tags := map[string]string{}
	a.MetricOutsideWindow = selfstat.Register("yandex_cloud_monitoring", "metric_outside_window", tags)

	return nil
}

// Close shuts down an any active connections
func (a *YandexCloudMonitoring) Close() error {
	a.client = nil
	return nil
}

// Write writes metrics to the remote endpoint
func (a *YandexCloudMonitoring) Write(metrics []telegraf.Metric) error {
	var yandexCloudMonitoringMetrics []yandexCloudMonitoringMetric
	for _, m := range metrics {
		for _, field := range m.FieldList() {
			yandexCloudMonitoringMetrics = append(
				yandexCloudMonitoringMetrics,
				yandexCloudMonitoringMetric{
					Name:   field.Key,
					Labels: m.Tags(),
					TS:     fmt.Sprint(m.Time().Format(time.RFC3339)),
					Value:  field.Value.(float64),
				},
			)
		}
	}

	var body []byte
	jsonBytes, err := json.Marshal(
		yandexCloudMonitoringMessage{
			Metrics: yandexCloudMonitoringMetrics,
		},
	)

	if err != nil {
		return err
	}
	body = append(jsonBytes, '\n')
	return a.send(body)
}

func getResponseFromMetadata(c *http.Client, metadataURL string) ([]byte, error) {
	req, err := http.NewRequest("GET", metadataURL, nil)
	if err != nil {
		return nil, fmt.Errorf("error creating request: %v", err)
	}
	req.Header.Set("Metadata-Flavor", "Google")
	resp, err := c.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode >= 300 || resp.StatusCode < 200 {
		return nil, fmt.Errorf("unable to fetch instance metadata: [%s] %d",
			metadataURL, resp.StatusCode)
	}
	return body, nil
}

func (a *YandexCloudMonitoring) getFolderIDFromMetadata() (string, error) {
	a.Log.Infof("getting folder ID in %s", a.MetadataFolderURL)
	body, err := getResponseFromMetadata(a.client, a.MetadataFolderURL)
	if err != nil {
		return "", err
	}
	folderID := string(body)
	if folderID == "" {
		return "", fmt.Errorf("unable to fetch folder id from URL %s: %v", a.MetadataFolderURL, err)
	}
	return folderID, nil
}

func (a *YandexCloudMonitoring) getIAMTokenFromMetadata() (string, int, error) {
	a.Log.Debugf("getting new IAM token in %s", a.MetadataTokenURL)
	body, err := getResponseFromMetadata(a.client, a.MetadataTokenURL)
	if err != nil {
		return "", 0, err
	}
	var metadata MetadataIamToken
	if err := json.Unmarshal(body, &metadata); err != nil {
		return "", 0, err
	}
	if metadata.AccessToken == "" || metadata.ExpiresIn == 0 {
		return "", 0, fmt.Errorf("unable to fetch authentication credentials %s: %v", a.MetadataTokenURL, err)
	}
	return metadata.AccessToken, int(metadata.ExpiresIn), nil
}

func (a *YandexCloudMonitoring) send(body []byte) error {
	req, err := http.NewRequest("POST", a.EndpointURL, bytes.NewBuffer(body))
	if err != nil {
		return err
	}
	q := req.URL.Query()
	q.Add("folderId", a.FolderID)
	q.Add("service", a.Service)
	req.URL.RawQuery = q.Encode()

	req.Header.Set("Content-Type", "application/json")
	isTokenExpired := !a.IamTokenExpirationTime.After(time.Now())
	if a.IAMToken == "" || isTokenExpired {
		token, expiresIn, err := a.getIAMTokenFromMetadata()
		if err != nil {
			return err
		}
		a.IamTokenExpirationTime = time.Now().Add(time.Duration(expiresIn) * time.Second)
		a.IAMToken = token
	}
	req.Header.Set("Authorization", "Bearer "+a.IAMToken)

	a.Log.Debugf("sending metrics to %s", req.URL.String())
	resp, err := a.client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	_, err = ioutil.ReadAll(resp.Body)
	if err != nil || resp.StatusCode < 200 || resp.StatusCode > 299 {
		return fmt.Errorf("failed to write batch: [%v] %s", resp.StatusCode, resp.Status)
	}

	return nil
}

func init() {
	outputs.Add("yandex_cloud_monitoring", func() telegraf.Output {
		return &YandexCloudMonitoring{
			timeFunc: time.Now,
		}
	})
}
<commit_msg>use correct compute metadata url to get folder-id (#9056)<commit_after>package yandex_cloud_monitoring

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/outputs"
	"github.com/influxdata/telegraf/selfstat"
)

// YandexCloudMonitoring allows publishing of metrics to the Yandex Cloud Monitoring custom metrics
// service
type YandexCloudMonitoring struct {
	Timeout     internal.Duration `toml:"timeout"`
	EndpointURL string            `toml:"endpoint_url"`
	Service     string            `toml:"service"`

	Log telegraf.Logger

	MetadataTokenURL       string
	MetadataFolderURL      string
	FolderID               string
	IAMToken               string
	IamTokenExpirationTime time.Time

	client *http.Client

	timeFunc func() time.Time

	MetricOutsideWindow selfstat.Stat
}

type yandexCloudMonitoringMessage struct {
	TS      string                        `json:"ts,omitempty"`
	Labels  map[string]string             `json:"labels,omitempty"`
	Metrics []yandexCloudMonitoringMetric `json:"metrics"`
}

type yandexCloudMonitoringMetric struct {
	Name       string            `json:"name"`
	Labels     map[string]string `json:"labels"`
	MetricType string            `json:"type,omitempty"` // DGAUGE|IGAUGE|COUNTER|RATE. Default: DGAUGE
	TS         string            `json:"ts,omitempty"`
	Value      float64           `json:"value"`
}

type MetadataIamToken struct {
	AccessToken string `json:"access_token"`
	ExpiresIn   int64  `json:"expires_in"`
	TokenType   string `json:"token_type"`
}

const (
	defaultRequestTimeout    = time.Second * 20
	defaultEndpointURL       = "https://monitoring.api.cloud.yandex.net/monitoring/v2/data/write"
	defaultMetadataTokenURL  = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token"
	defaultMetadataFolderURL = "http://169.254.169.254/computeMetadata/v1/yandex/folder-id"
)

var sampleConfig = `
  ## Timeout for HTTP writes.
  # timeout = "20s"

  ## Yandex.Cloud monitoring API endpoint. Normally should not be changed
  # endpoint_url = "https://monitoring.api.cloud.yandex.net/monitoring/v2/data/write"

  ## All user metrics should be sent with "custom" service specified. Normally should not be changed
  # service = "custom"
`

// Description provides a description of the plugin
func (a *YandexCloudMonitoring) Description() string {
	return "Send aggregated metrics to Yandex.Cloud Monitoring"
}

// SampleConfig provides a sample configuration for the plugin
func (a *YandexCloudMonitoring) SampleConfig() string {
	return sampleConfig
}

// Connect initializes the plugin and validates connectivity
func (a *YandexCloudMonitoring) Connect() error {
	if a.Timeout.Duration <= 0 {
		a.Timeout.Duration = defaultRequestTimeout
	}
	if a.EndpointURL == "" {
		a.EndpointURL = defaultEndpointURL
	}
	if a.Service == "" {
		a.Service = "custom"
	}
	if a.MetadataTokenURL == "" {
		a.MetadataTokenURL = defaultMetadataTokenURL
	}
	if a.MetadataFolderURL == "" {
		a.MetadataFolderURL = defaultMetadataFolderURL
	}

	a.client = &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
		},
		Timeout: a.Timeout.Duration,
	}

	var err error
	a.FolderID, err = a.getFolderIDFromMetadata()
	if err != nil {
		return err
	}

	a.Log.Infof("Writing to Yandex.Cloud Monitoring URL: %s", a.EndpointURL)

	tags := map[string]string{}
	a.MetricOutsideWindow = selfstat.Register("yandex_cloud_monitoring", "metric_outside_window", tags)

	return nil
}

// Close shuts down an any active connections
func (a *YandexCloudMonitoring) Close() error {
	a.client = nil
	return nil
}

// Write writes metrics to the remote endpoint
func (a *YandexCloudMonitoring) Write(metrics []telegraf.Metric) error {
	var yandexCloudMonitoringMetrics []yandexCloudMonitoringMetric
	for _, m := range metrics {
		for _, field := range m.FieldList() {
			yandexCloudMonitoringMetrics = append(
				yandexCloudMonitoringMetrics,
				yandexCloudMonitoringMetric{
					Name:   field.Key,
					Labels: m.Tags(),
					TS:     fmt.Sprint(m.Time().Format(time.RFC3339)),
					Value:  field.Value.(float64),
				},
			)
		}
	}

	var body []byte
	jsonBytes, err := json.Marshal(
		yandexCloudMonitoringMessage{
			Metrics: yandexCloudMonitoringMetrics,
		},
	)

	if err != nil {
		return err
	}
	body = append(jsonBytes, '\n')
	return a.send(body)
}

func getResponseFromMetadata(c *http.Client, metadataURL string) ([]byte, error) {
	req, err := http.NewRequest("GET", metadataURL, nil)
	if err != nil {
		return nil, fmt.Errorf("error creating request: %v", err)
	}
	req.Header.Set("Metadata-Flavor", "Google")
	resp, err := c.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode >= 300 || resp.StatusCode < 200 {
		return nil, fmt.Errorf("unable to fetch instance metadata: [%s] %d",
			metadataURL, resp.StatusCode)
	}
	return body, nil
}

func (a *YandexCloudMonitoring) getFolderIDFromMetadata() (string, error) {
	a.Log.Infof("getting folder ID in %s", a.MetadataFolderURL)
	body, err := getResponseFromMetadata(a.client, a.MetadataFolderURL)
	if err != nil {
		return "", err
	}
	folderID := string(body)
	if folderID == "" {
		return "", fmt.Errorf("unable to fetch folder id from URL %s: %v", a.MetadataFolderURL, err)
	}
	return folderID, nil
}

func (a *YandexCloudMonitoring) getIAMTokenFromMetadata() (string, int, error) {
	a.Log.Debugf("getting new IAM token in %s", a.MetadataTokenURL)
	body, err := getResponseFromMetadata(a.client, a.MetadataTokenURL)
	if err != nil {
		return "", 0, err
	}
	var metadata MetadataIamToken
	if err := json.Unmarshal(body, &metadata); err != nil {
		return "", 0, err
	}
	if metadata.AccessToken == "" || metadata.ExpiresIn == 0 {
		return "", 0, fmt.Errorf("unable to fetch authentication credentials %s: %v", a.MetadataTokenURL, err)
	}
	return metadata.AccessToken, int(metadata.ExpiresIn), nil
}

func (a *YandexCloudMonitoring) send(body []byte) error {
	req, err := http.NewRequest("POST", a.EndpointURL, bytes.NewBuffer(body))
	if err != nil {
		return err
	}
	q := req.URL.Query()
	q.Add("folderId", a.FolderID)
	q.Add("service", a.Service)
	req.URL.RawQuery = q.Encode()

	req.Header.Set("Content-Type", "application/json")
	isTokenExpired := !a.IamTokenExpirationTime.After(time.Now())
	if a.IAMToken == "" || isTokenExpired {
		token, expiresIn, err := a.getIAMTokenFromMetadata()
		if err != nil {
			return err
		}
		a.IamTokenExpirationTime = time.Now().Add(time.Duration(expiresIn) * time.Second)
		a.IAMToken = token
	}
	req.Header.Set("Authorization", "Bearer "+a.IAMToken)

	a.Log.Debugf("sending metrics to %s", req.URL.String())
	a.Log.Debugf("body: %s", body)
	resp, err := a.client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	_, err = ioutil.ReadAll(resp.Body)
	if err != nil || resp.StatusCode < 200 || resp.StatusCode > 299 {
		return fmt.Errorf("failed to write batch: [%v] %s", resp.StatusCode, resp.Status)
	}

	return nil
}

func init() {
	outputs.Add("yandex_cloud_monitoring", func() telegraf.Output {
		return &YandexCloudMonitoring{
			timeFunc: time.Now,
		}
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io/ioutil"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/registry"
	"k8s.io/kops/pkg/client/simple/vfsclientset"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kops/upup/pkg/fi/utils"
)

func up() error {
	clientset := vfsclientset.NewVFSClientset(registryBase)

	cluster := &api.Cluster{}
	cluster.ObjectMeta.Name = clusterName
	cluster.Spec = api.ClusterSpec{
		Channel:       "stable",
		CloudProvider: "aws",
		ConfigBase:    registryBase.Join(cluster.ObjectMeta.Name).Path(),
		Topology:      &api.TopologySpec{},
	}
	cluster.Spec.Topology.Masters = api.TopologyPublic
	cluster.Spec.Topology.Nodes = api.TopologyPublic

	for _, z := range nodeZones {
		cluster.Spec.Zones = append(cluster.Spec.Zones, &api.ClusterZoneSpec{
			Name: z,
		})
	}

	for _, etcdClusterName := range cloudup.EtcdClusters {
		etcdCluster := &api.EtcdClusterSpec{
			Name: etcdClusterName,
		}
		for _, masterZone := range masterZones {
			etcdMember := &api.EtcdMemberSpec{
				Name: masterZone,
				Zone: fi.String(masterZone),
			}
			etcdCluster.Members = append(etcdCluster.Members, etcdMember)
		}
		cluster.Spec.EtcdClusters = append(cluster.Spec.EtcdClusters, etcdCluster)
	}

	if err := cluster.PerformAssignments(); err != nil {
		return err
	}

	_, err := clientset.Clusters().Create(cluster)
	if err != nil {
		return err
	}

	// Create master ig
	{
		ig := &api.InstanceGroup{}
		ig.ObjectMeta.Name = "master"
		ig.Spec = api.InstanceGroupSpec{
			Role:  api.InstanceGroupRoleMaster,
			Zones: masterZones,
		}
		_, err := clientset.InstanceGroups(cluster.ObjectMeta.Name).Create(ig)
		if err != nil {
			return err
		}
	}

	// Create node ig
	{
		ig := &api.InstanceGroup{}
		ig.ObjectMeta.Name = "nodes"
		ig.Spec = api.InstanceGroupSpec{
			Role:  api.InstanceGroupRoleNode,
			Zones: nodeZones,
		}

		_, err := clientset.InstanceGroups(cluster.ObjectMeta.Name).Create(ig)
		if err != nil {
			return err
		}
	}

	keyStore, err := registry.KeyStore(cluster)
	if err != nil {
		return err
	}

	// Add a public key
	{
		f := utils.ExpandPath(sshPublicKey)
		pubKey, err := ioutil.ReadFile(f)
		if err != nil {
			return fmt.Errorf("error reading SSH key file %q: %v", f, err)
		}
		err = keyStore.AddSSHPublicKey(fi.SecretNameSSHPrimary, pubKey)
		if err != nil {
			return fmt.Errorf("error addding SSH public key: %v", err)
		}
	}

	return nil
}
<commit_msg>Fix example code<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io/ioutil"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/registry"
	"k8s.io/kops/pkg/client/simple/vfsclientset"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kops/upup/pkg/fi/utils"
)

func up() error {
	clientset := vfsclientset.NewVFSClientset(registryBase)

	cluster := &api.Cluster{}
	cluster.ObjectMeta.Name = clusterName
	cluster.Spec = api.ClusterSpec{
		Channel:       "stable",
		CloudProvider: "aws",
		ConfigBase:    registryBase.Join(cluster.ObjectMeta.Name).Path(),
		Topology:      &api.TopologySpec{},
	}
	cluster.Spec.Topology.Masters = api.TopologyPublic
	cluster.Spec.Topology.Nodes = api.TopologyPublic

	for _, z := range nodeZones {
		cluster.Spec.Subnets = append(cluster.Spec.Subnets, api.ClusterSubnetSpec{
			Name: z,
			Zone: z,
			Type: api.SubnetTypePublic,
		})
	}

	for _, etcdClusterName := range cloudup.EtcdClusters {
		etcdCluster := &api.EtcdClusterSpec{
			Name: etcdClusterName,
		}
		for _, masterZone := range masterZones {
			etcdMember := &api.EtcdMemberSpec{
				Name:          masterZone,
				InstanceGroup: fi.String(masterZone),
			}
			etcdCluster.Members = append(etcdCluster.Members, etcdMember)
		}
		cluster.Spec.EtcdClusters = append(cluster.Spec.EtcdClusters, etcdCluster)
	}

	if err := cloudup.PerformAssignments(cluster); err != nil {
		return err
	}

	_, err := clientset.Clusters().Create(cluster)
	if err != nil {
		return err
	}

	// Create master ig
	{
		ig := &api.InstanceGroup{}
		ig.ObjectMeta.Name = "master"
		ig.Spec = api.InstanceGroupSpec{
			Role:    api.InstanceGroupRoleMaster,
			Subnets: masterZones,
		}
		_, err := clientset.InstanceGroups(cluster.ObjectMeta.Name).Create(ig)
		if err != nil {
			return err
		}
	}

	// Create node ig
	{
		ig := &api.InstanceGroup{}
		ig.ObjectMeta.Name = "nodes"
		ig.Spec = api.InstanceGroupSpec{
			Role:    api.InstanceGroupRoleNode,
			Subnets: nodeZones,
		}

		_, err := clientset.InstanceGroups(cluster.ObjectMeta.Name).Create(ig)
		if err != nil {
			return err
		}
	}

	keyStore, err := registry.KeyStore(cluster)
	if err != nil {
		return err
	}

	// Add a public key
	{
		f := utils.ExpandPath(sshPublicKey)
		pubKey, err := ioutil.ReadFile(f)
		if err != nil {
			return fmt.Errorf("error reading SSH key file %q: %v", f, err)
		}
		err = keyStore.AddSSHPublicKey(fi.SecretNameSSHPrimary, pubKey)
		if err != nil {
			return fmt.Errorf("error addding SSH public key: %v", err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/base64"
	"io/ioutil"
	"log"
	"os"
)

// FileTokenStorage implements the TokenStorage interface.
// It is one possible implementation for storing user credentials.
type FileTokenStorage struct {
	token         string
	tokenFileName string
}

// SetFileName sets the file name used to store the token in
func (ts *FileTokenStorage) SetFileName(path string) {
	ts.tokenFileName = path
}

// Save persists the token to a file
func (ts *FileTokenStorage) Save(token token) error {
	log.Printf("Saving token [%s]", token.tokenString)
	encodedToken := base64.StdEncoding.EncodeToString([]byte(token.tokenString))
	log.Printf("Saving token (encoded) [%s]", encodedToken)
	ts.token = encodedToken
	return ioutil.WriteFile(ts.tokenFileName, []byte(encodedToken), 0644)
}

// Load loads the persisted token from file
func (ts *FileTokenStorage) Load() (*token, error) {
	if _,err := os.Stat(ts.tokenFileName); err == nil {
		out, err := ioutil.ReadFile(ts.tokenFileName)
		ts.token = string(out)
		log.Printf("Loaded token [%s]", ts.token)
		return Token(ts.token), err
	}
	token := Token("")
	token.Invalidate()
	return token, nil
}
<commit_msg>Removed loggong from file_token_storage<commit_after>package api

import (
	"encoding/base64"
	"io/ioutil"
	"log"
	"os"
)

// FileTokenStorage implements the TokenStorage interface.
// It is one possible implementation for storing user credentials.
type FileTokenStorage struct {
	token         string
	tokenFileName string
}

// SetFileName sets the file name used to store the token in
func (ts *FileTokenStorage) SetFileName(path string) {
	ts.tokenFileName = path
}

// Save persists the token to a file
func (ts *FileTokenStorage) Save(token token) error {
	encodedToken := base64.StdEncoding.EncodeToString([]byte(token.tokenString))
	ts.token = encodedToken
	return ioutil.WriteFile(ts.tokenFileName, []byte(encodedToken), 0644)
}

// Load loads the persisted token from file
func (ts *FileTokenStorage) Load() (*token, error) {
	if _,err := os.Stat(ts.tokenFileName); err == nil {
		out, err := ioutil.ReadFile(ts.tokenFileName)
		ts.token = string(out)
		return Token(ts.token), err
	}
	token := Token("")
	token.Invalidate()
	return token, nil
}
<|endoftext|>


<commit_before><commit_msg>scrape list of languages.<commit_after><|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"errors"
	//"fmt"
	"github.com/jmoiron/sqlx"
	"github.com/pborman/uuid"
	"net/http"
)

type Database struct {
	db *sqlx.DB
}

func (db *Database) readDataFromPlot(user string, plotId int) ([]Measurement, error) {
	measurements := []Measurement{}
	var sql = `
        WITH instruments as (
            SELECT key FROM
            instrument
            WHERE plot = $1
        )
        SELECT m.key, m.value, m.timestamp
        FROM measurement m, plot p
        WHERE m.timestamp >= p.start_time
        AND(p.end_time is null OR m.timestamp <= p.end_time)
        AND m.key IN (SELECT key from instruments)
        AND p.id = $1
        AND p.login = $2
        ORDER BY m.timestamp desc;
    `
	err := db.db.Select(&measurements, sql, plotId, user)
	return measurements, err
}

func (db *Database) readLatestDataFromPlot(user string, plotId int) ([]Measurement, error) {
	measurements := []Measurement{}
	var sql = `
        WITH
        instruments as (
            SELECT i.key AS keys
            FROM instrument i
            WHERE plot = $1
        ),
        latest_measurement as (

            SELECT max(m.timestamp) as timestamp
            FROM measurement m, plot p
            WHERE m.timestamp >= p.start_time
            AND(p.end_time is null OR m.timestamp <= p.end_time)
            AND p.id = $1
        )
        SELECT m.key, m.value, m.timestamp
        FROM measurement m, plot p
        WHERE m.timestamp >= p.start_time
        and m.timestamp = (select timestamp from latest_measurement)
        AND m.key IN (SELECT i.keys from instruments i)
        AND p.id = $1
        AND p.login = $2
        ORDER BY m.timestamp desc;
    `
	err := db.db.Select(&measurements, sql, plotId, user)
	return measurements, err
}

func (db *Database) readHourlyDataFromPlot(user string, plotId int) ([]Measurement, error) {
	measurements := []Measurement{}
	var sql = `
        WITH instruments as (
            SELECT key AS keys
            FROM instrument
            WHERE plot = $1
        )
        SELECT
            m.key,
            round(cast(avg(value) as numeric),0) AS value,
            m.timestamp::date::timestamp + make_interval(hours => DATE_PART('HOUR', m.timestamp)::integer) as timestamp
        FROM measurement m, plot p
        WHERE m.timestamp >= p.start_time
        AND m.key IN (SELECT i.keys from instruments i)
        AND p.id = $1
        AND p.login = $2
        GROUP BY m.key, timestamp
        ORDER BY timestamp desc;
    `
	err := db.db.Select(&measurements, sql, plotId, user)
	return measurements, err
}

func (db *Database) readMeasurements(user string, name string) ([]Measurement, error) {
	measurements := []Measurement{}

	var sql = `
        SELECT key, value, timestamp
        FROM measurement
        WHERE name = $1
        AND login = $2
        ORDER BY timestamp
    `

	err := db.db.Select(&measurements, sql, name, user)
	return measurements, err
}

func (db *Database) saveMeasurements(measurements []Measurement, user string) error {
	tx := db.db.MustBegin()
	var sql = `
        INSERT INTO measurement (key, value, timestamp, login)
        VALUES (:key, :value, :timestamp, :login)
    `
	for _, measurement := range measurements {
		measurement.Login = user
		tx.NamedExec(sql, &measurement)
	}

	err := tx.Commit()
	if err != nil {
		return err
	}
	return nil
}

func (db *Database) getPlots(user string) ([]Plot, error) {
	plots := []Plot{}

	var sql = `
        SELECT id, start_time, end_time, name, case when end_time IS null then true else false end as active
        FROM plot
        WHERE login = $1
        ORDER BY start_time DESC
    `

	err := db.db.Select(&plots, sql, user)
	return plots, err
}

func (db *Database) getInstruments(plotId int) ([]Instrument, error) {
	instruments := []Instrument{}

	var sql = `
        SELECT key, id, name, type
        FROM instrument
        WHERE plot = $1
    `

	err := db.db.Select(&instruments, sql, plotId)
	return instruments, err
}

func (db *Database) getPlot(id int, user string) (Plot, error) {
	plot := Plot{}

	var sql = `
        SELECT id, start_time, end_time, name
        FROM plot
        WHERE login = $1
        AND id = $2
    `

	err := db.db.Get(&plot, sql, user, id)
	return plot, err
}

func (db *Database) savePlot(plot Plot, user string) (Plot, error) {

	plot.Login = user

	var sql = `
        INSERT INTO plot (start_time, end_time, name, login) VALUES (:start_time, :end_time, :name, :login) RETURNING id
    `
	var id int
	rows, err := db.db.NamedQuery(sql, plot)
	if err != nil {
		return plot, err
	}
	if rows.Next() {
		rows.Scan(&id)
	}
	plot.Id = id
	tx := db.db.MustBegin()

	var sql2 = `
        INSERT INTO instrument (key, name, type, plot)
        VALUES (:key, :name, :type, :plot)
    `
	for _, instrument := range plot.Instruments {
		instrument.Plot = plot.Id
		tx.NamedExec(sql2, &instrument)
	}
	tx.Commit()
	return plot, err
}

func (db *Database) getUser(r *http.Request) (string, error) {
	key := r.Header.Get("X-PYTILT-KEY")
	return db.getUserForKey(key)
}

func (db *Database) getUserForKey(key string) (string, error) {
	var id string
	err := db.db.Get(&id, "SELECT id FROM login WHERE key = $1", key)

	if err == sql.ErrNoRows {
		return "", errors.New("unknown key")
	}
	return id, err
}

func (db *Database) getkeyForUser(user string) (string, error) {
	var key string
	err := db.db.Get(&key, "SELECT key FROM login WHERE id = $1", user)

	if err == sql.ErrNoRows {
		return "", errors.New("unknown user")
	}
	return key, err
}

func (db *Database) userExists(id string) (bool, error) {
	var uid string
	if err := db.db.QueryRow("SELECT id FROM login WHERE id = $1", id).Scan(&uid); err == nil {
		return true, nil
	} else if err == sql.ErrNoRows {
		return false, nil
	} else {
		return false, err
	}

}

func (db *Database) createUser(id string, email string, name string) error {
	tx := db.db.MustBegin()
	key := uuid.New()
	var sql = `
        INSERT INTO login (id, name, email, key)
        VALUES ($1, $2, $3, $4)
    `
	tx.MustExec(sql, id, name, email, key)
	tx.Commit()
	return nil
}
<commit_msg>include active prop on single plot<commit_after>package main

import (
	"database/sql"
	"errors"
	//"fmt"
	"github.com/jmoiron/sqlx"
	"github.com/pborman/uuid"
	"net/http"
)

type Database struct {
	db *sqlx.DB
}

func (db *Database) readDataFromPlot(user string, plotId int) ([]Measurement, error) {
	measurements := []Measurement{}
	var sql = `
        WITH instruments as (
            SELECT key FROM
            instrument
            WHERE plot = $1
        )
        SELECT m.key, m.value, m.timestamp
        FROM measurement m, plot p
        WHERE m.timestamp >= p.start_time
        AND(p.end_time is null OR m.timestamp <= p.end_time)
        AND m.key IN (SELECT key from instruments)
        AND p.id = $1
        AND p.login = $2
        ORDER BY m.timestamp desc;
    `
	err := db.db.Select(&measurements, sql, plotId, user)
	return measurements, err
}

func (db *Database) readLatestDataFromPlot(user string, plotId int) ([]Measurement, error) {
	measurements := []Measurement{}
	var sql = `
        WITH
        instruments as (
            SELECT i.key AS keys
            FROM instrument i
            WHERE plot = $1
        ),
        latest_measurement as (

            SELECT max(m.timestamp) as timestamp
            FROM measurement m, plot p
            WHERE m.timestamp >= p.start_time
            AND(p.end_time is null OR m.timestamp <= p.end_time)
            AND p.id = $1
        )
        SELECT m.key, m.value, m.timestamp
        FROM measurement m, plot p
        WHERE m.timestamp >= p.start_time
        and m.timestamp = (select timestamp from latest_measurement)
        AND m.key IN (SELECT i.keys from instruments i)
        AND p.id = $1
        AND p.login = $2
        ORDER BY m.timestamp desc;
    `
	err := db.db.Select(&measurements, sql, plotId, user)
	return measurements, err
}

func (db *Database) readHourlyDataFromPlot(user string, plotId int) ([]Measurement, error) {
	measurements := []Measurement{}
	var sql = `
        WITH instruments as (
            SELECT key AS keys
            FROM instrument
            WHERE plot = $1
        )
        SELECT
            m.key,
            round(cast(avg(value) as numeric),0) AS value,
            m.timestamp::date::timestamp + make_interval(hours => DATE_PART('HOUR', m.timestamp)::integer) as timestamp
        FROM measurement m, plot p
        WHERE m.timestamp >= p.start_time
        AND m.key IN (SELECT i.keys from instruments i)
        AND p.id = $1
        AND p.login = $2
        GROUP BY m.key, timestamp
        ORDER BY timestamp desc;
    `
	err := db.db.Select(&measurements, sql, plotId, user)
	return measurements, err
}

func (db *Database) readMeasurements(user string, name string) ([]Measurement, error) {
	measurements := []Measurement{}

	var sql = `
        SELECT key, value, timestamp
        FROM measurement
        WHERE name = $1
        AND login = $2
        ORDER BY timestamp
    `

	err := db.db.Select(&measurements, sql, name, user)
	return measurements, err
}

func (db *Database) saveMeasurements(measurements []Measurement, user string) error {
	tx := db.db.MustBegin()
	var sql = `
        INSERT INTO measurement (key, value, timestamp, login)
        VALUES (:key, :value, :timestamp, :login)
    `
	for _, measurement := range measurements {
		measurement.Login = user
		tx.NamedExec(sql, &measurement)
	}

	err := tx.Commit()
	if err != nil {
		return err
	}
	return nil
}

func (db *Database) getPlots(user string) ([]Plot, error) {
	plots := []Plot{}

	var sql = `
        SELECT id, start_time, end_time, name, case when end_time IS null then true else false end as active
        FROM plot
        WHERE login = $1
        ORDER BY start_time DESC
    `

	err := db.db.Select(&plots, sql, user)
	return plots, err
}

func (db *Database) getInstruments(plotId int) ([]Instrument, error) {
	instruments := []Instrument{}

	var sql = `
        SELECT key, id, name, type
        FROM instrument
        WHERE plot = $1
    `

	err := db.db.Select(&instruments, sql, plotId)
	return instruments, err
}

func (db *Database) getPlot(id int, user string) (Plot, error) {
	plot := Plot{}

	var sql = `
        SELECT id, start_time, end_time, name, case when end_time IS null then true else false end as active
        FROM plot
        WHERE login = $1
        AND id = $2
    `

	err := db.db.Get(&plot, sql, user, id)
	return plot, err
}

func (db *Database) savePlot(plot Plot, user string) (Plot, error) {

	plot.Login = user

	var sql = `
        INSERT INTO plot (start_time, end_time, name, login) VALUES (:start_time, :end_time, :name, :login) RETURNING id
    `
	var id int
	rows, err := db.db.NamedQuery(sql, plot)
	if err != nil {
		return plot, err
	}
	if rows.Next() {
		rows.Scan(&id)
	}
	plot.Id = id
	tx := db.db.MustBegin()

	var sql2 = `
        INSERT INTO instrument (key, name, type, plot)
        VALUES (:key, :name, :type, :plot)
    `
	for _, instrument := range plot.Instruments {
		instrument.Plot = plot.Id
		tx.NamedExec(sql2, &instrument)
	}
	tx.Commit()
	return plot, err
}

func (db *Database) getUser(r *http.Request) (string, error) {
	key := r.Header.Get("X-PYTILT-KEY")
	return db.getUserForKey(key)
}

func (db *Database) getUserForKey(key string) (string, error) {
	var id string
	err := db.db.Get(&id, "SELECT id FROM login WHERE key = $1", key)

	if err == sql.ErrNoRows {
		return "", errors.New("unknown key")
	}
	return id, err
}

func (db *Database) getkeyForUser(user string) (string, error) {
	var key string
	err := db.db.Get(&key, "SELECT key FROM login WHERE id = $1", user)

	if err == sql.ErrNoRows {
		return "", errors.New("unknown user")
	}
	return key, err
}

func (db *Database) userExists(id string) (bool, error) {
	var uid string
	if err := db.db.QueryRow("SELECT id FROM login WHERE id = $1", id).Scan(&uid); err == nil {
		return true, nil
	} else if err == sql.ErrNoRows {
		return false, nil
	} else {
		return false, err
	}

}

func (db *Database) createUser(id string, email string, name string) error {
	tx := db.db.MustBegin()
	key := uuid.New()
	var sql = `
        INSERT INTO login (id, name, email, key)
        VALUES ($1, $2, $3, $4)
    `
	tx.MustExec(sql, id, name, email, key)
	tx.Commit()
	return nil
}
<|endoftext|>


<commit_before>package gokeepasslib

import (
	"crypto/sha256"
	"errors"
	"fmt"
	"crypto/aes"
	"crypto/cipher"
)

// ErrUnsupportedStreamType is retured if no streamManager can be created
// due to an unsupported InnerRandomStreamID value
var ErrUnsupportedStreamType = errors.New("Type of stream manager unsupported")

// Database stores all contents nessesary for a keepass database file
type Database struct {
	Signature   *FileSignature
	Headers     *FileHeaders
	Credentials *DBCredentials
	Content     *DBContent
}

// NewDatabase creates a new database with some sensable default settings. To create a database with no settigns per-set, use gokeepasslib.Database{}
func NewDatabase() *Database {
	return &Database{
		Signature:   &DefaultSig,
		Headers:     NewFileHeaders(),
		Credentials: new(DBCredentials),
		Content:     NewDBContent(),
	}
}

func (db *Database) String() string {
	return fmt.Sprintf("Database:\nSignature: %s\n"+
		"Headers: %s\nCredentials: %s\nContent:\n%+v\n",
		db.Signature,
		db.Headers,
		db.Credentials,
		db.Content,
	)
}

// StreamManager returns a ProtectedStreamManager bassed on the db headers, or nil if the type is unsupported
// Can be used to lock only certain entries instead of calling
func (db *Database) StreamManager() ProtectedStreamManager {
	switch db.Headers.InnerRandomStreamID {
	case NoStreamID:
		return new(InsecureStreamManager)
	case SalsaStreamID:
		key := sha256.Sum256(db.Headers.ProtectedStreamKey)
		return NewSalsaManager(key)
	default:
		return nil
	}
}

// UnlockProtectedEntries goes through the entire database and encrypts
// any Values in entries with protected=true set.
// This should be called after decoding if you want to view plaintext password in an entry
// Warning: If you call this when entry values are already unlocked, it will cause them to be unreadable
func (db *Database) UnlockProtectedEntries() error {
	manager := db.StreamManager()
	if manager == nil {
		return ErrUnsupportedStreamType
	}
	UnlockProtectedGroups(manager, db.Content.Root.Groups)
	return nil
}

// LockProtectedEntries goes through the entire database and decrypts
// any Values in entries with protected=true set.
// Warning: Do not call this if entries are already locked
// Warning: Encoding a database calls LockProtectedEntries automatically
func (db *Database) LockProtectedEntries() error {
	manager := db.StreamManager()
	if manager == nil {
		return ErrUnsupportedStreamType
	}
	LockProtectedGroups(manager, db.Content.Root.Groups)
	return nil
}

func (db *Database) Decrypter() (cipher.BlockMode,error) {
	block,err := db.Cipher()
	if err != nil {
		return nil,err
	}
	return cipher.NewCBCDecrypter(block, db.Headers.EncryptionIV),nil
}
func (db *Database) Encrypter() (cipher.BlockMode,error) {
	block,err := db.Cipher()
	if err != nil {
		return nil,err
	}
	//Encrypts block data using AES block with initialization vector from header
	return cipher.NewCBCEncrypter(block, db.Headers.EncryptionIV),nil
}
func (db *Database) Cipher() (cipher.Block,error) {
	masterKey, err := db.Credentials.buildMasterKey(db)
	if err != nil {
		return nil,err
	}

	return aes.NewCipher(masterKey)
}
<commit_msg>Added error checking in encrypter/decrypter to avoid nil pointer panic<commit_after>package gokeepasslib

import (
	"crypto/sha256"
	"errors"
	"fmt"
	"crypto/aes"
	"crypto/cipher"
)

// ErrUnsupportedStreamType is retured if no streamManager can be created
// due to an unsupported InnerRandomStreamID value
var ErrUnsupportedStreamType = errors.New("Type of stream manager unsupported")

var ErrRequiredAttributeMissing string
func (e ErrRequiredAttributeMissing) Error () string {
	return fmt.Sprintf("gokeepasslib: operation can not be performed if database does not have %s",e)
}

// Database stores all contents nessesary for a keepass database file
type Database struct {
	Signature   *FileSignature
	Headers     *FileHeaders
	Credentials *DBCredentials
	Content     *DBContent
}

// NewDatabase creates a new database with some sensable default settings. To create a database with no settigns per-set, use gokeepasslib.Database{}
func NewDatabase() *Database {
	return &Database{
		Signature:   &DefaultSig,
		Headers:     NewFileHeaders(),
		Credentials: new(DBCredentials),
		Content:     NewDBContent(),
	}
}

func (db *Database) String() string {
	return fmt.Sprintf("Database:\nSignature: %s\n"+
		"Headers: %s\nCredentials: %s\nContent:\n%+v\n",
		db.Signature,
		db.Headers,
		db.Credentials,
		db.Content,
	)
}

// StreamManager returns a ProtectedStreamManager bassed on the db headers, or nil if the type is unsupported
// Can be used to lock only certain entries instead of calling
func (db *Database) StreamManager() ProtectedStreamManager {
	switch db.Headers.InnerRandomStreamID {
	case NoStreamID:
		return new(InsecureStreamManager)
	case SalsaStreamID:
		key := sha256.Sum256(db.Headers.ProtectedStreamKey)
		return NewSalsaManager(key)
	default:
		return nil
	}
}

// UnlockProtectedEntries goes through the entire database and encrypts
// any Values in entries with protected=true set.
// This should be called after decoding if you want to view plaintext password in an entry
// Warning: If you call this when entry values are already unlocked, it will cause them to be unreadable
func (db *Database) UnlockProtectedEntries() error {
	manager := db.StreamManager()
	if manager == nil {
		return ErrUnsupportedStreamType
	}
	UnlockProtectedGroups(manager, db.Content.Root.Groups)
	return nil
}

// LockProtectedEntries goes through the entire database and decrypts
// any Values in entries with protected=true set.
// Warning: Do not call this if entries are already locked
// Warning: Encoding a database calls LockProtectedEntries automatically
func (db *Database) LockProtectedEntries() error {
	manager := db.StreamManager()
	if manager == nil {
		return ErrUnsupportedStreamType
	}
	LockProtectedGroups(manager, db.Content.Root.Groups)
	return nil
}

func (db *Database) Decrypter() (cipher.BlockMode,error) {
	block,err := db.Cipher()
	if err != nil {
		return nil,err
	}
	return cipher.NewCBCDecrypter(block, db.Headers.EncryptionIV),nil
}
func (db *Database) Encrypter() (cipher.BlockMode,error) {
	if db.Headers == nil {
		return nil, ErrRequiredAttributeMissing("Headers")
	}
	if db.Headers.EncryptionIV == nil {
		return nil, ErrRequiredAttributeMissing("Headers.EncryptionIV")
	}
	block,err := db.Cipher()
	if err != nil {
		return nil,err
	}
	//Encrypts block data using AES block with initialization vector from header
	return cipher.NewCBCEncrypter(block, db.Headers.EncryptionIV),nil
}
func (db *Database) Cipher() (cipher.Block,error) {
	if db.Credentials == nil {
		return nil, ErrRequiredAttributeMissing("Credentials")
	}
	masterKey, err := db.Credentials.buildMasterKey(db)
	if err != nil {
		return nil,err
	}
	return aes.NewCipher(masterKey)
}
<|endoftext|>


<commit_before>package logmetrics

import (
	"fmt"
	"log"
	"strconv"
	"strings"
	"syseng/go-timemetrics"
	"time"
)

type dataPoint struct {
	name        string
	value       int64
	metric_type string
}

type dataPointTime struct {
	name string
	time int64
}

type tsdPoint struct {
	data             interface{}
	lastPush         time.Time
	lastCrunchedPush time.Time
}

func (lg *LogGroup) extractTags(data []string) []string {
	tags := make([]string, lg.getNbTags())

	i := 0

	//General tags
	for tagname, position := range lg.tags {
		tags[i] = fmt.Sprintf("%s=%s", tagname, data[position])
		i++
	}

	return tags
}

func (lg *LogGroup) getKeys(data []string) ([]dataPoint, time.Time) {
	y := time.Now().Year()

	tags := lg.extractTags(data)

	nbKeys := len(lg.metrics)
	dataPoints := make([]dataPoint, nbKeys)

	//Time
	var t time.Time
	if data[lg.date_position] == lg.last_date_str {
		t = lg.last_date
	} else {
		var err error
		t, err = time.Parse(lg.date_format, data[lg.date_position])
		if err != nil {
			log.Print(err)
			var nt time.Time
			return nil, nt
		}
	}

	//Keep time around to only parse new dates
	lg.last_date_str = data[lg.date_position]
	lg.last_date = t

	//Patch in year if missing - rfc3164
	if t.Year() == 0 {
		t = time.Date(y, t.Month(), t.Day(), t.Hour(), t.Minute(),
			t.Second(), t.Nanosecond(), t.Location())
	}

	//Make a first pass extracting the data, applying float->int conversion on multiplier
	values := make([]int64, lg.expected_matches+1)
	for position, keyType := range lg.metrics {
		if position == 0 {
			values[position] = 1
		} else {
			var val int64
			var err error
			if keyType.format == "float" {
				var val_float float64
				if val_float, err = strconv.ParseFloat(data[position], 64); err == nil {
					val = int64(val_float * float64(keyType.multiply))
				}
			} else {
				if val, err = strconv.ParseInt(data[position], 10, 64); err == nil {
					val = val * int64(keyType.multiply)
				}
			}

			if err != nil {
				log.Printf("Unable to extract data from value match, %s: %s", err, data[position])
				var nt time.Time
				return nil, nt
			} else {
				values[position] = val
			}
		}
	}

	//Second pass applies operation and create datapoints
	var i = 0
	for position, val := range values {
		//Is the value a metric?
		if keyType, ok := lg.metrics[position]; ok {
			//Key name
			key := fmt.Sprintf("%s.%s.%s %s %s", lg.key_prefix, keyType.key_suffix, "%s %d %s", strings.Join(tags, " "), keyType.tag)

			//Do we need to do any operation on this val?
			for op, opvalues := range keyType.operations {
				for _, op_position := range opvalues {
					//log.Printf("%s %d on pos %d, current val: %d", op, op_position, position, val)
					if op_position != 0 {
						switch op {
						case "add":
							val += values[op_position]

						case "sub":
							val -= values[op_position]
						}
					}
				}
			}

			if val < 0 && lg.fail_operation_warn {
				log.Printf("Values cannot be negative after applying operation. Offending line: %s", data[0])
				var nt time.Time
				return nil, nt
			}

			dataPoints[i] = dataPoint{name: key, value: val, metric_type: keyType.metric_type}
			i++
		}
	}

	return dataPoints, t
}

func getCounterKeys(name string, c timemetrics.Counter) []string {
	t := int(c.GetMaxTime().Unix())

	keys := make([]string, 1)
	keys[0] = fmt.Sprintf(name, "count", t, fmt.Sprintf("%d", c.Count()))

	return keys
}

func getMeterKeyCount(name string, m timemetrics.Meter) []string {
	t := int(m.GetMaxTime().Unix())

	keys := make([]string, 1)
	keys[0] = fmt.Sprintf(name, "count", t, fmt.Sprintf("%d", m.Count()))

	return keys
}

func getMeterKeyRates(name string, m timemetrics.Meter) []string {
	t := int(m.GetMaxEWMATime().Unix())

	keys := make([]string, 3)
	keys[0] = fmt.Sprintf(name, "rate._1min", t, fmt.Sprintf("%.4f", m.Rate1()))
	keys[1] = fmt.Sprintf(name, "rate._5min", t, fmt.Sprintf("%.4f", m.Rate5()))
	keys[2] = fmt.Sprintf(name, "rate._15min", t, fmt.Sprintf("%.4f", m.Rate15()))

	return keys
}

func getHistogramKeys(name string, h timemetrics.Histogram) []string {
	t := int(h.GetMaxTime().Unix())
	ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})

	keys := make([]string, 10)

	keys[0] = fmt.Sprintf(name, "min", t, fmt.Sprintf("%d", h.Min()))
	keys[1] = fmt.Sprintf(name, "max", t, fmt.Sprintf("%d", h.Max()))
	keys[2] = fmt.Sprintf(name, "mean", t, fmt.Sprintf("%.4f", h.Mean()))
	keys[3] = fmt.Sprintf(name, "std-dev", t, fmt.Sprintf("%.4f", h.StdDev()))
	keys[4] = fmt.Sprintf(name, "p50", t, fmt.Sprintf("%d", int64(ps[0])))
	keys[5] = fmt.Sprintf(name, "p75", t, fmt.Sprintf("%d", int64(ps[1])))
	keys[6] = fmt.Sprintf(name, "p95", t, fmt.Sprintf("%d", int64(ps[2])))
	keys[7] = fmt.Sprintf(name, "p99", t, fmt.Sprintf("%d", int64(ps[3])))
	keys[8] = fmt.Sprintf(name, "p999", t, fmt.Sprintf("%d", int64(ps[4])))
	keys[9] = fmt.Sprintf(name, "sample_size", t, fmt.Sprintf("%d", h.Sample().Size()))

	return keys
}

func (lg LogGroup) dataPoolHandler(channel_number int, tsd_pushers []chan []string, tsd_channel_number int) error {
	dataPool := make(map[string]*tsdPoint)
	tsd_push := tsd_pushers[tsd_channel_number]

	log.Printf("Datapool[%s:%d] started. Pushing keys to TsdPusher[%d]", lg.name, channel_number, tsd_channel_number)

	//Start the handler
	go func() {

		//Failsafe if anything goes really wrong
		//defer func() {
		//	if r := recover(); r != nil {
		//		log.Printf("Recovered error in %s: %s", lg.name, r)
		//	}
		//}()

		pushStats := make(chan bool)

		var lastTimePushed *time.Time
		lastNbKeys := 0
		for {
			select {
			case data := <-lg.tail_data[channel_number]:
				data_points, point_time := lg.getKeys(data)

				//To start things off
				if lastTimePushed == nil {
					lastTimePushed = &point_time
				}

				for _, data_point := range data_points {
					//New metrics, add
					if _, ok := dataPool[data_point.name]; !ok {
						switch data_point.metric_type {
						case "histogram":
							s := timemetrics.NewExpDecaySample(point_time, lg.histogram_size, lg.histogram_alpha_decay, lg.histogram_rescale_threshold_min)
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewHistogram(s),
								lastPush: point_time}
						case "counter":
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewCounter(point_time),
								lastPush: point_time}
						case "meter":
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewMeter(point_time, lg.interval),
								lastPush: point_time, lastCrunchedPush: point_time}
						default:
							log.Fatalf("Unexpected metric type %s!", data_point.metric_type)
						}
					}

					//Make sure data is ordered or we risk sending duplicate data
					if dataPool[data_point.name].lastPush.Unix() > point_time.Unix() {
						log.Printf("Non-ordered data detected in log file. Its key already had a update at %s in the future. Offending line: %s",
							dataPool[data_point.name].lastPush, data[0])
					}

					switch d := dataPool[data_point.name].data.(type) {
					case timemetrics.Histogram:
						d.Update(point_time, data_point.value)
					case timemetrics.Counter:
						d.Inc(point_time, data_point.value)
					case timemetrics.Meter:
						d.Mark(point_time, data_point.value)
					}
				}

				//Support for log playback - Push when <interval> has pass in the logs, not real time
				if point_time.Sub(*lastTimePushed) > (time.Duration(lg.interval) * time.Second) {
					interval := int(point_time.Sub(*lastTimePushed).Seconds())

					//Update EWMAs
					for _, tsdPoint := range dataPool {
						switch v := tsdPoint.data.(type) {
						case timemetrics.Meter:
							sec_since_last_value := point_time.Sub(v.GetMaxTime()).Unix()
							sec_since_last_ewma_crunch := point_time.Sub(v.GetMaxEWMATime()).Unix()

							if sec_since_last_value > float64(interval) || sec_since_last_ewma_crunch > float64(lg.ewmaInterval) {
								v.CrunchEWMA(point_time, interval)
							}
						}
					}

					lastTimePushed = &point_time

					go func() { pushStats <- true }()
				}

			case <-pushStats:
				nbKeys := 0
				for tsd_key, tsdPoint := range dataPool {
					switch v := tsdPoint.data.(type) {
					case timemetrics.Histogram:
						snap := v.Snapshot()

						if snap.GetMaxTime().Unix() > tsdPoint.lastPush.Unix() { //Only push updated metrics
							tsdPoint.lastPush = snap.GetMaxTime()
							keys := getHistogramKeys(tsd_key, snap)
							tsd_push <- keys
						}

						nbKeys += 10
					case timemetrics.Counter:
						snap := v.Snapshot()
						if snap.GetMaxTime().Unix() > tsdPoint.lastPush.Unix() {
							tsd_push <- getCounterKeys(tsd_key, snap)
						}
						nbKeys += 1
					case timemetrics.Meter:
						snap := v.Snapshot()

						if snap.GetMaxTime().Unix() > tsdPoint.lastPush.Unix() {
							tsdPoint.lastPush = snap.GetMaxTime()
							tsd_push <- getMeterKeyCount(tsd_key, snap)
						}

						if snap.GetMaxEWMATime().Unix() > tsdPoint.lastCrunchedPush.Unix() {
							tsdPoint.lastCrunchedPush = snap.GetMaxEWMATime()
							tsd_push <- getMeterKeyRates(tsd_key, snap)
						}

						nbKeys += 4
					}
				}

				if lastNbKeys != nbKeys {
					log.Printf("Datapool[%s:%d] currently tracking %d keys", lg.name, channel_number, nbKeys)
				}

				lastNbKeys = nbKeys
			}
		}
	}()

	return nil
}

func StartDataPools(config *Config, tsd_pushers []chan []string) {
	//Start a queryHandler by log group
	nb_tsd_push := 0
	for _, lg := range config.logGroups {
		for i := 0; i < lg.goroutines; i++ {
			lg.dataPoolHandler(i, tsd_pushers, nb_tsd_push)
			nb_tsd_push = (nb_tsd_push + 1) % config.GetPusherNumber()
		}
	}
}
<commit_msg>Another fix on EWMA refresh logic.<commit_after>package logmetrics

import (
	"fmt"
	"log"
	"strconv"
	"strings"
	"syseng/go-timemetrics"
	"time"
)

type dataPoint struct {
	name        string
	value       int64
	metric_type string
}

type dataPointTime struct {
	name string
	time int64
}

type tsdPoint struct {
	data             interface{}
	lastPush         time.Time
	lastCrunchedPush time.Time
}

func (lg *LogGroup) extractTags(data []string) []string {
	tags := make([]string, lg.getNbTags())

	i := 0

	//General tags
	for tagname, position := range lg.tags {
		tags[i] = fmt.Sprintf("%s=%s", tagname, data[position])
		i++
	}

	return tags
}

func (lg *LogGroup) getKeys(data []string) ([]dataPoint, time.Time) {
	y := time.Now().Year()

	tags := lg.extractTags(data)

	nbKeys := len(lg.metrics)
	dataPoints := make([]dataPoint, nbKeys)

	//Time
	var t time.Time
	if data[lg.date_position] == lg.last_date_str {
		t = lg.last_date
	} else {
		var err error
		t, err = time.Parse(lg.date_format, data[lg.date_position])
		if err != nil {
			log.Print(err)
			var nt time.Time
			return nil, nt
		}
	}

	//Keep time around to only parse new dates
	lg.last_date_str = data[lg.date_position]
	lg.last_date = t

	//Patch in year if missing - rfc3164
	if t.Year() == 0 {
		t = time.Date(y, t.Month(), t.Day(), t.Hour(), t.Minute(),
			t.Second(), t.Nanosecond(), t.Location())
	}

	//Make a first pass extracting the data, applying float->int conversion on multiplier
	values := make([]int64, lg.expected_matches+1)
	for position, keyType := range lg.metrics {
		if position == 0 {
			values[position] = 1
		} else {
			var val int64
			var err error
			if keyType.format == "float" {
				var val_float float64
				if val_float, err = strconv.ParseFloat(data[position], 64); err == nil {
					val = int64(val_float * float64(keyType.multiply))
				}
			} else {
				if val, err = strconv.ParseInt(data[position], 10, 64); err == nil {
					val = val * int64(keyType.multiply)
				}
			}

			if err != nil {
				log.Printf("Unable to extract data from value match, %s: %s", err, data[position])
				var nt time.Time
				return nil, nt
			} else {
				values[position] = val
			}
		}
	}

	//Second pass applies operation and create datapoints
	var i = 0
	for position, val := range values {
		//Is the value a metric?
		if keyType, ok := lg.metrics[position]; ok {
			//Key name
			key := fmt.Sprintf("%s.%s.%s %s %s", lg.key_prefix, keyType.key_suffix, "%s %d %s", strings.Join(tags, " "), keyType.tag)

			//Do we need to do any operation on this val?
			for op, opvalues := range keyType.operations {
				for _, op_position := range opvalues {
					//log.Printf("%s %d on pos %d, current val: %d", op, op_position, position, val)
					if op_position != 0 {
						switch op {
						case "add":
							val += values[op_position]

						case "sub":
							val -= values[op_position]
						}
					}
				}
			}

			if val < 0 && lg.fail_operation_warn {
				log.Printf("Values cannot be negative after applying operation. Offending line: %s", data[0])
				var nt time.Time
				return nil, nt
			}

			dataPoints[i] = dataPoint{name: key, value: val, metric_type: keyType.metric_type}
			i++
		}
	}

	return dataPoints, t
}

func getCounterKeys(name string, c timemetrics.Counter) []string {
	t := int(c.GetMaxTime().Unix())

	keys := make([]string, 1)
	keys[0] = fmt.Sprintf(name, "count", t, fmt.Sprintf("%d", c.Count()))

	return keys
}

func getMeterKeyCount(name string, m timemetrics.Meter) []string {
	t := int(m.GetMaxTime().Unix())

	keys := make([]string, 1)
	keys[0] = fmt.Sprintf(name, "count", t, fmt.Sprintf("%d", m.Count()))

	return keys
}

func getMeterKeyRates(name string, m timemetrics.Meter) []string {
	t := int(m.GetMaxEWMATime().Unix())

	keys := make([]string, 3)
	keys[0] = fmt.Sprintf(name, "rate._1min", t, fmt.Sprintf("%.4f", m.Rate1()))
	keys[1] = fmt.Sprintf(name, "rate._5min", t, fmt.Sprintf("%.4f", m.Rate5()))
	keys[2] = fmt.Sprintf(name, "rate._15min", t, fmt.Sprintf("%.4f", m.Rate15()))

	return keys
}

func getHistogramKeys(name string, h timemetrics.Histogram) []string {
	t := int(h.GetMaxTime().Unix())
	ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})

	keys := make([]string, 10)

	keys[0] = fmt.Sprintf(name, "min", t, fmt.Sprintf("%d", h.Min()))
	keys[1] = fmt.Sprintf(name, "max", t, fmt.Sprintf("%d", h.Max()))
	keys[2] = fmt.Sprintf(name, "mean", t, fmt.Sprintf("%.4f", h.Mean()))
	keys[3] = fmt.Sprintf(name, "std-dev", t, fmt.Sprintf("%.4f", h.StdDev()))
	keys[4] = fmt.Sprintf(name, "p50", t, fmt.Sprintf("%d", int64(ps[0])))
	keys[5] = fmt.Sprintf(name, "p75", t, fmt.Sprintf("%d", int64(ps[1])))
	keys[6] = fmt.Sprintf(name, "p95", t, fmt.Sprintf("%d", int64(ps[2])))
	keys[7] = fmt.Sprintf(name, "p99", t, fmt.Sprintf("%d", int64(ps[3])))
	keys[8] = fmt.Sprintf(name, "p999", t, fmt.Sprintf("%d", int64(ps[4])))
	keys[9] = fmt.Sprintf(name, "sample_size", t, fmt.Sprintf("%d", h.Sample().Size()))

	return keys
}

func (lg LogGroup) dataPoolHandler(channel_number int, tsd_pushers []chan []string, tsd_channel_number int) error {
	dataPool := make(map[string]*tsdPoint)
	tsd_push := tsd_pushers[tsd_channel_number]

	log.Printf("Datapool[%s:%d] started. Pushing keys to TsdPusher[%d]", lg.name, channel_number, tsd_channel_number)

	//Start the handler
	go func() {

		//Failsafe if anything goes really wrong
		//defer func() {
		//	if r := recover(); r != nil {
		//		log.Printf("Recovered error in %s: %s", lg.name, r)
		//	}
		//}()

		pushStats := make(chan bool)

		var lastTimePushed *time.Time
		lastNbKeys := 0
		for {
			select {
			case data := <-lg.tail_data[channel_number]:
				data_points, point_time := lg.getKeys(data)

				//To start things off
				if lastTimePushed == nil {
					lastTimePushed = &point_time
				}

				for _, data_point := range data_points {
					//New metrics, add
					if _, ok := dataPool[data_point.name]; !ok {
						switch data_point.metric_type {
						case "histogram":
							s := timemetrics.NewExpDecaySample(point_time, lg.histogram_size, lg.histogram_alpha_decay, lg.histogram_rescale_threshold_min)
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewHistogram(s),
								lastPush: point_time}
						case "counter":
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewCounter(point_time),
								lastPush: point_time}
						case "meter":
							dataPool[data_point.name] = &tsdPoint{data: timemetrics.NewMeter(point_time, lg.interval),
								lastPush: point_time, lastCrunchedPush: point_time}
						default:
							log.Fatalf("Unexpected metric type %s!", data_point.metric_type)
						}
					}

					//Make sure data is ordered or we risk sending duplicate data
					if dataPool[data_point.name].lastPush.Unix() > point_time.Unix() {
						log.Printf("Non-ordered data detected in log file. Its key already had a update at %s in the future. Offending line: %s",
							dataPool[data_point.name].lastPush, data[0])
					}

					switch d := dataPool[data_point.name].data.(type) {
					case timemetrics.Histogram:
						d.Update(point_time, data_point.value)
					case timemetrics.Counter:
						d.Inc(point_time, data_point.value)
					case timemetrics.Meter:
						d.Mark(point_time, data_point.value)
					}
				}

				//Support for log playback - Push when <interval> has pass in the logs, not real time
				if point_time.Sub(*lastTimePushed) > (time.Duration(lg.interval) * time.Second) {
					interval := int(point_time.Unix() - (*lastTimePushed).Unix())

					//Update EWMAs
					for _, tsdPoint := range dataPool {
						switch v := tsdPoint.data.(type) {
						case timemetrics.Meter:
							sec_since_last_value := int(point_time.Unix() - v.GetMaxTime().Unix())
							sec_since_last_ewma_crunch := int(point_time.Unix() - v.GetMaxEWMATime().Unix())

							if sec_since_last_value > interval || sec_since_last_ewma_crunch > lg.ewmaInterval {
								v.CrunchEWMA(point_time, interval)
							}
						}
					}

					lastTimePushed = &point_time

					go func() { pushStats <- true }()
				}

			case <-pushStats:
				nbKeys := 0
				for tsd_key, tsdPoint := range dataPool {
					switch v := tsdPoint.data.(type) {
					case timemetrics.Histogram:
						snap := v.Snapshot()

						if snap.GetMaxTime().Unix() > tsdPoint.lastPush.Unix() { //Only push updated metrics
							tsdPoint.lastPush = snap.GetMaxTime()
							keys := getHistogramKeys(tsd_key, snap)
							tsd_push <- keys
						}

						nbKeys += 10
					case timemetrics.Counter:
						snap := v.Snapshot()
						if snap.GetMaxTime().Unix() > tsdPoint.lastPush.Unix() {
							tsd_push <- getCounterKeys(tsd_key, snap)
						}
						nbKeys += 1
					case timemetrics.Meter:
						snap := v.Snapshot()

						if snap.GetMaxTime().Unix() > tsdPoint.lastPush.Unix() {
							tsdPoint.lastPush = snap.GetMaxTime()
							tsd_push <- getMeterKeyCount(tsd_key, snap)
						}

						if snap.GetMaxEWMATime().Unix() > tsdPoint.lastCrunchedPush.Unix() {
							tsdPoint.lastCrunchedPush = snap.GetMaxEWMATime()
							tsd_push <- getMeterKeyRates(tsd_key, snap)
						}

						nbKeys += 4
					}
				}

				if lastNbKeys != nbKeys {
					log.Printf("Datapool[%s:%d] currently tracking %d keys", lg.name, channel_number, nbKeys)
				}

				lastNbKeys = nbKeys
			}
		}
	}()

	return nil
}

func StartDataPools(config *Config, tsd_pushers []chan []string) {
	//Start a queryHandler by log group
	nb_tsd_push := 0
	for _, lg := range config.logGroups {
		for i := 0; i < lg.goroutines; i++ {
			lg.dataPoolHandler(i, tsd_pushers, nb_tsd_push)
			nb_tsd_push = (nb_tsd_push + 1) % config.GetPusherNumber()
		}
	}
}
<|endoftext|>