<commit_before>// Copyright (c) 2018 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controller

import (
	"fmt"

	"github.com/go-kit/kit/log/level"
	habv1beta1 "github.com/kinvolk/habitat-operator/pkg/apis/habitat/v1beta1"
	appsv1beta1 "k8s.io/api/apps/v1beta1"
	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/tools/cache"
)

const persistentVolumeName = "persistent"

func (hc *HabitatController) newStatefulSet(h *habv1beta1.Habitat) (*appsv1beta1.StatefulSet, error) {
	// This value needs to be passed as a *int32, so we convert it, assign it to a
	// variable and afterwards pass a pointer to it.
	count := int32(h.Spec.Count)

	// Set the service arguments we send to Habitat.
	var habArgs []string
	if h.Spec.Service.Group != "" {
		// When a service is started without explicitly naming the group,
		// it's assigned to the default group.
		habArgs = append(habArgs,
			"--group", h.Spec.Service.Group)
	}

	// As we want to label our pods with the
	// topology type we set standalone as the default one.
	// We do not need to pass this to habitat, as if no topology
	// is set, habitat by default sets standalone topology.
	topology := habv1beta1.TopologyStandalone

	if h.Spec.Service.Topology == habv1beta1.TopologyLeader {
		topology = habv1beta1.TopologyLeader
	}

	path := fmt.Sprintf("%s/%s", configMapDir, peerFilename)

	habArgs = append(habArgs,
		"--topology", topology.String(),
		"--peer-watch-file", path,
	)

	// Runtime binding.
	// One Service connects to another forming a producer/consumer relationship.
	for _, bind := range h.Spec.Service.Bind {
		// Pass --bind flag.
		bindArg := fmt.Sprintf("%s:%s.%s", bind.Name, bind.Service, bind.Group)
		habArgs = append(habArgs,
			"--bind", bindArg)
	}

	base := &appsv1beta1.StatefulSet{
		ObjectMeta: metav1.ObjectMeta{
			Name: h.Name,
		},
		Spec: appsv1beta1.StatefulSetSpec{
			Selector: &metav1.LabelSelector{
				MatchLabels: map[string]string{
					habv1beta1.HabitatNameLabel: h.Name,
				},
			},
			Replicas: &count,
			// Stateless Pods are allowed to be started/terminated in parallel
			PodManagementPolicy: appsv1beta1.ParallelPodManagement,
			Template: apiv1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						habv1beta1.HabitatLabel:     "true",
						habv1beta1.HabitatNameLabel: h.Name,
						habv1beta1.TopologyLabel:    topology.String(),
					},
				},
				Spec: apiv1.PodSpec{
					Containers: []apiv1.Container{
						{
							Name:  "habitat-service",
							Image: h.Spec.Image,
							Args:  habArgs,
							VolumeMounts: []apiv1.VolumeMount{
								{
									Name:      "config",
									MountPath: configMapDir,
									ReadOnly:  true,
								},
							},
							Env: h.Spec.Env,
						},
					},
					// Define the volume for the ConfigMap.
					Volumes: []apiv1.Volume{
						{
							Name: "config",
							VolumeSource: apiv1.VolumeSource{
								ConfigMap: &apiv1.ConfigMapVolumeSource{
									LocalObjectReference: apiv1.LocalObjectReference{
										Name: configMapName,
									},
									Items: []apiv1.KeyToPath{
										{
											Key:  peerFile,
											Path: peerFilename,
										},
									},
								},
							},
						},
					},
				},
			},
		},
	}

	// If we have a secret name present we should mount that secret.
	if h.Spec.Service.ConfigSecretName != "" {
		// Let's make sure our secret is there before mounting it.
		secret, err := hc.config.KubernetesClientset.CoreV1().Secrets(h.Namespace).Get(h.Spec.Service.ConfigSecretName, metav1.GetOptions{})
		if err != nil {
			return nil, err
		}

		secretVolume := &apiv1.Volume{
			Name: userConfigFilename,
			VolumeSource: apiv1.VolumeSource{
				Secret: &apiv1.SecretVolumeSource{
					SecretName: secret.Name,
					Items: []apiv1.KeyToPath{
						{
							Key:  userTOMLFile,
							Path: userTOMLFile,
						},
					},
				},
			},
		}

		secretVolumeMount := &apiv1.VolumeMount{
			Name: userConfigFilename,
			// The Habitat supervisor creates a directory for each service under /hab/svc/<servicename>.
			// We need to place the user.toml file in there in order for it to be detected.
			MountPath: fmt.Sprintf("/hab/user/%s/config", h.Spec.Service.Name),
			ReadOnly:  false,
		}

		base.Spec.Template.Spec.Containers[0].VolumeMounts = append(base.Spec.Template.Spec.Containers[0].VolumeMounts, *secretVolumeMount)
		base.Spec.Template.Spec.Volumes = append(base.Spec.Template.Spec.Volumes, *secretVolume)
	}

	// Mount Persistent Volume, if requested.
	if ps := h.Spec.PersistentStorage; ps != nil {
		vm := &apiv1.VolumeMount{
			Name:      persistentVolumeName,
			MountPath: ps.MountPath,
		}

		base.Spec.Template.Spec.Containers[0].VolumeMounts = append(base.Spec.Template.Spec.Containers[0].VolumeMounts, *vm)

		q, err := resource.ParseQuantity(ps.Size)
		if err != nil {
			return nil, err
		}

		base.Spec.VolumeClaimTemplates = []apiv1.PersistentVolumeClaim{
			apiv1.PersistentVolumeClaim{
				ObjectMeta: metav1.ObjectMeta{
					Name:      persistentVolumeName,
					Namespace: h.Namespace,
					Labels: map[string]string{
						habv1beta1.HabitatLabel:     "true",
						habv1beta1.HabitatNameLabel: h.Name,
					},
				},
				Spec: apiv1.PersistentVolumeClaimSpec{
					AccessModes: []apiv1.PersistentVolumeAccessMode{
						apiv1.ReadWriteOnce,
					},
					StorageClassName: &ps.StorageClassName,
					Resources: apiv1.ResourceRequirements{
						Requests: apiv1.ResourceList{
							apiv1.ResourceStorage: q,
						},
					},
				},
			},
		}
	}

	// Handle ring key, if one is specified.
	if ringSecretName := h.Spec.Service.RingSecretName; ringSecretName != "" {
		s, err := hc.config.KubernetesClientset.CoreV1().Secrets(apiv1.NamespaceDefault).Get(ringSecretName, metav1.GetOptions{})
		if err != nil {
			level.Error(hc.logger).Log("msg", "Could not find Secret containing ring key")
			return nil, err
		}

		// The filename under which the ring key is saved.
		ringKeyFile := fmt.Sprintf("%s.%s", ringSecretName, ringKeyFileExt)

		// Extract the bare ring name, by removing the revision.
		// Validation has already been performed by this point.
		ringName := ringRegexp.FindStringSubmatch(ringSecretName)[1]

		v := &apiv1.Volume{
			Name: ringSecretName,
			VolumeSource: apiv1.VolumeSource{
				Secret: &apiv1.SecretVolumeSource{
					SecretName: s.Name,
					Items: []apiv1.KeyToPath{
						{
							Key:  ringSecretKey,
							Path: ringKeyFile,
						},
					},
				},
			},
		}

		vm := &apiv1.VolumeMount{
			Name:      ringSecretName,
			MountPath: "/hab/cache/keys",
			// This directory cannot be made read-only, as the supervisor writes to
			// it during its operation.
			ReadOnly: false,
		}

		// Mount ring key file.
		base.Spec.Template.Spec.Volumes = append(base.Spec.Template.Spec.Volumes, *v)
		base.Spec.Template.Spec.Containers[0].VolumeMounts = append(base.Spec.Template.Spec.Containers[0].VolumeMounts, *vm)

		// Add --ring argument to supervisor invocation.
		base.Spec.Template.Spec.Containers[0].Args = append(base.Spec.Template.Spec.Containers[0].Args, "--ring", ringName)
	}

	return base, nil
}

func (hc *HabitatController) cacheStatefulSets() {
	source := newListWatchFromClientWithLabels(
		hc.config.KubernetesClientset.AppsV1beta1().RESTClient(),
		"statefulsets",
		apiv1.NamespaceAll,
		labelListOptions())

	hc.stsInformer = cache.NewSharedIndexInformer(
		source,
		&appsv1beta1.StatefulSet{},
		resyncPeriod,
		cache.Indexers{},
	)

	hc.stsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    hc.handleStsAdd,
		UpdateFunc: hc.handleStsUpdate,
		DeleteFunc: hc.handleStsDelete,
	})

	hc.stsInformerSynced = hc.stsInformer.HasSynced
}

func (hc *HabitatController) handleStsAdd(obj interface{}) {
	d, ok := obj.(*appsv1beta1.StatefulSet)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert StatefulSet", "obj", obj)
		return
	}

	h, err := hc.getHabitatFromLabeledResource(d)
	if err != nil {
		level.Error(hc.logger).Log("msg", "Could not find Habitat for StatefulSet", "name", d.Name)
		return
	}

	hc.enqueue(h)
}

func (hc *HabitatController) handleStsUpdate(oldObj, newObj interface{}) {
	d, ok := newObj.(*appsv1beta1.StatefulSet)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert StatefulSet", "obj", newObj)
		return
	}

	h, err := hc.getHabitatFromLabeledResource(d)
	if err != nil {
		level.Error(hc.logger).Log("msg", "Could not find Habitat for StatefulSet", "name", d.Name)
		return
	}

	hc.enqueue(h)
}

func (hc *HabitatController) handleStsDelete(obj interface{}) {
	d, ok := obj.(*appsv1beta1.StatefulSet)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert StatefulSet", "obj", obj)
		return
	}

	h, err := hc.getHabitatFromLabeledResource(d)
	if err != nil {
		// Could not find Habitat, it must have already been removed.
		level.Debug(hc.logger).Log("msg", "Could not find Habitat for StatefulSet", "name", d.Name)
		return
	}

	hc.enqueue(h)
}
<commit_msg>Add context to error<commit_after>// Copyright (c) 2018 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controller

import (
	"fmt"

	"github.com/go-kit/kit/log/level"
	habv1beta1 "github.com/kinvolk/habitat-operator/pkg/apis/habitat/v1beta1"
	appsv1beta1 "k8s.io/api/apps/v1beta1"
	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/tools/cache"
)

const persistentVolumeName = "persistent"

func (hc *HabitatController) newStatefulSet(h *habv1beta1.Habitat) (*appsv1beta1.StatefulSet, error) {
	// This value needs to be passed as a *int32, so we convert it, assign it to a
	// variable and afterwards pass a pointer to it.
	count := int32(h.Spec.Count)

	// Set the service arguments we send to Habitat.
	var habArgs []string
	if h.Spec.Service.Group != "" {
		// When a service is started without explicitly naming the group,
		// it's assigned to the default group.
		habArgs = append(habArgs,
			"--group", h.Spec.Service.Group)
	}

	// As we want to label our pods with the
	// topology type we set standalone as the default one.
	// We do not need to pass this to habitat, as if no topology
	// is set, habitat by default sets standalone topology.
	topology := habv1beta1.TopologyStandalone

	if h.Spec.Service.Topology == habv1beta1.TopologyLeader {
		topology = habv1beta1.TopologyLeader
	}

	path := fmt.Sprintf("%s/%s", configMapDir, peerFilename)

	habArgs = append(habArgs,
		"--topology", topology.String(),
		"--peer-watch-file", path,
	)

	// Runtime binding.
	// One Service connects to another forming a producer/consumer relationship.
	for _, bind := range h.Spec.Service.Bind {
		// Pass --bind flag.
		bindArg := fmt.Sprintf("%s:%s.%s", bind.Name, bind.Service, bind.Group)
		habArgs = append(habArgs,
			"--bind", bindArg)
	}

	base := &appsv1beta1.StatefulSet{
		ObjectMeta: metav1.ObjectMeta{
			Name: h.Name,
		},
		Spec: appsv1beta1.StatefulSetSpec{
			Selector: &metav1.LabelSelector{
				MatchLabels: map[string]string{
					habv1beta1.HabitatNameLabel: h.Name,
				},
			},
			Replicas: &count,
			// Stateless Pods are allowed to be started/terminated in parallel
			PodManagementPolicy: appsv1beta1.ParallelPodManagement,
			Template: apiv1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						habv1beta1.HabitatLabel:     "true",
						habv1beta1.HabitatNameLabel: h.Name,
						habv1beta1.TopologyLabel:    topology.String(),
					},
				},
				Spec: apiv1.PodSpec{
					Containers: []apiv1.Container{
						{
							Name:  "habitat-service",
							Image: h.Spec.Image,
							Args:  habArgs,
							VolumeMounts: []apiv1.VolumeMount{
								{
									Name:      "config",
									MountPath: configMapDir,
									ReadOnly:  true,
								},
							},
							Env: h.Spec.Env,
						},
					},
					// Define the volume for the ConfigMap.
					Volumes: []apiv1.Volume{
						{
							Name: "config",
							VolumeSource: apiv1.VolumeSource{
								ConfigMap: &apiv1.ConfigMapVolumeSource{
									LocalObjectReference: apiv1.LocalObjectReference{
										Name: configMapName,
									},
									Items: []apiv1.KeyToPath{
										{
											Key:  peerFile,
											Path: peerFilename,
										},
									},
								},
							},
						},
					},
				},
			},
		},
	}

	// If we have a secret name present we should mount that secret.
	if h.Spec.Service.ConfigSecretName != "" {
		// Let's make sure our secret is there before mounting it.
		secret, err := hc.config.KubernetesClientset.CoreV1().Secrets(h.Namespace).Get(h.Spec.Service.ConfigSecretName, metav1.GetOptions{})
		if err != nil {
			return nil, err
		}

		secretVolume := &apiv1.Volume{
			Name: userConfigFilename,
			VolumeSource: apiv1.VolumeSource{
				Secret: &apiv1.SecretVolumeSource{
					SecretName: secret.Name,
					Items: []apiv1.KeyToPath{
						{
							Key:  userTOMLFile,
							Path: userTOMLFile,
						},
					},
				},
			},
		}

		secretVolumeMount := &apiv1.VolumeMount{
			Name: userConfigFilename,
			// The Habitat supervisor creates a directory for each service under /hab/svc/<servicename>.
			// We need to place the user.toml file in there in order for it to be detected.
			MountPath: fmt.Sprintf("/hab/user/%s/config", h.Spec.Service.Name),
			ReadOnly:  false,
		}

		base.Spec.Template.Spec.Containers[0].VolumeMounts = append(base.Spec.Template.Spec.Containers[0].VolumeMounts, *secretVolumeMount)
		base.Spec.Template.Spec.Volumes = append(base.Spec.Template.Spec.Volumes, *secretVolume)
	}

	// Mount Persistent Volume, if requested.
	if ps := h.Spec.PersistentStorage; ps != nil {
		vm := &apiv1.VolumeMount{
			Name:      persistentVolumeName,
			MountPath: ps.MountPath,
		}

		base.Spec.Template.Spec.Containers[0].VolumeMounts = append(base.Spec.Template.Spec.Containers[0].VolumeMounts, *vm)

		q, err := resource.ParseQuantity(ps.Size)
		if err != nil {
			return nil, fmt.Errorf("Could not parse PersistentStorage.Size: %v", err)
		}

		base.Spec.VolumeClaimTemplates = []apiv1.PersistentVolumeClaim{
			apiv1.PersistentVolumeClaim{
				ObjectMeta: metav1.ObjectMeta{
					Name:      persistentVolumeName,
					Namespace: h.Namespace,
					Labels: map[string]string{
						habv1beta1.HabitatLabel:     "true",
						habv1beta1.HabitatNameLabel: h.Name,
					},
				},
				Spec: apiv1.PersistentVolumeClaimSpec{
					AccessModes: []apiv1.PersistentVolumeAccessMode{
						apiv1.ReadWriteOnce,
					},
					StorageClassName: &ps.StorageClassName,
					Resources: apiv1.ResourceRequirements{
						Requests: apiv1.ResourceList{
							apiv1.ResourceStorage: q,
						},
					},
				},
			},
		}
	}

	// Handle ring key, if one is specified.
	if ringSecretName := h.Spec.Service.RingSecretName; ringSecretName != "" {
		s, err := hc.config.KubernetesClientset.CoreV1().Secrets(apiv1.NamespaceDefault).Get(ringSecretName, metav1.GetOptions{})
		if err != nil {
			level.Error(hc.logger).Log("msg", "Could not find Secret containing ring key")
			return nil, err
		}

		// The filename under which the ring key is saved.
		ringKeyFile := fmt.Sprintf("%s.%s", ringSecretName, ringKeyFileExt)

		// Extract the bare ring name, by removing the revision.
		// Validation has already been performed by this point.
		ringName := ringRegexp.FindStringSubmatch(ringSecretName)[1]

		v := &apiv1.Volume{
			Name: ringSecretName,
			VolumeSource: apiv1.VolumeSource{
				Secret: &apiv1.SecretVolumeSource{
					SecretName: s.Name,
					Items: []apiv1.KeyToPath{
						{
							Key:  ringSecretKey,
							Path: ringKeyFile,
						},
					},
				},
			},
		}

		vm := &apiv1.VolumeMount{
			Name:      ringSecretName,
			MountPath: "/hab/cache/keys",
			// This directory cannot be made read-only, as the supervisor writes to
			// it during its operation.
			ReadOnly: false,
		}

		// Mount ring key file.
		base.Spec.Template.Spec.Volumes = append(base.Spec.Template.Spec.Volumes, *v)
		base.Spec.Template.Spec.Containers[0].VolumeMounts = append(base.Spec.Template.Spec.Containers[0].VolumeMounts, *vm)

		// Add --ring argument to supervisor invocation.
		base.Spec.Template.Spec.Containers[0].Args = append(base.Spec.Template.Spec.Containers[0].Args, "--ring", ringName)
	}

	return base, nil
}

func (hc *HabitatController) cacheStatefulSets() {
	source := newListWatchFromClientWithLabels(
		hc.config.KubernetesClientset.AppsV1beta1().RESTClient(),
		"statefulsets",
		apiv1.NamespaceAll,
		labelListOptions())

	hc.stsInformer = cache.NewSharedIndexInformer(
		source,
		&appsv1beta1.StatefulSet{},
		resyncPeriod,
		cache.Indexers{},
	)

	hc.stsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    hc.handleStsAdd,
		UpdateFunc: hc.handleStsUpdate,
		DeleteFunc: hc.handleStsDelete,
	})

	hc.stsInformerSynced = hc.stsInformer.HasSynced
}

func (hc *HabitatController) handleStsAdd(obj interface{}) {
	d, ok := obj.(*appsv1beta1.StatefulSet)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert StatefulSet", "obj", obj)
		return
	}

	h, err := hc.getHabitatFromLabeledResource(d)
	if err != nil {
		level.Error(hc.logger).Log("msg", "Could not find Habitat for StatefulSet", "name", d.Name)
		return
	}

	hc.enqueue(h)
}

func (hc *HabitatController) handleStsUpdate(oldObj, newObj interface{}) {
	d, ok := newObj.(*appsv1beta1.StatefulSet)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert StatefulSet", "obj", newObj)
		return
	}

	h, err := hc.getHabitatFromLabeledResource(d)
	if err != nil {
		level.Error(hc.logger).Log("msg", "Could not find Habitat for StatefulSet", "name", d.Name)
		return
	}

	hc.enqueue(h)
}

func (hc *HabitatController) handleStsDelete(obj interface{}) {
	d, ok := obj.(*appsv1beta1.StatefulSet)
	if !ok {
		level.Error(hc.logger).Log("msg", "Failed to type assert StatefulSet", "obj", obj)
		return
	}

	h, err := hc.getHabitatFromLabeledResource(d)
	if err != nil {
		// Could not find Habitat, it must have already been removed.
		level.Debug(hc.logger).Log("msg", "Could not find Habitat for StatefulSet", "name", d.Name)
		return
	}

	hc.enqueue(h)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1beta2

import (
	"fmt"

	habv1beta2 "github.com/habitat-sh/habitat-operator/pkg/apis/habitat/v1beta2"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/tools/cache"
)

const leaderFollowerTopologyMinCount = 3

type keyNotFoundError struct {
	key string
}

func (err keyNotFoundError) Error() string {
	return fmt.Sprintf("could not find Object with key %s in the cache", err.key)
}

func validateCustomObject(h habv1beta2.Habitat) error {
	spec := h.Spec

	switch spec.Service.Topology {
	case habv1beta2.TopologyStandalone:
	case habv1beta2.TopologyLeader:
		if spec.Count < leaderFollowerTopologyMinCount {
			return fmt.Errorf("too few instances: %d, leader-follower topology requires at least %d", spec.Count, leaderFollowerTopologyMinCount)
		}
	default:
		return fmt.Errorf("unknown topology: %s", spec.Service.Topology)
	}

	if rsn := spec.Service.RingSecretName; rsn != "" {
		ringParts := ringRegexp.FindStringSubmatch(rsn)

		// The ringParts slice should have a second element for the capturing group
		// in the ringRegexp regular expression, containing the ring's name.
		if len(ringParts) < 2 {
			return fmt.Errorf("malformed ring secret name: %s", rsn)
		}
	}

	return nil
}

// newListWatchFromClientWithLabels is a modified newListWatchFromClient function from listWatch.
// Instead of using fields to filter, we modify the function to use labels.
func newListWatchFromClientWithLabels(c cache.Getter, resource string, namespace string, op metav1.ListOptions) *cache.ListWatch {
	listFunc := func(_ metav1.ListOptions) (runtime.Object, error) {
		return c.Get().
			Namespace(namespace).
			Resource(resource).
			VersionedParams(&op, metav1.ParameterCodec).
			Do().
			Get()
	}
	watchFunc := func(_ metav1.ListOptions) (watch.Interface, error) {
		op.Watch = true
		return c.Get().
			Namespace(namespace).
			Resource(resource).
			VersionedParams(&op, metav1.ParameterCodec).
			Watch()
	}
	return &cache.ListWatch{ListFunc: listFunc, WatchFunc: watchFunc}
}

func labelListOptions() metav1.ListOptions {
	ls := labels.SelectorFromSet(labels.Set(map[string]string{
		habv1beta2.HabitatLabel: "true",
	}))

	return metav1.ListOptions{
		LabelSelector: ls.String(),
	}
}
<commit_msg>Add CreateCRD function to v1beta2 controller<commit_after>// Copyright (c) 2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1beta2

import (
	"fmt"
	"reflect"
	"time"

	"github.com/habitat-sh/habitat-operator/pkg/apis/habitat"
	habv1beta2 "github.com/habitat-sh/habitat-operator/pkg/apis/habitat/v1beta2"

	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/tools/cache"
)

const (
	leaderFollowerTopologyMinCount = 3
	pollInterval                   = 500 * time.Millisecond
	timeOut                        = 10 * time.Second
	habitatCRDName                 = habv1beta2.HabitatResourcePlural + "." + habitat.GroupName
)

type keyNotFoundError struct {
	key string
}

func (err keyNotFoundError) Error() string {
	return fmt.Sprintf("could not find Object with key %s in the cache", err.key)
}

func validateCustomObject(h habv1beta2.Habitat) error {
	spec := h.Spec

	switch spec.Service.Topology {
	case habv1beta2.TopologyStandalone:
	case habv1beta2.TopologyLeader:
		if spec.Count < leaderFollowerTopologyMinCount {
			return fmt.Errorf("too few instances: %d, leader-follower topology requires at least %d", spec.Count, leaderFollowerTopologyMinCount)
		}
	default:
		return fmt.Errorf("unknown topology: %s", spec.Service.Topology)
	}

	if rsn := spec.Service.RingSecretName; rsn != "" {
		ringParts := ringRegexp.FindStringSubmatch(rsn)

		// The ringParts slice should have a second element for the capturing group
		// in the ringRegexp regular expression, containing the ring's name.
		if len(ringParts) < 2 {
			return fmt.Errorf("malformed ring secret name: %s", rsn)
		}
	}

	return nil
}

// newListWatchFromClientWithLabels is a modified newListWatchFromClient function from listWatch.
// Instead of using fields to filter, we modify the function to use labels.
func newListWatchFromClientWithLabels(c cache.Getter, resource string, namespace string, op metav1.ListOptions) *cache.ListWatch {
	listFunc := func(_ metav1.ListOptions) (runtime.Object, error) {
		return c.Get().
			Namespace(namespace).
			Resource(resource).
			VersionedParams(&op, metav1.ParameterCodec).
			Do().
			Get()
	}
	watchFunc := func(_ metav1.ListOptions) (watch.Interface, error) {
		op.Watch = true
		return c.Get().
			Namespace(namespace).
			Resource(resource).
			VersionedParams(&op, metav1.ParameterCodec).
			Watch()
	}
	return &cache.ListWatch{ListFunc: listFunc, WatchFunc: watchFunc}
}

func labelListOptions() metav1.ListOptions {
	ls := labels.SelectorFromSet(labels.Set(map[string]string{
		habv1beta2.HabitatLabel: "true",
	}))

	return metav1.ListOptions{
		LabelSelector: ls.String(),
	}
}

func CreateCRD(clientset apiextensionsclient.Interface) (*apiextensionsv1beta1.CustomResourceDefinition, error) {
	crd := &apiextensionsv1beta1.CustomResourceDefinition{
		ObjectMeta: metav1.ObjectMeta{
			Name: habv1beta2.SchemeGroupVersion.String(),
		},
		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
			Group:   habv1beta2.SchemeGroupVersion.Group,
			Version: habv1beta2.SchemeGroupVersion.Version,
			Scope:   apiextensionsv1beta1.NamespaceScoped,
			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
				Plural:     habv1beta2.HabitatResourcePlural,
				Kind:       reflect.TypeOf(habv1beta2.Habitat{}).Name(),
				ShortNames: []string{habv1beta2.HabitatShortName},
			},
		},
	}

	_, err := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
	if err != nil {
		return nil, err
	}

	// wait for CRD being established.
	err = wait.Poll(pollInterval, timeOut, func() (bool, error) {
		crd, err = clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Get(habitatCRDName, metav1.GetOptions{})

		if err != nil {
			return false, err
		}

		for _, cond := range crd.Status.Conditions {
			switch cond.Type {
			case apiextensionsv1beta1.Established:
				if cond.Status == apiextensionsv1beta1.ConditionTrue {
					return true, err
				}
			case apiextensionsv1beta1.NamesAccepted:
				if cond.Status == apiextensionsv1beta1.ConditionFalse {
					// TODO re-introduce logging?
					// fmt.Printf("Error: Name conflict: %v\n", cond.Reason)
				}
			}
		}

		return false, err
	})

	// delete CRD if there was an error.
	if err != nil {
		deleteErr := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(habitatCRDName, nil)
		if deleteErr != nil {
			return nil, errors.NewAggregate([]error{err, deleteErr})
		}

		return nil, err
	}

	return crd, nil
}
<|endoftext|>


<commit_before>// +build !windows,!solaris

package listeners

import (
	"crypto/tls"
	"fmt"
	"net"
	"strconv"

	"github.com/coreos/go-systemd/activation"
	"github.com/docker/go-connections/sockets"
)

// Init creates new listeners for the server.
// TODO: Clean up the fact that socketGroup and tlsConfig aren't always used.
func Init(proto, addr, socketGroup string, tlsConfig *tls.Config) ([]net.Listener, error) {
	ls := []net.Listener{}

	switch proto {
	case "fd":
		fds, err := listenFD(addr, tlsConfig)
		if err != nil {
			return nil, err
		}
		ls = append(ls, fds...)
	case "tcp":
		l, err := sockets.NewTCPSocket(addr, tlsConfig)
		if err != nil {
			return nil, err
		}
		ls = append(ls, l)
	case "unix":
		l, err := sockets.NewUnixSocket(addr, socketGroup)
		if err != nil {
			return nil, fmt.Errorf("can't create unix socket %s: %v", addr, err)
		}
		ls = append(ls, l)
	default:
		return nil, fmt.Errorf("invalid protocol format: %q", proto)
	}

	return ls, nil
}

// listenFD returns the specified socket activated files as a slice of
// net.Listeners or all of the activated files if "*" is given.
func listenFD(addr string, tlsConfig *tls.Config) ([]net.Listener, error) {
	var (
		err       error
		listeners []net.Listener
	)
	// socket activation
	if tlsConfig != nil {
		listeners, err = activation.TLSListeners(false, tlsConfig)
	} else {
		listeners, err = activation.Listeners(false)
	}
	if err != nil {
		return nil, err
	}

	if len(listeners) == 0 {
		return nil, fmt.Errorf("no sockets found via socket activation: make sure the service was started by systemd")
	}

	// default to all fds just like unix:// and tcp://
	if addr == "" || addr == "*" {
		return listeners, nil
	}

	fdNum, err := strconv.Atoi(addr)
	if err != nil {
		return nil, fmt.Errorf("failed to parse systemd fd address: should be a number: %v", addr)
	}
	fdOffset := fdNum - 3
	if len(listeners) < int(fdOffset)+1 {
		return nil, fmt.Errorf("too few socket activated files passed in by systemd")
	}
	if listeners[fdOffset] == nil {
		return nil, fmt.Errorf("failed to listen on systemd activated file: fd %d", fdOffset+3)
	}
	for i, ls := range listeners {
		if i == fdOffset || ls == nil {
			continue
		}
		if err := ls.Close(); err != nil {
			return nil, fmt.Errorf("failed to close systemd activated file: fd %d: %v", fdOffset+3, err)
		}
	}
	return []net.Listener{listeners[fdOffset]}, nil
}
<commit_msg>Convert socket group to int<commit_after>// +build !windows,!solaris

package listeners

import (
	"crypto/tls"
	"fmt"
	"net"
	"strconv"

	"github.com/coreos/go-systemd/activation"
	"github.com/docker/go-connections/sockets"
)

// Init creates new listeners for the server.
// TODO: Clean up the fact that socketGroup and tlsConfig aren't always used.
func Init(proto, addr, socketGroup string, tlsConfig *tls.Config) ([]net.Listener, error) {
	ls := []net.Listener{}

	switch proto {
	case "fd":
		fds, err := listenFD(addr, tlsConfig)
		if err != nil {
			return nil, err
		}
		ls = append(ls, fds...)
	case "tcp":
		l, err := sockets.NewTCPSocket(addr, tlsConfig)
		if err != nil {
			return nil, err
		}
		ls = append(ls, l)
	case "unix":

		gid, err := strconv.Atoi(socketGroup)
		if err != nil {
			return nil, fmt.Errorf("failed to parse socket group id: should be a number: %v", socketGroup)
		}
		l, err := sockets.NewUnixSocket(addr, gid)
		if err != nil {
			return nil, fmt.Errorf("can't create unix socket %s: %v", addr, err)
		}
		ls = append(ls, l)
	default:
		return nil, fmt.Errorf("invalid protocol format: %q", proto)
	}

	return ls, nil
}

// listenFD returns the specified socket activated files as a slice of
// net.Listeners or all of the activated files if "*" is given.
func listenFD(addr string, tlsConfig *tls.Config) ([]net.Listener, error) {
	var (
		err       error
		listeners []net.Listener
	)
	// socket activation
	if tlsConfig != nil {
		listeners, err = activation.TLSListeners(false, tlsConfig)
	} else {
		listeners, err = activation.Listeners(false)
	}
	if err != nil {
		return nil, err
	}

	if len(listeners) == 0 {
		return nil, fmt.Errorf("no sockets found via socket activation: make sure the service was started by systemd")
	}

	// default to all fds just like unix:// and tcp://
	if addr == "" || addr == "*" {
		return listeners, nil
	}

	fdNum, err := strconv.Atoi(addr)
	if err != nil {
		return nil, fmt.Errorf("failed to parse systemd fd address: should be a number: %v", addr)
	}
	fdOffset := fdNum - 3
	if len(listeners) < int(fdOffset)+1 {
		return nil, fmt.Errorf("too few socket activated files passed in by systemd")
	}
	if listeners[fdOffset] == nil {
		return nil, fmt.Errorf("failed to listen on systemd activated file: fd %d", fdOffset+3)
	}
	for i, ls := range listeners {
		if i == fdOffset || ls == nil {
			continue
		}
		if err := ls.Close(); err != nil {
			return nil, fmt.Errorf("failed to close systemd activated file: fd %d: %v", fdOffset+3, err)
		}
	}
	return []net.Listener{listeners[fdOffset]}, nil
}
<|endoftext|>


<commit_before>package position

import (
	"encoding/json"
	"fmt"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/convert"
)

type Status string

type Position struct {
	Id                   int64
	Symbol               string
	Status               Status
	Amount               float64
	BasePrice            float64
	MarginFunding        float64
	MarginFundingType    int64
	ProfitLoss           float64
	ProfitLossPercentage float64
	LiquidationPrice     float64
	Leverage             float64
}

func FromRaw(raw []interface{}) (o *Position, err error) {
	if len(raw) < 6 {
		return o, fmt.Errorf("data slice too short for position: %#v", raw)
	}

	o = &Position{
		Symbol:            convert.SValOrEmpty(raw[0]),
		Status:            Status(convert.SValOrEmpty(raw[1])),
		Amount:            convert.F64ValOrZero(raw[2]),
		BasePrice:         convert.F64ValOrZero(raw[3]),
		MarginFunding:     convert.F64ValOrZero(raw[4]),
		MarginFundingType: convert.I64ValOrZero(raw[5]),
	}

	if len(raw) == 10 {
		o.ProfitLoss = convert.F64ValOrZero(raw[6])
		o.ProfitLossPercentage = convert.F64ValOrZero(raw[7])
		o.LiquidationPrice = convert.F64ValOrZero(raw[8])
		o.Leverage = convert.F64ValOrZero(raw[9])
		return
	}

	if len(raw) > 10 {
		o.ProfitLoss = convert.F64ValOrZero(raw[6])
		o.ProfitLossPercentage = convert.F64ValOrZero(raw[7])
		o.LiquidationPrice = convert.F64ValOrZero(raw[8])
		o.Leverage = convert.F64ValOrZero(raw[9])
		o.Id = int64(convert.F64ValOrZero(raw[11]))
	}

	return
}

type Snapshot struct {
	Snapshot []*Position
}
type New Position
type Update Position
type Cancel Position

func SnapshotFromRaw(raw []interface{}) (s *Snapshot, err error) {
	if len(raw) == 0 {
		return s, fmt.Errorf("data slice too short for position: %#v", raw)
	}

	ps := make([]*Position, 0)
	switch raw[0].(type) {
	case []interface{}:
		for _, v := range raw {
			if l, ok := v.([]interface{}); ok {
				p, err := FromRaw(l)
				if err != nil {
					return s, err
				}
				ps = append(ps, p)
			}
		}
	default:
		return s, fmt.Errorf("not a position snapshot")
	}
	s = &Snapshot{Snapshot: ps}

	return
}

type ClaimRequest struct {
	Id int64
}

func (o *ClaimRequest) ToJSON() ([]byte, error) {
	aux := struct {
		Id int64 `json:"id"`
	}{
		Id: o.Id,
	}
	return json.Marshal(aux)
}
<commit_msg>updating position model to reflect latest api response values<commit_after>package position

import (
	"encoding/json"
	"fmt"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/convert"
)

type Position struct {
	Id                   int64
	Symbol               string
	Status               string
	Amount               float64
	BasePrice            float64
	MarginFunding        float64
	MarginFundingType    int64
	ProfitLoss           float64
	ProfitLossPercentage float64
	LiquidationPrice     float64
	Leverage             float64
	Flag                 interface{}
	MtsCreate            int64
	MtsUpdate            int64
	Type                 string
	Collateral           float64
	CollateralMin        float64
	Meta                 map[string]interface{}
}

type New Position
type Update Position
type Cancel Position

type Snapshot struct {
	Snapshot []*Position
}

func FromRaw(raw []interface{}) (p *Position, err error) {
	if len(raw) < 20 {
		return p, fmt.Errorf("data slice too short for position: %#v", raw)
	}

	p = &Position{
		Symbol:               convert.SValOrEmpty(raw[0]),
		Status:               convert.SValOrEmpty(raw[1]),
		Amount:               convert.F64ValOrZero(raw[2]),
		BasePrice:            convert.F64ValOrZero(raw[3]),
		MarginFunding:        convert.F64ValOrZero(raw[4]),
		MarginFundingType:    convert.I64ValOrZero(raw[5]),
		ProfitLoss:           convert.F64ValOrZero(raw[6]),
		ProfitLossPercentage: convert.F64ValOrZero(raw[7]),
		LiquidationPrice:     convert.F64ValOrZero(raw[8]),
		Leverage:             convert.F64ValOrZero(raw[9]),
		Id:                   convert.I64ValOrZero(raw[11]),
		MtsCreate:            convert.I64ValOrZero(raw[12]),
		MtsUpdate:            convert.I64ValOrZero(raw[13]),
		Type:                 convert.SValOrEmpty(raw[15]),
		Collateral:           convert.F64ValOrZero(raw[17]),
		CollateralMin:        convert.F64ValOrZero(raw[18]),
	}

	if meta, ok := raw[19].(map[string]interface{}); ok {
		p.Meta = meta
	}

	return
}

func NewFromRaw(raw []interface{}) (New, error) {
	p, err := FromRaw(raw)
	if err != nil {
		return New{}, err
	}
	p.Type = "pn"
	return New(*p), nil
}

func UpdateFromRaw(raw []interface{}) (Update, error) {
	p, err := FromRaw(raw)
	if err != nil {
		return Update{}, err
	}
	p.Type = "pu"
	return Update(*p), nil
}

func CancelFromRaw(raw []interface{}) (Cancel, error) {
	p, err := FromRaw(raw)
	if err != nil {
		return Cancel{}, err
	}
	p.Type = "pc"
	return Cancel(*p), nil
}

func SnapshotFromRaw(raw []interface{}) (s *Snapshot, err error) {
	if len(raw) == 0 {
		return s, fmt.Errorf("data slice too short for position: %#v", raw)
	}

	ps := make([]*Position, 0)
	switch raw[0].(type) {
	case []interface{}:
		for _, v := range raw {
			if l, ok := v.([]interface{}); ok {
				p, err := FromRaw(l)
				p.Type = "ps"
				if err != nil {
					return s, err
				}
				ps = append(ps, p)
			}
		}
	default:
		return s, fmt.Errorf("not a position snapshot")
	}
	s = &Snapshot{Snapshot: ps}

	return
}

type ClaimRequest struct {
	Id int64
}

func (o *ClaimRequest) ToJSON() ([]byte, error) {
	aux := struct {
		Id int64 `json:"id"`
	}{
		Id: o.Id,
	}
	return json.Marshal(aux)
}
<|endoftext|>


<commit_before>// Copyright 2016 DeepFabric, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

// +build freebsd openbsd netbsd dragonfly linux

package storage

import (
	"bytes"

	"github.com/deepfabric/elasticell/pkg/log"
	"github.com/deepfabric/elasticell/pkg/pb/raftcmdpb"
	"github.com/deepfabric/elasticell/pkg/util"
	gonemo "github.com/deepfabric/go-nemo"
	"golang.org/x/net/context"
)

var (
	endScan = []byte("")
)

type nemoHashEngine struct {
	limiter *util.Limiter
	db      *gonemo.NEMO
}

func newNemoHashEngine(db *gonemo.NEMO, cfg *NemoCfg) HashEngine {
	return &nemoHashEngine{
		limiter: util.NewLimiter(cfg.LimitConcurrencyWrite),
		db:      db,
	}
}

func (e *nemoHashEngine) HSet(key, field, value []byte) (int64, error) {
	e.limiter.Wait(context.TODO())
	n, err := e.db.HSet(key, field, value)
	e.limiter.Release()

	return int64(n), err
}

func (e *nemoHashEngine) HGet(key, field []byte) ([]byte, error) {
	return e.db.HGet(key, field)
}

func (e *nemoHashEngine) HDel(key []byte, fields ...[]byte) (int64, error) {
	e.limiter.Wait(context.TODO())
	n, err := e.db.HDel(key, fields...)
	e.limiter.Release()

	return n, err
}

func (e *nemoHashEngine) HExists(key, field []byte) (bool, error) {
	return e.db.HExists(key, field)
}

func (e *nemoHashEngine) HKeys(key []byte) ([][]byte, error) {
	return e.db.HKeys(key)
}

func (e *nemoHashEngine) HVals(key []byte) ([][]byte, error) {
	return e.db.HVals(key)
}

func (e *nemoHashEngine) HScanGet(key, start []byte, count int) ([]*raftcmdpb.FVPair, error) {
	iter := e.db.HScan(key, start, endScan, true)

	var result []*raftcmdpb.FVPair
	for {
		if len(result) == count {
			break
		}

		if !iter.Valid() {
			break
		}

		field := iter.Field()
		value := iter.Value()
		if !bytes.Equal(start, field) {
			result = append(result, &raftcmdpb.FVPair{
				Field: field,
				Value: value,
			})

			log.Infof("iter returnd field <%s>, value <%s>, result: %+v",
				field,
				value,
				result)
		}

		iter.Next()
	}

	iter.Free()
	return result, nil
}

func (e *nemoHashEngine) HGetAll(key []byte) ([]*raftcmdpb.FVPair, error) {
	fields, values, err := e.db.HGetall(key)
	if err != nil {
		return nil, err
	}

	if nil == fields || nil == values {
		return nil, nil
	}

	pairs := make([]*raftcmdpb.FVPair, len(fields))
	for idx, field := range fields {
		pairs[idx] = &raftcmdpb.FVPair{
			Field: field,
			Value: values[idx],
		}
	}

	return pairs, nil
}

func (e *nemoHashEngine) HLen(key []byte) (int64, error) {
	return e.db.HLen(key)
}

func (e *nemoHashEngine) HMGet(key []byte, fields ...[]byte) ([][]byte, []error) {
	e.limiter.Wait(context.TODO())
	values, errors := e.db.HMGet(key, fields)
	e.limiter.Release()

	var errs []error
	if len(errors) > 0 {
		for _, err := range errors {
			if err != nil {
				errs = append(errs, err)
			}
		}
	}

	return values, errs
}

func (e *nemoHashEngine) HMSet(key []byte, fields, values [][]byte) error {
	e.limiter.Wait(context.TODO())
	_, err := e.db.HMSet(key, fields, values)
	e.limiter.Release()

	return err
}

func (e *nemoHashEngine) HSetNX(key, field, value []byte) (int64, error) {
	e.limiter.Wait(context.TODO())
	n, err := e.db.HSetnx(key, field, value)
	e.limiter.Release()

	return n, err
}

func (e *nemoHashEngine) HStrLen(key, field []byte) (int64, error) {
	return e.db.HStrlen(key, field)
}

func (e *nemoHashEngine) HIncrBy(key, field []byte, incrment int64) ([]byte, error) {
	e.limiter.Wait(context.TODO())
	value, err := e.db.HIncrby(key, field, incrment)
	e.limiter.Release()

	return value, err
}
<commit_msg>dev: remove debug log<commit_after>// Copyright 2016 DeepFabric, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

// +build freebsd openbsd netbsd dragonfly linux

package storage

import (
	"bytes"

	"github.com/deepfabric/elasticell/pkg/pb/raftcmdpb"
	"github.com/deepfabric/elasticell/pkg/util"
	gonemo "github.com/deepfabric/go-nemo"
	"golang.org/x/net/context"
)

var (
	endScan = []byte("")
)

type nemoHashEngine struct {
	limiter *util.Limiter
	db      *gonemo.NEMO
}

func newNemoHashEngine(db *gonemo.NEMO, cfg *NemoCfg) HashEngine {
	return &nemoHashEngine{
		limiter: util.NewLimiter(cfg.LimitConcurrencyWrite),
		db:      db,
	}
}

func (e *nemoHashEngine) HSet(key, field, value []byte) (int64, error) {
	e.limiter.Wait(context.TODO())
	n, err := e.db.HSet(key, field, value)
	e.limiter.Release()

	return int64(n), err
}

func (e *nemoHashEngine) HGet(key, field []byte) ([]byte, error) {
	return e.db.HGet(key, field)
}

func (e *nemoHashEngine) HDel(key []byte, fields ...[]byte) (int64, error) {
	e.limiter.Wait(context.TODO())
	n, err := e.db.HDel(key, fields...)
	e.limiter.Release()

	return n, err
}

func (e *nemoHashEngine) HExists(key, field []byte) (bool, error) {
	return e.db.HExists(key, field)
}

func (e *nemoHashEngine) HKeys(key []byte) ([][]byte, error) {
	return e.db.HKeys(key)
}

func (e *nemoHashEngine) HVals(key []byte) ([][]byte, error) {
	return e.db.HVals(key)
}

func (e *nemoHashEngine) HScanGet(key, start []byte, count int) ([]*raftcmdpb.FVPair, error) {
	iter := e.db.HScan(key, start, endScan, true)

	var result []*raftcmdpb.FVPair
	for {
		if len(result) == count {
			break
		}

		if !iter.Valid() {
			break
		}

		field := iter.Field()
		value := iter.Value()
		if !bytes.Equal(start, field) {
			result = append(result, &raftcmdpb.FVPair{
				Field: field,
				Value: value,
			})
		}

		iter.Next()
	}

	iter.Free()
	return result, nil
}

func (e *nemoHashEngine) HGetAll(key []byte) ([]*raftcmdpb.FVPair, error) {
	fields, values, err := e.db.HGetall(key)
	if err != nil {
		return nil, err
	}

	if nil == fields || nil == values {
		return nil, nil
	}

	pairs := make([]*raftcmdpb.FVPair, len(fields))
	for idx, field := range fields {
		pairs[idx] = &raftcmdpb.FVPair{
			Field: field,
			Value: values[idx],
		}
	}

	return pairs, nil
}

func (e *nemoHashEngine) HLen(key []byte) (int64, error) {
	return e.db.HLen(key)
}

func (e *nemoHashEngine) HMGet(key []byte, fields ...[]byte) ([][]byte, []error) {
	e.limiter.Wait(context.TODO())
	values, errors := e.db.HMGet(key, fields)
	e.limiter.Release()

	var errs []error
	if len(errors) > 0 {
		for _, err := range errors {
			if err != nil {
				errs = append(errs, err)
			}
		}
	}

	return values, errs
}

func (e *nemoHashEngine) HMSet(key []byte, fields, values [][]byte) error {
	e.limiter.Wait(context.TODO())
	_, err := e.db.HMSet(key, fields, values)
	e.limiter.Release()

	return err
}

func (e *nemoHashEngine) HSetNX(key, field, value []byte) (int64, error) {
	e.limiter.Wait(context.TODO())
	n, err := e.db.HSetnx(key, field, value)
	e.limiter.Release()

	return n, err
}

func (e *nemoHashEngine) HStrLen(key, field []byte) (int64, error) {
	return e.db.HStrlen(key, field)
}

func (e *nemoHashEngine) HIncrBy(key, field []byte, incrment int64) ([]byte, error) {
	e.limiter.Wait(context.TODO())
	value, err := e.db.HIncrby(key, field, incrment)
	e.limiter.Release()

	return value, err
}
<|endoftext|>


<commit_before>package main

import "testing"

func TestCount(t *testing.T) {
	cases := []struct {
		in       string
		expected int
	}{
		{"(", 1},
		{"()", 0},
		{"(((", 3},
		{"(()(()(", 3},
		{"))(((((", 3},
		{"())", -1},
		{"))(", -1},
		{")))", -3},
		{")())())", -3},
	}
	for _, c := range cases {
		result := Count(c.in)
		if result != c.expected {
			t.Errorf("Count(%q) == %d, expected %d", c.in, result, c.expected)
		}
	}
}

func TestEntryPoint(t *testing.T) {
	cases := []struct {
		input    string
		expected int
	}{
		{"", 0},
		{")", 1},
		{"()())", 5},
	}
	for _, c := range cases {
		basement := -1
		result, _ := FindEntryPoint(c.input, basement)
		if result != c.expected {
			t.Errorf("Floor(%d) reached at character %d, not %d", basement, result, c.expected)
		}
	}
}
<commit_msg>tweaked test to handle new code<commit_after>package main

import "testing"

func TestCount(t *testing.T) {
	cases := []struct {
		in       string
		expected int
	}{
		{"(", 1},
		{"()", 0},
		{"(((", 3},
		{"(()(()(", 3},
		{"))(((((", 3},
		{"())", -1},
		{"))(", -1},
		{")))", -3},
		{")())())", -3},
	}
	for _, c := range cases {
		result := count(c.in)
		if result != c.expected {
			t.Errorf("Count(%q) == %d, expected %d", c.in, result, c.expected)
		}
	}
}

func TestEntryPoint(t *testing.T) {
	cases := []struct {
		input    string
		expected int
	}{
		{"", 0},
		{")", 1},
		{"()())", 5},
	}
	for _, c := range cases {
		basement := -1
		_, result := findFloor(c.input, 0, basement)
		if result != c.expected {
			t.Errorf("Floor(%d) reached at character %d, not %d", basement, result, c.expected)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
    "web"
    "strings"
    "godis"
    "fmt"
    "os"
    "url"
)

const(
    // characters used for short-urls
    SYMBOLS = "0123456789abcdefghijklmnopqrsuvwxyzABCDEFGHIJKLMNOPQRSTUVXYZ"
    // special key in redis, that is our global counter
    COUNTER = "__counter__"
    HTTP = "http"
    HTTPS = "https"

)

// connecting to redis on localhost, db with id 0 and no password
var (
    redis *godis.Client
    config *Config
)

// function to resolve a shorturl and redirect
func resolve(ctx *web.Context, short string) {
    redirect, err := redis.Get(short)
    if err == nil {
        ctx.Redirect(301, redirect.String())
    } else {
        ctx.Redirect(301, "https://www.youtube.com/watch?v=jRHmvy5eaG4")
    }
}

func isValidUrl(rawurl string) (u *url.URL, err os.Error){
    if len(rawurl) == 0{
        return nil, os.NewError("empty url")
    }
    // XXX this needs some love...
    if !strings.HasPrefix(rawurl, HTTP){
        rawurl = fmt.Sprintf("%s://%s", HTTP, rawurl)
    }
    return url.Parse(rawurl)
}

// function to shorten and store a url
func shorten(ctx *web.Context, data string){
    const(
        jsntmpl = "{\"url\" : \"%s\", \"longurl\" : \"%s\"}\n"
    )
    host := config.GetStringDefault("hostname", "localhost")
    r, _ := ctx.Request.Params["url"]
    theUrl, err := isValidUrl(string(r))
    if err == nil{
        ctr, _ := redis.Incr(COUNTER)
        encoded := encode(ctr)
        // fire and forget
        go redis.Set(encoded, theUrl.Raw)

        ctx.SetHeader("Content-Type", "application/json", true)
        location := fmt.Sprintf("%s://%s/%s", HTTP, host, encoded)
        ctx.SetHeader("Location", location, true)
        ctx.StartResponse(201)
        ctx.WriteString(fmt.Sprintf(jsntmpl, location, theUrl.Raw))
    }else{
       ctx.Redirect(404, "/")
    }
}

// encodes a number into our *base* representation
// TODO can this be made better with some bitshifting?
func encode(number int64) string{
    const base = int64(len(SYMBOLS))
    rest := number % base
    // strings are a bit weird in go...
    result := string(SYMBOLS[rest])
    if number - rest != 0{
       newnumber := (number - rest ) / base
       result = encode(newnumber) + result
    }
    return result
}

func bootstrap(path string) os.Error {
    config = NewConfig(path)
    config.Parse()
    host := config.GetStringDefault("redis.address", "tcp:localhost:6379")
    db := config.GetIntDefault("redis.database", 0)
    passwd := config.GetStringDefault("redis.password", "")

    redis = godis.New(host, db, passwd)
    return nil
}

// main function that inits the routes in web.go
func main() {
    err := bootstrap("conf/kurz.conf")
    if err == nil {
        // this could go to bootstrap as well
        web.Post("/shorten/(.*)", shorten)
        web.Get("/(.*)", resolve)
        listen := config.GetStringDefault("listen", "0.0.0.0")
        port := config.GetStringDefault("port", "9999")
        web.Run(fmt.Sprintf("%s:%s", listen, port))
    }
}

<commit_msg>remove encoding bits from main file and use the new Encode function<commit_after>package main

import (
    "web"
    "strings"
    "godis"
    "fmt"
    "os"
    "url"
)

const(
    // special key in redis, that is our global counter
    COUNTER = "__counter__"
    HTTP = "http"
)

// connecting to redis on localhost, db with id 0 and no password
var (
    redis *godis.Client
    config *Config
)

// function to resolve a shorturl and redirect
func resolve(ctx *web.Context, short string) {
    redirect, err := redis.Get(short)
    if err == nil {
        ctx.Redirect(301, redirect.String())
    } else {
        ctx.Redirect(301, "https://www.youtube.com/watch?v=jRHmvy5eaG4")
    }
}

func isValidUrl(rawurl string) (u *url.URL, err os.Error){
    if len(rawurl) == 0{
        return nil, os.NewError("empty url")
    }
    // XXX this needs some love...
    if !strings.HasPrefix(rawurl, HTTP){
        rawurl = fmt.Sprintf("%s://%s", HTTP, rawurl)
    }
    return url.Parse(rawurl)
}

// function to shorten and store a url
func shorten(ctx *web.Context, data string){
    const(
        jsntmpl = "{\"url\" : \"%s\", \"longurl\" : \"%s\"}\n"
    )
    host := config.GetStringDefault("hostname", "localhost")
    r, _ := ctx.Request.Params["url"]
    theUrl, err := isValidUrl(string(r))
    if err == nil{
        ctr, _ := redis.Incr(COUNTER)
        encoded := Encode(ctr)
        // fire and forget
        go redis.Set(encoded, theUrl.Raw)

        ctx.SetHeader("Content-Type", "application/json", true)
        location := fmt.Sprintf("%s://%s/%s", HTTP, host, encoded)
        ctx.SetHeader("Location", location, true)
        ctx.StartResponse(201)
        ctx.WriteString(fmt.Sprintf(jsntmpl, location, theUrl.Raw))
    }else{
       ctx.Redirect(404, "/")
    }
}

func bootstrap(path string) os.Error {
    config = NewConfig(path)
    config.Parse()
    host := config.GetStringDefault("redis.address", "tcp:localhost:6379")
    db := config.GetIntDefault("redis.database", 0)
    passwd := config.GetStringDefault("redis.password", "")

    redis = godis.New(host, db, passwd)
    return nil
}

// main function that inits the routes in web.go
func main() {
    err := bootstrap("conf/kurz.conf")
    if err == nil {
        // this could go to bootstrap as well
        web.Post("/shorten/(.*)", shorten)
        web.Get("/(.*)", resolve)
        listen := config.GetStringDefault("listen", "0.0.0.0")
        port := config.GetStringDefault("port", "9999")
        web.Run(fmt.Sprintf("%s:%s", listen, port))
    }
}

<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"sync"
	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
	"os"
)

// LangValue struct for github.Repository.ListLanguages map
type LangValue struct {
	Lang string
	Value int
}

// RepoLang struct for identify repo with his used languages
type RepoLang struct {
	RepName string
    langs LangValue
}

// Stats for concurrency accessible slice from goroutines
type Stats struct {
	sync.RWMutex
	Statistic []RepoLang
}

func (s *Stats) add(name, login string, client *github.Client, wg *sync.WaitGroup, ) {
    defer s.Unlock()
    defer wg.Done()
	langs,_,err := client.Repositories.ListLanguages(login,name)
	if err != nil {
		fmt.Println(err)
	}
	s.Lock()
    for k,v := range langs {
		tlang := LangValue{Lang:k, Value: v}
		s.Statistic = append(s.Statistic, RepoLang{RepName:name, langs:tlang})
	}
}

func main() {
	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken:"some token here"})
	tc := oauth2.NewClient(oauth2.NoContext,ts)
	client := github.NewClient(tc)
	result := Stats{Statistic:make([]RepoLang,0,0)}
	var wg sync.WaitGroup
	repos, _, err := client.Repositories.List(os.Args[1], nil)
	if err != nil {
		fmt.Println(err)
	}
	for _, v := range repos {
		fmt.Println(*v.Name)
		wg.Add(1)
		go result.add(*v.Name,"envek",client,&wg)
	}
	wg.Wait()
	fmt.Println(result)
}
<commit_msg>Use os.Args for input user login and uses github Token<commit_after>package main

import (
	"fmt"
	"sync"
	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
	"os"
)

// LangValue struct for github.Repository.ListLanguages map
type LangValue struct {
	Lang string
	Value int
}

// RepoLang struct for identify repo with his used languages
type RepoLang struct {
	RepName string
    langs LangValue
}

// Stats for concurrency accessible slice from goroutines
type Stats struct {
	sync.RWMutex
	Statistic []RepoLang
}

func (s *Stats) add(name, login string, client *github.Client, wg *sync.WaitGroup, ) {
    defer s.Unlock()
    defer wg.Done()
	langs,_,err := client.Repositories.ListLanguages(login,name)
	if err != nil {
		fmt.Println(err)
	}
	s.Lock()
    for k,v := range langs {
		tlang := LangValue{Lang:k, Value: v}
		s.Statistic = append(s.Statistic, RepoLang{RepName:name, langs:tlang})
	}
}

func main() {
	login := os.Args[1]
	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken:"some token"})
	tc := oauth2.NewClient(oauth2.NoContext,ts)
	client := github.NewClient(tc)
	result := Stats{Statistic:make([]RepoLang,0,0)}
	var wg sync.WaitGroup
	repos, _, err := client.Repositories.List(login, nil)
	if err != nil {
		fmt.Println(err)
	}
	for _, v := range repos {
		wg.Add(1)
		go result.add(*v.Name,login,client,&wg)
	}
	wg.Wait()
	fmt.Println(result)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"net/http"
	"os"
	"regexp"
	"strconv"

	"github.com/Sirupsen/logrus"
	"github.com/infinityworks/prometheus-rancher-exporter/measure"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	namespace           = "rancher" // Used to prepand Prometheus metrics created by this exporter.
	defaultLabelsFilter = "^io.prometheus"
)

// Runtime variables, user controllable for targeting, authentication and filtering.
var (
	log = logrus.New()

	metricsPath   = getEnv("METRICS_PATH", "/metrics")            // Path under which to expose metrics
	listenAddress = getEnv("LISTEN_ADDRESS", ":9173")             // Address on which to expose metrics
	rancherURL    = os.Getenv("CATTLE_URL")                       // URL of Rancher Server API e.g. http://192.168.0.1:8080/v2-beta
	accessKey     = os.Getenv("CATTLE_ACCESS_KEY")                // Optional - Access Key for Rancher API
	secretKey     = os.Getenv("CATTLE_SECRET_KEY")                // Optional - Secret Key for Rancher API
	labelsFilter  = os.Getenv("LABELS_FILTER")                    // Optional - Filter for Rancher label names
	logLevel      = getEnv("LOG_LEVEL", "info")                   // Optional - Set the logging level
	resourceLimit = getEnv("API_LIMIT", "100")                    // Optional - Rancher API resource limit (default: 100)
	hideSys, _    = strconv.ParseBool(getEnv("HIDE_SYS", "true")) // hideSys - Optional - Flag that indicates if the environment variable `HIDE_SYS` is set to a boolean true value
)

// Predefined variables that are used throughout the exporter
var (
	agentStates     = []string{"activating", "active", "reconnecting", "disconnected", "disconnecting", "finishing-reconnect", "reconnected"}
	clusterStates   = []string{"active", "cordoned", "degraded", "disconnected", "drained", "draining", "healthy", "initializing", "locked", "purged", "purging", "reconnecting", "reinitializing", "removed", "running", "unavailable", "unhealthy", "upgraded", "upgrading"}
	hostStates      = []string{"activating", "active", "deactivating", "disconnected", "error", "erroring", "inactive", "provisioned", "purged", "purging", "reconnecting", "registering", "removed", "removing", "requested", "restoring", "updating_active", "updating_inactive"}
	stackStates     = []string{"activating", "active", "canceled_upgrade", "canceling_upgrade", "error", "erroring", "finishing_upgrade", "removed", "removing", "requested", "restarting", "rolling_back", "updating_active", "upgraded", "upgrading"}
	serviceStates   = []string{"activating", "active", "canceled_upgrade", "canceling_upgrade", "deactivating", "finishing_upgrade", "inactive", "registering", "removed", "removing", "requested", "restarting", "rolling_back", "updating_active", "updating_inactive", "upgraded", "upgrading"}
	healthStates    = []string{"healthy", "unhealthy", "initializing", "degraded", "started-once"}
	componentStatus = []string{"True", "False", "Unknown"}
	nodeStates      = []string{"active", "cordoned", "drained", "draining", "provisioning", "registering", "unavailable"}
	endpoints       = []string{"stacks", "services", "hosts"} // EndPoints the exporter will trawl
	endpointsV3     = []string{"clusters", "nodes"} // EndPoints the exporter will trawl]
	stackRef        = make(map[string]string)                 // Stores the StackID and StackName as a map, used to provide label dimensions to service metrics
	clusterRef      = make(map[string]string)	                // Stores the ClusterID and ClusterName as a map, used to provide label dimensions to node metrics
)

// getEnv - Allows us to supply a fallback option if nothing specified
func getEnv(key, fallback string) string {
	value := os.Getenv(key)
	if len(value) == 0 {
		return fallback
	}
	return value
}

func main() {
	flag.Parse()

	// Sets the logging value for the exporter, defaults to info
	setLogLevel(logLevel)

	// check the rancherURL ($CATTLE_URL) has been provided correctly
	if rancherURL == "" {
		log.Fatal("CATTLE_URL must be set and non-empty")
	}

	if labelsFilter == "" {
		labelsFilter = defaultLabelsFilter
	}

	labelsFilterRegexp, err := regexp.Compile(labelsFilter)
	if err != nil {
		log.Fatal("LABELS_FILTER must be valid regular expression")
	}

	log.Info("Starting Prometheus Exporter for Rancher")
	log.Info(
		"Runtime Configuration in-use: URL of Rancher Server: ",
		rancherURL,
		" Access key: ",
		accessKey,
		" System services hidden: ",
		hideSys,
		" Labels filter: ",
		labelsFilter,
	)

	// Register internal metrics used for tracking the exporter performance
	measure.Init()

	// Register a new Exporter
	exporter := newExporter(rancherURL, accessKey, secretKey, labelsFilterRegexp, hideSys, resourceLimit)

	// Register Metrics from each of the endpoints
	// This invokes the Collect method through the prometheus client libraries.
	prometheus.MustRegister(exporter)

	// Setup HTTP handler
	http.Handle(metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
		                <title>Rancher exporter</title>

rancher exporter

<a href='` + metricsPath + `'>Metrics

		              `))
	})
	log.Printf("Starting Server on port %s and path %s", listenAddress, metricsPath)
	log.Fatal(http.ListenAndServe(listenAddress, nil))
}
<commit_msg>updated to use prometheus/client_golang v1.0.0<commit_after>package main

import (
	"flag"
	"net/http"
	"os"
	"regexp"
	"strconv"

	"github.com/Sirupsen/logrus"
	"github.com/infinityworks/prometheus-rancher-exporter/measure"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

const (
	namespace           = "rancher" // Used to prepand Prometheus metrics created by this exporter.
	defaultLabelsFilter = "^io.prometheus"
)

// Runtime variables, user controllable for targeting, authentication and filtering.
var (
	log = logrus.New()

	metricsPath   = getEnv("METRICS_PATH", "/metrics")            // Path under which to expose metrics
	listenAddress = getEnv("LISTEN_ADDRESS", ":9173")             // Address on which to expose metrics
	rancherURL    = os.Getenv("CATTLE_URL")                       // URL of Rancher Server API e.g. http://192.168.0.1:8080/v2-beta
	accessKey     = os.Getenv("CATTLE_ACCESS_KEY")                // Optional - Access Key for Rancher API
	secretKey     = os.Getenv("CATTLE_SECRET_KEY")                // Optional - Secret Key for Rancher API
	labelsFilter  = os.Getenv("LABELS_FILTER")                    // Optional - Filter for Rancher label names
	logLevel      = getEnv("LOG_LEVEL", "info")                   // Optional - Set the logging level
	resourceLimit = getEnv("API_LIMIT", "100")                    // Optional - Rancher API resource limit (default: 100)
	hideSys, _    = strconv.ParseBool(getEnv("HIDE_SYS", "true")) // hideSys - Optional - Flag that indicates if the environment variable `HIDE_SYS` is set to a boolean true value
)

// Predefined variables that are used throughout the exporter
var (
	agentStates     = []string{"activating", "active", "reconnecting", "disconnected", "disconnecting", "finishing-reconnect", "reconnected"}
	clusterStates   = []string{"active", "cordoned", "degraded", "disconnected", "drained", "draining", "healthy", "initializing", "locked", "purged", "purging", "reconnecting", "reinitializing", "removed", "running", "unavailable", "unhealthy", "upgraded", "upgrading"}
	hostStates      = []string{"activating", "active", "deactivating", "disconnected", "error", "erroring", "inactive", "provisioned", "purged", "purging", "reconnecting", "registering", "removed", "removing", "requested", "restoring", "updating_active", "updating_inactive"}
	stackStates     = []string{"activating", "active", "canceled_upgrade", "canceling_upgrade", "error", "erroring", "finishing_upgrade", "removed", "removing", "requested", "restarting", "rolling_back", "updating_active", "upgraded", "upgrading"}
	serviceStates   = []string{"activating", "active", "canceled_upgrade", "canceling_upgrade", "deactivating", "finishing_upgrade", "inactive", "registering", "removed", "removing", "requested", "restarting", "rolling_back", "updating_active", "updating_inactive", "upgraded", "upgrading"}
	healthStates    = []string{"healthy", "unhealthy", "initializing", "degraded", "started-once"}
	componentStatus = []string{"True", "False", "Unknown"}
	nodeStates      = []string{"active", "cordoned", "drained", "draining", "provisioning", "registering", "unavailable"}
	endpoints       = []string{"stacks", "services", "hosts"} // EndPoints the exporter will trawl
	endpointsV3     = []string{"clusters", "nodes"} // EndPoints the exporter will trawl]
	stackRef        = make(map[string]string)                 // Stores the StackID and StackName as a map, used to provide label dimensions to service metrics
	clusterRef      = make(map[string]string)	                // Stores the ClusterID and ClusterName as a map, used to provide label dimensions to node metrics
)

// getEnv - Allows us to supply a fallback option if nothing specified
func getEnv(key, fallback string) string {
	value := os.Getenv(key)
	if len(value) == 0 {
		return fallback
	}
	return value
}

func main() {
	flag.Parse()

	// Sets the logging value for the exporter, defaults to info
	setLogLevel(logLevel)

	// check the rancherURL ($CATTLE_URL) has been provided correctly
	if rancherURL == "" {
		log.Fatal("CATTLE_URL must be set and non-empty")
	}

	if labelsFilter == "" {
		labelsFilter = defaultLabelsFilter
	}

	labelsFilterRegexp, err := regexp.Compile(labelsFilter)
	if err != nil {
		log.Fatal("LABELS_FILTER must be valid regular expression")
	}

	log.Info("Starting Prometheus Exporter for Rancher")
	log.Info(
		"Runtime Configuration in-use: URL of Rancher Server: ",
		rancherURL,
		" Access key: ",
		accessKey,
		" System services hidden: ",
		hideSys,
		" Labels filter: ",
		labelsFilter,
	)

	// Register internal metrics used for tracking the exporter performance
	measure.Init()

	// Register a new Exporter
	exporter := newExporter(rancherURL, accessKey, secretKey, labelsFilterRegexp, hideSys, resourceLimit)

	// Register Metrics from each of the endpoints
	// This invokes the Collect method through the prometheus client libraries.
	prometheus.MustRegister(exporter)

	// Setup HTTP handler
	http.Handle(metricsPath, promhttp.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
		                <title>Rancher exporter</title>

rancher exporter

<a href='` + metricsPath + `'>Metrics

		              `))
	})
	log.Printf("Starting Server on port %s and path %s", listenAddress, metricsPath)
	log.Fatal(http.ListenAndServe(listenAddress, nil))
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Huawei Technologies Co., Ltd. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package module

import (
	"encoding/json"
	"errors"
	"strconv"
	"strings"
	"time"

	"github.com/Huawei/containerops/pilotage/models"

	log "github.com/Sirupsen/logrus"
	"github.com/containerops/configure"
	"github.com/jinzhu/gorm"
)

type WorkflowVar struct {
	*models.WorkflowVar
}

type WorkflowVarLog struct {
	*models.WorkflowVarLog
}

var eventList = map[string]string{
	"CO_COMPONENT_START": "CO_COMPONENT_START",
	"CO_COMPONENT_STOP":  "CO_COMPONENT_STOP",
	"CO_TASK_START":      "CO_TASK_START",
	"CO_TASK_RESULT":     "CO_TASK_RESULT",
	"CO_TASK_STATUS":     "CO_TASK_STATUS",
	"CO_REGISTER_URL":    "CO_register",
}

var projectAddr = ""

func init() {
	if configure.GetString("projectaddr") == "" {
		projectAddr = "http://localhost"
	} else {
		projectAddr = configure.GetString("projectaddr")
	}
	projectAddr = strings.TrimSuffix(projectAddr, "/")
}

func setSystemEvent(db *gorm.DB, actionLog *models.ActionLog) error {
	if db == nil {
		db = models.GetDB().Begin()
		err := db.Error
		if err != nil {
			log.Error("[setSystemEvent]:when db.Begin():", err.Error())
			return err
		}
	}

	workflowLog := new(models.WorkflowLog)
	err := db.Model(&models.WorkflowLog{}).Where("id = ?", actionLog.Workflow).First(workflowLog).Error
	if err != nil {
		log.Error("[setSystemEvent]:error when get workflowlog info from db:", err.Error())
		rollbackErr := db.Rollback().Error
		if rollbackErr != nil {
			log.Error("[setSystemEvent]:when rollback in get workflowlog's info:", rollbackErr.Error())
			return errors.New("errors occur:\nerror1:" + err.Error() + "\nerror2:" + rollbackErr.Error())
		}
		return err
	}

	for key, value := range eventList {
		tempEvent := new(models.EventDefinition)
		tempEvent.Event = key
		tempEvent.Title = key
		tempEvent.Namespace = actionLog.Namespace
		tempEvent.Repository = actionLog.Repository
		tempEvent.Workflow = actionLog.Workflow
		tempEvent.Stage = actionLog.Stage
		tempEvent.Action = actionLog.ID
		tempEvent.Character = models.CharacterComponentEvent
		tempEvent.Type = models.TypeSystemEvent
		tempEvent.Source = models.SourceInnerEvent
		tempEvent.Definition = projectAddr + "/v2/" + actionLog.Namespace + "/" + actionLog.Repository + "/workflow/v1/runtime/event/" + workflowLog.Workflow + "/" + value

		err := db.Save(tempEvent).Error
		if err != nil {
			log.Error("[setSystemEvent]:error when save event definition to db:", err.Error())
			rollbackErr := db.Rollback().Error
			if rollbackErr != nil {
				log.Error("[setSystemEvent]:when rollback in get workflowlog's info:", rollbackErr.Error())
				return errors.New("errors occur:\nerror1:" + err.Error() + "\nerror2:" + rollbackErr.Error())
			}
			return err
		}
	}

	return nil
}

func getSystemEventList(actionID int64) ([]map[string]interface{}, error) {
	result := make([]map[string]interface{}, 0)

	eventDefineList := make([]models.EventDefinition, 0)

	err := new(models.EventDefinition).GetEventDefinition().Where("action = ?", actionID).Find(&eventDefineList).Error

	if err != nil {
		log.Error("[getSystemEventList]:error when get systemEventList from db:", err.Error())
		return nil, err
	}

	for _, eventDefine := range eventDefineList {
		tempMap := make(map[string]interface{})
		tempMap["name"] = eventDefine.Title
		tempMap["value"] = eventDefine.Definition
		tempMap["Title"] = eventDefine.Title
		tempMap["ID"] = eventDefine.ID

		result = append(result, tempMap)
	}

	return result, nil
}

func RecordEventInfo(eventDefineId, sequence int64, headerInfo, payload, authInfo string, eventDefineInfo ...string) error {
	eventDefine := new(models.EventDefinition)
	if eventDefineId < 0 {
		eventDefine.Type = models.TypeSystemEvent
		eventDefine.Source = models.SourceInnerEvent

		if len(eventDefineInfo) > 0 {
			eventDefine.Title = eventDefineInfo[0]
		}

		if len(eventDefineInfo) > 1 {
			characterInt, _ := strconv.ParseInt(eventDefineInfo[1], 10, 64)
			eventDefine.Character = characterInt
		}

		if len(eventDefineInfo) > 2 {
			eventDefine.Namespace = eventDefineInfo[2]
		}

		if len(eventDefineInfo) > 3 {
			eventDefine.Repository = eventDefineInfo[3]
		}

		if len(eventDefineInfo) > 4 {
			pipelinInt, _ := strconv.ParseInt(eventDefineInfo[4], 10, 64)
			eventDefine.Workflow = pipelinInt
		}

		if len(eventDefineInfo) > 5 {
			stageInt, _ := strconv.ParseInt(eventDefineInfo[5], 10, 64)
			eventDefine.Stage = stageInt
		}

		if len(eventDefineInfo) > 6 {
			actionInt, _ := strconv.ParseInt(eventDefineInfo[6], 10, 64)
			eventDefine.Action = actionInt
		}
	} else {
		err := eventDefine.GetEventDefinition().Where("id = ?", eventDefineId).First(eventDefine).Error
		if err != nil {
			log.Error("[event's RecordEventInfo]:error when get event define from db:", err.Error())
			return err
		}
	}

	event := new(models.Event)
	event.Definition = eventDefineId
	event.Title = eventDefine.Title
	event.Header = headerInfo
	event.Payload = payload
	event.Authorization = authInfo
	event.Type = eventDefine.Type
	event.Source = eventDefine.Source
	event.Character = eventDefine.Character
	event.Namespace = eventDefine.Namespace
	event.Repository = eventDefine.Repository
	event.Workflow = eventDefine.Workflow
	event.Stage = eventDefine.Stage
	event.Action = eventDefine.Action
	event.Sequence = sequence

	err := event.GetEvent().Save(event).Error
	if err != nil {
		log.Error("[event's RecordEventInfo]:error when save event info to db:", err.Error())
		return err
	}

	return nil
}

func SetWorkflowVarInfo(id int64, varMap map[string]interface{}) error {
	db := models.GetDB().Begin()
	err := db.Error
	if err != nil {
		log.Error("[workflowVar's SetWorkflowVarInfo]:when db.Begin():", err.Error())
		return errors.New("error when db.Begin")
	}

	err = db.Model(&models.WorkflowVar{}).Where("workflow = ?", id).Unscoped().Delete(&models.WorkflowVar{}).Error
	if err != nil {
		log.Error("[workflowVar's SetWorkflowVarInfo]:when delet var info from db:", err.Error())
		rollbackErr := db.Rollback().Error
		if rollbackErr != nil {
			log.Error("[workflowVar's SetWorkflowVarInfo]:when rollback in delet var info got err:", rollbackErr.Error())
			return errors.New("errors occur:\nerror1:" + err.Error() + "\nerror2:" + rollbackErr.Error())
		}
		return errors.New("error when delete var info from db:" + err.Error())
	}

	for key, defaultValue := range varMap {
		varSet := new(models.WorkflowVar)

		defaultValueStr, ok := defaultValue.(string)
		if !ok {
			log.Error("[workflowVar's SetWorkflowVarInfo]:error when pase default value, want a string,got:", defaultValue)
			return errors.New("var's vaule is not string")
		}

		varSet.Workflow = id
		varSet.Key = key
		varSet.Default = defaultValueStr

		err = db.Model(&models.WorkflowVar{}).Save(varSet).Error
		if err != nil {
			log.Error("[workflowVar's SetWorkflowVarInfo]:when save var info from db:", err.Error())
			rollbackErr := db.Rollback().Error
			if rollbackErr != nil {
				log.Error("[workflowVar's SetWorkflowVarInfo]:when rollback in save var info got err:", rollbackErr.Error())
				return errors.New("errors occur:\nerror1:" + err.Error() + "\nerror2:" + rollbackErr.Error())
			}
			return errors.New("error when save var info")
		}
	}

	db.Commit()
	return nil
}

func GetWorkflowVarInfo(id int64) (map[string]string, error) {
	resultMap := make(map[string]string)
	varList := make([]models.WorkflowVar, 0)

	err := new(models.WorkflowVar).GetWorkflowVar().Where("workflow = ?", id).Find(&varList).Error
	if err != nil {
		log.Error("[workflowVar's GetWorkflowVarInfo]:error when get var list from db:", err.Error())
		return nil, errors.New("error when get var info from db")
	}

	for _, varInfo := range varList {
		resultMap[varInfo.Key] = varInfo.Default
	}

	return resultMap, nil
}

func (workflowVar *WorkflowVar) GenerateNewLog(db *gorm.DB, workflowLog *models.WorkflowLog) error {
	if db == nil {
		db = models.GetDB()
		db = db.Begin()
	}

	changeLogMap := make(map[string]interface{})
	changeLogMap["user"] = "system"
	changeLogMap["time"] = time.Now().Format("2006-01-02 15:04:05")
	changeLogMap["action"] = "init data: set key:" + workflowVar.Key + " 's value to" + workflowVar.Default

	changeLogList := make([]interface{}, 0)
	changeLogList = append(changeLogList, changeLogMap)

	changeInfoBytes, _ := json.Marshal(changeLogList)

	varLog := new(models.WorkflowVarLog)
	varLog.Workflow = workflowLog.ID
	varLog.FromWorkflow = workflowLog.FromWorkflow
	varLog.Sequence = workflowLog.Sequence
	varLog.Key = workflowVar.Key
	varLog.Default = workflowVar.Default
	varLog.Vaule = varLog.Default
	varLog.ChangeLog = string(changeInfoBytes)

	err := varLog.GetWorkflowVarLog().Save(varLog).Error
	if err != nil {
		rollbackErr := db.Rollback().Error
		if rollbackErr != nil {
			log.Error("[Workflow's GenerateNewLog]:when rollback in save workflow var log:", rollbackErr.Error())
			return errors.New("errors occur:\nerror1:" + err.Error() + "\nerror2:" + rollbackErr.Error())
		}

		return err
	}

	return nil
}

func getWorkflowVarLogInfo(workflow, sequence int64, key string) (string, error) {
	varInfo := new(models.WorkflowVarLog)
	err := varInfo.GetWorkflowVarLog().Where("workflow = ?", workflow).Where("sequence = ?", sequence).Where("`key` = ?", key).First(varInfo).Error
	if err != nil {
		log.Error("[workflowVarLog's getWorkflowVarLogInfo]:get workflow var info from db error:", err.Error())
		return "", errors.New("stage's timeout is not a global value")
	}

	return varInfo.Vaule, nil
}
<commit_msg>bugfix<commit_after>/*
Copyright 2014 Huawei Technologies Co., Ltd. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package module

import (
	"encoding/json"
	"errors"
	"strconv"
	"strings"
	"time"

	"github.com/Huawei/containerops/pilotage/models"

	log "github.com/Sirupsen/logrus"
	"github.com/containerops/configure"
	"github.com/jinzhu/gorm"
)

type WorkflowVar struct {
	*models.WorkflowVar
}

type WorkflowVarLog struct {
	*models.WorkflowVarLog
}

var eventList = map[string]string{
	"CO_COMPONENT_START": "CO_COMPONENT_START",
	"CO_COMPONENT_STOP":  "CO_COMPONENT_STOP",
	"CO_TASK_START":      "CO_TASK_START",
	"CO_TASK_RESULT":     "CO_TASK_RESULT",
	"CO_TASK_STATUS":     "CO_TASK_STATUS",
	"CO_REGISTER_URL":    "register",
}

var projectAddr = ""

func init() {
	if configure.GetString("projectaddr") == "" {
		projectAddr = "http://localhost"
	} else {
		projectAddr = configure.GetString("projectaddr")
	}
	projectAddr = strings.TrimSuffix(projectAddr, "/")
}

func setSystemEvent(db *gorm.DB, actionLog *models.ActionLog) error {
	if db == nil {
		db = models.GetDB().Begin()
		err := db.Error
		if err != nil {
			log.Error("[setSystemEvent]:when db.Begin():", err.Error())
			return err
		}
	}

	workflowLog := new(models.WorkflowLog)
	err := db.Model(&models.WorkflowLog{}).Where("id = ?", actionLog.Workflow).First(workflowLog).Error
	if err != nil {
		log.Error("[setSystemEvent]:error when get workflowlog info from db:", err.Error())
		rollbackErr := db.Rollback().Error
		if rollbackErr != nil {
			log.Error("[setSystemEvent]:when rollback in get workflowlog's info:", rollbackErr.Error())
			return errors.New("errors occur:\nerror1:" + err.Error() + "\nerror2:" + rollbackErr.Error())
		}
		return err
	}

	for key, value := range eventList {
		tempEvent := new(models.EventDefinition)
		tempEvent.Event = key
		tempEvent.Title = key
		tempEvent.Namespace = actionLog.Namespace
		tempEvent.Repository = actionLog.Repository
		tempEvent.Workflow = actionLog.Workflow
		tempEvent.Stage = actionLog.Stage
		tempEvent.Action = actionLog.ID
		tempEvent.Character = models.CharacterComponentEvent
		tempEvent.Type = models.TypeSystemEvent
		tempEvent.Source = models.SourceInnerEvent
		tempEvent.Definition = projectAddr + "/v2/" + actionLog.Namespace + "/" + actionLog.Repository + "/workflow/v1/runtime/event/" + workflowLog.Workflow + "/" + value

		err := db.Save(tempEvent).Error
		if err != nil {
			log.Error("[setSystemEvent]:error when save event definition to db:", err.Error())
			rollbackErr := db.Rollback().Error
			if rollbackErr != nil {
				log.Error("[setSystemEvent]:when rollback in get workflowlog's info:", rollbackErr.Error())
				return errors.New("errors occur:\nerror1:" + err.Error() + "\nerror2:" + rollbackErr.Error())
			}
			return err
		}
	}

	return nil
}

func getSystemEventList(actionID int64) ([]map[string]interface{}, error) {
	result := make([]map[string]interface{}, 0)

	eventDefineList := make([]models.EventDefinition, 0)

	err := new(models.EventDefinition).GetEventDefinition().Where("action = ?", actionID).Find(&eventDefineList).Error

	if err != nil {
		log.Error("[getSystemEventList]:error when get systemEventList from db:", err.Error())
		return nil, err
	}

	for _, eventDefine := range eventDefineList {
		tempMap := make(map[string]interface{})
		tempMap["name"] = eventDefine.Title
		tempMap["value"] = eventDefine.Definition
		tempMap["Title"] = eventDefine.Title
		tempMap["ID"] = eventDefine.ID

		result = append(result, tempMap)
	}

	return result, nil
}

func RecordEventInfo(eventDefineId, sequence int64, headerInfo, payload, authInfo string, eventDefineInfo ...string) error {
	eventDefine := new(models.EventDefinition)
	if eventDefineId < 0 {
		eventDefine.Type = models.TypeSystemEvent
		eventDefine.Source = models.SourceInnerEvent

		if len(eventDefineInfo) > 0 {
			eventDefine.Title = eventDefineInfo[0]
		}

		if len(eventDefineInfo) > 1 {
			characterInt, _ := strconv.ParseInt(eventDefineInfo[1], 10, 64)
			eventDefine.Character = characterInt
		}

		if len(eventDefineInfo) > 2 {
			eventDefine.Namespace = eventDefineInfo[2]
		}

		if len(eventDefineInfo) > 3 {
			eventDefine.Repository = eventDefineInfo[3]
		}

		if len(eventDefineInfo) > 4 {
			pipelinInt, _ := strconv.ParseInt(eventDefineInfo[4], 10, 64)
			eventDefine.Workflow = pipelinInt
		}

		if len(eventDefineInfo) > 5 {
			stageInt, _ := strconv.ParseInt(eventDefineInfo[5], 10, 64)
			eventDefine.Stage = stageInt
		}

		if len(eventDefineInfo) > 6 {
			actionInt, _ := strconv.ParseInt(eventDefineInfo[6], 10, 64)
			eventDefine.Action = actionInt
		}
	} else {
		err := eventDefine.GetEventDefinition().Where("id = ?", eventDefineId).First(eventDefine).Error
		if err != nil {
			log.Error("[event's RecordEventInfo]:error when get event define from db:", err.Error())
			return err
		}
	}

	event := new(models.Event)
	event.Definition = eventDefineId
	event.Title = eventDefine.Title
	event.Header = headerInfo
	event.Payload = payload
	event.Authorization = authInfo
	event.Type = eventDefine.Type
	event.Source = eventDefine.Source
	event.Character = eventDefine.Character
	event.Namespace = eventDefine.Namespace
	event.Repository = eventDefine.Repository
	event.Workflow = eventDefine.Workflow
	event.Stage = eventDefine.Stage
	event.Action = eventDefine.Action
	event.Sequence = sequence

	err := event.GetEvent().Save(event).Error
	if err != nil {
		log.Error("[event's RecordEventInfo]:error when save event info to db:", err.Error())
		return err
	}

	return nil
}

func SetWorkflowVarInfo(id int64, varMap map[string]interface{}) error {
	db := models.GetDB().Begin()
	err := db.Error
	if err != nil {
		log.Error("[workflowVar's SetWorkflowVarInfo]:when db.Begin():", err.Error())
		return errors.New("error when db.Begin")
	}

	err = db.Model(&models.WorkflowVar{}).Where("workflow = ?", id).Unscoped().Delete(&models.WorkflowVar{}).Error
	if err != nil {
		log.Error("[workflowVar's SetWorkflowVarInfo]:when delet var info from db:", err.Error())
		rollbackErr := db.Rollback().Error
		if rollbackErr != nil {
			log.Error("[workflowVar's SetWorkflowVarInfo]:when rollback in delet var info got err:", rollbackErr.Error())
			return errors.New("errors occur:\nerror1:" + err.Error() + "\nerror2:" + rollbackErr.Error())
		}
		return errors.New("error when delete var info from db:" + err.Error())
	}

	for key, defaultValue := range varMap {
		varSet := new(models.WorkflowVar)

		defaultValueStr, ok := defaultValue.(string)
		if !ok {
			log.Error("[workflowVar's SetWorkflowVarInfo]:error when pase default value, want a string,got:", defaultValue)
			return errors.New("var's vaule is not string")
		}

		varSet.Workflow = id
		varSet.Key = key
		varSet.Default = defaultValueStr

		err = db.Model(&models.WorkflowVar{}).Save(varSet).Error
		if err != nil {
			log.Error("[workflowVar's SetWorkflowVarInfo]:when save var info from db:", err.Error())
			rollbackErr := db.Rollback().Error
			if rollbackErr != nil {
				log.Error("[workflowVar's SetWorkflowVarInfo]:when rollback in save var info got err:", rollbackErr.Error())
				return errors.New("errors occur:\nerror1:" + err.Error() + "\nerror2:" + rollbackErr.Error())
			}
			return errors.New("error when save var info")
		}
	}

	db.Commit()
	return nil
}

func GetWorkflowVarInfo(id int64) (map[string]string, error) {
	resultMap := make(map[string]string)
	varList := make([]models.WorkflowVar, 0)

	err := new(models.WorkflowVar).GetWorkflowVar().Where("workflow = ?", id).Find(&varList).Error
	if err != nil {
		log.Error("[workflowVar's GetWorkflowVarInfo]:error when get var list from db:", err.Error())
		return nil, errors.New("error when get var info from db")
	}

	for _, varInfo := range varList {
		resultMap[varInfo.Key] = varInfo.Default
	}

	return resultMap, nil
}

func (workflowVar *WorkflowVar) GenerateNewLog(db *gorm.DB, workflowLog *models.WorkflowLog) error {
	if db == nil {
		db = models.GetDB()
		db = db.Begin()
	}

	changeLogMap := make(map[string]interface{})
	changeLogMap["user"] = "system"
	changeLogMap["time"] = time.Now().Format("2006-01-02 15:04:05")
	changeLogMap["action"] = "init data: set key:" + workflowVar.Key + " 's value to" + workflowVar.Default

	changeLogList := make([]interface{}, 0)
	changeLogList = append(changeLogList, changeLogMap)

	changeInfoBytes, _ := json.Marshal(changeLogList)

	varLog := new(models.WorkflowVarLog)
	varLog.Workflow = workflowLog.ID
	varLog.FromWorkflow = workflowLog.FromWorkflow
	varLog.Sequence = workflowLog.Sequence
	varLog.Key = workflowVar.Key
	varLog.Default = workflowVar.Default
	varLog.Vaule = varLog.Default
	varLog.ChangeLog = string(changeInfoBytes)

	err := varLog.GetWorkflowVarLog().Save(varLog).Error
	if err != nil {
		rollbackErr := db.Rollback().Error
		if rollbackErr != nil {
			log.Error("[Workflow's GenerateNewLog]:when rollback in save workflow var log:", rollbackErr.Error())
			return errors.New("errors occur:\nerror1:" + err.Error() + "\nerror2:" + rollbackErr.Error())
		}

		return err
	}

	return nil
}

func getWorkflowVarLogInfo(workflow, sequence int64, key string) (string, error) {
	varInfo := new(models.WorkflowVarLog)
	err := varInfo.GetWorkflowVarLog().Where("workflow = ?", workflow).Where("sequence = ?", sequence).Where("`key` = ?", key).First(varInfo).Error
	if err != nil {
		log.Error("[workflowVarLog's getWorkflowVarLogInfo]:get workflow var info from db error:", err.Error())
		return "", errors.New("stage's timeout is not a global value")
	}

	return varInfo.Vaule, nil
}
<|endoftext|>