<commit_before>package operator

import (
	"context"
	"fmt"
	"os"
	"time"

	"github.com/golang/glog"
	v1meta "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/client-go/kubernetes"
	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/pkg/api"
	v1api "k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/flowcontrol"

	// These should be replaced with client-go equivilents when available
	kapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
	kleaderelection "k8s.io/kubernetes/pkg/client/leaderelection"
	kresourcelock "k8s.io/kubernetes/pkg/client/leaderelection/resourcelock"

	"github.com/coreos/container-linux-update-operator/pkg/constants"
	"github.com/coreos/container-linux-update-operator/pkg/k8sutil"
)

const (
	eventReasonRebootFailed            = "RebootFailed"
	eventSourceComponent               = "update-operator"
	leaderElectionEventSourceComponent = "update-operator-leader-election"
	// agentDefaultAppName is the label value for the 'app' key that agents are
	// expected to be labeled with.
	agentDefaultAppName = "container-linux-update-agent"
	maxRebootingNodes   = 1

	leaderElectionResourceName = "container-linux-update-operator-lock"

	// Arbitrarily copied from KVO
	leaderElectionLease = 90 * time.Second
)

var (
	// justRebootedSelector is a selector for combination of annotations
	// expected to be on a node after it has completed a reboot.
	//
	// The update-operator sets constants.AnnotationOkToReboot to true to
	// trigger a reboot, and the update-agent sets
	// constants.AnnotationRebootNeeded and
	// constants.AnnotationRebootInProgress to false when it has finished.
	justRebootedSelector = fields.Set(map[string]string{
		constants.AnnotationOkToReboot:       constants.True,
		constants.AnnotationRebootNeeded:     constants.False,
		constants.AnnotationRebootInProgress: constants.False,
	}).AsSelector()

	// wantsRebootSelector is a selector for the annotation expected to be on a node when it wants to be rebooted.
	//
	// The update-agent sets constants.AnnotationRebootNeeded to true when
	// it would like to reboot, and false when it starts up.
	//
	// If constants.AnnotationRebootPaused is set to "true", the update-agent will not consider it for rebooting.
	wantsRebootSelector = fields.ParseSelectorOrDie(constants.AnnotationRebootNeeded + "==" + constants.True + "," + constants.AnnotationRebootPaused + "!=" + constants.True)

	// stillRebootingSelector is a selector for the annotation set expected to be
	// on a node when it's in the process of rebooting
	stillRebootingSelector = fields.Set(map[string]string{
		constants.AnnotationOkToReboot:       constants.True,
		constants.AnnotationRebootNeeded:     constants.True,
		constants.AnnotationRebootInProgress: constants.True,
	}).AsSelector()
)

type Kontroller struct {
	kc *kubernetes.Clientset
	nc v1core.NodeInterface
	er record.EventRecorder

	leaderElectionClient        clientset.Interface
	leaderElectionEventRecorder record.EventRecorder
	// namespace is the kubernetes namespace any resources (e.g. locks,
	// configmaps, agents) should be created and read under.
	// It will be set to the namespace the operator is running in automatically.
	namespace string
}

func New() (*Kontroller, error) {
	// set up kubernetes in-cluster client
	kc, err := k8sutil.InClusterClient()
	if err != nil {
		return nil, fmt.Errorf("error creating Kubernetes client: %v", err)
	}

	// node interface
	nc := kc.Nodes()

	// create event emitter
	broadcaster := record.NewBroadcaster()
	broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kc.Events("")})
	er := broadcaster.NewRecorder(api.Scheme, v1api.EventSource{Component: eventSourceComponent})

	leaderElectionClientConfig, err := rest.InClusterConfig()
	if err != nil {
		return nil, fmt.Errorf("error creating leader election client config: %v", err)
	}
	leaderElectionClient, err := clientset.NewForConfig(leaderElectionClientConfig)
	if err != nil {
		return nil, fmt.Errorf("error creating leader election client: %v", err)
	}

	leaderElectionBroadcaster := record.NewBroadcaster()
	leaderElectionBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{
		Interface: v1core.New(leaderElectionClient.Core().RESTClient()).Events(""),
	})
	leaderElectionEventRecorder := leaderElectionBroadcaster.NewRecorder(kapi.Scheme, v1api.EventSource{
		Component: leaderElectionEventSourceComponent,
	})

	namespace := os.Getenv("POD_NAMESPACE")
	if namespace == "" {
		return nil, fmt.Errorf("unable to determine operator namespace: please ensure POD_NAMESPACE environment variable is set")
	}

	return &Kontroller{kc, nc, er, leaderElectionClient, leaderElectionEventRecorder, namespace}, nil
}

// withLeaderElection creates a new context which is cancelled when this
// operator does not hold a lock to operate on the cluster
func (k *Kontroller) withLeaderElection() error {
	// TODO: a better id might be necessary.
	// Currently, KVO uses env.POD_NAME and the upstream controller-manager uses this.
	// Both end up having the same value in general, but Hostname is
	// more likely to have a value.
	id, err := os.Hostname()
	if err != nil {
		return err
	}

	// TODO(euank): this should not use an endpoint due to performance reasons.
	// See https://github.com/kubernetes/client-go/issues/28#issuecomment-284032220
	// Currently, every controller/operator uses an endpoint, so we are not alone
	// in this
	// Once https://github.com/kubernetes/kubernetes/pull/42666 is merged, update
	// to use that. There will be version-cross-compatibility-dragons.
	resLock := &kresourcelock.EndpointsLock{
		EndpointsMeta: v1meta.ObjectMeta{
			Namespace: k.namespace,
			Name:      leaderElectionResourceName,
		},
		Client: k.leaderElectionClient,
		LockConfig: kresourcelock.ResourceLockConfig{
			Identity:      id,
			EventRecorder: k.leaderElectionEventRecorder,
		},
	}

	waitLeading := make(chan struct{})
	go func(waitLeading chan<- struct{}) {
		// Lease values inspired by a combination of
		// https://github.com/kubernetes/kubernetes/blob/f7c07a121d2afadde7aa15b12a9d02858b30a0a9/pkg/apis/componentconfig/v1alpha1/defaults.go#L163-L174
		// and the KVO values
		// See also
		// https://github.com/kubernetes/kubernetes/blob/fc31dae165f406026142f0dd9a98cada8474682a/pkg/client/leaderelection/leaderelection.go#L17
		kleaderelection.RunOrDie(kleaderelection.LeaderElectionConfig{
			Lock:          resLock,
			LeaseDuration: leaderElectionLease,
			RenewDeadline: leaderElectionLease * 2 / 3,
			RetryPeriod:   leaderElectionLease / 3,
			Callbacks: kleaderelection.LeaderCallbacks{
				OnStartedLeading: func(stop <-chan struct{}) {
					glog.V(5).Info("started leading")
					waitLeading <- struct{}{}
				},
				OnStoppedLeading: func() {
					glog.Fatalf("leaderelection lost")
				},
			},
		})
	}(waitLeading)

	<-waitLeading
	return nil
}

func (k *Kontroller) Run(ctx context.Context, manageAgent bool, agentImageRepo string) error {
	err := k.withLeaderElection()
	if err != nil {
		return err
	}
	glog.V(5).Info("Starting run loop")

	rl := flowcontrol.NewTokenBucketRateLimiter(0.2, 1)

	// Before doing anytihng else, make sure the associated agent daemonset is
	// ready if it's our responsibility.
	if manageAgent {
		err := k.runDaemonsetUpdate(agentImageRepo)
		if err != nil {
			glog.Errorf("unable to ensure managed agents are ready: %v", err)
			return err
		}
	}

	for {
		rl.Accept()
		glog.V(4).Info("Going through a loop cycle")

		nodelist, err := k.nc.List(v1meta.ListOptions{})
		if err != nil {
			glog.Infof("Failed listing nodes %v", err)
			continue
		}

		glog.V(6).Infof("Found nodes: %+v", nodelist.Items)

		justRebootedNodes := k8sutil.FilterNodesByAnnotation(nodelist.Items, justRebootedSelector)

		if len(justRebootedNodes) > 0 {
			glog.Infof("Found %d rebooted nodes, setting annotation %q to false", len(justRebootedNodes), constants.AnnotationOkToReboot)
		}

		for _, n := range justRebootedNodes {
			glog.V(5).Infof("Setting 'ok-to-reboot=false' for %v", n.Name)
			if err := k8sutil.SetNodeAnnotations(k.nc, n.Name, map[string]string{
				constants.AnnotationOkToReboot: constants.False,
			}); err != nil {
				glog.Infof("Failed setting annotation %q on node %q to false: %v", constants.AnnotationOkToReboot, n.Name, err)
			}
		}

		nodelist, err = k.nc.List(v1meta.ListOptions{})
		if err != nil {
			glog.Infof("Failed listing nodes: %v", err)
			continue
		}

		// Verify no nodes are still in the process of rebooting to avoid rebooting N > maxRebootingNodes
		rebootingNodes := k8sutil.FilterNodesByAnnotation(nodelist.Items, stillRebootingSelector)
		glog.V(6).Infof("Found %+v rebooting nodes", rebootingNodes)
		if len(rebootingNodes) >= maxRebootingNodes {
			glog.Infof("Found %d (of max %d) rebooting nodes; waiting for completion", len(rebootingNodes), maxRebootingNodes)
			continue
		}

		rebootableNodes := k8sutil.FilterNodesByAnnotation(nodelist.Items, wantsRebootSelector)
		if len(rebootableNodes) == 0 {
			continue
		}

		remainingRebootableCount := maxRebootingNodes - len(rebootingNodes)
		// pick N of the eligible candidates to reboot
		chosenNodes := make([]*v1api.Node, 0, remainingRebootableCount)
		for i := 0; i < remainingRebootableCount && i < len(rebootableNodes); i++ {
			chosenNodes = append(chosenNodes, &rebootableNodes[i])
		}

		glog.Infof("Found %d nodes that need a reboot", len(chosenNodes))
		for _, node := range chosenNodes {
			k.markNodeRebootable(node)
		}

	}
}

func (k *Kontroller) markNodeRebootable(n *v1api.Node) {
	glog.V(4).Infof("marking %s ok to reboot", n.Name)
	if err := k8sutil.UpdateNodeRetry(k.nc, n.Name, func(node *v1api.Node) {
		if !wantsRebootSelector.Matches(fields.Set(node.Annotations)) {
			glog.Warningf("Node %v no longer wanted to a reboot while we were trying to mark it so: %v", node.Name, node.Annotations)
			return
		}

		node.Annotations[constants.AnnotationOkToReboot] = constants.True
	}); err != nil {
		glog.Infof("Failed to set annotation %q on node %q: %v", constants.AnnotationOkToReboot, n.Name, err)
		return
	}
}
<commit_msg>pkg/operator: Show logs for individual node names<commit_after>package operator

import (
	"context"
	"fmt"
	"os"
	"time"

	"github.com/golang/glog"
	v1meta "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/client-go/kubernetes"
	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/pkg/api"
	v1api "k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/flowcontrol"

	// These should be replaced with client-go equivilents when available
	kapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
	kleaderelection "k8s.io/kubernetes/pkg/client/leaderelection"
	kresourcelock "k8s.io/kubernetes/pkg/client/leaderelection/resourcelock"

	"github.com/coreos/container-linux-update-operator/pkg/constants"
	"github.com/coreos/container-linux-update-operator/pkg/k8sutil"
)

const (
	eventReasonRebootFailed            = "RebootFailed"
	eventSourceComponent               = "update-operator"
	leaderElectionEventSourceComponent = "update-operator-leader-election"
	// agentDefaultAppName is the label value for the 'app' key that agents are
	// expected to be labeled with.
	agentDefaultAppName = "container-linux-update-agent"
	maxRebootingNodes   = 1

	leaderElectionResourceName = "container-linux-update-operator-lock"

	// Arbitrarily copied from KVO
	leaderElectionLease = 90 * time.Second
)

var (
	// justRebootedSelector is a selector for combination of annotations
	// expected to be on a node after it has completed a reboot.
	//
	// The update-operator sets constants.AnnotationOkToReboot to true to
	// trigger a reboot, and the update-agent sets
	// constants.AnnotationRebootNeeded and
	// constants.AnnotationRebootInProgress to false when it has finished.
	justRebootedSelector = fields.Set(map[string]string{
		constants.AnnotationOkToReboot:       constants.True,
		constants.AnnotationRebootNeeded:     constants.False,
		constants.AnnotationRebootInProgress: constants.False,
	}).AsSelector()

	// wantsRebootSelector is a selector for the annotation expected to be on a node when it wants to be rebooted.
	//
	// The update-agent sets constants.AnnotationRebootNeeded to true when
	// it would like to reboot, and false when it starts up.
	//
	// If constants.AnnotationRebootPaused is set to "true", the update-agent will not consider it for rebooting.
	wantsRebootSelector = fields.ParseSelectorOrDie(constants.AnnotationRebootNeeded + "==" + constants.True + "," + constants.AnnotationRebootPaused + "!=" + constants.True)

	// stillRebootingSelector is a selector for the annotation set expected to be
	// on a node when it's in the process of rebooting
	stillRebootingSelector = fields.Set(map[string]string{
		constants.AnnotationOkToReboot:       constants.True,
		constants.AnnotationRebootNeeded:     constants.True,
		constants.AnnotationRebootInProgress: constants.True,
	}).AsSelector()
)

type Kontroller struct {
	kc *kubernetes.Clientset
	nc v1core.NodeInterface
	er record.EventRecorder

	leaderElectionClient        clientset.Interface
	leaderElectionEventRecorder record.EventRecorder
	// namespace is the kubernetes namespace any resources (e.g. locks,
	// configmaps, agents) should be created and read under.
	// It will be set to the namespace the operator is running in automatically.
	namespace string
}

func New() (*Kontroller, error) {
	// set up kubernetes in-cluster client
	kc, err := k8sutil.InClusterClient()
	if err != nil {
		return nil, fmt.Errorf("error creating Kubernetes client: %v", err)
	}

	// node interface
	nc := kc.Nodes()

	// create event emitter
	broadcaster := record.NewBroadcaster()
	broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kc.Events("")})
	er := broadcaster.NewRecorder(api.Scheme, v1api.EventSource{Component: eventSourceComponent})

	leaderElectionClientConfig, err := rest.InClusterConfig()
	if err != nil {
		return nil, fmt.Errorf("error creating leader election client config: %v", err)
	}
	leaderElectionClient, err := clientset.NewForConfig(leaderElectionClientConfig)
	if err != nil {
		return nil, fmt.Errorf("error creating leader election client: %v", err)
	}

	leaderElectionBroadcaster := record.NewBroadcaster()
	leaderElectionBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{
		Interface: v1core.New(leaderElectionClient.Core().RESTClient()).Events(""),
	})
	leaderElectionEventRecorder := leaderElectionBroadcaster.NewRecorder(kapi.Scheme, v1api.EventSource{
		Component: leaderElectionEventSourceComponent,
	})

	namespace := os.Getenv("POD_NAMESPACE")
	if namespace == "" {
		return nil, fmt.Errorf("unable to determine operator namespace: please ensure POD_NAMESPACE environment variable is set")
	}

	return &Kontroller{kc, nc, er, leaderElectionClient, leaderElectionEventRecorder, namespace}, nil
}

// withLeaderElection creates a new context which is cancelled when this
// operator does not hold a lock to operate on the cluster
func (k *Kontroller) withLeaderElection() error {
	// TODO: a better id might be necessary.
	// Currently, KVO uses env.POD_NAME and the upstream controller-manager uses this.
	// Both end up having the same value in general, but Hostname is
	// more likely to have a value.
	id, err := os.Hostname()
	if err != nil {
		return err
	}

	// TODO(euank): this should not use an endpoint due to performance reasons.
	// See https://github.com/kubernetes/client-go/issues/28#issuecomment-284032220
	// Currently, every controller/operator uses an endpoint, so we are not alone
	// in this
	// Once https://github.com/kubernetes/kubernetes/pull/42666 is merged, update
	// to use that. There will be version-cross-compatibility-dragons.
	resLock := &kresourcelock.EndpointsLock{
		EndpointsMeta: v1meta.ObjectMeta{
			Namespace: k.namespace,
			Name:      leaderElectionResourceName,
		},
		Client: k.leaderElectionClient,
		LockConfig: kresourcelock.ResourceLockConfig{
			Identity:      id,
			EventRecorder: k.leaderElectionEventRecorder,
		},
	}

	waitLeading := make(chan struct{})
	go func(waitLeading chan<- struct{}) {
		// Lease values inspired by a combination of
		// https://github.com/kubernetes/kubernetes/blob/f7c07a121d2afadde7aa15b12a9d02858b30a0a9/pkg/apis/componentconfig/v1alpha1/defaults.go#L163-L174
		// and the KVO values
		// See also
		// https://github.com/kubernetes/kubernetes/blob/fc31dae165f406026142f0dd9a98cada8474682a/pkg/client/leaderelection/leaderelection.go#L17
		kleaderelection.RunOrDie(kleaderelection.LeaderElectionConfig{
			Lock:          resLock,
			LeaseDuration: leaderElectionLease,
			RenewDeadline: leaderElectionLease * 2 / 3,
			RetryPeriod:   leaderElectionLease / 3,
			Callbacks: kleaderelection.LeaderCallbacks{
				OnStartedLeading: func(stop <-chan struct{}) {
					glog.V(5).Info("started leading")
					waitLeading <- struct{}{}
				},
				OnStoppedLeading: func() {
					glog.Fatalf("leaderelection lost")
				},
			},
		})
	}(waitLeading)

	<-waitLeading
	return nil
}

func (k *Kontroller) Run(ctx context.Context, manageAgent bool, agentImageRepo string) error {
	err := k.withLeaderElection()
	if err != nil {
		return err
	}
	glog.V(5).Info("Starting run loop")

	rl := flowcontrol.NewTokenBucketRateLimiter(0.2, 1)

	// Before doing anytihng else, make sure the associated agent daemonset is
	// ready if it's our responsibility.
	if manageAgent {
		err := k.runDaemonsetUpdate(agentImageRepo)
		if err != nil {
			glog.Errorf("unable to ensure managed agents are ready: %v", err)
			return err
		}
	}

	for {
		rl.Accept()
		glog.V(4).Info("Going through a loop cycle")

		nodelist, err := k.nc.List(v1meta.ListOptions{})
		if err != nil {
			glog.Infof("Failed listing nodes %v", err)
			continue
		}

		glog.V(6).Infof("Found nodes: %+v", nodelist.Items)

		justRebootedNodes := k8sutil.FilterNodesByAnnotation(nodelist.Items, justRebootedSelector)

		if len(justRebootedNodes) > 0 {
			glog.Infof("Found %d rebooted nodes, setting annotation %q to false", len(justRebootedNodes), constants.AnnotationOkToReboot)
		}

		for _, n := range justRebootedNodes {
			glog.Infof("Setting 'ok-to-reboot=false' for %q", n.Name)
			if err := k8sutil.SetNodeAnnotations(k.nc, n.Name, map[string]string{
				constants.AnnotationOkToReboot: constants.False,
			}); err != nil {
				glog.Infof("Failed setting annotation %q on node %q to false: %v", constants.AnnotationOkToReboot, n.Name, err)
			}
		}

		nodelist, err = k.nc.List(v1meta.ListOptions{})
		if err != nil {
			glog.Infof("Failed listing nodes: %v", err)
			continue
		}

		// Verify no nodes are still in the process of rebooting to avoid rebooting N > maxRebootingNodes
		rebootingNodes := k8sutil.FilterNodesByAnnotation(nodelist.Items, stillRebootingSelector)
		glog.V(6).Infof("Found %+v rebooting nodes", rebootingNodes)
		if len(rebootingNodes) >= maxRebootingNodes {
			glog.Infof("Found %d (of max %d) rebooting nodes; waiting for completion", len(rebootingNodes), maxRebootingNodes)
			for _, n := range rebootingNodes {
				glog.Infof("Found node %q still rebooting, waiting", n.Name)
			}
			continue
		}

		rebootableNodes := k8sutil.FilterNodesByAnnotation(nodelist.Items, wantsRebootSelector)
		if len(rebootableNodes) == 0 {
			continue
		}

		remainingRebootableCount := maxRebootingNodes - len(rebootingNodes)
		// pick N of the eligible candidates to reboot
		chosenNodes := make([]*v1api.Node, 0, remainingRebootableCount)
		for i := 0; i < remainingRebootableCount && i < len(rebootableNodes); i++ {
			chosenNodes = append(chosenNodes, &rebootableNodes[i])
		}

		glog.Infof("Found %d nodes that need a reboot", len(chosenNodes))
		for _, node := range chosenNodes {
			k.markNodeRebootable(node)
		}

	}
}

func (k *Kontroller) markNodeRebootable(n *v1api.Node) {
	glog.Infof("marking %q ok to reboot", n.Name)
	if err := k8sutil.UpdateNodeRetry(k.nc, n.Name, func(node *v1api.Node) {
		if !wantsRebootSelector.Matches(fields.Set(node.Annotations)) {
			glog.Warningf("Node %v no longer wanted to reboot while we were trying to mark it so: %v", node.Name, node.Annotations)
			return
		}

		node.Annotations[constants.AnnotationOkToReboot] = constants.True
	}); err != nil {
		glog.Infof("Failed to set annotation %q on node %q: %v", constants.AnnotationOkToReboot, n.Name, err)
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/andyxning/eventarbiter/cmd/eventarbiter/conf"
	"github.com/andyxning/eventarbiter/cmd/eventarbiter/signal"
	"github.com/andyxning/eventarbiter/handler"
	_ "github.com/andyxning/eventarbiter/handler/node"
	_ "github.com/andyxning/eventarbiter/handler/pod"
	"github.com/andyxning/eventarbiter/models"
	"github.com/andyxning/eventarbiter/sink/callback"
	"github.com/andyxning/eventarbiter/sink/stdout"
	"github.com/andyxning/eventarbiter/source/kubernetes"
	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	"os"
	"strings"
	"sync"
	"time"
)

func mergeCmdFlags(conf *conf.Config) {
	var versionFlag bool

	flag.BoolVar(&versionFlag, "version", false, "version info")
	flag.UintVar(&conf.MaxProcs, "max_procs", conf.MaxProcs,
		"max cpu number arbiter will use. Can not exceeds the number of max cpu cores.")
	flag.DurationVar(&conf.LogFlushFrequency, "log_flush_frequency", conf.LogFlushFrequency,
		"Maximum number of seconds between glog flushes")

	flag.Var(&conf.Source, "source", "source to read event from. Currently only support Kubernetes")
	flag.Var(&conf.Sinks, "sink", "sink(s) that receive event alert")
	flag.Var(&conf.FilteredAlertEventReasons, "event_filter",
		"event alert reasons that should be filtered. Seperated by comma")
	flag.Var(&conf.Environment, "environment", "comma seperated key-value pairs. This map will be"+
		" as the 'Environment' field in callback http request body along side event alert.")

	flag.Parse()

	if versionFlag {
		fmt.Printf("version: %s\ncommit: %s\n", conf.Version, conf.CommitHash)
		os.Exit(0)
	}
}

func flushLogPeriodically(duration time.Duration) {
	glog.Infof("set glog flush interval to %v", duration)

	go func() {
		flusher := time.NewTicker(duration)
		for {
			select {
			case <-flusher.C:
				glog.Flush()
			}
		}
	}()
}

func filterAlertEvent() {
	for _, filteredEvent := range conf.Conf.FilteredAlertEventReasons.Value {
		glog.Infof("filter out %s event alert", filteredEvent)
		delete(handler.EventAlertHandlers, filteredEvent)
	}

	var eventAlertReasonEnabled []string
	for eventAlertReason := range handler.EventAlertHandlers {
		eventAlertReasonEnabled = append(eventAlertReasonEnabled, eventAlertReason)
	}

	glog.Infof("enable event alert for %s", strings.Join(eventAlertReasonEnabled, ","))

	var eventReasons []string
	for eventReason := range handler.EventReasons {
		eventReasons = append(eventReasons, eventReason)
	}

	glog.V(2).Infof("listening Kubernetes event: %s", strings.Join(eventReasons, ","))
}

func StartMain(sinks []models.Sink, eventChan <-chan *api.Event, stopWG *sync.WaitGroup) {
	go func() {
		for event := range eventChan {
			if _, exists := handler.EventReasons[event.Reason]; exists {
				glog.Infof("got %s. %#v", event.Reason, event)

				for alertEventReason, eventHandler := range handler.EventAlertHandlers {
					glog.V(2).Infof("sending event to %s", alertEventReason)
					stopWG.Add(1)

					// range variable eventHandler can not be captured by func literal
					go func(eventHandler models.EventHandler) {
						defer stopWG.Done()

						eventHandler.HandleEvent(sinks, event)
					}(eventHandler)
				}
			}
		}
	}()
}

func main() {
	mergeCmdFlags(&conf.Conf)

	err := conf.Conf.Validate()
	if err != nil {
		glog.Exit("arbiter encounters something that can not be handled internally. Exit")
	}

	conf.Conf.SetMaxProcs()

	flushLogPeriodically(conf.Conf.LogFlushFrequency)

	filterAlertEvent()

	var sinks []models.Sink
	for _, sinker := range conf.Conf.Sinks {
		if sinker.Key == "stdout" {
			sinks = append(sinks, stdout.NewStdout())
			glog.Info("start json sink")

			continue
		}

		if sinker.Key == "callback" {
			sinks = append(sinks, callback.NewCallback(sinker.Val.String()))
			glog.Infof("start callback sink on %s", sinker.Val.String())

			continue
		}

		glog.Exit("unrecognized sink %q. only support stdout and callback", sinker.Key)
	}

	source := kubernetes.MustNewKubernetes(&conf.Conf.Source.Val)

	eventChan := make(chan *api.Event, conf.Conf.EventChanLength)
	source.Start(eventChan)

	stopWG := sync.WaitGroup{}
	StartMain(sinks, eventChan, &stopWG)

	select {
	case exitSignal := <-signal.ExitChan:
		glog.Infof("receive signal %s", exitSignal)
		glog.Flush()

		source.Stop()

		glog.Info("stop sink")
		for {
			if len(eventChan) != 0 {
				time.Sleep(200 * time.Millisecond)
				glog.Infof("waiting for event chan to be empty")
				continue
			}

			break
		}
		stopWG.Wait()
		glog.Info("sink stopped")

		glog.Infoln("stop gracefully")
		glog.Flush()
		os.Exit(0)
	}
}
<commit_msg>make less log<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/andyxning/eventarbiter/cmd/eventarbiter/conf"
	"github.com/andyxning/eventarbiter/cmd/eventarbiter/signal"
	"github.com/andyxning/eventarbiter/handler"
	_ "github.com/andyxning/eventarbiter/handler/node"
	_ "github.com/andyxning/eventarbiter/handler/pod"
	"github.com/andyxning/eventarbiter/models"
	"github.com/andyxning/eventarbiter/sink/callback"
	"github.com/andyxning/eventarbiter/sink/stdout"
	"github.com/andyxning/eventarbiter/source/kubernetes"
	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	"os"
	"strings"
	"sync"
	"time"
)

func mergeCmdFlags(conf *conf.Config) {
	var versionFlag bool

	flag.BoolVar(&versionFlag, "version", false, "version info")
	flag.UintVar(&conf.MaxProcs, "max_procs", conf.MaxProcs,
		"max cpu number arbiter will use. Can not exceeds the number of max cpu cores.")
	flag.DurationVar(&conf.LogFlushFrequency, "log_flush_frequency", conf.LogFlushFrequency,
		"Maximum number of seconds between glog flushes")

	flag.Var(&conf.Source, "source", "source to read event from. Currently only support Kubernetes")
	flag.Var(&conf.Sinks, "sink", "sink(s) that receive event alert")
	flag.Var(&conf.FilteredAlertEventReasons, "event_filter",
		"event alert reasons that should be filtered. Seperated by comma")
	flag.Var(&conf.Environment, "environment", "comma seperated key-value pairs. This map will be"+
		" as the 'Environment' field in callback http request body along side event alert.")

	flag.Parse()

	if versionFlag {
		fmt.Printf("version: %s\ncommit: %s\n", conf.Version, conf.CommitHash)
		os.Exit(0)
	}
}

func flushLogPeriodically(duration time.Duration) {
	glog.Infof("set glog flush interval to %v", duration)

	go func() {
		flusher := time.NewTicker(duration)
		for {
			select {
			case <-flusher.C:
				glog.Flush()
			}
		}
	}()
}

func filterAlertEvent() {
	for _, filteredEvent := range conf.Conf.FilteredAlertEventReasons.Value {
		glog.Infof("filter out %s event alert", filteredEvent)
		delete(handler.EventAlertHandlers, filteredEvent)
	}

	var eventAlertReasonEnabled []string
	for eventAlertReason := range handler.EventAlertHandlers {
		eventAlertReasonEnabled = append(eventAlertReasonEnabled, eventAlertReason)
	}

	glog.Infof("enable event alert for %s", strings.Join(eventAlertReasonEnabled, ","))

	var eventReasons []string
	for eventReason := range handler.EventReasons {
		eventReasons = append(eventReasons, eventReason)
	}

	glog.V(2).Infof("listening Kubernetes event: %s", strings.Join(eventReasons, ","))
}

func StartMain(sinks []models.Sink, eventChan <-chan *api.Event, stopWG *sync.WaitGroup) {
	go func() {
		for event := range eventChan {
			if _, exists := handler.EventReasons[event.Reason]; exists {
				glog.Infof("got %s. %#v", event.Reason, event)

				for alertEventReason, eventHandler := range handler.EventAlertHandlers {
					glog.V(3).Infof("sending event to %s", alertEventReason)
					stopWG.Add(1)

					// range variable eventHandler can not be captured by func literal
					go func(eventHandler models.EventHandler) {
						defer stopWG.Done()

						eventHandler.HandleEvent(sinks, event)
					}(eventHandler)
				}
			}
		}
	}()
}

func main() {
	mergeCmdFlags(&conf.Conf)

	err := conf.Conf.Validate()
	if err != nil {
		glog.Exit("arbiter encounters something that can not be handled internally. Exit")
	}

	conf.Conf.SetMaxProcs()

	flushLogPeriodically(conf.Conf.LogFlushFrequency)

	filterAlertEvent()

	var sinks []models.Sink
	for _, sinker := range conf.Conf.Sinks {
		if sinker.Key == "stdout" {
			sinks = append(sinks, stdout.NewStdout())
			glog.Info("start json sink")

			continue
		}

		if sinker.Key == "callback" {
			sinks = append(sinks, callback.NewCallback(sinker.Val.String()))
			glog.Infof("start callback sink on %s", sinker.Val.String())

			continue
		}

		glog.Exit("unrecognized sink %q. only support stdout and callback", sinker.Key)
	}

	source := kubernetes.MustNewKubernetes(&conf.Conf.Source.Val)

	eventChan := make(chan *api.Event, conf.Conf.EventChanLength)
	source.Start(eventChan)

	stopWG := sync.WaitGroup{}
	StartMain(sinks, eventChan, &stopWG)

	select {
	case exitSignal := <-signal.ExitChan:
		glog.Infof("receive signal %s", exitSignal)
		glog.Flush()

		source.Stop()

		glog.Info("stop sink")
		for {
			if len(eventChan) != 0 {
				time.Sleep(200 * time.Millisecond)
				glog.Infof("waiting for event chan to be empty")
				continue
			}

			break
		}
		stopWG.Wait()
		glog.Info("sink stopped")

		glog.Infoln("stop gracefully")
		glog.Flush()
		os.Exit(0)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"time"

	"github.com/go-ggz/ggz/config"
	"github.com/go-ggz/ggz/version"

	"github.com/joho/godotenv"
	_ "github.com/joho/godotenv/autoload"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"gopkg.in/urfave/cli.v2"
)

func setupLogging() {
	zerolog.SetGlobalLevel(zerolog.InfoLevel)
	if config.Server.Debug {
		zerolog.SetGlobalLevel(zerolog.DebugLevel)
	}

	if config.Logs.Pretty {
		log.Logger = log.Output(
			zerolog.ConsoleWriter{
				Out:     os.Stderr,
				NoColor: !config.Logs.Color,
			},
		)
	}
}

func main() {
	if env := os.Getenv("GGZ_ENV_FILE"); env != "" {
		if err := godotenv.Load(env); err != nil {
			log.Fatal().Err(err).Msg("Cannot start load config from env")
		}
	}

	app := &cli.App{
		Name:      "gzz redirect",
		Usage:     "redirect service",
		Copyright: "Copyright (c) 2018 Bo-Yi Wu",
		Version:   version.PrintCLIVersion(),
		Compiled:  time.Now(),
		Authors: []*cli.Author{
			{
				Name:  "Bo-Yi Wu",
				Email: "appleboy.tw@gmail.com",
			},
		},

		Flags: []cli.Flag{
			&cli.BoolFlag{
				Name:        "debug",
				Value:       true,
				Usage:       "Activate debug information",
				EnvVars:     []string{"GGZ_SERVER_DEBUG"},
				Destination: &config.Server.Debug,
			},
			&cli.BoolFlag{
				Name:        "color",
				Value:       true,
				Usage:       "Enable pprof debugging server",
				EnvVars:     []string{"GGZ_LOGS_COLOR"},
				Destination: &config.Logs.Color,
			},
			&cli.BoolFlag{
				Name:        "pretty",
				Value:       true,
				Usage:       "Enable pprof debugging server",
				EnvVars:     []string{"GGZ_LOGS_PRETTY"},
				Destination: &config.Logs.Pretty,
			},
		},

		Before: func(c *cli.Context) error {
			setupLogging()

			return nil
		},

		Commands: []*cli.Command{
			Server(),
			Ping(),
		},
	}

	cli.HelpFlag = &cli.BoolFlag{
		Name:    "help",
		Aliases: []string{"h"},
		Usage:   "Show the help, so what you see now",
	}

	cli.VersionFlag = &cli.BoolFlag{
		Name:    "version",
		Aliases: []string{"v"},
		Usage:   "Print the current version of that tool",
	}

	if err := app.Run(os.Args); err != nil {
		os.Exit(1)
	}
}
<commit_msg>feat: support log level for redirect<commit_after>package main

import (
	"os"
	"strings"
	"time"

	"github.com/go-ggz/ggz/config"
	"github.com/go-ggz/ggz/version"

	"github.com/joho/godotenv"
	_ "github.com/joho/godotenv/autoload"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"gopkg.in/urfave/cli.v2"
)

func setupLogging() {
	switch strings.ToLower(config.Logs.Level) {
	case "panic":
		zerolog.SetGlobalLevel(zerolog.PanicLevel)
	case "fatal":
		zerolog.SetGlobalLevel(zerolog.FatalLevel)
	case "error":
		zerolog.SetGlobalLevel(zerolog.ErrorLevel)
	case "warn":
		zerolog.SetGlobalLevel(zerolog.WarnLevel)
	case "info":
		zerolog.SetGlobalLevel(zerolog.InfoLevel)
	case "debug":
		zerolog.SetGlobalLevel(zerolog.DebugLevel)
	default:
		zerolog.SetGlobalLevel(zerolog.InfoLevel)
	}

	if config.Logs.Pretty {
		log.Logger = log.Output(
			zerolog.ConsoleWriter{
				Out:     os.Stderr,
				NoColor: !config.Logs.Color,
			},
		)
	}
}

func main() {
	if env := os.Getenv("GGZ_ENV_FILE"); env != "" {
		if err := godotenv.Load(env); err != nil {
			log.Fatal().Err(err).Msg("Cannot start load config from env")
		}
	}

	app := &cli.App{
		Name:      "gzz redirect",
		Usage:     "redirect service",
		Copyright: "Copyright (c) 2018 Bo-Yi Wu",
		Version:   version.PrintCLIVersion(),
		Compiled:  time.Now(),
		Authors: []*cli.Author{
			{
				Name:  "Bo-Yi Wu",
				Email: "appleboy.tw@gmail.com",
			},
		},

		Flags: []cli.Flag{
			&cli.BoolFlag{
				Name:        "debug",
				Value:       true,
				Usage:       "Activate debug information",
				EnvVars:     []string{"GGZ_SERVER_DEBUG"},
				Destination: &config.Server.Debug,
			},
			&cli.BoolFlag{
				Name:        "log-color",
				Value:       true,
				Usage:       "enable colored logging",
				EnvVars:     []string{"GGZ_LOGS_COLOR"},
				Destination: &config.Logs.Color,
			},
			&cli.BoolFlag{
				Name:        "log-pretty",
				Value:       true,
				Usage:       "enable pretty logging",
				EnvVars:     []string{"GGZ_LOGS_PRETTY"},
				Destination: &config.Logs.Pretty,
			},
			&cli.StringFlag{
				Name:        "log-level",
				Value:       "info",
				Usage:       "set logging level",
				EnvVars:     []string{"GGZ_LOGS_LEVEL"},
				Destination: &config.Logs.Level,
			},
		},

		Before: func(c *cli.Context) error {
			setupLogging()

			return nil
		},

		Commands: []*cli.Command{
			Server(),
			Ping(),
		},
	}

	cli.HelpFlag = &cli.BoolFlag{
		Name:    "help",
		Aliases: []string{"h"},
		Usage:   "Show the help, so what you see now",
	}

	cli.VersionFlag = &cli.BoolFlag{
		Name:    "version",
		Aliases: []string{"v"},
		Usage:   "Print the current version of that tool",
	}

	if err := app.Run(os.Args); err != nil {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// This package contains data structures that hold all the information
// required by the alignment algorithm, and procedures for serializing
// and deserializing that information.
package alignmentprofile

import (
	a "github.com/hivdb/nucamino/types/amino"
	"sort"
)

type Gene string
type PositionalIndelScores map[int]([2]int)
type GenePositionalIndelScores map[Gene]PositionalIndelScores
type ReferenceSeqs map[Gene][]a.AminoAcid

// This stores the all the information needed to align a sequence to a
// reference: reference sequences, alignment parameters, and
// positional indel scores.
type AlignmentProfile struct {
	StopCodonPenalty         int
	GapOpeningPenalty        int
	GapExtensionPenalty      int
	IndelCodonOpeningBonus   int
	IndelCodonExtensionBonus int
	GeneIndelScores          GenePositionalIndelScores
	ReferenceSequences       ReferenceSeqs
}

// An array of all the genes supported by this alignment profile.
func (profile AlignmentProfile) Genes() []Gene {
	var genes = make([]Gene, 0, len(profile.ReferenceSequences))
	for gene, _ := range profile.ReferenceSequences {
		genes = append(genes, gene)
	}
	return genes
}

func (profile AlignmentProfile) rawIndelScores() map[string][]rawIndelScore {
	result := make(map[string][]rawIndelScore)
	for gene, positionalIndelScores := range profile.GeneIndelScores {
		rawIndelScores := make([]rawIndelScore, 0, len(positionalIndelScores))
		for posKey, scores := range positionalIndelScores {
			var pos int
			var kind string
			// Interpret the position key in the positional indel
			// scores map: the magnitude is the position, negative
			// indicates deletion.
			if posKey >= 0 {
				pos = posKey
				kind = "ins"
			} else {
				pos = -posKey
				kind = "del"
			}
			rawScore := rawIndelScore{
				Kind:     kind,
				Position: pos,
				Open:     scores[0],
				Extend:   scores[1],
			}
			rawIndelScores = append(rawIndelScores, rawScore)
		}
		sort.Sort(byPositionAndKind(rawIndelScores))
		result[string(gene)] = rawIndelScores
	}
	return result
}

func (profile AlignmentProfile) asRaw() rawAlignmentProfile {
	var raw rawAlignmentProfile
	raw.StopCodonPenalty = profile.StopCodonPenalty
	raw.GapOpeningPenalty = profile.GapOpeningPenalty
	raw.GapExtensionPenalty = profile.GapExtensionPenalty
	raw.IndelCodonOpeningBonus = profile.IndelCodonOpeningBonus
	raw.IndelCodonExtensionBonus = profile.IndelCodonExtensionBonus

	raw.ReferenceSequences = make(map[string]string)
	for gene, aaSeq := range profile.ReferenceSequences {
		raw.ReferenceSequences[string(gene)] = a.WriteString(aaSeq)
	}

	if profile.GeneIndelScores != nil {
		raw.RawIndelScores = profile.rawIndelScores()
	}
	return raw
}
<commit_msg>Restore a utility method on AlignmentProfile<commit_after>// This package contains data structures that hold all the information
// required by the alignment algorithm, and procedures for serializing
// and deserializing that information.
package alignmentprofile

import (
	a "github.com/hivdb/nucamino/types/amino"
	"sort"
)

type Gene string
type PositionalIndelScores map[int]([2]int)
type GenePositionalIndelScores map[Gene]PositionalIndelScores
type ReferenceSeqs map[Gene][]a.AminoAcid

// This stores the all the information needed to align a sequence to a
// reference: reference sequences, alignment parameters, and
// positional indel scores.
type AlignmentProfile struct {
	StopCodonPenalty         int
	GapOpeningPenalty        int
	GapExtensionPenalty      int
	IndelCodonOpeningBonus   int
	IndelCodonExtensionBonus int
	GeneIndelScores          GenePositionalIndelScores
	ReferenceSequences       ReferenceSeqs
}

// An array of all the genes supported by this alignment profile.
func (profile AlignmentProfile) Genes() []Gene {
	var genes = make([]Gene, 0, len(profile.ReferenceSequences))
	for gene, _ := range profile.ReferenceSequences {
		genes = append(genes, gene)
	}
	return genes
}

func (profile AlignmentProfile) rawIndelScores() map[string][]rawIndelScore {
	result := make(map[string][]rawIndelScore)
	for gene, positionalIndelScores := range profile.GeneIndelScores {
		rawIndelScores := make([]rawIndelScore, 0, len(positionalIndelScores))
		for posKey, scores := range positionalIndelScores {
			var pos int
			var kind string
			// Interpret the position key in the positional indel
			// scores map: the magnitude is the position, negative
			// indicates deletion.
			if posKey >= 0 {
				pos = posKey
				kind = "ins"
			} else {
				pos = -posKey
				kind = "del"
			}
			rawScore := rawIndelScore{
				Kind:     kind,
				Position: pos,
				Open:     scores[0],
				Extend:   scores[1],
			}
			rawIndelScores = append(rawIndelScores, rawScore)
		}
		sort.Sort(byPositionAndKind(rawIndelScores))
		result[string(gene)] = rawIndelScores
	}
	return result
}

func (profile AlignmentProfile) asRaw() rawAlignmentProfile {
	var raw rawAlignmentProfile
	raw.StopCodonPenalty = profile.StopCodonPenalty
	raw.GapOpeningPenalty = profile.GapOpeningPenalty
	raw.GapExtensionPenalty = profile.GapExtensionPenalty
	raw.IndelCodonOpeningBonus = profile.IndelCodonOpeningBonus
	raw.IndelCodonExtensionBonus = profile.IndelCodonExtensionBonus

	raw.ReferenceSequences = make(map[string]string)
	for gene, aaSeq := range profile.ReferenceSequences {
		raw.ReferenceSequences[string(gene)] = a.WriteString(aaSeq)
	}

	if profile.GeneIndelScores != nil {
		raw.RawIndelScores = profile.rawIndelScores()
	}
	return raw
}

// Retrieve the positional indel scores for a Gene.
func (profile *AlignmentProfile) PositionalIndelScoresFor(g Gene) (PositionalIndelScores, bool) {
	scores, found := profile.GeneIndelScores[g]
	return scores, found
}
<|endoftext|>


<commit_before>package realtime

import (
	"time"

	"github.com/cozy/cozy-stack/pkg/prefixer"
	"github.com/sirupsen/logrus"
)

type log struct {
	Time    time.Time              `json:"time"`
	Message string                 `json:"message"`
	Level   string                 `json:"level"`
	Data    map[string]interface{} `json:"data"`

	id      string
	docType string
}

func (l *log) DocType() string   { return l.docType }
func (l *log) ID() string        { return l.id }
func (l *log) Rev() string       { return "" }
func (l *log) SetID(id string)   {}
func (l *log) SetRev(rev string) {}

type logHook struct {
	Hub
	db      prefixer.Prefixer
	docType string
	docID   string
}

// LogHook creates a hook that transmits logs through redis pubsub
// messaging.
func LogHook(db prefixer.Prefixer, hub Hub, parentDocType, documentID string) logrus.Hook {
	return &logHook{
		Hub:     hub,
		db:      db,
		docType: parentDocType + ".logs",
		docID:   documentID,
	}
}

func (r *logHook) Levels() []logrus.Level {
	return []logrus.Level{
		logrus.DebugLevel,
		logrus.InfoLevel,
		logrus.WarnLevel,
		logrus.ErrorLevel,
		logrus.FatalLevel,
		logrus.PanicLevel,
	}
}

func (r *logHook) Fire(entry *logrus.Entry) error {
	doc := &log{
		Time:    entry.Time,
		Message: entry.Message,
		Level:   entry.Level.String(),
		Data:    entry.Data,
		docType: r.docType,
		id:      r.docID,
	}
	r.Hub.Publish(r.db, EventCreate, doc, nil)
	return nil
}
<commit_msg>Fix realtime events by settings log id on an _id field<commit_after>package realtime

import (
	"time"

	"github.com/cozy/cozy-stack/pkg/prefixer"
	"github.com/sirupsen/logrus"
)

type log struct {
	LogID   string                 `json:"_id"`
	Time    time.Time              `json:"time"`
	Message string                 `json:"message"`
	Level   string                 `json:"level"`
	Data    map[string]interface{} `json:"data"`

	docType string
}

func (l *log) DocType() string   { return l.docType }
func (l *log) ID() string        { return l.LogID }
func (l *log) Rev() string       { return "" }
func (l *log) SetID(id string)   { l.LogID = id }
func (l *log) SetRev(rev string) {}

type logHook struct {
	Hub
	db      prefixer.Prefixer
	docType string
	docID   string
}

// LogHook creates a hook that transmits logs through redis pubsub
// messaging.
func LogHook(db prefixer.Prefixer, hub Hub, parentDocType, documentID string) logrus.Hook {
	return &logHook{
		Hub:     hub,
		db:      db,
		docType: parentDocType + ".logs",
		docID:   documentID,
	}
}

func (r *logHook) Levels() []logrus.Level {
	return []logrus.Level{
		logrus.DebugLevel,
		logrus.InfoLevel,
		logrus.WarnLevel,
		logrus.ErrorLevel,
		logrus.FatalLevel,
		logrus.PanicLevel,
	}
}

func (r *logHook) Fire(entry *logrus.Entry) error {
	doc := &log{
		LogID:   r.docID,
		Time:    entry.Time,
		Message: entry.Message,
		Level:   entry.Level.String(),
		Data:    entry.Data,
		docType: r.docType,
	}
	r.Hub.Publish(r.db, EventCreate, doc, nil)
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"k8s.io/ingress-gce/pkg/controller"
	"k8s.io/ingress-gce/pkg/flags"
)

func RunHTTPServer(lbc *controller.LoadBalancerController) {
	http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
		if err := lbc.CloudClusterManager.IsHealthy(); err != nil {
			w.WriteHeader(500)
			w.Write([]byte(fmt.Sprintf("Cluster unhealthy: %v", err)))
			return
		}
		w.WriteHeader(200)
		w.Write([]byte("ok"))
	})
	http.Handle("/metrics", promhttp.Handler())
	http.HandleFunc("/delete-all-and-quit", func(w http.ResponseWriter, r *http.Request) {
		// TODO: Retry failures during shutdown.
		lbc.Stop(true)
	})

	glog.V(0).Infof("Running http server on :%v", flags.F.HealthzPort)
	glog.Fatal(http.ListenAndServe(fmt.Sprintf(":%v", flags.F.HealthzPort), nil))
}

func RunSIGTERMHandler(lbc *controller.LoadBalancerController, deleteAll bool) {
	// Multiple SIGTERMs will get dropped
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGTERM)
	glog.V(0).Infof("SIGTERM handler registered")
	<-signalChan
	glog.Infof("Received SIGTERM, shutting down")

	// TODO: Better retries than relying on restartPolicy.
	exitCode := 0
	if err := lbc.Stop(deleteAll); err != nil {
		glog.Infof("Error during shutdown %v", err)
		exitCode = 1
	}
	glog.Infof("Exiting with %v", exitCode)
	os.Exit(exitCode)
}
<commit_msg>Add endpoint for changing verbosity<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"flag"
	"fmt"
	"html/template"
	"net/http"
	"os"
	"os/signal"
	"syscall"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"k8s.io/ingress-gce/pkg/controller"
	"k8s.io/ingress-gce/pkg/flags"
	"k8s.io/ingress-gce/pkg/version"
)

func RunHTTPServer(lbc *controller.LoadBalancerController) {
	http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
		if err := lbc.CloudClusterManager.IsHealthy(); err != nil {
			w.WriteHeader(500)
			w.Write([]byte(fmt.Sprintf("Cluster unhealthy: %v", err)))
			return
		}
		w.WriteHeader(200)
		w.Write([]byte("ok"))
	})
	http.Handle("/metrics", promhttp.Handler())
	http.HandleFunc("/flag", flagHandler)
	http.HandleFunc("/delete-all-and-quit", func(w http.ResponseWriter, r *http.Request) {
		// TODO: Retry failures during shutdown.
		lbc.Stop(true)
	})

	glog.V(0).Infof("Running http server on :%v", flags.F.HealthzPort)
	glog.Fatal(http.ListenAndServe(fmt.Sprintf(":%v", flags.F.HealthzPort), nil))
}

func RunSIGTERMHandler(lbc *controller.LoadBalancerController, deleteAll bool) {
	// Multiple SIGTERMs will get dropped
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGTERM)
	glog.V(0).Infof("SIGTERM handler registered")
	<-signalChan
	glog.Infof("Received SIGTERM, shutting down")

	// TODO: Better retries than relying on restartPolicy.
	exitCode := 0
	if err := lbc.Stop(deleteAll); err != nil {
		glog.Infof("Error during shutdown %v", err)
		exitCode = 1
	}
	glog.Infof("Exiting with %v", exitCode)
	os.Exit(exitCode)
}

func flagHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		getFlagPage(w, r)
		return
	case "PUT":
		putFlag(w, r)
	default:
		w.WriteHeader(http.StatusBadRequest)
		return
	}
}

func putFlag(w http.ResponseWriter, r *http.Request) {
	for key, values := range r.URL.Query() {
		if len(values) != 1 {
			glog.Warningln("No query string params provided")
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		v := values[0]
		switch key {
		case "v":
			setVerbosity(v)
		default:
			glog.Warningf("Unrecognized key: %q", key)
			w.WriteHeader(http.StatusBadRequest)
			return
		}

		w.WriteHeader(http.StatusOK)
	}
}

func setVerbosity(v string) {
	flag.Lookup("v").Value.Set(v)
	glog.V(0).Infof("Setting verbosity level to %q", v)
}

func getFlagPage(w http.ResponseWriter, r *http.Request) {
	s := struct {
		Version   string
		Verbosity string
	}{
		Version:   version.Version,
		Verbosity: flag.Lookup("v").Value.String(),
	}
	flagPageTemplate.Execute(w, s)
}

var flagPageTemplate = template.Must(template.New("").Parse(`GCE Ingress Controller "GLBC"
Version: {{.Version}}

Verbosity ('v'): {{.Verbosity}}
`))
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"strings"
	"sync/atomic"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/api/v1"
	"github.com/funkygao/gafka/ctx"
)

var (
	c        int
	addr     string
	n        int64
	msgSize  int
	debug    bool
	async    bool
	step     int64
	appid    string
	topic    string
	key      string
	workerId string
	sleep    time.Duration
	tag      string
)

func init() {
	ip, _ := ctx.LocalIP()
	flag.IntVar(&c, "c", 10, "client concurrency")
	flag.IntVar(&msgSize, "sz", 100, "msg size")
	flag.StringVar(&appid, "appid", "app1", "app id")
	flag.DurationVar(&sleep, "sleep", 0, "sleep between pub")
	flag.StringVar(&addr, "h", fmt.Sprintf("%s:9191", ip.String()), "pub http addr")
	flag.Int64Var(&step, "step", 1, "display progress step")
	flag.StringVar(&key, "key", "", "message key")
	flag.BoolVar(&debug, "debug", false, "debug")
	flag.BoolVar(&async, "async", false, "async pub")
	flag.StringVar(&tag, "tag", "", "add tag to each message")
	flag.StringVar(&topic, "t", "foobar", "topic to pub")
	flag.StringVar(&workerId, "id", "1", "worker id")
	flag.Parse()
}

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	for i := 0; i < c; i++ {
		go pubGatewayLoop(i)
	}

	select {}
}

func pubGatewayLoop(seq int) {
	cf := api.DefaultConfig(appid, "mysecret")
	cf.Pub.Endpoint = addr
	cf.Debug = debug
	client := api.NewClient(cf)

	var (
		err error
		msg string
		no  int64
		sz  int
	)

	var opt api.PubOption
	opt.Topic = topic
	opt.Ver = "v1"
	opt.Async = async
	for {
		sz = msgSize + rand.Intn(msgSize)
		no = atomic.AddInt64(&n, 1)

		msg = fmt.Sprintf("topic is %s, %s w:%s seq:%-2d no:%-10d payload:%s",
			topic, time.Now(),
			workerId, seq, no, strings.Repeat("X", sz))
		if tag != "" {
			opt.Tag = tag
		}

		err = client.Pub(key, []byte(msg), opt)
		if err != nil {
			fmt.Println(err)
			no = atomic.AddInt64(&n, -1)
		} else {
			if no%step == 0 {
				log.Println(msg)
			}
		}

		if sleep > 0 {
			time.Sleep(sleep)
		}
	}

}
<commit_msg>add non-ascii in test script<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"strings"
	"sync/atomic"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/api/v1"
	"github.com/funkygao/gafka/ctx"
)

var (
	c        int
	addr     string
	n        int64
	msgSize  int
	debug    bool
	async    bool
	step     int64
	appid    string
	topic    string
	key      string
	workerId string
	sleep    time.Duration
	tag      string
)

func init() {
	ip, _ := ctx.LocalIP()
	flag.IntVar(&c, "c", 10, "client concurrency")
	flag.IntVar(&msgSize, "sz", 100, "msg size")
	flag.StringVar(&appid, "appid", "app1", "app id")
	flag.DurationVar(&sleep, "sleep", 0, "sleep between pub")
	flag.StringVar(&addr, "h", fmt.Sprintf("%s:9191", ip.String()), "pub http addr")
	flag.Int64Var(&step, "step", 1, "display progress step")
	flag.StringVar(&key, "key", "", "message key")
	flag.BoolVar(&debug, "debug", false, "debug")
	flag.BoolVar(&async, "async", false, "async pub")
	flag.StringVar(&tag, "tag", "", "add tag to each message")
	flag.StringVar(&topic, "t", "foobar", "topic to pub")
	flag.StringVar(&workerId, "id", "1", "worker id")
	flag.Parse()
}

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	for i := 0; i < c; i++ {
		go pubGatewayLoop(i)
	}

	select {}
}

func pubGatewayLoop(seq int) {
	cf := api.DefaultConfig(appid, "mysecret")
	cf.Pub.Endpoint = addr
	cf.Debug = debug
	client := api.NewClient(cf)

	var (
		err error
		msg string
		no  int64
		sz  int
	)

	var opt api.PubOption
	opt.Topic = topic
	opt.Ver = "v1"
	opt.Async = async
	for {
		sz = msgSize + rand.Intn(msgSize)
		no = atomic.AddInt64(&n, 1)

		msg = fmt.Sprintf("topic[哈哈] is %s, %s w:%s seq:%-2d no:%-10d payload:%s",
			topic, time.Now(),
			workerId, seq, no, strings.Repeat("X", sz))
		if tag != "" {
			opt.Tag = tag
		}

		err = client.Pub(key, []byte(msg), opt)
		if err != nil {
			fmt.Println(err)
			no = atomic.AddInt64(&n, -1)
		} else {
			if no%step == 0 {
				log.Println(msg)
			}
		}

		if sleep > 0 {
			time.Sleep(sleep)
		}
	}

}
<|endoftext|>


<commit_before>package sharing

import (
	"fmt"
	"strconv"
	"strings"
	"time"
)

type conflictStatus int

const (
	// NoConflict is the status when the rev is in the revisions chain (OK)
	NoConflict conflictStatus = iota
	// LostConflict is the status when rev is greater than the last revision of
	// the chain (the resolution is often to abort the update)
	LostConflict
	// WonConflict is the status when rev is not in the chain,
	// but the last revision of the chain is still (the resolution can be to
	// make the update but including rev in the revisions chain)
	WonConflict
)

// RevsStruct is a struct for revisions in bulk methods of CouchDB
type RevsStruct struct {
	Start int      `json:"start"`
	IDs   []string `json:"ids"`
}

// RevsTree is a tree of revisions, like CouchDB has.
// The revisions are sorted by growing generation (the number before the hyphen).
// http://docs.couchdb.org/en/stable/replication/conflicts.html#revision-tree
type RevsTree struct {
	// Rev is a revision, with the generation and the id
	// e.g. 1-1bad9a88f0a608ea78c12ab49882ac41
	Rev string `json:"rev"`

	// Branches is the list of revisions that have this revision for parent.
	// The general case is to have only one branch, but we can have more with
	// conflicts.
	Branches []RevsTree `json:"branches,omitempty"`
}

// Clone duplicates the RevsTree
func (rt *RevsTree) Clone() RevsTree {
	cloned := RevsTree{Rev: rt.Rev}
	cloned.Branches = make([]RevsTree, len(rt.Branches))
	for i, b := range rt.Branches {
		cloned.Branches[i] = b.Clone()
	}
	return cloned
}

// Generation returns the maximal generation of a revision in this tree
func (rt *RevsTree) Generation() int {
	if len(rt.Branches) == 0 {
		return RevGeneration(rt.Rev)
	}
	max := 0
	for _, b := range rt.Branches {
		if g := b.Generation(); g > max {
			max = g
		}
	}
	return max
}

// Find returns the sub-tree for the given revision, or nil if not found.
func (rt *RevsTree) Find(rev string) *RevsTree {
	if rt.Rev == rev {
		return rt
	}
	for i := range rt.Branches {
		if sub := rt.Branches[i].Find(rev); sub != nil {
			return sub
		}
	}
	return nil
}

// Add inserts the given revision in the main branch
func (rt *RevsTree) Add(rev string) *RevsTree {
	// TODO check generations (conflicts)
	if len(rt.Branches) > 0 {
		return rt.Branches[0].Add(rev)
	}
	rt.Branches = []RevsTree{
		{Rev: rev},
	}
	return &rt.Branches[0]
}

// InsertAfter inserts the given revision in the tree as a child of the second
// revision.
func (rt *RevsTree) InsertAfter(rev, parent string) {
	subtree := rt.Find(parent)
	if subtree == nil {
		subtree = rt.Add(parent)
	}
	for _, b := range subtree.Branches {
		if b.Rev == rev {
			return
		}
	}
	subtree.Branches = append(subtree.Branches, RevsTree{Rev: rev})
	// TODO rebalance (conflicts)
}

// InsertChain inserts a chain of revisions, ie the first revision is the
// parent of the second revision, which is itself the parent of the third
// revision, etc. The first revisions of the chain are very probably already in
// the tree, the last one is certainly not.
func (rt *RevsTree) InsertChain(chain []string) {
	if len(chain) == 0 {
		return
	}
	subtree := rt.Find(chain[0])
	if subtree == nil {
		subtree = rt.Add(chain[0])
	}
	for _, rev := range chain[1:] {
		if len(subtree.Branches) > 0 {
			found := false
			for i := range subtree.Branches {
				if subtree.Branches[i].Rev == rev {
					found = true
					subtree = &subtree.Branches[i]
					break
				}
			}
			if found {
				continue
			}
		}
		subtree.Branches = append(subtree.Branches, RevsTree{Rev: rev})
		subtree = &subtree.Branches[0]
	}
	// TODO rebalance (conflicts)
}

// RevGeneration returns the number before the hyphen, called the generation of a revision
func RevGeneration(rev string) int {
	parts := strings.SplitN(rev, "-", 2)
	gen, err := strconv.Atoi(parts[0])
	if err != nil {
		return 0
	}
	return gen
}

// revsMapToStruct builds a RevsStruct from a json unmarshaled to a map
func revsMapToStruct(revs interface{}) *RevsStruct {
	revisions, ok := revs.(map[string]interface{})
	if !ok {
		return nil
	}
	start, ok := revisions["start"].(float64)
	if !ok {
		return nil
	}
	slice, ok := revisions["ids"].([]interface{})
	if !ok {
		return nil
	}
	ids := make([]string, len(slice))
	for i, id := range slice {
		ids[i], _ = id.(string)
	}
	return &RevsStruct{
		Start: int(start),
		IDs:   ids,
	}
}

// revsChainToStruct transforms revisions from on format to another:
// ["2-aa", "3-bb", "4-cc"] -> { start: 4, ids: ["cc", "bb", "aa"] }
func revsChainToStruct(revs []string) RevsStruct {
	s := RevsStruct{
		IDs: make([]string, len(revs)),
	}
	var last string
	for i, rev := range revs {
		parts := strings.SplitN(rev, "-", 2)
		last = parts[0]
		s.IDs[len(s.IDs)-i-1] = parts[1]
	}
	s.Start, _ = strconv.Atoi(last)
	return s
}

// revsStructToChain is the reverse of revsChainToStruct:
// { start: 4, ids: ["cc", "bb", "aa"] } -> ["2-aa", "3-bb", "4-cc"]
func revsStructToChain(revs RevsStruct) []string {
	start := revs.Start
	ids := revs.IDs
	chain := make([]string, len(ids))
	for i, id := range ids {
		rev := fmt.Sprintf("%d-%s", start, id)
		chain[len(ids)-i-1] = rev
		start--
	}
	return chain
}

// detectConflict says if there is a conflict (ie rev is not in the revisions
// chain), and if it is the case, if the update should be made (WonConflict) or
// aborted (LostConflict)
func detectConflict(rev string, chain []string) conflictStatus {
	if len(chain) == 0 {
		return LostConflict
	}
	for _, r := range chain {
		if r == rev {
			return NoConflict
		}
	}

	last := chain[len(chain)-1]
	genl := RevGeneration(last)
	genr := RevGeneration(rev)
	if genl > genr {
		return WonConflict
	} else if genl < genr {
		return LostConflict
	} else if last > rev {
		return WonConflict
	}
	return LostConflict
}

// MixupChainToResolveConflict creates a new chain of revisions that can be
// used to resolve a conflict: the new chain will start the old rev and include
// other revisions from the chain with a greater generation.
func MixupChainToResolveConflict(rev string, chain []string) []string {
	gen := RevGeneration(rev)
	mixed := make([]string, 0)
	found := false
	for _, r := range chain {
		if found {
			mixed = append(mixed, r)
		} else if gen == RevGeneration(r) {
			mixed = append(mixed, rev)
			found = true
		}
	}
	return mixed
}

// conflictName generates a new name for a file/folder in conflict with another
// that has the same path.
func conflictName(name, suffix string) string {
	if suffix == "" {
		suffix = fmt.Sprintf("%d", time.Now().Unix())
	}
	return fmt.Sprintf("%s - conflict - %s", name, suffix)
}

// conflictID generates a new ID for a file/folder that has a conflict between
// two versions of its content.
func conflictID(id, rev string) string {
	parts := strings.SplitN(rev, "-", 2)
	key := []byte(parts[1])
	for i, c := range key {
		switch {
		case '0' <= c && c <= '9':
			key[i] = c - '0'
		case 'a' <= c && c <= 'f':
			key[i] = c - 'a' + 10
		case 'A' <= c && c <= 'F':
			key[i] = c - 'A' + 10
		}
	}
	return XorID(id, key)
}
<commit_msg>Add a small work-around for sharing bugs (#1792)<commit_after>package sharing

import (
	"fmt"
	"strconv"
	"strings"
	"time"
)

type conflictStatus int

const (
	// NoConflict is the status when the rev is in the revisions chain (OK)
	NoConflict conflictStatus = iota
	// LostConflict is the status when rev is greater than the last revision of
	// the chain (the resolution is often to abort the update)
	LostConflict
	// WonConflict is the status when rev is not in the chain,
	// but the last revision of the chain is still (the resolution can be to
	// make the update but including rev in the revisions chain)
	WonConflict
)

// RevsStruct is a struct for revisions in bulk methods of CouchDB
type RevsStruct struct {
	Start int      `json:"start"`
	IDs   []string `json:"ids"`
}

// RevsTree is a tree of revisions, like CouchDB has.
// The revisions are sorted by growing generation (the number before the hyphen).
// http://docs.couchdb.org/en/stable/replication/conflicts.html#revision-tree
type RevsTree struct {
	// Rev is a revision, with the generation and the id
	// e.g. 1-1bad9a88f0a608ea78c12ab49882ac41
	Rev string `json:"rev"`

	// Branches is the list of revisions that have this revision for parent.
	// The general case is to have only one branch, but we can have more with
	// conflicts.
	Branches []RevsTree `json:"branches,omitempty"`
}

// Clone duplicates the RevsTree
func (rt *RevsTree) Clone() RevsTree {
	cloned := RevsTree{Rev: rt.Rev}
	cloned.Branches = make([]RevsTree, len(rt.Branches))
	for i, b := range rt.Branches {
		cloned.Branches[i] = b.Clone()
	}
	return cloned
}

// Generation returns the maximal generation of a revision in this tree
func (rt *RevsTree) Generation() int {
	if len(rt.Branches) == 0 {
		return RevGeneration(rt.Rev)
	}
	max := 0
	for _, b := range rt.Branches {
		if g := b.Generation(); g > max {
			max = g
		}
	}
	return max
}

// Find returns the sub-tree for the given revision, or nil if not found.
func (rt *RevsTree) Find(rev string) *RevsTree {
	if rt.Rev == rev {
		return rt
	}
	for i := range rt.Branches {
		if sub := rt.Branches[i].Find(rev); sub != nil {
			return sub
		}
	}
	return nil
}

// Add inserts the given revision in the main branch
func (rt *RevsTree) Add(rev string) *RevsTree {
	// TODO check generations (conflicts)
	if len(rt.Branches) > 0 {
		return rt.Branches[0].Add(rev)
	}
	rt.Branches = []RevsTree{
		{Rev: rev},
	}
	return &rt.Branches[0]
}

// InsertAfter inserts the given revision in the tree as a child of the second
// revision.
func (rt *RevsTree) InsertAfter(rev, parent string) {
	subtree := rt.Find(parent)
	if subtree == nil {
		// XXX This condition shouldn't be true, but it can help to limit
		// damage in case bugs happen.
		if rt.Find(rev) != nil {
			return
		}
		subtree = rt.Add(parent)
	}
	for _, b := range subtree.Branches {
		if b.Rev == rev {
			return
		}
	}
	subtree.Branches = append(subtree.Branches, RevsTree{Rev: rev})
	// TODO rebalance (conflicts)
}

// InsertChain inserts a chain of revisions, ie the first revision is the
// parent of the second revision, which is itself the parent of the third
// revision, etc. The first revisions of the chain are very probably already in
// the tree, the last one is certainly not.
func (rt *RevsTree) InsertChain(chain []string) {
	if len(chain) == 0 {
		return
	}
	subtree := rt.Find(chain[0])
	if subtree == nil {
		subtree = rt.Add(chain[0])
	}
	for _, rev := range chain[1:] {
		if len(subtree.Branches) > 0 {
			found := false
			for i := range subtree.Branches {
				if subtree.Branches[i].Rev == rev {
					found = true
					subtree = &subtree.Branches[i]
					break
				}
			}
			if found {
				continue
			}
		}
		subtree.Branches = append(subtree.Branches, RevsTree{Rev: rev})
		subtree = &subtree.Branches[0]
	}
	// TODO rebalance (conflicts)
}

// RevGeneration returns the number before the hyphen, called the generation of a revision
func RevGeneration(rev string) int {
	parts := strings.SplitN(rev, "-", 2)
	gen, err := strconv.Atoi(parts[0])
	if err != nil {
		return 0
	}
	return gen
}

// revsMapToStruct builds a RevsStruct from a json unmarshaled to a map
func revsMapToStruct(revs interface{}) *RevsStruct {
	revisions, ok := revs.(map[string]interface{})
	if !ok {
		return nil
	}
	start, ok := revisions["start"].(float64)
	if !ok {
		return nil
	}
	slice, ok := revisions["ids"].([]interface{})
	if !ok {
		return nil
	}
	ids := make([]string, len(slice))
	for i, id := range slice {
		ids[i], _ = id.(string)
	}
	return &RevsStruct{
		Start: int(start),
		IDs:   ids,
	}
}

// revsChainToStruct transforms revisions from on format to another:
// ["2-aa", "3-bb", "4-cc"] -> { start: 4, ids: ["cc", "bb", "aa"] }
func revsChainToStruct(revs []string) RevsStruct {
	s := RevsStruct{
		IDs: make([]string, len(revs)),
	}
	var last string
	for i, rev := range revs {
		parts := strings.SplitN(rev, "-", 2)
		last = parts[0]
		s.IDs[len(s.IDs)-i-1] = parts[1]
	}
	s.Start, _ = strconv.Atoi(last)
	return s
}

// revsStructToChain is the reverse of revsChainToStruct:
// { start: 4, ids: ["cc", "bb", "aa"] } -> ["2-aa", "3-bb", "4-cc"]
func revsStructToChain(revs RevsStruct) []string {
	start := revs.Start
	ids := revs.IDs
	chain := make([]string, len(ids))
	for i, id := range ids {
		rev := fmt.Sprintf("%d-%s", start, id)
		chain[len(ids)-i-1] = rev
		start--
	}
	return chain
}

// detectConflict says if there is a conflict (ie rev is not in the revisions
// chain), and if it is the case, if the update should be made (WonConflict) or
// aborted (LostConflict)
func detectConflict(rev string, chain []string) conflictStatus {
	if len(chain) == 0 {
		return LostConflict
	}
	for _, r := range chain {
		if r == rev {
			return NoConflict
		}
	}

	last := chain[len(chain)-1]
	genl := RevGeneration(last)
	genr := RevGeneration(rev)
	if genl > genr {
		return WonConflict
	} else if genl < genr {
		return LostConflict
	} else if last > rev {
		return WonConflict
	}
	return LostConflict
}

// MixupChainToResolveConflict creates a new chain of revisions that can be
// used to resolve a conflict: the new chain will start the old rev and include
// other revisions from the chain with a greater generation.
func MixupChainToResolveConflict(rev string, chain []string) []string {
	gen := RevGeneration(rev)
	mixed := make([]string, 0)
	found := false
	for _, r := range chain {
		if found {
			mixed = append(mixed, r)
		} else if gen == RevGeneration(r) {
			mixed = append(mixed, rev)
			found = true
		}
	}
	return mixed
}

// conflictName generates a new name for a file/folder in conflict with another
// that has the same path.
func conflictName(name, suffix string) string {
	if suffix == "" {
		suffix = fmt.Sprintf("%d", time.Now().Unix())
	}
	return fmt.Sprintf("%s - conflict - %s", name, suffix)
}

// conflictID generates a new ID for a file/folder that has a conflict between
// two versions of its content.
func conflictID(id, rev string) string {
	parts := strings.SplitN(rev, "-", 2)
	key := []byte(parts[1])
	for i, c := range key {
		switch {
		case '0' <= c && c <= '9':
			key[i] = c - '0'
		case 'a' <= c && c <= 'f':
			key[i] = c - 'a' + 10
		case 'A' <= c && c <= 'F':
			key[i] = c - 'A' + 10
		}
	}
	return XorID(id, key)
}
<|endoftext|>


<commit_before>package shared

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"gopkg.in/robfig/cron.v2"

	"github.com/lxc/lxd/shared/units"
	"github.com/lxc/lxd/shared/validate"
)

// InstanceAction indicates the type of action being performed.
type InstanceAction string

// InstanceAction types.
const (
	Stop     InstanceAction = "stop"
	Start    InstanceAction = "start"
	Restart  InstanceAction = "restart"
	Freeze   InstanceAction = "freeze"
	Unfreeze InstanceAction = "unfreeze"
)

// ConfigVolatilePrefix indicates the prefix used for volatile config keys.
const ConfigVolatilePrefix = "volatile."

// IsRootDiskDevice returns true if the given device representation is configured as root disk for
// an instance. It typically get passed a specific entry of api.Instance.Devices.
func IsRootDiskDevice(device map[string]string) bool {
	// Root disk devices also need a non-empty "pool" property, but we can't check that here
	// because this function is used with clients talking to older servers where there was no
	// concept of a storage pool, and also it is used for migrating from old to new servers.
	// The validation of the non-empty "pool" property is done inside the disk device itself.
	if device["type"] == "disk" && device["path"] == "/" && device["source"] == "" {
		return true
	}

	return false
}

// ErrNoRootDisk means there is no root disk device found.
var ErrNoRootDisk = fmt.Errorf("No root device could be found")

// GetRootDiskDevice returns the instance device that is configured as root disk.
// Returns the device name and device config map.
func GetRootDiskDevice(devices map[string]map[string]string) (string, map[string]string, error) {
	var devName string
	var dev map[string]string

	for n, d := range devices {
		if IsRootDiskDevice(d) {
			if devName != "" {
				return "", nil, fmt.Errorf("More than one root device found")
			}

			devName = n
			dev = d
		}
	}

	if devName != "" {
		return devName, dev, nil
	}

	return "", nil, ErrNoRootDisk
}

// HugePageSizeKeys is a list of known hugepage size configuration keys.
var HugePageSizeKeys = [...]string{"limits.hugepages.64KB", "limits.hugepages.1MB", "limits.hugepages.2MB", "limits.hugepages.1GB"}

// HugePageSizeSuffix contains the list of known hugepage size suffixes.
var HugePageSizeSuffix = [...]string{"64KB", "1MB", "2MB", "1GB"}

// KnownInstanceConfigKeys maps all fully defined, well-known config keys
// to an appropriate checker function, which validates whether or not a
// given value is syntactically legal.
var KnownInstanceConfigKeys = map[string]func(value string) error{
	"boot.autostart":             validate.Optional(validate.IsBool),
	"boot.autostart.delay":       validate.Optional(validate.IsInt64),
	"boot.autostart.priority":    validate.Optional(validate.IsInt64),
	"boot.stop.priority":         validate.Optional(validate.IsInt64),
	"boot.host_shutdown_timeout": validate.Optional(validate.IsInt64),

	"limits.cpu": func(value string) error {
		if value == "" {
			return nil
		}

		// Validate the character set
		match, _ := regexp.MatchString("^[-,0-9]*$", value)
		if !match {
			return fmt.Errorf("Invalid CPU limit syntax")
		}

		// Validate first character
		if strings.HasPrefix(value, "-") || strings.HasPrefix(value, ",") {
			return fmt.Errorf("CPU limit can't start with a separator")
		}

		// Validate last character
		if strings.HasSuffix(value, "-") || strings.HasSuffix(value, ",") {
			return fmt.Errorf("CPU limit can't end with a separator")
		}

		return nil
	},
	"limits.cpu.allowance": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			// Percentage based allocation
			_, err := strconv.Atoi(strings.TrimSuffix(value, "%"))
			if err != nil {
				return err
			}

			return nil
		}

		// Time based allocation
		fields := strings.SplitN(value, "/", 2)
		if len(fields) != 2 {
			return fmt.Errorf("Invalid allowance: %s", value)
		}

		_, err := strconv.Atoi(strings.TrimSuffix(fields[0], "ms"))
		if err != nil {
			return err
		}

		_, err = strconv.Atoi(strings.TrimSuffix(fields[1], "ms"))
		if err != nil {
			return err
		}

		return nil
	},
	"limits.cpu.priority": validate.Optional(validate.IsPriority),

	"limits.disk.priority": validate.Optional(validate.IsPriority),

	"limits.hugepages.64KB": validate.Optional(validate.IsSize),
	"limits.hugepages.1MB":  validate.Optional(validate.IsSize),
	"limits.hugepages.2MB":  validate.Optional(validate.IsSize),
	"limits.hugepages.1GB":  validate.Optional(validate.IsSize),

	"limits.memory": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			_, err := strconv.ParseInt(strings.TrimSuffix(value, "%"), 10, 64)
			if err != nil {
				return err
			}

			return nil
		}

		_, err := units.ParseByteSizeString(value)
		if err != nil {
			return err
		}

		return nil
	},
	"limits.memory.enforce": func(value string) error {
		return validate.IsOneOf(value, []string{"soft", "hard"})
	},
	"limits.memory.swap":          validate.Optional(validate.IsBool),
	"limits.memory.swap.priority": validate.Optional(validate.IsPriority),
	"limits.memory.hugepages":     validate.Optional(validate.IsBool),

	"limits.network.priority": validate.Optional(validate.IsPriority),

	"limits.processes": validate.Optional(validate.IsInt64),

	"linux.kernel_modules": validate.IsAny,

	"migration.incremental.memory":            validate.Optional(validate.IsBool),
	"migration.incremental.memory.iterations": validate.Optional(validate.IsUint32),
	"migration.incremental.memory.goal":       validate.Optional(validate.IsUint32),

	"nvidia.runtime":             validate.Optional(validate.IsBool),
	"nvidia.driver.capabilities": validate.IsAny,
	"nvidia.require.cuda":        validate.IsAny,
	"nvidia.require.driver":      validate.IsAny,

	"security.nesting":       validate.Optional(validate.IsBool),
	"security.privileged":    validate.Optional(validate.IsBool),
	"security.devlxd":        validate.Optional(validate.IsBool),
	"security.devlxd.images": validate.Optional(validate.IsBool),

	"security.protection.delete": validate.Optional(validate.IsBool),
	"security.protection.shift":  validate.Optional(validate.IsBool),

	"security.idmap.base":     validate.Optional(validate.IsUint32),
	"security.idmap.isolated": validate.Optional(validate.IsBool),
	"security.idmap.size":     validate.Optional(validate.IsUint32),

	"security.secureboot": validate.Optional(validate.IsBool),

	"security.syscalls.allow":                   validate.IsAny,
	"security.syscalls.blacklist_default":       validate.Optional(validate.IsBool),
	"security.syscalls.blacklist_compat":        validate.Optional(validate.IsBool),
	"security.syscalls.blacklist":               validate.IsAny,
	"security.syscalls.deny_default":            validate.Optional(validate.IsBool),
	"security.syscalls.deny_compat":             validate.Optional(validate.IsBool),
	"security.syscalls.deny":                    validate.IsAny,
	"security.syscalls.intercept.bpf":           validate.Optional(validate.IsBool),
	"security.syscalls.intercept.bpf.devices":   validate.Optional(validate.IsBool),
	"security.syscalls.intercept.mknod":         validate.Optional(validate.IsBool),
	"security.syscalls.intercept.mount":         validate.Optional(validate.IsBool),
	"security.syscalls.intercept.mount.allowed": validate.IsAny,
	"security.syscalls.intercept.mount.fuse":    validate.IsAny,
	"security.syscalls.intercept.mount.shift":   validate.Optional(validate.IsBool),
	"security.syscalls.intercept.setxattr":      validate.Optional(validate.IsBool),
	"security.syscalls.whitelist":               validate.IsAny,

	"snapshots.schedule": func(value string) error {
		if value == "" {
			return nil
		}

		if len(strings.Split(value, " ")) != 5 {
			return fmt.Errorf("Schedule must be of the form: <minute> <hour> <day-of-month> <month> <day-of-week>")
		}

		_, err := cron.Parse(fmt.Sprintf("* %s", value))
		if err != nil {
			return errors.Wrap(err, "Error parsing schedule")
		}

		return nil
	},
	"snapshots.schedule.stopped": validate.Optional(validate.IsBool),
	"snapshots.pattern":          validate.IsAny,
	"snapshots.expiry": func(value string) error {
		// Validate expression
		_, err := GetSnapshotExpiry(time.Time{}, value)
		return err
	},

	// Caller is responsible for full validation of any raw.* value
	"raw.apparmor": validate.IsAny,
	"raw.idmap":    validate.IsAny,
	"raw.lxc":      validate.IsAny,
	"raw.qemu":     validate.IsAny,
	"raw.seccomp":  validate.IsAny,

	"volatile.apply_template":   validate.IsAny,
	"volatile.base_image":       validate.IsAny,
	"volatile.last_state.idmap": validate.IsAny,
	"volatile.last_state.power": validate.IsAny,
	"volatile.idmap.base":       validate.IsAny,
	"volatile.idmap.current":    validate.IsAny,
	"volatile.idmap.next":       validate.IsAny,
	"volatile.apply_quota":      validate.IsAny,
	"volatile.uuid":             validate.Optional(validate.IsUUID),
}

// ConfigKeyChecker returns a function that will check whether or not
// a provide value is valid for the associate config key.  Returns an
// error if the key is not known.  The checker function only performs
// syntactic checking of the value, semantic and usage checking must
// be done by the caller.  User defined keys are always considered to
// be valid, e.g. user.* and environment.* keys.
func ConfigKeyChecker(key string) (func(value string) error, error) {
	if f, ok := KnownInstanceConfigKeys[key]; ok {
		return f, nil
	}

	if strings.HasPrefix(key, ConfigVolatilePrefix) {
		if strings.HasSuffix(key, ".hwaddr") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".name") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".host_name") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".mtu") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".created") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".id") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".vlan") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".spoofcheck") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".apply_quota") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".ceph_rbd") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".driver") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".uuid") {
			return validate.IsAny, nil
		}
	}

	if strings.HasPrefix(key, "environment.") {
		return validate.IsAny, nil
	}

	if strings.HasPrefix(key, "user.") {
		return validate.IsAny, nil
	}

	if strings.HasPrefix(key, "image.") {
		return validate.IsAny, nil
	}

	if strings.HasPrefix(key, "limits.kernel.") &&
		(len(key) > len("limits.kernel.")) {
		return validate.IsAny, nil
	}

	return nil, fmt.Errorf("Unknown configuration key: %s", key)
}

// InstanceGetParentAndSnapshotName returns the parent instance name, snapshot name,
// and whether it actually was a snapshot name.
func InstanceGetParentAndSnapshotName(name string) (string, string, bool) {
	fields := strings.SplitN(name, SnapshotDelimiter, 2)
	if len(fields) == 1 {
		return name, "", false
	}

	return fields[0], fields[1], true
}

// InstanceIncludeWhenCopying is used to decide whether to include a config item or not when copying an instance.
// The remoteCopy argument indicates if the copy is remote (i.e between LXD nodes) as this affects the keys kept.
func InstanceIncludeWhenCopying(configKey string, remoteCopy bool) bool {
	if configKey == "volatile.base_image" {
		return true // Include volatile.base_image always as it can help optimize copies.
	}

	if configKey == "volatile.last_state.idmap" && !remoteCopy {
		return true // Include volatile.last_state.idmap when doing local copy to avoid needless remapping.
	}

	if strings.HasPrefix(configKey, ConfigVolatilePrefix) {
		return false // Exclude all other volatile keys.
	}

	return true // Keep all other keys.
}
<commit_msg>shared: Add migration.stateful<commit_after>package shared

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"gopkg.in/robfig/cron.v2"

	"github.com/lxc/lxd/shared/units"
	"github.com/lxc/lxd/shared/validate"
)

// InstanceAction indicates the type of action being performed.
type InstanceAction string

// InstanceAction types.
const (
	Stop     InstanceAction = "stop"
	Start    InstanceAction = "start"
	Restart  InstanceAction = "restart"
	Freeze   InstanceAction = "freeze"
	Unfreeze InstanceAction = "unfreeze"
)

// ConfigVolatilePrefix indicates the prefix used for volatile config keys.
const ConfigVolatilePrefix = "volatile."

// IsRootDiskDevice returns true if the given device representation is configured as root disk for
// an instance. It typically get passed a specific entry of api.Instance.Devices.
func IsRootDiskDevice(device map[string]string) bool {
	// Root disk devices also need a non-empty "pool" property, but we can't check that here
	// because this function is used with clients talking to older servers where there was no
	// concept of a storage pool, and also it is used for migrating from old to new servers.
	// The validation of the non-empty "pool" property is done inside the disk device itself.
	if device["type"] == "disk" && device["path"] == "/" && device["source"] == "" {
		return true
	}

	return false
}

// ErrNoRootDisk means there is no root disk device found.
var ErrNoRootDisk = fmt.Errorf("No root device could be found")

// GetRootDiskDevice returns the instance device that is configured as root disk.
// Returns the device name and device config map.
func GetRootDiskDevice(devices map[string]map[string]string) (string, map[string]string, error) {
	var devName string
	var dev map[string]string

	for n, d := range devices {
		if IsRootDiskDevice(d) {
			if devName != "" {
				return "", nil, fmt.Errorf("More than one root device found")
			}

			devName = n
			dev = d
		}
	}

	if devName != "" {
		return devName, dev, nil
	}

	return "", nil, ErrNoRootDisk
}

// HugePageSizeKeys is a list of known hugepage size configuration keys.
var HugePageSizeKeys = [...]string{"limits.hugepages.64KB", "limits.hugepages.1MB", "limits.hugepages.2MB", "limits.hugepages.1GB"}

// HugePageSizeSuffix contains the list of known hugepage size suffixes.
var HugePageSizeSuffix = [...]string{"64KB", "1MB", "2MB", "1GB"}

// KnownInstanceConfigKeys maps all fully defined, well-known config keys
// to an appropriate checker function, which validates whether or not a
// given value is syntactically legal.
var KnownInstanceConfigKeys = map[string]func(value string) error{
	"boot.autostart":             validate.Optional(validate.IsBool),
	"boot.autostart.delay":       validate.Optional(validate.IsInt64),
	"boot.autostart.priority":    validate.Optional(validate.IsInt64),
	"boot.stop.priority":         validate.Optional(validate.IsInt64),
	"boot.host_shutdown_timeout": validate.Optional(validate.IsInt64),

	"limits.cpu": func(value string) error {
		if value == "" {
			return nil
		}

		// Validate the character set
		match, _ := regexp.MatchString("^[-,0-9]*$", value)
		if !match {
			return fmt.Errorf("Invalid CPU limit syntax")
		}

		// Validate first character
		if strings.HasPrefix(value, "-") || strings.HasPrefix(value, ",") {
			return fmt.Errorf("CPU limit can't start with a separator")
		}

		// Validate last character
		if strings.HasSuffix(value, "-") || strings.HasSuffix(value, ",") {
			return fmt.Errorf("CPU limit can't end with a separator")
		}

		return nil
	},
	"limits.cpu.allowance": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			// Percentage based allocation
			_, err := strconv.Atoi(strings.TrimSuffix(value, "%"))
			if err != nil {
				return err
			}

			return nil
		}

		// Time based allocation
		fields := strings.SplitN(value, "/", 2)
		if len(fields) != 2 {
			return fmt.Errorf("Invalid allowance: %s", value)
		}

		_, err := strconv.Atoi(strings.TrimSuffix(fields[0], "ms"))
		if err != nil {
			return err
		}

		_, err = strconv.Atoi(strings.TrimSuffix(fields[1], "ms"))
		if err != nil {
			return err
		}

		return nil
	},
	"limits.cpu.priority": validate.Optional(validate.IsPriority),

	"limits.disk.priority": validate.Optional(validate.IsPriority),

	"limits.hugepages.64KB": validate.Optional(validate.IsSize),
	"limits.hugepages.1MB":  validate.Optional(validate.IsSize),
	"limits.hugepages.2MB":  validate.Optional(validate.IsSize),
	"limits.hugepages.1GB":  validate.Optional(validate.IsSize),

	"limits.memory": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			_, err := strconv.ParseInt(strings.TrimSuffix(value, "%"), 10, 64)
			if err != nil {
				return err
			}

			return nil
		}

		_, err := units.ParseByteSizeString(value)
		if err != nil {
			return err
		}

		return nil
	},
	"limits.memory.enforce": func(value string) error {
		return validate.IsOneOf(value, []string{"soft", "hard"})
	},
	"limits.memory.swap":          validate.Optional(validate.IsBool),
	"limits.memory.swap.priority": validate.Optional(validate.IsPriority),
	"limits.memory.hugepages":     validate.Optional(validate.IsBool),

	"limits.network.priority": validate.Optional(validate.IsPriority),

	"limits.processes": validate.Optional(validate.IsInt64),

	"linux.kernel_modules": validate.IsAny,

	"migration.incremental.memory":            validate.Optional(validate.IsBool),
	"migration.incremental.memory.iterations": validate.Optional(validate.IsUint32),
	"migration.incremental.memory.goal":       validate.Optional(validate.IsUint32),
	"migration.stateful":                      validate.Optional(validate.IsBool),

	"nvidia.runtime":             validate.Optional(validate.IsBool),
	"nvidia.driver.capabilities": validate.IsAny,
	"nvidia.require.cuda":        validate.IsAny,
	"nvidia.require.driver":      validate.IsAny,

	"security.nesting":       validate.Optional(validate.IsBool),
	"security.privileged":    validate.Optional(validate.IsBool),
	"security.devlxd":        validate.Optional(validate.IsBool),
	"security.devlxd.images": validate.Optional(validate.IsBool),

	"security.protection.delete": validate.Optional(validate.IsBool),
	"security.protection.shift":  validate.Optional(validate.IsBool),

	"security.idmap.base":     validate.Optional(validate.IsUint32),
	"security.idmap.isolated": validate.Optional(validate.IsBool),
	"security.idmap.size":     validate.Optional(validate.IsUint32),

	"security.secureboot": validate.Optional(validate.IsBool),

	"security.syscalls.allow":                   validate.IsAny,
	"security.syscalls.blacklist_default":       validate.Optional(validate.IsBool),
	"security.syscalls.blacklist_compat":        validate.Optional(validate.IsBool),
	"security.syscalls.blacklist":               validate.IsAny,
	"security.syscalls.deny_default":            validate.Optional(validate.IsBool),
	"security.syscalls.deny_compat":             validate.Optional(validate.IsBool),
	"security.syscalls.deny":                    validate.IsAny,
	"security.syscalls.intercept.bpf":           validate.Optional(validate.IsBool),
	"security.syscalls.intercept.bpf.devices":   validate.Optional(validate.IsBool),
	"security.syscalls.intercept.mknod":         validate.Optional(validate.IsBool),
	"security.syscalls.intercept.mount":         validate.Optional(validate.IsBool),
	"security.syscalls.intercept.mount.allowed": validate.IsAny,
	"security.syscalls.intercept.mount.fuse":    validate.IsAny,
	"security.syscalls.intercept.mount.shift":   validate.Optional(validate.IsBool),
	"security.syscalls.intercept.setxattr":      validate.Optional(validate.IsBool),
	"security.syscalls.whitelist":               validate.IsAny,

	"snapshots.schedule": func(value string) error {
		if value == "" {
			return nil
		}

		if len(strings.Split(value, " ")) != 5 {
			return fmt.Errorf("Schedule must be of the form: <minute> <hour> <day-of-month> <month> <day-of-week>")
		}

		_, err := cron.Parse(fmt.Sprintf("* %s", value))
		if err != nil {
			return errors.Wrap(err, "Error parsing schedule")
		}

		return nil
	},
	"snapshots.schedule.stopped": validate.Optional(validate.IsBool),
	"snapshots.pattern":          validate.IsAny,
	"snapshots.expiry": func(value string) error {
		// Validate expression
		_, err := GetSnapshotExpiry(time.Time{}, value)
		return err
	},

	// Caller is responsible for full validation of any raw.* value
	"raw.apparmor": validate.IsAny,
	"raw.idmap":    validate.IsAny,
	"raw.lxc":      validate.IsAny,
	"raw.qemu":     validate.IsAny,
	"raw.seccomp":  validate.IsAny,

	"volatile.apply_template":   validate.IsAny,
	"volatile.base_image":       validate.IsAny,
	"volatile.last_state.idmap": validate.IsAny,
	"volatile.last_state.power": validate.IsAny,
	"volatile.idmap.base":       validate.IsAny,
	"volatile.idmap.current":    validate.IsAny,
	"volatile.idmap.next":       validate.IsAny,
	"volatile.apply_quota":      validate.IsAny,
	"volatile.uuid":             validate.Optional(validate.IsUUID),
}

// ConfigKeyChecker returns a function that will check whether or not
// a provide value is valid for the associate config key.  Returns an
// error if the key is not known.  The checker function only performs
// syntactic checking of the value, semantic and usage checking must
// be done by the caller.  User defined keys are always considered to
// be valid, e.g. user.* and environment.* keys.
func ConfigKeyChecker(key string) (func(value string) error, error) {
	if f, ok := KnownInstanceConfigKeys[key]; ok {
		return f, nil
	}

	if strings.HasPrefix(key, ConfigVolatilePrefix) {
		if strings.HasSuffix(key, ".hwaddr") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".name") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".host_name") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".mtu") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".created") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".id") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".vlan") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".spoofcheck") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".apply_quota") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".ceph_rbd") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".driver") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".uuid") {
			return validate.IsAny, nil
		}
	}

	if strings.HasPrefix(key, "environment.") {
		return validate.IsAny, nil
	}

	if strings.HasPrefix(key, "user.") {
		return validate.IsAny, nil
	}

	if strings.HasPrefix(key, "image.") {
		return validate.IsAny, nil
	}

	if strings.HasPrefix(key, "limits.kernel.") &&
		(len(key) > len("limits.kernel.")) {
		return validate.IsAny, nil
	}

	return nil, fmt.Errorf("Unknown configuration key: %s", key)
}

// InstanceGetParentAndSnapshotName returns the parent instance name, snapshot name,
// and whether it actually was a snapshot name.
func InstanceGetParentAndSnapshotName(name string) (string, string, bool) {
	fields := strings.SplitN(name, SnapshotDelimiter, 2)
	if len(fields) == 1 {
		return name, "", false
	}

	return fields[0], fields[1], true
}

// InstanceIncludeWhenCopying is used to decide whether to include a config item or not when copying an instance.
// The remoteCopy argument indicates if the copy is remote (i.e between LXD nodes) as this affects the keys kept.
func InstanceIncludeWhenCopying(configKey string, remoteCopy bool) bool {
	if configKey == "volatile.base_image" {
		return true // Include volatile.base_image always as it can help optimize copies.
	}

	if configKey == "volatile.last_state.idmap" && !remoteCopy {
		return true // Include volatile.last_state.idmap when doing local copy to avoid needless remapping.
	}

	if strings.HasPrefix(configKey, ConfigVolatilePrefix) {
		return false // Exclude all other volatile keys.
	}

	return true // Keep all other keys.
}
<|endoftext|>


<commit_before>// The goal here is to make a script which can connect to a storage node and
// pull out the current schemas as defined and spit them back to the user
// in dataman format.
//
// For now this will simply be something that knows how to interact with just postgres
// but once we do a split of interfaces in the storage node we should be able to use
// any storage node to do so
package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"

	yaml "gopkg.in/yaml.v2"

	"github.com/jacksontj/dataman/storagenode"
	"github.com/jacksontj/dataman/storagenode/datasource"
	"github.com/jacksontj/dataman/storagenode/metadata"
	flags "github.com/jessevdk/go-flags"
	"github.com/sirupsen/logrus"
)

var opts struct {
	Databases []string `long:"databases"`
}

func main() {
	parser := flags.NewParser(&opts, flags.Default)
	if _, err := parser.Parse(); err != nil {
		logrus.Fatalf("Error parsing flags: %v", err)
	}

	meta := metadata.NewMeta()

	// TODO: actually have these come through CLI args or something
	config := &storagenode.Config{}
	configBytes, err := ioutil.ReadFile("../storagenode/config.yaml")
	if err != nil {
		logrus.Fatalf("unable to find config: %v", err)
	}
	err = yaml.Unmarshal([]byte(configBytes), &config)
	if err != nil {
		logrus.Fatalf("invalid config: %v", err)
	}

	var datasourceInstanceConfig *storagenode.DatasourceInstanceConfig
	for _, c := range config.Datasources {
		datasourceInstanceConfig = c
		break
	}

	// Note: since we are soely doing schema *export* we don't define a meta func
	// this means that all writes will fail as there is no schema to compare to
	store, err := datasourceInstanceConfig.GetStore(nil)
	storeSchema := store.(datasource.SchemaInterface)

	for _, databasename := range opts.Databases {
		meta.Databases[databasename] = storeSchema.GetDatabase(context.Background(), databasename)
	}

	// TODO: sort? it'd be nice to have the files not change if there was no schema change
	bytes, _ := json.MarshalIndent(meta, "", "  ")
	fmt.Println(string(bytes))

}
<commit_msg>Make `databases` a required field to schemaexport<commit_after>// The goal here is to make a script which can connect to a storage node and
// pull out the current schemas as defined and spit them back to the user
// in dataman format.
//
// For now this will simply be something that knows how to interact with just postgres
// but once we do a split of interfaces in the storage node we should be able to use
// any storage node to do so
package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"

	yaml "gopkg.in/yaml.v2"

	"github.com/jacksontj/dataman/storagenode"
	"github.com/jacksontj/dataman/storagenode/datasource"
	"github.com/jacksontj/dataman/storagenode/metadata"
	flags "github.com/jessevdk/go-flags"
	"github.com/sirupsen/logrus"
)

var opts struct {
	Databases []string `long:"databases" required:"true"`
}

func main() {
	parser := flags.NewParser(&opts, flags.Default)
	if _, err := parser.Parse(); err != nil {
		logrus.Fatalf("Error parsing flags: %v", err)
	}

	meta := metadata.NewMeta()

	// TODO: actually have these come through CLI args or something
	config := &storagenode.Config{}
	configBytes, err := ioutil.ReadFile("../storagenode/config.yaml")
	if err != nil {
		logrus.Fatalf("unable to find config: %v", err)
	}
	err = yaml.Unmarshal([]byte(configBytes), &config)
	if err != nil {
		logrus.Fatalf("invalid config: %v", err)
	}

	var datasourceInstanceConfig *storagenode.DatasourceInstanceConfig
	for _, c := range config.Datasources {
		datasourceInstanceConfig = c
		break
	}

	// Note: since we are soely doing schema *export* we don't define a meta func
	// this means that all writes will fail as there is no schema to compare to
	store, err := datasourceInstanceConfig.GetStore(nil)
	storeSchema := store.(datasource.SchemaInterface)

	for _, databasename := range opts.Databases {
		meta.Databases[databasename] = storeSchema.GetDatabase(context.Background(), databasename)
	}

	// TODO: sort? it'd be nice to have the files not change if there was no schema change
	bytes, _ := json.MarshalIndent(meta, "", "  ")
	fmt.Println(string(bytes))

}
<|endoftext|>