<commit_before>package service

import (
	"context"
	"encoding/json"
	"time"

	kithttp "github.com/go-kit/kit/transport/http"

	"github.com/kolide/fleet/server/kolide"
)

func (mw loggingMiddleware) EnrollAgent(ctx context.Context, enrollSecret string, hostIdentifier string, hostDetails map[string](map[string]string)) (string, error) {
	var (
		nodeKey string
		err     error
	)

	defer func(begin time.Time) {
		_ = mw.loggerInfo(err).Log(
			"method", "EnrollAgent",
			"ip_addr", ctx.Value(kithttp.ContextKeyRequestRemoteAddr).(string),
			"x_for_ip_addr", ctx.Value(kithttp.ContextKeyRequestXForwardedFor).(string),
			"err", err,
			"took", time.Since(begin),
		)
	}(time.Now())

	nodeKey, err = mw.Service.EnrollAgent(ctx, enrollSecret, hostIdentifier, hostDetails)
	return nodeKey, err
}

func (mw loggingMiddleware) AuthenticateHost(ctx context.Context, nodeKey string) (*kolide.Host, error) {
	var (
		host *kolide.Host
		err  error
	)

	defer func(begin time.Time) {
		_ = mw.loggerDebug(err).Log(
			"method", "AuthenticateHost",
			"ip_addr", ctx.Value(kithttp.ContextKeyRequestRemoteAddr).(string),
			"x_for_ip_addr", ctx.Value(kithttp.ContextKeyRequestXForwardedFor).(string),
			"err", err,
			"took", time.Since(begin),
		)
	}(time.Now())

	host, err = mw.Service.AuthenticateHost(ctx, nodeKey)
	return host, err
}

func (mw loggingMiddleware) GetClientConfig(ctx context.Context) (map[string]interface{}, error) {
	var (
		config map[string]interface{}
		err    error
	)

	defer func(begin time.Time) {
		_ = mw.loggerDebug(err).Log(
			"method", "GetClientConfig",
			"ip_addr", ctx.Value(kithttp.ContextKeyRequestRemoteAddr).(string),
			"x_for_ip_addr", ctx.Value(kithttp.ContextKeyRequestXForwardedFor).(string),
			"err", err,
			"took", time.Since(begin),
		)
	}(time.Now())

	config, err = mw.Service.GetClientConfig(ctx)
	return config, err
}

func (mw loggingMiddleware) GetDistributedQueries(ctx context.Context) (map[string]string, uint, error) {
	var (
		queries    map[string]string
		err        error
		accelerate uint
	)

	defer func(begin time.Time) {
		_ = mw.loggerDebug(err).Log(
			"method", "GetDistributedQueries",
			"ip_addr", ctx.Value(kithttp.ContextKeyRequestRemoteAddr).(string),
			"x_for_ip_addr", ctx.Value(kithttp.ContextKeyRequestXForwardedFor).(string),
			"err", err,
			"took", time.Since(begin),
		)
	}(time.Now())

	queries, accelerate, err = mw.Service.GetDistributedQueries(ctx)
	return queries, accelerate, err
}

func (mw loggingMiddleware) SubmitDistributedQueryResults(ctx context.Context, results kolide.OsqueryDistributedQueryResults, statuses map[string]kolide.OsqueryStatus) error {
	var (
		err error
	)

	defer func(begin time.Time) {
		_ = mw.loggerDebug(err).Log(
			"method", "SubmitDistributedQueryResults",
			"ip_addr", ctx.Value(kithttp.ContextKeyRequestRemoteAddr).(string),
			"x_for_ip_addr", ctx.Value(kithttp.ContextKeyRequestXForwardedFor).(string),
			"err", err,
			"took", time.Since(begin),
		)
	}(time.Now())

	err = mw.Service.SubmitDistributedQueryResults(ctx, results, statuses)
	return err
}

func (mw loggingMiddleware) SubmitStatusLogs(ctx context.Context, logs []json.RawMessage) error {
	var (
		err error
	)

	defer func(begin time.Time) {
		_ = mw.loggerDebug(err).Log(
			"method", "SubmitStatusLogs",
			"ip_addr", ctx.Value(kithttp.ContextKeyRequestRemoteAddr).(string),
			"x_for_ip_addr", ctx.Value(kithttp.ContextKeyRequestXForwardedFor).(string),
			"err", err,
			"took", time.Since(begin),
		)
	}(time.Now())

	err = mw.Service.SubmitStatusLogs(ctx, logs)
	return err
}

func (mw loggingMiddleware) SubmitResultLogs(ctx context.Context, logs []json.RawMessage) error {
	var (
		err error
	)

	defer func(begin time.Time) {
		_ = mw.loggerDebug(err).Log(
			"method", "SubmitResultLogs",
			"ip_addr", ctx.Value(kithttp.ContextKeyRequestRemoteAddr).(string),
			"x_for_ip_addr", ctx.Value(kithttp.ContextKeyRequestXForwardedFor).(string),
			"err", err,
			"took", time.Since(begin),
		)
	}(time.Now())

	err = mw.Service.SubmitResultLogs(ctx, logs)
	return err
}
<commit_msg>Log host identifier in EnrollAgent service logging (#2313)<commit_after>package service

import (
	"context"
	"encoding/json"
	"time"

	kithttp "github.com/go-kit/kit/transport/http"

	"github.com/kolide/fleet/server/kolide"
)

func (mw loggingMiddleware) EnrollAgent(ctx context.Context, enrollSecret string, hostIdentifier string, hostDetails map[string](map[string]string)) (string, error) {
	var (
		nodeKey string
		err     error
	)

	defer func(begin time.Time) {
		_ = mw.loggerInfo(err).Log(
			"method", "EnrollAgent",
			"ip_addr", ctx.Value(kithttp.ContextKeyRequestRemoteAddr).(string),
			"x_for_ip_addr", ctx.Value(kithttp.ContextKeyRequestXForwardedFor).(string),
			"host_identifier", hostIdentifier,
			"err", err,
			"took", time.Since(begin),
		)
	}(time.Now())

	nodeKey, err = mw.Service.EnrollAgent(ctx, enrollSecret, hostIdentifier, hostDetails)
	return nodeKey, err
}

func (mw loggingMiddleware) AuthenticateHost(ctx context.Context, nodeKey string) (*kolide.Host, error) {
	var (
		host *kolide.Host
		err  error
	)

	defer func(begin time.Time) {
		_ = mw.loggerDebug(err).Log(
			"method", "AuthenticateHost",
			"ip_addr", ctx.Value(kithttp.ContextKeyRequestRemoteAddr).(string),
			"x_for_ip_addr", ctx.Value(kithttp.ContextKeyRequestXForwardedFor).(string),
			"err", err,
			"took", time.Since(begin),
		)
	}(time.Now())

	host, err = mw.Service.AuthenticateHost(ctx, nodeKey)
	return host, err
}

func (mw loggingMiddleware) GetClientConfig(ctx context.Context) (map[string]interface{}, error) {
	var (
		config map[string]interface{}
		err    error
	)

	defer func(begin time.Time) {
		_ = mw.loggerDebug(err).Log(
			"method", "GetClientConfig",
			"ip_addr", ctx.Value(kithttp.ContextKeyRequestRemoteAddr).(string),
			"x_for_ip_addr", ctx.Value(kithttp.ContextKeyRequestXForwardedFor).(string),
			"err", err,
			"took", time.Since(begin),
		)
	}(time.Now())

	config, err = mw.Service.GetClientConfig(ctx)
	return config, err
}

func (mw loggingMiddleware) GetDistributedQueries(ctx context.Context) (map[string]string, uint, error) {
	var (
		queries    map[string]string
		err        error
		accelerate uint
	)

	defer func(begin time.Time) {
		_ = mw.loggerDebug(err).Log(
			"method", "GetDistributedQueries",
			"ip_addr", ctx.Value(kithttp.ContextKeyRequestRemoteAddr).(string),
			"x_for_ip_addr", ctx.Value(kithttp.ContextKeyRequestXForwardedFor).(string),
			"err", err,
			"took", time.Since(begin),
		)
	}(time.Now())

	queries, accelerate, err = mw.Service.GetDistributedQueries(ctx)
	return queries, accelerate, err
}

func (mw loggingMiddleware) SubmitDistributedQueryResults(ctx context.Context, results kolide.OsqueryDistributedQueryResults, statuses map[string]kolide.OsqueryStatus) error {
	var (
		err error
	)

	defer func(begin time.Time) {
		_ = mw.loggerDebug(err).Log(
			"method", "SubmitDistributedQueryResults",
			"ip_addr", ctx.Value(kithttp.ContextKeyRequestRemoteAddr).(string),
			"x_for_ip_addr", ctx.Value(kithttp.ContextKeyRequestXForwardedFor).(string),
			"err", err,
			"took", time.Since(begin),
		)
	}(time.Now())

	err = mw.Service.SubmitDistributedQueryResults(ctx, results, statuses)
	return err
}

func (mw loggingMiddleware) SubmitStatusLogs(ctx context.Context, logs []json.RawMessage) error {
	var (
		err error
	)

	defer func(begin time.Time) {
		_ = mw.loggerDebug(err).Log(
			"method", "SubmitStatusLogs",
			"ip_addr", ctx.Value(kithttp.ContextKeyRequestRemoteAddr).(string),
			"x_for_ip_addr", ctx.Value(kithttp.ContextKeyRequestXForwardedFor).(string),
			"err", err,
			"took", time.Since(begin),
		)
	}(time.Now())

	err = mw.Service.SubmitStatusLogs(ctx, logs)
	return err
}

func (mw loggingMiddleware) SubmitResultLogs(ctx context.Context, logs []json.RawMessage) error {
	var (
		err error
	)

	defer func(begin time.Time) {
		_ = mw.loggerDebug(err).Log(
			"method", "SubmitResultLogs",
			"ip_addr", ctx.Value(kithttp.ContextKeyRequestRemoteAddr).(string),
			"x_for_ip_addr", ctx.Value(kithttp.ContextKeyRequestXForwardedFor).(string),
			"err", err,
			"took", time.Since(begin),
		)
	}(time.Now())

	err = mw.Service.SubmitResultLogs(ctx, logs)
	return err
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The OpenEBS Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package poolcontroller

import (
	"fmt"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/runtime"
	kubeinformers "k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"
	"k8s.io/klog"

	"github.com/openebs/maya/cmd/cstor-pool-mgmt/controller/common"

	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
	clientset "github.com/openebs/maya/pkg/client/generated/clientset/versioned"

	//openebsScheme "github.com/openebs/maya/pkg/client/clientset/versioned/scheme"
	openebsScheme "github.com/openebs/maya/pkg/client/generated/clientset/versioned/scheme"
	//informers "github.com/openebs/maya/pkg/client/informers/externalversions"
	informers "github.com/openebs/maya/pkg/client/generated/informers/externalversions"
)

const poolControllerName = "CStorPool"

// CStorPoolController is the controller implementation for CStorPool resources.
type CStorPoolController struct {
	// kubeclientset is a standard kubernetes clientset
	kubeclientset kubernetes.Interface

	// clientset is a openebs custom resource package generated for custom API group.
	clientset clientset.Interface

	// cStorPoolSynced is used for caches sync to get populated
	cStorPoolSynced cache.InformerSynced

	// workqueue is a rate limited work queue. This is used to queue work to be
	// processed instead of performing it as soon as a change happens. This
	// means we can ensure we only process a fixed amount of resources at a
	// time, and makes it easy to ensure we are never processing the same item
	// simultaneously in two different workers.
	workqueue workqueue.RateLimitingInterface
	// recorder is an event recorder for recording Event resources to the
	// Kubernetes API.
	recorder record.EventRecorder
}

// NewCStorPoolController returns a new instance of CStorPool controller
func NewCStorPoolController(
	kubeclientset kubernetes.Interface,
	clientset clientset.Interface,
	kubeInformerFactory kubeinformers.SharedInformerFactory,
	cStorInformerFactory informers.SharedInformerFactory) *CStorPoolController {

	// obtain references to shared index informers for the cStorPool resources
	cStorPoolInformer := cStorInformerFactory.Openebs().V1alpha1().CStorPools()

	err := openebsScheme.AddToScheme(scheme.Scheme)
	if err != nil {
		klog.Errorf("failed to add to scheme: error {%v}", err)
	}
	// Create event broadcaster to receive events and send them to any EventSink, watcher, or log.
	// Add NewCstorPoolController types to the default Kubernetes Scheme so Events can be
	// logged for CstorPool Controller types.
	klog.V(4).Info("Creating event broadcaster")
	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(klog.Infof)

	// StartEventWatcher starts sending events received from this EventBroadcaster to the given
	// event handler function. The return value can be ignored or used to stop recording, if
	// desired. Events("") denotes empty namespace
	eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeclientset.CoreV1().Events("")})
	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: poolControllerName})

	controller := &CStorPoolController{
		kubeclientset:   kubeclientset,
		clientset:       clientset,
		cStorPoolSynced: cStorPoolInformer.Informer().HasSynced,
		workqueue:       workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "CStorPool"),
		recorder:        recorder,
	}

	klog.Info("Setting up event handlers")

	// Instantiating QueueLoad before entering workqueue.
	q := common.QueueLoad{}

	// Set up an event handler for when CstorPool resources change.
	cStorPoolInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			cStorPool := obj.(*apis.CStorPool)
			if !IsRightCStorPoolMgmt(cStorPool) {
				return
			}
			if IsDeletionFailedBefore(cStorPool) || IsErrorDuplicate(cStorPool) {
				return
			}
			if cStorPool.Annotations[string(apis.OpenEBSDisableReconcileKey)] == "true" {
				message := fmt.Sprintf("reconcile is disabled via %q annotation", string(apis.OpenEBSDisableReconcileKey))
				controller.recorder.Event(cStorPool, corev1.EventTypeWarning, "Create", message)
				return
			}
			q.Operation = common.QOpAdd
			klog.Infof("cStorPool Added event : %v, %v", cStorPool.ObjectMeta.Name, string(cStorPool.ObjectMeta.UID))
			controller.recorder.Event(cStorPool, corev1.EventTypeNormal, string(common.SuccessSynced), string(common.MessageCreateSynced))

			if !IsCStorPoolCreateStatuses(cStorPool) {
				cStorPool.Status.Phase = apis.CStorPoolStatusPending
			}

			cStorPool, _ = controller.clientset.OpenebsV1alpha1().CStorPools().Update(cStorPool)
			controller.enqueueCStorPool(cStorPool, q)
		},
		UpdateFunc: func(old, new interface{}) {
			newCStorPool := new.(*apis.CStorPool)
			oldCStorPool := old.(*apis.CStorPool)
			if !IsRightCStorPoolMgmt(newCStorPool) {
				return
			}
			if IsOnlyStatusChange(oldCStorPool, newCStorPool) {
				klog.V(4).Infof("Only cStorPool status change: %v, %v ", newCStorPool.ObjectMeta.Name, string(newCStorPool.ObjectMeta.UID))
				return
			}
			if IsDeletionFailedBefore(newCStorPool) || IsErrorDuplicate(newCStorPool) {
				return
			}
			if newCStorPool.Annotations[string(apis.OpenEBSDisableReconcileKey)] == "true" {
				message := fmt.Sprintf("reconcile is disabled via %q annotation", string(apis.OpenEBSDisableReconcileKey))
				controller.recorder.Event(newCStorPool, corev1.EventTypeWarning, "Update", message)
				return
			}
			// Periodic resync will send update events for all known CStorPool.
			// Two different versions of the same CStorPool will always have different RVs.
			if newCStorPool.ResourceVersion == oldCStorPool.ResourceVersion {
				// Synchronize Cstor pool status
				q.Operation = common.QOpSync
				klog.V(4).Infof("cStorPool sync event for %s", newCStorPool.ObjectMeta.Name)
			} else if IsDestroyEvent(newCStorPool) {
				q.Operation = common.QOpDestroy
				klog.Infof("cStorPool Destroy event : %v, %v ", newCStorPool.ObjectMeta.Name, string(newCStorPool.ObjectMeta.UID))
				controller.recorder.Event(newCStorPool, corev1.EventTypeNormal, string(common.SuccessSynced), string(common.MessageDestroySynced))
			} else {
				q.Operation = common.QOpModify
				klog.Infof("cStorPool Modify event : %v, %v", newCStorPool.ObjectMeta.Name, string(newCStorPool.ObjectMeta.UID))
				controller.recorder.Event(newCStorPool, corev1.EventTypeNormal, string(common.SuccessSynced), string(common.MessageModifySynced))
				return // will be removed once modify is implemented
			}
			controller.enqueueCStorPool(newCStorPool, q)
		},
		DeleteFunc: func(obj interface{}) {
			cStorPool := obj.(*apis.CStorPool)
			if !IsRightCStorPoolMgmt(cStorPool) {
				return
			}
			if cStorPool.Annotations[string(apis.OpenEBSDisableReconcileKey)] == "true" {
				message := fmt.Sprintf("reconcile is disabled via %q annotation", string(apis.OpenEBSDisableReconcileKey))
				controller.recorder.Event(cStorPool, corev1.EventTypeWarning, "Delete", message)
				return
			}
			klog.Infof("cStorPool Resource deleted event: %v, %v", cStorPool.ObjectMeta.Name, string(cStorPool.ObjectMeta.UID))
		},
	})

	return controller
}

// enqueueCstorPool takes a CStorPool resource and converts it into a namespace/name
// string which is then put onto the work queue. This method should *not* be
// passed resources of any type other than CStorPools.
func (c *CStorPoolController) enqueueCStorPool(obj *apis.CStorPool, q common.QueueLoad) {
	var key string
	var err error
	if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil {
		runtime.HandleError(err)
		return
	}
	q.Key = key
	c.workqueue.AddRateLimited(q)
}
<commit_msg>fix(cstor-pool-mgmt): reconcile even after deletion timestamp is set on CSP (#1695)<commit_after>/*
Copyright 2018 The OpenEBS Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package poolcontroller

import (
	"fmt"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/runtime"
	kubeinformers "k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"
	"k8s.io/klog"

	"github.com/openebs/maya/cmd/cstor-pool-mgmt/controller/common"

	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
	clientset "github.com/openebs/maya/pkg/client/generated/clientset/versioned"

	//openebsScheme "github.com/openebs/maya/pkg/client/clientset/versioned/scheme"
	openebsScheme "github.com/openebs/maya/pkg/client/generated/clientset/versioned/scheme"
	//informers "github.com/openebs/maya/pkg/client/informers/externalversions"
	informers "github.com/openebs/maya/pkg/client/generated/informers/externalversions"
)

const poolControllerName = "CStorPool"

// CStorPoolController is the controller implementation for CStorPool resources.
type CStorPoolController struct {
	// kubeclientset is a standard kubernetes clientset
	kubeclientset kubernetes.Interface

	// clientset is a openebs custom resource package generated for custom API group.
	clientset clientset.Interface

	// cStorPoolSynced is used for caches sync to get populated
	cStorPoolSynced cache.InformerSynced

	// workqueue is a rate limited work queue. This is used to queue work to be
	// processed instead of performing it as soon as a change happens. This
	// means we can ensure we only process a fixed amount of resources at a
	// time, and makes it easy to ensure we are never processing the same item
	// simultaneously in two different workers.
	workqueue workqueue.RateLimitingInterface
	// recorder is an event recorder for recording Event resources to the
	// Kubernetes API.
	recorder record.EventRecorder
}

// NewCStorPoolController returns a new instance of CStorPool controller
func NewCStorPoolController(
	kubeclientset kubernetes.Interface,
	clientset clientset.Interface,
	kubeInformerFactory kubeinformers.SharedInformerFactory,
	cStorInformerFactory informers.SharedInformerFactory) *CStorPoolController {

	// obtain references to shared index informers for the cStorPool resources
	cStorPoolInformer := cStorInformerFactory.Openebs().V1alpha1().CStorPools()

	err := openebsScheme.AddToScheme(scheme.Scheme)
	if err != nil {
		klog.Errorf("failed to add to scheme: error {%v}", err)
	}
	// Create event broadcaster to receive events and send them to any EventSink, watcher, or log.
	// Add NewCstorPoolController types to the default Kubernetes Scheme so Events can be
	// logged for CstorPool Controller types.
	klog.V(4).Info("Creating event broadcaster")
	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(klog.Infof)

	// StartEventWatcher starts sending events received from this EventBroadcaster to the given
	// event handler function. The return value can be ignored or used to stop recording, if
	// desired. Events("") denotes empty namespace
	eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeclientset.CoreV1().Events("")})
	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: poolControllerName})

	controller := &CStorPoolController{
		kubeclientset:   kubeclientset,
		clientset:       clientset,
		cStorPoolSynced: cStorPoolInformer.Informer().HasSynced,
		workqueue:       workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "CStorPool"),
		recorder:        recorder,
	}

	klog.Info("Setting up event handlers")

	// Instantiating QueueLoad before entering workqueue.
	q := common.QueueLoad{}

	// Set up an event handler for when CstorPool resources change.
	cStorPoolInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			cStorPool := obj.(*apis.CStorPool)
			if !IsRightCStorPoolMgmt(cStorPool) {
				return
			}
			if IsDeletionFailedBefore(cStorPool) || IsErrorDuplicate(cStorPool) {
				return
			}
			if cStorPool.Annotations[string(apis.OpenEBSDisableReconcileKey)] == "true" {
				message := fmt.Sprintf("reconcile is disabled via %q annotation", string(apis.OpenEBSDisableReconcileKey))
				controller.recorder.Event(cStorPool, corev1.EventTypeWarning, "Create", message)
				return
			}

			q.Operation = common.QOpAdd
			klog.Infof("cStorPool Added event : %v, %v", cStorPool.ObjectMeta.Name, string(cStorPool.ObjectMeta.UID))
			controller.recorder.Event(cStorPool, corev1.EventTypeNormal, string(common.SuccessSynced), string(common.MessageCreateSynced))
			if !IsCStorPoolCreateStatuses(cStorPool) {
				cStorPool.Status.Phase = apis.CStorPoolStatusPending
			}

			cStorPool, _ = controller.clientset.OpenebsV1alpha1().CStorPools().Update(cStorPool)
			controller.enqueueCStorPool(cStorPool, q)
		},
		UpdateFunc: func(old, new interface{}) {
			newCStorPool := new.(*apis.CStorPool)
			oldCStorPool := old.(*apis.CStorPool)
			if !IsRightCStorPoolMgmt(newCStorPool) {
				return
			}

			// When CSP is marked for deletion no need to call IsOnlyStatusChange because when
			// CSP marked for deletion we need to enqueue such CSPs
			if !IsDestroyEvent(newCStorPool) && IsOnlyStatusChange(oldCStorPool, newCStorPool) {
				klog.V(4).Infof("Only cStorPool status change: %v, %v ", newCStorPool.ObjectMeta.Name, string(newCStorPool.ObjectMeta.UID))
				return
			}
			if IsErrorDuplicate(newCStorPool) {
				return
			}
			if newCStorPool.Annotations[string(apis.OpenEBSDisableReconcileKey)] == "true" {
				message := fmt.Sprintf("reconcile is disabled via %q annotation", string(apis.OpenEBSDisableReconcileKey))
				controller.recorder.Event(newCStorPool, corev1.EventTypeWarning, "Update", message)
				return
			}
			// Periodic resync will send update events for all known CStorPool.
			// Two different versions of the same CStorPool will always have different RVs.
			if IsDestroyEvent(newCStorPool) {
				q.Operation = common.QOpDestroy
				klog.Infof("cStorPool Destroy event : %v, %v ", newCStorPool.ObjectMeta.Name, string(newCStorPool.ObjectMeta.UID))
				controller.recorder.Event(newCStorPool, corev1.EventTypeNormal, string(common.SuccessSynced), string(common.MessageDestroySynced))
			} else if newCStorPool.ResourceVersion == oldCStorPool.ResourceVersion {
				// Synchronize Cstor pool status
				q.Operation = common.QOpSync
				klog.V(4).Infof("cStorPool sync event for %s", newCStorPool.ObjectMeta.Name)
			} else {
				q.Operation = common.QOpModify
				klog.Infof("cStorPool Modify event : %v, %v", newCStorPool.ObjectMeta.Name, string(newCStorPool.ObjectMeta.UID))
				controller.recorder.Event(newCStorPool, corev1.EventTypeNormal, string(common.SuccessSynced), string(common.MessageModifySynced))
				return // will be removed once modify is implemented
			}
			controller.enqueueCStorPool(newCStorPool, q)
		},
		DeleteFunc: func(obj interface{}) {
			cStorPool := obj.(*apis.CStorPool)
			if !IsRightCStorPoolMgmt(cStorPool) {
				return
			}
			if cStorPool.Annotations[string(apis.OpenEBSDisableReconcileKey)] == "true" {
				message := fmt.Sprintf("reconcile is disabled via %q annotation", string(apis.OpenEBSDisableReconcileKey))
				controller.recorder.Event(cStorPool, corev1.EventTypeWarning, "Delete", message)
				return
			}
			klog.Infof("cStorPool Resource deleted event: %v, %v", cStorPool.ObjectMeta.Name, string(cStorPool.ObjectMeta.UID))
		},
	})

	return controller
}

// enqueueCstorPool takes a CStorPool resource and converts it into a namespace/name
// string which is then put onto the work queue. This method should *not* be
// passed resources of any type other than CStorPools.
func (c *CStorPoolController) enqueueCStorPool(obj *apis.CStorPool, q common.QueueLoad) {
	var key string
	var err error
	if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil {
		runtime.HandleError(err)
		return
	}
	q.Key = key
	c.workqueue.AddRateLimited(q)
}
<|endoftext|>


<commit_before>package ginServer

import (
	"encoding/base64"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/DanielRenne/GoCore/core/extensions"
	"github.com/DanielRenne/GoCore/core/serverSettings"
	"github.com/gin-gonic/contrib/sessions"
	"github.com/gin-gonic/gin"
)

const TimeFormat = "Mon, 02 Jan 2006 15:04:05 GMT"
const StatusNotModified = 304 // RFC 7232, 4.1

var unixEpochTime = time.Unix(0, 0)

type ErrorResponse struct {
	Message string `json:"message"`
}

type LocaleLanguage struct {
	Locale   string
	Language string
}

func GetSessionKey(c *gin.Context, key string) string {
	session := sessions.Default(c)
	if strings.Contains(c.Request.Host, ".com") {
		session.Options(sessions.Options{MaxAge: 86400 * serverSettings.WebConfig.Application.SessionExpirationDays,
			Secure: serverSettings.WebConfig.Application.SessionSecureCookie,
			Domain: ginCookieDomain})
	}
	value := session.Get(key)
	if value == nil {
		return ""
	} else {
		return session.Get(key).(string)
	}
}

func SetSessionKey(c *gin.Context, key string, value string) {
	session := sessions.Default(c)
	if strings.Contains(c.Request.Host, ".com") {
		session.Options(sessions.Options{MaxAge: 86400 * serverSettings.WebConfig.Application.SessionExpirationDays,
			Secure: serverSettings.WebConfig.Application.SessionSecureCookie,
			Domain: ginCookieDomain})
	}
	session.Set(key, value)
	session.Save()
}

func SaveSession(c *gin.Context) {
	session := sessions.Default(c)
	if strings.Contains(c.Request.Host, ".com") {
		session.Options(sessions.Options{MaxAge: 86400 * serverSettings.WebConfig.Application.SessionExpirationDays,
			Secure: serverSettings.WebConfig.Application.SessionSecureCookie,
			Domain: ginCookieDomain})
	}
	session.Save()
}

func ClearSession(c *gin.Context) {
	session := sessions.Default(c)
	if strings.Contains(c.Request.Host, ".com") {
		session.Options(sessions.Options{MaxAge: 86400 * serverSettings.WebConfig.Application.SessionExpirationDays,
			Secure: serverSettings.WebConfig.Application.SessionSecureCookie,
			Domain: ginCookieDomain})
	}
	session.Clear()
}

func GetLocaleLanguage(c *gin.Context) (ll LocaleLanguage) {
	header := c.Request.Header.Get("Accept-Language")
	allLanguages := strings.Split(header, ";")

	locals := strings.Split(allLanguages[0], ",")
	localsSplit := strings.Split(locals[0], "-")

	if len(localsSplit) == 1 && len(locals) == 2 {
		localsSplit = strings.Split(locals[1], "-")
	}

	ll.Language = localsSplit[0]
	if len(localsSplit) == 2 {
		ll.Locale = localsSplit[1]
	}
	return
}

func GetRequestBody(c *gin.Context) ([]byte, error) {
	body := c.Request.Body
	return ioutil.ReadAll(body)
}

// Reads a file from the path parameter and returns to the client as text/html.
func ReadHTMLFile(path string, c *gin.Context) {
	page, err := ioutil.ReadFile(path)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	pageHTML := string(page)

	c.Header("Content-Type", "text/html")
	c.String(http.StatusOK, pageHTML)
}

func ReadJSFile(path string, c *gin.Context) {
	page, err := ioutil.ReadFile(path)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	pageHTML := string(page)

	c.Header("Content-Type", "text/javascript")
	c.String(http.StatusOK, pageHTML)
}

//Reads a file and responds with a base64 encoded string.  Primarily used for jquery ajax response binary data blob encoding.
func ReadFileBase64(path string, c *gin.Context) {
	page, err := ioutil.ReadFile(path)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	data := base64.StdEncoding.EncodeToString(page)
	c.Writer.Header().Set("Content-Length", extensions.IntToString(len(data)))
	c.String(http.StatusOK, data)
}

// Takes a string and returns to the client as text/html.
func RenderHTML(html string, c *gin.Context) {
	c.Header("Content-Type", "text/html")
	c.String(http.StatusOK, html)
}

// Reads a file from the path parameter and returns to the client application/json
func ReadJSONFile(path string, c *gin.Context) {
	js, err := ioutil.ReadFile(path)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}
	c.Header("Content-Type", "application/json")
	c.Writer.Write(js)
}

// Returns to the client application/json format for the passed interface.
func RespondJSON(v interface{}, c *gin.Context) {

	if v == nil {
		c.JSON(http.StatusNotFound, v)
		return
	}
	c.JSON(http.StatusOK, v)
}

//  Returns an byte array comprised of a JSON formated object with the error message.
func RespondError(message string) []byte {
	var msg ErrorResponse
	msg.Message = message
	b, _ := json.Marshal(msg)
	return b
}

func ReadGzipJSFile(path string, c *gin.Context) {

	c.Header("Content-Type", "application/javascript")
	c.Header("Content-Encoding", "gzip")
	c.File(path)
}

func RespondGzipJSFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "application/javascript")
	c.Header("Content-Encoding", "gzip")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func RespondJSFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "application/javascript")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func RespondTtfFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "application/x-font-ttf")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func RespondOtfFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "application/x-font-otf")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func RespondWoffFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "application/font-woff")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func RespondWoff2File(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "application/font-woff2")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func RespondEotFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "application/vnd.ms-fontobject")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func RespondSvgFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "image/svg+xml")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func ReadGzipCSSFile(path string, c *gin.Context) {

	c.Header("Content-Type", "text/css")
	c.Header("Content-Encoding", "gzip")
	c.File(path)

}

func RespondGzipCSSFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "text/css")
	c.Header("Content-Encoding", "gzip")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func ReadPngFile(path string, c *gin.Context) {
	c.Header("Content-Type", "image/png")
	c.File(path)
}

func ReadJpgFile(path string, c *gin.Context) {
	c.Header("Content-Type", "image/jpeg")
	c.File(path)
}

// modtime is the modification time of the resource to be served, or IsZero().
// return value is whether this request is now complete.
func CheckLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool {
	if modtime.IsZero() || modtime.Equal(unixEpochTime) {
		// If the file doesn't have a modtime (IsZero), or the modtime
		// is obviously garbage (Unix time == 0), then ignore modtimes
		// and don't process the If-Modified-Since header.
		return false
	}

	// The Date-Modified header truncates sub-second precision, so
	// use mtime < t+1s instead of mtime <= t to check for unmodified.
	if t, err := time.Parse(TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) {
		h := w.Header()
		delete(h, "Content-Type")
		delete(h, "Content-Length")
		w.WriteHeader(StatusNotModified)
		return true
	}
	w.Header().Set("Last-Modified", modtime.UTC().Format(TimeFormat))
	return false
}
<commit_msg>Catch concurrent map read writes in gin<commit_after>package ginServer

import (
	"encoding/base64"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/DanielRenne/GoCore/core/extensions"
	"github.com/DanielRenne/GoCore/core/serverSettings"
	"github.com/gin-gonic/contrib/sessions"
	"github.com/gin-gonic/gin"
)

const TimeFormat = "Mon, 02 Jan 2006 15:04:05 GMT"
const StatusNotModified = 304 // RFC 7232, 4.1

var unixEpochTime = time.Unix(0, 0)

type ErrorResponse struct {
	Message string `json:"message"`
}

type LocaleLanguage struct {
	Locale   string
	Language string
}

func GetSessionKey(c *gin.Context, key string) (sessionKey string) {
	//https://github.com/gin-gonic/gin/issues/700
	// defer needed to catch session.Get concurrent map read write.
	defer func() {
	    if r := recover(); r != nil {
	        return
	    }
	}()
	session := sessions.Default(c)
	if strings.Contains(c.Request.Host, ".com") {
		session.Options(sessions.Options{MaxAge: 86400 * serverSettings.WebConfig.Application.SessionExpirationDays,
			Secure: serverSettings.WebConfig.Application.SessionSecureCookie,
			Domain: ginCookieDomain})
	}
	value := session.Get(key)
	if value == nil {
		sessionKey = ""
		return
	} else {
		sessionKey = session.Get(key).(string)
		return
	}
}

func SetSessionKey(c *gin.Context, key string, value string) {
	session := sessions.Default(c)
	if strings.Contains(c.Request.Host, ".com") {
		session.Options(sessions.Options{MaxAge: 86400 * serverSettings.WebConfig.Application.SessionExpirationDays,
			Secure: serverSettings.WebConfig.Application.SessionSecureCookie,
			Domain: ginCookieDomain})
	}
	session.Set(key, value)
	session.Save()
}

func SaveSession(c *gin.Context) {
	session := sessions.Default(c)
	if strings.Contains(c.Request.Host, ".com") {
		session.Options(sessions.Options{MaxAge: 86400 * serverSettings.WebConfig.Application.SessionExpirationDays,
			Secure: serverSettings.WebConfig.Application.SessionSecureCookie,
			Domain: ginCookieDomain})
	}
	session.Save()
}

func ClearSession(c *gin.Context) {
	session := sessions.Default(c)
	if strings.Contains(c.Request.Host, ".com") {
		session.Options(sessions.Options{MaxAge: 86400 * serverSettings.WebConfig.Application.SessionExpirationDays,
			Secure: serverSettings.WebConfig.Application.SessionSecureCookie,
			Domain: ginCookieDomain})
	}
	session.Clear()
}

func GetLocaleLanguage(c *gin.Context) (ll LocaleLanguage) {
	header := c.Request.Header.Get("Accept-Language")
	allLanguages := strings.Split(header, ";")

	locals := strings.Split(allLanguages[0], ",")
	localsSplit := strings.Split(locals[0], "-")

	if len(localsSplit) == 1 && len(locals) == 2 {
		localsSplit = strings.Split(locals[1], "-")
	}

	ll.Language = localsSplit[0]
	if len(localsSplit) == 2 {
		ll.Locale = localsSplit[1]
	}
	return
}

func GetRequestBody(c *gin.Context) ([]byte, error) {
	body := c.Request.Body
	return ioutil.ReadAll(body)
}

// Reads a file from the path parameter and returns to the client as text/html.
func ReadHTMLFile(path string, c *gin.Context) {
	page, err := ioutil.ReadFile(path)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	pageHTML := string(page)

	c.Header("Content-Type", "text/html")
	c.String(http.StatusOK, pageHTML)
}

func ReadJSFile(path string, c *gin.Context) {
	page, err := ioutil.ReadFile(path)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	pageHTML := string(page)

	c.Header("Content-Type", "text/javascript")
	c.String(http.StatusOK, pageHTML)
}

//Reads a file and responds with a base64 encoded string.  Primarily used for jquery ajax response binary data blob encoding.
func ReadFileBase64(path string, c *gin.Context) {
	page, err := ioutil.ReadFile(path)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	data := base64.StdEncoding.EncodeToString(page)
	c.Writer.Header().Set("Content-Length", extensions.IntToString(len(data)))
	c.String(http.StatusOK, data)
}

// Takes a string and returns to the client as text/html.
func RenderHTML(html string, c *gin.Context) {
	c.Header("Content-Type", "text/html")
	c.String(http.StatusOK, html)
}

// Reads a file from the path parameter and returns to the client application/json
func ReadJSONFile(path string, c *gin.Context) {
	js, err := ioutil.ReadFile(path)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}
	c.Header("Content-Type", "application/json")
	c.Writer.Write(js)
}

// Returns to the client application/json format for the passed interface.
func RespondJSON(v interface{}, c *gin.Context) {

	if v == nil {
		c.JSON(http.StatusNotFound, v)
		return
	}
	c.JSON(http.StatusOK, v)
}

//  Returns an byte array comprised of a JSON formated object with the error message.
func RespondError(message string) []byte {
	var msg ErrorResponse
	msg.Message = message
	b, _ := json.Marshal(msg)
	return b
}

func ReadGzipJSFile(path string, c *gin.Context) {

	c.Header("Content-Type", "application/javascript")
	c.Header("Content-Encoding", "gzip")
	c.File(path)
}

func RespondGzipJSFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "application/javascript")
	c.Header("Content-Encoding", "gzip")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func RespondJSFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "application/javascript")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func RespondTtfFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "application/x-font-ttf")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func RespondOtfFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "application/x-font-otf")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func RespondWoffFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "application/font-woff")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func RespondWoff2File(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "application/font-woff2")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func RespondEotFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "application/vnd.ms-fontobject")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func RespondSvgFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "image/svg+xml")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func ReadGzipCSSFile(path string, c *gin.Context) {

	c.Header("Content-Type", "text/css")
	c.Header("Content-Encoding", "gzip")
	c.File(path)

}

func RespondGzipCSSFile(data []byte, modTime time.Time, c *gin.Context) {
	c.Header("Content-Type", "text/css")
	c.Header("Content-Encoding", "gzip")
	CheckLastModified(c.Writer, c.Request, modTime)
	c.Writer.Write(data)
}

func ReadPngFile(path string, c *gin.Context) {
	c.Header("Content-Type", "image/png")
	c.File(path)
}

func ReadJpgFile(path string, c *gin.Context) {
	c.Header("Content-Type", "image/jpeg")
	c.File(path)
}

// modtime is the modification time of the resource to be served, or IsZero().
// return value is whether this request is now complete.
func CheckLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool {
	if modtime.IsZero() || modtime.Equal(unixEpochTime) {
		// If the file doesn't have a modtime (IsZero), or the modtime
		// is obviously garbage (Unix time == 0), then ignore modtimes
		// and don't process the If-Modified-Since header.
		return false
	}

	// The Date-Modified header truncates sub-second precision, so
	// use mtime < t+1s instead of mtime <= t to check for unmodified.
	if t, err := time.Parse(TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) {
		h := w.Header()
		delete(h, "Content-Type")
		delete(h, "Content-Length")
		w.WriteHeader(StatusNotModified)
		return true
	}
	w.Header().Set("Last-Modified", modtime.UTC().Format(TimeFormat))
	return false
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cloudimagemetadata

import (
	"fmt"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	jujutxn "github.com/juju/txn"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"

	"github.com/juju/juju/environs/imagemetadata"
)

var logger = loggo.GetLogger("juju.state.cloudimagemetadata")

type cloudImageStorage struct {
	envUUID            string
	metadataCollection *mgo.Collection
	txnRunner          jujutxn.Runner
}

var _ Storage = (*cloudImageStorage)(nil)

// NewStorage constructs a new Storage that stores  image metadata
// in the provided collection using the provided transaction runner.
func NewStorage(
	envUUID string,
	metadataCollection *mgo.Collection,
	runner jujutxn.Runner,
) Storage {
	return &cloudImageStorage{
		envUUID:            envUUID,
		metadataCollection: metadataCollection,
		txnRunner:          runner,
	}
}

// AddMetadata implements Storage.AddMetadata.
func (s *cloudImageStorage) AddMetadata(metadata Metadata) error {
	newDoc := metadata.doc()
	buildTxn := func(attempt int) ([]txn.Op, error) {
		op := txn.Op{
			C:  s.metadataCollection.Name,
			Id: newDoc.Id,
		}
		if attempt == 0 {
			// On the first attempt we assume we're adding new cloud image metadata.
			op.Assert = txn.DocMissing
			op.Insert = &newDoc
			logger.Debugf("inserting cloud image metadata for %v", newDoc.Id)
		} else {
			// Subsequent attempts to add metadata will update the fields.
			op.Assert = txn.DocExists
			op.Update = bson.D{{"$set", newDoc.updates()}}
			logger.Debugf("updating cloud image metadata for %v", newDoc.Id)
		}
		return []txn.Op{op}, nil
	}

	err := s.txnRunner.Run(buildTxn)
	if err != nil {
		return errors.Annotatef(err, "cannot add cloud image metadata for %v", newDoc.Id)
	}
	return nil
}

// Metadata implements Storage.Metadata.
func (s *cloudImageStorage) Metadata(stream, series, arch string) (Metadata, error) {
	desiredID := createKey(stream, series, arch)
	doc, err := s.imagesMetadata(desiredID)
	if err != nil {
		return Metadata{}, errors.Trace(err)
	}
	return doc.metadata(), nil
}

// AllMetadata implements Storage.AllMetadata.
func (s *cloudImageStorage) AllMetadata() ([]Metadata, error) {
	var docs []imagesMetadataDoc
	if err := s.metadataCollection.Find(nil).All(&docs); err != nil {
		return nil, errors.Trace(err)
	}
	metadata := make([]Metadata, len(docs))
	for i, doc := range docs {
		metadata[i] = doc.metadata()
	}
	return metadata, nil
}

func (c *cloudImageStorage) imagesMetadata(desiredID string) (imagesMetadataDoc, error) {
	var doc imagesMetadataDoc
	logger.Infof("looking for cloud image metadata with id %v", desiredID)
	err := c.metadataCollection.Find(bson.D{{"_id", desiredID}}).One(&doc)
	if err == mgo.ErrNotFound {
		return doc, errors.NotFoundf("cloud image metadata with id %v", desiredID)
	}
	return doc, err
}

type imagesMetadataDoc struct {
	Id          string `bson:"_id"`
	Arch        string `bson:"arch"`
	Series      string `bson:"series"`
	Storage     string `bson:"root_store,omitempty"`
	VirtType    string `bson:"virt,omitempty"`
	RegionAlias string `bson:"crsn,omitempty"`
	RegionName  string `bson:"region,omitempty"`
	Endpoint    string `bson:"endpoint,omitempty"`
	Stream      string `bson:"stream,omitempty"`
}

func (m imagesMetadataDoc) metadata() Metadata {
	return Metadata{
		Series:      m.Series,
		Storage:     m.Storage,
		VirtType:    m.VirtType,
		Arch:        m.Arch,
		RegionAlias: m.RegionAlias,
		RegionName:  m.RegionName,
		Endpoint:    m.Endpoint,
		Stream:      m.Stream,
	}
}
func (m imagesMetadataDoc) updates() bson.D {
	return bson.D{
		{"root_store", m.Storage},
		{"virt", m.VirtType},
		{"arch", m.Arch},
		{"series", m.Series},
		{"crsn", m.RegionAlias},
		{"region", m.RegionName},
		{"endpoint", m.Endpoint},
		{"stream", m.Stream},
	}
}

func (m *Metadata) doc() imagesMetadataDoc {
	return imagesMetadataDoc{
		Id:          m.key(),
		Series:      m.Series,
		Storage:     m.Storage,
		VirtType:    m.VirtType,
		Arch:        m.Arch,
		RegionAlias: m.RegionAlias,
		RegionName:  m.RegionName,
		Endpoint:    m.Endpoint,
		Stream:      m.Stream,
	}
}

func streamKey(stream string) string {
	//stream is optional, default is empty.
	if stream != "" && stream != imagemetadata.ReleasedStream {
		return "." + stream
	}
	return ""
}

var createKey = func(stream, series, arch string) string {
	return fmt.Sprintf("%s-%s-%s", streamKey(stream), series, arch)
}

func (im *Metadata) key() string {
	return createKey(im.Stream, im.Series, im.Arch)
}
<commit_msg>Renamed method for clarity.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cloudimagemetadata

import (
	"fmt"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	jujutxn "github.com/juju/txn"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"

	"github.com/juju/juju/environs/imagemetadata"
)

var logger = loggo.GetLogger("juju.state.cloudimagemetadata")

type cloudImageStorage struct {
	envUUID            string
	metadataCollection *mgo.Collection
	txnRunner          jujutxn.Runner
}

var _ Storage = (*cloudImageStorage)(nil)

// NewStorage constructs a new Storage that stores  image metadata
// in the provided collection using the provided transaction runner.
func NewStorage(
	envUUID string,
	metadataCollection *mgo.Collection,
	runner jujutxn.Runner,
) Storage {
	return &cloudImageStorage{
		envUUID:            envUUID,
		metadataCollection: metadataCollection,
		txnRunner:          runner,
	}
}

// AddMetadata implements Storage.AddMetadata.
func (s *cloudImageStorage) AddMetadata(metadata Metadata) error {
	newDoc := metadata.mongoDoc()
	buildTxn := func(attempt int) ([]txn.Op, error) {
		op := txn.Op{
			C:  s.metadataCollection.Name,
			Id: newDoc.Id,
		}
		if attempt == 0 {
			// On the first attempt we assume we're adding new cloud image metadata.
			op.Assert = txn.DocMissing
			op.Insert = &newDoc
			logger.Debugf("inserting cloud image metadata for %v", newDoc.Id)
		} else {
			// Subsequent attempts to add metadata will update the fields.
			op.Assert = txn.DocExists
			op.Update = bson.D{{"$set", newDoc.updates()}}
			logger.Debugf("updating cloud image metadata for %v", newDoc.Id)
		}
		return []txn.Op{op}, nil
	}

	err := s.txnRunner.Run(buildTxn)
	if err != nil {
		return errors.Annotatef(err, "cannot add cloud image metadata for %v", newDoc.Id)
	}
	return nil
}

// Metadata implements Storage.Metadata.
func (s *cloudImageStorage) Metadata(stream, series, arch string) (Metadata, error) {
	desiredID := createKey(stream, series, arch)
	doc, err := s.imagesMetadata(desiredID)
	if err != nil {
		return Metadata{}, errors.Trace(err)
	}
	return doc.metadata(), nil
}

// AllMetadata implements Storage.AllMetadata.
func (s *cloudImageStorage) AllMetadata() ([]Metadata, error) {
	var docs []imagesMetadataDoc
	if err := s.metadataCollection.Find(nil).All(&docs); err != nil {
		return nil, errors.Trace(err)
	}
	metadata := make([]Metadata, len(docs))
	for i, doc := range docs {
		metadata[i] = doc.metadata()
	}
	return metadata, nil
}

func (c *cloudImageStorage) imagesMetadata(desiredID string) (imagesMetadataDoc, error) {
	var doc imagesMetadataDoc
	logger.Infof("looking for cloud image metadata with id %v", desiredID)
	err := c.metadataCollection.Find(bson.D{{"_id", desiredID}}).One(&doc)
	if err == mgo.ErrNotFound {
		return doc, errors.NotFoundf("cloud image metadata with id %v", desiredID)
	}
	return doc, err
}

type imagesMetadataDoc struct {
	Id          string `bson:"_id"`
	Arch        string `bson:"arch"`
	Series      string `bson:"series"`
	Storage     string `bson:"root_store,omitempty"`
	VirtType    string `bson:"virt,omitempty"`
	RegionAlias string `bson:"crsn,omitempty"`
	RegionName  string `bson:"region,omitempty"`
	Endpoint    string `bson:"endpoint,omitempty"`
	Stream      string `bson:"stream,omitempty"`
}

func (m imagesMetadataDoc) metadata() Metadata {
	return Metadata{
		Series:      m.Series,
		Storage:     m.Storage,
		VirtType:    m.VirtType,
		Arch:        m.Arch,
		RegionAlias: m.RegionAlias,
		RegionName:  m.RegionName,
		Endpoint:    m.Endpoint,
		Stream:      m.Stream,
	}
}
func (m imagesMetadataDoc) updates() bson.D {
	return bson.D{
		{"root_store", m.Storage},
		{"virt", m.VirtType},
		{"arch", m.Arch},
		{"series", m.Series},
		{"crsn", m.RegionAlias},
		{"region", m.RegionName},
		{"endpoint", m.Endpoint},
		{"stream", m.Stream},
	}
}

func (m *Metadata) mongoDoc() imagesMetadataDoc {
	return imagesMetadataDoc{
		Id:          m.key(),
		Series:      m.Series,
		Storage:     m.Storage,
		VirtType:    m.VirtType,
		Arch:        m.Arch,
		RegionAlias: m.RegionAlias,
		RegionName:  m.RegionName,
		Endpoint:    m.Endpoint,
		Stream:      m.Stream,
	}
}

func streamKey(stream string) string {
	//stream is optional, default is empty.
	if stream != "" && stream != imagemetadata.ReleasedStream {
		return "." + stream
	}
	return ""
}

var createKey = func(stream, series, arch string) string {
	return fmt.Sprintf("%s-%s-%s", streamKey(stream), series, arch)
}

func (im *Metadata) key() string {
	return createKey(im.Stream, im.Series, im.Arch)
}
<|endoftext|>


<commit_before>package web

import (
	"io"
	"log"
	"math"
	"net/http"
	"strconv"
	"strings"

	"github.com/bmizerany/pat"

	"github.com/go-imsto/imagid"
	"github.com/go-imsto/imsto/config"
	"github.com/go-imsto/imsto/storage"
)

// Handler ...
func Handler() http.Handler {
	mux := pat.New()
	mux.Get("/imsto/roofs", http.HandlerFunc(roofsHandler))

	mux.Post("/imsto/ticket", storage.CheckAPIKey(http.HandlerFunc(ticketHandlerPost)))
	mux.Get("/imsto/ticket", storage.CheckAPIKey(http.HandlerFunc(ticketHandlerGet)))

	mux.Post("/imsto/token", storage.CheckAPIKey(http.HandlerFunc(tokenHandler)))

	mux.Post("/imsto/:roof", storage.CheckAPIKey(secure(whiteList, storedHandler)))
	mux.Del("/imsto/:roof/:id", storage.CheckAPIKey(secure(whiteList, deleteHandler)))
	mux.Get("/imsto/:roof/id", http.HandlerFunc(GetOrHeadHandler))
	mux.Get("/imsto/:roof/metas/count", http.HandlerFunc(countHandler))
	mux.Get("/imsto/:roof/metas", http.HandlerFunc(browseHandler))
	// mux.Post("/imsto/:roof/token", http.HandlerFunc(tokenHandler))
	// mux.Post("/imsto/:roof/ticket", http.HandlerFunc(ticketHandler))

	return mux
}

func StageHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("X-Server", "IMSTO STAGE")

	item, err := storage.LoadPath(r.URL.Path)

	if err != nil {
		logger().Warnw("loadPath fail", "ref", r.Referer(), "err", err)
		if he, ok := err.(*storage.HttpError); ok {
			if he.Code == 302 {
				logger().Infow("redirect", "path", he.Path)
				http.Redirect(w, r, he.Path, he.Code)
				return
			}
		}
		writeJsonError(w, r, err)

		return
	}

	// log.Print(item)

	c := func(file io.ReadSeeker) {
		http.ServeContent(w, r, item.Name(), item.Modified(), file)
	}
	err = item.Walk(c)
	if err != nil {
		logger().Warnw("item walk fail", "item", item, "err", err)
		writeJsonError(w, r, err)
		return
	}
}

func roofsHandler(w http.ResponseWriter, r *http.Request) {
	m := newApiMeta(true)
	// m["roofs"] = config.Sections()
	writeJsonQuiet(w, r, newApiRes(m, config.GetSections()))
}

func browseHandler(w http.ResponseWriter, r *http.Request) {
	roof := r.URL.Query().Get(":roof")
	logger().Debugw("browse", "roof", roof, "query", r.URL.Query())
	var (
		limit  uint64
		offset uint64
	)

	if str := r.FormValue("rows"); str != "" {
		limit, _ = strconv.ParseUint(str, 10, 32)
		if limit < 1 {
			limit = 1
		}
	} else {
		limit = 20
	}

	if str := r.FormValue("skip"); str != "" {
		offset, _ = strconv.ParseUint(str, 10, 32)
	} else {
		var page uint64
		if str := r.FormValue("page"); str != "" {
			page, _ = strconv.ParseUint(str, 10, 32)
		}
		if page < 1 {
			page = 1
		}

		offset = limit * (page - 1)
	}

	sort := make(map[string]int)
	sort_name := r.FormValue("sort_name")
	sort_order := r.FormValue("sort_order")
	if sort_name != "" {
		var o int
		if strings.ToUpper(sort_order) == "ASC" {
			o = storage.ASCENDING
		} else {
			o = storage.DESCENDING
		}
		sort[sort_name] = o
	}

	filter := storage.MetaFilter{Tags: r.FormValue("tags")}

	mw := storage.NewMetaWrapper(roof)
	t, err := mw.Count(filter)
	if err != nil {
		// w.WriteHeader(http.StatusInternalServerError)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}

	a, err := mw.Browse(int(limit), int(offset), sort, filter)
	if err != nil {
		// w.WriteHeader(http.StatusInternalServerError)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}

	m := newApiMeta(true)
	m["rows"] = limit
	m["page"] = (offset + 1)
	m["skip"] = offset
	m["page_count"] = uint(math.Ceil(float64(t) / float64(limit)))

	m["total"] = t

	m["url_prefix"] = getURL(r.URL.Scheme, roof, "") + "/"
	m["version"] = config.Version
	writeJsonQuiet(w, r, newApiRes(m, a))
}

func countHandler(w http.ResponseWriter, r *http.Request) {
	roof := r.FormValue("roof")

	filter := storage.MetaFilter{Tags: r.FormValue("tags")}

	mw := storage.NewMetaWrapper(roof)
	t, err := mw.Count(filter)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}

	m := newApiMeta(true)
	m["total"] = t
	m["version"] = config.Version
	writeJsonQuiet(w, r, newApiRes(m, nil))
}

func GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
	id, err := imagid.ParseID(r.URL.Query().Get(":id"))
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}

	roof := r.URL.Query().Get(":roof")
	mw := storage.NewMetaWrapper(roof)
	entry, err := mw.GetMeta(id.String())
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}

	if r.Method == "HEAD" {
		return
	}
	url := getURL(r.URL.Scheme, roof, "orig/"+entry.Path)
	log.Printf("Get entry: %v", entry.Id)
	meta := newApiMeta(true)
	obj := struct {
		*storage.Entry
		OrigUrl string `json:"orig_url,omitempty"`
	}{
		Entry:   entry,
		OrigUrl: url,
	}
	writeJsonQuiet(w, r, newApiRes(meta, obj))
}

func getURL(scheme, roof, size string) string {
	return storage.GetURL(scheme, size)
}

func storedHandler(w http.ResponseWriter, r *http.Request) {
	var us uploadSchema
	err := Bind(r, &us)
	if err != nil {
		writeJsonError(w, r, err)
		return
	}
	if us.Roof == "" {
		us.Roof = r.URL.Query().Get(":roof")
	}
	if err = r.ParseMultipartForm(storage.DefaultMaxMemory); err != nil {
		log.Print("multipart form parse error:", err)
		writeJsonError(w, r, err)
		return
	}
	app, appOK := storage.AppFromContext(r.Context())
	if !appOK {
		w.WriteHeader(400)
		writeJson(w, r, "app error")
		return
	}
	_, err = app.VerifyToken(us.Token)
	if err != nil {
		writeJsonError(w, r, err)
		return
	}

	tags, _ := storage.ParseTags(us.Tags)
	entries := make(map[string][]*storage.Entry)
	for k, fhs := range r.MultipartForm.File {
		entries[k] = make([]*storage.Entry, len(fhs))
		for i, fh := range fhs {
			entries[k][i] = new(storage.Entry)
			log.Printf("%d name: %s, ctype: %s", i, fh.Filename, fh.Header.Get("Content-Type"))
			mime := fh.Header.Get("Content-Type")
			file, fe := fh.Open()
			if fe != nil {
				entries[k][i].Err = fe.Error()
			}

			logger().Infow("post upload", "name", fh.Filename, "mime", mime, "size", fh.Size)

			entry, ee := storage.PrepareReader(file, fh.Filename)
			if ee != nil {
				logger().Infow("prepare upload fail", "name", fh.Filename)
				entries[k][i].Err = ee.Error()
				continue
			}
			entry.AppId = app.Id
			entry.Author = storage.Author(us.User)
			// entry.Modified = lastModified
			entry.Tags = tags
			ee = <-entry.Store(us.Roof)
			if ee != nil {
				logger().Infow("stored fail", "i", i, "roof", us.Roof, "id", entry.Id, "err", ee)
				entries[k][i].Err = ee.Error()
				continue
			}
			logger().Infow("stored", "i", i, "roof", us.Roof, "id", entry.Id, "path", entry.Path)
			entries[k][i] = entry
		}
	}

	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}
	// log.Print(entries[0].Path)
	meta := newApiMeta(true)
	var roof = r.FormValue("roof")

	meta["stage_host"] = config.Current.StageHost
	meta["url_prefix"] = getURL(r.URL.Scheme, roof, "") + "/"
	meta["version"] = config.Version

	writeJsonQuiet(w, r, newApiRes(meta, entries))
}

func deleteHandler(w http.ResponseWriter, r *http.Request) {
	err := storage.Delete(r.URL.Query().Get(":roof"), r.URL.Query().Get(":id"))
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}

	meta := newApiMeta(true)
	writeJsonQuiet(w, r, newApiRes(meta, nil))
}

func tokenHandler(w http.ResponseWriter, r *http.Request) {
	var param tokenSchema
	if err := Bind(r, &param); err != nil {
		writeJsonError(w, r, err)
		return
	}
	app, appOK := storage.AppFromContext(r.Context())
	if !appOK {
		w.WriteHeader(400)
		writeJson(w, r, "app error")
		return
	}

	token := app.RequestNewToken(param.User)
	meta := newApiMeta(token != "")
	meta["token"] = token
	writeJsonQuiet(w, r, newApiRes(meta, nil))
}

func ticketHandlerPost(w http.ResponseWriter, r *http.Request) {
	var param ticketSchema
	if err := Bind(r, &param); err != nil {
		writeJsonError(w, r, err)
		return
	}
	app, appOK := storage.AppFromContext(r.Context())
	if !appOK {
		w.WriteHeader(400)
		writeJson(w, r, "app error")
		return
	}

	token, err := app.TicketRequestNew(param.Roof, param.Token, param.User, param.Prompt)
	if err != nil {
		writeJsonError(w, r, err)
		return
	}
	str := token.String()
	meta := newApiMeta(str != "")
	meta["token"] = str
	writeJsonQuiet(w, r, newApiRes(meta, nil))
}

func ticketHandlerGet(w http.ResponseWriter, r *http.Request) {
	var param ticketSchema
	if err := Bind(r, &param); err != nil {
		writeJsonError(w, r, err)
		return
	}
	app, appOK := storage.AppFromContext(r.Context())
	if !appOK {
		w.WriteHeader(400)
		writeJson(w, r, "app error")
		return
	}
	ticket, err := app.TicketRequestLoad(param.Token, param.Roof, param.User)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}
	meta := newApiMeta(false)
	meta["ok"] = true
	meta["ticket"] = ticket

	writeJsonQuiet(w, r, newApiRes(meta, nil))
}
<commit_msg>Fix getURL()<commit_after>package web

import (
	"io"
	"log"
	"math"
	"net/http"
	"strconv"
	"strings"

	"github.com/bmizerany/pat"

	"github.com/go-imsto/imagid"
	"github.com/go-imsto/imsto/config"
	"github.com/go-imsto/imsto/storage"
)

// Handler ...
func Handler() http.Handler {
	mux := pat.New()
	mux.Get("/imsto/roofs", http.HandlerFunc(roofsHandler))

	mux.Post("/imsto/ticket", storage.CheckAPIKey(http.HandlerFunc(ticketHandlerPost)))
	mux.Get("/imsto/ticket", storage.CheckAPIKey(http.HandlerFunc(ticketHandlerGet)))

	mux.Post("/imsto/token", storage.CheckAPIKey(http.HandlerFunc(tokenHandler)))

	mux.Post("/imsto/:roof", storage.CheckAPIKey(secure(whiteList, storedHandler)))
	mux.Del("/imsto/:roof/:id", storage.CheckAPIKey(secure(whiteList, deleteHandler)))
	mux.Get("/imsto/:roof/id", http.HandlerFunc(GetOrHeadHandler))
	mux.Get("/imsto/:roof/metas/count", http.HandlerFunc(countHandler))
	mux.Get("/imsto/:roof/metas", http.HandlerFunc(browseHandler))
	// mux.Post("/imsto/:roof/token", http.HandlerFunc(tokenHandler))
	// mux.Post("/imsto/:roof/ticket", http.HandlerFunc(ticketHandler))

	return mux
}

func StageHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("X-Server", "IMSTO STAGE")

	item, err := storage.LoadPath(r.URL.Path)

	if err != nil {
		logger().Warnw("loadPath fail", "ref", r.Referer(), "err", err)
		if he, ok := err.(*storage.HttpError); ok {
			if he.Code == 302 {
				logger().Infow("redirect", "path", he.Path)
				http.Redirect(w, r, he.Path, he.Code)
				return
			}
		}
		writeJsonError(w, r, err)

		return
	}

	// log.Print(item)

	c := func(file io.ReadSeeker) {
		http.ServeContent(w, r, item.Name(), item.Modified(), file)
	}
	err = item.Walk(c)
	if err != nil {
		logger().Warnw("item walk fail", "item", item, "err", err)
		writeJsonError(w, r, err)
		return
	}
}

func roofsHandler(w http.ResponseWriter, r *http.Request) {
	m := newApiMeta(true)
	// m["roofs"] = config.Sections()
	writeJsonQuiet(w, r, newApiRes(m, config.GetSections()))
}

func browseHandler(w http.ResponseWriter, r *http.Request) {
	roof := r.URL.Query().Get(":roof")
	logger().Debugw("browse", "roof", roof, "query", r.URL.Query())
	var (
		limit  uint64
		offset uint64
	)

	if str := r.FormValue("rows"); str != "" {
		limit, _ = strconv.ParseUint(str, 10, 32)
		if limit < 1 {
			limit = 1
		}
	} else {
		limit = 20
	}

	if str := r.FormValue("skip"); str != "" {
		offset, _ = strconv.ParseUint(str, 10, 32)
	} else {
		var page uint64
		if str := r.FormValue("page"); str != "" {
			page, _ = strconv.ParseUint(str, 10, 32)
		}
		if page < 1 {
			page = 1
		}

		offset = limit * (page - 1)
	}

	sort := make(map[string]int)
	sort_name := r.FormValue("sort_name")
	sort_order := r.FormValue("sort_order")
	if sort_name != "" {
		var o int
		if strings.ToUpper(sort_order) == "ASC" {
			o = storage.ASCENDING
		} else {
			o = storage.DESCENDING
		}
		sort[sort_name] = o
	}

	filter := storage.MetaFilter{Tags: r.FormValue("tags")}

	mw := storage.NewMetaWrapper(roof)
	t, err := mw.Count(filter)
	if err != nil {
		// w.WriteHeader(http.StatusInternalServerError)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}

	a, err := mw.Browse(int(limit), int(offset), sort, filter)
	if err != nil {
		// w.WriteHeader(http.StatusInternalServerError)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}

	m := newApiMeta(true)
	m["rows"] = limit
	m["page"] = (offset + 1)
	m["skip"] = offset
	m["page_count"] = uint(math.Ceil(float64(t) / float64(limit)))

	m["total"] = t

	m["url_prefix"] = getURL(r, "") + "/"
	m["version"] = config.Version
	writeJsonQuiet(w, r, newApiRes(m, a))
}

func countHandler(w http.ResponseWriter, r *http.Request) {
	roof := r.FormValue("roof")

	filter := storage.MetaFilter{Tags: r.FormValue("tags")}

	mw := storage.NewMetaWrapper(roof)
	t, err := mw.Count(filter)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}

	m := newApiMeta(true)
	m["total"] = t
	m["version"] = config.Version
	writeJsonQuiet(w, r, newApiRes(m, nil))
}

func GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
	id, err := imagid.ParseID(r.URL.Query().Get(":id"))
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}

	roof := r.URL.Query().Get(":roof")
	mw := storage.NewMetaWrapper(roof)
	entry, err := mw.GetMeta(id.String())
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}

	if r.Method == "HEAD" {
		return
	}
	url := getURL(r, "orig/"+entry.Path)
	log.Printf("Get entry: %v", entry.Id)
	meta := newApiMeta(true)
	obj := struct {
		*storage.Entry
		OrigUrl string `json:"orig_url,omitempty"`
	}{
		Entry:   entry,
		OrigUrl: url,
	}
	writeJsonQuiet(w, r, newApiRes(meta, obj))
}

func getURL(r *http.Request, size string) string {
	scheme := r.URL.Scheme
	if strings.HasPrefix(r.Header.Get("X-Scheme"), "https") {
		scheme = "https"
	}
	return storage.GetURL(scheme, size)
}

func storedHandler(w http.ResponseWriter, r *http.Request) {
	var us uploadSchema
	err := Bind(r, &us)
	if err != nil {
		writeJsonError(w, r, err)
		return
	}
	if us.Roof == "" {
		us.Roof = r.URL.Query().Get(":roof")
	}
	if err = r.ParseMultipartForm(storage.DefaultMaxMemory); err != nil {
		log.Print("multipart form parse error:", err)
		writeJsonError(w, r, err)
		return
	}
	app, appOK := storage.AppFromContext(r.Context())
	if !appOK {
		w.WriteHeader(400)
		writeJson(w, r, "app error")
		return
	}
	_, err = app.VerifyToken(us.Token)
	if err != nil {
		writeJsonError(w, r, err)
		return
	}

	tags, _ := storage.ParseTags(us.Tags)
	entries := make(map[string][]*storage.Entry)
	for k, fhs := range r.MultipartForm.File {
		entries[k] = make([]*storage.Entry, len(fhs))
		for i, fh := range fhs {
			entries[k][i] = new(storage.Entry)
			log.Printf("%d name: %s, ctype: %s", i, fh.Filename, fh.Header.Get("Content-Type"))
			mime := fh.Header.Get("Content-Type")
			file, fe := fh.Open()
			if fe != nil {
				entries[k][i].Err = fe.Error()
			}

			logger().Infow("post upload", "name", fh.Filename, "mime", mime, "size", fh.Size)

			entry, ee := storage.PrepareReader(file, fh.Filename)
			if ee != nil {
				logger().Infow("prepare upload fail", "name", fh.Filename)
				entries[k][i].Err = ee.Error()
				continue
			}
			entry.AppId = app.Id
			entry.Author = storage.Author(us.User)
			// entry.Modified = lastModified
			entry.Tags = tags
			ee = <-entry.Store(us.Roof)
			if ee != nil {
				logger().Infow("stored fail", "i", i, "roof", us.Roof, "id", entry.Id, "err", ee)
				entries[k][i].Err = ee.Error()
				continue
			}
			logger().Infow("stored", "i", i, "roof", us.Roof, "id", entry.Id, "path", entry.Path)
			entries[k][i] = entry
		}
	}

	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}
	// log.Print(entries[0].Path)
	meta := newApiMeta(true)

	meta["stage_host"] = config.Current.StageHost
	meta["url_prefix"] = getURL(r, "") + "/"
	meta["version"] = config.Version

	writeJsonQuiet(w, r, newApiRes(meta, entries))
}

func deleteHandler(w http.ResponseWriter, r *http.Request) {
	err := storage.Delete(r.URL.Query().Get(":roof"), r.URL.Query().Get(":id"))
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}

	meta := newApiMeta(true)
	writeJsonQuiet(w, r, newApiRes(meta, nil))
}

func tokenHandler(w http.ResponseWriter, r *http.Request) {
	var param tokenSchema
	if err := Bind(r, &param); err != nil {
		writeJsonError(w, r, err)
		return
	}
	app, appOK := storage.AppFromContext(r.Context())
	if !appOK {
		w.WriteHeader(400)
		writeJson(w, r, "app error")
		return
	}

	token := app.RequestNewToken(param.User)
	meta := newApiMeta(token != "")
	meta["token"] = token
	writeJsonQuiet(w, r, newApiRes(meta, nil))
}

func ticketHandlerPost(w http.ResponseWriter, r *http.Request) {
	var param ticketSchema
	if err := Bind(r, &param); err != nil {
		writeJsonError(w, r, err)
		return
	}
	app, appOK := storage.AppFromContext(r.Context())
	if !appOK {
		w.WriteHeader(400)
		writeJson(w, r, "app error")
		return
	}

	token, err := app.TicketRequestNew(param.Roof, param.Token, param.User, param.Prompt)
	if err != nil {
		writeJsonError(w, r, err)
		return
	}
	str := token.String()
	meta := newApiMeta(str != "")
	meta["token"] = str
	writeJsonQuiet(w, r, newApiRes(meta, nil))
}

func ticketHandlerGet(w http.ResponseWriter, r *http.Request) {
	var param ticketSchema
	if err := Bind(r, &param); err != nil {
		writeJsonError(w, r, err)
		return
	}
	app, appOK := storage.AppFromContext(r.Context())
	if !appOK {
		w.WriteHeader(400)
		writeJson(w, r, "app error")
		return
	}
	ticket, err := app.TicketRequestLoad(param.Token, param.Roof, param.User)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		log.Printf("ERROR: %s", err)
		writeJsonError(w, r, err)
		return
	}
	meta := newApiMeta(false)
	meta["ok"] = true
	meta["ticket"] = ticket

	writeJsonQuiet(w, r, newApiRes(meta, nil))
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package avalanche

import (
	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/snow/choices"
	"github.com/ava-labs/gecko/snow/consensus/avalanche"
	"github.com/ava-labs/gecko/snow/engine/common"
	"github.com/ava-labs/gecko/snow/engine/common/queue"
	"github.com/ava-labs/gecko/utils/formatting"
	"github.com/prometheus/client_golang/prometheus"
)

// BootstrapConfig ...
type BootstrapConfig struct {
	common.Config

	// VtxBlocked tracks operations that are blocked on vertices
	// TxBlocked tracks operations that are blocked on transactions
	VtxBlocked, TxBlocked *queue.Jobs

	State State
	VM    DAGVM
}

type bootstrapper struct {
	BootstrapConfig
	metrics
	common.Bootstrapper

	// vtxReqs prevents asking validators for the same vertex
	vtxReqs common.Requests

	pending    ids.Set
	finished   bool
	onFinished func()
}

// Initialize this engine.
func (b *bootstrapper) Initialize(config BootstrapConfig) {
	b.BootstrapConfig = config

	b.VtxBlocked.SetParser(&vtxParser{
		numAccepted: b.numBootstrappedVtx,
		numDropped:  b.numDroppedVtx,
		state:       b.State,
	})

	b.TxBlocked.SetParser(&txParser{
		numAccepted: b.numBootstrappedTx,
		numDropped:  b.numDroppedTx,
		vm:          b.VM,
	})

	config.Bootstrapable = b
	b.Bootstrapper.Initialize(config.Config)
}

// CurrentAcceptedFrontier ...
func (b *bootstrapper) CurrentAcceptedFrontier() ids.Set {
	acceptedFrontier := ids.Set{}
	acceptedFrontier.Add(b.State.Edge()...)
	return acceptedFrontier
}

// FilterAccepted ...
func (b *bootstrapper) FilterAccepted(containerIDs ids.Set) ids.Set {
	acceptedVtxIDs := ids.Set{}
	for _, vtxID := range containerIDs.List() {
		if vtx, err := b.State.GetVertex(vtxID); err == nil && vtx.Status() == choices.Accepted {
			acceptedVtxIDs.Add(vtxID)
		}
	}
	return acceptedVtxIDs
}

// ForceAccepted ...
func (b *bootstrapper) ForceAccepted(acceptedContainerIDs ids.Set) {
	for _, vtxID := range acceptedContainerIDs.List() {
		b.fetch(vtxID)
	}

	if numPending := b.pending.Len(); numPending == 0 {
		// TODO: This typically indicates bootstrapping has failed, so this
		// should be handled appropriately
		b.finish()
	}
}

// Put ...
func (b *bootstrapper) Put(vdr ids.ShortID, requestID uint32, vtxID ids.ID, vtxBytes []byte) {
	b.BootstrapConfig.Context.Log.Verbo("Put called for vertexID %s", vtxID)

	vtx, err := b.State.ParseVertex(vtxBytes)
	if err != nil {
		b.BootstrapConfig.Context.Log.Debug("ParseVertex failed due to %s for block:\n%s",
			err,
			formatting.DumpBytes{Bytes: vtxBytes})

		b.GetFailed(vdr, requestID)
		return
	}

	if !b.pending.Contains(vtx.ID()) {
		b.BootstrapConfig.Context.Log.Debug("Validator %s sent an unrequested vertex:\n%s",
			vdr,
			formatting.DumpBytes{Bytes: vtxBytes})

		b.GetFailed(vdr, requestID)
		return
	}

	b.addVertex(vtx)
}

// GetFailed ...
func (b *bootstrapper) GetFailed(vdr ids.ShortID, requestID uint32) {
	vtxID, ok := b.vtxReqs.Remove(vdr, requestID)
	if !ok {
		b.BootstrapConfig.Context.Log.Debug("GetFailed called without sending the corresponding Get message from %s",
			vdr)
		return
	}

	b.sendRequest(vtxID)
}

func (b *bootstrapper) fetch(vtxID ids.ID) {
	if b.pending.Contains(vtxID) {
		return
	}

	vtx, err := b.State.GetVertex(vtxID)
	if err != nil {
		b.sendRequest(vtxID)
		return
	}
	b.storeVertex(vtx)
}

func (b *bootstrapper) sendRequest(vtxID ids.ID) {
	validators := b.BootstrapConfig.Validators.Sample(1)
	if len(validators) == 0 {
		b.BootstrapConfig.Context.Log.Error("Dropping request for %s as there are no validators", vtxID)
		return
	}
	validatorID := validators[0].ID()
	b.RequestID++

	b.vtxReqs.RemoveAny(vtxID)
	b.vtxReqs.Add(validatorID, b.RequestID, vtxID)

	b.pending.Add(vtxID)
	b.BootstrapConfig.Sender.Get(validatorID, b.RequestID, vtxID)

	b.numPendingRequests.Set(float64(b.pending.Len()))
}

func (b *bootstrapper) addVertex(vtx avalanche.Vertex) {
	b.storeVertex(vtx)

	if numPending := b.pending.Len(); numPending == 0 {
		b.finish()
	}
}

func (b *bootstrapper) storeVertex(vtx avalanche.Vertex) {
	vts := []avalanche.Vertex{vtx}

	for len(vts) > 0 {
		newLen := len(vts) - 1
		vtx := vts[newLen]
		vts = vts[:newLen]

		vtxID := vtx.ID()
		switch status := vtx.Status(); status {
		case choices.Unknown:
			b.sendRequest(vtxID)
		case choices.Processing:
			b.pending.Remove(vtxID)

			if err := b.VtxBlocked.Push(&vertexJob{
				numAccepted: b.numBootstrappedVtx,
				numDropped:  b.numDroppedVtx,
				vtx:         vtx,
			}); err == nil {
				b.numBlockedVtx.Inc()
			}
			for _, tx := range vtx.Txs() {
				if err := b.TxBlocked.Push(&txJob{
					numAccepted: b.numBootstrappedVtx,
					numDropped:  b.numDroppedVtx,
					tx:          tx,
				}); err == nil {
					b.numBlockedTx.Inc()
				}
			}

			vts = append(vts, vtx.Parents()...)
		case choices.Accepted:
			b.BootstrapConfig.Context.Log.Verbo("Bootstrapping confirmed %s", vtxID)
		case choices.Rejected:
			b.BootstrapConfig.Context.Log.Error("Bootstrapping wants to accept %s, however it was previously rejected", vtxID)
		}
	}

	numPending := b.pending.Len()
	b.numPendingRequests.Set(float64(numPending))
}

func (b *bootstrapper) finish() {
	if b.finished {
		return
	}

	b.executeAll(b.TxBlocked, b.numBlockedTx)
	b.executeAll(b.VtxBlocked, b.numBlockedVtx)

	// Start consensus
	b.onFinished()
	b.finished = true
}

func (b *bootstrapper) executeAll(jobs *queue.Jobs, numBlocked prometheus.Gauge) {
	for job, err := jobs.Pop(); err == nil; job, err = jobs.Pop() {
		numBlocked.Dec()
		b.BootstrapConfig.Context.Log.Debug("Executing: %s", job.ID())
		if err := jobs.Execute(job); err != nil {
			b.BootstrapConfig.Context.Log.Warn("Error executing: %s", err)
		}
	}
}
<commit_msg>periodically print bootstrap progress to log. don't fetch ancestry of same vertex multiple times<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package avalanche

import (
	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/snow/choices"
	"github.com/ava-labs/gecko/snow/consensus/avalanche"
	"github.com/ava-labs/gecko/snow/engine/common"
	"github.com/ava-labs/gecko/snow/engine/common/queue"
	"github.com/ava-labs/gecko/utils/formatting"
	"github.com/prometheus/client_golang/prometheus"
)

// BootstrapConfig ...
type BootstrapConfig struct {
	common.Config

	// VtxBlocked tracks operations that are blocked on vertices
	// TxBlocked tracks operations that are blocked on transactions
	VtxBlocked, TxBlocked *queue.Jobs

	State State
	VM    DAGVM
}

type bootstrapper struct {
	BootstrapConfig
	metrics
	common.Bootstrapper

	// IDs of vertices that we're already in the process of getting
	// TODO: Find a better way to track; this keeps every single vertex's ID in memory when bootstrapping from nothing
	seen ids.Set

	numFetched uint64 // number of vertices that have been fetched from validators

	// vtxReqs prevents asking validators for the same vertex
	vtxReqs common.Requests

	// IDs of vertices that we have requested from other validators but haven't received
	pending    ids.Set
	finished   bool
	onFinished func()
}

// Initialize this engine.
func (b *bootstrapper) Initialize(config BootstrapConfig) {
	b.BootstrapConfig = config

	b.VtxBlocked.SetParser(&vtxParser{
		numAccepted: b.numBootstrappedVtx,
		numDropped:  b.numDroppedVtx,
		state:       b.State,
	})

	b.TxBlocked.SetParser(&txParser{
		numAccepted: b.numBootstrappedTx,
		numDropped:  b.numDroppedTx,
		vm:          b.VM,
	})

	config.Bootstrapable = b
	b.Bootstrapper.Initialize(config.Config)
}

// CurrentAcceptedFrontier ...
func (b *bootstrapper) CurrentAcceptedFrontier() ids.Set {
	acceptedFrontier := ids.Set{}
	acceptedFrontier.Add(b.State.Edge()...)
	return acceptedFrontier
}

// FilterAccepted ...
func (b *bootstrapper) FilterAccepted(containerIDs ids.Set) ids.Set {
	acceptedVtxIDs := ids.Set{}
	for _, vtxID := range containerIDs.List() {
		if vtx, err := b.State.GetVertex(vtxID); err == nil && vtx.Status() == choices.Accepted {
			acceptedVtxIDs.Add(vtxID)
		}
	}
	return acceptedVtxIDs
}

// ForceAccepted ...
func (b *bootstrapper) ForceAccepted(acceptedContainerIDs ids.Set) {
	for _, vtxID := range acceptedContainerIDs.List() {
		b.fetch(vtxID)
	}

	if numPending := b.pending.Len(); numPending == 0 {
		// TODO: This typically indicates bootstrapping has failed, so this
		// should be handled appropriately
		b.finish()
	}
}

// Put ...
func (b *bootstrapper) Put(vdr ids.ShortID, requestID uint32, vtxID ids.ID, vtxBytes []byte) {
	vtx, err := b.State.ParseVertex(vtxBytes)
	if err != nil {
		b.BootstrapConfig.Context.Log.Debug("ParseVertex failed due to %s for block:\n%s",
			err,
			formatting.DumpBytes{Bytes: vtxBytes})

		b.GetFailed(vdr, requestID)
		return
	}

	if !b.pending.Contains(vtx.ID()) {
		b.BootstrapConfig.Context.Log.Debug("Validator %s sent an unrequested vertex:\n%s",
			vdr,
			formatting.DumpBytes{Bytes: vtxBytes})

		b.GetFailed(vdr, requestID)
		return
	}

	b.addVertex(vtx)
}

// GetFailed ...
func (b *bootstrapper) GetFailed(vdr ids.ShortID, requestID uint32) {
	vtxID, ok := b.vtxReqs.Remove(vdr, requestID)
	if !ok {
		b.BootstrapConfig.Context.Log.Debug("GetFailed called without sending the corresponding Get message from %s",
			vdr)
		return
	}

	b.sendRequest(vtxID)
}

func (b *bootstrapper) fetch(vtxID ids.ID) {
	if b.pending.Contains(vtxID) {
		return
	}

	vtx, err := b.State.GetVertex(vtxID)
	if err != nil {
		b.sendRequest(vtxID)
		return
	}
	b.storeVertex(vtx)
}

func (b *bootstrapper) sendRequest(vtxID ids.ID) {
	validators := b.BootstrapConfig.Validators.Sample(1)
	if len(validators) == 0 {
		b.BootstrapConfig.Context.Log.Error("Dropping request for %s as there are no validators", vtxID)
		return
	}
	validatorID := validators[0].ID()
	b.RequestID++

	b.vtxReqs.RemoveAny(vtxID)
	b.vtxReqs.Add(validatorID, b.RequestID, vtxID)

	b.pending.Add(vtxID)
	b.BootstrapConfig.Sender.Get(validatorID, b.RequestID, vtxID)

	b.numPendingRequests.Set(float64(b.pending.Len()))
}

func (b *bootstrapper) addVertex(vtx avalanche.Vertex) {
	b.storeVertex(vtx)

	if numPending := b.pending.Len(); numPending == 0 {
		b.finish()
	}
}

func (b *bootstrapper) storeVertex(vtx avalanche.Vertex) {
	vts := []avalanche.Vertex{vtx}
	b.numFetched++
	if b.numFetched%2500 == 0 { // perioidcally inform user of progress
		b.BootstrapConfig.Context.Log.Info("bootstrapping has fetched %d vertices", b.numFetched)
	}

	for len(vts) > 0 {
		newLen := len(vts) - 1
		vtx := vts[newLen]
		vts = vts[:newLen]

		vtxID := vtx.ID()
		switch status := vtx.Status(); status {
		case choices.Unknown:
			b.sendRequest(vtxID)
		case choices.Processing:
			b.pending.Remove(vtxID)

			if err := b.VtxBlocked.Push(&vertexJob{
				numAccepted: b.numBootstrappedVtx,
				numDropped:  b.numDroppedVtx,
				vtx:         vtx,
			}); err == nil {
				b.numBlockedVtx.Inc()
			} else {
				b.BootstrapConfig.Context.Log.Verbo("couldn't push to vtxBlocked")
			}
			for _, tx := range vtx.Txs() {
				if err := b.TxBlocked.Push(&txJob{
					numAccepted: b.numBootstrappedVtx,
					numDropped:  b.numDroppedVtx,
					tx:          tx,
				}); err == nil {
					b.numBlockedTx.Inc()
				} else {
					b.BootstrapConfig.Context.Log.Verbo("couldn't push to txBlocked")
				}
			}
			parentsToAdd := []avalanche.Vertex{}
			parentsToAddIDs := ids.Set{} // ToDO remove...only here for debug
			for _, parent := range vtx.Parents() {
				if !b.seen.Contains(parent.ID()) {
					parentsToAdd = append(parentsToAdd, parent)
					parentsToAddIDs.Add(parent.ID())
				}
			}
			vts = append(vts, parentsToAdd...)
			b.seen.Add(parentsToAddIDs.List()...)
		case choices.Accepted:
			b.BootstrapConfig.Context.Log.Verbo("Bootstrapping confirmed %s", vtxID)
		case choices.Rejected:
			b.BootstrapConfig.Context.Log.Error("Bootstrapping wants to accept %s, however it was previously rejected", vtxID)
		}
	}

	numPending := b.pending.Len()
	b.numPendingRequests.Set(float64(numPending))
}

func (b *bootstrapper) finish() {
	if b.finished {
		return
	}
	b.BootstrapConfig.Context.Log.Info("bootstrapping finished fetching vertices. executing state transitions...")

	b.executeAll(b.TxBlocked, b.numBlockedTx)
	b.executeAll(b.VtxBlocked, b.numBlockedVtx)

	// Start consensus
	b.BootstrapConfig.Context.Log.Info("done bootstrapping")
	b.onFinished()
	b.seen = ids.Set{}
	b.finished = true
}

func (b *bootstrapper) executeAll(jobs *queue.Jobs, numBlocked prometheus.Gauge) {
	for job, err := jobs.Pop(); err == nil; job, err = jobs.Pop() {
		numBlocked.Dec()
		b.BootstrapConfig.Context.Log.Debug("Executing: %s", job.ID())
		if err := jobs.Execute(job); err != nil {
			b.BootstrapConfig.Context.Log.Warn("Error executing: %s", err)
		}
	}
}
<|endoftext|>


<commit_before>/*
                       WARNING WARNING WARNING

                Attention all potential contributors

   This testfile is not in the best state. We've been slowly transitioning
   from the built in "testing" package to using Ginkgo. As you can see, we've
   changed the format, but a lot of the setup, test body, descriptions, etc
   are either hardcoded, completely lacking, or misleading.

   For example:

   Describe("Testing with ginkgo"...)      // This is not a great description
   It("TestDoesSoemthing"...)              // This is a horrible description

   Describe("create-user command"...       // Describe the actual object under test
   It("creates a user when provided ..."   // this is more descriptive

   For good examples of writing Ginkgo tests for the cli, refer to

   src/cf/commands/application/delete_app_test.go
   src/cf/terminal/ui_test.go
   src/github.com/cloudfoundry/loggregator_consumer/consumer_test.go
*/

package space_test

import (
	. "cf/commands/space"
	"cf/commands/user"
	"cf/models"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	testapi "testhelpers/api"
	testcmd "testhelpers/commands"
	testconfig "testhelpers/configuration"
	"testhelpers/maker"
	testreq "testhelpers/requirements"
	testterm "testhelpers/terminal"

	. "testhelpers/matchers"
)

var (
	defaultReqFactory *testreq.FakeReqFactory
	configSpace       models.SpaceFields
	configOrg         models.OrganizationFields
	defaultSpace      models.Space
	defaultSpaceRepo  *testapi.FakeSpaceRepository
	defaultOrgRepo    *testapi.FakeOrgRepository
	defaultUserRepo   *testapi.FakeUserRepository
)

func resetSpaceDefaults() {
	defaultReqFactory = &testreq.FakeReqFactory{LoginSuccess: true, TargetedOrgSuccess: true}
	configOrg = models.OrganizationFields{}
	configOrg.Name = "my-org"
	configOrg.Guid = "my-org-guid"

	configSpace = models.SpaceFields{}
	configSpace.Name = "config-space"
	configSpace.Guid = "config-space-guid"

	defaultSpace = models.Space{}
	defaultSpace.Name = "my-space"
	defaultSpace.Guid = "my-space-guid"
	defaultSpace.Organization = configOrg

	defaultSpaceRepo = &testapi.FakeSpaceRepository{
		CreateSpaceSpace: defaultSpace,
	}

	defaultUserRepo = &testapi.FakeUserRepository{}
	defaultOrgRepo = &testapi.FakeOrgRepository{}
}

func callCreateSpace(args []string, requirementsFactory *testreq.FakeReqFactory, spaceRepo *testapi.FakeSpaceRepository, orgRepo *testapi.FakeOrgRepository, userRepo *testapi.FakeUserRepository) (ui *testterm.FakeUI) {
	ui = new(testterm.FakeUI)
	ctxt := testcmd.NewContext("create-space", args)

	configRepo := testconfig.NewRepositoryWithDefaults()
	spaceRoleSetter := user.NewSetSpaceRole(ui, configRepo, spaceRepo, userRepo)
	cmd := NewCreateSpace(ui, configRepo, spaceRoleSetter, spaceRepo, orgRepo, userRepo)
	testcmd.RunCommand(cmd, ctxt, requirementsFactory)
	return
}

var _ = Describe("Testing with ginkgo", func() {
	It("TestCreateSpaceFailsWithUsage", func() {
		resetSpaceDefaults()
		requirementsFactory := &testreq.FakeReqFactory{}

		ui := callCreateSpace([]string{}, requirementsFactory, defaultSpaceRepo, defaultOrgRepo, defaultUserRepo)
		Expect(ui.FailedWithUsage).To(BeTrue())

		ui = callCreateSpace([]string{"my-space"}, requirementsFactory, defaultSpaceRepo, defaultOrgRepo, defaultUserRepo)
		Expect(ui.FailedWithUsage).To(BeFalse())
	})
	It("TestCreateSpaceRequirements", func() {

		resetSpaceDefaults()
		requirementsFactory := &testreq.FakeReqFactory{LoginSuccess: true, TargetedOrgSuccess: true}
		callCreateSpace([]string{"my-space"}, requirementsFactory, defaultSpaceRepo, defaultOrgRepo, defaultUserRepo)
		Expect(testcmd.CommandDidPassRequirements).To(BeTrue())

		requirementsFactory = &testreq.FakeReqFactory{LoginSuccess: true, TargetedOrgSuccess: false}
		callCreateSpace([]string{"my-space"}, requirementsFactory, defaultSpaceRepo, defaultOrgRepo, defaultUserRepo)
		Expect(testcmd.CommandDidPassRequirements).To(BeFalse())

		requirementsFactory = &testreq.FakeReqFactory{LoginSuccess: false, TargetedOrgSuccess: true}
		callCreateSpace([]string{"my-space"}, requirementsFactory, defaultSpaceRepo, defaultOrgRepo, defaultUserRepo)
		Expect(testcmd.CommandDidPassRequirements).To(BeFalse())

		requirementsFactory = &testreq.FakeReqFactory{LoginSuccess: true, TargetedOrgSuccess: false}
		callCreateSpace([]string{"-o", "some-org", "my-space"}, requirementsFactory, defaultSpaceRepo, defaultOrgRepo, defaultUserRepo)
		Expect(testcmd.CommandDidPassRequirements).To(BeTrue())
	})
	It("TestCreateSpace", func() {

		resetSpaceDefaults()
		ui := callCreateSpace([]string{"my-space"}, defaultReqFactory, defaultSpaceRepo, defaultOrgRepo, defaultUserRepo)

		Expect(ui.Outputs).To(ContainSubstrings(
			[]string{"Creating space", "my-space", "my-org", "my-user"},
			[]string{"OK"},
			[]string{"Assigning", "my-user", "my-space", models.SpaceRoleToUserInput[models.SPACE_MANAGER]},
			[]string{"Assigning", "my-user", "my-space", models.SpaceRoleToUserInput[models.SPACE_DEVELOPER]},
			[]string{"TIP"},
		))

		Expect(defaultSpaceRepo.CreateSpaceName).To(Equal("my-space"))
		Expect(defaultSpaceRepo.CreateSpaceOrgGuid).To(Equal("my-org-guid"))
		Expect(defaultUserRepo.SetSpaceRoleUserGuid).To(Equal("my-user-guid"))
		Expect(defaultUserRepo.SetSpaceRoleSpaceGuid).To(Equal("my-space-guid"))
		Expect(defaultUserRepo.SetSpaceRoleRole).To(Equal(models.SPACE_DEVELOPER))
	})

	It("TestCreateSpaceInOrg", func() {

		resetSpaceDefaults()

		org := maker.NewOrg(maker.Overrides{"name": "other-org"})
		defaultOrgRepo.Organizations = []models.Organization{org}

		ui := callCreateSpace([]string{"-o", "other-org", "my-space"}, defaultReqFactory, defaultSpaceRepo, defaultOrgRepo, defaultUserRepo)

		Expect(ui.Outputs).To(ContainSubstrings(
			[]string{"Creating space", "my-space", "other-org", "my-user"},
			[]string{"OK"},
			[]string{"Assigning", "my-user", "my-space", models.SpaceRoleToUserInput[models.SPACE_MANAGER]},
			[]string{"Assigning", "my-user", "my-space", models.SpaceRoleToUserInput[models.SPACE_DEVELOPER]},
			[]string{"TIP"},
		))

		Expect(defaultSpaceRepo.CreateSpaceName).To(Equal("my-space"))
		Expect(defaultSpaceRepo.CreateSpaceOrgGuid).To(Equal(org.Guid))
		Expect(defaultUserRepo.SetSpaceRoleUserGuid).To(Equal("my-user-guid"))
		Expect(defaultUserRepo.SetSpaceRoleSpaceGuid).To(Equal("my-space-guid"))
		Expect(defaultUserRepo.SetSpaceRoleRole).To(Equal(models.SPACE_DEVELOPER))
	})

	It("TestCreateSpaceInOrgWhenTheOrgDoesNotExist", func() {

		resetSpaceDefaults()

		defaultOrgRepo.FindByNameNotFound = true

		ui := callCreateSpace([]string{"-o", "cool-organization", "my-space"}, defaultReqFactory, defaultSpaceRepo, defaultOrgRepo, defaultUserRepo)

		Expect(ui.Outputs).To(ContainSubstrings(
			[]string{"FAILED"},
			[]string{"cool-organization", "does not exist"},
		))

		Expect(defaultSpaceRepo.CreateSpaceName).To(Equal(""))
	})

	It("TestCreateSpaceInOrgWhenErrorFindingOrg", func() {

		resetSpaceDefaults()

		defaultOrgRepo.FindByNameErr = true

		ui := callCreateSpace([]string{"-o", "cool-organization", "my-space"}, defaultReqFactory, defaultSpaceRepo, defaultOrgRepo, defaultUserRepo)

		Expect(ui.Outputs).To(ContainSubstrings(
			[]string{"FAILED"},
			[]string{"error"},
		))

		Expect(defaultSpaceRepo.CreateSpaceName).To(Equal(""))
	})

	It("TestCreateSpaceWhenItAlreadyExists", func() {

		resetSpaceDefaults()
		defaultSpaceRepo.CreateSpaceExists = true
		ui := callCreateSpace([]string{"my-space"}, defaultReqFactory, defaultSpaceRepo, defaultOrgRepo, defaultUserRepo)

		Expect(ui.Outputs).To(ContainSubstrings(
			[]string{"Creating space", "my-space"},
			[]string{"OK"},
			[]string{"my-space", "already exists"},
		))

		Expect(ui.Outputs).ToNot(ContainSubstrings(
			[]string{"Assigning", "my-user", "my-space", models.SpaceRoleToUserInput[models.SPACE_MANAGER]},
		))

		Expect(defaultSpaceRepo.CreateSpaceName).To(Equal(""))
		Expect(defaultSpaceRepo.CreateSpaceOrgGuid).To(Equal(""))
		Expect(defaultUserRepo.SetSpaceRoleUserGuid).To(Equal(""))
		Expect(defaultUserRepo.SetSpaceRoleSpaceGuid).To(Equal(""))
	})
})
<commit_msg>cleanup create-space tests<commit_after>package space_test

import (
	"cf/commands/user"
	"cf/configuration"
	"cf/models"
	testapi "testhelpers/api"
	testcmd "testhelpers/commands"
	testconfig "testhelpers/configuration"
	"testhelpers/maker"
	testreq "testhelpers/requirements"
	testterm "testhelpers/terminal"

	. "cf/commands/space"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "testhelpers/matchers"
)

var _ = Describe("create-space command", func() {
	var (
		ui                  *testterm.FakeUI
		requirementsFactory *testreq.FakeReqFactory
		configSpace         models.SpaceFields
		configOrg           models.OrganizationFields
		configRepo          configuration.ReadWriter
		spaceRepo           *testapi.FakeSpaceRepository
		orgRepo             *testapi.FakeOrgRepository
		userRepo            *testapi.FakeUserRepository
		spaceRoleSetter     user.SpaceRoleSetter
	)

	runCommand := func(args ...string) {
		cmd := NewCreateSpace(ui, configRepo, spaceRoleSetter, spaceRepo, orgRepo, userRepo)
		testcmd.RunCommand(cmd, testcmd.NewContext("create-space", args), requirementsFactory)
	}

	BeforeEach(func() {
		ui = &testterm.FakeUI{}
		configRepo = testconfig.NewRepositoryWithDefaults()

		orgRepo = &testapi.FakeOrgRepository{}
		userRepo = &testapi.FakeUserRepository{}
		spaceRoleSetter = user.NewSetSpaceRole(ui, configRepo, spaceRepo, userRepo)

		requirementsFactory = &testreq.FakeReqFactory{LoginSuccess: true, TargetedOrgSuccess: true}
		configOrg = models.OrganizationFields{
			Name: "my-org",
			Guid: "my-org-guid",
		}

		configSpace = models.SpaceFields{
			Name: "config-space",
			Guid: "config-space-guid",
		}

		spaceRepo = &testapi.FakeSpaceRepository{
			CreateSpaceSpace: maker.NewSpace(maker.Overrides{"name": "my-space", "guid": "my-space-guid", "organization": configOrg}),
		}
		Expect(spaceRepo.CreateSpaceSpace.Name).To(Equal("my-space"))
	})

	Describe("Requirements", func() {
		It("fails with usage when no arguments are passed", func() {
			runCommand()
			Expect(ui.FailedWithUsage).To(BeTrue())
		})

		Context("when not logged in", func() {
			BeforeEach(func() {
				requirementsFactory.LoginSuccess = false
			})

			It("fails requirements", func() {
				runCommand("some-space")
				Expect(testcmd.CommandDidPassRequirements).To(BeFalse())
			})
		})

		Context("when a org is not targeted", func() {
			BeforeEach(func() {
				requirementsFactory.TargetedOrgSuccess = false
			})

			It("fails requirements", func() {
				runCommand("what-is-space?")
				Expect(testcmd.CommandDidPassRequirements).To(BeFalse())
			})
		})
	})

	It("creates a space", func() {
		runCommand("my-space")
		Expect(ui.Outputs).To(ContainSubstrings(
			[]string{"Creating space", "my-space", "my-org", "my-user"},
			[]string{"OK"},
			[]string{"Assigning", models.SpaceRoleToUserInput[models.SPACE_MANAGER], "my-user", "my-space"},
			[]string{"Assigning", models.SpaceRoleToUserInput[models.SPACE_DEVELOPER], "my-user", "my-space"},
			[]string{"TIP"},
		))

		Expect(spaceRepo.CreateSpaceName).To(Equal("my-space"))
		Expect(spaceRepo.CreateSpaceOrgGuid).To(Equal("my-org-guid"))
		Expect(userRepo.SetSpaceRoleUserGuid).To(Equal("my-user-guid"))
		Expect(userRepo.SetSpaceRoleSpaceGuid).To(Equal("my-space-guid"))
		Expect(userRepo.SetSpaceRoleRole).To(Equal(models.SPACE_DEVELOPER))
	})

	It("warns the user when a space with that name already exists", func() {
		spaceRepo.CreateSpaceExists = true
		runCommand("my-space")

		Expect(ui.Outputs).To(ContainSubstrings(
			[]string{"Creating space", "my-space"},
			[]string{"OK"},
		))
		Expect(ui.WarnOutputs).To(ContainSubstrings([]string{"my-space", "already exists"}))
		Expect(ui.Outputs).ToNot(ContainSubstrings(
			[]string{"Assigning", "my-user", "my-space", models.SpaceRoleToUserInput[models.SPACE_MANAGER]},
		))

		Expect(spaceRepo.CreateSpaceName).To(Equal(""))
		Expect(spaceRepo.CreateSpaceOrgGuid).To(Equal(""))
		Expect(userRepo.SetSpaceRoleUserGuid).To(Equal(""))
		Expect(userRepo.SetSpaceRoleSpaceGuid).To(Equal(""))
	})

	Context("when the -o flag is provided", func() {
		It("creates a space within that org", func() {
			org := maker.NewOrg(maker.Overrides{"name": "other-org"})
			orgRepo.Organizations = []models.Organization{org}

			runCommand("-o", "other-org", "my-space")

			Expect(ui.Outputs).To(ContainSubstrings(
				[]string{"Creating space", "my-space", "other-org", "my-user"},
				[]string{"OK"},
				[]string{"Assigning", "my-user", "my-space", models.SpaceRoleToUserInput[models.SPACE_MANAGER]},
				[]string{"Assigning", "my-user", "my-space", models.SpaceRoleToUserInput[models.SPACE_DEVELOPER]},
				[]string{"TIP"},
			))

			Expect(spaceRepo.CreateSpaceName).To(Equal("my-space"))
			Expect(spaceRepo.CreateSpaceOrgGuid).To(Equal(org.Guid))
			Expect(userRepo.SetSpaceRoleUserGuid).To(Equal("my-user-guid"))
			Expect(userRepo.SetSpaceRoleSpaceGuid).To(Equal("my-space-guid"))
			Expect(userRepo.SetSpaceRoleRole).To(Equal(models.SPACE_DEVELOPER))
		})

		It("fails when the org provided does not exist", func() {
			orgRepo.FindByNameNotFound = true
			runCommand("-o", "cool-organization", "my-space")

			Expect(ui.Outputs).To(ContainSubstrings(
				[]string{"FAILED"},
				[]string{"cool-organization", "does not exist"},
			))

			Expect(spaceRepo.CreateSpaceName).To(Equal(""))
		})

		It("fails when finding the org returns an error", func() {
			orgRepo.FindByNameErr = true
			runCommand("-o", "cool-organization", "my-space")

			Expect(ui.Outputs).To(ContainSubstrings(
				[]string{"FAILED"},
				[]string{"Error"},
			))

			Expect(spaceRepo.CreateSpaceName).To(Equal(""))
		})
	})
})
<|endoftext|>