<commit_before>package lib

import (
	"strconv"
	"strings"
	"time"
)

type BaseItem struct {
	*HaveID
	*HaveProjectID
	Content string `json:"content"`
	UserID  int    `json:"user_id"`
}

type CompletedItem struct {
	*BaseItem
	CompletedDate string      `json:"completed_date"`
	MetaData      interface{} `json:"meta_data"`
	TaskID        int         `json:"task_id"`
}

func (item CompletedItem) CompletedDateTime() time.Time {
	t, _ := time.Parse(DateFormat, item.CompletedDate)
	return t
}

type CompletedItems []CompletedItem

type Item struct {
	*BaseItem
	AllDay         bool        `json:"all_day"`
	AssignedByUID  int         `json:"assigned_by_uid"`
	Checked        int         `json:"checked"`
	Collapsed      int         `json:"collapsed"`
	DateAdded      string      `json:"date_added"`
	DateLang       string      `json:"date_lang"`
	DateString     string      `json:"date_string"`
	DayOrder       int         `json:"day_order"`
	DueDateUtc     string      `json:"due_date_utc"`
	HasMoreNotes   bool        `json:"has_more_notes"`
	InHistory      int         `json:"in_history"`
	Indent         int         `json:"indent"`
	IsArchived     int         `json:"is_archived"`
	IsDeleted      int         `json:"is_deleted"`
	ItemOrder      int         `json:"item_order"`
	LabelIDs       []int       `json:"labels"`
	ParentID       interface{} `json:"parent_id"`
	Priority       int         `json:"priority"`
	ResponsibleUID interface{} `json:"responsible_uid"`
	SyncID         interface{} `json:"sync_id"`
}

type Items []Item

func (item Item) DueDateTime() time.Time {
	t, _ := time.Parse(DateFormat, item.DueDateUtc)
	return t
}

func (items Items) FindByID(id int) (Item, error) {
	for _, item := range items {
		if item.ID == id {
			return item, nil
		}
	}
	return Item{}, FindFailed
}

func (item Item) AddParam() interface{} {
	param := map[string]interface{}{}
	if item.Content != "" {
		param["content"] = item.Content
	}
	if item.DateString != "" {
		param["date_string"] = item.DateString
	}
	if len(item.LabelIDs) != 0 {
		param["labels"] = item.LabelIDs
	}
	if item.Priority != 0 {
		param["priority"] = item.Priority
	}
	if item.ProjectID != 0 {
		param["project_id"] = item.ProjectID
	}
	return param
}

func (item Item) UpdateParam() interface{} {
	param := map[string]interface{}{}
	if item.ID != 0 {
		param["id"] = item.ID
	}
	if item.Content != "" {
		param["content"] = item.Content
	}
	if item.DateString != "" {
		param["date_string"] = item.DateString
	}
	if len(item.LabelIDs) != 0 {
		param["labels"] = item.LabelIDs
	}
	if item.Priority != 0 {
		param["priority"] = item.Priority
	}
	return param
}

func (item Item) MoveParam(to_project Project) interface{} {
	param := map[string]interface{}{
		"project_items": map[string][]int{
			strconv.Itoa(item.ProjectID): []int{item.ID},
		},
		"to_project": to_project.ID,
	}
	return param
}

func (item Item) LabelsString(labels Labels) string {
	label_names := make([]string, 0)
	for _, label_id := range item.LabelIDs {
		label, err := labels.FindByID(label_id)
		if err != nil {
			return "Error"
		}
		label_names = append(label_names, "@"+label.Name)
	}
	return strings.Join(label_names, ",")
}

func AddItem(item Item, token string) error {
	commands := Commands{
		NewCommand("item_add", item.AddParam()),
	}
	_, err := SyncRequest(commands.UrlValues(token))
	return err
}

func UpdateItem(item Item, token string) error {
	commands := Commands{
		NewCommand("item_update", item.UpdateParam()),
	}
	_, err := SyncRequest(commands.UrlValues(token))
	return err
}

func CloseItem(ids []int, token string) error {
	var commands Commands
	for _, id := range ids {
		command := NewCommand("item_close", map[string]interface{}{"id": id})
		commands = append(commands, command)
	}
	_, err := SyncRequest(commands.UrlValues(token))
	return err
}

func MoveItem(item Item, to_project Project, token string) error {
	commands := Commands{
		NewCommand("item_move", item.MoveParam(to_project)),
	}
	_, err := SyncRequest(commands.UrlValues(token))
	return err
}
<commit_msg>Modify item struct attributes<commit_after>package lib

import (
	"strconv"
	"strings"
	"time"
)

type BaseItem struct {
	HaveID
	HaveProjectID
	Content string `json:"content"`
	UserID  int    `json:"user_id"`
}

type CompletedItem struct {
	BaseItem
	CompletedDate string      `json:"completed_date"`
	MetaData      interface{} `json:"meta_data"`
	TaskID        int         `json:"task_id"`
}

func (item CompletedItem) CompletedDateTime() time.Time {
	t, _ := time.Parse(DateFormat, item.CompletedDate)
	return t
}

type CompletedItems []CompletedItem

type Item struct {
	BaseItem
	AllDay         bool        `json:"all_day"`
	AssignedByUID  int         `json:"assigned_by_uid"`
	Checked        int         `json:"checked"`
	Collapsed      int         `json:"collapsed"`
	DateAdded      string      `json:"date_added"`
	DateLang       string      `json:"date_lang"`
	DateString     string      `json:"date_string"`
	DayOrder       int         `json:"day_order"`
	DueDateUtc     string      `json:"due_date_utc"`
	HasMoreNotes   bool        `json:"has_more_notes"`
	InHistory      int         `json:"in_history"`
	Indent         int         `json:"indent"`
	IsArchived     int         `json:"is_archived"`
	IsDeleted      int         `json:"is_deleted"`
	ItemOrder      int         `json:"item_order"`
	LabelIDs       []int       `json:"labels"`
	ParentID       interface{} `json:"parent_id"`
	Priority       int         `json:"priority"`
	ResponsibleUID interface{} `json:"responsible_uid"`
	SyncID         interface{} `json:"sync_id"`
}

type Items []Item

func (item Item) DueDateTime() time.Time {
	t, _ := time.Parse(DateFormat, item.DueDateUtc)
	return t
}

func (items Items) FindByID(id int) (Item, error) {
	for _, item := range items {
		if item.ID == id {
			return item, nil
		}
	}
	return Item{}, FindFailed
}

func (item Item) AddParam() interface{} {
	param := map[string]interface{}{}
	if item.Content != "" {
		param["content"] = item.Content
	}
	if item.DateString != "" {
		param["date_string"] = item.DateString
	}
	if len(item.LabelIDs) != 0 {
		param["labels"] = item.LabelIDs
	}
	if item.Priority != 0 {
		param["priority"] = item.Priority
	}
	if item.ProjectID != 0 {
		param["project_id"] = item.ProjectID
	}
	return param
}

func (item Item) UpdateParam() interface{} {
	param := map[string]interface{}{}
	if item.ID != 0 {
		param["id"] = item.ID
	}
	if item.Content != "" {
		param["content"] = item.Content
	}
	if item.DateString != "" {
		param["date_string"] = item.DateString
	}
	if len(item.LabelIDs) != 0 {
		param["labels"] = item.LabelIDs
	}
	if item.Priority != 0 {
		param["priority"] = item.Priority
	}
	return param
}

func (item Item) MoveParam(to_project Project) interface{} {
	param := map[string]interface{}{
		"project_items": map[string][]int{
			strconv.Itoa(item.ProjectID): []int{item.ID},
		},
		"to_project": to_project.ID,
	}
	return param
}

func (item Item) LabelsString(labels Labels) string {
	label_names := make([]string, 0)
	for _, label_id := range item.LabelIDs {
		label, err := labels.FindByID(label_id)
		if err != nil {
			return "Error"
		}
		label_names = append(label_names, "@"+label.Name)
	}
	return strings.Join(label_names, ",")
}

func AddItem(item Item, token string) error {
	commands := Commands{
		NewCommand("item_add", item.AddParam()),
	}
	_, err := SyncRequest(commands.UrlValues(token))
	return err
}

func UpdateItem(item Item, token string) error {
	commands := Commands{
		NewCommand("item_update", item.UpdateParam()),
	}
	_, err := SyncRequest(commands.UrlValues(token))
	return err
}

func CloseItem(ids []int, token string) error {
	var commands Commands
	for _, id := range ids {
		command := NewCommand("item_close", map[string]interface{}{"id": id})
		commands = append(commands, command)
	}
	_, err := SyncRequest(commands.UrlValues(token))
	return err
}

func MoveItem(item Item, to_project Project, token string) error {
	commands := Commands{
		NewCommand("item_move", item.MoveParam(to_project)),
	}
	_, err := SyncRequest(commands.UrlValues(token))
	return err
}
<|endoftext|>


<commit_before>package controller

import (
	"fmt"
	"strings"

	kerrors "github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors"
	"github.com/golang/glog"

	kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"

	buildapi "github.com/openshift/origin/pkg/build/api"
	buildclient "github.com/openshift/origin/pkg/build/client"
	buildutil "github.com/openshift/origin/pkg/build/util"
	imageapi "github.com/openshift/origin/pkg/image/api"
)

// ImageChangeControllerFatalError represents a fatal error while handling an image change
type ImageChangeControllerFatalError struct {
	Reason string
	Err    error
}

func (e ImageChangeControllerFatalError) Error() string {
	return fmt.Sprintf("fatal error handling ImageStream change: %s, root error was: %s", e.Reason, e.Err.Error())
}

// ImageChangeController watches for changes to ImageRepositories and triggers
// builds when a new version of a tag referenced by a BuildConfig
// is available.
type ImageChangeController struct {
	BuildConfigStore        cache.Store
	BuildConfigInstantiator buildclient.BuildConfigInstantiator
	// Stop is an optional channel that controls when the controller exits
	Stop <-chan struct{}
}

// getImageStreamNameFromReference strips off the :tag or @id suffix
// from an ImageStream[Tag,Image,''].Name
func getImageStreamNameFromReference(ref *kapi.ObjectReference) string {
	name := strings.Split(ref.Name, ":")[0]
	return strings.Split(name, "@")[0]
}

// HandleImageRepo processes the next ImageStream event.
func (c *ImageChangeController) HandleImageRepo(repo *imageapi.ImageStream) error {
	glog.V(4).Infof("Build image change controller detected ImageStream change %s", repo.Status.DockerImageRepository)

	// Loop through all build configurations and record if there was an error
	// instead of breaking the loop. The error will be returned in the end, so the
	// retry controller can retry. Any BuildConfigs that were processed successfully
	// should have had their LastTriggeredImageID updated, so the retry should result
	// in a no-op for them.
	hasError := false

	// TODO: this is inefficient
	for _, bc := range c.BuildConfigStore.List() {
		config := bc.(*buildapi.BuildConfig)

		from := buildutil.GetImageStreamForStrategy(config.Parameters.Strategy)
		if from == nil || from.Kind != "ImageStreamTag" {
			continue
		}

		shouldBuild := false
		triggeredImage := ""
		// For every ImageChange trigger find the latest tagged image from the image repository and replace that value
		// throughout the build strategies. A new build is triggered only if the latest tagged image id or pull spec
		// differs from the last triggered build recorded on the build config.
		for _, trigger := range config.Triggers {
			if trigger.Type != buildapi.ImageChangeBuildTriggerType {
				continue
			}
			fromStreamName := getImageStreamNameFromReference(from)

			fromNamespace := from.Namespace
			if len(fromNamespace) == 0 {
				fromNamespace = config.Namespace
			}

			// only trigger a build if this image repo matches the name and namespace of the ref in the build trigger
			// also do not trigger if the imagerepo does not have a valid DockerImageRepository value for us to pull
			// the image from
			if len(repo.Status.DockerImageRepository) == 0 || fromStreamName != repo.Name || fromNamespace != repo.Namespace {
				continue
			}

			// This split is safe because ImageStreamTag names always have the form
			// name:tag.
			tag := strings.Split(from.Name, ":")[1]
			latest := imageapi.LatestTaggedImage(repo, tag)
			if latest == nil {
				util.HandleError(fmt.Errorf("unable to find tagged image: no image recorded for %s/%s:%s", repo.Namespace, repo.Name, tag))
				continue
			}
			glog.V(4).Infof("Found ImageStream %s/%s with tag %s", repo.Namespace, repo.Name, tag)

			// (must be different) to trigger a build
			last := trigger.ImageChange.LastTriggeredImageID
			next := latest.DockerImageReference

			if len(last) == 0 || (len(next) > 0 && next != last) {
				triggeredImage = next
				shouldBuild = true
				// it doesn't really make sense to have multiple image change triggers any more,
				// so just exit the loop now
				break
			}
		}

		if shouldBuild {
			glog.V(4).Infof("Running build for BuildConfig %s/%s", config.Namespace, config.Name)
			// instantiate new build
			request := &buildapi.BuildRequest{
				ObjectMeta: kapi.ObjectMeta{
					Name: config.Name,
				},
				TriggeredByImage: &kapi.ObjectReference{
					Kind: "DockerImage",
					Name: triggeredImage,
				},
			}
			if _, err := c.BuildConfigInstantiator.Instantiate(config.Namespace, request); err != nil {
				if kerrors.IsConflict(err) {
					util.HandleError(fmt.Errorf("unable to instantiate Build for BuildConfig %s/%s due to a conflicting update: %v", config.Namespace, config.Name, err))
				} else {
					util.HandleError(fmt.Errorf("error instantiating Build from BuildConfig %s/%s: %v", config.Namespace, config.Name, err))
				}
				hasError = true
				continue
			}
		}
	}
	if hasError {
		return fmt.Errorf("an error occurred processing 1 or more build configurations; the image change trigger for image stream %s will be retried", repo.Status.DockerImageRepository)
	}
	return nil
}
<commit_msg>lower log level of missing tag for image message<commit_after>package controller

import (
	"fmt"
	"strings"

	kerrors "github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors"
	"github.com/golang/glog"

	kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"

	buildapi "github.com/openshift/origin/pkg/build/api"
	buildclient "github.com/openshift/origin/pkg/build/client"
	buildutil "github.com/openshift/origin/pkg/build/util"
	imageapi "github.com/openshift/origin/pkg/image/api"
)

// ImageChangeControllerFatalError represents a fatal error while handling an image change
type ImageChangeControllerFatalError struct {
	Reason string
	Err    error
}

func (e ImageChangeControllerFatalError) Error() string {
	return fmt.Sprintf("fatal error handling ImageStream change: %s, root error was: %s", e.Reason, e.Err.Error())
}

// ImageChangeController watches for changes to ImageRepositories and triggers
// builds when a new version of a tag referenced by a BuildConfig
// is available.
type ImageChangeController struct {
	BuildConfigStore        cache.Store
	BuildConfigInstantiator buildclient.BuildConfigInstantiator
	// Stop is an optional channel that controls when the controller exits
	Stop <-chan struct{}
}

// getImageStreamNameFromReference strips off the :tag or @id suffix
// from an ImageStream[Tag,Image,''].Name
func getImageStreamNameFromReference(ref *kapi.ObjectReference) string {
	name := strings.Split(ref.Name, ":")[0]
	return strings.Split(name, "@")[0]
}

// HandleImageRepo processes the next ImageStream event.
func (c *ImageChangeController) HandleImageRepo(repo *imageapi.ImageStream) error {
	glog.V(4).Infof("Build image change controller detected ImageStream change %s", repo.Status.DockerImageRepository)

	// Loop through all build configurations and record if there was an error
	// instead of breaking the loop. The error will be returned in the end, so the
	// retry controller can retry. Any BuildConfigs that were processed successfully
	// should have had their LastTriggeredImageID updated, so the retry should result
	// in a no-op for them.
	hasError := false

	// TODO: this is inefficient
	for _, bc := range c.BuildConfigStore.List() {
		config := bc.(*buildapi.BuildConfig)

		from := buildutil.GetImageStreamForStrategy(config.Parameters.Strategy)
		if from == nil || from.Kind != "ImageStreamTag" {
			continue
		}

		shouldBuild := false
		triggeredImage := ""
		// For every ImageChange trigger find the latest tagged image from the image repository and replace that value
		// throughout the build strategies. A new build is triggered only if the latest tagged image id or pull spec
		// differs from the last triggered build recorded on the build config.
		for _, trigger := range config.Triggers {
			if trigger.Type != buildapi.ImageChangeBuildTriggerType {
				continue
			}
			fromStreamName := getImageStreamNameFromReference(from)

			fromNamespace := from.Namespace
			if len(fromNamespace) == 0 {
				fromNamespace = config.Namespace
			}

			// only trigger a build if this image repo matches the name and namespace of the ref in the build trigger
			// also do not trigger if the imagerepo does not have a valid DockerImageRepository value for us to pull
			// the image from
			if len(repo.Status.DockerImageRepository) == 0 || fromStreamName != repo.Name || fromNamespace != repo.Namespace {
				continue
			}

			// This split is safe because ImageStreamTag names always have the form
			// name:tag.
			tag := strings.Split(from.Name, ":")[1]
			latest := imageapi.LatestTaggedImage(repo, tag)
			if latest == nil {
				glog.V(4).Infof("unable to find tagged image: no image recorded for %s/%s:%s", repo.Namespace, repo.Name, tag)
				continue
			}
			glog.V(4).Infof("Found ImageStream %s/%s with tag %s", repo.Namespace, repo.Name, tag)

			// (must be different) to trigger a build
			last := trigger.ImageChange.LastTriggeredImageID
			next := latest.DockerImageReference

			if len(last) == 0 || (len(next) > 0 && next != last) {
				triggeredImage = next
				shouldBuild = true
				// it doesn't really make sense to have multiple image change triggers any more,
				// so just exit the loop now
				break
			}
		}

		if shouldBuild {
			glog.V(4).Infof("Running build for BuildConfig %s/%s", config.Namespace, config.Name)
			// instantiate new build
			request := &buildapi.BuildRequest{
				ObjectMeta: kapi.ObjectMeta{
					Name: config.Name,
				},
				TriggeredByImage: &kapi.ObjectReference{
					Kind: "DockerImage",
					Name: triggeredImage,
				},
			}
			if _, err := c.BuildConfigInstantiator.Instantiate(config.Namespace, request); err != nil {
				if kerrors.IsConflict(err) {
					util.HandleError(fmt.Errorf("unable to instantiate Build for BuildConfig %s/%s due to a conflicting update: %v", config.Namespace, config.Name, err))
				} else {
					util.HandleError(fmt.Errorf("error instantiating Build from BuildConfig %s/%s: %v", config.Namespace, config.Name, err))
				}
				hasError = true
				continue
			}
		}
	}
	if hasError {
		return fmt.Errorf("an error occurred processing 1 or more build configurations; the image change trigger for image stream %s will be retried", repo.Status.DockerImageRepository)
	}
	return nil
}
<|endoftext|>


<commit_before>package secret

import (
	"context"
	"fmt"
	"strings"

	"k8s.io/apimachinery/pkg/runtime"

	"github.com/rancher/norman/controller"
	v1 "github.com/rancher/rancher/pkg/generated/norman/core/v1"
	v3 "github.com/rancher/rancher/pkg/generated/norman/management.cattle.io/v3"
	"github.com/rancher/rancher/pkg/types/config"
	"github.com/sirupsen/logrus"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
)

// SecretController listens for secret CUD in management API
// and propagates the changes to all corresponding namespaces in cluster API

// NamespaceController listens to cluster namespace events,
// reads secrets from the management namespace of corresponding project,
// and creates the secrets in the cluster namespace

const (
	projectIDLabel             = "field.cattle.io/projectId"
	create                     = "create"
	update                     = "update"
	projectNamespaceAnnotation = "management.cattle.io/system-namespace"
	userSecretAnnotation       = "secret.user.cattle.io/secret"
)

type Controller struct {
	secrets                   v1.SecretInterface
	clusterNamespaceLister    v1.NamespaceLister
	managementNamespaceLister v1.NamespaceLister
	projectLister             v3.ProjectLister
	clusterName               string
}

func Register(ctx context.Context, cluster *config.UserContext) {
	starter := cluster.DeferredStart(ctx, func(ctx context.Context) error {
		registerDeferred(ctx, cluster)
		return nil
	})

	projectLister := cluster.Management.Management.Projects("").Controller().Lister()
	secrets := cluster.Management.Core.Secrets("")
	secrets.AddHandler(ctx, "secret-deferred", func(key string, obj *corev1.Secret) (runtime.Object, error) {
		if obj == nil {
			return nil, nil
		}

		_, err := projectLister.Get(cluster.ClusterName, obj.Namespace)
		if errors.IsNotFound(err) {
			return obj, nil
		} else if err != nil {
			return obj, err
		}

		if !strings.HasPrefix(obj.Name, "default-token-") {
			return obj, starter()
		}

		return obj, nil
	})
}

func registerDeferred(ctx context.Context, cluster *config.UserContext) {
	clusterSecretsClient := cluster.Core.Secrets("")
	s := &Controller{
		secrets:                   clusterSecretsClient,
		clusterNamespaceLister:    cluster.Core.Namespaces("").Controller().Lister(),
		managementNamespaceLister: cluster.Management.Core.Namespaces("").Controller().Lister(),
		projectLister:             cluster.Management.Management.Projects(cluster.ClusterName).Controller().Lister(),
		clusterName:               cluster.ClusterName,
	}

	n := &NamespaceController{
		clusterSecretsClient: clusterSecretsClient,
		managementSecrets:    cluster.Management.Core.Secrets("").Controller().Lister(),
	}
	cluster.Core.Namespaces("").AddHandler(ctx, "secretsController", n.sync)

	sync := v1.NewSecretLifecycleAdapter(fmt.Sprintf("secretsController_%s", cluster.ClusterName), true,
		cluster.Management.Core.Secrets(""), s)

	cluster.Management.Core.Secrets("").AddHandler(ctx, "secretsController", func(key string, obj *corev1.Secret) (runtime.Object, error) {
		if obj == nil {
			logrus.Tracef("secretsController: AddHandler: obj is nil, calling sync")
			return sync(key, nil)
		}
		if !controller.ObjectInCluster(cluster.ClusterName, obj) {
			logrus.Tracef("secretsController: AddHandler: obj [%s] is not in cluster [%s], returning nil", obj.Name, cluster.ClusterName)
			return nil, nil
		}

		if obj.Type == corev1.SecretTypeServiceAccountToken {
			logrus.Tracef("secretsController: AddHandler: obj [%s] is Service Account token, skipping", obj.Name)
			return nil, nil
		}

		if obj.Labels != nil {
			if obj.Labels["cattle.io/creator"] == "norman" {
				logrus.Tracef("secretsController: AddHandler: obj [%s] labels in [%s] contain cattle.io/creator=norman, calling sync", obj.Name, cluster.ClusterName)
				return sync(key, obj)
			}
		}

		return nil, nil
	})
}

type NamespaceController struct {
	clusterSecretsClient v1.SecretInterface
	managementSecrets    v1.SecretLister
}

func (n *NamespaceController) sync(key string, obj *corev1.Namespace) (runtime.Object, error) {
	if obj == nil || obj.DeletionTimestamp != nil {
		return nil, nil
	}
	logrus.Tracef("secretsController: sync called for key [%s] in namespace [%s]", key, obj.Name)
	// field.cattle.io/projectId value is <cluster name>:<project name>
	logrus.Tracef("secretsController: sync: key [%s], obj.Annotations[projectIDLabel]: [%s]", key, obj.Annotations[projectIDLabel])
	if obj.Annotations[projectIDLabel] != "" {
		parts := strings.Split(obj.Annotations[projectIDLabel], ":")
		if len(parts) == 2 {
			if parts[1] == "" {
				logrus.Debugf("[NamspaceController|sync] empty project name found in obj.Annotations[projectIDLabel] for cluster: %s", parts[0])
				return nil, nil
			}
			// on the management side, secret's namespace name equals to project name
			secrets, err := n.managementSecrets.List(parts[1], labels.NewSelector())
			if err != nil {
				return nil, err
			}
			logrus.Tracef("secretsController: sync: length of secrets for [%s] in namespace [%s] is %d", parts[1], obj.Name, len(secrets))
			for _, secret := range secrets {
				// skip service account token secrets
				if secret.Type == corev1.SecretTypeServiceAccountToken {
					logrus.Tracef("secretsController: AddHandler: secret [%s] is Service Account token, skipping", secret.Name)
					continue
				}
				namespacedSecret := getNamespacedSecret(secret, obj.Name)
				logrus.Infof("Creating secret [%s] into namespace [%s]", namespacedSecret.Name, obj.Name)
				_, err := n.clusterSecretsClient.Create(namespacedSecret)
				if err != nil && !errors.IsAlreadyExists(err) {
					return nil, err
				}
			}
		}
	}
	return nil, nil
}

func (s *Controller) Create(obj *corev1.Secret) (runtime.Object, error) {
	logrus.Tracef("secretsController: Create called for [%s]", obj.Name)
	return nil, s.createOrUpdate(obj, create)
}

func (s *Controller) Updated(obj *corev1.Secret) (runtime.Object, error) {
	logrus.Tracef("secretsController: Updated called for [%s]", obj.Name)
	return nil, s.createOrUpdate(obj, update)
}

func (s *Controller) Remove(obj *corev1.Secret) (runtime.Object, error) {
	logrus.Tracef("secretsController: Remove called for [%s]", obj.Name)
	clusterNamespaces, err := s.getClusterNamespaces(obj)
	if err != nil {
		return nil, err
	}

	for _, namespace := range clusterNamespaces {
		logrus.Infof("Deleting secret [%s] in namespace [%s]", obj.Name, namespace.Name)
		if err := s.secrets.DeleteNamespaced(namespace.Name, obj.Name, &metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
			return nil, err
		}
	}
	return nil, nil
}

func (s *Controller) getClusterNamespaces(obj *corev1.Secret) ([]*corev1.Namespace, error) {
	logrus.Tracef("secretsController: getClusterNamespaces called for [%s]", obj.Name)
	var toReturn []*corev1.Namespace
	projectNamespace, err := s.managementNamespaceLister.Get("", obj.Namespace)
	if err != nil {
		if errors.IsNotFound(err) {
			logrus.Warnf("Project namespace [%s] can't be found", obj.Namespace)
			return toReturn, nil
		}
		return toReturn, err
	}
	if projectNamespace.Annotations == nil {
		return toReturn, nil
	}
	if val, ok := projectNamespace.Annotations[projectNamespaceAnnotation]; !(ok && val == "true") {
		return toReturn, nil
	}

	// Ignore projects from other clusters. Project namespace name = project name, so use it to locate the project
	_, err = s.projectLister.Get(s.clusterName, projectNamespace.Name)
	if err != nil {
		if errors.IsNotFound(err) {
			return toReturn, nil
		}
		return toReturn, err
	}

	namespaces, err := s.clusterNamespaceLister.List("", labels.NewSelector())
	if err != nil {
		return toReturn, err
	}
	// system project namespace name == project.Name
	projectID := obj.Namespace

	for _, namespace := range namespaces {
		parts := strings.Split(namespace.Annotations[projectIDLabel], ":")
		if len(parts) == 2 && parts[1] == projectID {
			toReturn = append(toReturn, namespace)
		}
	}
	return toReturn, nil
}

func (s *Controller) createOrUpdate(obj *corev1.Secret, action string) error {
	logrus.Tracef("secretsController: createOrUpdate called for [%s]", obj.Name)
	if obj.Annotations[projectIDLabel] != "" {
		parts := strings.Split(obj.Annotations[projectIDLabel], ":")
		if len(parts) == 2 {
			if parts[0] != s.clusterName {
				return nil
			}
		}
	}
	clusterNamespaces, err := s.getClusterNamespaces(obj)
	if err != nil {
		return err
	}
	for _, namespace := range clusterNamespaces {
		if !namespace.DeletionTimestamp.IsZero() {
			continue
		}
		// copy the secret into namespace
		namespacedSecret := getNamespacedSecret(obj, namespace.Name)
		switch action {
		case create:
			logrus.Infof("Copying secret [%s] into namespace [%s]", namespacedSecret.Name, namespace.Name)
			_, err := s.secrets.Create(namespacedSecret)
			if err != nil && !errors.IsAlreadyExists(err) {
				return err
			}
		case update:
			logrus.Infof("Updating secret [%s] into namespace [%s]", namespacedSecret.Name, namespace.Name)
			_, err := s.secrets.Update(namespacedSecret)
			if err != nil && !errors.IsNotFound(err) {
				return err
			} else if errors.IsNotFound(err) {
				logrus.Infof("Updating secret [%s] returned NotFound, creating secret [%s] into namespace [%s]", namespacedSecret.Name, namespacedSecret.Name, namespace.Name)
				_, err := s.secrets.Create(namespacedSecret)
				if err != nil {
					return err
				}
			}
		}
	}

	return nil
}

func getNamespacedSecret(obj *corev1.Secret, namespace string) *corev1.Secret {
	namespacedSecret := &corev1.Secret{}
	namespacedSecret.Name = obj.Name
	namespacedSecret.Kind = obj.Kind
	namespacedSecret.Data = obj.Data
	namespacedSecret.StringData = obj.StringData
	namespacedSecret.Namespace = namespace
	namespacedSecret.Type = obj.Type
	namespacedSecret.Annotations = make(map[string]string)
	namespacedSecret.Labels = make(map[string]string)
	copyMap(namespacedSecret.Annotations, obj.Annotations)
	copyMap(namespacedSecret.Labels, obj.Labels)
	namespacedSecret.Annotations[userSecretAnnotation] = "true"
	return namespacedSecret
}

func copyMap(dst map[string]string, src map[string]string) {
	for k, v := range src {
		dst[k] = v
	}
}
<commit_msg>Check cache before create/update of project secrets<commit_after>package secret

import (
	"context"
	"fmt"
	"reflect"
	"strings"

	"k8s.io/apimachinery/pkg/runtime"

	"github.com/rancher/norman/controller"
	v1 "github.com/rancher/rancher/pkg/generated/norman/core/v1"
	v3 "github.com/rancher/rancher/pkg/generated/norman/management.cattle.io/v3"
	"github.com/rancher/rancher/pkg/types/config"
	"github.com/sirupsen/logrus"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
)

// SecretController listens for secret CUD in management API
// and propagates the changes to all corresponding namespaces in cluster API

// NamespaceController listens to cluster namespace events,
// reads secrets from the management namespace of corresponding project,
// and creates the secrets in the cluster namespace

const (
	projectIDLabel             = "field.cattle.io/projectId"
	create                     = "create"
	update                     = "update"
	projectNamespaceAnnotation = "management.cattle.io/system-namespace"
	userSecretAnnotation       = "secret.user.cattle.io/secret"
)

type Controller struct {
	secrets                   v1.SecretInterface
	secretLister              v1.SecretLister
	clusterNamespaceLister    v1.NamespaceLister
	managementNamespaceLister v1.NamespaceLister
	projectLister             v3.ProjectLister
	clusterName               string
}

func Register(ctx context.Context, cluster *config.UserContext) {
	starter := cluster.DeferredStart(ctx, func(ctx context.Context) error {
		registerDeferred(ctx, cluster)
		return nil
	})

	projectLister := cluster.Management.Management.Projects("").Controller().Lister()
	secrets := cluster.Management.Core.Secrets("")
	secrets.AddHandler(ctx, "secret-deferred", func(key string, obj *corev1.Secret) (runtime.Object, error) {
		if obj == nil {
			return nil, nil
		}

		_, err := projectLister.Get(cluster.ClusterName, obj.Namespace)
		if errors.IsNotFound(err) {
			return obj, nil
		} else if err != nil {
			return obj, err
		}

		if !strings.HasPrefix(obj.Name, "default-token-") {
			return obj, starter()
		}

		return obj, nil
	})
}

func registerDeferred(ctx context.Context, cluster *config.UserContext) {
	clusterSecretsClient := cluster.Core.Secrets("")
	s := &Controller{
		secrets:                   clusterSecretsClient,
		secretLister:              clusterSecretsClient.Controller().Lister(),
		clusterNamespaceLister:    cluster.Core.Namespaces("").Controller().Lister(),
		managementNamespaceLister: cluster.Management.Core.Namespaces("").Controller().Lister(),
		projectLister:             cluster.Management.Management.Projects(cluster.ClusterName).Controller().Lister(),
		clusterName:               cluster.ClusterName,
	}

	n := &NamespaceController{
		clusterSecretsClient: clusterSecretsClient,
		clusterSecretsLister: clusterSecretsClient.Controller().Lister(),
		managementSecrets:    cluster.Management.Core.Secrets("").Controller().Lister(),
	}
	cluster.Core.Namespaces("").AddHandler(ctx, "secretsController", n.sync)

	sync := v1.NewSecretLifecycleAdapter(fmt.Sprintf("secretsController_%s", cluster.ClusterName), true,
		cluster.Management.Core.Secrets(""), s)

	cluster.Management.Core.Secrets("").AddHandler(ctx, "secretsController", func(key string, obj *corev1.Secret) (runtime.Object, error) {
		if obj == nil {
			logrus.Tracef("secretsController: AddHandler: obj is nil, calling sync")
			return sync(key, nil)
		}
		if !controller.ObjectInCluster(cluster.ClusterName, obj) {
			logrus.Tracef("secretsController: AddHandler: obj [%s] is not in cluster [%s], returning nil", obj.Name, cluster.ClusterName)
			return nil, nil
		}

		if obj.Type == corev1.SecretTypeServiceAccountToken {
			logrus.Tracef("secretsController: AddHandler: obj [%s] is Service Account token, skipping", obj.Name)
			return nil, nil
		}

		if obj.Labels != nil {
			if obj.Labels["cattle.io/creator"] == "norman" {
				logrus.Tracef("secretsController: AddHandler: obj [%s] labels in [%s] contain cattle.io/creator=norman, calling sync", obj.Name, cluster.ClusterName)
				return sync(key, obj)
			}
		}

		return nil, nil
	})
}

type NamespaceController struct {
	clusterSecretsClient v1.SecretInterface
	clusterSecretsLister v1.SecretLister
	managementSecrets    v1.SecretLister
}

func (n *NamespaceController) sync(key string, obj *corev1.Namespace) (runtime.Object, error) {
	if obj == nil || obj.DeletionTimestamp != nil {
		return nil, nil
	}
	logrus.Tracef("secretsController: sync called for key [%s] in namespace [%s]", key, obj.Name)
	// field.cattle.io/projectId value is <cluster name>:<project name>
	logrus.Tracef("secretsController: sync: key [%s], obj.Annotations[projectIDLabel]: [%s]", key, obj.Annotations[projectIDLabel])
	if obj.Annotations[projectIDLabel] != "" {
		parts := strings.Split(obj.Annotations[projectIDLabel], ":")
		if len(parts) == 2 {
			if parts[1] == "" {
				logrus.Debugf("[NamspaceController|sync] empty project name found in obj.Annotations[projectIDLabel] for cluster: %s", parts[0])
				return nil, nil
			}
			// on the management side, secret's namespace name equals to project name
			secrets, err := n.managementSecrets.List(parts[1], labels.NewSelector())
			if err != nil {
				return nil, err
			}
			logrus.Tracef("secretsController: sync: length of secrets for [%s] in namespace [%s] is %d", parts[1], obj.Name, len(secrets))
			for _, secret := range secrets {
				// skip service account token secrets
				if secret.Type == corev1.SecretTypeServiceAccountToken {
					logrus.Tracef("secretsController: AddHandler: secret [%s] is Service Account token, skipping", secret.Name)
					continue
				}
				namespacedSecret := getNamespacedSecret(secret, obj.Name)
				if _, err := n.clusterSecretsLister.Get(namespacedSecret.Namespace, namespacedSecret.Name); err == nil {
					continue
				}
				logrus.Infof("Creating secret [%s] into namespace [%s]", namespacedSecret.Name, obj.Name)
				_, err := n.clusterSecretsClient.Create(namespacedSecret)
				if err != nil && !errors.IsAlreadyExists(err) {
					return nil, err
				}
			}
		}
	}
	return nil, nil
}

func (s *Controller) Create(obj *corev1.Secret) (runtime.Object, error) {
	logrus.Tracef("secretsController: Create called for [%s]", obj.Name)
	return nil, s.createOrUpdate(obj, create)
}

func (s *Controller) Updated(obj *corev1.Secret) (runtime.Object, error) {
	logrus.Tracef("secretsController: Updated called for [%s]", obj.Name)
	return nil, s.createOrUpdate(obj, update)
}

func (s *Controller) Remove(obj *corev1.Secret) (runtime.Object, error) {
	logrus.Tracef("secretsController: Remove called for [%s]", obj.Name)
	clusterNamespaces, err := s.getClusterNamespaces(obj)
	if err != nil {
		return nil, err
	}

	for _, namespace := range clusterNamespaces {
		logrus.Infof("Deleting secret [%s] in namespace [%s]", obj.Name, namespace.Name)
		if err := s.secrets.DeleteNamespaced(namespace.Name, obj.Name, &metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
			return nil, err
		}
	}
	return nil, nil
}

func (s *Controller) getClusterNamespaces(obj *corev1.Secret) ([]*corev1.Namespace, error) {
	logrus.Tracef("secretsController: getClusterNamespaces called for [%s]", obj.Name)
	var toReturn []*corev1.Namespace
	projectNamespace, err := s.managementNamespaceLister.Get("", obj.Namespace)
	if err != nil {
		if errors.IsNotFound(err) {
			logrus.Warnf("Project namespace [%s] can't be found", obj.Namespace)
			return toReturn, nil
		}
		return toReturn, err
	}
	if projectNamespace.Annotations == nil {
		return toReturn, nil
	}
	if val, ok := projectNamespace.Annotations[projectNamespaceAnnotation]; !(ok && val == "true") {
		return toReturn, nil
	}

	// Ignore projects from other clusters. Project namespace name = project name, so use it to locate the project
	_, err = s.projectLister.Get(s.clusterName, projectNamespace.Name)
	if err != nil {
		if errors.IsNotFound(err) {
			return toReturn, nil
		}
		return toReturn, err
	}

	namespaces, err := s.clusterNamespaceLister.List("", labels.NewSelector())
	if err != nil {
		return toReturn, err
	}
	// system project namespace name == project.Name
	projectID := obj.Namespace

	for _, namespace := range namespaces {
		parts := strings.Split(namespace.Annotations[projectIDLabel], ":")
		if len(parts) == 2 && parts[1] == projectID {
			toReturn = append(toReturn, namespace)
		}
	}
	return toReturn, nil
}

func (s *Controller) createOrUpdate(obj *corev1.Secret, action string) error {
	logrus.Tracef("secretsController: createOrUpdate called for [%s]", obj.Name)
	if obj.Annotations[projectIDLabel] != "" {
		parts := strings.Split(obj.Annotations[projectIDLabel], ":")
		if len(parts) == 2 {
			if parts[0] != s.clusterName {
				return nil
			}
		}
	}
	clusterNamespaces, err := s.getClusterNamespaces(obj)
	if err != nil {
		return err
	}
	for _, namespace := range clusterNamespaces {
		if !namespace.DeletionTimestamp.IsZero() {
			continue
		}
		// copy the secret into namespace
		namespacedSecret := getNamespacedSecret(obj, namespace.Name)
		switch action {
		case create:
			if _, err := s.secretLister.Get(namespacedSecret.Namespace, namespacedSecret.Name); err == nil {
				continue
			}
			logrus.Infof("Copying secret [%s] into namespace [%s]", namespacedSecret.Name, namespace.Name)
			_, err := s.secrets.Create(namespacedSecret)
			if err != nil && !errors.IsAlreadyExists(err) {
				return err
			}
		case update:
			if existing, err := s.secretLister.Get(namespacedSecret.Namespace, namespacedSecret.Name); err == nil &&
				reflect.DeepEqual(existing.Data, namespacedSecret.Data) {
				continue
			}
			logrus.Infof("Updating secret [%s] into namespace [%s]", namespacedSecret.Name, namespace.Name)
			_, err := s.secrets.Update(namespacedSecret)
			if err != nil && !errors.IsNotFound(err) {
				return err
			} else if errors.IsNotFound(err) {
				logrus.Infof("Updating secret [%s] returned NotFound, creating secret [%s] into namespace [%s]", namespacedSecret.Name, namespacedSecret.Name, namespace.Name)
				_, err := s.secrets.Create(namespacedSecret)
				if err != nil {
					return err
				}
			}
		}
	}

	return nil
}

func getNamespacedSecret(obj *corev1.Secret, namespace string) *corev1.Secret {
	namespacedSecret := &corev1.Secret{}
	namespacedSecret.Name = obj.Name
	namespacedSecret.Kind = obj.Kind
	namespacedSecret.Data = obj.Data
	namespacedSecret.StringData = obj.StringData
	namespacedSecret.Namespace = namespace
	namespacedSecret.Type = obj.Type
	namespacedSecret.Annotations = make(map[string]string)
	namespacedSecret.Labels = make(map[string]string)
	copyMap(namespacedSecret.Annotations, obj.Annotations)
	copyMap(namespacedSecret.Labels, obj.Labels)
	namespacedSecret.Annotations[userSecretAnnotation] = "true"
	return namespacedSecret
}

func copyMap(dst map[string]string, src map[string]string) {
	for k, v := range src {
		dst[k] = v
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gettext

import (
	"errors"
	"fmt"
	"sort"

	"code.google.com/p/gorilla/gettext/pluralforms"
)

var ErrMissingContext = errors.New("The message doesn't have a context.")

// Key represents a key for a Catalog translation.
type Key struct {
	Src    string // message
	Ctx    string // message context
	HasCtx bool   // differentiates empty context from no context
}

// NewCatalog returns a new Catalog, initializing internal fields.
func NewCatalog() *Catalog {
	return &Catalog{
		Header:     make(map[string]string),
		Messages:   make(map[Key]Message),
		PluralFunc: pluralforms.DefaultPluralFunc,
	}
}

// Catalog stores gettext translations.
//
// Catalog messages can't be modified in-place; they must be removed and
// re-added using Add() after the modifications, because they message key
// depends on the content of the message.
type Catalog struct {
	Header     map[string]string      // meta-data
	Messages   map[Key]Message        // translations
	PluralFunc pluralforms.PluralFunc // used to select the plural form index
	ctx        string                 // active context
	hasCtx     bool                   // whether to use a context
}

// Add adds a message to the catalog.
func (c *Catalog) Add(msg Message) {
	c.Messages[msg.Key()] = msg
}

// Clone returns a copy of the catalog.
func (c *Catalog) Clone() *Catalog {
	clone := NewCatalog()
	clone.PluralFunc = c.PluralFunc
	clone.ctx = c.ctx
	clone.hasCtx = c.hasCtx
	for k, v := range c.Messages {
		clone.Messages[k] = v.Clone()
	}
	return clone
}

// SetContext activates a given context for messages.
func (c *Catalog) SetContext(ctx string) {
	c.ctx = ctx
	c.hasCtx = true
}

// RemoveContext deactivates any context for messages.
func (c *Catalog) RemoveContext() {
	c.ctx = ""
	c.hasCtx = false
}

// Get returns a translation for the given key, or an empty string if the
// key is not found.
//
// Extra arguments or optional, used to format the translation.
func (c *Catalog) Get(key string, a ...interface{}) string {
	if msg, ok := c.Messages[Key{Src: key, Ctx: c.ctx, HasCtx: c.hasCtx}]; ok {
		if a == nil {
			return msg.Get()
		}
		return msg.Format(msg.Get(), a...)
	}
	return ""
}

// GetPlural returns a plural translation for the given key and number,
// or an empty string if the key is not found.
//
// Extra arguments or optional, used to format the translation.
func (c *Catalog) GetPlural(key string, num int, a ...interface{}) string {
	if msg, ok := c.Messages[Key{Src: key, Ctx: c.ctx, HasCtx: c.hasCtx}]; ok {
		if a == nil {
			return msg.GetPlural(c.PluralFunc(num))
		}
		return msg.Format(msg.GetPlural(c.PluralFunc(num)), a...)
	}
	return ""
}

// sortedMessages returns a slice of messages sorted by key for a catalog.
func sortedMessages(c *Catalog) []Message {
	var msgs []Message
	keyMap := make(map[string][]Message)
	for k, v := range c.Messages {
		keyMap[k.Src] = append(keyMap[k.Src], v)
	}
	for _, v := range sortedMessageKeys(keyMap) {
		// No context first, then the ones with context, sorted.
		ctxMap := make(map[string][]Message)
		for _, msg := range keyMap[v] {
			if ctx, err := msg.Context(); err != nil {
				msgs = append(msgs, msg)
			} else {
				ctxMap[ctx] = append(ctxMap[ctx], msg)
			}
		}
		for _, ctx := range sortedMessageKeys(ctxMap) {
			msgs = append(msgs, ctxMap[ctx]...)
		}
	}
	return msgs
}

func sortedMessageKeys(m map[string][]Message) []string {
	keys := make([]string, len(m))
	i := 0
	for k, _ := range m {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	return keys
}

// ----------------------------------------------------------------------------

// Message represents a translation, including meta-data.
type Message interface {
	// Key returns the message's key.
	Key() Key
	// Context returns the message context, which can be an empty string.
	// If there's no context it returns an error.
	Context() (string, error)
	// Get returns a translation for the message.
	Get() string
	// GetPlural returns a plural translation for the message.
	GetPlural(index int) string
	// Format formats the message. Each message can use a specific formatter.
	Format(s string, a ...interface{}) string
	// Clone returns a copy of the message.
	Clone() Message
	// Info returns the message's meta-data, which can be changed in-place.
	Info() *MessageInfo
}

// ----------------------------------------------------------------------------

type MessageInfo struct {
	UserComments   []string // translator comments. prefix: #
	SourceComments []string // extracted comments.  prefix: #.
	References     []string // reference file/line. prefix: #:
	Flags          []string // flags.               prefix: #,
	PrevSingular   string   //                      prefix: #|
	PrevPlural     string   //                      prefix: #|
	PrevCtx        string   //                      prefix: #|
	HasPrevCtx     bool
}

func (m *MessageInfo) Clone() *MessageInfo {
	clone := &MessageInfo{
		PrevSingular: m.PrevSingular,
		PrevPlural:   m.PrevPlural,
		PrevCtx:      m.PrevCtx,
		HasPrevCtx:   m.HasPrevCtx,
	}
	if m.UserComments != nil {
		clone.UserComments = make([]string, len(m.UserComments))
		copy(clone.UserComments, m.UserComments)
	}
	if m.SourceComments != nil {
		clone.SourceComments = make([]string, len(m.SourceComments))
		copy(clone.SourceComments, m.SourceComments)
	}
	if m.References != nil {
		clone.References = make([]string, len(m.References))
		copy(clone.References, m.References)
	}
	if m.Flags != nil {
		clone.Flags = make([]string, len(m.Flags))
		copy(clone.Flags, m.Flags)
	}
	return clone
}

// ----------------------------------------------------------------------------

// SimpleMessage is a message without plural forms.
type SimpleMessage struct {
	Src    string
	Dst    string
	Ctx    string
	HasCtx bool
	info   *MessageInfo
}

func (m *SimpleMessage) Key() Key {
	if m.HasCtx {
		return Key{Src: m.Src, Ctx: m.Ctx, HasCtx: true}
	}
	return Key{Src: m.Src}
}

func (m *SimpleMessage) Context() (string, error) {
	if m.HasCtx {
		return m.Ctx, nil
	}
	return "", ErrMissingContext
}

func (m *SimpleMessage) Get() string {
	return m.Dst
}

func (m *SimpleMessage) GetPlural(idx int) string {
	return ""
}

func (m *SimpleMessage) Format(s string, a ...interface{}) string {
	// TODO: use message formatter
	return fmt.Sprintf(s, a...)
}

func (m *SimpleMessage) Clone() Message {
	clone := &SimpleMessage{
		Src:    m.Src,
		Dst:    m.Dst,
		Ctx:    m.Ctx,
		HasCtx: m.HasCtx,
	}
	if m.info != nil {
		clone.info = m.info.Clone()
	}
	return clone
}

func (m *SimpleMessage) Info() *MessageInfo {
	if m.info == nil {
		m.info = &MessageInfo{}
	}
	return m.info
}

// ----------------------------------------------------------------------------

// PluralMessage is a message with plural forms.
type PluralMessage struct {
	Src    []string
	Dst    []string
	Ctx    string
	HasCtx bool
	info   *MessageInfo
}

func (m *PluralMessage) Key() Key {
	src := ""
	if len(m.Src) > 0 {
		src = m.Src[0]
	}
	if m.HasCtx {
		return Key{Src: src, Ctx: m.Ctx, HasCtx: true}
	}
	return Key{Src: src}
}

func (m *PluralMessage) Context() (string, error) {
	if m.HasCtx {
		return m.Ctx, nil
	}
	return "", ErrMissingContext
}

func (m *PluralMessage) Get() string {
	return ""
}

func (m *PluralMessage) GetPlural(index int) string {
	if index >= 0 && index < len(m.Dst) {
		return m.Dst[index]
	}
	return ""
}

func (m *PluralMessage) Format(s string, a ...interface{}) string {
	// TODO: use message formatter
	return fmt.Sprintf(s, a...)
}

func (m *PluralMessage) Clone() Message {
	src := make([]string, len(m.Src))
	copy(src, m.Src)
	dst := make([]string, len(m.Dst))
	copy(dst, m.Dst)
	clone := &PluralMessage{
		Src:    src,
		Dst:    dst,
		Ctx:    m.Ctx,
		HasCtx: m.HasCtx,
	}
	if m.info != nil {
		clone.info = m.info.Clone()
	}
	return clone
}

func (m *PluralMessage) Info() *MessageInfo {
	if m.info == nil {
		m.info = &MessageInfo{}
	}
	return m.info
}
<commit_msg>gettext: docs<commit_after>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gettext

import (
	"errors"
	"fmt"
	"sort"

	"code.google.com/p/gorilla/gettext/pluralforms"
)

var ErrMissingContext = errors.New("The message doesn't have a context.")

// Key represents a key for a Catalog translation.
type Key struct {
	Src    string // message
	Ctx    string // message context
	HasCtx bool   // differentiates empty context from no context
}

// NewCatalog returns a new Catalog, initializing internal fields.
func NewCatalog() *Catalog {
	return &Catalog{
		Header:     make(map[string]string),
		Messages:   make(map[Key]Message),
		PluralFunc: pluralforms.DefaultPluralFunc,
	}
}

// Catalog stores gettext translations.
//
// Catalog messages can't be modified in-place; they must be removed and
// re-added using Add() after the modifications, because they message key
// depends on the content of the message.
type Catalog struct {
	Header     map[string]string      // meta-data
	Messages   map[Key]Message        // translations
	PluralFunc pluralforms.PluralFunc // used to select the plural form index
	ctx        string                 // active context
	hasCtx     bool                   // whether to use a context
}

// Add adds a message to the catalog.
func (c *Catalog) Add(msg Message) {
	c.Messages[msg.Key()] = msg
}

// Clone returns a copy of the catalog.
func (c *Catalog) Clone() *Catalog {
	clone := NewCatalog()
	clone.PluralFunc = c.PluralFunc
	clone.ctx = c.ctx
	clone.hasCtx = c.hasCtx
	for k, v := range c.Messages {
		clone.Messages[k] = v.Clone()
	}
	return clone
}

// SetContext activates a given context for messages.
func (c *Catalog) SetContext(ctx string) {
	c.ctx = ctx
	c.hasCtx = true
}

// RemoveContext deactivates any context for messages.
func (c *Catalog) RemoveContext() {
	c.ctx = ""
	c.hasCtx = false
}

// Get returns a translation for the given key, or an empty string if the
// key is not found.
//
// Extra arguments or optional, used to format the translation.
func (c *Catalog) Get(key string, a ...interface{}) string {
	if msg, ok := c.Messages[Key{Src: key, Ctx: c.ctx, HasCtx: c.hasCtx}]; ok {
		if a == nil {
			return msg.Get()
		}
		return msg.Format(msg.Get(), a...)
	}
	return ""
}

// GetPlural returns a plural translation for the given key and number,
// or an empty string if the key is not found.
//
// Extra arguments or optional, used to format the translation.
func (c *Catalog) GetPlural(key string, num int, a ...interface{}) string {
	if msg, ok := c.Messages[Key{Src: key, Ctx: c.ctx, HasCtx: c.hasCtx}]; ok {
		if a == nil {
			return msg.GetPlural(c.PluralFunc(num))
		}
		return msg.Format(msg.GetPlural(c.PluralFunc(num)), a...)
	}
	return ""
}

// sortedMessages returns a slice of messages sorted by key for a catalog.
func sortedMessages(c *Catalog) []Message {
	var msgs []Message
	keyMap := make(map[string][]Message)
	for k, v := range c.Messages {
		keyMap[k.Src] = append(keyMap[k.Src], v)
	}
	for _, v := range sortedMessageKeys(keyMap) {
		// No context first, then the ones with context, sorted.
		ctxMap := make(map[string][]Message)
		for _, msg := range keyMap[v] {
			if ctx, err := msg.Context(); err != nil {
				msgs = append(msgs, msg)
			} else {
				ctxMap[ctx] = append(ctxMap[ctx], msg)
			}
		}
		for _, ctx := range sortedMessageKeys(ctxMap) {
			msgs = append(msgs, ctxMap[ctx]...)
		}
	}
	return msgs
}

// sortedMessageKeys returns a slice of sorted keys for a map of messages.
func sortedMessageKeys(m map[string][]Message) []string {
	keys := make([]string, len(m))
	i := 0
	for k, _ := range m {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	return keys
}

// ----------------------------------------------------------------------------

// Message represents a translation, including meta-data.
type Message interface {
	// Key returns the message's key.
	Key() Key
	// Context returns the message context, which can be an empty string.
	// If there's no context it returns an error.
	Context() (string, error)
	// Get returns a translation for the message.
	Get() string
	// GetPlural returns a plural translation for the message.
	GetPlural(index int) string
	// Format formats the message. Each message can use a specific formatter.
	Format(s string, a ...interface{}) string
	// Clone returns a copy of the message.
	Clone() Message
	// Info returns the message's meta-data, which can be changed in-place.
	Info() *MessageInfo
}

// ----------------------------------------------------------------------------

// MessageInfo stores meta-data found in GNU PO files, such as translator
// comments and references to the source of the translatable strings.
type MessageInfo struct {
	UserComments   []string // translator comments. prefix: #
	SourceComments []string // extracted comments.  prefix: #.
	References     []string // reference file/line. prefix: #:
	Flags          []string // flags.               prefix: #,
	PrevSingular   string   //                      prefix: #|
	PrevPlural     string   //                      prefix: #|
	PrevCtx        string   //                      prefix: #|
	HasPrevCtx     bool
}

// Clone returns a copy of the meta-data.
func (m *MessageInfo) Clone() *MessageInfo {
	clone := &MessageInfo{
		PrevSingular: m.PrevSingular,
		PrevPlural:   m.PrevPlural,
		PrevCtx:      m.PrevCtx,
		HasPrevCtx:   m.HasPrevCtx,
	}
	if m.UserComments != nil {
		clone.UserComments = make([]string, len(m.UserComments))
		copy(clone.UserComments, m.UserComments)
	}
	if m.SourceComments != nil {
		clone.SourceComments = make([]string, len(m.SourceComments))
		copy(clone.SourceComments, m.SourceComments)
	}
	if m.References != nil {
		clone.References = make([]string, len(m.References))
		copy(clone.References, m.References)
	}
	if m.Flags != nil {
		clone.Flags = make([]string, len(m.Flags))
		copy(clone.Flags, m.Flags)
	}
	return clone
}

// ----------------------------------------------------------------------------

// SimpleMessage is a message without plural forms.
//
// See the Message interface for a description of the methods.
type SimpleMessage struct {
	Src    string
	Dst    string
	Ctx    string
	HasCtx bool
	info   *MessageInfo
}

func (m *SimpleMessage) Key() Key {
	if m.HasCtx {
		return Key{Src: m.Src, Ctx: m.Ctx, HasCtx: true}
	}
	return Key{Src: m.Src}
}

func (m *SimpleMessage) Context() (string, error) {
	if m.HasCtx {
		return m.Ctx, nil
	}
	return "", ErrMissingContext
}

func (m *SimpleMessage) Get() string {
	return m.Dst
}

func (m *SimpleMessage) GetPlural(idx int) string {
	return ""
}

func (m *SimpleMessage) Format(s string, a ...interface{}) string {
	// TODO: use message formatter
	return fmt.Sprintf(s, a...)
}

func (m *SimpleMessage) Clone() Message {
	clone := &SimpleMessage{
		Src:    m.Src,
		Dst:    m.Dst,
		Ctx:    m.Ctx,
		HasCtx: m.HasCtx,
	}
	if m.info != nil {
		clone.info = m.info.Clone()
	}
	return clone
}

func (m *SimpleMessage) Info() *MessageInfo {
	if m.info == nil {
		m.info = &MessageInfo{}
	}
	return m.info
}

// ----------------------------------------------------------------------------

// PluralMessage is a message with plural forms.
//
// See the Message interface for a description of the methods.
type PluralMessage struct {
	Src    []string
	Dst    []string
	Ctx    string
	HasCtx bool
	info   *MessageInfo
}

func (m *PluralMessage) Key() Key {
	src := ""
	if len(m.Src) > 0 {
		src = m.Src[0]
	}
	if m.HasCtx {
		return Key{Src: src, Ctx: m.Ctx, HasCtx: true}
	}
	return Key{Src: src}
}

func (m *PluralMessage) Context() (string, error) {
	if m.HasCtx {
		return m.Ctx, nil
	}
	return "", ErrMissingContext
}

func (m *PluralMessage) Get() string {
	return ""
}

func (m *PluralMessage) GetPlural(index int) string {
	if index >= 0 && index < len(m.Dst) {
		return m.Dst[index]
	}
	return ""
}

func (m *PluralMessage) Format(s string, a ...interface{}) string {
	// TODO: use message formatter
	return fmt.Sprintf(s, a...)
}

func (m *PluralMessage) Clone() Message {
	src := make([]string, len(m.Src))
	copy(src, m.Src)
	dst := make([]string, len(m.Dst))
	copy(dst, m.Dst)
	clone := &PluralMessage{
		Src:    src,
		Dst:    dst,
		Ctx:    m.Ctx,
		HasCtx: m.HasCtx,
	}
	if m.info != nil {
		clone.info = m.info.Clone()
	}
	return clone
}

func (m *PluralMessage) Info() *MessageInfo {
	if m.info == nil {
		m.info = &MessageInfo{}
	}
	return m.info
}
<|endoftext|>


<commit_before>// Copyright 2013 errnoh. All rights reserved.
// Use of this source code is governed by
// MIT License that can be found in the LICENSE file.

// Small wrapper for Reittiopas API ( http://developer.reittiopas.fi/pages/en/http-get-interface-version-2.php )
package ro

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
)

var (
	api_user, api_pass string
)

func SetCredentials(user, pass string) {
	api_user = user
	api_pass = pass
}

func GetLocation(loc string) (LocationResponse, bool) {
	if loc == "" {
		return nil, false
	}

	var resp LocationResponse

	s := fmt.Sprintf("request=geocode&key=%s", url.QueryEscape(loc))
	ok := Get(s, &resp)
	if !ok {
		return nil, false
	}

	return resp, true
}

func GetRoute(from Location, to Location, time string, date string, optimize string, limit int) RouteResponse {
	var (
		resp   RouteResponse
		buffer bytes.Buffer
	)

	buffer.WriteString(fmt.Sprintf("request=route&from=%s&to=%s", from.Coords, to.Coords))

	if time != "" {
		buffer.WriteString(fmt.Sprintf("&time=%s", time))
	}
	if date != "" {
		buffer.WriteString(fmt.Sprintf("&date=%s", date))
	}
	if optimize != "" {
		buffer.WriteString(fmt.Sprintf("&optimize=%s", optimize))
	}
	if limit != 3 && limit >= 0 && limit <= 5 {
		buffer.WriteString(fmt.Sprintf("&show=%d", limit))
	}

	Get(buffer.String(), &resp)

	resp.fixStartEndLocations(from, to)

	return resp
}

// Add route start & end locations to the route information instead of displaying empty fields.
func (r RouteResponse) fixStartEndLocations(from Location, to Location) {
	for i := 0; i < len(r); i++ {
		r[i][0].Legs[0].Locs[0].Name = from.Name
		r[i][0].Legs[len(r[i][0].Legs)-1].Locs[len(r[i][0].Legs[len(r[i][0].Legs)-1].Locs)-1].Name = to.Name
	}
}

func Get(s string, v interface{}) bool {
	url := fmt.Sprintf("http://api.reittiopas.fi/hsl/prod/?user=%s&pass=%s&%s", api_user, api_pass, s)

	resp, err := http.Get(url)
	if err != nil {
		log.Println(err)
		return false
	}

	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		b, _ := ioutil.ReadAll(resp.Body)
		if string(b) != "" {
			log.Println(string(b))
		}
		return false
	}

	err = json.NewDecoder(resp.Body).Decode(v)
	if err != nil {
		// These aren't fatal errors most of the time.
		log.Printf(err.Error())
	}

	return true
}

func (l Legs) Line() string {
	switch l.Code {
	case "1", "2", "3", "4":
		return l.Type[2:][:4]
	case "5", "8":
		return l.Type[1:][:4]
	case "6":
		return "metro"
	case "12":
		return l.Type[4:5]
	case "walk":
		return l.Code
	}
	return l.Type
}
<commit_msg>Fixed: Two variables had been switched around<commit_after>// Copyright 2013 errnoh. All rights reserved.
// Use of this source code is governed by
// MIT License that can be found in the LICENSE file.

// Small wrapper for Reittiopas API ( http://developer.reittiopas.fi/pages/en/http-get-interface-version-2.php )
package ro

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
)

var (
	api_user, api_pass string
)

func SetCredentials(user, pass string) {
	api_user = user
	api_pass = pass
}

func GetLocation(loc string) (LocationResponse, bool) {
	if loc == "" {
		return nil, false
	}

	var resp LocationResponse

	s := fmt.Sprintf("request=geocode&key=%s", url.QueryEscape(loc))
	ok := Get(s, &resp)
	if !ok {
		return nil, false
	}

	return resp, true
}

func GetRoute(from Location, to Location, time string, date string, optimize string, limit int) RouteResponse {
	var (
		resp   RouteResponse
		buffer bytes.Buffer
	)

	buffer.WriteString(fmt.Sprintf("request=route&from=%s&to=%s", from.Coords, to.Coords))

	if time != "" {
		buffer.WriteString(fmt.Sprintf("&time=%s", time))
	}
	if date != "" {
		buffer.WriteString(fmt.Sprintf("&date=%s", date))
	}
	if optimize != "" {
		buffer.WriteString(fmt.Sprintf("&optimize=%s", optimize))
	}
	if limit != 3 && limit >= 0 && limit <= 5 {
		buffer.WriteString(fmt.Sprintf("&show=%d", limit))
	}

	Get(buffer.String(), &resp)

	resp.fixStartEndLocations(from, to)

	return resp
}

// Add route start & end locations to the route information instead of displaying empty fields.
func (r RouteResponse) fixStartEndLocations(from Location, to Location) {
	for i := 0; i < len(r); i++ {
		r[i][0].Legs[0].Locs[0].Name = from.Name
		r[i][0].Legs[len(r[i][0].Legs)-1].Locs[len(r[i][0].Legs[len(r[i][0].Legs)-1].Locs)-1].Name = to.Name
	}
}

func Get(s string, v interface{}) bool {
	url := fmt.Sprintf("http://api.reittiopas.fi/hsl/prod/?user=%s&pass=%s&%s", api_user, api_pass, s)

	resp, err := http.Get(url)
	if err != nil {
		log.Println(err)
		return false
	}

	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		b, _ := ioutil.ReadAll(resp.Body)
		if string(b) != "" {
			log.Println(string(b))
		}
		return false
	}

	err = json.NewDecoder(resp.Body).Decode(v)
	if err != nil {
		// These aren't fatal errors most of the time.
		log.Printf(err.Error())
	}

	return true
}

func (l Legs) Line() string {
	switch l.Type {
	case "1", "2", "3", "4":
		return l.Code[2:][:4]
	case "5", "8":
		return l.Code[1:][:4]
	case "6":
		return "metro"
	case "12":
		return l.Code[4:5]
	case "walk":
		return l.Type
	}
	return l.Code
}
<|endoftext|>


<commit_before>package dns

import (
	"strings"
)

// Helper functions

// Only return those RR with the type t
func SieveRR(rr []RR, t uint16) (l []RR) {
	for _, r := range rr {
		if r.Header().Rrtype == t {
			l = append(l, r)
		}
	}
	return
}

// DnameLength returns the length of a packed dname.
func DomainNameLength(s string) int { // TODO better name
	// Special case for '.'
	if s == "." {
		return 1
	}

	// Add trailing dot to canonicalize name.
	if n := len(s); n == 0 || s[n-1] != '.' {
		return n + 1
	} else {
		return n + 1
	}
	panic("not reached")
	return 0
}

// Return a slice with the full name, one label shorter,
// shorter, until we, hit the root
// Each name is fully qualified.
// The root labels is added as last
func LabelSlice(s string) []string {
	if s[len(s)-1] != '.' {
		s += "."
	}
	ss := strings.Split(s, ".")
	labels := make([]string, 0)
	for i := 0; i < len(ss)-1; i++ {
		labels = append(labels, strings.Join(ss[i:], "."))
	}
	labels = append(labels, ".")
	return labels
}

// LabelSliceReverse reverse a label slice
func LabelSliceReverse(l []string) []string {
	for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 {
		l[i], l[j] = l[j], l[i]
	}
	return l
}

// Labels returns i labels from the left side of the name.
// The counting start with 0. The returned names are fully
// qualified.
// So for "a.miek.nl." it returns [ a.miek.nl. miek.nl. nl. . ]
func Labels(s string, i int) string {
	ss := strings.Split(s, ".")
	return strings.Join(ss[:i+1], ".")
	// TODO fully qualified
}

// Labels returns the number of labels in a domain name.
func LabelCount(a string) (c uint8) {
	// walk the string and count the dots
	// except when it is escaped
	esc := false
	for _, v := range a {
		switch v {
		case '.':
			if esc {
				esc = !esc
				continue
			}
			c++
		case '\\':
			esc = true
		}
	}
	return
}
<commit_msg>And remove it again<commit_after><|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package instancepoller

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/juju/errors"
	jc "github.com/juju/testing/checkers"
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/testing/testbase"
)

type aggregateSuite struct {
	testbase.LoggingSuite
}

var _ = gc.Suite(&aggregateSuite{})

type testInstance struct {
	instance.Instance
	addresses []instance.Address
	status    string
	err       error
}

var _ instance.Instance = (*testInstance)(nil)

func (t *testInstance) Addresses() ([]instance.Address, error) {
	if t.err != nil {
		return nil, t.err
	}
	return t.addresses, nil
}

func (t *testInstance) Status() string {
	return t.status
}

type testInstanceGetter struct {
	// ids is set when the Instances method is called.
	ids     []instance.Id
	results []instance.Instance
	err     error
	counter int32
}

func (i *testInstanceGetter) Instances(ids []instance.Id) (result []instance.Instance, err error) {
	i.ids = ids
	atomic.AddInt32(&i.counter, 1)
	return i.results, i.err
}

func newTestInstance(status string, addresses []string) *testInstance {
	thisInstance := testInstance{status: status}
	thisInstance.addresses = instance.NewAddresses(addresses...)
	return &thisInstance
}

func (s *aggregateSuite) TestSingleRequest(c *gc.C) {
	testGetter := new(testInstanceGetter)
	instance1 := newTestInstance("foobar", []string{"127.0.0.1", "192.168.1.1"})
	testGetter.results = []instance.Instance{instance1}
	aggregator := newAggregator(testGetter)

	info, err := aggregator.instanceInfo("foo")
	c.Assert(err, gc.IsNil)
	c.Assert(info, gc.DeepEquals, instanceInfo{
		status:    "foobar",
		addresses: instance1.addresses,
	})
	c.Assert(testGetter.ids, gc.DeepEquals, []instance.Id{"foo"})
}

func (s *aggregateSuite) TestMultipleResponseHandling(c *gc.C) {
	s.PatchValue(&gatherTime, 30*time.Millisecond)
	testGetter := new(testInstanceGetter)

	instance1 := newTestInstance("foobar", []string{"127.0.0.1", "192.168.1.1"})
	testGetter.results = []instance.Instance{instance1}
	aggregator := newAggregator(testGetter)

	replyChan := make(chan instanceInfoReply)
	req := instanceInfoReq{
		reply:  replyChan,
		instId: instance.Id("foo"),
	}
	aggregator.reqc <- req
	reply := <-replyChan
	c.Assert(reply.err, gc.IsNil)

	instance2 := newTestInstance("not foobar", []string{"192.168.1.2"})
	instance3 := newTestInstance("ok-ish", []string{"192.168.1.3"})
	testGetter.results = []instance.Instance{instance2, instance3}

	var wg sync.WaitGroup
	checkInfo := func(id instance.Id, expectStatus string) {
		info, err := aggregator.instanceInfo(id)
		c.Check(err, gc.IsNil)
		c.Check(info.status, gc.Equals, expectStatus)
		wg.Done()
	}

	wg.Add(2)
	go checkInfo("foo2", "not foobar")
	go checkInfo("foo3", "ok-ish")
	wg.Wait()

	c.Assert(len(testGetter.ids), gc.DeepEquals, 2)
}

func (s *aggregateSuite) TestBatching(c *gc.C) {
	s.PatchValue(&gatherTime, 10*time.Millisecond)
	testGetter := new(testInstanceGetter)

	aggregator := newAggregator(testGetter)
	for i := 0; i < 100; i++ {
		testGetter.results = append(testGetter.results, newTestInstance("foobar", []string{"127.0.0.1", "192.168.1.1"}))
	}
	var wg sync.WaitGroup
	makeRequest := func() {
		_, err := aggregator.instanceInfo("foo")
		c.Check(err, gc.IsNil)
		wg.Done()
	}
	startTime := time.Now()
	wg.Add(100)
	for i := 0; i < 100; i++ {
		go makeRequest()
		time.Sleep(time.Millisecond)
	}
	wg.Wait()
	totalTime := time.Now().Sub(startTime)
	// +1 because we expect one extra call for the first request
	expectedMax := int32((totalTime / (10 * time.Millisecond)) + 1)
	c.Assert(testGetter.counter, jc.LessThan, expectedMax+1)
	c.Assert(testGetter.counter, jc.GreaterThan, 10)
}

func (s *aggregateSuite) TestError(c *gc.C) {
	testGetter := new(testInstanceGetter)
	ourError := fmt.Errorf("Some error")
	testGetter.err = ourError

	aggregator := newAggregator(testGetter)

	_, err := aggregator.instanceInfo("foo")
	c.Assert(err, gc.Equals, ourError)
}

func (s *aggregateSuite) TestPartialErrResponse(c *gc.C) {
	testGetter := new(testInstanceGetter)
	testGetter.err = environs.ErrPartialInstances
	testGetter.results = []instance.Instance{nil}

	aggregator := newAggregator(testGetter)
	_, err := aggregator.instanceInfo("foo")

	c.Assert(err, gc.ErrorMatches, "instance foo not found")
	c.Assert(err, jc.Satisfies, errors.IsNotFound)
}

func (s *aggregateSuite) TestAddressesError(c *gc.C) {
	testGetter := new(testInstanceGetter)
	instance1 := newTestInstance("foobar", []string{"127.0.0.1", "192.168.1.1"})
	ourError := fmt.Errorf("gotcha")
	instance1.err = ourError
	testGetter.results = []instance.Instance{instance1}

	aggregator := newAggregator(testGetter)
	_, err := aggregator.instanceInfo("foo")
	c.Assert(err, gc.Equals, ourError)
}

func (s *aggregateSuite) TestKillAndWait(c *gc.C) {
	testGetter := new(testInstanceGetter)
	aggregator := newAggregator(testGetter)
	aggregator.Kill()
	err := aggregator.Wait()
	c.Assert(err, gc.IsNil)
}
<commit_msg>[r=axwalk],[bug=1319652] worker/instancepoller: TestBatching less race-y<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package instancepoller

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/juju/errors"
	jc "github.com/juju/testing/checkers"
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/testing/testbase"
)

type aggregateSuite struct {
	testbase.LoggingSuite
}

var _ = gc.Suite(&aggregateSuite{})

type testInstance struct {
	instance.Instance
	addresses []instance.Address
	status    string
	err       error
}

var _ instance.Instance = (*testInstance)(nil)

func (t *testInstance) Addresses() ([]instance.Address, error) {
	if t.err != nil {
		return nil, t.err
	}
	return t.addresses, nil
}

func (t *testInstance) Status() string {
	return t.status
}

type testInstanceGetter struct {
	// ids is set when the Instances method is called.
	ids     []instance.Id
	results []instance.Instance
	err     error
	counter int32
}

func (i *testInstanceGetter) Instances(ids []instance.Id) (result []instance.Instance, err error) {
	i.ids = ids
	atomic.AddInt32(&i.counter, 1)
	return i.results, i.err
}

func newTestInstance(status string, addresses []string) *testInstance {
	thisInstance := testInstance{status: status}
	thisInstance.addresses = instance.NewAddresses(addresses...)
	return &thisInstance
}

func (s *aggregateSuite) TestSingleRequest(c *gc.C) {
	testGetter := new(testInstanceGetter)
	instance1 := newTestInstance("foobar", []string{"127.0.0.1", "192.168.1.1"})
	testGetter.results = []instance.Instance{instance1}
	aggregator := newAggregator(testGetter)

	info, err := aggregator.instanceInfo("foo")
	c.Assert(err, gc.IsNil)
	c.Assert(info, gc.DeepEquals, instanceInfo{
		status:    "foobar",
		addresses: instance1.addresses,
	})
	c.Assert(testGetter.ids, gc.DeepEquals, []instance.Id{"foo"})
}

func (s *aggregateSuite) TestMultipleResponseHandling(c *gc.C) {
	s.PatchValue(&gatherTime, 30*time.Millisecond)
	testGetter := new(testInstanceGetter)

	instance1 := newTestInstance("foobar", []string{"127.0.0.1", "192.168.1.1"})
	testGetter.results = []instance.Instance{instance1}
	aggregator := newAggregator(testGetter)

	replyChan := make(chan instanceInfoReply)
	req := instanceInfoReq{
		reply:  replyChan,
		instId: instance.Id("foo"),
	}
	aggregator.reqc <- req
	reply := <-replyChan
	c.Assert(reply.err, gc.IsNil)

	instance2 := newTestInstance("not foobar", []string{"192.168.1.2"})
	instance3 := newTestInstance("ok-ish", []string{"192.168.1.3"})
	testGetter.results = []instance.Instance{instance2, instance3}

	var wg sync.WaitGroup
	checkInfo := func(id instance.Id, expectStatus string) {
		info, err := aggregator.instanceInfo(id)
		c.Check(err, gc.IsNil)
		c.Check(info.status, gc.Equals, expectStatus)
		wg.Done()
	}

	wg.Add(2)
	go checkInfo("foo2", "not foobar")
	go checkInfo("foo3", "ok-ish")
	wg.Wait()

	c.Assert(len(testGetter.ids), gc.DeepEquals, 2)
}

type batchingInstanceGetter struct {
	testInstanceGetter
	wg         sync.WaitGroup
	aggregator *aggregator
	batchSize  int
	started    int
}

func (g *batchingInstanceGetter) Instances(ids []instance.Id) ([]instance.Instance, error) {
	insts, err := g.testInstanceGetter.Instances(ids)
	g.startRequests()
	return insts, err
}

func (g *batchingInstanceGetter) startRequests() {
	n := len(g.results) - g.started
	if n > g.batchSize {
		n = g.batchSize
	}
	for i := 0; i < n; i++ {
		g.startRequest()
	}
}

func (g *batchingInstanceGetter) startRequest() {
	g.started++
	go func() {
		_, err := g.aggregator.instanceInfo("foo")
		if err != nil {
			panic(err)
		}
		g.wg.Done()
	}()
}

func (s *aggregateSuite) TestBatching(c *gc.C) {
	s.PatchValue(&gatherTime, 10*time.Millisecond)
	var testGetter batchingInstanceGetter
	testGetter.aggregator = newAggregator(&testGetter)
	testGetter.results = make([]instance.Instance, 100)
	for i := range testGetter.results {
		testGetter.results[i] = newTestInstance("foobar", []string{"127.0.0.1", "192.168.1.1"})
	}
	testGetter.batchSize = 10
	testGetter.wg.Add(len(testGetter.results))
	testGetter.startRequest()
	testGetter.wg.Wait()
	c.Assert(testGetter.counter, gc.Equals, int32(len(testGetter.results)/testGetter.batchSize)+1)
}

func (s *aggregateSuite) TestError(c *gc.C) {
	testGetter := new(testInstanceGetter)
	ourError := fmt.Errorf("Some error")
	testGetter.err = ourError

	aggregator := newAggregator(testGetter)

	_, err := aggregator.instanceInfo("foo")
	c.Assert(err, gc.Equals, ourError)
}

func (s *aggregateSuite) TestPartialErrResponse(c *gc.C) {
	testGetter := new(testInstanceGetter)
	testGetter.err = environs.ErrPartialInstances
	testGetter.results = []instance.Instance{nil}

	aggregator := newAggregator(testGetter)
	_, err := aggregator.instanceInfo("foo")

	c.Assert(err, gc.ErrorMatches, "instance foo not found")
	c.Assert(err, jc.Satisfies, errors.IsNotFound)
}

func (s *aggregateSuite) TestAddressesError(c *gc.C) {
	testGetter := new(testInstanceGetter)
	instance1 := newTestInstance("foobar", []string{"127.0.0.1", "192.168.1.1"})
	ourError := fmt.Errorf("gotcha")
	instance1.err = ourError
	testGetter.results = []instance.Instance{instance1}

	aggregator := newAggregator(testGetter)
	_, err := aggregator.instanceInfo("foo")
	c.Assert(err, gc.Equals, ourError)
}

func (s *aggregateSuite) TestKillAndWait(c *gc.C) {
	testGetter := new(testInstanceGetter)
	aggregator := newAggregator(testGetter)
	aggregator.Kill()
	err := aggregator.Wait()
	c.Assert(err, gc.IsNil)
}
<|endoftext|>


<commit_before>package actions

import (
	"errors"

	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/gaia-docker/tugbot/container"
	"github.com/samalba/dockerclient"
	"strings"
)

// Run looks at Docker containers to see if any of the images
// used to start those containers is a test container.
// For each test container it'll create and start a new container according
// to tugbots' labels.
func Run(client container.Client, names []string, e *dockerclient.Event) error {
	var ret []string
	if !container.IsSwarmTask(e) && !container.IsCreatedByTugbot(e) {
		candidates, err := client.ListContainers(containerFilter(names))
		if err != nil {
			return err
		}
		for _, candidate := range candidates {
			if candidate.IsEventListener(e) {
				if err := client.StartContainerFrom(candidate); err != nil {
					log.Error(err)
					ret = append(ret, err.Error())
				}
			}
		}
	}

	return errors.New(fmt.Errorf(strings.Join(ret, "\n")))
}

func containerFilter(names []string) container.Filter {

	return func(c container.Container) bool {
		return nameFilter(names)(c) && c.IsTugbotCandidate()
	}
}

func nameFilter(names []string) container.Filter {

	if len(names) == 0 {
		// all containers
		return func(container.Container) bool {
			return true
		}
	}

	return func(c container.Container) bool {
		for _, name := range names {
			if (name == c.Name()) || (name == c.Name()[1:]) {
				return true
			}
		}
		return false
	}
}
<commit_msg>fix build;<commit_after>package actions

import (
	"fmt"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/gaia-docker/tugbot/container"
	"github.com/samalba/dockerclient"
)

// Run looks at Docker containers to see if any of the images
// used to start those containers is a test container.
// For each test container it'll create and start a new container according
// to tugbots' labels.
func Run(client container.Client, names []string, e *dockerclient.Event) error {
	var ret []string
	if !container.IsSwarmTask(e) && !container.IsCreatedByTugbot(e) {
		candidates, err := client.ListContainers(containerFilter(names))
		if err != nil {
			return err
		}
		for _, candidate := range candidates {
			if candidate.IsEventListener(e) {
				if err := client.StartContainerFrom(candidate); err != nil {
					log.Error(err)
					ret = append(ret, err.Error())
				}
			}
		}
	}

	return fmt.Errorf(strings.Join(ret, "\n"))
}

func containerFilter(names []string) container.Filter {

	return func(c container.Container) bool {
		return nameFilter(names)(c) && c.IsTugbotCandidate()
	}
}

func nameFilter(names []string) container.Filter {

	if len(names) == 0 {
		// all containers
		return func(container.Container) bool {
			return true
		}
	}

	return func(c container.Container) bool {
		for _, name := range names {
			if (name == c.Name()) || (name == c.Name()[1:]) {
				return true
			}
		}
		return false
	}
}
<|endoftext|>


<commit_before>// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package main

import (
	"flag"
	mathrand "math/rand"
	"os"
	"time"

	"github.com/aws/amazon-ecs-agent/agent/acs"
	"github.com/aws/amazon-ecs-agent/agent/api"
	"github.com/aws/amazon-ecs-agent/agent/auth"
	"github.com/aws/amazon-ecs-agent/agent/config"
	"github.com/aws/amazon-ecs-agent/agent/engine"
	"github.com/aws/amazon-ecs-agent/agent/eventhandler"
	"github.com/aws/amazon-ecs-agent/agent/handlers"
	"github.com/aws/amazon-ecs-agent/agent/logger"
	"github.com/aws/amazon-ecs-agent/agent/sighandlers"
	"github.com/aws/amazon-ecs-agent/agent/statemanager"
	"github.com/aws/amazon-ecs-agent/agent/utils"
)

func init() {
	mathrand.Seed(time.Now().UnixNano())
}

func main() {
	acceptInsecureCert := flag.Bool("k", false, "Do not verify ssl certs")
	logLevel := flag.String("loglevel", "", "Loglevel: [<crit>|<error>|<warn>|<info>|<debug>]")
	flag.Parse()
	logger.SetLevel(*logLevel)
	log := logger.ForModule("main")

	log.Info("Starting Agent")

	log.Info("Loading configuration")
	cfg, err := config.NewConfig()
	if err != nil {
		log.Error("Error loading config", "err", err)
		os.Exit(1)
	}

	containerInstanceArn := ""
	taskEngine := engine.NewTaskEngine(cfg)

	// Load any state from disk *before* talking to the docker daemon or any
	// network services such that the information we get from them is applied to
	// the correct state
	var state_manager statemanager.StateManager
	if !cfg.Checkpoint {
		state_manager = statemanager.NewNoopStateManager()
	} else {
		state_manager, err = statemanager.NewStateManager(cfg, statemanager.AddSaveable("TaskEngine", taskEngine), statemanager.AddSaveable("ContainerInstanceArn", &containerInstanceArn))
		if err != nil {
			log.Crit("Error creating state manager", "err", err)
			os.Exit(1)
		}
	}
	err = state_manager.Load()
	if err != nil {
		log.Crit("Error loading initial state", "err", err)
		os.Exit(1)
	}

	// Begin listening to the docker daemon and saving changes
	taskEngine.SetSaver(state_manager)
	taskEngine.MustInit()

	credentialProvider := auth.NewBasicAWSCredentialProvider()
	client := api.NewECSClient(credentialProvider, cfg, *acceptInsecureCert)

	if containerInstanceArn == "" {
		log.Info("Registering Instance with ECS")
		containerInstanceArn, err = client.RegisterContainerInstance()
		if err != nil {
			log.Error("Error registering", "err", err)
			os.Exit(1)
		}
		log.Info("Registration completed successfully", "containerInstance", containerInstanceArn, "cluster", cfg.ClusterArn)
		// Save our shiny new containerInstanceArn
		state_manager.Save()
	} else {
		log.Info("Restored state", "containerInstance", containerInstanceArn, "cluster", cfg.ClusterArn)
	}

	sighandlers.StartTerminationHandler(state_manager)

	// Agent introspection api
	go handlers.ServeHttp(&containerInstanceArn, taskEngine, cfg)

	// Start sending events to the backend
	go eventhandler.HandleEngineEvents(taskEngine, client, state_manager)

	log.Info("Beginning Polling for updates")
	// Todo, split into separate package
	for {
		backoff := utils.NewSimpleBackoff(time.Second, 1*time.Minute, 0.2, 2)
		utils.RetryWithBackoff(backoff, func() error {
			acsEndpoint, err := client.DiscoverPollEndpoint(containerInstanceArn)
			if err != nil {
				log.Error("Could not discover poll endpoint", "err", err)
				return err
			}
			log.Info("Discovered poll endpoint", "endpoint", acsEndpoint)
			acsObj := acs.NewAgentCommunicationClient(acsEndpoint, cfg, credentialProvider, containerInstanceArn)

			state_changes, errc, err := acsObj.Poll(*acceptInsecureCert)
			if err != nil {
				log.Error("Error polling; retrying", "err", err)
				return err
			}

			var err_ok bool
			for state_changes != nil {
				select {
				case state, state_ok := <-state_changes:
					if state_ok {
						backoff.Reset()

						go func(payload *acs.Payload) {
							for _, task := range payload.Tasks {
								taskEngine.AddTask(task)
							}

							err = state_manager.Save()
							if err != nil {
								log.Error("Error saving state", "err", err)
							}
							acsObj.Ack(payload)
						}(state)
					} else {
						// Break out of the loop to reconnect
						state_changes = nil
					}
				case err, err_ok = <-errc:
					if !err_ok {
						log.Error("Error channel unexpectedly closed")

						state_changes = nil // break out
					}
					log.Error("Error in state", "err", err)
				}
			}
			if err != nil {
				log.Warn("Error polling. Waiting and retrying")
				return err
			}
			// Shouldn't happen
			return nil
		})
	}
}
<commit_msg>SaveState: Save cluster as well<commit_after>// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package main

import (
	"flag"
	mathrand "math/rand"
	"os"
	"time"

	"github.com/aws/amazon-ecs-agent/agent/acs"
	"github.com/aws/amazon-ecs-agent/agent/api"
	"github.com/aws/amazon-ecs-agent/agent/auth"
	"github.com/aws/amazon-ecs-agent/agent/config"
	"github.com/aws/amazon-ecs-agent/agent/engine"
	"github.com/aws/amazon-ecs-agent/agent/eventhandler"
	"github.com/aws/amazon-ecs-agent/agent/handlers"
	"github.com/aws/amazon-ecs-agent/agent/logger"
	"github.com/aws/amazon-ecs-agent/agent/sighandlers"
	"github.com/aws/amazon-ecs-agent/agent/statemanager"
	"github.com/aws/amazon-ecs-agent/agent/utils"
)

func init() {
	mathrand.Seed(time.Now().UnixNano())
}

func main() {
	acceptInsecureCert := flag.Bool("k", false, "Do not verify ssl certs")
	logLevel := flag.String("loglevel", "", "Loglevel: [<crit>|<error>|<warn>|<info>|<debug>]")
	flag.Parse()
	logger.SetLevel(*logLevel)
	log := logger.ForModule("main")

	log.Info("Starting Agent")

	log.Info("Loading configuration")
	cfg, err := config.NewConfig()
	if err != nil {
		log.Error("Error loading config", "err", err)
		os.Exit(1)
	}

	containerInstanceArn := ""
	configuredCluster := cfg.ClusterArn // Save a copy of the cluster so we can verify the loaded one matches
	taskEngine := engine.NewTaskEngine(cfg)

	// Load any state from disk *before* talking to the docker daemon or any
	// network services such that the information we get from them is applied to
	// the correct state
	var state_manager statemanager.StateManager
	if !cfg.Checkpoint {
		state_manager = statemanager.NewNoopStateManager()
	} else {
		state_manager, err = statemanager.NewStateManager(cfg,
			statemanager.AddSaveable("TaskEngine", taskEngine),
			statemanager.AddSaveable("ContainerInstanceArn", &containerInstanceArn),
			statemanager.AddSaveable("ClusterArn", &cfg.ClusterArn),
		)
		if err != nil {
			log.Crit("Error creating state manager", "err", err)
			os.Exit(1)
		}
	}
	err = state_manager.Load()
	if err != nil {
		log.Crit("Error loading initial state", "err", err)
		os.Exit(1)
	}

	if cfg.Checkpoint && configuredCluster != "" && configuredCluster != cfg.ClusterArn {
		log.Crit("Cluster mismatch; saved cluster does not match configured cluster", "configured", configuredCluster, "saved", cfg.ClusterArn)
		os.Exit(1)
	}

	// Begin listening to the docker daemon and saving changes
	taskEngine.SetSaver(state_manager)
	taskEngine.MustInit()

	credentialProvider := auth.NewBasicAWSCredentialProvider()
	client := api.NewECSClient(credentialProvider, cfg, *acceptInsecureCert)

	if containerInstanceArn == "" {
		log.Info("Registering Instance with ECS")
		containerInstanceArn, err = client.RegisterContainerInstance()
		if err != nil {
			log.Error("Error registering", "err", err)
			os.Exit(1)
		}
		log.Info("Registration completed successfully", "containerInstance", containerInstanceArn, "cluster", cfg.ClusterArn)
		// Save our shiny new containerInstanceArn
		state_manager.Save()
	} else {
		log.Info("Restored state", "containerInstance", containerInstanceArn, "cluster", cfg.ClusterArn)
	}

	sighandlers.StartTerminationHandler(state_manager)

	// Agent introspection api
	go handlers.ServeHttp(&containerInstanceArn, taskEngine, cfg)

	// Start sending events to the backend
	go eventhandler.HandleEngineEvents(taskEngine, client, state_manager)

	log.Info("Beginning Polling for updates")
	// Todo, split into separate package
	for {
		backoff := utils.NewSimpleBackoff(time.Second, 1*time.Minute, 0.2, 2)
		utils.RetryWithBackoff(backoff, func() error {
			acsEndpoint, err := client.DiscoverPollEndpoint(containerInstanceArn)
			if err != nil {
				log.Error("Could not discover poll endpoint", "err", err)
				return err
			}
			log.Info("Discovered poll endpoint", "endpoint", acsEndpoint)
			acsObj := acs.NewAgentCommunicationClient(acsEndpoint, cfg, credentialProvider, containerInstanceArn)

			state_changes, errc, err := acsObj.Poll(*acceptInsecureCert)
			if err != nil {
				log.Error("Error polling; retrying", "err", err)
				return err
			}

			var err_ok bool
			for state_changes != nil {
				select {
				case state, state_ok := <-state_changes:
					if state_ok {
						backoff.Reset()

						go func(payload *acs.Payload) {
							for _, task := range payload.Tasks {
								taskEngine.AddTask(task)
							}

							err = state_manager.Save()
							if err != nil {
								log.Error("Error saving state", "err", err)
							}
							acsObj.Ack(payload)
						}(state)
					} else {
						// Break out of the loop to reconnect
						state_changes = nil
					}
				case err, err_ok = <-errc:
					if !err_ok {
						log.Error("Error channel unexpectedly closed")

						state_changes = nil // break out
					}
					log.Error("Error in state", "err", err)
				}
			}
			if err != nil {
				log.Warn("Error polling. Waiting and retrying")
				return err
			}
			// Shouldn't happen
			return nil
		})
	}
}
<|endoftext|>