<commit_before>// Copyright 2017 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"context"
	"crypto/tls"
	"errors"
	"fmt"
	"path"

	"github.com/coreos/etcd-operator/pkg/backup/util"
	"github.com/coreos/etcd-operator/pkg/backup/writer"
	"github.com/coreos/etcd-operator/pkg/util/constants"
	"github.com/coreos/etcd-operator/pkg/util/etcdutil"
	"github.com/coreos/etcd-operator/pkg/util/k8sutil"

	"github.com/coreos/etcd/clientv3"
	"github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	"k8s.io/client-go/kubernetes"
)

// BackupManager backups an etcd cluster.
type BackupManager struct {
	kubecli kubernetes.Interface

	clusterName   string
	namespace     string
	etcdTLSConfig *tls.Config

	bw writer.Writer
}

// NewBackupManagerFromWriter creates a BackupManager with backup writer.
func NewBackupManagerFromWriter(kubecli kubernetes.Interface, bw writer.Writer, clusterName, namespace string) *BackupManager {
	return &BackupManager{
		kubecli:     kubecli,
		clusterName: clusterName,
		namespace:   namespace,
		bw:          bw,
	}
}

// SaveSnapWithPrefix uses backup writer to save latest snapshot to a path prepended with the given prefix
// and returns file size and full path.
// the full path has the format of prefix/<etcd_version>_<snapshot_reversion>_etcd.backup
// e.g prefix = etcd-backups/v1/default/example-etcd-cluster and
// backup object name = 3.2.10_0000000000000001_etcd.backup
// full path is "etcd-backups/v1/default/example-etcd-cluster/3.2.10_0000000000000001_etcd.backup".
func (bm *BackupManager) SaveSnapWithPrefix(prefix string) (string, error) {
	etcdcli, rev, err := bm.etcdClientWithMaxRevision()
	if err != nil {
		return "", fmt.Errorf("create etcd client failed: %v", err)
	}
	defer etcdcli.Close()

	ctx, cancel := context.WithTimeout(context.Background(), constants.DefaultSnapshotTimeout)
	defer cancel() // Can't cancel() after Snapshot() because that will close the reader.
	rc, err := etcdcli.Snapshot(ctx)
	if err != nil {
		return "", fmt.Errorf("failed to receive snapshot (%v)", err)
	}
	defer rc.Close()

	version, err := getEtcdVersion(etcdcli.Maintenance, etcdcli.Endpoints()[0])
	if err != nil {
		return "", err
	}
	fullPath := path.Join(prefix, util.MakeBackupName(version, rev))
	_, err = bm.bw.Write(fullPath, rc)
	if err != nil {
		return "", fmt.Errorf("failed to write snapshot (%v)", err)
	}
	return fullPath, nil
}

func getEtcdVersion(mcli clientv3.Maintenance, endpoint string) (string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), constants.DefaultSnapshotTimeout)
	resp, err := mcli.Status(ctx, endpoint)
	cancel()
	if err != nil {
		return "", fmt.Errorf("failed to receive etcd version (%v)", err)
	}
	return resp.Version, nil
}

// etcdClientWithMaxRevision gets the etcd member with the maximum kv store revision
// and returns the etcd client and the rev of that member.
func (bm *BackupManager) etcdClientWithMaxRevision() (*clientv3.Client, int64, error) {
	podList, err := bm.kubecli.Core().Pods(bm.namespace).List(k8sutil.ClusterListOpt(bm.clusterName))
	if err != nil {
		return nil, 0, err
	}

	var pods []*v1.Pod
	for i := range podList.Items {
		pod := &podList.Items[i]
		if pod.Status.Phase == v1.PodRunning {
			pods = append(pods, pod)
		}
	}

	if len(pods) == 0 {
		return nil, 0, errors.New("no running etcd pods found")
	}
	member, rev := getMemberWithMaxRev(pods, bm.etcdTLSConfig)
	if member == nil {
		return nil, 0, errors.New("no reachable member")
	}

	etcdcli, err := createEtcdClient(member.ClientURL(), bm.etcdTLSConfig)
	if err != nil {
		return nil, 0, fmt.Errorf("create etcd client failed: %v", err)
	}
	return etcdcli, rev, nil
}

func getMemberWithMaxRev(pods []*v1.Pod, tc *tls.Config) (*etcdutil.Member, int64) {
	var member *etcdutil.Member
	maxRev := int64(0)
	for _, pod := range pods {
		m := &etcdutil.Member{
			Name:         pod.Name,
			Namespace:    pod.Namespace,
			SecureClient: tc != nil,
		}
		cfg := clientv3.Config{
			Endpoints:   []string{m.ClientURL()},
			DialTimeout: constants.DefaultDialTimeout,
			TLS:         tc,
		}
		etcdcli, err := clientv3.New(cfg)
		if err != nil {
			logrus.Warningf("failed to create etcd client for pod (%v): %v", pod.Name, err)
			continue
		}
		defer etcdcli.Close()

		ctx, cancel := context.WithTimeout(context.Background(), constants.DefaultRequestTimeout)
		resp, err := etcdcli.Get(ctx, "/", clientv3.WithSerializable())
		cancel()
		if err != nil {
			logrus.Warningf("getMaxRev: failed to get revision from member %s (%s)", m.Name, m.ClientURL())
			continue
		}

		logrus.Infof("getMaxRev: member %s revision (%d)", m.Name, resp.Header.Revision)
		if resp.Header.Revision > maxRev {
			maxRev = resp.Header.Revision
			member = m
		}
	}
	return member, maxRev
}

func createEtcdClient(url string, tlsConfig *tls.Config) (*clientv3.Client, error) {
	cfg := clientv3.Config{
		Endpoints:   []string{url},
		DialTimeout: constants.DefaultDialTimeout,
		TLS:         tlsConfig,
	}
	return clientv3.New(cfg)
}
<commit_msg>backup: create backup manager now takes in tls config<commit_after>// Copyright 2017 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"context"
	"crypto/tls"
	"errors"
	"fmt"
	"path"

	"github.com/coreos/etcd-operator/pkg/backup/util"
	"github.com/coreos/etcd-operator/pkg/backup/writer"
	"github.com/coreos/etcd-operator/pkg/util/constants"
	"github.com/coreos/etcd-operator/pkg/util/etcdutil"
	"github.com/coreos/etcd-operator/pkg/util/k8sutil"

	"github.com/coreos/etcd/clientv3"
	"github.com/sirupsen/logrus"
	"k8s.io/api/core/v1"
	"k8s.io/client-go/kubernetes"
)

// BackupManager backups an etcd cluster.
type BackupManager struct {
	kubecli kubernetes.Interface

	clusterName   string
	namespace     string
	etcdTLSConfig *tls.Config

	bw writer.Writer
}

// NewBackupManagerFromWriter creates a BackupManager with backup writer.
func NewBackupManagerFromWriter(kubecli kubernetes.Interface, bw writer.Writer, tls *tls.Config, clusterName, namespace string) *BackupManager {
	return &BackupManager{
		kubecli:       kubecli,
		clusterName:   clusterName,
		namespace:     namespace,
		etcdTLSConfig: tls,
		bw:            bw,
	}
}

// SaveSnapWithPrefix uses backup writer to save latest snapshot to a path prepended with the given prefix
// and returns file size and full path.
// the full path has the format of prefix/<etcd_version>_<snapshot_reversion>_etcd.backup
// e.g prefix = etcd-backups/v1/default/example-etcd-cluster and
// backup object name = 3.2.10_0000000000000001_etcd.backup
// full path is "etcd-backups/v1/default/example-etcd-cluster/3.2.10_0000000000000001_etcd.backup".
func (bm *BackupManager) SaveSnapWithPrefix(prefix string) (string, error) {
	etcdcli, rev, err := bm.etcdClientWithMaxRevision()
	if err != nil {
		return "", fmt.Errorf("create etcd client failed: %v", err)
	}
	defer etcdcli.Close()

	ctx, cancel := context.WithTimeout(context.Background(), constants.DefaultSnapshotTimeout)
	defer cancel() // Can't cancel() after Snapshot() because that will close the reader.
	rc, err := etcdcli.Snapshot(ctx)
	if err != nil {
		return "", fmt.Errorf("failed to receive snapshot (%v)", err)
	}
	defer rc.Close()

	version, err := getEtcdVersion(etcdcli.Maintenance, etcdcli.Endpoints()[0])
	if err != nil {
		return "", err
	}
	fullPath := path.Join(prefix, util.MakeBackupName(version, rev))
	_, err = bm.bw.Write(fullPath, rc)
	if err != nil {
		return "", fmt.Errorf("failed to write snapshot (%v)", err)
	}
	return fullPath, nil
}

func getEtcdVersion(mcli clientv3.Maintenance, endpoint string) (string, error) {
	ctx, cancel := context.WithTimeout(context.Background(), constants.DefaultSnapshotTimeout)
	resp, err := mcli.Status(ctx, endpoint)
	cancel()
	if err != nil {
		return "", fmt.Errorf("failed to receive etcd version (%v)", err)
	}
	return resp.Version, nil
}

// etcdClientWithMaxRevision gets the etcd member with the maximum kv store revision
// and returns the etcd client and the rev of that member.
func (bm *BackupManager) etcdClientWithMaxRevision() (*clientv3.Client, int64, error) {
	podList, err := bm.kubecli.Core().Pods(bm.namespace).List(k8sutil.ClusterListOpt(bm.clusterName))
	if err != nil {
		return nil, 0, err
	}

	var pods []*v1.Pod
	for i := range podList.Items {
		pod := &podList.Items[i]
		if pod.Status.Phase == v1.PodRunning {
			pods = append(pods, pod)
		}
	}

	if len(pods) == 0 {
		return nil, 0, errors.New("no running etcd pods found")
	}
	member, rev := getMemberWithMaxRev(pods, bm.etcdTLSConfig)
	if member == nil {
		return nil, 0, errors.New("no reachable member")
	}

	etcdcli, err := createEtcdClient(member.ClientURL(), bm.etcdTLSConfig)
	if err != nil {
		return nil, 0, fmt.Errorf("create etcd client failed: %v", err)
	}
	return etcdcli, rev, nil
}

func getMemberWithMaxRev(pods []*v1.Pod, tc *tls.Config) (*etcdutil.Member, int64) {
	var member *etcdutil.Member
	maxRev := int64(0)
	for _, pod := range pods {
		m := &etcdutil.Member{
			Name:         pod.Name,
			Namespace:    pod.Namespace,
			SecureClient: tc != nil,
		}
		cfg := clientv3.Config{
			Endpoints:   []string{m.ClientURL()},
			DialTimeout: constants.DefaultDialTimeout,
			TLS:         tc,
		}
		etcdcli, err := clientv3.New(cfg)
		if err != nil {
			logrus.Warningf("failed to create etcd client for pod (%v): %v", pod.Name, err)
			continue
		}
		defer etcdcli.Close()

		ctx, cancel := context.WithTimeout(context.Background(), constants.DefaultRequestTimeout)
		resp, err := etcdcli.Get(ctx, "/", clientv3.WithSerializable())
		cancel()
		if err != nil {
			logrus.Warningf("getMaxRev: failed to get revision from member %s (%s)", m.Name, m.ClientURL())
			continue
		}

		logrus.Infof("getMaxRev: member %s revision (%d)", m.Name, resp.Header.Revision)
		if resp.Header.Revision > maxRev {
			maxRev = resp.Header.Revision
			member = m
		}
	}
	return member, maxRev
}

func createEtcdClient(url string, tlsConfig *tls.Config) (*clientv3.Client, error) {
	cfg := clientv3.Config{
		Endpoints:   []string{url},
		DialTimeout: constants.DefaultDialTimeout,
		TLS:         tlsConfig,
	}
	return clientv3.New(cfg)
}
<|endoftext|>


<commit_before>package controller

import (
	"context"
	"fmt"
	"time"

	clientset "github.com/matt-tyler/elasticsearch-operator/pkg/client/clientset/versioned"
	"github.com/matt-tyler/elasticsearch-operator/pkg/client/clientset/versioned/scheme"
	informers "github.com/matt-tyler/elasticsearch-operator/pkg/client/informers/externalversions"
	listers "github.com/matt-tyler/elasticsearch-operator/pkg/client/listers/es/v1"
	"github.com/matt-tyler/elasticsearch-operator/pkg/log"
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/util/runtime"

	clusterscheme "github.com/matt-tyler/elasticsearch-operator/pkg/client/clientset/versioned/scheme"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	kubeinformers "k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	appslisters "k8s.io/client-go/listers/apps/v1beta2"
	corelisters "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"
)

const maxRetries = 5
const controllerAgentName = "elasticsearch-cluster-controller"

const (
	// SuccessSynced is used as part of the Event 'reason' when a cluster is synced
	SuccessSynced = "Synced"

	// ErrResourceExists is used as part of the Event 'reason' when a cluster fails
	// to sync due to a resource already existing
	ErrResourceExists = "ErrResourceExists"

	// MessageResourceExists is the message used for events when a resource
	// fails to sync due to it already existing
	MessageResourceExists = "Resource %q already exists and is not managed by controller"

	// MessageResourceSynced is the messaged used for an Event fire when a Cluster
	// is successfully synced.
	MessageResourceSynced = "Resource %q synced successfully"
)

type Controller struct {
	log.Logger

	kubeclientset kubernetes.Interface
	esclientset   clientset.Interface

	kubeInformerFactory kubeinformers.SharedInformerFactory
	esInformerFactory   informers.SharedInformerFactory

	clustersSynced    cache.InformerSynced
	servicesSynced    cache.InformerSynced
	deploymentsSynced cache.InformerSynced

	clusterLister    listers.ClusterLister
	serviceLister    corelisters.ServiceLister
	deploymentLister appslisters.DeploymentLister

	queue workqueue.RateLimitingInterface

	recorder record.EventRecorder
}

func NewController(config *rest.Config) *Controller {
	queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())

	kubeclientset := kubernetes.NewForConfigOrDie(config)
	esclientset := clientset.NewForConfigOrDie(config)

	resyncPeriod := 0 * time.Second

	kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeclientset, resyncPeriod)
	esInformerFactory := informers.NewSharedInformerFactory(esclientset, resyncPeriod)

	clusterInformer := esInformerFactory.Es().V1().Clusters()
	serviceInformer := kubeInformerFactory.Core().V1().Services()
	deploymentInformer := kubeInformerFactory.Apps().V1beta2().Deployments()

	clusterInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			if key, err := cache.MetaNamespaceKeyFunc(obj); err == nil {
				queue.Add(key)
			}
		},
		UpdateFunc: func(oldObj interface{}, newObj interface{}) {
			if key, err := cache.MetaNamespaceKeyFunc(newObj); err == nil {
				queue.Add(key)
			}
		},
		DeleteFunc: func(obj interface{}) {
			if key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj); err == nil {
				queue.Add(key)
			}
		},
	})

	ctx := context.Background()

	go kubeInformerFactory.Start(ctx.Done())
	go esInformerFactory.Start(ctx.Done())

	logger := log.NewLogger()

	clusterscheme.AddToScheme(scheme.Scheme)
	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(logger.Infof)
	eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeclientset.CoreV1().Events("")})
	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName})

	return &Controller{
		logger,
		kubeclientset,
		esclientset,
		kubeInformerFactory,
		esInformerFactory,
		clusterInformer.Informer().HasSynced,
		serviceInformer.Informer().HasSynced,
		deploymentInformer.Informer().HasSynced,
		clusterInformer.Lister(),
		serviceInformer.Lister(),
		deploymentInformer.Lister(),
		queue,
		recorder,
	}
}

func (c *Controller) runWorker() {
	for c.processNextItem() {
		// continue looping
	}
}

func (c *Controller) processNextItem() bool {
	obj, shutdown := c.queue.Get()

	if shutdown {
		return false
	}

	err := func(obj interface{}) error {
		defer c.queue.Done(obj)

		var key string
		var ok bool

		if key, ok = obj.(string); !ok {
			c.queue.Forget(obj)
			runtime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj))
			return nil
		}

		if err := c.sync(key); err != nil {
			return fmt.Errorf("error syncing '%s': %s", key, err.Error())
		}

		c.queue.Forget(obj)
		return nil
	}(obj)

	if err != nil {
		runtime.HandleError(err)
	}

	return true
}

func (c *Controller) sync(key string) error {
	c.Infof("Processing change to %s", key)

	namespace, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		runtime.HandleError(fmt.Errorf("invalid resource key: %s", key))
		return nil
	}

	cluster, err := c.clusterLister.Clusters(namespace).Get(name)
	if err != nil {
		if errors.IsNotFound(err) {
			runtime.HandleError(fmt.Errorf("foo '%s' in work queue no longer exists", key))
			return nil
		}
		return err
	}

	c.Infof("Object: %#v", cluster)

	c.Infof("create master discovery service...")
	masterServiceName := fmt.Sprintf("%v-master-service", cluster.Name)
	masterService, err := c.serviceLister.Services(cluster.Namespace).Get(masterServiceName)
	if errors.IsNotFound(err) {
		masterService, err = c.kubeclientset.CoreV1().Services(cluster.Namespace).Create(newMasterService(cluster))
	}

	if err != nil {
		return err
	}

	if !metav1.IsControlledBy(masterService, cluster) {
		msg := fmt.Sprintf(MessageResourceExists, masterService.Name)
		c.recorder.Event(cluster, corev1.EventTypeWarning, ErrResourceExists, msg)
		return fmt.Errorf(msg)
	}

	serviceURL := fmt.Sprintf("%v.%v.svc.cluster.local", masterServiceName, cluster.Namespace)

	c.Infof("Creating master node deployment...")
	masterDeploymentName := fmt.Sprintf("%v-master-deployment", cluster.Name)
	masterDeployment, err := c.deploymentLister.Deployments(cluster.Namespace).Get(masterDeploymentName)
	if errors.IsNotFound(err) {
		masterDeployment, err = c.kubeclientset.AppsV1beta2().Deployments(cluster.Namespace).Create(newMasterDeployment(cluster, serviceURL))
	}

	if err != nil {
		return err
	}

	if !metav1.IsControlledBy(masterDeployment, cluster) {
		msg := fmt.Sprintf(MessageResourceExists, masterDeployment.Name)
		c.recorder.Event(cluster, corev1.EventTypeWarning, ErrResourceExists, msg)
		return fmt.Errorf(msg)
	}

	msg := fmt.Sprintf(MessageResourceSynced, cluster.Name)
	c.recorder.Event(cluster, corev1.EventTypeNormal, SuccessSynced, msg)

	return nil
}

func (c *Controller) Run(ctx context.Context) {
	defer utilruntime.HandleCrash()
	defer c.queue.ShutDown()

	c.Infof("Starting Controller...")

	if !cache.WaitForCacheSync(ctx.Done(), c.clustersSynced, c.servicesSynced, c.deploymentsSynced) {
		utilruntime.HandleError(fmt.Errorf("Timed out waiting for cache to sync"))
		return
	}

	wait.Until(c.runWorker, time.Second, ctx.Done())
}
<commit_msg>Add additional logging to controller<commit_after>package controller

import (
	"context"
	"fmt"
	"time"

	clientset "github.com/matt-tyler/elasticsearch-operator/pkg/client/clientset/versioned"
	"github.com/matt-tyler/elasticsearch-operator/pkg/client/clientset/versioned/scheme"
	informers "github.com/matt-tyler/elasticsearch-operator/pkg/client/informers/externalversions"
	listers "github.com/matt-tyler/elasticsearch-operator/pkg/client/listers/es/v1"
	"github.com/matt-tyler/elasticsearch-operator/pkg/log"
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/util/runtime"

	clusterscheme "github.com/matt-tyler/elasticsearch-operator/pkg/client/clientset/versioned/scheme"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	kubeinformers "k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	appslisters "k8s.io/client-go/listers/apps/v1beta2"
	corelisters "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"
)

const maxRetries = 5
const controllerAgentName = "elasticsearch-cluster-controller"

const (
	// SuccessSynced is used as part of the Event 'reason' when a cluster is synced
	SuccessSynced = "Synced"

	// ErrResourceExists is used as part of the Event 'reason' when a cluster fails
	// to sync due to a resource already existing
	ErrResourceExists = "ErrResourceExists"

	// MessageResourceExists is the message used for events when a resource
	// fails to sync due to it already existing
	MessageResourceExists = "Resource %q already exists and is not managed by controller"

	// MessageResourceSynced is the messaged used for an Event fire when a Cluster
	// is successfully synced.
	MessageResourceSynced = "Resource %q synced successfully"
)

type Controller struct {
	log.Logger

	kubeclientset kubernetes.Interface
	esclientset   clientset.Interface

	kubeInformerFactory kubeinformers.SharedInformerFactory
	esInformerFactory   informers.SharedInformerFactory

	clustersSynced    cache.InformerSynced
	servicesSynced    cache.InformerSynced
	deploymentsSynced cache.InformerSynced

	clusterLister    listers.ClusterLister
	serviceLister    corelisters.ServiceLister
	deploymentLister appslisters.DeploymentLister

	queue workqueue.RateLimitingInterface

	recorder record.EventRecorder
}

func NewController(config *rest.Config) *Controller {
	queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())

	kubeclientset := kubernetes.NewForConfigOrDie(config)
	esclientset := clientset.NewForConfigOrDie(config)

	resyncPeriod := 0 * time.Second

	kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeclientset, resyncPeriod)
	esInformerFactory := informers.NewSharedInformerFactory(esclientset, resyncPeriod)

	clusterInformer := esInformerFactory.Es().V1().Clusters()
	serviceInformer := kubeInformerFactory.Core().V1().Services()
	deploymentInformer := kubeInformerFactory.Apps().V1beta2().Deployments()

	clusterInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			if key, err := cache.MetaNamespaceKeyFunc(obj); err == nil {
				queue.Add(key)
			}
		},
		UpdateFunc: func(oldObj interface{}, newObj interface{}) {
			if key, err := cache.MetaNamespaceKeyFunc(newObj); err == nil {
				queue.Add(key)
			}
		},
		DeleteFunc: func(obj interface{}) {
			if key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj); err == nil {
				queue.Add(key)
			}
		},
	})

	ctx := context.Background()

	go kubeInformerFactory.Start(ctx.Done())
	go esInformerFactory.Start(ctx.Done())

	logger := log.NewLogger()

	clusterscheme.AddToScheme(scheme.Scheme)
	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(logger.Infof)
	eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeclientset.CoreV1().Events("")})
	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName})

	return &Controller{
		logger,
		kubeclientset,
		esclientset,
		kubeInformerFactory,
		esInformerFactory,
		clusterInformer.Informer().HasSynced,
		serviceInformer.Informer().HasSynced,
		deploymentInformer.Informer().HasSynced,
		clusterInformer.Lister(),
		serviceInformer.Lister(),
		deploymentInformer.Lister(),
		queue,
		recorder,
	}
}

func (c *Controller) runWorker() {
	c.Infof("Processing items")
	for c.processNextItem() {
		// continue looping
	}
}

func (c *Controller) processNextItem() bool {
	obj, shutdown := c.queue.Get()

	if shutdown {
		return false
	}

	err := func(obj interface{}) error {
		defer c.queue.Done(obj)

		var key string
		var ok bool

		if key, ok = obj.(string); !ok {
			c.queue.Forget(obj)
			runtime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj))
			return nil
		}

		if err := c.sync(key); err != nil {
			return fmt.Errorf("error syncing '%s': %s", key, err.Error())
		}

		c.queue.Forget(obj)
		return nil
	}(obj)

	if err != nil {
		runtime.HandleError(err)
	}

	return true
}

func (c *Controller) sync(key string) error {
	c.Infof("Processing change to %s", key)

	namespace, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		runtime.HandleError(fmt.Errorf("invalid resource key: %s", key))
		return nil
	}

	cluster, err := c.clusterLister.Clusters(namespace).Get(name)
	if err != nil {
		if errors.IsNotFound(err) {
			runtime.HandleError(fmt.Errorf("foo '%s' in work queue no longer exists", key))
			return nil
		}
		return err
	}

	c.Infof("Object: %#v", cluster)

	c.Infof("create master discovery service...")
	masterServiceName := fmt.Sprintf("%v-master-service", cluster.Name)
	masterService, err := c.serviceLister.Services(cluster.Namespace).Get(masterServiceName)
	if errors.IsNotFound(err) {
		masterService, err = c.kubeclientset.CoreV1().Services(cluster.Namespace).Create(newMasterService(cluster))
	}

	if err != nil {
		return err
	}

	if !metav1.IsControlledBy(masterService, cluster) {
		msg := fmt.Sprintf(MessageResourceExists, masterService.Name)
		c.recorder.Event(cluster, corev1.EventTypeWarning, ErrResourceExists, msg)
		return fmt.Errorf(msg)
	}

	serviceURL := fmt.Sprintf("%v.%v.svc.cluster.local", masterServiceName, cluster.Namespace)

	c.Infof("Creating master node deployment...")
	masterDeploymentName := fmt.Sprintf("%v-master-deployment", cluster.Name)
	masterDeployment, err := c.deploymentLister.Deployments(cluster.Namespace).Get(masterDeploymentName)
	if errors.IsNotFound(err) {
		masterDeployment, err = c.kubeclientset.AppsV1beta2().Deployments(cluster.Namespace).Create(newMasterDeployment(cluster, serviceURL))
	}

	if err != nil {
		return err
	}

	if !metav1.IsControlledBy(masterDeployment, cluster) {
		msg := fmt.Sprintf(MessageResourceExists, masterDeployment.Name)
		c.recorder.Event(cluster, corev1.EventTypeWarning, ErrResourceExists, msg)
		return fmt.Errorf(msg)
	}

	msg := fmt.Sprintf(MessageResourceSynced, cluster.Name)
	c.recorder.Event(cluster, corev1.EventTypeNormal, SuccessSynced, msg)

	return nil
}

func (c *Controller) Run(ctx context.Context) {
	defer utilruntime.HandleCrash()
	defer c.queue.ShutDown()

	c.Infof("Starting Controller...")

	if !cache.WaitForCacheSync(ctx.Done(), c.clustersSynced, c.servicesSynced, c.deploymentsSynced) {
		utilruntime.HandleError(fmt.Errorf("Timed out waiting for cache to sync"))
		return
	}

	c.Infof("Controller started")

	wait.Until(c.runWorker, time.Second, ctx.Done())
}
<|endoftext|>


<commit_before>/*
Package flexvolume implements utility code for Kubernetes flexvolumes.

https://github.com/kubernetes/kubernetes/pull/13840
https://github.com/kubernetes/kubernetes/tree/master/examples/flexvolume
*/
package flexvolume
<commit_msg>add public types and functions for flexvolume<commit_after>/*
Package flexvolume implements utility code for Kubernetes flexvolumes.

https://github.com/kubernetes/kubernetes/pull/13840
https://github.com/kubernetes/kubernetes/tree/master/examples/flexvolume
*/
package flexvolume

// Client is the client for a flexvolume implementation.
//
// It is both called from the wrapper cli tool, and implemented by a given implementation.
type Client interface {
	Init() error
	Attach(jsonOptions map[string]interface{}) error
	Detach(mountDevice string) error
	Mount(targetMountDir string, mountDevice string, jsonOptions map[string]interface{}) error
	Unmount(mountDir string) error
}

// NewClient returns a new Client for the given APIClient.
func NewClient(apiClient APIClient) Client {
	return nil
}

// NewLocalAPIClient returns a new APIClient for the given APIServer.
func NewLocalAPIClient(apiServer APIServer) APIClient {
	return nil
}

// NewAPIServer returns a new APIServer for the given Client.
func NewAPIServer(client Client) APIServer {
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logs

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"sync"
	"time"

	"github.com/spf13/cobra"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	"k8s.io/client-go/rest"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/kubectl/polymorphichelpers"
	"k8s.io/kubernetes/pkg/kubectl/scheme"
	"k8s.io/kubernetes/pkg/kubectl/util"
	"k8s.io/kubernetes/pkg/kubectl/util/i18n"
	"k8s.io/kubernetes/pkg/kubectl/util/templates"
)

const (
	logsUsageStr = "logs [-f] [-p] (POD | TYPE/NAME) [-c CONTAINER]"
)

var (
	logsExample = templates.Examples(i18n.T(`
		# Return snapshot logs from pod nginx with only one container
		kubectl logs nginx

		# Return snapshot logs from pod nginx with multi containers
		kubectl logs nginx --all-containers=true

		# Return snapshot logs from all containers in pods defined by label app=nginx
		kubectl logs -lapp=nginx --all-containers=true

		# Return snapshot of previous terminated ruby container logs from pod web-1
		kubectl logs -p -c ruby web-1

		# Begin streaming the logs of the ruby container in pod web-1
		kubectl logs -f -c ruby web-1

		# Begin streaming the logs from all containers in pods defined by label app=nginx
		kubectl logs -f -lapp=nginx --all-containers=true

		# Display only the most recent 20 lines of output in pod nginx
		kubectl logs --tail=20 nginx

		# Show all logs from pod nginx written in the last hour
		kubectl logs --since=1h nginx

		# Return snapshot logs from first container of a job named hello
		kubectl logs job/hello

		# Return snapshot logs from container nginx-1 of a deployment named nginx
		kubectl logs deployment/nginx -c nginx-1`))

	selectorTail    int64 = 10
	logsUsageErrStr       = fmt.Sprintf("expected '%s'.\nPOD or TYPE/NAME is a required argument for the logs command", logsUsageStr)
)

const (
	defaultPodLogsTimeout = 20 * time.Second
)

type LogsOptions struct {
	Namespace     string
	ResourceArg   string
	AllContainers bool
	Options       runtime.Object
	Resources     []string

	ConsumeRequestFn func(rest.ResponseWrapper, io.Writer) error

	// PodLogOptions
	SinceTime       string
	SinceSeconds    time.Duration
	Follow          bool
	Previous        bool
	Timestamps      bool
	IgnoreLogErrors bool
	LimitBytes      int64
	Tail            int64
	Container       string

	// whether or not a container name was given via --container
	ContainerNameSpecified bool
	Selector               string
	MaxFollowConcurency    int

	Object           runtime.Object
	GetPodTimeout    time.Duration
	RESTClientGetter genericclioptions.RESTClientGetter
	LogsForObject    polymorphichelpers.LogsForObjectFunc

	genericclioptions.IOStreams
}

func NewLogsOptions(streams genericclioptions.IOStreams, allContainers bool) *LogsOptions {
	return &LogsOptions{
		IOStreams:           streams,
		AllContainers:       allContainers,
		Tail:                -1,
		MaxFollowConcurency: 5,
	}
}

// NewCmdLogs creates a new pod logs command
func NewCmdLogs(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command {
	o := NewLogsOptions(streams, false)

	cmd := &cobra.Command{
		Use:                   logsUsageStr,
		DisableFlagsInUseLine: true,
		Short:                 i18n.T("Print the logs for a container in a pod"),
		Long:                  "Print the logs for a container in a pod or specified resource. If the pod has only one container, the container name is optional.",
		Example:               logsExample,
		PreRun: func(cmd *cobra.Command, args []string) {
			if len(os.Args) > 1 && os.Args[1] == "log" {
				fmt.Fprintf(o.ErrOut, "%s is DEPRECATED and will be removed in a future version. Use %s instead.\n", "log", "logs")
			}
		},
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(o.Complete(f, cmd, args))
			cmdutil.CheckErr(o.Validate())
			cmdutil.CheckErr(o.RunLogs())
		},
		Aliases: []string{"log"},
	}
	cmd.Flags().BoolVar(&o.AllContainers, "all-containers", o.AllContainers, "Get all containers' logs in the pod(s).")
	cmd.Flags().BoolVarP(&o.Follow, "follow", "f", o.Follow, "Specify if the logs should be streamed.")
	cmd.Flags().BoolVar(&o.Timestamps, "timestamps", o.Timestamps, "Include timestamps on each line in the log output")
	cmd.Flags().Int64Var(&o.LimitBytes, "limit-bytes", o.LimitBytes, "Maximum bytes of logs to return. Defaults to no limit.")
	cmd.Flags().BoolVarP(&o.Previous, "previous", "p", o.Previous, "If true, print the logs for the previous instance of the container in a pod if it exists.")
	cmd.Flags().Int64Var(&o.Tail, "tail", o.Tail, "Lines of recent log file to display. Defaults to -1 with no selector, showing all log lines otherwise 10, if a selector is provided.")
	cmd.Flags().BoolVar(&o.IgnoreLogErrors, "ignore-errors", o.IgnoreLogErrors, "If watching / following pod logs, allow for any errors that occur to be non-fatal")
	cmd.Flags().StringVar(&o.SinceTime, "since-time", o.SinceTime, i18n.T("Only return logs after a specific date (RFC3339). Defaults to all logs. Only one of since-time / since may be used."))
	cmd.Flags().DurationVar(&o.SinceSeconds, "since", o.SinceSeconds, "Only return logs newer than a relative duration like 5s, 2m, or 3h. Defaults to all logs. Only one of since-time / since may be used.")
	cmd.Flags().StringVarP(&o.Container, "container", "c", o.Container, "Print the logs of this container")
	cmdutil.AddPodRunningTimeoutFlag(cmd, defaultPodLogsTimeout)
	cmd.Flags().StringVarP(&o.Selector, "selector", "l", o.Selector, "Selector (label query) to filter on.")
	cmd.Flags().IntVar(&o.MaxFollowConcurency, "max-log-requests", o.MaxFollowConcurency, "Specify maximum number of concurrent logs to follow when using by a selector. Defaults to 5.")
	return cmd
}

func (o *LogsOptions) ToLogOptions() (*corev1.PodLogOptions, error) {
	logOptions := &corev1.PodLogOptions{
		Container:  o.Container,
		Follow:     o.Follow,
		Previous:   o.Previous,
		Timestamps: o.Timestamps,
	}

	if len(o.SinceTime) > 0 {
		t, err := util.ParseRFC3339(o.SinceTime, metav1.Now)
		if err != nil {
			return nil, err
		}

		logOptions.SinceTime = &t
	}

	if o.LimitBytes != 0 {
		logOptions.LimitBytes = &o.LimitBytes
	}

	if o.SinceSeconds != 0 {
		// round up to the nearest second
		sec := int64(o.SinceSeconds.Round(time.Second).Seconds())
		logOptions.SinceSeconds = &sec
	}

	if len(o.Selector) > 0 && o.Tail == -1 {
		logOptions.TailLines = &selectorTail
	} else if o.Tail != -1 {
		logOptions.TailLines = &o.Tail
	}

	return logOptions, nil
}

func (o *LogsOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error {
	o.ContainerNameSpecified = cmd.Flag("container").Changed
	o.Resources = args

	switch len(args) {
	case 0:
		if len(o.Selector) == 0 {
			return cmdutil.UsageErrorf(cmd, "%s", logsUsageErrStr)
		}
	case 1:
		o.ResourceArg = args[0]
		if len(o.Selector) != 0 {
			return cmdutil.UsageErrorf(cmd, "only a selector (-l) or a POD name is allowed")
		}
	case 2:
		o.ResourceArg = args[0]
		o.Container = args[1]
	default:
		return cmdutil.UsageErrorf(cmd, "%s", logsUsageErrStr)
	}
	var err error
	o.Namespace, _, err = f.ToRawKubeConfigLoader().Namespace()
	if err != nil {
		return err
	}

	o.ConsumeRequestFn = DefaultConsumeRequest

	o.GetPodTimeout, err = cmdutil.GetPodRunningTimeoutFlag(cmd)
	if err != nil {
		return err
	}

	o.Options, err = o.ToLogOptions()
	if err != nil {
		return err
	}

	o.RESTClientGetter = f
	o.LogsForObject = polymorphichelpers.LogsForObjectFn

	if o.Object == nil {
		builder := f.NewBuilder().
			WithScheme(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...).
			NamespaceParam(o.Namespace).DefaultNamespace().
			SingleResourceType()
		if o.ResourceArg != "" {
			builder.ResourceNames("pods", o.ResourceArg)
		}
		if o.Selector != "" {
			builder.ResourceTypes("pods").LabelSelectorParam(o.Selector)
		}
		infos, err := builder.Do().Infos()
		if err != nil {
			return err
		}
		if o.Selector == "" && len(infos) != 1 {
			return errors.New("expected a resource")
		}
		o.Object = infos[0].Object
	}

	return nil
}

func (o LogsOptions) Validate() error {
	if len(o.SinceTime) > 0 && o.SinceSeconds != 0 {
		return fmt.Errorf("at most one of `sinceTime` or `sinceSeconds` may be specified")
	}

	logsOptions, ok := o.Options.(*corev1.PodLogOptions)
	if !ok {
		return errors.New("unexpected logs options object")
	}
	if o.AllContainers && len(logsOptions.Container) > 0 {
		return fmt.Errorf("--all-containers=true should not be specified with container name %s", logsOptions.Container)
	}

	if o.ContainerNameSpecified && len(o.Resources) == 2 {
		return fmt.Errorf("only one of -c or an inline [CONTAINER] arg is allowed")
	}

	if o.LimitBytes < 0 {
		return fmt.Errorf("--limit-bytes must be greater than 0")
	}

	if logsOptions.SinceSeconds != nil && *logsOptions.SinceSeconds < int64(0) {
		return fmt.Errorf("--since must be greater than 0")
	}

	if logsOptions.TailLines != nil && *logsOptions.TailLines < 0 {
		return fmt.Errorf("TailLines must be greater than or equal to 0")
	}

	return nil
}

// RunLogs retrieves a pod log
func (o LogsOptions) RunLogs() error {
	requests, err := o.LogsForObject(o.RESTClientGetter, o.Object, o.Options, o.GetPodTimeout, o.AllContainers)
	if err != nil {
		return err
	}

	if o.Follow && len(requests) > 1 {
		if len(requests) > o.MaxFollowConcurency {
			return fmt.Errorf(
				"you are attempting to follow %d log streams, but maximum allowed concurency is %d, use --max-log-requests to increase the limit",
				len(requests), o.MaxFollowConcurency,
			)
		}

		return o.parallelConsumeRequest(requests)
	}

	return o.sequentialConsumeRequest(requests)
}

func (o LogsOptions) parallelConsumeRequest(requests []rest.ResponseWrapper) error {
	reader, writer := io.Pipe()
	wg := &sync.WaitGroup{}
	wg.Add(len(requests))
	for _, request := range requests {
		go func(request rest.ResponseWrapper) {
			if err := o.ConsumeRequestFn(request, writer); err != nil {
				if !o.IgnoreLogErrors {
					writer.CloseWithError(err)

					// It's important to return here to propagate the error via the pipe
					return
				}

				fmt.Fprintf(writer, "error: %v\n", err)
			}

			wg.Done()
		}(request)
	}

	go func() {
		wg.Wait()
		writer.Close()
	}()

	_, err := io.Copy(o.Out, reader)
	return err
}

func (o LogsOptions) sequentialConsumeRequest(requests []rest.ResponseWrapper) error {
	for _, request := range requests {
		if err := o.ConsumeRequestFn(request, o.Out); err != nil {
			return err
		}
	}

	return nil
}

// DefaultConsumeRequest reads the data from request and writes into
// the out writer. It buffers data from requests until the newline or io.EOF
// occurs in the data, so it doesn't interleave logs sub-line
// when running concurrently.
//
// A successful read returns err == nil, not err == io.EOF.
// Because the function is defined to read from request until io.EOF, it does
// not treat an io.EOF as an error to be reported.
func DefaultConsumeRequest(request rest.ResponseWrapper, out io.Writer) error {
	readCloser, err := request.Stream()
	if err != nil {
		return err
	}
	defer readCloser.Close()

	r := bufio.NewReader(readCloser)
	for {
		bytes, err := r.ReadBytes('\n')
		if _, err := out.Write(bytes); err != nil {
			return err
		}

		if err != nil {
			if err != io.EOF {
				return err
			}
			return nil
		}
	}
}
<commit_msg>added defer keyword to wg.Done() in logs.go<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logs

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"sync"
	"time"

	"github.com/spf13/cobra"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	"k8s.io/client-go/rest"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/kubectl/polymorphichelpers"
	"k8s.io/kubernetes/pkg/kubectl/scheme"
	"k8s.io/kubernetes/pkg/kubectl/util"
	"k8s.io/kubernetes/pkg/kubectl/util/i18n"
	"k8s.io/kubernetes/pkg/kubectl/util/templates"
)

const (
	logsUsageStr = "logs [-f] [-p] (POD | TYPE/NAME) [-c CONTAINER]"
)

var (
	logsExample = templates.Examples(i18n.T(`
		# Return snapshot logs from pod nginx with only one container
		kubectl logs nginx

		# Return snapshot logs from pod nginx with multi containers
		kubectl logs nginx --all-containers=true

		# Return snapshot logs from all containers in pods defined by label app=nginx
		kubectl logs -lapp=nginx --all-containers=true

		# Return snapshot of previous terminated ruby container logs from pod web-1
		kubectl logs -p -c ruby web-1

		# Begin streaming the logs of the ruby container in pod web-1
		kubectl logs -f -c ruby web-1

		# Begin streaming the logs from all containers in pods defined by label app=nginx
		kubectl logs -f -lapp=nginx --all-containers=true

		# Display only the most recent 20 lines of output in pod nginx
		kubectl logs --tail=20 nginx

		# Show all logs from pod nginx written in the last hour
		kubectl logs --since=1h nginx

		# Return snapshot logs from first container of a job named hello
		kubectl logs job/hello

		# Return snapshot logs from container nginx-1 of a deployment named nginx
		kubectl logs deployment/nginx -c nginx-1`))

	selectorTail    int64 = 10
	logsUsageErrStr       = fmt.Sprintf("expected '%s'.\nPOD or TYPE/NAME is a required argument for the logs command", logsUsageStr)
)

const (
	defaultPodLogsTimeout = 20 * time.Second
)

type LogsOptions struct {
	Namespace     string
	ResourceArg   string
	AllContainers bool
	Options       runtime.Object
	Resources     []string

	ConsumeRequestFn func(rest.ResponseWrapper, io.Writer) error

	// PodLogOptions
	SinceTime       string
	SinceSeconds    time.Duration
	Follow          bool
	Previous        bool
	Timestamps      bool
	IgnoreLogErrors bool
	LimitBytes      int64
	Tail            int64
	Container       string

	// whether or not a container name was given via --container
	ContainerNameSpecified bool
	Selector               string
	MaxFollowConcurency    int

	Object           runtime.Object
	GetPodTimeout    time.Duration
	RESTClientGetter genericclioptions.RESTClientGetter
	LogsForObject    polymorphichelpers.LogsForObjectFunc

	genericclioptions.IOStreams
}

func NewLogsOptions(streams genericclioptions.IOStreams, allContainers bool) *LogsOptions {
	return &LogsOptions{
		IOStreams:           streams,
		AllContainers:       allContainers,
		Tail:                -1,
		MaxFollowConcurency: 5,
	}
}

// NewCmdLogs creates a new pod logs command
func NewCmdLogs(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command {
	o := NewLogsOptions(streams, false)

	cmd := &cobra.Command{
		Use:                   logsUsageStr,
		DisableFlagsInUseLine: true,
		Short:                 i18n.T("Print the logs for a container in a pod"),
		Long:                  "Print the logs for a container in a pod or specified resource. If the pod has only one container, the container name is optional.",
		Example:               logsExample,
		PreRun: func(cmd *cobra.Command, args []string) {
			if len(os.Args) > 1 && os.Args[1] == "log" {
				fmt.Fprintf(o.ErrOut, "%s is DEPRECATED and will be removed in a future version. Use %s instead.\n", "log", "logs")
			}
		},
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(o.Complete(f, cmd, args))
			cmdutil.CheckErr(o.Validate())
			cmdutil.CheckErr(o.RunLogs())
		},
		Aliases: []string{"log"},
	}
	cmd.Flags().BoolVar(&o.AllContainers, "all-containers", o.AllContainers, "Get all containers' logs in the pod(s).")
	cmd.Flags().BoolVarP(&o.Follow, "follow", "f", o.Follow, "Specify if the logs should be streamed.")
	cmd.Flags().BoolVar(&o.Timestamps, "timestamps", o.Timestamps, "Include timestamps on each line in the log output")
	cmd.Flags().Int64Var(&o.LimitBytes, "limit-bytes", o.LimitBytes, "Maximum bytes of logs to return. Defaults to no limit.")
	cmd.Flags().BoolVarP(&o.Previous, "previous", "p", o.Previous, "If true, print the logs for the previous instance of the container in a pod if it exists.")
	cmd.Flags().Int64Var(&o.Tail, "tail", o.Tail, "Lines of recent log file to display. Defaults to -1 with no selector, showing all log lines otherwise 10, if a selector is provided.")
	cmd.Flags().BoolVar(&o.IgnoreLogErrors, "ignore-errors", o.IgnoreLogErrors, "If watching / following pod logs, allow for any errors that occur to be non-fatal")
	cmd.Flags().StringVar(&o.SinceTime, "since-time", o.SinceTime, i18n.T("Only return logs after a specific date (RFC3339). Defaults to all logs. Only one of since-time / since may be used."))
	cmd.Flags().DurationVar(&o.SinceSeconds, "since", o.SinceSeconds, "Only return logs newer than a relative duration like 5s, 2m, or 3h. Defaults to all logs. Only one of since-time / since may be used.")
	cmd.Flags().StringVarP(&o.Container, "container", "c", o.Container, "Print the logs of this container")
	cmdutil.AddPodRunningTimeoutFlag(cmd, defaultPodLogsTimeout)
	cmd.Flags().StringVarP(&o.Selector, "selector", "l", o.Selector, "Selector (label query) to filter on.")
	cmd.Flags().IntVar(&o.MaxFollowConcurency, "max-log-requests", o.MaxFollowConcurency, "Specify maximum number of concurrent logs to follow when using by a selector. Defaults to 5.")
	return cmd
}

func (o *LogsOptions) ToLogOptions() (*corev1.PodLogOptions, error) {
	logOptions := &corev1.PodLogOptions{
		Container:  o.Container,
		Follow:     o.Follow,
		Previous:   o.Previous,
		Timestamps: o.Timestamps,
	}

	if len(o.SinceTime) > 0 {
		t, err := util.ParseRFC3339(o.SinceTime, metav1.Now)
		if err != nil {
			return nil, err
		}

		logOptions.SinceTime = &t
	}

	if o.LimitBytes != 0 {
		logOptions.LimitBytes = &o.LimitBytes
	}

	if o.SinceSeconds != 0 {
		// round up to the nearest second
		sec := int64(o.SinceSeconds.Round(time.Second).Seconds())
		logOptions.SinceSeconds = &sec
	}

	if len(o.Selector) > 0 && o.Tail == -1 {
		logOptions.TailLines = &selectorTail
	} else if o.Tail != -1 {
		logOptions.TailLines = &o.Tail
	}

	return logOptions, nil
}

func (o *LogsOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error {
	o.ContainerNameSpecified = cmd.Flag("container").Changed
	o.Resources = args

	switch len(args) {
	case 0:
		if len(o.Selector) == 0 {
			return cmdutil.UsageErrorf(cmd, "%s", logsUsageErrStr)
		}
	case 1:
		o.ResourceArg = args[0]
		if len(o.Selector) != 0 {
			return cmdutil.UsageErrorf(cmd, "only a selector (-l) or a POD name is allowed")
		}
	case 2:
		o.ResourceArg = args[0]
		o.Container = args[1]
	default:
		return cmdutil.UsageErrorf(cmd, "%s", logsUsageErrStr)
	}
	var err error
	o.Namespace, _, err = f.ToRawKubeConfigLoader().Namespace()
	if err != nil {
		return err
	}

	o.ConsumeRequestFn = DefaultConsumeRequest

	o.GetPodTimeout, err = cmdutil.GetPodRunningTimeoutFlag(cmd)
	if err != nil {
		return err
	}

	o.Options, err = o.ToLogOptions()
	if err != nil {
		return err
	}

	o.RESTClientGetter = f
	o.LogsForObject = polymorphichelpers.LogsForObjectFn

	if o.Object == nil {
		builder := f.NewBuilder().
			WithScheme(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...).
			NamespaceParam(o.Namespace).DefaultNamespace().
			SingleResourceType()
		if o.ResourceArg != "" {
			builder.ResourceNames("pods", o.ResourceArg)
		}
		if o.Selector != "" {
			builder.ResourceTypes("pods").LabelSelectorParam(o.Selector)
		}
		infos, err := builder.Do().Infos()
		if err != nil {
			return err
		}
		if o.Selector == "" && len(infos) != 1 {
			return errors.New("expected a resource")
		}
		o.Object = infos[0].Object
	}

	return nil
}

func (o LogsOptions) Validate() error {
	if len(o.SinceTime) > 0 && o.SinceSeconds != 0 {
		return fmt.Errorf("at most one of `sinceTime` or `sinceSeconds` may be specified")
	}

	logsOptions, ok := o.Options.(*corev1.PodLogOptions)
	if !ok {
		return errors.New("unexpected logs options object")
	}
	if o.AllContainers && len(logsOptions.Container) > 0 {
		return fmt.Errorf("--all-containers=true should not be specified with container name %s", logsOptions.Container)
	}

	if o.ContainerNameSpecified && len(o.Resources) == 2 {
		return fmt.Errorf("only one of -c or an inline [CONTAINER] arg is allowed")
	}

	if o.LimitBytes < 0 {
		return fmt.Errorf("--limit-bytes must be greater than 0")
	}

	if logsOptions.SinceSeconds != nil && *logsOptions.SinceSeconds < int64(0) {
		return fmt.Errorf("--since must be greater than 0")
	}

	if logsOptions.TailLines != nil && *logsOptions.TailLines < 0 {
		return fmt.Errorf("TailLines must be greater than or equal to 0")
	}

	return nil
}

// RunLogs retrieves a pod log
func (o LogsOptions) RunLogs() error {
	requests, err := o.LogsForObject(o.RESTClientGetter, o.Object, o.Options, o.GetPodTimeout, o.AllContainers)
	if err != nil {
		return err
	}

	if o.Follow && len(requests) > 1 {
		if len(requests) > o.MaxFollowConcurency {
			return fmt.Errorf(
				"you are attempting to follow %d log streams, but maximum allowed concurency is %d, use --max-log-requests to increase the limit",
				len(requests), o.MaxFollowConcurency,
			)
		}

		return o.parallelConsumeRequest(requests)
	}

	return o.sequentialConsumeRequest(requests)
}

func (o LogsOptions) parallelConsumeRequest(requests []rest.ResponseWrapper) error {
	reader, writer := io.Pipe()
	wg := &sync.WaitGroup{}
	wg.Add(len(requests))
	for _, request := range requests {
		go func(request rest.ResponseWrapper) {
			defer wg.Done()
			if err := o.ConsumeRequestFn(request, writer); err != nil {
				if !o.IgnoreLogErrors {
					writer.CloseWithError(err)

					// It's important to return here to propagate the error via the pipe
					return
				}

				fmt.Fprintf(writer, "error: %v\n", err)
			}

		}(request)
	}

	go func() {
		wg.Wait()
		writer.Close()
	}()

	_, err := io.Copy(o.Out, reader)
	return err
}

func (o LogsOptions) sequentialConsumeRequest(requests []rest.ResponseWrapper) error {
	for _, request := range requests {
		if err := o.ConsumeRequestFn(request, o.Out); err != nil {
			return err
		}
	}

	return nil
}

// DefaultConsumeRequest reads the data from request and writes into
// the out writer. It buffers data from requests until the newline or io.EOF
// occurs in the data, so it doesn't interleave logs sub-line
// when running concurrently.
//
// A successful read returns err == nil, not err == io.EOF.
// Because the function is defined to read from request until io.EOF, it does
// not treat an io.EOF as an error to be reported.
func DefaultConsumeRequest(request rest.ResponseWrapper, out io.Writer) error {
	readCloser, err := request.Stream()
	if err != nil {
		return err
	}
	defer readCloser.Close()

	r := bufio.NewReader(readCloser)
	for {
		bytes, err := r.ReadBytes('\n')
		if _, err := out.Write(bytes); err != nil {
			return err
		}

		if err != nil {
			if err != io.EOF {
				return err
			}
			return nil
		}
	}
}
<|endoftext|>


<commit_before>package node

import (
	"fmt"
	"net"
	"sync"
	"syscall"

	"github.com/golang/glog"

	"k8s.io/apimachinery/pkg/util/sets"
	utilwait "k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/cache"

	networkapi "github.com/openshift/origin/pkg/network/apis/network"
	"github.com/openshift/origin/pkg/network/common"
	networkclient "github.com/openshift/origin/pkg/network/generated/internalclientset"

	"github.com/vishvananda/netlink"
)

type nodeEgress struct {
	nodeIP    string
	egressIPs sets.String
}

type namespaceEgress struct {
	vnid uint32

	// claimedIP is the egress IP it wants (NetNamespace.EgressIP[0]), or "" for none
	claimedIP string
	// assignedIP is an egress IP actually in use on nodeIP
	assignedIP string
	nodeIP     string
}

type egressIPWatcher struct {
	sync.Mutex

	localIP string
	oc      *ovsController

	networkClient networkclient.Interface
	iptables      *NodeIPTables

	// from HostSubnets
	nodesByNodeIP   map[string]*nodeEgress
	nodesByEgressIP map[string]*nodeEgress

	// From NetNamespaces
	namespacesByVNID     map[uint32]*namespaceEgress
	namespacesByEgressIP map[string]*namespaceEgress

	localEgressLink      netlink.Link
	localEgressIPMaskLen int

	testModeChan chan string
}

func newEgressIPWatcher(localIP string, oc *ovsController) *egressIPWatcher {
	return &egressIPWatcher{
		localIP: localIP,
		oc:      oc,

		nodesByNodeIP:   make(map[string]*nodeEgress),
		nodesByEgressIP: make(map[string]*nodeEgress),

		namespacesByVNID:     make(map[uint32]*namespaceEgress),
		namespacesByEgressIP: make(map[string]*namespaceEgress),
	}
}

func (eip *egressIPWatcher) Start(networkClient networkclient.Interface, iptables *NodeIPTables) error {
	if err := eip.findEgressLink(); err != nil {
		return fmt.Errorf("could not find egress network interface: %v", err)
	}

	eip.iptables = iptables
	eip.networkClient = networkClient

	go utilwait.Forever(eip.watchHostSubnets, 0)
	go utilwait.Forever(eip.watchNetNamespaces, 0)
	return nil
}

func (eip *egressIPWatcher) findEgressLink() error {
	links, err := netlink.LinkList()
	if err != nil {
		return err
	}
	for _, link := range links {
		addrs, err := netlink.AddrList(link, syscall.AF_INET)
		if err != nil {
			glog.Warningf("Could not get addresses of interface %q while trying to find egress interface: %v", link.Attrs().Name, err)
			continue
		}

		for _, addr := range addrs {
			if addr.IP.String() == eip.localIP {
				eip.localEgressLink = link
				eip.localEgressIPMaskLen, _ = addr.Mask.Size()
				return nil
			}
		}
	}

	return fmt.Errorf("could not find network interface with the address %q", eip.localIP)
}

func ipToHex(ip string) string {
	bytes := net.ParseIP(ip)
	if bytes == nil {
		return "invalid IP: shouldn't happen"
	}
	bytes = bytes.To4()
	return fmt.Sprintf("0x%02x%02x%02x%02x", bytes[0], bytes[1], bytes[2], bytes[3])
}

func (eip *egressIPWatcher) watchHostSubnets() {
	common.RunEventQueue(eip.networkClient.Network().RESTClient(), common.HostSubnets, func(delta cache.Delta) error {
		hs := delta.Object.(*networkapi.HostSubnet)

		var egressIPs []string
		if delta.Type != cache.Deleted {
			egressIPs = hs.EgressIPs
		}

		eip.updateNodeEgress(hs.HostIP, egressIPs)
		return nil
	})
}

func (eip *egressIPWatcher) updateNodeEgress(nodeIP string, nodeEgressIPs []string) {
	eip.Lock()
	defer eip.Unlock()

	node := eip.nodesByNodeIP[nodeIP]
	if node == nil {
		if len(nodeEgressIPs) == 0 {
			return
		}
		node = &nodeEgress{nodeIP: nodeIP, egressIPs: sets.NewString()}
		eip.nodesByNodeIP[nodeIP] = node
	} else if len(nodeEgressIPs) == 0 {
		delete(eip.nodesByNodeIP, nodeIP)
	}
	oldEgressIPs := node.egressIPs
	node.egressIPs = sets.NewString(nodeEgressIPs...)

	// Process new EgressIPs
	for _, ip := range node.egressIPs.Difference(oldEgressIPs).UnsortedList() {
		if oldNode := eip.nodesByEgressIP[ip]; oldNode != nil {
			glog.Errorf("Multiple nodes claiming EgressIP %q (nodes %q, %q)", ip, node.nodeIP, oldNode.nodeIP)
			continue
		}

		eip.nodesByEgressIP[ip] = node
		hex := ipToHex(ip)
		claimedNodeIP := nodeIP

		if nodeIP == eip.localIP {
			if err := eip.claimEgressIP(ip, hex); err != nil {
				glog.Errorf("Error claiming Egress IP %q: %v", ip, err)
				claimedNodeIP = ""
			}
		}

		if ns, exists := eip.namespacesByEgressIP[ip]; exists {
			if ns.assignedIP == "" {
				ns.assignedIP = ip
				ns.nodeIP = claimedNodeIP
				err := eip.oc.UpdateNamespaceEgressRules(ns.vnid, claimedNodeIP, hex)
				if err != nil {
					glog.Errorf("Error updating Namespace egress rules: %v", err)
				}
			}
		}
	}

	// Process removed EgressIPs
	for _, ip := range oldEgressIPs.Difference(node.egressIPs).UnsortedList() {
		delete(eip.nodesByEgressIP, ip)
		hex := ipToHex(ip)

		if nodeIP == eip.localIP {
			if err := eip.releaseEgressIP(ip, hex); err != nil {
				glog.Errorf("Error releasing Egress IP %q: %v", ip, err)
			}
		}

		if ns, exists := eip.namespacesByEgressIP[ip]; exists {
			if ns.assignedIP == ip {
				ns.assignedIP = ""
				ns.nodeIP = ""
				err := eip.oc.UpdateNamespaceEgressRules(ns.vnid, "", hex)
				if err != nil {
					glog.Errorf("Error updating Namespace egress rules: %v", err)
				}
			}
		}
	}
}

func (eip *egressIPWatcher) watchNetNamespaces() {
	common.RunEventQueue(eip.networkClient.Network().RESTClient(), common.NetNamespaces, func(delta cache.Delta) error {
		netns := delta.Object.(*networkapi.NetNamespace)

		if delta.Type != cache.Deleted && len(netns.EgressIPs) != 0 {
			if len(netns.EgressIPs) > 1 {
				glog.Warningf("Ignoring extra EgressIPs (%v) in NetNamespace %q", netns.EgressIPs[1:], netns.Name)
			}
			eip.updateNamespaceEgress(netns.NetID, netns.EgressIPs[0])
		} else {
			eip.deleteNamespaceEgress(netns.NetID)
		}
		return nil
	})
}

func (eip *egressIPWatcher) updateNamespaceEgress(vnid uint32, egressIP string) {
	eip.Lock()
	defer eip.Unlock()

	ns := eip.namespacesByVNID[vnid]
	if ns == nil {
		ns = &namespaceEgress{vnid: vnid}
		eip.namespacesByVNID[vnid] = ns
	}
	if ns.claimedIP == egressIP {
		return
	}
	if oldNS := eip.namespacesByEgressIP[egressIP]; oldNS != nil {
		glog.Errorf("Multiple NetNamespaces claiming EgressIP %q (NetIDs %d, %d)", egressIP, ns.vnid, oldNS.vnid)
		return
	}

	if ns.claimedIP != "" {
		delete(eip.namespacesByEgressIP, ns.claimedIP)
		ns.assignedIP = ""
		ns.nodeIP = ""
	}
	ns.claimedIP = egressIP
	eip.namespacesByEgressIP[egressIP] = ns
	if node := eip.nodesByEgressIP[egressIP]; node != nil {
		ns.assignedIP = egressIP
		ns.nodeIP = node.nodeIP
	}

	err := eip.oc.UpdateNamespaceEgressRules(ns.vnid, ns.nodeIP, ipToHex(egressIP))
	if err != nil {
		glog.Errorf("Error updating Namespace egress rules: %v", err)
	}
}

func (eip *egressIPWatcher) deleteNamespaceEgress(vnid uint32) {
	eip.Lock()
	defer eip.Unlock()

	ns := eip.namespacesByVNID[vnid]
	if ns == nil {
		return
	}

	if ns.claimedIP != "" {
		delete(eip.namespacesByEgressIP, ns.claimedIP)
	}
	delete(eip.namespacesByVNID, vnid)

	err := eip.oc.UpdateNamespaceEgressRules(ns.vnid, "", "")
	if err != nil {
		glog.Errorf("Error updating Namespace egress rules: %v", err)
	}
}

func (eip *egressIPWatcher) claimEgressIP(egressIP, egressHex string) error {
	if egressIP == eip.localIP {
		return fmt.Errorf("desired egress IP %q is the node IP", egressIP)
	}

	if eip.testModeChan != nil {
		eip.testModeChan <- fmt.Sprintf("claim %s", egressIP)
		return nil
	}

	egressIPNet := fmt.Sprintf("%s/%d", egressIP, eip.localEgressIPMaskLen)
	addr, err := netlink.ParseAddr(egressIPNet)
	if err != nil {
		return fmt.Errorf("could not parse egress IP %q: %v", egressIPNet, err)
	}
	err = netlink.AddrAdd(eip.localEgressLink, addr)
	if err != nil {
		return fmt.Errorf("could not add egress IP %q to %s: %v", egressIPNet, eip.localEgressLink.Attrs().Name, err)
	}

	if err := eip.iptables.AddEgressIPRules(egressIP, egressHex); err != nil {
		return fmt.Errorf("could not add egress IP iptables rule: %v", err)
	}

	return nil
}

func (eip *egressIPWatcher) releaseEgressIP(egressIP, egressHex string) error {
	if egressIP == eip.localIP {
		return nil
	}

	if eip.testModeChan != nil {
		eip.testModeChan <- fmt.Sprintf("release %s", egressIP)
		return nil
	}

	egressIPNet := fmt.Sprintf("%s/%d", egressIP, eip.localEgressIPMaskLen)
	addr, err := netlink.ParseAddr(egressIPNet)
	if err != nil {
		return fmt.Errorf("could not parse egress IP %q: %v", egressIPNet, err)
	}
	err = netlink.AddrDel(eip.localEgressLink, addr)
	if err != nil {
		if err == syscall.EADDRNOTAVAIL {
			glog.V(2).Infof("Could not delete egress IP %q from %s: no such address", egressIPNet, eip.localEgressLink.Attrs().Name)
		} else {
			return fmt.Errorf("could not delete egress IP %q from %s: %v", egressIPNet, eip.localEgressLink.Attrs().Name, err)
		}
	}

	if err := eip.iptables.DeleteEgressIPRules(egressIP, egressHex); err != nil {
		return fmt.Errorf("could not delete egress IP iptables rule: %v", err)
	}

	return nil
}
<commit_msg>Verify that EgressIPs are on the expected subnet<commit_after>package node

import (
	"fmt"
	"net"
	"sync"
	"syscall"

	"github.com/golang/glog"

	"k8s.io/apimachinery/pkg/util/sets"
	utilwait "k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/cache"

	networkapi "github.com/openshift/origin/pkg/network/apis/network"
	"github.com/openshift/origin/pkg/network/common"
	networkclient "github.com/openshift/origin/pkg/network/generated/internalclientset"

	"github.com/vishvananda/netlink"
)

type nodeEgress struct {
	nodeIP    string
	egressIPs sets.String
}

type namespaceEgress struct {
	vnid uint32

	// claimedIP is the egress IP it wants (NetNamespace.EgressIP[0]), or "" for none
	claimedIP string
	// assignedIP is an egress IP actually in use on nodeIP
	assignedIP string
	nodeIP     string
}

type egressIPWatcher struct {
	sync.Mutex

	localIP string
	oc      *ovsController

	networkClient networkclient.Interface
	iptables      *NodeIPTables

	// from HostSubnets
	nodesByNodeIP   map[string]*nodeEgress
	nodesByEgressIP map[string]*nodeEgress

	// From NetNamespaces
	namespacesByVNID     map[uint32]*namespaceEgress
	namespacesByEgressIP map[string]*namespaceEgress

	localEgressLink      netlink.Link
	localEgressNet       *net.IPNet
	localEgressIPMaskLen int

	testModeChan chan string
}

func newEgressIPWatcher(localIP string, oc *ovsController) *egressIPWatcher {
	return &egressIPWatcher{
		localIP: localIP,
		oc:      oc,

		nodesByNodeIP:   make(map[string]*nodeEgress),
		nodesByEgressIP: make(map[string]*nodeEgress),

		namespacesByVNID:     make(map[uint32]*namespaceEgress),
		namespacesByEgressIP: make(map[string]*namespaceEgress),
	}
}

func (eip *egressIPWatcher) Start(networkClient networkclient.Interface, iptables *NodeIPTables) error {
	if err := eip.findEgressLink(); err != nil {
		return fmt.Errorf("could not find egress network interface: %v", err)
	}

	eip.iptables = iptables
	eip.networkClient = networkClient

	go utilwait.Forever(eip.watchHostSubnets, 0)
	go utilwait.Forever(eip.watchNetNamespaces, 0)
	return nil
}

func (eip *egressIPWatcher) findEgressLink() error {
	links, err := netlink.LinkList()
	if err != nil {
		return err
	}
	for _, link := range links {
		addrs, err := netlink.AddrList(link, syscall.AF_INET)
		if err != nil {
			glog.Warningf("Could not get addresses of interface %q while trying to find egress interface: %v", link.Attrs().Name, err)
			continue
		}

		for _, addr := range addrs {
			if addr.IP.String() == eip.localIP {
				_, eip.localEgressNet, err = net.ParseCIDR(addr.IPNet.String())
				if err != nil {
					return fmt.Errorf("could not parse CIDR network from address %q: %v", addr.IP.String(), err)
				}
				eip.localEgressLink = link
				eip.localEgressIPMaskLen, _ = addr.Mask.Size()
				return nil
			}
		}
	}

	return fmt.Errorf("could not find network interface with the address %q", eip.localIP)
}

func ipToHex(ip string) string {
	bytes := net.ParseIP(ip)
	if bytes == nil {
		return "invalid IP: shouldn't happen"
	}
	bytes = bytes.To4()
	return fmt.Sprintf("0x%02x%02x%02x%02x", bytes[0], bytes[1], bytes[2], bytes[3])
}

func (eip *egressIPWatcher) watchHostSubnets() {
	common.RunEventQueue(eip.networkClient.Network().RESTClient(), common.HostSubnets, func(delta cache.Delta) error {
		hs := delta.Object.(*networkapi.HostSubnet)

		var egressIPs []string
		if delta.Type != cache.Deleted {
			egressIPs = hs.EgressIPs
		}

		eip.updateNodeEgress(hs.HostIP, egressIPs)
		return nil
	})
}

func (eip *egressIPWatcher) updateNodeEgress(nodeIP string, nodeEgressIPs []string) {
	eip.Lock()
	defer eip.Unlock()

	node := eip.nodesByNodeIP[nodeIP]
	if node == nil {
		if len(nodeEgressIPs) == 0 {
			return
		}
		node = &nodeEgress{nodeIP: nodeIP, egressIPs: sets.NewString()}
		eip.nodesByNodeIP[nodeIP] = node
	} else if len(nodeEgressIPs) == 0 {
		delete(eip.nodesByNodeIP, nodeIP)
	}
	oldEgressIPs := node.egressIPs
	node.egressIPs = sets.NewString(nodeEgressIPs...)

	// Process new EgressIPs
	for _, ip := range node.egressIPs.Difference(oldEgressIPs).UnsortedList() {
		if oldNode := eip.nodesByEgressIP[ip]; oldNode != nil {
			glog.Errorf("Multiple nodes claiming EgressIP %q (nodes %q, %q)", ip, node.nodeIP, oldNode.nodeIP)
			continue
		}

		eip.nodesByEgressIP[ip] = node
		hex := ipToHex(ip)
		claimedNodeIP := nodeIP

		if nodeIP == eip.localIP {
			if err := eip.claimEgressIP(ip, hex); err != nil {
				glog.Errorf("Error claiming Egress IP %q: %v", ip, err)
				claimedNodeIP = ""
			}
		}

		if ns, exists := eip.namespacesByEgressIP[ip]; exists {
			if ns.assignedIP == "" {
				ns.assignedIP = ip
				ns.nodeIP = claimedNodeIP
				err := eip.oc.UpdateNamespaceEgressRules(ns.vnid, claimedNodeIP, hex)
				if err != nil {
					glog.Errorf("Error updating Namespace egress rules: %v", err)
				}
			}
		}
	}

	// Process removed EgressIPs
	for _, ip := range oldEgressIPs.Difference(node.egressIPs).UnsortedList() {
		delete(eip.nodesByEgressIP, ip)
		hex := ipToHex(ip)

		if nodeIP == eip.localIP {
			if err := eip.releaseEgressIP(ip, hex); err != nil {
				glog.Errorf("Error releasing Egress IP %q: %v", ip, err)
			}
		}

		if ns, exists := eip.namespacesByEgressIP[ip]; exists {
			if ns.assignedIP == ip {
				ns.assignedIP = ""
				ns.nodeIP = ""
				err := eip.oc.UpdateNamespaceEgressRules(ns.vnid, "", hex)
				if err != nil {
					glog.Errorf("Error updating Namespace egress rules: %v", err)
				}
			}
		}
	}
}

func (eip *egressIPWatcher) watchNetNamespaces() {
	common.RunEventQueue(eip.networkClient.Network().RESTClient(), common.NetNamespaces, func(delta cache.Delta) error {
		netns := delta.Object.(*networkapi.NetNamespace)

		if delta.Type != cache.Deleted && len(netns.EgressIPs) != 0 {
			if len(netns.EgressIPs) > 1 {
				glog.Warningf("Ignoring extra EgressIPs (%v) in NetNamespace %q", netns.EgressIPs[1:], netns.Name)
			}
			eip.updateNamespaceEgress(netns.NetID, netns.EgressIPs[0])
		} else {
			eip.deleteNamespaceEgress(netns.NetID)
		}
		return nil
	})
}

func (eip *egressIPWatcher) updateNamespaceEgress(vnid uint32, egressIP string) {
	eip.Lock()
	defer eip.Unlock()

	ns := eip.namespacesByVNID[vnid]
	if ns == nil {
		ns = &namespaceEgress{vnid: vnid}
		eip.namespacesByVNID[vnid] = ns
	}
	if ns.claimedIP == egressIP {
		return
	}
	if oldNS := eip.namespacesByEgressIP[egressIP]; oldNS != nil {
		glog.Errorf("Multiple NetNamespaces claiming EgressIP %q (NetIDs %d, %d)", egressIP, ns.vnid, oldNS.vnid)
		return
	}

	if ns.claimedIP != "" {
		delete(eip.namespacesByEgressIP, ns.claimedIP)
		ns.assignedIP = ""
		ns.nodeIP = ""
	}
	ns.claimedIP = egressIP
	eip.namespacesByEgressIP[egressIP] = ns
	if node := eip.nodesByEgressIP[egressIP]; node != nil {
		ns.assignedIP = egressIP
		ns.nodeIP = node.nodeIP
	}

	err := eip.oc.UpdateNamespaceEgressRules(ns.vnid, ns.nodeIP, ipToHex(egressIP))
	if err != nil {
		glog.Errorf("Error updating Namespace egress rules: %v", err)
	}
}

func (eip *egressIPWatcher) deleteNamespaceEgress(vnid uint32) {
	eip.Lock()
	defer eip.Unlock()

	ns := eip.namespacesByVNID[vnid]
	if ns == nil {
		return
	}

	if ns.claimedIP != "" {
		delete(eip.namespacesByEgressIP, ns.claimedIP)
	}
	delete(eip.namespacesByVNID, vnid)

	err := eip.oc.UpdateNamespaceEgressRules(ns.vnid, "", "")
	if err != nil {
		glog.Errorf("Error updating Namespace egress rules: %v", err)
	}
}

func (eip *egressIPWatcher) claimEgressIP(egressIP, egressHex string) error {
	if egressIP == eip.localIP {
		return fmt.Errorf("desired egress IP %q is the node IP", egressIP)
	}

	if eip.testModeChan != nil {
		eip.testModeChan <- fmt.Sprintf("claim %s", egressIP)
		return nil
	}

	egressIPNet := fmt.Sprintf("%s/%d", egressIP, eip.localEgressIPMaskLen)
	addr, err := netlink.ParseAddr(egressIPNet)
	if err != nil {
		return fmt.Errorf("could not parse egress IP %q: %v", egressIPNet, err)
	}
	if !eip.localEgressNet.Contains(addr.IP) {
		return fmt.Errorf("egress IP %q is not in local network %s of interface %s", egressIP, eip.localEgressNet.String(), eip.localEgressLink.Attrs().Name)
	}
	err = netlink.AddrAdd(eip.localEgressLink, addr)
	if err != nil {
		return fmt.Errorf("could not add egress IP %q to %s: %v", egressIPNet, eip.localEgressLink.Attrs().Name, err)
	}

	if err := eip.iptables.AddEgressIPRules(egressIP, egressHex); err != nil {
		return fmt.Errorf("could not add egress IP iptables rule: %v", err)
	}

	return nil
}

func (eip *egressIPWatcher) releaseEgressIP(egressIP, egressHex string) error {
	if egressIP == eip.localIP {
		return nil
	}

	if eip.testModeChan != nil {
		eip.testModeChan <- fmt.Sprintf("release %s", egressIP)
		return nil
	}

	egressIPNet := fmt.Sprintf("%s/%d", egressIP, eip.localEgressIPMaskLen)
	addr, err := netlink.ParseAddr(egressIPNet)
	if err != nil {
		return fmt.Errorf("could not parse egress IP %q: %v", egressIPNet, err)
	}
	err = netlink.AddrDel(eip.localEgressLink, addr)
	if err != nil {
		if err == syscall.EADDRNOTAVAIL {
			glog.V(2).Infof("Could not delete egress IP %q from %s: no such address", egressIPNet, eip.localEgressLink.Attrs().Name)
		} else {
			return fmt.Errorf("could not delete egress IP %q from %s: %v", egressIPNet, eip.localEgressLink.Attrs().Name, err)
		}
	}

	if err := eip.iptables.DeleteEgressIPRules(egressIP, egressHex); err != nil {
		return fmt.Errorf("could not delete egress IP iptables rule: %v", err)
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"encoding/json"
	"fmt"
	"io"
	"path"
	"path/filepath"
	"strings"

	"github.com/containers/image/types"

	"github.com/containers/image/manifest"

	"github.com/containers/image/docker"

	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/util"
	"github.com/moby/moby/builder/dockerfile/parser"
	"github.com/moby/moby/builder/dockerfile/shell"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	add  = "add"
	copy = "copy"
	env  = "env"
	from = "from"
)

// For testing.
var RetrieveConfig = retrieveImageConfig

// GetDockerfileDependencies parses a dockerfile and returns the full paths
// of all the source files that the resulting docker image depends on.
func GetDockerfileDependencies(workspace string, r io.Reader) ([]string, error) {
	res, err := parser.Parse(r)
	if err != nil {
		return nil, errors.Wrap(err, "parsing dockerfile")
	}
	envs := map[string]string{}
	depMap := map[string]struct{}{}
	// First process onbuilds, if present.
	onbuilds := []string{}
	for _, value := range res.AST.Children {
		switch value.Value {
		case from:
			onbuilds, err = processBaseImage(value)
			if err != nil {
				logrus.Warnf("Error processing base image for onbuild triggers: %s. Dependencies may be incomplete.", err)
			}
		}
	}

	var dispatchInstructions = func(r *parser.Result) {
		for _, value := range r.AST.Children {
			switch value.Value {
			case add, copy:
				processCopy(workspace, value, depMap, envs)
			case env:
				envs[value.Next.Value] = value.Next.Next.Value
			}
		}
	}
	for _, ob := range onbuilds {
		obRes, err := parser.Parse(strings.NewReader(ob))
		if err != nil {
			return nil, err
		}
		dispatchInstructions(obRes)
	}

	dispatchInstructions(res)

	deps := []string{}
	for dep := range depMap {
		deps = append(deps, dep)
	}
	logrus.Infof("Found dependencies for dockerfile %s", deps)
	expandedDeps, err := util.ExpandPaths(workspace, deps)
	if err != nil {
		return nil, errors.Wrap(err, "expanding dockerfile paths")
	}

	// Look for .dockerignore.
	ignorePath := filepath.Join(workspace, ".dockerignore")

	filteredDeps, err := util.ApplyDockerIgnore(expandedDeps, ignorePath)
	if err != nil {
		return nil, errors.Wrap(err, "applying dockerignore")
	}
	return filteredDeps, nil
}

func processBaseImage(value *parser.Node) ([]string, error) {
	base := value.Next.Value
	logrus.Debugf("Checking base image %s for ONBUILD triggers.", base)
	if strings.ToLower(base) == "scratch" {
		logrus.Debugf("SCRATCH base image found, skipping check: %s", base)
		return nil, nil
	}
	cfg, err := RetrieveConfig(base)
	if err != nil {
		return nil, err
	}
	logrus.Debugf("Found onbuild triggers %v in image %s", cfg.Config.OnBuild, base)
	return cfg.Config.OnBuild, nil
}

func retrieveImageConfig(image string) (*manifest.Schema2Image, error) {
	ref, err := docker.ParseReference("//" + image)
	if err != nil {
		return nil, err
	}

	// Hardcode the OS and Arch in case the image returns a manifest list.
	context := &types.SystemContext{
		OSChoice:           "linux",
		ArchitectureChoice: "amd64",
	}
	img, err := ref.NewImage(context)
	if err != nil {
		return nil, err
	}

	cfgBytes, err := img.ConfigBlob()
	if err != nil {
		return nil, err
	}
	cfg := &manifest.Schema2Image{}
	if err := json.Unmarshal(cfgBytes, cfg); err != nil {
		return nil, err
	}
	return cfg, nil
}

func processCopy(workspace string, value *parser.Node, paths map[string]struct{}, envs map[string]string) error {
	slex := shell.NewLex('\\')
	for {
		// Skip last node, since it is the destination, and stop if we arrive at a comment
		if value.Next.Next == nil || strings.HasPrefix(value.Next.Next.Value, "#") {
			break
		}
		src, err := processShellWord(slex, value.Next.Value, envs)
		if err != nil {
			return errors.Wrap(err, "processing word")
		}
		// If the --from flag is provided, we are dealing with a multi-stage dockerfile
		// Adding a dependency from a different stage does not imply a source dependency
		if hasMultiStageFlag(value.Flags) {
			return nil
		}
		dep := path.Join(workspace, src)
		paths[dep] = struct{}{}
		value = value.Next
	}
	return nil
}

func processShellWord(lex *shell.Lex, word string, envs map[string]string) (string, error) {
	envSlice := []string{}
	for envKey, envVal := range envs {
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", envKey, envVal))
	}
	return lex.ProcessWord(word, envSlice)
}

func hasMultiStageFlag(flags []string) bool {
	for _, f := range flags {
		if strings.HasPrefix(f, "--from=") {
			return true
		}
	}
	return false
}
<commit_msg>The Image has to be closed (#142)<commit_after>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"encoding/json"
	"fmt"
	"io"
	"path"
	"path/filepath"
	"strings"

	"github.com/containers/image/types"

	"github.com/containers/image/manifest"

	"github.com/containers/image/docker"

	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/util"
	"github.com/moby/moby/builder/dockerfile/parser"
	"github.com/moby/moby/builder/dockerfile/shell"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	add  = "add"
	copy = "copy"
	env  = "env"
	from = "from"
)

// For testing.
var RetrieveConfig = retrieveImageConfig

// GetDockerfileDependencies parses a dockerfile and returns the full paths
// of all the source files that the resulting docker image depends on.
func GetDockerfileDependencies(workspace string, r io.Reader) ([]string, error) {
	res, err := parser.Parse(r)
	if err != nil {
		return nil, errors.Wrap(err, "parsing dockerfile")
	}
	envs := map[string]string{}
	depMap := map[string]struct{}{}
	// First process onbuilds, if present.
	onbuilds := []string{}
	for _, value := range res.AST.Children {
		switch value.Value {
		case from:
			onbuilds, err = processBaseImage(value)
			if err != nil {
				logrus.Warnf("Error processing base image for onbuild triggers: %s. Dependencies may be incomplete.", err)
			}
		}
	}

	var dispatchInstructions = func(r *parser.Result) {
		for _, value := range r.AST.Children {
			switch value.Value {
			case add, copy:
				processCopy(workspace, value, depMap, envs)
			case env:
				envs[value.Next.Value] = value.Next.Next.Value
			}
		}
	}
	for _, ob := range onbuilds {
		obRes, err := parser.Parse(strings.NewReader(ob))
		if err != nil {
			return nil, err
		}
		dispatchInstructions(obRes)
	}

	dispatchInstructions(res)

	deps := []string{}
	for dep := range depMap {
		deps = append(deps, dep)
	}
	logrus.Infof("Found dependencies for dockerfile %s", deps)
	expandedDeps, err := util.ExpandPaths(workspace, deps)
	if err != nil {
		return nil, errors.Wrap(err, "expanding dockerfile paths")
	}

	// Look for .dockerignore.
	ignorePath := filepath.Join(workspace, ".dockerignore")

	filteredDeps, err := util.ApplyDockerIgnore(expandedDeps, ignorePath)
	if err != nil {
		return nil, errors.Wrap(err, "applying dockerignore")
	}
	return filteredDeps, nil
}

func processBaseImage(value *parser.Node) ([]string, error) {
	base := value.Next.Value
	logrus.Debugf("Checking base image %s for ONBUILD triggers.", base)
	if strings.ToLower(base) == "scratch" {
		logrus.Debugf("SCRATCH base image found, skipping check: %s", base)
		return nil, nil
	}
	cfg, err := RetrieveConfig(base)
	if err != nil {
		return nil, err
	}
	logrus.Debugf("Found onbuild triggers %v in image %s", cfg.Config.OnBuild, base)
	return cfg.Config.OnBuild, nil
}

func retrieveImageConfig(image string) (*manifest.Schema2Image, error) {
	ref, err := docker.ParseReference("//" + image)
	if err != nil {
		return nil, err
	}

	// Hardcode the OS and Arch in case the image returns a manifest list.
	context := &types.SystemContext{
		OSChoice:           "linux",
		ArchitectureChoice: "amd64",
	}
	img, err := ref.NewImage(context)
	if err != nil {
		return nil, err
	}
	defer img.Close()

	cfgBytes, err := img.ConfigBlob()
	if err != nil {
		return nil, err
	}
	cfg := &manifest.Schema2Image{}
	if err := json.Unmarshal(cfgBytes, cfg); err != nil {
		return nil, err
	}
	return cfg, nil
}

func processCopy(workspace string, value *parser.Node, paths map[string]struct{}, envs map[string]string) error {
	slex := shell.NewLex('\\')
	for {
		// Skip last node, since it is the destination, and stop if we arrive at a comment
		if value.Next.Next == nil || strings.HasPrefix(value.Next.Next.Value, "#") {
			break
		}
		src, err := processShellWord(slex, value.Next.Value, envs)
		if err != nil {
			return errors.Wrap(err, "processing word")
		}
		// If the --from flag is provided, we are dealing with a multi-stage dockerfile
		// Adding a dependency from a different stage does not imply a source dependency
		if hasMultiStageFlag(value.Flags) {
			return nil
		}
		dep := path.Join(workspace, src)
		paths[dep] = struct{}{}
		value = value.Next
	}
	return nil
}

func processShellWord(lex *shell.Lex, word string, envs map[string]string) (string, error) {
	envSlice := []string{}
	for envKey, envVal := range envs {
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", envKey, envVal))
	}
	return lex.ProcessWord(word, envSlice)
}

func hasMultiStageFlag(flags []string) bool {
	for _, f := range flags {
		if strings.HasPrefix(f, "--from=") {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package webhook

import (
	"context"
	"reflect"
	"testing"

	configv1alpha1 "github.com/open-policy-agent/gatekeeper/apis/config/v1alpha1"
	mutationsunversioned "github.com/open-policy-agent/gatekeeper/apis/mutations/unversioned"
	"github.com/open-policy-agent/gatekeeper/pkg/controller/config/process"
	"github.com/open-policy-agent/gatekeeper/pkg/mutation"
	"github.com/open-policy-agent/gatekeeper/pkg/mutation/match"
	"github.com/open-policy-agent/gatekeeper/pkg/mutation/mutators"
	"github.com/open-policy-agent/gatekeeper/pkg/mutation/mutators/assign"
	"github.com/open-policy-agent/gatekeeper/pkg/mutation/mutators/assignmeta"
	"github.com/open-policy-agent/gatekeeper/pkg/mutation/types"
	admissionv1 "k8s.io/api/admission/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
	atypes "sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)

func makeValue(v interface{}) mutationsunversioned.AssignField {
	return mutationsunversioned.AssignField{Value: &types.Anything{Value: v}}
}

func TestWebhookAssign(t *testing.T) {
	sys := mutation.NewSystem(mutation.SystemOpts{})

	v := &mutationsunversioned.Assign{
		ObjectMeta: metav1.ObjectMeta{Name: "AddFoo"},
		Spec: mutationsunversioned.AssignSpec{
			ApplyTo:  []match.ApplyTo{{Groups: []string{""}, Versions: []string{"v1"}, Kinds: []string{"Pod"}}},
			Location: "spec.value",
			Parameters: mutationsunversioned.Parameters{
				Assign: makeValue("foo"),
			},
		},
	}
	if err := assign.IsValidAssign(v); err != nil {
		t.Fatal(err)
	}

	m, err := mutators.MutatorForAssign(v)
	if err != nil {
		t.Fatal(err)
	}
	if err := sys.Upsert(m); err != nil {
		t.Fatal(err)
	}

	h := &mutationHandler{
		webhookHandler: webhookHandler{
			injectedConfig: &configv1alpha1.Config{
				Spec: configv1alpha1.ConfigSpec{
					Validation: configv1alpha1.Validation{
						Traces: []configv1alpha1.Trace{},
					},
				},
			},
			client:          &nsGetter{},
			reader:          &nsGetter{},
			processExcluder: process.New(),
		},
		mutationSystem: sys,
		deserializer:   codecs.UniversalDeserializer(),
	}

	raw := []byte(`{"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "acbd","namespace": "ns1"}}`)

	req := atypes.Request{
		AdmissionRequest: admissionv1.AdmissionRequest{
			Kind: metav1.GroupVersionKind{
				Group:   "",
				Version: "v1",
				Kind:    "Pod",
			},
			Object:    runtime.RawExtension{Raw: raw},
			Namespace: "ns1",
			Operation: admissionv1.Create,
		},
	}

	resp := h.Handle(context.Background(), req)

	expectedVal := []byte(`{"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "acbd","namespace": "ns1"}, "spec": {"value": "foo"}}`)
	expected := admission.PatchResponseFromRaw(raw, expectedVal)

	if !reflect.DeepEqual(resp, expected) {
		t.Errorf("unexpected response: %+v\n\nexpected: %+v", resp, expected)
	}
}

func TestWebhookAssignMetadata(t *testing.T) {
	sys := mutation.NewSystem(mutation.SystemOpts{})

	v := &mutationsunversioned.AssignMetadata{
		ObjectMeta: metav1.ObjectMeta{Name: "AddFoo"},
		Spec: mutationsunversioned.AssignMetadataSpec{
			Location: "metadata.labels.foo",
			Parameters: mutationsunversioned.MetadataParameters{
				Assign: makeValue("bar"),
			},
		},
	}
	if err := assignmeta.IsValidAssignMetadata(v); err != nil {
		t.Fatal(err)
	}

	m, err := mutators.MutatorForAssignMetadata(v)
	if err != nil {
		t.Fatal(err)
	}
	if err := sys.Upsert(m); err != nil {
		t.Fatal(err)
	}

	h := &mutationHandler{
		webhookHandler: webhookHandler{
			injectedConfig: &configv1alpha1.Config{
				Spec: configv1alpha1.ConfigSpec{
					Validation: configv1alpha1.Validation{
						Traces: []configv1alpha1.Trace{},
					},
				},
			},
			client:          &nsGetter{},
			reader:          &nsGetter{},
			processExcluder: process.New(),
		},
		mutationSystem: sys,
		deserializer:   codecs.UniversalDeserializer(),
	}

	raw := []byte(`{"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "acbd","namespace": "ns1"}}`)

	req := atypes.Request{
		AdmissionRequest: admissionv1.AdmissionRequest{
			Kind: metav1.GroupVersionKind{
				Group:   "",
				Version: "v1",
				Kind:    "Pod",
			},
			Object:    runtime.RawExtension{Raw: raw},
			Namespace: "ns1",
			Operation: admissionv1.Create,
		},
	}

	resp := h.Handle(context.Background(), req)

	expectedVal := []byte(`{"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "acbd", "namespace": "ns1", "labels": {"foo":"bar"}}}`)
	expected := admission.PatchResponseFromRaw(raw, expectedVal)

	if !reflect.DeepEqual(resp, expected) {
		t.Errorf("unexpected response: %+v\n\nexpected: %+v", resp, expected)
	}
}
<commit_msg>Remove duplicate import (#1694)<commit_after>package webhook

import (
	"context"
	"reflect"
	"testing"

	configv1alpha1 "github.com/open-policy-agent/gatekeeper/apis/config/v1alpha1"
	mutationsunversioned "github.com/open-policy-agent/gatekeeper/apis/mutations/unversioned"
	"github.com/open-policy-agent/gatekeeper/pkg/controller/config/process"
	"github.com/open-policy-agent/gatekeeper/pkg/mutation"
	"github.com/open-policy-agent/gatekeeper/pkg/mutation/match"
	"github.com/open-policy-agent/gatekeeper/pkg/mutation/mutators"
	"github.com/open-policy-agent/gatekeeper/pkg/mutation/mutators/assign"
	"github.com/open-policy-agent/gatekeeper/pkg/mutation/mutators/assignmeta"
	"github.com/open-policy-agent/gatekeeper/pkg/mutation/types"
	admissionv1 "k8s.io/api/admission/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)

func makeValue(v interface{}) mutationsunversioned.AssignField {
	return mutationsunversioned.AssignField{Value: &types.Anything{Value: v}}
}

func TestWebhookAssign(t *testing.T) {
	sys := mutation.NewSystem(mutation.SystemOpts{})

	v := &mutationsunversioned.Assign{
		ObjectMeta: metav1.ObjectMeta{Name: "AddFoo"},
		Spec: mutationsunversioned.AssignSpec{
			ApplyTo:  []match.ApplyTo{{Groups: []string{""}, Versions: []string{"v1"}, Kinds: []string{"Pod"}}},
			Location: "spec.value",
			Parameters: mutationsunversioned.Parameters{
				Assign: makeValue("foo"),
			},
		},
	}
	if err := assign.IsValidAssign(v); err != nil {
		t.Fatal(err)
	}

	m, err := mutators.MutatorForAssign(v)
	if err != nil {
		t.Fatal(err)
	}
	if err := sys.Upsert(m); err != nil {
		t.Fatal(err)
	}

	h := &mutationHandler{
		webhookHandler: webhookHandler{
			injectedConfig: &configv1alpha1.Config{
				Spec: configv1alpha1.ConfigSpec{
					Validation: configv1alpha1.Validation{
						Traces: []configv1alpha1.Trace{},
					},
				},
			},
			client:          &nsGetter{},
			reader:          &nsGetter{},
			processExcluder: process.New(),
		},
		mutationSystem: sys,
		deserializer:   codecs.UniversalDeserializer(),
	}

	raw := []byte(`{"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "acbd","namespace": "ns1"}}`)

	req := admission.Request{
		AdmissionRequest: admissionv1.AdmissionRequest{
			Kind: metav1.GroupVersionKind{
				Group:   "",
				Version: "v1",
				Kind:    "Pod",
			},
			Object:    runtime.RawExtension{Raw: raw},
			Namespace: "ns1",
			Operation: admissionv1.Create,
		},
	}

	resp := h.Handle(context.Background(), req)

	expectedVal := []byte(`{"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "acbd","namespace": "ns1"}, "spec": {"value": "foo"}}`)
	expected := admission.PatchResponseFromRaw(raw, expectedVal)

	if !reflect.DeepEqual(resp, expected) {
		t.Errorf("unexpected response: %+v\n\nexpected: %+v", resp, expected)
	}
}

func TestWebhookAssignMetadata(t *testing.T) {
	sys := mutation.NewSystem(mutation.SystemOpts{})

	v := &mutationsunversioned.AssignMetadata{
		ObjectMeta: metav1.ObjectMeta{Name: "AddFoo"},
		Spec: mutationsunversioned.AssignMetadataSpec{
			Location: "metadata.labels.foo",
			Parameters: mutationsunversioned.MetadataParameters{
				Assign: makeValue("bar"),
			},
		},
	}
	if err := assignmeta.IsValidAssignMetadata(v); err != nil {
		t.Fatal(err)
	}

	m, err := mutators.MutatorForAssignMetadata(v)
	if err != nil {
		t.Fatal(err)
	}
	if err := sys.Upsert(m); err != nil {
		t.Fatal(err)
	}

	h := &mutationHandler{
		webhookHandler: webhookHandler{
			injectedConfig: &configv1alpha1.Config{
				Spec: configv1alpha1.ConfigSpec{
					Validation: configv1alpha1.Validation{
						Traces: []configv1alpha1.Trace{},
					},
				},
			},
			client:          &nsGetter{},
			reader:          &nsGetter{},
			processExcluder: process.New(),
		},
		mutationSystem: sys,
		deserializer:   codecs.UniversalDeserializer(),
	}

	raw := []byte(`{"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "acbd","namespace": "ns1"}}`)

	req := admission.Request{
		AdmissionRequest: admissionv1.AdmissionRequest{
			Kind: metav1.GroupVersionKind{
				Group:   "",
				Version: "v1",
				Kind:    "Pod",
			},
			Object:    runtime.RawExtension{Raw: raw},
			Namespace: "ns1",
			Operation: admissionv1.Create,
		},
	}

	resp := h.Handle(context.Background(), req)

	expectedVal := []byte(`{"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "acbd", "namespace": "ns1", "labels": {"foo":"bar"}}}`)
	expected := admission.PatchResponseFromRaw(raw, expectedVal)

	if !reflect.DeepEqual(resp, expected) {
		t.Errorf("unexpected response: %+v\n\nexpected: %+v", resp, expected)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/event"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/order"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/wallet"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/mux"
)

func main() {
	m := mux.New().
		TransformRaw().
		WithAPIKEY("YOUR_API_KEY").
		WithAPISEC("YOUR_API_SECRET").
		Start()

	crash := make(chan error)

	go func() {
		crash <- m.Listen(func(msg interface{}, err error) {
			if err != nil {
				log.Printf("error received: %s\n", err)
			}

			switch v := msg.(type) {
			case event.Info:
				log.Printf("%T: %+v\n", v, v)
			case *order.New:
				log.Printf("%T: %+v\n", v, v)
			case *order.Snapshot:
				log.Printf("%T: %+v\n", v, v)
				for _, ss := range v.Snapshot {
					log.Printf("%T item: %+v\n", ss, ss)
				}
			case *order.Update:
				log.Printf("%T: %+v\n", v, v)
			case *order.Cancel:
				log.Printf("%T: %+v\n", v, v)
			case *wallet.Update:
				log.Printf("%T: %+v\n", v, v)
			case *wallet.Snapshot:
				log.Printf("%T: %+v\n", v, v)
				for _, ss := range v.Snapshot {
					log.Printf("%T item: %+v\n", ss, ss)
				}
			default:
				log.Printf("raw/unhandled: %T: %+v\n", v, v)
			}
		})
	}()

	log.Fatal(<-crash)
}
<commit_msg>updating authenticated feed ingest example<commit_after>package main

import (
	"log"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/balanceinfo"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/event"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/order"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/wallet"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/mux"
)

func main() {
	m := mux.New().
		TransformRaw().
		WithAPIKEY("YOUR_API_KEY").
		WithAPISEC("YOUR_API_SECRET").
		Start()

	crash := make(chan error)

	go func() {
		crash <- m.Listen(func(msg interface{}, err error) {
			if err != nil {
				log.Printf("error received: %s\n", err)
			}

			switch v := msg.(type) {
			case event.Info:
				log.Printf("%T: %+v\n", v, v)
			case *order.New:
				log.Printf("%T: %+v\n", v, v)
			case *order.Snapshot:
				log.Printf("%T: %+v\n", v, v)
				for _, ss := range v.Snapshot {
					log.Printf("%T item: %+v\n", ss, ss)
				}
			case *order.Update:
				log.Printf("%T: %+v\n", v, v)
			case *order.Cancel:
				log.Printf("%T: %+v\n", v, v)
			case *wallet.Update:
				log.Printf("%T: %+v\n", v, v)
			case *wallet.Snapshot:
				log.Printf("%T: %+v\n", v, v)
				for _, ss := range v.Snapshot {
					log.Printf("%T item: %+v\n", ss, ss)
				}
			case balanceinfo.Update:
				log.Printf("%T: %+v\n", v, v)
			default:
				log.Printf("raw/unhandled: %T: %+v\n", v, v)
			}
		})
	}()

	log.Fatal(<-crash)
}
<|endoftext|>


<commit_before>package gw

import (
	"time"

	"github.com/brocaar/lorawan"
	"github.com/brocaar/lorawan/band"
)

// RXPacket contains the PHYPayload received from the gateway.
type RXPacket struct {
	RXInfo     RXInfo             `json:"rxInfo"`
	PHYPayload lorawan.PHYPayload `json:"phyPayload"`
}

// RXPacketBytes contains the PHYPayload as []byte received from the gateway.
// The JSON output is compatible with RXPacket.
type RXPacketBytes struct {
	RXInfo     RXInfo `json:"rxInfo"`
	PHYPayload []byte `json:"phyPayload"`
}

// RXInfo contains the RX information.
type RXInfo struct {
	MAC       lorawan.EUI64 `json:"mac"`            // MAC address of the gateway
	Time      time.Time     `json:"time,omitempty"` // receive time
	Timestamp uint32        `json:"timestamp"`      // gateway internal receive timestamp with microsecond precision, will rollover every ~ 72 minutes
	Frequency int           `json:"frequency"`      // frequency in Hz
	Channel   int           `json:"channel"`        // concentrator IF channel used for RX
	RFChain   int           `json:"rfChain"`        // RF chain used for RX
	CRCStatus int           `json:"crcStatus"`      // 1 = OK, -1 = fail, 0 = no CRC
	CodeRate  string        `json:"codeRate"`       // ECC code rate
	RSSI      int           `json:"rssi"`           // RSSI in dBm
	LoRaSNR   float64       `json:"loRaSNR"`        // LoRa signal-to-noise ratio in dB
	Size      int           `json:"size"`           // packet payload size
	DataRate  band.DataRate `json:"dataRate"`       // RX datarate (either LoRa or FSK)
}

// TXPacket contains the PHYPayload which should be send to the
// gateway.
type TXPacket struct {
	TXInfo     TXInfo             `json:"txInfo"`
	PHYPayload lorawan.PHYPayload `json:"phyPayload"`
}

// TXPacketBytes contains the PHYPayload as []byte which should be send to the
// gateway. The JSON output is compatible with TXPacket.
type TXPacketBytes struct {
	TXInfo     TXInfo `json:"txInfo"`
	PHYPayload []byte `json:"phyPayload"`
}

// TXInfo contains the information used for TX.
type TXInfo struct {
	MAC         lorawan.EUI64 `json:"mac"`         // MAC address of the gateway
	Immediately bool          `json:"immediately"` // send the packet immediately (ignore Time)
	Timestamp   uint32        `json:"timestamp"`   // gateway internal receive timestamp with microsecond precision, will rollover every ~ 72 minutes
	Frequency   int           `json:"frequency"`   // frequency in Hz
	Power       int           `json:"power"`       // TX power to use in dBm
	DataRate    band.DataRate `json:"dataRate"`    // TX datarate (either LoRa or FSK)
	CodeRate    string        `json:"codeRate"`    // ECC code rate
	IPol        *bool         `json:"iPol"`        // when left nil, the gateway-bridge will use the default (true for LoRa modulation)
}

// GatewayStatsPacket contains the information of a gateway.
type GatewayStatsPacket struct {
	MAC                 lorawan.EUI64          `json:"mac"`
	Time                time.Time              `json:"time,omitempty"`
	Latitude            float64                `json:"latitude"`
	Longitude           float64                `json:"longitude"`
	Altitude            float64                `json:"altitude"`
	RXPacketsReceived   int                    `json:"rxPacketsReceived"`
	RXPacketsReceivedOK int                    `json:"rxPacketsReceivedOK"`
	TXPacketsEmitted    int                    `json:"txPacketsEmitted"`
	CustomData          map[string]interface{} `json:"customData"` // custom fields defined by alternative packet_forwarder versions (e.g. TTN sends platform, contactEmail, and description)
}
<commit_msg>Add TXPacketsReceived field to GatewayStatsPacket.<commit_after>package gw

import (
	"time"

	"github.com/brocaar/lorawan"
	"github.com/brocaar/lorawan/band"
)

// RXPacket contains the PHYPayload received from the gateway.
type RXPacket struct {
	RXInfo     RXInfo             `json:"rxInfo"`
	PHYPayload lorawan.PHYPayload `json:"phyPayload"`
}

// RXPacketBytes contains the PHYPayload as []byte received from the gateway.
// The JSON output is compatible with RXPacket.
type RXPacketBytes struct {
	RXInfo     RXInfo `json:"rxInfo"`
	PHYPayload []byte `json:"phyPayload"`
}

// RXInfo contains the RX information.
type RXInfo struct {
	MAC       lorawan.EUI64 `json:"mac"`            // MAC address of the gateway
	Time      time.Time     `json:"time,omitempty"` // receive time
	Timestamp uint32        `json:"timestamp"`      // gateway internal receive timestamp with microsecond precision, will rollover every ~ 72 minutes
	Frequency int           `json:"frequency"`      // frequency in Hz
	Channel   int           `json:"channel"`        // concentrator IF channel used for RX
	RFChain   int           `json:"rfChain"`        // RF chain used for RX
	CRCStatus int           `json:"crcStatus"`      // 1 = OK, -1 = fail, 0 = no CRC
	CodeRate  string        `json:"codeRate"`       // ECC code rate
	RSSI      int           `json:"rssi"`           // RSSI in dBm
	LoRaSNR   float64       `json:"loRaSNR"`        // LoRa signal-to-noise ratio in dB
	Size      int           `json:"size"`           // packet payload size
	DataRate  band.DataRate `json:"dataRate"`       // RX datarate (either LoRa or FSK)
}

// TXPacket contains the PHYPayload which should be send to the
// gateway.
type TXPacket struct {
	TXInfo     TXInfo             `json:"txInfo"`
	PHYPayload lorawan.PHYPayload `json:"phyPayload"`
}

// TXPacketBytes contains the PHYPayload as []byte which should be send to the
// gateway. The JSON output is compatible with TXPacket.
type TXPacketBytes struct {
	TXInfo     TXInfo `json:"txInfo"`
	PHYPayload []byte `json:"phyPayload"`
}

// TXInfo contains the information used for TX.
type TXInfo struct {
	MAC         lorawan.EUI64 `json:"mac"`         // MAC address of the gateway
	Immediately bool          `json:"immediately"` // send the packet immediately (ignore Time)
	Timestamp   uint32        `json:"timestamp"`   // gateway internal receive timestamp with microsecond precision, will rollover every ~ 72 minutes
	Frequency   int           `json:"frequency"`   // frequency in Hz
	Power       int           `json:"power"`       // TX power to use in dBm
	DataRate    band.DataRate `json:"dataRate"`    // TX datarate (either LoRa or FSK)
	CodeRate    string        `json:"codeRate"`    // ECC code rate
	IPol        *bool         `json:"iPol"`        // when left nil, the gateway-bridge will use the default (true for LoRa modulation)
}

// GatewayStatsPacket contains the information of a gateway.
type GatewayStatsPacket struct {
	MAC                 lorawan.EUI64          `json:"mac"`
	Time                time.Time              `json:"time,omitempty"`
	Latitude            float64                `json:"latitude"`
	Longitude           float64                `json:"longitude"`
	Altitude            float64                `json:"altitude"`
	RXPacketsReceived   int                    `json:"rxPacketsReceived"`
	RXPacketsReceivedOK int                    `json:"rxPacketsReceivedOK"`
	TXPacketsReceived   int                    `json:"txPacketsReceived"`
	TXPacketsEmitted    int                    `json:"txPacketsEmitted"`
	CustomData          map[string]interface{} `json:"customData"` // custom fields defined by alternative packet_forwarder versions (e.g. TTN sends platform, contactEmail, and description)
}
<|endoftext|>