<commit_before>package controller

import (
	"fmt"
	"time"

	"github.com/golang/glog"
	. "github.com/kubevirt/containerized-data-importer/pkg/common"
	"github.com/pkg/errors"
	"k8s.io/api/core/v1"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
)

const DataVolName = "cdi-data-vol"

// return a pvc pointer based on the passed-in work queue key.
func (c *Controller) pvcFromKey(key interface{}) (*v1.PersistentVolumeClaim, error) {
	keyString, ok := key.(string)
	if !ok {
		return nil, errors.New("key object not of type string")
	}
	obj, ok, err := c.pvcInformer.GetIndexer().GetByKey(keyString)
	if err != nil {
		return nil, errors.Wrapf(err, "Error getting key %q from cache", keyString)
	}
	if !ok {
		return nil, errors.Errorf("key %q not found in cache", keyString)
	}
	pvc, ok := obj.(*v1.PersistentVolumeClaim)
	if !ok {
		return nil, errors.New("Object not of type *v1.PersistentVolumeClaim")
	}
	return pvc, nil
}

func (c *Controller) podFromKey(key interface{}) (*v1.Pod, error) {
	keyString, ok := key.(string)
	if !ok {
		return nil, errors.New("keys is not of type string")
	}
	obj, ok, err := c.podInformer.GetIndexer().GetByKey(keyString)
	if err != nil {
		return nil, errors.Wrap(err, "error getting pod obj from store")
	}
	if !ok {
		return nil, errors.New("pod not found in store")
	}
	pod, ok := obj.(*v1.Pod)
	if !ok {
		return nil, errors.New("error casting object to type \"v1.Pod\"")
	}
	return pod, nil
}

// checkPVC verifies that the passed-in pvc is one we care about. Specifically, it must have the
// endpoint annotation and it must not already be "in-progress". If the pvc passes these filters
// then true is returned and the importer pod will be created. `AnnEndPoint` indicates that the
// pvc is targeted for the importer pod. `AnnImportPod` indicates the  pvc is being processed.
// Note: there is a race condition where the AnnImportPod annotation is not seen in time and as
//   a result the importer pod can be created twice (or more, presumably). To reduce this window
//   a Get api call can be requested in order to get the latest copy of the pvc before verifying
//   its annotations.
func (c *Controller) checkPVC(pvc *v1.PersistentVolumeClaim, get bool) (ok bool, newPvc *v1.PersistentVolumeClaim, err error) {
	// check if we have proper AnnEndPoint annotation
	if !metav1.HasAnnotation(pvc.ObjectMeta, AnnEndpoint) {
		glog.V(Vadmin).Infof("pvc annotation %q not found, skipping pvc\n", AnnEndpoint)
		return false, pvc, nil
	}
	//check if the pvc is being processed
	if metav1.HasAnnotation(pvc.ObjectMeta, AnnImportPod) {
		glog.V(Vadmin).Infof("pvc annotation %q exists indicating in-progress or completed, skipping pvc\n", AnnImportPod)
		return false, pvc, nil
	}

	if !get {
		return true, pvc, nil // done checking this pvc, assume it's good to go
	}

	// get latest pvc object to help mitigate race and timing issues with latency between the
	// store and work queue to double check if we are already processing
	glog.V(Vdebug).Infof("checkPVC: getting latest version of pvc for in-process annotation")
	newPvc, err = c.clientset.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{})
	if err != nil {
		glog.Infof("checkPVC: pvc %q Get error: %v\n", pvc.Name, err)
		return false, pvc, err
	}
	// check if we are processing this pvc now that we have the lastest copy
	if metav1.HasAnnotation(newPvc.ObjectMeta, AnnImportPod) {
		glog.V(Vadmin).Infof("latest pvc annotation %q exists indicating in-progress or completed, skipping pvc\n", AnnImportPod)
		return false, newPvc, nil
	}
	//continue to process pvc
	return true, newPvc, nil
}

// returns the endpoint string which contains the full path URI of the target object to be copied.
func getEndpoint(pvc *v1.PersistentVolumeClaim) (string, error) {
	ep, found := pvc.Annotations[AnnEndpoint]
	if !found || ep == "" {
		verb := "empty"
		if !found {
			verb = "missing"
		}
		return ep, errors.Errorf("annotation %q in pvc \"%s/%s\" is %s\n", AnnEndpoint, pvc.Namespace, pvc.Name, verb)
	}
	return ep, nil
}

// returns the name of the secret containing endpoint credentials consumed by the importer pod.
// A value of "" implies there are no credentials for the endpoint being used. A returned error
// causes processNextItem() to stop.
func (c *Controller) getSecretName(pvc *v1.PersistentVolumeClaim) (string, error) {
	ns := pvc.Namespace
	name, found := pvc.Annotations[AnnSecret]
	if !found || name == "" {
		msg := "getEndpointSecret: "
		if !found {
			msg += "annotation %q is missing in pvc \"%s/%s\""
		} else {
			msg += "secret name is missing from annotation %q in pvc \"%s/%s\""
		}
		glog.V(Vadmin).Infof(msg+"\n", AnnSecret, ns, pvc.Name)
		return "", nil // importer pod will not contain secret credentials
	}
	glog.V(Vdebug).Infof("getEndpointSecret: retrieving Secret \"%s/%s\"\n", ns, name)
	_, err := c.clientset.CoreV1().Secrets(ns).Get(name, metav1.GetOptions{})
	if apierrs.IsNotFound(err) {
		glog.V(Vuser).Infof("secret %q defined in pvc \"%s/%s\" is missing. Importer pod will run once this secret is created\n", name, ns, pvc.Name)
		return name, nil
	}
	if err != nil {
		return "", errors.Wrapf(err, "error getting secret %q defined in pvc \"%s/%s\"", name, ns, pvc.Name)
	}
	glog.V(Vuser).Infof("retrieved secret %q defined in pvc \"%s/%s\"\n", name, ns, pvc.Name)
	return name, nil
}

// Update and return a copy of the passed-in pvc. Only one of the annotation or label maps is requred though
// both can be passed.
// Note: the only pvc changes supported are annotations and labels.
func (c *Controller) updatePVC(pvc *v1.PersistentVolumeClaim, anno, label map[string]string) (*v1.PersistentVolumeClaim, error) {
	glog.V(Vdebug).Infof("updatePVC: updating pvc \"%s/%s\" with anno: %+v and label: %+v", pvc.Namespace, pvc.Name, anno, label)

	applyUpdt := func(claim *v1.PersistentVolumeClaim, a, l map[string]string) {
		if a != nil {
			claim.ObjectMeta.Annotations = addToMap(claim.ObjectMeta.Annotations, a)
		}
		if l != nil {
			claim.ObjectMeta.Labels = addToMap(claim.ObjectMeta.Labels, l)
		}
	}

	var updtPvc *v1.PersistentVolumeClaim
	nsName := fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Name)
	// don't mutate the passed-in pvc since it's likely from the shared informer
	pvcCopy := pvc.DeepCopy()

	// loop a few times in case the pvc is stale
	err := wait.PollImmediate(time.Second*1, time.Second*10, func() (bool, error) {
		var e error
		applyUpdt(pvcCopy, anno, label)
		updtPvc, e = c.clientset.CoreV1().PersistentVolumeClaims(pvc.Namespace).Update(pvcCopy)
		if e == nil {
			return true, nil // successful update
		}
		if apierrs.IsConflict(e) { // pvc is likely stale
			glog.V(Vdebug).Infof("pvc %q is stale, re-trying\n", nsName)
			pvcCopy, e = c.clientset.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{})
			if e == nil {
				return false, nil // retry update
			}
			// Get failed, start over
			pvcCopy = pvc.DeepCopy()
		}
		glog.Errorf("%q update/get error: %v\n", nsName, e)
		return false, nil // retry
	})

	if err == nil {
		glog.V(Vdebug).Infof("updatePVC: pvc %q updated", nsName)
		return updtPvc, nil
	}
	return pvc, errors.Wrapf(err, "error updating pvc %q\n", nsName)
}

// Sets an annotation `key: val` in the given pvc. Returns the updated pvc.
func (c *Controller) setPVCAnnotation(pvc *v1.PersistentVolumeClaim, key, val string) (*v1.PersistentVolumeClaim, error) {
	glog.V(Vdebug).Infof("setPVCAnnotation: adding annotation \"%s: %s\" to pvc \"%s/%s\"\n", key, val, pvc.Namespace, pvc.Name)
	return c.updatePVC(pvc, map[string]string{key: val}, nil)
}

// checks if annotation `key` has a value of `val`.
func (c *Controller) checkIfAnnoExists(pvc *v1.PersistentVolumeClaim, key string, val string) bool {
	value, exists := pvc.ObjectMeta.Annotations[key]
	if exists && value == val {
		return true
	}
	return false
}

// checks if particular label exists in pvc
func (c *Controller) checkIfLabelExists(pvc *v1.PersistentVolumeClaim, lbl string, val string) bool {
	value, exists := pvc.ObjectMeta.Labels[lbl]
	if exists && value == val {
		return true
	}
	return false
}

// return a pointer to a pod which is created based on the passed-in endpoint, secret
// name, and pvc. A nil secret means the endpoint credentials are not passed to the
// importer pod.
func (c *Controller) createImporterPod(ep, secretName string, pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) {
	ns := pvc.Namespace
	pod := c.makeImporterPodSpec(ep, secretName, pvc)

	pod, err := c.clientset.CoreV1().Pods(ns).Create(pod)
	if err != nil {
		return nil, errors.Wrap(err, "importer pod API create errored")
	}
	glog.V(Vuser).Infof("importer pod \"%s/%s\" (image: %q) created\n", pod.Namespace, pod.Name, c.importerImage)
	return pod, nil
}

// return the importer pod spec based on the passed-in endpoint, secret and pvc.
func (c *Controller) makeImporterPodSpec(ep, secret string, pvc *v1.PersistentVolumeClaim) *v1.Pod {
	// importer pod name contains the pvc name
	podName := fmt.Sprintf("%s-%s-", IMPORTER_PODNAME, pvc.Name)

	pod := &v1.Pod{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Pod",
			APIVersion: "v1",
		},
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: podName,
			Annotations: map[string]string{
				AnnCreatedBy: "yes",
			},
			Labels: map[string]string{
				CDI_LABEL_KEY: CDI_LABEL_VALUE,
			},
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:            IMPORTER_PODNAME,
					Image:           c.importerImage,
					ImagePullPolicy: v1.PullPolicy(c.pullPolicy),
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      DataVolName,
							MountPath: IMPORTER_DATA_DIR,
						},
					},
					Args: []string{"-v=" + c.verbose},
				},
			},
			RestartPolicy: v1.RestartPolicyNever,
			Volumes: []v1.Volume{
				{
					Name: DataVolName,
					VolumeSource: v1.VolumeSource{
						PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
							ClaimName: pvc.Name,
							ReadOnly:  false,
						},
					},
				},
			},
		},
	}
	pod.Spec.Containers[0].Env = makeEnv(ep, secret)
	return pod
}

// return the Env portion for the importer container.
func makeEnv(endpoint, secret string) []v1.EnvVar {
	env := []v1.EnvVar{
		{
			Name:  IMPORTER_ENDPOINT,
			Value: endpoint,
		},
	}
	if secret != "" {
		env = append(env, v1.EnvVar{
			Name: IMPORTER_ACCESS_KEY_ID,
			ValueFrom: &v1.EnvVarSource{
				SecretKeyRef: &v1.SecretKeySelector{
					LocalObjectReference: v1.LocalObjectReference{
						Name: secret,
					},
					Key: KeyAccess,
				},
			},
		}, v1.EnvVar{
			Name: IMPORTER_SECRET_KEY,
			ValueFrom: &v1.EnvVarSource{
				SecretKeyRef: &v1.SecretKeySelector{
					LocalObjectReference: v1.LocalObjectReference{
						Name: secret,
					},
					Key: KeySecret,
				},
			},
		})
	}
	return env
}

// Return a new map consisting of map1 with map2 added. In general, map2 is expected to have a single key. eg
// a single annotation or label. If map1 has the same key as map2 then map2's value is used.
func addToMap(m1, m2 map[string]string) map[string]string {
	if m1 == nil {
		m1 = make(map[string]string)
	}
	for k, v := range m2 {
		m1[k] = v
	}
	return m1
}
<commit_msg>Update util.go<commit_after>package controller

import (
	"fmt"
	"time"

	"github.com/golang/glog"
	. "github.com/kubevirt/containerized-data-importer/pkg/common"
	"github.com/pkg/errors"
	"k8s.io/api/core/v1"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
)

const DataVolName = "cdi-data-vol"

// return a pvc pointer based on the passed-in work queue key.
func (c *Controller) pvcFromKey(key interface{}) (*v1.PersistentVolumeClaim, error) {
	keyString, ok := key.(string)
	if !ok {
		return nil, errors.New("key object not of type string")
	}
	obj, ok, err := c.pvcInformer.GetIndexer().GetByKey(keyString)
	if err != nil {
		return nil, errors.Wrapf(err, "Error getting key %q from cache", keyString)
	}
	if !ok {
		return nil, errors.Errorf("key %q not found in cache", keyString)
	}
	pvc, ok := obj.(*v1.PersistentVolumeClaim)
	if !ok {
		return nil, errors.New("Object not of type *v1.PersistentVolumeClaim")
	}
	return pvc, nil
}

func (c *Controller) podFromKey(key interface{}) (*v1.Pod, error) {
	keyString, ok := key.(string)
	if !ok {
		return nil, errors.New("keys is not of type string")
	}
	obj, ok, err := c.podInformer.GetIndexer().GetByKey(keyString)
	if err != nil {
		return nil, errors.Wrap(err, "error getting pod obj from store")
	}
	if !ok {
		return nil, errors.New("pod not found in store")
	}
	pod, ok := obj.(*v1.Pod)
	if !ok {
		return nil, errors.New("error casting object to type \"v1.Pod\"")
	}
	return pod, nil
}

// checkPVC verifies that the passed-in pvc is one we care about. Specifically, it must have the
// endpoint annotation and it must not already be "in-progress". If the pvc passes these filters
// then true is returned and the importer pod will be created. `AnnEndPoint` indicates that the
// pvc is targeted for the importer pod. `AnnImportPod` indicates the  pvc is being processed.
// Note: there is a race condition where the AnnImportPod annotation is not seen in time and as
//   a result the importer pod can be created twice (or more, presumably). To reduce this window
//   a Get api call can be requested in order to get the latest copy of the pvc before verifying
//   its annotations.
func (c *Controller) checkPVC(pvc *v1.PersistentVolumeClaim, get bool) (ok bool, newPvc *v1.PersistentVolumeClaim, err error) {
	// check if we have proper AnnEndPoint annotation
	if !metav1.HasAnnotation(pvc.ObjectMeta, AnnEndpoint) {
		glog.V(Vadmin).Infof("pvc annotation %q not found, skipping pvc\n", AnnEndpoint)
		return false, pvc, nil
	}
	//check if the pvc is being processed
	if metav1.HasAnnotation(pvc.ObjectMeta, AnnImportPod) {
		glog.V(Vadmin).Infof("pvc annotation %q exists indicating in-progress or completed, skipping pvc\n", AnnImportPod)
		return false, pvc, nil
	}

	if !get {
		return true, pvc, nil // done checking this pvc, assume it's good to go
	}

	// get latest pvc object to help mitigate race and timing issues with latency between the
	// store and work queue to double check if we are already processing
	glog.V(Vdebug).Infof("checkPVC: getting latest version of pvc for in-process annotation")
	newPvc, err = c.clientset.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{})
	if err != nil {
		glog.Infof("checkPVC: pvc %q Get error: %v\n", pvc.Name, err)
		return false, pvc, err
	}
	// check if we are processing this pvc now that we have the lastest copy
	if metav1.HasAnnotation(newPvc.ObjectMeta, AnnImportPod) {
		glog.V(Vadmin).Infof("latest pvc annotation %q exists indicating in-progress or completed, skipping pvc\n", AnnImportPod)
		return false, newPvc, nil
	}
	//continue to process pvc
	return true, newPvc, nil
}

// returns the endpoint string which contains the full path URI of the target object to be copied.
func getEndpoint(pvc *v1.PersistentVolumeClaim) (string, error) {
	ep, found := pvc.Annotations[AnnEndpoint]
	if !found || ep == "" {
		verb := "empty"
		if !found {
			verb = "missing"
		}
		return ep, errors.Errorf("annotation %q in pvc \"%s/%s\" is %s\n", AnnEndpoint, pvc.Namespace, pvc.Name, verb)
	}
	return ep, nil
}

// returns the name of the secret containing endpoint credentials consumed by the importer pod.
// A value of "" implies there are no credentials for the endpoint being used. A returned error
// causes processNextItem() to stop.
func (c *Controller) getSecretName(pvc *v1.PersistentVolumeClaim) (string, error) {
	ns := pvc.Namespace
	name, found := pvc.Annotations[AnnSecret]
	if !found || name == "" {
		msg := "getEndpointSecret: "
		if !found {
			msg += "annotation %q is missing in pvc \"%s/%s\""
		} else {
			msg += "secret name is missing from annotation %q in pvc \"%s/%s\""
		}
		glog.V(Vadmin).Infof(msg+"\n", AnnSecret, ns, pvc.Name)
		return "", nil // importer pod will not contain secret credentials
	}
	glog.V(Vdebug).Infof("getEndpointSecret: retrieving Secret \"%s/%s\"\n", ns, name)
	_, err := c.clientset.CoreV1().Secrets(ns).Get(name, metav1.GetOptions{})
	if apierrs.IsNotFound(err) {
		glog.V(Vuser).Infof("secret %q defined in pvc \"%s/%s\" is missing. Importer pod will run once this secret is created\n", name, ns, pvc.Name)
		return name, nil
	}
	if err != nil {
		return "", errors.Wrapf(err, "error getting secret %q defined in pvc \"%s/%s\"", name, ns, pvc.Name)
	}
	glog.V(Vuser).Infof("retrieved secret %q defined in pvc \"%s/%s\"\n", name, ns, pvc.Name)
	return name, nil
}

// Update and return a copy of the passed-in pvc. Only one of the annotation or label maps is required though
// both can be passed.
// Note: the only pvc changes supported are annotations and labels.
func (c *Controller) updatePVC(pvc *v1.PersistentVolumeClaim, anno, label map[string]string) (*v1.PersistentVolumeClaim, error) {
	glog.V(Vdebug).Infof("updatePVC: updating pvc \"%s/%s\" with anno: %+v and label: %+v", pvc.Namespace, pvc.Name, anno, label)

	applyUpdt := func(claim *v1.PersistentVolumeClaim, a, l map[string]string) {
		if a != nil {
			claim.ObjectMeta.Annotations = addToMap(claim.ObjectMeta.Annotations, a)
		}
		if l != nil {
			claim.ObjectMeta.Labels = addToMap(claim.ObjectMeta.Labels, l)
		}
	}

	var updtPvc *v1.PersistentVolumeClaim
	nsName := fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Name)
	// don't mutate the passed-in pvc since it's likely from the shared informer
	pvcCopy := pvc.DeepCopy()

	// loop a few times in case the pvc is stale
	err := wait.PollImmediate(time.Second*1, time.Second*10, func() (bool, error) {
		var e error
		applyUpdt(pvcCopy, anno, label)
		updtPvc, e = c.clientset.CoreV1().PersistentVolumeClaims(pvc.Namespace).Update(pvcCopy)
		if e == nil {
			return true, nil // successful update
		}
		if apierrs.IsConflict(e) { // pvc is likely stale
			glog.V(Vdebug).Infof("pvc %q is stale, re-trying\n", nsName)
			pvcCopy, e = c.clientset.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{})
			if e == nil {
				return false, nil // retry update
			}
			// Get failed, start over
			pvcCopy = pvc.DeepCopy()
		}
		glog.Errorf("%q update/get error: %v\n", nsName, e)
		return false, nil // retry
	})

	if err == nil {
		glog.V(Vdebug).Infof("updatePVC: pvc %q updated", nsName)
		return updtPvc, nil
	}
	return pvc, errors.Wrapf(err, "error updating pvc %q\n", nsName)
}

// Sets an annotation `key: val` in the given pvc. Returns the updated pvc.
func (c *Controller) setPVCAnnotation(pvc *v1.PersistentVolumeClaim, key, val string) (*v1.PersistentVolumeClaim, error) {
	glog.V(Vdebug).Infof("setPVCAnnotation: adding annotation \"%s: %s\" to pvc \"%s/%s\"\n", key, val, pvc.Namespace, pvc.Name)
	return c.updatePVC(pvc, map[string]string{key: val}, nil)
}

// checks if annotation `key` has a value of `val`.
func (c *Controller) checkIfAnnoExists(pvc *v1.PersistentVolumeClaim, key string, val string) bool {
	value, exists := pvc.ObjectMeta.Annotations[key]
	if exists && value == val {
		return true
	}
	return false
}

// checks if particular label exists in pvc
func (c *Controller) checkIfLabelExists(pvc *v1.PersistentVolumeClaim, lbl string, val string) bool {
	value, exists := pvc.ObjectMeta.Labels[lbl]
	if exists && value == val {
		return true
	}
	return false
}

// return a pointer to a pod which is created based on the passed-in endpoint, secret
// name, and pvc. A nil secret means the endpoint credentials are not passed to the
// importer pod.
func (c *Controller) createImporterPod(ep, secretName string, pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) {
	ns := pvc.Namespace
	pod := c.makeImporterPodSpec(ep, secretName, pvc)

	pod, err := c.clientset.CoreV1().Pods(ns).Create(pod)
	if err != nil {
		return nil, errors.Wrap(err, "importer pod API create errored")
	}
	glog.V(Vuser).Infof("importer pod \"%s/%s\" (image: %q) created\n", pod.Namespace, pod.Name, c.importerImage)
	return pod, nil
}

// return the importer pod spec based on the passed-in endpoint, secret and pvc.
func (c *Controller) makeImporterPodSpec(ep, secret string, pvc *v1.PersistentVolumeClaim) *v1.Pod {
	// importer pod name contains the pvc name
	podName := fmt.Sprintf("%s-%s-", IMPORTER_PODNAME, pvc.Name)

	pod := &v1.Pod{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Pod",
			APIVersion: "v1",
		},
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: podName,
			Annotations: map[string]string{
				AnnCreatedBy: "yes",
			},
			Labels: map[string]string{
				CDI_LABEL_KEY: CDI_LABEL_VALUE,
			},
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{
				{
					Name:            IMPORTER_PODNAME,
					Image:           c.importerImage,
					ImagePullPolicy: v1.PullPolicy(c.pullPolicy),
					VolumeMounts: []v1.VolumeMount{
						{
							Name:      DataVolName,
							MountPath: IMPORTER_DATA_DIR,
						},
					},
					Args: []string{"-v=" + c.verbose},
				},
			},
			RestartPolicy: v1.RestartPolicyNever,
			Volumes: []v1.Volume{
				{
					Name: DataVolName,
					VolumeSource: v1.VolumeSource{
						PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
							ClaimName: pvc.Name,
							ReadOnly:  false,
						},
					},
				},
			},
		},
	}
	pod.Spec.Containers[0].Env = makeEnv(ep, secret)
	return pod
}

// return the Env portion for the importer container.
func makeEnv(endpoint, secret string) []v1.EnvVar {
	env := []v1.EnvVar{
		{
			Name:  IMPORTER_ENDPOINT,
			Value: endpoint,
		},
	}
	if secret != "" {
		env = append(env, v1.EnvVar{
			Name: IMPORTER_ACCESS_KEY_ID,
			ValueFrom: &v1.EnvVarSource{
				SecretKeyRef: &v1.SecretKeySelector{
					LocalObjectReference: v1.LocalObjectReference{
						Name: secret,
					},
					Key: KeyAccess,
				},
			},
		}, v1.EnvVar{
			Name: IMPORTER_SECRET_KEY,
			ValueFrom: &v1.EnvVarSource{
				SecretKeyRef: &v1.SecretKeySelector{
					LocalObjectReference: v1.LocalObjectReference{
						Name: secret,
					},
					Key: KeySecret,
				},
			},
		})
	}
	return env
}

// Return a new map consisting of map1 with map2 added. In general, map2 is expected to have a single key. eg
// a single annotation or label. If map1 has the same key as map2 then map2's value is used.
func addToMap(m1, m2 map[string]string) map[string]string {
	if m1 == nil {
		m1 = make(map[string]string)
	}
	for k, v := range m2 {
		m1[k] = v
	}
	return m1
}
<|endoftext|>


<commit_before>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package envoy

import (
	"sort"

	envoyAPI "github.com/cilium/cilium/pkg/envoy/cilium"
	"github.com/cilium/cilium/pkg/envoy/xds"
	"github.com/cilium/cilium/pkg/identity"
	"github.com/cilium/cilium/pkg/ipcache"
	"github.com/cilium/cilium/pkg/logging/logfields"

	"github.com/sirupsen/logrus"
)

const (
	// ListenerTypeURL is the type URL of Listener resources.
	ListenerTypeURL = "type.googleapis.com/envoy.api.v2.Listener"

	// NetworkPolicyTypeURL is the type URL of NetworkPolicy resources.
	NetworkPolicyTypeURL = "type.googleapis.com/cilium.NetworkPolicy"

	// NetworkPolicyHostsTypeURL is the type URL of NetworkPolicyHosts resources.
	NetworkPolicyHostsTypeURL = "type.googleapis.com/cilium.NetworkPolicyHosts"
)

// NPHDSCache is a cache of resources in the Network Policy Hosts Discovery
// Service.
type NPHDSCache struct {
	*xds.Cache
}

func newNPHDSCache() NPHDSCache {
	return NPHDSCache{Cache: xds.NewCache()}
}

var (
	// NetworkPolicyHostsCache is the global cache of resources of type
	// NetworkPolicyHosts. Resources in this cache must have the
	// NetworkPolicyHostsTypeURL type URL.
	NetworkPolicyHostsCache = newNPHDSCache()
)

// OnIPIdentityCacheGC is required to implement IPIdentityMappingListener.
func (cache *NPHDSCache) OnIPIdentityCacheGC() {
	// We don't have anything to synchronize in this case.
}

// OnIPIdentityCacheChange pushes modifications to the IP<->Identity mapping
// into the Network Policy Host Discovery Service (NPHDS).
func (cache *NPHDSCache) OnIPIdentityCacheChange(
	modType ipcache.CacheModification, ipIDPair identity.IPIdentityPair) {

	scopedLog := log.WithFields(logrus.Fields{
		logfields.IPAddr:       ipIDPair.IP,
		logfields.Identity:     ipIDPair.ID,
		logfields.Modification: modType,
	})
	// Look up the current resources for the specified Identity.
	msg, err := cache.Lookup(NetworkPolicyHostsTypeURL, ipIDPair.ID.StringID())
	if err != nil {
		scopedLog.WithError(err).Warning("Can't lookup NPHDS cache")
		return
	}

	switch modType {
	case ipcache.Upsert:
		var npHost *envoyAPI.NetworkPolicyHosts
		if msg == nil {
			npHost = &envoyAPI.NetworkPolicyHosts{Policy: uint64(ipIDPair.ID), HostAddresses: make([]string, 0, 1)}
		} else {
			npHost = msg.(*envoyAPI.NetworkPolicyHosts)
		}
		npHost.HostAddresses = append(npHost.HostAddresses, ipIDPair.PrefixString())
		sort.Strings(npHost.HostAddresses)
		if err := npHost.Validate(); err != nil {
			scopedLog.WithError(err).WithFields(logrus.Fields{
				logfields.XDSResource: npHost,
			}).Warning("Could not validate NPHDS resource update on upsert")
			return
		}
		cache.Upsert(NetworkPolicyHostsTypeURL, ipIDPair.ID.StringID(), npHost, false)
	case ipcache.Delete:
		if msg == nil {
			// Doesn't exist; already deleted.
			return
		}
		cache.handleIPDelete(msg.(*envoyAPI.NetworkPolicyHosts), ipIDPair.ID.StringID(), ipIDPair.PrefixString())
	}
}

// handleIPUpsert deletes elements from the NPHDS cache with the specified peer IP->ID mapping.
func (cache *NPHDSCache) handleIPDelete(npHost *envoyAPI.NetworkPolicyHosts, peerIdentity, peerIP string) {
	targetIndex := -1

	scopedLog := log.WithFields(logrus.Fields{
		logfields.IPAddr:       peerIP,
		logfields.Identity:     peerIdentity,
		logfields.Modification: ipcache.Delete,
	})
	for i, endpointIP := range npHost.HostAddresses {
		if endpointIP == peerIP {
			targetIndex = i
			break
		}
	}
	if targetIndex < 0 {
		scopedLog.Fatal("Can't find IP in NPHDS cache")
		return
	}

	// If removing this host would result in empty list, delete it.
	// Otherwise, update to a list that doesn't contain the target IP
	if len(npHost.HostAddresses) <= 1 {
		cache.Delete(NetworkPolicyHostsTypeURL, peerIdentity, false)
	} else {
		if len(npHost.HostAddresses) == targetIndex {
			npHost.HostAddresses = npHost.HostAddresses[0:targetIndex]
		} else {
			npHost.HostAddresses = append(npHost.HostAddresses[0:targetIndex], npHost.HostAddresses[targetIndex+1:]...)
		}
		if err := npHost.Validate(); err != nil {
			scopedLog.WithError(err).Warning("Could not validate NPHDS resource update on delete")
			return
		}
		cache.Upsert(NetworkPolicyHostsTypeURL, peerIdentity, npHost, false)
	}
}
<commit_msg>ipcache: Create copies of NPHDS cache resources when updating<commit_after>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package envoy

import (
	"sort"

	envoyAPI "github.com/cilium/cilium/pkg/envoy/cilium"
	"github.com/cilium/cilium/pkg/envoy/xds"
	"github.com/cilium/cilium/pkg/identity"
	"github.com/cilium/cilium/pkg/ipcache"
	"github.com/cilium/cilium/pkg/logging/logfields"

	"github.com/sirupsen/logrus"
)

const (
	// ListenerTypeURL is the type URL of Listener resources.
	ListenerTypeURL = "type.googleapis.com/envoy.api.v2.Listener"

	// NetworkPolicyTypeURL is the type URL of NetworkPolicy resources.
	NetworkPolicyTypeURL = "type.googleapis.com/cilium.NetworkPolicy"

	// NetworkPolicyHostsTypeURL is the type URL of NetworkPolicyHosts resources.
	NetworkPolicyHostsTypeURL = "type.googleapis.com/cilium.NetworkPolicyHosts"
)

// NPHDSCache is a cache of resources in the Network Policy Hosts Discovery
// Service.
type NPHDSCache struct {
	*xds.Cache
}

func newNPHDSCache() NPHDSCache {
	return NPHDSCache{Cache: xds.NewCache()}
}

var (
	// NetworkPolicyHostsCache is the global cache of resources of type
	// NetworkPolicyHosts. Resources in this cache must have the
	// NetworkPolicyHostsTypeURL type URL.
	NetworkPolicyHostsCache = newNPHDSCache()
)

// OnIPIdentityCacheGC is required to implement IPIdentityMappingListener.
func (cache *NPHDSCache) OnIPIdentityCacheGC() {
	// We don't have anything to synchronize in this case.
}

// OnIPIdentityCacheChange pushes modifications to the IP<->Identity mapping
// into the Network Policy Host Discovery Service (NPHDS).
func (cache *NPHDSCache) OnIPIdentityCacheChange(
	modType ipcache.CacheModification, ipIDPair identity.IPIdentityPair) {

	scopedLog := log.WithFields(logrus.Fields{
		logfields.IPAddr:       ipIDPair.IP,
		logfields.Identity:     ipIDPair.ID,
		logfields.Modification: modType,
	})
	// Look up the current resources for the specified Identity.
	msg, err := cache.Lookup(NetworkPolicyHostsTypeURL, ipIDPair.ID.StringID())
	if err != nil {
		scopedLog.WithError(err).Warning("Can't lookup NPHDS cache")
		return
	}

	switch modType {
	case ipcache.Upsert:
		var hostAddresses []string
		if msg == nil {
			hostAddresses = make([]string, 0, 1)
		} else {
			// If the resource already exists, create a copy of it and insert
			// the new IP address into its HostAddresses list.
			npHost := msg.(*envoyAPI.NetworkPolicyHosts)
			hostAddresses = make([]string, 0, len(npHost.HostAddresses)+1)
			hostAddresses = append(hostAddresses, npHost.HostAddresses...)
		}
		hostAddresses = append(hostAddresses, ipIDPair.PrefixString())
		sort.Strings(hostAddresses)

		newNpHost := envoyAPI.NetworkPolicyHosts{
			Policy:        uint64(ipIDPair.ID),
			HostAddresses: hostAddresses,
		}
		if err := newNpHost.Validate(); err != nil {
			scopedLog.WithError(err).WithFields(logrus.Fields{
				logfields.XDSResource: newNpHost,
			}).Warning("Could not validate NPHDS resource update on upsert")
			return
		}
		cache.Upsert(NetworkPolicyHostsTypeURL, ipIDPair.ID.StringID(), &newNpHost, false)
	case ipcache.Delete:
		if msg == nil {
			// Doesn't exist; already deleted.
			return
		}
		cache.handleIPDelete(msg.(*envoyAPI.NetworkPolicyHosts), ipIDPair.ID.StringID(), ipIDPair.PrefixString())
	}
}

// handleIPUpsert deletes elements from the NPHDS cache with the specified peer IP->ID mapping.
func (cache *NPHDSCache) handleIPDelete(npHost *envoyAPI.NetworkPolicyHosts, peerIdentity, peerIP string) {
	targetIndex := -1

	scopedLog := log.WithFields(logrus.Fields{
		logfields.IPAddr:       peerIP,
		logfields.Identity:     peerIdentity,
		logfields.Modification: ipcache.Delete,
	})
	for i, endpointIP := range npHost.HostAddresses {
		if endpointIP == peerIP {
			targetIndex = i
			break
		}
	}
	if targetIndex < 0 {
		scopedLog.Fatal("Can't find IP in NPHDS cache")
		return
	}

	// If removing this host would result in empty list, delete it.
	// Otherwise, update to a list that doesn't contain the target IP
	if len(npHost.HostAddresses) <= 1 {
		cache.Delete(NetworkPolicyHostsTypeURL, peerIdentity, false)
	} else {
		// If the resource is to be updated, create a copy of it before
		// removing the IP address from its HostAddresses list.
		var hostAddresses []string
		if len(npHost.HostAddresses) == targetIndex {
			hostAddresses = npHost.HostAddresses[0:targetIndex]
		} else {
			hostAddresses = append(npHost.HostAddresses[0:targetIndex], npHost.HostAddresses[targetIndex+1:]...)
		}

		newNpHost := envoyAPI.NetworkPolicyHosts{
			Policy:        uint64(npHost.Policy),
			HostAddresses: hostAddresses,
		}
		if err := newNpHost.Validate(); err != nil {
			scopedLog.WithError(err).Warning("Could not validate NPHDS resource update on delete")
			return
		}
		cache.Upsert(NetworkPolicyHostsTypeURL, peerIdentity, &newNpHost, false)
	}
}
<|endoftext|>


<commit_before>package event

import (
	"github.com/Aptomi/aptomi/pkg/errors"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"github.com/Sirupsen/logrus"
	"io/ioutil"
)

// Fields is a set of named fields. Fields are attached to every log record
type Fields map[string]interface{}

// attachedObjects is a list of core aptomi lang objects attached to a set of log records (e.g. dependency, user, contract, context, key)
type attachedObjects struct {
	objects []interface{}
}

// Log is an buffered event log.
// It stores all log entries in memory first, then allows them to be processed and stored
type Log struct {
	logger     *logrus.Logger
	attachedTo *attachedObjects
	hookMemory *HookMemory
	scope      string
	level      logrus.Level
	log        bool
}

// NewLog creates a new instance of event log.
// Initially it just buffers all entries and doesn't write them.
// It needs to buffer all entries, so that the context can be later attached to them
// before they get serialized and written to an external source
func NewLog(level logrus.Level, scope string, logToConsole bool) *Log {
	logger := &logrus.Logger{
		Out:       ioutil.Discard,
		Formatter: new(logrus.TextFormatter),
		Hooks:     make(logrus.LevelHooks),
		Level:     level,
	}

	hookMemory := &HookMemory{}
	logger.Hooks.Add(hookMemory)

	if logToConsole {
		logger.Hooks.Add(NewHookConsole())
	}

	return &Log{
		logger:     logger,
		attachedTo: &attachedObjects{},
		hookMemory: hookMemory,
		scope:      scope,
		level:      level,
		log:        logToConsole,
	}
}

// Replaces base objects with their string key value
func fieldValue(data interface{}) interface{} {
	if baseObject, ok := data.(runtime.Storable); ok {
		return runtime.KeyForStorable(baseObject)
	}
	return data
}

// GetLevel returns log level for the event log
func (eventLog *Log) GetLevel() logrus.Level {
	return eventLog.level
}

// GetScope returns scope for event log
func (eventLog *Log) GetScope() string {
	return eventLog.scope
}

// IsLog returns true if current event log writes logs on the go
func (eventLog *Log) IsLog() bool {
	return eventLog.log
}

// WithFields creates a new log entry with a given set of fields
func (eventLog *Log) WithFields(fields Fields) *logrus.Entry {
	// see if there is any details which have to added to the log from the error
	for key, value := range fields {
		if errWithDetails, ok := value.(*errors.ErrorWithDetails); ok {
			// put details from the error into the same log record
			for dKey, dValue := range errWithDetails.Details() {
				fields[dKey] = fieldValue(dValue)
			}
			fields[key] = errWithDetails.Error()
		} else {
			fields[key] = fieldValue(value)
		}
	}

	if len(eventLog.scope) > 0 {
		fields["scope"] = eventLog.scope
	}

	return eventLog.logger.WithFields(logrus.Fields(fields))
}

// AttachTo attaches all entries in this event log to a certain object
// E.g. dependency, user, contract, context, serviceKey
func (eventLog *Log) AttachTo(object interface{}) {
	eventLog.attachedTo.objects = append(eventLog.attachedTo.objects, object)
}

// Append adds entries to the event logs
func (eventLog *Log) Append(that *Log) {
	eventLog.hookMemory.entries = append(eventLog.hookMemory.entries, that.hookMemory.entries...)
}

// LogError logs an error. Errors with details are processed specially, their details get unfolded as record fields
func (eventLog *Log) LogError(err error) {
	errWithDetails, isErrorWithDetails := err.(*errors.ErrorWithDetails)
	if isErrorWithDetails {
		eventLog.WithFields(Fields(errWithDetails.Details())).Error(err.Error())
	} else {
		eventLog.WithFields(Fields{}).Error(err.Error())
	}
}

// LogWarning logs a warning. Errors with details are processed specially, their details get unfolded as record fields
func (eventLog *Log) LogWarning(err error) {
	errWithDetails, isErrorWithDetails := err.(*errors.ErrorWithDetails)
	if isErrorWithDetails {
		eventLog.WithFields(Fields(errWithDetails.Details())).Warning(err.Error())
	} else {
		eventLog.WithFields(Fields{}).Warning(err.Error())
	}
}

// Save takes all buffered event log entries and saves them
func (eventLog *Log) Save(hook logrus.Hook) {
	for _, e := range eventLog.hookMemory.entries {
		e.Data["attachedTo"] = eventLog.attachedTo
		err := hook.Fire(e)
		if err != nil {
			panic(err) // is it ok to panic here?
		}
	}
}
<commit_msg>removed log field and IsLog() from event log<commit_after>package event

import (
	"github.com/Aptomi/aptomi/pkg/errors"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"github.com/Sirupsen/logrus"
	"io/ioutil"
)

// Fields is a set of named fields. Fields are attached to every log record
type Fields map[string]interface{}

// attachedObjects is a list of core aptomi lang objects attached to a set of log records (e.g. dependency, user, contract, context, key)
type attachedObjects struct {
	objects []interface{}
}

// Log is an buffered event log.
// It stores all log entries in memory first, then allows them to be processed and stored
type Log struct {
	logger     *logrus.Logger
	attachedTo *attachedObjects
	hookMemory *HookMemory
	scope      string
	level      logrus.Level
}

// NewLog creates a new instance of event log.
// Initially it just buffers all entries and doesn't write them.
// It needs to buffer all entries, so that the context can be later attached to them
// before they get serialized and written to an external source
func NewLog(level logrus.Level, scope string, logToConsole bool) *Log {
	logger := &logrus.Logger{
		Out:       ioutil.Discard,
		Formatter: new(logrus.TextFormatter),
		Hooks:     make(logrus.LevelHooks),
		Level:     level,
	}

	hookMemory := &HookMemory{}
	logger.Hooks.Add(hookMemory)

	if logToConsole {
		logger.Hooks.Add(NewHookConsole())
	}

	return &Log{
		logger:     logger,
		attachedTo: &attachedObjects{},
		hookMemory: hookMemory,
		scope:      scope,
		level:      level,
	}
}

// GetLevel returns log level for the event log
func (eventLog *Log) GetLevel() logrus.Level {
	return eventLog.level
}

// GetScope returns scope for event log
func (eventLog *Log) GetScope() string {
	return eventLog.scope
}

// Replaces base objects with their string key value
func fieldValue(data interface{}) interface{} {
	if baseObject, ok := data.(runtime.Storable); ok {
		return runtime.KeyForStorable(baseObject)
	}
	return data
}

// WithFields creates a new log entry with a given set of fields
func (eventLog *Log) WithFields(fields Fields) *logrus.Entry {
	// see if there is any details which have to added to the log from the error
	for key, value := range fields {
		if errWithDetails, ok := value.(*errors.ErrorWithDetails); ok {
			// put details from the error into the same log record
			for dKey, dValue := range errWithDetails.Details() {
				fields[dKey] = fieldValue(dValue)
			}
			fields[key] = errWithDetails.Error()
		} else {
			fields[key] = fieldValue(value)
		}
	}

	if len(eventLog.scope) > 0 {
		fields["scope"] = eventLog.scope
	}

	return eventLog.logger.WithFields(logrus.Fields(fields))
}

// AttachTo attaches all entries in this event log to a certain object
// E.g. dependency, user, contract, context, serviceKey
func (eventLog *Log) AttachTo(object interface{}) {
	eventLog.attachedTo.objects = append(eventLog.attachedTo.objects, object)
}

// Append adds entries to the event logs
func (eventLog *Log) Append(that *Log) {
	eventLog.hookMemory.entries = append(eventLog.hookMemory.entries, that.hookMemory.entries...)
}

// LogError logs an error. Errors with details are processed specially, their details get unfolded as record fields
func (eventLog *Log) LogError(err error) {
	errWithDetails, isErrorWithDetails := err.(*errors.ErrorWithDetails)
	if isErrorWithDetails {
		eventLog.WithFields(Fields(errWithDetails.Details())).Error(err.Error())
	} else {
		eventLog.WithFields(Fields{}).Error(err.Error())
	}
}

// LogWarning logs a warning. Errors with details are processed specially, their details get unfolded as record fields
func (eventLog *Log) LogWarning(err error) {
	errWithDetails, isErrorWithDetails := err.(*errors.ErrorWithDetails)
	if isErrorWithDetails {
		eventLog.WithFields(Fields(errWithDetails.Details())).Warning(err.Error())
	} else {
		eventLog.WithFields(Fields{}).Warning(err.Error())
	}
}

// Save takes all buffered event log entries and saves them
func (eventLog *Log) Save(hook logrus.Hook) {
	for _, e := range eventLog.hookMemory.entries {
		e.Data["attachedTo"] = eventLog.attachedTo
		err := hook.Fire(e)
		if err != nil {
			panic(err) // is it ok to panic here?
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package handler

import (
	"bytes"
	"compress/gzip"
	"crypto/sha512"
	"encoding/json"
	"fmt"
	"mime"
	"net/http"
	"strings"
	"sync"
	"time"

	"bitbucket.org/ww/goautoneg"

	yaml "gopkg.in/yaml.v2"

	"github.com/NYTimes/gziphandler"
	restful "github.com/emicklei/go-restful"
	"github.com/go-openapi/spec"
	"github.com/golang/protobuf/proto"
	openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2"
	"github.com/googleapis/gnostic/compiler"

	"k8s.io/kube-openapi/pkg/builder"
	"k8s.io/kube-openapi/pkg/common"
)

const (
	jsonExt = ".json"

	mimeJson = "application/json"
	// TODO(mehdy): change @68f4ded to a version tag when gnostic add version tags.
	mimePb   = "application/com.github.googleapis.gnostic.OpenAPIv2@68f4ded+protobuf"
	mimePbGz = "application/x-gzip"
)

// OpenAPIService is the service responsible for serving OpenAPI spec. It has
// the ability to safely change the spec while serving it.
type OpenAPIService struct {
	// rwMutex protects All members of this service.
	rwMutex sync.RWMutex

	orgSpec      *spec.Swagger
	lastModified time.Time

	specBytes []byte
	specPb    []byte
	specPbGz  []byte

	specBytesETag string
	specPbETag    string
	specPbGzETag  string
}

func init() {
	mime.AddExtensionType(".json", mimeJson)
	mime.AddExtensionType(".pb-v1", mimePb)
	mime.AddExtensionType(".gz", mimePbGz)
}

func computeETag(data []byte) string {
	return fmt.Sprintf("\"%X\"", sha512.Sum512(data))
}

// NOTE: [DEPRECATION] We will announce deprecation for format-separated endpoints for OpenAPI spec,
// and switch to a single /openapi/v2 endpoint in Kubernetes 1.10. The design doc and deprecation process
// are tracked at: https://docs.google.com/document/d/19lEqE9lc4yHJ3WJAJxS_G7TcORIJXGHyq3wpwcH28nU.
//
// BuildAndRegisterOpenAPIService builds the spec and registers a handler to provide access to it.
// Use this method if your OpenAPI spec is static. If you want to update the spec, use BuildOpenAPISpec then RegisterOpenAPIService.
func BuildAndRegisterOpenAPIService(servePath string, webServices []*restful.WebService, config *common.Config, handler common.PathHandler) (*OpenAPIService, error) {
	spec, err := builder.BuildOpenAPISpec(webServices, config)
	if err != nil {
		return nil, err
	}
	return RegisterOpenAPIService(spec, servePath, handler)
}

// NOTE: [DEPRECATION] We will announce deprecation for format-separated endpoints for OpenAPI spec,
// and switch to a single /openapi/v2 endpoint in Kubernetes 1.10. The design doc and deprecation process
// are tracked at: https://docs.google.com/document/d/19lEqE9lc4yHJ3WJAJxS_G7TcORIJXGHyq3wpwcH28nU.
//
// RegisterOpenAPIService registers a handler to provide access to provided swagger spec.
// Note: servePath should end with ".json" as the RegisterOpenAPIService assume it is serving a
// json file and will also serve .pb and .gz files.
func RegisterOpenAPIService(openapiSpec *spec.Swagger, servePath string, handler common.PathHandler) (*OpenAPIService, error) {
	if !strings.HasSuffix(servePath, jsonExt) {
		return nil, fmt.Errorf("serving path must end with \"%s\"", jsonExt)
	}

	servePathBase := strings.TrimSuffix(servePath, jsonExt)

	o := OpenAPIService{}
	if err := o.UpdateSpec(openapiSpec); err != nil {
		return nil, err
	}

	type fileInfo struct {
		ext            string
		getDataAndETag func() ([]byte, string, time.Time)
	}

	files := []fileInfo{
		{".json", o.getSwaggerBytes},
		{"-2.0.0.json", o.getSwaggerBytes},
		{"-2.0.0.pb-v1", o.getSwaggerPbBytes},
		{"-2.0.0.pb-v1.gz", o.getSwaggerPbGzBytes},
	}

	for _, file := range files {
		path := servePathBase + file.ext
		getDataAndETag := file.getDataAndETag
		handler.Handle(path, gziphandler.GzipHandler(http.HandlerFunc(
			func(w http.ResponseWriter, r *http.Request) {
				data, etag, lastModified := getDataAndETag()
				w.Header().Set("Etag", etag)

				// ServeContent will take care of caching using eTag.
				http.ServeContent(w, r, path, lastModified, bytes.NewReader(data))
			}),
		))
	}

	return &o, nil
}

func (o *OpenAPIService) getSwaggerBytes() ([]byte, string, time.Time) {
	o.rwMutex.RLock()
	defer o.rwMutex.RUnlock()
	return o.specBytes, o.specBytesETag, o.lastModified
}

func (o *OpenAPIService) getSwaggerPbBytes() ([]byte, string, time.Time) {
	o.rwMutex.RLock()
	defer o.rwMutex.RUnlock()
	return o.specPb, o.specPbETag, o.lastModified
}

func (o *OpenAPIService) getSwaggerPbGzBytes() ([]byte, string, time.Time) {
	o.rwMutex.RLock()
	defer o.rwMutex.RUnlock()
	return o.specPbGz, o.specPbGzETag, o.lastModified
}

func (o *OpenAPIService) UpdateSpec(openapiSpec *spec.Swagger) (err error) {
	orgSpec := openapiSpec
	specBytes, err := json.MarshalIndent(openapiSpec, " ", " ")
	if err != nil {
		return err
	}
	specPb, err := toProtoBinary(specBytes)
	if err != nil {
		return err
	}
	specPbGz := toGzip(specPb)

	specBytesETag := computeETag(specBytes)
	specPbETag := computeETag(specPb)
	specPbGzETag := computeETag(specPbGz)

	lastModified := time.Now()

	o.rwMutex.Lock()
	defer o.rwMutex.Unlock()

	o.orgSpec = orgSpec
	o.specBytes = specBytes
	o.specPb = specPb
	o.specPbGz = specPbGz
	o.specBytesETag = specBytesETag
	o.specPbETag = specPbETag
	o.specPbGzETag = specPbGzETag
	o.lastModified = lastModified

	return nil
}

func toProtoBinary(spec []byte) ([]byte, error) {
	var info yaml.MapSlice
	err := yaml.Unmarshal(spec, &info)
	if err != nil {
		return nil, err
	}
	document, err := openapi_v2.NewDocument(info, compiler.NewContext("$root", nil))
	if err != nil {
		return nil, err
	}
	return proto.Marshal(document)
}

func toGzip(data []byte) []byte {
	var buf bytes.Buffer
	zw := gzip.NewWriter(&buf)
	zw.Write(data)
	zw.Close()
	return buf.Bytes()
}

// RegisterOpenAPIVersionedService registers a handler to provide access to provided swagger spec.
func RegisterOpenAPIVersionedService(openapiSpec *spec.Swagger, servePath string, handler common.PathHandler) (*OpenAPIService, error) {
	o := OpenAPIService{}
	if err := o.UpdateSpec(openapiSpec); err != nil {
		return nil, err
	}

	accepted := []struct {
		Type           string
		SubType        string
		GetDataAndETag func() ([]byte, string, time.Time)
	}{
		{"application", "json", o.getSwaggerBytes},
		{"application", "com.github.proto-openapi.spec.v2@v1.0+protobuf", o.getSwaggerPbBytes},
	}

	handler.Handle(servePath, gziphandler.GzipHandler(http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			decipherableFormats := r.Header.Get("Accept")
			if decipherableFormats == "" {
				decipherableFormats = "*/*"
			}
			clauses := goautoneg.ParseAccept(decipherableFormats)
			w.Header().Add("Vary", "Accept")
			for _, clause := range clauses {
				for _, accepts := range accepted {
					if clause.Type != accepts.Type && clause.Type != "*" {
						continue
					}
					if clause.SubType != accepts.SubType && clause.SubType != "*" {
						continue
					}

					// serve the first matching media type in the sorted clause list
					data, etag, lastModified := accepts.GetDataAndETag()
					w.Header().Set("Etag", etag)
					// ServeContent will take care of caching using eTag.
					http.ServeContent(w, r, servePath, lastModified, bytes.NewReader(data))
					return
				}
			}
			// Return 406 for not acceptable format
			w.WriteHeader(406)
			return
		}),
	))

	return &o, nil
}

// BuildAndRegisterOpenAPIVersionedService builds the spec and registers a handler to provide access to it.
// Use this method if your OpenAPI spec is static. If you want to update the spec, use BuildOpenAPISpec then RegisterOpenAPIVersionedService.
func BuildAndRegisterOpenAPIVersionedService(servePath string, webServices []*restful.WebService, config *common.Config, handler common.PathHandler) (*OpenAPIService, error) {
	spec, err := builder.BuildOpenAPISpec(webServices, config)
	if err != nil {
		return nil, err
	}
	return RegisterOpenAPIVersionedService(spec, servePath, handler)
}
<commit_msg>Remove unnecessarily stored spec<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package handler

import (
	"bytes"
	"compress/gzip"
	"crypto/sha512"
	"encoding/json"
	"fmt"
	"mime"
	"net/http"
	"strings"
	"sync"
	"time"

	"bitbucket.org/ww/goautoneg"

	yaml "gopkg.in/yaml.v2"

	"github.com/NYTimes/gziphandler"
	restful "github.com/emicklei/go-restful"
	"github.com/go-openapi/spec"
	"github.com/golang/protobuf/proto"
	openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2"
	"github.com/googleapis/gnostic/compiler"

	"k8s.io/kube-openapi/pkg/builder"
	"k8s.io/kube-openapi/pkg/common"
)

const (
	jsonExt = ".json"

	mimeJson = "application/json"
	// TODO(mehdy): change @68f4ded to a version tag when gnostic add version tags.
	mimePb   = "application/com.github.googleapis.gnostic.OpenAPIv2@68f4ded+protobuf"
	mimePbGz = "application/x-gzip"
)

// OpenAPIService is the service responsible for serving OpenAPI spec. It has
// the ability to safely change the spec while serving it.
type OpenAPIService struct {
	// rwMutex protects All members of this service.
	rwMutex sync.RWMutex

	lastModified time.Time

	specBytes []byte
	specPb    []byte
	specPbGz  []byte

	specBytesETag string
	specPbETag    string
	specPbGzETag  string
}

func init() {
	mime.AddExtensionType(".json", mimeJson)
	mime.AddExtensionType(".pb-v1", mimePb)
	mime.AddExtensionType(".gz", mimePbGz)
}

func computeETag(data []byte) string {
	return fmt.Sprintf("\"%X\"", sha512.Sum512(data))
}

// NOTE: [DEPRECATION] We will announce deprecation for format-separated endpoints for OpenAPI spec,
// and switch to a single /openapi/v2 endpoint in Kubernetes 1.10. The design doc and deprecation process
// are tracked at: https://docs.google.com/document/d/19lEqE9lc4yHJ3WJAJxS_G7TcORIJXGHyq3wpwcH28nU.
//
// BuildAndRegisterOpenAPIService builds the spec and registers a handler to provide access to it.
// Use this method if your OpenAPI spec is static. If you want to update the spec, use BuildOpenAPISpec then RegisterOpenAPIService.
func BuildAndRegisterOpenAPIService(servePath string, webServices []*restful.WebService, config *common.Config, handler common.PathHandler) (*OpenAPIService, error) {
	spec, err := builder.BuildOpenAPISpec(webServices, config)
	if err != nil {
		return nil, err
	}
	return RegisterOpenAPIService(spec, servePath, handler)
}

// NOTE: [DEPRECATION] We will announce deprecation for format-separated endpoints for OpenAPI spec,
// and switch to a single /openapi/v2 endpoint in Kubernetes 1.10. The design doc and deprecation process
// are tracked at: https://docs.google.com/document/d/19lEqE9lc4yHJ3WJAJxS_G7TcORIJXGHyq3wpwcH28nU.
//
// RegisterOpenAPIService registers a handler to provide access to provided swagger spec.
// Note: servePath should end with ".json" as the RegisterOpenAPIService assume it is serving a
// json file and will also serve .pb and .gz files.
func RegisterOpenAPIService(openapiSpec *spec.Swagger, servePath string, handler common.PathHandler) (*OpenAPIService, error) {
	if !strings.HasSuffix(servePath, jsonExt) {
		return nil, fmt.Errorf("serving path must end with \"%s\"", jsonExt)
	}

	servePathBase := strings.TrimSuffix(servePath, jsonExt)

	o := OpenAPIService{}
	if err := o.UpdateSpec(openapiSpec); err != nil {
		return nil, err
	}

	type fileInfo struct {
		ext            string
		getDataAndETag func() ([]byte, string, time.Time)
	}

	files := []fileInfo{
		{".json", o.getSwaggerBytes},
		{"-2.0.0.json", o.getSwaggerBytes},
		{"-2.0.0.pb-v1", o.getSwaggerPbBytes},
		{"-2.0.0.pb-v1.gz", o.getSwaggerPbGzBytes},
	}

	for _, file := range files {
		path := servePathBase + file.ext
		getDataAndETag := file.getDataAndETag
		handler.Handle(path, gziphandler.GzipHandler(http.HandlerFunc(
			func(w http.ResponseWriter, r *http.Request) {
				data, etag, lastModified := getDataAndETag()
				w.Header().Set("Etag", etag)

				// ServeContent will take care of caching using eTag.
				http.ServeContent(w, r, path, lastModified, bytes.NewReader(data))
			}),
		))
	}

	return &o, nil
}

func (o *OpenAPIService) getSwaggerBytes() ([]byte, string, time.Time) {
	o.rwMutex.RLock()
	defer o.rwMutex.RUnlock()
	return o.specBytes, o.specBytesETag, o.lastModified
}

func (o *OpenAPIService) getSwaggerPbBytes() ([]byte, string, time.Time) {
	o.rwMutex.RLock()
	defer o.rwMutex.RUnlock()
	return o.specPb, o.specPbETag, o.lastModified
}

func (o *OpenAPIService) getSwaggerPbGzBytes() ([]byte, string, time.Time) {
	o.rwMutex.RLock()
	defer o.rwMutex.RUnlock()
	return o.specPbGz, o.specPbGzETag, o.lastModified
}

func (o *OpenAPIService) UpdateSpec(openapiSpec *spec.Swagger) (err error) {
	specBytes, err := json.MarshalIndent(openapiSpec, " ", " ")
	if err != nil {
		return err
	}
	specPb, err := toProtoBinary(specBytes)
	if err != nil {
		return err
	}
	specPbGz := toGzip(specPb)

	specBytesETag := computeETag(specBytes)
	specPbETag := computeETag(specPb)
	specPbGzETag := computeETag(specPbGz)

	lastModified := time.Now()

	o.rwMutex.Lock()
	defer o.rwMutex.Unlock()

	o.specBytes = specBytes
	o.specPb = specPb
	o.specPbGz = specPbGz
	o.specBytesETag = specBytesETag
	o.specPbETag = specPbETag
	o.specPbGzETag = specPbGzETag
	o.lastModified = lastModified

	return nil
}

func toProtoBinary(spec []byte) ([]byte, error) {
	var info yaml.MapSlice
	err := yaml.Unmarshal(spec, &info)
	if err != nil {
		return nil, err
	}
	document, err := openapi_v2.NewDocument(info, compiler.NewContext("$root", nil))
	if err != nil {
		return nil, err
	}
	return proto.Marshal(document)
}

func toGzip(data []byte) []byte {
	var buf bytes.Buffer
	zw := gzip.NewWriter(&buf)
	zw.Write(data)
	zw.Close()
	return buf.Bytes()
}

// RegisterOpenAPIVersionedService registers a handler to provide access to provided swagger spec.
func RegisterOpenAPIVersionedService(openapiSpec *spec.Swagger, servePath string, handler common.PathHandler) (*OpenAPIService, error) {
	o := OpenAPIService{}
	if err := o.UpdateSpec(openapiSpec); err != nil {
		return nil, err
	}

	accepted := []struct {
		Type           string
		SubType        string
		GetDataAndETag func() ([]byte, string, time.Time)
	}{
		{"application", "json", o.getSwaggerBytes},
		{"application", "com.github.proto-openapi.spec.v2@v1.0+protobuf", o.getSwaggerPbBytes},
	}

	handler.Handle(servePath, gziphandler.GzipHandler(http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			decipherableFormats := r.Header.Get("Accept")
			if decipherableFormats == "" {
				decipherableFormats = "*/*"
			}
			clauses := goautoneg.ParseAccept(decipherableFormats)
			w.Header().Add("Vary", "Accept")
			for _, clause := range clauses {
				for _, accepts := range accepted {
					if clause.Type != accepts.Type && clause.Type != "*" {
						continue
					}
					if clause.SubType != accepts.SubType && clause.SubType != "*" {
						continue
					}

					// serve the first matching media type in the sorted clause list
					data, etag, lastModified := accepts.GetDataAndETag()
					w.Header().Set("Etag", etag)
					// ServeContent will take care of caching using eTag.
					http.ServeContent(w, r, servePath, lastModified, bytes.NewReader(data))
					return
				}
			}
			// Return 406 for not acceptable format
			w.WriteHeader(406)
			return
		}),
	))

	return &o, nil
}

// BuildAndRegisterOpenAPIVersionedService builds the spec and registers a handler to provide access to it.
// Use this method if your OpenAPI spec is static. If you want to update the spec, use BuildOpenAPISpec then RegisterOpenAPIVersionedService.
func BuildAndRegisterOpenAPIVersionedService(servePath string, webServices []*restful.WebService, config *common.Config, handler common.PathHandler) (*OpenAPIService, error) {
	spec, err := builder.BuildOpenAPISpec(webServices, config)
	if err != nil {
		return nil, err
	}
	return RegisterOpenAPIVersionedService(spec, servePath, handler)
}
<|endoftext|>


<commit_before>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logging

import (
	"fmt"
	"log/syslog"
	"net"
	"os"
	"regexp"
	"strconv"
	"time"

	"github.com/evalphobia/logrus_fluent"
	"github.com/sirupsen/logrus"
	logrus_syslog "github.com/sirupsen/logrus/hooks/syslog"
)

const (
	fAddr  = "fluentd.address"
	fTag   = "fluentd.tag"
	fLevel = "fluentd.level"

	SLevel = "syslog.level"

	lAddr     = "logstash.address"
	lLevel    = "logstash.level"
	lProtocol = "logstash.protocol"

	Syslog   = "syslog"
	Fluentd  = "fluentd"
	Logstash = "logstash"
)

var (
	// DefaultLogger is the base logrus logger. It is different from the logrus
	// default to avoid external dependencies from writing out unexpectedly
	DefaultLogger = InitializeDefaultLogger()

	// DefaultLogLevel is the alternative we provide to Debug
	DefaultLogLevel = logrus.InfoLevel

	// syslogOpts is the set of supported options for syslog configuration.
	syslogOpts = map[string]bool{
		"syslog.level": true,
	}

	// fluentDOpts is the set of supported options for fluentD configuration.
	fluentDOpts = map[string]bool{
		fAddr:  true,
		fTag:   true,
		fLevel: true,
	}

	// logstashOpts is the set of supported options for logstash configuration.
	logstashOpts = map[string]bool{
		lAddr:     true,
		lLevel:    true,
		lProtocol: true,
	}

	// syslogLevelMap maps logrus.Level values to syslog.Priority levels.
	syslogLevelMap = map[logrus.Level]syslog.Priority{
		logrus.PanicLevel: syslog.LOG_ALERT,
		logrus.FatalLevel: syslog.LOG_CRIT,
		logrus.ErrorLevel: syslog.LOG_ERR,
		logrus.WarnLevel:  syslog.LOG_WARNING,
		logrus.InfoLevel:  syslog.LOG_INFO,
		logrus.DebugLevel: syslog.LOG_DEBUG,
	}
)

// setFireLevels returns a slice of logrus.Level values higher in priority
// and including level, excluding any levels lower in priority.
func setFireLevels(level logrus.Level) []logrus.Level {
	switch level {
	case logrus.PanicLevel:
		return []logrus.Level{logrus.PanicLevel}
	case logrus.FatalLevel:
		return []logrus.Level{logrus.PanicLevel, logrus.FatalLevel}
	case logrus.ErrorLevel:
		return []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel}
	case logrus.WarnLevel:
		return []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel, logrus.WarnLevel}
	case logrus.InfoLevel:
		return []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel, logrus.WarnLevel, logrus.InfoLevel}
	case logrus.DebugLevel:
		return []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel, logrus.WarnLevel, logrus.InfoLevel, logrus.DebugLevel}
	default:
		logrus.Infof("logrus level %v is not supported at this time; defaulting to info level", level)
		return []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel, logrus.WarnLevel, logrus.InfoLevel}
	}
}

// InitializeDefaultLogger returns a logrus Logger with a custom text formatter.
func InitializeDefaultLogger() *logrus.Logger {
	logger := logrus.New()
	logger.Formatter = setupFormatter()
	return logger
}

// SetupLogging sets up each logging service provided in loggers and configures
// each logger with the provided logOpts.
func SetupLogging(loggers []string, logOpts map[string]string, tag string, debug bool) error {
	// Set default logger to output to stdout if no loggers are provided.
	if len(loggers) == 0 {
		// TODO: switch to a per-logger version when we upgrade to logrus >1.0.3
		logrus.SetOutput(os.Stdout)
	}

	SetLogLevel(DefaultLogLevel)
	ToggleDebugLogs(debug)

	// always suppress the default logger so libraries don't print things
	logrus.SetLevel(logrus.PanicLevel)

	// Iterate through all provided loggers and configure them according
	// to user-provided settings.
	for _, logger := range loggers {
		valuesToValidate := getLogDriverConfig(logger, logOpts)
		switch logger {
		case Syslog:
			valuesToValidate := getLogDriverConfig(Syslog, logOpts)
			err := validateOpts(Syslog, valuesToValidate, syslogOpts)
			if err != nil {
				return err
			}
			setupSyslog(valuesToValidate, tag, debug)
		case Fluentd:
			err := validateOpts(logger, valuesToValidate, fluentDOpts)
			if err != nil {
				return err
			}
			setupFluentD(valuesToValidate, debug)
			//TODO - need to finish logstash integration.
		/*case Logstash:
		fmt.Printf("SetupLogging: in logstash case\n")
		err := validateOpts(logger, valuesToValidate, logstashOpts)
		fmt.Printf("SetupLogging: validating options for logstash complete\n")
		if err != nil {
			fmt.Printf("SetupLogging: error validating logstash opts %v\n", err.Error())
			return err
		}
		fmt.Printf("SetupLogging: about to setup logstash\n")
		setupLogstash(valuesToValidate)
		*/
		default:
			return fmt.Errorf("provided log driver %q is not a supported log driver", logger)
		}
	}

	return nil
}

// SetLogLevel sets the log level on DefaultLogger. This logger is, by
// convention, the base logger for package specific ones thus setting the level
// here impacts the default logging behaviour.
// This function is thread-safe when logging, reading DefaultLogger.Level is
// not protected this way, however.
func SetLogLevel(level logrus.Level) {
	DefaultLogger.SetLevel(level)
}

// ToggleDebugLogs switches on or off debugging logs. It will select
// DefaultLogLevel when turning debug off.
// It is thread-safe.
func ToggleDebugLogs(debug bool) {
	if debug {
		SetLogLevel(logrus.DebugLevel)
	} else {
		SetLogLevel(DefaultLogLevel)
	}
}

// setupSyslog sets up and configures syslog with the provided options in
// logOpts. If some options are not provided, sensible defaults are used.
func setupSyslog(logOpts map[string]string, tag string, debug bool) {
	logLevel, ok := logOpts[SLevel]
	if !ok {
		if debug {
			logLevel = "debug"
		} else {
			logLevel = "info"
		}
	}

	//Validate provided log level.
	level, err := logrus.ParseLevel(logLevel)
	if err != nil {
		DefaultLogger.Fatal(err)
	}

	DefaultLogger.SetLevel(level)
	// Create syslog hook.
	h, err := logrus_syslog.NewSyslogHook("", "", syslogLevelMap[level], tag)
	if err != nil {
		DefaultLogger.Fatal(err)
	}
	// TODO: switch to a per-logger version when we upgrade to logrus >1.0.3
	logrus.AddHook(h)
}

// setupFormatter sets up the text formatting for logs output by logrus.
func setupFormatter() logrus.Formatter {
	fileFormat := new(logrus.TextFormatter)
	fileFormat.DisableTimestamp = true
	fileFormat.DisableColors = true
	switch os.Getenv("INITSYSTEM") {
	case "SYSTEMD":
		fileFormat.FullTimestamp = true
	default:
		fileFormat.TimestampFormat = time.RFC3339
	}

	// TODO: switch to a per-logger version when we upgrade to logrus >1.0.3
	return fileFormat
}

// setupFluentD sets up and configures FluentD with the provided options in
// logOpts. If some options are not provided, sensible defaults are used.
func setupFluentD(logOpts map[string]string, debug bool) {
	//If no logging level set for fluentd, use debug value if it is set.
	// Logging level set for fluentd takes precedence over debug flag
	// fluent.level provided.
	logLevel, ok := logOpts[fLevel]
	if !ok {
		if debug {
			logLevel = "debug"
		} else {
			logLevel = "info"
		}
	}
	level, err := logrus.ParseLevel(logLevel)
	if err != nil {
		DefaultLogger.Fatal(err)
	}

	hostAndPort, ok := logOpts[fAddr]
	if !ok {
		hostAndPort = "localhost:24224"
	}

	host, strPort, err := net.SplitHostPort(hostAndPort)
	if err != nil {
		DefaultLogger.Fatal(err)
	}
	port, err := strconv.Atoi(strPort)
	if err != nil {
		DefaultLogger.Fatal(err)
	}

	h, err := logrus_fluent.New(host, port)
	if err != nil {
		DefaultLogger.Fatal(err)
	}

	tag, ok := logOpts[fTag]
	if ok {
		h.SetTag(tag)
	}

	// set custom fire level
	h.SetLevels(setFireLevels(level))
	// TODO: switch to a per-logger version when we upgrade to logrus >1.0.3
	logrus.AddHook(h)
}

// setupLogstash sets up and configures Logstash with the provided options in
// logOpts. If some options are not provided, sensible defaults are used.
/// FIXME GH-1578 - needs to be tested with a working logstash setup.
//func setupLogstash(logOpts map[string]string) {
//	hostAndPort, ok := logOpts[lAddr]
//	if !ok {
//		hostAndPort = "172.17.0.2:999"
//	}
//
//	protocol, ok := logOpts[lProtocol]
//	if !ok {
//		protocol = "tcp"
//	}
//
//	h, err := logrustash.NewHook(protocol, hostAndPort, "cilium")
//	if err != nil {
//		DefaultLogger.Fatal(err)
//	}
//
//	DefaultLogger.AddHook(h)
//}

// validateOpts iterates through all of the keys in logOpts, and errors out if
// the key in logOpts is not a key in supportedOpts.
func validateOpts(logDriver string, logOpts map[string]string, supportedOpts map[string]bool) error {
	for k := range logOpts {
		if !supportedOpts[k] {
			return fmt.Errorf("provided configuration value %q is not supported as a logging option for log driver %s", k, logDriver)
		}
	}
	return nil
}

// getLogDriverConfig returns a map containing the key-value pairs that start
// with string logDriver from map logOpts.
func getLogDriverConfig(logDriver string, logOpts map[string]string) map[string]string {
	keysToValidate := make(map[string]string)
	for k, v := range logOpts {
		ok, err := regexp.MatchString(logDriver+".*", k)
		if err != nil {
			DefaultLogger.Fatal(err)
		}
		if ok {
			keysToValidate[k] = v
		}
	}
	return keysToValidate
}
<commit_msg>Revert "pkg/logging: disable timestamp by default"<commit_after>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logging

import (
	"fmt"
	"log/syslog"
	"net"
	"os"
	"regexp"
	"strconv"
	"time"

	"github.com/evalphobia/logrus_fluent"
	"github.com/sirupsen/logrus"
	logrus_syslog "github.com/sirupsen/logrus/hooks/syslog"
)

const (
	fAddr  = "fluentd.address"
	fTag   = "fluentd.tag"
	fLevel = "fluentd.level"

	SLevel = "syslog.level"

	lAddr     = "logstash.address"
	lLevel    = "logstash.level"
	lProtocol = "logstash.protocol"

	Syslog   = "syslog"
	Fluentd  = "fluentd"
	Logstash = "logstash"
)

var (
	// DefaultLogger is the base logrus logger. It is different from the logrus
	// default to avoid external dependencies from writing out unexpectedly
	DefaultLogger = InitializeDefaultLogger()

	// DefaultLogLevel is the alternative we provide to Debug
	DefaultLogLevel = logrus.InfoLevel

	// syslogOpts is the set of supported options for syslog configuration.
	syslogOpts = map[string]bool{
		"syslog.level": true,
	}

	// fluentDOpts is the set of supported options for fluentD configuration.
	fluentDOpts = map[string]bool{
		fAddr:  true,
		fTag:   true,
		fLevel: true,
	}

	// logstashOpts is the set of supported options for logstash configuration.
	logstashOpts = map[string]bool{
		lAddr:     true,
		lLevel:    true,
		lProtocol: true,
	}

	// syslogLevelMap maps logrus.Level values to syslog.Priority levels.
	syslogLevelMap = map[logrus.Level]syslog.Priority{
		logrus.PanicLevel: syslog.LOG_ALERT,
		logrus.FatalLevel: syslog.LOG_CRIT,
		logrus.ErrorLevel: syslog.LOG_ERR,
		logrus.WarnLevel:  syslog.LOG_WARNING,
		logrus.InfoLevel:  syslog.LOG_INFO,
		logrus.DebugLevel: syslog.LOG_DEBUG,
	}
)

// setFireLevels returns a slice of logrus.Level values higher in priority
// and including level, excluding any levels lower in priority.
func setFireLevels(level logrus.Level) []logrus.Level {
	switch level {
	case logrus.PanicLevel:
		return []logrus.Level{logrus.PanicLevel}
	case logrus.FatalLevel:
		return []logrus.Level{logrus.PanicLevel, logrus.FatalLevel}
	case logrus.ErrorLevel:
		return []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel}
	case logrus.WarnLevel:
		return []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel, logrus.WarnLevel}
	case logrus.InfoLevel:
		return []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel, logrus.WarnLevel, logrus.InfoLevel}
	case logrus.DebugLevel:
		return []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel, logrus.WarnLevel, logrus.InfoLevel, logrus.DebugLevel}
	default:
		logrus.Infof("logrus level %v is not supported at this time; defaulting to info level", level)
		return []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel, logrus.WarnLevel, logrus.InfoLevel}
	}
}

// InitializeDefaultLogger returns a logrus Logger with a custom text formatter.
func InitializeDefaultLogger() *logrus.Logger {
	logger := logrus.New()
	logger.Formatter = setupFormatter()
	return logger
}

// SetupLogging sets up each logging service provided in loggers and configures
// each logger with the provided logOpts.
func SetupLogging(loggers []string, logOpts map[string]string, tag string, debug bool) error {
	// Set default logger to output to stdout if no loggers are provided.
	if len(loggers) == 0 {
		// TODO: switch to a per-logger version when we upgrade to logrus >1.0.3
		logrus.SetOutput(os.Stdout)
	}

	SetLogLevel(DefaultLogLevel)
	ToggleDebugLogs(debug)

	// always suppress the default logger so libraries don't print things
	logrus.SetLevel(logrus.PanicLevel)

	// Iterate through all provided loggers and configure them according
	// to user-provided settings.
	for _, logger := range loggers {
		valuesToValidate := getLogDriverConfig(logger, logOpts)
		switch logger {
		case Syslog:
			valuesToValidate := getLogDriverConfig(Syslog, logOpts)
			err := validateOpts(Syslog, valuesToValidate, syslogOpts)
			if err != nil {
				return err
			}
			setupSyslog(valuesToValidate, tag, debug)
		case Fluentd:
			err := validateOpts(logger, valuesToValidate, fluentDOpts)
			if err != nil {
				return err
			}
			setupFluentD(valuesToValidate, debug)
			//TODO - need to finish logstash integration.
		/*case Logstash:
		fmt.Printf("SetupLogging: in logstash case\n")
		err := validateOpts(logger, valuesToValidate, logstashOpts)
		fmt.Printf("SetupLogging: validating options for logstash complete\n")
		if err != nil {
			fmt.Printf("SetupLogging: error validating logstash opts %v\n", err.Error())
			return err
		}
		fmt.Printf("SetupLogging: about to setup logstash\n")
		setupLogstash(valuesToValidate)
		*/
		default:
			return fmt.Errorf("provided log driver %q is not a supported log driver", logger)
		}
	}

	return nil
}

// SetLogLevel sets the log level on DefaultLogger. This logger is, by
// convention, the base logger for package specific ones thus setting the level
// here impacts the default logging behaviour.
// This function is thread-safe when logging, reading DefaultLogger.Level is
// not protected this way, however.
func SetLogLevel(level logrus.Level) {
	DefaultLogger.SetLevel(level)
}

// ToggleDebugLogs switches on or off debugging logs. It will select
// DefaultLogLevel when turning debug off.
// It is thread-safe.
func ToggleDebugLogs(debug bool) {
	if debug {
		SetLogLevel(logrus.DebugLevel)
	} else {
		SetLogLevel(DefaultLogLevel)
	}
}

// setupSyslog sets up and configures syslog with the provided options in
// logOpts. If some options are not provided, sensible defaults are used.
func setupSyslog(logOpts map[string]string, tag string, debug bool) {
	logLevel, ok := logOpts[SLevel]
	if !ok {
		if debug {
			logLevel = "debug"
		} else {
			logLevel = "info"
		}
	}

	//Validate provided log level.
	level, err := logrus.ParseLevel(logLevel)
	if err != nil {
		DefaultLogger.Fatal(err)
	}

	DefaultLogger.SetLevel(level)
	// Create syslog hook.
	h, err := logrus_syslog.NewSyslogHook("", "", syslogLevelMap[level], tag)
	if err != nil {
		DefaultLogger.Fatal(err)
	}
	// TODO: switch to a per-logger version when we upgrade to logrus >1.0.3
	logrus.AddHook(h)
}

// setupFormatter sets up the text formatting for logs output by logrus.
func setupFormatter() logrus.Formatter {
	fileFormat := new(logrus.TextFormatter)
	fileFormat.DisableColors = true
	switch os.Getenv("INITSYSTEM") {
	case "SYSTEMD":
		fileFormat.DisableTimestamp = true
		fileFormat.FullTimestamp = true
	default:
		fileFormat.TimestampFormat = time.RFC3339
	}

	// TODO: switch to a per-logger version when we upgrade to logrus >1.0.3
	return fileFormat
}

// setupFluentD sets up and configures FluentD with the provided options in
// logOpts. If some options are not provided, sensible defaults are used.
func setupFluentD(logOpts map[string]string, debug bool) {
	//If no logging level set for fluentd, use debug value if it is set.
	// Logging level set for fluentd takes precedence over debug flag
	// fluent.level provided.
	logLevel, ok := logOpts[fLevel]
	if !ok {
		if debug {
			logLevel = "debug"
		} else {
			logLevel = "info"
		}
	}
	level, err := logrus.ParseLevel(logLevel)
	if err != nil {
		DefaultLogger.Fatal(err)
	}

	hostAndPort, ok := logOpts[fAddr]
	if !ok {
		hostAndPort = "localhost:24224"
	}

	host, strPort, err := net.SplitHostPort(hostAndPort)
	if err != nil {
		DefaultLogger.Fatal(err)
	}
	port, err := strconv.Atoi(strPort)
	if err != nil {
		DefaultLogger.Fatal(err)
	}

	h, err := logrus_fluent.New(host, port)
	if err != nil {
		DefaultLogger.Fatal(err)
	}

	tag, ok := logOpts[fTag]
	if ok {
		h.SetTag(tag)
	}

	// set custom fire level
	h.SetLevels(setFireLevels(level))
	// TODO: switch to a per-logger version when we upgrade to logrus >1.0.3
	logrus.AddHook(h)
}

// setupLogstash sets up and configures Logstash with the provided options in
// logOpts. If some options are not provided, sensible defaults are used.
/// FIXME GH-1578 - needs to be tested with a working logstash setup.
//func setupLogstash(logOpts map[string]string) {
//	hostAndPort, ok := logOpts[lAddr]
//	if !ok {
//		hostAndPort = "172.17.0.2:999"
//	}
//
//	protocol, ok := logOpts[lProtocol]
//	if !ok {
//		protocol = "tcp"
//	}
//
//	h, err := logrustash.NewHook(protocol, hostAndPort, "cilium")
//	if err != nil {
//		DefaultLogger.Fatal(err)
//	}
//
//	DefaultLogger.AddHook(h)
//}

// validateOpts iterates through all of the keys in logOpts, and errors out if
// the key in logOpts is not a key in supportedOpts.
func validateOpts(logDriver string, logOpts map[string]string, supportedOpts map[string]bool) error {
	for k := range logOpts {
		if !supportedOpts[k] {
			return fmt.Errorf("provided configuration value %q is not supported as a logging option for log driver %s", k, logDriver)
		}
	}
	return nil
}

// getLogDriverConfig returns a map containing the key-value pairs that start
// with string logDriver from map logOpts.
func getLogDriverConfig(logDriver string, logOpts map[string]string) map[string]string {
	keysToValidate := make(map[string]string)
	for k, v := range logOpts {
		ok, err := regexp.MatchString(logDriver+".*", k)
		if err != nil {
			DefaultLogger.Fatal(err)
		}
		if ok {
			keysToValidate[k] = v
		}
	}
	return keysToValidate
}
<|endoftext|>


<commit_before>package project

var (
	description string = "The azure-operator manages Kubernetes clusters on Azure."
	gitSHA             = "n/a"
	name        string = "azure-operator"
	source      string = "https://github.com/giantswarm/azure-operator"
	version            = "5.9.0"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<commit_msg>Bump version to 5.9.1-dev (#1531)<commit_after>package project

var (
	description string = "The azure-operator manages Kubernetes clusters on Azure."
	gitSHA             = "n/a"
	name        string = "azure-operator"
	source      string = "https://github.com/giantswarm/azure-operator"
	version            = "5.9.1-dev"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<|endoftext|>


<commit_before>package project

var (
	description        = "The aws-operator manages Kubernetes clusters running on AWS."
	gitSHA             = "n/a"
	name        string = "aws-operator"
	source      string = "https://github.com/giantswarm/aws-operator"
	version            = "9.1.2-dev-calvix"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<commit_msg>revert version name (#2791)<commit_after>package project

var (
	description        = "The aws-operator manages Kubernetes clusters running on AWS."
	gitSHA             = "n/a"
	name        string = "aws-operator"
	source      string = "https://github.com/giantswarm/aws-operator"
	version            = "9.1.2-dev"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<|endoftext|>


<commit_before>// Copyright 2019 the Service Broker Project Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"sort"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/gcp-service-broker/pkg/broker"
	"github.com/GoogleCloudPlatform/gcp-service-broker/pkg/client"
)

func GetAllCompleteServiceExamples(registry broker.BrokerRegistry) ([]client.CompleteServiceExample, error) {

	var allExamples []client.CompleteServiceExample

	services := registry.GetAllServices()

	for _, service := range services {

		serviceExamples, err := client.GetExamplesForAService(service)

		if err != nil {
			return nil, err
		}

		allExamples = append(allExamples, serviceExamples...)
	}

	// Sort by ServiceName and ExampleName so there's a consistent order in the UI and tests.
	sort.Slice(allExamples, func(i int, j int) bool {
		if strings.Compare(allExamples[i].ServiceName, allExamples[j].ServiceName) != 0 {
			return allExamples[i].ServiceName < allExamples[j].ServiceName
		} else {
			return allExamples[i].ServiceExample.Name < allExamples[j].ServiceExample.Name
		}
	})

	return allExamples, nil
}

func GetExamplesFromServer() []client.CompleteServiceExample {

	var allExamples []client.CompleteServiceExample
	url := "http://localhost:8000/examples"

	serverClient := http.Client{
		Timeout: time.Second * 2,
	}

	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		log.Fatal(err)
	}

	resp, err := serverClient.Do(req)
	if err != nil {
		log.Fatal(err)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	err = json.Unmarshal(body, &allExamples)
	if err != nil {
		log.Fatal(err)
	}

	return allExamples
}

func NewExampleHandler(registry broker.BrokerRegistry) http.HandlerFunc {
	allExamples, err := GetAllCompleteServiceExamples(registry)

	if err != nil {
		return func(w http.ResponseWriter, rep *http.Request) {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	}

	exampleJSON, err := json.Marshal(allExamples)

	if err != nil {
		return func(w http.ResponseWriter, rep *http.Request) {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	}

	return func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)
		w.Write(exampleJSON)
	}
}
<commit_msg>Use Viper to get hostname and port variables<commit_after>// Copyright 2019 the Service Broker Project Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"sort"
	"strings"
	"time"

	"github.com/spf13/viper"
	"github.com/GoogleCloudPlatform/gcp-service-broker/pkg/broker"
	"github.com/GoogleCloudPlatform/gcp-service-broker/pkg/client"
)

func GetAllCompleteServiceExamples(registry broker.BrokerRegistry) ([]client.CompleteServiceExample, error) {

	var allExamples []client.CompleteServiceExample

	services := registry.GetAllServices()

	for _, service := range services {

		serviceExamples, err := client.GetExamplesForAService(service)

		if err != nil {
			return nil, err
		}

		allExamples = append(allExamples, serviceExamples...)
	}

	// Sort by ServiceName and ExampleName so there's a consistent order in the UI and tests.
	sort.Slice(allExamples, func(i int, j int) bool {
		if strings.Compare(allExamples[i].ServiceName, allExamples[j].ServiceName) != 0 {
			return allExamples[i].ServiceName < allExamples[j].ServiceName
		} else {
			return allExamples[i].ServiceExample.Name < allExamples[j].ServiceExample.Name
		}
	})

	return allExamples, nil
}

func GetExamplesFromServer() []client.CompleteServiceExample {

	var allExamples []client.CompleteServiceExample
	url := fmt.Sprintf("http://%s:%d/examples", viper.GetString("api.hostname"), viper.GetInt("api.port"))

	serverClient := http.Client{
		Timeout: time.Second * 2,
	}

	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		log.Fatal(err)
	}

	resp, err := serverClient.Do(req)
	if err != nil {
		log.Fatal(err)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	err = json.Unmarshal(body, &allExamples)
	if err != nil {
		log.Fatal(err)
	}

	return allExamples
}

func NewExampleHandler(registry broker.BrokerRegistry) http.HandlerFunc {
	allExamples, err := GetAllCompleteServiceExamples(registry)

	if err != nil {
		return func(w http.ResponseWriter, rep *http.Request) {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	}

	exampleJSON, err := json.Marshal(allExamples)

	if err != nil {
		return func(w http.ResponseWriter, rep *http.Request) {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	}

	return func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)
		w.Write(exampleJSON)
	}
}
<|endoftext|>


<commit_before>package structs

// Temporary struct storing custom claims until JWT creation.
type CustomClaims struct {
	Claims map[string]interface{}
}

// UserI each *User struct must prepare the data for being placed in the JWT
type UserI interface {
	PrepareUserData()
}

// User is inherited.
type User struct {
	// TODO: set Provider here so that we can pass it to db
	// populated by db (via mapstructure) or from provider (via json)
	// Provider   string `json:"provider",mapstructure:"provider"`
	Username   string `json:"username" mapstructure:"username"`
	Name       string `json:"name" mapstructure:"name"`
	Email      string `json:"email" mapstructure:"email"`
	CreatedOn  int64  `json:"createdon"`
	LastUpdate int64  `json:"lastupdate"`
	ID         int    `json:"id" mapstructure:"id"`
	// jwt.StandardClaims
}

// PrepareUserData implement PersonalData interface
func (u *User) PrepareUserData() {
	u.Username = u.Email
}

// GoogleUser is a retrieved and authentiacted user from Google.
// unused!
// TODO: see if these should be pointers to the *User object as per
// https://golang.org/doc/effective_go.html#embedding
type GoogleUser struct {
	User
	Sub           string `json:"sub"`
	GivenName     string `json:"given_name"`
	FamilyName    string `json:"family_name"`
	Profile       string `json:"profile"`
	Picture       string `json:"picture"`
	EmailVerified bool   `json:"email_verified"`
	Gender        string `json:"gender"`
	HostDomain    string `json:"hd"`
	// jwt.StandardClaims
}

// PrepareUserData implement PersonalData interface
func (u *GoogleUser) PrepareUserData() {
	u.Username = u.Email
}

// ADFSUser Active Directory user record
type ADFSUser struct {
	User
	Sub string `json:"sub"`
	UPN string `json:"upn"`
	// UniqueName string `json:"unique_name"`
	// PwdExp     string `json:"pwd_exp"`
	// SID        string `json:"sid"`
	// Groups     string `json:"groups"`
	// jwt.StandardClaims
}

// PrepareUserData implement PersonalData interface
func (u *ADFSUser) PrepareUserData() {
	u.Username = u.UPN
}

// GitHubUser is a retrieved and authentiacted user from GitHub.
type GitHubUser struct {
	User
	Login   string `json:"login"`
	Picture string `json:"avatar_url"`
	// jwt.StandardClaims
}

// PrepareUserData implement PersonalData interface
func (u *GitHubUser) PrepareUserData() {
	// always use the u.Login as the u.Username
	u.Username = u.Login
}

// IndieAuthUser see indieauth.net
type IndieAuthUser struct {
	User
	URL string `json:"me"`
}

// PrepareUserData implement PersonalData interface
func (u *IndieAuthUser) PrepareUserData() {
	u.Username = u.URL
}

// Team has members and provides acess to sites
type Team struct {
	Name       string   `json:"name" mapstructure:"name"`
	Members    []string `json:"members" mapstructure:"members"` // just the emails
	Sites      []string `json:"sites" mapstructure:"sites"`     // just the domains
	CreatedOn  int64    `json:"createdon" mapstructure:"createdon"`
	LastUpdate int64    `json:"lastupdate" mapstructure:"lastupdate"`
	ID         int      `json:"id" mapstructure:"id"`
}

// Site is the basic unit of auth
type Site struct {
	Domain     string `json:"domain"`
	CreatedOn  int64  `json:"createdon"`
	LastUpdate int64  `json:"lastupdate"`
	ID         int    `json:"id" mapstructure:"id"`
}

type PTokens struct {
	PAccessToken string
	PIdToken     string
}
<commit_msg>don't clobber username if present<commit_after>package structs

// Temporary struct storing custom claims until JWT creation.
type CustomClaims struct {
	Claims map[string]interface{}
}

// UserI each *User struct must prepare the data for being placed in the JWT
type UserI interface {
	PrepareUserData()
}

// User is inherited.
type User struct {
	// TODO: set Provider here so that we can pass it to db
	// populated by db (via mapstructure) or from provider (via json)
	// Provider   string `json:"provider",mapstructure:"provider"`
	Username   string `json:"username" mapstructure:"username"`
	Name       string `json:"name" mapstructure:"name"`
	Email      string `json:"email" mapstructure:"email"`
	CreatedOn  int64  `json:"createdon"`
	LastUpdate int64  `json:"lastupdate"`
	ID         int    `json:"id" mapstructure:"id"`
	// jwt.StandardClaims
}

// PrepareUserData implement PersonalData interface
func (u *User) PrepareUserData() {
	if u.Username == "" {
	    u.Username = u.Email
    }
}

// GoogleUser is a retrieved and authentiacted user from Google.
// unused!
// TODO: see if these should be pointers to the *User object as per
// https://golang.org/doc/effective_go.html#embedding
type GoogleUser struct {
	User
	Sub           string `json:"sub"`
	GivenName     string `json:"given_name"`
	FamilyName    string `json:"family_name"`
	Profile       string `json:"profile"`
	Picture       string `json:"picture"`
	EmailVerified bool   `json:"email_verified"`
	Gender        string `json:"gender"`
	HostDomain    string `json:"hd"`
	// jwt.StandardClaims
}

// PrepareUserData implement PersonalData interface
func (u *GoogleUser) PrepareUserData() {
	u.Username = u.Email
}

// ADFSUser Active Directory user record
type ADFSUser struct {
	User
	Sub string `json:"sub"`
	UPN string `json:"upn"`
	// UniqueName string `json:"unique_name"`
	// PwdExp     string `json:"pwd_exp"`
	// SID        string `json:"sid"`
	// Groups     string `json:"groups"`
	// jwt.StandardClaims
}

// PrepareUserData implement PersonalData interface
func (u *ADFSUser) PrepareUserData() {
	u.Username = u.UPN
}

// GitHubUser is a retrieved and authentiacted user from GitHub.
type GitHubUser struct {
	User
	Login   string `json:"login"`
	Picture string `json:"avatar_url"`
	// jwt.StandardClaims
}

// PrepareUserData implement PersonalData interface
func (u *GitHubUser) PrepareUserData() {
	// always use the u.Login as the u.Username
	u.Username = u.Login
}

// IndieAuthUser see indieauth.net
type IndieAuthUser struct {
	User
	URL string `json:"me"`
}

// PrepareUserData implement PersonalData interface
func (u *IndieAuthUser) PrepareUserData() {
	u.Username = u.URL
}

// Team has members and provides acess to sites
type Team struct {
	Name       string   `json:"name" mapstructure:"name"`
	Members    []string `json:"members" mapstructure:"members"` // just the emails
	Sites      []string `json:"sites" mapstructure:"sites"`     // just the domains
	CreatedOn  int64    `json:"createdon" mapstructure:"createdon"`
	LastUpdate int64    `json:"lastupdate" mapstructure:"lastupdate"`
	ID         int      `json:"id" mapstructure:"id"`
}

// Site is the basic unit of auth
type Site struct {
	Domain     string `json:"domain"`
	CreatedOn  int64  `json:"createdon"`
	LastUpdate int64  `json:"lastupdate"`
	ID         int    `json:"id" mapstructure:"id"`
}

type PTokens struct {
	PAccessToken string
	PIdToken     string
}
<|endoftext|>