<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cronjob

import (
	"fmt"
	"time"

	"github.com/golang/glog"
	"github.com/robfig/cron"

	batchv1 "k8s.io/api/batch/v1"
	batchv1beta1 "k8s.io/api/batch/v1beta1"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	ref "k8s.io/client-go/tools/reference"
	"k8s.io/kubernetes/pkg/api/legacyscheme"
)

// Utilities for dealing with Jobs and CronJobs and time.

func inActiveList(sj batchv1beta1.CronJob, uid types.UID) bool {
	for _, j := range sj.Status.Active {
		if j.UID == uid {
			return true
		}
	}
	return false
}

func deleteFromActiveList(sj *batchv1beta1.CronJob, uid types.UID) {
	if sj == nil {
		return
	}
	newActive := []v1.ObjectReference{}
	for _, j := range sj.Status.Active {
		if j.UID != uid {
			newActive = append(newActive, j)
		}
	}
	sj.Status.Active = newActive
}

// getParentUIDFromJob extracts UID of job's parent and whether it was found
func getParentUIDFromJob(j batchv1.Job) (types.UID, bool) {
	controllerRef := metav1.GetControllerOf(&j)

	if controllerRef == nil {
		return types.UID(""), false
	}

	if controllerRef.Kind != "CronJob" {
		glog.V(4).Infof("Job with non-CronJob parent, name %s namespace %s", j.Name, j.Namespace)
		return types.UID(""), false
	}

	return controllerRef.UID, true
}

// groupJobsByParent groups jobs into a map keyed by the job parent UID (e.g. scheduledJob).
// It has no receiver, to facilitate testing.
func groupJobsByParent(js []batchv1.Job) map[types.UID][]batchv1.Job {
	jobsBySj := make(map[types.UID][]batchv1.Job)
	for _, job := range js {
		parentUID, found := getParentUIDFromJob(job)
		if !found {
			glog.V(4).Infof("Unable to get parent uid from job %s in namespace %s", job.Name, job.Namespace)
			continue
		}
		jobsBySj[parentUID] = append(jobsBySj[parentUID], job)
	}
	return jobsBySj
}

// getNextStartTimeAfter gets the latest scheduled start time that is less than "now", or an error.
func getNextStartTimeAfter(schedule string, now time.Time) (time.Time, error) {
	// Using robfig/cron for cron scheduled parsing and next runtime
	// computation. Not using the entire library because:
	// - I want to detect when we missed a runtime due to being down.
	//   - How do I set the time such that I can detect the last known runtime?
	// - I guess the functions could launch a go-routine to start the job and
	// then return.
	// How to handle concurrency control.
	// How to detect changes to schedules or deleted schedules and then
	// update the jobs?
	sched, err := cron.Parse(schedule)
	if err != nil {
		return time.Unix(0, 0), fmt.Errorf("Unparseable schedule: %s : %s", schedule, err)
	}
	return sched.Next(now), nil
}

// getRecentUnmetScheduleTimes gets a slice of times (from oldest to latest) that have passed when a Job should have started but did not.
//
// If there are too many (>100) unstarted times, just give up and return an empty slice.
// If there were missed times prior to the last known start time, then those are not returned.
func getRecentUnmetScheduleTimes(sj batchv1beta1.CronJob, now time.Time) ([]time.Time, error) {
	starts := []time.Time{}
	sched, err := cron.ParseStandard(sj.Spec.Schedule)
	if err != nil {
		return starts, fmt.Errorf("Unparseable schedule: %s : %s", sj.Spec.Schedule, err)
	}

	var earliestTime time.Time
	if sj.Status.LastScheduleTime != nil {
		earliestTime = sj.Status.LastScheduleTime.Time
	} else {
		// If none found, then this is either a recently created scheduledJob,
		// or the active/completed info was somehow lost (contract for status
		// in kubernetes says it may need to be recreated), or that we have
		// started a job, but have not noticed it yet (distributed systems can
		// have arbitrary delays).  In any case, use the creation time of the
		// CronJob as last known start time.
		earliestTime = sj.ObjectMeta.CreationTimestamp.Time
	}
	if sj.Spec.StartingDeadlineSeconds != nil {
		// Controller is not going to schedule anything below this point
		schedulingDeadline := now.Add(-time.Second * time.Duration(*sj.Spec.StartingDeadlineSeconds))

		if schedulingDeadline.After(earliestTime) {
			earliestTime = schedulingDeadline
		}
	}
	if earliestTime.After(now) {
		return []time.Time{}, nil
	}

	for t := sched.Next(earliestTime); !t.After(now); t = sched.Next(t) {
		starts = append(starts, t)
		// An object might miss several starts. For example, if
		// controller gets wedged on friday at 5:01pm when everyone has
		// gone home, and someone comes in on tuesday AM and discovers
		// the problem and restarts the controller, then all the hourly
		// jobs, more than 80 of them for one hourly scheduledJob, should
		// all start running with no further intervention (if the scheduledJob
		// allows concurrency and late starts).
		//
		// However, if there is a bug somewhere, or incorrect clock
		// on controller's server or apiservers (for setting creationTimestamp)
		// then there could be so many missed start times (it could be off
		// by decades or more), that it would eat up all the CPU and memory
		// of this controller. In that case, we want to not try to list
		// all the missed start times.
		//
		// I've somewhat arbitrarily picked 100, as more than 80,
		// but less than "lots".
		if len(starts) > 100 {
			// We can't get the most recent times so just return an empty slice
			return []time.Time{}, fmt.Errorf("Too many missed start time (> 100). Set or decrease .spec.startingDeadlineSeconds or check clock skew.")
		}
	}
	return starts, nil
}

// getJobFromTemplate makes a Job from a CronJob
func getJobFromTemplate(sj *batchv1beta1.CronJob, scheduledTime time.Time) (*batchv1.Job, error) {
	// TODO: consider adding the following labels:
	// nominal-start-time=$RFC_3339_DATE_OF_INTENDED_START -- for user convenience
	// scheduled-job-name=$SJ_NAME -- for user convenience
	labels := copyLabels(&sj.Spec.JobTemplate)
	annotations := copyAnnotations(&sj.Spec.JobTemplate)
	// We want job names for a given nominal start time to have a deterministic name to avoid the same job being created twice
	name := fmt.Sprintf("%s-%d", sj.Name, getTimeHash(scheduledTime))

	job := &batchv1.Job{
		ObjectMeta: metav1.ObjectMeta{
			Labels:          labels,
			Annotations:     annotations,
			Name:            name,
			OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(sj, controllerKind)},
		},
	}
	if err := legacyscheme.Scheme.Convert(&sj.Spec.JobTemplate.Spec, &job.Spec, nil); err != nil {
		return nil, fmt.Errorf("unable to convert job template: %v", err)
	}
	return job, nil
}

// getTimeHash returns Unix Epoch Time
func getTimeHash(scheduledTime time.Time) int64 {
	return scheduledTime.Unix()
}

// makeCreatedByRefJson makes a json string with an object reference for use in "created-by" annotation value
func makeCreatedByRefJson(object runtime.Object) (string, error) {
	createdByRef, err := ref.GetReference(legacyscheme.Scheme, object)
	if err != nil {
		return "", fmt.Errorf("unable to get controller reference: %v", err)
	}

	// TODO: this code was not safe previously - as soon as new code came along that switched to v2, old clients
	//   would be broken upon reading it. This is explicitly hardcoded to v1 to guarantee predictable deployment.
	//   We need to consistently handle this case of annotation versioning.
	codec := legacyscheme.Codecs.LegacyCodec(schema.GroupVersion{Group: v1.GroupName, Version: "v1"})

	createdByRefJson, err := runtime.Encode(codec, &v1.SerializedReference{
		Reference: *createdByRef,
	})
	if err != nil {
		return "", fmt.Errorf("unable to serialize controller reference: %v", err)
	}
	return string(createdByRefJson), nil
}

func getFinishedStatus(j *batchv1.Job) (bool, batchv1.JobConditionType) {
	for _, c := range j.Status.Conditions {
		if (c.Type == batchv1.JobComplete || c.Type == batchv1.JobFailed) && c.Status == v1.ConditionTrue {
			return true, c.Type
		}
	}
	return false, ""
}

func IsJobFinished(j *batchv1.Job) bool {
	isFinished, _ := getFinishedStatus(j)
	return isFinished
}

// byJobStartTime sorts a list of jobs by start timestamp, using their names as a tie breaker.
type byJobStartTime []batchv1.Job

func (o byJobStartTime) Len() int      { return len(o) }
func (o byJobStartTime) Swap(i, j int) { o[i], o[j] = o[j], o[i] }

func (o byJobStartTime) Less(i, j int) bool {
	if o[j].Status.StartTime == nil {
		return o[i].Status.StartTime != nil
	}

	if o[i].Status.StartTime.Equal(o[j].Status.StartTime) {
		return o[i].Name < o[j].Name
	}

	return o[i].Status.StartTime.Before(o[j].Status.StartTime)
}
<commit_msg>cronjob_remove_getNextStartTimeAfter<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cronjob

import (
	"fmt"
	"time"

	"github.com/golang/glog"
	"github.com/robfig/cron"

	batchv1 "k8s.io/api/batch/v1"
	batchv1beta1 "k8s.io/api/batch/v1beta1"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	ref "k8s.io/client-go/tools/reference"
	"k8s.io/kubernetes/pkg/api/legacyscheme"
)

// Utilities for dealing with Jobs and CronJobs and time.

func inActiveList(sj batchv1beta1.CronJob, uid types.UID) bool {
	for _, j := range sj.Status.Active {
		if j.UID == uid {
			return true
		}
	}
	return false
}

func deleteFromActiveList(sj *batchv1beta1.CronJob, uid types.UID) {
	if sj == nil {
		return
	}
	newActive := []v1.ObjectReference{}
	for _, j := range sj.Status.Active {
		if j.UID != uid {
			newActive = append(newActive, j)
		}
	}
	sj.Status.Active = newActive
}

// getParentUIDFromJob extracts UID of job's parent and whether it was found
func getParentUIDFromJob(j batchv1.Job) (types.UID, bool) {
	controllerRef := metav1.GetControllerOf(&j)

	if controllerRef == nil {
		return types.UID(""), false
	}

	if controllerRef.Kind != "CronJob" {
		glog.V(4).Infof("Job with non-CronJob parent, name %s namespace %s", j.Name, j.Namespace)
		return types.UID(""), false
	}

	return controllerRef.UID, true
}

// groupJobsByParent groups jobs into a map keyed by the job parent UID (e.g. scheduledJob).
// It has no receiver, to facilitate testing.
func groupJobsByParent(js []batchv1.Job) map[types.UID][]batchv1.Job {
	jobsBySj := make(map[types.UID][]batchv1.Job)
	for _, job := range js {
		parentUID, found := getParentUIDFromJob(job)
		if !found {
			glog.V(4).Infof("Unable to get parent uid from job %s in namespace %s", job.Name, job.Namespace)
			continue
		}
		jobsBySj[parentUID] = append(jobsBySj[parentUID], job)
	}
	return jobsBySj
}

// getRecentUnmetScheduleTimes gets a slice of times (from oldest to latest) that have passed when a Job should have started but did not.
//
// If there are too many (>100) unstarted times, just give up and return an empty slice.
// If there were missed times prior to the last known start time, then those are not returned.
func getRecentUnmetScheduleTimes(sj batchv1beta1.CronJob, now time.Time) ([]time.Time, error) {
	starts := []time.Time{}
	sched, err := cron.ParseStandard(sj.Spec.Schedule)
	if err != nil {
		return starts, fmt.Errorf("Unparseable schedule: %s : %s", sj.Spec.Schedule, err)
	}

	var earliestTime time.Time
	if sj.Status.LastScheduleTime != nil {
		earliestTime = sj.Status.LastScheduleTime.Time
	} else {
		// If none found, then this is either a recently created scheduledJob,
		// or the active/completed info was somehow lost (contract for status
		// in kubernetes says it may need to be recreated), or that we have
		// started a job, but have not noticed it yet (distributed systems can
		// have arbitrary delays).  In any case, use the creation time of the
		// CronJob as last known start time.
		earliestTime = sj.ObjectMeta.CreationTimestamp.Time
	}
	if sj.Spec.StartingDeadlineSeconds != nil {
		// Controller is not going to schedule anything below this point
		schedulingDeadline := now.Add(-time.Second * time.Duration(*sj.Spec.StartingDeadlineSeconds))

		if schedulingDeadline.After(earliestTime) {
			earliestTime = schedulingDeadline
		}
	}
	if earliestTime.After(now) {
		return []time.Time{}, nil
	}

	for t := sched.Next(earliestTime); !t.After(now); t = sched.Next(t) {
		starts = append(starts, t)
		// An object might miss several starts. For example, if
		// controller gets wedged on friday at 5:01pm when everyone has
		// gone home, and someone comes in on tuesday AM and discovers
		// the problem and restarts the controller, then all the hourly
		// jobs, more than 80 of them for one hourly scheduledJob, should
		// all start running with no further intervention (if the scheduledJob
		// allows concurrency and late starts).
		//
		// However, if there is a bug somewhere, or incorrect clock
		// on controller's server or apiservers (for setting creationTimestamp)
		// then there could be so many missed start times (it could be off
		// by decades or more), that it would eat up all the CPU and memory
		// of this controller. In that case, we want to not try to list
		// all the missed start times.
		//
		// I've somewhat arbitrarily picked 100, as more than 80,
		// but less than "lots".
		if len(starts) > 100 {
			// We can't get the most recent times so just return an empty slice
			return []time.Time{}, fmt.Errorf("Too many missed start time (> 100). Set or decrease .spec.startingDeadlineSeconds or check clock skew.")
		}
	}
	return starts, nil
}

// getJobFromTemplate makes a Job from a CronJob
func getJobFromTemplate(sj *batchv1beta1.CronJob, scheduledTime time.Time) (*batchv1.Job, error) {
	// TODO: consider adding the following labels:
	// nominal-start-time=$RFC_3339_DATE_OF_INTENDED_START -- for user convenience
	// scheduled-job-name=$SJ_NAME -- for user convenience
	labels := copyLabels(&sj.Spec.JobTemplate)
	annotations := copyAnnotations(&sj.Spec.JobTemplate)
	// We want job names for a given nominal start time to have a deterministic name to avoid the same job being created twice
	name := fmt.Sprintf("%s-%d", sj.Name, getTimeHash(scheduledTime))

	job := &batchv1.Job{
		ObjectMeta: metav1.ObjectMeta{
			Labels:          labels,
			Annotations:     annotations,
			Name:            name,
			OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(sj, controllerKind)},
		},
	}
	if err := legacyscheme.Scheme.Convert(&sj.Spec.JobTemplate.Spec, &job.Spec, nil); err != nil {
		return nil, fmt.Errorf("unable to convert job template: %v", err)
	}
	return job, nil
}

// getTimeHash returns Unix Epoch Time
func getTimeHash(scheduledTime time.Time) int64 {
	return scheduledTime.Unix()
}

// makeCreatedByRefJson makes a json string with an object reference for use in "created-by" annotation value
func makeCreatedByRefJson(object runtime.Object) (string, error) {
	createdByRef, err := ref.GetReference(legacyscheme.Scheme, object)
	if err != nil {
		return "", fmt.Errorf("unable to get controller reference: %v", err)
	}

	// TODO: this code was not safe previously - as soon as new code came along that switched to v2, old clients
	//   would be broken upon reading it. This is explicitly hardcoded to v1 to guarantee predictable deployment.
	//   We need to consistently handle this case of annotation versioning.
	codec := legacyscheme.Codecs.LegacyCodec(schema.GroupVersion{Group: v1.GroupName, Version: "v1"})

	createdByRefJson, err := runtime.Encode(codec, &v1.SerializedReference{
		Reference: *createdByRef,
	})
	if err != nil {
		return "", fmt.Errorf("unable to serialize controller reference: %v", err)
	}
	return string(createdByRefJson), nil
}

func getFinishedStatus(j *batchv1.Job) (bool, batchv1.JobConditionType) {
	for _, c := range j.Status.Conditions {
		if (c.Type == batchv1.JobComplete || c.Type == batchv1.JobFailed) && c.Status == v1.ConditionTrue {
			return true, c.Type
		}
	}
	return false, ""
}

func IsJobFinished(j *batchv1.Job) bool {
	isFinished, _ := getFinishedStatus(j)
	return isFinished
}

// byJobStartTime sorts a list of jobs by start timestamp, using their names as a tie breaker.
type byJobStartTime []batchv1.Job

func (o byJobStartTime) Len() int      { return len(o) }
func (o byJobStartTime) Swap(i, j int) { o[i], o[j] = o[j], o[i] }

func (o byJobStartTime) Less(i, j int) bool {
	if o[j].Status.StartTime == nil {
		return o[i].Status.StartTime != nil
	}

	if o[i].Status.StartTime.Equal(o[j].Status.StartTime) {
		return o[i].Name < o[j].Name
	}

	return o[i].Status.StartTime.Before(o[j].Status.StartTime)
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 ActiveState Software Inc. All rights reserved.

package watch

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/ActiveState/tail/util"
	"gopkg.in/fsnotify.v0"
	"gopkg.in/tomb.v1"
)

// InotifyFileWatcher uses inotify to monitor file changes.
type InotifyFileWatcher struct {
	Filename string
	Size     int64
	w        *fsnotify.Watcher
}

func NewInotifyFileWatcher(filename string, w *fsnotify.Watcher) *InotifyFileWatcher {
	fw := &InotifyFileWatcher{filename, 0, w}
	return fw
}

func (fw *InotifyFileWatcher) BlockUntilExists(t *tomb.Tomb) error {
	dirname := filepath.Dir(fw.Filename)

	// Watch for new files to be created in the parent directory.
	err := fw.w.WatchFlags(dirname, fsnotify.FSN_CREATE)
	if err != nil {
		return err
	}
	defer fw.w.RemoveWatch(dirname)

	// Do a real check now as the file might have been created before
	// calling `WatchFlags` above.
	if _, err = os.Stat(fw.Filename); !os.IsNotExist(err) {
		// file exists, or stat returned an error.
		return err
	}

	for {
		select {
		case evt, ok := <-fw.w.Event:
			if !ok {
				return fmt.Errorf("inotify watcher has been closed")
			} else if filepath.Base(evt.Name) == filepath.Base(fw.Filename) {
				return nil
			}
		case <-t.Dying():
			return tomb.ErrDying
		}
	}
	panic("unreachable")
}

func (fw *InotifyFileWatcher) ChangeEvents(t *tomb.Tomb, fi os.FileInfo) *FileChanges {
	changes := NewFileChanges()

	err := fw.w.Watch(fw.Filename)
	if err != nil {
		util.Fatal("Error watching %v: %v", fw.Filename, err)
	}

	fw.Size = fi.Size()

	go func() {
		defer fw.w.RemoveWatch(fw.Filename)
		defer changes.Close()

		for {
			prevSize := fw.Size

			var evt *fsnotify.FileEvent
			var ok bool

			select {
			case evt, ok = <-fw.w.Event:
				if !ok {
					return
				}
			case <-t.Dying():
				return
			}

			switch {
			case evt.IsDelete():
				fallthrough

			case evt.IsRename():
				changes.NotifyDeleted()
				return

			case evt.IsModify():
				fi, err := os.Stat(fw.Filename)
				if err != nil {
					if os.IsNotExist(err) {
						changes.NotifyDeleted()
						return
					}
					// XXX: report this error back to the user
					util.Fatal("Failed to stat file %v: %v", fw.Filename, err)
				}
				fw.Size = fi.Size()

				if prevSize > 0 && prevSize > fw.Size {
					changes.NotifyTruncated()
				} else {
					changes.NotifyModified()
				}
				prevSize = fw.Size
			}
		}
	}()

	return changes
}
<commit_msg>Watch the *directory* for file deletions instead of the file itself.<commit_after>// Copyright (c) 2013 ActiveState Software Inc. All rights reserved.

package watch

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/ActiveState/tail/util"
	"gopkg.in/fsnotify.v0"
	"gopkg.in/tomb.v1"
)

// InotifyFileWatcher uses inotify to monitor file changes.
type InotifyFileWatcher struct {
	Filename string
	Size     int64
	w        *fsnotify.Watcher
}

func NewInotifyFileWatcher(filename string, w *fsnotify.Watcher) *InotifyFileWatcher {
	fw := &InotifyFileWatcher{filename, 0, w}
	return fw
}

func (fw *InotifyFileWatcher) BlockUntilExists(t *tomb.Tomb) error {
	dirname := filepath.Dir(fw.Filename)

	// Watch for new files to be created in the parent directory.
	err := fw.w.WatchFlags(dirname, fsnotify.FSN_CREATE)
	if err != nil {
		return err
	}
	defer fw.w.RemoveWatch(dirname)

	// Do a real check now as the file might have been created before
	// calling `WatchFlags` above.
	if _, err = os.Stat(fw.Filename); !os.IsNotExist(err) {
		// file exists, or stat returned an error.
		return err
	}

	for {
		select {
		case evt, ok := <-fw.w.Event:
			if !ok {
				return fmt.Errorf("inotify watcher has been closed")
			} else if filepath.Base(evt.Name) == filepath.Base(fw.Filename) {
				return nil
			}
		case <-t.Dying():
			return tomb.ErrDying
		}
	}
	panic("unreachable")
}

func (fw *InotifyFileWatcher) ChangeEvents(t *tomb.Tomb, fi os.FileInfo) *FileChanges {
	changes := NewFileChanges()

	if err := fw.w.Watch(fw.Filename); err != nil {
		util.Fatal("Error watching %v: %v", fw.Filename, err)
	}

	// Watch the directory to be notified when the file is deleted since the file
	// watch is on the inode, not the path.
	dirname := filepath.Dir(fw.Filename)
	if err := fw.w.WatchFlags(dirname, fsnotify.FSN_DELETE); err != nil {
		util.Fatal("Error watching %v: %v", dirname, err)
	}

	fw.Size = fi.Size()

	go func() {
		defer fw.w.RemoveWatch(fw.Filename)
		defer fw.w.RemoveWatch(dirname)
		defer changes.Close()

		for {
			prevSize := fw.Size

			var evt *fsnotify.FileEvent
			var ok bool

			select {
			case evt, ok = <-fw.w.Event:
				if !ok {
					return
				}
			case <-t.Dying():
				return
			}

			switch {
			case evt.IsDelete():
				if filepath.Base(evt.Name) != filepath.Base(fw.Filename) {
					continue
				}
				fallthrough

			case evt.IsRename():
				changes.NotifyDeleted()
				return

			case evt.IsModify():
				fi, err := os.Stat(fw.Filename)
				if err != nil {
					if os.IsNotExist(err) {
						changes.NotifyDeleted()
						return
					}
					// XXX: report this error back to the user
					util.Fatal("Failed to stat file %v: %v", fw.Filename, err)
				}
				fw.Size = fi.Size()

				if prevSize > 0 && prevSize > fw.Size {
					changes.NotifyTruncated()
				} else {
					changes.NotifyModified()
				}
				prevSize = fw.Size
			}
		}
	}()

	return changes
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package elliptic

import (
	"crypto/elliptic/internal/fiat"
	"math/big"
)

type p521Curve struct {
	*CurveParams
}

var p521 p521Curve
var p521Params *CurveParams

func initP521() {
	// See FIPS 186-3, section D.2.5
	p521.CurveParams = &CurveParams{Name: "P-521"}
	p521.P, _ = new(big.Int).SetString("6864797660130609714981900799081393217269435300143305409394463459185543183397656052122559640661454554977296311391480858037121987999716643812574028291115057151", 10)
	p521.N, _ = new(big.Int).SetString("6864797660130609714981900799081393217269435300143305409394463459185543183397655394245057746333217197532963996371363321113864768612440380340372808892707005449", 10)
	p521.B, _ = new(big.Int).SetString("051953eb9618e1c9a1f929a21a0b68540eea2da725b99b315f3b8b489918ef109e156193951ec7e937b1652c0bd3bb1bf073573df883d2c34f1ef451fd46b503f00", 16)
	p521.Gx, _ = new(big.Int).SetString("c6858e06b70404e9cd9e3ecb662395b4429c648139053fb521f828af606b4d3dbaa14b5e77efe75928fe1dc127a2ffa8de3348b3c1856a429bf97e7e31c2e5bd66", 16)
	p521.Gy, _ = new(big.Int).SetString("11839296a789a3bc0045c8a5fb42c7d1bd998f54449579b446817afbd17273e662c97ee72995ef42640c550b9013fad0761353c7086a272c24088be94769fd16650", 16)
	p521.BitSize = 521
}

func (curve p521Curve) Params() *CurveParams {
	return curve.CurveParams
}

func (curve p521Curve) IsOnCurve(x, y *big.Int) bool {
	x1 := bigIntToFiatP521(x)
	y1 := bigIntToFiatP521(y)
	b := bigIntToFiatP521(curve.B) // TODO: precompute this value.

	// x³ - 3x + b.
	x3 := new(fiat.P521Element).Square(x1)
	x3.Mul(x3, x1)

	threeX := new(fiat.P521Element).Add(x1, x1)
	threeX.Add(threeX, x1)

	x3.Sub(x3, threeX)
	x3.Add(x3, b)

	// y² = x³ - 3x + b
	y2 := new(fiat.P521Element).Square(y1)

	return x3.Equal(y2) == 1
}

type p512Point struct {
	x, y, z *fiat.P521Element
}

func fiatP521ToBigInt(x *fiat.P521Element) *big.Int {
	xBytes := x.Bytes()
	for i := range xBytes[:len(xBytes)/2] {
		xBytes[i], xBytes[len(xBytes)-i-1] = xBytes[len(xBytes)-i-1], xBytes[i]
	}
	return new(big.Int).SetBytes(xBytes)
}

// affineFromJacobian brings a point in Jacobian coordinates back to affine
// coordinates, with (0, 0) representing infinity by convention. It also goes
// back to big.Int values to match the exposed API.
func (curve p521Curve) affineFromJacobian(p *p512Point) (x, y *big.Int) {
	if p.z.IsZero() == 1 {
		return new(big.Int), new(big.Int)
	}

	zinv := new(fiat.P521Element).Invert(p.z)
	zinvsq := new(fiat.P521Element).Mul(zinv, zinv)

	xx := new(fiat.P521Element).Mul(p.x, zinvsq)
	zinvsq.Mul(zinvsq, zinv)
	yy := new(fiat.P521Element).Mul(p.y, zinvsq)

	return fiatP521ToBigInt(xx), fiatP521ToBigInt(yy)
}

func bigIntToFiatP521(x *big.Int) *fiat.P521Element {
	xBytes := new(big.Int).Mod(x, p521.P).FillBytes(make([]byte, 66))
	for i := range xBytes[:len(xBytes)/2] {
		xBytes[i], xBytes[len(xBytes)-i-1] = xBytes[len(xBytes)-i-1], xBytes[i]
	}
	x1, err := new(fiat.P521Element).SetBytes(xBytes)
	if err != nil {
		// The input is reduced modulo P and encoded in a fixed size bytes
		// slice, this should be impossible.
		panic("internal error: bigIntToFiatP521")
	}
	return x1
}

// jacobianFromAffine converts (x, y) affine coordinates into (x, y, z) Jacobian
// coordinates. It also converts from big.Int to fiat, which is necessarily a
// messy and variable-time operation, which we can't avoid due to the exposed API.
func (curve p521Curve) jacobianFromAffine(x, y *big.Int) *p512Point {
	// (0, 0) is by convention the point at infinity, which can't be represented
	// in affine coordinates, but is (0, 0, 0) in Jacobian.
	if x.Sign() == 0 && y.Sign() == 0 {
		return &p512Point{
			x: new(fiat.P521Element),
			y: new(fiat.P521Element),
			z: new(fiat.P521Element),
		}
	}
	return &p512Point{
		x: bigIntToFiatP521(x),
		y: bigIntToFiatP521(y),
		z: new(fiat.P521Element).One(),
	}
}

func (curve p521Curve) Add(x1, y1, x2, y2 *big.Int) (*big.Int, *big.Int) {
	p1 := curve.jacobianFromAffine(x1, y1)
	p2 := curve.jacobianFromAffine(x2, y2)
	return curve.affineFromJacobian(p1.addJacobian(p1, p2))
}

// addJacobian sets q = p1 + p2, and returns q. The points may overlap.
func (q *p512Point) addJacobian(p1, p2 *p512Point) *p512Point {
	// https://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#addition-add-2007-bl
	if p1.z.IsZero() == 1 {
		q.x.Set(p2.x)
		q.y.Set(p2.y)
		q.z.Set(p2.z)
		return q
	}
	if p2.z.IsZero() == 1 {
		q.x.Set(p1.x)
		q.y.Set(p1.y)
		q.z.Set(p1.z)
		return q
	}

	z1z1 := new(fiat.P521Element).Square(p1.z)
	z2z2 := new(fiat.P521Element).Square(p2.z)

	u1 := new(fiat.P521Element).Mul(p1.x, z2z2)
	u2 := new(fiat.P521Element).Mul(p2.x, z1z1)
	h := new(fiat.P521Element).Sub(u2, u1)
	xEqual := h.IsZero() == 1
	i := new(fiat.P521Element).Add(h, h)
	i.Square(i)
	j := new(fiat.P521Element).Mul(h, i)

	s1 := new(fiat.P521Element).Mul(p1.y, p2.z)
	s1.Mul(s1, z2z2)
	s2 := new(fiat.P521Element).Mul(p2.y, p1.z)
	s2.Mul(s2, z1z1)
	r := new(fiat.P521Element).Sub(s2, s1)
	yEqual := r.IsZero() == 1
	if xEqual && yEqual {
		return q.doubleJacobian(p1)
	}
	r.Add(r, r)
	v := new(fiat.P521Element).Mul(u1, i)

	q.x.Set(r)
	q.x.Square(q.x)
	q.x.Sub(q.x, j)
	q.x.Sub(q.x, v)
	q.x.Sub(q.x, v)

	q.y.Set(r)
	v.Sub(v, q.x)
	q.y.Mul(q.y, v)
	s1.Mul(s1, j)
	s1.Add(s1, s1)
	q.y.Sub(q.y, s1)

	q.z.Add(p1.z, p2.z)
	q.z.Square(q.z)
	q.z.Sub(q.z, z1z1)
	q.z.Sub(q.z, z2z2)
	q.z.Mul(q.z, h)

	return q
}

func (curve p521Curve) Double(x1, y1 *big.Int) (*big.Int, *big.Int) {
	p := curve.jacobianFromAffine(x1, y1)
	return curve.affineFromJacobian(p.doubleJacobian(p))
}

// doubleJacobian sets q = p + p, and returns q. The points may overlap.
func (q *p512Point) doubleJacobian(p *p512Point) *p512Point {
	// https://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#doubling-dbl-2001-b
	delta := new(fiat.P521Element).Square(p.z)
	gamma := new(fiat.P521Element).Square(p.y)
	alpha := new(fiat.P521Element).Sub(p.x, delta)
	alpha2 := new(fiat.P521Element).Add(p.x, delta)
	alpha.Mul(alpha, alpha2)
	alpha2.Set(alpha)
	alpha.Add(alpha, alpha)
	alpha.Add(alpha, alpha2)

	beta := alpha2.Mul(p.x, gamma)

	q.x.Square(alpha)
	beta8 := new(fiat.P521Element).Add(beta, beta)
	beta8.Add(beta8, beta8)
	beta8.Add(beta8, beta8)
	q.x.Sub(q.x, beta8)

	q.z.Add(p.y, p.z)
	q.z.Square(q.z)
	q.z.Sub(q.z, gamma)
	q.z.Sub(q.z, delta)

	beta.Add(beta, beta)
	beta.Add(beta, beta)
	beta.Sub(beta, q.x)
	q.y.Mul(alpha, beta)

	gamma.Square(gamma)
	gamma.Add(gamma, gamma)
	gamma.Add(gamma, gamma)
	gamma.Add(gamma, gamma)

	q.y.Sub(q.y, gamma)

	return q
}

func (curve p521Curve) ScalarMult(Bx, By *big.Int, k []byte) (*big.Int, *big.Int) {
	B := curve.jacobianFromAffine(Bx, By)
	p := &p512Point{
		x: new(fiat.P521Element),
		y: new(fiat.P521Element),
		z: new(fiat.P521Element),
	}

	for _, byte := range k {
		for bitNum := 0; bitNum < 8; bitNum++ {
			p.doubleJacobian(p)
			if byte&0x80 == 0x80 {
				p.addJacobian(B, p)
			}
			byte <<= 1
		}
	}

	return curve.affineFromJacobian(p)
}

func (curve p521Curve) ScalarBaseMult(k []byte) (*big.Int, *big.Int) {
	return curve.ScalarMult(curve.Gx, curve.Gy, k)
}
<commit_msg>crypto/elliptic: make P-521 scalar multiplication constant time<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package elliptic

import (
	"crypto/elliptic/internal/fiat"
	"math/big"
)

type p521Curve struct {
	*CurveParams
}

var p521 p521Curve
var p521Params *CurveParams

func initP521() {
	// See FIPS 186-3, section D.2.5
	p521.CurveParams = &CurveParams{Name: "P-521"}
	p521.P, _ = new(big.Int).SetString("6864797660130609714981900799081393217269435300143305409394463459185543183397656052122559640661454554977296311391480858037121987999716643812574028291115057151", 10)
	p521.N, _ = new(big.Int).SetString("6864797660130609714981900799081393217269435300143305409394463459185543183397655394245057746333217197532963996371363321113864768612440380340372808892707005449", 10)
	p521.B, _ = new(big.Int).SetString("051953eb9618e1c9a1f929a21a0b68540eea2da725b99b315f3b8b489918ef109e156193951ec7e937b1652c0bd3bb1bf073573df883d2c34f1ef451fd46b503f00", 16)
	p521.Gx, _ = new(big.Int).SetString("c6858e06b70404e9cd9e3ecb662395b4429c648139053fb521f828af606b4d3dbaa14b5e77efe75928fe1dc127a2ffa8de3348b3c1856a429bf97e7e31c2e5bd66", 16)
	p521.Gy, _ = new(big.Int).SetString("11839296a789a3bc0045c8a5fb42c7d1bd998f54449579b446817afbd17273e662c97ee72995ef42640c550b9013fad0761353c7086a272c24088be94769fd16650", 16)
	p521.BitSize = 521
}

func (curve p521Curve) Params() *CurveParams {
	return curve.CurveParams
}

func (curve p521Curve) IsOnCurve(x, y *big.Int) bool {
	x1 := bigIntToFiatP521(x)
	y1 := bigIntToFiatP521(y)
	b := bigIntToFiatP521(curve.B) // TODO: precompute this value.

	// x³ - 3x + b.
	x3 := new(fiat.P521Element).Square(x1)
	x3.Mul(x3, x1)

	threeX := new(fiat.P521Element).Add(x1, x1)
	threeX.Add(threeX, x1)

	x3.Sub(x3, threeX)
	x3.Add(x3, b)

	// y² = x³ - 3x + b
	y2 := new(fiat.P521Element).Square(y1)

	return x3.Equal(y2) == 1
}

type p512Point struct {
	x, y, z *fiat.P521Element
}

func fiatP521ToBigInt(x *fiat.P521Element) *big.Int {
	xBytes := x.Bytes()
	for i := range xBytes[:len(xBytes)/2] {
		xBytes[i], xBytes[len(xBytes)-i-1] = xBytes[len(xBytes)-i-1], xBytes[i]
	}
	return new(big.Int).SetBytes(xBytes)
}

// affineFromJacobian brings a point in Jacobian coordinates back to affine
// coordinates, with (0, 0) representing infinity by convention. It also goes
// back to big.Int values to match the exposed API.
func (curve p521Curve) affineFromJacobian(p *p512Point) (x, y *big.Int) {
	if p.z.IsZero() == 1 {
		return new(big.Int), new(big.Int)
	}

	zinv := new(fiat.P521Element).Invert(p.z)
	zinvsq := new(fiat.P521Element).Mul(zinv, zinv)

	xx := new(fiat.P521Element).Mul(p.x, zinvsq)
	zinvsq.Mul(zinvsq, zinv)
	yy := new(fiat.P521Element).Mul(p.y, zinvsq)

	return fiatP521ToBigInt(xx), fiatP521ToBigInt(yy)
}

func bigIntToFiatP521(x *big.Int) *fiat.P521Element {
	xBytes := new(big.Int).Mod(x, p521.P).FillBytes(make([]byte, 66))
	for i := range xBytes[:len(xBytes)/2] {
		xBytes[i], xBytes[len(xBytes)-i-1] = xBytes[len(xBytes)-i-1], xBytes[i]
	}
	x1, err := new(fiat.P521Element).SetBytes(xBytes)
	if err != nil {
		// The input is reduced modulo P and encoded in a fixed size bytes
		// slice, this should be impossible.
		panic("internal error: bigIntToFiatP521")
	}
	return x1
}

// jacobianFromAffine converts (x, y) affine coordinates into (x, y, z) Jacobian
// coordinates. It also converts from big.Int to fiat, which is necessarily a
// messy and variable-time operation, which we can't avoid due to the exposed API.
func (curve p521Curve) jacobianFromAffine(x, y *big.Int) *p512Point {
	// (0, 0) is by convention the point at infinity, which can't be represented
	// in affine coordinates, but is (0, 0, 0) in Jacobian.
	if x.Sign() == 0 && y.Sign() == 0 {
		return &p512Point{
			x: new(fiat.P521Element),
			y: new(fiat.P521Element),
			z: new(fiat.P521Element),
		}
	}
	return &p512Point{
		x: bigIntToFiatP521(x),
		y: bigIntToFiatP521(y),
		z: new(fiat.P521Element).One(),
	}
}

func (curve p521Curve) Add(x1, y1, x2, y2 *big.Int) (*big.Int, *big.Int) {
	p1 := curve.jacobianFromAffine(x1, y1)
	p2 := curve.jacobianFromAffine(x2, y2)
	return curve.affineFromJacobian(p1.addJacobian(p1, p2))
}

// addJacobian sets q = p1 + p2, and returns q. The points may overlap.
func (q *p512Point) addJacobian(p1, p2 *p512Point) *p512Point {
	// https://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#addition-add-2007-bl
	z1IsZero := p1.z.IsZero()
	z2IsZero := p2.z.IsZero()

	z1z1 := new(fiat.P521Element).Square(p1.z)
	z2z2 := new(fiat.P521Element).Square(p2.z)

	u1 := new(fiat.P521Element).Mul(p1.x, z2z2)
	u2 := new(fiat.P521Element).Mul(p2.x, z1z1)
	h := new(fiat.P521Element).Sub(u2, u1)
	xEqual := h.IsZero() == 1
	i := new(fiat.P521Element).Add(h, h)
	i.Square(i)
	j := new(fiat.P521Element).Mul(h, i)

	s1 := new(fiat.P521Element).Mul(p1.y, p2.z)
	s1.Mul(s1, z2z2)
	s2 := new(fiat.P521Element).Mul(p2.y, p1.z)
	s2.Mul(s2, z1z1)
	r := new(fiat.P521Element).Sub(s2, s1)
	yEqual := r.IsZero() == 1
	if xEqual && yEqual && z1IsZero == 0 && z2IsZero == 0 {
		return q.doubleJacobian(p1)
	}
	r.Add(r, r)
	v := new(fiat.P521Element).Mul(u1, i)

	x := new(fiat.P521Element).Set(r)
	x.Square(x)
	x.Sub(x, j)
	x.Sub(x, v)
	x.Sub(x, v)

	y := new(fiat.P521Element).Set(r)
	v.Sub(v, x)
	y.Mul(y, v)
	s1.Mul(s1, j)
	s1.Add(s1, s1)
	y.Sub(y, s1)

	z := new(fiat.P521Element).Add(p1.z, p2.z)
	z.Square(z)
	z.Sub(z, z1z1)
	z.Sub(z, z2z2)
	z.Mul(z, h)

	x.Select(p2.x, x, z1IsZero)
	x.Select(p1.x, x, z2IsZero)
	y.Select(p2.y, y, z1IsZero)
	y.Select(p1.y, y, z2IsZero)
	z.Select(p2.z, z, z1IsZero)
	z.Select(p1.z, z, z2IsZero)

	q.x.Set(x)
	q.y.Set(y)
	q.z.Set(z)
	return q
}

func (curve p521Curve) Double(x1, y1 *big.Int) (*big.Int, *big.Int) {
	p := curve.jacobianFromAffine(x1, y1)
	return curve.affineFromJacobian(p.doubleJacobian(p))
}

// doubleJacobian sets q = p + p, and returns q. The points may overlap.
func (q *p512Point) doubleJacobian(p *p512Point) *p512Point {
	// https://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-3.html#doubling-dbl-2001-b
	delta := new(fiat.P521Element).Square(p.z)
	gamma := new(fiat.P521Element).Square(p.y)
	alpha := new(fiat.P521Element).Sub(p.x, delta)
	alpha2 := new(fiat.P521Element).Add(p.x, delta)
	alpha.Mul(alpha, alpha2)
	alpha2.Set(alpha)
	alpha.Add(alpha, alpha)
	alpha.Add(alpha, alpha2)

	beta := alpha2.Mul(p.x, gamma)

	q.x.Square(alpha)
	beta8 := new(fiat.P521Element).Add(beta, beta)
	beta8.Add(beta8, beta8)
	beta8.Add(beta8, beta8)
	q.x.Sub(q.x, beta8)

	q.z.Add(p.y, p.z)
	q.z.Square(q.z)
	q.z.Sub(q.z, gamma)
	q.z.Sub(q.z, delta)

	beta.Add(beta, beta)
	beta.Add(beta, beta)
	beta.Sub(beta, q.x)
	q.y.Mul(alpha, beta)

	gamma.Square(gamma)
	gamma.Add(gamma, gamma)
	gamma.Add(gamma, gamma)
	gamma.Add(gamma, gamma)

	q.y.Sub(q.y, gamma)

	return q
}

func (curve p521Curve) ScalarMult(Bx, By *big.Int, scalar []byte) (*big.Int, *big.Int) {
	B := curve.jacobianFromAffine(Bx, By)
	p, t := &p512Point{
		x: new(fiat.P521Element),
		y: new(fiat.P521Element),
		z: new(fiat.P521Element),
	}, &p512Point{
		x: new(fiat.P521Element),
		y: new(fiat.P521Element),
		z: new(fiat.P521Element),
	}

	for _, byte := range scalar {
		for bitNum := 0; bitNum < 8; bitNum++ {
			p.doubleJacobian(p)
			bit := (byte >> (7 - bitNum)) & 1
			t.addJacobian(p, B)
			p.x.Select(t.x, p.x, int(bit))
			p.y.Select(t.y, p.y, int(bit))
			p.z.Select(t.z, p.z, int(bit))
		}
	}

	return curve.affineFromJacobian(p)
}

func (curve p521Curve) ScalarBaseMult(k []byte) (*big.Int, *big.Int) {
	return curve.ScalarMult(curve.Gx, curve.Gy, k)
}
<|endoftext|>


<commit_before>package skunk

import (
	"encoding/json"
	"math"
	"strconv"
	"time"
)

// Body represents the POSTed body of a NewRelic plugin's metrics data.
type Body struct {
	Agent      AgentRep     `json:"agent"`
	Components []*Component `json:"components"`
}

// AgentRep describes a NewRelic agent.
type AgentRep struct {
	Host string `json:"host"`
	// PID zero is treated as an erroneous/nonexistent PID. If you're shoving NewRelic into a Go-based scheduler in
	// the kernel, I guess you could open an issue for this. Otherwise, this seems reasonable to me.
	PID     int    `json:"pid,omitempty"`
	Version string `json:"version"`
}

// Seconds is a convenience wrapper around a time.Duration to allow any duration to be used when describing the duration
// of components' metrics.
type Seconds struct{ time.Duration }

func (s Seconds) MarshalJSON() ([]byte, error) {
	var i int64
	if int, frac := math.Modf(s.Seconds()); frac >= 0.5 {
		i = int64(int) + 1
	} else {
		i = int64(int)
	}
	return strconv.AppendInt(nil, i, 10), nil
}

// Component describes a component in a NewRelic agent. It must contain a minimum of at least one metric, otherwise the
// component is culled from its parent Body before constructing a JSON payload. All fields of the Component are
// read-only once initialized.
type Component struct {
	Name string `json:"name"`
	GUID string `json:"guid"`
	// Duration is the time elapsed, in seconds, for this snapshot of the component. The duration is rounded to the
	// nearest second. This is only used when constructing a payload using a copy of a Component.
	Duration Seconds           `json:"duration"`
	Metrics  map[string]Metric `json:"metrics"`

	// start is the time that the first metric was recorded. If start.IsZero is true, the time needs to be set to
	// the current time once a metric is added. The start time is cleared upon an agent successfully sending
	// a payload to NewRelic.
	start time.Time

	// agent is a pointer to the Agent that owns this component.
	agent *Agent
}

// AddMetric adds a single metric to the Component. If the metric already exists by name in the Component, the value is
// added to the existing metric, otherwise the metric is added as a ScalarMetric.
func (c *Component) AddMetric(name string, value float64) {
	c.agent.ops <- func(*Agent) error {
		if m, ok := c.Metrics[name]; ok {
			c.Metrics[name] = m.Add(value)
		} else {
			c.Metrics[name] = ScalarMetric(value)
		}

		// Set the time over which the metric was gathered.
		if c.start.IsZero() {
			c.start = time.Now()
			c.Duration = Seconds{0}
		} else {
			c.Duration.Duration = time.Since(c.start)
		}

		return nil
	}
}

// Metric describes any metric that can have an additional value added to it. All metrics must be marshallable as JSON,
// but are not required to implement MarshalJSON (e.g., RangeMetric).
//
// The Add method of a Metric is used to get the result of adding an additional value to a metric. Metrics themselves
// should be considered immutable, so the result must be a new Metric.
type Metric interface {
	Add(value float64) Metric
}

// ScalarMetric is any singular metric that does not cover a range a values. Adding to a ScalarMetric produces
// a RangeMetric.
type ScalarMetric float64

func (s ScalarMetric) Add(value float64) Metric {
	f := float64(s)
	return RangeMetric{
		Total:  f + value,
		Count:  2,
		Min:    math.Min(value, f),
		Max:    math.Max(value, f),
		Square: math.Pow(f, 2) + math.Pow(value, 2),
	}
}

func (s ScalarMetric) MarshalJSON() ([]byte, error) {
	return json.Marshal(float64(s))
}

// RangeMetric is any metric that covers a range of values. Adding to a RangeMetric produces a new RangeMetric.
type RangeMetric struct {
	Total float64 `json:"total"`
	Count int     `json:"count"`
	Min   float64 `json:"min"`
	Max   float64 `json:"max"`
	// Square is the sum of squares of all values recorded for the metric. This is simply A₁² + A₂² + Aₙ² where A is
	// the set of numbers recorded for this metric.
	Square float64 `json:"sum_of_squares"`
}

func (r RangeMetric) Add(value float64) Metric {
	return RangeMetric{
		Total:  r.Total + value,
		Count:  r.Count + 1,
		Min:    math.Min(value, r.Min),
		Max:    math.Max(value, r.Max),
		Square: r.Square + math.Pow(value, 2),
	}
}
<commit_msg>Add metric merging to components<commit_after>package skunk

import (
	"encoding/json"
	"math"
	"strconv"
	"time"
)

// Body represents the POSTed body of a NewRelic plugin's metrics data.
type Body struct {
	Agent      AgentRep     `json:"agent"`
	Components []*Component `json:"components"`
}

// AgentRep describes a NewRelic agent.
type AgentRep struct {
	Host string `json:"host"`
	// PID zero is treated as an erroneous/nonexistent PID. If you're shoving NewRelic into a Go-based scheduler in
	// the kernel, I guess you could open an issue for this. Otherwise, this seems reasonable to me.
	PID     int    `json:"pid,omitempty"`
	Version string `json:"version"`
}

// Seconds is a convenience wrapper around a time.Duration to allow any duration to be used when describing the duration
// of components' metrics.
type Seconds struct{ time.Duration }

func (s Seconds) MarshalJSON() ([]byte, error) {
	var i int64
	if int, frac := math.Modf(s.Seconds()); frac >= 0.5 {
		i = int64(int) + 1
	} else {
		i = int64(int)
	}
	return strconv.AppendInt(nil, i, 10), nil
}

// Component describes a component in a NewRelic agent. It must contain a minimum of at least one metric, otherwise the
// component is culled from its parent Body before constructing a JSON payload. All fields of the Component are
// read-only once initialized.
type Component struct {
	Name string `json:"name"`
	GUID string `json:"guid"`
	// Duration is the time elapsed, in seconds, for this snapshot of the component. The duration is rounded to the
	// nearest second. This is only used when constructing a payload using a copy of a Component.
	Duration Seconds           `json:"duration"`
	Metrics  map[string]Metric `json:"metrics"`

	// start is the time that the first metric was recorded. If start.IsZero is true, the time needs to be set to
	// the current time once a metric is added. The start time is cleared upon an agent successfully sending
	// a payload to NewRelic.
	start time.Time

	// agent is a pointer to the Agent that owns this component.
	agent *Agent
}

// AddMetric adds a single metric to the Component. If the metric already exists by name in the Component, the value is
// added to the existing metric, otherwise the metric is added as a ScalarMetric.
func (c *Component) AddMetric(name string, value float64) {
	c.MergeMetric(name, ScalarMetric(value))
}

// AddMetric adds a single metric to the Component. If the metric already exists by name in the Component, the value is
// added to the existing metric, otherwise the metric is added as a ScalarMetric.
func (c *Component) MergeMetric(name string, value Metric) {
	c.agent.ops <- func(*Agent) error {
		if m, ok := c.Metrics[name]; ok {
			c.Metrics[name] = m.Merge(value)
		} else {
			c.Metrics[name] = value
		}

		// Set the time over which the metric was gathered.
		if c.start.IsZero() {
			c.start = time.Now()
			c.Duration = Seconds{0}
		} else {
			c.Duration.Duration = time.Since(c.start)
		}

		return nil
	}
}

// Metric describes any metric that can have an additional value added to it. All metrics must be marshallable as JSON,
// but are not required to implement MarshalJSON (e.g., RangeMetric).
//
// The Add method of a Metric is used to get the result of adding an additional value to a metric. Metrics themselves
// should be considered immutable, so the result must be a new Metric.
//
// This shouldn't be implemented by other libraries.
type Metric interface {
	Add(value float64) Metric
	Merge(Metric) Metric
}

// ScalarMetric is any singular metric that does not cover a range a values. Adding to a ScalarMetric produces
// a RangeMetric.
type ScalarMetric float64

func (s ScalarMetric) Add(value float64) Metric {
	f := float64(s)
	return RangeMetric{
		Total:  f + value,
		Count:  2,
		Min:    math.Min(value, f),
		Max:    math.Max(value, f),
		Square: math.Pow(f, 2) + math.Pow(value, 2),
	}
}

func (s ScalarMetric) Merge(value Metric) Metric {
	f := float64(s)
	return value.Merge(RangeMetric{Total: f, Count: 1, Min: f, Max: f, Square: math.Pow(f, 2)})
}

func (s ScalarMetric) MarshalJSON() ([]byte, error) {
	return json.Marshal(float64(s))
}

// RangeMetric is any metric that covers a range of values. Adding to a RangeMetric produces a new RangeMetric.
type RangeMetric struct {
	Total float64 `json:"total"`
	Count int     `json:"count"`
	Min   float64 `json:"min"`
	Max   float64 `json:"max"`
	// Square is the sum of squares of all values recorded for the metric. This is simply A₁² + A₂² + Aₙ² where A is
	// the set of numbers recorded for this metric.
	Square float64 `json:"sum_of_squares"`
}

func (r RangeMetric) Add(value float64) Metric {
	return RangeMetric{
		Total:  r.Total + value,
		Count:  r.Count + 1,
		Min:    math.Min(value, r.Min),
		Max:    math.Max(value, r.Max),
		Square: r.Square + math.Pow(value, 2),
	}
}

func (r RangeMetric) Merge(value Metric) Metric {
	switch o := value.(type) {
	case ScalarMetric:
		f := float64(o)
		r.Total += f
		r.Count++
		r.Min = math.Min(r.Min, f)
		r.Max = math.Max(r.Max, f)
		r.Square += math.Pow(f, 2)
	case RangeMetric:
		r.Total += o.Total
		r.Count += o.Count
		r.Min = math.Min(r.Min, o.Min)
		r.Max = math.Max(r.Max, o.Max)
		r.Square += o.Square
	default:
		// Defer to the other metric to attempt the merge.
		return value.Merge(r)
	}
	return r
}
<|endoftext|>


<commit_before>// Copyright 2016-2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !privileged_tests

package labelsfilter

import (
	"testing"

	"github.com/cilium/cilium/pkg/checker"
	k8sConst "github.com/cilium/cilium/pkg/k8s/apis/cilium.io"
	"github.com/cilium/cilium/pkg/labels"

	. "gopkg.in/check.v1"
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) {
	TestingT(t)
}

type LabelsPrefCfgSuite struct{}

var _ = Suite(&LabelsPrefCfgSuite{})

func (s *LabelsPrefCfgSuite) TestFilterLabels(c *C) {
	wanted := labels.Labels{
		"id.lizards":                  labels.NewLabel("id.lizards", "web", labels.LabelSourceContainer),
		"id.lizards.k8s":              labels.NewLabel("id.lizards.k8s", "web", labels.LabelSourceK8s),
		"io.kubernetes.pod.namespace": labels.NewLabel("io.kubernetes.pod.namespace", "default", labels.LabelSourceContainer),
		"app.kubernetes.io":           labels.NewLabel("app.kubernetes.io", "my-nginx", labels.LabelSourceContainer),
		"foo2.lizards.k8s":            labels.NewLabel("foo2.lizards.k8s", "web", labels.LabelSourceK8s),
	}

	err := ParseLabelPrefixCfg([]string{":!ignor[eE]", "id.*", "foo"}, "")
	c.Assert(err, IsNil)
	dlpcfg := validLabelPrefixes
	allNormalLabels := map[string]string{
		"io.kubernetes.container.hash":                              "cf58006d",
		"io.kubernetes.container.name":                              "POD",
		"io.kubernetes.container.restartCount":                      "0",
		"io.kubernetes.container.terminationMessagePath":            "",
		"io.kubernetes.pod.name":                                    "my-nginx-3800858182-07i3n",
		"io.kubernetes.pod.namespace":                               "default",
		"app.kubernetes.io":                                         "my-nginx",
		"kubernetes.io.foo":                                         "foo",
		"beta.kubernetes.io.foo":                                    "foo",
		"annotation.kubectl.kubernetes.io":                          "foo",
		"annotation.hello":                                          "world",
		"annotation." + k8sConst.CiliumIdentityAnnotationDeprecated: "12356",
		"io.kubernetes.pod.terminationGracePeriod":                  "30",
		"io.kubernetes.pod.uid":                                     "c2e22414-dfc3-11e5-9792-080027755f5a",
		"ignore":                                                    "foo",
		"ignorE":                                                    "foo",
		"annotation.kubernetes.io/config.seen":                      "2017-05-30T14:22:17.691491034Z",
		"controller-revision-hash":                                  "123456",
	}
	allLabels := labels.Map2Labels(allNormalLabels, labels.LabelSourceContainer)
	filtered, _ := dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 2)
	allLabels["id.lizards"] = labels.NewLabel("id.lizards", "web", labels.LabelSourceContainer)
	allLabels["id.lizards.k8s"] = labels.NewLabel("id.lizards.k8s", "web", labels.LabelSourceK8s)
	filtered, _ = dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 4)
	// Checking that it does not need to an exact match of "foo", but "foo2" also works since it's not a regex
	allLabels["foo2.lizards.k8s"] = labels.NewLabel("foo2.lizards.k8s", "web", labels.LabelSourceK8s)
	filtered, _ = dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 5)
	// Checking that "foo" only works if it's the prefix of a label
	allLabels["lizards.foo.lizards.k8s"] = labels.NewLabel("lizards.foo.lizards.k8s", "web", labels.LabelSourceK8s)
	filtered, _ = dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 5)
	c.Assert(filtered, checker.DeepEquals, wanted)
	// Making sure we are deep copying the labels
	allLabels["id.lizards"] = labels.NewLabel("id.lizards", "web", "I can change this and doesn't affect any one")
	c.Assert(filtered, checker.DeepEquals, wanted)
}

func (s *LabelsPrefCfgSuite) TestDefaultFilterLabels(c *C) {
	wanted := labels.Labels{
		"app.kubernetes.io":           labels.NewLabel("app.kubernetes.io", "my-nginx", labels.LabelSourceContainer),
		"id.lizards.k8s":              labels.NewLabel("id.lizards.k8s", "web", labels.LabelSourceK8s),
		"id.lizards":                  labels.NewLabel("id.lizards", "web", labels.LabelSourceContainer),
		"ignorE":                      labels.NewLabel("ignorE", "foo", labels.LabelSourceContainer),
		"ignore":                      labels.NewLabel("ignore", "foo", labels.LabelSourceContainer),
		"reserved:host":               labels.NewLabel("reserved:host", "", labels.LabelSourceAny),
		"io.kubernetes.pod.namespace": labels.NewLabel("io.kubernetes.pod.namespace", "default", labels.LabelSourceContainer),
	}

	err := ParseLabelPrefixCfg([]string{}, "")
	c.Assert(err, IsNil)
	dlpcfg := validLabelPrefixes
	allNormalLabels := map[string]string{
		"io.kubernetes.container.hash":                              "cf58006d",
		"io.kubernetes.container.name":                              "POD",
		"io.kubernetes.container.restartCount":                      "0",
		"io.kubernetes.container.terminationMessagePath":            "",
		"io.kubernetes.pod.name":                                    "my-nginx-3800858182-07i3n",
		"io.kubernetes.pod.namespace":                               "default",
		"app.kubernetes.io":                                         "my-nginx",
		"kubernetes.io.foo":                                         "foo",
		"beta.kubernetes.io.foo":                                    "foo",
		"annotation.kubectl.kubernetes.io":                          "foo",
		"annotation.hello":                                          "world",
		"annotation." + k8sConst.CiliumIdentityAnnotationDeprecated: "12356",
		"io.kubernetes.pod.terminationGracePeriod":                  "30",
		"io.kubernetes.pod.uid":                                     "c2e22414-dfc3-11e5-9792-080027755f5a",
		"ignore":                                                    "foo",
		"ignorE":                                                    "foo",
		"annotation.kubernetes.io/config.seen":                      "2017-05-30T14:22:17.691491034Z",
		"controller-revision-hash":                                  "123456",
	}
	allLabels := labels.Map2Labels(allNormalLabels, labels.LabelSourceContainer)
	allLabels["reserved:host"] = labels.NewLabel("reserved:host", "", labels.LabelSourceAny)
	filtered, _ := dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 5)
	allLabels["id.lizards"] = labels.NewLabel("id.lizards", "web", labels.LabelSourceContainer)
	allLabels["id.lizards.k8s"] = labels.NewLabel("id.lizards.k8s", "web", labels.LabelSourceK8s)
	filtered, _ = dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 7)
	c.Assert(filtered, checker.DeepEquals, wanted)
}

func (s *LabelsPrefCfgSuite) TestFilterLabelsDocExample(c *C) {
	wanted := labels.Labels{
		"io.cilium.k8s.namespace.labels": labels.NewLabel("io.cilium.k8s.namespace.labels", "foo", labels.LabelSourceK8s),
		"k8s-app-team":                   labels.NewLabel("k8s-app-team", "foo", labels.LabelSourceK8s),
		"app-production":                 labels.NewLabel("app-production", "foo", labels.LabelSourceK8s),
		"name-defined":                   labels.NewLabel("name-defined", "foo", labels.LabelSourceK8s),
		"host":                           labels.NewLabel("host", "", labels.LabelSourceReserved),
		"io.kubernetes.pod.namespace":    labels.NewLabel("io.kubernetes.pod.namespace", "docker", labels.LabelSourceAny),
	}

	err := ParseLabelPrefixCfg([]string{"k8s:io.kubernetes.pod.namespace", "k8s:k8s-app", "k8s:app", "k8s:name"}, "")
	c.Assert(err, IsNil)
	dlpcfg := validLabelPrefixes
	allNormalLabels := map[string]string{
		"io.cilium.k8s.namespace.labels": "foo",
		"k8s-app-team":                   "foo",
		"app-production":                 "foo",
		"name-defined":                   "foo",
	}
	allLabels := labels.Map2Labels(allNormalLabels, labels.LabelSourceK8s)
	filtered, _ := dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 4)

	// Reserved labels are included.
	allLabels["host"] = labels.NewLabel("host", "", labels.LabelSourceReserved)
	filtered, _ = dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 5)

	// io.kubernetes.pod.namespace=docker matches because the default list has any:io.kubernetes.pod.namespace.
	allLabels["io.kubernetes.pod.namespace"] = labels.NewLabel("io.kubernetes.pod.namespace", "docker", labels.LabelSourceAny)
	filtered, _ = dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 6)

	// container:k8s-app-role=foo doesn't match because it doesn't have source k8s.
	allLabels["k8s-app-role"] = labels.NewLabel("k8s-app-role", "foo", labels.LabelSourceContainer)
	filtered, _ = dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 6)
	c.Assert(filtered, checker.DeepEquals, wanted)
}
<commit_msg>labelsfilter: Fix test for default filters<commit_after>// Copyright 2016-2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !privileged_tests

package labelsfilter

import (
	"testing"

	"github.com/cilium/cilium/pkg/checker"
	k8sConst "github.com/cilium/cilium/pkg/k8s/apis/cilium.io"
	"github.com/cilium/cilium/pkg/labels"

	. "gopkg.in/check.v1"
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) {
	TestingT(t)
}

type LabelsPrefCfgSuite struct{}

var _ = Suite(&LabelsPrefCfgSuite{})

func (s *LabelsPrefCfgSuite) TestFilterLabels(c *C) {
	wanted := labels.Labels{
		"id.lizards":                  labels.NewLabel("id.lizards", "web", labels.LabelSourceContainer),
		"id.lizards.k8s":              labels.NewLabel("id.lizards.k8s", "web", labels.LabelSourceK8s),
		"io.kubernetes.pod.namespace": labels.NewLabel("io.kubernetes.pod.namespace", "default", labels.LabelSourceContainer),
		"app.kubernetes.io":           labels.NewLabel("app.kubernetes.io", "my-nginx", labels.LabelSourceContainer),
		"foo2.lizards.k8s":            labels.NewLabel("foo2.lizards.k8s", "web", labels.LabelSourceK8s),
	}

	err := ParseLabelPrefixCfg([]string{":!ignor[eE]", "id.*", "foo"}, "")
	c.Assert(err, IsNil)
	dlpcfg := validLabelPrefixes
	allNormalLabels := map[string]string{
		"io.kubernetes.container.hash":                              "cf58006d",
		"io.kubernetes.container.name":                              "POD",
		"io.kubernetes.container.restartCount":                      "0",
		"io.kubernetes.container.terminationMessagePath":            "",
		"io.kubernetes.pod.name":                                    "my-nginx-3800858182-07i3n",
		"io.kubernetes.pod.namespace":                               "default",
		"app.kubernetes.io":                                         "my-nginx",
		"kubernetes.io.foo":                                         "foo",
		"beta.kubernetes.io.foo":                                    "foo",
		"annotation.kubectl.kubernetes.io":                          "foo",
		"annotation.hello":                                          "world",
		"annotation." + k8sConst.CiliumIdentityAnnotationDeprecated: "12356",
		"io.kubernetes.pod.terminationGracePeriod":                  "30",
		"io.kubernetes.pod.uid":                                     "c2e22414-dfc3-11e5-9792-080027755f5a",
		"ignore":                                                    "foo",
		"ignorE":                                                    "foo",
		"annotation.kubernetes.io/config.seen":                      "2017-05-30T14:22:17.691491034Z",
		"controller-revision-hash":                                  "123456",
	}
	allLabels := labels.Map2Labels(allNormalLabels, labels.LabelSourceContainer)
	filtered, _ := dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 2)
	allLabels["id.lizards"] = labels.NewLabel("id.lizards", "web", labels.LabelSourceContainer)
	allLabels["id.lizards.k8s"] = labels.NewLabel("id.lizards.k8s", "web", labels.LabelSourceK8s)
	filtered, _ = dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 4)
	// Checking that it does not need to an exact match of "foo", but "foo2" also works since it's not a regex
	allLabels["foo2.lizards.k8s"] = labels.NewLabel("foo2.lizards.k8s", "web", labels.LabelSourceK8s)
	filtered, _ = dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 5)
	// Checking that "foo" only works if it's the prefix of a label
	allLabels["lizards.foo.lizards.k8s"] = labels.NewLabel("lizards.foo.lizards.k8s", "web", labels.LabelSourceK8s)
	filtered, _ = dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 5)
	c.Assert(filtered, checker.DeepEquals, wanted)
	// Making sure we are deep copying the labels
	allLabels["id.lizards"] = labels.NewLabel("id.lizards", "web", "I can change this and doesn't affect any one")
	c.Assert(filtered, checker.DeepEquals, wanted)
}

func (s *LabelsPrefCfgSuite) TestDefaultFilterLabels(c *C) {
	wanted := labels.Labels{
		"app.kubernetes.io":           labels.NewLabel("app.kubernetes.io", "my-nginx", labels.LabelSourceContainer),
		"id.lizards.k8s":              labels.NewLabel("id.lizards.k8s", "web", labels.LabelSourceK8s),
		"id.lizards":                  labels.NewLabel("id.lizards", "web", labels.LabelSourceContainer),
		"ignorE":                      labels.NewLabel("ignorE", "foo", labels.LabelSourceContainer),
		"ignore":                      labels.NewLabel("ignore", "foo", labels.LabelSourceContainer),
		"host":                        labels.NewLabel("host", "", labels.LabelSourceReserved),
		"io.kubernetes.pod.namespace": labels.NewLabel("io.kubernetes.pod.namespace", "default", labels.LabelSourceContainer),
	}

	err := ParseLabelPrefixCfg([]string{}, "")
	c.Assert(err, IsNil)
	dlpcfg := validLabelPrefixes
	allNormalLabels := map[string]string{
		"io.kubernetes.container.hash":                              "cf58006d",
		"io.kubernetes.container.name":                              "POD",
		"io.kubernetes.container.restartCount":                      "0",
		"io.kubernetes.container.terminationMessagePath":            "",
		"io.kubernetes.pod.name":                                    "my-nginx-3800858182-07i3n",
		"io.kubernetes.pod.namespace":                               "default",
		"app.kubernetes.io":                                         "my-nginx",
		"kubernetes.io.foo":                                         "foo",
		"beta.kubernetes.io.foo":                                    "foo",
		"annotation.kubectl.kubernetes.io":                          "foo",
		"annotation.hello":                                          "world",
		"annotation." + k8sConst.CiliumIdentityAnnotationDeprecated: "12356",
		"io.kubernetes.pod.terminationGracePeriod":                  "30",
		"io.kubernetes.pod.uid":                                     "c2e22414-dfc3-11e5-9792-080027755f5a",
		"ignore":                                                    "foo",
		"ignorE":                                                    "foo",
		"annotation.kubernetes.io/config.seen":                      "2017-05-30T14:22:17.691491034Z",
		"controller-revision-hash":                                  "123456",
	}
	allLabels := labels.Map2Labels(allNormalLabels, labels.LabelSourceContainer)
	allLabels["host"] = labels.NewLabel("host", "", labels.LabelSourceReserved)
	filtered, _ := dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 5)
	allLabels["id.lizards"] = labels.NewLabel("id.lizards", "web", labels.LabelSourceContainer)
	allLabels["id.lizards.k8s"] = labels.NewLabel("id.lizards.k8s", "web", labels.LabelSourceK8s)
	filtered, _ = dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 7)
	c.Assert(filtered, checker.DeepEquals, wanted)
}

func (s *LabelsPrefCfgSuite) TestFilterLabelsDocExample(c *C) {
	wanted := labels.Labels{
		"io.cilium.k8s.namespace.labels": labels.NewLabel("io.cilium.k8s.namespace.labels", "foo", labels.LabelSourceK8s),
		"k8s-app-team":                   labels.NewLabel("k8s-app-team", "foo", labels.LabelSourceK8s),
		"app-production":                 labels.NewLabel("app-production", "foo", labels.LabelSourceK8s),
		"name-defined":                   labels.NewLabel("name-defined", "foo", labels.LabelSourceK8s),
		"host":                           labels.NewLabel("host", "", labels.LabelSourceReserved),
		"io.kubernetes.pod.namespace":    labels.NewLabel("io.kubernetes.pod.namespace", "docker", labels.LabelSourceAny),
	}

	err := ParseLabelPrefixCfg([]string{"k8s:io.kubernetes.pod.namespace", "k8s:k8s-app", "k8s:app", "k8s:name"}, "")
	c.Assert(err, IsNil)
	dlpcfg := validLabelPrefixes
	allNormalLabels := map[string]string{
		"io.cilium.k8s.namespace.labels": "foo",
		"k8s-app-team":                   "foo",
		"app-production":                 "foo",
		"name-defined":                   "foo",
	}
	allLabels := labels.Map2Labels(allNormalLabels, labels.LabelSourceK8s)
	filtered, _ := dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 4)

	// Reserved labels are included.
	allLabels["host"] = labels.NewLabel("host", "", labels.LabelSourceReserved)
	filtered, _ = dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 5)

	// io.kubernetes.pod.namespace=docker matches because the default list has any:io.kubernetes.pod.namespace.
	allLabels["io.kubernetes.pod.namespace"] = labels.NewLabel("io.kubernetes.pod.namespace", "docker", labels.LabelSourceAny)
	filtered, _ = dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 6)

	// container:k8s-app-role=foo doesn't match because it doesn't have source k8s.
	allLabels["k8s-app-role"] = labels.NewLabel("k8s-app-role", "foo", labels.LabelSourceContainer)
	filtered, _ = dlpcfg.filterLabels(allLabels)
	c.Assert(len(filtered), Equals, 6)
	c.Assert(filtered, checker.DeepEquals, wanted)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	prow_config "k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/testgrid/config/yaml2proto"
)

func main() {
	args := os.Args[1:]

	if len(args) != 3 {
		fmt.Println("Missing args - usage: go run jenkins_validate.go <path/to/job_collection> <path/to/prow> <path/to/testgrid_config>")
		os.Exit(1)
	}

	jobPath := args[0]
	prowPath := args[1]
	configPath := args[2]

	jobs := make(map[string]bool)
	files, err := filepath.Glob(jobPath + "/*")
	if err != nil {
		fmt.Println("Failed to collect outputs.")
		os.Exit(1)
	}

	for _, file := range files {
		file = strings.TrimPrefix(file, jobPath+"/")
		jobs[file] = false
	}

	data, err := ioutil.ReadFile(configPath)
	if err != nil {
		fmt.Printf("Failed reading %v\n", configPath)
		os.Exit(1)
	}

	c := yaml2proto.Config{}
	if err := c.Update(data); err != nil {
		fmt.Printf("Failed to convert yaml to protobuf: %v\n", err)
		os.Exit(1)
	}

	config, err := c.Raw()
	if err != nil {
		fmt.Printf("Error validating config: %v\n", err)
		os.Exit(1)
	}

	prowConfig, err := prow_config.Load(prowPath + "/config.yaml")
	if err != nil {
		fmt.Printf("Could not load prow configs: %v\n", err)
		os.Exit(1)
	}

	// Also check k/k presubmit, prow postsubmit and periodic jobs
	for _, job := range prowConfig.AllPresubmits([]string{}) {
		jobs[job.Name] = false
	}

	for _, job := range prowConfig.AllPostsubmits([]string{}) {
		if job.Agent != "jenkins" {
			jobs[job.Name] = false
		}
	}

	for _, job := range prowConfig.AllPeriodics() {
		if job.Agent != "jenkins" {
			jobs[job.Name] = false
		}
	}

	// For now anything outsite k8s-jenkins/(pr-)logs are considered to be fine
	testgroups := make(map[string]bool)
	for _, testgroup := range config.TestGroups {
		if strings.Contains(testgroup.GcsPrefix, "kubernetes-jenkins/logs/") {
			job := strings.TrimPrefix(testgroup.GcsPrefix, "kubernetes-jenkins/logs/")
			testgroups[job] = false
		}

		if strings.Contains(testgroup.GcsPrefix, "kubernetes-jenkins/pr-logs/directory/") {
			job := strings.TrimPrefix(testgroup.GcsPrefix, "kubernetes-jenkins/pr-logs/directory/")
			testgroups[job] = false
		}
	}

	// Cross check
	// -- Each job need to have a match testgrid group
	for job := range jobs {
		if _, ok := testgroups[job]; ok {
			testgroups[job] = true
			jobs[job] = true
		}
	}

	// Conclusion
	badjobs := []string{}
	for job, valid := range jobs {
		if !valid {
			badjobs = append(badjobs, job)
			fmt.Printf("Job %v does not have a matching testgrid testgroup\n", job)
		}
	}

	badconfigs := []string{}
	for testgroup, valid := range testgroups {
		if !valid {
			badconfigs = append(badconfigs, testgroup)
			fmt.Printf("Testgrid group %v does not have a matching jenkins or prow job\n", testgroup)
		}
	}

	if len(badconfigs) > 0 {
		fmt.Printf("Total bad config(s) - %v\n", len(badconfigs))
	}

	if len(badjobs) > 0 {
		fmt.Printf("Total bad job(s) - %v\n", len(badjobs))
	}

	if len(badconfigs) > 0 || len(badjobs) > 0 {
		os.Exit(1)
	}
}
<commit_msg>Revert "* Try to fix failure"<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	prow_config "k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/testgrid/config/yaml2proto"
)

func main() {
	args := os.Args[1:]

	if len(args) != 3 {
		fmt.Println("Missing args - usage: go run jenkins_validate.go <path/to/job_collection> <path/to/prow> <path/to/testgrid_config>")
		os.Exit(1)
	}

	jobPath := args[0]
	prowPath := args[1]
	configPath := args[2]

	jobs := make(map[string]bool)
	files, err := filepath.Glob(jobPath + "/*")
	if err != nil {
		fmt.Println("Failed to collect outputs.")
		os.Exit(1)
	}

	for _, file := range files {
		file = strings.TrimPrefix(file, jobPath+"/")
		jobs[file] = false
	}

	data, err := ioutil.ReadFile(configPath)
	if err != nil {
		fmt.Printf("Failed reading %v\n", configPath)
		os.Exit(1)
	}

	c := yaml2proto.Config{}
	if err := c.Update(data); err != nil {
		fmt.Printf("Failed to convert yaml to protobuf: %v\n", err)
		os.Exit(1)
	}

	config, err := c.Raw()
	if err != nil {
		fmt.Printf("Error validating config: %v\n", err)
		os.Exit(1)
	}

	prowConfig, err := prow_config.Load(prowPath + "/config.yaml")
	if err != nil {
		fmt.Printf("Could not load prow configs: %v\n", err)
		os.Exit(1)
	}

	// Also check k/k presubmit, prow postsubmit and periodic jobs
	for _, job := range prowConfig.AllPresubmits([]string{"kubernetes/kubernetes"}) {
		jobs[job.Name] = false
	}

	for _, job := range prowConfig.AllPostsubmits([]string{}) {
		if job.Agent != "jenkins" {
			jobs[job.Name] = false
		}
	}

	for _, job := range prowConfig.AllPeriodics() {
		if job.Agent != "jenkins" {
			jobs[job.Name] = false
		}
	}

	// For now anything outsite k8s-jenkins/(pr-)logs are considered to be fine
	testgroups := make(map[string]bool)
	for _, testgroup := range config.TestGroups {
		if strings.Contains(testgroup.GcsPrefix, "kubernetes-jenkins/logs/") {
			job := strings.TrimPrefix(testgroup.GcsPrefix, "kubernetes-jenkins/logs/")
			testgroups[job] = false
		}

		if strings.Contains(testgroup.GcsPrefix, "kubernetes-jenkins/pr-logs/directory/") {
			job := strings.TrimPrefix(testgroup.GcsPrefix, "kubernetes-jenkins/pr-logs/directory/")
			testgroups[job] = false
		}
	}

	// Cross check
	// -- Each job need to have a match testgrid group
	for job := range jobs {
		if _, ok := testgroups[job]; ok {
			testgroups[job] = true
			jobs[job] = true
		}
	}

	// Conclusion
	badjobs := []string{}
	for job, valid := range jobs {
		if !valid {
			badjobs = append(badjobs, job)
			fmt.Printf("Job %v does not have a matching testgrid testgroup\n", job)
		}
	}

	badconfigs := []string{}
	for testgroup, valid := range testgroups {
		if !valid {
			badconfigs = append(badconfigs, testgroup)
			fmt.Printf("Testgrid group %v does not have a matching jenkins or prow job\n", testgroup)
		}
	}

	if len(badconfigs) > 0 {
		fmt.Printf("Total bad config(s) - %v\n", len(badconfigs))
	}

	if len(badjobs) > 0 {
		fmt.Printf("Total bad job(s) - %v\n", len(badjobs))
	}

	if len(badconfigs) > 0 || len(badjobs) > 0 {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"
	"text/template"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/flag/stringsetflag"
)

type app struct {
	out io.Writer

	packageName string
	typeNames   stringsetflag.Flag
	outFile     string
	header      string
}

const help = `Usage of %s:

%s is a go-generator program that generates PropertyConverter implementations
for types produced by protoc. It can be used in a go generation file like:

  //go:generate <protoc command>
  //go:generate proto-gae -type MessageType -type OtherMessageType

This will produce a new file which implements the ToProperty and FromProperty
methods for the named types.

Options:
`

const copyright = `// Copyright 2016 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.
`

func (a *app) parseArgs(fs *flag.FlagSet, args []string) error {
	fs.SetOutput(a.out)
	fs.Usage = func() {
		fmt.Fprintf(a.out, help, args[0], args[0])
		fs.PrintDefaults()
	}

	fs.Var(&a.typeNames, "type",
		"A generated proto.Message type to generate stubs for (required, repeatable)")
	fs.StringVar(&a.outFile, "out", "proto_gae.gen.go",
		"The name of the output file")
	fs.StringVar(&a.header, "header", copyright, "Header text to put at the top of "+
		"the generated file. Defaults to the LUCI Authors copyright.")

	if err := fs.Parse(args[1:]); err != nil {
		return err
	}
	fail := errors.MultiError(nil)
	if a.typeNames.Data == nil || a.typeNames.Data.Len() == 0 {
		fail = append(fail, errors.New("must specify one or more -type"))
	}
	if !strings.HasSuffix(a.outFile, ".go") {
		fail = append(fail, errors.New("-output must end with '.go'"))
	}
	if len(fail) > 0 {
		for _, e := range fail {
			fmt.Fprintln(a.out, "error:", e)
		}
		fmt.Fprintln(a.out)
		fs.Usage()
		return fail
	}
	return nil
}

var tmpl = template.Must(
	template.New("main").Parse(`{{if index . "header"}}{{index . "header"}}
{{end}}// AUTOGENERATED: Do not edit

package {{index . "package"}}

import (
	"github.com/golang/protobuf/proto"

	"go.chromium.org/gae/service/datastore"
){{range index . "types"}}

var _ datastore.PropertyConverter = (*{{.}})(nil)

// ToProperty implements datastore.PropertyConverter. It causes an embedded
// '{{.}}' to serialize to an unindexed '[]byte' when used with the
// "go.chromium.org/gae" library.
func (p *{{.}}) ToProperty() (prop datastore.Property, err error) {
	data, err := proto.Marshal(p)
	if err == nil {
		prop.SetValue(data, datastore.NoIndex)
	}
	return
}

// FromProperty implements datastore.PropertyConverter. It parses a '[]byte'
// into an embedded '{{.}}' when used with the "go.chromium.org/gae" library.
func (p *{{.}}) FromProperty(prop datastore.Property) error {
	data, err := prop.Project(datastore.PTBytes)
	if err != nil {
		return err
	}
	return proto.Unmarshal(data.([]byte), p)
}{{end}}
`))

func (a *app) writeTo(w io.Writer) error {
	typeNames := a.typeNames.Data.ToSlice()
	sort.Strings(typeNames)

	return tmpl.Execute(w, map[string]interface{}{
		"package": a.packageName,
		"types":   typeNames,
		"header":  a.header,
	})
}

func (a *app) main() {
	if err := a.parseArgs(flag.NewFlagSet(os.Args[0], flag.ContinueOnError), os.Args); err != nil {
		os.Exit(1)
	}
	ofile, err := os.Create(a.outFile)
	if err != nil {
		fmt.Fprintf(a.out, "error: %s", err)
		os.Exit(2)
	}
	closeFn := func(delete bool) {
		if ofile != nil {
			if err := ofile.Close(); err != nil {
				fmt.Fprintf(a.out, "error while closing file: %s", err)
			}
			if delete {
				if err := os.Remove(a.outFile); err != nil {
					fmt.Fprintf(a.out, "failed to remove file!")
				}
			}
		}
		ofile = nil
	}
	defer closeFn(false)
	buf := bufio.NewWriter(ofile)
	err = a.writeTo(buf)
	if err != nil {
		fmt.Fprintf(a.out, "error while writing: %s", err)
		closeFn(true)
		os.Exit(3)
	}
	if err := buf.Flush(); err != nil {
		fmt.Fprintf(a.out, "error while writing: %s", err)
		closeFn(true)
		os.Exit(4)
	}
}

func main() {
	(&app{out: os.Stderr, packageName: os.Getenv("GOPACKAGE")}).main()
}
<commit_msg>[proto-gae] Update default copyright header.<commit_after>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"
	"text/template"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/flag/stringsetflag"
)

type app struct {
	out io.Writer

	packageName string
	typeNames   stringsetflag.Flag
	outFile     string
	header      string
}

const help = `Usage of %s:

%s is a go-generator program that generates PropertyConverter implementations
for types produced by protoc. It can be used in a go generation file like:

  //go:generate <protoc command>
  //go:generate proto-gae -type MessageType -type OtherMessageType

This will produce a new file which implements the ToProperty and FromProperty
methods for the named types.

Options:
`

const copyright = `// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
`

func (a *app) parseArgs(fs *flag.FlagSet, args []string) error {
	fs.SetOutput(a.out)
	fs.Usage = func() {
		fmt.Fprintf(a.out, help, args[0], args[0])
		fs.PrintDefaults()
	}

	fs.Var(&a.typeNames, "type",
		"A generated proto.Message type to generate stubs for (required, repeatable)")
	fs.StringVar(&a.outFile, "out", "proto_gae.gen.go",
		"The name of the output file")
	fs.StringVar(&a.header, "header", copyright, "Header text to put at the top of "+
		"the generated file. Defaults to the LUCI Authors copyright.")

	if err := fs.Parse(args[1:]); err != nil {
		return err
	}
	fail := errors.MultiError(nil)
	if a.typeNames.Data == nil || a.typeNames.Data.Len() == 0 {
		fail = append(fail, errors.New("must specify one or more -type"))
	}
	if !strings.HasSuffix(a.outFile, ".go") {
		fail = append(fail, errors.New("-output must end with '.go'"))
	}
	if len(fail) > 0 {
		for _, e := range fail {
			fmt.Fprintln(a.out, "error:", e)
		}
		fmt.Fprintln(a.out)
		fs.Usage()
		return fail
	}
	return nil
}

var tmpl = template.Must(
	template.New("main").Parse(`{{if index . "header"}}{{index . "header"}}
{{end}}// AUTOGENERATED: Do not edit

package {{index . "package"}}

import (
	"github.com/golang/protobuf/proto"

	"go.chromium.org/gae/service/datastore"
){{range index . "types"}}

var _ datastore.PropertyConverter = (*{{.}})(nil)

// ToProperty implements datastore.PropertyConverter. It causes an embedded
// '{{.}}' to serialize to an unindexed '[]byte' when used with the
// "go.chromium.org/gae" library.
func (p *{{.}}) ToProperty() (prop datastore.Property, err error) {
	data, err := proto.Marshal(p)
	if err == nil {
		prop.SetValue(data, datastore.NoIndex)
	}
	return
}

// FromProperty implements datastore.PropertyConverter. It parses a '[]byte'
// into an embedded '{{.}}' when used with the "go.chromium.org/gae" library.
func (p *{{.}}) FromProperty(prop datastore.Property) error {
	data, err := prop.Project(datastore.PTBytes)
	if err != nil {
		return err
	}
	return proto.Unmarshal(data.([]byte), p)
}{{end}}
`))

func (a *app) writeTo(w io.Writer) error {
	typeNames := a.typeNames.Data.ToSlice()
	sort.Strings(typeNames)

	return tmpl.Execute(w, map[string]interface{}{
		"package": a.packageName,
		"types":   typeNames,
		"header":  a.header,
	})
}

func (a *app) main() {
	if err := a.parseArgs(flag.NewFlagSet(os.Args[0], flag.ContinueOnError), os.Args); err != nil {
		os.Exit(1)
	}
	ofile, err := os.Create(a.outFile)
	if err != nil {
		fmt.Fprintf(a.out, "error: %s", err)
		os.Exit(2)
	}
	closeFn := func(delete bool) {
		if ofile != nil {
			if err := ofile.Close(); err != nil {
				fmt.Fprintf(a.out, "error while closing file: %s", err)
			}
			if delete {
				if err := os.Remove(a.outFile); err != nil {
					fmt.Fprintf(a.out, "failed to remove file!")
				}
			}
		}
		ofile = nil
	}
	defer closeFn(false)
	buf := bufio.NewWriter(ofile)
	err = a.writeTo(buf)
	if err != nil {
		fmt.Fprintf(a.out, "error while writing: %s", err)
		closeFn(true)
		os.Exit(3)
	}
	if err := buf.Flush(); err != nil {
		fmt.Fprintf(a.out, "error while writing: %s", err)
		closeFn(true)
		os.Exit(4)
	}
}

func main() {
	(&app{out: os.Stderr, packageName: os.Getenv("GOPACKAGE")}).main()
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"testing"
	"time"
)

const requestTimeout = time.Second * 5

var (
	edgeHost    = flag.String("edgeHost", "www.gov.uk", "Hostname of edge")
	originPort  = flag.Int("originPort", 8080, "Origin port to listen on for requests")
	insecureTLS = flag.Bool("insecureTLS", false, "Whether to check server certificates")

	client       *http.Transport
	originServer *CDNServeMux
)

// Setup clients and servers.
func init() {

	flag.Parse()

	tlsOptions := &tls.Config{}
	if *insecureTLS {
		tlsOptions.InsecureSkipVerify = true
	}

	client = &http.Transport{
		ResponseHeaderTimeout: requestTimeout,
		TLSClientConfig:       tlsOptions,
	}
	originServer = StartServer(*originPort)

	log.Println("Confirming that CDN has successfully probed Origin")
	err := confirmOriginIsEnabled(originServer, *edgeHost)
	if err != nil {
		log.Fatal(err)
	}
}

func TestHelpers(t *testing.T) {
	testHelpersCDNServeMuxHandlers(t, originServer)
	testHelpersCDNServeMuxProbes(t, originServer)
}

// Should redirect from HTTP to HTTPS without hitting origin.
func TestProtocolRedirect(t *testing.T) {
	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		t.Error("Request should not have made it to origin")
	})

	sourceUrl := fmt.Sprintf("http://%s/foo/bar", *edgeHost)
	destUrl := fmt.Sprintf("https://%s/foo/bar", *edgeHost)

	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 301 {
		t.Errorf("Status code expected 301, got %d", resp.StatusCode)
	}
	if d := resp.Header.Get("Location"); d != destUrl {
		t.Errorf("Location header expected %s, got %s", destUrl, d)
	}
}

// Should send request to origin by default
func TestRequestsGoToOriginByDefault(t *testing.T) {
	uuid := NewUUID()
	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" && r.URL.Path == fmt.Sprintf("/%s", uuid) {
			w.Header().Set("EnsureOriginServed", uuid)
		}
	})

	sourceUrl := fmt.Sprintf("https://%s/%s", *edgeHost, uuid)

	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 200 {
		t.Errorf("Status code expected 200, got %d", resp.StatusCode)
	}
	if d := resp.Header.Get("EnsureOriginServed"); d != uuid {
		t.Errorf("EnsureOriginServed header has not come from Origin: expected %q, got %q", uuid, d)
	}

}

// Should cache first response and return it on second request without
// hitting origin again.
func TestFirstResponseCached(t *testing.T) {
	const bodyExpected = "first request"
	const requestsExpectedCount = 1
	requestsReceivedCount := 0

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if requestsReceivedCount == 0 {
			w.Write([]byte(bodyExpected))
		} else {
			w.Write([]byte("subsequent request"))
		}

		requestsReceivedCount++
	})

	url := fmt.Sprintf("https://%s/%s", *edgeHost, NewUUID())
	req, _ := http.NewRequest("GET", url, nil)

	for i := 0; i < 2; i++ {
		resp, err := client.RoundTrip(req)
		if err != nil {
			t.Fatal(err)
		}

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Fatal(err)
		}
		if string(body) != bodyExpected {
			t.Errorf("Incorrect response body. Expected %q, got %q", bodyExpected, body)
		}
	}

	if requestsReceivedCount > requestsExpectedCount {
		t.Errorf("originServer got too many requests. Expected %d requests, got %d", requestsExpectedCount, requestsReceivedCount)
	}
}

// Should return 403 for PURGE requests from IPs not in the whitelist.
func TestRestrictPurgeRequests(t *testing.T) {
	t.Error("Not implemented")
}

// Should create an X-Forwarded-For header containing the client's IP.
func TestHeaderCreateXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should append client's IP to existing X-Forwarded-For header.
func TestHeaderAppendXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should create a True-Client-IP header containing the client's IP
// address, discarding the value provided in the original request.
func TestHeaderUnspoofableClientIP(t *testing.T) {
	t.Error("Not implemented")
}

// Should not modify Host header from original request.
func TestHeaderHostUnmodified(t *testing.T) {
	t.Error("Not implemented")
}

// Should set a default TTL if the response doesn't set one.
func TestDefaultTTL(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin is down and
// object is beyond TTL but still in cache.
func TestFailoverOriginDownServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin returns a 5xx
// response and object is beyond TTL but still in cache.
func TestFailoverOrigin5xxServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin is down and object is not in
// cache (active or stale).
func TestFailoverOriginDownUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin returns 5xx response and object
// is not in cache (active or stale).
func TestFailoverOrigin5xxUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror are
// down.
func TestFailoverOriginDownFirstMirrorDownUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror return
// 5xx responses.
func TestFailoverOrigin5xxFirstMirror5xxUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should not fallback to mirror if origin returns a 5xx response with a
// No-Fallback header.
func TestFailoverNoFallbackHeader(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve a known static error page if cannot serve a page
// from origin, stale or any mirror.
// NB: ideally this should be a page that we control that has a mechanism
//     to alert us that it has been served.
func TestErrorPageIsServedWhenNoBackendAvailable(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Set-Cookie a header.
func TestNoCacheHeaderSetCookie(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Cache-Control: private header.
func TestNoCacheHeaderCacheControlPrivate(t *testing.T) {
	t.Error("Not implemented")
}

// ---------------------------------------------------------
// Test that useful common cache-related parameters are sent to the
// client by this CDN provider.

// Should set an Age header itself rather than passing the Age header from origin.
func TestAgeHeaderIsSetByProviderNotOrigin(t *testing.T) {
	t.Error("Not implemented")
}

// Should set an X-Cache header containing HIT/MISS from 'origin, itself'
func TestXCacheHeaderContainsHitMissFromBothProviderAndOrigin(t *testing.T) {
	t.Error("Not implemented")
}

// Should set an X-Served-By header giving information on the node and location served from.
func TestXServedByHeaderContainsANodeIdAndLocation(t *testing.T) {
	t.Error("Not implemented")
}

// Should set an X-Cache-Hits header containing hit count for this object,
// from the provider not origin
func TestXCacheHitsContainsProviderHitCountForThisObject(t *testing.T) {
	t.Error("Not implemented")
}
<commit_msg>Test that for X-Served-By is set by Edge<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"testing"
	"time"
)

const requestTimeout = time.Second * 5

var (
	edgeHost    = flag.String("edgeHost", "www.gov.uk", "Hostname of edge")
	originPort  = flag.Int("originPort", 8080, "Origin port to listen on for requests")
	insecureTLS = flag.Bool("insecureTLS", false, "Whether to check server certificates")

	client       *http.Transport
	originServer *CDNServeMux
)

// Setup clients and servers.
func init() {

	flag.Parse()

	tlsOptions := &tls.Config{}
	if *insecureTLS {
		tlsOptions.InsecureSkipVerify = true
	}

	client = &http.Transport{
		ResponseHeaderTimeout: requestTimeout,
		TLSClientConfig:       tlsOptions,
	}
	originServer = StartServer(*originPort)

	log.Println("Confirming that CDN has successfully probed Origin")
	err := confirmOriginIsEnabled(originServer, *edgeHost)
	if err != nil {
		log.Fatal(err)
	}
}

func TestHelpers(t *testing.T) {
	testHelpersCDNServeMuxHandlers(t, originServer)
	testHelpersCDNServeMuxProbes(t, originServer)
}

// Should redirect from HTTP to HTTPS without hitting origin.
func TestProtocolRedirect(t *testing.T) {
	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		t.Error("Request should not have made it to origin")
	})

	sourceUrl := fmt.Sprintf("http://%s/foo/bar", *edgeHost)
	destUrl := fmt.Sprintf("https://%s/foo/bar", *edgeHost)

	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 301 {
		t.Errorf("Status code expected 301, got %d", resp.StatusCode)
	}
	if d := resp.Header.Get("Location"); d != destUrl {
		t.Errorf("Location header expected %s, got %s", destUrl, d)
	}
}

// Should send request to origin by default
func TestRequestsGoToOriginByDefault(t *testing.T) {
	uuid := NewUUID()
	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" && r.URL.Path == fmt.Sprintf("/%s", uuid) {
			w.Header().Set("EnsureOriginServed", uuid)
		}
	})

	sourceUrl := fmt.Sprintf("https://%s/%s", *edgeHost, uuid)

	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)

	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 200 {
		t.Errorf("Status code expected 200, got %d", resp.StatusCode)
	}
	if d := resp.Header.Get("EnsureOriginServed"); d != uuid {
		t.Errorf("EnsureOriginServed header has not come from Origin: expected %q, got %q", uuid, d)
	}

}

// Should cache first response and return it on second request without
// hitting origin again.
func TestFirstResponseCached(t *testing.T) {
	const bodyExpected = "first request"
	const requestsExpectedCount = 1
	requestsReceivedCount := 0

	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if requestsReceivedCount == 0 {
			w.Write([]byte(bodyExpected))
		} else {
			w.Write([]byte("subsequent request"))
		}

		requestsReceivedCount++
	})

	url := fmt.Sprintf("https://%s/%s", *edgeHost, NewUUID())
	req, _ := http.NewRequest("GET", url, nil)

	for i := 0; i < 2; i++ {
		resp, err := client.RoundTrip(req)
		if err != nil {
			t.Fatal(err)
		}

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Fatal(err)
		}
		if string(body) != bodyExpected {
			t.Errorf("Incorrect response body. Expected %q, got %q", bodyExpected, body)
		}
	}

	if requestsReceivedCount > requestsExpectedCount {
		t.Errorf("originServer got too many requests. Expected %d requests, got %d", requestsExpectedCount, requestsReceivedCount)
	}
}

// Should return 403 for PURGE requests from IPs not in the whitelist.
func TestRestrictPurgeRequests(t *testing.T) {
	t.Error("Not implemented")
}

// Should create an X-Forwarded-For header containing the client's IP.
func TestHeaderCreateXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should append client's IP to existing X-Forwarded-For header.
func TestHeaderAppendXFF(t *testing.T) {
	t.Error("Not implemented")
}

// Should create a True-Client-IP header containing the client's IP
// address, discarding the value provided in the original request.
func TestHeaderUnspoofableClientIP(t *testing.T) {
	t.Error("Not implemented")
}

// Should not modify Host header from original request.
func TestHeaderHostUnmodified(t *testing.T) {
	t.Error("Not implemented")
}

// Should set a default TTL if the response doesn't set one.
func TestDefaultTTL(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin is down and
// object is beyond TTL but still in cache.
func TestFailoverOriginDownServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve stale object and not hit mirror(s) if origin returns a 5xx
// response and object is beyond TTL but still in cache.
func TestFailoverOrigin5xxServeStale(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin is down and object is not in
// cache (active or stale).
func TestFailoverOriginDownUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to first mirror if origin returns 5xx response and object
// is not in cache (active or stale).
func TestFailoverOrigin5xxUseFirstMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror are
// down.
func TestFailoverOriginDownFirstMirrorDownUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should fallback to second mirror if both origin and first mirror return
// 5xx responses.
func TestFailoverOrigin5xxFirstMirror5xxUseSecondMirror(t *testing.T) {
	t.Error("Not implemented")
}

// Should not fallback to mirror if origin returns a 5xx response with a
// No-Fallback header.
func TestFailoverNoFallbackHeader(t *testing.T) {
	t.Error("Not implemented")
}

// Should serve a known static error page if cannot serve a page
// from origin, stale or any mirror.
// NB: ideally this should be a page that we control that has a mechanism
//     to alert us that it has been served.
func TestErrorPageIsServedWhenNoBackendAvailable(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Set-Cookie a header.
func TestNoCacheHeaderSetCookie(t *testing.T) {
	t.Error("Not implemented")
}

// Should not cache a response with a Cache-Control: private header.
func TestNoCacheHeaderCacheControlPrivate(t *testing.T) {
	t.Error("Not implemented")
}

// ---------------------------------------------------------
// Test that useful common cache-related parameters are sent to the
// client by this CDN provider.

// Should set an Age header itself rather than passing the Age header from origin.
func TestAgeHeaderIsSetByProviderNotOrigin(t *testing.T) {
	t.Error("Not implemented")
}

// Should set an X-Cache header containing HIT/MISS from 'origin, itself'
func TestXCacheHeaderContainsHitMissFromBothProviderAndOrigin(t *testing.T) {
	t.Error("Not implemented")
}

// Should set an X-Served-By header giving information on the node and location served from.
func TestXServedByHeaderContainsANodeIdAndLocation(t *testing.T) {

	uuid := NewUUID()
	originServer.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" && r.URL.Path == fmt.Sprintf("/%s", uuid) {
			w.WriteHeader(200)
		}
	})

	sourceUrl := fmt.Sprintf("https://%s/%s", *edgeHost, uuid)

	// Get first request, will come from origin. Edge Hit Count 0
	req, _ := http.NewRequest("GET", sourceUrl, nil)
	resp, err := client.RoundTrip(req)
	if err != nil {
		t.Fatal(err)
	}

	actualHeader := resp.Header.Get("X-Served-By")
	if actualHeader == "" {
		t.Error("X-Served-By header has not been set by Edge")
	}

	re := regexp.MustCompile("^cache-[a-z0-9]+-[A-Z]{3}$")
	if re.FindString(actualHeader) != actualHeader {
		t.Errorf("X-Served-By is not as expected: got %q", actualHeader)
	}

}

// Should set an X-Cache-Hits header containing hit count for this object,
// from the provider not origin
func TestXCacheHitsContainsProviderHitCountForThisObject(t *testing.T) {
	t.Error("Not implemented")
}
<|endoftext|>