<commit_before>package metrics

import (
	"context"

	"github.com/juju/errors"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	errCollectorHasBeenRegistered = "metrics: collector has been registered"
	errCollector                  = "metrics: register collector"
)

var collectors map[string]prometheus.Collector = make(map[string]prometheus.Collector, 0)

type Collector struct {
	ClusterCollector *ClusterCollector
}

func NewCollector(reg prometheus.Registerer, descs []*prometheus.Desc, zone string, hosts []string) *Collector {
	c := &ClusterCollector{
		descs: descs,
		hosts: hosts,
	}
	cc := &Collector{
		ClusterCollector: c,
	}
	prometheus.WrapRegistererWith(prometheus.Labels{"zone": zone}, reg).MustRegister(c)
	reg.MustRegister(
		prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
		prometheus.NewGoCollector(),
	)
	return cc
}

func (i *Collector) Prepare(ctx context.Context) (context.Context, error) {
	return ctx, nil
}
func (i *Collector) Initiate(ctx context.Context) (context.Context, error) {
	return ctx, nil
}
func (i *Collector) OnStartup(ctx context.Context) (context.Context, error) {
	return ctx, nil
}
func (i *Collector) OnShutdown(ctx context.Context) (context.Context, error) {
	return ctx, nil
}

func (i *Collector) RegisterCollector(name string, cs prometheus.Collector) error {
	if _, ok := collectors[name]; !ok {
		collectors[name] = cs
		prometheus.MustRegister(cs)
	} else {
		return errors.New(errCollectorHasBeenRegistered)
	}
	return nil
}
func (i *Collector) GetCollector(name string) (cs prometheus.Collector) {
	if _, ok := collectors[name]; ok {
		return collectors[name]
	} else {
		return nil
	}
}

type ClusterCollector struct {
	descs []*prometheus.Desc
	hosts []string
}

func (i *ClusterCollector) Describe(ch chan<- *prometheus.Desc) {
	prometheus.DescribeByCollect(i, ch)
}

func (i *ClusterCollector) Collect(ch chan<- prometheus.Metric) {
	for _, desc := range i.descs {
		for index, host := range i.hosts {
			ch <- prometheus.MustNewConstMetric(
				desc,
				prometheus.CounterValue,
				float64(index),
				host,
			)
		}

	}
}
<commit_msg>metrics<commit_after>package metrics

import (
	"context"

	"github.com/juju/errors"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	errCollectorHasRegistered  = "metrics: collector '%s' has registered"
	errRegisterCollector       = "metrics: register collector '%s'"
	errRegisterCustomCollector = "metrics: register custom collector '%s'"
)

type Metrics struct {
	Collectors map[string]prometheus.Collector
}

func (i *Metrics) Prepare(ctx context.Context) (context.Context, error) {
	return ctx, nil
}
func (i *Metrics) Initiate(ctx context.Context) (context.Context, error) {
	i.Collectors = make(map[string]prometheus.Collector, 0)
	return ctx, nil
}
func (i *Metrics) OnStartup(ctx context.Context) (context.Context, error) {
	return ctx, nil
}
func (i *Metrics) OnShutdown(ctx context.Context) (context.Context, error) {
	return ctx, nil
}

func (i *Metrics) RegisterCollector(name string, cs prometheus.Collector) error {
	if _, ok := i.Collectors[name]; !ok {
		i.Collectors[name] = cs
		prometheus.MustRegister(cs)
	} else {
		return errors.Annotate(errors.Errorf(errCollectorHasRegistered, name), errRegisterCollector)
	}
	return nil
}
func (i *Metrics) GetCollector(name string) (cs prometheus.Collector) {
	if _, ok := i.Collectors[name]; ok {
		return i.Collectors[name]
	} else {
		return nil
	}
}

func (i *Metrics) RegisterCustomCollector(name string, reg prometheus.Registerer, descs []*prometheus.Desc, zone string, hosts []string) error {
	if _, ok := i.Collectors[name]; !ok {
		c := &ClusterCollector{
			descs: descs,
			hosts: hosts,
		}
		prometheus.WrapRegistererWith(prometheus.Labels{"zone": zone}, reg).MustRegister(c)
		reg.MustRegister(
			prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
			prometheus.NewGoCollector(),
		)
		i.Collectors[name] = c
	} else {
		return errors.Annotate(errors.Errorf(errCollectorHasRegistered, name), errRegisterCustomCollector)
	}
	return nil
}

type ClusterCollector struct {
	descs []*prometheus.Desc
	hosts []string
}

func (i *ClusterCollector) Describe(ch chan<- *prometheus.Desc) {
	prometheus.DescribeByCollect(i, ch)
}

func (i *ClusterCollector) Collect(ch chan<- prometheus.Metric) {
	for _, desc := range i.descs {
		for index, host := range i.hosts {
			ch <- prometheus.MustNewConstMetric(
				desc,
				prometheus.CounterValue,
				float64(index),
				host,
			)
		}

	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"sync"

	"k8s.io/component-base/metrics"
)

/*
 * By default, all the following metrics are defined as falling under
 * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
 *
 * Promoting the stability level of the metric is a responsibility of the component owner, since it
 * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
 * the metric stability policy.
 */
var (
	unavailableGaugeDesc = metrics.NewDesc(
		"aggregator_unavailable_apiservice",
		"Gauge of APIServices which are marked as unavailable broken down by APIService name.",
		[]string{"name"},
		nil,
		metrics.ALPHA,
		"",
	)
)

type availabilityMetrics struct {
	unavailableCounter *metrics.CounterVec

	*availabilityCollector
}

func newAvailabilityMetrics() *availabilityMetrics {
	return &availabilityMetrics{
		unavailableCounter: metrics.NewCounterVec(
			&metrics.CounterOpts{
				Name:           "aggregator_unavailable_apiservice_total",
				Help:           "Counter of APIServices which are marked as unavailable broken down by APIService name and reason.",
				StabilityLevel: metrics.ALPHA,
			},
			[]string{"name", "reason"},
		),
		availabilityCollector: newAvailabilityCollector(),
	}
}

// Register registers apiservice availability metrics.
func (m *availabilityMetrics) Register(
	registrationFunc func(metrics.Registerable) error,
	customRegistrationFunc func(metrics.StableCollector) error,
) error {
	err := registrationFunc(m.unavailableCounter)
	if err != nil {
		return err
	}

	err = customRegistrationFunc(m.availabilityCollector)
	if err != nil {
		return err
	}

	return nil
}

// UnavailableCounter returns a counter to track apiservices marked as unavailable.
func (m *availabilityMetrics) UnavailableCounter(apiServiceName, reason string) metrics.CounterMetric {
	return m.unavailableCounter.WithLabelValues(apiServiceName, reason)
}

type availabilityCollector struct {
	metrics.BaseStableCollector

	mtx            sync.RWMutex
	availabilities map[string]bool
}

// Check if apiServiceStatusCollector implements necessary interface.
var _ metrics.StableCollector = &availabilityCollector{}

func newAvailabilityCollector() *availabilityCollector {
	return &availabilityCollector{
		availabilities: make(map[string]bool),
	}
}

// DescribeWithStability implements the metrics.StableCollector interface.
func (c *availabilityCollector) DescribeWithStability(ch chan<- *metrics.Desc) {
	ch <- unavailableGaugeDesc
}

// CollectWithStability implements the metrics.StableCollector interface.
func (c *availabilityCollector) CollectWithStability(ch chan<- metrics.Metric) {
	c.mtx.RLock()
	defer c.mtx.RUnlock()

	for apiServiceName, isAvailable := range c.availabilities {
		gaugeValue := 1.0
		if isAvailable {
			gaugeValue = 0.0
		}
		ch <- metrics.NewLazyConstMetric(
			unavailableGaugeDesc,
			metrics.GaugeValue,
			gaugeValue,
			apiServiceName,
		)
	}
}

// SetAPIServiceAvailable sets the given apiservice availability gauge to available.
func (c *availabilityCollector) SetAPIServiceAvailable(apiServiceKey string) {
	c.setAPIServiceAvailability(apiServiceKey, true)
}

// SetAPIServiceUnavailable sets the given apiservice availability gauge to unavailable.
func (c *availabilityCollector) SetAPIServiceUnavailable(apiServiceKey string) {
	c.setAPIServiceAvailability(apiServiceKey, false)
}

func (c *availabilityCollector) setAPIServiceAvailability(apiServiceKey string, availability bool) {
	c.mtx.Lock()
	defer c.mtx.Unlock()

	c.availabilities[apiServiceKey] = availability
}

// ForgetAPIService removes the availability gauge of the given apiservice.
func (c *availabilityCollector) ForgetAPIService(apiServiceKey string) {
	c.mtx.Lock()
	defer c.mtx.Unlock()

	delete(c.availabilities, apiServiceKey)
}
<commit_msg>fix broken link in some files<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"sync"

	"k8s.io/component-base/metrics"
)

/*
 * By default, all the following metrics are defined as falling under
 * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/kubernetes-control-plane-metrics-stability.md#stability-classes)
 *
 * Promoting the stability level of the metric is a responsibility of the component owner, since it
 * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
 * the metric stability policy.
 */
var (
	unavailableGaugeDesc = metrics.NewDesc(
		"aggregator_unavailable_apiservice",
		"Gauge of APIServices which are marked as unavailable broken down by APIService name.",
		[]string{"name"},
		nil,
		metrics.ALPHA,
		"",
	)
)

type availabilityMetrics struct {
	unavailableCounter *metrics.CounterVec

	*availabilityCollector
}

func newAvailabilityMetrics() *availabilityMetrics {
	return &availabilityMetrics{
		unavailableCounter: metrics.NewCounterVec(
			&metrics.CounterOpts{
				Name:           "aggregator_unavailable_apiservice_total",
				Help:           "Counter of APIServices which are marked as unavailable broken down by APIService name and reason.",
				StabilityLevel: metrics.ALPHA,
			},
			[]string{"name", "reason"},
		),
		availabilityCollector: newAvailabilityCollector(),
	}
}

// Register registers apiservice availability metrics.
func (m *availabilityMetrics) Register(
	registrationFunc func(metrics.Registerable) error,
	customRegistrationFunc func(metrics.StableCollector) error,
) error {
	err := registrationFunc(m.unavailableCounter)
	if err != nil {
		return err
	}

	err = customRegistrationFunc(m.availabilityCollector)
	if err != nil {
		return err
	}

	return nil
}

// UnavailableCounter returns a counter to track apiservices marked as unavailable.
func (m *availabilityMetrics) UnavailableCounter(apiServiceName, reason string) metrics.CounterMetric {
	return m.unavailableCounter.WithLabelValues(apiServiceName, reason)
}

type availabilityCollector struct {
	metrics.BaseStableCollector

	mtx            sync.RWMutex
	availabilities map[string]bool
}

// Check if apiServiceStatusCollector implements necessary interface.
var _ metrics.StableCollector = &availabilityCollector{}

func newAvailabilityCollector() *availabilityCollector {
	return &availabilityCollector{
		availabilities: make(map[string]bool),
	}
}

// DescribeWithStability implements the metrics.StableCollector interface.
func (c *availabilityCollector) DescribeWithStability(ch chan<- *metrics.Desc) {
	ch <- unavailableGaugeDesc
}

// CollectWithStability implements the metrics.StableCollector interface.
func (c *availabilityCollector) CollectWithStability(ch chan<- metrics.Metric) {
	c.mtx.RLock()
	defer c.mtx.RUnlock()

	for apiServiceName, isAvailable := range c.availabilities {
		gaugeValue := 1.0
		if isAvailable {
			gaugeValue = 0.0
		}
		ch <- metrics.NewLazyConstMetric(
			unavailableGaugeDesc,
			metrics.GaugeValue,
			gaugeValue,
			apiServiceName,
		)
	}
}

// SetAPIServiceAvailable sets the given apiservice availability gauge to available.
func (c *availabilityCollector) SetAPIServiceAvailable(apiServiceKey string) {
	c.setAPIServiceAvailability(apiServiceKey, true)
}

// SetAPIServiceUnavailable sets the given apiservice availability gauge to unavailable.
func (c *availabilityCollector) SetAPIServiceUnavailable(apiServiceKey string) {
	c.setAPIServiceAvailability(apiServiceKey, false)
}

func (c *availabilityCollector) setAPIServiceAvailability(apiServiceKey string, availability bool) {
	c.mtx.Lock()
	defer c.mtx.Unlock()

	c.availabilities[apiServiceKey] = availability
}

// ForgetAPIService removes the availability gauge of the given apiservice.
func (c *availabilityCollector) ForgetAPIService(apiServiceKey string) {
	c.mtx.Lock()
	defer c.mtx.Unlock()

	delete(c.availabilities, apiServiceKey)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package stats

import (
	"k8s.io/kubernetes/pkg/api/resource"
	"k8s.io/kubernetes/pkg/api/unversioned"
)

// Summary is a top-level container for holding NodeStats and PodStats.
type Summary struct {
	// Overall node stats.
	Node NodeStats `json:"node"`
	// Per-pod stats.
	Pods []PodStats `json:"pods"`
}

// NodeStats holds node-level unprocessed sample stats.
type NodeStats struct {
	// Reference to the measured Node.
	NodeName string `json:"nodeName"`
	// Overall node stats.
	Total []NodeSample `json:"total,omitempty" patchStrategy:"merge" patchMergeKey:"sampleTime"`
	// Stats of system daemons tracked as raw containers.
	// The system containers are named according to the SystemContainer* constants.
	SystemContainers []ContainerStats `json:"systemContainers,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
}

const (
	// Container name for the system container tracking Kubelet usage.
	SystemContainerKubelet = "/kubelet"
	// Container name for the system container tracking the runtime (e.g. docker or rkt) usage.
	SystemContainerRuntime = "/runtime"
	// Container name for the system container tracking non-kubernetes processes.
	SystemContainerMisc = "/misc"
)

// PodStats holds pod-level unprocessed sample stats.
type PodStats struct {
	// Reference to the measured Pod.
	PodRef NonLocalObjectReference `json:"podRef"`
	// Stats of containers in the measured pod.
	Containers []ContainerStats `json:"containers" patchStrategy:"merge" patchMergeKey:"name"`
	// Historical stat samples of pod-level resources.
	Samples []PodSample `json:"samples" patchStrategy:"merge" patchMergeKey:"sampleTime"`
}

// ContainerStats holds container-level unprocessed sample stats.
type ContainerStats struct {
	// Reference to the measured container.
	Name string `json:"name"`
	// Historical stat samples gathered from the container.
	Samples []ContainerSample `json:"samples" patchStrategy:"merge" patchMergeKey:"sampleTime"`
}

// NonLocalObjectReference contains enough information to locate the referenced object.
type NonLocalObjectReference struct {
	Name      string `json:"name"`
	Namespace string `json:"namespace"`
}

// Sample defines metadata common to all sample types.
// Samples may not be nested within other samples.
type Sample struct {
	// The time this data point was collected at.
	SampleTime unversioned.Time `json:"sampleTime"`
}

// NodeSample contains a sample point of data aggregated over a node.
type NodeSample struct {
	Sample `json:",inline"`
	// Stats pertaining to CPU resources.
	CPU *CPUStats `json:"cpu,omitempty"`
	// Stats pertaining to memory (RAM) resources.
	Memory *MemoryStats `json:"memory,omitempty"`
	// Stats pertaining to network resources.
	Network *NetworkStats `json:"network,omitempty"`
	// Stats pertaining to filesystem resources. Reported per-device.
	Filesystem []FilesystemStats `json:"filesystem,omitempty" patchStrategy:"merge" patchMergeKey:"device"`
}

// PodSample contains a sample point of pod-level resources.
type PodSample struct {
	Sample `json:",inline"`
	// Stats pertaining to network resources.
	Network *NetworkStats `json:"network,omitempty"`
}

// ContainerSample contains a sample point of container-level resources.
type ContainerSample struct {
	Sample `json:",inline"`
	// Stats pertaining to CPU resources.
	CPU *CPUStats `json:"cpu,omitempty"`
	// Stats pertaining to memory (RAM) resources.
	Memory *MemoryStats `json:"memory,omitempty"`
	// Stats pertaining to filesystem resources. Reported per-device.
	Filesystem []FilesystemStats `json:"filesystem,omitempty" patchStrategy:"merge" patchMergeKey:"device"`
}

// NetworkStats contains data about network resources.
type NetworkStats struct {
	// Cumulative count of bytes received.
	RxBytes *resource.Quantity `json:"rxBytes,omitempty"`
	// Cumulative count of receive errors encountered.
	RxErrors *int64 `json:"rxErrors,omitempty"`
	// Cumulative count of bytes transmitted.
	TxBytes *resource.Quantity `json:"txBytes,omitempty"`
	// Cumulative count of transmit errors encountered.
	TxErrors *int64 `json:"txErrors,omitempty"`
}

// CPUStats contains data about CPU usage.
type CPUStats struct {
	// Total CPU usage (sum of all cores) averaged over the sample window.
	// The "core" unit can be interpreted as CPU core-seconds per second.
	UsageCores *resource.Quantity `json:"usageCores,omitempty"`
	// Cumulative CPU usage (sum of all cores) since object creation.
	UsageCoreSeconds *resource.Quantity `json:"usageCoreSeconds,omitempty"`
}

// MemoryStats contains data about memory usage.
type MemoryStats struct {
	// Total memory in use. This includes all memory regardless of when it was accessed.
	UsageBytes *resource.Quantity `json:"usageBytes,omitempty"`
	// The amount of working set memory. This includes recently accessed memory,
	// dirty memory, and kernel memory. UsageBytes is <= TotalBytes.
	WorkingSetBytes *resource.Quantity `json:"workingSetBytes,omitempty"`
	// Cumulative number of minor page faults.
	PageFaults *int64 `json:"pageFaults,omitempty"`
	// Cumulative number of major page faults.
	MajorPageFaults *int64 `json:"majorPageFaults,omitempty"`
}

// FilesystemStats contains data about filesystem usage.
type FilesystemStats struct {
	// The block device name associated with the filesystem.
	Device string `json:"device"`
	// Number of bytes that is consumed by the container on this filesystem.
	UsageBytes *resource.Quantity `json:"usageBytes,omitempty"`
	// Number of bytes that can be consumed by the container on this filesystem.
	LimitBytes *resource.Quantity `json:"limitBytes,omitempty"`
}

// StatsOptions are the query options for raw stats endpoints.
type StatsOptions struct {
	// Only include samples with sampleTime equal to or more recent than this time.
	// This does not affect cumulative values, which are cumulative from object creation.
	SinceTime *unversioned.Time `json:"sinceTime,omitempty"`
	// Only include samples with sampleTime less recent than this time.
	UntilTime *unversioned.Time `json:"untilTime,omitempty"`
	// Specifies the maximum number of elements in any list of samples.
	// When the total number of samples exceeds the maximum the most recent MaxSamples samples are
	// returned.
	MaxSamples int `json:"maxSamples,omitempty"`
}
<commit_msg>Update from offline discussions<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package stats

import (
	"k8s.io/kubernetes/pkg/api/resource"
	"k8s.io/kubernetes/pkg/api/unversioned"
)

// Summary is a top-level container for holding NodeStats and PodStats.
type Summary struct {
	// The time the most recent data included in this summary was collect at, rounded to the nearest
	// second.
	Time unversioned.Time `json:"time"`
	// Overall node stats.
	Node NodeStats `json:"node"`
	// Per-pod stats.
	Pods []PodStats `json:"pods"`
}

// NodeStats holds node-level unprocessed sample stats.
type NodeStats struct {
	// Reference to the measured Node.
	NodeName string `json:"nodeName"`
	// Stats of system daemons tracked as raw containers.
	// The system containers are named according to the SystemContainer* constants.
	SystemContainers []ContainerStats `json:"systemContainers,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
	// Stats pertaining to CPU resources.
	CPU *CPUStats `json:"cpu,omitempty"`
	// Stats pertaining to memory (RAM) resources.
	Memory *MemoryStats `json:"memory,omitempty"`
	// Stats pertaining to network resources.
	Network *NetworkStats `json:"network,omitempty"`
}

const (
	// Container name for the system container tracking Kubelet usage.
	SystemContainerKubelet = "kubelet"
	// Container name for the system container tracking the runtime (e.g. docker or rkt) usage.
	SystemContainerRuntime = "runtime"
	// Container name for the system container tracking non-kubernetes processes.
	SystemContainerMisc = "misc"
)

// PodStats holds pod-level unprocessed sample stats.
type PodStats struct {
	// Reference to the measured Pod.
	PodRef NonLocalObjectReference `json:"podRef"`
	// Stats of containers in the measured pod.
	Containers []ContainerStats `json:"containers" patchStrategy:"merge" patchMergeKey:"name"`
	// Stats pertaining to network resources.
	Network *NetworkStats `json:"network,omitempty"`
}

// ContainerStats holds container-level unprocessed sample stats.
type ContainerStats struct {
	// Reference to the measured container.
	Name string `json:"name"`
	// Stats pertaining to CPU resources.
	CPU *CPUStats `json:"cpu,omitempty"`
	// Stats pertaining to memory (RAM) resources.
	Memory *MemoryStats `json:"memory,omitempty"`
}

// NonLocalObjectReference contains enough information to locate the referenced object.
type NonLocalObjectReference struct {
	Name      string `json:"name"`
	Namespace string `json:"namespace"`
}

// NetworkStats contains data about network resources.
type NetworkStats struct {
	// Cumulative count of bytes received.
	RxBytes *resource.Quantity `json:"rxBytes,omitempty"`
	// Cumulative count of receive errors encountered.
	RxErrors *int64 `json:"rxErrors,omitempty"`
	// Cumulative count of bytes transmitted.
	TxBytes *resource.Quantity `json:"txBytes,omitempty"`
	// Cumulative count of transmit errors encountered.
	TxErrors *int64 `json:"txErrors,omitempty"`
}

// CPUStats contains data about CPU usage.
type CPUStats struct {
	// Total CPU usage (sum of all cores) averaged over the sample window.
	// The "core" unit can be interpreted as CPU core-seconds per second.
	UsageCores *resource.Quantity `json:"usageCores,omitempty"`
	// Cumulative CPU usage (sum of all cores) since object creation.
	UsageCoreSeconds *resource.Quantity `json:"usageCoreSeconds,omitempty"`
}

// MemoryStats contains data about memory usage.
type MemoryStats struct {
	// Total memory in use. This includes all memory regardless of when it was accessed.
	UsageBytes *resource.Quantity `json:"usageBytes,omitempty"`
	// The amount of working set memory. This includes recently accessed memory,
	// dirty memory, and kernel memory. UsageBytes is <= TotalBytes.
	WorkingSetBytes *resource.Quantity `json:"workingSetBytes,omitempty"`
	// Cumulative number of minor page faults.
	PageFaults *int64 `json:"pageFaults,omitempty"`
	// Cumulative number of major page faults.
	MajorPageFaults *int64 `json:"majorPageFaults,omitempty"`
}
<|endoftext|>


<commit_before>package alerting

import (
	"math/rand"
	"strconv"
	"time"

	//"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/log"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/setting"
	"sync"
)

func Init() {
	if !setting.AlertingEnabled {
		return
	}

	log.Info("Alerting: Initializing scheduler...")

	scheduler := NewScheduler()
	go scheduler.Dispatch(&AlertRuleReader{})
	go scheduler.Executor(&DummieExecutor{})
	go scheduler.HandleResponses()
}

type Scheduler struct {
	jobs          map[int64]*AlertJob
	runQueue      chan *AlertJob
	responseQueue chan *AlertResult
	mtx           sync.RWMutex

	alertRuleFetcher RuleReader

	serverId       string
	serverPosition int
	clusterSize    int
}

func NewScheduler() *Scheduler {
	return &Scheduler{
		jobs:          make(map[int64]*AlertJob, 0),
		runQueue:      make(chan *AlertJob, 1000),
		responseQueue: make(chan *AlertResult, 1000),
		serverId:      strconv.Itoa(rand.Intn(1000)),
	}
}

func (this *Scheduler) heartBeat() {

	//Lets cheat on this until we focus on clustering
	log.Info("Heartbeat: Sending heartbeat from " + this.serverId)
	this.clusterSize = 1
	this.serverPosition = 1

	/*
		cmd := &m.HeartBeatCommand{ServerId: this.serverId}
		err := bus.Dispatch(cmd)

		if err != nil {
			log.Error(1, "Failed to send heartbeat.")
		} else {
			this.clusterSize = cmd.Result.ClusterSize
			this.serverPosition = cmd.Result.UptimePosition
		}
	*/
}

func (this *Scheduler) Dispatch(reader RuleReader) {
	reschedule := time.NewTicker(time.Second * 100)
	secondTicker := time.NewTicker(time.Second)
	heartbeat := time.NewTicker(time.Second * 5)

	this.heartBeat()
	this.updateJobs(reader.Fetch)

	for {
		select {
		case <-secondTicker.C:
			this.queueJobs()
		case <-reschedule.C:
			this.updateJobs(reader.Fetch)
		case <-heartbeat.C:
			this.heartBeat()
		}
	}
}

func (this *Scheduler) updateJobs(f func() []m.AlertRule) {
	log.Debug("Scheduler: UpdateJobs()")

	jobs := make(map[int64]*AlertJob, 0)
	rules := f()

	this.mtx.Lock()
	defer this.mtx.Unlock()

	for i := this.serverPosition - 1; i < len(rules); i += this.clusterSize {
		rule := rules[i]
		jobs[rule.Id] = &AlertJob{rule: rule, offset: int64(len(jobs))}
	}

	log.Debug("Scheduler: Selected %d jobs", len(jobs))

	this.jobs = jobs
}

func (this *Scheduler) queueJobs() {
	now := time.Now().Unix()

	for _, job := range this.jobs {
		if now%job.rule.Frequency == 0 && job.running == false {
			log.Info("Scheduler: Putting job on to run queue: %s", job.rule.Title)
			this.runQueue <- job
		}
	}
}

func (this *Scheduler) Executor(executor Executor) {
	for job := range this.runQueue {
		log.Info("Executor: queue length %d", len(this.runQueue))
		log.Info("Executor: executing %s", job.rule.Title)
		this.jobs[job.rule.Id].running = true
		go this.Measure(executor, job)
	}
}

func (this *Scheduler) HandleResponses() {
	for response := range this.responseQueue {
		log.Info("Response: alert %d returned %s", response.id, response.state)
		this.jobs[response.id].running = false
	}
}

func (this *Scheduler) Measure(exec Executor, rule *AlertJob) {
	now := time.Now()
	exec.Execute(rule.rule, this.responseQueue)
	elapsed := time.Since(now)
	log.Info("Schedular: exeuction took %v milli seconds", elapsed.Nanoseconds()/1000000)
}

type AlertJob struct {
	offset  int64
	delay   bool
	running bool
	rule    m.AlertRule
}

type AlertResult struct {
	id       int64
	state    string
	duration time.Time
}
<commit_msg>feat(alerting): make sure the map contains the responding alert<commit_after>package alerting

import (
	"math/rand"
	"strconv"
	"time"

	//"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/log"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/setting"
	"sync"
)

func Init() {
	if !setting.AlertingEnabled {
		return
	}

	log.Info("Alerting: Initializing scheduler...")

	scheduler := NewScheduler()
	go scheduler.Dispatch(&AlertRuleReader{})
	go scheduler.Executor(&DummieExecutor{})
	go scheduler.HandleResponses()
}

type Scheduler struct {
	jobs          map[int64]*AlertJob
	runQueue      chan *AlertJob
	responseQueue chan *AlertResult
	mtx           sync.RWMutex

	alertRuleFetcher RuleReader

	serverId       string
	serverPosition int
	clusterSize    int
}

func NewScheduler() *Scheduler {
	return &Scheduler{
		jobs:          make(map[int64]*AlertJob, 0),
		runQueue:      make(chan *AlertJob, 1000),
		responseQueue: make(chan *AlertResult, 1000),
		serverId:      strconv.Itoa(rand.Intn(1000)),
	}
}

func (this *Scheduler) heartBeat() {

	//Lets cheat on this until we focus on clustering
	log.Info("Heartbeat: Sending heartbeat from " + this.serverId)
	this.clusterSize = 1
	this.serverPosition = 1

	/*
		cmd := &m.HeartBeatCommand{ServerId: this.serverId}
		err := bus.Dispatch(cmd)

		if err != nil {
			log.Error(1, "Failed to send heartbeat.")
		} else {
			this.clusterSize = cmd.Result.ClusterSize
			this.serverPosition = cmd.Result.UptimePosition
		}
	*/
}

func (this *Scheduler) Dispatch(reader RuleReader) {
	reschedule := time.NewTicker(time.Second * 100)
	secondTicker := time.NewTicker(time.Second)
	heartbeat := time.NewTicker(time.Second * 5)

	this.heartBeat()
	this.updateJobs(reader.Fetch)

	for {
		select {
		case <-secondTicker.C:
			this.queueJobs()
		case <-reschedule.C:
			this.updateJobs(reader.Fetch)
		case <-heartbeat.C:
			this.heartBeat()
		}
	}
}

func (this *Scheduler) updateJobs(f func() []m.AlertRule) {
	log.Debug("Scheduler: UpdateJobs()")

	jobs := make(map[int64]*AlertJob, 0)
	rules := f()

	this.mtx.Lock()
	defer this.mtx.Unlock()

	for i := this.serverPosition - 1; i < len(rules); i += this.clusterSize {
		rule := rules[i]
		jobs[rule.Id] = &AlertJob{rule: rule, offset: int64(len(jobs))}
	}

	log.Debug("Scheduler: Selected %d jobs", len(jobs))

	this.jobs = jobs
}

func (this *Scheduler) queueJobs() {
	now := time.Now().Unix()

	for _, job := range this.jobs {
		if now%job.rule.Frequency == 0 && job.running == false {
			log.Info("Scheduler: Putting job on to run queue: %s", job.rule.Title)
			this.runQueue <- job
		}
	}
}

func (this *Scheduler) Executor(executor Executor) {
	for job := range this.runQueue {
		log.Info("Executor: queue length %d", len(this.runQueue))
		log.Info("Executor: executing %s", job.rule.Title)
		this.jobs[job.rule.Id].running = true
		go this.Measure(executor, job)
	}
}

func (this *Scheduler) HandleResponses() {
	for response := range this.responseQueue {
		log.Info("Response: alert %d returned %s", response.id, response.state)
		if this.jobs[response.id] != nil {
			this.jobs[response.id].running = false
		}
	}
}

func (this *Scheduler) Measure(exec Executor, rule *AlertJob) {
	now := time.Now()
	exec.Execute(rule.rule, this.responseQueue)
	elapsed := time.Since(now)
	log.Info("Schedular: exeuction took %v milli seconds", elapsed.Nanoseconds()/1000000)
}

type AlertJob struct {
	offset  int64
	delay   bool
	running bool
	rule    m.AlertRule
}

type AlertResult struct {
	id       int64
	state    string
	duration time.Time
}
<|endoftext|>


<commit_before>package repositoriesmanager

import (
	"fmt"

	"github.com/go-gorp/gorp"
	"github.com/mitchellh/mapstructure"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/database"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

//ReceiveEvents has to be launched as a goroutine.
func ReceiveEvents() {
	for {
		e := sdk.Event{}
		cache.Dequeue("events_repositoriesmanager", &e)
		db := database.DBMap(database.DB())
		if db != nil {
			if err := processEvent(db, e); err != nil {
				log.Error("ReceiveEvents> err while processing error=%s : %v", err, e)
				retryEvent(&e, err)
			}
			continue
		}
		retryEvent(&e, nil)
	}
}

func retryEvent(e *sdk.Event, err error) {
	e.Attempts++
	if e.Attempts > 2 {
		log.Error("ReceiveEvents> Aborting event processing %v: %v", err, e)
		return
	}
	cache.Enqueue("events_repositoriesmanager", e)
}

func processEvent(db gorp.SqlExecutor, event sdk.Event) error {
	log.Debug("repositoriesmanager>processEvent> receive: type:%s all: %+v", event.EventType, event)

	if event.EventType != fmt.Sprintf("%T", sdk.EventPipelineBuild{}) {
		return nil
	}

	var eventpb sdk.EventPipelineBuild
	if err := mapstructure.Decode(event.Payload, &eventpb); err != nil {
		log.Error("Error during consumption: %s", err)
		return err
	}

	if eventpb.RepositoryManagerName == "" {
		return nil
	}

	log.Debug("repositoriesmanager>processEvent> event:%+v", event)

	c, erra := AuthorizedClient(db, eventpb.ProjectKey, eventpb.RepositoryManagerName)
	if erra != nil {
		return fmt.Errorf("repositoriesmanager>processEvent> AuthorizedClient (%s, %s) > err:%s", eventpb.ProjectKey, eventpb.RepositoryManagerName, erra)
	}

	if err := c.SetStatus(event); err != nil {
		retryEvent(&event, err)
		return fmt.Errorf("repositoriesmanager>processEvent> SetStatus > err:%s", err)
	}

	retryEvent(&event, nil)

	return nil
}
<commit_msg>fix (api): do not retry when success (#597)<commit_after>package repositoriesmanager

import (
	"fmt"

	"github.com/go-gorp/gorp"
	"github.com/mitchellh/mapstructure"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/database"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

//ReceiveEvents has to be launched as a goroutine.
func ReceiveEvents() {
	for {
		e := sdk.Event{}
		cache.Dequeue("events_repositoriesmanager", &e)
		db := database.DBMap(database.DB())
		if db != nil {
			if err := processEvent(db, e); err != nil {
				log.Error("ReceiveEvents> err while processing error=%s : %v", err, e)
				retryEvent(&e, err)
			}
			continue
		}
		retryEvent(&e, nil)
	}
}

func retryEvent(e *sdk.Event, err error) {
	e.Attempts++
	if e.Attempts > 2 {
		log.Error("ReceiveEvents> Aborting event processing %v: %v", err, e)
		return
	}
	cache.Enqueue("events_repositoriesmanager", e)
}

func processEvent(db gorp.SqlExecutor, event sdk.Event) error {
	log.Debug("repositoriesmanager>processEvent> receive: type:%s all: %+v", event.EventType, event)

	if event.EventType != fmt.Sprintf("%T", sdk.EventPipelineBuild{}) {
		return nil
	}

	var eventpb sdk.EventPipelineBuild
	if err := mapstructure.Decode(event.Payload, &eventpb); err != nil {
		log.Error("Error during consumption: %s", err)
		return err
	}

	if eventpb.RepositoryManagerName == "" {
		return nil
	}

	log.Debug("repositoriesmanager>processEvent> event:%+v", event)

	c, erra := AuthorizedClient(db, eventpb.ProjectKey, eventpb.RepositoryManagerName)
	if erra != nil {
		return fmt.Errorf("repositoriesmanager>processEvent> AuthorizedClient (%s, %s) > err:%s", eventpb.ProjectKey, eventpb.RepositoryManagerName, erra)
	}

	if err := c.SetStatus(event); err != nil {
		retryEvent(&event, err)
		return fmt.Errorf("repositoriesmanager>processEvent> SetStatus > err:%s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2010 The draw2d Authors. All rights reserved.
// created: 21/11/2010 by Laurent Le Goff

// Draw an android avatar to android.png
package main

import (
	"fmt"
	"image/color"
	"math"

	"github.com/stanim/draw2d"
	"github.com/stanim/draw2d/pdf2d"
	"github.com/stanim/gofpdf"
)

func main() {
	// Initialize the graphic context on a pdf document
	pdf := gofpdf.New("P", "mm", "A4", "../font")
	pdf.AddPage()
	gc := pdf2d.NewGraphicContext(pdf)

	// set the fill and stroke color of the droid
	gc.SetFillColor(color.RGBA{0x44, 0xff, 0x44, 0xff})
	gc.SetStrokeColor(color.RGBA{0x44, 0x44, 0x44, 0xff})

	// Draw the droid
	DrawDroid(gc, 10, 10)

	// Save to pdf
	pdf2d.SaveToPdfFile("android.pdf", pdf)
}

func DrawDroid(gc draw2d.GraphicContext, x, y float64) {
	gc.SetLineCap(draw2d.RoundCap)
	gc.SetLineWidth(5)

	fmt.Println("\nhead")
	gc.MoveTo(x+30, y+70)
	gc.ArcTo(x+80, y+70, 50, 50, 180*(math.Pi/180), 180*(math.Pi/180))
	gc.Close()
	gc.FillStroke()
	gc.MoveTo(x+60, y+25)
	gc.LineTo(x+50, y+10)
	gc.MoveTo(x+100, y+25)
	gc.LineTo(x+110, y+10)
	gc.Stroke()

	fmt.Println("\nleft eye")
	draw2d.Circle(gc, x+60, y+45, 5)
	gc.FillStroke()

	fmt.Println("\nright eye")
	draw2d.Circle(gc, x+100, y+45, 5)
	gc.FillStroke()

	fmt.Println("\nbody")
	draw2d.RoundRect(gc, x+30, y+75, x+30+100, y+75+90, 10, 10)
	gc.FillStroke()
	draw2d.Rect(gc, x+30, y+75, x+30+100, y+75+80)
	gc.FillStroke()

	fmt.Println("\nleft arm")
	draw2d.RoundRect(gc, x+5, y+80, x+5+20, y+80+70, 10, 10)
	gc.FillStroke()

	fmt.Println("\nright arm")
	draw2d.RoundRect(gc, x+135, y+80, x+135+20, y+80+70, 10, 10)
	gc.FillStroke()

	fmt.Println("\nleft leg")
	draw2d.RoundRect(gc, x+50, y+150, x+50+20, y+150+50, 10, 10)
	gc.FillStroke()

	fmt.Println("\nright leg")
	draw2d.RoundRect(gc, x+90, y+150, x+90+20, y+150+50, 10, 10)
	gc.FillStroke()

}
<commit_msg>move android in its own folder<commit_after>// Copyright 2010 The draw2d Authors. All rights reserved.
// created: 21/11/2010 by Laurent Le Goff, Stani Michiels

// Draw an android avatar to android.png
package main

import (
	"fmt"
	"image/color"
	"math"

	"github.com/stanim/draw2d"
	"github.com/stanim/draw2d/pdf2d"
	"github.com/stanim/gofpdf"
)

func main() {
	// Initialize the graphic context on a pdf document
	pdf := gofpdf.New("P", "mm", "A4", "../font")
	pdf.AddPage()
	gc := pdf2d.NewGraphicContext(pdf)

	// set the fill and stroke color of the droid
	gc.SetFillColor(color.RGBA{0x44, 0xff, 0x44, 0xff})
	gc.SetStrokeColor(color.RGBA{0x44, 0x44, 0x44, 0xff})

	// Draw the droid
	DrawDroid(gc, 10, 10)

	// Save to pdf
	pdf2d.SaveToPdfFile("android.pdf", pdf)
}

func DrawDroid(gc draw2d.GraphicContext, x, y float64) {
	gc.SetLineCap(draw2d.RoundCap)
	gc.SetLineWidth(5)

	fmt.Println("\nhead")
	gc.MoveTo(x+30, y+70)
	gc.ArcTo(x+80, y+70, 50, 50, 180*(math.Pi/180), 180*(math.Pi/180))
	gc.Close()
	gc.FillStroke()
	gc.MoveTo(x+60, y+25)
	gc.LineTo(x+50, y+10)
	gc.MoveTo(x+100, y+25)
	gc.LineTo(x+110, y+10)
	gc.Stroke()

	fmt.Println("\nleft eye")
	draw2d.Circle(gc, x+60, y+45, 5)
	gc.FillStroke()

	fmt.Println("\nright eye")
	draw2d.Circle(gc, x+100, y+45, 5)
	gc.FillStroke()

	fmt.Println("\nbody")
	draw2d.RoundRect(gc, x+30, y+75, x+30+100, y+75+90, 10, 10)
	gc.FillStroke()
	draw2d.Rect(gc, x+30, y+75, x+30+100, y+75+80)
	gc.FillStroke()

	fmt.Println("\nleft arm")
	draw2d.RoundRect(gc, x+5, y+80, x+5+20, y+80+70, 10, 10)
	gc.FillStroke()

	fmt.Println("\nright arm")
	draw2d.RoundRect(gc, x+135, y+80, x+135+20, y+80+70, 10, 10)
	gc.FillStroke()

	fmt.Println("\nleft leg")
	draw2d.RoundRect(gc, x+50, y+150, x+50+20, y+150+50, 10, 10)
	gc.FillStroke()

	fmt.Println("\nright leg")
	draw2d.RoundRect(gc, x+90, y+150, x+90+20, y+150+50, 10, 10)
	gc.FillStroke()

}
<|endoftext|>


<commit_before>// roshi-server provides a REST-y HTTP service to interact with a farm.
package main

import (
	"encoding/json"
	_ "expvar"
	"flag"
	"fmt"
	logpkg "log"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/gorilla/pat"
	"github.com/peterbourgon/g2s"
	"github.com/soundcloud/roshi/cluster"
	"github.com/soundcloud/roshi/common"
	"github.com/soundcloud/roshi/farm"
	"github.com/soundcloud/roshi/instrumentation/statsd"
	"github.com/soundcloud/roshi/shard"
	"github.com/streadway/handy/breaker"
)

var (
	stats = g2s.Noop()
	log   = logpkg.New(os.Stdout, "", logpkg.Lmicroseconds)
)

func main() {
	var (
		redisInstances            = flag.String("redis.instances", "", "Semicolon-separated list of comma-separated lists of Redis instances")
		redisConnectTimeout       = flag.Duration("redis.connect.timeout", 3*time.Second, "Redis connect timeout")
		redisReadTimeout          = flag.Duration("redis.read.timeout", 3*time.Second, "Redis read timeout")
		redisWriteTimeout         = flag.Duration("redis.write.timeout", 3*time.Second, "Redis write timeout")
		redisMCPI                 = flag.Int("redis.mcpi", 10, "Max connections per Redis instance")
		redisHash                 = flag.String("redis.hash", "murmur3", "Redis hash function: murmur3, fnv, fnva")
		redisReadStrategy         = flag.String("redis.read.strategy", "SendAllReadAll", "Redis read strategy: SendAllReadAll, SendOneReadOne, SendAllReadFirstLinger, SendVarReadFirstLinger")
		redisReadThresholdRate    = flag.Int("redis.read.threshold.rate", 10, "Baseline SendAll reads per sec, additional reads are SendOne (SendVarReadFirstLinger strategy only)")
		redisReadThresholdLatency = flag.Duration("redis.read.threshold.latency", 50*time.Millisecond, "If a SendOne read has not returned anything after this latency, it's promoted to SendAll (SendVarReadFirstLinger strategy only)")
		redisRepairer             = flag.String("redis.repairer", "RateLimited", "Redis repairer: RateLimited, Nop")
		redisMaxRepairRate        = flag.Int("redis.repair.maxrate", 10, "Max repairs per second (RateLimited repairer only)")
		redisMaxRepairBacklog     = flag.Int("redis.repair.maxbacklog", 100000, "Max number of queued repairs (RateLimited repairer only)")
		maxSize                   = flag.Int("max.size", 10000, "Maximum number of events per key")
		statsdAddress             = flag.String("statsd.address", "", "Statsd address (blank to disable)")
		statsdSampleRate          = flag.Float64("statsd.sample.rate", 0.1, "Statsd sample rate for normal metrics")
		statsdBucketPrefix        = flag.String("statsd.bucket.prefix", "myservice.", "Statsd bucket key prefix, including trailing period")
		httpCircuitBreaker        = flag.Bool("http.circuit.breaker", true, "Enable HTTP server circuit breaker")
		httpAddress               = flag.String("http.address", ":6301", "HTTP listen address")
	)
	flag.Parse()
	log.Printf("GOMAXPROCS %d", runtime.GOMAXPROCS(-1))

	// Set up statsd instrumentation, if it's specified.
	if *statsdAddress != "" {
		var err error
		stats, err = g2s.Dial("udp", *statsdAddress)
		if err != nil {
			log.Fatal(err)
		}
	}

	// Parse read strategy.
	var readStrategy farm.ReadStrategy
	switch strings.ToLower(*redisReadStrategy) {
	case "sendallreadall":
		readStrategy = farm.SendAllReadAll
	case "sendonereadone":
		readStrategy = farm.SendOneReadOne
	case "sendallreadfirstlinger":
		readStrategy = farm.SendAllReadFirstLinger
	case "sendvarreadfirstlinger":
		readStrategy = farm.SendVarReadFirstLinger(*redisReadThresholdRate, *redisReadThresholdLatency)
	default:
		log.Fatalf("unknown read strategy '%s'", *redisReadStrategy)
	}
	log.Printf("using %s read strategy", *redisReadStrategy)

	// Parse repairer.
	var repairer farm.Repairer
	switch strings.ToLower(*redisRepairer) {
	case "nop":
		repairer = farm.NopRepairer
	case "ratelimited":
		repairer = farm.RateLimitedRepairer(*redisMaxRepairRate, *redisMaxRepairBacklog)
	default:
		log.Fatalf("unknown repairer '%s'", *redisRepairer)
	}

	// Parse hash function.
	var hashFunc func(string) uint32
	switch strings.ToLower(*redisHash) {
	case "murmur3":
		hashFunc = shard.Murmur3
	case "fnv":
		hashFunc = shard.FNV
	case "fnva":
		hashFunc = shard.FNVa
	default:
		log.Fatalf("unknown hash '%s'", *redisHash)
	}

	// Build the farm.
	farm, err := newFarm(
		*redisInstances,
		*redisConnectTimeout, *redisReadTimeout, *redisWriteTimeout,
		*redisMCPI,
		hashFunc,
		readStrategy,
		repairer,
		*maxSize,
		*statsdSampleRate,
		*statsdBucketPrefix,
	)
	if err != nil {
		log.Fatal(err)
	}

	// Build the HTTP server.
	r := pat.New()
	r.Add("GET", "/debug", http.DefaultServeMux)
	r.Get("/", handleSelect(farm))
	r.Post("/", handleInsert(farm))
	r.Delete("/", handleDelete(farm))
	h := http.Handler(r)
	if *httpCircuitBreaker {
		log.Printf("using HTTP circuit breaker")
		h = breaker.DefaultBreaker(h)
	}

	// Go for it.
	log.Printf("listening on %s", *httpAddress)
	log.Fatal(http.ListenAndServe(*httpAddress, h))
}

func newFarm(
	redisInstances string,
	connectTimeout, readTimeout, writeTimeout time.Duration,
	redisMCPI int,
	hash func(string) uint32,
	readStrategy farm.ReadStrategy,
	repairer farm.Repairer,
	maxSize int,
	statsdSampleRate float64,
	bucketPrefix string,
) (*farm.Farm, error) {
	instr := statsd.New(stats, float32(statsdSampleRate), bucketPrefix)

	clusters := []cluster.Cluster{}
	for i, clusterInstances := range strings.Split(redisInstances, ";") {
		addresses := stripBlank(strings.Split(clusterInstances, ","))
		if len(addresses) <= 0 {
			continue
		}
		clusters = append(clusters, cluster.New(
			shard.New(
				addresses,
				connectTimeout, readTimeout, writeTimeout,
				redisMCPI,
				hash,
			),
			maxSize,
			instr,
		))
		log.Printf("Redis cluster %d: %d instance(s)", i+1, len(addresses))
	}
	if len(clusters) <= 0 {
		return nil, fmt.Errorf("no cluster(s)")
	}

	return farm.New(
		clusters,
		readStrategy,
		repairer,
		instr,
	), nil
}

func handleSelect(selecter farm.Selecter) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		began := time.Now()

		if err := r.ParseForm(); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
			return
		}
		offset := parseInt(r.Form, "offset", 0)
		limit := parseInt(r.Form, "limit", 10)
		coalesce := parseBool(r.Form, "coalesce", false)

		var keys [][]byte
		defer r.Body.Close()
		if err := json.NewDecoder(r.Body).Decode(&keys); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusBadRequest, err)
			return
		}

		keyStrings := make([]string, len(keys))
		for i := range keys {
			keyStrings[i] = string(keys[i])
		}

		var records interface{}
		if coalesce {
			// We need to Select from 0 to offset+limit, flatten the map to a
			// single ordered slice, and then cut off the last limit elements.
			m, err := selecter.Select(keyStrings, 0, offset+limit)
			if err != nil {
				respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
				return
			}
			records = flatten(m, offset, limit)
		} else {
			// We can directly Select using the given offset and limit.
			m, err := selecter.Select(keyStrings, offset, limit)
			if err != nil {
				respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
				return
			}
			records = m
		}

		respondSelected(w, keys, offset, limit, records, time.Since(began))
	}
}

func handleInsert(inserter cluster.Inserter) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		began := time.Now()

		var tuples []common.KeyScoreMember
		if err := json.NewDecoder(r.Body).Decode(&tuples); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusBadRequest, err)
			return
		}

		if err := inserter.Insert(tuples); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
			return
		}

		respondInserted(w, len(tuples), time.Since(began))
	}
}

func handleDelete(deleter cluster.Deleter) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		began := time.Now()

		var tuples []common.KeyScoreMember
		if err := json.NewDecoder(r.Body).Decode(&tuples); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusBadRequest, err)
			return
		}

		if err := deleter.Delete(tuples); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
			return
		}

		respondDeleted(w, len(tuples), time.Since(began))
	}
}

func flatten(m map[string][]common.KeyScoreMember, offset, limit int) []common.KeyScoreMember {
	a := common.KeyScoreMembers{}
	for _, tuples := range m {
		a = append(a, tuples...)
	}

	sort.Sort(a)

	if len(a) < offset {
		return []common.KeyScoreMember{}
	}
	a = a[offset:]

	if len(a) > limit {
		a = a[:limit]
	}

	return a
}

func parseInt(values url.Values, key string, defaultValue int) int {
	value, err := strconv.ParseInt(values.Get(key), 10, 64)
	if err != nil {
		return defaultValue
	}
	return int(value)
}

func parseBool(values url.Values, key string, defaultValue bool) bool {
	value, err := strconv.ParseBool(values.Get(key))
	if err != nil {
		return defaultValue
	}
	return value
}

func respondInserted(w http.ResponseWriter, n int, duration time.Duration) {
	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(map[string]interface{}{
		"inserted": n,
		"duration": duration.String(),
	})
}

func respondSelected(w http.ResponseWriter, keys [][]byte, offset, limit int, records interface{}, duration time.Duration) {
	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(map[string]interface{}{
		"keys":     keys,
		"offset":   offset,
		"limit":    limit,
		"records":  records,
		"duration": duration.String(),
	})
}

func respondDeleted(w http.ResponseWriter, n int, duration time.Duration) {
	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(map[string]interface{}{
		"deleted":  n,
		"duration": duration.String(),
	})
}

func respondError(w http.ResponseWriter, method, url string, code int, err error) {
	log.Printf("%s %s: HTTP %d: %s", method, url, code, err)
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	json.NewEncoder(w).Encode(map[string]interface{}{
		"error":       err.Error(),
		"code":        code,
		"description": http.StatusText(code),
	})
}

func stripBlank(src []string) []string {
	dst := []string{}
	for _, s := range src {
		if s == "" {
			continue
		}
		dst = append(dst, s)
	}
	return dst
}

type writer struct{ *logpkg.Logger }

func (w writer) Write(p []byte) (int, error) { w.Print(string(p)); return len(p), nil }
<commit_msg>roshi-server default listen :6302<commit_after>// roshi-server provides a REST-y HTTP service to interact with a farm.
package main

import (
	"encoding/json"
	_ "expvar"
	"flag"
	"fmt"
	logpkg "log"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/gorilla/pat"
	"github.com/peterbourgon/g2s"
	"github.com/soundcloud/roshi/cluster"
	"github.com/soundcloud/roshi/common"
	"github.com/soundcloud/roshi/farm"
	"github.com/soundcloud/roshi/instrumentation/statsd"
	"github.com/soundcloud/roshi/shard"
	"github.com/streadway/handy/breaker"
)

var (
	stats = g2s.Noop()
	log   = logpkg.New(os.Stdout, "", logpkg.Lmicroseconds)
)

func main() {
	var (
		redisInstances            = flag.String("redis.instances", "", "Semicolon-separated list of comma-separated lists of Redis instances")
		redisConnectTimeout       = flag.Duration("redis.connect.timeout", 3*time.Second, "Redis connect timeout")
		redisReadTimeout          = flag.Duration("redis.read.timeout", 3*time.Second, "Redis read timeout")
		redisWriteTimeout         = flag.Duration("redis.write.timeout", 3*time.Second, "Redis write timeout")
		redisMCPI                 = flag.Int("redis.mcpi", 10, "Max connections per Redis instance")
		redisHash                 = flag.String("redis.hash", "murmur3", "Redis hash function: murmur3, fnv, fnva")
		redisReadStrategy         = flag.String("redis.read.strategy", "SendAllReadAll", "Redis read strategy: SendAllReadAll, SendOneReadOne, SendAllReadFirstLinger, SendVarReadFirstLinger")
		redisReadThresholdRate    = flag.Int("redis.read.threshold.rate", 10, "Baseline SendAll reads per sec, additional reads are SendOne (SendVarReadFirstLinger strategy only)")
		redisReadThresholdLatency = flag.Duration("redis.read.threshold.latency", 50*time.Millisecond, "If a SendOne read has not returned anything after this latency, it's promoted to SendAll (SendVarReadFirstLinger strategy only)")
		redisRepairer             = flag.String("redis.repairer", "RateLimited", "Redis repairer: RateLimited, Nop")
		redisMaxRepairRate        = flag.Int("redis.repair.maxrate", 10, "Max repairs per second (RateLimited repairer only)")
		redisMaxRepairBacklog     = flag.Int("redis.repair.maxbacklog", 100000, "Max number of queued repairs (RateLimited repairer only)")
		maxSize                   = flag.Int("max.size", 10000, "Maximum number of events per key")
		statsdAddress             = flag.String("statsd.address", "", "Statsd address (blank to disable)")
		statsdSampleRate          = flag.Float64("statsd.sample.rate", 0.1, "Statsd sample rate for normal metrics")
		statsdBucketPrefix        = flag.String("statsd.bucket.prefix", "myservice.", "Statsd bucket key prefix, including trailing period")
		httpCircuitBreaker        = flag.Bool("http.circuit.breaker", true, "Enable HTTP server circuit breaker")
		httpAddress               = flag.String("http.address", ":6302", "HTTP listen address")
	)
	flag.Parse()
	log.Printf("GOMAXPROCS %d", runtime.GOMAXPROCS(-1))

	// Set up statsd instrumentation, if it's specified.
	if *statsdAddress != "" {
		var err error
		stats, err = g2s.Dial("udp", *statsdAddress)
		if err != nil {
			log.Fatal(err)
		}
	}

	// Parse read strategy.
	var readStrategy farm.ReadStrategy
	switch strings.ToLower(*redisReadStrategy) {
	case "sendallreadall":
		readStrategy = farm.SendAllReadAll
	case "sendonereadone":
		readStrategy = farm.SendOneReadOne
	case "sendallreadfirstlinger":
		readStrategy = farm.SendAllReadFirstLinger
	case "sendvarreadfirstlinger":
		readStrategy = farm.SendVarReadFirstLinger(*redisReadThresholdRate, *redisReadThresholdLatency)
	default:
		log.Fatalf("unknown read strategy '%s'", *redisReadStrategy)
	}
	log.Printf("using %s read strategy", *redisReadStrategy)

	// Parse repairer.
	var repairer farm.Repairer
	switch strings.ToLower(*redisRepairer) {
	case "nop":
		repairer = farm.NopRepairer
	case "ratelimited":
		repairer = farm.RateLimitedRepairer(*redisMaxRepairRate, *redisMaxRepairBacklog)
	default:
		log.Fatalf("unknown repairer '%s'", *redisRepairer)
	}

	// Parse hash function.
	var hashFunc func(string) uint32
	switch strings.ToLower(*redisHash) {
	case "murmur3":
		hashFunc = shard.Murmur3
	case "fnv":
		hashFunc = shard.FNV
	case "fnva":
		hashFunc = shard.FNVa
	default:
		log.Fatalf("unknown hash '%s'", *redisHash)
	}

	// Build the farm.
	farm, err := newFarm(
		*redisInstances,
		*redisConnectTimeout, *redisReadTimeout, *redisWriteTimeout,
		*redisMCPI,
		hashFunc,
		readStrategy,
		repairer,
		*maxSize,
		*statsdSampleRate,
		*statsdBucketPrefix,
	)
	if err != nil {
		log.Fatal(err)
	}

	// Build the HTTP server.
	r := pat.New()
	r.Add("GET", "/debug", http.DefaultServeMux)
	r.Get("/", handleSelect(farm))
	r.Post("/", handleInsert(farm))
	r.Delete("/", handleDelete(farm))
	h := http.Handler(r)
	if *httpCircuitBreaker {
		log.Printf("using HTTP circuit breaker")
		h = breaker.DefaultBreaker(h)
	}

	// Go for it.
	log.Printf("listening on %s", *httpAddress)
	log.Fatal(http.ListenAndServe(*httpAddress, h))
}

func newFarm(
	redisInstances string,
	connectTimeout, readTimeout, writeTimeout time.Duration,
	redisMCPI int,
	hash func(string) uint32,
	readStrategy farm.ReadStrategy,
	repairer farm.Repairer,
	maxSize int,
	statsdSampleRate float64,
	bucketPrefix string,
) (*farm.Farm, error) {
	instr := statsd.New(stats, float32(statsdSampleRate), bucketPrefix)

	clusters := []cluster.Cluster{}
	for i, clusterInstances := range strings.Split(redisInstances, ";") {
		addresses := stripBlank(strings.Split(clusterInstances, ","))
		if len(addresses) <= 0 {
			continue
		}
		clusters = append(clusters, cluster.New(
			shard.New(
				addresses,
				connectTimeout, readTimeout, writeTimeout,
				redisMCPI,
				hash,
			),
			maxSize,
			instr,
		))
		log.Printf("Redis cluster %d: %d instance(s)", i+1, len(addresses))
	}
	if len(clusters) <= 0 {
		return nil, fmt.Errorf("no cluster(s)")
	}

	return farm.New(
		clusters,
		readStrategy,
		repairer,
		instr,
	), nil
}

func handleSelect(selecter farm.Selecter) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		began := time.Now()

		if err := r.ParseForm(); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
			return
		}
		offset := parseInt(r.Form, "offset", 0)
		limit := parseInt(r.Form, "limit", 10)
		coalesce := parseBool(r.Form, "coalesce", false)

		var keys [][]byte
		defer r.Body.Close()
		if err := json.NewDecoder(r.Body).Decode(&keys); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusBadRequest, err)
			return
		}

		keyStrings := make([]string, len(keys))
		for i := range keys {
			keyStrings[i] = string(keys[i])
		}

		var records interface{}
		if coalesce {
			// We need to Select from 0 to offset+limit, flatten the map to a
			// single ordered slice, and then cut off the last limit elements.
			m, err := selecter.Select(keyStrings, 0, offset+limit)
			if err != nil {
				respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
				return
			}
			records = flatten(m, offset, limit)
		} else {
			// We can directly Select using the given offset and limit.
			m, err := selecter.Select(keyStrings, offset, limit)
			if err != nil {
				respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
				return
			}
			records = m
		}

		respondSelected(w, keys, offset, limit, records, time.Since(began))
	}
}

func handleInsert(inserter cluster.Inserter) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		began := time.Now()

		var tuples []common.KeyScoreMember
		if err := json.NewDecoder(r.Body).Decode(&tuples); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusBadRequest, err)
			return
		}

		if err := inserter.Insert(tuples); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
			return
		}

		respondInserted(w, len(tuples), time.Since(began))
	}
}

func handleDelete(deleter cluster.Deleter) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		began := time.Now()

		var tuples []common.KeyScoreMember
		if err := json.NewDecoder(r.Body).Decode(&tuples); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusBadRequest, err)
			return
		}

		if err := deleter.Delete(tuples); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
			return
		}

		respondDeleted(w, len(tuples), time.Since(began))
	}
}

func flatten(m map[string][]common.KeyScoreMember, offset, limit int) []common.KeyScoreMember {
	a := common.KeyScoreMembers{}
	for _, tuples := range m {
		a = append(a, tuples...)
	}

	sort.Sort(a)

	if len(a) < offset {
		return []common.KeyScoreMember{}
	}
	a = a[offset:]

	if len(a) > limit {
		a = a[:limit]
	}

	return a
}

func parseInt(values url.Values, key string, defaultValue int) int {
	value, err := strconv.ParseInt(values.Get(key), 10, 64)
	if err != nil {
		return defaultValue
	}
	return int(value)
}

func parseBool(values url.Values, key string, defaultValue bool) bool {
	value, err := strconv.ParseBool(values.Get(key))
	if err != nil {
		return defaultValue
	}
	return value
}

func respondInserted(w http.ResponseWriter, n int, duration time.Duration) {
	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(map[string]interface{}{
		"inserted": n,
		"duration": duration.String(),
	})
}

func respondSelected(w http.ResponseWriter, keys [][]byte, offset, limit int, records interface{}, duration time.Duration) {
	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(map[string]interface{}{
		"keys":     keys,
		"offset":   offset,
		"limit":    limit,
		"records":  records,
		"duration": duration.String(),
	})
}

func respondDeleted(w http.ResponseWriter, n int, duration time.Duration) {
	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(map[string]interface{}{
		"deleted":  n,
		"duration": duration.String(),
	})
}

func respondError(w http.ResponseWriter, method, url string, code int, err error) {
	log.Printf("%s %s: HTTP %d: %s", method, url, code, err)
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	json.NewEncoder(w).Encode(map[string]interface{}{
		"error":       err.Error(),
		"code":        code,
		"description": http.StatusText(code),
	})
}

func stripBlank(src []string) []string {
	dst := []string{}
	for _, s := range src {
		if s == "" {
			continue
		}
		dst = append(dst, s)
	}
	return dst
}

type writer struct{ *logpkg.Logger }

func (w writer) Write(p []byte) (int, error) { w.Print(string(p)); return len(p), nil }
<|endoftext|>


<commit_before>package builds

import (
	"fmt"
	"sync"
	"time"

	"k8s.io/kubernetes/pkg/util/wait"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	buildapi "github.com/openshift/origin/pkg/build/api"
	exutil "github.com/openshift/origin/test/extended/util"
)

var _ = g.Describe("builds: parallel: oc start-build", func() {
	defer g.GinkgoRecover()
	var (
		buildFixture   = exutil.FixturePath("..", "extended", "fixtures", "test-build.json")
		exampleGemfile = exutil.FixturePath("..", "extended", "fixtures", "test-build-app", "Gemfile")
		exampleBuild   = exutil.FixturePath("..", "extended", "fixtures", "test-build-app")
		oc             = exutil.NewCLI("cli-start-build", exutil.KubeConfigPath())
	)

	g.JustBeforeEach(func() {
		g.By("waiting for builder service account")
		err := exutil.WaitForBuilderAccount(oc.KubeREST().ServiceAccounts(oc.Namespace()))
		o.Expect(err).NotTo(o.HaveOccurred())
		oc.Run("create").Args("-f", buildFixture).Execute()
	})

	g.Describe("oc start-build --wait", func() {
		g.It("should start a build and wait for the build to complete", func() {
			g.By("starting the build with --wait flag")
			out, err := oc.Run("start-build").Args("sample-build", "--wait").Output()
			o.Expect(err).NotTo(o.HaveOccurred())

			g.By(fmt.Sprintf("verifying the build %q status", out))
			build, err := oc.REST().Builds(oc.Namespace()).Get(out)
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
		})

		g.It("should start a build and wait for the build to fail", func() {
			g.By("starting the build with --wait flag but wrong --commit")
			out, err := oc.Run("start-build").
				Args("sample-build", "--wait", "--commit", "fffffff").
				Output()
			o.Expect(err).To(o.HaveOccurred())
			o.Expect(out).Should(o.ContainSubstring(`status is "Failed"`))
		})
	})

	g.Describe("binary builds", func() {
		g.It("should accept --from-file as input", func() {
			g.By("starting the build with a Dockerfile")
			out, err := oc.Run("start-build").Args("sample-build", "--follow", "--wait", fmt.Sprintf("--from-file=%s", exampleGemfile)).Output()
			g.By(fmt.Sprintf("verifying the build %q status", out))
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(out).To(o.ContainSubstring("Uploading file"))
			o.Expect(out).To(o.ContainSubstring("as binary input for the build ..."))
			o.Expect(out).To(o.ContainSubstring("Your bundle is complete"))

			build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
		})

		g.It("should accept --from-dir as input", func() {
			g.By("starting the build with a directory")
			out, err := oc.Run("start-build").Args("sample-build", "--follow", "--wait", fmt.Sprintf("--from-dir=%s", exampleBuild)).Output()
			g.By(fmt.Sprintf("verifying the build %q status", out))
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(out).To(o.ContainSubstring("Uploading directory"))
			o.Expect(out).To(o.ContainSubstring("as binary input for the build ..."))
			o.Expect(out).To(o.ContainSubstring("Your bundle is complete"))

			build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
		})

		g.It("should accept --from-repo as input", func() {
			g.By("starting the build with a Git repository")
			out, err := oc.Run("start-build").Args("sample-build", "--follow", "--wait", fmt.Sprintf("--from-repo=%s", exampleBuild)).Output()
			g.By(fmt.Sprintf("verifying the build %q status", out))
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(out).To(o.ContainSubstring("Uploading Git repository"))
			o.Expect(out).To(o.ContainSubstring("as binary input for the build ..."))
			o.Expect(out).To(o.ContainSubstring("Your bundle is complete"))

			build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
		})
	})

	g.Describe("cancelling build started by oc start-build --wait", func() {
		g.It("should start a build and wait for the build to cancel", func() {
			g.By("starting the build with --wait flag")
			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				defer g.GinkgoRecover()
				out, err := oc.Run("start-build").Args("sample-build", "--wait").Output()
				defer wg.Done()
				o.Expect(err).To(o.HaveOccurred())
				o.Expect(out).Should(o.ContainSubstring(`status is "Cancelled"`))
			}()

			g.By("getting the build name")
			var buildName string
			wait.Poll(time.Duration(100*time.Millisecond), time.Duration(60*time.Second), func() (bool, error) {
				out, err := oc.Run("get").
					Args("build", "--template", "{{ (index .items 0).metadata.name }}").Output()
				// Give it second chance in case the build resource was not created yet
				if err != nil || len(out) == 0 {
					return false, nil
				}
				buildName = out
				return true, nil
			})

			o.Expect(buildName).ToNot(o.BeEmpty())

			g.By(fmt.Sprintf("cancelling the build %q", buildName))
			err := oc.Run("cancel-build").Args(buildName).Execute()
			o.Expect(err).ToNot(o.HaveOccurred())
			wg.Wait()
		})

	})

})
<commit_msg>Fix extended test for start-build<commit_after>package builds

import (
	"fmt"
	"sync"
	"time"

	"k8s.io/kubernetes/pkg/util/wait"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	buildapi "github.com/openshift/origin/pkg/build/api"
	exutil "github.com/openshift/origin/test/extended/util"
)

var _ = g.Describe("builds: parallel: oc start-build", func() {
	defer g.GinkgoRecover()
	var (
		buildFixture   = exutil.FixturePath("..", "extended", "fixtures", "test-build.json")
		exampleGemfile = exutil.FixturePath("..", "extended", "fixtures", "test-build-app", "Gemfile")
		exampleBuild   = exutil.FixturePath("..", "extended", "fixtures", "test-build-app")
		oc             = exutil.NewCLI("cli-start-build", exutil.KubeConfigPath())
	)

	g.JustBeforeEach(func() {
		g.By("waiting for builder service account")
		err := exutil.WaitForBuilderAccount(oc.KubeREST().ServiceAccounts(oc.Namespace()))
		o.Expect(err).NotTo(o.HaveOccurred())
		oc.Run("create").Args("-f", buildFixture).Execute()
	})

	g.Describe("oc start-build --wait", func() {
		g.It("should start a build and wait for the build to complete", func() {
			g.By("starting the build with --wait flag")
			out, err := oc.Run("start-build").Args("sample-build", "--wait").Output()
			o.Expect(err).NotTo(o.HaveOccurred())

			g.By(fmt.Sprintf("verifying the build %q status", out))
			build, err := oc.REST().Builds(oc.Namespace()).Get(out)
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
		})

		g.It("should start a build and wait for the build to fail", func() {
			g.By("starting the build with --wait flag but wrong --commit")
			out, err := oc.Run("start-build").
				Args("sample-build", "--wait", "--commit", "fffffff").
				Output()
			o.Expect(err).To(o.HaveOccurred())
			o.Expect(out).Should(o.ContainSubstring(`status is "Failed"`))
		})
	})

	g.Describe("binary builds", func() {
		g.It("should accept --from-file as input", func() {
			g.By("starting the build with a Dockerfile")
			out, err := oc.Run("start-build").Args("sample-build", "--follow", "--wait", fmt.Sprintf("--from-file=%s", exampleGemfile)).Output()
			g.By(fmt.Sprintf("verifying the build %q status", out))
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(out).To(o.ContainSubstring("Uploading file"))
			o.Expect(out).To(o.ContainSubstring("as binary input for the build ..."))
			o.Expect(out).To(o.ContainSubstring("Your bundle is complete"))

			build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
		})

		g.It("should accept --from-dir as input", func() {
			g.By("starting the build with a directory")
			out, err := oc.Run("start-build").Args("sample-build", "--follow", "--wait", fmt.Sprintf("--from-dir=%s", exampleBuild)).Output()
			g.By(fmt.Sprintf("verifying the build %q status", out))
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(out).To(o.ContainSubstring("Uploading directory"))
			o.Expect(out).To(o.ContainSubstring("as binary input for the build ..."))
			o.Expect(out).To(o.ContainSubstring("Your bundle is complete"))

			build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
		})

		g.It("should accept --from-repo as input", func() {
			g.By("starting the build with a Git repository")
			out, err := oc.Run("start-build").Args("sample-build", "--follow", "--wait", fmt.Sprintf("--from-repo=%s", exampleBuild)).Output()
			g.By(fmt.Sprintf("verifying the build %q status", out))
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(out).To(o.ContainSubstring("Uploading"))
			o.Expect(out).To(o.ContainSubstring(`at commit "HEAD"`))
			o.Expect(out).To(o.ContainSubstring("as binary input for the build ..."))
			o.Expect(out).To(o.ContainSubstring("Your bundle is complete"))

			build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
		})

		g.It("should accept --from-repo with --commit as input", func() {
			g.By("starting the build with a Git repository")
			// NOTE: This actually takes the commit from the origin repository. If the
			// test-build-app changes, this commit has to be bumped.
			out, err := oc.Run("start-build").Args("sample-build", "--follow", "--commit=4b7de05", "--wait", fmt.Sprintf("--from-repo=%s", exampleBuild)).Output()
			g.By(fmt.Sprintf("verifying the build %q status", out))
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(out).To(o.ContainSubstring("Uploading"))
			o.Expect(out).To(o.ContainSubstring(`at commit "4b7de05"`))
			o.Expect(out).To(o.ContainSubstring(`"commit":"4b7de05d4abb7570fc03f8ac2e27e5bba1e9c390"`))
			o.Expect(out).To(o.ContainSubstring("as binary input for the build ..."))
			o.Expect(out).To(o.ContainSubstring("Your bundle is complete"))

			build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
			o.Expect(err).NotTo(o.HaveOccurred())
			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
		})
	})

	g.Describe("cancelling build started by oc start-build --wait", func() {
		g.It("should start a build and wait for the build to cancel", func() {
			g.By("starting the build with --wait flag")
			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				defer g.GinkgoRecover()
				out, err := oc.Run("start-build").Args("sample-build", "--wait").Output()
				defer wg.Done()
				o.Expect(err).To(o.HaveOccurred())
				o.Expect(out).Should(o.ContainSubstring(`status is "Cancelled"`))
			}()

			g.By("getting the build name")
			var buildName string
			wait.Poll(time.Duration(100*time.Millisecond), time.Duration(60*time.Second), func() (bool, error) {
				out, err := oc.Run("get").
					Args("build", "--template", "{{ (index .items 0).metadata.name }}").Output()
				// Give it second chance in case the build resource was not created yet
				if err != nil || len(out) == 0 {
					return false, nil
				}
				buildName = out
				return true, nil
			})

			o.Expect(buildName).ToNot(o.BeEmpty())

			g.By(fmt.Sprintf("cancelling the build %q", buildName))
			err := oc.Run("cancel-build").Args(buildName).Execute()
			o.Expect(err).ToNot(o.HaveOccurred())
			wg.Wait()
		})

	})

})
<|endoftext|>


<commit_before>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package common

import (
	"bytes"
	"compress/zlib"
	"context"
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"google.golang.org/grpc/codes"

	"go.chromium.org/luci/auth/identity"
	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/grpc/grpcutil"
	"go.chromium.org/luci/server/auth"
)

// MergeStrings merges multiple string slices together into a single slice,
// removing duplicates.
func MergeStrings(sss ...[]string) []string {
	result := []string{}
	seen := map[string]bool{}
	for _, ss := range sss {
		for _, s := range ss {
			if seen[s] {
				continue
			}
			seen[s] = true
			result = append(result, s)
		}
	}
	sort.Strings(result)
	return result
}

// ObfuscateEmail converts a string containing email address email@address.com
// into email<junk>@address.com.
func ObfuscateEmail(email string) template.HTML {
	email = template.HTMLEscapeString(email)
	return template.HTML(strings.Replace(
		email, "@", "ohnoyoudont@", -1))
}

// ShortenEmail shortens Google emails.
func ShortenEmail(email string) string {
	return strings.Replace(email, "@google.com", "", -1)
}

// TagGRPC annotates some gRPC with Milo specific semantics, specifically:
// * Marks the error as Unauthorized if the user is not logged in,
// and the underlying error was a 403 or 404.
// * Otherwise, tag the error with the original error code.
func TagGRPC(c context.Context, err error) error {
	loggedIn := auth.CurrentIdentity(c) != identity.AnonymousIdentity
	code := grpcutil.Code(err)
	if code == codes.NotFound || code == codes.PermissionDenied {
		// Mask the errors, so they look the same.
		if loggedIn {
			return errors.Reason("not found").Tag(grpcutil.NotFoundTag).Err()
		}
		return errors.Reason("not logged in").Tag(grpcutil.UnauthenticatedTag).Err()
	}
	return grpcutil.ToGRPCErr(err)
}

// ParseIntFromForm parses an integer from a form.
func ParseIntFromForm(form url.Values, key string, base int, bitSize int) (int64, error) {
	input, err := ReadExactOneFromForm(form, key)
	if err != nil {
		return 0, err
	}
	ret, err := strconv.ParseInt(input, 10, 64)
	if err != nil {
		return 0, errors.Annotate(err, "invalid %v; expected an integer; actual value: %v", key, input).Err()
	}
	return ret, nil
}

// ReadExactOneFromForm read a string from a form.
// There must be exactly one and non-empty entry of the given key in the form.
func ReadExactOneFromForm(form url.Values, key string) (string, error) {
	input := form[key]
	if len(input) != 1 || input[0] == "" {
		return "", fmt.Errorf("multiple or missing %v; actual value: %v", key, input)
	}
	return input[0], nil
}

// LegacyBuilderIDString returns a legacy string identifying the builder.
// It is used in the Milo datastore.
func LegacyBuilderIDString(bid *buildbucketpb.BuilderID) string {
	return fmt.Sprintf("buildbucket/luci.%s.%s/%s", bid.Project, bid.Bucket, bid.Builder)
}

var ErrInvalidLegacyBuilderID = errors.New("the string is not a valid legacy builder ID")
var legacyBuilderIDRe = regexp.MustCompile(`^buildbucket/luci\.([^./]+)\.([^/]+)/([^/]+)$`)

// ParseLegacyBuilderID parses the legacy builder ID
// (e.g. `buildbucket/luci.<project>.<bucket>/<builder>`) and returns the
// BuilderID struct.
func ParseLegacyBuilderID(bid string) (*buildbucketpb.BuilderID, error) {
	match := legacyBuilderIDRe.FindStringSubmatch(bid)
	if len(match) == 0 {
		return nil, ErrInvalidLegacyBuilderID
	}
	return &buildbucketpb.BuilderID{
		Project: match[1],
		Bucket:  match[2],
		Builder: match[3],
	}, nil
}

var ErrInvalidBuilderID = errors.New("the string is not a valid builder ID")
var builderIDRe = regexp.MustCompile("^([^/]+)/([^/]+)/([^/]+)$")

// ParseBuilderID parses the canonical builder ID
// (e.g. `<project>/<bucket>/<builder>`) and returns the BuilderID struct.
func ParseBuilderID(bid string) (*buildbucketpb.BuilderID, error) {
	match := builderIDRe.FindStringSubmatch(bid)
	if len(match) == 0 {
		return nil, ErrInvalidBuilderID
	}
	return &buildbucketpb.BuilderID{
		Project: match[1],
		Bucket:  match[2],
		Builder: match[3],
	}, nil
}

var legacyBuildIDWithBuildNumRe = regexp.MustCompile(`^buildbucket/luci\.([^./]+)\.([^/]+)/([^/]+)/(\d+)$`)
var ErrInvalidLegacyBuildID = errors.New("the string is not a valid legacy build ID")

// ParseLegacyBuildID parses the legacy build ID
// (e.g. `buildbucket/luci.<project>.<bucket>/<builder>/<number>`)
func ParseLegacyBuildID(bid string) (builderID *buildbucketpb.BuilderID, number int32, err error) {
	match := legacyBuildIDWithBuildNumRe.FindStringSubmatch(bid)
	if len(match) == 0 {
		return nil, 0, ErrInvalidLegacyBuildID
	}
	builderID = &buildbucketpb.BuilderID{
		Project: match[1],
		Bucket:  match[2],
		Builder: match[3],
	}
	buildNum, err := strconv.ParseInt(match[4], 10, 32)
	if err != nil {
		return nil, 0, ErrInvalidLegacyBuildID
	}
	return builderID, int32(buildNum), nil
}

// JSONMarshalCompressed converts a message into compressed JSON form, suitable for storing in memcache.
func JSONMarshalCompressed(message interface{}) ([]byte, error) {
	// Compress using zlib.
	b := bytes.Buffer{}
	w := zlib.NewWriter(&b)
	enc := json.NewEncoder(w)

	if err := enc.Encode(message); err != nil {
		return nil, err
	}

	// Flush remaining bytes in the zlib writer.
	if err := w.Close(); err != nil {
		return nil, err
	}

	return b.Bytes(), nil
}

// JSONUnmarshalCompressed converts a message back from compressed JSON form.
func JSONUnmarshalCompressed(serialized []byte, out interface{}) error {
	// Decompress using zlib.
	r, err := zlib.NewReader(bytes.NewReader(serialized))
	if err != nil {
		return err
	}

	dec := json.NewDecoder(r)
	return dec.Decode(out)
}

// GetJSONData fetches data from the given URL, parses the response body to `out`.
// It follows redirection and returns an error if the status code is 4xx or 5xx.
func GetJSONData(client *http.Client, url string, out interface{}) (err error) {
	response, err := client.Get(url)
	if err != nil {
		return err
	}
	defer func() {
		err = errors.Flatten(errors.NewMultiError(response.Body.Close(), err))
	}()

	if response.StatusCode >= 400 && response.StatusCode <= 599 {
		return fmt.Errorf("failed to fetch data: %q returned code %q", url, response.Status)
	}

	dec := json.NewDecoder(response.Body)
	return dec.Decode(out)
}
<commit_msg>[milo] implement BucketResourceID<commit_after>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package common

import (
	"bytes"
	"compress/zlib"
	"context"
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"google.golang.org/grpc/codes"

	"go.chromium.org/luci/auth/identity"
	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/grpc/grpcutil"
	"go.chromium.org/luci/server/auth"
)

// MergeStrings merges multiple string slices together into a single slice,
// removing duplicates.
func MergeStrings(sss ...[]string) []string {
	result := []string{}
	seen := map[string]bool{}
	for _, ss := range sss {
		for _, s := range ss {
			if seen[s] {
				continue
			}
			seen[s] = true
			result = append(result, s)
		}
	}
	sort.Strings(result)
	return result
}

// ObfuscateEmail converts a string containing email address email@address.com
// into email<junk>@address.com.
func ObfuscateEmail(email string) template.HTML {
	email = template.HTMLEscapeString(email)
	return template.HTML(strings.Replace(
		email, "@", "ohnoyoudont@", -1))
}

// ShortenEmail shortens Google emails.
func ShortenEmail(email string) string {
	return strings.Replace(email, "@google.com", "", -1)
}

// TagGRPC annotates some gRPC with Milo specific semantics, specifically:
// * Marks the error as Unauthorized if the user is not logged in,
// and the underlying error was a 403 or 404.
// * Otherwise, tag the error with the original error code.
func TagGRPC(c context.Context, err error) error {
	loggedIn := auth.CurrentIdentity(c) != identity.AnonymousIdentity
	code := grpcutil.Code(err)
	if code == codes.NotFound || code == codes.PermissionDenied {
		// Mask the errors, so they look the same.
		if loggedIn {
			return errors.Reason("not found").Tag(grpcutil.NotFoundTag).Err()
		}
		return errors.Reason("not logged in").Tag(grpcutil.UnauthenticatedTag).Err()
	}
	return grpcutil.ToGRPCErr(err)
}

// ParseIntFromForm parses an integer from a form.
func ParseIntFromForm(form url.Values, key string, base int, bitSize int) (int64, error) {
	input, err := ReadExactOneFromForm(form, key)
	if err != nil {
		return 0, err
	}
	ret, err := strconv.ParseInt(input, 10, 64)
	if err != nil {
		return 0, errors.Annotate(err, "invalid %v; expected an integer; actual value: %v", key, input).Err()
	}
	return ret, nil
}

// ReadExactOneFromForm read a string from a form.
// There must be exactly one and non-empty entry of the given key in the form.
func ReadExactOneFromForm(form url.Values, key string) (string, error) {
	input := form[key]
	if len(input) != 1 || input[0] == "" {
		return "", fmt.Errorf("multiple or missing %v; actual value: %v", key, input)
	}
	return input[0], nil
}

// BucketResourceID returns a string identifying the bucket resource.
// It is used when checking bucket permission.
func BucketResourceID(project, bucket string) string {
	return fmt.Sprintf("luci.%s.%s", project, bucket)
}

// LegacyBuilderIDString returns a legacy string identifying the builder.
// It is used in the Milo datastore.
func LegacyBuilderIDString(bid *buildbucketpb.BuilderID) string {
	return fmt.Sprintf("buildbucket/%s/%s", BucketResourceID(bid.Project, bid.Bucket), bid.Builder)
}

var ErrInvalidLegacyBuilderID = errors.New("the string is not a valid legacy builder ID")
var legacyBuilderIDRe = regexp.MustCompile(`^buildbucket/luci\.([^./]+)\.([^/]+)/([^/]+)$`)

// ParseLegacyBuilderID parses the legacy builder ID
// (e.g. `buildbucket/luci.<project>.<bucket>/<builder>`) and returns the
// BuilderID struct.
func ParseLegacyBuilderID(bid string) (*buildbucketpb.BuilderID, error) {
	match := legacyBuilderIDRe.FindStringSubmatch(bid)
	if len(match) == 0 {
		return nil, ErrInvalidLegacyBuilderID
	}
	return &buildbucketpb.BuilderID{
		Project: match[1],
		Bucket:  match[2],
		Builder: match[3],
	}, nil
}

var ErrInvalidBuilderID = errors.New("the string is not a valid builder ID")
var builderIDRe = regexp.MustCompile("^([^/]+)/([^/]+)/([^/]+)$")

// ParseBuilderID parses the canonical builder ID
// (e.g. `<project>/<bucket>/<builder>`) and returns the BuilderID struct.
func ParseBuilderID(bid string) (*buildbucketpb.BuilderID, error) {
	match := builderIDRe.FindStringSubmatch(bid)
	if len(match) == 0 {
		return nil, ErrInvalidBuilderID
	}
	return &buildbucketpb.BuilderID{
		Project: match[1],
		Bucket:  match[2],
		Builder: match[3],
	}, nil
}

var legacyBuildIDWithBuildNumRe = regexp.MustCompile(`^buildbucket/luci\.([^./]+)\.([^/]+)/([^/]+)/(\d+)$`)
var ErrInvalidLegacyBuildID = errors.New("the string is not a valid legacy build ID")

// ParseLegacyBuildID parses the legacy build ID
// (e.g. `buildbucket/luci.<project>.<bucket>/<builder>/<number>`)
func ParseLegacyBuildID(bid string) (builderID *buildbucketpb.BuilderID, number int32, err error) {
	match := legacyBuildIDWithBuildNumRe.FindStringSubmatch(bid)
	if len(match) == 0 {
		return nil, 0, ErrInvalidLegacyBuildID
	}
	builderID = &buildbucketpb.BuilderID{
		Project: match[1],
		Bucket:  match[2],
		Builder: match[3],
	}
	buildNum, err := strconv.ParseInt(match[4], 10, 32)
	if err != nil {
		return nil, 0, ErrInvalidLegacyBuildID
	}
	return builderID, int32(buildNum), nil
}

// JSONMarshalCompressed converts a message into compressed JSON form, suitable for storing in memcache.
func JSONMarshalCompressed(message interface{}) ([]byte, error) {
	// Compress using zlib.
	b := bytes.Buffer{}
	w := zlib.NewWriter(&b)
	enc := json.NewEncoder(w)

	if err := enc.Encode(message); err != nil {
		return nil, err
	}

	// Flush remaining bytes in the zlib writer.
	if err := w.Close(); err != nil {
		return nil, err
	}

	return b.Bytes(), nil
}

// JSONUnmarshalCompressed converts a message back from compressed JSON form.
func JSONUnmarshalCompressed(serialized []byte, out interface{}) error {
	// Decompress using zlib.
	r, err := zlib.NewReader(bytes.NewReader(serialized))
	if err != nil {
		return err
	}

	dec := json.NewDecoder(r)
	return dec.Decode(out)
}

// GetJSONData fetches data from the given URL, parses the response body to `out`.
// It follows redirection and returns an error if the status code is 4xx or 5xx.
func GetJSONData(client *http.Client, url string, out interface{}) (err error) {
	response, err := client.Get(url)
	if err != nil {
		return err
	}
	defer func() {
		err = errors.Flatten(errors.NewMultiError(response.Body.Close(), err))
	}()

	if response.StatusCode >= 400 && response.StatusCode <= 599 {
		return fmt.Errorf("failed to fetch data: %q returned code %q", url, response.Status)
	}

	dec := json.NewDecoder(response.Body)
	return dec.Decode(out)
}
<|endoftext|>


<commit_before>// Windows file locking implementation based on (but heavily modified from)
// https://github.com/golang/build/blob/4821e1d4e1dd5d386f53f1e869ced293dd18f44a/cmd/builder/filemutex_windows.go.
//
// The original code license:
//
// Copyright (c) 2009 The Go Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//    * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//    * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//    * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// The original license header inside the code itself:
//
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package filesystem

import (
	"syscall"
	"unsafe"
)

var (
	kernel32     = syscall.NewLazyDLL("kernel32.dll")
	lockFileEx   = kernel32.NewProc("LockFileEx")
	unlockFileEx = kernel32.NewProc("UnlockFileEx")
)

const (
	LOCKFILE_EXCLUSIVE_LOCK   = 2
	LOCKFILE_FAIL_IMMEDIATELY = 1
)

func callLockFileEx(
	handle syscall.Handle,
	flags,
	reserved,
	lockLow,
	lockHigh uint32,
	overlapped *syscall.Overlapped,
) (err error) {
	r1, _, e1 := syscall.Syscall6(
		lockFileEx.Addr(),
		6,
		uintptr(handle),
		uintptr(flags),
		uintptr(reserved),
		uintptr(lockLow),
		uintptr(lockHigh),
		uintptr(unsafe.Pointer(overlapped)),
	)
	if r1 == 0 {
		if e1 != 0 {
			err = error(e1)
		} else {
			err = syscall.EINVAL
		}
	}
	return
}

func callunlockFileEx(
	handle syscall.Handle,
	reserved,
	lockLow,
	lockHigh uint32,
	overlapped *syscall.Overlapped,
) (err error) {
	r1, _, e1 := syscall.Syscall6(
		unlockFileEx.Addr(),
		5,
		uintptr(handle),
		uintptr(reserved),
		uintptr(lockLow),
		uintptr(lockHigh),
		uintptr(unsafe.Pointer(overlapped)),
		0,
	)
	if r1 == 0 {
		if e1 != 0 {
			err = error(e1)
		} else {
			err = syscall.EINVAL
		}
	}
	return
}

// Lock attempts to acquire the file lock.
func (l *Locker) Lock(block bool) error {
	var ol syscall.Overlapped
	flags := uint32(LOCKFILE_EXCLUSIVE_LOCK)
	if !block {
		flags |= LOCKFILE_FAIL_IMMEDIATELY
	}
	return callLockFileEx(syscall.Handle(l.file.Fd()), flags, 0, 1, 0, &ol)
}

// Unlock releases the file lock.
func (l *Locker) Unlock() error {
	var ol syscall.Overlapped
	return callunlockFileEx(syscall.Handle(l.file.Fd()), 0, 1, 0, &ol)
}
<commit_msg>Incorporated fix for CVE-2019-9634.<commit_after>// Windows file locking implementation based on (but heavily modified from)
// https://github.com/golang/build/blob/4821e1d4e1dd5d386f53f1e869ced293dd18f44a/cmd/builder/filemutex_windows.go.
//
// The original code license:
//
// Copyright (c) 2009 The Go Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//    * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//    * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//    * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// The original license header inside the code itself:
//
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package filesystem

import (
	"syscall"
	"unsafe"

	"golang.org/x/sys/windows"
)

var (
	kernel32     = windows.NewLazySystemDLL("kernel32.dll")
	lockFileEx   = kernel32.NewProc("LockFileEx")
	unlockFileEx = kernel32.NewProc("UnlockFileEx")
)

const (
	LOCKFILE_EXCLUSIVE_LOCK   = 2
	LOCKFILE_FAIL_IMMEDIATELY = 1
)

func callLockFileEx(
	handle syscall.Handle,
	flags,
	reserved,
	lockLow,
	lockHigh uint32,
	overlapped *syscall.Overlapped,
) (err error) {
	r1, _, e1 := syscall.Syscall6(
		lockFileEx.Addr(),
		6,
		uintptr(handle),
		uintptr(flags),
		uintptr(reserved),
		uintptr(lockLow),
		uintptr(lockHigh),
		uintptr(unsafe.Pointer(overlapped)),
	)
	if r1 == 0 {
		if e1 != 0 {
			err = error(e1)
		} else {
			err = syscall.EINVAL
		}
	}
	return
}

func callunlockFileEx(
	handle syscall.Handle,
	reserved,
	lockLow,
	lockHigh uint32,
	overlapped *syscall.Overlapped,
) (err error) {
	r1, _, e1 := syscall.Syscall6(
		unlockFileEx.Addr(),
		5,
		uintptr(handle),
		uintptr(reserved),
		uintptr(lockLow),
		uintptr(lockHigh),
		uintptr(unsafe.Pointer(overlapped)),
		0,
	)
	if r1 == 0 {
		if e1 != 0 {
			err = error(e1)
		} else {
			err = syscall.EINVAL
		}
	}
	return
}

// Lock attempts to acquire the file lock.
func (l *Locker) Lock(block bool) error {
	var ol syscall.Overlapped
	flags := uint32(LOCKFILE_EXCLUSIVE_LOCK)
	if !block {
		flags |= LOCKFILE_FAIL_IMMEDIATELY
	}
	return callLockFileEx(syscall.Handle(l.file.Fd()), flags, 0, 1, 0, &ol)
}

// Unlock releases the file lock.
func (l *Locker) Unlock() error {
	var ol syscall.Overlapped
	return callunlockFileEx(syscall.Handle(l.file.Fd()), 0, 1, 0, &ol)
}
<|endoftext|>