<commit_before>package metrics2

/*
   Convenience utilities for working with InfluxDB.
*/

import (
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/influxdb"
	"go.skia.org/infra/go/util"
)

const (
	DEFAULT_REPORT_FREQUENCY = time.Minute
	PUSH_FREQUENCY           = time.Minute
)

var (
	DefaultClient *Client = &Client{
		aggMetrics: map[string]*aggregateMetric{},
		counters:   map[string]*Counter{},
		metrics:    map[string]*rawMetric{},
	}
)

// Init() initializes the metrics package.
func Init(appName string, influxClient *influxdb.Client) error {
	hostName, err := os.Hostname()
	if err != nil {
		return fmt.Errorf("Unable to retrieve hostname: %s", err)
	}
	tags := map[string]string{
		"app":  appName,
		"host": hostName,
	}
	c, err := NewClient(influxClient, tags, DEFAULT_REPORT_FREQUENCY)
	if err != nil {
		return err
	}
	// Some metrics may already be registered with DefaultClient. Copy them
	// over.
	c.aggMetrics = DefaultClient.aggMetrics
	c.counters = DefaultClient.counters
	c.metrics = DefaultClient.metrics

	// Set the default client.
	DefaultClient = c
	return nil
}

// Client is a struct used for communicating with an InfluxDB instance.
type Client struct {
	aggMetrics    map[string]*aggregateMetric
	aggMetricsMtx sync.Mutex

	counters    map[string]*Counter
	countersMtx sync.Mutex

	influxClient *influxdb.Client
	defaultTags  map[string]string

	metrics    map[string]*rawMetric
	metricsMtx sync.Mutex

	reportFrequency time.Duration
	values          *influxdb.BatchPoints
	valuesMtx       sync.Mutex
}

// NewClient returns a Client which uses the given influxdb.Client to push data.
// defaultTags specifies a set of default tag keys and values which are applied
// to all data points. reportFrequency specifies how often metrics should create
// data points.
func NewClient(influxClient *influxdb.Client, defaultTags map[string]string, reportFrequency time.Duration) (*Client, error) {
	values, err := influxClient.NewBatchPoints()
	if err != nil {
		return nil, err
	}
	c := &Client{
		aggMetrics:      map[string]*aggregateMetric{},
		counters:        map[string]*Counter{},
		influxClient:    influxClient,
		defaultTags:     defaultTags,
		metrics:         map[string]*rawMetric{},
		reportFrequency: reportFrequency,
		values:          values,
	}
	go func() {
		for _ = range time.Tick(PUSH_FREQUENCY) {
			byMeasurement, err := c.pushData()
			if err != nil {
				glog.Errorf("Failed to push data into InfluxDB: %s", err)
			} else {
				total := int64(0)
				for k, v := range byMeasurement {
					c.GetInt64Metric("metrics.points-pushed.by-measurement", map[string]string{"measurement": k}).Update(v)
					total += v
				}
				c.GetInt64Metric("metrics.points-pushed.total", nil).Update(total)
			}
		}
	}()
	go func() {
		for _ = range time.Tick(reportFrequency) {
			c.collectMetrics()
			c.collectAggregateMetrics()
		}
	}()
	return c, nil
}

// collectMetrics collects data points from all raw metrics.
func (c *Client) collectMetrics() {
	c.metricsMtx.Lock()
	defer c.metricsMtx.Unlock()
	for _, m := range c.metrics {
		c.addPoint(m.measurement, m.tags, m.get())
	}
}

// collectAggregateMetrics collects data points from all aggregate metrics.
func (c *Client) collectAggregateMetrics() {
	c.aggMetricsMtx.Lock()
	defer c.aggMetricsMtx.Unlock()
	for _, m := range c.aggMetrics {
		c.addPoint(m.measurement, m.tags, m.reset())
	}
}

// addPointAtTime adds a data point with the given timestamp.
func (c *Client) addPointAtTime(measurement string, tags map[string]string, value interface{}, ts time.Time) {
	c.valuesMtx.Lock()
	defer c.valuesMtx.Unlock()
	if c.values == nil {
		glog.Errorf("Metrics client not initialized; cannot add points.")
		return
	}
	if tags == nil {
		tags = map[string]string{}
	}
	allTags := make(map[string]string, len(tags)+len(c.defaultTags))
	for k, v := range c.defaultTags {
		allTags[k] = v
	}
	for k, v := range tags {
		allTags[k] = v
	}
	if err := c.values.AddPoint(measurement, allTags, map[string]interface{}{"value": value}, ts); err != nil {
		glog.Errorf("Failed to add data point: %s", err)
	}
}

// addPoint adds a data point.
func (c *Client) addPoint(measurement string, tags map[string]string, value interface{}) {
	c.addPointAtTime(measurement, tags, value, time.Now())
}

// RawAddInt64PointAtTime adds an int64 data point to the default client at the
// given time. When possible, use one of the helpers instead.
func RawAddInt64PointAtTime(measurement string, tags map[string]string, value int64, ts time.Time) {
	DefaultClient.addPointAtTime(measurement, tags, value, ts)
}

// pushData pushes all queued data into InfluxDB.
func (c *Client) pushData() (map[string]int64, error) {
	c.valuesMtx.Lock()
	defer c.valuesMtx.Unlock()
	if c.influxClient == nil {
		return nil, fmt.Errorf("InfluxDB client is nil! Cannot push data. Did you initialize the metrics2 package?")
	}

	// Push the points.
	if err := c.influxClient.WriteBatch(c.values); err != nil {
		return nil, err
	}

	// Record the number of points.
	byMeasurement := map[string]int64{}
	points := c.values.Points()
	for _, pt := range points {
		count := byMeasurement[pt.Name()]
		byMeasurement[pt.Name()] = count + 1
	}

	// Get a fresh BatchPoints.
	newValues, err := c.influxClient.NewBatchPoints()
	if err != nil {
		return nil, err
	}
	c.values = newValues
	return byMeasurement, nil
}

// rawMetric is a metric which has no explicit type.
type rawMetric struct {
	measurement string
	mtx         sync.RWMutex
	tags        map[string]string
	value       interface{}
}

// get returns the current value of the metric.
func (m *rawMetric) get() interface{} {
	m.mtx.RLock()
	defer m.mtx.RUnlock()
	return m.value
}

// update adds a data point to the metric.
func (m *rawMetric) update(v interface{}) {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	m.value = v
}

// getRawMetric creates or retrieves a metric with the given measurement name
// and tag set and returns it.
func (c *Client) getRawMetric(measurement string, tagsList []map[string]string, initial interface{}) *rawMetric {
	c.metricsMtx.Lock()
	defer c.metricsMtx.Unlock()

	// Make a copy of the concatenation of all provided tags.
	tags := util.AddParams(map[string]string{}, tagsList...)
	md5, err := util.MD5Params(tags)
	if err != nil {
		glog.Errorf("Failed to encode measurement tags: %s", err)
	}
	key := fmt.Sprintf("%s_%s", measurement, md5)
	m, ok := c.metrics[key]
	if !ok {
		m = &rawMetric{
			measurement: measurement,
			tags:        tags,
			value:       initial,
		}
		c.metrics[key] = m
	}
	return m
}

// getAggregateMetric creates or retrieves an aggregateMetric with the given
// measurement name and tag set and returns it.
func (c *Client) getAggregateMetric(measurement string, tagsList []map[string]string, aggFn func([]interface{}) interface{}) *aggregateMetric {
	c.aggMetricsMtx.Lock()
	defer c.aggMetricsMtx.Unlock()

	// Make a copy of the concatenation of all provided tags.
	tags := util.AddParams(map[string]string{}, tagsList...)
	md5, err := util.MD5Params(tags)
	if err != nil {
		glog.Errorf("Failed to encode measurement tags: %s", err)
	}
	key := fmt.Sprintf("%s_%s", measurement, md5)
	m, ok := c.aggMetrics[key]
	if !ok {
		m = &aggregateMetric{
			aggFn:       aggFn,
			measurement: measurement,
			tags:        tags,
			values:      []interface{}{},
		}
		c.aggMetrics[key] = m
	}
	return m
}
<commit_msg>metrics2: Always clear out data, even when the push fails<commit_after>package metrics2

/*
   Convenience utilities for working with InfluxDB.
*/

import (
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/influxdb"
	"go.skia.org/infra/go/util"
)

const (
	DEFAULT_REPORT_FREQUENCY = time.Minute
	PUSH_FREQUENCY           = time.Minute
)

var (
	DefaultClient *Client = &Client{
		aggMetrics: map[string]*aggregateMetric{},
		counters:   map[string]*Counter{},
		metrics:    map[string]*rawMetric{},
	}
)

// Init() initializes the metrics package.
func Init(appName string, influxClient *influxdb.Client) error {
	hostName, err := os.Hostname()
	if err != nil {
		return fmt.Errorf("Unable to retrieve hostname: %s", err)
	}
	tags := map[string]string{
		"app":  appName,
		"host": hostName,
	}
	c, err := NewClient(influxClient, tags, DEFAULT_REPORT_FREQUENCY)
	if err != nil {
		return err
	}
	// Some metrics may already be registered with DefaultClient. Copy them
	// over.
	c.aggMetrics = DefaultClient.aggMetrics
	c.counters = DefaultClient.counters
	c.metrics = DefaultClient.metrics

	// Set the default client.
	DefaultClient = c
	return nil
}

// Client is a struct used for communicating with an InfluxDB instance.
type Client struct {
	aggMetrics    map[string]*aggregateMetric
	aggMetricsMtx sync.Mutex

	counters    map[string]*Counter
	countersMtx sync.Mutex

	influxClient *influxdb.Client
	defaultTags  map[string]string

	metrics    map[string]*rawMetric
	metricsMtx sync.Mutex

	reportFrequency time.Duration
	values          *influxdb.BatchPoints
	valuesMtx       sync.Mutex
}

// NewClient returns a Client which uses the given influxdb.Client to push data.
// defaultTags specifies a set of default tag keys and values which are applied
// to all data points. reportFrequency specifies how often metrics should create
// data points.
func NewClient(influxClient *influxdb.Client, defaultTags map[string]string, reportFrequency time.Duration) (*Client, error) {
	values, err := influxClient.NewBatchPoints()
	if err != nil {
		return nil, err
	}
	c := &Client{
		aggMetrics:      map[string]*aggregateMetric{},
		counters:        map[string]*Counter{},
		influxClient:    influxClient,
		defaultTags:     defaultTags,
		metrics:         map[string]*rawMetric{},
		reportFrequency: reportFrequency,
		values:          values,
	}
	go func() {
		for _ = range time.Tick(PUSH_FREQUENCY) {
			byMeasurement, err := c.pushData()
			if err != nil {
				glog.Errorf("Failed to push data into InfluxDB: %s", err)
			} else {
				total := int64(0)
				for k, v := range byMeasurement {
					c.GetInt64Metric("metrics.points-pushed.by-measurement", map[string]string{"measurement": k}).Update(v)
					total += v
				}
				c.GetInt64Metric("metrics.points-pushed.total", nil).Update(total)
			}
		}
	}()
	go func() {
		for _ = range time.Tick(reportFrequency) {
			c.collectMetrics()
			c.collectAggregateMetrics()
		}
	}()
	return c, nil
}

// collectMetrics collects data points from all raw metrics.
func (c *Client) collectMetrics() {
	c.metricsMtx.Lock()
	defer c.metricsMtx.Unlock()
	for _, m := range c.metrics {
		c.addPoint(m.measurement, m.tags, m.get())
	}
}

// collectAggregateMetrics collects data points from all aggregate metrics.
func (c *Client) collectAggregateMetrics() {
	c.aggMetricsMtx.Lock()
	defer c.aggMetricsMtx.Unlock()
	for _, m := range c.aggMetrics {
		c.addPoint(m.measurement, m.tags, m.reset())
	}
}

// addPointAtTime adds a data point with the given timestamp.
func (c *Client) addPointAtTime(measurement string, tags map[string]string, value interface{}, ts time.Time) {
	c.valuesMtx.Lock()
	defer c.valuesMtx.Unlock()
	if c.values == nil {
		glog.Errorf("Metrics client not initialized; cannot add points.")
		return
	}
	if tags == nil {
		tags = map[string]string{}
	}
	allTags := make(map[string]string, len(tags)+len(c.defaultTags))
	for k, v := range c.defaultTags {
		allTags[k] = v
	}
	for k, v := range tags {
		allTags[k] = v
	}
	if err := c.values.AddPoint(measurement, allTags, map[string]interface{}{"value": value}, ts); err != nil {
		glog.Errorf("Failed to add data point: %s", err)
	}
}

// addPoint adds a data point.
func (c *Client) addPoint(measurement string, tags map[string]string, value interface{}) {
	c.addPointAtTime(measurement, tags, value, time.Now())
}

// RawAddInt64PointAtTime adds an int64 data point to the default client at the
// given time. When possible, use one of the helpers instead.
func RawAddInt64PointAtTime(measurement string, tags map[string]string, value int64, ts time.Time) {
	DefaultClient.addPointAtTime(measurement, tags, value, ts)
}

// pushData pushes all queued data into InfluxDB.
func (c *Client) pushData() (map[string]int64, error) {
	c.valuesMtx.Lock()
	defer c.valuesMtx.Unlock()

	// Always clear out the values after pushing, even if we failed.
	newValues, err := c.influxClient.NewBatchPoints()
	if err != nil {
		return nil, err
	}
	defer func() {
		c.values = newValues
	}()

	if c.influxClient == nil {
		return nil, fmt.Errorf("InfluxDB client is nil! Cannot push data. Did you initialize the metrics2 package?")
	}

	// Push the points.
	if err := c.influxClient.WriteBatch(c.values); err != nil {
		return nil, err
	}

	// Record the number of points.
	byMeasurement := map[string]int64{}
	points := c.values.Points()
	for _, pt := range points {
		count := byMeasurement[pt.Name()]
		byMeasurement[pt.Name()] = count + 1
	}

	return byMeasurement, nil
}

// rawMetric is a metric which has no explicit type.
type rawMetric struct {
	measurement string
	mtx         sync.RWMutex
	tags        map[string]string
	value       interface{}
}

// get returns the current value of the metric.
func (m *rawMetric) get() interface{} {
	m.mtx.RLock()
	defer m.mtx.RUnlock()
	return m.value
}

// update adds a data point to the metric.
func (m *rawMetric) update(v interface{}) {
	m.mtx.Lock()
	defer m.mtx.Unlock()
	m.value = v
}

// getRawMetric creates or retrieves a metric with the given measurement name
// and tag set and returns it.
func (c *Client) getRawMetric(measurement string, tagsList []map[string]string, initial interface{}) *rawMetric {
	c.metricsMtx.Lock()
	defer c.metricsMtx.Unlock()

	// Make a copy of the concatenation of all provided tags.
	tags := util.AddParams(map[string]string{}, tagsList...)
	md5, err := util.MD5Params(tags)
	if err != nil {
		glog.Errorf("Failed to encode measurement tags: %s", err)
	}
	key := fmt.Sprintf("%s_%s", measurement, md5)
	m, ok := c.metrics[key]
	if !ok {
		m = &rawMetric{
			measurement: measurement,
			tags:        tags,
			value:       initial,
		}
		c.metrics[key] = m
	}
	return m
}

// getAggregateMetric creates or retrieves an aggregateMetric with the given
// measurement name and tag set and returns it.
func (c *Client) getAggregateMetric(measurement string, tagsList []map[string]string, aggFn func([]interface{}) interface{}) *aggregateMetric {
	c.aggMetricsMtx.Lock()
	defer c.aggMetricsMtx.Unlock()

	// Make a copy of the concatenation of all provided tags.
	tags := util.AddParams(map[string]string{}, tagsList...)
	md5, err := util.MD5Params(tags)
	if err != nil {
		glog.Errorf("Failed to encode measurement tags: %s", err)
	}
	key := fmt.Sprintf("%s_%s", measurement, md5)
	m, ok := c.aggMetrics[key]
	if !ok {
		m = &aggregateMetric{
			aggFn:       aggFn,
			measurement: measurement,
			tags:        tags,
			values:      []interface{}{},
		}
		c.aggMetrics[key] = m
	}
	return m
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"flag"
	"os"
	"strings"
)

func new_token(username string, timestamp int, secret string) string {
	h := hmac.New(sha256.New, []byte(secret))
	unauthed_token := fmt.Sprint(timestamp,":",username)
	h.Write([]byte(unauthed_token))
	return fmt.Sprint(hex.EncodeToString(h.Sum(nil)), ":", timestamp, ":", username)
}
func converTime2GraphiteFormat(time2convert int64) string {
	timeRange := time.Unix(time.Now().Unix() - time2convert, 0)
	return strings.Replace(timeRange.Format("15:04 20060102")," ","_", -1) //10:0020150923
}

func main() {
	var username, authToken, metric, url string
	var thresholdWarningI, thresholdCriticalI, thresholdWarningD, thresholdCriticalD int
	var range1FromAgo, range1UntilAgo, range2FromAgo, range2UntilAgo int64
	var debug bool
	flag.StringVar(&username, "u", "graphite", "User, which has rights to access Graphite")
	flag.StringVar(&authToken, "a", "", "AuthToken to access the graphite-API. For example 'qqq'")
	flag.StringVar(&metric, "m", "", "Name of metric or metric filter e.g. qqqq.test.leoleovich.currentProblems")
	flag.StringVar(&url, "U", "", "Base address of your graphite server e.g. https://graphite.protury.info/")

	flag.Int64Var(&range1FromAgo, "range1From", 90000, "Amount of seconds ago for the 1st range (from)")
	flag.Int64Var(&range1UntilAgo, "range1Until", 86400, "Amount of seconds ago for the 1st range (until)")
	flag.Int64Var(&range2FromAgo, "range2From", 3600, "Amount of seconds ago for the 2st range (from)")
	flag.Int64Var(&range2UntilAgo, "range2Until", 0, "Amount of seconds ago for the 2st range (until)")

	flag.IntVar(&thresholdWarningI, "wi", 20, "Metrics above this threshold will be marked as warning")
	flag.IntVar(&thresholdCriticalI, "ci", 40, "Metrics above this threshold will be marked as critical")
	flag.IntVar(&thresholdWarningD, "wd", 20, "Metrics below this threshold will be marked as warning")
	flag.IntVar(&thresholdCriticalD, "cd", 40, "Metrics below this threshold will be marked as critical")

	flag.BoolVar(&debug, "d", false, "Debug mode will print a lot of additinal info")
	flag.Parse()

	if authToken == "" ||  metric == "" || url == "" {
		fmt.Println("URL (-U), authToken (-a) and metric (-m) attributes are required")
		os.Exit(5)
	}
	if thresholdCriticalD < thresholdWarningD || thresholdCriticalI < thresholdWarningI {
		fmt.Println("Critical threshold can not be less, than warning")
		os.Exit(5)
	}
	range1FromS := converTime2GraphiteFormat(range1FromAgo)
	range1UntilS := converTime2GraphiteFormat(range1UntilAgo)
	range2FromS := converTime2GraphiteFormat(range2FromAgo)
	range2UntilS := converTime2GraphiteFormat(range2UntilAgo)

	if debug {
		fmt.Println("Token: " + authToken)
		fmt.Println("Range1: " + range1FromS + " - " + range1UntilS)
		fmt.Println("Range2: " + range2FromS + " - " + range2UntilS)
	}

	cm := CompareMetrics{
		GraphiteClient{new_token(username, int(time.Now().Unix()), authToken), url},
		metric,
		range1FromS,
		range1UntilS,
		range2FromS,
		range2UntilS,
		thresholdWarningI,
		thresholdCriticalI,
		thresholdWarningD,
		thresholdCriticalD}

	// Compare metrics and return result
	result, returnCode := cm.analysisOfMetrics(debug)

	// Print and exit
	fmt.Println(result)
	os.Exit(returnCode)
}
<commit_msg>Fix for convertTime2GraphiteFormat(). Use UTC.<commit_after>package main

import (
	"fmt"
	"time"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"flag"
	"os"
	"strings"
)

func new_token(username string, timestamp int, secret string) string {
	h := hmac.New(sha256.New, []byte(secret))
	unauthed_token := fmt.Sprint(timestamp,":",username)
	h.Write([]byte(unauthed_token))
	return fmt.Sprint(hex.EncodeToString(h.Sum(nil)), ":", timestamp, ":", username)
}
func converTime2GraphiteFormat(time2convert int64) string {
	timeRange := time.Unix(time.Now().UTC().Unix() - time2convert, 0)
	return strings.Replace(timeRange.Format("15:04 20060102")," ","_", -1) //10:0020150923
}

func main() {
	var username, authToken, metric, url string
	var thresholdWarningI, thresholdCriticalI, thresholdWarningD, thresholdCriticalD int
	var range1FromAgo, range1UntilAgo, range2FromAgo, range2UntilAgo int64
	var debug bool
	flag.StringVar(&username, "u", "graphite", "User, which has rights to access Graphite")
	flag.StringVar(&authToken, "a", "", "AuthToken to access the graphite-API. For example 'qqq'")
	flag.StringVar(&metric, "m", "", "Name of metric or metric filter e.g. qqqq.test.leoleovich.currentProblems")
	flag.StringVar(&url, "U", "", "Base address of your graphite server e.g. https://graphite.protury.info/")

	flag.Int64Var(&range1FromAgo, "range1From", 90000, "Amount of seconds ago for the 1st range (from)")
	flag.Int64Var(&range1UntilAgo, "range1Until", 86400, "Amount of seconds ago for the 1st range (until)")
	flag.Int64Var(&range2FromAgo, "range2From", 3600, "Amount of seconds ago for the 2st range (from)")
	flag.Int64Var(&range2UntilAgo, "range2Until", 0, "Amount of seconds ago for the 2st range (until)")

	flag.IntVar(&thresholdWarningI, "wi", 20, "Metrics above this threshold will be marked as warning")
	flag.IntVar(&thresholdCriticalI, "ci", 40, "Metrics above this threshold will be marked as critical")
	flag.IntVar(&thresholdWarningD, "wd", 20, "Metrics below this threshold will be marked as warning")
	flag.IntVar(&thresholdCriticalD, "cd", 40, "Metrics below this threshold will be marked as critical")

	flag.BoolVar(&debug, "d", false, "Debug mode will print a lot of additinal info")
	flag.Parse()

	if authToken == "" ||  metric == "" || url == "" {
		fmt.Println("URL (-U), authToken (-a) and metric (-m) attributes are required")
		os.Exit(5)
	}
	if thresholdCriticalD < thresholdWarningD || thresholdCriticalI < thresholdWarningI {
		fmt.Println("Critical threshold can not be less, than warning")
		os.Exit(5)
	}
	range1FromS := converTime2GraphiteFormat(range1FromAgo)
	range1UntilS := converTime2GraphiteFormat(range1UntilAgo)
	range2FromS := converTime2GraphiteFormat(range2FromAgo)
	range2UntilS := converTime2GraphiteFormat(range2UntilAgo)

	if debug {
		fmt.Println("Token: " + authToken)
		fmt.Println("Range1: " + range1FromS + " - " + range1UntilS)
		fmt.Println("Range2: " + range2FromS + " - " + range2UntilS)
	}

	cm := CompareMetrics{
		GraphiteClient{new_token(username, int(time.Now().Unix()), authToken), url},
		metric,
		range1FromS,
		range1UntilS,
		range2FromS,
		range2UntilS,
		thresholdWarningI,
		thresholdCriticalI,
		thresholdWarningD,
		thresholdCriticalD}

	// Compare metrics and return result
	result, returnCode := cm.analysisOfMetrics(debug)

	// Print and exit
	fmt.Println(result)
	os.Exit(returnCode)
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2019 Red Hat, Inc.
 *
 */
package v1

import (
	"os"

	extv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/client-go/kubernetes/scheme"
)

// GroupName is the group name use in this package
const GroupName = "kubevirt.io"
const SubresourceGroupName = "subresources.kubevirt.io"
const KubeVirtClientGoSchemeRegistrationVersionEnvVar = "KUBEVIRT_CLIENT_GO_SCHEME_REGISTRATION_VERSION"

var (
	ApiLatestVersion            = "v1"
	ApiSupportedWebhookVersions = []string{"v1alpha3", "v1"}
	ApiStorageVersion           = "v1alpha3"
	ApiSupportedVersions        = []extv1beta1.CustomResourceDefinitionVersion{
		{
			Name:    "v1",
			Served:  true,
			Storage: false,
		},
		{
			Name:    "v1alpha3",
			Served:  true,
			Storage: true,
		},
	}
)

var (
	// GroupVersion is the latest group version for the KubeVirt api
	GroupVersion = schema.GroupVersion{Group: GroupName, Version: ApiLatestVersion}

	// StorageGroupVersion is the group version our api is persistented internally as
	StorageGroupVersion = schema.GroupVersion{Group: GroupName, Version: ApiStorageVersion}

	// GroupVersions is group version list used to register these objects
	// The preferred group version is the first item in the list.
	GroupVersions = []schema.GroupVersion{{Group: GroupName, Version: "v1"}, {Group: GroupName, Version: "v1alpha3"}}

	// SubresourceGroupVersions is group version list used to register these objects
	// The preferred group version is the first item in the list.
	SubresourceGroupVersions = []schema.GroupVersion{{Group: SubresourceGroupName, Version: ApiLatestVersion}, {Group: SubresourceGroupName, Version: ApiStorageVersion}}

	// SubresourceStorageGroupVersion is the group version our api is persistented internally as
	SubresourceStorageGroupVersion = schema.GroupVersion{Group: SubresourceGroupName, Version: ApiStorageVersion}
)

var (
	// GroupVersionKind
	VirtualMachineInstanceGroupVersionKind           = schema.GroupVersionKind{Group: GroupName, Version: GroupVersion.Version, Kind: "VirtualMachineInstance"}
	VirtualMachineInstanceReplicaSetGroupVersionKind = schema.GroupVersionKind{Group: GroupName, Version: GroupVersion.Version, Kind: "VirtualMachineInstanceReplicaSet"}
	VirtualMachineInstancePresetGroupVersionKind     = schema.GroupVersionKind{Group: GroupName, Version: GroupVersion.Version, Kind: "VirtualMachineInstancePreset"}
	VirtualMachineGroupVersionKind                   = schema.GroupVersionKind{Group: GroupName, Version: GroupVersion.Version, Kind: "VirtualMachine"}
	VirtualMachineInstanceMigrationGroupVersionKind  = schema.GroupVersionKind{Group: GroupName, Version: GroupVersion.Version, Kind: "VirtualMachineInstanceMigration"}
	KubeVirtGroupVersionKind                         = schema.GroupVersionKind{Group: GroupName, Version: GroupVersion.Version, Kind: "KubeVirt"}
)

var (
	SchemeBuilder  = runtime.NewSchemeBuilder(addKnownTypes)
	Scheme         = runtime.NewScheme()
	AddToScheme    = SchemeBuilder.AddToScheme
	Codecs         = serializer.NewCodecFactory(Scheme)
	ParameterCodec = runtime.NewParameterCodec(Scheme)
)

func init() {
	AddToScheme(Scheme)
	AddToScheme(scheme.Scheme)
}

// Adds the list of known types to api.Scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
	registerGroupVersions := []schema.GroupVersion{}

	// This allows consumers of the KubeVirt client go package to
	// customize what version the client uses. Without specifying a
	// version, all versions are registered. While this techincally
	// file to register all versions, so k8s ecosystem libraries
	// do not work well with this. By explicitly setting the env var,
	// consumers of our client go can avoid these scenarios by only
	// registering a single version
	registerVersion := os.Getenv(KubeVirtClientGoSchemeRegistrationVersionEnvVar)
	if registerVersion != "" {
		registerGroupVersions = append(registerGroupVersions, schema.GroupVersion{Group: GroupName, Version: registerVersion})
	} else {
		registerGroupVersions = append(registerGroupVersions, GroupVersions...)
	}

	for _, groupVersion := range registerGroupVersions {
		scheme.AddKnownTypes(groupVersion,
			&VirtualMachineInstance{},
			&VirtualMachineInstanceList{},
			&VirtualMachineInstanceReplicaSet{},
			&VirtualMachineInstanceReplicaSetList{},
			&VirtualMachineInstancePreset{},
			&VirtualMachineInstancePresetList{},
			&VirtualMachineInstanceMigration{},
			&VirtualMachineInstanceMigrationList{},
			&VirtualMachine{},
			&VirtualMachineList{},
			&KubeVirt{},
			&KubeVirtList{},
		)
		metav1.AddToGroupVersion(scheme, groupVersion)
	}

	return nil
}

// Resource takes an unqualified resource and returns a Group qualified GroupResource
func Resource(resource string) schema.GroupResource {
	return GroupVersion.WithResource(resource).GroupResource()
}
<commit_msg>Define SchemeGroupVersion in client-go<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2019 Red Hat, Inc.
 *
 */
package v1

import (
	"os"

	extv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/client-go/kubernetes/scheme"
)

// GroupName is the group name use in this package
const GroupName = "kubevirt.io"
const SubresourceGroupName = "subresources.kubevirt.io"
const KubeVirtClientGoSchemeRegistrationVersionEnvVar = "KUBEVIRT_CLIENT_GO_SCHEME_REGISTRATION_VERSION"

var (
	ApiLatestVersion            = "v1"
	ApiSupportedWebhookVersions = []string{"v1alpha3", "v1"}
	ApiStorageVersion           = "v1alpha3"
	ApiSupportedVersions        = []extv1beta1.CustomResourceDefinitionVersion{
		{
			Name:    "v1",
			Served:  true,
			Storage: false,
		},
		{
			Name:    "v1alpha3",
			Served:  true,
			Storage: true,
		},
	}
)

var (
	// GroupVersion is the latest group version for the KubeVirt api
	GroupVersion       = schema.GroupVersion{Group: GroupName, Version: ApiLatestVersion}
	SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: ApiLatestVersion}

	// StorageGroupVersion is the group version our api is persistented internally as
	StorageGroupVersion = schema.GroupVersion{Group: GroupName, Version: ApiStorageVersion}

	// GroupVersions is group version list used to register these objects
	// The preferred group version is the first item in the list.
	GroupVersions = []schema.GroupVersion{{Group: GroupName, Version: "v1"}, {Group: GroupName, Version: "v1alpha3"}}

	// SubresourceGroupVersions is group version list used to register these objects
	// The preferred group version is the first item in the list.
	SubresourceGroupVersions = []schema.GroupVersion{{Group: SubresourceGroupName, Version: ApiLatestVersion}, {Group: SubresourceGroupName, Version: ApiStorageVersion}}

	// SubresourceStorageGroupVersion is the group version our api is persistented internally as
	SubresourceStorageGroupVersion = schema.GroupVersion{Group: SubresourceGroupName, Version: ApiStorageVersion}
)

var (
	// GroupVersionKind
	VirtualMachineInstanceGroupVersionKind           = schema.GroupVersionKind{Group: GroupName, Version: GroupVersion.Version, Kind: "VirtualMachineInstance"}
	VirtualMachineInstanceReplicaSetGroupVersionKind = schema.GroupVersionKind{Group: GroupName, Version: GroupVersion.Version, Kind: "VirtualMachineInstanceReplicaSet"}
	VirtualMachineInstancePresetGroupVersionKind     = schema.GroupVersionKind{Group: GroupName, Version: GroupVersion.Version, Kind: "VirtualMachineInstancePreset"}
	VirtualMachineGroupVersionKind                   = schema.GroupVersionKind{Group: GroupName, Version: GroupVersion.Version, Kind: "VirtualMachine"}
	VirtualMachineInstanceMigrationGroupVersionKind  = schema.GroupVersionKind{Group: GroupName, Version: GroupVersion.Version, Kind: "VirtualMachineInstanceMigration"}
	KubeVirtGroupVersionKind                         = schema.GroupVersionKind{Group: GroupName, Version: GroupVersion.Version, Kind: "KubeVirt"}
)

var (
	SchemeBuilder  = runtime.NewSchemeBuilder(addKnownTypes)
	Scheme         = runtime.NewScheme()
	AddToScheme    = SchemeBuilder.AddToScheme
	Codecs         = serializer.NewCodecFactory(Scheme)
	ParameterCodec = runtime.NewParameterCodec(Scheme)
)

func init() {
	AddToScheme(Scheme)
	AddToScheme(scheme.Scheme)
}

// Adds the list of known types to api.Scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
	registerGroupVersions := []schema.GroupVersion{}

	// This allows consumers of the KubeVirt client go package to
	// customize what version the client uses. Without specifying a
	// version, all versions are registered. While this techincally
	// file to register all versions, so k8s ecosystem libraries
	// do not work well with this. By explicitly setting the env var,
	// consumers of our client go can avoid these scenarios by only
	// registering a single version
	registerVersion := os.Getenv(KubeVirtClientGoSchemeRegistrationVersionEnvVar)
	if registerVersion != "" {
		registerGroupVersions = append(registerGroupVersions, schema.GroupVersion{Group: GroupName, Version: registerVersion})
	} else {
		registerGroupVersions = append(registerGroupVersions, GroupVersions...)
	}

	for _, groupVersion := range registerGroupVersions {
		scheme.AddKnownTypes(groupVersion,
			&VirtualMachineInstance{},
			&VirtualMachineInstanceList{},
			&VirtualMachineInstanceReplicaSet{},
			&VirtualMachineInstanceReplicaSetList{},
			&VirtualMachineInstancePreset{},
			&VirtualMachineInstancePresetList{},
			&VirtualMachineInstanceMigration{},
			&VirtualMachineInstanceMigrationList{},
			&VirtualMachine{},
			&VirtualMachineList{},
			&KubeVirt{},
			&KubeVirtList{},
		)
		metav1.AddToGroupVersion(scheme, groupVersion)
	}

	return nil
}

// Resource takes an unqualified resource and returns a Group qualified GroupResource
func Resource(resource string) schema.GroupResource {
	return GroupVersion.WithResource(resource).GroupResource()
}
<|endoftext|>


<commit_before>package varint

import (
	"io"
)

// WriteTo writes the given value to the given io.Writer as a varint encoded
// byte array.
//
// On success, the function returns the number of bytes written to the writer,
// and nil.
func WriteTo(w io.Writer, value int64) (int64, error) {
	var length int
	var buff [2]byte
	if value <= 0x7F {
		buff[0] = byte(value)
		length = 1
	} else if value <= 0x3FFF {
		buff[0] = byte(((value >> 8) & 0x3F) | 0x80)
		buff[1] = byte(value & 0xFF)
		length = 2
	}
	if length > 0 {
		if n, err := w.Write(buff[:length]); err != nil {
			return int64(n), err
		} else {
			return int64(n), nil
		}
	}
	return 0, ErrOutOfRange
}
<commit_msg>increase varint.WriteTo range<commit_after>package varint

import (
	"io"
)

// WriteTo writes the given value to the given io.Writer as a varint encoded
// byte array.
//
// On success, the function returns the number of bytes written to the writer,
// and nil.
func WriteTo(w io.Writer, value int64) (int64, error) {
	var length int
	var buff [4]byte
	if value <= 0x7F {
		buff[0] = byte(value)
		length = 1
	} else if value <= 0x3FFF {
		buff[0] = byte(((value >> 8) & 0x3F) | 0x80)
		buff[1] = byte(value & 0xFF)
		length = 2
	} else if value <= 0x1FFFFF {
		buff[0] = byte((value >> 16) & 0x1F | 0xC0)
		buff[1] = byte((value >> 8) & 0xFF)
		buff[2] = byte(value & 0xFF)
		length = 3
	} else if value <= 0xFFFFFFF {
		buff[0] = byte((value >> 24) & 0xF | 0xE0)
		buff[1] = byte((value >> 16) & 0xFF)
		buff[2] = byte((value >> 8) & 0xFF)
		buff[3] = byte(value & 0xFF)
		length = 4
	}
	if length > 0 {
		if n, err := w.Write(buff[:length]); err != nil {
			return int64(n), err
		} else {
			return int64(n), nil
		}
	}
	return 0, ErrOutOfRange
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"time"

	log "github.com/cihub/seelog"
	"github.com/stampzilla/gocast"
	"github.com/stampzilla/gocast/events"
	"github.com/stampzilla/gocast/handlers"
	"github.com/stampzilla/gocast/responses"
)

type Chromecast struct {
	Id    string
	Name_ string `json:"Name"`

	PrimaryApp      string
	PrimaryEndpoint string
	Playing         bool
	//Paused          bool

	IsStandBy     bool
	IsActiveInput bool

	Volume float64
	Muted  bool

	Addr net.IP
	Port int

	publish func()

	mediaHandler           *handlers.Media
	mediaConnectionHandler *handlers.Connection

	appLaunch chan string

	*gocast.Device
}

func NewChromecast(d *gocast.Device) *Chromecast {
	c := &Chromecast{
		Device: d,
	}

	d.OnEvent(c.Event)

	c.mediaHandler = &handlers.Media{}
	c.mediaConnectionHandler = &handlers.Connection{}
	c.appLaunch = make(chan string)

	return c
}

func (c *Chromecast) Play() {
	c.mediaHandler.Play()
}
func (c *Chromecast) Pause() {
	c.mediaHandler.Pause()
}
func (c *Chromecast) Stop() {
	c.mediaHandler.Stop()
}

func (c *Chromecast) PlayUrl(url string, contentType string) {
	err := c.Device.ReceiverHandler.LaunchApp(gocast.AppMedia)
	if err != nil && err != handlers.ErrAppAlreadyLaunched {
		log.Error(err)
		return
	}

	if err != handlers.ErrAppAlreadyLaunched {
		//Wait for new media connection to launched app
		if err := c.waitForAppLaunch(gocast.AppMedia); err != nil {
			log.Error(err)
			return
		}
	}

	if contentType == "" {
		contentType = "audio/mpeg"
	}
	item := responses.MediaItem{
		ContentId:   url,
		StreamType:  "BUFFERED",
		ContentType: contentType,
	}
	err = c.mediaHandler.LoadMedia(item, 0, true, map[string]interface{}{})
	if err != nil {
		log.Error(err)
		return
	}
}

func (c *Chromecast) waitForAppLaunch(app string) error {
	select {
	case launchedApp := <-c.appLaunch:
		if app == launchedApp {
			return nil
		}
		return fmt.Errorf("Wrong app launched. Expected %s got %s", app, launchedApp)
	case <-time.After(time.Second * 20):
		return fmt.Errorf("timeout waiting for app launch after 20 seconds")

	}

}
func (c *Chromecast) appLaunched(app string) {
	select {
	case c.appLaunch <- app:
		log.Info("Notified c.appLaunch")
	default:
		log.Info("No one is waiting for appLaunch event")
	}
}

func (c *Chromecast) Event(event events.Event) {
	switch data := event.(type) {
	case events.Connected:
		log.Info(c.Name(), "- Connected")

		c.Addr = c.Ip()
		c.Port = c.Device.Port()
		c.Id = c.Uuid()
		c.Name_ = c.Name()

		state.Add(c)
	case events.Disconnected:
		log.Warn(c.Name(), "- Disconnected")

		state.Remove(c)
	case events.AppStarted:
		log.Info(c.Name(), "- App started:", data.DisplayName, "(", data.AppID, ")")
		//spew.Dump("Data:", data)

		c.PrimaryApp = data.DisplayName
		c.PrimaryEndpoint = data.TransportId

		//If the app supports media controls lets subscribe to it
		if data.HasNamespace("urn:x-cast:com.google.cast.media") {
			log.Info(c.Name(), "- Subscribe cast.tp.connection:", data.DisplayName, "(", data.AppID, ")")
			c.Subscribe("urn:x-cast:com.google.cast.tp.connection", data.TransportId, c.mediaConnectionHandler)
			log.Info(c.Name(), "- Subscribe cast.media:", data.DisplayName, "(", data.AppID, ")")
			c.Subscribe("urn:x-cast:com.google.cast.media", data.TransportId, c.mediaHandler)
		}
		c.appLaunched(data.AppID)
		log.Info(c.Name(), "- Notifying appLanunched:", data.DisplayName, "(", data.AppID, ")")

	case events.AppStopped:
		log.Info(c.Name(), "- App stopped:", data.DisplayName, "(", data.AppID, ")")
		//spew.Dump("Data:", data)

		//unsubscribe from old channels
		for _, v := range data.Namespaces {
			c.UnsubscribeByUrnAndDestinationId(v.Name, data.TransportId)
		}
		c.PrimaryApp = ""
		c.PrimaryEndpoint = ""
		c.Playing = false

	case events.ReceiverStatus:
		c.IsStandBy = data.Status.IsStandBy
		c.IsActiveInput = data.Status.IsActiveInput
		c.Volume = data.Status.Volume.Level
		c.Muted = data.Status.Volume.Muted
	case events.Media:
		playing := c.Playing
		if data.PlayerState == "PLAYING" {
			c.Playing = true
		} else {
			c.Playing = false
		}

		//Only publish if playing state changed
		if playing != c.Playing {
			c.publish()
		}
		return

	//gocast.MediaEvent:
	default:
		log.Warn("unexpected event %T: %#v\n", data, data)
	}

	c.publish()
}
<commit_msg>Possible bugfix<commit_after>package main

import (
	"fmt"
	"net"
	"time"

	log "github.com/cihub/seelog"
	"github.com/stampzilla/gocast"
	"github.com/stampzilla/gocast/events"
	"github.com/stampzilla/gocast/handlers"
	"github.com/stampzilla/gocast/responses"
)

type Chromecast struct {
	Id    string
	Name_ string `json:"Name"`

	PrimaryApp      string
	PrimaryEndpoint string
	Playing         bool
	//Paused          bool

	IsStandBy     bool
	IsActiveInput bool

	Volume float64
	Muted  bool

	Addr net.IP
	Port int

	publish func()

	mediaHandler           *handlers.Media
	mediaConnectionHandler *handlers.Connection

	appLaunch chan string

	*gocast.Device
}

func NewChromecast(d *gocast.Device) *Chromecast {
	c := &Chromecast{
		Device:                 d,
		mediaHandler:           &handlers.Media{},
		mediaConnectionHandler: &handlers.Connection{},
		appLaunch:              make(chan string),
	}

	d.OnEvent(c.Event)
	return c
}

func (c *Chromecast) Play() {
	c.mediaHandler.Play()
}
func (c *Chromecast) Pause() {
	c.mediaHandler.Pause()
}
func (c *Chromecast) Stop() {
	c.mediaHandler.Stop()
}

func (c *Chromecast) PlayUrl(url string, contentType string) {
	err := c.Device.ReceiverHandler.LaunchApp(gocast.AppMedia)
	if err != nil && err != handlers.ErrAppAlreadyLaunched {
		log.Error(err)
		return
	}

	if err != handlers.ErrAppAlreadyLaunched {
		//Wait for new media connection to launched app
		if err := c.waitForAppLaunch(gocast.AppMedia); err != nil {
			log.Error(err)
			return
		}
	}

	if contentType == "" {
		contentType = "audio/mpeg"
	}
	item := responses.MediaItem{
		ContentId:   url,
		StreamType:  "BUFFERED",
		ContentType: contentType,
	}
	err = c.mediaHandler.LoadMedia(item, 0, true, map[string]interface{}{})
	if err != nil {
		log.Error(err)
		return
	}
}

func (c *Chromecast) waitForAppLaunch(app string) error {
	select {
	case launchedApp := <-c.appLaunch:
		if app == launchedApp {
			return nil
		}
		return fmt.Errorf("Wrong app launched. Expected %s got %s", app, launchedApp)
	case <-time.After(time.Second * 20):
		return fmt.Errorf("timeout waiting for app launch after 20 seconds")

	}

}
func (c *Chromecast) appLaunched(app string) {
	select {
	case c.appLaunch <- app:
		log.Info("Notified c.appLaunch")
	default:
		log.Info("No one is waiting for appLaunch event")
	}
}

func (c *Chromecast) Event(event events.Event) {
	switch data := event.(type) {
	case events.Connected:
		log.Info(c.Name(), "- Connected")

		c.Addr = c.Ip()
		c.Port = c.Device.Port()
		c.Id = c.Uuid()
		c.Name_ = c.Name()

		state.Add(c)
	case events.Disconnected:
		log.Warn(c.Name(), "- Disconnected")

		state.Remove(c)
	case events.AppStarted:
		log.Info(c.Name(), "- App started:", data.DisplayName, "(", data.AppID, ")")
		//spew.Dump("Data:", data)

		c.PrimaryApp = data.DisplayName
		c.PrimaryEndpoint = data.TransportId

		//If the app supports media controls lets subscribe to it
		if data.HasNamespace("urn:x-cast:com.google.cast.media") {
			log.Info(c.Name(), "- Subscribe cast.tp.connection:", data.DisplayName, "(", data.AppID, ")")
			c.Subscribe("urn:x-cast:com.google.cast.tp.connection", data.TransportId, c.mediaConnectionHandler)
			log.Info(c.Name(), "- Subscribe cast.media:", data.DisplayName, "(", data.AppID, ")")
			c.Subscribe("urn:x-cast:com.google.cast.media", data.TransportId, c.mediaHandler)
		}
		c.appLaunched(data.AppID)
		log.Info(c.Name(), "- Notifying appLanunched:", data.DisplayName, "(", data.AppID, ")")

	case events.AppStopped:
		log.Info(c.Name(), "- App stopped:", data.DisplayName, "(", data.AppID, ")")
		//spew.Dump("Data:", data)

		//unsubscribe from old channels
		for _, v := range data.Namespaces {
			c.UnsubscribeByUrnAndDestinationId(v.Name, data.TransportId)
		}
		c.PrimaryApp = ""
		c.PrimaryEndpoint = ""
		c.Playing = false

	case events.ReceiverStatus:
		c.IsStandBy = data.Status.IsStandBy
		c.IsActiveInput = data.Status.IsActiveInput
		c.Volume = data.Status.Volume.Level
		c.Muted = data.Status.Volume.Muted
	case events.Media:
		playing := c.Playing
		if data.PlayerState == "PLAYING" {
			c.Playing = true
		} else {
			c.Playing = false
		}

		//Only publish if playing state changed
		if playing != c.Playing {
			c.publish()
		}
		return

	//gocast.MediaEvent:
	default:
		log.Warn("unexpected event %T: %#v\n", data, data)
	}

	c.publish()
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Simple logging package. It defines a type, Logger, with methods
// for formatting output. It also has a predefined 'standard' Logger
// accessible through helper functions Print[f|ln], Exit[f|ln], and
// Panic[f|ln], which are easier to use than creating a Logger manually.
// That logger writes to standard error and prints the date and time
// of each logged message.
// The Exit functions call os.Exit(1) after writing the log message.
// The Panic functions call panic after writing the log message.
package log

import (
	"bytes"
	"fmt"
	"io"
	"runtime"
	"os"
	"time"
)

// These flags define the output Loggers produce.
const (
	// Bits or'ed together to control what's printed. There is no control over the
	// order they appear (the order listed here) or the format they present (as
	// described in the comments).  A colon appears after these items:
	//	2009/0123 01:23:23.123123 /a/b/c/d.go:23: message
	Ldate         = 1 << iota // the date: 2009/0123
	Ltime                     // the time: 01:23:23
	Lmicroseconds             // microsecond resolution: 01:23:23.123123.  assumes Ltime.
	Llongfile                 // full file name and line number: /a/b/c/d.go:23
	Lshortfile                // final file name element and line number: d.go:23. overrides Llongfile
)

// Logger represents an active logging object.
type Logger struct {
	out    io.Writer // destination for output
	prefix string    // prefix to write at beginning of each line
	flag   int       // properties
}

// New creates a new Logger.   The out variable sets the
// destination to which log data will be written.
// The prefix appears at the beginning of each generated log line.
// The flag argument defines the logging properties.
func New(out io.Writer, prefix string, flag int) *Logger {
	return &Logger{out, prefix, flag}
}

var std = New(os.Stderr, "", Ldate|Ltime)

// Cheap integer to fixed-width decimal ASCII.  Give a negative width to avoid zero-padding.
// Knows the buffer has capacity.
func itoa(buf *bytes.Buffer, i int, wid int) {
	var u uint = uint(i)
	if u == 0 && wid <= 1 {
		buf.WriteByte('0')
		return
	}

	// Assemble decimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; u > 0 || wid > 0; u /= 10 {
		bp--
		wid--
		b[bp] = byte(u%10) + '0'
	}

	// avoid slicing b to avoid an allocation.
	for bp < len(b) {
		buf.WriteByte(b[bp])
		bp++
	}
}

func (l *Logger) formatHeader(buf *bytes.Buffer, ns int64, calldepth int) {
	buf.WriteString(l.prefix)
	if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 {
		t := time.SecondsToLocalTime(ns / 1e9)
		if l.flag&Ldate != 0 {
			itoa(buf, int(t.Year), 4)
			buf.WriteByte('/')
			itoa(buf, int(t.Month), 2)
			buf.WriteByte('/')
			itoa(buf, int(t.Day), 2)
			buf.WriteByte(' ')
		}
		if l.flag&(Ltime|Lmicroseconds) != 0 {
			itoa(buf, int(t.Hour), 2)
			buf.WriteByte(':')
			itoa(buf, int(t.Minute), 2)
			buf.WriteByte(':')
			itoa(buf, int(t.Second), 2)
			if l.flag&Lmicroseconds != 0 {
				buf.WriteByte('.')
				itoa(buf, int(ns%1e9)/1e3, 6)
			}
			buf.WriteByte(' ')
		}
	}
	if l.flag&(Lshortfile|Llongfile) != 0 {
		_, file, line, ok := runtime.Caller(calldepth)
		if ok {
			if l.flag&Lshortfile != 0 {
				short := file
				for i := len(file) - 1; i > 0; i-- {
					if file[i] == '/' {
						short = file[i+1:]
						break
					}
				}
				file = short
			}
		} else {
			file = "???"
			line = 0
		}
		buf.WriteString(file)
		buf.WriteByte(':')
		itoa(buf, line, -1)
		buf.WriteString(": ")
	}
}

// Output writes the output for a logging event.  The string s contains the text to print after
// the time stamp;  calldepth is used to recover the PC.  It is provided for generality, although
// at the moment on all pre-defined paths it will be 2.
func (l *Logger) Output(calldepth int, s string) os.Error {
	now := time.Nanoseconds() // get this early.
	buf := new(bytes.Buffer)
	l.formatHeader(buf, now, calldepth+1)
	buf.WriteString(s)
	if len(s) > 0 && s[len(s)-1] != '\n' {
		buf.WriteByte('\n')
	}
	_, err := l.out.Write(buf.Bytes())
	return err
}

// Printf prints to the logger in the manner of fmt.Printf.
func (l *Logger) Printf(format string, v ...interface{}) {
	l.Output(2, fmt.Sprintf(format, v...))
}

// Print prints to the logger in the manner of fmt.Print.
func (l *Logger) Print(v ...interface{}) { l.Output(2, fmt.Sprint(v...)) }

// Println prints to the logger in the manner of fmt.Println.
func (l *Logger) Println(v ...interface{}) { l.Output(2, fmt.Sprintln(v...)) }

// SetOutput sets the output destination for the standard logger.
func SetOutput(w io.Writer) {
	std.out = w
}

// SetFlags sets the output flags for the standard logger.
func SetFlags(flag int) {
	std.flag = flag
}

// SetPrefix sets the output prefix for the standard logger.
func SetPrefix(prefix string) {
	std.prefix = prefix
}

// These functions write to the standard logger.

// Print prints to the standard logger in the manner of fmt.Print.
func Print(v ...interface{}) {
	std.Output(2, fmt.Sprint(v...))
}

// Printf prints to the standard logger in the manner of fmt.Printf.
func Printf(format string, v ...interface{}) {
	std.Output(2, fmt.Sprintf(format, v...))
}

// Println prints to the standard logger in the manner of fmt.Println.
func Println(v ...interface{}) {
	std.Output(2, fmt.Sprintln(v...))
}

// Exit is equivalent to Print() followed by a call to os.Exit(1).
func Exit(v ...interface{}) {
	std.Output(2, fmt.Sprint(v...))
	os.Exit(1)
}

// Exitf is equivalent to Printf() followed by a call to os.Exit(1).
func Exitf(format string, v ...interface{}) {
	std.Output(2, fmt.Sprintf(format, v...))
	os.Exit(1)
}

// Exitln is equivalent to Println() followed by a call to os.Exit(1).
func Exitln(v ...interface{}) {
	std.Output(2, fmt.Sprintln(v...))
	os.Exit(1)
}

// Panic is equivalent to Print() followed by a call to panic().
func Panic(v ...interface{}) {
	s := fmt.Sprint(v...)
	std.Output(2, s)
	panic(s)
}

// Panicf is equivalent to Printf() followed by a call to panic().
func Panicf(format string, v ...interface{}) {
	s := fmt.Sprintf(format, v...)
	std.Output(2, s)
	panic(s)
}

// Panicln is equivalent to Println() followed by a call to panic().
func Panicln(v ...interface{}) {
	s := fmt.Sprintln(v...)
	std.Output(2, s)
	panic(s)
}
<commit_msg>log: roll back deprecation of old API to apply fix to log.Output in public release.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Simple logging package. It defines a type, Logger, with methods
// for formatting output. It also has a predefined 'standard' Logger
// accessible through helper functions Print[f|ln], Exit[f|ln], and
// Panic[f|ln], which are easier to use than creating a Logger manually.
// That logger writes to standard error and prints the date and time
// of each logged message.
// The Exit functions call os.Exit(1) after writing the log message.
// The Panic functions call panic after writing the log message.
package log

import (
	"bytes"
	"fmt"
	"io"
	"runtime"
	"os"
	"time"
)

// These flags define the output Loggers produce.
const (
	// Bits or'ed together to control what's printed. There is no control over the
	// order they appear (the order listed here) or the format they present (as
	// described in the comments).  A colon appears after these items:
	//	2009/0123 01:23:23.123123 /a/b/c/d.go:23: message
	Ldate         = 1 << iota // the date: 2009/0123
	Ltime                     // the time: 01:23:23
	Lmicroseconds             // microsecond resolution: 01:23:23.123123.  assumes Ltime.
	Llongfile                 // full file name and line number: /a/b/c/d.go:23
	Lshortfile                // final file name element and line number: d.go:23. overrides Llongfile
)

// Logger represents an active logging object.
type Logger struct {
	out    io.Writer // destination for output
	prefix string    // prefix to write at beginning of each line
	flag   int       // properties
}

// New creates a new Logger.   The out variable sets the
// destination to which log data will be written.
// The prefix appears at the beginning of each generated log line.
// The flag argument defines the logging properties.
func New(out io.Writer, prefix string, flag int) *Logger {
	return &Logger{out, prefix, flag}
}

var (
	std    = New(os.Stderr, "", Ldate|Ltime)
	stdout = New(os.Stdout, "", Ldate|Ltime) // Deprecated.
)

// Cheap integer to fixed-width decimal ASCII.  Give a negative width to avoid zero-padding.
// Knows the buffer has capacity.
func itoa(buf *bytes.Buffer, i int, wid int) {
	var u uint = uint(i)
	if u == 0 && wid <= 1 {
		buf.WriteByte('0')
		return
	}

	// Assemble decimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; u > 0 || wid > 0; u /= 10 {
		bp--
		wid--
		b[bp] = byte(u%10) + '0'
	}

	// avoid slicing b to avoid an allocation.
	for bp < len(b) {
		buf.WriteByte(b[bp])
		bp++
	}
}

func (l *Logger) formatHeader(buf *bytes.Buffer, ns int64, calldepth int) {
	buf.WriteString(l.prefix)
	if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 {
		t := time.SecondsToLocalTime(ns / 1e9)
		if l.flag&Ldate != 0 {
			itoa(buf, int(t.Year), 4)
			buf.WriteByte('/')
			itoa(buf, int(t.Month), 2)
			buf.WriteByte('/')
			itoa(buf, int(t.Day), 2)
			buf.WriteByte(' ')
		}
		if l.flag&(Ltime|Lmicroseconds) != 0 {
			itoa(buf, int(t.Hour), 2)
			buf.WriteByte(':')
			itoa(buf, int(t.Minute), 2)
			buf.WriteByte(':')
			itoa(buf, int(t.Second), 2)
			if l.flag&Lmicroseconds != 0 {
				buf.WriteByte('.')
				itoa(buf, int(ns%1e9)/1e3, 6)
			}
			buf.WriteByte(' ')
		}
	}
	if l.flag&(Lshortfile|Llongfile) != 0 {
		_, file, line, ok := runtime.Caller(calldepth)
		if ok {
			if l.flag&Lshortfile != 0 {
				short := file
				for i := len(file) - 1; i > 0; i-- {
					if file[i] == '/' {
						short = file[i+1:]
						break
					}
				}
				file = short
			}
		} else {
			file = "???"
			line = 0
		}
		buf.WriteString(file)
		buf.WriteByte(':')
		itoa(buf, line, -1)
		buf.WriteString(": ")
	}
}

// Output writes the output for a logging event.  The string s contains the text to print after
// the time stamp;  calldepth is used to recover the PC.  It is provided for generality, although
// at the moment on all pre-defined paths it will be 2.
func (l *Logger) Output(calldepth int, s string) os.Error {
	now := time.Nanoseconds() // get this early.
	buf := new(bytes.Buffer)
	l.formatHeader(buf, now, calldepth+1)
	buf.WriteString(s)
	if len(s) > 0 && s[len(s)-1] != '\n' {
		buf.WriteByte('\n')
	}
	_, err := l.out.Write(buf.Bytes())
	return err
}

// Printf prints to the logger in the manner of fmt.Printf.
func (l *Logger) Printf(format string, v ...interface{}) {
	l.Output(2, fmt.Sprintf(format, v...))
}

// Print prints to the logger in the manner of fmt.Print.
func (l *Logger) Print(v ...interface{}) { l.Output(2, fmt.Sprint(v...)) }

// Println prints to the logger in the manner of fmt.Println.
func (l *Logger) Println(v ...interface{}) { l.Output(2, fmt.Sprintln(v...)) }

// SetOutput sets the output destination for the standard logger.
func SetOutput(w io.Writer) {
	std.out = w
}

// SetFlags sets the output flags for the standard logger.
func SetFlags(flag int) {
	std.flag = flag
}

// SetPrefix sets the output prefix for the standard logger.
func SetPrefix(prefix string) {
	std.prefix = prefix
}

// These functions write to the standard logger.

// Print prints to the standard logger in the manner of fmt.Print.
func Print(v ...interface{}) {
	std.Output(2, fmt.Sprint(v...))
}

// Printf prints to the standard logger in the manner of fmt.Printf.
func Printf(format string, v ...interface{}) {
	std.Output(2, fmt.Sprintf(format, v...))
}

// Println prints to the standard logger in the manner of fmt.Println.
func Println(v ...interface{}) {
	std.Output(2, fmt.Sprintln(v...))
}

// Exit is equivalent to Print() followed by a call to os.Exit(1).
func Exit(v ...interface{}) {
	std.Output(2, fmt.Sprint(v...))
	os.Exit(1)
}

// Exitf is equivalent to Printf() followed by a call to os.Exit(1).
func Exitf(format string, v ...interface{}) {
	std.Output(2, fmt.Sprintf(format, v...))
	os.Exit(1)
}

// Exitln is equivalent to Println() followed by a call to os.Exit(1).
func Exitln(v ...interface{}) {
	std.Output(2, fmt.Sprintln(v...))
	os.Exit(1)
}

// Panic is equivalent to Print() followed by a call to panic().
func Panic(v ...interface{}) {
	s := fmt.Sprint(v...)
	std.Output(2, s)
	panic(s)
}

// Panicf is equivalent to Printf() followed by a call to panic().
func Panicf(format string, v ...interface{}) {
	s := fmt.Sprintf(format, v...)
	std.Output(2, s)
	panic(s)
}

// Panicln is equivalent to Println() followed by a call to panic().
func Panicln(v ...interface{}) {
	s := fmt.Sprintln(v...)
	std.Output(2, s)
	panic(s)
}

// Everything from here on is deprecated and will be removed after the next release.

// Logf is analogous to Printf() for a Logger.
// Deprecated.
func (l *Logger) Logf(format string, v ...interface{}) {
	l.Output(2, fmt.Sprintf(format, v...))
}

// Log is analogous to Print() for a Logger.
// Deprecated.
func (l *Logger) Log(v ...interface{}) { l.Output(2, fmt.Sprintln(v...)) }

// Stdout is a helper function for easy logging to stdout. It is analogous to Print().
// Deprecated.
func Stdout(v ...interface{}) { stdout.Output(2, fmt.Sprint(v...)) }

// Stderr is a helper function for easy logging to stderr. It is analogous to Fprint(os.Stderr).
// Deprecated.
func Stderr(v ...interface{}) { std.Output(2, fmt.Sprintln(v...)) }

// Stdoutf is a helper functions for easy formatted logging to stdout. It is analogous to Printf().
// Deprecated.
func Stdoutf(format string, v ...interface{}) { stdout.Output(2, fmt.Sprintf(format, v...)) }

// Stderrf is a helper function for easy formatted logging to stderr. It is analogous to Fprintf(os.Stderr).
// Deprecated.
func Stderrf(format string, v ...interface{}) { std.Output(2, fmt.Sprintf(format, v...)) }

// Crash is equivalent to Stderr() followed by a call to panic().
// Deprecated.
func Crash(v ...interface{}) { Panicln(v...) }

// Crashf is equivalent to Stderrf() followed by a call to panic().
// Deprecated.
func Crashf(format string, v ...interface{}) { Panicf(format, v...) }
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package state

import (
	"encoding/gob"
	"io"
)

// State that should be saved between runs of the program.
type State struct {
	// The set of scores that are known to already exist in the blob store,
	// represented in hex. It is unnecessary to again store any blob with one of
	// these scores.
	ExistingScores StringSet

	// A version number for the above set of scores. Any time state is saved,
	// this version should first be updated to a random number and saved to the
	// backup registry, making sure that the old version was still the current
	// one. This protects us from drifting out of date if another process is
	// concurrently adding scores to the blob store.
	ExistingScoresVersion uint64

	// A map from file system info to the scores that were seen for a given file
	// last time.
	ScoresForFiles ScoreMap
}

func LoadState(r io.Reader) (state State, err error) {
	decoder := gob.NewDecoder(r)
	err = decoder.Decode(&state)
	return
}

func SaveState(w io.Writer, state State) (err error) {
	encoder := gob.NewEncoder(w)
	err = encoder.Encode(state)
	return
}
<commit_msg>Added a TODO.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package state

import (
	"encoding/gob"
	"io"
)

// State that should be saved between runs of the program.
type State struct {
	// The set of scores that are known to already exist in the blob store,
	// represented in hex. It is unnecessary to again store any blob with one of
	// these scores.
	ExistingScores StringSet

	// A version number for the above set of scores. Any time state is saved,
	// this version should first be updated to a random number and saved to the
	// backup registry, making sure that the old version was still the current
	// one. This protects us from drifting out of date if another process is
	// concurrently adding scores to the blob store.
	//
	// TODO(jacobsa): Do we really need this silliness? See issue #15.
	ExistingScoresVersion uint64

	// A map from file system info to the scores that were seen for a given file
	// last time.
	ScoresForFiles ScoreMap
}

func LoadState(r io.Reader) (state State, err error) {
	decoder := gob.NewDecoder(r)
	err = decoder.Decode(&state)
	return
}

func SaveState(w io.Writer, state State) (err error) {
	encoder := gob.NewEncoder(w)
	err = encoder.Encode(state)
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package acme

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	cmacme "github.com/jetstack/cert-manager/pkg/apis/acme/v1alpha2"
	cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2"
	cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
	"github.com/jetstack/cert-manager/test/e2e/framework"
	"github.com/jetstack/cert-manager/test/e2e/framework/addon"
	"github.com/jetstack/cert-manager/test/e2e/framework/addon/pebble"
	"github.com/jetstack/cert-manager/test/e2e/framework/addon/tiller"
	"github.com/jetstack/cert-manager/test/e2e/suite/conformance/certificates"
	"github.com/jetstack/cert-manager/test/e2e/suite/issuers/acme/dnsproviders"
)

var _ = framework.ConformanceDescribe("Certificates", func() {
	// unsupportedHTTP01Features is a list of features that are not supported by the ACME
	// issuer type using HTTP01
	var unsupportedHTTP01Features = certificates.NewFeatureSet(
		certificates.IPAddressFeature,
		certificates.DurationFeature,
		certificates.WildcardsFeature,
		certificates.URISANsFeature,
		certificates.CommonNameFeature,
		certificates.KeyUsagesFeature,
	)

	// unsupportedDNS01Features is a list of features that are not supported by the ACME
	// issuer type using DNS01
	var unsupportedDNS01Features = certificates.NewFeatureSet(
		certificates.IPAddressFeature,
		certificates.DurationFeature,
		certificates.URISANsFeature,
		certificates.CommonNameFeature,
		certificates.KeyUsagesFeature,
	)

	provisionerHTTP01 := new(acmeIssuerProvisioner)
	(&certificates.Suite{
		Name:                "ACME HTTP01 Issuer",
		CreateIssuerFunc:    provisionerHTTP01.createHTTP01Issuer,
		DeleteIssuerFunc:    provisionerHTTP01.delete,
		UnsupportedFeatures: unsupportedHTTP01Features,
	}).Define()

	provisionerDNS01 := new(acmeIssuerProvisioner)
	(&certificates.Suite{
		Name:                "ACME DNS01 Issuer",
		CreateIssuerFunc:    provisionerDNS01.createDNS01Issuer,
		DeleteIssuerFunc:    provisionerDNS01.delete,
		UnsupportedFeatures: unsupportedDNS01Features,
	}).Define()

	(&certificates.Suite{
		Name:                "ACME HTTP01 ClusterIssuer",
		CreateIssuerFunc:    provisionerHTTP01.createHTTP01ClusterIssuer,
		DeleteIssuerFunc:    provisionerHTTP01.delete,
		UnsupportedFeatures: unsupportedHTTP01Features,
	}).Define()

	(&certificates.Suite{
		Name:                "ACME DNS01 ClusterIssuer",
		CreateIssuerFunc:    provisionerDNS01.createDNS01ClusterIssuer,
		DeleteIssuerFunc:    provisionerDNS01.delete,
		UnsupportedFeatures: unsupportedDNS01Features,
	}).Define()
})

type acmeIssuerProvisioner struct {
	tiller     *tiller.Tiller
	pebble     *pebble.Pebble
	cloudflare *dnsproviders.Cloudflare
}

func (a *acmeIssuerProvisioner) delete(f *framework.Framework, ref cmmeta.ObjectReference) {
	if a.pebble != nil {
		Expect(a.pebble.Deprovision()).NotTo(HaveOccurred(), "failed to deprovision pebble")
	}
	if a.cloudflare != nil {
		Expect(a.cloudflare.Deprovision()).NotTo(HaveOccurred(), "failed to deprovision cloudflare")
	}
	Expect(a.tiller.Deprovision()).NotTo(HaveOccurred(), "failed to deprovision tiller")

	if ref.Kind == "ClusterIssuer" {
		err := f.CertManagerClientSet.CertmanagerV1alpha2().ClusterIssuers().Delete(ref.Name, nil)
		Expect(err).NotTo(HaveOccurred())
	}
}

// createXXX will deploy the required components to run an ACME issuer based test.
// This includes:
// - tiller
// - pebble
// - a properly configured Issuer resource

func (a *acmeIssuerProvisioner) createHTTP01Issuer(f *framework.Framework) cmmeta.ObjectReference {
	a.deployTiller(f, "http01")

	By("Creating an ACME HTTP01 Issuer")
	issuer := &cmapi.Issuer{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "acme-issuer-http01-",
		},
		Spec: a.createHTTP01IssuerSpec(),
	}

	issuer, err := f.CertManagerClientSet.CertmanagerV1alpha2().Issuers(f.Namespace.Name).Create(issuer)
	Expect(err).NotTo(HaveOccurred(), "failed to create acme HTTP01 issuer")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.IssuerKind,
		Name:  issuer.Name,
	}
}

func (a *acmeIssuerProvisioner) createHTTP01ClusterIssuer(f *framework.Framework) cmmeta.ObjectReference {
	a.deployTiller(f, "http01")

	By("Creating an ACME HTTP01 ClusterIssuer")
	issuer := &cmapi.ClusterIssuer{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "acme-cluster-issuer-http01-",
		},
		Spec: a.createHTTP01IssuerSpec(),
	}

	issuer, err := f.CertManagerClientSet.CertmanagerV1alpha2().ClusterIssuers().Create(issuer)
	Expect(err).NotTo(HaveOccurred(), "failed to create acme HTTP01 cluster issuer")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.ClusterIssuerKind,
		Name:  issuer.Name,
	}
}

func (a *acmeIssuerProvisioner) createHTTP01IssuerSpec() cmapi.IssuerSpec {
	return cmapi.IssuerSpec{
		IssuerConfig: cmapi.IssuerConfig{
			ACME: &cmacme.ACMEIssuer{
				Server:        addon.Pebble.Details().Host,
				SkipTLSVerify: true,
				PrivateKey: cmmeta.SecretKeySelector{
					LocalObjectReference: cmmeta.LocalObjectReference{
						Name: "acme-private-key-http01",
					},
				},
				Solvers: []cmacme.ACMEChallengeSolver{
					{
						HTTP01: &cmacme.ACMEChallengeSolverHTTP01{
							// Not setting the Class or Name field will cause cert-manager to create
							// new ingress resources that do not specify a class to solve challenges,
							// which means all Ingress controllers should act on the ingresses.
							Ingress: &cmacme.ACMEChallengeSolverHTTP01Ingress{},
						},
					},
				},
			},
		},
	}
}

func (a *acmeIssuerProvisioner) createDNS01Issuer(f *framework.Framework) cmmeta.ObjectReference {
	a.deployTiller(f, "dns01")

	a.cloudflare = &dnsproviders.Cloudflare{
		Namespace: f.Namespace.Name,
	}
	Expect(a.cloudflare.Setup(f.Config)).NotTo(HaveOccurred(), "failed to setup cloudflare")
	Expect(a.cloudflare.Provision()).NotTo(HaveOccurred(), "failed to provision cloudflare")

	By("Creating an ACME DNS01 Issuer")
	issuer := &cmapi.Issuer{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "acme-issuer-dns01-",
		},
		Spec: a.createDNS01IssuerSpec(),
	}
	issuer, err := f.CertManagerClientSet.CertmanagerV1alpha2().Issuers(f.Namespace.Name).Create(issuer)
	Expect(err).NotTo(HaveOccurred(), "failed to create acme DNS01 Issuer")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.IssuerKind,
		Name:  issuer.Name,
	}
}

func (a *acmeIssuerProvisioner) createDNS01ClusterIssuer(f *framework.Framework) cmmeta.ObjectReference {
	a.deployTiller(f, "dns01")

	a.cloudflare = &dnsproviders.Cloudflare{
		Namespace: f.Namespace.Name,
	}
	Expect(a.cloudflare.Setup(f.Config)).NotTo(HaveOccurred(), "failed to setup cloudflare")
	Expect(a.cloudflare.Provision()).NotTo(HaveOccurred(), "failed to provision cloudflare")

	By("Creating an ACME DNS01 ClusterIssuer")
	issuer := &cmapi.ClusterIssuer{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "acme-cluster-issuer-dns01-",
		},
		Spec: a.createDNS01IssuerSpec(),
	}
	issuer, err := f.CertManagerClientSet.CertmanagerV1alpha2().ClusterIssuers().Create(issuer)
	Expect(err).NotTo(HaveOccurred(), "failed to create acme DNS01 ClusterIssuer")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.ClusterIssuerKind,
		Name:  issuer.Name,
	}
}

func (a *acmeIssuerProvisioner) createDNS01IssuerSpec() cmapi.IssuerSpec {
	return cmapi.IssuerSpec{
		IssuerConfig: cmapi.IssuerConfig{
			ACME: &cmacme.ACMEIssuer{
				// Hardcode this to the acme staging endpoint now due to issues with pebble dns resolution
				Server:        "https://acme-staging-v02.api.letsencrypt.org/directory",
				SkipTLSVerify: true,
				PrivateKey: cmmeta.SecretKeySelector{
					LocalObjectReference: cmmeta.LocalObjectReference{
						Name: "acme-private-key",
					},
				},
				Solvers: []cmacme.ACMEChallengeSolver{
					{
						DNS01: &a.cloudflare.Details().ProviderConfig,
					},
				},
			},
		},
	}
}

func (a *acmeIssuerProvisioner) deployTiller(f *framework.Framework, solverType string) {
	a.tiller = &tiller.Tiller{
		Name:               "tiller-deploy-" + solverType,
		ClusterPermissions: false,
		Namespace:          f.Namespace.Name,
	}
	Expect(a.tiller.Setup(f.Config)).NotTo(HaveOccurred(), "failed to setup tiller")
	Expect(a.tiller.Provision()).NotTo(HaveOccurred(), "failed to provision tiller")
}
<commit_msg>Use cloud flare service account in cluster namespace when using cluster issuer<commit_after>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package acme

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	cmacme "github.com/jetstack/cert-manager/pkg/apis/acme/v1alpha2"
	cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2"
	cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
	"github.com/jetstack/cert-manager/test/e2e/framework"
	"github.com/jetstack/cert-manager/test/e2e/framework/addon"
	"github.com/jetstack/cert-manager/test/e2e/framework/addon/pebble"
	"github.com/jetstack/cert-manager/test/e2e/framework/addon/tiller"
	"github.com/jetstack/cert-manager/test/e2e/suite/conformance/certificates"
	"github.com/jetstack/cert-manager/test/e2e/suite/issuers/acme/dnsproviders"
)

var _ = framework.ConformanceDescribe("Certificates", func() {
	// unsupportedHTTP01Features is a list of features that are not supported by the ACME
	// issuer type using HTTP01
	var unsupportedHTTP01Features = certificates.NewFeatureSet(
		certificates.IPAddressFeature,
		certificates.DurationFeature,
		certificates.WildcardsFeature,
		certificates.URISANsFeature,
		certificates.CommonNameFeature,
		certificates.KeyUsagesFeature,
	)

	// unsupportedDNS01Features is a list of features that are not supported by the ACME
	// issuer type using DNS01
	var unsupportedDNS01Features = certificates.NewFeatureSet(
		certificates.IPAddressFeature,
		certificates.DurationFeature,
		certificates.URISANsFeature,
		certificates.CommonNameFeature,
		certificates.KeyUsagesFeature,
	)

	provisionerHTTP01 := new(acmeIssuerProvisioner)
	(&certificates.Suite{
		Name:                "ACME HTTP01 Issuer",
		CreateIssuerFunc:    provisionerHTTP01.createHTTP01Issuer,
		DeleteIssuerFunc:    provisionerHTTP01.delete,
		UnsupportedFeatures: unsupportedHTTP01Features,
	}).Define()

	provisionerDNS01 := new(acmeIssuerProvisioner)
	(&certificates.Suite{
		Name:                "ACME DNS01 Issuer",
		CreateIssuerFunc:    provisionerDNS01.createDNS01Issuer,
		DeleteIssuerFunc:    provisionerDNS01.delete,
		UnsupportedFeatures: unsupportedDNS01Features,
	}).Define()

	(&certificates.Suite{
		Name:                "ACME HTTP01 ClusterIssuer",
		CreateIssuerFunc:    provisionerHTTP01.createHTTP01ClusterIssuer,
		DeleteIssuerFunc:    provisionerHTTP01.delete,
		UnsupportedFeatures: unsupportedHTTP01Features,
	}).Define()

	(&certificates.Suite{
		Name:                "ACME DNS01 ClusterIssuer",
		CreateIssuerFunc:    provisionerDNS01.createDNS01ClusterIssuer,
		DeleteIssuerFunc:    provisionerDNS01.delete,
		UnsupportedFeatures: unsupportedDNS01Features,
	}).Define()
})

type acmeIssuerProvisioner struct {
	tiller     *tiller.Tiller
	pebble     *pebble.Pebble
	cloudflare *dnsproviders.Cloudflare
}

func (a *acmeIssuerProvisioner) delete(f *framework.Framework, ref cmmeta.ObjectReference) {
	if a.pebble != nil {
		Expect(a.pebble.Deprovision()).NotTo(HaveOccurred(), "failed to deprovision pebble")
	}
	if a.cloudflare != nil {
		Expect(a.cloudflare.Deprovision()).NotTo(HaveOccurred(), "failed to deprovision cloudflare")
	}
	Expect(a.tiller.Deprovision()).NotTo(HaveOccurred(), "failed to deprovision tiller")

	if ref.Kind == "ClusterIssuer" {
		err := f.CertManagerClientSet.CertmanagerV1alpha2().ClusterIssuers().Delete(ref.Name, nil)
		Expect(err).NotTo(HaveOccurred())
	}
}

// createXXX will deploy the required components to run an ACME issuer based test.
// This includes:
// - tiller
// - pebble
// - a properly configured Issuer resource

func (a *acmeIssuerProvisioner) createHTTP01Issuer(f *framework.Framework) cmmeta.ObjectReference {
	a.deployTiller(f, "http01")

	By("Creating an ACME HTTP01 Issuer")
	issuer := &cmapi.Issuer{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "acme-issuer-http01-",
		},
		Spec: a.createHTTP01IssuerSpec(),
	}

	issuer, err := f.CertManagerClientSet.CertmanagerV1alpha2().Issuers(f.Namespace.Name).Create(issuer)
	Expect(err).NotTo(HaveOccurred(), "failed to create acme HTTP01 issuer")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.IssuerKind,
		Name:  issuer.Name,
	}
}

func (a *acmeIssuerProvisioner) createHTTP01ClusterIssuer(f *framework.Framework) cmmeta.ObjectReference {
	a.deployTiller(f, "http01")

	By("Creating an ACME HTTP01 ClusterIssuer")
	issuer := &cmapi.ClusterIssuer{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "acme-cluster-issuer-http01-",
		},
		Spec: a.createHTTP01IssuerSpec(),
	}

	issuer, err := f.CertManagerClientSet.CertmanagerV1alpha2().ClusterIssuers().Create(issuer)
	Expect(err).NotTo(HaveOccurred(), "failed to create acme HTTP01 cluster issuer")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.ClusterIssuerKind,
		Name:  issuer.Name,
	}
}

func (a *acmeIssuerProvisioner) createHTTP01IssuerSpec() cmapi.IssuerSpec {
	return cmapi.IssuerSpec{
		IssuerConfig: cmapi.IssuerConfig{
			ACME: &cmacme.ACMEIssuer{
				Server:        addon.Pebble.Details().Host,
				SkipTLSVerify: true,
				PrivateKey: cmmeta.SecretKeySelector{
					LocalObjectReference: cmmeta.LocalObjectReference{
						Name: "acme-private-key-http01",
					},
				},
				Solvers: []cmacme.ACMEChallengeSolver{
					{
						HTTP01: &cmacme.ACMEChallengeSolverHTTP01{
							// Not setting the Class or Name field will cause cert-manager to create
							// new ingress resources that do not specify a class to solve challenges,
							// which means all Ingress controllers should act on the ingresses.
							Ingress: &cmacme.ACMEChallengeSolverHTTP01Ingress{},
						},
					},
				},
			},
		},
	}
}

func (a *acmeIssuerProvisioner) createDNS01Issuer(f *framework.Framework) cmmeta.ObjectReference {
	a.deployTiller(f, "dns01")

	a.cloudflare = &dnsproviders.Cloudflare{
		Namespace: f.Namespace.Name,
	}
	Expect(a.cloudflare.Setup(f.Config)).NotTo(HaveOccurred(), "failed to setup cloudflare")
	Expect(a.cloudflare.Provision()).NotTo(HaveOccurred(), "failed to provision cloudflare")

	By("Creating an ACME DNS01 Issuer")
	issuer := &cmapi.Issuer{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "acme-issuer-dns01-",
		},
		Spec: a.createDNS01IssuerSpec(),
	}
	issuer, err := f.CertManagerClientSet.CertmanagerV1alpha2().Issuers(f.Namespace.Name).Create(issuer)
	Expect(err).NotTo(HaveOccurred(), "failed to create acme DNS01 Issuer")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.IssuerKind,
		Name:  issuer.Name,
	}
}

func (a *acmeIssuerProvisioner) createDNS01ClusterIssuer(f *framework.Framework) cmmeta.ObjectReference {
	a.deployTiller(f, "dns01")

	a.cloudflare = &dnsproviders.Cloudflare{
		Namespace: addon.CertManager.Namespace,
	}
	Expect(a.cloudflare.Setup(f.Config)).NotTo(HaveOccurred(), "failed to setup cloudflare")
	Expect(a.cloudflare.Provision()).NotTo(HaveOccurred(), "failed to provision cloudflare")

	By("Creating an ACME DNS01 ClusterIssuer")
	issuer := &cmapi.ClusterIssuer{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "acme-cluster-issuer-dns01-",
		},
		Spec: a.createDNS01IssuerSpec(),
	}
	issuer, err := f.CertManagerClientSet.CertmanagerV1alpha2().ClusterIssuers().Create(issuer)
	Expect(err).NotTo(HaveOccurred(), "failed to create acme DNS01 ClusterIssuer")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.ClusterIssuerKind,
		Name:  issuer.Name,
	}
}

func (a *acmeIssuerProvisioner) createDNS01IssuerSpec() cmapi.IssuerSpec {
	return cmapi.IssuerSpec{
		IssuerConfig: cmapi.IssuerConfig{
			ACME: &cmacme.ACMEIssuer{
				// Hardcode this to the acme staging endpoint now due to issues with pebble dns resolution
				Server:        "https://acme-staging-v02.api.letsencrypt.org/directory",
				SkipTLSVerify: true,
				PrivateKey: cmmeta.SecretKeySelector{
					LocalObjectReference: cmmeta.LocalObjectReference{
						Name: "acme-private-key",
					},
				},
				Solvers: []cmacme.ACMEChallengeSolver{
					{
						DNS01: &a.cloudflare.Details().ProviderConfig,
					},
				},
			},
		},
	}
}

func (a *acmeIssuerProvisioner) deployTiller(f *framework.Framework, solverType string) {
	a.tiller = &tiller.Tiller{
		Name:               "tiller-deploy-" + solverType,
		ClusterPermissions: false,
		Namespace:          f.Namespace.Name,
	}
	Expect(a.tiller.Setup(f.Config)).NotTo(HaveOccurred(), "failed to setup tiller")
	Expect(a.tiller.Provision()).NotTo(HaveOccurred(), "failed to provision tiller")
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package avalanche

import (
	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/snow/choices"
	"github.com/ava-labs/gecko/snow/consensus/avalanche"
	"github.com/ava-labs/gecko/snow/engine/common"
	"github.com/ava-labs/gecko/snow/engine/common/queue"
	"github.com/ava-labs/gecko/utils/formatting"
	"github.com/prometheus/client_golang/prometheus"
)

// BootstrapConfig ...
type BootstrapConfig struct {
	common.Config

	// VtxBlocked tracks operations that are blocked on vertices
	// TxBlocked tracks operations that are blocked on transactions
	VtxBlocked, TxBlocked *queue.Jobs

	State State
	VM    DAGVM
}

type bootstrapper struct {
	BootstrapConfig
	metrics
	common.Bootstrapper

	// IDs of vertices that we're already in the process of getting
	// TODO: Find a better way to track; this keeps every single vertex's ID in memory when bootstrapping from nothing
	seen ids.Set

	numFetched uint64 // number of vertices that have been fetched from validators

	// vtxReqs prevents asking validators for the same vertex
	vtxReqs common.Requests

	// IDs of vertices that we have requested from other validators but haven't received
	pending    ids.Set
	finished   bool
	onFinished func()
}

// Initialize this engine.
func (b *bootstrapper) Initialize(config BootstrapConfig) {
	b.BootstrapConfig = config

	b.VtxBlocked.SetParser(&vtxParser{
		numAccepted: b.numBootstrappedVtx,
		numDropped:  b.numDroppedVtx,
		state:       b.State,
	})

	b.TxBlocked.SetParser(&txParser{
		numAccepted: b.numBootstrappedTx,
		numDropped:  b.numDroppedTx,
		vm:          b.VM,
	})

	config.Bootstrapable = b
	b.Bootstrapper.Initialize(config.Config)
}

// CurrentAcceptedFrontier ...
func (b *bootstrapper) CurrentAcceptedFrontier() ids.Set {
	acceptedFrontier := ids.Set{}
	acceptedFrontier.Add(b.State.Edge()...)
	return acceptedFrontier
}

// FilterAccepted ...
func (b *bootstrapper) FilterAccepted(containerIDs ids.Set) ids.Set {
	acceptedVtxIDs := ids.Set{}
	for _, vtxID := range containerIDs.List() {
		if vtx, err := b.State.GetVertex(vtxID); err == nil && vtx.Status() == choices.Accepted {
			acceptedVtxIDs.Add(vtxID)
		}
	}
	return acceptedVtxIDs
}

// ForceAccepted ...
func (b *bootstrapper) ForceAccepted(acceptedContainerIDs ids.Set) {
	for _, vtxID := range acceptedContainerIDs.List() {
		b.fetch(vtxID)
	}

	if numPending := b.pending.Len(); numPending == 0 {
		// TODO: This typically indicates bootstrapping has failed, so this
		// should be handled appropriately
		b.finish()
	}
}

// Put ...
func (b *bootstrapper) Put(vdr ids.ShortID, requestID uint32, vtxID ids.ID, vtxBytes []byte) {
	vtx, err := b.State.ParseVertex(vtxBytes)
	if err != nil {
		b.BootstrapConfig.Context.Log.Debug("ParseVertex failed due to %s for block:\n%s",
			err,
			formatting.DumpBytes{Bytes: vtxBytes})

		b.GetFailed(vdr, requestID)
		return
	}

	if !b.pending.Contains(vtx.ID()) {
		b.BootstrapConfig.Context.Log.Debug("Validator %s sent an unrequested vertex:\n%s",
			vdr,
			formatting.DumpBytes{Bytes: vtxBytes})

		b.GetFailed(vdr, requestID)
		return
	}

	b.addVertex(vtx)
}

// GetFailed ...
func (b *bootstrapper) GetFailed(vdr ids.ShortID, requestID uint32) {
	vtxID, ok := b.vtxReqs.Remove(vdr, requestID)
	if !ok {
		b.BootstrapConfig.Context.Log.Debug("GetFailed called without sending the corresponding Get message from %s",
			vdr)
		return
	}

	b.sendRequest(vtxID)
}

func (b *bootstrapper) fetch(vtxID ids.ID) {
	if b.pending.Contains(vtxID) {
		return
	}

	vtx, err := b.State.GetVertex(vtxID)
	if err != nil {
		b.sendRequest(vtxID)
		return
	}
	b.storeVertex(vtx)
}

func (b *bootstrapper) sendRequest(vtxID ids.ID) {
	validators := b.BootstrapConfig.Validators.Sample(1)
	if len(validators) == 0 {
		b.BootstrapConfig.Context.Log.Error("Dropping request for %s as there are no validators", vtxID)
		return
	}
	validatorID := validators[0].ID()
	b.RequestID++

	b.vtxReqs.RemoveAny(vtxID)
	b.vtxReqs.Add(validatorID, b.RequestID, vtxID)

	b.pending.Add(vtxID)
	b.BootstrapConfig.Sender.Get(validatorID, b.RequestID, vtxID)

	b.numPendingRequests.Set(float64(b.pending.Len()))
}

func (b *bootstrapper) addVertex(vtx avalanche.Vertex) {
	b.storeVertex(vtx)

	if numPending := b.pending.Len(); numPending == 0 {
		b.finish()
	}
}

func (b *bootstrapper) storeVertex(vtx avalanche.Vertex) {
	vts := []avalanche.Vertex{vtx}
	b.numFetched++
	if b.numFetched%2500 == 0 { // perioidcally inform user of progress
		b.BootstrapConfig.Context.Log.Info("bootstrapping has fetched %d vertices", b.numFetched)
	}

	for len(vts) > 0 {
		newLen := len(vts) - 1
		vtx := vts[newLen]
		vts = vts[:newLen]

		vtxID := vtx.ID()
		switch status := vtx.Status(); status {
		case choices.Unknown:
			b.sendRequest(vtxID)
		case choices.Processing:
			b.pending.Remove(vtxID)

			if err := b.VtxBlocked.Push(&vertexJob{
				numAccepted: b.numBootstrappedVtx,
				numDropped:  b.numDroppedVtx,
				vtx:         vtx,
			}); err == nil {
				b.numBlockedVtx.Inc()
			} else {
				b.BootstrapConfig.Context.Log.Verbo("couldn't push to vtxBlocked")
			}
			for _, tx := range vtx.Txs() {
				if err := b.TxBlocked.Push(&txJob{
					numAccepted: b.numBootstrappedVtx,
					numDropped:  b.numDroppedVtx,
					tx:          tx,
				}); err == nil {
					b.numBlockedTx.Inc()
				} else {
					b.BootstrapConfig.Context.Log.Verbo("couldn't push to txBlocked")
				}
			}
			parentsToAdd := []avalanche.Vertex{}
			parentsToAddIDs := ids.Set{} // ToDO remove...only here for debug
			for _, parent := range vtx.Parents() {
				if !b.seen.Contains(parent.ID()) {
					parentsToAdd = append(parentsToAdd, parent)
					parentsToAddIDs.Add(parent.ID())
				}
			}
			vts = append(vts, parentsToAdd...)
			b.seen.Add(parentsToAddIDs.List()...)
		case choices.Accepted:
			b.BootstrapConfig.Context.Log.Verbo("Bootstrapping confirmed %s", vtxID)
		case choices.Rejected:
			b.BootstrapConfig.Context.Log.Error("Bootstrapping wants to accept %s, however it was previously rejected", vtxID)
		}
	}

	numPending := b.pending.Len()
	b.numPendingRequests.Set(float64(numPending))
}

func (b *bootstrapper) finish() {
	if b.finished {
		return
	}
	b.BootstrapConfig.Context.Log.Info("bootstrapping finished fetching vertices. executing state transitions...")

	b.executeAll(b.TxBlocked, b.numBlockedTx)
	b.executeAll(b.VtxBlocked, b.numBlockedVtx)

	// Start consensus
	b.BootstrapConfig.Context.Log.Info("done bootstrapping")
	b.onFinished()
	b.seen = ids.Set{}
	b.finished = true
}

func (b *bootstrapper) executeAll(jobs *queue.Jobs, numBlocked prometheus.Gauge) {
	for job, err := jobs.Pop(); err == nil; job, err = jobs.Pop() {
		numBlocked.Dec()
		b.BootstrapConfig.Context.Log.Debug("Executing: %s", job.ID())
		if err := jobs.Execute(job); err != nil {
			b.BootstrapConfig.Context.Log.Warn("Error executing: %s", err)
		}
	}
}
<commit_msg>minor code cleanup<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package avalanche

import (
	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/snow/choices"
	"github.com/ava-labs/gecko/snow/consensus/avalanche"
	"github.com/ava-labs/gecko/snow/engine/common"
	"github.com/ava-labs/gecko/snow/engine/common/queue"
	"github.com/ava-labs/gecko/utils/formatting"
	"github.com/prometheus/client_golang/prometheus"
)

// BootstrapConfig ...
type BootstrapConfig struct {
	common.Config

	// VtxBlocked tracks operations that are blocked on vertices
	// TxBlocked tracks operations that are blocked on transactions
	VtxBlocked, TxBlocked *queue.Jobs

	State State
	VM    DAGVM
}

type bootstrapper struct {
	BootstrapConfig
	metrics
	common.Bootstrapper

	// IDs of vertices that we're already in the process of getting
	// TODO: Find a better way to track; this keeps every single vertex's ID in memory when bootstrapping from nothing
	seen ids.Set

	numFetched uint64 // number of vertices that have been fetched from validators

	// vtxReqs prevents asking validators for the same vertex
	vtxReqs common.Requests

	// IDs of vertices that we have requested from other validators but haven't received
	pending    ids.Set
	finished   bool
	onFinished func()
}

// Initialize this engine.
func (b *bootstrapper) Initialize(config BootstrapConfig) {
	b.BootstrapConfig = config

	b.VtxBlocked.SetParser(&vtxParser{
		numAccepted: b.numBootstrappedVtx,
		numDropped:  b.numDroppedVtx,
		state:       b.State,
	})

	b.TxBlocked.SetParser(&txParser{
		numAccepted: b.numBootstrappedTx,
		numDropped:  b.numDroppedTx,
		vm:          b.VM,
	})

	config.Bootstrapable = b
	b.Bootstrapper.Initialize(config.Config)
}

// CurrentAcceptedFrontier ...
func (b *bootstrapper) CurrentAcceptedFrontier() ids.Set {
	acceptedFrontier := ids.Set{}
	acceptedFrontier.Add(b.State.Edge()...)
	return acceptedFrontier
}

// FilterAccepted ...
func (b *bootstrapper) FilterAccepted(containerIDs ids.Set) ids.Set {
	acceptedVtxIDs := ids.Set{}
	for _, vtxID := range containerIDs.List() {
		if vtx, err := b.State.GetVertex(vtxID); err == nil && vtx.Status() == choices.Accepted {
			acceptedVtxIDs.Add(vtxID)
		}
	}
	return acceptedVtxIDs
}

// ForceAccepted ...
func (b *bootstrapper) ForceAccepted(acceptedContainerIDs ids.Set) {
	for _, vtxID := range acceptedContainerIDs.List() {
		b.fetch(vtxID)
	}

	if numPending := b.pending.Len(); numPending == 0 {
		// TODO: This typically indicates bootstrapping has failed, so this
		// should be handled appropriately
		b.finish()
	}
}

// Put ...
func (b *bootstrapper) Put(vdr ids.ShortID, requestID uint32, vtxID ids.ID, vtxBytes []byte) {
	vtx, err := b.State.ParseVertex(vtxBytes)
	if err != nil {
		b.BootstrapConfig.Context.Log.Debug("ParseVertex failed due to %s for block:\n%s",
			err,
			formatting.DumpBytes{Bytes: vtxBytes})

		b.GetFailed(vdr, requestID)
		return
	}

	if !b.pending.Contains(vtx.ID()) {
		b.BootstrapConfig.Context.Log.Debug("Validator %s sent an unrequested vertex:\n%s",
			vdr,
			formatting.DumpBytes{Bytes: vtxBytes})

		b.GetFailed(vdr, requestID)
		return
	}

	b.addVertex(vtx)
}

// GetFailed ...
func (b *bootstrapper) GetFailed(vdr ids.ShortID, requestID uint32) {
	vtxID, ok := b.vtxReqs.Remove(vdr, requestID)
	if !ok {
		b.BootstrapConfig.Context.Log.Debug("GetFailed called without sending the corresponding Get message from %s",
			vdr)
		return
	}

	b.sendRequest(vtxID)
}

func (b *bootstrapper) fetch(vtxID ids.ID) {
	if b.pending.Contains(vtxID) {
		return
	}

	vtx, err := b.State.GetVertex(vtxID)
	if err != nil {
		b.sendRequest(vtxID)
		return
	}
	b.storeVertex(vtx)
}

func (b *bootstrapper) sendRequest(vtxID ids.ID) {
	validators := b.BootstrapConfig.Validators.Sample(1)
	if len(validators) == 0 {
		b.BootstrapConfig.Context.Log.Error("Dropping request for %s as there are no validators", vtxID)
		return
	}
	validatorID := validators[0].ID()
	b.RequestID++

	b.vtxReqs.RemoveAny(vtxID)
	b.vtxReqs.Add(validatorID, b.RequestID, vtxID)

	b.pending.Add(vtxID)
	b.BootstrapConfig.Sender.Get(validatorID, b.RequestID, vtxID)

	b.numPendingRequests.Set(float64(b.pending.Len()))
}

func (b *bootstrapper) addVertex(vtx avalanche.Vertex) {
	b.storeVertex(vtx)

	if numPending := b.pending.Len(); numPending == 0 {
		b.finish()
	}
}

func (b *bootstrapper) storeVertex(vtx avalanche.Vertex) {
	vts := []avalanche.Vertex{vtx}
	b.numFetched++
	if b.numFetched%2500 == 0 { // perioidcally inform user of progress
		b.BootstrapConfig.Context.Log.Info("bootstrapping has fetched %d vertices", b.numFetched)
	}

	for len(vts) > 0 {
		newLen := len(vts) - 1
		vtx := vts[newLen]
		vts = vts[:newLen]

		vtxID := vtx.ID()
		switch status := vtx.Status(); status {
		case choices.Unknown:
			b.sendRequest(vtxID)
		case choices.Processing:
			b.pending.Remove(vtxID)

			if err := b.VtxBlocked.Push(&vertexJob{
				numAccepted: b.numBootstrappedVtx,
				numDropped:  b.numDroppedVtx,
				vtx:         vtx,
			}); err == nil {
				b.numBlockedVtx.Inc()
			} else {
				b.BootstrapConfig.Context.Log.Verbo("couldn't push to vtxBlocked")
			}
			for _, tx := range vtx.Txs() {
				if err := b.TxBlocked.Push(&txJob{
					numAccepted: b.numBootstrappedVtx,
					numDropped:  b.numDroppedVtx,
					tx:          tx,
				}); err == nil {
					b.numBlockedTx.Inc()
				} else {
					b.BootstrapConfig.Context.Log.Verbo("couldn't push to txBlocked")
				}
			}
			for _, parent := range vtx.Parents() {
				if parentID := parent.ID(); !b.seen.Contains(parentID) {
					b.seen.Add(parentID)
					vts = append(vts, parent)
				}
			}
		case choices.Accepted:
			b.BootstrapConfig.Context.Log.Verbo("Bootstrapping confirmed %s", vtxID)
		case choices.Rejected:
			b.BootstrapConfig.Context.Log.Error("Bootstrapping wants to accept %s, however it was previously rejected", vtxID)
		}
	}

	numPending := b.pending.Len()
	b.numPendingRequests.Set(float64(numPending))
}

func (b *bootstrapper) finish() {
	if b.finished {
		return
	}
	b.BootstrapConfig.Context.Log.Info("bootstrapping finished fetching vertices. executing state transitions...")

	b.executeAll(b.TxBlocked, b.numBlockedTx)
	b.executeAll(b.VtxBlocked, b.numBlockedVtx)

	// Start consensus
	b.onFinished()
	b.seen = ids.Set{}
	b.finished = true
}

func (b *bootstrapper) executeAll(jobs *queue.Jobs, numBlocked prometheus.Gauge) {
	for job, err := jobs.Pop(); err == nil; job, err = jobs.Pop() {
		numBlocked.Dec()
		b.BootstrapConfig.Context.Log.Debug("Executing: %s", job.ID())
		if err := jobs.Execute(job); err != nil {
			b.BootstrapConfig.Context.Log.Warn("Error executing: %s", err)
		}
	}
}
<|endoftext|>