<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/grafana/grafana/pkg/log"
	"github.com/raintank/raintank-metric/metric_tank/consolidation"
	"runtime"
	"sort"
)

// doRecover is the handler that turns panics into returns from the top level of getTarget.
func doRecover(errp *error) {
	e := recover()
	if e != nil {
		if _, ok := e.(runtime.Error); ok {
			panic(e)
		}
		if err, ok := e.(error); ok {
			*errp = err
		} else if errStr, ok := e.(string); ok {
			*errp = errors.New(errStr)
		} else {
			*errp = fmt.Errorf("%v", e)
		}
	}
	return
}

func divide(pointsA, pointsB []Point) []Point {
	if len(pointsA) != len(pointsB) {
		panic(fmt.Errorf("divide of a series with len %d by a series with len %d", len(pointsA), len(pointsB)))
	}
	out := make([]Point, len(pointsA))
	for i, a := range pointsA {
		b := pointsB[i]
		out[i] = Point{a.Val / b.Val, a.Ts}
	}
	return out
}

func consolidate(in []Point, num int, consolidator consolidation.Consolidator) []Point {
	aggFunc := consolidation.GetAggFunc(consolidator)
	buf := make([]float64, num)
	bufpos := -1
	points := make([]Point, (len(in)/num)+1)
	for inpos, p := range in {
		bufpos = inpos % num
		buf[bufpos] = p.Val
		if bufpos == num-1 {
			points = append(points, Point{aggFunc(buf), p.Ts})
		}
	}
	if bufpos != -1 && bufpos < num-1 {
		// we have an incomplete buf of some points that didn't get aggregated yet
		points = append(points, Point{aggFunc(buf[:bufpos+1]), in[len(in)-1].Ts})
	}
	return points
}

// returns how many points should be aggregated together so that you end up with as many points as possible,
// but never more than maxPoints
func aggEvery(numPoints, maxPoints uint32) int {
	return int(numPoints / maxPoints)
}

type planOption struct {
	archive  string
	interval uint32
	intestim bool
	points   uint32
	comment  string
}

type plan []planOption

func (a plan) Len() int           { return len(a) }
func (a plan) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a plan) Less(i, j int) bool { return a[i].points > a[j].points }

func getTarget(req Req, aggSettings []aggSetting, metaCache *MetaCache) (points []Point, err error) {
	defer doRecover(&err)
	archive := -1 // -1 means original data, 0 last agg level, 1 2nd last, etc.

	p := make([]planOption, len(aggSettings)+1)
	guess := false

	// note: the metacache is clearly not a perfect all-knowning entity, it just knows the last interval of metrics seen since program start
	// and we assume we can use that interval through history.
	// TODO: no support for interval changes, metrics not seen yet, missing datablocks, ...
	meta := metaCache.Get(req.key)
	interval := uint32(meta.interval)

	// we don't have the data yet, let's assume the interval is 10 seconds
	if interval == 0 {
		guess = true
		interval = 10
	}
	numPoints := (req.to - req.from) / interval

	p[0] = planOption{"raw", interval, guess, numPoints, ""}

	aggs := aggSettingsSpanDesc(aggSettings)
	sort.Sort(aggs)
	finished := false
	for i, aggSetting := range aggs {
		numPointsHere := (req.to - req.from) / aggSetting.span
		p[i+1] = planOption{fmt.Sprintf("agg %d", i), aggSetting.span, false, numPointsHere, ""}
		if numPointsHere >= req.minPoints && !finished {
			archive = i
			interval = aggSetting.span
			numPoints = numPointsHere
			finished = true
		}
	}

	p[archive+1].comment = "<-- chosen"

	// note, it should always be safe to dynamically switch on/off consolidation based on how well our data stacks up against the request
	// i.e. whether your data got consolidated or not, it should be pretty equivalent.
	// for that reason, stdev should not be done as a consolidation. but sos is still useful for when we explicitly (and always, not optionally) want the stdev.

	readConsolidated := (archive != -1)                 // do we need to read from a downsampled series?
	runtimeConsolidation := (numPoints > req.maxPoints) // do we need to compress any points at runtime?

	log.Debug("getTarget()         %s", req)
	log.Debug("type   interval   points")
	sortedPlan := plan(p)
	sort.Sort(sortedPlan)
	for _, opt := range p {
		iStr := fmt.Sprintf("%d", opt.interval)
		if opt.intestim {
			iStr = fmt.Sprintf("%d (guess)", opt.interval)
		}
		log.Debug("%-6s %-10s %-6d %s", opt.archive, iStr, opt.points, opt.comment)
	}
	log.Debug("runtimeConsolidation: %t", runtimeConsolidation)

	if !readConsolidated && !runtimeConsolidation {
		return getSeries(req.key, consolidation.None, 0, req.from, req.to), nil
	} else if !readConsolidated && runtimeConsolidation {
		return consolidate(
			getSeries(req.key, consolidation.None, 0, req.from, req.to),
			aggEvery(numPoints, req.maxPoints),
			req.consolidator), nil
	} else if readConsolidated && !runtimeConsolidation {
		if req.consolidator == consolidation.Avg {
			return divide(
				getSeries(req.key, consolidation.Sum, interval, req.from, req.to),
				getSeries(req.key, consolidation.Cnt, interval, req.from, req.to),
			), nil
		} else {
			return getSeries(req.key, req.consolidator, interval, req.from, req.to), nil
		}
	} else {
		// readConsolidated && runtimeConsolidation
		aggNum := aggEvery(numPoints, req.maxPoints)
		if req.consolidator == consolidation.Avg {
			return divide(
				consolidate(
					getSeries(req.key, consolidation.Sum, interval, req.from, req.to),
					aggNum,
					consolidation.Sum),
				consolidate(
					getSeries(req.key, consolidation.Cnt, interval, req.from, req.to),
					aggNum,
					consolidation.Cnt),
			), nil
		} else {
			return consolidate(
				getSeries(req.key, req.consolidator, interval, req.from, req.to),
				aggNum, req.consolidator), nil
		}
	}
}

func logLoad(typ, key string, from, to uint32) {
	log.Debug("load from %-6s %-20s %d - %d (%s - %s) span:%ds", typ, key, from, to, TS(from), TS(to), to-from-1)
}

func aggMetricKey(key, archive string, aggSpan uint32) string {
	return fmt.Sprintf("%s_%s_%d", key, archive, aggSpan)
}

// getSeries just gets the needed raw iters from mem and/or cassandra, based on from/to
// it can query for data within aggregated archives, by using fn min/max/sos/sum/cnt and providing the matching agg span.
func getSeries(key string, consolidator consolidation.Consolidator, aggSpan, fromUnix, toUnix uint32) []Point {
	iters := make([]Iter, 0)
	memIters := make([]Iter, 0)
	oldest := toUnix
	if metric, ok := metrics.Get(key); ok {
		if consolidator != consolidation.None {
			logLoad("memory", aggMetricKey(key, consolidator.Archive(), aggSpan), fromUnix, toUnix)
			oldest, memIters = metric.GetAggregated(consolidator, aggSpan, fromUnix, toUnix)
		} else {
			logLoad("memory", key, fromUnix, toUnix)
			oldest, memIters = metric.Get(fromUnix, toUnix)
		}
	}
	if oldest > fromUnix {
		reqSpanBoth.Value(int64(toUnix - fromUnix))
		if consolidator != consolidation.None {
			key = aggMetricKey(key, consolidator.Archive(), aggSpan)
		}
		logLoad("cassan", key, fromUnix, oldest)
		storeIters, err := searchCassandra(key, fromUnix, oldest)
		if err != nil {
			panic(err)
		}
		iters = append(iters, storeIters...)
	} else {
		reqSpanMem.Value(int64(toUnix - fromUnix))
	}
	iters = append(iters, memIters...)

	points := make([]Point, 0)
	for _, iter := range iters {
		total := 0
		good := 0
		for iter.Next() {
			total += 1
			ts, val := iter.Values()
			if ts >= fromUnix && ts < toUnix {
				good += 1
				points = append(points, Point{val, ts})
			}
		}
		log.Debug("getSeries: iter %s  values good/total %d/%d", iter.cmt, good, total)
	}
	return points
}
<commit_msg>fix aggEvery logic<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/grafana/grafana/pkg/log"
	"github.com/raintank/raintank-metric/metric_tank/consolidation"
	"runtime"
	"sort"
)

// doRecover is the handler that turns panics into returns from the top level of getTarget.
func doRecover(errp *error) {
	e := recover()
	if e != nil {
		if _, ok := e.(runtime.Error); ok {
			panic(e)
		}
		if err, ok := e.(error); ok {
			*errp = err
		} else if errStr, ok := e.(string); ok {
			*errp = errors.New(errStr)
		} else {
			*errp = fmt.Errorf("%v", e)
		}
	}
	return
}

func divide(pointsA, pointsB []Point) []Point {
	if len(pointsA) != len(pointsB) {
		panic(fmt.Errorf("divide of a series with len %d by a series with len %d", len(pointsA), len(pointsB)))
	}
	out := make([]Point, len(pointsA))
	for i, a := range pointsA {
		b := pointsB[i]
		out[i] = Point{a.Val / b.Val, a.Ts}
	}
	return out
}

func consolidate(in []Point, num int, consolidator consolidation.Consolidator) []Point {
	aggFunc := consolidation.GetAggFunc(consolidator)
	buf := make([]float64, num)
	bufpos := -1
	points := make([]Point, (len(in)/num)+1)
	for inpos, p := range in {
		bufpos = inpos % num
		buf[bufpos] = p.Val
		if bufpos == num-1 {
			points = append(points, Point{aggFunc(buf), p.Ts})
		}
	}
	if bufpos != -1 && bufpos < num-1 {
		// we have an incomplete buf of some points that didn't get aggregated yet
		points = append(points, Point{aggFunc(buf[:bufpos+1]), in[len(in)-1].Ts})
	}
	return points
}

// returns how many points should be aggregated together so that you end up with as many points as possible,
// but never more than maxPoints
func aggEvery(numPoints, maxPoints uint32) int {
	return int((numPoints + maxPoints - 1) / maxPoints)
}

type planOption struct {
	archive  string
	interval uint32
	intestim bool
	points   uint32
	comment  string
}

type plan []planOption

func (a plan) Len() int           { return len(a) }
func (a plan) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a plan) Less(i, j int) bool { return a[i].points > a[j].points }

func getTarget(req Req, aggSettings []aggSetting, metaCache *MetaCache) (points []Point, err error) {
	defer doRecover(&err)
	archive := -1 // -1 means original data, 0 last agg level, 1 2nd last, etc.

	p := make([]planOption, len(aggSettings)+1)
	guess := false

	// note: the metacache is clearly not a perfect all-knowning entity, it just knows the last interval of metrics seen since program start
	// and we assume we can use that interval through history.
	// TODO: no support for interval changes, metrics not seen yet, missing datablocks, ...
	meta := metaCache.Get(req.key)
	interval := uint32(meta.interval)

	// we don't have the data yet, let's assume the interval is 10 seconds
	if interval == 0 {
		guess = true
		interval = 10
	}
	numPoints := (req.to - req.from) / interval

	p[0] = planOption{"raw", interval, guess, numPoints, ""}

	aggs := aggSettingsSpanDesc(aggSettings)
	sort.Sort(aggs)
	finished := false
	for i, aggSetting := range aggs {
		numPointsHere := (req.to - req.from) / aggSetting.span
		p[i+1] = planOption{fmt.Sprintf("agg %d", i), aggSetting.span, false, numPointsHere, ""}
		if numPointsHere >= req.minPoints && !finished {
			archive = i
			interval = aggSetting.span
			numPoints = numPointsHere
			finished = true
		}
	}

	p[archive+1].comment = "<-- chosen"

	// note, it should always be safe to dynamically switch on/off consolidation based on how well our data stacks up against the request
	// i.e. whether your data got consolidated or not, it should be pretty equivalent.
	// for that reason, stdev should not be done as a consolidation. but sos is still useful for when we explicitly (and always, not optionally) want the stdev.

	readConsolidated := (archive != -1)                 // do we need to read from a downsampled series?
	runtimeConsolidation := (numPoints > req.maxPoints) // do we need to compress any points at runtime?

	log.Debug("getTarget()         %s", req)
	log.Debug("type   interval   points")
	sortedPlan := plan(p)
	sort.Sort(sortedPlan)
	for _, opt := range p {
		iStr := fmt.Sprintf("%d", opt.interval)
		if opt.intestim {
			iStr = fmt.Sprintf("%d (guess)", opt.interval)
		}
		log.Debug("%-6s %-10s %-6d %s", opt.archive, iStr, opt.points, opt.comment)
	}
	log.Debug("runtimeConsolidation: %t", runtimeConsolidation)

	if !readConsolidated && !runtimeConsolidation {
		return getSeries(req.key, consolidation.None, 0, req.from, req.to), nil
	} else if !readConsolidated && runtimeConsolidation {
		return consolidate(
			getSeries(req.key, consolidation.None, 0, req.from, req.to),
			aggEvery(numPoints, req.maxPoints),
			req.consolidator), nil
	} else if readConsolidated && !runtimeConsolidation {
		if req.consolidator == consolidation.Avg {
			return divide(
				getSeries(req.key, consolidation.Sum, interval, req.from, req.to),
				getSeries(req.key, consolidation.Cnt, interval, req.from, req.to),
			), nil
		} else {
			return getSeries(req.key, req.consolidator, interval, req.from, req.to), nil
		}
	} else {
		// readConsolidated && runtimeConsolidation
		aggNum := aggEvery(numPoints, req.maxPoints)
		if req.consolidator == consolidation.Avg {
			return divide(
				consolidate(
					getSeries(req.key, consolidation.Sum, interval, req.from, req.to),
					aggNum,
					consolidation.Sum),
				consolidate(
					getSeries(req.key, consolidation.Cnt, interval, req.from, req.to),
					aggNum,
					consolidation.Cnt),
			), nil
		} else {
			return consolidate(
				getSeries(req.key, req.consolidator, interval, req.from, req.to),
				aggNum, req.consolidator), nil
		}
	}
}

func logLoad(typ, key string, from, to uint32) {
	log.Debug("load from %-6s %-20s %d - %d (%s - %s) span:%ds", typ, key, from, to, TS(from), TS(to), to-from-1)
}

func aggMetricKey(key, archive string, aggSpan uint32) string {
	return fmt.Sprintf("%s_%s_%d", key, archive, aggSpan)
}

// getSeries just gets the needed raw iters from mem and/or cassandra, based on from/to
// it can query for data within aggregated archives, by using fn min/max/sos/sum/cnt and providing the matching agg span.
func getSeries(key string, consolidator consolidation.Consolidator, aggSpan, fromUnix, toUnix uint32) []Point {
	iters := make([]Iter, 0)
	memIters := make([]Iter, 0)
	oldest := toUnix
	if metric, ok := metrics.Get(key); ok {
		if consolidator != consolidation.None {
			logLoad("memory", aggMetricKey(key, consolidator.Archive(), aggSpan), fromUnix, toUnix)
			oldest, memIters = metric.GetAggregated(consolidator, aggSpan, fromUnix, toUnix)
		} else {
			logLoad("memory", key, fromUnix, toUnix)
			oldest, memIters = metric.Get(fromUnix, toUnix)
		}
	}
	if oldest > fromUnix {
		reqSpanBoth.Value(int64(toUnix - fromUnix))
		if consolidator != consolidation.None {
			key = aggMetricKey(key, consolidator.Archive(), aggSpan)
		}
		logLoad("cassan", key, fromUnix, oldest)
		storeIters, err := searchCassandra(key, fromUnix, oldest)
		if err != nil {
			panic(err)
		}
		iters = append(iters, storeIters...)
	} else {
		reqSpanMem.Value(int64(toUnix - fromUnix))
	}
	iters = append(iters, memIters...)

	points := make([]Point, 0)
	for _, iter := range iters {
		total := 0
		good := 0
		for iter.Next() {
			total += 1
			ts, val := iter.Values()
			if ts >= fromUnix && ts < toUnix {
				good += 1
				points = append(points, Point{val, ts})
			}
		}
		log.Debug("getSeries: iter %s  values good/total %d/%d", iter.cmt, good, total)
	}
	return points
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package venaficloud

import (
	"context"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
	cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
	"github.com/jetstack/cert-manager/test/e2e/framework"
	"github.com/jetstack/cert-manager/test/e2e/framework/helper/featureset"
	"github.com/jetstack/cert-manager/test/e2e/framework/util/errors"
	"github.com/jetstack/cert-manager/test/e2e/suite/conformance/certificates"
	vaddon "github.com/jetstack/cert-manager/test/e2e/suite/issuers/venafi/addon"
)

var _ = framework.ConformanceDescribe("[Feature:Issuers:Venafi:Cloud] Certificates", func() {
	// unsupportedFeatures is a list of features that are not supported by the
	// Venafi Cloud issuer.
	var unsupportedFeatures = featureset.NewFeatureSet(
		// Venafi Cloud does not allow setting duration in request
		featureset.DurationFeature,
		// Venafi Cloud has no ECDSA support
		featureset.ECDSAFeature,
		// Alternate SANS are currently not supported in Venafi Cloud
		featureset.EmailSANsFeature,
		featureset.IPAddressFeature,
		featureset.URISANsFeature,
		// Venafi doesn't allow certs with empty CN & DN
		featureset.OnlySAN,
	)

	provisioner := new(venafiProvisioner)
	(&certificates.Suite{
		Name:                "Venafi Cloud Issuer",
		CreateIssuerFunc:    provisioner.createIssuer,
		DeleteIssuerFunc:    provisioner.delete,
		UnsupportedFeatures: unsupportedFeatures,
	}).Define()

	(&certificates.Suite{
		Name:                "Venafi Cloud ClusterIssuer",
		CreateIssuerFunc:    provisioner.createClusterIssuer,
		DeleteIssuerFunc:    provisioner.delete,
		UnsupportedFeatures: unsupportedFeatures,
	}).Define()
})

type venafiProvisioner struct {
	cloud *vaddon.VenafiCloud
}

func (v *venafiProvisioner) delete(f *framework.Framework, ref cmmeta.ObjectReference) {
	Expect(v.cloud.Deprovision()).NotTo(HaveOccurred(), "failed to deprovision cloud venafi")

	if ref.Kind == "ClusterIssuer" {
		err := f.CertManagerClientSet.CertmanagerV1().ClusterIssuers().Delete(context.TODO(), ref.Name, metav1.DeleteOptions{})
		Expect(err).NotTo(HaveOccurred())
	}
}

func (v *venafiProvisioner) createIssuer(f *framework.Framework) cmmeta.ObjectReference {
	By("Creating a Venafi Cloud Issuer")

	v.cloud = &vaddon.VenafiCloud{
		Namespace: f.Namespace.Name,
	}

	err := v.cloud.Setup(f.Config)
	if errors.IsSkip(err) {
		framework.Skipf("Skipping test as addon could not be setup: %v", err)
	}
	Expect(err).NotTo(HaveOccurred(), "failed to provision venafi cloud issuer")

	Expect(v.cloud.Provision()).NotTo(HaveOccurred(), "failed to provision tpp venafi")

	issuer := v.cloud.Details().BuildIssuer()
	issuer, err = f.CertManagerClientSet.CertmanagerV1().Issuers(f.Namespace.Name).Create(context.TODO(), issuer, metav1.CreateOptions{})
	Expect(err).NotTo(HaveOccurred(), "failed to create issuer for venafi")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.IssuerKind,
		Name:  issuer.Name,
	}
}

func (v *venafiProvisioner) createClusterIssuer(f *framework.Framework) cmmeta.ObjectReference {
	By("Creating a Venafi ClusterIssuer")

	v.cloud = &vaddon.VenafiCloud{
		Namespace: f.Config.Addons.CertManager.ClusterResourceNamespace,
	}

	err := v.cloud.Setup(f.Config)
	if errors.IsSkip(err) {
		framework.Skipf("Skipping test as addon could not be setup: %v", err)
	}
	Expect(err).NotTo(HaveOccurred(), "failed to setup tpp venafi")

	Expect(v.cloud.Provision()).NotTo(HaveOccurred(), "failed to provision tpp venafi")

	issuer := v.cloud.Details().BuildClusterIssuer()
	issuer, err = f.CertManagerClientSet.CertmanagerV1().ClusterIssuers().Create(context.TODO(), issuer, metav1.CreateOptions{})
	Expect(err).NotTo(HaveOccurred(), "failed to create issuer for venafi")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.ClusterIssuerKind,
		Name:  issuer.Name,
	}
}
<commit_msg>Enable the Venafi Cloud E2E tests<commit_after>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package venaficloud

import (
	"context"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
	cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
	"github.com/jetstack/cert-manager/test/e2e/framework"
	"github.com/jetstack/cert-manager/test/e2e/framework/helper/featureset"
	"github.com/jetstack/cert-manager/test/e2e/framework/util/errors"
	"github.com/jetstack/cert-manager/test/e2e/suite/conformance/certificates"
	vaddon "github.com/jetstack/cert-manager/test/e2e/suite/issuers/venafi/addon"
)

var _ = framework.ConformanceDescribe("Certificates", func() {
	// unsupportedFeatures is a list of features that are not supported by the
	// Venafi Cloud issuer.
	var unsupportedFeatures = featureset.NewFeatureSet(
		// Venafi Cloud does not allow setting duration in request
		featureset.DurationFeature,
		// Venafi Cloud has no ECDSA support
		featureset.ECDSAFeature,
		// Alternate SANS are currently not supported in Venafi Cloud
		featureset.EmailSANsFeature,
		featureset.IPAddressFeature,
		featureset.URISANsFeature,
		// Venafi doesn't allow certs with empty CN & DN
		featureset.OnlySAN,
	)

	provisioner := new(venafiProvisioner)
	(&certificates.Suite{
		Name:                "Venafi Cloud Issuer",
		CreateIssuerFunc:    provisioner.createIssuer,
		DeleteIssuerFunc:    provisioner.delete,
		UnsupportedFeatures: unsupportedFeatures,
	}).Define()

	(&certificates.Suite{
		Name:                "Venafi Cloud ClusterIssuer",
		CreateIssuerFunc:    provisioner.createClusterIssuer,
		DeleteIssuerFunc:    provisioner.delete,
		UnsupportedFeatures: unsupportedFeatures,
	}).Define()
})

type venafiProvisioner struct {
	cloud *vaddon.VenafiCloud
}

func (v *venafiProvisioner) delete(f *framework.Framework, ref cmmeta.ObjectReference) {
	Expect(v.cloud.Deprovision()).NotTo(HaveOccurred(), "failed to deprovision cloud venafi")

	if ref.Kind == "ClusterIssuer" {
		err := f.CertManagerClientSet.CertmanagerV1().ClusterIssuers().Delete(context.TODO(), ref.Name, metav1.DeleteOptions{})
		Expect(err).NotTo(HaveOccurred())
	}
}

func (v *venafiProvisioner) createIssuer(f *framework.Framework) cmmeta.ObjectReference {
	By("Creating a Venafi Cloud Issuer")

	v.cloud = &vaddon.VenafiCloud{
		Namespace: f.Namespace.Name,
	}

	err := v.cloud.Setup(f.Config)
	if errors.IsSkip(err) {
		framework.Skipf("Skipping test as addon could not be setup: %v", err)
	}
	Expect(err).NotTo(HaveOccurred(), "failed to provision venafi cloud issuer")

	Expect(v.cloud.Provision()).NotTo(HaveOccurred(), "failed to provision tpp venafi")

	issuer := v.cloud.Details().BuildIssuer()
	issuer, err = f.CertManagerClientSet.CertmanagerV1().Issuers(f.Namespace.Name).Create(context.TODO(), issuer, metav1.CreateOptions{})
	Expect(err).NotTo(HaveOccurred(), "failed to create issuer for venafi")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.IssuerKind,
		Name:  issuer.Name,
	}
}

func (v *venafiProvisioner) createClusterIssuer(f *framework.Framework) cmmeta.ObjectReference {
	By("Creating a Venafi ClusterIssuer")

	v.cloud = &vaddon.VenafiCloud{
		Namespace: f.Config.Addons.CertManager.ClusterResourceNamespace,
	}

	err := v.cloud.Setup(f.Config)
	if errors.IsSkip(err) {
		framework.Skipf("Skipping test as addon could not be setup: %v", err)
	}
	Expect(err).NotTo(HaveOccurred(), "failed to setup tpp venafi")

	Expect(v.cloud.Provision()).NotTo(HaveOccurred(), "failed to provision tpp venafi")

	issuer := v.cloud.Details().BuildClusterIssuer()
	issuer, err = f.CertManagerClientSet.CertmanagerV1().ClusterIssuers().Create(context.TODO(), issuer, metav1.CreateOptions{})
	Expect(err).NotTo(HaveOccurred(), "failed to create issuer for venafi")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.ClusterIssuerKind,
		Name:  issuer.Name,
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package venaficloud

import (
	"context"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
	cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
	"github.com/jetstack/cert-manager/test/e2e/framework"
	"github.com/jetstack/cert-manager/test/e2e/framework/helper/featureset"
	"github.com/jetstack/cert-manager/test/e2e/framework/util/errors"
	"github.com/jetstack/cert-manager/test/e2e/suite/conformance/certificates"
	vaddon "github.com/jetstack/cert-manager/test/e2e/suite/issuers/venafi/addon"
)

var _ = framework.ConformanceDescribe("Certificates", func() {
	// unsupportedFeatures is a list of features that are not supported by the
	// Venafi Cloud issuer.
	var unsupportedFeatures = featureset.NewFeatureSet(
		// Venafi Cloud does not allow setting duration in request
		featureset.DurationFeature,
		// Venafi Cloud has no ECDSA support
		featureset.ECDSAFeature,
		// Alternate SANS are currently not supported in Venafi Cloud
		featureset.EmailSANsFeature,
		featureset.IPAddressFeature,
		featureset.URISANsFeature,
		// Venafi doesn't allow certs with empty CN & DN
		featureset.OnlySAN,
	)

	provisioner := new(venafiProvisioner)
	(&certificates.Suite{
		Name:                "Venafi Cloud Issuer",
		CreateIssuerFunc:    provisioner.createIssuer,
		DeleteIssuerFunc:    provisioner.delete,
		UnsupportedFeatures: unsupportedFeatures,
	}).Define()

	(&certificates.Suite{
		Name:                "Venafi Cloud ClusterIssuer",
		CreateIssuerFunc:    provisioner.createClusterIssuer,
		DeleteIssuerFunc:    provisioner.delete,
		UnsupportedFeatures: unsupportedFeatures,
	}).Define()
})

type venafiProvisioner struct {
	cloud *vaddon.VenafiCloud
}

func (v *venafiProvisioner) delete(f *framework.Framework, ref cmmeta.ObjectReference) {
	Expect(v.cloud.Deprovision()).NotTo(HaveOccurred(), "failed to deprovision cloud venafi")

	if ref.Kind == "ClusterIssuer" {
		err := f.CertManagerClientSet.CertmanagerV1().ClusterIssuers().Delete(context.TODO(), ref.Name, metav1.DeleteOptions{})
		Expect(err).NotTo(HaveOccurred())
	}
}

func (v *venafiProvisioner) createIssuer(f *framework.Framework) cmmeta.ObjectReference {
	By("Creating a Venafi Cloud Issuer")

	v.cloud = &vaddon.VenafiCloud{
		Namespace: f.Namespace.Name,
	}

	err := v.cloud.Setup(f.Config)
	if errors.IsSkip(err) {
		framework.Skipf("Skipping test as addon could not be setup: %v", err)
	}
	Expect(err).NotTo(HaveOccurred(), "failed to provision venafi cloud issuer")

	Expect(v.cloud.Provision()).NotTo(HaveOccurred(), "failed to provision tpp venafi")

	issuer := v.cloud.Details().BuildIssuer()
	issuer, err = f.CertManagerClientSet.CertmanagerV1().Issuers(f.Namespace.Name).Create(context.TODO(), issuer, metav1.CreateOptions{})
	Expect(err).NotTo(HaveOccurred(), "failed to create issuer for venafi")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.IssuerKind,
		Name:  issuer.Name,
	}
}

func (v *venafiProvisioner) createClusterIssuer(f *framework.Framework) cmmeta.ObjectReference {
	By("Creating a Venafi ClusterIssuer")

	v.cloud = &vaddon.VenafiCloud{
		Namespace: f.Config.Addons.CertManager.ClusterResourceNamespace,
	}

	err := v.cloud.Setup(f.Config)
	if errors.IsSkip(err) {
		framework.Skipf("Skipping test as addon could not be setup: %v", err)
	}
	Expect(err).NotTo(HaveOccurred(), "failed to setup tpp venafi")

	Expect(v.cloud.Provision()).NotTo(HaveOccurred(), "failed to provision tpp venafi")

	issuer := v.cloud.Details().BuildClusterIssuer()
	issuer, err = f.CertManagerClientSet.CertmanagerV1().ClusterIssuers().Create(context.TODO(), issuer, metav1.CreateOptions{})
	Expect(err).NotTo(HaveOccurred(), "failed to create issuer for venafi")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.ClusterIssuerKind,
		Name:  issuer.Name,
	}
}
<commit_msg>Temporarily revert "Enable the Venafi Cloud E2E tests"<commit_after>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package venaficloud

import (
	"context"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	cmapi "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
	cmmeta "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
	"github.com/jetstack/cert-manager/test/e2e/framework"
	"github.com/jetstack/cert-manager/test/e2e/framework/helper/featureset"
	"github.com/jetstack/cert-manager/test/e2e/framework/util/errors"
	"github.com/jetstack/cert-manager/test/e2e/suite/conformance/certificates"
	vaddon "github.com/jetstack/cert-manager/test/e2e/suite/issuers/venafi/addon"
)

var _ = framework.ConformanceDescribe("[Feature:Issuers:Venafi:Cloud] Certificates", func() {
	// unsupportedFeatures is a list of features that are not supported by the
	// Venafi Cloud issuer.
	var unsupportedFeatures = featureset.NewFeatureSet(
		// Venafi Cloud does not allow setting duration in request
		featureset.DurationFeature,
		// Venafi Cloud has no ECDSA support
		featureset.ECDSAFeature,
		// Alternate SANS are currently not supported in Venafi Cloud
		featureset.EmailSANsFeature,
		featureset.IPAddressFeature,
		featureset.URISANsFeature,
		// Venafi doesn't allow certs with empty CN & DN
		featureset.OnlySAN,
	)

	provisioner := new(venafiProvisioner)
	(&certificates.Suite{
		Name:                "Venafi Cloud Issuer",
		CreateIssuerFunc:    provisioner.createIssuer,
		DeleteIssuerFunc:    provisioner.delete,
		UnsupportedFeatures: unsupportedFeatures,
	}).Define()

	(&certificates.Suite{
		Name:                "Venafi Cloud ClusterIssuer",
		CreateIssuerFunc:    provisioner.createClusterIssuer,
		DeleteIssuerFunc:    provisioner.delete,
		UnsupportedFeatures: unsupportedFeatures,
	}).Define()
})

type venafiProvisioner struct {
	cloud *vaddon.VenafiCloud
}

func (v *venafiProvisioner) delete(f *framework.Framework, ref cmmeta.ObjectReference) {
	Expect(v.cloud.Deprovision()).NotTo(HaveOccurred(), "failed to deprovision cloud venafi")

	if ref.Kind == "ClusterIssuer" {
		err := f.CertManagerClientSet.CertmanagerV1().ClusterIssuers().Delete(context.TODO(), ref.Name, metav1.DeleteOptions{})
		Expect(err).NotTo(HaveOccurred())
	}
}

func (v *venafiProvisioner) createIssuer(f *framework.Framework) cmmeta.ObjectReference {
	By("Creating a Venafi Cloud Issuer")

	v.cloud = &vaddon.VenafiCloud{
		Namespace: f.Namespace.Name,
	}

	err := v.cloud.Setup(f.Config)
	if errors.IsSkip(err) {
		framework.Skipf("Skipping test as addon could not be setup: %v", err)
	}
	Expect(err).NotTo(HaveOccurred(), "failed to provision venafi cloud issuer")

	Expect(v.cloud.Provision()).NotTo(HaveOccurred(), "failed to provision tpp venafi")

	issuer := v.cloud.Details().BuildIssuer()
	issuer, err = f.CertManagerClientSet.CertmanagerV1().Issuers(f.Namespace.Name).Create(context.TODO(), issuer, metav1.CreateOptions{})
	Expect(err).NotTo(HaveOccurred(), "failed to create issuer for venafi")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.IssuerKind,
		Name:  issuer.Name,
	}
}

func (v *venafiProvisioner) createClusterIssuer(f *framework.Framework) cmmeta.ObjectReference {
	By("Creating a Venafi ClusterIssuer")

	v.cloud = &vaddon.VenafiCloud{
		Namespace: f.Config.Addons.CertManager.ClusterResourceNamespace,
	}

	err := v.cloud.Setup(f.Config)
	if errors.IsSkip(err) {
		framework.Skipf("Skipping test as addon could not be setup: %v", err)
	}
	Expect(err).NotTo(HaveOccurred(), "failed to setup tpp venafi")

	Expect(v.cloud.Provision()).NotTo(HaveOccurred(), "failed to provision tpp venafi")

	issuer := v.cloud.Details().BuildClusterIssuer()
	issuer, err = f.CertManagerClientSet.CertmanagerV1().ClusterIssuers().Create(context.TODO(), issuer, metav1.CreateOptions{})
	Expect(err).NotTo(HaveOccurred(), "failed to create issuer for venafi")

	return cmmeta.ObjectReference{
		Group: cmapi.SchemeGroupVersion.Group,
		Kind:  cmapi.ClusterIssuerKind,
		Name:  issuer.Name,
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Mark Nevill. All Rights Reserved.
// See LICENSE for licensing terms.

package http_prometheus

import (
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/improbable-eng/go-httpwares/metrics"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	clientStarted = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "http_tripper_started_requests_total",
			Help: "Count of started requests.",
		},
		[]string{"name", "handler", "host", "path", "method"},
	)
	clientCompleted = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "http_tripper_completed_requests_total",
			Help: "Count of completed requests.",
		},
		[]string{"name", "handler", "host", "path", "method", "status"},
	)
	clientLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "http_tripper_completed_latency_seconds",
			Help:    "Latency of completed requests.",
			Buckets: []float64{.01, .03, .1, .3, 1, 3, 10, 30, 100, 300},
		},
		[]string{"name", "handler", "host", "path", "method", "status"},
	)
	clientRequestSize = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "http_tripper_request_size_bytes",
			Help:    "Size of sent requests.",
			Buckets: prometheus.ExponentialBuckets(32, 32, 6),
		},
		[]string{"name", "handler", "host", "path", "method", "status"},
	)
	clientResponseSize = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "http_tripper_response_size_bytes",
			Help:    "Size of received responses.",
			Buckets: prometheus.ExponentialBuckets(32, 32, 6),
		},
		[]string{"name", "handler", "host", "path", "method", "status"},
	)

	clientInit     sync.Once
	clientHistInit sync.Once
	clientSizeInit sync.Once
)

func ClientMetrics(opts ...opt) http_metrics.Reporter {
	o := evalOpts(opts)
	clientInit.Do(func() {
		prometheus.MustRegister(clientStarted)
		prometheus.MustRegister(clientCompleted)
	})
	if o.latency {
		clientHistInit.Do(func() {
			prometheus.MustRegister(clientLatency)
		})
	}
	if o.sizes {
		clientSizeInit.Do(func() {
			prometheus.MustRegister(clientRequestSize)
			prometheus.MustRegister(clientResponseSize)
		})
	}
	return &clientReporter{opts: o}
}

type clientReporter struct {
	opts *options
}

func (r *clientReporter) Track(req *http.Request) http_metrics.Tracker {
	return &clientTracker{
		opts: r.opts,
		meta: reqMeta(req, r.opts, false),
	}
}

type clientTracker struct {
	opts *options
	*meta
}

func (t *clientTracker) RequestStarted() {
	clientStarted.WithLabelValues(t.name, t.handler, t.host, t.path, t.method).Inc()
}

func (t *clientTracker) RequestRead(duration time.Duration, size int) {
	if t.opts.sizes {
		clientRequestSize.WithLabelValues(t.name, t.handler, t.host, t.path, t.method).Observe(float64(size))
	}
}

func (t *clientTracker) ResponseStarted(duration time.Duration, code int, header http.Header) {
	status := strconv.Itoa(code)
	clientCompleted.WithLabelValues(t.name, t.handler, t.host, t.path, t.method, status).Inc()
	if t.opts.latency {
		clientLatency.WithLabelValues(t.name, t.handler, t.host, t.path, t.method, status).Observe(duration.Seconds())
	}
}

func (t *clientTracker) ResponseDone(duration time.Duration, code int, size int) {
	if t.opts.sizes {
		clientResponseSize.WithLabelValues(t.name, t.handler, t.host, t.path, t.method, strconv.Itoa(code)).Observe(float64(size))
	}
}
<commit_msg>Remove status code from request size metric (#54)<commit_after>// Copyright 2017 Mark Nevill. All Rights Reserved.
// See LICENSE for licensing terms.

package http_prometheus

import (
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/improbable-eng/go-httpwares/metrics"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	clientStarted = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "http_tripper_started_requests_total",
			Help: "Count of started requests.",
		},
		[]string{"name", "handler", "host", "path", "method"},
	)
	clientCompleted = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "http_tripper_completed_requests_total",
			Help: "Count of completed requests.",
		},
		[]string{"name", "handler", "host", "path", "method", "status"},
	)
	clientLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "http_tripper_completed_latency_seconds",
			Help:    "Latency of completed requests.",
			Buckets: []float64{.01, .03, .1, .3, 1, 3, 10, 30, 100, 300},
		},
		[]string{"name", "handler", "host", "path", "method", "status"},
	)
	clientRequestSize = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "http_tripper_request_size_bytes",
			Help:    "Size of sent requests.",
			Buckets: prometheus.ExponentialBuckets(32, 32, 6),
		},
		[]string{"name", "handler", "host", "path", "method"},
	)
	clientResponseSize = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "http_tripper_response_size_bytes",
			Help:    "Size of received responses.",
			Buckets: prometheus.ExponentialBuckets(32, 32, 6),
		},
		[]string{"name", "handler", "host", "path", "method", "status"},
	)

	clientInit     sync.Once
	clientHistInit sync.Once
	clientSizeInit sync.Once
)

func ClientMetrics(opts ...opt) http_metrics.Reporter {
	o := evalOpts(opts)
	clientInit.Do(func() {
		prometheus.MustRegister(clientStarted)
		prometheus.MustRegister(clientCompleted)
	})
	if o.latency {
		clientHistInit.Do(func() {
			prometheus.MustRegister(clientLatency)
		})
	}
	if o.sizes {
		clientSizeInit.Do(func() {
			prometheus.MustRegister(clientRequestSize)
			prometheus.MustRegister(clientResponseSize)
		})
	}
	return &clientReporter{opts: o}
}

type clientReporter struct {
	opts *options
}

func (r *clientReporter) Track(req *http.Request) http_metrics.Tracker {
	return &clientTracker{
		opts: r.opts,
		meta: reqMeta(req, r.opts, false),
	}
}

type clientTracker struct {
	opts *options
	*meta
}

func (t *clientTracker) RequestStarted() {
	clientStarted.WithLabelValues(t.name, t.handler, t.host, t.path, t.method).Inc()
}

func (t *clientTracker) RequestRead(duration time.Duration, size int) {
	if t.opts.sizes {
		clientRequestSize.WithLabelValues(t.name, t.handler, t.host, t.path, t.method).Observe(float64(size))
	}
}

func (t *clientTracker) ResponseStarted(duration time.Duration, code int, header http.Header) {
	status := strconv.Itoa(code)
	clientCompleted.WithLabelValues(t.name, t.handler, t.host, t.path, t.method, status).Inc()
	if t.opts.latency {
		clientLatency.WithLabelValues(t.name, t.handler, t.host, t.path, t.method, status).Observe(duration.Seconds())
	}
}

func (t *clientTracker) ResponseDone(duration time.Duration, code int, size int) {
	if t.opts.sizes {
		clientResponseSize.WithLabelValues(t.name, t.handler, t.host, t.path, t.method, strconv.Itoa(code)).Observe(float64(size))
	}
}
<|endoftext|>


<commit_before>package plugins

import (
	"strings"

	"time"

	"fmt"

	"github.com/Seklfreak/Robyul2/cache"
	"github.com/Seklfreak/Robyul2/helpers"
	"github.com/Seklfreak/Robyul2/models"
	"github.com/bwmarrin/discordgo"
	"github.com/dustin/go-humanize"
	"github.com/globalsign/mgo/bson"
	"github.com/sirupsen/logrus"
)

type botStatusAction func(args []string, in *discordgo.Message, out **discordgo.MessageSend) (next botStatusAction)

type BotStatus struct{}

func (bs *BotStatus) Commands() []string {
	return []string{
		"bot-status",
	}
}

func (bs *BotStatus) Init(session *discordgo.Session) {
	go func() {
		defer helpers.Recover()

		time.Sleep(time.Second * 60)
		go bs.gameStatusRotationLoop()
	}()
}

func (bs *BotStatus) gameStatusRotationLoop() {
	defer helpers.Recover()
	defer func() {
		go func() {
			bs.logger().Error("The gameStatusRotationLoop died. Please investigate! Will be restarted in 60 seconds")
			time.Sleep(60 * time.Second)
			bs.gameStatusRotationLoop()
		}()
	}()

	var err error
	var newStatus string
	for {
		var entryBucket models.BotStatusEntry
		// TODO: pipe aggregation
		err = helpers.MdbCollection(models.BotStatusTable).Pipe([]bson.M{{"$sample": bson.M{"size": 1}}}).One(&entryBucket)
		if err != nil {
			if !strings.Contains(err.Error(), "not found") {
				helpers.RelaxLog(err)
			}
			time.Sleep(60 * time.Second)
			continue
		}

		newStatus = bs.replaceText(entryBucket.Text)

		err = cache.GetSession().UpdateStatusComplex(discordgo.UpdateStatusData{
			Game: &discordgo.Game{
				Name: newStatus,
				Type: entryBucket.Type,
			},
			Status: "online",
		})
		helpers.RelaxLog(err)

		bs.logger().Infof("set the Bot Status to: \"%s\" using the rotation loop", newStatus)

		time.Sleep(45 * time.Minute)
	}
}

func (bs *BotStatus) Action(command string, content string, msg *discordgo.Message, session *discordgo.Session) {
	session.ChannelTyping(msg.ChannelID)

	var result *discordgo.MessageSend
	args := strings.Fields(content)

	action := bs.actionStart
	for action != nil {
		action = action(args, msg, &result)
	}
}

func (bs *BotStatus) actionStart(args []string, in *discordgo.Message, out **discordgo.MessageSend) botStatusAction {
	cache.GetSession().ChannelTyping(in.ChannelID)

	if len(args) < 1 {
		*out = bs.newMsg("bot.arguments.too-few")
		return bs.actionFinish
	}

	switch args[0] {
	case "add":
		return bs.actionAdd
	case "remove", "rem", "delete", "del":
		return bs.actionRemove
	case "set":
		return bs.actionSet
	case "list":
		return bs.actionList
	}

	*out = bs.newMsg("bot.arguments.invalid")
	return bs.actionFinish
}

func (bs *BotStatus) replaceText(text string) (result string) {
	users := make(map[string]string)
	channels := make(map[string]string)
	for _, guild := range cache.GetSession().State.Guilds {
		for _, u := range guild.Members {
			users[u.User.ID] = u.User.Username
		}
		for _, c := range guild.Channels {
			channels[c.ID] = c.Name
		}
	}

	text = strings.Replace(text, "{GUILD_COUNT}", humanize.Comma(int64(len(cache.GetSession().State.Guilds))), -1)
	text = strings.Replace(text, "{MEMBER_COUNT}", humanize.Comma(int64(len(users))), -1)
	text = strings.Replace(text, "{CHANNEL_COUNT}", humanize.Comma(int64(len(channels))), -1)

	return text
}

// [p]bot-status set <status text>
func (bs *BotStatus) actionSet(args []string, in *discordgo.Message, out **discordgo.MessageSend) botStatusAction {
	if !helpers.IsRobyulMod(in.Author.ID) {
		*out = bs.newMsg("robyulmod.no_permission")
		return bs.actionFinish
	}

	if len(args) <= 0 {
		*out = bs.newMsg("bot.arguments.too-few")
		return bs.actionFinish
	}

	parts := strings.Split(in.Content, args[0])
	if len(parts) < 2 {
		*out = bs.newMsg("bot.arguments.too-few")
		return bs.actionFinish
	}
	statusMessage := strings.TrimSpace(strings.Join(parts[1:], args[0]))

	newStatus := bs.replaceText(statusMessage)
	err := cache.GetSession().UpdateStatus(0, newStatus)
	helpers.Relax(err)

	bs.logger().WithField("UserID", in.Author.ID).Infof("Set the Bot Status to: \"%s\" using the set command", newStatus)

	*out = bs.newMsg(helpers.GetTextF("plugins.botstatus.set-success", newStatus))
	return bs.actionFinish
}

// [p]bot-status add <status text>
func (bs *BotStatus) actionAdd(args []string, in *discordgo.Message, out **discordgo.MessageSend) botStatusAction {
	if !helpers.IsRobyulMod(in.Author.ID) {
		*out = bs.newMsg("robyulmod.no_permission")
		return bs.actionFinish
	}

	if len(args) <= 0 {
		*out = bs.newMsg("bot.arguments.too-few")
		return bs.actionFinish
	}

	parts := strings.Split(in.Content, args[0])
	if len(parts) < 2 {
		*out = bs.newMsg("bot.arguments.too-few")
		return bs.actionFinish
	}
	statusMessage := strings.TrimSpace(strings.Join(parts[1:], args[0]))

	_, err := helpers.MDbInsert(
		models.BotStatusTable,
		models.BotStatusEntry{
			AddedByUserID: in.Author.ID,
			AddedAt:       time.Now(),
			Text:          statusMessage,
			Type:          discordgo.GameTypeGame,
		},
	)
	helpers.Relax(err)

	*out = bs.newMsg(helpers.GetTextF("plugins.botstatus.add-success", statusMessage))
	return bs.actionFinish
}

// [p]bot-status remove <status id>
func (bs *BotStatus) actionRemove(args []string, in *discordgo.Message, out **discordgo.MessageSend) botStatusAction {
	if !helpers.IsRobyulMod(in.Author.ID) {
		*out = bs.newMsg("robyulmod.no_permission")
		return bs.actionFinish
	}

	if len(args) < 2 {
		*out = bs.newMsg("bot.arguments.too-few")
		return bs.actionFinish
	}

	var entryBucket models.BotStatusEntry
	err := helpers.MdbOne(
		helpers.MdbCollection(models.BotStatusTable).Find(bson.M{"_id": helpers.HumanToMdbId(args[1])}),
		&entryBucket,
	)
	if err != nil {
		if !strings.Contains(err.Error(), "not found") {
			helpers.RelaxLog(err)
		}
		*out = bs.newMsg("bot.arguments.invalid")
		return bs.actionFinish
	}

	err = helpers.MDbDelete(models.BotStatusTable, entryBucket.ID)
	helpers.Relax(err)

	*out = bs.newMsg(helpers.GetTextF("plugins.botstatus.remove-success", entryBucket.Text))
	return bs.actionFinish
}

// [p]bot-status list
func (bs *BotStatus) actionList(args []string, in *discordgo.Message, out **discordgo.MessageSend) botStatusAction {
	if !helpers.IsRobyulMod(in.Author.ID) {
		*out = bs.newMsg("robyulmod.no_permission")
		return bs.actionFinish
	}

	var entryBucket []models.BotStatusEntry
	err := helpers.MDbIter(helpers.MdbCollection(models.BotStatusTable).Find(nil)).All(&entryBucket)
	helpers.Relax(err)

	if entryBucket == nil || len(entryBucket) <= 0 {
		*out = bs.newMsg("plugins.botstatus.list-empty")
		return bs.actionFinish
	}

	var message string
	for _, botStatus := range entryBucket {
		message += fmt.Sprintf("`%s`: `%s`\n", helpers.MdbIdToHuman(botStatus.ID), botStatus.Text)
	}
	message += fmt.Sprintf("_found %d statuses in total_\n", len(entryBucket))

	*out = bs.newMsg(message)
	return bs.actionFinish
}

func (bs *BotStatus) actionFinish(args []string, in *discordgo.Message, out **discordgo.MessageSend) botStatusAction {
	_, err := helpers.SendComplex(in.ChannelID, *out)
	helpers.Relax(err)

	return nil
}

func (bs *BotStatus) newMsg(content string) *discordgo.MessageSend {
	return &discordgo.MessageSend{Content: helpers.GetText(content)}
}

func (bs *BotStatus) logger() *logrus.Entry {
	return cache.GetLogger().WithField("module", "botstatus")
}
<commit_msg>[botstatus] adds spport for listening and watching 🎧<commit_after>package plugins

import (
	"strings"

	"time"

	"fmt"

	"github.com/Seklfreak/Robyul2/cache"
	"github.com/Seklfreak/Robyul2/helpers"
	"github.com/Seklfreak/Robyul2/models"
	"github.com/bwmarrin/discordgo"
	"github.com/dustin/go-humanize"
	"github.com/globalsign/mgo/bson"
	"github.com/sirupsen/logrus"
)

type botStatusAction func(args []string, in *discordgo.Message, out **discordgo.MessageSend) (next botStatusAction)

type BotStatus struct{}

func (bs *BotStatus) Commands() []string {
	return []string{
		"bot-status",
	}
}

func (bs *BotStatus) Init(session *discordgo.Session) {
	go func() {
		defer helpers.Recover()

		time.Sleep(time.Second * 60)
		go bs.gameStatusRotationLoop()
	}()
}

func (bs *BotStatus) gameStatusRotationLoop() {
	defer helpers.Recover()
	defer func() {
		go func() {
			bs.logger().Error("The gameStatusRotationLoop died. Please investigate! Will be restarted in 60 seconds")
			time.Sleep(60 * time.Second)
			bs.gameStatusRotationLoop()
		}()
	}()

	var err error
	var newStatus string
	for {
		var entryBucket models.BotStatusEntry
		// TODO: pipe aggregation
		err = helpers.MdbCollection(models.BotStatusTable).Pipe([]bson.M{{"$sample": bson.M{"size": 1}}}).One(&entryBucket)
		if err != nil {
			if !strings.Contains(err.Error(), "not found") {
				helpers.RelaxLog(err)
			}
			time.Sleep(60 * time.Second)
			continue
		}

		newStatus = bs.replaceText(entryBucket.Text)

		err = cache.GetSession().UpdateStatusComplex(discordgo.UpdateStatusData{
			Game: &discordgo.Game{
				Name: newStatus,
				Type: entryBucket.Type,
			},
			Status: "online",
		})
		helpers.RelaxLog(err)

		bs.logger().Infof("set the Bot Status to: \"%s\" using the rotation loop", newStatus)

		time.Sleep(45 * time.Minute)
	}
}

func (bs *BotStatus) Action(command string, content string, msg *discordgo.Message, session *discordgo.Session) {
	session.ChannelTyping(msg.ChannelID)

	var result *discordgo.MessageSend
	args := strings.Fields(content)

	action := bs.actionStart
	for action != nil {
		action = action(args, msg, &result)
	}
}

func (bs *BotStatus) actionStart(args []string, in *discordgo.Message, out **discordgo.MessageSend) botStatusAction {
	cache.GetSession().ChannelTyping(in.ChannelID)

	if len(args) < 1 {
		*out = bs.newMsg("bot.arguments.too-few")
		return bs.actionFinish
	}

	switch args[0] {
	case "add":
		return bs.actionAdd
	case "remove", "rem", "delete", "del":
		return bs.actionRemove
	case "set":
		return bs.actionSet
	case "list":
		return bs.actionList
	}

	*out = bs.newMsg("bot.arguments.invalid")
	return bs.actionFinish
}

func (bs *BotStatus) replaceText(text string) (result string) {
	users := make(map[string]string)
	channels := make(map[string]string)
	for _, guild := range cache.GetSession().State.Guilds {
		for _, u := range guild.Members {
			users[u.User.ID] = u.User.Username
		}
		for _, c := range guild.Channels {
			channels[c.ID] = c.Name
		}
	}

	text = strings.Replace(text, "{GUILD_COUNT}", humanize.Comma(int64(len(cache.GetSession().State.Guilds))), -1)
	text = strings.Replace(text, "{MEMBER_COUNT}", humanize.Comma(int64(len(users))), -1)
	text = strings.Replace(text, "{CHANNEL_COUNT}", humanize.Comma(int64(len(channels))), -1)

	return text
}

// [p]bot-status set <status text>
func (bs *BotStatus) actionSet(args []string, in *discordgo.Message, out **discordgo.MessageSend) botStatusAction {
	if !helpers.IsRobyulMod(in.Author.ID) {
		*out = bs.newMsg("robyulmod.no_permission")
		return bs.actionFinish
	}

	if len(args) < 3 {
		*out = bs.newMsg("bot.arguments.too-few")
		return bs.actionFinish
	}

	statusMessage := strings.TrimSpace(strings.Replace(strings.Join(args, " "), strings.Join(args[:2], " "), "", 1))
	statusType := bs.textToGameType(args[1])

	newStatus := bs.replaceText(statusMessage)

	err := cache.GetSession().UpdateStatusComplex(discordgo.UpdateStatusData{
		Game: &discordgo.Game{
			Name: newStatus,
			Type: statusType,
		},
		Status: "online",
	})
	helpers.Relax(err)

	bs.logger().WithField("UserID", in.Author.ID).Infof("Set the Bot Status to: \"%s\" using the set command", newStatus)

	*out = bs.newMsg(helpers.GetTextF("plugins.botstatus.set-success", newStatus))
	return bs.actionFinish
}

// [p]bot-status add <status text>
func (bs *BotStatus) actionAdd(args []string, in *discordgo.Message, out **discordgo.MessageSend) botStatusAction {
	if !helpers.IsRobyulMod(in.Author.ID) {
		*out = bs.newMsg("robyulmod.no_permission")
		return bs.actionFinish
	}

	if len(args) < 3 {
		*out = bs.newMsg("bot.arguments.too-few")
		return bs.actionFinish
	}

	statusMessage := strings.TrimSpace(strings.Replace(strings.Join(args, " "), strings.Join(args[:2], " "), "", 1))
	statusType := bs.textToGameType(args[1])

	_, err := helpers.MDbInsert(
		models.BotStatusTable,
		models.BotStatusEntry{
			AddedByUserID: in.Author.ID,
			AddedAt:       time.Now(),
			Text:          statusMessage,
			Type:          statusType,
		},
	)
	helpers.Relax(err)

	*out = bs.newMsg(helpers.GetTextF("plugins.botstatus.add-success", statusMessage))
	return bs.actionFinish
}

// [p]bot-status remove <status id>
func (bs *BotStatus) actionRemove(args []string, in *discordgo.Message, out **discordgo.MessageSend) botStatusAction {
	if !helpers.IsRobyulMod(in.Author.ID) {
		*out = bs.newMsg("robyulmod.no_permission")
		return bs.actionFinish
	}

	if len(args) < 2 {
		*out = bs.newMsg("bot.arguments.too-few")
		return bs.actionFinish
	}

	var entryBucket models.BotStatusEntry
	err := helpers.MdbOne(
		helpers.MdbCollection(models.BotStatusTable).Find(bson.M{"_id": helpers.HumanToMdbId(args[1])}),
		&entryBucket,
	)
	if err != nil {
		if !strings.Contains(err.Error(), "not found") {
			helpers.RelaxLog(err)
		}
		*out = bs.newMsg("bot.arguments.invalid")
		return bs.actionFinish
	}

	err = helpers.MDbDelete(models.BotStatusTable, entryBucket.ID)
	helpers.Relax(err)

	*out = bs.newMsg(helpers.GetTextF("plugins.botstatus.remove-success", entryBucket.Text))
	return bs.actionFinish
}

// [p]bot-status list
func (bs *BotStatus) actionList(args []string, in *discordgo.Message, out **discordgo.MessageSend) botStatusAction {
	if !helpers.IsRobyulMod(in.Author.ID) {
		*out = bs.newMsg("robyulmod.no_permission")
		return bs.actionFinish
	}

	var entryBucket []models.BotStatusEntry
	err := helpers.MDbIter(helpers.MdbCollection(models.BotStatusTable).Find(nil)).All(&entryBucket)
	helpers.Relax(err)

	if entryBucket == nil || len(entryBucket) <= 0 {
		*out = bs.newMsg("plugins.botstatus.list-empty")
		return bs.actionFinish
	}

	var message string
	for _, botStatus := range entryBucket {
		message += fmt.Sprintf("`%s`: %s `%s`\n",
			helpers.MdbIdToHuman(botStatus.ID), bs.gameTypeToText(botStatus.Type), botStatus.Text)
	}
	message += fmt.Sprintf("_found %d statuses in total_\n", len(entryBucket))

	*out = bs.newMsg(message)
	return bs.actionFinish
}

func (bs *BotStatus) textToGameType(text string) (gameType discordgo.GameType) {
	switch strings.ToLower(text) {
	case "playing", "game":
		return discordgo.GameTypeGame
	case "listening":
		return discordgo.GameTypeListening
	case "watching":
		return discordgo.GameTypeWatching
	}
	return discordgo.GameTypeGame
}

func (bs *BotStatus) gameTypeToText(gameType discordgo.GameType) (text string) {
	switch gameType {
	case discordgo.GameTypeGame:
		return "playing"
	case discordgo.GameTypeListening:
		return "listening to"
	case discordgo.GameTypeWatching:
		return "watching"
	}
	return ""
}

func (bs *BotStatus) actionFinish(args []string, in *discordgo.Message, out **discordgo.MessageSend) botStatusAction {
	_, err := helpers.SendComplex(in.ChannelID, *out)
	helpers.Relax(err)

	return nil
}

func (bs *BotStatus) newMsg(content string) *discordgo.MessageSend {
	return &discordgo.MessageSend{Content: helpers.GetText(content)}
}

func (bs *BotStatus) logger() *logrus.Entry {
	return cache.GetLogger().WithField("module", "botstatus")
}
<|endoftext|>


<commit_before>package triggers

type Donators struct{}

func (d *Donators) Triggers() []string {
    return []string{
        "donators",
        "donations",
        "donate",
        "supporters",
        "support",
        "patreon",
        "patreons",
        "credits",
    }
}

func (d *Donators) Response(trigger string, content string) string {
    return "<:robyulblush:327206930437373952> **These awesome people support me:**\nKakkela 💕\nSunny 💓\nsomicidal minaiac 💞\nOokami 💖\nKeldra 💗\nTN 💝\nseulguille 💘\nSlenn 💜\nFugu ❣\nWoori 💞\nhikari 💙\nAshton 💖\nKay 💝\nThank you so much!\n_You want to be in this list? <https://www.patreon.com/sekl>!_"
}
<commit_msg>[donators] adds jamie!<commit_after>package triggers

type Donators struct{}

func (d *Donators) Triggers() []string {
    return []string{
        "donators",
        "donations",
        "donate",
        "supporters",
        "support",
        "patreon",
        "patreons",
        "credits",
    }
}

func (d *Donators) Response(trigger string, content string) string {
    return "<:robyulblush:327206930437373952> **These awesome people support me:**\nKakkela 💕\nSunny 💓\nsomicidal minaiac 💞\nOokami 💖\nKeldra 💗\nTN 💝\nseulguille 💘\nSlenn 💜\nFugu ❣\nWoori 💞\nhikari 💙\nAshton 💖\nKay 💝\njamie 💓\nThank you so much!\n_You want to be in this list? <https://www.patreon.com/sekl>!_"
}
<|endoftext|>


<commit_before>package v1batch

import (
	"fmt"
	"net/http"
	"strconv"
	"time"

	v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload"
)

//ClientTaskInterface is an interface so client task calls can be mocked.
type ClientTaskInterface interface {
	StartTask(projectID, queueID, payload string) (output *v1payload.StartTaskOutput, err error)
	StopTask(projectID, queueID string, taskID int64) (output *v1payload.StopTaskOutput, err error)
	ListTasks(projectID, queueID string) (output *v1payload.ListTasksOutput, err error)
	DescribeTask(projectID, queueID string, taskID int64) (output *v1payload.DescribeTaskOutput, err error)
	ReceiveTaskRuns(projectID, queueID string, timeout time.Duration, queueOps QueueOps) (output []*v1payload.Run, err error)
}

// QueueOps is an interface that includes queue operations.
type QueueOps interface {
	ReceiveMessages(queueURL string, maxNoOfMessages, waitTimeSeconds int64) (messages []interface{}, err error)
	UnmarshalMessage(message interface{}, v interface{}) error
	DeleteMessage(queueURL string, message interface{}) error
}

//DescribeTask will create an execute a new task
func (c *Client) DescribeTask(projectID, queueID string, taskID int64) (output *v1payload.DescribeTaskOutput, err error) {
	output = &v1payload.DescribeTaskOutput{}
	input := &v1payload.DescribeTaskInput{
		ProjectID: projectID,
		QueueID:   queueID,
		TaskID:    taskID,
	}

	return output, c.doRequest(http.MethodGet, createPath(projectID, queuesEndpoint, queueID, "tasks", strconv.FormatInt(taskID, 10)), input, output)
}

//StartTask will create an execute a new task
func (c *Client) StartTask(projectID, queueID, payload string) (output *v1payload.StartTaskOutput, err error) {
	output = &v1payload.StartTaskOutput{}
	input := &v1payload.StartTaskInput{
		QueueID:   queueID,
		ProjectID: projectID,
		Payload:   payload,
	}

	return output, c.doRequest(http.MethodPost, createPath(projectID, queuesEndpoint, queueID, "tasks"), input, output)
}

//StopTask will create queue
func (c *Client) StopTask(projectID, queueID string, taskID int64) (output *v1payload.StopTaskOutput, err error) {
	output = &v1payload.StopTaskOutput{}
	input := &v1payload.StopTaskInput{
		ProjectID: projectID,
		QueueID:   queueID,
		TaskID:    taskID,
	}

	return output, c.doRequest(http.MethodDelete, createPath(projectID, queuesEndpoint, queueID, "tasks"), input, output)
}

// ListTasks will return all tasks in a queue
func (c *Client) ListTasks(projectID, queueID string) (output *v1payload.ListTasksOutput, err error) {
	output = &v1payload.ListTasksOutput{}
	input := &v1payload.ListTasksInput{
		ProjectID: projectID,
		QueueID:   queueID,
	}

	return output, c.doRequest(http.MethodGet, createPath(projectID, queuesEndpoint, queueID, "tasks"), input, output)
}

//ReceiveTaskRuns will long poll the aws sqs queue for the availability of new runs. It will receive and delete messages once decoded
func (c *Client) ReceiveTaskRuns(projectID, queueID string, timeout time.Duration, queueOps QueueOps) (output []*v1payload.Run, err error) {
	queue, err := c.DescribeQueue(projectID, queueID)
	if err != nil {
		return nil, fmt.Errorf("failed to describe queue: %+v", err)
	}

	toCh := time.After(timeout)
	for {
		select {
		case <-toCh:
			return output, nil
		default:
		}

		out, err := queueOps.ReceiveMessages(queue.QueueURL, 1, 5)
		if err != nil {
			return nil, fmt.Errorf("failed to receive runs: %+v", err)
		}

		for _, msg := range out {
			r := &v1payload.Run{}
			err = queueOps.UnmarshalMessage(msg, r)
			if err != nil {
				return nil, fmt.Errorf("failed to decode message: %+v", err)
			}

			if err = queueOps.DeleteMessage(queue.QueueURL, msg); err != nil {
				return nil, fmt.Errorf("failed to receive runs: %+v", err)
			}

			_, err = c.SendRunHeartbeat(r.ProjectID, r.QueueID, r.TaskID, r.Token)
			if err != nil {
				continue //we will not consider this run at all, it must be expired
			}

			output = append(output, r)
			return output, nil
		}
	}
}
<commit_msg>fixed task stop endpoint<commit_after>package v1batch

import (
	"fmt"
	"net/http"
	"strconv"
	"time"

	v1payload "github.com/nerdalize/nerd/nerd/client/batch/v1/payload"
)

//ClientTaskInterface is an interface so client task calls can be mocked.
type ClientTaskInterface interface {
	StartTask(projectID, queueID, payload string) (output *v1payload.StartTaskOutput, err error)
	StopTask(projectID, queueID string, taskID int64) (output *v1payload.StopTaskOutput, err error)
	ListTasks(projectID, queueID string) (output *v1payload.ListTasksOutput, err error)
	DescribeTask(projectID, queueID string, taskID int64) (output *v1payload.DescribeTaskOutput, err error)
	ReceiveTaskRuns(projectID, queueID string, timeout time.Duration, queueOps QueueOps) (output []*v1payload.Run, err error)
}

// QueueOps is an interface that includes queue operations.
type QueueOps interface {
	ReceiveMessages(queueURL string, maxNoOfMessages, waitTimeSeconds int64) (messages []interface{}, err error)
	UnmarshalMessage(message interface{}, v interface{}) error
	DeleteMessage(queueURL string, message interface{}) error
}

//DescribeTask will create an execute a new task
func (c *Client) DescribeTask(projectID, queueID string, taskID int64) (output *v1payload.DescribeTaskOutput, err error) {
	output = &v1payload.DescribeTaskOutput{}
	input := &v1payload.DescribeTaskInput{
		ProjectID: projectID,
		QueueID:   queueID,
		TaskID:    taskID,
	}

	return output, c.doRequest(http.MethodGet, createPath(projectID, queuesEndpoint, queueID, "tasks", strconv.FormatInt(taskID, 10)), input, output)
}

//StartTask will create an execute a new task
func (c *Client) StartTask(projectID, queueID, payload string) (output *v1payload.StartTaskOutput, err error) {
	output = &v1payload.StartTaskOutput{}
	input := &v1payload.StartTaskInput{
		QueueID:   queueID,
		ProjectID: projectID,
		Payload:   payload,
	}

	return output, c.doRequest(http.MethodPost, createPath(projectID, queuesEndpoint, queueID, "tasks"), input, output)
}

//StopTask will create queue
func (c *Client) StopTask(projectID, queueID string, taskID int64) (output *v1payload.StopTaskOutput, err error) {
	output = &v1payload.StopTaskOutput{}
	input := &v1payload.StopTaskInput{
		ProjectID: projectID,
		QueueID:   queueID,
		TaskID:    taskID,
	}

	return output, c.doRequest(http.MethodDelete, createPath(projectID, queuesEndpoint, queueID, "tasks", strconv.FormatInt(taskID, 10)), input, output)
}

// ListTasks will return all tasks in a queue
func (c *Client) ListTasks(projectID, queueID string) (output *v1payload.ListTasksOutput, err error) {
	output = &v1payload.ListTasksOutput{}
	input := &v1payload.ListTasksInput{
		ProjectID: projectID,
		QueueID:   queueID,
	}

	return output, c.doRequest(http.MethodGet, createPath(projectID, queuesEndpoint, queueID, "tasks"), input, output)
}

//ReceiveTaskRuns will long poll the aws sqs queue for the availability of new runs. It will receive and delete messages once decoded
func (c *Client) ReceiveTaskRuns(projectID, queueID string, timeout time.Duration, queueOps QueueOps) (output []*v1payload.Run, err error) {
	queue, err := c.DescribeQueue(projectID, queueID)
	if err != nil {
		return nil, fmt.Errorf("failed to describe queue: %+v", err)
	}

	toCh := time.After(timeout)
	for {
		select {
		case <-toCh:
			return output, nil
		default:
		}

		out, err := queueOps.ReceiveMessages(queue.QueueURL, 1, 5)
		if err != nil {
			return nil, fmt.Errorf("failed to receive runs: %+v", err)
		}

		for _, msg := range out {
			r := &v1payload.Run{}
			err = queueOps.UnmarshalMessage(msg, r)
			if err != nil {
				return nil, fmt.Errorf("failed to decode message: %+v", err)
			}

			if err = queueOps.DeleteMessage(queue.QueueURL, msg); err != nil {
				return nil, fmt.Errorf("failed to receive runs: %+v", err)
			}

			_, err = c.SendRunHeartbeat(r.ProjectID, r.QueueID, r.TaskID, r.Token)
			if err != nil {
				continue //we will not consider this run at all, it must be expired
			}

			output = append(output, r)
			return output, nil
		}
	}
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package libsnnet

import (
	"fmt"
	"math/rand"
	"net"
	"strings"
	"time"

	"github.com/vishvananda/netlink"
)

const (
	prefixBridge   = "sbr"
	prefixVnic     = "svn"
	prefixVnicCont = "svp"
	prefixVnicHost = "svn"
	prefixCnciVnic = "svc"
	prefixGretap   = "sgt"
)

const ifaceRetryLimit = 10

var (
	ifaceRseed rand.Source
	ifaceRsrc  *rand.Rand
)

func init() {
	ifaceRseed = rand.NewSource(time.Now().UnixNano())
	ifaceRsrc = rand.New(ifaceRseed)
}

func validSnPrefix(s string) bool {
	switch {
	case strings.HasPrefix(s, prefixBridge):
	case strings.HasPrefix(s, prefixVnic):
	case strings.HasPrefix(s, prefixVnicCont):
	case strings.HasPrefix(s, prefixVnicHost):
	case strings.HasPrefix(s, prefixCnciVnic):
	case strings.HasPrefix(s, prefixGretap):
	default:
		return false
	}

	return true
}

func getPrefix(device interface{}) (string, error) {

	prefix := ""

	switch d := device.(type) {
	case *Bridge:
		prefix = prefixBridge
	case *Vnic:
		switch d.Role {
		case TenantVM:
			prefix = prefixVnic
		case TenantContainer:
			prefix = prefixVnicHost
		}
	case *GreTunEP:
		prefix = prefixGretap
	case *CnciVnic:
		prefix = prefixCnciVnic
	}

	if prefix == "" {
		return prefix, fmt.Errorf("invalid device type %T %v", device, device)
	}

	return prefix, nil

}

// GenIface generates locally unique interface names based on the
// type of device passed in. It will additionally check if the
// interface name exists on the localhost based on unique
// When uniqueness is specified error will be returned
// if it is not possible to generate a locally unique name within
// a finite number of retries
func genIface(device interface{}, unique bool) (string, error) {

	prefix, err := getPrefix(device)
	if err != nil {
		return "", err
	}

	if !unique {
		iface := fmt.Sprintf("%s_%x", prefix, ifaceRsrc.Uint32())
		return iface, nil
	}

	for i := 0; i < ifaceRetryLimit; i++ {
		iface := fmt.Sprintf("%s_%x", prefix, ifaceRsrc.Uint32())
		if _, err := netlink.LinkByName(iface); err != nil {
			return iface, nil
		}
	}

	// The chances of the failure are remote
	return "", fmt.Errorf("unable to create unique interface name")
}

func validPhysicalLink(link netlink.Link) bool {
	phyDevice := true

	switch link.Type() {
	case "device":
	case "bond":
	case "vlan":
	case "bridge":
	default:
		phyDevice = false
	}

	if (link.Attrs().Flags & net.FlagLoopback) != 0 {
		return false
	}

	if travisCI {
		return true
	}
	return phyDevice
}
<commit_msg>Networking: Exclude docker bridges in auto-configuration<commit_after>//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package libsnnet

import (
	"fmt"
	"math/rand"
	"net"
	"strings"
	"time"

	"github.com/vishvananda/netlink"
)

const (
	prefixBridge   = "sbr"
	prefixVnic     = "svn"
	prefixVnicCont = "svp"
	prefixVnicHost = "svn"
	prefixCnciVnic = "svc"
	prefixGretap   = "sgt"
)

const ifaceRetryLimit = 10

var (
	ifaceRseed rand.Source
	ifaceRsrc  *rand.Rand
)

func init() {
	ifaceRseed = rand.NewSource(time.Now().UnixNano())
	ifaceRsrc = rand.New(ifaceRseed)
}

func validSnPrefix(s string) bool {
	switch {
	case strings.HasPrefix(s, prefixBridge):
	case strings.HasPrefix(s, prefixVnic):
	case strings.HasPrefix(s, prefixVnicCont):
	case strings.HasPrefix(s, prefixVnicHost):
	case strings.HasPrefix(s, prefixCnciVnic):
	case strings.HasPrefix(s, prefixGretap):
	default:
		return false
	}

	return true
}

func getPrefix(device interface{}) (string, error) {

	prefix := ""

	switch d := device.(type) {
	case *Bridge:
		prefix = prefixBridge
	case *Vnic:
		switch d.Role {
		case TenantVM:
			prefix = prefixVnic
		case TenantContainer:
			prefix = prefixVnicHost
		}
	case *GreTunEP:
		prefix = prefixGretap
	case *CnciVnic:
		prefix = prefixCnciVnic
	}

	if prefix == "" {
		return prefix, fmt.Errorf("invalid device type %T %v", device, device)
	}

	return prefix, nil

}

// GenIface generates locally unique interface names based on the
// type of device passed in. It will additionally check if the
// interface name exists on the localhost based on unique
// When uniqueness is specified error will be returned
// if it is not possible to generate a locally unique name within
// a finite number of retries
func genIface(device interface{}, unique bool) (string, error) {

	prefix, err := getPrefix(device)
	if err != nil {
		return "", err
	}

	if !unique {
		iface := fmt.Sprintf("%s_%x", prefix, ifaceRsrc.Uint32())
		return iface, nil
	}

	for i := 0; i < ifaceRetryLimit; i++ {
		iface := fmt.Sprintf("%s_%x", prefix, ifaceRsrc.Uint32())
		if _, err := netlink.LinkByName(iface); err != nil {
			return iface, nil
		}
	}

	// The chances of the failure are remote
	return "", fmt.Errorf("unable to create unique interface name")
}

func validPhysicalLink(link netlink.Link) bool {
	phyDevice := true

	switch link.Type() {
	case "device":
	case "bond":
	case "vlan":
	case "bridge":
		if strings.HasPrefix(link.Attrs().Name, "docker") {
			phyDevice = false
		}
	default:
		phyDevice = false
	}

	if (link.Attrs().Flags & net.FlagLoopback) != 0 {
		return false
	}

	if travisCI {
		return true
	}
	return phyDevice
}
<|endoftext|>


<commit_before>package nomad

import (
	"fmt"
	"math/rand"
	"time"

	metrics "github.com/armon/go-metrics"
	log "github.com/hashicorp/go-hclog"
	memdb "github.com/hashicorp/go-memdb"
	cstructs "github.com/hashicorp/nomad/client/structs"
)

// ClientCSI is used to forward RPC requests to the targed Nomad client's
// CSIController endpoint.
type ClientCSI struct {
	srv    *Server
	logger log.Logger
}

func (a *ClientCSI) ControllerAttachVolume(args *cstructs.ClientCSIControllerAttachVolumeRequest, reply *cstructs.ClientCSIControllerAttachVolumeResponse) error {
	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "attach_volume"}, time.Now())
	// Get a Nomad client node for the controller
	nodeID, err := a.nodeForController(args.PluginID, args.ControllerNodeID)
	if err != nil {
		return err
	}
	args.ControllerNodeID = nodeID

	// Get the connection to the client
	state, ok := a.srv.getNodeConn(args.ControllerNodeID)
	if !ok {
		return findNodeConnAndForward(a.srv, args.ControllerNodeID, "ClientCSI.ControllerAttachVolume", args, reply)
	}

	// Make the RPC
	err = NodeRpc(state.Session, "CSI.ControllerAttachVolume", args, reply)
	if err != nil {
		return fmt.Errorf("controller attach volume: %v", err)
	}
	return nil
}

func (a *ClientCSI) ControllerValidateVolume(args *cstructs.ClientCSIControllerValidateVolumeRequest, reply *cstructs.ClientCSIControllerValidateVolumeResponse) error {
	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "validate_volume"}, time.Now())

	// Get a Nomad client node for the controller
	nodeID, err := a.nodeForController(args.PluginID, args.ControllerNodeID)
	if err != nil {
		return err
	}
	args.ControllerNodeID = nodeID

	// Get the connection to the client
	state, ok := a.srv.getNodeConn(args.ControllerNodeID)
	if !ok {
		return findNodeConnAndForward(a.srv, args.ControllerNodeID, "ClientCSI.ControllerValidateVolume", args, reply)
	}

	// Make the RPC
	err = NodeRpc(state.Session, "CSI.ControllerValidateVolume", args, reply)
	if err != nil {
		return fmt.Errorf("validate volume: %v", err)
	}
	return nil
}

func (a *ClientCSI) ControllerDetachVolume(args *cstructs.ClientCSIControllerDetachVolumeRequest, reply *cstructs.ClientCSIControllerDetachVolumeResponse) error {
	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "detach_volume"}, time.Now())

	// Get a Nomad client node for the controller
	nodeID, err := a.nodeForController(args.PluginID, args.ControllerNodeID)
	if err != nil {
		return err
	}
	args.ControllerNodeID = nodeID

	// Get the connection to the client
	state, ok := a.srv.getNodeConn(args.ControllerNodeID)
	if !ok {
		return findNodeConnAndForward(a.srv, args.ControllerNodeID, "ClientCSI.ControllerDetachVolume", args, reply)
	}

	// Make the RPC
	err = NodeRpc(state.Session, "CSI.ControllerDetachVolume", args, reply)
	if err != nil {
		return fmt.Errorf("controller detach volume: %v", err)
	}
	return nil

}

func (a *ClientCSI) NodeDetachVolume(args *cstructs.ClientCSINodeDetachVolumeRequest, reply *cstructs.ClientCSINodeDetachVolumeResponse) error {
	defer metrics.MeasureSince([]string{"nomad", "client_csi_node", "detach_volume"}, time.Now())

	// Make sure Node is valid and new enough to support RPC
	snap, err := a.srv.State().Snapshot()
	if err != nil {
		return err
	}

	_, err = getNodeForRpc(snap, args.NodeID)
	if err != nil {
		return err
	}

	// Get the connection to the client
	state, ok := a.srv.getNodeConn(args.NodeID)
	if !ok {
		return findNodeConnAndForward(a.srv, args.NodeID, "ClientCSI.NodeDetachVolume", args, reply)
	}

	// Make the RPC
	err = NodeRpc(state.Session, "CSI.NodeDetachVolume", args, reply)
	if err != nil {
		return fmt.Errorf("node detach volume: %v", err)
	}
	return nil

}

// nodeForController validates that the Nomad client node ID for
// a plugin exists and is new enough to support client RPC. If no node
// ID is passed, select a random node ID for the controller to load-balance
// long blocking RPCs across client nodes.
func (a *ClientCSI) nodeForController(pluginID, nodeID string) (string, error) {

	snap, err := a.srv.State().Snapshot()
	if err != nil {
		return "", err
	}

	if nodeID != "" {
		_, err = getNodeForRpc(snap, nodeID)
		if err == nil {
			return nodeID, nil
		} else {
			// we'll fall-through and select a node at random
			a.logger.Trace("%s could not be used for client RPC: %v", nodeID, err)
		}
	}

	if pluginID == "" {
		return "", fmt.Errorf("missing plugin ID")
	}

	ws := memdb.NewWatchSet()

	// note: plugin IDs are not scoped to region/DC but volumes are.
	// so any node we get for a controller is already in the same
	// region/DC for the volume.
	plugin, err := snap.CSIPluginByID(ws, pluginID)
	if err != nil {
		return "", fmt.Errorf("error getting plugin: %s, %v", pluginID, err)
	}
	if plugin == nil {
		return "", fmt.Errorf("plugin missing: %s %v", pluginID, err)
	}
	count := len(plugin.Controllers)
	if count == 0 {
		return "", fmt.Errorf("no controllers available for plugin %q", plugin.ID)
	}

	// iterating maps is "random" but unspecified and isn't particularly
	// random with small maps, so not well-suited for load balancing.
	// so we shuffle the keys and iterate over them.
	clientIDs := make([]string, 0, count)
	for clientID := range plugin.Controllers {
		clientIDs = append(clientIDs, clientID)
	}
	rand.Shuffle(count, func(i, j int) {
		clientIDs[i], clientIDs[j] = clientIDs[j], clientIDs[i]
	})

	for _, clientID := range clientIDs {
		controller := plugin.Controllers[clientID]
		if !controller.IsController() {
			// we don't have separate types for CSIInfo depending on
			// whether it's a controller or node. this error shouldn't
			// make it to production but is to aid developers during
			// development
			err = fmt.Errorf("plugin is not a controller")
			continue
		}
		_, err = getNodeForRpc(snap, clientID)
		if err != nil {
			continue
		}
		return clientID, nil
	}

	return "", err
}
<commit_msg>fix a trace logline<commit_after>package nomad

import (
	"fmt"
	"math/rand"
	"time"

	metrics "github.com/armon/go-metrics"
	log "github.com/hashicorp/go-hclog"
	memdb "github.com/hashicorp/go-memdb"
	cstructs "github.com/hashicorp/nomad/client/structs"
)

// ClientCSI is used to forward RPC requests to the targed Nomad client's
// CSIController endpoint.
type ClientCSI struct {
	srv    *Server
	logger log.Logger
}

func (a *ClientCSI) ControllerAttachVolume(args *cstructs.ClientCSIControllerAttachVolumeRequest, reply *cstructs.ClientCSIControllerAttachVolumeResponse) error {
	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "attach_volume"}, time.Now())
	// Get a Nomad client node for the controller
	nodeID, err := a.nodeForController(args.PluginID, args.ControllerNodeID)
	if err != nil {
		return err
	}
	args.ControllerNodeID = nodeID

	// Get the connection to the client
	state, ok := a.srv.getNodeConn(args.ControllerNodeID)
	if !ok {
		return findNodeConnAndForward(a.srv, args.ControllerNodeID, "ClientCSI.ControllerAttachVolume", args, reply)
	}

	// Make the RPC
	err = NodeRpc(state.Session, "CSI.ControllerAttachVolume", args, reply)
	if err != nil {
		return fmt.Errorf("controller attach volume: %v", err)
	}
	return nil
}

func (a *ClientCSI) ControllerValidateVolume(args *cstructs.ClientCSIControllerValidateVolumeRequest, reply *cstructs.ClientCSIControllerValidateVolumeResponse) error {
	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "validate_volume"}, time.Now())

	// Get a Nomad client node for the controller
	nodeID, err := a.nodeForController(args.PluginID, args.ControllerNodeID)
	if err != nil {
		return err
	}
	args.ControllerNodeID = nodeID

	// Get the connection to the client
	state, ok := a.srv.getNodeConn(args.ControllerNodeID)
	if !ok {
		return findNodeConnAndForward(a.srv, args.ControllerNodeID, "ClientCSI.ControllerValidateVolume", args, reply)
	}

	// Make the RPC
	err = NodeRpc(state.Session, "CSI.ControllerValidateVolume", args, reply)
	if err != nil {
		return fmt.Errorf("validate volume: %v", err)
	}
	return nil
}

func (a *ClientCSI) ControllerDetachVolume(args *cstructs.ClientCSIControllerDetachVolumeRequest, reply *cstructs.ClientCSIControllerDetachVolumeResponse) error {
	defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "detach_volume"}, time.Now())

	// Get a Nomad client node for the controller
	nodeID, err := a.nodeForController(args.PluginID, args.ControllerNodeID)
	if err != nil {
		return err
	}
	args.ControllerNodeID = nodeID

	// Get the connection to the client
	state, ok := a.srv.getNodeConn(args.ControllerNodeID)
	if !ok {
		return findNodeConnAndForward(a.srv, args.ControllerNodeID, "ClientCSI.ControllerDetachVolume", args, reply)
	}

	// Make the RPC
	err = NodeRpc(state.Session, "CSI.ControllerDetachVolume", args, reply)
	if err != nil {
		return fmt.Errorf("controller detach volume: %v", err)
	}
	return nil

}

func (a *ClientCSI) NodeDetachVolume(args *cstructs.ClientCSINodeDetachVolumeRequest, reply *cstructs.ClientCSINodeDetachVolumeResponse) error {
	defer metrics.MeasureSince([]string{"nomad", "client_csi_node", "detach_volume"}, time.Now())

	// Make sure Node is valid and new enough to support RPC
	snap, err := a.srv.State().Snapshot()
	if err != nil {
		return err
	}

	_, err = getNodeForRpc(snap, args.NodeID)
	if err != nil {
		return err
	}

	// Get the connection to the client
	state, ok := a.srv.getNodeConn(args.NodeID)
	if !ok {
		return findNodeConnAndForward(a.srv, args.NodeID, "ClientCSI.NodeDetachVolume", args, reply)
	}

	// Make the RPC
	err = NodeRpc(state.Session, "CSI.NodeDetachVolume", args, reply)
	if err != nil {
		return fmt.Errorf("node detach volume: %v", err)
	}
	return nil

}

// nodeForController validates that the Nomad client node ID for
// a plugin exists and is new enough to support client RPC. If no node
// ID is passed, select a random node ID for the controller to load-balance
// long blocking RPCs across client nodes.
func (a *ClientCSI) nodeForController(pluginID, nodeID string) (string, error) {

	snap, err := a.srv.State().Snapshot()
	if err != nil {
		return "", err
	}

	if nodeID != "" {
		_, err = getNodeForRpc(snap, nodeID)
		if err == nil {
			return nodeID, nil
		} else {
			// we'll fall-through and select a node at random
			a.logger.Trace("could not be used for client RPC", "node", nodeID, "error", err)
		}
	}

	if pluginID == "" {
		return "", fmt.Errorf("missing plugin ID")
	}

	ws := memdb.NewWatchSet()

	// note: plugin IDs are not scoped to region/DC but volumes are.
	// so any node we get for a controller is already in the same
	// region/DC for the volume.
	plugin, err := snap.CSIPluginByID(ws, pluginID)
	if err != nil {
		return "", fmt.Errorf("error getting plugin: %s, %v", pluginID, err)
	}
	if plugin == nil {
		return "", fmt.Errorf("plugin missing: %s %v", pluginID, err)
	}
	count := len(plugin.Controllers)
	if count == 0 {
		return "", fmt.Errorf("no controllers available for plugin %q", plugin.ID)
	}

	// iterating maps is "random" but unspecified and isn't particularly
	// random with small maps, so not well-suited for load balancing.
	// so we shuffle the keys and iterate over them.
	clientIDs := make([]string, 0, count)
	for clientID := range plugin.Controllers {
		clientIDs = append(clientIDs, clientID)
	}
	rand.Shuffle(count, func(i, j int) {
		clientIDs[i], clientIDs[j] = clientIDs[j], clientIDs[i]
	})

	for _, clientID := range clientIDs {
		controller := plugin.Controllers[clientID]
		if !controller.IsController() {
			// we don't have separate types for CSIInfo depending on
			// whether it's a controller or node. this error shouldn't
			// make it to production but is to aid developers during
			// development
			err = fmt.Errorf("plugin is not a controller")
			continue
		}
		_, err = getNodeForRpc(snap, clientID)
		if err != nil {
			continue
		}
		return clientID, nil
	}

	return "", err
}
<|endoftext|>


<commit_before>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package num

import (
	"testing"

	"gosl/chk"
)

func TestNlsConfig01(tst *testing.T) {

	// verbose()
	chk.PrintTitle("NlsConfig01")

	c := NewNlSolverConfig()

	// flags
	res := []bool{
		c.Verbose,
		c.ConstantJacobian,
		c.LineSearch,
		c.EnforceConvRate,
		c.useDenseSolver,
		c.hasJacobianFunction,
		c.LinSolConfig.Symmetric,
		c.LinSolConfig.SymPosDef,
		c.LinSolConfig.Verbose,
	}
	correct := []bool{false, false, false, false, false, false, false, false, false}
	chk.Bools(tst, "flags", res, correct)

	// tolerances
	chk.Float64(tst, "atol", 1e-15, c.atol, 1e-8)
	chk.Float64(tst, "rtol", 1e-15, c.rtol, 1e-8)
	chk.Float64(tst, "ftol", 1e-15, c.ftol, 1e-9)
	chk.Float64(tst, "fnewt", 1e-15, c.fnewt, 0.0001)
}
<commit_msg>Fix test in num about la flags<commit_after>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package num

import (
	"testing"

	"gosl/chk"
)

func TestNlsConfig01(tst *testing.T) {

	// verbose()
	chk.PrintTitle("NlsConfig01")

	c := NewNlSolverConfig()

	// flags
	res := []bool{
		c.Verbose,
		c.ConstantJacobian,
		c.LineSearch,
		c.EnforceConvRate,
		c.useDenseSolver,
		c.hasJacobianFunction,
		c.LinSolConfig.Verbose,
	}
	correct := []bool{false, false, false, false, false, false, false}
	chk.Bools(tst, "flags", res, correct)

	// tolerances
	chk.Float64(tst, "atol", 1e-15, c.atol, 1e-8)
	chk.Float64(tst, "rtol", 1e-15, c.rtol, 1e-8)
	chk.Float64(tst, "ftol", 1e-15, c.ftol, 1e-9)
	chk.Float64(tst, "fnewt", 1e-15, c.fnewt, 0.0001)
}
<|endoftext|>