<commit_before>// The matrix package contains various utilities for dealing with raw matrices.
// The interface is loosely based on the NumPy package in Python.
//
// The key interfaces here are:
// * NDArray – A multidimensional array, with dense and (2D-only) sparse
//             implementations.
// * Matrix – A two-dimensional array, with various methods only available when
//            working in two dimensions.  A two-dimensional NDArray can be
//            trivially converted to the Matrix type by calling arr.M().
//
// When possible, function implementations take advantage of matrix sparsity.
// For instance, MProd(), the matrix multiplication function, performs the
// minimum amount of work required based on the types of its arguments.
//
// Certain linear algebra methods, particularly in the Matrix interface, rely
// on BLAS. In order to use it, you will need to register an appropriate engine.
// See the documentation at https://github.com/gonum/blas for details. You can
// register a default (native Go) engine by calling InitDefaultBlas().
// If you see a panic message like
// "mat64: no blas engine registered: call Register()"
// then you need to register a BLAS engine. If you don't see this error, then
// you probably don't need to worry about it.
package matrix

import (
	"fmt"
	"math/rand"
	"time"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

// ArraySparsity indicates the representation type of the matrix
type ArraySparsity int

const (
	DenseArray ArraySparsity = iota
	SparseCooMatrix
	SparseDiagMatrix
)

// A NDArray is an n-dimensional array of numbers which can be manipulated in
// various ways. Concrete implementations can differ; for instance, sparse
// and dense representations are possible.
type NDArray interface {

	// Return the element-wise sum of this array and one or more others
	Add(others ...NDArray) NDArray

	// Returns true if and only if all items are nonzero
	All() bool

	// Returns true if f is true for all array elements
	AllF(f func(v float64) bool) bool

	// Returns true if f is true for all pairs of array elements in the same position
	AllF2(f func(v1, v2 float64) bool, other NDArray) bool

	// Returns true if and only if any item is nonzero
	Any() bool

	// Returns true if f is true for any array element
	AnyF(f func(v float64) bool) bool

	// Returns true if f is true for any pair of array elements in the same position
	AnyF2(f func(v1, v2 float64) bool, other NDArray) bool

	// Return the result of applying a function to all elements
	Apply(f func(float64) float64) NDArray

	// Get the matrix data as a flattened 1D array; sparse matrices will make
	// a copy first.
	Array() []float64

	// Create a new array by concatenating this with another array along the
	// specified axis. The array shapes must be equal along all other axes.
	// It is legal to add a new axis.
	Concat(axis int, others ...NDArray) NDArray

	// Returns a duplicate of this array
	Copy() NDArray

	// Counts the number of nonzero elements in the array
	CountNonzero() int

	// Returns a dense copy of the array
	Dense() NDArray

	// Return the element-wise quotient of this array and one or more others.
	// This function defines 0 / 0 = 0, so it's useful for sparse arrays.
	Div(others ...NDArray) NDArray

	// Returns true if and only if all elements in the two arrays are equal
	Equal(other NDArray) bool

	// Set all array elements to the given value
	Fill(value float64)

	// Get an array element in a flattened verison of this array
	FlatItem(index int) float64

	// Set an array element in a flattened version of this array
	FlatItemSet(value float64, index int)

	// Return an iterator over populated matrix entries
	FlatIter() FlatNDArrayIterator

	// Get an array element
	Item(index ...int) float64

	// Return the result of adding a scalar value to each array element
	ItemAdd(value float64) NDArray

	// Return the result of dividing each array element by a scalar value
	ItemDiv(value float64) NDArray

	// Return the reuslt of multiplying each array element by a scalar value
	ItemProd(value float64) NDArray

	// Return the result of subtracting a scalar value from each array element
	ItemSub(value float64) NDArray

	// Set an array element
	ItemSet(value float64, index ...int)

	// Return an iterator over populated matrix entries
	Iter() CoordNDArrayIterator

	// Returns the array as a matrix. This is only possible for 1D and 2D arrays;
	// 1D arrays of length n are converted into n x 1 vectors.
	M() Matrix

	// Get the value of the largest array element
	Max() float64

	// Get the value of the smallest array element
	Min() float64

	// Return the element-wise product of this array and one or more others
	Prod(others ...NDArray) NDArray

	// The number of dimensions in the matrix
	NDim() int

	// Return a copy of the array, normalized to sum to 1
	Normalize() NDArray

	// Get a 1D copy of the array, in 'C' order: rightmost axes change fastest
	Ravel() NDArray

	// A slice giving the size of all array dimensions
	Shape() []int

	// The total number of elements in the matrix
	Size() int

	// Get an array containing a rectangular slice of this array.
	// `from` and `to` should both have one index per axis. The indices
	// in `from` and `to` define the first and just-past-last indices you wish
	// to select along each axis. Negative indexing is supported: when slicing,
	// index -1 refers to the item just past the last and -arr.Size() refers to
	// the first element.
	Slice(from []int, to []int) NDArray

	// Ask whether the matrix has a sparse representation (useful for optimization)
	Sparsity() ArraySparsity

	// Return the element-wise difference of this array and one or more others
	Sub(others ...NDArray) NDArray

	// Return the sum of all array elements
	Sum() float64

	// Return the same matrix, but with axes transposed. The same data is used,
	// for speed and memory efficiency. Use Copy() to create a new array.
	// A 1D array is unchanged; create a 2D analog to rotate a vector.
	T() NDArray
}

// Create an array from literal data
func A(shape []int, values ...float64) NDArray {
	size := 1
	for _, sz := range shape {
		size *= sz
	}
	if len(values) != size {
		panic(fmt.Sprintf("Expected %d array elements but got %d", size, len(values)))
	}
	array := &DenseF64Array{
		shape: shape,
		array: make([]float64, len(values)),
	}
	copy(array.array[:], values[:])
	return array
}

// Create a 1D array
func A1(values ...float64) NDArray {
	return A([]int{len(values)}, values...)
}

// Create a 2D array
func A2(values ...[]float64) NDArray {
	array := &DenseF64Array{
		shape: []int{len(values), len(values[0])},
		array: make([]float64, len(values)*len(values[0])),
	}
	for i0 := 0; i0 < array.shape[0]; i0++ {
		if len(values[i0]) != array.shape[1] {
			panic(fmt.Sprintf("A2 got inconsistent array lengths %d and %d", array.shape[1], len(values[i0])))
		}
		for i1 := 0; i1 < array.shape[1]; i1++ {
			array.ItemSet(values[i0][i1], i0, i1)
		}
	}
	return array
}

// Create an NDArray of float64 values, initialized to zero
func Dense(size ...int) NDArray {
	totalSize := 1
	for _, sz := range size {
		totalSize *= sz
	}
	return &DenseF64Array{
		shape: size,
		array: make([]float64, totalSize),
	}
}

// Create an NDArray of float64 values, initialized to value
func WithValue(value float64, size ...int) NDArray {
	array := Dense(size...)
	array.Fill(value)
	return array
}

// Create an NDArray of float64 values, initialized to zero
func Zeros(size ...int) NDArray {
	return Dense(size...)
}

// Create an NDArray of float64 values, initialized to one
func Ones(size ...int) NDArray {
	return WithValue(1.0, size...)
}

// Create a dense NDArray of float64 values, initialized to uniformly random
// values in [0, 1).
func Rand(size ...int) NDArray {
	array := Dense(size...)

	max := array.Size()
	for i := 0; i < max; i++ {
		array.FlatItemSet(rand.Float64(), i)
	}

	return array
}

// Create a dense NDArray of float64 values, initialized to random values in
// [-math.MaxFloat64, +math.MaxFloat64] distributed on the standard Normal
// distribution.
func RandN(size ...int) NDArray {
	array := Dense(size...)

	max := array.Size()
	for i := 0; i < max; i++ {
		array.FlatItemSet(rand.NormFloat64(), i)
	}

	return array
}
<commit_msg>updated package docs<commit_after>// The matrix package contains various utilities for dealing with raw matrices.
// The interface is loosely based on the NumPy package in Python.
//
// The key interfaces here are:
// * NDArray – A multidimensional array, with dense and (2D-only) sparse
//   implementations.
// * Matrix – A two-dimensional array, with various methods only available when
//   working in two dimensions.  A two-dimensional NDArray can be
//   trivially converted to the Matrix type by calling arr.M().
//
// When possible, function implementations take advantage of matrix sparsity.
// For instance, MProd(), the matrix multiplication function, performs the
// minimum amount of work required based on the types of its arguments.
//
// Certain linear algebra methods, particularly in the Matrix interface, rely
// on BLAS. In order to use it, you will need to register an appropriate engine.
// See the documentation at https://github.com/gonum/blas for details. You can
// register a default (native Go) engine by calling InitDefaultBlas().
// If you see a panic message like
// "mat64: no blas engine registered: call Register()"
// then you need to register a BLAS engine. If you don't see this error, then
// you probably don't need to worry about it.
package matrix

import (
	"fmt"
	"math/rand"
	"time"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

// ArraySparsity indicates the representation type of the matrix
type ArraySparsity int

const (
	DenseArray ArraySparsity = iota
	SparseCooMatrix
	SparseDiagMatrix
)

// A NDArray is an n-dimensional array of numbers which can be manipulated in
// various ways. Concrete implementations can differ; for instance, sparse
// and dense representations are possible.
type NDArray interface {

	// Return the element-wise sum of this array and one or more others
	Add(others ...NDArray) NDArray

	// Returns true if and only if all items are nonzero
	All() bool

	// Returns true if f is true for all array elements
	AllF(f func(v float64) bool) bool

	// Returns true if f is true for all pairs of array elements in the same position
	AllF2(f func(v1, v2 float64) bool, other NDArray) bool

	// Returns true if and only if any item is nonzero
	Any() bool

	// Returns true if f is true for any array element
	AnyF(f func(v float64) bool) bool

	// Returns true if f is true for any pair of array elements in the same position
	AnyF2(f func(v1, v2 float64) bool, other NDArray) bool

	// Return the result of applying a function to all elements
	Apply(f func(float64) float64) NDArray

	// Get the matrix data as a flattened 1D array; sparse matrices will make
	// a copy first.
	Array() []float64

	// Create a new array by concatenating this with another array along the
	// specified axis. The array shapes must be equal along all other axes.
	// It is legal to add a new axis.
	Concat(axis int, others ...NDArray) NDArray

	// Returns a duplicate of this array
	Copy() NDArray

	// Counts the number of nonzero elements in the array
	CountNonzero() int

	// Returns a dense copy of the array
	Dense() NDArray

	// Return the element-wise quotient of this array and one or more others.
	// This function defines 0 / 0 = 0, so it's useful for sparse arrays.
	Div(others ...NDArray) NDArray

	// Returns true if and only if all elements in the two arrays are equal
	Equal(other NDArray) bool

	// Set all array elements to the given value
	Fill(value float64)

	// Get an array element in a flattened verison of this array
	FlatItem(index int) float64

	// Set an array element in a flattened version of this array
	FlatItemSet(value float64, index int)

	// Return an iterator over populated matrix entries
	FlatIter() FlatNDArrayIterator

	// Get an array element
	Item(index ...int) float64

	// Return the result of adding a scalar value to each array element
	ItemAdd(value float64) NDArray

	// Return the result of dividing each array element by a scalar value
	ItemDiv(value float64) NDArray

	// Return the reuslt of multiplying each array element by a scalar value
	ItemProd(value float64) NDArray

	// Return the result of subtracting a scalar value from each array element
	ItemSub(value float64) NDArray

	// Set an array element
	ItemSet(value float64, index ...int)

	// Return an iterator over populated matrix entries
	Iter() CoordNDArrayIterator

	// Returns the array as a matrix. This is only possible for 1D and 2D arrays;
	// 1D arrays of length n are converted into n x 1 vectors.
	M() Matrix

	// Get the value of the largest array element
	Max() float64

	// Get the value of the smallest array element
	Min() float64

	// Return the element-wise product of this array and one or more others
	Prod(others ...NDArray) NDArray

	// The number of dimensions in the matrix
	NDim() int

	// Return a copy of the array, normalized to sum to 1
	Normalize() NDArray

	// Get a 1D copy of the array, in 'C' order: rightmost axes change fastest
	Ravel() NDArray

	// A slice giving the size of all array dimensions
	Shape() []int

	// The total number of elements in the matrix
	Size() int

	// Get an array containing a rectangular slice of this array.
	// `from` and `to` should both have one index per axis. The indices
	// in `from` and `to` define the first and just-past-last indices you wish
	// to select along each axis. Negative indexing is supported: when slicing,
	// index -1 refers to the item just past the last and -arr.Size() refers to
	// the first element.
	Slice(from []int, to []int) NDArray

	// Ask whether the matrix has a sparse representation (useful for optimization)
	Sparsity() ArraySparsity

	// Return the element-wise difference of this array and one or more others
	Sub(others ...NDArray) NDArray

	// Return the sum of all array elements
	Sum() float64

	// Return the same matrix, but with axes transposed. The same data is used,
	// for speed and memory efficiency. Use Copy() to create a new array.
	// A 1D array is unchanged; create a 2D analog to rotate a vector.
	T() NDArray
}

// Create an array from literal data
func A(shape []int, values ...float64) NDArray {
	size := 1
	for _, sz := range shape {
		size *= sz
	}
	if len(values) != size {
		panic(fmt.Sprintf("Expected %d array elements but got %d", size, len(values)))
	}
	array := &DenseF64Array{
		shape: shape,
		array: make([]float64, len(values)),
	}
	copy(array.array[:], values[:])
	return array
}

// Create a 1D array
func A1(values ...float64) NDArray {
	return A([]int{len(values)}, values...)
}

// Create a 2D array
func A2(values ...[]float64) NDArray {
	array := &DenseF64Array{
		shape: []int{len(values), len(values[0])},
		array: make([]float64, len(values)*len(values[0])),
	}
	for i0 := 0; i0 < array.shape[0]; i0++ {
		if len(values[i0]) != array.shape[1] {
			panic(fmt.Sprintf("A2 got inconsistent array lengths %d and %d", array.shape[1], len(values[i0])))
		}
		for i1 := 0; i1 < array.shape[1]; i1++ {
			array.ItemSet(values[i0][i1], i0, i1)
		}
	}
	return array
}

// Create an NDArray of float64 values, initialized to zero
func Dense(size ...int) NDArray {
	totalSize := 1
	for _, sz := range size {
		totalSize *= sz
	}
	return &DenseF64Array{
		shape: size,
		array: make([]float64, totalSize),
	}
}

// Create an NDArray of float64 values, initialized to value
func WithValue(value float64, size ...int) NDArray {
	array := Dense(size...)
	array.Fill(value)
	return array
}

// Create an NDArray of float64 values, initialized to zero
func Zeros(size ...int) NDArray {
	return Dense(size...)
}

// Create an NDArray of float64 values, initialized to one
func Ones(size ...int) NDArray {
	return WithValue(1.0, size...)
}

// Create a dense NDArray of float64 values, initialized to uniformly random
// values in [0, 1).
func Rand(size ...int) NDArray {
	array := Dense(size...)

	max := array.Size()
	for i := 0; i < max; i++ {
		array.FlatItemSet(rand.Float64(), i)
	}

	return array
}

// Create a dense NDArray of float64 values, initialized to random values in
// [-math.MaxFloat64, +math.MaxFloat64] distributed on the standard Normal
// distribution.
func RandN(size ...int) NDArray {
	array := Dense(size...)

	max := array.Size()
	for i := 0; i < max; i++ {
		array.FlatItemSet(rand.NormFloat64(), i)
	}

	return array
}
<|endoftext|>


<commit_before>/*
Copyright 2016 Christian Grabowski All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pipeline

import (
	"errors"
	"log"

	git "gopkg.in/libgit2/git2go.v22"
)

// DepTree is a dependency tree to determine whether or not to build services
type DepTree struct {
	CurrNode *DepService
}

// TraverseTree traverses a dependency tree
func TraverseTree(depSrv *DepService, repo *git.Repository, lastBuildCommit *string) error {
	if depSrv == nil {
		return errors.New("Service is nil in tree")
	}
	shouldBuild, buildErr := depSrv.build.ShouldBuild(repo, lastBuildCommit)
	if buildErr != nil {
		return buildErr
	}
	if shouldBuild {
		for i := range depSrv.Children {
			depSrv.Children[i].build.shouldBuild = true
			travErr := TraverseTree(depSrv.Children[i], repo, lastBuildCommit)
			if travErr != nil {
				return travErr
			}
		}
	}
	return nil
}

// DepService represents a service in the tree
type DepService struct {
	build    *Service
	Parent   *DepService
	Children map[string]*DepService
}

func getDependencies(depSrv *DepService, created map[string]*DepService, proj *Project) {
	for j := range depSrv.build.conf.DependsOn {
		if created[depSrv.build.conf.DependsOn[j]] != nil {
			depSrv.Parent = created[depSrv.build.conf.DependsOn[j]]
			if created[depSrv.build.conf.DependsOn[j]].Children[depSrv.build.conf.Name] == nil {
				created[depSrv.build.conf.DependsOn[j]].Children[depSrv.build.conf.Name] = depSrv
			}
		} else {
			parent := &DepService{build: proj.Services[depSrv.build.conf.DependsOn[j]]}
			depSrv.Parent = parent
			if parent.Children[depSrv.build.conf.Name] == nil {
				parent.Children[depSrv.build.conf.Name] = depSrv
			}
		}
	}
}

// NewTreeList returns a list of dependency trees
func NewTreeList(proj *Project) []*DepTree {
	var newTrees []*DepTree
	created := make(map[string]*DepService)
	log.Println("Creating a dependency tree")
	for i := range proj.Services {
		log.Println("Finding a spot in the tree for ", proj.Services[i].conf.Name)
		if created[proj.Services[i].conf.Name] != nil {
			depSrv := created[proj.Services[i].conf.Name]
			getDependencies(depSrv, created, proj)
		}
		depSrv := &DepService{build: proj.Services[i], Children: make(map[string]*DepService)}
		if len(depSrv.build.conf.DependsOn) == 0 {
			newTrees = append(newTrees, &DepTree{CurrNode: depSrv})
			created[depSrv.build.conf.Name] = depSrv
		} else {
			getDependencies(depSrv, created, proj)
		}
	}
	return newTrees
}
<commit_msg>make lowest parent direct dependency<commit_after>/*
Copyright 2016 Christian Grabowski All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pipeline

import (
	"errors"
	"log"

	git "gopkg.in/libgit2/git2go.v22"
)

// DepTree is a dependency tree to determine whether or not to build services
type DepTree struct {
	CurrNode *DepService
}

// TraverseTree traverses a dependency tree
func TraverseTree(depSrv *DepService, repo *git.Repository, lastBuildCommit *string) error {
	if depSrv == nil {
		return errors.New("Service is nil in tree")
	}
	shouldBuild, buildErr := depSrv.build.ShouldBuild(repo, lastBuildCommit)
	if buildErr != nil {
		return buildErr
	}
	if shouldBuild {
		for i := range depSrv.Children {
			depSrv.Children[i].build.shouldBuild = true
			travErr := TraverseTree(depSrv.Children[i], repo, lastBuildCommit)
			if travErr != nil {
				return travErr
			}
		}
	}
	return nil
}

// DepService represents a service in the tree
type DepService struct {
	build    *Service
	Parent   *DepService
	Children map[string]*DepService
}

func dependsOnChild(child, parent *DepService) int {
	for j := range child.build.conf.DependsOn {
		if parent.Children[child.build.conf.DependsOn[j]] != nil {
			return j
		}
	}
	return -1
}

func getDependencies(depSrv *DepService, created map[string]*DepService, proj *Project) {
	for j := range depSrv.build.conf.DependsOn {
		if created[depSrv.build.conf.DependsOn[j]] != nil {
			youngerParentIndex := dependsOnChild(depSrv, created[depSrv.build.conf.DependsOn[j]])
			if youngerParentIndex > -1 {
				depSrv.Parent = created[depSrv.build.conf.DependsOn[j]].Children[depSrv.build.conf.DependsOn[youngerParentIndex]]
				depSrv.Children[depSrv.build.conf.Name] = depSrv
			} else {
				depSrv.Parent = created[depSrv.build.conf.DependsOn[j]]
				if depSrv.Parent.Children[depSrv.build.conf.Name] == nil {
					depSrv.Parent.Children[depSrv.build.conf.Name] = depSrv
				}
			}
		} else {
			parent := &DepService{build: proj.Services[depSrv.build.conf.DependsOn[j]]}
			depSrv.Parent = parent
			if parent.Children[depSrv.build.conf.Name] == nil {
				parent.Children[depSrv.build.conf.Name] = depSrv
			}
		}
	}
}

// NewTreeList returns a list of dependency trees
func NewTreeList(proj *Project) []*DepTree {
	var newTrees []*DepTree
	created := make(map[string]*DepService)
	log.Println("Creating a dependency tree")
	for i := range proj.Services {
		log.Println("Finding a spot in the tree for ", proj.Services[i].conf.Name)
		if created[proj.Services[i].conf.Name] != nil {
			depSrv := created[proj.Services[i].conf.Name]
			getDependencies(depSrv, created, proj)
		}
		depSrv := &DepService{build: proj.Services[i], Children: make(map[string]*DepService)}
		if len(depSrv.build.conf.DependsOn) == 0 {
			newTrees = append(newTrees, &DepTree{CurrNode: depSrv})
			created[depSrv.build.conf.Name] = depSrv
		} else {
			getDependencies(depSrv, created, proj)
		}
	}
	return newTrees
}
<|endoftext|>


<commit_before>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helm

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"sort"
	"strings"

	jsonpatch "github.com/evanphx/json-patch"
	"github.com/ghodss/yaml"
	"k8s.io/helm/pkg/chartutil"
	"k8s.io/helm/pkg/engine"
	"k8s.io/helm/pkg/proto/hapi/chart"
	"k8s.io/helm/pkg/timeconv"

	"istio.io/operator/pkg/util"
	"istio.io/pkg/log"
)

const (
	// YAMLSeparator is a separator for multi-document YAML files.
	YAMLSeparator = "\n---\n"

	// DefaultProfileString is the name of the default profile.
	DefaultProfileString = "default"
)

// TemplateRenderer defines a helm template renderer interface.
type TemplateRenderer interface {
	// Run starts the renderer and should be called before using it.
	Run() error
	// RenderManifest renders the associated helm charts with the given values YAML string and returns the resulting
	// string.
	RenderManifest(values string) (string, error)
}

// NewHelmRenderer creates a new helm renderer with the given parameters and returns an interface to it.
// The format of helmBaseDir and profile strings determines the type of helm renderer returned (compiled-in, file,
// HTTP etc.)
func NewHelmRenderer(chartsRootDir, helmBaseDir, componentName, namespace string) (TemplateRenderer, error) {
	// filepath would remove leading slash here if chartsRootDir is empty.
	dir := chartsRootDir + "/" + helmBaseDir
	switch {
	case chartsRootDir == "":
		return NewVFSRenderer(helmBaseDir, componentName, namespace), nil
	case util.IsFilePath(dir):
		return NewFileTemplateRenderer(dir, componentName, namespace), nil
	default:
		return nil, fmt.Errorf("unknown helm renderer with chartsRoot=%s", chartsRootDir)
	}
}

// ReadProfileYAML reads the YAML values associated with the given profile. It uses an appropriate reader for the
// profile format (compiled-in, file, HTTP, etc.).
func ReadProfileYAML(profile string) (string, error) {
	var err error
	var globalValues string
	if profile == "" {
		log.Infof("ReadProfileYAML for profile name: [Empty]")
	} else {
		log.Infof("ReadProfileYAML for profile name: %s", profile)
	}

	// Get global values from profile.
	switch {
	case isBuiltinProfileName(profile):
		if globalValues, err = LoadValuesVFS(profile); err != nil {
			return "", err
		}
	case util.IsFilePath(profile):
		log.Infof("Loading values from local filesystem at path %s", profile)
		if globalValues, err = readFile(profile); err != nil {
			return "", err
		}
	default:
		return "", fmt.Errorf("unsupported Profile type: %s", profile)
	}

	return globalValues, nil
}

// renderChart renders the given chart with the given values and returns the resulting YAML manifest string.
func renderChart(namespace, values string, chrt *chart.Chart) (string, error) {
	config := &chart.Config{Raw: values, Values: map[string]*chart.Value{}}
	options := chartutil.ReleaseOptions{
		Name:      "istio",
		Time:      timeconv.Now(),
		Namespace: namespace,
	}

	vals, err := chartutil.ToRenderValuesCaps(chrt, config, options, nil)
	if err != nil {
		return "", err
	}

	files, err := engine.New().Render(chrt, vals)
	if err != nil {
		return "", err
	}

	// Create sorted array of keys to iterate over, to stabilize the order of the rendered templates
	keys := make([]string, 0, len(files))
	for k := range files {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	var sb strings.Builder
	for i := 0; i < len(keys); i++ {
		f := files[keys[i]]
		// add yaml separator if the rendered file doesn't have one at the end
		if !strings.HasSuffix(strings.TrimSpace(f)+"\n", YAMLSeparator) {
			f += YAMLSeparator
		}
		_, err := sb.WriteString(f)
		if err != nil {
			return "", err
		}
	}

	return sb.String(), nil
}

// OverlayYAML patches the overlay tree over the base tree and returns the result. All trees are expressed as YAML
// strings.
func OverlayYAML(base, overlay string) (string, error) {
	if overlay == "" {
		return base, nil
	}
	bj, err := yaml.YAMLToJSON([]byte(base))
	if err != nil {
		return "", fmt.Errorf("yamlToJSON error in base: %s\n%s", err, bj)
	}
	oj, err := yaml.YAMLToJSON([]byte(overlay))
	if err != nil {
		return "", fmt.Errorf("yamlToJSON error in overlay: %s\n%s", err, oj)
	}
	if overlay == "" {
		oj = []byte("{}")
	}

	merged, err := jsonpatch.MergePatch(bj, oj)
	if err != nil {
		return "", fmt.Errorf("json merge error (%s) for base object: \n%s\n override object: \n%s", err, bj, oj)
	}
	my, err := yaml.JSONToYAML(merged)
	if err != nil {
		return "", fmt.Errorf("jsonToYAML error (%s) for merged object: \n%s", err, merged)
	}

	return string(my), nil
}

// DefaultFilenameForProfile returns the profile name of the default profile for the given profile.
func DefaultFilenameForProfile(profile string) (string, error) {
	switch {
	case util.IsFilePath(profile):
		return filepath.Join(filepath.Dir(profile), DefaultProfileFilename), nil
	default:
		if _, ok := ProfileNames[profile]; ok || profile == "" {
			return DefaultProfileString, nil
		}
		return "", fmt.Errorf("bad profile string %s", profile)
	}
}

// IsDefaultProfile reports whether the given profile is the default profile.
func IsDefaultProfile(profile string) bool {
	return profile == "" || profile == DefaultProfileString || filepath.Base(profile) == DefaultProfileFilename
}

func readFile(path string) (string, error) {
	b, err := ioutil.ReadFile(path)
	return string(b), err
}
<commit_msg>remove notes file from render output. (#280)<commit_after>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helm

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"sort"
	"strings"

	jsonpatch "github.com/evanphx/json-patch"
	"github.com/ghodss/yaml"
	"k8s.io/helm/pkg/chartutil"
	"k8s.io/helm/pkg/engine"
	"k8s.io/helm/pkg/proto/hapi/chart"
	"k8s.io/helm/pkg/timeconv"

	"istio.io/operator/pkg/util"
	"istio.io/pkg/log"
)

const (
	// YAMLSeparator is a separator for multi-document YAML files.
	YAMLSeparator = "\n---\n"

	// DefaultProfileString is the name of the default profile.
	DefaultProfileString = "default"

	// notes file name suffix for the helm chart.
	NotesFileNameSuffix = ".txt"
)

// TemplateRenderer defines a helm template renderer interface.
type TemplateRenderer interface {
	// Run starts the renderer and should be called before using it.
	Run() error
	// RenderManifest renders the associated helm charts with the given values YAML string and returns the resulting
	// string.
	RenderManifest(values string) (string, error)
}

// NewHelmRenderer creates a new helm renderer with the given parameters and returns an interface to it.
// The format of helmBaseDir and profile strings determines the type of helm renderer returned (compiled-in, file,
// HTTP etc.)
func NewHelmRenderer(chartsRootDir, helmBaseDir, componentName, namespace string) (TemplateRenderer, error) {
	// filepath would remove leading slash here if chartsRootDir is empty.
	dir := chartsRootDir + "/" + helmBaseDir
	switch {
	case chartsRootDir == "":
		return NewVFSRenderer(helmBaseDir, componentName, namespace), nil
	case util.IsFilePath(dir):
		return NewFileTemplateRenderer(dir, componentName, namespace), nil
	default:
		return nil, fmt.Errorf("unknown helm renderer with chartsRoot=%s", chartsRootDir)
	}
}

// ReadProfileYAML reads the YAML values associated with the given profile. It uses an appropriate reader for the
// profile format (compiled-in, file, HTTP, etc.).
func ReadProfileYAML(profile string) (string, error) {
	var err error
	var globalValues string
	if profile == "" {
		log.Infof("ReadProfileYAML for profile name: [Empty]")
	} else {
		log.Infof("ReadProfileYAML for profile name: %s", profile)
	}

	// Get global values from profile.
	switch {
	case isBuiltinProfileName(profile):
		if globalValues, err = LoadValuesVFS(profile); err != nil {
			return "", err
		}
	case util.IsFilePath(profile):
		log.Infof("Loading values from local filesystem at path %s", profile)
		if globalValues, err = readFile(profile); err != nil {
			return "", err
		}
	default:
		return "", fmt.Errorf("unsupported Profile type: %s", profile)
	}

	return globalValues, nil
}

// renderChart renders the given chart with the given values and returns the resulting YAML manifest string.
func renderChart(namespace, values string, chrt *chart.Chart) (string, error) {
	config := &chart.Config{Raw: values, Values: map[string]*chart.Value{}}
	options := chartutil.ReleaseOptions{
		Name:      "istio",
		Time:      timeconv.Now(),
		Namespace: namespace,
	}

	vals, err := chartutil.ToRenderValuesCaps(chrt, config, options, nil)
	if err != nil {
		return "", err
	}

	files, err := engine.New().Render(chrt, vals)
	if err != nil {
		return "", err
	}

	// Create sorted array of keys to iterate over, to stabilize the order of the rendered templates
	keys := make([]string, 0, len(files))
	for k := range files {
		if strings.HasSuffix(k, NotesFileNameSuffix) {
			continue
		}
		keys = append(keys, k)
	}
	sort.Strings(keys)

	var sb strings.Builder
	for i := 0; i < len(keys); i++ {
		f := files[keys[i]]
		// add yaml separator if the rendered file doesn't have one at the end
		if !strings.HasSuffix(strings.TrimSpace(f)+"\n", YAMLSeparator) {
			f += YAMLSeparator
		}
		_, err := sb.WriteString(f)
		if err != nil {
			return "", err
		}
	}

	return sb.String(), nil
}

// OverlayYAML patches the overlay tree over the base tree and returns the result. All trees are expressed as YAML
// strings.
func OverlayYAML(base, overlay string) (string, error) {
	if overlay == "" {
		return base, nil
	}
	bj, err := yaml.YAMLToJSON([]byte(base))
	if err != nil {
		return "", fmt.Errorf("yamlToJSON error in base: %s\n%s", err, bj)
	}
	oj, err := yaml.YAMLToJSON([]byte(overlay))
	if err != nil {
		return "", fmt.Errorf("yamlToJSON error in overlay: %s\n%s", err, oj)
	}
	if overlay == "" {
		oj = []byte("{}")
	}

	merged, err := jsonpatch.MergePatch(bj, oj)
	if err != nil {
		return "", fmt.Errorf("json merge error (%s) for base object: \n%s\n override object: \n%s", err, bj, oj)
	}
	my, err := yaml.JSONToYAML(merged)
	if err != nil {
		return "", fmt.Errorf("jsonToYAML error (%s) for merged object: \n%s", err, merged)
	}

	return string(my), nil
}

// DefaultFilenameForProfile returns the profile name of the default profile for the given profile.
func DefaultFilenameForProfile(profile string) (string, error) {
	switch {
	case util.IsFilePath(profile):
		return filepath.Join(filepath.Dir(profile), DefaultProfileFilename), nil
	default:
		if _, ok := ProfileNames[profile]; ok || profile == "" {
			return DefaultProfileString, nil
		}
		return "", fmt.Errorf("bad profile string %s", profile)
	}
}

// IsDefaultProfile reports whether the given profile is the default profile.
func IsDefaultProfile(profile string) bool {
	return profile == "" || profile == DefaultProfileString || filepath.Base(profile) == DefaultProfileFilename
}

func readFile(path string) (string, error) {
	b, err := ioutil.ReadFile(path)
	return string(b), err
}
<|endoftext|>


<commit_before>package plugin

import (
	"os/exec"

	plugin "github.com/hashicorp/go-plugin"
	"github.com/hashicorp/terraform/plugin/discovery"
)

// ClientConfig returns a configuration object that can be used to instantiate
// a client for the plugin described by the given metadata.
func ClientConfig(m discovery.PluginMeta) *plugin.ClientConfig {
	return &plugin.ClientConfig{
		Cmd:             exec.Command(m.Path),
		HandshakeConfig: Handshake,
		Managed:         true,
		Plugins:         PluginMap,
	}
}

// Client returns a plugin client for the plugin described by the given metadata.
func Client(m discovery.PluginMeta) *plugin.Client {
	return plugin.NewClient(ClientConfig(m))
}
<commit_msg>specify a logger for go-plugin<commit_after>package plugin

import (
	"os"
	"os/exec"

	hclog "github.com/hashicorp/go-hclog"
	plugin "github.com/hashicorp/go-plugin"
	"github.com/hashicorp/terraform/plugin/discovery"
)

// ClientConfig returns a configuration object that can be used to instantiate
// a client for the plugin described by the given metadata.
func ClientConfig(m discovery.PluginMeta) *plugin.ClientConfig {
	logger := hclog.New(&hclog.LoggerOptions{
		Name:   "plugin",
		Level:  hclog.Trace,
		Output: os.Stderr,
	})

	return &plugin.ClientConfig{
		Cmd:             exec.Command(m.Path),
		HandshakeConfig: Handshake,
		Managed:         true,
		Plugins:         PluginMap,
		Logger:          logger,
	}
}

// Client returns a plugin client for the plugin described by the given metadata.
func Client(m discovery.PluginMeta) *plugin.Client {
	return plugin.NewClient(ClientConfig(m))
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Yasuhiro Matsumoto <mattn.jp@gmail.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// +build go1.8

package sqlite3

import (
	"context"
	"database/sql"
	"fmt"
	"math/rand"
	"os"
	"testing"
	"time"
)

func TestNamedParams(t *testing.T) {
	tempFilename := TempFilename(t)
	defer os.Remove(tempFilename)
	db, err := sql.Open("sqlite3", tempFilename)
	if err != nil {
		t.Fatal("Failed to open database:", err)
	}
	defer db.Close()

	_, err = db.Exec(`
	create table foo (id integer, name text, extra text);
	`)
	if err != nil {
		t.Error("Failed to call db.Query:", err)
	}

	_, err = db.Exec(`insert into foo(id, name, extra) values(:id, :name, :name)`, sql.Named("name", "foo"), sql.Named("id", 1))
	if err != nil {
		t.Error("Failed to call db.Exec:", err)
	}

	row := db.QueryRow(`select id, extra from foo where id = :id and extra = :extra`, sql.Named("id", 1), sql.Named("extra", "foo"))
	if row == nil {
		t.Error("Failed to call db.QueryRow")
	}
	var id int
	var extra string
	err = row.Scan(&id, &extra)
	if err != nil {
		t.Error("Failed to db.Scan:", err)
	}
	if id != 1 || extra != "foo" {
		t.Error("Failed to db.QueryRow: not matched results")
	}
}

var (
	testTableStatements = []string{
		`DROP TABLE IF EXISTS test_table`,
		`
CREATE TABLE IF NOT EXISTS test_table (
	key1      VARCHAR(64) PRIMARY KEY,
	key_id    VARCHAR(64) NOT NULL,
	key2      VARCHAR(64) NOT NULL,
	key3      VARCHAR(64) NOT NULL,
	key4      VARCHAR(64) NOT NULL,
	key5      VARCHAR(64) NOT NULL,
	key6      VARCHAR(64) NOT NULL,
	data      BLOB        NOT NULL
);`,
	}
	letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
)

func randStringBytes(n int) string {
	b := make([]byte, n)
	for i := range b {
		b[i] = letterBytes[rand.Intn(len(letterBytes))]
	}
	return string(b)
}

func initDatabase(t *testing.T, db *sql.DB, rowCount int64) {
	t.Logf("Executing db initializing statements")
	for _, query := range testTableStatements {
		_, err := db.Exec(query)
		if err != nil {
			t.Fatal(err)
		}
	}
	for i := int64(0); i < rowCount; i++ {
		query := `INSERT INTO test_table
			(key1, key_id, key2, key3, key4, key5, key6, data)
			VALUES
			(?, ?, ?, ?, ?, ?, ?, ?);`
		args := []interface{}{
			randStringBytes(50),
			fmt.Sprint(i),
			randStringBytes(50),
			randStringBytes(50),
			randStringBytes(50),
			randStringBytes(50),
			randStringBytes(50),
			randStringBytes(50),
			randStringBytes(2048),
		}
		_, err := db.Exec(query, args...)
		if err != nil {
			t.Fatal(err)
		}
	}
}

func TestShortTimeout(t *testing.T) {
	srcTempFilename := TempFilename(t)
	defer os.Remove(srcTempFilename)

	db, err := sql.Open("sqlite3", srcTempFilename)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()
	initDatabase(t, db, 100)

	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Microsecond)
	defer cancel()
	query := `SELECT key1, key_id, key2, key3, key4, key5, key6, data
		FROM test_table
		ORDER BY key2 ASC`
	rows, err := db.QueryContext(ctx, query)
	if err != nil && context.DeadlineExceeded {
		t.Fatal(err)
	}
	if ctx.Err() != nil && context.DeadlineExceeded != ctx.Err() {
		t.Fatalf(ctx.Err())
	}
	rows.Close()
}
<commit_msg>fix broken test<commit_after>// Copyright (C) 2014 Yasuhiro Matsumoto <mattn.jp@gmail.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// +build go1.8

package sqlite3

import (
	"context"
	"database/sql"
	"fmt"
	"math/rand"
	"os"
	"testing"
	"time"
)

func TestNamedParams(t *testing.T) {
	tempFilename := TempFilename(t)
	defer os.Remove(tempFilename)
	db, err := sql.Open("sqlite3", tempFilename)
	if err != nil {
		t.Fatal("Failed to open database:", err)
	}
	defer db.Close()

	_, err = db.Exec(`
	create table foo (id integer, name text, extra text);
	`)
	if err != nil {
		t.Error("Failed to call db.Query:", err)
	}

	_, err = db.Exec(`insert into foo(id, name, extra) values(:id, :name, :name)`, sql.Named("name", "foo"), sql.Named("id", 1))
	if err != nil {
		t.Error("Failed to call db.Exec:", err)
	}

	row := db.QueryRow(`select id, extra from foo where id = :id and extra = :extra`, sql.Named("id", 1), sql.Named("extra", "foo"))
	if row == nil {
		t.Error("Failed to call db.QueryRow")
	}
	var id int
	var extra string
	err = row.Scan(&id, &extra)
	if err != nil {
		t.Error("Failed to db.Scan:", err)
	}
	if id != 1 || extra != "foo" {
		t.Error("Failed to db.QueryRow: not matched results")
	}
}

var (
	testTableStatements = []string{
		`DROP TABLE IF EXISTS test_table`,
		`
CREATE TABLE IF NOT EXISTS test_table (
	key1      VARCHAR(64) PRIMARY KEY,
	key_id    VARCHAR(64) NOT NULL,
	key2      VARCHAR(64) NOT NULL,
	key3      VARCHAR(64) NOT NULL,
	key4      VARCHAR(64) NOT NULL,
	key5      VARCHAR(64) NOT NULL,
	key6      VARCHAR(64) NOT NULL,
	data      BLOB        NOT NULL
);`,
	}
	letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
)

func randStringBytes(n int) string {
	b := make([]byte, n)
	for i := range b {
		b[i] = letterBytes[rand.Intn(len(letterBytes))]
	}
	return string(b)
}

func initDatabase(t *testing.T, db *sql.DB, rowCount int64) {
	t.Logf("Executing db initializing statements")
	for _, query := range testTableStatements {
		_, err := db.Exec(query)
		if err != nil {
			t.Fatal(err)
		}
	}
	for i := int64(0); i < rowCount; i++ {
		query := `INSERT INTO test_table
			(key1, key_id, key2, key3, key4, key5, key6, data)
			VALUES
			(?, ?, ?, ?, ?, ?, ?, ?);`
		args := []interface{}{
			randStringBytes(50),
			fmt.Sprint(i),
			randStringBytes(50),
			randStringBytes(50),
			randStringBytes(50),
			randStringBytes(50),
			randStringBytes(50),
			randStringBytes(50),
			randStringBytes(2048),
		}
		_, err := db.Exec(query, args...)
		if err != nil {
			t.Fatal(err)
		}
	}
}

func TestShortTimeout(t *testing.T) {
	srcTempFilename := TempFilename(t)
	defer os.Remove(srcTempFilename)

	db, err := sql.Open("sqlite3", srcTempFilename)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()
	initDatabase(t, db, 100)

	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Microsecond)
	defer cancel()
	query := `SELECT key1, key_id, key2, key3, key4, key5, key6, data
		FROM test_table
		ORDER BY key2 ASC`
	rows, err := db.QueryContext(ctx, query)
	if err != nil && err != context.DeadlineExceeded {
		t.Fatal(err)
	}
	if ctx.Err() != nil && ctx.Err() != context.DeadlineExceeded {
		t.Fatal(ctx.Err())
	}
	rows.Close()
}
<|endoftext|>


<commit_before>package sqsbroker

import (
	"fmt"
)

const minAllocatedStorage = 5
const maxAllocatedStorage = 6144

type Catalog struct {
	Services []Service `json:"services,omitempty"`
}

type Service struct {
	ID              string           `json:"id"`
	Name            string           `json:"name"`
	Description     string           `json:"description"`
	Bindable        bool             `json:"bindable,omitempty"`
	Tags            []string         `json:"tags,omitempty"`
	Metadata        *ServiceMetadata `json:"metadata,omitempty"`
	Requires        []string         `json:"requires,omitempty"`
	PlanUpdateable  bool             `json:"plan_updateable"`
	Plans           []ServicePlan    `json:"plans,omitempty"`
	DashboardClient *DashboardClient `json:"dashboard_client,omitempty"`
}

type ServiceMetadata struct {
	DisplayName         string `json:"displayName,omitempty"`
	ImageURL            string `json:"imageUrl,omitempty"`
	LongDescription     string `json:"longDescription,omitempty"`
	ProviderDisplayName string `json:"providerDisplayName,omitempty"`
	DocumentationURL    string `json:"documentationUrl,omitempty"`
	SupportURL          string `json:"supportUrl,omitempty"`
}

type ServicePlan struct {
	ID            string               `json:"id"`
	Name          string               `json:"name"`
	Description   string               `json:"description"`
	Metadata      *ServicePlanMetadata `json:"metadata,omitempty"`
	Free          bool                 `json:"free"`
	SQSProperties SQSProperties        `json:"sqs_properties,omitempty"`
}

type ServicePlanMetadata struct {
	Bullets     []string `json:"bullets,omitempty"`
	Costs       []Cost   `json:"costs,omitempty"`
	DisplayName string   `json:"displayName,omitempty"`
}

type DashboardClient struct {
	ID          string `json:"id,omitempty"`
	Secret      string `json:"secret,omitempty"`
	RedirectURI string `json:"redirect_uri,omitempty"`
}

type Cost struct {
	Amount map[string]interface{} `json:"amount,omitempty"`
	Unit   string                 `json:"unit,omitempty"`
}

type SQSProperties struct {
	DelaySeconds                  string `json:"delay_seconds"`
	MaximumMessageSize            string `json:"maximum_message_size"`
	MessageRetentionPeriod        string `json:"message_retention_period"`
	ReceiveMessageWaitTimeSeconds string `json:"receive_message_wait_time_seconds"`
	VisibilityTimeout             string `json:"visibility_timeout"`
}

func (c Catalog) Validate() error {
	for _, service := range c.Services {
		if err := service.Validate(); err != nil {
			return fmt.Errorf("Validating Services configuration: %s", err)
		}
	}

	return nil
}

func (c Catalog) FindService(serviceID string) (service Service, found bool) {
	for _, service := range c.Services {
		if service.ID == serviceID {
			return service, true
		}
	}

	return service, false
}

func (c Catalog) FindServicePlan(planID string) (plan ServicePlan, found bool) {
	for _, service := range c.Services {
		for _, plan := range service.Plans {
			if plan.ID == planID {
				return plan, true
			}
		}
	}

	return plan, false
}

func (s Service) Validate() error {
	if s.ID == "" {
		return fmt.Errorf("Must provide a non-empty ID (%+v)", s)
	}

	if s.Name == "" {
		return fmt.Errorf("Must provide a non-empty Name (%+v)", s)
	}

	if s.Description == "" {
		return fmt.Errorf("Must provide a non-empty Description (%+v)", s)
	}

	for _, servicePlan := range s.Plans {
		if err := servicePlan.Validate(); err != nil {
			return fmt.Errorf("Validating Plans configuration: %s", err)
		}
	}

	return nil
}

func (sp ServicePlan) Validate() error {
	if sp.ID == "" {
		return fmt.Errorf("Must provide a non-empty ID (%+v)", sp)
	}

	if sp.Name == "" {
		return fmt.Errorf("Must provide a non-empty Name (%+v)", sp)
	}

	if sp.Description == "" {
		return fmt.Errorf("Must provide a non-empty Description (%+v)", sp)
	}

	if err := sp.SQSProperties.Validate(); err != nil {
		return fmt.Errorf("Validating SQS Properties configuration: %s", err)
	}

	return nil
}

func (sq SQSProperties) Validate() error {

	return nil
}
<commit_msg>Add omitempty to optional fields<commit_after>package sqsbroker

import (
	"fmt"
)

const minAllocatedStorage = 5
const maxAllocatedStorage = 6144

type Catalog struct {
	Services []Service `json:"services,omitempty"`
}

type Service struct {
	ID              string           `json:"id"`
	Name            string           `json:"name"`
	Description     string           `json:"description"`
	Bindable        bool             `json:"bindable,omitempty"`
	Tags            []string         `json:"tags,omitempty"`
	Metadata        *ServiceMetadata `json:"metadata,omitempty"`
	Requires        []string         `json:"requires,omitempty"`
	PlanUpdateable  bool             `json:"plan_updateable"`
	Plans           []ServicePlan    `json:"plans,omitempty"`
	DashboardClient *DashboardClient `json:"dashboard_client,omitempty"`
}

type ServiceMetadata struct {
	DisplayName         string `json:"displayName,omitempty"`
	ImageURL            string `json:"imageUrl,omitempty"`
	LongDescription     string `json:"longDescription,omitempty"`
	ProviderDisplayName string `json:"providerDisplayName,omitempty"`
	DocumentationURL    string `json:"documentationUrl,omitempty"`
	SupportURL          string `json:"supportUrl,omitempty"`
}

type ServicePlan struct {
	ID            string               `json:"id"`
	Name          string               `json:"name"`
	Description   string               `json:"description"`
	Metadata      *ServicePlanMetadata `json:"metadata,omitempty"`
	Free          bool                 `json:"free"`
	SQSProperties SQSProperties        `json:"sqs_properties,omitempty"`
}

type ServicePlanMetadata struct {
	Bullets     []string `json:"bullets,omitempty"`
	Costs       []Cost   `json:"costs,omitempty"`
	DisplayName string   `json:"displayName,omitempty"`
}

type DashboardClient struct {
	ID          string `json:"id,omitempty"`
	Secret      string `json:"secret,omitempty"`
	RedirectURI string `json:"redirect_uri,omitempty"`
}

type Cost struct {
	Amount map[string]interface{} `json:"amount,omitempty"`
	Unit   string                 `json:"unit,omitempty"`
}

type SQSProperties struct {
	DelaySeconds                  string `json:"delay_seconds,omitempty"`
	MaximumMessageSize            string `json:"maximum_message_size,omitempty"`
	MessageRetentionPeriod        string `json:"message_retention_period,omitempty"`
	ReceiveMessageWaitTimeSeconds string `json:"receive_message_wait_time_seconds,omitempty"`
	VisibilityTimeout             string `json:"visibility_timeout,omitempty"`
}

func (c Catalog) Validate() error {
	for _, service := range c.Services {
		if err := service.Validate(); err != nil {
			return fmt.Errorf("Validating Services configuration: %s", err)
		}
	}

	return nil
}

func (c Catalog) FindService(serviceID string) (service Service, found bool) {
	for _, service := range c.Services {
		if service.ID == serviceID {
			return service, true
		}
	}

	return service, false
}

func (c Catalog) FindServicePlan(planID string) (plan ServicePlan, found bool) {
	for _, service := range c.Services {
		for _, plan := range service.Plans {
			if plan.ID == planID {
				return plan, true
			}
		}
	}

	return plan, false
}

func (s Service) Validate() error {
	if s.ID == "" {
		return fmt.Errorf("Must provide a non-empty ID (%+v)", s)
	}

	if s.Name == "" {
		return fmt.Errorf("Must provide a non-empty Name (%+v)", s)
	}

	if s.Description == "" {
		return fmt.Errorf("Must provide a non-empty Description (%+v)", s)
	}

	for _, servicePlan := range s.Plans {
		if err := servicePlan.Validate(); err != nil {
			return fmt.Errorf("Validating Plans configuration: %s", err)
		}
	}

	return nil
}

func (sp ServicePlan) Validate() error {
	if sp.ID == "" {
		return fmt.Errorf("Must provide a non-empty ID (%+v)", sp)
	}

	if sp.Name == "" {
		return fmt.Errorf("Must provide a non-empty Name (%+v)", sp)
	}

	if sp.Description == "" {
		return fmt.Errorf("Must provide a non-empty Description (%+v)", sp)
	}

	if err := sp.SQSProperties.Validate(); err != nil {
		return fmt.Errorf("Validating SQS Properties configuration: %s", err)
	}

	return nil
}

func (sq SQSProperties) Validate() error {

	return nil
}
<|endoftext|>


<commit_before>package manager

import (
	"sort"
	"sync"
	"time"

	"github.com/prometheus/common/model"
	"github.com/prometheus/log"
)

// Dispatcher dispatches alerts. It is absed on the alert's data
// rather than the time they arrive. Thus it can recover it's state
// without persistence.
type Dispatcher struct {
	state State

	aggrGroups map[model.Fingerprint]*aggrGroup
}

func NewDispatcher(state State) *Dispatcher {
	return &Dispatcher{
		state:      state,
		aggrGroups: map[model.Fingerprint]*aggrGroup{},
	}
}

func (d *Dispatcher) notify(name string, alerts ...*Alert) {
	n := &LogNotifier{}
	i := []interface{}{name, "::"}
	for _, a := range alerts {
		i = append(i, a)
	}
	n.Send(i...)
}

func (d *Dispatcher) Run() {
	for {
		log.Infoln("waiting")
		alert := d.state.Alert().Next()

		log.Infoln("received:", alert)

		conf, err := d.state.Config().Get()
		if err != nil {
			log.Error(err)
			continue
		}
		log.Infoln("retrieved config")

		for _, m := range conf.Routes.Match(alert.Labels) {
			d.processAlert(alert, m)
		}
		log.Infoln("processing done")
	}
}

func (d *Dispatcher) processAlert(alert *Alert, opts *RouteOpts) {
	group := model.LabelSet{}

	for ln, lv := range alert.Labels {
		if _, ok := opts.GroupBy[ln]; ok {
			group[ln] = lv
		}
	}

	fp := group.Fingerprint()

	ag, ok := d.aggrGroups[fp]
	if !ok {
		ag = &aggrGroup{
			dispatcher: d,

			labels:    group,
			wait:      opts.GroupWait(),
			waitTimer: time.NewTimer(time.Hour),
			notify:    opts.SendTo,
		}
		d.aggrGroups[fp] = ag

		go ag.run()
	}

	ag.insert(alert)
}

// aggrGroup aggregates alerts into groups based on
// common values for a set of labels.
type aggrGroup struct {
	dispatcher *Dispatcher

	labels    model.LabelSet
	alertsOld alertTimeline
	alertsNew alertTimeline
	notify    string

	wait      time.Duration
	waitTimer *time.Timer

	done chan bool
	mtx  sync.RWMutex
}

func (ag *aggrGroup) run() {

	ag.waitTimer.Stop()

	if ag.wait > 0 {
		ag.waitTimer.Reset(ag.wait)
	}

	for {
		select {
		case <-ag.waitTimer.C:
			log.Infoln("timer flush")
			ag.flush()
		case <-ag.done:
			return
		}
	}
}

func (ag *aggrGroup) stop() {
	ag.waitTimer.Stop()
	close(ag.done)
}

func (ag *aggrGroup) fingerprint() model.Fingerprint {
	return ag.labels.Fingerprint()
}

func (ag *aggrGroup) insert(alert *Alert) {
	log.Infoln("insert:", alert)

	ag.alertsNew = append(ag.alertsNew, alert)
	sort.Sort(ag.alertsNew)

	if alert.Timestamp.Add(ag.wait).Before(time.Now()) {
		ag.flush()
	}

	if ag.wait > 0 {
		ag.waitTimer.Reset(ag.wait)
	}
}

func (ag *aggrGroup) flush() {
	log.Infoln("flush")

	ag.mtx.Lock()
	defer ag.mtx.Unlock()

	ag.dispatcher.notify(ag.notify, ag.alertsNew...)

	ag.alertsOld = append(ag.alertsOld, ag.alertsNew...)
	ag.alertsNew = ag.alertsNew[:0]
}

type alertTimeline []*Alert

func (at alertTimeline) Len() int {
	return len(at)
}

func (at alertTimeline) Less(i, j int) bool {
	return at[i].Timestamp.Before(at[j].Timestamp)
}

func (at alertTimeline) Swap(i, j int) {
	at[i], at[j] = at[j], at[i]
}
<commit_msg>improve dispatcher code and fix concurrency<commit_after>package manager

import (
	"sort"
	"sync"
	"time"

	"github.com/prometheus/common/model"
	"github.com/prometheus/log"
)

// Dispatcher dispatches alerts. It is absed on the alert's data
// rather than the time they arrive. Thus it can recover it's state
// without persistence.
type Dispatcher struct {
	state State

	aggrGroups map[model.Fingerprint]*aggrGroup
}

func NewDispatcher(state State) *Dispatcher {
	return &Dispatcher{
		state:      state,
		aggrGroups: map[model.Fingerprint]*aggrGroup{},
	}
}

func (d *Dispatcher) notify(name string, alerts ...*Alert) {
	n := &LogNotifier{}
	i := []interface{}{name, "::"}
	for _, a := range alerts {
		i = append(i, a)
	}
	n.Send(i...)
}

func (d *Dispatcher) Run() {
	for {
		alert := d.state.Alert().Next()

		conf, err := d.state.Config().Get()
		if err != nil {
			log.Error(err)
			continue
		}

		for _, m := range conf.Routes.Match(alert.Labels) {
			d.processAlert(alert, m)
		}
	}
}

func (d *Dispatcher) processAlert(alert *Alert, opts *RouteOpts) {
	group := model.LabelSet{}

	for ln, lv := range alert.Labels {
		if _, ok := opts.GroupBy[ln]; ok {
			group[ln] = lv
		}
	}

	fp := group.Fingerprint()

	ag, ok := d.aggrGroups[fp]
	if !ok {
		ag = newAggrGroup(group, opts)
		d.aggrGroups[fp] = ag
	}

	ag.insert(alert)

	if ag.empty() {
		ag.stop()
		delete(d.aggrGroups, fp)
	}
}

// aggrGroup aggregates alerts into groups based on
// common values for a set of labels.
type aggrGroup struct {
	dispatcher *Dispatcher

	labels    model.LabelSet
	alertsOld alertTimeline
	alertsNew alertTimeline
	notify    string

	wait      time.Duration
	waitTimer *time.Timer

	done chan bool
	mtx  sync.RWMutex
}

// newAggrGroup returns a new aggregation group and starts background processing
// that sends notifications about the contained alerts.
func newAggrGroup(labels model.LabelSet, opts *RouteOpts) *aggrGroup {
	ag := &aggrGroup{
		dispatcher: d,

		labels:    group,
		wait:      opts.GroupWait(),
		waitTimer: time.NewTimer(opts.GroupWait()),
		notify:    opts.SendTo,
		done:      make(chan bool),
	}
	if ag.wait == 0 {
		ag.waitTimer.Stop()
	}

	go ag.run()

	return ag
}

func (ag *aggrGroup) run() {
	for {
		select {
		case <-ag.waitTimer.C:
			ag.flush()
		case <-ag.done:
			return
		}
	}
}

func (ag *aggrGroup) stop() {
	ag.waitTimer.Stop()
	close(ag.done)
}

func (ag *aggrGroup) fingerprint() model.Fingerprint {
	return ag.labels.Fingerprint()
}

// insert the alert into the aggregation group. If the aggregation group
// is empty afterwards, true is returned.
func (ag *aggrGroup) insert(alert *Alert) {
	ag.mtx.Lock()

	ag.alertsNew = append(ag.alertsNew, alert)
	sort.Sort(ag.alertsNew)

	ag.mtx.Unlock()

	// Immediately trigger a flush if the wait duration for this
	// alert is already over.
	if alert.Timestamp.Add(ag.wait).Before(time.Now()) {
		ag.flush()
	}

	if ag.wait > 0 {
		ag.waitTimer.Reset(ag.wait)
	}
}

func (ag *aggrGroup) empty() bool {
	ag.mtx.RLock()
	defer ag.mtx.RUnlock()

	return len(ag.alertsNew)+len(ag.alertsOld) == 0
}

// flush sends notifications for all new alerts.
func (ag *aggrGroup) flush() {
	ag.mtx.Lock()
	defer ag.mtx.Unlock()

	ag.dispatcher.notify(ag.notify, ag.alertsNew...)

	ag.alertsOld = append(ag.alertsOld, ag.alertsNew...)
	ag.alertsNew = ag.alertsNew[:0]
}

// alertTimeline is a list of alerts sorted by their timestamp.
type alertTimeline []*Alert

func (at alertTimeline) Len() int           { return len(at) }
func (at alertTimeline) Less(i, j int) bool { return at[i].Timestamp.Before(at[j].Timestamp) }
func (at alertTimeline) Swap(i, j int)      { at[i], at[j] = at[j], at[i] }
<|endoftext|>


<commit_before>package manager

import (
	"fmt"
	"sync"
	"time"

	"github.com/prometheus/common/model"
	"github.com/prometheus/log"
)

const ResolveTimeout = 15 * time.Second

// Dispatcher dispatches alerts. It is absed on the alert's data
// rather than the time they arrive. Thus it can recover it's state
// without persistence.
type Dispatcher struct {
	state State

	aggrGroups map[model.Fingerprint]*aggrGroup
	notifiers  map[string]Notifier

	mtx sync.RWMutex
}

func NewDispatcher(state State, notifiers []Notifier) *Dispatcher {
	disp := &Dispatcher{
		state:      state,
		aggrGroups: map[model.Fingerprint]*aggrGroup{},
		notifiers:  map[string]Notifier{},
	}

	for _, n := range notifiers {
		disp.notifiers[n.Name()] = n
	}

	return disp
}

func (d *Dispatcher) notify(name string, alerts ...*Alert) error {
	if len(alerts) == 0 {
		return nil
	}

	d.mtx.RLock()
	notifier, ok := d.notifiers[name]
	d.mtx.RUnlock()

	if !ok {
		return fmt.Errorf("notifier %q does not exist", name)
	}

	return notifier.Send(alerts...)
}

func (d *Dispatcher) Run() {

	updates := d.state.Alert().Iter()
	cleanup := time.Tick(30 * time.Second)

	for {
		select {
		case <-cleanup:
			// Cleanup routine.
			for _, ag := range d.aggrGroups {
				if ag.empty() {
					ag.stop()
					delete(d.aggrGroups, ag.fingerprint())
				}
			}

			// now := time.Now()

			// list, err := d.state.Alert().GetAll()
			// if err != nil {
			// 	log.Error(err)
			// }

			// for _, a := range list {
			// 	if a.Resolved() && a.ResolvedAt.Before(now.Sub(ResolveTimeout)) {
			// 		if err := d.state.Alert().Del(a.Fingerprint()); err != nil {
			// 			log.Errorf("error cleaning resolved alerts: %s", err)
			// 		}
			// 	}
			// }

		case alert := <-updates:

			conf, err := d.state.Config().Get()
			if err != nil {
				log.Error(err)
				continue
			}

			for _, m := range conf.Routes.Match(alert.Labels) {
				d.processAlert(alert, m)
			}

			if !alert.Resolved() {
				a := *alert
				a.ResolvedAt = alert.CreatedAt.Add(ResolveTimeout)

				// After the constant timeout update the alert to be resolved.
				go func(a Alert) {
					now := time.Now()

					if a.ResolvedAt.After(now) {
						time.Sleep(a.ResolvedAt.Sub(now))
					}
					if err := d.state.Alert().Add(&a); err != nil {
						log.Errorf("alert auto-resolve failed: %s", err)
					}
				}(a)
			}
		}
	}
}

func (d *Dispatcher) processAlert(alert *Alert, opts *RouteOpts) {
	group := model.LabelSet{}

	for ln, lv := range alert.Labels {
		if _, ok := opts.GroupBy[ln]; ok {
			group[ln] = lv
		}
	}

	fp := group.Fingerprint()

	ag, ok := d.aggrGroups[fp]
	if !ok {
		ag = newAggrGroup(d, group, opts)
		d.aggrGroups[fp] = ag
	}

	ag.insert(alert)
}

// Alert models an action triggered by Prometheus.
type Alert struct {
	// Label value pairs for purpose of aggregation, matching, and disposition
	// dispatching. This must minimally include an "alertname" label.
	Labels model.LabelSet `json:"labels"`

	// Extra key/value information which is not used for aggregation.
	Payload     map[string]string `json:"payload,omitempty"`
	Summary     string            `json:"summary,omitempty"`
	Description string            `json:"description,omitempty"`
	Runbook     string            `json:"runbook,omitempty"`

	CreatedAt  time.Time `json:"created_at,omitempty"`
	ResolvedAt time.Time `json:"resolved_at,omitempty"`

	// The authoritative timestamp.
	Timestamp time.Time `json:"timestamp"`
}

// Name returns the name of the alert. It is equivalent to the "alertname" label.
func (a *Alert) Name() string {
	return string(a.Labels[model.AlertNameLabel])
}

// Fingerprint returns a unique hash for the alert. It is equivalent to
// the fingerprint of the alert's label set.
func (a *Alert) Fingerprint() model.Fingerprint {
	return a.Labels.Fingerprint()
}

func (a *Alert) String() string {
	s := fmt.Sprintf("%s[%x]", a.Name(), a.Fingerprint())
	if a.Resolved() {
		return s + "[resolved]"
	}
	return s + "[active]"
}

func (a *Alert) Resolved() bool {
	return a.ResolvedAt.After(a.CreatedAt)
}

// aggrGroup aggregates alerts into groups based on
// common values for a set of labels.
type aggrGroup struct {
	dispatcher *Dispatcher

	labels model.LabelSet
	opts   *RouteOpts

	next *time.Timer
	done chan struct{}

	mtx     sync.RWMutex
	alerts  map[model.Fingerprint]struct{}
	hasSent bool
}

// newAggrGroup returns a new aggregation group and starts background processing
// that sends notifications about the contained alerts.
func newAggrGroup(d *Dispatcher, labels model.LabelSet, opts *RouteOpts) *aggrGroup {
	ag := &aggrGroup{
		dispatcher: d,

		labels: labels,
		opts:   opts,

		alerts: map[model.Fingerprint]struct{}{},
		done:   make(chan struct{}),

		// Set an initial one-time wait before flushing
		// the first batch of notifications.
		next: time.NewTimer(opts.GroupWait),
	}

	go ag.run()

	return ag
}

func (ag *aggrGroup) run() {

	defer ag.next.Stop()

	for {
		select {
		case <-ag.next.C:
			ag.flush()
			// Wait the configured interval before calling flush again.
			ag.next.Reset(ag.opts.GroupInterval)

		case <-ag.done:
			return
		}
	}
}

func (ag *aggrGroup) stop() {
	close(ag.done)
}

func (ag *aggrGroup) fingerprint() model.Fingerprint {
	return ag.labels.Fingerprint()
}

// insert the alert into the aggregation group. If the aggregation group
// is empty afterwards, true is returned.
func (ag *aggrGroup) insert(alert *Alert) {
	fp := alert.Fingerprint()

	ag.mtx.Lock()
	ag.alerts[fp] = struct{}{}
	ag.mtx.Unlock()

	// Immediately trigger a flush if the wait duration for this
	// alert is already over.
	if !ag.hasSent && alert.Timestamp.Add(ag.opts.GroupWait).Before(time.Now()) {
		ag.next.Reset(0)
	}
}

func (ag *aggrGroup) empty() bool {
	ag.mtx.RLock()
	defer ag.mtx.RUnlock()

	return len(ag.alerts) == 0
}

// flush sends notifications for all new alerts.
func (ag *aggrGroup) flush() {
	ag.mtx.Lock()
	defer ag.mtx.Unlock()

	var alerts []*Alert
	for fp := range ag.alerts {
		a, err := ag.dispatcher.state.Alert().Get(fp)
		if err != nil {
			log.Error(err)
			continue
		}
		// TODO(fabxc): only delete if notify successful.
		if a.Resolved() {
			delete(ag.alerts, fp)
		}
		alerts = append(alerts, a)
	}

	ag.dispatcher.notify(ag.opts.SendTo, alerts...)
	ag.hasSent = true
}

// alertTimeline is a list of alerts sorted by their timestamp.
type alertTimeline []*Alert

func (at alertTimeline) Len() int           { return len(at) }
func (at alertTimeline) Less(i, j int) bool { return at[i].Timestamp.Before(at[j].Timestamp) }
func (at alertTimeline) Swap(i, j int)      { at[i], at[j] = at[j], at[i] }
<commit_msg>calculate stale resolve time on initial insert<commit_after>package manager

import (
	"fmt"
	"sync"
	"time"

	"github.com/prometheus/common/model"
	"github.com/prometheus/log"
)

const ResolveTimeout = 15 * time.Second

// Dispatcher dispatches alerts. It is absed on the alert's data
// rather than the time they arrive. Thus it can recover it's state
// without persistence.
type Dispatcher struct {
	state State

	aggrGroups map[model.Fingerprint]*aggrGroup
	notifiers  map[string]Notifier

	mtx sync.RWMutex
}

func NewDispatcher(state State, notifiers []Notifier) *Dispatcher {
	disp := &Dispatcher{
		state:      state,
		aggrGroups: map[model.Fingerprint]*aggrGroup{},
		notifiers:  map[string]Notifier{},
	}

	for _, n := range notifiers {
		disp.notifiers[n.Name()] = n
	}

	return disp
}

func (d *Dispatcher) notify(name string, alerts ...*Alert) error {
	if len(alerts) == 0 {
		return nil
	}

	d.mtx.RLock()
	notifier, ok := d.notifiers[name]
	d.mtx.RUnlock()

	if !ok {
		return fmt.Errorf("notifier %q does not exist", name)
	}

	return notifier.Send(alerts...)
}

func (d *Dispatcher) Run() {

	updates := d.state.Alert().Iter()
	cleanup := time.Tick(30 * time.Second)

	for {
		select {
		case <-cleanup:
			// Cleanup routine.
			for _, ag := range d.aggrGroups {
				if ag.empty() {
					ag.stop()
					delete(d.aggrGroups, ag.fingerprint())
				}
			}

			// now := time.Now()

			// list, err := d.state.Alert().GetAll()
			// if err != nil {
			// 	log.Error(err)
			// }

			// for _, a := range list {
			// 	if a.Resolved() && a.ResolvedAt.Before(now.Sub(ResolveTimeout)) {
			// 		if err := d.state.Alert().Del(a.Fingerprint()); err != nil {
			// 			log.Errorf("error cleaning resolved alerts: %s", err)
			// 		}
			// 	}
			// }

		case alert := <-updates:

			conf, err := d.state.Config().Get()
			if err != nil {
				log.Error(err)
				continue
			}

			for _, m := range conf.Routes.Match(alert.Labels) {
				d.processAlert(alert, m)
			}

			if alert.ResolvedAt.IsZero() {
				alert.ResolvedAt = alert.CreatedAt.Add(ResolveTimeout)
			}
		}
	}
}

func (d *Dispatcher) processAlert(alert *Alert, opts *RouteOpts) {
	group := model.LabelSet{}

	for ln, lv := range alert.Labels {
		if _, ok := opts.GroupBy[ln]; ok {
			group[ln] = lv
		}
	}

	fp := group.Fingerprint()

	ag, ok := d.aggrGroups[fp]
	if !ok {
		ag = newAggrGroup(d, group, opts)
		d.aggrGroups[fp] = ag
	}

	ag.insert(alert)
}

// Alert models an action triggered by Prometheus.
type Alert struct {
	// Label value pairs for purpose of aggregation, matching, and disposition
	// dispatching. This must minimally include an "alertname" label.
	Labels model.LabelSet `json:"labels"`

	// Extra key/value information which is not used for aggregation.
	Payload     map[string]string `json:"payload,omitempty"`
	Summary     string            `json:"summary,omitempty"`
	Description string            `json:"description,omitempty"`
	Runbook     string            `json:"runbook,omitempty"`

	CreatedAt  time.Time `json:"created_at,omitempty"`
	ResolvedAt time.Time `json:"resolved_at,omitempty"`

	// The authoritative timestamp.
	Timestamp time.Time `json:"timestamp"`
}

// Name returns the name of the alert. It is equivalent to the "alertname" label.
func (a *Alert) Name() string {
	return string(a.Labels[model.AlertNameLabel])
}

// Fingerprint returns a unique hash for the alert. It is equivalent to
// the fingerprint of the alert's label set.
func (a *Alert) Fingerprint() model.Fingerprint {
	return a.Labels.Fingerprint()
}

func (a *Alert) String() string {
	s := fmt.Sprintf("%s[%s]", a.Name(), a.Fingerprint())
	if a.Resolved() {
		return s + "[resolved]"
	}
	return s + "[active]"
}

func (a *Alert) Resolved() bool {
	if a.ResolvedAt.IsZero() {
		return false
	}
	return !a.ResolvedAt.After(time.Now())
}

// aggrGroup aggregates alerts into groups based on
// common values for a set of labels.
type aggrGroup struct {
	dispatcher *Dispatcher

	labels model.LabelSet
	opts   *RouteOpts

	next *time.Timer
	done chan struct{}

	mtx     sync.RWMutex
	alerts  map[model.Fingerprint]struct{}
	hasSent bool
}

// newAggrGroup returns a new aggregation group and starts background processing
// that sends notifications about the contained alerts.
func newAggrGroup(d *Dispatcher, labels model.LabelSet, opts *RouteOpts) *aggrGroup {
	ag := &aggrGroup{
		dispatcher: d,

		labels: labels,
		opts:   opts,

		alerts: map[model.Fingerprint]struct{}{},
		done:   make(chan struct{}),

		// Set an initial one-time wait before flushing
		// the first batch of notifications.
		next: time.NewTimer(opts.GroupWait),
	}

	go ag.run()

	return ag
}

func (ag *aggrGroup) run() {

	defer ag.next.Stop()

	for {
		select {
		case <-ag.next.C:
			ag.flush()
			// Wait the configured interval before calling flush again.
			ag.next.Reset(ag.opts.GroupInterval)

		case <-ag.done:
			return
		}
	}
}

func (ag *aggrGroup) stop() {
	close(ag.done)
}

func (ag *aggrGroup) fingerprint() model.Fingerprint {
	return ag.labels.Fingerprint()
}

// insert the alert into the aggregation group. If the aggregation group
// is empty afterwards, true is returned.
func (ag *aggrGroup) insert(alert *Alert) {
	fp := alert.Fingerprint()

	ag.mtx.Lock()
	ag.alerts[fp] = struct{}{}
	ag.mtx.Unlock()

	// Immediately trigger a flush if the wait duration for this
	// alert is already over.
	if !ag.hasSent && alert.Timestamp.Add(ag.opts.GroupWait).Before(time.Now()) {
		ag.next.Reset(0)
	}
}

func (ag *aggrGroup) empty() bool {
	ag.mtx.RLock()
	defer ag.mtx.RUnlock()

	return len(ag.alerts) == 0
}

// flush sends notifications for all new alerts.
func (ag *aggrGroup) flush() {
	ag.mtx.Lock()
	defer ag.mtx.Unlock()

	var alerts []*Alert
	for fp := range ag.alerts {
		a, err := ag.dispatcher.state.Alert().Get(fp)
		if err != nil {
			log.Error(err)
			continue
		}
		// TODO(fabxc): only delete if notify successful.
		if a.Resolved() {
			delete(ag.alerts, fp)
		}
		alerts = append(alerts, a)
	}

	ag.dispatcher.notify(ag.opts.SendTo, alerts...)
	ag.hasSent = true
}

// alertTimeline is a list of alerts sorted by their timestamp.
type alertTimeline []*Alert

func (at alertTimeline) Len() int           { return len(at) }
func (at alertTimeline) Less(i, j int) bool { return at[i].Timestamp.Before(at[j].Timestamp) }
func (at alertTimeline) Swap(i, j int)      { at[i], at[j] = at[j], at[i] }
<|endoftext|>


<commit_before>// A package for mocking the JSON HTTP POST requests.
package poster

import (
	"io"
	"net/http"
	"net/http/httptest"
)

// A Poster is an interface for creating JSON HTTP POST requests.
type Poster interface {
	// Create a POST request.
	Post(url string, r io.Reader) (*http.Response, error)
}

type httpPoster struct {
	client *http.Client
}

func (self httpPoster) Post(url string, r io.Reader) (*http.Response, error) {
	return self.client.Post(url, "text/json", r)
}

// Create a Poster implementation with standart net/http library.
func Http() Poster {
	return httpPoster{&http.Client{}}
}

type PosterCloser interface {
	io.Closer
	Poster
}

type handlerPoster struct {
	srv *httptest.Server
}

func (self handlerPoster) Post(url string, r io.Reader) (*http.Response, error) {
	return http.Post(url, "text/json", r)
}

func (self handlerPoster) Close() error {
	self.srv.Close()
	return nil
}

// Create a Poster implementation with standart net/http/httptest library
// that creates POST requests to a specific http.Handler.
// You need to Close it after you're done.
// Close always returns nil for this implementation.
func Handle(h http.Handler) (PosterCloser, string) {
	res := handlerPoster{httptest.NewServer(h)}
	return res, res.srv.URL
}
<commit_msg>better docs for poster<commit_after>// A package for mocking the JSON HTTP POST requests.
package poster

import (
	"io"
	"net/http"
	"net/http/httptest"
)

// A Poster is an interface for creating JSON HTTP POST requests.
type Poster interface {
	// Create a POST request.
	Post(url string, r io.Reader) (*http.Response, error)
}

type httpPoster struct {
	client *http.Client
}

func (self httpPoster) Post(url string, r io.Reader) (*http.Response, error) {
	return self.client.Post(url, "text/json", r)
}

// Create a Poster implementation with standart net/http library.
func Http() Poster {
	return httpPoster{&http.Client{}}
}

type PosterCloser interface {
	io.Closer
	Poster
}

type handlerPoster struct {
	srv *httptest.Server
}

func (self handlerPoster) Post(url string, r io.Reader) (*http.Response, error) {
	return http.Post(url, "text/json", r)
}

func (self handlerPoster) Close() error {
	self.srv.Close()
	return nil
}

/*
Create a Poster implementation with standart net/http/httptest library
that creates POST requests to a specific http.Handler.

You need to Close it after you're done.
Close always returns nil for this implementation.
*/
func Handle(h http.Handler) (PosterCloser, string) {
	res := handlerPoster{httptest.NewServer(h)}
	return res, res.srv.URL
}
<|endoftext|>


<commit_before>package dagr

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"testing"
)

func TestCompiledPoint(t *testing.T) {
	const required = `service.some_event,host=example.local,pid=1234 depth=123.456,msg="a \"string\" of sorts",on=T,value=123i 1136214245000000000` + "\n"

	defer prepareLogger(t)()

	integer := new(Int)
	boolean := new(Bool)
	float := new(Float)
	str := new(String)

	integer.Set(123)
	boolean.Set(true)
	float.Set(123.456)
	str.Set(`a "string" of sorts`)

	var rb bytes.Buffer
	m := NewPoint(
		"service.some_event",
		Tags{"pid": fmt.Sprint(1234), "host": "example.local"},
		Fields{"value": integer, "depth": float, "on": boolean, "msg": str},
	)

	if _, err := m.WriteTo(&rb); err != nil {
		t.Fatal(err)
	}

	normal := rb.String()
	if normal != required {
		t.Errorf("[N] Expected %q\nGot %q", required, normal)
	}

	c := m.compile()
	var cb bytes.Buffer
	if _, err := c.WriteTo(&cb); err != nil {
		t.Fatal(err)
	}

	compiled := cb.String()
	if compiled != required {
		t.Errorf("[C] Expected %q\nGot %q", required, compiled)
	}

	if compiled != normal {
		t.Errorf("[CN] Expected %q\nGot %q", normal, compiled)
	}
}

func BenchmarkWriteMeasurement(b *testing.B) {
	const result = `service.some_event,host=example.local,pid=1234 depth=123.456,msg="a \"string\" of sorts",on=T,value=123i 1136214245000000000` + "\n"

	defer prepareLogger(b)()

	integer := new(Int)
	boolean := new(Bool)
	float := new(Float)
	str := new(String)

	integer.Set(123)
	boolean.Set(true)
	float.Set(123.456)
	str.Set(`a "string" of sorts`)

	m := NewPoint(
		"service.some_event",
		Tags{"pid": fmt.Sprint(1234), "host": "example.local"},
		Fields{"value": integer, "depth": float, "on": boolean, "msg": str},
	)

	for i := b.N; i > 0; i-- {
		WriteMeasurement(ioutil.Discard, m)
	}
}

func BenchmarkWriteMeasurement_Compiled(b *testing.B) {
	const result = `service.some_event,host=example.local,pid=1234 depth=123.456,msg="a \"string\" of sorts",on=T,value=123i 1136214245000000000` + "\n"

	defer prepareLogger(b)()

	integer := new(Int)
	boolean := new(Bool)
	float := new(Float)
	str := new(String)

	integer.Set(123)
	boolean.Set(true)
	float.Set(123.456)
	str.Set(`a "string" of sorts`)

	m := NewPoint(
		"service.some_event",
		Tags{"pid": fmt.Sprint(1234), "host": "example.local"},
		Fields{"value": integer, "depth": float, "on": boolean, "msg": str},
	).Compiled()

	for i := b.N; i > 0; i-- {
		WriteMeasurement(ioutil.Discard, m)
	}
}

func BenchmarkWriteMeasurement_Parallel(b *testing.B) {
	const result = `service.some_event,host=example.local,pid=1234 depth=123.456,msg="a \"string\" of sorts",on=T,value=123i 1136214245000000000` + "\n"

	defer prepareLogger(b)()

	integer := new(Int)
	boolean := new(Bool)
	float := new(Float)
	str := new(String)

	integer.Set(123)
	boolean.Set(true)
	float.Set(123.456)
	str.Set(`a "string" of sorts`)

	m := NewPoint(
		"service.some_event",
		Tags{"pid": fmt.Sprint(1234), "host": "example.local"},
		Fields{"value": integer, "depth": float, "on": boolean, "msg": str},
	)

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			WriteMeasurement(ioutil.Discard, m)
		}
	})
}

func BenchmarkWriteMeasurement_ParallelCompiled(b *testing.B) {
	const result = `service.some_event,host=example.local,pid=1234 depth=123.456,msg="a \"string\" of sorts",on=T,value=123i 1136214245000000000` + "\n"

	defer prepareLogger(b)()

	integer := new(Int)
	boolean := new(Bool)
	float := new(Float)
	str := new(String)

	integer.Set(123)
	boolean.Set(true)
	float.Set(123.456)
	str.Set(`a "string" of sorts`)

	m := NewPoint(
		"service.some_event",
		Tags{"pid": fmt.Sprint(1234), "host": "example.local"},
		Fields{"value": integer, "depth": float, "on": boolean, "msg": str},
	).Compiled()

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			WriteMeasurement(ioutil.Discard, m)
		}
	})
}
<commit_msg>Add TestWriteMeasurement<commit_after>package dagr

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"testing"
)

func TestWriteMeasurement(t *testing.T) {
	const required = `balderdash.things_baldered,host=example.local,pid=1234 on=T,value=123i 1136214245000000000` + "\n"

	defer prepareLogger(t)()

	integer := new(Int)
	boolean := new(Bool)

	boolean.Set(true)
	integer.Set(123)

	var rb bytes.Buffer
	m := NewPoint(
		"balderdash.things_baldered",
		Tags{"pid": fmt.Sprint(1234), "host": "example.local"},
		Fields{"value": integer, "on": boolean},
	)

	_, err := WriteMeasurement(&rb, m)
	if err != nil {
		t.Fatal(err)
	}

	if bs := rb.String(); bs != required {
		t.Errorf("Expected ---\n%s\n---\n\nGot ---\n%s\n---", required, bs)
	}
}

func TestCompiledPoint(t *testing.T) {
	const required = `service.some_event,host=example.local,pid=1234 depth=123.456,msg="a \"string\" of sorts",on=T,value=123i 1136214245000000000` + "\n"

	defer prepareLogger(t)()

	integer := new(Int)
	boolean := new(Bool)
	float := new(Float)
	str := new(String)

	integer.Set(123)
	boolean.Set(true)
	float.Set(123.456)
	str.Set(`a "string" of sorts`)

	var rb bytes.Buffer
	m := NewPoint(
		"service.some_event",
		Tags{"pid": fmt.Sprint(1234), "host": "example.local"},
		Fields{"value": integer, "depth": float, "on": boolean, "msg": str},
	)

	if _, err := m.WriteTo(&rb); err != nil {
		t.Fatal(err)
	}

	normal := rb.String()
	if normal != required {
		t.Errorf("[N] Expected %q\nGot %q", required, normal)
	}

	c := m.compile()
	var cb bytes.Buffer
	if _, err := c.WriteTo(&cb); err != nil {
		t.Fatal(err)
	}

	compiled := cb.String()
	if compiled != required {
		t.Errorf("[C] Expected %q\nGot %q", required, compiled)
	}

	if compiled != normal {
		t.Errorf("[CN] Expected %q\nGot %q", normal, compiled)
	}
}

func BenchmarkWriteMeasurement(b *testing.B) {
	const result = `service.some_event,host=example.local,pid=1234 depth=123.456,msg="a \"string\" of sorts",on=T,value=123i 1136214245000000000` + "\n"

	defer prepareLogger(b)()

	integer := new(Int)
	boolean := new(Bool)
	float := new(Float)
	str := new(String)

	integer.Set(123)
	boolean.Set(true)
	float.Set(123.456)
	str.Set(`a "string" of sorts`)

	m := NewPoint(
		"service.some_event",
		Tags{"pid": fmt.Sprint(1234), "host": "example.local"},
		Fields{"value": integer, "depth": float, "on": boolean, "msg": str},
	)

	for i := b.N; i > 0; i-- {
		WriteMeasurement(ioutil.Discard, m)
	}
}

func BenchmarkWriteMeasurement_Compiled(b *testing.B) {
	const result = `service.some_event,host=example.local,pid=1234 depth=123.456,msg="a \"string\" of sorts",on=T,value=123i 1136214245000000000` + "\n"

	defer prepareLogger(b)()

	integer := new(Int)
	boolean := new(Bool)
	float := new(Float)
	str := new(String)

	integer.Set(123)
	boolean.Set(true)
	float.Set(123.456)
	str.Set(`a "string" of sorts`)

	m := NewPoint(
		"service.some_event",
		Tags{"pid": fmt.Sprint(1234), "host": "example.local"},
		Fields{"value": integer, "depth": float, "on": boolean, "msg": str},
	).Compiled()

	for i := b.N; i > 0; i-- {
		WriteMeasurement(ioutil.Discard, m)
	}
}

func BenchmarkWriteMeasurement_Parallel(b *testing.B) {
	const result = `service.some_event,host=example.local,pid=1234 depth=123.456,msg="a \"string\" of sorts",on=T,value=123i 1136214245000000000` + "\n"

	defer prepareLogger(b)()

	integer := new(Int)
	boolean := new(Bool)
	float := new(Float)
	str := new(String)

	integer.Set(123)
	boolean.Set(true)
	float.Set(123.456)
	str.Set(`a "string" of sorts`)

	m := NewPoint(
		"service.some_event",
		Tags{"pid": fmt.Sprint(1234), "host": "example.local"},
		Fields{"value": integer, "depth": float, "on": boolean, "msg": str},
	)

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			WriteMeasurement(ioutil.Discard, m)
		}
	})
}

func BenchmarkWriteMeasurement_ParallelCompiled(b *testing.B) {
	const result = `service.some_event,host=example.local,pid=1234 depth=123.456,msg="a \"string\" of sorts",on=T,value=123i 1136214245000000000` + "\n"

	defer prepareLogger(b)()

	integer := new(Int)
	boolean := new(Bool)
	float := new(Float)
	str := new(String)

	integer.Set(123)
	boolean.Set(true)
	float.Set(123.456)
	str.Set(`a "string" of sorts`)

	m := NewPoint(
		"service.some_event",
		Tags{"pid": fmt.Sprint(1234), "host": "example.local"},
		Fields{"value": integer, "depth": float, "on": boolean, "msg": str},
	).Compiled()

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			WriteMeasurement(ioutil.Discard, m)
		}
	})
}
<|endoftext|>


<commit_before>package trash

import (
	"runtime"
	"time"

	"github.com/cozy/cozy-stack/model/job"
	"github.com/cozy/cozy-stack/model/vfs"
)

func init() {
	job.AddWorker(&job.WorkerConfig{
		WorkerType:   "trash-files",
		Concurrency:  runtime.NumCPU() * 4,
		MaxExecCount: 2,
		Reserved:     true,
		Timeout:      2 * time.Hour,
		WorkerFunc:   WorkerTrashFiles,
	})
}

// WorkerTrashFiles is a worker to remove files in Swift after they have been
// removed from CouchDB. It is used when cleaning the trash, as removing a lot
// of files from Swift can take some time.
func WorkerTrashFiles(ctx *job.WorkerContext) error {
	opts := vfs.TrashJournal{}
	err := ctx.UnmarshalMessage(&opts)
	if err != nil {
		return err
	}
	fs := ctx.Instance.VFS()
	return fs.EnsureErased(opts)
}
<commit_msg>Add a critical log if trash-files fails (#2112)<commit_after>package trash

import (
	"runtime"
	"time"

	"github.com/cozy/cozy-stack/model/job"
	"github.com/cozy/cozy-stack/model/vfs"
)

func init() {
	job.AddWorker(&job.WorkerConfig{
		WorkerType:   "trash-files",
		Concurrency:  runtime.NumCPU() * 4,
		MaxExecCount: 2,
		Reserved:     true,
		Timeout:      2 * time.Hour,
		WorkerFunc:   WorkerTrashFiles,
	})
}

// WorkerTrashFiles is a worker to remove files in Swift after they have been
// removed from CouchDB. It is used when cleaning the trash, as removing a lot
// of files from Swift can take some time.
func WorkerTrashFiles(ctx *job.WorkerContext) error {
	opts := vfs.TrashJournal{}
	if err := ctx.UnmarshalMessage(&opts); err != nil {
		return err
	}
	fs := ctx.Instance.VFS()
	if err := fs.EnsureErased(opts); err != nil {
		ctx.Logger().WithField("critical", "true").
			Errorf("Error: %s", err)
		return err
	}
	return nil
}
<|endoftext|>