<commit_before>// Package local provides the default implementation for volumes. It
// is used to mount data volume containers and directories local to
// the host server.
package local // import "github.com/docker/docker/volume/local"

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"sync"

	"github.com/docker/docker/daemon/names"
	"github.com/docker/docker/errdefs"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/quota"
	"github.com/docker/docker/volume"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	// volumeDataPathName is the name of the directory where the volume data is stored.
	// It uses a very distinctive name to avoid collisions migrating data between
	// Docker versions.
	volumeDataPathName = "_data"
	volumesPathName    = "volumes"
)

var (
	// ErrNotFound is the typed error returned when the requested volume name can't be found
	ErrNotFound = fmt.Errorf("volume not found")
	// volumeNameRegex ensures the name assigned for the volume is valid.
	// This name is used to create the bind directory, so we need to avoid characters that
	// would make the path to escape the root directory.
	volumeNameRegex = names.RestrictedNamePattern
)

type activeMount struct {
	count   uint64
	mounted bool
}

// New instantiates a new Root instance with the provided scope. Scope
// is the base path that the Root instance uses to store its
// volumes. The base path is created here if it does not exist.
func New(scope string, rootIdentity idtools.Identity) (*Root, error) {
	r := &Root{
		path:         filepath.Join(scope, volumesPathName),
		volumes:      make(map[string]*localVolume),
		rootIdentity: rootIdentity,
	}

	if err := idtools.MkdirAllAndChown(r.path, 0701, idtools.CurrentIdentity()); err != nil {
		return nil, err
	}

	dirs, err := os.ReadDir(r.path)
	if err != nil {
		return nil, err
	}

	if r.quotaCtl, err = quota.NewControl(r.path); err != nil {
		logrus.Debugf("No quota support for local volumes in %s: %v", r.path, err)
	}

	for _, d := range dirs {
		if !d.IsDir() {
			continue
		}

		name := d.Name()
		v := &localVolume{
			driverName: r.Name(),
			name:       name,
			path:       r.DataPath(name),
			quotaCtl:   r.quotaCtl,
		}
		r.volumes[name] = v
		if b, err := os.ReadFile(filepath.Join(r.path, name, "opts.json")); err == nil {
			opts := optsConfig{}
			if err := json.Unmarshal(b, &opts); err != nil {
				return nil, errors.Wrapf(err, "error while unmarshaling volume options for volume: %s", name)
			}
			// Make sure this isn't an empty optsConfig.
			// This could be empty due to buggy behavior in older versions of Docker.
			if !reflect.DeepEqual(opts, optsConfig{}) {
				v.opts = &opts
			}
			// unmount anything that may still be mounted (for example, from an
			// unclean shutdown). This is a no-op on windows
			unmount(v.path)
		}
	}

	return r, nil
}

// Root implements the Driver interface for the volume package and
// manages the creation/removal of volumes. It uses only standard vfs
// commands to create/remove dirs within its provided scope.
type Root struct {
	m            sync.Mutex
	path         string
	quotaCtl     *quota.Control
	volumes      map[string]*localVolume
	rootIdentity idtools.Identity
}

// List lists all the volumes
func (r *Root) List() ([]volume.Volume, error) {
	var ls []volume.Volume
	r.m.Lock()
	for _, v := range r.volumes {
		ls = append(ls, v)
	}
	r.m.Unlock()
	return ls, nil
}

// DataPath returns the constructed path of this volume.
func (r *Root) DataPath(volumeName string) string {
	return filepath.Join(r.path, volumeName, volumeDataPathName)
}

// Name returns the name of Root, defined in the volume package in the DefaultDriverName constant.
func (r *Root) Name() string {
	return volume.DefaultDriverName
}

// Create creates a new volume.Volume with the provided name, creating
// the underlying directory tree required for this volume in the
// process.
func (r *Root) Create(name string, opts map[string]string) (volume.Volume, error) {
	if err := r.validateName(name); err != nil {
		return nil, err
	}

	r.m.Lock()
	defer r.m.Unlock()

	v, exists := r.volumes[name]
	if exists {
		return v, nil
	}

	path := r.DataPath(name)
	volRoot := filepath.Dir(path)
	// Root dir does not need to be accessed by the remapped root
	if err := idtools.MkdirAllAndChown(volRoot, 0701, idtools.CurrentIdentity()); err != nil {
		return nil, errors.Wrapf(errdefs.System(err), "error while creating volume root path '%s'", volRoot)
	}

	// Remapped root does need access to the data path
	if err := idtools.MkdirAllAndChown(path, 0755, r.rootIdentity); err != nil {
		return nil, errors.Wrapf(errdefs.System(err), "error while creating volume data path '%s'", path)
	}

	var err error
	defer func() {
		if err != nil {
			os.RemoveAll(filepath.Dir(path))
		}
	}()

	v = &localVolume{
		driverName: r.Name(),
		name:       name,
		path:       path,
		quotaCtl:   r.quotaCtl,
	}

	if len(opts) != 0 {
		if err = setOpts(v, opts); err != nil {
			return nil, err
		}
		var b []byte
		b, err = json.Marshal(v.opts)
		if err != nil {
			return nil, err
		}
		if err = os.WriteFile(filepath.Join(filepath.Dir(path), "opts.json"), b, 0600); err != nil {
			return nil, errdefs.System(errors.Wrap(err, "error while persisting volume options"))
		}
	}

	r.volumes[name] = v
	return v, nil
}

// Remove removes the specified volume and all underlying data. If the
// given volume does not belong to this driver and an error is
// returned. The volume is reference counted, if all references are
// not released then the volume is not removed.
func (r *Root) Remove(v volume.Volume) error {
	r.m.Lock()
	defer r.m.Unlock()

	lv, ok := v.(*localVolume)
	if !ok {
		return errdefs.System(errors.Errorf("unknown volume type %T", v))
	}

	if lv.active.count > 0 {
		return errdefs.System(errors.Errorf("volume has active mounts"))
	}

	if err := lv.unmount(); err != nil {
		return err
	}

	realPath, err := filepath.EvalSymlinks(lv.path)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
		realPath = filepath.Dir(lv.path)
	}

	if realPath == r.path || !strings.HasPrefix(realPath, r.path) {
		return errdefs.System(errors.Errorf("unable to remove a directory outside of the local volume root %s: %s", r.path, realPath))
	}

	if err := removePath(realPath); err != nil {
		return err
	}

	delete(r.volumes, lv.name)
	return removePath(filepath.Dir(lv.path))
}

func removePath(path string) error {
	if err := os.RemoveAll(path); err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return errdefs.System(errors.Wrapf(err, "error removing volume path '%s'", path))
	}
	return nil
}

// Get looks up the volume for the given name and returns it if found
func (r *Root) Get(name string) (volume.Volume, error) {
	r.m.Lock()
	v, exists := r.volumes[name]
	r.m.Unlock()
	if !exists {
		return nil, ErrNotFound
	}
	return v, nil
}

// Scope returns the local volume scope
func (r *Root) Scope() string {
	return volume.LocalScope
}

func (r *Root) validateName(name string) error {
	if len(name) == 1 {
		return errdefs.InvalidParameter(errors.New("volume name is too short, names should be at least two alphanumeric characters"))
	}
	if !volumeNameRegex.MatchString(name) {
		return errdefs.InvalidParameter(errors.Errorf("%q includes invalid characters for a local volume name, only %q are allowed. If you intended to pass a host directory, use absolute path", name, names.RestrictedNameChars))
	}
	return nil
}

// localVolume implements the Volume interface from the volume package and
// represents the volumes created by Root.
type localVolume struct {
	m sync.Mutex
	// unique name of the volume
	name string
	// path is the path on the host where the data lives
	path string
	// driverName is the name of the driver that created the volume.
	driverName string
	// opts is the parsed list of options used to create the volume
	opts *optsConfig
	// active refcounts the active mounts
	active activeMount
	// reference to Root instances quotaCtl
	quotaCtl *quota.Control
}

// Name returns the name of the given Volume.
func (v *localVolume) Name() string {
	return v.name
}

// DriverName returns the driver that created the given Volume.
func (v *localVolume) DriverName() string {
	return v.driverName
}

// Path returns the data location.
func (v *localVolume) Path() string {
	return v.path
}

// CachedPath returns the data location
func (v *localVolume) CachedPath() string {
	return v.path
}

// Mount implements the localVolume interface, returning the data location.
// If there are any provided mount options, the resources will be mounted at this point
func (v *localVolume) Mount(id string) (string, error) {
	v.m.Lock()
	defer v.m.Unlock()
	if v.needsMount() {
		if !v.active.mounted {
			if err := v.mount(); err != nil {
				return "", errdefs.System(err)
			}
			v.active.mounted = true
		}
		v.active.count++
	}
	if err := v.postMount(); err != nil {
		return "", err
	}
	return v.path, nil
}

// Unmount dereferences the id, and if it is the last reference will unmount any resources
// that were previously mounted.
func (v *localVolume) Unmount(id string) error {
	v.m.Lock()
	defer v.m.Unlock()

	// Always decrement the count, even if the unmount fails
	// Essentially docker doesn't care if this fails, it will send an error, but
	// ultimately there's nothing that can be done. If we don't decrement the count
	// this volume can never be removed until a daemon restart occurs.
	if v.needsMount() {
		v.active.count--
	}

	if v.active.count > 0 {
		return nil
	}

	return v.unmount()
}

func (v *localVolume) Status() map[string]interface{} {
	return nil
}

// getAddress finds out address/hostname from options
func getAddress(opts string) string {
	optsList := strings.Split(opts, ",")
	for i := 0; i < len(optsList); i++ {
		if strings.HasPrefix(optsList[i], "addr=") {
			addr := strings.SplitN(optsList[i], "=", 2)[1]
			return addr
		}
	}
	return ""
}

// getPassword finds out a password from options
func getPassword(opts string) string {
	optsList := strings.Split(opts, ",")
	for i := 0; i < len(optsList); i++ {
		if strings.HasPrefix(optsList[i], "password=") {
			passwd := strings.SplitN(optsList[i], "=", 2)[1]
			return passwd
		}
	}
	return ""
}
<commit_msg>volume/local.New(): don't register volume before we're done<commit_after>// Package local provides the default implementation for volumes. It
// is used to mount data volume containers and directories local to
// the host server.
package local // import "github.com/docker/docker/volume/local"

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"sync"

	"github.com/docker/docker/daemon/names"
	"github.com/docker/docker/errdefs"
	"github.com/docker/docker/pkg/idtools"
	"github.com/docker/docker/quota"
	"github.com/docker/docker/volume"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	// volumeDataPathName is the name of the directory where the volume data is stored.
	// It uses a very distinctive name to avoid collisions migrating data between
	// Docker versions.
	volumeDataPathName = "_data"
	volumesPathName    = "volumes"
)

var (
	// ErrNotFound is the typed error returned when the requested volume name can't be found
	ErrNotFound = fmt.Errorf("volume not found")
	// volumeNameRegex ensures the name assigned for the volume is valid.
	// This name is used to create the bind directory, so we need to avoid characters that
	// would make the path to escape the root directory.
	volumeNameRegex = names.RestrictedNamePattern
)

type activeMount struct {
	count   uint64
	mounted bool
}

// New instantiates a new Root instance with the provided scope. Scope
// is the base path that the Root instance uses to store its
// volumes. The base path is created here if it does not exist.
func New(scope string, rootIdentity idtools.Identity) (*Root, error) {
	r := &Root{
		path:         filepath.Join(scope, volumesPathName),
		volumes:      make(map[string]*localVolume),
		rootIdentity: rootIdentity,
	}

	if err := idtools.MkdirAllAndChown(r.path, 0701, idtools.CurrentIdentity()); err != nil {
		return nil, err
	}

	dirs, err := os.ReadDir(r.path)
	if err != nil {
		return nil, err
	}

	if r.quotaCtl, err = quota.NewControl(r.path); err != nil {
		logrus.Debugf("No quota support for local volumes in %s: %v", r.path, err)
	}

	for _, d := range dirs {
		if !d.IsDir() {
			continue
		}

		name := d.Name()
		v := &localVolume{
			driverName: r.Name(),
			name:       name,
			path:       r.DataPath(name),
			quotaCtl:   r.quotaCtl,
		}
		if b, err := os.ReadFile(filepath.Join(r.path, name, "opts.json")); err == nil {
			opts := optsConfig{}
			if err := json.Unmarshal(b, &opts); err != nil {
				return nil, errors.Wrapf(err, "error while unmarshaling volume options for volume: %s", name)
			}
			// Make sure this isn't an empty optsConfig.
			// This could be empty due to buggy behavior in older versions of Docker.
			if !reflect.DeepEqual(opts, optsConfig{}) {
				v.opts = &opts
			}
			// unmount anything that may still be mounted (for example, from an
			// unclean shutdown). This is a no-op on windows
			unmount(v.path)
		}
		r.volumes[name] = v
	}

	return r, nil
}

// Root implements the Driver interface for the volume package and
// manages the creation/removal of volumes. It uses only standard vfs
// commands to create/remove dirs within its provided scope.
type Root struct {
	m            sync.Mutex
	path         string
	quotaCtl     *quota.Control
	volumes      map[string]*localVolume
	rootIdentity idtools.Identity
}

// List lists all the volumes
func (r *Root) List() ([]volume.Volume, error) {
	var ls []volume.Volume
	r.m.Lock()
	for _, v := range r.volumes {
		ls = append(ls, v)
	}
	r.m.Unlock()
	return ls, nil
}

// DataPath returns the constructed path of this volume.
func (r *Root) DataPath(volumeName string) string {
	return filepath.Join(r.path, volumeName, volumeDataPathName)
}

// Name returns the name of Root, defined in the volume package in the DefaultDriverName constant.
func (r *Root) Name() string {
	return volume.DefaultDriverName
}

// Create creates a new volume.Volume with the provided name, creating
// the underlying directory tree required for this volume in the
// process.
func (r *Root) Create(name string, opts map[string]string) (volume.Volume, error) {
	if err := r.validateName(name); err != nil {
		return nil, err
	}

	r.m.Lock()
	defer r.m.Unlock()

	v, exists := r.volumes[name]
	if exists {
		return v, nil
	}

	path := r.DataPath(name)
	volRoot := filepath.Dir(path)
	// Root dir does not need to be accessed by the remapped root
	if err := idtools.MkdirAllAndChown(volRoot, 0701, idtools.CurrentIdentity()); err != nil {
		return nil, errors.Wrapf(errdefs.System(err), "error while creating volume root path '%s'", volRoot)
	}

	// Remapped root does need access to the data path
	if err := idtools.MkdirAllAndChown(path, 0755, r.rootIdentity); err != nil {
		return nil, errors.Wrapf(errdefs.System(err), "error while creating volume data path '%s'", path)
	}

	var err error
	defer func() {
		if err != nil {
			os.RemoveAll(filepath.Dir(path))
		}
	}()

	v = &localVolume{
		driverName: r.Name(),
		name:       name,
		path:       path,
		quotaCtl:   r.quotaCtl,
	}

	if len(opts) != 0 {
		if err = setOpts(v, opts); err != nil {
			return nil, err
		}
		var b []byte
		b, err = json.Marshal(v.opts)
		if err != nil {
			return nil, err
		}
		if err = os.WriteFile(filepath.Join(filepath.Dir(path), "opts.json"), b, 0600); err != nil {
			return nil, errdefs.System(errors.Wrap(err, "error while persisting volume options"))
		}
	}

	r.volumes[name] = v
	return v, nil
}

// Remove removes the specified volume and all underlying data. If the
// given volume does not belong to this driver and an error is
// returned. The volume is reference counted, if all references are
// not released then the volume is not removed.
func (r *Root) Remove(v volume.Volume) error {
	r.m.Lock()
	defer r.m.Unlock()

	lv, ok := v.(*localVolume)
	if !ok {
		return errdefs.System(errors.Errorf("unknown volume type %T", v))
	}

	if lv.active.count > 0 {
		return errdefs.System(errors.Errorf("volume has active mounts"))
	}

	if err := lv.unmount(); err != nil {
		return err
	}

	realPath, err := filepath.EvalSymlinks(lv.path)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
		realPath = filepath.Dir(lv.path)
	}

	if realPath == r.path || !strings.HasPrefix(realPath, r.path) {
		return errdefs.System(errors.Errorf("unable to remove a directory outside of the local volume root %s: %s", r.path, realPath))
	}

	if err := removePath(realPath); err != nil {
		return err
	}

	delete(r.volumes, lv.name)
	return removePath(filepath.Dir(lv.path))
}

func removePath(path string) error {
	if err := os.RemoveAll(path); err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return errdefs.System(errors.Wrapf(err, "error removing volume path '%s'", path))
	}
	return nil
}

// Get looks up the volume for the given name and returns it if found
func (r *Root) Get(name string) (volume.Volume, error) {
	r.m.Lock()
	v, exists := r.volumes[name]
	r.m.Unlock()
	if !exists {
		return nil, ErrNotFound
	}
	return v, nil
}

// Scope returns the local volume scope
func (r *Root) Scope() string {
	return volume.LocalScope
}

func (r *Root) validateName(name string) error {
	if len(name) == 1 {
		return errdefs.InvalidParameter(errors.New("volume name is too short, names should be at least two alphanumeric characters"))
	}
	if !volumeNameRegex.MatchString(name) {
		return errdefs.InvalidParameter(errors.Errorf("%q includes invalid characters for a local volume name, only %q are allowed. If you intended to pass a host directory, use absolute path", name, names.RestrictedNameChars))
	}
	return nil
}

// localVolume implements the Volume interface from the volume package and
// represents the volumes created by Root.
type localVolume struct {
	m sync.Mutex
	// unique name of the volume
	name string
	// path is the path on the host where the data lives
	path string
	// driverName is the name of the driver that created the volume.
	driverName string
	// opts is the parsed list of options used to create the volume
	opts *optsConfig
	// active refcounts the active mounts
	active activeMount
	// reference to Root instances quotaCtl
	quotaCtl *quota.Control
}

// Name returns the name of the given Volume.
func (v *localVolume) Name() string {
	return v.name
}

// DriverName returns the driver that created the given Volume.
func (v *localVolume) DriverName() string {
	return v.driverName
}

// Path returns the data location.
func (v *localVolume) Path() string {
	return v.path
}

// CachedPath returns the data location
func (v *localVolume) CachedPath() string {
	return v.path
}

// Mount implements the localVolume interface, returning the data location.
// If there are any provided mount options, the resources will be mounted at this point
func (v *localVolume) Mount(id string) (string, error) {
	v.m.Lock()
	defer v.m.Unlock()
	if v.needsMount() {
		if !v.active.mounted {
			if err := v.mount(); err != nil {
				return "", errdefs.System(err)
			}
			v.active.mounted = true
		}
		v.active.count++
	}
	if err := v.postMount(); err != nil {
		return "", err
	}
	return v.path, nil
}

// Unmount dereferences the id, and if it is the last reference will unmount any resources
// that were previously mounted.
func (v *localVolume) Unmount(id string) error {
	v.m.Lock()
	defer v.m.Unlock()

	// Always decrement the count, even if the unmount fails
	// Essentially docker doesn't care if this fails, it will send an error, but
	// ultimately there's nothing that can be done. If we don't decrement the count
	// this volume can never be removed until a daemon restart occurs.
	if v.needsMount() {
		v.active.count--
	}

	if v.active.count > 0 {
		return nil
	}

	return v.unmount()
}

func (v *localVolume) Status() map[string]interface{} {
	return nil
}

// getAddress finds out address/hostname from options
func getAddress(opts string) string {
	optsList := strings.Split(opts, ",")
	for i := 0; i < len(optsList); i++ {
		if strings.HasPrefix(optsList[i], "addr=") {
			addr := strings.SplitN(optsList[i], "=", 2)[1]
			return addr
		}
	}
	return ""
}

// getPassword finds out a password from options
func getPassword(opts string) string {
	optsList := strings.Split(opts, ",")
	for i := 0; i < len(optsList); i++ {
		if strings.HasPrefix(optsList[i], "password=") {
			passwd := strings.SplitN(optsList[i], "=", 2)[1]
			return passwd
		}
	}
	return ""
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package action

import (
	"bytes"
	"fmt"
	"time"

	"github.com/pkg/errors"

	"helm.sh/helm/pkg/release"
)

// ReleaseTesting is the action for testing a release.
//
// It provides the implementation of 'helm test'.
type ReleaseTesting struct {
	cfg *Configuration

	Timeout time.Duration
	Cleanup bool
}

// NewReleaseTesting creates a new ReleaseTesting object with the given configuration.
func NewReleaseTesting(cfg *Configuration) *ReleaseTesting {
	return &ReleaseTesting{
		cfg: cfg,
	}
}

// Run executes 'helm test' against the given release.
func (r *ReleaseTesting) Run(name string) error {
	if err := validateReleaseName(name); err != nil {
		return errors.Errorf("releaseTest: Release name is invalid: %s", name)
	}

	// finds the non-deleted release with the given name
	rel, err := r.cfg.Releases.Last(name)
	if err != nil {
		return err
	}

	if err := r.cfg.execHook(rel, release.HookTest, r.Timeout); err != nil {
		r.cfg.Releases.Update(rel)
		return err
	}

	if r.Cleanup {
		for _, h := range rel.Hooks {
			for _, e := range h.Events {
				if e == release.HookTest {
					hookResource, err := r.cfg.KubeClient.Build(bytes.NewBufferString(h.Manifest))
					if err != nil {
						return errors.Wrapf(err, "unable to build kubernetes object for %v hook %s", h, h.Path)
					}
					if _, errs := r.cfg.KubeClient.Delete(hookResource); errs != nil {
						return fmt.Errorf("unable to delete kubernetes object for %v hook %s: %s", h, h.Path, joinErrors(errs))
					}
				}
			}
		}
	}

	return r.cfg.Releases.Update(rel)
}
<commit_msg>ref(test): join all hook manifests before building<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package action

import (
	"bytes"
	"fmt"
	"strings"
	"time"

	"github.com/pkg/errors"

	"helm.sh/helm/pkg/release"
)

// ReleaseTesting is the action for testing a release.
//
// It provides the implementation of 'helm test'.
type ReleaseTesting struct {
	cfg *Configuration

	Timeout time.Duration
	Cleanup bool
}

// NewReleaseTesting creates a new ReleaseTesting object with the given configuration.
func NewReleaseTesting(cfg *Configuration) *ReleaseTesting {
	return &ReleaseTesting{
		cfg: cfg,
	}
}

// Run executes 'helm test' against the given release.
func (r *ReleaseTesting) Run(name string) error {
	if err := validateReleaseName(name); err != nil {
		return errors.Errorf("releaseTest: Release name is invalid: %s", name)
	}

	// finds the non-deleted release with the given name
	rel, err := r.cfg.Releases.Last(name)
	if err != nil {
		return err
	}

	if err := r.cfg.execHook(rel, release.HookTest, r.Timeout); err != nil {
		r.cfg.Releases.Update(rel)
		return err
	}

	if r.Cleanup {
		var manifestsToDelete strings.Builder
		for _, h := range rel.Hooks {
			for _, e := range h.Events {
				if e == release.HookTest {
					fmt.Fprintf(&manifestsToDelete, "\n---\n%s", h.Manifest)
				}
			}
		}
		hooks, err := r.cfg.KubeClient.Build(bytes.NewBufferString(manifestsToDelete.String()))
		if err != nil {
			return fmt.Errorf("unable to build test hooks: %v", err)
		}
		if _, errs := r.cfg.KubeClient.Delete(hooks); errs != nil {
			return fmt.Errorf("unable to delete test hooks: %v", joinErrors(errs))
		}
	}

	return r.cfg.Releases.Update(rel)
}
<|endoftext|>


<commit_before>package cloudwatch

import (
	"encoding/json"
	"sort"

	"github.com/grafana/grafana/pkg/middleware"
	"github.com/grafana/grafana/pkg/util"
)

var metricsMap map[string][]string
var dimensionsMap map[string][]string

func init() {
	metricsMap = map[string][]string{
		"AWS/AutoScaling": {"GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity", "GroupInServiceInstances", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"},
		"AWS/Billing":     {"EstimatedCharges"},
		"AWS/CloudFront":  {"Requests", "BytesDownloaded", "BytesUploaded", "TotalErrorRate", "4xxErrorRate", "5xxErrorRate"},
		"AWS/CloudSearch": {"SuccessfulRequests", "SearchableDocuments", "IndexUtilization", "Partitions"},
		"AWS/DynamoDB":    {"ConditionalCheckFailedRequests", "ConsumedReadCapacityUnits", "ConsumedWriteCapacityUnits", "OnlineIndexConsumedWriteCapacity", "OnlineIndexPercentageProgress", "OnlineIndexThrottleEvents", "ProvisionedReadCapacityUnits", "ProvisionedWriteCapacityUnits", "ReadThrottleEvents", "ReturnedItemCount", "SuccessfulRequestLatency", "SystemErrors", "ThrottledRequests", "UserErrors", "WriteThrottleEvents"},
		"AWS/ECS":         {"CPUUtilization", "MemoryUtilization"},
		"AWS/ElastiCache": {
			"CPUUtilization", "SwapUsage", "FreeableMemory", "NetworkBytesIn", "NetworkBytesOut",
			"BytesUsedForCacheItems", "BytesReadIntoMemcached", "BytesWrittenOutFromMemcached", "CasBadval", "CasHits", "CasMisses", "CmdFlush", "CmdGet", "CmdSet", "CurrConnections", "CurrItems", "DecrHits", "DecrMisses", "DeleteHits", "DeleteMisses", "Evictions", "GetHits", "GetMisses", "IncrHits", "IncrMisses", "Reclaimed",
			"CurrConnections", "Evictions", "Reclaimed", "NewConnections", "BytesUsedForCache", "CacheHits", "CacheMisses", "ReplicationLag", "GetTypeCmds", "SetTypeCmds", "KeyBasedCmds", "StringBasedCmds", "HashBasedCmds", "ListBasedCmds", "SetBasedCmds", "SortedSetBasedCmds", "CurrItems",
		},
		"AWS/EBS":              {"VolumeReadBytes", "VolumeWriteBytes", "VolumeReadOps", "VolumeWriteOps", "VolumeTotalReadTime", "VolumeTotalWriteTime", "VolumeIdleTime", "VolumeQueueLength", "VolumeThroughputPercentage", "VolumeConsumedReadWriteOps"},
		"AWS/EC2":              {"CPUCreditUsage", "CPUCreditBalance", "CPUUtilization", "DiskReadOps", "DiskWriteOps", "DiskReadBytes", "DiskWriteBytes", "NetworkIn", "NetworkOut", "StatusCheckFailed", "StatusCheckFailed_Instance", "StatusCheckFailed_System"},
		"AWS/ELB":              {"HealthyHostCount", "UnHealthyHostCount", "RequestCount", "Latency", "HTTPCode_ELB_4XX", "HTTPCode_ELB_5XX", "HTTPCode_Backend_2XX", "HTTPCode_Backend_3XX", "HTTPCode_Backend_4XX", "HTTPCode_Backend_5XX", "BackendConnectionErrors", "SurgeQueueLength", "SpilloverCount"},
		"AWS/ElasticMapReduce": {"CoreNodesPending", "CoreNodesRunning", "HBaseBackupFailed", "HBaseMostRecentBackupDuration", "HBaseTimeSinceLastSuccessfulBackup", "HDFSBytesRead", "HDFSBytesWritten", "HDFSUtilization", "IsIdle", "JobsFailed", "JobsRunning", "LiveDataNodes", "LiveTaskTrackers", "MapSlotsOpen", "MissingBlocks", "ReduceSlotsOpen", "RemainingMapTasks", "RemainingMapTasksPerSlot", "RemainingReduceTasks", "RunningMapTasks", "RunningReduceTasks", "S3BytesRead", "S3BytesWritten", "TaskNodesPending", "TaskNodesRunning", "TotalLoad"},
		"AWS/Kinesis":          {"PutRecord.Bytes", "PutRecord.Latency", "PutRecord.Success", "PutRecords.Bytes", "PutRecords.Latency", "PutRecords.Records", "PutRecords.Success", "IncomingBytes", "IncomingRecords", "GetRecords.Bytes", "GetRecords.IteratorAgeMilliseconds", "GetRecords.Latency", "GetRecords.Success"},
		"AWS/ML":               {"PredictCount", "PredictFailureCount"},
		"AWS/OpsWorks":         {"cpu_idle", "cpu_nice", "cpu_system", "cpu_user", "cpu_waitio", "load_1", "load_5", "load_15", "memory_buffers", "memory_cached", "memory_free", "memory_swap", "memory_total", "memory_used", "procs"},
		"AWS/Redshift":         {"CPUUtilization", "DatabaseConnections", "HealthStatus", "MaintenanceMode", "NetworkReceiveThroughput", "NetworkTransmitThroughput", "PercentageDiskSpaceUsed", "ReadIOPS", "ReadLatency", "ReadThroughput", "WriteIOPS", "WriteLatency", "WriteThroughput"},
		"AWS/RDS":              {"BinLogDiskUsage", "CPUUtilization", "DatabaseConnections", "DiskQueueDepth", "FreeableMemory", "FreeStorageSpace", "ReplicaLag", "SwapUsage", "ReadIOPS", "WriteIOPS", "ReadLatency", "WriteLatency", "ReadThroughput", "WriteThroughput", "NetworkReceiveThroughput", "NetworkTransmitThroughput"},
		"AWS/Route53":          {"HealthCheckStatus", "HealthCheckPercentageHealthy"},
		"AWS/SNS":              {"NumberOfMessagesPublished", "PublishSize", "NumberOfNotificationsDelivered", "NumberOfNotificationsFailed"},
		"AWS/SQS":              {"NumberOfMessagesSent", "SentMessageSize", "NumberOfMessagesReceived", "NumberOfEmptyReceives", "NumberOfMessagesDeleted", "ApproximateNumberOfMessagesDelayed", "ApproximateNumberOfMessagesVisible", "ApproximateNumberOfMessagesNotVisible"},
		"AWS/S3":               {"BucketSizeBytes", "NumberOfObjects"},
		"AWS/SWF":              {"DecisionTaskScheduleToStartTime", "DecisionTaskStartToCloseTime", "DecisionTasksCompleted", "StartedDecisionTasksTimedOutOnClose", "WorkflowStartToCloseTime", "WorkflowsCanceled", "WorkflowsCompleted", "WorkflowsContinuedAsNew", "WorkflowsFailed", "WorkflowsTerminated", "WorkflowsTimedOut"},
		"AWS/StorageGateway":   {"CacheHitPercent", "CachePercentUsed", "CachePercentDirty", "CloudBytesDownloaded", "CloudDownloadLatency", "CloudBytesUploaded", "UploadBufferFree", "UploadBufferPercentUsed", "UploadBufferUsed", "QueuedWrites", "ReadBytes", "ReadTime", "TotalCacheSize", "WriteBytes", "WriteTime", "WorkingStorageFree", "WorkingStoragePercentUsed", "WorkingStorageUsed", "CacheHitPercent", "CachePercentUsed", "CachePercentDirty", "ReadBytes", "ReadTime", "WriteBytes", "WriteTime", "QueuedWrites"},
		"AWS/WorkSpaces":       {"Available", "Unhealthy", "ConnectionAttempt", "ConnectionSuccess", "ConnectionFailure", "SessionLaunchTime", "InSessionLatency", "SessionDisconnect"},
	}
	dimensionsMap = map[string][]string{
		"AWS/AutoScaling":      {"AutoScalingGroupName"},
		"AWS/Billing":          {"ServiceName", "LinkedAccount", "Currency"},
		"AWS/CloudFront":       {"DistributionId", "Region"},
		"AWS/CloudSearch":      {},
		"AWS/DynamoDB":         {"TableName", "GlobalSecondaryIndexName", "Operation"},
		"AWS/ECS":              {"ClusterName", "ServiceName"},
		"AWS/ElastiCache":      {"CacheClusterId", "CacheNodeId"},
		"AWS/EBS":              {"VolumeId"},
		"AWS/EC2":              {"AutoScalingGroupName", "ImageId", "InstanceId", "InstanceType"},
		"AWS/ELB":              {"LoadBalancerName", "AvailabilityZone"},
		"AWS/ElasticMapReduce": {"ClusterId", "JobId"},
		"AWS/Kinesis":          {"StreamName"},
		"AWS/ML":               {"MLModelId", "RequestMode"},
		"AWS/OpsWorks":         {"StackId", "LayerId", "InstanceId"},
		"AWS/Redshift":         {"NodeID", "ClusterIdentifier"},
		"AWS/RDS":              {"DBInstanceIdentifier", "DatabaseClass", "EngineName"},
		"AWS/Route53":          {"HealthCheckId"},
		"AWS/SNS":              {"Application", "Platform", "TopicName"},
		"AWS/SQS":              {"QueueName"},
		"AWS/S3":               {"BucketName", "StorageType"},
		"AWS/SWF":              {"Domain", "ActivityTypeName", "ActivityTypeVersion"},
		"AWS/StorageGateway":   {"GatewayId", "GatewayName", "VolumeId"},
		"AWS/WorkSpaces":       {"DirectoryId", "WorkspaceId"},
	}
}

// Whenever this list is updated, frontend list should also be updated.
// Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html
func handleGetRegions(req *cwRequest, c *middleware.Context) {
	regions := []string{
		"ap-northeast-1", "ap-southeast-1", "ap-southeast-2", "cn-north-1",
		"eu-central-1", "eu-west-1", "sa-east-1", "us-east-1", "us-west-1", "us-west-2",
	}

	result := []interface{}{}
	for _, region := range regions {
		result = append(result, util.DynMap{"text": region, "value": region})
	}

	c.JSON(200, result)
}

func handleGetNamespaces(req *cwRequest, c *middleware.Context) {
	keys := []string{}
	for key := range metricsMap {
		keys = append(keys, key)
	}
	sort.Sort(sort.StringSlice(keys))

	result := []interface{}{}
	for _, key := range keys {
		result = append(result, util.DynMap{"text": key, "value": key})
	}

	c.JSON(200, result)
}

func handleGetMetrics(req *cwRequest, c *middleware.Context) {
	reqParam := &struct {
		Parameters struct {
			Namespace string `json:"namespace"`
		} `json:"parameters"`
	}{}

	json.Unmarshal(req.Body, reqParam)

	namespaceMetrics, exists := metricsMap[reqParam.Parameters.Namespace]
	if !exists {
		c.JsonApiErr(404, "Unable to find namespace "+reqParam.Parameters.Namespace, nil)
		return
	}

	result := []interface{}{}
	for _, name := range namespaceMetrics {
		result = append(result, util.DynMap{"text": name, "value": name})
	}

	c.JSON(200, result)
}

func handleGetDimensions(req *cwRequest, c *middleware.Context) {
	reqParam := &struct {
		Parameters struct {
			Namespace string `json:"namespace"`
		} `json:"parameters"`
	}{}

	json.Unmarshal(req.Body, reqParam)

	dimensionValues, exists := dimensionsMap[reqParam.Parameters.Namespace]
	if !exists {
		c.JsonApiErr(404, "Unable to find dimension "+reqParam.Parameters.Namespace, nil)
		return
	}

	result := []interface{}{}
	for _, name := range dimensionValues {
		result = append(result, util.DynMap{"text": name, "value": name})
	}

	c.JSON(200, result)
}
<commit_msg>update supported metrics and dimensions<commit_after>package cloudwatch

import (
	"encoding/json"
	"sort"

	"github.com/grafana/grafana/pkg/middleware"
	"github.com/grafana/grafana/pkg/util"
)

var metricsMap map[string][]string
var dimensionsMap map[string][]string

func init() {
	metricsMap = map[string][]string{
		"AWS/AutoScaling": {"GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity", "GroupInServiceInstances", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"},
		"AWS/Billing":     {"EstimatedCharges"},
		"AWS/CloudFront":  {"Requests", "BytesDownloaded", "BytesUploaded", "TotalErrorRate", "4xxErrorRate", "5xxErrorRate"},
		"AWS/CloudSearch": {"SuccessfulRequests", "SearchableDocuments", "IndexUtilization", "Partitions"},
		"AWS/DynamoDB":    {"ConditionalCheckFailedRequests", "ConsumedReadCapacityUnits", "ConsumedWriteCapacityUnits", "OnlineIndexConsumedWriteCapacity", "OnlineIndexPercentageProgress", "OnlineIndexThrottleEvents", "ProvisionedReadCapacityUnits", "ProvisionedWriteCapacityUnits", "ReadThrottleEvents", "ReturnedItemCount", "SuccessfulRequestLatency", "SystemErrors", "ThrottledRequests", "UserErrors", "WriteThrottleEvents"},
		"AWS/ECS":         {"CPUUtilization", "MemoryUtilization"},
		"AWS/ElastiCache": {
			"CPUUtilization", "FreeableMemory", "NetworkBytesIn", "NetworkBytesOut", "SwapUsage",
			"BytesUsedForCacheItems", "BytesReadIntoMemcached", "BytesWrittenOutFromMemcached", "CasBadval", "CasHits", "CasMisses", "CmdFlush", "CmdGet", "CmdSet", "CurrConnections", "CurrItems", "DecrHits", "DecrMisses", "DeleteHits", "DeleteMisses", "Evictions", "GetHits", "GetMisses", "IncrHits", "IncrMisses", "Reclaimed",
			"BytesUsedForHash", "CmdConfigGet", "CmdConfigSet", "CmdTouch", "CurrConfig", "EvictedUnfetched", "ExpiredUnfetched", "SlabsMoved", "TouchHits", "TouchMisses",
			"NewConnections", "NewItems", "UnusedMemory",
			"BytesUsedForCache", "CacheHits", "CacheMisses", "CurrConnections", "Evictions", "HyperLogLogBasedCmds", "NewConnections", "Reclaimed", "ReplicationBytes", "ReplicationLag", "SaveInProgress",
			"CurrItems", "GetTypeCmds", "HashBasedCmds", "KeyBasedCmds", "ListBasedCmds", "SetBasedCmds", "SetTypeCmds", "SortedSetBasedCmds", "StringBasedCmds",
		},
		"AWS/EBS": {"VolumeReadBytes", "VolumeWriteBytes", "VolumeReadOps", "VolumeWriteOps", "VolumeTotalReadTime", "VolumeTotalWriteTime", "VolumeIdleTime", "VolumeQueueLength", "VolumeThroughputPercentage", "VolumeConsumedReadWriteOps"},
		"AWS/EC2": {"CPUCreditUsage", "CPUCreditBalance", "CPUUtilization", "DiskReadOps", "DiskWriteOps", "DiskReadBytes", "DiskWriteBytes", "NetworkIn", "NetworkOut", "StatusCheckFailed", "StatusCheckFailed_Instance", "StatusCheckFailed_System"},
		"AWS/ELB": {"HealthyHostCount", "UnHealthyHostCount", "RequestCount", "Latency", "HTTPCode_ELB_4XX", "HTTPCode_ELB_5XX", "HTTPCode_Backend_2XX", "HTTPCode_Backend_3XX", "HTTPCode_Backend_4XX", "HTTPCode_Backend_5XX", "BackendConnectionErrors", "SurgeQueueLength", "SpilloverCount"},
		"AWS/ElasticMapReduce": {"IsIdle", "JobsRunning", "JobsFailed",
			"MapTasksRunning", "MapTasksRemaining", "MapSlotsOpen", "RemainingMapTasksPerSlot", "ReduceTasksRunning", "ReduceTasksRemaining", "ReduceSlotsOpen",
			"CoreNodesRunning", "CoreNodesPending", "LiveDataNodes", "TaskNodesRunning", "TaskNodesPending", "LiveTaskTrackers",
			"S3BytesWritten", "S3BytesRead", "HDFSUtilization", "HDFSBytesRead", "HDFSBytesWritten", "MissingBlocks", "TotalLoad",
			"BackupFailed", "MostRecentBackupDuration", "TimeSinceLastSuccessfulBackup",
			"IsIdle", "ContainerAllocated", "ContainerReserved", "ContainerPending", "AppsCompleted", "AppsFailed", "AppsKilled", "AppsPending", "AppsRunning", "AppsSubmitted",
			"CoreNodesRunning", "CoreNodesPending", "LiveDataNodes", "MRTotalNodes", "MRActiveNodes", "MRLostNodes", "MRUnhealthyNodes", "MRDecommissionedNodes", "MRRebootedNodes",
			"S3BytesWritten", "S3BytesRead", "HDFSUtilization", "HDFSBytesRead", "HDFSBytesWritten", "MissingBlocks", "CorruptBlocks", "TotalLoad", "MemoryTotalMB", "MemoryReservedMB", "MemoryAvailableMB", "MemoryAllocatedMB", "PendingDeletionBlocks", "UnderReplicatedBlocks", "DfsPendingReplicationBlocks", "CapacityRemainingGB",
			"HbaseBackupFailed", "MostRecentBackupDuration", "TimeSinceLastSuccessfulBackup"},
		"AWS/ES":       {"ClusterStatus.green", "ClusterStatus.yellow", "ClusterStatus.red", "Nodes", "SearchableDocuments", "DeletedDocuments", "CPUUtilization", "FreeStorageSpace", "JVMMemoryPressure", "AutomatedSnapshotFailure", "MasterCPUUtilization", "MasterFreeStorageSpace", "MasterJVMMemoryPressure", "ReadLatency", "WriteLatency", "ReadThroughput", "WriteThroughput", "DiskQueueLength", "ReadIOPS", "WriteIOPS"},
		"AWS/Kinesis":  {"PutRecord.Bytes", "PutRecord.Latency", "PutRecord.Success", "PutRecords.Bytes", "PutRecords.Latency", "PutRecords.Records", "PutRecords.Success", "IncomingBytes", "IncomingRecords", "GetRecords.Bytes", "GetRecords.IteratorAgeMilliseconds", "GetRecords.Latency", "GetRecords.Success"},
		"AWS/Lambda":   {"Invocations", "Errors", "Duration", "Throttles"},
		"AWS/ML":       {"PredictCount", "PredictFailureCount"},
		"AWS/OpsWorks": {"cpu_idle", "cpu_nice", "cpu_system", "cpu_user", "cpu_waitio", "load_1", "load_5", "load_15", "memory_buffers", "memory_cached", "memory_free", "memory_swap", "memory_total", "memory_used", "procs"},
		"AWS/Redshift": {"CPUUtilization", "DatabaseConnections", "HealthStatus", "MaintenanceMode", "NetworkReceiveThroughput", "NetworkTransmitThroughput", "PercentageDiskSpaceUsed", "ReadIOPS", "ReadLatency", "ReadThroughput", "WriteIOPS", "WriteLatency", "WriteThroughput"},
		"AWS/RDS":      {"BinLogDiskUsage", "CPUUtilization", "CPUCreditUsage", "CPUCreditBalance", "DatabaseConnections", "DiskQueueDepth", "FreeableMemory", "FreeStorageSpace", "ReplicaLag", "SwapUsage", "ReadIOPS", "WriteIOPS", "ReadLatency", "WriteLatency", "ReadThroughput", "WriteThroughput", "NetworkReceiveThroughput", "NetworkTransmitThroughput"},
		"AWS/Route53":  {"HealthCheckStatus", "HealthCheckPercentageHealthy"},
		"AWS/SNS":      {"NumberOfMessagesPublished", "PublishSize", "NumberOfNotificationsDelivered", "NumberOfNotificationsFailed"},
		"AWS/SQS":      {"NumberOfMessagesSent", "SentMessageSize", "NumberOfMessagesReceived", "NumberOfEmptyReceives", "NumberOfMessagesDeleted", "ApproximateNumberOfMessagesDelayed", "ApproximateNumberOfMessagesVisible", "ApproximateNumberOfMessagesNotVisible"},
		"AWS/S3":       {"BucketSizeBytes", "NumberOfObjects"},
		"AWS/SWF": {"DecisionTaskScheduleToStartTime", "DecisionTaskStartToCloseTime", "DecisionTasksCompleted", "StartedDecisionTasksTimedOutOnClose", "WorkflowStartToCloseTime", "WorkflowsCanceled", "WorkflowsCompleted", "WorkflowsContinuedAsNew", "WorkflowsFailed", "WorkflowsTerminated", "WorkflowsTimedOut",
			"ActivityTaskScheduleToCloseTime", "ActivityTaskScheduleToStartTime", "ActivityTaskStartToCloseTime", "ActivityTasksCanceled", "ActivityTasksCompleted", "ActivityTasksFailed", "ScheduledActivityTasksTimedOutOnClose", "ScheduledActivityTasksTimedOutOnStart", "StartedActivityTasksTimedOutOnClose", "StartedActivityTasksTimedOutOnHeartbeat"},
		"AWS/StorageGateway": {"CacheHitPercent", "CachePercentUsed", "CachePercentDirty", "CloudBytesDownloaded", "CloudDownloadLatency", "CloudBytesUploaded", "UploadBufferFree", "UploadBufferPercentUsed", "UploadBufferUsed", "QueuedWrites", "ReadBytes", "ReadTime", "TotalCacheSize", "WriteBytes", "WriteTime", "TimeSinceLastRecoveryPoint", "WorkingStorageFree", "WorkingStoragePercentUsed", "WorkingStorageUsed",
			"CacheHitPercent", "CachePercentUsed", "CachePercentDirty", "ReadBytes", "ReadTime", "WriteBytes", "WriteTime", "QueuedWrites"},
		"AWS/WAF":        {"AllowedRequests", "BlockedRequests", "CountedRequests"},
		"AWS/WorkSpaces": {"Available", "Unhealthy", "ConnectionAttempt", "ConnectionSuccess", "ConnectionFailure", "SessionLaunchTime", "InSessionLatency", "SessionDisconnect"},
	}
	dimensionsMap = map[string][]string{
		"AWS/AutoScaling":      {"AutoScalingGroupName"},
		"AWS/Billing":          {"ServiceName", "LinkedAccount", "Currency"},
		"AWS/CloudFront":       {"DistributionId", "Region"},
		"AWS/CloudSearch":      {},
		"AWS/DynamoDB":         {"TableName", "GlobalSecondaryIndexName", "Operation"},
		"AWS/ECS":              {"ClusterName", "ServiceName"},
		"AWS/ElastiCache":      {"CacheClusterId", "CacheNodeId"},
		"AWS/EBS":              {"VolumeId"},
		"AWS/EC2":              {"AutoScalingGroupName", "ImageId", "InstanceId", "InstanceType"},
		"AWS/ELB":              {"LoadBalancerName", "AvailabilityZone"},
		"AWS/ElasticMapReduce": {"ClusterId", "JobFlowId", "JobId"},
		"AWS/ES":               {},
		"AWS/Kinesis":          {"StreamName"},
		"AWS/Lambda":           {"FunctionName"},
		"AWS/ML":               {"MLModelId", "RequestMode"},
		"AWS/OpsWorks":         {"StackId", "LayerId", "InstanceId"},
		"AWS/Redshift":         {"NodeID", "ClusterIdentifier"},
		"AWS/RDS":              {"DBInstanceIdentifier", "DatabaseClass", "EngineName"},
		"AWS/Route53":          {"HealthCheckId"},
		"AWS/SNS":              {"Application", "Platform", "TopicName"},
		"AWS/SQS":              {"QueueName"},
		"AWS/S3":               {"BucketName", "StorageType"},
		"AWS/SWF":              {"Domain", "WorkflowTypeName", "WorkflowTypeVersion", "ActivityTypeName", "ActivityTypeVersion"},
		"AWS/StorageGateway":   {"GatewayId", "GatewayName", "VolumeId"},
		"AWS/WAF":              {"Rule", "WebACL"},
		"AWS/WorkSpaces":       {"DirectoryId", "WorkspaceId"},
	}
}

// Whenever this list is updated, frontend list should also be updated.
// Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html
func handleGetRegions(req *cwRequest, c *middleware.Context) {
	regions := []string{
		"ap-northeast-1", "ap-southeast-1", "ap-southeast-2", "cn-north-1",
		"eu-central-1", "eu-west-1", "sa-east-1", "us-east-1", "us-west-1", "us-west-2",
	}

	result := []interface{}{}
	for _, region := range regions {
		result = append(result, util.DynMap{"text": region, "value": region})
	}

	c.JSON(200, result)
}

func handleGetNamespaces(req *cwRequest, c *middleware.Context) {
	keys := []string{}
	for key := range metricsMap {
		keys = append(keys, key)
	}
	sort.Sort(sort.StringSlice(keys))

	result := []interface{}{}
	for _, key := range keys {
		result = append(result, util.DynMap{"text": key, "value": key})
	}

	c.JSON(200, result)
}

func handleGetMetrics(req *cwRequest, c *middleware.Context) {
	reqParam := &struct {
		Parameters struct {
			Namespace string `json:"namespace"`
		} `json:"parameters"`
	}{}

	json.Unmarshal(req.Body, reqParam)

	namespaceMetrics, exists := metricsMap[reqParam.Parameters.Namespace]
	if !exists {
		c.JsonApiErr(404, "Unable to find namespace "+reqParam.Parameters.Namespace, nil)
		return
	}

	result := []interface{}{}
	for _, name := range namespaceMetrics {
		result = append(result, util.DynMap{"text": name, "value": name})
	}

	c.JSON(200, result)
}

func handleGetDimensions(req *cwRequest, c *middleware.Context) {
	reqParam := &struct {
		Parameters struct {
			Namespace string `json:"namespace"`
		} `json:"parameters"`
	}{}

	json.Unmarshal(req.Body, reqParam)

	dimensionValues, exists := dimensionsMap[reqParam.Parameters.Namespace]
	if !exists {
		c.JsonApiErr(404, "Unable to find dimension "+reqParam.Parameters.Namespace, nil)
		return
	}

	result := []interface{}{}
	for _, name := range dimensionValues {
		result = append(result, util.DynMap{"text": name, "value": name})
	}

	c.JSON(200, result)
}
<|endoftext|>


<commit_before>package daemon

import (
	"context"
	"math/rand"
	"net"
	"strings"
	"sync"
	"time"

	dns2 "github.com/miekg/dns"
	"google.golang.org/protobuf/types/known/durationpb"

	"github.com/datawire/dlib/dgroup"
	"github.com/datawire/dlib/dlog"
	rpc "github.com/telepresenceio/telepresence/rpc/v2/daemon"
	"github.com/telepresenceio/telepresence/rpc/v2/manager"
	"github.com/telepresenceio/telepresence/v2/pkg/client"
	"github.com/telepresenceio/telepresence/v2/pkg/client/scout"
	"github.com/telepresenceio/telepresence/v2/pkg/iputil"
)

type awaitLookupResult struct {
	done   chan struct{}
	result iputil.IPs
}

// outbound does stuff, idk, I didn't write it.
//
// A zero outbound is invalid; you must use newOutbound.
type outbound struct {
	noSearch bool
	router   *tunRouter

	// Namespaces, accessible using <service-name>.<namespace-name>
	namespaces map[string]struct{}
	domains    map[string]struct{}
	search     []string

	// The domainsLock locks usage of namespaces, domains, and search
	domainsLock sync.RWMutex

	searchPathCh chan []string

	// dnsQueriesInProgress unique set of DNS queries currently in progress.
	dnsInProgress  map[string]*awaitLookupResult
	dnsQueriesLock sync.Mutex

	dnsConfig *rpc.DNSConfig

	scout chan scout.ScoutReport
}

func newLocalUDPListener(c context.Context) (net.PacketConn, error) {
	lc := &net.ListenConfig{}
	return lc.ListenPacket(c, "udp", "127.0.0.1:0")
}

// newOutbound returns a new properly initialized outbound object.
//
// If dnsIP is empty, it will be detected from /etc/resolv.conf
func newOutbound(c context.Context, dnsIPStr string, noSearch bool, scout chan scout.ScoutReport) (*outbound, error) {
	// seed random generator (used when shuffling IPs)
	rand.Seed(time.Now().UnixNano())

	ret := &outbound{
		dnsConfig: &rpc.DNSConfig{
			LocalIp: iputil.Parse(dnsIPStr),
		},
		noSearch:      noSearch,
		namespaces:    make(map[string]struct{}),
		domains:       make(map[string]struct{}),
		dnsInProgress: make(map[string]*awaitLookupResult),
		search:        []string{""},
		searchPathCh:  make(chan []string),
		scout:         scout,
	}

	var err error
	if ret.router, err = newTunRouter(c); err != nil {
		return nil, err
	}
	return ret, nil
}

// On a macOS, Docker uses its own search-path for single label names. This means that the search path that is declared
// in the macOS resolver is ignored although the rest of the DNS-resolution works OK. Since the search-path is likely to
// change during a session, a stable fake domain is needed to emulate the search-path. That fake-domain can then be used
// in the search path declared in the Docker config. The "tel2-search" domain fills this purpose and a request for
// "<single label name>.tel2-search." will be resolved as "<single label name>." using the search path of this resolver.
const tel2SubDomain = "tel2-search"
const tel2SubDomainDot = tel2SubDomain + "."

var localhostIPv6 = []net.IP{{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}}
var localhostIPv4 = []net.IP{{127, 0, 0, 1}}

func (o *outbound) shouldDoClusterLookup(query string) bool {
	if strings.HasSuffix(query, "."+o.router.clusterDomain) && strings.Count(query, ".") < 4 {
		return false
	}

	query = query[:len(query)-1] // skip last dot

	// Always include configured includeSuffixes
	for _, sfx := range o.dnsConfig.IncludeSuffixes {
		if strings.HasSuffix(query, sfx) {
			return true
		}
	}

	// Skip configured excludeSuffixes
	for _, sfx := range o.dnsConfig.ExcludeSuffixes {
		if strings.HasSuffix(query, sfx) {
			return false
		}
	}
	return true
}

func (o *outbound) resolveInCluster(c context.Context, qType uint16, query string) (results []net.IP) {
	query = strings.ToLower(query)
	query = strings.TrimSuffix(query, tel2SubDomainDot)

	if query == "localhost." {
		// BUG(lukeshu): I have no idea why a lookup
		// for localhost even makes it to here on my
		// home WiFi when connecting to a k3sctl
		// cluster (but not a kubernaut.io cluster).
		// But it does, so I need this in order to be
		// productive at home.  We should really
		// root-cause this, because it's weird.
		if qType == dns2.TypeAAAA {
			return localhostIPv6
		}
		return localhostIPv4
	}

	if !o.shouldDoClusterLookup(query) {
		return nil
	}
	// Don't report queries that won't be resolved in-cluster, since that'll report every single DNS query on the user's machine
	defer func() {
		o.scout <- scout.ScoutReport{
			Action: "incluster_dns_query",
			Metadata: map[string]interface{}{
				"had_results": results != nil,
			},
		}
	}()

	var firstLookupResult *awaitLookupResult
	o.dnsQueriesLock.Lock()
	awaitResult := o.dnsInProgress[query]
	if awaitResult == nil {
		firstLookupResult = &awaitLookupResult{done: make(chan struct{})}
		o.dnsInProgress[query] = firstLookupResult
	}
	o.dnsQueriesLock.Unlock()

	if awaitResult != nil {
		// Wait for this query to complete. Then return its value
		select {
		case <-awaitResult.done:
			return awaitResult.result
		case <-c.Done():
			return nil
		}
	}

	// Give the cluster lookup a reasonable timeout.
	c, cancel := context.WithTimeout(c, o.dnsConfig.LookupTimeout.AsDuration())
	defer func() {
		cancel()
		o.dnsQueriesLock.Lock()
		delete(o.dnsInProgress, query)
		o.dnsQueriesLock.Unlock()
		close(firstLookupResult.done)
	}()

	queryWithNoTrailingDot := query[:len(query)-1]
	dlog.Debugf(c, "LookupHost %q", queryWithNoTrailingDot)
	response, err := o.router.managerClient.LookupHost(c, &manager.LookupHostRequest{
		Session: o.router.session,
		Host:    queryWithNoTrailingDot,
	})
	if err != nil {
		dlog.Error(c, client.CheckTimeout(c, err))
		return nil
	}
	if len(response.Ips) == 0 {
		return nil
	}
	ips := make(iputil.IPs, len(response.Ips))
	for i, ip := range response.Ips {
		ips[i] = ip
	}
	firstLookupResult.result = ips
	return ips
}

func (o *outbound) setInfo(ctx context.Context, info *rpc.OutboundInfo) error {
	if info.Dns == nil {
		info.Dns = &rpc.DNSConfig{}
	}
	if oldIP := o.dnsConfig.GetLocalIp(); len(oldIP) > 0 {
		info.Dns.LocalIp = oldIP
	}
	if len(info.Dns.ExcludeSuffixes) == 0 {
		info.Dns.ExcludeSuffixes = []string{
			".arpa",
			".com",
			".io",
			".net",
			".org",
			".ru",
		}
	}
	if info.Dns.LookupTimeout.AsDuration() <= 0 {
		info.Dns.LookupTimeout = durationpb.New(4 * time.Second)
	}
	o.dnsConfig = info.Dns
	return o.router.setOutboundInfo(ctx, info)
}

func (o *outbound) getInfo() *rpc.OutboundInfo {
	info := rpc.OutboundInfo{
		Dns: &rpc.DNSConfig{
			RemoteIp: o.router.dnsIP,
		},
	}
	if o.dnsConfig != nil {
		info.Dns.LocalIp = o.dnsConfig.LocalIp
		info.Dns.ExcludeSuffixes = o.dnsConfig.ExcludeSuffixes
		info.Dns.IncludeSuffixes = o.dnsConfig.IncludeSuffixes
		info.Dns.LookupTimeout = o.dnsConfig.LookupTimeout
	}

	if len(o.router.alsoProxySubnets) > 0 {
		info.AlsoProxySubnets = make([]*manager.IPNet, len(o.router.alsoProxySubnets))
		for i, ap := range o.router.alsoProxySubnets {
			info.AlsoProxySubnets[i] = iputil.IPNetToRPC(ap)
		}
	}

	return &info
}

// SetSearchPath updates the DNS search path used by the resolver
func (o *outbound) setSearchPath(_ context.Context, paths, namespaces []string) {
	// Provide direct access to intercepted namespaces
	for _, ns := range namespaces {
		paths = append(paths, ns+".svc."+o.router.clusterDomain)
	}
	o.searchPathCh <- paths
}

func (o *outbound) processSearchPaths(g *dgroup.Group, processor func(context.Context, []string) error) {
	g.Go("SearchPaths", func(c context.Context) error {
		var prevPaths []string
		unchanged := func(paths []string) bool {
			if len(paths) != len(prevPaths) {
				return false
			}
			for i, path := range paths {
				if path != prevPaths[i] {
					return false
				}
			}
			return true
		}

		for {
			select {
			case <-c.Done():
				return nil
			case paths := <-o.searchPathCh:
				if !unchanged(paths) {
					dlog.Debugf(c, "%v -> %v", prevPaths, paths)
					prevPaths = make([]string, len(paths))
					copy(prevPaths, paths)
					if err := processor(c, paths); err != nil {
						return err
					}
				}
			}
		}
	})
}

// splitToUDPAddr splits the given address into an UDPAddr. It's
// an  error if the address is based on a hostname rather than an IP.
func splitToUDPAddr(netAddr net.Addr) (*net.UDPAddr, error) {
	ip, port, err := iputil.SplitToIPPort(netAddr)
	if err != nil {
		return nil, err
	}
	return &net.UDPAddr{IP: ip, Port: int(port)}, nil
}
<commit_msg>Ensure that the SetSearchPath method never hangs<commit_after>package daemon

import (
	"context"
	"math/rand"
	"net"
	"strings"
	"sync"
	"time"

	dns2 "github.com/miekg/dns"
	"google.golang.org/protobuf/types/known/durationpb"

	"github.com/datawire/dlib/dgroup"
	"github.com/datawire/dlib/dlog"
	rpc "github.com/telepresenceio/telepresence/rpc/v2/daemon"
	"github.com/telepresenceio/telepresence/rpc/v2/manager"
	"github.com/telepresenceio/telepresence/v2/pkg/client"
	"github.com/telepresenceio/telepresence/v2/pkg/client/scout"
	"github.com/telepresenceio/telepresence/v2/pkg/iputil"
)

type awaitLookupResult struct {
	done   chan struct{}
	result iputil.IPs
}

// outbound does stuff, idk, I didn't write it.
//
// A zero outbound is invalid; you must use newOutbound.
type outbound struct {
	noSearch bool
	router   *tunRouter

	// Namespaces, accessible using <service-name>.<namespace-name>
	namespaces map[string]struct{}
	domains    map[string]struct{}
	search     []string

	// The domainsLock locks usage of namespaces, domains, and search
	domainsLock sync.RWMutex

	searchPathCh chan []string

	// dnsQueriesInProgress unique set of DNS queries currently in progress.
	dnsInProgress  map[string]*awaitLookupResult
	dnsQueriesLock sync.Mutex

	dnsConfig *rpc.DNSConfig

	scout chan scout.ScoutReport
}

func newLocalUDPListener(c context.Context) (net.PacketConn, error) {
	lc := &net.ListenConfig{}
	return lc.ListenPacket(c, "udp", "127.0.0.1:0")
}

// newOutbound returns a new properly initialized outbound object.
//
// If dnsIP is empty, it will be detected from /etc/resolv.conf
func newOutbound(c context.Context, dnsIPStr string, noSearch bool, scout chan scout.ScoutReport) (*outbound, error) {
	// seed random generator (used when shuffling IPs)
	rand.Seed(time.Now().UnixNano())

	ret := &outbound{
		dnsConfig: &rpc.DNSConfig{
			LocalIp: iputil.Parse(dnsIPStr),
		},
		noSearch:      noSearch,
		namespaces:    make(map[string]struct{}),
		domains:       make(map[string]struct{}),
		dnsInProgress: make(map[string]*awaitLookupResult),
		search:        []string{""},
		searchPathCh:  make(chan []string, 5),
		scout:         scout,
	}

	var err error
	if ret.router, err = newTunRouter(c); err != nil {
		return nil, err
	}
	return ret, nil
}

// On a macOS, Docker uses its own search-path for single label names. This means that the search path that is declared
// in the macOS resolver is ignored although the rest of the DNS-resolution works OK. Since the search-path is likely to
// change during a session, a stable fake domain is needed to emulate the search-path. That fake-domain can then be used
// in the search path declared in the Docker config. The "tel2-search" domain fills this purpose and a request for
// "<single label name>.tel2-search." will be resolved as "<single label name>." using the search path of this resolver.
const tel2SubDomain = "tel2-search"
const tel2SubDomainDot = tel2SubDomain + "."

var localhostIPv6 = []net.IP{{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}}
var localhostIPv4 = []net.IP{{127, 0, 0, 1}}

func (o *outbound) shouldDoClusterLookup(query string) bool {
	if strings.HasSuffix(query, "."+o.router.clusterDomain) && strings.Count(query, ".") < 4 {
		return false
	}

	query = query[:len(query)-1] // skip last dot

	// Always include configured includeSuffixes
	for _, sfx := range o.dnsConfig.IncludeSuffixes {
		if strings.HasSuffix(query, sfx) {
			return true
		}
	}

	// Skip configured excludeSuffixes
	for _, sfx := range o.dnsConfig.ExcludeSuffixes {
		if strings.HasSuffix(query, sfx) {
			return false
		}
	}
	return true
}

func (o *outbound) resolveInCluster(c context.Context, qType uint16, query string) (results []net.IP) {
	query = strings.ToLower(query)
	query = strings.TrimSuffix(query, tel2SubDomainDot)

	if query == "localhost." {
		// BUG(lukeshu): I have no idea why a lookup
		// for localhost even makes it to here on my
		// home WiFi when connecting to a k3sctl
		// cluster (but not a kubernaut.io cluster).
		// But it does, so I need this in order to be
		// productive at home.  We should really
		// root-cause this, because it's weird.
		if qType == dns2.TypeAAAA {
			return localhostIPv6
		}
		return localhostIPv4
	}

	if !o.shouldDoClusterLookup(query) {
		return nil
	}
	// Don't report queries that won't be resolved in-cluster, since that'll report every single DNS query on the user's machine
	defer func() {
		o.scout <- scout.ScoutReport{
			Action: "incluster_dns_query",
			Metadata: map[string]interface{}{
				"had_results": results != nil,
			},
		}
	}()

	var firstLookupResult *awaitLookupResult
	o.dnsQueriesLock.Lock()
	awaitResult := o.dnsInProgress[query]
	if awaitResult == nil {
		firstLookupResult = &awaitLookupResult{done: make(chan struct{})}
		o.dnsInProgress[query] = firstLookupResult
	}
	o.dnsQueriesLock.Unlock()

	if awaitResult != nil {
		// Wait for this query to complete. Then return its value
		select {
		case <-awaitResult.done:
			return awaitResult.result
		case <-c.Done():
			return nil
		}
	}

	// Give the cluster lookup a reasonable timeout.
	c, cancel := context.WithTimeout(c, o.dnsConfig.LookupTimeout.AsDuration())
	defer func() {
		cancel()
		o.dnsQueriesLock.Lock()
		delete(o.dnsInProgress, query)
		o.dnsQueriesLock.Unlock()
		close(firstLookupResult.done)
	}()

	queryWithNoTrailingDot := query[:len(query)-1]
	dlog.Debugf(c, "LookupHost %q", queryWithNoTrailingDot)
	response, err := o.router.managerClient.LookupHost(c, &manager.LookupHostRequest{
		Session: o.router.session,
		Host:    queryWithNoTrailingDot,
	})
	if err != nil {
		dlog.Error(c, client.CheckTimeout(c, err))
		return nil
	}
	if len(response.Ips) == 0 {
		return nil
	}
	ips := make(iputil.IPs, len(response.Ips))
	for i, ip := range response.Ips {
		ips[i] = ip
	}
	firstLookupResult.result = ips
	return ips
}

func (o *outbound) setInfo(ctx context.Context, info *rpc.OutboundInfo) error {
	if info.Dns == nil {
		info.Dns = &rpc.DNSConfig{}
	}
	if oldIP := o.dnsConfig.GetLocalIp(); len(oldIP) > 0 {
		info.Dns.LocalIp = oldIP
	}
	if len(info.Dns.ExcludeSuffixes) == 0 {
		info.Dns.ExcludeSuffixes = []string{
			".arpa",
			".com",
			".io",
			".net",
			".org",
			".ru",
		}
	}
	if info.Dns.LookupTimeout.AsDuration() <= 0 {
		info.Dns.LookupTimeout = durationpb.New(4 * time.Second)
	}
	o.dnsConfig = info.Dns
	return o.router.setOutboundInfo(ctx, info)
}

func (o *outbound) getInfo() *rpc.OutboundInfo {
	info := rpc.OutboundInfo{
		Dns: &rpc.DNSConfig{
			RemoteIp: o.router.dnsIP,
		},
	}
	if o.dnsConfig != nil {
		info.Dns.LocalIp = o.dnsConfig.LocalIp
		info.Dns.ExcludeSuffixes = o.dnsConfig.ExcludeSuffixes
		info.Dns.IncludeSuffixes = o.dnsConfig.IncludeSuffixes
		info.Dns.LookupTimeout = o.dnsConfig.LookupTimeout
	}

	if len(o.router.alsoProxySubnets) > 0 {
		info.AlsoProxySubnets = make([]*manager.IPNet, len(o.router.alsoProxySubnets))
		for i, ap := range o.router.alsoProxySubnets {
			info.AlsoProxySubnets[i] = iputil.IPNetToRPC(ap)
		}
	}

	return &info
}

// SetSearchPath updates the DNS search path used by the resolver
func (o *outbound) setSearchPath(ctx context.Context, paths, namespaces []string) {
	// Provide direct access to intercepted namespaces
	for _, ns := range namespaces {
		paths = append(paths, ns+".svc."+o.router.clusterDomain)
	}
	select {
	case <-ctx.Done():
	case o.searchPathCh <- paths:
	}
}

func (o *outbound) processSearchPaths(g *dgroup.Group, processor func(context.Context, []string) error) {
	g.Go("SearchPaths", func(c context.Context) error {
		var prevPaths []string
		unchanged := func(paths []string) bool {
			if len(paths) != len(prevPaths) {
				return false
			}
			for i, path := range paths {
				if path != prevPaths[i] {
					return false
				}
			}
			return true
		}

		for {
			select {
			case <-c.Done():
				return nil
			case paths := <-o.searchPathCh:
				if len(o.searchPathCh) > 0 {
					// Only interested in the last one
					continue
				}
				if !unchanged(paths) {
					dlog.Debugf(c, "%v -> %v", prevPaths, paths)
					prevPaths = make([]string, len(paths))
					copy(prevPaths, paths)
					if err := processor(c, paths); err != nil {
						return err
					}
				}
			}
		}
	})
}

// splitToUDPAddr splits the given address into an UDPAddr. It's
// an  error if the address is based on a hostname rather than an IP.
func splitToUDPAddr(netAddr net.Addr) (*net.UDPAddr, error) {
	ip, port, err := iputil.SplitToIPPort(netAddr)
	if err != nil {
		return nil, err
	}
	return &net.UDPAddr{IP: ip, Port: int(port)}, nil
}
<|endoftext|>


<commit_before>package template

import (
	"fmt"
	"io/ioutil"
	"path/filepath"

	"github.com/flosch/pongo2"
	"github.com/franela/goreq"
)

func DownloadTemplateFromFilePath(filePath string) (string, error) {
	buf, err := ioutil.ReadFile(filepath.Clean(filePath))
	if err != nil {
		return "", err
	}

	return string(buf), nil
}

func DownloadTemplateFromURL(url string) (string, error) {
	req := goreq.Request{
		Method: "GET",
		Uri:    url,
	}

	var err error
	var resp *goreq.Response
	if resp, err = req.Do(); err != nil {
		return "", err
	}

	if !(resp.StatusCode >= 200 && resp.StatusCode < 300) {
		return "", fmt.Errorf("unsuccessful request: %s", resp.Status)
	}

	var body string
	if body, err = resp.Body.ToString(); err != nil {
		return "", err
	}

	return body, nil
}

func ParseTextTemplateFromURL(url string, context map[string]interface{}) (string, error) {
	var body string
	var err error
	if body, err = DownloadTemplateFromURL(url); err != nil {
		return "", err
	}

	return ParseTextTemplate(body, context)
}

func ParseHTMLTemplateFromURL(url string, context map[string]interface{}) (string, error) {
	var body string
	var err error
	if body, err = DownloadTemplateFromURL(url); err != nil {
		return "", err
	}

	return ParseHTMLTemplate(body, context)
}

func ParseTextTemplate(templateString string, context map[string]interface{}) (out string, err error) {
	if templateString == "" {
		return
	}

	// turn off auto html escape
	autoEscapeOffTemplate := `{%% autoescape off %%}%s{%% endautoescape %%}`
	autoEscapeOffTemplateString := fmt.Sprintf(autoEscapeOffTemplate, templateString)

	return ParseHTMLTemplate(autoEscapeOffTemplateString, context)
}

func ParseHTMLTemplate(templateString string, context map[string]interface{}) (out string, err error) {
	if templateString == "" {
		return
	}

	var t *pongo2.Template
	if t, err = pongo2.FromString(templateString); err != nil {
		return
	}

	if out, err = t.Execute(context); err != nil {
		return
	}

	return
}
<commit_msg>Limit template maximum size<commit_after>package template

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"

	"github.com/flosch/pongo2"
)

const MaxTemplateSize = 1024 * 1024 * 1

func DownloadTemplateFromFilePath(filePath string) (string, error) {
	filePath = filepath.Clean(filePath)

	f, err := os.Open(filePath)
	if err != nil {
		return "", err
	}
	defer f.Close()

	b, err := ioutil.ReadAll(io.LimitReader(f, MaxTemplateSize))
	if err != nil {
		return "", err
	}

	return string(b), nil
}

func DownloadTemplateFromURL(url string) (string, error) {
	// FIXME(sec): validate URL to be trusted URL
	// nolint: gosec
	resp, err := http.Get(url)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return "", err
	}

	if !(resp.StatusCode >= 200 && resp.StatusCode < 300) {
		return "", fmt.Errorf("unsuccessful request: %s", resp.Status)
	}

	body, err := ioutil.ReadAll(io.LimitReader(resp.Body, MaxTemplateSize))
	if err != nil {
		return "", err
	}

	return string(body), nil
}

func ParseTextTemplateFromURL(url string, context map[string]interface{}) (string, error) {
	var body string
	var err error
	if body, err = DownloadTemplateFromURL(url); err != nil {
		return "", err
	}

	return ParseTextTemplate(body, context)
}

func ParseHTMLTemplateFromURL(url string, context map[string]interface{}) (string, error) {
	var body string
	var err error
	if body, err = DownloadTemplateFromURL(url); err != nil {
		return "", err
	}

	return ParseHTMLTemplate(body, context)
}

func ParseTextTemplate(templateString string, context map[string]interface{}) (out string, err error) {
	if templateString == "" {
		return
	}

	// turn off auto html escape
	autoEscapeOffTemplate := `{%% autoescape off %%}%s{%% endautoescape %%}`
	autoEscapeOffTemplateString := fmt.Sprintf(autoEscapeOffTemplate, templateString)

	return ParseHTMLTemplate(autoEscapeOffTemplateString, context)
}

func ParseHTMLTemplate(templateString string, context map[string]interface{}) (out string, err error) {
	if templateString == "" {
		return
	}

	var t *pongo2.Template
	if t, err = pongo2.FromString(templateString); err != nil {
		return
	}

	if out, err = t.Execute(context); err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>package client

import (
	"flag"
	"io"
	"time"

	cortex_client "github.com/cortexproject/cortex/pkg/ingester/client"
	"github.com/cortexproject/cortex/pkg/util/grpcclient"
	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
	opentracing "github.com/opentracing/opentracing-go"
	"github.com/weaveworks/common/middleware"
	"google.golang.org/grpc"
	"google.golang.org/grpc/health/grpc_health_v1"

	"github.com/grafana/loki/pkg/logproto"
)

// Config for an ingester client.
type Config struct {
	PoolConfig       cortex_client.PoolConfig `yaml:"pool_config,omitempty"`
	RemoteTimeout    time.Duration            `yaml:"remote_timeout,omitempty"`
	GRPCClientConfig grpcclient.Config        `yaml:"grpc_client_config"`
}

// RegisterFlags registers flags.
func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
	cfg.GRPCClientConfig.RegisterFlags("ingester.client", f)
	cfg.PoolConfig.RegisterFlags(f)

	f.DurationVar(&cfg.PoolConfig.RemoteTimeout, "ingester.client.healthcheck-timeout", 1*time.Second, "Timeout for healthcheck rpcs.")
	f.DurationVar(&cfg.RemoteTimeout, "ingester.client.timeout", 5*time.Second, "Timeout for ingester client RPCs.")
}

// New returns a new ingester client.
func New(cfg Config, addr string) (grpc_health_v1.HealthClient, error) {
	opts := []grpc.DialOption{
		grpc.WithInsecure(),
		grpc.WithDefaultCallOptions(
			grpc.UseCompressor("gzip"),
		),
	}
	opts = append(opts, cfg.GRPCClientConfig.DialOption(instrumentation())...)
	conn, err := grpc.Dial(addr, opts...)
	if err != nil {
		return nil, err
	}
	return struct {
		logproto.PusherClient
		logproto.QuerierClient
		logproto.IngesterClient
		grpc_health_v1.HealthClient
		io.Closer
	}{
		PusherClient:   logproto.NewPusherClient(conn),
		QuerierClient:  logproto.NewQuerierClient(conn),
		IngesterClient: logproto.NewIngesterClient(conn),
		HealthClient:   grpc_health_v1.NewHealthClient(conn),
		Closer:         conn,
	}, nil
}

func instrumentation() ([]grpc.UnaryClientInterceptor, []grpc.StreamClientInterceptor) {
	return []grpc.UnaryClientInterceptor{
			otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()),
			middleware.ClientUserHeaderInterceptor,
		}, []grpc.StreamClientInterceptor{
			otgrpc.OpenTracingStreamClientInterceptor(opentracing.GlobalTracer()),
			middleware.StreamClientUserHeaderInterceptor,
		}
}
<commit_msg>Use ingester client GRPC call options from config. (#1797)<commit_after>package client

import (
	"flag"
	"io"
	"time"

	cortex_client "github.com/cortexproject/cortex/pkg/ingester/client"
	"github.com/cortexproject/cortex/pkg/util/grpcclient"
	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
	opentracing "github.com/opentracing/opentracing-go"
	"github.com/weaveworks/common/middleware"
	"google.golang.org/grpc"
	"google.golang.org/grpc/health/grpc_health_v1"

	"github.com/grafana/loki/pkg/logproto"
)

// Config for an ingester client.
type Config struct {
	PoolConfig       cortex_client.PoolConfig `yaml:"pool_config,omitempty"`
	RemoteTimeout    time.Duration            `yaml:"remote_timeout,omitempty"`
	GRPCClientConfig grpcclient.Config        `yaml:"grpc_client_config"`
}

// RegisterFlags registers flags.
func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
	cfg.GRPCClientConfig.RegisterFlags("ingester.client", f)
	cfg.PoolConfig.RegisterFlags(f)

	f.DurationVar(&cfg.PoolConfig.RemoteTimeout, "ingester.client.healthcheck-timeout", 1*time.Second, "Timeout for healthcheck rpcs.")
	f.DurationVar(&cfg.RemoteTimeout, "ingester.client.timeout", 5*time.Second, "Timeout for ingester client RPCs.")
}

// New returns a new ingester client.
func New(cfg Config, addr string) (grpc_health_v1.HealthClient, error) {
	opts := []grpc.DialOption{
		grpc.WithInsecure(),
		grpc.WithDefaultCallOptions(cfg.GRPCClientConfig.CallOptions()...),
	}
	opts = append(opts, cfg.GRPCClientConfig.DialOption(instrumentation())...)
	conn, err := grpc.Dial(addr, opts...)
	if err != nil {
		return nil, err
	}
	return struct {
		logproto.PusherClient
		logproto.QuerierClient
		logproto.IngesterClient
		grpc_health_v1.HealthClient
		io.Closer
	}{
		PusherClient:   logproto.NewPusherClient(conn),
		QuerierClient:  logproto.NewQuerierClient(conn),
		IngesterClient: logproto.NewIngesterClient(conn),
		HealthClient:   grpc_health_v1.NewHealthClient(conn),
		Closer:         conn,
	}, nil
}

func instrumentation() ([]grpc.UnaryClientInterceptor, []grpc.StreamClientInterceptor) {
	return []grpc.UnaryClientInterceptor{
			otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()),
			middleware.ClientUserHeaderInterceptor,
		}, []grpc.StreamClientInterceptor{
			otgrpc.OpenTracingStreamClientInterceptor(opentracing.GlobalTracer()),
			middleware.StreamClientUserHeaderInterceptor,
		}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/cruntime"
	"k8s.io/minikube/pkg/minikube/sysinit"
	"k8s.io/minikube/pkg/util/retry"
)

// Pause pauses a Kubernetes cluster, retrying if necessary
func Pause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) {
	var ids []string
	tryPause := func() (err error) {
		ids, err = pause(cr, r, namespaces)
		return err
	}

	if err := retry.Expo(tryPause, 250*time.Millisecond, 2*time.Second); err != nil {
		return ids, err
	}
	return ids, nil
}

// pause pauses a Kubernetes cluster
func pause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) {
	ids := []string{}

	// Disable the kubelet so it does not attempt to restart paused pods
	sm := sysinit.New(r)
	if err := sm.Disable("kubelet"); err != nil {
		return ids, errors.Wrap(err, "kubelet disable")
	}

	if err := sm.Stop("kubelet"); err != nil {
		return ids, errors.Wrap(err, "kubelet stop")
	}

	ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Running, Namespaces: namespaces})
	if err != nil {
		return ids, errors.Wrap(err, "list running")
	}

	if len(ids) == 0 {
		glog.Warningf("no running containers to pause")
		return ids, nil
	}

	return ids, cr.PauseContainers(ids)
}

// Unpause unpauses a Kubernetes cluster, retrying if necessary
func Unpause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) {
	var ids []string
	tryUnpause := func() (err error) {
		ids, err = unpause(cr, r, namespaces)
		return err
	}

	if err := retry.Expo(tryUnpause, 250*time.Millisecond, 2*time.Second); err != nil {
		return ids, err
	}
	return ids, nil
}

// unpause unpauses a Kubernetes cluster
func unpause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) {
	ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Paused, Namespaces: namespaces})
	if err != nil {
		return ids, errors.Wrap(err, "list paused")
	}

	if len(ids) == 0 {
		glog.Warningf("no paused containers found")
	} else if err := cr.UnpauseContainers(ids); err != nil {
		return ids, errors.Wrap(err, "unpause")
	}

	sm := sysinit.New(r)
	if err := sm.Enable("kubelet"); err != nil {
		return ids, errors.Wrap(err, "kubelet enable")
	}

	if err := sm.Start("kubelet"); err != nil {
		return ids, errors.Wrap(err, "kubelet start")
	}

	return ids, nil
}
<commit_msg>Remove Enable(kubelet) from unpause<commit_after>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/cruntime"
	"k8s.io/minikube/pkg/minikube/sysinit"
	"k8s.io/minikube/pkg/util/retry"
)

// Pause pauses a Kubernetes cluster, retrying if necessary
func Pause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) {
	var ids []string
	tryPause := func() (err error) {
		ids, err = pause(cr, r, namespaces)
		return err
	}

	if err := retry.Expo(tryPause, 250*time.Millisecond, 2*time.Second); err != nil {
		return ids, err
	}
	return ids, nil
}

// pause pauses a Kubernetes cluster
func pause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) {
	ids := []string{}

	// Disable the kubelet so it does not attempt to restart paused pods
	sm := sysinit.New(r)
	if err := sm.Disable("kubelet"); err != nil {
		return ids, errors.Wrap(err, "kubelet disable")
	}

	if err := sm.Stop("kubelet"); err != nil {
		return ids, errors.Wrap(err, "kubelet stop")
	}

	ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Running, Namespaces: namespaces})
	if err != nil {
		return ids, errors.Wrap(err, "list running")
	}

	if len(ids) == 0 {
		glog.Warningf("no running containers to pause")
		return ids, nil
	}

	return ids, cr.PauseContainers(ids)
}

// Unpause unpauses a Kubernetes cluster, retrying if necessary
func Unpause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) {
	var ids []string
	tryUnpause := func() (err error) {
		ids, err = unpause(cr, r, namespaces)
		return err
	}

	if err := retry.Expo(tryUnpause, 250*time.Millisecond, 2*time.Second); err != nil {
		return ids, err
	}
	return ids, nil
}

// unpause unpauses a Kubernetes cluster
func unpause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) {
	ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Paused, Namespaces: namespaces})
	if err != nil {
		return ids, errors.Wrap(err, "list paused")
	}

	if len(ids) == 0 {
		glog.Warningf("no paused containers found")
	} else if err := cr.UnpauseContainers(ids); err != nil {
		return ids, errors.Wrap(err, "unpause")
	}

	sm := sysinit.New(r)

	if err := sm.Start("kubelet"); err != nil {
		return ids, errors.Wrap(err, "kubelet start")
	}

	return ids, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package notify

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"runtime"
	"strings"
	"time"

	"github.com/blang/semver"
	"github.com/golang/glog"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/util/lock"
	"k8s.io/minikube/pkg/version"
)

const updateLinkPrefix = "https://github.com/kubernetes/minikube/releases/tag/v"

var (
	timeLayout              = time.RFC1123
	lastUpdateCheckFilePath = constants.MakeMiniPath("last_update_check")
)

// MaybePrintUpdateTextFromGithub prints update text if needed, from github
func MaybePrintUpdateTextFromGithub() {
	MaybePrintUpdateText(constants.GithubMinikubeReleasesURL, lastUpdateCheckFilePath)
}

// MaybePrintUpdateText prints update text if needed
func MaybePrintUpdateText(url string, lastUpdatePath string) {
	if !shouldCheckURLVersion(lastUpdatePath) {
		return
	}
	latestVersion, err := getLatestVersionFromURL(url)
	if err != nil {
		glog.Warning(err)
		return
	}
	localVersion, err := version.GetSemverVersion()
	if err != nil {
		glog.Warning(err)
		return
	}
	if localVersion.Compare(latestVersion) < 0 {
		if err := writeTimeToFile(lastUpdateCheckFilePath, time.Now().UTC()); err != nil {
			glog.Errorf("write time failed: %v", err)
		}
		url := fmt.Sprintf("%s/%s", updateLinkPrefix, latestVersion)
		out.ErrT(out.WarningType, `minikube {{.version}} is available! Download it: {{.url}}`, out.V{"version": latestVersion, "url": url})
		out.T(out.Tip, "To disable this notice, run: 'minikube config set WantUpdateNotification false'")
	}
}

func shouldCheckURLVersion(filePath string) bool {
	if !viper.GetBool(config.WantUpdateNotification) {
		return false
	}
	lastUpdateTime := getTimeFromFileIfExists(filePath)
	return time.Since(lastUpdateTime).Hours() >= viper.GetFloat64(config.ReminderWaitPeriodInHours)
}

// Release represents a release
type Release struct {
	Name      string
	Checksums map[string]string
}

// Releases represents several release
type Releases []Release

func getJSON(url string, target *Releases) error {
	client := &http.Client{}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return errors.Wrap(err, "error creating new http request")
	}
	ua := fmt.Sprintf("Minikube/%s Minikube-OS/%s",
		version.GetVersion(), runtime.GOOS)

	req.Header.Set("User-Agent", ua)

	resp, err := client.Do(req)
	if err != nil {
		return errors.Wrapf(err, "error with http GET for endpoint %s", url)
	}

	defer resp.Body.Close()
	return json.NewDecoder(resp.Body).Decode(target)
}

func getLatestVersionFromURL(url string) (semver.Version, error) {
	r, err := GetAllVersionsFromURL(url)
	if err != nil {
		return semver.Version{}, err
	}
	return semver.Make(strings.TrimPrefix(r[0].Name, version.VersionPrefix))
}

// GetAllVersionsFromURL get all versions from a JSON URL
func GetAllVersionsFromURL(url string) (Releases, error) {
	var releases Releases
	glog.Info("Checking for updates...")
	if err := getJSON(url, &releases); err != nil {
		return releases, errors.Wrap(err, "Error getting json from minikube version url")
	}
	if len(releases) == 0 {
		return releases, errors.Errorf("There were no json releases at the url specified: %s", url)
	}
	return releases, nil
}

func writeTimeToFile(path string, inputTime time.Time) error {
	err := lock.WriteFile(path, []byte(inputTime.Format(timeLayout)), 0644)
	if err != nil {
		return errors.Wrap(err, "Error writing current update time to file: ")
	}
	return nil
}

func getTimeFromFileIfExists(path string) time.Time {
	lastUpdateCheckTime, err := ioutil.ReadFile(path)
	if err != nil {
		return time.Time{}
	}
	timeInFile, err := time.Parse(timeLayout, string(lastUpdateCheckTime))
	if err != nil {
		return time.Time{}
	}
	return timeInFile
}
<commit_msg>Update URL should be concatenated without a /<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package notify

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"runtime"
	"strings"
	"time"

	"github.com/blang/semver"
	"github.com/golang/glog"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/util/lock"
	"k8s.io/minikube/pkg/version"
)

var (
	timeLayout              = time.RFC1123
	lastUpdateCheckFilePath = constants.MakeMiniPath("last_update_check")
)

// MaybePrintUpdateTextFromGithub prints update text if needed, from github
func MaybePrintUpdateTextFromGithub() {
	MaybePrintUpdateText(constants.GithubMinikubeReleasesURL, lastUpdateCheckFilePath)
}

// MaybePrintUpdateText prints update text if needed
func MaybePrintUpdateText(url string, lastUpdatePath string) {
	if !shouldCheckURLVersion(lastUpdatePath) {
		return
	}
	latestVersion, err := getLatestVersionFromURL(url)
	if err != nil {
		glog.Warning(err)
		return
	}
	localVersion, err := version.GetSemverVersion()
	if err != nil {
		glog.Warning(err)
		return
	}
	if localVersion.Compare(latestVersion) < 0 {
		if err := writeTimeToFile(lastUpdateCheckFilePath, time.Now().UTC()); err != nil {
			glog.Errorf("write time failed: %v", err)
		}
		url := "https://github.com/kubernetes/minikube/releases/tag/v" + latestVersion.String()
		out.ErrT(out.WarningType, `minikube {{.version}} is available! Download it: {{.url}}`, out.V{"version": latestVersion, "url": url})
		out.T(out.Tip, "To disable this notice, run: 'minikube config set WantUpdateNotification false'")
	}
}

func shouldCheckURLVersion(filePath string) bool {
	if !viper.GetBool(config.WantUpdateNotification) {
		return false
	}
	lastUpdateTime := getTimeFromFileIfExists(filePath)
	return time.Since(lastUpdateTime).Hours() >= viper.GetFloat64(config.ReminderWaitPeriodInHours)
}

// Release represents a release
type Release struct {
	Name      string
	Checksums map[string]string
}

// Releases represents several release
type Releases []Release

func getJSON(url string, target *Releases) error {
	client := &http.Client{}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return errors.Wrap(err, "error creating new http request")
	}
	ua := fmt.Sprintf("Minikube/%s Minikube-OS/%s",
		version.GetVersion(), runtime.GOOS)

	req.Header.Set("User-Agent", ua)

	resp, err := client.Do(req)
	if err != nil {
		return errors.Wrapf(err, "error with http GET for endpoint %s", url)
	}

	defer resp.Body.Close()
	return json.NewDecoder(resp.Body).Decode(target)
}

func getLatestVersionFromURL(url string) (semver.Version, error) {
	r, err := GetAllVersionsFromURL(url)
	if err != nil {
		return semver.Version{}, err
	}
	return semver.Make(strings.TrimPrefix(r[0].Name, version.VersionPrefix))
}

// GetAllVersionsFromURL get all versions from a JSON URL
func GetAllVersionsFromURL(url string) (Releases, error) {
	var releases Releases
	glog.Info("Checking for updates...")
	if err := getJSON(url, &releases); err != nil {
		return releases, errors.Wrap(err, "Error getting json from minikube version url")
	}
	if len(releases) == 0 {
		return releases, errors.Errorf("There were no json releases at the url specified: %s", url)
	}
	return releases, nil
}

func writeTimeToFile(path string, inputTime time.Time) error {
	err := lock.WriteFile(path, []byte(inputTime.Format(timeLayout)), 0644)
	if err != nil {
		return errors.Wrap(err, "Error writing current update time to file: ")
	}
	return nil
}

func getTimeFromFileIfExists(path string) time.Time {
	lastUpdateCheckTime, err := ioutil.ReadFile(path)
	if err != nil {
		return time.Time{}
	}
	timeInFile, err := time.Parse(timeLayout, string(lastUpdateCheckTime))
	if err != nil {
		return time.Time{}
	}
	return timeInFile
}
<|endoftext|>


<commit_before>package paxos

import (
	"doozer/assert"
	"testing"
)

func TestCoordIgnoreOldMessages(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))

	co.Put(newTick()) // force the start of a new round

	got = Msg{}
	co.Put(newRsvpFrom(1, 1, 0, ""))
	co.Put(newRsvpFrom(2, 1, 0, ""))
	co.Put(newRsvpFrom(3, 1, 0, ""))
	co.Put(newRsvpFrom(4, 1, 0, ""))
	co.Put(newRsvpFrom(5, 1, 0, ""))
	co.Put(newRsvpFrom(6, 1, 0, ""))
	assert.Equal(t, Msg{}, got)
}

func TestCoordStart(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))
	assert.Equal(t, newInvite(1), got)
}

// This is here mainly for triangulation.  It ensures we're not
// hardcoding crnd.
func TestCoordStartAlt(t *testing.T) {
	var got Msg
	cx := newCluster("c", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))
	assert.Equal(t, newInvite(2), got)
}

func TestCoordTargetNomination(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))

	co.Put(newRsvpFrom(2, 1, 0, ""))
	co.Put(newRsvpFrom(3, 1, 0, ""))
	co.Put(newRsvpFrom(4, 1, 0, ""))
	co.Put(newRsvpFrom(5, 1, 0, ""))
	co.Put(newRsvpFrom(6, 1, 0, ""))
	co.Put(newRsvpFrom(7, 1, 0, ""))
	assert.Equal(t, newNominate(1, "foo"), got)
}

func TestCoordRestart(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))

	// never reach majority (force timeout)
	co.Put(newRsvpFrom(2, 1, 0, ""))
	co.Put(newRsvpFrom(3, 1, 0, ""))
	co.Put(newRsvpFrom(4, 1, 0, ""))
	co.Put(newRsvpFrom(5, 1, 0, ""))
	co.Put(newRsvpFrom(6, 1, 0, ""))

	co.Put(newTick()) // force the start of a new round
	assert.Equal(t, newInvite(11), got)
}

func TestCoordNonTargetNomination(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))

	co.Put(newRsvpFrom(1, 1, 0, ""))
	co.Put(newRsvpFrom(2, 1, 0, ""))
	co.Put(newRsvpFrom(3, 1, 0, ""))
	co.Put(newRsvpFrom(4, 1, 0, ""))
	co.Put(newRsvpFrom(5, 1, 0, ""))
	co.Put(newRsvpFrom(6, 1, 1, "bar"))
	assert.Equal(t, newNominate(1, "bar"), got)
}

func TestCoordOneNominationPerRound(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))

	co.Put(newRsvpFrom(1, 1, 0, ""))
	co.Put(newRsvpFrom(2, 1, 0, ""))
	co.Put(newRsvpFrom(3, 1, 0, ""))
	co.Put(newRsvpFrom(4, 1, 0, ""))
	co.Put(newRsvpFrom(5, 1, 0, ""))
	co.Put(newRsvpFrom(6, 1, 0, ""))
	assert.Equal(t, newNominate(1, "foo"), got)

	got = Msg{}
	co.Put(newRsvpFrom(7, 1, 0, ""))
	assert.Equal(t, Msg{}, got)
}

func TestCoordEachRoundResetsCval(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))

	co.Put(newRsvpFrom(1, 1, 0, ""))
	co.Put(newRsvpFrom(2, 1, 0, ""))
	co.Put(newRsvpFrom(3, 1, 0, ""))
	co.Put(newRsvpFrom(4, 1, 0, ""))
	co.Put(newRsvpFrom(5, 1, 0, ""))
	co.Put(newRsvpFrom(6, 1, 0, ""))

	co.Put(newTick()) // force the start of a new round

	co.Put(newRsvpFrom(1, 11, 0, ""))
	co.Put(newRsvpFrom(2, 11, 0, ""))
	co.Put(newRsvpFrom(3, 11, 0, ""))
	co.Put(newRsvpFrom(4, 11, 0, ""))
	co.Put(newRsvpFrom(5, 11, 0, ""))
	co.Put(newRsvpFrom(6, 11, 0, ""))

	assert.Equal(t, newNominate(11, "foo"), got)
}

func TestCoordStartRsvp(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newRsvpFrom(1, 1, 0, ""))
	co.Put(newRsvpFrom(2, 1, 0, ""))
	co.Put(newRsvpFrom(3, 1, 0, ""))
	co.Put(newRsvpFrom(4, 1, 0, ""))
	co.Put(newRsvpFrom(5, 1, 0, ""))
	co.Put(newRsvpFrom(6, 1, 0, ""))

	co.Put(newPropose("foo"))

	// If the RSVPs were ignored, this will be an invite. Otherwise, it'll be a
	// nominate.
	assert.Equal(t, newInvite(1), got)
}
<commit_msg>paxos: test that coordinator waits for quorum<commit_after>package paxos

import (
	"doozer/assert"
	"testing"
)

func TestCoordIgnoreOldMessages(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))

	co.Put(newTick()) // force the start of a new round

	got = Msg{}
	co.Put(newRsvpFrom(1, 1, 0, ""))
	co.Put(newRsvpFrom(2, 1, 0, ""))
	co.Put(newRsvpFrom(3, 1, 0, ""))
	co.Put(newRsvpFrom(4, 1, 0, ""))
	co.Put(newRsvpFrom(5, 1, 0, ""))
	co.Put(newRsvpFrom(6, 1, 0, ""))
	assert.Equal(t, Msg{}, got)
}

func TestCoordStart(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))
	assert.Equal(t, newInvite(1), got)
}

// This is here mainly for triangulation.  It ensures we're not
// hardcoding crnd.
func TestCoordStartAlt(t *testing.T) {
	var got Msg
	cx := newCluster("c", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))
	assert.Equal(t, newInvite(2), got)
}

func TestCoordQuorum(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))

	got = nil
	co.Put(newRsvpFrom(2, 1, 0, ""))
	assert.Equal(t, Msg(nil), got)
	co.Put(newRsvpFrom(3, 1, 0, ""))
	assert.Equal(t, Msg(nil), got)
	co.Put(newRsvpFrom(4, 1, 0, ""))
	assert.Equal(t, Msg(nil), got)
	co.Put(newRsvpFrom(5, 1, 0, ""))
	assert.Equal(t, Msg(nil), got)
	co.Put(newRsvpFrom(6, 1, 0, ""))
	assert.Equal(t, Msg(nil), got)
}

func TestCoordTargetNomination(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))

	co.Put(newRsvpFrom(2, 1, 0, ""))
	co.Put(newRsvpFrom(3, 1, 0, ""))
	co.Put(newRsvpFrom(4, 1, 0, ""))
	co.Put(newRsvpFrom(5, 1, 0, ""))
	co.Put(newRsvpFrom(6, 1, 0, ""))
	co.Put(newRsvpFrom(7, 1, 0, ""))
	assert.Equal(t, newNominate(1, "foo"), got)
}

func TestCoordRestart(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))

	// never reach majority (force timeout)
	co.Put(newRsvpFrom(2, 1, 0, ""))
	co.Put(newRsvpFrom(3, 1, 0, ""))
	co.Put(newRsvpFrom(4, 1, 0, ""))
	co.Put(newRsvpFrom(5, 1, 0, ""))
	co.Put(newRsvpFrom(6, 1, 0, ""))

	co.Put(newTick()) // force the start of a new round
	assert.Equal(t, newInvite(11), got)
}

func TestCoordNonTargetNomination(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))

	co.Put(newRsvpFrom(1, 1, 0, ""))
	co.Put(newRsvpFrom(2, 1, 0, ""))
	co.Put(newRsvpFrom(3, 1, 0, ""))
	co.Put(newRsvpFrom(4, 1, 0, ""))
	co.Put(newRsvpFrom(5, 1, 0, ""))
	co.Put(newRsvpFrom(6, 1, 1, "bar"))
	assert.Equal(t, newNominate(1, "bar"), got)
}

func TestCoordOneNominationPerRound(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))

	co.Put(newRsvpFrom(1, 1, 0, ""))
	co.Put(newRsvpFrom(2, 1, 0, ""))
	co.Put(newRsvpFrom(3, 1, 0, ""))
	co.Put(newRsvpFrom(4, 1, 0, ""))
	co.Put(newRsvpFrom(5, 1, 0, ""))
	co.Put(newRsvpFrom(6, 1, 0, ""))
	assert.Equal(t, newNominate(1, "foo"), got)

	got = Msg{}
	co.Put(newRsvpFrom(7, 1, 0, ""))
	assert.Equal(t, Msg{}, got)
}

func TestCoordEachRoundResetsCval(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newPropose("foo"))

	co.Put(newRsvpFrom(1, 1, 0, ""))
	co.Put(newRsvpFrom(2, 1, 0, ""))
	co.Put(newRsvpFrom(3, 1, 0, ""))
	co.Put(newRsvpFrom(4, 1, 0, ""))
	co.Put(newRsvpFrom(5, 1, 0, ""))
	co.Put(newRsvpFrom(6, 1, 0, ""))

	co.Put(newTick()) // force the start of a new round

	co.Put(newRsvpFrom(1, 11, 0, ""))
	co.Put(newRsvpFrom(2, 11, 0, ""))
	co.Put(newRsvpFrom(3, 11, 0, ""))
	co.Put(newRsvpFrom(4, 11, 0, ""))
	co.Put(newRsvpFrom(5, 11, 0, ""))
	co.Put(newRsvpFrom(6, 11, 0, ""))

	assert.Equal(t, newNominate(11, "foo"), got)
}

func TestCoordStartRsvp(t *testing.T) {
	var got Msg
	cx := newCluster("b", tenNodes, tenIds, nil)
	co := coordinator{cx: cx, crnd: uint64(cx.SelfIndex()), outs: msgSlot{&got}}

	co.Put(newRsvpFrom(1, 1, 0, ""))
	co.Put(newRsvpFrom(2, 1, 0, ""))
	co.Put(newRsvpFrom(3, 1, 0, ""))
	co.Put(newRsvpFrom(4, 1, 0, ""))
	co.Put(newRsvpFrom(5, 1, 0, ""))
	co.Put(newRsvpFrom(6, 1, 0, ""))

	co.Put(newPropose("foo"))

	// If the RSVPs were ignored, this will be an invite. Otherwise, it'll be a
	// nominate.
	assert.Equal(t, newInvite(1), got)
}
<|endoftext|>


<commit_before>// Package object includes all unified Object stuff and ways to persist it
package object

import "strings"

// Metadata represents standard metadata for unified objects.
// It implements Base interface and it's enough to include it into any struct to make object DB and API
// layers compatible.
type Metadata struct {
	Namespace  string
	Kind       string
	Name       string
	RandAddon  string
	Generation Generation
	// TODO(slukjanov): do we need CreatedAt string? I think yes
	// TODO(slukjanov): should any object have owner?
}

// GetKey returns object's Key
func (meta *Metadata) GetKey() string {
	// todo fix it!!1 separator and slice creation, cache key?
	return strings.Join([]string{meta.Namespace, meta.Kind, meta.Name, meta.Generation.String()}, "#")
}

func (meta *Metadata) GetRef() string {
	return meta.Namespace + KeySeparator + meta.Kind + KeySeparator + meta.Name
}

// GetNamespace returns object's Namespace
func (meta *Metadata) GetNamespace() string {
	return meta.Namespace
}

// GetKind returns object's Kind
func (meta *Metadata) GetKind() string {
	return meta.Kind
}

// GetName returns object's Name
func (meta *Metadata) GetName() string {
	return meta.Name
}

func (meta *Metadata) GetRandAddon() string {
	return meta.RandAddon
}

// GetGeneration returns object's Generation ("version")
func (meta *Metadata) GetGeneration() Generation {
	return meta.Generation
}
<commit_msg>Fix object/metadata<commit_after>package object

import "strings"

type Metadata struct {
	Namespace  string
	Kind       string
	Name       string
	Generation Generation
	// TODO(slukjanov): do we need CreatedAt string? I think yes
	// TODO(slukjanov): should any object have owner?
}

func (meta *Metadata) GetKey() string {
	return strings.Join([]string{meta.Namespace, meta.Kind, meta.Name}, KeySeparator)
}

// GetNamespace returns object's Namespace
func (meta *Metadata) GetNamespace() string {
	return meta.Namespace
}

// GetKind returns object's Kind
func (meta *Metadata) GetKind() string {
	return meta.Kind
}

// GetName returns object's Name
func (meta *Metadata) GetName() string {
	return meta.Name
}

// GetGeneration returns object's Generation ("version")
func (meta *Metadata) GetGeneration() Generation {
	return meta.Generation
}
<|endoftext|>