<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package helm // import "k8s.io/helm/pkg/helm"

import (
	"errors"
	"path/filepath"
	"reflect"
	"testing"

	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"

	"k8s.io/helm/pkg/chartutil"
	cpb "k8s.io/helm/pkg/proto/hapi/chart"
	rls "k8s.io/helm/pkg/proto/hapi/release"
	tpb "k8s.io/helm/pkg/proto/hapi/services"
)

// path to example charts relative to pkg/helm.
const chartsDir = "../../docs/examples/"

// sentinel error to indicate to the helm client to not send the request to tiller.
var errSkip = errors.New("test: skip")

// Verify ReleaseListOption's are applied to a ListReleasesRequest correctly.
func TestListReleases_VerifyOptions(t *testing.T) {
	// Options testdata
	var limit = 2
	var offset = "offset"
	var filter = "filter"
	var sortBy = int32(2)
	var sortOrd = int32(1)
	var codes = []rls.Status_Code{
		rls.Status_FAILED,
		rls.Status_DELETED,
		rls.Status_DEPLOYED,
		rls.Status_SUPERSEDED,
	}
	var namespace = "namespace"

	// Expected ListReleasesRequest message
	exp := &tpb.ListReleasesRequest{
		Limit:       int64(limit),
		Offset:      offset,
		Filter:      filter,
		SortBy:      tpb.ListSort_SortBy(sortBy),
		SortOrder:   tpb.ListSort_SortOrder(sortOrd),
		StatusCodes: codes,
		Namespace:   namespace,
	}

	// Options used in ListReleases
	ops := []ReleaseListOption{
		ReleaseListSort(sortBy),
		ReleaseListOrder(sortOrd),
		ReleaseListLimit(limit),
		ReleaseListOffset(offset),
		ReleaseListFilter(filter),
		ReleaseListStatuses(codes),
		ReleaseListNamespace(namespace),
	}

	// BeforeCall option to intercept helm client ListReleasesRequest
	b4c := BeforeCall(func(_ context.Context, msg proto.Message) error {
		switch act := msg.(type) {
		case *tpb.ListReleasesRequest:
			t.Logf("ListReleasesRequest: %#+v\n", act)
			assert(t, exp, act)
		default:
			t.Fatalf("expected message of type ListReleasesRequest, got %T\n", act)
		}
		return errSkip
	})

	NewClient(b4c).ListReleases(ops...)
}

// Verify InstallOption's are applied to an InstallReleaseRequest correctly.
func TestInstallRelease_VerifyOptions(t *testing.T) {
	// Options testdata
	var disableHooks = true
	var releaseName = "test"
	var namespace = "default"
	var reuseName = true
	var dryRun = true
	var chartName = "alpine"
	var overrides = []byte("key1=value1,key2=value2")

	// Expected InstallReleaseRequest message
	exp := &tpb.InstallReleaseRequest{
		Chart:        loadChart(t, chartName),
		Values:       &cpb.Config{Raw: string(overrides)},
		DryRun:       dryRun,
		Name:         releaseName,
		DisableHooks: disableHooks,
		Namespace:    namespace,
		ReuseName:    reuseName,
	}

	// Options used in InstallRelease
	ops := []InstallOption{
		ValueOverrides(overrides),
		InstallDryRun(dryRun),
		ReleaseName(releaseName),
		InstallReuseName(reuseName),
		InstallDisableHooks(disableHooks),
	}

	// BeforeCall option to intercept helm client InstallReleaseRequest
	b4c := BeforeCall(func(_ context.Context, msg proto.Message) error {
		switch act := msg.(type) {
		case *tpb.InstallReleaseRequest:
			t.Logf("InstallReleaseRequest: %#+v\n", act)
			assert(t, exp, act)
		default:
			t.Fatalf("expected message of type InstallReleaseRequest, got %T\n", act)
		}
		return errSkip
	})

	NewClient(b4c).InstallRelease(chartName, namespace, ops...)
}

// Verify DeleteOptions's are applied to an UninstallReleaseRequest correctly.
func TestDeleteRelease_VerifyOptions(t *testing.T) {
	// Options testdata
	var releaseName = "test"
	var disableHooks = true
	var purgeFlag = true

	// Expected DeleteReleaseRequest message
	exp := &tpb.UninstallReleaseRequest{
		Name:         releaseName,
		Purge:        purgeFlag,
		DisableHooks: disableHooks,
	}

	// Options used in DeleteRelease
	ops := []DeleteOption{
		DeletePurge(purgeFlag),
		DeleteDisableHooks(disableHooks),
	}

	// BeforeCall option to intercept helm client DeleteReleaseRequest
	b4c := BeforeCall(func(_ context.Context, msg proto.Message) error {
		switch act := msg.(type) {
		case *tpb.UninstallReleaseRequest:
			t.Logf("UninstallReleaseRequest: %#+v\n", act)
			assert(t, exp, act)
		default:
			t.Fatalf("expected message of type UninstallReleaseRequest, got %T\n", act)
		}
		return errSkip
	})

	NewClient(b4c).DeleteRelease(releaseName, ops...)
}

// Verify UpdateOption's are applied to an UpdateReleaseRequest correctly.
func TestUpdateRelease_VerifyOptions(t *testing.T) {
	// Options testdata
	var chartName = "alpine"
	var releaseName = "test"
	var disableHooks = true
	var overrides = []byte("key1=value1,key2=value2")
	var dryRun = false

	// Expected UpdateReleaseRequest message
	exp := &tpb.UpdateReleaseRequest{
		Name:         releaseName,
		Chart:        loadChart(t, chartName),
		Values:       &cpb.Config{Raw: string(overrides)},
		DryRun:       dryRun,
		DisableHooks: disableHooks,
	}

	// Options used in UpdateRelease
	ops := []UpdateOption{
		UpgradeDryRun(dryRun),
		UpdateValueOverrides(overrides),
		UpgradeDisableHooks(disableHooks),
	}

	// BeforeCall option to intercept helm client UpdateReleaseRequest
	b4c := BeforeCall(func(_ context.Context, msg proto.Message) error {
		switch act := msg.(type) {
		case *tpb.UpdateReleaseRequest:
			t.Logf("UpdateReleaseRequest: %#+v\n", act)
			assert(t, exp, act)
		default:
			t.Fatalf("expected message of type UpdateReleaseRequest, got %T\n", act)
		}
		return errSkip
	})

	NewClient(b4c).UpdateRelease(releaseName, chartName, ops...)
}

// Verify RollbackOption's are applied to a RollbackReleaseRequest correctly.
func TestRollbackRelease_VerifyOptions(t *testing.T) {
	// Options testdata
	var disableHooks = true
	var releaseName = "test"
	var revision = int32(2)
	var dryRun = true

	// Expected RollbackReleaseRequest message
	exp := &tpb.RollbackReleaseRequest{
		Name:         releaseName,
		DryRun:       dryRun,
		Version:      revision,
		DisableHooks: disableHooks,
	}

	// Options used in RollbackRelease
	ops := []RollbackOption{
		RollbackDryRun(dryRun),
		RollbackVersion(revision),
		RollbackDisableHooks(disableHooks),
	}

	// BeforeCall option to intercept helm client RollbackReleaseRequest
	b4c := BeforeCall(func(_ context.Context, msg proto.Message) error {
		switch act := msg.(type) {
		case *tpb.RollbackReleaseRequest:
			t.Logf("RollbackReleaseRequest: %#+v\n", act)
			assert(t, exp, act)
		default:
			t.Fatalf("expected message of type RollbackReleaseRequest, got %T\n", act)
		}
		return errSkip
	})

	NewClient(b4c).RollbackRelease(releaseName, ops...)
}

// Verify StatusOption's are applied to a GetReleaseStatusRequest correctly.
func TestReleaseStatus_VerifyOptions(t *testing.T) {
	// Options testdata
	var releaseName = "test"
	var revision = int32(2)

	// Expected GetReleaseStatusRequest message
	exp := &tpb.GetReleaseStatusRequest{
		Name:    releaseName,
		Version: revision,
	}

	// BeforeCall option to intercept helm client GetReleaseStatusRequest
	b4c := BeforeCall(func(_ context.Context, msg proto.Message) error {
		switch act := msg.(type) {
		case *tpb.GetReleaseStatusRequest:
			t.Logf("GetReleaseStatusRequest: %#+v\n", act)
			assert(t, exp, act)
		default:
			t.Fatalf("expected message of type GetReleaseStatusRequest, got %T\n", act)
		}
		return errSkip
	})

	NewClient(b4c).ReleaseStatus(releaseName, StatusReleaseVersion(revision))
}

// Verify ContentOption's are applied to a GetReleaseContentRequest correctly.
func TestReleaseContent_VerifyOptions(t *testing.T) {
	// Options testdata
	var releaseName = "test"
	var revision = int32(2)

	// Expected GetReleaseContentRequest message
	exp := &tpb.GetReleaseContentRequest{
		Name:    releaseName,
		Version: revision,
	}

	// BeforeCall option to intercept helm client GetReleaseContentRequest
	b4c := BeforeCall(func(_ context.Context, msg proto.Message) error {
		switch act := msg.(type) {
		case *tpb.GetReleaseContentRequest:
			t.Logf("GetReleaseContentRequest: %#+v\n", act)
			assert(t, exp, act)
		default:
			t.Fatalf("expected message of type GetReleaseContentRequest, got %T\n", act)
		}
		return errSkip
	})

	NewClient(b4c).ReleaseContent(releaseName, ContentReleaseVersion(revision))
}

func assert(t *testing.T, expect, actual interface{}) {
	if !reflect.DeepEqual(expect, actual) {
		t.Fatalf("expected %#+v, actual %#+v\n", expect, actual)
	}
}

func loadChart(t *testing.T, name string) *cpb.Chart {
	c, err := chartutil.Load(filepath.Join(chartsDir, name))
	if err != nil {
		t.Fatalf("failed to load test chart (%q): %s\n", name, err)
	}
	return c
}
<commit_msg>Fixes TestInstallRelease_VerifyOptions & TestUpdateRelease_VerifyOptions<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package helm // import "k8s.io/helm/pkg/helm"

import (
	"errors"
	"path/filepath"
	"reflect"
	"testing"

	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"

	"k8s.io/helm/pkg/chartutil"
	cpb "k8s.io/helm/pkg/proto/hapi/chart"
	rls "k8s.io/helm/pkg/proto/hapi/release"
	tpb "k8s.io/helm/pkg/proto/hapi/services"
)

// path to example charts relative to pkg/helm.
const chartsDir = "../../docs/examples/"

// sentinel error to indicate to the helm client to not send the request to tiller.
var errSkip = errors.New("test: skip")

// Verify ReleaseListOption's are applied to a ListReleasesRequest correctly.
func TestListReleases_VerifyOptions(t *testing.T) {
	// Options testdata
	var limit = 2
	var offset = "offset"
	var filter = "filter"
	var sortBy = int32(2)
	var sortOrd = int32(1)
	var codes = []rls.Status_Code{
		rls.Status_FAILED,
		rls.Status_DELETED,
		rls.Status_DEPLOYED,
		rls.Status_SUPERSEDED,
	}
	var namespace = "namespace"

	// Expected ListReleasesRequest message
	exp := &tpb.ListReleasesRequest{
		Limit:       int64(limit),
		Offset:      offset,
		Filter:      filter,
		SortBy:      tpb.ListSort_SortBy(sortBy),
		SortOrder:   tpb.ListSort_SortOrder(sortOrd),
		StatusCodes: codes,
		Namespace:   namespace,
	}

	// Options used in ListReleases
	ops := []ReleaseListOption{
		ReleaseListSort(sortBy),
		ReleaseListOrder(sortOrd),
		ReleaseListLimit(limit),
		ReleaseListOffset(offset),
		ReleaseListFilter(filter),
		ReleaseListStatuses(codes),
		ReleaseListNamespace(namespace),
	}

	// BeforeCall option to intercept helm client ListReleasesRequest
	b4c := BeforeCall(func(_ context.Context, msg proto.Message) error {
		switch act := msg.(type) {
		case *tpb.ListReleasesRequest:
			t.Logf("ListReleasesRequest: %#+v\n", act)
			assert(t, exp, act)
		default:
			t.Fatalf("expected message of type ListReleasesRequest, got %T\n", act)
		}
		return errSkip
	})

	if _, err := NewClient(b4c).ListReleases(ops...); err != errSkip {
		t.Fatalf("did not expect error but got (%v)\n``", err)
	}
}

// Verify InstallOption's are applied to an InstallReleaseRequest correctly.
func TestInstallRelease_VerifyOptions(t *testing.T) {
	// Options testdata
	var disableHooks = true
	var releaseName = "test"
	var namespace = "default"
	var reuseName = true
	var dryRun = true
	var chartName = "alpine"
	var chartPath = filepath.Join(chartsDir, chartName)
	var overrides = []byte("key1=value1,key2=value2")

	// Expected InstallReleaseRequest message
	exp := &tpb.InstallReleaseRequest{
		Chart:        loadChart(t, chartName),
		Values:       &cpb.Config{Raw: string(overrides)},
		DryRun:       dryRun,
		Name:         releaseName,
		DisableHooks: disableHooks,
		Namespace:    namespace,
		ReuseName:    reuseName,
	}

	// Options used in InstallRelease
	ops := []InstallOption{
		ValueOverrides(overrides),
		InstallDryRun(dryRun),
		ReleaseName(releaseName),
		InstallReuseName(reuseName),
		InstallDisableHooks(disableHooks),
	}

	// BeforeCall option to intercept helm client InstallReleaseRequest
	b4c := BeforeCall(func(_ context.Context, msg proto.Message) error {
		switch act := msg.(type) {
		case *tpb.InstallReleaseRequest:
			t.Logf("InstallReleaseRequest: %#+v\n", act)
			assert(t, exp, act)
		default:
			t.Fatalf("expected message of type InstallReleaseRequest, got %T\n", act)
		}
		return errSkip
	})

	if _, err := NewClient(b4c).InstallRelease(chartPath, namespace, ops...); err != errSkip {
		t.Fatalf("did not expect error but got (%v)\n``", err)
	}
}

// Verify DeleteOptions's are applied to an UninstallReleaseRequest correctly.
func TestDeleteRelease_VerifyOptions(t *testing.T) {
	// Options testdata
	var releaseName = "test"
	var disableHooks = true
	var purgeFlag = true

	// Expected DeleteReleaseRequest message
	exp := &tpb.UninstallReleaseRequest{
		Name:         releaseName,
		Purge:        purgeFlag,
		DisableHooks: disableHooks,
	}

	// Options used in DeleteRelease
	ops := []DeleteOption{
		DeletePurge(purgeFlag),
		DeleteDisableHooks(disableHooks),
	}

	// BeforeCall option to intercept helm client DeleteReleaseRequest
	b4c := BeforeCall(func(_ context.Context, msg proto.Message) error {
		switch act := msg.(type) {
		case *tpb.UninstallReleaseRequest:
			t.Logf("UninstallReleaseRequest: %#+v\n", act)
			assert(t, exp, act)
		default:
			t.Fatalf("expected message of type UninstallReleaseRequest, got %T\n", act)
		}
		return errSkip
	})

	if _, err := NewClient(b4c).DeleteRelease(releaseName, ops...); err != errSkip {
		t.Fatalf("did not expect error but got (%v)\n``", err)
	}
}

// Verify UpdateOption's are applied to an UpdateReleaseRequest correctly.
func TestUpdateRelease_VerifyOptions(t *testing.T) {
	// Options testdata
	var chartName = "alpine"
	var chartPath = filepath.Join(chartsDir, chartName)
	var releaseName = "test"
	var disableHooks = true
	var overrides = []byte("key1=value1,key2=value2")
	var dryRun = false

	// Expected UpdateReleaseRequest message
	exp := &tpb.UpdateReleaseRequest{
		Name:         releaseName,
		Chart:        loadChart(t, chartName),
		Values:       &cpb.Config{Raw: string(overrides)},
		DryRun:       dryRun,
		DisableHooks: disableHooks,
	}

	// Options used in UpdateRelease
	ops := []UpdateOption{
		UpgradeDryRun(dryRun),
		UpdateValueOverrides(overrides),
		UpgradeDisableHooks(disableHooks),
	}

	// BeforeCall option to intercept helm client UpdateReleaseRequest
	b4c := BeforeCall(func(_ context.Context, msg proto.Message) error {
		switch act := msg.(type) {
		case *tpb.UpdateReleaseRequest:
			t.Logf("UpdateReleaseRequest: %#+v\n", act)
			assert(t, exp, act)
		default:
			t.Fatalf("expected message of type UpdateReleaseRequest, got %T\n", act)
		}
		return errSkip
	})

	if _, err := NewClient(b4c).UpdateRelease(releaseName, chartPath, ops...); err != errSkip {
		t.Fatalf("did not expect error but got (%v)\n``", err)
	}
}

// Verify RollbackOption's are applied to a RollbackReleaseRequest correctly.
func TestRollbackRelease_VerifyOptions(t *testing.T) {
	// Options testdata
	var disableHooks = true
	var releaseName = "test"
	var revision = int32(2)
	var dryRun = true

	// Expected RollbackReleaseRequest message
	exp := &tpb.RollbackReleaseRequest{
		Name:         releaseName,
		DryRun:       dryRun,
		Version:      revision,
		DisableHooks: disableHooks,
	}

	// Options used in RollbackRelease
	ops := []RollbackOption{
		RollbackDryRun(dryRun),
		RollbackVersion(revision),
		RollbackDisableHooks(disableHooks),
	}

	// BeforeCall option to intercept helm client RollbackReleaseRequest
	b4c := BeforeCall(func(_ context.Context, msg proto.Message) error {
		switch act := msg.(type) {
		case *tpb.RollbackReleaseRequest:
			t.Logf("RollbackReleaseRequest: %#+v\n", act)
			assert(t, exp, act)
		default:
			t.Fatalf("expected message of type RollbackReleaseRequest, got %T\n", act)
		}
		return errSkip
	})

	if _, err := NewClient(b4c).RollbackRelease(releaseName, ops...); err != errSkip {
		t.Fatalf("did not expect error but got (%v)\n``", err)
	}
}

// Verify StatusOption's are applied to a GetReleaseStatusRequest correctly.
func TestReleaseStatus_VerifyOptions(t *testing.T) {
	// Options testdata
	var releaseName = "test"
	var revision = int32(2)

	// Expected GetReleaseStatusRequest message
	exp := &tpb.GetReleaseStatusRequest{
		Name:    releaseName,
		Version: revision,
	}

	// BeforeCall option to intercept helm client GetReleaseStatusRequest
	b4c := BeforeCall(func(_ context.Context, msg proto.Message) error {
		switch act := msg.(type) {
		case *tpb.GetReleaseStatusRequest:
			t.Logf("GetReleaseStatusRequest: %#+v\n", act)
			assert(t, exp, act)
		default:
			t.Fatalf("expected message of type GetReleaseStatusRequest, got %T\n", act)
		}
		return errSkip
	})

	if _, err := NewClient(b4c).ReleaseStatus(releaseName, StatusReleaseVersion(revision)); err != errSkip {
		t.Fatalf("did not expect error but got (%v)\n``", err)
	}
}

// Verify ContentOption's are applied to a GetReleaseContentRequest correctly.
func TestReleaseContent_VerifyOptions(t *testing.T) {
	// Options testdata
	var releaseName = "test"
	var revision = int32(2)

	// Expected GetReleaseContentRequest message
	exp := &tpb.GetReleaseContentRequest{
		Name:    releaseName,
		Version: revision,
	}

	// BeforeCall option to intercept helm client GetReleaseContentRequest
	b4c := BeforeCall(func(_ context.Context, msg proto.Message) error {
		switch act := msg.(type) {
		case *tpb.GetReleaseContentRequest:
			t.Logf("GetReleaseContentRequest: %#+v\n", act)
			assert(t, exp, act)
		default:
			t.Fatalf("expected message of type GetReleaseContentRequest, got %T\n", act)
		}
		return errSkip
	})

	if _, err := NewClient(b4c).ReleaseContent(releaseName, ContentReleaseVersion(revision)); err != errSkip {
		t.Fatalf("did not expect error but got (%v)\n``", err)
	}
}

func assert(t *testing.T, expect, actual interface{}) {
	if !reflect.DeepEqual(expect, actual) {
		t.Fatalf("expected %#+v, actual %#+v\n", expect, actual)
	}
}

func loadChart(t *testing.T, name string) *cpb.Chart {
	c, err := chartutil.Load(filepath.Join(chartsDir, name))
	if err != nil {
		t.Fatalf("failed to load test chart (%q): %s\n", name, err)
	}
	return c
}
<|endoftext|>


<commit_before>// Copyright (c) 2017-2019 Snowflake Computing Inc. All right reserved.

package gosnowflake

// SnowflakeGoDriverVersion is the version of Go Snowflake Driver.
const SnowflakeGoDriverVersion = "1.3.10"
<commit_msg>Bump GoSnowflake version to 1.3.11 (#349)<commit_after>// Copyright (c) 2017-2019 Snowflake Computing Inc. All right reserved.

package gosnowflake

// SnowflakeGoDriverVersion is the version of Go Snowflake Driver.
const SnowflakeGoDriverVersion = "1.3.11"
<|endoftext|>


<commit_before>// Package consolidation provides an abstraction for consolidators
package consolidation

import (
	"errors"
	"fmt"

	"github.com/raintank/schema"

	"github.com/grafana/metrictank/batch"
)

// consolidator is a highlevel description of a point consolidation method
// mostly for use by the http api, but can also be used internally for data processing
//go:generate msgp
type Consolidator int

var errUnknownConsolidationFunction = errors.New("unknown consolidation function")

const (
	None Consolidator = iota
	Avg
	Sum
	Lst
	Max
	Min
	Cnt // not available through http api
	Mult
	Med
	Diff
	StdDev
	Range
)

// String provides human friendly names
func (c Consolidator) String() string {
	switch c {
	case None:
		return "NoneConsolidator"
	case Avg:
		return "AverageConsolidator"
	case Cnt:
		return "CountConsolidator"
	case Lst:
		return "LastConsolidator"
	case Min:
		return "MinimumConsolidator"
	case Max:
		return "MaximumConsolidator"
	case Mult:
		return "MultiplyConsolidator"
	case Med:
		return "MedianConsolidator"
	case Diff:
		return "DifferenceConsolidator"
	case StdDev:
		return "StdDevConsolidator"
	case Range:
		return "RangeConsolidator"
	case Sum:
		return "SumConsolidator"
	}
	panic(fmt.Sprintf("Consolidator.String(): unknown consolidator %d", c))
}

// provide the name of a stored archive
// see aggregator.go for which archives are available
func (c Consolidator) Archive() schema.Method {
	switch c {
	case None:
		panic("cannot get an archive for no consolidation")
	case Avg:
		panic("avg consolidator has no matching Archive(). you need sum and cnt")
	case Cnt:
		return schema.Cnt
	case Lst:
		return schema.Lst
	case Min:
		return schema.Min
	case Max:
		return schema.Max
	case Sum:
		return schema.Sum
	}
	panic(fmt.Sprintf("Consolidator.Archive(): unknown consolidator %q", c))
}

func FromArchive(archive schema.Method) Consolidator {
	switch archive {
	case schema.Cnt:
		return Cnt
	case schema.Lst:
		return Lst
	case schema.Min:
		return Min
	case schema.Max:
		return Max
	case schema.Sum:
		return Sum
	}
	return None
}

func FromConsolidateBy(c string) Consolidator {
	switch c {
	case "avg", "average":
		return Avg
	case "cnt":
		return Cnt // bonus. not supported by graphite
	case "lst", "last", "current":
		return Lst
	case "min":
		return Min
	case "max":
		return Max
	case "mult", "multiply":
		return Mult
	case "med", "median":
		return Med
	case "diff":
		return Diff
	case "stddev":
		return StdDev
	case "range", "rangeOf":
		return Range
	case "sum", "total":
		return Sum
	}
	return None
}

// map the consolidation to the respective aggregation function, if applicable.
func GetAggFunc(consolidator Consolidator) batch.AggFunc {
	var consFunc batch.AggFunc
	switch consolidator {
	case Avg:
		consFunc = batch.Avg
	case Cnt:
		consFunc = batch.Cnt
	case Lst:
		consFunc = batch.Lst
	case Min:
		consFunc = batch.Min
	case Max:
		consFunc = batch.Max
	case Mult:
		consFunc = batch.Mult
	case Med:
		consFunc = batch.Med
	case Diff:
		consFunc = batch.Diff
	case StdDev:
		consFunc = batch.StdDev
	case Range:
		consFunc = batch.Range
	case Sum:
		consFunc = batch.Sum
	}
	return consFunc
}

func Validate(fn string) error {
	if fn == "avg" || fn == "average" ||
		fn == "count" ||
		fn == "last" || fn == "current" ||
		fn == "min" ||
		fn == "max" ||
		fn == "mult" || fn == "multiply" ||
		fn == "med" || fn == "median" ||
		fn == "diff" ||
		fn == "stddev" ||
		fn == "range" || fn == "rangeOf" ||
		fn == "sum" || fn == "total" {
		return nil
	}
	return errUnknownConsolidationFunction
}
<commit_msg>Align count behavior for Validate and FromConsolidateBy<commit_after>// Package consolidation provides an abstraction for consolidators
package consolidation

import (
	"errors"
	"fmt"

	"github.com/raintank/schema"

	"github.com/grafana/metrictank/batch"
)

// consolidator is a highlevel description of a point consolidation method
// mostly for use by the http api, but can also be used internally for data processing
//go:generate msgp
type Consolidator int

var errUnknownConsolidationFunction = errors.New("unknown consolidation function")

const (
	None Consolidator = iota
	Avg
	Sum
	Lst
	Max
	Min
	Cnt // not available through http api
	Mult
	Med
	Diff
	StdDev
	Range
)

// String provides human friendly names
func (c Consolidator) String() string {
	switch c {
	case None:
		return "NoneConsolidator"
	case Avg:
		return "AverageConsolidator"
	case Cnt:
		return "CountConsolidator"
	case Lst:
		return "LastConsolidator"
	case Min:
		return "MinimumConsolidator"
	case Max:
		return "MaximumConsolidator"
	case Mult:
		return "MultiplyConsolidator"
	case Med:
		return "MedianConsolidator"
	case Diff:
		return "DifferenceConsolidator"
	case StdDev:
		return "StdDevConsolidator"
	case Range:
		return "RangeConsolidator"
	case Sum:
		return "SumConsolidator"
	}
	panic(fmt.Sprintf("Consolidator.String(): unknown consolidator %d", c))
}

// provide the name of a stored archive
// see aggregator.go for which archives are available
func (c Consolidator) Archive() schema.Method {
	switch c {
	case None:
		panic("cannot get an archive for no consolidation")
	case Avg:
		panic("avg consolidator has no matching Archive(). you need sum and cnt")
	case Cnt:
		return schema.Cnt
	case Lst:
		return schema.Lst
	case Min:
		return schema.Min
	case Max:
		return schema.Max
	case Sum:
		return schema.Sum
	}
	panic(fmt.Sprintf("Consolidator.Archive(): unknown consolidator %q", c))
}

func FromArchive(archive schema.Method) Consolidator {
	switch archive {
	case schema.Cnt:
		return Cnt
	case schema.Lst:
		return Lst
	case schema.Min:
		return Min
	case schema.Max:
		return Max
	case schema.Sum:
		return Sum
	}
	return None
}

func FromConsolidateBy(c string) Consolidator {
	switch c {
	case "avg", "average":
		return Avg
	case "cnt", "count":
		return Cnt // bonus. not supported by graphite
	case "lst", "last", "current":
		return Lst
	case "min":
		return Min
	case "max":
		return Max
	case "mult", "multiply":
		return Mult
	case "med", "median":
		return Med
	case "diff":
		return Diff
	case "stddev":
		return StdDev
	case "range", "rangeOf":
		return Range
	case "sum", "total":
		return Sum
	}
	return None
}

// map the consolidation to the respective aggregation function, if applicable.
func GetAggFunc(consolidator Consolidator) batch.AggFunc {
	var consFunc batch.AggFunc
	switch consolidator {
	case Avg:
		consFunc = batch.Avg
	case Cnt:
		consFunc = batch.Cnt
	case Lst:
		consFunc = batch.Lst
	case Min:
		consFunc = batch.Min
	case Max:
		consFunc = batch.Max
	case Mult:
		consFunc = batch.Mult
	case Med:
		consFunc = batch.Med
	case Diff:
		consFunc = batch.Diff
	case StdDev:
		consFunc = batch.StdDev
	case Range:
		consFunc = batch.Range
	case Sum:
		consFunc = batch.Sum
	}
	return consFunc
}

func Validate(fn string) error {
	if fn == "avg" || fn == "average" ||
		fn == "count" || fn == "cnt" ||
		fn == "last" || fn == "current" ||
		fn == "min" ||
		fn == "max" ||
		fn == "mult" || fn == "multiply" ||
		fn == "med" || fn == "median" ||
		fn == "diff" ||
		fn == "stddev" ||
		fn == "range" || fn == "rangeOf" ||
		fn == "sum" || fn == "total" {
		return nil
	}
	return errUnknownConsolidationFunction
}
<|endoftext|>


<commit_before>package godata

const (
	ALLPAGES = "allpages"
	NONE     = "none"
)

func ParseInlineCountString(inlinecount string) (*GoDataInlineCountQuery, error) {
	result := GoDataInlineCountQuery(inlinecount)
	if inlinecount == ALLPAGES {
		return &result, nil
	} else if inlinecount == NONE {
		return &result, nil
	} else {
		return nil, BadRequestError("Could not parse orderby query.")
	}
}
<commit_msg>Fix inlinecount error message<commit_after>package godata

const (
	ALLPAGES = "allpages"
	NONE     = "none"
)

func ParseInlineCountString(inlinecount string) (*GoDataInlineCountQuery, error) {
	result := GoDataInlineCountQuery(inlinecount)
	if inlinecount == ALLPAGES {
		return &result, nil
	} else if inlinecount == NONE {
		return &result, nil
	} else {
		return nil, BadRequestError("Invalid inlinecount query.")
	}
}
<|endoftext|>


<commit_before>package boltql

import (
	"bytes"
	"errors"
	"fmt"

	"github.com/boltdb/bolt"
	"github.com/gobs/typedbuffer"
)

var (
	NO_TABLE         = bolt.ErrBucketNotFound
	NO_INDEX         = bolt.ErrBucketNotFound
	ALREADY_EXISTS   = bolt.ErrBucketExists
	NO_SCHEMA        = errors.New("no schema for table")
	SCHEMA_CORRUPTED = errors.New("schema corrupted")
	NO_KEY           = errors.New("key not found")
	BAD_VALUES       = errors.New("bad values")
)

type DataStore bolt.DB

type DataRecord interface {
	ToFieldList() []interface{}
	FromFieldList([]interface{})
}

// Open the database (create if it doesn't exist)
func Open(dbfile string) (*DataStore, error) {
	db, err := bolt.Open(dbfile, 0666, nil)
	if err != nil {
		return nil, err
	}

	return (*DataStore)(db), nil
}

// Close the database
func (d *DataStore) Close() {
	d.Close()
}

func indices(name string) []byte {
	return []byte(name + "_idx")
}

func schema(name string) []byte {
	return []byte(name + "_schema")
}

//
// A Table is a container for the table name and indices
//
type Table struct {
	name    string
	indices map[string][]uint64

	d *DataStore
}

func (t *Table) String() string {
	return fmt.Sprintf("Table{name: %q, indices: %v}", t.name, t.indices)
}

//
// Create table if doesn't exist
//
func (d *DataStore) CreateTable(name string) (*Table, error) {
	db := (*bolt.DB)(d)

	err := db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucket([]byte(name))
		if err != nil {
			return err
		}
		_, err = tx.CreateBucket(schema(name))
		if err != nil {
			return err
		}

		return nil
	})

	if err == nil {
		return &Table{name: name, indices: map[string][]uint64{}, d: d}, nil
	} else {
		return nil, err
	}
}

// Get table info
func (d *DataStore) GetTable(name string) (*Table, error) {
	db := (*bolt.DB)(d)
	table := Table{name: name, indices: map[string][]uint64{}, d: d}

	err := db.View(func(tx *bolt.Tx) error {
		if tx.Bucket([]byte(name)) == nil {
			return NO_TABLE
		}

		b := tx.Bucket(schema(name))
		if b == nil {
			return NO_SCHEMA
		}

		b.ForEach(func(k, v []byte) error {
			name := string(k)
			fields, err := typedbuffer.DecodeUintArray(v)
			if err != nil {
				return SCHEMA_CORRUPTED
			}

			table.indices[name] = fields
			return nil
		})

		return nil
	})

	if err == nil {
		return &table, nil
	} else {
		return nil, err
	}
}

func (t *Table) CreateIndex(index string, fields ...uint64) error {
	db := (*bolt.DB)(t.d)

	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(schema(t.name))
		if b == nil {
			return NO_TABLE
		}

		enc, err := typedbuffer.Encode(fields)
		if err != nil {
			return BAD_VALUES
		}

		if err := b.Put([]byte(index), enc); err != nil {
			return err
		}

		if _, err := tx.CreateBucket(indices(index)); err != nil {
			return err
		}

		return nil
	})

	if err == nil {
		t.indices[index] = fields
	}

	return err
}

func marshalKeyValue(keys []uint64, fields []interface{}) (key, value []byte, err error) {
	vkey := make([]interface{}, len(keys))
	vval := make([]interface{}, len(fields)-len(keys)) // only the entries not stored as keys

	ki := 0
	vi := 0

	for fi, fv := range fields {
		if len(keys) > 0 && fi == int(keys[0]) {
			vkey[ki] = fv
			ki += 1
			keys = keys[1:]
		} else {
			vval[vi] = fv
			vi += 1
		}
	}

	if len(vkey) > 0 {
		if key, err = typedbuffer.Encode(vkey...); err != nil {
			return
		}
	}

	value, err = typedbuffer.Encode(vval...)
	return
}

// Add a record to the table (using sequential record number)
func (t *Table) Put(rec DataRecord) (uint64, error) {
	db := (*bolt.DB)(t.d)

	var key uint64

	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(t.name))
		if b == nil {
			return NO_TABLE
		}

		fields := rec.ToFieldList()

		data, err := typedbuffer.Encode(fields...)
		if err != nil {
			return err
		}

		key, err = b.NextSequence()
		if err != nil {
			return err
		}

		if err = b.Put(typedbuffer.EncodeUint64(key), data); err != nil {
			return err
		}

		for index, keys := range t.indices {
			ib := tx.Bucket(indices(index))
			if ib == nil {
				return NO_TABLE
			}

			key, val, err := marshalKeyValue(keys, fields)
			if err != nil {
				return err
			}

			if key == nil {
				continue
			}

			if err := ib.Put(key, val); err != nil {
				return err
			}
		}

		return nil
	})

	return key, err
}

// Get a record from the table (using sequential record number)
func (t *Table) Get(key uint64, record DataRecord) error {
	db := (*bolt.DB)(t.d)

	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(t.name))
		if b == nil {
			return NO_TABLE
		}

		data := b.Get(typedbuffer.EncodeUint64(key))
		if data == nil {
			return NO_KEY
		}

		fields, err := typedbuffer.DecodeAll(data)
		if err != nil {
			return err
		}

		record.FromFieldList(fields)
		return nil
	})

	return err
}

// Delete a record from the table (using sequential record number)
func (t *Table) Delete(key uint64) error {
	db := (*bolt.DB)(t.d)

	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(t.name))
		if b == nil {
			return NO_TABLE
		}

		enkey := typedbuffer.EncodeUint64(key)

		c := b.Cursor()
		k, v := c.Seek(enkey)

		// Seek will return the next key if there is no match
		// so make sure we check we got the right record

		if bytes.Equal(enkey, k) {
			if err := c.Delete(); err != nil {
				return err
			}

			fields, err := typedbuffer.DecodeAll(v)
			if err != nil {
				return err
			}

			for index, keys := range t.indices {
				b := tx.Bucket(indices(index))
				if b == nil {
					continue
				}

				vkey := make([]interface{}, len(keys))
				for i, j := range keys {
					vkey[i] = fields[j]
				}

				dkey, err := typedbuffer.Encode(vkey...)
				if err != nil {
					return err
				}

				if err := b.Delete(dkey); err != nil {
					return err
				}
			}
		}

		return nil
	})

	return err
}

// Get all records sorted by sequential id (ascending or descending)
// Call user function with key (position) and record content
func (t *Table) ScanSequential(ascending bool, res DataRecord, callback func(uint64, DataRecord, error) bool) error {
	db := (*bolt.DB)(t.d)

	return db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(t.name))
		if b == nil {
			return NO_TABLE
		}

		c := b.Cursor()

		// ascending
		first := c.First
		next := c.Next

		// descending
		if !ascending {
			first = c.Last
			next = c.Prev
		}

		for k, v := first(); k != nil; k, v = next() {
			pos, _, _ := typedbuffer.Decode(k)

			fields, err := typedbuffer.DecodeAll(v)
			if err == nil {
				res.FromFieldList(fields)
			}

			if !callback(pos.(uint64), res, err) {
				break
			}
		}

		return nil
	})
}

// Get all records sorted by index (ascending or descending)
// Call user function with record content
func (t *Table) ScanIndex(index string, ascending bool, start, res DataRecord, callback func(DataRecord, error) bool) error {
	db := (*bolt.DB)(t.d)

	return db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(indices(index))
		if b == nil {
			return NO_INDEX
		}

		c := b.Cursor()

		keys := t.indices[index]

		var k, v []byte

		if start != nil {
			key, _, err := marshalKeyValue(keys, start.ToFieldList())
			if err != nil {
				return err
			}

			if key != nil {
				k, v = c.Seek(key)
				if !ascending && !bytes.Equal(key, k) {
					// if descending and keys don't match we want to start from the first key
					// in range (previous)

					k, v = c.Prev()
				}
			}
		}

		if k == nil {
			if ascending {
				k, v = c.First()
			} else {
				k, v = c.Last()
			}
		}

		var next func() (key []byte, value []byte)

		if ascending {
			next = c.Next
		} else {
			next = c.Prev
		}

		for ; k != nil; k, v = next() {
			vkey, err := typedbuffer.DecodeAll(k)
			if err != nil {
				return err
			}

			vval, err := typedbuffer.DecodeAll(v)
			if err != nil {
				return err
			}

			lkey := len(vkey)
			lval := len(vval)

			fields := []interface{}{}

			var ival interface{}

			ik := 0
			lk := len(keys)

			for i := 0; i < lkey+lval; i++ {
				if ik < lk && i == int(keys[ik]) {
					ival = vkey[0]
					vkey = vkey[1:]
					ik += 1
				} else {
					ival = vval[0]
					vval = vval[1:]
				}

				fields = append(fields, ival)
			}

			res.FromFieldList(fields)

			if !callback(res, err) {
				break
			}
		}

		return nil
	})
}

func (t *Table) ForEach(index string, callback func(k, b []byte) error) error {
	db := (*bolt.DB)(t.d)

	return db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(indices(index))
		if b == nil {
			return NO_INDEX
		}

		return b.ForEach(callback)
	})
}
<commit_msg>Fixed Close() that was just recusing and marshalKeyValue, that was rewriting the table indices.<commit_after>package boltql

import (
	"bytes"
	"errors"
	"fmt"

	"github.com/boltdb/bolt"
	"github.com/gobs/typedbuffer"
)

var (
	NO_TABLE         = bolt.ErrBucketNotFound
	NO_INDEX         = bolt.ErrBucketNotFound
	ALREADY_EXISTS   = bolt.ErrBucketExists
	NO_SCHEMA        = errors.New("no schema for table")
	SCHEMA_CORRUPTED = errors.New("schema corrupted")
	NO_KEY           = errors.New("key not found")
	BAD_VALUES       = errors.New("bad values")
)

type DataStore bolt.DB

type DataRecord interface {
	ToFieldList() []interface{}
	FromFieldList([]interface{})
}

// Open the database (create if it doesn't exist)
func Open(dbfile string) (*DataStore, error) {
	db, err := bolt.Open(dbfile, 0666, nil)
	if err != nil {
		return nil, err
	}

	return (*DataStore)(db), nil
}

// Close the database
func (d *DataStore) Close() error {
	db := (*bolt.DB)(d)
	return db.Close()
}

func indices(name string) []byte {
	return []byte(name + "_idx")
}

func schema(name string) []byte {
	return []byte(name + "_schema")
}

//
// A Table is a container for the table name and indices
//
type Table struct {
	name    string
	indices map[string][]uint64

	d *DataStore
}

func (t *Table) String() string {
	return fmt.Sprintf("Table{name: %q, indices: %v}", t.name, t.indices)
}

//
// Create table if doesn't exist
//
func (d *DataStore) CreateTable(name string) (*Table, error) {
	db := (*bolt.DB)(d)

	err := db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucket([]byte(name))
		if err != nil {
			return err
		}
		_, err = tx.CreateBucket(schema(name))
		if err != nil {
			return err
		}

		return nil
	})

	if err == nil {
		return &Table{name: name, indices: map[string][]uint64{}, d: d}, nil
	} else {
		return nil, err
	}
}

// Get table info
func (d *DataStore) GetTable(name string) (*Table, error) {
	db := (*bolt.DB)(d)
	table := Table{name: name, indices: map[string][]uint64{}, d: d}

	err := db.View(func(tx *bolt.Tx) error {
		if tx.Bucket([]byte(name)) == nil {
			return NO_TABLE
		}

		b := tx.Bucket(schema(name))
		if b == nil {
			return NO_SCHEMA
		}

		b.ForEach(func(k, v []byte) error {
			name := string(k)
			fields, err := typedbuffer.DecodeUintArray(v)
			if err != nil {
				return SCHEMA_CORRUPTED
			}

			table.indices[name] = fields
			return nil
		})

		return nil
	})

	if err == nil {
		return &table, nil
	} else {
		return nil, err
	}
}

func (t *Table) CreateIndex(index string, fields ...uint64) error {
	db := (*bolt.DB)(t.d)

	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(schema(t.name))
		if b == nil {
			return NO_TABLE
		}

		enc, err := typedbuffer.Encode(fields)
		if err != nil {
			return BAD_VALUES
		}

		if err := b.Put([]byte(index), enc); err != nil {
			return err
		}

		if _, err := tx.CreateBucket(indices(index)); err != nil {
			return err
		}

		return nil
	})

	if err == nil {
		t.indices[index] = fields
	}

	return err
}

func marshalKeyValue(keys []uint64, fields []interface{}) (key, value []byte, err error) {
	if len(keys) == 0 {
		return
	}

	var vkey, vval []interface{}

	kk, lk := 0, len(keys)

	for fi, fv := range fields {
		if kk < lk && fi == int(keys[kk]) {
			vkey = append(vkey, fv)
			kk += 1
		} else {
			vval = append(vval, fv)
		}
	}

	if len(vkey) > 0 {
		if key, err = typedbuffer.Encode(vkey...); err != nil {
			return
		}
	}

	if len(vval) > 0 {
		value, err = typedbuffer.Encode(vval...)
	}

	return
}

// Add a record to the table (using sequential record number)
func (t *Table) Put(rec DataRecord) (uint64, error) {
	db := (*bolt.DB)(t.d)

	var key uint64

	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(t.name))
		if b == nil {
			return NO_TABLE
		}

		fields := rec.ToFieldList()

		data, err := typedbuffer.Encode(fields...)
		if err != nil {
			return err
		}

		key, err = b.NextSequence()
		if err != nil {
			return err
		}

		if err = b.Put(typedbuffer.EncodeUint64(key), data); err != nil {
			return err
		}

		for index, keys := range t.indices {
			ib := tx.Bucket(indices(index))
			if ib == nil {
				return NO_TABLE
			}

			key, val, err := marshalKeyValue(keys, fields)
			if err != nil {
				return err
			}

			if key == nil {
				continue
			}

			if err := ib.Put(key, val); err != nil {
				return err
			}
		}

		return nil
	})

	return key, err
}

// Get a record from the table (using sequential record number)
func (t *Table) Get(key uint64, record DataRecord) error {
	db := (*bolt.DB)(t.d)

	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(t.name))
		if b == nil {
			return NO_TABLE
		}

		data := b.Get(typedbuffer.EncodeUint64(key))
		if data == nil {
			return NO_KEY
		}

		fields, err := typedbuffer.DecodeAll(data)
		if err != nil {
			return err
		}

		record.FromFieldList(fields)
		return nil
	})

	return err
}

// Delete a record from the table (using sequential record number)
func (t *Table) Delete(key uint64) error {
	db := (*bolt.DB)(t.d)

	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(t.name))
		if b == nil {
			return NO_TABLE
		}

		enkey := typedbuffer.EncodeUint64(key)

		c := b.Cursor()
		k, v := c.Seek(enkey)

		// Seek will return the next key if there is no match
		// so make sure we check we got the right record

		if bytes.Equal(enkey, k) {
			if err := c.Delete(); err != nil {
				return err
			}

			fields, err := typedbuffer.DecodeAll(v)
			if err != nil {
				return err
			}

			for index, keys := range t.indices {
				b := tx.Bucket(indices(index))
				if b == nil {
					continue
				}

				vkey := make([]interface{}, len(keys))
				for i, j := range keys {
					vkey[i] = fields[j]
				}

				dkey, err := typedbuffer.Encode(vkey...)
				if err != nil {
					return err
				}

				if err := b.Delete(dkey); err != nil {
					return err
				}
			}
		}

		return nil
	})

	return err
}

// Get all records sorted by sequential id (ascending or descending)
// Call user function with key (position) and record content
func (t *Table) ScanSequential(ascending bool, res DataRecord, callback func(uint64, DataRecord, error) bool) error {
	db := (*bolt.DB)(t.d)

	return db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(t.name))
		if b == nil {
			return NO_TABLE
		}

		c := b.Cursor()

		// ascending
		first := c.First
		next := c.Next

		// descending
		if !ascending {
			first = c.Last
			next = c.Prev
		}

		for k, v := first(); k != nil; k, v = next() {
			pos, _, _ := typedbuffer.Decode(k)

			fields, err := typedbuffer.DecodeAll(v)
			if err == nil {
				res.FromFieldList(fields)
			}

			if !callback(pos.(uint64), res, err) {
				break
			}
		}

		return nil
	})
}

// Get all records sorted by index (ascending or descending)
// Call user function with record content
func (t *Table) ScanIndex(index string, ascending bool, start, res DataRecord, callback func(DataRecord, error) bool) error {
	db := (*bolt.DB)(t.d)

	return db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(indices(index))
		if b == nil {
			return NO_INDEX
		}

		c := b.Cursor()

		keys := t.indices[index]

		var k, v []byte

		if start != nil {
			key, _, err := marshalKeyValue(keys, start.ToFieldList())
			if err != nil {
				return err
			}

			if key != nil {
				k, v = c.Seek(key)
				if !ascending && !bytes.Equal(key, k) {
					// if descending and keys don't match we want to start from the first key
					// in range (previous)

					k, v = c.Prev()
				}
			}
		}

		if k == nil {
			if ascending {
				k, v = c.First()
			} else {
				k, v = c.Last()
			}
		}

		var next func() (key []byte, value []byte)

		if ascending {
			next = c.Next
		} else {
			next = c.Prev
		}

		for ; k != nil; k, v = next() {
			vkey, err := typedbuffer.DecodeAll(k)
			if err != nil {
				return err
			}

			vval, err := typedbuffer.DecodeAll(v)
			if err != nil {
				return err
			}

			lkey := len(vkey)
			lval := len(vval)

			fields := []interface{}{}

			var ival interface{}

			ik := 0
			lk := len(keys)

			for i := 0; i < lkey+lval; i++ {
				if ik < lk && i == int(keys[ik]) {
					ival = vkey[0]
					vkey = vkey[1:]
					ik += 1
				} else {
					ival = vval[0]
					vval = vval[1:]
				}

				fields = append(fields, ival)
			}

			res.FromFieldList(fields)

			if !callback(res, err) {
				break
			}
		}

		return nil
	})
}

func (t *Table) ForEach(index string, callback func(k, b []byte) error) error {
	db := (*bolt.DB)(t.d)

	return db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(indices(index))
		if b == nil {
			return NO_INDEX
		}

		return b.ForEach(callback)
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"log"
	"json"
	"os"
	"path/filepath"
	"sync"

	"camli/blobref"
	"camli/client"
	"camli/schema"
	"camli/third_party/github.com/hanwen/go-fuse/fuse"
)

var _ = fmt.Println
var _ = log.Println

type CamliFileSystem struct {
	fuse.DefaultPathFilesystem

	fetcher blobref.Fetcher
	root   *blobref.BlobRef

	lk         sync.Mutex
	nameToBlob map[string]*blobref.BlobRef
}

func NewCamliFileSystem(client *client.Client, root *blobref.BlobRef) *CamliFileSystem {
	return &CamliFileSystem{
	fetcher: client,
	root: root,
	nameToBlob: make(map[string]*blobref.BlobRef),
	}
}

// Where name == "" for root,
// Returns nil on failure
func (fs *CamliFileSystem) blobRefFromNameCached(name string) *blobref.BlobRef {
	fs.lk.Lock()
	defer fs.lk.Unlock()
	return fs.nameToBlob[name]
}

func (fs *CamliFileSystem) fetchSchemaSuperset(br *blobref.BlobRef) (*schema.Superset, os.Error) {
	rsc, _, err := fs.fetcher.Fetch(br)
	if err != nil {
		return nil, err
	}
	defer rsc.Close()
	jd := json.NewDecoder(rsc)
	ss := new(schema.Superset)
        err = jd.Decode(ss)
        if err != nil {
		log.Printf("Error parsing %s as schema blob: %v", br, err)
                return nil, os.EINVAL
        }
	return ss, nil
}

// Where name == "" for root,
// Returns fuse.Status == fuse.OK on success or anything else on failure.
func (fs *CamliFileSystem) blobRefFromName(name string) (*blobref.BlobRef, fuse.Status) {
	if name == "" {
		return fs.root, fuse.OK
	}
	if br := fs.blobRefFromNameCached(name); br != nil {
		return br, fuse.OK
	}
	dir, fileName := filepath.Split(name)
	dirBlob, fuseStatus := fs.blobRefFromName(dir)
	if fuseStatus != fuse.OK {
		return nil, fuseStatus
	}

	dirss, err := fs.fetchSchemaSuperset(dirBlob)
	switch {
	case err == os.ENOENT:
		log.Printf("Failed to find directory %s", dirBlob)
		return nil, fuse.ENOENT
	case err == os.EINVAL:
		log.Printf("Failed to parse directory %s", dirBlob)
		return nil, fuse.ENOTDIR
	case err != nil:
		panic(fmt.Sprintf("Invalid fetcher error: %v", err))
	case dirss == nil:
		panic("nil dirss")
	case dirss.Type != "directory":
		log.Printf("Expected %s to be a directory; actually a %s",
			dirBlob, dirss.Type)
		return nil, fuse.ENOTDIR
	}

	if dirss.Entries == "" {
		log.Printf("Expected %s to have 'entries'", dirBlob)
		return nil, fuse.ENOTDIR
	}
	entriesBlob := blobref.Parse(dirss.Entries)
	if entriesBlob == nil {
		log.Printf("Blob %s had invalid blobref %q for its 'entries'", dirBlob, dirss.Entries)
		return nil, fuse.ENOTDIR
	}

	entss, err := fs.fetchSchemaSuperset(entriesBlob)
	switch {
	case err == os.ENOENT:
		log.Printf("Failed to find entries %s via directory %s", entriesBlob, dirBlob)
		return nil, fuse.ENOENT
	case err == os.EINVAL:
		log.Printf("Failed to parse entries %s via directory %s", entriesBlob, dirBlob)
		return nil, fuse.ENOTDIR
	case err != nil:
		panic(fmt.Sprintf("Invalid fetcher error: %v", err))
	case entss == nil:
		panic("nil entss")
	case entss.Type != "static-set":
		log.Printf("Expected %s to be a directory; actually a %s",
			dirBlob, dirss.Type)
		return nil, fuse.ENOTDIR
	}

	wg := new(sync.WaitGroup)
	foundCh := make(chan *blobref.BlobRef)
	for _, m := range entss.Members {
		wg.Add(1)
		go func(memberBlobstr string) {
			childss, err := fs.fetchSchemaSuperset(entriesBlob)
			if err == nil && childss.HasFilename(fileName) {
				foundCh <- entriesBlob
			}
			wg.Done()
		}(m)
	}
	failCh := make(chan string)
	go func() {
		wg.Wait()
		failCh <- "ENOENT"
	}()
	select {
	case found := <-foundCh:
		fs.lk.Lock()
		defer fs.lk.Unlock()
		fs.nameToBlob[name] = found
		return found, fuse.OK
	case <-failCh:
	}
	// TODO: negative cache
	return nil, fuse.ENOENT
}

func (fs *CamliFileSystem) Mount(connector *fuse.PathFileSystemConnector) fuse.Status {
	log.Printf("cammount: Mount")
	return fuse.OK
}

func (fs *CamliFileSystem) Unmount() {
	log.Printf("cammount: Unmount.")
}

func (fs *CamliFileSystem) GetAttr(name string) (*fuse.Attr, fuse.Status) {
	log.Printf("cammount: GetAttr(%q)", name)
	blobref, errStatus := fs.blobRefFromName(name)
	log.Printf("cammount: GetAttr(%q), blobRefFromName err=%v", name, errStatus)
	if errStatus != fuse.OK {
		return nil, errStatus
	}
	log.Printf("cammount: got blob %s", blobref)

	// TODO: this is redundant with what blobRefFromName already
	// did.  we should at least keep this in RAM (pre-de-JSON'd)
	// so we don't have to fetch + unmarshal it again.
	ss, err := fs.fetchSchemaSuperset(blobref)
	if err != nil {
		log.Printf("cammount: GetAttr(%q, %s): fetch schema error: %v", name, blobref, err)
		return nil, fuse.EIO
	}

	out := new(fuse.Attr)
	var fi os.FileInfo

	fi.Mode = ss.UnixMode()

	// TODO: have a mode to set permissions equal to mounting user?
	fi.Uid = ss.UnixOwnerId
	fi.Gid = ss.UnixGroupId

	// TODO: other types
	if ss.Type == "file" {
		fi.Size = ss.Size
	}

	// TODO: mtime and such

	fuse.CopyFileInfo(&fi, out)
	return out, fuse.OK
}

func (fs *CamliFileSystem) Access(name string, mode uint32) fuse.Status {
	log.Printf("cammount: Access(%q, %d)", name, mode)
	return fuse.OK
}

func (fs *CamliFileSystem) Open(name string, flags uint32) (file fuse.RawFuseFile, code fuse.Status) {
	log.Printf("cammount: Open(%q, %d)", name, flags)
	// TODO
	return nil, fuse.EACCES
}

func (fs *CamliFileSystem) OpenDir(name string) (stream chan fuse.DirEntry, code fuse.Status) {
	dirBlob, errStatus := fs.blobRefFromName(name)
	log.Printf("cammount: OpenDir(%q), dirBlob=%s err=%v", name, dirBlob, errStatus)
	if errStatus != fuse.OK {
		return nil, errStatus
	}

	// TODO: this is redundant with what blobRefFromName already
	// did.  we should at least keep this in RAM (pre-de-JSON'd)
	// so we don't have to fetch + unmarshal it again.
	dirss, err := fs.fetchSchemaSuperset(dirBlob)
	log.Printf("dirss blob: %v, err=%v", dirss, err)
	if err != nil {
		log.Printf("cammount: OpenDir(%q, %s): fetch schema error: %v", name, dirBlob, err)
		return nil, fuse.EIO
	}

	if dirss.Entries == "" {
		log.Printf("Expected %s to have 'entries'", dirBlob)
		return nil, fuse.ENOTDIR
	}
	entriesBlob := blobref.Parse(dirss.Entries)
	if entriesBlob == nil {
		log.Printf("Blob %s had invalid blobref %q for its 'entries'", dirBlob, dirss.Entries)
		return nil, fuse.ENOTDIR
	}

	entss, err := fs.fetchSchemaSuperset(entriesBlob)
	log.Printf("entries blob: %v, err=%v", entss, err)
	switch {
	case err == os.ENOENT:
		log.Printf("Failed to find entries %s via directory %s", entriesBlob, dirBlob)
		return nil, fuse.ENOENT
	case err == os.EINVAL:
		log.Printf("Failed to parse entries %s via directory %s", entriesBlob, dirBlob)
		return nil, fuse.ENOTDIR
	case err != nil:
		panic(fmt.Sprintf("Invalid fetcher error: %v", err))
	case entss == nil:
		panic("nil entss")
	case entss.Type != "static-set":
		log.Printf("Expected %s to be a directory; actually a %s",
			dirBlob, entss.Type)
		return nil, fuse.ENOTDIR
	}

	retch := make(chan fuse.DirEntry, 20)
	wg := new(sync.WaitGroup)
	for _, m := range entss.Members {
		wg.Add(1)
		go func(memberBlobstr string) {
			defer wg.Done()
			memberBlob := blobref.Parse(memberBlobstr)
			if memberBlob == nil {
				log.Printf("invalid blobref of %q in static set %s", memberBlobstr, entss)
				return
			}
			childss, err := fs.fetchSchemaSuperset(memberBlob)
			if err == nil {
				if childss.FileName != "" {
					mode := childss.UnixMode()
					//log.Printf("adding to dir %s: file=%q, mode=%d", dirBlob, childss.FileName, mode)
					retch <- fuse.DirEntry{Name: childss.FileName, Mode: mode}
				} else {
					log.Printf("Blob %s had no filename", childss.FileName)
				}
			} else {
				log.Printf("Error fetching %s: %v", memberBlobstr, err)
			}
		}(m)
	}
	go func() {
		wg.Wait()
		close(retch)
	}()
	return retch, fuse.OK
}

func (fs *CamliFileSystem) Readlink(name string) (string, fuse.Status) {
	log.Printf("cammount: Readlink(%q)", name)
	// TODO
	return "", fuse.EACCES
}
<commit_msg>cammount: working OpenDir!<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"log"
	"json"
	"os"
	"path/filepath"
	"sync"

	"camli/blobref"
	"camli/client"
	"camli/schema"
	"camli/third_party/github.com/hanwen/go-fuse/fuse"
)

var _ = fmt.Println
var _ = log.Println

type CamliFileSystem struct {
	fuse.DefaultPathFilesystem

	fetcher blobref.Fetcher
	root   *blobref.BlobRef

	lk         sync.Mutex
	nameToBlob map[string]*blobref.BlobRef
}

func NewCamliFileSystem(client *client.Client, root *blobref.BlobRef) *CamliFileSystem {
	return &CamliFileSystem{
	fetcher: client,
	root: root,
	nameToBlob: make(map[string]*blobref.BlobRef),
	}
}

// Where name == "" for root,
// Returns nil on failure
func (fs *CamliFileSystem) blobRefFromNameCached(name string) *blobref.BlobRef {
	fs.lk.Lock()
	defer fs.lk.Unlock()
	return fs.nameToBlob[name]
}

func (fs *CamliFileSystem) fetchSchemaSuperset(br *blobref.BlobRef) (*schema.Superset, os.Error) {
	rsc, _, err := fs.fetcher.Fetch(br)
	if err != nil {
		return nil, err
	}
	defer rsc.Close()
	jd := json.NewDecoder(rsc)
	ss := new(schema.Superset)
        err = jd.Decode(ss)
        if err != nil {
		log.Printf("Error parsing %s as schema blob: %v", br, err)
                return nil, os.EINVAL
        }
	return ss, nil
}

// Where name == "" for root,
// Returns fuse.Status == fuse.OK on success or anything else on failure.
func (fs *CamliFileSystem) blobRefFromName(name string) (retbr *blobref.BlobRef, retstatus fuse.Status) {
	if name == "" {
		return fs.root, fuse.OK
	}
	if br := fs.blobRefFromNameCached(name); br != nil {
		return br, fuse.OK
	}

	log.Printf("blobRefFromName(%q) = ...", name)
	defer func() {
		log.Printf("blobRefFromName(%q) = %s, %v", retbr, retstatus)
	}()

	dir, fileName := filepath.Split(name)
	if len(dir) > 0 {
		dir = dir[:len(dir)-1] // remove trailing "/" or whatever
	}
	dirBlob, fuseStatus := fs.blobRefFromName(dir)
	if fuseStatus != fuse.OK {
		return nil, fuseStatus
	}

	dirss, err := fs.fetchSchemaSuperset(dirBlob)
	switch {
	case err == os.ENOENT:
		log.Printf("Failed to find directory %s", dirBlob)
		return nil, fuse.ENOENT
	case err == os.EINVAL:
		log.Printf("Failed to parse directory %s", dirBlob)
		return nil, fuse.ENOTDIR
	case err != nil:
		panic(fmt.Sprintf("Invalid fetcher error: %v", err))
	case dirss == nil:
		panic("nil dirss")
	case dirss.Type != "directory":
		log.Printf("Expected %s to be a directory; actually a %s",
			dirBlob, dirss.Type)
		return nil, fuse.ENOTDIR
	}

	if dirss.Entries == "" {
		log.Printf("Expected %s to have 'entries'", dirBlob)
		return nil, fuse.ENOTDIR
	}
	entriesBlob := blobref.Parse(dirss.Entries)
	if entriesBlob == nil {
		log.Printf("Blob %s had invalid blobref %q for its 'entries'", dirBlob, dirss.Entries)
		return nil, fuse.ENOTDIR
	}

	entss, err := fs.fetchSchemaSuperset(entriesBlob)
	switch {
	case err == os.ENOENT:
		log.Printf("Failed to find entries %s via directory %s", entriesBlob, dirBlob)
		return nil, fuse.ENOENT
	case err == os.EINVAL:
		log.Printf("Failed to parse entries %s via directory %s", entriesBlob, dirBlob)
		return nil, fuse.ENOTDIR
	case err != nil:
		panic(fmt.Sprintf("Invalid fetcher error: %v", err))
	case entss == nil:
		panic("nil entss")
	case entss.Type != "static-set":
		log.Printf("Expected %s to be a directory; actually a %s",
			dirBlob, dirss.Type)
		return nil, fuse.ENOTDIR
	}

	wg := new(sync.WaitGroup)
	foundCh := make(chan *blobref.BlobRef)
	for _, m := range entss.Members {
		wg.Add(1)
		go func(memberBlobstr string) {
			defer wg.Done()
			memberBlob := blobref.Parse(memberBlobstr)
			if memberBlob == nil {
				log.Printf("invalid blobref of %q in static set %s", memberBlobstr, entss)
				return
			}
			childss, err := fs.fetchSchemaSuperset(memberBlob)
			if err == nil && childss.HasFilename(fileName) {
				foundCh <- memberBlob
			}
		}(m)
	}
	failCh := make(chan string)
	go func() {
		wg.Wait()
		failCh <- "ENOENT"
	}()
	select {
	case found := <-foundCh:
		fs.lk.Lock()
		defer fs.lk.Unlock()
		fs.nameToBlob[name] = found
		return found, fuse.OK
	case <-failCh:
	}
	// TODO: negative cache
	return nil, fuse.ENOENT
}

func (fs *CamliFileSystem) Mount(connector *fuse.PathFileSystemConnector) fuse.Status {
	log.Printf("cammount: Mount")
	return fuse.OK
}

func (fs *CamliFileSystem) Unmount() {
	log.Printf("cammount: Unmount.")
}

func (fs *CamliFileSystem) GetAttr(name string) (*fuse.Attr, fuse.Status) {
	log.Printf("cammount: GetAttr(%q)", name)
	blobref, errStatus := fs.blobRefFromName(name)
	log.Printf("cammount: GetAttr(%q), blobRefFromName err=%v", name, errStatus)
	if errStatus != fuse.OK {
		return nil, errStatus
	}
	log.Printf("cammount: got blob %s", blobref)

	// TODO: this is redundant with what blobRefFromName already
	// did.  we should at least keep this in RAM (pre-de-JSON'd)
	// so we don't have to fetch + unmarshal it again.
	ss, err := fs.fetchSchemaSuperset(blobref)
	if err != nil {
		log.Printf("cammount: GetAttr(%q, %s): fetch schema error: %v", name, blobref, err)
		return nil, fuse.EIO
	}

	out := new(fuse.Attr)
	var fi os.FileInfo

	fi.Mode = ss.UnixMode()

	// TODO: have a mode to set permissions equal to mounting user?
	fi.Uid = ss.UnixOwnerId
	fi.Gid = ss.UnixGroupId

	// TODO: other types
	if ss.Type == "file" {
		fi.Size = ss.Size
	}

	// TODO: mtime and such

	fuse.CopyFileInfo(&fi, out)
	return out, fuse.OK
}

func (fs *CamliFileSystem) Access(name string, mode uint32) fuse.Status {
	log.Printf("cammount: Access(%q, %d)", name, mode)
	return fuse.OK
}

func (fs *CamliFileSystem) Open(name string, flags uint32) (file fuse.RawFuseFile, code fuse.Status) {
	log.Printf("cammount: Open(%q, %d)", name, flags)
	// TODO
	return nil, fuse.EACCES
}

func (fs *CamliFileSystem) OpenDir(name string) (stream chan fuse.DirEntry, code fuse.Status) {
	dirBlob, errStatus := fs.blobRefFromName(name)
	log.Printf("cammount: OpenDir(%q), dirBlob=%s err=%v", name, dirBlob, errStatus)
	if errStatus != fuse.OK {
		return nil, errStatus
	}

	// TODO: this is redundant with what blobRefFromName already
	// did.  we should at least keep this in RAM (pre-de-JSON'd)
	// so we don't have to fetch + unmarshal it again.
	dirss, err := fs.fetchSchemaSuperset(dirBlob)
	log.Printf("dirss blob: %v, err=%v", dirss, err)
	if err != nil {
		log.Printf("cammount: OpenDir(%q, %s): fetch schema error: %v", name, dirBlob, err)
		return nil, fuse.EIO
	}

	if dirss.Entries == "" {
		log.Printf("Expected %s to have 'entries'", dirBlob)
		return nil, fuse.ENOTDIR
	}
	entriesBlob := blobref.Parse(dirss.Entries)
	if entriesBlob == nil {
		log.Printf("Blob %s had invalid blobref %q for its 'entries'", dirBlob, dirss.Entries)
		return nil, fuse.ENOTDIR
	}

	entss, err := fs.fetchSchemaSuperset(entriesBlob)
	log.Printf("entries blob: %v, err=%v", entss, err)
	switch {
	case err == os.ENOENT:
		log.Printf("Failed to find entries %s via directory %s", entriesBlob, dirBlob)
		return nil, fuse.ENOENT
	case err == os.EINVAL:
		log.Printf("Failed to parse entries %s via directory %s", entriesBlob, dirBlob)
		return nil, fuse.ENOTDIR
	case err != nil:
		panic(fmt.Sprintf("Invalid fetcher error: %v", err))
	case entss == nil:
		panic("nil entss")
	case entss.Type != "static-set":
		log.Printf("Expected %s to be a directory; actually a %s",
			dirBlob, entss.Type)
		return nil, fuse.ENOTDIR
	}

	retch := make(chan fuse.DirEntry, 20)
	wg := new(sync.WaitGroup)
	for _, m := range entss.Members {
		wg.Add(1)
		go func(memberBlobstr string) {
			defer wg.Done()
			memberBlob := blobref.Parse(memberBlobstr)
			if memberBlob == nil {
				log.Printf("invalid blobref of %q in static set %s", memberBlobstr, entss)
				return
			}
			childss, err := fs.fetchSchemaSuperset(memberBlob)
			if err == nil {
				if childss.FileName != "" {
					mode := childss.UnixMode()
					//log.Printf("adding to dir %s: file=%q, mode=%d", dirBlob, childss.FileName, mode)
					retch <- fuse.DirEntry{Name: childss.FileName, Mode: mode}
				} else {
					log.Printf("Blob %s had no filename", childss.FileName)
				}
			} else {
				log.Printf("Error fetching %s: %v", memberBlobstr, err)
			}
		}(m)
	}
	go func() {
		wg.Wait()
		close(retch)
	}()
	return retch, fuse.OK
}

func (fs *CamliFileSystem) Readlink(name string) (string, fuse.Status) {
	log.Printf("cammount: Readlink(%q)", name)
	// TODO
	return "", fuse.EACCES
}
<|endoftext|>


<commit_before>package clipboard_test

import (
	"fmt"

	"github.com/go-vgo/robotgo/clipboard"
	// "github.com/atotto/clipboard"
)

func Example() {
	clipboard.WriteAll("日本語")
	text, _ := clipboard.ReadAll()
	fmt.Println(text)

	// Output:
	// 日本語
}
<commit_msg>Update clipboard test<commit_after>package clipboard_test

import (
	"fmt"

	"github.com/go-vgo/robotgo/clipboard"
	// "github.com/atotto/clipboard"
)

func Example() {
	clipboard.WriteAll("日本語")
	text, err := clipboard.ReadAll()
	if err == nil {
		fmt.Println(text)
	}

	// Output:
	// 日本語
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package exec

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/coreos/pkg/capnslog"
)

// Executor is the main interface for all the exec commands
type Executor interface {
	ExecuteCommand(command string, arg ...string) error
	ExecuteCommandWithEnv(env []string, command string, arg ...string) error
	ExecuteCommandWithOutput(command string, arg ...string) (string, error)
	ExecuteCommandWithCombinedOutput(command string, arg ...string) (string, error)
	ExecuteCommandWithOutputFile(command, outfileArg string, arg ...string) (string, error)
	ExecuteCommandWithOutputFileTimeout(timeout time.Duration, command, outfileArg string, arg ...string) (string, error)
	ExecuteCommandWithTimeout(timeout time.Duration, command string, arg ...string) (string, error)
}

// CommandExecutor is the type of the Executor
type CommandExecutor struct {
}

// ExecuteCommand starts a process and wait for its completion
func (c *CommandExecutor) ExecuteCommand(command string, arg ...string) error {
	return c.ExecuteCommandWithEnv([]string{}, command, arg...)
}

// ExecuteCommandWithEnv starts a process with env variables and wait for its completion
func (*CommandExecutor) ExecuteCommandWithEnv(env []string, command string, arg ...string) error {
	cmd, stdout, stderr, err := startCommand(env, command, arg...)
	if err != nil {
		return err
	}

	logOutput(stdout, stderr)

	if err := cmd.Wait(); err != nil {
		return err
	}

	return nil
}

// ExecuteCommandWithTimeout starts a process and wait for its completion with timeout.
func (*CommandExecutor) ExecuteCommandWithTimeout(timeout time.Duration, command string, arg ...string) (string, error) {
	logCommand(command, arg...)
	// #nosec G204 Rook controls the input to the exec arguments
	cmd := exec.Command(command, arg...)

	var b bytes.Buffer
	cmd.Stdout = &b
	cmd.Stderr = &b

	if err := cmd.Start(); err != nil {
		return "", err
	}

	done := make(chan error, 1)
	go func() {
		done <- cmd.Wait()
	}()

	interruptSent := false
	for {
		select {
		case <-time.After(timeout):
			if interruptSent {
				logger.Infof("timeout waiting for process %s to return after interrupt signal was sent. Sending kill signal to the process", command)
				var e error
				if err := cmd.Process.Kill(); err != nil {
					logger.Errorf("Failed to kill process %s: %v", command, err)
					e = fmt.Errorf("timeout waiting for the command %s to return after interrupt signal was sent. Tried to kill the process but that failed: %v", command, err)
				} else {
					e = fmt.Errorf("timeout waiting for the command %s to return", command)
				}
				return strings.TrimSpace(b.String()), e
			}

			logger.Infof("timeout waiting for process %s to return. Sending interrupt signal to the process", command)
			if err := cmd.Process.Signal(os.Interrupt); err != nil {
				logger.Errorf("Failed to send interrupt signal to process %s: %v", command, err)
				// kill signal will be sent next loop
			}
			interruptSent = true
		case err := <-done:
			if err != nil {
				return strings.TrimSpace(b.String()), err
			}
			if interruptSent {
				return strings.TrimSpace(b.String()), fmt.Errorf("timeout waiting for the command %s to return", command)
			}
			return strings.TrimSpace(b.String()), nil
		}
	}
}

// ExecuteCommandWithOutput executes a command with output
func (*CommandExecutor) ExecuteCommandWithOutput(command string, arg ...string) (string, error) {
	logCommand(command, arg...)
	// #nosec G204 Rook controls the input to the exec arguments
	cmd := exec.Command(command, arg...)
	return runCommandWithOutput(cmd, false)
}

// ExecuteCommandWithCombinedOutput executes a command with combined output
func (*CommandExecutor) ExecuteCommandWithCombinedOutput(command string, arg ...string) (string, error) {
	logCommand(command, arg...)
	// #nosec G204 Rook controls the input to the exec arguments
	cmd := exec.Command(command, arg...)
	return runCommandWithOutput(cmd, true)
}

// ExecuteCommandWithOutputFileTimeout Same as ExecuteCommandWithOutputFile but with a timeout limit.
// #nosec G307 Calling defer to close the file without checking the error return is not a risk for a simple file open and close
func (*CommandExecutor) ExecuteCommandWithOutputFileTimeout(timeout time.Duration,
	command, outfileArg string, arg ...string) (string, error) {

	outFile, err := ioutil.TempFile("", "")
	if err != nil {
		return "", fmt.Errorf("failed to open output file: %+v", err)
	}
	defer outFile.Close()
	defer os.Remove(outFile.Name())

	arg = append(arg, outfileArg, outFile.Name())
	logCommand(command, arg...)

	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()

	// #nosec G204 Rook controls the input to the exec arguments
	cmd := exec.CommandContext(ctx, command, arg...)
	cmdOut, err := cmd.CombinedOutput()

	// if there was anything that went to stdout/stderr then log it, even before
	// we return an error
	if string(cmdOut) != "" {
		logger.Debug(string(cmdOut))
	}

	if ctx.Err() == context.DeadlineExceeded {
		return string(cmdOut), ctx.Err()
	}

	if err != nil {
		return string(cmdOut), err
	}

	fileOut, err := ioutil.ReadAll(outFile)
	if err := outFile.Close(); err != nil {
		return "", err
	}
	return string(fileOut), err
}

// ExecuteCommandWithOutputFile executes a command with output on a file
// #nosec G307 Calling defer to close the file without checking the error return is not a risk for a simple file open and close
func (*CommandExecutor) ExecuteCommandWithOutputFile(command, outfileArg string, arg ...string) (string, error) {

	// create a temporary file to serve as the output file for the command to be run and ensure
	// it is cleaned up after this function is done
	outFile, err := ioutil.TempFile("", "")
	if err != nil {
		return "", fmt.Errorf("failed to open output file: %+v", err)
	}
	defer outFile.Close()
	defer os.Remove(outFile.Name())

	// append the output file argument to the list or args
	arg = append(arg, outfileArg, outFile.Name())

	logCommand(command, arg...)
	// #nosec G204 Rook controls the input to the exec arguments
	cmd := exec.Command(command, arg...)
	cmdOut, err := cmd.CombinedOutput()
	if err != nil {
		cmdOut = []byte(fmt.Sprintf("%s. %s", string(cmdOut), assertErrorType(err)))
	}
	// if there was anything that went to stdout/stderr then log it, even before we return an error
	if string(cmdOut) != "" {
		logger.Debug(string(cmdOut))
	}
	if err != nil {
		return string(cmdOut), err
	}

	// read the entire output file and return that to the caller
	fileOut, err := ioutil.ReadAll(outFile)
	if err := outFile.Close(); err != nil {
		return "", err
	}
	return string(fileOut), err
}

func startCommand(env []string, command string, arg ...string) (*exec.Cmd, io.ReadCloser, io.ReadCloser, error) {
	logCommand(command, arg...)

	// #nosec G204 Rook controls the input to the exec arguments
	cmd := exec.Command(command, arg...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		logger.Warningf("failed to open stdout pipe: %+v", err)
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		logger.Warningf("failed to open stderr pipe: %+v", err)
	}

	if len(env) > 0 {
		cmd.Env = env
	}

	err = cmd.Start()

	return cmd, stdout, stderr, err
}

// read from reader line by line and write it to the log
func logFromReader(logger *capnslog.PackageLogger, reader io.ReadCloser) {
	in := bufio.NewScanner(reader)
	lastLine := ""
	for in.Scan() {
		lastLine = in.Text()
		logger.Debug(lastLine)
	}
}

func logOutput(stdout, stderr io.ReadCloser) {
	if stdout == nil || stderr == nil {
		logger.Warningf("failed to collect stdout and stderr")
		return
	}

	// The child processes should appropriately be outputting at the desired global level.  Therefore,
	// we always log at INFO level here, so that log statements from child procs at higher levels
	// (e.g., WARNING) will still be displayed.  We are relying on the child procs to output appropriately.
	childLogger := capnslog.NewPackageLogger("github.com/rook/rook", "exec")
	if !childLogger.LevelAt(capnslog.INFO) {
		rl, err := capnslog.GetRepoLogger("github.com/rook/rook")
		if err == nil {
			rl.SetLogLevel(map[string]capnslog.LogLevel{"exec": capnslog.INFO})
		}
	}

	go logFromReader(childLogger, stderr)
	logFromReader(childLogger, stdout)
}

func runCommandWithOutput(cmd *exec.Cmd, combinedOutput bool) (string, error) {
	var output []byte
	var err error
	var out string

	if combinedOutput {
		output, err = cmd.CombinedOutput()
	} else {
		output, err = cmd.Output()
		if err != nil {
			output = []byte(fmt.Sprintf("%s. %s", string(output), assertErrorType(err)))
		}
	}

	out = strings.TrimSpace(string(output))

	if err != nil {
		return out, err
	}

	return out, nil
}

func logCommand(command string, arg ...string) {
	logger.Debugf("Running command: %s %s", command, strings.Join(arg, " "))
}

func assertErrorType(err error) string {
	switch errType := err.(type) {
	case *exec.ExitError:
		return string(errType.Stderr)
	case *exec.Error:
		return errType.Error()
	}

	return ""
}
<commit_msg>ceph: use errors pkg instead of fmt<commit_after>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package exec

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/coreos/pkg/capnslog"
	"github.com/pkg/errors"
)

// Executor is the main interface for all the exec commands
type Executor interface {
	ExecuteCommand(command string, arg ...string) error
	ExecuteCommandWithEnv(env []string, command string, arg ...string) error
	ExecuteCommandWithOutput(command string, arg ...string) (string, error)
	ExecuteCommandWithCombinedOutput(command string, arg ...string) (string, error)
	ExecuteCommandWithOutputFile(command, outfileArg string, arg ...string) (string, error)
	ExecuteCommandWithOutputFileTimeout(timeout time.Duration, command, outfileArg string, arg ...string) (string, error)
	ExecuteCommandWithTimeout(timeout time.Duration, command string, arg ...string) (string, error)
}

// CommandExecutor is the type of the Executor
type CommandExecutor struct {
}

// ExecuteCommand starts a process and wait for its completion
func (c *CommandExecutor) ExecuteCommand(command string, arg ...string) error {
	return c.ExecuteCommandWithEnv([]string{}, command, arg...)
}

// ExecuteCommandWithEnv starts a process with env variables and wait for its completion
func (*CommandExecutor) ExecuteCommandWithEnv(env []string, command string, arg ...string) error {
	cmd, stdout, stderr, err := startCommand(env, command, arg...)
	if err != nil {
		return err
	}

	logOutput(stdout, stderr)

	if err := cmd.Wait(); err != nil {
		return err
	}

	return nil
}

// ExecuteCommandWithTimeout starts a process and wait for its completion with timeout.
func (*CommandExecutor) ExecuteCommandWithTimeout(timeout time.Duration, command string, arg ...string) (string, error) {
	logCommand(command, arg...)
	// #nosec G204 Rook controls the input to the exec arguments
	cmd := exec.Command(command, arg...)

	var b bytes.Buffer
	cmd.Stdout = &b
	cmd.Stderr = &b

	if err := cmd.Start(); err != nil {
		return "", err
	}

	done := make(chan error, 1)
	go func() {
		done <- cmd.Wait()
	}()

	interruptSent := false
	for {
		select {
		case <-time.After(timeout):
			if interruptSent {
				logger.Infof("timeout waiting for process %s to return after interrupt signal was sent. Sending kill signal to the process", command)
				var e error
				if err := cmd.Process.Kill(); err != nil {
					logger.Errorf("Failed to kill process %s: %v", command, err)
					e = fmt.Errorf("timeout waiting for the command %s to return after interrupt signal was sent. Tried to kill the process but that failed: %v", command, err)
				} else {
					e = fmt.Errorf("timeout waiting for the command %s to return", command)
				}
				return strings.TrimSpace(b.String()), e
			}

			logger.Infof("timeout waiting for process %s to return. Sending interrupt signal to the process", command)
			if err := cmd.Process.Signal(os.Interrupt); err != nil {
				logger.Errorf("Failed to send interrupt signal to process %s: %v", command, err)
				// kill signal will be sent next loop
			}
			interruptSent = true
		case err := <-done:
			if err != nil {
				return strings.TrimSpace(b.String()), err
			}
			if interruptSent {
				return strings.TrimSpace(b.String()), fmt.Errorf("timeout waiting for the command %s to return", command)
			}
			return strings.TrimSpace(b.String()), nil
		}
	}
}

// ExecuteCommandWithOutput executes a command with output
func (*CommandExecutor) ExecuteCommandWithOutput(command string, arg ...string) (string, error) {
	logCommand(command, arg...)
	// #nosec G204 Rook controls the input to the exec arguments
	cmd := exec.Command(command, arg...)
	return runCommandWithOutput(cmd, false)
}

// ExecuteCommandWithCombinedOutput executes a command with combined output
func (*CommandExecutor) ExecuteCommandWithCombinedOutput(command string, arg ...string) (string, error) {
	logCommand(command, arg...)
	// #nosec G204 Rook controls the input to the exec arguments
	cmd := exec.Command(command, arg...)
	return runCommandWithOutput(cmd, true)
}

// ExecuteCommandWithOutputFileTimeout Same as ExecuteCommandWithOutputFile but with a timeout limit.
// #nosec G307 Calling defer to close the file without checking the error return is not a risk for a simple file open and close
func (*CommandExecutor) ExecuteCommandWithOutputFileTimeout(timeout time.Duration,
	command, outfileArg string, arg ...string) (string, error) {

	outFile, err := ioutil.TempFile("", "")
	if err != nil {
		return "", errors.Wrap(err, "failed to open output file")
	}
	defer outFile.Close()
	defer os.Remove(outFile.Name())

	arg = append(arg, outfileArg, outFile.Name())
	logCommand(command, arg...)

	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()

	// #nosec G204 Rook controls the input to the exec arguments
	cmd := exec.CommandContext(ctx, command, arg...)
	cmdOut, err := cmd.CombinedOutput()

	// if there was anything that went to stdout/stderr then log it, even before
	// we return an error
	if string(cmdOut) != "" {
		logger.Debug(string(cmdOut))
	}

	if ctx.Err() == context.DeadlineExceeded {
		return string(cmdOut), ctx.Err()
	}

	if err != nil {
		return string(cmdOut), err
	}

	fileOut, err := ioutil.ReadAll(outFile)
	if err := outFile.Close(); err != nil {
		return "", err
	}
	return string(fileOut), err
}

// ExecuteCommandWithOutputFile executes a command with output on a file
// #nosec G307 Calling defer to close the file without checking the error return is not a risk for a simple file open and close
func (*CommandExecutor) ExecuteCommandWithOutputFile(command, outfileArg string, arg ...string) (string, error) {

	// create a temporary file to serve as the output file for the command to be run and ensure
	// it is cleaned up after this function is done
	outFile, err := ioutil.TempFile("", "")
	if err != nil {
		return "", errors.Wrap(err, "failed to open output file")
	}
	defer outFile.Close()
	defer os.Remove(outFile.Name())

	// append the output file argument to the list or args
	arg = append(arg, outfileArg, outFile.Name())

	logCommand(command, arg...)
	// #nosec G204 Rook controls the input to the exec arguments
	cmd := exec.Command(command, arg...)
	cmdOut, err := cmd.CombinedOutput()
	if err != nil {
		cmdOut = []byte(fmt.Sprintf("%s. %s", string(cmdOut), assertErrorType(err)))
	}
	// if there was anything that went to stdout/stderr then log it, even before we return an error
	if string(cmdOut) != "" {
		logger.Debug(string(cmdOut))
	}
	if err != nil {
		return string(cmdOut), err
	}

	// read the entire output file and return that to the caller
	fileOut, err := ioutil.ReadAll(outFile)
	if err := outFile.Close(); err != nil {
		return "", err
	}
	return string(fileOut), err
}

func startCommand(env []string, command string, arg ...string) (*exec.Cmd, io.ReadCloser, io.ReadCloser, error) {
	logCommand(command, arg...)

	// #nosec G204 Rook controls the input to the exec arguments
	cmd := exec.Command(command, arg...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		logger.Warningf("failed to open stdout pipe: %+v", err)
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		logger.Warningf("failed to open stderr pipe: %+v", err)
	}

	if len(env) > 0 {
		cmd.Env = env
	}

	err = cmd.Start()

	return cmd, stdout, stderr, err
}

// read from reader line by line and write it to the log
func logFromReader(logger *capnslog.PackageLogger, reader io.ReadCloser) {
	in := bufio.NewScanner(reader)
	lastLine := ""
	for in.Scan() {
		lastLine = in.Text()
		logger.Debug(lastLine)
	}
}

func logOutput(stdout, stderr io.ReadCloser) {
	if stdout == nil || stderr == nil {
		logger.Warningf("failed to collect stdout and stderr")
		return
	}

	// The child processes should appropriately be outputting at the desired global level.  Therefore,
	// we always log at INFO level here, so that log statements from child procs at higher levels
	// (e.g., WARNING) will still be displayed.  We are relying on the child procs to output appropriately.
	childLogger := capnslog.NewPackageLogger("github.com/rook/rook", "exec")
	if !childLogger.LevelAt(capnslog.INFO) {
		rl, err := capnslog.GetRepoLogger("github.com/rook/rook")
		if err == nil {
			rl.SetLogLevel(map[string]capnslog.LogLevel{"exec": capnslog.INFO})
		}
	}

	go logFromReader(childLogger, stderr)
	logFromReader(childLogger, stdout)
}

func runCommandWithOutput(cmd *exec.Cmd, combinedOutput bool) (string, error) {
	var output []byte
	var err error
	var out string

	if combinedOutput {
		output, err = cmd.CombinedOutput()
	} else {
		output, err = cmd.Output()
		if err != nil {
			output = []byte(fmt.Sprintf("%s. %s", string(output), assertErrorType(err)))
		}
	}

	out = strings.TrimSpace(string(output))

	if err != nil {
		return out, err
	}

	return out, nil
}

func logCommand(command string, arg ...string) {
	logger.Debugf("Running command: %s %s", command, strings.Join(arg, " "))
}

func assertErrorType(err error) string {
	switch errType := err.(type) {
	case *exec.ExitError:
		return string(errType.Stderr)
	case *exec.Error:
		return errType.Error()
	}

	return ""
}
<|endoftext|>


<commit_before>// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package balancer

import (
	"context"
	"fmt"
	"testing"

	"github.com/coreos/etcd/clientv3/balancer/picker"
	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/pkg/mock/mockserver"

	"go.uber.org/zap"
	"google.golang.org/grpc"
	"google.golang.org/grpc/peer"
	"google.golang.org/grpc/resolver"
	"google.golang.org/grpc/resolver/manual"
)

// TestRoundRobinBalancedResolvableNoFailover ensures that
// requests to a resolvable endpoint can be balanced between
// multiple, if any, nodes. And there needs be no failover.
func TestRoundRobinBalancedResolvableNoFailover(t *testing.T) {
	testCases := []struct {
		name        string
		serverCount int
		reqN        int
	}{
		{name: "rrBalanced_1", serverCount: 1, reqN: 5},
		{name: "rrBalanced_3", serverCount: 3, reqN: 7},
		{name: "rrBalanced_5", serverCount: 5, reqN: 10},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			ms, err := mockserver.StartMockServers(tc.serverCount)
			if err != nil {
				t.Fatalf("failed to start mock servers: %v", err)
			}
			defer ms.Stop()
			var resolvedAddrs []resolver.Address
			for _, svr := range ms {
				resolvedAddrs = append(resolvedAddrs, resolver.Address{Addr: svr.Address})
			}

			rsv, closeResolver := manual.GenerateAndRegisterManualResolver()
			defer closeResolver()
			cfg := Config{
				Policy:    picker.RoundrobinBalanced,
				Name:      genName(),
				Logger:    zap.NewExample(),
				Endpoints: []string{fmt.Sprintf("%s:///mock.server", rsv.Scheme())},
			}
			rrb := New(cfg)
			conn, err := grpc.Dial(cfg.Endpoints[0], grpc.WithInsecure(), grpc.WithBalancerName(rrb.Name()))
			if err != nil {
				t.Fatalf("failed to dial mock server: %s", err)
			}
			defer conn.Close()
			rsv.NewAddress(resolvedAddrs)
			cli := pb.NewKVClient(conn)

			reqFunc := func(ctx context.Context) (picked string, err error) {
				var p peer.Peer
				_, err = cli.Range(ctx, &pb.RangeRequest{Key: []byte("/x")}, grpc.Peer(&p))
				if p.Addr != nil {
					picked = p.Addr.String()
				}
				return picked, err
			}

			prev, switches := "", 0
			for i := 0; i < tc.reqN; i++ {
				picked, err := reqFunc(context.Background())
				if err != nil {
					t.Fatalf("#%d: unexpected failure %v", i, err)
				}
				if prev == "" {
					prev = picked
					continue
				}
				if prev != picked {
					switches++
				}
				prev = picked
			}
			if tc.serverCount > 1 && switches < tc.reqN-3 { // -3 for initial resolutions
				t.Fatalf("expected balanced loads for %d requests, got switches %d", tc.reqN, switches)
			}
		})
	}
}
<commit_msg>clientv3/balancer: use new mock server in tests<commit_after>// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package balancer

import (
	"context"
	"fmt"
	"testing"

	"github.com/coreos/etcd/clientv3/balancer/picker"
	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/pkg/mock/mockserver"

	"go.uber.org/zap"
	"google.golang.org/grpc"
	"google.golang.org/grpc/peer"
	"google.golang.org/grpc/resolver"
	"google.golang.org/grpc/resolver/manual"
)

// TestRoundRobinBalancedResolvableNoFailover ensures that
// requests to a resolvable endpoint can be balanced between
// multiple, if any, nodes. And there needs be no failover.
func TestRoundRobinBalancedResolvableNoFailover(t *testing.T) {
	testCases := []struct {
		name        string
		serverCount int
		reqN        int
	}{
		{name: "rrBalanced_1", serverCount: 1, reqN: 5},
		{name: "rrBalanced_3", serverCount: 3, reqN: 7},
		{name: "rrBalanced_5", serverCount: 5, reqN: 10},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			ms, err := mockserver.StartMockServers(tc.serverCount)
			if err != nil {
				t.Fatalf("failed to start mock servers: %v", err)
			}
			defer ms.Stop()
			var resolvedAddrs []resolver.Address
			for _, svr := range ms.Servers {
				resolvedAddrs = append(resolvedAddrs, resolver.Address{Addr: svr.Address})
			}

			rsv, closeResolver := manual.GenerateAndRegisterManualResolver()
			defer closeResolver()
			cfg := Config{
				Policy:    picker.RoundrobinBalanced,
				Name:      genName(),
				Logger:    zap.NewExample(),
				Endpoints: []string{fmt.Sprintf("%s:///mock.server", rsv.Scheme())},
			}
			rrb := New(cfg)
			conn, err := grpc.Dial(cfg.Endpoints[0], grpc.WithInsecure(), grpc.WithBalancerName(rrb.Name()))
			if err != nil {
				t.Fatalf("failed to dial mock server: %s", err)
			}
			defer conn.Close()
			rsv.NewAddress(resolvedAddrs)
			cli := pb.NewKVClient(conn)

			reqFunc := func(ctx context.Context) (picked string, err error) {
				var p peer.Peer
				_, err = cli.Range(ctx, &pb.RangeRequest{Key: []byte("/x")}, grpc.Peer(&p))
				if p.Addr != nil {
					picked = p.Addr.String()
				}
				return picked, err
			}

			prev, switches := "", 0
			for i := 0; i < tc.reqN; i++ {
				picked, err := reqFunc(context.Background())
				if err != nil {
					t.Fatalf("#%d: unexpected failure %v", i, err)
				}
				if prev == "" {
					prev = picked
					continue
				}
				if prev != picked {
					switches++
				}
				prev = picked
			}
			if tc.serverCount > 1 && switches < tc.reqN-3 { // -3 for initial resolutions
				t.Fatalf("expected balanced loads for %d requests, got switches %d", tc.reqN, switches)
			}
		})
	}
}
<|endoftext|>


<commit_before>// Package generate provides the ability to generate things that need to be random,
// including salt, token and password values.
package generate

import (
	"crypto/rand"

	"github.com/davidcarboni/cryptolite/bytearray"
)

// TokenBits is the length for tokens.
var TokenBits = 256

// SaltBytes is the length for salt values.
var SaltBytes = 16

// Work out the right number of bytes for random tokens:
var tokenLengthBytes = TokenBits / 8

// Characters for pasword generation:
var passwordCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"

// ByteArray is a convenience method to instantiate and populate a byte array of the specified length.
//
//The length parameter sets the length of the returned slice.
func ByteArray(length int) []byte {
	byteArray := make([]byte, length)
	bytes := 0
	for bytes < 8 {
		read, err := rand.Read(byteArray)
		if err != nil {
			panic(err)
		}
		bytes += read
	}
	return byteArray
}

// Token generates a random token.
// Returns A 256-bit (32 byte) random token as a hexadecimal string.
func Token() string {
	tokenBytes := ByteArray(tokenLengthBytes)
	token := bytearray.ToHex(tokenBytes)
	return token
}

// Password generates a random password.
//
// The length parameter specifies the length of the password to be returned.
// Returns A password of the specified length, selected from passwordCharacters.
func Password(length int) string {

	result := ""
	values := byte_array(length)
	// We use a modulus of an increasing index rather than of the byte values
	// to avoid certain characters coming up more often.
	index := 0

	for i = 0; i < length; i++ {
		index += values[i]
		// We're not using any double-byte characters, so byte length is fine:
		index = index % len(passwordCharacters)
		result += passwordCharacters[index]
	}

	return result
}

// Salt generates a random salt value.
// If a salt value is needed by an API call,
// the documentation of that method should reference this method. Other than than,
// it should not be necessary to call this in normal usage of this library.
//
// Returns a random salt value of SaltBytes length, as a base64-encoded
// string (for easy storage).
func Salt() string {
	salt := ByteArray(SaltBytes)
	return bytearray.ToBase64(salt)
}
<commit_msg>Fixed up for build.<commit_after>// Package generate provides the ability to generate things that need to be random,
// including salt, token and password values.
package generate

import (
	"crypto/rand"

	"github.com/davidcarboni/cryptolite/bytearray"
)

// TokenBits is the length for tokens.
var TokenBits = 256

// SaltBytes is the length for salt values.
var SaltBytes = 16

// Work out the right number of bytes for random tokens:
var tokenLengthBytes = TokenBits / 8

// Characters for pasword generation:
var passwordCharacters = []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789")

// ByteArray instantiates and populates a byte array of the specified length.
//
// The length parameter sets the length of the returned slice.
func ByteArray(length int) []byte {
	byteArray := make([]byte, length)
	bytes := 0
	for bytes < 8 {
		read, err := rand.Read(byteArray)
		if err != nil {
			panic(err)
		}
		bytes += read
	}
	return byteArray
}

// Token generates a random token.
// Returns A 256-bit (32 byte) random token as a hexadecimal string.
func Token() string {
	tokenBytes := ByteArray(tokenLengthBytes)
	token := bytearray.ToHex(tokenBytes)
	return token
}

// Password generates a random password.
//
// The length parameter specifies the length of the password to be returned.
// Returns A password of the specified length, selected from passwordCharacters.
func Password(length int) string {

	result := ""
	values := ByteArray(length)
	// We use a modulus of an increasing index rather than of the byte values
	// to avoid certain characters coming up more often.
	index := 0

	for i := 0; i < length; i++ {
		index += int(values[i])
		// We're not using any complex characters, so glyph length is fine:
		index = index % len(passwordCharacters)
		result += string(passwordCharacters[index])
	}

	return result
}

// Salt generates a random salt value.
// If a salt value is needed by an API call,
// the documentation of that method should reference this method. Other than than,
// it should not be necessary to call this in normal usage of this library.
//
// Returns a random salt value of SaltBytes length, as a base64-encoded
// string (for easy storage).
func Salt() string {
	salt := ByteArray(SaltBytes)
	return bytearray.ToBase64(salt)
}
<|endoftext|>