<commit_before>// Package config contains the model and loader of the goreleaser configuration
// file.
package config

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"strings"

	"github.com/apex/log"
	yaml "gopkg.in/yaml.v2"
)

// Repo represents any kind of repo (github, gitlab, etc)
type Repo struct {
	Owner string `yaml:",omitempty"`
	Name  string `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// String of the repo, e.g. owner/name
func (r Repo) String() string {
	return r.Owner + "/" + r.Name
}

// Homebrew contains the brew section
type Homebrew struct {
	GitHub       Repo     `yaml:",omitempty"`
	Folder       string   `yaml:",omitempty"`
	Caveats      string   `yaml:",omitempty"`
	Plist        string   `yaml:",omitempty"`
	Install      string   `yaml:",omitempty"`
	Dependencies []string `yaml:",omitempty"`
	Conflicts    []string `yaml:",omitempty"`
	Description  string   `yaml:",omitempty"`
	Homepage     string   `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// Hooks define actions to run before and/or after something
type Hooks struct {
	Pre  string `yaml:",omitempty"`
	Post string `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// IgnoredBuild represents a build ignored by the user
type IgnoredBuild struct {
	Goos, Goarch, Goarm string

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// Build contains the build configuration section
type Build struct {
	Goos    []string       `yaml:",omitempty"`
	Goarch  []string       `yaml:",omitempty"`
	Goarm   []string       `yaml:",omitempty"`
	Ignore  []IgnoredBuild `yaml:",omitempty"`
	Main    string         `yaml:",omitempty"`
	Ldflags string         `yaml:",omitempty"`
	Flags   string         `yaml:",omitempty"`
	Binary  string         `yaml:",omitempty"`
	Hooks   Hooks          `yaml:",omitempty"`
	Env     []string       `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// FormatOverride is used to specify a custom format for a specific GOOS.
type FormatOverride struct {
	Goos   string `yaml:",omitempty"`
	Format string `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// Archive config used for the archive
type Archive struct {
	Format          string            `yaml:",omitempty"`
	FormatOverrides []FormatOverride  `yaml:"format_overrides,omitempty"`
	NameTemplate    string            `yaml:"name_template,omitempty"`
	Replacements    map[string]string `yaml:",omitempty"`
	Files           []string          `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// Release config used for the GitHub release
type Release struct {
	GitHub Repo `yaml:",omitempty"`
	Draft  bool `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// FPM config
type FPM struct {
	Formats      []string `yaml:",omitempty"`
	Dependencies []string `yaml:",omitempty"`
	Conflicts    []string `yaml:",omitempty"`
	Vendor       string   `yaml:",omitempty"`
	Homepage     string   `yaml:",omitempty"`
	Maintainer   string   `yaml:",omitempty"`
	Description  string   `yaml:",omitempty"`
	License      string   `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// Snapshot config
type Snapshot struct {
	NameTemplate string `yaml:"name_template,omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// Project includes all project configuration
type Project struct {
	ProjectName string   `yaml:"project_name,omitempty"`
	Release     Release  `yaml:",omitempty"`
	Brew        Homebrew `yaml:",omitempty"`
	Builds      []Build  `yaml:",omitempty"`
	Archive     Archive  `yaml:",omitempty"`
	FPM         FPM      `yaml:",omitempty"`
	Snapshot    Snapshot `yaml:",omitempty"`

	// this is a hack ¯\_(ツ)_/¯
	SingleBuild Build `yaml:"build,omitempty"`

	// test only property indicating the path to the dist folder
	Dist string `yaml:"-"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// Load config file
func Load(file string) (config Project, err error) {
	f, err := os.Open(file)
	if err != nil {
		return
	}
	log.WithField("file", file).Info("loading config file")
	return LoadReader(f)
}

// LoadReader config via io.Reader
func LoadReader(fd io.Reader) (config Project, err error) {
	data, err := ioutil.ReadAll(fd)
	if err != nil {
		return config, err
	}
	err = yaml.Unmarshal(data, &config)
	log.WithField("config", config).Debug("loaded config file")

	err = checkOverflows(config)
	return
}

func checkOverflows(config Project) error {
	var checker = &overflowChecker{}
	checker.check(config.XXX, "")
	checker.check(config.Archive.XXX, "archive")
	for i, ov := range config.Archive.FormatOverrides {
		checker.check(ov.XXX, fmt.Sprintf("archive.format_overrides[%d]", i))
	}
	checker.check(config.Brew.XXX, "brew")
	checker.check(config.Brew.GitHub.XXX, "brew.github")
	for i, build := range config.Builds {
		checker.check(build.XXX, fmt.Sprintf("builds[%d]", i))
		checker.check(build.Hooks.XXX, fmt.Sprintf("builds[%d].hooks", i))
		for j, ignored := range build.Ignore {
			checker.check(ignored.XXX, fmt.Sprintf("builds[%d].ignored_builds[%d]", i, j))
		}
	}
	checker.check(config.FPM.XXX, "fpm")
	checker.check(config.Release.XXX, "release")
	checker.check(config.Release.GitHub.XXX, "release.github")
	checker.check(config.SingleBuild.XXX, "build")
	checker.check(config.SingleBuild.Hooks.XXX, "builds.hooks")
	for i, ignored := range config.SingleBuild.Ignore {
		checker.check(ignored.XXX, fmt.Sprintf("builds.ignored_builds[%d]", i))
	}
	checker.check(config.Snapshot.XXX, "snapshot")
	return checker.err()
}

type overflowChecker struct {
	fields []string
}

func (o *overflowChecker) check(m map[string]interface{}, ctx string) {
	for k := range m {
		var key = fmt.Sprintf("%s.%s", ctx, k)
		if ctx == "" {
			key = k
		}
		o.fields = append(o.fields, key)
	}
}

func (o *overflowChecker) err() error {
	if len(o.fields) == 0 {
		return nil
	}
	return fmt.Errorf(
		"unknown fields in the config file: %s",
		strings.Join(o.fields, ", "),
	)
}
<commit_msg>format file<commit_after>// Package config contains the model and loader of the goreleaser configuration
// file.
package config

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	"github.com/apex/log"
	yaml "gopkg.in/yaml.v2"
)

// Repo represents any kind of repo (github, gitlab, etc)
type Repo struct {
	Owner string `yaml:",omitempty"`
	Name  string `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// String of the repo, e.g. owner/name
func (r Repo) String() string {
	return r.Owner + "/" + r.Name
}

// Homebrew contains the brew section
type Homebrew struct {
	GitHub       Repo     `yaml:",omitempty"`
	Folder       string   `yaml:",omitempty"`
	Caveats      string   `yaml:",omitempty"`
	Plist        string   `yaml:",omitempty"`
	Install      string   `yaml:",omitempty"`
	Dependencies []string `yaml:",omitempty"`
	Conflicts    []string `yaml:",omitempty"`
	Description  string   `yaml:",omitempty"`
	Homepage     string   `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// Hooks define actions to run before and/or after something
type Hooks struct {
	Pre  string `yaml:",omitempty"`
	Post string `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// IgnoredBuild represents a build ignored by the user
type IgnoredBuild struct {
	Goos, Goarch, Goarm string

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// Build contains the build configuration section
type Build struct {
	Goos    []string       `yaml:",omitempty"`
	Goarch  []string       `yaml:",omitempty"`
	Goarm   []string       `yaml:",omitempty"`
	Ignore  []IgnoredBuild `yaml:",omitempty"`
	Main    string         `yaml:",omitempty"`
	Ldflags string         `yaml:",omitempty"`
	Flags   string         `yaml:",omitempty"`
	Binary  string         `yaml:",omitempty"`
	Hooks   Hooks          `yaml:",omitempty"`
	Env     []string       `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// FormatOverride is used to specify a custom format for a specific GOOS.
type FormatOverride struct {
	Goos   string `yaml:",omitempty"`
	Format string `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// Archive config used for the archive
type Archive struct {
	Format          string            `yaml:",omitempty"`
	FormatOverrides []FormatOverride  `yaml:"format_overrides,omitempty"`
	NameTemplate    string            `yaml:"name_template,omitempty"`
	Replacements    map[string]string `yaml:",omitempty"`
	Files           []string          `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// Release config used for the GitHub release
type Release struct {
	GitHub Repo `yaml:",omitempty"`
	Draft  bool `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// FPM config
type FPM struct {
	Formats      []string `yaml:",omitempty"`
	Dependencies []string `yaml:",omitempty"`
	Conflicts    []string `yaml:",omitempty"`
	Vendor       string   `yaml:",omitempty"`
	Homepage     string   `yaml:",omitempty"`
	Maintainer   string   `yaml:",omitempty"`
	Description  string   `yaml:",omitempty"`
	License      string   `yaml:",omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// Snapshot config
type Snapshot struct {
	NameTemplate string `yaml:"name_template,omitempty"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// Project includes all project configuration
type Project struct {
	ProjectName string   `yaml:"project_name,omitempty"`
	Release     Release  `yaml:",omitempty"`
	Brew        Homebrew `yaml:",omitempty"`
	Builds      []Build  `yaml:",omitempty"`
	Archive     Archive  `yaml:",omitempty"`
	FPM         FPM      `yaml:",omitempty"`
	Snapshot    Snapshot `yaml:",omitempty"`

	// this is a hack ¯\_(ツ)_/¯
	SingleBuild Build `yaml:"build,omitempty"`

	// test only property indicating the path to the dist folder
	Dist string `yaml:"-"`

	// Capture all undefined fields and should be empty after loading
	XXX map[string]interface{} `yaml:",inline"`
}

// Load config file
func Load(file string) (config Project, err error) {
	f, err := os.Open(file)
	if err != nil {
		return
	}
	log.WithField("file", file).Info("loading config file")
	return LoadReader(f)
}

// LoadReader config via io.Reader
func LoadReader(fd io.Reader) (config Project, err error) {
	data, err := ioutil.ReadAll(fd)
	if err != nil {
		return config, err
	}
	err = yaml.Unmarshal(data, &config)
	log.WithField("config", config).Debug("loaded config file")

	err = checkOverflows(config)
	return
}

func checkOverflows(config Project) error {
	var checker = &overflowChecker{}
	checker.check(config.XXX, "")
	checker.check(config.Archive.XXX, "archive")
	for i, ov := range config.Archive.FormatOverrides {
		checker.check(ov.XXX, fmt.Sprintf("archive.format_overrides[%d]", i))
	}
	checker.check(config.Brew.XXX, "brew")
	checker.check(config.Brew.GitHub.XXX, "brew.github")
	for i, build := range config.Builds {
		checker.check(build.XXX, fmt.Sprintf("builds[%d]", i))
		checker.check(build.Hooks.XXX, fmt.Sprintf("builds[%d].hooks", i))
		for j, ignored := range build.Ignore {
			checker.check(ignored.XXX, fmt.Sprintf("builds[%d].ignored_builds[%d]", i, j))
		}
	}
	checker.check(config.FPM.XXX, "fpm")
	checker.check(config.Release.XXX, "release")
	checker.check(config.Release.GitHub.XXX, "release.github")
	checker.check(config.SingleBuild.XXX, "build")
	checker.check(config.SingleBuild.Hooks.XXX, "builds.hooks")
	for i, ignored := range config.SingleBuild.Ignore {
		checker.check(ignored.XXX, fmt.Sprintf("builds.ignored_builds[%d]", i))
	}
	checker.check(config.Snapshot.XXX, "snapshot")
	return checker.err()
}

type overflowChecker struct {
	fields []string
}

func (o *overflowChecker) check(m map[string]interface{}, ctx string) {
	for k := range m {
		var key = fmt.Sprintf("%s.%s", ctx, k)
		if ctx == "" {
			key = k
		}
		o.fields = append(o.fields, key)
	}
}

func (o *overflowChecker) err() error {
	if len(o.fields) == 0 {
		return nil
	}
	return fmt.Errorf(
		"unknown fields in the config file: %s",
		strings.Join(o.fields, ", "),
	)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package config

import (
	"errors"
	"time"

	"github.com/m3db/m3cluster/client/etcd"
	clusterKV "github.com/m3db/m3cluster/kv"
	"github.com/m3db/m3ctl/auth"
	"github.com/m3db/m3ctl/r2"
	"github.com/m3db/m3ctl/r2/kv"
	"github.com/m3db/m3ctl/r2/stub"
	"github.com/m3db/m3metrics/filters"
	"github.com/m3db/m3metrics/metric"
	"github.com/m3db/m3metrics/policy"
	"github.com/m3db/m3metrics/rules"
	"github.com/m3db/m3x/instrument"
	"github.com/m3db/m3x/log"
)

var (
	errKVConfigRequired = errors.New("must provide kv configuration if not using stub store")
)

// Configuration is the global configuration for r2ctl.
type Configuration struct {
	// Logging configuration.
	Logging log.Configuration `yaml:"logging"`

	// HTTP server configuration.
	HTTP serverConfig `yaml:"http"`

	// Metrics configuration.
	Metrics instrument.MetricsConfiguration `yaml:"metrics"`

	// Store configuration.
	Store r2StoreConfiguration `yaml:"store"`

	// Simple Auth Config.
	Auth *auth.SimpleAuthConfig `yaml:"auth"`
}

// r2StoreConfiguration has all the fields necessary for an R2 store.
type r2StoreConfiguration struct {
	// Stub means use the stub store.
	Stub bool `yaml:"stub"`

	// KV is the configuration for the etcd backed implementation of the kv store.
	KV *kvStoreConfig `yaml:"kv,omitempty"`
}

// NewR2Store creates a new R2 store.
func (c r2StoreConfiguration) NewR2Store(instrumentOpts instrument.Options) (r2.Store, error) {
	if c.Stub {
		return stub.NewStore(instrumentOpts), nil
	}

	if c.KV == nil {
		return nil, errKVConfigRequired
	}

	return c.KV.NewStore(instrumentOpts)
}

// kvStoreConfig is the configuration for the kv backed implementation of the R2 store.
type kvStoreConfig struct {
	// KVClient configures the client for key value store.
	KVClient *etcd.Configuration `yaml:"kvClient" validate:"nonzero"`

	// KVConfig configures the namespace and the environment.
	KVConfig clusterKV.Configuration `yaml:"kvConfig"`

	// Propagation delay for rule updates.
	PropagationDelay time.Duration `yaml:"propagationDelay" validate:"nonzero"`

	// NamespacesKey is where the namespace data lives in kv
	NamespacesKey string `yaml:"namespacesKey" validate:"nonzero"`

	// RuleSetKey fmt
	RuleSetKeyFmt string `yaml:"ruleSetKeyFmt" validate:"nonzero"`

	// Validation configuration.
	Validation *validationConfiguration `yaml:"validation"`
}

// NewStore creates a new kv backed store.
func (c kvStoreConfig) NewStore(instrumentOpts instrument.Options) (r2.Store, error) {
	// Create rules store.
	client, err := c.KVClient.NewClient(instrumentOpts)
	if err != nil {
		return nil, err
	}
	store, err := client.TxnStore(c.KVConfig.NewOptions())
	if err != nil {
		return nil, err
	}
	var validator rules.Validator
	if c.Validation != nil {
		validator = c.Validation.NewValidator()
	}
	rulesStoreOpts := rules.NewStoreOptions(c.NamespacesKey, c.RuleSetKeyFmt, validator)
	rulesStore := rules.NewStore(store, rulesStoreOpts)

	// Create kv store.
	kvStoreOpts := kv.NewStoreOptions().
		SetInstrumentOptions(instrumentOpts).
		SetRuleUpdatePropagationDelay(c.PropagationDelay)
	return kv.NewStore(rulesStore, kvStoreOpts), nil
}

type validationConfiguration struct {
	MetricTypes metricTypesValidationConfiguration `yaml:"metricTypes"`
	Policies    policiesValidationConfiguration    `yaml:"policies"`
}

func (c validationConfiguration) NewValidator() rules.Validator {
	opts := rules.NewValidatorOptions().
		SetMetricTypesFn(c.MetricTypes.NewMetricTypesFn()).
		SetDefaultAllowedStoragePolicies(c.Policies.DefaultAllowed.StoragePolicies).
		SetDefaultAllowedCustomAggregationTypes(c.Policies.DefaultAllowed.AggregationTypes)
	for _, override := range c.Policies.Overrides {
		opts = opts.
			SetAllowedStoragePoliciesFor(override.Type, override.Allowed.StoragePolicies).
			SetAllowedCustomAggregationTypesFor(override.Type, override.Allowed.AggregationTypes)
	}
	return rules.NewValidator(opts)
}

type metricTypesValidationConfiguration struct {
	// Metric type tag.
	TypeTag string `yaml:"typeTag"`

	// Allowed metric types.
	Allowed []metric.Type `yaml:"allowed"`
}

func (c metricTypesValidationConfiguration) NewMetricTypesFn() rules.MetricTypesFn {
	return func(tagFilters map[string]string) ([]metric.Type, error) {
		allowed := make([]metric.Type, 0, len(c.Allowed))
		filterStr, exists := tagFilters[c.TypeTag]
		if !exists {
			// If there is not type filter provided, the filter may match any allowed type.
			allowed = append(allowed, c.Allowed...)
			return allowed, nil
		}
		f, err := filters.NewFilter([]byte(filterStr))
		if err != nil {
			return nil, err
		}
		for _, t := range c.Allowed {
			if f.Matches([]byte(t.String())) {
				allowed = append(allowed, t)
			}
		}
		return allowed, nil
	}
}

type policiesValidationConfiguration struct {
	// DefaultAllowed defines the policies allowed by default.
	DefaultAllowed policiesConfiguration `yaml:"defaultAllowed"`

	// Overrides define the metric type specific policy overrides.
	Overrides []policiesOverrideConfiguration `yaml:"overrides"`
}

type policiesOverrideConfiguration struct {
	Type    metric.Type           `yaml:"type"`
	Allowed policiesConfiguration `yaml:"allowed"`
}

type policiesConfiguration struct {
	StoragePolicies  []policy.StoragePolicy  `yaml:"storagePolicies"`
	AggregationTypes policy.AggregationTypes `yaml:"aggregationTypes"`
}
<commit_msg>Use a slice of aggregation types for easier configuration (#17)<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package config

import (
	"errors"
	"time"

	"github.com/m3db/m3cluster/client/etcd"
	clusterKV "github.com/m3db/m3cluster/kv"
	"github.com/m3db/m3ctl/auth"
	"github.com/m3db/m3ctl/r2"
	"github.com/m3db/m3ctl/r2/kv"
	"github.com/m3db/m3ctl/r2/stub"
	"github.com/m3db/m3metrics/filters"
	"github.com/m3db/m3metrics/metric"
	"github.com/m3db/m3metrics/policy"
	"github.com/m3db/m3metrics/rules"
	"github.com/m3db/m3x/instrument"
	"github.com/m3db/m3x/log"
)

var (
	errKVConfigRequired = errors.New("must provide kv configuration if not using stub store")
)

// Configuration is the global configuration for r2ctl.
type Configuration struct {
	// Logging configuration.
	Logging log.Configuration `yaml:"logging"`

	// HTTP server configuration.
	HTTP serverConfig `yaml:"http"`

	// Metrics configuration.
	Metrics instrument.MetricsConfiguration `yaml:"metrics"`

	// Store configuration.
	Store r2StoreConfiguration `yaml:"store"`

	// Simple Auth Config.
	Auth *auth.SimpleAuthConfig `yaml:"auth"`
}

// r2StoreConfiguration has all the fields necessary for an R2 store.
type r2StoreConfiguration struct {
	// Stub means use the stub store.
	Stub bool `yaml:"stub"`

	// KV is the configuration for the etcd backed implementation of the kv store.
	KV *kvStoreConfig `yaml:"kv,omitempty"`
}

// NewR2Store creates a new R2 store.
func (c r2StoreConfiguration) NewR2Store(instrumentOpts instrument.Options) (r2.Store, error) {
	if c.Stub {
		return stub.NewStore(instrumentOpts), nil
	}

	if c.KV == nil {
		return nil, errKVConfigRequired
	}

	return c.KV.NewStore(instrumentOpts)
}

// kvStoreConfig is the configuration for the kv backed implementation of the R2 store.
type kvStoreConfig struct {
	// KVClient configures the client for key value store.
	KVClient *etcd.Configuration `yaml:"kvClient" validate:"nonzero"`

	// KVConfig configures the namespace and the environment.
	KVConfig clusterKV.Configuration `yaml:"kvConfig"`

	// Propagation delay for rule updates.
	PropagationDelay time.Duration `yaml:"propagationDelay" validate:"nonzero"`

	// NamespacesKey is where the namespace data lives in kv
	NamespacesKey string `yaml:"namespacesKey" validate:"nonzero"`

	// RuleSetKey fmt
	RuleSetKeyFmt string `yaml:"ruleSetKeyFmt" validate:"nonzero"`

	// Validation configuration.
	Validation *validationConfiguration `yaml:"validation"`
}

// NewStore creates a new kv backed store.
func (c kvStoreConfig) NewStore(instrumentOpts instrument.Options) (r2.Store, error) {
	// Create rules store.
	client, err := c.KVClient.NewClient(instrumentOpts)
	if err != nil {
		return nil, err
	}
	store, err := client.TxnStore(c.KVConfig.NewOptions())
	if err != nil {
		return nil, err
	}
	var validator rules.Validator
	if c.Validation != nil {
		validator = c.Validation.NewValidator()
	}
	rulesStoreOpts := rules.NewStoreOptions(c.NamespacesKey, c.RuleSetKeyFmt, validator)
	rulesStore := rules.NewStore(store, rulesStoreOpts)

	// Create kv store.
	kvStoreOpts := kv.NewStoreOptions().
		SetInstrumentOptions(instrumentOpts).
		SetRuleUpdatePropagationDelay(c.PropagationDelay)
	return kv.NewStore(rulesStore, kvStoreOpts), nil
}

type validationConfiguration struct {
	MetricTypes metricTypesValidationConfiguration `yaml:"metricTypes"`
	Policies    policiesValidationConfiguration    `yaml:"policies"`
}

func (c validationConfiguration) NewValidator() rules.Validator {
	opts := rules.NewValidatorOptions().
		SetMetricTypesFn(c.MetricTypes.NewMetricTypesFn()).
		SetDefaultAllowedStoragePolicies(c.Policies.DefaultAllowed.StoragePolicies).
		SetDefaultAllowedCustomAggregationTypes(c.Policies.DefaultAllowed.AggregationTypes)
	for _, override := range c.Policies.Overrides {
		opts = opts.
			SetAllowedStoragePoliciesFor(override.Type, override.Allowed.StoragePolicies).
			SetAllowedCustomAggregationTypesFor(override.Type, override.Allowed.AggregationTypes)
	}
	return rules.NewValidator(opts)
}

type metricTypesValidationConfiguration struct {
	// Metric type tag.
	TypeTag string `yaml:"typeTag"`

	// Allowed metric types.
	Allowed []metric.Type `yaml:"allowed"`
}

func (c metricTypesValidationConfiguration) NewMetricTypesFn() rules.MetricTypesFn {
	return func(tagFilters map[string]string) ([]metric.Type, error) {
		allowed := make([]metric.Type, 0, len(c.Allowed))
		filterStr, exists := tagFilters[c.TypeTag]
		if !exists {
			// If there is not type filter provided, the filter may match any allowed type.
			allowed = append(allowed, c.Allowed...)
			return allowed, nil
		}
		f, err := filters.NewFilter([]byte(filterStr))
		if err != nil {
			return nil, err
		}
		for _, t := range c.Allowed {
			if f.Matches([]byte(t.String())) {
				allowed = append(allowed, t)
			}
		}
		return allowed, nil
	}
}

type policiesValidationConfiguration struct {
	// DefaultAllowed defines the policies allowed by default.
	DefaultAllowed policiesConfiguration `yaml:"defaultAllowed"`

	// Overrides define the metric type specific policy overrides.
	Overrides []policiesOverrideConfiguration `yaml:"overrides"`
}

type policiesOverrideConfiguration struct {
	Type    metric.Type           `yaml:"type"`
	Allowed policiesConfiguration `yaml:"allowed"`
}

type policiesConfiguration struct {
	StoragePolicies  []policy.StoragePolicy   `yaml:"storagePolicies"`
	AggregationTypes []policy.AggregationType `yaml:"aggregationTypes"`
}
<|endoftext|>


<commit_before>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/google/syzkaller/fileutil"
	"github.com/google/syzkaller/sys"
	"github.com/google/syzkaller/vm"
)

type Config struct {
	Name    string // Instance name (used for identification and as GCE instance prefix)
	Http    string // TCP address to serve HTTP stats page (e.g. "localhost:50000")
	Rpc     string // TCP address to serve RPC for fuzzer processes (optional, only useful for type "none")
	Workdir string
	Vmlinux string
	Kernel  string // e.g. arch/x86/boot/bzImage
	Tag     string // arbitrary optional tag that is saved along with crash reports (e.g. kernel branch/commit)
	Cmdline string // kernel command line
	Image   string // linux image for VMs
	Initrd  string // linux initial ramdisk. (optional)
	Cpu     int    // number of VM CPUs
	Mem     int    // amount of VM memory in MBs
	Sshkey  string // root ssh key for the image
	Bin     string // qemu/lkvm binary name
	Debug   bool   // dump all VM output to console
	Output  string // one of stdout/dmesg/file (useful only for local VM)

	Syzkaller string   // path to syzkaller checkout (syz-manager will look for binaries in bin subdir)
	Type      string   // VM type (qemu, kvm, local)
	Count     int      // number of VMs (don't secify for adb, instead specify devices)
	Devices   []string // device IDs for adb
	Procs     int      // number of parallel processes inside of every VM

	Sandbox string // type of sandbox to use during fuzzing:
	// "none": don't do anything special (has false positives, e.g. due to killing init)
	// "setuid": impersonate into user nobody (65534), default
	// "namespace": create a new namespace for fuzzer using CLONE_NEWNS/CLONE_NEWNET/CLONE_NEWPID/etc,
	//	requires building kernel with CONFIG_NAMESPACES, CONFIG_UTS_NS, CONFIG_USER_NS, CONFIG_PID_NS and CONFIG_NET_NS.

	Machine_Type string // GCE machine type (e.g. "n1-highcpu-2")

	Cover bool // use kcov coverage (default: true)
	Leak  bool // do memory leak checking

	Enable_Syscalls  []string
	Disable_Syscalls []string
	Suppressions     []string
}

func Parse(filename string) (*Config, map[int]bool, []*regexp.Regexp, error) {
	if filename == "" {
		return nil, nil, nil, fmt.Errorf("supply config in -config flag")
	}
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("failed to read config file: %v", err)
	}
	return parse(data)
}

func parse(data []byte) (*Config, map[int]bool, []*regexp.Regexp, error) {
	unknown, err := checkUnknownFields(data)
	if err != nil {
		return nil, nil, nil, err
	}
	if unknown != "" {
		return nil, nil, nil, fmt.Errorf("unknown field '%v' in config", unknown)
	}
	cfg := new(Config)
	cfg.Cover = true
	cfg.Sandbox = "setuid"
	if err := json.Unmarshal(data, cfg); err != nil {
		return nil, nil, nil, fmt.Errorf("failed to parse config file: %v", err)
	}
	if _, err := os.Stat(filepath.Join(cfg.Syzkaller, "bin/syz-fuzzer")); err != nil {
		return nil, nil, nil, fmt.Errorf("bad config syzkaller param: can't find bin/syz-fuzzer")
	}
	if _, err := os.Stat(filepath.Join(cfg.Syzkaller, "bin/syz-executor")); err != nil {
		return nil, nil, nil, fmt.Errorf("bad config syzkaller param: can't find bin/syz-executor")
	}
	if cfg.Http == "" {
		return nil, nil, nil, fmt.Errorf("config param http is empty")
	}
	if cfg.Workdir == "" {
		return nil, nil, nil, fmt.Errorf("config param workdir is empty")
	}
	if cfg.Vmlinux == "" {
		return nil, nil, nil, fmt.Errorf("config param vmlinux is empty")
	}
	if cfg.Type == "" {
		return nil, nil, nil, fmt.Errorf("config param type is empty")
	}
	switch cfg.Type {
	case "none":
		if cfg.Count != 0 {
			return nil, nil, nil, fmt.Errorf("invalid config param count: %v, type \"none\" does not support param count", cfg.Count)
		}
		if cfg.Rpc == "" {
			return nil, nil, nil, fmt.Errorf("config param rpc is empty (required for type \"none\")")
		}
		if len(cfg.Devices) != 0 {
			return nil, nil, nil, fmt.Errorf("type %v does not support devices param", cfg.Type)
		}
	case "adb":
		if cfg.Count != 0 {
			return nil, nil, nil, fmt.Errorf("don't specify count for adb, instead specify devices")
		}
		if len(cfg.Devices) == 0 {
			return nil, nil, nil, fmt.Errorf("specify at least 1 adb device")
		}
		cfg.Count = len(cfg.Devices)
	case "gce":
		if cfg.Machine_Type == "" {
			return nil, nil, nil, fmt.Errorf("machine_type parameter is empty (required for gce)")
		}
		fallthrough
	default:
		if cfg.Count <= 0 || cfg.Count > 1000 {
			return nil, nil, nil, fmt.Errorf("invalid config param count: %v, want (1, 1000]", cfg.Count)
		}
		if len(cfg.Devices) != 0 {
			return nil, nil, nil, fmt.Errorf("type %v does not support devices param", cfg.Type)
		}
	}
	if cfg.Rpc == "" {
		cfg.Rpc = "localhost:0"
	}
	if cfg.Procs <= 0 {
		cfg.Procs = 1
	}
	if cfg.Output == "" {
		if cfg.Type == "local" {
			cfg.Output = "none"
		} else {
			cfg.Output = "stdout"
		}
	}
	switch cfg.Output {
	case "none", "stdout", "dmesg", "file":
	default:
		return nil, nil, nil, fmt.Errorf("config param output must contain one of none/stdout/dmesg/file")
	}
	switch cfg.Sandbox {
	case "none", "setuid", "namespace":
	default:
		return nil, nil, nil, fmt.Errorf("config param sandbox must contain one of none/setuid/namespace")
	}

	syscalls, err := parseSyscalls(cfg)
	if err != nil {
		return nil, nil, nil, err
	}

	suppressions, err := parseSuppressions(cfg)
	if err != nil {
		return nil, nil, nil, err
	}

	return cfg, syscalls, suppressions, nil
}

func parseSyscalls(cfg *Config) (map[int]bool, error) {
	match := func(call *sys.Call, str string) bool {
		if str == call.CallName || str == call.Name {
			return true
		}
		if len(str) > 1 && str[len(str)-1] == '*' && strings.HasPrefix(call.Name, str[:len(str)-1]) {
			return true
		}
		return false
	}

	syscalls := make(map[int]bool)
	if len(cfg.Enable_Syscalls) != 0 {
		for _, c := range cfg.Enable_Syscalls {
			n := 0
			for _, call := range sys.Calls {
				if match(call, c) {
					syscalls[call.ID] = true
					n++
				}
			}
			if n == 0 {
				return nil, fmt.Errorf("unknown enabled syscall: %v", c)
			}
		}
	} else {
		for _, call := range sys.Calls {
			syscalls[call.ID] = true
		}
	}
	for _, c := range cfg.Disable_Syscalls {
		n := 0
		for _, call := range sys.Calls {
			if match(call, c) {
				delete(syscalls, call.ID)
				n++
			}
		}
		if n == 0 {
			return nil, fmt.Errorf("unknown disabled syscall: %v", c)
		}
	}
	// They will be generated anyway.
	syscalls[sys.CallMap["mmap"].ID] = true
	syscalls[sys.CallMap["clock_gettime"].ID] = true

	return syscalls, nil
}

func parseSuppressions(cfg *Config) ([]*regexp.Regexp, error) {
	// Add some builtin suppressions.
	supp := append(cfg.Suppressions, []string{
		"panic: failed to start executor binary",
		"panic: executor failed: pthread_create failed",
		"panic: failed to create temp dir",
		"fatal error: runtime: out of memory",
		"fatal error: unexpected signal during runtime execution", // presubmably OOM turned into SIGBUS
		"Out of memory: Kill process .* \\(syz-fuzzer\\)",
		"lowmemorykiller: Killing 'syz-fuzzer'",
		//"INFO: lockdep is turned off", // printed by some sysrq that dumps scheduler state, but also on all lockdep reports
	}...)
	var suppressions []*regexp.Regexp
	for _, s := range supp {
		re, err := regexp.Compile(s)
		if err != nil {
			return nil, fmt.Errorf("failed to compile suppression '%v': %v", s, err)
		}
		suppressions = append(suppressions, re)
	}

	return suppressions, nil
}

func CreateVMConfig(cfg *Config, index int) (*vm.Config, error) {
	if index < 0 || index >= cfg.Count {
		return nil, fmt.Errorf("invalid VM index %v (count %v)", index, cfg.Count)
	}
	workdir, index, err := fileutil.ProcessTempDir(cfg.Workdir)
	if err != nil {
		return nil, fmt.Errorf("failed to create instance temp dir: %v", err)
	}
	vmCfg := &vm.Config{
		Name:        fmt.Sprintf("%v-%v", cfg.Type, index),
		Index:       index,
		Workdir:     workdir,
		Bin:         cfg.Bin,
		Kernel:      cfg.Kernel,
		Cmdline:     cfg.Cmdline,
		Image:       cfg.Image,
		Initrd:      cfg.Initrd,
		Sshkey:      cfg.Sshkey,
		Executor:    filepath.Join(cfg.Syzkaller, "bin", "syz-executor"),
		Cpu:         cfg.Cpu,
		Mem:         cfg.Mem,
		Debug:       cfg.Debug,
		MachineType: cfg.Machine_Type,
	}
	if len(cfg.Devices) != 0 {
		vmCfg.Device = cfg.Devices[index]
	}
	return vmCfg, nil
}

func checkUnknownFields(data []byte) (string, error) {
	// While https://github.com/golang/go/issues/15314 is not resolved
	// we don't have a better way than to enumerate all known fields.
	var fields = []string{
		"Http",
		"Rpc",
		"Workdir",
		"Vmlinux",
		"Kernel",
		"Tag",
		"Cmdline",
		"Image",
		"Cpu",
		"Mem",
		"Sshkey",
		"Bin",
		"Debug",
		"Output",
		"Syzkaller",
		"Type",
		"Count",
		"Devices",
		"Procs",
		"Cover",
		"Sandbox",
		"Leak",
		"Enable_Syscalls",
		"Disable_Syscalls",
		"Suppressions",
		"Initrd",
		"Machine_Type",
	}
	f := make(map[string]interface{})
	if err := json.Unmarshal(data, &f); err != nil {
		return "", fmt.Errorf("failed to parse config file: %v", err)
	}
	for k := range f {
		ok := false
		for _, k1 := range fields {
			if strings.ToLower(k) == strings.ToLower(k1) {
				ok = true
				break
			}
		}
		if !ok {
			return k, nil
		}
	}
	return "", nil
}
<commit_msg>config: whitelist name parameter<commit_after>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/google/syzkaller/fileutil"
	"github.com/google/syzkaller/sys"
	"github.com/google/syzkaller/vm"
)

type Config struct {
	Name    string // Instance name (used for identification and as GCE instance prefix)
	Http    string // TCP address to serve HTTP stats page (e.g. "localhost:50000")
	Rpc     string // TCP address to serve RPC for fuzzer processes (optional, only useful for type "none")
	Workdir string
	Vmlinux string
	Kernel  string // e.g. arch/x86/boot/bzImage
	Tag     string // arbitrary optional tag that is saved along with crash reports (e.g. kernel branch/commit)
	Cmdline string // kernel command line
	Image   string // linux image for VMs
	Initrd  string // linux initial ramdisk. (optional)
	Cpu     int    // number of VM CPUs
	Mem     int    // amount of VM memory in MBs
	Sshkey  string // root ssh key for the image
	Bin     string // qemu/lkvm binary name
	Debug   bool   // dump all VM output to console
	Output  string // one of stdout/dmesg/file (useful only for local VM)

	Syzkaller string   // path to syzkaller checkout (syz-manager will look for binaries in bin subdir)
	Type      string   // VM type (qemu, kvm, local)
	Count     int      // number of VMs (don't secify for adb, instead specify devices)
	Devices   []string // device IDs for adb
	Procs     int      // number of parallel processes inside of every VM

	Sandbox string // type of sandbox to use during fuzzing:
	// "none": don't do anything special (has false positives, e.g. due to killing init)
	// "setuid": impersonate into user nobody (65534), default
	// "namespace": create a new namespace for fuzzer using CLONE_NEWNS/CLONE_NEWNET/CLONE_NEWPID/etc,
	//	requires building kernel with CONFIG_NAMESPACES, CONFIG_UTS_NS, CONFIG_USER_NS, CONFIG_PID_NS and CONFIG_NET_NS.

	Machine_Type string // GCE machine type (e.g. "n1-highcpu-2")

	Cover bool // use kcov coverage (default: true)
	Leak  bool // do memory leak checking

	Enable_Syscalls  []string
	Disable_Syscalls []string
	Suppressions     []string
}

func Parse(filename string) (*Config, map[int]bool, []*regexp.Regexp, error) {
	if filename == "" {
		return nil, nil, nil, fmt.Errorf("supply config in -config flag")
	}
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("failed to read config file: %v", err)
	}
	return parse(data)
}

func parse(data []byte) (*Config, map[int]bool, []*regexp.Regexp, error) {
	unknown, err := checkUnknownFields(data)
	if err != nil {
		return nil, nil, nil, err
	}
	if unknown != "" {
		return nil, nil, nil, fmt.Errorf("unknown field '%v' in config", unknown)
	}
	cfg := new(Config)
	cfg.Cover = true
	cfg.Sandbox = "setuid"
	if err := json.Unmarshal(data, cfg); err != nil {
		return nil, nil, nil, fmt.Errorf("failed to parse config file: %v", err)
	}
	if _, err := os.Stat(filepath.Join(cfg.Syzkaller, "bin/syz-fuzzer")); err != nil {
		return nil, nil, nil, fmt.Errorf("bad config syzkaller param: can't find bin/syz-fuzzer")
	}
	if _, err := os.Stat(filepath.Join(cfg.Syzkaller, "bin/syz-executor")); err != nil {
		return nil, nil, nil, fmt.Errorf("bad config syzkaller param: can't find bin/syz-executor")
	}
	if cfg.Http == "" {
		return nil, nil, nil, fmt.Errorf("config param http is empty")
	}
	if cfg.Workdir == "" {
		return nil, nil, nil, fmt.Errorf("config param workdir is empty")
	}
	if cfg.Vmlinux == "" {
		return nil, nil, nil, fmt.Errorf("config param vmlinux is empty")
	}
	if cfg.Type == "" {
		return nil, nil, nil, fmt.Errorf("config param type is empty")
	}
	switch cfg.Type {
	case "none":
		if cfg.Count != 0 {
			return nil, nil, nil, fmt.Errorf("invalid config param count: %v, type \"none\" does not support param count", cfg.Count)
		}
		if cfg.Rpc == "" {
			return nil, nil, nil, fmt.Errorf("config param rpc is empty (required for type \"none\")")
		}
		if len(cfg.Devices) != 0 {
			return nil, nil, nil, fmt.Errorf("type %v does not support devices param", cfg.Type)
		}
	case "adb":
		if cfg.Count != 0 {
			return nil, nil, nil, fmt.Errorf("don't specify count for adb, instead specify devices")
		}
		if len(cfg.Devices) == 0 {
			return nil, nil, nil, fmt.Errorf("specify at least 1 adb device")
		}
		cfg.Count = len(cfg.Devices)
	case "gce":
		if cfg.Machine_Type == "" {
			return nil, nil, nil, fmt.Errorf("machine_type parameter is empty (required for gce)")
		}
		fallthrough
	default:
		if cfg.Count <= 0 || cfg.Count > 1000 {
			return nil, nil, nil, fmt.Errorf("invalid config param count: %v, want (1, 1000]", cfg.Count)
		}
		if len(cfg.Devices) != 0 {
			return nil, nil, nil, fmt.Errorf("type %v does not support devices param", cfg.Type)
		}
	}
	if cfg.Rpc == "" {
		cfg.Rpc = "localhost:0"
	}
	if cfg.Procs <= 0 {
		cfg.Procs = 1
	}
	if cfg.Output == "" {
		if cfg.Type == "local" {
			cfg.Output = "none"
		} else {
			cfg.Output = "stdout"
		}
	}
	switch cfg.Output {
	case "none", "stdout", "dmesg", "file":
	default:
		return nil, nil, nil, fmt.Errorf("config param output must contain one of none/stdout/dmesg/file")
	}
	switch cfg.Sandbox {
	case "none", "setuid", "namespace":
	default:
		return nil, nil, nil, fmt.Errorf("config param sandbox must contain one of none/setuid/namespace")
	}

	syscalls, err := parseSyscalls(cfg)
	if err != nil {
		return nil, nil, nil, err
	}

	suppressions, err := parseSuppressions(cfg)
	if err != nil {
		return nil, nil, nil, err
	}

	return cfg, syscalls, suppressions, nil
}

func parseSyscalls(cfg *Config) (map[int]bool, error) {
	match := func(call *sys.Call, str string) bool {
		if str == call.CallName || str == call.Name {
			return true
		}
		if len(str) > 1 && str[len(str)-1] == '*' && strings.HasPrefix(call.Name, str[:len(str)-1]) {
			return true
		}
		return false
	}

	syscalls := make(map[int]bool)
	if len(cfg.Enable_Syscalls) != 0 {
		for _, c := range cfg.Enable_Syscalls {
			n := 0
			for _, call := range sys.Calls {
				if match(call, c) {
					syscalls[call.ID] = true
					n++
				}
			}
			if n == 0 {
				return nil, fmt.Errorf("unknown enabled syscall: %v", c)
			}
		}
	} else {
		for _, call := range sys.Calls {
			syscalls[call.ID] = true
		}
	}
	for _, c := range cfg.Disable_Syscalls {
		n := 0
		for _, call := range sys.Calls {
			if match(call, c) {
				delete(syscalls, call.ID)
				n++
			}
		}
		if n == 0 {
			return nil, fmt.Errorf("unknown disabled syscall: %v", c)
		}
	}
	// They will be generated anyway.
	syscalls[sys.CallMap["mmap"].ID] = true
	syscalls[sys.CallMap["clock_gettime"].ID] = true

	return syscalls, nil
}

func parseSuppressions(cfg *Config) ([]*regexp.Regexp, error) {
	// Add some builtin suppressions.
	supp := append(cfg.Suppressions, []string{
		"panic: failed to start executor binary",
		"panic: executor failed: pthread_create failed",
		"panic: failed to create temp dir",
		"fatal error: runtime: out of memory",
		"fatal error: unexpected signal during runtime execution", // presubmably OOM turned into SIGBUS
		"Out of memory: Kill process .* \\(syz-fuzzer\\)",
		"lowmemorykiller: Killing 'syz-fuzzer'",
		//"INFO: lockdep is turned off", // printed by some sysrq that dumps scheduler state, but also on all lockdep reports
	}...)
	var suppressions []*regexp.Regexp
	for _, s := range supp {
		re, err := regexp.Compile(s)
		if err != nil {
			return nil, fmt.Errorf("failed to compile suppression '%v': %v", s, err)
		}
		suppressions = append(suppressions, re)
	}

	return suppressions, nil
}

func CreateVMConfig(cfg *Config, index int) (*vm.Config, error) {
	if index < 0 || index >= cfg.Count {
		return nil, fmt.Errorf("invalid VM index %v (count %v)", index, cfg.Count)
	}
	workdir, index, err := fileutil.ProcessTempDir(cfg.Workdir)
	if err != nil {
		return nil, fmt.Errorf("failed to create instance temp dir: %v", err)
	}
	vmCfg := &vm.Config{
		Name:        fmt.Sprintf("%v-%v", cfg.Type, index),
		Index:       index,
		Workdir:     workdir,
		Bin:         cfg.Bin,
		Kernel:      cfg.Kernel,
		Cmdline:     cfg.Cmdline,
		Image:       cfg.Image,
		Initrd:      cfg.Initrd,
		Sshkey:      cfg.Sshkey,
		Executor:    filepath.Join(cfg.Syzkaller, "bin", "syz-executor"),
		Cpu:         cfg.Cpu,
		Mem:         cfg.Mem,
		Debug:       cfg.Debug,
		MachineType: cfg.Machine_Type,
	}
	if len(cfg.Devices) != 0 {
		vmCfg.Device = cfg.Devices[index]
	}
	return vmCfg, nil
}

func checkUnknownFields(data []byte) (string, error) {
	// While https://github.com/golang/go/issues/15314 is not resolved
	// we don't have a better way than to enumerate all known fields.
	var fields = []string{
		"Name",
		"Http",
		"Rpc",
		"Workdir",
		"Vmlinux",
		"Kernel",
		"Tag",
		"Cmdline",
		"Image",
		"Cpu",
		"Mem",
		"Sshkey",
		"Bin",
		"Debug",
		"Output",
		"Syzkaller",
		"Type",
		"Count",
		"Devices",
		"Procs",
		"Cover",
		"Sandbox",
		"Leak",
		"Enable_Syscalls",
		"Disable_Syscalls",
		"Suppressions",
		"Initrd",
		"Machine_Type",
	}
	f := make(map[string]interface{})
	if err := json.Unmarshal(data, &f); err != nil {
		return "", fmt.Errorf("failed to parse config file: %v", err)
	}
	for k := range f {
		ok := false
		for _, k1 := range fields {
			if strings.ToLower(k) == strings.ToLower(k1) {
				ok = true
				break
			}
		}
		if !ok {
			return k, nil
		}
	}
	return "", nil
}
<|endoftext|>


<commit_before>package conf

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

const (
	srcConf  = "load_conf_test.conf"
	destConf = "load_conf_test2.conf"
)

func doTestTrimComments(t *testing.T) {
	ast := assert.New(t)
	var (
		err         error
		b, b1       []byte
		conf, conf1 interface{}
	)
	err = LoadEx(&conf, srcConf)
	ast.Nil(err)
	tb, err := ioutil.ReadFile(destConf)
	assert.Nil(t, err)
	err = json.Unmarshal(tb, &conf1)
	ast.Nil(err)
	b, err = json.Marshal(conf)
	ast.Nil(err)
	b1, err = json.Marshal(conf1)
	ast.Nil(err)
	ast.Equal(b1, b)
	return
}

func TestDo(t *testing.T) {
	doTestTrimComments(t)
}

func Test_getEnv(t *testing.T) {
	t.Parallel()
	tests := []struct {
		name   []string
		expect string
	}{
		{},
		{
			name:   []string{"name1", "name2", "name3"},
			expect: "name1name2name3",
		},
		{
			name:   []string{"name1"},
			expect: "name1",
		},
	}

	for _, test := range tests {
		for _, n := range test.name {
			os.Setenv(n, n)
		}
		actual := getEnv(test.name)
		assert.EqualValues(t, test.expect, actual)
	}
}

func TestGetConfigDir(t *testing.T) {
	t.Parallel()
	tests := []struct {
		app         string
		homeEnvName string
		expectErr   error
		expectDir   string
	}{
		{
			app:         "",
			homeEnvName: "",
			expectErr:   ErrHomeNotFound,
			expectDir:   "",
		},
		{
			app:         "app",
			homeEnvName: "TEST",
			expectErr:   nil,
			expectDir:   "TEST/.app",
		},
	}

	for _, test := range tests {
		tmp := homeEnvNames
		homeEnvNames = [][]string{{test.homeEnvName}}
		os.Setenv(test.homeEnvName, test.homeEnvName)
		actualDir, actualErr := GetConfigDir(test.app)
		assert.EqualValues(t, test.expectErr, actualErr)
		assert.EqualValues(t, test.expectDir, actualDir)
		homeEnvNames = tmp
	}
}

func TestInit(t *testing.T) {
	t.Parallel()
	tests := []struct {
		cflag       string
		app         string
		defaultConf string
		homeEnvName string
		expect      string
	}{
		{
			expect: "/",
		},
		{
			cflag:       "f",
			app:         "app",
			defaultConf: "defaultConf",
			homeEnvName: "TEST",
			expect:      "TEST/.app/TEST",
		},
	}
	for _, test := range tests {
		tmp := homeEnvNames
		homeEnvNames = [][]string{{test.homeEnvName}}
		os.Setenv(test.homeEnvName, test.homeEnvName)
		Init(test.cflag, test.app, test.homeEnvName)
		assert.NotEmpty(t, *confName)
		homeEnvNames = tmp
	}
}

type Config struct {
	MaxProcs   int      `json:"max_procs"`
	DebugLevel int      `json:"debug_level"`
	ConfsPath  []string `json:"confs_path"`
}

func TestLoad(t *testing.T) {
	t.Parallel()
	tests := []struct {
		content  string
		confName string
		conf     interface{}
		expect   error
	}{
		{
			content: "",
			conf:    &Config{},
			expect:  errors.New("no such file or directory"),
		},
		{
			confName: "./TestLoad",
			content: `{
				"max_procs": 2,
				"debug_level": 1,
				"confs_path": ["conf1","conf2"]
			}`,
			conf: &Config{
				MaxProcs:   2,
				DebugLevel: 1,
				ConfsPath:  []string{"conf1", "conf2"},
			},
			expect: nil,
		},
	}

	for _, test := range tests {
		confName = &test.confName
		if test.confName != "" {
			err := ioutil.WriteFile(test.confName, []byte(test.content), 0600)
			assert.Nil(t, err)
			time.Sleep(3 * time.Second)
		}
		Init("f", "app", test.confName)
		err := Load(test.conf)
		if err != nil {
			t.Log("load conf  ", test.confName, " failed: ", err)
			if !strings.Contains(err.Error(), test.expect.Error()) {
				t.Fatalf("expect contains: %s, but got: %s", test.expect.Error(), err.Error())
			}
		}
		os.RemoveAll(test.confName)
	}
}

func TestLoadEx(t *testing.T) {
	t.Parallel()
	tests := []struct {
		content  string
		confName string
		conf     interface{}
		expect   error
	}{
		{
			content: "",
			conf:    &Config{},
			expect:  errors.New("open : no such file or directory"),
		},
		{
			confName: "./TestLoadEx",
			content: `{
"max_procs": 2,
"debug_level": 1,
"confs_path": ["conf1","conf2"]
}`,
			conf: &Config{
				MaxProcs:   2,
				DebugLevel: 1,
				ConfsPath:  []string{"conf1", "conf2"},
			},
			expect: nil,
		},
		{
			confName: "./TestLoadEx",
			content: `{
"max_procs": "2",
"debug_level": 1,
"confs_path": ["conf1","conf2"]
}`,
			conf: &Config{
				MaxProcs:   2,
				DebugLevel: 1,
				ConfsPath:  []string{"conf1", "conf2"},
			},
			expect: errors.New("json: cannot unmarshal string into Go struct field Config.max_procs of type int"),
		},
	}

	for _, test := range tests {
		if test.confName != "" {
			err := ioutil.WriteFile(test.confName, []byte(test.content), 0600)
			assert.Nil(t, err)
		}
		err := LoadEx(test.conf, test.confName)
		assert.EqualValues(t, fmt.Sprintf("%v", test.expect), fmt.Sprintf("%v", err))
		os.RemoveAll(test.confName)
	}
}

func Test_trimComments(t *testing.T) {
	t.Parallel()
	tests := []struct {
		data   []byte
		expect []byte
	}{
		{
			data: []byte(`{
"max_procs": "2",
"debug_level": 1,
"confs_path": ["conf1","conf2"]
}`),
			expect: []byte(`{
"max_procs": "2",
"debug_level": 1,
"confs_path": ["conf1","conf2"]
}`),
		},
	}

	for _, test := range tests {
		actual := trimComments(test.data)
		assert.EqualValues(t, test.expect, actual)
	}

}

func Test_trimCommentsLine(t *testing.T) {
	tests := []struct {
		line   []byte
		expect []byte
	}{
		{
			line: []byte(`{
"max_procs": "2",
"debug_level": 1,
"confs_path": ["conf1","conf2"]
}`),
			expect: []byte(`{
"max_procs": "2",
"debug_level": 1,
"confs_path": ["conf1","conf2"]
}`),
		},
	}

	for _, test := range tests {
		actual := trimCommentsLine(test.line)
		assert.EqualValues(t, test.expect, actual)
	}
}

func TestLoadFile(t *testing.T) {
	t.Parallel()
	confName := "./TestLoadFile"
	conf := &Config{}
	err := ioutil.WriteFile(confName, []byte(`{
"max_procs": 2,
"debug_level": 1,
"confs_path": ["conf1","conf2"]
}`), 0755)
	assert.Nil(t, err)
	defer os.RemoveAll(confName)
	err = LoadFile(conf, confName)
	assert.Nil(t, err)
	assert.EqualValues(t, []string{"conf1", "conf2"}, conf.ConfsPath)
	assert.EqualValues(t, 2, conf.MaxProcs)
	assert.EqualValues(t, 1, conf.DebugLevel)
}
<commit_msg>fix utest<commit_after>package conf

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

const (
	srcConf  = "load_conf_test.conf"
	destConf = "load_conf_test2.conf"
)

func doTestTrimComments(t *testing.T) {
	ast := assert.New(t)
	var (
		err         error
		b, b1       []byte
		conf, conf1 interface{}
	)
	err = LoadEx(&conf, srcConf)
	ast.Nil(err)
	tb, err := ioutil.ReadFile(destConf)
	assert.Nil(t, err)
	err = json.Unmarshal(tb, &conf1)
	ast.Nil(err)
	b, err = json.Marshal(conf)
	ast.Nil(err)
	b1, err = json.Marshal(conf1)
	ast.Nil(err)
	ast.Equal(b1, b)
	return
}

func TestDo(t *testing.T) {
	doTestTrimComments(t)
}

func Test_getEnv(t *testing.T) {
	t.Parallel()
	tests := []struct {
		name   []string
		expect string
	}{
		{},
		{
			name:   []string{"name1", "name2", "name3"},
			expect: "name1name2name3",
		},
		{
			name:   []string{"name1"},
			expect: "name1",
		},
	}

	for _, test := range tests {
		for _, n := range test.name {
			os.Setenv(n, n)
		}
		actual := getEnv(test.name)
		assert.EqualValues(t, test.expect, actual)
	}
}

func TestGetConfigDir(t *testing.T) {
	t.Parallel()
	tests := []struct {
		app         string
		homeEnvName string
		expectErr   error
		expectDir   string
	}{
		{
			app:         "",
			homeEnvName: "",
			expectErr:   ErrHomeNotFound,
			expectDir:   "",
		},
		{
			app:         "app",
			homeEnvName: "TEST",
			expectErr:   nil,
			expectDir:   "TEST/.app",
		},
	}

	for _, test := range tests {
		tmp := homeEnvNames
		homeEnvNames = [][]string{{test.homeEnvName}}
		os.Setenv(test.homeEnvName, test.homeEnvName)
		actualDir, actualErr := GetConfigDir(test.app)
		assert.EqualValues(t, test.expectErr, actualErr)
		assert.EqualValues(t, test.expectDir, actualDir)
		homeEnvNames = tmp
	}
}

func TestInit(t *testing.T) {
	t.Parallel()
	tests := []struct {
		cflag       string
		app         string
		defaultConf string
		homeEnvName string
		expect      string
	}{
		{
			expect: "/",
		},
		{
			cflag:       "f",
			app:         "app",
			defaultConf: "defaultConf",
			homeEnvName: "TEST",
			expect:      "TEST/.app/TEST",
		},
	}
	for _, test := range tests {
		tmp := homeEnvNames
		homeEnvNames = [][]string{{test.homeEnvName}}
		os.Setenv(test.homeEnvName, test.homeEnvName)
		Init(test.cflag, test.app, test.homeEnvName)
		assert.NotEmpty(t, *confName)
		homeEnvNames = tmp
	}
}

type Config struct {
	MaxProcs   int      `json:"max_procs"`
	DebugLevel int      `json:"debug_level"`
	ConfsPath  []string `json:"confs_path"`
}

func TestLoad(t *testing.T) {
	t.Parallel()
	tests := []struct {
		content  string
		confName string
		conf     interface{}
		expect   error
	}{
		{
			content: "",
			conf:    &Config{},
			expect:  errors.New("no such file or directory"),
		},
		{
			confName: "./TestLoad",
			content: `{
				"max_procs": 2,
				"debug_level": 1,
				"confs_path": ["conf1","conf2"]
			}`,
			conf: &Config{
				MaxProcs:   2,
				DebugLevel: 1,
				ConfsPath:  []string{"conf1", "conf2"},
			},
			expect: nil,
		},
	}

	for _, test := range tests {
		confName = &test.confName
		if test.confName != "" {
			err := ioutil.WriteFile(test.confName, []byte(test.content), 0600)
			assert.Nil(t, err)
			time.Sleep(3 * time.Second)
		}
		err := Load(test.conf)
		if err != nil {
			t.Log("load conf  ", test.confName, " failed: ", err)
			if test.expect != nil && !strings.Contains(err.Error(), test.expect.Error()) {
				t.Fatalf("expect contains: %s, but got: %s", test.expect.Error(), err.Error())
			}
		}
		os.RemoveAll(test.confName)
	}
}

func TestLoadEx(t *testing.T) {
	t.Parallel()
	tests := []struct {
		content  string
		confName string
		conf     interface{}
		expect   error
	}{
		{
			content: "",
			conf:    &Config{},
			expect:  errors.New("open : no such file or directory"),
		},
		{
			confName: "./TestLoadEx",
			content: `{
"max_procs": 2,
"debug_level": 1,
"confs_path": ["conf1","conf2"]
}`,
			conf: &Config{
				MaxProcs:   2,
				DebugLevel: 1,
				ConfsPath:  []string{"conf1", "conf2"},
			},
			expect: nil,
		},
		{
			confName: "./TestLoadEx",
			content: `{
"max_procs": "2",
"debug_level": 1,
"confs_path": ["conf1","conf2"]
}`,
			conf: &Config{
				MaxProcs:   2,
				DebugLevel: 1,
				ConfsPath:  []string{"conf1", "conf2"},
			},
			expect: errors.New("json: cannot unmarshal string into Go struct field Config.max_procs of type int"),
		},
	}

	for _, test := range tests {
		if test.confName != "" {
			err := ioutil.WriteFile(test.confName, []byte(test.content), 0600)
			assert.Nil(t, err)
		}
		err := LoadEx(test.conf, test.confName)
		assert.EqualValues(t, fmt.Sprintf("%v", test.expect), fmt.Sprintf("%v", err))
		os.RemoveAll(test.confName)
	}
}

func Test_trimComments(t *testing.T) {
	t.Parallel()
	tests := []struct {
		data   []byte
		expect []byte
	}{
		{
			data: []byte(`{
"max_procs": "2",
"debug_level": 1,
"confs_path": ["conf1","conf2"]
}`),
			expect: []byte(`{
"max_procs": "2",
"debug_level": 1,
"confs_path": ["conf1","conf2"]
}`),
		},
	}

	for _, test := range tests {
		actual := trimComments(test.data)
		assert.EqualValues(t, test.expect, actual)
	}

}

func Test_trimCommentsLine(t *testing.T) {
	tests := []struct {
		line   []byte
		expect []byte
	}{
		{
			line: []byte(`{
"max_procs": "2",
"debug_level": 1,
"confs_path": ["conf1","conf2"]
}`),
			expect: []byte(`{
"max_procs": "2",
"debug_level": 1,
"confs_path": ["conf1","conf2"]
}`),
		},
	}

	for _, test := range tests {
		actual := trimCommentsLine(test.line)
		assert.EqualValues(t, test.expect, actual)
	}
}

func TestLoadFile(t *testing.T) {
	t.Parallel()
	confName := "./TestLoadFile"
	conf := &Config{}
	err := ioutil.WriteFile(confName, []byte(`{
"max_procs": 2,
"debug_level": 1,
"confs_path": ["conf1","conf2"]
}`), 0755)
	assert.Nil(t, err)
	defer os.RemoveAll(confName)
	err = LoadFile(conf, confName)
	assert.Nil(t, err)
	assert.EqualValues(t, []string{"conf1", "conf2"}, conf.ConfsPath)
	assert.EqualValues(t, 2, conf.MaxProcs)
	assert.EqualValues(t, 1, conf.DebugLevel)
}
<|endoftext|>


<commit_before>package config

import (
	"log"
	"os"

	"github.com/blevesearch/bleve"
	mgo "gopkg.in/mgo.v2"
)

//Conf nbfmjh
type Conf struct {
	MongoDB     string
	MongoServer string
	Database    *mgo.Database
	BleveFile   string
	BleveIndex  bleve.Index
}

var (
	config Conf
)

const (
	ADVERT  = "advert"
	LISTING = "listing"

	FACEBOOK = "facebook"
	GOOGLE   = "google"

	USERSCOLLECTION    = "Users"
	LISTINGSCOLLECTION = "Listings"
)

func CreateBleveIndex() (bleve.Index, error) {
	mapping := bleve.NewIndexMapping()
	listingsMapping := bleve.NewDocumentMapping()

	nameFieldMapping := bleve.NewTextFieldMapping()
	nameFieldMapping.Analyzer = "en"
	listingsMapping.AddFieldMappingsAt("companyname", nameFieldMapping)

	descriptionFieldMapping := bleve.NewTextFieldMapping()
	descriptionFieldMapping.Analyzer = "en"
	listingsMapping.AddFieldMappingsAt("about", descriptionFieldMapping)

	specialisationFieldMapping := bleve.NewTextFieldMapping()
	listingsMapping.AddFieldMappingsAt("about", specialisationFieldMapping)

	mapping.AddDocumentMapping("listing", listingsMapping)
	log.Println(config.BleveFile)
	bIndex, err := bleve.New(config.BleveFile, mapping)
	if err != nil {
		log.Println(err)
		return bIndex, err
	}

	return bIndex, nil
}

func Init() {
	MONGOSERVER := os.Getenv("MONGO_URL")
	MONGODB := os.Getenv("MONGODB")
	if MONGOSERVER == "" {
		log.Println("No mongo server address set, resulting to default address")
		MONGOSERVER = "127.0.0.1:27017"
		MONGODB = "yellowListings"
	}

	session, err := mgo.Dial(MONGOSERVER)
	if err != nil {
		log.Println(err)
	}

	log.Printf("mongoserver %s", MONGOSERVER)

	config = Conf{
		MongoDB:     MONGODB,
		MongoServer: MONGOSERVER,
		Database:    session.DB(MONGODB),
	}

	bleveFile := os.Getenv("BLEVE_PATH")
	if bleveFile == "" {
		log.Println("Blevefile not set, resulting to default address")
		bleveFile = "./yellowpages.bleve"
	}
	config.BleveFile = bleveFile

	log.Printf("bleve file: %s", bleveFile)
	bleveIndex, err := bleve.Open(bleveFile)
	if err != nil {
		log.Println("create bleve index")
		log.Println(err)
		bleveIndex, err = CreateBleveIndex()
	}
	config.BleveIndex = bleveIndex

}

func Get() *Conf {
	return &config
}
<commit_msg>index fixes<commit_after>package config

import (
	"log"
	"os"

	"github.com/blevesearch/bleve"
	mgo "gopkg.in/mgo.v2"
)

//Conf nbfmjh
type Conf struct {
	MongoDB     string
	MongoServer string
	Database    *mgo.Database
	BleveFile   string
	BleveIndex  bleve.Index
}

var (
	config Conf
)

const (
	ADVERT  = "advert"
	LISTING = "listing"

	FACEBOOK = "facebook"
	GOOGLE   = "google"

	USERSCOLLECTION    = "Users"
	LISTINGSCOLLECTION = "Listings"
)

func CreateBleveIndex() (bleve.Index, error) {
	mapping := bleve.NewIndexMapping()
	listingsMapping := bleve.NewDocumentMapping()

	nameFieldMapping := bleve.NewTextFieldMapping()
	nameFieldMapping.Analyzer = "en"
	listingsMapping.AddFieldMappingsAt("companyname", nameFieldMapping)

	categoryFieldMapping := bleve.NewTextFieldMapping()
	listingsMapping.AddFieldMappingsAt("category", categoryFieldMapping)

	descriptionFieldMapping := bleve.NewTextFieldMapping()
	listingsMapping.AddFieldMappingsAt("about", descriptionFieldMapping)

	specialisationFieldMapping := bleve.NewTextFieldMapping()
	listingsMapping.AddFieldMappingsAt("specialisation", specialisationFieldMapping)

	mapping.AddDocumentMapping("listing", listingsMapping)
	log.Println(config.BleveFile)
	bIndex, err := bleve.New(config.BleveFile, mapping)
	if err != nil {
		log.Println(err)
		return bIndex, err
	}

	return bIndex, nil
}

func Init() {
	MONGOSERVER := os.Getenv("MONGO_URL")
	MONGODB := os.Getenv("MONGODB")
	if MONGOSERVER == "" {
		log.Println("No mongo server address set, resulting to default address")
		MONGOSERVER = "127.0.0.1:27017"
		MONGODB = "yellowListings"
	}

	session, err := mgo.Dial(MONGOSERVER)
	if err != nil {
		log.Println(err)
	}

	log.Printf("mongoserver %s", MONGOSERVER)

	config = Conf{
		MongoDB:     MONGODB,
		MongoServer: MONGOSERVER,
		Database:    session.DB(MONGODB),
	}

	bleveFile := os.Getenv("BLEVE_PATH")
	if bleveFile == "" {
		log.Println("Blevefile not set, resulting to default address")
		bleveFile = "./yellowpages.bleve"
	}
	config.BleveFile = bleveFile

	log.Printf("bleve file: %s", bleveFile)
	bleveIndex, err := bleve.Open(bleveFile)
	if err != nil {
		log.Println("create bleve index")
		log.Println(err)
		bleveIndex, err = CreateBleveIndex()
	}
	config.BleveIndex = bleveIndex

}

func Get() *Conf {
	return &config
}
<|endoftext|>


<commit_before>package config

import (
	"crypto/tls"
	"fmt"
	"net/url"

	"github.com/cloudfoundry-incubator/candiedyaml"
	token_fetcher "github.com/cloudfoundry-incubator/uaa-token-fetcher"
	steno "github.com/cloudfoundry/gosteno"
	"github.com/pivotal-golang/localip"

	"io/ioutil"
	"runtime"
	"strconv"
	"strings"
	"time"
)

type StatusConfig struct {
	Port uint16 `yaml:"port"`
	User string `yaml:"user"`
	Pass string `yaml:"pass"`
}

var defaultStatusConfig = StatusConfig{
	Port: 8082,
	User: "",
	Pass: "",
}

type NatsConfig struct {
	Host string `yaml:"host"`
	Port uint16 `yaml:"port"`
	User string `yaml:"user"`
	Pass string `yaml:"pass"`
}

type RoutingApiConfig struct {
	Uri  string `yaml:"uri"`
	Port int    `yaml:"port"`
}

var defaultNatsConfig = NatsConfig{
	Host: "localhost",
	Port: 4222,
	User: "",
	Pass: "",
}

type LoggingConfig struct {
	File               string `yaml:"file"`
	Syslog             string `yaml:"syslog"`
	Level              string `yaml:"level"`
	LoggregatorEnabled bool   `yaml:"loggregator_enabled"`
	MetronAddress      string `yaml:"metron_address"`

	// This field is populated by the `Process` function.
	JobName string `yaml:"-"`
}

var defaultLoggingConfig = LoggingConfig{
	Level:         "debug",
	MetronAddress: "localhost:3457",
}

type Config struct {
	Status  StatusConfig  `yaml:"status"`
	Nats    []NatsConfig  `yaml:"nats"`
	Logging LoggingConfig `yaml:"logging"`

	Port              uint16 `yaml:"port"`
	Index             uint   `yaml:"index"`
	Zone              string `yaml:"zone"`
	GoMaxProcs        int    `yaml:"go_max_procs,omitempty"`
	TraceKey          string `yaml:"trace_key"`
	AccessLog         string `yaml:"access_log"`
	DebugAddr         string `yaml:"debug_addr"`
	EnableSSL         bool   `yaml:"enable_ssl"`
	SSLPort           uint16 `yaml:"ssl_port"`
	SSLCertPath       string `yaml:"ssl_cert_path"`
	SSLKeyPath        string `yaml:"ssl_key_path"`
	SSLCertificate    tls.Certificate
	SSLSkipValidation bool `yaml:"ssl_skip_validation"`

	CipherString string `yaml:"cipher_suites"`
	CipherSuites []uint16

	PublishStartMessageIntervalInSeconds int `yaml:"publish_start_message_interval"`
	PruneStaleDropletsIntervalInSeconds  int `yaml:"prune_stale_droplets_interval"`
	DropletStaleThresholdInSeconds       int `yaml:"droplet_stale_threshold"`
	PublishActiveAppsIntervalInSeconds   int `yaml:"publish_active_apps_interval"`
	StartResponseDelayIntervalInSeconds  int `yaml:"start_response_delay_interval"`
	EndpointTimeoutInSeconds             int `yaml:"endpoint_timeout"`
	RouteServiceTimeoutInSeconds         int `yaml:"route_service_timeout"`

	DrainTimeoutInSeconds int  `yaml:"drain_timeout,omitempty"`
	SecureCookies         bool `yaml:"secure_cookies"`

	OAuth                  token_fetcher.OAuthConfig `yaml:"oauth"`
	RoutingApi             RoutingApiConfig          `yaml:"routing_api"`
	RouteServiceSecret     string                    `yaml:"route_services_secret"`
	RouteServiceSecretPrev string                    `yaml:"route_services_secret_decrypt_only"`

	// These fields are populated by the `Process` function.
	PruneStaleDropletsInterval time.Duration `yaml:"-"`
	DropletStaleThreshold      time.Duration `yaml:"-"`
	PublishActiveAppsInterval  time.Duration `yaml:"-"`
	StartResponseDelayInterval time.Duration `yaml:"-"`
	EndpointTimeout            time.Duration `yaml:"-"`
	RouteServiceTimeout        time.Duration `yaml:"-"`
	DrainTimeout               time.Duration `yaml:"-"`
	Ip                         string        `yaml:"-"`
	RouteServiceEnabled        bool          `yaml:"-"`

	ExtraHeadersToLog []string `yaml:"extra_headers_to_log"`
}

var defaultConfig = Config{
	Status:  defaultStatusConfig,
	Nats:    []NatsConfig{defaultNatsConfig},
	Logging: defaultLoggingConfig,

	Port:       8081,
	Index:      0,
	GoMaxProcs: -1,
	EnableSSL:  false,
	SSLPort:    443,

	EndpointTimeoutInSeconds:     60,
	RouteServiceTimeoutInSeconds: 60,

	PublishStartMessageIntervalInSeconds: 30,
	PruneStaleDropletsIntervalInSeconds:  30,
	DropletStaleThresholdInSeconds:       120,
	PublishActiveAppsIntervalInSeconds:   0,
	StartResponseDelayIntervalInSeconds:  5,
}

func DefaultConfig() *Config {
	c := defaultConfig

	c.Process()

	return &c
}

func (c *Config) Process() {
	var err error

	if c.GoMaxProcs == -1 {
		c.GoMaxProcs = runtime.NumCPU()
	}

	c.PruneStaleDropletsInterval = time.Duration(c.PruneStaleDropletsIntervalInSeconds) * time.Second
	c.DropletStaleThreshold = time.Duration(c.DropletStaleThresholdInSeconds) * time.Second
	c.PublishActiveAppsInterval = time.Duration(c.PublishActiveAppsIntervalInSeconds) * time.Second
	c.StartResponseDelayInterval = time.Duration(c.StartResponseDelayIntervalInSeconds) * time.Second
	c.EndpointTimeout = time.Duration(c.EndpointTimeoutInSeconds) * time.Second
	c.RouteServiceTimeout = time.Duration(c.RouteServiceTimeoutInSeconds) * time.Second
	c.Logging.JobName = "router_" + c.Zone + "_" + strconv.Itoa(int(c.Index))

	if c.StartResponseDelayInterval > c.DropletStaleThreshold {
		c.DropletStaleThreshold = c.StartResponseDelayInterval
		log := steno.NewLogger("config.logger")
		log.Warnf("DropletStaleThreshold (%s) cannot be less than StartResponseDelayInterval (%s); setting both equal to StartResponseDelayInterval and continuing", c.DropletStaleThreshold, c.StartResponseDelayInterval)
	}

	drain := c.DrainTimeoutInSeconds
	if drain == 0 {
		drain = c.EndpointTimeoutInSeconds
	}
	c.DrainTimeout = time.Duration(drain) * time.Second

	c.Ip, err = localip.LocalIP()
	if err != nil {
		panic(err)
	}

	if c.EnableSSL {
		c.CipherSuites = c.processCipherSuites()
		cert, err := tls.LoadX509KeyPair(c.SSLCertPath, c.SSLKeyPath)
		if err != nil {
			panic(err)
		}
		c.SSLCertificate = cert
	}

	if c.RouteServiceSecret != "" {
		c.RouteServiceEnabled = true
	}
}

func (c *Config) processCipherSuites() []uint16 {
	cipherMap := map[string]uint16{
		"TLS_RSA_WITH_RC4_128_SHA":                0x0005,
		"TLS_RSA_WITH_AES_128_CBC_SHA":            0x002f,
		"TLS_RSA_WITH_AES_256_CBC_SHA":            0x0035,
		"TLS_ECDHE_ECDSA_WITH_RC4_128_SHA":        0xc007,
		"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA":    0xc009,
		"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA":    0xc00a,
		"TLS_ECDHE_RSA_WITH_RC4_128_SHA":          0xc011,
		"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA":      0xc013,
		"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA":      0xc014,
		"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256":   0xc02f,
		"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": 0xc02b,
	}

	defaultCiphers := []string{
		"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
		"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
		"TLS_ECDHE_RSA_WITH_RC4_128_SHA",
		"TLS_ECDHE_ECDSA_WITH_RC4_128_SHA",
		"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA",
		"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA",
		"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",
		"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA",
		"TLS_RSA_WITH_RC4_128_SHA",
		"TLS_RSA_WITH_AES_128_CBC_SHA",
		"TLS_RSA_WITH_AES_256_CBC_SHA",
	}

	var ciphers []string

	if len(strings.TrimSpace(c.CipherString)) == 0 {
		ciphers = defaultCiphers
	} else {
		ciphers = strings.Split(c.CipherString, ":")
	}

	return convertCipherStringToInt(ciphers, cipherMap)
}

func convertCipherStringToInt(cipherStrs []string, cipherMap map[string]uint16) []uint16 {
	ciphers := []uint16{}
	for _, cipher := range cipherStrs {
		if val, ok := cipherMap[cipher]; ok {
			ciphers = append(ciphers, val)
		} else {
			panic("invalid cipher string configuration")
		}
	}

	return ciphers
}

func (c *Config) NatsServers() []string {
	var natsServers []string
	for _, info := range c.Nats {
		uri := url.URL{
			Scheme: "nats",
			User:   url.UserPassword(info.User, info.Pass),
			Host:   fmt.Sprintf("%s:%d", info.Host, info.Port),
		}
		natsServers = append(natsServers, uri.String())
	}

	return natsServers
}

func (c *Config) RoutingApiEnabled() bool {
	return (c.RoutingApi.Uri != "") && (c.RoutingApi.Port != 0)
}

func (c *Config) Initialize(configYAML []byte) error {
	c.Nats = []NatsConfig{}
	return candiedyaml.Unmarshal(configYAML, &c)
}

func InitConfigFromFile(path string) *Config {
	var c *Config = DefaultConfig()
	var e error

	b, e := ioutil.ReadFile(path)
	if e != nil {
		panic(e.Error())
	}

	e = c.Initialize(b)
	if e != nil {
		panic(e.Error())
	}

	c.Process()

	return c
}
<commit_msg>Improve panic error message<commit_after>package config

import (
	"crypto/tls"
	"fmt"
	"net/url"

	"github.com/cloudfoundry-incubator/candiedyaml"
	token_fetcher "github.com/cloudfoundry-incubator/uaa-token-fetcher"
	steno "github.com/cloudfoundry/gosteno"
	"github.com/pivotal-golang/localip"

	"io/ioutil"
	"runtime"
	"strconv"
	"strings"
	"time"
)

type StatusConfig struct {
	Port uint16 `yaml:"port"`
	User string `yaml:"user"`
	Pass string `yaml:"pass"`
}

var defaultStatusConfig = StatusConfig{
	Port: 8082,
	User: "",
	Pass: "",
}

type NatsConfig struct {
	Host string `yaml:"host"`
	Port uint16 `yaml:"port"`
	User string `yaml:"user"`
	Pass string `yaml:"pass"`
}

type RoutingApiConfig struct {
	Uri  string `yaml:"uri"`
	Port int    `yaml:"port"`
}

var defaultNatsConfig = NatsConfig{
	Host: "localhost",
	Port: 4222,
	User: "",
	Pass: "",
}

type LoggingConfig struct {
	File               string `yaml:"file"`
	Syslog             string `yaml:"syslog"`
	Level              string `yaml:"level"`
	LoggregatorEnabled bool   `yaml:"loggregator_enabled"`
	MetronAddress      string `yaml:"metron_address"`

	// This field is populated by the `Process` function.
	JobName string `yaml:"-"`
}

var defaultLoggingConfig = LoggingConfig{
	Level:         "debug",
	MetronAddress: "localhost:3457",
}

type Config struct {
	Status  StatusConfig  `yaml:"status"`
	Nats    []NatsConfig  `yaml:"nats"`
	Logging LoggingConfig `yaml:"logging"`

	Port              uint16 `yaml:"port"`
	Index             uint   `yaml:"index"`
	Zone              string `yaml:"zone"`
	GoMaxProcs        int    `yaml:"go_max_procs,omitempty"`
	TraceKey          string `yaml:"trace_key"`
	AccessLog         string `yaml:"access_log"`
	DebugAddr         string `yaml:"debug_addr"`
	EnableSSL         bool   `yaml:"enable_ssl"`
	SSLPort           uint16 `yaml:"ssl_port"`
	SSLCertPath       string `yaml:"ssl_cert_path"`
	SSLKeyPath        string `yaml:"ssl_key_path"`
	SSLCertificate    tls.Certificate
	SSLSkipValidation bool `yaml:"ssl_skip_validation"`

	CipherString string `yaml:"cipher_suites"`
	CipherSuites []uint16

	PublishStartMessageIntervalInSeconds int `yaml:"publish_start_message_interval"`
	PruneStaleDropletsIntervalInSeconds  int `yaml:"prune_stale_droplets_interval"`
	DropletStaleThresholdInSeconds       int `yaml:"droplet_stale_threshold"`
	PublishActiveAppsIntervalInSeconds   int `yaml:"publish_active_apps_interval"`
	StartResponseDelayIntervalInSeconds  int `yaml:"start_response_delay_interval"`
	EndpointTimeoutInSeconds             int `yaml:"endpoint_timeout"`
	RouteServiceTimeoutInSeconds         int `yaml:"route_service_timeout"`

	DrainTimeoutInSeconds int  `yaml:"drain_timeout,omitempty"`
	SecureCookies         bool `yaml:"secure_cookies"`

	OAuth                  token_fetcher.OAuthConfig `yaml:"oauth"`
	RoutingApi             RoutingApiConfig          `yaml:"routing_api"`
	RouteServiceSecret     string                    `yaml:"route_services_secret"`
	RouteServiceSecretPrev string                    `yaml:"route_services_secret_decrypt_only"`

	// These fields are populated by the `Process` function.
	PruneStaleDropletsInterval time.Duration `yaml:"-"`
	DropletStaleThreshold      time.Duration `yaml:"-"`
	PublishActiveAppsInterval  time.Duration `yaml:"-"`
	StartResponseDelayInterval time.Duration `yaml:"-"`
	EndpointTimeout            time.Duration `yaml:"-"`
	RouteServiceTimeout        time.Duration `yaml:"-"`
	DrainTimeout               time.Duration `yaml:"-"`
	Ip                         string        `yaml:"-"`
	RouteServiceEnabled        bool          `yaml:"-"`

	ExtraHeadersToLog []string `yaml:"extra_headers_to_log"`
}

var defaultConfig = Config{
	Status:  defaultStatusConfig,
	Nats:    []NatsConfig{defaultNatsConfig},
	Logging: defaultLoggingConfig,

	Port:       8081,
	Index:      0,
	GoMaxProcs: -1,
	EnableSSL:  false,
	SSLPort:    443,

	EndpointTimeoutInSeconds:     60,
	RouteServiceTimeoutInSeconds: 60,

	PublishStartMessageIntervalInSeconds: 30,
	PruneStaleDropletsIntervalInSeconds:  30,
	DropletStaleThresholdInSeconds:       120,
	PublishActiveAppsIntervalInSeconds:   0,
	StartResponseDelayIntervalInSeconds:  5,
}

func DefaultConfig() *Config {
	c := defaultConfig

	c.Process()

	return &c
}

func (c *Config) Process() {
	var err error

	if c.GoMaxProcs == -1 {
		c.GoMaxProcs = runtime.NumCPU()
	}

	c.PruneStaleDropletsInterval = time.Duration(c.PruneStaleDropletsIntervalInSeconds) * time.Second
	c.DropletStaleThreshold = time.Duration(c.DropletStaleThresholdInSeconds) * time.Second
	c.PublishActiveAppsInterval = time.Duration(c.PublishActiveAppsIntervalInSeconds) * time.Second
	c.StartResponseDelayInterval = time.Duration(c.StartResponseDelayIntervalInSeconds) * time.Second
	c.EndpointTimeout = time.Duration(c.EndpointTimeoutInSeconds) * time.Second
	c.RouteServiceTimeout = time.Duration(c.RouteServiceTimeoutInSeconds) * time.Second
	c.Logging.JobName = "router_" + c.Zone + "_" + strconv.Itoa(int(c.Index))

	if c.StartResponseDelayInterval > c.DropletStaleThreshold {
		c.DropletStaleThreshold = c.StartResponseDelayInterval
		log := steno.NewLogger("config.logger")
		log.Warnf("DropletStaleThreshold (%s) cannot be less than StartResponseDelayInterval (%s); setting both equal to StartResponseDelayInterval and continuing", c.DropletStaleThreshold, c.StartResponseDelayInterval)
	}

	drain := c.DrainTimeoutInSeconds
	if drain == 0 {
		drain = c.EndpointTimeoutInSeconds
	}
	c.DrainTimeout = time.Duration(drain) * time.Second

	c.Ip, err = localip.LocalIP()
	if err != nil {
		panic(err)
	}

	if c.EnableSSL {
		c.CipherSuites = c.processCipherSuites()
		cert, err := tls.LoadX509KeyPair(c.SSLCertPath, c.SSLKeyPath)
		if err != nil {
			panic(err)
		}
		c.SSLCertificate = cert
	}

	if c.RouteServiceSecret != "" {
		c.RouteServiceEnabled = true
	}
}

func (c *Config) processCipherSuites() []uint16 {
	cipherMap := map[string]uint16{
		"TLS_RSA_WITH_RC4_128_SHA":                0x0005,
		"TLS_RSA_WITH_AES_128_CBC_SHA":            0x002f,
		"TLS_RSA_WITH_AES_256_CBC_SHA":            0x0035,
		"TLS_ECDHE_ECDSA_WITH_RC4_128_SHA":        0xc007,
		"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA":    0xc009,
		"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA":    0xc00a,
		"TLS_ECDHE_RSA_WITH_RC4_128_SHA":          0xc011,
		"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA":      0xc013,
		"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA":      0xc014,
		"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256":   0xc02f,
		"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": 0xc02b,
	}

	defaultCiphers := []string{
		"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
		"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
		"TLS_ECDHE_RSA_WITH_RC4_128_SHA",
		"TLS_ECDHE_ECDSA_WITH_RC4_128_SHA",
		"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA",
		"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA",
		"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",
		"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA",
		"TLS_RSA_WITH_RC4_128_SHA",
		"TLS_RSA_WITH_AES_128_CBC_SHA",
		"TLS_RSA_WITH_AES_256_CBC_SHA",
	}

	var ciphers []string

	if len(strings.TrimSpace(c.CipherString)) == 0 {
		ciphers = defaultCiphers
	} else {
		ciphers = strings.Split(c.CipherString, ":")
	}

	return convertCipherStringToInt(ciphers, cipherMap)
}

func convertCipherStringToInt(cipherStrs []string, cipherMap map[string]uint16) []uint16 {
	ciphers := []uint16{}
	for _, cipher := range cipherStrs {
		if val, ok := cipherMap[cipher]; ok {
			ciphers = append(ciphers, val)
		} else {
			panic("invalid cipher string configuration: " + cipher)
		}
	}

	return ciphers
}

func (c *Config) NatsServers() []string {
	var natsServers []string
	for _, info := range c.Nats {
		uri := url.URL{
			Scheme: "nats",
			User:   url.UserPassword(info.User, info.Pass),
			Host:   fmt.Sprintf("%s:%d", info.Host, info.Port),
		}
		natsServers = append(natsServers, uri.String())
	}

	return natsServers
}

func (c *Config) RoutingApiEnabled() bool {
	return (c.RoutingApi.Uri != "") && (c.RoutingApi.Port != 0)
}

func (c *Config) Initialize(configYAML []byte) error {
	c.Nats = []NatsConfig{}
	return candiedyaml.Unmarshal(configYAML, &c)
}

func InitConfigFromFile(path string) *Config {
	var c *Config = DefaultConfig()
	var e error

	b, e := ioutil.ReadFile(path)
	if e != nil {
		panic(e.Error())
	}

	e = c.Initialize(b)
	if e != nil {
		panic(e.Error())
	}

	c.Process()

	return c
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"sync"

	log "github.com/couchbaselabs/clog"
	"github.com/couchbaselabs/go-couchbase"
)

type ManagerEventHandlers interface {
	OnRegisterPIndex(pindex *PIndex)
	OnUnregisterPIndex(pindex *PIndex)
}

type Manager struct {
	dataDir  string
	server   string
	m        sync.Mutex
	feeds    map[string]Feed
	pindexes map[string]*PIndex

	meh ManagerEventHandlers
}

func NewManager(dataDir, server string, meh ManagerEventHandlers) *Manager {
	return &Manager{
		dataDir:  dataDir,
		server:   server,
		feeds:    make(map[string]Feed),
		pindexes: make(map[string]*PIndex),
		meh:      meh,
	}
}

func (mgr *Manager) DataDir() string {
	return mgr.dataDir
}

func (mgr *Manager) PIndexPath(pindexName string) string {
	return PIndexPath(mgr.dataDir, pindexName)
}

func (mgr *Manager) ParsePIndexPath(pindexPath string) (string, bool) {
	return ParsePIndexPath(mgr.dataDir, pindexPath)
}

func (mgr *Manager) Start() error {
	// TODO: if we one day handle multiple "seed" servers, what if
	// those seeds actually come from different clusters?  Can we have
	// multiple clusters fan-in to a single cbft?
	//
	// First, check if couchbase server is valid, else exit early.
	// Afterwards, any later loss of couchbase conns, in contrast,
	// won't exit the server, where cbft will instead retry/reconnect.
	// Empty server ("") allows for unit testing.
	if mgr.server != "" {
		_, err := couchbase.Connect(mgr.server)
		if err != nil {
			return fmt.Errorf("error: could not connect to couchbase server URL: %s, err: %v",
				mgr.server, err)
		}
	}

	// walk the data dir and register pindexes
	log.Printf("scanning dataDir...")
	dirEntries, err := ioutil.ReadDir(mgr.dataDir)
	if err != nil {
		return fmt.Errorf("error: could not read dataDir: %s, err: %v",
			mgr.dataDir, err)
	}

	for _, dirInfo := range dirEntries {
		path := mgr.dataDir + string(os.PathSeparator) + dirInfo.Name()
		name, ok := mgr.ParsePIndexPath(path)
		if !ok {
			log.Printf("  skipping: %s", dirInfo.Name())
			continue
		}

		log.Printf("  opening pindex: %s", name)
		pindex, err := OpenPIndex(name, path)
		if err != nil {
			log.Printf("error: could not open pindex: %s, err: %v",
				path, err)
			continue
		}

		// TODO: Need bucket UUID.
		err = mgr.FeedPIndex(pindex) // TODO: err handling.
		if err != nil {
			log.Printf("error: could not feed pindex: %s, err: %v",
				path, err)
			continue
		}
	}

	return nil
}

// Creates a logical index, which might be comprised of many PIndex objects.
func (mgr *Manager) CreateIndex(bucketName, bucketUUID,
	indexName string, indexMappingBytes []byte) error {
	// TODO: a logical index might map to multiple PIndexes, not the current 1-to-1.
	indexPath := mgr.PIndexPath(indexName)

	// TODO: need to check if this pindex already exists?
	// TODO: need to alloc a version/uuid for the pindex?
	// TODO: need to save feed reconstruction info (bucketName, bucketUUID, etc)
	//   with the pindex
	pindex, err := NewPIndex(indexName, indexPath, indexMappingBytes)
	if err != nil {
		return fmt.Errorf("error running pindex: %v", err)
	}

	err = mgr.FeedPIndex(pindex)
	if err != nil {
		// TODO: cleanup?
		return fmt.Errorf("error feeding pindex: %v", err)
	}

	// TODO: Create a uuid for the index?

	return nil
}

// Deletes a logical index, which might be comprised of many PIndex objects.
func (mgr *Manager) DeleteIndex(indexName string) error {
	// try to stop the feed
	// TODO: should be future, multiple feeds
	feed := mgr.UnregisterFeed(indexName)
	if feed != nil {
		err := feed.Close()
		if err != nil {
			log.Printf("error closing stream: %v", err)
		}
		// not returning error here
		// because we still want to try and delete it
	}

	pindex := mgr.UnregisterPIndex(indexName)
	if pindex != nil {
		// TODO: if we closed the stream right now, then the feed might
		// incorrectly try writing to a closed channel.
		// If there is multiple feeds going into one stream (fan-in)
		// then need to know how to count down to the final Close().
		// err := stream.Close()
		// if err != nil {
		// 	log.Printf("error closing pindex: %v", err)
		// }
		// not returning error here
		// because we still want to try and delete it
	}

	// TODO: what about any inflight queries or ops?

	// close the index
	// TODO: looks like the pindex should be responsible
	// for the final bleve.Close()
	// and actual subdirectory deletes?
	// indexToDelete := bleveHttp.UnregisterIndexByName(indexName)
	// if indexToDelete == nil {
	// 	log.Printf("no such index '%s'", indexName)
	//  }
	// indexToDelete.Close()
	pindex.BIndex().Close()

	// now delete it
	// TODO: should really send a msg to PIndex who's responsible for
	// actual file / subdir to do the deletion rather than here.

	return os.RemoveAll(mgr.PIndexPath(indexName))
}

func (mgr *Manager) FeedPIndex(pindex *PIndex) error {
	indexName := pindex.name // TODO: bad assumption of 1-to-1 pindex.name to indexName

	mgr.RegisterPIndex(indexName, pindex)

	bucketName := indexName // TODO: read bucketName out of bleve storage.
	bucketUUID := ""        // TODO: read bucketUUID and vbucket list out of bleve storage.
	feed, err := NewTAPFeed(mgr.server, "default", bucketName, bucketUUID)
	if err != nil {
		return fmt.Errorf("error: could not prepare TAP stream to server: %s,"+
			" bucketName: %s, indexName: %s, err: %v",
			mgr.server, bucketName, indexName, err)
		// TODO: need a way to collect these errors so REST api
		// can show them to user ("hey, perhaps you deleted a bucket
		// and should delete these related full-text indexes?
		// or the couchbase cluster is just down.");
		// perhaps as specialized clog writer?
		// TODO: cleanup on error?
	}

	if err = feed.Start(); err != nil {
		// TODO: need way to track dead cows (non-beef)
		// TODO: cleanup?
		return fmt.Errorf("error: could not start feed, server: %s, err: %v",
			mgr.server, err)
	}

	mgr.RegisterFeed(indexName, feed) // TODO: Need to figure out feed names.

	return nil
}

func (mgr *Manager) RegisterFeed(name string, feed Feed) {
	mgr.m.Lock()
	defer mgr.m.Unlock()
	mgr.feeds[name] = feed
}

func (mgr *Manager) UnregisterFeed(name string) Feed {
	mgr.m.Lock()
	defer mgr.m.Unlock()
	rv, ok := mgr.feeds[name]
	if ok {
		delete(mgr.feeds, name)
		return rv
	}
	return nil
}

func (mgr *Manager) RegisterPIndex(name string, pindex *PIndex) {
	mgr.m.Lock()
	defer mgr.m.Unlock()
	mgr.pindexes[name] = pindex
	if mgr.meh != nil {
		mgr.meh.OnRegisterPIndex(pindex)
	}
}

func (mgr *Manager) UnregisterPIndex(name string) *PIndex {
	mgr.m.Lock()
	defer mgr.m.Unlock()
	pindex, ok := mgr.pindexes[name]
	if ok {
		delete(mgr.pindexes, name)
		if mgr.meh != nil {
			mgr.meh.OnUnregisterPIndex(pindex)
		}
		return pindex
	}
	return nil
}
<commit_msg>sketch placeholder for janitor concept<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"sync"

	log "github.com/couchbaselabs/clog"
	"github.com/couchbaselabs/go-couchbase"
)

type ManagerEventHandlers interface {
	OnRegisterPIndex(pindex *PIndex)
	OnUnregisterPIndex(pindex *PIndex)
}

type Manager struct {
	dataDir   string
	server    string
	m         sync.Mutex
	feeds     map[string]Feed
	pindexes  map[string]*PIndex
	janitorCh chan bool // Used to kick the janitor awake.
	meh       ManagerEventHandlers
}

func NewManager(dataDir, server string, meh ManagerEventHandlers) *Manager {
	return &Manager{
		dataDir:   dataDir,
		server:    server,
		feeds:     make(map[string]Feed),
		pindexes:  make(map[string]*PIndex),
		janitorCh: make(chan bool),
		meh:       meh,
	}
}

func (mgr *Manager) DataDir() string {
	return mgr.dataDir
}

func (mgr *Manager) PIndexPath(pindexName string) string {
	return PIndexPath(mgr.dataDir, pindexName)
}

func (mgr *Manager) ParsePIndexPath(pindexPath string) (string, bool) {
	return ParsePIndexPath(mgr.dataDir, pindexPath)
}

func (mgr *Manager) Start() error {
	// TODO: if we one day handle multiple "seed" servers, what if
	// those seeds actually come from different clusters?  Can we have
	// multiple clusters fan-in to a single cbft?
	//
	// First, check if couchbase server is valid, else exit early.
	// Afterwards, any later loss of couchbase conns, in contrast,
	// won't exit the server, where cbft will instead retry/reconnect.
	// Empty server ("") allows for unit testing.
	if mgr.server != "" {
		_, err := couchbase.Connect(mgr.server)
		if err != nil {
			return fmt.Errorf("error: could not connect to couchbase server URL: %s, err: %v",
				mgr.server, err)
		}
	}

	// walk the data dir and register pindexes
	log.Printf("scanning dataDir...")
	dirEntries, err := ioutil.ReadDir(mgr.dataDir)
	if err != nil {
		return fmt.Errorf("error: could not read dataDir: %s, err: %v",
			mgr.dataDir, err)
	}

	for _, dirInfo := range dirEntries {
		path := mgr.dataDir + string(os.PathSeparator) + dirInfo.Name()
		name, ok := mgr.ParsePIndexPath(path)
		if !ok {
			log.Printf("  skipping: %s", dirInfo.Name())
			continue
		}

		log.Printf("  opening pindex: %s", name)
		pindex, err := OpenPIndex(name, path)
		if err != nil {
			log.Printf("error: could not open pindex: %s, err: %v",
				path, err)
			continue
		}

		// TODO: Need bucket UUID.
		err = mgr.FeedPIndex(pindex) // TODO: err handling.
		if err != nil {
			log.Printf("error: could not feed pindex: %s, err: %v",
				path, err)
			continue
		}
	}

	return nil
}

// Creates a logical index, which might be comprised of many PIndex objects.
func (mgr *Manager) CreateIndex(bucketName, bucketUUID,
	indexName string, indexMappingBytes []byte) error {
	// TODO: a logical index might map to multiple PIndexes, not the current 1-to-1.
	indexPath := mgr.PIndexPath(indexName)

	// TODO: need to check if this pindex already exists?
	// TODO: need to alloc a version/uuid for the pindex?
	// TODO: need to save feed reconstruction info (bucketName, bucketUUID, etc)
	//   with the pindex
	pindex, err := NewPIndex(indexName, indexPath, indexMappingBytes)
	if err != nil {
		return fmt.Errorf("error running pindex: %v", err)
	}

	err = mgr.FeedPIndex(pindex)
	if err != nil {
		// TODO: cleanup?
		return fmt.Errorf("error feeding pindex: %v", err)
	}

	// TODO: Create a uuid for the index?

	return nil
}

// Deletes a logical index, which might be comprised of many PIndex objects.
func (mgr *Manager) DeleteIndex(indexName string) error {
	// try to stop the feed
	// TODO: should be future, multiple feeds
	feed := mgr.UnregisterFeed(indexName)
	if feed != nil {
		err := feed.Close()
		if err != nil {
			log.Printf("error closing stream: %v", err)
		}
		// not returning error here
		// because we still want to try and delete it
	}

	pindex := mgr.UnregisterPIndex(indexName)
	if pindex != nil {
		// TODO: if we closed the stream right now, then the feed might
		// incorrectly try writing to a closed channel.
		// If there is multiple feeds going into one stream (fan-in)
		// then need to know how to count down to the final Close().
		// err := stream.Close()
		// if err != nil {
		// 	log.Printf("error closing pindex: %v", err)
		// }
		// not returning error here
		// because we still want to try and delete it
	}

	// TODO: what about any inflight queries or ops?

	// close the index
	// TODO: looks like the pindex should be responsible
	// for the final bleve.Close()
	// and actual subdirectory deletes?
	// indexToDelete := bleveHttp.UnregisterIndexByName(indexName)
	// if indexToDelete == nil {
	// 	log.Printf("no such index '%s'", indexName)
	//  }
	// indexToDelete.Close()
	pindex.BIndex().Close()

	// now delete it
	// TODO: should really send a msg to PIndex who's responsible for
	// actual file / subdir to do the deletion rather than here.

	return os.RemoveAll(mgr.PIndexPath(indexName))
}

// Maintains feeds, creating and deleting as necessary.
func (mgr *Manager) StartFeedsJanitor(pindex *PIndex) {
	go func() {
		for _ = range mgr.janitorCh {
			startFeeds := make(map[string]Feed)
			startPIndexes := make(map[string]*PIndex)

			// Get a snapshot of our current feeds and pindexes.
			mgr.m.Lock()
			for k, v := range mgr.feeds {
				startFeeds[k] = v
			}
			for k, v := range mgr.pindexes {
				startPIndexes[k] = v
			}
			mgr.m.Unlock()

			// TODO: create feeds that we're missing.
			// TODO: teardown feeds that we have too much of.
		}
	}()
}

func (mgr *Manager) FeedPIndex(pindex *PIndex) error {
	indexName := pindex.name // TODO: bad assumption of 1-to-1 pindex.name to indexName

	mgr.RegisterPIndex(indexName, pindex)

	bucketName := indexName // TODO: read bucketName out of bleve storage.
	bucketUUID := ""        // TODO: read bucketUUID and vbucket list out of bleve storage.
	feed, err := NewTAPFeed(mgr.server, "default", bucketName, bucketUUID)
	if err != nil {
		return fmt.Errorf("error: could not prepare TAP stream to server: %s,"+
			" bucketName: %s, indexName: %s, err: %v",
			mgr.server, bucketName, indexName, err)
		// TODO: need a way to collect these errors so REST api
		// can show them to user ("hey, perhaps you deleted a bucket
		// and should delete these related full-text indexes?
		// or the couchbase cluster is just down.");
		// perhaps as specialized clog writer?
		// TODO: cleanup on error?
	}

	if err = feed.Start(); err != nil {
		// TODO: need way to track dead cows (non-beef)
		// TODO: cleanup?
		return fmt.Errorf("error: could not start feed, server: %s, err: %v",
			mgr.server, err)
	}

	mgr.RegisterFeed(indexName, feed) // TODO: Need to figure out feed names.

	return nil
}

func (mgr *Manager) RegisterFeed(name string, feed Feed) {
	mgr.m.Lock()
	defer mgr.m.Unlock()
	mgr.feeds[name] = feed
}

func (mgr *Manager) UnregisterFeed(name string) Feed {
	mgr.m.Lock()
	defer mgr.m.Unlock()
	rv, ok := mgr.feeds[name]
	if ok {
		delete(mgr.feeds, name)
		return rv
	}
	return nil
}

func (mgr *Manager) RegisterPIndex(name string, pindex *PIndex) {
	mgr.m.Lock()
	defer mgr.m.Unlock()
	mgr.pindexes[name] = pindex
	if mgr.meh != nil {
		mgr.meh.OnRegisterPIndex(pindex)
	}
}

func (mgr *Manager) UnregisterPIndex(name string) *PIndex {
	mgr.m.Lock()
	defer mgr.m.Unlock()
	pindex, ok := mgr.pindexes[name]
	if ok {
		delete(mgr.pindexes, name)
		if mgr.meh != nil {
			mgr.meh.OnUnregisterPIndex(pindex)
		}
		return pindex
	}
	return nil
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path"
	"time"
)

type cacheEntry struct {
	ModTime  time.Time
	Contents interface{}
}

type Config struct {
	ConfigDir string
	Cache     map[string]cacheEntry
}

type Context struct {
	Network string
	Target  string
	Source  string
}

var C Config

func init() {
	log.Println("Initializing configuration cache")
	C.Cache = make(map[string]cacheEntry)
}

func lookupvar(key, path string) interface{} {
	var f interface{}
	i, err := os.Stat(path)
	if err != nil {
		return nil
	}

	_, ok := C.Cache[path]
	if C.Cache[path].ModTime.Before(i.ModTime()) || !ok {
		log.Println("Stale cache for", path)
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return nil
		}

		err = json.Unmarshal(data, &f)
		if err != nil {
			log.Println("Cannot parse", path)
			return nil
		}

		log.Println("Updating cache for", path)
		C.Cache[path] = cacheEntry{
			ModTime:  i.ModTime(),
			Contents: f,
		}

		return f.(map[string]interface{})[key]
	} else {
		log.Println("Found cache for", path)
		return C.Cache[path].Contents.(map[string]interface{})[key]
	}
}

func (c *Config) Lookup(context Context, key string) interface{} {
	var fpath string
	var value interface{}
	if context.Network != "" {
		if context.Source != "" {
			if context.Target != "" {
				fpath = path.Join(c.ConfigDir, context.Network, context.Source, context.Target+".json")

				log.Println("Context:", context, "Looking up", key, "in", fpath)
				value = lookupvar(key, fpath)
				if value != nil {
					log.Println("Context:", context, "Found", key, value)
					return value
				}
			}

			fpath = path.Join(c.ConfigDir, context.Network, context.Source+".json")

			log.Println("Context:", context, "Looking up", key, "in", fpath)
			value = lookupvar(key, fpath)
			if value != nil {
				log.Println("Context:", context, "Found", key, value)
				return value
			}
		}

		fpath = path.Join(c.ConfigDir, context.Network+".json")

		log.Println("Context:", context, "Looking up", key, "in", fpath)
		value = lookupvar(key, fpath)
		if value != nil {
			log.Println("Context:", context, "Found", key, value)
			return value
		}
	}

	fpath = path.Join(c.ConfigDir, "common.json")

	log.Println("Context:", context, "Looking up", key, "in", fpath)
	value = lookupvar(key, fpath)
	if value != nil {
		log.Println("Context:", context, "Found", key, value)
		return value
	} else {
		log.Println("Key", key, "not found")
		return nil
	}
}
<commit_msg>Implement cache purging<commit_after>package config

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path"
	"time"
)

type cacheEntry struct {
	ModTime  time.Time
	Contents interface{}
}

type Config struct {
	ConfigDir string
	Cache     map[string]cacheEntry
}

type Context struct {
	Network string
	Target  string
	Source  string
}

var C Config

func init() {
	log.Println("Initializing configuration cache")
	C.Cache = make(map[string]cacheEntry)
}

func lookupvar(key, path string) interface{} {
	var f interface{}
	i, err := os.Stat(path)
	_, ok := C.Cache[path]
	if os.IsNotExist(err) {
		log.Println("Config does not exist", path)
		if ok {
			log.Println("Purging", path, "from cache")
			delete(C.Cache, path)
		}
	}
	if err != nil {
		return nil
	}

	if C.Cache[path].ModTime.Before(i.ModTime()) || !ok {
		log.Println("Stale cache for", path)
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return nil
		}

		err = json.Unmarshal(data, &f)
		if err != nil {
			log.Println("Cannot parse", path)
			return nil
		}

		log.Println("Updating cache for", path)
		C.Cache[path] = cacheEntry{
			ModTime:  i.ModTime(),
			Contents: f,
		}

		return f.(map[string]interface{})[key]
	} else {
		log.Println("Found cache for", path)
		return C.Cache[path].Contents.(map[string]interface{})[key]
	}
}

func (c *Config) Lookup(context Context, key string) interface{} {
	var fpath string
	var value interface{}
	if context.Network != "" {
		if context.Source != "" {
			if context.Target != "" {
				fpath = path.Join(c.ConfigDir, context.Network, context.Source, context.Target+".json")

				log.Println("Context:", context, "Looking up", key, "in", fpath)
				value = lookupvar(key, fpath)
				if value != nil {
					log.Println("Context:", context, "Found", key, value)
					return value
				}
			}

			fpath = path.Join(c.ConfigDir, context.Network, context.Source+".json")

			log.Println("Context:", context, "Looking up", key, "in", fpath)
			value = lookupvar(key, fpath)
			if value != nil {
				log.Println("Context:", context, "Found", key, value)
				return value
			}
		}

		fpath = path.Join(c.ConfigDir, context.Network+".json")

		log.Println("Context:", context, "Looking up", key, "in", fpath)
		value = lookupvar(key, fpath)
		if value != nil {
			log.Println("Context:", context, "Found", key, value)
			return value
		}
	}

	fpath = path.Join(c.ConfigDir, "common.json")

	log.Println("Context:", context, "Looking up", key, "in", fpath)
	value = lookupvar(key, fpath)
	if value != nil {
		log.Println("Context:", context, "Found", key, value)
		return value
	} else {
		log.Println("Key", key, "not found")
		return nil
	}
}
<|endoftext|>