<commit_before>package sub

import (
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/triggers"
	"github.com/Symantec/Dominator/proto/common"
	"github.com/Symantec/Dominator/sub/scanner"
)

type Configuration struct {
	ScanSpeedPercent    uint
	NetworkSpeedPercent uint
	ScanExclusionList   []string
}

type FetchRequest struct {
	ServerAddress string
	Hashes        []hash.Hash
}

type FetchResponse common.StatusResponse

type GetConfigurationRequest struct {
}

type GetConfigurationResponse Configuration

type PollRequest struct {
	HaveGeneration uint64
}

type PollResponse struct {
	NetworkSpeed     uint64
	FetchInProgress  bool // Fetch() and Update() are mutually exclusive.
	UpdateInProgress bool
	GenerationCount  uint64
	FileSystem       *scanner.FileSystem
}

type SetConfigurationRequest Configuration

type SetConfigurationResponse common.StatusResponse

type Directory struct {
	Name string
	Mode filesystem.FileMode
	Uid  uint32
	Gid  uint32
}

type Hardlink struct {
	Source string
	Target string
}

type Inode struct {
	Name string
	filesystem.GenericInode
}

type UpdateRequest struct {
	PathsToDelete       []string
	DirectoriesToMake   []Directory
	DirectoriesToChange []Directory
	InodesToChange      []Inode
	HardlinksToMake     []Hardlink
	Triggers            *triggers.Triggers
}

type UpdateResponse struct{}
<commit_msg>Add InodesToMake to UpdateRequest RPC message.<commit_after>package sub

import (
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/triggers"
	"github.com/Symantec/Dominator/proto/common"
	"github.com/Symantec/Dominator/sub/scanner"
)

type Configuration struct {
	ScanSpeedPercent    uint
	NetworkSpeedPercent uint
	ScanExclusionList   []string
}

type FetchRequest struct {
	ServerAddress string
	Hashes        []hash.Hash
}

type FetchResponse common.StatusResponse

type GetConfigurationRequest struct {
}

type GetConfigurationResponse Configuration

type PollRequest struct {
	HaveGeneration uint64
}

type PollResponse struct {
	NetworkSpeed     uint64
	FetchInProgress  bool // Fetch() and Update() are mutually exclusive.
	UpdateInProgress bool
	GenerationCount  uint64
	FileSystem       *scanner.FileSystem
}

type SetConfigurationRequest Configuration

type SetConfigurationResponse common.StatusResponse

type Directory struct {
	Name string
	Mode filesystem.FileMode
	Uid  uint32
	Gid  uint32
}

type Hardlink struct {
	Source string
	Target string
}

type Inode struct {
	Name string
	filesystem.GenericInode
}

type UpdateRequest struct {
	PathsToDelete       []string
	DirectoriesToMake   []Directory
	DirectoriesToChange []Directory
	InodesToChange      []Inode
	InodesToMake        []Inode
	HardlinksToMake     []Hardlink
	Triggers            *triggers.Triggers
}

type UpdateResponse struct{}
<|endoftext|>


<commit_before>package config

import (
	"code.google.com/p/gcfg"
	"fmt"
	"os"
)

type Config struct {
	Worker struct {
		Dir string
	}
	Site struct {
		Port    string
		Ip      string
		Gif_Dir string
	}
	Redis struct {
		Port string
		Ip   string
	}
}

var conf Config

func Get() Config {
	// if we loaded it before, return that instead
	if conf.Worker.Dir != "" {
		return conf
	}
	fmt.Println(os.Getwd())
	if err := gcfg.ReadFileInto(&conf, "config.txt"); err != nil {
		panic(err)
	}
	return conf
}
<commit_msg>hook up logger<commit_after>package config

import (
	"code.google.com/p/gcfg"
	"github.com/Stantheman/youtube-gif-go/logger"
)

type Config struct {
	Worker struct {
		Dir string
	}
	Site struct {
		Port    string
		Ip      string
		Gif_Dir string
	}
	Redis struct {
		Port string
		Ip   string
	}
}

var conf Config

func Get() Config {
	// if we loaded it before, return that instead
	if conf.Worker.Dir != "" {
		return conf
	}
	if err := gcfg.ReadFileInto(&conf, "config.txt"); err != nil {
		panic(err)
	}
	logger.Get().Info("Loaded configuration")
	return conf
}
<|endoftext|>


<commit_before>/*
Package config contains convenience functions for reading and managing viper configs.

Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config

import (
	log "github.com/sirupsen/logrus"
	"github.com/spf13/viper"
	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
)

var (
	// Logrus structured logging setup
	logFields = log.Fields{
		"app":       "openmatch",
		"component": "config",
	}
	cfgLog = log.WithFields(logFields)

	// Map of the config file keys to environment variable names populated by
	// k8s into pods. Examples of redis-related env vars as written by k8s
	// REDIS_SENTINEL_PORT_6379_TCP=tcp://10.55.253.195:6379
	// REDIS_SENTINEL_PORT=tcp://10.55.253.195:6379
	// REDIS_SENTINEL_PORT_6379_TCP_ADDR=10.55.253.195
	// REDIS_SENTINEL_SERVICE_PORT=6379
	// REDIS_SENTINEL_PORT_6379_TCP_PORT=6379
	// REDIS_SENTINEL_PORT_6379_TCP_PROTO=tcp
	// REDIS_SENTINEL_SERVICE_HOST=10.55.253.195
	//
	// MMFs are expected to get their configuation from env vars instead
	// of reading the config file.  So, config parameters that are required
	// by MMFs should be populated to env vars.
	envMappings = map[string]string{
		"redis.hostname":         "REDIS_SERVICE_HOST",
		"redis.port":             "REDIS_SERVICE_PORT",
		"redis.pool.maxIdle":     "REDIS_POOL_MAXIDLE",
		"redis.pool.maxActive":   "REDIS_POOL_MAXACTIVE",
		"redis.pool.idleTimeout": "REDIS_POOL_IDLETIMEOUT",
		"api.mmlogic.hostname":   "OM_MMLOGICAPI_SERVICE_HOST",
		"api.mmlogic.port":       "OM_MMLOGICAPI_SERVICE_PORT",
	}

	// Viper config management setup
	cfg = viper.New()

	// OpenCensus
	cfgVarCount = stats.Int64("config/vars_total", "Number of config vars read during initialization", "1")
	// CfgVarCountView is the Open Census view for the cfgVarCount measure.
	CfgVarCountView = &view.View{
		Name:        "config/vars_total",
		Measure:     cfgVarCount,
		Description: "The number of config vars read during initialization",
		Aggregation: view.Count(),
	}
)

// Read reads a config file into a viper.Viper instance and associates environment vars defined in
// config.envMappings
func Read() (*viper.Viper, error) {

	// Viper config management initialization
	// Support either json or yaml file types (json for backwards compatibility
	// with previous versions)
	cfg.SetConfigType("json")
	cfg.SetConfigType("yaml")
	cfg.SetConfigName("matchmaker_config")
	cfg.AddConfigPath(".")

	// Read in config file using Viper
	err := cfg.ReadInConfig()
	if err != nil {
		cfgLog.WithFields(log.Fields{
			"error": err.Error(),
		}).Fatal("Fatal error reading config file")
	}

	// Bind this envvars to viper config vars.
	// https://github.com/spf13/viper#working-with-environment-variables
	// One important thing to recognize when working with ENV variables is
	// that the value will be read each time it is accessed. Viper does not
	// fix the value when the BindEnv is called.
	for cfgKey, envVar := range envMappings {
		err = cfg.BindEnv(cfgKey, envVar)

		if err != nil {
			cfgLog.WithFields(log.Fields{
				"configkey": cfgKey,
				"envvar":    envVar,
				"error":     err.Error(),
				"module":    "config",
			}).Warn("Unable to bind environment var as a config variable")

		} else {
			cfgLog.WithFields(log.Fields{
				"configkey": cfgKey,
				"envvar":    envVar,
				"module":    "config",
			}).Info("Binding environment var as a config variable")

		}

	}

	// Look for updates to the config; in Kubernetes, this is implemented using
	// a ConfigMap that is written to the matchmaker_config.yaml file, which is
	// what the Open Match components using Viper monitor for changes.
	// More details about Open Match's use of Kubernetes ConfigMaps at:
	// https://github.com/GoogleCloudPlatform/open-match/issues/42
	cfg.WatchConfig() // Watch and re-read config file.
	return cfg, err
}
<commit_msg>Add config/ in the search path for configuration so that PWD/config can be used as a ConfigMap mount path.<commit_after>/*
Package config contains convenience functions for reading and managing viper configs.

Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config

import (
	log "github.com/sirupsen/logrus"
	"github.com/spf13/viper"
	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
)

var (
	// Logrus structured logging setup
	logFields = log.Fields{
		"app":       "openmatch",
		"component": "config",
	}
	cfgLog = log.WithFields(logFields)

	// Map of the config file keys to environment variable names populated by
	// k8s into pods. Examples of redis-related env vars as written by k8s
	// REDIS_SENTINEL_PORT_6379_TCP=tcp://10.55.253.195:6379
	// REDIS_SENTINEL_PORT=tcp://10.55.253.195:6379
	// REDIS_SENTINEL_PORT_6379_TCP_ADDR=10.55.253.195
	// REDIS_SENTINEL_SERVICE_PORT=6379
	// REDIS_SENTINEL_PORT_6379_TCP_PORT=6379
	// REDIS_SENTINEL_PORT_6379_TCP_PROTO=tcp
	// REDIS_SENTINEL_SERVICE_HOST=10.55.253.195
	//
	// MMFs are expected to get their configuation from env vars instead
	// of reading the config file.  So, config parameters that are required
	// by MMFs should be populated to env vars.
	envMappings = map[string]string{
		"redis.hostname":         "REDIS_SERVICE_HOST",
		"redis.port":             "REDIS_SERVICE_PORT",
		"redis.pool.maxIdle":     "REDIS_POOL_MAXIDLE",
		"redis.pool.maxActive":   "REDIS_POOL_MAXACTIVE",
		"redis.pool.idleTimeout": "REDIS_POOL_IDLETIMEOUT",
		"api.mmlogic.hostname":   "OM_MMLOGICAPI_SERVICE_HOST",
		"api.mmlogic.port":       "OM_MMLOGICAPI_SERVICE_PORT",
	}

	// Viper config management setup
	cfg = viper.New()

	// OpenCensus
	cfgVarCount = stats.Int64("config/vars_total", "Number of config vars read during initialization", "1")
	// CfgVarCountView is the Open Census view for the cfgVarCount measure.
	CfgVarCountView = &view.View{
		Name:        "config/vars_total",
		Measure:     cfgVarCount,
		Description: "The number of config vars read during initialization",
		Aggregation: view.Count(),
	}
)

// Read reads a config file into a viper.Viper instance and associates environment vars defined in
// config.envMappings
func Read() (*viper.Viper, error) {

	// Viper config management initialization
	// Support either json or yaml file types (json for backwards compatibility
	// with previous versions)
	cfg.SetConfigType("json")
	cfg.SetConfigType("yaml")
	cfg.SetConfigName("matchmaker_config")
	cfg.AddConfigPath(".")
	cfg.AddConfigPath("config")

	// Read in config file using Viper
	err := cfg.ReadInConfig()
	if err != nil {
		cfgLog.WithFields(log.Fields{
			"error": err.Error(),
		}).Fatal("Fatal error reading config file")
	}

	// Bind this envvars to viper config vars.
	// https://github.com/spf13/viper#working-with-environment-variables
	// One important thing to recognize when working with ENV variables is
	// that the value will be read each time it is accessed. Viper does not
	// fix the value when the BindEnv is called.
	for cfgKey, envVar := range envMappings {
		err = cfg.BindEnv(cfgKey, envVar)

		if err != nil {
			cfgLog.WithFields(log.Fields{
				"configkey": cfgKey,
				"envvar":    envVar,
				"error":     err.Error(),
				"module":    "config",
			}).Warn("Unable to bind environment var as a config variable")

		} else {
			cfgLog.WithFields(log.Fields{
				"configkey": cfgKey,
				"envvar":    envVar,
				"module":    "config",
			}).Info("Binding environment var as a config variable")

		}

	}

	// Look for updates to the config; in Kubernetes, this is implemented using
	// a ConfigMap that is written to the matchmaker_config.yaml file, which is
	// what the Open Match components using Viper monitor for changes.
	// More details about Open Match's use of Kubernetes ConfigMaps at:
	// https://github.com/GoogleCloudPlatform/open-match/issues/42
	cfg.WatchConfig() // Watch and re-read config file.
	return cfg, err
}
<|endoftext|>


<commit_before>package config

import (
	"errors"
	"io/ioutil"
	"path/filepath"
	"regexp"

	homedir "github.com/mitchellh/go-homedir"
	"github.com/prydonius/karn/repo"
	yaml "gopkg.in/yaml.v2"
)

const (
	CONFIG_FILE   = "~/.karn.yml"
	FILE_READ_ERR = "Unable to open karn configuration file. Did you create a ~/.karn.yml in your" +
		" home directory?"
)

type Dirs map[string]*repo.Identity

func GetConfig() (Dirs, error) {
	file, err := homedir.Expand(CONFIG_FILE)
	if err != nil {
		return nil, err
	}

	source, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, errors.New(FILE_READ_ERR)
	}

	dirs := make(Dirs)
	err = yaml.Unmarshal(source, dirs)
	if err != nil {
		return dirs, err
	}

	return dirs, nil
}

func GetIdentity(path string, dirs Dirs) (*repo.Identity, error) {
	for {
		for dir, _ := range dirs {
			verdict, err := regexp.MatchString(path+"/?", dir)
			if err != nil {
				return nil, err
			}

			if verdict {
				return dirs[dir], nil
			}
		}

		path, _ = filepath.Split(path)
		length := len(path)

		if length == 1 {
			break
		}

		// Remove trailing slash
		path = path[:length-1]
	}

	return nil, nil
}
<commit_msg>Rewrite GetIdentity to make it more readable<commit_after>package config

import (
	"errors"
	"io/ioutil"
	"path/filepath"
	"regexp"

	homedir "github.com/mitchellh/go-homedir"
	"github.com/prydonius/karn/repo"
	yaml "gopkg.in/yaml.v2"
)

const (
	CONFIG_FILE   = "~/.karn.yml"
	FILE_READ_ERR = "Unable to open karn configuration file. Did you create a ~/.karn.yml in your" +
		" home directory?"
)

type Dirs map[string]*repo.Identity

func GetConfig() (Dirs, error) {
	file, err := homedir.Expand(CONFIG_FILE)
	if err != nil {
		return nil, err
	}

	source, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, errors.New(FILE_READ_ERR)
	}

	dirs := make(Dirs)
	err = yaml.Unmarshal(source, dirs)
	if err != nil {
		return dirs, err
	}

	return dirs, nil
}

func GetIdentity(path string, dirs Dirs) (*repo.Identity, error) {
	if len(path) == 1 {
		return dirs[path], nil
	}

	// Traverse directory -> id map from config
	for dir, _ := range dirs {
		// Expects the path to not have a trailing slash
		match, err := regexp.MatchString("^"+path+"/?$", dir)
		if err != nil {
			return nil, err
		}

		if match {
			return dirs[dir], nil
		}
	}

	// No match, try parent directory
	return GetIdentity(filepath.Dir(path), dirs)
}
<|endoftext|>


<commit_before>package config

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"reflect"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/davecgh/go-spew/spew"
	"github.com/go-fsnotify/fsnotify"

	"github.com/juju/loggo"
	"gopkg.in/alecthomas/kingpin.v1"
)

var (
	config   map[string]interface{}
	log      = loggo.GetLogger("config") // avoid the wrapper as it uses this module and will cause a loop
	dataPath = "/data"
	lock     = &sync.WaitGroup{}
)

func init() {
	// in snappy, we default to using the snappy data path
	snappDataPath := os.Getenv("SNAPP_APP_DATA_PATH")
	if snappDataPath != "" {
		dataPath = snappDataPath
	}

	MustRefresh()

	if Bool(false, "dumpConfig") {
		spew.Dump(GetAll(false))
	}

	go func() {

		watcher, err := fsnotify.NewWatcher()
		if err != nil {
			panic("Failed to create watcher: " + err.Error())
		}
		watcher.Add(dataPath + "/etc/opt/ninja")

		for {
			select {
			case ev := <-watcher.Events:
				log.Infof("Config updated: %s", ev.Name)
				MustRefresh()
			case err := <-watcher.Errors:
				log.Warningf("Config Watcher error: %s", err)
			}
		}

	}()
}

func GetAll(flatten bool) map[string]interface{} {
	if flatten {
		return config
	}
	return unflatten(config)
}

var serial string
var sphereVersion string

func Serial() string {

	if serial == "" {

		if HasString("serial") {
			serial = String("serial")
		} else {

			cmd := exec.Command("sphere-serial", os.Args[1:]...)

			var out bytes.Buffer
			cmd.Stdout = &out

			err := cmd.Run()
			if err == nil {
				serial = out.String()
			} else {

				if runtime.GOOS == "darwin" {
					serial = darwinSerial()
				} else {
					log.Errorf("Failed to get sphere serial (sphere-serial must be in the PATH) error:%s", err)
					panic(err)
				}
			}

		}
	}

	return serial
}

func darwinSerial() string {
	cmd := exec.Command("/bin/sh", "-c", "system_profiler SPHardwareDataType | sed -n 's/.*Serial Number (system).*: /OSX/p'")
	bytes, err := cmd.Output()
	if err != nil {
		log.Errorf("Failed to get darwin serial: %s ", err)
		panic("No darwin serial")
	}

	return string(bytes[0 : len(bytes)-1])
}

func SphereVersion() string {
	if sphereVersion == "" {
		if HasString("sphere-version") {

			sphereVersion = String("sphere-version")

		} else {

			cmd := exec.Command("sphere-version", os.Args[1:]...)

			var out bytes.Buffer
			cmd.Stdout = &out

			err := cmd.Run()
			if err != nil {
				log.Errorf("Failed to get sphere version (sphere-version must be in the PATH) error:%s", err)
				sphereVersion = "[unknown]"
			} else {
				sphereVersion = strings.TrimSpace(out.String())
			}
		}

	}
	return sphereVersion
}

func IsPaired() bool {
	return /*HasString("sphereNetworkKey") && */ HasString("token") && HasString("userId")
}

func NoCloud() bool {
	return Bool(false, "noCloud")
}

func String(def string, path ...string) string {
	val := get(path...)
	if val == nil {
		return def
	}
	return val.(string)
}

// MustString returns the string property at the path
func MustString(path ...string) string {
	return mustGet(path...).(string)
}

// Duration returns the string property at the path, as a time.Duration
func Duration(def time.Duration, path ...string) time.Duration {
	s := String(hey, path...)
	if s == hey {
		return def
	}
	d, err := time.ParseDuration(s)
	if err != nil {
		log.Infof("Failed to parse duration '%s': %s", s, err)
		return def
	}
	return d
}

// MustDuration returns the string property at the path, as a time.Duration
func MustDuration(path ...string) time.Duration {
	s := MustString(path...)
	d, err := time.ParseDuration(s)
	if err != nil {
		log.Infof("Failed to parse duration '%s': %s", s, err)
	}
	return d
}

// MustStringArray returns the string array property at the path
func MustStringArray(path ...string) []string {
	a := mustGet(path...).([]interface{})
	b := make([]string, len(a))
	for i := range a {
		b[i] = a[i].(string)
	}
	return b
}

// Int returns the integer property at the path, with a default
func Int(def int, path ...string) int {
	val := get(path...)
	if val == nil {
		return def
	}
	return int(val.(float64))
}

// MustInt returns the string property at the path
func MustInt(path ...string) int {
	return int(mustGet(path...).(float64))
}

// Float returns the float property at the path, with a default
func Float(def float64, path ...string) float64 {
	val := get(path...)
	if val == nil {
		return def
	}
	return val.(float64)
}

func MustFloat(path ...string) float64 {
	return mustGet(path...).(float64)
}

// Bool returns the boolean property at the path, with a default
func Bool(def bool, path ...string) bool {
	val := get(path...)
	if val == nil {
		return def
	}
	return val.(bool)
}

// MustBool returns the boolean property at the path
func MustBool(path ...string) bool {
	return mustGet(path...).(bool)
}

var hey = "what's up buddy?"

func HasString(path ...string) bool {
	lock.Wait()

	val, ok := config[strings.Join(path, ".")]
	if !ok {
		return false
	}

	_, ok = val.(string)
	return ok
}

func mustGet(path ...string) interface{} {
	lock.Wait()

	val, ok := config[strings.Join(path, ".")]
	if !ok {
		log.Errorf("expected value for %v but found nothing", path)
		panic(fmt.Errorf("expected value for %v but found nothing", path))
	}
	return val
}

func get(path ...string) interface{} {
	lock.Wait()

	val, ok := config[strings.Join(path, ".")]
	if !ok {
		return nil
	}
	return val
}

func MustRefresh() {
	lock.Wait()
	lock.Add(1)
	defer lock.Done()

	flat := make(map[string]interface{})

	// cli overrides
	addArgs(flat)

	// load environments (no value args) from cli args
	environments := []string{}
	for name, value := range flat {
		if value == nil {
			environments = append(environments, name)
		} else {
			if ok, boolValue := value.(bool); ok {
				if boolValue {
					environments = append(environments, name)
				}
			}
		}
	}

	// env vars (if starting with "sphere_")
	addEnv(flat)

	// If there aren't any environments set via cli, see if any were set in env var
	if len(environments) == 0 {
		if v, ok := flat["env"]; ok {
			environments = append(environments, strings.Split(v.(string), ",")...)
		}
	}

	environments = append(environments, "default")

	flat["env"] = environments

	log.Infof("Environments: %s", strings.Join(environments, ", "))

	userHome := getUserHome()

	// anything that can be parsed as a number, is a number
	parseNumbers(flat)

	installDir := "/opt/ninjablocks"
	if val, ok := flat["installDirectory"]; ok {
		installDir = val.(string)
	}

	if _, err := os.Stat(installDir); err != nil {
		// check for installation in snappy, apply different default path
		snappAppPath := os.Getenv("SNAPP_APP_PATH")
		if snappAppPath != "" {
			installDir = snappAppPath
		}
	}

	flat["installDirectory"] = installDir

	if _, err := os.Stat(installDir); err != nil {
		log.Warningf("Couldn't load sphere install directory. Override with env var sphere_installDirectory. error:%s", err)
	}

	// User overrides (json)
	addFile(dataPath+"/config.json", flat)

	files, _ := ioutil.ReadDir(dataPath + "/etc/opt/ninja")
	for _, f := range files {
		if strings.HasSuffix(f.Name(), ".json") {
			addFile(dataPath+"/etc/opt/ninja/"+f.Name(), flat)
		}
	}

	// home directory environment(s) config
	for i := len(environments) - 1; i >= 0; i-- {
		addFile(filepath.Join(userHome, ".sphere", environments[i]+".json"), flat)
	}

	// current directory environment(s) config
	for i := len(environments) - 1; i >= 0; i-- {
		addFile(filepath.Join(".", "config", environments[i]+".json"), flat)
	}

	// common environment(s) config
	for i := len(environments) - 1; i >= 0; i-- {
		addFile(filepath.Join(installDir, "config", environments[i]+".json"), flat)
	}

	//log.Debugf("Loaded config: %v", flat)

	config = flat

	if nc, ok := config["noCloud"]; ok {
		if nc.(bool) {
			config["userId"] = "nouser"
			config["token"] = "notoken"
			config["sphereNetworkKey"] = "nonetworkkey"
			config["siteId"] = "nomesh" + Serial()
			config["masterNodeId"] = Serial()
		}
	}

}

func addEnv(config map[string]interface{}) {
	prefix := "sphere_"
	for _, v := range os.Environ() {

		re := regexp.MustCompile("([^=]*)=(.*)")
		split := re.FindStringSubmatch(v)
		if split != nil {
			name, value := split[1], split[2]

			if strings.HasPrefix(name, prefix) {
				name = strings.TrimPrefix(name, prefix)
				name = strings.Replace(name, "_", ".", -1)

				if _, ok := config[name]; !ok {
					config[name] = value
				}
			}
		}
	}
}

func addArgs(config map[string]interface{}) {

	parser := kingpin.Tokenize(os.Args[1:]).Tokens

	for token, parser := parser.Peek(), parser.Next(); token.Type != kingpin.TokenEOL; token, parser = parser.Peek(), parser.Next() {

		if token.IsFlag() {
			var value interface{}
			name := token.Value

			next := parser.Peek()
			if next.Type == kingpin.TokenArg {
				if next.Value == "false" {
					value = false
				} else if next.Value == "true" {
					value = true
				} else {
					value = next.Value
				}
			} else {
				value = true
				// It's an environment indicator... like --cloud-production
			}

			config[name] = value
		}

	}

}

func addFile(path string, config map[string]interface{}) error {
	//log.Debugf("Loading config file: %s", path)

	file, e := ioutil.ReadFile(path)
	if e != nil {
		return fmt.Errorf("Failed to load file: %s error: %s", path, e)
	}

	content := make(map[string]interface{})
	e = json.Unmarshal(file, &content)
	if e != nil {
		return fmt.Errorf("Failed to read file: %s error: %s", path, e)
	}

	//spew.Dump(path, content)

	flatten(content, nil, config)
	return nil
}

func flatten(input interface{}, lpath []string, flattened map[string]interface{}) {
	if lpath == nil {
		lpath = []string{}
	}

	if reflect.ValueOf(input).Kind() == reflect.Map {
		for rkey, value := range input.(map[string]interface{}) {
			flatten(value, append(lpath, rkey), flattened)
		}
	} else {
		if _, ok := flattened[strings.Join(lpath, ".")]; !ok {
			flattened[strings.Join(lpath, ".")] = input
		}
	}
}

func unflatten(flat map[string]interface{}) map[string]interface{} {
	out := make(map[string]interface{})
	obj := out

	for key, val := range flat {

		obj = out
		keys := strings.Split(key, ".")

		for i, k := range keys {
			if i == len(keys)-1 {
				obj[k] = val
			} else {
				next, ok := obj[k]
				if !ok {
					next = make(map[string]interface{})
				}
				obj[k] = next
				obj = next.(map[string]interface{})
			}
		}

	}

	return out
}

func parseNumbers(config map[string]interface{}) {
	for name, val := range config {
		stringVal, ok := val.(string)
		if ok {
			floatVal, err := strconv.ParseFloat(stringVal, 64)
			if err == nil {
				config[name] = floatVal
			}
		}
	}
}

func getUserHome() string {
	usr, err := user.Current()
	if err != nil {
		return "/root"
	}
	return usr.HomeDir
}

func IsSlave() bool {
	masterNodeId := String("", "masterNodeId")
	serial := Serial()
	return masterNodeId != "" && serial != masterNodeId
}

func IsMaster() bool {
	masterNodeId := String("", "masterNodeId")
	serial := Serial()
	return masterNodeId != "" && serial == masterNodeId
}
<commit_msg>fix up configuration override order so that the following are used in order:<commit_after>package config

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"reflect"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/davecgh/go-spew/spew"
	"github.com/go-fsnotify/fsnotify"

	"github.com/juju/loggo"
	"gopkg.in/alecthomas/kingpin.v1"
)

var (
	config   map[string]interface{}
	log      = loggo.GetLogger("config") // avoid the wrapper as it uses this module and will cause a loop
	dataPath = "/data"
	lock     = &sync.WaitGroup{}
)

func init() {
	// in snappy, we default to using the snappy data path
	snappDataPath := os.Getenv("SNAPP_APP_DATA_PATH")
	if snappDataPath != "" {
		dataPath = snappDataPath
	}

	MustRefresh()

	if Bool(false, "dumpConfig") {
		spew.Dump(GetAll(false))
	}

	go func() {

		watcher, err := fsnotify.NewWatcher()
		if err != nil {
			panic("Failed to create watcher: " + err.Error())
		}
		watcher.Add(dataPath + "/etc/opt/ninja")

		for {
			select {
			case ev := <-watcher.Events:
				log.Infof("Config updated: %s", ev.Name)
				MustRefresh()
			case err := <-watcher.Errors:
				log.Warningf("Config Watcher error: %s", err)
			}
		}

	}()
}

func GetAll(flatten bool) map[string]interface{} {
	if flatten {
		return config
	}
	return unflatten(config)
}

var serial string
var sphereVersion string

func Serial() string {

	if serial == "" {

		if HasString("serial") {
			serial = String("serial")
		} else {

			cmd := exec.Command("sphere-serial", os.Args[1:]...)

			var out bytes.Buffer
			cmd.Stdout = &out

			err := cmd.Run()
			if err == nil {
				serial = out.String()
			} else {

				if runtime.GOOS == "darwin" {
					serial = darwinSerial()
				} else {
					log.Errorf("Failed to get sphere serial (sphere-serial must be in the PATH) error:%s", err)
					panic(err)
				}
			}

		}
	}

	return serial
}

func darwinSerial() string {
	cmd := exec.Command("/bin/sh", "-c", "system_profiler SPHardwareDataType | sed -n 's/.*Serial Number (system).*: /OSX/p'")
	bytes, err := cmd.Output()
	if err != nil {
		log.Errorf("Failed to get darwin serial: %s ", err)
		panic("No darwin serial")
	}

	return string(bytes[0 : len(bytes)-1])
}

func SphereVersion() string {
	if sphereVersion == "" {
		if HasString("sphere-version") {

			sphereVersion = String("sphere-version")

		} else {

			cmd := exec.Command("sphere-version", os.Args[1:]...)

			var out bytes.Buffer
			cmd.Stdout = &out

			err := cmd.Run()
			if err != nil {
				log.Errorf("Failed to get sphere version (sphere-version must be in the PATH) error:%s", err)
				sphereVersion = "[unknown]"
			} else {
				sphereVersion = strings.TrimSpace(out.String())
			}
		}

	}
	return sphereVersion
}

func IsPaired() bool {
	return /*HasString("sphereNetworkKey") && */ HasString("token") && HasString("userId")
}

func NoCloud() bool {
	return Bool(false, "noCloud")
}

func String(def string, path ...string) string {
	val := get(path...)
	if val == nil {
		return def
	}
	return val.(string)
}

// MustString returns the string property at the path
func MustString(path ...string) string {
	return mustGet(path...).(string)
}

// Duration returns the string property at the path, as a time.Duration
func Duration(def time.Duration, path ...string) time.Duration {
	s := String(hey, path...)
	if s == hey {
		return def
	}
	d, err := time.ParseDuration(s)
	if err != nil {
		log.Infof("Failed to parse duration '%s': %s", s, err)
		return def
	}
	return d
}

// MustDuration returns the string property at the path, as a time.Duration
func MustDuration(path ...string) time.Duration {
	s := MustString(path...)
	d, err := time.ParseDuration(s)
	if err != nil {
		log.Infof("Failed to parse duration '%s': %s", s, err)
	}
	return d
}

// MustStringArray returns the string array property at the path
func MustStringArray(path ...string) []string {
	a := mustGet(path...).([]interface{})
	b := make([]string, len(a))
	for i := range a {
		b[i] = a[i].(string)
	}
	return b
}

// Int returns the integer property at the path, with a default
func Int(def int, path ...string) int {
	val := get(path...)
	if val == nil {
		return def
	}
	return int(val.(float64))
}

// MustInt returns the string property at the path
func MustInt(path ...string) int {
	return int(mustGet(path...).(float64))
}

// Float returns the float property at the path, with a default
func Float(def float64, path ...string) float64 {
	val := get(path...)
	if val == nil {
		return def
	}
	return val.(float64)
}

func MustFloat(path ...string) float64 {
	return mustGet(path...).(float64)
}

// Bool returns the boolean property at the path, with a default
func Bool(def bool, path ...string) bool {
	val := get(path...)
	if val == nil {
		return def
	}
	return val.(bool)
}

// MustBool returns the boolean property at the path
func MustBool(path ...string) bool {
	return mustGet(path...).(bool)
}

var hey = "what's up buddy?"

func HasString(path ...string) bool {
	lock.Wait()

	val, ok := config[strings.Join(path, ".")]
	if !ok {
		return false
	}

	_, ok = val.(string)
	return ok
}

func mustGet(path ...string) interface{} {
	lock.Wait()

	val, ok := config[strings.Join(path, ".")]
	if !ok {
		log.Errorf("expected value for %v but found nothing", path)
		panic(fmt.Errorf("expected value for %v but found nothing", path))
	}
	return val
}

func get(path ...string) interface{} {
	lock.Wait()

	val, ok := config[strings.Join(path, ".")]
	if !ok {
		return nil
	}
	return val
}

func MustRefresh() {
	lock.Wait()
	lock.Add(1)
	defer lock.Done()

	flat := make(map[string]interface{})

	// cli overrides
	addArgs(flat)

	// env vars (if starting with "sphere_")
	addEnv(flat)

	// initialise the list
	environments := []string{"default"}

	// add environments read from the env variable
	if v, ok := flat["env"]; ok {
		environments = append(environments, strings.Split(v.(string), ",")...)
	}

	// then add any found in cli arguments
	for name, value := range flat {
		if value == nil {
			environments = append(environments, name)
		} else {
			if ok, boolValue := value.(bool); ok {
				if boolValue {
					environments = append(environments, name)
				}
			}
		}
	}

	flat["env"] = environments

	log.Infof("Environments: %s", strings.Join(environments, ", "))

	userHome := getUserHome()

	// anything that can be parsed as a number, is a number
	parseNumbers(flat)

	installDir := "/opt/ninjablocks"
	if val, ok := flat["installDirectory"]; ok {
		installDir = val.(string)
	}

	if _, err := os.Stat(installDir); err != nil {
		// check for installation in snappy, apply different default path
		snappAppPath := os.Getenv("SNAPP_APP_PATH")
		if snappAppPath != "" {
			installDir = snappAppPath
		}
	}

	flat["installDirectory"] = installDir

	if _, err := os.Stat(installDir); err != nil {
		log.Warningf("Couldn't load sphere install directory. Override with env var sphere_installDirectory. error:%s", err)
	}

	// User overrides (json)
	addFile(dataPath+"/config.json", flat)

	files, _ := ioutil.ReadDir(dataPath + "/etc/opt/ninja")
	for _, f := range files {
		if strings.HasSuffix(f.Name(), ".json") {
			addFile(dataPath+"/etc/opt/ninja/"+f.Name(), flat)
		}
	}

	// home directory environment(s) config
	for i := len(environments) - 1; i >= 0; i-- {
		addFile(filepath.Join(userHome, ".sphere", environments[i]+".json"), flat)
	}

	// current directory environment(s) config
	for i := len(environments) - 1; i >= 0; i-- {
		addFile(filepath.Join(".", "config", environments[i]+".json"), flat)
	}

	// common environment(s) config
	for i := len(environments) - 1; i >= 0; i-- {
		addFile(filepath.Join(installDir, "config", environments[i]+".json"), flat)
	}

	//log.Debugf("Loaded config: %v", flat)

	config = flat

	if nc, ok := config["noCloud"]; ok {
		if nc.(bool) {
			config["userId"] = "nouser"
			config["token"] = "notoken"
			config["sphereNetworkKey"] = "nonetworkkey"
			config["siteId"] = "nomesh" + Serial()
			config["masterNodeId"] = Serial()
		}
	}

}

func addEnv(config map[string]interface{}) {
	prefix := "sphere_"
	for _, v := range os.Environ() {

		re := regexp.MustCompile("([^=]*)=(.*)")
		split := re.FindStringSubmatch(v)
		if split != nil {
			name, value := split[1], split[2]

			if strings.HasPrefix(name, prefix) {
				name = strings.TrimPrefix(name, prefix)
				name = strings.Replace(name, "_", ".", -1)

				if _, ok := config[name]; !ok {
					config[name] = value
				}
			}
		}
	}
}

func addArgs(config map[string]interface{}) {

	parser := kingpin.Tokenize(os.Args[1:]).Tokens

	for token, parser := parser.Peek(), parser.Next(); token.Type != kingpin.TokenEOL; token, parser = parser.Peek(), parser.Next() {

		if token.IsFlag() {
			var value interface{}
			name := token.Value

			next := parser.Peek()
			if next.Type == kingpin.TokenArg {
				if next.Value == "false" {
					value = false
				} else if next.Value == "true" {
					value = true
				} else {
					value = next.Value
				}
			} else {
				value = true
				// It's an environment indicator... like --cloud-production
			}

			config[name] = value
		}

	}

}

func addFile(path string, config map[string]interface{}) error {
	//log.Debugf("Loading config file: %s", path)

	file, e := ioutil.ReadFile(path)
	if e != nil {
		return fmt.Errorf("Failed to load file: %s error: %s", path, e)
	}

	content := make(map[string]interface{})
	e = json.Unmarshal(file, &content)
	if e != nil {
		return fmt.Errorf("Failed to read file: %s error: %s", path, e)
	}

	//spew.Dump(path, content)

	flatten(content, nil, config)
	return nil
}

func flatten(input interface{}, lpath []string, flattened map[string]interface{}) {
	if lpath == nil {
		lpath = []string{}
	}

	if reflect.ValueOf(input).Kind() == reflect.Map {
		for rkey, value := range input.(map[string]interface{}) {
			flatten(value, append(lpath, rkey), flattened)
		}
	} else {
		if _, ok := flattened[strings.Join(lpath, ".")]; !ok {
			flattened[strings.Join(lpath, ".")] = input
		}
	}
}

func unflatten(flat map[string]interface{}) map[string]interface{} {
	out := make(map[string]interface{})
	obj := out

	for key, val := range flat {

		obj = out
		keys := strings.Split(key, ".")

		for i, k := range keys {
			if i == len(keys)-1 {
				obj[k] = val
			} else {
				next, ok := obj[k]
				if !ok {
					next = make(map[string]interface{})
				}
				obj[k] = next
				obj = next.(map[string]interface{})
			}
		}

	}

	return out
}

func parseNumbers(config map[string]interface{}) {
	for name, val := range config {
		stringVal, ok := val.(string)
		if ok {
			floatVal, err := strconv.ParseFloat(stringVal, 64)
			if err == nil {
				config[name] = floatVal
			}
		}
	}
}

func getUserHome() string {
	usr, err := user.Current()
	if err != nil {
		return "/root"
	}
	return usr.HomeDir
}

func IsSlave() bool {
	masterNodeId := String("", "masterNodeId")
	serial := Serial()
	return masterNodeId != "" && serial != masterNodeId
}

func IsMaster() bool {
	masterNodeId := String("", "masterNodeId")
	serial := Serial()
	return masterNodeId != "" && serial == masterNodeId
}
<|endoftext|>


<commit_before>package config

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	yaml "gopkg.in/yaml.v2"
)

// S3CloudStorageConfig stores the configuration of an S3 API proxy backend.
type S3CloudStorageConfig struct {
	Endpoint        string `yaml:"endpoint"`
	Bucket          string `yaml:"bucket"`
	Prefix          string `yaml:"prefix"`
	AccessKeyID     string `yaml:"access_key_id"`
	SecretAccessKey string `yaml:"secret_access_key"`
	DisableSSL      bool   `yaml:"disable_ssl"`
	IAMRoleEndpoint string `yaml:"iam_role_endpoint"`
	Region          string `yaml:"region"`
}

// GoogleCloudStorageConfig stores the configuration of a GCS proxy backend.
type GoogleCloudStorageConfig struct {
	Bucket                string `yaml:"bucket"`
	UseDefaultCredentials bool   `yaml:"use_default_credentials"`
	JSONCredentialsFile   string `yaml:"json_credentials_file"`
}

// HTTPBackendConfig stores the configuration for a HTTP proxy backend.
type HTTPBackendConfig struct {
	BaseURL string `yaml:"url"`
}

// Config holds the top-level configuration for bazel-remote.
type Config struct {
	Host                    string                    `yaml:"host"`
	Port                    int                       `yaml:"port"`
	GRPCPort                int                       `yaml:"grpc_port"`
	ProfileHost             string                    `yaml:"profile_host"`
	ProfilePort             int                       `yaml:"profile_port"`
	Dir                     string                    `yaml:"dir"`
	MaxSize                 int                       `yaml:"max_size"`
	HtpasswdFile            string                    `yaml:"htpasswd_file"`
	TLSCertFile             string                    `yaml:"tls_cert_file"`
	TLSKeyFile              string                    `yaml:"tls_key_file"`
	S3CloudStorage          *S3CloudStorageConfig     `yaml:"s3_proxy"`
	GoogleCloudStorage      *GoogleCloudStorageConfig `yaml:"gcs_proxy"`
	HTTPBackend             *HTTPBackendConfig        `yaml:"http_proxy"`
	IdleTimeout             time.Duration             `yaml:"idle_timeout"`
	DisableHTTPACValidation bool                      `yaml:"disable_http_ac_validation"`
	DisableGRPCACDepsCheck  bool                      `yaml:"disable_grpc_ac_deps_check"`
}

// New ...
func New(dir string, maxSize int, host string, port int, grpcPort int,
	profileHost string, profilePort int, htpasswdFile string,
	tlsCertFile string, tlsKeyFile string, idleTimeout time.Duration,
	s3 *S3CloudStorageConfig, disableHTTPACValidation bool,
	disableGRPCACDepsCheck bool) (*Config, error) {
	c := Config{
		Host:                    host,
		Port:                    port,
		GRPCPort:                grpcPort,
		ProfileHost:             profileHost,
		ProfilePort:             profilePort,
		Dir:                     dir,
		MaxSize:                 maxSize,
		HtpasswdFile:            htpasswdFile,
		TLSCertFile:             tlsCertFile,
		TLSKeyFile:              tlsKeyFile,
		S3CloudStorage:          s3,
		GoogleCloudStorage:      nil,
		HTTPBackend:             nil,
		IdleTimeout:             idleTimeout,
		DisableHTTPACValidation: disableHTTPACValidation,
		DisableGRPCACDepsCheck:  disableGRPCACDepsCheck,
	}

	err := validateConfig(&c)
	if err != nil {
		return nil, err
	}
	return &c, nil
}

// NewFromYamlFile ...
func NewFromYamlFile(path string) (*Config, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("Failed to open config file '%s': %v", path, err)
	}
	defer file.Close()

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, fmt.Errorf("Failed to read config file '%s': %v", path, err)
	}

	return newFromYaml(data)
}

func newFromYaml(data []byte) (*Config, error) {
	c := Config{}
	err := yaml.Unmarshal(data, &c)
	if err != nil {
		return nil, fmt.Errorf("Failed to parse YAML config: %v", err)
	}

	err = validateConfig(&c)
	if err != nil {
		return nil, err
	}
	return &c, nil
}

func validateConfig(c *Config) error {
	if c.Dir == "" {
		return errors.New("The 'dir' flag/key is required")
	}

	if c.MaxSize <= 0 {
		return errors.New("The 'max_size' flag/key must be set to a value > 0")
	}

	if c.Port == 0 {
		return errors.New("A valid 'port' flag/key must be specified")
	}

	if c.GRPCPort < 0 {
		return errors.New("The 'grpc_port' flag/key must be 0 (disabled) or a positive integer")
	}

	if (c.TLSCertFile != "" && c.TLSKeyFile == "") || (c.TLSCertFile == "" && c.TLSKeyFile != "") {
		return errors.New("When enabling TLS one must specify both " +
			"'tls_key_file' and 'tls_cert_file'")
	}

	if c.GoogleCloudStorage != nil && c.HTTPBackend != nil && c.S3CloudStorage != nil {
		return errors.New("One can specify at most one proxying backend")
	}

	if c.GoogleCloudStorage != nil {
		if c.GoogleCloudStorage.Bucket == "" {
			return errors.New("The 'bucket' field is required for 'gcs_proxy'")
		}
	}

	if c.HTTPBackend != nil {
		if c.HTTPBackend.BaseURL == "" {
			return errors.New("The 'url' field is required for 'http_proxy'")
		}
	}

	if c.S3CloudStorage != nil {
		if c.S3CloudStorage.AccessKeyID != "" && c.S3CloudStorage.IAMRoleEndpoint != "" {
			return errors.New("Expected either 's3.access_key_id' or 's3.iam_role_endpoint', found both")
		}
	}

	return nil
}
<commit_msg>config: add non-stub comments for New and NewFromYamlFile<commit_after>package config

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	yaml "gopkg.in/yaml.v2"
)

// S3CloudStorageConfig stores the configuration of an S3 API proxy backend.
type S3CloudStorageConfig struct {
	Endpoint        string `yaml:"endpoint"`
	Bucket          string `yaml:"bucket"`
	Prefix          string `yaml:"prefix"`
	AccessKeyID     string `yaml:"access_key_id"`
	SecretAccessKey string `yaml:"secret_access_key"`
	DisableSSL      bool   `yaml:"disable_ssl"`
	IAMRoleEndpoint string `yaml:"iam_role_endpoint"`
	Region          string `yaml:"region"`
}

// GoogleCloudStorageConfig stores the configuration of a GCS proxy backend.
type GoogleCloudStorageConfig struct {
	Bucket                string `yaml:"bucket"`
	UseDefaultCredentials bool   `yaml:"use_default_credentials"`
	JSONCredentialsFile   string `yaml:"json_credentials_file"`
}

// HTTPBackendConfig stores the configuration for a HTTP proxy backend.
type HTTPBackendConfig struct {
	BaseURL string `yaml:"url"`
}

// Config holds the top-level configuration for bazel-remote.
type Config struct {
	Host                    string                    `yaml:"host"`
	Port                    int                       `yaml:"port"`
	GRPCPort                int                       `yaml:"grpc_port"`
	ProfileHost             string                    `yaml:"profile_host"`
	ProfilePort             int                       `yaml:"profile_port"`
	Dir                     string                    `yaml:"dir"`
	MaxSize                 int                       `yaml:"max_size"`
	HtpasswdFile            string                    `yaml:"htpasswd_file"`
	TLSCertFile             string                    `yaml:"tls_cert_file"`
	TLSKeyFile              string                    `yaml:"tls_key_file"`
	S3CloudStorage          *S3CloudStorageConfig     `yaml:"s3_proxy"`
	GoogleCloudStorage      *GoogleCloudStorageConfig `yaml:"gcs_proxy"`
	HTTPBackend             *HTTPBackendConfig        `yaml:"http_proxy"`
	IdleTimeout             time.Duration             `yaml:"idle_timeout"`
	DisableHTTPACValidation bool                      `yaml:"disable_http_ac_validation"`
	DisableGRPCACDepsCheck  bool                      `yaml:"disable_grpc_ac_deps_check"`
}

// New returns a validated Config with the specified values, and an error
// if there were any problems with the validation.
func New(dir string, maxSize int, host string, port int, grpcPort int,
	profileHost string, profilePort int, htpasswdFile string,
	tlsCertFile string, tlsKeyFile string, idleTimeout time.Duration,
	s3 *S3CloudStorageConfig, disableHTTPACValidation bool,
	disableGRPCACDepsCheck bool) (*Config, error) {
	c := Config{
		Host:                    host,
		Port:                    port,
		GRPCPort:                grpcPort,
		ProfileHost:             profileHost,
		ProfilePort:             profilePort,
		Dir:                     dir,
		MaxSize:                 maxSize,
		HtpasswdFile:            htpasswdFile,
		TLSCertFile:             tlsCertFile,
		TLSKeyFile:              tlsKeyFile,
		S3CloudStorage:          s3,
		GoogleCloudStorage:      nil,
		HTTPBackend:             nil,
		IdleTimeout:             idleTimeout,
		DisableHTTPACValidation: disableHTTPACValidation,
		DisableGRPCACDepsCheck:  disableGRPCACDepsCheck,
	}

	err := validateConfig(&c)
	if err != nil {
		return nil, err
	}
	return &c, nil
}

// NewFromYamlFile reads configuration settings from a YAML file then returns
// a validated Config with those settings, and an error if there were any
// problems.
func NewFromYamlFile(path string) (*Config, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("Failed to open config file '%s': %v", path, err)
	}
	defer file.Close()

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, fmt.Errorf("Failed to read config file '%s': %v", path, err)
	}

	return newFromYaml(data)
}

func newFromYaml(data []byte) (*Config, error) {
	c := Config{}
	err := yaml.Unmarshal(data, &c)
	if err != nil {
		return nil, fmt.Errorf("Failed to parse YAML config: %v", err)
	}

	err = validateConfig(&c)
	if err != nil {
		return nil, err
	}
	return &c, nil
}

func validateConfig(c *Config) error {
	if c.Dir == "" {
		return errors.New("The 'dir' flag/key is required")
	}

	if c.MaxSize <= 0 {
		return errors.New("The 'max_size' flag/key must be set to a value > 0")
	}

	if c.Port == 0 {
		return errors.New("A valid 'port' flag/key must be specified")
	}

	if c.GRPCPort < 0 {
		return errors.New("The 'grpc_port' flag/key must be 0 (disabled) or a positive integer")
	}

	if (c.TLSCertFile != "" && c.TLSKeyFile == "") || (c.TLSCertFile == "" && c.TLSKeyFile != "") {
		return errors.New("When enabling TLS one must specify both " +
			"'tls_key_file' and 'tls_cert_file'")
	}

	if c.GoogleCloudStorage != nil && c.HTTPBackend != nil && c.S3CloudStorage != nil {
		return errors.New("One can specify at most one proxying backend")
	}

	if c.GoogleCloudStorage != nil {
		if c.GoogleCloudStorage.Bucket == "" {
			return errors.New("The 'bucket' field is required for 'gcs_proxy'")
		}
	}

	if c.HTTPBackend != nil {
		if c.HTTPBackend.BaseURL == "" {
			return errors.New("The 'url' field is required for 'http_proxy'")
		}
	}

	if c.S3CloudStorage != nil {
		if c.S3CloudStorage.AccessKeyID != "" && c.S3CloudStorage.IAMRoleEndpoint != "" {
			return errors.New("Expected either 's3.access_key_id' or 's3.iam_role_endpoint', found both")
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package inbloom

import (
	"errors"
	"fmt"
	"hash"
	"hash/fnv"
	"math"
)

//ProbabilisticSet represents an abstraction of a Probabilistic
type ProbabilisticSet interface {
	Add(obj *[]byte) error
	Test(obj *[]byte) (bool, error)
}

//BloomFilter is a space-efficient probabilistic data structure, conceived by Burton Howard Bloom in 1970, that is used to test whether an element is a member of a set.
type BloomFilter struct {
	vector         []byte
	baseHashFn     hash.Hash64
	numberOfHashes uint64
}

//Add an object to the set
func (filter BloomFilter) Add(obj *[]byte) error {

	hashValues, err := filter.getHashVector(obj)

	for i := 0; i < len(hashValues); i++ {
		hashVal := hashValues[i]
		filter.vector[hashVal] = 1
	}
	return err
}

func (filter *BloomFilter) getHashVector(obj *[]byte) ([]uint64, error) {

	defer filter.baseHashFn.Reset()

	if filter == nil {
		fmt.Println("filter is null")
	}

	if filter.baseHashFn == nil {
		fmt.Println("hash function is null")
	}
	_, e1 := filter.baseHashFn.Write(*obj)

	if e1 != nil {
		empty := make([]uint64, 0)
		return empty, errors.New("failed to add object to filter")
	}

	seed1 := filter.baseHashFn.Sum64()
	// simulate output of two hash functions that will serve as base hashes for simulating the output of n hash functions
	upperBits := seed1 >> 32 << 32
	lowerBits := seed1 >> 32 << 32
	hashValues := make([]uint64, filter.numberOfHashes)
	for i := uint64(0); i < filter.numberOfHashes; i++ {

		h := (upperBits + lowerBits*i) % filter.numberOfHashes
		hashValues[i] = h
	}

	return hashValues, nil
}

//Test if an element is in the set
func (filter BloomFilter) Test(obj *[]byte) (bool, error) {

	hashVales, e := filter.getHashVector(obj)

	for i := 0; i < len(hashVales); i++ {
		hashVal := hashVales[i]
		if filter.vector[hashVal] == 0 {
			return false, e
		}
	}

	return true, e
}

//NewFilter creates a new BloomFilter. p is the error rate
//and n is the estimated number of elements that will be handled by the filter
func NewFilter(p float64, n int64) ProbabilisticSet {

	/*
		Given:

		n: how many items you expect to have in your filter (e.g. 216,553)
		p: your acceptable false positive rate {0..1} (e.g. 0.01 → 1%)
		we want to calculate:

		m: the number of bits needed in the bloom filter
		k: the number of hash functions we should apply
		The formulas:

		m = -n*ln(p) / (ln(2)^2) the number of bits
		k = m/n * ln(2) the number of hash functions

	*/

	m := -float64(n) * math.Log(p) / math.Pow(math.Ln2, 2)
	k := uint64(m / float64(n) * math.Ln2)

	return BloomFilter{vector: make([]byte, int64(m), int64(m)),
		baseHashFn:     fnv.New64(),
		numberOfHashes: k}
}
<commit_msg>fixed bug in hashing function, and cleaned up some code leftovers<commit_after>package inbloom

import (
	"errors"
	"hash"
	"hash/fnv"
	"math"
)

//ProbabilisticSet represents an abstraction of a Probabilistic
type ProbabilisticSet interface {
	Add(obj *[]byte) error
	Test(obj *[]byte) (bool, error)
}

//BloomFilter is a space-efficient probabilistic data structure, conceived by Burton Howard Bloom in 1970, that is used to test whether an element is a member of a set.
type BloomFilter struct {
	vector         []byte
	baseHashFn     hash.Hash64
	numberOfHashes uint64
}

//Add an object to the set
func (filter BloomFilter) Add(obj *[]byte) error {

	hashValues, err := filter.getHashVector(obj)

	for i := 0; i < len(hashValues); i++ {
		hashVal := hashValues[i]
		filter.vector[hashVal] = 1
	}
	return err
}

func (filter *BloomFilter) getHashVector(obj *[]byte) ([]uint64, error) {

	defer filter.baseHashFn.Reset()

	_, e1 := filter.baseHashFn.Write(*obj)

	if e1 != nil {
		empty := make([]uint64, 0)
		return empty, errors.New("failed to add object to filter")
	}

	// simulate output of two hash functions that will serve as base hashes for simulating the output of n hash functions
	// based on https://www.eecs.harvard.edu/~michaelm/postscripts/rsa2008.pdf

	// instead of generating 2 hash values we optimize and generate only 1 and splitting it's value
	// into 2 distinct values simulating the values of two separate hash functions
	seed1 := filter.baseHashFn.Sum64()

	upperBits := seed1 >> 32 << 32
	lowerBits := seed1 >> 32 << 32
	hashValues := make([]uint64, filter.numberOfHashes)

	for i := uint64(0); i < filter.numberOfHashes; i++ {

		h := (upperBits + lowerBits*i + uint64(math.Pow(float64(i), 2))) % filter.numberOfHashes
		hashValues[i] = h
	}

	return hashValues, nil
}

//Test if an element is in the set
func (filter BloomFilter) Test(obj *[]byte) (bool, error) {

	hashVales, e := filter.getHashVector(obj)

	for i := 0; i < len(hashVales); i++ {
		hashVal := hashVales[i]
		if filter.vector[hashVal] == 0 {
			return false, e
		}
	}

	return true, e
}

//NewFilter creates a new BloomFilter. p is the error rate
//and n is the estimated number of elements that will be handled by the filter
func NewFilter(p float64, n int64) ProbabilisticSet {

	/*
		Given:

		n: how many items you expect to have in your filter (e.g. 216,553)
		p: your acceptable false positive rate {0..1} (e.g. 0.01 → 1%)
		we want to calculate:

		m: the number of bits needed in the bloom filter
		k: the number of hash functions we should apply
		The formulas:

		m = -n*ln(p) / (ln(2)^2) the number of bits
		k = m/n * ln(2) the number of hash functions

	*/

	m := -float64(n) * math.Log(p) / math.Pow(math.Ln2, 2)
	k := uint64(m / float64(n) * math.Ln2)

	return BloomFilter{vector: make([]byte, int64(m), int64(m)),
		baseHashFn:     fnv.New64(),
		numberOfHashes: k}
}
<|endoftext|>


<commit_before>// Copyright 2016-2019 Granitic. All rights reserved.
// Use of this source code is governed by an Apache 2.0 license that can be found in the LICENSE file at the root of this project.

/*
Package config provides functionality for working with configuration files and command line arguments to a Granitic application.

Grantic uses JSON files to store component definitions (declarations of, and relationships between, components to
run in the IoC container) and configuration (variables used by IoC components that may vary between environments and settings
for Grantic's built-in facilities). A definition of the use and syntax of these files are outside of the scope of a GoDoc page,
but are described in detail at https://granitic.io/ref/component-definition-files and https://granitic.io/ref/configuration-files

This package defines functionality for loading a JSON file (from a filesystem or via HTTP) and merging multiple files into
a single view. This is a key concept in Granitic.

Given a folder of configuration files called conf:
	conf/x.json
	conf/sub/a.json
	conf/sub/b.json

starting a Grantic application with:

	-c http://example.com/base.json,conf,http://example.com/myinstance.json

The following will take place. Firstly the files would be expanded into a flat list of paths/URIs

	http://example.com/base.json
	conf/sub/a.json
	conf/sub/b.json
	conf/x.json
	http://example.com/myinstance.json

The the files will be merged together from left, using the the first file as a base. In this example,  http://example.com/base.json
and conf/sub/a.json will be merged together, then result of that merge will be merged with conf/sub/b.json and so on.

For named fields (in a JSON object/map), the process of merging is fairly obvious. When merging files A and B, a field that
is defined in both files will have the value of the field used in file B in the merged output. For example,

	a.json

	{
		"database": {
			"host": "localhost",
			"port": 3306,
			"flags": ["a", "b", "c"]
		}
	}

and

	b.json

	{
		"database": {
			"host": "remotehost",
			"flags": ["d"]
		}
	}

woud merge to:

	{
		"database": {
			"host": "remotehost",
			"port": 3306,
			"flags": ["d"]
		}
	}

The merging of configuration files occurs exactly above, but when component definition files are merged, arrays are joined, not overwritten.
For example:

	{ "methods": ["GET"] }

merged with;

	{ "methods": ["POST"] }

would result in:

	{ "methods": ["GET", "POST"] }

Another core concept used by the types in this package is a config path. This is the absolute path to field in the
eventual merged configuration file with a dot-delimited notation. E.g "database.host".
*/
package config

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/graniticio/granitic/v2/logging"
	"reflect"
	"strings"
)

// JSONPathSeparator is the character used to delimit paths to config values.
const JSONPathSeparator string = "."

// Used by code that needs to know what type of JSON data structure resides at a particular path
// before operating on it.
const (
	Unset       = -2
	JSONUnknown = -1
	JSONString  = 1
	JSONArray   = 2
	JSONMap     = 3
	JSONBool    = 4
)

// MissingPathError indicates that the a problem was caused by there being no value at the supplied
// config path
type MissingPathError struct {
	message string
}

func (mp MissingPathError) Error() string {
	return mp.message
}

// A Accessor provides access to a merged view of configuration files during the initialisation and
// configuration of the Granitic IoC container.
type Accessor struct {
	// The merged JSON configuration in object form.
	JSONData map[string]interface{}

	// Logger used by Granitic framework components. Automatically injected.
	FrameworkLogger logging.Logger
}

// Flush removes internal references to the (potentially very large) merged JSON data so the associated
// memory can be recovered during the next garbage collection.
func (ac *Accessor) Flush() {
	ac.JSONData = nil
}

// PathExists check to see whether the supplied dot-delimited path exists in the configuration and points to a non-null JSON value.
func (ac *Accessor) PathExists(path string) bool {
	value := ac.Value(path)

	return value != nil
}

// Value returns the JSON value at the supplied path or nil if the path does not exist of points to a null JSON value.
func (ac *Accessor) Value(path string) interface{} {

	splitPath := strings.Split(path, JSONPathSeparator)

	return ac.configVal(splitPath, ac.JSONData)

}

// ObjectVal returns a map representing a JSON object or nil if the path does not exist of points to a null JSON value. An error
// is returned if the value cannot be interpreted as a JSON object.
func (ac *Accessor) ObjectVal(path string) (map[string]interface{}, error) {

	value := ac.Value(path)

	if value == nil {
		return nil, nil
	} else if v, found := value.(map[string]interface{}); found {
		return v, nil
	}

	return nil, fmt.Errorf("unable to convert the value at %s to a JSON map/object", path)

}

// StringVal returns the string value of the JSON string at the supplied path. Does not convert other types to
// a string, so will return an error if the value is not a JSON string.
func (ac *Accessor) StringVal(path string) (string, error) {

	v := ac.Value(path)

	if v == nil {
		return "", errors.New("No string value found at " + path)
	}

	s, found := v.(string)

	if found {
		return s, nil
	}

	return "", fmt.Errorf("Value at %s is %q and cannot be converted to a string", path, v)

}

// IntVal returns the int value of the JSON number at the supplied path. JSON numbers
// are internally represented by Go as a float64, so no error will be returned, but data might be lost
// if the JSON number does not actually represent an int. An error will be returned if the value is not a JSON number
// or cannot be converted to an int.
func (ac *Accessor) IntVal(path string) (int, error) {

	v := ac.Value(path)

	if v == nil {
		return 0, errors.New("No such path " + path)
	} else if f, found := v.(float64); found {
		return int(f), nil
	}

	return 0, fmt.Errorf("alue at %s is %q and cannot be converted to an int", path, v)

}

// Float64Val returns the float64 value of the JSON number at the supplied path. An error will be returned if the value is not a JSON number.
func (ac *Accessor) Float64Val(path string) (float64, error) {

	v := ac.Value(path)

	if v == nil {
		return 0, errors.New("No such path " + path)
	} else if f, found := v.(float64); found {
		return f, nil
	}

	return 0, fmt.Errorf("value at %s is %q and cannot be converted to a float64", path, v)
}

// Array returns the value of an array of JSON obects at the supplied path. Caution should be used when calling this method
// as behaviour is undefined for JSON arrays of JSON types other than object.
func (ac *Accessor) Array(path string) ([]interface{}, error) {

	value := ac.Value(path)

	if value == nil {
		return nil, nil
	} else if v, found := value.([]interface{}); found {
		return v, nil
	}

	return nil, fmt.Errorf("unable to convert the value at %s to a JSON array", path)

}

// BoolVal returns the bool value of the JSON bool at the supplied path. An error will be returned if the value is not a JSON bool.
// Note this method only suports the JSON definition of bools (true, false) not the Go definition (true, false, 1, 0 etc).
func (ac *Accessor) BoolVal(path string) (bool, error) {

	v := ac.Value(path)

	if v == nil {
		return false, errors.New("No such path " + path)
	}

	if b, found := v.(bool); found {
		return b, nil
	}

	return false, fmt.Errorf("Value at %s is %q and cannot be converted to a bool", path, v)

}

// JSONType determines the apparent JSONType of the supplied Go interface.
func JSONType(value interface{}) int {

	switch value.(type) {
	case string:
		return JSONString
	case map[string]interface{}:
		return JSONMap
	case bool:
		return JSONBool
	case []interface{}:
		return JSONArray
	default:
		return JSONUnknown
	}
}

func (ac *Accessor) configVal(path []string, jsonMap map[string]interface{}) interface{} {

	var result interface{}
	result = jsonMap[path[0]]

	if result == nil {
		return nil
	}

	if len(path) == 1 {
		return result
	}

	remainPath := path[1:len(path)]
	return ac.configVal(remainPath, result.(map[string]interface{}))
}

// SetField takes a target Go interface and uses the data a the supplied path to populated the named field on the
// target. The target must be a pointer to a struct. The field must be a string, bool, int, float63, string[interface{}] map
// or a slice of one of those types. An eror will be returned if the target field, is missing, not settable or incompatible
// with the JSON value at the supplied path.
func (ac *Accessor) SetField(fieldName string, path string, target interface{}) error {

	if !ac.PathExists(path) {
		return MissingPathError{message: "No value found at " + path}
	}

	targetReflect := reflect.ValueOf(target).Elem()
	targetField := targetReflect.FieldByName(fieldName)

	k := targetField.Type().Kind()

	switch k {
	case reflect.String:
		s, _ := ac.StringVal(path)
		targetField.SetString(s)
	case reflect.Bool:
		b, _ := ac.BoolVal(path)
		targetField.SetBool(b)
	case reflect.Int:
		i, _ := ac.IntVal(path)
		targetField.SetInt(int64(i))
	case reflect.Float64:
		f, _ := ac.Float64Val(path)
		targetField.SetFloat(f)
	case reflect.Map:

		if v, err := ac.ObjectVal(path); err == nil {
			if err = ac.populateMapField(targetField, v); err != nil {
				return err
			}
		} else {
			return err
		}
	case reflect.Slice:
		ac.populateSlice(targetField, path, target)

	default:
		m := fmt.Sprintf("Unable to use value at path %s as target field %s is not a suppported type (%s)", path, fieldName, k)
		ac.FrameworkLogger.LogErrorf(m)

		return errors.New(m)
	}

	return nil
}

func (ac *Accessor) populateSlice(targetField reflect.Value, path string, target interface{}) {

	v := ac.Value(path)

	data, _ := json.Marshal(v)

	vt := targetField.Type()
	nt := reflect.New(vt)

	jTarget := nt.Interface()
	json.Unmarshal(data, &jTarget)

	vr := reflect.ValueOf(jTarget)
	targetField.Set(vr.Elem())

}

func (ac *Accessor) populateMapField(targetField reflect.Value, contents map[string]interface{}) error {
	var err error

	m := reflect.MakeMap(targetField.Type())
	targetField.Set(m)

	for k, v := range contents {

		keyVal := reflect.ValueOf(k)
		vVal := reflect.ValueOf(v)

		if vVal.Kind() == reflect.Slice {
			vVal, err = ac.arrayVal(vVal)

			if err != nil {
				return err
			}
		}

		m.SetMapIndex(keyVal, vVal)

	}

	return nil
}

func (ac *Accessor) arrayVal(a reflect.Value) (reflect.Value, error) {

	v := a.Interface().([]interface{})
	l := len(v)

	if l == 0 {

		return reflect.Zero(reflect.TypeOf(ac)), errors.New("cannot use an empty array as a value in a Map")

	}

	var s reflect.Value

	switch t := v[0].(type) {
	case string:
		s = reflect.MakeSlice(reflect.TypeOf([]string{}), 0, 0)
	default:
		m := fmt.Sprintf("Cannot use an array of %T as a value in a Map.", t)
		return reflect.Zero(reflect.TypeOf(ac)), errors.New(m)
	}

	for _, elem := range v {

		s = reflect.Append(s, reflect.ValueOf(elem))

	}

	return s, nil
}

// Populate sets the fields on the supplied target object using the JSON data
// at the supplied path. This is achieved using Go's json.Marshal to convert the data
// back into text JSON and then json.Unmarshal to unmarshal back into the target.
func (ac *Accessor) Populate(path string, target interface{}) error {
	exists := ac.PathExists(path)

	if !exists {
		return errors.New("No such path: " + path)
	}

	//Already check if path exists
	object, _ := ac.ObjectVal(path)

	if data, err := json.Marshal(object); err != nil {
		m := fmt.Sprintf("%T cannot be marshalled to JSON", object)
		return errors.New(m)
	} else if json.Unmarshal(data, target); err != nil {
		return fmt.Errorf("%T cannot be populated with %v to JSON", object, data)
	}

	return nil

}
<commit_msg>Run gofmt.<commit_after>// Copyright 2016-2019 Granitic. All rights reserved.
// Use of this source code is governed by an Apache 2.0 license that can be found in the LICENSE file at the root of this project.

/*
Package config provides functionality for working with configuration files and command line arguments to a Granitic application.

Grantic uses JSON files to store component definitions (declarations of, and relationships between, components to
run in the IoC container) and configuration (variables used by IoC components that may vary between environments and settings
for Grantic's built-in facilities). A definition of the use and syntax of these files are outside of the scope of a GoDoc page,
but are described in detail at https://granitic.io/ref/component-definition-files and https://granitic.io/ref/configuration-files

This package defines functionality for loading a JSON file (from a filesystem or via HTTP) and merging multiple files into
a single view. This is a key concept in Granitic.

Given a folder of configuration files called conf:
	conf/x.json
	conf/sub/a.json
	conf/sub/b.json

starting a Grantic application with:

	-c http://example.com/base.json,conf,http://example.com/myinstance.json

The following will take place. Firstly the files would be expanded into a flat list of paths/URIs

	http://example.com/base.json
	conf/sub/a.json
	conf/sub/b.json
	conf/x.json
	http://example.com/myinstance.json

The the files will be merged together from left, using the the first file as a base. In this example,  http://example.com/base.json
and conf/sub/a.json will be merged together, then result of that merge will be merged with conf/sub/b.json and so on.

For named fields (in a JSON object/map), the process of merging is fairly obvious. When merging files A and B, a field that
is defined in both files will have the value of the field used in file B in the merged output. For example,

	a.json

	{
		"database": {
			"host": "localhost",
			"port": 3306,
			"flags": ["a", "b", "c"]
		}
	}

and

	b.json

	{
		"database": {
			"host": "remotehost",
			"flags": ["d"]
		}
	}

woud merge to:

	{
		"database": {
			"host": "remotehost",
			"port": 3306,
			"flags": ["d"]
		}
	}

The merging of configuration files occurs exactly above, but when component definition files are merged, arrays are joined, not overwritten.
For example:

	{ "methods": ["GET"] }

merged with;

	{ "methods": ["POST"] }

would result in:

	{ "methods": ["GET", "POST"] }

Another core concept used by the types in this package is a config path. This is the absolute path to field in the
eventual merged configuration file with a dot-delimited notation. E.g "database.host".
*/
package config

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/graniticio/granitic/v2/logging"
	"reflect"
	"strings"
)

// JSONPathSeparator is the character used to delimit paths to config values.
const JSONPathSeparator string = "."

// Used by code that needs to know what type of JSON data structure resides at a particular path
// before operating on it.
const (
	Unset       = -2
	JSONUnknown = -1
	JSONString  = 1
	JSONArray   = 2
	JSONMap     = 3
	JSONBool    = 4
)

// MissingPathError indicates that the a problem was caused by there being no value at the supplied
// config path
type MissingPathError struct {
	message string
}

func (mp MissingPathError) Error() string {
	return mp.message
}

// A Accessor provides access to a merged view of configuration files during the initialisation and
// configuration of the Granitic IoC container.
type Accessor struct {
	// The merged JSON configuration in object form.
	JSONData map[string]interface{}

	// Logger used by Granitic framework components. Automatically injected.
	FrameworkLogger logging.Logger
}

// Flush removes internal references to the (potentially very large) merged JSON data so the associated
// memory can be recovered during the next garbage collection.
func (ac *Accessor) Flush() {
	ac.JSONData = nil
}

// PathExists check to see whether the supplied dot-delimited path exists in the configuration and points to a non-null JSON value.
func (ac *Accessor) PathExists(path string) bool {
	value := ac.Value(path)

	return value != nil
}

// Value returns the JSON value at the supplied path or nil if the path does not exist of points to a null JSON value.
func (ac *Accessor) Value(path string) interface{} {

	splitPath := strings.Split(path, JSONPathSeparator)

	return ac.configVal(splitPath, ac.JSONData)

}

// ObjectVal returns a map representing a JSON object or nil if the path does not exist of points to a null JSON value. An error
// is returned if the value cannot be interpreted as a JSON object.
func (ac *Accessor) ObjectVal(path string) (map[string]interface{}, error) {

	value := ac.Value(path)

	if value == nil {
		return nil, nil
	} else if v, found := value.(map[string]interface{}); found {
		return v, nil
	}

	return nil, fmt.Errorf("unable to convert the value at %s to a JSON map/object", path)

}

// StringVal returns the string value of the JSON string at the supplied path. Does not convert other types to
// a string, so will return an error if the value is not a JSON string.
func (ac *Accessor) StringVal(path string) (string, error) {

	v := ac.Value(path)

	if v == nil {
		return "", errors.New("No string value found at " + path)
	}

	s, found := v.(string)

	if found {
		return s, nil
	}

	return "", fmt.Errorf("Value at %s is %q and cannot be converted to a string", path, v)

}

// IntVal returns the int value of the JSON number at the supplied path. JSON numbers
// are internally represented by Go as a float64, so no error will be returned, but data might be lost
// if the JSON number does not actually represent an int. An error will be returned if the value is not a JSON number
// or cannot be converted to an int.
func (ac *Accessor) IntVal(path string) (int, error) {

	v := ac.Value(path)

	if v == nil {
		return 0, errors.New("No such path " + path)
	} else if f, found := v.(float64); found {
		return int(f), nil
	}

	return 0, fmt.Errorf("alue at %s is %q and cannot be converted to an int", path, v)

}

// Float64Val returns the float64 value of the JSON number at the supplied path. An error will be returned if the value is not a JSON number.
func (ac *Accessor) Float64Val(path string) (float64, error) {

	v := ac.Value(path)

	if v == nil {
		return 0, errors.New("No such path " + path)
	} else if f, found := v.(float64); found {
		return f, nil
	}

	return 0, fmt.Errorf("value at %s is %q and cannot be converted to a float64", path, v)
}

// Array returns the value of an array of JSON obects at the supplied path. Caution should be used when calling this method
// as behaviour is undefined for JSON arrays of JSON types other than object.
func (ac *Accessor) Array(path string) ([]interface{}, error) {

	value := ac.Value(path)

	if value == nil {
		return nil, nil
	} else if v, found := value.([]interface{}); found {
		return v, nil
	}

	return nil, fmt.Errorf("unable to convert the value at %s to a JSON array", path)

}

// BoolVal returns the bool value of the JSON bool at the supplied path. An error will be returned if the value is not a JSON bool.
// Note this method only suports the JSON definition of bools (true, false) not the Go definition (true, false, 1, 0 etc).
func (ac *Accessor) BoolVal(path string) (bool, error) {

	v := ac.Value(path)

	if v == nil {
		return false, errors.New("No such path " + path)
	}

	if b, found := v.(bool); found {
		return b, nil
	}

	return false, fmt.Errorf("Value at %s is %q and cannot be converted to a bool", path, v)

}

// JSONType determines the apparent JSONType of the supplied Go interface.
func JSONType(value interface{}) int {

	switch value.(type) {
	case string:
		return JSONString
	case map[string]interface{}:
		return JSONMap
	case bool:
		return JSONBool
	case []interface{}:
		return JSONArray
	default:
		return JSONUnknown
	}
}

func (ac *Accessor) configVal(path []string, jsonMap map[string]interface{}) interface{} {

	var result interface{}
	result = jsonMap[path[0]]

	if result == nil {
		return nil
	}

	if len(path) == 1 {
		return result
	}

	remainPath := path[1:]
	return ac.configVal(remainPath, result.(map[string]interface{}))
}

// SetField takes a target Go interface and uses the data a the supplied path to populated the named field on the
// target. The target must be a pointer to a struct. The field must be a string, bool, int, float63, string[interface{}] map
// or a slice of one of those types. An eror will be returned if the target field, is missing, not settable or incompatible
// with the JSON value at the supplied path.
func (ac *Accessor) SetField(fieldName string, path string, target interface{}) error {

	if !ac.PathExists(path) {
		return MissingPathError{message: "No value found at " + path}
	}

	targetReflect := reflect.ValueOf(target).Elem()
	targetField := targetReflect.FieldByName(fieldName)

	k := targetField.Type().Kind()

	switch k {
	case reflect.String:
		s, _ := ac.StringVal(path)
		targetField.SetString(s)
	case reflect.Bool:
		b, _ := ac.BoolVal(path)
		targetField.SetBool(b)
	case reflect.Int:
		i, _ := ac.IntVal(path)
		targetField.SetInt(int64(i))
	case reflect.Float64:
		f, _ := ac.Float64Val(path)
		targetField.SetFloat(f)
	case reflect.Map:

		if v, err := ac.ObjectVal(path); err == nil {
			if err = ac.populateMapField(targetField, v); err != nil {
				return err
			}
		} else {
			return err
		}
	case reflect.Slice:
		ac.populateSlice(targetField, path, target)

	default:
		m := fmt.Sprintf("Unable to use value at path %s as target field %s is not a suppported type (%s)", path, fieldName, k)
		ac.FrameworkLogger.LogErrorf(m)

		return errors.New(m)
	}

	return nil
}

func (ac *Accessor) populateSlice(targetField reflect.Value, path string, target interface{}) {

	v := ac.Value(path)

	data, _ := json.Marshal(v)

	vt := targetField.Type()
	nt := reflect.New(vt)

	jTarget := nt.Interface()
	json.Unmarshal(data, &jTarget)

	vr := reflect.ValueOf(jTarget)
	targetField.Set(vr.Elem())

}

func (ac *Accessor) populateMapField(targetField reflect.Value, contents map[string]interface{}) error {
	var err error

	m := reflect.MakeMap(targetField.Type())
	targetField.Set(m)

	for k, v := range contents {

		keyVal := reflect.ValueOf(k)
		vVal := reflect.ValueOf(v)

		if vVal.Kind() == reflect.Slice {
			vVal, err = ac.arrayVal(vVal)

			if err != nil {
				return err
			}
		}

		m.SetMapIndex(keyVal, vVal)

	}

	return nil
}

func (ac *Accessor) arrayVal(a reflect.Value) (reflect.Value, error) {

	v := a.Interface().([]interface{})
	l := len(v)

	if l == 0 {

		return reflect.Zero(reflect.TypeOf(ac)), errors.New("cannot use an empty array as a value in a Map")

	}

	var s reflect.Value

	switch t := v[0].(type) {
	case string:
		s = reflect.MakeSlice(reflect.TypeOf([]string{}), 0, 0)
	default:
		m := fmt.Sprintf("Cannot use an array of %T as a value in a Map.", t)
		return reflect.Zero(reflect.TypeOf(ac)), errors.New(m)
	}

	for _, elem := range v {

		s = reflect.Append(s, reflect.ValueOf(elem))

	}

	return s, nil
}

// Populate sets the fields on the supplied target object using the JSON data
// at the supplied path. This is achieved using Go's json.Marshal to convert the data
// back into text JSON and then json.Unmarshal to unmarshal back into the target.
func (ac *Accessor) Populate(path string, target interface{}) error {
	exists := ac.PathExists(path)

	if !exists {
		return errors.New("No such path: " + path)
	}

	//Already check if path exists
	object, _ := ac.ObjectVal(path)

	if data, err := json.Marshal(object); err != nil {
		m := fmt.Sprintf("%T cannot be marshalled to JSON", object)
		return errors.New(m)
	} else if json.Unmarshal(data, target); err != nil {
		return fmt.Errorf("%T cannot be populated with %v to JSON", object, data)
	}

	return nil

}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"io/ioutil"
	"log"
	"os/user"

	yaml "gopkg.in/yaml.v2"

	"github.com/spf13/viper"
)

// ClusterName is the name of an ECS Cluster (example: "mountain")
type ClusterName string

// FilePath is the path to a yaml file on disk
type FilePath string

// Keys is a map of cluster names
type Keys map[ClusterName]FilePath

// Config represents global application configuration
type Config struct {
	Keys Keys `yaml:"keys"`
}

// Storage is a mechanism for storing ECS Commander Config!
type Storage interface {
	ReadKeys() (Keys, error)
	SaveKeys(Keys) error
	IsModified() bool
}

// ReadKeys returns you the existing keys if they exist,
// if not, will return a new, blank, Keys struct
func ReadKeys(adapter Storage) (Keys, error) {
	return adapter.ReadKeys()
}

// YAMLFile stores config in a YAML file on disk,
// this is the default.  It implements the storage
// adapter interface.
type YAMLFile struct {
	path     string
	content  []byte
	modified bool
}

// GetYAMLConfig gets, or creates, a yaml configuration
// file from disk
func GetYAMLConfig() *YAMLFile {
	file := viper.ConfigFileUsed()
	configFile := NewYAMLFile()
	if file != "" {
		configFile = ReadYAMLFile(file)
	}
	return configFile
}

// NewYAMLFile creates a new and empty YAML file
func NewYAMLFile() *YAMLFile {
	usr, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	return &YAMLFile{path: fmt.Sprintf("%s/.ecs-commander.yaml", usr.HomeDir)}
}

// ReadYAMLFile Sets the file to be used for storing config
func ReadYAMLFile(path string) *YAMLFile {
	content, err := ioutil.ReadFile(path)
	if err != nil {
		panic(fmt.Errorf("Tried to read config file from %s, but failed. %v", content, err))
	}
	yamlFile := &YAMLFile{path: path, content: content}
	return yamlFile
}

// ReadKeys implements a key reader for yaml files
func (yamlFile *YAMLFile) ReadKeys() (Keys, error) {
	config := &Config{}
	err := yaml.Unmarshal(yamlFile.content, &config)
	if err != nil {
		return nil, err
	}
	return config.Keys, nil
}

// SaveKeys implements a key writer for yaml files
func (yamlFile *YAMLFile) SaveKeys(keys Keys) error {
	config := &Config{}
	err := yaml.Unmarshal(yamlFile.content, &config)
	if err != nil {
		return err
	}
	config.Keys = keys
	bytes, err := yaml.Marshal(config)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(yamlFile.path, bytes, 0644)
	if err != nil {
		return err
	}
	return nil
}

// IsModified implements a modified
func (yamlFile *YAMLFile) IsModified() bool {
	return true
}

// GetClusterKey returns the registered key path for a cluster
func GetClusterKey(cluster string) string {
	out := fmt.Sprintf("~/%s.pem", cluster)
	allKeys, err := GetYAMLConfig().ReadKeys()
	if err != nil {
		return out
	}
	if key, ok := allKeys[ClusterName(cluster)]; ok {
		out = string(key)
	}
	return out
}
<commit_msg>Fix bug with empty config<commit_after>package config

import (
	"fmt"
	"io/ioutil"
	"log"
	"os/user"

	yaml "gopkg.in/yaml.v2"

	"github.com/spf13/viper"
)

// ClusterName is the name of an ECS Cluster (example: "mountain")
type ClusterName string

// FilePath is the path to a yaml file on disk
type FilePath string

// Keys is a map of cluster names
type Keys map[ClusterName]FilePath

// Config represents global application configuration
type Config struct {
	Keys Keys `yaml:"keys"`
}

// Storage is a mechanism for storing ECS Commander Config!
type Storage interface {
	ReadKeys() (Keys, error)
	SaveKeys(Keys) error
	IsModified() bool
}

// ReadKeys returns you the existing keys if they exist,
// if not, will return a new, blank, Keys struct
func ReadKeys(adapter Storage) (Keys, error) {
	return adapter.ReadKeys()
}

// YAMLFile stores config in a YAML file on disk,
// this is the default.  It implements the storage
// adapter interface.
type YAMLFile struct {
	path     string
	content  []byte
	modified bool
}

// GetYAMLConfig gets, or creates, a yaml configuration
// file from disk
func GetYAMLConfig() *YAMLFile {
	file := viper.ConfigFileUsed()
	configFile := NewYAMLFile()
	if file != "" {
		configFile = ReadYAMLFile(file)
	}
	return configFile
}

// NewYAMLFile creates a new and empty YAML file
func NewYAMLFile() *YAMLFile {
	usr, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	return &YAMLFile{path: fmt.Sprintf("%s/.ecs-commander.yaml", usr.HomeDir)}
}

// ReadYAMLFile Sets the file to be used for storing config
func ReadYAMLFile(path string) *YAMLFile {
	content, err := ioutil.ReadFile(path)
	if err != nil {
		panic(fmt.Errorf("Tried to read config file from %s, but failed. %v", content, err))
	}
	yamlFile := &YAMLFile{path: path, content: content}
	return yamlFile
}

// ReadKeys implements a key reader for yaml files
func (yamlFile *YAMLFile) ReadKeys() (Keys, error) {
	config := &Config{}
	err := yaml.Unmarshal(yamlFile.content, &config)
	if err != nil {
		return nil, err
	}
	if config.Keys == nil {
		config.Keys = make(Keys)
	}
	return config.Keys, nil
}

// SaveKeys implements a key writer for yaml files
func (yamlFile *YAMLFile) SaveKeys(keys Keys) error {
	config := &Config{}
	err := yaml.Unmarshal(yamlFile.content, &config)
	if err != nil {
		return err
	}
	config.Keys = keys
	bytes, err := yaml.Marshal(config)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(yamlFile.path, bytes, 0644)
	if err != nil {
		return err
	}
	return nil
}

// IsModified implements a modified
func (yamlFile *YAMLFile) IsModified() bool {
	return true
}

// GetClusterKey returns the registered key path for a cluster
func GetClusterKey(cluster string) string {
	out := fmt.Sprintf("~/%s.pem", cluster)
	allKeys, err := GetYAMLConfig().ReadKeys()
	if err != nil {
		return out
	}
	if key, ok := allKeys[ClusterName(cluster)]; ok {
		out = string(key)
	}
	return out
}
<|endoftext|>