<commit_before>package config

import (
	"github.com/BurntSushi/toml"
	"github.com/sirupsen/logrus"
)

var Logger *logrus.Entry

type Config struct {
	Slack         Settings
	Mattermost    Settings
	Debug         bool
	Bind          string
	BindInterface string
	BindPort      int
	TLSBind       string
	TLSDir        string
	Insecure      bool
	SkipTLSVerify bool
	DefaultServer string
	DefaultTeam   string
	Restrict      string
	JoinExclude   []string
	JoinInclude   []string
	PartFake      bool
}

type Settings struct {
	Restrict []string
}

func LoadConfig(cfgfile string, defaultcfg Config) *Config {
	if _, err := toml.DecodeFile(cfgfile, &defaultcfg); err != nil {
		Logger.Fatalf("Error loading config file %s: %s", cfgfile, err)
	}
	Logger.Infof("Loaded config from %s", cfgfile)
	return &defaultcfg
}
<commit_msg>Add UseDisplayName config setting (slack)<commit_after>package config

import (
	"github.com/BurntSushi/toml"
	"github.com/sirupsen/logrus"
)

var Logger *logrus.Entry

type Config struct {
	Slack         Settings
	Mattermost    Settings
	Debug         bool
	Bind          string
	BindInterface string
	BindPort      int
	TLSBind       string
	TLSDir        string
	Insecure      bool
	SkipTLSVerify bool
	DefaultServer string
	DefaultTeam   string
	Restrict      string
	JoinExclude   []string
	JoinInclude   []string
	PartFake      bool
}

type Settings struct {
	Restrict       []string
	UseDisplayName bool
}

func LoadConfig(cfgfile string, defaultcfg Config) *Config {
	if _, err := toml.DecodeFile(cfgfile, &defaultcfg); err != nil {
		Logger.Fatalf("Error loading config file %s: %s", cfgfile, err)
	}
	Logger.Infof("Loaded config from %s", cfgfile)
	return &defaultcfg
}
<|endoftext|>


<commit_before>//  Copyright (c) 2015 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	// token maps
	_ "github.com/blevesearch/bleve/analysis/tokenmap"

	// fragment formatters
	_ "github.com/blevesearch/bleve/search/highlight/format/ansi"
	_ "github.com/blevesearch/bleve/search/highlight/format/html"

	// fragmenters
	_ "github.com/blevesearch/bleve/search/highlight/fragmenter/simple"

	// highlighters
	_ "github.com/blevesearch/bleve/search/highlight/highlighter/ansi"
	_ "github.com/blevesearch/bleve/search/highlight/highlighter/html"
	_ "github.com/blevesearch/bleve/search/highlight/highlighter/simple"

	// char filters
	_ "github.com/blevesearch/bleve/analysis/char/asciifolding"
	_ "github.com/blevesearch/bleve/analysis/char/html"
	_ "github.com/blevesearch/bleve/analysis/char/regexp"
	_ "github.com/blevesearch/bleve/analysis/char/zerowidthnonjoiner"

	// analyzers
	_ "github.com/blevesearch/bleve/analysis/analyzer/custom"
	_ "github.com/blevesearch/bleve/analysis/analyzer/keyword"
	_ "github.com/blevesearch/bleve/analysis/analyzer/simple"
	_ "github.com/blevesearch/bleve/analysis/analyzer/standard"
	_ "github.com/blevesearch/bleve/analysis/analyzer/web"

	// token filters
	_ "github.com/blevesearch/bleve/analysis/token/apostrophe"
	_ "github.com/blevesearch/bleve/analysis/token/compound"
	_ "github.com/blevesearch/bleve/analysis/token/edgengram"
	_ "github.com/blevesearch/bleve/analysis/token/elision"
	_ "github.com/blevesearch/bleve/analysis/token/keyword"
	_ "github.com/blevesearch/bleve/analysis/token/length"
	_ "github.com/blevesearch/bleve/analysis/token/lowercase"
	_ "github.com/blevesearch/bleve/analysis/token/ngram"
	_ "github.com/blevesearch/bleve/analysis/token/reverse"
	_ "github.com/blevesearch/bleve/analysis/token/shingle"
	_ "github.com/blevesearch/bleve/analysis/token/stop"
	_ "github.com/blevesearch/bleve/analysis/token/truncate"
	_ "github.com/blevesearch/bleve/analysis/token/unicodenorm"

	// tokenizers
	_ "github.com/blevesearch/bleve/analysis/tokenizer/exception"
	_ "github.com/blevesearch/bleve/analysis/tokenizer/regexp"
	_ "github.com/blevesearch/bleve/analysis/tokenizer/single"
	_ "github.com/blevesearch/bleve/analysis/tokenizer/unicode"
	_ "github.com/blevesearch/bleve/analysis/tokenizer/web"
	_ "github.com/blevesearch/bleve/analysis/tokenizer/whitespace"

	// date time parsers
	_ "github.com/blevesearch/bleve/analysis/datetime/flexible"
	_ "github.com/blevesearch/bleve/analysis/datetime/optional"

	// languages
	_ "github.com/blevesearch/bleve/analysis/lang/ar"
	_ "github.com/blevesearch/bleve/analysis/lang/bg"
	_ "github.com/blevesearch/bleve/analysis/lang/ca"
	_ "github.com/blevesearch/bleve/analysis/lang/cjk"
	_ "github.com/blevesearch/bleve/analysis/lang/ckb"
	_ "github.com/blevesearch/bleve/analysis/lang/cs"
	_ "github.com/blevesearch/bleve/analysis/lang/da"
	_ "github.com/blevesearch/bleve/analysis/lang/de"
	_ "github.com/blevesearch/bleve/analysis/lang/el"
	_ "github.com/blevesearch/bleve/analysis/lang/en"
	_ "github.com/blevesearch/bleve/analysis/lang/es"
	_ "github.com/blevesearch/bleve/analysis/lang/eu"
	_ "github.com/blevesearch/bleve/analysis/lang/fa"
	_ "github.com/blevesearch/bleve/analysis/lang/fi"
	_ "github.com/blevesearch/bleve/analysis/lang/fr"
	_ "github.com/blevesearch/bleve/analysis/lang/ga"
	_ "github.com/blevesearch/bleve/analysis/lang/gl"
	_ "github.com/blevesearch/bleve/analysis/lang/hi"
	_ "github.com/blevesearch/bleve/analysis/lang/hu"
	_ "github.com/blevesearch/bleve/analysis/lang/hy"
	_ "github.com/blevesearch/bleve/analysis/lang/id"
	_ "github.com/blevesearch/bleve/analysis/lang/in"
	_ "github.com/blevesearch/bleve/analysis/lang/it"
	_ "github.com/blevesearch/bleve/analysis/lang/nl"
	_ "github.com/blevesearch/bleve/analysis/lang/no"
	_ "github.com/blevesearch/bleve/analysis/lang/pt"
	_ "github.com/blevesearch/bleve/analysis/lang/ro"
	_ "github.com/blevesearch/bleve/analysis/lang/ru"
	_ "github.com/blevesearch/bleve/analysis/lang/sv"
	_ "github.com/blevesearch/bleve/analysis/lang/tr"

	// kv stores
	_ "github.com/blevesearch/bleve/index/store/boltdb"
	_ "github.com/blevesearch/bleve/index/store/goleveldb"
	_ "github.com/blevesearch/bleve/index/store/gtreap"
	_ "github.com/blevesearch/bleve/index/store/moss"

	// index types
	_ "github.com/blevesearch/bleve/index/upsidedown"
)
<commit_msg>Include missing token filters<commit_after>//  Copyright (c) 2015 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	// token maps
	_ "github.com/blevesearch/bleve/analysis/tokenmap"

	// fragment formatters
	_ "github.com/blevesearch/bleve/search/highlight/format/ansi"
	_ "github.com/blevesearch/bleve/search/highlight/format/html"

	// fragmenters
	_ "github.com/blevesearch/bleve/search/highlight/fragmenter/simple"

	// highlighters
	_ "github.com/blevesearch/bleve/search/highlight/highlighter/ansi"
	_ "github.com/blevesearch/bleve/search/highlight/highlighter/html"
	_ "github.com/blevesearch/bleve/search/highlight/highlighter/simple"

	// char filters
	_ "github.com/blevesearch/bleve/analysis/char/asciifolding"
	_ "github.com/blevesearch/bleve/analysis/char/html"
	_ "github.com/blevesearch/bleve/analysis/char/regexp"
	_ "github.com/blevesearch/bleve/analysis/char/zerowidthnonjoiner"

	// analyzers
	_ "github.com/blevesearch/bleve/analysis/analyzer/custom"
	_ "github.com/blevesearch/bleve/analysis/analyzer/keyword"
	_ "github.com/blevesearch/bleve/analysis/analyzer/simple"
	_ "github.com/blevesearch/bleve/analysis/analyzer/standard"
	_ "github.com/blevesearch/bleve/analysis/analyzer/web"

	// token filters
	_ "github.com/blevesearch/bleve/analysis/token/apostrophe"
	_ "github.com/blevesearch/bleve/analysis/token/camelcase"
	_ "github.com/blevesearch/bleve/analysis/token/compound"
	_ "github.com/blevesearch/bleve/analysis/token/edgengram"
	_ "github.com/blevesearch/bleve/analysis/token/elision"
	_ "github.com/blevesearch/bleve/analysis/token/keyword"
	_ "github.com/blevesearch/bleve/analysis/token/length"
	_ "github.com/blevesearch/bleve/analysis/token/lowercase"
	_ "github.com/blevesearch/bleve/analysis/token/ngram"
	_ "github.com/blevesearch/bleve/analysis/token/reverse"
	_ "github.com/blevesearch/bleve/analysis/token/shingle"
	_ "github.com/blevesearch/bleve/analysis/token/stop"
	_ "github.com/blevesearch/bleve/analysis/token/truncate"
	_ "github.com/blevesearch/bleve/analysis/token/unicodenorm"
	_ "github.com/blevesearch/bleve/analysis/token/unique"

	// tokenizers
	_ "github.com/blevesearch/bleve/analysis/tokenizer/exception"
	_ "github.com/blevesearch/bleve/analysis/tokenizer/regexp"
	_ "github.com/blevesearch/bleve/analysis/tokenizer/single"
	_ "github.com/blevesearch/bleve/analysis/tokenizer/unicode"
	_ "github.com/blevesearch/bleve/analysis/tokenizer/web"
	_ "github.com/blevesearch/bleve/analysis/tokenizer/whitespace"

	// date time parsers
	_ "github.com/blevesearch/bleve/analysis/datetime/flexible"
	_ "github.com/blevesearch/bleve/analysis/datetime/optional"

	// languages
	_ "github.com/blevesearch/bleve/analysis/lang/ar"
	_ "github.com/blevesearch/bleve/analysis/lang/bg"
	_ "github.com/blevesearch/bleve/analysis/lang/ca"
	_ "github.com/blevesearch/bleve/analysis/lang/cjk"
	_ "github.com/blevesearch/bleve/analysis/lang/ckb"
	_ "github.com/blevesearch/bleve/analysis/lang/cs"
	_ "github.com/blevesearch/bleve/analysis/lang/da"
	_ "github.com/blevesearch/bleve/analysis/lang/de"
	_ "github.com/blevesearch/bleve/analysis/lang/el"
	_ "github.com/blevesearch/bleve/analysis/lang/en"
	_ "github.com/blevesearch/bleve/analysis/lang/es"
	_ "github.com/blevesearch/bleve/analysis/lang/eu"
	_ "github.com/blevesearch/bleve/analysis/lang/fa"
	_ "github.com/blevesearch/bleve/analysis/lang/fi"
	_ "github.com/blevesearch/bleve/analysis/lang/fr"
	_ "github.com/blevesearch/bleve/analysis/lang/ga"
	_ "github.com/blevesearch/bleve/analysis/lang/gl"
	_ "github.com/blevesearch/bleve/analysis/lang/hi"
	_ "github.com/blevesearch/bleve/analysis/lang/hu"
	_ "github.com/blevesearch/bleve/analysis/lang/hy"
	_ "github.com/blevesearch/bleve/analysis/lang/id"
	_ "github.com/blevesearch/bleve/analysis/lang/in"
	_ "github.com/blevesearch/bleve/analysis/lang/it"
	_ "github.com/blevesearch/bleve/analysis/lang/nl"
	_ "github.com/blevesearch/bleve/analysis/lang/no"
	_ "github.com/blevesearch/bleve/analysis/lang/pt"
	_ "github.com/blevesearch/bleve/analysis/lang/ro"
	_ "github.com/blevesearch/bleve/analysis/lang/ru"
	_ "github.com/blevesearch/bleve/analysis/lang/sv"
	_ "github.com/blevesearch/bleve/analysis/lang/tr"

	// kv stores
	_ "github.com/blevesearch/bleve/index/store/boltdb"
	_ "github.com/blevesearch/bleve/index/store/goleveldb"
	_ "github.com/blevesearch/bleve/index/store/gtreap"
	_ "github.com/blevesearch/bleve/index/store/moss"

	// index types
	_ "github.com/blevesearch/bleve/index/upsidedown"
)
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"os/user"
)

type iobeamConfig struct {
	Name string `json:"profile"`
}

const (
	CLI_VERSION      = "0.3.0"
	pathSeparator    = string(os.PathSeparator)
	dotDirName       = ".iobeam"
	defaultConfig    = "profile"
	profileFileName  = "profile.config"
	DefaultApiServer = "https://api.iobeam.com"
)

func InitConfig() (*iobeamConfig, error) {
	c := &iobeamConfig{
		Name: "default",
	}
	err := c.save()
	if err != nil {
		return nil, err
	}
	return c, nil
}

func GetDotDir() string {
	user, err := user.Current()

	if err != nil {
		return os.TempDir()
	}

	return user.HomeDir + pathSeparator + dotDirName
}

func defaultConfigPath() string {
	return GetDotDir() + pathSeparator + defaultConfig
}

func makeAllOnPath(path string) error {
	err := os.MkdirAll(path, 0700)
	if err != nil {
		return err
	}
	return nil
}

func saveJson(path string, obj interface{}) error {
	_ = os.Remove(path) // error only if it does not exist, ignore.

	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		return err
	}

	defer file.Close()
	return json.NewEncoder(file).Encode(obj)
}

func readJson(path string, obj interface{}) error {
	file, err := os.Open(path)

	if err != nil {
		return err
	}

	defer file.Close()
	return json.NewDecoder(file).Decode(obj)
}

// save writes the config to disk in the user's .iobeam directory.
func (c *iobeamConfig) save() error {
	err := makeAllOnPath(GetDotDir())
	if err != nil {
		return err
	}
	return saveJson(defaultConfigPath(), c)
}

func (c *iobeamConfig) read(path string) error {
	return readJson(path, c)
}

func readConfig(path string) (*iobeamConfig, error) {
	c := new(iobeamConfig)

	err := c.read(path)
	if err != nil {
		return nil, err
	}

	return c, err
}

func ReadDefaultConfig() (*iobeamConfig, error) {
	return readConfig(defaultConfigPath())
}

type Profile struct {
	Name            string `json:"-"`
	Server          string `json:"server"`
	ActiveProject   uint64 `json:"active_project"`
	ActiveUser      uint64 `json:"active_user"`
	ActiveUserEmail string `json:"activer_user_email"`
	// TODO: Don't export active fields.
}

func InitProfile(name string) (*Profile, error) {
	return InitProfileWithServer(name, DefaultApiServer)
}

func InitProfileWithServer(name, server string) (*Profile, error) {
	p := &Profile{
		Name:          name,
		Server:        server,
		ActiveProject: 0,
		ActiveUser:    0,
	}
	err := p.save()
	if err != nil {
		return nil, err
	}
	return p, nil
}

func baseProfilePath(name string) string {
	return GetDotDir() + pathSeparator + name
}

func profilePath(name string) string {
	// e.x. ~user/.iobeam/default/profile.config
	return baseProfilePath(name) + pathSeparator + profileFileName
}

func (p *Profile) save() error {
	err := makeAllOnPath(p.GetDir())
	if err != nil {
		return err
	}
	return saveJson(profilePath(p.Name), p)
}

func (p *Profile) read() error {
	return readJson(p.GetFile(), p)
}

func (p *Profile) GetDir() string {
	return baseProfilePath(p.Name)
}

func (p *Profile) GetFile() string {
	return profilePath(p.Name)
}

func (p *Profile) UpdateActiveUser(uid uint64, email string) error {
	p.ActiveUser = uid
	p.ActiveUserEmail = email
	return p.save()
}

func (p *Profile) UpdateActiveProject(pid uint64) error {
	p.ActiveProject = pid
	return p.save()
}

func ReadProfile(name string) (*Profile, error) {
	p := new(Profile)
	p.Name = name

	err := p.read()
	if err != nil {
		return nil, err
	}

	return p, nil
}

func GetProfileList() ([]string, error) {
	files, err := ioutil.ReadDir(GetDotDir())
	if err != nil {
		return nil, err
	}

	list := make([]string, 0)
	for _, f := range files {
		if f.IsDir() {
			list = append(list, f.Name())
		}
	}

	return list, nil
}

func SwitchProfile(name string) error {
	path := baseProfilePath(name)
	_, err := os.Stat(path)
	if err != nil {
		return err
	}

	c := &iobeamConfig{
		Name: name,
	}
	err = c.save()
	return err
}
<commit_msg>Unable to get dot dir should panic, not use temp.<commit_after>package config

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"os/user"
)

type iobeamConfig struct {
	Name string `json:"profile"`
}

const (
	CLI_VERSION      = "0.3.0"
	pathSeparator    = string(os.PathSeparator)
	dotDirName       = ".iobeam"
	defaultConfig    = "profile"
	profileFileName  = "profile.config"
	DefaultApiServer = "https://api.iobeam.com"
)

func InitConfig() (*iobeamConfig, error) {
	c := &iobeamConfig{
		Name: "default",
	}
	err := c.save()
	if err != nil {
		return nil, err
	}
	return c, nil
}

func getDotDir() string {
	user, err := user.Current()

	if err != nil {
		// We cannot gracefully use the temp directory with profiles.
		panic(err)
	}

	return user.HomeDir + pathSeparator + dotDirName
}

func defaultConfigPath() string {
	return getDotDir() + pathSeparator + defaultConfig
}

func makeAllOnPath(path string) error {
	err := os.MkdirAll(path, 0700)
	if err != nil {
		return err
	}
	return nil
}

func saveJson(path string, obj interface{}) error {
	_ = os.Remove(path) // error only if it does not exist, ignore.

	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		return err
	}

	defer file.Close()
	return json.NewEncoder(file).Encode(obj)
}

func readJson(path string, obj interface{}) error {
	file, err := os.Open(path)

	if err != nil {
		return err
	}

	defer file.Close()
	return json.NewDecoder(file).Decode(obj)
}

// save writes the config to disk in the user's .iobeam directory.
func (c *iobeamConfig) save() error {
	err := makeAllOnPath(getDotDir())
	if err != nil {
		return err
	}
	return saveJson(defaultConfigPath(), c)
}

func (c *iobeamConfig) read(path string) error {
	return readJson(path, c)
}

func readConfig(path string) (*iobeamConfig, error) {
	c := new(iobeamConfig)

	err := c.read(path)
	if err != nil {
		return nil, err
	}

	return c, err
}

func ReadDefaultConfig() (*iobeamConfig, error) {
	return readConfig(defaultConfigPath())
}

type Profile struct {
	Name            string `json:"-"`
	Server          string `json:"server"`
	ActiveProject   uint64 `json:"active_project"`
	ActiveUser      uint64 `json:"active_user"`
	ActiveUserEmail string `json:"activer_user_email"`
	// TODO: Don't export active fields.
}

func InitProfile(name string) (*Profile, error) {
	return InitProfileWithServer(name, DefaultApiServer)
}

func InitProfileWithServer(name, server string) (*Profile, error) {
	p := &Profile{
		Name:          name,
		Server:        server,
		ActiveProject: 0,
		ActiveUser:    0,
	}
	err := p.save()
	if err != nil {
		return nil, err
	}
	return p, nil
}

func baseProfilePath(name string) string {
	return getDotDir() + pathSeparator + name
}

func profilePath(name string) string {
	// e.x. ~user/.iobeam/default/profile.config
	return baseProfilePath(name) + pathSeparator + profileFileName
}

func (p *Profile) save() error {
	err := makeAllOnPath(p.GetDir())
	if err != nil {
		return err
	}
	return saveJson(profilePath(p.Name), p)
}

func (p *Profile) read() error {
	return readJson(p.GetFile(), p)
}

func (p *Profile) GetDir() string {
	return baseProfilePath(p.Name)
}

func (p *Profile) GetFile() string {
	return profilePath(p.Name)
}

func (p *Profile) UpdateActiveUser(uid uint64, email string) error {
	p.ActiveUser = uid
	p.ActiveUserEmail = email
	return p.save()
}

func (p *Profile) UpdateActiveProject(pid uint64) error {
	p.ActiveProject = pid
	return p.save()
}

func ReadProfile(name string) (*Profile, error) {
	p := new(Profile)
	p.Name = name

	err := p.read()
	if err != nil {
		return nil, err
	}

	return p, nil
}

func GetProfileList() ([]string, error) {
	files, err := ioutil.ReadDir(getDotDir())
	if err != nil {
		return nil, err
	}

	list := make([]string, 0)
	for _, f := range files {
		if f.IsDir() {
			list = append(list, f.Name())
		}
	}

	return list, nil
}

func SwitchProfile(name string) error {
	path := baseProfilePath(name)
	_, err := os.Stat(path)
	if err != nil {
		return err
	}

	c := &iobeamConfig{
		Name: name,
	}
	err = c.save()
	return err
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"

	homedir "github.com/mitchellh/go-homedir"
)

type iobeamConfig struct {
	Name string `json:"profile"`
}

const (
	// CLIVersion is the version of the CLI.
	CLIVersion = "0.6.1"
	// DefaultApiServer is the default iobeam server.
	DefaultApiServer = "https://api.iobeam.com"

	pathSeparator   = string(os.PathSeparator)
	dotDirName      = ".iobeam"
	defaultConfig   = "profile"
	profileFileName = "profile.config"
)

// InitConfig sets up the default config.
func InitConfig() (*iobeamConfig, error) {
	c := &iobeamConfig{
		Name: "default",
	}
	err := c.save()
	if err != nil {
		return nil, err
	}
	return c, nil
}

func getDotDir() string {
	homeDir, err := homedir.Dir()

	if err != nil {
		// We cannot gracefully use the temp directory with profiles.
		panic(err)
	}

	return homeDir + pathSeparator + dotDirName
}

func defaultConfigPath() string {
	return getDotDir() + pathSeparator + defaultConfig
}

func makeAllOnPath(path string) error {
	err := os.MkdirAll(path, 0700)
	if err != nil {
		return err
	}
	return nil
}

func saveJson(path string, obj interface{}) error {
	_ = os.Remove(path) // error only if it does not exist, ignore.

	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		return err
	}

	defer file.Close()
	return json.NewEncoder(file).Encode(obj)
}

func readJson(path string, obj interface{}) error {
	file, err := os.Open(path)

	if err != nil {
		return err
	}

	defer file.Close()
	return json.NewDecoder(file).Decode(obj)
}

// save writes the config to disk in the user's .iobeam directory.
func (c *iobeamConfig) save() error {
	err := makeAllOnPath(getDotDir())
	if err != nil {
		return err
	}
	return saveJson(defaultConfigPath(), c)
}

func (c *iobeamConfig) read(path string) error {
	return readJson(path, c)
}

func readConfig(path string) (*iobeamConfig, error) {
	c := new(iobeamConfig)

	err := c.read(path)
	if err != nil {
		return nil, err
	}

	return c, err
}

func ReadDefaultConfig() (*iobeamConfig, error) {
	return readConfig(defaultConfigPath())
}

// Profile represents a CLI profile, which is similar to a workspace
// that tracks active user, project, and other metadata.
type Profile struct {
	Name            string `json:"-"`
	Server          string `json:"server"`
	ActiveProject   uint64 `json:"active_project"`
	ActiveUser      uint64 `json:"active_user"`
	ActiveUserEmail string `json:"activer_user_email"`
	// TODO: Don't export active fields.
}

// InitProfile creates a new profile on the system named 'name'.
func InitProfile(name string) (*Profile, error) {
	return InitProfileWithServer(name, DefaultApiServer)
}

// InitProfileWithServer creates a new profile on the system named
// 'name' and uses 'server' for the API server.
func InitProfileWithServer(name, server string) (*Profile, error) {
	p := &Profile{
		Name:          name,
		Server:        server,
		ActiveProject: 0,
		ActiveUser:    0,
	}
	err := p.save()
	if err != nil {
		return nil, err
	}
	return p, nil
}

func baseProfilePath(name string) string {
	return getDotDir() + pathSeparator + name
}

func profilePath(name string) string {
	// e.x. ~user/.iobeam/default/profile.config
	return baseProfilePath(name) + pathSeparator + profileFileName
}

func (p *Profile) save() error {
	err := makeAllOnPath(p.GetDir())
	if err != nil {
		return err
	}
	return saveJson(profilePath(p.Name), p)
}

func (p *Profile) read() error {
	return readJson(p.GetFile(), p)
}

// GetDir returns the path to where p's data is stored.
func (p *Profile) GetDir() string {
	return baseProfilePath(p.Name)
}

// GetFile returns the path to where p's metadata is stored.
func (p *Profile) GetFile() string {
	return profilePath(p.Name)
}

// UpdateActiveUser changes the active user id and email of p.
func (p *Profile) UpdateActiveUser(uid uint64, email string) error {
	p.ActiveUser = uid
	p.ActiveUserEmail = email
	return p.save()
}

// UpdateActiveProject changes the active project id of p.
func (p *Profile) UpdateActiveProject(pid uint64) error {
	p.ActiveProject = pid
	return p.save()
}

// ReadProfile attempts to read and create a *Profile object.
func ReadProfile(name string) (*Profile, error) {
	p := new(Profile)
	p.Name = name

	err := p.read()
	if err != nil {
		return nil, err
	}

	return p, nil
}

// GetProfileList returns a list of available profiles.
func GetProfileList() ([]string, error) {
	files, err := ioutil.ReadDir(getDotDir())
	if err != nil {
		return nil, err
	}

	var list []string
	for _, f := range files {
		if f.IsDir() {
			list = append(list, f.Name())
		}
	}

	return list, nil
}

// SwitchProfile attempts to change the active profile.
func SwitchProfile(name string) error {
	path := baseProfilePath(name)
	_, err := os.Stat(path)
	if os.IsNotExist(err) {
		return fmt.Errorf("Profile '%s' does not exist", name)
	}

	if err == nil {
		c := &iobeamConfig{
			Name: name,
		}
		err = c.save()
	}
	return err
}

// DeleteProfile removes a profile from the system.
func DeleteProfile(name string) error {
	path := baseProfilePath(name)
	_, err := os.Stat(path)
	if err == nil {
		return os.RemoveAll(path)
	} else if os.IsNotExist(err) {
		return fmt.Errorf("Profile '%s' does not exist", name)
	} else {
		return err
	}
}
<commit_msg>release: v0.6.2<commit_after>package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"

	homedir "github.com/mitchellh/go-homedir"
)

type iobeamConfig struct {
	Name string `json:"profile"`
}

const (
	// CLIVersion is the version of the CLI.
	CLIVersion = "0.6.2"
	// DefaultApiServer is the default iobeam server.
	DefaultApiServer = "https://api.iobeam.com"

	pathSeparator   = string(os.PathSeparator)
	dotDirName      = ".iobeam"
	defaultConfig   = "profile"
	profileFileName = "profile.config"
)

// InitConfig sets up the default config.
func InitConfig() (*iobeamConfig, error) {
	c := &iobeamConfig{
		Name: "default",
	}
	err := c.save()
	if err != nil {
		return nil, err
	}
	return c, nil
}

func getDotDir() string {
	homeDir, err := homedir.Dir()

	if err != nil {
		// We cannot gracefully use the temp directory with profiles.
		panic(err)
	}

	return homeDir + pathSeparator + dotDirName
}

func defaultConfigPath() string {
	return getDotDir() + pathSeparator + defaultConfig
}

func makeAllOnPath(path string) error {
	err := os.MkdirAll(path, 0700)
	if err != nil {
		return err
	}
	return nil
}

func saveJson(path string, obj interface{}) error {
	_ = os.Remove(path) // error only if it does not exist, ignore.

	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		return err
	}

	defer file.Close()
	return json.NewEncoder(file).Encode(obj)
}

func readJson(path string, obj interface{}) error {
	file, err := os.Open(path)

	if err != nil {
		return err
	}

	defer file.Close()
	return json.NewDecoder(file).Decode(obj)
}

// save writes the config to disk in the user's .iobeam directory.
func (c *iobeamConfig) save() error {
	err := makeAllOnPath(getDotDir())
	if err != nil {
		return err
	}
	return saveJson(defaultConfigPath(), c)
}

func (c *iobeamConfig) read(path string) error {
	return readJson(path, c)
}

func readConfig(path string) (*iobeamConfig, error) {
	c := new(iobeamConfig)

	err := c.read(path)
	if err != nil {
		return nil, err
	}

	return c, err
}

func ReadDefaultConfig() (*iobeamConfig, error) {
	return readConfig(defaultConfigPath())
}

// Profile represents a CLI profile, which is similar to a workspace
// that tracks active user, project, and other metadata.
type Profile struct {
	Name            string `json:"-"`
	Server          string `json:"server"`
	ActiveProject   uint64 `json:"active_project"`
	ActiveUser      uint64 `json:"active_user"`
	ActiveUserEmail string `json:"activer_user_email"`
	// TODO: Don't export active fields.
}

// InitProfile creates a new profile on the system named 'name'.
func InitProfile(name string) (*Profile, error) {
	return InitProfileWithServer(name, DefaultApiServer)
}

// InitProfileWithServer creates a new profile on the system named
// 'name' and uses 'server' for the API server.
func InitProfileWithServer(name, server string) (*Profile, error) {
	p := &Profile{
		Name:          name,
		Server:        server,
		ActiveProject: 0,
		ActiveUser:    0,
	}
	err := p.save()
	if err != nil {
		return nil, err
	}
	return p, nil
}

func baseProfilePath(name string) string {
	return getDotDir() + pathSeparator + name
}

func profilePath(name string) string {
	// e.x. ~user/.iobeam/default/profile.config
	return baseProfilePath(name) + pathSeparator + profileFileName
}

func (p *Profile) save() error {
	err := makeAllOnPath(p.GetDir())
	if err != nil {
		return err
	}
	return saveJson(profilePath(p.Name), p)
}

func (p *Profile) read() error {
	return readJson(p.GetFile(), p)
}

// GetDir returns the path to where p's data is stored.
func (p *Profile) GetDir() string {
	return baseProfilePath(p.Name)
}

// GetFile returns the path to where p's metadata is stored.
func (p *Profile) GetFile() string {
	return profilePath(p.Name)
}

// UpdateActiveUser changes the active user id and email of p.
func (p *Profile) UpdateActiveUser(uid uint64, email string) error {
	p.ActiveUser = uid
	p.ActiveUserEmail = email
	return p.save()
}

// UpdateActiveProject changes the active project id of p.
func (p *Profile) UpdateActiveProject(pid uint64) error {
	p.ActiveProject = pid
	return p.save()
}

// ReadProfile attempts to read and create a *Profile object.
func ReadProfile(name string) (*Profile, error) {
	p := new(Profile)
	p.Name = name

	err := p.read()
	if err != nil {
		return nil, err
	}

	return p, nil
}

// GetProfileList returns a list of available profiles.
func GetProfileList() ([]string, error) {
	files, err := ioutil.ReadDir(getDotDir())
	if err != nil {
		return nil, err
	}

	var list []string
	for _, f := range files {
		if f.IsDir() {
			list = append(list, f.Name())
		}
	}

	return list, nil
}

// SwitchProfile attempts to change the active profile.
func SwitchProfile(name string) error {
	path := baseProfilePath(name)
	_, err := os.Stat(path)
	if os.IsNotExist(err) {
		return fmt.Errorf("Profile '%s' does not exist", name)
	}

	if err == nil {
		c := &iobeamConfig{
			Name: name,
		}
		err = c.save()
	}
	return err
}

// DeleteProfile removes a profile from the system.
func DeleteProfile(name string) error {
	path := baseProfilePath(name)
	_, err := os.Stat(path)
	if err == nil {
		return os.RemoveAll(path)
	} else if os.IsNotExist(err) {
		return fmt.Errorf("Profile '%s' does not exist", name)
	} else {
		return err
	}
}
<|endoftext|>


<commit_before>package config

import "time"

type MessageBusServer struct {
	Host     string `yaml:"host"`
	User     string `yaml:"user"`
	Password string `yaml:"password"`
}

type HealthCheckerConf struct {
	Name              string `yaml:"name"`
	HealthcheckScript string `yaml:"healthcheck_script_path"`
}

type Config struct {
	MessageBusServers []MessageBusServer `yaml:"message_bus_servers"`
	HealthChecker     *HealthCheckerConf `yaml:"health_checker"`
	Routes            []Route            `yaml:"routes"`
	RefreshInterval   time.Duration      `yaml:"refresh_interval"`
	Host              string             `yaml:"host"`
}

type Route struct {
	Name string   `yaml:"name"`
	Port int      `yaml:"port"`
	Tags []Tag    `yaml:"tags"`
	URIs []string `yaml:"uris"`
}

type Tag map[string]string
<commit_msg>Change type of Tags field in config<commit_after>package config

import "time"

type MessageBusServer struct {
	Host     string `yaml:"host"`
	User     string `yaml:"user"`
	Password string `yaml:"password"`
}

type HealthCheckerConf struct {
	Name              string `yaml:"name"`
	HealthcheckScript string `yaml:"healthcheck_script_path"`
}

type Config struct {
	MessageBusServers []MessageBusServer `yaml:"message_bus_servers"`
	HealthChecker     *HealthCheckerConf `yaml:"health_checker"`
	Routes            []Route            `yaml:"routes"`
	RefreshInterval   time.Duration      `yaml:"refresh_interval"`
	Host              string             `yaml:"host"`
}

type Route struct {
	Name string            `yaml:"name"`
	Port int               `yaml:"port"`
	Tags map[string]string `yaml:"tags"`
	URIs []string          `yaml:"uris"`
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/inominate/eve-api-proxy/apicache"
)

var apiClient apicache.Client

// worker tracking
var activeWorkerCount, workerCount int32
var workCount []int32

type apiReq struct {
	apiReq  *apicache.Request
	apiResp *apicache.Response

	apiErr apicache.APIError

	expires time.Time

	worker   int
	httpCode int
	err      error
	respChan chan apiReq
}

// Channel for sending jobs to workers
var workChan chan apiReq

func APIReq(url string, params map[string]string) (*apicache.Response, error) {
	var errorStr string

	if atomic.LoadInt32(&workerCount) <= 0 {
		panic("No workers!")
	}

	// Build the request
	apireq := apicache.NewRequest(url)
	for k, v := range params {
		switch k {
		case "force":
			if v != "" {
				apireq.Force = true
			}
		default:
			apireq.Set(k, v)
		}
	}

	workerID := "C"
	// Don't send it to a worker if we can just yank it fromm the cache
	apiResp, err := apireq.GetCached()
	if err != nil || apireq.Force {
		respChan := make(chan apiReq)
		req := apiReq{apiReq: apireq, respChan: respChan}
		workChan <- req

		resp := <-respChan
		close(respChan)

		apiResp = resp.apiResp
		err = resp.err
		workerID = fmt.Sprintf("%d", resp.worker)
	}

	// This is similar to the request log, but knows more about where it came from.
	if debug {
		if apiResp.Error.ErrorCode != 0 {
			errorStr = fmt.Sprintf(" Error %d: %s", apiResp.Error.ErrorCode, apiResp.Error.ErrorText)
		}
		logParams := ""
		var paramVal string
		for k, _ := range params {
			if conf.Logging.CensorLog && strings.ToLower(k) == "vcode" && len(params[k]) > 8 {
				paramVal = params[k][0:8] + "..."
			} else {
				paramVal = params[k]
			}
			logParams = fmt.Sprintf("%s&%s=%s", logParams, k, paramVal)
		}
		if logParams != "" {
			logParams = "?" + logParams[1:]
		}

		debugLog.Printf("w%s: %s%s HTTP: %d Expires: %s%s", workerID, url, logParams, apiResp.HTTPCode, apiResp.Expires.Format("2006-01-02 15:04:05"), errorStr)
	}
	return apiResp, err
}

func worker(reqChan chan apiReq, workerID int) {
	atomic.AddInt32(&workerCount, 1)
	var err error

	var eErr error
	var rErr error

	var errStr string
	for req := range reqChan {
		atomic.AddInt32(&activeWorkerCount, 1)

		// Run both of the error limiters simultaneously rather than in
		// sequence. Still need both before we continue.
		errorLimiter := make(chan error)
		rpsLimiter := make(chan error)
		go func() {
			err := errorRateLimiter.Start(60 * time.Second)
			errorLimiter <- err
		}()
		go func() {
			err := rateLimiter.Start(60 * time.Second)
			rpsLimiter <- err
		}()
		eErr = <-errorLimiter
		rErr = <-rpsLimiter

		// Check the error limiter for timeouts
		if eErr != nil {
			err = eErr
			errStr = "error throttling"

			// If the rate limiter didn't timeout be sure to signal it that we
			// didn't do anything.
			if rErr == nil {
				rateLimiter.Finish(true)
			}
		}
		if rErr != nil {
			err = rErr
			errStr = "rate limiting"

			// If the error limiter didn't also timeout be sure to signal it that we
			// didn't do anything.
			if eErr == nil {
				errorRateLimiter.Finish(true)
			}
		}
		// We're left with a single err and errStr for returning an error to the client.
		if err != nil {
			req.apiResp = &apicache.Response{
				Data: apicache.SynthesizeAPIError(500,
					fmt.Sprintf("APIProxy Error: Proxy timeout due to %s.", errStr),
					5*time.Minute),
				Expires: time.Now().Add(5 * time.Minute),
				Error: apicache.APIError{500,
					fmt.Sprintf("APIProxy Error: Proxy timeout due to %s.", errStr)},
				HTTPCode: 504,
			}
			req.err = err
		} else {
			resp, err := req.apiReq.Do()
			req.apiResp = resp
			req.err = err
			if resp.Error.ErrorCode == 0 {
				// Finish, but skip recording the event in the rate limiter
				// when there is no error.
				errorRateLimiter.Finish(true)
			} else {
				errorRateLimiter.Finish(false)
			}
			rateLimiter.Finish(false)
		}

		req.worker = workerID
		req.respChan <- req
		atomic.AddInt32(&workCount[workerID], 1)
		atomic.AddInt32(&activeWorkerCount, -1)
	}
	atomic.AddInt32(&workerCount, -1)
}

var startWorkersOnce = &sync.Once{}

func startWorkers() {
	startWorkersOnce.Do(realStartWorkers)
}

func realStartWorkers() {
	log.Printf("Starting %d Workers...", conf.Workers)
	workChan = make(chan apiReq)
	workCount = make([]int32, conf.Workers+1)

	for i := 1; i <= conf.Workers; i++ {
		debugLog.Printf("Starting worker #%d.", i)
		go worker(workChan, i)
	}
}

func stopWorkers() {
	close(workChan)
	for atomic.LoadInt32(&workerCount) > 0 {
		time.Sleep(10 * time.Millisecond)
	}
	startWorkersOnce = &sync.Once{}
}

func PrintWorkerStats(w io.Writer) {
	active, loaded := GetWorkerStats()
	fmt.Fprintf(w, "%d workers idle, %d workers active.\n", loaded-active, active)

	for i := int32(1); i <= atomic.LoadInt32(&workerCount); i++ {
		count := atomic.LoadInt32(&workCount[i])
		fmt.Fprintf(w, "   %d: %d\n", i, count)
	}
}

func GetWorkerStats() (int32, int32) {
	return atomic.LoadInt32(&activeWorkerCount), atomic.LoadInt32(&workerCount)
}
<commit_msg>added rate and error rate reporting to stats<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/inominate/eve-api-proxy/apicache"
)

var apiClient apicache.Client

// worker tracking
var activeWorkerCount, workerCount int32
var workCount []int32

type apiReq struct {
	apiReq  *apicache.Request
	apiResp *apicache.Response

	apiErr apicache.APIError

	expires time.Time

	worker   int
	httpCode int
	err      error
	respChan chan apiReq
}

// Channel for sending jobs to workers
var workChan chan apiReq

func APIReq(url string, params map[string]string) (*apicache.Response, error) {
	var errorStr string

	if atomic.LoadInt32(&workerCount) <= 0 {
		panic("No workers!")
	}

	// Build the request
	apireq := apicache.NewRequest(url)
	for k, v := range params {
		switch k {
		case "force":
			if v != "" {
				apireq.Force = true
			}
		default:
			apireq.Set(k, v)
		}
	}

	workerID := "C"
	// Don't send it to a worker if we can just yank it fromm the cache
	apiResp, err := apireq.GetCached()
	if err != nil || apireq.Force {
		respChan := make(chan apiReq)
		req := apiReq{apiReq: apireq, respChan: respChan}
		workChan <- req

		resp := <-respChan
		close(respChan)

		apiResp = resp.apiResp
		err = resp.err
		workerID = fmt.Sprintf("%d", resp.worker)
	}

	// This is similar to the request log, but knows more about where it came from.
	if debug {
		if apiResp.Error.ErrorCode != 0 {
			errorStr = fmt.Sprintf(" Error %d: %s", apiResp.Error.ErrorCode, apiResp.Error.ErrorText)
		}
		logParams := ""
		var paramVal string
		for k, _ := range params {
			if conf.Logging.CensorLog && strings.ToLower(k) == "vcode" && len(params[k]) > 8 {
				paramVal = params[k][0:8] + "..."
			} else {
				paramVal = params[k]
			}
			logParams = fmt.Sprintf("%s&%s=%s", logParams, k, paramVal)
		}
		if logParams != "" {
			logParams = "?" + logParams[1:]
		}

		debugLog.Printf("w%s: %s%s HTTP: %d Expires: %s%s", workerID, url, logParams, apiResp.HTTPCode, apiResp.Expires.Format("2006-01-02 15:04:05"), errorStr)
	}
	return apiResp, err
}

func worker(reqChan chan apiReq, workerID int) {
	atomic.AddInt32(&workerCount, 1)
	var err error

	var eErr error
	var rErr error

	var errStr string
	for req := range reqChan {
		atomic.AddInt32(&activeWorkerCount, 1)

		// Run both of the error limiters simultaneously rather than in
		// sequence. Still need both before we continue.
		errorLimiter := make(chan error)
		rpsLimiter := make(chan error)
		go func() {
			err := errorRateLimiter.Start(60 * time.Second)
			errorLimiter <- err
		}()
		go func() {
			err := rateLimiter.Start(60 * time.Second)
			rpsLimiter <- err
		}()
		eErr = <-errorLimiter
		rErr = <-rpsLimiter

		// Check the error limiter for timeouts
		if eErr != nil {
			err = eErr
			errStr = "error throttling"

			// If the rate limiter didn't timeout be sure to signal it that we
			// didn't do anything.
			if rErr == nil {
				rateLimiter.Finish(true)
			}
		}
		if rErr != nil {
			err = rErr
			errStr = "rate limiting"

			// If the error limiter didn't also timeout be sure to signal it that we
			// didn't do anything.
			if eErr == nil {
				errorRateLimiter.Finish(true)
			}
		}
		// We're left with a single err and errStr for returning an error to the client.
		if err != nil {
			req.apiResp = &apicache.Response{
				Data: apicache.SynthesizeAPIError(500,
					fmt.Sprintf("APIProxy Error: Proxy timeout due to %s.", errStr),
					5*time.Minute),
				Expires: time.Now().Add(5 * time.Minute),
				Error: apicache.APIError{500,
					fmt.Sprintf("APIProxy Error: Proxy timeout due to %s.", errStr)},
				HTTPCode: 504,
			}
			req.err = err
		} else {
			resp, err := req.apiReq.Do()
			req.apiResp = resp
			req.err = err
			if resp.Error.ErrorCode == 0 {
				// Finish, but skip recording the event in the rate limiter
				// when there is no error.
				errorRateLimiter.Finish(true)
			} else {
				errorRateLimiter.Finish(false)
			}
			rateLimiter.Finish(false)
		}

		req.worker = workerID
		req.respChan <- req
		atomic.AddInt32(&workCount[workerID], 1)
		atomic.AddInt32(&activeWorkerCount, -1)
	}
	atomic.AddInt32(&workerCount, -1)
}

var startWorkersOnce = &sync.Once{}

func startWorkers() {
	startWorkersOnce.Do(realStartWorkers)
}

func realStartWorkers() {
	log.Printf("Starting %d Workers...", conf.Workers)
	workChan = make(chan apiReq)
	workCount = make([]int32, conf.Workers+1)

	for i := 1; i <= conf.Workers; i++ {
		debugLog.Printf("Starting worker #%d.", i)
		go worker(workChan, i)
	}
}

func stopWorkers() {
	close(workChan)
	for atomic.LoadInt32(&workerCount) > 0 {
		time.Sleep(10 * time.Millisecond)
	}
	startWorkersOnce = &sync.Once{}
}

func PrintWorkerStats(w io.Writer) {
	active, loaded := GetWorkerStats()
	fmt.Fprintf(w, "%d workers idle, %d workers active.\n", loaded-active, active)

	rateCount := rateLimiter.Count()
	errorCount := errorRateLimiter.Count()
	fmt.Fprintf(w, "%d requests in the last second.\n", rateCount)
	fmt.Fprintf(w, "%d errors over last %d seconds.\n", errorCount, conf.ErrorPeriod)

	for i := int32(1); i <= atomic.LoadInt32(&workerCount); i++ {
		count := atomic.LoadInt32(&workCount[i])
		fmt.Fprintf(w, "   %d: %d\n", i, count)
	}
}

func GetWorkerStats() (int32, int32) {
	return atomic.LoadInt32(&activeWorkerCount), atomic.LoadInt32(&workerCount)
}
<|endoftext|>


<commit_before>package main_test

import (
	"encoding/json"
	"fmt"
	"syscall"
	"time"

	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"github.com/cloudfoundry-incubator/bbs"
	bbsrunner "github.com/cloudfoundry-incubator/bbs/cmd/bbs/testrunner"
	"github.com/cloudfoundry-incubator/bbs/models"
	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/consuladapter/consulrunner"
	convergerrunner "github.com/cloudfoundry-incubator/converger/cmd/converger/testrunner"
	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	oldmodels "github.com/cloudfoundry-incubator/runtime-schema/models"
)

type BinPaths struct {
	Converger string
	Bbs       string
}

var _ = Describe("Converger", func() {
	const (
		exitDuration                = 4 * time.Second
		convergeRepeatInterval      = 500 * time.Millisecond
		taskKickInterval            = convergeRepeatInterval
		expireCompletedTaskDuration = 3 * convergeRepeatInterval
		expirePendingTaskDuration   = 30 * time.Minute
	)

	var (
		binPaths         BinPaths
		etcdRunner       *etcdstorerunner.ETCDClusterRunner
		bbsArgs          bbsrunner.Args
		bbsProcess       ifrit.Process
		bbsClient        bbs.Client
		legacyBBS        *Bbs.BBS
		convergerConfig  *convergerrunner.Config
		convergerProcess ifrit.Process
		runner           *ginkgomon.Runner

		consulRunner  *consulrunner.ClusterRunner
		consulSession *consuladapter.Session

		etcdClient storeadapter.StoreAdapter

		logger lager.Logger
	)

	SynchronizedBeforeSuite(func() []byte {
		convergerBinPath, err := Build("github.com/cloudfoundry-incubator/converger/cmd/converger", "-race")
		Expect(err).NotTo(HaveOccurred())
		bbsBinPath, err := Build("github.com/cloudfoundry-incubator/bbs/cmd/bbs", "-race")
		Expect(err).NotTo(HaveOccurred())
		bytes, err := json.Marshal(BinPaths{
			Converger: convergerBinPath,
			Bbs:       bbsBinPath,
		})
		Expect(err).NotTo(HaveOccurred())
		return bytes
	}, func(bytes []byte) {
		binPaths = BinPaths{}
		err := json.Unmarshal(bytes, &binPaths)
		Expect(err).NotTo(HaveOccurred())

		etcdPort := 5001 + config.GinkgoConfig.ParallelNode
		etcdCluster := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
		etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, nil)

		etcdClient = etcdRunner.Adapter(nil)

		consulRunner = consulrunner.NewClusterRunner(
			9001+config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
			1,
			"http",
		)

		logger = lagertest.NewTestLogger("test")

		convergerConfig = &convergerrunner.Config{
			BinPath:                     binPaths.Converger,
			ConvergeRepeatInterval:      convergeRepeatInterval.String(),
			KickTaskDuration:            taskKickInterval.String(),
			ExpirePendingTaskDuration:   expirePendingTaskDuration.String(),
			ExpireCompletedTaskDuration: expireCompletedTaskDuration.String(),
			EtcdCluster:                 etcdCluster,
			ConsulCluster:               consulRunner.ConsulCluster(),
			LogLevel:                    "info",
		}

		bbsArgs = bbsrunner.Args{
			Address:           fmt.Sprintf("127.0.0.1:%d", 13000+GinkgoParallelNode()),
			AuctioneerAddress: "some-address",
			EtcdCluster:       etcdCluster,
			ConsulCluster:     consulRunner.ConsulCluster(),
		}
	})

	SynchronizedAfterSuite(func() {
	}, func() {
		CleanupBuildArtifacts()
	})

	BeforeEach(func() {
		etcdRunner.Start()
		consulRunner.Start()
		consulRunner.WaitUntilReady()

		bbsProcess = ginkgomon.Invoke(bbsrunner.New(binPaths.Bbs, bbsArgs))
		bbsClient = bbs.NewClient(fmt.Sprint("http://", bbsArgs.Address))

		consulSession = consulRunner.NewSession("a-session")
		legacyBBS = Bbs.NewBBS(etcdClient, consulSession, "http://receptor.bogus.com", clock.NewClock(), logger)

		capacity := oldmodels.NewCellCapacity(512, 1024, 124)
		cellPresence := oldmodels.NewCellPresence("the-cell-id", "1.2.3.4", "the-zone", capacity, []string{}, []string{})

		value, err := oldmodels.ToJSON(cellPresence)
		Expect(err).NotTo(HaveOccurred())

		_, err = consulSession.SetPresence(shared.CellSchemaPath(cellPresence.CellID), value)
		Expect(err).NotTo(HaveOccurred())

	})

	AfterEach(func() {
		ginkgomon.Kill(bbsProcess)
		ginkgomon.Kill(convergerProcess)
		consulRunner.Stop()
		etcdRunner.Stop()
	})

	startConverger := func() {
		runner = convergerrunner.New(convergerConfig)
		convergerProcess = ginkgomon.Invoke(runner)
		time.Sleep(convergeRepeatInterval)
	}

	createRunningTaskWithDeadCell := func() {
		task := oldmodels.Task{
			Domain: "tests",

			TaskGuid: "task-guid",
			RootFS:   "some:rootfs",
			Action: &oldmodels.RunAction{
				User: "me",
				Path: "cat",
				Args: []string{"/tmp/file"},
			},
		}

		err := legacyBBS.DesireTask(logger, task)
		Expect(err).NotTo(HaveOccurred())

		_, err = legacyBBS.StartTask(logger, task.TaskGuid, "dead-cell")
		Expect(err).NotTo(HaveOccurred())
	}

	itIsInactive := func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("does not converge the task", func() {
				Consistently(func() []*models.Task {
					return getTasksByState(bbsClient, models.Task_Completed)
				}, 10*convergeRepeatInterval).Should(BeEmpty())
			})
		})
	}

	Context("when the converger has the lock", func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("marks the task as completed and failed", func() {
				Consistently(func() []*models.Task {
					return getTasksByState(bbsClient, models.Task_Completed)
				}, 0.5).Should(BeEmpty())

				startConverger()

				Eventually(func() []*models.Task {
					return getTasksByState(bbsClient, models.Task_Completed)
				}, 10*convergeRepeatInterval).Should(HaveLen(1))
			})
		})
	})

	Context("when the converger loses the lock", func() {
		BeforeEach(func() {
			startConverger()
			Eventually(runner, 5*time.Second).Should(gbytes.Say("acquire-lock-succeeded"))

			consulRunner.Reset()
		})

		It("exits with an error", func() {
			Eventually(runner, exitDuration).Should(Exit(1))
		})
	})

	Context("when the converger initially does not have the lock", func() {
		var otherSession *consuladapter.Session

		BeforeEach(func() {
			otherSession = consulRunner.NewSession("other-session")
			err := otherSession.AcquireLock(shared.LockSchemaPath("converge_lock"), []byte("something-else"))
			Expect(err).NotTo(HaveOccurred())

			startConverger()
		})

		itIsInactive()

		Describe("when the lock becomes available", func() {
			BeforeEach(func() {
				otherSession.Destroy()
				time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
			})

			Describe("when a running task with a dead cell is present", func() {
				JustBeforeEach(createRunningTaskWithDeadCell)

				It("eventually marks the task as failed", func() {
					Eventually(func() []*models.Task {
						completedTasks := getTasksByState(bbsClient, models.Task_Completed)
						return failedTasks(completedTasks)
					}, 10*convergeRepeatInterval).Should(HaveLen(1))
				})
			})
		})
	})

	Describe("signal handling", func() {
		BeforeEach(func() {
			startConverger()
		})

		Describe("when it receives SIGINT", func() {
			It("exits successfully", func() {
				convergerProcess.Signal(syscall.SIGINT)
				Eventually(runner, exitDuration).Should(Exit(0))
			})
		})

		Describe("when it receives SIGTERM", func() {
			It("exits successfully", func() {
				convergerProcess.Signal(syscall.SIGTERM)
				Eventually(runner, exitDuration).Should(Exit(0))
			})
		})
	})

	Context("when etcd is down", func() {
		BeforeEach(func() {
			etcdRunner.Stop()
			startConverger()
		})

		It("starts", func() {
			Consistently(runner).ShouldNot(Exit())
		})
	})
})

func getTasksByState(client bbs.Client, state models.Task_State) []*models.Task {
	tasks, err := client.Tasks()
	Expect(err).NotTo(HaveOccurred())

	filteredTasks := make([]*models.Task, 0)
	for _, task := range tasks {
		if task.State == state {
			filteredTasks = append(filteredTasks, task)
		}
	}
	return filteredTasks
}

func failedTasks(tasks []*models.Task) []*models.Task {
	failedTasks := make([]*models.Task, 0)

	for _, task := range tasks {
		if task.Failed {
			failedTasks = append(failedTasks, task)
		}
	}

	return failedTasks
}
<commit_msg>wired bbsClient start and desire task<commit_after>package main_test

import (
	"encoding/json"
	"fmt"
	"syscall"
	"time"

	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"github.com/cloudfoundry-incubator/bbs"
	bbsrunner "github.com/cloudfoundry-incubator/bbs/cmd/bbs/testrunner"
	"github.com/cloudfoundry-incubator/bbs/models"
	"github.com/cloudfoundry-incubator/bbs/models/internal/model_helpers"
	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/consuladapter/consulrunner"
	convergerrunner "github.com/cloudfoundry-incubator/converger/cmd/converger/testrunner"
	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	oldmodels "github.com/cloudfoundry-incubator/runtime-schema/models"
)

type BinPaths struct {
	Converger string
	Bbs       string
}

var _ = Describe("Converger", func() {
	const (
		exitDuration                = 4 * time.Second
		convergeRepeatInterval      = 500 * time.Millisecond
		taskKickInterval            = convergeRepeatInterval
		expireCompletedTaskDuration = 3 * convergeRepeatInterval
		expirePendingTaskDuration   = 30 * time.Minute
	)

	var (
		binPaths         BinPaths
		etcdRunner       *etcdstorerunner.ETCDClusterRunner
		bbsArgs          bbsrunner.Args
		bbsProcess       ifrit.Process
		bbsClient        bbs.Client
		legacyBBS        *Bbs.BBS
		convergerConfig  *convergerrunner.Config
		convergerProcess ifrit.Process
		runner           *ginkgomon.Runner

		consulRunner  *consulrunner.ClusterRunner
		consulSession *consuladapter.Session

		etcdClient storeadapter.StoreAdapter

		logger lager.Logger
	)

	SynchronizedBeforeSuite(func() []byte {
		convergerBinPath, err := Build("github.com/cloudfoundry-incubator/converger/cmd/converger", "-race")
		Expect(err).NotTo(HaveOccurred())
		bbsBinPath, err := Build("github.com/cloudfoundry-incubator/bbs/cmd/bbs", "-race")
		Expect(err).NotTo(HaveOccurred())
		bytes, err := json.Marshal(BinPaths{
			Converger: convergerBinPath,
			Bbs:       bbsBinPath,
		})
		Expect(err).NotTo(HaveOccurred())
		return bytes
	}, func(bytes []byte) {
		binPaths = BinPaths{}
		err := json.Unmarshal(bytes, &binPaths)
		Expect(err).NotTo(HaveOccurred())

		etcdPort := 5001 + config.GinkgoConfig.ParallelNode
		etcdCluster := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
		etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, nil)

		etcdClient = etcdRunner.Adapter(nil)

		consulRunner = consulrunner.NewClusterRunner(
			9001+config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
			1,
			"http",
		)

		logger = lagertest.NewTestLogger("test")

		convergerConfig = &convergerrunner.Config{
			BinPath:                     binPaths.Converger,
			ConvergeRepeatInterval:      convergeRepeatInterval.String(),
			KickTaskDuration:            taskKickInterval.String(),
			ExpirePendingTaskDuration:   expirePendingTaskDuration.String(),
			ExpireCompletedTaskDuration: expireCompletedTaskDuration.String(),
			EtcdCluster:                 etcdCluster,
			ConsulCluster:               consulRunner.ConsulCluster(),
			LogLevel:                    "info",
		}

		bbsArgs = bbsrunner.Args{
			Address:           fmt.Sprintf("127.0.0.1:%d", 13000+GinkgoParallelNode()),
			AuctioneerAddress: "some-address",
			EtcdCluster:       etcdCluster,
			ConsulCluster:     consulRunner.ConsulCluster(),
		}
	})

	SynchronizedAfterSuite(func() {
	}, func() {
		CleanupBuildArtifacts()
	})

	BeforeEach(func() {
		etcdRunner.Start()
		consulRunner.Start()
		consulRunner.WaitUntilReady()

		bbsProcess = ginkgomon.Invoke(bbsrunner.New(binPaths.Bbs, bbsArgs))
		bbsClient = bbs.NewClient(fmt.Sprint("http://", bbsArgs.Address))

		consulSession = consulRunner.NewSession("a-session")
		legacyBBS = Bbs.NewBBS(etcdClient, consulSession, "http://receptor.bogus.com", clock.NewClock(), logger)

		capacity := oldmodels.NewCellCapacity(512, 1024, 124)
		cellPresence := oldmodels.NewCellPresence("the-cell-id", "1.2.3.4", "the-zone", capacity, []string{}, []string{})

		value, err := oldmodels.ToJSON(cellPresence)
		Expect(err).NotTo(HaveOccurred())

		_, err = consulSession.SetPresence(shared.CellSchemaPath(cellPresence.CellID), value)
		Expect(err).NotTo(HaveOccurred())

	})

	AfterEach(func() {
		ginkgomon.Kill(bbsProcess)
		ginkgomon.Kill(convergerProcess)
		consulRunner.Stop()
		etcdRunner.Stop()
	})

	startConverger := func() {
		runner = convergerrunner.New(convergerConfig)
		convergerProcess = ginkgomon.Invoke(runner)
		time.Sleep(convergeRepeatInterval)
	}

	createRunningTaskWithDeadCell := func() {
		task := model_helpers.NewValidTask("task-guid")

		err := bbsClient.DesireTask(task.TaskGuid, task.Domain, task.TaskDefinition)
		Expect(err).NotTo(HaveOccurred())

		_, err = bbsClient.StartTask(task.TaskGuid, "dead-cell")
		Expect(err).NotTo(HaveOccurred())
	}

	itIsInactive := func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("does not converge the task", func() {
				Consistently(func() []*models.Task {
					return getTasksByState(bbsClient, models.Task_Completed)
				}, 10*convergeRepeatInterval).Should(BeEmpty())
			})
		})
	}

	Context("when the converger has the lock", func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("marks the task as completed and failed", func() {
				Consistently(func() []*models.Task {
					return getTasksByState(bbsClient, models.Task_Completed)
				}, 0.5).Should(BeEmpty())

				startConverger()

				Eventually(func() []*models.Task {
					return getTasksByState(bbsClient, models.Task_Completed)
				}, 10*convergeRepeatInterval).Should(HaveLen(1))
			})
		})
	})

	Context("when the converger loses the lock", func() {
		BeforeEach(func() {
			startConverger()
			Eventually(runner, 5*time.Second).Should(gbytes.Say("acquire-lock-succeeded"))

			consulRunner.Reset()
		})

		It("exits with an error", func() {
			Eventually(runner, exitDuration).Should(Exit(1))
		})
	})

	Context("when the converger initially does not have the lock", func() {
		var otherSession *consuladapter.Session

		BeforeEach(func() {
			otherSession = consulRunner.NewSession("other-session")
			err := otherSession.AcquireLock(shared.LockSchemaPath("converge_lock"), []byte("something-else"))
			Expect(err).NotTo(HaveOccurred())

			startConverger()
		})

		itIsInactive()

		Describe("when the lock becomes available", func() {
			BeforeEach(func() {
				otherSession.Destroy()
				time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
			})

			Describe("when a running task with a dead cell is present", func() {
				JustBeforeEach(createRunningTaskWithDeadCell)

				It("eventually marks the task as failed", func() {
					Eventually(func() []*models.Task {
						completedTasks := getTasksByState(bbsClient, models.Task_Completed)
						return failedTasks(completedTasks)
					}, 10*convergeRepeatInterval).Should(HaveLen(1))
				})
			})
		})
	})

	Describe("signal handling", func() {
		BeforeEach(func() {
			startConverger()
		})

		Describe("when it receives SIGINT", func() {
			It("exits successfully", func() {
				convergerProcess.Signal(syscall.SIGINT)
				Eventually(runner, exitDuration).Should(Exit(0))
			})
		})

		Describe("when it receives SIGTERM", func() {
			It("exits successfully", func() {
				convergerProcess.Signal(syscall.SIGTERM)
				Eventually(runner, exitDuration).Should(Exit(0))
			})
		})
	})

	Context("when etcd is down", func() {
		BeforeEach(func() {
			etcdRunner.Stop()
			startConverger()
		})

		It("starts", func() {
			Consistently(runner).ShouldNot(Exit())
		})
	})
})

func getTasksByState(client bbs.Client, state models.Task_State) []*models.Task {
	tasks, err := client.Tasks()
	Expect(err).NotTo(HaveOccurred())

	filteredTasks := make([]*models.Task, 0)
	for _, task := range tasks {
		if task.State == state {
			filteredTasks = append(filteredTasks, task)
		}
	}
	return filteredTasks
}

func failedTasks(tasks []*models.Task) []*models.Task {
	failedTasks := make([]*models.Task, 0)

	for _, task := range tasks {
		if task.Failed {
			failedTasks = append(failedTasks, task)
		}
	}

	return failedTasks
}
<|endoftext|>


<commit_before>package config

import (
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/bobtfish/AWSnycast/aws"
	"github.com/bobtfish/AWSnycast/healthcheck"
	"github.com/bobtfish/AWSnycast/instancemetadata"
	"gopkg.in/yaml.v2"
	"io/ioutil"
)

type Config struct {
	PollTime                   uint                                `yaml:"poll_time"`
	Healthchecks               map[string]*healthcheck.Healthcheck `yaml:"healthchecks"`
	RemoteHealthcheckTemplates map[string]*healthcheck.Healthcheck `yaml:"remote_healthchecks"`
	RouteTables                map[string]*RouteTable              `yaml:"routetables"`
}

type RouteTable struct {
	Find           RouteTableFindSpec      `yaml:"find"`
	ManageRoutes   []*aws.ManageRoutesSpec `yaml:"manage_routes"`
	ec2RouteTables []*ec2.RouteTable
}

type RouteTableFindSpec struct {
	NoResultsOk bool                   `yaml:"no_results_ok"`
	Type        string                 `yaml:"type"`
	Not         bool                   `yaml:"not"`
	Config      map[string]interface{} `yaml:"config"`
}

var routeFindTypes map[string]func(RouteTableFindSpec) (aws.RouteTableFilter, error)

func init() {
	routeFindTypes = make(map[string]func(RouteTableFindSpec) (aws.RouteTableFilter, error))
	routeFindTypes["by_tag"] = func(spec RouteTableFindSpec) (aws.RouteTableFilter, error) {
		if _, ok := spec.Config["key"]; !ok {
			return nil, errors.New("No key in config for by_tag route table finder")
		}
		if _, ok := spec.Config["value"]; !ok {
			return nil, errors.New("No value in config for by_tag route table finder")
		}
		return aws.RouteTableFilterTagMatch{
			Key:   spec.Config["key"].(string),
			Value: spec.Config["value"].(string),
		}, nil
	}
	routeFindTypes["and"] = func(spec RouteTableFindSpec) (aws.RouteTableFilter, error) {
		filters, ok := getFiltersListForSpec(spec)
		if ok != nil {
			return nil, errors.New("No filters in config for and route table finder")
		}
		return aws.RouteTableFilterAnd{filters}, nil
	}
	routeFindTypes["or"] = func(spec RouteTableFindSpec) (aws.RouteTableFilter, error) {
		filters, ok := getFiltersListForSpec(spec)
		if ok != nil {
			return nil, errors.New("No filters in config for or route table finder")
		}
		return aws.RouteTableFilterOr{filters}, nil
	}
	routeFindTypes["main"] = func(spec RouteTableFindSpec) (aws.RouteTableFilter, error) {
		return aws.RouteTableFilterMain{}, nil
	}
	routeFindTypes["subnet"] = func(spec RouteTableFindSpec) (aws.RouteTableFilter, error) {
		if _, ok := spec.Config["subnet_id"]; !ok {
			return nil, errors.New("No subnet_id in config for subnet route table finder")
		}
		return aws.RouteTableFilterSubnet{spec.Config["subnet_id"].(string)}, nil
	}
	routeFindTypes["has_route_to"] = func(spec RouteTableFindSpec) (aws.RouteTableFilter, error) {
		if _, ok := spec.Config["cidr"]; !ok {
			return nil, errors.New("No cidr in config for has_route_to route table finder")
		}
		return aws.RouteTableFilterDestinationCidrBlock{DestinationCidrBlock: spec.Config["cidr"].(string)}, nil
	}
}

func getFiltersListForSpec(spec RouteTableFindSpec) ([]aws.RouteTableFilter, error) {
	v, ok := spec.Config["filters"]
	if !ok {
		return nil, errors.New("No filters in config for route table finder")
	}
	var filters []aws.RouteTableFilter
	switch t := v.(type) {
	default:
		return filters, errors.New(fmt.Sprintf("unexpected type %T\n", t))
	case []interface{}:
		for _, filter := range t { // I REGRET NOTHING
			filterRepacked, err := yaml.Marshal(filter)
			if err != nil {
				return nil, err
			}
			var spec RouteTableFindSpec
			err = yaml.Unmarshal(filterRepacked, &spec)
			if err != nil {
				return nil, err
			}
			filter, err := spec.GetFilter()
			if err != nil {
				return nil, err
			}
			filters = append(filters, filter)
		} // End lack of regret
	}
	return filters, nil
}

func (spec RouteTableFindSpec) GetFilter() (aws.RouteTableFilter, error) {
	if genFilter, found := routeFindTypes[spec.Type]; found {
		filter, err := genFilter(spec)
		if err != nil {
			return filter, err
		}
		if spec.Not {
			return aws.RouteTableFilterNot{filter}, nil
		}
		return filter, nil
	}
	return nil, errors.New(fmt.Sprintf("Route table finder type '%s' not found in the registry", spec.Type))
}

func (r *RouteTable) UpdateEc2RouteTables(rt []*ec2.RouteTable) error {
	filter, err := r.Find.GetFilter()
	if err != nil {
		return err
	}
	r.ec2RouteTables = aws.FilterRouteTables(filter, rt)
	if len(r.ec2RouteTables) == 0 {
		if r.Find.NoResultsOk {
			return nil
		}
		return errors.New("No route table in AWS matched filter spec")
	}
	for _, manage := range r.ManageRoutes {
		manage.UpdateEc2RouteTables(r.ec2RouteTables)
	}
	return nil
}

func (r *RouteTable) RunEc2Updates(manager aws.RouteTableManager, noop bool) error {
	for _, rtb := range r.ec2RouteTables {
		contextLogger := log.WithFields(log.Fields{
			"rtb": *(rtb.RouteTableId),
		})
		contextLogger.Debug("Finder found route table")
		for _, manageRoute := range r.ManageRoutes {
			contextLogger.WithFields(log.Fields{"cidr": manageRoute.Cidr}).Debug("Trying to manage route")
			if err := manager.ManageInstanceRoute(*rtb, *manageRoute, noop); err != nil {
				return err
			}
		}
	}
	return nil
}

func (c *Config) Default(im instancemetadata.InstanceMetadata, manager aws.RouteTableManager) {
	if c.PollTime == 0 {
		c.PollTime = 300 // Default to every 5m
	}
	if c.Healthchecks == nil {
		c.Healthchecks = make(map[string]*healthcheck.Healthcheck)
	}
	if c.RemoteHealthcheckTemplates == nil {
		c.RemoteHealthcheckTemplates = make(map[string]*healthcheck.Healthcheck)
	}
	if c.RouteTables != nil {
		for _, v := range c.RouteTables {
			v.Default(im.Instance, manager)
		}
	} else {
		c.RouteTables = make(map[string]*RouteTable)
	}
	for _, v := range c.Healthchecks {
		v.Default()
	}
	for _, v := range c.RemoteHealthcheckTemplates {
		v.Default()
	}
}

func (c Config) Validate() error {
	if c.RouteTables == nil {
		return errors.New("No route_tables key in config")
	}
	if len(c.RouteTables) == 0 {
		return errors.New("No route_tables defined in config")
	}
	if c.Healthchecks != nil {
		for k, v := range c.Healthchecks {
			if err := v.Validate(k, false); err != nil {
				return err
			}
		}
	}
	for k, v := range c.RouteTables {
		if err := v.Validate(k, c.Healthchecks); err != nil {
			return err
		}
	}
	return nil
}

func (r *RouteTableFindSpec) Default() {
	if r.Config == nil {
		r.Config = make(map[string]interface{})
	}
}
func (r *RouteTableFindSpec) Validate(name string) error {
	if r.Type == "" {
		return errors.New(fmt.Sprintf("Route find spec %s needs a type key", name))
	}
	if r.Type != "by_tag" {
		return errors.New(fmt.Sprintf("Route find spec %s type '%s' not known", name, r.Type))
	}
	if r.Config == nil {
		return errors.New("No config supplied")
	}
	return nil
}

func (r *RouteTable) Default(instance string, manager aws.RouteTableManager) {
	r.Find.Default()
	if r.ManageRoutes == nil {
		r.ManageRoutes = make([]*aws.ManageRoutesSpec, 0)
	}
	for _, v := range r.ManageRoutes {
		v.Default(instance, manager)
	}
	if r.ec2RouteTables == nil {
		r.ec2RouteTables = make([]*ec2.RouteTable, 0)
	}
}
func (r RouteTable) Validate(name string, healthchecks map[string]*healthcheck.Healthcheck) error {
	if r.ManageRoutes == nil || len(r.ManageRoutes) == 0 {
		return errors.New(fmt.Sprintf("No manage_routes key in route table '%s'", name))
	}
	for _, v := range r.ManageRoutes {
		if err := v.Validate(name, healthchecks); err != nil {
			return err
		}
	}
	return nil
}

func New(filename string, im instancemetadata.InstanceMetadata, manager aws.RouteTableManager) (*Config, error) {
	c := new(Config)
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return c, err
	}
	err = yaml.Unmarshal(data, &c)
	if err == nil {
		c.Default(im, manager)
		err = c.Validate()
	}
	return c, err
}
<commit_msg>Make remote healthchecks themselves a thing<commit_after>package config

import (
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/bobtfish/AWSnycast/aws"
	"github.com/bobtfish/AWSnycast/healthcheck"
	"github.com/bobtfish/AWSnycast/instancemetadata"
	"gopkg.in/yaml.v2"
	"io/ioutil"
)

type Config struct {
	PollTime                   uint                                           `yaml:"poll_time"`
	Healthchecks               map[string]*healthcheck.Healthcheck            `yaml:"healthchecks"`
	RemoteHealthcheckTemplates map[string]*healthcheck.Healthcheck            `yaml:"remote_healthchecks"`
	RemoteHealthchecks         map[string]map[string]*healthcheck.Healthcheck `yaml:"-"`
	RouteTables                map[string]*RouteTable                         `yaml:"routetables"`
}

type RouteTable struct {
	Find           RouteTableFindSpec      `yaml:"find"`
	ManageRoutes   []*aws.ManageRoutesSpec `yaml:"manage_routes"`
	ec2RouteTables []*ec2.RouteTable
}

type RouteTableFindSpec struct {
	NoResultsOk bool                   `yaml:"no_results_ok"`
	Type        string                 `yaml:"type"`
	Not         bool                   `yaml:"not"`
	Config      map[string]interface{} `yaml:"config"`
}

var routeFindTypes map[string]func(RouteTableFindSpec) (aws.RouteTableFilter, error)

func init() {
	routeFindTypes = make(map[string]func(RouteTableFindSpec) (aws.RouteTableFilter, error))
	routeFindTypes["by_tag"] = func(spec RouteTableFindSpec) (aws.RouteTableFilter, error) {
		if _, ok := spec.Config["key"]; !ok {
			return nil, errors.New("No key in config for by_tag route table finder")
		}
		if _, ok := spec.Config["value"]; !ok {
			return nil, errors.New("No value in config for by_tag route table finder")
		}
		return aws.RouteTableFilterTagMatch{
			Key:   spec.Config["key"].(string),
			Value: spec.Config["value"].(string),
		}, nil
	}
	routeFindTypes["and"] = func(spec RouteTableFindSpec) (aws.RouteTableFilter, error) {
		filters, ok := getFiltersListForSpec(spec)
		if ok != nil {
			return nil, errors.New("No filters in config for and route table finder")
		}
		return aws.RouteTableFilterAnd{filters}, nil
	}
	routeFindTypes["or"] = func(spec RouteTableFindSpec) (aws.RouteTableFilter, error) {
		filters, ok := getFiltersListForSpec(spec)
		if ok != nil {
			return nil, errors.New("No filters in config for or route table finder")
		}
		return aws.RouteTableFilterOr{filters}, nil
	}
	routeFindTypes["main"] = func(spec RouteTableFindSpec) (aws.RouteTableFilter, error) {
		return aws.RouteTableFilterMain{}, nil
	}
	routeFindTypes["subnet"] = func(spec RouteTableFindSpec) (aws.RouteTableFilter, error) {
		if _, ok := spec.Config["subnet_id"]; !ok {
			return nil, errors.New("No subnet_id in config for subnet route table finder")
		}
		return aws.RouteTableFilterSubnet{spec.Config["subnet_id"].(string)}, nil
	}
	routeFindTypes["has_route_to"] = func(spec RouteTableFindSpec) (aws.RouteTableFilter, error) {
		if _, ok := spec.Config["cidr"]; !ok {
			return nil, errors.New("No cidr in config for has_route_to route table finder")
		}
		return aws.RouteTableFilterDestinationCidrBlock{DestinationCidrBlock: spec.Config["cidr"].(string)}, nil
	}
}

func getFiltersListForSpec(spec RouteTableFindSpec) ([]aws.RouteTableFilter, error) {
	v, ok := spec.Config["filters"]
	if !ok {
		return nil, errors.New("No filters in config for route table finder")
	}
	var filters []aws.RouteTableFilter
	switch t := v.(type) {
	default:
		return filters, errors.New(fmt.Sprintf("unexpected type %T\n", t))
	case []interface{}:
		for _, filter := range t { // I REGRET NOTHING
			filterRepacked, err := yaml.Marshal(filter)
			if err != nil {
				return nil, err
			}
			var spec RouteTableFindSpec
			err = yaml.Unmarshal(filterRepacked, &spec)
			if err != nil {
				return nil, err
			}
			filter, err := spec.GetFilter()
			if err != nil {
				return nil, err
			}
			filters = append(filters, filter)
		} // End lack of regret
	}
	return filters, nil
}

func (spec RouteTableFindSpec) GetFilter() (aws.RouteTableFilter, error) {
	if genFilter, found := routeFindTypes[spec.Type]; found {
		filter, err := genFilter(spec)
		if err != nil {
			return filter, err
		}
		if spec.Not {
			return aws.RouteTableFilterNot{filter}, nil
		}
		return filter, nil
	}
	return nil, errors.New(fmt.Sprintf("Route table finder type '%s' not found in the registry", spec.Type))
}

func (r *RouteTable) UpdateEc2RouteTables(rt []*ec2.RouteTable) error {
	filter, err := r.Find.GetFilter()
	if err != nil {
		return err
	}
	r.ec2RouteTables = aws.FilterRouteTables(filter, rt)
	if len(r.ec2RouteTables) == 0 {
		if r.Find.NoResultsOk {
			return nil
		}
		return errors.New("No route table in AWS matched filter spec")
	}
	for _, manage := range r.ManageRoutes {
		manage.UpdateEc2RouteTables(r.ec2RouteTables)
	}
	return nil
}

func (r *RouteTable) RunEc2Updates(manager aws.RouteTableManager, noop bool) error {
	for _, rtb := range r.ec2RouteTables {
		contextLogger := log.WithFields(log.Fields{
			"rtb": *(rtb.RouteTableId),
		})
		contextLogger.Debug("Finder found route table")
		for _, manageRoute := range r.ManageRoutes {
			contextLogger.WithFields(log.Fields{"cidr": manageRoute.Cidr}).Debug("Trying to manage route")
			if err := manager.ManageInstanceRoute(*rtb, *manageRoute, noop); err != nil {
				return err
			}
		}
	}
	return nil
}

func (c *Config) Default(im instancemetadata.InstanceMetadata, manager aws.RouteTableManager) {
	if c.PollTime == 0 {
		c.PollTime = 300 // Default to every 5m
	}
	if c.Healthchecks == nil {
		c.Healthchecks = make(map[string]*healthcheck.Healthcheck)
	}
	if c.RemoteHealthcheckTemplates == nil {
		c.RemoteHealthcheckTemplates = make(map[string]*healthcheck.Healthcheck)
	}
	c.RemoteHealthchecks = make(map[string]map[string]*healthcheck.Healthcheck)
	if c.RouteTables != nil {
		for _, v := range c.RouteTables {
			v.Default(im.Instance, manager)
		}
	} else {
		c.RouteTables = make(map[string]*RouteTable)
	}
	for _, v := range c.Healthchecks {
		v.Default()
	}
	for _, v := range c.RemoteHealthcheckTemplates {
		v.Default()
	}
}

func (c Config) Validate() error {
	if c.RouteTables == nil {
		return errors.New("No route_tables key in config")
	}
	if len(c.RouteTables) == 0 {
		return errors.New("No route_tables defined in config")
	}
	if c.Healthchecks != nil {
		for k, v := range c.Healthchecks {
			if err := v.Validate(k, false); err != nil {
				return err
			}
		}
	}
	for k, v := range c.RouteTables {
		if err := v.Validate(k, c.Healthchecks); err != nil {
			return err
		}
	}
	return nil
}

func (r *RouteTableFindSpec) Default() {
	if r.Config == nil {
		r.Config = make(map[string]interface{})
	}
}
func (r *RouteTableFindSpec) Validate(name string) error {
	if r.Type == "" {
		return errors.New(fmt.Sprintf("Route find spec %s needs a type key", name))
	}
	if r.Type != "by_tag" {
		return errors.New(fmt.Sprintf("Route find spec %s type '%s' not known", name, r.Type))
	}
	if r.Config == nil {
		return errors.New("No config supplied")
	}
	return nil
}

func (r *RouteTable) Default(instance string, manager aws.RouteTableManager) {
	r.Find.Default()
	if r.ManageRoutes == nil {
		r.ManageRoutes = make([]*aws.ManageRoutesSpec, 0)
	}
	for _, v := range r.ManageRoutes {
		v.Default(instance, manager)
	}
	if r.ec2RouteTables == nil {
		r.ec2RouteTables = make([]*ec2.RouteTable, 0)
	}
}
func (r RouteTable) Validate(name string, healthchecks map[string]*healthcheck.Healthcheck) error {
	if r.ManageRoutes == nil || len(r.ManageRoutes) == 0 {
		return errors.New(fmt.Sprintf("No manage_routes key in route table '%s'", name))
	}
	for _, v := range r.ManageRoutes {
		if err := v.Validate(name, healthchecks); err != nil {
			return err
		}
	}
	return nil
}

func New(filename string, im instancemetadata.InstanceMetadata, manager aws.RouteTableManager) (*Config, error) {
	c := new(Config)
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return c, err
	}
	err = yaml.Unmarshal(data, &c)
	if err == nil {
		c.Default(im, manager)
		err = c.Validate()
	}
	return c, err
}
<|endoftext|>


<commit_before>package serve

import (
    "fmt"
    "log"
    "net/http"
    "net/url"
    "strings"
)

var rewriteJsTpl = "function pproxy_rewrite(req){\n%s\nreturn req;\n}"

func (ser *ProxyServe) parseAndSaveRewriteJs(jsStr string) error {
    rewriteJs := fmt.Sprintf(rewriteJsTpl, jsStr)
    js.Run(rewriteJs)
    jsFn, err := js.Get("pproxy_rewrite")
    if err == nil {
        ser.RewriteJs = jsStr
        ser.RewriteJsFn = jsFn
    }
    return err
}

func (ser *ProxyServe) reqRewriteByjs(req *http.Request) {
    if ser.RewriteJs == "" {
        return
    }
    urlObj, _ := js.Object(`req={}`)
    urlObj.Set("url", req.URL.String())
    urlObj.Set("schema", req.URL.Scheme)

    host_info := strings.Split(req.URL.Host, ":")
    urlObj.Set("host", host_info[0])
    if len(host_info) == 2 {
        urlObj.Set("port", host_info[1])
    } else {
        urlObj.Set("port", "")
    }

    urlObj.Set("path", req.URL.Path)
    urlObj.Set("rawquery", req.URL.RawQuery)
    username := ""
    psw := ""
    if req.URL.User != nil {
        username = req.URL.User.Username()
        psw, _ = req.URL.User.Password()
    }
    urlObj.Set("username", username)
    urlObj.Set("password", psw)

    js_ret, err_js := ser.RewriteJsFn.Call(ser.RewriteJsFn, urlObj)

    if err_js == nil {
        if js_ret.IsObject() {
            obj, export_err := js_ret.Export()
            if export_err == nil {
                url_obj := obj.(map[string]interface{})
                schema := getMapValStr(url_obj, "schema")
                url_new := schema + "://"
                username := getMapValStr(url_obj, "username")
                if username != "" {
                    url_new += fmt.Sprintf("%s:%s@", username, getMapValStr(url_obj, "password"))
                }
                host := getMapValStr(url_obj, "host")
                port := getMapValStr(url_obj, "port")
                if port != "" {
                    host += ":" + port
                }
                url_new += fmt.Sprintf("%s%s", host, getMapValStr(url_obj, "path"))

                rawquery := getMapValStr(url_obj, "rawquery")
                if rawquery != "" {
                    url_new += "?" + rawquery
                }
                if url_new == req.URL.String() {
                    return
                }
                host_addr := getMapValStr(url_obj, "host_addr")

                var url_err error
                req.URL, url_err = url.Parse(url_new)
                if ser.Debug {
                    log.Println("DEBUG req_rewrite,url_new:", url_new, "req_new:", req.URL)
                }
                if url_err != nil {
                    log.Println("js filter err:", js_ret, url_err)
                } else {
                    req.Host = req.URL.Host
                    if host_addr != "" {
                        req.URL.Host = host_addr
                    }
                }
            } else {
                log.Println("js filter result wrong", js_ret.String())
            }
        }
    } else {
        log.Println("js filter err:", err_js, js_ret)
    }
}

func (ser *ProxyServe) reqRewrite(req *http.Request) {
    ser.reqRewriteByjs(req)
    ser.reqRewriteByHosts(req)
}

func (ser *ProxyServe) reqRewriteByHosts(req *http.Request) {
    if ser.hosts == nil {
        return
    }
    if host, has := ser.hosts[req.Host]; has {
        log.Println("rewrite host:", req.Host, "==>", host)
        req.URL.Host = host
        return
    }
    host_info := strings.Split(req.URL.Host, ":")
    if len(host_info) == 1 {
        if req.URL.Scheme == "http" {
            host_info = append(host_info, "80")
        }
    }
    req_host := strings.Join(host_info, ":")
    if host, has := ser.hosts[req_host]; has {
        log.Println("rewrite host:", req.Host, "==>", host)
        req.URL.Host = host
        return
    }
    if host, has := ser.hosts[host_info[0]]; has {
        log.Println("rewrite host:", req.Host, "==>", host)
        req.URL.Host = host
        return
    }
}
<commit_msg>fix hosts<commit_after>package serve

import (
    "fmt"
    "log"
    "net/http"
    "net/url"
    "strings"
)

var rewriteJsTpl = "function pproxy_rewrite(req){\n%s\nreturn req;\n}"

func (ser *ProxyServe) parseAndSaveRewriteJs(jsStr string) error {
    rewriteJs := fmt.Sprintf(rewriteJsTpl, jsStr)
    js.Run(rewriteJs)
    jsFn, err := js.Get("pproxy_rewrite")
    if err == nil {
        ser.RewriteJs = jsStr
        ser.RewriteJsFn = jsFn
    }
    return err
}

func (ser *ProxyServe) reqRewriteByjs(req *http.Request) {
    if ser.RewriteJs == "" {
        return
    }
    urlObj, _ := js.Object(`req={}`)
    urlObj.Set("url", req.URL.String())
    urlObj.Set("schema", req.URL.Scheme)

    host_info := strings.Split(req.URL.Host, ":")
    urlObj.Set("host", host_info[0])
    if len(host_info) == 2 {
        urlObj.Set("port", host_info[1])
    } else {
        urlObj.Set("port", "")
    }

    urlObj.Set("path", req.URL.Path)
    urlObj.Set("rawquery", req.URL.RawQuery)
    username := ""
    psw := ""
    if req.URL.User != nil {
        username = req.URL.User.Username()
        psw, _ = req.URL.User.Password()
    }
    urlObj.Set("username", username)
    urlObj.Set("password", psw)

    js_ret, err_js := ser.RewriteJsFn.Call(ser.RewriteJsFn, urlObj)

    if err_js == nil {
        if js_ret.IsObject() {
            obj, export_err := js_ret.Export()
            if export_err == nil {
                url_obj := obj.(map[string]interface{})
                schema := getMapValStr(url_obj, "schema")
                url_new := schema + "://"
                username := getMapValStr(url_obj, "username")
                if username != "" {
                    url_new += fmt.Sprintf("%s:%s@", username, getMapValStr(url_obj, "password"))
                }
                host := getMapValStr(url_obj, "host")
                port := getMapValStr(url_obj, "port")
                if port != "" {
                    host += ":" + port
                }
                url_new += fmt.Sprintf("%s%s", host, getMapValStr(url_obj, "path"))

                rawquery := getMapValStr(url_obj, "rawquery")
                if rawquery != "" {
                    url_new += "?" + rawquery
                }
                if url_new == req.URL.String() {
                    return
                }
                host_addr := getMapValStr(url_obj, "host_addr")

                var url_err error
                req.URL, url_err = url.Parse(url_new)
                if ser.Debug {
                    log.Println("DEBUG req_rewrite,url_new:", url_new, "req_new:", req.URL)
                }
                if url_err != nil {
                    log.Println("js filter err:", js_ret, url_err)
                } else {
                    req.Host = req.URL.Host
                    if host_addr != "" {
                        req.URL.Host = host_addr
                    }
                }
            } else {
                log.Println("js filter result wrong", js_ret.String())
            }
        }
    } else {
        log.Println("js filter err:", err_js, js_ret)
    }
}

func (ser *ProxyServe) reqRewrite(req *http.Request) {
    ser.reqRewriteByjs(req)
    ser.reqRewriteByHosts(req)
}

func (ser *ProxyServe) reqRewriteByHosts(req *http.Request) {
    if ser.hosts == nil {
        return
    }
    if host, has := ser.hosts[req.URL.Host]; has {
        log.Println("rewrite host:", req.URL.Host, "==>", host)
        req.URL.Host = host
        return
    }
    host_info := strings.Split(req.URL.Host, ":")
    if len(host_info) == 1 {
        if req.URL.Scheme == "http" {
            host_info = append(host_info, "80")
        }
    }
    req_host := strings.Join(host_info, ":")
    if host, has := ser.hosts[req_host]; has {
        log.Println("rewrite host:", req.Host, "==>", host)
        req.URL.Host = host
        return
    }
    if host, has := ser.hosts[host_info[0]]; has {
        log.Println("rewrite host:", req.Host, "==>", host)
        req.URL.Host = host
        return
    }
}
<|endoftext|>


<commit_before>package sms // import "github.com/mndrix/sms-over-xmpp"
import (
	"errors"

	xco "github.com/mndrix/go-xco"
)

// StaticConfig intends to implement the Config interface
var _ Config = &StaticConfig{}

type StaticConfig struct {
	Http HttpConfig `toml:"http"`

	Xmpp StaticConfigXmpp `toml:"xmpp"`

	// Phones maps an E.164 phone number to an XMPP address.  If a
	// mapping is not found here, the inverse of Users is considered.
	Phones map[string]string `toml:"phones"`

	// Users maps an XMPP address to an E.164 phone number.
	Users map[string]string `toml:"users"`

	// Twilio contains optional account details for making API calls via the
	// Twilio service.
	Twilio *TwilioConfig `toml:"twilio"`
}

type HttpConfig struct {
	Host string `toml:"host"`
	Port int    `toml:"port"`
}

type StaticConfigXmpp struct {
	Host   string `toml:"host"`
	Name   string `toml:"name"`
	Port   int    `toml:"port"`
	Secret string `toml:"secret"`
}

type TwilioConfig struct {
	AccountSid string `toml:"account-sid"`
	KeySid     string `toml:"key-sid"`
	KeySecret  string `toml:"key-secret"`
}

func (self *StaticConfig) ComponentName() string {
	return self.Xmpp.Name
}

func (self *StaticConfig) SharedSecret() string {
	return self.Xmpp.Secret
}

func (self *StaticConfig) HttpHost() string {
	return self.Http.Host
}

func (self *StaticConfig) HttpPort() int {
	port := self.Http.Port
	if port == 0 {
		port = 9677
	}
	return port
}

func (self *StaticConfig) XmppHost() string {
	return self.Xmpp.Host
}

func (self *StaticConfig) XmppPort() int {
	return self.Xmpp.Port
}

func (self *StaticConfig) AddressToPhone(addr xco.Address) (string, error) {
	e164, ok := self.Users[addr.LocalPart+"@"+addr.DomainPart]
	if ok {
		return e164, nil
	}

	// assume the name is already a phone number
	return addr.LocalPart, nil
}

func (self *StaticConfig) PhoneToAddress(e164 string) (xco.Address, error) {
	// is there an explicit mapping?
	jid, ok := self.Phones[e164]
	if ok {
		return xco.ParseAddress(jid)
	}

	// maybe there's an implicit mapping
	for jid, phone := range self.Users {
		if phone == e164 {
			return xco.ParseAddress(jid)
		}
	}

	// assume the phone number is the user name
	addr := xco.Address{
		LocalPart:  e164,
		DomainPart: self.Xmpp.Name,
	}
	return addr, nil
}

func (self *StaticConfig) SmsProvider() (SmsProvider, error) {
	if self.Twilio == nil {
		return nil, errors.New("Need to configure an SMS provider")
	}
	twilio := &Twilio{
		accountSid: self.Twilio.AccountSid,
		keySid:     self.Twilio.KeySid,
		keySecret:  self.Twilio.KeySecret,
	}
	return twilio, nil
}
<commit_msg>Default for HTTP listen host<commit_after>package sms // import "github.com/mndrix/sms-over-xmpp"
import (
	"errors"

	xco "github.com/mndrix/go-xco"
)

// StaticConfig intends to implement the Config interface
var _ Config = &StaticConfig{}

type StaticConfig struct {
	Http HttpConfig `toml:"http"`

	Xmpp StaticConfigXmpp `toml:"xmpp"`

	// Phones maps an E.164 phone number to an XMPP address.  If a
	// mapping is not found here, the inverse of Users is considered.
	Phones map[string]string `toml:"phones"`

	// Users maps an XMPP address to an E.164 phone number.
	Users map[string]string `toml:"users"`

	// Twilio contains optional account details for making API calls via the
	// Twilio service.
	Twilio *TwilioConfig `toml:"twilio"`
}

type HttpConfig struct {
	Host string `toml:"host"`
	Port int    `toml:"port"`
}

type StaticConfigXmpp struct {
	Host   string `toml:"host"`
	Name   string `toml:"name"`
	Port   int    `toml:"port"`
	Secret string `toml:"secret"`
}

type TwilioConfig struct {
	AccountSid string `toml:"account-sid"`
	KeySid     string `toml:"key-sid"`
	KeySecret  string `toml:"key-secret"`
}

func (self *StaticConfig) ComponentName() string {
	return self.Xmpp.Name
}

func (self *StaticConfig) SharedSecret() string {
	return self.Xmpp.Secret
}

func (self *StaticConfig) HttpHost() string {
	host := self.Http.Host
	if host == "" {
		host = "127.0.0.1"
	}
	return host
}

func (self *StaticConfig) HttpPort() int {
	port := self.Http.Port
	if port == 0 {
		port = 9677
	}
	return port
}

func (self *StaticConfig) XmppHost() string {
	return self.Xmpp.Host
}

func (self *StaticConfig) XmppPort() int {
	return self.Xmpp.Port
}

func (self *StaticConfig) AddressToPhone(addr xco.Address) (string, error) {
	e164, ok := self.Users[addr.LocalPart+"@"+addr.DomainPart]
	if ok {
		return e164, nil
	}

	// assume the name is already a phone number
	return addr.LocalPart, nil
}

func (self *StaticConfig) PhoneToAddress(e164 string) (xco.Address, error) {
	// is there an explicit mapping?
	jid, ok := self.Phones[e164]
	if ok {
		return xco.ParseAddress(jid)
	}

	// maybe there's an implicit mapping
	for jid, phone := range self.Users {
		if phone == e164 {
			return xco.ParseAddress(jid)
		}
	}

	// assume the phone number is the user name
	addr := xco.Address{
		LocalPart:  e164,
		DomainPart: self.Xmpp.Name,
	}
	return addr, nil
}

func (self *StaticConfig) SmsProvider() (SmsProvider, error) {
	if self.Twilio == nil {
		return nil, errors.New("Need to configure an SMS provider")
	}
	twilio := &Twilio{
		accountSid: self.Twilio.AccountSid,
		keySid:     self.Twilio.KeySid,
		keySecret:  self.Twilio.KeySecret,
	}
	return twilio, nil
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package main

import (
	"database/sql"
	"log"
	"net"
	"os"
	"runtime"
	"sync"
	"time"

	_ "github.com/go-sql-driver/mysql"

	"github.com/go-gorp/gorp"
	"github.com/jcjones/ct-sql/sqldb"
	"github.com/jcjones/ct-sql/utils"
	"github.com/oschwald/geoip2-golang"
)

type ResolutionEntry struct {
	NameID uint64
	Name   string
	Time   *time.Time
	Ipaddr *string
}

var (
	config = utils.NewCTConfig()
)

func main() {
	log.SetFlags(0)
	log.SetPrefix("")
	dbConnectStr, err := sqldb.RecombineURLForDB(*config.DbConnect)
	if err != nil {
		log.Printf("unable to parse %s: %s", *config.DbConnect, err)
	}

	if len(dbConnectStr) == 0 || len(*config.GeoipDbPath) == 0 {
		config.Usage()
		os.Exit(2)
	}

	db, err := sql.Open("mysql", dbConnectStr)
	if err != nil {
		log.Fatalf("unable to open SQL: %s: %s", dbConnectStr, err)
	}
	if err = db.Ping(); err != nil {
		log.Fatalf("unable to ping SQL: %s: %s", dbConnectStr, err)
	}

	dialect := gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"}
	dbMap := &gorp.DbMap{Db: db, Dialect: dialect}
	entriesDb := &sqldb.EntriesDatabase{DbMap: dbMap,
		Verbose: *config.Verbose, KnownIssuers: make(map[string]int)}
	err = entriesDb.InitTables()
	if err != nil {
		log.Fatalf("unable to prepare SQL DB. dbConnectStr=%s: %s", dbConnectStr, err)
	}

	geoDB, err := geoip2.Open(*config.GeoipDbPath)
	if err != nil {
		log.Fatalf("unable to prepare GeoIP DB. geoipDbPath=%s: %s", *config.GeoipDbPath, err)
	}
	defer geoDB.Close()

	netscan := &NetScan{
		wg:    new(sync.WaitGroup),
		db:    entriesDb,
		geodb: geoDB,
	}

	if *config.Limit == 0 {
		// Didn't include a mandatory action, so print usage and exit.
		log.Fatalf("You must set a limit")
	}

	oldestAllowed := time.Now().AddDate(-1, 0, 0)

	var entries []ResolutionEntry
	_, err = dbMap.Select(&entries,
		`SELECT f.nameID, f.name, r.time, r.ipaddr FROM
          fqdn AS f
          NATURAL LEFT JOIN resolvedname AS r
      WHERE
          r.time < :oldestAllowed OR r.time IS NULL
      LIMIT :limit`,
		map[string]interface{}{
			"oldestAllowed": oldestAllowed,
			"limit":         *config.Limit,
		})

	if err != nil {
		log.Fatalf("unable to execute SQL: %s", err)
	}

	err = netscan.processEntries(entries)

	if err != nil {
		log.Fatalf("error while running importer: %s", err)
	}

	netscan.wg.Wait()
	os.Exit(0)
}

type NetScan struct {
	db    *sqldb.EntriesDatabase
	wg    *sync.WaitGroup
	geodb *geoip2.Reader
}

func (ns *NetScan) resolveWorker(entries <-chan ResolutionEntry) {
	ns.wg.Add(1)
	defer ns.wg.Done()
	for e := range entries {
		ips, err := net.LookupIP(e.Name)
		if err != nil {
			if *config.Verbose {
				log.Printf("Could not lookup host %s: %s", e.Name, err)
			}

			// Insert a blank record so we know this one didn't work in the future
			ns.db.InsertResolvedName(e.NameID, "")
			// Can't proceed with the geo work since the IP didn't resolve
			continue
		}

		// Log each resolved IP
		for _, ip := range ips {
			ns.db.InsertResolvedName(e.NameID, ip.String())
		}

		// Look up the geo-ip data for the first resolved IP
		geoRecord, err := ns.geodb.City(ips[0])
		if err != nil {
			if *config.Verbose {
				log.Printf("Could not lookup geo-ip record for host %s: %s", e.Name, err)
			}
			continue
		}

		// Log the geo-ip data
		ns.db.InsertResolvedPlace(e.NameID, geoRecord.City.Names["en"],
			geoRecord.Country.IsoCode, geoRecord.Continent.Names["en"])
	}
}

func (ns *NetScan) processEntries(entries []ResolutionEntry) error {
	entryChan := make(chan ResolutionEntry, 10)
	defer close(entryChan)
	ns.wg.Add(1)
	defer ns.wg.Done()
	progressDisplay := utils.NewProgressDisplay()
	defer progressDisplay.Close()

	progressDisplay.StartDisplay(ns.wg)

	numWorkers := *config.NumThreads * runtime.NumCPU()
	for i := 0; i < numWorkers; i++ {
		go ns.resolveWorker(entryChan)
	}

	for i, entry := range entries {
		entryChan <- entry
		if i%256 == 0 {
			progressDisplay.UpdateProgress(0, int64(i), int64(len(entries)))
		}
	}

	return nil
}
<commit_msg>Special-case wildcards in ct-netscan<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package main

import (
	"database/sql"
	"log"
	"net"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	_ "github.com/go-sql-driver/mysql"

	"github.com/go-gorp/gorp"
	"github.com/jcjones/ct-sql/sqldb"
	"github.com/jcjones/ct-sql/utils"
	"github.com/oschwald/geoip2-golang"
)

type ResolutionEntry struct {
	NameID uint64
	Name   string
	Time   *time.Time
	Ipaddr *string
}

var (
	config = utils.NewCTConfig()
)

func main() {
	log.SetFlags(0)
	log.SetPrefix("")
	dbConnectStr, err := sqldb.RecombineURLForDB(*config.DbConnect)
	if err != nil {
		log.Printf("unable to parse %s: %s", *config.DbConnect, err)
	}

	if len(dbConnectStr) == 0 || len(*config.GeoipDbPath) == 0 {
		config.Usage()
		os.Exit(2)
	}

	db, err := sql.Open("mysql", dbConnectStr)
	if err != nil {
		log.Fatalf("unable to open SQL: %s: %s", dbConnectStr, err)
	}
	if err = db.Ping(); err != nil {
		log.Fatalf("unable to ping SQL: %s: %s", dbConnectStr, err)
	}

	dialect := gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"}
	dbMap := &gorp.DbMap{Db: db, Dialect: dialect}
	entriesDb := &sqldb.EntriesDatabase{DbMap: dbMap,
		Verbose: *config.Verbose, KnownIssuers: make(map[string]int)}
	err = entriesDb.InitTables()
	if err != nil {
		log.Fatalf("unable to prepare SQL DB. dbConnectStr=%s: %s", dbConnectStr, err)
	}

	geoDB, err := geoip2.Open(*config.GeoipDbPath)
	if err != nil {
		log.Fatalf("unable to prepare GeoIP DB. geoipDbPath=%s: %s", *config.GeoipDbPath, err)
	}
	defer geoDB.Close()

	netscan := &NetScan{
		wg:    new(sync.WaitGroup),
		db:    entriesDb,
		geodb: geoDB,
	}

	if *config.Limit == 0 {
		// Didn't include a mandatory action, so print usage and exit.
		log.Fatalf("You must set a limit")
	}

	oldestAllowed := time.Now().AddDate(-1, 0, 0)

	var entries []ResolutionEntry
	_, err = dbMap.Select(&entries,
		`SELECT f.nameID, f.name, r.time, r.ipaddr FROM
          fqdn AS f
          NATURAL LEFT JOIN resolvedname AS r
      WHERE
          r.time < :oldestAllowed OR r.time IS NULL
      LIMIT :limit`,
		map[string]interface{}{
			"oldestAllowed": oldestAllowed,
			"limit":         *config.Limit,
		})

	if err != nil {
		log.Fatalf("unable to execute SQL: %s", err)
	}

	err = netscan.processEntries(entries)

	if err != nil {
		log.Fatalf("error while running importer: %s", err)
	}

	netscan.wg.Wait()
	os.Exit(0)
}

type NetScan struct {
	db    *sqldb.EntriesDatabase
	wg    *sync.WaitGroup
	geodb *geoip2.Reader
}

func (ns *NetScan) resolveWorker(entries <-chan ResolutionEntry) {
	ns.wg.Add(1)
	defer ns.wg.Done()
	for e := range entries {
		if strings.Contains(e.Name, "*") {
			// Is a wildcard, we can't resolve it.
			ns.db.InsertResolvedName(e.NameID, "")
			continue
		}

		ips, err := net.LookupIP(e.Name)
		if err != nil {
			if *config.Verbose {
				log.Printf("Could not lookup host %s: %s", e.Name, err)
			}

			// Insert a blank record so we know this one didn't work in the future
			ns.db.InsertResolvedName(e.NameID, "")
			// Can't proceed with the geo work since the IP didn't resolve
			continue
		}

		// Log each resolved IP
		for _, ip := range ips {
			ns.db.InsertResolvedName(e.NameID, ip.String())
		}

		// Look up the geo-ip data for the first resolved IP
		geoRecord, err := ns.geodb.City(ips[0])
		if err != nil {
			if *config.Verbose {
				log.Printf("Could not lookup geo-ip record for host %s: %s", e.Name, err)
			}
			continue
		}

		// Log the geo-ip data
		ns.db.InsertResolvedPlace(e.NameID, geoRecord.City.Names["en"],
			geoRecord.Country.IsoCode, geoRecord.Continent.Names["en"])
	}
}

func (ns *NetScan) processEntries(entries []ResolutionEntry) error {
	entryChan := make(chan ResolutionEntry, 10)
	defer close(entryChan)
	ns.wg.Add(1)
	defer ns.wg.Done()
	progressDisplay := utils.NewProgressDisplay()
	defer progressDisplay.Close()

	progressDisplay.StartDisplay(ns.wg)

	numWorkers := *config.NumThreads * runtime.NumCPU()
	for i := 0; i < numWorkers; i++ {
		go ns.resolveWorker(entryChan)
	}

	for i, entry := range entries {
		entryChan <- entry
		if i%256 == 0 {
			progressDisplay.UpdateProgress(0, int64(i), int64(len(entries)))
		}
	}

	return nil
}
<|endoftext|>