<commit_before>package main

import (
	"errors"
	"fmt"
	"log"

	alks "github.com/Cox-Automotive/alks-go"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
)

// Config stores ALKS configuration and credentials
type Config struct {
	URL           string
	AccessKey     string
	SecretKey     string
	Token         string
	CredsFilename string
	Profile       string
}

func getCredentials(c *Config) *credentials.Credentials {
	// Follow the  same priority as the AWS Terraform Provider
	// https://www.terraform.io/docs/providers/aws/#authentication

	providers := []credentials.Provider{
		&credentials.StaticProvider{Value: credentials.Value{
			AccessKeyID:     c.AccessKey,
			SecretAccessKey: c.SecretKey,
			SessionToken:    c.Token,
		}},
		&credentials.EnvProvider{},
		&credentials.SharedCredentialsProvider{
			Filename: c.CredsFilename,
			Profile:  c.Profile,
		},
		&ec2rolecreds.EC2RoleProvider{},
	}

	return credentials.NewChainCredentials(providers)
}

// Client returns a properly configured ALKS client or an appropriate error if initialization fails
func (c *Config) Client() (*alks.Client, error) {
	log.Println("[DEBUG] Validting STS credentials")

	// lookup credentials
	creds := getCredentials(c)
	cp, cpErr := creds.Get()

	// validate we have credentials
	if cpErr != nil {
		return nil, errors.New(`No valid credential sources found for ALKS Provider.
Please see https://github.com/Cox-Automotive/terraform-provider-alks#authentication for more information on
providing credentials for the ALKS Provider`)
	}

	// create a new session to test credentails
	sess, err := session.NewSession(&aws.Config{
		Region:      aws.String("us-east-1"),
		Credentials: creds,
	})

	// validate session
	if err != nil {
		return nil, fmt.Errorf("Error creating session from STS. (%v)", err)
	}

	// make a basic api call to test creds are valid
	stsconn := sts.New(sess)
	_, serr := stsconn.GetCallerIdentity(&sts.GetCallerIdentityInput{})

	// check for valid creds
	if serr != nil {
		return nil, serr
	}

	// got good creds, create alks sts client
	client, err := alks.NewSTSClient(c.URL, cp.AccessKeyID, cp.SecretAccessKey, cp.SessionToken)

	if err != nil {
		return nil, err
	}

	log.Println("[INFO] ALKS Client configured")

	return client, nil
}
<commit_msg>Adds EC2 metadata session for the EC2 role provider<commit_after>package main

import (
	"errors"
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws/ec2metadata"

	alks "github.com/Cox-Automotive/alks-go"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
)

// Config stores ALKS configuration and credentials
type Config struct {
	URL           string
	AccessKey     string
	SecretKey     string
	Token         string
	CredsFilename string
	Profile       string
}

func getCredentials(c *Config) *credentials.Credentials {
	// Follow the  same priority as the AWS Terraform Provider
	// https://www.terraform.io/docs/providers/aws/#authentication

	// needed for the EC2MetaData service
	sess := session.Must(session.NewSession())

	providers := []credentials.Provider{
		&credentials.StaticProvider{Value: credentials.Value{
			AccessKeyID:     c.AccessKey,
			SecretAccessKey: c.SecretKey,
			SessionToken:    c.Token,
		}},
		&credentials.EnvProvider{},
		&credentials.SharedCredentialsProvider{
			Filename: c.CredsFilename,
			Profile:  c.Profile,
		},
		&ec2rolecreds.EC2RoleProvider{
			Client: ec2metadata.New(sess),
		},
	}

	return credentials.NewChainCredentials(providers)
}

// Client returns a properly configured ALKS client or an appropriate error if initialization fails
func (c *Config) Client() (*alks.Client, error) {
	log.Println("[DEBUG] Validting STS credentials")

	// lookup credentials
	creds := getCredentials(c)
	cp, cpErr := creds.Get()

	// validate we have credentials
	if cpErr != nil {
		return nil, errors.New(`No valid credential sources found for ALKS Provider.
Please see https://github.com/Cox-Automotive/terraform-provider-alks#authentication for more information on
providing credentials for the ALKS Provider`)
	}

	// create a new session to test credentails
	sess, err := session.NewSession(&aws.Config{
		Region:      aws.String("us-east-1"),
		Credentials: creds,
	})

	// validate session
	if err != nil {
		return nil, fmt.Errorf("Error creating session from STS. (%v)", err)
	}

	// make a basic api call to test creds are valid
	stsconn := sts.New(sess)
	_, serr := stsconn.GetCallerIdentity(&sts.GetCallerIdentityInput{})

	// check for valid creds
	if serr != nil {
		return nil, serr
	}

	// got good creds, create alks sts client
	client, err := alks.NewSTSClient(c.URL, cp.AccessKeyID, cp.SecretAccessKey, cp.SessionToken)

	if err != nil {
		return nil, err
	}

	log.Println("[INFO] ALKS Client configured")

	return client, nil
}
<|endoftext|>


<commit_before>package lazycache

import (
	"fmt"
	flag "github.com/spf13/pflag"
	"github.com/spf13/viper"
	"strings"
)

func ViperConfiguration() {

	// Configuration
	viper.SetDefault("port", 8080)
	viper.SetDefault("bind", "0.0.0.0")
	viper.SetDefault("imagestore", "")
	viper.SetDefault("imagestore.bucket", "camhd-image-cache")

	viper.SetDefault("quicktimestore", "")
	viper.SetDefault("directorystore", "")
	viper.SetDefault("redishost", "localhost:6379")

	viper.SetConfigName("lazycache")
	viper.AddConfigPath("/etc/lazycache")
	viper.AddConfigPath(".")
	err := viper.ReadInConfig() // Find and read the config file
	if err != nil {             // Handle errors reading the config file
		switch err.(type) {
		case viper.ConfigFileNotFoundError:
			// ignore
		default:
			panic(fmt.Errorf("Fatal error config file: %s \n", err))
		}
	}

	viper.SetEnvPrefix("lazycache")
	viper.AutomaticEnv()
	// Convert '.' to '_' in configuration variable names
	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))

	// var (
	// 	bindFlag          = flag.String("bind", "0.0.0.0", "Network interface to bind to (defaults to 0.0.0.0)")
	// 	ImageStoreFlag   = flag.String("image-store", "", "Type of image store (none, google)")
	// 	ImageBucketFlag = flag.String("image-store-bucket", "", "Bucket used for Google image store")
	// )
	flag.Int("port", 80, "Network port to listen on (default: 8080)")
	flag.String("bind", "0.0.0.0", "Network interface to bind to (defaults to 0.0.0.0)")

	flag.String("image-store", "", "Type of image store (none, local, google)")
	flag.String("image-store-bucket", "camhd-image-cache", "Bucket used for Google image store")
	flag.String("image-local-root", "", "Bucket used for Google image store")
	flag.String("image-url-root", "", "Bucket used for Google image store")

	flag.String("quicktime-store", "", "Type of quicktime store (none, redis)")
	flag.String("directory-store", "", "Type of directory store (none, redis)")
	flag.String("redis-host", "localhost:6379", "Host used for redis store")

	viper.BindPFlag("port", flag.Lookup("port"))
	viper.BindPFlag("bind", flag.Lookup("bind"))

	viper.BindPFlag("imagestore", flag.Lookup("image-store"))
	viper.BindPFlag("imagestore.bucket", flag.Lookup("image-store-bucket"))
	viper.BindPFlag("imagestore.localroot", flag.Lookup("image-local-root"))

	viper.BindPFlag("directorystore", flag.Lookup("directory-store"))
	viper.BindPFlag("quicktimestore", flag.Lookup("quicktime-store"))
	viper.BindPFlag("redishost", flag.Lookup("redis-host"))

	flag.Parse()
}

func ConfigureImageStoreFromViper() {
	storeKey := viper.GetString("imagestore")
	DefaultLogger.Log("msg", fmt.Sprintf("Configuring image store with type \"%s\"", storeKey))
	switch strings.ToLower(storeKey) {
	default:
		DefaultLogger.Log("msg", fmt.Sprintf("Unable to determine type of image store from \"%s\"", storeKey))
		DefaultImageStore = NullImageStore{}
	case "", "none":
		DefaultLogger.Log("msg", "No image store configured.")
		DefaultImageStore = NullImageStore{}
	case "local":
		DefaultImageStore = CreateLocalStore(viper.GetString("imagestore.localRoot"),
			viper.GetString("imagestore.bind"))
	case "google":
		DefaultImageStore = CreateGoogleStore(viper.GetString("imagestore.bucket"))
	}
}

func ConfigureQuicktimeStoreFromViper() {
	storeKey := viper.GetString("quicktimestore")
	DefaultLogger.Log("msg", fmt.Sprintf("Configuring quicktime store with type \"%s\"", storeKey))

	switch strings.ToLower(storeKey) {
	default:
		DefaultLogger.Log("msg", fmt.Sprintf("Unable to determine type of image store from \"%s\"", storeKey))
		QTMetadataStore = CreateMapJSONStore()
	case "", "none":
		DefaultLogger.Log("msg", "Using default QuicktimeStore.")
		QTMetadataStore = CreateMapJSONStore()
	case "redis":
		hostname := viper.GetString("redishost")
		DefaultLogger.Log("msg", fmt.Sprintf("Connecting to redis host \"%s\"", hostname))
		redis, err := CreateRedisJSONStore(hostname, "qt")
		if err != nil {
			DefaultLogger.Log("msg", fmt.Sprintf("Failed to configure Redis Quicktime store to host \"%s\"", hostname))
		}

		DefaultLogger.Log("msg", fmt.Sprintf("Logging movie metadata to Redis at %s", hostname))
		QTMetadataStore = redis
	}
}

func ConfigureDirectoryStoreFromViper() {
	storeKey := viper.GetString("directorystore")
	DefaultLogger.Log("msg", fmt.Sprintf("Configuring directory store with type \"%s\"", storeKey))

	switch strings.ToLower(storeKey) {
	default:
		DefaultLogger.Log("msg", fmt.Sprintf("Unable to determine type of directory store from \"%s\"", storeKey))
		DirKeyStore = CreateMapJSONStore()
	case "", "none":
		DefaultLogger.Log("msg", "Using default directory store.")
		DirKeyStore = CreateMapJSONStore()
	case "redis":
		hostname := viper.GetString("redishost")
		DefaultLogger.Log("msg", fmt.Sprintf("Connecting to redis host \"%s\"", hostname))
		redis, err := CreateRedisJSONStore(hostname, "dir")
		if err != nil {
			DefaultLogger.Log("msg", fmt.Sprintf("Failed to configure Redis directory store to host \"%s\"", hostname))
		}

		DefaultLogger.Log("msg", fmt.Sprintf("Logging directory metadata to Redis at %s", hostname))
		DirKeyStore = redis
	}
}

func ConfigureFromViper() {
	DefaultLogger.Log("msg", "In ConfigureFromViper")
	ConfigureImageStoreFromViper()
	ConfigureDirectoryStoreFromViper()
	ConfigureQuicktimeStoreFromViper()
}
<commit_msg>Moved ViperConfiguration into ConfigureFromViper<commit_after>package lazycache

import (
	"fmt"
	flag "github.com/spf13/pflag"
	"github.com/spf13/viper"
	"strings"
)

func ViperConfiguration() {

	// Configuration
	viper.SetDefault("port", 8080)
	viper.SetDefault("bind", "0.0.0.0")
	viper.SetDefault("imagestore", "")
	viper.SetDefault("imagestore.bucket", "camhd-image-cache")

	viper.SetDefault("quicktimestore", "")
	viper.SetDefault("directorystore", "")
	viper.SetDefault("redishost", "localhost:6379")

	viper.SetConfigName("lazycache")
	viper.AddConfigPath("/etc/lazycache")
	viper.AddConfigPath(".")
	err := viper.ReadInConfig() // Find and read the config file
	if err != nil {             // Handle errors reading the config file
		switch err.(type) {
		case viper.ConfigFileNotFoundError:
			// ignore
		default:
			panic(fmt.Errorf("Fatal error config file: %s \n", err))
		}
	}

	viper.SetEnvPrefix("lazycache")
	viper.AutomaticEnv()
	// Convert '.' to '_' in configuration variable names
	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))

	// var (
	// 	bindFlag          = flag.String("bind", "0.0.0.0", "Network interface to bind to (defaults to 0.0.0.0)")
	// 	ImageStoreFlag   = flag.String("image-store", "", "Type of image store (none, google)")
	// 	ImageBucketFlag = flag.String("image-store-bucket", "", "Bucket used for Google image store")
	// )
	flag.Int("port", 80, "Network port to listen on (default: 8080)")
	flag.String("bind", "0.0.0.0", "Network interface to bind to (defaults to 0.0.0.0)")

	flag.String("image-store", "", "Type of image store (none, local, google)")
	flag.String("image-store-bucket", "camhd-image-cache", "Bucket used for Google image store")
	flag.String("image-local-root", "", "Bucket used for Google image store")
	flag.String("image-url-root", "", "Bucket used for Google image store")

	flag.String("quicktime-store", "", "Type of quicktime store (none, redis)")
	flag.String("directory-store", "", "Type of directory store (none, redis)")
	flag.String("redis-host", "localhost:6379", "Host used for redis store")

	viper.BindPFlag("port", flag.Lookup("port"))
	viper.BindPFlag("bind", flag.Lookup("bind"))

	viper.BindPFlag("imagestore", flag.Lookup("image-store"))
	viper.BindPFlag("imagestore.bucket", flag.Lookup("image-store-bucket"))
	viper.BindPFlag("imagestore.localroot", flag.Lookup("image-local-root"))

	viper.BindPFlag("directorystore", flag.Lookup("directory-store"))
	viper.BindPFlag("quicktimestore", flag.Lookup("quicktime-store"))
	viper.BindPFlag("redishost", flag.Lookup("redis-host"))

	flag.Parse()
}

func ConfigureImageStoreFromViper() {
	storeKey := viper.GetString("imagestore")
	DefaultLogger.Log("msg", fmt.Sprintf("Configuring image store with type \"%s\"", storeKey))
	switch strings.ToLower(storeKey) {
	default:
		DefaultLogger.Log("msg", fmt.Sprintf("Unable to determine type of image store from \"%s\"", storeKey))
		DefaultImageStore = NullImageStore{}
	case "", "none":
		DefaultLogger.Log("msg", "No image store configured.")
		DefaultImageStore = NullImageStore{}
	case "local":
		DefaultImageStore = CreateLocalStore(viper.GetString("imagestore.localRoot"),
			viper.GetString("imagestore.bind"))
	case "google":
		DefaultImageStore = CreateGoogleStore(viper.GetString("imagestore.bucket"))
	}
}

func ConfigureQuicktimeStoreFromViper() {
	storeKey := viper.GetString("quicktimestore")
	DefaultLogger.Log("msg", fmt.Sprintf("Configuring quicktime store with type \"%s\"", storeKey))

	switch strings.ToLower(storeKey) {
	default:
		DefaultLogger.Log("msg", fmt.Sprintf("Unable to determine type of image store from \"%s\"", storeKey))
		QTMetadataStore = CreateMapJSONStore()
	case "", "none":
		DefaultLogger.Log("msg", "Using default QuicktimeStore.")
		QTMetadataStore = CreateMapJSONStore()
	case "redis":
		hostname := viper.GetString("redishost")
		DefaultLogger.Log("msg", fmt.Sprintf("Connecting to redis host \"%s\"", hostname))
		redis, err := CreateRedisJSONStore(hostname, "qt")
		if err != nil {
			DefaultLogger.Log("msg", fmt.Sprintf("Failed to configure Redis Quicktime store to host \"%s\"", hostname))
		}

		DefaultLogger.Log("msg", fmt.Sprintf("Logging movie metadata to Redis at %s", hostname))
		QTMetadataStore = redis
	}
}

func ConfigureDirectoryStoreFromViper() {
	storeKey := viper.GetString("directorystore")
	DefaultLogger.Log("msg", fmt.Sprintf("Configuring directory store with type \"%s\"", storeKey))

	switch strings.ToLower(storeKey) {
	default:
		DefaultLogger.Log("msg", fmt.Sprintf("Unable to determine type of directory store from \"%s\"", storeKey))
		DirKeyStore = CreateMapJSONStore()
	case "", "none":
		DefaultLogger.Log("msg", "Using default directory store.")
		DirKeyStore = CreateMapJSONStore()
	case "redis":
		hostname := viper.GetString("redishost")
		DefaultLogger.Log("msg", fmt.Sprintf("Connecting to redis host \"%s\"", hostname))
		redis, err := CreateRedisJSONStore(hostname, "dir")
		if err != nil {
			DefaultLogger.Log("msg", fmt.Sprintf("Failed to configure Redis directory store to host \"%s\"", hostname))
		}

		DefaultLogger.Log("msg", fmt.Sprintf("Logging directory metadata to Redis at %s", hostname))
		DirKeyStore = redis
	}
}

func ConfigureFromViper() {
	ViperConfiguration()

	DefaultLogger.Log("msg", "In ConfigureFromViper")
	ConfigureImageStoreFromViper()
	ConfigureDirectoryStoreFromViper()
	ConfigureQuicktimeStoreFromViper()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"os"
	"sync"
)

type Server struct {
	ID         int
	Name, Path string
	Args       []string
	status     string
	Map        int
}

type Map struct {
	ID         int
	Name, Path string
	Status     string
	Server     int
}

type Config struct {
	mu         sync.RWMutex
	filename   string
	ServerName string
	ServersDir string
	MapsDir    string
	Port       uint16
	Servers    serverMap
	Maps       mapMap
	selected   int
}

type serverMap map[int]Server

func (m serverMap) MarshalJSON() ([]byte, error) {
	s := make([]Server, 0, len(m))
	for _, v := range m {
		s = append(s, v)
	}
	return json.Marshal(s)
}

func (m serverMap) UnmarshalJSON(b []byte) error {
	var s []Server
	err := json.Unmarshal(b, &s)
	if err != nil {
		return err
	}
	for _, v := range s {
		m[v.ID] = v
	}
	return nil
}

type mapMap map[int]Map

func (m mapMap) MarshalJSON() ([]byte, error) {
	s := make([]Map, 0, len(m))
	for _, v := range m {
		s = append(s, v)
	}
	return json.Marshal(s)
}

func (m mapMap) UnmarshalJSON(b []byte) error {
	var s []Map
	err := json.Unmarshal(b, &s)
	if err != nil {
		return err
	}
	for _, v := range s {
		m[v.ID] = v
	}
	return nil
}

func loadConfig(filename string) (*Config, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	c := &Config{
		filename:   filename,
		ServerName: "Minecraft",
		Port:       8080,
		Servers:    make(serverMap),
		Maps:       make(mapMap),
		selected:   -1,
	}
	err = json.NewDecoder(f).Decode(c)
	if err != nil {
		return nil, err
	}
	return c, nil
}

func (c *Config) save() error {
	f, err := os.Create(c.filename)
	if err != nil {
		return err
	}
	defer f.Close()
	return json.NewEncoder(f).Encode(c)
}

func (c *Config) createServer(name, path string) int {
	c.mu.Lock()
	defer c.mu.Unlock()
	defer c.save()
	id := 0
	for {
		_, ok := c.Servers[id]
		if !ok {
			break
		}
		id++
	}
	c.Servers[id] = Server{ID: id, Name: name, Path: path, Args: []string{"-Xmx1024M", "-Xms1024M"}}
	return id
}

func (c *Config) newMap(name, path string) int {
	c.mu.Lock()
	defer c.mu.Unlock()
	defer c.save()
	id := 0
	for {
		_, ok := c.Maps[id]
		if !ok {
			break
		}
		id++
	}
	c.Maps[id] = Map{ID: id, Name: name, Path: path, Server: -1}
	return id
}

func (c *Config) serverStatus(id int, status string) {
	c.mu.Lock()
	defer c.mu.Unlock()
	s, ok := c.Servers[id]
	if !ok {
		return
	}
	s.status = status
	c.Servers[id] = s
}
<commit_msg>Added default -1 for Map in Server config<commit_after>package main

import (
	"encoding/json"
	"os"
	"sync"
)

type Server struct {
	ID         int
	Name, Path string
	Args       []string
	status     string
	Map        int
}

type Map struct {
	ID         int
	Name, Path string
	Status     string
	Server     int
}

type Config struct {
	mu         sync.RWMutex
	filename   string
	ServerName string
	ServersDir string
	MapsDir    string
	Port       uint16
	Servers    serverMap
	Maps       mapMap
	selected   int
}

type serverMap map[int]Server

func (m serverMap) MarshalJSON() ([]byte, error) {
	s := make([]Server, 0, len(m))
	for _, v := range m {
		s = append(s, v)
	}
	return json.Marshal(s)
}

func (m serverMap) UnmarshalJSON(b []byte) error {
	var s []Server
	err := json.Unmarshal(b, &s)
	if err != nil {
		return err
	}
	for _, v := range s {
		m[v.ID] = v
	}
	return nil
}

type mapMap map[int]Map

func (m mapMap) MarshalJSON() ([]byte, error) {
	s := make([]Map, 0, len(m))
	for _, v := range m {
		s = append(s, v)
	}
	return json.Marshal(s)
}

func (m mapMap) UnmarshalJSON(b []byte) error {
	var s []Map
	err := json.Unmarshal(b, &s)
	if err != nil {
		return err
	}
	for _, v := range s {
		m[v.ID] = v
	}
	return nil
}

func loadConfig(filename string) (*Config, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	c := &Config{
		filename:   filename,
		ServerName: "Minecraft",
		Port:       8080,
		Servers:    make(serverMap),
		Maps:       make(mapMap),
		selected:   -1,
	}
	err = json.NewDecoder(f).Decode(c)
	if err != nil {
		return nil, err
	}
	return c, nil
}

func (c *Config) save() error {
	f, err := os.Create(c.filename)
	if err != nil {
		return err
	}
	defer f.Close()
	return json.NewEncoder(f).Encode(c)
}

func (c *Config) createServer(name, path string) int {
	c.mu.Lock()
	defer c.mu.Unlock()
	defer c.save()
	id := 0
	for {
		_, ok := c.Servers[id]
		if !ok {
			break
		}
		id++
	}
	c.Servers[id] = Server{ID: id, Name: name, Path: path, Args: []string{"-Xmx1024M", "-Xms1024M"}, Map: -1}
	return id
}

func (c *Config) newMap(name, path string) int {
	c.mu.Lock()
	defer c.mu.Unlock()
	defer c.save()
	id := 0
	for {
		_, ok := c.Maps[id]
		if !ok {
			break
		}
		id++
	}
	c.Maps[id] = Map{ID: id, Name: name, Path: path, Server: -1}
	return id
}

func (c *Config) serverStatus(id int, status string) {
	c.mu.Lock()
	defer c.mu.Unlock()
	s, ok := c.Servers[id]
	if !ok {
		return
	}
	s.status = status
	c.Servers[id] = s
}
<|endoftext|>


<commit_before>package main

import "os"

// Configuration - config information to pass to various functions
type Configuration struct {
	Dbname            string
	Dbaddress         string
	TagPostTable      string
	DBPAPIKey					string
}

// ImportConfig instantiates the above structure info
func ImportConfig() Configuration {

	// initialize default configuration
	configuration := Configuration{
		Dbname:        "bibletagapi",
		Dbaddress:     "localhost:28015",
		TagPostTable:  "tags",
		DBPAPIKey:		 ""
	}

	// override defaults if environmental vars available
	if os.Getenv("BIBLETAGAPI_DBNAME") != "" {
		configuration.Dbname = os.Getenv("BIBLETAGAPI_DBNAME")
	}
	if os.Getenv("BIBLETAGAPI_DBADDRESS") != "" {
		configuration.Dbaddress = os.Getenv("BIBLETAGAPI_DBADDRESS")
	}
	if os.Getenv("BIBLETAGAPI_TAGPOSTTABLE") != "" {
		configuration.TagPostTable = os.Getenv("BIBLETAGAPI_TAGPOSTTABLE")
	}
	if os.Getenv("BIBLETAGAPI_DBP_API_KEY") != "" {
		configuration.DBPAPIKey = os.Getenv("BIBLETAGAPI_DBP_API_KEY")
	}

	return configuration
}<commit_msg>Apparently, Go requires a comma after the last element in a struct<commit_after>package main

import "os"

// Configuration - config information to pass to various functions
type Configuration struct {
	Dbname            string
	Dbaddress         string
	TagPostTable      string
	DBPAPIKey					string
}

// ImportConfig instantiates the above structure info
func ImportConfig() Configuration {

	// initialize default configuration
	configuration := Configuration{
		Dbname:        "bibletagapi",
		Dbaddress:     "localhost:28015",
		TagPostTable:  "tags",
		DBPAPIKey:		 "",
	}

	// override defaults if environmental vars available
	if os.Getenv("BIBLETAGAPI_DBNAME") != "" {
		configuration.Dbname = os.Getenv("BIBLETAGAPI_DBNAME")
	}
	if os.Getenv("BIBLETAGAPI_DBADDRESS") != "" {
		configuration.Dbaddress = os.Getenv("BIBLETAGAPI_DBADDRESS")
	}
	if os.Getenv("BIBLETAGAPI_TAGPOSTTABLE") != "" {
		configuration.TagPostTable = os.Getenv("BIBLETAGAPI_TAGPOSTTABLE")
	}
	if os.Getenv("BIBLETAGAPI_DBP_API_KEY") != "" {
		configuration.DBPAPIKey = os.Getenv("BIBLETAGAPI_DBP_API_KEY")
	}

	return configuration
}<|endoftext|>


<commit_before>package main

import (
	"log"
	"strings"

	"github.com/kelseyhightower/envconfig"
)

// Configs ...
type Configs struct {
	DockerHostURL             string  `split_words:"true"`
	CatalogLimit              string  `split_words:"true" default:"50000"`
	KeepLatestTag             bool    `split_words:"true" default:"true"`
	ImageTagExcemptionTestAPI string  `split_words:"true"`
	ImageTagExcemption        bool    `split_words:"true" default:"true"`
	MaxImageLifetime          float64 `split_words:"true" default:"720"`
}

// GlobalConfigs ...
var GlobalConfigs Configs

// CheckAndGetConfigs ...
func CheckAndGetConfigs() {
	envconfig.Process("cleanistry", &GlobalConfigs)
	if GlobalConfigs.DockerHostURL == "" {
		log.Fatal("Environment variable CLEANISTRY_DOCKER_HOST_URL not found")
	}
	if !AbleToConnect("tcp", GlobalConfigs.DockerHostURL) {
		log.Fatalf("Unable to connect to %s", GlobalConfigs.DockerHostURL)
	}
	if ShouldWeKeepLatestTag() {
		log.Println("Keeping all images with the 'latest' tag")
	}
	if IsImageTagExcemptionAPIPresent() {
		if GlobalConfigs.ImageTagExcemptionTestAPI == "" {
			log.Fatalf("Ensure an ImageTag excemption API is configured using CLEANISTRY_IMAGE_TAG_EXCEMPTION_TEST_API or set CLEANISTRY_IMAGE_TAG_EXCEMPTION to false")
		} else {
			if !AbleToConnect("tcp", strings.Split(GlobalConfigs.ImageTagExcemptionTestAPI, "/")[0]) {
				log.Fatalf("Unable to connect to %s", GlobalConfigs.ImageTagExcemptionTestAPI)
			}
		}
	}
}

// GetDockerHostURL ...
func GetDockerHostURL() string {
	return GlobalConfigs.DockerHostURL
}

// GetCatalogLimit ...
func GetCatalogLimit() string {
	return GlobalConfigs.CatalogLimit
}

// ShouldWeKeepLatestTag ...
func ShouldWeKeepLatestTag() bool {
	return GlobalConfigs.KeepLatestTag
}

// GetImageTagExcemptionTestAPI ...
func GetImageTagExcemptionTestAPI() string {
	return GlobalConfigs.ImageTagExcemptionTestAPI
}

// IsImageTagExcemptionAPIPresent ...
func IsImageTagExcemptionAPIPresent() bool {
	return GlobalConfigs.ImageTagExcemption
}

// GetMaxImageLifetime ...
func GetMaxImageLifetime() float64 {
	return GlobalConfigs.MaxImageLifetime
}
<commit_msg>appending the 80 port if not mentioned<commit_after>package main

import (
	"log"
	"strings"

	"github.com/kelseyhightower/envconfig"
)

// Configs ...
type Configs struct {
	DockerHostURL             string  `split_words:"true"`
	CatalogLimit              string  `split_words:"true" default:"50000"`
	KeepLatestTag             bool    `split_words:"true" default:"true"`
	ImageTagExcemptionTestAPI string  `split_words:"true"`
	ImageTagExcemption        bool    `split_words:"true" default:"true"`
	MaxImageLifetime          float64 `split_words:"true" default:"720"`
}

// GlobalConfigs ...
var GlobalConfigs Configs

// CheckAndGetConfigs ...
func CheckAndGetConfigs() {
	envconfig.Process("cleanistry", &GlobalConfigs)
	if GlobalConfigs.DockerHostURL == "" {
		log.Fatal("Environment variable CLEANISTRY_DOCKER_HOST_URL not found")
	}
	if !AbleToConnect("tcp", GlobalConfigs.DockerHostURL) {
		log.Fatalf("Unable to connect to %s", GlobalConfigs.DockerHostURL)
	}
	if ShouldWeKeepLatestTag() {
		log.Println("Keeping all images with the 'latest' tag")
	}
	if IsImageTagExcemptionAPIPresent() {
		if GlobalConfigs.ImageTagExcemptionTestAPI == "" {
			log.Fatalf("Ensure an ImageTag excemption API is configured using CLEANISTRY_IMAGE_TAG_EXCEMPTION_TEST_API or set CLEANISTRY_IMAGE_TAG_EXCEMPTION to false")
		} else {
			address := strings.Split(GlobalConfigs.ImageTagExcemptionTestAPI, "/")[0]
			if !strings.Contains(address, ":") {
				address += ":80"
			}
			if !AbleToConnect("tcp", address) {
				log.Fatalf("Unable to connect to %s", address)
			}
		}
	}
}

// GetDockerHostURL ...
func GetDockerHostURL() string {
	return GlobalConfigs.DockerHostURL
}

// GetCatalogLimit ...
func GetCatalogLimit() string {
	return GlobalConfigs.CatalogLimit
}

// ShouldWeKeepLatestTag ...
func ShouldWeKeepLatestTag() bool {
	return GlobalConfigs.KeepLatestTag
}

// GetImageTagExcemptionTestAPI ...
func GetImageTagExcemptionTestAPI() string {
	return GlobalConfigs.ImageTagExcemptionTestAPI
}

// IsImageTagExcemptionAPIPresent ...
func IsImageTagExcemptionAPIPresent() bool {
	return GlobalConfigs.ImageTagExcemption
}

// GetMaxImageLifetime ...
func GetMaxImageLifetime() float64 {
	return GlobalConfigs.MaxImageLifetime
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/gogap/logs"
)

type InletHTTPAPIConfig struct {
	HTTP               HTTPConfig      `json:"http"`
	IncludeConfigFiles []string        `json:"include_config_files"`
	Address            []AddressConfig `json:"address"`
	Graphs             []GraphsConfig  `json:"graphs"`
}

type HTTPConfig struct {
	Address       string   `json:"address"`
	Server        string   `json:"server"`
	APIHeader     string   `json:"api_header"`
	CookiesDomain string   `json:"cookies_domain"`
	EnableStat    bool     `json:"enable_stat"`
	P3P           string   `json:"p3p"`
	AllowOrigins  []string `json:"allow_origins"`
	AllowHeaders  []string `json:"allow_headers"`
	PATH          string   `json:"path"`

	allowOrigins    map[string]bool   `json:"-"`
	responseHeaders map[string]string `json:"-"`
}

type AddressConfig struct {
	Name string `json:"name"`
	Type string `json:"type"`
	Url  string `json:"url"`
}

type GraphsConfig struct {
	API              string   `json:"api"`
	Graph            []string `json:"graph"`
	IsProxy          bool     `json:"is_proxy,omitempty"`
	ErrorAddressName string   `json:"error_address_name"`
}

func parseRefer(url string) (protocol string, domain string) {
	url = strings.TrimSpace(url)

	if len(url) > 0 {
		start0 := strings.Index(url, "://")
		url0 := url[start0+3 : len(url)]
		surls := strings.Split(url0, "/")

		if len(surls) > 0 {
			domain = surls[0]
		}

		protocol = url[0:start0]
	}

	return
}

func (p *HTTPConfig) ParseOrigin(refer string) (protocol string, domin string, isAllow bool) {
	if _, err := url.Parse(refer); err == nil {
		refProtocol, refDomain := parseRefer(refer)
		if p.allowOrigins["*"] ||
			p.allowOrigins[refDomain] {
			return refProtocol, refDomain, true
		}
		return refProtocol, refDomain, false
	}

	return "", "", false
}

func (p *HTTPConfig) allowHeaders() string {
	if p.AllowHeaders != nil {
		return strings.Join(p.AllowHeaders, ",")
	}
	return ""
}

func isFileOrDir(filename string, decideDir bool) bool {
	fileInfo, err := os.Stat(filename)
	if err != nil {
		return false
	}
	isDir := fileInfo.IsDir()
	if decideDir {
		return isDir
	}
	return !isDir
}

func loadIncludeFile(filename string, conf *InletHTTPAPIConfig) {

	bFile, e := ioutil.ReadFile(filename)
	if e != nil {
		panic(e)
	}
	exConf := InletHTTPAPIConfig{}
	if e = json.Unmarshal(bFile, &exConf); e != nil {
		panic(e)
	}

	if exConf.Address != nil && len(exConf.Address) > 0 {
		conf.Address = append(conf.Address, exConf.Address...)
	}

	if exConf.Graphs != nil && len(exConf.Graphs) > 0 {
		conf.Graphs = append(conf.Graphs, exConf.Graphs...)
	}

	logs.Info("config file loaded:", filename)

	return
}

func LoadConfig(filename string) InletHTTPAPIConfig {
	bFile, e := ioutil.ReadFile(filename)
	if e != nil {
		panic(e)
	}

	conf := InletHTTPAPIConfig{}
	if e = json.Unmarshal(bFile, &conf); e != nil {
		panic(e)
	}

	conf.HTTP.allowOrigins = make(map[string]bool)

	for _, allowOrigin := range conf.HTTP.AllowOrigins {
		conf.HTTP.allowOrigins[allowOrigin] = true
	}

	if conf.HTTP.responseHeaders == nil {
		conf.HTTP.responseHeaders = make(map[string]string)
	}

	if conf.HTTP.P3P != "" {
		conf.HTTP.responseHeaders["P3P"] = conf.HTTP.P3P
	}

	if conf.HTTP.Server == "" {
		conf.HTTP.responseHeaders["Server"] = conf.HTTP.Server
	} else {
		conf.HTTP.responseHeaders["Server"] = "spirit"
	}

	logs.Info("config file loaded:", filename)

	//read include configs
	if conf.IncludeConfigFiles != nil && len(conf.IncludeConfigFiles) > 0 {
		for _, filename := range conf.IncludeConfigFiles {
			if isFileOrDir(filename, true) {
				if f, e := os.Open(filename); e != nil {
					panic(e)
				} else if names, e := f.Readdirnames(-1); e != nil {
					panic(e)
				} else {
					for _, name := range names {
						filename = strings.TrimRight(filename, "/")
						if filepath.Ext(name) == ".conf" {
							loadIncludeFile(filename+"/"+name, &conf)
						}
					}
				}
			} else {
				loadIncludeFile(filename, &conf)
			}
		}
	}

	for _, graph := range conf.Graphs {
		if graph.IsProxy {
			proxyAPI[graph.API] = true
		}
	}

	return conf
}
<commit_msg>improve config panic description<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/gogap/logs"
)

type InletHTTPAPIConfig struct {
	HTTP               HTTPConfig      `json:"http"`
	IncludeConfigFiles []string        `json:"include_config_files"`
	Address            []AddressConfig `json:"address"`
	Graphs             []GraphsConfig  `json:"graphs"`
}

type HTTPConfig struct {
	Address       string   `json:"address"`
	Server        string   `json:"server"`
	APIHeader     string   `json:"api_header"`
	CookiesDomain string   `json:"cookies_domain"`
	EnableStat    bool     `json:"enable_stat"`
	P3P           string   `json:"p3p"`
	AllowOrigins  []string `json:"allow_origins"`
	AllowHeaders  []string `json:"allow_headers"`
	PATH          string   `json:"path"`

	allowOrigins    map[string]bool   `json:"-"`
	responseHeaders map[string]string `json:"-"`
}

type AddressConfig struct {
	Name string `json:"name"`
	Type string `json:"type"`
	Url  string `json:"url"`
}

type GraphsConfig struct {
	API              string   `json:"api"`
	Graph            []string `json:"graph"`
	IsProxy          bool     `json:"is_proxy,omitempty"`
	ErrorAddressName string   `json:"error_address_name"`
}

func parseRefer(url string) (protocol string, domain string) {
	url = strings.TrimSpace(url)

	if len(url) > 0 {
		start0 := strings.Index(url, "://")
		url0 := url[start0+3 : len(url)]
		surls := strings.Split(url0, "/")

		if len(surls) > 0 {
			domain = surls[0]
		}

		protocol = url[0:start0]
	}

	return
}

func (p *HTTPConfig) ParseOrigin(refer string) (protocol string, domin string, isAllow bool) {
	if _, err := url.Parse(refer); err == nil {
		refProtocol, refDomain := parseRefer(refer)
		if p.allowOrigins["*"] ||
			p.allowOrigins[refDomain] {
			return refProtocol, refDomain, true
		}
		return refProtocol, refDomain, false
	}

	return "", "", false
}

func (p *HTTPConfig) allowHeaders() string {
	if p.AllowHeaders != nil {
		return strings.Join(p.AllowHeaders, ",")
	}
	return ""
}

func isFileOrDir(filename string, decideDir bool) bool {
	fileInfo, err := os.Stat(filename)
	if err != nil {
		return false
	}
	isDir := fileInfo.IsDir()
	if decideDir {
		return isDir
	}
	return !isDir
}

func loadIncludeFile(filename string, conf *InletHTTPAPIConfig) {

	bFile, e := ioutil.ReadFile(filename)
	if e != nil {
		e = fmt.Errorf("read config file of %s failed, error: %s", filename, e)
		panic(e)
	}
	exConf := InletHTTPAPIConfig{}
	if e = json.Unmarshal(bFile, &exConf); e != nil {
		e = fmt.Errorf("unmarshal config file of %s to object failed, error: %s", filename, e)
		panic(e)
	}

	if exConf.Address != nil && len(exConf.Address) > 0 {
		conf.Address = append(conf.Address, exConf.Address...)
	}

	if exConf.Graphs != nil && len(exConf.Graphs) > 0 {
		conf.Graphs = append(conf.Graphs, exConf.Graphs...)
	}

	logs.Info("config file loaded:", filename)

	return
}

func LoadConfig(filename string) InletHTTPAPIConfig {
	bFile, e := ioutil.ReadFile(filename)
	if e != nil {
		panic(e)
	}

	conf := InletHTTPAPIConfig{}
	if e = json.Unmarshal(bFile, &conf); e != nil {
		e = fmt.Errorf("unmarshal config file of %s to object failed, error: %s", filename, e)
		panic(e)
	}

	conf.HTTP.allowOrigins = make(map[string]bool)

	for _, allowOrigin := range conf.HTTP.AllowOrigins {
		conf.HTTP.allowOrigins[allowOrigin] = true
	}

	if conf.HTTP.responseHeaders == nil {
		conf.HTTP.responseHeaders = make(map[string]string)
	}

	if conf.HTTP.P3P != "" {
		conf.HTTP.responseHeaders["P3P"] = conf.HTTP.P3P
	}

	if conf.HTTP.Server == "" {
		conf.HTTP.responseHeaders["Server"] = conf.HTTP.Server
	} else {
		conf.HTTP.responseHeaders["Server"] = "spirit"
	}

	logs.Info("config file loaded:", filename)

	//read include configs
	if conf.IncludeConfigFiles != nil && len(conf.IncludeConfigFiles) > 0 {
		for _, filename := range conf.IncludeConfigFiles {
			if isFileOrDir(filename, true) {
				if f, e := os.Open(filename); e != nil {
					panic(e)
				} else if names, e := f.Readdirnames(-1); e != nil {
					panic(e)
				} else {
					for _, name := range names {
						filename = strings.TrimRight(filename, "/")
						if filepath.Ext(name) == ".conf" {
							loadIncludeFile(filename+"/"+name, &conf)
						}
					}
				}
			} else {
				loadIncludeFile(filename, &conf)
			}
		}
	}

	for _, graph := range conf.Graphs {
		if graph.IsProxy {
			proxyAPI[graph.API] = true
		}
	}

	return conf
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"strings"

	"github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/go-multierror"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/aws/aws-sdk-go/service/cloudtrail"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/aws/aws-sdk-go/service/codedeploy"
	"github.com/aws/aws-sdk-go/service/directoryservice"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/ecs"
	"github.com/aws/aws-sdk-go/service/efs"
	"github.com/aws/aws-sdk-go/service/elasticache"
	elasticsearch "github.com/aws/aws-sdk-go/service/elasticsearchservice"
	"github.com/aws/aws-sdk-go/service/elb"
	"github.com/aws/aws-sdk-go/service/glacier"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/aws/aws-sdk-go/service/kinesis"
	"github.com/aws/aws-sdk-go/service/lambda"
	"github.com/aws/aws-sdk-go/service/opsworks"
	"github.com/aws/aws-sdk-go/service/rds"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/sns"
	"github.com/aws/aws-sdk-go/service/sqs"
)

type Config struct {
	AccessKey  string
	SecretKey  string
	Token      string
	Region     string
	MaxRetries int

	AllowedAccountIds   []interface{}
	ForbiddenAccountIds []interface{}

	DynamoDBEndpoint string
	KinesisEndpoint  string
}

type AWSClient struct {
	cfconn             *cloudformation.CloudFormation
	cloudtrailconn     *cloudtrail.CloudTrail
	cloudwatchconn     *cloudwatch.CloudWatch
	cloudwatchlogsconn *cloudwatchlogs.CloudWatchLogs
	dsconn             *directoryservice.DirectoryService
	dynamodbconn       *dynamodb.DynamoDB
	ec2conn            *ec2.EC2
	ecsconn            *ecs.ECS
	efsconn            *efs.EFS
	elbconn            *elb.ELB
	esconn             *elasticsearch.ElasticsearchService
	autoscalingconn    *autoscaling.AutoScaling
	s3conn             *s3.S3
	sqsconn            *sqs.SQS
	snsconn            *sns.SNS
	r53conn            *route53.Route53
	region             string
	rdsconn            *rds.RDS
	iamconn            *iam.IAM
	kinesisconn        *kinesis.Kinesis
	elasticacheconn    *elasticache.ElastiCache
	lambdaconn         *lambda.Lambda
	opsworksconn       *opsworks.OpsWorks
	glacierconn        *glacier.Glacier
	codedeployconn     *codedeploy.CodeDeploy
}

// Client configures and returns a fully initialized AWSClient
func (c *Config) Client() (interface{}, error) {
	var client AWSClient

	// Get the auth and region. This can fail if keys/regions were not
	// specified and we're attempting to use the environment.
	var errs []error

	log.Println("[INFO] Building AWS region structure")
	err := c.ValidateRegion()
	if err != nil {
		errs = append(errs, err)
	}

	if len(errs) == 0 {
		// store AWS region in client struct, for region specific operations such as
		// bucket storage in S3
		client.region = c.Region

		log.Println("[INFO] Building AWS auth structure")
		// We fetched all credential sources in Provider. If they are
		// available, they'll already be in c. See Provider definition.
		creds := credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, c.Token)
		awsConfig := &aws.Config{
			Credentials: creds,
			Region:      aws.String(c.Region),
			MaxRetries:  aws.Int(c.MaxRetries),
			HTTPClient:  cleanhttp.DefaultClient(),
		}

		log.Println("[INFO] Initializing IAM Connection")
		client.iamconn = iam.New(awsConfig)

		err := c.ValidateCredentials(client.iamconn)
		if err != nil {
			errs = append(errs, err)
		}

		// Some services exist only in us-east-1, e.g. because they manage
		// resources that can span across multiple regions, or because
		// signature format v4 requires region to be us-east-1 for global
		// endpoints:
		// http://docs.aws.amazon.com/general/latest/gr/sigv4_changes.html
		usEast1AwsConfig := &aws.Config{
			Credentials: creds,
			Region:      aws.String("us-east-1"),
			MaxRetries:  aws.Int(c.MaxRetries),
			HTTPClient:  cleanhttp.DefaultClient(),
		}

		awsDynamoDBConfig := *awsConfig
		awsDynamoDBConfig.Endpoint = aws.String(c.DynamoDBEndpoint)

		log.Println("[INFO] Initializing DynamoDB connection")
		client.dynamodbconn = dynamodb.New(&awsDynamoDBConfig)

		log.Println("[INFO] Initializing ELB connection")
		client.elbconn = elb.New(awsConfig)

		log.Println("[INFO] Initializing S3 connection")
		client.s3conn = s3.New(awsConfig)

		log.Println("[INFO] Initializing SQS connection")
		client.sqsconn = sqs.New(awsConfig)

		log.Println("[INFO] Initializing SNS connection")
		client.snsconn = sns.New(awsConfig)

		log.Println("[INFO] Initializing RDS Connection")
		client.rdsconn = rds.New(awsConfig)

		awsKinesisConfig := *awsConfig
		awsKinesisConfig.Endpoint = aws.String(c.KinesisEndpoint)

		log.Println("[INFO] Initializing Kinesis Connection")
		client.kinesisconn = kinesis.New(&awsKinesisConfig)

		authErr := c.ValidateAccountId(client.iamconn)
		if authErr != nil {
			errs = append(errs, authErr)
		}

		log.Println("[INFO] Initializing AutoScaling connection")
		client.autoscalingconn = autoscaling.New(awsConfig)

		log.Println("[INFO] Initializing EC2 Connection")
		client.ec2conn = ec2.New(awsConfig)

		log.Println("[INFO] Initializing ECS Connection")
		client.ecsconn = ecs.New(awsConfig)

		log.Println("[INFO] Initializing EFS Connection")
		client.efsconn = efs.New(awsConfig)

		log.Println("[INFO] Initializing ElasticSearch Connection")
		client.esconn = elasticsearch.New(awsConfig)

		log.Println("[INFO] Initializing Route 53 connection")
		client.r53conn = route53.New(usEast1AwsConfig)

		log.Println("[INFO] Initializing Elasticache Connection")
		client.elasticacheconn = elasticache.New(awsConfig)

		log.Println("[INFO] Initializing Lambda Connection")
		client.lambdaconn = lambda.New(awsConfig)

		log.Println("[INFO] Initializing Cloudformation Connection")
		client.cfconn = cloudformation.New(awsConfig)

		log.Println("[INFO] Initializing CloudWatch SDK connection")
		client.cloudwatchconn = cloudwatch.New(awsConfig)

		log.Println("[INFO] Initializing CloudTrail connection")
		client.cloudtrailconn = cloudtrail.New(awsConfig)

		log.Println("[INFO] Initializing CloudWatch Logs connection")
		client.cloudwatchlogsconn = cloudwatchlogs.New(awsConfig)

		log.Println("[INFO] Initializing OpsWorks Connection")
		client.opsworksconn = opsworks.New(usEast1AwsConfig)

		log.Println("[INFO] Initializing Directory Service connection")
		client.dsconn = directoryservice.New(awsConfig)

		log.Println("[INFO] Initializing Glacier connection")
		client.glacierconn = glacier.New(awsConfig)

		log.Println("[INFO] Initializing CodeDeploy Connection")
		client.codedeployconn = codedeploy.New(awsConfig)
	}

	if len(errs) > 0 {
		return nil, &multierror.Error{Errors: errs}
	}

	return &client, nil
}

// ValidateRegion returns an error if the configured region is not a
// valid aws region and nil otherwise.
func (c *Config) ValidateRegion() error {
	var regions = [11]string{"us-east-1", "us-west-2", "us-west-1", "eu-west-1",
		"eu-central-1", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1",
		"sa-east-1", "cn-north-1", "us-gov-west-1"}

	for _, valid := range regions {
		if c.Region == valid {
			return nil
		}
	}
	return fmt.Errorf("Not a valid region: %s", c.Region)
}

// Validate credentials early and fail before we do any graph walking.
// In the case of an IAM role/profile with insuffecient privileges, fail
// silently
func (c *Config) ValidateCredentials(iamconn *iam.IAM) error {
	_, err := iamconn.GetUser(nil)

	if awsErr, ok := err.(awserr.Error); ok {

		if awsErr.Code() == "AccessDenied" || awsErr.Code() == "ValidationError" {
			log.Printf("[WARN] AccessDenied Error with iam.GetUser, assuming IAM profile")
			// User may be an IAM instance profile, or otherwise IAM role without the
			// GetUser permissions, so fail silently
			return nil
		}

		if awsErr.Code() == "SignatureDoesNotMatch" {
			return fmt.Errorf("Failed authenticating with AWS: please verify credentials")
		}
	}

	return err
}

// ValidateAccountId returns a context-specific error if the configured account
// id is explicitly forbidden or not authorised; and nil if it is authorised.
func (c *Config) ValidateAccountId(iamconn *iam.IAM) error {
	if c.AllowedAccountIds == nil && c.ForbiddenAccountIds == nil {
		return nil
	}

	log.Printf("[INFO] Validating account ID")

	out, err := iamconn.GetUser(nil)

	if err != nil {
		awsErr, _ := err.(awserr.Error)
		if awsErr.Code() == "ValidationError" {
			log.Printf("[WARN] ValidationError with iam.GetUser, assuming its an IAM profile")
			// User may be an IAM instance profile, so fail silently.
			// If it is an IAM instance profile
			// validating account might be superfluous
			return nil
		} else {
			return fmt.Errorf("Failed getting account ID from IAM: %s", err)
			// return error if the account id is explicitly not authorised
		}
	}

	account_id := strings.Split(*out.User.Arn, ":")[4]

	if c.ForbiddenAccountIds != nil {
		for _, id := range c.ForbiddenAccountIds {
			if id == account_id {
				return fmt.Errorf("Forbidden account ID (%s)", id)
			}
		}
	}

	if c.AllowedAccountIds != nil {
		for _, id := range c.AllowedAccountIds {
			if id == account_id {
				return nil
			}
		}
		return fmt.Errorf("Account ID not allowed (%s)", account_id)
	}

	return nil
}
<commit_msg>aws: fix build after upstream breaking change<commit_after>package aws

import (
	"fmt"
	"log"
	"strings"

	"github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/go-multierror"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/aws/aws-sdk-go/service/cloudtrail"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/aws/aws-sdk-go/service/codedeploy"
	"github.com/aws/aws-sdk-go/service/directoryservice"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/ecs"
	"github.com/aws/aws-sdk-go/service/efs"
	"github.com/aws/aws-sdk-go/service/elasticache"
	elasticsearch "github.com/aws/aws-sdk-go/service/elasticsearchservice"
	"github.com/aws/aws-sdk-go/service/elb"
	"github.com/aws/aws-sdk-go/service/glacier"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/aws/aws-sdk-go/service/kinesis"
	"github.com/aws/aws-sdk-go/service/lambda"
	"github.com/aws/aws-sdk-go/service/opsworks"
	"github.com/aws/aws-sdk-go/service/rds"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/sns"
	"github.com/aws/aws-sdk-go/service/sqs"
)

type Config struct {
	AccessKey  string
	SecretKey  string
	Token      string
	Region     string
	MaxRetries int

	AllowedAccountIds   []interface{}
	ForbiddenAccountIds []interface{}

	DynamoDBEndpoint string
	KinesisEndpoint  string
}

type AWSClient struct {
	cfconn             *cloudformation.CloudFormation
	cloudtrailconn     *cloudtrail.CloudTrail
	cloudwatchconn     *cloudwatch.CloudWatch
	cloudwatchlogsconn *cloudwatchlogs.CloudWatchLogs
	dsconn             *directoryservice.DirectoryService
	dynamodbconn       *dynamodb.DynamoDB
	ec2conn            *ec2.EC2
	ecsconn            *ecs.ECS
	efsconn            *efs.EFS
	elbconn            *elb.ELB
	esconn             *elasticsearch.ElasticsearchService
	autoscalingconn    *autoscaling.AutoScaling
	s3conn             *s3.S3
	sqsconn            *sqs.SQS
	snsconn            *sns.SNS
	r53conn            *route53.Route53
	region             string
	rdsconn            *rds.RDS
	iamconn            *iam.IAM
	kinesisconn        *kinesis.Kinesis
	elasticacheconn    *elasticache.ElastiCache
	lambdaconn         *lambda.Lambda
	opsworksconn       *opsworks.OpsWorks
	glacierconn        *glacier.Glacier
	codedeployconn     *codedeploy.CodeDeploy
}

// Client configures and returns a fully initialized AWSClient
func (c *Config) Client() (interface{}, error) {
	var client AWSClient

	// Get the auth and region. This can fail if keys/regions were not
	// specified and we're attempting to use the environment.
	var errs []error

	log.Println("[INFO] Building AWS region structure")
	err := c.ValidateRegion()
	if err != nil {
		errs = append(errs, err)
	}

	if len(errs) == 0 {
		// store AWS region in client struct, for region specific operations such as
		// bucket storage in S3
		client.region = c.Region

		log.Println("[INFO] Building AWS auth structure")
		// We fetched all credential sources in Provider. If they are
		// available, they'll already be in c. See Provider definition.
		creds := credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, c.Token)
		awsConfig := &aws.Config{
			Credentials: creds,
			Region:      aws.String(c.Region),
			MaxRetries:  aws.Int(c.MaxRetries),
			HTTPClient:  cleanhttp.DefaultClient(),
		}

		log.Println("[INFO] Initializing IAM Connection")
		sess := session.New(awsConfig)
		client.iamconn = iam.New(sess)

		err := c.ValidateCredentials(client.iamconn)
		if err != nil {
			errs = append(errs, err)
		}

		// Some services exist only in us-east-1, e.g. because they manage
		// resources that can span across multiple regions, or because
		// signature format v4 requires region to be us-east-1 for global
		// endpoints:
		// http://docs.aws.amazon.com/general/latest/gr/sigv4_changes.html
		usEast1AwsConfig := &aws.Config{
			Credentials: creds,
			Region:      aws.String("us-east-1"),
			MaxRetries:  aws.Int(c.MaxRetries),
			HTTPClient:  cleanhttp.DefaultClient(),
		}
		usEast1Sess := session.New(usEast1AwsConfig)

		awsDynamoDBConfig := *awsConfig
		awsDynamoDBConfig.Endpoint = aws.String(c.DynamoDBEndpoint)

		log.Println("[INFO] Initializing DynamoDB connection")
		dynamoSess := session.New(&awsDynamoDBConfig)
		client.dynamodbconn = dynamodb.New(dynamoSess)

		log.Println("[INFO] Initializing ELB connection")
		client.elbconn = elb.New(sess)

		log.Println("[INFO] Initializing S3 connection")
		client.s3conn = s3.New(sess)

		log.Println("[INFO] Initializing SQS connection")
		client.sqsconn = sqs.New(sess)

		log.Println("[INFO] Initializing SNS connection")
		client.snsconn = sns.New(sess)

		log.Println("[INFO] Initializing RDS Connection")
		client.rdsconn = rds.New(sess)

		awsKinesisConfig := *awsConfig
		awsKinesisConfig.Endpoint = aws.String(c.KinesisEndpoint)

		log.Println("[INFO] Initializing Kinesis Connection")
		kinesisSess := session.New(&awsKinesisConfig)
		client.kinesisconn = kinesis.New(kinesisSess)

		authErr := c.ValidateAccountId(client.iamconn)
		if authErr != nil {
			errs = append(errs, authErr)
		}

		log.Println("[INFO] Initializing AutoScaling connection")
		client.autoscalingconn = autoscaling.New(sess)

		log.Println("[INFO] Initializing EC2 Connection")
		client.ec2conn = ec2.New(sess)

		log.Println("[INFO] Initializing ECS Connection")
		client.ecsconn = ecs.New(sess)

		log.Println("[INFO] Initializing EFS Connection")
		client.efsconn = efs.New(sess)

		log.Println("[INFO] Initializing ElasticSearch Connection")
		client.esconn = elasticsearch.New(sess)

		log.Println("[INFO] Initializing Route 53 connection")
		client.r53conn = route53.New(usEast1Sess)

		log.Println("[INFO] Initializing Elasticache Connection")
		client.elasticacheconn = elasticache.New(sess)

		log.Println("[INFO] Initializing Lambda Connection")
		client.lambdaconn = lambda.New(sess)

		log.Println("[INFO] Initializing Cloudformation Connection")
		client.cfconn = cloudformation.New(sess)

		log.Println("[INFO] Initializing CloudWatch SDK connection")
		client.cloudwatchconn = cloudwatch.New(sess)

		log.Println("[INFO] Initializing CloudTrail connection")
		client.cloudtrailconn = cloudtrail.New(sess)

		log.Println("[INFO] Initializing CloudWatch Logs connection")
		client.cloudwatchlogsconn = cloudwatchlogs.New(sess)

		log.Println("[INFO] Initializing OpsWorks Connection")
		client.opsworksconn = opsworks.New(usEast1Sess)

		log.Println("[INFO] Initializing Directory Service connection")
		client.dsconn = directoryservice.New(sess)

		log.Println("[INFO] Initializing Glacier connection")
		client.glacierconn = glacier.New(sess)

		log.Println("[INFO] Initializing CodeDeploy Connection")
		client.codedeployconn = codedeploy.New(sess)
	}

	if len(errs) > 0 {
		return nil, &multierror.Error{Errors: errs}
	}

	return &client, nil
}

// ValidateRegion returns an error if the configured region is not a
// valid aws region and nil otherwise.
func (c *Config) ValidateRegion() error {
	var regions = [11]string{"us-east-1", "us-west-2", "us-west-1", "eu-west-1",
		"eu-central-1", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1",
		"sa-east-1", "cn-north-1", "us-gov-west-1"}

	for _, valid := range regions {
		if c.Region == valid {
			return nil
		}
	}
	return fmt.Errorf("Not a valid region: %s", c.Region)
}

// Validate credentials early and fail before we do any graph walking.
// In the case of an IAM role/profile with insuffecient privileges, fail
// silently
func (c *Config) ValidateCredentials(iamconn *iam.IAM) error {
	_, err := iamconn.GetUser(nil)

	if awsErr, ok := err.(awserr.Error); ok {

		if awsErr.Code() == "AccessDenied" || awsErr.Code() == "ValidationError" {
			log.Printf("[WARN] AccessDenied Error with iam.GetUser, assuming IAM profile")
			// User may be an IAM instance profile, or otherwise IAM role without the
			// GetUser permissions, so fail silently
			return nil
		}

		if awsErr.Code() == "SignatureDoesNotMatch" {
			return fmt.Errorf("Failed authenticating with AWS: please verify credentials")
		}
	}

	return err
}

// ValidateAccountId returns a context-specific error if the configured account
// id is explicitly forbidden or not authorised; and nil if it is authorised.
func (c *Config) ValidateAccountId(iamconn *iam.IAM) error {
	if c.AllowedAccountIds == nil && c.ForbiddenAccountIds == nil {
		return nil
	}

	log.Printf("[INFO] Validating account ID")

	out, err := iamconn.GetUser(nil)

	if err != nil {
		awsErr, _ := err.(awserr.Error)
		if awsErr.Code() == "ValidationError" {
			log.Printf("[WARN] ValidationError with iam.GetUser, assuming its an IAM profile")
			// User may be an IAM instance profile, so fail silently.
			// If it is an IAM instance profile
			// validating account might be superfluous
			return nil
		} else {
			return fmt.Errorf("Failed getting account ID from IAM: %s", err)
			// return error if the account id is explicitly not authorised
		}
	}

	account_id := strings.Split(*out.User.Arn, ":")[4]

	if c.ForbiddenAccountIds != nil {
		for _, id := range c.ForbiddenAccountIds {
			if id == account_id {
				return fmt.Errorf("Forbidden account ID (%s)", id)
			}
		}
	}

	if c.AllowedAccountIds != nil {
		for _, id := range c.AllowedAccountIds {
			if id == account_id {
				return nil
			}
		}
		return fmt.Errorf("Account ID not allowed (%s)", account_id)
	}

	return nil
}
<|endoftext|>


<commit_before>package config

import (
	"errors"
	"os/user"
	"path/filepath"
	"strings"
)

type ConfigNamespace struct {
	Organization string // optional additional namespace for orgs.
	Namespace    string // usually project name.
}

type Config interface {
	Load() error
}

const UserBase string = "~/.config/"

func ExpandUser(path string) (exPath string, err error) {
	// Acts kind of like os.path.expanduser in Python, except only supports
	// expanding "~/" or "$HOME"
	usr, err := user.Current()
	if err != nil {
		return
	}

	dir := usr.HomeDir

	if path[:2] == "~/" {
		exPath = strings.Replace(path, "~/", dir, 1)
	} else if path[:5] == "$HOME" {
		exPath = strings.Replace(path, "$HOME", dir, 1)
	} else {
		err = errors.New("No expandable path provided.")
	}

	if err != nil {
		return "", err
	}

	exPath, err = filepath.Abs(filepath.Clean(exPath))

	if err != nil {
		return "", err
	}

	return
}

func (c ConfigNamespace) Path() (path string, err error) {
	userBase, err := ExpandUser(UserBase)
	if err != nil {
		return "", err
	}

	path = filepath.Join(userBase, c.Organization, c.Namespace, "config.yaml")
	return
}
<commit_msg>add Load() function<commit_after>package config

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os/user"
	"path/filepath"
	"reflect"
	"strings"

	"gopkg.in/yaml.v2"
)

type ConfigNamespace struct {
	Organization string // optional additional namespace for orgs.
	Namespace    string // usually project name.
}

type Config interface {
	Load(src string, dst interface{}) error
}

const UserBase string = "~/.config/"

// Load expands the provided src path using config.ExpandUser, then reads
// the file and unmarshals into dst using go-yaml.
func Load(src string, dst interface{}) (err error) {
	dstv := reflect.ValueOf(dst)

	if dstv.Kind() != reflect.Ptr {
		err = errors.New("config: not a pointer.")
	} else if dstv.IsNil() {
		err = fmt.Errorf("nil %s.", reflect.TypeOf(dstv).String())
	}
	if err != nil {
		return
	}

	path, err := ExpandUser(src)
	if err != nil {
		return
	}

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	err = yaml.Unmarshal(data, &dst)
	return
}

func ExpandUser(path string) (exPath string, err error) {
	// Acts kind of like os.path.expanduser in Python, except only supports
	// expanding "~/" or "$HOME"
	usr, err := user.Current()
	if err != nil {
		return
	}

	dir := usr.HomeDir

	if path[:2] == "~/" {
		exPath = strings.Replace(path, "~/", dir, 1)
	} else if path[:5] == "$HOME" {
		exPath = strings.Replace(path, "$HOME", dir, 1)
	} else {
		err = errors.New("No expandable path provided.")
	}

	if err != nil {
		return "", err
	}

	exPath, err = filepath.Abs(filepath.Clean(exPath))

	if err != nil {
		return "", err
	}

	return
}

func (c ConfigNamespace) Path() (path string, err error) {
	userBase, err := ExpandUser(UserBase)
	if err != nil {
		return "", err
	}

	path = filepath.Join(userBase, c.Organization, c.Namespace, "config.yaml")
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/Jeffail/gabs"
	"github.com/fsnotify/fsnotify"
	"github.com/ghodss/yaml"
	"github.com/hashicorp/go-multierror"
)

const (
	delimStart = "### Zap Shortcuts :start ##\n"
	delimEnd   = "### Zap Shortcuts :end ##\n"
	expandKey  = "expand"
	queryKey   = "query"
	sslKey     = "ssl_off"

)

// Sentinel value used to indicate set membership.
var exists = struct{}{}

// parseYaml takes a file name and returns a gabs config object.
func parseYaml(fname string) (*gabs.Container, error) {
	data, err := ioutil.ReadFile(fname)
	if err != nil {
		fmt.Printf("Unable to read file: %s\n", err.Error())
		return nil, err
	}
	d, jsonErr := yaml.YAMLToJSON([]byte(data))
	if jsonErr != nil {
		fmt.Printf("Error encoding input to JSON.\n%s\n", jsonErr.Error())
		return nil, jsonErr
	}
	j, _ := gabs.ParseJSON(d)
	return j, nil
}

// validateConfig verifies that there are no unexpected values in the config file.
// at each level of the config, we should either have a KV for expansions, or a leaf node
// with the values oneof "expand", "query", "ssl_off" that map to a string.
func validateConfig(c *gabs.Container) (error) {
	var errors *multierror.Error
	children, _ := c.ChildrenMap()
	seenKeys := make(map[string]struct{})
	for k,v := range children {
		// Check if key already seen
		if _, ok := seenKeys[k]; ok {
			log.Printf("%s detected again", k)
			errors = multierror.Append(errors, fmt.Errorf("duplicate key detected %s", k))
		} else {
			seenKeys[k] = exists // mark key as seen
		}

		// Validate all children
		switch k {
		case
			"expand",
			"query":
			// check that v is a string, else return error.
			if _, ok := v.Data().(string); !ok {
				errors = multierror.Append(errors, fmt.Errorf("expected string value for %T, got: %v", k, v.Data()))
			}
		case "ssl_off":
			// check that v is a boolean, else return error.
			if _, ok := v.Data().(bool); !ok {
				errors = multierror.Append(errors, fmt.Errorf("expected bool value for %T, got: %v", k, v.Data()))
			}
		default:
			// Check if we have an unknown string here.
			if _, ok := v.Data().(string); ok {
				fmt.Printf("unexpected key %s", k)
				errors = multierror.Append(errors, fmt.Errorf("unexpected string value under key %s, got: %v", k, v.Data()))
			}
			// recurse, collect any errors.
			if err := validateConfig(v); err != nil {
				errors = multierror.Append(errors, err)
			}
		}
	}
	return errors.ErrorOrNil()
}

func watchChanges(watcher *fsnotify.Watcher, fname string, cb func()) {
	for {
		select {
		case event := <-watcher.Events:
			// You may wonder why we can't just listen for "Write" events. The reason is that vim (and other editors)
			// will create swap files, and when you write they delete the original and rename the swap file. This is great
			// for resolving system crashes, but also completely incompatible with inotify and other fswatch implementations.
			// Thus, we check that the file of interest might be created as well.
			updated := event.Op&fsnotify.Create == fsnotify.Create || event.Op&fsnotify.Write == fsnotify.Write
			zapconf := filepath.Clean(event.Name) == fname
			if updated && zapconf {
				cb()
			}
		case e := <-watcher.Errors:
			log.Println("error:", e)
		}
	}
}

// TODO: add tests. simulate touching a file.
// updateHosts will attempt to write the zap list of shortcuts
// to /etc/hosts. It will gracefully fail if there are not enough
// permissions to do so.
func updateHosts(c *context) {
	hostPath := "/etc/hosts"

	// 1. read file, prep buffer.
	data, err := ioutil.ReadFile(hostPath)
	if err != nil {
		log.Println("open config: ", err)
	}
	var replacement bytes.Buffer

	// 2. generate payload.
	replacement.WriteString(delimStart)
	children, _ := c.config.ChildrenMap()
	for k := range children {
		replacement.WriteString(fmt.Sprintf("127.0.0.1 %s\n", k))
	}
	replacement.WriteString(delimEnd)

	// 3. Generate new file content
	var updatedFile string
	if !strings.Contains(string(data), delimStart) {
		updatedFile = string(data) + replacement.String()
	} else {
		zapBlock := regexp.MustCompile("(###(.*)##)\n(.|\n)*(###(.*)##\n)")
		updatedFile = zapBlock.ReplaceAllString(string(data), replacement.String())
	}

	// 4. Attempt write to file.
	err = ioutil.WriteFile(hostPath, []byte(updatedFile), 0644)
	if err != nil {
		log.Printf("Error writing to '%s': %s\n", hostPath, err.Error())
	}
}

// makeCallback returns a func that that updates global state.
func makeCallback(c *context, configName string) func() {
	return func() {
		data, err := parseYaml(configName)
		if err != nil {
			log.Printf("Error in new config: %s. Fallback to old config.", err)
			return
		}

		// Update config atomically
		c.configMtx.Lock()
		c.config = data
		c.configMtx.Unlock()

		// Sync DNS entries.
		updateHosts(c)
		return
	}
}
<commit_msg>enable validation checker in hot reload<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/Jeffail/gabs"
	"github.com/fsnotify/fsnotify"
	"github.com/ghodss/yaml"
	"github.com/hashicorp/go-multierror"
)

const (
	delimStart = "### Zap Shortcuts :start ##\n"
	delimEnd   = "### Zap Shortcuts :end ##\n"
	expandKey  = "expand"
	queryKey   = "query"
	sslKey     = "ssl_off"

)

// Sentinel value used to indicate set membership.
var exists = struct{}{}

// parseYaml takes a file name and returns a gabs config object.
func parseYaml(fname string) (*gabs.Container, error) {
	data, err := ioutil.ReadFile(fname)
	if err != nil {
		fmt.Printf("Unable to read file: %s\n", err.Error())
		return nil, err
	}
	d, jsonErr := yaml.YAMLToJSON([]byte(data))
	if jsonErr != nil {
		fmt.Printf("Error encoding input to JSON.\n%s\n", jsonErr.Error())
		return nil, jsonErr
	}
	j, _ := gabs.ParseJSON(d)
	return j, nil
}

// validateConfig verifies that there are no unexpected values in the config file.
// at each level of the config, we should either have a KV for expansions, or a leaf node
// with the values oneof "expand", "query", "ssl_off" that map to a string.
func validateConfig(c *gabs.Container) (error) {
	var errors *multierror.Error
	children, _ := c.ChildrenMap()
	seenKeys := make(map[string]struct{})
	for k,v := range children {
		// Check if key already seen
		if _, ok := seenKeys[k]; ok {
			log.Printf("%s detected again", k)
			errors = multierror.Append(errors, fmt.Errorf("duplicate key detected %s", k))
		} else {
			seenKeys[k] = exists // mark key as seen
		}

		// Validate all children
		switch k {
		case
			"expand",
			"query":
			// check that v is a string, else return error.
			if _, ok := v.Data().(string); !ok {
				errors = multierror.Append(errors, fmt.Errorf("expected string value for %T, got: %v", k, v.Data()))
			}
		case "ssl_off":
			// check that v is a boolean, else return error.
			if _, ok := v.Data().(bool); !ok {
				errors = multierror.Append(errors, fmt.Errorf("expected bool value for %T, got: %v", k, v.Data()))
			}
		default:
			// Check if we have an unknown string here.
			if _, ok := v.Data().(string); ok {
				fmt.Printf("unexpected key %s", k)
				errors = multierror.Append(errors, fmt.Errorf("unexpected string value under key %s, got: %v", k, v.Data()))
			}
			// recurse, collect any errors.
			if err := validateConfig(v); err != nil {
				errors = multierror.Append(errors, err)
			}
		}
	}
	return errors.ErrorOrNil()
}

func watchChanges(watcher *fsnotify.Watcher, fname string, cb func()) {
	for {
		select {
		case event := <-watcher.Events:
			// You may wonder why we can't just listen for "Write" events. The reason is that vim (and other editors)
			// will create swap files, and when you write they delete the original and rename the swap file. This is great
			// for resolving system crashes, but also completely incompatible with inotify and other fswatch implementations.
			// Thus, we check that the file of interest might be created as well.
			updated := event.Op&fsnotify.Create == fsnotify.Create || event.Op&fsnotify.Write == fsnotify.Write
			zapconf := filepath.Clean(event.Name) == fname
			if updated && zapconf {
				cb()
			}
		case e := <-watcher.Errors:
			log.Println("error:", e)
		}
	}
}

// TODO: add tests. simulate touching a file.
// updateHosts will attempt to write the zap list of shortcuts
// to /etc/hosts. It will gracefully fail if there are not enough
// permissions to do so.
func updateHosts(c *context) {
	hostPath := "/etc/hosts"

	// 1. read file, prep buffer.
	data, err := ioutil.ReadFile(hostPath)
	if err != nil {
		log.Println("open config: ", err)
	}
	var replacement bytes.Buffer

	// 2. generate payload.
	replacement.WriteString(delimStart)
	children, _ := c.config.ChildrenMap()
	for k := range children {
		replacement.WriteString(fmt.Sprintf("127.0.0.1 %s\n", k))
	}
	replacement.WriteString(delimEnd)

	// 3. Generate new file content
	var updatedFile string
	if !strings.Contains(string(data), delimStart) {
		updatedFile = string(data) + replacement.String()
	} else {
		zapBlock := regexp.MustCompile("(###(.*)##)\n(.|\n)*(###(.*)##\n)")
		updatedFile = zapBlock.ReplaceAllString(string(data), replacement.String())
	}

	// 4. Attempt write to file.
	err = ioutil.WriteFile(hostPath, []byte(updatedFile), 0644)
	if err != nil {
		log.Printf("Error writing to '%s': %s\n", hostPath, err.Error())
	}
}

// makeCallback returns a func that that updates global state.
func makeCallback(c *context, configName string) func() {
	return func() {
		data, err := parseYaml(configName)
		if err != nil {
			log.Printf("Error loading new config: %s. Fallback to old config.", err)
			return
		}
		err = validateConfig(data)
		if err != nil {
			log.Printf("Error validating new config: %s. Fallback to old config.", err)
			return
		}

		// Update config atomically
		c.configMtx.Lock()
		c.config = data
		c.configMtx.Unlock()

		// Sync DNS entries.
		updateHosts(c)
		return
	}
}
<|endoftext|>


<commit_before>package govuk_crawler_worker

import (
	"github.com/streadway/amqp"
)

type QueueConnection struct {
	Connection *amqp.Connection
	Channel    *amqp.Channel
}

func NewQueueConnection(amqpURI string) (*QueueConnection, error) {
	connection, err := amqp.Dial(amqpURI)
	if err != nil {
		return nil, err
	}

	channel, err := connection.Channel()
	if err != nil {
		return nil, err
	}

	return &QueueConnection{
		Connection: connection,
		Channel: channel,
	}, nil
}

func (c *QueueConnection) Close() error {
	err := c.Channel.Close()
	if err != nil {
		return err
	}

	return c.Connection.Close()
}

func (c *QueueConnection) ExchangeDeclare(exchangeName string, exchangeType string) error {
	return c.Channel.ExchangeDeclare(
		exchangeName, // name of the exchange
		exchangeType, // type
		true,         // durable
		false,        // delete when complete
		false,        // internal
		false,        // noWait
		nil,          // arguments
	)
}

func (c *QueueConnection) QueueDeclare(name string) (amqp.Queue, error) {
	queue, err := c.Channel.QueueDeclare(
		name,  // name of the queue
		true,  // durable
		false, // delete when usused
		false, // exclusive
		false, // noWait
		nil)   // arguments
	if err != nil {
		return amqp.Queue{
			Name: name,
		}, err
	}

	return queue, nil
}
<commit_msg>Be consistent with parameter naming: name -> queueName<commit_after>package govuk_crawler_worker

import (
	"github.com/streadway/amqp"
)

type QueueConnection struct {
	Connection *amqp.Connection
	Channel    *amqp.Channel
}

func NewQueueConnection(amqpURI string) (*QueueConnection, error) {
	connection, err := amqp.Dial(amqpURI)
	if err != nil {
		return nil, err
	}

	channel, err := connection.Channel()
	if err != nil {
		return nil, err
	}

	return &QueueConnection{
		Connection: connection,
		Channel: channel,
	}, nil
}

func (c *QueueConnection) Close() error {
	err := c.Channel.Close()
	if err != nil {
		return err
	}

	return c.Connection.Close()
}

func (c *QueueConnection) ExchangeDeclare(exchangeName string, exchangeType string) error {
	return c.Channel.ExchangeDeclare(
		exchangeName, // name of the exchange
		exchangeType, // type
		true,         // durable
		false,        // delete when complete
		false,        // internal
		false,        // noWait
		nil,          // arguments
	)
}

func (c *QueueConnection) QueueDeclare(queueName string) (amqp.Queue, error) {
	queue, err := c.Channel.QueueDeclare(
		queueName, // name of the queue
		true,      // durable
		false,     // delete when usused
		false,     // exclusive
		false,     // noWait
		nil)       // arguments
	if err != nil {
		return amqp.Queue{
			Name: queueName,
		}, err
	}

	return queue, nil
}
<|endoftext|>


<commit_before>//
//  q_cmd.go
//  sed
//
// Copyright (c) 2009 Geoffrey Clements
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//

package sed

import (
	"fmt"
	"os"
	"strconv"
)

type q_cmd struct {
	addr		*address
	exit_code	int
}

func (c *q_cmd) match(line []byte, lineNumber int) bool {
	return c.addr.match(line, lineNumber)
}

func (c *q_cmd) String() string {
	if c != nil {
		if c.addr != nil {
			return fmt.Sprintf("{Quit Cmd addr:%s with exit code: %d}", c.addr.String(), c.exit_code)
		}
		return fmt.Sprintf("{Quit Cmd with exit code: %d}", c.exit_code)
	}
	return fmt.Sprint("{Quit Cmd}")
}

func NewQCmd(pieces [][]byte, addr *address) (c *q_cmd, err os.Error) {
	err = nil
	switch len(pieces) {
	case 2:
		c = new(q_cmd)
		c.addr = addr
		c.exit_code, err = strconv.Atoi(string(pieces[1]))
		if err != nil {
			c = nil
		}
	case 1:
		c = new(q_cmd)
		c.addr = addr
		c.exit_code = 0
	default:
		c, err = nil, WrongNumberOfCommandParameters
	}
	return c, err
}

func (c *q_cmd) processLine(s *Sed) (stop bool, err os.Error) {
	os.Exit(c.exit_code)
	return false, nil
}
<commit_msg>print command nicer<commit_after>//
//  q_cmd.go
//  sed
//
// Copyright (c) 2009 Geoffrey Clements
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//

package sed

import (
	"fmt"
	"os"
	"strconv"
)

type q_cmd struct {
	addr		*address
	exit_code	int
}

func (c *q_cmd) match(line []byte, lineNumber int) bool {
	return c.addr.match(line, lineNumber)
}

func (c *q_cmd) String() string {
	if c != nil {
		if c.addr != nil {
			return fmt.Sprintf("{q command addr:%s with exit code: %d}", c.addr.String(), c.exit_code)
		}
		return fmt.Sprintf("{q command with exit code: %d}", c.exit_code)
	}
	return fmt.Sprint("{q command}")
}

func NewQCmd(pieces [][]byte, addr *address) (c *q_cmd, err os.Error) {
	err = nil
	switch len(pieces) {
	case 2:
		c = new(q_cmd)
		c.addr = addr
		c.exit_code, err = strconv.Atoi(string(pieces[1]))
		if err != nil {
			c = nil
		}
	case 1:
		c = new(q_cmd)
		c.addr = addr
		c.exit_code = 0
	default:
		c, err = nil, WrongNumberOfCommandParameters
	}
	return c, err
}

func (c *q_cmd) processLine(s *Sed) (stop bool, err os.Error) {
	os.Exit(c.exit_code)
	return false, nil
}
<|endoftext|>


<commit_before>package raft

import "github.com/relab/raft/proto/gorums"

// QuorumSpec holds information about the quorum size of the current configuration
// and allows us to invoke QRPCs.
type QuorumSpec struct {
	N int
	Q int
}

// RequestVoteQF gathers RequestVoteResponses
// and delivers a reply when a higher term is seen or a quorum of votes is received.
func (qspec *QuorumSpec) RequestVoteQF(replies []*gorums.RequestVoteResponse) (*gorums.RequestVoteResponse, bool) {
	votes := 0
	response := &gorums.RequestVoteResponse{Term: replies[0].RequestTerm}

	for _, reply := range replies {
		if reply.Term > response.Term {
			response.Term = reply.Term

			return response, true
		}

		if reply.VoteGranted {
			votes++
		}

		if votes >= qspec.Q {
			response.VoteGranted = true
			return response, true
		}
	}

	return response, false
}

// TODO GoDoc
func (qspec *QuorumSpec) AppendEntriesQF(replies []*gorums.AppendEntriesResponse) (*gorums.AppendEntriesResponse, bool) {
	numSuccess := 0
	response := &gorums.AppendEntriesResponse{}

	var term uint64
	var matchIndex uint64

	for _, reply := range replies {
		if reply.MatchIndex < matchIndex || matchIndex == 0 {
			response.MatchIndex = reply.MatchIndex
		}

		if reply.Term < term || term == 0 {
			response.Term = reply.Term
		}

		if reply.Success {
			numSuccess++
			response.FollowerID = append(response.FollowerID, reply.FollowerID[0])
		}

		if numSuccess >= qspec.Q {
			reply.FollowerID = response.FollowerID

			return reply, true
		}
	}

	response.Success = false
	// FollowerID is now irrelevant.
	// Set to nil as accessing it is a bug.
	response.FollowerID = nil

	return response, false
}
<commit_msg>Response must contain the highest term seen<commit_after>package raft

import "github.com/relab/raft/proto/gorums"

// QuorumSpec holds information about the quorum size of the current configuration
// and allows us to invoke QRPCs.
type QuorumSpec struct {
	N int
	Q int
}

// RequestVoteQF gathers RequestVoteResponses
// and delivers a reply when a higher term is seen or a quorum of votes is received.
func (qspec *QuorumSpec) RequestVoteQF(replies []*gorums.RequestVoteResponse) (*gorums.RequestVoteResponse, bool) {
	votes := 0
	response := &gorums.RequestVoteResponse{Term: replies[0].RequestTerm}

	for _, reply := range replies {
		if reply.Term > response.Term {
			response.Term = reply.Term

			return response, true
		}

		if reply.VoteGranted {
			votes++
		}

		if votes >= qspec.Q {
			response.VoteGranted = true
			return response, true
		}
	}

	return response, false
}

// TODO GoDoc
func (qspec *QuorumSpec) AppendEntriesQF(replies []*gorums.AppendEntriesResponse) (*gorums.AppendEntriesResponse, bool) {
	numSuccess := 0
	response := &gorums.AppendEntriesResponse{}

	var term uint64
	var matchIndex uint64

	for _, reply := range replies {
		if reply.MatchIndex < matchIndex || matchIndex == 0 {
			response.MatchIndex = reply.MatchIndex
		}

		if reply.Term > term || term == 0 {
			response.Term = reply.Term
		}

		if reply.Success {
			numSuccess++
			response.FollowerID = append(response.FollowerID, reply.FollowerID[0])
		}

		if numSuccess >= qspec.Q {
			reply.FollowerID = response.FollowerID

			return reply, true
		}
	}

	response.Success = false
	// FollowerID is now irrelevant.
	// Set to nil as accessing it is a bug.
	response.FollowerID = nil

	return response, false
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/briandowns/spinner"
	"github.com/fatih/color"
	proxier "golang.org/x/net/proxy"
)

var (
	voiceURL = "https://dict.youdao.com/dictvoice?audio=%s&type=2"
)

func query(words []string, withVoice, withMore, isMulti bool) {
	var url string
	var doc *goquery.Document
	var voiceBody io.ReadCloser

	queryString := strings.Join(words, " ")
	voiceString := strings.Join(words, "+")

	isChinese := isChinese(queryString)

	if isChinese {
		url = "http://dict.youdao.com/w/eng/%s"
	} else {
		url = "http://dict.youdao.com/w/%s"
	}

	//Init spinner
	s := spinner.New(spinner.CharSets[35], 100*time.Millisecond)
	s.Prefix = "Querying... "
	s.Color("green")
	s.Start()

	//Check proxy
	if proxy != "" {
		client := &http.Client{}
		dialer, err := proxier.SOCKS5("tcp", proxy, nil, proxier.Direct)

		if err != nil {
			color.Red("Can't connect to the proxy: %s", err)
			os.Exit(1)
		}

		httpTransport := &http.Transport{}
		client.Transport = httpTransport
		httpTransport.Dial = dialer.Dial

		resp, err := client.Get(fmt.Sprintf(url, queryString))

		if err != nil {
			color.Red("Query failed with err: %s", err.Error())
			os.Exit(1)
		}

		doc, _ = goquery.NewDocumentFromResponse(resp)

		if withVoice && isAvailableOS() {
			if resp, err := client.Get(fmt.Sprintf(voiceURL, voiceString)); err == nil {
				voiceBody = resp.Body
			}
		}
	} else {
		var err error
		doc, err = goquery.NewDocument(fmt.Sprintf(url, queryString))

		if err != nil {
			log.Fatal(err)
			os.Exit(1)
		}

		if withVoice && isAvailableOS() {
			if resp, err := http.Get(fmt.Sprintf(voiceURL, voiceString)); err == nil {
				voiceBody = resp.Body
			}
		}
	}

	s.Stop()

	if isChinese {
		// Find the result
		fmt.Println()
		doc.Find(".trans-container > ul > p > span.contentTitle").Each(func(i int, s *goquery.Selection) {
			color.Green("    %s", s.Find(".search-js").Text())
		})
	} else {

		// Check for typos
		if hint := getHint(doc); hint != nil {
			color.Blue("\r\n    word '%s' not found, do you mean?", queryString)
			fmt.Println()
			for _, guess := range hint {
				color.Green("    %s", guess[0])
				color.Magenta("    %s", guess[1])
			}
			fmt.Println()
			return
		}

		// Find the pronounce
		if !isMulti {
			color.Green("\r\n    %s", getPronounce(doc))
		}

		// Find the result
		result := doc.Find("div#phrsListTab > div.trans-container > ul").Text()
		color.Green(result)
	}

	// Show examples
	sentences := getSentences(words, doc, isChinese, withMore)
	if len(sentences) > 0 {
		fmt.Println()
		for i, sentence := range sentences {
			color.Green(" %2d.%s", i+1, sentence[0])
			color.Magenta("    %s", sentence[1])
		}
		fmt.Println()
	}

	if withVoice && isAvailableOS() {
		playVoice(voiceBody)
	}
}

func playVoice(body io.ReadCloser) {
	tmpfile, err := ioutil.TempFile("", "ydict")
	if err != nil {
		log.Fatal(err)
	}

	defer os.Remove(tmpfile.Name()) // clean up

	data, err := ioutil.ReadAll(body)
	if err != nil {
		log.Fatal(err)
	}

	if _, err := tmpfile.Write(data); err != nil {
		log.Fatal(err)
	}

	if err := tmpfile.Close(); err != nil {
		fmt.Println(err)
	}

	cmd := exec.Command("mpg123", tmpfile.Name())

	if err := cmd.Start(); err != nil {
		fmt.Println(err)
	}

	if err := cmd.Wait(); err != nil {
		fmt.Println(err)
	}
}

func getPronounce(doc *goquery.Document) string {
	var pronounce string
	doc.Find("div.baav > span.pronounce").Each(func(i int, s *goquery.Selection) {

		if i == 0 {
			p := fmt.Sprintf("英: %s    ", s.Find("span.phonetic").Text())
			pronounce += p
		}

		if i == 1 {
			p := fmt.Sprintf("美: %s", s.Find("span.phonetic").Text())
			pronounce += p
		}
	})

	return pronounce
}

func getHint(doc *goquery.Document) [][]string {
	typos := doc.Find(".typo-rel")
	if typos.Length() == 0 {
		return nil
	}
	result := [][]string{}
	typos.Each(func(_ int, s *goquery.Selection) {
		word := strings.TrimSpace(s.Find("a").Text())
		s.Children().Remove()
		mean := strings.TrimSpace(s.Text())
		result = append(result, []string{word, mean})
	})
	return result
}

func getSentences(words []string, doc *goquery.Document, isChinese, withMore bool) [][]string {
	result := [][]string{}
	if withMore {
		url := fmt.Sprintf("http://dict.youdao.com/example/blng/eng/%s", strings.Join(words, "_"))
		var err error
		doc, err = goquery.NewDocument(url)
		if err != nil {
			return result
		}
	}
	doc.Find("#bilingual ul li").Each(func(_ int, s *goquery.Selection) {
		r := []string{}
		s.Children().Each(func(ii int, ss *goquery.Selection) {
			// Ignore source
			if ii == 2 {
				return
			}
			var sentence string
			ss.Children().Each(func(iii int, sss *goquery.Selection) {
				if text := strings.TrimSpace(sss.Text()); text != "" {
					addSpace := (ii == 1 && isChinese) || (ii == 0 && !isChinese) && iii != 0 && text != "."
					if addSpace {
						text = " " + text
					}
					sentence += text
				}
			})
			r = append(r, sentence)
		})
		if len(r) == 2 {
			result = append(result, r)
		}
	})
	return result
}
<commit_msg>Show word's part of speech<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/briandowns/spinner"
	"github.com/fatih/color"
	proxier "golang.org/x/net/proxy"
)

var (
	voiceURL = "https://dict.youdao.com/dictvoice?audio=%s&type=2"
)

func query(words []string, withVoice, withMore, isMulti bool) {
	var url string
	var doc *goquery.Document
	var voiceBody io.ReadCloser

	queryString := strings.Join(words, " ")
	voiceString := strings.Join(words, "+")

	isChinese := isChinese(queryString)

	if isChinese {
		url = "http://dict.youdao.com/w/eng/%s"
	} else {
		url = "http://dict.youdao.com/w/%s"
	}

	//Init spinner
	s := spinner.New(spinner.CharSets[35], 100*time.Millisecond)
	s.Prefix = "Querying... "
	s.Color("green")
	s.Start()

	//Check proxy
	if proxy != "" {
		client := &http.Client{}
		dialer, err := proxier.SOCKS5("tcp", proxy, nil, proxier.Direct)

		if err != nil {
			color.Red("Can't connect to the proxy: %s", err)
			os.Exit(1)
		}

		httpTransport := &http.Transport{}
		client.Transport = httpTransport
		httpTransport.Dial = dialer.Dial

		resp, err := client.Get(fmt.Sprintf(url, queryString))

		if err != nil {
			color.Red("Query failed with err: %s", err.Error())
			os.Exit(1)
		}

		doc, _ = goquery.NewDocumentFromResponse(resp)

		if withVoice && isAvailableOS() {
			if resp, err := client.Get(fmt.Sprintf(voiceURL, voiceString)); err == nil {
				voiceBody = resp.Body
			}
		}
	} else {
		var err error
		doc, err = goquery.NewDocument(fmt.Sprintf(url, queryString))

		if err != nil {
			log.Fatal(err)
			os.Exit(1)
		}

		if withVoice && isAvailableOS() {
			if resp, err := http.Get(fmt.Sprintf(voiceURL, voiceString)); err == nil {
				voiceBody = resp.Body
			}
		}
	}

	s.Stop()

	if isChinese {
		// Find the result
		fmt.Println()
		doc.Find(".trans-container > ul > p").Each(func(i int, s *goquery.Selection) {
			partOfSpeech := s.Children().Not(".contentTitle").Text()
			if partOfSpeech != "" {
				fmt.Printf("%14s ", color.MagentaString(partOfSpeech))
			}

			meanings := []string{}
			s.Find(".contentTitle > .search-js").Each(func(ii int, ss *goquery.Selection) {
				meanings = append(meanings, ss.Text())
			})
			fmt.Printf("%s\n", color.GreenString(strings.Join(meanings, "; ")))
		})
	} else {

		// Check for typos
		if hint := getHint(doc); hint != nil {
			color.Blue("\r\n    word '%s' not found, do you mean?", queryString)
			fmt.Println()
			for _, guess := range hint {
				color.Green("    %s", guess[0])
				color.Magenta("    %s", guess[1])
			}
			fmt.Println()
			return
		}

		// Find the pronounce
		if !isMulti {
			color.Green("\r\n    %s", getPronounce(doc))
		}

		// Find the result
		result := doc.Find("div#phrsListTab > div.trans-container > ul").Text()
		color.Green(result)
	}

	// Show examples
	sentences := getSentences(words, doc, isChinese, withMore)
	if len(sentences) > 0 {
		fmt.Println()
		for i, sentence := range sentences {
			color.Green(" %2d.%s", i+1, sentence[0])
			color.Magenta("    %s", sentence[1])
		}
		fmt.Println()
	}

	if withVoice && isAvailableOS() {
		playVoice(voiceBody)
	}
}

func playVoice(body io.ReadCloser) {
	tmpfile, err := ioutil.TempFile("", "ydict")
	if err != nil {
		log.Fatal(err)
	}

	defer os.Remove(tmpfile.Name()) // clean up

	data, err := ioutil.ReadAll(body)
	if err != nil {
		log.Fatal(err)
	}

	if _, err := tmpfile.Write(data); err != nil {
		log.Fatal(err)
	}

	if err := tmpfile.Close(); err != nil {
		fmt.Println(err)
	}

	cmd := exec.Command("mpg123", tmpfile.Name())

	if err := cmd.Start(); err != nil {
		fmt.Println(err)
	}

	if err := cmd.Wait(); err != nil {
		fmt.Println(err)
	}
}

func getPronounce(doc *goquery.Document) string {
	var pronounce string
	doc.Find("div.baav > span.pronounce").Each(func(i int, s *goquery.Selection) {

		if i == 0 {
			p := fmt.Sprintf("英: %s    ", s.Find("span.phonetic").Text())
			pronounce += p
		}

		if i == 1 {
			p := fmt.Sprintf("美: %s", s.Find("span.phonetic").Text())
			pronounce += p
		}
	})

	return pronounce
}

func getHint(doc *goquery.Document) [][]string {
	typos := doc.Find(".typo-rel")
	if typos.Length() == 0 {
		return nil
	}
	result := [][]string{}
	typos.Each(func(_ int, s *goquery.Selection) {
		word := strings.TrimSpace(s.Find("a").Text())
		s.Children().Remove()
		mean := strings.TrimSpace(s.Text())
		result = append(result, []string{word, mean})
	})
	return result
}

func getSentences(words []string, doc *goquery.Document, isChinese, withMore bool) [][]string {
	result := [][]string{}
	if withMore {
		url := fmt.Sprintf("http://dict.youdao.com/example/blng/eng/%s", strings.Join(words, "_"))
		var err error
		doc, err = goquery.NewDocument(url)
		if err != nil {
			return result
		}
	}
	doc.Find("#bilingual ul li").Each(func(_ int, s *goquery.Selection) {
		r := []string{}
		s.Children().Each(func(ii int, ss *goquery.Selection) {
			// Ignore source
			if ii == 2 {
				return
			}
			var sentence string
			ss.Children().Each(func(iii int, sss *goquery.Selection) {
				if text := strings.TrimSpace(sss.Text()); text != "" {
					addSpace := (ii == 1 && isChinese) || (ii == 0 && !isChinese) && iii != 0 && text != "."
					if addSpace {
						text = " " + text
					}
					sentence += text
				}
			})
			r = append(r, sentence)
		})
		if len(r) == 2 {
			result = append(result, r)
		}
	})
	return result
}
<|endoftext|>


<commit_before>// Package queue is a FIFO queue using the circularly linked list package in
// Go's standard library. There is also a thread safe version.
package queue

import (
	"container/list"
	"sync"
)

// Queue represents the FIFO queue.
type Queue struct {
	l *list.List
}

type AsyncQueue struct {
	q    *Queue
	lock sync.Mutex
}

// Returns an initialized Queue.
func New() Queue {
	return Queue{list.New()}
}

func NewSync() AsyncQueue {
	return AsyncQueue{q: &Queue{list.New()}}
}

// Pushes a new item to the back of the Queue.
func (q Queue) Push(o interface{}) {
	q.l.PushBack(o)
}

// Removes an item from the front of the Queue and returns it's value or nil.
func (q Queue) Pop() interface{} {
	e := q.l.Front()
	if e == nil {
		return nil
	}

	return q.l.Remove(e)
}

// Checks to see if the Queue is empty.
func (q Queue) IsEmpty() bool {
	return q.l.Len() == 0
}

// Returns the current length of the Queue.
func (q Queue) Len() int {
	return q.l.Len()
}

// Returns the item at the front of the Queue or nil.
// The item is a *list.Element from the 'container/list' package.
func (q Queue) Front() *list.Element {
	return q.l.Front()
}

// Returns the item after e or nil it is the last item or nil.
// The item is a *list.Element from the 'container/list' package.
// Even though it is possible to call e.Next() directly, don't. This behavior
// may not be supported moving forward.
func (q Queue) Next(e *list.Element) *list.Element {
	if e == nil {
		return e
	}

	return e.Next()
}

func (q AsyncQueue) Push(o interface{}) {
	q.lock.Lock()
	defer q.lock.Unlock()

	q.q.Push(o)
}

func (q AsyncQueue) Pop() interface{} {
	q.lock.Lock()
	defer q.lock.Unlock()

	return q.q.Pop()
}

func (q AsyncQueue) IsEmpty() bool {
	return q.q.IsEmpty()
}

func (q AsyncQueue) Len() int {
	return q.q.Len()
}

func (q AsyncQueue) Front() *list.Element {
	return q.q.Front()
}

func (q AsyncQueue) Next(e *list.Element) *list.Element {
	return q.q.Next(e)
}
<commit_msg>Documenting the thread safe queue<commit_after>// Package queue is a FIFO queue using the circularly linked list package in
// Go's standard library. There is also a thread safe version.
package queue

import (
	"container/list"
	"sync"
)

// Queue represents the FIFO queue.
type Queue struct {
	l *list.List
}

// A thread safe version of Queue.
type AsyncQueue struct {
	q    *Queue
	lock sync.Mutex
}

// Returns an initialized Queue.
func New() Queue {
	return Queue{list.New()}
}

// Returns an initialized SyncQueue
func NewSync() AsyncQueue {
	return AsyncQueue{q: &Queue{list.New()}}
}

// Pushes a new item to the back of the Queue.
func (q Queue) Push(o interface{}) {
	q.l.PushBack(o)
}

// Removes an item from the front of the Queue and returns it's value or nil.
func (q Queue) Pop() interface{} {
	e := q.l.Front()
	if e == nil {
		return nil
	}

	return q.l.Remove(e)
}

// Checks to see if the Queue is empty.
func (q Queue) IsEmpty() bool {
	return q.l.Len() == 0
}

// Returns the current length of the Queue.
func (q Queue) Len() int {
	return q.l.Len()
}

// Returns the item at the front of the Queue or nil.
// The item is a *list.Element from the 'container/list' package.
func (q Queue) Front() *list.Element {
	return q.l.Front()
}

// Returns the item after e or nil it is the last item or nil.
// The item is a *list.Element from the 'container/list' package.
// Even though it is possible to call e.Next() directly, don't. This behavior
// may not be supported moving forward.
func (q Queue) Next(e *list.Element) *list.Element {
	if e == nil {
		return e
	}

	return e.Next()
}

// Same as Push for Queue, except it is thread safe.
func (q AsyncQueue) Push(o interface{}) {
	q.lock.Lock()
	defer q.lock.Unlock()

	q.q.Push(o)
}

// Same as Pop for Queue, except it is thread safe.
func (q AsyncQueue) Pop() interface{} {
	q.lock.Lock()
	defer q.lock.Unlock()

	return q.q.Pop()
}

// Same as IsEmpty for Queue, except it is thread safe.
func (q AsyncQueue) IsEmpty() bool {
	return q.q.IsEmpty()
}

// Same as Len for Queue, except it is thread safe.
func (q AsyncQueue) Len() int {
	return q.q.Len()
}

// Same as Front for Queue, except it is thread safe.
func (q AsyncQueue) Front() *list.Element {
	return q.q.Front()
}

// Same as Next for Queue, except it is thread safe.
func (q AsyncQueue) Next(e *list.Element) *list.Element {
	return q.q.Next(e)
}
<|endoftext|>


<commit_before>//go:generate protoc ./minion/pb/pb.proto --go_out=plugins=grpc:.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	l_mod "log"
	"os"
	"strings"

	"github.com/NetSys/quilt/api"
	"github.com/NetSys/quilt/api/server"
	"github.com/NetSys/quilt/cluster"
	"github.com/NetSys/quilt/db"
	"github.com/NetSys/quilt/minion"
	"github.com/NetSys/quilt/quiltctl"
	"github.com/NetSys/quilt/util"

	"google.golang.org/grpc/grpclog"

	log "github.com/Sirupsen/logrus"
)

func main() {
	flag.Usage = func() {
		fmt.Println("Usage: quilt " +
			"[-log-level=<level> | -l=<level>] [-H=<listen_address>] " +
			"[daemon | inspect <stitch> | run <stitch> | minion | " +
			"stop <namespace> | get <import_path> | " +
			"machines | containers | ssh <machine>]")
		fmt.Println("\nWhen provided a stitch, quilt takes responsibility\n" +
			"for deploying it as specified.  Alternatively, quilt may be\n" +
			"instructed to stop all deployments in a given namespace,\n" +
			"or the default namespace if none is provided.\n")
		flag.PrintDefaults()
		fmt.Println("        Valid logger levels are:\n" +
			"            debug, info, warn, error, fatal or panic.")
	}

	var logLevel = flag.String("log-level", "info", "level to set logger to")
	flag.StringVar(logLevel, "l", "info", "level to set logger to")
	var lAddr = flag.String("H", api.DefaultSocket,
		"Socket to listen for API requests on.")
	flag.Parse()

	level, err := parseLogLevel(*logLevel)
	if err != nil {
		fmt.Println(err)
		usage()
	}
	log.SetLevel(level)
	log.SetFormatter(util.Formatter{})

	// GRPC spews a lot of useless log messages so we tell to eat its logs, unless
	// we are in debug mode
	grpclog.SetLogger(l_mod.New(ioutil.Discard, "", 0))
	if level == log.DebugLevel {
		grpclog.SetLogger(log.StandardLogger())
	}

	if len(flag.Args()) == 0 {
		usage()
	}

	subcommand := flag.Arg(0)
	switch {
	case subcommand == "minion":
		minion.Run()
	case subcommand == "daemon":
		runDaemon(*lAddr)
	case quiltctl.HasSubcommand(subcommand):
		quiltctl.Run(flag.Args())
	default:
		usage()
	}
}

func runDaemon(lAddr string) {
	conn := db.New()
	go server.Run(conn, lAddr)
	cluster.Run(conn)
}

func usage() {
	flag.Usage()
	os.Exit(1)
}

// parseLogLevel returns the log.Level type corresponding to the given string
// (case insensitive).
// If no such matching string is found, it returns log.InfoLevel (default) and an error.
func parseLogLevel(logLevel string) (log.Level, error) {
	logLevel = strings.ToLower(logLevel)
	switch logLevel {
	case "debug":
		return log.DebugLevel, nil
	case "info":
		return log.InfoLevel, nil
	case "warn":
		return log.WarnLevel, nil
	case "error":
		return log.ErrorLevel, nil
	case "fatal":
		return log.FatalLevel, nil
	case "panic":
		return log.PanicLevel, nil
	}
	return log.InfoLevel, fmt.Errorf("bad log level: '%v'", logLevel)
}
<commit_msg>quilt: Add option to send logs to a file<commit_after>//go:generate protoc ./minion/pb/pb.proto --go_out=plugins=grpc:.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	l_mod "log"
	"os"
	"strings"

	"github.com/NetSys/quilt/api"
	"github.com/NetSys/quilt/api/server"
	"github.com/NetSys/quilt/cluster"
	"github.com/NetSys/quilt/db"
	"github.com/NetSys/quilt/minion"
	"github.com/NetSys/quilt/quiltctl"
	"github.com/NetSys/quilt/util"

	"google.golang.org/grpc/grpclog"

	log "github.com/Sirupsen/logrus"
)

func main() {
	flag.Usage = func() {
		fmt.Println("Usage: quilt " +
			"[-log-level=<level> | -l=<level>] [-H=<listen_address>] " +
			"[log-file=<log_output_file>] " +
			"[daemon | inspect <stitch> | run <stitch> | minion | " +
			"stop <namespace> | get <import_path> | " +
			"machines | containers | ssh <machine>]")
		fmt.Println("\nWhen provided a stitch, quilt takes responsibility\n" +
			"for deploying it as specified.  Alternatively, quilt may be\n" +
			"instructed to stop all deployments in a given namespace,\n" +
			"or the default namespace if none is provided.\n")
		flag.PrintDefaults()
		fmt.Println("        Valid logger levels are:\n" +
			"            debug, info, warn, error, fatal or panic.")
	}

	var logOut = flag.String("log-file", "", "log output file (will be overwritten)")
	var logLevel = flag.String("log-level", "info", "level to set logger to")
	flag.StringVar(logLevel, "l", "info", "level to set logger to")
	var lAddr = flag.String("H", api.DefaultSocket,
		"Socket to listen for API requests on.")
	flag.Parse()

	level, err := parseLogLevel(*logLevel)
	if err != nil {
		fmt.Println(err)
		usage()
	}
	log.SetLevel(level)
	log.SetFormatter(util.Formatter{})

	if *logOut != "" {
		file, err := os.Create(*logOut)
		if err != nil {
			fmt.Printf("Failed to create file %s\n", *logOut)
			os.Exit(1)
		}
		defer file.Close()
		log.SetOutput(file)
	}

	// GRPC spews a lot of useless log messages so we tell to eat its logs, unless
	// we are in debug mode
	grpclog.SetLogger(l_mod.New(ioutil.Discard, "", 0))
	if level == log.DebugLevel {
		grpclog.SetLogger(log.StandardLogger())
	}

	if len(flag.Args()) == 0 {
		usage()
	}

	subcommand := flag.Arg(0)
	switch {
	case subcommand == "minion":
		minion.Run()
	case subcommand == "daemon":
		runDaemon(*lAddr)
	case quiltctl.HasSubcommand(subcommand):
		quiltctl.Run(flag.Args())
	default:
		usage()
	}
}

func runDaemon(lAddr string) {
	conn := db.New()
	go server.Run(conn, lAddr)
	cluster.Run(conn)
}

func usage() {
	flag.Usage()
	os.Exit(1)
}

// parseLogLevel returns the log.Level type corresponding to the given string
// (case insensitive).
// If no such matching string is found, it returns log.InfoLevel (default) and an error.
func parseLogLevel(logLevel string) (log.Level, error) {
	logLevel = strings.ToLower(logLevel)
	switch logLevel {
	case "debug":
		return log.DebugLevel, nil
	case "info":
		return log.InfoLevel, nil
	case "warn":
		return log.WarnLevel, nil
	case "error":
		return log.ErrorLevel, nil
	case "fatal":
		return log.FatalLevel, nil
	case "panic":
		return log.PanicLevel, nil
	}
	return log.InfoLevel, fmt.Errorf("bad log level: '%v'", logLevel)
}
<|endoftext|>