<commit_before>package pgmgr

import (
	"fmt"
	"io/ioutil"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"../pgmgr"
)

const (
	testDBName      = "pgmgr_testdb"
	migrationFolder = "/tmp/migrations/"
	dumpFile        = "/tmp/pgmgr_dump.sql"
)

func globalConfig() *pgmgr.Config {
	return &pgmgr.Config{
		Database:        testDBName,
		Host:            "localhost",
		Port:            5432,
		DumpFile:        dumpFile,
		MigrationFolder: migrationFolder,
	}
}

func TestCreate(t *testing.T) {
	if err := dropDB(t); err != nil {
		t.Fatal("dropdb failed: ", err)
	}

	if err := pgmgr.Create(globalConfig()); err != nil {
		t.Log(err)
		t.Fatal("Could not create database")
	}

	// if we can't remove that db, it couldn't have been created by us above.
	if err := dropDB(t); err != nil {
		t.Fatal("database doesn't seem to have been created!")
	}
}

func TestDrop(t *testing.T) {
	if err := createDB(t); err != nil {
		t.Fatal("createdb failed: ", err)
	}

	if err := pgmgr.Drop(globalConfig()); err != nil {
		t.Log(err)
		t.Fatal("Could not drop database")
	}

	if err := createDB(t); err != nil {
		t.Fatal("database doesn't seem to have been dropped!")
	}
}

func TestDump(t *testing.T) {
	resetDB(t)
	testSh(t, "psql", []string{"-d", testDBName, "-c", "CREATE TABLE bars (bar_id INTEGER);"})
	testSh(t, "psql", []string{"-d", testDBName, "-c", "INSERT INTO bars (bar_id) VALUES (123), (456);"})
	testSh(t, "psql", []string{"-d", testDBName, "-c", "CREATE TABLE foos (foo_id INTEGER);"})
	testSh(t, "psql", []string{"-d", testDBName, "-c", "INSERT INTO foos (foo_id) VALUES (789);"})

	c := globalConfig()
	err := pgmgr.Dump(c)

	if err != nil {
		t.Log(err)
		t.Fatal("Could not dump database to file")
	}

	file, err := ioutil.ReadFile(dumpFile)
	if err != nil {
		t.Log(err)
		t.Fatal("Could not read dump")
	}

	if !strings.Contains(string(file), "CREATE TABLE bars") {
		t.Fatal("dump does not contain the table definition")
	}

	if !strings.Contains(string(file), "123") {
		t.Fatal("dump does not contain the table data when --seed-tables is not specified")
	}

	c.SeedTables = append(c.SeedTables, "foos")
	err = pgmgr.Dump(c)

	if err != nil {
		t.Log(err)
		t.Fatal("Could not dump database to file")
	}

	file, err = ioutil.ReadFile(dumpFile)
	if err != nil {
		t.Log(err)
		t.Fatal("Could not read dump")
	}

	if strings.Contains(string(file), "123") {
		t.Fatal("dump contains table data for non-seed tables, when --seed-tables was given")
	}

	if !strings.Contains(string(file), "789") {
		t.Fatal("dump does not contain table data for seed tables, when --seed-tables was given")
	}
}

func TestLoad(t *testing.T) {
	resetDB(t)

	ioutil.WriteFile(dumpFile, []byte(`
		CREATE TABLE foos (foo_id INTEGER);
		INSERT INTO foos (foo_id) VALUES (1), (2), (3);
	`), 0644)

	err := pgmgr.Load(globalConfig())

	if err != nil {
		t.Log(err)
		t.Fatal("Could not load database from file")
	}

	err = testSh(t, "psql", []string{"-d", testDBName, "-c", "SELECT * FROM foos;"})
	if err != nil {
		t.Log(err)
		t.Fatal("Could not query the table; schema didn't load, probably")
	}
}

func TestVersion(t *testing.T) {
	resetDB(t)

	version, err := pgmgr.Version(globalConfig())

	if err != nil {
		t.Log(err)
		t.Fatal("Could not fetch version info")
	}

	if version != -1 {
		t.Fatal("expected version to be -1 before table exists, got", version)
	}

	pgmgr.Initialize(globalConfig())

	testSh(t, "psql", []string{"-e", "-d", testDBName, "-c", "INSERT INTO schema_migrations (version) VALUES (1)"})

	version, err = pgmgr.Version(globalConfig())

	if version != 1 {
		t.Fatal("expected version to be 1, got", version)
	}
}

func TestColumnTypeString(t *testing.T) {
	resetDB(t)

	config := globalConfig()
	config.ColumnType = "string"
	pgmgr.Initialize(config)

	testSh(t, "psql", []string{"-e", "-d", testDBName, "-c", "INSERT INTO schema_migrations (version) VALUES ('20150910120933')"})
	version, err := pgmgr.Version(config)
	if err != nil {
		t.Fatal(err)
	}

	if version != 20150910120933 {
		t.Fatal("expected version to be 20150910120933, got", version)
	}
}

func TestMigrate(t *testing.T) {
	// start with an empty DB
	resetDB(t)
	clearMigrationFolder(t)

	// add our first migration
	writeMigration(t, "002_this_is_a_migration.up.sql", `
		CREATE TABLE foos (foo_id INTEGER);
		INSERT INTO foos (foo_id) VALUES (1), (2), (3);
	`)

	writeMigration(t, "002_this_is_a_migration.down.sql", `DROP TABLE foos;`)

	err := pgmgr.Migrate(globalConfig())

	if err != nil {
		t.Log(err)
		t.Fatal("Migrations failed to run.")
	}

	// test simple idempotency
	err = pgmgr.Migrate(globalConfig())
	if err != nil {
		t.Log(err)
		t.Fatal("Running migrations again was not idempotent!")
	}

	err = testSh(t, "psql", []string{"-d", testDBName, "-c", "SELECT * FROM foos;"})
	if err != nil {
		t.Log(err)
		t.Fatal("Could not query the table; migration didn't apply, probably")
	}

	// add a new migration with an older version, as if another dev's branch was merged in
	writeMigration(t, "001_this_is_an_older_migration.up.sql", `
		CREATE TABLE bars (bar_id INTEGER);
		INSERT INTO bars (bar_id) VALUES (4), (5), (6);
	`)

	err = pgmgr.Migrate(globalConfig())
	if err != nil {
		t.Log(err)
		t.Fatal("Could not apply second migration!")
	}

	err = testSh(t, "psql", []string{"-d", testDBName, "-c", "SELECT * FROM bars;"})
	if err != nil {
		t.Log(err)
		t.Fatal("Could not query the table; second migration didn't apply, probably")
	}

	// rollback the initial migration, since it has the latest version
	err = pgmgr.Rollback(globalConfig())

	err = testSh(t, "psql", []string{"-d", testDBName, "-c", "SELECT * FROM foos;"})
	if err == nil {
		t.Log(err)
		t.Fatal("Could query the table; migration didn't downgrade")
	}

	v, err := pgmgr.Version(globalConfig())
	if err != nil || v != 1 {
		t.Log(err)
		t.Fatal("Rollback did not reset version! Still on version ", v)
	}
}

func TestMigrateColumnTypeString(t *testing.T) {
	// start with an empty DB
	resetDB(t)
	clearMigrationFolder(t)

	config := globalConfig()
	config.ColumnType = "string"

	// migrate up
	writeMigration(t, "20150910120933_some_migration.up.sql", `
		CREATE TABLE foos (foo_id INTEGER);
		INSERT INTO foos (foo_id) VALUES (1), (2), (3);
	`)

	err := pgmgr.Migrate(config)
	if err != nil {
		t.Fatal(err)
	}

	v, err := pgmgr.Version(config)
	if err != nil {
		t.Fatal(err)
	}

	if v != 20150910120933 {
		t.Fatal("Expected version 20150910120933 after migration, got", v)
	}

	// migrate down
	writeMigration(t, "20150910120933_some_migration.down.sql", `DROP TABLE foos;`)

	err = pgmgr.Rollback(config)
	if err != nil {
		t.Fatal(err)
	}

	v, err = pgmgr.Version(config)
	if err != nil {
		t.Fatal(err)
	}

	if v != -1 {
		t.Fatal("Expected version -1 after rollback, got", v)
	}
}

func TestMigrateNoTransaction(t *testing.T) {
	// start with an empty DB
	resetDB(t)
	clearMigrationFolder(t)

	// CREATE INDEX CONCURRENTLY can not run inside a transaction, so we can assert
	// that no transaction was used by verifying it ran successfully.
	writeMigration(t, "001_create_foos.up.sql", `CREATE TABLE foos (foo_id INTEGER);`)
	writeMigration(t, "002_index_foos.no_txn.up.sql", `CREATE INDEX CONCURRENTLY idx_foo_id ON foos(foo_id);`)

	err := pgmgr.Migrate(globalConfig())
	if err != nil {
		t.Fatal(err)
	}
}

func TestCreateMigration(t *testing.T) {
	clearMigrationFolder(t)

	assertFileExists := func(filename string) {
		err := testSh(t, "stat", []string{filepath.Join(migrationFolder, filename)})
		if err != nil {
			t.Fatal(err)
		}
	}

	expectedVersion := time.Now().Unix()
	err := pgmgr.CreateMigration(globalConfig(), "new_migration", false)
	if err != nil {
		t.Fatal(err)
	}

	assertFileExists(fmt.Sprint(expectedVersion, "_new_migration.up.sql"))
	assertFileExists(fmt.Sprint(expectedVersion, "_new_migration.down.sql"))

	expectedStringVersion := time.Now().Format(datetimeFormat)
	config := globalConfig()
	config.Format = "datetime"
	err = pgmgr.CreateMigration(config, "rails_style", false)
	if err != nil {
		t.Fatal(err)
	}

	assertFileExists(fmt.Sprint(expectedStringVersion, "_rails_style.up.sql"))
	assertFileExists(fmt.Sprint(expectedStringVersion, "_rails_style.down.sql"))

	err = pgmgr.CreateMigration(config, "create_index", true)
	if err != nil {
		t.Fatal(err)
	}

	assertFileExists(fmt.Sprint(expectedStringVersion, "_create_index.no_txn.up.sql"))
	assertFileExists(fmt.Sprint(expectedStringVersion, "_create_index.no_txn.down.sql"))
}

// redundant, but I'm also lazy
func testSh(t *testing.T, command string, args []string) error {
	c := exec.Command(command, args...)
	output, err := c.CombinedOutput()
	t.Log(string(output))
	if err != nil {
		return err
	}

	return nil
}

func resetDB(t *testing.T) {
	if err := dropDB(t); err != nil {
		t.Fatal("dropdb failed: ", err)
	}

	if err := createDB(t); err != nil {
		t.Fatal("createdb failed: ", err)
	}
}

func dropDB(t *testing.T) error {
	return testSh(t, "dropdb", []string{testDBName})
}

func createDB(t *testing.T) error {
	return testSh(t, "createdb", []string{testDBName})
}

func clearMigrationFolder(t *testing.T) {
	if err := testSh(t, "rm", []string{"-r", migrationFolder}); err != nil {
		t.Fatalf("Could not remove directory %s: %s", migrationFolder, err)
	}

	if err := testSh(t, "mkdir", []string{migrationFolder}); err != nil {
		t.Fatalf("Could not create directory %s: %s", migrationFolder, err)
	}
}

func writeMigration(t *testing.T, name, contents string) {
	filename := path.Join(migrationFolder, name)
	err := ioutil.WriteFile(filename, []byte(contents), 0644)
	if err != nil {
		t.Fatalf("Failed to write %s: %s", filename, err)
	}
}
<commit_msg>psqlExec, psqlMustExec test helpers<commit_after>package pgmgr

import (
	"fmt"
	"io/ioutil"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"../pgmgr"
)

const (
	testDBName      = "pgmgr_testdb"
	migrationFolder = "/tmp/migrations/"
	dumpFile        = "/tmp/pgmgr_dump.sql"
)

func globalConfig() *pgmgr.Config {
	return &pgmgr.Config{
		Database:        testDBName,
		Host:            "localhost",
		Port:            5432,
		DumpFile:        dumpFile,
		MigrationFolder: migrationFolder,
	}
}

func TestCreate(t *testing.T) {
	if err := dropDB(t); err != nil {
		t.Fatal("dropdb failed: ", err)
	}

	if err := pgmgr.Create(globalConfig()); err != nil {
		t.Log(err)
		t.Fatal("Could not create database")
	}

	// if we can't remove that db, it couldn't have been created by us above.
	if err := dropDB(t); err != nil {
		t.Fatal("database doesn't seem to have been created!")
	}
}

func TestDrop(t *testing.T) {
	if err := createDB(t); err != nil {
		t.Fatal("createdb failed: ", err)
	}

	if err := pgmgr.Drop(globalConfig()); err != nil {
		t.Log(err)
		t.Fatal("Could not drop database")
	}

	if err := createDB(t); err != nil {
		t.Fatal("database doesn't seem to have been dropped!")
	}
}

func TestDump(t *testing.T) {
	resetDB(t)
	psqlMustExec(t, `CREATE TABLE bars (bar_id INTEGER);`)
	psqlMustExec(t, `INSERT INTO bars (bar_id) VALUES (123), (456);`)
	psqlMustExec(t, `CREATE TABLE foos (foo_id INTEGER);`)
	psqlMustExec(t, `INSERT INTO foos (foo_id) VALUES (789);`)

	c := globalConfig()
	err := pgmgr.Dump(c)

	if err != nil {
		t.Log(err)
		t.Fatal("Could not dump database to file")
	}

	file, err := ioutil.ReadFile(dumpFile)
	if err != nil {
		t.Log(err)
		t.Fatal("Could not read dump")
	}

	if !strings.Contains(string(file), "CREATE TABLE bars") {
		t.Fatal("dump does not contain the table definition")
	}

	if !strings.Contains(string(file), "123") {
		t.Fatal("dump does not contain the table data when --seed-tables is not specified")
	}

	c.SeedTables = append(c.SeedTables, "foos")
	err = pgmgr.Dump(c)

	if err != nil {
		t.Log(err)
		t.Fatal("Could not dump database to file")
	}

	file, err = ioutil.ReadFile(dumpFile)
	if err != nil {
		t.Log(err)
		t.Fatal("Could not read dump")
	}

	if strings.Contains(string(file), "123") {
		t.Fatal("dump contains table data for non-seed tables, when --seed-tables was given")
	}

	if !strings.Contains(string(file), "789") {
		t.Fatal("dump does not contain table data for seed tables, when --seed-tables was given")
	}
}

func TestLoad(t *testing.T) {
	resetDB(t)

	ioutil.WriteFile(dumpFile, []byte(`
		CREATE TABLE foos (foo_id INTEGER);
		INSERT INTO foos (foo_id) VALUES (1), (2), (3);
	`), 0644)

	err := pgmgr.Load(globalConfig())

	if err != nil {
		t.Log(err)
		t.Fatal("Could not load database from file")
	}

	psqlMustExec(t, `SELECT * FROM foos;`)
}

func TestVersion(t *testing.T) {
	resetDB(t)

	version, err := pgmgr.Version(globalConfig())

	if err != nil {
		t.Log(err)
		t.Fatal("Could not fetch version info")
	}

	if version != -1 {
		t.Fatal("expected version to be -1 before table exists, got", version)
	}

	pgmgr.Initialize(globalConfig())
	psqlMustExec(t, `INSERT INTO schema_migrations (version) VALUES (1);`)

	version, err = pgmgr.Version(globalConfig())
	if version != 1 {
		t.Fatal("expected version to be 1, got", version)
	}
}

func TestColumnTypeString(t *testing.T) {
	resetDB(t)

	config := globalConfig()
	config.ColumnType = "string"
	pgmgr.Initialize(config)

	psqlMustExec(t, `INSERT INTO schema_migrations (version) VALUES ('20150910120933');`)
	version, err := pgmgr.Version(config)
	if err != nil {
		t.Fatal(err)
	}

	if version != 20150910120933 {
		t.Fatal("expected version to be 20150910120933, got", version)
	}
}

func TestMigrate(t *testing.T) {
	// start with an empty DB
	resetDB(t)
	clearMigrationFolder(t)

	// add our first migration
	writeMigration(t, "002_this_is_a_migration.up.sql", `
		CREATE TABLE foos (foo_id INTEGER);
		INSERT INTO foos (foo_id) VALUES (1), (2), (3);
	`)

	writeMigration(t, "002_this_is_a_migration.down.sql", `DROP TABLE foos;`)

	err := pgmgr.Migrate(globalConfig())

	if err != nil {
		t.Log(err)
		t.Fatal("Migrations failed to run.")
	}

	// test simple idempotency
	err = pgmgr.Migrate(globalConfig())
	if err != nil {
		t.Log(err)
		t.Fatal("Running migrations again was not idempotent!")
	}

	psqlMustExec(t, `SELECT * FROM foos;`)

	// add a new migration with an older version, as if another dev's branch was merged in
	writeMigration(t, "001_this_is_an_older_migration.up.sql", `
		CREATE TABLE bars (bar_id INTEGER);
		INSERT INTO bars (bar_id) VALUES (4), (5), (6);
	`)

	err = pgmgr.Migrate(globalConfig())
	if err != nil {
		t.Log(err)
		t.Fatal("Could not apply second migration!")
	}

	psqlMustExec(t, `SELECT * FROM bars;`)

	// rollback the initial migration, since it has the latest version
	err = pgmgr.Rollback(globalConfig())

	if err := psqlExec(t, `SELECT * FROM foos;`); err == nil {
		t.Fatal("Should not have been able to select from foos table")
	}

	v, err := pgmgr.Version(globalConfig())
	if err != nil || v != 1 {
		t.Log(err)
		t.Fatal("Rollback did not reset version! Still on version ", v)
	}
}

func TestMigrateColumnTypeString(t *testing.T) {
	// start with an empty DB
	resetDB(t)
	clearMigrationFolder(t)

	config := globalConfig()
	config.ColumnType = "string"

	// migrate up
	writeMigration(t, "20150910120933_some_migration.up.sql", `
		CREATE TABLE foos (foo_id INTEGER);
		INSERT INTO foos (foo_id) VALUES (1), (2), (3);
	`)

	err := pgmgr.Migrate(config)
	if err != nil {
		t.Fatal(err)
	}

	v, err := pgmgr.Version(config)
	if err != nil {
		t.Fatal(err)
	}

	if v != 20150910120933 {
		t.Fatal("Expected version 20150910120933 after migration, got", v)
	}

	// migrate down
	writeMigration(t, "20150910120933_some_migration.down.sql", `DROP TABLE foos;`)

	err = pgmgr.Rollback(config)
	if err != nil {
		t.Fatal(err)
	}

	v, err = pgmgr.Version(config)
	if err != nil {
		t.Fatal(err)
	}

	if v != -1 {
		t.Fatal("Expected version -1 after rollback, got", v)
	}
}

func TestMigrateNoTransaction(t *testing.T) {
	// start with an empty DB
	resetDB(t)
	clearMigrationFolder(t)

	// CREATE INDEX CONCURRENTLY can not run inside a transaction, so we can assert
	// that no transaction was used by verifying it ran successfully.
	writeMigration(t, "001_create_foos.up.sql", `CREATE TABLE foos (foo_id INTEGER);`)
	writeMigration(t, "002_index_foos.no_txn.up.sql", `CREATE INDEX CONCURRENTLY idx_foo_id ON foos(foo_id);`)

	err := pgmgr.Migrate(globalConfig())
	if err != nil {
		t.Fatal(err)
	}
}

func TestCreateMigration(t *testing.T) {
	clearMigrationFolder(t)

	assertFileExists := func(filename string) {
		err := testSh(t, "stat", []string{filepath.Join(migrationFolder, filename)})
		if err != nil {
			t.Fatal(err)
		}
	}

	expectedVersion := time.Now().Unix()
	err := pgmgr.CreateMigration(globalConfig(), "new_migration", false)
	if err != nil {
		t.Fatal(err)
	}

	assertFileExists(fmt.Sprint(expectedVersion, "_new_migration.up.sql"))
	assertFileExists(fmt.Sprint(expectedVersion, "_new_migration.down.sql"))

	expectedStringVersion := time.Now().Format(datetimeFormat)
	config := globalConfig()
	config.Format = "datetime"
	err = pgmgr.CreateMigration(config, "rails_style", false)
	if err != nil {
		t.Fatal(err)
	}

	assertFileExists(fmt.Sprint(expectedStringVersion, "_rails_style.up.sql"))
	assertFileExists(fmt.Sprint(expectedStringVersion, "_rails_style.down.sql"))

	err = pgmgr.CreateMigration(config, "create_index", true)
	if err != nil {
		t.Fatal(err)
	}

	assertFileExists(fmt.Sprint(expectedStringVersion, "_create_index.no_txn.up.sql"))
	assertFileExists(fmt.Sprint(expectedStringVersion, "_create_index.no_txn.down.sql"))
}

// redundant, but I'm also lazy
func testSh(t *testing.T, command string, args []string) error {
	c := exec.Command(command, args...)
	output, err := c.CombinedOutput()
	t.Log(string(output))
	if err != nil {
		return err
	}

	return nil
}

func psqlExec(t *testing.T, statement string) error {
	return testSh(t, "psql", []string{"-d", testDBName, "-c", statement})
}

func psqlMustExec(t *testing.T, statement string) {
	err := psqlExec(t, statement)
	if err != nil {
		t.Fatalf("Failed to execute statement: '%s': %s", statement, err)
	}
}

func resetDB(t *testing.T) {
	if err := dropDB(t); err != nil {
		t.Fatal("dropdb failed: ", err)
	}

	if err := createDB(t); err != nil {
		t.Fatal("createdb failed: ", err)
	}
}

func dropDB(t *testing.T) error {
	return testSh(t, "dropdb", []string{testDBName})
}

func createDB(t *testing.T) error {
	return testSh(t, "createdb", []string{testDBName})
}

func clearMigrationFolder(t *testing.T) {
	if err := testSh(t, "rm", []string{"-r", migrationFolder}); err != nil {
		t.Fatalf("Could not remove directory %s: %s", migrationFolder, err)
	}

	if err := testSh(t, "mkdir", []string{migrationFolder}); err != nil {
		t.Fatalf("Could not create directory %s: %s", migrationFolder, err)
	}
}

func writeMigration(t *testing.T, name, contents string) {
	filename := path.Join(migrationFolder, name)
	err := ioutil.WriteFile(filename, []byte(contents), 0644)
	if err != nil {
		t.Fatalf("Failed to write %s: %s", filename, err)
	}
}
<|endoftext|>


<commit_before>package godata
<commit_msg>added dummy go test case<commit_after>package godata

import (
	"fmt"
	"testing"
)

const TEST_SQLITE3_CREATE_STATEMENT_A = "CREATE TABLE FOO ( X INTEGER, Y TEXT PRIMARY KEY)"

func TestNewMetaTableFromCreateStatement(t *testing.T) {
	meta, err := NewMetaTableFromCreateStatement(TEST_SQLITE3_CREATE_STATEMENT_A)
	if err != nil {
		t.Error(err)
	}

	fmt.Println(meta.columns)
}
<|endoftext|>


<commit_before>package common

import (
	"io"
	"time"
)

// Context defines the context object passed around
type Context struct {
	Config               Config
	StackManager         StackManager
	ClusterManager       ClusterManager
	InstanceManager      InstanceManager
	ElbManager           ElbManager
	RdsManager           RdsManager
	ParamManager         ParamManager
	LocalPipelineManager PipelineManager // instance that ignores region/profile/role
	PipelineManager      PipelineManager
	LogsManager          LogsManager
	DockerManager        DockerManager
	DockerOut            io.Writer
	TaskManager          TaskManager
	ArtifactManager      ArtifactManager
	RolesetManager       RolesetManager
	ExtensionsManager    ExtensionsManager
}

// Config defines the structure of the yml file for the mu config
type Config struct {
	Namespace    string        `yaml:"namespace,omitempty"`
	Environments []Environment `yaml:"environments,omitempty"`
	Service      Service       `yaml:"service,omitempty"`
	Basedir      string        `yaml:"-"`
	RelMuFile    string        `yaml:"-"`
	Repo         struct {
		Name     string
		Slug     string
		Revision string
		Branch   string
		Provider string
	} `yaml:"-"`
	Templates  map[string]interface{}       `yaml:"templates,omitempty"`
	Parameters map[string]map[string]string `yaml:"parameters,omitempty"`
	Tags       map[string]map[string]string `yaml:"tags,omitempty"`
	Extensions []Extension                  `yaml:"extensions,omitempty"`
	DisableIAM bool                         `yaml:"disableIAM,omitempty"`
	Roles      struct {
		CloudFormation string `yaml:"cloudFormation,omitempty"`
	} `yaml:"roles,omitempty"`
}

// Extension defines the structure of the yml file for an extension
type Extension struct {
	URL   string `yaml:"url,omitempty"`
	Image string `yaml:"image,omitempty"`
}

// Environment defines the structure of the yml file for an environment
type Environment struct {
	Name         string      `yaml:"name,omitempty"`
	Provider     EnvProvider `yaml:"provider,omitempty"`
	Loadbalancer struct {
		HostedZone  string `yaml:"hostedzone,omitempty"`
		Name        string `yaml:"name,omitempty"`
		Certificate string `yaml:"certificate,omitempty"`
		Internal    bool   `yaml:"internal,omitempty"`
	} `yaml:"loadbalancer,omitempty"`
	Cluster struct {
		InstanceType            string `yaml:"instanceType,omitempty"`
		ImageID                 string `yaml:"imageId,omitempty"`
		ImageOsType             string `yaml:"osType,omitempty"`
		InstanceTenancy         string `yaml:"instanceTenancy,omitempty"`
		DesiredCapacity         int    `yaml:"desiredCapacity,omitempty"`
		MinSize                 int    `yaml:"minSize,omitempty"`
		MaxSize                 int    `yaml:"maxSize,omitempty"`
		KeyName                 string `yaml:"keyName,omitempty"`
		SSHAllow                string `yaml:"sshAllow,omitempty"`
		TargetCPUReservation    int    `yaml:"targetCPUReservation,omitempty"`
		TargetMemoryReservation int    `yaml:"targetMemoryReservation,omitempty"`
		HTTPProxy               string `yaml:"httpProxy,omitempty"`
	} `yaml:"cluster,omitempty"`
	Discovery struct {
		Provider      string            `yaml:"provider,omitempty"`
		Configuration map[string]string `yaml:"configuration,omitempty"`
	} `yaml:"discovery,omitempty"`
	VpcTarget struct {
		VpcID             string   `yaml:"vpcId,omitempty"`
		InstanceSubnetIds []string `yaml:"instanceSubnetIds,omitempty"`
		ElbSubnetIds      []string `yaml:"elbSubnetIds,omitempty"`
	} `yaml:"vpcTarget,omitempty"`
	Roles struct {
		EcsInstance      string `yaml:"ecsInstance,omitempty"`
		ConsulClientTask string `yaml:"consulClientTask,omitempty"`
		ConsulInstance   string `yaml:"consulInstance,omitempty"`
		ConsulServerTask string `yaml:"consulServerTask,omitempty"`
	} `yaml:"roles,omitempty"`
}

// Service defines the structure of the yml file for a service
type Service struct {
	Name                 string                 `yaml:"name,omitempty"`
	DesiredCount         int                    `yaml:"desiredCount,omitempty"`
	MinSize              int                    `yaml:"minSize,omitempty"`
	MaxSize              int                    `yaml:"maxSize,omitempty"`
	Dockerfile           string                 `yaml:"dockerfile,omitempty"`
	ImageRepository      string                 `yaml:"imageRepository,omitempty"`
	Port                 int                    `yaml:"port,omitempty"`
	Protocol             string                 `yaml:"protocol,omitempty"`
	HealthEndpoint       string                 `yaml:"healthEndpoint,omitempty"`
	CPU                  int                    `yaml:"cpu,omitempty"`
	Memory               int                    `yaml:"memory,omitempty"`
	NetworkMode          string                 `yaml:"networkMode,omitempty"`
	Environment          map[string]interface{} `yaml:"environment,omitempty"`
	PathPatterns         []string               `yaml:"pathPatterns,omitempty"`
	HostPatterns         []string               `yaml:"hostPatterns,omitempty"`
	Priority             int                    `yaml:"priority,omitempty"`
	Pipeline             Pipeline               `yaml:"pipeline,omitempty"`
	Database             Database               `yaml:"database,omitempty"`
	Schedule             []Schedule             `yaml:"schedules,omitempty"`
	TargetCPUUtilization int                    `yaml:"targetCPUUtilization,omitempty"`
	Roles                struct {
		Ec2Instance            string `yaml:"ec2Instance,omitempty"`
		CodeDeploy             string `yaml:"codeDeploy,omitempty"`
		EcsEvents              string `yaml:"ecsEvents,omitempty"`
		EcsService             string `yaml:"ecsService,omitempty"`
		EcsTask                string `yaml:"ecsTask,omitempty"`
		ApplicationAutoScaling string `yaml:"applicationAutoScaling,omitempty"`
	} `yaml:"roles,omitempty"`
}

// Database definition
type Database struct {
	Name              string `yaml:"name,omitempty"`
	InstanceClass     string `yaml:"instanceClass,omitempty"`
	Engine            string `yaml:"engine,omitempty"`
	IamAuthentication bool   `yaml:"iamAuthentication,omitempty"`
	MasterUsername    string `yaml:"masterUsername,omitempty"`
	AllocatedStorage  string `yaml:"allocatedStorage,omitempty"`
}

// Schedule definition
type Schedule struct {
	Name       string   `yaml:"name,omitempty"`
	Expression string   `yaml:"expression,omitempty"`
	Command    []string `yaml:"command,omitempty"`
}

// Pipeline definition
type Pipeline struct {
	Source struct {
		Provider string `yaml:"provider,omitempty"`
		Repo     string `yaml:"repo,omitempty"`
		Branch   string `yaml:"branch,omitempty"`
	} `yaml:"source,omitempty"`
	Build struct {
		Disabled    bool   `yaml:"disabled,omitempty"`
		Type        string `yaml:"type,omitempty"`
		ComputeType string `yaml:"computeType,omitempty"`
		Image       string `yaml:"image,omitempty"`
		Bucket      string `yaml:"bucket,omitempty"`
	} `yaml:"build,omitempty"`
	Acceptance struct {
		Disabled    bool   `yaml:"disabled,omitempty"`
		Environment string `yaml:"environment,omitempty"`
		Type        string `yaml:"type,omitempty"`
		ComputeType string `yaml:"computeType,omitempty"`
		Image       string `yaml:"image,omitempty"`
		Roles       struct {
			CodeBuild string `yaml:"codeBuild,omitempty"`
			Mu        string `yaml:"mu,omitempty"`
		} `yaml:"roles,omitempty"`
	} `yaml:"acceptance,omitempty"`
	Production struct {
		Disabled    bool   `yaml:"disabled,omitempty"`
		Environment string `yaml:"environment,omitempty"`
		Roles       struct {
			CodeBuild string `yaml:"codeBuild,omitempty"`
			Mu        string `yaml:"mu,omitempty"`
		} `yaml:"roles,omitempty"`
	} `yaml:"production,omitempty"`
	MuBaseurl string `yaml:"muBaseurl,omitempty"`
	MuVersion string `yaml:"muVersion,omitempty"`
	Roles     struct {
		Pipeline string `yaml:"pipeline,omitempty"`
		Build    string `yaml:"build,omitempty"`
	} `yaml:"roles,omitempty"`
	Bucket string `yaml:"bucket,omitempty"`
}

// Stack summary
type Stack struct {
	ID             string
	Name           string
	Status         string
	StatusReason   string
	LastUpdateTime time.Time
	Tags           map[string]string
	Outputs        map[string]string
	Parameters     map[string]string
}

const (
	// StackStatusCreateInProgress is a StackStatus enum value
	StackStatusCreateInProgress = "CREATE_IN_PROGRESS"

	// StackStatusCreateFailed is a StackStatus enum value
	StackStatusCreateFailed = "CREATE_FAILED"

	// StackStatusCreateComplete is a StackStatus enum value
	StackStatusCreateComplete = "CREATE_COMPLETE"

	// StackStatusRollbackInProgress is a StackStatus enum value
	StackStatusRollbackInProgress = "ROLLBACK_IN_PROGRESS"

	// StackStatusRollbackFailed is a StackStatus enum value
	StackStatusRollbackFailed = "ROLLBACK_FAILED"

	// StackStatusRollbackComplete is a StackStatus enum value
	StackStatusRollbackComplete = "ROLLBACK_COMPLETE"

	// StackStatusDeleteInProgress is a StackStatus enum value
	StackStatusDeleteInProgress = "DELETE_IN_PROGRESS"

	// StackStatusDeleteFailed is a StackStatus enum value
	StackStatusDeleteFailed = "DELETE_FAILED"

	// StackStatusDeleteComplete is a StackStatus enum value
	StackStatusDeleteComplete = "DELETE_COMPLETE"

	// StackStatusUpdateInProgress is a StackStatus enum value
	StackStatusUpdateInProgress = "UPDATE_IN_PROGRESS"

	// StackStatusUpdateCompleteCleanupInProgress is a StackStatus enum value
	StackStatusUpdateCompleteCleanupInProgress = "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS"

	// StackStatusUpdateComplete is a StackStatus enum value
	StackStatusUpdateComplete = "UPDATE_COMPLETE"

	// StackStatusUpdateRollbackInProgress is a StackStatus enum value
	StackStatusUpdateRollbackInProgress = "UPDATE_ROLLBACK_IN_PROGRESS"

	// StackStatusUpdateRollbackFailed is a StackStatus enum value
	StackStatusUpdateRollbackFailed = "UPDATE_ROLLBACK_FAILED"

	// StackStatusUpdateRollbackCompleteCleanupInProgress is a StackStatus enum value
	StackStatusUpdateRollbackCompleteCleanupInProgress = "UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS"

	// StackStatusUpdateRollbackComplete is a StackStatus enum value
	StackStatusUpdateRollbackComplete = "UPDATE_ROLLBACK_COMPLETE"

	// StackStatusReviewInProgress is a StackStatus enum value
	StackStatusReviewInProgress = "REVIEW_IN_PROGRESS"
)

// StackType describes supported stack types
type StackType string

// List of valid stack types
const (
	StackTypeVpc          StackType = "vpc"
	StackTypeTarget                 = "target"
	StackTypeIam                    = "iam"
	StackTypeEnv                    = "environment"
	StackTypeLoadBalancer           = "loadbalancer"
	StackTypeConsul                 = "consul"
	StackTypeRepo                   = "repo"
	StackTypeApp                    = "app"
	StackTypeService                = "service"
	StackTypePipeline               = "pipeline"
	StackTypeDatabase               = "database"
	StackTypeSchedule               = "schedule"
	StackTypeBucket                 = "bucket"
)

// EnvProvider describes supported environment strategies
type EnvProvider string

// List of valid environment strategies
const (
	EnvProviderEcs        EnvProvider = "ecs"
	EnvProviderEcsFargate             = "ecs-fargate"
	EnvProviderEc2                    = "ec2"
)

// ArtifactProvider describes supported artifact strategies
type ArtifactProvider string

// List of valid artifact providers
const (
	ArtifactProviderEcr ArtifactProvider = "ecr"
	ArtifactProviderS3                   = "s3"
)

// Container describes container details
type Container struct {
	Name     string
	Instance string
}

// Task describes task definition
type Task struct {
	Name           string
	Environment    string
	Service        string
	TaskDefinition string
	Cluster        string
	Command        []string
	Containers     []Container
}

// JSONOutput common json definition
type JSONOutput struct {
	Values [1]struct {
		Key   string `json:"key"`
		Value string `json:"value"`
	} `json:"values"`
}

// Int64Value returns the value of the int64 pointer passed in or
// 0 if the pointer is nil.
func Int64Value(v *int64) int64 {
	if v != nil {
		return *v
	}
	return 0
}

// StringValue returns the value of the string pointer passed in or
// "" if the pointer is nil.
func StringValue(v *string) string {
	if v != nil {
		return *v
	}
	return ""
}

// BoolValue returns the value of the bool pointer passed in or
// false if the pointer is nil.
func BoolValue(v *bool) bool {
	if v != nil {
		return *v
	}
	return false
}

// TimeValue returns the value of the time.Time pointer passed in or
// time.Time{} if the pointer is nil.
func TimeValue(v *time.Time) time.Time {
	if v != nil {
		return *v
	}
	return time.Time{}
}

// CPUMemory represents valid cpu/memory structure
type CPUMemory struct {
	CPU    int
	Memory []int
}

// CPUMemorySupport represents valid ECS combinations
var CPUMemorySupport = []CPUMemory{
	{CPU: 256, Memory: []int{512, 1, 2}},
	{CPU: 512, Memory: []int{1, 2, 3, 4}},
	{CPU: 1024, Memory: []int{2, 3, 4, 5, 6, 7, 8}},
	{CPU: 2048, Memory: []int{4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}},
	{CPU: 4096, Memory: []int{8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30}},
}
<commit_msg>update MB to be measured in GB<commit_after>package common

import (
	"io"
	"time"
)

// Context defines the context object passed around
type Context struct {
	Config               Config
	StackManager         StackManager
	ClusterManager       ClusterManager
	InstanceManager      InstanceManager
	ElbManager           ElbManager
	RdsManager           RdsManager
	ParamManager         ParamManager
	LocalPipelineManager PipelineManager // instance that ignores region/profile/role
	PipelineManager      PipelineManager
	LogsManager          LogsManager
	DockerManager        DockerManager
	DockerOut            io.Writer
	TaskManager          TaskManager
	ArtifactManager      ArtifactManager
	RolesetManager       RolesetManager
	ExtensionsManager    ExtensionsManager
}

// Config defines the structure of the yml file for the mu config
type Config struct {
	Namespace    string        `yaml:"namespace,omitempty"`
	Environments []Environment `yaml:"environments,omitempty"`
	Service      Service       `yaml:"service,omitempty"`
	Basedir      string        `yaml:"-"`
	RelMuFile    string        `yaml:"-"`
	Repo         struct {
		Name     string
		Slug     string
		Revision string
		Branch   string
		Provider string
	} `yaml:"-"`
	Templates  map[string]interface{}       `yaml:"templates,omitempty"`
	Parameters map[string]map[string]string `yaml:"parameters,omitempty"`
	Tags       map[string]map[string]string `yaml:"tags,omitempty"`
	Extensions []Extension                  `yaml:"extensions,omitempty"`
	DisableIAM bool                         `yaml:"disableIAM,omitempty"`
	Roles      struct {
		CloudFormation string `yaml:"cloudFormation,omitempty"`
	} `yaml:"roles,omitempty"`
}

// Extension defines the structure of the yml file for an extension
type Extension struct {
	URL   string `yaml:"url,omitempty"`
	Image string `yaml:"image,omitempty"`
}

// Environment defines the structure of the yml file for an environment
type Environment struct {
	Name         string      `yaml:"name,omitempty"`
	Provider     EnvProvider `yaml:"provider,omitempty"`
	Loadbalancer struct {
		HostedZone  string `yaml:"hostedzone,omitempty"`
		Name        string `yaml:"name,omitempty"`
		Certificate string `yaml:"certificate,omitempty"`
		Internal    bool   `yaml:"internal,omitempty"`
	} `yaml:"loadbalancer,omitempty"`
	Cluster struct {
		InstanceType            string `yaml:"instanceType,omitempty"`
		ImageID                 string `yaml:"imageId,omitempty"`
		ImageOsType             string `yaml:"osType,omitempty"`
		InstanceTenancy         string `yaml:"instanceTenancy,omitempty"`
		DesiredCapacity         int    `yaml:"desiredCapacity,omitempty"`
		MinSize                 int    `yaml:"minSize,omitempty"`
		MaxSize                 int    `yaml:"maxSize,omitempty"`
		KeyName                 string `yaml:"keyName,omitempty"`
		SSHAllow                string `yaml:"sshAllow,omitempty"`
		TargetCPUReservation    int    `yaml:"targetCPUReservation,omitempty"`
		TargetMemoryReservation int    `yaml:"targetMemoryReservation,omitempty"`
		HTTPProxy               string `yaml:"httpProxy,omitempty"`
	} `yaml:"cluster,omitempty"`
	Discovery struct {
		Provider      string            `yaml:"provider,omitempty"`
		Configuration map[string]string `yaml:"configuration,omitempty"`
	} `yaml:"discovery,omitempty"`
	VpcTarget struct {
		VpcID             string   `yaml:"vpcId,omitempty"`
		InstanceSubnetIds []string `yaml:"instanceSubnetIds,omitempty"`
		ElbSubnetIds      []string `yaml:"elbSubnetIds,omitempty"`
	} `yaml:"vpcTarget,omitempty"`
	Roles struct {
		EcsInstance      string `yaml:"ecsInstance,omitempty"`
		ConsulClientTask string `yaml:"consulClientTask,omitempty"`
		ConsulInstance   string `yaml:"consulInstance,omitempty"`
		ConsulServerTask string `yaml:"consulServerTask,omitempty"`
	} `yaml:"roles,omitempty"`
}

// Service defines the structure of the yml file for a service
type Service struct {
	Name                 string                 `yaml:"name,omitempty"`
	DesiredCount         int                    `yaml:"desiredCount,omitempty"`
	MinSize              int                    `yaml:"minSize,omitempty"`
	MaxSize              int                    `yaml:"maxSize,omitempty"`
	Dockerfile           string                 `yaml:"dockerfile,omitempty"`
	ImageRepository      string                 `yaml:"imageRepository,omitempty"`
	Port                 int                    `yaml:"port,omitempty"`
	Protocol             string                 `yaml:"protocol,omitempty"`
	HealthEndpoint       string                 `yaml:"healthEndpoint,omitempty"`
	CPU                  int                    `yaml:"cpu,omitempty"`
	Memory               int                    `yaml:"memory,omitempty"`
	NetworkMode          string                 `yaml:"networkMode,omitempty"`
	Environment          map[string]interface{} `yaml:"environment,omitempty"`
	PathPatterns         []string               `yaml:"pathPatterns,omitempty"`
	HostPatterns         []string               `yaml:"hostPatterns,omitempty"`
	Priority             int                    `yaml:"priority,omitempty"`
	Pipeline             Pipeline               `yaml:"pipeline,omitempty"`
	Database             Database               `yaml:"database,omitempty"`
	Schedule             []Schedule             `yaml:"schedules,omitempty"`
	TargetCPUUtilization int                    `yaml:"targetCPUUtilization,omitempty"`
	Roles                struct {
		Ec2Instance            string `yaml:"ec2Instance,omitempty"`
		CodeDeploy             string `yaml:"codeDeploy,omitempty"`
		EcsEvents              string `yaml:"ecsEvents,omitempty"`
		EcsService             string `yaml:"ecsService,omitempty"`
		EcsTask                string `yaml:"ecsTask,omitempty"`
		ApplicationAutoScaling string `yaml:"applicationAutoScaling,omitempty"`
	} `yaml:"roles,omitempty"`
}

// Database definition
type Database struct {
	Name              string `yaml:"name,omitempty"`
	InstanceClass     string `yaml:"instanceClass,omitempty"`
	Engine            string `yaml:"engine,omitempty"`
	IamAuthentication bool   `yaml:"iamAuthentication,omitempty"`
	MasterUsername    string `yaml:"masterUsername,omitempty"`
	AllocatedStorage  string `yaml:"allocatedStorage,omitempty"`
}

// Schedule definition
type Schedule struct {
	Name       string   `yaml:"name,omitempty"`
	Expression string   `yaml:"expression,omitempty"`
	Command    []string `yaml:"command,omitempty"`
}

// Pipeline definition
type Pipeline struct {
	Source struct {
		Provider string `yaml:"provider,omitempty"`
		Repo     string `yaml:"repo,omitempty"`
		Branch   string `yaml:"branch,omitempty"`
	} `yaml:"source,omitempty"`
	Build struct {
		Disabled    bool   `yaml:"disabled,omitempty"`
		Type        string `yaml:"type,omitempty"`
		ComputeType string `yaml:"computeType,omitempty"`
		Image       string `yaml:"image,omitempty"`
		Bucket      string `yaml:"bucket,omitempty"`
	} `yaml:"build,omitempty"`
	Acceptance struct {
		Disabled    bool   `yaml:"disabled,omitempty"`
		Environment string `yaml:"environment,omitempty"`
		Type        string `yaml:"type,omitempty"`
		ComputeType string `yaml:"computeType,omitempty"`
		Image       string `yaml:"image,omitempty"`
		Roles       struct {
			CodeBuild string `yaml:"codeBuild,omitempty"`
			Mu        string `yaml:"mu,omitempty"`
		} `yaml:"roles,omitempty"`
	} `yaml:"acceptance,omitempty"`
	Production struct {
		Disabled    bool   `yaml:"disabled,omitempty"`
		Environment string `yaml:"environment,omitempty"`
		Roles       struct {
			CodeBuild string `yaml:"codeBuild,omitempty"`
			Mu        string `yaml:"mu,omitempty"`
		} `yaml:"roles,omitempty"`
	} `yaml:"production,omitempty"`
	MuBaseurl string `yaml:"muBaseurl,omitempty"`
	MuVersion string `yaml:"muVersion,omitempty"`
	Roles     struct {
		Pipeline string `yaml:"pipeline,omitempty"`
		Build    string `yaml:"build,omitempty"`
	} `yaml:"roles,omitempty"`
	Bucket string `yaml:"bucket,omitempty"`
}

// Stack summary
type Stack struct {
	ID             string
	Name           string
	Status         string
	StatusReason   string
	LastUpdateTime time.Time
	Tags           map[string]string
	Outputs        map[string]string
	Parameters     map[string]string
}

const (
	// StackStatusCreateInProgress is a StackStatus enum value
	StackStatusCreateInProgress = "CREATE_IN_PROGRESS"

	// StackStatusCreateFailed is a StackStatus enum value
	StackStatusCreateFailed = "CREATE_FAILED"

	// StackStatusCreateComplete is a StackStatus enum value
	StackStatusCreateComplete = "CREATE_COMPLETE"

	// StackStatusRollbackInProgress is a StackStatus enum value
	StackStatusRollbackInProgress = "ROLLBACK_IN_PROGRESS"

	// StackStatusRollbackFailed is a StackStatus enum value
	StackStatusRollbackFailed = "ROLLBACK_FAILED"

	// StackStatusRollbackComplete is a StackStatus enum value
	StackStatusRollbackComplete = "ROLLBACK_COMPLETE"

	// StackStatusDeleteInProgress is a StackStatus enum value
	StackStatusDeleteInProgress = "DELETE_IN_PROGRESS"

	// StackStatusDeleteFailed is a StackStatus enum value
	StackStatusDeleteFailed = "DELETE_FAILED"

	// StackStatusDeleteComplete is a StackStatus enum value
	StackStatusDeleteComplete = "DELETE_COMPLETE"

	// StackStatusUpdateInProgress is a StackStatus enum value
	StackStatusUpdateInProgress = "UPDATE_IN_PROGRESS"

	// StackStatusUpdateCompleteCleanupInProgress is a StackStatus enum value
	StackStatusUpdateCompleteCleanupInProgress = "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS"

	// StackStatusUpdateComplete is a StackStatus enum value
	StackStatusUpdateComplete = "UPDATE_COMPLETE"

	// StackStatusUpdateRollbackInProgress is a StackStatus enum value
	StackStatusUpdateRollbackInProgress = "UPDATE_ROLLBACK_IN_PROGRESS"

	// StackStatusUpdateRollbackFailed is a StackStatus enum value
	StackStatusUpdateRollbackFailed = "UPDATE_ROLLBACK_FAILED"

	// StackStatusUpdateRollbackCompleteCleanupInProgress is a StackStatus enum value
	StackStatusUpdateRollbackCompleteCleanupInProgress = "UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS"

	// StackStatusUpdateRollbackComplete is a StackStatus enum value
	StackStatusUpdateRollbackComplete = "UPDATE_ROLLBACK_COMPLETE"

	// StackStatusReviewInProgress is a StackStatus enum value
	StackStatusReviewInProgress = "REVIEW_IN_PROGRESS"
)

// StackType describes supported stack types
type StackType string

// List of valid stack types
const (
	StackTypeVpc          StackType = "vpc"
	StackTypeTarget                 = "target"
	StackTypeIam                    = "iam"
	StackTypeEnv                    = "environment"
	StackTypeLoadBalancer           = "loadbalancer"
	StackTypeConsul                 = "consul"
	StackTypeRepo                   = "repo"
	StackTypeApp                    = "app"
	StackTypeService                = "service"
	StackTypePipeline               = "pipeline"
	StackTypeDatabase               = "database"
	StackTypeSchedule               = "schedule"
	StackTypeBucket                 = "bucket"
)

// EnvProvider describes supported environment strategies
type EnvProvider string

// List of valid environment strategies
const (
	EnvProviderEcs        EnvProvider = "ecs"
	EnvProviderEcsFargate             = "ecs-fargate"
	EnvProviderEc2                    = "ec2"
)

// ArtifactProvider describes supported artifact strategies
type ArtifactProvider string

// List of valid artifact providers
const (
	ArtifactProviderEcr ArtifactProvider = "ecr"
	ArtifactProviderS3                   = "s3"
)

// Container describes container details
type Container struct {
	Name     string
	Instance string
}

// Task describes task definition
type Task struct {
	Name           string
	Environment    string
	Service        string
	TaskDefinition string
	Cluster        string
	Command        []string
	Containers     []Container
}

// JSONOutput common json definition
type JSONOutput struct {
	Values [1]struct {
		Key   string `json:"key"`
		Value string `json:"value"`
	} `json:"values"`
}

// Int64Value returns the value of the int64 pointer passed in or
// 0 if the pointer is nil.
func Int64Value(v *int64) int64 {
	if v != nil {
		return *v
	}
	return 0
}

// StringValue returns the value of the string pointer passed in or
// "" if the pointer is nil.
func StringValue(v *string) string {
	if v != nil {
		return *v
	}
	return ""
}

// BoolValue returns the value of the bool pointer passed in or
// false if the pointer is nil.
func BoolValue(v *bool) bool {
	if v != nil {
		return *v
	}
	return false
}

// TimeValue returns the value of the time.Time pointer passed in or
// time.Time{} if the pointer is nil.
func TimeValue(v *time.Time) time.Time {
	if v != nil {
		return *v
	}
	return time.Time{}
}

// CPUMemory represents valid cpu/memory structure
type CPUMemory struct {
	CPU    int
	Memory []int
}

// GB count of MB
var GB = 1024

// CPUMemorySupport represents valid ECS combinations
var CPUMemorySupport = []CPUMemory{
	{CPU: 256, Memory: []int{512, 1 * GB, 2 * GB}},
	{CPU: 512, Memory: []int{1 * GB, 2 * GB, 3 * GB, 4 * GB}},
	{CPU: 1024, Memory: []int{2 * GB, 3 * GB, 4 * GB, 5 * GB, 6 * GB, 7 * GB, 8 * GB}},
	{CPU: 2048, Memory: []int{4 * GB, 5 * GB, 6 * GB, 7 * GB, 8 * GB, 9 * GB, 10 * GB, 11 * GB, 12 * GB, 13 * GB, 14 * GB, 15 * GB, 16 * GB}},
	{CPU: 4096, Memory: []int{8 * GB, 9 * GB, 10 * GB, 11 * GB, 12 * GB, 13 * GB, 14 * GB, 15 * GB, 16 * GB, 17 * GB, 18 * GB, 19 * GB, 20 * GB,
		21 * GB, 22 * GB, 23 * GB, 24 * GB, 25 * GB, 26 * GB, 27 * GB, 28 * GB, 29 * GB, 30 * GB}},
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// TODO(cAdvisor): Package comment.
package raw

import (
	"fmt"
	"io/ioutil"
	"path"
	"strconv"
	"strings"

	"code.google.com/p/go.exp/inotify"
	"github.com/docker/libcontainer/cgroups"
	"github.com/docker/libcontainer/cgroups/fs"
	"github.com/golang/glog"
	"github.com/google/cadvisor/container"
	"github.com/google/cadvisor/container/libcontainer"
	"github.com/google/cadvisor/info"
	"github.com/google/cadvisor/utils"
)

type rawContainerHandler struct {
	name               string
	cgroup             *cgroups.Cgroup
	cgroupSubsystems   *cgroupSubsystems
	machineInfoFactory info.MachineInfoFactory
	watcher            *inotify.Watcher
	stopWatcher        chan error
	watches            map[string]struct{}
}

func newRawContainerHandler(name string, cgroupSubsystems *cgroupSubsystems, machineInfoFactory info.MachineInfoFactory) (container.ContainerHandler, error) {
	return &rawContainerHandler{
		name: name,
		cgroup: &cgroups.Cgroup{
			Parent: "/",
			Name:   name,
		},
		cgroupSubsystems:   cgroupSubsystems,
		machineInfoFactory: machineInfoFactory,
		stopWatcher:        make(chan error),
		watches:            make(map[string]struct{}),
	}, nil
}

func (self *rawContainerHandler) ContainerReference() (info.ContainerReference, error) {
	// We only know the container by its one name.
	return info.ContainerReference{
		Name: self.name,
	}, nil
}

func readString(dirpath string, file string) string {
	cgroupFile := path.Join(dirpath, file)

	// Ignore non-existent files
	if !utils.FileExists(cgroupFile) {
		return ""
	}

	// Read
	out, err := ioutil.ReadFile(cgroupFile)
	if err != nil {
		glog.Errorf("raw driver: Failed to read %q: %s", cgroupFile, err)
		return ""
	}
	return string(out)
}

func readInt64(dirpath string, file string) uint64 {
	out := readString(dirpath, file)
	if out == "" {
		return 0
	}

	val, err := strconv.ParseUint(strings.TrimSpace(out), 10, 64)
	if err != nil {
		glog.Errorf("raw driver: Failed to parse int %q from file %q: %s", out, path.Join(dirpath, file), err)
		return 0
	}

	return val
}

func (self *rawContainerHandler) GetSpec() (info.ContainerSpec, error) {
	var spec info.ContainerSpec

	// The raw driver assumes unified hierarchy containers.

	// Get machine info.
	mi, err := self.machineInfoFactory.GetMachineInfo()
	if err != nil {
		return spec, err
	}

	// CPU.
	cpuRoot, ok := self.cgroupSubsystems.mountPoints["cpu"]
	if ok {
		cpuRoot = path.Join(cpuRoot, self.name)
		if utils.FileExists(cpuRoot) {
			spec.HasCpu = true
			spec.Cpu.Limit = readInt64(cpuRoot, "cpu.shares")
		}
	}

	// Cpu Mask.
	// This will fail for non-unified hierarchies. We'll return the whole machine mask in that case.
	cpusetRoot, ok := self.cgroupSubsystems.mountPoints["cpuset"]
	if ok {
		cpusetRoot = path.Join(cpusetRoot, self.name)
		if utils.FileExists(cpusetRoot) {
			spec.HasCpu = true
			spec.Cpu.Mask = readString(cpusetRoot, "cpuset.cpus")
			if spec.Cpu.Mask == "" {
				spec.Cpu.Mask = fmt.Sprintf("0-%d", mi.NumCores-1)
			}
		}
	}

	// Memory.
	memoryRoot, ok := self.cgroupSubsystems.mountPoints["memory"]
	if ok {
		memoryRoot = path.Join(memoryRoot, self.name)
		if utils.FileExists(memoryRoot) {
			spec.HasMemory = true
			spec.Memory.Limit = readInt64(memoryRoot, "memory.limit_in_bytes")
			spec.Memory.SwapLimit = readInt64(memoryRoot, "memory.memsw.limit_in_bytes")
		}
	}

	return spec, nil
}

func (self *rawContainerHandler) GetStats() (stats *info.ContainerStats, err error) {
	return libcontainer.GetStatsCgroupOnly(self.cgroup)
}

// Lists all directories under "path" and outputs the results as children of "parent".
func listDirectories(dirpath string, parent string, recursive bool, output map[string]struct{}) error {
	// Ignore if this hierarchy does not exist.
	if !utils.FileExists(dirpath) {
		return nil
	}

	entries, err := ioutil.ReadDir(dirpath)
	if err != nil {
		return err
	}
	for _, entry := range entries {
		// We only grab directories.
		if entry.IsDir() {
			name := path.Join(parent, entry.Name())
			output[name] = struct{}{}

			// List subcontainers if asked to.
			if recursive {
				err := listDirectories(path.Join(dirpath, entry.Name()), name, true, output)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func (self *rawContainerHandler) ListContainers(listType container.ListType) ([]info.ContainerReference, error) {
	containers := make(map[string]struct{})
	for _, subsystem := range self.cgroupSubsystems.mounts {
		err := listDirectories(path.Join(subsystem.Mountpoint, self.name), self.name, listType == container.ListRecursive, containers)
		if err != nil {
			return nil, err
		}
	}

	// Make into container references.
	ret := make([]info.ContainerReference, 0, len(containers))
	for cont := range containers {
		ret = append(ret, info.ContainerReference{
			Name: cont,
		})
	}

	return ret, nil
}

func (self *rawContainerHandler) ListThreads(listType container.ListType) ([]int, error) {
	// TODO(vmarmol): Implement
	return nil, nil
}

func (self *rawContainerHandler) ListProcesses(listType container.ListType) ([]int, error) {
	return fs.GetPids(self.cgroup)
}

func (self *rawContainerHandler) watchDirectory(dir string, containerName string) error {
	err := self.watcher.AddWatch(dir, inotify.IN_CREATE|inotify.IN_DELETE|inotify.IN_MOVE)
	if err != nil {
		return err
	}
	self.watches[containerName] = struct{}{}

	// Watch subdirectories as well.
	entries, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}
	for _, entry := range entries {
		if entry.IsDir() {
			err = self.watchDirectory(path.Join(dir, entry.Name()), path.Join(containerName, entry.Name()))
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func (self *rawContainerHandler) processEvent(event *inotify.Event, events chan container.SubcontainerEvent) error {
	// Convert the inotify event type to a container create or delete.
	var eventType container.SubcontainerEventType
	switch {
	case (event.Mask & inotify.IN_CREATE) > 0:
		eventType = container.SubcontainerAdd
	case (event.Mask & inotify.IN_DELETE) > 0:
		eventType = container.SubcontainerDelete
	case (event.Mask & inotify.IN_MOVED_FROM) > 0:
		eventType = container.SubcontainerDelete
	case (event.Mask & inotify.IN_MOVED_TO) > 0:
		eventType = container.SubcontainerAdd
	default:
		// Ignore other events.
		return nil
	}

	// Derive the container name from the path name.
	var containerName string
	for _, mount := range self.cgroupSubsystems.mounts {
		mountLocation := path.Clean(mount.Mountpoint) + "/"
		if strings.HasPrefix(event.Name, mountLocation) {
			containerName = event.Name[len(mountLocation)-1:]
			break
		}
	}
	if containerName == "" {
		return fmt.Errorf("unable to detect container from watch event on directory %q", event.Name)
	}

	// Maintain the watch for the new or deleted container.
	switch {
	case eventType == container.SubcontainerAdd:
		// If we've already seen this event, return.
		if _, ok := self.watches[containerName]; ok {
			return nil
		}

		// New container was created, watch it.
		err := self.watchDirectory(event.Name, containerName)
		if err != nil {
			return err
		}
	case eventType == container.SubcontainerDelete:
		// If we've already seen this event, return.
		if _, ok := self.watches[containerName]; !ok {
			return nil
		}
		delete(self.watches, containerName)

		// Container was deleted, stop watching for it.
		if err := self.watcher.RemoveWatch(event.Name); err != nil {
			return err
		}
	default:
		return fmt.Errorf("unknown event type %v", eventType)
	}

	// Deliver the event.
	events <- container.SubcontainerEvent{
		EventType: eventType,
		Name:      containerName,
	}

	return nil
}

func (self *rawContainerHandler) WatchSubcontainers(events chan container.SubcontainerEvent) error {
	// Lazily initialize the watcher so we don't use it when not asked to.
	if self.watcher == nil {
		w, err := inotify.NewWatcher()
		if err != nil {
			return err
		}
		self.watcher = w
	}

	// Watch this container (all its cgroups) and all subdirectories.
	for _, mnt := range self.cgroupSubsystems.mounts {
		err := self.watchDirectory(path.Join(mnt.Mountpoint, self.name), self.name)
		if err != nil {
			return err
		}
	}

	// Process the events received from the kernel.
	go func() {
		for {
			select {
			case event := <-self.watcher.Event:
				err := self.processEvent(event, events)
				if err != nil {
					glog.Warningf("Error while processing event (%+v): %v", event, err)
				}
			case err := <-self.watcher.Error:
				glog.Warningf("Error while watching %q:", self.name, err)
			case <-self.stopWatcher:
				err := self.watcher.Close()
				if err == nil {
					self.stopWatcher <- err
					self.watcher = nil
					return
				}
			}
		}
	}()

	return nil
}

func (self *rawContainerHandler) StopWatchingSubcontainers() error {
	if self.watcher == nil {
		return fmt.Errorf("can't stop watch that has not started for container %q", self.name)
	}

	// Rendezvous with the watcher thread.
	self.stopWatcher <- nil
	return <-self.stopWatcher
}
<commit_msg>And yet another if.<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// TODO(cAdvisor): Package comment.
package raw

import (
	"fmt"
	"io/ioutil"
	"path"
	"strconv"
	"strings"

	"code.google.com/p/go.exp/inotify"
	"github.com/docker/libcontainer/cgroups"
	"github.com/docker/libcontainer/cgroups/fs"
	"github.com/golang/glog"
	"github.com/google/cadvisor/container"
	"github.com/google/cadvisor/container/libcontainer"
	"github.com/google/cadvisor/info"
	"github.com/google/cadvisor/utils"
)

type rawContainerHandler struct {
	name               string
	cgroup             *cgroups.Cgroup
	cgroupSubsystems   *cgroupSubsystems
	machineInfoFactory info.MachineInfoFactory
	watcher            *inotify.Watcher
	stopWatcher        chan error
	watches            map[string]struct{}
}

func newRawContainerHandler(name string, cgroupSubsystems *cgroupSubsystems, machineInfoFactory info.MachineInfoFactory) (container.ContainerHandler, error) {
	return &rawContainerHandler{
		name: name,
		cgroup: &cgroups.Cgroup{
			Parent: "/",
			Name:   name,
		},
		cgroupSubsystems:   cgroupSubsystems,
		machineInfoFactory: machineInfoFactory,
		stopWatcher:        make(chan error),
		watches:            make(map[string]struct{}),
	}, nil
}

func (self *rawContainerHandler) ContainerReference() (info.ContainerReference, error) {
	// We only know the container by its one name.
	return info.ContainerReference{
		Name: self.name,
	}, nil
}

func readString(dirpath string, file string) string {
	cgroupFile := path.Join(dirpath, file)

	// Ignore non-existent files
	if !utils.FileExists(cgroupFile) {
		return ""
	}

	// Read
	out, err := ioutil.ReadFile(cgroupFile)
	if err != nil {
		glog.Errorf("raw driver: Failed to read %q: %s", cgroupFile, err)
		return ""
	}
	return string(out)
}

func readInt64(dirpath string, file string) uint64 {
	out := readString(dirpath, file)
	if out == "" {
		return 0
	}

	val, err := strconv.ParseUint(strings.TrimSpace(out), 10, 64)
	if err != nil {
		glog.Errorf("raw driver: Failed to parse int %q from file %q: %s", out, path.Join(dirpath, file), err)
		return 0
	}

	return val
}

func (self *rawContainerHandler) GetSpec() (info.ContainerSpec, error) {
	var spec info.ContainerSpec

	// The raw driver assumes unified hierarchy containers.

	// Get machine info.
	mi, err := self.machineInfoFactory.GetMachineInfo()
	if err != nil {
		return spec, err
	}

	// CPU.
	cpuRoot, ok := self.cgroupSubsystems.mountPoints["cpu"]
	if ok {
		cpuRoot = path.Join(cpuRoot, self.name)
		if utils.FileExists(cpuRoot) {
			spec.HasCpu = true
			spec.Cpu.Limit = readInt64(cpuRoot, "cpu.shares")
		}
	}

	// Cpu Mask.
	// This will fail for non-unified hierarchies. We'll return the whole machine mask in that case.
	cpusetRoot, ok := self.cgroupSubsystems.mountPoints["cpuset"]
	if ok {
		cpusetRoot = path.Join(cpusetRoot, self.name)
		if utils.FileExists(cpusetRoot) {
			spec.HasCpu = true
			spec.Cpu.Mask = readString(cpusetRoot, "cpuset.cpus")
			if spec.Cpu.Mask == "" {
				spec.Cpu.Mask = fmt.Sprintf("0-%d", mi.NumCores-1)
			}
		}
	}

	// Memory.
	memoryRoot, ok := self.cgroupSubsystems.mountPoints["memory"]
	if ok {
		memoryRoot = path.Join(memoryRoot, self.name)
		if utils.FileExists(memoryRoot) {
			spec.HasMemory = true
			spec.Memory.Limit = readInt64(memoryRoot, "memory.limit_in_bytes")
			spec.Memory.SwapLimit = readInt64(memoryRoot, "memory.memsw.limit_in_bytes")
		}
	}

	return spec, nil
}

func (self *rawContainerHandler) GetStats() (stats *info.ContainerStats, err error) {
	return libcontainer.GetStatsCgroupOnly(self.cgroup)
}

// Lists all directories under "path" and outputs the results as children of "parent".
func listDirectories(dirpath string, parent string, recursive bool, output map[string]struct{}) error {
	// Ignore if this hierarchy does not exist.
	if !utils.FileExists(dirpath) {
		return nil
	}

	entries, err := ioutil.ReadDir(dirpath)
	if err != nil {
		return err
	}
	for _, entry := range entries {
		// We only grab directories.
		if entry.IsDir() {
			name := path.Join(parent, entry.Name())
			output[name] = struct{}{}

			// List subcontainers if asked to.
			if recursive {
				err := listDirectories(path.Join(dirpath, entry.Name()), name, true, output)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func (self *rawContainerHandler) ListContainers(listType container.ListType) ([]info.ContainerReference, error) {
	containers := make(map[string]struct{})
	for _, subsystem := range self.cgroupSubsystems.mounts {
		err := listDirectories(path.Join(subsystem.Mountpoint, self.name), self.name, listType == container.ListRecursive, containers)
		if err != nil {
			return nil, err
		}
	}

	// Make into container references.
	ret := make([]info.ContainerReference, 0, len(containers))
	for cont := range containers {
		ret = append(ret, info.ContainerReference{
			Name: cont,
		})
	}

	return ret, nil
}

func (self *rawContainerHandler) ListThreads(listType container.ListType) ([]int, error) {
	// TODO(vmarmol): Implement
	return nil, nil
}

func (self *rawContainerHandler) ListProcesses(listType container.ListType) ([]int, error) {
	return fs.GetPids(self.cgroup)
}

func (self *rawContainerHandler) watchDirectory(dir string, containerName string) error {
	err := self.watcher.AddWatch(dir, inotify.IN_CREATE|inotify.IN_DELETE|inotify.IN_MOVE)
	if err != nil {
		return err
	}
	self.watches[containerName] = struct{}{}

	// Watch subdirectories as well.
	entries, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}
	for _, entry := range entries {
		if entry.IsDir() {
			err = self.watchDirectory(path.Join(dir, entry.Name()), path.Join(containerName, entry.Name()))
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func (self *rawContainerHandler) processEvent(event *inotify.Event, events chan container.SubcontainerEvent) error {
	// Convert the inotify event type to a container create or delete.
	var eventType container.SubcontainerEventType
	switch {
	case (event.Mask & inotify.IN_CREATE) > 0:
		eventType = container.SubcontainerAdd
	case (event.Mask & inotify.IN_DELETE) > 0:
		eventType = container.SubcontainerDelete
	case (event.Mask & inotify.IN_MOVED_FROM) > 0:
		eventType = container.SubcontainerDelete
	case (event.Mask & inotify.IN_MOVED_TO) > 0:
		eventType = container.SubcontainerAdd
	default:
		// Ignore other events.
		return nil
	}

	// Derive the container name from the path name.
	var containerName string
	for _, mount := range self.cgroupSubsystems.mounts {
		mountLocation := path.Clean(mount.Mountpoint) + "/"
		if strings.HasPrefix(event.Name, mountLocation) {
			containerName = event.Name[len(mountLocation)-1:]
			break
		}
	}
	if containerName == "" {
		return fmt.Errorf("unable to detect container from watch event on directory %q", event.Name)
	}

	// Maintain the watch for the new or deleted container.
	switch {
	case eventType == container.SubcontainerAdd:
		// If we've already seen this event, return.
		if _, ok := self.watches[containerName]; ok {
			return nil
		}

		// New container was created, watch it.
		err := self.watchDirectory(event.Name, containerName)
		if err != nil {
			return err
		}
	case eventType == container.SubcontainerDelete:
		// If we've already seen this event, return.
		if _, ok := self.watches[containerName]; !ok {
			return nil
		}
		delete(self.watches, containerName)

		// Container was deleted, stop watching for it.
		err := self.watcher.RemoveWatch(event.Name)
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("unknown event type %v", eventType)
	}

	// Deliver the event.
	events <- container.SubcontainerEvent{
		EventType: eventType,
		Name:      containerName,
	}

	return nil
}

func (self *rawContainerHandler) WatchSubcontainers(events chan container.SubcontainerEvent) error {
	// Lazily initialize the watcher so we don't use it when not asked to.
	if self.watcher == nil {
		w, err := inotify.NewWatcher()
		if err != nil {
			return err
		}
		self.watcher = w
	}

	// Watch this container (all its cgroups) and all subdirectories.
	for _, mnt := range self.cgroupSubsystems.mounts {
		err := self.watchDirectory(path.Join(mnt.Mountpoint, self.name), self.name)
		if err != nil {
			return err
		}
	}

	// Process the events received from the kernel.
	go func() {
		for {
			select {
			case event := <-self.watcher.Event:
				err := self.processEvent(event, events)
				if err != nil {
					glog.Warningf("Error while processing event (%+v): %v", event, err)
				}
			case err := <-self.watcher.Error:
				glog.Warningf("Error while watching %q:", self.name, err)
			case <-self.stopWatcher:
				err := self.watcher.Close()
				if err == nil {
					self.stopWatcher <- err
					self.watcher = nil
					return
				}
			}
		}
	}()

	return nil
}

func (self *rawContainerHandler) StopWatchingSubcontainers() error {
	if self.watcher == nil {
		return fmt.Errorf("can't stop watch that has not started for container %q", self.name)
	}

	// Rendezvous with the watcher thread.
	self.stopWatcher <- nil
	return <-self.stopWatcher
}
<|endoftext|>


<commit_before>// Copyright © 2017 Pennock Tech, LLC.
// All rights reserved, except as granted under license.
// Licensed per file LICENSE.txt

package main

import (
	"errors"
	"fmt"
	"net"
	"os"
	"regexp"
	"strconv"
	"sync"

	"go.pennock.tech/smtpdane/internal/errorlist"

	"github.com/miekg/dns"
)

const EnvKeyDNSResolver = "DNS_RESOLVER"

var dnsSettings struct {
	sync.RWMutex
	conf   *dns.ClientConfig
	client *dns.Client
}

func initDNS() (*dns.ClientConfig, *dns.Client, error) {
	dnsSettings.RLock()
	if dnsSettings.client != nil {
		defer dnsSettings.RUnlock()
		return dnsSettings.conf, dnsSettings.client, nil
	}
	dnsSettings.RUnlock()
	dnsSettings.Lock()
	defer dnsSettings.Unlock()
	if dnsSettings.client != nil {
		return dnsSettings.conf, dnsSettings.client, nil
	}

	var (
		conf *dns.ClientConfig
		err  error
	)
	if os.Getenv(EnvKeyDNSResolver) == "" {
		conf, err = dns.ClientConfigFromFile("/etc/resolv.conf")
		if err != nil {
			return nil, nil, err
		}
	}

	c := new(dns.Client)

	dnsSettings.conf = conf
	dnsSettings.client = c

	return dnsSettings.conf, dnsSettings.client, nil
}

func resolversFromList(input []string, defDNSPort string) []string {
	r := make([]string, len(input))
	for i := range input {
		r[i] = HostPortWithDefaultPort(input[i], defDNSPort)
	}
	return r
}

var resolverSplitRE *regexp.Regexp

func init() {
	resolverSplitRE = regexp.MustCompile(`[,\s]+`)
}

func resolversFromString(input string) []string {
	return resolversFromList(resolverSplitRE.Split(input, -1), "53")
}

// FIXME: This is not doing DNS validation locally.
// It's resolving DNS, delegating trust in validation to the resolver by
// trusting the AD bit.
// I want to get this working without needing a validating resolver.
// This should be a standalone monitoring tool.
func resolveRRSecure(
	// the cbfunc is called the the confirmed RR type and the rr and the rrname;
	// it should return an item to be added to the resolveRRSecure return list,
	// and an error; non-nil error inhibits appending to the list.
	cbfunc func(typ uint16, rr dns.RR, rrname string) (interface{}, error),
	rrname string,
	typlist ...uint16,
) ([]interface{}, error) {
	config, c, err := initDNS()
	if err != nil {
		return nil, err
	}

	var resolvers []string
	if r := os.Getenv(EnvKeyDNSResolver); r != "" {
		resolvers = resolversFromString(r)
	} else {
		resolvers = resolversFromList(config.Servers, config.Port)
	}

	resultList := make([]interface{}, 0, 20)
	errList := errorlist.New()

	m := new(dns.Msg)
	m.SetEdns0(dns.DefaultMsgSize, true)

	// why is this uint16 ipv dns.Type ?  Infelicity stuck in API?
DNS_RRTYPE_LOOP:
	for _, typ := range typlist {
		m.SetQuestion(rrname, typ)

		var (
			r   *dns.Msg
			err error
		)

		for _, resolver := range resolvers {
			// TODO: add retries
			r, _, err = c.Exchange(m, resolver)
			if err != nil {
				errList.Add(err)
				r = nil
				continue
			}
			if r.Rcode != dns.RcodeSuccess {
				failure, known := dns.RcodeToString[r.Rcode]
				if !known {
					failure = fmt.Sprintf("Rcode<%d> (unknown)", r.Rcode)
				}
				errList.AddErrorf("DNS lookup non-successful [resolver %v]: %v", resolver, failure)
				r = nil
				continue
			}
			// Here we depend upon AD bit and so are still secure, assuming secure
			// link to trusted resolver.
			if !r.AuthenticatedData {
				errList.AddErrorf("not AD set for results from %v for %q/%v query", resolver, rrname, dns.Type(typ))
				r = nil
				continue
			}
		}

		if r == nil {
			errList.AddErrorf("[%q/%v]: all DNS resolver queries failed, unable to get authentic result", rrname, dns.Type(typ))
			// seems likely might be SERVFAIL from broken auth servers for AAAA records
			continue DNS_RRTYPE_LOOP
		}

		for _, rr := range r.Answer {
			// TODO: CNAME?
			if rr.Header().Rrtype != typ {
				continue
			}
			x, err := cbfunc(typ, dns.Copy(rr), rrname)
			if err != nil {
				errList.Add(err)
			} else {
				resultList = append(resultList, x)
			}
		}
	}

	if len(resultList) == 0 {
		errList.Add(errors.New("no results found"))
		return nil, errList
	}
	return resultList, errList.Maybe()
}

// There's a lot of repetition/boilerplate in the below.
// If we expand beyond where we are at now, then we really should consider reflection; more complexity, less repetition.

func cbRRTypeAddr(typ uint16, rr dns.RR, rrname string) (interface{}, error) {
	switch typ {
	case dns.TypeA:
		if ip, ok := rr.(*dns.A); ok {
			return ip.A, nil
		} else {
			return nil, fmt.Errorf("A record failed to cast to *dns.A [%q/%v]", rrname, dns.Type(typ))
		}
	case dns.TypeAAAA:
		if ip, ok := rr.(*dns.AAAA); ok {
			return ip.AAAA, nil
		} else {
			return nil, fmt.Errorf("AAAA record failed to cast to *dns.AAAA [%q/%v]", rrname, dns.Type(typ))
		}
	}
	return nil, fmt.Errorf("BUG: cbRRTypeAddr(%v,..,%q) called, expected A/AAAA", dns.Type(typ), rrname)
}

func ResolveAddrSecure(hostname string) ([]net.IP, error) {
	rl, e := resolveRRSecure(cbRRTypeAddr, dns.Fqdn(hostname), dns.TypeAAAA, dns.TypeA)
	if e != nil {
		return nil, e
	}
	addrList := make([]net.IP, len(rl))
	for i := range rl {
		addrList[i] = rl[i].(net.IP)
	}
	return addrList, nil
}

type TLSAset struct {
	RRs       []*dns.TLSA
	name      string
	foundName string
}

func cbRRTypeTLSA(typ uint16, rr dns.RR, rrname string) (interface{}, error) {
	switch typ {
	case dns.TypeTLSA:
		if tlsa, ok := rr.(*dns.TLSA); ok {
			return tlsa, nil
		} else {
			return nil, fmt.Errorf("TLSA record failed to cast to *dns.TLSA [%q/%v]", rrname, dns.Type(typ))
		}
	}
	return nil, fmt.Errorf("BUG: cbRRTypeTLSA(%v,..,%q) called, expected TLSA", dns.Type(typ), rrname)
}

func ResolveTLSA(hostname string, port int) (*TLSAset, error) {
	tlsaName := fmt.Sprintf("_%d._tcp.%s", port, dns.Fqdn(hostname))
	rl, e := resolveRRSecure(cbRRTypeTLSA, tlsaName, dns.TypeTLSA)
	if e != nil {
		return nil, e
	}

	TLSAList := make([]*dns.TLSA, len(rl))
	for i := range rl {
		TLSAList[i] = rl[i].(*dns.TLSA)
	}

	return &TLSAset{
		RRs:       TLSAList,
		name:      tlsaName,
		foundName: TLSAList[0].Hdr.Name,
	}, nil
}

// TLSAShortString provides something suitable for output without showing the
// full contents; for our uses, we don't need the RR_Header and for
// full-certs-in-DNS we don't _want_ to print it all.
// Viktor points out that for full certs in DNS, the start of the record will
// be less useful, so show the _last_ 16 octets
// TLSAShortString is "enough to probably fit on a line with much other text".
func TLSAShortString(rr *dns.TLSA) string {
	offset := len(rr.Certificate) - 16
	prefix := "..."
	if offset < 0 {
		offset = 0
		prefix = ""
	}
	return strconv.Itoa(int(rr.Usage)) + " " +
		strconv.Itoa(int(rr.Selector)) + " " +
		strconv.Itoa(int(rr.MatchingType)) + " " +
		prefix + rr.Certificate[offset:]
}

// TLSAMediumString is for where the TLSA record is probably all that's on a line.
// Assume 2 leading spaces, 1 digit for each of the three leading fields, a space
// after each, that's 8, allow for 70.
func TLSAMediumString(rr *dns.TLSA) string {
	var rest, prefix string
	if len(rr.Certificate) <= 70 {
		rest = rr.Certificate
	} else {
		prefix = "..."
		rest = rr.Certificate[(len(rr.Certificate) - 67):]
	}
	return strconv.Itoa(int(rr.Usage)) + " " +
		strconv.Itoa(int(rr.Selector)) + " " +
		strconv.Itoa(int(rr.MatchingType)) + " " +
		prefix + rest
}

func cbRRTypeMX(typ uint16, rr dns.RR, rrname string) (interface{}, error) {
	switch typ {
	case dns.TypeMX:
		if mx, ok := rr.(*dns.MX); ok {
			return mx.Mx, nil
		} else {
			return nil, fmt.Errorf("MX record failed to cast to *dns.MX [%q/%v]", rrname, dns.Type(typ))
		}
	}
	return nil, fmt.Errorf("BUG: cbRRTypeMX(%v,..,%q) called, expected MX", dns.Type(typ), rrname)
}

// ResolveMX only returns the hostnames, we don't care about the Preference
func ResolveMX(hostname string) ([]string, error) {
	rl, e := resolveRRSecure(cbRRTypeMX, dns.Fqdn(hostname), dns.TypeMX)
	if e != nil {
		return nil, e
	}
	hostnameList := make([]string, len(rl))
	for i := range rl {
		hostnameList[i] = rl[i].(string)
	}
	return hostnameList, nil
}

func cbRRTypeSRV(typ uint16, rr dns.RR, rrname string) (interface{}, error) {
	switch typ {
	case dns.TypeSRV:
		if srv, ok := rr.(*dns.SRV); ok {
			return srv, nil
		} else {
			return nil, fmt.Errorf("SRV record failed to cast to *dns.SRV [%q/%v]", rrname, dns.Type(typ))
		}
	}
	return nil, fmt.Errorf("BUG: cbRRTypeSRV(%v,..,%q) called, expected SRV", dns.Type(typ), rrname)
}

// ResolveSRV returns MX records, we need at least the Port, not just the Target
func ResolveSRV(lookup string) ([]*dns.SRV, error) {
	rl, e := resolveRRSecure(cbRRTypeSRV, dns.Fqdn(lookup), dns.TypeSRV)
	if e != nil {
		return nil, e
	}
	srvList := make([]*dns.SRV, len(rl))
	for i := range rl {
		srvList[i] = rl[i].(*dns.SRV)
	}
	return srvList, nil
}
<commit_msg>DNS: retries and TCP fallback<commit_after>// Copyright © 2017 Pennock Tech, LLC.
// All rights reserved, except as granted under license.
// Licensed per file LICENSE.txt

package main

import (
	"errors"
	"fmt"
	"net"
	"os"
	"regexp"
	"strconv"
	"sync"

	"go.pennock.tech/smtpdane/internal/errorlist"

	"github.com/miekg/dns"
)

const EnvKeyDNSResolver = "DNS_RESOLVER"

var dnsSettings struct {
	sync.RWMutex
	conf   *dns.ClientConfig
	client *dns.Client
}

func initDNS() (*dns.ClientConfig, *dns.Client, error) {
	dnsSettings.RLock()
	if dnsSettings.client != nil {
		defer dnsSettings.RUnlock()
		return dnsSettings.conf, dnsSettings.client, nil
	}
	dnsSettings.RUnlock()
	dnsSettings.Lock()
	defer dnsSettings.Unlock()
	if dnsSettings.client != nil {
		return dnsSettings.conf, dnsSettings.client, nil
	}

	var (
		conf *dns.ClientConfig
		err  error
	)
	if os.Getenv(EnvKeyDNSResolver) == "" {
		conf, err = dns.ClientConfigFromFile("/etc/resolv.conf")
		if err != nil {
			return nil, nil, err
		}
	}

	c := new(dns.Client)

	dnsSettings.conf = conf
	dnsSettings.client = c

	return dnsSettings.conf, dnsSettings.client, nil
}

func resolversFromList(input []string, defDNSPort string) []string {
	r := make([]string, len(input))
	for i := range input {
		r[i] = HostPortWithDefaultPort(input[i], defDNSPort)
	}
	return r
}

var resolverSplitRE *regexp.Regexp

func init() {
	resolverSplitRE = regexp.MustCompile(`[,\s]+`)
}

func resolversFromString(input string) []string {
	return resolversFromList(resolverSplitRE.Split(input, -1), "53")
}

// FIXME: This is not doing DNS validation locally.
// It's resolving DNS, delegating trust in validation to the resolver by
// trusting the AD bit.
// I want to get this working without needing a validating resolver.
// This should be a standalone monitoring tool.
func resolveRRSecure(
	// the cbfunc is called the the confirmed RR type and the rr and the rrname;
	// it should return an item to be added to the resolveRRSecure return list,
	// and an error; non-nil error inhibits appending to the list.
	cbfunc func(typ uint16, rr dns.RR, rrname string) (interface{}, error),
	rrname string,
	typlist ...uint16,
) ([]interface{}, error) {
	config, c, err := initDNS()
	if err != nil {
		return nil, err
	}

	var resolvers []string
	if r := os.Getenv(EnvKeyDNSResolver); r != "" {
		resolvers = resolversFromString(r)
	} else {
		resolvers = resolversFromList(config.Servers, config.Port)
	}

	resultList := make([]interface{}, 0, 20)
	errList := errorlist.New()

	m := new(dns.Msg)
	m.SetEdns0(dns.DefaultMsgSize, true)

	// why is this uint16 ipv dns.Type ?  Infelicity stuck in API?
DNS_RRTYPE_LOOP:
	for _, typ := range typlist {
		m.SetQuestion(rrname, typ)

		var (
			r   *dns.Msg
			err error
		)

	DNS_RESOLVER_LOOP:
		for _, resolver := range resolvers {
			c.Net = "udp"
		RETRY_DNS_LOOKUP:
			for i := 0; i < config.Attempts; i++ {
				r, _, err = c.Exchange(m, resolver)
				if err != nil {
					if nerr, ok := err.(net.Error); ok && nerr.Timeout() && i < config.Attempts {
						continue
					}
					errList.Add(err)
					r = nil
					continue DNS_RESOLVER_LOOP
				}
			}

			if r.Rcode != dns.RcodeSuccess {
				failure, known := dns.RcodeToString[r.Rcode]
				if !known {
					failure = fmt.Sprintf("Rcode<%d> (unknown)", r.Rcode)
				}
				errList.AddErrorf("DNS lookup non-successful [resolver %v]: %v", resolver, failure)
				r = nil
				continue DNS_RESOLVER_LOOP
			}

			// Check for truncation first, in case some bad servers truncate
			// the DNSSEC data needed to be AD.
			if r.Truncated {
				c.Net = "tcp"
				goto RETRY_DNS_LOOKUP
			}

			// Here we depend upon AD bit and so are still secure, assuming secure
			// link to trusted resolver.
			// Assume all our resolvers are equivalent for AD/not, so if not AD, try the
			// next type (because some DNS servers break horribly on AAAA).
			if !r.AuthenticatedData {
				errList.AddErrorf("not AD set for results from %v for %q/%v query", resolver, rrname, dns.Type(typ))
				r = nil
				continue DNS_RRTYPE_LOOP
			}
		}

		if r == nil {
			errList.AddErrorf("[%q/%v]: all DNS resolver queries failed, unable to get authentic result", rrname, dns.Type(typ))
			// seems likely might be SERVFAIL from broken auth servers for AAAA records
			continue DNS_RRTYPE_LOOP
		}

		for _, rr := range r.Answer {
			// TODO: CNAME?
			if rr.Header().Rrtype != typ {
				continue
			}
			x, err := cbfunc(typ, dns.Copy(rr), rrname)
			if err != nil {
				errList.Add(err)
			} else {
				resultList = append(resultList, x)
			}
		}
	}

	if len(resultList) == 0 {
		errList.Add(errors.New("no results found"))
		return nil, errList
	}
	return resultList, errList.Maybe()
}

// There's a lot of repetition/boilerplate in the below.
// If we expand beyond where we are at now, then we really should consider reflection; more complexity, less repetition.

func cbRRTypeAddr(typ uint16, rr dns.RR, rrname string) (interface{}, error) {
	switch typ {
	case dns.TypeA:
		if ip, ok := rr.(*dns.A); ok {
			return ip.A, nil
		} else {
			return nil, fmt.Errorf("A record failed to cast to *dns.A [%q/%v]", rrname, dns.Type(typ))
		}
	case dns.TypeAAAA:
		if ip, ok := rr.(*dns.AAAA); ok {
			return ip.AAAA, nil
		} else {
			return nil, fmt.Errorf("AAAA record failed to cast to *dns.AAAA [%q/%v]", rrname, dns.Type(typ))
		}
	}
	return nil, fmt.Errorf("BUG: cbRRTypeAddr(%v,..,%q) called, expected A/AAAA", dns.Type(typ), rrname)
}

func ResolveAddrSecure(hostname string) ([]net.IP, error) {
	rl, e := resolveRRSecure(cbRRTypeAddr, dns.Fqdn(hostname), dns.TypeAAAA, dns.TypeA)
	if e != nil {
		return nil, e
	}
	addrList := make([]net.IP, len(rl))
	for i := range rl {
		addrList[i] = rl[i].(net.IP)
	}
	return addrList, nil
}

type TLSAset struct {
	RRs       []*dns.TLSA
	name      string
	foundName string
}

func cbRRTypeTLSA(typ uint16, rr dns.RR, rrname string) (interface{}, error) {
	switch typ {
	case dns.TypeTLSA:
		if tlsa, ok := rr.(*dns.TLSA); ok {
			return tlsa, nil
		} else {
			return nil, fmt.Errorf("TLSA record failed to cast to *dns.TLSA [%q/%v]", rrname, dns.Type(typ))
		}
	}
	return nil, fmt.Errorf("BUG: cbRRTypeTLSA(%v,..,%q) called, expected TLSA", dns.Type(typ), rrname)
}

func ResolveTLSA(hostname string, port int) (*TLSAset, error) {
	tlsaName := fmt.Sprintf("_%d._tcp.%s", port, dns.Fqdn(hostname))
	rl, e := resolveRRSecure(cbRRTypeTLSA, tlsaName, dns.TypeTLSA)
	if e != nil {
		return nil, e
	}

	TLSAList := make([]*dns.TLSA, len(rl))
	for i := range rl {
		TLSAList[i] = rl[i].(*dns.TLSA)
	}

	return &TLSAset{
		RRs:       TLSAList,
		name:      tlsaName,
		foundName: TLSAList[0].Hdr.Name,
	}, nil
}

// TLSAShortString provides something suitable for output without showing the
// full contents; for our uses, we don't need the RR_Header and for
// full-certs-in-DNS we don't _want_ to print it all.
// Viktor points out that for full certs in DNS, the start of the record will
// be less useful, so show the _last_ 16 octets
// TLSAShortString is "enough to probably fit on a line with much other text".
func TLSAShortString(rr *dns.TLSA) string {
	offset := len(rr.Certificate) - 16
	prefix := "..."
	if offset < 0 {
		offset = 0
		prefix = ""
	}
	return strconv.Itoa(int(rr.Usage)) + " " +
		strconv.Itoa(int(rr.Selector)) + " " +
		strconv.Itoa(int(rr.MatchingType)) + " " +
		prefix + rr.Certificate[offset:]
}

// TLSAMediumString is for where the TLSA record is probably all that's on a line.
// Assume 2 leading spaces, 1 digit for each of the three leading fields, a space
// after each, that's 8, allow for 70.
func TLSAMediumString(rr *dns.TLSA) string {
	var rest, prefix string
	if len(rr.Certificate) <= 70 {
		rest = rr.Certificate
	} else {
		prefix = "..."
		rest = rr.Certificate[(len(rr.Certificate) - 67):]
	}
	return strconv.Itoa(int(rr.Usage)) + " " +
		strconv.Itoa(int(rr.Selector)) + " " +
		strconv.Itoa(int(rr.MatchingType)) + " " +
		prefix + rest
}

func cbRRTypeMX(typ uint16, rr dns.RR, rrname string) (interface{}, error) {
	switch typ {
	case dns.TypeMX:
		if mx, ok := rr.(*dns.MX); ok {
			return mx.Mx, nil
		} else {
			return nil, fmt.Errorf("MX record failed to cast to *dns.MX [%q/%v]", rrname, dns.Type(typ))
		}
	}
	return nil, fmt.Errorf("BUG: cbRRTypeMX(%v,..,%q) called, expected MX", dns.Type(typ), rrname)
}

// ResolveMX only returns the hostnames, we don't care about the Preference
func ResolveMX(hostname string) ([]string, error) {
	rl, e := resolveRRSecure(cbRRTypeMX, dns.Fqdn(hostname), dns.TypeMX)
	if e != nil {
		return nil, e
	}
	hostnameList := make([]string, len(rl))
	for i := range rl {
		hostnameList[i] = rl[i].(string)
	}
	return hostnameList, nil
}

func cbRRTypeSRV(typ uint16, rr dns.RR, rrname string) (interface{}, error) {
	switch typ {
	case dns.TypeSRV:
		if srv, ok := rr.(*dns.SRV); ok {
			return srv, nil
		} else {
			return nil, fmt.Errorf("SRV record failed to cast to *dns.SRV [%q/%v]", rrname, dns.Type(typ))
		}
	}
	return nil, fmt.Errorf("BUG: cbRRTypeSRV(%v,..,%q) called, expected SRV", dns.Type(typ), rrname)
}

// ResolveSRV returns MX records, we need at least the Port, not just the Target
func ResolveSRV(lookup string) ([]*dns.SRV, error) {
	rl, e := resolveRRSecure(cbRRTypeSRV, dns.Fqdn(lookup), dns.TypeSRV)
	if e != nil {
		return nil, e
	}
	srvList := make([]*dns.SRV, len(rl))
	for i := range rl {
		srvList[i] = rl[i].(*dns.SRV)
	}
	return srvList, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Kazuhisa TAKEI<xtakei@me.com>. All rights reserved.
// Use of this source code is governed by MPL-2.0 license tha can be
// found in the LICENSE file
//
// Package buffer_list implements a double linked list with sequencial buffer data.
//
// implement AList(alias list)
// example
//   bl :=  buffer_list.New(Hoge{}, 10000)
//   alist := Alist{parent: bl}
//   ae := alist.NewElem()
//   alist.Front().Insert(ae)
//   v := ae.Value().(*Hoge)
//   v.Commit()

package buffer_list

import (
	"sync"
)

type AList struct {
	m      sync.Mutex
	parent *List
	root   *AElement
	Len    int
	e2ae   map[*Element]*AElement
}

// AElement is an element of alias linked list.
type AElement struct {
	list   *AList
	prev   *AElement
	next   *AElement
	parent *Element
}

// create new alias element with element
func (al *AList) NewElem() (ae *AElement) {
	ae = &AElement{list: al, parent: al.parent.InsertLast()}
	return ae
}

// add element to last of list
func (al *AList) Push(ae *AElement) bool {
	al.m.Lock()
	defer al.m.Unlock()
	if al.root == nil {
		al.root = ae
		ae.prev = ae
		ae.next = nil
	} else {
		ae.prev = al.root.prev
		al.root.prev.next = ae
		al.root.prev = ae
		ae.next = nil
	}
	al.Len++
	return true
}

// first element of list
func (al *AList) Front() *AElement {
	return al.root
}

// last element of list
func (al *AList) Back() *AElement {
	return al.root.prev
}

// add elemnet after at_element
func (at *AElement) Insert(e *AElement) *AElement {
	l := at.list

	l.m.Lock()
	defer l.m.Unlock()

	e.list = at.list
	n := at.next
	at.next = e
	e.prev = at
	if n != nil {
		n.prev = e
		e.next = n
	} else {
		e.list.root.prev = e
	}
	l.Len++
	return e
}

// return value of reel element object
func (ae *AElement) Value() interface{} {
	if ae.list.e2ae == nil {
		ae.list.e2ae = make(map[*Element]*AElement)
	}
	ae.list.e2ae[ae.parent] = ae

	return ae.parent.Value()
}
func (ae *AElement) InitValue() {
	ae.parent.InitValue()
}

// register pointer protect GC
func (ae *AElement) Commit() {
	ae.parent.Commit()
}

// remove element from alias list
func (e *AElement) Remove() bool {
	e.list.m.Lock()
	defer e.list.m.Unlock()

	delete(e.list.e2ae, e.parent)

	at := e.prev
	n := e.next
	if at.next == e {
		at.next = n
	}
	if n != nil {
		n.prev = at
	}
	e.list.Len -= 1
	e.list = nil
	return true
}

// Next return next list element or nil
func (e *AElement) Next() *AElement {
	return e.next
}

// Prev return previous list element or nil
func (e *AElement) Prev() *AElement {
	return e.prev
}

// remove  from alias list and free real list
func (e *AElement) Free() {
	e.Remove()
	e.parent.Free()
	e.parent = nil
}

func (e *AElement) Base() *Element {
	return e.parent
}

func (l *AList) ElemByValue(v interface{}) *AElement {

	e := l.parent.ElemByValue(v)
	if e == nil {
		return nil
	}
	if l.e2ae == nil {
		return nil
	}
	return l.e2ae[e]
}
<commit_msg> add AElement.List()<commit_after>// Copyright 2015 Kazuhisa TAKEI<xtakei@me.com>. All rights reserved.
// Use of this source code is governed by MPL-2.0 license tha can be
// found in the LICENSE file
//
// Package buffer_list implements a double linked list with sequencial buffer data.
//
// implement AList(alias list)
// example
//   bl :=  buffer_list.New(Hoge{}, 10000)
//   alist := Alist{parent: bl}
//   ae := alist.NewElem()
//   alist.Front().Insert(ae)
//   v := ae.Value().(*Hoge)
//   v.Commit()

package buffer_list

import (
	"sync"
)

type AList struct {
	m      sync.Mutex
	parent *List
	root   *AElement
	Len    int
	e2ae   map[*Element]*AElement
}

// AElement is an element of alias linked list.
type AElement struct {
	list   *AList
	prev   *AElement
	next   *AElement
	parent *Element
}

// create new alias element with element
func (al *AList) NewElem() (ae *AElement) {
	ae = &AElement{list: al, parent: al.parent.InsertLast()}
	return ae
}

// add element to last of list
func (al *AList) Push(ae *AElement) bool {
	al.m.Lock()
	defer al.m.Unlock()
	if al.root == nil {
		al.root = ae
		ae.prev = ae
		ae.next = nil
	} else {
		ae.prev = al.root.prev
		al.root.prev.next = ae
		al.root.prev = ae
		ae.next = nil
	}
	al.Len++
	return true
}

// first element of list
func (al *AList) Front() *AElement {
	return al.root
}

// last element of list
func (al *AList) Back() *AElement {
	return al.root.prev
}

// add elemnet after at_element
func (at *AElement) Insert(e *AElement) *AElement {
	l := at.list

	l.m.Lock()
	defer l.m.Unlock()

	e.list = at.list
	n := at.next
	at.next = e
	e.prev = at
	if n != nil {
		n.prev = e
		e.next = n
	} else {
		e.list.root.prev = e
	}
	l.Len++
	return e
}

// return value of reel element object
func (ae *AElement) Value() interface{} {
	if ae.list.e2ae == nil {
		ae.list.e2ae = make(map[*Element]*AElement)
	}
	ae.list.e2ae[ae.parent] = ae

	return ae.parent.Value()
}
func (ae *AElement) InitValue() {
	ae.parent.InitValue()
}

// register pointer protect GC
func (ae *AElement) Commit() {
	ae.parent.Commit()
}

// remove element from alias list
func (e *AElement) Remove() bool {
	e.list.m.Lock()
	defer e.list.m.Unlock()

	delete(e.list.e2ae, e.parent)

	at := e.prev
	n := e.next
	if at.next == e {
		at.next = n
	}
	if n != nil {
		n.prev = at
	}
	e.list.Len -= 1
	e.list = nil
	return true
}

// Next return next list element or nil
func (e *AElement) Next() *AElement {
	return e.next
}

// Prev return previous list element or nil
func (e *AElement) Prev() *AElement {
	return e.prev
}

// remove  from alias list and free real list
func (e *AElement) Free() {
	e.Remove()
	e.parent.Free()
	e.parent = nil
}

func (e *AElement) Base() *Element {
	return e.parent
}
func (e *AElement) List() *AList {
	return e.list
}

func (l *AList) ElemByValue(v interface{}) *AElement {

	e := l.parent.ElemByValue(v)
	if e == nil {
		return nil
	}
	if l.e2ae == nil {
		return nil
	}
	return l.e2ae[e]
}
<|endoftext|>


<commit_before>package consul

import (
	"fmt"
	"log"
	"net"
	"net/url"
	"strings"

	consulapi "github.com/hashicorp/consul/api"
	"github.com/42wim/registrator-work/bridge"
)

const DefaultInterval = "10s"

func init() {
	bridge.Register(new(Factory), "consul")
}

func (r *ConsulAdapter) interpolateService(script string, service *bridge.Service) string {
	withIp := strings.Replace(script, "$SERVICE_IP", service.Origin.HostIP, -1)
	withPort := strings.Replace(withIp, "$SERVICE_PORT", service.Origin.HostPort, -1)
	return withPort
}

type Factory struct{}

func (f *Factory) New(uri *url.URL) bridge.RegistryAdapter {
	config := consulapi.DefaultConfig()
	if uri.Host != "" {
		config.Address = uri.Host
	}
	client, err := consulapi.NewClient(config)
	if err != nil {
		log.Fatal("consul: ", uri.Scheme)
	}
	return &ConsulAdapter{client: client}
}

type ConsulAdapter struct {
	client *consulapi.Client
}

// Ping will try to connect to consul by attempting to retrieve the current leader.
func (r *ConsulAdapter) Ping() error {
	status := r.client.Status()
	leader, err := status.Leader()
	if err != nil {
		return err
	}
	log.Println("consul: current leader ", leader)

	return nil
}

func (r *ConsulAdapter) Register(service *bridge.Service) error {
	registration := new(consulapi.AgentServiceRegistration)
	registration.ID = service.ID
	registration.Name = service.Name
	registration.Port = service.Port
	registration.Tags = service.Tags
	registration.Address = service.IP
	registration.Check = r.buildCheck(service)
	return r.client.Agent().ServiceRegister(registration)
}

func (r *ConsulAdapter) buildCheck(service *bridge.Service) *consulapi.AgentServiceCheck {
	check := new(consulapi.AgentServiceCheck)
	if path := service.Attrs["check_http"]; path != "" {
		if net.ParseIP(service.IP).To4() == nil { // verify if its an ipv6 address
			service.IP = fmt.Sprintf("[%s]", service.IP)
		}
		check.HTTP = fmt.Sprintf("http://%s:%d%s", service.IP, service.Port, path)
		if timeout := service.Attrs["check_timeout"]; timeout != "" {
			check.Timeout = timeout
		}
	} else if cmd := service.Attrs["check_cmd"]; cmd != "" {
		check.Script = fmt.Sprintf("check-cmd %s %s %s", service.Origin.ContainerID[:12], service.Origin.ExposedPort, cmd)
	} else if script := service.Attrs["check_script"]; script != "" {
		check.Script = r.interpolateService(script, service)
	} else if ttl := service.Attrs["check_ttl"]; ttl != "" {
		check.TTL = ttl
	} else {
		return nil
	}
	if check.Script != "" || check.HTTP != "" {
		if interval := service.Attrs["check_interval"]; interval != "" {
			check.Interval = interval
		} else {
			check.Interval = DefaultInterval
		}
	}
	return check
}

func (r *ConsulAdapter) Deregister(service *bridge.Service) error {
	return r.client.Agent().ServiceDeregister(service.ID)
}

func (r *ConsulAdapter) Refresh(service *bridge.Service) error {
	return nil
}

func (r *ConsulAdapter) Services() ([]*bridge.Service, error) {
	services, err := r.client.Agent().Services()
	if err != nil {
		return []*bridge.Service{}, err
	}
	out := make([]*bridge.Service, len(services))
	i := 0
	for _, v := range services {
		s := &bridge.Service{
			ID:   v.ID,
			Name: v.Service,
			Port: v.Port,
			Tags: v.Tags,
			IP:   v.Address,
		}
		out[i] = s
		i++
	}
	return out, nil
}
<commit_msg>Don't add square brackets every refresh<commit_after>package consul

import (
	"fmt"
	"log"
	"net"
	"net/url"
	"strings"

	"github.com/42wim/registrator-work/bridge"
	consulapi "github.com/hashicorp/consul/api"
)

const DefaultInterval = "10s"

func init() {
	bridge.Register(new(Factory), "consul")
}

func (r *ConsulAdapter) interpolateService(script string, service *bridge.Service) string {
	withIp := strings.Replace(script, "$SERVICE_IP", service.Origin.HostIP, -1)
	withPort := strings.Replace(withIp, "$SERVICE_PORT", service.Origin.HostPort, -1)
	return withPort
}

type Factory struct{}

func (f *Factory) New(uri *url.URL) bridge.RegistryAdapter {
	config := consulapi.DefaultConfig()
	if uri.Host != "" {
		config.Address = uri.Host
	}
	client, err := consulapi.NewClient(config)
	if err != nil {
		log.Fatal("consul: ", uri.Scheme)
	}
	return &ConsulAdapter{client: client}
}

type ConsulAdapter struct {
	client *consulapi.Client
}

// Ping will try to connect to consul by attempting to retrieve the current leader.
func (r *ConsulAdapter) Ping() error {
	status := r.client.Status()
	leader, err := status.Leader()
	if err != nil {
		return err
	}
	log.Println("consul: current leader ", leader)

	return nil
}

func (r *ConsulAdapter) Register(service *bridge.Service) error {
	registration := new(consulapi.AgentServiceRegistration)
	registration.ID = service.ID
	registration.Name = service.Name
	registration.Port = service.Port
	registration.Tags = service.Tags
	registration.Address = service.IP
	registration.Check = r.buildCheck(service)
	return r.client.Agent().ServiceRegister(registration)
}

func (r *ConsulAdapter) buildCheck(service *bridge.Service) *consulapi.AgentServiceCheck {
	check := new(consulapi.AgentServiceCheck)
	if path := service.Attrs["check_http"]; path != "" {
		if net.ParseIP(service.IP).To4() == nil && service.IP[0] != '[' { // verify if its an ipv6 address
			service.IP = fmt.Sprintf("[%s]", service.IP)
		}
		check.HTTP = fmt.Sprintf("http://%s:%d%s", service.IP, service.Port, path)
		if timeout := service.Attrs["check_timeout"]; timeout != "" {
			check.Timeout = timeout
		}
	} else if cmd := service.Attrs["check_cmd"]; cmd != "" {
		check.Script = fmt.Sprintf("check-cmd %s %s %s", service.Origin.ContainerID[:12], service.Origin.ExposedPort, cmd)
	} else if script := service.Attrs["check_script"]; script != "" {
		check.Script = r.interpolateService(script, service)
	} else if ttl := service.Attrs["check_ttl"]; ttl != "" {
		check.TTL = ttl
	} else {
		return nil
	}
	if check.Script != "" || check.HTTP != "" {
		if interval := service.Attrs["check_interval"]; interval != "" {
			check.Interval = interval
		} else {
			check.Interval = DefaultInterval
		}
	}
	return check
}

func (r *ConsulAdapter) Deregister(service *bridge.Service) error {
	return r.client.Agent().ServiceDeregister(service.ID)
}

func (r *ConsulAdapter) Refresh(service *bridge.Service) error {
	return nil
}

func (r *ConsulAdapter) Services() ([]*bridge.Service, error) {
	services, err := r.client.Agent().Services()
	if err != nil {
		return []*bridge.Service{}, err
	}
	out := make([]*bridge.Service, len(services))
	i := 0
	for _, v := range services {
		s := &bridge.Service{
			ID:   v.ID,
			Name: v.Service,
			Port: v.Port,
			Tags: v.Tags,
			IP:   v.Address,
		}
		out[i] = s
		i++
	}
	return out, nil
}
<|endoftext|>


<commit_before>package dchan

import (
	"net"
	"time"

	"github.com/chzyer/flow"
	"github.com/chzyer/next/packet"
	kcp "github.com/xtaci/kcp-go"
)

type UdpChanFactory struct {
	port int32
}

func NewUdpChanFactory() *UdpChanFactory {
	return &UdpChanFactory{
		port: 10000,
	}
}

type wrapLn struct {
	*kcp.Listener
}

func (w *wrapLn) Accept() (net.Conn, error) {
	sess, err := w.Listener.Accept()
	if err != nil {
		return nil, err
	}
	sess.SetMtu(1350)
	sess.SetNoDelay(1, 10, 2, 1)
	return sess, nil
}

func (u *UdpChanFactory) Listen(f *flow.Flow) (net.Listener, error) {
	ln, err := kcp.Listen(":0")
	if err != nil {
		return nil, err
	}
	return &wrapLn{ln}, nil
}

func (UdpChanFactory) DialTimeout(host string, timeout time.Duration) (net.Conn, error) {
	sess, err := kcp.Dial(host)
	if err != nil {
		return nil, err
	}
	sess.SetMtu(1350)
	sess.SetNoDelay(1, 10, 2, 1)
	return sess, nil
}

func (UdpChanFactory) NewClient(f *flow.Flow, session *packet.Session, conn net.Conn, out packet.SendChan) Channel {
	return NewHttpChanClient(f, session, conn, out)
}

func (UdpChanFactory) NewServer(f *flow.Flow, session *packet.Session, conn net.Conn, delegate SvrInitDelegate) Channel {
	return NewHttpChanServer(f, session, conn, delegate)
}
<commit_msg>update window size<commit_after>package dchan

import (
	"net"
	"time"

	"github.com/chzyer/flow"
	"github.com/chzyer/next/packet"
	kcp "github.com/xtaci/kcp-go"
)

type UdpChanFactory struct {
	port int32
}

func NewUdpChanFactory() *UdpChanFactory {
	return &UdpChanFactory{
		port: 10000,
	}
}

type wrapLn struct {
	*kcp.Listener
}

func (w *wrapLn) Accept() (net.Conn, error) {
	sess, err := w.Listener.Accept()
	if err != nil {
		return nil, err
	}
	sess.SetMtu(1350)
	sess.SetNoDelay(1, 10, 2, 1)
	sess.SetWindowSize(2048, 2048)
	return sess, nil
}

func (u *UdpChanFactory) Listen(f *flow.Flow) (net.Listener, error) {
	ln, err := kcp.Listen(":0")
	if err != nil {
		return nil, err
	}
	return &wrapLn{ln}, nil
}

func (UdpChanFactory) DialTimeout(host string, timeout time.Duration) (net.Conn, error) {
	sess, err := kcp.Dial(host)
	if err != nil {
		return nil, err
	}
	sess.SetMtu(1350)
	sess.SetNoDelay(1, 10, 2, 1)
	sess.SetWindowSize(256, 2048)
	return sess, nil
}

func (UdpChanFactory) NewClient(f *flow.Flow, session *packet.Session, conn net.Conn, out packet.SendChan) Channel {
	return NewHttpChanClient(f, session, conn, out)
}

func (UdpChanFactory) NewServer(f *flow.Flow, session *packet.Session, conn net.Conn, delegate SvrInitDelegate) Channel {
	return NewHttpChanServer(f, session, conn, delegate)
}
<|endoftext|>