<commit_before>package benchmarkbbs_test

import (
	"context"
	"crypto/rand"
	"database/sql"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"runtime"
	"strings"
	"time"

	"code.cloudfoundry.org/bbs"
	"code.cloudfoundry.org/bbs/db"
	"code.cloudfoundry.org/bbs/db/sqldb"
	"code.cloudfoundry.org/bbs/encryption"
	"code.cloudfoundry.org/bbs/format"
	"code.cloudfoundry.org/bbs/guidprovider"
	"code.cloudfoundry.org/bbs/models"
	benchmarkconfig "code.cloudfoundry.org/benchmarkbbs/config"
	"code.cloudfoundry.org/benchmarkbbs/generator"
	"code.cloudfoundry.org/benchmarkbbs/reporter"
	"code.cloudfoundry.org/cfhttp"
	"code.cloudfoundry.org/clock"
	fakes "code.cloudfoundry.org/diego-logging-client/testhelpers"
	"code.cloudfoundry.org/executor"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/locket"
	locketmodels "code.cloudfoundry.org/locket/models"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/zorkian/go-datadog-api"

	_ "github.com/go-sql-driver/mysql"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"testing"
)

var (
	expectedLRPCount int

	expectedActualLRPCounts map[string]int
	config                  benchmarkconfig.BenchmarkBBSConfig

	logger          lager.Logger
	locketClient    locketmodels.LocketClient
	sqlDB           *sqldb.SQLDB
	activeDB        db.DB
	bbsClient       bbs.InternalClient
	dataDogClient   *datadog.Client
	dataDogReporter reporter.DataDogReporter
	reporters       []Reporter
)

const (
	DEBUG = "debug"
	INFO  = "info"
	ERROR = "error"
	FATAL = "fatal"
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	configFile := flag.String("config", "", "config file")
	flag.Parse()

	var err error
	config, err = benchmarkconfig.NewBenchmarkBBSConfig(*configFile)
	if err != nil {
		panic(err)
	}

	if config.BBSAddress == "" {
		log.Fatal("bbsAddress is required")
	}

	BenchmarkTests(config.NumReps, config.NumTrials, config.LocalRouteEmitters)
}

func TestBenchmarkBbs(t *testing.T) {
	var lagerLogLevel lager.LogLevel
	switch config.LogLevel {
	case DEBUG:
		lagerLogLevel = lager.DEBUG
	case INFO:
		lagerLogLevel = lager.INFO
	case ERROR:
		lagerLogLevel = lager.ERROR
	case FATAL:
		lagerLogLevel = lager.FATAL
	default:
		panic(fmt.Errorf("unknown log level: %s", config.LogLevel))
	}

	var logWriter io.Writer
	if config.LogFilename == "" {
		logWriter = GinkgoWriter
	} else {
		logFile, err := os.OpenFile(config.LogFilename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
		if err != nil {
			panic("Error opening file, err: " + err.Error())
		}
		defer logFile.Close()

		logWriter = logFile
	}

	logger = lager.NewLogger("bbs-benchmarks-test")
	logger.RegisterSink(lager.NewWriterSink(logWriter, lagerLogLevel))

	reporters = []Reporter{}

	if config.DataDogAPIKey != "" && config.DataDogAppKey != "" {
		dataDogClient = datadog.NewClient(config.DataDogAPIKey, config.DataDogAppKey)
		dataDogReporter = reporter.NewDataDogReporter(logger, config.MetricPrefix, dataDogClient)
		reporters = append(reporters, &dataDogReporter)
	}

	if config.AwsAccessKeyID != "" && config.AwsSecretAccessKey != "" && config.AwsBucketName != "" {
		creds := credentials.NewStaticCredentials(config.AwsAccessKeyID, config.AwsSecretAccessKey, "")
		s3Client := s3.New(&aws.Config{
			Region:      &config.AwsRegion,
			Credentials: creds,
		})
		uploader := s3manager.NewUploader(&s3manager.UploadOptions{S3: s3Client})
		reporter := reporter.NewS3Reporter(logger, config.AwsBucketName, uploader)
		reporters = append(reporters, &reporter)
	}

	RegisterFailHandler(Fail)
	RunSpecsWithDefaultAndCustomReporters(t, "Benchmark BBS Suite", reporters)
}

type expectedLRPCounts struct {
	DesiredLRPCount int

	ActualLRPCounts map[string]int
}

func initializeActiveDB() *sql.DB {
	if activeDB != nil {
		return nil
	}

	if config.DatabaseConnectionString == "" {
		logger.Fatal("no-sql-configuration", errors.New("no-sql-configuration"))
	}

	if config.DatabaseDriver == "postgres" && !strings.Contains(config.DatabaseConnectionString, "sslmode") {
		config.DatabaseConnectionString = fmt.Sprintf("%s?sslmode=disable", config.DatabaseConnectionString)
	}

	sqlConn, err := sql.Open(config.DatabaseDriver, config.DatabaseConnectionString)
	if err != nil {
		logger.Fatal("failed-to-open-sql", err)
	}
	sqlConn.SetMaxOpenConns(1)
	sqlConn.SetMaxIdleConns(1)

	err = sqlConn.Ping()
	Expect(err).NotTo(HaveOccurred())

	sqlDB = initializeSQLDB(logger, sqlConn)
	activeDB = sqlDB
	return sqlConn
}

var _ = BeforeSuite(func() {
	bbsClient = initializeBBSClient(logger, time.Duration(config.BBSClientHTTPTimeout))

	conn := initializeActiveDB()
	initializeLocketClient()

	resp, err := locketClient.FetchAll(context.Background(), &locketmodels.FetchAllRequest{TypeCode: locketmodels.PRESENCE})
	Expect(err).NotTo(HaveOccurred())

	cells := make(map[string]struct{})
	for i := 0; i < config.NumReps; i++ {
		cells[fmt.Sprintf("cell-%d", i)] = struct{}{}
	}

	if config.ReseedDatabase || config.NumReps != len(resp.Resources) {
		var err error

		cleanupSQLDB(conn)
		resetLocketDB(locketClient, cells, resp)

		if config.DesiredLRPs > 0 {
			desiredLRPGenerator := generator.NewDesiredLRPGenerator(
				config.ErrorTolerance,
				config.MetricPrefix,
				config.NumPopulateWorkers,
				bbsClient,
				dataDogClient,
			)

			expectedLRPCount, expectedActualLRPCounts, err = desiredLRPGenerator.Generate(logger, config.NumReps, config.DesiredLRPs)
			Expect(err).NotTo(HaveOccurred())
		}
	} else {
		resetUnclaimedActualLRPs(conn, cells)

		query := `
			SELECT
				COUNT(*)
			FROM desired_lrps
		`
		res := conn.QueryRow(query)
		err := res.Scan(&expectedLRPCount)
		Expect(err).NotTo(HaveOccurred())

		expectedActualLRPCounts = make(map[string]int)
		query = `
			SELECT
				cell_id, COUNT(*)
			FROM actual_lrps
			GROUP BY cell_id
		`
		rows, err := conn.Query(query)
		Expect(err).NotTo(HaveOccurred())

		defer rows.Close()
		for rows.Next() {
			var cellId string
			var count int
			err = rows.Scan(&cellId, &count)
			Expect(err).NotTo(HaveOccurred())

			expectedActualLRPCounts[cellId] = count
		}
	}

	if float64(expectedLRPCount) < float64(config.DesiredLRPs)*config.ErrorTolerance {
		Fail(fmt.Sprintf("Error rate of %.3f for actuals exceeds tolerance of %.3f", float64(expectedLRPCount)/float64(config.DesiredLRPs), config.ErrorTolerance))
	}
})

func initializeSQLDB(logger lager.Logger, sqlConn *sql.DB) *sqldb.SQLDB {
	key, keys, err := config.EncryptionConfig.Parse()
	if err != nil {
		logger.Fatal("cannot-setup-encryption", err)
	}
	keyManager, err := encryption.NewKeyManager(key, keys)
	if err != nil {
		logger.Fatal("cannot-setup-encryption", err)
	}
	cryptor := encryption.NewCryptor(keyManager, rand.Reader)

	return sqldb.NewSQLDB(
		sqlConn,
		1000,
		1000,
		format.ENCODED_PROTO,
		cryptor,
		guidprovider.DefaultGuidProvider,
		clock.NewClock(),
		config.DatabaseDriver,
		&fakes.FakeIngressClient{},
	)
}

func initializeLocketClient() {
	var err error
	if config.SkipCertVerify {
		locketClient, err = locket.NewClientSkipCertVerify(logger, config.ClientLocketConfig)
	} else {
		locketClient, err = locket.NewClient(logger, config.ClientLocketConfig)
	}
	Expect(err).NotTo(HaveOccurred())
}

func initializeBBSClient(logger lager.Logger, bbsClientHTTPTimeout time.Duration) bbs.InternalClient {
	bbsURL, err := url.Parse(config.BBSAddress)
	if err != nil {
		logger.Fatal("Invalid BBS URL", err)
	}

	if bbsURL.Scheme != "https" {
		return bbs.NewClient(config.BBSAddress)
	}

	cfhttp.Initialize(bbsClientHTTPTimeout)
	var bbsClient bbs.InternalClient
	if config.SkipCertVerify {
		bbsClient, err = bbs.NewSecureSkipVerifyClient(
			config.BBSAddress,
			config.BBSClientCert,
			config.BBSClientKey,
			1,
			25000,
		)
	} else {
		bbsClient, err = bbs.NewSecureClient(
			config.BBSAddress,
			config.BBSCACert,
			config.BBSClientCert,
			config.BBSClientKey,
			1,
			25000,
		)
	}
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}

func cleanupSQLDB(conn *sql.DB) {
	_, err := conn.Exec("TRUNCATE actual_lrps")
	Expect(err).NotTo(HaveOccurred())
	_, err = conn.Exec("TRUNCATE desired_lrps")
	Expect(err).NotTo(HaveOccurred())
}

func resetLocketDB(locketClient locketmodels.LocketClient, cells map[string]struct{}, resp *locketmodels.FetchAllResponse) {
	for _, resource := range resp.Resources {
		if _, ok := cells[resource.Owner]; !ok {
			_, err := locketClient.Release(context.Background(), &locketmodels.ReleaseRequest{Resource: resource})
			Expect(err).NotTo(HaveOccurred())
		}
	}

	for cellId, _ := range cells {
		_, err := locketClient.Lock(context.Background(), &locketmodels.LockRequest{Resource: lockResource(cellId), TtlInSeconds: longTermTtl})
		Expect(err).NotTo(HaveOccurred())
	}
}

func lockResource(cellID string) *locketmodels.Resource {
	resources := executor.ExecutorResources{
		MemoryMB:   1000,
		DiskMB:     2000,
		Containers: 300,
	}
	cellCapacity := models.NewCellCapacity(int32(resources.MemoryMB), int32(resources.DiskMB), int32(resources.Containers))
	cellPresence := models.NewCellPresence(cellID, cellID+".address", cellID+".url",
		"z1", cellCapacity, []string{"providers"},
		[]string{"cflinuxfs9"}, []string{}, []string{})

	payload, err := json.Marshal(cellPresence)
	Expect(err).NotTo(HaveOccurred())

	return &locketmodels.Resource{
		Key:   cellID,
		Owner: cellID,
		Value: string(payload),
		Type:  locketmodels.PresenceType,
	}
}

func resetUnclaimedActualLRPs(conn *sql.DB, cells map[string]struct{}) {
	existingCellIds := make(map[string]struct{})

	query := `
			SELECT
				cell_id
			FROM actual_lrps
			GROUP BY cell_id
		`
	rows, err := conn.Query(query)
	Expect(err).NotTo(HaveOccurred())

	defer rows.Close()
	for rows.Next() {
		var cellId string
		err = rows.Scan(&cellId)
		Expect(err).NotTo(HaveOccurred())

		existingCellIds[cellId] = struct{}{}
	}

	missingCells := findMissingCells(existingCellIds, cells)

	query = `
				SELECT
					process_guid, domain
				FROM actual_lrps WHERE cell_id = ''
			`

	type guidAndDomain struct {
		ProcessGuid string
		Domain      string
	}
	var lrps []guidAndDomain

	rows, err = conn.Query(query)
	Expect(err).NotTo(HaveOccurred())

	defer rows.Close()
	for rows.Next() {
		var guid, domain string
		err = rows.Scan(&guid, &domain)
		Expect(err).NotTo(HaveOccurred())
		lrps = append(lrps, guidAndDomain{ProcessGuid: guid, Domain: domain})
	}

	for i, lrp := range lrps {
		cellID := missingCells[i%len(missingCells)]
		actualLRPInstanceKey := &models.ActualLRPInstanceKey{InstanceGuid: lrp.ProcessGuid + "-i", CellId: cellID}
		netInfo := models.NewActualLRPNetInfo("1.2.3.4", "2.2.2.2", models.NewPortMapping(61999, 8080))
		err := bbsClient.StartActualLRP(logger, &models.ActualLRPKey{Domain: lrp.Domain, ProcessGuid: lrp.ProcessGuid, Index: 0}, actualLRPInstanceKey, &netInfo)
		Expect(err).NotTo(HaveOccurred())
	}
}

func findMissingCells(existing map[string]struct{}, expected map[string]struct{}) []string {
	var missing []string
	for cell, _ := range expected {
		if _, ok := existing[cell]; !ok {
			missing = append(missing, cell)
		}
	}

	return missing
}
<commit_msg>update benchmarkbbs with new SQL helpers interfaces<commit_after>package benchmarkbbs_test

import (
	"context"
	"crypto/rand"
	"database/sql"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"runtime"
	"strings"
	"time"

	"code.cloudfoundry.org/bbs"
	"code.cloudfoundry.org/bbs/db"
	"code.cloudfoundry.org/bbs/db/sqldb"
	"code.cloudfoundry.org/bbs/db/sqldb/helpers"
	"code.cloudfoundry.org/bbs/encryption"
	"code.cloudfoundry.org/bbs/format"
	"code.cloudfoundry.org/bbs/guidprovider"
	"code.cloudfoundry.org/bbs/models"
	benchmarkconfig "code.cloudfoundry.org/benchmarkbbs/config"
	"code.cloudfoundry.org/benchmarkbbs/generator"
	"code.cloudfoundry.org/benchmarkbbs/reporter"
	"code.cloudfoundry.org/cfhttp"
	"code.cloudfoundry.org/clock"
	fakes "code.cloudfoundry.org/diego-logging-client/testhelpers"
	"code.cloudfoundry.org/executor"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/locket"
	locketmodels "code.cloudfoundry.org/locket/models"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/zorkian/go-datadog-api"

	_ "github.com/go-sql-driver/mysql"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"testing"
)

var (
	expectedLRPCount int

	expectedActualLRPCounts map[string]int
	config                  benchmarkconfig.BenchmarkBBSConfig

	logger          lager.Logger
	locketClient    locketmodels.LocketClient
	sqlDB           *sqldb.SQLDB
	activeDB        db.DB
	bbsClient       bbs.InternalClient
	dataDogClient   *datadog.Client
	dataDogReporter reporter.DataDogReporter
	reporters       []Reporter
)

const (
	DEBUG = "debug"
	INFO  = "info"
	ERROR = "error"
	FATAL = "fatal"
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	configFile := flag.String("config", "", "config file")
	flag.Parse()

	var err error
	config, err = benchmarkconfig.NewBenchmarkBBSConfig(*configFile)
	if err != nil {
		panic(err)
	}

	if config.BBSAddress == "" {
		log.Fatal("bbsAddress is required")
	}

	BenchmarkTests(config.NumReps, config.NumTrials, config.LocalRouteEmitters)
}

func TestBenchmarkBbs(t *testing.T) {
	var lagerLogLevel lager.LogLevel
	switch config.LogLevel {
	case DEBUG:
		lagerLogLevel = lager.DEBUG
	case INFO:
		lagerLogLevel = lager.INFO
	case ERROR:
		lagerLogLevel = lager.ERROR
	case FATAL:
		lagerLogLevel = lager.FATAL
	default:
		panic(fmt.Errorf("unknown log level: %s", config.LogLevel))
	}

	var logWriter io.Writer
	if config.LogFilename == "" {
		logWriter = GinkgoWriter
	} else {
		logFile, err := os.OpenFile(config.LogFilename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
		if err != nil {
			panic("Error opening file, err: " + err.Error())
		}
		defer logFile.Close()

		logWriter = logFile
	}

	logger = lager.NewLogger("bbs-benchmarks-test")
	logger.RegisterSink(lager.NewWriterSink(logWriter, lagerLogLevel))

	reporters = []Reporter{}

	if config.DataDogAPIKey != "" && config.DataDogAppKey != "" {
		dataDogClient = datadog.NewClient(config.DataDogAPIKey, config.DataDogAppKey)
		dataDogReporter = reporter.NewDataDogReporter(logger, config.MetricPrefix, dataDogClient)
		reporters = append(reporters, &dataDogReporter)
	}

	if config.AwsAccessKeyID != "" && config.AwsSecretAccessKey != "" && config.AwsBucketName != "" {
		creds := credentials.NewStaticCredentials(config.AwsAccessKeyID, config.AwsSecretAccessKey, "")
		s3Client := s3.New(&aws.Config{
			Region:      &config.AwsRegion,
			Credentials: creds,
		})
		uploader := s3manager.NewUploader(&s3manager.UploadOptions{S3: s3Client})
		reporter := reporter.NewS3Reporter(logger, config.AwsBucketName, uploader)
		reporters = append(reporters, &reporter)
	}

	RegisterFailHandler(Fail)
	RunSpecsWithDefaultAndCustomReporters(t, "Benchmark BBS Suite", reporters)
}

type expectedLRPCounts struct {
	DesiredLRPCount int

	ActualLRPCounts map[string]int
}

func initializeActiveDB() *sql.DB {
	if activeDB != nil {
		return nil
	}

	if config.DatabaseConnectionString == "" {
		logger.Fatal("no-sql-configuration", errors.New("no-sql-configuration"))
	}

	if config.DatabaseDriver == "postgres" && !strings.Contains(config.DatabaseConnectionString, "sslmode") {
		config.DatabaseConnectionString = fmt.Sprintf("%s?sslmode=disable", config.DatabaseConnectionString)
	}

	sqlConn, err := sql.Open(config.DatabaseDriver, config.DatabaseConnectionString)
	if err != nil {
		logger.Fatal("failed-to-open-sql", err)
	}
	sqlConn.SetMaxOpenConns(1)
	sqlConn.SetMaxIdleConns(1)

	err = sqlConn.Ping()
	Expect(err).NotTo(HaveOccurred())

	wrappedDB := helpers.NewMonitoredDB(sqlConn, helpers.NewQueryMonitor())
	sqlDB = initializeSQLDB(logger, wrappedDB)
	activeDB = sqlDB
	return sqlConn
}

var _ = BeforeSuite(func() {
	bbsClient = initializeBBSClient(logger, time.Duration(config.BBSClientHTTPTimeout))

	conn := initializeActiveDB()
	initializeLocketClient()

	resp, err := locketClient.FetchAll(context.Background(), &locketmodels.FetchAllRequest{TypeCode: locketmodels.PRESENCE})
	Expect(err).NotTo(HaveOccurred())

	cells := make(map[string]struct{})
	for i := 0; i < config.NumReps; i++ {
		cells[fmt.Sprintf("cell-%d", i)] = struct{}{}
	}

	if config.ReseedDatabase || config.NumReps != len(resp.Resources) {
		var err error

		cleanupSQLDB(conn)
		resetLocketDB(locketClient, cells, resp)

		if config.DesiredLRPs > 0 {
			desiredLRPGenerator := generator.NewDesiredLRPGenerator(
				config.ErrorTolerance,
				config.MetricPrefix,
				config.NumPopulateWorkers,
				bbsClient,
				dataDogClient,
			)

			expectedLRPCount, expectedActualLRPCounts, err = desiredLRPGenerator.Generate(logger, config.NumReps, config.DesiredLRPs)
			Expect(err).NotTo(HaveOccurred())
		}
	} else {
		resetUnclaimedActualLRPs(conn, cells)

		query := `
			SELECT
				COUNT(*)
			FROM desired_lrps
		`
		res := conn.QueryRow(query)
		err := res.Scan(&expectedLRPCount)
		Expect(err).NotTo(HaveOccurred())

		expectedActualLRPCounts = make(map[string]int)
		query = `
			SELECT
				cell_id, COUNT(*)
			FROM actual_lrps
			GROUP BY cell_id
		`
		rows, err := conn.Query(query)
		Expect(err).NotTo(HaveOccurred())

		defer rows.Close()
		for rows.Next() {
			var cellId string
			var count int
			err = rows.Scan(&cellId, &count)
			Expect(err).NotTo(HaveOccurred())

			expectedActualLRPCounts[cellId] = count
		}
	}

	if float64(expectedLRPCount) < float64(config.DesiredLRPs)*config.ErrorTolerance {
		Fail(fmt.Sprintf("Error rate of %.3f for actuals exceeds tolerance of %.3f", float64(expectedLRPCount)/float64(config.DesiredLRPs), config.ErrorTolerance))
	}
})

func initializeSQLDB(logger lager.Logger, sqlConn helpers.DB) *sqldb.SQLDB {
	key, keys, err := config.EncryptionConfig.Parse()
	if err != nil {
		logger.Fatal("cannot-setup-encryption", err)
	}
	keyManager, err := encryption.NewKeyManager(key, keys)
	if err != nil {
		logger.Fatal("cannot-setup-encryption", err)
	}
	cryptor := encryption.NewCryptor(keyManager, rand.Reader)

	return sqldb.NewSQLDB(
		sqlConn,
		1000,
		1000,
		format.ENCODED_PROTO,
		cryptor,
		guidprovider.DefaultGuidProvider,
		clock.NewClock(),
		config.DatabaseDriver,
		&fakes.FakeIngressClient{},
	)
}

func initializeLocketClient() {
	var err error
	if config.SkipCertVerify {
		locketClient, err = locket.NewClientSkipCertVerify(logger, config.ClientLocketConfig)
	} else {
		locketClient, err = locket.NewClient(logger, config.ClientLocketConfig)
	}
	Expect(err).NotTo(HaveOccurred())
}

func initializeBBSClient(logger lager.Logger, bbsClientHTTPTimeout time.Duration) bbs.InternalClient {
	bbsURL, err := url.Parse(config.BBSAddress)
	if err != nil {
		logger.Fatal("Invalid BBS URL", err)
	}

	if bbsURL.Scheme != "https" {
		return bbs.NewClient(config.BBSAddress)
	}

	cfhttp.Initialize(bbsClientHTTPTimeout)
	var bbsClient bbs.InternalClient
	if config.SkipCertVerify {
		bbsClient, err = bbs.NewSecureSkipVerifyClient(
			config.BBSAddress,
			config.BBSClientCert,
			config.BBSClientKey,
			1,
			25000,
		)
	} else {
		bbsClient, err = bbs.NewSecureClient(
			config.BBSAddress,
			config.BBSCACert,
			config.BBSClientCert,
			config.BBSClientKey,
			1,
			25000,
		)
	}
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}

func cleanupSQLDB(conn *sql.DB) {
	_, err := conn.Exec("TRUNCATE actual_lrps")
	Expect(err).NotTo(HaveOccurred())
	_, err = conn.Exec("TRUNCATE desired_lrps")
	Expect(err).NotTo(HaveOccurred())
}

func resetLocketDB(locketClient locketmodels.LocketClient, cells map[string]struct{}, resp *locketmodels.FetchAllResponse) {
	for _, resource := range resp.Resources {
		if _, ok := cells[resource.Owner]; !ok {
			_, err := locketClient.Release(context.Background(), &locketmodels.ReleaseRequest{Resource: resource})
			Expect(err).NotTo(HaveOccurred())
		}
	}

	for cellId, _ := range cells {
		_, err := locketClient.Lock(context.Background(), &locketmodels.LockRequest{Resource: lockResource(cellId), TtlInSeconds: longTermTtl})
		Expect(err).NotTo(HaveOccurred())
	}
}

func lockResource(cellID string) *locketmodels.Resource {
	resources := executor.ExecutorResources{
		MemoryMB:   1000,
		DiskMB:     2000,
		Containers: 300,
	}
	cellCapacity := models.NewCellCapacity(int32(resources.MemoryMB), int32(resources.DiskMB), int32(resources.Containers))
	cellPresence := models.NewCellPresence(cellID, cellID+".address", cellID+".url",
		"z1", cellCapacity, []string{"providers"},
		[]string{"cflinuxfs9"}, []string{}, []string{})

	payload, err := json.Marshal(cellPresence)
	Expect(err).NotTo(HaveOccurred())

	return &locketmodels.Resource{
		Key:   cellID,
		Owner: cellID,
		Value: string(payload),
		Type:  locketmodels.PresenceType,
	}
}

func resetUnclaimedActualLRPs(conn *sql.DB, cells map[string]struct{}) {
	existingCellIds := make(map[string]struct{})

	query := `
			SELECT
				cell_id
			FROM actual_lrps
			GROUP BY cell_id
		`
	rows, err := conn.Query(query)
	Expect(err).NotTo(HaveOccurred())

	defer rows.Close()
	for rows.Next() {
		var cellId string
		err = rows.Scan(&cellId)
		Expect(err).NotTo(HaveOccurred())

		existingCellIds[cellId] = struct{}{}
	}

	missingCells := findMissingCells(existingCellIds, cells)

	query = `
				SELECT
					process_guid, domain
				FROM actual_lrps WHERE cell_id = ''
			`

	type guidAndDomain struct {
		ProcessGuid string
		Domain      string
	}
	var lrps []guidAndDomain

	rows, err = conn.Query(query)
	Expect(err).NotTo(HaveOccurred())

	defer rows.Close()
	for rows.Next() {
		var guid, domain string
		err = rows.Scan(&guid, &domain)
		Expect(err).NotTo(HaveOccurred())
		lrps = append(lrps, guidAndDomain{ProcessGuid: guid, Domain: domain})
	}

	for i, lrp := range lrps {
		cellID := missingCells[i%len(missingCells)]
		actualLRPInstanceKey := &models.ActualLRPInstanceKey{InstanceGuid: lrp.ProcessGuid + "-i", CellId: cellID}
		netInfo := models.NewActualLRPNetInfo("1.2.3.4", "2.2.2.2", models.NewPortMapping(61999, 8080))
		err := bbsClient.StartActualLRP(logger, &models.ActualLRPKey{Domain: lrp.Domain, ProcessGuid: lrp.ProcessGuid, Index: 0}, actualLRPInstanceKey, &netInfo)
		Expect(err).NotTo(HaveOccurred())
	}
}

func findMissingCells(existing map[string]struct{}, expected map[string]struct{}) []string {
	var missing []string
	for cell, _ := range expected {
		if _, ok := existing[cell]; !ok {
			missing = append(missing, cell)
		}
	}

	return missing
}
<|endoftext|>


<commit_before>package benchmark_bbs_test

import (
	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"strings"

	"github.com/cloudfoundry-incubator/bbs"
	etcddb "github.com/cloudfoundry-incubator/bbs/db/etcd"
	"github.com/cloudfoundry-incubator/bbs/encryption"
	"github.com/cloudfoundry-incubator/benchmark-bbs/generator"
	"github.com/cloudfoundry-incubator/cf-lager"
	"github.com/coreos/go-etcd/etcd"
	"github.com/pivotal-golang/lager"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"testing"
)

var bbsAddress string
var bbsClientCert string
var bbsClientKey string
var etcdFlags *ETCDFlags
var desiredLRPs int
var actualLRPs int
var encryptionFlags *encryption.EncryptionFlags

var etcdClient *etcd.Client
var bbsClient bbs.Client
var logger lager.Logger

func init() {
	flag.StringVar(&bbsAddress, "bbsAddress", "", "Address of the BBS Server")
	flag.StringVar(&bbsClientCert, "bbsClientCert", "", "bbs client ssl certificate")
	flag.StringVar(&bbsClientKey, "bbsClientKey", "", "bbs client ssl key")

	flag.IntVar(&desiredLRPs, "desiredLRPs", 0, "number of DesiredLRPs to create")
	flag.IntVar(&actualLRPs, "actualLRPs", 0, "number of ActualLRPs to create")

	cf_lager.AddFlags(flag.CommandLine)
	etcdFlags = AddETCDFlags(flag.CommandLine)
	encryptionFlags = encryption.AddEncryptionFlags(flag.CommandLine)
}

func TestBenchmarkBbs(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "BenchmarkBbs Suite")
}

var _ = BeforeSuite(func() {
	etcdOptions, err := etcdFlags.Validate()
	Expect(err).NotTo(HaveOccurred())

	logger = lager.NewLogger("test")
	logger.RegisterSink(lager.NewWriterSink(GinkgoWriter, lager.DEBUG))

	etcdClient = initializeEtcdClient(logger, etcdOptions)
	bbsClient = initializeBBSClient(logger)

	cleanUpDesiredLRPs()
	cleanUpActualLRPs()

	_, err = bbsClient.Domains()
	Expect(err).NotTo(HaveOccurred())

	if desiredLRPs > 0 {
		desiredLRPGenerator := generator.NewDesiredLRPGenerator(logger, bbsClient, *etcdClient)
		err := desiredLRPGenerator.Generate(desiredLRPs)
		Expect(err).NotTo(HaveOccurred())
	}

	if actualLRPs > 0 {
		actualLRPGenerator := generator.NewActualLRPGenerator(logger, bbsClient, *etcdClient)
		err := actualLRPGenerator.Generate(actualLRPs)
		Expect(err).NotTo(HaveOccurred())
	}
})

var _ = AfterSuite(func() {
	cleanUpDesiredLRPs()
	cleanUpActualLRPs()
})

type ETCDFlags struct {
	etcdCertFile           string
	etcdKeyFile            string
	etcdCaFile             string
	clusterUrls            string
	clientSessionCacheSize int
	maxIdleConnsPerHost    int
}

func AddETCDFlags(flagSet *flag.FlagSet) *ETCDFlags {
	flags := &ETCDFlags{}

	flagSet.StringVar(
		&flags.clusterUrls,
		"etcdCluster",
		"http://127.0.0.1:4001",
		"comma-separated list of etcd URLs (scheme://ip:port)",
	)
	flagSet.StringVar(
		&flags.etcdCertFile,
		"etcdCertFile",
		"",
		"Location of the client certificate for mutual auth",
	)
	flagSet.StringVar(
		&flags.etcdKeyFile,
		"etcdKeyFile",
		"",
		"Location of the client key for mutual auth",
	)
	flagSet.StringVar(
		&flags.etcdCaFile,
		"etcdCaFile",
		"",
		"Location of the CA certificate for mutual auth",
	)

	flagSet.IntVar(
		&flags.clientSessionCacheSize,
		"etcdSessionCacheSize",
		0,
		"Capacity of the ClientSessionCache option on the TLS configuration. If zero, golang's default will be used",
	)
	flagSet.IntVar(
		&flags.maxIdleConnsPerHost,
		"etcdMaxIdleConnsPerHost",
		0,
		"Controls the maximum number of idle (keep-alive) connctions per host. If zero, golang's default will be used",
	)
	return flags
}

func (flags *ETCDFlags) Validate() (*etcddb.ETCDOptions, error) {
	scheme := ""
	clusterUrls := strings.Split(flags.clusterUrls, ",")
	for i, uString := range clusterUrls {
		uString = strings.TrimSpace(uString)
		clusterUrls[i] = uString
		u, err := url.Parse(uString)
		if err != nil {
			return nil, fmt.Errorf("Invalid cluster URL: '%s', error: [%s]", uString, err.Error())
		}
		if scheme == "" {
			if u.Scheme != "http" && u.Scheme != "https" {
				return nil, errors.New("Invalid scheme: " + uString)
			}
			scheme = u.Scheme
		} else if scheme != u.Scheme {
			return nil, fmt.Errorf("Multiple url schemes provided: %s", flags.clusterUrls)
		}
	}

	isSSL := false
	if scheme == "https" {
		isSSL = true
		if flags.etcdCertFile == "" {
			return nil, errors.New("Cert file must be provided for https connections")
		}
		if flags.etcdKeyFile == "" {
			return nil, errors.New("Key file must be provided for https connections")
		}
	}

	return &etcddb.ETCDOptions{
		CertFile:    flags.etcdCertFile,
		KeyFile:     flags.etcdKeyFile,
		CAFile:      flags.etcdCaFile,
		ClusterUrls: clusterUrls,
		IsSSL:       isSSL,
		ClientSessionCacheSize: flags.clientSessionCacheSize,
		MaxIdleConnsPerHost:    flags.maxIdleConnsPerHost,
	}, nil
}

func initializeEtcdClient(logger lager.Logger, etcdOptions *etcddb.ETCDOptions) *etcd.Client {
	var etcdClient *etcd.Client
	var tr *http.Transport

	if etcdOptions.IsSSL {
		if etcdOptions.CertFile == "" || etcdOptions.KeyFile == "" {
			logger.Fatal("failed-to-construct-etcd-tls-client", errors.New("Require both cert and key path"))
		}

		var err error
		etcdClient, err = etcd.NewTLSClient(etcdOptions.ClusterUrls, etcdOptions.CertFile, etcdOptions.KeyFile, etcdOptions.CAFile)
		if err != nil {
			logger.Fatal("failed-to-construct-etcd-tls-client", err)
		}

		tlsCert, err := tls.LoadX509KeyPair(etcdOptions.CertFile, etcdOptions.KeyFile)
		if err != nil {
			logger.Fatal("failed-to-construct-etcd-tls-client", err)
		}

		tlsConfig := &tls.Config{
			Certificates:       []tls.Certificate{tlsCert},
			InsecureSkipVerify: true,
			ClientSessionCache: tls.NewLRUClientSessionCache(etcdOptions.ClientSessionCacheSize),
		}
		tr = &http.Transport{
			TLSClientConfig:     tlsConfig,
			Dial:                etcdClient.DefaultDial,
			MaxIdleConnsPerHost: etcdOptions.MaxIdleConnsPerHost,
		}
		etcdClient.SetTransport(tr)
	} else {
		etcdClient = etcd.NewClient(etcdOptions.ClusterUrls)
	}
	etcdClient.SetConsistency(etcd.STRONG_CONSISTENCY)

	return etcdClient
}

func initializeBBSClient(logger lager.Logger) bbs.Client {
	if bbsAddress == "" {
		log.Fatal("bbsAddress is required")
	}

	bbsURL, err := url.Parse(bbsAddress)
	if err != nil {
		logger.Fatal("Invalid BBS URL", err)
	}

	if bbsURL.Scheme != "https" {
		return bbs.NewClient(bbsAddress)
	}

	bbsClient, err := bbs.NewSecureSkipVerifyClient(bbsAddress, bbsClientCert, bbsClientKey, 1, 1)
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}

func cleanUpDesiredLRPs() {
	_, err := etcdClient.Delete("/v1/desired_lrp/", true)
	Expect(err).ToNot(HaveOccurred())
}

func cleanUpActualLRPs() {
	_, err := etcdClient.Delete("/v1/actual/", true)
	Expect(err).ToNot(HaveOccurred())
}
<commit_msg>clean up methods -> purge method<commit_after>package benchmark_bbs_test

import (
	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"strings"

	"github.com/cloudfoundry-incubator/bbs"
	etcddb "github.com/cloudfoundry-incubator/bbs/db/etcd"
	"github.com/cloudfoundry-incubator/bbs/encryption"
	"github.com/cloudfoundry-incubator/benchmark-bbs/generator"
	"github.com/cloudfoundry-incubator/cf-lager"
	"github.com/coreos/go-etcd/etcd"
	"github.com/pivotal-golang/lager"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"testing"
)

var bbsAddress string
var bbsClientCert string
var bbsClientKey string
var etcdFlags *ETCDFlags
var desiredLRPs int
var actualLRPs int
var encryptionFlags *encryption.EncryptionFlags

var etcdClient *etcd.Client
var bbsClient bbs.Client
var logger lager.Logger

func init() {
	flag.StringVar(&bbsAddress, "bbsAddress", "", "Address of the BBS Server")
	flag.StringVar(&bbsClientCert, "bbsClientCert", "", "bbs client ssl certificate")
	flag.StringVar(&bbsClientKey, "bbsClientKey", "", "bbs client ssl key")

	flag.IntVar(&desiredLRPs, "desiredLRPs", 0, "number of DesiredLRPs to create")
	flag.IntVar(&actualLRPs, "actualLRPs", 0, "number of ActualLRPs to create")

	cf_lager.AddFlags(flag.CommandLine)
	etcdFlags = AddETCDFlags(flag.CommandLine)
	encryptionFlags = encryption.AddEncryptionFlags(flag.CommandLine)
}

func TestBenchmarkBbs(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "BenchmarkBbs Suite")
}

var _ = BeforeSuite(func() {
	etcdOptions, err := etcdFlags.Validate()
	Expect(err).NotTo(HaveOccurred())

	logger = lager.NewLogger("test")
	logger.RegisterSink(lager.NewWriterSink(GinkgoWriter, lager.DEBUG))

	etcdClient = initializeEtcdClient(logger, etcdOptions)
	bbsClient = initializeBBSClient(logger)

	purge("/v1/desired_lrp")
	purge("/v1/actual")

	_, err = bbsClient.Domains()
	Expect(err).NotTo(HaveOccurred())

	if desiredLRPs > 0 {
		desiredLRPGenerator := generator.NewDesiredLRPGenerator(logger, bbsClient, *etcdClient)
		err := desiredLRPGenerator.Generate(desiredLRPs)
		Expect(err).NotTo(HaveOccurred())
	}

	if actualLRPs > 0 {
		actualLRPGenerator := generator.NewActualLRPGenerator(logger, bbsClient, *etcdClient)
		err := actualLRPGenerator.Generate(actualLRPs)
		Expect(err).NotTo(HaveOccurred())
	}
})

var _ = AfterSuite(func() {
	purge("/v1/desired_lrp")
	purge("/v1/actual")
})

type ETCDFlags struct {
	etcdCertFile           string
	etcdKeyFile            string
	etcdCaFile             string
	clusterUrls            string
	clientSessionCacheSize int
	maxIdleConnsPerHost    int
}

func AddETCDFlags(flagSet *flag.FlagSet) *ETCDFlags {
	flags := &ETCDFlags{}

	flagSet.StringVar(
		&flags.clusterUrls,
		"etcdCluster",
		"http://127.0.0.1:4001",
		"comma-separated list of etcd URLs (scheme://ip:port)",
	)
	flagSet.StringVar(
		&flags.etcdCertFile,
		"etcdCertFile",
		"",
		"Location of the client certificate for mutual auth",
	)
	flagSet.StringVar(
		&flags.etcdKeyFile,
		"etcdKeyFile",
		"",
		"Location of the client key for mutual auth",
	)
	flagSet.StringVar(
		&flags.etcdCaFile,
		"etcdCaFile",
		"",
		"Location of the CA certificate for mutual auth",
	)

	flagSet.IntVar(
		&flags.clientSessionCacheSize,
		"etcdSessionCacheSize",
		0,
		"Capacity of the ClientSessionCache option on the TLS configuration. If zero, golang's default will be used",
	)
	flagSet.IntVar(
		&flags.maxIdleConnsPerHost,
		"etcdMaxIdleConnsPerHost",
		0,
		"Controls the maximum number of idle (keep-alive) connctions per host. If zero, golang's default will be used",
	)
	return flags
}

func (flags *ETCDFlags) Validate() (*etcddb.ETCDOptions, error) {
	scheme := ""
	clusterUrls := strings.Split(flags.clusterUrls, ",")
	for i, uString := range clusterUrls {
		uString = strings.TrimSpace(uString)
		clusterUrls[i] = uString
		u, err := url.Parse(uString)
		if err != nil {
			return nil, fmt.Errorf("Invalid cluster URL: '%s', error: [%s]", uString, err.Error())
		}
		if scheme == "" {
			if u.Scheme != "http" && u.Scheme != "https" {
				return nil, errors.New("Invalid scheme: " + uString)
			}
			scheme = u.Scheme
		} else if scheme != u.Scheme {
			return nil, fmt.Errorf("Multiple url schemes provided: %s", flags.clusterUrls)
		}
	}

	isSSL := false
	if scheme == "https" {
		isSSL = true
		if flags.etcdCertFile == "" {
			return nil, errors.New("Cert file must be provided for https connections")
		}
		if flags.etcdKeyFile == "" {
			return nil, errors.New("Key file must be provided for https connections")
		}
	}

	return &etcddb.ETCDOptions{
		CertFile:    flags.etcdCertFile,
		KeyFile:     flags.etcdKeyFile,
		CAFile:      flags.etcdCaFile,
		ClusterUrls: clusterUrls,
		IsSSL:       isSSL,
		ClientSessionCacheSize: flags.clientSessionCacheSize,
		MaxIdleConnsPerHost:    flags.maxIdleConnsPerHost,
	}, nil
}

func initializeEtcdClient(logger lager.Logger, etcdOptions *etcddb.ETCDOptions) *etcd.Client {
	var etcdClient *etcd.Client
	var tr *http.Transport

	if etcdOptions.IsSSL {
		if etcdOptions.CertFile == "" || etcdOptions.KeyFile == "" {
			logger.Fatal("failed-to-construct-etcd-tls-client", errors.New("Require both cert and key path"))
		}

		var err error
		etcdClient, err = etcd.NewTLSClient(etcdOptions.ClusterUrls, etcdOptions.CertFile, etcdOptions.KeyFile, etcdOptions.CAFile)
		if err != nil {
			logger.Fatal("failed-to-construct-etcd-tls-client", err)
		}

		tlsCert, err := tls.LoadX509KeyPair(etcdOptions.CertFile, etcdOptions.KeyFile)
		if err != nil {
			logger.Fatal("failed-to-construct-etcd-tls-client", err)
		}

		tlsConfig := &tls.Config{
			Certificates:       []tls.Certificate{tlsCert},
			InsecureSkipVerify: true,
			ClientSessionCache: tls.NewLRUClientSessionCache(etcdOptions.ClientSessionCacheSize),
		}
		tr = &http.Transport{
			TLSClientConfig:     tlsConfig,
			Dial:                etcdClient.DefaultDial,
			MaxIdleConnsPerHost: etcdOptions.MaxIdleConnsPerHost,
		}
		etcdClient.SetTransport(tr)
	} else {
		etcdClient = etcd.NewClient(etcdOptions.ClusterUrls)
	}
	etcdClient.SetConsistency(etcd.STRONG_CONSISTENCY)

	return etcdClient
}

func initializeBBSClient(logger lager.Logger) bbs.Client {
	if bbsAddress == "" {
		log.Fatal("bbsAddress is required")
	}

	bbsURL, err := url.Parse(bbsAddress)
	if err != nil {
		logger.Fatal("Invalid BBS URL", err)
	}

	if bbsURL.Scheme != "https" {
		return bbs.NewClient(bbsAddress)
	}

	bbsClient, err := bbs.NewSecureSkipVerifyClient(bbsAddress, bbsClientCert, bbsClientKey, 1, 1)
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}

func purge(key string) {
	_, err := etcdClient.Delete(key, true)
	if err != nil {
		matches, matchErr := regexp.Match(".*Key not found.*", []byte(err.Error()))
		if matchErr != nil {
			Fail(matchErr.Error())
		}
		if !matches {
			Fail(err.Error())
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and

package cmdimpl

import (
	"encoding/json"
	"fmt"
	"github.com/contiv/vpp/plugins/contiv/model/node"
	"github.com/contiv/vpp/plugins/crd/cache/telemetrymodel"
	"github.com/contiv/vpp/plugins/ksr/model/pod"
	"github.com/contiv/vpp/plugins/netctl/http"
	"github.com/ligato/cn-infra/db/keyval/etcd"
	"github.com/ligato/vpp-agent/plugins/vpp/model/interfaces"
	"os"
	"strings"
	"text/tabwriter"
)

type nodeData struct {
	ipam *telemetrymodel.IPamEntry
	ifcs telemetrymodel.NodeInterfaces
}
type nodeDataCache map[string]*nodeData

type podGetter struct {
	ndCache nodeDataCache
	db      *etcd.BytesConnectionEtcd
	pods    []*pod.Pod
}

// PrintAllPods will print out all of the non local pods in a network in
// a table format.
func PrintAllPods() {
	w := getWriter("HOST-NAME")
	pg := newPodGetter()
	pg.printAllPods(w)
	pg.db.Close()
	w.Flush()
}

//PrintPodsPerNode will print out all of the non-local pods for a certain
// pods along with their tap interface ip address
func PrintPodsPerNode(input string) {
	w := getWriter("")
	pg := newPodGetter()
	pg.printPodsPerNode(w, input, "")
	pg.db.Close()
	w.Flush()
}

func newPodGetter() *podGetter {
	pg := &podGetter{
		ndCache: make(nodeDataCache, 0),
		db:      getEtcdBroker(),
	}

	pg.pods = make([]*pod.Pod, 0)
	itr, err := pg.db.ListValues("/vnf-agent/contiv-ksr/k8s/pod/")
	if err != nil {
		fmt.Printf("Failed to get pods from etcd, error %s", err)
		os.Exit(2)
	}

	for {
		kv, stop := itr.GetNext()
		if stop {
			break
		}
		buf := kv.GetValue()
		podInfo := &pod.Pod{}
		if err = json.Unmarshal(buf, podInfo); err != nil {
			fmt.Printf("Failed to unmarshall pod, error %s", err)
			continue
		}
		pg.pods = append(pg.pods, podInfo)
	}

	return pg
}

func (pg *podGetter) printAllPods(w *tabwriter.Writer) {

	itr, err := pg.db.ListValues("/vnf-agent/contiv-ksr/allocatedIDs/")
	if err != nil {
		fmt.Printf("Error getting values")
		return
	}

	for {
		kv, stop := itr.GetNext()
		if stop {
			fmt.Println()
			break
		}
		buf := kv.GetValue()
		nodeInfo := &node.NodeInfo{}
		err = json.Unmarshal(buf, nodeInfo)
		nodeID := fmt.Sprintf("%s (%s):", nodeInfo.Name, nodeInfo.ManagementIpAddress)
		fmt.Fprintf(w, "%s\t\t\t\t\t\t\t\n", nodeID)
		fmt.Fprintf(w, "%s\t\t\t\t\t\t\t\n", strings.Repeat("-", len(nodeID)))

		pg.printPodsPerNode(w, nodeInfo.ManagementIpAddress, nodeInfo.Name)
		fmt.Fprintln(w, "\t\t\t\t\t\t\t\t")
	}
}

func (pg *podGetter) printPodsPerNode(w *tabwriter.Writer, nodeNameOrIP string, nodeName string) {
	hostIP := resolveNodeOrIP(nodeNameOrIP)

	fmt.Fprintf(w, "POD-NAME\tNAMESPACE\tPOD-IP\tVPP-IP\tIF-IDX\tIF-NAME\tINTERNAL-IF-NAME\n")

	for _, podInfo := range pg.pods {
		if podInfo.HostIpAddress != hostIP {
			continue
		} else if podInfo.IpAddress == hostIP {
			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
				podInfo.Name,
				podInfo.Namespace,
				podInfo.IpAddress,
				"", "", "", "")
		} else {
			ipAddress, ifIndex, intName, name := pg.getTapInterfaceForPod(podInfo)
			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\t%s\n",
				podInfo.Name,
				podInfo.Namespace,
				podInfo.IpAddress,
				strings.Split(ipAddress, "/")[0],
				ifIndex,
				intName,
				name)
		}
	}
}

func (pg *podGetter) getTapInterfaceForPod(podInfo *pod.Pod) (string, uint32, string, string) {
	// when we see a pod from a given node for the first time, retrieve its
	// IPAM and interface info
	if pg.ndCache[podInfo.HostIpAddress] == nil {

		// Get ipam data for the node where the pod is hosted
		b, err := http.GetNodeInfo(podInfo.HostIpAddress, getIpamDataCmd)
		if err != nil {
			fmt.Printf("Host '%s', Pod '%s' - failed to get ipam, err %s\n",
				podInfo.HostIpAddress, podInfo.Name, err)
			return "N/A", 0, "N/A", "N/A"
		}

		ipam := &telemetrymodel.IPamEntry{}
		if err := json.Unmarshal(b, ipam); err != nil {
			fmt.Printf("Host '%s', Pod '%s' - failed to decode ipam, err %s\n",
				podInfo.HostIpAddress, podInfo.Name, err)
			return "N/A", 0, "N/A", "N/A"
		}

		// Get interfaces data for the node where the pod is hosted
		b, err = http.GetNodeInfo(podInfo.HostIpAddress, getInterfaceDataCmd)
		intfs := make(telemetrymodel.NodeInterfaces)
		if err := json.Unmarshal(b, &intfs); err != nil {
			fmt.Printf("Host '%s', Pod '%s' - failed to get pod's interface, err %s\n",
				podInfo.HostIpAddress, podInfo.Name, err)
			return "N/A", 0, "N/A", "N/A"
		}

		pg.ndCache[podInfo.HostIpAddress] = &nodeData{
			ipam: ipam,
			ifcs: intfs,
		}
	}

	// Determine the tap interface on VPP that connects the pod to the VPP
	podPfxLen := pg.ndCache[podInfo.HostIpAddress].ipam.Config.VppHostNetworkPrefixLen
	podMask := maskLength2Mask(int(podPfxLen))

	podIPSubnet, podIPMask, err := getIPAddressAndMask(pg.ndCache[podInfo.HostIpAddress].ipam.Config.PodSubnetCIDR)
	if err != nil {
		fmt.Printf("Host '%s', Pod '%s' - invalid PodSubnetCIDR address %s, err %s\n",
			podInfo.HostIpAddress, podInfo.Name, pg.ndCache[podInfo.HostIpAddress].ipam.Config.PodSubnetCIDR, err)
		// Do not return - we can still continue if this error happens
	}

	if podMask != podIPMask {
		fmt.Printf("Host '%s', Pod '%s' - vppHostNetworkPrefixLen mismatch: "+
			"PodSubnetCIDR '%s', podNetworkPrefixLen '%d'\n",
			podInfo.HostIpAddress, podInfo.Name,
			pg.ndCache[podInfo.HostIpAddress].ipam.Config.PodSubnetCIDR,
			pg.ndCache[podInfo.HostIpAddress].ipam.Config.PodNetworkPrefixLen)
		// Do not return - we can still continue if this error happens
	}

	podIfIPAddress, podIfIPMask, err := getIPAddressAndMask(pg.ndCache[podInfo.HostIpAddress].ipam.Config.PodIfIPCIDR)
	if err != nil {
		fmt.Printf("Host '%s', Pod '%s' - invalid PodIfIPCIDR address %s, err %s\n",
			podInfo.HostIpAddress, podInfo.Name, pg.ndCache[podInfo.HostIpAddress].ipam.Config.PodIfIPCIDR, err)
		return "N/A", 0, "N/A", "N/A"
	}

	if podMask != podIfIPMask {
		fmt.Printf("Host '%s', Pod '%s' - vppHostNetworkPrefixLen mismatch: "+
			"PodIfIPCIDR '%s', podNetworkPrefixLen '%d'\n",
			podInfo.HostIpAddress, podInfo.Name,
			pg.ndCache[podInfo.HostIpAddress].ipam.Config.PodIfIPCIDR,
			pg.ndCache[podInfo.HostIpAddress].ipam.Config.PodNetworkPrefixLen)
		// Do not return - we can still continue if this error happens
	}

	podIfIPPrefix := podIfIPAddress &^ podMask
	podAddr, err := ip2uint32(podInfo.IpAddress)
	if err != nil {
		fmt.Printf("Host '%s', Pod '%s' - invalid podInfo.IpAddress %s, err %s",
			podInfo.HostIpAddress, podInfo.Name, podInfo.IpAddress, err)
		return "N/A", 0, "N/A", "N/A"
	}

	podAddrSuffix := podAddr & podMask

	if podAddr&^podMask != podIPSubnet {
		fmt.Printf("Host '%s', Pod '%s' - pod IP address %s not from PodSubnetCIDR %s\n",
			podInfo.HostIpAddress, podInfo.Name, podInfo.IpAddress,
			pg.ndCache[podInfo.HostIpAddress].ipam.Config.PodSubnetCIDR)
		// Do not return - we can still continue if this error happens
	}

	for _, intf := range pg.ndCache[podInfo.HostIpAddress].ifcs {
		if intf.If.IfType == interfaces.InterfaceType_TAP_INTERFACE {
			for _, ip := range intf.If.IPAddresses {
				ifIPAddr, iffIPMask, err := getIPAddressAndMask(ip)
				if err != nil {
					continue
				}
				if iffIPMask != 0 {
					// TODO: do some error handling
					continue
				}

				ifIPAdrPrefix := ifIPAddr &^ podMask
				ifIPAdrSuffix := ifIPAddr & podMask
				if (podIfIPPrefix == ifIPAdrPrefix) && (ifIPAdrSuffix == podAddrSuffix) {
					return ip, intf.IfMeta.SwIfIndex, intf.IfMeta.VppInternalName, intf.If.Name
				}
			}
		}
	}

	return "N/A", 0, "N/A", "N/A"
}

func getWriter(hostName string) *tabwriter.Writer {
	w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0)
	return w
}
<commit_msg>Fix a bug in netctl where IPAM config.podSubnetCIDR was used instead of podNetwork<commit_after>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and

package cmdimpl

import (
	"encoding/json"
	"fmt"
	"github.com/contiv/vpp/plugins/contiv/model/node"
	"github.com/contiv/vpp/plugins/crd/cache/telemetrymodel"
	"github.com/contiv/vpp/plugins/ksr/model/pod"
	"github.com/contiv/vpp/plugins/netctl/http"
	"github.com/ligato/cn-infra/db/keyval/etcd"
	"github.com/ligato/vpp-agent/plugins/vpp/model/interfaces"
	"os"
	"strings"
	"text/tabwriter"
)

type nodeData struct {
	ipam *telemetrymodel.IPamEntry
	ifcs telemetrymodel.NodeInterfaces
}
type nodeDataCache map[string]*nodeData

type podGetter struct {
	ndCache nodeDataCache
	db      *etcd.BytesConnectionEtcd
	pods    []*pod.Pod
}

// PrintAllPods will print out all of the non local pods in a network in
// a table format.
func PrintAllPods() {
	w := getWriter("HOST-NAME")
	pg := newPodGetter()
	pg.printAllPods(w)
	pg.db.Close()
	w.Flush()
}

//PrintPodsPerNode will print out all of the non-local pods for a certain
// pods along with their tap interface ip address
func PrintPodsPerNode(input string) {
	w := getWriter("")
	pg := newPodGetter()
	pg.printPodsPerNode(w, input, "")
	pg.db.Close()
	w.Flush()
}

func newPodGetter() *podGetter {
	pg := &podGetter{
		ndCache: make(nodeDataCache, 0),
		db:      getEtcdBroker(),
	}

	pg.pods = make([]*pod.Pod, 0)
	itr, err := pg.db.ListValues("/vnf-agent/contiv-ksr/k8s/pod/")
	if err != nil {
		fmt.Printf("Failed to get pods from etcd, error %s", err)
		os.Exit(2)
	}

	for {
		kv, stop := itr.GetNext()
		if stop {
			break
		}
		buf := kv.GetValue()
		podInfo := &pod.Pod{}
		if err = json.Unmarshal(buf, podInfo); err != nil {
			fmt.Printf("Failed to unmarshall pod, error %s", err)
			continue
		}
		pg.pods = append(pg.pods, podInfo)
	}

	return pg
}

func (pg *podGetter) printAllPods(w *tabwriter.Writer) {

	itr, err := pg.db.ListValues("/vnf-agent/contiv-ksr/allocatedIDs/")
	if err != nil {
		fmt.Printf("Error getting values")
		return
	}

	for {
		kv, stop := itr.GetNext()
		if stop {
			fmt.Println()
			break
		}
		buf := kv.GetValue()
		nodeInfo := &node.NodeInfo{}
		err = json.Unmarshal(buf, nodeInfo)
		nodeID := fmt.Sprintf("%s (%s):", nodeInfo.Name, nodeInfo.ManagementIpAddress)
		fmt.Fprintf(w, "%s\t\t\t\t\t\t\t\n", nodeID)
		fmt.Fprintf(w, "%s\t\t\t\t\t\t\t\n", strings.Repeat("-", len(nodeID)))

		pg.printPodsPerNode(w, nodeInfo.ManagementIpAddress, nodeInfo.Name)
		fmt.Fprintln(w, "\t\t\t\t\t\t\t\t")
	}
}

func (pg *podGetter) printPodsPerNode(w *tabwriter.Writer, nodeNameOrIP string, nodeName string) {
	hostIP := resolveNodeOrIP(nodeNameOrIP)

	fmt.Fprintf(w, "POD-NAME\tNAMESPACE\tPOD-IP\tVPP-IP\tIF-IDX\tIF-NAME\tINTERNAL-IF-NAME\n")

	for _, podInfo := range pg.pods {
		if podInfo.HostIpAddress != hostIP {
			continue
		} else if podInfo.IpAddress == hostIP {
			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
				podInfo.Name,
				podInfo.Namespace,
				podInfo.IpAddress,
				"", "", "", "")
		} else {
			ipAddress, ifIndex, intName, name := pg.getTapInterfaceForPod(podInfo)
			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\t%s\n",
				podInfo.Name,
				podInfo.Namespace,
				podInfo.IpAddress,
				strings.Split(ipAddress, "/")[0],
				ifIndex,
				intName,
				name)
		}
	}
}

func (pg *podGetter) getTapInterfaceForPod(podInfo *pod.Pod) (string, uint32, string, string) {
	// when we see a pod from a given node for the first time, retrieve its
	// IPAM and interface info
	if pg.ndCache[podInfo.HostIpAddress] == nil {

		// Get ipam data for the node where the pod is hosted
		b, err := http.GetNodeInfo(podInfo.HostIpAddress, getIpamDataCmd)
		if err != nil {
			fmt.Printf("Host '%s', Pod '%s' - failed to get ipam, err %s\n",
				podInfo.HostIpAddress, podInfo.Name, err)
			return "N/A", 0, "N/A", "N/A"
		}

		ipam := &telemetrymodel.IPamEntry{}
		if err := json.Unmarshal(b, ipam); err != nil {
			fmt.Printf("Host '%s', Pod '%s' - failed to decode ipam, err %s\n",
				podInfo.HostIpAddress, podInfo.Name, err)
			return "N/A", 0, "N/A", "N/A"
		}

		// Get interfaces data for the node where the pod is hosted
		b, err = http.GetNodeInfo(podInfo.HostIpAddress, getInterfaceDataCmd)
		intfs := make(telemetrymodel.NodeInterfaces)
		if err := json.Unmarshal(b, &intfs); err != nil {
			fmt.Printf("Host '%s', Pod '%s' - failed to get pod's interface, err %s\n",
				podInfo.HostIpAddress, podInfo.Name, err)
			return "N/A", 0, "N/A", "N/A"
		}

		pg.ndCache[podInfo.HostIpAddress] = &nodeData{
			ipam: ipam,
			ifcs: intfs,
		}
	}

	// Determine the tap interface on VPP that connects the pod to the VPP
	podPfxLen := pg.ndCache[podInfo.HostIpAddress].ipam.Config.VppHostNetworkPrefixLen
	podMask := maskLength2Mask(int(podPfxLen))

	podNetwork, podIPMask, err := getIPAddressAndMask(pg.ndCache[podInfo.HostIpAddress].ipam.PodNetwork)
	if err != nil {
		fmt.Printf("Host '%s', Pod '%s' - invalid PodNetwork address %s, err %s\n",
			podInfo.HostIpAddress, podInfo.Name, pg.ndCache[podInfo.HostIpAddress].ipam.PodNetwork, err)
		// Do not return - we can still continue if this error happens
	}

	if podMask != podIPMask {
		fmt.Printf("Host '%s', Pod '%s' - vppHostNetworkPrefixLen mismatch: "+
			"PodNetwork '%s', podNetworkPrefixLen '%d'\n",
			podInfo.HostIpAddress, podInfo.Name,
			pg.ndCache[podInfo.HostIpAddress].ipam.PodNetwork,
			pg.ndCache[podInfo.HostIpAddress].ipam.Config.PodNetworkPrefixLen)
		// Do not return - we can still continue if this error happens
	}

	podIfIPAddress, podIfIPMask, err := getIPAddressAndMask(pg.ndCache[podInfo.HostIpAddress].ipam.Config.PodIfIPCIDR)
	if err != nil {
		fmt.Printf("Host '%s', Pod '%s' - invalid PodIfIPCIDR address %s, err %s\n",
			podInfo.HostIpAddress, podInfo.Name, pg.ndCache[podInfo.HostIpAddress].ipam.Config.PodIfIPCIDR, err)
		return "N/A", 0, "N/A", "N/A"
	}

	if podMask != podIfIPMask {
		fmt.Printf("Host '%s', Pod '%s' - vppHostNetworkPrefixLen mismatch: "+
			"PodIfIPCIDR '%s', podNetworkPrefixLen '%d'\n",
			podInfo.HostIpAddress, podInfo.Name,
			pg.ndCache[podInfo.HostIpAddress].ipam.Config.PodIfIPCIDR,
			pg.ndCache[podInfo.HostIpAddress].ipam.Config.PodNetworkPrefixLen)
		// Do not return - we can still continue if this error happens
	}

	podIfIPPrefix := podIfIPAddress &^ podMask
	podAddr, err := ip2uint32(podInfo.IpAddress)
	if err != nil {
		fmt.Printf("Host '%s', Pod '%s' - invalid podInfo.IpAddress %s, err %s",
			podInfo.HostIpAddress, podInfo.Name, podInfo.IpAddress, err)
		return "N/A", 0, "N/A", "N/A"
	}

	podAddrSuffix := podAddr & podMask

	if podAddr&^podMask != podNetwork {
		fmt.Printf("Host '%s', Pod '%s' - pod IP address %s not from PodNetwork subnet %s\n",
			podInfo.HostIpAddress, podInfo.Name, podInfo.IpAddress,
			pg.ndCache[podInfo.HostIpAddress].ipam.PodNetwork)
		// Do not return - we can still continue if this error happens
	}

	for _, intf := range pg.ndCache[podInfo.HostIpAddress].ifcs {
		if intf.If.IfType == interfaces.InterfaceType_TAP_INTERFACE {
			for _, ip := range intf.If.IPAddresses {
				ifIPAddr, iffIPMask, err := getIPAddressAndMask(ip)
				if err != nil {
					continue
				}
				if iffIPMask != 0 {
					// TODO: do some error handling
					continue
				}

				ifIPAdrPrefix := ifIPAddr &^ podMask
				ifIPAdrSuffix := ifIPAddr & podMask
				if (podIfIPPrefix == ifIPAdrPrefix) && (ifIPAdrSuffix == podAddrSuffix) {
					return ip, intf.IfMeta.SwIfIndex, intf.IfMeta.VppInternalName, intf.If.Name
				}
			}
		}
	}

	return "N/A", 0, "N/A", "N/A"
}

func getWriter(hostName string) *tabwriter.Writer {
	w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0)
	return w
}
<|endoftext|>


<commit_before>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bootkube

import (
	"fmt"
	"strings"
	"time"

	"github.com/coreos/mantle/kola/cluster"
	"github.com/coreos/mantle/pluton"
	"github.com/coreos/mantle/pluton/spawn"
	"github.com/coreos/mantle/util"
)

func etcdScale(tc cluster.TestCluster) error {
	// create cluster with self-hosted etcd
	c, err := spawn.MakeBootkubeCluster(tc, 1, true)
	if err != nil {
		return err
	}

	// add two master nodes to cluster
	if err := c.AddMasters(2); err != nil {
		return err
	}

	// scale up etcd operator
	if err := resizeSelfHostedEtcd(c, 3); err != nil {
		return err
	}

	// todo check that each pod runs on a different master node
	if err := checkEtcdPodDistribution(c, 3); err != nil {
		return err
	}

	// scale back to 1
	if err := resizeSelfHostedEtcd(c, 1); err != nil {
		return err
	}

	// run an nginx deployment and ping it
	if err := nginxCheck(c); err != nil {
		return fmt.Errorf("nginxCheck: %v", err)
	}
	return nil
}

// resizes self-hosted etcd and checks that the desired number of pods are in a running state
func resizeSelfHostedEtcd(c *pluton.Cluster, size int) error {
	const (
		tprGroup   = "etcd.coreos.com"
		apiVersion = "v1beta1"
		tprKind    = "clusters"
	)
	var tprEndpoint = fmt.Sprintf("http://127.0.0.1:8080/apis/%s/%s/namespaces/kube-system/%s/kube-etcd",
		tprGroup, apiVersion, tprKind)

	scaleCmds := []string{
		fmt.Sprintf("curl -H 'Content-Type: application/json' -X GET %v > body.json", tprEndpoint),
		// delete resourceVersion field before curling back
		fmt.Sprintf("jq 'recurse(.metadata) |= del(.resourceVersion)' < body.json | jq .spec.size=%v > newbody.json", size),
		fmt.Sprintf("curl -H 'Content-Type: application/json' -X PUT --data @newbody.json %v", tprEndpoint),
	}
	for _, cmd := range scaleCmds {
		sout, serr, err := c.SSH(cmd)
		if err != nil {
			return fmt.Errorf("error in scale up command: %v:\nSTDERR: %s\nSTDOUT: %s", cmd, serr, sout)
		}
	}

	// check that all pods are running
	podsReady := func() error {
		out, err := c.Kubectl(`get po -l etcd_cluster=kube-etcd -o jsonpath='{.items[*].status.phase}' --namespace=kube-system`)
		if err != nil {
			return err
		}
		phases := strings.Split(out, " ")
		if len(phases) != size {
			return fmt.Errorf("expected %d etcd pods got %d: %v", size, len(phases), phases)
		}
		for _, phase := range phases {
			if phase != "Running" {
				return fmt.Errorf("one or more etcd pods not in a 'Running' phase")
			}
		}
		return nil
	}

	if err := util.Retry(15, 10*time.Second, podsReady); err != nil {
		return err
	}
	return nil
}

// checks that self-hosted etcd pods are scheduled on different master nodes
// when possible
func checkEtcdPodDistribution(c *pluton.Cluster, etcdClusterSize int) error {
	// check that number of unique nodes etcd pods run on is equal to the
	// lesser value betweeen total number of master nodes and total number
	// of etcd pods
	out, err := c.Kubectl(`get po -l etcd_cluster=kube-etcd -o jsonpath='{.items[*].status.hostIP}' --namespace=kube-system`)
	if err != nil {
		return err
	}

	nodeIPs := strings.Split(out, " ")
	nodeSet := map[string]struct{}{}
	for _, node := range nodeIPs {
		nodeSet[node] = struct{}{}
	}

	var expectedUniqueNodes int
	if len(c.Masters) > etcdClusterSize {
		expectedUniqueNodes = etcdClusterSize
	} else {
		expectedUniqueNodes = len(c.Masters)
	}

	if len(nodeSet) != expectedUniqueNodes {
		return fmt.Errorf("self-hosted etcd pods not properly distributed")
	}

	// check that each node in nodeSet is a master node
	masterSet := map[string]struct{}{}
	for _, m := range c.Masters {
		masterSet[m.PrivateIP()] = struct{}{}
	}

	for k, _ := range nodeSet {
		if _, ok := masterSet[k]; !ok {
			// Just warn instead of erroring until/if supported
			plog.Infof("detected self-hosted etcd pod running on non-master node %v %v", masterSet, nodeSet)
		}
	}

	return nil
}
<commit_msg>pluton/tests/bootkube: self-hosted etcd only scales to master nodes<commit_after>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bootkube

import (
	"fmt"
	"strings"
	"time"

	"github.com/coreos/mantle/kola/cluster"
	"github.com/coreos/mantle/pluton"
	"github.com/coreos/mantle/pluton/spawn"
	"github.com/coreos/mantle/util"
)

func etcdScale(tc cluster.TestCluster) error {
	// create cluster with self-hosted etcd
	c, err := spawn.MakeBootkubeCluster(tc, 1, true)
	if err != nil {
		return err
	}

	// add two master nodes to cluster
	if err := c.AddMasters(2); err != nil {
		return err
	}

	// scale up etcd operator
	if err := resizeSelfHostedEtcd(c, 3); err != nil {
		return err
	}

	// todo check that each pod runs on a different master node
	if err := checkEtcdPodDistribution(c, 3); err != nil {
		return err
	}

	// scale back to 1
	if err := resizeSelfHostedEtcd(c, 1); err != nil {
		return err
	}

	// run an nginx deployment and ping it
	if err := nginxCheck(c); err != nil {
		return fmt.Errorf("nginxCheck: %v", err)
	}
	return nil
}

// resizes self-hosted etcd and checks that the desired number of pods are in a running state
func resizeSelfHostedEtcd(c *pluton.Cluster, size int) error {
	const (
		tprGroup   = "etcd.coreos.com"
		apiVersion = "v1beta1"
		tprKind    = "clusters"
	)
	var tprEndpoint = fmt.Sprintf("http://127.0.0.1:8080/apis/%s/%s/namespaces/kube-system/%s/kube-etcd",
		tprGroup, apiVersion, tprKind)

	scaleCmds := []string{
		fmt.Sprintf("curl -H 'Content-Type: application/json' -X GET %v > body.json", tprEndpoint),
		// delete resourceVersion field before curling back
		fmt.Sprintf("jq 'recurse(.metadata) |= del(.resourceVersion)' < body.json | jq .spec.size=%v > newbody.json", size),
		fmt.Sprintf("curl -H 'Content-Type: application/json' -X PUT --data @newbody.json %v", tprEndpoint),
	}
	for _, cmd := range scaleCmds {
		sout, serr, err := c.SSH(cmd)
		if err != nil {
			return fmt.Errorf("error in scale up command: %v:\nSTDERR: %s\nSTDOUT: %s", cmd, serr, sout)
		}
	}

	// check that all pods are running
	podsReady := func() error {
		out, err := c.Kubectl(`get po -l etcd_cluster=kube-etcd -o jsonpath='{.items[*].status.phase}' --namespace=kube-system`)
		if err != nil {
			return err
		}
		phases := strings.Split(out, " ")
		if len(phases) != size {
			return fmt.Errorf("expected %d etcd pods got %d: %v", size, len(phases), phases)
		}
		for _, phase := range phases {
			if phase != "Running" {
				return fmt.Errorf("one or more etcd pods not in a 'Running' phase")
			}
		}
		return nil
	}

	if err := util.Retry(15, 10*time.Second, podsReady); err != nil {
		return err
	}
	return nil
}

// checks that self-hosted etcd pods are scheduled on different master nodes
// when possible
func checkEtcdPodDistribution(c *pluton.Cluster, etcdClusterSize int) error {
	// check that number of unique nodes etcd pods run on is equal to the
	// lesser value betweeen total number of master nodes and total number
	// of etcd pods
	out, err := c.Kubectl(`get po -l etcd_cluster=kube-etcd -o jsonpath='{.items[*].status.hostIP}' --namespace=kube-system`)
	if err != nil {
		return err
	}

	nodeIPs := strings.Split(out, " ")
	nodeSet := map[string]struct{}{}
	for _, node := range nodeIPs {
		nodeSet[node] = struct{}{}
	}

	var expectedUniqueNodes int
	if len(c.Masters) > etcdClusterSize {
		expectedUniqueNodes = etcdClusterSize
	} else {
		expectedUniqueNodes = len(c.Masters)
	}

	if len(nodeSet) != expectedUniqueNodes {
		return fmt.Errorf("self-hosted etcd pods not properly distributed")
	}

	// check that each node in nodeSet is a master node
	masterSet := map[string]struct{}{}
	for _, m := range c.Masters {
		masterSet[m.PrivateIP()] = struct{}{}
	}

	for k, _ := range nodeSet {
		if _, ok := masterSet[k]; !ok {
			return fmt.Errorf("detected self-hosted etcd pod running on non-master node %v %v", masterSet, nodeSet)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package bootstrap

import (
	"testing"
)

type TestAction struct {
	Foo string
	Bar int
}

func (t *TestAction) Run(state *State) error {
	return nil
}

func init() {
	Register("test-action", &TestAction{})
}

func TestRun(t *testing.T) {
	manifest := []byte(`[{"id":"1", "action":"test-action", "Foo":"bar"}, {"id":"2", "action":"test-action", "Bar":2}]`)
	err := Run(manifest)
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>bootstrap: Remove outdated test<commit_after><|endoftext|>


<commit_before>package btelegram

import (
	"strconv"

	"github.com/42wim/matterbridge/bridge/config"
	log "github.com/Sirupsen/logrus"
	"github.com/go-telegram-bot-api/telegram-bot-api"
)

type Btelegram struct {
	c       *tgbotapi.BotAPI
	Config  *config.Protocol
	Remote  chan config.Message
	Account string
}

var flog *log.Entry
var protocol = "telegram"

func init() {
	flog = log.WithFields(log.Fields{"module": protocol})
}

func New(cfg config.Protocol, account string, c chan config.Message) *Btelegram {
	b := &Btelegram{}
	b.Config = &cfg
	b.Remote = c
	b.Account = account
	return b
}

func (b *Btelegram) Connect() error {
	var err error
	flog.Info("Connecting")
	b.c, err = tgbotapi.NewBotAPI(b.Config.Token)
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	updates, err := b.c.GetUpdatesChan(tgbotapi.NewUpdate(0))
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	flog.Info("Connection succeeded")
	go b.handleRecv(updates)
	return nil
}

func (b *Btelegram) Disconnect() error {
	return nil

}

func (b *Btelegram) JoinChannel(channel string) error {
	return nil
}

func (b *Btelegram) Send(msg config.Message) error {
	flog.Debugf("Receiving %#v", msg)
	chatid, err := strconv.ParseInt(msg.Channel, 10, 64)
	if err != nil {
		return err
	}

	if b.Config.MessageFormat == "HTML" {
		msg.Text = makeHTML(msg.Text)
	}
	m := tgbotapi.NewMessage(chatid, msg.Username+msg.Text)
	if b.Config.MessageFormat == "HTML" {
		m.ParseMode = tgbotapi.ModeHTML
	}
	_, err = b.c.Send(m)
	return err
}

func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
	for update := range updates {
		var message *tgbotapi.Message
		username := ""
		channel := ""
		text := ""
		// handle channels
		if update.ChannelPost != nil {
			message = update.ChannelPost
		}
		if update.EditedChannelPost != nil && !b.Config.EditDisable {
			message = update.EditedChannelPost
			message.Text = message.Text + b.Config.EditSuffix
		}
		// handle groups
		if update.Message != nil {
			message = update.Message
		}
		if update.EditedMessage != nil && !b.Config.EditDisable {
			message = update.EditedMessage
			message.Text = message.Text + b.Config.EditSuffix
		}
		if message.From != nil {
			if b.Config.UseFirstName {
				username = message.From.FirstName
			}
			if username == "" {
				username = message.From.UserName
				if username == "" {
					username = message.From.FirstName
				}
			}
			text = message.Text
			channel = strconv.FormatInt(message.Chat.ID, 10)
		}

		if username == "" {
			username = "unknown"
		}
		if message.Sticker != nil {
			text = text + " " + b.getFileDirectURL(message.Sticker.FileID)
		}
		if message.Video != nil {
			text = text + " " + b.getFileDirectURL(message.Video.FileID)
		}
		if message.Photo != nil {
			photos := *message.Photo
			// photo 3 is the biggest
			if len(photos) == 3 {
				text = text + " " + b.getFileDirectURL(photos[2].FileID)
			}
		}
		if message.Document != nil {
			text = text + " " + message.Document.FileName + " : " + b.getFileDirectURL(message.Document.FileID)
		}
		if text != "" {
			flog.Debugf("Sending message from %s on %s to gateway", username, b.Account)
			b.Remote <- config.Message{Username: username, Text: text, Channel: channel, Account: b.Account, UserID: strconv.Itoa(message.From.ID)}
		}
	}
}

func (b *Btelegram) getFileDirectURL(id string) string {
	res, err := b.c.GetFileDirectURL(id)
	if err != nil {
		return ""
	}
	return res
}
<commit_msg>Use the last (and biggest) photo to relay (telegram). Closes #184<commit_after>package btelegram

import (
	"strconv"

	"github.com/42wim/matterbridge/bridge/config"
	log "github.com/Sirupsen/logrus"
	"github.com/go-telegram-bot-api/telegram-bot-api"
)

type Btelegram struct {
	c       *tgbotapi.BotAPI
	Config  *config.Protocol
	Remote  chan config.Message
	Account string
}

var flog *log.Entry
var protocol = "telegram"

func init() {
	flog = log.WithFields(log.Fields{"module": protocol})
}

func New(cfg config.Protocol, account string, c chan config.Message) *Btelegram {
	b := &Btelegram{}
	b.Config = &cfg
	b.Remote = c
	b.Account = account
	return b
}

func (b *Btelegram) Connect() error {
	var err error
	flog.Info("Connecting")
	b.c, err = tgbotapi.NewBotAPI(b.Config.Token)
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	updates, err := b.c.GetUpdatesChan(tgbotapi.NewUpdate(0))
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	flog.Info("Connection succeeded")
	go b.handleRecv(updates)
	return nil
}

func (b *Btelegram) Disconnect() error {
	return nil

}

func (b *Btelegram) JoinChannel(channel string) error {
	return nil
}

func (b *Btelegram) Send(msg config.Message) error {
	flog.Debugf("Receiving %#v", msg)
	chatid, err := strconv.ParseInt(msg.Channel, 10, 64)
	if err != nil {
		return err
	}

	if b.Config.MessageFormat == "HTML" {
		msg.Text = makeHTML(msg.Text)
	}
	m := tgbotapi.NewMessage(chatid, msg.Username+msg.Text)
	if b.Config.MessageFormat == "HTML" {
		m.ParseMode = tgbotapi.ModeHTML
	}
	_, err = b.c.Send(m)
	return err
}

func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
	for update := range updates {
		var message *tgbotapi.Message
		username := ""
		channel := ""
		text := ""
		// handle channels
		if update.ChannelPost != nil {
			message = update.ChannelPost
		}
		if update.EditedChannelPost != nil && !b.Config.EditDisable {
			message = update.EditedChannelPost
			message.Text = message.Text + b.Config.EditSuffix
		}
		// handle groups
		if update.Message != nil {
			message = update.Message
		}
		if update.EditedMessage != nil && !b.Config.EditDisable {
			message = update.EditedMessage
			message.Text = message.Text + b.Config.EditSuffix
		}
		if message.From != nil {
			if b.Config.UseFirstName {
				username = message.From.FirstName
			}
			if username == "" {
				username = message.From.UserName
				if username == "" {
					username = message.From.FirstName
				}
			}
			text = message.Text
			channel = strconv.FormatInt(message.Chat.ID, 10)
		}

		if username == "" {
			username = "unknown"
		}
		if message.Sticker != nil {
			text = text + " " + b.getFileDirectURL(message.Sticker.FileID)
		}
		if message.Video != nil {
			text = text + " " + b.getFileDirectURL(message.Video.FileID)
		}
		if message.Photo != nil {
			photos := *message.Photo
			// last photo is the biggest
			text = text + " " + b.getFileDirectURL(photos[len(photos)-1].FileID)
		}
		if message.Document != nil {
			text = text + " " + message.Document.FileName + " : " + b.getFileDirectURL(message.Document.FileID)
		}
		if text != "" {
			flog.Debugf("Sending message from %s on %s to gateway", username, b.Account)
			b.Remote <- config.Message{Username: username, Text: text, Channel: channel, Account: b.Account, UserID: strconv.Itoa(message.From.ID)}
		}
	}
}

func (b *Btelegram) getFileDirectURL(id string) string {
	res, err := b.c.GetFileDirectURL(id)
	if err != nil {
		return ""
	}
	return res
}
<|endoftext|>


<commit_before>// Copyright 2017 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package kubernetes

import (
	"encoding/json"
	"strings"

	"github.com/pkg/errors"
	"github.com/tsuru/tsuru/provision"
	"github.com/tsuru/tsuru/set"
	"github.com/tsuru/tsuru/volume"
	"github.com/ugorji/go/codec"
	apiv1 "k8s.io/api/core/v1"
	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type volumeOptions struct {
	Plugin       string
	StorageClass string `json:"storage-class"`
	Capacity     resource.Quantity
	AccessModes  string `json:"access-modes"`
}

var allowedNonPersistentVolumes = set.FromValues("emptyDir")

func (opts *volumeOptions) isPersistent() bool {
	return !allowedNonPersistentVolumes.Includes(opts.Plugin)
}

func createVolumesForApp(client *ClusterClient, app provision.App) ([]apiv1.Volume, []apiv1.VolumeMount, error) {
	volumes, err := volume.ListByApp(app.GetName())
	if err != nil {
		return nil, nil, errors.WithStack(err)
	}
	var kubeVolumes []apiv1.Volume
	var kubeMounts []apiv1.VolumeMount
	for i := range volumes {
		opts, err := validateVolume(&volumes[i])
		if err != nil {
			return nil, nil, err
		}
		if opts.isPersistent() {
			err = createVolume(client, &volumes[i], opts, app)
			if err != nil {
				return nil, nil, err
			}
		}
		volume, mounts, err := bindsForVolume(&volumes[i], opts, app.GetName())
		if err != nil {
			return nil, nil, err
		}
		kubeMounts = append(kubeMounts, mounts...)
		kubeVolumes = append(kubeVolumes, *volume)
	}
	return kubeVolumes, kubeMounts, nil
}

func bindsForVolume(v *volume.Volume, opts *volumeOptions, appName string) (*apiv1.Volume, []apiv1.VolumeMount, error) {
	var kubeMounts []apiv1.VolumeMount
	binds, err := v.LoadBindsForApp(appName)
	if err != nil {
		return nil, nil, errors.WithStack(err)
	}
	allReadOnly := true
	for _, b := range binds {
		kubeMounts = append(kubeMounts, apiv1.VolumeMount{
			Name:      volumeName(v.Name),
			MountPath: b.ID.MountPoint,
			ReadOnly:  b.ReadOnly,
		})
		if !b.ReadOnly {
			allReadOnly = false
		}
	}
	kubeVol := apiv1.Volume{
		Name: volumeName(v.Name),
	}
	if opts.isPersistent() {
		kubeVol.VolumeSource = apiv1.VolumeSource{
			PersistentVolumeClaim: &apiv1.PersistentVolumeClaimVolumeSource{
				ClaimName: volumeClaimName(v.Name),
				ReadOnly:  allReadOnly,
			},
		}
	} else {
		kubeVol.VolumeSource, err = nonPersistentVolume(v, opts)
		if err != nil {
			return nil, nil, err
		}
	}
	return &kubeVol, kubeMounts, nil
}

func nonPersistentVolume(v *volume.Volume, opts *volumeOptions) (apiv1.VolumeSource, error) {
	var volumeSrc apiv1.VolumeSource
	data, err := json.Marshal(map[string]interface{}{
		opts.Plugin: v.Opts,
	})
	if err != nil {
		return volumeSrc, errors.WithStack(err)
	}
	h := &codec.JsonHandle{}
	dec := codec.NewDecoderBytes(data, h)
	err = dec.Decode(&volumeSrc)
	if err != nil {
		return volumeSrc, errors.WithStack(err)
	}
	return volumeSrc, nil
}

func validateVolume(v *volume.Volume) (*volumeOptions, error) {
	var opts volumeOptions
	err := v.UnmarshalPlan(&opts)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	if opts.Plugin != "" && opts.StorageClass != "" {
		return nil, errors.New("both volume plan plugin and storage-class cannot be set")
	}
	if opts.Plugin == "" && opts.StorageClass == "" {
		return nil, errors.New("both volume plan plugin and storage-class are empty")
	}
	if !opts.isPersistent() {
		return &opts, nil
	}
	if capRaw, ok := v.Opts["capacity"]; ok {
		delete(v.Opts, "capacity")
		opts.Capacity, err = resource.ParseQuantity(capRaw)
		if err != nil {
			return nil, errors.Wrap(err, "unable to parse `capacity` opt")
		}
	}
	if opts.Capacity.IsZero() {
		return nil, errors.New("capacity is mandatory either in plan or as volume opts")
	}
	if accessModesRaw, ok := v.Opts["access-modes"]; ok {
		delete(v.Opts, "access-modes")
		opts.AccessModes = accessModesRaw
	}
	if opts.AccessModes == "" {
		return nil, errors.New("access-modes is mandatory either in plan or as volume opts")
	}
	return &opts, nil
}

func deleteVolume(client *ClusterClient, name string) error {
	v, err := volume.Load(name)
	if err != nil {
		if err == volume.ErrVolumeNotFound {
			return nil
		}
		return err
	}
	namespace, err := getNamespaceForVolume(client, v)
	if err != nil {
		return err
	}
	err = client.CoreV1().PersistentVolumes().Delete(volumeName(name), &metav1.DeleteOptions{
		PropagationPolicy: propagationPtr(metav1.DeletePropagationForeground),
	})
	if err != nil && !k8sErrors.IsNotFound(err) {
		return errors.WithStack(err)
	}
	err = client.CoreV1().PersistentVolumeClaims(namespace).Delete(volumeClaimName(name), &metav1.DeleteOptions{
		PropagationPolicy: propagationPtr(metav1.DeletePropagationForeground),
	})
	if err != nil && !k8sErrors.IsNotFound(err) {
		return errors.WithStack(err)
	}
	return nil
}

func createVolume(client *ClusterClient, v *volume.Volume, opts *volumeOptions, app provision.App) error {
	namespace, err := getNamespaceForVolume(client, v)
	if err != nil {
		return err
	}
	labelSet := provision.VolumeLabels(provision.VolumeLabelsOpts{
		Name:        v.Name,
		Provisioner: provisionerName,
		Prefix:      tsuruLabelPrefix,
		Pool:        v.Pool,
		Plan:        v.Plan.Name,
	})
	data, err := json.Marshal(map[string]interface{}{
		opts.Plugin: v.Opts,
	})
	if err != nil {
		return errors.WithStack(err)
	}
	h := &codec.JsonHandle{}
	dec := codec.NewDecoderBytes(data, h)
	pvSpec := apiv1.PersistentVolumeSpec{}
	err = dec.Decode(&pvSpec)
	if err != nil {
		return errors.Wrapf(err, "unable to decode as pv spec: %s", string(data))
	}
	pvSpec.Capacity = apiv1.ResourceList{
		apiv1.ResourceStorage: opts.Capacity,
	}
	for _, am := range strings.Split(opts.AccessModes, ",") {
		pvSpec.AccessModes = append(pvSpec.AccessModes, apiv1.PersistentVolumeAccessMode(am))
	}
	var volName string
	var selector *metav1.LabelSelector
	if opts.Plugin != "" {
		pv := &apiv1.PersistentVolume{
			ObjectMeta: metav1.ObjectMeta{
				Name:   volumeName(v.Name),
				Labels: labelSet.ToLabels(),
			},
			Spec: pvSpec,
		}
		_, err = client.CoreV1().PersistentVolumes().Create(pv)
		if err != nil && !k8sErrors.IsAlreadyExists(err) {
			return errors.WithStack(err)
		}
		selector = &metav1.LabelSelector{
			MatchLabels: labelSet.ToVolumeSelector(),
		}
		volName = volumeName(v.Name)
	}
	pvc := &apiv1.PersistentVolumeClaim{
		ObjectMeta: metav1.ObjectMeta{
			Name:   volumeClaimName(v.Name),
			Labels: labelSet.ToLabels(),
		},
		Spec: apiv1.PersistentVolumeClaimSpec{
			Resources: apiv1.ResourceRequirements{
				Requests: pvSpec.Capacity,
			},
			AccessModes:      pvSpec.AccessModes,
			Selector:         selector,
			VolumeName:       volName,
			StorageClassName: &opts.StorageClass,
		},
	}
	_, err = client.CoreV1().PersistentVolumeClaims(namespace).Create(pvc)
	if err != nil && !k8sErrors.IsAlreadyExists(err) {
		return errors.WithStack(err)
	}
	return nil
}

func volumeExists(client *ClusterClient, name string) (bool, error) {
	v, err := volume.Load(name)
	if err != nil {
		if err == volume.ErrVolumeNotFound {
			return false, nil
		}
		return false, err
	}
	namespace, err := getNamespaceForVolume(client, v)
	if err != nil {
		return false, err
	}
	_, pvErr := client.CoreV1().PersistentVolumes().Get(volumeName(name), metav1.GetOptions{})
	_, pvcErr := client.CoreV1().PersistentVolumeClaims(namespace).Get(volumeClaimName(name), metav1.GetOptions{})
	if k8sErrors.IsNotFound(pvErr) && k8sErrors.IsNotFound(pvcErr) {
		return false, nil
	}
	if pvErr != nil {
		return false, errors.WithStack(pvErr)
	}
	if pvcErr != nil {
		return false, errors.WithStack(pvcErr)
	}
	return true, nil
}

func getNamespaceForVolume(client *ClusterClient, v *volume.Volume) (string, error) {
	binds, err := v.LoadBinds()
	if err != nil {
		return "", err
	}
	if len(binds) == 0 {
		return client.PoolNamespace(v.Pool), nil
	}
	var namespace string
	for _, b := range binds {
		ns, err := client.appNamespaceByName(b.ID.App)
		if err != nil {
			return "", err
		}
		if namespace == "" {
			namespace = ns
			continue
		}
		if ns != namespace {
			return "", errors.New("multiple namespaces for volume")
		}
	}
	return namespace, nil
}
<commit_msg>provision/kubernetes: fix volume create when no plugin is specified<commit_after>// Copyright 2017 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package kubernetes

import (
	"encoding/json"
	"strings"

	"github.com/pkg/errors"
	"github.com/tsuru/tsuru/provision"
	"github.com/tsuru/tsuru/set"
	"github.com/tsuru/tsuru/volume"
	"github.com/ugorji/go/codec"
	apiv1 "k8s.io/api/core/v1"
	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type volumeOptions struct {
	Plugin       string
	StorageClass string `json:"storage-class"`
	Capacity     resource.Quantity
	AccessModes  string `json:"access-modes"`
}

var allowedNonPersistentVolumes = set.FromValues("emptyDir")

func (opts *volumeOptions) isPersistent() bool {
	return !allowedNonPersistentVolumes.Includes(opts.Plugin)
}

func createVolumesForApp(client *ClusterClient, app provision.App) ([]apiv1.Volume, []apiv1.VolumeMount, error) {
	volumes, err := volume.ListByApp(app.GetName())
	if err != nil {
		return nil, nil, errors.WithStack(err)
	}
	var kubeVolumes []apiv1.Volume
	var kubeMounts []apiv1.VolumeMount
	for i := range volumes {
		opts, err := validateVolume(&volumes[i])
		if err != nil {
			return nil, nil, err
		}
		if opts.isPersistent() {
			err = createVolume(client, &volumes[i], opts, app)
			if err != nil {
				return nil, nil, err
			}
		}
		volume, mounts, err := bindsForVolume(&volumes[i], opts, app.GetName())
		if err != nil {
			return nil, nil, err
		}
		kubeMounts = append(kubeMounts, mounts...)
		kubeVolumes = append(kubeVolumes, *volume)
	}
	return kubeVolumes, kubeMounts, nil
}

func bindsForVolume(v *volume.Volume, opts *volumeOptions, appName string) (*apiv1.Volume, []apiv1.VolumeMount, error) {
	var kubeMounts []apiv1.VolumeMount
	binds, err := v.LoadBindsForApp(appName)
	if err != nil {
		return nil, nil, errors.WithStack(err)
	}
	allReadOnly := true
	for _, b := range binds {
		kubeMounts = append(kubeMounts, apiv1.VolumeMount{
			Name:      volumeName(v.Name),
			MountPath: b.ID.MountPoint,
			ReadOnly:  b.ReadOnly,
		})
		if !b.ReadOnly {
			allReadOnly = false
		}
	}
	kubeVol := apiv1.Volume{
		Name: volumeName(v.Name),
	}
	if opts.isPersistent() {
		kubeVol.VolumeSource = apiv1.VolumeSource{
			PersistentVolumeClaim: &apiv1.PersistentVolumeClaimVolumeSource{
				ClaimName: volumeClaimName(v.Name),
				ReadOnly:  allReadOnly,
			},
		}
	} else {
		kubeVol.VolumeSource, err = nonPersistentVolume(v, opts)
		if err != nil {
			return nil, nil, err
		}
	}
	return &kubeVol, kubeMounts, nil
}

func nonPersistentVolume(v *volume.Volume, opts *volumeOptions) (apiv1.VolumeSource, error) {
	var volumeSrc apiv1.VolumeSource
	data, err := json.Marshal(map[string]interface{}{
		opts.Plugin: v.Opts,
	})
	if err != nil {
		return volumeSrc, errors.WithStack(err)
	}
	h := &codec.JsonHandle{}
	dec := codec.NewDecoderBytes(data, h)
	err = dec.Decode(&volumeSrc)
	if err != nil {
		return volumeSrc, errors.WithStack(err)
	}
	return volumeSrc, nil
}

func validateVolume(v *volume.Volume) (*volumeOptions, error) {
	var opts volumeOptions
	err := v.UnmarshalPlan(&opts)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	if opts.Plugin != "" && opts.StorageClass != "" {
		return nil, errors.New("both volume plan plugin and storage-class cannot be set")
	}
	if opts.Plugin == "" && opts.StorageClass == "" {
		return nil, errors.New("both volume plan plugin and storage-class are empty")
	}
	if !opts.isPersistent() {
		return &opts, nil
	}
	if capRaw, ok := v.Opts["capacity"]; ok {
		delete(v.Opts, "capacity")
		opts.Capacity, err = resource.ParseQuantity(capRaw)
		if err != nil {
			return nil, errors.Wrap(err, "unable to parse `capacity` opt")
		}
	}
	if opts.Capacity.IsZero() {
		return nil, errors.New("capacity is mandatory either in plan or as volume opts")
	}
	if accessModesRaw, ok := v.Opts["access-modes"]; ok {
		delete(v.Opts, "access-modes")
		opts.AccessModes = accessModesRaw
	}
	if opts.AccessModes == "" {
		return nil, errors.New("access-modes is mandatory either in plan or as volume opts")
	}
	return &opts, nil
}

func deleteVolume(client *ClusterClient, name string) error {
	v, err := volume.Load(name)
	if err != nil {
		if err == volume.ErrVolumeNotFound {
			return nil
		}
		return err
	}
	namespace, err := getNamespaceForVolume(client, v)
	if err != nil {
		return err
	}
	err = client.CoreV1().PersistentVolumes().Delete(volumeName(name), &metav1.DeleteOptions{
		PropagationPolicy: propagationPtr(metav1.DeletePropagationForeground),
	})
	if err != nil && !k8sErrors.IsNotFound(err) {
		return errors.WithStack(err)
	}
	err = client.CoreV1().PersistentVolumeClaims(namespace).Delete(volumeClaimName(name), &metav1.DeleteOptions{
		PropagationPolicy: propagationPtr(metav1.DeletePropagationForeground),
	})
	if err != nil && !k8sErrors.IsNotFound(err) {
		return errors.WithStack(err)
	}
	return nil
}

func createVolume(client *ClusterClient, v *volume.Volume, opts *volumeOptions, app provision.App) error {
	namespace, err := getNamespaceForVolume(client, v)
	if err != nil {
		return err
	}
	labelSet := provision.VolumeLabels(provision.VolumeLabelsOpts{
		Name:        v.Name,
		Provisioner: provisionerName,
		Prefix:      tsuruLabelPrefix,
		Pool:        v.Pool,
		Plan:        v.Plan.Name,
	})
	capacity := apiv1.ResourceList{
		apiv1.ResourceStorage: opts.Capacity,
	}
	var accessModes []apiv1.PersistentVolumeAccessMode
	for _, am := range strings.Split(opts.AccessModes, ",") {
		accessModes = append(accessModes, apiv1.PersistentVolumeAccessMode(am))
	}
	var volName string
	var selector *metav1.LabelSelector
	if opts.Plugin != "" {
		pvSpec := apiv1.PersistentVolumeSpec{}
		var data []byte
		data, err = json.Marshal(map[string]interface{}{
			opts.Plugin: v.Opts,
		})
		if err != nil {
			return errors.WithStack(err)
		}
		h := &codec.JsonHandle{}
		dec := codec.NewDecoderBytes(data, h)
		err = dec.Decode(&pvSpec)
		if err != nil {
			return errors.Wrapf(err, "unable to decode as pv spec: %s", string(data))
		}
		pvSpec.Capacity = capacity
		pvSpec.AccessModes = accessModes
		pv := &apiv1.PersistentVolume{
			ObjectMeta: metav1.ObjectMeta{
				Name:   volumeName(v.Name),
				Labels: labelSet.ToLabels(),
			},
			Spec: pvSpec,
		}
		_, err = client.CoreV1().PersistentVolumes().Create(pv)
		if err != nil && !k8sErrors.IsAlreadyExists(err) {
			return errors.WithStack(err)
		}
		selector = &metav1.LabelSelector{
			MatchLabels: labelSet.ToVolumeSelector(),
		}
		volName = volumeName(v.Name)
	}
	pvc := &apiv1.PersistentVolumeClaim{
		ObjectMeta: metav1.ObjectMeta{
			Name:   volumeClaimName(v.Name),
			Labels: labelSet.ToLabels(),
		},
		Spec: apiv1.PersistentVolumeClaimSpec{
			Resources: apiv1.ResourceRequirements{
				Requests: capacity,
			},
			AccessModes:      accessModes,
			Selector:         selector,
			VolumeName:       volName,
			StorageClassName: &opts.StorageClass,
		},
	}
	_, err = client.CoreV1().PersistentVolumeClaims(namespace).Create(pvc)
	if err != nil && !k8sErrors.IsAlreadyExists(err) {
		return errors.WithStack(err)
	}
	return nil
}

func volumeExists(client *ClusterClient, name string) (bool, error) {
	v, err := volume.Load(name)
	if err != nil {
		if err == volume.ErrVolumeNotFound {
			return false, nil
		}
		return false, err
	}
	namespace, err := getNamespaceForVolume(client, v)
	if err != nil {
		return false, err
	}
	_, pvErr := client.CoreV1().PersistentVolumes().Get(volumeName(name), metav1.GetOptions{})
	_, pvcErr := client.CoreV1().PersistentVolumeClaims(namespace).Get(volumeClaimName(name), metav1.GetOptions{})
	if k8sErrors.IsNotFound(pvErr) && k8sErrors.IsNotFound(pvcErr) {
		return false, nil
	}
	if pvErr != nil {
		return false, errors.WithStack(pvErr)
	}
	if pvcErr != nil {
		return false, errors.WithStack(pvcErr)
	}
	return true, nil
}

func getNamespaceForVolume(client *ClusterClient, v *volume.Volume) (string, error) {
	binds, err := v.LoadBinds()
	if err != nil {
		return "", err
	}
	if len(binds) == 0 {
		return client.PoolNamespace(v.Pool), nil
	}
	var namespace string
	for _, b := range binds {
		ns, err := client.appNamespaceByName(b.ID.App)
		if err != nil {
			return "", err
		}
		if namespace == "" {
			namespace = ns
			continue
		}
		if ns != namespace {
			return "", errors.New("multiple namespaces for volume")
		}
	}
	return namespace, nil
}
<|endoftext|>


<commit_before>package build

import (
	"os"
	"path/filepath"

	"github.com/sirupsen/logrus"
)

func (b *Builder) prepTarget() error {
	// Create output directory if not exists
	outputDir := filepath.Join(b.Root, filepath.Dir(b.Bin))
	logrus.Debugf("preparing target dir %s", outputDir)
	os.RemoveAll(outputDir)

	if _, err := os.Stat(outputDir); os.IsNotExist(err) {
		os.MkdirAll(outputDir, 0776)
		logrus.Debugf("creating target dir %s", outputDir)
	}
	return nil
}
<commit_msg>running `buffalo build` with `-o` flag deletes working directory fixes #827<commit_after>package build

import (
	"os"
	"path/filepath"

	"github.com/sirupsen/logrus"
)

func (b *Builder) prepTarget() error {
	// Create output directory if not exists
	outputDir := filepath.Join(b.Root, filepath.Dir(b.Bin))
	if _, err := os.Stat(outputDir); os.IsNotExist(err) {
		os.MkdirAll(outputDir, 0776)
		logrus.Debugf("creating target dir %s", outputDir)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright © 2017 Michael Lihs
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package client_test

import (
	//. "github.com/michaellihs/golab/client"

	. "github.com/onsi/ginkgo"
	//. "github.com/onsi/gomega"
)

var _ = Describe("GitlabClient", func() {

})
<commit_msg>deleted unused client folder<commit_after><|endoftext|>


<commit_before>package main

import (
    "bufio"
    "fmt"
    "net/http"
    "strings"
    "time"
)

func main() {
    status()
}

func status() {
    host := "http://127.0.0.1:19071"
    path := "/ApplicationStatus"
    description := "Config server"
    //response := request(host, path, description)

    client := &http.Client{
	    Timeout: time.Second * 30,
    }
    resp, err := client.Get(host + path)
    if err != nil {
        error("Could not connect to", strings.ToLower(description), "at", host)
        detail(err.Error())
        return
    }
    defer resp.Body.Close()

    scanner := bufio.NewScanner(resp.Body)

    if err := scanner.Err(); err != nil {
        error("Error reading data from", strings.ToLower(description), "at", host)
        detail(err.Error())
    } else if resp.StatusCode != 200 {
        error(description, "at", host, "is not ready")
        detail("Response status:", resp.Status)
    } else {
        success(description, "at", host, "is ready")
    }
}

func request(host string, path string, description string) {
}

func error(messages ...string) {
    print("\033[31m", messages)
}

func success(messages ...string) {
    print("\033[32m", messages)
}

func detail(messages ...string) {
    print("\033[33m", messages)
}

func print(prefix string, messages []string) {
   fmt.Print(prefix)
    for i := 0; i < len(messages); i++ {
        fmt.Print(messages[i])
        fmt.Print(" ")
    }
    fmt.Println("")
}
<commit_msg>Extract http get error handling<commit_after>package main

import (
    "bufio"
    "fmt"
    "net/http"
    "strings"
    "time"
)

func main() {
    status()
}

func status() {
    host := "http://127.0.0.1:19071"
    path := "/ApplicationStatus"
    description := "Config server"
    response := request(host, path, description)
    if (response == nil) {
        return
    }

    if response.StatusCode != 200 {
        printError(description, "at", host, "is not ready")
        printDetail("Response status:", response.Status)
    } else {
        printSuccess(description, "at", host, "is ready")
    }
}

func request(host string, path string, description string) (response *http.Response) {
    client := &http.Client{
	    Timeout: time.Second * 30,
    }
    response, error := client.Get(host + path)
    if error != nil {
        printError("Could not connect to", strings.ToLower(description), "at", host)
        printDetail(error.Error())
        return
    }
    defer response.Body.Close()

    scanner := bufio.NewScanner(response.Body)

    if error := scanner.Err(); error != nil {
        printError("Error reading data from", strings.ToLower(description), "at", host)
        printDetail(error.Error())
        return nil
    } else {
        return response
    }
}

func printError(messages ...string) {
    print("\033[31m", messages)
}

func printSuccess(messages ...string) {
    print("\033[32m", messages)
}

func printDetail(messages ...string) {
    print("\033[33m", messages)
}

func print(prefix string, messages []string) {
   fmt.Print(prefix)
    for i := 0; i < len(messages); i++ {
        fmt.Print(messages[i])
        fmt.Print(" ")
    }
    fmt.Println("")
}
<|endoftext|>