<commit_before>package commands

import (
	"fmt"
	"os"
	"log"
	"net/url"

	"github.com/andygrunwald/perseus/config"
	"github.com/andygrunwald/perseus/downloader"
	"github.com/andygrunwald/perseus/packagist"
	"github.com/andygrunwald/perseus/perseus"
)

// AddCommand reflects the business logic and the Command interface to add a new package.
// This command is independent from an human interface (CLI, HTTP, etc.)
// The human interfaces will interact with this command.
type AddCommand struct {
	// WithDependencies decides if the dependencies of an external package needs to be mirrored as well
	WithDependencies bool
	// Package is the package to mirror
	Package string
	// Config is the main medusa configuration
	Config *config.Medusa
	// Log represents a logger to log messages
	Log *log.Logger
}

// downloadResult represents the result of a download
type downloadResult struct {
	Package string
	Error   error
}

// Run is the business logic of AddCommand.
func (c *AddCommand) Run() error {
	c.Log.Printf("Running \"add\" command for package \"%s\"", c.Package)
	p, err := perseus.NewPackage(c.Package)
	if err != nil {
		return err
	}

	// We don't respect the error here.
	// OH: "WTF? Why? You claim 'Serious error handling' in the README!"
	// Yep, you are right. And we still do.
	// In this case, it is okay, if p is not configured or no repositories are configured at all.
	// When this happen, we will ask Packagist fot the repository url.
	// If this package is not available on packagist, this will be shift to an error.
	p.Repository, _ = c.Config.GetRepositoryURLOfPackage(p)
	if p.Repository == nil {

		dependencies := []*perseus.Package{p}
		if c.WithDependencies {
			pUrl := "https://packagist.org"
			c.Log.Printf("Loading dependencies for package \"%s\" from %s", c.Package, pUrl)
			packagistClient, err := packagist.New(pUrl, nil)
			if err != nil {
				return err
			}

			// TODO Okay, here we don't take error handling serious.
			//	Why? Easy. If an API request fails, we don't know it.
			//	Why? Easy. Which packages will be skipped? e.g. "php" ?
			//	We really have to refactor this. Checkout the articles / links
			//	That are mentioned IN the depdency resolver comments
			//	But you know. 1. Make it work. 2. Make it fast. 3. Make it beautiful
			// 	And this works for now.
			d := perseus.NewDependencyResolver(p.Name, packagistClient)
			dependencies = d.Resolve()
			c.Log.Printf("%d dependencies found for package \"%s\" on %s", len(dependencies), c.Package, pUrl)
		}

		// Download package incl. dependencies concurrent
		dependencyCount := len(dependencies)
		downloadsChan := make(chan downloadResult, dependencyCount)
		c.startConcurrentDownloads(dependencies, downloadsChan)

		// Check which dependencies where download successful and which not
		c.processFinishedDownloads(downloadsChan, dependencyCount)
		close(downloadsChan)

	} else {
		c.Log.Printf("Mirroring of package \"%s\" from repository \"%s\" started", p.Name, p.Repository)
		// TODO: downloadPackage will write to p (name + Repository url), we should test this with a package that is deprecated.
		// Afaik Packagist will forward you to the new one.
		// Facebook SDK is one of those
		err := c.downloadPackage(p)
		if err != nil {
			return err
		}
		c.Log.Printf("Mirroring of package \"%s\" successful", p.Name)

		// TODO updateSatisConfig(packet)
	}

	// TODO Implement everything and remove this
	c.Log.Println("=============================")
	c.Log.Println("Add command runs successful. Fuck Yeah!")
	c.Log.Println("Important: This command is not complete yet. Write command of Satis configuration is missing.")
	c.Log.Println("=============================")

	return nil
}

func (c *AddCommand) downloadPackage(p *perseus.Package) error {
	repoDir := c.Config.GetString("repodir")
	// TODO Path traversal in p.Name possible?
	targetDir := fmt.Sprintf("%s/%s.git", repoDir, p.Name)

	// Does targetDir already exist?
	if _, err := os.Stat(targetDir); err != nil {
		if os.IsExist(err) {
			return fmt.Errorf("The repository %s already exists in %s. Try updating it instead.", p.Name, targetDir)
		}
	}

	if p.Repository == nil {
		packagistClient, err := packagist.New("https://packagist.org/", nil)
		if err != nil {
			return fmt.Errorf("Packagist client creation failed: %s", err)
		}
		packagistPackage, resp, err := packagistClient.GetPackage(p.Name)
		if err != nil {
			return fmt.Errorf("Failed to retrieve information about package \"%s\" from Packagist. Called %s. Error: %s", p.Name, resp.Request.URL.String(), err)
		}

		// Check if URL is empty
		if len(packagistPackage.Repository) == 0 {
			// TODO What happens if Packagist rewrite the package? E.g. the facebook example? We should output here both names
			return fmt.Errorf("Received empty URL for package %s from Packagist", p.Name)
		}

		// Overwriting values from Packagist
		p.Name = packagistPackage.Name
		u, err := url.Parse(packagistPackage.Repository)
		if err != nil {
			return fmt.Errorf("URL conversion of %s to a net/url.URL object failed: %s", packagistPackage.Repository, err)
		}
		p.Repository = u
	}

	downloadClient, err := downloader.NewGit(p.Repository.String())
	if err != nil {
		return fmt.Errorf("Downloader client creation failed for package %s: %s", p.Name, err)
	}
	return downloadClient.Download(targetDir)
}

func (c *AddCommand) startConcurrentDownloads(dependencies []*perseus.Package, downloadChan chan<- downloadResult) {
	// Loop over all dependencies and download them concurrent
	for _, packet := range dependencies {
		c.Log.Printf("Mirroring of package \"%s\" started", packet.Name)

		go func(singlePacket *perseus.Package, ch chan<- downloadResult) {
			err := c.downloadPackage(singlePacket)
			if err != nil {
				ch <- downloadResult{
					Package: singlePacket.Name,
					Error:   err,
				}
				return
			}

			// Successful result
			ch <- downloadResult{
				Package: singlePacket.Name,
				Error:   nil,
			}
			// TODO updateSatisConfig(packet) per package
		}(packet, downloadChan)
	}
}

func (c *AddCommand) processFinishedDownloads(ch <-chan downloadResult, dependencyCount int) {
	for i:= 0; i < dependencyCount; i++ {
		download := <-ch
		if download.Error == nil {
			c.Log.Printf("Mirroring of package \"%s\" successful", download.Package)
		} else {
			c.Log.Printf("Error while mirroring package \"%s\": %s", download.Package, download.Error)
		}
	}
}
<commit_msg>Add handling for "Folder already exists" situations. In those cases we will skip the downloading process for this package<commit_after>package commands

import (
	"fmt"
	"os"
	"log"
	"net/url"

	"github.com/andygrunwald/perseus/config"
	"github.com/andygrunwald/perseus/downloader"
	"github.com/andygrunwald/perseus/packagist"
	"github.com/andygrunwald/perseus/perseus"
)

// AddCommand reflects the business logic and the Command interface to add a new package.
// This command is independent from an human interface (CLI, HTTP, etc.)
// The human interfaces will interact with this command.
type AddCommand struct {
	// WithDependencies decides if the dependencies of an external package needs to be mirrored as well
	WithDependencies bool
	// Package is the package to mirror
	Package string
	// Config is the main medusa configuration
	Config *config.Medusa
	// Log represents a logger to log messages
	Log *log.Logger
}

// downloadResult represents the result of a download
type downloadResult struct {
	Package string
	Error   error
}

// Run is the business logic of AddCommand.
func (c *AddCommand) Run() error {
	c.Log.Printf("Running \"add\" command for package \"%s\"", c.Package)
	p, err := perseus.NewPackage(c.Package)
	if err != nil {
		return err
	}

	// We don't respect the error here.
	// OH: "WTF? Why? You claim 'Serious error handling' in the README!"
	// Yep, you are right. And we still do.
	// In this case, it is okay, if p is not configured or no repositories are configured at all.
	// When this happen, we will ask Packagist fot the repository url.
	// If this package is not available on packagist, this will be shift to an error.
	p.Repository, _ = c.Config.GetRepositoryURLOfPackage(p)
	if p.Repository == nil {

		dependencies := []*perseus.Package{p}
		if c.WithDependencies {
			pUrl := "https://packagist.org"
			c.Log.Printf("Loading dependencies for package \"%s\" from %s", c.Package, pUrl)
			packagistClient, err := packagist.New(pUrl, nil)
			if err != nil {
				return err
			}

			// TODO Okay, here we don't take error handling serious.
			//	Why? Easy. If an API request fails, we don't know it.
			//	Why? Easy. Which packages will be skipped? e.g. "php" ?
			//	We really have to refactor this. Checkout the articles / links
			//	That are mentioned IN the depdency resolver comments
			//	But you know. 1. Make it work. 2. Make it fast. 3. Make it beautiful
			// 	And this works for now.
			d := perseus.NewDependencyResolver(p.Name, packagistClient)
			dependencies = d.Resolve()
			c.Log.Printf("%d dependencies found for package \"%s\" on %s", len(dependencies), c.Package, pUrl)
		}

		// Download package incl. dependencies concurrent
		dependencyCount := len(dependencies)
		downloadsChan := make(chan downloadResult, dependencyCount)
		c.startConcurrentDownloads(dependencies, downloadsChan)

		// Check which dependencies where download successful and which not
		c.processFinishedDownloads(downloadsChan, dependencyCount)
		close(downloadsChan)

	} else {
		c.Log.Printf("Mirroring of package \"%s\" from repository \"%s\" started", p.Name, p.Repository)
		// TODO: downloadPackage will write to p (name + Repository url), we should test this with a package that is deprecated.
		// Afaik Packagist will forward you to the new one.
		// Facebook SDK is one of those
		err := c.downloadPackage(p)
		if err != nil {
			return err
		}
		c.Log.Printf("Mirroring of package \"%s\" successful", p.Name)

		// TODO updateSatisConfig(packet)
	}

	// TODO Implement everything and remove this
	c.Log.Println("=============================")
	c.Log.Println("Add command runs successful. Fuck Yeah!")
	c.Log.Println("Important: This command is not complete yet. Write command of Satis configuration is missing.")
	c.Log.Println("=============================")

	return nil
}

func (c *AddCommand) downloadPackage(p *perseus.Package) error {
	repoDir := c.Config.GetString("repodir")
	// TODO Path traversal in p.Name possible?
	targetDir := fmt.Sprintf("%s/%s.git", repoDir, p.Name)

	// Does targetDir already exist?
	if _, err := os.Stat(targetDir); err != nil {
		return err
		if !os.IsNotExist(err) {
			return err
		}
	} else {
		return os.ErrExist
	}

	if p.Repository == nil {
		packagistClient, err := packagist.New("https://packagist.org/", nil)
		if err != nil {
			return fmt.Errorf("Packagist client creation failed: %s", err)
		}
		packagistPackage, resp, err := packagistClient.GetPackage(p.Name)
		if err != nil {
			return fmt.Errorf("Failed to retrieve information about package \"%s\" from Packagist. Called %s. Error: %s", p.Name, resp.Request.URL.String(), err)
		}

		// Check if URL is empty
		if len(packagistPackage.Repository) == 0 {
			// TODO What happens if Packagist rewrite the package? E.g. the facebook example? We should output here both names
			return fmt.Errorf("Received empty URL for package %s from Packagist", p.Name)
		}

		// Overwriting values from Packagist
		p.Name = packagistPackage.Name
		u, err := url.Parse(packagistPackage.Repository)
		if err != nil {
			return fmt.Errorf("URL conversion of %s to a net/url.URL object failed: %s", packagistPackage.Repository, err)
		}
		p.Repository = u
	}

	downloadClient, err := downloader.NewGit(p.Repository.String())
	if err != nil {
		return fmt.Errorf("Downloader client creation failed for package %s: %s", p.Name, err)
	}
	return downloadClient.Download(targetDir)
}

func (c *AddCommand) startConcurrentDownloads(dependencies []*perseus.Package, downloadChan chan<- downloadResult) {
	// Loop over all dependencies and download them concurrent
	for _, packet := range dependencies {
		c.Log.Printf("Mirroring of package \"%s\" started", packet.Name)

		go func(singlePacket *perseus.Package, ch chan<- downloadResult) {
			err := c.downloadPackage(singlePacket)
			if err != nil {
				ch <- downloadResult{
					Package: singlePacket.Name,
					Error:   err,
				}
				return
			}

			// Successful result
			ch <- downloadResult{
				Package: singlePacket.Name,
				Error:   nil,
			}
			// TODO updateSatisConfig(packet) per package
		}(packet, downloadChan)
	}
}

func (c *AddCommand) processFinishedDownloads(ch <-chan downloadResult, dependencyCount int) {
	for i:= 0; i < dependencyCount; i++ {
		download := <-ch
		if download.Error == nil {
			c.Log.Printf("Mirroring of package \"%s\" successful", download.Package)
		} else {
			if os.IsExist(download.Error) {
				c.Log.Printf("Package \"%s\" exists on disk. Try updating it instead. Skipping.", download.Package)
			} else {
				c.Log.Printf("Error while mirroring package \"%s\": %s", download.Package, download.Error)
			}
		}
	}
}
<|endoftext|>


<commit_before>package queue_test

import (
	. "github.com/alphagov/govuk_crawler_worker/queue"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"net/url"

	"github.com/alphagov/govuk_crawler_worker/util"
	"github.com/streadway/amqp"
)

var _ = Describe("QueueConnection", func() {
	amqpAddr := util.GetEnvDefault("AMQP_ADDRESS", "amqp://guest:guest@localhost:5672/")

	It("fails if it can't connect to an AMQP server", func() {
		connection, err := NewQueueConnection("amqp://guest:guest@localhost:50000/")

		Expect(err).ToNot(BeNil())
		Expect(connection).To(BeNil())
	})

	Describe("Connection errors", func() {
		var (
			connection       *QueueConnection
			proxy            *util.ProxyTCP
			proxyAddr        string           = "localhost:5673"
			queueName        string           = "govuk_crawler_worker-test-crawler-queue"
			fatalErrs        chan *amqp.Error = make(chan *amqp.Error)
			channelCloseMsgs chan string      = make(chan string)
		)

		BeforeEach(func() {
			proxyDest, err := addrFromURL(amqpAddr)
			Expect(err).To(BeNil())
			proxyURL, err := urlChangeAddr(amqpAddr, proxyAddr)
			Expect(err).To(BeNil())

			proxy, err = util.NewProxyTCP(proxyAddr, proxyDest)
			Expect(err).To(BeNil())
			Expect(proxy).ToNot(BeNil())

			connection, err = NewQueueConnection(proxyURL)
			Expect(err).To(BeNil())
			Expect(connection).ToNot(BeNil())

			connection.HandleFatalError = func(err *amqp.Error) {
				fatalErrs <- err
			}

			connection.HandleChannelClose = func(message string) {
				channelCloseMsgs <- message
			}

			_, err = connection.QueueDeclare(queueName)
			Expect(err).To(BeNil())
		})

		AfterEach(func() {
			defer connection.Close()
			defer proxy.Close()

			// Assume existing connection is dead.
			connection.Close()
			connection, _ = NewQueueConnection(amqpAddr)

			deleted, err := connection.Channel.QueueDelete(queueName, false, false, false)
			Expect(err).To(BeNil())
			Expect(deleted).To(Equal(0))
		})

		It("should call connection.HandleChannelClose() on recoverable errors", func(done Done) {
			connection.Channel.Close()

			// check connection.HandleChannelClose is called
			message := <-channelCloseMsgs
			Expect(message).To(Equal("Channel closed"))

			// Connection no longer works
			_, err := connection.Channel.QueueInspect(queueName)
			Expect(err).To(Equal(amqp.ErrClosed))

			close(done)
		})

		It("should exit on non-recoverable errors", func(done Done) {
			const expectedError = "Exception \\(501\\) Reason: \"EOF\"|connection reset by peer"

			proxy.KillConnected()

			_, err := connection.Channel.QueueInspect(queueName)
			Expect(err.Error()).To(MatchRegexp(expectedError))

			// We'd normally log.Fatalln() here to exit.
			amqpErr := <-fatalErrs
			Expect(amqpErr.Error()).To(MatchRegexp(expectedError))
			Expect(amqpErr.Recover).To(Equal(false))

			// Connection no longer works.
			_, err = connection.Channel.QueueInspect(queueName)
			Expect(err).To(Equal(amqp.ErrClosed))

			close(done)
		})
	})

	Describe("Connecting to a running AMQP service", func() {
		var (
			connection    *QueueConnection
			connectionErr error
		)

		BeforeEach(func() {
			connection, connectionErr = NewQueueConnection(amqpAddr)
			connection.HandleChannelClose = func(_ string) {}
		})

		AfterEach(func() {
			defer connection.Close()
		})

		It("successfully connects to an AMQP service", func() {
			Expect(connectionErr).To(BeNil())
			Expect(connection).ToNot(BeNil())
		})

		It("can close the connection without errors", func() {
			Expect(connection.Close()).To(BeNil())
		})

		It("can declare an exchange", func() {
			var err error
			exchange := "govuk_crawler_worker-some-exchange"

			err = connection.ExchangeDeclare(exchange, "direct")
			Expect(err).To(BeNil())

			err = connection.Channel.ExchangeDelete(exchange, false, false)
			Expect(err).To(BeNil())
		})

		It("can declare a queue", func() {
			var (
				err   error
				queue amqp.Queue
				name  = "govuk_crawler_worker-some-queue"
			)

			queue, err = connection.QueueDeclare(name)
			Expect(err).To(BeNil())
			Expect(queue.Name).To(Equal(name))

			deleted, err := connection.Channel.QueueDelete(name, false, false, false)
			Expect(err).To(BeNil())
			Expect(deleted).To(Equal(0))
		})

		It("can bind a queue to an exchange", func() {
			var err error

			exchangeName := "govuk_crawler_worker-some-binding-exchange"
			queueName := "govuk_crawler_worker-some-binding-queue"

			err = connection.ExchangeDeclare(exchangeName, "direct")
			Expect(err).To(BeNil())

			_, err = connection.QueueDeclare(queueName)
			Expect(err).To(BeNil())

			err = connection.BindQueueToExchange(queueName, exchangeName)
			Expect(err).To(BeNil())

			deleted, err := connection.Channel.QueueDelete(queueName, false, false, false)
			Expect(err).To(BeNil())
			Expect(deleted).To(Equal(0))

			err = connection.Channel.ExchangeDelete(exchangeName, false, false)
			Expect(err).To(BeNil())
		})
	})

	Describe("working with messages on the queue", func() {
		var (
			publisher *QueueConnection
			consumer  *QueueConnection
			err       error
		)

		exchangeName := "govuk_crawler_worker-test-crawler-exchange"
		queueName := "govuk_crawler_worker-test-crawler-queue"

		BeforeEach(func() {
			publisher, err = NewQueueConnection(amqpAddr)
			Expect(err).To(BeNil())
			Expect(publisher).ToNot(BeNil())

			consumer, err = NewQueueConnection(amqpAddr)
			Expect(err).To(BeNil())
			Expect(consumer).ToNot(BeNil())

			publisher.HandleChannelClose = func(_ string) {}
			consumer.HandleChannelClose = func(_ string) {}
		})

		AfterEach(func() {
			// Consumer must Cancel() or Close() before deleting.
			consumer.Close()
			defer publisher.Close()

			deleted, err := publisher.Channel.QueueDelete(queueName, false, false, false)
			Expect(err).To(BeNil())
			Expect(deleted).To(Equal(0))

			err = publisher.Channel.ExchangeDelete(exchangeName, false, false)
			Expect(err).To(BeNil())
		})

		It("should consume and publish messages onto the provided queue and exchange", func(done Done) {
			err = consumer.ExchangeDeclare(exchangeName, "direct")
			Expect(err).To(BeNil())

			_, err = consumer.QueueDeclare(queueName)
			Expect(err).To(BeNil())

			err = consumer.BindQueueToExchange(queueName, exchangeName)
			Expect(err).To(BeNil())

			deliveries, err := consumer.Consume(queueName)
			Expect(err).To(BeNil())

			err = publisher.Publish(exchangeName, "#", "text/plain", "foo")
			Expect(err).To(BeNil())

			item := <-deliveries
			Expect(string(item.Body)).To(Equal("foo"))
			item.Ack(false)
			close(done)
		})
	})
})

// addrFromURL extracts the addr (host:port) from a URL string.
func addrFromURL(URL string) (string, error) {
	parsedURL, err := url.Parse(URL)
	if err != nil {
		return "", err
	}

	return parsedURL.Host, nil
}

// urlChangeAddr changes the addr (host:port) of a URL string.
func urlChangeAddr(origURL, newHost string) (string, error) {
	parsedURL, err := url.Parse(origURL)
	if err != nil {
		return "", err
	}

	parsedURL.Host = newHost
	return parsedURL.String(), nil
}
<commit_msg>Prefer BeTrue(), BeFalse() to Equal() for booleans<commit_after>package queue_test

import (
	. "github.com/alphagov/govuk_crawler_worker/queue"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"net/url"

	"github.com/alphagov/govuk_crawler_worker/util"
	"github.com/streadway/amqp"
)

var _ = Describe("QueueConnection", func() {
	amqpAddr := util.GetEnvDefault("AMQP_ADDRESS", "amqp://guest:guest@localhost:5672/")

	It("fails if it can't connect to an AMQP server", func() {
		connection, err := NewQueueConnection("amqp://guest:guest@localhost:50000/")

		Expect(err).ToNot(BeNil())
		Expect(connection).To(BeNil())
	})

	Describe("Connection errors", func() {
		var (
			connection       *QueueConnection
			proxy            *util.ProxyTCP
			proxyAddr        string           = "localhost:5673"
			queueName        string           = "govuk_crawler_worker-test-crawler-queue"
			fatalErrs        chan *amqp.Error = make(chan *amqp.Error)
			channelCloseMsgs chan string      = make(chan string)
		)

		BeforeEach(func() {
			proxyDest, err := addrFromURL(amqpAddr)
			Expect(err).To(BeNil())
			proxyURL, err := urlChangeAddr(amqpAddr, proxyAddr)
			Expect(err).To(BeNil())

			proxy, err = util.NewProxyTCP(proxyAddr, proxyDest)
			Expect(err).To(BeNil())
			Expect(proxy).ToNot(BeNil())

			connection, err = NewQueueConnection(proxyURL)
			Expect(err).To(BeNil())
			Expect(connection).ToNot(BeNil())

			connection.HandleFatalError = func(err *amqp.Error) {
				fatalErrs <- err
			}

			connection.HandleChannelClose = func(message string) {
				channelCloseMsgs <- message
			}

			_, err = connection.QueueDeclare(queueName)
			Expect(err).To(BeNil())
		})

		AfterEach(func() {
			defer connection.Close()
			defer proxy.Close()

			// Assume existing connection is dead.
			connection.Close()
			connection, _ = NewQueueConnection(amqpAddr)

			deleted, err := connection.Channel.QueueDelete(queueName, false, false, false)
			Expect(err).To(BeNil())
			Expect(deleted).To(Equal(0))
		})

		It("should call connection.HandleChannelClose() on recoverable errors", func(done Done) {
			connection.Channel.Close()

			// check connection.HandleChannelClose is called
			message := <-channelCloseMsgs
			Expect(message).To(Equal("Channel closed"))

			// Connection no longer works
			_, err := connection.Channel.QueueInspect(queueName)
			Expect(err).To(Equal(amqp.ErrClosed))

			close(done)
		})

		It("should exit on non-recoverable errors", func(done Done) {
			const expectedError = "Exception \\(501\\) Reason: \"EOF\"|connection reset by peer"

			proxy.KillConnected()

			_, err := connection.Channel.QueueInspect(queueName)
			Expect(err.Error()).To(MatchRegexp(expectedError))

			// We'd normally log.Fatalln() here to exit.
			amqpErr := <-fatalErrs
			Expect(amqpErr.Error()).To(MatchRegexp(expectedError))
			Expect(amqpErr.Recover).To(BeFalse())

			// Connection no longer works.
			_, err = connection.Channel.QueueInspect(queueName)
			Expect(err).To(Equal(amqp.ErrClosed))

			close(done)
		})
	})

	Describe("Connecting to a running AMQP service", func() {
		var (
			connection    *QueueConnection
			connectionErr error
		)

		BeforeEach(func() {
			connection, connectionErr = NewQueueConnection(amqpAddr)
			connection.HandleChannelClose = func(_ string) {}
		})

		AfterEach(func() {
			defer connection.Close()
		})

		It("successfully connects to an AMQP service", func() {
			Expect(connectionErr).To(BeNil())
			Expect(connection).ToNot(BeNil())
		})

		It("can close the connection without errors", func() {
			Expect(connection.Close()).To(BeNil())
		})

		It("can declare an exchange", func() {
			var err error
			exchange := "govuk_crawler_worker-some-exchange"

			err = connection.ExchangeDeclare(exchange, "direct")
			Expect(err).To(BeNil())

			err = connection.Channel.ExchangeDelete(exchange, false, false)
			Expect(err).To(BeNil())
		})

		It("can declare a queue", func() {
			var (
				err   error
				queue amqp.Queue
				name  = "govuk_crawler_worker-some-queue"
			)

			queue, err = connection.QueueDeclare(name)
			Expect(err).To(BeNil())
			Expect(queue.Name).To(Equal(name))

			deleted, err := connection.Channel.QueueDelete(name, false, false, false)
			Expect(err).To(BeNil())
			Expect(deleted).To(Equal(0))
		})

		It("can bind a queue to an exchange", func() {
			var err error

			exchangeName := "govuk_crawler_worker-some-binding-exchange"
			queueName := "govuk_crawler_worker-some-binding-queue"

			err = connection.ExchangeDeclare(exchangeName, "direct")
			Expect(err).To(BeNil())

			_, err = connection.QueueDeclare(queueName)
			Expect(err).To(BeNil())

			err = connection.BindQueueToExchange(queueName, exchangeName)
			Expect(err).To(BeNil())

			deleted, err := connection.Channel.QueueDelete(queueName, false, false, false)
			Expect(err).To(BeNil())
			Expect(deleted).To(Equal(0))

			err = connection.Channel.ExchangeDelete(exchangeName, false, false)
			Expect(err).To(BeNil())
		})
	})

	Describe("working with messages on the queue", func() {
		var (
			publisher *QueueConnection
			consumer  *QueueConnection
			err       error
		)

		exchangeName := "govuk_crawler_worker-test-crawler-exchange"
		queueName := "govuk_crawler_worker-test-crawler-queue"

		BeforeEach(func() {
			publisher, err = NewQueueConnection(amqpAddr)
			Expect(err).To(BeNil())
			Expect(publisher).ToNot(BeNil())

			consumer, err = NewQueueConnection(amqpAddr)
			Expect(err).To(BeNil())
			Expect(consumer).ToNot(BeNil())

			publisher.HandleChannelClose = func(_ string) {}
			consumer.HandleChannelClose = func(_ string) {}
		})

		AfterEach(func() {
			// Consumer must Cancel() or Close() before deleting.
			consumer.Close()
			defer publisher.Close()

			deleted, err := publisher.Channel.QueueDelete(queueName, false, false, false)
			Expect(err).To(BeNil())
			Expect(deleted).To(Equal(0))

			err = publisher.Channel.ExchangeDelete(exchangeName, false, false)
			Expect(err).To(BeNil())
		})

		It("should consume and publish messages onto the provided queue and exchange", func(done Done) {
			err = consumer.ExchangeDeclare(exchangeName, "direct")
			Expect(err).To(BeNil())

			_, err = consumer.QueueDeclare(queueName)
			Expect(err).To(BeNil())

			err = consumer.BindQueueToExchange(queueName, exchangeName)
			Expect(err).To(BeNil())

			deliveries, err := consumer.Consume(queueName)
			Expect(err).To(BeNil())

			err = publisher.Publish(exchangeName, "#", "text/plain", "foo")
			Expect(err).To(BeNil())

			item := <-deliveries
			Expect(string(item.Body)).To(Equal("foo"))
			item.Ack(false)
			close(done)
		})
	})
})

// addrFromURL extracts the addr (host:port) from a URL string.
func addrFromURL(URL string) (string, error) {
	parsedURL, err := url.Parse(URL)
	if err != nil {
		return "", err
	}

	return parsedURL.Host, nil
}

// urlChangeAddr changes the addr (host:port) of a URL string.
func urlChangeAddr(origURL, newHost string) (string, error) {
	parsedURL, err := url.Parse(origURL)
	if err != nil {
		return "", err
	}

	parsedURL.Host = newHost
	return parsedURL.String(), nil
}
<|endoftext|>


<commit_before>package chClient

import (
	"fmt"

	"git.containerum.net/ch/kube-client/pkg/cherry"
	"git.containerum.net/ch/kube-client/pkg/cherry/auth"
	"git.containerum.net/ch/kube-client/pkg/cherry/kube-api"
	kubeClientModels "git.containerum.net/ch/kube-client/pkg/model"
	"github.com/containerum/chkit/pkg/chkitErrors"
	"github.com/containerum/chkit/pkg/model/namespace"
)

const (
	// ErrUnableToGetNamespace -- unable to get namespace
	ErrUnableToGetNamespace chkitErrors.Err = "unable to get namespace"
	// ErrYouDoNotHaveAccessToNamespace -- you don't have access to namespace
	ErrYouDoNotHaveAccessToNamespace chkitErrors.Err = "you don't have access to namespace"
	// ErrNamespaceNotExists -- namespace not exists
	ErrNamespaceNotExists chkitErrors.Err = "namespace not exists"
)

// GetNamespace -- returns info of namespace with given label.
// Returns:
// 	- ErrNamespaceNotExists
//  - ErrWrongPasswordLoginCombination
//  - ErrUserNotExist
func (client *Client) GetNamespace(label string) (namespace.Namespace, error) {
	var err error
	var kubeNamespace kubeClientModels.Namespace
	for i := uint(0); i == 0 || (i < 4 && err != nil); i++ {
		kubeNamespace, err = client.kubeAPIClient.GetNamespace(label)
		switch {
		case err == nil:
			break
		case cherry.Equals(err, kubeErrors.ErrResourceNotExist()) ||
			cherry.Equals(err, kubeErrors.ErrAccessError()) ||
			cherry.Equals(err, kubeErrors.ErrUnableGetResource()):
			return namespace.Namespace{}, ErrNamespaceNotExists
		case cherry.Equals(err, autherr.ErrInvalidToken()) ||
			cherry.Equals(err, autherr.ErrTokenNotFound()):
			switch client.Auth() {
			case ErrWrongPasswordLoginCombination, ErrUserNotExist:
				return namespace.Namespace{}, err
			}
		}
		waitNextAttempt(i)
	}
	return namespace.NamespaceFromKube(kubeNamespace), err
}

func (client *Client) GetNamespaceList() (namespace.NamespaceList, error) {
	var err error
	var list []kubeClientModels.Namespace
	for i := uint(0); i == 0 || (i < 4 && err != nil); i++ {
		list, err = client.kubeAPIClient.GetNamespaceList(nil)
		switch {
		case err == nil:
			break
		case cherry.Equals(err, autherr.ErrInvalidToken()) ||
			cherry.Equals(err, autherr.ErrTokenNotFound()):
			fmt.Printf("reauth: %v\n", err)
			err = client.Auth()
			switch err {
			case ErrWrongPasswordLoginCombination, ErrUserNotExist:
				return []namespace.Namespace{}, err
			default:
				fmt.Println(err)
			}
		case cherry.Equals(err, kubeErrors.ErrAccessError()):
			return namespace.NamespaceList{}, ErrYouDoNotHaveAccessToNamespace
		}
		waitNextAttempt(i)
	}
	return namespace.NamespaceListFromKube(list), err
}
<commit_msg>fix errors<commit_after>package chClient

import (
	"fmt"

	"git.containerum.net/ch/kube-client/pkg/cherry"
	"git.containerum.net/ch/kube-client/pkg/cherry/auth"
	"git.containerum.net/ch/kube-client/pkg/cherry/kube-api"
	kubeClientModels "git.containerum.net/ch/kube-client/pkg/model"
	"github.com/containerum/chkit/pkg/chkitErrors"
	"github.com/containerum/chkit/pkg/model/namespace"
)

const (
	// ErrUnableToGetNamespace -- unable to get namespace
	ErrUnableToGetNamespace chkitErrors.Err = "unable to get namespace"
	// ErrYouDoNotHaveAccessToNamespace -- you don't have access to namespace
	ErrYouDoNotHaveAccessToNamespace chkitErrors.Err = "you don't have access to namespace"
	// ErrNamespaceNotExists -- namespace not exists
	ErrNamespaceNotExists chkitErrors.Err = "namespace not exists"
)

// GetNamespace -- returns info of namespace with given label.
// Returns:
// 	- ErrNamespaceNotExists
//  - ErrWrongPasswordLoginCombination
//  - ErrUserNotExist
func (client *Client) GetNamespace(label string) (namespace.Namespace, error) {
	var err error
	var kubeNamespace kubeClientModels.Namespace
retry:
	for i := uint(0); i == 0 || (i < 4 && err != nil); i++ {
		kubeNamespace, err = client.kubeAPIClient.GetNamespace(label)
		switch {
		case err == nil:
			break retry
		case cherry.In(err,
			kubeErrors.ErrResourceNotExist(),
			kubeErrors.ErrAccessError(),
			kubeErrors.ErrUnableGetResource()):
			return namespace.Namespace{}, ErrNamespaceNotExists
		case cherry.In(err, autherr.ErrInvalidToken(),
			autherr.ErrTokenNotFound()):
			if er := client.Auth(); er != nil {
				return namespace.Namespace{}, er
			}
		}
		waitNextAttempt(i)
	}
	return namespace.NamespaceFromKube(kubeNamespace), err
}

func (client *Client) GetNamespaceList() (namespace.NamespaceList, error) {
	var err error
	var list []kubeClientModels.Namespace
	for i := uint(0); i == 0 || (i < 4 && err != nil); i++ {
		list, err = client.kubeAPIClient.GetNamespaceList(nil)
		switch {
		case err == nil:
			break
		case cherry.Equals(err, autherr.ErrInvalidToken()) ||
			cherry.Equals(err, autherr.ErrTokenNotFound()):
			fmt.Printf("reauth: %v\n", err)
			err = client.Auth()
			switch err {
			case ErrWrongPasswordLoginCombination, ErrUserNotExist:
				return []namespace.Namespace{}, err
			default:
				fmt.Println(err)
			}
		case cherry.Equals(err, kubeErrors.ErrAccessError()):
			return namespace.NamespaceList{}, ErrYouDoNotHaveAccessToNamespace
		}
		waitNextAttempt(i)
	}
	return namespace.NamespaceListFromKube(list), err
}
<|endoftext|>


<commit_before>package config

import ()

// SnmpDeviceCfg contains all snmp related device definitions
type SnmpDeviceCfg struct {
	ID string `xorm:"'id' unique" binding:"Required"`
	//snmp connection config
	Host    string `xorm:"host" binding:"Required"`
	Port    int    `xorm:"port" binding:"Required"`
	Retries int    `xorm:"retries"`
	Timeout int    `xorm:"timeout"`
	Repeat  int    `xorm:"repeat"`
	Active  bool   `xorm:"'active' default 1"`
	//snmp auth  config
	SnmpVersion string `xorm:"snmpversion" binding:"Required;In(1,2c,3)"`
	Community   string `xorm:"community"`
	V3SecLevel  string `xorm:"v3seclevel"`
	V3AuthUser  string `xorm:"v3authuser"`
	V3AuthPass  string `xorm:"v3authpass"`
	V3AuthProt  string `xorm:"v3authprot"`
	V3PrivPass  string `xorm:"v3privpass"`
	V3PrivProt  string `xorm:"v3privprot"`
	//snmp workarround for some devices
	DisableBulk    bool  `xorm:"'disablebulk' default 0"`
	MaxRepetitions uint8 `xorm:"'maxrepetitions' default 50" binding:"Default(50);IntegerNotZero"`
	//snmp runtime config
	Freq             int  `xorm:"'freq' default 60" binding:"Default(60);IntegerNotZero"`
	UpdateFltFreq    int  `xorm:"'update_flt_freq' default 60" binding:"Default(60);UIntegerAndLessOne"`
	ConcurrentGather bool `xorm:"'concurrent_gather' default 1" binding:"Default(1)"`

	OutDB    string `xorm:"outdb"`
	LogLevel string `xorm:"loglevel" binding:"Default(info)"`
	LogFile  string `xorm:"logfile"`

	SnmpDebug bool `xorm:"snmpdebug" binding:"Default(0)"`
	//influx tags
	DeviceTagName  string   `xorm:"devicetagname" binding:"Default(hostname)"`
	DeviceTagValue string   `xorm:"devicetagvalue" binding:"Default(id)"`
	ExtraTags      []string `xorm:"extra-tags"`
	Description    string   `xorm:"description"`
	//Filters for measurements
	MeasurementGroups []string `xorm:"-"`
	MeasFilters       []string `xorm:"-"`
}

// InfluxCfg is the main configuration for any InfluxDB TSDB
type InfluxCfg struct {
	ID          string `xorm:"'id' unique" binding:"Required"`
	Host        string `xorm:"host" binding:"Required"`
	Port        int    `xorm:"port" binding:"Required;IntegerNotZero"`
	DB          string `xorm:"db" binding:"Required"`
	User        string `xorm:"user" binding:"Required"`
	Password    string `xorm:"password" binding:"Required"`
	Retention   string `xorm:"'retention' default 'autogen'" binding:"Required"`
	Precision   string `xorm:"'precision' default 's'" binding:"Default(s);OmitEmpty;In(h,m,s,ms,u,ns)"` //posible values [h,m,s,ms,u,ns] default seconds for the nature of data
	Timeout     int    `xorm:"'timeout' default 30" binding:"Default(30);IntegerNotZero"`
	UserAgent   string `xorm:"useragent" binding:"Default(snmpcollector)"`
	Description string `xorm:"description"`
}

//MeasFilterCfg the filter configuration
type MeasFilterCfg struct {
	ID               string `xorm:"'id' unique" binding:"Required"`
	IDMeasurementCfg string `xorm:"id_measurement_cfg"`
	FType            string `xorm:"filter_type" binding:"Required"` //file/OIDCondition/CustomFilter
	FilterName       string `xorm:"filter_name" binding:"Required"` // valid identificator for the filter depending on the type
	EnableAlias      bool   `xorm:"enable_alias"`                   //only valid if file/Custom
	Description      string `xorm:"description"`
}

//MeasurementFieldCfg the metrics contained on each measurement (to initialize on the fieldMetric array)
type MeasurementFieldCfg struct {
	IDMeasurementCfg string `xorm:"id_measurement_cfg"`
	IDMetricCfg      string `xorm:"id_metric_cfg"`
	Report           int    `xorm:"'report' default 1"`
}

// CUSTOM FILTER TYPES

// CustomFilterItems  list of items on each custom filter
type CustomFilterItems struct {
	CustomID string `xorm:"customid"`
	TagID    string `xorm:"tagid"`
	Alias    string `xorm:"alias"`
}

// CustomFilterCfg table with user custom choosed indexes
type CustomFilterCfg struct {
	ID          string `xorm:"'id' unique" binding:"Required"`
	Description string `xorm:"description"`
	RelatedDev  string `xorm:"related_dev"`
	RelatedMeas string `xorm:"related_meas"`
	Items       []struct {
		TagID string
		Alias string
	} `xorm:"-"`
}

// OidConditionCfg condition config for filters and metrics
type OidConditionCfg struct {
	ID          string `xorm:"'id' unique" binding:"Required"`
	IsMultiple  bool   `xorm:"is_multiple"`
	OIDCond     string `xorm:"cond_oid" binding:"Required"`
	CondType    string `xorm:"cond_type"`
	CondValue   string `xorm:"cond_value"`
	Description string `xorm:"description"`
}

//SnmpDevFilters filters to use with indexed measurement
type SnmpDevFilters struct {
	IDSnmpDev string `xorm:"id_snmpdev"`
	IDFilter  string `xorm:"id_filter"`
}

//MGroupsCfg measurement groups to asign to devices
type MGroupsCfg struct {
	ID           string   `xorm:"'id' unique" binding:"Required"`
	Measurements []string `xorm:"-"`
	Description  string   `xorm:"description"`
}

//MGroupsMeasurements measurements contained on each Measurement Group
type MGroupsMeasurements struct {
	IDMGroupCfg      string `xorm:"id_mgroup_cfg"`
	IDMeasurementCfg string `xorm:"id_measurement_cfg"`
}

// SnmpDevMGroups Mgroups defined on each SnmpDevice
type SnmpDevMGroups struct {
	IDSnmpDev   string `xorm:"id_snmpdev"`
	IDMGroupCfg string `xorm:"id_mgroup_cfg"`
}

// SQLConfig read from DB
type SQLConfig struct {
	Metrics      map[string]*SnmpMetricCfg
	Measurements map[string]*MeasurementCfg
	MFilters     map[string]*MeasFilterCfg
	GetGroups    map[string]*MGroupsCfg
	SnmpDevice   map[string]*SnmpDeviceCfg
	Influxdb     map[string]*InfluxCfg
}

/*
initMetricsCfg this function does 2 things
1.- Initialice id from key of maps for all SnmpMetricCfg and InfluxMeasurementCfg objects
2.- Initialice references between InfluxMeasurementCfg and SnmpMetricGfg objects
*/
// InitMetricsCfg xx
func InitMetricsCfg(cfg *SQLConfig) error {
	//TODO:
	// - check duplicates OID's => warning messages
	//Initialize references to SnmpMetricGfg into InfluxMeasurementCfg
	log.Debug("--------------------Initializing Config metrics-------------------")
	log.Debug("Initializing SNMPMetricconfig...")
	for mKey, mVal := range cfg.Metrics {
		err := mVal.Init()
		if err != nil {
			log.Warnln("Error in Metric config:", err)
			//if some error int the format the metric is deleted from the config
			delete(cfg.Metrics, mKey)
		}
	}
	log.Debug("Initializing MEASSUREMENTSconfig...")
	for mKey, mVal := range cfg.Measurements {
		err := mVal.Init(&cfg.Metrics)
		if err != nil {
			log.Warnln("Error in Measurement config:", err)
			//if some error int the format the metric is deleted from the config
			delete(cfg.Metrics, mKey)
		}

		log.Debugf("FIELDMETRICS: %+v", mVal.FieldMetric)
	}
	log.Debug("-----------------------END Config metrics----------------------")
	return nil
}

//var DBConfig SQLConfig
<commit_msg>fix #209<commit_after>package config

import ()

// SnmpDeviceCfg contains all snmp related device definitions
type SnmpDeviceCfg struct {
	ID string `xorm:"'id' unique" binding:"Required"`
	//snmp connection config
	Host    string `xorm:"host" binding:"Required"`
	Port    int    `xorm:"port" binding:"Required"`
	Retries int    `xorm:"retries"`
	Timeout int    `xorm:"timeout"`
	Repeat  int    `xorm:"repeat"`
	Active  bool   `xorm:"'active' default 1"`
	//snmp auth  config
	SnmpVersion string `xorm:"snmpversion" binding:"Required;In(1,2c,3)"`
	Community   string `xorm:"community"`
	V3SecLevel  string `xorm:"v3seclevel"`
	V3AuthUser  string `xorm:"v3authuser"`
	V3AuthPass  string `xorm:"v3authpass"`
	V3AuthProt  string `xorm:"v3authprot"`
	V3PrivPass  string `xorm:"v3privpass"`
	V3PrivProt  string `xorm:"v3privprot"`
	//snmp workarround for some devices
	DisableBulk    bool  `xorm:"'disablebulk' default 0"`
	MaxRepetitions uint8 `xorm:"'maxrepetitions' default 50" binding:"Default(50);IntegerNotZero"`
	//snmp runtime config
	Freq             int  `xorm:"'freq' default 60" binding:"Default(60);IntegerNotZero"`
	UpdateFltFreq    int  `xorm:"'update_flt_freq' default 60" binding:"Default(60);UIntegerAndLessOne"`
	ConcurrentGather bool `xorm:"'concurrent_gather' default 1"`

	OutDB    string `xorm:"outdb"`
	LogLevel string `xorm:"loglevel" binding:"Default(info)"`
	LogFile  string `xorm:"logfile"`

	SnmpDebug bool `xorm:"'snmpdebug' default 0"`
	//influx tags
	DeviceTagName  string   `xorm:"devicetagname" binding:"Default(hostname)"`
	DeviceTagValue string   `xorm:"devicetagvalue" binding:"Default(id)"`
	ExtraTags      []string `xorm:"extra-tags"`
	Description    string   `xorm:"description"`
	//Filters for measurements
	MeasurementGroups []string `xorm:"-"`
	MeasFilters       []string `xorm:"-"`
}

// InfluxCfg is the main configuration for any InfluxDB TSDB
type InfluxCfg struct {
	ID          string `xorm:"'id' unique" binding:"Required"`
	Host        string `xorm:"host" binding:"Required"`
	Port        int    `xorm:"port" binding:"Required;IntegerNotZero"`
	DB          string `xorm:"db" binding:"Required"`
	User        string `xorm:"user" binding:"Required"`
	Password    string `xorm:"password" binding:"Required"`
	Retention   string `xorm:"'retention' default 'autogen'" binding:"Required"`
	Precision   string `xorm:"'precision' default 's'" binding:"Default(s);OmitEmpty;In(h,m,s,ms,u,ns)"` //posible values [h,m,s,ms,u,ns] default seconds for the nature of data
	Timeout     int    `xorm:"'timeout' default 30" binding:"Default(30);IntegerNotZero"`
	UserAgent   string `xorm:"useragent" binding:"Default(snmpcollector)"`
	Description string `xorm:"description"`
}

//MeasFilterCfg the filter configuration
type MeasFilterCfg struct {
	ID               string `xorm:"'id' unique" binding:"Required"`
	IDMeasurementCfg string `xorm:"id_measurement_cfg"`
	FType            string `xorm:"filter_type" binding:"Required"` //file/OIDCondition/CustomFilter
	FilterName       string `xorm:"filter_name" binding:"Required"` // valid identificator for the filter depending on the type
	EnableAlias      bool   `xorm:"enable_alias"`                   //only valid if file/Custom
	Description      string `xorm:"description"`
}

//MeasurementFieldCfg the metrics contained on each measurement (to initialize on the fieldMetric array)
type MeasurementFieldCfg struct {
	IDMeasurementCfg string `xorm:"id_measurement_cfg"`
	IDMetricCfg      string `xorm:"id_metric_cfg"`
	Report           int    `xorm:"'report' default 1"`
}

// CUSTOM FILTER TYPES

// CustomFilterItems  list of items on each custom filter
type CustomFilterItems struct {
	CustomID string `xorm:"customid"`
	TagID    string `xorm:"tagid"`
	Alias    string `xorm:"alias"`
}

// CustomFilterCfg table with user custom choosed indexes
type CustomFilterCfg struct {
	ID          string `xorm:"'id' unique" binding:"Required"`
	Description string `xorm:"description"`
	RelatedDev  string `xorm:"related_dev"`
	RelatedMeas string `xorm:"related_meas"`
	Items       []struct {
		TagID string
		Alias string
	} `xorm:"-"`
}

// OidConditionCfg condition config for filters and metrics
type OidConditionCfg struct {
	ID          string `xorm:"'id' unique" binding:"Required"`
	IsMultiple  bool   `xorm:"is_multiple"`
	OIDCond     string `xorm:"cond_oid" binding:"Required"`
	CondType    string `xorm:"cond_type"`
	CondValue   string `xorm:"cond_value"`
	Description string `xorm:"description"`
}

//SnmpDevFilters filters to use with indexed measurement
type SnmpDevFilters struct {
	IDSnmpDev string `xorm:"id_snmpdev"`
	IDFilter  string `xorm:"id_filter"`
}

//MGroupsCfg measurement groups to asign to devices
type MGroupsCfg struct {
	ID           string   `xorm:"'id' unique" binding:"Required"`
	Measurements []string `xorm:"-"`
	Description  string   `xorm:"description"`
}

//MGroupsMeasurements measurements contained on each Measurement Group
type MGroupsMeasurements struct {
	IDMGroupCfg      string `xorm:"id_mgroup_cfg"`
	IDMeasurementCfg string `xorm:"id_measurement_cfg"`
}

// SnmpDevMGroups Mgroups defined on each SnmpDevice
type SnmpDevMGroups struct {
	IDSnmpDev   string `xorm:"id_snmpdev"`
	IDMGroupCfg string `xorm:"id_mgroup_cfg"`
}

// SQLConfig read from DB
type SQLConfig struct {
	Metrics      map[string]*SnmpMetricCfg
	Measurements map[string]*MeasurementCfg
	MFilters     map[string]*MeasFilterCfg
	GetGroups    map[string]*MGroupsCfg
	SnmpDevice   map[string]*SnmpDeviceCfg
	Influxdb     map[string]*InfluxCfg
}

/*
initMetricsCfg this function does 2 things
1.- Initialice id from key of maps for all SnmpMetricCfg and InfluxMeasurementCfg objects
2.- Initialice references between InfluxMeasurementCfg and SnmpMetricGfg objects
*/
// InitMetricsCfg xx
func InitMetricsCfg(cfg *SQLConfig) error {
	//TODO:
	// - check duplicates OID's => warning messages
	//Initialize references to SnmpMetricGfg into InfluxMeasurementCfg
	log.Debug("--------------------Initializing Config metrics-------------------")
	log.Debug("Initializing SNMPMetricconfig...")
	for mKey, mVal := range cfg.Metrics {
		err := mVal.Init()
		if err != nil {
			log.Warnln("Error in Metric config:", err)
			//if some error int the format the metric is deleted from the config
			delete(cfg.Metrics, mKey)
		}
	}
	log.Debug("Initializing MEASSUREMENTSconfig...")
	for mKey, mVal := range cfg.Measurements {
		err := mVal.Init(&cfg.Metrics)
		if err != nil {
			log.Warnln("Error in Measurement config:", err)
			//if some error int the format the metric is deleted from the config
			delete(cfg.Metrics, mKey)
		}

		log.Debugf("FIELDMETRICS: %+v", mVal.FieldMetric)
	}
	log.Debug("-----------------------END Config metrics----------------------")
	return nil
}

//var DBConfig SQLConfig
<|endoftext|>


<commit_before>package dbg

import (
	"fmt"
	"io/ioutil"
	"os"
	"runtime"
	"runtime/pprof"
	"time"

	"github.com/sirupsen/logrus"
)

const (
	path = "/var/cores/"
)

// DumpGoMemoryTrace output memory profile to logs.
func DumpGoMemoryTrace() {
	m := &runtime.MemStats{}
	runtime.ReadMemStats(m)
	res := fmt.Sprintf("%#v", m)
	logrus.Infof("==== Dumping Memory Profile ===")
	logrus.Infof(res)
}

// DumpGoProfile output goroutines to file.
func DumpGoProfile() error {
	trace := make([]byte, 5120*1024)
	len := runtime.Stack(trace, true)
	return ioutil.WriteFile(path+time.Now().String()+".stack", trace[:len], 0644)
}

func DumpHeap() {
	f, err := os.Create(path + time.Now().String() + ".heap")
	if err != nil {
		logrus.Errorf("could not create memory profile: %v", err)
		return
	}
	defer f.Close()
	if err := pprof.WriteHeapProfile(f); err != nil {
		logrus.Errorf("could not write memory profile: %v", err)
	}
}
<commit_msg>PWX-5813: Remove whitespace from .stack & .heap filenames.<commit_after>package dbg

import (
	"fmt"
	"io/ioutil"
	"os"
	"runtime"
	"runtime/pprof"
	"time"

	"github.com/sirupsen/logrus"
)

const (
	path = "/var/cores/"
)

// DumpGoMemoryTrace output memory profile to logs.
func DumpGoMemoryTrace() {
	m := &runtime.MemStats{}
	runtime.ReadMemStats(m)
	res := fmt.Sprintf("%#v", m)
	logrus.Infof("==== Dumping Memory Profile ===")
	logrus.Infof(res)
}

// DumpGoProfile output goroutines to file.
func DumpGoProfile() error {
	trace := make([]byte, 5120*1024)
	len := runtime.Stack(trace, true)
	return ioutil.WriteFile(path+time.Now().Format("2006-01-02T15:04:05.999999-0700MST")+".stack", trace[:len], 0644)
}

func DumpHeap() {
	f, err := os.Create(path + time.Now().Format("2006-01-02T15:04:05.999999-0700MST") + ".heap")
	if err != nil {
		logrus.Errorf("could not create memory profile: %v", err)
		return
	}
	defer f.Close()
	if err := pprof.WriteHeapProfile(f); err != nil {
		logrus.Errorf("could not write memory profile: %v", err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The HAProxy Ingress Controller Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package haproxy

import (
	"fmt"
	"os/exec"
	"reflect"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/jcmoraisjr/haproxy-ingress/pkg/acme"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/template"
	hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types"
	hautils "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/utils"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/types"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/utils"
)

// InstanceOptions ...
type InstanceOptions struct {
	AcmeSigner        acme.Signer
	AcmeQueue         utils.Queue
	LeaderElector     types.LeaderElector
	MaxOldConfigFiles int
	HAProxyCmd        string
	HAProxyConfigFile string
	Metrics           types.Metrics
	ReloadCmd         string
	ReloadStrategy    string
	ValidateConfig    bool
}

// Instance ...
type Instance interface {
	AcmePeriodicCheck()
	ParseTemplates() error
	Config() Config
	CalcIdleMetric()
	Update(timer *utils.Timer)
}

// CreateInstance ...
func CreateInstance(logger types.Logger, options InstanceOptions) Instance {
	return &instance{
		logger:       logger,
		options:      &options,
		templates:    template.CreateConfig(),
		mapsTemplate: template.CreateConfig(),
		mapsDir:      "/etc/haproxy/maps",
		metrics:      options.Metrics,
	}
}

type instance struct {
	logger       types.Logger
	options      *InstanceOptions
	templates    *template.Config
	mapsTemplate *template.Config
	mapsDir      string
	oldConfig    Config
	curConfig    Config
	metrics      types.Metrics
}

func (i *instance) AcmePeriodicCheck() {
	if i.oldConfig == nil || i.options.AcmeQueue == nil {
		return
	}
	hasAccount := i.acmeEnsureConfig(i.oldConfig.AcmeData())
	if !hasAccount {
		return
	}
	le := i.options.LeaderElector
	if !le.IsLeader() {
		i.logger.Info("skipping acme periodic check, leader is %s", le.LeaderName())
		return
	}
	i.logger.Info("starting periodic certificate check")
	var count int
	for storage, domains := range i.oldConfig.AcmeData().Certs {
		i.acmeAddCert(storage, domains)
		count++
	}
	if count == 0 {
		i.logger.Info("certificate list is empty")
	} else {
		i.logger.Info("finish adding %d certificate(s) to the work queue", count)
	}
}

func (i *instance) acmeEnsureConfig(acmeConfig *hatypes.AcmeData) bool {
	signer := i.options.AcmeSigner
	signer.AcmeConfig(acmeConfig.Expiring)
	signer.AcmeAccount(acmeConfig.Endpoint, acmeConfig.Emails, acmeConfig.TermsAgreed)
	return signer.HasAccount()
}

func (i *instance) acmeBuildCert(storage string, domains map[string]struct{}) string {
	cert := make([]string, len(domains))
	n := 0
	for dom := range domains {
		cert[n] = dom
		n++
	}
	sort.Slice(cert, func(i, j int) bool {
		return cert[i] < cert[j]
	})
	return strings.Join(cert, ",")
}

func (i *instance) acmeAddCert(storage string, domains map[string]struct{}) {
	strcert := i.acmeBuildCert(storage, domains)
	i.logger.InfoV(2, "enqueue certificate for processing: storage=%s domain(s)=%s",
		storage, strcert)
	i.options.AcmeQueue.Add(storage + "," + strcert)
}

func (i *instance) acmeRemoveCert(storage string, domains map[string]struct{}) {
	strcert := i.acmeBuildCert(storage, domains)
	i.options.AcmeQueue.Remove(storage + "," + strcert)
}

func (i *instance) ParseTemplates() error {
	i.templates.ClearTemplates()
	i.mapsTemplate.ClearTemplates()
	if err := i.templates.NewTemplate(
		"spoe-modsecurity.tmpl",
		"/etc/haproxy/modsecurity/spoe-modsecurity.tmpl",
		"/etc/haproxy/spoe-modsecurity.conf",
		0,
		1024,
	); err != nil {
		return err
	}
	if err := i.templates.NewTemplate(
		"haproxy.tmpl",
		"/etc/haproxy/template/haproxy.tmpl",
		"/etc/haproxy/haproxy.cfg",
		i.options.MaxOldConfigFiles,
		16384,
	); err != nil {
		return err
	}
	err := i.mapsTemplate.NewTemplate(
		"map.tmpl",
		"/etc/haproxy/maptemplate/map.tmpl",
		"",
		0,
		2048,
	)
	return err
}

func (i *instance) Config() Config {
	if i.curConfig == nil {
		config := createConfig(options{
			mapsTemplate: i.mapsTemplate,
			mapsDir:      i.mapsDir,
		})
		i.curConfig = config
	}
	return i.curConfig
}

var idleRegex = regexp.MustCompile(`Idle_pct: ([0-9]+)`)

func (i *instance) CalcIdleMetric() {
	if i.oldConfig == nil {
		return
	}
	msg, err := hautils.HAProxyCommand(i.oldConfig.Global().AdminSocket, i.metrics.HAProxyShowInfoResponseTime, "show info")
	if err != nil {
		i.logger.Error("error reading admin socket: %v", err)
		return
	}
	idleStr := idleRegex.FindStringSubmatch(msg[0])
	if len(idleStr) < 2 {
		i.logger.Error("cannot find Idle_pct field in the show info socket command")
		return
	}
	idle, err := strconv.Atoi(idleStr[1])
	if err != nil {
		i.logger.Error("Idle_pct has an invalid integer: %s", idleStr[1])
	}
	i.metrics.AddIdleFactor(idle)
}

func (i *instance) Update(timer *utils.Timer) {
	i.acmeUpdate()
	i.haproxyUpdate(timer)
}

func (i *instance) acmeUpdate() {
	if i.oldConfig == nil || i.curConfig == nil || i.options.AcmeQueue == nil {
		return
	}
	le := i.options.LeaderElector
	if le.IsLeader() {
		hasAccount := i.acmeEnsureConfig(i.curConfig.AcmeData())
		if !hasAccount {
			return
		}
	}
	var updated bool
	oldCerts := i.oldConfig.AcmeData().Certs
	curCerts := i.curConfig.AcmeData().Certs
	// Remove from the retry queue certs that was removed from the config
	for storage, domains := range oldCerts {
		curdomains, found := curCerts[storage]
		if !found || !reflect.DeepEqual(domains, curdomains) {
			if le.IsLeader() {
				i.acmeRemoveCert(storage, domains)
			}
			updated = true
		}
	}
	// Add new certs to the work queue
	for storage, domains := range curCerts {
		olddomains, found := oldCerts[storage]
		if !found || !reflect.DeepEqual(domains, olddomains) {
			if le.IsLeader() {
				i.acmeAddCert(storage, domains)
			}
			updated = true
		}
	}
	if updated && !le.IsLeader() {
		i.logger.InfoV(2, "skipping acme update check, leader is %s", le.LeaderName())
	}
}

func (i *instance) haproxyUpdate(timer *utils.Timer) {
	// nil config, just ignore
	if i.curConfig == nil {
		i.logger.Info("new configuration is empty")
		return
	}
	//
	// this should be taken into account when refactoring this func:
	//   - dynUpdater might change config state, so it should be called before templates.Write()
	//   - i.metrics.IncUpdate<Status>() should be called always, but only once
	//   - i.metrics.UpdateSuccessful(<bool>) should be called only if haproxy is reloaded or cfg is validated
	//
	defer i.rotateConfig()
	if err := i.curConfig.BuildFrontendGroup(); err != nil {
		i.logger.Error("error building configuration group: %v", err)
		i.metrics.IncUpdateNoop()
		return
	}
	if err := i.curConfig.BuildBackendMaps(); err != nil {
		i.logger.Error("error building backend maps: %v", err)
		i.metrics.IncUpdateNoop()
		return
	}
	if i.curConfig.Equals(i.oldConfig) {
		i.logger.InfoV(2, "old and new configurations match, skipping reload")
		i.metrics.IncUpdateNoop()
		return
	}
	updater := i.newDynUpdater()
	updated := updater.update()
	if !updated || updater.cmdCnt > 0 {
		// only need to rewrtite config files if:
		//   - !updated           - there are changes that cannot be dynamically applied
		//   - updater.cmdCnt > 0 - there are changes that was dynamically applied
		err := i.templates.Write(i.curConfig)
		timer.Tick("write_tmpl")
		if err != nil {
			i.logger.Error("error writing configuration: %v", err)
			i.metrics.IncUpdateNoop()
			return
		}
	}
	if updated {
		if updater.cmdCnt > 0 {
			if i.options.ValidateConfig {
				var err error
				if err = i.check(); err != nil {
					i.logger.Error("error validating config file:\n%v", err)
				}
				timer.Tick("validate_cfg")
				i.metrics.UpdateSuccessful(err == nil)
			}
			i.logger.Info("HAProxy updated without needing to reload. Commands sent: %d", updater.cmdCnt)
			i.metrics.IncUpdateDynamic()
		} else {
			i.logger.Info("old and new configurations match")
			i.metrics.IncUpdateNoop()
		}
		return
	}
	i.updateCertExpiring()
	i.metrics.IncUpdateFull()
	if err := i.reload(); err != nil {
		i.logger.Error("error reloading server:\n%v", err)
		i.metrics.UpdateSuccessful(false)
		return
	}
	timer.Tick("reload_haproxy")
	i.metrics.UpdateSuccessful(true)
	i.logger.Info("HAProxy successfully reloaded")
}

func (i *instance) updateCertExpiring() {
	// TODO move to dynupdate when dynamic crt update is implemented
	if i.oldConfig == nil {
		for _, curHost := range i.curConfig.Hosts() {
			if curHost.TLS.HasTLS() {
				i.metrics.SetCertExpireDate(curHost.Hostname, curHost.TLS.TLSCommonName, &curHost.TLS.TLSNotAfter)
			}
		}
		return
	}
	for _, oldHost := range i.oldConfig.Hosts() {
		if !oldHost.TLS.HasTLS() {
			continue
		}
		curHost := i.curConfig.FindHost(oldHost.Hostname)
		if curHost == nil || oldHost.TLS.TLSCommonName != curHost.TLS.TLSCommonName {
			i.metrics.SetCertExpireDate(oldHost.Hostname, oldHost.TLS.TLSCommonName, nil)
		}
	}
	for _, curHost := range i.curConfig.Hosts() {
		if !curHost.TLS.HasTLS() {
			continue
		}
		oldHost := i.oldConfig.FindHost(curHost.Hostname)
		if oldHost == nil || oldHost.TLS.TLSCommonName != curHost.TLS.TLSCommonName {
			i.metrics.SetCertExpireDate(curHost.Hostname, curHost.TLS.TLSCommonName, &curHost.TLS.TLSNotAfter)
		}
	}
}

func (i *instance) check() error {
	if i.options.HAProxyCmd == "" {
		i.logger.Info("(test) check was skipped")
		return nil
	}
	out, err := exec.Command(i.options.HAProxyCmd, "-c", "-f", i.options.HAProxyConfigFile).CombinedOutput()
	outstr := filterOutput(out)
	if err != nil {
		return fmt.Errorf(outstr)
	}
	return nil
}

func (i *instance) reload() error {
	if i.options.ReloadCmd == "" {
		i.logger.Info("(test) reload was skipped")
		return nil
	}
	out, err := exec.Command(i.options.ReloadCmd, i.options.ReloadStrategy, i.options.HAProxyConfigFile).CombinedOutput()
	outstr := filterOutput(out)
	if len(outstr) > 0 {
		i.logger.Warn("output from haproxy:\n%v", outstr)
	}
	if err != nil {
		return err
	}
	return nil
}

func filterOutput(out []byte) string {
	// workaround a misplaced warn from haproxy until the fix is merged
	skip := "contains no embedded dots nor does not start with a dot"
	outstr := ""
	for _, line := range strings.Split(string(out), "\n") {
		if line != "" && strings.Index(line, skip) < 0 {
			outstr += line + "\n"
		}
	}
	return outstr
}

func (i *instance) rotateConfig() {
	// TODO releaseConfig (old support files, ...)
	i.oldConfig = i.curConfig
	i.curConfig = nil
}
<commit_msg>remove haproxy warning filter<commit_after>/*
Copyright 2019 The HAProxy Ingress Controller Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package haproxy

import (
	"fmt"
	"os/exec"
	"reflect"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/jcmoraisjr/haproxy-ingress/pkg/acme"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/template"
	hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types"
	hautils "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/utils"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/types"
	"github.com/jcmoraisjr/haproxy-ingress/pkg/utils"
)

// InstanceOptions ...
type InstanceOptions struct {
	AcmeSigner        acme.Signer
	AcmeQueue         utils.Queue
	LeaderElector     types.LeaderElector
	MaxOldConfigFiles int
	HAProxyCmd        string
	HAProxyConfigFile string
	Metrics           types.Metrics
	ReloadCmd         string
	ReloadStrategy    string
	ValidateConfig    bool
}

// Instance ...
type Instance interface {
	AcmePeriodicCheck()
	ParseTemplates() error
	Config() Config
	CalcIdleMetric()
	Update(timer *utils.Timer)
}

// CreateInstance ...
func CreateInstance(logger types.Logger, options InstanceOptions) Instance {
	return &instance{
		logger:       logger,
		options:      &options,
		templates:    template.CreateConfig(),
		mapsTemplate: template.CreateConfig(),
		mapsDir:      "/etc/haproxy/maps",
		metrics:      options.Metrics,
	}
}

type instance struct {
	logger       types.Logger
	options      *InstanceOptions
	templates    *template.Config
	mapsTemplate *template.Config
	mapsDir      string
	oldConfig    Config
	curConfig    Config
	metrics      types.Metrics
}

func (i *instance) AcmePeriodicCheck() {
	if i.oldConfig == nil || i.options.AcmeQueue == nil {
		return
	}
	hasAccount := i.acmeEnsureConfig(i.oldConfig.AcmeData())
	if !hasAccount {
		return
	}
	le := i.options.LeaderElector
	if !le.IsLeader() {
		i.logger.Info("skipping acme periodic check, leader is %s", le.LeaderName())
		return
	}
	i.logger.Info("starting periodic certificate check")
	var count int
	for storage, domains := range i.oldConfig.AcmeData().Certs {
		i.acmeAddCert(storage, domains)
		count++
	}
	if count == 0 {
		i.logger.Info("certificate list is empty")
	} else {
		i.logger.Info("finish adding %d certificate(s) to the work queue", count)
	}
}

func (i *instance) acmeEnsureConfig(acmeConfig *hatypes.AcmeData) bool {
	signer := i.options.AcmeSigner
	signer.AcmeConfig(acmeConfig.Expiring)
	signer.AcmeAccount(acmeConfig.Endpoint, acmeConfig.Emails, acmeConfig.TermsAgreed)
	return signer.HasAccount()
}

func (i *instance) acmeBuildCert(storage string, domains map[string]struct{}) string {
	cert := make([]string, len(domains))
	n := 0
	for dom := range domains {
		cert[n] = dom
		n++
	}
	sort.Slice(cert, func(i, j int) bool {
		return cert[i] < cert[j]
	})
	return strings.Join(cert, ",")
}

func (i *instance) acmeAddCert(storage string, domains map[string]struct{}) {
	strcert := i.acmeBuildCert(storage, domains)
	i.logger.InfoV(2, "enqueue certificate for processing: storage=%s domain(s)=%s",
		storage, strcert)
	i.options.AcmeQueue.Add(storage + "," + strcert)
}

func (i *instance) acmeRemoveCert(storage string, domains map[string]struct{}) {
	strcert := i.acmeBuildCert(storage, domains)
	i.options.AcmeQueue.Remove(storage + "," + strcert)
}

func (i *instance) ParseTemplates() error {
	i.templates.ClearTemplates()
	i.mapsTemplate.ClearTemplates()
	if err := i.templates.NewTemplate(
		"spoe-modsecurity.tmpl",
		"/etc/haproxy/modsecurity/spoe-modsecurity.tmpl",
		"/etc/haproxy/spoe-modsecurity.conf",
		0,
		1024,
	); err != nil {
		return err
	}
	if err := i.templates.NewTemplate(
		"haproxy.tmpl",
		"/etc/haproxy/template/haproxy.tmpl",
		"/etc/haproxy/haproxy.cfg",
		i.options.MaxOldConfigFiles,
		16384,
	); err != nil {
		return err
	}
	err := i.mapsTemplate.NewTemplate(
		"map.tmpl",
		"/etc/haproxy/maptemplate/map.tmpl",
		"",
		0,
		2048,
	)
	return err
}

func (i *instance) Config() Config {
	if i.curConfig == nil {
		config := createConfig(options{
			mapsTemplate: i.mapsTemplate,
			mapsDir:      i.mapsDir,
		})
		i.curConfig = config
	}
	return i.curConfig
}

var idleRegex = regexp.MustCompile(`Idle_pct: ([0-9]+)`)

func (i *instance) CalcIdleMetric() {
	if i.oldConfig == nil {
		return
	}
	msg, err := hautils.HAProxyCommand(i.oldConfig.Global().AdminSocket, i.metrics.HAProxyShowInfoResponseTime, "show info")
	if err != nil {
		i.logger.Error("error reading admin socket: %v", err)
		return
	}
	idleStr := idleRegex.FindStringSubmatch(msg[0])
	if len(idleStr) < 2 {
		i.logger.Error("cannot find Idle_pct field in the show info socket command")
		return
	}
	idle, err := strconv.Atoi(idleStr[1])
	if err != nil {
		i.logger.Error("Idle_pct has an invalid integer: %s", idleStr[1])
	}
	i.metrics.AddIdleFactor(idle)
}

func (i *instance) Update(timer *utils.Timer) {
	i.acmeUpdate()
	i.haproxyUpdate(timer)
}

func (i *instance) acmeUpdate() {
	if i.oldConfig == nil || i.curConfig == nil || i.options.AcmeQueue == nil {
		return
	}
	le := i.options.LeaderElector
	if le.IsLeader() {
		hasAccount := i.acmeEnsureConfig(i.curConfig.AcmeData())
		if !hasAccount {
			return
		}
	}
	var updated bool
	oldCerts := i.oldConfig.AcmeData().Certs
	curCerts := i.curConfig.AcmeData().Certs
	// Remove from the retry queue certs that was removed from the config
	for storage, domains := range oldCerts {
		curdomains, found := curCerts[storage]
		if !found || !reflect.DeepEqual(domains, curdomains) {
			if le.IsLeader() {
				i.acmeRemoveCert(storage, domains)
			}
			updated = true
		}
	}
	// Add new certs to the work queue
	for storage, domains := range curCerts {
		olddomains, found := oldCerts[storage]
		if !found || !reflect.DeepEqual(domains, olddomains) {
			if le.IsLeader() {
				i.acmeAddCert(storage, domains)
			}
			updated = true
		}
	}
	if updated && !le.IsLeader() {
		i.logger.InfoV(2, "skipping acme update check, leader is %s", le.LeaderName())
	}
}

func (i *instance) haproxyUpdate(timer *utils.Timer) {
	// nil config, just ignore
	if i.curConfig == nil {
		i.logger.Info("new configuration is empty")
		return
	}
	//
	// this should be taken into account when refactoring this func:
	//   - dynUpdater might change config state, so it should be called before templates.Write()
	//   - i.metrics.IncUpdate<Status>() should be called always, but only once
	//   - i.metrics.UpdateSuccessful(<bool>) should be called only if haproxy is reloaded or cfg is validated
	//
	defer i.rotateConfig()
	if err := i.curConfig.BuildFrontendGroup(); err != nil {
		i.logger.Error("error building configuration group: %v", err)
		i.metrics.IncUpdateNoop()
		return
	}
	if err := i.curConfig.BuildBackendMaps(); err != nil {
		i.logger.Error("error building backend maps: %v", err)
		i.metrics.IncUpdateNoop()
		return
	}
	if i.curConfig.Equals(i.oldConfig) {
		i.logger.InfoV(2, "old and new configurations match, skipping reload")
		i.metrics.IncUpdateNoop()
		return
	}
	updater := i.newDynUpdater()
	updated := updater.update()
	if !updated || updater.cmdCnt > 0 {
		// only need to rewrtite config files if:
		//   - !updated           - there are changes that cannot be dynamically applied
		//   - updater.cmdCnt > 0 - there are changes that was dynamically applied
		err := i.templates.Write(i.curConfig)
		timer.Tick("write_tmpl")
		if err != nil {
			i.logger.Error("error writing configuration: %v", err)
			i.metrics.IncUpdateNoop()
			return
		}
	}
	if updated {
		if updater.cmdCnt > 0 {
			if i.options.ValidateConfig {
				var err error
				if err = i.check(); err != nil {
					i.logger.Error("error validating config file:\n%v", err)
				}
				timer.Tick("validate_cfg")
				i.metrics.UpdateSuccessful(err == nil)
			}
			i.logger.Info("HAProxy updated without needing to reload. Commands sent: %d", updater.cmdCnt)
			i.metrics.IncUpdateDynamic()
		} else {
			i.logger.Info("old and new configurations match")
			i.metrics.IncUpdateNoop()
		}
		return
	}
	i.updateCertExpiring()
	i.metrics.IncUpdateFull()
	if err := i.reload(); err != nil {
		i.logger.Error("error reloading server:\n%v", err)
		i.metrics.UpdateSuccessful(false)
		return
	}
	timer.Tick("reload_haproxy")
	i.metrics.UpdateSuccessful(true)
	i.logger.Info("HAProxy successfully reloaded")
}

func (i *instance) updateCertExpiring() {
	// TODO move to dynupdate when dynamic crt update is implemented
	if i.oldConfig == nil {
		for _, curHost := range i.curConfig.Hosts() {
			if curHost.TLS.HasTLS() {
				i.metrics.SetCertExpireDate(curHost.Hostname, curHost.TLS.TLSCommonName, &curHost.TLS.TLSNotAfter)
			}
		}
		return
	}
	for _, oldHost := range i.oldConfig.Hosts() {
		if !oldHost.TLS.HasTLS() {
			continue
		}
		curHost := i.curConfig.FindHost(oldHost.Hostname)
		if curHost == nil || oldHost.TLS.TLSCommonName != curHost.TLS.TLSCommonName {
			i.metrics.SetCertExpireDate(oldHost.Hostname, oldHost.TLS.TLSCommonName, nil)
		}
	}
	for _, curHost := range i.curConfig.Hosts() {
		if !curHost.TLS.HasTLS() {
			continue
		}
		oldHost := i.oldConfig.FindHost(curHost.Hostname)
		if oldHost == nil || oldHost.TLS.TLSCommonName != curHost.TLS.TLSCommonName {
			i.metrics.SetCertExpireDate(curHost.Hostname, curHost.TLS.TLSCommonName, &curHost.TLS.TLSNotAfter)
		}
	}
}

func (i *instance) check() error {
	if i.options.HAProxyCmd == "" {
		i.logger.Info("(test) check was skipped")
		return nil
	}
	out, err := exec.Command(i.options.HAProxyCmd, "-c", "-f", i.options.HAProxyConfigFile).CombinedOutput()
	outstr := string(out)
	if err != nil {
		return fmt.Errorf(outstr)
	}
	return nil
}

func (i *instance) reload() error {
	if i.options.ReloadCmd == "" {
		i.logger.Info("(test) reload was skipped")
		return nil
	}
	out, err := exec.Command(i.options.ReloadCmd, i.options.ReloadStrategy, i.options.HAProxyConfigFile).CombinedOutput()
	outstr := string(out)
	if len(outstr) > 0 {
		i.logger.Warn("output from haproxy:\n%v", outstr)
	}
	if err != nil {
		return err
	}
	return nil
}

func (i *instance) rotateConfig() {
	// TODO releaseConfig (old support files, ...)
	i.oldConfig = i.curConfig
	i.curConfig = nil
}
<|endoftext|>


<commit_before>package proxy

import (
	"encoding/json"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/hellofresh/janus/pkg/router"
)

// Route is the container for a proxy and it's handlers
type Route struct {
	proxy    *Definition
	handlers []router.Constructor
}

type routeJSONProxy struct {
	Proxy *Definition `json:"proxy"`
}

// NewRoute creates an instance of Route
func NewRoute(proxy *Definition, handlers ...router.Constructor) *Route {
	return &Route{proxy, handlers}
}

// JSONMarshal encodes route struct to JSON
func (r *Route) JSONMarshal() ([]byte, error) {
	return json.Marshal(routeJSONProxy{r.proxy})
}

// JSONUnmarshalRoute decodes route struct from JSON
func JSONUnmarshalRoute(rawRoute []byte) (*Route, error) {
	var proxyRoute routeJSONProxy
	if err := json.Unmarshal(rawRoute, &proxyRoute); err != nil {
		return nil, err
	}
	return NewRoute(proxyRoute.Proxy), nil
}

// Definition defines proxy rules for a route
type Definition struct {
	PreserveHostHeader          bool     `bson:"preserve_host_header" json:"preserve_host_header"`
	ListenPath                  string   `bson:"listen_path" json:"listen_path" valid:"required"`
	TargetURL                   string   `bson:"target_url" json:"target_url" valid:"url,required"`
	StripListenPath             bool     `bson:"strip_listen_path" json:"strip_listen_path"`
	AppendListenPath            bool     `bson:"append_listen_path" json:"append_listen_path"`
	EnableLoadBalancing         bool     `bson:"enable_load_balancing" json:"enable_load_balancing"`
	TargetList                  []string `bson:"target_list" json:"target_list"`
	CheckHostAgainstUptimeTests bool     `bson:"check_host_against_uptime_tests" json:"check_host_against_uptime_tests"`
	Methods                     []string `bson:"methods" json:"methods"`
}

// Validate validates proxy data
func Validate(proxy *Definition) bool {
	if proxy.ListenPath == "" {
		log.Warning("Listen path is empty")
		return false
	}

	if strings.Contains(proxy.ListenPath, " ") {
		log.Warning("Listen path contains spaces, is invalid")
		return false
	}

	return true
}
<commit_msg>Avoiding errors<commit_after>package proxy

import (
	"encoding/json"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/hellofresh/janus/pkg/router"
)

// Route is the container for a proxy and it's handlers
type Route struct {
	proxy    *Definition
	handlers []router.Constructor
}

type routeJSONProxy struct {
	Proxy *Definition `json:"proxy"`
}

// NewRoute creates an instance of Route
func NewRoute(proxy *Definition, handlers ...router.Constructor) *Route {
	return &Route{proxy, handlers}
}

// JSONMarshal encodes route struct to JSON
func (r *Route) JSONMarshal() ([]byte, error) {
	return json.Marshal(routeJSONProxy{r.proxy})
}

// JSONUnmarshalRoute decodes route struct from JSON
func JSONUnmarshalRoute(rawRoute []byte) (*Route, error) {
	var proxyRoute routeJSONProxy
	if err := json.Unmarshal(rawRoute, &proxyRoute); err != nil {
		return nil, err
	}
	return NewRoute(proxyRoute.Proxy), nil
}

// Definition defines proxy rules for a route
type Definition struct {
	PreserveHostHeader          bool     `bson:"preserve_host_header" json:"preserve_host_header"`
	ListenPath                  string   `bson:"listen_path" json:"listen_path" valid:"required"`
	TargetURL                   string   `bson:"target_url" json:"target_url" valid:"url,required"`
	StripListenPath             bool     `bson:"strip_listen_path" json:"strip_listen_path"`
	AppendListenPath            bool     `bson:"append_listen_path" json:"append_listen_path"`
	EnableLoadBalancing         bool     `bson:"enable_load_balancing" json:"enable_load_balancing"`
	TargetList                  []string `bson:"target_list" json:"target_list"`
	CheckHostAgainstUptimeTests bool     `bson:"check_host_against_uptime_tests" json:"check_host_against_uptime_tests"`
	Methods                     []string `bson:"methods" json:"methods"`
}

// Validate validates proxy data
func Validate(proxy *Definition) bool {
	if nil == proxy {
		return false
	}

	if proxy.ListenPath == "" {
		log.Warning("Listen path is empty")
		return false
	}

	if strings.Contains(proxy.ListenPath, " ") {
		log.Warning("Listen path contains spaces, is invalid")
		return false
	}

	return true
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package schema

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"

	"camlistore.org/pkg/blob"
)

// A DirReader reads the entries of a "directory" schema blob's
// referenced "static-set" blob.
type DirReader struct {
	fetcher blob.Fetcher
	ss      *superset

	staticSet []blob.Ref
	current   int
}

// NewDirReader creates a new directory reader and prepares to
// fetch the static-set entries
func NewDirReader(fetcher blob.Fetcher, dirBlobRef blob.Ref) (*DirReader, error) {
	ss := new(superset)
	err := ss.setFromBlobRef(fetcher, dirBlobRef)
	if err != nil {
		return nil, err
	}
	if ss.Type != "directory" {
		return nil, fmt.Errorf("schema/dirreader: expected \"directory\" schema blob for %s, got %q", dirBlobRef, ss.Type)
	}
	dr, err := ss.NewDirReader(fetcher)
	if err != nil {
		return nil, fmt.Errorf("schema/dirreader: creating DirReader for %s: %v", dirBlobRef, err)
	}
	dr.current = 0
	return dr, nil
}

func (b *Blob) NewDirReader(fetcher blob.Fetcher) (*DirReader, error) {
	return b.ss.NewDirReader(fetcher)
}

func (ss *superset) NewDirReader(fetcher blob.Fetcher) (*DirReader, error) {
	if ss.Type != "directory" {
		return nil, fmt.Errorf("Superset not of type \"directory\"")
	}
	return &DirReader{fetcher: fetcher, ss: ss}, nil
}

func (ss *superset) setFromBlobRef(fetcher blob.Fetcher, blobRef blob.Ref) error {
	if !blobRef.Valid() {
		return errors.New("schema/dirreader: blobref invalid")
	}
	ss.BlobRef = blobRef
	rc, _, err := fetcher.Fetch(blobRef)
	if err != nil {
		return fmt.Errorf("schema/dirreader: fetching schema blob %s: %v", blobRef, err)
	}
	defer rc.Close()
	if err := json.NewDecoder(rc).Decode(ss); err != nil {
		return fmt.Errorf("schema/dirreader: decoding schema blob %s: %v", blobRef, err)
	}
	return nil
}

// StaticSet returns the whole of the static set members of that directory
func (dr *DirReader) StaticSet() ([]blob.Ref, error) {
	if dr.staticSet != nil {
		return dr.staticSet, nil
	}
	staticSetBlobref := dr.ss.Entries
	if !staticSetBlobref.Valid() {
		return nil, errors.New("schema/dirreader: Invalid blobref")
	}
	rsc, _, err := dr.fetcher.Fetch(staticSetBlobref)
	if err != nil {
		return nil, fmt.Errorf("schema/dirreader: fetching schema blob %s: %v", staticSetBlobref, err)
	}
	defer rsc.Close()
	ss, err := parseSuperset(rsc)
	if err != nil {
		return nil, fmt.Errorf("schema/dirreader: decoding schema blob %s: %v", staticSetBlobref, err)
	}
	if ss.Type != "static-set" {
		return nil, fmt.Errorf("schema/dirreader: expected \"static-set\" schema blob for %s, got %q", staticSetBlobref, ss.Type)
	}
	for _, member := range ss.Members {
		if !member.Valid() {
			return nil, fmt.Errorf("schema/dirreader: invalid (static-set member) blobref referred by \"static-set\" schema blob %v", staticSetBlobref)
		}
		dr.staticSet = append(dr.staticSet, member)
	}
	return dr.staticSet, nil
}

// Readdir implements the Directory interface.
func (dr *DirReader) Readdir(n int) (entries []DirectoryEntry, err error) {
	sts, err := dr.StaticSet()
	if err != nil {
		return nil, fmt.Errorf("schema/dirreader: can't get StaticSet: %v", err)
	}
	up := dr.current + n
	if n <= 0 {
		dr.current = 0
		up = len(sts)
	} else {
		if n > (len(sts) - dr.current) {
			err = io.EOF
			up = len(sts)
		}
	}

	// TODO(bradfitz): push down information to the fetcher
	// (e.g. cachingfetcher -> remote client http) that we're
	// going to load a bunch, so the HTTP client (if not using
	// SPDY) can do discovery and see if the server supports a
	// batch handler, then get them all in one round-trip, rather
	// than attacking the server with hundreds of parallel TLS
	// setups.

	type res struct {
		ent DirectoryEntry
		err error
	}
	var cs []chan res

	// Kick off all directory entry loads.
	// TODO: bound this?
	for _, entRef := range sts[dr.current:up] {
		c := make(chan res, 1)
		cs = append(cs, c)
		go func(entRef blob.Ref) {
			entry, err := NewDirectoryEntryFromBlobRef(dr.fetcher, entRef)
			c <- res{entry, err}
		}(entRef)
	}

	for _, c := range cs {
		res := <-c
		if res.err != nil {
			return nil, fmt.Errorf("schema/dirreader: can't create dirEntry: %v", err)
		}
		entries = append(entries, res.ent)
	}
	return entries, nil
}
<commit_msg>DirReader.Readdir incorrectly logs the contents of err where it should really log the contents of res.err<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package schema

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"

	"camlistore.org/pkg/blob"
)

// A DirReader reads the entries of a "directory" schema blob's
// referenced "static-set" blob.
type DirReader struct {
	fetcher blob.Fetcher
	ss      *superset

	staticSet []blob.Ref
	current   int
}

// NewDirReader creates a new directory reader and prepares to
// fetch the static-set entries
func NewDirReader(fetcher blob.Fetcher, dirBlobRef blob.Ref) (*DirReader, error) {
	ss := new(superset)
	err := ss.setFromBlobRef(fetcher, dirBlobRef)
	if err != nil {
		return nil, err
	}
	if ss.Type != "directory" {
		return nil, fmt.Errorf("schema/dirreader: expected \"directory\" schema blob for %s, got %q", dirBlobRef, ss.Type)
	}
	dr, err := ss.NewDirReader(fetcher)
	if err != nil {
		return nil, fmt.Errorf("schema/dirreader: creating DirReader for %s: %v", dirBlobRef, err)
	}
	dr.current = 0
	return dr, nil
}

func (b *Blob) NewDirReader(fetcher blob.Fetcher) (*DirReader, error) {
	return b.ss.NewDirReader(fetcher)
}

func (ss *superset) NewDirReader(fetcher blob.Fetcher) (*DirReader, error) {
	if ss.Type != "directory" {
		return nil, fmt.Errorf("Superset not of type \"directory\"")
	}
	return &DirReader{fetcher: fetcher, ss: ss}, nil
}

func (ss *superset) setFromBlobRef(fetcher blob.Fetcher, blobRef blob.Ref) error {
	if !blobRef.Valid() {
		return errors.New("schema/dirreader: blobref invalid")
	}
	ss.BlobRef = blobRef
	rc, _, err := fetcher.Fetch(blobRef)
	if err != nil {
		return fmt.Errorf("schema/dirreader: fetching schema blob %s: %v", blobRef, err)
	}
	defer rc.Close()
	if err := json.NewDecoder(rc).Decode(ss); err != nil {
		return fmt.Errorf("schema/dirreader: decoding schema blob %s: %v", blobRef, err)
	}
	return nil
}

// StaticSet returns the whole of the static set members of that directory
func (dr *DirReader) StaticSet() ([]blob.Ref, error) {
	if dr.staticSet != nil {
		return dr.staticSet, nil
	}
	staticSetBlobref := dr.ss.Entries
	if !staticSetBlobref.Valid() {
		return nil, errors.New("schema/dirreader: Invalid blobref")
	}
	rsc, _, err := dr.fetcher.Fetch(staticSetBlobref)
	if err != nil {
		return nil, fmt.Errorf("schema/dirreader: fetching schema blob %s: %v", staticSetBlobref, err)
	}
	defer rsc.Close()
	ss, err := parseSuperset(rsc)
	if err != nil {
		return nil, fmt.Errorf("schema/dirreader: decoding schema blob %s: %v", staticSetBlobref, err)
	}
	if ss.Type != "static-set" {
		return nil, fmt.Errorf("schema/dirreader: expected \"static-set\" schema blob for %s, got %q", staticSetBlobref, ss.Type)
	}
	for _, member := range ss.Members {
		if !member.Valid() {
			return nil, fmt.Errorf("schema/dirreader: invalid (static-set member) blobref referred by \"static-set\" schema blob %v", staticSetBlobref)
		}
		dr.staticSet = append(dr.staticSet, member)
	}
	return dr.staticSet, nil
}

// Readdir implements the Directory interface.
func (dr *DirReader) Readdir(n int) (entries []DirectoryEntry, err error) {
	sts, err := dr.StaticSet()
	if err != nil {
		return nil, fmt.Errorf("schema/dirreader: can't get StaticSet: %v", err)
	}
	up := dr.current + n
	if n <= 0 {
		dr.current = 0
		up = len(sts)
	} else {
		if n > (len(sts) - dr.current) {
			err = io.EOF
			up = len(sts)
		}
	}

	// TODO(bradfitz): push down information to the fetcher
	// (e.g. cachingfetcher -> remote client http) that we're
	// going to load a bunch, so the HTTP client (if not using
	// SPDY) can do discovery and see if the server supports a
	// batch handler, then get them all in one round-trip, rather
	// than attacking the server with hundreds of parallel TLS
	// setups.

	type res struct {
		ent DirectoryEntry
		err error
	}
	var cs []chan res

	// Kick off all directory entry loads.
	// TODO: bound this?
	for _, entRef := range sts[dr.current:up] {
		c := make(chan res, 1)
		cs = append(cs, c)
		go func(entRef blob.Ref) {
			entry, err := NewDirectoryEntryFromBlobRef(dr.fetcher, entRef)
			c <- res{entry, err}
		}(entRef)
	}

	for _, c := range cs {
		res := <-c
		if res.err != nil {
			return nil, fmt.Errorf("schema/dirreader: can't create dirEntry: %v", res.err)
		}
		entries = append(entries, res.ent)
	}
	return entries, nil
}
<|endoftext|>


<commit_before>package template

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/Masterminds/semver/v3"
	"github.com/giantswarm/apiextensions/v3/pkg/apis/release/v1alpha1"
	"github.com/giantswarm/microerror"
)

func BuildImages(registryDomain string, versions Versions) Images {
	return Images{
		CalicoCNI:                    buildImage(registryDomain, "giantswarm/cni", versions.Calico, ""),
		CalicoKubeControllers:        buildImage(registryDomain, "giantswarm/kube-controllers", versions.Calico, ""),
		CalicoNode:                   buildImage(registryDomain, "giantswarm/node", versions.Calico, ""),
		Etcd:                         buildImage(registryDomain, "giantswarm/etcd", versions.Etcd, ""),
		Hyperkube:                    buildImage(registryDomain, "giantswarm/hyperkube", strings.TrimPrefix(versions.Kubernetes, "v"), ""),
		KubeApiserver:                buildImage(registryDomain, "giantswarm/kube-apiserver", versions.Kubernetes, "-giantswarm"),
		KubeControllerManager:        buildImage(registryDomain, "giantswarm/kube-controller-manager", versions.Kubernetes, ""),
		KubeProxy:                    buildImage(registryDomain, "giantswarm/kube-proxy", versions.Kubernetes, ""),
		KubeScheduler:                buildImage(registryDomain, "giantswarm/kube-scheduler", versions.Kubernetes, ""),
		KubernetesAPIHealthz:         buildImage(registryDomain, "giantswarm/k8s-api-healthz", versions.KubernetesAPIHealthz, ""),
		KubernetesNetworkSetupDocker: buildImage(registryDomain, "giantswarm/k8s-setup-network-environment", versions.KubernetesNetworkSetupDocker, ""),
		Pause:                        buildImage(registryDomain, "giantswarm/pause", "3.2", ""),
	}
}

func ExtractComponentVersions(releaseComponents []v1alpha1.ReleaseSpecComponent) (Versions, error) {
	var versions Versions

	{
		component, err := findComponent(releaseComponents, "kubernetes")
		if err != nil {
			return Versions{}, err
		}
		// cri-tools is released for each k8s minor version
		parsedVersion, err := semver.NewVersion(component.Version)
		if err != nil {
			return Versions{}, err
		}
		versions.CRITools = fmt.Sprintf("v%d.%d.0", parsedVersion.Major(), parsedVersion.Minor())
		versions.Kubernetes = fmt.Sprintf("v%s", component.Version)
	}

	{
		component, err := findComponent(releaseComponents, "etcd")
		if err != nil {
			return Versions{}, err
		}
		versions.Etcd = fmt.Sprintf("v%s", component.Version)
	}

	{
		component, err := findComponent(releaseComponents, "calico")
		if err != nil {
			return Versions{}, err
		}
		versions.Calico = fmt.Sprintf("v%s", component.Version)
	}

	return versions, nil
}

func buildImage(registryDomain, repo, tag, suffix string) string {
	return fmt.Sprintf("%s/%s:%s%s", registryDomain, repo, tag, suffix)
}

func findComponent(releaseComponents []v1alpha1.ReleaseSpecComponent, name string) (*v1alpha1.ReleaseSpecComponent, error) {
	for _, component := range releaseComponents {
		if component.Name == name {
			return &component, nil
		}
	}
	return nil, componentNotFoundError
}

func validateImagesRegsitry(images Images, mirrors []string) error {
	data, err := json.Marshal(images)
	if err != nil {
		return microerror.Mask(err)
	}

	var m map[string]string
	err = json.Unmarshal(data, &m)
	if err != nil {
		return microerror.Mask(err)
	}

	var firstImage string
	var firstKey string
	var firstRegistry string

	for k, image := range m {
		split := strings.Split(image, "/")
		r := split[0]
		if firstImage == "" {
			firstImage = image
			firstKey = k
			firstRegistry = r
		}

		if r == "" {
			return microerror.Maskf(invalidConfigError, "%T.%s image %#q registry domain must not be empty", images, k, image)
		}

		if len(mirrors) > 0 && r != "docker.io" {
			return microerror.Maskf(invalidConfigError, "%T.%s image %#q registry domain must be %#q when mirrors are set", images, k, image, "docker.io")
		}

		if r != firstRegistry {
			return microerror.Maskf(invalidConfigError, "%T.%s image %#q and %T.%s image %#q have different registries domains", images, firstKey, firstImage, images, k, image)
		}
	}

	return nil
}
<commit_msg>drop apiserver suffix (#857)<commit_after>package template

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/Masterminds/semver/v3"
	"github.com/giantswarm/apiextensions/v3/pkg/apis/release/v1alpha1"
	"github.com/giantswarm/microerror"
)

func BuildImages(registryDomain string, versions Versions) Images {
	return Images{
		CalicoCNI:                    buildImage(registryDomain, "giantswarm/cni", versions.Calico, ""),
		CalicoKubeControllers:        buildImage(registryDomain, "giantswarm/kube-controllers", versions.Calico, ""),
		CalicoNode:                   buildImage(registryDomain, "giantswarm/node", versions.Calico, ""),
		Etcd:                         buildImage(registryDomain, "giantswarm/etcd", versions.Etcd, ""),
		Hyperkube:                    buildImage(registryDomain, "giantswarm/hyperkube", strings.TrimPrefix(versions.Kubernetes, "v"), ""),
		KubeApiserver:                buildImage(registryDomain, "giantswarm/kube-apiserver", versions.Kubernetes, ""),
		KubeControllerManager:        buildImage(registryDomain, "giantswarm/kube-controller-manager", versions.Kubernetes, ""),
		KubeProxy:                    buildImage(registryDomain, "giantswarm/kube-proxy", versions.Kubernetes, ""),
		KubeScheduler:                buildImage(registryDomain, "giantswarm/kube-scheduler", versions.Kubernetes, ""),
		KubernetesAPIHealthz:         buildImage(registryDomain, "giantswarm/k8s-api-healthz", versions.KubernetesAPIHealthz, ""),
		KubernetesNetworkSetupDocker: buildImage(registryDomain, "giantswarm/k8s-setup-network-environment", versions.KubernetesNetworkSetupDocker, ""),
		Pause:                        buildImage(registryDomain, "giantswarm/pause", "3.2", ""),
	}
}

func ExtractComponentVersions(releaseComponents []v1alpha1.ReleaseSpecComponent) (Versions, error) {
	var versions Versions

	{
		component, err := findComponent(releaseComponents, "kubernetes")
		if err != nil {
			return Versions{}, err
		}
		// cri-tools is released for each k8s minor version
		parsedVersion, err := semver.NewVersion(component.Version)
		if err != nil {
			return Versions{}, err
		}
		versions.CRITools = fmt.Sprintf("v%d.%d.0", parsedVersion.Major(), parsedVersion.Minor())
		versions.Kubernetes = fmt.Sprintf("v%s", component.Version)
	}

	{
		component, err := findComponent(releaseComponents, "etcd")
		if err != nil {
			return Versions{}, err
		}
		versions.Etcd = fmt.Sprintf("v%s", component.Version)
	}

	{
		component, err := findComponent(releaseComponents, "calico")
		if err != nil {
			return Versions{}, err
		}
		versions.Calico = fmt.Sprintf("v%s", component.Version)
	}

	return versions, nil
}

func buildImage(registryDomain, repo, tag, suffix string) string {
	return fmt.Sprintf("%s/%s:%s%s", registryDomain, repo, tag, suffix)
}

func findComponent(releaseComponents []v1alpha1.ReleaseSpecComponent, name string) (*v1alpha1.ReleaseSpecComponent, error) {
	for _, component := range releaseComponents {
		if component.Name == name {
			return &component, nil
		}
	}
	return nil, componentNotFoundError
}

func validateImagesRegsitry(images Images, mirrors []string) error {
	data, err := json.Marshal(images)
	if err != nil {
		return microerror.Mask(err)
	}

	var m map[string]string
	err = json.Unmarshal(data, &m)
	if err != nil {
		return microerror.Mask(err)
	}

	var firstImage string
	var firstKey string
	var firstRegistry string

	for k, image := range m {
		split := strings.Split(image, "/")
		r := split[0]
		if firstImage == "" {
			firstImage = image
			firstKey = k
			firstRegistry = r
		}

		if r == "" {
			return microerror.Maskf(invalidConfigError, "%T.%s image %#q registry domain must not be empty", images, k, image)
		}

		if len(mirrors) > 0 && r != "docker.io" {
			return microerror.Maskf(invalidConfigError, "%T.%s image %#q registry domain must be %#q when mirrors are set", images, k, image, "docker.io")
		}

		if r != firstRegistry {
			return microerror.Maskf(invalidConfigError, "%T.%s image %#q and %T.%s image %#q have different registries domains", images, firstKey, firstImage, images, k, image)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package test

import (
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"time"

	"github.com/cerana/cerana/acomm"
	"github.com/cerana/cerana/coordinator"
	"github.com/cerana/cerana/provider"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

// Coordinator holds a coordinator server and a provider server with one or
// more registered mock Providers to be used for testing.
type Coordinator struct {
	SocketDir      string
	coordinatorURL string
	coordinator    *coordinator.Server
	providerName   string
	providerServer *provider.Server
}

// NewCoordinator creates a new Coordinator. The coordinator sever will be
// given a temporary socket directory and external port.
func NewCoordinator(baseDir string) (*Coordinator, error) {
	coordinatorName := "testCoordinator"
	socketDir, err := ioutil.TempDir(baseDir, coordinatorName)
	if err != nil {
		return nil, err
	}

	coordinatorViper := viper.New()
	coordinatorViper.Set("service_name", coordinatorName)
	coordinatorViper.Set("socket_dir", socketDir)
	coordinatorViper.Set("external_port", 1024+rand.Intn(65535-1024))
	coordinatorViper.Set("request_timeout", 20)
	coordinatorViper.Set("log_level", "fatal")

	flags := pflag.NewFlagSet(coordinatorName, pflag.ContinueOnError)
	coordinatorConfig := coordinator.NewConfig(flags, coordinatorViper)
	if err = flags.Parse([]string{}); err != nil {
		return nil, err
	}
	_ = coordinatorConfig.SetupLogging()

	coordinatorServer, err := coordinator.NewServer(coordinatorConfig)
	if err != nil {
		return nil, err
	}

	coordinatorSocket := "unix://" + filepath.Join(
		coordinatorConfig.SocketDir(),
		"coordinator",
		coordinatorConfig.ServiceName()+".sock")

	c := &Coordinator{
		SocketDir:      socketDir,
		coordinatorURL: coordinatorSocket,
		coordinator:    coordinatorServer,
	}

	c.providerName = "testProvider"
	providerFlags := pflag.NewFlagSet(c.providerName, pflag.ContinueOnError)
	providerConfig := provider.NewConfig(providerFlags, c.NewProviderViper())
	if err = providerFlags.Parse([]string{}); err != nil {
		return nil, err
	}

	providerServer, err := provider.NewServer(providerConfig)
	if err != nil {
		return nil, err
	}
	c.providerServer = providerServer

	return c, nil
}

// NewProviderViper prepares a basic viper instance for a Provider, setting
// appropriate values corresponding to the coordinator and provider server.
func (c *Coordinator) NewProviderViper() *viper.Viper {
	v := viper.New()
	v.Set("service_name", c.providerName)
	v.Set("socket_dir", c.SocketDir)
	v.Set("coordinator_url", c.coordinatorURL)
	v.Set("request_timeout", 20)
	v.Set("log_level", "fatal")
	return v
}

// ProviderTracker returns the tracker of the provider server.
func (c *Coordinator) ProviderTracker() *acomm.Tracker {
	return c.providerServer.Tracker()
}

// RegisterProvider registers a Provider's tasks with the internal Provider
// server.
func (c *Coordinator) RegisterProvider(p provider.Provider) {
	p.RegisterTasks(c.providerServer)
}

// Start starts the Coordinator and Provider servers.
func (c *Coordinator) Start() error {
	if err := c.coordinator.Start(); err != nil {
		return err
	}
	if err := c.providerServer.Start(); err != nil {
		return err
	}
	return nil
}

// Stop stops the Coordinator and Provider servers.
func (c *Coordinator) Stop() {
	c.providerServer.Stop()
	c.coordinator.Stop()
}

// Cleanup removes the temporary socket directory.
func (c *Coordinator) Cleanup() error {
	return os.RemoveAll(c.SocketDir)
}

func init() {
	rand.Seed(time.Now().UnixNano())
}
<commit_msg>Expose the external port in test coordinator<commit_after>package test

import (
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"time"

	"github.com/cerana/cerana/acomm"
	"github.com/cerana/cerana/coordinator"
	"github.com/cerana/cerana/provider"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

// Coordinator holds a coordinator server and a provider server with one or
// more registered mock Providers to be used for testing.
type Coordinator struct {
	SocketDir      string
	HTTPPort       int
	coordinatorURL string
	coordinator    *coordinator.Server
	providerName   string
	providerServer *provider.Server
}

// NewCoordinator creates a new Coordinator. The coordinator sever will be
// given a temporary socket directory and external port.
func NewCoordinator(baseDir string) (*Coordinator, error) {
	coordinatorName := "testCoordinator"
	socketDir, err := ioutil.TempDir(baseDir, coordinatorName)
	if err != nil {
		return nil, err
	}

	port := 1024 + rand.Intn(65535-1024)

	coordinatorViper := viper.New()
	coordinatorViper.Set("service_name", coordinatorName)
	coordinatorViper.Set("socket_dir", socketDir)
	coordinatorViper.Set("external_port", port)
	coordinatorViper.Set("request_timeout", 20)
	coordinatorViper.Set("log_level", "fatal")

	flags := pflag.NewFlagSet(coordinatorName, pflag.ContinueOnError)
	coordinatorConfig := coordinator.NewConfig(flags, coordinatorViper)
	if err = flags.Parse([]string{}); err != nil {
		return nil, err
	}
	_ = coordinatorConfig.SetupLogging()

	coordinatorServer, err := coordinator.NewServer(coordinatorConfig)
	if err != nil {
		return nil, err
	}

	coordinatorSocket := "unix://" + filepath.Join(
		coordinatorConfig.SocketDir(),
		"coordinator",
		coordinatorConfig.ServiceName()+".sock")

	c := &Coordinator{
		SocketDir:      socketDir,
		HTTPPort:       port,
		coordinatorURL: coordinatorSocket,
		coordinator:    coordinatorServer,
	}

	c.providerName = "testProvider"
	providerFlags := pflag.NewFlagSet(c.providerName, pflag.ContinueOnError)
	providerConfig := provider.NewConfig(providerFlags, c.NewProviderViper())
	if err = providerFlags.Parse([]string{}); err != nil {
		return nil, err
	}

	providerServer, err := provider.NewServer(providerConfig)
	if err != nil {
		return nil, err
	}
	c.providerServer = providerServer

	return c, nil
}

// NewProviderViper prepares a basic viper instance for a Provider, setting
// appropriate values corresponding to the coordinator and provider server.
func (c *Coordinator) NewProviderViper() *viper.Viper {
	v := viper.New()
	v.Set("service_name", c.providerName)
	v.Set("socket_dir", c.SocketDir)
	v.Set("coordinator_url", c.coordinatorURL)
	v.Set("request_timeout", 20)
	v.Set("log_level", "fatal")
	return v
}

// ProviderTracker returns the tracker of the provider server.
func (c *Coordinator) ProviderTracker() *acomm.Tracker {
	return c.providerServer.Tracker()
}

// RegisterProvider registers a Provider's tasks with the internal Provider
// server.
func (c *Coordinator) RegisterProvider(p provider.Provider) {
	p.RegisterTasks(c.providerServer)
}

// Start starts the Coordinator and Provider servers.
func (c *Coordinator) Start() error {
	if err := c.coordinator.Start(); err != nil {
		return err
	}
	if err := c.providerServer.Start(); err != nil {
		return err
	}
	return nil
}

// Stop stops the Coordinator and Provider servers.
func (c *Coordinator) Stop() {
	c.providerServer.Stop()
	c.coordinator.Stop()
}

// Cleanup removes the temporary socket directory.
func (c *Coordinator) Cleanup() error {
	return os.RemoveAll(c.SocketDir)
}

func init() {
	rand.Seed(time.Now().UnixNano())
}
<|endoftext|>