<commit_before>package gocql

import (
	"fmt"
	"log"
	"strings"
	"sync"
	"time"
)

/*ConnectionPool represents the interface gocql will use to work with a collection of connections.

Purpose

The connection pool in gocql opens and closes connections as well as selects an available connection
for gocql to execute a query against. The pool is also respnsible for handling connection errors that
are caught by the connection experiencing the error.

A connection pool should make a copy of the variables used from the ClusterConfig provided to the pool
upon creation. ClusterConfig is a pointer and can be modified after the creation of the pool. This can
lead to issues with variables being modified outside the expectations of the ConnectionPool type.

Example of Single Connection Pool:

	type SingleConnection struct {
		conn *Conn
		cfg *ClusterConfig
	}

	func NewSingleConnection(cfg *ClusterConfig) ConnectionPool {
		addr := strings.TrimSpace(cfg.Hosts[0])
		if strings.Index(addr, ":") < 0 {
			addr = fmt.Sprintf("%s:%d", addr, cfg.DefaultPort)
		}
		connCfg := ConnConfig{
			ProtoVersion:  cfg.ProtoVersion,
			CQLVersion:    cfg.CQLVersion,
			Timeout:       cfg.Timeout,
			NumStreams:    cfg.NumStreams,
			Compressor:    cfg.Compressor,
			Authenticator: cfg.Authenticator,
			Keepalive:     cfg.SocketKeepalive,
		}
		pool := SingleConnection{cfg:cfg}
		pool.conn = Connect(addr,connCfg,pool)
		return &pool
	}

	func (s *SingleConnection) HandleError(conn *Conn, err error, closed bool) {
		if closed {
			connCfg := ConnConfig{
				ProtoVersion:  cfg.ProtoVersion,
				CQLVersion:    cfg.CQLVersion,
				Timeout:       cfg.Timeout,
				NumStreams:    cfg.NumStreams,
				Compressor:    cfg.Compressor,
				Authenticator: cfg.Authenticator,
				Keepalive:     cfg.SocketKeepalive,
			}
			s.conn = Connect(conn.Address(),connCfg,s)
		}
	}

	func (s *SingleConnection) Pick(qry *Query) *Conn {
		if s.conn.isClosed {
			return nil
		}
		return s.conn
	}

	func (s *SingleConnection) Size() int {
		return 1
	}

	func (s *SingleConnection) Close() {
		s.conn.Close()
	}

This is a very simple example of a type that exposes the connection pool interface. To assign
this type as the connection pool to use you would assign it to the ClusterConfig like so:

		cluster := NewCluster("127.0.0.1")
		cluster.ConnPoolType = NewSingleConnection
		...
		session, err := cluster.CreateSession()

To see a more complete example of a ConnectionPool implementation please see the SimplePool type.
*/
type ConnectionPool interface {
	Pick(*Query) *Conn
	Size() int
	HandleError(*Conn, error, bool)
	Close()
	SetHosts(host []HostInfo)
}

//NewPoolFunc is the type used by ClusterConfig to create a pool of a specific type.
type NewPoolFunc func(*ClusterConfig) ConnectionPool

//SimplePool is the current implementation of the connection pool inside gocql. This
//pool is meant to be a simple default used by gocql so users can get up and running
//quickly.
type SimplePool struct {
	cfg      *ClusterConfig
	hostPool *RoundRobin
	connPool map[string]*RoundRobin
	conns    map[*Conn]struct{}
	keyspace string

	hostMu sync.RWMutex
	// this is the set of current hosts which the pool will attempt to connect to
	hosts map[string]*HostInfo

	// protects hostpool, connPoll, conns, quit
	mu sync.Mutex

	cFillingPool chan int

	quit     bool
	quitWait chan bool
	quitOnce sync.Once
}

//NewSimplePool is the function used by gocql to create the simple connection pool.
//This is the default if no other pool type is specified.
func NewSimplePool(cfg *ClusterConfig) ConnectionPool {
	pool := &SimplePool{
		cfg:          cfg,
		hostPool:     NewRoundRobin(),
		connPool:     make(map[string]*RoundRobin),
		conns:        make(map[*Conn]struct{}),
		quitWait:     make(chan bool),
		cFillingPool: make(chan int, 1),
		keyspace:     cfg.Keyspace,
		hosts:        make(map[string]*HostInfo),
	}

	for _, host := range cfg.Hosts {
		// seed hosts have unknown topology
		// TODO: Handle populating this during SetHosts
		pool.hosts[host] = &HostInfo{Peer: host}
	}

	//Walk through connecting to hosts. As soon as one host connects
	//defer the remaining connections to cluster.fillPool()
	for i := 0; i < len(cfg.Hosts); i++ {
		addr := strings.TrimSpace(cfg.Hosts[i])
		if strings.Index(addr, ":") < 0 {
			addr = fmt.Sprintf("%s:%d", addr, cfg.DefaultPort)
		}

		if pool.connect(addr) == nil {
			pool.cFillingPool <- 1
			go pool.fillPool()
			break
		}
	}

	return pool
}

func (c *SimplePool) newConn(addr string) (*Conn, error) {
	cfg := ConnConfig{
		ProtoVersion:  c.cfg.ProtoVersion,
		CQLVersion:    c.cfg.CQLVersion,
		Timeout:       c.cfg.Timeout,
		NumStreams:    c.cfg.NumStreams,
		Compressor:    c.cfg.Compressor,
		Authenticator: c.cfg.Authenticator,
		Keepalive:     c.cfg.SocketKeepalive,
	}

	conn, err := Connect(addr, cfg, c)
	if err != nil {
		log.Printf("newConn: failed to connect to %q: %v", addr, err)
		return nil, err
	}

	return conn, nil
}

func (c *SimplePool) connect(addr string) error {
	if conn, err := c.newConn(addr); err != nil {
		log.Printf("connect: failed to connect to %q: %v", addr, err)
		return err
	} else {
		return c.addConn(conn)
	}
}

func (c *SimplePool) addConn(conn *Conn) error {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.quit {
		conn.Close()
		return nil
	}

	//Set the connection's keyspace if any before adding it to the pool
	if c.keyspace != "" {
		if err := conn.UseKeyspace(c.keyspace); err != nil {
			log.Printf("error setting connection keyspace. %v", err)
			conn.Close()
			return err
		}
	}

	connPool := c.connPool[conn.Address()]
	if connPool == nil {
		connPool = NewRoundRobin()
		c.connPool[conn.Address()] = connPool
		c.hostPool.AddNode(connPool)
	}

	connPool.AddNode(conn)
	c.conns[conn] = struct{}{}

	return nil
}

//fillPool manages the pool of connections making sure that each host has the correct
//amount of connections defined. Also the method will test a host with one connection
//instead of flooding the host with number of connections defined in the cluster config
func (c *SimplePool) fillPool() {
	//Debounce large amounts of requests to fill pool
	select {
	case <-time.After(1 * time.Millisecond):
		return
	case <-c.cFillingPool:
		defer func() { c.cFillingPool <- 1 }()
	}

	c.mu.Lock()
	isClosed := c.quit
	c.mu.Unlock()
	//Exit if cluster(session) is closed
	if isClosed {
		return
	}

	c.hostMu.RLock()

	//Walk through list of defined hosts
	for host := range c.hosts {
		addr := strings.TrimSpace(host)
		if strings.Index(addr, ":") < 0 {
			addr = fmt.Sprintf("%s:%d", addr, c.cfg.DefaultPort)
		}

		numConns := 1
		//See if the host already has connections in the pool
		c.mu.Lock()
		conns, ok := c.connPool[addr]
		c.mu.Unlock()

		if ok {
			//if the host has enough connections just exit
			numConns = conns.Size()
			if numConns >= c.cfg.NumConns {
				continue
			}
		} else {
			//See if the host is reachable
			if err := c.connect(addr); err != nil {
				continue
			}
		}

		//This is reached if the host is responsive and needs more connections
		//Create connections for host synchronously to mitigate flooding the host.
		go func(a string, conns int) {
			for ; conns < c.cfg.NumConns; conns++ {
				c.connect(addr)
			}
		}(addr, numConns)
	}

	c.hostMu.RUnlock()
}

// Should only be called if c.mu is locked
func (c *SimplePool) removeConnLocked(conn *Conn) {
	conn.Close()
	connPool := c.connPool[conn.addr]
	if connPool == nil {
		return
	}
	connPool.RemoveNode(conn)
	if connPool.Size() == 0 {
		c.hostPool.RemoveNode(connPool)
		delete(c.connPool, conn.addr)
	}
	delete(c.conns, conn)
}

func (c *SimplePool) removeConn(conn *Conn) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.removeConnLocked(conn)
}

//HandleError is called by a Connection object to report to the pool an error has occured.
//Logic is then executed within the pool to clean up the erroroneous connection and try to
//top off the pool.
func (c *SimplePool) HandleError(conn *Conn, err error, closed bool) {
	if !closed {
		// ignore all non-fatal errors
		return
	}
	c.removeConn(conn)
	if !c.quit {
		go c.fillPool() // top off pool.
	}
}

//Pick selects a connection to be used by the query.
func (c *SimplePool) Pick(qry *Query) *Conn {
	//Check if connections are available
	c.mu.Lock()
	conns := len(c.conns)
	c.mu.Unlock()

	if conns == 0 {
		//try to populate the pool before returning.
		c.fillPool()
	}

	return c.hostPool.Pick(qry)
}

//Size returns the number of connections currently active in the pool
func (p *SimplePool) Size() int {
	p.mu.Lock()
	conns := len(p.conns)
	p.mu.Unlock()
	return conns
}

//Close kills the pool and all associated connections.
func (c *SimplePool) Close() {
	c.quitOnce.Do(func() {
		c.mu.Lock()
		defer c.mu.Unlock()
		c.quit = true
		close(c.quitWait)
		for conn := range c.conns {
			c.removeConnLocked(conn)
		}
	})
}

func (c *SimplePool) SetHosts(hosts []HostInfo) {
	c.hostMu.Lock()
	toRemove := make(map[string]struct{})
	for k := range c.hosts {
		toRemove[k] = struct{}{}
	}

	for _, host := range hosts {
		host := host

		delete(toRemove, host.Peer)
		// we already have it
		if _, ok := c.hosts[host.Peer]; ok {
			// TODO: Check rack, dc, token range is consistent, trigger topology change
			// update stored host
			continue
		}

		c.hosts[host.Peer] = &host
	}

	// can we hold c.mu whilst iterating this loop?
	for addr := range toRemove {
		c.removeHostLocked(addr)
	}
	c.hostMu.Unlock()
}

func (c *SimplePool) removeHostLocked(addr string) {
	if _, ok := c.hosts[addr]; !ok {
		return
	}
	delete(c.hosts, addr)

	c.mu.Lock()
	defer c.mu.Unlock()

	if _, ok := c.connPool[addr]; !ok {
		return
	}

	for conn := range c.conns {
		if conn.Address() == addr {
			c.removeConnLocked(conn)
		}
	}
}
<commit_msg>remove SimplePool.newConn<commit_after>package gocql

import (
	"fmt"
	"log"
	"strings"
	"sync"
	"time"
)

/*ConnectionPool represents the interface gocql will use to work with a collection of connections.

Purpose

The connection pool in gocql opens and closes connections as well as selects an available connection
for gocql to execute a query against. The pool is also respnsible for handling connection errors that
are caught by the connection experiencing the error.

A connection pool should make a copy of the variables used from the ClusterConfig provided to the pool
upon creation. ClusterConfig is a pointer and can be modified after the creation of the pool. This can
lead to issues with variables being modified outside the expectations of the ConnectionPool type.

Example of Single Connection Pool:

	type SingleConnection struct {
		conn *Conn
		cfg *ClusterConfig
	}

	func NewSingleConnection(cfg *ClusterConfig) ConnectionPool {
		addr := strings.TrimSpace(cfg.Hosts[0])
		if strings.Index(addr, ":") < 0 {
			addr = fmt.Sprintf("%s:%d", addr, cfg.DefaultPort)
		}
		connCfg := ConnConfig{
			ProtoVersion:  cfg.ProtoVersion,
			CQLVersion:    cfg.CQLVersion,
			Timeout:       cfg.Timeout,
			NumStreams:    cfg.NumStreams,
			Compressor:    cfg.Compressor,
			Authenticator: cfg.Authenticator,
			Keepalive:     cfg.SocketKeepalive,
		}
		pool := SingleConnection{cfg:cfg}
		pool.conn = Connect(addr,connCfg,pool)
		return &pool
	}

	func (s *SingleConnection) HandleError(conn *Conn, err error, closed bool) {
		if closed {
			connCfg := ConnConfig{
				ProtoVersion:  cfg.ProtoVersion,
				CQLVersion:    cfg.CQLVersion,
				Timeout:       cfg.Timeout,
				NumStreams:    cfg.NumStreams,
				Compressor:    cfg.Compressor,
				Authenticator: cfg.Authenticator,
				Keepalive:     cfg.SocketKeepalive,
			}
			s.conn = Connect(conn.Address(),connCfg,s)
		}
	}

	func (s *SingleConnection) Pick(qry *Query) *Conn {
		if s.conn.isClosed {
			return nil
		}
		return s.conn
	}

	func (s *SingleConnection) Size() int {
		return 1
	}

	func (s *SingleConnection) Close() {
		s.conn.Close()
	}

This is a very simple example of a type that exposes the connection pool interface. To assign
this type as the connection pool to use you would assign it to the ClusterConfig like so:

		cluster := NewCluster("127.0.0.1")
		cluster.ConnPoolType = NewSingleConnection
		...
		session, err := cluster.CreateSession()

To see a more complete example of a ConnectionPool implementation please see the SimplePool type.
*/
type ConnectionPool interface {
	Pick(*Query) *Conn
	Size() int
	HandleError(*Conn, error, bool)
	Close()
	SetHosts(host []HostInfo)
}

//NewPoolFunc is the type used by ClusterConfig to create a pool of a specific type.
type NewPoolFunc func(*ClusterConfig) ConnectionPool

//SimplePool is the current implementation of the connection pool inside gocql. This
//pool is meant to be a simple default used by gocql so users can get up and running
//quickly.
type SimplePool struct {
	cfg      *ClusterConfig
	hostPool *RoundRobin
	connPool map[string]*RoundRobin
	conns    map[*Conn]struct{}
	keyspace string

	hostMu sync.RWMutex
	// this is the set of current hosts which the pool will attempt to connect to
	hosts map[string]*HostInfo

	// protects hostpool, connPoll, conns, quit
	mu sync.Mutex

	cFillingPool chan int

	quit     bool
	quitWait chan bool
	quitOnce sync.Once
}

//NewSimplePool is the function used by gocql to create the simple connection pool.
//This is the default if no other pool type is specified.
func NewSimplePool(cfg *ClusterConfig) ConnectionPool {
	pool := &SimplePool{
		cfg:          cfg,
		hostPool:     NewRoundRobin(),
		connPool:     make(map[string]*RoundRobin),
		conns:        make(map[*Conn]struct{}),
		quitWait:     make(chan bool),
		cFillingPool: make(chan int, 1),
		keyspace:     cfg.Keyspace,
		hosts:        make(map[string]*HostInfo),
	}

	for _, host := range cfg.Hosts {
		// seed hosts have unknown topology
		// TODO: Handle populating this during SetHosts
		pool.hosts[host] = &HostInfo{Peer: host}
	}

	//Walk through connecting to hosts. As soon as one host connects
	//defer the remaining connections to cluster.fillPool()
	for i := 0; i < len(cfg.Hosts); i++ {
		addr := strings.TrimSpace(cfg.Hosts[i])
		if strings.Index(addr, ":") < 0 {
			addr = fmt.Sprintf("%s:%d", addr, cfg.DefaultPort)
		}

		if pool.connect(addr) == nil {
			pool.cFillingPool <- 1
			go pool.fillPool()
			break
		}
	}

	return pool
}

func (c *SimplePool) connect(addr string) error {

	cfg := ConnConfig{
		ProtoVersion:  c.cfg.ProtoVersion,
		CQLVersion:    c.cfg.CQLVersion,
		Timeout:       c.cfg.Timeout,
		NumStreams:    c.cfg.NumStreams,
		Compressor:    c.cfg.Compressor,
		Authenticator: c.cfg.Authenticator,
		Keepalive:     c.cfg.SocketKeepalive,
	}

	conn, err := Connect(addr, cfg, c)
	if err != nil {
		log.Printf("connect: failed to connect to %q: %v", addr, err)
		return err
	}

	return c.addConn(conn)
}

func (c *SimplePool) addConn(conn *Conn) error {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.quit {
		conn.Close()
		return nil
	}

	//Set the connection's keyspace if any before adding it to the pool
	if c.keyspace != "" {
		if err := conn.UseKeyspace(c.keyspace); err != nil {
			log.Printf("error setting connection keyspace. %v", err)
			conn.Close()
			return err
		}
	}

	connPool := c.connPool[conn.Address()]
	if connPool == nil {
		connPool = NewRoundRobin()
		c.connPool[conn.Address()] = connPool
		c.hostPool.AddNode(connPool)
	}

	connPool.AddNode(conn)
	c.conns[conn] = struct{}{}

	return nil
}

//fillPool manages the pool of connections making sure that each host has the correct
//amount of connections defined. Also the method will test a host with one connection
//instead of flooding the host with number of connections defined in the cluster config
func (c *SimplePool) fillPool() {
	//Debounce large amounts of requests to fill pool
	select {
	case <-time.After(1 * time.Millisecond):
		return
	case <-c.cFillingPool:
		defer func() { c.cFillingPool <- 1 }()
	}

	c.mu.Lock()
	isClosed := c.quit
	c.mu.Unlock()
	//Exit if cluster(session) is closed
	if isClosed {
		return
	}

	c.hostMu.RLock()

	//Walk through list of defined hosts
	for host := range c.hosts {
		addr := strings.TrimSpace(host)
		if strings.Index(addr, ":") < 0 {
			addr = fmt.Sprintf("%s:%d", addr, c.cfg.DefaultPort)
		}

		numConns := 1
		//See if the host already has connections in the pool
		c.mu.Lock()
		conns, ok := c.connPool[addr]
		c.mu.Unlock()

		if ok {
			//if the host has enough connections just exit
			numConns = conns.Size()
			if numConns >= c.cfg.NumConns {
				continue
			}
		} else {
			//See if the host is reachable
			if err := c.connect(addr); err != nil {
				continue
			}
		}

		//This is reached if the host is responsive and needs more connections
		//Create connections for host synchronously to mitigate flooding the host.
		go func(a string, conns int) {
			for ; conns < c.cfg.NumConns; conns++ {
				c.connect(addr)
			}
		}(addr, numConns)
	}

	c.hostMu.RUnlock()
}

// Should only be called if c.mu is locked
func (c *SimplePool) removeConnLocked(conn *Conn) {
	conn.Close()
	connPool := c.connPool[conn.addr]
	if connPool == nil {
		return
	}
	connPool.RemoveNode(conn)
	if connPool.Size() == 0 {
		c.hostPool.RemoveNode(connPool)
		delete(c.connPool, conn.addr)
	}
	delete(c.conns, conn)
}

func (c *SimplePool) removeConn(conn *Conn) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.removeConnLocked(conn)
}

//HandleError is called by a Connection object to report to the pool an error has occured.
//Logic is then executed within the pool to clean up the erroroneous connection and try to
//top off the pool.
func (c *SimplePool) HandleError(conn *Conn, err error, closed bool) {
	if !closed {
		// ignore all non-fatal errors
		return
	}
	c.removeConn(conn)
	if !c.quit {
		go c.fillPool() // top off pool.
	}
}

//Pick selects a connection to be used by the query.
func (c *SimplePool) Pick(qry *Query) *Conn {
	//Check if connections are available
	c.mu.Lock()
	conns := len(c.conns)
	c.mu.Unlock()

	if conns == 0 {
		//try to populate the pool before returning.
		c.fillPool()
	}

	return c.hostPool.Pick(qry)
}

//Size returns the number of connections currently active in the pool
func (p *SimplePool) Size() int {
	p.mu.Lock()
	conns := len(p.conns)
	p.mu.Unlock()
	return conns
}

//Close kills the pool and all associated connections.
func (c *SimplePool) Close() {
	c.quitOnce.Do(func() {
		c.mu.Lock()
		defer c.mu.Unlock()
		c.quit = true
		close(c.quitWait)
		for conn := range c.conns {
			c.removeConnLocked(conn)
		}
	})
}

func (c *SimplePool) SetHosts(hosts []HostInfo) {
	c.hostMu.Lock()
	toRemove := make(map[string]struct{})
	for k := range c.hosts {
		toRemove[k] = struct{}{}
	}

	for _, host := range hosts {
		host := host

		delete(toRemove, host.Peer)
		// we already have it
		if _, ok := c.hosts[host.Peer]; ok {
			// TODO: Check rack, dc, token range is consistent, trigger topology change
			// update stored host
			continue
		}

		c.hosts[host.Peer] = &host
	}

	// can we hold c.mu whilst iterating this loop?
	for addr := range toRemove {
		c.removeHostLocked(addr)
	}
	c.hostMu.Unlock()
}

func (c *SimplePool) removeHostLocked(addr string) {
	if _, ok := c.hosts[addr]; !ok {
		return
	}
	delete(c.hosts, addr)

	c.mu.Lock()
	defer c.mu.Unlock()

	if _, ok := c.connPool[addr]; !ok {
		return
	}

	for conn := range c.conns {
		if conn.Address() == addr {
			c.removeConnLocked(conn)
		}
	}
}
<|endoftext|>


<commit_before>package kloud

import (
	"errors"
	"strconv"
	"time"

	"github.com/koding/kite"
)

// Builder is used to create and provisiong a single image or machine for a
// given Provider.
type Builder interface {
	// Prepare is responsible of configuring the builder and validating the
	// given configuration prior Build.
	Prepare(...interface{}) error

	// Build is creating a image and a machine.
	Build(...interface{}) (map[string]interface{}, error)
}

type BuildResponse struct {
	MachineName string `json:"machineName" mapstructure:"machineName"`
	MachineId   int    `json:"machineId" mapstructure:"machineId"`
	KiteId      string `json:"kiteId" mapstructure:"kiteId"`
	IpAddress   string `json:"ipAddress" mapstructure:"ipAddress"`
}

type BuildArgs struct {
	MachineId    string
	SnapshotName string
	MachineName  string
}

func (k *Kloud) build(r *kite.Request) (interface{}, error) {
	args := &BuildArgs{}
	if err := r.Args.One().Unmarshal(args); err != nil {
		return nil, err
	}

	if args.MachineId == "" {
		return nil, errors.New("machineId is missing.")
	}

	// this locks are important to prevent consecutive calls from the same user
	k.idlock.Get(r.Username).Lock()
	defer k.idlock.Get(r.Username).Unlock()

	machineData, err := k.Storage.Get(args.MachineId)
	if err != nil {
		return nil, err
	}

	p, ok := providers[machineData.Provider]
	if !ok {
		return nil, errors.New("provider not supported")
	}

	provider, ok := p.(Builder)
	if !ok {
		return nil, errors.New("provider doesn't satisfy the builder interface.")
	}

	if err := provider.Prepare(machineData.Credential, machineData.Builders); err != nil {
		return nil, err
	}

	snapshotName := defaultSnapshotName
	if args.SnapshotName != "" {
		snapshotName = args.SnapshotName
	}

	signFunc := func() (string, string, error) {
		return createKey(r.Username, k.KontrolURL, k.KontrolPrivateKey, k.KontrolPublicKey)
	}

	machineName := r.Username + "-" + strconv.FormatInt(time.Now().UTC().UnixNano(), 10)
	if args.MachineName != "" {
		machineName = args.MachineName
	}

	artifact, err := provider.Build(snapshotName, machineName, signFunc)
	if err != nil {
		return nil, err
	}

	if err := k.Storage.Update(args.MachineId, artifact); err != nil {
		return nil, err
	}

	return artifact, nil
}
<commit_msg>kloud/builder: refactor build function<commit_after>package kloud

import (
	"errors"
	"strconv"
	"time"

	"github.com/koding/kite"
)

// Builder is used to create and provisiong a single image or machine for a
// given Provider.
type Builder interface {
	// Prepare is responsible of configuring the builder and validating the
	// given configuration prior Build.
	Prepare(...interface{}) error

	// Build is creating a image and a machine.
	Build(...interface{}) (map[string]interface{}, error)
}

type BuildResponse struct {
	MachineName string `json:"machineName" mapstructure:"machineName"`
	MachineId   int    `json:"machineId" mapstructure:"machineId"`
	KiteId      string `json:"kiteId" mapstructure:"kiteId"`
	IpAddress   string `json:"ipAddress" mapstructure:"ipAddress"`
}

type BuildArgs struct {
	MachineId    string
	SnapshotName string
	MachineName  string
}

func (k *Kloud) build(r *kite.Request) (interface{}, error) {
	args := &BuildArgs{}
	if err := r.Args.One().Unmarshal(args); err != nil {
		return nil, err
	}

	if args.MachineId == "" {
		return nil, errors.New("machineId is missing.")
	}

	// this locks are important to prevent consecutive calls from the same user
	k.idlock.Get(r.Username).Lock()
	defer k.idlock.Get(r.Username).Unlock()

	snapshotName := defaultSnapshotName
	if args.SnapshotName != "" {
		snapshotName = args.SnapshotName
	}

	signFunc := func() (string, string, error) {
		return createKey(r.Username, k.KontrolURL, k.KontrolPrivateKey, k.KontrolPublicKey)
	}

	machineName := r.Username + "-" + strconv.FormatInt(time.Now().UTC().UnixNano(), 10)
	if args.MachineName != "" {
		machineName = args.MachineName
	}

	provider, err := k.provider(args.MachineId)
	if err != nil {
		return nil, err
	}

	artifact, err := provider.Build(snapshotName, machineName, signFunc)
	if err != nil {
		return nil, err
	}

	if err := k.Storage.Update(args.MachineId, artifact); err != nil {
		return nil, err
	}

	return artifact, nil
}
<|endoftext|>


<commit_before>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package binlogplayer contains the code that plays a filtered replication
// stream on a client database. It usually runs inside the destination master
// vttablet process.
package binlogplayer

import (
	"fmt"
	"io"
	"sync"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/mysql"
	mproto "github.com/youtube/vitess/go/mysql/proto"
	"github.com/youtube/vitess/go/stats"
	"github.com/youtube/vitess/go/sync2"
	"github.com/youtube/vitess/go/vt/binlog/proto"
	"github.com/youtube/vitess/go/vt/key"
	myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto"
)

var (
	// we will log anything that's higher than that
	SLOW_QUERY_THRESHOLD = time.Duration(100 * time.Millisecond)

	// keys for the stats map
	BLPL_QUERY       = "Query"
	BLPL_TRANSACTION = "Transaction"

	// flags for the blp_checkpoint table. The database entry is just
	// a join(",") of these flags.
	BLP_FLAG_DONT_START = "DontStart"
)

// BinlogPlayerStats is the internal stats of a player. It is a different
// structure that is passed in so stats can be collected over the life
// of multiple individual players.
type BinlogPlayerStats struct {
	// Stats about the player, keys used are BLPL_QUERY and BLPL_TRANSACTION
	Timings *stats.Timings
	Rates   *stats.Rates

	// Last saved status
	LastGTID            myproto.GTID
	LastGTIDMutex       sync.Mutex
	SecondsBehindMaster sync2.AtomicInt64
}

func (bps *BinlogPlayerStats) SetLastGTID(gtid myproto.GTID) {
	bps.LastGTIDMutex.Lock()
	defer bps.LastGTIDMutex.Unlock()
	bps.LastGTID = gtid
}

// NewBinlogPlayerStats creates a new BinlogPlayerStats structure
func NewBinlogPlayerStats() *BinlogPlayerStats {
	bps := &BinlogPlayerStats{}
	bps.Timings = stats.NewTimings("")
	bps.Rates = stats.NewRates("", bps.Timings, 15, 60e9)
	return bps
}

// BinlogPlayer is handling reading a stream of updates from BinlogServer
type BinlogPlayer struct {
	addr     string
	dbClient VtClient

	// for key range base requests
	keyspaceIdType key.KeyspaceIdType
	keyRange       key.KeyRange

	// for table base requests
	tables []string

	// common to all
	blpPos     proto.BlpPosition
	stopAtGTID myproto.GTID
	blplStats  *BinlogPlayerStats
}

// NewBinlogPlayerKeyRange returns a new BinlogPlayer pointing at the server
// replicating the provided keyrange, starting at the startPosition.GTID,
// and updating _vt.blp_checkpoint with uid=startPosition.Uid.
// If stopAtGTID != nil, it will stop when reaching that GTID.
func NewBinlogPlayerKeyRange(dbClient VtClient, addr string, keyspaceIdType key.KeyspaceIdType, keyRange key.KeyRange, startPosition *proto.BlpPosition, stopAtGTID myproto.GTID, blplStats *BinlogPlayerStats) *BinlogPlayer {
	return &BinlogPlayer{
		addr:           addr,
		dbClient:       dbClient,
		keyspaceIdType: keyspaceIdType,
		keyRange:       keyRange,
		blpPos:         *startPosition,
		stopAtGTID:     stopAtGTID,
		blplStats:      blplStats,
	}
}

// NewBinlogPlayerTables returns a new BinlogPlayer pointing at the server
// replicating the provided tables, starting at the startPosition.GTID,
// and updating _vt.blp_checkpoint with uid=startPosition.Uid.
// If stopAtGTID != nil, it will stop when reaching that GTID.
func NewBinlogPlayerTables(dbClient VtClient, addr string, tables []string, startPosition *proto.BlpPosition, stopAtGTID myproto.GTID, blplStats *BinlogPlayerStats) *BinlogPlayer {
	return &BinlogPlayer{
		addr:       addr,
		dbClient:   dbClient,
		tables:     tables,
		blpPos:     *startPosition,
		stopAtGTID: stopAtGTID,
		blplStats:  blplStats,
	}
}

// writeRecoveryPosition will write the current GTID as the recovery position
// for the next transaction.
// We will also try to get the timestamp for the transaction. Two cases:
// - we have statements, and they start with a SET TIMESTAMP that we
//   can parse: then we update transaction_timestamp in blp_checkpoint
//   with it, and set SecondsBehindMaster to now() - transaction_timestamp
// - otherwise (the statements are probably filtered out), we leave
//   transaction_timestamp alone (keeping the old value), and we don't
//   change SecondsBehindMaster
func (blp *BinlogPlayer) writeRecoveryPosition(tx *proto.BinlogTransaction) error {
	now := time.Now().Unix()

	blp.blpPos.GTID = tx.GTID
	updateRecovery := ""
	if tx.Timestamp != 0 {
		updateRecovery = fmt.Sprintf(
			"UPDATE _vt.blp_checkpoint SET gtid='%v', time_updated=%v, transaction_timestamp=%v WHERE source_shard_uid=%v",
			tx.GTID,
			now,
			tx.Timestamp,
			blp.blpPos.Uid)
	} else {
		updateRecovery = fmt.Sprintf(
			"UPDATE _vt.blp_checkpoint SET gtid='%v', time_updated=%v WHERE source_shard_uid=%v",
			tx.GTID,
			now,
			blp.blpPos.Uid)
	}

	qr, err := blp.exec(updateRecovery)
	if err != nil {
		return fmt.Errorf("Error %v in writing recovery info %v", err, updateRecovery)
	}
	if qr.RowsAffected != 1 {
		return fmt.Errorf("Cannot update blp_recovery table, affected %v rows", qr.RowsAffected)
	}
	blp.blplStats.SetLastGTID(tx.GTID)
	if tx.Timestamp != 0 {
		blp.blplStats.SecondsBehindMaster.Set(now - tx.Timestamp)
	}
	return nil
}

// ReadStartPosition will return the current start position and the flags for
// the provided binlog player.
func ReadStartPosition(dbClient VtClient, uid uint32) (*proto.BlpPosition, string, error) {
	selectRecovery := QueryBlpCheckpoint(uid)
	qr, err := dbClient.ExecuteFetch(selectRecovery, 1, true)
	if err != nil {
		return nil, "", fmt.Errorf("error %v in selecting from recovery table %v", err, selectRecovery)
	}
	if qr.RowsAffected != 1 {
		return nil, "", fmt.Errorf("checkpoint information not available in db for %v", uid)
	}
	gtid, err := myproto.DecodeGTID(qr.Rows[0][0].String())
	if err != nil {
		return nil, "", err
	}
	return &proto.BlpPosition{
		Uid:  uid,
		GTID: myproto.GTIDField{gtid},
	}, string(qr.Rows[0][1].Raw()), nil
}

func (blp *BinlogPlayer) processTransaction(tx *proto.BinlogTransaction) (ok bool, err error) {
	txnStartTime := time.Now()
	if err = blp.dbClient.Begin(); err != nil {
		return false, fmt.Errorf("failed query BEGIN, err: %s", err)
	}
	if err = blp.writeRecoveryPosition(tx); err != nil {
		return false, err
	}
	for _, stmt := range tx.Statements {
		if _, err = blp.exec(string(stmt.Sql)); err == nil {
			continue
		}
		if sqlErr, ok := err.(*mysql.SqlError); ok && sqlErr.Number() == 1213 {
			// Deadlock: ask for retry
			log.Infof("Deadlock: %v", err)
			if err = blp.dbClient.Rollback(); err != nil {
				return false, err
			}
			return false, nil
		}
		return false, err
	}
	if err = blp.dbClient.Commit(); err != nil {
		return false, fmt.Errorf("failed query COMMIT, err: %s", err)
	}
	blp.blplStats.Timings.Record(BLPL_TRANSACTION, txnStartTime)
	return true, nil
}

func (blp *BinlogPlayer) exec(sql string) (*mproto.QueryResult, error) {
	queryStartTime := time.Now()
	qr, err := blp.dbClient.ExecuteFetch(sql, 0, false)
	blp.blplStats.Timings.Record(BLPL_QUERY, queryStartTime)
	if time.Now().Sub(queryStartTime) > SLOW_QUERY_THRESHOLD {
		log.Infof("SLOW QUERY '%s'", sql)
	}
	return qr, err
}

// ApplyBinlogEvents makes a gob rpc request to BinlogServer
// and processes the events. It will return nil if 'interrupted'
// was closed, or if we reached the stopping point.
// It will return io.EOF if the server stops sending us updates.
// It may return any other error it encounters.
func (blp *BinlogPlayer) ApplyBinlogEvents(interrupted chan struct{}) error {
	if len(blp.tables) > 0 {
		log.Infof("BinlogPlayer client %v for tables %v starting @ '%v', server: %v",
			blp.blpPos.Uid,
			blp.tables,
			blp.blpPos.GTID,
			blp.addr,
		)
	} else {
		log.Infof("BinlogPlayer client %v for keyrange '%v-%v' starting @ '%v', server: %v",
			blp.blpPos.Uid,
			blp.keyRange.Start.Hex(),
			blp.keyRange.End.Hex(),
			blp.blpPos.GTID,
			blp.addr,
		)
	}
	if blp.stopAtGTID != nil {
		// We need to stop at some point. Sanity check the point.
		//
		// TODO(enisoc): Stop trying to compare GTIDs, since we can't do that
		// reliably for some MySQL flavors.
		cmp, err := blp.blpPos.GTID.TryCompare(blp.stopAtGTID)
		if err != nil {
			return err
		}
		if cmp > 0 { // blp.blpPos.GTID > blp.stopAtGTID
			return fmt.Errorf("starting point %v greater than stopping point %v", blp.blpPos.GTID, blp.stopAtGTID)
		} else if cmp == 0 { // blp.blpPos.GTID == blp.stopAtGTID
			log.Infof("Not starting BinlogPlayer, we're already at the desired position %v", blp.stopAtGTID)
			return nil
		}
		log.Infof("Will stop player when reaching %v", blp.stopAtGTID)
	}

	binlogPlayerClientFactory, ok := binlogPlayerClientFactories[*binlogPlayerProtocol]
	if !ok {
		return fmt.Errorf("no binlog player client factory named %v", *binlogPlayerProtocol)
	}
	blplClient := binlogPlayerClientFactory()
	err := blplClient.Dial(blp.addr, *binlogPlayerConnTimeout)
	if err != nil {
		log.Errorf("Error dialing binlog server: %v", err)
		return fmt.Errorf("error dialing binlog server: %v", err)
	}
	defer blplClient.Close()

	responseChan := make(chan *proto.BinlogTransaction)
	var resp BinlogPlayerResponse
	if len(blp.tables) > 0 {
		req := &proto.TablesRequest{
			Tables: blp.tables,
			GTID:   blp.blpPos.GTID,
		}
		resp = blplClient.StreamTables(req, responseChan)
	} else {
		req := &proto.KeyRangeRequest{
			KeyspaceIdType: blp.keyspaceIdType,
			KeyRange:       blp.keyRange,
			GTID:           blp.blpPos.GTID,
		}
		resp = blplClient.StreamKeyRange(req, responseChan)
	}

processLoop:
	for {
		select {
		case response, ok := <-responseChan:
			if !ok {
				break processLoop
			}
			for {
				ok, err = blp.processTransaction(response)
				if err != nil {
					return fmt.Errorf("Error in processing binlog event %v", err)
				}
				if ok {
					if blp.stopAtGTID != nil {
						// TODO(enisoc): Stop trying to compare GTIDs.
						cmp, err := blp.blpPos.GTID.TryCompare(blp.stopAtGTID)
						if err != nil {
							return err
						}
						if cmp >= 0 { // blp.blpPos.GTID >= blp.stopAtGTID
							log.Infof("Reached stopping position, done playing logs")
							return nil
						}
					}
					break
				}
				log.Infof("Retrying txn")
				time.Sleep(1 * time.Second)
			}
		case <-interrupted:
			return nil
		}
	}
	if resp.Error() != nil {
		return fmt.Errorf("Error received from ServeBinlog %v", resp.Error())
	}
	return io.EOF
}

// CreateBlpCheckpoint returns the statements required to create
// the _vt.blp_checkpoint table
func CreateBlpCheckpoint() []string {
	return []string{
		"CREATE DATABASE IF NOT EXISTS _vt",
		`CREATE TABLE IF NOT EXISTS _vt.blp_checkpoint (
  source_shard_uid INT(10) UNSIGNED NOT NULL,
  gtid VARCHAR(250) DEFAULT NULL,
  time_updated BIGINT UNSIGNED NOT NULL,
  transaction_timestamp BIGINT UNSIGNED NOT NULL,
  flags VARCHAR(250) DEFAULT NULL,
  PRIMARY KEY (source_shard_uid)) ENGINE=InnoDB`}
}

// PopulateBlpCheckpoint returns a statement to populate the first value into
// the _vt.blp_checkpoint table
func PopulateBlpCheckpoint(index uint32, gtid myproto.GTID, timeUpdated int64, flags string) string {
	return fmt.Sprintf("INSERT INTO _vt.blp_checkpoint (source_shard_uid, gtid, time_updated, transaction_timestamp, flags) VALUES (%v, '%v', %v, 0, '%v')", index, gtid, timeUpdated, flags)
}

// QueryBlpCheckpoint returns a statement to query the gtid and flags for a
// given shard from the _vt.blp_checkpoint table
func QueryBlpCheckpoint(index uint32) string {
	return fmt.Sprintf("SELECT gtid, flags FROM _vt.blp_checkpoint WHERE source_shard_uid=%v", index)
}
<commit_msg>Add synchronized GetLastGTID() method.<commit_after>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package binlogplayer contains the code that plays a filtered replication
// stream on a client database. It usually runs inside the destination master
// vttablet process.
package binlogplayer

import (
	"fmt"
	"io"
	"sync"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/mysql"
	mproto "github.com/youtube/vitess/go/mysql/proto"
	"github.com/youtube/vitess/go/stats"
	"github.com/youtube/vitess/go/sync2"
	"github.com/youtube/vitess/go/vt/binlog/proto"
	"github.com/youtube/vitess/go/vt/key"
	myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto"
)

var (
	// we will log anything that's higher than that
	SLOW_QUERY_THRESHOLD = time.Duration(100 * time.Millisecond)

	// keys for the stats map
	BLPL_QUERY       = "Query"
	BLPL_TRANSACTION = "Transaction"

	// flags for the blp_checkpoint table. The database entry is just
	// a join(",") of these flags.
	BLP_FLAG_DONT_START = "DontStart"
)

// BinlogPlayerStats is the internal stats of a player. It is a different
// structure that is passed in so stats can be collected over the life
// of multiple individual players.
type BinlogPlayerStats struct {
	// Stats about the player, keys used are BLPL_QUERY and BLPL_TRANSACTION
	Timings *stats.Timings
	Rates   *stats.Rates

	// Last saved status
	LastGTID            myproto.GTID
	LastGTIDMutex       sync.RWMutex
	SecondsBehindMaster sync2.AtomicInt64
}

func (bps *BinlogPlayerStats) SetLastGTID(gtid myproto.GTID) {
	bps.LastGTIDMutex.Lock()
	defer bps.LastGTIDMutex.Unlock()
	bps.LastGTID = gtid
}

func (bps *BinlogPlayerStats) GetLastGTID() myproto.GTID {
	bps.LastGTIDMutex.RLock()
	defer bps.LastGTIDMutex.RUnlock()
	return bps.LastGTID
}

// NewBinlogPlayerStats creates a new BinlogPlayerStats structure
func NewBinlogPlayerStats() *BinlogPlayerStats {
	bps := &BinlogPlayerStats{}
	bps.Timings = stats.NewTimings("")
	bps.Rates = stats.NewRates("", bps.Timings, 15, 60e9)
	return bps
}

// BinlogPlayer is handling reading a stream of updates from BinlogServer
type BinlogPlayer struct {
	addr     string
	dbClient VtClient

	// for key range base requests
	keyspaceIdType key.KeyspaceIdType
	keyRange       key.KeyRange

	// for table base requests
	tables []string

	// common to all
	blpPos     proto.BlpPosition
	stopAtGTID myproto.GTID
	blplStats  *BinlogPlayerStats
}

// NewBinlogPlayerKeyRange returns a new BinlogPlayer pointing at the server
// replicating the provided keyrange, starting at the startPosition.GTID,
// and updating _vt.blp_checkpoint with uid=startPosition.Uid.
// If stopAtGTID != nil, it will stop when reaching that GTID.
func NewBinlogPlayerKeyRange(dbClient VtClient, addr string, keyspaceIdType key.KeyspaceIdType, keyRange key.KeyRange, startPosition *proto.BlpPosition, stopAtGTID myproto.GTID, blplStats *BinlogPlayerStats) *BinlogPlayer {
	return &BinlogPlayer{
		addr:           addr,
		dbClient:       dbClient,
		keyspaceIdType: keyspaceIdType,
		keyRange:       keyRange,
		blpPos:         *startPosition,
		stopAtGTID:     stopAtGTID,
		blplStats:      blplStats,
	}
}

// NewBinlogPlayerTables returns a new BinlogPlayer pointing at the server
// replicating the provided tables, starting at the startPosition.GTID,
// and updating _vt.blp_checkpoint with uid=startPosition.Uid.
// If stopAtGTID != nil, it will stop when reaching that GTID.
func NewBinlogPlayerTables(dbClient VtClient, addr string, tables []string, startPosition *proto.BlpPosition, stopAtGTID myproto.GTID, blplStats *BinlogPlayerStats) *BinlogPlayer {
	return &BinlogPlayer{
		addr:       addr,
		dbClient:   dbClient,
		tables:     tables,
		blpPos:     *startPosition,
		stopAtGTID: stopAtGTID,
		blplStats:  blplStats,
	}
}

// writeRecoveryPosition will write the current GTID as the recovery position
// for the next transaction.
// We will also try to get the timestamp for the transaction. Two cases:
// - we have statements, and they start with a SET TIMESTAMP that we
//   can parse: then we update transaction_timestamp in blp_checkpoint
//   with it, and set SecondsBehindMaster to now() - transaction_timestamp
// - otherwise (the statements are probably filtered out), we leave
//   transaction_timestamp alone (keeping the old value), and we don't
//   change SecondsBehindMaster
func (blp *BinlogPlayer) writeRecoveryPosition(tx *proto.BinlogTransaction) error {
	now := time.Now().Unix()

	blp.blpPos.GTID = tx.GTID
	updateRecovery := ""
	if tx.Timestamp != 0 {
		updateRecovery = fmt.Sprintf(
			"UPDATE _vt.blp_checkpoint SET gtid='%v', time_updated=%v, transaction_timestamp=%v WHERE source_shard_uid=%v",
			tx.GTID,
			now,
			tx.Timestamp,
			blp.blpPos.Uid)
	} else {
		updateRecovery = fmt.Sprintf(
			"UPDATE _vt.blp_checkpoint SET gtid='%v', time_updated=%v WHERE source_shard_uid=%v",
			tx.GTID,
			now,
			blp.blpPos.Uid)
	}

	qr, err := blp.exec(updateRecovery)
	if err != nil {
		return fmt.Errorf("Error %v in writing recovery info %v", err, updateRecovery)
	}
	if qr.RowsAffected != 1 {
		return fmt.Errorf("Cannot update blp_recovery table, affected %v rows", qr.RowsAffected)
	}
	blp.blplStats.SetLastGTID(tx.GTID)
	if tx.Timestamp != 0 {
		blp.blplStats.SecondsBehindMaster.Set(now - tx.Timestamp)
	}
	return nil
}

// ReadStartPosition will return the current start position and the flags for
// the provided binlog player.
func ReadStartPosition(dbClient VtClient, uid uint32) (*proto.BlpPosition, string, error) {
	selectRecovery := QueryBlpCheckpoint(uid)
	qr, err := dbClient.ExecuteFetch(selectRecovery, 1, true)
	if err != nil {
		return nil, "", fmt.Errorf("error %v in selecting from recovery table %v", err, selectRecovery)
	}
	if qr.RowsAffected != 1 {
		return nil, "", fmt.Errorf("checkpoint information not available in db for %v", uid)
	}
	gtid, err := myproto.DecodeGTID(qr.Rows[0][0].String())
	if err != nil {
		return nil, "", err
	}
	return &proto.BlpPosition{
		Uid:  uid,
		GTID: myproto.GTIDField{gtid},
	}, string(qr.Rows[0][1].Raw()), nil
}

func (blp *BinlogPlayer) processTransaction(tx *proto.BinlogTransaction) (ok bool, err error) {
	txnStartTime := time.Now()
	if err = blp.dbClient.Begin(); err != nil {
		return false, fmt.Errorf("failed query BEGIN, err: %s", err)
	}
	if err = blp.writeRecoveryPosition(tx); err != nil {
		return false, err
	}
	for _, stmt := range tx.Statements {
		if _, err = blp.exec(string(stmt.Sql)); err == nil {
			continue
		}
		if sqlErr, ok := err.(*mysql.SqlError); ok && sqlErr.Number() == 1213 {
			// Deadlock: ask for retry
			log.Infof("Deadlock: %v", err)
			if err = blp.dbClient.Rollback(); err != nil {
				return false, err
			}
			return false, nil
		}
		return false, err
	}
	if err = blp.dbClient.Commit(); err != nil {
		return false, fmt.Errorf("failed query COMMIT, err: %s", err)
	}
	blp.blplStats.Timings.Record(BLPL_TRANSACTION, txnStartTime)
	return true, nil
}

func (blp *BinlogPlayer) exec(sql string) (*mproto.QueryResult, error) {
	queryStartTime := time.Now()
	qr, err := blp.dbClient.ExecuteFetch(sql, 0, false)
	blp.blplStats.Timings.Record(BLPL_QUERY, queryStartTime)
	if time.Now().Sub(queryStartTime) > SLOW_QUERY_THRESHOLD {
		log.Infof("SLOW QUERY '%s'", sql)
	}
	return qr, err
}

// ApplyBinlogEvents makes a gob rpc request to BinlogServer
// and processes the events. It will return nil if 'interrupted'
// was closed, or if we reached the stopping point.
// It will return io.EOF if the server stops sending us updates.
// It may return any other error it encounters.
func (blp *BinlogPlayer) ApplyBinlogEvents(interrupted chan struct{}) error {
	if len(blp.tables) > 0 {
		log.Infof("BinlogPlayer client %v for tables %v starting @ '%v', server: %v",
			blp.blpPos.Uid,
			blp.tables,
			blp.blpPos.GTID,
			blp.addr,
		)
	} else {
		log.Infof("BinlogPlayer client %v for keyrange '%v-%v' starting @ '%v', server: %v",
			blp.blpPos.Uid,
			blp.keyRange.Start.Hex(),
			blp.keyRange.End.Hex(),
			blp.blpPos.GTID,
			blp.addr,
		)
	}
	if blp.stopAtGTID != nil {
		// We need to stop at some point. Sanity check the point.
		//
		// TODO(enisoc): Stop trying to compare GTIDs, since we can't do that
		// reliably for some MySQL flavors.
		cmp, err := blp.blpPos.GTID.TryCompare(blp.stopAtGTID)
		if err != nil {
			return err
		}
		if cmp > 0 { // blp.blpPos.GTID > blp.stopAtGTID
			return fmt.Errorf("starting point %v greater than stopping point %v", blp.blpPos.GTID, blp.stopAtGTID)
		} else if cmp == 0 { // blp.blpPos.GTID == blp.stopAtGTID
			log.Infof("Not starting BinlogPlayer, we're already at the desired position %v", blp.stopAtGTID)
			return nil
		}
		log.Infof("Will stop player when reaching %v", blp.stopAtGTID)
	}

	binlogPlayerClientFactory, ok := binlogPlayerClientFactories[*binlogPlayerProtocol]
	if !ok {
		return fmt.Errorf("no binlog player client factory named %v", *binlogPlayerProtocol)
	}
	blplClient := binlogPlayerClientFactory()
	err := blplClient.Dial(blp.addr, *binlogPlayerConnTimeout)
	if err != nil {
		log.Errorf("Error dialing binlog server: %v", err)
		return fmt.Errorf("error dialing binlog server: %v", err)
	}
	defer blplClient.Close()

	responseChan := make(chan *proto.BinlogTransaction)
	var resp BinlogPlayerResponse
	if len(blp.tables) > 0 {
		req := &proto.TablesRequest{
			Tables: blp.tables,
			GTID:   blp.blpPos.GTID,
		}
		resp = blplClient.StreamTables(req, responseChan)
	} else {
		req := &proto.KeyRangeRequest{
			KeyspaceIdType: blp.keyspaceIdType,
			KeyRange:       blp.keyRange,
			GTID:           blp.blpPos.GTID,
		}
		resp = blplClient.StreamKeyRange(req, responseChan)
	}

processLoop:
	for {
		select {
		case response, ok := <-responseChan:
			if !ok {
				break processLoop
			}
			for {
				ok, err = blp.processTransaction(response)
				if err != nil {
					return fmt.Errorf("Error in processing binlog event %v", err)
				}
				if ok {
					if blp.stopAtGTID != nil {
						// TODO(enisoc): Stop trying to compare GTIDs.
						cmp, err := blp.blpPos.GTID.TryCompare(blp.stopAtGTID)
						if err != nil {
							return err
						}
						if cmp >= 0 { // blp.blpPos.GTID >= blp.stopAtGTID
							log.Infof("Reached stopping position, done playing logs")
							return nil
						}
					}
					break
				}
				log.Infof("Retrying txn")
				time.Sleep(1 * time.Second)
			}
		case <-interrupted:
			return nil
		}
	}
	if resp.Error() != nil {
		return fmt.Errorf("Error received from ServeBinlog %v", resp.Error())
	}
	return io.EOF
}

// CreateBlpCheckpoint returns the statements required to create
// the _vt.blp_checkpoint table
func CreateBlpCheckpoint() []string {
	return []string{
		"CREATE DATABASE IF NOT EXISTS _vt",
		`CREATE TABLE IF NOT EXISTS _vt.blp_checkpoint (
  source_shard_uid INT(10) UNSIGNED NOT NULL,
  gtid VARCHAR(250) DEFAULT NULL,
  time_updated BIGINT UNSIGNED NOT NULL,
  transaction_timestamp BIGINT UNSIGNED NOT NULL,
  flags VARCHAR(250) DEFAULT NULL,
  PRIMARY KEY (source_shard_uid)) ENGINE=InnoDB`}
}

// PopulateBlpCheckpoint returns a statement to populate the first value into
// the _vt.blp_checkpoint table
func PopulateBlpCheckpoint(index uint32, gtid myproto.GTID, timeUpdated int64, flags string) string {
	return fmt.Sprintf("INSERT INTO _vt.blp_checkpoint (source_shard_uid, gtid, time_updated, transaction_timestamp, flags) VALUES (%v, '%v', %v, 0, '%v')", index, gtid, timeUpdated, flags)
}

// QueryBlpCheckpoint returns a statement to query the gtid and flags for a
// given shard from the _vt.blp_checkpoint table
func QueryBlpCheckpoint(index uint32) string {
	return fmt.Sprintf("SELECT gtid, flags FROM _vt.blp_checkpoint WHERE source_shard_uid=%v", index)
}
<|endoftext|>


<commit_before>package middleware

import (
	"fmt"
	"io"
	"net/http"
	"path"

	"github.com/labstack/echo"
)

type (
	StaticOptions struct {
		Root   string `json:"root"`
		Index  string `json:"index"`
		Browse bool   `json:"browse"`
	}
)

func Static(root string, options ...*StaticOptions) echo.MiddlewareFunc {
	return func(next echo.Handler) echo.Handler {
		// Default options
		opts := new(StaticOptions)
		if len(options) > 0 {
			opts = options[0]
		}
		if opts.Index == "" {
			opts.Index = "index.html"
		}

		return echo.HandlerFunc(func(c echo.Context) error {
			fs := http.Dir(root)
			file := path.Clean(c.Request().URL().Path())
			f, err := fs.Open(file)
			if err != nil {
				return next.Handle(c)
			}
			defer f.Close()

			fi, err := f.Stat()
			if err != nil {
				return err
			}

			if fi.IsDir() {
				/* NOTE:
				Not checking the Last-Modified header as it caches the response `304` when
				changing differnt directories for the same path.
				*/
				d := f

				// Index file
				file = path.Join(file, opts.Index)
				f, err = fs.Open(file)
				if err != nil {
					if opts.Browse {
						dirs, err := d.Readdir(-1)
						if err != nil {
							return err
						}

						// Create a directory index
						res := c.Response()
						res.Header().Set(echo.ContentType, echo.TextHTMLCharsetUTF8)
						if _, err = fmt.Fprintf(res, "
\n"); err != nil {
							return err
						}
						for _, d := range dirs {
							name := d.Name()
							color := "#212121"
							if d.IsDir() {
								color = "#e91e63"
								name += "/"
							}
							if _, err = fmt.Fprintf(res, "%s\n", name, color, name); err != nil {
								return err
							}
						}
						_, err = fmt.Fprintf(res, "
\n")
						return err
					}
					return next.Handle(c)
				}
				fi, _ = f.Stat() // Index file stat
			}
			c.Response().WriteHeader(http.StatusOK)
			io.Copy(c.Response(), f)
			return nil
			// TODO:
			// http.ServeContent(c.Response(), c.Request(), fi.Name(), fi.ModTime(), f)
		})
	}
}

// Favicon serves the default favicon - GET /favicon.ico.
func Favicon() echo.HandlerFunc {
	return func(c echo.Context) error {
		return nil
	}
}
<commit_msg>Content type for static middleware<commit_after>package middleware

import (
	"fmt"
	"io"
	"mime"
	"net/http"
	"path"
	"path/filepath"

	"github.com/labstack/echo"
)

type (
	StaticOptions struct {
		Root   string `json:"root"`
		Index  string `json:"index"`
		Browse bool   `json:"browse"`
	}
)

func Static(root string, options ...*StaticOptions) echo.MiddlewareFunc {
	return func(next echo.Handler) echo.Handler {
		// Default options
		opts := new(StaticOptions)
		if len(options) > 0 {
			opts = options[0]
		}
		if opts.Index == "" {
			opts.Index = "index.html"
		}

		return echo.HandlerFunc(func(c echo.Context) error {
			fs := http.Dir(root)
			file := path.Clean(c.Request().URL().Path())
			f, err := fs.Open(file)
			if err != nil {
				return next.Handle(c)
			}
			defer f.Close()

			fi, err := f.Stat()
			if err != nil {
				return err
			}

			if fi.IsDir() {
				/* NOTE:
				Not checking the Last-Modified header as it caches the response `304` when
				changing differnt directories for the same path.
				*/
				d := f

				// Index file
				file = path.Join(file, opts.Index)
				f, err = fs.Open(file)
				if err != nil {
					if opts.Browse {
						dirs, err := d.Readdir(-1)
						if err != nil {
							return err
						}

						// Create a directory index
						res := c.Response()
						res.Header().Set(echo.ContentType, echo.TextHTMLCharsetUTF8)
						if _, err = fmt.Fprintf(res, "
\n"); err != nil {
							return err
						}
						for _, d := range dirs {
							name := d.Name()
							color := "#212121"
							if d.IsDir() {
								color = "#e91e63"
								name += "/"
							}
							if _, err = fmt.Fprintf(res, "%s\n", name, color, name); err != nil {
								return err
							}
						}
						_, err = fmt.Fprintf(res, "
\n")
						return err
					}
					return next.Handle(c)
				}
				fi, _ = f.Stat() // Index file stat
			}
			ct := mime.TypeByExtension(filepath.Ext(fi.Name()))
			if ct == "" {
				ct = echo.OctetStream
			}
			c.Response().Header().Set(echo.ContentType, ct)
			c.Response().WriteHeader(http.StatusOK)
			_, err = io.Copy(c.Response(), f)
			return err
			// TODO:
			// http.ServeContent(c.Response(), c.Request(), fi.Name(), fi.ModTime(), f)
		})
	}
}

// Favicon serves the default favicon - GET /favicon.ico.
func Favicon() echo.HandlerFunc {
	return func(c echo.Context) error {
		return nil
	}
}
<|endoftext|>


<commit_before>package models

// BuildValidate describes a request to the build validate service.
type BuildValidate struct {
	Mapping  *Mapping `json:"mapping"`
	Policies []Policy `json:"policies"`
}

// BuildValidateReponse describes a response from the build validate service.
type BuildValidateResponse struct {
	Version    string     `json:"version"`
	Controls   []Control  `json:"controls"`
	Statistics Statistics `json:"statistics"`
}

// Control describes an individual test within a build validation.
type Control struct {
	ID        string `json:"id"`
	ProfileID string `json:"profile_id"`
	Status    string `json:"status"`
	CodeDesc  string `json:"code_desc"`
	Message   string `json:"message"`
}

// Statistics describes stats for the build validate service.
type Statistics struct {
	Duration float64 `json:"duration"`
}

// Passed : returns true if validation rules passed
func (b *BuildValidateResponse) Passed() bool {
	for _, e := range b.Controls {
		if e.Status == "failed" {
			return false
		}
	}

	return true
}
<commit_msg>updated model to include more detail on compliance output<commit_after>package models

// BuildValidate describes a request to the build validate service.
type BuildValidate struct {
	Mapping  *Mapping `json:"mapping"`
	Policies []Policy `json:"policies"`
}

// BuildValidateReponse describes a response from the build validate service.
type BuildValidateResponse struct {
	Version    string     `json:"version"`
	Controls   []Control  `json:"controls"`
	Profiles   []Profile  `json:"profiles"`
	Statistics Statistics `json:"statistics"`
}

// Profile describes the policy document and its test results
type Profile struct {
	Supports []string         `json:"supports"`
	Title    string           `json:"title"`
	Name     string           `json:"name"`
	Controls []ControlDetails `json:"controls"`
}

// ControlDetails describes additional information about a control
type ControlDetails struct {
	ID             string    `json:"id"`
	Title          string    `json:"title"`
	Description    string    `json:"desc"`
	Impact         string    `json:"impact"`
	References     []string  `json:"refs"`
	Tags           []string  `json:"tags"`
	Code           string    `json:"code"`
	Results        []Control `json:"results"`
	Groups         []Group   `json:"groups"`
	Attributes     []string  `json:"attribues"`
	SHA256         string    `json:"sha256"`
	SourceLocation struct {
		Reference string `json:"ref"`
		Line      string `json:"line"`
	} `json:"source_location"`
}

// Control describes an individual test within a build validation.
type Control struct {
	ID        string  `json:"id"`
	ProfileID string  `json:"profile_id"`
	Status    string  `json:"status"`
	CodeDesc  string  `json:"code_desc"`
	RunTime   string  `json:"run_time"`
	StartTime float64 `json:"start_time"`
	Message   string  `json:"message"`
}

type Group struct {
	ID       string   `json:"id"`
	Title    string   `json:"title"`
	Controls []string `json:"controls"`
}

// Statistics describes stats for the build validate service.
type Statistics struct {
	Duration float64 `json:"duration"`
}

// Passed : returns true if validation rules passed
func (b *BuildValidateResponse) Passed() bool {
	for _, e := range b.Controls {
		if e.Status == "failed" {
			return false
		}
	}

	return true
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sdb

import (
	"github.com/jacobsa/aws/exp/sdb/conn"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"sort"
	"testing"
)

func TestPut(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func getSortedKeys(r conn.Request) []string {
	result := sort.StringSlice{}
	for key, _ := range r {
		result = append(result, key)
	}

	sort.Sort(result)
	return result
}

////////////////////////////////////////////////////////////////////////
// PutAttributes
////////////////////////////////////////////////////////////////////////

type PutTest struct {
	domainTest

	item ItemName
	updates []PutUpdate
	preconditions []Precondition

	err error
}

func init() { RegisterTestSuite(&PutTest{}) }

func (t *PutTest) SetUp(i *TestInfo) {
	// Call common setup code.
	t.domainTest.SetUp(i)

	// Make the request legal by default.
	t.item = "foo"
	t.updates = []PutUpdate{PutUpdate{"bar", "baz", false}}
}

func (t *PutTest) callDomain() {
	t.err = t.domain.PutAttributes(t.item, t.updates, t.preconditions)
}

func (t *PutTest) EmptyItemName() {
	t.item = ""

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("item name")))
}

func (t *PutTest) InvalidItemName() {
	t.item = "taco\x80\x81\x82"

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("item name")))
	ExpectThat(t.err, Error(HasSubstr(string(t.item))))
}

func (t *PutTest) ZeroUpdates() {
	t.updates = []PutUpdate{}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("number")))
	ExpectThat(t.err, Error(HasSubstr("updates")))
	ExpectThat(t.err, Error(HasSubstr("0")))
}

func (t *PutTest) TooManyUpdates() {
	t.updates = make([]PutUpdate, 257)

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("number")))
	ExpectThat(t.err, Error(HasSubstr("updates")))
	ExpectThat(t.err, Error(HasSubstr("256")))
}

func (t *PutTest) OneAttributeNameEmpty() {
	t.updates = []PutUpdate{
		PutUpdate{Name: "foo"},
		PutUpdate{Name: "", Value: "taco"},
		PutUpdate{Name: "bar"},
	}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("attribute")))
	ExpectThat(t.err, Error(HasSubstr("name")))
	ExpectThat(t.err, Error(HasSubstr("taco")))
}

func (t *PutTest) OneAttributeNameInvalid() {
	t.updates = []PutUpdate{
		PutUpdate{Name: "foo"},
		PutUpdate{Name: "taco\x80\x81\x82"},
		PutUpdate{Name: "bar"},
	}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("attribute")))
	ExpectThat(t.err, Error(HasSubstr("name")))
	ExpectThat(t.err, Error(HasSubstr(t.updates[1].Name)))
}

func (t *PutTest) OneAttributeValueInvalid() {
	t.updates = []PutUpdate{
		PutUpdate{Name: "foo"},
		PutUpdate{Name: "bar", Value: "taco\x80\x81\x82"},
		PutUpdate{Name: "baz"},
	}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("attribute")))
	ExpectThat(t.err, Error(HasSubstr("value")))
	ExpectThat(t.err, Error(HasSubstr(t.updates[1].Value)))
}

func (t *PutTest) OnePreconditionNameEmpty() {
	t.preconditions = []Precondition{
		Precondition{Name: "foo", Exists: new(bool)},
		Precondition{Name: "", Exists: new(bool)},
		Precondition{Name: "baz", Exists: new(bool)},
	}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("attribute")))
	ExpectThat(t.err, Error(HasSubstr("name")))
}

func (t *PutTest) OnePreconditionNameInvalid() {
	t.preconditions = []Precondition{
		Precondition{Name: "foo", Exists: new(bool)},
		Precondition{Name: "taco\x80\x81\x82", Exists: new(bool)},
		Precondition{Name: "baz", Exists: new(bool)},
	}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("attribute")))
	ExpectThat(t.err, Error(HasSubstr("name")))
	ExpectThat(t.err, Error(HasSubstr(t.preconditions[1].Name)))
}

func (t *PutTest) OnePreconditionValueInvalid() {
	t.preconditions = []Precondition{
		Precondition{Name: "foo", Value: new(string)},
		Precondition{Name: "bar", Value: new(string)},
		Precondition{Name: "baz", Value: new(string)},
	}

	*t.preconditions[0].Value = ""
	*t.preconditions[1].Value = "taco\x80\x81\x82"
	*t.preconditions[2].Value = "qux"

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("attribute")))
	ExpectThat(t.err, Error(HasSubstr("value")))
	ExpectThat(t.err, Error(HasSubstr(*t.preconditions[1].Value)))
}

func (t *PutTest) OnePreconditionMissingOperand() {
	t.preconditions = []Precondition{
		Precondition{Name: "foo", Exists: new(bool)},
		Precondition{Name: "bar"},
		Precondition{Name: "baz", Exists: new(bool)},
	}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("precondition")))
	ExpectThat(t.err, Error(HasSubstr("bar")))
}

func (t *PutTest) OnePreconditionHasTwoOperands() {
	t.preconditions = []Precondition{
		Precondition{Name: "foo", Exists: new(bool)},
		Precondition{Name: "bar", Exists: new(bool), Value: new(string)},
		Precondition{Name: "baz", Exists: new(bool)},
	}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("precondition")))
	ExpectThat(t.err, Error(HasSubstr("bar")))
}

func (t *PutTest) BasicParameters() {
	t.item = "some_item"
	t.updates = []PutUpdate{
		PutUpdate{Name: "foo"},
		PutUpdate{Name: "bar", Value: "taco", Replace: true},
		PutUpdate{Name: "baz", Value: "burrito"},
	}

	// Call
	t.callDomain()
	AssertNe(nil, t.c.req)

	AssertThat(
		getSortedKeys(t.c.req),
		ElementsAre(
			"Attribute.1.Name",
			"Attribute.1.Value",
			"Attribute.2.Name",
			"Attribute.2.Replace",
			"Attribute.2.Value",
			"Attribute.3.Name",
			"Attribute.3.Value",
			"DomainName",
			"ItemName",
		),
	)

	ExpectEq("foo", t.c.req["Attribute.1.Name"])
	ExpectEq("bar", t.c.req["Attribute.2.Name"])
	ExpectEq("baz", t.c.req["Attribute.3.Name"])

	ExpectEq("", t.c.req["Attribute.1.Value"])
	ExpectEq("taco", t.c.req["Attribute.2.Value"])
	ExpectEq("burrito", t.c.req["Attribute.3.Value"])

	ExpectEq("true", t.c.req["Attribute.2.Replace"])

	ExpectEq("some_item", t.c.req["ItemName"])
	ExpectEq(t.name, t.c.req["DomainName"])
}

func (t *PutTest) NoPreconditions() {
	// Call
	t.callDomain()
	AssertNe(nil, t.c.req)

	ExpectThat(getSortedKeys(t.c.req), Not(Contains(HasSubstr("Expected"))))
}

func (t *PutTest) SomePreconditions() {
	ExpectEq("TODO", "")
}

func (t *PutTest) ConnReturnsError() {
	ExpectEq("TODO", "")
}

func (t *PutTest) ConnSaysOkay() {
	ExpectEq("TODO", "")
}

////////////////////////////////////////////////////////////////////////
// BatchPutAttributes
////////////////////////////////////////////////////////////////////////

type BatchPutTest struct {
	domainTest
}

func init() { RegisterTestSuite(&BatchPutTest{}) }

func (t *BatchPutTest) DoesFoo() {
	ExpectEq("TODO", "")
}
<commit_msg>PutTest.SomePreconditions<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sdb

import (
	"github.com/jacobsa/aws/exp/sdb/conn"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"sort"
	"testing"
)

func TestPut(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func getSortedKeys(r conn.Request) []string {
	result := sort.StringSlice{}
	for key, _ := range r {
		result = append(result, key)
	}

	sort.Sort(result)
	return result
}

////////////////////////////////////////////////////////////////////////
// PutAttributes
////////////////////////////////////////////////////////////////////////

type PutTest struct {
	domainTest

	item ItemName
	updates []PutUpdate
	preconditions []Precondition

	err error
}

func init() { RegisterTestSuite(&PutTest{}) }

func (t *PutTest) SetUp(i *TestInfo) {
	// Call common setup code.
	t.domainTest.SetUp(i)

	// Make the request legal by default.
	t.item = "foo"
	t.updates = []PutUpdate{PutUpdate{"bar", "baz", false}}
}

func (t *PutTest) callDomain() {
	t.err = t.domain.PutAttributes(t.item, t.updates, t.preconditions)
}

func (t *PutTest) EmptyItemName() {
	t.item = ""

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("item name")))
}

func (t *PutTest) InvalidItemName() {
	t.item = "taco\x80\x81\x82"

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("item name")))
	ExpectThat(t.err, Error(HasSubstr(string(t.item))))
}

func (t *PutTest) ZeroUpdates() {
	t.updates = []PutUpdate{}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("number")))
	ExpectThat(t.err, Error(HasSubstr("updates")))
	ExpectThat(t.err, Error(HasSubstr("0")))
}

func (t *PutTest) TooManyUpdates() {
	t.updates = make([]PutUpdate, 257)

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("number")))
	ExpectThat(t.err, Error(HasSubstr("updates")))
	ExpectThat(t.err, Error(HasSubstr("256")))
}

func (t *PutTest) OneAttributeNameEmpty() {
	t.updates = []PutUpdate{
		PutUpdate{Name: "foo"},
		PutUpdate{Name: "", Value: "taco"},
		PutUpdate{Name: "bar"},
	}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("attribute")))
	ExpectThat(t.err, Error(HasSubstr("name")))
	ExpectThat(t.err, Error(HasSubstr("taco")))
}

func (t *PutTest) OneAttributeNameInvalid() {
	t.updates = []PutUpdate{
		PutUpdate{Name: "foo"},
		PutUpdate{Name: "taco\x80\x81\x82"},
		PutUpdate{Name: "bar"},
	}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("attribute")))
	ExpectThat(t.err, Error(HasSubstr("name")))
	ExpectThat(t.err, Error(HasSubstr(t.updates[1].Name)))
}

func (t *PutTest) OneAttributeValueInvalid() {
	t.updates = []PutUpdate{
		PutUpdate{Name: "foo"},
		PutUpdate{Name: "bar", Value: "taco\x80\x81\x82"},
		PutUpdate{Name: "baz"},
	}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("attribute")))
	ExpectThat(t.err, Error(HasSubstr("value")))
	ExpectThat(t.err, Error(HasSubstr(t.updates[1].Value)))
}

func (t *PutTest) OnePreconditionNameEmpty() {
	t.preconditions = []Precondition{
		Precondition{Name: "foo", Exists: new(bool)},
		Precondition{Name: "", Exists: new(bool)},
		Precondition{Name: "baz", Exists: new(bool)},
	}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("attribute")))
	ExpectThat(t.err, Error(HasSubstr("name")))
}

func (t *PutTest) OnePreconditionNameInvalid() {
	t.preconditions = []Precondition{
		Precondition{Name: "foo", Exists: new(bool)},
		Precondition{Name: "taco\x80\x81\x82", Exists: new(bool)},
		Precondition{Name: "baz", Exists: new(bool)},
	}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("attribute")))
	ExpectThat(t.err, Error(HasSubstr("name")))
	ExpectThat(t.err, Error(HasSubstr(t.preconditions[1].Name)))
}

func (t *PutTest) OnePreconditionValueInvalid() {
	t.preconditions = []Precondition{
		Precondition{Name: "foo", Value: new(string)},
		Precondition{Name: "bar", Value: new(string)},
		Precondition{Name: "baz", Value: new(string)},
	}

	*t.preconditions[0].Value = ""
	*t.preconditions[1].Value = "taco\x80\x81\x82"
	*t.preconditions[2].Value = "qux"

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("attribute")))
	ExpectThat(t.err, Error(HasSubstr("value")))
	ExpectThat(t.err, Error(HasSubstr(*t.preconditions[1].Value)))
}

func (t *PutTest) OnePreconditionMissingOperand() {
	t.preconditions = []Precondition{
		Precondition{Name: "foo", Exists: new(bool)},
		Precondition{Name: "bar"},
		Precondition{Name: "baz", Exists: new(bool)},
	}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("precondition")))
	ExpectThat(t.err, Error(HasSubstr("bar")))
}

func (t *PutTest) OnePreconditionHasTwoOperands() {
	t.preconditions = []Precondition{
		Precondition{Name: "foo", Exists: new(bool)},
		Precondition{Name: "bar", Exists: new(bool), Value: new(string)},
		Precondition{Name: "baz", Exists: new(bool)},
	}

	// Call
	t.callDomain()

	ExpectThat(t.err, Error(HasSubstr("Invalid")))
	ExpectThat(t.err, Error(HasSubstr("precondition")))
	ExpectThat(t.err, Error(HasSubstr("bar")))
}

func (t *PutTest) BasicParameters() {
	t.item = "some_item"
	t.updates = []PutUpdate{
		PutUpdate{Name: "foo"},
		PutUpdate{Name: "bar", Value: "taco", Replace: true},
		PutUpdate{Name: "baz", Value: "burrito"},
	}

	// Call
	t.callDomain()
	AssertNe(nil, t.c.req)

	AssertThat(
		getSortedKeys(t.c.req),
		ElementsAre(
			"Attribute.1.Name",
			"Attribute.1.Value",
			"Attribute.2.Name",
			"Attribute.2.Replace",
			"Attribute.2.Value",
			"Attribute.3.Name",
			"Attribute.3.Value",
			"DomainName",
			"ItemName",
		),
	)

	ExpectEq("foo", t.c.req["Attribute.1.Name"])
	ExpectEq("bar", t.c.req["Attribute.2.Name"])
	ExpectEq("baz", t.c.req["Attribute.3.Name"])

	ExpectEq("", t.c.req["Attribute.1.Value"])
	ExpectEq("taco", t.c.req["Attribute.2.Value"])
	ExpectEq("burrito", t.c.req["Attribute.3.Value"])

	ExpectEq("true", t.c.req["Attribute.2.Replace"])

	ExpectEq("some_item", t.c.req["ItemName"])
	ExpectEq(t.name, t.c.req["DomainName"])
}

func (t *PutTest) NoPreconditions() {
	// Call
	t.callDomain()
	AssertNe(nil, t.c.req)

	ExpectThat(getSortedKeys(t.c.req), Not(Contains(HasSubstr("Expected"))))
}

func (t *PutTest) SomePreconditions() {
	t.preconditions = []Precondition{
		Precondition{Name: "foo", Exists: new(bool)},
		Precondition{Name: "bar", Value: new(string)},
		Precondition{Name: "baz", Exists: new(bool)},
	}

	*t.preconditions[0].Exists = false
	*t.preconditions[1].Value = "taco"
	*t.preconditions[2].Exists = true

	// Call
	t.callDomain()
	AssertNe(nil, t.c.req)

	AssertThat(
		getSortedKeys(t.c.req),
		AllOf(
			Contains("Expected.1.Name"),
			Contains("Expected.2.Name"),
			Contains("Expected.3.Name"),
			Contains("Expected.1.Exists"),
			Contains("Expected.2.Value"),
			Contains("Expected.3.Exists"),
		),
	)

	ExpectEq("foo", t.c.req["Expected.1.Name"])
	ExpectEq("bar", t.c.req["Expected.2.Name"])
	ExpectEq("baz", t.c.req["Expected.3.Name"])

	ExpectEq("false", t.c.req["Expected.1.Exists"])
	ExpectEq("taco", t.c.req["Expected.2.Value"])
	ExpectEq("true", t.c.req["Expected.3.Exists"])
}

func (t *PutTest) ConnReturnsError() {
	ExpectEq("TODO", "")
}

func (t *PutTest) ConnSaysOkay() {
	ExpectEq("TODO", "")
}

////////////////////////////////////////////////////////////////////////
// BatchPutAttributes
////////////////////////////////////////////////////////////////////////

type BatchPutTest struct {
	domainTest
}

func init() { RegisterTestSuite(&BatchPutTest{}) }

func (t *BatchPutTest) DoesFoo() {
	ExpectEq("TODO", "")
}
<|endoftext|>


<commit_before>package monitor

import (
	"context"
	"sync"
	"time"
)

// Scheduler is an object which performs availability polling once every interval
// and writes the availability statuses into a channel.
type Scheduler struct {
	// The Poller object which will be used to do polling.
	Poller *Poller
	// Source of lists of targets.
	Targets TargetsGetter
	// Time interval between targets polling.
	Interval time.Duration
	// Maximum number of parallel http requests.
	ParallelPolls uint
	// The channel into which the scheduler will write the polling results.
	Statuses chan TargetStatus
	// The optional channel into which the scheduler should write errors of
	// TargetsGetter. If this field is set to nil, the scheduler will ignore
	// TargetsGetter errors.
	Errors chan error
}

// NewScheduler constructs a new Scheduler with given TargetsGetter and default
// fields values.
func NewScheduler(targets TargetsGetter) *Scheduler {
	return &Scheduler{
		Poller:        NewPoller(),
		Targets:       targets,
		Interval:      5 * time.Second,
		ParallelPolls: 5,
		Statuses:      make(chan TargetStatus, 1),
		Errors:        nil,
	}
}

// Run start infinite looop of targets polling in foreground. Call this method
// in a goroutine to do polling in background.
// If the context argument is not nil, the scheduler will stop the loop when
// it receives a signal from context.Done().
func (s *Scheduler) Run(context context.Context) {
	var done <-chan struct{}
	if context != nil {
		done = context.Done()
	}

	ticker := time.NewTicker(s.Interval)

	for {
		select {
		case <-ticker.C:
			s.PollTargets()
		case <-done:
			return
		}
	}
}

// PollTargets does single cycle of targets polling in foreground, which includes:
// - getting the targets list from s.Targets;
// - polling each target with s.Poller;
// - writing results into s.Statuses channel.
// If the s.Targets returns an error, which method will no perform polling
// and will attempt to send the error to s.Errors channel, if it's not nil.
func (s *Scheduler) PollTargets() {
	targets, err := s.Targets.GetTargets()
	if err != nil {
		if s.Errors != nil {
			s.Errors <- err
		}
		return
	}

	// This chanell is used to limit number of workers working at the same time.
	workersPool := make(chan struct{}, s.ParallelPolls)

	// This WaitGroup is used to wait for all workers to finish their job.
	var workersDone sync.WaitGroup

	for _, target := range targets {
		target := target
		workersDone.Add(1)
		workersPool <- struct{}{}
		go func() {
			status := s.Poller.PollService(target.URL)
			if s.Statuses != nil {
				s.Statuses <- TargetStatus{target, status}
			}
			<-workersPool
			workersDone.Done()
		}()
	}

	workersDone.Wait()
}
<commit_msg>:hammer: Make Scheduler.Errors into a getter method<commit_after>package monitor

import (
	"context"
	"sync"
	"time"
)

// Scheduler is an object which performs availability polling once every interval
// and writes the availability statuses into a channel.
type Scheduler struct {
	// The Poller object which will be used to do polling.
	Poller *Poller
	// Source of lists of targets.
	Targets TargetsGetter
	// Time interval between targets polling.
	Interval time.Duration
	// Maximum number of parallel http requests.
	ParallelPolls uint
	// The channel into which the scheduler will write the polling results.
	Statuses chan TargetStatus

	errors chan error
}

// NewScheduler constructs a new Scheduler with given TargetsGetter and default
// fields values.
func NewScheduler(targets TargetsGetter) *Scheduler {
	return &Scheduler{
		Poller:        NewPoller(),
		Targets:       targets,
		Interval:      5 * time.Second,
		ParallelPolls: 5,
		Statuses:      make(chan TargetStatus, 1),
		errors:        nil,
	}
}

// Errors retruns a channel through which the Scheduler will send all errors,
// appearing in the process.
func (s *Scheduler) Errors() chan error {
	if s.errors == nil {
		s.errors = make(chan error, 10)
	}
	return s.errors
}

// Run start infinite looop of targets polling in foreground. Call this method
// in a goroutine to do polling in background.
// If the context argument is not nil, the scheduler will stop the loop when
// it receives a signal from context.Done().
func (s *Scheduler) Run(context context.Context) {
	var done <-chan struct{}
	if context != nil {
		done = context.Done()
	}

	ticker := time.NewTicker(s.Interval)

	for {
		select {
		case <-ticker.C:
			s.PollTargets()
		case <-done:
			return
		}
	}
}

// PollTargets does single cycle of targets polling in foreground, which includes:
// - getting the targets list from s.Targets;
// - polling each target with s.Poller;
// - writing results into s.Statuses channel.
// If the s.Targets returns an error, which method will no perform polling
// and will attempt to send the error to s.Errors channel, if it's not nil.
func (s *Scheduler) PollTargets() {
	targets, err := s.Targets.GetTargets()
	if err != nil {
		if s.errors != nil {
			s.errors <- err
		}
		return
	}

	// This chanell is used to limit number of workers working at the same time.
	workersPool := make(chan struct{}, s.ParallelPolls)

	// This WaitGroup is used to wait for all workers to finish their job.
	var workersDone sync.WaitGroup

	for _, target := range targets {
		target := target
		workersDone.Add(1)
		workersPool <- struct{}{}
		go func() {
			status := s.Poller.PollService(target.URL)
			if s.Statuses != nil {
				s.Statuses <- TargetStatus{target, status}
			}
			<-workersPool
			workersDone.Done()
		}()
	}

	workersDone.Wait()
}
<|endoftext|>


<commit_before>package movingmedian

import (
	"log"
	"math"
	"math/rand"
	"sort"
	"testing"
)

func TestUnit(t *testing.T) {
	tests := []struct {
		name       string
		windowSize int
		data       []float64
		want       []float64
	}{
		{
			"OneWindowSize",
			1,
			[]float64{1, 3, 5, 7, 9, 11, math.NaN()},
			[]float64{1, 3, 5, 7, 9, 11, math.NaN()},
		},
		{
			"OddWindowSize",
			3,
			[]float64{1, 3, 5, 7, 9, 11},
			[]float64{1, 2, 3, 5, 7, 9},
		},
		{
			"EvenWindowSize",
			4,
			[]float64{1, 3, 5, 7, 9, 11},
			[]float64{1, 2, 3, 4, 6, 8},
		},
		{
			"DecreasingValues",
			4,
			[]float64{19, 17, 15, 13, 11, 9},
			[]float64{19, 18, 17, 16, 14, 12},
		},
		{
			"DecreasingIncreasingValues",
			4,
			[]float64{21, 19, 17, 15, 13, 11, 13, 15, 17, 19},
			[]float64{21, 20, 19, 18, 16, 14, 13, 13, 14, 16},
		},
		{
			"IncreasingDecreasingValues",
			4,
			[]float64{11, 13, 15, 17, 19, 21, 19, 17, 15, 13},
			[]float64{11, 12, 13, 14, 16, 18, 19, 19, 18, 16},
		},
		{

			"ZigZag",
			4,
			[]float64{21, 23, 17, 27, 13, 31, 9, 35, 5, 39, 1},
			[]float64{21, 22, 21, 22, 20, 22, 20, 22, 20, 22, 20},
		},
		{

			"NewValuesInBetween",
			4,
			[]float64{21, 21, 19, 19, 21, 21, 19, 19, 19, 19},
			[]float64{21, 21, 21, 20, 20, 20, 20, 20, 19, 19},
		},
		{
			"SameNumberInBothHeaps3Times",
			4,
			[]float64{11, 13, 13, 13, 25, 27, 29, 31},
			[]float64{11, 12, 13, 13, 13, 19, 26, 28},
		},
		{
			"SameNumberInBothHeaps3TimesDecreasing",
			4,
			[]float64{31, 29, 29, 29, 17, 15, 13, 11},
			[]float64{31, 30, 29, 29, 29, 23, 16, 14},
		},
		{
			"SameNumberInBothHeaps4Times",
			4,
			[]float64{11, 13, 13, 13, 13, 25, 27, 29, 31},
			[]float64{11, 12, 13, 13, 13, 13, 19, 26, 28},
		},
	}

	for _, test := range tests {
		log.Println("test name", test.name)
		m := NewMovingMedian(test.windowSize)
		for i, v := range test.data {
			m.Push(v)
			actual := m.Median()
			if test.want[i] != actual && !(math.IsNaN(actual) && math.IsNaN(test.want[i])) {
				firstElement := 1 + i - test.windowSize
				if firstElement < 0 {
					firstElement = 0
				}
				t.Errorf("failed on test %s index %d the median of %f is %f and not %f",
					test.name,
					i,
					test.data[firstElement:1+i],
					test.want[i],
					actual)
			}
		}
	}
}

func TestRandom(t *testing.T) {
	rangeSize := 100
	for windowSize := 1; windowSize < 5; windowSize++ {
		data := getData(rangeSize, windowSize)
		intData := make([]int, rangeSize)
		for i, v := range data {
			intData[i] = int(v)
		}

		log.Println("test name random test window size", windowSize)
		m := NewMovingMedian(windowSize)
		for i, v := range data {
			want := median(data, i, windowSize)

			m.Push(v)
			actual := m.Median()
			if want != actual {
				firstElement := 1 + i - windowSize
				if firstElement < 0 {
					firstElement = 0
				}

				t.Errorf("failed on test random window size %d index %d the median of %d is %f and not %f",
					windowSize,
					i,
					intData[firstElement:1+i],
					want,
					actual)
			}
		}
	}
}

func Benchmark_10values_windowsize1(b *testing.B) {
	benchmark(b, 10, 1)
}

func Benchmark_100values_windowsize10(b *testing.B) {
	benchmark(b, 100, 10)
}

func Benchmark_10Kvalues_windowsize100(b *testing.B) {
	benchmark(b, 10000, 100)
}

func Benchmark_10Kvalues_windowsize1000(b *testing.B) {
	benchmark(b, 10000, 1000)
}

func benchmark(b *testing.B, numberOfValues, windowSize int) {
	data := getData(numberOfValues, windowSize)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		m := NewMovingMedian(windowSize)
		for _, v := range data {
			m.Push(v)
			m.Median()
		}
	}
}

func getData(rangeSize, windowSize int) []float64 {
	var data = make([]float64, rangeSize)
	var r = rand.New(rand.NewSource(99))
	for i, _ := range data {
		data[i] = math.Floor(1000 * r.Float64())
	}

	return data
}

func median(data []float64, i, windowSize int) float64 {
	min := 1 + i - windowSize
	if min < 0 {
		min = 0
	}

	if len(data) == 0 {
		return math.NaN()
	}

	window := make([]float64, 1+i-min)
	copy(window, data[min:i+1])

	if len(window) == 1 {
		return window[0]
	}

	sort.Float64s(window)

	k := len(window) / 2
	if len(window)%2 == 1 {
		return window[k]
	}

	return 0.5*window[k-1] + 0.5*window[k]
}
<commit_msg>expand range of randomize tests<commit_after>package movingmedian

import (
	"log"
	"math"
	"math/rand"
	"sort"
	"testing"
)

func TestUnit(t *testing.T) {
	tests := []struct {
		name       string
		windowSize int
		data       []float64
		want       []float64
	}{
		{
			"OneWindowSize",
			1,
			[]float64{1, 3, 5, 7, 9, 11, math.NaN()},
			[]float64{1, 3, 5, 7, 9, 11, math.NaN()},
		},
		{
			"OddWindowSize",
			3,
			[]float64{1, 3, 5, 7, 9, 11},
			[]float64{1, 2, 3, 5, 7, 9},
		},
		{
			"EvenWindowSize",
			4,
			[]float64{1, 3, 5, 7, 9, 11},
			[]float64{1, 2, 3, 4, 6, 8},
		},
		{
			"DecreasingValues",
			4,
			[]float64{19, 17, 15, 13, 11, 9},
			[]float64{19, 18, 17, 16, 14, 12},
		},
		{
			"DecreasingIncreasingValues",
			4,
			[]float64{21, 19, 17, 15, 13, 11, 13, 15, 17, 19},
			[]float64{21, 20, 19, 18, 16, 14, 13, 13, 14, 16},
		},
		{
			"IncreasingDecreasingValues",
			4,
			[]float64{11, 13, 15, 17, 19, 21, 19, 17, 15, 13},
			[]float64{11, 12, 13, 14, 16, 18, 19, 19, 18, 16},
		},
		{

			"ZigZag",
			4,
			[]float64{21, 23, 17, 27, 13, 31, 9, 35, 5, 39, 1},
			[]float64{21, 22, 21, 22, 20, 22, 20, 22, 20, 22, 20},
		},
		{

			"NewValuesInBetween",
			4,
			[]float64{21, 21, 19, 19, 21, 21, 19, 19, 19, 19},
			[]float64{21, 21, 21, 20, 20, 20, 20, 20, 19, 19},
		},
		{
			"SameNumberInBothHeaps3Times",
			4,
			[]float64{11, 13, 13, 13, 25, 27, 29, 31},
			[]float64{11, 12, 13, 13, 13, 19, 26, 28},
		},
		{
			"SameNumberInBothHeaps3TimesDecreasing",
			4,
			[]float64{31, 29, 29, 29, 17, 15, 13, 11},
			[]float64{31, 30, 29, 29, 29, 23, 16, 14},
		},
		{
			"SameNumberInBothHeaps4Times",
			4,
			[]float64{11, 13, 13, 13, 13, 25, 27, 29, 31},
			[]float64{11, 12, 13, 13, 13, 13, 19, 26, 28},
		},
	}

	for _, test := range tests {
		log.Println("test name", test.name)
		m := NewMovingMedian(test.windowSize)
		for i, v := range test.data {
			m.Push(v)
			actual := m.Median()
			if test.want[i] != actual && !(math.IsNaN(actual) && math.IsNaN(test.want[i])) {
				firstElement := 1 + i - test.windowSize
				if firstElement < 0 {
					firstElement = 0
				}
				t.Errorf("failed on test %s index %d the median of %f is %f and not %f",
					test.name,
					i,
					test.data[firstElement:1+i],
					test.want[i],
					actual)
			}
		}
	}
}

func TestRandom(t *testing.T) {
	rangeSize := 10000
	for windowSize := 1; windowSize < 100; windowSize++ {
		data := getData(rangeSize, windowSize)
		intData := make([]int, rangeSize)
		for i, v := range data {
			intData[i] = int(v)
		}

		log.Println("test name random test window size", windowSize)
		m := NewMovingMedian(windowSize)
		for i, v := range data {
			want := median(data, i, windowSize)

			m.Push(v)
			actual := m.Median()
			if want != actual {
				firstElement := 1 + i - windowSize
				if firstElement < 0 {
					firstElement = 0
				}

				t.Errorf("failed on test random window size %d index %d the median of %d is %f and not %f",
					windowSize,
					i,
					intData[firstElement:1+i],
					want,
					actual)
			}
		}
	}
}

func Benchmark_10values_windowsize1(b *testing.B) {
	benchmark(b, 10, 1)
}

func Benchmark_100values_windowsize10(b *testing.B) {
	benchmark(b, 100, 10)
}

func Benchmark_10Kvalues_windowsize100(b *testing.B) {
	benchmark(b, 10000, 100)
}

func Benchmark_10Kvalues_windowsize1000(b *testing.B) {
	benchmark(b, 10000, 1000)
}

func benchmark(b *testing.B, numberOfValues, windowSize int) {
	data := getData(numberOfValues, windowSize)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		m := NewMovingMedian(windowSize)
		for _, v := range data {
			m.Push(v)
			m.Median()
		}
	}
}

func getData(rangeSize, windowSize int) []float64 {
	var data = make([]float64, rangeSize)
	var r = rand.New(rand.NewSource(99))
	for i, _ := range data {
		data[i] = math.Floor(1000 * r.Float64())
	}

	return data
}

func median(data []float64, i, windowSize int) float64 {
	min := 1 + i - windowSize
	if min < 0 {
		min = 0
	}

	if len(data) == 0 {
		return math.NaN()
	}

	window := make([]float64, 1+i-min)
	copy(window, data[min:i+1])

	if len(window) == 1 {
		return window[0]
	}

	sort.Float64s(window)

	k := len(window) / 2
	if len(window)%2 == 1 {
		return window[k]
	}

	return 0.5*window[k-1] + 0.5*window[k]
}
<|endoftext|>