<commit_before>// Package postgresql implements a micro Store backed by sql
package postgresql

import (
	"database/sql"
	"fmt"
	"time"
	"unicode"

	"github.com/lib/pq"
	"github.com/pkg/errors"

	"github.com/micro/go-micro/config/options"
	"github.com/micro/go-micro/store"
)

// DefaultNamespace is the namespace that the sql store
// will use if no namespace is provided.
const DefaultNamespace = "micro"

type sqlStore struct {
	db *sql.DB

	table string
	options.Options
}

// List all the known records
func (s *sqlStore) List() ([]*store.Record, error) {
	q, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM micro.%s;", s.table))
	if err != nil {
		return nil, err
	}
	var records []*store.Record
	var timehelper pq.NullTime
	rows, err := q.Query()
	if err != nil {
		if err == sql.ErrNoRows {
			return records, nil
		}
		return nil, err
	}
	defer rows.Close()
	for rows.Next() {
		record := &store.Record{}
		if err := rows.Scan(&record.Key, &record.Value, &timehelper); err != nil {
			return records, err
		}
		if timehelper.Valid {
			if timehelper.Time.Before(time.Now()) {
				// record has expired
				go s.Delete(record.Key)
			} else {
				record.Expiry = time.Until(timehelper.Time)
				records = append(records, record)
			}
		} else {
			records = append(records, record)
		}

	}
	rowErr := rows.Close()
	if rowErr != nil {
		// transaction rollback or something
		return records, rowErr
	}
	if err := rows.Err(); err != nil {
		return records, err
	}
	return records, nil
}

// Read all records with keys
func (s *sqlStore) Read(keys ...string) ([]*store.Record, error) {
	q, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM micro.%s WHERE key = $1;", s.table))
	if err != nil {
		return nil, err
	}
	var records []*store.Record
	var timehelper pq.NullTime
	for _, key := range keys {
		row := q.QueryRow(key)
		record := &store.Record{}
		if err := row.Scan(&record.Key, &record.Value, &timehelper); err != nil {
			if err == sql.ErrNoRows {
				return records, store.ErrNotFound
			}
			return records, err
		}
		if timehelper.Valid {
			if timehelper.Time.Before(time.Now()) {
				// record has expired
				go s.Delete(key)
				return records, store.ErrNotFound
			}
			record.Expiry = time.Until(timehelper.Time)
			records = append(records, record)
		} else {
			records = append(records, record)
		}
	}
	return records, nil
}

// Write records
func (s *sqlStore) Write(rec ...*store.Record) error {
	q, err := s.db.Prepare(fmt.Sprintf(`INSERT INTO micro.%s(key, value, expiry)
		VALUES ($1, $2::bytea, $3)
		ON CONFLICT (key)
		DO UPDATE
		SET value = EXCLUDED.value, expiry = EXCLUDED.expiry;`, s.table))
	if err != nil {
		return err
	}
	for _, r := range rec {
		var err error
		if r.Expiry != 0 {
			_, err = q.Exec(r.Key, r.Value, time.Now().Add(r.Expiry))
		} else {
			_, err = q.Exec(r.Key, r.Value, nil)
		}
		if err != nil {
			return errors.Wrap(err, "Couldn't insert record "+r.Key)
		}
	}

	return nil
}

// Delete records with keys
func (s *sqlStore) Delete(keys ...string) error {
	q, err := s.db.Prepare(fmt.Sprintf("DELETE FROM micro.%s WHERE key = $1;", s.table))
	if err != nil {
		return err
	}
	for _, key := range keys {
		result, err := q.Exec(key)
		if err != nil {
			return err
		}
		_, err = result.RowsAffected()
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *sqlStore) initDB(options options.Options) error {
	// Get the store.namespace option, or use sql.DefaultNamespace
	namespaceOpt, found := options.Values().Get("store.namespace")
	if !found {
		s.table = DefaultNamespace
	} else {
		if namespace, ok := namespaceOpt.(string); ok {
			s.table = namespace
		} else {
			return errors.New("store.namespace option must be a string")
		}
	}

	// Create "micro" schema
	schema, err := s.db.Prepare("CREATE SCHEMA IF NOT EXISTS micro ;")
	if err != nil {
		return err
	}
	_, err = schema.Exec()
	if err != nil {
		return errors.Wrap(err, "Couldn't create Schema")
	}

	// Create a table for the Store namespace
	tableq, err := s.db.Prepare(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS micro.%s
	(
		key text COLLATE "default" NOT NULL,
		value bytea,
		expiry timestamp with time zone,
		CONSTRAINT %s_pkey PRIMARY KEY (key)
	);`, s.table, s.table))
	_, err = tableq.Exec()
	if err != nil {
		return errors.Wrap(err, "Couldn't create table")
	}

	return nil
}

// New returns a new micro Store backed by sql
func New(opts ...options.Option) (store.Store, error) {
	options := options.NewOptions(opts...)
	driver, dataSourceName, err := validateOptions(options)
	if err != nil {
		return nil, err
	}
	db, err := sql.Open(driver, dataSourceName)
	if err != nil {
		return nil, err
	}
	if err := db.Ping(); err != nil {
		return nil, err
	}
	s := &sqlStore{
		db: db,
	}

	return s, s.initDB(options)
}

// validateOptions checks whether the provided options are valid, then returns the driver
// and data source name.
func validateOptions(options options.Options) (driver, dataSourceName string, err error) {
	driverOpt, found := options.Values().Get("store.sql.driver")
	if !found {
		return "", "", errors.New("No store.sql.driver option specified")
	}
	nodesOpt, found := options.Values().Get("store.nodes")
	if !found {
		return "", "", errors.New("No store.nodes option specified (expected a database connection string)")
	}
	driver, ok := driverOpt.(string)
	if !ok {
		return "", "", errors.New("store.sql.driver option must be a string")
	}
	nodes, ok := nodesOpt.([]string)
	if !ok {
		return "", "", errors.New("store.nodes option must be a []string")
	}
	if len(nodes) != 1 {
		return "", "", errors.New("expected only 1 store.nodes option")
	}
	namespaceOpt, found := options.Values().Get("store.namespace")
	if found {
		namespace, ok := namespaceOpt.(string)
		if !ok {
			return "", "", errors.New("store.namespace must me a string")
		}
		for _, r := range namespace {
			if !unicode.IsLetter(r) {
				return "", "", errors.New("store.namespace must only contain letters")
			}
		}
	}
	return driver, nodes[0], nil
}
<commit_msg>Let people connect with just a hostname<commit_after>// Package postgresql implements a micro Store backed by sql
package postgresql

import (
	"database/sql"
	"fmt"
	"strings"
	"time"
	"unicode"

	"github.com/lib/pq"
	"github.com/pkg/errors"

	"github.com/micro/go-micro/config/options"
	"github.com/micro/go-micro/store"
)

// DefaultNamespace is the namespace that the sql store
// will use if no namespace is provided.
const DefaultNamespace = "micro"

type sqlStore struct {
	db *sql.DB

	table string
	options.Options
}

// List all the known records
func (s *sqlStore) List() ([]*store.Record, error) {
	q, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM micro.%s;", s.table))
	if err != nil {
		return nil, err
	}
	var records []*store.Record
	var timehelper pq.NullTime
	rows, err := q.Query()
	if err != nil {
		if err == sql.ErrNoRows {
			return records, nil
		}
		return nil, err
	}
	defer rows.Close()
	for rows.Next() {
		record := &store.Record{}
		if err := rows.Scan(&record.Key, &record.Value, &timehelper); err != nil {
			return records, err
		}
		if timehelper.Valid {
			if timehelper.Time.Before(time.Now()) {
				// record has expired
				go s.Delete(record.Key)
			} else {
				record.Expiry = time.Until(timehelper.Time)
				records = append(records, record)
			}
		} else {
			records = append(records, record)
		}

	}
	rowErr := rows.Close()
	if rowErr != nil {
		// transaction rollback or something
		return records, rowErr
	}
	if err := rows.Err(); err != nil {
		return records, err
	}
	return records, nil
}

// Read all records with keys
func (s *sqlStore) Read(keys ...string) ([]*store.Record, error) {
	q, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM micro.%s WHERE key = $1;", s.table))
	if err != nil {
		return nil, err
	}
	var records []*store.Record
	var timehelper pq.NullTime
	for _, key := range keys {
		row := q.QueryRow(key)
		record := &store.Record{}
		if err := row.Scan(&record.Key, &record.Value, &timehelper); err != nil {
			if err == sql.ErrNoRows {
				return records, store.ErrNotFound
			}
			return records, err
		}
		if timehelper.Valid {
			if timehelper.Time.Before(time.Now()) {
				// record has expired
				go s.Delete(key)
				return records, store.ErrNotFound
			}
			record.Expiry = time.Until(timehelper.Time)
			records = append(records, record)
		} else {
			records = append(records, record)
		}
	}
	return records, nil
}

// Write records
func (s *sqlStore) Write(rec ...*store.Record) error {
	q, err := s.db.Prepare(fmt.Sprintf(`INSERT INTO micro.%s(key, value, expiry)
		VALUES ($1, $2::bytea, $3)
		ON CONFLICT (key)
		DO UPDATE
		SET value = EXCLUDED.value, expiry = EXCLUDED.expiry;`, s.table))
	if err != nil {
		return err
	}
	for _, r := range rec {
		var err error
		if r.Expiry != 0 {
			_, err = q.Exec(r.Key, r.Value, time.Now().Add(r.Expiry))
		} else {
			_, err = q.Exec(r.Key, r.Value, nil)
		}
		if err != nil {
			return errors.Wrap(err, "Couldn't insert record "+r.Key)
		}
	}

	return nil
}

// Delete records with keys
func (s *sqlStore) Delete(keys ...string) error {
	q, err := s.db.Prepare(fmt.Sprintf("DELETE FROM micro.%s WHERE key = $1;", s.table))
	if err != nil {
		return err
	}
	for _, key := range keys {
		result, err := q.Exec(key)
		if err != nil {
			return err
		}
		_, err = result.RowsAffected()
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *sqlStore) initDB(options options.Options) error {
	// Get the store.namespace option, or use sql.DefaultNamespace
	namespaceOpt, found := options.Values().Get("store.namespace")
	if !found {
		s.table = DefaultNamespace
	} else {
		if namespace, ok := namespaceOpt.(string); ok {
			s.table = namespace
		} else {
			return errors.New("store.namespace option must be a string")
		}
	}

	// Create "micro" schema
	schema, err := s.db.Prepare("CREATE SCHEMA IF NOT EXISTS micro ;")
	if err != nil {
		return err
	}
	_, err = schema.Exec()
	if err != nil {
		return errors.Wrap(err, "Couldn't create Schema")
	}

	// Create a table for the Store namespace
	tableq, err := s.db.Prepare(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS micro.%s
	(
		key text COLLATE "default" NOT NULL,
		value bytea,
		expiry timestamp with time zone,
		CONSTRAINT %s_pkey PRIMARY KEY (key)
	);`, s.table, s.table))
	_, err = tableq.Exec()
	if err != nil {
		return errors.Wrap(err, "Couldn't create table")
	}

	return nil
}

// New returns a new micro Store backed by sql
func New(opts ...options.Option) (store.Store, error) {
	options := options.NewOptions(opts...)
	driver, dataSourceName, err := validateOptions(options)
	if err != nil {
		return nil, err
	}
	if !strings.Contains(dataSourceName, " ") {
		dataSourceName = fmt.Sprintf("host=%s", dataSourceName)
	}
	db, err := sql.Open(driver, dataSourceName)
	if err != nil {
		return nil, err
	}
	if err := db.Ping(); err != nil {
		return nil, err
	}
	s := &sqlStore{
		db: db,
	}

	return s, s.initDB(options)
}

// validateOptions checks whether the provided options are valid, then returns the driver
// and data source name.
func validateOptions(options options.Options) (driver, dataSourceName string, err error) {
	driverOpt, found := options.Values().Get("store.sql.driver")
	if !found {
		return "", "", errors.New("No store.sql.driver option specified")
	}
	nodesOpt, found := options.Values().Get("store.nodes")
	if !found {
		return "", "", errors.New("No store.nodes option specified (expected a database connection string)")
	}
	driver, ok := driverOpt.(string)
	if !ok {
		return "", "", errors.New("store.sql.driver option must be a string")
	}
	nodes, ok := nodesOpt.([]string)
	if !ok {
		return "", "", errors.New("store.nodes option must be a []string")
	}
	if len(nodes) != 1 {
		return "", "", errors.New("expected only 1 store.nodes option")
	}
	namespaceOpt, found := options.Values().Get("store.namespace")
	if found {
		namespace, ok := namespaceOpt.(string)
		if !ok {
			return "", "", errors.New("store.namespace must me a string")
		}
		for _, r := range namespace {
			if !unicode.IsLetter(r) {
				return "", "", errors.New("store.namespace must only contain letters")
			}
		}
	}
	return driver, nodes[0], nil
}
<|endoftext|>


<commit_before>package consul

import (
	"math/rand"
	"sync/atomic"
	"time"

	"github.com/hashicorp/consul/lib"
)

type consulServerEventTypes int

const (
	// consulServersNodeJoin is used to notify of a new consulServer.
	// The primary effect of this is a reshuffling of consulServers and
	// finding a new preferredServer.
	consulServersNodeJoin = iota

	// consulServersRebalance is used to signal we should rebalance our
	// connection load across servers
	consulServersRebalance

	// consulServersRPCError is used to signal when a server has either
	// timed out or returned an error and we would like to have the
	// server manager find a new preferredServer.
	consulServersRPCError
)

// serverCfg is the thread-safe configuration structure that is used to
// maintain the list of consul servers in Client.
//
// NOTE(sean@): We are explicitly relying on the fact that this is copied.
// Please keep this structure light.
type serverConfig struct {
	// servers tracks the locally known servers
	servers []*serverParts

	// Timer used to control rebalancing of servers
	rebalanceTimer *time.Timer
}

// consulServersManager is used to automatically shuffle and rebalance the
// list of consulServers.  This maintenance happens either when a new server
// is added or when a duration has been exceed.
func (c *Client) consulServersManager() {
	defaultTimeout := 5 * time.Second // FIXME(sean@): This is a bullshit value
	var rebalanceTimer *time.Timer
	func(c *Client) {
		c.serverConfigLock.Lock()
		defer c.serverConfigLock.Unlock()

		serverCfgPtr := c.serverConfigValue.Load()
		if serverCfgPtr == nil {
			panic("server config has not been initialized")
		}
		var serverCfg serverConfig
		serverCfg = serverCfgPtr.(serverConfig)
		rebalanceTimer = time.NewTimer(defaultTimeout)
		serverCfg.rebalanceTimer = rebalanceTimer
	}(c)

	for {
		select {
		case e := <-c.consulServersCh:
			switch e {
			case consulServersNodeJoin:
				c.logger.Printf("[INFO] consul: new node joined cluster")
				c.RebalanceServers()
			case consulServersRebalance:
				c.logger.Printf("[INFO] consul: rebalancing servers by request")
				c.RebalanceServers()
			case consulServersRPCError:
				c.logger.Printf("[INFO] consul: need to find a new server to talk with")
				c.CycleFailedServers()
				// FIXME(sean@): wtb preemptive Status.Ping
				// of servers, ideally parallel fan-out of N
				// nodes, then settle on the first node which
				// responds successfully.
				//
				// Is there a distinction between slow and
				// offline?  Do we run the Status.Ping with a
				// fixed timeout (say 30s) that way we can
				// alert administrators that they've set
				// their RPC time too low even though the
				// Ping did return successfully?
			default:
				c.logger.Printf("[WARN] consul: unhandled LAN Serf Event: %#v", e)
			}
		case <-rebalanceTimer.C:
			c.logger.Printf("[INFO] consul: server rebalance timeout")
			c.RebalanceServers()

		case <-c.shutdownCh:
			return
		}
	}
}

func (c *Client) AddServer(server *serverParts) {
	c.serverConfigLock.Lock()
	defer c.serverConfigLock.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	// Check if this server is known
	found := false
	for idx, existing := range serverCfg.servers {
		if existing.Name == server.Name {
			// Overwrite the existing server parts in order to
			// possibly update metadata (i.e. server version)
			serverCfg.servers[idx] = server
			found = true
			break
		}
	}

	// Add to the list if not known
	if !found {
		serverCfg.servers = append(serverCfg.servers, server)

		// Notify the server maintenance task of a new server
		c.consulServersCh <- consulServersNodeJoin
	}

	c.serverConfigValue.Store(serverCfg)

}

func (c *Client) CycleFailedServers() {
	c.serverConfigLock.Lock()
	defer c.serverConfigLock.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	for i := range serverCfg.servers {
		failCount := atomic.LoadUint64(&(serverCfg.servers[i].Disabled))
		if failCount == 0 {
			break
		} else if failCount > 0 {
			serverCfg.servers = serverCfg.cycleServer()
		}
	}

	serverCfg.resetRebalanceTimer(c)
	c.serverConfigValue.Store(serverCfg)
}

func (sc *serverConfig) cycleServer() (servers []*serverParts) {
	numServers := len(servers)
	if numServers < 2 {
		// No action required for zero or one server situations
		return servers
	}

	var failedNode *serverParts
	failedNode, servers = servers[0], servers[1:]
	servers = append(servers, failedNode)
	return servers
}

func (c *Client) RebalanceServers() {
	c.serverConfigLock.Lock()
	defer c.serverConfigLock.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	// Shuffle the server list on server join.  Servers are selected from
	// the head of the list and are moved to the end of the list on
	// failure.
	for i := len(serverCfg.servers) - 1; i > 0; i-- {
		j := rand.Int31n(int32(i + 1))
		serverCfg.servers[i], serverCfg.servers[j] = serverCfg.servers[j], serverCfg.servers[i]
	}

	serverCfg.resetRebalanceTimer(c)
	c.serverConfigValue.Store(serverCfg)
}

func (c *Client) RemoveServer(server *serverParts) {
	c.serverConfigLock.Lock()
	defer c.serverConfigLock.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	// Remove the server if known
	n := len(serverCfg.servers)
	for i := 0; i < n; i++ {
		if serverCfg.servers[i].Name == server.Name {
			serverCfg.servers[i], serverCfg.servers[n-1] = serverCfg.servers[n-1], nil
			serverCfg.servers = serverCfg.servers[:n-1]
			break
		}
	}

	c.serverConfigValue.Store(serverCfg)

}

// resetRebalanceTimer assumes:
//
// 1) the serverConfigLock is already held by the caller.
// 2) the caller will call serverConfigValue.Store()
func (sc *serverConfig) resetRebalanceTimer(c *Client) {
	numConsulServers := len(sc.servers)
	// Limit this connection's life based on the size (and health) of the
	// cluster.  Never rebalance a connection more frequently than
	// connReuseLowWatermarkDuration, and make sure we never exceed
	// clusterWideRebalanceConnsPerSec operations/s across numLANMembers.
	clusterWideRebalanceConnsPerSec := float64(numConsulServers * newRebalanceConnsPerSecPerServer)
	connReuseLowWatermarkDuration := clientRPCMinReuseDuration + lib.RandomStagger(clientRPCMinReuseDuration/clientRPCJitterFraction)
	numLANMembers := len(c.LANMembers())
	connRebalanceTimeout := lib.RateScaledInterval(clusterWideRebalanceConnsPerSec, connReuseLowWatermarkDuration, numLANMembers)
	c.logger.Printf("[DEBUG] consul: connection will be rebalanced in %v", connRebalanceTimeout)

	sc.rebalanceTimer.Reset(connRebalanceTimeout)
}
<commit_msg>Move consul.serverConfig out of the consul package<commit_after><|endoftext|>


<commit_before>package mapgen

import (
	"fmt"

	"buildblast/lib/coords"
)

type Chunk [][][]Block

func generateChunk(bg blockGenerator, cc coords.Chunk) (Chunk, []coords.World) {
	cw := coords.ChunkWidth
	ch := coords.ChunkHeight
	cd := coords.ChunkDepth

	spawns := make([]coords.World, 0)
	blocks := make([][][]Block, cw)
	for ocX := 0; ocX < cw; ocX++ {
		blocks[ocX] = make([][]Block, ch)
		for ocY := 0; ocY < ch; ocY++ {
			blocks[ocX][ocY] = make([]Block, cd)
			for ocZ := 0; ocZ < cd; ocZ++ {
				bc := coords.Block{
					X: ocX + cc.X*cw,
					Y: ocY + cc.Y*ch,
					Z: ocZ + cc.Z*cd,
				}
				block, isSpawn := bg.Block(bc)
				blocks[ocX][ocY][ocZ] = block
				if isSpawn {
					spawns = append(spawns, bc.Center())
				}
			}
		}
	}
	return blocks, spawns
}

func (c Chunk) Block(oc coords.Offset) Block {
	return c[oc.X][oc.Y][oc.Z]
}

func (c Chunk) SetBlock(oc coords.Offset, newBlock Block) {
	c[oc.X][oc.Y][oc.Z] = newBlock
}

// Flatten returns the chunk data as a string. It
// can be used for various forms of serialization
// where a valid UTF-8 string is required, and
// efficiency (in terms of size) is not hugely
// important. We use this format for chunk data
// in JSON documents because:
//  a) It's smaller, at least half the size, of a
//     normal JSON array, potentially more.
//     (depending on the contents)
//  b) It's much faster - the go JSON implementation
//     isn't particulilly fast at serializing large
//     arrays of numbers.
func (c Chunk) Flatten() string {
	cw := coords.ChunkWidth
	ch := coords.ChunkHeight
	cd := coords.ChunkDepth
	data := make([]byte, cw*ch*cd)
	for ocX := 0; ocX < cw; ocX++ {
		for ocY := 0; ocY < ch; ocY++ {
			for ocZ := 0; ocZ < cd; ocZ++ {
				// 35: # charater. Control charaters
				// are not allowed in JSON strings, and
				// we want to avoid '"', which requires
				// escaping.
				value := byte(c[ocX][ocY][ocZ] + 35)
				if value >= 127 || value < 35 {
					panic(fmt.Sprintf("Attempted to encode out of range value of '%d' to chunk data. (It might work but we need to test it)", value))
				}
				data[ocX*cw*ch+ocY*cw+ocZ] = value
			}
		}
	}
	return string(data)
}

func (c Chunk) Clone() Chunk {
	cw := coords.ChunkWidth
	ch := coords.ChunkHeight
	cd := coords.ChunkDepth

	blocks := make([][][]Block, cw)
	for ocX := 0; ocX < cw; ocX++ {
		blocks[ocX] = make([][]Block, ch)
		for ocY := 0; ocY < ch; ocY++ {
			blocks[ocX][ocY] = make([]Block, cd)
			for ocZ := 0; ocZ < cd; ocZ++ {
				blocks[ocX][ocY][ocZ] = c[ocX][ocY][ocZ]
			}
		}
	}
	return blocks
}
<commit_msg>Make chunks allocated flat.<commit_after>package mapgen

import (
	"fmt"

	"buildblast/lib/coords"
)

type Chunk []Block
const blocksPerChunk = coords.ChunkWidth * coords.ChunkHeight * coords.ChunkDepth

func allocChunk() (Chunk) {
	return make([]Block, blocksPerChunk)
}

func generateChunk(bg blockGenerator, cc coords.Chunk) (Chunk, []coords.World) {
	cw := coords.ChunkWidth
	ch := coords.ChunkHeight
	cd := coords.ChunkDepth

	spawns := make([]coords.World, 0)
	chunk := allocChunk();
	for ocX := 0; ocX < cw; ocX++ {
		for ocY := 0; ocY < ch; ocY++ {
			for ocZ := 0; ocZ < cd; ocZ++ {
				bc := coords.Block{
					X: ocX + cc.X*cw,
					Y: ocY + cc.Y*ch,
					Z: ocZ + cc.Z*cd,
				}
				block, isSpawn := bg.Block(bc)
				chunk.SetBlock(bc.Offset(), block)
				if isSpawn {
					spawns = append(spawns, bc.Center())
				}
			}
		}
	}
	return chunk, spawns
}

func (c Chunk) Block(oc coords.Offset) Block {
	cw := coords.ChunkWidth
	ch := coords.ChunkHeight
	return c[oc.X*cw*ch + oc.Y*cw + oc.Z]
}

func (c Chunk) SetBlock(oc coords.Offset, newBlock Block) {
	cw := coords.ChunkWidth
	ch := coords.ChunkHeight
	c[oc.X*cw*ch + oc.Y*cw + oc.Z] = newBlock
}

// Flatten returns the chunk data as a string. It
// can be used for various forms of serialization
// where a valid UTF-8 string is required, and
// efficiency (in terms of size) is not hugely
// important. We use this format for chunk data
// in JSON documents because:
//  a) It's smaller, at least half the size, of a
//     normal JSON array, potentially more.
//     (depending on the contents)
//  b) It's much faster - the go JSON implementation
//     isn't particulilly fast at serializing large
//     arrays of numbers.
func (c Chunk) Flatten() string {
	data := make([]byte, blocksPerChunk)
	for i := 0; i < blocksPerChunk; i++ {
		// 35: # charater. Control charaters are not allowed in JSON
		// strings, and we want to avoid '"', which requires escaping.
		value := byte(c[i] + 35)
		if value >= 127 || value < 35 {
			panic(fmt.Sprintf("Attempted to encode out of range value of '%d' to chunk data. (It might work but we need to test it)", value))
		}
		data[i] = value
	}
	return string(data)
}

func (c Chunk) Clone() Chunk {
	newChunk := allocChunk()
	for i := 0; i < blocksPerChunk; i++ {
		newChunk[i] = c[i];
	}
	return newChunk
}
<|endoftext|>


<commit_before>package apps

import (
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/assets"
)

var _ = Describe("Buildpacks", func() {
	var appName string

	BeforeEach(func() {
		appName = generator.RandomName()
	})

	AfterEach(func() {
		Expect(cf.Cf("delete", appName, "-f").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
	})

	Describe("node", func() {
		It("makes the app reachable via its bound route", func() {
			Expect(cf.Cf("push", appName, "-p", assets.NewAssets().Node, "-c", "node app.js").Wait(CF_PUSH_TIMEOUT)).To(Exit(0))

			Eventually(func() string {
				return helpers.CurlAppRoot(appName)
			}, DEFAULT_TIMEOUT).Should(ContainSubstring("Hello from a node app!"))
		})
	})

	Describe("java", func() {
		It("makes the app reachable via its bound route", func() {
			Expect(cf.Cf("push", appName, "-p", assets.NewAssets().Java).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))

			Eventually(func() string {
				return helpers.CurlAppRoot(appName)
			}, DEFAULT_TIMEOUT).Should(ContainSubstring("Hello, from your friendly neighborhood Java JSP!"))
		})
	})

	Describe("golang", func() {
		It("makes the app reachable via its bound route", func() {
			Expect(cf.Cf("push", appName, "-p", assets.NewAssets().Golang).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))

			Eventually(func() string {
				return helpers.CurlAppRoot(appName)
			}, DEFAULT_TIMEOUT).Should(ContainSubstring("go, world"))
		})
	})

	Describe("python", func() {
		It("makes the app reachable via its bound route", func() {
			Expect(cf.Cf("push", appName, "-p", assets.NewAssets().Python).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))

			Eventually(func() string {
				return helpers.CurlAppRoot(appName)
			}, DEFAULT_TIMEOUT).Should(ContainSubstring("python, world"))
		})
	})

	Describe("php", func() {
		// This test requires more time during push, because the php buildpack is slower than your average bear
		var phpPushTimeout = CF_PUSH_TIMEOUT + 6*time.Minute

		It("makes the app reachable via its bound route", func() {
			Expect(cf.Cf("push", appName, "-p", assets.NewAssets().Php).Wait(phpPushTimeout)).To(Exit(0))

			Eventually(func() string {
				return helpers.CurlAppRoot(appName)
			}, DEFAULT_TIMEOUT).Should(ContainSubstring("Hello from php"))
		})
	})
})
<commit_msg>Added 'cf api' to the before for the build-pack tests<commit_after>package apps

import (
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/assets"
)

var _ = Describe("Buildpacks", func() {
	var appName string

	BeforeEach(func() {
		appName = generator.RandomName()
		apiEndpoint := helpers.LoadConfig().ApiEndpoint
		Expect(cf.Cf("api", apiEndpoint, "--skip-ssl-validation").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
	})

	AfterEach(func() {
		Expect(cf.Cf("delete", appName, "-f").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
	})

	Describe("node", func() {
		It("makes the app reachable via its bound route", func() {
			Expect(cf.Cf("push", appName, "-p", assets.NewAssets().Node, "-c", "node app.js").Wait(CF_PUSH_TIMEOUT)).To(Exit(0))

			Eventually(func() string {
				return helpers.CurlAppRoot(appName)
			}, DEFAULT_TIMEOUT).Should(ContainSubstring("Hello from a node app!"))
		})
	})

	Describe("java", func() {
		It("makes the app reachable via its bound route", func() {
			Expect(cf.Cf("push", appName, "-p", assets.NewAssets().Java).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))

			Eventually(func() string {
				return helpers.CurlAppRoot(appName)
			}, DEFAULT_TIMEOUT).Should(ContainSubstring("Hello, from your friendly neighborhood Java JSP!"))
		})
	})

	Describe("golang", func() {
		It("makes the app reachable via its bound route", func() {
			Expect(cf.Cf("push", appName, "-p", assets.NewAssets().Golang).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))

			Eventually(func() string {
				return helpers.CurlAppRoot(appName)
			}, DEFAULT_TIMEOUT).Should(ContainSubstring("go, world"))
		})
	})

	Describe("python", func() {
		It("makes the app reachable via its bound route", func() {
			Expect(cf.Cf("push", appName, "-p", assets.NewAssets().Python).Wait(CF_PUSH_TIMEOUT)).To(Exit(0))

			Eventually(func() string {
				return helpers.CurlAppRoot(appName)
			}, DEFAULT_TIMEOUT).Should(ContainSubstring("python, world"))
		})
	})

	Describe("php", func() {
		// This test requires more time during push, because the php buildpack is slower than your average bear
		var phpPushTimeout = CF_PUSH_TIMEOUT + 6*time.Minute

		It("makes the app reachable via its bound route", func() {
			Expect(cf.Cf("push", appName, "-p", assets.NewAssets().Php).Wait(phpPushTimeout)).To(Exit(0))

			Eventually(func() string {
				return helpers.CurlAppRoot(appName)
			}, DEFAULT_TIMEOUT).Should(ContainSubstring("Hello from php"))
		})
	})
})
<|endoftext|>


<commit_before>package wedge

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"
)

const (
	HTTP = iota
	JSON
)

const (
	FileChunks = 1024
)

var (
	routes []*url
)

type handlertype int

type appServer struct {
	port    string
	routes  []*url
	timeout time.Duration
}

// Handler functions should match this signature
type view func(*http.Request) (string, int)

// This is the main 'event loop' for the web server. All requests are
// sent to this handler, which checks the incoming request against
// all the routes we have setup if it finds a match it will invoke
// the handler which is attached to that match.
func (self *appServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	request := req.URL.Path

	for _, route := range self.routes {
		matches := route.match.FindAllStringSubmatch(request, 1)
		if len(matches) > 0 {
			log.Println("Request:", route.name)
			resp, err := route.handler(req)
			if err == 404 {
				http.NotFound(w, req)
				return
			}

			switch route.viewtype {
			case HTTP:
				io.WriteString(w, resp)
				return
			case JSON:
				w.Header().Set("Content-type", "application/json")
				json.NewEncoder(w).Encode(map[string]string{
					"message": resp,
				})
				return
			default:
				panic("Unknown handler type!")
			}
		}
	}
	log.Println("404")
	http.NotFound(w, req)
}

// Basic URL struct which holds a match, a name and a handler function
//
// match:
//     Match is a *regexp.Regexp which we will use to check incoming
//     request URLs against and return the handler on any that match
// name:
//     Name is a simple string of what the url should be referred to as
// handler:
//     Handler is a wedge.view function which we will use against any
//     requests that match `match`.
type url struct {
	match *regexp.Regexp
	name string
	handler view
	viewtype handlertype
	rawre string
}

func (u *url) String() string {
	return fmt.Sprintf(
		"{\n  URL: %s\n  Handler: %s\n}", u.match, u.name,
	)
}

// URL is a function which returns a URL value.
// re:
//     re is a string which will be compiled to a *regexp.Regexp
//     and will panic if the regular expression cannot be compiled
// name:
//     Name is a simple string of what the url should be referred to as
// handler:
//     Handler is a wedge.view function which we will use against any
//     requests that match `match`.
func URL(re, name string, v view, t handlertype)  *url {
	match := regexp.MustCompile(re)
	return &url{
		match: match,
		name: name,
		handler: v,
		viewtype: t,
		rawre: re,
	}
}

// StaticFiles is a not so light wrapper around the URL function
//
// We start off receiving an 'as' string which marks the URL to which
// we match against. We then take a []string which is filepaths to all
// the locations in which an incoming file request should be checked
// against. The file is read in chunks as per the module level constant
// FileChunk.
//
// This function will return a file in a string format ready to be sent
// across the wire.
func StaticFiles(as string, paths ...string) *url {

	return URL(as, "Static File", func(req *http.Request) (string, int) {
		log.Println(req.URL.Path)
		filename := req.URL.Path[len(as):]
		b := []string{}

		for _, path := range paths {
			// Prevent Directory Traversal Attacks
			if len(strings.Split(path, "..")) > 1 {
				return "", http.StatusNotFound
			}

			// Attempt to open the file in using one of the paths
			file, err := os.Open(filepath.Join(path, filename))
			if err != nil {
				continue
			}

			// if we're here, the file exists and we just need to send
			// it to the client.
			for {
				reader := make([]byte, FileChunks)
				count, err := file.Read(reader)
				if err != nil {
					return strings.Join(b, ""), http.StatusOK
				}

				b = append(b, string(reader[:count]))
			}
		}
		return "", http.StatusNotFound
	}, HTTP)

}

// Patterns is a helper function which returns a *[]*url.
func Patterns(urls ...*url) (*[]*url) {
	r := make([]*url, 0)
	for _, url := range urls {
		r = append(r, url)
	}

	return &r
}

// Starts the server running on PORT `port` with the timeout duration
func Run(patterns *[]*url, port string, timeout time.Duration) {
	app := &appServer{port, *patterns, timeout}
	server := http.Server{
		Addr: ":"+app.port,
		Handler: app,
		ReadTimeout: app.timeout * time.Second,
	}
	fmt.Printf("Serving on PORT: %s", port)
	fmt.Println("\n")
	err := server.ListenAndServe()
	if err != nil {
		fmt.Println(err)
	}
}<commit_msg>Gofmt. And added a new request handle type: STATIC. Static will be used when handling static files since static files require that their content header be set.<commit_after>package wedge

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"
)

const (
	HTTP = iota
	JSON
	STATIC
)

const (
	FileChunks = 1024
)

var (
	routes []*url
)

type handlertype int

type appServer struct {
	port    string
	routes  []*url
	timeout time.Duration
}

// Handler functions should match this signature
type view func(*http.Request) (string, int)

// This is the main 'event loop' for the web server. All requests are
// sent to this handler, which checks the incoming request against
// all the routes we have setup if it finds a match it will invoke
// the handler which is attached to that match.
func (self *appServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	request := req.URL.Path

	for _, route := range self.routes {
		matches := route.match.FindAllStringSubmatch(request, 1)
		if len(matches) > 0 {
			log.Println("Request:", route.name)
			resp, err := route.handler(req)
			if err == 404 {
				http.NotFound(w, req)
				return
			}

			switch route.viewtype {
			case HTTP:
				io.WriteString(w, resp)
				return
			case JSON:
				w.Header().Set("Content-type", "application/json")
				json.NewEncoder(w).Encode(map[string]string{
					"message": resp,
				})
				return
			case STATIC:
				if w.Header().Get("Content-Type") == "" {
					reqstr := req.URL.Path[len(route.rawre):]
					ctype := mime.TypeByExtension(filepath.Ext(reqstr))
					w.Header().Set("Content-Type", ctype)
				}
				io.WriteString(w, resp)
				return
			default:
				panic("Unknown handler type!")
			}
		}
	}
	log.Println("404")
	http.NotFound(w, req)
}

// Basic URL struct which holds a match, a name and a handler function
//
// match:
//     Match is a *regexp.Regexp which we will use to check incoming
//     request URLs against and return the handler on any that match
// name:
//     Name is a simple string of what the url should be referred to as
// handler:
//     Handler is a wedge.view function which we will use against any
//     requests that match `match`.
type url struct {
	match    *regexp.Regexp
	name     string
	handler  view
	viewtype handlertype
	rawre    string
}

func (u *url) String() string {
	return fmt.Sprintf(
		"{\n  URL: %s\n  Handler: %s\n}", u.match, u.name,
	)
}

// URL is a function which returns a URL value.
// re:
//     re is a string which will be compiled to a *regexp.Regexp
//     and will panic if the regular expression cannot be compiled
// name:
//     Name is a simple string of what the url should be referred to as
// handler:
//     Handler is a wedge.view function which we will use against any
//     requests that match `match`.
func URL(re, name string, v view, t handlertype) *url {
	match := regexp.MustCompile(re)
	return &url{
		match:    match,
		name:     name,
		handler:  v,
		viewtype: t,
		rawre:    re,
	}
}

// StaticFiles is a not so light wrapper around the URL function
//
// We start off receiving an 'as' string which marks the URL to which
// we match against. We then take a []string which is filepaths to all
// the locations in which an incoming file request should be checked
// against. The file is read in chunks as per the module level constant
// FileChunk.
//
// This function will return a file in a string format ready to be sent
// across the wire.
func StaticFiles(as string, paths ...string) *url {

	return URL(as, "Static File", func(req *http.Request) (string, int) {
		log.Println(req.URL.Path)
		filename := req.URL.Path[len(as):]
		b := []string{}

		for _, path := range paths {
			// Prevent Directory Traversal Attacks
			if len(strings.Split(path, "..")) > 1 {
				return "", http.StatusNotFound
			}

			// Attempt to open the file in using one of the paths
			file, err := os.Open(filepath.Join(path, filename))
			if err != nil {
				continue
			}

			// if we're here, the file exists and we just need to send
			// it to the client.
			for {
				reader := make([]byte, FileChunks)
				count, err := file.Read(reader)
				if err != nil {
					return strings.Join(b, ""), http.StatusOK
				}

				b = append(b, string(reader[:count]))
			}
		}
		return "", http.StatusNotFound
	}, STATIC)

}

// Patterns is a helper function which returns a *[]*url.
func Patterns(urls ...*url) *[]*url {
	r := make([]*url, 0)
	for _, url := range urls {
		r = append(r, url)
	}

	return &r
}

// Starts the server running on PORT `port` with the timeout duration
func Run(patterns *[]*url, port string, timeout time.Duration) {
	app := &appServer{port, *patterns, timeout}
	server := http.Server{
		Addr:        ":" + app.port,
		Handler:     app,
		ReadTimeout: app.timeout * time.Second,
	}
	fmt.Printf("Serving on PORT: %s", port)
	fmt.Println("\n")
	err := server.ListenAndServe()
	if err != nil {
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before>package parser

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"testing"

	"cloud.google.com/go/bigquery"
)

func TestNDTParser(t *testing.T) {
	// Load test data.
	rawData, err := ioutil.ReadFile("testdata/c2s_snaplog")

	n := NDTParser{tmpDir: "./"}
	values, err := n.Parse("filename", "table", rawData)
	if err != nil {
		t.Fatalf(err.Error())
	}

	results := values.(map[string]bigquery.Value)
	// TODO(dev): find a better way to verify the returned values are correct.
	expectedValues := map[string]bigquery.Value{
		"web100_log_entry.version":                    "2.5.27 201001301335 net100",
		"web100_log_entry.snap.RemAddress":            "45.56.98.222",
		"web100_log_entry.connection_spec.local_port": int64(43685),
	}
	for key, value := range expectedValues {
		// Raw bigquery.Value instances do not compare.
		if results[key] != value {
			t.Errorf("Wrong value for %q: got %q; want %q", key, str(results[key]), str(value))
		}
	}
	// TODO(dev): remove print of entire snaplog.
	prettyPrint(r)
}

// TODO(dev): is there a better way to display these values?
func str(v bigquery.Value) string {
	switch v.(type) {
	case string:
		s := v.(string)
		return s
	case int64:
		i := v.(int64)
		return fmt.Sprintf("%d", i)
	default:
		panic("Only string and int64 types are supported.")
	}
}

func prettyPrint(results map[string]bigquery.Value) {
	b, err := json.MarshalIndent(results, "", "  ")
	if err != nil {
		fmt.Println("error:", err)
	}
	fmt.Print(string(b))
}
<commit_msg>Fix variable name in ndt test.<commit_after>package parser

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"testing"

	"cloud.google.com/go/bigquery"
)

func TestNDTParser(t *testing.T) {
	// Load test data.
	rawData, err := ioutil.ReadFile("testdata/c2s_snaplog")

	n := NDTParser{tmpDir: "./"}
	values, err := n.Parse("filename", "table", rawData)
	if err != nil {
		t.Fatalf(err.Error())
	}

	results := values.(map[string]bigquery.Value)
	// TODO(dev): find a better way to verify the returned values are correct.
	expectedValues := map[string]bigquery.Value{
		"web100_log_entry.version":                    "2.5.27 201001301335 net100",
		"web100_log_entry.snap.RemAddress":            "45.56.98.222",
		"web100_log_entry.connection_spec.local_port": int64(43685),
	}
	for key, value := range expectedValues {
		// Raw bigquery.Value instances do not compare.
		if results[key] != value {
			t.Errorf("Wrong value for %q: got %q; want %q", key, str(results[key]), str(value))
		}
	}
	// TODO(dev): remove print of entire snaplog.
	prettyPrint(results)
}

// TODO(dev): is there a better way to display these values?
func str(v bigquery.Value) string {
	switch v.(type) {
	case string:
		s := v.(string)
		return s
	case int64:
		i := v.(int64)
		return fmt.Sprintf("%d", i)
	default:
		panic("Only string and int64 types are supported.")
	}
}

func prettyPrint(results map[string]bigquery.Value) {
	b, err := json.MarshalIndent(results, "", "  ")
	if err != nil {
		fmt.Println("error:", err)
	}
	fmt.Print(string(b))
}
<|endoftext|>


<commit_before>package routines

import (
	"sync"
	"time"

	"github.com/Shopify/sarama"
	log "github.com/Sirupsen/logrus"
	saramaC "github.com/d33d33/sarama-cluster"
	"github.com/d33d33/viper" // FIXME https://github.com/spf13/viper/pull/285

	"github.com/runabove/metronome/src/metronome/constants"
	"github.com/runabove/metronome/src/metronome/models"
)

// TaskConsumer handle the internal states of the consumer
type TaskConsumer struct {
	client   *saramaC.Client
	consumer *saramaC.Consumer
	drained  bool
	drainWg  sync.WaitGroup
	tasks    chan models.Task
}

// NewTaskComsumer return a new task consumer
func NewTaskComsumer() (*TaskConsumer, error) {
	brokers := viper.GetStringSlice("kafka.brokers")

	config := saramaC.NewConfig()
	config.ClientID = "metronome-scheduler"
	config.Consumer.Offsets.Initial = sarama.OffsetOldest

	client, err := saramaC.NewClient(brokers, config)
	if err != nil {
		return nil, err
	}

	consumer, err := saramaC.NewConsumerFromClient(client, "schedulers", []string{constants.KafkaTopicTasks})
	if err != nil {
		return nil, err
	}

	tc := &TaskConsumer{
		client:   client,
		consumer: consumer,
		tasks:    make(chan models.Task),
	}

	hwm := tc.highWaterMarks()
	offsets := make(map[int32]int64)
	messages := 0

	tc.drainWg.Add(1)

	// Progress display
	ticker := time.NewTicker(500 * time.Millisecond)
	if log.GetLevel() == log.DebugLevel {
		go func() {
			for {
				select {
				case <-ticker.C:
					log.WithField("count", messages).Debug("Loading tasks")
				}
			}
		}()
	}

	// Drain timeout
	timeout := time.NewTicker(700 * time.Millisecond)

	go func() {
		for {
			select {
			case msg, ok := <-consumer.Messages():
				if !ok { // shuting down
					return
				}
				messages++
				tc.handleMsg(msg)

				offsets[msg.Partition] = msg.Offset
				timeout.Stop()
				if !tc.drained && tc.isDrained(hwm, offsets) {
					ticker.Stop()
					tc.drained = true
					tc.drainWg.Done()
				}
			case <-timeout.C:
				if !tc.drained {
					tc.drained = true
					ticker.Stop()
					timeout.Stop()
					tc.drainWg.Done()
				}
			}
		}
	}()

	return tc, nil
}

// Tasks return the incomming task channel
func (tc *TaskConsumer) Tasks() <-chan models.Task {
	return tc.tasks
}

// WaitForDrain wait for consumer to EOF partitions
func (tc *TaskConsumer) WaitForDrain() {
	tc.drainWg.Wait()
}

// Close the task consumer
func (tc *TaskConsumer) Close() (err error) {
	if e := tc.consumer.Close(); e != nil {
		err = e
	}
	if e := tc.client.Close(); e != nil {
		err = e
	}
	return
}

// Handle incomming messages
func (tc *TaskConsumer) handleMsg(msg *sarama.ConsumerMessage) {
	var t models.Task
	if err := t.FromKafka(msg); err != nil {
		log.Error(err)
		return
	}
	log.Debugf("Task received: %v", t.ToJSON())
	tc.tasks <- t
}

// Retrive highWaterMarks for each partition
func (tc *TaskConsumer) highWaterMarks() map[int32]int64 {
	res := make(map[int32]int64)

	parts, err := tc.client.Partitions(constants.KafkaTopicTasks)
	if err != nil {
		log.Panic(err)
	}

	for p := range parts {
		i, err := tc.client.GetOffset(constants.KafkaTopicTasks, int32(p), sarama.OffsetNewest)
		if err != nil {
			log.Panic(err)
		}

		res[int32(p)] = i
	}

	return res
}

// Check if consumer reach EOF on all the partitions
func (tc *TaskConsumer) isDrained(hwm, offsets map[int32]int64) bool {
	subs := tc.consumer.Subscriptions()[constants.KafkaTopicTasks]

	for partition := range subs {
		part := int32(partition)
		if _, ok := hwm[part]; !ok {
			log.Panicf("Missing HighWaterMarks for partition %v", part)
		}
		if hwm[part] == 0 {
			continue
		}
		// No message received for partiton
		if _, ok := offsets[part]; !ok {
			return false
		}
		// Check offset
		if (offsets[part] + 1) < hwm[part] {
			return false
		}
	}

	return true
}
<commit_msg>fix(scheduler): make scheduler wait for topic creation<commit_after>package routines

import (
	"sync"
	"time"

	"github.com/Shopify/sarama"
	log "github.com/Sirupsen/logrus"
	saramaC "github.com/d33d33/sarama-cluster"
	"github.com/d33d33/viper" // FIXME https://github.com/spf13/viper/pull/285

	"github.com/runabove/metronome/src/metronome/constants"
	"github.com/runabove/metronome/src/metronome/models"
)

// TaskConsumer handle the internal states of the consumer
type TaskConsumer struct {
	client   *saramaC.Client
	consumer *saramaC.Consumer
	drained  bool
	drainWg  sync.WaitGroup
	tasks    chan models.Task
}

// NewTaskComsumer return a new task consumer
func NewTaskComsumer() (*TaskConsumer, error) {
	brokers := viper.GetStringSlice("kafka.brokers")

	config := saramaC.NewConfig()
	config.ClientID = "metronome-scheduler"
	config.Consumer.Offsets.Initial = sarama.OffsetOldest

	client, err := saramaC.NewClient(brokers, config)
	if err != nil {
		return nil, err
	}

	consumer, err := saramaC.NewConsumerFromClient(client, "schedulers", []string{constants.KafkaTopicTasks})
	if err != nil {
		return nil, err
	}

	tc := &TaskConsumer{
		client:   client,
		consumer: consumer,
		tasks:    make(chan models.Task),
	}

	hwm := <-tc.highWaterMarks()
	offsets := make(map[int32]int64)
	messages := 0

	tc.drainWg.Add(1)

	// Progress display
	ticker := time.NewTicker(500 * time.Millisecond)
	if log.GetLevel() == log.DebugLevel {
		go func() {
			for {
				select {
				case <-ticker.C:
					log.WithField("count", messages).Debug("Loading tasks")
				}
			}
		}()
	}

	// Drain timeout
	timeout := time.NewTicker(700 * time.Millisecond)

	go func() {
		for {
			select {
			case msg, ok := <-consumer.Messages():
				if !ok { // shuting down
					return
				}
				messages++
				tc.handleMsg(msg)

				offsets[msg.Partition] = msg.Offset
				timeout.Stop()
				if !tc.drained && tc.isDrained(hwm, offsets) {
					ticker.Stop()
					tc.drained = true
					tc.drainWg.Done()
				}
			case <-timeout.C:
				if !tc.drained {
					tc.drained = true
					ticker.Stop()
					timeout.Stop()
					tc.drainWg.Done()
				}
			}
		}
	}()

	return tc, nil
}

// Tasks return the incomming task channel
func (tc *TaskConsumer) Tasks() <-chan models.Task {
	return tc.tasks
}

// WaitForDrain wait for consumer to EOF partitions
func (tc *TaskConsumer) WaitForDrain() {
	tc.drainWg.Wait()
}

// Close the task consumer
func (tc *TaskConsumer) Close() (err error) {
	if e := tc.consumer.Close(); e != nil {
		err = e
	}
	if e := tc.client.Close(); e != nil {
		err = e
	}
	return
}

// Handle incomming messages
func (tc *TaskConsumer) handleMsg(msg *sarama.ConsumerMessage) {
	var t models.Task
	if err := t.FromKafka(msg); err != nil {
		log.Error(err)
		return
	}
	log.Debugf("Task received: %v", t.ToJSON())
	tc.tasks <- t
}

// Retrive highWaterMarks for each partition
func (tc *TaskConsumer) highWaterMarks() chan map[int32]int64 {
	resChan := make(chan map[int32]int64)

	go func() {
		for {
			parts, err := tc.client.Partitions(constants.KafkaTopicTasks)
			if err != nil {
				log.Warn("Can't get topic. Retry")
				continue
			}

			res := make(map[int32]int64)
			for p := range parts {
				i, err := tc.client.GetOffset(constants.KafkaTopicTasks, int32(p), sarama.OffsetNewest)
				if err != nil {
					log.Panic(err)
				}

				res[int32(p)] = i
			}

			resChan <- res
			close(resChan)
			break
		}
	}()

	return resChan
}

// Check if consumer reach EOF on all the partitions
func (tc *TaskConsumer) isDrained(hwm, offsets map[int32]int64) bool {
	subs := tc.consumer.Subscriptions()[constants.KafkaTopicTasks]

	for partition := range subs {
		part := int32(partition)
		if _, ok := hwm[part]; !ok {
			log.Panicf("Missing HighWaterMarks for partition %v", part)
		}
		if hwm[part] == 0 {
			continue
		}
		// No message received for partiton
		if _, ok := offsets[part]; !ok {
			return false
		}
		// Check offset
		if (offsets[part] + 1) < hwm[part] {
			return false
		}
	}

	return true
}
<|endoftext|>


<commit_before>// A `pass` based credential helper. Passwords are stored as arguments to pass
// of the form: "$PASS_FOLDER/base64-url(serverURL)/username". We base64-url
// encode the serverURL, because under the hood pass uses files and folders, so
// /s will get translated into additional folders.
package pass

import (
	"encoding/base64"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"

	"github.com/docker/docker-credential-helpers/credentials"
)

const PASS_FOLDER = "docker-credential-helpers"

var (
	passInitialized bool
)

func init() {
	passInitialized = exec.Command("pass").Run() == nil
}

func runPass(stdinContent string, args ...string) (string, error) {
	cmd := exec.Command("pass", args...)

	stdin, err := cmd.StdinPipe()
	if err != nil {
		return "", err
	}
	defer stdin.Close()

	stderr, err := cmd.StderrPipe()
	if err != nil {
		return "", err
	}
	defer stderr.Close()

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return "", err
	}
	defer stdout.Close()

	err = cmd.Start()
	if err != nil {
		return "", err
	}

	_, err = stdin.Write([]byte(stdinContent))
	if err != nil {
		return "", err
	}
	stdin.Close()

	errContent, err := ioutil.ReadAll(stderr)
	if err != nil {
		return "", fmt.Errorf("error reading stderr: %s", err)
	}

	result, err := ioutil.ReadAll(stdout)
	if err != nil {
		return "", fmt.Errorf("Error reading stdout: %s", err)
	}

	cmdErr := cmd.Wait()
	if cmdErr != nil {
		return "", fmt.Errorf("%s: %s", cmdErr, errContent)
	}

	return string(result), nil
}

// Pass handles secrets using Linux secret-service as a store.
type Pass struct{}

// Add adds new credentials to the keychain.
func (h Pass) Add(creds *credentials.Credentials) error {
	if !passInitialized {
		return errors.New("pass store is uninitialized")
	}

	if creds == nil {
		return errors.New("missing credentials")
	}

	encoded := base64.URLEncoding.EncodeToString([]byte(creds.ServerURL))

	_, err := runPass(creds.Secret, "insert", "-f", "-m", path.Join(PASS_FOLDER, encoded, creds.Username))
	return err
}

// Delete removes credentials from the store.
func (h Pass) Delete(serverURL string) error {
	if !passInitialized {
		return errors.New("pass store is uninitialized")
	}

	if serverURL == "" {
		return errors.New("missing server url")
	}

	encoded := base64.URLEncoding.EncodeToString([]byte(serverURL))
	_, err := runPass("", "rm", "-rf", path.Join(PASS_FOLDER, encoded))
	return err
}

// listPassDir lists all the contents of a directory in the password store.
// Pass uses fancy unicode to emit stuff to stdout, so rather than try
// and parse this, let's just look at the directory structure instead.
func listPassDir(args ...string) ([]os.FileInfo, error) {
	passDir := os.ExpandEnv("$HOME/.password-store")
	for _, e := range os.Environ() {
		parts := strings.SplitN(e, "=", 2)
		if len(parts) < 2 {
			continue
		}

		if parts[0] != "PASSWORD_STORE_DIR" {
			continue
		}

		passDir = parts[1]
		break
	}

	p := path.Join(append([]string{passDir, PASS_FOLDER}, args...)...)
	contents, err := ioutil.ReadDir(p)
	if err != nil {
		if os.IsNotExist(err) {
			return []os.FileInfo{}, nil
		}

		return nil, err
	}

	return contents, nil
}

// Get returns the username and secret to use for a given registry server URL.
func (h Pass) Get(serverURL string) (string, string, error) {
	if !passInitialized {
		return "", "", errors.New("pass store is uninitialized")
	}

	if serverURL == "" {
		return "", "", errors.New("missing server url")
	}

	encoded := base64.URLEncoding.EncodeToString([]byte(serverURL))

	usernames, err := listPassDir(encoded)
	if err != nil {
		return "", "", err
	}

	if len(usernames) < 1 {
		return "", "", fmt.Errorf("no usernames for %s", serverURL)
	}

	actual := strings.TrimSuffix(usernames[0].Name(), ".gpg")
	secret, err := runPass("", "show", path.Join(PASS_FOLDER, encoded, actual))
	return actual, secret, err
}

// List returns the stored URLs and corresponding usernames for a given credentials label
func (h Pass) List() (map[string]string, error) {
	if !passInitialized {
		return nil, errors.New("pass store is uninitialized")
	}

	servers, err := listPassDir()
	if err != nil {
		return nil, err
	}

	resp := map[string]string{}

	for _, server := range servers {
		if !server.IsDir() {
			continue
		}

		serverURL, err := base64.URLEncoding.DecodeString(server.Name())
		if err != nil {
			return nil, err
		}

		usernames, err := listPassDir(server.Name())
		if err != nil {
			return nil, err
		}

		if len(usernames) < 1 {
			return nil, fmt.Errorf("no usernames for %s", serverURL)
		}

		resp[string(serverURL)] = strings.TrimSuffix(usernames[0].Name(), ".gpg")
	}

	return resp, nil
}
<commit_msg>pass backend: expose pass initialized flag<commit_after>// A `pass` based credential helper. Passwords are stored as arguments to pass
// of the form: "$PASS_FOLDER/base64-url(serverURL)/username". We base64-url
// encode the serverURL, because under the hood pass uses files and folders, so
// /s will get translated into additional folders.
package pass

import (
	"encoding/base64"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"

	"github.com/docker/docker-credential-helpers/credentials"
)

const PASS_FOLDER = "docker-credential-helpers"

var (
	PassInitialized bool
)

func init() {
	PassInitialized = exec.Command("pass").Run() == nil
}

func runPass(stdinContent string, args ...string) (string, error) {
	cmd := exec.Command("pass", args...)

	stdin, err := cmd.StdinPipe()
	if err != nil {
		return "", err
	}
	defer stdin.Close()

	stderr, err := cmd.StderrPipe()
	if err != nil {
		return "", err
	}
	defer stderr.Close()

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return "", err
	}
	defer stdout.Close()

	err = cmd.Start()
	if err != nil {
		return "", err
	}

	_, err = stdin.Write([]byte(stdinContent))
	if err != nil {
		return "", err
	}
	stdin.Close()

	errContent, err := ioutil.ReadAll(stderr)
	if err != nil {
		return "", fmt.Errorf("error reading stderr: %s", err)
	}

	result, err := ioutil.ReadAll(stdout)
	if err != nil {
		return "", fmt.Errorf("Error reading stdout: %s", err)
	}

	cmdErr := cmd.Wait()
	if cmdErr != nil {
		return "", fmt.Errorf("%s: %s", cmdErr, errContent)
	}

	return string(result), nil
}

// Pass handles secrets using Linux secret-service as a store.
type Pass struct{}

// Add adds new credentials to the keychain.
func (h Pass) Add(creds *credentials.Credentials) error {
	if !PassInitialized {
		return errors.New("pass store is uninitialized")
	}

	if creds == nil {
		return errors.New("missing credentials")
	}

	encoded := base64.URLEncoding.EncodeToString([]byte(creds.ServerURL))

	_, err := runPass(creds.Secret, "insert", "-f", "-m", path.Join(PASS_FOLDER, encoded, creds.Username))
	return err
}

// Delete removes credentials from the store.
func (h Pass) Delete(serverURL string) error {
	if !PassInitialized {
		return errors.New("pass store is uninitialized")
	}

	if serverURL == "" {
		return errors.New("missing server url")
	}

	encoded := base64.URLEncoding.EncodeToString([]byte(serverURL))
	_, err := runPass("", "rm", "-rf", path.Join(PASS_FOLDER, encoded))
	return err
}

// listPassDir lists all the contents of a directory in the password store.
// Pass uses fancy unicode to emit stuff to stdout, so rather than try
// and parse this, let's just look at the directory structure instead.
func listPassDir(args ...string) ([]os.FileInfo, error) {
	passDir := os.ExpandEnv("$HOME/.password-store")
	for _, e := range os.Environ() {
		parts := strings.SplitN(e, "=", 2)
		if len(parts) < 2 {
			continue
		}

		if parts[0] != "PASSWORD_STORE_DIR" {
			continue
		}

		passDir = parts[1]
		break
	}

	p := path.Join(append([]string{passDir, PASS_FOLDER}, args...)...)
	contents, err := ioutil.ReadDir(p)
	if err != nil {
		if os.IsNotExist(err) {
			return []os.FileInfo{}, nil
		}

		return nil, err
	}

	return contents, nil
}

// Get returns the username and secret to use for a given registry server URL.
func (h Pass) Get(serverURL string) (string, string, error) {
	if !PassInitialized {
		return "", "", errors.New("pass store is uninitialized")
	}

	if serverURL == "" {
		return "", "", errors.New("missing server url")
	}

	encoded := base64.URLEncoding.EncodeToString([]byte(serverURL))

	usernames, err := listPassDir(encoded)
	if err != nil {
		return "", "", err
	}

	if len(usernames) < 1 {
		return "", "", fmt.Errorf("no usernames for %s", serverURL)
	}

	actual := strings.TrimSuffix(usernames[0].Name(), ".gpg")
	secret, err := runPass("", "show", path.Join(PASS_FOLDER, encoded, actual))
	return actual, secret, err
}

// List returns the stored URLs and corresponding usernames for a given credentials label
func (h Pass) List() (map[string]string, error) {
	if !PassInitialized {
		return nil, errors.New("pass store is uninitialized")
	}

	servers, err := listPassDir()
	if err != nil {
		return nil, err
	}

	resp := map[string]string{}

	for _, server := range servers {
		if !server.IsDir() {
			continue
		}

		serverURL, err := base64.URLEncoding.DecodeString(server.Name())
		if err != nil {
			return nil, err
		}

		usernames, err := listPassDir(server.Name())
		if err != nil {
			return nil, err
		}

		if len(usernames) < 1 {
			return nil, fmt.Errorf("no usernames for %s", serverURL)
		}

		resp[string(serverURL)] = strings.TrimSuffix(usernames[0].Name(), ".gpg")
	}

	return resp, nil
}
<|endoftext|>


<commit_before>package pathhelper_test

import (
	"fmt"
	"github.com/northbright/pathhelper"
	"os"
)

func ExampleGetCurrentExecDir() {
	dir, err := pathhelper.GetCurrentExecDir()
	if err != nil {
		return
	}

	// temp test folder will be changed each time, output to os.Stderr.
	// Ex: /tmp/go-build662839082/github.com/northbright/pathhelper/_test
	fmt.Fprintf(os.Stderr, "current dir: %v\n", dir)
	// Output:
}

func ExampleGetAbsPath() {
	// Relative path will be joined to a absolute path with current test executable in under /tmp.
	// Ex: /tmp/go-build662839082/github.com/northbright/pathhelper/_test/config.json
	pathArr := []string{"/", "/var/log/boot.log", "config.json", "./example/test.go"}

	for _, v := range pathArr {
		if p, err := pathhelper.GetAbsPath(v); err != nil {
			fmt.Fprintf(os.Stderr, "GetAbsPath(%v) error: %v\n", v, err)
		} else {
			fmt.Fprintf(os.Stderr, "%v\n", p)
		}
	}
	// Output:
}

func ExampleGetAbsPaths() {
	var err error
	// Init absolute path map:
	// Join all relative paths with current executable dir.
	dirMap := map[string]string{
		"uploadDir": "./uploads",
		"photoDir":  "/tmp/photos",
	}

	absDirMap := map[string]string{}

	if absDirMap, err = pathhelper.GetAbsPaths(dirMap); err != nil {
		fmt.Fprintf(os.Stderr, "GetAbsPaths() error: %v\n", err)
	}

	fmt.Fprintf(os.Stderr, "GetAbsPaths():\n")
	for k, v := range absDirMap {
		fmt.Fprintf(os.Stderr, "k: %v, v: %v\n", k, v)
	}
	// Output:
}

// Use "go test -c && ./pathhelper.test" to test.
func ExampleCreateDirs() {
	m := map[string]string{
		"uploadDir": "./uploads",
		"photoDir":  "/tmp/photos",
	}

	// Create absolute dirs with 0755 permission bits.
	if err := pathhelper.CreateDirs(m, 0755); err != nil {
		fmt.Fprintf(os.Stderr, "CreateDirs() error: %v\n", err)
	}

	fmt.Fprintf(os.Stderr, "CreateDirs() successfully.\n")
	for k, v := range m {
		fmt.Fprintf(os.Stderr, "k: %v, v: %v\n", k, v)
	}
	// Output:
}

func ExampleGetFileNameWithoutExt() {
	p := "/a/b/c.apk"
	f := pathhelper.GetFileNameWithoutExt(p)
	fmt.Println(f)
	// Output:
	// c
}

func ExamplePathFileExist() {
	arr := []string{"/usr/", "~/go", "./", "~/xxx"}

	for _, v := range arr {
		b := pathhelper.PathFileExist(v)
		fmt.Printf("%v: %v\n", v, b)
	}

	// Output:
	// /usr/: true
	// ~/go: false
	// ./: true
	// ~/xxx: false
}
<commit_msg>Add 'go test -c && ./pathhelper.test' comments<commit_after>package pathhelper_test

import (
	"fmt"
	"github.com/northbright/pathhelper"
	"os"
)

// 1. Run "go test -c && ./pathhelper.test" to generate test binary under root of project dir and start test.
// 2. Run "go test" will generate test binary under "/tmp".
//    Temp test folder will be changed each time, output to os.Stderr.
//    Ex: /tmp/go-build662839082/github.com/northbright/pathhelper/_test
func ExampleGetCurrentExecDir() {
	dir, err := pathhelper.GetCurrentExecDir()
	if err != nil {
		return
	}

	fmt.Fprintf(os.Stderr, "current dir: %v\n", dir)
	// Output:
}

// 1. Run "go test -c && ./pathhelper.test" to generate test binary under root of project dir and start test.
// 2. Run "go test" will generate test binary under "/tmp".
//    Relative path will be joined to a absolute path with current test executable in under "/tmp".
//    Ex: /tmp/go-build662839082/github.com/northbright/pathhelper/_test/config.json
func ExampleGetAbsPath() {
	pathArr := []string{"/", "/var/log/boot.log", "config.json", "./example/test.go"}

	for _, v := range pathArr {
		if p, err := pathhelper.GetAbsPath(v); err != nil {
			fmt.Fprintf(os.Stderr, "GetAbsPath(%v) error: %v\n", v, err)
		} else {
			fmt.Fprintf(os.Stderr, "%v\n", p)
		}
	}
	// Output:
}

func ExampleGetAbsPaths() {
	var err error
	// Init absolute path map:
	// Join all relative paths with current executable dir.
	dirMap := map[string]string{
		"uploadDir": "./uploads",
		"photoDir":  "/tmp/photos",
	}

	absDirMap := map[string]string{}

	if absDirMap, err = pathhelper.GetAbsPaths(dirMap); err != nil {
		fmt.Fprintf(os.Stderr, "GetAbsPaths() error: %v\n", err)
	}

	fmt.Fprintf(os.Stderr, "GetAbsPaths():\n")
	for k, v := range absDirMap {
		fmt.Fprintf(os.Stderr, "k: %v, v: %v\n", k, v)
	}
	// Output:
}

// Use "go test -c && ./pathhelper.test" to test.
func ExampleCreateDirs() {
	m := map[string]string{
		"uploadDir": "./uploads",
		"photoDir":  "/tmp/photos",
	}

	// Create absolute dirs with 0755 permission bits.
	if err := pathhelper.CreateDirs(m, 0755); err != nil {
		fmt.Fprintf(os.Stderr, "CreateDirs() error: %v\n", err)
	}

	fmt.Fprintf(os.Stderr, "CreateDirs() successfully.\n")
	for k, v := range m {
		fmt.Fprintf(os.Stderr, "k: %v, v: %v\n", k, v)
	}
	// Output:
}

func ExampleGetFileNameWithoutExt() {
	p := "/a/b/c.apk"
	f := pathhelper.GetFileNameWithoutExt(p)
	fmt.Println(f)
	// Output:
	// c
}

func ExamplePathFileExist() {
	arr := []string{"/usr/", "~/go", "./", "~/xxx"}

	for _, v := range arr {
		b := pathhelper.PathFileExist(v)
		fmt.Printf("%v: %v\n", v, b)
	}

	// Output:
	// /usr/: true
	// ~/go: false
	// ./: true
	// ~/xxx: false
}
<|endoftext|>


<commit_before>package coordinator

import (
	"code.google.com/p/go.crypto/bcrypt"
	"regexp"
)

type Matcher struct {
	isRegex bool
	name    string
}

func (self *Matcher) Matches(name string) bool {
	if self.isRegex {
		matches, _ := regexp.MatchString(self.name, name)
		return matches
	}
	return self.name == name
}

type CommonUser struct {
	Name          string `json:"name"`
	Hash          string `json:"hash"`
	IsUserDeleted bool   `json:"is_deleted"`
}

func (self *CommonUser) GetName() string {
	return self.Name
}

func (self *CommonUser) IsDeleted() bool {
	return self.IsUserDeleted
}

func (self *CommonUser) changePassword(password string) error {
	hash, err := hashPassword(password)
	if err != nil {
		return err
	}
	self.Hash = string(hash)
	return nil
}

func (self *CommonUser) isValidPwd(password string) bool {
	return bcrypt.CompareHashAndPassword([]byte(self.Hash), []byte(password)) == nil
}

func (self *CommonUser) IsClusterAdmin() bool {
	return false
}

func (self *CommonUser) IsDbAdmin(db string) bool {
	return false
}

func (self *CommonUser) GetDb() string {
	return ""
}

func (self *CommonUser) HasWriteAccess(name string) bool {
	return false
}

func (self *CommonUser) HasReadAccess(name string) bool {
	return false
}

type clusterAdmin struct {
	CommonUser `josn:"common"`
}

func (self *clusterAdmin) IsClusterAdmin() bool {
	return true
}

type dbUser struct {
	CommonUser `json:"common"`
	Db         string     `json:"db"`
	WriteTo    []*Matcher `json:"write_matchers"`
	ReadFrom   []*Matcher `json:"read_matchers"`
	IsAdmin    bool       `json:"is_admin"`
}

func (self *dbUser) IsDbAdmin(db string) bool {
	return self.IsAdmin && self.Db == db
}

func (self *dbUser) HasWriteAccess(name string) bool {
	for _, matcher := range self.WriteTo {
		if matcher.Matches(name) {
			return true
		}
	}

	return false
}

func (self *dbUser) HasReadAccess(name string) bool {
	for _, matcher := range self.ReadFrom {
		if matcher.Matches(name) {
			return true
		}
	}

	return false
}

func (self *dbUser) GetDb() string {
	return self.Db
}

// private funcs

func hashPassword(password string) ([]byte, error) {
	// The second arg is the cost of the hashing, higher is slower but makes it harder
	// to brute force, since it will be really slow and impractical
	return bcrypt.GenerateFromPassword([]byte(password), 10)
}
<commit_msg>fix the matcher serialization.<commit_after>package coordinator

import (
	"code.google.com/p/go.crypto/bcrypt"
	"regexp"
)

type Matcher struct {
	IsRegex bool
	Name    string
}

func (self *Matcher) Matches(name string) bool {
	if self.IsRegex {
		matches, _ := regexp.MatchString(self.Name, name)
		return matches
	}
	return self.Name == name
}

type CommonUser struct {
	Name          string `json:"name"`
	Hash          string `json:"hash"`
	IsUserDeleted bool   `json:"is_deleted"`
}

func (self *CommonUser) GetName() string {
	return self.Name
}

func (self *CommonUser) IsDeleted() bool {
	return self.IsUserDeleted
}

func (self *CommonUser) changePassword(password string) error {
	hash, err := hashPassword(password)
	if err != nil {
		return err
	}
	self.Hash = string(hash)
	return nil
}

func (self *CommonUser) isValidPwd(password string) bool {
	return bcrypt.CompareHashAndPassword([]byte(self.Hash), []byte(password)) == nil
}

func (self *CommonUser) IsClusterAdmin() bool {
	return false
}

func (self *CommonUser) IsDbAdmin(db string) bool {
	return false
}

func (self *CommonUser) GetDb() string {
	return ""
}

func (self *CommonUser) HasWriteAccess(name string) bool {
	return false
}

func (self *CommonUser) HasReadAccess(name string) bool {
	return false
}

type clusterAdmin struct {
	CommonUser `josn:"common"`
}

func (self *clusterAdmin) IsClusterAdmin() bool {
	return true
}

type dbUser struct {
	CommonUser `json:"common"`
	Db         string     `json:"db"`
	WriteTo    []*Matcher `json:"write_matchers"`
	ReadFrom   []*Matcher `json:"read_matchers"`
	IsAdmin    bool       `json:"is_admin"`
}

func (self *dbUser) IsDbAdmin(db string) bool {
	return self.IsAdmin && self.Db == db
}

func (self *dbUser) HasWriteAccess(name string) bool {
	for _, matcher := range self.WriteTo {
		if matcher.Matches(name) {
			return true
		}
	}

	return false
}

func (self *dbUser) HasReadAccess(name string) bool {
	for _, matcher := range self.ReadFrom {
		if matcher.Matches(name) {
			return true
		}
	}

	return false
}

func (self *dbUser) GetDb() string {
	return self.Db
}

// private funcs

func hashPassword(password string) ([]byte, error) {
	// The second arg is the cost of the hashing, higher is slower but makes it harder
	// to brute force, since it will be really slow and impractical
	return bcrypt.GenerateFromPassword([]byte(password), 10)
}
<|endoftext|>


<commit_before>package api_availability

import (
	"fmt"
	"net/url"
	"os"
	"regexp"
	"time"

	"platform/availability/helpers"
	"platform/availability/monitor"

	"github.com/cloudfoundry-community/go-cfclient"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	maxConcourseConnectionFailures = 5
	maxWarnings                    = 5
	numWorkers                     = 4
	taskRatePerSecond              = 2
)

func lg(things ...interface{}) {
	fmt.Fprintln(os.Stdout, things...)
}

var warningMatchers = []*regexp.Regexp{
	regexp.MustCompile("cannot fetch token: 503 Service Unavailable"),
	regexp.MustCompile(`error \(200002\): CF-StatsUnavailable`),
}

var _ = Describe("API Availability Monitoring", func() {

	It("should have uninterupted access to cloudfoundry api during deploy", func() {
		cfConfig := &cfclient.Config{
			ApiAddress:        fmt.Sprintf("https://api.%s", helpers.MustGetenv("SYSTEM_DNS_ZONE_NAME")),
			Username:          helpers.MustGetenv("CF_USER"),
			Password:          helpers.MustGetenv("CF_PASS"),
			SkipSslValidation: helpers.MustGetenv("SKIP_SSL_VALIDATION") == "true",
		}
		monitor := monitor.NewMonitor(cfConfig, os.Stdout, numWorkers, warningMatchers, taskRatePerSecond)
		deployment := helpers.ConcourseDeployment()

		monitor.Add("Listing all apps in a space", func(cfg *cfclient.Config) error {
			cf, err := cfclient.NewClient(cfg)
			if err != nil {
				return fmt.Errorf("Failed to connect to Cloud Foundry API: %s", err)
			}
			org, err := cf.GetOrgByName("admin")
			if err != nil {
				return fmt.Errorf("Failed to fetch 'admin' org: %s", err)
			}

			space, err := cf.GetSpaceByName("healthchecks", org.Guid)
			if err != nil {
				return fmt.Errorf("Failed to fetch 'healthchecks' space within 'admin' org: %s", err)
			}
			apps, err := cf.ListAppsByQuery(url.Values{"q": []string{
				"organization_guid:" + org.Guid,
				"space_guid:" + space.Guid,
			}})
			if err != nil {
				return fmt.Errorf("Failed to query apps within space 'healthchecks' in org 'admin': %s", err)
			} else if len(apps) < 1 {
				return fmt.Errorf("Failed to find any apps in the 'healthchecks' space, expected at least one to be returned")
			}

			return nil
		})

		monitor.Add("Fetching detailed app information", func(cfg *cfclient.Config) error {
			cf, err := cfclient.NewClient(cfg)
			if err != nil {
				return fmt.Errorf("Failed to connect to Cloud Foundry API: %s", err)
			}
			org, err := cf.GetOrgByName("admin")
			if err != nil {
				return fmt.Errorf("Failed to fetch 'admin' org")
			}

			apps, err := cf.ListAppsByQuery(url.Values{"q": []string{
				"name:" + appName,
				"organization_guid:" + org.Guid,
			}})
			if err != nil {
				return fmt.Errorf("Failed to query app by name within 'admin' org: %s", err)
			} else if len(apps) == 0 {
				return fmt.Errorf("Failed to find the app named '%s' within 'admin' org", appName)
			}
			app := apps[0]

			if _, err := cf.GetAppStats(app.Guid); err != nil {
				return fmt.Errorf("Failed to fetch app stats: %s", err)
			}

			if _, err := cf.GetAppInstances(app.Guid); err != nil {
				return fmt.Errorf("Failed to fetch app instances: %s", err)
			}

			if _, err := cf.GetAppRoutes(app.Guid); err != nil {
				return fmt.Errorf("Failed to fetch app routes: %s", err)
			}

			return nil
		})

		// poll concourse job status til done
		go func(concourseConnectionAttemptsRemaining int64) {
			defer GinkgoRecover()
			for {
				<-time.After(2 * time.Second)
				if done, err := deployment.Complete(); err != nil {
					concourseConnectionAttemptsRemaining--
					if concourseConnectionAttemptsRemaining <= 0 {
						monitor.Stop()
						Expect(err).ToNot(HaveOccurred())
						return
					}
					lg("failed to get status from concourse [", concourseConnectionAttemptsRemaining, " attempts remaining]", err)
				} else if done {
					concourseConnectionAttemptsRemaining = maxConcourseConnectionFailures
					lg("detected deployment job completed, stopping monitor")
					monitor.Stop()
					return
				}
			}
		}(maxConcourseConnectionFailures)

		report := monitor.Run()
		lg(report.String())
		Expect(report.Errors).To(BeEmpty(), "expected no errors")
		Expect(report.SuccessCount).To(BeNumerically(">", int64(0)), "expected at least one success")
		Expect(report.FailureCount).To(Equal(int64(0)), "expected 0 failures")
		Expect(report.WarningCount).To(BeNumerically("<=", int64(maxWarnings)), "expected at most %d warnings", maxWarnings)
	})
})
<commit_msg>Disable HTTP Keepalive in API availability tests<commit_after>package api_availability

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"time"

	"platform/availability/helpers"
	"platform/availability/monitor"

	"github.com/cloudfoundry-community/go-cfclient"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	maxConcourseConnectionFailures = 5
	maxWarnings                    = 5
	numWorkers                     = 4
	taskRatePerSecond              = 2
)

func lg(things ...interface{}) {
	fmt.Fprintln(os.Stdout, things...)
}

var warningMatchers = []*regexp.Regexp{
	regexp.MustCompile("cannot fetch token: 503 Service Unavailable"),
	regexp.MustCompile(`error \(200002\): CF-StatsUnavailable`),
}

var _ = Describe("API Availability Monitoring", func() {

	It("should have uninterupted access to cloudfoundry api during deploy", func() {
		cfConfig := &cfclient.Config{
			ApiAddress:        fmt.Sprintf("https://api.%s", helpers.MustGetenv("SYSTEM_DNS_ZONE_NAME")),
			Username:          helpers.MustGetenv("CF_USER"),
			Password:          helpers.MustGetenv("CF_PASS"),
			SkipSslValidation: helpers.MustGetenv("SKIP_SSL_VALIDATION") == "true",
			HttpClient: &http.Client{
				Transport: &http.Transport{
					DisableKeepAlives: true,
				},
			},
		}
		monitor := monitor.NewMonitor(cfConfig, os.Stdout, numWorkers, warningMatchers, taskRatePerSecond)
		deployment := helpers.ConcourseDeployment()

		monitor.Add("Listing all apps in a space", func(cfg *cfclient.Config) error {
			cf, err := cfclient.NewClient(cfg)
			if err != nil {
				return fmt.Errorf("Failed to connect to Cloud Foundry API: %s", err)
			}
			org, err := cf.GetOrgByName("admin")
			if err != nil {
				return fmt.Errorf("Failed to fetch 'admin' org: %s", err)
			}

			space, err := cf.GetSpaceByName("healthchecks", org.Guid)
			if err != nil {
				return fmt.Errorf("Failed to fetch 'healthchecks' space within 'admin' org: %s", err)
			}
			apps, err := cf.ListAppsByQuery(url.Values{"q": []string{
				"organization_guid:" + org.Guid,
				"space_guid:" + space.Guid,
			}})
			if err != nil {
				return fmt.Errorf("Failed to query apps within space 'healthchecks' in org 'admin': %s", err)
			} else if len(apps) < 1 {
				return fmt.Errorf("Failed to find any apps in the 'healthchecks' space, expected at least one to be returned")
			}

			return nil
		})

		monitor.Add("Fetching detailed app information", func(cfg *cfclient.Config) error {
			cf, err := cfclient.NewClient(cfg)
			if err != nil {
				return fmt.Errorf("Failed to connect to Cloud Foundry API: %s", err)
			}
			org, err := cf.GetOrgByName("admin")
			if err != nil {
				return fmt.Errorf("Failed to fetch 'admin' org")
			}

			apps, err := cf.ListAppsByQuery(url.Values{"q": []string{
				"name:" + appName,
				"organization_guid:" + org.Guid,
			}})
			if err != nil {
				return fmt.Errorf("Failed to query app by name within 'admin' org: %s", err)
			} else if len(apps) == 0 {
				return fmt.Errorf("Failed to find the app named '%s' within 'admin' org", appName)
			}
			app := apps[0]

			if _, err := cf.GetAppStats(app.Guid); err != nil {
				return fmt.Errorf("Failed to fetch app stats: %s", err)
			}

			if _, err := cf.GetAppInstances(app.Guid); err != nil {
				return fmt.Errorf("Failed to fetch app instances: %s", err)
			}

			if _, err := cf.GetAppRoutes(app.Guid); err != nil {
				return fmt.Errorf("Failed to fetch app routes: %s", err)
			}

			return nil
		})

		// poll concourse job status til done
		go func(concourseConnectionAttemptsRemaining int64) {
			defer GinkgoRecover()
			for {
				<-time.After(2 * time.Second)
				if done, err := deployment.Complete(); err != nil {
					concourseConnectionAttemptsRemaining--
					if concourseConnectionAttemptsRemaining <= 0 {
						monitor.Stop()
						Expect(err).ToNot(HaveOccurred())
						return
					}
					lg("failed to get status from concourse [", concourseConnectionAttemptsRemaining, " attempts remaining]", err)
				} else if done {
					concourseConnectionAttemptsRemaining = maxConcourseConnectionFailures
					lg("detected deployment job completed, stopping monitor")
					monitor.Stop()
					return
				}
			}
		}(maxConcourseConnectionFailures)

		report := monitor.Run()
		lg(report.String())
		Expect(report.Errors).To(BeEmpty(), "expected no errors")
		Expect(report.SuccessCount).To(BeNumerically(">", int64(0)), "expected at least one success")
		Expect(report.FailureCount).To(Equal(int64(0)), "expected 0 failures")
		Expect(report.WarningCount).To(BeNumerically("<=", int64(maxWarnings)), "expected at most %d warnings", maxWarnings)
	})
})
<|endoftext|>


<commit_before>package api

import (
	"net/http"

	"github.com/hellofresh/janus/pkg/errors"
	"github.com/hellofresh/janus/pkg/middleware"
	"github.com/hellofresh/janus/pkg/request"
	"github.com/hellofresh/janus/pkg/response"
	"github.com/hellofresh/janus/pkg/router"
	"gopkg.in/mgo.v2"
)

// Controller is the api rest controller
type Controller struct {
	loader *Loader
}

// NewController creates a new instance of Controller
func NewController(loader *Loader) *Controller {
	return &Controller{loader}
}

func (u *Controller) Get() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		repo := u.getRepository(u.getDatabase(r))

		data, err := repo.FindAll()
		if err != nil {
			panic(err.Error())
		}

		response.JSON(w, http.StatusOK, data)
	}
}

func (u *Controller) GetBy() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		id := router.FromContext(r.Context()).ByName("id")
		repo := u.getRepository(u.getDatabase(r))

		data, err := repo.FindByID(id)
		if data.ID == "" {
			panic(ErrAPIDefinitionNotFound)
		}

		if err != nil {
			panic(errors.New(http.StatusInternalServerError, err.Error()))
		}

		response.JSON(w, http.StatusOK, data)
	}
}

func (u *Controller) PutBy() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		var err error

		id := router.FromContext(r.Context()).ByName("id")
		repo := u.getRepository(u.getDatabase(r))
		definition, err := repo.FindByID(id)
		if definition.ID == "" {
			panic(ErrAPIDefinitionNotFound)
		}

		if err != nil {
			panic(errors.New(http.StatusInternalServerError, err.Error()))
		}

		err = request.BindJSON(r, definition)
		if nil != err {
			panic(errors.New(http.StatusInternalServerError, err.Error()))
		}

		err = repo.Add(definition)
		if nil != err {
			panic(errors.New(http.StatusBadRequest, err.Error()))
		}

		u.loader.Load()

		response.JSON(w, http.StatusOK, nil)
	}
}

func (u *Controller) Post() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		repo := u.getRepository(u.getDatabase(r))
		definition := NewDefinition()

		err := request.BindJSON(r, definition)
		if nil != err {
			panic(errors.New(http.StatusInternalServerError, err.Error()))
		}

		def, err := repo.FindByListenPath(definition.Proxy.ListenPath)
		if nil != err && err != mgo.ErrNotFound {
			panic(errors.New(http.StatusBadRequest, err.Error()))
		}

		if def != nil {
			panic(errors.ErrProxyExists)
		}

		err = repo.Add(definition)
		if nil != err {

			panic(errors.New(http.StatusBadRequest, err.Error()))
		}

		u.loader.Load()
		response.JSON(w, http.StatusCreated, nil)
	}
}

func (u *Controller) DeleteBy() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		id := router.FromContext(r.Context()).ByName("id")
		repo := u.getRepository(u.getDatabase(r))

		err := repo.Remove(id)
		if err != nil {
			panic(errors.New(http.StatusInternalServerError, err.Error()))
		}

		u.loader.Load()
		response.JSON(w, http.StatusNoContent, nil)
	}
}

func (u *Controller) getDatabase(r *http.Request) *mgo.Database {
	db := r.Context().Value(middleware.ContextKeyDatabase)

	if nil == db {
		panic(ErrDBContextNotSet)
	}

	return db.(*mgo.Database)
}

// GetRepository gets the repository for the handlers
func (u *Controller) getRepository(db *mgo.Database) *MongoAPISpecRepository {
	repo, err := NewMongoAppRepository(db)
	if err != nil {
		panic(errors.New(http.StatusInternalServerError, err.Error()))
	}

	return repo
}
<commit_msg>Removed line<commit_after>package api

import (
	"net/http"

	"github.com/hellofresh/janus/pkg/errors"
	"github.com/hellofresh/janus/pkg/middleware"
	"github.com/hellofresh/janus/pkg/request"
	"github.com/hellofresh/janus/pkg/response"
	"github.com/hellofresh/janus/pkg/router"
	"gopkg.in/mgo.v2"
)

// Controller is the api rest controller
type Controller struct {
	loader *Loader
}

// NewController creates a new instance of Controller
func NewController(loader *Loader) *Controller {
	return &Controller{loader}
}

func (u *Controller) Get() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		repo := u.getRepository(u.getDatabase(r))

		data, err := repo.FindAll()
		if err != nil {
			panic(err.Error())
		}

		response.JSON(w, http.StatusOK, data)
	}
}

func (u *Controller) GetBy() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		id := router.FromContext(r.Context()).ByName("id")
		repo := u.getRepository(u.getDatabase(r))

		data, err := repo.FindByID(id)
		if data.ID == "" {
			panic(ErrAPIDefinitionNotFound)
		}

		if err != nil {
			panic(errors.New(http.StatusInternalServerError, err.Error()))
		}

		response.JSON(w, http.StatusOK, data)
	}
}

func (u *Controller) PutBy() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		var err error

		id := router.FromContext(r.Context()).ByName("id")
		repo := u.getRepository(u.getDatabase(r))
		definition, err := repo.FindByID(id)
		if definition.ID == "" {
			panic(ErrAPIDefinitionNotFound)
		}

		if err != nil {
			panic(errors.New(http.StatusInternalServerError, err.Error()))
		}

		err = request.BindJSON(r, definition)
		if nil != err {
			panic(errors.New(http.StatusInternalServerError, err.Error()))
		}

		err = repo.Add(definition)
		if nil != err {
			panic(errors.New(http.StatusBadRequest, err.Error()))
		}

		u.loader.Load()

		response.JSON(w, http.StatusOK, nil)
	}
}

func (u *Controller) Post() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		repo := u.getRepository(u.getDatabase(r))
		definition := NewDefinition()

		err := request.BindJSON(r, definition)
		if nil != err {
			panic(errors.New(http.StatusInternalServerError, err.Error()))
		}

		def, err := repo.FindByListenPath(definition.Proxy.ListenPath)
		if nil != err && err != mgo.ErrNotFound {
			panic(errors.New(http.StatusBadRequest, err.Error()))
		}

		if def != nil {
			panic(errors.ErrProxyExists)
		}

		err = repo.Add(definition)
		if nil != err {
			panic(errors.New(http.StatusBadRequest, err.Error()))
		}

		u.loader.Load()
		response.JSON(w, http.StatusCreated, nil)
	}
}

func (u *Controller) DeleteBy() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		id := router.FromContext(r.Context()).ByName("id")
		repo := u.getRepository(u.getDatabase(r))

		err := repo.Remove(id)
		if err != nil {
			panic(errors.New(http.StatusInternalServerError, err.Error()))
		}

		u.loader.Load()
		response.JSON(w, http.StatusNoContent, nil)
	}
}

func (u *Controller) getDatabase(r *http.Request) *mgo.Database {
	db := r.Context().Value(middleware.ContextKeyDatabase)

	if nil == db {
		panic(ErrDBContextNotSet)
	}

	return db.(*mgo.Database)
}

// GetRepository gets the repository for the handlers
func (u *Controller) getRepository(db *mgo.Database) *MongoAPISpecRepository {
	repo, err := NewMongoAppRepository(db)
	if err != nil {
		panic(errors.New(http.StatusInternalServerError, err.Error()))
	}

	return repo
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2018 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package certs

import (
	"crypto/tls"
	"path/filepath"
	"sync"

	"github.com/rjeczalik/notify"
)

// A Certs represents a certificate manager able to watch certificate
// and key pairs for changes.
type Certs struct {
	sync.RWMutex
	// user input params.
	certFile string
	keyFile  string
	loadCert LoadX509KeyPairFunc

	// points to the latest certificate.
	cert tls.Certificate

	// internal param to track for events, also
	// used to close the watcher.
	e chan notify.EventInfo
}

// LoadX509KeyPairFunc - provides a type for custom cert loader function.
type LoadX509KeyPairFunc func(certFile, keyFile string) (tls.Certificate, error)

// New initializes a new certs monitor.
func New(certFile, keyFile string, loadCert LoadX509KeyPairFunc) (*Certs, error) {
	c := &Certs{
		certFile: certFile,
		keyFile:  keyFile,
		loadCert: loadCert,
		// Make the channel buffered to ensure no event is dropped. Notify will drop
		// an event if the receiver is not able to keep up the sending pace.
		e: make(chan notify.EventInfo, 1),
	}

	if err := c.watch(); err != nil {
		return nil, err
	}

	return c, nil
}

// watch starts watching for changes to the certificate
// and key files. On any change the certificate and key
// are reloaded. If there is an issue the loading will fail
// and the old (if any) certificates and keys will continue
// to be used.
func (c *Certs) watch() (err error) {
	defer func() {
		if err != nil {
			// Stop any watches previously setup after an error.
			notify.Stop(c.e)
		}
	}()

	// Windows doesn't allow for watching file changes but instead allows
	// for directory changes only, while we can still watch for changes
	// on files on other platforms. Watch parent directory on all platforms
	// for simplicity.
	if err = notify.Watch(filepath.Dir(c.certFile), c.e, eventWrite...); err != nil {
		return err
	}
	if err = notify.Watch(filepath.Dir(c.keyFile), c.e, eventWrite...); err != nil {
		return err
	}
	c.Lock()
	c.cert, err = c.loadCert(c.certFile, c.keyFile)
	c.Unlock()
	if err != nil {
		return err
	}
	go c.run()
	return nil
}

func (c *Certs) run() {
	for event := range c.e {
		base := filepath.Base(event.Path())
		if isWriteEvent(event.Event()) {
			certChanged := base == filepath.Base(c.certFile)
			keyChanged := base == filepath.Base(c.keyFile)
			if certChanged || keyChanged {
				cert, err := c.loadCert(c.certFile, c.keyFile)
				if err != nil {
					// ignore the error continue to use
					// old certificates.
					continue
				}
				c.Lock()
				c.cert = cert
				c.Unlock()
			}
		}
	}
}

// GetCertificateFunc provides a GetCertificate type for custom client implementations.
type GetCertificateFunc func(hello *tls.ClientHelloInfo) (*tls.Certificate, error)

// GetCertificate returns the loaded certificate for use by
// the TLSConfig fields GetCertificate field in a http.Server.
func (c *Certs) GetCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
	c.RLock()
	defer c.RUnlock()
	return &c.cert, nil
}

// Stop tells loader to stop watching for changes to the
// certificate and key files.
func (c *Certs) Stop() {
	if c != nil {
		notify.Stop(c.e)
	}
}
<commit_msg>Watch for symlinked certs in container envs (#6282)<commit_after>/*
 * Minio Cloud Storage, (C) 2018 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package certs

import (
	"crypto/tls"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/rjeczalik/notify"
)

// A Certs represents a certificate manager able to watch certificate
// and key pairs for changes.
type Certs struct {
	sync.RWMutex
	// user input params.
	certFile string
	keyFile  string
	loadCert LoadX509KeyPairFunc

	// points to the latest certificate.
	cert tls.Certificate

	// internal param to track for events, also
	// used to close the watcher.
	e chan notify.EventInfo
}

// LoadX509KeyPairFunc - provides a type for custom cert loader function.
type LoadX509KeyPairFunc func(certFile, keyFile string) (tls.Certificate, error)

// New initializes a new certs monitor.
func New(certFile, keyFile string, loadCert LoadX509KeyPairFunc) (*Certs, error) {
	certFileIsLink, err := checkSymlink(certFile)
	if err != nil {
		return nil, err
	}
	keyFileIsLink, err := checkSymlink(keyFile)
	if err != nil {
		return nil, err
	}
	c := &Certs{
		certFile: certFile,
		keyFile:  keyFile,
		loadCert: loadCert,
		// Make the channel buffered to ensure no event is dropped. Notify will drop
		// an event if the receiver is not able to keep up the sending pace.
		e: make(chan notify.EventInfo, 1),
	}

	if certFileIsLink && keyFileIsLink {
		if err := c.watchSymlinks(); err != nil {
			return nil, err
		}
	} else {
		if err := c.watch(); err != nil {
			return nil, err
		}
	}

	return c, nil
}

func checkSymlink(file string) (bool, error) {
	st, err := os.Lstat(file)
	if err != nil {
		return false, err
	}
	return st.Mode()&os.ModeSymlink == os.ModeSymlink, nil
}

// watchSymlinks reloads symlinked files since fsnotify cannot watch
// on symbolic links.
func (c *Certs) watchSymlinks() (err error) {
	c.Lock()
	c.cert, err = c.loadCert(c.certFile, c.keyFile)
	c.Unlock()
	if err != nil {
		return err
	}
	go func() {
		for {
			select {
			case <-c.e:
				// Once stopped exits this routine.
				return
			case <-time.After(24 * time.Hour):
				cert, cerr := c.loadCert(c.certFile, c.keyFile)
				if cerr != nil {
					continue
				}
				c.Lock()
				c.cert = cert
				c.Unlock()
			}
		}
	}()
	return nil
}

// watch starts watching for changes to the certificate
// and key files. On any change the certificate and key
// are reloaded. If there is an issue the loading will fail
// and the old (if any) certificates and keys will continue
// to be used.
func (c *Certs) watch() (err error) {
	defer func() {
		if err != nil {
			// Stop any watches previously setup after an error.
			notify.Stop(c.e)
		}
	}()

	// Windows doesn't allow for watching file changes but instead allows
	// for directory changes only, while we can still watch for changes
	// on files on other platforms. Watch parent directory on all platforms
	// for simplicity.
	if err = notify.Watch(filepath.Dir(c.certFile), c.e, eventWrite...); err != nil {
		return err
	}
	if err = notify.Watch(filepath.Dir(c.keyFile), c.e, eventWrite...); err != nil {
		return err
	}
	c.Lock()
	c.cert, err = c.loadCert(c.certFile, c.keyFile)
	c.Unlock()
	if err != nil {
		return err
	}
	go c.run()
	return nil
}

func (c *Certs) run() {
	for event := range c.e {
		base := filepath.Base(event.Path())
		if isWriteEvent(event.Event()) {
			certChanged := base == filepath.Base(c.certFile)
			keyChanged := base == filepath.Base(c.keyFile)
			if certChanged || keyChanged {
				cert, err := c.loadCert(c.certFile, c.keyFile)
				if err != nil {
					// ignore the error continue to use
					// old certificates.
					continue
				}
				c.Lock()
				c.cert = cert
				c.Unlock()
			}
		}
	}
}

// GetCertificateFunc provides a GetCertificate type for custom client implementations.
type GetCertificateFunc func(hello *tls.ClientHelloInfo) (*tls.Certificate, error)

// GetCertificate returns the loaded certificate for use by
// the TLSConfig fields GetCertificate field in a http.Server.
func (c *Certs) GetCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
	c.RLock()
	defer c.RUnlock()
	return &c.cert, nil
}

// Stop tells loader to stop watching for changes to the
// certificate and key files.
func (c *Certs) Stop() {
	if c != nil {
		notify.Stop(c.e)
	}
}
<|endoftext|>