<commit_before>package cache

import (
	"context"
	"errors"
	"sync/atomic"
	"time"

	"github.com/go-redis/cache/internal"
	"github.com/go-redis/cache/internal/lrucache"
	"github.com/go-redis/cache/internal/singleflight"

	"github.com/go-redis/redis"
)

var ErrCacheMiss = errors.New("cache: key is missing")
var errRedisLocalCacheNil = errors.New("cache: both Redis and LocalCache are nil")

func SetLogger(logger internal.Logger) {
	internal.Log = logger
}

type rediser interface {
	Set(key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	Get(key string) *redis.StringCmd
	Del(keys ...string) *redis.IntCmd
}

type Item struct {
	Ctx context.Context

	Key    string
	Object interface{}

	// Func returns object to be cached.
	Func func() (interface{}, error)

	// Expiration is the cache expiration time.
	// Default expiration is 1 hour.
	Expiration time.Duration
}

func (item *Item) object() (interface{}, error) {
	if item.Object != nil {
		return item.Object, nil
	}
	if item.Func != nil {
		return item.Func()
	}
	return nil, nil
}

func (item *Item) exp() time.Duration {
	if item.Expiration < 0 {
		return 0
	}
	if item.Expiration < time.Second {
		return time.Hour
	}
	return item.Expiration
}

type Codec struct {
	Redis rediser

	hooks      []Hook
	localCache *lrucache.Cache

	Marshal   func(interface{}) ([]byte, error)
	Unmarshal func([]byte, interface{}) error

	group singleflight.Group

	hits        uint64
	misses      uint64
	localHits   uint64
	localMisses uint64
}

// UseLocalCache causes Codec to cache items in local LRU cache.
func (cd *Codec) UseLocalCache(maxLen int, expiration time.Duration) {
	cd.localCache = lrucache.New(maxLen, expiration)
}

// Set caches the item.
func (cd *Codec) Set(item *Item) error {
	cd.beforeSet(item)
	_, err := cd.setItem(item)
	cd.afterSet(item)
	return err
}

func (cd *Codec) setItem(item *Item) ([]byte, error) {
	object, err := item.object()
	if err != nil {
		return nil, err
	}

	b, err := cd.Marshal(object)
	if err != nil {
		internal.Log.Printf("cache: Marshal key=%q failed: %s", item.Key, err)
		return nil, err
	}

	if cd.localCache != nil {
		cd.localCache.Set(item.Key, b)
	}

	if cd.Redis == nil {
		if cd.localCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return b, nil
	}

	err = cd.Redis.Set(item.Key, b, item.exp()).Err()
	if err != nil {
		internal.Log.Printf("cache: Set key=%q failed: %s", item.Key, err)
	}
	return b, err
}

// Exists reports whether object for the given key exists.
func (cd *Codec) Exists(key string) bool {
	return cd.Get(key, nil) == nil
}

// Get gets the object for the given key.
func (cd *Codec) Get(key string, object interface{}) error {
	return cd.get(nil, key, object)
}

func (cd *Codec) GetContext(c context.Context, key string, object interface{}) error {
	return cd.get(c, key, object)
}

func (cd *Codec) get(c context.Context, key string, object interface{}) error {
	cd.beforeGet(c, key, object)
	err := cd._get(key, object, false)
	cd.afterGet(c, key, object)
	return err
}

func (cd *Codec) _get(key string, object interface{}, onlyLocalCache bool) error {
	b, err := cd.getBytes(key, onlyLocalCache)
	if err != nil {
		return err
	}

	if object == nil || len(b) == 0 {
		return nil
	}

	err = cd.Unmarshal(b, object)
	if err != nil {
		internal.Log.Printf("cache: key=%q Unmarshal(%T) failed: %s", key, object, err)
		return err
	}

	return nil
}

func (cd *Codec) getBytes(key string, onlyLocalCache bool) ([]byte, error) {
	if cd.localCache != nil {
		b, ok := cd.localCache.Get(key)
		if ok {
			atomic.AddUint64(&cd.localHits, 1)
			return b, nil
		}
		atomic.AddUint64(&cd.localMisses, 1)
	}

	if onlyLocalCache {
		return nil, ErrCacheMiss
	}
	if cd.Redis == nil {
		if cd.localCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return nil, ErrCacheMiss
	}

	b, err := cd.Redis.Get(key).Bytes()
	if err != nil {
		atomic.AddUint64(&cd.misses, 1)
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		internal.Log.Printf("cache: Get key=%q failed: %s", key, err)
		return nil, err
	}
	atomic.AddUint64(&cd.hits, 1)

	if cd.localCache != nil {
		cd.localCache.Set(key, b)
	}
	return b, nil
}

// Once gets the item.Object for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Codec) Once(item *Item) error {
	cd.beforeOnce(item)
	err := cd.once(item)
	cd.afterOnce(item)
	return err
}

func (cd *Codec) once(item *Item) error {
	b, cached, err := cd.getSetItemBytesOnce(item)
	if err != nil {
		return err
	}

	if item.Object == nil || len(b) == 0 {
		return nil
	}

	err = cd.Unmarshal(b, item.Object)
	if err != nil {
		internal.Log.Printf("cache: key=%q Unmarshal(%T) failed: %s", item.Key, item.Object, err)
		if cached {
			_ = cd._delete(item.Key)
			return cd.once(item)
		}
		return err
	}

	return nil
}

func (cd *Codec) getSetItemBytesOnce(item *Item) (b []byte, cached bool, err error) {
	if cd.localCache != nil {
		b, err := cd.getItemBytesFast(item)
		if err == nil {
			return b, true, nil
		}
	}

	obj, err := cd.group.Do(item.Key, func() (interface{}, error) {
		b, err := cd.getItemBytes(item)
		if err == nil {
			cached = true
			return b, nil
		}

		obj, err := item.Func()
		if err != nil {
			return nil, err
		}

		b, err = cd.setItem(&Item{
			Key:        item.Key,
			Object:     obj,
			Expiration: item.Expiration,
		})
		if b != nil {
			// Ignore error if we have the result.
			return b, nil
		}
		return nil, err
	})
	if err != nil {
		return nil, false, err
	}
	return obj.([]byte), cached, nil
}

func (cd *Codec) getItemBytes(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, false)
}

func (cd *Codec) getItemBytesFast(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, true)
}

func (cd *Codec) Delete(key string) error {
	return cd.delete(nil, key)
}

func (cd *Codec) DeleteContext(c context.Context, key string) error {
	return cd.delete(c, key)
}

func (cd *Codec) delete(c context.Context, key string) error {
	cd.beforeDelete(c, key)
	err := cd._delete(key)
	cd.afterDelete(c, key)
	return err
}

func (cd *Codec) _delete(key string) error {
	if cd.localCache != nil {
		cd.localCache.Delete(key)
	}

	if cd.Redis == nil {
		if cd.localCache == nil {
			return errRedisLocalCacheNil
		}
		return nil
	}

	deleted, err := cd.Redis.Del(key).Result()
	if err != nil {
		internal.Log.Printf("cache: Del key=%q failed: %s", key, err)
		return err
	}
	if deleted == 0 {
		return ErrCacheMiss
	}
	return nil
}

type Hook interface {
	BeforeSet(item *Item)
	AfterSet(item *Item)

	BeforeGet(c context.Context, key string, object interface{})
	AfterGet(c context.Context, key string, object interface{})

	BeforeDelete(c context.Context, key string)
	AfterDelete(c context.Context, key string)

	BeforeOnce(item *Item)
	AfterOnce(item *Item)
}

func (cd *Codec) AddHook(h Hook) {
	cd.hooks = append(cd.hooks, h)
}

func (cd *Codec) beforeSet(item *Item) {
	for _, h := range cd.hooks {
		h.BeforeSet(item)
	}
}

func (cd *Codec) afterSet(item *Item) {
	for _, h := range cd.hooks {
		h.AfterSet(item)
	}
}

func (cd *Codec) beforeGet(c context.Context, key string, object interface{}) {
	for _, h := range cd.hooks {
		h.BeforeGet(c, key, object)
	}
}

func (cd *Codec) afterGet(c context.Context, key string, object interface{}) {
	for _, h := range cd.hooks {
		h.AfterGet(c, key, object)
	}
}

func (cd *Codec) beforeDelete(c context.Context, key string) {
	for _, h := range cd.hooks {
		h.BeforeDelete(c, key)
	}
}

func (cd *Codec) afterDelete(c context.Context, key string) {
	for _, h := range cd.hooks {
		h.AfterDelete(c, key)
	}
}

func (cd *Codec) beforeOnce(item *Item) {
	for _, h := range cd.hooks {
		h.BeforeOnce(item)
	}
}

func (cd *Codec) afterOnce(item *Item) {
	for _, h := range cd.hooks {
		h.AfterOnce(item)
	}
}

type Stats struct {
	Hits        uint64
	Misses      uint64
	LocalHits   uint64
	LocalMisses uint64
}

// Stats returns cache statistics.
func (cd *Codec) Stats() *Stats {
	stats := Stats{
		Hits:   atomic.LoadUint64(&cd.hits),
		Misses: atomic.LoadUint64(&cd.misses),
	}
	if cd.localCache != nil {
		stats.LocalHits = atomic.LoadUint64(&cd.localHits)
		stats.LocalMisses = atomic.LoadUint64(&cd.localMisses)
	}
	return &stats
}
<commit_msg>Tweak hook<commit_after>package cache

import (
	"context"
	"errors"
	"sync/atomic"
	"time"

	"github.com/go-redis/cache/internal"
	"github.com/go-redis/cache/internal/lrucache"
	"github.com/go-redis/cache/internal/singleflight"

	"github.com/go-redis/redis"
)

var ErrCacheMiss = errors.New("cache: key is missing")
var errRedisLocalCacheNil = errors.New("cache: both Redis and LocalCache are nil")

func SetLogger(logger internal.Logger) {
	internal.Log = logger
}

type rediser interface {
	Set(key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	Get(key string) *redis.StringCmd
	Del(keys ...string) *redis.IntCmd
}

type Item struct {
	Ctx context.Context

	Key    string
	Object interface{}

	// Func returns object to be cached.
	Func func() (interface{}, error)

	// Expiration is the cache expiration time.
	// Default expiration is 1 hour.
	Expiration time.Duration
}

func (item *Item) object() (interface{}, error) {
	if item.Object != nil {
		return item.Object, nil
	}
	if item.Func != nil {
		return item.Func()
	}
	return nil, nil
}

func (item *Item) exp() time.Duration {
	if item.Expiration < 0 {
		return 0
	}
	if item.Expiration < time.Second {
		return time.Hour
	}
	return item.Expiration
}

type Codec struct {
	Redis rediser

	hooks      []Hook
	localCache *lrucache.Cache

	Marshal   func(interface{}) ([]byte, error)
	Unmarshal func([]byte, interface{}) error

	group singleflight.Group

	hits        uint64
	misses      uint64
	localHits   uint64
	localMisses uint64
}

// UseLocalCache causes Codec to cache items in local LRU cache.
func (cd *Codec) UseLocalCache(maxLen int, expiration time.Duration) {
	cd.localCache = lrucache.New(maxLen, expiration)
}

// Set caches the item.
func (cd *Codec) Set(item *Item) error {
	if err := cd.beforeSet(item); err != nil {
		return err
	}

	_, err := cd.setItem(item)

	if err := cd.afterSet(item); err != nil {
		return err
	}
	return err
}

func (cd *Codec) setItem(item *Item) ([]byte, error) {
	object, err := item.object()
	if err != nil {
		return nil, err
	}

	b, err := cd.Marshal(object)
	if err != nil {
		internal.Log.Printf("cache: Marshal key=%q failed: %s", item.Key, err)
		return nil, err
	}

	if cd.localCache != nil {
		cd.localCache.Set(item.Key, b)
	}

	if cd.Redis == nil {
		if cd.localCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return b, nil
	}

	err = cd.Redis.Set(item.Key, b, item.exp()).Err()
	if err != nil {
		internal.Log.Printf("cache: Set key=%q failed: %s", item.Key, err)
	}
	return b, err
}

// Exists reports whether object for the given key exists.
func (cd *Codec) Exists(key string) bool {
	return cd.Get(key, nil) == nil
}

// Get gets the object for the given key.
func (cd *Codec) Get(key string, object interface{}) error {
	return cd.get(nil, key, object)
}

func (cd *Codec) GetContext(c context.Context, key string, object interface{}) error {
	return cd.get(c, key, object)
}

func (cd *Codec) get(c context.Context, key string, object interface{}) error {
	c, err := cd.beforeGet(c, key, object)
	if err != nil {
		return err
	}

	err = cd._get(key, object, false)

	if _, err := cd.afterGet(c, key, object); err != nil {
		return err
	}

	return err
}

func (cd *Codec) _get(key string, object interface{}, onlyLocalCache bool) error {
	b, err := cd.getBytes(key, onlyLocalCache)
	if err != nil {
		return err
	}

	if object == nil || len(b) == 0 {
		return nil
	}

	err = cd.Unmarshal(b, object)
	if err != nil {
		internal.Log.Printf("cache: key=%q Unmarshal(%T) failed: %s", key, object, err)
		return err
	}

	return nil
}

func (cd *Codec) getBytes(key string, onlyLocalCache bool) ([]byte, error) {
	if cd.localCache != nil {
		b, ok := cd.localCache.Get(key)
		if ok {
			atomic.AddUint64(&cd.localHits, 1)
			return b, nil
		}
		atomic.AddUint64(&cd.localMisses, 1)
	}

	if onlyLocalCache {
		return nil, ErrCacheMiss
	}
	if cd.Redis == nil {
		if cd.localCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return nil, ErrCacheMiss
	}

	b, err := cd.Redis.Get(key).Bytes()
	if err != nil {
		atomic.AddUint64(&cd.misses, 1)
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		internal.Log.Printf("cache: Get key=%q failed: %s", key, err)
		return nil, err
	}
	atomic.AddUint64(&cd.hits, 1)

	if cd.localCache != nil {
		cd.localCache.Set(key, b)
	}
	return b, nil
}

// Once gets the item.Object for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Codec) Once(item *Item) error {
	if err := cd.beforeOnce(item); err != nil {
		return err
	}

	err := cd.once(item)

	if err := cd.afterOnce(item); err != nil {
		return err
	}

	return err
}

func (cd *Codec) once(item *Item) error {
	b, cached, err := cd.getSetItemBytesOnce(item)
	if err != nil {
		return err
	}

	if item.Object == nil || len(b) == 0 {
		return nil
	}

	err = cd.Unmarshal(b, item.Object)
	if err != nil {
		internal.Log.Printf("cache: key=%q Unmarshal(%T) failed: %s", item.Key, item.Object, err)
		if cached {
			_ = cd._delete(item.Key)
			return cd.once(item)
		}
		return err
	}

	return nil
}

func (cd *Codec) getSetItemBytesOnce(item *Item) (b []byte, cached bool, err error) {
	if cd.localCache != nil {
		b, err := cd.getItemBytesFast(item)
		if err == nil {
			return b, true, nil
		}
	}

	obj, err := cd.group.Do(item.Key, func() (interface{}, error) {
		b, err := cd.getItemBytes(item)
		if err == nil {
			cached = true
			return b, nil
		}

		obj, err := item.Func()
		if err != nil {
			return nil, err
		}

		b, err = cd.setItem(&Item{
			Key:        item.Key,
			Object:     obj,
			Expiration: item.Expiration,
		})
		if b != nil {
			// Ignore error if we have the result.
			return b, nil
		}
		return nil, err
	})
	if err != nil {
		return nil, false, err
	}
	return obj.([]byte), cached, nil
}

func (cd *Codec) getItemBytes(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, false)
}

func (cd *Codec) getItemBytesFast(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, true)
}

func (cd *Codec) Delete(key string) error {
	return cd.delete(nil, key)
}

func (cd *Codec) DeleteContext(c context.Context, key string) error {
	return cd.delete(c, key)
}

func (cd *Codec) delete(c context.Context, key string) error {
	c, err := cd.beforeDelete(c, key)
	if err != nil {
		return err
	}

	err = cd._delete(key)

	if _, err := cd.afterDelete(c, key); err != nil {
		return err
	}

	return err
}

func (cd *Codec) _delete(key string) error {
	if cd.localCache != nil {
		cd.localCache.Delete(key)
	}

	if cd.Redis == nil {
		if cd.localCache == nil {
			return errRedisLocalCacheNil
		}
		return nil
	}

	deleted, err := cd.Redis.Del(key).Result()
	if err != nil {
		internal.Log.Printf("cache: Del key=%q failed: %s", key, err)
		return err
	}
	if deleted == 0 {
		return ErrCacheMiss
	}
	return nil
}

//------------------------------------------------------------------------------

type Hook interface {
	BeforeSet(item *Item) error
	AfterSet(item *Item) error

	BeforeGet(c context.Context, key string, object interface{}) (context.Context, error)
	AfterGet(c context.Context, key string, object interface{}) (context.Context, error)

	BeforeDelete(c context.Context, key string) (context.Context, error)
	AfterDelete(c context.Context, key string) (context.Context, error)

	BeforeOnce(item *Item) error
	AfterOnce(item *Item) error
}

func (cd *Codec) AddHook(h Hook) {
	cd.hooks = append(cd.hooks, h)
}

func (cd *Codec) beforeSet(item *Item) error {
	for _, h := range cd.hooks {
		err := h.BeforeSet(item)
		if err != nil {
			return err
		}
	}
	return nil
}

func (cd *Codec) afterSet(item *Item) error {
	for _, h := range cd.hooks {
		err := h.AfterSet(item)
		if err != nil {
			return err
		}
	}
	return nil
}

func (cd *Codec) beforeGet(c context.Context, key string, object interface{}) (context.Context, error) {
	for _, h := range cd.hooks {
		var err error
		c, err = h.BeforeGet(c, key, object)
		if err != nil {
			return nil, err
		}
	}
	return c, nil
}

func (cd *Codec) afterGet(c context.Context, key string, object interface{}) (context.Context, error) {
	for _, h := range cd.hooks {
		var err error
		c, err = h.AfterGet(c, key, object)
		if err != nil {
			return nil, err
		}
	}
	return c, nil
}

func (cd *Codec) beforeDelete(c context.Context, key string) (context.Context, error) {
	for _, h := range cd.hooks {
		var err error
		c, err = h.BeforeDelete(c, key)
		if err != nil {
			return nil, err
		}
	}
	return c, nil
}

func (cd *Codec) afterDelete(c context.Context, key string) (context.Context, error) {
	for _, h := range cd.hooks {
		var err error
		c, err = h.AfterDelete(c, key)
		if err != nil {
			return nil, err
		}
	}
	return c, nil
}

func (cd *Codec) beforeOnce(item *Item) error {
	for _, h := range cd.hooks {
		err := h.BeforeOnce(item)
		if err != nil {
			return err
		}
	}
	return nil
}

func (cd *Codec) afterOnce(item *Item) error {
	for _, h := range cd.hooks {
		err := h.AfterOnce(item)
		if err != nil {
			return err
		}
	}
	return nil
}

//------------------------------------------------------------------------------

type Stats struct {
	Hits        uint64
	Misses      uint64
	LocalHits   uint64
	LocalMisses uint64
}

// Stats returns cache statistics.
func (cd *Codec) Stats() *Stats {
	stats := Stats{
		Hits:   atomic.LoadUint64(&cd.hits),
		Misses: atomic.LoadUint64(&cd.misses),
	}
	if cd.localCache != nil {
		stats.LocalHits = atomic.LoadUint64(&cd.localHits)
		stats.LocalMisses = atomic.LoadUint64(&cd.localMisses)
	}
	return &stats
}
<|endoftext|>


<commit_before>package imageserver

type Cache interface {
	Get(key string, parameters Parameters) (image *Image, err error)
	Set(key string, image *Image, parameters Parameters) error
}
<commit_msg>update cache<commit_after>package imageserver

type Cache interface {
	Get(key string, parameters Parameters) (image *Image, err error)
	Set(key string, image *Image, parameters Parameters) (err error)
}
<|endoftext|>


<commit_before>package blog

import (
	"html/template"
	"io"
	"io/ioutil"
	"sort"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/gogits/git"
	"github.com/russross/blackfriday"
)

type commitInfo struct {
	created time.Time
	commit  string
	tree    string
}

type node struct {
	Created time.Time

	IndexTemplate   *template.Template
	ArticleTemplate *template.Template

	Index    Index
	Articles map[string]*Article
	Tree     *git.Tree
}

// Cache gets and caches file trees and articles
type Cache struct {
	Repo *git.Repository `inject:""`

	lock  sync.RWMutex
	once  sync.Once
	cache map[string]node

	Branches map[string]*commitInfo
	Commits  map[string]*commitInfo
}

// BranchInfo gets the commit and tree ids of a branch
func (c *Cache) BranchInfo(branch string) (tid string, id string, err error) {
	c.lock.RLock()
	if c.Branches != nil {
		if info, ok := c.Branches[branch]; ok && time.Since(info.created) > time.Second {
			c.lock.RUnlock()
			return info.tree, info.commit, nil
		}
	}
	c.lock.RUnlock()

	commit, err := c.Repo.GetCommitOfBranch(branch)
	if err != nil {
		return "", "", err
	}

	c.lock.Lock()
	defer c.lock.Unlock()

	if c.Branches == nil {
		c.Branches = make(map[string]*commitInfo)
	}

	info := &commitInfo{
		created: time.Now(),
		commit:  commit.Id.String(),
		tree:    commit.TreeId().String(),
	}

	c.Branches[branch] = info

	return info.tree, info.commit, nil
}

// CommitInfo gets the commit and tree ids of a commit
func (c *Cache) CommitInfo(commitID string) (tid string, id string, err error) {
	c.lock.RLock()
	if c.Commits != nil {
		if info, ok := c.Commits[commitID]; ok && time.Since(info.created) > time.Second {
			c.lock.RUnlock()
			return info.tree, info.commit, nil
		}
	}
	c.lock.RUnlock()

	commit, err := c.Repo.GetCommit(commitID)
	if err != nil {
		return "", "", err
	}

	c.lock.Lock()
	defer c.lock.Unlock()

	if c.Commits == nil {
		c.Commits = make(map[string]*commitInfo)
	}

	info := &commitInfo{
		created: time.Now(),
		commit:  commit.Id.String(),
		tree:    commit.TreeId().String(),
	}

	c.Commits[commitID] = info

	return info.tree, info.commit, nil
}

// Clear clears the cache
func (c *Cache) Clear() {
	c.lock.Lock()
	defer c.lock.Unlock()

	c.cache = make(map[string]node)
}

// Clean removes old cached items
func (c *Cache) Clean() {
	c.lock.Lock()
	defer c.lock.Unlock()

	logrus.Info("Starting cache clean")

	for k, n := range c.cache {
		if time.Since(n.Created) > (time.Minute * 5) {
			logrus.WithField("id", k).Info("Old item removed from cache")
			delete(c.cache, k)
		}
	}
}

func (c *Cache) startClean() {
	runner := func() {
		ticker := time.NewTicker(time.Minute * 5)
		for {
			<-ticker.C
			c.Clean()
		}
	}
	go runner()
}

// ClearOne clears the cache for one commit id
func (c *Cache) ClearOne(id string) {
	c.lock.Lock()
	defer c.lock.Unlock()

	if c.cache == nil {
		return
	}

	delete(c.cache, id)
}

// GetIndex gets an Index from tree and commit ids
func (c *Cache) GetIndex(tid string, id string) (Index, bool) {
	if c.exists(id) {
		return c.getIndex(id)
	}

	if c.Build(tid, id) {
		return c.getIndex(id)
	}

	return nil, false
}

func (c *Cache) getIndex(id string) (Index, bool) {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.cache == nil {
		return nil, false
	}

	if n, ok := c.cache[id]; ok {
		return n.Index, true
	}

	return nil, false
}

// GetFile gets a file from tree and commit ids
func (c *Cache) GetFile(tid string, id string, path string) (io.Reader, bool) {
	if c.exists(id) {
		return c.getFile(id, path)
	}

	if c.Build(tid, id) {
		return c.getFile(id, path)
	}

	return nil, false
}

func (c *Cache) getFile(id string, path string) (io.Reader, bool) {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.cache == nil {
		return nil, false
	}

	if n, ok := c.cache[id]; ok {
		blob, err := n.Tree.GetBlobByPath(path)
		if err != nil {
			return nil, false
		}

		r, err := blob.Data()
		if err != nil {
			return nil, false
		}

		return r, true
	}

	return nil, false
}

// GetIndexTemplate gets the template from the cache
func (c *Cache) GetIndexTemplate(tid string, id string) *template.Template {
	if c.exists(id) {
		if tpl, ok := c.getIndexTemplate(id); ok {
			return tpl
		}
	} else if c.Build(tid, id) {
		if tpl, ok := c.getIndexTemplate(id); ok {
			return tpl
		}
	}

	return IndexTemplate
}

func (c *Cache) getIndexTemplate(id string) (*template.Template, bool) {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.cache == nil {
		return nil, false
	}

	if n, ok := c.cache[id]; ok {
		return n.IndexTemplate, true
	}

	return nil, false
}

// GetArticleTemplate gets the template from the cache
func (c *Cache) GetArticleTemplate(tid string, id string) *template.Template {
	if c.exists(id) {
		if tpl, ok := c.getArticleTemplate(id); ok {
			return tpl
		}
	} else if c.Build(tid, id) {
		if tpl, ok := c.getArticleTemplate(id); ok {
			return tpl
		}
	}

	return ArticleTemplate
}

func (c *Cache) getArticleTemplate(id string) (*template.Template, bool) {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.cache == nil {
		return nil, false
	}

	if n, ok := c.cache[id]; ok {
		return n.ArticleTemplate, true
	}

	return nil, false
}

func (c *Cache) buildIndexTemplate(tree *git.Tree) *template.Template {
	blob, err := tree.GetBlobByPath("index.tpl")
	if err != nil {
		logrus.WithError(err).Error("Could not read template blob")
		return IndexTemplate
	}

	reader, err := blob.Data()
	if err != nil {
		logrus.WithError(err).Error("Could not read template blob data")
		return IndexTemplate
	}

	bytes, err := ioutil.ReadAll(reader)
	if err != nil {
		logrus.WithError(err).Error("Could not read template data")
		return IndexTemplate
	}

	tpl, err := template.New("index").Parse(string(bytes))
	if err != nil {
		logrus.WithError(err).Error("Could parse template")
		return IndexTemplate
	}

	return tpl
}

func (c *Cache) buildArticleTemplate(tree *git.Tree) *template.Template {
	blob, err := tree.GetBlobByPath("article.tpl")
	if err != nil {
		logrus.WithError(err).Error("Could not read template blob")
		return ArticleTemplate
	}

	reader, err := blob.Data()
	if err != nil {
		logrus.WithError(err).Error("Could not read template blob data")
		return ArticleTemplate
	}

	bytes, err := ioutil.ReadAll(reader)
	if err != nil {
		logrus.WithError(err).Error("Could not read template data")
		return ArticleTemplate
	}

	tpl, err := template.New("index").Parse(string(bytes))
	if err != nil {
		logrus.WithError(err).Error("Could parse template")
		return ArticleTemplate
	}

	return tpl
}

// GetArticle gets an article from tree and commit ids
func (c *Cache) GetArticle(tid string, id string, article string) (*Article, bool) {
	if c.exists(id) {
		return c.getArticle(id, article)
	}

	if c.Build(tid, id) {
		return c.getArticle(id, article)
	}

	return nil, false
}

func (c *Cache) getArticle(id string, article string) (*Article, bool) {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.cache == nil {
		return nil, false
	}

	if n, ok := c.cache[id]; ok {
		if article, ok := n.Articles[article]; ok {
			return article, true
		}
	}

	return nil, false
}

func (c *Cache) exists(id string) bool {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.cache == nil {
		return false
	}

	n, ok := c.cache[id]

	return ok && time.Since(n.Created) < (time.Minute*5)
}

// Build gets and caches information on a tree and commit id combo
func (c *Cache) Build(tid string, id string) bool {
	c.lock.Lock()
	defer c.lock.Unlock()

	c.once.Do(c.startClean)

	logrus.WithField("commit", id).WithField("tree", tid).Info("Building cache")

	if c.cache == nil {
		c.cache = make(map[string]node)
	}

	sha1, err := git.NewIdFromString(tid)
	if err != nil {
		logrus.WithError(err).WithField("tree", tid).Error("Could not build data")
		return false
	}

	tree := git.NewTree(c.Repo, sha1)

	n := node{
		Articles: make(map[string]*Article),
		Tree:     tree,
		Created:  time.Now(),
	}

	scanner, err := tree.Scanner()
	if err != nil {
		logrus.WithError(err).WithField("tree", tid).Error("Could not build data")
		return false
	}

	for scanner.Scan() {
		entry := scanner.TreeEntry()

		name := entry.Name()

		if entry.IsDir() {
			logrus.
				WithField("tree", tid).
				WithField("directory", name).
				Info("Directory ignored")

			continue
		}

		if len(name) <= 3 || name[len(name)-3:] != ".md" {
			logrus.
				WithField("tree", tid).
				WithField("filename", name).
				Info("Non markdown file ignored")

			continue
		}

		reader, err := entry.Blob().Data()
		if err != nil {
			logrus.
				WithError(err).
				WithField("tree", tid).
				WithField("filename", name).
				Warn("File blob could not be generated")

			continue
		}

		markdown, err := ioutil.ReadAll(reader)
		if err != nil {
			logrus.
				WithError(err).
				WithField("tree", tid).
				WithField("filename", name).
				Warn("File blob could not be read")

			continue
		}

		commit, err := c.Repo.GetCommit(id)
		if err != nil {
			logrus.
				WithError(err).
				WithField("commit", id).
				Warn("Could not get commit")

			continue
		}

		fileCommit, err := commit.GetCommitOfRelPath(name)
		if err != nil {
			logrus.
				WithError(err).
				WithField("commit", id).
				WithField("filename", name).
				Warn("Could not get relative commit")

			continue
		}

		if fileCommit.Committer == nil {
			logrus.
				WithError(err).
				WithField("commit", fileCommit.Id.String()).
				Warn("Committer information not set")

			continue
		}

		article := Article{
			Name: name[:len(name)-3],
			Mod:  fileCommit.Committer.When,
			Data: blackfriday.MarkdownCommon(markdown),
		}

		logrus.
			WithField("commit", id).
			WithField("tree", tid).
			WithField("article", article.Name).
			Info("Article cached")

		n.Index = append(n.Index, article)
		n.Articles[article.Name] = &article
	}

	sort.Sort(n.Index)

	n.IndexTemplate = c.buildIndexTemplate(tree)
	n.ArticleTemplate = c.buildArticleTemplate(tree)

	c.cache[id] = n

	logrus.WithField("commit", id).WithField("tree", tid).Info("Cache built")

	return true
}
<commit_msg>Fixed cache issue<commit_after>package blog

import (
	"html/template"
	"io"
	"io/ioutil"
	"sort"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/gogits/git"
	"github.com/russross/blackfriday"
)

type commitInfo struct {
	created time.Time
	commit  string
	tree    string
}

type node struct {
	Created time.Time

	IndexTemplate   *template.Template
	ArticleTemplate *template.Template

	Index    Index
	Articles map[string]*Article
	Tree     *git.Tree
}

// Cache gets and caches file trees and articles
type Cache struct {
	Repo *git.Repository `inject:""`

	lock  sync.RWMutex
	once  sync.Once
	cache map[string]node

	Branches map[string]*commitInfo
	Commits  map[string]*commitInfo
}

// BranchInfo gets the commit and tree ids of a branch
func (c *Cache) BranchInfo(branch string) (tid string, id string, err error) {
	c.lock.RLock()
	if c.Branches != nil {
		if info, ok := c.Branches[branch]; ok && time.Since(info.created) < time.Second {
			c.lock.RUnlock()
			return info.tree, info.commit, nil
		}
	}
	c.lock.RUnlock()

	commit, err := c.Repo.GetCommitOfBranch(branch)
	if err != nil {
		return "", "", err
	}

	c.lock.Lock()
	defer c.lock.Unlock()

	if c.Branches == nil {
		c.Branches = make(map[string]*commitInfo)
	}

	info := &commitInfo{
		created: time.Now(),
		commit:  commit.Id.String(),
		tree:    commit.TreeId().String(),
	}

	c.Branches[branch] = info

	return info.tree, info.commit, nil
}

// CommitInfo gets the commit and tree ids of a commit
func (c *Cache) CommitInfo(commitID string) (tid string, id string, err error) {
	c.lock.RLock()
	if c.Commits != nil {
		if info, ok := c.Commits[commitID]; ok && time.Since(info.created) < time.Second {
			c.lock.RUnlock()
			return info.tree, info.commit, nil
		}
	}
	c.lock.RUnlock()

	commit, err := c.Repo.GetCommit(commitID)
	if err != nil {
		return "", "", err
	}

	c.lock.Lock()
	defer c.lock.Unlock()

	if c.Commits == nil {
		c.Commits = make(map[string]*commitInfo)
	}

	info := &commitInfo{
		created: time.Now(),
		commit:  commit.Id.String(),
		tree:    commit.TreeId().String(),
	}

	c.Commits[commitID] = info

	return info.tree, info.commit, nil
}

// Clear clears the cache
func (c *Cache) Clear() {
	c.lock.Lock()
	defer c.lock.Unlock()

	c.cache = make(map[string]node)
}

// Clean removes old cached items
func (c *Cache) Clean() {
	c.lock.Lock()
	defer c.lock.Unlock()

	logrus.Info("Starting cache clean")

	for k, n := range c.cache {
		if time.Since(n.Created) > (time.Minute * 5) {
			logrus.WithField("id", k).Info("Old item removed from cache")
			delete(c.cache, k)
		}
	}
}

func (c *Cache) startClean() {
	runner := func() {
		ticker := time.NewTicker(time.Minute * 5)
		for {
			<-ticker.C
			c.Clean()
		}
	}
	go runner()
}

// ClearOne clears the cache for one commit id
func (c *Cache) ClearOne(id string) {
	c.lock.Lock()
	defer c.lock.Unlock()

	if c.cache == nil {
		return
	}

	delete(c.cache, id)
}

// GetIndex gets an Index from tree and commit ids
func (c *Cache) GetIndex(tid string, id string) (Index, bool) {
	if c.exists(id) {
		return c.getIndex(id)
	}

	if c.Build(tid, id) {
		return c.getIndex(id)
	}

	return nil, false
}

func (c *Cache) getIndex(id string) (Index, bool) {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.cache == nil {
		return nil, false
	}

	if n, ok := c.cache[id]; ok {
		return n.Index, true
	}

	return nil, false
}

// GetFile gets a file from tree and commit ids
func (c *Cache) GetFile(tid string, id string, path string) (io.Reader, bool) {
	if c.exists(id) {
		return c.getFile(id, path)
	}

	if c.Build(tid, id) {
		return c.getFile(id, path)
	}

	return nil, false
}

func (c *Cache) getFile(id string, path string) (io.Reader, bool) {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.cache == nil {
		return nil, false
	}

	if n, ok := c.cache[id]; ok {
		blob, err := n.Tree.GetBlobByPath(path)
		if err != nil {
			return nil, false
		}

		r, err := blob.Data()
		if err != nil {
			return nil, false
		}

		return r, true
	}

	return nil, false
}

// GetIndexTemplate gets the template from the cache
func (c *Cache) GetIndexTemplate(tid string, id string) *template.Template {
	if c.exists(id) {
		if tpl, ok := c.getIndexTemplate(id); ok {
			return tpl
		}
	} else if c.Build(tid, id) {
		if tpl, ok := c.getIndexTemplate(id); ok {
			return tpl
		}
	}

	return IndexTemplate
}

func (c *Cache) getIndexTemplate(id string) (*template.Template, bool) {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.cache == nil {
		return nil, false
	}

	if n, ok := c.cache[id]; ok {
		return n.IndexTemplate, true
	}

	return nil, false
}

// GetArticleTemplate gets the template from the cache
func (c *Cache) GetArticleTemplate(tid string, id string) *template.Template {
	if c.exists(id) {
		if tpl, ok := c.getArticleTemplate(id); ok {
			return tpl
		}
	} else if c.Build(tid, id) {
		if tpl, ok := c.getArticleTemplate(id); ok {
			return tpl
		}
	}

	return ArticleTemplate
}

func (c *Cache) getArticleTemplate(id string) (*template.Template, bool) {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.cache == nil {
		return nil, false
	}

	if n, ok := c.cache[id]; ok {
		return n.ArticleTemplate, true
	}

	return nil, false
}

func (c *Cache) buildIndexTemplate(tree *git.Tree) *template.Template {
	blob, err := tree.GetBlobByPath("index.tpl")
	if err != nil {
		logrus.WithError(err).Error("Could not read template blob")
		return IndexTemplate
	}

	reader, err := blob.Data()
	if err != nil {
		logrus.WithError(err).Error("Could not read template blob data")
		return IndexTemplate
	}

	bytes, err := ioutil.ReadAll(reader)
	if err != nil {
		logrus.WithError(err).Error("Could not read template data")
		return IndexTemplate
	}

	tpl, err := template.New("index").Parse(string(bytes))
	if err != nil {
		logrus.WithError(err).Error("Could parse template")
		return IndexTemplate
	}

	return tpl
}

func (c *Cache) buildArticleTemplate(tree *git.Tree) *template.Template {
	blob, err := tree.GetBlobByPath("article.tpl")
	if err != nil {
		logrus.WithError(err).Error("Could not read template blob")
		return ArticleTemplate
	}

	reader, err := blob.Data()
	if err != nil {
		logrus.WithError(err).Error("Could not read template blob data")
		return ArticleTemplate
	}

	bytes, err := ioutil.ReadAll(reader)
	if err != nil {
		logrus.WithError(err).Error("Could not read template data")
		return ArticleTemplate
	}

	tpl, err := template.New("index").Parse(string(bytes))
	if err != nil {
		logrus.WithError(err).Error("Could parse template")
		return ArticleTemplate
	}

	return tpl
}

// GetArticle gets an article from tree and commit ids
func (c *Cache) GetArticle(tid string, id string, article string) (*Article, bool) {
	if c.exists(id) {
		return c.getArticle(id, article)
	}

	if c.Build(tid, id) {
		return c.getArticle(id, article)
	}

	return nil, false
}

func (c *Cache) getArticle(id string, article string) (*Article, bool) {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.cache == nil {
		return nil, false
	}

	if n, ok := c.cache[id]; ok {
		if article, ok := n.Articles[article]; ok {
			return article, true
		}
	}

	return nil, false
}

func (c *Cache) exists(id string) bool {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if c.cache == nil {
		return false
	}

	n, ok := c.cache[id]

	return ok && time.Since(n.Created) < (time.Minute*5)
}

// Build gets and caches information on a tree and commit id combo
func (c *Cache) Build(tid string, id string) bool {
	c.lock.Lock()
	defer c.lock.Unlock()

	c.once.Do(c.startClean)

	logrus.WithField("commit", id).WithField("tree", tid).Info("Building cache")

	if c.cache == nil {
		c.cache = make(map[string]node)
	}

	sha1, err := git.NewIdFromString(tid)
	if err != nil {
		logrus.WithError(err).WithField("tree", tid).Error("Could not build data")
		return false
	}

	tree := git.NewTree(c.Repo, sha1)

	n := node{
		Articles: make(map[string]*Article),
		Tree:     tree,
		Created:  time.Now(),
	}

	scanner, err := tree.Scanner()
	if err != nil {
		logrus.WithError(err).WithField("tree", tid).Error("Could not build data")
		return false
	}

	for scanner.Scan() {
		entry := scanner.TreeEntry()

		name := entry.Name()

		if entry.IsDir() {
			logrus.
				WithField("tree", tid).
				WithField("directory", name).
				Info("Directory ignored")

			continue
		}

		if len(name) <= 3 || name[len(name)-3:] != ".md" {
			logrus.
				WithField("tree", tid).
				WithField("filename", name).
				Info("Non markdown file ignored")

			continue
		}

		reader, err := entry.Blob().Data()
		if err != nil {
			logrus.
				WithError(err).
				WithField("tree", tid).
				WithField("filename", name).
				Warn("File blob could not be generated")

			continue
		}

		markdown, err := ioutil.ReadAll(reader)
		if err != nil {
			logrus.
				WithError(err).
				WithField("tree", tid).
				WithField("filename", name).
				Warn("File blob could not be read")

			continue
		}

		commit, err := c.Repo.GetCommit(id)
		if err != nil {
			logrus.
				WithError(err).
				WithField("commit", id).
				Warn("Could not get commit")

			continue
		}

		fileCommit, err := commit.GetCommitOfRelPath(name)
		if err != nil {
			logrus.
				WithError(err).
				WithField("commit", id).
				WithField("filename", name).
				Warn("Could not get relative commit")

			continue
		}

		if fileCommit.Committer == nil {
			logrus.
				WithError(err).
				WithField("commit", fileCommit.Id.String()).
				Warn("Committer information not set")

			continue
		}

		article := Article{
			Name: name[:len(name)-3],
			Mod:  fileCommit.Committer.When,
			Data: blackfriday.MarkdownCommon(markdown),
		}

		logrus.
			WithField("commit", id).
			WithField("tree", tid).
			WithField("article", article.Name).
			Info("Article cached")

		n.Index = append(n.Index, article)
		n.Articles[article.Name] = &article
	}

	sort.Sort(n.Index)

	n.IndexTemplate = c.buildIndexTemplate(tree)
	n.ArticleTemplate = c.buildArticleTemplate(tree)

	c.cache[id] = n

	logrus.WithField("commit", id).WithField("tree", tid).Info("Cache built")

	return true
}
<|endoftext|>


<commit_before>//Simple caching library with expiration capabilities
package cache

import (
	"errors"
	"sync"
	"time"
)

type expiringCacheEntry interface {
	XCache(key string, expire time.Duration, value expiringCacheEntry)
	KeepAlive()
}

// Structure that must be embeded in the objectst that must be cached with expiration.
// If the expiration is not needed this can be ignored
type XEntry struct {
	sync.Mutex
	key            string
	keepAlive      bool
	expireDuration time.Duration
}

var (
	xcache = make(map[string]expiringCacheEntry)
	cache  = make(map[string]interface{})
)

// The main function to cache with expiration
func (xe *XEntry) XCache(key string, expire time.Duration, value expiringCacheEntry) {
	xe.keepAlive = true
	xe.key = key
	xe.expireDuration = expire
	xcache[key] = value
	go xe.expire()
}

// The internal mechanism for expiartion
func (xe *XEntry) expire() {
	for xe.keepAlive {
		xe.Lock()
		xe.keepAlive = false
		xe.Unlock()
		t := time.NewTimer(xe.expireDuration)
		<-t.C
		xe.Lock()
		if !xe.keepAlive {
			delete(xcache, xe.key)
		}
		xe.Unlock()
	}
}

// Mark entry to be kept another expirationDuration period
func (xe *XEntry) KeepAlive() {
	xe.Lock()
	defer xe.Unlock()
	xe.keepAlive = true
}

// Get an entry from the expiration cache and mark it for keeping alive
func GetXCached(key string) (ece expiringCacheEntry, err error) {
	if r, ok := xcache[key]; ok {
		r.KeepAlive()
		return r, nil
	}
	return nil, errors.New("not found")
}

// The function to be used to cache a key/value pair when expiration is not needed
func Cache(key string, value interface{}) {
	cache[key] = value
}

// The function to extract a value for a key that never expire
func GetCached(key string) (v interface{}, err error) {
	if r, ok := cache[key]; ok {
		return r, nil
	}
	return nil, errors.New("not found")
}
<commit_msg>added flush methods<commit_after>//Simple caching library with expiration capabilities
package cache

import (
	"errors"
	"sync"
	"time"
)

type expiringCacheEntry interface {
	XCache(key string, expire time.Duration, value expiringCacheEntry)
	KeepAlive()
}

// Structure that must be embeded in the objectst that must be cached with expiration.
// If the expiration is not needed this can be ignored
type XEntry struct {
	sync.Mutex
	key            string
	keepAlive      bool
	expireDuration time.Duration
}

var (
	xcache = make(map[string]expiringCacheEntry)
	cache  = make(map[string]interface{})
)

// The main function to cache with expiration
func (xe *XEntry) XCache(key string, expire time.Duration, value expiringCacheEntry) {
	xe.keepAlive = true
	xe.key = key
	xe.expireDuration = expire
	xcache[key] = value
	go xe.expire()
}

// The internal mechanism for expiartion
func (xe *XEntry) expire() {
	for xe.keepAlive {
		xe.Lock()
		xe.keepAlive = false
		xe.Unlock()
		t := time.NewTimer(xe.expireDuration)
		<-t.C
		xe.Lock()
		if !xe.keepAlive {
			delete(xcache, xe.key)
		}
		xe.Unlock()
	}
}

// Mark entry to be kept another expirationDuration period
func (xe *XEntry) KeepAlive() {
	xe.Lock()
	defer xe.Unlock()
	xe.keepAlive = true
}

// Delete all keys from expiraton cache
func (xe *XEntry) Flush() {
	xcache = make(map[string]expiringCacheEntry)
}

// Get an entry from the expiration cache and mark it for keeping alive
func GetXCached(key string) (ece expiringCacheEntry, err error) {
	if r, ok := xcache[key]; ok {
		r.KeepAlive()
		return r, nil
	}
	return nil, errors.New("not found")
}

// The function to be used to cache a key/value pair when expiration is not needed
func Cache(key string, value interface{}) {
	cache[key] = value
}

// The function to extract a value for a key that never expire
func GetCached(key string) (v interface{}, err error) {
	if r, ok := cache[key]; ok {
		return r, nil
	}
	return nil, errors.New("not found")
}

// Delete all keys from cache
func Flush() {
	cache = make(map[string]interface{})
}
<|endoftext|>


<commit_before>package syncer_test

import (
	"errors"
	"os"
	"sync"
	"time"

	"github.com/apcera/nats"
	"github.com/cloudfoundry-incubator/route-emitter/nats_emitter/fake_nats_emitter"
	"github.com/cloudfoundry-incubator/route-emitter/routing_table"
	"github.com/cloudfoundry-incubator/route-emitter/routing_table/fake_routing_table"
	. "github.com/cloudfoundry-incubator/route-emitter/syncer"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/fake_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/gibson"
	"github.com/cloudfoundry/yagnats/fakeyagnats"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Syncer", func() {
	var (
		bbs          *fake_bbs.FakeRouteEmitterBBS
		natsClient   *fakeyagnats.FakeApceraWrapper
		emitter      *fake_nats_emitter.FakeNATSEmitter
		table        *fake_routing_table.FakeRoutingTable
		syncer       *Syncer
		process      ifrit.Process
		syncMessages routing_table.MessagesToEmit
		emitMessages routing_table.MessagesToEmit
		syncDuration time.Duration

		routerStartMessages chan<- *nats.Msg
	)

	BeforeEach(func() {
		bbs = fake_bbs.NewFakeRouteEmitterBBS()
		natsClient = fakeyagnats.NewApceraClientWrapper()
		emitter = &fake_nats_emitter.FakeNATSEmitter{}
		table = &fake_routing_table.FakeRoutingTable{}
		syncDuration = 10 * time.Second

		startMessages := make(chan *nats.Msg)
		routerStartMessages = startMessages

		natsClient.WhenSubscribing("router.start", func(callback nats.MsgHandler) error {
			go func() {
				for msg := range startMessages {
					callback(msg)
				}
			}()

			return nil
		})

		//what follows is fake data to distinguish between
		//the "sync" and "emit" codepaths
		dummyContainer := routing_table.Container{Host: "1.1.1.1", Port: 11}
		dummyMessage := routing_table.RegistryMessageFor(dummyContainer, "foo.com", "bar.com")
		syncMessages = routing_table.MessagesToEmit{
			RegistrationMessages: []gibson.RegistryMessage{dummyMessage},
		}

		dummyContainer = routing_table.Container{Host: "2.2.2.2", Port: 22}
		dummyMessage = routing_table.RegistryMessageFor(dummyContainer, "baz.com")
		emitMessages = routing_table.MessagesToEmit{
			RegistrationMessages: []gibson.RegistryMessage{dummyMessage},
		}

		table.SyncReturns(syncMessages)
		table.MessagesToEmitReturns(emitMessages)

		//Set up some BBS data
		bbs.AllActualLRPs = []models.ActualLRP{
			{
				ProcessGuid:  "process-guid-1",
				Index:        0,
				InstanceGuid: "instance-guid-1",
				Host:         "1.2.3.4",
				Ports: []models.PortMapping{
					{
						HostPort:      1234,
						ContainerPort: 5678,
					},
				},
			},
		}

		bbs.AllDesiredLRPs = []models.DesiredLRP{
			{
				ProcessGuid: "process-guid-1",
				Routes:      []string{"route-1", "route-2"},
			},
		}
	})

	JustBeforeEach(func() {
		logger := lagertest.NewTestLogger("test")
		syncer = NewSyncer(bbs, table, emitter, syncDuration, natsClient, logger)
		process = ifrit.Envoke(syncer)
	})

	AfterEach(func() {
		process.Signal(os.Interrupt)
		Eventually(process.Wait()).Should(Receive(BeNil()))
		close(routerStartMessages)
	})

	Describe("on startup", func() {
		It("should sync the table", func() {
			Ω(table.SyncCallCount()).Should(Equal(1))
			routes, containers := table.SyncArgsForCall(0)
			Ω(routes["process-guid-1"]).Should(Equal([]string{"route-1", "route-2"}))
			Ω(containers["process-guid-1"]).Should(Equal([]routing_table.Container{
				{Host: "1.2.3.4", Port: 1234},
			}))

			Ω(emitter.EmitCallCount()).Should(Equal(1))
			Ω(emitter.EmitArgsForCall(0)).Should(Equal(syncMessages))
		})
	})

	Describe("getting the heartbeat interval from the router", func() {
		var greetings chan *nats.Msg
		BeforeEach(func() {
			greetings = make(chan *nats.Msg, 3)
			natsClient.WhenPublishing("router.greet", func(msg *nats.Msg) error {
				greetings <- msg
				return nil
			})
		})

		Context("when the router emits a router.start", func() {
			JustBeforeEach(func() {
				routerStartMessages <- &nats.Msg{
					Data: []byte(`{"minimumRegisterIntervalInSeconds":1}`),
				}
			})

			It("should emit routes with the frequency of the passed-in-interval", func() {
				Eventually(emitter.EmitCallCount, 2).Should(Equal(2))
				Ω(emitter.EmitArgsForCall(1)).Should(Equal(emitMessages))
				t1 := time.Now()

				Eventually(emitter.EmitCallCount, 2).Should(Equal(3))
				Ω(emitter.EmitArgsForCall(2)).Should(Equal(emitMessages))
				t2 := time.Now()

				Ω(t2.Sub(t1)).Should(BeNumerically("~", 1*time.Second, 200*time.Millisecond))
			})

			It("should only greet the router once", func() {
				Eventually(greetings).Should(Receive())
				Consistently(greetings, 1).ShouldNot(Receive())
			})
		})

		Context("when the router does not emit a router.start", func() {
			It("should keep greeting the router until it gets an interval", func() {
				//get the first greeting
				Eventually(greetings, 2).Should(Receive())

				//get the second greeting, and respond
				var msg *nats.Msg
				Eventually(greetings, 2).Should(Receive(&msg))
				go natsClient.Publish(msg.Reply, []byte(`{"minimumRegisterIntervalInSeconds":1}`))

				// go natsClient.Subscriptions(msg.Reply)[0].Callback(&nats.Msg{
				// 	Data: []byte(`{"minimumRegisterIntervalInSeconds":1}`),
				// })

				//shold now be emittingn regularly at the specified interval
				Eventually(emitter.EmitCallCount, 2).Should(Equal(2))
				Ω(emitter.EmitArgsForCall(1)).Should(Equal(emitMessages))
				t1 := time.Now()

				Eventually(emitter.EmitCallCount, 2).Should(Equal(3))
				Ω(emitter.EmitArgsForCall(2)).Should(Equal(emitMessages))
				t2 := time.Now()

				Ω(t2.Sub(t1)).Should(BeNumerically("~", 1*time.Second, 200*time.Millisecond))

				//should no longer be greeting the router
				Consistently(greetings).ShouldNot(Receive())
			})
		})

		Context("after getting the first interval, when a second interval arrives", func() {
			JustBeforeEach(func() {
				routerStartMessages <- &nats.Msg{
					Data: []byte(`{"minimumRegisterIntervalInSeconds":1}`),
				}
			})

			It("should modify its update rate", func() {
				routerStartMessages <- &nats.Msg{
					Data: []byte(`{"minimumRegisterIntervalInSeconds":2}`),
				}

				//first emit should be pretty quick, it is in response to the incoming heartbeat interval
				Eventually(emitter.EmitCallCount, 0.2).Should(Equal(2))
				Ω(emitter.EmitArgsForCall(1)).Should(Equal(emitMessages))
				t1 := time.Now()

				//subsequent emit should follow the interval
				Eventually(emitter.EmitCallCount, 3).Should(Equal(3))
				Ω(emitter.EmitArgsForCall(2)).Should(Equal(emitMessages))
				t2 := time.Now()
				Ω(t2.Sub(t1)).Should(BeNumerically("~", 2*time.Second, 200*time.Millisecond))
			})
		})

		Context("if it never hears anything from a router anywhere", func() {
			It("should still be able to shutdown", func() {
				process.Signal(os.Interrupt)
				Eventually(process.Wait()).Should(Receive(BeNil()))
			})
		})
	})

	Describe("syncing", func() {
		BeforeEach(func() {
			syncDuration = 500 * time.Millisecond
		})

		It("should sync on the specified interval", func() {
			//we set the emit interval real high to avoid colliding with our sync interval
			routerStartMessages <- &nats.Msg{
				Data: []byte(`{"minimumRegisterIntervalInSeconds":10}`),
			}

			Eventually(table.SyncCallCount).Should(Equal(2))
			Eventually(emitter.EmitCallCount).Should(Equal(2))
			t1 := time.Now()

			Eventually(table.SyncCallCount).Should(Equal(3))
			Eventually(emitter.EmitCallCount).Should(Equal(3))
			t2 := time.Now()

			Ω(emitter.EmitArgsForCall(1)).Should(Equal(syncMessages))
			Ω(emitter.EmitArgsForCall(2)).Should(Equal(syncMessages))
			Ω(t2.Sub(t1)).Should(BeNumerically("~", 500*time.Millisecond, 100*time.Millisecond))
		})

		Context("when fetching actuals fails", func() {
			BeforeEach(func() {
				lock := &sync.Mutex{}
				calls := 0
				bbs.WhenGettingRunningActualLRPs = func() ([]models.ActualLRP, error) {
					lock.Lock()
					defer lock.Unlock()
					if calls == 0 {
						calls++
						return nil, errors.New("bam")
					}
					return bbs.AllActualLRPs, nil
				}
			})

			It("should not call sync until the error resolves", func() {
				Ω(table.SyncCallCount()).Should(Equal(0))

				routerStartMessages <- &nats.Msg{
					Data: []byte(`{"minimumRegisterIntervalInSeconds":10}`),
				}

				Eventually(table.SyncCallCount).Should(Equal(1))
			})
		})

		Context("when fetching desireds fails", func() {
			BeforeEach(func() {
				lock := &sync.Mutex{}
				calls := 0
				bbs.WhenGettingAllDesiredLRPs = func() ([]models.DesiredLRP, error) {
					lock.Lock()
					defer lock.Unlock()
					if calls == 0 {
						calls++
						return nil, errors.New("bam")
					}
					return bbs.AllDesiredLRPs, nil
				}
			})

			It("should not call sync until the error resolves", func() {
				Ω(table.SyncCallCount()).Should(Equal(0))

				routerStartMessages <- &nats.Msg{
					Data: []byte(`{"minimumRegisterIntervalInSeconds":10}`),
				}

				Eventually(table.SyncCallCount).Should(Equal(1))
			})
		})
	})
})
<commit_msg>Fix/delete some comments in tests<commit_after>package syncer_test

import (
	"errors"
	"os"
	"sync"
	"time"

	"github.com/apcera/nats"
	"github.com/cloudfoundry-incubator/route-emitter/nats_emitter/fake_nats_emitter"
	"github.com/cloudfoundry-incubator/route-emitter/routing_table"
	"github.com/cloudfoundry-incubator/route-emitter/routing_table/fake_routing_table"
	. "github.com/cloudfoundry-incubator/route-emitter/syncer"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/fake_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/gibson"
	"github.com/cloudfoundry/yagnats/fakeyagnats"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Syncer", func() {
	var (
		bbs          *fake_bbs.FakeRouteEmitterBBS
		natsClient   *fakeyagnats.FakeApceraWrapper
		emitter      *fake_nats_emitter.FakeNATSEmitter
		table        *fake_routing_table.FakeRoutingTable
		syncer       *Syncer
		process      ifrit.Process
		syncMessages routing_table.MessagesToEmit
		emitMessages routing_table.MessagesToEmit
		syncDuration time.Duration

		routerStartMessages chan<- *nats.Msg
	)

	BeforeEach(func() {
		bbs = fake_bbs.NewFakeRouteEmitterBBS()
		natsClient = fakeyagnats.NewApceraClientWrapper()
		emitter = &fake_nats_emitter.FakeNATSEmitter{}
		table = &fake_routing_table.FakeRoutingTable{}
		syncDuration = 10 * time.Second

		startMessages := make(chan *nats.Msg)
		routerStartMessages = startMessages

		natsClient.WhenSubscribing("router.start", func(callback nats.MsgHandler) error {
			go func() {
				for msg := range startMessages {
					callback(msg)
				}
			}()

			return nil
		})

		//what follows is fake data to distinguish between
		//the "sync" and "emit" codepaths
		dummyContainer := routing_table.Container{Host: "1.1.1.1", Port: 11}
		dummyMessage := routing_table.RegistryMessageFor(dummyContainer, "foo.com", "bar.com")
		syncMessages = routing_table.MessagesToEmit{
			RegistrationMessages: []gibson.RegistryMessage{dummyMessage},
		}

		dummyContainer = routing_table.Container{Host: "2.2.2.2", Port: 22}
		dummyMessage = routing_table.RegistryMessageFor(dummyContainer, "baz.com")
		emitMessages = routing_table.MessagesToEmit{
			RegistrationMessages: []gibson.RegistryMessage{dummyMessage},
		}

		table.SyncReturns(syncMessages)
		table.MessagesToEmitReturns(emitMessages)

		//Set up some BBS data
		bbs.AllActualLRPs = []models.ActualLRP{
			{
				ProcessGuid:  "process-guid-1",
				Index:        0,
				InstanceGuid: "instance-guid-1",
				Host:         "1.2.3.4",
				Ports: []models.PortMapping{
					{
						HostPort:      1234,
						ContainerPort: 5678,
					},
				},
			},
		}

		bbs.AllDesiredLRPs = []models.DesiredLRP{
			{
				ProcessGuid: "process-guid-1",
				Routes:      []string{"route-1", "route-2"},
			},
		}
	})

	JustBeforeEach(func() {
		logger := lagertest.NewTestLogger("test")
		syncer = NewSyncer(bbs, table, emitter, syncDuration, natsClient, logger)
		process = ifrit.Envoke(syncer)
	})

	AfterEach(func() {
		process.Signal(os.Interrupt)
		Eventually(process.Wait()).Should(Receive(BeNil()))
		close(routerStartMessages)
	})

	Describe("on startup", func() {
		It("should sync the table", func() {
			Ω(table.SyncCallCount()).Should(Equal(1))
			routes, containers := table.SyncArgsForCall(0)
			Ω(routes["process-guid-1"]).Should(Equal([]string{"route-1", "route-2"}))
			Ω(containers["process-guid-1"]).Should(Equal([]routing_table.Container{
				{Host: "1.2.3.4", Port: 1234},
			}))

			Ω(emitter.EmitCallCount()).Should(Equal(1))
			Ω(emitter.EmitArgsForCall(0)).Should(Equal(syncMessages))
		})
	})

	Describe("getting the heartbeat interval from the router", func() {
		var greetings chan *nats.Msg
		BeforeEach(func() {
			greetings = make(chan *nats.Msg, 3)
			natsClient.WhenPublishing("router.greet", func(msg *nats.Msg) error {
				greetings <- msg
				return nil
			})
		})

		Context("when the router emits a router.start", func() {
			JustBeforeEach(func() {
				routerStartMessages <- &nats.Msg{
					Data: []byte(`{"minimumRegisterIntervalInSeconds":1}`),
				}
			})

			It("should emit routes with the frequency of the passed-in-interval", func() {
				Eventually(emitter.EmitCallCount, 2).Should(Equal(2))
				Ω(emitter.EmitArgsForCall(1)).Should(Equal(emitMessages))
				t1 := time.Now()

				Eventually(emitter.EmitCallCount, 2).Should(Equal(3))
				Ω(emitter.EmitArgsForCall(2)).Should(Equal(emitMessages))
				t2 := time.Now()

				Ω(t2.Sub(t1)).Should(BeNumerically("~", 1*time.Second, 200*time.Millisecond))
			})

			It("should only greet the router once", func() {
				Eventually(greetings).Should(Receive())
				Consistently(greetings, 1).ShouldNot(Receive())
			})
		})

		Context("when the router does not emit a router.start", func() {
			It("should keep greeting the router until it gets an interval", func() {
				//get the first greeting
				Eventually(greetings, 2).Should(Receive())

				//get the second greeting, and respond
				var msg *nats.Msg
				Eventually(greetings, 2).Should(Receive(&msg))
				go natsClient.Publish(msg.Reply, []byte(`{"minimumRegisterIntervalInSeconds":1}`))

				//should now be emitting regularly at the specified interval
				Eventually(emitter.EmitCallCount, 2).Should(Equal(2))
				Ω(emitter.EmitArgsForCall(1)).Should(Equal(emitMessages))
				t1 := time.Now()

				Eventually(emitter.EmitCallCount, 2).Should(Equal(3))
				Ω(emitter.EmitArgsForCall(2)).Should(Equal(emitMessages))
				t2 := time.Now()

				Ω(t2.Sub(t1)).Should(BeNumerically("~", 1*time.Second, 200*time.Millisecond))

				//should no longer be greeting the router
				Consistently(greetings).ShouldNot(Receive())
			})
		})

		Context("after getting the first interval, when a second interval arrives", func() {
			JustBeforeEach(func() {
				routerStartMessages <- &nats.Msg{
					Data: []byte(`{"minimumRegisterIntervalInSeconds":1}`),
				}
			})

			It("should modify its update rate", func() {
				routerStartMessages <- &nats.Msg{
					Data: []byte(`{"minimumRegisterIntervalInSeconds":2}`),
				}

				//first emit should be pretty quick, it is in response to the incoming heartbeat interval
				Eventually(emitter.EmitCallCount, 0.2).Should(Equal(2))
				Ω(emitter.EmitArgsForCall(1)).Should(Equal(emitMessages))
				t1 := time.Now()

				//subsequent emit should follow the interval
				Eventually(emitter.EmitCallCount, 3).Should(Equal(3))
				Ω(emitter.EmitArgsForCall(2)).Should(Equal(emitMessages))
				t2 := time.Now()
				Ω(t2.Sub(t1)).Should(BeNumerically("~", 2*time.Second, 200*time.Millisecond))
			})
		})

		Context("if it never hears anything from a router anywhere", func() {
			It("should still be able to shutdown", func() {
				process.Signal(os.Interrupt)
				Eventually(process.Wait()).Should(Receive(BeNil()))
			})
		})
	})

	Describe("syncing", func() {
		BeforeEach(func() {
			syncDuration = 500 * time.Millisecond
		})

		It("should sync on the specified interval", func() {
			//we set the emit interval real high to avoid colliding with our sync interval
			routerStartMessages <- &nats.Msg{
				Data: []byte(`{"minimumRegisterIntervalInSeconds":10}`),
			}

			Eventually(table.SyncCallCount).Should(Equal(2))
			Eventually(emitter.EmitCallCount).Should(Equal(2))
			t1 := time.Now()

			Eventually(table.SyncCallCount).Should(Equal(3))
			Eventually(emitter.EmitCallCount).Should(Equal(3))
			t2 := time.Now()

			Ω(emitter.EmitArgsForCall(1)).Should(Equal(syncMessages))
			Ω(emitter.EmitArgsForCall(2)).Should(Equal(syncMessages))
			Ω(t2.Sub(t1)).Should(BeNumerically("~", 500*time.Millisecond, 100*time.Millisecond))
		})

		Context("when fetching actuals fails", func() {
			BeforeEach(func() {
				lock := &sync.Mutex{}
				calls := 0
				bbs.WhenGettingRunningActualLRPs = func() ([]models.ActualLRP, error) {
					lock.Lock()
					defer lock.Unlock()
					if calls == 0 {
						calls++
						return nil, errors.New("bam")
					}
					return bbs.AllActualLRPs, nil
				}
			})

			It("should not call sync until the error resolves", func() {
				Ω(table.SyncCallCount()).Should(Equal(0))

				routerStartMessages <- &nats.Msg{
					Data: []byte(`{"minimumRegisterIntervalInSeconds":10}`),
				}

				Eventually(table.SyncCallCount).Should(Equal(1))
			})
		})

		Context("when fetching desireds fails", func() {
			BeforeEach(func() {
				lock := &sync.Mutex{}
				calls := 0
				bbs.WhenGettingAllDesiredLRPs = func() ([]models.DesiredLRP, error) {
					lock.Lock()
					defer lock.Unlock()
					if calls == 0 {
						calls++
						return nil, errors.New("bam")
					}
					return bbs.AllDesiredLRPs, nil
				}
			})

			It("should not call sync until the error resolves", func() {
				Ω(table.SyncCallCount()).Should(Equal(0))

				routerStartMessages <- &nats.Msg{
					Data: []byte(`{"minimumRegisterIntervalInSeconds":10}`),
				}

				Eventually(table.SyncCallCount).Should(Equal(1))
			})
		})
	})
})
<|endoftext|>


<commit_before>package command

import (
	"flag"
	"fmt"
	"strings"

	"github.com/funkygao/columnize"
	"github.com/funkygao/dbus/pkg/cluster"
	czk "github.com/funkygao/dbus/pkg/cluster/zk"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gocli"
)

type Resources struct {
	Ui  cli.Ui
	Cmd string

	zone        string
	addResource string
}

func (this *Resources) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("resources", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.addResource, "add", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	mgr := czk.NewManager(ctx.ZoneZkAddrs(this.zone))
	if err := mgr.Open(); err != nil {
		panic(err)
	}
	defer mgr.Close()

	if len(this.addResource) != 0 {
		tuples := strings.SplitN(this.addResource, "-", 2)
		if len(tuples) != 2 {
			this.Ui.Error("invalid resource fmt")
			return 2
		}

		this.doAddResource(mgr, tuples[0], tuples[1])
		return
	}

	// list all resources
	resources, err := mgr.RegisteredResources()
	if err != nil {
		this.Ui.Error(err.Error())
		return
	}

	lines := []string{"InputPlugin|Resources"}
	for _, res := range resources {
		lines = append(lines, fmt.Sprintf("%s|%s", res.InputPlugin, res.Name))
	}
	if len(lines) > 1 {
		this.Ui.Output(columnize.SimpleFormat(lines))
	}

	return
}

func (this *Resources) doAddResource(mgr cluster.Manager, input, resource string) {
	res := cluster.Resource{
		Name:        resource,
		InputPlugin: input,
	}
	if err := mgr.RegisterResource(res); err != nil {
		this.Ui.Error(err.Error())
	} else {
		this.Ui.Info("ok")
	}
}

func (*Resources) Synopsis() string {
	return "Define cluster resources"
}

func (this *Resources) Help() string {
	help := fmt.Sprintf(`
Usage: %s resources [options]

    %s

Options:

    -z zone

    -add input-resource

`, this.Cmd, this.Synopsis())
	return strings.TrimSpace(help)
}
<commit_msg>add sample DSN scheme<commit_after>package command

import (
	"flag"
	"fmt"
	"strings"

	"github.com/funkygao/columnize"
	"github.com/funkygao/dbus/pkg/cluster"
	czk "github.com/funkygao/dbus/pkg/cluster/zk"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gocli"
)

type Resources struct {
	Ui  cli.Ui
	Cmd string

	zone        string
	addResource string
}

func (this *Resources) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("resources", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.addResource, "add", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	mgr := czk.NewManager(ctx.ZoneZkAddrs(this.zone))
	if err := mgr.Open(); err != nil {
		panic(err)
	}
	defer mgr.Close()

	if len(this.addResource) != 0 {
		tuples := strings.SplitN(this.addResource, "-", 2)
		if len(tuples) != 2 {
			this.Ui.Error("invalid resource fmt")
			return 2
		}

		this.doAddResource(mgr, tuples[0], tuples[1])
		return
	}

	// list all resources
	resources, err := mgr.RegisteredResources()
	if err != nil {
		this.Ui.Error(err.Error())
		return
	}

	lines := []string{"InputPlugin|Resources"}
	for _, res := range resources {
		lines = append(lines, fmt.Sprintf("%s|%s", res.InputPlugin, res.Name))
	}
	if len(lines) > 1 {
		this.Ui.Output(columnize.SimpleFormat(lines))
	}

	return
}

func (this *Resources) doAddResource(mgr cluster.Manager, input, resource string) {
	res := cluster.Resource{
		Name:        resource,
		InputPlugin: input,
	}
	if err := mgr.RegisterResource(res); err != nil {
		this.Ui.Error(err.Error())
	} else {
		this.Ui.Info("ok")
	}
}

func (*Resources) Synopsis() string {
	return "Define cluster resources"
}

func (this *Resources) Help() string {
	help := fmt.Sprintf(`
Usage: %s resources [options]

    %s

Options:

    -z zone

    -add input-resource
      resource DSN
      mysql zone://user:pass@host:port/db1,db2,...,dbn
      kafka zone://cluster/topic#partition

`, this.Cmd, this.Synopsis())
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>// Sample
package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/m-lab/etl/bq"
	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/parser"
	"github.com/m-lab/etl/storage"
	"github.com/m-lab/etl/task"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	// Enable profiling. For more background and usage information, see:
	//   https://blog.golang.org/profiling-go-programs
	_ "net/http/pprof"
)

// Task Queue can always submit to an admin restricted URL.
//   login: admin
// Return 200 status code.
// Track reqeusts that last longer than 24 hrs.
// Is task handling idempotent?

// Useful headers added by AppEngine when sending Tasks via Push.
//   X-AppEngine-QueueName
//   X-AppEngine-TaskETA
//   X-AppEngine-TaskName
//   X-AppEngine-TaskRetryCount
//   X-AppEngine-TaskExecutionCount

func handler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.NotFound(w, r)
		return
	}
	fmt.Fprint(w, "Hello world!")
}

func worker(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	// Log request data.
	for key, value := range r.Form {
		log.Printf("Form:   %q == %q\n", key, value)
	}
	// Log headers.
	//for key, value := range r.Header {
	//	log.Printf("Header: %q == %q\n", key, value)
	//}

	// TODO(dev): log the originating task queue name from headers.
	log.Printf("Received filename: %q\n", r.FormValue("filename"))

	// TODO(dev) Create reusable Client.
	tr, err := storage.NewGCSTarReader(nil, r.FormValue("filename"))
	if err != nil {
		log.Printf("%v", err)
		log.Printf("Bailing out")
		fmt.Fprintf(w, `{"message": "Bailing out"}`)
		// TODO - something better.
	}
	parser := new(parser.TestParser)
	ins, err := bq.NewInserter("mlab-sandbox", "mlab_sandbox", "test3")
	if err != nil {
		log.Printf("%v", err)
		log.Printf("Bailing out")
		fmt.Fprintf(w, `{"message": "Bailing out"}`)
		// TODO - something better.
	}
	tsk := task.NewTask(tr, parser, ins, "test3")

	log.Printf("Calling ProcessAllTests")
	tsk.ProcessAllTests()
	log.Printf("Done")
	tr.Close()

	fmt.Fprintf(w, `{"message": "Success"}`)
}

func healthCheckHandler(w http.ResponseWriter, r *http.Request) {
	// TODO(soltesz): provide a real health check.
	fmt.Fprint(w, "ok")
}

func main() {
	http.HandleFunc("/", handler)
	http.HandleFunc("/worker", metrics.DurationHandler("generic", worker))
	http.HandleFunc("/_ah/health", healthCheckHandler)

	// Assign the default prometheus handler to the standard exporter path.
	http.Handle("/metrics", promhttp.Handler())
	http.ListenAndServe(":8080", nil)
}
<commit_msg>Pass client in to task<commit_after>// Sample
package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/m-lab/etl/bq"
	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/parser"
	"github.com/m-lab/etl/storage"
	"github.com/m-lab/etl/task"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	// Enable profiling. For more background and usage information, see:
	//   https://blog.golang.org/profiling-go-programs
	_ "net/http/pprof"
)

// Task Queue can always submit to an admin restricted URL.
//   login: admin
// Return 200 status code.
// Track reqeusts that last longer than 24 hrs.
// Is task handling idempotent?

// Useful headers added by AppEngine when sending Tasks via Push.
//   X-AppEngine-QueueName
//   X-AppEngine-TaskETA
//   X-AppEngine-TaskName
//   X-AppEngine-TaskRetryCount
//   X-AppEngine-TaskExecutionCount

func handler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.NotFound(w, r)
		return
	}
	fmt.Fprint(w, "Hello world!")
}

func worker(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	// Log request data.
	for key, value := range r.Form {
		log.Printf("Form:   %q == %q\n", key, value)
	}
	// Log headers.
	//for key, value := range r.Header {
	//	log.Printf("Header: %q == %q\n", key, value)
	//}

	// TODO(dev): log the originating task queue name from headers.
	log.Printf("Received filename: %q\n", r.FormValue("filename"))

	client, err := storage.GetStorageClient(false)
	if err != nil {
		fmt.Fprintf(w, `{"message": "Could not create client."}`)
		w.WriteHeader(503) // Service Unavailable
		return
	}

	// TODO(dev) Create reusable Client.
	tr, err := storage.NewGCSTarReader(client, r.FormValue("filename"))
	if err != nil {
		log.Printf("%v", err)
		log.Printf("Bailing out")
		fmt.Fprintf(w, `{"message": "Bailing out"}`)
		return
		// TODO - something better.
	}
	parser := new(parser.TestParser)
	ins, err := bq.NewInserter("mlab-sandbox", "mlab_sandbox", "test3")
	if err != nil {
		log.Printf("%v", err)
		log.Printf("Bailing out")
		fmt.Fprintf(w, `{"message": "Bailing out"}`)
		return
		// TODO - something better.
	}
	tsk := task.NewTask(tr, parser, ins, "test3")

	log.Printf("Calling ProcessAllTests")
	tsk.ProcessAllTests()
	log.Printf("Done")
	tr.Close()

	// TODO - if there are any errors, consider sending back a meaningful response
	// for web browser and queue-pusher debugging.
	fmt.Fprintf(w, `{"message": "Success"}`)
}

func healthCheckHandler(w http.ResponseWriter, r *http.Request) {
	// TODO(soltesz): provide a real health check.
	fmt.Fprint(w, "ok")
}

func main() {
	http.HandleFunc("/", handler)
	http.HandleFunc("/worker", metrics.DurationHandler("generic", worker))
	http.HandleFunc("/_ah/health", healthCheckHandler)

	// Assign the default prometheus handler to the standard exporter path.
	http.Handle("/metrics", promhttp.Handler())
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"k8s.io/kops/upup/pkg/fi/cloudup"

	"github.com/ghodss/yaml"
	"github.com/spf13/cobra"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/klog"
	"k8s.io/kops/cmd/kops/util"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/validation"
	"k8s.io/kops/util/pkg/tables"
)

type ValidateClusterOptions struct {
	output     string
	wait       time.Duration
	count      int
	kubeconfig string
}

func (o *ValidateClusterOptions) InitDefaults() {
	o.output = OutputTable
}

func NewCmdValidateCluster(f *util.Factory, out io.Writer) *cobra.Command {
	options := &ValidateClusterOptions{}
	options.InitDefaults()

	cmd := &cobra.Command{
		Use:     "cluster",
		Short:   validateShort,
		Long:    validateLong,
		Example: validateExample,
		Run: func(cmd *cobra.Command, args []string) {
			result, err := RunValidateCluster(f, cmd, args, os.Stdout, options)
			if err != nil {
				exitWithError(fmt.Errorf("Validation failed: %v", err))
			}
			// We want the validate command to exit non-zero if validation found a problem,
			// even if we didn't really hit an error during validation.
			if len(result.Failures) != 0 {
				os.Exit(2)
			}
		},
	}

	cmd.Flags().StringVarP(&options.output, "output", "o", options.output, "Output format. One of json|yaml|table.")
	cmd.Flags().DurationVar(&options.wait, "wait", options.wait, "If set, will wait for cluster to be ready")
	cmd.Flags().IntVar(&options.count, "count", options.count, "If set, will validate the cluster consecutive times")
	cmd.Flags().StringVar(&options.kubeconfig, "kubeconfig", "", "Path to the kubeconfig file")

	return cmd
}

func RunValidateCluster(f *util.Factory, cmd *cobra.Command, args []string, out io.Writer, options *ValidateClusterOptions) (*validation.ValidationCluster, error) {
	err := rootCommand.ProcessArgs(args)
	if err != nil {
		return nil, err
	}

	cluster, err := rootCommand.Cluster()
	if err != nil {
		return nil, err
	}

	cloud, err := cloudup.BuildCloud(cluster)
	if err != nil {
		return nil, err
	}

	clientSet, err := f.Clientset()
	if err != nil {
		return nil, err
	}

	list, err := clientSet.InstanceGroupsFor(cluster).List(metav1.ListOptions{})
	if err != nil {
		return nil, fmt.Errorf("cannot get InstanceGroups for %q: %v", cluster.ObjectMeta.Name, err)
	}

	if options.output == OutputTable {
		fmt.Fprintf(out, "Validating cluster %v\n\n", cluster.ObjectMeta.Name)
	}

	var instanceGroups []api.InstanceGroup
	for _, ig := range list.Items {
		instanceGroups = append(instanceGroups, ig)
		klog.V(2).Infof("instance group: %#v\n\n", ig.Spec)
	}

	if len(instanceGroups) == 0 {
		return nil, fmt.Errorf("no InstanceGroup objects found")
	}

	// TODO: Refactor into util.Factory
	contextName := cluster.ObjectMeta.Name
	configLoadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
	if options.kubeconfig != "" {
		configLoadingRules.ExplicitPath = options.kubeconfig
	}
	config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		configLoadingRules,
		&clientcmd.ConfigOverrides{CurrentContext: contextName}).ClientConfig()
	if err != nil {
		return nil, fmt.Errorf("cannot load kubecfg settings for %q: %v", contextName, err)
	}

	k8sClient, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, fmt.Errorf("cannot build kubernetes api client for %q: %v", contextName, err)
	}

	timeout := time.Now().Add(options.wait)
	pollInterval := 10 * time.Second

	validator, err := validation.NewClusterValidator(cluster, cloud, list, k8sClient)
	if err != nil {
		return nil, fmt.Errorf("unexpected error creating validatior: %v", err)
	}

	consecutive := 0
	for {
		if options.wait > 0 && time.Now().After(timeout) {
			return nil, fmt.Errorf("wait time exceeded during validation")
		}

		result, err := validator.Validate()
		if err != nil {
			consecutive = 0
			if options.wait > 0 {
				klog.Warningf("(will retry): unexpected error during validation: %v", err)
				time.Sleep(pollInterval)
				continue
			} else {
				return nil, fmt.Errorf("unexpected error during validation: %v", err)
			}
		}

		switch options.output {
		case OutputTable:
			if err := validateClusterOutputTable(result, cluster, instanceGroups, out); err != nil {
				return nil, err
			}
		case OutputYaml:
			y, err := yaml.Marshal(result)
			if err != nil {
				return nil, fmt.Errorf("unable to marshal YAML: %v", err)
			}
			if _, err := out.Write(y); err != nil {
				return nil, fmt.Errorf("error writing to output: %v", err)
			}
		case OutputJSON:
			j, err := json.Marshal(result)
			if err != nil {
				return nil, fmt.Errorf("unable to marshal JSON: %v", err)
			}
			if _, err := out.Write(j); err != nil {
				return nil, fmt.Errorf("error writing to output: %v", err)
			}
		default:
			return nil, fmt.Errorf("unknown output format: %q", options.output)
		}

		if len(result.Failures) == 0 {
			consecutive++
			if consecutive < options.count {
				klog.Infof("(will retry): cluster passed validation %d consecutive times", consecutive)
				if options.wait > 0 {
					time.Sleep(pollInterval)
					continue
				} else {
					return nil, fmt.Errorf("cluster passed validation %d consecutive times", consecutive)
				}
			} else {
				return result, nil
			}
		} else {
			consecutive = 0
			if options.wait > 0 {
				klog.Warningf("(will retry): cluster not yet healthy")
				time.Sleep(pollInterval)
				continue
			} else {
				return nil, fmt.Errorf("cluster not yet healthy")
			}
		}
	}
}

func validateClusterOutputTable(result *validation.ValidationCluster, cluster *api.Cluster, instanceGroups []api.InstanceGroup, out io.Writer) error {
	t := &tables.Table{}
	t.AddColumn("NAME", func(c api.InstanceGroup) string {
		return c.ObjectMeta.Name
	})
	t.AddColumn("ROLE", func(c api.InstanceGroup) string {
		return string(c.Spec.Role)
	})
	t.AddColumn("MACHINETYPE", func(c api.InstanceGroup) string {
		return c.Spec.MachineType
	})
	t.AddColumn("SUBNETS", func(c api.InstanceGroup) string {
		return strings.Join(c.Spec.Subnets, ",")
	})
	t.AddColumn("MIN", func(c api.InstanceGroup) string {
		return int32PointerToString(c.Spec.MinSize)
	})
	t.AddColumn("MAX", func(c api.InstanceGroup) string {
		return int32PointerToString(c.Spec.MaxSize)
	})

	fmt.Fprintln(out, "INSTANCE GROUPS")
	err := t.Render(instanceGroups, out, "NAME", "ROLE", "MACHINETYPE", "MIN", "MAX", "SUBNETS")
	if err != nil {
		return fmt.Errorf("cannot render nodes for %q: %v", cluster.Name, err)
	}

	{
		nodeTable := &tables.Table{}
		nodeTable.AddColumn("NAME", func(n *validation.ValidationNode) string {
			return n.Name
		})

		nodeTable.AddColumn("READY", func(n *validation.ValidationNode) v1.ConditionStatus {
			return n.Status
		})

		nodeTable.AddColumn("ROLE", func(n *validation.ValidationNode) string {
			return n.Role
		})

		fmt.Fprintln(out, "\nNODE STATUS")
		if err := nodeTable.Render(result.Nodes, out, "NAME", "ROLE", "READY"); err != nil {
			return fmt.Errorf("cannot render nodes for %q: %v", cluster.Name, err)
		}
	}

	if len(result.Failures) != 0 {
		failuresTable := &tables.Table{}
		failuresTable.AddColumn("KIND", func(e *validation.ValidationError) string {
			return e.Kind
		})
		failuresTable.AddColumn("NAME", func(e *validation.ValidationError) string {
			return e.Name
		})
		failuresTable.AddColumn("MESSAGE", func(e *validation.ValidationError) string {
			return e.Message
		})

		fmt.Fprintln(out, "\nVALIDATION ERRORS")
		if err := failuresTable.Render(result.Failures, out, "KIND", "NAME", "MESSAGE"); err != nil {
			return fmt.Errorf("error rendering failures table: %v", err)
		}

		fmt.Fprintf(out, "\nValidation Failed\n")
	} else {
		fmt.Fprintf(out, "\nYour cluster %s is ready\n", cluster.Name)
	}

	return nil
}
<commit_msg>Fail validation if any consecutive validation fails<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"k8s.io/kops/upup/pkg/fi/cloudup"

	"github.com/ghodss/yaml"
	"github.com/spf13/cobra"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/klog"
	"k8s.io/kops/cmd/kops/util"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/validation"
	"k8s.io/kops/util/pkg/tables"
)

type ValidateClusterOptions struct {
	output     string
	wait       time.Duration
	count      int
	kubeconfig string
}

func (o *ValidateClusterOptions) InitDefaults() {
	o.output = OutputTable
}

func NewCmdValidateCluster(f *util.Factory, out io.Writer) *cobra.Command {
	options := &ValidateClusterOptions{}
	options.InitDefaults()

	cmd := &cobra.Command{
		Use:     "cluster",
		Short:   validateShort,
		Long:    validateLong,
		Example: validateExample,
		Run: func(cmd *cobra.Command, args []string) {
			result, err := RunValidateCluster(f, cmd, args, os.Stdout, options)
			if err != nil {
				exitWithError(fmt.Errorf("Validation failed: %v", err))
			}
			// We want the validate command to exit non-zero if validation found a problem,
			// even if we didn't really hit an error during validation.
			if len(result.Failures) != 0 {
				os.Exit(2)
			}
		},
	}

	cmd.Flags().StringVarP(&options.output, "output", "o", options.output, "Output format. One of json|yaml|table.")
	cmd.Flags().DurationVar(&options.wait, "wait", options.wait, "If set, will wait for cluster to be ready")
	cmd.Flags().IntVar(&options.count, "count", options.count, "If set, will validate the cluster consecutive times")
	cmd.Flags().StringVar(&options.kubeconfig, "kubeconfig", "", "Path to the kubeconfig file")

	return cmd
}

func RunValidateCluster(f *util.Factory, cmd *cobra.Command, args []string, out io.Writer, options *ValidateClusterOptions) (*validation.ValidationCluster, error) {
	err := rootCommand.ProcessArgs(args)
	if err != nil {
		return nil, err
	}

	cluster, err := rootCommand.Cluster()
	if err != nil {
		return nil, err
	}

	cloud, err := cloudup.BuildCloud(cluster)
	if err != nil {
		return nil, err
	}

	clientSet, err := f.Clientset()
	if err != nil {
		return nil, err
	}

	list, err := clientSet.InstanceGroupsFor(cluster).List(metav1.ListOptions{})
	if err != nil {
		return nil, fmt.Errorf("cannot get InstanceGroups for %q: %v", cluster.ObjectMeta.Name, err)
	}

	if options.output == OutputTable {
		fmt.Fprintf(out, "Validating cluster %v\n\n", cluster.ObjectMeta.Name)
	}

	var instanceGroups []api.InstanceGroup
	for _, ig := range list.Items {
		instanceGroups = append(instanceGroups, ig)
		klog.V(2).Infof("instance group: %#v\n\n", ig.Spec)
	}

	if len(instanceGroups) == 0 {
		return nil, fmt.Errorf("no InstanceGroup objects found")
	}

	// TODO: Refactor into util.Factory
	contextName := cluster.ObjectMeta.Name
	configLoadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
	if options.kubeconfig != "" {
		configLoadingRules.ExplicitPath = options.kubeconfig
	}
	config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		configLoadingRules,
		&clientcmd.ConfigOverrides{CurrentContext: contextName}).ClientConfig()
	if err != nil {
		return nil, fmt.Errorf("cannot load kubecfg settings for %q: %v", contextName, err)
	}

	k8sClient, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, fmt.Errorf("cannot build kubernetes api client for %q: %v", contextName, err)
	}

	timeout := time.Now().Add(options.wait)
	pollInterval := 10 * time.Second

	validator, err := validation.NewClusterValidator(cluster, cloud, list, k8sClient)
	if err != nil {
		return nil, fmt.Errorf("unexpected error creating validatior: %v", err)
	}

	consecutive := 0
	for {
		if options.wait > 0 && time.Now().After(timeout) {
			return nil, fmt.Errorf("wait time exceeded during validation")
		}

		result, err := validator.Validate()
		if err != nil {
			consecutive = 0
			if options.wait > 0 {
				klog.Warningf("(will retry): unexpected error during validation: %v", err)
				time.Sleep(pollInterval)
				continue
			} else {
				return nil, fmt.Errorf("unexpected error during validation: %v", err)
			}
		}

		switch options.output {
		case OutputTable:
			if err := validateClusterOutputTable(result, cluster, instanceGroups, out); err != nil {
				return nil, err
			}
		case OutputYaml:
			y, err := yaml.Marshal(result)
			if err != nil {
				return nil, fmt.Errorf("unable to marshal YAML: %v", err)
			}
			if _, err := out.Write(y); err != nil {
				return nil, fmt.Errorf("error writing to output: %v", err)
			}
		case OutputJSON:
			j, err := json.Marshal(result)
			if err != nil {
				return nil, fmt.Errorf("unable to marshal JSON: %v", err)
			}
			if _, err := out.Write(j); err != nil {
				return nil, fmt.Errorf("error writing to output: %v", err)
			}
		default:
			return nil, fmt.Errorf("unknown output format: %q", options.output)
		}

		if len(result.Failures) == 0 {
			consecutive++
			if consecutive < options.count {
				klog.Infof("(will retry): cluster passed validation %d consecutive times", consecutive)
				if options.wait > 0 {
					time.Sleep(pollInterval)
					continue
				} else {
					return nil, fmt.Errorf("cluster passed validation %d consecutive times", consecutive)
				}
			} else {
				return result, nil
			}
		} else {
			if options.wait > 0 && consecutive == 0 {
				klog.Warningf("(will retry): cluster not yet healthy")
				time.Sleep(pollInterval)
				continue
			} else {
				return nil, fmt.Errorf("cluster not yet healthy")
			}
		}
	}
}

func validateClusterOutputTable(result *validation.ValidationCluster, cluster *api.Cluster, instanceGroups []api.InstanceGroup, out io.Writer) error {
	t := &tables.Table{}
	t.AddColumn("NAME", func(c api.InstanceGroup) string {
		return c.ObjectMeta.Name
	})
	t.AddColumn("ROLE", func(c api.InstanceGroup) string {
		return string(c.Spec.Role)
	})
	t.AddColumn("MACHINETYPE", func(c api.InstanceGroup) string {
		return c.Spec.MachineType
	})
	t.AddColumn("SUBNETS", func(c api.InstanceGroup) string {
		return strings.Join(c.Spec.Subnets, ",")
	})
	t.AddColumn("MIN", func(c api.InstanceGroup) string {
		return int32PointerToString(c.Spec.MinSize)
	})
	t.AddColumn("MAX", func(c api.InstanceGroup) string {
		return int32PointerToString(c.Spec.MaxSize)
	})

	fmt.Fprintln(out, "INSTANCE GROUPS")
	err := t.Render(instanceGroups, out, "NAME", "ROLE", "MACHINETYPE", "MIN", "MAX", "SUBNETS")
	if err != nil {
		return fmt.Errorf("cannot render nodes for %q: %v", cluster.Name, err)
	}

	{
		nodeTable := &tables.Table{}
		nodeTable.AddColumn("NAME", func(n *validation.ValidationNode) string {
			return n.Name
		})

		nodeTable.AddColumn("READY", func(n *validation.ValidationNode) v1.ConditionStatus {
			return n.Status
		})

		nodeTable.AddColumn("ROLE", func(n *validation.ValidationNode) string {
			return n.Role
		})

		fmt.Fprintln(out, "\nNODE STATUS")
		if err := nodeTable.Render(result.Nodes, out, "NAME", "ROLE", "READY"); err != nil {
			return fmt.Errorf("cannot render nodes for %q: %v", cluster.Name, err)
		}
	}

	if len(result.Failures) != 0 {
		failuresTable := &tables.Table{}
		failuresTable.AddColumn("KIND", func(e *validation.ValidationError) string {
			return e.Kind
		})
		failuresTable.AddColumn("NAME", func(e *validation.ValidationError) string {
			return e.Name
		})
		failuresTable.AddColumn("MESSAGE", func(e *validation.ValidationError) string {
			return e.Message
		})

		fmt.Fprintln(out, "\nVALIDATION ERRORS")
		if err := failuresTable.Render(result.Failures, out, "KIND", "NAME", "MESSAGE"); err != nil {
			return fmt.Errorf("error rendering failures table: %v", err)
		}

		fmt.Fprintf(out, "\nValidation Failed\n")
	} else {
		fmt.Fprintf(out, "\nYour cluster %s is ready\n", cluster.Name)
	}

	return nil
}
<|endoftext|>


<commit_before>// Command pkg-diff-example implements a subset of the diff command using
// github.com/pkg/diff.
package main

import (
	"bufio"
	"context"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"

	"github.com/pkg/diff"
)

var (
	color   = flag.Bool("color", false, "colorize the output")
	timeout = flag.Duration("timeout", 0, "timeout")
	unified = flag.Int("unified", 3, "lines of unified context")
)

// check logs a fatal error and exits if err is not nil.
func check(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

// fileLines returns the lines of the file called name.
func fileLines(name string) ([]string, error) {
	f, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	var lines []string
	s := bufio.NewScanner(f)
	for s.Scan() {
		lines = append(lines, s.Text())
	}
	return lines, s.Err()
}

func main() {
	flag.Parse()
	if len(flag.Args()) != 2 {
		fmt.Printf("syntax: %s name1 name2\n", filepath.Base(os.Args[0]))
		os.Exit(2)
	}

	aName := flag.Arg(0)
	aLines, err := fileLines(aName)
	check(err)

	bName := flag.Arg(1)
	bLines, err := fileLines(bName)
	check(err)

	ab := diff.Strings(aLines, bLines)
	ctx := context.Background()
	if *timeout != 0 {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, *timeout)
		defer cancel()
	}
	e := diff.Myers(ctx, ab)
	opts := []diff.WriteOpt{
		diff.Names(aName, bName),
	}
	if *color {
		opts = append(opts, diff.TerminalColor())
	}
	_, err = e.WithContextSize(*unified).WriteUnified(os.Stdout, ab, opts...)
	check(err)
}
<commit_msg>cmd/pkg-diff-example: make a few minor tweaks<commit_after>// Command pkg-diff-example implements a subset of the diff command using
// github.com/pkg/diff.
package main

import (
	"bufio"
	"context"
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/pkg/diff"
)

var (
	color   = flag.Bool("color", false, "colorize the output")
	timeout = flag.Duration("timeout", 0, "timeout")
	unified = flag.Int("unified", 3, "lines of unified context")
)

// check logs a fatal error and exits if err is not nil.
func check(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

// fileLines returns the lines of the file called name.
func fileLines(name string) ([]string, error) {
	f, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	var lines []string
	s := bufio.NewScanner(f)
	for s.Scan() {
		lines = append(lines, s.Text())
	}
	return lines, s.Err()
}

func usage() {
	fmt.Fprintf(os.Stderr, "pkg-diff-example [flags] file1 file2\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	log.SetPrefix("pkg-diff-example: ")
	log.SetFlags(0)

	flag.Usage = usage
	flag.Parse()
	if len(flag.Args()) != 2 {
		flag.Usage()
	}

	aName := flag.Arg(0)
	aLines, err := fileLines(aName)
	check(err)

	bName := flag.Arg(1)
	bLines, err := fileLines(bName)
	check(err)

	ab := diff.Strings(aLines, bLines)
	ctx := context.Background()
	if *timeout != 0 {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, *timeout)
		defer cancel()
	}
	e := diff.Myers(ctx, ab)
	e = e.WithContextSize(*unified) // limit amount of output context
	opts := []diff.WriteOpt{
		diff.Names(aName, bName),
	}
	if *color {
		opts = append(opts, diff.TerminalColor())
	}
	_, err = e.WriteUnified(os.Stdout, ab, opts...)
	check(err)
}
<|endoftext|>


<commit_before>package main

func main() {
	//Questions: simplify server's cache with hashmap? how to handle negative names?

	//Generate zonefiles
	//Generate Traces
	//Generate Mapping from IP to channel
	//Initialize and start authoritative server and load zonefile.
	//Initialize caching resolvers with the correct public and private keys and root server addr (channel)
	//Optional: load some values into the caching resolver's cache
	//Start caching resolver
	//Start clients with trace => (start a go routine that issues a new go routine that sends the query
	//in the client's name and tracks how long it takes to get an answer.)
}
<commit_msg>STart servers and clients in simulation<commit_after>package main

import "github.com/netsec-ethz/rains/internal/pkg/rainsd"

func main() {
	nofNamingServers := 2
	nofResolvers := 1
	nofClients := 5
	idToServer := make(map[int]*rainsd.Server)

	for i := 0; i < nofNamingServers; i++ {
		server, err := rainsd.New("config/namingServer.conf", log.LvlDebug, i)
		idToServer[i] = server
		go server.Start(false)
	}
	for i := 0; i < nofResolvers; i++ {
		server, err := rainsd.New("config/resolver.conf", log.LvlDebug, i)
		idToServer[i] = server
		go server.Start(false)
	}
	for i := 0; i < nofClients; i++ {
		go startClient()
	}

	//Generate zonefiles
	//Generate Traces
	//Generate Mapping from IP to channel
	//Initialize and start authoritative server and load zonefile.
	//Initialize caching resolvers with the correct public and private keys and root server addr (channel)
	//Optional: load some values into the caching resolver's cache
	//Start caching resolver
	//Start clients with trace => (start a go routine that issues a new go routine that sends the query
	//in the client's name and tracks how long it takes to get an answer.)
}

func startClient() {
	//send queries based on trace. log delay
}
<|endoftext|>