<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package cache

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"

	"github.com/golang/groupcache/lru"
	"github.com/pkg/errors"
)

const (
	defaultMaxLRUCacheEntry = 10
	defaultMaxCacheFds      = 10
)

type DirectoryCacheConfig struct {
	MaxLRUCacheEntry int  `toml:"max_lru_cache_entry"`
	MaxCacheFds      int  `toml:"max_cache_fds"`
	SyncAdd          bool `toml:"sync_add"`
}

// TODO: contents validation.

type BlobCache interface {
	Add(key string, p []byte, opts ...Option)
	FetchAt(key string, offset int64, p []byte, opts ...Option) (n int, err error)
}

type cacheOpt struct {
	direct bool
}

type Option func(o *cacheOpt) *cacheOpt

// When Direct option is specified for FetchAt and Add methods, these operation
// won't use on-memory caches. When you know that the targeting value won't be
// used immediately, you can prevent the limited space of on-memory caches from
// being polluted by these unimportant values.
func Direct() Option {
	return func(o *cacheOpt) *cacheOpt {
		o.direct = true
		return o
	}
}

func NewDirectoryCache(directory string, config DirectoryCacheConfig) (BlobCache, error) {
	maxEntry := config.MaxLRUCacheEntry
	if maxEntry == 0 {
		maxEntry = defaultMaxLRUCacheEntry
	}
	maxFds := config.MaxCacheFds
	if maxFds == 0 {
		maxFds = defaultMaxCacheFds
	}
	if err := os.MkdirAll(directory, os.ModePerm); err != nil {
		return nil, err
	}
	dc := &directoryCache{
		cache:     newObjectCache(maxEntry),
		fileCache: newObjectCache(maxFds),
		directory: directory,
		bufPool: sync.Pool{
			New: func() interface{} {
				return new(bytes.Buffer)
			},
		},
	}
	dc.cache.finalize = func(value interface{}) {
		dc.bufPool.Put(value)
	}
	dc.fileCache.finalize = func(value interface{}) {
		value.(*os.File).Close()
	}
	dc.syncAdd = config.SyncAdd
	return dc, nil
}

// directoryCache is a cache implementation which backend is a directory.
type directoryCache struct {
	cache     *objectCache
	fileCache *objectCache
	directory string

	bufPool sync.Pool

	syncAdd bool
}

func (dc *directoryCache) FetchAt(key string, offset int64, p []byte, opts ...Option) (n int, err error) {
	opt := &cacheOpt{}
	for _, o := range opts {
		opt = o(opt)
	}

	if !opt.direct {
		// Get data from memory
		if b, done, ok := dc.cache.get(key); ok {
			defer done()
			data := b.(*bytes.Buffer).Bytes()
			if int64(len(data)) < offset {
				return 0, fmt.Errorf("invalid offset %d exceeds chunk size %d",
					offset, len(data))
			}
			return copy(p, data[offset:]), nil
		}

		// Get data from disk. If the file is already opened, use it.
		if f, done, ok := dc.fileCache.get(key); ok {
			defer done()
			return f.(*os.File).ReadAt(p, offset)
		}
	}

	// Open the cache file and read the target region
	// TODO: If the target cache is write-in-progress, should we wait for the completion
	//       or simply report the cache miss?
	file, err := os.Open(dc.cachePath(key))
	if err != nil {
		return 0, errors.Wrapf(err, "failed to open blob file for %q", key)
	}
	if n, err = file.ReadAt(p, offset); err == io.EOF {
		err = nil
	}

	// Cache the opened file for future use. If "direct" option is specified, this
	// won't be done. This option is useful for preventing file cache from being
	// polluted by data that won't be accessed immediately.
	if opt.direct || !dc.fileCache.add(key, file) {
		file.Close()
	}

	// TODO: should we cache the entire file data on memory?
	//       but making I/O (possibly huge) on every fetching
	//       might be costly.

	return n, err
}

func (dc *directoryCache) Add(key string, p []byte, opts ...Option) {
	opt := &cacheOpt{}
	for _, o := range opts {
		opt = o(opt)
	}

	if !opt.direct {
		// Cache the passed data on memory. This enables to serve this data even
		// during writing it to the disk. If "direct" option is specified, this
		// won't be done. This option is useful for preventing memory cache from being
		// polluted by data that won't be accessed immediately.
		b := dc.bufPool.Get().(*bytes.Buffer)
		b.Reset()
		b.Write(p)
		if !dc.cache.add(key, b) {
			dc.bufPool.Put(b) // Already exists. No need to cache.
		}
	}

	// Cache the passed data to disk.
	b2 := dc.bufPool.Get().(*bytes.Buffer)
	b2.Reset()
	b2.Write(p)
	addFunc := func() {
		defer dc.bufPool.Put(b2)

		var (
			c   = dc.cachePath(key)
			wip = dc.wipPath(key)
		)
		if _, err := os.Stat(wip); err == nil {
			return // Write in progress
		}
		if _, err := os.Stat(c); err == nil {
			return // Already exists.
		}

		// Write the contents to a temporary file
		if err := os.MkdirAll(filepath.Dir(wip), os.ModePerm); err != nil {
			fmt.Printf("Warning: Failed to Create blob cache directory %q: %v\n", c, err)
			return
		}
		wipfile, err := os.Create(wip)
		if err != nil {
			fmt.Printf("Warning: failed to prepare temp file for storing cache %q", key)
			return
		}
		defer func() {
			wipfile.Close()
			os.Remove(wipfile.Name())
		}()
		want := b2.Len()
		if _, err := io.CopyN(wipfile, b2, int64(want)); err != nil {
			fmt.Printf("Warning: failed to write cache: %v\n", err)
			return
		}

		// Commit the cache contents
		if err := os.MkdirAll(filepath.Dir(c), os.ModePerm); err != nil {
			fmt.Printf("Warning: Failed to Create blob cache directory %q: %v\n", c, err)
			return
		}
		if err := os.Rename(wipfile.Name(), c); err != nil {
			fmt.Printf("Warning: failed to commit cache to %q: %v\n", c, err)
			return
		}
		file, err := os.Open(c)
		if err != nil {
			fmt.Printf("Warning: failed to open cache on %q: %v\n", c, err)
			return
		}

		// Cache the opened file for future use. If "direct" option is specified, this
		// won't be done. This option is useful for preventing file cache from being
		// polluted by data that won't be accessed immediately.
		if opt.direct || !dc.fileCache.add(key, file) {
			file.Close()
		}
	}

	if dc.syncAdd {
		addFunc()
	} else {
		go addFunc()
	}
}

func (dc *directoryCache) cachePath(key string) string {
	return filepath.Join(dc.directory, key[:2], key)
}

func (dc *directoryCache) wipPath(key string) string {
	return filepath.Join(dc.directory, key[:2], "w", key)
}

func newObjectCache(maxEntries int) *objectCache {
	oc := &objectCache{
		cache: lru.New(maxEntries),
	}
	oc.cache.OnEvicted = func(key lru.Key, value interface{}) {
		value.(*object).release() // Decrease ref count incremented in add operation.
	}
	return oc
}

type objectCache struct {
	cache    *lru.Cache
	cacheMu  sync.Mutex
	finalize func(interface{})
}

func (oc *objectCache) get(key string) (value interface{}, done func(), ok bool) {
	oc.cacheMu.Lock()
	defer oc.cacheMu.Unlock()
	o, ok := oc.cache.Get(key)
	if !ok {
		return nil, nil, false
	}
	o.(*object).use()
	return o.(*object).v, func() { o.(*object).release() }, true
}

func (oc *objectCache) add(key string, value interface{}) bool {
	oc.cacheMu.Lock()
	defer oc.cacheMu.Unlock()
	if _, ok := oc.cache.Get(key); ok {
		return false // TODO: should we swap the object?
	}
	o := &object{
		v:        value,
		finalize: oc.finalize,
	}
	o.use() // Keep this object having at least 1 ref count (will be decreased on eviction)
	oc.cache.Add(key, o)
	return true
}

type object struct {
	v interface{}

	refCounts int64
	finalize  func(interface{})

	mu sync.Mutex
}

func (o *object) use() {
	o.mu.Lock()
	defer o.mu.Unlock()
	o.refCounts++
}

func (o *object) release() {
	o.mu.Lock()
	defer o.mu.Unlock()
	o.refCounts--
	if o.refCounts <= 0 && o.finalize != nil {
		// nobody will refer this object
		o.finalize(o.v)
	}
}

func NewMemoryCache() BlobCache {
	return &memoryCache{
		membuf: map[string]string{},
	}
}

// memoryCache is a cache implementation which backend is a memory.
type memoryCache struct {
	membuf map[string]string // read-only []byte map is more ideal but we don't have it in golang...
	mu     sync.Mutex
}

func (mc *memoryCache) FetchAt(key string, offset int64, p []byte, opts ...Option) (n int, err error) {
	mc.mu.Lock()
	defer mc.mu.Unlock()

	cache, ok := mc.membuf[key]
	if !ok {
		return 0, fmt.Errorf("Missed cache: %q", key)
	}
	return copy(p, cache[offset:]), nil
}

func (mc *memoryCache) Add(key string, p []byte, opts ...Option) {
	mc.mu.Lock()
	defer mc.mu.Unlock()
	mc.membuf[key] = string(p)
}
<commit_msg>Avoid race condition on writing cache contents to files<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package cache

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"

	"github.com/golang/groupcache/lru"
	"github.com/pkg/errors"
)

const (
	defaultMaxLRUCacheEntry = 10
	defaultMaxCacheFds      = 10
)

type DirectoryCacheConfig struct {
	MaxLRUCacheEntry int  `toml:"max_lru_cache_entry"`
	MaxCacheFds      int  `toml:"max_cache_fds"`
	SyncAdd          bool `toml:"sync_add"`
}

// TODO: contents validation.

type BlobCache interface {
	Add(key string, p []byte, opts ...Option)
	FetchAt(key string, offset int64, p []byte, opts ...Option) (n int, err error)
}

type cacheOpt struct {
	direct bool
}

type Option func(o *cacheOpt) *cacheOpt

// When Direct option is specified for FetchAt and Add methods, these operation
// won't use on-memory caches. When you know that the targeting value won't be
// used immediately, you can prevent the limited space of on-memory caches from
// being polluted by these unimportant values.
func Direct() Option {
	return func(o *cacheOpt) *cacheOpt {
		o.direct = true
		return o
	}
}

func NewDirectoryCache(directory string, config DirectoryCacheConfig) (BlobCache, error) {
	maxEntry := config.MaxLRUCacheEntry
	if maxEntry == 0 {
		maxEntry = defaultMaxLRUCacheEntry
	}
	maxFds := config.MaxCacheFds
	if maxFds == 0 {
		maxFds = defaultMaxCacheFds
	}
	if err := os.MkdirAll(directory, os.ModePerm); err != nil {
		return nil, err
	}
	dc := &directoryCache{
		cache:     newObjectCache(maxEntry),
		fileCache: newObjectCache(maxFds),
		wipLock:   &namedLock{},
		directory: directory,
		bufPool: sync.Pool{
			New: func() interface{} {
				return new(bytes.Buffer)
			},
		},
	}
	dc.cache.finalize = func(value interface{}) {
		dc.bufPool.Put(value)
	}
	dc.fileCache.finalize = func(value interface{}) {
		value.(*os.File).Close()
	}
	dc.syncAdd = config.SyncAdd
	return dc, nil
}

// directoryCache is a cache implementation which backend is a directory.
type directoryCache struct {
	cache     *objectCache
	fileCache *objectCache
	directory string
	wipLock   *namedLock

	bufPool sync.Pool

	syncAdd bool
}

func (dc *directoryCache) FetchAt(key string, offset int64, p []byte, opts ...Option) (n int, err error) {
	opt := &cacheOpt{}
	for _, o := range opts {
		opt = o(opt)
	}

	if !opt.direct {
		// Get data from memory
		if b, done, ok := dc.cache.get(key); ok {
			defer done()
			data := b.(*bytes.Buffer).Bytes()
			if int64(len(data)) < offset {
				return 0, fmt.Errorf("invalid offset %d exceeds chunk size %d",
					offset, len(data))
			}
			return copy(p, data[offset:]), nil
		}

		// Get data from disk. If the file is already opened, use it.
		if f, done, ok := dc.fileCache.get(key); ok {
			defer done()
			return f.(*os.File).ReadAt(p, offset)
		}
	}

	// Open the cache file and read the target region
	// TODO: If the target cache is write-in-progress, should we wait for the completion
	//       or simply report the cache miss?
	file, err := os.Open(dc.cachePath(key))
	if err != nil {
		return 0, errors.Wrapf(err, "failed to open blob file for %q", key)
	}
	if n, err = file.ReadAt(p, offset); err == io.EOF {
		err = nil
	}

	// Cache the opened file for future use. If "direct" option is specified, this
	// won't be done. This option is useful for preventing file cache from being
	// polluted by data that won't be accessed immediately.
	if opt.direct || !dc.fileCache.add(key, file) {
		file.Close()
	}

	// TODO: should we cache the entire file data on memory?
	//       but making I/O (possibly huge) on every fetching
	//       might be costly.

	return n, err
}

func (dc *directoryCache) Add(key string, p []byte, opts ...Option) {
	opt := &cacheOpt{}
	for _, o := range opts {
		opt = o(opt)
	}

	if !opt.direct {
		// Cache the passed data on memory. This enables to serve this data even
		// during writing it to the disk. If "direct" option is specified, this
		// won't be done. This option is useful for preventing memory cache from being
		// polluted by data that won't be accessed immediately.
		b := dc.bufPool.Get().(*bytes.Buffer)
		b.Reset()
		b.Write(p)
		if !dc.cache.add(key, b) {
			dc.bufPool.Put(b) // Already exists. No need to cache.
		}
	}

	// Cache the passed data to disk.
	b2 := dc.bufPool.Get().(*bytes.Buffer)
	b2.Reset()
	b2.Write(p)
	addFunc := func() {
		defer dc.bufPool.Put(b2)

		var (
			c   = dc.cachePath(key)
			wip = dc.wipPath(key)
		)

		dc.wipLock.lock(key)
		if _, err := os.Stat(wip); err == nil {
			dc.wipLock.unlock(key)
			return // Write in progress
		}
		if _, err := os.Stat(c); err == nil {
			dc.wipLock.unlock(key)
			return // Already exists.
		}

		// Write the contents to a temporary file
		if err := os.MkdirAll(filepath.Dir(wip), os.ModePerm); err != nil {
			fmt.Printf("Warning: Failed to Create blob cache directory %q: %v\n", c, err)
			dc.wipLock.unlock(key)
			return
		}
		wipfile, err := os.Create(wip)
		if err != nil {
			fmt.Printf("Warning: failed to prepare temp file for storing cache %q", key)
			dc.wipLock.unlock(key)
			return
		}
		dc.wipLock.unlock(key)

		defer func() {
			wipfile.Close()
			os.Remove(wipfile.Name())
		}()
		want := b2.Len()
		if _, err := io.CopyN(wipfile, b2, int64(want)); err != nil {
			fmt.Printf("Warning: failed to write cache: %v\n", err)
			return
		}

		// Commit the cache contents
		if err := os.MkdirAll(filepath.Dir(c), os.ModePerm); err != nil {
			fmt.Printf("Warning: Failed to Create blob cache directory %q: %v\n", c, err)
			return
		}
		if err := os.Rename(wipfile.Name(), c); err != nil {
			fmt.Printf("Warning: failed to commit cache to %q: %v\n", c, err)
			return
		}
		file, err := os.Open(c)
		if err != nil {
			fmt.Printf("Warning: failed to open cache on %q: %v\n", c, err)
			return
		}

		// Cache the opened file for future use. If "direct" option is specified, this
		// won't be done. This option is useful for preventing file cache from being
		// polluted by data that won't be accessed immediately.
		if opt.direct || !dc.fileCache.add(key, file) {
			file.Close()
		}
	}

	if dc.syncAdd {
		addFunc()
	} else {
		go addFunc()
	}
}

func (dc *directoryCache) cachePath(key string) string {
	return filepath.Join(dc.directory, key[:2], key)
}

func (dc *directoryCache) wipPath(key string) string {
	return filepath.Join(dc.directory, key[:2], "w", key)
}

type namedLock struct {
	muMap  map[string]*sync.Mutex
	refMap map[string]int

	mu sync.Mutex
}

func (nl *namedLock) lock(name string) {
	nl.mu.Lock()
	if nl.muMap == nil {
		nl.muMap = make(map[string]*sync.Mutex)
	}
	if nl.refMap == nil {
		nl.refMap = make(map[string]int)
	}
	if _, ok := nl.muMap[name]; !ok {
		nl.muMap[name] = &sync.Mutex{}
	}
	mu := nl.muMap[name]
	nl.refMap[name]++
	nl.mu.Unlock()
	mu.Lock()
}

func (nl *namedLock) unlock(name string) {
	nl.mu.Lock()
	mu := nl.muMap[name]
	nl.refMap[name]--
	if nl.refMap[name] <= 0 {
		delete(nl.muMap, name)
		delete(nl.refMap, name)
	}
	nl.mu.Unlock()
	mu.Unlock()
}

func newObjectCache(maxEntries int) *objectCache {
	oc := &objectCache{
		cache: lru.New(maxEntries),
	}
	oc.cache.OnEvicted = func(key lru.Key, value interface{}) {
		value.(*object).release() // Decrease ref count incremented in add operation.
	}
	return oc
}

type objectCache struct {
	cache    *lru.Cache
	cacheMu  sync.Mutex
	finalize func(interface{})
}

func (oc *objectCache) get(key string) (value interface{}, done func(), ok bool) {
	oc.cacheMu.Lock()
	defer oc.cacheMu.Unlock()
	o, ok := oc.cache.Get(key)
	if !ok {
		return nil, nil, false
	}
	o.(*object).use()
	return o.(*object).v, func() { o.(*object).release() }, true
}

func (oc *objectCache) add(key string, value interface{}) bool {
	oc.cacheMu.Lock()
	defer oc.cacheMu.Unlock()
	if _, ok := oc.cache.Get(key); ok {
		return false // TODO: should we swap the object?
	}
	o := &object{
		v:        value,
		finalize: oc.finalize,
	}
	o.use() // Keep this object having at least 1 ref count (will be decreased on eviction)
	oc.cache.Add(key, o)
	return true
}

type object struct {
	v interface{}

	refCounts int64
	finalize  func(interface{})

	mu sync.Mutex
}

func (o *object) use() {
	o.mu.Lock()
	defer o.mu.Unlock()
	o.refCounts++
}

func (o *object) release() {
	o.mu.Lock()
	defer o.mu.Unlock()
	o.refCounts--
	if o.refCounts <= 0 && o.finalize != nil {
		// nobody will refer this object
		o.finalize(o.v)
	}
}

func NewMemoryCache() BlobCache {
	return &memoryCache{
		membuf: map[string]string{},
	}
}

// memoryCache is a cache implementation which backend is a memory.
type memoryCache struct {
	membuf map[string]string // read-only []byte map is more ideal but we don't have it in golang...
	mu     sync.Mutex
}

func (mc *memoryCache) FetchAt(key string, offset int64, p []byte, opts ...Option) (n int, err error) {
	mc.mu.Lock()
	defer mc.mu.Unlock()

	cache, ok := mc.membuf[key]
	if !ok {
		return 0, fmt.Errorf("Missed cache: %q", key)
	}
	return copy(p, cache[offset:]), nil
}

func (mc *memoryCache) Add(key string, p []byte, opts ...Option) {
	mc.mu.Lock()
	defer mc.mu.Unlock()
	mc.membuf[key] = string(p)
}
<|endoftext|>


<commit_before>package cache

import "time"

type Backend int

const (
    INMEMORY = iota
    REDIS
)

type Store interface {
    // Get retrieves item from cache, i.e., (item, true).
    // If the key is not found, return (nil, false).
    Get(key string) (interface{}, error)

    // Set sets item to cache.
    // If the key exists, replace the item.
    Set(key string, value interface{}, expire time.Duration) error

    // Delete removes item from cache.
    // If the key does not exist, do nothing.
    Delete(key string) error

    // Clear all items from cache.
    Clear() error
}
<commit_msg>Update<commit_after>package cache

import (
    "time"
    "errors"
)

type Backend int

const (
    MEMORY = iota
    REDIS
)

const (
    DEFAULT = time.Duration(0)
    FOREVER = time.Duration(-1)
)

var ErrCacheMiss = errors.New("cache missing")

type Store interface {
    // Get retrieves item from cache, and return nil.
    // If the key is not found, return ErrCacheMiss.
    // Value must be a pointer.
    Get(key string, ptr interface{}) error

    // Set sets item to cache.
    // If the key exists, replace the item.
    Set(key string, value interface{}, expire time.Duration) error

    // Delete removes item from cache.
    // If the key does not exist, do nothing.
    Delete(key string) error

    // Clear all items from
    Clear() error
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"github.com/fzzy/radix/extra/pool"
	"github.com/fzzy/radix/redis"
	"github.com/minotar/minecraft"
	"image/png"
)

type CacheRedis struct {
	Client *redis.Client
	Pool   *pool.Pool
}

func (c *CacheRedis) setup() {
	pool, err := pool.NewPool("tcp", config.Redis.Address, config.Redis.PoolSize)
	if err != nil {
		log.Error("Error connecting to redis database")
		return
	}

	c.Pool = pool
	client, err := c.Pool.Get()
	if err != nil {
		log.Error(err.Error())
	}
	defer c.Pool.Put(client)

	_ = client.Cmd("AUTH", config.Redis.Auth)

	log.Info("Loaded Redis cache (pool: " + fmt.Sprintf("%v", config.Redis.PoolSize) + ")")
}

func (c *CacheRedis) has(username string) bool {
	client, err := c.Pool.Get()
	if err != nil {
		log.Error(err.Error())
	}
	defer c.Pool.Put(client)

	res := client.Cmd("EXISTS", config.Redis.Prefix+username)

	exists, err := res.Bool()
	if err != nil {
		log.Error(err.Error())
		return false
	}

	return exists
}

func (c *CacheRedis) pull(username string) minecraft.Skin {
	client, err := c.Pool.Get()
	if err != nil {
		log.Error(err.Error())
	}
	defer c.Pool.Put(client)

	resp := client.Cmd("GET", config.Redis.Prefix+username)

	skin, err := getSkinFromReply(resp)
	if err != nil {
		log.Error(err.Error())

		c.remove(username)
		char, _ := minecraft.FetchSkinForChar()

		return char
	}

	return skin
}

func (c *CacheRedis) add(username string, skin minecraft.Skin) {
	client, err := c.Pool.Get()
	if err != nil {
		log.Error(err.Error())
	}
	defer c.Pool.Put(client)

	skinBuf := new(bytes.Buffer)
	_ = png.Encode(skinBuf, skin.Image)

	_ = client.Cmd("SETEX", "skins:"+username, config.Redis.Ttl, skinBuf.Bytes())
}

func (c *CacheRedis) remove(username string) {
	client, err := c.Pool.Get()
	if err != nil {
		log.Error(err.Error())
	}
	defer c.Pool.Put(client)

	_ = client.Cmd("DEL", config.Redis.Prefix+username)
}

func getSkinFromReply(resp *redis.Reply) (minecraft.Skin, error) {
	respBytes, respErr := resp.Bytes()
	if respErr != nil {
		return minecraft.Skin{}, respErr
	}

	imgBuf := bytes.NewBuffer(respBytes)

	skin, skinErr := minecraft.DecodeSkin(imgBuf)
	if skinErr != nil {
		return minecraft.Skin{}, skinErr
	}

	return skin, nil
}
<commit_msg>Auth before every command<commit_after>package main

import (
	"bytes"
	"fmt"
	"github.com/fzzy/radix/extra/pool"
	"github.com/fzzy/radix/redis"
	"github.com/minotar/minecraft"
	"image/png"
)

type CacheRedis struct {
	Client *redis.Client
	Pool   *pool.Pool
}

func (c *CacheRedis) setup() {
	pool, err := pool.NewPool("tcp", config.Redis.Address, config.Redis.PoolSize)
	if err != nil {
		log.Error("Error connecting to redis database")
		return
	}

	c.Pool = pool
	client, err := c.Pool.Get()
	if err != nil {
		log.Error(err.Error())
	}
	defer c.Pool.Put(client)

	log.Info("Loaded Redis cache (pool: " + fmt.Sprintf("%v", config.Redis.PoolSize) + ")")
}

func (c *CacheRedis) has(username string) bool {
	client, err := c.Pool.Get()
	if err != nil {
		log.Error(err.Error())
	}
	defer c.Pool.Put(client)

	_ = client.Cmd("AUTH", config.Redis.Auth)
	res := client.Cmd("EXISTS", config.Redis.Prefix+username)

	exists, err := res.Bool()
	if err != nil {
		log.Error(err.Error())
		return false
	}

	return exists
}

func (c *CacheRedis) pull(username string) minecraft.Skin {
	client, err := c.Pool.Get()
	if err != nil {
		log.Error(err.Error())
	}
	defer c.Pool.Put(client)

	_ = client.Cmd("AUTH", config.Redis.Auth)
	resp := client.Cmd("GET", config.Redis.Prefix+username)

	skin, err := getSkinFromReply(resp)
	if err != nil {
		log.Error(err.Error())

		c.remove(username)
		char, _ := minecraft.FetchSkinForChar()

		return char
	}

	return skin
}

func (c *CacheRedis) add(username string, skin minecraft.Skin) {
	client, err := c.Pool.Get()
	if err != nil {
		log.Error(err.Error())
	}
	defer c.Pool.Put(client)

	skinBuf := new(bytes.Buffer)
	_ = png.Encode(skinBuf, skin.Image)

	_ = client.Cmd("AUTH", config.Redis.Auth)
	_ = client.Cmd("SETEX", "skins:"+username, config.Redis.Ttl, skinBuf.Bytes())
}

func (c *CacheRedis) remove(username string) {
	client, err := c.Pool.Get()
	if err != nil {
		log.Error(err.Error())
	}
	defer c.Pool.Put(client)

	_ = client.Cmd("AUTH", config.Redis.Auth)
	_ = client.Cmd("DEL", config.Redis.Prefix+username)
}

func getSkinFromReply(resp *redis.Reply) (minecraft.Skin, error) {
	respBytes, respErr := resp.Bytes()
	if respErr != nil {
		return minecraft.Skin{}, respErr
	}

	imgBuf := bytes.NewBuffer(respBytes)

	skin, skinErr := minecraft.DecodeSkin(imgBuf)
	if skinErr != nil {
		return minecraft.Skin{}, skinErr
	}

	return skin, nil
}
<|endoftext|>


<commit_before>package rds

import (
	"strconv"
	"strings"
	"testing"

	"github.com/denverdino/aliyungo/common"
)

func TestDBPostpaidClassicInstanceCreationAndDeletion(t *testing.T) {

	if TestIAmRich == false {
		// Avoid payment
		return
	}

	client := NewTestClient()

	args := CreateOrderArgs{
		RegionId:          TestRegionID,
		CommodityCode:     Bards,
		Engine:            MySQL,
		EngineVersion:     EngineVersion,
		DBInstanceClass:   DBInstanceClass,
		DBInstanceStorage: 10,
		Quantity:          1,
		PayType:           Postpaid,
		DBInstanceNetType: common.Intranet,
		Resource:          DefaultResource,
	}

	resp, err := client.CreateOrder(&args)
	if err != nil {
		t.Errorf("Failed to create db instance %v", err)
	}
	instanceId := resp.DBInstanceId
	t.Logf("Instance %s is created successfully.", instanceId)

	arrtArgs := DescribeDBInstancesArgs{
		DBInstanceId: instanceId,
	}
	attrResp, err := client.DescribeDBInstanceAttribute(&arrtArgs)
	t.Logf("Instance: %++v  %v", attrResp, err)

	err = client.WaitForInstance(instanceId, Running, 500)
	if err != nil {
		t.Errorf("Failed to create instance %s: %v", instanceId, err)
	}

	err = client.DeleteInstance(instanceId)

	if err != nil {
		t.Errorf("Failed to delete instance %s: %v", instanceId, err)
	}
	t.Logf("Instance %s is deleted successfully.", instanceId)
}

func TestDBPrepaidInstanceCreation(t *testing.T) {

	if TestIAmRich == false {
		// Avoid payment
		return
	}

	client := NewTestClient()

	args := CreateOrderArgs{
		RegionId:          TestRegionID,
		CommodityCode:     Rds,
		Engine:            MySQL,
		EngineVersion:     EngineVersion,
		DBInstanceClass:   DBInstanceClass,
		DBInstanceStorage: 10,
		Quantity:          1,
		PayType:           Prepaid,
		DBInstanceNetType: common.Intranet,
		Resource:          DefaultResource,
		TimeType:          common.Month,
		UsedTime:          1,
		AutoPay:           strconv.FormatBool(false),
	}

	resp, err := client.CreateOrder(&args)
	if err != nil {
		t.Errorf("Failed to create db instance %v", err)
	}

	instanceId := resp.DBInstanceId
	t.Logf("Instance %s is created successfully.", instanceId)

	arrtArgs := DescribeDBInstancesArgs{
		DBInstanceId: instanceId,
	}
	attrResp, err := client.DescribeDBInstanceAttribute(&arrtArgs)
	t.Logf("Instance: %++v  %v", attrResp, err)

	err = client.WaitForInstance(instanceId, Running, 500)
	if err != nil {
		t.Errorf("Failed to create instance %s: %v", instanceId, err)
	}
	t.Logf("Instance %s is running successfully.", instanceId)
}

func TestDBPostpaidVpcInstanceCreationAndDeletion(t *testing.T) {

	if TestIAmRich == false {
		// Avoid payment
		return
	}

	client := NewTestClient()

	args := CreateOrderArgs{
		RegionId:            TestRegionID,
		ZoneId:              ZoneId,
		CommodityCode:       Bards,
		Engine:              MySQL,
		EngineVersion:       EngineVersion,
		DBInstanceClass:     DBInstanceClass,
		DBInstanceStorage:   10,
		Quantity:            1,
		PayType:             Postpaid,
		DBInstanceNetType:   common.Intranet,
		InstanceNetworkType: common.VPC,
		VPCId:               VPCId,
		VSwitchId:           VSwitchId,
		SecurityIPList:      "127.0.0.1",
		Resource:            DefaultResource,
	}

	resp, err := client.CreateOrder(&args)
	if err != nil {
		t.Errorf("Failed to create db instance %v", err)
	}
	instanceId := resp.DBInstanceId
	t.Logf("Instance %s is created successfully.", instanceId)

	arrtArgs := DescribeDBInstancesArgs{
		DBInstanceId: instanceId,
	}
	attrResp, err := client.DescribeDBInstanceAttribute(&arrtArgs)
	t.Logf("Instance: %++v  %v", attrResp, err)

	err = client.WaitForInstance(instanceId, Running, 600)
	if err != nil {
		t.Errorf("Failed to create instance %s: %v", instanceId, err)
	}

	err = client.DeleteInstance(instanceId)

	if err != nil {
		t.Errorf("Failed to delete instance %s: %v", instanceId, err)
	}
	t.Logf("Instance %s is deleted successfully.", instanceId)
}

func TestGetZonesByRegionId(t *testing.T) {

	client := NewTestClient()
	resp, err := client.DescribeRegions()
	if err != nil {
		t.Errorf("Failed to describe rds regions %v", err)
	}

	regions := resp.Regions.RDSRegion
	t.Logf("all regions %++v.", regions)

	zoneIds := []string{}
	for _, r := range regions {
		if strings.Contains(r.RegionId, string(TestRegionID)) {
			zoneIds = append(zoneIds, r.ZoneId)
		}
	}
	t.Logf("all zones %++v of current region.", zoneIds)
}

func TestDatabaseCreationAndDeletion(t *testing.T) {
	client := NewTestClient()

	args := CreateDatabaseArgs{
		DBInstanceId:     DBInstanceId,
		DBName:           DBName,
		CharacterSetName: "utf8",
		DBDescription:    "test",
	}

	_, err := client.CreateDatabase(&args)
	if err != nil {
		t.Errorf("Failed to create db instance %v", err)
	}
	t.Logf("Database %s is created successfully.", DBName)

	q := [1]string{DBName}
	err = client.WaitForAllDatabase(DBInstanceId, q[:], Running, 600)
	if err != nil {
		t.Errorf("Failed to create database %s: %v", DBName, err)
	}

	err = client.DeleteDatabase(DBInstanceId, DBName)

	if err != nil {
		t.Errorf("Failed to delete database %s: %v", DBName, err)
	}
	t.Logf("Database %s is deleted successfully.", DBName)
}

func TestAccountCreationAndDeletion(t *testing.T) {
	client := NewTestClient()

	args := CreateAccountArgs{
		DBInstanceId:       DBInstanceId,
		AccountName:        AccountName,
		AccountPassword:    AccountPassword,
		AccountDescription: "test",
	}

	_, err := client.CreateAccount(&args)
	err = client.WaitForAccount(DBInstanceId, AccountName, Available, 600)
	if err != nil {
		t.Errorf("Failed to create account %s: %v", AccountName, err)
	}
	t.Logf("Account %s is created successfully.", AccountName)

	pargs := GrantAccountPrivilegeArgs{
		DBInstanceId:     DBInstanceId,
		AccountName:      AccountName,
		DBName:           DBName,
		AccountPrivilege: ReadWrite,
	}
	_, err = client.GrantAccountPrivilege(&pargs)
	if err != nil {
		t.Errorf("Failed to grant privilege to account %v", err)
	}
	t.Logf("Grant privilege to account %s successfully.", AccountName)

	err = client.WaitForAccountPrivilege(DBInstanceId, AccountName, DBName, ReadWrite, 200)
	if err != nil {
		t.Errorf("Failed to grant privilege to account %s: %v", AccountName, err)
	}
	t.Logf("Grant privilege to account %s successfully.", AccountName)

	_, err = client.DeleteAccount(DBInstanceId, AccountName)

	if err != nil {
		t.Errorf("Failed to delete account %s: %v", AccountName, err)
	}
	t.Logf("Account %s is deleted successfully.", AccountName)
}

func TestAccountAllocatePublicConnection(t *testing.T) {
	client := NewTestClient()

	args := AllocateInstancePublicConnectionArgs{
		DBInstanceId:           DBInstanceId,
		ConnectionStringPrefix: DBInstanceId + "o",
		Port: "3306",
	}

	_, err := client.AllocateInstancePublicConnection(&args)
	err = client.WaitForPublicConnection(DBInstanceId, 600)
	if err != nil {
		t.Errorf("Failed to allocate public connection: %v", err)
	}
	t.Logf("Allocate public connection successfully.")

}

func TestModifyBackupPolicy(t *testing.T) {
	client := NewTestClient()

	bargs := BackupPolicy{
		PreferredBackupTime:   "00:00Z-01:00Z",
		PreferredBackupPeriod: "Wednesday",
		BackupRetentionPeriod: 9,
	}
	args := ModifyBackupPolicyArgs{
		DBInstanceId: DBInstanceId,
		BackupPolicy: bargs,
	}

	_, err := client.ModifyBackupPolicy(&args)
	if err != nil {
		t.Errorf("Failed to modify backup policy: %v", err)
	}
	t.Logf("Modify backup policy successfully.")

}

func TestModifySecurityIps(t *testing.T) {
	client := NewTestClient()

	securityIps := "127.0.0.1"

	args := ModifySecurityIpsArgs{
		DBInstanceId: DBInstanceId,
		SecurityIps:  securityIps,
	}

	_, err := client.ModifySecurityIps(&args)
	if err != nil {
		t.Errorf("Failed to modify security ips: %v", err)
	}
	t.Logf("Modify security ips successfully.")

}

func TestModifyDBInstanceSpec(t *testing.T) {
	client := NewTestClient()

	args := ModifyDBInstanceSpecArgs{
		DBInstanceId:    DBInstanceUpgradeId,
		PayType:         Postpaid,
		DBInstanceClass: DBInstanceUpgradeClass,
	}

	_, err := client.ModifyDBInstanceSpec(&args)
	if err != nil {
		t.Errorf("Failed to modify db instance spec: %v", err)
	}

	err = client.WaitForInstance(DBInstanceUpgradeId, Running, 600)
	if err != nil {
		t.Errorf("Failed to modify db instance %s: %v", DBInstanceUpgradeId, err)
	}

	arrtArgs := DescribeDBInstancesArgs{
		DBInstanceId: DBInstanceUpgradeId,
	}
	attrResp, err := client.DescribeDBInstanceAttribute(&arrtArgs)
	t.Logf("Instance: %++v  %v", attrResp, err)

	if attrResp.Items.DBInstanceAttribute[0].DBInstanceClass != DBInstanceUpgradeClass {
		t.Errorf("Failed to modify db instance spec: %v", err)
	}

	t.Logf("Modify db instance spec successfully.")
}

func TestClient_DescribeRegions(t *testing.T) {
	client := NewTestClientForDebug()
	client.SetSecurityToken(TestSecurityToken)

	regions, err := client.DescribeRegions()
	if err != nil {
		t.Fatalf("Error %++v", err)
	} else {
		t.Logf("Result = %++v", regions)
	}
}
<commit_msg>fix rds test issue<commit_after>package rds

import (
	"strconv"
	"strings"
	"testing"

	"github.com/denverdino/aliyungo/common"
)

func TestDBPostpaidClassicInstanceCreationAndDeletion(t *testing.T) {

	if TestIAmRich == false {
		// Avoid payment
		return
	}

	client := NewTestClient()

	args := CreateOrderArgs{
		RegionId:          TestRegionID,
		CommodityCode:     Bards,
		Engine:            MySQL,
		EngineVersion:     EngineVersion,
		DBInstanceClass:   DBInstanceClass,
		DBInstanceStorage: 10,
		Quantity:          1,
		PayType:           Postpaid,
		DBInstanceNetType: common.Intranet,
		Resource:          DefaultResource,
	}

	resp, err := client.CreateOrder(&args)
	if err != nil {
		t.Errorf("Failed to create db instance %v", err)
	}
	instanceId := resp.DBInstanceId
	t.Logf("Instance %s is created successfully.", instanceId)

	arrtArgs := DescribeDBInstanceAttributeArgs{
		DBInstanceId: instanceId,
	}
	attrResp, err := client.DescribeDBInstanceAttribute(&arrtArgs)
	t.Logf("Instance: %++v  %v", attrResp, err)

	err = client.WaitForInstance(instanceId, Running, 500)
	if err != nil {
		t.Errorf("Failed to create instance %s: %v", instanceId, err)
	}

	err = client.DeleteInstance(instanceId)

	if err != nil {
		t.Errorf("Failed to delete instance %s: %v", instanceId, err)
	}
	t.Logf("Instance %s is deleted successfully.", instanceId)
}

func TestDBPrepaidInstanceCreation(t *testing.T) {

	if TestIAmRich == false {
		// Avoid payment
		return
	}

	client := NewTestClient()

	args := CreateOrderArgs{
		RegionId:          TestRegionID,
		CommodityCode:     Rds,
		Engine:            MySQL,
		EngineVersion:     EngineVersion,
		DBInstanceClass:   DBInstanceClass,
		DBInstanceStorage: 10,
		Quantity:          1,
		PayType:           Prepaid,
		DBInstanceNetType: common.Intranet,
		Resource:          DefaultResource,
		TimeType:          common.Month,
		UsedTime:          1,
		AutoPay:           strconv.FormatBool(false),
	}

	resp, err := client.CreateOrder(&args)
	if err != nil {
		t.Errorf("Failed to create db instance %v", err)
	}

	instanceId := resp.DBInstanceId
	t.Logf("Instance %s is created successfully.", instanceId)

	arrtArgs := DescribeDBInstanceAttributeArgs{
		DBInstanceId: instanceId,
	}
	attrResp, err := client.DescribeDBInstanceAttribute(&arrtArgs)
	t.Logf("Instance: %++v  %v", attrResp, err)

	err = client.WaitForInstance(instanceId, Running, 500)
	if err != nil {
		t.Errorf("Failed to create instance %s: %v", instanceId, err)
	}
	t.Logf("Instance %s is running successfully.", instanceId)
}

func TestDBPostpaidVpcInstanceCreationAndDeletion(t *testing.T) {

	if TestIAmRich == false {
		// Avoid payment
		return
	}

	client := NewTestClient()

	args := CreateOrderArgs{
		RegionId:            TestRegionID,
		ZoneId:              ZoneId,
		CommodityCode:       Bards,
		Engine:              MySQL,
		EngineVersion:       EngineVersion,
		DBInstanceClass:     DBInstanceClass,
		DBInstanceStorage:   10,
		Quantity:            1,
		PayType:             Postpaid,
		DBInstanceNetType:   common.Intranet,
		InstanceNetworkType: common.VPC,
		VPCId:               VPCId,
		VSwitchId:           VSwitchId,
		SecurityIPList:      "127.0.0.1",
		Resource:            DefaultResource,
	}

	resp, err := client.CreateOrder(&args)
	if err != nil {
		t.Errorf("Failed to create db instance %v", err)
	}
	instanceId := resp.DBInstanceId
	t.Logf("Instance %s is created successfully.", instanceId)

	arrtArgs := DescribeDBInstanceAttributeArgs{
		DBInstanceId: instanceId,
	}
	attrResp, err := client.DescribeDBInstanceAttribute(&arrtArgs)
	t.Logf("Instance: %++v  %v", attrResp, err)

	err = client.WaitForInstance(instanceId, Running, 600)
	if err != nil {
		t.Errorf("Failed to create instance %s: %v", instanceId, err)
	}

	err = client.DeleteInstance(instanceId)

	if err != nil {
		t.Errorf("Failed to delete instance %s: %v", instanceId, err)
	}
	t.Logf("Instance %s is deleted successfully.", instanceId)
}

func TestGetZonesByRegionId(t *testing.T) {

	client := NewTestClient()
	resp, err := client.DescribeRegions()
	if err != nil {
		t.Errorf("Failed to describe rds regions %v", err)
	}

	regions := resp.Regions.RDSRegion
	t.Logf("all regions %++v.", regions)

	zoneIds := []string{}
	for _, r := range regions {
		if strings.Contains(r.RegionId, string(TestRegionID)) {
			zoneIds = append(zoneIds, r.ZoneId)
		}
	}
	t.Logf("all zones %++v of current region.", zoneIds)
}

func TestDatabaseCreationAndDeletion(t *testing.T) {
	client := NewTestClient()

	args := CreateDatabaseArgs{
		DBInstanceId:     DBInstanceId,
		DBName:           DBName,
		CharacterSetName: "utf8",
		DBDescription:    "test",
	}

	_, err := client.CreateDatabase(&args)
	if err != nil {
		t.Errorf("Failed to create db instance %v", err)
	}
	t.Logf("Database %s is created successfully.", DBName)

	q := [1]string{DBName}
	err = client.WaitForAllDatabase(DBInstanceId, q[:], Running, 600)
	if err != nil {
		t.Errorf("Failed to create database %s: %v", DBName, err)
	}

	err = client.DeleteDatabase(DBInstanceId, DBName)

	if err != nil {
		t.Errorf("Failed to delete database %s: %v", DBName, err)
	}
	t.Logf("Database %s is deleted successfully.", DBName)
}

func TestAccountCreationAndDeletion(t *testing.T) {
	client := NewTestClient()

	args := CreateAccountArgs{
		DBInstanceId:       DBInstanceId,
		AccountName:        AccountName,
		AccountPassword:    AccountPassword,
		AccountDescription: "test",
	}

	_, err := client.CreateAccount(&args)
	err = client.WaitForAccount(DBInstanceId, AccountName, Available, 600)
	if err != nil {
		t.Errorf("Failed to create account %s: %v", AccountName, err)
	}
	t.Logf("Account %s is created successfully.", AccountName)

	pargs := GrantAccountPrivilegeArgs{
		DBInstanceId:     DBInstanceId,
		AccountName:      AccountName,
		DBName:           DBName,
		AccountPrivilege: ReadWrite,
	}
	_, err = client.GrantAccountPrivilege(&pargs)
	if err != nil {
		t.Errorf("Failed to grant privilege to account %v", err)
	}
	t.Logf("Grant privilege to account %s successfully.", AccountName)

	err = client.WaitForAccountPrivilege(DBInstanceId, AccountName, DBName, ReadWrite, 200)
	if err != nil {
		t.Errorf("Failed to grant privilege to account %s: %v", AccountName, err)
	}
	t.Logf("Grant privilege to account %s successfully.", AccountName)

	_, err = client.DeleteAccount(DBInstanceId, AccountName)

	if err != nil {
		t.Errorf("Failed to delete account %s: %v", AccountName, err)
	}
	t.Logf("Account %s is deleted successfully.", AccountName)
}

func TestAccountAllocatePublicConnection(t *testing.T) {
	client := NewTestClient()

	args := AllocateInstancePublicConnectionArgs{
		DBInstanceId:           DBInstanceId,
		ConnectionStringPrefix: DBInstanceId + "o",
		Port: "3306",
	}

	_, err := client.AllocateInstancePublicConnection(&args)
	err = client.WaitForPublicConnection(DBInstanceId, 600)
	if err != nil {
		t.Errorf("Failed to allocate public connection: %v", err)
	}
	t.Logf("Allocate public connection successfully.")

}

func TestModifyBackupPolicy(t *testing.T) {
	client := NewTestClient()

	bargs := BackupPolicy{
		PreferredBackupTime:   "00:00Z-01:00Z",
		PreferredBackupPeriod: "Wednesday",
		BackupRetentionPeriod: 9,
	}
	args := ModifyBackupPolicyArgs{
		DBInstanceId: DBInstanceId,
		BackupPolicy: bargs,
	}

	_, err := client.ModifyBackupPolicy(&args)
	if err != nil {
		t.Errorf("Failed to modify backup policy: %v", err)
	}
	t.Logf("Modify backup policy successfully.")

}

func TestModifySecurityIps(t *testing.T) {
	client := NewTestClient()

	securityIps := "127.0.0.1"

	args := ModifySecurityIpsArgs{
		DBInstanceId: DBInstanceId,
		SecurityIps:  securityIps,
	}

	_, err := client.ModifySecurityIps(&args)
	if err != nil {
		t.Errorf("Failed to modify security ips: %v", err)
	}
	t.Logf("Modify security ips successfully.")

}

func TestModifyDBInstanceSpec(t *testing.T) {
	client := NewTestClient()

	args := ModifyDBInstanceSpecArgs{
		DBInstanceId:    DBInstanceUpgradeId,
		PayType:         Postpaid,
		DBInstanceClass: DBInstanceUpgradeClass,
	}

	_, err := client.ModifyDBInstanceSpec(&args)
	if err != nil {
		t.Errorf("Failed to modify db instance spec: %v", err)
	}

	err = client.WaitForInstance(DBInstanceUpgradeId, Running, 600)
	if err != nil {
		t.Errorf("Failed to modify db instance %s: %v", DBInstanceUpgradeId, err)
	}

	arrtArgs := DescribeDBInstanceAttributeArgs{
		DBInstanceId: DBInstanceUpgradeId,
	}
	attrResp, err := client.DescribeDBInstanceAttribute(&arrtArgs)
	t.Logf("Instance: %++v  %v", attrResp, err)

	if attrResp.Items.DBInstanceAttribute[0].DBInstanceClass != DBInstanceUpgradeClass {
		t.Errorf("Failed to modify db instance spec: %v", err)
	}

	t.Logf("Modify db instance spec successfully.")
}

func TestClient_DescribeRegions(t *testing.T) {
	client := NewTestClientForDebug()
	client.SetSecurityToken(TestSecurityToken)

	regions, err := client.DescribeRegions()
	if err != nil {
		t.Fatalf("Error %++v", err)
	} else {
		t.Logf("Result = %++v", regions)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/juju/ratelimit"
	"github.com/pkg/errors"
	"github.com/tomnomnom/linkheader"
)

func PubSubMakeCallbackURL(baseURL, id string) string {
	return baseURL + "/" + id
}

type PubSubSubscription struct {
	ID          string
	Hub         string
	Topic       string
	CallbackURL string
	CreatedAt   time.Time
	UpdatedAt   time.Time
	ExpiresAt   *time.Time
}

type PubSubState interface {
	All() (subscriptions []PubSubSubscription, err error)
	Add(hub, topic, baseURL string) (subscription *PubSubSubscription, oldCallbackURL string, err error)
	Get(hub, topic string) (subscription *PubSubSubscription, err error)
	GetByID(id string) (subscription *PubSubSubscription, err error)
	Set(hub, topic string, updatedAt, expiresAt time.Time) (err error)
	Del(hub, topic string) (err error)
}

type PubSubMessageHandler func(id string, s *PubSubSubscription, rd io.ReadCloser)

type PubSubClient struct {
	CallbackURL string
	State       PubSubState
	OnMessage   PubSubMessageHandler
}

func NewPubSubClient(callbackURL string, state PubSubState, onMessage PubSubMessageHandler) *PubSubClient {
	return &PubSubClient{
		CallbackURL: callbackURL,
		State:       state,
		OnMessage:   onMessage,
	}
}

func (c *PubSubClient) Refresh(forceUpdate bool, interval time.Duration) error {
	a, err := c.State.All()
	if err != nil {
		return errors.Wrap(err, "PubSubClient.Refresh")
	}

	logrus.WithField("count", len(a)).Debug("pubsub: got subscriptions to refresh")

	m := make(map[string]*ratelimit.Bucket)
	var l sync.RWMutex

	getBucket := func(host string) *ratelimit.Bucket {
		l.RLock()
		if b, ok := m[host]; ok {
			l.RUnlock()
			return b
		}
		l.RUnlock()

		l.Lock()
		if b, ok := m[host]; ok {
			l.Unlock()
			return b
		}
		defer l.Unlock()

		m[host] = ratelimit.NewBucket(time.Second*30, 4)

		return m[host]
	}

	var g WorkerGroup

	for _, e := range a {
		e := e

		callbackURL := c.CallbackURL + "/" + e.ID

		l := logrus.WithFields(logrus.Fields{
			"id":               e.ID,
			"hub":              e.Hub,
			"topic":            e.Topic,
			"callback_url":     e.CallbackURL,
			"new_callback_url": callbackURL,
			"created_at":       e.CreatedAt,
			"updated_at":       e.UpdatedAt,
			"expires_at":       e.ExpiresAt,
			"force_update":     forceUpdate,
		})

		if e.ExpiresAt == nil {
			l.Debug("pubsub: not refreshing subscription which has never been verified")
			continue
		}

		if forceUpdate || e.ExpiresAt.Sub(time.Now()) < interval || e.CallbackURL != callbackURL {
			l.Debug("pubsub: refreshing subscription")

			g.Add(func() error {
				u, err := url.Parse(e.Hub)
				if err != nil {
					l.WithError(err).Warn("pubsub: couldn't parse hub url")
					return errors.Wrap(err, "PubSubClient.RefreshWorker")
				}

				if dur, skip := getBucket(u.Host).TakeMaxDuration(1, *pubsubRefreshInterval); skip {
					l.Debug("pubsub: skipping renewing for now as we'd have to wait too long")
					return nil
				} else if dur > 0 {
					l.WithField("duration", dur).Debug("pubsub: waiting so as not to overwhelm the endpoint")
					time.Sleep(dur)
				}

				if err := c.Subscribe(e.Hub, e.Topic); err != nil {
					l.WithError(err).Warn("pubsub: couldn't subscribe to topic")
					return errors.Wrap(err, "PubSubClient.RefreshWorker")
				}

				l.Debug("pubsub: subscribed successfully")
				return nil
			})
		}
	}

	return errors.Wrap(g.Run(4), "PubSubClient.Refresh")
}

func (c *PubSubClient) Subscribe(hub, topic string) error {
	logrus.WithFields(logrus.Fields{"hub": hub, "topic": topic}).Debug("pubsub: subscribing")

	s, oldCallbackURL, err := c.State.Add(hub, topic, c.CallbackURL)
	if err != nil {
		return errors.Wrap(err, "PubSubClient.Subscribe")
	}

	if oldCallbackURL != s.CallbackURL {
		if oldCallbackURL != "" {
			if err := PubSubUnsubscribe(hub, topic, oldCallbackURL); err != nil {
				return errors.Wrap(err, "PubSubClient.Subscribe")
			}
		}

		if err := PubSubSubscribe(hub, topic, s.CallbackURL); err != nil {
			return errors.Wrap(err, "PubSubClient.Subscribe")
		}
	}

	return nil
}

func (c *PubSubClient) Unsubscribe(hub, topic string) error {
	logrus.WithFields(logrus.Fields{"hub": hub, "topic": topic}).Debug("pubsub: unsubscribing")

	s, err := c.State.Get(hub, topic)
	if err != nil {
		return errors.Wrap(err, "PubSubClient.Unsubscribe")
	}

	if s != nil {
		if err := PubSubUnsubscribe(hub, topic, s.CallbackURL); err != nil {
			return errors.Wrap(err, "PubSubClient.Unsubscribe")
		}

		if err := c.State.Del(s.Hub, s.Topic); err != nil {
			return errors.Wrap(err, "PubSubClient.Unsubscribe")
		}
	}

	return nil
}

func (c *PubSubClient) Handler() *PubSubHandler {
	return &PubSubHandler{
		OnChallenge: func(id, topic, mode string, leaseTime time.Duration) error {
			l := logrus.WithFields(logrus.Fields{
				"id":         id,
				"topic":      topic,
				"mode":       mode,
				"lease_time": leaseTime,
			})

			l.Debug("pubsub: received challenge")

			s, err := c.State.GetByID(id)
			if err != nil {
				l.WithError(err).Warn("pubsub: error fetching subscription during challenge")
				return err
			}

			if s == nil {
				l.WithError(err).Warn("pubsub: subscription not found during challenge")
				return errors.Errorf("PubSubClient.Handler: subscription not found")
			}

			return errors.Wrap(c.State.Set(s.Hub, s.Topic, time.Now(), time.Now().Add(leaseTime)), "PubSubClient.Handler")
		},
		OnMessage: func(id, topic string, rd io.ReadCloser) {
			l := logrus.WithFields(logrus.Fields{
				"id":    id,
				"topic": topic,
			})

			l.Debug("pubsub: received message")

			defer rd.Close()

			if c.OnMessage != nil {
				s, err := c.State.GetByID(id)
				if err != nil {
					l.WithError(err).Warn("pubsub: error fetching subscription during reception")
					return
				}

				c.OnMessage(id, s, rd)
			}
		},
	}
}

func PubSubAlter(hub, topic, callbackURL, mode string) error {
	res, err := http.PostForm(hub, url.Values{
		"hub.callback":      []string{callbackURL},
		"hub.mode":          []string{mode},
		"hub.topic":         []string{topic},
		"hub.verify":        []string{"async"},
		"hub.lease_seconds": []string{"604800"},
	})

	if err != nil {
		return errors.Wrap(err, "PubSubAlter")
	}

	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return errors.Errorf("PubSubAlter: invalid status code; expected 2xx but got %d", res.StatusCode)
	}

	return nil
}

func PubSubSubscribe(hub, topic, callbackURL string) error {
	return errors.Wrap(PubSubAlter(hub, topic, callbackURL, "subscribe"), "PubSubSubscribe")
}

func PubSubUnsubscribe(hub, topic, callbackURL string) error {
	return errors.Wrap(PubSubAlter(hub, topic, callbackURL, "unsubscribe"), "PubSubSubscribe")
}

type PubSubHandler struct {
	OnChallenge func(id, topic, mode string, leaseTime time.Duration) error
	OnMessage   func(id, topic string, rd io.ReadCloser)
}

func (h *PubSubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	q := r.URL.Query()

	bits := strings.Split(r.URL.Path, "/")
	id := bits[len(bits)-1]

	if r.Method == http.MethodGet && q.Get("hub.challenge") != "" {
		if h.OnChallenge != nil {
			leaseTime := time.Hour * 12
			if n, err := strconv.ParseInt(q.Get("hub.lease_seconds"), 10, 64); err == nil {
				leaseTime = time.Second * time.Duration(n)
			}

			if err := h.OnChallenge(id, q.Get("hub.topic"), q.Get("hub.mode"), leaseTime); err != nil {
				rw.WriteHeader(http.StatusNotFound)
				return
			}
		}

		rw.Write([]byte(r.URL.Query().Get("hub.challenge")))
		return
	}

	if r.Method != http.MethodPost {
		rw.WriteHeader(http.StatusMethodNotAllowed)
		return
	}

	links := linkheader.ParseMultiple(r.Header["Link"])

	var topic string
	if a := links.FilterByRel("self"); len(a) > 0 {
		topic = a[0].URL
	}

	rw.WriteHeader(http.StatusAccepted)

	if h.OnMessage != nil {
		h.OnMessage(id, topic, r.Body)
	}
}
<commit_msg>only log when a refresh is actually starting<commit_after>package main

import (
	"io"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/juju/ratelimit"
	"github.com/pkg/errors"
	"github.com/tomnomnom/linkheader"
)

func PubSubMakeCallbackURL(baseURL, id string) string {
	return baseURL + "/" + id
}

type PubSubSubscription struct {
	ID          string
	Hub         string
	Topic       string
	CallbackURL string
	CreatedAt   time.Time
	UpdatedAt   time.Time
	ExpiresAt   *time.Time
}

type PubSubState interface {
	All() (subscriptions []PubSubSubscription, err error)
	Add(hub, topic, baseURL string) (subscription *PubSubSubscription, oldCallbackURL string, err error)
	Get(hub, topic string) (subscription *PubSubSubscription, err error)
	GetByID(id string) (subscription *PubSubSubscription, err error)
	Set(hub, topic string, updatedAt, expiresAt time.Time) (err error)
	Del(hub, topic string) (err error)
}

type PubSubMessageHandler func(id string, s *PubSubSubscription, rd io.ReadCloser)

type PubSubClient struct {
	CallbackURL string
	State       PubSubState
	OnMessage   PubSubMessageHandler
}

func NewPubSubClient(callbackURL string, state PubSubState, onMessage PubSubMessageHandler) *PubSubClient {
	return &PubSubClient{
		CallbackURL: callbackURL,
		State:       state,
		OnMessage:   onMessage,
	}
}

func (c *PubSubClient) Refresh(forceUpdate bool, interval time.Duration) error {
	a, err := c.State.All()
	if err != nil {
		return errors.Wrap(err, "PubSubClient.Refresh")
	}

	logrus.WithField("count", len(a)).Debug("pubsub: got subscriptions to refresh")

	m := make(map[string]*ratelimit.Bucket)
	var l sync.RWMutex

	getBucket := func(host string) *ratelimit.Bucket {
		l.RLock()
		if b, ok := m[host]; ok {
			l.RUnlock()
			return b
		}
		l.RUnlock()

		l.Lock()
		if b, ok := m[host]; ok {
			l.Unlock()
			return b
		}
		defer l.Unlock()

		m[host] = ratelimit.NewBucket(time.Second*30, 4)

		return m[host]
	}

	var g WorkerGroup

	for _, e := range a {
		e := e

		callbackURL := c.CallbackURL + "/" + e.ID

		l := logrus.WithFields(logrus.Fields{
			"id":               e.ID,
			"hub":              e.Hub,
			"topic":            e.Topic,
			"callback_url":     e.CallbackURL,
			"new_callback_url": callbackURL,
			"created_at":       e.CreatedAt,
			"updated_at":       e.UpdatedAt,
			"expires_at":       e.ExpiresAt,
			"force_update":     forceUpdate,
		})

		if e.ExpiresAt == nil {
			l.Debug("pubsub: not refreshing subscription which has never been verified")
			continue
		}

		if forceUpdate || e.ExpiresAt.Sub(time.Now()) < interval || e.CallbackURL != callbackURL {
			g.Add(func() error {
				u, err := url.Parse(e.Hub)
				if err != nil {
					l.WithError(err).Warn("pubsub: couldn't parse hub url")
					return errors.Wrap(err, "PubSubClient.RefreshWorker")
				}

				if dur, skip := getBucket(u.Host).TakeMaxDuration(1, *pubsubRefreshInterval); skip {
					l.Debug("pubsub: skipping renewing for now as we'd have to wait too long")
					return nil
				} else if dur > 0 {
					l.WithField("duration", dur).Debug("pubsub: waiting so as not to overwhelm the endpoint")
					time.Sleep(dur)
				}

				l.Debug("pubsub: refreshing subscription")

				if err := c.Subscribe(e.Hub, e.Topic); err != nil {
					l.WithError(err).Warn("pubsub: couldn't subscribe to topic")
					return errors.Wrap(err, "PubSubClient.RefreshWorker")
				}

				l.Debug("pubsub: subscribed successfully")
				return nil
			})
		}
	}

	return errors.Wrap(g.Run(4), "PubSubClient.Refresh")
}

func (c *PubSubClient) Subscribe(hub, topic string) error {
	logrus.WithFields(logrus.Fields{"hub": hub, "topic": topic}).Debug("pubsub: subscribing")

	s, oldCallbackURL, err := c.State.Add(hub, topic, c.CallbackURL)
	if err != nil {
		return errors.Wrap(err, "PubSubClient.Subscribe")
	}

	if oldCallbackURL != s.CallbackURL {
		if oldCallbackURL != "" {
			if err := PubSubUnsubscribe(hub, topic, oldCallbackURL); err != nil {
				return errors.Wrap(err, "PubSubClient.Subscribe")
			}
		}

		if err := PubSubSubscribe(hub, topic, s.CallbackURL); err != nil {
			return errors.Wrap(err, "PubSubClient.Subscribe")
		}
	}

	return nil
}

func (c *PubSubClient) Unsubscribe(hub, topic string) error {
	logrus.WithFields(logrus.Fields{"hub": hub, "topic": topic}).Debug("pubsub: unsubscribing")

	s, err := c.State.Get(hub, topic)
	if err != nil {
		return errors.Wrap(err, "PubSubClient.Unsubscribe")
	}

	if s != nil {
		if err := PubSubUnsubscribe(hub, topic, s.CallbackURL); err != nil {
			return errors.Wrap(err, "PubSubClient.Unsubscribe")
		}

		if err := c.State.Del(s.Hub, s.Topic); err != nil {
			return errors.Wrap(err, "PubSubClient.Unsubscribe")
		}
	}

	return nil
}

func (c *PubSubClient) Handler() *PubSubHandler {
	return &PubSubHandler{
		OnChallenge: func(id, topic, mode string, leaseTime time.Duration) error {
			l := logrus.WithFields(logrus.Fields{
				"id":         id,
				"topic":      topic,
				"mode":       mode,
				"lease_time": leaseTime,
			})

			l.Debug("pubsub: received challenge")

			s, err := c.State.GetByID(id)
			if err != nil {
				l.WithError(err).Warn("pubsub: error fetching subscription during challenge")
				return err
			}

			if s == nil {
				l.WithError(err).Warn("pubsub: subscription not found during challenge")
				return errors.Errorf("PubSubClient.Handler: subscription not found")
			}

			return errors.Wrap(c.State.Set(s.Hub, s.Topic, time.Now(), time.Now().Add(leaseTime)), "PubSubClient.Handler")
		},
		OnMessage: func(id, topic string, rd io.ReadCloser) {
			l := logrus.WithFields(logrus.Fields{
				"id":    id,
				"topic": topic,
			})

			l.Debug("pubsub: received message")

			defer rd.Close()

			if c.OnMessage != nil {
				s, err := c.State.GetByID(id)
				if err != nil {
					l.WithError(err).Warn("pubsub: error fetching subscription during reception")
					return
				}

				c.OnMessage(id, s, rd)
			}
		},
	}
}

func PubSubAlter(hub, topic, callbackURL, mode string) error {
	res, err := http.PostForm(hub, url.Values{
		"hub.callback":      []string{callbackURL},
		"hub.mode":          []string{mode},
		"hub.topic":         []string{topic},
		"hub.verify":        []string{"async"},
		"hub.lease_seconds": []string{"604800"},
	})

	if err != nil {
		return errors.Wrap(err, "PubSubAlter")
	}

	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return errors.Errorf("PubSubAlter: invalid status code; expected 2xx but got %d", res.StatusCode)
	}

	return nil
}

func PubSubSubscribe(hub, topic, callbackURL string) error {
	return errors.Wrap(PubSubAlter(hub, topic, callbackURL, "subscribe"), "PubSubSubscribe")
}

func PubSubUnsubscribe(hub, topic, callbackURL string) error {
	return errors.Wrap(PubSubAlter(hub, topic, callbackURL, "unsubscribe"), "PubSubSubscribe")
}

type PubSubHandler struct {
	OnChallenge func(id, topic, mode string, leaseTime time.Duration) error
	OnMessage   func(id, topic string, rd io.ReadCloser)
}

func (h *PubSubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	q := r.URL.Query()

	bits := strings.Split(r.URL.Path, "/")
	id := bits[len(bits)-1]

	if r.Method == http.MethodGet && q.Get("hub.challenge") != "" {
		if h.OnChallenge != nil {
			leaseTime := time.Hour * 12
			if n, err := strconv.ParseInt(q.Get("hub.lease_seconds"), 10, 64); err == nil {
				leaseTime = time.Second * time.Duration(n)
			}

			if err := h.OnChallenge(id, q.Get("hub.topic"), q.Get("hub.mode"), leaseTime); err != nil {
				rw.WriteHeader(http.StatusNotFound)
				return
			}
		}

		rw.Write([]byte(r.URL.Query().Get("hub.challenge")))
		return
	}

	if r.Method != http.MethodPost {
		rw.WriteHeader(http.StatusMethodNotAllowed)
		return
	}

	links := linkheader.ParseMultiple(r.Header["Link"])

	var topic string
	if a := links.FilterByRel("self"); len(a) > 0 {
		topic = a[0].URL
	}

	rw.WriteHeader(http.StatusAccepted)

	if h.OnMessage != nil {
		h.OnMessage(id, topic, r.Body)
	}
}
<|endoftext|>


<commit_before>package cblog

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
)

const LOG_OUTPUT_BUFFER = 1024

type logMesg struct {
	mesg  string
	fatal bool
}

type loggerHandler interface {
	setup(config map[string]interface{}) error
	write(mesg *logMesg)
}

type Logger struct {
	messages chan *logMesg
	outputs  map[string]loggerHandler
}

func New() *Logger {
	l := &Logger{
		messages: make(chan *logMesg, LOG_OUTPUT_BUFFER),
		outputs:  make(map[string]loggerHandler),
	}
	go l.run()
	return l
}

func (l *Logger) SetLogger(handlerType string, cfg map[string]interface{}) {
	var handler loggerHandler
	switch handlerType {
	case "console":
		handler = newConsoleHandler()
	case "file":
		handler = newFileHandler()
	default:
		panic("Unknown log handler.")
	}

	handler.setup(cfg)
	l.outputs[handlerType] = handler
}

func (l *Logger) run() {
	for {
		select {
		case mesg := <-l.messages:
			for _, handler := range l.outputs {
				handler.write(mesg)
			}
		}
	}
}

func (l *Logger) writeMesg(mesg string, fatal bool) {
	lm := &logMesg{
		mesg:  mesg,
		fatal: fatal,
	}
	l.messages <- lm
}

func (l *Logger) Debug(format string, v ...interface{}) {
	mesg := fmt.Sprintf("[DEBUG] "+format, v...)
	l.writeMesg(mesg, false)
}

func (l *Logger) Info(format string, v ...interface{}) {
	mesg := fmt.Sprintf("[INFO] "+format, v...)
	l.writeMesg(mesg, false)
}

func (l *Logger) Notice(format string, v ...interface{}) {
	mesg := fmt.Sprintf("[NOTICE] "+format, v...)
	l.writeMesg(mesg, false)
}

func (l *Logger) Warn(format string, v ...interface{}) {
	mesg := fmt.Sprintf("[WARN] "+format, v...)
	l.writeMesg(mesg, false)
}

func (l *Logger) Error(format string, v ...interface{}) {
	mesg := fmt.Sprintf("[ERROR] "+format, v...)
	l.writeMesg(mesg, false)
}

func (l *Logger) Fatal(format string, v ...interface{}) {
	mesg := fmt.Sprintf("[FATAL] "+format, v...)
	l.writeMesg(mesg, true)
}

type consoleHandler struct {
	logger *log.Logger
}

func newConsoleHandler() loggerHandler {
	return new(consoleHandler)
}

func (h *consoleHandler) setup(cfg map[string]interface{}) error {
	h.logger = log.New(os.Stdout, "", log.Ldate|log.Ltime)
	return nil
}

func (h *consoleHandler) write(lm *logMesg) {
	if !lm.fatal {
		h.logger.Println(lm.mesg)
	} else {
		h.logger.Fatalln(lm.mesg)
	}
}

type fileHandler struct {
	file   string
	logger *log.Logger
}

func newFileHandler() loggerHandler {
	return new(fileHandler)
}

func (h *fileHandler) setup(config map[string]interface{}) error {
	if file, ok := config["file"]; ok {
		h.file = file.(string)
		if _, err := os.Stat(h.file); os.IsNotExist(err) {
			_ = os.MkdirAll(filepath.Dir(h.file), 0755)
			if _, err := os.Create(h.file); err != nil {
				return err
			}
		}

		output, _ := os.Create(h.file)
		h.logger = log.New(output, "", log.Ldate|log.Ltime)
	}

	return nil
}

func (h *fileHandler) write(lm *logMesg) {
	if h.logger == nil {
		return
	}

	if !lm.fatal {
		h.logger.Println(lm.mesg)
	} else {
		h.logger.Fatalln(lm.mesg)
	}
}
<commit_msg>gnocco: fixed go lint issues with cblog<commit_after>// Package cblog provides a channel based logger.
package cblog

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
)

// LOG_OUTPUT_BUFFER is the size of the channel buffer used for logging.
const LOG_OUTPUT_BUFFER = 1024

type logMesg struct {
	mesg  string
	fatal bool
}

type loggerHandler interface {
	setup(config map[string]interface{}) error
	write(mesg *logMesg)
}

// Logger is the logging object.
type Logger struct {
	messages chan *logMesg
	outputs  map[string]loggerHandler
}

// New creates a new Logger.
func New() *Logger {
	l := &Logger{
		messages: make(chan *logMesg, LOG_OUTPUT_BUFFER),
		outputs:  make(map[string]loggerHandler),
	}
	go l.run()
	return l
}

// SetLogger sets the Logger object output.
func (l *Logger) SetLogger(handlerType string, cfg map[string]interface{}) {
	// BUG(karasz): SetLogger should be replaced with SetOutput in order to become stdlib compatible
	var handler loggerHandler
	switch handlerType {
	case "console":
		handler = newConsoleHandler()
	case "file":
		handler = newFileHandler()
	default:
		panic("Unknown log handler.")
	}

	handler.setup(cfg)
	l.outputs[handlerType] = handler
}

func (l *Logger) run() {
	for {
		select {
		case mesg := <-l.messages:
			for _, handler := range l.outputs {
				handler.write(mesg)
			}
		}
	}
}

func (l *Logger) writeMesg(mesg string, fatal bool) {
	lm := &logMesg{
		mesg:  mesg,
		fatal: fatal,
	}
	l.messages <- lm
}

// Debug calls l.writeMesg prefixing the message with [DEBUG]
func (l *Logger) Debug(format string, v ...interface{}) {
	mesg := fmt.Sprintf("[DEBUG] "+format, v...)
	l.writeMesg(mesg, false)
}

// Info calls l.writeMesg prefixing the message with [INFO]
func (l *Logger) Info(format string, v ...interface{}) {
	mesg := fmt.Sprintf("[INFO] "+format, v...)
	l.writeMesg(mesg, false)
}

// Notice calls l.writeMesg prefixing the message with [NOTICE]
func (l *Logger) Notice(format string, v ...interface{}) {
	mesg := fmt.Sprintf("[NOTICE] "+format, v...)
	l.writeMesg(mesg, false)
}

// Warn calls l.writeMesg prefixing the message with [WARN]
func (l *Logger) Warn(format string, v ...interface{}) {
	mesg := fmt.Sprintf("[WARN] "+format, v...)
	l.writeMesg(mesg, false)
}

// Error calls l.writeMesg prefixing the message with [ERROR]
func (l *Logger) Error(format string, v ...interface{}) {
	mesg := fmt.Sprintf("[ERROR] "+format, v...)
	l.writeMesg(mesg, false)
}

// Fatal calls l.writeMesg prefixing the message with [FATAL]
func (l *Logger) Fatal(format string, v ...interface{}) {
	mesg := fmt.Sprintf("[FATAL] "+format, v...)
	l.writeMesg(mesg, true)
}

type consoleHandler struct {
	logger *log.Logger
}

func newConsoleHandler() loggerHandler {
	return new(consoleHandler)
}

func (h *consoleHandler) setup(cfg map[string]interface{}) error {
	h.logger = log.New(os.Stdout, "", log.Ldate|log.Ltime)
	return nil
}

func (h *consoleHandler) write(lm *logMesg) {
	if !lm.fatal {
		h.logger.Println(lm.mesg)
	} else {
		h.logger.Fatalln(lm.mesg)
	}
}

type fileHandler struct {
	file   string
	logger *log.Logger
}

func newFileHandler() loggerHandler {
	return new(fileHandler)
}

func (h *fileHandler) setup(config map[string]interface{}) error {
	if file, ok := config["file"]; ok {
		h.file = file.(string)
		if _, err := os.Stat(h.file); os.IsNotExist(err) {
			_ = os.MkdirAll(filepath.Dir(h.file), 0755)
			if _, err := os.Create(h.file); err != nil {
				return err
			}
		}

		output, _ := os.Create(h.file)
		h.logger = log.New(output, "", log.Ldate|log.Ltime)
	}

	return nil
}

func (h *fileHandler) write(lm *logMesg) {
	if h.logger == nil {
		return
	}

	if !lm.fatal {
		h.logger.Println(lm.mesg)
	} else {
		h.logger.Fatalln(lm.mesg)
	}
}
<|endoftext|>


<commit_before>package hstspreload

import (
	"fmt"
	"strconv"
	"strings"
)

const (
	// Value that indictes when HSTSHeader.MaxAge is invalid.
	MAX_AGE_NOT_PRESENT = (-1)

	// 18 weeks
	hstsMinimumMaxAge = 10886400 // seconds

	// 1 year: https://code.google.com/p/chromium/codesearch#chromium/src/net/http/http_security_headers.h&q=kMaxHSTSAgeSecs
	hstsChromeMaxAgeCapOneYear = 86400 * 365 // seconds
)

// Unless all values are known at initialization time, use
// NewHSTSHeader() instead of constructing an `HSTSHeader` directly.
// This makes sure that `MaxAge` is initialized to
// `MAX_AGE_NOT_PRESENT`.
type HSTSHeader struct {
	// MaxAge == MAX_AGE_NOT_PRESENT indicates that this value is invalid.
	// A valid `maxAge` value is a non-negative integer.
	MaxAge            int64
	IncludeSubDomains bool
	Preload           bool
}

func NewHSTSHeader() HSTSHeader {
	return HSTSHeader{
		Preload:           false,
		IncludeSubDomains: false,
		MaxAge:            MAX_AGE_NOT_PRESENT,
	}
}

// Mainly useful for testing.
func headersEqual(header1 HSTSHeader, header2 HSTSHeader) bool {
	if header1.Preload != header2.Preload {
		return false
	}

	if header1.IncludeSubDomains != header2.IncludeSubDomains {
		return false
	}

	if header1.MaxAge != header2.MaxAge {
		return false
	}

	return true
}

// Iff Issues has no errors, the output integer is the max-age in seconds.
func parseMaxAge(directive string) (int64, Issues) {
	issues := NewIssues()

	maxAgeNumericalString := directive[8:]

	// TODO: Use more concise validation code to parse a digit string to a signed int.
	for i, c := range maxAgeNumericalString {
		if i == 0 && c == '0' && len(maxAgeNumericalString) > 1 {
			issues = issues.addWarning(fmt.Sprintf("Syntax warning: max-age value contains a leading 0: `%s`", directive))
		}
		if c < '0' || c > '9' {
			return MAX_AGE_NOT_PRESENT, issues.addError(fmt.Sprintf("Syntax error: max-age value contains characters that are not digits: `%s`", directive))
		}
	}

	maxAge, err := strconv.ParseInt(maxAgeNumericalString, 10, 64)

	if err != nil {
		return MAX_AGE_NOT_PRESENT, issues.addError(fmt.Sprintf("Syntax error: Could not parse max-age value `%s`.", maxAgeNumericalString))
	}

	if maxAge < 0 {
		return MAX_AGE_NOT_PRESENT, issues.addError(fmt.Sprintf("Internal error: unexpected negative integer: `%d`", maxAge))
	}

	return maxAge, issues
}

// This function parses an HSTS header.
//
// It will report syntax errors and warnings, but does NOT calculate
// whether the header value is semantically valid.
//
// To interpret the issues, see the list of conventions in the
// documentation for `Issues`.
//
// Example Usage:
//
//     hstsHeader, issues := ParseHeaderString("includeSubDomains; max-age;")
//
//     issues.Errors[0] == []string{"Syntax error: A max-age directive name is present without an associated value."}
//     issues.Warnings[0] == []string{"Syntax warning: Header includes an empty directive or extra semicolon."}
func ParseHeaderString(headerString string) (HSTSHeader, Issues) {
	hstsHeader := NewHSTSHeader()
	issues := NewIssues()

	directives := strings.Split(headerString, ";")
	for i, directive := range directives {
		// TODO: this trims more than spaces and tabs (LWS). https://crbug.com/596561#c10
		directives[i] = strings.TrimSpace(directive)
	}

	// If strings.Split() is given whitespace, it still returns an (empty) directive.
	// So we handle this case separately.
	if len(directives) == 1 && directives[0] == "" {
		// Return immediately, because all the extra information is redundant.
		return hstsHeader, issues.addWarning("Syntax warning: Header is empty.")
	}

	for _, directive := range directives {
		directiveEqualsIgnoringCase := func(s string) bool {
			return strings.ToLower(directive) == strings.ToLower(s)
		}

		directiveHasPrefixIgnoringCase := func(prefix string) bool {
			return strings.HasPrefix(strings.ToLower(directive), strings.ToLower(prefix))
		}

		switch {
		case directiveEqualsIgnoringCase("preload"):
			if hstsHeader.Preload {
				issues = issues.addUniqueWarning("Syntax warning: Header contains a repeated directive: `preload`")
			} else {
				hstsHeader.Preload = true
			}

		case directiveHasPrefixIgnoringCase("preload"):
			issues = issues.addUniqueWarning("Syntax warning: Header contains a `preload` directive with extra directives.")

		case directiveEqualsIgnoringCase("includeSubDomains"):
			if hstsHeader.IncludeSubDomains {
				issues = issues.addUniqueWarning("Syntax warning: Header contains a repeated directive: `includeSubDomains`")
			} else {
				hstsHeader.IncludeSubDomains = true
				if directive != "includeSubDomains" {
					issues = issues.addUniqueWarning(fmt.Sprintf(
						"Syntax warning: Header contains the token `%s`. The recommended capitalization is `includeSubDomains`.",
						directive,
					))
				}
			}

		case directiveHasPrefixIgnoringCase("includeSubDomains"):
			issues = issues.addUniqueWarning("Syntax warning: Header contains an `includeSubDomains` directive with extra directives.")

		case directiveHasPrefixIgnoringCase("max-age="):
			maxAge, maxAgeIssues := parseMaxAge(directive)
			issues = combineIssues(issues, maxAgeIssues)

			if len(maxAgeIssues.Errors) > 0 {
				continue
			}

			if hstsHeader.MaxAge == MAX_AGE_NOT_PRESENT {
				hstsHeader.MaxAge = maxAge
			} else {
				issues = issues.addUniqueWarning(fmt.Sprintf("Syntax warning: Header contains a repeated directive: `max-age`"))
			}

		case directiveHasPrefixIgnoringCase("max-age"):
			issues = issues.addUniqueError("Syntax error: A max-age directive name is present without an associated value.")

		case directiveEqualsIgnoringCase(""):
			issues = issues.addUniqueWarning("Syntax warning: Header includes an empty directive or extra semicolon.")

		default:
			issues = issues.addWarning(fmt.Sprintf("Syntax warning: Header contains an unknown directive: `%s`", directive))
		}
	}
	return hstsHeader, issues
}

// This function checks whether the `HSTSHeader` matches all
// requirements for preloading in Chromium.
//
// To interpret the result, see the list of conventions in the
// documentation for `Issues`.
//
// Most of the time, you'll probably want to use `CheckHeaderString()` instead.
func CheckHeader(hstsHeader HSTSHeader) Issues {
	issues := NewIssues()

	if !hstsHeader.IncludeSubDomains {
		issues = issues.addError("Header requirement error: Header must contain the `includeSubDomains` directive.")
	}

	if !hstsHeader.Preload {
		issues = issues.addError("Header requirement error: Header must contain the `preload` directive.")
	}

	switch {
	case hstsHeader.MaxAge == MAX_AGE_NOT_PRESENT:
		issues = issues.addError("Header requirement error: Header must contain a valid `max-age` directive.")

	case hstsHeader.MaxAge < 0:
		issues = issues.addError(fmt.Sprintf("Internal error: encountered an HSTSHeader with a negative max-age that does not equal MAX_AGE_NOT_PRESENT: %d", hstsHeader.MaxAge))

	case hstsHeader.MaxAge < hstsMinimumMaxAge:
		issues = issues.addError(fmt.Sprintf(
			"Header requirement error: The max-age must be at least 10886400 seconds (== 18 weeks), but the header only had max-age=%d.",
			hstsHeader.MaxAge,
		))

	case hstsHeader.MaxAge > hstsChromeMaxAgeCapOneYear:
		issues = issues.addWarning(fmt.Sprintf(
			"Header FYI: The max-age (%d seconds) is longer than a year. Note that Chrome will round HSTS header max-age values down to 1 year (%d seconds).",
			hstsHeader.MaxAge,
			hstsChromeMaxAgeCapOneYear,
		))

	}

	return issues
}

// This convenience function calls ParseHeaderString() and then calls on
// the parsed headerCheckHeader(). It returns all issues from both calls, combined.
//
// To interpret the result, see the list of conventions in the
// documentation for `Issues`.
//
// Example Usage:
//
//     hstsHeader, issues := ParseHeaderString("includeSubDomains; max-age;")
//
//     hstsHeader.Errors[0] == "Header requirement error: Header must contain the `preload` directive."
//     hstsHeader.Warnings[0] == "Header FYI: The max-age (31536001 seconds) is longer than a year. Note that Chrome will round HSTS header max-age values down to 1 year (31536000 seconds)."
func CheckHeaderString(headerString string) Issues {
	hstsHeader, issues := ParseHeaderString(headerString)
	return combineIssues(issues, CheckHeader(hstsHeader))
}
<commit_msg>Fix a MaxAge capitalization type in the HSTSHeader struct doc comment.<commit_after>package hstspreload

import (
	"fmt"
	"strconv"
	"strings"
)

const (
	// Value that indictes when HSTSHeader.MaxAge is invalid.
	MAX_AGE_NOT_PRESENT = (-1)

	// 18 weeks
	hstsMinimumMaxAge = 10886400 // seconds

	// 1 year: https://code.google.com/p/chromium/codesearch#chromium/src/net/http/http_security_headers.h&q=kMaxHSTSAgeSecs
	hstsChromeMaxAgeCapOneYear = 86400 * 365 // seconds
)

// Unless all values are known at initialization time, use
// NewHSTSHeader() instead of constructing an `HSTSHeader` directly.
// This makes sure that `MaxAge` is initialized to
// `MAX_AGE_NOT_PRESENT`.
type HSTSHeader struct {
	// MaxAge == MAX_AGE_NOT_PRESENT indicates that this value is invalid.
	// A valid `MaxAge` value is a non-negative integer.
	MaxAge            int64
	IncludeSubDomains bool
	Preload           bool
}

func NewHSTSHeader() HSTSHeader {
	return HSTSHeader{
		Preload:           false,
		IncludeSubDomains: false,
		MaxAge:            MAX_AGE_NOT_PRESENT,
	}
}

// Mainly useful for testing.
func headersEqual(header1 HSTSHeader, header2 HSTSHeader) bool {
	if header1.Preload != header2.Preload {
		return false
	}

	if header1.IncludeSubDomains != header2.IncludeSubDomains {
		return false
	}

	if header1.MaxAge != header2.MaxAge {
		return false
	}

	return true
}

// Iff Issues has no errors, the output integer is the max-age in seconds.
func parseMaxAge(directive string) (int64, Issues) {
	issues := NewIssues()

	maxAgeNumericalString := directive[8:]

	// TODO: Use more concise validation code to parse a digit string to a signed int.
	for i, c := range maxAgeNumericalString {
		if i == 0 && c == '0' && len(maxAgeNumericalString) > 1 {
			issues = issues.addWarning(fmt.Sprintf("Syntax warning: max-age value contains a leading 0: `%s`", directive))
		}
		if c < '0' || c > '9' {
			return MAX_AGE_NOT_PRESENT, issues.addError(fmt.Sprintf("Syntax error: max-age value contains characters that are not digits: `%s`", directive))
		}
	}

	maxAge, err := strconv.ParseInt(maxAgeNumericalString, 10, 64)

	if err != nil {
		return MAX_AGE_NOT_PRESENT, issues.addError(fmt.Sprintf("Syntax error: Could not parse max-age value `%s`.", maxAgeNumericalString))
	}

	if maxAge < 0 {
		return MAX_AGE_NOT_PRESENT, issues.addError(fmt.Sprintf("Internal error: unexpected negative integer: `%d`", maxAge))
	}

	return maxAge, issues
}

// This function parses an HSTS header.
//
// It will report syntax errors and warnings, but does NOT calculate
// whether the header value is semantically valid.
//
// To interpret the issues, see the list of conventions in the
// documentation for `Issues`.
//
// Example Usage:
//
//     hstsHeader, issues := ParseHeaderString("includeSubDomains; max-age;")
//
//     issues.Errors[0] == []string{"Syntax error: A max-age directive name is present without an associated value."}
//     issues.Warnings[0] == []string{"Syntax warning: Header includes an empty directive or extra semicolon."}
func ParseHeaderString(headerString string) (HSTSHeader, Issues) {
	hstsHeader := NewHSTSHeader()
	issues := NewIssues()

	directives := strings.Split(headerString, ";")
	for i, directive := range directives {
		// TODO: this trims more than spaces and tabs (LWS). https://crbug.com/596561#c10
		directives[i] = strings.TrimSpace(directive)
	}

	// If strings.Split() is given whitespace, it still returns an (empty) directive.
	// So we handle this case separately.
	if len(directives) == 1 && directives[0] == "" {
		// Return immediately, because all the extra information is redundant.
		return hstsHeader, issues.addWarning("Syntax warning: Header is empty.")
	}

	for _, directive := range directives {
		directiveEqualsIgnoringCase := func(s string) bool {
			return strings.ToLower(directive) == strings.ToLower(s)
		}

		directiveHasPrefixIgnoringCase := func(prefix string) bool {
			return strings.HasPrefix(strings.ToLower(directive), strings.ToLower(prefix))
		}

		switch {
		case directiveEqualsIgnoringCase("preload"):
			if hstsHeader.Preload {
				issues = issues.addUniqueWarning("Syntax warning: Header contains a repeated directive: `preload`")
			} else {
				hstsHeader.Preload = true
			}

		case directiveHasPrefixIgnoringCase("preload"):
			issues = issues.addUniqueWarning("Syntax warning: Header contains a `preload` directive with extra directives.")

		case directiveEqualsIgnoringCase("includeSubDomains"):
			if hstsHeader.IncludeSubDomains {
				issues = issues.addUniqueWarning("Syntax warning: Header contains a repeated directive: `includeSubDomains`")
			} else {
				hstsHeader.IncludeSubDomains = true
				if directive != "includeSubDomains" {
					issues = issues.addUniqueWarning(fmt.Sprintf(
						"Syntax warning: Header contains the token `%s`. The recommended capitalization is `includeSubDomains`.",
						directive,
					))
				}
			}

		case directiveHasPrefixIgnoringCase("includeSubDomains"):
			issues = issues.addUniqueWarning("Syntax warning: Header contains an `includeSubDomains` directive with extra directives.")

		case directiveHasPrefixIgnoringCase("max-age="):
			maxAge, maxAgeIssues := parseMaxAge(directive)
			issues = combineIssues(issues, maxAgeIssues)

			if len(maxAgeIssues.Errors) > 0 {
				continue
			}

			if hstsHeader.MaxAge == MAX_AGE_NOT_PRESENT {
				hstsHeader.MaxAge = maxAge
			} else {
				issues = issues.addUniqueWarning(fmt.Sprintf("Syntax warning: Header contains a repeated directive: `max-age`"))
			}

		case directiveHasPrefixIgnoringCase("max-age"):
			issues = issues.addUniqueError("Syntax error: A max-age directive name is present without an associated value.")

		case directiveEqualsIgnoringCase(""):
			issues = issues.addUniqueWarning("Syntax warning: Header includes an empty directive or extra semicolon.")

		default:
			issues = issues.addWarning(fmt.Sprintf("Syntax warning: Header contains an unknown directive: `%s`", directive))
		}
	}
	return hstsHeader, issues
}

// This function checks whether the `HSTSHeader` matches all
// requirements for preloading in Chromium.
//
// To interpret the result, see the list of conventions in the
// documentation for `Issues`.
//
// Most of the time, you'll probably want to use `CheckHeaderString()` instead.
func CheckHeader(hstsHeader HSTSHeader) Issues {
	issues := NewIssues()

	if !hstsHeader.IncludeSubDomains {
		issues = issues.addError("Header requirement error: Header must contain the `includeSubDomains` directive.")
	}

	if !hstsHeader.Preload {
		issues = issues.addError("Header requirement error: Header must contain the `preload` directive.")
	}

	switch {
	case hstsHeader.MaxAge == MAX_AGE_NOT_PRESENT:
		issues = issues.addError("Header requirement error: Header must contain a valid `max-age` directive.")

	case hstsHeader.MaxAge < 0:
		issues = issues.addError(fmt.Sprintf("Internal error: encountered an HSTSHeader with a negative max-age that does not equal MAX_AGE_NOT_PRESENT: %d", hstsHeader.MaxAge))

	case hstsHeader.MaxAge < hstsMinimumMaxAge:
		issues = issues.addError(fmt.Sprintf(
			"Header requirement error: The max-age must be at least 10886400 seconds (== 18 weeks), but the header only had max-age=%d.",
			hstsHeader.MaxAge,
		))

	case hstsHeader.MaxAge > hstsChromeMaxAgeCapOneYear:
		issues = issues.addWarning(fmt.Sprintf(
			"Header FYI: The max-age (%d seconds) is longer than a year. Note that Chrome will round HSTS header max-age values down to 1 year (%d seconds).",
			hstsHeader.MaxAge,
			hstsChromeMaxAgeCapOneYear,
		))

	}

	return issues
}

// This convenience function calls ParseHeaderString() and then calls on
// the parsed headerCheckHeader(). It returns all issues from both calls, combined.
//
// To interpret the result, see the list of conventions in the
// documentation for `Issues`.
//
// Example Usage:
//
//     hstsHeader, issues := ParseHeaderString("includeSubDomains; max-age;")
//
//     hstsHeader.Errors[0] == "Header requirement error: Header must contain the `preload` directive."
//     hstsHeader.Warnings[0] == "Header FYI: The max-age (31536001 seconds) is longer than a year. Note that Chrome will round HSTS header max-age values down to 1 year (31536000 seconds)."
func CheckHeaderString(headerString string) Issues {
	hstsHeader, issues := ParseHeaderString(headerString)
	return combineIssues(issues, CheckHeader(hstsHeader))
}
<|endoftext|>


<commit_before>package compiler

import (
	boshmodels "bosh/agent/applier/models"
	fakepa "bosh/agent/applier/packageapplier/fakes"
	fakeblobstore "bosh/blobstore/fakes"
	fakecmd "bosh/platform/commands/fakes"
	boshdirs "bosh/settings/directories"
	boshsys "bosh/system"
	fakesys "bosh/system/fakes"
	"github.com/stretchr/testify/assert"
	"os"
	"testing"
)

func TestCompileReturnsBlobIdAndSha1(t *testing.T) {
	_, blobstore, _, _, _, compiler := buildCompiler()

	blobstore.CreateBlobId = "my-blob-id"
	blobstore.CreateFingerprint = "blob-sha1"
	pkg, deps := getCompileArgs()

	blobId, sha1, err := compiler.Compile(pkg, deps)
	assert.NoError(t, err)

	assert.Equal(t, "my-blob-id", blobId)
	assert.Equal(t, "blob-sha1", sha1)
}

func TestCompileFetchesSourcePackageFromBlobstore(t *testing.T) {
	_, blobstore, _, _, _, compiler := buildCompiler()

	pkg, deps := getCompileArgs()

	_, _, err := compiler.Compile(pkg, deps)
	assert.NoError(t, err)

	assert.Equal(t, "blobstore_id", blobstore.GetBlobIds[0])
	assert.Equal(t, "sha1", blobstore.GetFingerprints[0])
}

func TestCompileInstallsDependentPackages(t *testing.T) {
	_, _, _, _, packageApplier, compiler := buildCompiler()

	pkg, deps := getCompileArgs()

	_, _, err := compiler.Compile(pkg, deps)
	assert.NoError(t, err)

	assert.Equal(t, packageApplier.AppliedPackages, deps)
}

func TestCompileExtractsSourcePkgToCompileDir(t *testing.T) {
	compressor, blobstore, fs, _, _, compiler := buildCompiler()

	pkg, deps := getCompileArgs()

	blobstore.GetFileName = "/dev/null"

	_, _, err := compiler.Compile(pkg, deps)
	assert.NoError(t, err)

	assert.True(t, fs.FileExists("/fake-dir/data/compile/pkg_name"))
	assert.Equal(t, compressor.DecompressFileToDirDirs[0], "/fake-dir/data/compile/pkg_name-bosh-agent-unpack")
	assert.Equal(t, compressor.DecompressFileToDirTarballPaths[0], blobstore.GetFileName)

	assert.Equal(t, fs.RenameOldPaths[0], "/fake-dir/data/compile/pkg_name-bosh-agent-unpack")
	assert.Equal(t, fs.RenameNewPaths[0], "/fake-dir/data/compile/pkg_name")
}

func TestCompileCreatesInstallDir(t *testing.T) {
	_, _, fs, _, _, compiler := buildCompiler()

	pkg, deps := getCompileArgs()

	installDir := "/fake-dir/data/packages/pkg_name/pkg_version"

	assert.False(t, fs.FileExists(installDir))

	_, _, err := compiler.Compile(pkg, deps)
	assert.NoError(t, err)

	assert.True(t, fs.FileExists(installDir))
	installDirStats := fs.GetFileTestStat(installDir)
	assert.Equal(t, os.FileMode(0755), installDirStats.FileMode.Perm())
}

func TestCompileRecreatesInstallDir(t *testing.T) {
	_, _, fs, _, _, compiler := buildCompiler()

	pkg, deps := getCompileArgs()

	err := fs.MkdirAll("/fake-dir/data/packages/pkg_name/pkg_version", os.FileMode(0755))
	assert.NoError(t, err)

	_, err = fs.WriteToFile("/fake-dir/data/packages/pkg_name/pkg_version/should_be_deleted", "test")
	assert.NoError(t, err)

	assert.True(t, fs.FileExists("/fake-dir/data/packages/pkg_name/pkg_version/should_be_deleted"))

	_, _, err = compiler.Compile(pkg, deps)
	assert.NoError(t, err)

	assert.False(t, fs.FileExists("/fake-dir/data/packages/pkg_name/pkg_version/should_be_deleted"))
}

func TestCompileSymlinksInstallDir(t *testing.T) {
	_, _, fs, _, _, compiler := buildCompiler()

	pkg, deps := getCompileArgs()

	_, _, err := compiler.Compile(pkg, deps)
	assert.NoError(t, err)

	fileStats := fs.GetFileTestStat("/fake-dir/packages/pkg_name")
	assert.NotNil(t, fileStats)
	assert.Equal(t, fakesys.FakeFileTypeSymlink, fileStats.FileType)
	assert.Equal(t, "/fake-dir/data/packages/pkg_name/pkg_version", fileStats.SymlinkTarget)
}

func TestCompileCompressesCompiledPackage(t *testing.T) {
	compressor, _, _, _, _, compiler := buildCompiler()

	pkg, deps := getCompileArgs()

	_, _, err := compiler.Compile(pkg, deps)
	assert.NoError(t, err)

	assert.Equal(t, "/fake-dir/data/packages/pkg_name/pkg_version", compressor.CompressFilesInDirDir)
}

func TestCompileWhenScriptDoesNotExist(t *testing.T) {
	_, _, _, runner, _, compiler := buildCompiler()

	pkg, deps := getCompileArgs()

	_, _, err := compiler.Compile(pkg, deps)
	assert.NoError(t, err)

	assert.Empty(t, runner.RunCommands)
}

func TestCompileWhenScriptExists(t *testing.T) {
	compressor, _, fs, runner, _, compiler := buildCompiler()

	pkg, deps := getCompileArgs()

	compressor.DecompressFileToDirCallBack = func() {
		fs.WriteToFile("/fake-dir/data/compile/pkg_name/packaging", "hi")
	}

	_, _, err := compiler.Compile(pkg, deps)
	assert.NoError(t, err)

	expectedCmd := boshsys.Command{
		Name: "bash",
		Args: []string{"-x", "packaging"},
		Env: map[string]string{
			"BOSH_COMPILE_TARGET":  "/fake-dir/data/compile/pkg_name",
			"BOSH_INSTALL_TARGET":  "/fake-dir/data/packages/pkg_name/pkg_version",
			"BOSH_PACKAGE_NAME":    "pkg_name",
			"BOSH_PACKAGE_VERSION": "pkg_version",
		},
		WorkingDir: "/fake-dir/data/compile/pkg_name",
	}

	assert.Equal(t, 1, len(runner.RunComplexCommands))
	assert.Equal(t, expectedCmd, runner.RunComplexCommands[0])
}

func TestCompileUploadsCompressedPackage(t *testing.T) {
	compressor, blobstore, _, _, _, compiler := buildCompiler()

	pkg, deps := getCompileArgs()

	compressor.CompressFilesInDirTarballPath = "/tmp/foo"

	_, _, err := compiler.Compile(pkg, deps)
	assert.NoError(t, err)
	assert.Equal(t, "/tmp/foo", blobstore.CreateFileName)
}

func getCompileArgs() (pkg Package, deps []boshmodels.Package) {
	pkg = Package{
		BlobstoreId: "blobstore_id",
		Sha1:        "sha1",
		Name:        "pkg_name",
		Version:     "pkg_version",
	}
	deps = []boshmodels.Package{
		{
			Name:    "first_dep",
			Version: "first_dep_version",
			Source: boshmodels.Source{
				Sha1:        "first_dep_sha1",
				BlobstoreId: "first_dep_blobstore_id",
			},
		},
		{
			Name:    "sec_dep",
			Version: "sec_dep_version",
			Source: boshmodels.Source{
				Sha1:        "sec_dep_sha1",
				BlobstoreId: "sec_dep_blobstore_id",
			},
		},
	}
	return
}

func buildCompiler() (
	compressor *fakecmd.FakeCompressor,
	blobstore *fakeblobstore.FakeBlobstore,
	fs *fakesys.FakeFileSystem,
	runner *fakesys.FakeCmdRunner,
	packageApplier *fakepa.FakePackageApplier,
	compiler Compiler,
) {
	compressor = fakecmd.NewFakeCompressor()
	blobstore = &fakeblobstore.FakeBlobstore{}
	fs = fakesys.NewFakeFileSystem()
	runner = fakesys.NewFakeCmdRunner()
	packageApplier = fakepa.NewFakePackageApplier()
	compiler = NewConcreteCompiler(compressor, blobstore, fs, runner, boshdirs.NewDirectoriesProvider("/fake-dir"), packageApplier)
	return
}
<commit_msg>use dependency struct to simplify tests for concreteCompiler<commit_after>package compiler

import (
	boshmodels "bosh/agent/applier/models"
	fakepa "bosh/agent/applier/packageapplier/fakes"
	fakeblobstore "bosh/blobstore/fakes"
	fakecmd "bosh/platform/commands/fakes"
	boshdirs "bosh/settings/directories"
	boshsys "bosh/system"
	fakesys "bosh/system/fakes"
	"github.com/stretchr/testify/assert"
	"os"
	"testing"
)

func TestCompileReturnsBlobIdAndSha1(t *testing.T) {
	deps, compiler := buildCompiler()

	deps.blobstore.CreateBlobId = "my-blob-id"
	deps.blobstore.CreateFingerprint = "blob-sha1"
	pkg, pkgDeps := getCompileArgs()

	blobId, sha1, err := compiler.Compile(pkg, pkgDeps)
	assert.NoError(t, err)

	assert.Equal(t, "my-blob-id", blobId)
	assert.Equal(t, "blob-sha1", sha1)
}

func TestCompileFetchesSourcePackageFromBlobstore(t *testing.T) {
	deps, compiler := buildCompiler()

	pkg, pkgDeps := getCompileArgs()

	_, _, err := compiler.Compile(pkg, pkgDeps)
	assert.NoError(t, err)

	assert.Equal(t, "blobstore_id", deps.blobstore.GetBlobIds[0])
	assert.Equal(t, "sha1", deps.blobstore.GetFingerprints[0])
}

func TestCompileInstallsDependentPackages(t *testing.T) {
	deps, compiler := buildCompiler()

	pkg, pkgDeps := getCompileArgs()

	_, _, err := compiler.Compile(pkg, pkgDeps)
	assert.NoError(t, err)

	assert.Equal(t, deps.packageApplier.AppliedPackages, pkgDeps)
}

func TestCompileExtractsSourcePkgToCompileDir(t *testing.T) {
	deps, compiler := buildCompiler()

	pkg, pkgDeps := getCompileArgs()

	deps.blobstore.GetFileName = "/dev/null"

	_, _, err := compiler.Compile(pkg, pkgDeps)
	assert.NoError(t, err)

	assert.True(t, deps.fs.FileExists("/fake-dir/data/compile/pkg_name"))
	assert.Equal(t, deps.compressor.DecompressFileToDirDirs[0], "/fake-dir/data/compile/pkg_name-bosh-agent-unpack")
	assert.Equal(t, deps.compressor.DecompressFileToDirTarballPaths[0], deps.blobstore.GetFileName)

	assert.Equal(t, deps.fs.RenameOldPaths[0], "/fake-dir/data/compile/pkg_name-bosh-agent-unpack")
	assert.Equal(t, deps.fs.RenameNewPaths[0], "/fake-dir/data/compile/pkg_name")
}

func TestCompileCreatesInstallDir(t *testing.T) {
	deps, compiler := buildCompiler()

	pkg, pkgDeps := getCompileArgs()

	installDir := "/fake-dir/data/packages/pkg_name/pkg_version"

	assert.False(t, deps.fs.FileExists(installDir))

	_, _, err := compiler.Compile(pkg, pkgDeps)
	assert.NoError(t, err)

	assert.True(t, deps.fs.FileExists(installDir))
	installDirStats := deps.fs.GetFileTestStat(installDir)
	assert.Equal(t, os.FileMode(0755), installDirStats.FileMode.Perm())
}

func TestCompileRecreatesInstallDir(t *testing.T) {
	deps, compiler := buildCompiler()

	pkg, pkgDeps := getCompileArgs()

	err := deps.fs.MkdirAll("/fake-dir/data/packages/pkg_name/pkg_version", os.FileMode(0755))
	assert.NoError(t, err)

	_, err = deps.fs.WriteToFile("/fake-dir/data/packages/pkg_name/pkg_version/should_be_deleted", "test")
	assert.NoError(t, err)

	assert.True(t, deps.fs.FileExists("/fake-dir/data/packages/pkg_name/pkg_version/should_be_deleted"))

	_, _, err = compiler.Compile(pkg, pkgDeps)
	assert.NoError(t, err)

	assert.False(t, deps.fs.FileExists("/fake-dir/data/packages/pkg_name/pkg_version/should_be_deleted"))
}

func TestCompileSymlinksInstallDir(t *testing.T) {
	deps, compiler := buildCompiler()

	pkg, pkgDeps := getCompileArgs()

	_, _, err := compiler.Compile(pkg, pkgDeps)
	assert.NoError(t, err)

	fileStats := deps.fs.GetFileTestStat("/fake-dir/packages/pkg_name")
	assert.NotNil(t, fileStats)
	assert.Equal(t, fakesys.FakeFileTypeSymlink, fileStats.FileType)
	assert.Equal(t, "/fake-dir/data/packages/pkg_name/pkg_version", fileStats.SymlinkTarget)
}

func TestCompileCompressesCompiledPackage(t *testing.T) {
	deps, compiler := buildCompiler()

	pkg, pkgDeps := getCompileArgs()

	_, _, err := compiler.Compile(pkg, pkgDeps)
	assert.NoError(t, err)

	assert.Equal(t, "/fake-dir/data/packages/pkg_name/pkg_version", deps.compressor.CompressFilesInDirDir)
}

func TestCompileWhenScriptDoesNotExist(t *testing.T) {
	deps, compiler := buildCompiler()

	pkg, pkgDeps := getCompileArgs()

	_, _, err := compiler.Compile(pkg, pkgDeps)
	assert.NoError(t, err)

	assert.Empty(t, deps.runner.RunCommands)
}

func TestCompileWhenScriptExists(t *testing.T) {
	deps, compiler := buildCompiler()

	pkg, pkgDeps := getCompileArgs()

	deps.compressor.DecompressFileToDirCallBack = func() {
		deps.fs.WriteToFile("/fake-dir/data/compile/pkg_name/packaging", "hi")
	}

	_, _, err := compiler.Compile(pkg, pkgDeps)
	assert.NoError(t, err)

	expectedCmd := boshsys.Command{
		Name: "bash",
		Args: []string{"-x", "packaging"},
		Env: map[string]string{
			"BOSH_COMPILE_TARGET":  "/fake-dir/data/compile/pkg_name",
			"BOSH_INSTALL_TARGET":  "/fake-dir/data/packages/pkg_name/pkg_version",
			"BOSH_PACKAGE_NAME":    "pkg_name",
			"BOSH_PACKAGE_VERSION": "pkg_version",
		},
		WorkingDir: "/fake-dir/data/compile/pkg_name",
	}

	assert.Equal(t, 1, len(deps.runner.RunComplexCommands))
	assert.Equal(t, expectedCmd, deps.runner.RunComplexCommands[0])
}

func TestCompileUploadsCompressedPackage(t *testing.T) {
	deps, compiler := buildCompiler()

	pkg, pkgDeps := getCompileArgs()

	deps.compressor.CompressFilesInDirTarballPath = "/tmp/foo"

	_, _, err := compiler.Compile(pkg, pkgDeps)
	assert.NoError(t, err)
	assert.Equal(t, "/tmp/foo", deps.blobstore.CreateFileName)
}

func getCompileArgs() (pkg Package, pkgDeps []boshmodels.Package) {
	pkg = Package{
		BlobstoreId: "blobstore_id",
		Sha1:        "sha1",
		Name:        "pkg_name",
		Version:     "pkg_version",
	}
	pkgDeps = []boshmodels.Package{
		{
			Name:    "first_dep",
			Version: "first_dep_version",
			Source: boshmodels.Source{
				Sha1:        "first_dep_sha1",
				BlobstoreId: "first_dep_blobstore_id",
			},
		},
		{
			Name:    "sec_dep",
			Version: "sec_dep_version",
			Source: boshmodels.Source{
				Sha1:        "sec_dep_sha1",
				BlobstoreId: "sec_dep_blobstore_id",
			},
		},
	}
	return
}

type compilerDeps struct {
	compressor     *fakecmd.FakeCompressor
	blobstore      *fakeblobstore.FakeBlobstore
	fs             *fakesys.FakeFileSystem
	runner         *fakesys.FakeCmdRunner
	packageApplier *fakepa.FakePackageApplier
}

func buildCompiler() (
	deps compilerDeps,
	compiler Compiler,
) {
	deps.compressor = fakecmd.NewFakeCompressor()
	deps.blobstore = &fakeblobstore.FakeBlobstore{}
	deps.fs = fakesys.NewFakeFileSystem()
	deps.runner = fakesys.NewFakeCmdRunner()
	deps.packageApplier = fakepa.NewFakePackageApplier()

	compiler = NewConcreteCompiler(
		deps.compressor,
		deps.blobstore,
		deps.fs,
		deps.runner,
		boshdirs.NewDirectoriesProvider("/fake-dir"),
		deps.packageApplier,
	)
	return
}
<|endoftext|>