<commit_before>package expirecache

import (
	"math/rand"
	"sync"
	"time"
)

type element struct {
	validUntil time.Time
	data       interface{}
	size       uint64
}

type Cache struct {
	sync.Mutex
	cache     map[string]element
	keys      []string
	totalSize uint64
	maxSize   uint64
}

func New(maxSize uint64) *Cache {
	return &Cache{
		cache:   make(map[string]element),
		maxSize: maxSize,
	}
}

func (ec *Cache) Size() uint64 {
	ec.Lock()
	s := ec.totalSize
	ec.Unlock()
	return s
}

func (ec *Cache) Items() int {
	ec.Lock()
	k := len(ec.keys)
	ec.Unlock()
	return k
}

func (ec *Cache) Get(k string) (interface{}, bool) {
	ec.Lock()
	v, ok := ec.cache[k]
	ec.Unlock()
	if !ok || v.validUntil.Before(timeNow()) {
		// Can't actually delete this element from the cache here since
		// we can't remove the key from ec.keys without a linear search.
		// It'll get removed during the next cleanup
		return nil, false
	}
	return v.data, ok
}

func (ec *Cache) Set(k string, v interface{}, size uint64, expire int32) {
	ec.Lock()
	oldv, ok := ec.cache[k]
	if !ok {
		ec.keys = append(ec.keys, k)
	} else {
		ec.totalSize -= oldv.size
	}

	ec.totalSize += size
	ec.cache[k] = element{validUntil: timeNow().Add(time.Duration(expire) * time.Second), data: v, size: size}

	for ec.maxSize > 0 && ec.totalSize > ec.maxSize {
		ec.randomEvict()
	}

	ec.Unlock()
}

func (ec *Cache) randomEvict() {
	slot := rand.Intn(len(ec.keys))
	k := ec.keys[slot]

	ec.keys[slot] = ec.keys[len(ec.keys)-1]
	ec.keys = ec.keys[:len(ec.keys)-1]

	v := ec.cache[k]
	ec.totalSize -= v.size

	delete(ec.cache, k)
}

func (ec *Cache) Cleaner(d time.Duration) {

	for {
		cleanerSleep(d)

		now := timeNow()
		ec.Lock()

		// We could potentially be holding this lock for a long time,
		// but since we keep the cache expiration times small, we
		// expect only a small number of elements here to loop over

		for i := 0; i < len(ec.keys); i++ {
			k := ec.keys[i]
			v := ec.cache[k]
			if v.validUntil.Before(now) {
				ec.totalSize -= v.size
				delete(ec.cache, k)

				ec.keys[i] = ec.keys[len(ec.keys)-1]
				ec.keys = ec.keys[:len(ec.keys)-1]
				i-- // so we reprocess this index
			}
		}

		ec.Unlock()
		cleanerDone()
	}
}

var (
	timeNow      = time.Now
	cleanerSleep = time.Sleep
	cleanerDone  = func() {}
)
<commit_msg>use RWMutex instead of plain Mutex<commit_after>package expirecache

import (
	"math/rand"
	"sync"
	"time"
)

type element struct {
	validUntil time.Time
	data       interface{}
	size       uint64
}

type Cache struct {
	sync.RWMutex
	cache     map[string]element
	keys      []string
	totalSize uint64
	maxSize   uint64
}

func New(maxSize uint64) *Cache {
	return &Cache{
		cache:   make(map[string]element),
		maxSize: maxSize,
	}
}

func (ec *Cache) Size() uint64 {
	ec.RLock()
	s := ec.totalSize
	ec.RUnlock()
	return s
}

func (ec *Cache) Items() int {
	ec.RLock()
	k := len(ec.keys)
	ec.RUnlock()
	return k
}

func (ec *Cache) Get(k string) (interface{}, bool) {
	ec.RLock()
	v, ok := ec.cache[k]
	ec.RUnlock()
	if !ok || v.validUntil.Before(timeNow()) {
		// Can't actually delete this element from the cache here since
		// we can't remove the key from ec.keys without a linear search.
		// It'll get removed during the next cleanup
		return nil, false
	}
	return v.data, ok
}

func (ec *Cache) Set(k string, v interface{}, size uint64, expire int32) {
	ec.Lock()
	oldv, ok := ec.cache[k]
	if !ok {
		ec.keys = append(ec.keys, k)
	} else {
		ec.totalSize -= oldv.size
	}

	ec.totalSize += size
	ec.cache[k] = element{validUntil: timeNow().Add(time.Duration(expire) * time.Second), data: v, size: size}

	for ec.maxSize > 0 && ec.totalSize > ec.maxSize {
		ec.randomEvict()
	}

	ec.Unlock()
}

func (ec *Cache) randomEvict() {
	slot := rand.Intn(len(ec.keys))
	k := ec.keys[slot]

	ec.keys[slot] = ec.keys[len(ec.keys)-1]
	ec.keys = ec.keys[:len(ec.keys)-1]

	v := ec.cache[k]
	ec.totalSize -= v.size

	delete(ec.cache, k)
}

func (ec *Cache) Cleaner(d time.Duration) {

	for {
		cleanerSleep(d)

		now := timeNow()
		ec.Lock()

		// We could potentially be holding this lock for a long time,
		// but since we keep the cache expiration times small, we
		// expect only a small number of elements here to loop over

		for i := 0; i < len(ec.keys); i++ {
			k := ec.keys[i]
			v := ec.cache[k]
			if v.validUntil.Before(now) {
				ec.totalSize -= v.size
				delete(ec.cache, k)

				ec.keys[i] = ec.keys[len(ec.keys)-1]
				ec.keys = ec.keys[:len(ec.keys)-1]
				i-- // so we reprocess this index
			}
		}

		ec.Unlock()
		cleanerDone()
	}
}

var (
	timeNow      = time.Now
	cleanerSleep = time.Sleep
	cleanerDone  = func() {}
)
<|endoftext|>


<commit_before>// Copyright 2017 Giulio Iotti. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"io"
	"log"
	"sort"
	"time"
)

type page struct {
	n      offset
	body   []byte
	expire time.Time
	cached bool
}

func newPage(n offset, body []byte) *page {
	return &page{
		n:    n,
		body: body,
	}
}

func (p *page) WriteTo(w io.Writer) (int64, error) {
	n, err := w.Write(p.body)
	return int64(n), err
}

type group string

type entry struct {
	deadline time.Time
	data     []byte
}

func newEntry(data []byte, d time.Duration) *entry {
	return &entry{
		deadline: time.Now().Add(d),
		data:     data,
	}
}

func (ce *entry) invalid(t time.Time) bool {
	return !ce.deadline.After(t)
}

func (ce *entry) asPage(n offset) *page {
	p := newPage(n, ce.data)
	p.expire = ce.deadline
	return p
}

type entries struct {
	ents map[group]map[offset]*entry
}

func newEntries() *entries {
	return &entries{
		ents: make(map[group]map[offset]*entry),
	}
}

func (e *entries) count() int {
	tot := 0
	for k := range e.ents {
		tot += len(e.ents[k])
	}
	return tot
}

func (e *entries) get(cg group, n offset) (*entry, bool) {
	pages, ok := e.ents[cg]
	if !ok {
		return nil, false
	}
	ce, ok := pages[n]
	return ce, ok
}

func (e *entries) put(cg group, n offset, ce *entry) {
	_, ok := e.ents[cg]
	if !ok {
		e.ents[cg] = make(map[offset]*entry)
	}
	e.ents[cg][n] = ce
}

func (e *entries) has(cg group, n offset, t time.Time) bool {
	_, ok := e.ents[cg]
	if !ok {
		return false
	}
	ent, ok := e.ents[cg][n]
	if !ok {
		return false
	}
	return !ent.invalid(t)
}

// remove assumes the entry exists
func (e *entries) remove(cg group, n offset) {
	delete(e.ents[cg], n)
	if len(e.ents[cg]) == 0 {
		delete(e.ents, cg)
	}
}

func (e *entries) purge(cg group, st *stats) {
	st.mem(-e.sizeof(cg))
	delete(e.ents, cg)
}

func (e *entries) sizeof(cg group) int {
	var s int
	for n := range e.ents[cg] {
		s += len(e.ents[cg][n].data)
	}
	return s
}

func (e *entries) oldestDeadline(cg group) time.Time {
	var t time.Time
	for off := range e.ents[cg] {
		entry := e.ents[cg][off]
		if t.IsZero() || t.After(entry.deadline) {
			t = entry.deadline
		}
	}
	return t
}

func (e *entries) gc(t time.Time, st *stats) {
	for cg, ents := range e.ents {
		for n := range ents {
			if ents[n].invalid(t) {
				st.mem(-len(ents[n].data))
				e.remove(cg, n)
			}
		}
	}
}

type cacheFunc func() error

type cache struct {
	entries *entries
	waits   *waiters
	fetcher *fetcher
	config  *config
	stat    *stats
	events  chan cacheFunc
	debug   func(string, ...interface{})
}

func newCache(f *fetcher, logs *logbuf, cf *config) *cache {
	c := &cache{
		fetcher: f,
		config:  cf,
		events:  make(chan cacheFunc),
		entries: newEntries(),
		waits:   newWaiters(),
		stat:    newStats(),
		debug:   logs.debug,
	}
	go c.gc(cf.gcpause)
	go c.run()
	return c
}

func (c *cache) run() {
	for f := range c.events {
		if err := f(); err != nil {
			log.Print("cache: ", err)
		}
	}
}

func (c *cache) gc(d time.Duration) {
	done := make(chan struct{})
	for {
		time.Sleep(d)
		c.events <- func() error {
			c.debug("running garbage collector cycle, memory is %d", c.stat.Mem)
			c.entries.gc(time.Now(), c.stat)
			c.debug("garbage collection done, memory is %d", c.stat.Mem)
			done <- struct{}{}
			return nil
		}
		<-done
	}
}

type timeGroup struct {
	t  time.Time
	cg group
}

type timeGroups struct {
	entries []timeGroup
}

type byTime []timeGroup

func (a byTime) Len() int           { return len(a) }
func (a byTime) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byTime) Less(i, j int) bool { return a[i].t.After(a[j].t) }

func makeTimeGroups(e *entries) timeGroups {
	var i int
	tg := make([]timeGroup, len(e.ents))
	for cg := range e.ents {
		tg[i] = timeGroup{
			t:  e.oldestDeadline(cg),
			cg: cg,
		}
		i++
	}
	sort.Sort(byTime(tg))
	return timeGroups{tg}
}

func (tg *timeGroups) purgeOldest(c *cache) {
	entry, ts := tg.entries[len(tg.entries)-1], tg.entries[0:len(tg.entries)-1]
	c.entries.purge(entry.cg, c.stat)
	tg.entries = ts
}

func (tg *timeGroups) empty() bool {
	return len(tg.entries) == 0
}

func (c *cache) oom(target int64) {
	c.debug("OOM called: using %d, limit is %d", c.stat.Mem, target)
	tg := makeTimeGroups(c.entries)
	for {
		tg.purgeOldest(c)
		// Repeat unless we have no more entries or we are using less memory than target
		if tg.empty() || !c.stat.above(target) {
			break
		}
	}
	c.debug("OOM: mem now %d", c.stat.Mem)
}

// put inserts a page into the cache (after it was fetched).
func (c *cache) put(cg group, p *page, err error) {
	c.events <- func() error {
		ce := newEntry(p.body, c.config.lifetime)
		// If an entry exists, it will be overwritten
		if ent, ok := c.entries.get(cg, p.n); ok {
			c.stat.mem(-len(ent.data))
		}
		c.entries.put(cg, p.n, ce)
		c.stat.mem(len(p.body))
		c.debug("added page %s/%d", cg, p.n)
		if c.stat.above(c.config.maxMemory) {
			go func() {
				c.events <- func() error {
					c.oom(c.config.maxMemory)
					return nil
				}
			}()
		}
		// If there were waiters, signal that the wait is over
		c.waits.done(cg, p.n)
		return err
	}
}

// prefetch requests pages (n-m, n) and (n, n+m) if not already fetched
func (c *cache) prefetch(cg group, n, m int, t time.Time) {
	i := n - m
	if i < 0 {
		i = 0
	}
	for ; i < n+m; i++ {
		if i == n {
			// just fetched
			continue
		}
		off := offset(i * c.config.incr)
		if c.entries.has(cg, off, t) || c.waits.has(cg, off) {
			// already fetched or requested
			continue
		}
		c.waits.wait(cg, off)
		res := newResource(c.config.tmpl, cg, off)
		c.fetcher.request(newJob(res, c))
	}
}

func (c *cache) request(cg group, n int, t time.Time) chan struct{} {
	off := offset(n * c.config.incr)
	wait := c.waits.wait(cg, off)
	res := newResource(c.config.tmpl, cg, off)
	c.fetcher.request(newJob(res, c))
	c.prefetch(cg, n, c.config.npref, t)
	return wait
}

func (c *cache) stats() *stats {
	var st *stats
	wait := make(chan struct{})
	c.events <- func() error {
		c.stat.Entries = c.entries.count()
		c.stat.Waiters = c.waits.count()
		st = c.stat.clone()
		wait <- struct{}{}
		return nil
	}
	<-wait
	return st
}

func (c *cache) get(cg group, n int) *page {
	var (
		page *page
		wait chan struct{}
	)
	cached := true
	requested := make(chan struct{})
	off := offset(n * c.config.incr)
	c.debug("%s/%d: requesting from cache", cg, off)
	for {
		wait = nil
		c.events <- func() error {
			defer func() { requested <- struct{}{} }()
			if c.waits.has(cg, off) {
				c.waits.wait(cg, off)
			}
			var now time.Time
			ce, ok := c.entries.get(cg, off)
			if ok {
				now = time.Now()
			}
			if !ok || ce.invalid(now) {
				c.debug("%s/%d: not cached, requested", cg, off)
				wait = c.request(cg, n, now)
				return nil
			}
			c.debug("%s/%d: found", cg, off)
			c.prefetch(cg, n, c.config.npref, now)
			c.stat.hit(cached)
			page = ce.asPage(off)
			return nil
		}
		<-requested
		// content was already in cache, return it
		if wait == nil {
			page.cached = cached
			return page
		}
		// We needed to request the object, it was not cached
		cached = false
		// content is being fetched, wait and try to get again
		<-wait
	}
}
<commit_msg>cache: trust stat and don't run GC when nothing to do<commit_after>// Copyright 2017 Giulio Iotti. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"io"
	"log"
	"sort"
	"time"
)

type page struct {
	n      offset
	body   []byte
	expire time.Time
	cached bool
}

func newPage(n offset, body []byte) *page {
	return &page{
		n:    n,
		body: body,
	}
}

func (p *page) WriteTo(w io.Writer) (int64, error) {
	n, err := w.Write(p.body)
	return int64(n), err
}

type group string

type entry struct {
	deadline time.Time
	data     []byte
}

func newEntry(data []byte, d time.Duration) *entry {
	return &entry{
		deadline: time.Now().Add(d),
		data:     data,
	}
}

func (ce *entry) invalid(t time.Time) bool {
	return !ce.deadline.After(t)
}

func (ce *entry) asPage(n offset) *page {
	p := newPage(n, ce.data)
	p.expire = ce.deadline
	return p
}

type entries struct {
	ents map[group]map[offset]*entry
}

func newEntries() *entries {
	return &entries{
		ents: make(map[group]map[offset]*entry),
	}
}

func (e *entries) count() int {
	tot := 0
	for k := range e.ents {
		tot += len(e.ents[k])
	}
	return tot
}

func (e *entries) get(cg group, n offset) (*entry, bool) {
	pages, ok := e.ents[cg]
	if !ok {
		return nil, false
	}
	ce, ok := pages[n]
	return ce, ok
}

func (e *entries) put(cg group, n offset, ce *entry) {
	_, ok := e.ents[cg]
	if !ok {
		e.ents[cg] = make(map[offset]*entry)
	}
	e.ents[cg][n] = ce
}

func (e *entries) has(cg group, n offset, t time.Time) bool {
	_, ok := e.ents[cg]
	if !ok {
		return false
	}
	ent, ok := e.ents[cg][n]
	if !ok {
		return false
	}
	return !ent.invalid(t)
}

// remove assumes the entry exists
func (e *entries) remove(cg group, n offset) {
	delete(e.ents[cg], n)
	if len(e.ents[cg]) == 0 {
		delete(e.ents, cg)
	}
}

func (e *entries) purge(cg group, st *stats) {
	st.mem(-e.sizeof(cg))
	delete(e.ents, cg)
}

func (e *entries) sizeof(cg group) int {
	var s int
	for n := range e.ents[cg] {
		s += len(e.ents[cg][n].data)
	}
	return s
}

func (e *entries) oldestDeadline(cg group) time.Time {
	var t time.Time
	for off := range e.ents[cg] {
		entry := e.ents[cg][off]
		if t.IsZero() || t.After(entry.deadline) {
			t = entry.deadline
		}
	}
	return t
}

func (e *entries) gc(t time.Time, st *stats) {
	for cg, ents := range e.ents {
		for n := range ents {
			if ents[n].invalid(t) {
				st.mem(-len(ents[n].data))
				e.remove(cg, n)
			}
		}
	}
}

type cacheFunc func() error

type cache struct {
	entries *entries
	waits   *waiters
	fetcher *fetcher
	config  *config
	stat    *stats
	events  chan cacheFunc
	debug   func(string, ...interface{})
}

func newCache(f *fetcher, logs *logbuf, cf *config) *cache {
	c := &cache{
		fetcher: f,
		config:  cf,
		events:  make(chan cacheFunc),
		entries: newEntries(),
		waits:   newWaiters(),
		stat:    newStats(),
		debug:   logs.debug,
	}
	go c.gc(cf.gcpause)
	go c.run()
	return c
}

func (c *cache) run() {
	for f := range c.events {
		if err := f(); err != nil {
			log.Print("cache: ", err)
		}
	}
}

func (c *cache) gc(d time.Duration) {
	done := make(chan struct{})
	for {
		time.Sleep(d)
		c.events <- func() error {
			if c.stat.Mem > 0 {
				c.debug("running garbage collector cycle, memory is %d", c.stat.Mem)
				c.entries.gc(time.Now(), c.stat)
				c.debug("garbage collection done, memory is %d", c.stat.Mem)
			}
			done <- struct{}{}
			return nil
		}
		<-done
	}
}

type timeGroup struct {
	t  time.Time
	cg group
}

type timeGroups struct {
	entries []timeGroup
}

type byTime []timeGroup

func (a byTime) Len() int           { return len(a) }
func (a byTime) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byTime) Less(i, j int) bool { return a[i].t.After(a[j].t) }

func makeTimeGroups(e *entries) timeGroups {
	var i int
	tg := make([]timeGroup, len(e.ents))
	for cg := range e.ents {
		tg[i] = timeGroup{
			t:  e.oldestDeadline(cg),
			cg: cg,
		}
		i++
	}
	sort.Sort(byTime(tg))
	return timeGroups{tg}
}

func (tg *timeGroups) purgeOldest(c *cache) {
	entry, ts := tg.entries[len(tg.entries)-1], tg.entries[0:len(tg.entries)-1]
	c.entries.purge(entry.cg, c.stat)
	tg.entries = ts
}

func (tg *timeGroups) empty() bool {
	return len(tg.entries) == 0
}

func (c *cache) oom(target int64) {
	c.debug("OOM called: using %d, limit is %d", c.stat.Mem, target)
	tg := makeTimeGroups(c.entries)
	for {
		tg.purgeOldest(c)
		// Repeat unless we have no more entries or we are using less memory than target
		if tg.empty() || !c.stat.above(target) {
			break
		}
	}
	c.debug("OOM: mem now %d", c.stat.Mem)
}

// put inserts a page into the cache (after it was fetched).
func (c *cache) put(cg group, p *page, err error) {
	c.events <- func() error {
		ce := newEntry(p.body, c.config.lifetime)
		// If an entry exists, it will be overwritten
		if ent, ok := c.entries.get(cg, p.n); ok {
			c.stat.mem(-len(ent.data))
		}
		c.entries.put(cg, p.n, ce)
		c.stat.mem(len(p.body))
		c.debug("added page %s/%d", cg, p.n)
		if c.stat.above(c.config.maxMemory) {
			go func() {
				c.events <- func() error {
					c.oom(c.config.maxMemory)
					return nil
				}
			}()
		}
		// If there were waiters, signal that the wait is over
		c.waits.done(cg, p.n)
		return err
	}
}

// prefetch requests pages (n-m, n) and (n, n+m) if not already fetched
func (c *cache) prefetch(cg group, n, m int, t time.Time) {
	i := n - m
	if i < 0 {
		i = 0
	}
	for ; i < n+m; i++ {
		if i == n {
			// just fetched
			continue
		}
		off := offset(i * c.config.incr)
		if c.entries.has(cg, off, t) || c.waits.has(cg, off) {
			// already fetched or requested
			continue
		}
		c.waits.wait(cg, off)
		res := newResource(c.config.tmpl, cg, off)
		c.fetcher.request(newJob(res, c))
	}
}

func (c *cache) request(cg group, n int, t time.Time) chan struct{} {
	off := offset(n * c.config.incr)
	wait := c.waits.wait(cg, off)
	res := newResource(c.config.tmpl, cg, off)
	c.fetcher.request(newJob(res, c))
	c.prefetch(cg, n, c.config.npref, t)
	return wait
}

func (c *cache) stats() *stats {
	var st *stats
	wait := make(chan struct{})
	c.events <- func() error {
		c.stat.Entries = c.entries.count()
		c.stat.Waiters = c.waits.count()
		st = c.stat.clone()
		wait <- struct{}{}
		return nil
	}
	<-wait
	return st
}

func (c *cache) get(cg group, n int) *page {
	var (
		page *page
		wait chan struct{}
	)
	cached := true
	requested := make(chan struct{})
	off := offset(n * c.config.incr)
	c.debug("%s/%d: requesting from cache", cg, off)
	for {
		wait = nil
		c.events <- func() error {
			defer func() { requested <- struct{}{} }()
			if c.waits.has(cg, off) {
				c.waits.wait(cg, off)
			}
			var now time.Time
			ce, ok := c.entries.get(cg, off)
			if ok {
				now = time.Now()
			}
			if !ok || ce.invalid(now) {
				c.debug("%s/%d: not cached, requested", cg, off)
				wait = c.request(cg, n, now)
				return nil
			}
			c.debug("%s/%d: found", cg, off)
			c.prefetch(cg, n, c.config.npref, now)
			c.stat.hit(cached)
			page = ce.asPage(off)
			return nil
		}
		<-requested
		// content was already in cache, return it
		if wait == nil {
			page.cached = cached
			return page
		}
		// We needed to request the object, it was not cached
		cached = false
		// content is being fetched, wait and try to get again
		<-wait
	}
}
<|endoftext|>


<commit_before>// Package runtime assembles the Elvish runtime.
package runtime

import (
	"errors"
	"fmt"
	"net/rpc"
	"os"
	"path/filepath"
	"time"

	"github.com/boltdb/bolt"
	"github.com/elves/elvish/daemon"
	"github.com/elves/elvish/eval"
	daemonmod "github.com/elves/elvish/eval/daemon"
	"github.com/elves/elvish/eval/re"
	daemonp "github.com/elves/elvish/program/daemon"
	"github.com/elves/elvish/store/storedefs"
	"github.com/elves/elvish/util"
)

var logger = util.GetLogger("[runtime] ")

const (
	daemonWaitLoops   = 100
	daemonWaitPerLoop = 10 * time.Millisecond
)

type daemonStatus int

const (
	daemonOK daemonStatus = iota
	sockfileMissing
	sockfileOtherError
	connectionShutdown
	connectionOtherError
	daemonInvalidDB
	daemonOutdated
)

const (
	daemonWontWorkMsg     = "Daemon-related functions will likely not work."
	connectionShutdownFmt = "Socket file %s exists but is not responding to request. This is likely due to abnormal shutdown of the daemon. Going to remove socket file and re-spawn a daemon.\n"
)

var errInvalidDB = errors.New("daemon reported that database is invalid. If you upgraded Elvish from a pre-0.10 version, you need to upgrade your database by following instructions in https://github.com/elves/upgrade-db-for-0.10/")

// InitRuntime initializes the runtime. The caller is responsible for calling
// CleanupRuntime at some point.
func InitRuntime(binpath, sockpath, dbpath string) (*eval.Evaler, string) {
	var dataDir string
	var err error

	// Determine data directory.
	dataDir, err = storedefs.EnsureDataDir()
	if err != nil {
		fmt.Fprintln(os.Stderr, "warning: cannot create data directory ~/.elvish")
	} else {
		if dbpath == "" {
			dbpath = filepath.Join(dataDir, "db")
		}
	}

	// Determine runtime directory.
	runDir, err := getSecureRunDir()
	if err != nil {
		fmt.Fprintln(os.Stderr, "cannot get runtime dir /tmp/elvish-$uid, falling back to data dir ~/.elvish:", err)
		runDir = dataDir
	}
	if sockpath == "" {
		sockpath = filepath.Join(runDir, "sock")
	}

	ev := eval.NewEvaler()
	ev.SetLibDir(filepath.Join(dataDir, "lib"))
	ev.InstallModule("re", re.Ns())
	if sockpath != "" && dbpath != "" {
		spawner := &daemonp.Daemon{
			BinPath:       binpath,
			DbPath:        dbpath,
			SockPath:      sockpath,
			LogPathPrefix: filepath.Join(runDir, "daemon.log-"),
		}
		// TODO(xiaq): Connect to daemon and install daemon module
		// asynchronously.
		client, err := connectToDaemon(sockpath, spawner)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Cannot connect to daemon:", err)
			fmt.Fprintln(os.Stderr, daemonWontWorkMsg)
		}
		// Even if error is not nil, we install daemon-related functionalities
		// anyway. Daemon may eventually come online and become functional.
		ev.InstallDaemonClient(client)
		ev.InstallModule("daemon", daemonmod.Ns(client, spawner))
	}
	return ev, dataDir
}

func connectToDaemon(sockpath string, spawner *daemonp.Daemon) (*daemon.Client, error) {
	cl := daemon.NewClient(sockpath)
	status, err := detectDaemon(sockpath, cl)
	shouldSpawn := false

	switch status {
	case daemonOK:
	case sockfileMissing:
		shouldSpawn = true
	case sockfileOtherError:
		return cl, fmt.Errorf("socket file %s inaccessible: %v", sockpath, err)
	case connectionShutdown:
		fmt.Fprintf(os.Stderr, connectionShutdownFmt, sockpath)
		err := os.Remove(sockpath)
		if err != nil {
			return cl, fmt.Errorf("failed to remove socket file: %v", err)
		}
		shouldSpawn = true
	case connectionOtherError:
		return cl, fmt.Errorf("unexpected RPC error on socket %s: %v", sockpath, err)
	case daemonInvalidDB:
		return cl, errInvalidDB
	case daemonOutdated:
		fmt.Fprintln(os.Stderr, "Daemon is outdated; going to kill old daemon and re-spawn")
		err := killDaemon(cl)
		if err != nil {
			return cl, fmt.Errorf("failed to kill old daemon: %v", err)
		}
		shouldSpawn = true
	default:
		return cl, fmt.Errorf("code bug: unknown daemon status %d", status)
	}

	if !shouldSpawn {
		return cl, nil
	}

	err = spawner.Spawn()
	if err != nil {
		return cl, fmt.Errorf("failed to spawn daemon: %v", err)
	}
	fmt.Fprintln(os.Stderr, "Spawned daemon")

	// Wait for daemon to come online
	for i := 0; i <= daemonWaitLoops; i++ {
		cl.ResetConn()
		status, err := detectDaemon(sockpath, cl)

		switch status {
		case daemonOK:
			return cl, nil
		case sockfileMissing:
			// Continue waiting
		case sockfileOtherError:
			return cl, fmt.Errorf("socket file %s inaccessible: %v", sockpath, err)
		case connectionShutdown:
			// Continue waiting
		case connectionOtherError:
			return cl, fmt.Errorf("unexpected RPC error on socket %s: %v", sockpath, err)
		case daemonInvalidDB:
			return cl, errInvalidDB
		case daemonOutdated:
			return cl, fmt.Errorf("code bug: newly spawned daemon is outdated")
		default:
			return cl, fmt.Errorf("code bug: unknown daemon status %d", status)
		}
		time.Sleep(daemonWaitPerLoop)
	}
	return cl, fmt.Errorf("daemon unreachable after waiting for %s", daemonWaitLoops*daemonWaitPerLoop)
}

func detectDaemon(sockpath string, cl *daemon.Client) (daemonStatus, error) {
	_, err := os.Stat(sockpath)
	if err != nil {
		if os.IsNotExist(err) {
			return sockfileMissing, err
		}
		return sockfileOtherError, err
	}

	version, err := cl.Version()
	if err != nil {
		switch {
		case err == rpc.ErrShutdown:
			return connectionShutdown, err
		case err.Error() == bolt.ErrInvalid.Error():
			return daemonInvalidDB, err
		default:
			return connectionOtherError, err
		}
	}
	if version < daemon.Version {
		return daemonOutdated, nil
	}
	return daemonOK, nil
}

func killDaemon(cl *daemon.Client) error {
	pid, err := cl.Pid()
	if err != nil {
		return fmt.Errorf("cannot get pid of daemon: %v", err)
	}
	process, err := os.FindProcess(pid)
	if err != nil {
		return fmt.Errorf("cannot find daemon process (pid=%d): %v", pid, err)
	}
	return process.Kill()
}

// CleanupRuntime cleans up the runtime.
func CleanupRuntime(ev *eval.Evaler) {
	if ev.DaemonClient != nil {
		err := ev.DaemonClient.Close()
		if err != nil {
			fmt.Fprintln(os.Stderr, "warning: failed to close connection to daemon:", err)
		}
	}
	ev.Close()
}

var (
	ErrBadOwner      = errors.New("bad owner")
	ErrBadPermission = errors.New("bad permission")
)
<commit_msg>Spawn daemon silently.<commit_after>// Package runtime assembles the Elvish runtime.
package runtime

import (
	"errors"
	"fmt"
	"net/rpc"
	"os"
	"path/filepath"
	"time"

	"github.com/boltdb/bolt"
	"github.com/elves/elvish/daemon"
	"github.com/elves/elvish/eval"
	daemonmod "github.com/elves/elvish/eval/daemon"
	"github.com/elves/elvish/eval/re"
	daemonp "github.com/elves/elvish/program/daemon"
	"github.com/elves/elvish/store/storedefs"
	"github.com/elves/elvish/util"
)

var logger = util.GetLogger("[runtime] ")

const (
	daemonWaitLoops   = 100
	daemonWaitPerLoop = 10 * time.Millisecond
)

type daemonStatus int

const (
	daemonOK daemonStatus = iota
	sockfileMissing
	sockfileOtherError
	connectionShutdown
	connectionOtherError
	daemonInvalidDB
	daemonOutdated
)

const (
	daemonWontWorkMsg     = "Daemon-related functions will likely not work."
	connectionShutdownFmt = "Socket file %s exists but is not responding to request. This is likely due to abnormal shutdown of the daemon. Going to remove socket file and re-spawn a daemon.\n"
)

var errInvalidDB = errors.New("daemon reported that database is invalid. If you upgraded Elvish from a pre-0.10 version, you need to upgrade your database by following instructions in https://github.com/elves/upgrade-db-for-0.10/")

// InitRuntime initializes the runtime. The caller is responsible for calling
// CleanupRuntime at some point.
func InitRuntime(binpath, sockpath, dbpath string) (*eval.Evaler, string) {
	var dataDir string
	var err error

	// Determine data directory.
	dataDir, err = storedefs.EnsureDataDir()
	if err != nil {
		fmt.Fprintln(os.Stderr, "warning: cannot create data directory ~/.elvish")
	} else {
		if dbpath == "" {
			dbpath = filepath.Join(dataDir, "db")
		}
	}

	// Determine runtime directory.
	runDir, err := getSecureRunDir()
	if err != nil {
		fmt.Fprintln(os.Stderr, "cannot get runtime dir /tmp/elvish-$uid, falling back to data dir ~/.elvish:", err)
		runDir = dataDir
	}
	if sockpath == "" {
		sockpath = filepath.Join(runDir, "sock")
	}

	ev := eval.NewEvaler()
	ev.SetLibDir(filepath.Join(dataDir, "lib"))
	ev.InstallModule("re", re.Ns())
	if sockpath != "" && dbpath != "" {
		spawner := &daemonp.Daemon{
			BinPath:       binpath,
			DbPath:        dbpath,
			SockPath:      sockpath,
			LogPathPrefix: filepath.Join(runDir, "daemon.log-"),
		}
		// TODO(xiaq): Connect to daemon and install daemon module
		// asynchronously.
		client, err := connectToDaemon(sockpath, spawner)
		if err != nil {
			fmt.Fprintln(os.Stderr, "Cannot connect to daemon:", err)
			fmt.Fprintln(os.Stderr, daemonWontWorkMsg)
		}
		// Even if error is not nil, we install daemon-related functionalities
		// anyway. Daemon may eventually come online and become functional.
		ev.InstallDaemonClient(client)
		ev.InstallModule("daemon", daemonmod.Ns(client, spawner))
	}
	return ev, dataDir
}

func connectToDaemon(sockpath string, spawner *daemonp.Daemon) (*daemon.Client, error) {
	cl := daemon.NewClient(sockpath)
	status, err := detectDaemon(sockpath, cl)
	shouldSpawn := false

	switch status {
	case daemonOK:
	case sockfileMissing:
		shouldSpawn = true
	case sockfileOtherError:
		return cl, fmt.Errorf("socket file %s inaccessible: %v", sockpath, err)
	case connectionShutdown:
		fmt.Fprintf(os.Stderr, connectionShutdownFmt, sockpath)
		err := os.Remove(sockpath)
		if err != nil {
			return cl, fmt.Errorf("failed to remove socket file: %v", err)
		}
		shouldSpawn = true
	case connectionOtherError:
		return cl, fmt.Errorf("unexpected RPC error on socket %s: %v", sockpath, err)
	case daemonInvalidDB:
		return cl, errInvalidDB
	case daemonOutdated:
		fmt.Fprintln(os.Stderr, "Daemon is outdated; going to kill old daemon and re-spawn")
		err := killDaemon(cl)
		if err != nil {
			return cl, fmt.Errorf("failed to kill old daemon: %v", err)
		}
		shouldSpawn = true
	default:
		return cl, fmt.Errorf("code bug: unknown daemon status %d", status)
	}

	if !shouldSpawn {
		return cl, nil
	}

	err = spawner.Spawn()
	if err != nil {
		return cl, fmt.Errorf("failed to spawn daemon: %v", err)
	}
	logger.Println("Spawned daemon")

	// Wait for daemon to come online
	for i := 0; i <= daemonWaitLoops; i++ {
		cl.ResetConn()
		status, err := detectDaemon(sockpath, cl)

		switch status {
		case daemonOK:
			return cl, nil
		case sockfileMissing:
			// Continue waiting
		case sockfileOtherError:
			return cl, fmt.Errorf("socket file %s inaccessible: %v", sockpath, err)
		case connectionShutdown:
			// Continue waiting
		case connectionOtherError:
			return cl, fmt.Errorf("unexpected RPC error on socket %s: %v", sockpath, err)
		case daemonInvalidDB:
			return cl, errInvalidDB
		case daemonOutdated:
			return cl, fmt.Errorf("code bug: newly spawned daemon is outdated")
		default:
			return cl, fmt.Errorf("code bug: unknown daemon status %d", status)
		}
		time.Sleep(daemonWaitPerLoop)
	}
	return cl, fmt.Errorf("daemon unreachable after waiting for %s", daemonWaitLoops*daemonWaitPerLoop)
}

func detectDaemon(sockpath string, cl *daemon.Client) (daemonStatus, error) {
	_, err := os.Stat(sockpath)
	if err != nil {
		if os.IsNotExist(err) {
			return sockfileMissing, err
		}
		return sockfileOtherError, err
	}

	version, err := cl.Version()
	if err != nil {
		switch {
		case err == rpc.ErrShutdown:
			return connectionShutdown, err
		case err.Error() == bolt.ErrInvalid.Error():
			return daemonInvalidDB, err
		default:
			return connectionOtherError, err
		}
	}
	if version < daemon.Version {
		return daemonOutdated, nil
	}
	return daemonOK, nil
}

func killDaemon(cl *daemon.Client) error {
	pid, err := cl.Pid()
	if err != nil {
		return fmt.Errorf("cannot get pid of daemon: %v", err)
	}
	process, err := os.FindProcess(pid)
	if err != nil {
		return fmt.Errorf("cannot find daemon process (pid=%d): %v", pid, err)
	}
	return process.Kill()
}

// CleanupRuntime cleans up the runtime.
func CleanupRuntime(ev *eval.Evaler) {
	if ev.DaemonClient != nil {
		err := ev.DaemonClient.Close()
		if err != nil {
			fmt.Fprintln(os.Stderr, "warning: failed to close connection to daemon:", err)
		}
	}
	ev.Close()
}

var (
	ErrBadOwner      = errors.New("bad owner")
	ErrBadPermission = errors.New("bad permission")
)
<|endoftext|>


<commit_before>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

// To enable debug output when testing, use "go test -debug"

package parse

import (
	"encoding/json"
	"github.com/demizer/go-elog"
	"os"
	"testing"
	// "fmt"
)

var lexParseTests LexTests

func TestSectionLevelsAdd(t *testing.T) {
	var p sectionLevels
	lvl := p.Add('=', true, 5)
	if lvl != 1 {
		t.Errorf("Improper level on first add, Got level: %d, expected: %d", lvl, 1)
	}
}

func TestSectionLevelsString(t *testing.T) {
	var p sectionLevels
	p.Add('=', true, 5)
	p.Add('-', true, 8)
	p.Add('~', false, 6)
	out := p.String()
	expect := "level: 1, rune: '=', overline: true, length: 5\nlevel: 2, rune: '-', " +
		"overline: true, length: 8\nlevel: 3, rune: '~', overline: false, length: 6\n"
	if out != expect {
		t.Errorf("String output mismatch!\nExpect:\n\n\t%q,\nGot:\n\n\t%q\n", expect, out)
	}
}

func TestSectionLevelsFind(t *testing.T) {
	var p sectionLevels
	p.Add('=', true, 5)
	p.Add('-', true, 8)
	p.Add('~', false, 6)
	lvl := p.Find('-')
	if lvl == -1 {
		t.Errorf("Level not found!\nExpect:\n\n\t%d\nGot:\n\n\t%t\n", 2, lvl)
	}
	if lvl != 2 {
		t.Errorf("Level not correct!\nExpect:\n\n\t%d\nGot:\n\n\t%d\n", 2, lvl)
	}
}

func TestSectionLevelsFindNoResult(t *testing.T) {
	var p sectionLevels
	lvl := p.Find('-')
	if lvl > 0 {
		t.Errorf("Should not find any levels!\nExpect:\n\n\t%d\nGot:\n\n\t%d\n", -1, lvl)
	}
}

func TestSectionLevelsLevelEmpty(t *testing.T) {
	var p sectionLevels
	lvl := p.Level()
	expect := 0
	if lvl != expect {
		t.Errorf("Empty sectionLevels should return \"%d\"!\nExpect:\t%d\nGot:\t%d\n",
			expect, expect, lvl)
	}

}

func TestSectionLevelsLevel(t *testing.T) {
	var p sectionLevels
	p.Add('=', true, 5)
	p.Add('-', true, 8)
	p.Add('~', false, 6)
	lvl := p.Level()
	if lvl != 3 {
		t.Errorf("Level() returned incorrect level!\nExpect:\n\n\t%d\nGot:\n\n\t%d\n", 3, lvl)
	}
}

func parseTest(t *testing.T, testName string) (tree *Tree, err error) {
	if lexParseTests == nil {
		lexParseTests, err = ParseTestData("../testdata/test_lex_sections.dat")
		if err != nil {
			t.Fatal(err)
		}
	}
	test := lexParseTests.SearchByName(testName)
	if test != nil {
		log.Debugf("Test Name: \t%s\n", test.name)
		log.Debugf("Description: \t%s\n", test.description)
		log.Debugf("Test Input:\n-----------\n%s\n----------\n", test.data)
		tree, err = Parse(test.name, test.data)
		if err != nil {
			t.Fatal(err)
		}
	}
	return
}
<commit_msg>parse_test.go: Start implementing compareNodes()<commit_after>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

// To enable debug output when testing, use "go test -debug"

package parse

import (
	"encoding/json"
	"fmt"
	"github.com/demizer/go-elog"
	"reflect"
	"strings"
	"testing"
)

var lexParseTests LexTests

func TestSectionLevelsAdd(t *testing.T) {
	var p sectionLevels
	lvl := p.Add('=', true, 5)
	if lvl != 1 {
		t.Errorf("Improper level on first add, Got level: %d, expected: %d", lvl, 1)
	}
}

func TestSectionLevelsString(t *testing.T) {
	var p sectionLevels
	p.Add('=', true, 5)
	p.Add('-', true, 8)
	p.Add('~', false, 6)
	out := p.String()
	expect := "level: 1, rune: '=', overline: true, length: 5\nlevel: 2, rune: '-', " +
		"overline: true, length: 8\nlevel: 3, rune: '~', overline: false, length: 6\n"
	if out != expect {
		t.Errorf("String output mismatch!\nExpect:\n\n\t%q,\nGot:\n\n\t%q\n", expect, out)
	}
}

func TestSectionLevelsFind(t *testing.T) {
	var p sectionLevels
	p.Add('=', true, 5)
	p.Add('-', true, 8)
	p.Add('~', false, 6)
	lvl := p.Find('-')
	if lvl == -1 {
		t.Errorf("Level not found!\nExpect:\n\n\t%d\nGot:\n\n\t%t\n", 2, lvl)
	}
	if lvl != 2 {
		t.Errorf("Level not correct!\nExpect:\n\n\t%d\nGot:\n\n\t%d\n", 2, lvl)
	}
}

func TestSectionLevelsFindNoResult(t *testing.T) {
	var p sectionLevels
	lvl := p.Find('-')
	if lvl > 0 {
		t.Errorf("Should not find any levels!\nExpect:\n\n\t%d\nGot:\n\n\t%d\n", -1, lvl)
	}
}

func TestSectionLevelsLevelEmpty(t *testing.T) {
	var p sectionLevels
	lvl := p.Level()
	expect := 0
	if lvl != expect {
		t.Errorf("Empty sectionLevels should return \"%d\"!\nExpect:\t%d\nGot:\t%d\n",
			expect, expect, lvl)
	}

}

func TestSectionLevelsLevel(t *testing.T) {
	var p sectionLevels
	p.Add('=', true, 5)
	p.Add('-', true, 8)
	p.Add('~', false, 6)
	lvl := p.Level()
	if lvl != 3 {
		t.Errorf("Level() returned incorrect level!\nExpect:\n\n\t%d\nGot:\n\n\t%d\n", 3, lvl)
	}
}

func parseTest(t *testing.T, testName string) (tree *Tree, err error) {
	if lexParseTests == nil {
		lexParseTests, err = ParseTestData("../testdata/test_lex_sections.dat")
		if err != nil {
			t.Fatal(err)
		}
	}
	test := lexParseTests.SearchByName(testName)
	if test != nil {
		log.Debugf("Test Name: \t%s\n", test.name)
		log.Debugf("Description: \t%s\n", test.description)
		log.Debugf("Test Input:\n-----------\n%s\n----------\n", test.data)
		tree, err = Parse(test.name, test.data)
		if err != nil {
			t.Fatal(err)
		}
	}
	return
}

func compareNodes(t *testing.T, nodes *NodeList, testName string) (err []error) {
	test := lexParseTests.SearchByName(testName)
	var nodeList []interface{}
	jErr := json.Unmarshal([]byte(test.expectTree), &nodeList)
	if jErr != nil {
		return append(err, jErr)
	}
	for num, node := range *nodes {
		nVal := reflect.ValueOf(node).Elem()
		nType := nVal.Type()
		for i := 0; i < nVal.NumField(); i++ {
			structName := nType.Name()
			fieldName := nType.Field(i).Name
			eName := strings.ToLower(string(fieldName[0])) + fieldName[1:]
			gVal := nVal.Field(i)
			eVal := nodeList[num].(map[string]interface{})[eName]
			eType := reflect.TypeOf(eVal)

			// SectionNode.OverLine and SectionNode.UnderLine can be null
			if eVal == nil && eName != "overLine" && eName != "underLine" {
				err = append(err, fmt.Errorf("\"%s\" property does not exist in "+
					"%s.expectTree!\n", fieldName, testName))
				continue
			}

			var val interface{}
			var cErr error

			match := false

			// log.Println(gVal.Kind())

			switch gVal.Kind() {
			case reflect.String:
				if gVal.String() != eVal || gVal.Kind() != reflect.String ||
					eType.Kind() != reflect.String {
					cErr = fmt.Errorf("Got Type: %s.%s = %#v (%s),\n\t"+
						"Expect Type: %s.%s = %#v (%s)\n", structName,
						fieldName, gVal.String(), gVal.Type(), testName, eName,
						eVal, eType.Kind())
					break
				}
				match = true
			case reflect.Int:
				// Check for the "Type" field name, this requires a conversion from
				// string to int.
				if fieldName == "Type" {
					gNodeType := NodeTypeFromString(eVal.(string))
					if gVal.Int() == int64(gNodeType) {
						match = true
						break
					} else {
						cErr = fmt.Errorf(
							"Got: %s.%s = %#v,\n\tExpect: %s.%s = %#v\n",
							structName, fieldName, val, testName, eName,
							eVal)
						break
					}
				}
				// Check for matching types betwen the parsed value and expected
				// value.
				val = gVal.Int()
				if eType.Kind() != reflect.Float64 || gVal.Kind() != reflect.Int ||
					eType.Kind() != reflect.Float64 {
					cErr = fmt.Errorf("Got Type: %s.%s = %#v (%s),\n\t"+
						"Expect Type: %s.%s = %f (%s)\n", structName,
						fieldName, gVal.Int(), gVal.Type(), testName, eName,
						eVal, eType.Kind())
					break
				}
				// Finally, check the actual values
				match = int(val.(int64)) == int(eVal.(float64))
			case reflect.Ptr:
				// A pointer in the struct is most likely another struct such as
				// an AdornmentNode.
				cErr = fmt.Errorf("reflect.Ptr for %s.%s not implemented yet!",
					structName, fieldName)
			case reflect.Slice:
				// A silce in the struct is probably a NodeList.
				cErr = fmt.Errorf("reflect.Slice for %s.%s not implemented yet!",
					structName, fieldName)
			}

			if match == false {
				err = append(err, cErr)
			}
		}
	}
	return
}

func TestParseSectionTitlePara(t *testing.T) {
	testName := "SectionTitlePara"
	tree, err := parseTest(t, testName)
	if err != nil {
		t.Error(err)
	}
	errors := compareNodes(t, tree.Document, testName)
	if errors != nil {
		for _, err := range errors {
			t.Error(err)
		}
	}
}
<|endoftext|>


<commit_before>package paths

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"testing"

	. "github.com/VonC/godbg"
	"github.com/VonC/senvgo/prgs"
	. "github.com/smartystreets/goconvey/convey"
)

func TestPath(t *testing.T) {

	FocusConvey("Tests for Path", t, func() {
		Convey("An empty path remains empty", func() {
			SetBuffers(nil)
			p := NewPath("")
			So(p.path, ShouldEqual, "")
			So(NoOutput(), ShouldBeTrue)
		})

		Convey("An http path remains unchanged", func() {
			SetBuffers(nil)
			p := NewPath(`http://a\b/../c`)
			So(p.path, ShouldEqual, `http://a\b/../c`)
			So(NoOutput(), ShouldBeTrue)
		})
		Convey("A path without trailing / must have one if it is an existing folder", func() {
			SetBuffers(nil)
			p := NewPath(`../paths`)
			So(p.path, ShouldEqual, `..\paths\`)
			So(NoOutput(), ShouldBeTrue)
		})
		Convey("A path without trailing / must keep it even if it is not an existing folder", func() {
			SetBuffers(nil)
			p := NewPath(`xxx\`)
			p = NewPath(`xxx/`)
			So(p.path, ShouldEqual, `xxx\`)
			So(NoOutput(), ShouldBeTrue)
		})

		Convey("A Path can test if it is a Dir", func() {
			SetBuffers(nil)
			p := NewPath("")
			So(p.IsDir(), ShouldBeFalse)
			So(OutString(), ShouldBeEmpty)
			So(ErrString(), ShouldEqual, `open : The system cannot find the file specified.
`)

			// Existing files
			SetBuffers(nil)
			p = NewPath("paths_test.go")
			So(p.IsDir(), ShouldBeFalse)
			So(NoOutput(), ShouldBeTrue)
			So(p.path, ShouldEqual, `paths_test.go`)

			// Existing folders
			SetBuffers(nil)
			p = NewPath("..")
			So(p.IsDir(), ShouldBeTrue)
			So(NoOutput(), ShouldBeTrue)
			So(p.path, ShouldEqual, `..\`)
			p = NewPath("../paths")
			So(p.IsDir(), ShouldBeTrue)
			So(NoOutput(), ShouldBeTrue)
			So(p.path, ShouldEqual, `..\paths\`)

			// Errors
			SetBuffers(nil)
			fstat = testerrfstat
			p = NewPath("..")
			So(p.IsDir(), ShouldBeFalse)
			So(OutString(), ShouldBeEmpty)
			So(ErrString(), ShouldEqual, `fstat error on '..'
fstat error on '..'
`)
			So(p.path, ShouldEqual, `..`)
			fstat = ifstat
		})

		FocusConvey("A Path can test if it exists", func() {
			// non-Existing paths (files or folders)
			SetBuffers(nil)
			p := NewPath("")
			So(p.Exists(), ShouldBeFalse)
			So(NoOutput(), ShouldBeTrue)
			So(p.path, ShouldEqual, ``)

			SetBuffers(nil)
			p = NewPath("xxx")
			So(p.Exists(), ShouldBeFalse)
			So(NoOutput(), ShouldBeTrue)
			So(p.path, ShouldEqual, `xxx`)

			// Existing paths (files or folders)
			SetBuffers(nil)
			p = NewPath("paths_test.go")
			So(p.Exists(), ShouldBeTrue)
			So(NoOutput(), ShouldBeTrue)
			So(p.path, ShouldEqual, `paths_test.go`)

			SetBuffers(nil)
			p = NewPath("../paths")
			So(p.Exists(), ShouldBeTrue)
			So(NoOutput(), ShouldBeTrue)
			So(p.path, ShouldEqual, `..\paths\`)

			// Stat error on path
			SetBuffers(nil)
			fosstat = testerrosfstat
			p = NewPath("test")
			So(p.Exists(), ShouldBeFalse)
			So(OutString(), ShouldBeEmpty)
			So(ErrString(), ShouldEqual, ``)
			So(p.path, ShouldEqual, `test`)
			fosstat = ifosstat
		})
	})
}

func testerrfstat(f *os.File) (fi os.FileInfo, err error) {
	return nil, fmt.Errorf("fstat error on '%+v'", f.Name())
}
func testerrosfstat(name string) (fi os.FileInfo, err error) {
	return nil, fmt.Errorf("os.Stat() error on '%+v'", name)
}

type testPrg struct{ name string }

func (tp *testPrg) Name() string { return tp.name }

type testPathWriter struct{ b *bytes.Buffer }

type testWriter struct{ w io.Writer }

func (tw *testWriter) Write(p []byte) (n int, err error) {
	s := string(p)
	if s == "prg2" {
		return 0, fmt.Errorf("Error writing '%s'", s)
	}
	return tw.w.Write(p)
}

func (tpw *testPathWriter) WritePath(prgs []prgs.Prg, w io.Writer) error {
	if err := pw.WritePath(prgs, w); err != nil {
		return err
	}
	return nil
}

func TestMain(t *testing.T) {
	tpw := &testPathWriter{b: bytes.NewBuffer(nil)}
	prgs := []prgs.Prg{&testPrg{name: "prg1"}, &testPrg{name: "prg2"}}
	Convey("Tests for Path Writer", t, func() {

		Convey("A Path writer writes any empty path if no prgs", func() {
			SetBuffers(nil)
			err := tpw.WritePath(prgs, tpw.b)
			So(err, ShouldBeNil)
			So(tpw.b.String(), ShouldEqual, "prg1prg2")
			So(NoOutput(), ShouldBeTrue)
		})

		Convey("A Path writer can report error during writing", func() {
			SetBuffers(nil)
			tpw.b = bytes.NewBuffer(nil)
			tw := &testWriter{w: tpw.b}
			err := tpw.WritePath(prgs, tw)
			So(err.Error(), ShouldEqual, "Error writing 'prg2'")
			So(tpw.b.String(), ShouldEqual, "prg1")
			So(NoOutput(), ShouldBeTrue)
		})
	})
}
<commit_msg>Test Path.String()<commit_after>package paths

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"testing"

	. "github.com/VonC/godbg"
	"github.com/VonC/senvgo/prgs"
	. "github.com/smartystreets/goconvey/convey"
)

func TestPath(t *testing.T) {

	FocusConvey("Tests for Path", t, func() {
		Convey("An empty path remains empty", func() {
			SetBuffers(nil)
			p := NewPath("")
			So(p.path, ShouldEqual, "")
			So(NoOutput(), ShouldBeTrue)
		})

		Convey("An http path remains unchanged", func() {
			SetBuffers(nil)
			p := NewPath(`http://a\b/../c`)
			So(p.path, ShouldEqual, `http://a\b/../c`)
			So(NoOutput(), ShouldBeTrue)
		})
		Convey("A path without trailing / must have one if it is an existing folder", func() {
			SetBuffers(nil)
			p := NewPath(`../paths`)
			So(p.path, ShouldEqual, `..\paths\`)
			So(NoOutput(), ShouldBeTrue)
		})
		Convey("A path without trailing / must keep it even if it is not an existing folder", func() {
			SetBuffers(nil)
			p := NewPath(`xxx\`)
			p = NewPath(`xxx/`)
			So(p.path, ShouldEqual, `xxx\`)
			So(NoOutput(), ShouldBeTrue)
		})

		Convey("A Path can test if it is a Dir", func() {
			SetBuffers(nil)
			p := NewPath("")
			So(p.IsDir(), ShouldBeFalse)
			So(OutString(), ShouldBeEmpty)
			So(ErrString(), ShouldEqual, `open : The system cannot find the file specified.
`)

			// Existing files
			SetBuffers(nil)
			p = NewPath("paths_test.go")
			So(p.IsDir(), ShouldBeFalse)
			So(NoOutput(), ShouldBeTrue)
			So(p.path, ShouldEqual, `paths_test.go`)

			// Existing folders
			SetBuffers(nil)
			p = NewPath("..")
			So(p.IsDir(), ShouldBeTrue)
			So(NoOutput(), ShouldBeTrue)
			So(p.path, ShouldEqual, `..\`)
			p = NewPath("../paths")
			So(p.IsDir(), ShouldBeTrue)
			So(NoOutput(), ShouldBeTrue)
			So(p.path, ShouldEqual, `..\paths\`)

			// Errors
			SetBuffers(nil)
			fstat = testerrfstat
			p = NewPath("..")
			So(p.IsDir(), ShouldBeFalse)
			So(OutString(), ShouldBeEmpty)
			So(ErrString(), ShouldEqual, `fstat error on '..'
fstat error on '..'
`)
			So(p.path, ShouldEqual, `..`)
			fstat = ifstat
		})

		Convey("A Path can test if it exists", func() {
			// non-Existing paths (files or folders)
			SetBuffers(nil)
			p := NewPath("")
			So(p.Exists(), ShouldBeFalse)
			So(NoOutput(), ShouldBeTrue)
			So(p.path, ShouldEqual, ``)

			SetBuffers(nil)
			p = NewPath("xxx")
			So(p.Exists(), ShouldBeFalse)
			So(NoOutput(), ShouldBeTrue)
			So(p.path, ShouldEqual, `xxx`)

			// Existing paths (files or folders)
			SetBuffers(nil)
			p = NewPath("paths_test.go")
			So(p.Exists(), ShouldBeTrue)
			So(NoOutput(), ShouldBeTrue)
			So(p.path, ShouldEqual, `paths_test.go`)

			SetBuffers(nil)
			p = NewPath("../paths")
			So(p.Exists(), ShouldBeTrue)
			So(NoOutput(), ShouldBeTrue)
			So(p.path, ShouldEqual, `..\paths\`)

			// Stat error on path
			SetBuffers(nil)
			fosstat = testerrosfstat
			p = NewPath("test")
			So(p.Exists(), ShouldBeFalse)
			So(OutString(), ShouldBeEmpty)
			So(ErrString(), ShouldEqual, ``)
			So(p.path, ShouldEqual, `test`)
			fosstat = ifosstat
		})

		FocusConvey("A Path can print itself", func() {

			// nil path is '<nil>'
			SetBuffers(nil)
			var p *Path
			So(p.String(), ShouldEqual, `<nil>`)
			So(NoOutput(), ShouldBeTrue)

			// file or folder are unchanged
			SetBuffers(nil)
			p = NewPath("test")
			So(p.String(), ShouldEqual, `test`)
			So(NoOutput(), ShouldBeTrue)

			// long file or folder are truncated
			SetBuffers(nil)
			var data []byte
			data = append(data, ([]byte("long string with "))...)
			for i := 0; i < 100; i++ {
				data = append(data, ([]byte("abcd"))...)
			}
			p = NewPath(string(data))
			So(p.String(), ShouldEqual, `long string with abc (417)`)
			So(NoOutput(), ShouldBeTrue)
		})
	})
}

func testerrfstat(f *os.File) (fi os.FileInfo, err error) {
	return nil, fmt.Errorf("fstat error on '%+v'", f.Name())
}
func testerrosfstat(name string) (fi os.FileInfo, err error) {
	return nil, fmt.Errorf("os.Stat() error on '%+v'", name)
}

type testPrg struct{ name string }

func (tp *testPrg) Name() string { return tp.name }

type testPathWriter struct{ b *bytes.Buffer }

type testWriter struct{ w io.Writer }

func (tw *testWriter) Write(p []byte) (n int, err error) {
	s := string(p)
	if s == "prg2" {
		return 0, fmt.Errorf("Error writing '%s'", s)
	}
	return tw.w.Write(p)
}

func (tpw *testPathWriter) WritePath(prgs []prgs.Prg, w io.Writer) error {
	if err := pw.WritePath(prgs, w); err != nil {
		return err
	}
	return nil
}

func TestMain(t *testing.T) {
	tpw := &testPathWriter{b: bytes.NewBuffer(nil)}
	prgs := []prgs.Prg{&testPrg{name: "prg1"}, &testPrg{name: "prg2"}}
	Convey("Tests for Path Writer", t, func() {

		Convey("A Path writer writes any empty path if no prgs", func() {
			SetBuffers(nil)
			err := tpw.WritePath(prgs, tpw.b)
			So(err, ShouldBeNil)
			So(tpw.b.String(), ShouldEqual, "prg1prg2")
			So(NoOutput(), ShouldBeTrue)
		})

		Convey("A Path writer can report error during writing", func() {
			SetBuffers(nil)
			tpw.b = bytes.NewBuffer(nil)
			tw := &testWriter{w: tpw.b}
			err := tpw.WritePath(prgs, tw)
			So(err.Error(), ShouldEqual, "Error writing 'prg2'")
			So(tpw.b.String(), ShouldEqual, "prg1")
			So(NoOutput(), ShouldBeTrue)
		})
	})
}
<|endoftext|>


<commit_before>package isumm

import (
	"fmt"
	"sort"
	"time"
)

// TimeseriesChart represents a chart where the x-axis is a timeseries.
type TimeseriesChart []timeSeriesPoint

// timeSeriesPoint represents a chart point where x-axis is a timeseries.
// All fields are private because interesting result is the String().
type timeSeriesPoint struct {
	date  time.Time
	value float32
}

func (g timeSeriesPoint) String() string {
	return fmt.Sprintf("[%d, %.2f]", g.date.UnixNano()/1000000, g.value)
}

func AmountSummaryChart(invs []*Investment) TimeseriesChart {
	summs := AggregateByDate(invs)
	var chart TimeseriesChart
	for _, s := range summs {
		chart = append(chart, timeSeriesPoint{s.Date, s.Balance})
	}
	return chart
}

func InterestRateChart(invs []*Investment) TimeseriesChart {
	summs := AggregateByDate(invs)
	// Interest rates implies at least two reference points.
	if len(summs) < 2 {
		return TimeseriesChart{}
	}
	var chart TimeseriesChart
	for i, s := range summs {
		t := timeSeriesPoint{date: s.Date}
		switch {
		case i == 0:
			t.value = 0
		default:
			t.value = float32(s.Balance-s.Change-summs[i-1].Balance) / float32(s.Change+summs[i-1].Balance) * 100.0
		}
		chart = append(chart, t)
	}
	return chart
}

// Returns the aggregated summaries sorted by date.
func AggregateByDate(invs []*Investment) Summaries {
	summByDate := make(map[time.Time]Summary)
	for _, i := range invs {
		for _, s := range i.Ops.Summarize() {
			v := summByDate[s.Date]
			v.Date = s.Date
			v.Balance += s.Balance
			v.Change += s.Change
			summByDate[s.Date] = v
		}
	}
	var summs Summaries
	for _, s := range summByDate {
		summs = append(summs, s)
	}
	// More information at: https://github.com/danielfireman/isumm/issues/2
	sort.Sort(summs)
	switch len(summs) {
	case 1:
		if summs[0].Balance == 0 {
			// There only one point and it is zero. This probably means that it's the first month the user
			// and the month hasn't changed.
			return Summaries{}
		}
	default:
		// This prevents the corner case where the last month hasn't ended and there already deposits or
		// widrawals. There is a valid case for that where we have zeroed the balance because of a Widrawal.
		last := summs[len(summs)-1]
		previous := summs[len(summs)-2]
		if last.Balance == 0 && (previous.Balance+last.Change != 0) {
			summs = summs[:len(summs)-2]
		}
	}
	return summs
}
<commit_msg>Small fix<commit_after>package isumm

import (
	"fmt"
	"sort"
	"time"
)

// TimeseriesChart represents a chart where the x-axis is a timeseries.
type TimeseriesChart []timeSeriesPoint

// timeSeriesPoint represents a chart point where x-axis is a timeseries.
// All fields are private because interesting result is the String().
type timeSeriesPoint struct {
	date  time.Time
	value float32
}

func (g timeSeriesPoint) String() string {
	return fmt.Sprintf("[%d, %.2f]", g.date.UnixNano()/1000000, g.value)
}

func AmountSummaryChart(invs []*Investment) TimeseriesChart {
	summs := AggregateByDate(invs)
	var chart TimeseriesChart
	for _, s := range summs {
		chart = append(chart, timeSeriesPoint{s.Date, s.Balance})
	}
	return chart
}

func InterestRateChart(invs []*Investment) TimeseriesChart {
	summs := AggregateByDate(invs)
	// Interest rates implies at least two reference points.
	if len(summs) < 2 {
		return TimeseriesChart{}
	}
	var chart TimeseriesChart
	for i, s := range summs {
		t := timeSeriesPoint{date: s.Date}
		switch {
		case i == 0:
			t.value = 0
		default:
			t.value = float32(s.Balance-s.Change-summs[i-1].Balance) / float32(s.Change+summs[i-1].Balance) * 100.0
		}
		chart = append(chart, t)
	}
	return chart
}

// Returns the aggregated summaries sorted by date.
func AggregateByDate(invs []*Investment) Summaries {
	summByDate := make(map[time.Time]Summary)
	for _, i := range invs {
		for _, s := range i.Ops.Summarize() {
			v := summByDate[s.Date]
			v.Date = s.Date
			v.Balance += s.Balance
			v.Change += s.Change
			summByDate[s.Date] = v
		}
	}
	var summs Summaries
	for _, s := range summByDate {
		summs = append(summs, s)
	}
	// More information at: https://github.com/danielfireman/isumm/issues/2
	sort.Sort(summs)
	switch len(summs) {
	case 1:
		if summs[0].Balance == 0 {
			// There only one point and it is zero. This probably means that it's the first month the user
			// and the month hasn't changed.
			return Summaries{}
		}
	default:
		// This prevents the corner case where the last month hasn't ended and there already deposits or
		// widrawals. There is a valid case for that where we have zeroed the balance because of a Widrawal.
		last := summs[len(summs)-1]
		previous := summs[len(summs)-2]
		if last.Balance == 0 && (previous.Balance+last.Change != 0) {
			summs = summs[:len(summs)-1]
		}
	}
	return summs
}
<|endoftext|>


<commit_before>package ttlcache

import (
	"io"
	"sync"
	"time"
)

const (
	NeverExpires = time.Duration(-1)
)

type ttlCacheEntry struct {
	value  interface{}
	expiry *time.Time
	lock   sync.RWMutex
}

func (e *ttlCacheEntry) Close() {
	c, ok := e.value.(io.Closer)
	if ok {
		c.Close()
	}
}

type TtlCache struct {
	gcInterval time.Duration
	cache      map[string]*ttlCacheEntry
	lock       sync.RWMutex
	exit       chan struct{}
	exited     chan struct{}
	isClosed   bool
}

func NewTtlCache(gcInterval time.Duration) *TtlCache {
	c := &TtlCache{
		gcInterval: gcInterval,
		cache:      make(map[string]*ttlCacheEntry),
		exit:       make(chan struct{}, 1),
		exited:     make(chan struct{}, 1),
		isClosed:   false,
	}
	go c.startCleaner()
	return c
}

func (c *TtlCache) Close() {
	c.lock.RLock()
	isClosed := c.isClosed
	c.lock.RUnlock()

	if isClosed {
		return
	}

	c.lock.Lock()
	c.isClosed = true
	c.lock.Unlock()

	c.exit <- struct{}{}
	<-c.exited
}

func (c *TtlCache) startCleaner() {
	if c == nil {
		return
	}
	if c.gcInterval > 0 {
		ticker := time.NewTicker(c.gcInterval)
	gcLoop:
		for {
			select {
			case <-c.exit:
				ticker.Stop()
				break gcLoop
			case now := <-ticker.C:
				if c == nil {
					return
				}
				c.lock.Lock()
				for id, entry := range c.cache {
					entry.lock.RLock()
					expiry := entry.expiry
					entry.lock.RUnlock()

					if expiry != nil && expiry.Before(now) {
						entry.Close()
						delete(c.cache, id)
					}
				}
				c.lock.Unlock()
			}
		}
	} else {
		<-c.exit
	}
	for id, entry := range c.cache {
		entry.Close()
		delete(c.cache, id)
	}
	c.exited <- struct{}{}
}

func (c *TtlCache) ensureEntry(id string) (entry *ttlCacheEntry) {
	c.lock.RLock()
	entry, ok := c.cache[id]
	c.lock.RUnlock()
	if ok {
		return
	}
	c.lock.Lock()
	defer c.lock.Unlock()
	entry, ok = c.cache[id]
	if ok {
		return
	}
	entry = &ttlCacheEntry{}
	c.cache[id] = entry
	return
}

func (c *TtlCache) Get(id string) interface{} {
	c.lock.RLock()
	entry, ok := c.cache[id]
	c.lock.RUnlock()

	if !ok {
		return nil
	}

	entry.lock.RLock()
	defer entry.lock.RUnlock()

	if ok && (entry.expiry == nil || entry.expiry.After(time.Now())) {
		return entry.value
	} else {
		return nil
	}
}

func (c *TtlCache) Set(id string, value interface{}, ttl time.Duration) {
	var expiry *time.Time
	if ttl != NeverExpires {
		expiryTime := time.Now().Add(ttl)
		expiry = &expiryTime
	}
	entry := c.ensureEntry(id)

	entry.lock.Lock()
	defer entry.lock.Unlock()

	entry.Close() // close potential existing
	entry.value = value
	entry.expiry = expiry
}

func (c *TtlCache) Delete(id string) {
	c.lock.Lock()

	elem, ok := c.cache[id]
	if !ok {
		c.lock.Unlock()

		return
	}

	delete(c.cache, id)

	c.lock.Unlock()

	elem.Close()
}

func (c *TtlCache) GetOrElseUpdate(id string, ttl time.Duration,
	create func() (interface{}, error)) (value interface{}, err error) {

	value = c.Get(id)
	if value != nil {
		return
	}

	entry := c.ensureEntry(id)
	entry.lock.Lock()
	defer entry.lock.Unlock()

	if entry.value != nil && (entry.expiry == nil || entry.expiry.After(time.Now())) {
		return entry.value, nil
	} else {
		value, err = create()
		if err != nil {
			nonCached, ok := IsDoNotCache(err)
			if ok {
				expiry := time.Unix(0, 0)
				entry.expiry = &expiry //will be GCed if nobody else is using it
				value = nonCached
				err = nil
			}
			return
		}
		entry.Close()
		entry.value = value

		var expiry *time.Time
		if ttl != NeverExpires {
			expiryTime := time.Now().Add(ttl)
			expiry = &expiryTime
		}
		entry.expiry = expiry
	}
	return
}

func (c *TtlCache) Foreach(f func(string, interface{})) {
	c.lock.RLock()
	i := 0
	keys := make([]string, len(c.cache))
	for key := range c.cache {
		keys[i] = key
		i++
	}
	c.lock.RUnlock()

	for _, key := range keys {
		c.lock.RLock()
		entry, ok := c.cache[key]
		c.lock.RUnlock()
		if ok {
			f(key, entry.value)
		}
	}
}

type DoNotCache struct {
	Value interface{}
}

func (d DoNotCache) Error() string {
	return "This contains an uncachable value"
}

func IsDoNotCache(err error) (value interface{}, ok bool) {
	dnc, ok := err.(DoNotCache)
	if !ok {
		return
	}
	value = dnc.Value
	return
}
<commit_msg>Cleanup<commit_after>package ttlcache

import (
	"io"
	"sync"
	"time"
)

const (
	NeverExpires = time.Duration(-1)
)

type ttlCacheEntry struct {
	value  interface{}
	expiry *time.Time
	lock   sync.RWMutex
}

func (e *ttlCacheEntry) Close() {
	c, ok := e.value.(io.Closer)
	if ok {
		c.Close()
	}
}

type TtlCache struct {
	gcInterval time.Duration
	cache      map[string]*ttlCacheEntry
	lock       sync.RWMutex
	exit       chan struct{}
	exited     chan struct{}
	isClosed   bool
}

func NewTtlCache(gcInterval time.Duration) *TtlCache {
	c := &TtlCache{
		gcInterval: gcInterval,
		cache:      make(map[string]*ttlCacheEntry),
		exit:       make(chan struct{}, 1),
		exited:     make(chan struct{}, 1),
		isClosed:   false,
	}
	go c.startCleaner()
	return c
}

func (c *TtlCache) Close() {
	c.lock.RLock()
	isClosed := c.isClosed
	c.lock.RUnlock()

	if isClosed {
		return
	}

	c.lock.Lock()
	c.isClosed = true
	c.lock.Unlock()

	c.exit <- struct{}{}
	<-c.exited
}

func (c *TtlCache) startCleaner() {
	if c == nil {
		return
	}
	if c.gcInterval > 0 {
		ticker := time.NewTicker(c.gcInterval)
	gcLoop:
		for {
			select {
			case <-c.exit:
				ticker.Stop()
				break gcLoop
			case now := <-ticker.C:
				if c == nil {
					return
				}
				c.lock.Lock()
				for id, entry := range c.cache {
					entry.lock.RLock()
					expiry := entry.expiry
					entry.lock.RUnlock()

					if expiry != nil && expiry.Before(now) {
						entry.Close()
						delete(c.cache, id)
					}
				}
				c.lock.Unlock()
			}
		}
	} else {
		<-c.exit
	}
	for id, entry := range c.cache {
		entry.Close()
		delete(c.cache, id)
	}
	c.exited <- struct{}{}
}

func (c *TtlCache) ensureEntry(id string) (entry *ttlCacheEntry) {
	c.lock.RLock()
	entry, ok := c.cache[id]
	c.lock.RUnlock()
	if ok {
		return
	}
	c.lock.Lock()
	defer c.lock.Unlock()
	entry, ok = c.cache[id]
	if ok {
		return
	}
	entry = &ttlCacheEntry{}
	c.cache[id] = entry
	return
}

func (c *TtlCache) Get(id string) interface{} {
	c.lock.RLock()
	entry, ok := c.cache[id]
	c.lock.RUnlock()

	if !ok {
		return nil
	}

	entry.lock.RLock()
	defer entry.lock.RUnlock()

	if entry.expiry != nil && !entry.expiry.After(time.Now()) {
		return nil
	}

	return entry.value
}

func (c *TtlCache) Set(id string, value interface{}, ttl time.Duration) {
	var expiry *time.Time
	if ttl != NeverExpires {
		expiryTime := time.Now().Add(ttl)
		expiry = &expiryTime
	}
	entry := c.ensureEntry(id)

	entry.lock.Lock()
	defer entry.lock.Unlock()

	entry.Close() // close potential existing
	entry.value = value
	entry.expiry = expiry
}

func (c *TtlCache) Delete(id string) {
	c.lock.Lock()

	elem, ok := c.cache[id]
	if !ok {
		c.lock.Unlock()

		return
	}

	delete(c.cache, id)

	c.lock.Unlock()

	elem.Close()
}

func (c *TtlCache) GetOrElseUpdate(id string, ttl time.Duration,
	create func() (interface{}, error)) (value interface{}, err error) {

	value = c.Get(id)
	if value != nil {
		return
	}

	entry := c.ensureEntry(id)
	entry.lock.Lock()
	defer entry.lock.Unlock()

	if entry.value != nil && (entry.expiry == nil || entry.expiry.After(time.Now())) {
		return entry.value, nil
	} else {
		value, err = create()
		if err != nil {
			nonCached, ok := IsDoNotCache(err)
			if ok {
				expiry := time.Unix(0, 0)
				entry.expiry = &expiry //will be GCed if nobody else is using it
				value = nonCached
				err = nil
			}
			return
		}
		entry.Close()
		entry.value = value

		var expiry *time.Time
		if ttl != NeverExpires {
			expiryTime := time.Now().Add(ttl)
			expiry = &expiryTime
		}
		entry.expiry = expiry
	}
	return
}

func (c *TtlCache) Foreach(f func(string, interface{})) {
	c.lock.RLock()
	i := 0
	keys := make([]string, len(c.cache))
	for key := range c.cache {
		keys[i] = key
		i++
	}
	c.lock.RUnlock()

	for _, key := range keys {
		c.lock.RLock()
		entry, ok := c.cache[key]
		c.lock.RUnlock()
		if ok {
			f(key, entry.value)
		}
	}
}

type DoNotCache struct {
	Value interface{}
}

func (d DoNotCache) Error() string {
	return "This contains an uncachable value"
}

func IsDoNotCache(err error) (value interface{}, ok bool) {
	dnc, ok := err.(DoNotCache)
	if !ok {
		return
	}
	value = dnc.Value
	return
}
<|endoftext|>


<commit_before>// Copyright 2013 Beego Authors
// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package cache a middleware that provides the cache manager of Macaron.
package cache

// NOTE: last sync b1b4dbb on Aug 08, 2014.

import (
	"fmt"

	"github.com/Unknwon/macaron"
)

// Cache interface contains all behaviors for cache adapter.
// usage:
//	cache.Register("file",cache.NewFileCache()) // this operation is run in init method of file.go.
//	c, err := cache.NewCache("file","{....}")
//	c.Put("key",value,3600)
//	v := c.Get("key")
//
//	c.Incr("counter")  // now is 1
//	c.Incr("counter")  // now is 2
//	count := c.Get("counter").(int)
type Cache interface {
	// get cached value by key.
	Get(key string) interface{}
	// set cached value with key and expire time.
	Put(key string, val interface{}, timeout int64) error
	// delete cached value by key.
	Delete(key string) error
	// increase cached int value by key, as a counter.
	Incr(key string) error
	// decrease cached int value by key, as a counter.
	Decr(key string) error
	// check if cached value exists or not.
	IsExist(key string) bool
	// clear all cache.
	ClearAll() error
	// start gc routine based on config string settings.
	StartAndGC(config string) error
}

type Options struct {
	// Name of adapter. Default is "memory".
	Adapter string
	// GC interval for memory adapter. Default is 60.
	Interval int
	// Connection string for non-memory adapter.
	Conn string
}

func prepareOptions(options []Options) Options {
	var opt Options
	if len(options) > 0 {
		opt = options[0]
	}

	// Defaults.
	if len(opt.Adapter) == 0 {
		opt.Adapter = "memory"
	}

	if opt.Adapter == "memory" {
		if opt.Interval == 0 {
			opt.Interval = 60
		}
	} else {
		if len(opt.Conn) == 0 {
			panic("no connection string is given for non-memory cache adapter")
		}
	}

	return opt
}

// Create a new cache driver by adapter name and config string.
// config need to be correct JSON as string: {"interval":360}.
// it will start gc automatically.
func NewCache(adapterName, config string) (Cache, error) {
	adapter, ok := adapters[adapterName]
	if !ok {
		return nil, fmt.Errorf("cache: unknown adapter name %q (forgot to import?)", adapterName)
	}
	if err := adapter.StartAndGC(config); err != nil {
		return nil, err
	}
	return adapter, nil
}

// Cacher is a middleware that maps a cache.Cache service into the Macaron handler chain.
// An single variadic cache.Options struct can be optionally provided to configure.
func Cacher(options ...Options) macaron.Handler {
	opt := prepareOptions(options)
	cache, err := NewCache(opt.Adapter,
		fmt.Sprintf(`{"interval":%d,"conn":"%s"}`, opt.Interval, opt.Conn))
	if err != nil {
		panic(err)
	}
	return func(ctx *macaron.Context) {
		ctx.Map(cache)
	}
}

var adapters = make(map[string]Cache)

// Register makes a cache adapter available by the adapter name.
// If Register is called twice with the same name or if driver is nil,
// it panics.
func Register(name string, adapter Cache) {
	if adapter == nil {
		panic("cache: Register adapter is nil")
	}
	if _, dup := adapters[name]; dup {
		panic("cache: Register called twice for adapter " + name)
	}
	adapters[name] = adapter
}
<commit_msg>Sync with upstream<commit_after>// Copyright 2013 Beego Authors
// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package cache a middleware that provides the cache manager of Macaron.
package cache

// NOTE: last sync a144769 on Aug 18, 2014.

import (
	"fmt"

	"github.com/Unknwon/macaron"
)

// Cache interface contains all behaviors for cache adapter.
// usage:
//	cache.Register("file",cache.NewFileCache()) // this operation is run in init method of file.go.
//	c, err := cache.NewCache("file","{....}")
//	c.Put("key",value,3600)
//	v := c.Get("key")
//
//	c.Incr("counter")  // now is 1
//	c.Incr("counter")  // now is 2
//	count := c.Get("counter").(int)
type Cache interface {
	// get cached value by key.
	Get(key string) interface{}
	// set cached value with key and expire time.
	Put(key string, val interface{}, timeout int64) error
	// delete cached value by key.
	Delete(key string) error
	// increase cached int value by key, as a counter.
	Incr(key string) error
	// decrease cached int value by key, as a counter.
	Decr(key string) error
	// check if cached value exists or not.
	IsExist(key string) bool
	// clear all cache.
	ClearAll() error
	// start gc routine based on config string settings.
	StartAndGC(config string) error
}

type Options struct {
	// Name of adapter. Default is "memory".
	Adapter string
	// GC interval for memory adapter. Default is 60.
	Interval int
	// Connection string for non-memory adapter.
	Conn string
}

func prepareOptions(options []Options) Options {
	var opt Options
	if len(options) > 0 {
		opt = options[0]
	}

	// Defaults.
	if len(opt.Adapter) == 0 {
		opt.Adapter = "memory"
	}

	if opt.Adapter == "memory" {
		if opt.Interval == 0 {
			opt.Interval = 60
		}
	} else {
		if len(opt.Conn) == 0 {
			panic("no connection string is given for non-memory cache adapter")
		}
	}

	return opt
}

// Create a new cache driver by adapter name and config string.
// config need to be correct JSON as string: {"interval":360}.
// it will start gc automatically.
func NewCache(adapterName, config string) (Cache, error) {
	adapter, ok := adapters[adapterName]
	if !ok {
		return nil, fmt.Errorf("cache: unknown adapter name %q (forgot to import?)", adapterName)
	}
	if err := adapter.StartAndGC(config); err != nil {
		return nil, err
	}
	return adapter, nil
}

// Cacher is a middleware that maps a cache.Cache service into the Macaron handler chain.
// An single variadic cache.Options struct can be optionally provided to configure.
func Cacher(options ...Options) macaron.Handler {
	opt := prepareOptions(options)
	cache, err := NewCache(opt.Adapter,
		fmt.Sprintf(`{"interval":%d,"conn":"%s"}`, opt.Interval, opt.Conn))
	if err != nil {
		panic(err)
	}
	return func(ctx *macaron.Context) {
		ctx.Map(cache)
	}
}

var adapters = make(map[string]Cache)

// Register makes a cache adapter available by the adapter name.
// If Register is called twice with the same name or if driver is nil,
// it panics.
func Register(name string, adapter Cache) {
	if adapter == nil {
		panic("cache: Register adapter is nil")
	}
	if _, dup := adapters[name]; dup {
		panic("cache: Register called twice for adapter " + name)
	}
	adapters[name] = adapter
}
<|endoftext|>


<commit_before>package gcache

import (
	"errors"
	"github.com/bluele/gcache/singleflight"
	"sync"
	"time"
)

var (
	TYPE_SIMPLE = "simple"
	TYPE_LRU    = "lru"
	TYPE_LFU    = "lfu"
	TYPE_ARC    = "arc"
)

var NotFoundKeyError = errors.New("Not found error.")

type Cache interface {
	Set(interface{}, interface{})
	Get(interface{}) (interface{}, error)
	Remove(interface{}) bool
	Purge()
	Keys() []interface{}
	Len() int
	gc()
}

type baseCache struct {
	size        int
	loaderFunc  *LoaderFunc
	evictedFunc *EvictedFunc
	addedFunc   *AddedFunc
	expiration  *time.Duration
	mu          sync.RWMutex
	loadGroup   singleflight.Group
}

type LoaderFunc func(interface{}) (interface{}, error)

type EvictedFunc func(interface{}, interface{})

type AddedFunc func(interface{}, interface{})

type CacheBuilder struct {
	tp          string
	size        int
	loaderFunc  *LoaderFunc
	evictedFunc *EvictedFunc
	addedFunc   *AddedFunc
	expiration  *time.Duration
	gcInterval  *time.Duration
}

func New(size int) *CacheBuilder {
	if size <= 0 {
		panic("gcache: size <= 0")
	}
	return &CacheBuilder{
		tp:   TYPE_SIMPLE,
		size: size,
	}
}

func (cb *CacheBuilder) LoaderFunc(loaderFunc LoaderFunc) *CacheBuilder {
	cb.loaderFunc = &loaderFunc
	return cb
}

func (cb *CacheBuilder) EnableGC(interval time.Duration) *CacheBuilder {
	cb.gcInterval = &interval
	return cb
}

func (cb *CacheBuilder) Simple() *CacheBuilder {
	cb.tp = TYPE_SIMPLE
	return cb
}

func (cb *CacheBuilder) LRU() *CacheBuilder {
	cb.tp = TYPE_LRU
	return cb
}

func (cb *CacheBuilder) LFU() *CacheBuilder {
	cb.tp = TYPE_LFU
	return cb
}

func (cb *CacheBuilder) ARC() *CacheBuilder {
	cb.tp = TYPE_ARC
	return cb
}

func (cb *CacheBuilder) EvictedFunc(evictedFunc EvictedFunc) *CacheBuilder {
	cb.evictedFunc = &evictedFunc
	return cb
}

func (cb *CacheBuilder) AddedFunc(addedFunc AddedFunc) *CacheBuilder {
	cb.addedFunc = &addedFunc
	return cb
}

func (cb *CacheBuilder) Expiration(expiration time.Duration) *CacheBuilder {
	cb.expiration = &expiration
	return cb
}

func (cb *CacheBuilder) Build() Cache {
	cache := cb.build()
	if cb.gcInterval != nil {
		go func() {
			t := time.NewTicker(*cb.gcInterval)
			for {
				select {
				case <-t.C:
					go cache.gc()
				}
			}
			t.Stop()
		}()
	}
	return cache
}

func (cb *CacheBuilder) build() Cache {
	switch cb.tp {
	case TYPE_SIMPLE:
		return newSimpleCache(cb)
	case TYPE_LRU:
		return newLRUCache(cb)
	case TYPE_LFU:
		return newLFUCache(cb)
	case TYPE_ARC:
		return newARC(cb)
	default:
		panic("gcache: Unknown type " + cb.tp)
	}
}

func buildCache(c *baseCache, cb *CacheBuilder) {
	c.size = cb.size
	c.loaderFunc = cb.loaderFunc
	c.expiration = cb.expiration
	c.addedFunc = cb.addedFunc
	c.evictedFunc = cb.evictedFunc
}

// load a new value using by specified key.
func (c *baseCache) load(key interface{}, cb func(interface{}, error) (interface{}, error)) (interface{}, error) {
	v, err := c.loadGroup.Do(key, func() (interface{}, error) {
		return cb((*c.loaderFunc)(key))
	})
	if err != nil {
		return nil, err
	}
	return v, nil
}
<commit_msg>add func `EvictType` for setting eviction type.<commit_after>package gcache

import (
	"errors"
	"github.com/bluele/gcache/singleflight"
	"sync"
	"time"
)

const (
	TYPE_SIMPLE = "simple"
	TYPE_LRU    = "lru"
	TYPE_LFU    = "lfu"
	TYPE_ARC    = "arc"
)

var NotFoundKeyError = errors.New("Not found error.")

type Cache interface {
	Set(interface{}, interface{})
	Get(interface{}) (interface{}, error)
	Remove(interface{}) bool
	Purge()
	Keys() []interface{}
	Len() int
	gc()
}

type baseCache struct {
	size        int
	loaderFunc  *LoaderFunc
	evictedFunc *EvictedFunc
	addedFunc   *AddedFunc
	expiration  *time.Duration
	mu          sync.RWMutex
	loadGroup   singleflight.Group
}

type LoaderFunc func(interface{}) (interface{}, error)

type EvictedFunc func(interface{}, interface{})

type AddedFunc func(interface{}, interface{})

type CacheBuilder struct {
	tp          string
	size        int
	loaderFunc  *LoaderFunc
	evictedFunc *EvictedFunc
	addedFunc   *AddedFunc
	expiration  *time.Duration
	gcInterval  *time.Duration
}

func New(size int) *CacheBuilder {
	if size <= 0 {
		panic("gcache: size <= 0")
	}
	return &CacheBuilder{
		tp:   TYPE_SIMPLE,
		size: size,
	}
}

func (cb *CacheBuilder) LoaderFunc(loaderFunc LoaderFunc) *CacheBuilder {
	cb.loaderFunc = &loaderFunc
	return cb
}

func (cb *CacheBuilder) EnableGC(interval time.Duration) *CacheBuilder {
	cb.gcInterval = &interval
	return cb
}

func (cb *CacheBuilder) EvictType(tp string) *CacheBuilder {
	cb.tp = tp
	return cb
}

func (cb *CacheBuilder) Simple() *CacheBuilder {
	return cb.EvictType(TYPE_SIMPLE)
}

func (cb *CacheBuilder) LRU() *CacheBuilder {
	return cb.EvictType(TYPE_LRU)
}

func (cb *CacheBuilder) LFU() *CacheBuilder {
	return cb.EvictType(TYPE_LFU)
}

func (cb *CacheBuilder) ARC() *CacheBuilder {
	return cb.EvictType(TYPE_ARC)
}

func (cb *CacheBuilder) EvictedFunc(evictedFunc EvictedFunc) *CacheBuilder {
	cb.evictedFunc = &evictedFunc
	return cb
}

func (cb *CacheBuilder) AddedFunc(addedFunc AddedFunc) *CacheBuilder {
	cb.addedFunc = &addedFunc
	return cb
}

func (cb *CacheBuilder) Expiration(expiration time.Duration) *CacheBuilder {
	cb.expiration = &expiration
	return cb
}

func (cb *CacheBuilder) Build() Cache {
	cache := cb.build()
	if cb.gcInterval != nil {
		go func() {
			t := time.NewTicker(*cb.gcInterval)
			for {
				select {
				case <-t.C:
					go cache.gc()
				}
			}
			t.Stop()
		}()
	}
	return cache
}

func (cb *CacheBuilder) build() Cache {
	switch cb.tp {
	case TYPE_SIMPLE:
		return newSimpleCache(cb)
	case TYPE_LRU:
		return newLRUCache(cb)
	case TYPE_LFU:
		return newLFUCache(cb)
	case TYPE_ARC:
		return newARC(cb)
	default:
		panic("gcache: Unknown type " + cb.tp)
	}
}

func buildCache(c *baseCache, cb *CacheBuilder) {
	c.size = cb.size
	c.loaderFunc = cb.loaderFunc
	c.expiration = cb.expiration
	c.addedFunc = cb.addedFunc
	c.evictedFunc = cb.evictedFunc
}

// load a new value using by specified key.
func (c *baseCache) load(key interface{}, cb func(interface{}, error) (interface{}, error)) (interface{}, error) {
	v, err := c.loadGroup.Do(key, func() (interface{}, error) {
		return cb((*c.loaderFunc)(key))
	})
	if err != nil {
		return nil, err
	}
	return v, nil
}
<|endoftext|>


<commit_before>package pilosa

import (
	"bytes"
	"fmt"
	"io"
	"sort"
	"sync"
	"time"

	"github.com/golang/groupcache/lru"
	"github.com/pilosa/pilosa/internal"
)

const (
	// ThresholdFactor is used to calculate the threshold for new items entering the cache
	ThresholdFactor = 1.1
)

// Cache represents a cache for bitmap counts.
type Cache interface {
	Add(bitmapID uint64, n uint64)
	BulkAdd(bitmapID uint64, n uint64)
	Get(bitmapID uint64) uint64
	Len() int

	// Returns a list of all bitmap IDs.
	BitmapIDs() []uint64

	// Updates the cache, if necessary.
	Invalidate()

	// Rebuilds the cache
	Recalculate()

	// Returns an ordered list of the top ranked bitmaps.
	Top() []BitmapPair
}

// LRUCache represents a least recently used Cache implemenation.
type LRUCache struct {
	cache  *lru.Cache
	counts map[uint64]uint64
}

// NewLRUCache returns a new instance of LRUCache.
func NewLRUCache(maxEntries uint32) *LRUCache {
	c := &LRUCache{
		cache:  lru.New(int(maxEntries)),
		counts: make(map[uint64]uint64),
	}
	c.cache.OnEvicted = c.onEvicted
	return c
}

func (c *LRUCache) BulkAdd(bitmapID, n uint64) {
	c.Add(bitmapID, n)
}

// Add adds a bitmap to the cache.
func (c *LRUCache) Add(bitmapID, n uint64) {
	c.cache.Add(bitmapID, n)
	c.counts[bitmapID] = n
}

// Get returns a bitmap with a given id.
func (c *LRUCache) Get(bitmapID uint64) uint64 {
	n, _ := c.cache.Get(bitmapID)
	nn, _ := n.(uint64)
	return nn
}

// Len returns the number of items in the cache.
func (c *LRUCache) Len() int { return c.cache.Len() }

// Invalidate is a no-op.
func (c *LRUCache) Invalidate() {}

// Recalculate is a no-op.
func (c *LRUCache) Recalculate() {}

// BitmapIDs returns a list of all bitmap IDs in the cache.
func (c *LRUCache) BitmapIDs() []uint64 {
	a := make([]uint64, 0, len(c.counts))
	for id := range c.counts {
		a = append(a, id)
	}
	sort.Sort(uint64Slice(a))
	return a
}

// Top returns all counts in the cache.
func (c *LRUCache) Top() []BitmapPair {
	a := make([]BitmapPair, 0, len(c.counts))
	for id, n := range c.counts {
		a = append(a, BitmapPair{
			ID:    id,
			Count: uint64(n),
		})
	}
	sort.Sort(BitmapPairs(a))
	return a
}

func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) }

// Ensure LRUCache implements Cache.
var _ Cache = &LRUCache{}

// RankCache represents a cache with sorted entries.
type RankCache struct {
	mu       sync.Mutex
	entries  map[uint64]uint64
	rankings []BitmapPair // cached, ordered list

	updateN    int
	updateTime time.Time

	// maxEntries is the user defined size of the cache
	maxEntries uint32

	// thresholdBuffer is used the calculate the lowest cached threshold value
	// This threshold determines what new items are added to the cache
	thresholdBuffer int

	// thresholdValue is the value of the last item in the cache
	thresholdValue uint64
}

// NewRankCache returns a new instance of RankCache.
func NewRankCache(maxEntries uint32) *RankCache {
	return &RankCache{
		maxEntries:      maxEntries,
		thresholdBuffer: int(ThresholdFactor * float64(maxEntries)),
		entries:         make(map[uint64]uint64),
	}
}

// Add adds a bitmap to the cache.
func (c *RankCache) Add(bitmapID uint64, n uint64) {
	c.mu.Lock()
	defer c.mu.Unlock()
	// Ignore if the bit count on the bitmap is below the threshold.
	if n < c.thresholdValue {
		return
	}

	c.entries[bitmapID] = n

	c.invalidate()
}

// BulkAdd adds a bitmap to the cache unsorted. You should Invalidate after completion.
func (c *RankCache) BulkAdd(bitmapID uint64, n uint64) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if n < c.thresholdValue {
		return
	}

	c.entries[bitmapID] = n
}

// Get returns a bitmap with a given id.
func (c *RankCache) Get(bitmapID uint64) uint64 {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.entries[bitmapID]
}

// Len returns the number of items in the cache.
func (c *RankCache) Len() int {
	c.mu.Lock()
	defer c.mu.Unlock()
	return len(c.entries)
}

// BitmapIDs returns a list of all bitmap IDs in the cache.
func (c *RankCache) BitmapIDs() []uint64 {
	c.mu.Lock()
	defer c.mu.Unlock()
	a := make([]uint64, 0, len(c.entries))
	for id := range c.entries {
		a = append(a, id)
	}
	sort.Sort(uint64Slice(a))
	return a
}

// Invalidate recalculates the the entries by rank.
func (c *RankCache) Invalidate() {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.invalidate()
}

func (c *RankCache) Recalculate() {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.recalculate()
}

func (c *RankCache) invalidate() {
	// Don't invalidate more than once every X seconds.
	// TODO: consider making this configurable.
	if time.Now().Sub(c.updateTime).Seconds() < 10 {
		return
	}
	c.recalculate()
}

func (c *RankCache) recalculate() {
	// Convert cache to a sorted list.
	rankings := make([]BitmapPair, 0, len(c.entries))
	for id, cnt := range c.entries {
		rankings = append(rankings, BitmapPair{
			ID:    id,
			Count: cnt,
		})
	}
	sort.Sort(BitmapPairs(rankings))

	// Store the count of the item at the threshold index.
	c.rankings = rankings
	if len(c.rankings) > int(c.maxEntries) {
		c.thresholdValue = rankings[c.maxEntries].Count
		c.rankings = c.rankings[0:c.maxEntries]
	} else {
		c.thresholdValue = 1
	}

	// Reset counters.
	c.updateTime, c.updateN = time.Now(), 0

	// If size is larger than the threshold then trim it.
	if len(c.entries) > c.thresholdBuffer {
		for id, cnt := range c.entries {
			if cnt <= c.thresholdValue {
				delete(c.entries, id)
			}
		}
	}
}

// Top returns an ordered list of bitmaps.
func (c *RankCache) Top() []BitmapPair { return c.rankings }

// WriteTo writes the cache to w.
func (c *RankCache) WriteTo(w io.Writer) (n int64, err error) {
	panic("FIXME: TODO")
}

// ReadFrom read from r into the cache.
func (c *RankCache) ReadFrom(r io.Reader) (n int64, err error) {
	panic("FIXME: TODO")
}

// Ensure RankCache implements Cache.
var _ Cache = &RankCache{}

// BitmapPair represents a bitmap with an associated identifier.
type BitmapPair struct {
	ID    uint64
	Count uint64
}

// BitmapPairs is a sortable list of BitmapPair objects.
type BitmapPairs []BitmapPair

func (p BitmapPairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p BitmapPairs) Len() int           { return len(p) }
func (p BitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }

// Pair holds a bitmap id and its count.
type Pair struct {
	ID    uint64 `json:"id"`
	Count uint64 `json:"count"`
}

func encodePair(p Pair) *internal.Pair {
	return &internal.Pair{
		Key:   p.ID,
		Count: p.Count,
	}
}

func decodePair(pb *internal.Pair) Pair {
	return Pair{
		ID:    pb.Key,
		Count: pb.Count,
	}
}

// Pairs is a sortable slice of Pair objects.
type Pairs []Pair

func (p Pairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p Pairs) Len() int           { return len(p) }
func (p Pairs) Less(i, j int) bool { return p[i].Count > p[j].Count }

// PairHeap is a heap implementation over a group of Pairs.
type PairHeap struct {
	Pairs
}

func (p PairHeap) Less(i, j int) bool { return p.Pairs[i].Count < p.Pairs[j].Count }

func (h *Pairs) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(Pair))
}

func (h *Pairs) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// Add merges other into p and returns a new slice.
func (p Pairs) Add(other []Pair) []Pair {
	// Create lookup of key/counts.
	m := make(map[uint64]uint64, len(p))
	for _, pair := range p {
		m[pair.ID] = pair.Count
	}

	// Add/merge from other.
	for _, pair := range other {
		m[pair.ID] += pair.Count
	}

	// Convert back to slice.
	a := make([]Pair, 0, len(m))
	for k, v := range m {
		a = append(a, Pair{ID: k, Count: v})
	}
	return a
}

// Keys returns a slice of all keys in p.
func (p Pairs) Keys() []uint64 {
	a := make([]uint64, len(p))
	for i := range p {
		a[i] = p[i].ID
	}
	return a
}

func (p Pairs) String() string {
	var buf bytes.Buffer
	buf.WriteString("Pairs(")
	for i := range p {
		fmt.Fprintf(&buf, "%d/%d", p[i].ID, p[i].Count)
		if i < len(p)-1 {
			buf.WriteString(", ")
		}
	}
	buf.WriteString(")")
	return buf.String()
}

func encodePairs(a Pairs) []*internal.Pair {
	other := make([]*internal.Pair, len(a))
	for i := range a {
		other[i] = encodePair(a[i])
	}
	return other
}

func decodePairs(a []*internal.Pair) []Pair {
	other := make([]Pair, len(a))
	for i := range a {
		other[i] = decodePair(a[i])
	}
	return other
}

// uint64Slice represents a sortable slice of uint64 numbers.
type uint64Slice []uint64

func (p uint64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p uint64Slice) Len() int           { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }

// merge combines p and other to a unique sorted set of values.
// p and other must both have unique sets and be sorted.
func (p uint64Slice) merge(other []uint64) []uint64 {
	ret := make([]uint64, 0, len(p))

	i, j := 0, 0
	for i < len(p) && j < len(other) {
		a, b := p[i], other[j]
		if a == b {
			ret = append(ret, a)
			i, j = i+1, j+1
		} else if a < b {
			ret = append(ret, a)
			i++
		} else {
			ret = append(ret, b)
			j++
		}
	}

	if i < len(p) {
		ret = append(ret, p[i:]...)
	} else if j < len(other) {
		ret = append(ret, other[j:]...)
	}

	return ret
}

// BitmapCache provides an interface for caching full bitmaps.
type BitmapCache interface {
	Fetch(id uint64) (*Bitmap, bool)
	Add(id uint64, b *Bitmap)
}

// SimpleCache implements BitmapCache
// it is meant to be a short-lived cache for cases where writes are continuing to access
// the same bit within a short time frame (i.e. good for write-heavy loads)
// A read-heavy use case would cause the cache to get bigger, potentially causing the
// node to run out of memory.
type SimpleCache struct {
	cache map[uint64]*Bitmap
}

// Fetch retrieves the bitmap at the id in the cache.
func (s *SimpleCache) Fetch(id uint64) (*Bitmap, bool) {
	m, ok := s.cache[id]
	return m, ok
}

// Add adds the bitmap to the cache, keyed on the id.
func (s *SimpleCache) Add(id uint64, b *Bitmap) {
	s.cache[id] = b
}
<commit_msg>add cache stats<commit_after>package pilosa

import (
	"bytes"
	"fmt"
	"io"
	"sort"
	"sync"
	"time"

	"github.com/golang/groupcache/lru"
	"github.com/pilosa/pilosa/internal"
)

const (
	// ThresholdFactor is used to calculate the threshold for new items entering the cache
	ThresholdFactor = 1.1
)

// Cache represents a cache for bitmap counts.
type Cache interface {
	Add(bitmapID uint64, n uint64)
	BulkAdd(bitmapID uint64, n uint64)
	Get(bitmapID uint64) uint64
	Len() int

	// Returns a list of all bitmap IDs.
	BitmapIDs() []uint64

	// Updates the cache, if necessary.
	Invalidate()

	// Rebuilds the cache
	Recalculate()

	// Returns an ordered list of the top ranked bitmaps.
	Top() []BitmapPair

	// SetStats defines the stats client used in the cache
	SetStats(s StatsClient)
}

// LRUCache represents a least recently used Cache implemenation.
type LRUCache struct {
	cache  *lru.Cache
	counts map[uint64]uint64
	stats  StatsClient
}

// NewLRUCache returns a new instance of LRUCache.
func NewLRUCache(maxEntries uint32) *LRUCache {
	c := &LRUCache{
		cache:  lru.New(int(maxEntries)),
		counts: make(map[uint64]uint64),
		stats:  NopStatsClient,
	}
	c.cache.OnEvicted = c.onEvicted
	return c
}

func (c *LRUCache) BulkAdd(bitmapID, n uint64) {
	c.Add(bitmapID, n)
}

// Add adds a bitmap to the cache.
func (c *LRUCache) Add(bitmapID, n uint64) {
	c.cache.Add(bitmapID, n)
	c.counts[bitmapID] = n
}

// Get returns a bitmap with a given id.
func (c *LRUCache) Get(bitmapID uint64) uint64 {
	n, _ := c.cache.Get(bitmapID)
	nn, _ := n.(uint64)
	return nn
}

// Len returns the number of items in the cache.
func (c *LRUCache) Len() int { return c.cache.Len() }

// Invalidate is a no-op.
func (c *LRUCache) Invalidate() {
	c.stats.Gauge("LRUCache", float64(c.cache.Len()))
}

// Recalculate is a no-op.
func (c *LRUCache) Recalculate() {
	c.stats.Gauge("LRUCache", float64(c.cache.Len()))
}

// BitmapIDs returns a list of all bitmap IDs in the cache.
func (c *LRUCache) BitmapIDs() []uint64 {
	a := make([]uint64, 0, len(c.counts))
	for id := range c.counts {
		a = append(a, id)
	}
	sort.Sort(uint64Slice(a))
	return a
}

// Top returns all counts in the cache.
func (c *LRUCache) Top() []BitmapPair {
	a := make([]BitmapPair, 0, len(c.counts))
	for id, n := range c.counts {
		a = append(a, BitmapPair{
			ID:    id,
			Count: uint64(n),
		})
	}
	sort.Sort(BitmapPairs(a))
	return a
}

// SetStats passes the stats client used the the frame that owns the cache
func (c *LRUCache) SetStats(s StatsClient) {
	c.stats = s
}

func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) { delete(c.counts, key.(uint64)) }

// Ensure LRUCache implements Cache.
var _ Cache = &LRUCache{}

// RankCache represents a cache with sorted entries.
type RankCache struct {
	mu       sync.Mutex
	entries  map[uint64]uint64
	rankings []BitmapPair // cached, ordered list

	updateN    int
	updateTime time.Time

	// maxEntries is the user defined size of the cache
	maxEntries uint32

	// thresholdBuffer is used the calculate the lowest cached threshold value
	// This threshold determines what new items are added to the cache
	thresholdBuffer int

	// thresholdValue is the value of the last item in the cache
	thresholdValue uint64

	stats StatsClient
}

// NewRankCache returns a new instance of RankCache.
func NewRankCache(maxEntries uint32) *RankCache {
	return &RankCache{
		maxEntries:      maxEntries,
		thresholdBuffer: int(ThresholdFactor * float64(maxEntries)),
		entries:         make(map[uint64]uint64),
		stats:           NopStatsClient,
	}
}

// Add adds a bitmap to the cache.
func (c *RankCache) Add(bitmapID uint64, n uint64) {
	c.mu.Lock()
	defer c.mu.Unlock()
	// Ignore if the bit count on the bitmap is below the threshold.
	if n < c.thresholdValue {
		return
	}

	c.entries[bitmapID] = n

	c.invalidate()
}

// BulkAdd adds a bitmap to the cache unsorted. You should Invalidate after completion.
func (c *RankCache) BulkAdd(bitmapID uint64, n uint64) {
	c.mu.Lock()
	defer c.mu.Unlock()
	if n < c.thresholdValue {
		return
	}

	c.entries[bitmapID] = n
}

// Get returns a bitmap with a given id.
func (c *RankCache) Get(bitmapID uint64) uint64 {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.entries[bitmapID]
}

// Len returns the number of items in the cache.
func (c *RankCache) Len() int {
	c.mu.Lock()
	defer c.mu.Unlock()
	return len(c.entries)
}

// BitmapIDs returns a list of all bitmap IDs in the cache.
func (c *RankCache) BitmapIDs() []uint64 {
	c.mu.Lock()
	defer c.mu.Unlock()
	a := make([]uint64, 0, len(c.entries))
	for id := range c.entries {
		a = append(a, id)
	}
	sort.Sort(uint64Slice(a))
	return a
}

// Invalidate recalculates the the entries by rank.
func (c *RankCache) Invalidate() {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.invalidate()
}

func (c *RankCache) Recalculate() {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.recalculate()
}

func (c *RankCache) invalidate() {
	// Don't invalidate more than once every X seconds.
	// TODO: consider making this configurable.
	if time.Now().Sub(c.updateTime).Seconds() < 10 {
		return
	}
	c.recalculate()
}

func (c *RankCache) recalculate() {
	// Convert cache to a sorted list.
	rankings := make([]BitmapPair, 0, len(c.entries))
	for id, cnt := range c.entries {
		rankings = append(rankings, BitmapPair{
			ID:    id,
			Count: cnt,
		})
	}
	sort.Sort(BitmapPairs(rankings))

	// Store the count of the item at the threshold index.
	c.rankings = rankings
	length := len(c.rankings)
	c.stats.Gauge("RankCache", float64(length))

	if length > int(c.maxEntries) {
		c.thresholdValue = rankings[c.maxEntries].Count
		c.rankings = c.rankings[0:c.maxEntries]
	} else {
		c.thresholdValue = 1
	}

	// Reset counters.
	c.updateTime, c.updateN = time.Now(), 0

	// If size is larger than the threshold then trim it.
	if len(c.entries) > c.thresholdBuffer {
		for id, cnt := range c.entries {
			if cnt <= c.thresholdValue {
				delete(c.entries, id)
			}
		}
	}
}

// SetStats passes the stats client used the the frame that owns the cache
func (c *RankCache) SetStats(s StatsClient) {
	c.stats = s
}

// Top returns an ordered list of bitmaps.
func (c *RankCache) Top() []BitmapPair { return c.rankings }

// WriteTo writes the cache to w.
func (c *RankCache) WriteTo(w io.Writer) (n int64, err error) {
	panic("FIXME: TODO")
}

// ReadFrom read from r into the cache.
func (c *RankCache) ReadFrom(r io.Reader) (n int64, err error) {
	panic("FIXME: TODO")
}

// Ensure RankCache implements Cache.
var _ Cache = &RankCache{}

// BitmapPair represents a bitmap with an associated identifier.
type BitmapPair struct {
	ID    uint64
	Count uint64
}

// BitmapPairs is a sortable list of BitmapPair objects.
type BitmapPairs []BitmapPair

func (p BitmapPairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p BitmapPairs) Len() int           { return len(p) }
func (p BitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }

// Pair holds a bitmap id and its count.
type Pair struct {
	ID    uint64 `json:"id"`
	Count uint64 `json:"count"`
}

func encodePair(p Pair) *internal.Pair {
	return &internal.Pair{
		Key:   p.ID,
		Count: p.Count,
	}
}

func decodePair(pb *internal.Pair) Pair {
	return Pair{
		ID:    pb.Key,
		Count: pb.Count,
	}
}

// Pairs is a sortable slice of Pair objects.
type Pairs []Pair

func (p Pairs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p Pairs) Len() int           { return len(p) }
func (p Pairs) Less(i, j int) bool { return p[i].Count > p[j].Count }

// PairHeap is a heap implementation over a group of Pairs.
type PairHeap struct {
	Pairs
}

func (p PairHeap) Less(i, j int) bool { return p.Pairs[i].Count < p.Pairs[j].Count }

func (h *Pairs) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(Pair))
}

func (h *Pairs) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// Add merges other into p and returns a new slice.
func (p Pairs) Add(other []Pair) []Pair {
	// Create lookup of key/counts.
	m := make(map[uint64]uint64, len(p))
	for _, pair := range p {
		m[pair.ID] = pair.Count
	}

	// Add/merge from other.
	for _, pair := range other {
		m[pair.ID] += pair.Count
	}

	// Convert back to slice.
	a := make([]Pair, 0, len(m))
	for k, v := range m {
		a = append(a, Pair{ID: k, Count: v})
	}
	return a
}

// Keys returns a slice of all keys in p.
func (p Pairs) Keys() []uint64 {
	a := make([]uint64, len(p))
	for i := range p {
		a[i] = p[i].ID
	}
	return a
}

func (p Pairs) String() string {
	var buf bytes.Buffer
	buf.WriteString("Pairs(")
	for i := range p {
		fmt.Fprintf(&buf, "%d/%d", p[i].ID, p[i].Count)
		if i < len(p)-1 {
			buf.WriteString(", ")
		}
	}
	buf.WriteString(")")
	return buf.String()
}

func encodePairs(a Pairs) []*internal.Pair {
	other := make([]*internal.Pair, len(a))
	for i := range a {
		other[i] = encodePair(a[i])
	}
	return other
}

func decodePairs(a []*internal.Pair) []Pair {
	other := make([]Pair, len(a))
	for i := range a {
		other[i] = decodePair(a[i])
	}
	return other
}

// uint64Slice represents a sortable slice of uint64 numbers.
type uint64Slice []uint64

func (p uint64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p uint64Slice) Len() int           { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }

// merge combines p and other to a unique sorted set of values.
// p and other must both have unique sets and be sorted.
func (p uint64Slice) merge(other []uint64) []uint64 {
	ret := make([]uint64, 0, len(p))

	i, j := 0, 0
	for i < len(p) && j < len(other) {
		a, b := p[i], other[j]
		if a == b {
			ret = append(ret, a)
			i, j = i+1, j+1
		} else if a < b {
			ret = append(ret, a)
			i++
		} else {
			ret = append(ret, b)
			j++
		}
	}

	if i < len(p) {
		ret = append(ret, p[i:]...)
	} else if j < len(other) {
		ret = append(ret, other[j:]...)
	}

	return ret
}

// BitmapCache provides an interface for caching full bitmaps.
type BitmapCache interface {
	Fetch(id uint64) (*Bitmap, bool)
	Add(id uint64, b *Bitmap)
}

// SimpleCache implements BitmapCache
// it is meant to be a short-lived cache for cases where writes are continuing to access
// the same bit within a short time frame (i.e. good for write-heavy loads)
// A read-heavy use case would cause the cache to get bigger, potentially causing the
// node to run out of memory.
type SimpleCache struct {
	cache map[uint64]*Bitmap
}

// Fetch retrieves the bitmap at the id in the cache.
func (s *SimpleCache) Fetch(id uint64) (*Bitmap, bool) {
	m, ok := s.cache[id]
	return m, ok
}

// Add adds the bitmap to the cache, keyed on the id.
func (s *SimpleCache) Add(id uint64, b *Bitmap) {
	s.cache[id] = b
}
<|endoftext|>