<commit_before>package cache

import (
	"fmt"
	"reflect"
	"runtime"
	"sync"
	"time"
)

// go-cache is an in-memory key:value store/cache similar to memcached that is suitable for
// applications running on a single machine. Any object can be stored, for a given duration
// or forever, and the cache can be used safely by multiple goroutines.
//
// Installation:
//     goinstall github.com/pmylund/go-cache
//
// Usage:
//     // Create a cache with a default expiration time of 5 minutes, and which purges
//     // expired items every 30 seconds
//     c := cache.New(5*time.Minute, 30*time.Second)
//
//     // Set the value of the key "foo" to "bar", with the default expiration time
//     c.Set("foo", "bar", 0)
//
//     // Set the value of the key "baz" to "yes", with no expiration time (the item
//     // won't be removed until it is re-set, or removed using c.Delete("baz")
//     c.Set("baz", "yes", -1)
//
//     // Get the string associated with the key "foo" from the cache
//     foo, found := c.Get("foo")
//     if found {
//             fmt.Println(foo)
//     }
//
//     // Since Go is statically typed, and cache values can be anything, type assertion
//     // is needed when values are being passed to functions that don't take arbitrary types,
//     // (i.e. interface{}). The simplest way to do this for values which will only be used
//     // once--e.g. for passing to another function--is:
//     foo, found := c.Get("foo")
//     if found {
//             MyFunction(foo.(string))
//     }
//
//     // This gets tedious if the value is used several times in the same function. You
//     // might do either of the following instead:
//     if x, found := c.Get("foo"); found {
//             foo := x.(string)
//             ...
//     }
//     // or
//     var foo string
//     if x, found := c.Get("foo"); found {
//             foo = x.(string)
//     }
//     ...
//     // foo can then be passed around freely as a string
//
//     // Want performance? Store pointers!
//     c.Set("foo", &MyStruct, 0)
//     if x, found := c.Get("foo"); found {
//             foo := x.(*MyStruct)
//             ...
//     }
//
// If you store a reference type like a pointer, slice, map or channel, you do not need to
// run Set if you modify the underlying data. The cache does not serialize its data, so if
// you modify a struct whose pointer you've stored in the cache, retrieving that pointer
// with Get will point you to the same data:
//
//     foo := &MyStruct{Num: 1}
//     c.Set("foo", foo, 0)
//     ...
//     x, _ := c.Get("foo")
//     foo := x.(MyStruct)
//     fmt.Println(foo.Num)
//     ...
//     foo.Num++
//     ...
//     x, _ := c.Get("foo")
//     foo := x.(MyStruct)
//     foo.Println(foo.Num)
//
// will print:
//     1
//     2

type Cache struct {
	*cache
	// If this is confusing, see the comment at the bottom of the New() function
}

type cache struct {
	DefaultExpiration time.Duration
	Items             map[string]*Item
	mu                *sync.Mutex
	janitor           *janitor
}

type Item struct {
	Object     interface{}
	Expires    bool
	Expiration *time.Time
}

type janitor struct {
	Interval time.Duration
	stop     chan bool
}

// Adds an item to the cache, replacing any existing item. If the duration is 0, the
// cache's default expiration time is used. If it is -1, the item never expires.
func (c *cache) Set(k string, x interface{}, d time.Duration) {
	c.mu.Lock()
	defer c.mu.Unlock()

	var e *time.Time
	expires := true
	if d == 0 {
		d = c.DefaultExpiration
	}
	if d == -1 {
		expires = false
	} else {
		t := time.Now().Add(d)
		e = &t
	}
	c.Items[k] = &Item{
		Object:     x,
		Expires:    expires,
		Expiration: e,
	}
}

// TODO: Add and Replace aren't completely atomic

// Adds an item to the cache only if an item doesn't already exist for the given key,
// or if the existing item has expired. Returns an error if not.
func (c *cache) Add(k string, x interface{}, d time.Duration) error {
	_, found := c.Get(k)
	if found {
		return fmt.Errorf("Item %s already exists", k)
	}
	c.Set(k, x, d)
	return nil
}

// Sets a new value for the cache item only if it already exists. Returns an error if
// it does not.
func (c *cache) Replace(k string, x interface{}, d time.Duration) error {
	_, found := c.Get(k)
	if !found {
		return fmt.Errorf("Item %s doesn't exist", k)
	}
	c.Set(k, x, d)
	return nil
}

// Gets an item from the cache.
func (c *cache) Get(k string) (interface{}, bool) {
	c.mu.Lock()
	defer c.mu.Unlock()

	item, found := c.Items[k]
	if !found {
		return nil, false
	}
	if item.Expired() {
		delete(c.Items, k)
		return nil, false
	}
	return item.Object, true
}

// Increment an item of type int, int8, int16, int32, int64, uintptr, uint, uint8,
// uint32, uint64, float32 or float64 by n. Returns an error if the item's value is
// not an integer, if it was not found, or if it is not possible to increment it by
// n.
func (c *cache) IncrementFloat(k string, n float64) error {
	c.mu.Lock()
	defer c.mu.Unlock()

	v, found := c.Items[k]
	if !found {
		return fmt.Errorf("V not found")
	}

	t := reflect.TypeOf(v.Object)
	switch t.Kind() {
	default:
		return fmt.Errorf("The value of %s is not an integer", k)
	case reflect.Uint:
		v.Object = v.Object.(uint) + uint(n)
	case reflect.Uintptr:
		v.Object = v.Object.(uintptr) + uintptr(n)
	case reflect.Uint8:
		v.Object = v.Object.(uint8) + uint8(n)
	case reflect.Uint16:
		v.Object = v.Object.(uint16) + uint16(n)
	case reflect.Uint32:
		v.Object = v.Object.(uint32) + uint32(n)
	case reflect.Uint64:
		v.Object = v.Object.(uint64) + uint64(n)
	case reflect.Int:
		v.Object = v.Object.(int) + int(n)
	case reflect.Int8:
		v.Object = v.Object.(int8) + int8(n)
	case reflect.Int16:
		v.Object = v.Object.(int16) + int16(n)
	case reflect.Int32:
		v.Object = v.Object.(int32) + int32(n)
	case reflect.Int64:
		v.Object = v.Object.(int64) + int64(n)
	case reflect.Float32:
		v.Object = v.Object.(float32) + float32(n)
	case reflect.Float64:
		v.Object = v.Object.(float64) + n
	}
	return nil
}

// Increment an item of type int, int8, int16, int32, int64, uintptr, uint, uint8,
// uint32, or uint64, float32 or float64 by n. Returns an error if the item's value
// is not an integer, if it was not found, or if it is not possible to increment it
// by n.
func (c *cache) Increment(k string, n int64) error {
	return c.IncrementFloat(k, float64(n))
}

// Decrement an item of type int, int8, int16, int32, int64, uintptr, uint, uint8,
// uint32, or uint64, float32 or float64 by n. Returns an error if the item's value
// is not an integer, if it was not found, or if it is not possible to decrement it
// by n.
func (c *cache) Decrement(k string, n int64) error {
	return c.Increment(k, n*-1)
}

// Deletes an item from the cache. Does nothing if the item does not exist in the cache.
func (c *cache) Delete(k string) {
	c.mu.Lock()
	defer c.mu.Unlock()

	delete(c.Items, k)
}

// Deletes all expired items from the cache.
func (c *cache) DeleteExpired() {
	c.mu.Lock()
	defer c.mu.Unlock()

	for k, v := range c.Items {
		if v.Expired() {
			delete(c.Items, k)
		}
	}
}

// Deletes all items in the cache
func (c *cache) Flush() {
	c.mu.Lock()
	defer c.mu.Unlock()

	c.Items = map[string]*Item{}
}

// Returns true if the item has expired.
func (i *Item) Expired() bool {
	if i.Expiration == nil {
		return false
	}
	return i.Expiration.Before(time.Now())
}

func (j *janitor) Run(c *cache) {
	j.stop = make(chan bool)
	tick := time.Tick(j.Interval)
	for {
		select {
		case <-tick:
			c.DeleteExpired()
		case <-j.stop:
			return
		}
	}
}

func (j *janitor) Stop() {
	j.stop <- true
}

func stopJanitor(c *Cache) {
	c.janitor.Stop()
}

// Returns a new cache with a given default expiration duration and default cleanup
// interval. If the expiration duration is less than 1, the items in the cache never expire
// and have to be deleted manually. If the cleanup interval is less than one, expired
// items are not deleted from the cache before their next lookup or before calling
// DeleteExpired.
func New(de, ci time.Duration) *Cache {
	if de == 0 {
		de = -1
	}
	c := &cache{
		DefaultExpiration: de,
		Items:             map[string]*Item{},
		mu:                &sync.Mutex{},
	}
	if ci > 0 {
		j := &janitor{
			Interval: ci,
		}
		c.janitor = j
		go j.Run(c)
	}
	// This trick ensures that the janitor goroutine (which--granted it was enabled--is
	// running DeleteExpired on c forever) does not keep the returned C object from being
	// garbage collected. When it is garbage collected, the finalizer stops the janitor
	// goroutine, after which c is collected.
	C := &Cache{c}
	if ci > 0 {
		runtime.SetFinalizer(C, stopJanitor)
	}
	return C
}
<commit_msg>V should be Item<commit_after>package cache

import (
	"fmt"
	"reflect"
	"runtime"
	"sync"
	"time"
)

// go-cache is an in-memory key:value store/cache similar to memcached that is suitable for
// applications running on a single machine. Any object can be stored, for a given duration
// or forever, and the cache can be used safely by multiple goroutines.
//
// Installation:
//     goinstall github.com/pmylund/go-cache
//
// Usage:
//     // Create a cache with a default expiration time of 5 minutes, and which purges
//     // expired items every 30 seconds
//     c := cache.New(5*time.Minute, 30*time.Second)
//
//     // Set the value of the key "foo" to "bar", with the default expiration time
//     c.Set("foo", "bar", 0)
//
//     // Set the value of the key "baz" to "yes", with no expiration time (the item
//     // won't be removed until it is re-set, or removed using c.Delete("baz")
//     c.Set("baz", "yes", -1)
//
//     // Get the string associated with the key "foo" from the cache
//     foo, found := c.Get("foo")
//     if found {
//             fmt.Println(foo)
//     }
//
//     // Since Go is statically typed, and cache values can be anything, type assertion
//     // is needed when values are being passed to functions that don't take arbitrary types,
//     // (i.e. interface{}). The simplest way to do this for values which will only be used
//     // once--e.g. for passing to another function--is:
//     foo, found := c.Get("foo")
//     if found {
//             MyFunction(foo.(string))
//     }
//
//     // This gets tedious if the value is used several times in the same function. You
//     // might do either of the following instead:
//     if x, found := c.Get("foo"); found {
//             foo := x.(string)
//             ...
//     }
//     // or
//     var foo string
//     if x, found := c.Get("foo"); found {
//             foo = x.(string)
//     }
//     ...
//     // foo can then be passed around freely as a string
//
//     // Want performance? Store pointers!
//     c.Set("foo", &MyStruct, 0)
//     if x, found := c.Get("foo"); found {
//             foo := x.(*MyStruct)
//             ...
//     }
//
// If you store a reference type like a pointer, slice, map or channel, you do not need to
// run Set if you modify the underlying data. The cache does not serialize its data, so if
// you modify a struct whose pointer you've stored in the cache, retrieving that pointer
// with Get will point you to the same data:
//
//     foo := &MyStruct{Num: 1}
//     c.Set("foo", foo, 0)
//     ...
//     x, _ := c.Get("foo")
//     foo := x.(MyStruct)
//     fmt.Println(foo.Num)
//     ...
//     foo.Num++
//     ...
//     x, _ := c.Get("foo")
//     foo := x.(MyStruct)
//     foo.Println(foo.Num)
//
// will print:
//     1
//     2

type Cache struct {
	*cache
	// If this is confusing, see the comment at the bottom of the New() function
}

type cache struct {
	DefaultExpiration time.Duration
	Items             map[string]*Item
	mu                *sync.Mutex
	janitor           *janitor
}

type Item struct {
	Object     interface{}
	Expires    bool
	Expiration *time.Time
}

type janitor struct {
	Interval time.Duration
	stop     chan bool
}

// Adds an item to the cache, replacing any existing item. If the duration is 0, the
// cache's default expiration time is used. If it is -1, the item never expires.
func (c *cache) Set(k string, x interface{}, d time.Duration) {
	c.mu.Lock()
	defer c.mu.Unlock()

	var e *time.Time
	expires := true
	if d == 0 {
		d = c.DefaultExpiration
	}
	if d == -1 {
		expires = false
	} else {
		t := time.Now().Add(d)
		e = &t
	}
	c.Items[k] = &Item{
		Object:     x,
		Expires:    expires,
		Expiration: e,
	}
}

// TODO: Add and Replace aren't completely atomic

// Adds an item to the cache only if an item doesn't already exist for the given key,
// or if the existing item has expired. Returns an error if not.
func (c *cache) Add(k string, x interface{}, d time.Duration) error {
	_, found := c.Get(k)
	if found {
		return fmt.Errorf("Item %s already exists", k)
	}
	c.Set(k, x, d)
	return nil
}

// Sets a new value for the cache item only if it already exists. Returns an error if
// it does not.
func (c *cache) Replace(k string, x interface{}, d time.Duration) error {
	_, found := c.Get(k)
	if !found {
		return fmt.Errorf("Item %s doesn't exist", k)
	}
	c.Set(k, x, d)
	return nil
}

// Gets an item from the cache.
func (c *cache) Get(k string) (interface{}, bool) {
	c.mu.Lock()
	defer c.mu.Unlock()

	item, found := c.Items[k]
	if !found {
		return nil, false
	}
	if item.Expired() {
		delete(c.Items, k)
		return nil, false
	}
	return item.Object, true
}

// Increment an item of type int, int8, int16, int32, int64, uintptr, uint, uint8,
// uint32, uint64, float32 or float64 by n. Returns an error if the item's value is
// not an integer, if it was not found, or if it is not possible to increment it by
// n.
func (c *cache) IncrementFloat(k string, n float64) error {
	c.mu.Lock()
	defer c.mu.Unlock()

	v, found := c.Items[k]
	if !found {
		return fmt.Errorf("Item not found")
	}

	t := reflect.TypeOf(v.Object)
	switch t.Kind() {
	default:
		return fmt.Errorf("The value of %s is not an integer", k)
	case reflect.Uint:
		v.Object = v.Object.(uint) + uint(n)
	case reflect.Uintptr:
		v.Object = v.Object.(uintptr) + uintptr(n)
	case reflect.Uint8:
		v.Object = v.Object.(uint8) + uint8(n)
	case reflect.Uint16:
		v.Object = v.Object.(uint16) + uint16(n)
	case reflect.Uint32:
		v.Object = v.Object.(uint32) + uint32(n)
	case reflect.Uint64:
		v.Object = v.Object.(uint64) + uint64(n)
	case reflect.Int:
		v.Object = v.Object.(int) + int(n)
	case reflect.Int8:
		v.Object = v.Object.(int8) + int8(n)
	case reflect.Int16:
		v.Object = v.Object.(int16) + int16(n)
	case reflect.Int32:
		v.Object = v.Object.(int32) + int32(n)
	case reflect.Int64:
		v.Object = v.Object.(int64) + int64(n)
	case reflect.Float32:
		v.Object = v.Object.(float32) + float32(n)
	case reflect.Float64:
		v.Object = v.Object.(float64) + n
	}
	return nil
}

// Increment an item of type int, int8, int16, int32, int64, uintptr, uint, uint8,
// uint32, or uint64, float32 or float64 by n. Returns an error if the item's value
// is not an integer, if it was not found, or if it is not possible to increment it
// by n.
func (c *cache) Increment(k string, n int64) error {
	return c.IncrementFloat(k, float64(n))
}

// Decrement an item of type int, int8, int16, int32, int64, uintptr, uint, uint8,
// uint32, or uint64, float32 or float64 by n. Returns an error if the item's value
// is not an integer, if it was not found, or if it is not possible to decrement it
// by n.
func (c *cache) Decrement(k string, n int64) error {
	return c.Increment(k, n*-1)
}

// Deletes an item from the cache. Does nothing if the item does not exist in the cache.
func (c *cache) Delete(k string) {
	c.mu.Lock()
	defer c.mu.Unlock()

	delete(c.Items, k)
}

// Deletes all expired items from the cache.
func (c *cache) DeleteExpired() {
	c.mu.Lock()
	defer c.mu.Unlock()

	for k, v := range c.Items {
		if v.Expired() {
			delete(c.Items, k)
		}
	}
}

// Deletes all items in the cache
func (c *cache) Flush() {
	c.mu.Lock()
	defer c.mu.Unlock()

	c.Items = map[string]*Item{}
}

// Returns true if the item has expired.
func (i *Item) Expired() bool {
	if i.Expiration == nil {
		return false
	}
	return i.Expiration.Before(time.Now())
}

func (j *janitor) Run(c *cache) {
	j.stop = make(chan bool)
	tick := time.Tick(j.Interval)
	for {
		select {
		case <-tick:
			c.DeleteExpired()
		case <-j.stop:
			return
		}
	}
}

func (j *janitor) Stop() {
	j.stop <- true
}

func stopJanitor(c *Cache) {
	c.janitor.Stop()
}

// Returns a new cache with a given default expiration duration and default cleanup
// interval. If the expiration duration is less than 1, the items in the cache never expire
// and have to be deleted manually. If the cleanup interval is less than one, expired
// items are not deleted from the cache before their next lookup or before calling
// DeleteExpired.
func New(de, ci time.Duration) *Cache {
	if de == 0 {
		de = -1
	}
	c := &cache{
		DefaultExpiration: de,
		Items:             map[string]*Item{},
		mu:                &sync.Mutex{},
	}
	if ci > 0 {
		j := &janitor{
			Interval: ci,
		}
		c.janitor = j
		go j.Run(c)
	}
	// This trick ensures that the janitor goroutine (which--granted it was enabled--is
	// running DeleteExpired on c forever) does not keep the returned C object from being
	// garbage collected. When it is garbage collected, the finalizer stops the janitor
	// goroutine, after which c is collected.
	C := &Cache{c}
	if ci > 0 {
		runtime.SetFinalizer(C, stopJanitor)
	}
	return C
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"io/ioutil"
	log "github.com/Sirupsen/logrus"
	"net/http"
	"net/http/httputil"
	"net/url"
)

// Item, the cached item
type Item struct {
	Key string
	Value []byte
}

// Cacher, something that can store and retrive to and from a cache
type Cacher interface {
	Get(string) (*Item, error)
	Set(*Item) error
}

type cacheTransport struct {
	// wrappedTransport: the transport we are caching
	wrappedTransport http.RoundTripper
	// cacheKey: function returning the key to cache on
	cacheKey func(*http.Request) (string, error)
	// cache: the Cache
	cache Cacher
}

func (ct *cacheTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	// first check the cache
	key, err := ct.cacheKey(r)
	if err != nil {
		log.Fatal("cache key error:", err)
	}
	item, err := ct.cache.Get(key)
	if err != nil {
		// there may not be a url in the request
		if r.URL == nil {
			r.URL = &url.URL{}
		}
		log.Println(
			"cache miss:",
			err,
			key,
			r.Method,
			r.URL.Host,
			r.URL.Path,
			r.URL.RawQuery,
			r.Header,
		)
	}

	// if we have an item, return that as the response
	if item != nil {
		reader := bufio.NewReader(bytes.NewReader(item.Value))
		response, err := http.ReadResponse(reader, r)
		if err != nil {
			log.Println("error reading response from cache:", err)
		} else {
			return response, nil
		}
	}

	// cache miss or cache error, gotta actually do the request
	response, err := ct.wrappedTransport.RoundTrip(r)
	if err != nil {
		log.Fatal("cacheTransport upstream error:", err)
	}
	// and make sure we cache a copy of the response but don't wait to
	// return the response
	cacheCopy := new(http.Response)
	*cacheCopy = *response
	if response.Body != nil {
		// TODO: extract this to a helper function
		bodyBytes, err := ioutil.ReadAll(response.Body)
		if err != nil {
			log.Fatal("error reading response body:", err)
		}
		cacheBytes := make([]byte, len(bodyBytes))
		copy(cacheBytes, bodyBytes)
		response.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
		cacheCopy.Body = ioutil.NopCloser(bytes.NewBuffer(cacheBytes))
	}

	go func () {
		dump, err := httputil.DumpResponse(cacheCopy, true)
		if err != nil {
			log.Fatal("could not dump response", err)
		}
		ct.cache.Set(&Item{Key: key, Value: dump})
	}()

	return response, nil
}
<commit_msg>only cache 200 responses<commit_after>package main

import (
	"bufio"
	"bytes"
	"io/ioutil"
	log "github.com/Sirupsen/logrus"
	"net/http"
	"net/http/httputil"
	"net/url"
)

// Item, the cached item
type Item struct {
	Key string
	Value []byte
}

// Cacher, something that can store and retrive to and from a cache
type Cacher interface {
	Get(string) (*Item, error)
	Set(*Item) error
}

type cacheTransport struct {
	// wrappedTransport: the transport we are caching
	wrappedTransport http.RoundTripper
	// cacheKey: function returning the key to cache on
	cacheKey func(*http.Request) (string, error)
	// cache: the Cache
	cache Cacher
}

func (ct *cacheTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	// first check the cache
	key, err := ct.cacheKey(r)
	if err != nil {
		log.Fatal("cache key error:", err)
	}
	item, err := ct.cache.Get(key)
	if err != nil {
		// there may not be a url in the request
		if r.URL == nil {
			r.URL = &url.URL{}
		}
		log.Println(
			"cache miss:",
			err,
			key,
			r.Method,
			r.URL.Host,
			r.URL.Path,
			r.URL.RawQuery,
			r.Header,
		)
	}

	// if we have an item, return that as the response
	if item != nil {
		reader := bufio.NewReader(bytes.NewReader(item.Value))
		response, err := http.ReadResponse(reader, r)
		if err != nil {
			log.Println("error reading response from cache:", err)
		} else {
			return response, nil
		}
	}

	// cache miss or cache error, gotta actually do the request
	response, err := ct.wrappedTransport.RoundTrip(r)
	if err != nil {
		log.Fatal("cacheTransport upstream error:", err)
	}
	// and make sure we cache a copy of the response if it's good
	// but don't wait to return the response
	if response.StatusCode != 200 {
		return response, nil
	}
	cacheCopy := new(http.Response)
	*cacheCopy = *response
	if response.Body != nil {
		// TODO: extract this to a helper function
		bodyBytes, err := ioutil.ReadAll(response.Body)
		if err != nil {
			log.Fatal("error reading response body:", err)
		}
		cacheBytes := make([]byte, len(bodyBytes))
		copy(cacheBytes, bodyBytes)
		response.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
		cacheCopy.Body = ioutil.NopCloser(bytes.NewBuffer(cacheBytes))
	}

	go func () {
		dump, err := httputil.DumpResponse(cacheCopy, true)
		if err != nil {
			log.Fatal("could not dump response", err)
		}
		ct.cache.Set(&Item{Key: key, Value: dump})
	}()

	return response, nil
}
<|endoftext|>


<commit_before>// -*- tab-width: 4; -*-

package main

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"os"
)

type Cached struct {
	Tweets       Tweets
	Lastmodified string
}

// key: url
type Cache map[string]Cached

func (c Cache) Store(configpath string) {
	b := new(bytes.Buffer)
	enc := gob.NewEncoder(b)
	err := enc.Encode(c)
	if err != nil {
		panic(err)
	}

	f, err := os.OpenFile(fmt.Sprintf("%s/cache", configpath),
		os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	if _, err = f.Write(b.Bytes()); err != nil {
		panic(err)
	}
}

func Loadcache(configpath string) Cache {
	cache := make(Cache)

	f, err := os.Open(fmt.Sprintf("%s/cache", configpath))
	if err != nil {
		return cache
	}
	defer f.Close()

	dec := gob.NewDecoder(f)
	err = dec.Decode(&cache)
	if err != nil {
		panic(err)
	}
	return cache
}
<commit_msg>The way to distinguish no-such-file-error from something worse<commit_after>// -*- tab-width: 4; -*-

package main

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"os"
)

type Cached struct {
	Tweets       Tweets
	Lastmodified string
}

// key: url
type Cache map[string]Cached

func (c Cache) Store(configpath string) {
	b := new(bytes.Buffer)
	enc := gob.NewEncoder(b)
	err := enc.Encode(c)
	if err != nil {
		panic(err)
	}

	f, err := os.OpenFile(fmt.Sprintf("%s/cache", configpath),
		os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	if _, err = f.Write(b.Bytes()); err != nil {
		panic(err)
	}
}

func Loadcache(configpath string) Cache {
	cache := make(Cache)

	f, err := os.Open(fmt.Sprintf("%s/cache", configpath))
	if err != nil {
		if os.IsNotExist(err) {
			return cache
		}
		panic(err)
	}
	defer f.Close()

	dec := gob.NewDecoder(f)
	err = dec.Decode(&cache)
	if err != nil {
		panic(err)
	}
	return cache
}
<|endoftext|>


<commit_before>package objcache

import (
	"container/list"
	"sync"
	"time"
)

type pair struct {
	Object interface{}
	expire int64
	key    string
}

type ObjCache struct {
	mu        sync.RWMutex
	items     map[string]*list.Element
	list      *list.List
	itemCount int
	config    Config
}

func (c *ObjCache) removeExpired() {
	e := time.Now().UnixNano()
	for {
		elem := c.list.Front()
		if elem == nil {
			break
		}
		v := elem.Value.(pair)
		if v.expire < e {
			c.itemCount = c.itemCount - 1
			delete(c.items, v.key)
			c.list.Remove(elem)
		} else {
			break
		}
	}
}

func (c *ObjCache) removeOldest() {
	c.itemCount = c.itemCount - 1
	elem := c.list.Front()
	v := elem.Value.(pair)
	delete(c.items, v.key)
	c.list.Remove(elem)
}

func (c *ObjCache) Set(k string, x interface{}, d time.Duration) error {
	if d == 0 {
		d = c.config.Expiration
	}
	c.mu.Lock()

	if _, ok := c.items[k]; !ok {

		c.removeExpired()

		if c.itemCount >= c.config.MaxEntryLimit {
			c.removeOldest()
		}

		p := pair{
			Object: x,
			key:    k,
			expire: time.Now().Add(d).UnixNano(),
		}
		c.items[k] = c.list.PushBack(p)
		c.itemCount = c.itemCount + 1
	} else {
		c.list.MoveToBack(c.items[k])
	}

	c.mu.Unlock()
	return nil
}

func (c *ObjCache) Get(k string) (interface{}, bool) {
	c.mu.RLock()
	elem, ok := c.items[k]
	if !ok {
		//TODO: delete if expired
		c.mu.RUnlock()
		return nil, false
	}
	c.mu.RUnlock()
	return elem.Value.(pair).Object, true
}

func (c *ObjCache) Del(k string) bool {
	c.mu.Lock()
	item, ok := c.items[k]
	if ok {
		c.itemCount = c.itemCount - 1
		delete(c.items, k)
		c.list.Remove(item)
	}
	c.mu.Unlock()
	return ok
}

func New(config Config) (*ObjCache, error) {
	l := list.New()
	cache := &ObjCache{
		items:     make(map[string]*list.Element),
		itemCount: 0,
		list:      l,
		config:    config,
	}
	return cache, nil
}
<commit_msg>expire item when try to get and add comments<commit_after>package objcache

import (
	"container/list"
	"sync"
	"time"
)

type pair struct {
	Object interface{}
	expire int64
	key    string
}

// ObjCache is a struct for managing cache.
// If a user call objcache.New(), returns an instance of this struct.
type ObjCache struct {
	mu        sync.RWMutex
	items     map[string]*list.Element
	list      *list.List
	itemCount int
	config    Config
}

func (c *ObjCache) removeExpired() {
	e := time.Now().UnixNano()
	for {
		elem := c.list.Front()
		if elem == nil {
			break
		}
		v := elem.Value.(pair)
		if v.expire < e {
			c.itemCount = c.itemCount - 1
			delete(c.items, v.key)
			c.list.Remove(elem)
		} else {
			break
		}
	}
}

func (c *ObjCache) removeOldest() {
	c.itemCount = c.itemCount - 1
	elem := c.list.Front()
	v := elem.Value.(pair)
	delete(c.items, v.key)
	c.list.Remove(elem)
}

// Set a value for key. if d is 0, the Expiration time would be default time.
func (c *ObjCache) Set(k string, x interface{}, d time.Duration) error {
	if d == 0 {
		d = c.config.Expiration
	}
	c.mu.Lock()

	if _, ok := c.items[k]; !ok {

		c.removeExpired()

		if c.itemCount >= c.config.MaxEntryLimit {
			c.removeOldest()
		}

		p := pair{
			Object: x,
			key:    k,
			expire: time.Now().Add(d).UnixNano(),
		}
		c.items[k] = c.list.PushBack(p)
		c.itemCount = c.itemCount + 1
	} else {
		c.list.MoveToBack(c.items[k])
	}

	c.mu.Unlock()
	return nil
}

// Get the object of key.
func (c *ObjCache) Get(k string) (interface{}, bool) {
	c.mu.RLock()
	elem, ok := c.items[k]
	if !ok {
		c.mu.RUnlock()
		return nil, false
	}
	v := elem.Value.(pair)

	if v.expire < time.Now().UnixNano() {
		c.itemCount = c.itemCount - 1
		delete(c.items, k)
		c.list.Remove(elem)
		c.mu.RUnlock()
		return nil, false
	}
	c.mu.RUnlock()
	return v.Object, true
}

// Del delete an item for some key.
func (c *ObjCache) Del(k string) bool {
	c.mu.Lock()
	item, ok := c.items[k]
	if ok {
		c.itemCount = c.itemCount - 1
		delete(c.items, k)
		c.list.Remove(item)
	}
	c.mu.Unlock()
	return ok
}

// New makes an cache object and returns it.
func New(config Config) (*ObjCache, error) {
	l := list.New()
	cache := &ObjCache{
		items:     make(map[string]*list.Element),
		itemCount: 0,
		list:      l,
		config:    config,
	}
	return cache, nil
}
<|endoftext|>


<commit_before><commit_msg>Improve code generated by protoc-gen-operatorcmd slightly<commit_after><|endoftext|>


<commit_before>package utils

import (
	"io"
	"net/http"
	"os"

	davclient "github.com/cloudfoundry/bosh-davcli/client"
	davconfig "github.com/cloudfoundry/bosh-davcli/config"
	"github.com/cloudfoundry/bosh-utils/uuid"
)

type BlobClient struct {
	dav           davclient.Client
	uuidGenerator uuid.Generator
}

func (b BlobClient) Create(filepath string) (string, error) {
	uuid, err := b.uuidGenerator.Generate()
	if err != nil {
		return "", err
	}
	file, err := os.Open(filepath)
	if err != nil {
		return "", err
	}
	info, err := file.Stat()
	if err != nil {
		return "", err
	}

	err = b.dav.Put(uuid, file, info.Size())
	if err != nil {
		return "", err
	}
	return uuid, nil
}

func (b BlobClient) Get(uuid string, destinationPath string) error {
	readCloser, err := b.dav.Get(uuid)
	if err != nil {
		return err
	}
	defer readCloser.Close()

	targetFile, err := os.Create(destinationPath)
	if err != nil {
		return err
	}
	defer targetFile.Close()

	_, err = io.Copy(targetFile, readCloser)

	return err
}

func NewBlobstore(uri string) BlobClient {
	config := davconfig.Config{Endpoint: uri}
	client := davclient.NewClient(config, http.DefaultClient)

	return BlobClient{
		dav:           client,
		uuidGenerator: uuid.NewGenerator(),
	}
}
<commit_msg>Add missing davcli parameter in windows integration<commit_after>package utils

import (
	"io"
	"net/http"
	"os"

	davclient "github.com/cloudfoundry/bosh-davcli/client"
	davconfig "github.com/cloudfoundry/bosh-davcli/config"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"
	"github.com/cloudfoundry/bosh-utils/uuid"
)

type BlobClient struct {
	dav           davclient.Client
	uuidGenerator uuid.Generator
}

func (b BlobClient) Create(filepath string) (string, error) {
	uuid, err := b.uuidGenerator.Generate()
	if err != nil {
		return "", err
	}
	file, err := os.Open(filepath)
	if err != nil {
		return "", err
	}
	info, err := file.Stat()
	if err != nil {
		return "", err
	}

	err = b.dav.Put(uuid, file, info.Size())
	if err != nil {
		return "", err
	}
	return uuid, nil
}

func (b BlobClient) Get(uuid string, destinationPath string) error {
	readCloser, err := b.dav.Get(uuid)
	if err != nil {
		return err
	}
	defer readCloser.Close()

	targetFile, err := os.Create(destinationPath)
	if err != nil {
		return err
	}
	defer targetFile.Close()

	_, err = io.Copy(targetFile, readCloser)

	return err
}

func NewBlobstore(uri string) BlobClient {
	config := davconfig.Config{Endpoint: uri}
	logger := boshlog.NewLogger(boshlog.LevelNone)
	client := davclient.NewClient(config, http.DefaultClient, logger)

	return BlobClient{
		dav:           client,
		uuidGenerator: uuid.NewGenerator(),
	}
}
<|endoftext|>


<commit_before>package mango

import (
	"net/http"
	"fmt"
	"strings"
	"io/ioutil"
	"bytes"
	"mime/multipart"
	"net/url"
)

type HTTPRequest struct {
	Args []string		//route params
	Method string		//HTTP method
	RemoteAddr string	//ip
	Path string
	Host string
	Params url.Values	//GET or POST params
	Body []byte			//Content-Type=application/octet-stream, Body=http body
	Files map[string][]*multipart.FileHeader		//Content-Type=multipart/form-data

	W http.ResponseWriter
	R *http.Request
}

func (self *HTTPRequest) Init(w http.ResponseWriter, r *http.Request, a []string) {
	self.W = w
	self.R = r
	self.Method = r.Method
	self.Path = r.URL.Path
	self.Args = a
	self.Host = r.Host

	ip := r.Header.Get("X-Forwarded-For")
	if ip == "" {
		ip = r.RemoteAddr
	}
	self.RemoteAddr = strings.Split(ip, ":")[0]
	self.parseBodyArguments()
}

func (self *HTTPRequest) UserAgent() string {
	return self.R.Header.Get("User-Agent")
}

func (self *HTTPRequest) Header(s string) string {
	return self.R.Header.Get(s)
}

func (self *HTTPRequest) SetHeader(k, v string) {
	self.W.Header().Set(k, v)
}

func (self *HTTPRequest) SetStatus(st int) {
	self.W.WriteHeader(st)
}

func (self *HTTPRequest) Cookie(k string) string {
	ck, err := self.R.Cookie(k)
	if err != nil {
		return ""
	}
	return ck.Value
}

var cks = strings.NewReplacer("\n", "-", "\r", "-")
var cvs = strings.NewReplacer("\n", " ", "\r", " ", ";", " ")

func (self *HTTPRequest) SetCookie(k, v string, params ...interface{}) {
	var b bytes.Buffer
	fmt.Fprintf(&b, "%s=%s", cks.Replace(k), cvs.Replace(v))
	if len(params) > 0 {
		switch v := params[0].(type) {
		case int:
			if v > 0 {
				fmt.Fprintf(&b, "; Max-Age=%d", v)
			} else if v < 0 {
				fmt.Fprintf(&b, "; Max-Age=0")
			}
		case int64:
			if v > 0 {
				fmt.Fprintf(&b, "; Max-Age=%d", v)
			} else if v < 0 {
				fmt.Fprintf(&b, "; Max-Age=0")
			}
		case int32:
			if v > 0 {
				fmt.Fprintf(&b, "; Max-Age=%d", v)
			} else if v < 0 {
				fmt.Fprintf(&b, "; Max-Age=0")
			}
		}
	}
	// Path, Domain, Secure, HttpOnly

	// default "/"
	if len(params) > 1 {
		if v, ok := params[1].(string); ok && len(v) > 0 {
			fmt.Fprintf(&b, "; Path=%s", cvs.Replace(v))
		}
	} else {
		fmt.Fprintf(&b, "; Path=%s", "/")
	}

	// default empty
	if len(params) > 2 {
		if v, ok := params[2].(string); ok && len(v) > 0 {
			fmt.Fprintf(&b, "; Domain=%s", cvs.Replace(v))
		}
	}

	// default empty
	if len(params) > 3 {
		var secure bool
		switch v := params[3].(type) {
		case bool:
			secure = v
		default:
			if params[3] != nil {
				secure = true
			}
		}
		if secure {
			fmt.Fprintf(&b, "; Secure")
		}
	}

	// default false.
	httponly := false
	if len(params) > 4 {
		if v, ok := params[4].(bool); ok && v {
			// HttpOnly = true
			httponly = true
		}
	}

	if httponly {
		fmt.Fprintf(&b, "; HttpOnly")
	}

	self.W.Header().Add("Set-Cookie", b.String())
}

func (self *HTTPRequest) Scheme() string {
	if self.R.URL.Scheme != "" {
		return self.R.URL.Scheme
	} else if self.R.TLS == nil {
		return "http"
	} else {
		return "https"
	}
}

func (self *HTTPRequest) Redirect(newloc string) {
	self.SetHeader("Location", newloc)
	self.SetStatus(303)
}

func (self *HTTPRequest) Query() map[string][]string {
	return self.R.URL.Query()
}

func (self *HTTPRequest) QueryString() string {
	if s := strings.Split(self.R.RequestURI, "?"); len(s) > 1 {
		return s[1]
	}else{
		return ""
	}
}

func (self *HTTPRequest) parseBodyArguments() {
	qs := self.R.URL.Query()
	var pt url.Values
	ct := self.Header("Content-Type")
	if strings.HasPrefix(ct, "application/x-www-form-urlencoded") {
		if err := self.R.ParseForm(); err != nil {
			fmt.Println("Error parsing request body:", err)
		} else {
			pt = self.R.Form
		}
	} else if strings.HasPrefix(ct, "multipart/form-data") {
		// TODO: Extract the multipart form param so app can set it.
		if err := self.R.ParseMultipartForm(32 << 20 /* 32 MB */); err != nil {
			fmt.Println("Error parsing request body:", err)
		} else {
			self.Files = self.R.MultipartForm.File
		}
	} else if strings.HasPrefix(ct, "application/octet-stream") {
		self.Body = self.Data()
	}

	num := len(qs) + len(pt)

	if num == 0 {
		self.Params = make(url.Values, 0)
		return
	}
	values := make(url.Values, num)
	for k, v := range qs {
		values[k] = append(values[k], v...)
	}
	for k, v := range pt {
		values[k] = append(values[k], v...)
	}
	self.Params = values
}

func (self *HTTPRequest) Data() []byte {
	body, _ := ioutil.ReadAll(self.R.Body)
	self.R.Body.Close()
	bf := bytes.NewBuffer(body)
	self.R.Body = ioutil.NopCloser(bf)
	return body
}

func (self *HTTPRequest) Write(s string) {
	self.W.Write([]byte(s))
}

func (self *HTTPRequest) Render(name string, data interface{}) {
	Template[name].Execute(self.W, data)
}

func (self *HTTPRequest) GET() {
	http.Error(self.W, "Method Not Allowed", 405)
}

func (self *HTTPRequest) POST() {
	http.Error(self.W, "Method Not Allowed", 405)
}

func (self *HTTPRequest) HEAD() {
	http.Error(self.W, "Method Not Allowed", 405)
}

func (self *HTTPRequest) OPTION() {
	http.Error(self.W, "Method Not Allowed", 405)
}

func (self *HTTPRequest) PUT() {
	http.Error(self.W, "Method Not Allowed", 405)
}
<commit_msg>fix http method<commit_after>package mango

import (
	"net/http"
	"fmt"
	"strings"
	"io/ioutil"
	"bytes"
	"mime/multipart"
	"net/url"
)

type HTTPRequest struct {
	Args []string		//route params
	Method string		//HTTP method
	RemoteAddr string	//ip
	Path string
	Host string
	Params url.Values	//GET or POST params
	Body []byte			//Content-Type=application/octet-stream, Body=http body
	Files map[string][]*multipart.FileHeader		//Content-Type=multipart/form-data

	W http.ResponseWriter
	R *http.Request
}

func (self *HTTPRequest) Init(w http.ResponseWriter, r *http.Request, a []string) {
	self.W = w
	self.R = r
	self.Method = r.Method
	self.Path = r.URL.Path
	self.Args = a
	self.Host = r.Host

	ip := r.Header.Get("X-Forwarded-For")
	if ip == "" {
		ip = r.RemoteAddr
	}
	self.RemoteAddr = strings.Split(ip, ":")[0]
	self.parseBodyArguments()
}

func (self *HTTPRequest) UserAgent() string {
	return self.R.Header.Get("User-Agent")
}

func (self *HTTPRequest) Header(s string) string {
	return self.R.Header.Get(s)
}

func (self *HTTPRequest) SetHeader(k, v string) {
	self.W.Header().Set(k, v)
}

func (self *HTTPRequest) SetStatus(st int) {
	self.W.WriteHeader(st)
}

func (self *HTTPRequest) Cookie(k string) string {
	ck, err := self.R.Cookie(k)
	if err != nil {
		return ""
	}
	return ck.Value
}

var cks = strings.NewReplacer("\n", "-", "\r", "-")
var cvs = strings.NewReplacer("\n", " ", "\r", " ", ";", " ")

func (self *HTTPRequest) SetCookie(k, v string, params ...interface{}) {
	var b bytes.Buffer
	fmt.Fprintf(&b, "%s=%s", cks.Replace(k), cvs.Replace(v))
	if len(params) > 0 {
		switch v := params[0].(type) {
		case int:
			if v > 0 {
				fmt.Fprintf(&b, "; Max-Age=%d", v)
			} else if v < 0 {
				fmt.Fprintf(&b, "; Max-Age=0")
			}
		case int64:
			if v > 0 {
				fmt.Fprintf(&b, "; Max-Age=%d", v)
			} else if v < 0 {
				fmt.Fprintf(&b, "; Max-Age=0")
			}
		case int32:
			if v > 0 {
				fmt.Fprintf(&b, "; Max-Age=%d", v)
			} else if v < 0 {
				fmt.Fprintf(&b, "; Max-Age=0")
			}
		}
	}
	// Path, Domain, Secure, HttpOnly

	// default "/"
	if len(params) > 1 {
		if v, ok := params[1].(string); ok && len(v) > 0 {
			fmt.Fprintf(&b, "; Path=%s", cvs.Replace(v))
		}
	} else {
		fmt.Fprintf(&b, "; Path=%s", "/")
	}

	// default empty
	if len(params) > 2 {
		if v, ok := params[2].(string); ok && len(v) > 0 {
			fmt.Fprintf(&b, "; Domain=%s", cvs.Replace(v))
		}
	}

	// default empty
	if len(params) > 3 {
		var secure bool
		switch v := params[3].(type) {
		case bool:
			secure = v
		default:
			if params[3] != nil {
				secure = true
			}
		}
		if secure {
			fmt.Fprintf(&b, "; Secure")
		}
	}

	// default false.
	httponly := false
	if len(params) > 4 {
		if v, ok := params[4].(bool); ok && v {
			// HttpOnly = true
			httponly = true
		}
	}

	if httponly {
		fmt.Fprintf(&b, "; HttpOnly")
	}

	self.W.Header().Add("Set-Cookie", b.String())
}

func (self *HTTPRequest) Scheme() string {
	if self.R.URL.Scheme != "" {
		return self.R.URL.Scheme
	} else if self.R.TLS == nil {
		return "http"
	} else {
		return "https"
	}
}

func (self *HTTPRequest) Redirect(newloc string) {
	self.SetHeader("Location", newloc)
	self.SetStatus(303)
}

func (self *HTTPRequest) Query() map[string][]string {
	return self.R.URL.Query()
}

func (self *HTTPRequest) QueryString() string {
	if s := strings.Split(self.R.RequestURI, "?"); len(s) > 1 {
		return s[1]
	}else{
		return ""
	}
}

func (self *HTTPRequest) parseBodyArguments() {
	qs := self.R.URL.Query()
	var pt url.Values
	ct := self.Header("Content-Type")
	if strings.HasPrefix(ct, "application/x-www-form-urlencoded") {
		if err := self.R.ParseForm(); err != nil {
			fmt.Println("Error parsing request body:", err)
		} else {
			pt = self.R.Form
		}
	} else if strings.HasPrefix(ct, "multipart/form-data") {
		// TODO: Extract the multipart form param so app can set it.
		if err := self.R.ParseMultipartForm(32 << 20 /* 32 MB */); err != nil {
			fmt.Println("Error parsing request body:", err)
		} else {
			self.Files = self.R.MultipartForm.File
		}
	} else if strings.HasPrefix(ct, "application/octet-stream") {
		self.Body = self.Data()
	}

	num := len(qs) + len(pt)

	if num == 0 {
		self.Params = make(url.Values, 0)
		return
	}
	values := make(url.Values, num)
	for k, v := range qs {
		values[k] = append(values[k], v...)
	}
	for k, v := range pt {
		values[k] = append(values[k], v...)
	}
	self.Params = values
}

func (self *HTTPRequest) Data() []byte {
	body, _ := ioutil.ReadAll(self.R.Body)
	self.R.Body.Close()
	bf := bytes.NewBuffer(body)
	self.R.Body = ioutil.NopCloser(bf)
	return body
}

func (self *HTTPRequest) Write(s string) {
	self.W.Write([]byte(s))
}

func (self *HTTPRequest) Render(name string, data interface{}) {
	Template[name].Execute(self.W, data)
}

func (self *HTTPRequest) GET() {
	http.Error(self.W, "Method GET Not Allowed", 405)
}

func (self *HTTPRequest) POST() {
	http.Error(self.W, "Method POST Not Allowed", 405)
}

func (self *HTTPRequest) HEAD() {
	http.Error(self.W, "Method HEAD Not Allowed", 405)
}

func (self *HTTPRequest) OPTION() {
	http.Error(self.W, "Method OPTION Not Allowed", 405)
}

func (self *HTTPRequest) PUT() {
	http.Error(self.W, "Method PUT Not Allowed", 405)
}

func (self *HTTPRequest) DELETE() {
	http.Error(self.W, "Method DELETE Not Allowed", 405)
}
<|endoftext|>


<commit_before>package main

// Slack outgoing webhooks are handled here. Requests come in and are run through
// the markov chain to generate a response, which is sent back to Slack.
//
// Create an outgoing webhook in your Slack here:
// https://my.slack.com/services/new/outgoing-webhook

import (
	"encoding/json"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"
//	"fmt"
)

type whr struct {
	Token		 string `json:"token"`
	Team_ID	 string `json:"team_id"`
	ChannelID  string `json:"channel_id"`
	ChannelName string `json:"channel_name"`
	TS				string `json:"timestamp"`
	UserID		string `json:"user_id"`
	Username string `json:"user_name"`
	Text     string `json:"text"`
	Trigger  string `json:"trigger_word"`
}

func init() {
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
	//	incomingText := r.PostFormValue("text")

			err := r.ParseForm()
			if err != nil {
				panic(err)
			}
			//log.Println("r.Form", r.Form)

			txt := r.Form["text"]
			usr := r.Form["user_name"]
			incomingText := txt[0]
			rUsername := usr[0]
		if incomingText != "" && rUsername != "" && rUsername != "slackbot" {
			text := parseText(incomingText)
			log.Printf("Handling incoming request: %s from %s", text,rUsername )

			if text != "" {
				markovChain.Write(text)
			}

			go func() {
				markovChain.Save(stateFile)
			}()

			if rand.Intn(100) <= responseChance || strings.HasPrefix(text, botUsername) || strings.Contains(text,"Dingus") {
				var response whr
				response.Username = botUsername
				response.Text = markovChain.Generate(numWords)
				log.Printf("Sending response: %s", response.Text)

				b, err := json.Marshal(response)
				if err != nil {
					log.Fatal(err)
				}

				time.Sleep(5 * time.Second)
				w.Write(b)
				r.Body.Close()
			}
		}
	})
}

func StartServer(port int) {
	log.Printf("Starting HTTP server on %d", port)
	err := http.ListenAndServe(":"+strconv.Itoa(port), nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<commit_msg>dingus now compliments when given Dingus: compliment me as a command<commit_after>package main

// Slack outgoing webhooks are handled here. Requests come in and are run through
// the markov chain to generate a response, which is sent back to Slack.
//
// Create an outgoing webhook in your Slack here:
// https://my.slack.com/services/new/outgoing-webhook

import (
	"encoding/json"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"
//	"fmt"
	"io/ioutil"
)

type whr struct {
	Token		 string `json:"token"`
	Team_ID	 string `json:"team_id"`
	ChannelID  string `json:"channel_id"`
	ChannelName string `json:"channel_name"`
	TS				string `json:"timestamp"`
	UserID		string `json:"user_id"`
	Username string `json:"user_name"`
	Text     string `json:"text"`
	Trigger  string `json:"trigger_word"`
}

func init() {
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
	//	incomingText := r.PostFormValue("text")

			err := r.ParseForm()
			if err != nil {
				panic(err)
			}
			//log.Println("r.Form", r.Form)

			txt := r.Form["text"]
			usr := r.Form["user_name"]
			incomingText := txt[0]
			rUsername := usr[0]
		if incomingText != "" && rUsername != "" && rUsername != "slackbot" {
			text := parseText(incomingText)
			log.Printf("Handling incoming request: %s from %s", text,rUsername )

			if text != "" {
				markovChain.Write(text)
			}

			go func() {
				markovChain.Save(stateFile)
			}()

			if rand.Intn(100) <= responseChance || strings.HasPrefix(text, botUsername) || strings.Contains(text,"Dingus") {
				var response whr
				response.Username = botUsername

				response.Text = markovChain.Generate(numWords)
				if strings.HasPrefix(text,"Dingus: compliment me"){
					url := "http://127.0.0.1:56735"
					comp, err := http.Get(url)
					if err != nil {
						log.Fatal(err)
					}
					defer comp.Body.Close()
					compData, err := ioutil.ReadAll(comp.Body)
					if err != nil {
						log.Fatal(err)
					}
					compString := string(compData)
					response.Text = rUsername + ": " + compString
				}
				log.Printf("Sending response: %s", response.Text)

				b, err := json.Marshal(response)
				if err != nil {
					log.Fatal(err)
				}

				time.Sleep(5 * time.Second)
				w.Write(b)
				r.Body.Close()
			}
		}
	})
}

func StartServer(port int) {
	log.Printf("Starting HTTP server on %d", port)
	err := http.ListenAndServe(":"+strconv.Itoa(port), nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<|endoftext|>


<commit_before>package xff

import (
	"log"
	"net"
	"net/http"
	"os"
	"strings"
)

// list of private subnets
var privateMasks, _ = toMasks([]string{
	"10.0.0.0/8",
	"172.16.0.0/12",
	"192.168.0.0/16",
	"fc00::/7",
})

// converts a list of subnets' string to a list of net.IPNet.
func toMasks(ips []string) (masks []net.IPNet, err error) {
	for _, cidr := range ips {
		var network *net.IPNet
		_, network, err = net.ParseCIDR(cidr)
		if err != nil {
			return
		}
		masks = append(masks, *network)
	}
	return
}

// checks if a net.IP is in a list of net.IPNet
func ipInMasks(ip net.IP, masks []net.IPNet) bool {
	for _, mask := range masks {
		if mask.Contains(ip) {
			return true
		}
	}
	return false
}

// IsPublicIP returns true if the given IP can be routed on the Internet.
func IsPublicIP(ip net.IP) bool {
	if !ip.IsGlobalUnicast() {
		return false
	}
	return !ipInMasks(ip, privateMasks)
}

// Parse parses the value of the X-Forwarded-For Header and returns the IP address.
func Parse(ipList string) string {
	for _, ip := range strings.Split(ipList, ",") {
		ip = strings.TrimSpace(ip)
		if IP := net.ParseIP(ip); IP != nil && IsPublicIP(IP) {
			return ip
		}
	}
	return ""
}

// GetRemoteAddr parses the given request, resolves the X-Forwarded-For header
// and returns the resolved remote address.
func GetRemoteAddr(r *http.Request) string {
	return GetRemoteAddrIfAllowed(r, func(sip string) bool { return true })
}

// GetRemoteAddrIfAllowed parses the given request, resolves the X-Forwarded-For header
// and returns the resolved remote address if allowed.
func GetRemoteAddrIfAllowed(r *http.Request, allowed func(sip string) bool) string {
	if xffh := r.Header.Get("X-Forwarded-For"); xffh != "" {
		if sip, sport, err := net.SplitHostPort(r.RemoteAddr); err == nil && sip != "" {
			if allowed(sip) {
				if xip := Parse(xffh); xip != "" {
					return net.JoinHostPort(xip, sport)
				}
			}
		}
	}
	return r.RemoteAddr
}

// Options is a configuration container to setup the XFF middleware.
type Options struct {
	// AllowedSubnets is a list of Subnets from which we will accept the
	// X-Forwarded-For header.
	// If this list is empty we will accept every Subnets (default).
	AllowedSubnets []string
	// Debugging flag adds additional output to debug server side XFF issues.
	Debug bool
}

// XFF http handler
type XFF struct {
	// Debug logger
	Log *log.Logger
	// Set to true if all IPs or Subnets are allowed.
	allowAll bool
	// List of IP subnets that are allowed.
	allowedMasks []net.IPNet
}

// New creates a new XFF handler with the provided options.
func New(options Options) (*XFF, error) {
	allowedMasks, err := toMasks(options.AllowedSubnets)
	if err != nil {
		return nil, err
	}
	xff := &XFF{
		allowAll:     len(options.AllowedSubnets) == 0,
		allowedMasks: allowedMasks,
	}
	if options.Debug {
		xff.Log = log.New(os.Stdout, "[xff] ", log.LstdFlags)
	}
	return xff, nil
}

// Default creates a new XFF handler with default options.
func Default() (*XFF, error) {
	return New(Options{})
}

// Handler updates RemoteAdd from X-Fowarded-For Headers.
func (xff *XFF) Handler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		r.RemoteAddr = GetRemoteAddrIfAllowed(r, xff.allowed)
		h.ServeHTTP(w, r)
	})
}

// Negroni compatible interface
func (xff *XFF) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	r.RemoteAddr = GetRemoteAddrIfAllowed(r, xff.allowed)
	next(w, r)
}

// HandlerFunc provides Martini compatible handler
func (xff *XFF) HandlerFunc(w http.ResponseWriter, r *http.Request) {
	r.RemoteAddr = GetRemoteAddrIfAllowed(r, xff.allowed)
}

// checks that the IP is allowed.
func (xff *XFF) allowed(sip string) bool {
	if xff.allowAll {
		return true
	} else if ip := net.ParseIP(sip); ip != nil && ipInMasks(ip, xff.allowedMasks) {
		return true
	}
	return false
}

// convenience method. checks if debugging is turned on before printing
func (xff *XFF) logf(format string, a ...interface{}) {
	if xff.Log != nil {
		xff.Log.Printf(format, a...)
	}
}
<commit_msg>add loopback range to default private masks<commit_after>package xff

import (
	"log"
	"net"
	"net/http"
	"os"
	"strings"
)

// list of private subnets
var privateMasks, _ = toMasks([]string{
	"127.0.0.0/8",
	"10.0.0.0/8",
	"172.16.0.0/12",
	"192.168.0.0/16",
	"fc00::/7",
})

// converts a list of subnets' string to a list of net.IPNet.
func toMasks(ips []string) (masks []net.IPNet, err error) {
	for _, cidr := range ips {
		var network *net.IPNet
		_, network, err = net.ParseCIDR(cidr)
		if err != nil {
			return
		}
		masks = append(masks, *network)
	}
	return
}

// checks if a net.IP is in a list of net.IPNet
func ipInMasks(ip net.IP, masks []net.IPNet) bool {
	for _, mask := range masks {
		if mask.Contains(ip) {
			return true
		}
	}
	return false
}

// IsPublicIP returns true if the given IP can be routed on the Internet.
func IsPublicIP(ip net.IP) bool {
	if !ip.IsGlobalUnicast() {
		return false
	}
	return !ipInMasks(ip, privateMasks)
}

// Parse parses the value of the X-Forwarded-For Header and returns the IP address.
func Parse(ipList string) string {
	for _, ip := range strings.Split(ipList, ",") {
		ip = strings.TrimSpace(ip)
		if IP := net.ParseIP(ip); IP != nil && IsPublicIP(IP) {
			return ip
		}
	}
	return ""
}

// GetRemoteAddr parses the given request, resolves the X-Forwarded-For header
// and returns the resolved remote address.
func GetRemoteAddr(r *http.Request) string {
	return GetRemoteAddrIfAllowed(r, func(sip string) bool { return true })
}

// GetRemoteAddrIfAllowed parses the given request, resolves the X-Forwarded-For header
// and returns the resolved remote address if allowed.
func GetRemoteAddrIfAllowed(r *http.Request, allowed func(sip string) bool) string {
	if xffh := r.Header.Get("X-Forwarded-For"); xffh != "" {
		if sip, sport, err := net.SplitHostPort(r.RemoteAddr); err == nil && sip != "" {
			if allowed(sip) {
				if xip := Parse(xffh); xip != "" {
					return net.JoinHostPort(xip, sport)
				}
			}
		}
	}
	return r.RemoteAddr
}

// Options is a configuration container to setup the XFF middleware.
type Options struct {
	// AllowedSubnets is a list of Subnets from which we will accept the
	// X-Forwarded-For header.
	// If this list is empty we will accept every Subnets (default).
	AllowedSubnets []string
	// Debugging flag adds additional output to debug server side XFF issues.
	Debug bool
}

// XFF http handler
type XFF struct {
	// Debug logger
	Log *log.Logger
	// Set to true if all IPs or Subnets are allowed.
	allowAll bool
	// List of IP subnets that are allowed.
	allowedMasks []net.IPNet
}

// New creates a new XFF handler with the provided options.
func New(options Options) (*XFF, error) {
	allowedMasks, err := toMasks(options.AllowedSubnets)
	if err != nil {
		return nil, err
	}
	xff := &XFF{
		allowAll:     len(options.AllowedSubnets) == 0,
		allowedMasks: allowedMasks,
	}
	if options.Debug {
		xff.Log = log.New(os.Stdout, "[xff] ", log.LstdFlags)
	}
	return xff, nil
}

// Default creates a new XFF handler with default options.
func Default() (*XFF, error) {
	return New(Options{})
}

// Handler updates RemoteAdd from X-Fowarded-For Headers.
func (xff *XFF) Handler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		r.RemoteAddr = GetRemoteAddrIfAllowed(r, xff.allowed)
		h.ServeHTTP(w, r)
	})
}

// Negroni compatible interface
func (xff *XFF) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	r.RemoteAddr = GetRemoteAddrIfAllowed(r, xff.allowed)
	next(w, r)
}

// HandlerFunc provides Martini compatible handler
func (xff *XFF) HandlerFunc(w http.ResponseWriter, r *http.Request) {
	r.RemoteAddr = GetRemoteAddrIfAllowed(r, xff.allowed)
}

// checks that the IP is allowed.
func (xff *XFF) allowed(sip string) bool {
	if xff.allowAll {
		return true
	} else if ip := net.ParseIP(sip); ip != nil && ipInMasks(ip, xff.allowedMasks) {
		return true
	}
	return false
}

// convenience method. checks if debugging is turned on before printing
func (xff *XFF) logf(format string, a ...interface{}) {
	if xff.Log != nil {
		xff.Log.Printf(format, a...)
	}
}
<|endoftext|>


<commit_before>package archiver

import (
	"archive/zip"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"path"
	"strings"

	"github.com/dsnet/compress/bzip2"
	"github.com/klauspost/compress/zstd"
	"github.com/ulikunitz/xz"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/encoding/japanese"
	"golang.org/x/text/encoding/korean"
	"golang.org/x/text/encoding/simplifiedchinese"
	"golang.org/x/text/encoding/traditionalchinese"
	"golang.org/x/text/encoding/unicode"
)

func init() {
	RegisterFormat(Zip{})

	// TODO: What about custom flate levels too
	zip.RegisterCompressor(ZipMethodBzip2, func(out io.Writer) (io.WriteCloser, error) {
		return bzip2.NewWriter(out, &bzip2.WriterConfig{ /*TODO: Level: z.CompressionLevel*/ })
	})
	zip.RegisterCompressor(ZipMethodZstd, func(out io.Writer) (io.WriteCloser, error) {
		return zstd.NewWriter(out)
	})
	zip.RegisterCompressor(ZipMethodXz, func(out io.Writer) (io.WriteCloser, error) {
		return xz.NewWriter(out)
	})

	zip.RegisterDecompressor(ZipMethodBzip2, func(r io.Reader) io.ReadCloser {
		bz2r, err := bzip2.NewReader(r, nil)
		if err != nil {
			return nil
		}
		return bz2r
	})
	zip.RegisterDecompressor(ZipMethodZstd, func(r io.Reader) io.ReadCloser {
		zr, err := zstd.NewReader(r)
		if err != nil {
			return nil
		}
		return zr.IOReadCloser()
	})
	zip.RegisterDecompressor(ZipMethodXz, func(r io.Reader) io.ReadCloser {
		xr, err := xz.NewReader(r)
		if err != nil {
			return nil
		}
		return io.NopCloser(xr)
	})
}

type Zip struct {
	// Only compress files which are not already in a
	// compressed format (determined simply by examining
	// file extension).
	SelectiveCompression bool

	// The method or algorithm for compressing stored files.
	Compression uint16

	// If true, errors encountered during reading or writing
	// a file within an archive will be logged and the
	// operation will continue on remaining files.
	ContinueOnError bool

	// For files in zip archives that do not have UTF-8
	// encoded filenames and comments, specify the character
	// encoding here.
	TextEncoding string
}

func (z Zip) Name() string { return ".zip" }

func (z Zip) Match(filename string, stream io.Reader) (MatchResult, error) {
	var mr MatchResult

	// match filename
	if strings.Contains(strings.ToLower(filename), z.Name()) {
		mr.ByName = true
	}

	// match file header
	buf := make([]byte, len(zipHeader))
	if _, err := io.ReadFull(stream, buf); err != nil {
		return mr, err
	}
	mr.ByStream = bytes.Equal(buf, zipHeader)

	return mr, nil
}

func (z Zip) Archive(ctx context.Context, output io.Writer, files []File) error {
	if ctx == nil {
		ctx = context.Background()
	}

	zw := zip.NewWriter(output)
	defer zw.Close()

	for i, file := range files {
		if err := ctx.Err(); err != nil {
			return err // honor context cancellation
		}

		hdr, err := zip.FileInfoHeader(file)
		if err != nil {
			return fmt.Errorf("getting info for file %d: %s: %w", i, file.Name(), err)
		}

		// customize header based on file properties
		if file.IsDir() {
			hdr.Name += "/" // required - strangely no mention of this in zip spec? but is in godoc...
			hdr.Method = zip.Store
		} else if z.SelectiveCompression {
			// only enable compression on compressable files
			ext := strings.ToLower(path.Ext(hdr.Name))
			if _, ok := compressedFormats[ext]; ok {
				hdr.Method = zip.Store
			} else {
				hdr.Method = z.Compression
			}
		}

		w, err := zw.CreateHeader(hdr)
		if err != nil {
			return fmt.Errorf("creating header for file %d: %s: %w", i, file.Name(), err)
		}

		// directories have no file body
		if file.IsDir() {
			continue
		}
		if err := openAndCopyFile(file, w); err != nil {
			return fmt.Errorf("writing file %d: %s: %w", i, file.Name(), err)
		}
	}

	return nil
}

func (z Zip) Extract(ctx context.Context, sourceArchive io.Reader, pathsInArchive []string, handleFile FileHandler) error {
	if ctx == nil {
		ctx = context.Background()
	}

	sra, ok := sourceArchive.(seekReaderAt)
	if !ok {
		return fmt.Errorf("input type must be an io.ReaderAt and io.Seeker because of zip format constraints")
	}

	size, err := streamSizeBySeeking(sra)
	if err != nil {
		return fmt.Errorf("determining stream size: %w", err)
	}

	zr, err := zip.NewReader(sra, size)
	if err != nil {
		return err
	}

	// important to initialize to non-nil, empty value due to how fileIsIncluded works
	skipDirs := skipList{}

	for i, f := range zr.File {
		if err := ctx.Err(); err != nil {
			return err // honor context cancellation
		}
		if !fileIsIncluded(pathsInArchive, f.Name) {
			continue
		}
		if fileIsIncluded(skipDirs, f.Name) {
			continue
		}

		// ensure filename and comment are UTF-8 encoded (issue #147)
		z.decodeText(&f.FileHeader)

		file := File{
			FileInfo:      f.FileInfo(),
			Header:        f.FileHeader,
			NameInArchive: f.Name,
			Open:          func() (io.ReadCloser, error) { return f.Open() },
		}

		err := handleFile(ctx, file)
		if errors.Is(err, fs.SkipDir) {
			// if a directory, skip this path; if a file, skip the folder path
			dirPath := f.Name
			if !file.IsDir() {
				dirPath = path.Dir(f.Name)
			}
			skipDirs.add(dirPath)
		} else if err != nil {
			return fmt.Errorf("handling file %d: %s: %w", i, f.Name, err)
		}
	}

	return nil
}

// decodeText decodes the name and comment fields from hdr into UTF-8.
// It is a no-op if the text is already UTF-8 encoded or if z.TextEncoding
// is not specified.
func (z Zip) decodeText(hdr *zip.FileHeader) {
	if hdr.NonUTF8 && z.TextEncoding != "" {
		filename, err := decodeText(hdr.Name, z.TextEncoding)
		if err == nil {
			hdr.Name = filename
		}
		if hdr.Comment != "" {
			comment, err := decodeText(hdr.Comment, z.TextEncoding)
			if err == nil {
				hdr.Comment = comment
			}
		}
	}
}

type seekReaderAt interface {
	io.ReaderAt
	io.Seeker
}

func streamSizeBySeeking(s io.Seeker) (int64, error) {
	currentPosition, err := s.Seek(0, io.SeekCurrent)
	if err != nil {
		return 0, fmt.Errorf("getting current offset: %w", err)
	}
	maxPosition, err := s.Seek(0, io.SeekEnd)
	if err != nil {
		return 0, fmt.Errorf("fast-forwarding to end: %w", err)
	}
	_, err = s.Seek(currentPosition, io.SeekStart)
	if err != nil {
		return 0, fmt.Errorf("returning to prior offset %d: %w", currentPosition, err)
	}
	return maxPosition, nil
}

// Additional compression methods not offered by archive/zip.
// See https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT section 4.4.5.
const (
	ZipMethodBzip2 = 12
	// TODO: LZMA: Disabled - because 7z isn't able to unpack ZIP+LZMA ZIP+LZMA2 archives made this way - and vice versa.
	// ZipMethodLzma     = 14
	ZipMethodZstd = 93
	ZipMethodXz   = 95
)

// compressedFormats is a (non-exhaustive) set of lowercased
// file extensions for formats that are typically already
// compressed. Compressing files that are already compressed
// is inefficient, so use this set of extensions to avoid that.
var compressedFormats = map[string]struct{}{
	".7z":   {},
	".avi":  {},
	".br":   {},
	".bz2":  {},
	".cab":  {},
	".docx": {},
	".gif":  {},
	".gz":   {},
	".jar":  {},
	".jpeg": {},
	".jpg":  {},
	".lz":   {},
	".lz4":  {},
	".lzma": {},
	".m4v":  {},
	".mov":  {},
	".mp3":  {},
	".mp4":  {},
	".mpeg": {},
	".mpg":  {},
	".png":  {},
	".pptx": {},
	".rar":  {},
	".sz":   {},
	".tbz2": {},
	".tgz":  {},
	".tsz":  {},
	".txz":  {},
	".xlsx": {},
	".xz":   {},
	".zip":  {},
	".zipx": {},
}

var encodings = map[string]encoding.Encoding{
	"ibm866":            charmap.CodePage866,
	"iso8859_2":         charmap.ISO8859_2,
	"iso8859_3":         charmap.ISO8859_3,
	"iso8859_4":         charmap.ISO8859_4,
	"iso8859_5":         charmap.ISO8859_5,
	"iso8859_6":         charmap.ISO8859_6,
	"iso8859_7":         charmap.ISO8859_7,
	"iso8859_8":         charmap.ISO8859_8,
	"iso8859_8I":        charmap.ISO8859_8I,
	"iso8859_10":        charmap.ISO8859_10,
	"iso8859_13":        charmap.ISO8859_13,
	"iso8859_14":        charmap.ISO8859_14,
	"iso8859_15":        charmap.ISO8859_15,
	"iso8859_16":        charmap.ISO8859_16,
	"koi8r":             charmap.KOI8R,
	"koi8u":             charmap.KOI8U,
	"macintosh":         charmap.Macintosh,
	"windows874":        charmap.Windows874,
	"windows1250":       charmap.Windows1250,
	"windows1251":       charmap.Windows1251,
	"windows1252":       charmap.Windows1252,
	"windows1253":       charmap.Windows1253,
	"windows1254":       charmap.Windows1254,
	"windows1255":       charmap.Windows1255,
	"windows1256":       charmap.Windows1256,
	"windows1257":       charmap.Windows1257,
	"windows1258":       charmap.Windows1258,
	"macintoshcyrillic": charmap.MacintoshCyrillic,
	"gbk":               simplifiedchinese.GBK,
	"gb18030":           simplifiedchinese.GB18030,
	"big5":              traditionalchinese.Big5,
	"eucjp":             japanese.EUCJP,
	"iso2022jp":         japanese.ISO2022JP,
	"shiftjis":          japanese.ShiftJIS,
	"euckr":             korean.EUCKR,
	"utf16be":           unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM),
	"utf16le":           unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM),
}

// decodeText returns UTF-8 encoded text from the given charset.
// Thanks to @zxdvd for contributing non-UTF-8 encoding logic in #149.
func decodeText(input, charset string) (string, error) {
	if enc, ok := encodings[charset]; ok {
		return enc.NewDecoder().String(input)
	}
	return "", fmt.Errorf("unrecognized charset %s", charset)
}

var zipHeader = []byte("PK\x03\x04") // TODO: headers of empty zip files might end with 0x05,0x06 or 0x06,0x06 instead of 0x03,0x04
<commit_msg>zip: Document interface requirements for Extract<commit_after>package archiver

import (
	"archive/zip"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"path"
	"strings"

	"github.com/dsnet/compress/bzip2"
	"github.com/klauspost/compress/zstd"
	"github.com/ulikunitz/xz"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/encoding/japanese"
	"golang.org/x/text/encoding/korean"
	"golang.org/x/text/encoding/simplifiedchinese"
	"golang.org/x/text/encoding/traditionalchinese"
	"golang.org/x/text/encoding/unicode"
)

func init() {
	RegisterFormat(Zip{})

	// TODO: What about custom flate levels too
	zip.RegisterCompressor(ZipMethodBzip2, func(out io.Writer) (io.WriteCloser, error) {
		return bzip2.NewWriter(out, &bzip2.WriterConfig{ /*TODO: Level: z.CompressionLevel*/ })
	})
	zip.RegisterCompressor(ZipMethodZstd, func(out io.Writer) (io.WriteCloser, error) {
		return zstd.NewWriter(out)
	})
	zip.RegisterCompressor(ZipMethodXz, func(out io.Writer) (io.WriteCloser, error) {
		return xz.NewWriter(out)
	})

	zip.RegisterDecompressor(ZipMethodBzip2, func(r io.Reader) io.ReadCloser {
		bz2r, err := bzip2.NewReader(r, nil)
		if err != nil {
			return nil
		}
		return bz2r
	})
	zip.RegisterDecompressor(ZipMethodZstd, func(r io.Reader) io.ReadCloser {
		zr, err := zstd.NewReader(r)
		if err != nil {
			return nil
		}
		return zr.IOReadCloser()
	})
	zip.RegisterDecompressor(ZipMethodXz, func(r io.Reader) io.ReadCloser {
		xr, err := xz.NewReader(r)
		if err != nil {
			return nil
		}
		return io.NopCloser(xr)
	})
}

type Zip struct {
	// Only compress files which are not already in a
	// compressed format (determined simply by examining
	// file extension).
	SelectiveCompression bool

	// The method or algorithm for compressing stored files.
	Compression uint16

	// If true, errors encountered during reading or writing
	// a file within an archive will be logged and the
	// operation will continue on remaining files.
	ContinueOnError bool

	// For files in zip archives that do not have UTF-8
	// encoded filenames and comments, specify the character
	// encoding here.
	TextEncoding string
}

func (z Zip) Name() string { return ".zip" }

func (z Zip) Match(filename string, stream io.Reader) (MatchResult, error) {
	var mr MatchResult

	// match filename
	if strings.Contains(strings.ToLower(filename), z.Name()) {
		mr.ByName = true
	}

	// match file header
	buf := make([]byte, len(zipHeader))
	if _, err := io.ReadFull(stream, buf); err != nil {
		return mr, err
	}
	mr.ByStream = bytes.Equal(buf, zipHeader)

	return mr, nil
}

func (z Zip) Archive(ctx context.Context, output io.Writer, files []File) error {
	if ctx == nil {
		ctx = context.Background()
	}

	zw := zip.NewWriter(output)
	defer zw.Close()

	for i, file := range files {
		if err := ctx.Err(); err != nil {
			return err // honor context cancellation
		}

		hdr, err := zip.FileInfoHeader(file)
		if err != nil {
			return fmt.Errorf("getting info for file %d: %s: %w", i, file.Name(), err)
		}

		// customize header based on file properties
		if file.IsDir() {
			hdr.Name += "/" // required - strangely no mention of this in zip spec? but is in godoc...
			hdr.Method = zip.Store
		} else if z.SelectiveCompression {
			// only enable compression on compressable files
			ext := strings.ToLower(path.Ext(hdr.Name))
			if _, ok := compressedFormats[ext]; ok {
				hdr.Method = zip.Store
			} else {
				hdr.Method = z.Compression
			}
		}

		w, err := zw.CreateHeader(hdr)
		if err != nil {
			return fmt.Errorf("creating header for file %d: %s: %w", i, file.Name(), err)
		}

		// directories have no file body
		if file.IsDir() {
			continue
		}
		if err := openAndCopyFile(file, w); err != nil {
			return fmt.Errorf("writing file %d: %s: %w", i, file.Name(), err)
		}
	}

	return nil
}

// Extract extracts files from z, implementing the Extractor interface. Uniquely, however,
// sourceArchive must be an io.ReaderAt and io.Seeker, which are oddly disjoint interfaces
// from io.Reader which is what the method signature requires. We chose this signature for
// the interface because we figure you can Read() from anything you can ReadAt() or Seek()
// with. Due to the nature of the zip archive format, if sourceArchive is not an io.Seeker
// and io.ReaderAt, an error is returned.
func (z Zip) Extract(ctx context.Context, sourceArchive io.Reader, pathsInArchive []string, handleFile FileHandler) error {
	if ctx == nil {
		ctx = context.Background()
	}

	sra, ok := sourceArchive.(seekReaderAt)
	if !ok {
		return fmt.Errorf("input type must be an io.ReaderAt and io.Seeker because of zip format constraints")
	}

	size, err := streamSizeBySeeking(sra)
	if err != nil {
		return fmt.Errorf("determining stream size: %w", err)
	}

	zr, err := zip.NewReader(sra, size)
	if err != nil {
		return err
	}

	// important to initialize to non-nil, empty value due to how fileIsIncluded works
	skipDirs := skipList{}

	for i, f := range zr.File {
		if err := ctx.Err(); err != nil {
			return err // honor context cancellation
		}
		if !fileIsIncluded(pathsInArchive, f.Name) {
			continue
		}
		if fileIsIncluded(skipDirs, f.Name) {
			continue
		}

		// ensure filename and comment are UTF-8 encoded (issue #147)
		z.decodeText(&f.FileHeader)

		file := File{
			FileInfo:      f.FileInfo(),
			Header:        f.FileHeader,
			NameInArchive: f.Name,
			Open:          func() (io.ReadCloser, error) { return f.Open() },
		}

		err := handleFile(ctx, file)
		if errors.Is(err, fs.SkipDir) {
			// if a directory, skip this path; if a file, skip the folder path
			dirPath := f.Name
			if !file.IsDir() {
				dirPath = path.Dir(f.Name)
			}
			skipDirs.add(dirPath)
		} else if err != nil {
			return fmt.Errorf("handling file %d: %s: %w", i, f.Name, err)
		}
	}

	return nil
}

// decodeText decodes the name and comment fields from hdr into UTF-8.
// It is a no-op if the text is already UTF-8 encoded or if z.TextEncoding
// is not specified.
func (z Zip) decodeText(hdr *zip.FileHeader) {
	if hdr.NonUTF8 && z.TextEncoding != "" {
		filename, err := decodeText(hdr.Name, z.TextEncoding)
		if err == nil {
			hdr.Name = filename
		}
		if hdr.Comment != "" {
			comment, err := decodeText(hdr.Comment, z.TextEncoding)
			if err == nil {
				hdr.Comment = comment
			}
		}
	}
}

type seekReaderAt interface {
	io.ReaderAt
	io.Seeker
}

func streamSizeBySeeking(s io.Seeker) (int64, error) {
	currentPosition, err := s.Seek(0, io.SeekCurrent)
	if err != nil {
		return 0, fmt.Errorf("getting current offset: %w", err)
	}
	maxPosition, err := s.Seek(0, io.SeekEnd)
	if err != nil {
		return 0, fmt.Errorf("fast-forwarding to end: %w", err)
	}
	_, err = s.Seek(currentPosition, io.SeekStart)
	if err != nil {
		return 0, fmt.Errorf("returning to prior offset %d: %w", currentPosition, err)
	}
	return maxPosition, nil
}

// Additional compression methods not offered by archive/zip.
// See https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT section 4.4.5.
const (
	ZipMethodBzip2 = 12
	// TODO: LZMA: Disabled - because 7z isn't able to unpack ZIP+LZMA ZIP+LZMA2 archives made this way - and vice versa.
	// ZipMethodLzma     = 14
	ZipMethodZstd = 93
	ZipMethodXz   = 95
)

// compressedFormats is a (non-exhaustive) set of lowercased
// file extensions for formats that are typically already
// compressed. Compressing files that are already compressed
// is inefficient, so use this set of extensions to avoid that.
var compressedFormats = map[string]struct{}{
	".7z":   {},
	".avi":  {},
	".br":   {},
	".bz2":  {},
	".cab":  {},
	".docx": {},
	".gif":  {},
	".gz":   {},
	".jar":  {},
	".jpeg": {},
	".jpg":  {},
	".lz":   {},
	".lz4":  {},
	".lzma": {},
	".m4v":  {},
	".mov":  {},
	".mp3":  {},
	".mp4":  {},
	".mpeg": {},
	".mpg":  {},
	".png":  {},
	".pptx": {},
	".rar":  {},
	".sz":   {},
	".tbz2": {},
	".tgz":  {},
	".tsz":  {},
	".txz":  {},
	".xlsx": {},
	".xz":   {},
	".zip":  {},
	".zipx": {},
}

var encodings = map[string]encoding.Encoding{
	"ibm866":            charmap.CodePage866,
	"iso8859_2":         charmap.ISO8859_2,
	"iso8859_3":         charmap.ISO8859_3,
	"iso8859_4":         charmap.ISO8859_4,
	"iso8859_5":         charmap.ISO8859_5,
	"iso8859_6":         charmap.ISO8859_6,
	"iso8859_7":         charmap.ISO8859_7,
	"iso8859_8":         charmap.ISO8859_8,
	"iso8859_8I":        charmap.ISO8859_8I,
	"iso8859_10":        charmap.ISO8859_10,
	"iso8859_13":        charmap.ISO8859_13,
	"iso8859_14":        charmap.ISO8859_14,
	"iso8859_15":        charmap.ISO8859_15,
	"iso8859_16":        charmap.ISO8859_16,
	"koi8r":             charmap.KOI8R,
	"koi8u":             charmap.KOI8U,
	"macintosh":         charmap.Macintosh,
	"windows874":        charmap.Windows874,
	"windows1250":       charmap.Windows1250,
	"windows1251":       charmap.Windows1251,
	"windows1252":       charmap.Windows1252,
	"windows1253":       charmap.Windows1253,
	"windows1254":       charmap.Windows1254,
	"windows1255":       charmap.Windows1255,
	"windows1256":       charmap.Windows1256,
	"windows1257":       charmap.Windows1257,
	"windows1258":       charmap.Windows1258,
	"macintoshcyrillic": charmap.MacintoshCyrillic,
	"gbk":               simplifiedchinese.GBK,
	"gb18030":           simplifiedchinese.GB18030,
	"big5":              traditionalchinese.Big5,
	"eucjp":             japanese.EUCJP,
	"iso2022jp":         japanese.ISO2022JP,
	"shiftjis":          japanese.ShiftJIS,
	"euckr":             korean.EUCKR,
	"utf16be":           unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM),
	"utf16le":           unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM),
}

// decodeText returns UTF-8 encoded text from the given charset.
// Thanks to @zxdvd for contributing non-UTF-8 encoding logic in #149.
func decodeText(input, charset string) (string, error) {
	if enc, ok := encodings[charset]; ok {
		return enc.NewDecoder().String(input)
	}
	return "", fmt.Errorf("unrecognized charset %s", charset)
}

var zipHeader = []byte("PK\x03\x04") // TODO: headers of empty zip files might end with 0x05,0x06 or 0x06,0x06 instead of 0x03,0x04
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"encoding/json"
	"errors"
	"log"
	"net/http"
	"strings"

	cb "github.com/couchbaselabs/go-couchbase"
)

var missingFile = errors.New("missing file")

type namedFile struct {
	name string
	meta fileMeta
	err  error
}

func pathGenerator(from string, ch chan *namedFile,
	errs chan error, quit chan bool) {
	defer close(ch)
	defer close(errs)

	parts := strings.Split(from, "/")

	viewRes := struct {
		Rows []struct {
			Key []string
			Id  string
		}
		Errors []cb.ViewError
	}{}

	limit := 1000
	startKey := parts
	done := false
	for !done {
		err := couchbase.ViewCustom("cbfs", "file_browse",
			map[string]interface{}{
				"stale":    false,
				"reduce":   false,
				"limit":    limit,
				"startkey": startKey,
			}, &viewRes)
		if err != nil {
			log.Printf("View error: %v", err)
			errs <- err
			return
		}
		for _, e := range viewRes.Errors {
			select {
			case errs <- e:
			case <-quit:
				return
			}
		}

		done = len(viewRes.Rows) < limit

		paths := map[string]*namedFile{}
		keys := []string{}

		for _, r := range viewRes.Rows {
			k := r.Id
			if !strings.HasPrefix(k, from) {
				done = true
				break
			}
			startKey = r.Key

			nf := namedFile{name: k, err: missingFile}

			paths[k] = &nf
			keys = append(keys, k)
		}

		for k, res := range couchbase.GetBulk(keys) {
			ob := paths[k]
			ob.err = json.Unmarshal(res.Body, &ob.meta)
			if ob.err != nil {
				log.Printf("Error unmarshaling %v: %v", k, ob.err)
			}
			select {
			case <-quit:
				return
			case ch <- ob:
				// We sent one.
			}
		}
	}
}

func doZipDocs(w http.ResponseWriter, req *http.Request,
	path string) {

	quit := make(chan bool)
	defer close(quit)
	ch := make(chan *namedFile)
	cherr := make(chan error)

	go pathGenerator(path, ch, cherr, quit)
	go logErrors("zip", cherr)

	w.Header().Set("Content-Type", "application/zip")
	w.WriteHeader(200)

	zw := zip.NewWriter(w)
	for nf := range ch {
		if nf.err != nil {
			log.Printf("Error on %v: %v", nf.name, nf.err)
			continue
		}

		fh := zip.FileHeader{
			Name:             nf.name,
			Method:           zip.Deflate,
			UncompressedSize: uint32(nf.meta.Length),
			Comment:          nf.meta.OID,
		}
		fh.SetModTime(nf.meta.Modified)

		zf, err := zw.CreateHeader(&fh)
		if err != nil {
			log.Printf("Error making zip file: %v", err)
			// Client will get a broken zip file
			return
		}

		err = copyBlob(zf, nf.meta.OID)
		if err != nil {
			log.Printf("Error copying blob for %v: %v",
				nf.name, err)
			// Client will get a broken zip file
			return
		}
	}
	zw.Close()
}
<commit_msg>Stream the view results through mc reqs faster.<commit_after>package main

import (
	"archive/zip"
	"errors"
	"log"
	"net/http"
	"strings"
	"sync"

	cb "github.com/couchbaselabs/go-couchbase"
)

var missingFile = errors.New("missing file")

type namedFile struct {
	name string
	meta fileMeta
	err  error
}

func pathDataFetcher(wg *sync.WaitGroup, quit <-chan bool,
	in <-chan string, out chan<- *namedFile) {
	defer wg.Done()

	for {
		select {
		case s, ok := <-in:
			if !ok {
				return
			}
			ob := namedFile{name: s}
			ob.err = couchbase.Get(s, &ob.meta)
			out <- &ob
		case <-quit:
			return
		}
	}
}

func pathGenerator(from string, ch chan *namedFile,
	errs chan error, quit chan bool) {

	parts := strings.Split(from, "/")

	viewRes := struct {
		Rows []struct {
			Key []string
			Id  string
		}
		Errors []cb.ViewError
	}{}

	limit := 1000
	fetchch := make(chan string, limit)
	startKey := parts
	done := false

	wg := &sync.WaitGroup{}
	for i := 0; i < 8; i++ {
		wg.Add(1)
		go pathDataFetcher(wg, quit, fetchch, ch)
	}
	defer func() {
		close(fetchch)
		wg.Wait()
		close(ch)
		close(errs)
	}()

	for !done {
		err := couchbase.ViewCustom("cbfs", "file_browse",
			map[string]interface{}{
				"stale":    false,
				"reduce":   false,
				"limit":    limit,
				"startkey": startKey,
			}, &viewRes)
		if err != nil {
			log.Printf("View error: %v", err)
			errs <- err
			return
		}
		for _, e := range viewRes.Errors {
			select {
			case errs <- e:
			case <-quit:
				return
			}
		}

		done = len(viewRes.Rows) < limit

		for _, r := range viewRes.Rows {
			k := r.Id
			if !strings.HasPrefix(k, from) {
				done = true
				break
			}
			startKey = r.Key

			fetchch <- k
		}
	}
}

func doZipDocs(w http.ResponseWriter, req *http.Request,
	path string) {

	quit := make(chan bool)
	defer close(quit)
	ch := make(chan *namedFile)
	cherr := make(chan error)

	go pathGenerator(path, ch, cherr, quit)
	go logErrors("zip", cherr)

	w.Header().Set("Content-Type", "application/zip")
	w.WriteHeader(200)

	zw := zip.NewWriter(w)
	for nf := range ch {
		if nf.err != nil {
			log.Printf("Error on %v: %v", nf.name, nf.err)
			continue
		}

		fh := zip.FileHeader{
			Name:             nf.name,
			Method:           zip.Deflate,
			UncompressedSize: uint32(nf.meta.Length),
			Comment:          nf.meta.OID,
		}
		fh.SetModTime(nf.meta.Modified)

		zf, err := zw.CreateHeader(&fh)
		if err != nil {
			log.Printf("Error making zip file: %v", err)
			// Client will get a broken zip file
			return
		}

		err = copyBlob(zf, nf.meta.OID)
		if err != nil {
			log.Printf("Error copying blob for %v: %v",
				nf.name, err)
			// Client will get a broken zip file
			return
		}
	}
	zw.Close()
}
<|endoftext|>


<commit_before>// libmbd contains data structures and helper routines for extracting reaction
// data from binary mcell reaction data output files
package libmbd

import (
	"fmt"
	"github.com/haskelladdict/mbdr/parser/util"
	"regexp"
)

// enumeration describing the output type of the included data
const (
	Step uint16 = 1 << iota
	TimeListType
	IterationListType
)

// list of currently know API versions
const API1 = "MCELL_BINARY_API_1"
const API2 = "MCELL_BINARY_API_2"

// MCellData tracks the data contained in the binary mcell file as well as
// relevant metadata to retrieve specific data items.
// NOTE: Depending on the API version of the binary output data not all fields
// are defined
type MCellData struct {
	Buffer         util.ReadBuf
	OutputListType uint16
	BlockSize      uint64
	StepSize       float64
	TimeList       []float64
	NumBlocks      uint64
	BlockNames     []string
	BlockNameMap   map[string]uint64
	API            string
	API1Data
	API2Data
}

// API1Data are data items specific to API version 1 of the mcell binary output
// format.
type API1Data struct {
	Offset       uint64 // offset into data buffer
	BlockEntries []BlockEntry
}

// BlockEntry is used for API version 1. It stores the beginning and end of
// each data block within the data buffer.
type BlockEntry struct {
	Type  byte
	Start uint64
	End   uint64
}

// API1Data are data items specific to API version 1 of the mcell binary output
// format.
type API2Data struct {
	OutputBufSize uint64
	TotalNumCols  uint64
	BlockInfo     []BlockData
}

// BlockData is used for API version 2. It stores metadata for a given data block
// such as the data name, number of data columns, the type of data
// stored (int/double), and the internal offset into the buffer at which the
// data can be found.
type BlockData struct {
	Name      string
	NumCols   uint64
	DataTypes []uint16
	Offset    uint64
}

// CountData is a container holding the data corresponding to a reaction data
// output block consisting of a number of columns
type CountData struct {
	Col       [][]float64
	DataTypes []uint16
}

// DataNames returns the list of available blocknames
func (d *MCellData) DataNames() []string {
	return d.BlockNames
}

// IDtoBlockName returns the blockname corresponding to the given id
func (d *MCellData) IDtoBlockName(id uint64) (string, error) {
	if id < 0 || id >= uint64(len(d.BlockNames)) {
		return "", fmt.Errorf("requested id is out of range")
	}
	return d.BlockNames[id], nil
}

// NumDataBlocks returns the number of available datablocks
func (d *MCellData) NumDataBlocks() uint64 {
	return d.NumBlocks
}

// BlockLen returns the number of output iterations per datablock
func (d *MCellData) BlockLen() uint64 {
	return d.BlockSize
}

// OutputType returns the output type (STEP, ITERATION_LIST/TIME_LIST)
func (d *MCellData) OutputType() uint16 {
	return d.OutputListType
}

// OutputStepLen returns the output step length. NOTE: The returns value is only
// meaningful is OutputListType == Step, otherwise this function returns 0
func (d *MCellData) OutputStepLen() float64 {
	return d.StepSize
}

// OutputTimes returns a slice with the output times corresponding to the
// column data (either computed from STEP or via ITERATION_LIST/TIME_LIST)
// NOTE: In the case of STEP we cache the timelist after the first request
func (d *MCellData) OutputTimes() []float64 {
	if d.OutputType() == Step && len(d.TimeList) == 0 {
		d.TimeList = make([]float64, d.BlockLen())
		for i := uint64(0); i < d.BlockLen(); i++ {
			d.TimeList[i] = d.OutputStepLen() * float64(i)
		}
	}
	return d.TimeList
}

// BlockDataByRegex returns a map with all datasets whose name matched the
// supplied regex. The map keys are the dataset names, the values are the
// corresponding count data items.
func (d *MCellData) BlockDataByRegex(selection string) (map[string]*CountData, error) {

	regex, err := regexp.Compile(selection)
	if err != nil {
		return nil, err
	}

	outputData := make(map[string]*CountData)
	names := d.DataNames()
	for _, n := range names {
		if regex.MatchString(n) {
			countData, err := d.BlockDataByName(n)
			if err != nil {
				return nil, err
			}
			outputData[n] = countData
		}
	}
	return outputData, nil
}

// BlockDataByName returns the data stored in the data block of the given name
// as a CountData struct
func (d *MCellData) BlockDataByName(name string) (*CountData, error) {
	id, ok := d.BlockNameMap[name]
	if !ok {
		return nil, fmt.Errorf("dataset %s not found", name)
	}

	return d.BlockDataByID(id)
}

// BlockDataByID returns the data stored in the data block of the given ID
// as a CountData struct
// NOTE: This is the only method of MCellData which is API sensitive
func (d *MCellData) BlockDataByID(id uint64) (*CountData, error) {
	if id < 0 || id >= d.NumBlocks {
		return nil, fmt.Errorf("supplied data ID %d is out of range", id)
	}

	var c *CountData
	var e error
	switch d.API {
	case API1:
		c, e = d.blockDataAPI1(id)
	case API2:
		c, e = d.blockDataAPI2(id)
	default:
		c = nil
		e = fmt.Errorf("unknown API type %s in BlockDataByID\n", d.API)
	}
	return c, e
}

// blockDataAPI1 returns count data for mcell binary API version 2. It returns
// the data stored in the data block of the given ID as a CountData struct
func (d *MCellData) blockDataAPI1(id uint64) (*CountData, error) {

	entry := d.BlockEntries[id]
	output := &CountData{}
	output.Col = make([][]float64, 1)
	output.Col[0] = make([]float64, 0, d.BlockSize)
	output.DataTypes = append(output.DataTypes, uint16(entry.Type))

	loc := entry.Start - d.Offset
	var buf util.ReadBuf
	switch entry.Type {
	case 0:
		var val uint32
		for i := uint64(0); i < d.BlockSize; i++ {
			buf = (d.Buffer)[loc:]
			val = buf.Uint32()
			output.Col[0] = append(output.Col[0], float64(val))
			loc += util.LenUint32
		}
		// sanity check
		if loc != entry.End-d.Offset {
			return nil, fmt.Errorf("did not properly reach end of data block %d\n", id)
		}

	case 1:
		var val float64
		for i := uint64(0); i < d.BlockSize; i++ {
			buf = (d.Buffer)[loc:]
			val = buf.Float64()
			output.Col[0] = append(output.Col[0], val)
			loc += util.LenFloat64
		}
		// sanity check
		if loc != entry.End-d.Offset {
			return nil, fmt.Errorf("did not properly reach end of data block %d\n", id)
		}
	}
	return output, nil
}

// blockDataAPI2 returns count data for mcell binary API version 2. It returns
// the data stored in the data block of the given ID as a CountData struct
func (d *MCellData) blockDataAPI2(id uint64) (*CountData, error) {

	entry := d.BlockInfo[id]
	output := &CountData{}
	output.Col = make([][]float64, entry.NumCols)
	for i := uint64(0); i < entry.NumCols; i++ {
		output.Col[i] = make([]float64, 0, d.BlockSize)
		output.DataTypes = append(output.DataTypes, entry.DataTypes[i])
	}

	row := uint64(0)
	stream := uint64(1)
	loc := d.OutputBufSize * util.LenFloat64 * entry.Offset
	// read all rows until we hit the total blockSize
	for row < d.BlockSize {

		// forward to the next stream block if we're done parsing the current one
		if row >= stream*d.OutputBufSize {
			offset := d.OutputBufSize
			// last partial block of length <= d.outputBufSize requires special treatment
			if d.BlockSize-row < d.OutputBufSize {
				offset = d.BlockSize - row
			}
			// forward to beginning of stream block
			loc = stream * d.OutputBufSize * d.TotalNumCols * util.LenFloat64

			// forward to location within stream block
			loc += offset * entry.Offset * util.LenFloat64

			stream++
		}

		// read current row
		for i := uint64(0); i < entry.NumCols; i++ {
			buf := (d.Buffer)[loc:]
			val := buf.Float64() //d.buffer[bufLoc:].float64NoSlice()
			output.Col[i] = append(output.Col[i], val)
			loc += util.LenFloat64
		}
		row++
	}

	return output, nil
}
<commit_msg>Properly calculate offset for binary output files when checkpointing.<commit_after>// libmbd contains data structures and helper routines for extracting reaction
// data from binary mcell reaction data output files
package libmbd

import (
	"fmt"
	"github.com/haskelladdict/mbdr/parser/util"
	"regexp"
)

// enumeration describing the output type of the included data
const (
	Step uint16 = 1 << iota
	TimeListType
	IterationListType
)

// list of currently know API versions
const API1 = "MCELL_BINARY_API_1"
const API2 = "MCELL_BINARY_API_2"

// MCellData tracks the data contained in the binary mcell file as well as
// relevant metadata to retrieve specific data items.
// NOTE: Depending on the API version of the binary output data not all fields
// are defined
type MCellData struct {
	Buffer         util.ReadBuf
	OutputListType uint16
	BlockSize      uint64
	StepSize       float64
	TimeList       []float64
	NumBlocks      uint64
	BlockNames     []string
	BlockNameMap   map[string]uint64
	API            string
	API1Data
	API2Data
}

// API1Data are data items specific to API version 1 of the mcell binary output
// format.
type API1Data struct {
	Offset       uint64 // offset into data buffer
	BlockEntries []BlockEntry
}

// BlockEntry is used for API version 1. It stores the beginning and end of
// each data block within the data buffer.
type BlockEntry struct {
	Type  byte
	Start uint64
	End   uint64
}

// API1Data are data items specific to API version 1 of the mcell binary output
// format.
type API2Data struct {
	OutputBufSize uint64
	TotalNumCols  uint64
	BlockInfo     []BlockData
}

// BlockData is used for API version 2. It stores metadata for a given data block
// such as the data name, number of data columns, the type of data
// stored (int/double), and the internal offset into the buffer at which the
// data can be found.
type BlockData struct {
	Name      string
	NumCols   uint64
	DataTypes []uint16
	Offset    uint64
}

// CountData is a container holding the data corresponding to a reaction data
// output block consisting of a number of columns
type CountData struct {
	Col       [][]float64
	DataTypes []uint16
}

// DataNames returns the list of available blocknames
func (d *MCellData) DataNames() []string {
	return d.BlockNames
}

// IDtoBlockName returns the blockname corresponding to the given id
func (d *MCellData) IDtoBlockName(id uint64) (string, error) {
	if id < 0 || id >= uint64(len(d.BlockNames)) {
		return "", fmt.Errorf("requested id is out of range")
	}
	return d.BlockNames[id], nil
}

// NumDataBlocks returns the number of available datablocks
func (d *MCellData) NumDataBlocks() uint64 {
	return d.NumBlocks
}

// BlockLen returns the number of output iterations per datablock
func (d *MCellData) BlockLen() uint64 {
	return d.BlockSize
}

// OutputType returns the output type (STEP, ITERATION_LIST/TIME_LIST)
func (d *MCellData) OutputType() uint16 {
	return d.OutputListType
}

// OutputStepLen returns the output step length. NOTE: The returns value is only
// meaningful is OutputListType == Step, otherwise this function returns 0
func (d *MCellData) OutputStepLen() float64 {
	return d.StepSize
}

// OutputTimes returns a slice with the output times corresponding to the
// column data (either computed from STEP or via ITERATION_LIST/TIME_LIST)
// NOTE: In the case of STEP we cache the timelist after the first request
func (d *MCellData) OutputTimes() []float64 {
	if d.OutputType() == Step && len(d.TimeList) == 0 {
		d.TimeList = make([]float64, d.BlockLen())
		for i := uint64(0); i < d.BlockLen(); i++ {
			d.TimeList[i] = d.OutputStepLen() * float64(i)
		}
	}
	return d.TimeList
}

// BlockDataByRegex returns a map with all datasets whose name matched the
// supplied regex. The map keys are the dataset names, the values are the
// corresponding count data items.
func (d *MCellData) BlockDataByRegex(selection string) (map[string]*CountData, error) {

	regex, err := regexp.Compile(selection)
	if err != nil {
		return nil, err
	}

	outputData := make(map[string]*CountData)
	names := d.DataNames()
	for _, n := range names {
		if regex.MatchString(n) {
			countData, err := d.BlockDataByName(n)
			if err != nil {
				return nil, err
			}
			outputData[n] = countData
		}
	}
	return outputData, nil
}

// BlockDataByName returns the data stored in the data block of the given name
// as a CountData struct
func (d *MCellData) BlockDataByName(name string) (*CountData, error) {
	id, ok := d.BlockNameMap[name]
	if !ok {
		return nil, fmt.Errorf("dataset %s not found", name)
	}

	return d.BlockDataByID(id)
}

// BlockDataByID returns the data stored in the data block of the given ID
// as a CountData struct
// NOTE: This is the only method of MCellData which is API sensitive
func (d *MCellData) BlockDataByID(id uint64) (*CountData, error) {
	if id < 0 || id >= d.NumBlocks {
		return nil, fmt.Errorf("supplied data ID %d is out of range", id)
	}

	var c *CountData
	var e error
	switch d.API {
	case API1:
		c, e = d.blockDataAPI1(id)
	case API2:
		c, e = d.blockDataAPI2(id)
	default:
		c = nil
		e = fmt.Errorf("unknown API type %s in BlockDataByID\n", d.API)
	}
	return c, e
}

// blockDataAPI1 returns count data for mcell binary API version 2. It returns
// the data stored in the data block of the given ID as a CountData struct
func (d *MCellData) blockDataAPI1(id uint64) (*CountData, error) {

	entry := d.BlockEntries[id]
	output := &CountData{}
	output.Col = make([][]float64, 1)
	output.Col[0] = make([]float64, 0, d.BlockSize)
	output.DataTypes = append(output.DataTypes, uint16(entry.Type))

	loc := entry.Start - d.Offset
	var buf util.ReadBuf
	switch entry.Type {
	case 0:
		var val uint32
		for i := uint64(0); i < d.BlockSize; i++ {
			buf = (d.Buffer)[loc:]
			val = buf.Uint32()
			output.Col[0] = append(output.Col[0], float64(val))
			loc += util.LenUint32
		}
		// sanity check
		if loc != entry.End-d.Offset {
			return nil, fmt.Errorf("did not properly reach end of data block %d\n", id)
		}

	case 1:
		var val float64
		for i := uint64(0); i < d.BlockSize; i++ {
			buf = (d.Buffer)[loc:]
			val = buf.Float64()
			output.Col[0] = append(output.Col[0], val)
			loc += util.LenFloat64
		}
		// sanity check
		if loc != entry.End-d.Offset {
			return nil, fmt.Errorf("did not properly reach end of data block %d\n", id)
		}
	}
	return output, nil
}

// blockDataAPI2 returns count data for mcell binary API version 2. It returns
// the data stored in the data block of the given ID as a CountData struct
func (d *MCellData) blockDataAPI2(id uint64) (*CountData, error) {

	entry := d.BlockInfo[id]
	output := &CountData{}
	output.Col = make([][]float64, entry.NumCols)
	for i := uint64(0); i < entry.NumCols; i++ {
		output.Col[i] = make([]float64, 0, d.BlockSize)
		output.DataTypes = append(output.DataTypes, entry.DataTypes[i])
	}

	row := uint64(0)
	stream := uint64(1)
	var loc uint64
	// NOTE: We need this to be able to deal with checkpoint files for which
	// the total number of items may be smaller than the output buffer size
	if d.BlockSize < d.OutputBufSize {
		loc = d.BlockSize * util.LenFloat64 * entry.Offset
	} else {
		loc = d.OutputBufSize * util.LenFloat64 * entry.Offset
	}
	// read all rows until we hit the total blockSize
	for row < d.BlockSize {

		// forward to the next stream block if we're done parsing the current one
		if row >= stream*d.OutputBufSize {
			offset := d.OutputBufSize
			// last partial block of length <= d.outputBufSize requires special treatment
			if d.BlockSize-row < d.OutputBufSize {
				offset = d.BlockSize - row
			}
			// forward to beginning of stream block
			loc = stream * d.OutputBufSize * d.TotalNumCols * util.LenFloat64

			// forward to location within stream block
			loc += offset * entry.Offset * util.LenFloat64

			stream++
		}

		// read current row
		for i := uint64(0); i < entry.NumCols; i++ {
			buf := (d.Buffer)[loc:]
			val := buf.Float64() //d.buffer[bufLoc:].float64NoSlice()
			output.Col[i] = append(output.Col[i], val)
			loc += util.LenFloat64
		}
		row++
	}

	return output, nil
}
<|endoftext|>