<commit_before>package types

import (
	"testing"
)

var tError = TypeError{}
var tInt = TypeIdent{"Int"}
var tBool = TypeIdent{"Bool"}
var tOpt = TypeOptional{tBool}
var tList = TypeList{tInt}
var tTuple = TypeTuple{[]Type{tInt, tBool, tOpt, tList}}
var tFunc = TypeFunction{tTuple, tList}

func TestTypeError(t *testing.T) {
	expectNotEquivalentType(t, tError, tError)
	expectNotEquivalentType(t, tError, tInt)

	expectString(t, tError.String(), "ERROR")
	expectBool(t, tError.IsError(), true)
	tError.isType()
}

func TestTypeVoid(t *testing.T) {
	expectEquivalentType(t, TypeVoid{}, TypeVoid{})
	expectNotEquivalentType(t, TypeVoid{}, TypeError{})
	expectNotEquivalentType(t, TypeVoid{}, tInt)

	expectString(t, (TypeVoid{}).String(), "Void")
	expectBool(t, (TypeVoid{}).IsError(), false)
	(TypeVoid{}).isType()
}

func TestTypeFunction(t *testing.T) {
	expectEquivalentType(t, tFunc, tFunc)
	expectNotEquivalentType(t, tFunc, tList)
	expectNotEquivalentType(t, tFunc, tError)
	expectNotEquivalentType(t, tFunc, TypeFunction{tTuple, tBool})
	expectNotEquivalentType(t, tFunc, TypeFunction{TypeTuple{}, tList})

	expectString(t, tFunc.String(), "(Int Bool Bool? [Int]) => [Int]")
	expectBool(t, tFunc.IsError(), false)
	tFunc.isType()
}

func TestTypeTuple(t *testing.T) {
	expectEquivalentType(t, tTuple, tTuple)
	expectNotEquivalentType(t, tTuple, tInt)
	expectNotEquivalentType(t, tTuple, tError)
	expectNotEquivalentType(t, tTuple, TypeTuple{[]Type{tInt, tBool, tOpt}})
	expectNotEquivalentType(t, tTuple, TypeTuple{[]Type{tInt, tBool, tOpt, tOpt}})

	expectString(t, tTuple.String(), "(Int Bool Bool? [Int])")
	expectBool(t, tTuple.IsError(), false)
	tTuple.isType()
}

func TestTypeList(t *testing.T) {
	expectEquivalentType(t, TypeList{tInt}, TypeList{tInt})
	expectEquivalentType(t, TypeList{tOpt}, TypeList{tOpt})
	expectNotEquivalentType(t, TypeList{tInt}, tInt)
	expectNotEquivalentType(t, TypeList{tInt}, tError)
	expectNotEquivalentType(t, TypeList{tInt}, TypeList{tBool})

	expectString(t, TypeList{tOpt}.String(), "[Bool?]")
	expectBool(t, tList.IsError(), false)
	tList.isType()
}

func TestTypeOptional(t *testing.T) {
	expectEquivalentType(t, TypeOptional{tInt}, TypeOptional{tInt})
	expectNotEquivalentType(t, TypeOptional{tInt}, tInt)
	expectNotEquivalentType(t, TypeOptional{tInt}, tError)
	expectNotEquivalentType(t, TypeOptional{tInt}, TypeOptional{tBool})

	expectString(t, tOpt.String(), "Bool?")
	expectBool(t, tOpt.IsError(), false)
	tOpt.isType()
}

func TestTypeIdent(t *testing.T) {
	expectEquivalentType(t, tInt, tInt)
	expectNotEquivalentType(t, tInt, tError)
	expectNotEquivalentType(t, tInt, tBool)
	expectNotEquivalentType(t, tInt, tOpt)

	expectString(t, tInt.String(), "Int")
	expectString(t, tBool.String(), "Bool")
	expectBool(t, tInt.IsError(), false)
	tInt.isType()
}

func TestConcatTypes(t *testing.T) {
	expectString(t, concatTypes(nil), "")
	expectString(t, concatTypes([]Type{}), "")
	expectString(t, concatTypes([]Type{tInt, tBool, tInt}), "Int Bool Int")
}

func expectEquivalentType(t *testing.T, t1 Type, t2 Type) {
	same := t1.Equals(t2)
	commutative := t1.Equals(t2) == t2.Equals(t1)

	if commutative == false {
		if same {
			t.Errorf("%s == %s, but %s != %s", t1, t2, t2, t1)
		} else {
			t.Errorf("%s == %s, but %s != %s", t2, t1, t1, t2)
		}
	}

	if same == false {
		t.Errorf("Expected %s == %s, got %t", t1, t2, same)
	}
}

func expectNotEquivalentType(t *testing.T, t1 Type, t2 Type) {
	same := t1.Equals(t2)
	commutative := t1.Equals(t2) == t2.Equals(t1)

	if commutative == false {
		if same {
			t.Errorf("%s == %s, but %s != %s", t1, t2, t2, t1)
		} else {
			t.Errorf("%s == %s, but %s != %s", t2, t1, t1, t2)
		}
	}

	if same == true {
		t.Errorf("Expected %s != %s, got %t", t1, t2, same)
	}
}

func expectString(t *testing.T, got string, exp string) {
	if exp != got {
		t.Errorf("Expected '%s', got '%s'", exp, got)
	}
}

func expectBool(t *testing.T, got bool, exp bool) {
	if exp != got {
		t.Errorf("Expected %t, got %t", exp, got)
	}
}
<commit_msg>fix unit test<commit_after>package types

import (
	"testing"
)

var tError = TypeError{}
var tInt = TypeIdent{"Int"}
var tBool = TypeIdent{"Bool"}
var tOpt = TypeOptional{tBool}
var tList = TypeList{tInt}
var tTuple = TypeTuple{[]Type{tInt, tBool, tOpt, tList}}
var tFunc = TypeFunction{tTuple, tList}

func TestTypeError(t *testing.T) {
	expectEquivalentType(t, tError, tError)
	expectNotEquivalentType(t, tError, tInt)

	expectString(t, tError.String(), "ERROR")
	expectBool(t, tError.IsError(), true)
	tError.isType()
}

func TestTypeVoid(t *testing.T) {
	expectEquivalentType(t, TypeVoid{}, TypeVoid{})
	expectNotEquivalentType(t, TypeVoid{}, TypeError{})
	expectNotEquivalentType(t, TypeVoid{}, tInt)

	expectString(t, (TypeVoid{}).String(), "Void")
	expectBool(t, (TypeVoid{}).IsError(), false)
	(TypeVoid{}).isType()
}

func TestTypeFunction(t *testing.T) {
	expectEquivalentType(t, tFunc, tFunc)
	expectNotEquivalentType(t, tFunc, tList)
	expectNotEquivalentType(t, tFunc, tError)
	expectNotEquivalentType(t, tFunc, TypeFunction{tTuple, tBool})
	expectNotEquivalentType(t, tFunc, TypeFunction{TypeTuple{}, tList})

	expectString(t, tFunc.String(), "(Int Bool Bool? [Int]) => [Int]")
	expectBool(t, tFunc.IsError(), false)
	tFunc.isType()
}

func TestTypeTuple(t *testing.T) {
	expectEquivalentType(t, tTuple, tTuple)
	expectNotEquivalentType(t, tTuple, tInt)
	expectNotEquivalentType(t, tTuple, tError)
	expectNotEquivalentType(t, tTuple, TypeTuple{[]Type{tInt, tBool, tOpt}})
	expectNotEquivalentType(t, tTuple, TypeTuple{[]Type{tInt, tBool, tOpt, tOpt}})

	expectString(t, tTuple.String(), "(Int Bool Bool? [Int])")
	expectBool(t, tTuple.IsError(), false)
	tTuple.isType()
}

func TestTypeList(t *testing.T) {
	expectEquivalentType(t, TypeList{tInt}, TypeList{tInt})
	expectEquivalentType(t, TypeList{tOpt}, TypeList{tOpt})
	expectNotEquivalentType(t, TypeList{tInt}, tInt)
	expectNotEquivalentType(t, TypeList{tInt}, tError)
	expectNotEquivalentType(t, TypeList{tInt}, TypeList{tBool})

	expectString(t, TypeList{tOpt}.String(), "[Bool?]")
	expectBool(t, tList.IsError(), false)
	tList.isType()
}

func TestTypeOptional(t *testing.T) {
	expectEquivalentType(t, TypeOptional{tInt}, TypeOptional{tInt})
	expectNotEquivalentType(t, TypeOptional{tInt}, tInt)
	expectNotEquivalentType(t, TypeOptional{tInt}, tError)
	expectNotEquivalentType(t, TypeOptional{tInt}, TypeOptional{tBool})

	expectString(t, tOpt.String(), "Bool?")
	expectBool(t, tOpt.IsError(), false)
	tOpt.isType()
}

func TestTypeIdent(t *testing.T) {
	expectEquivalentType(t, tInt, tInt)
	expectNotEquivalentType(t, tInt, tError)
	expectNotEquivalentType(t, tInt, tBool)
	expectNotEquivalentType(t, tInt, tOpt)

	expectString(t, tInt.String(), "Int")
	expectString(t, tBool.String(), "Bool")
	expectBool(t, tInt.IsError(), false)
	tInt.isType()
}

func TestConcatTypes(t *testing.T) {
	expectString(t, concatTypes(nil), "")
	expectString(t, concatTypes([]Type{}), "")
	expectString(t, concatTypes([]Type{tInt, tBool, tInt}), "Int Bool Int")
}

func expectEquivalentType(t *testing.T, t1 Type, t2 Type) {
	same := t1.Equals(t2)
	commutative := t1.Equals(t2) == t2.Equals(t1)

	if commutative == false {
		if same {
			t.Errorf("%s == %s, but %s != %s", t1, t2, t2, t1)
		} else {
			t.Errorf("%s == %s, but %s != %s", t2, t1, t1, t2)
		}
	}

	if same == false {
		t.Errorf("Expected %s == %s, got %t", t1, t2, same)
	}
}

func expectNotEquivalentType(t *testing.T, t1 Type, t2 Type) {
	same := t1.Equals(t2)
	commutative := t1.Equals(t2) == t2.Equals(t1)

	if commutative == false {
		if same {
			t.Errorf("%s == %s, but %s != %s", t1, t2, t2, t1)
		} else {
			t.Errorf("%s == %s, but %s != %s", t2, t1, t1, t2)
		}
	}

	if same == true {
		t.Errorf("Expected %s != %s, got %t", t1, t2, same)
	}
}

func expectString(t *testing.T, got string, exp string) {
	if exp != got {
		t.Errorf("Expected '%s', got '%s'", exp, got)
	}
}

func expectBool(t *testing.T, got bool, exp bool) {
	if exp != got {
		t.Errorf("Expected %t, got %t", exp, got)
	}
}
<|endoftext|>


<commit_before>package libkbfs

// Version is the current version (should be MAJOR.MINOR.PATCH)
const Version = "1.0.0"

// Build is the current build number
const Build = "24"
<commit_msg>New version 1.0.0-25<commit_after>package libkbfs

// Version is the current version (should be MAJOR.MINOR.PATCH)
const Version = "1.0.0"

// Build is the current build number
const Build = "25"
<|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"flag"
	"fmt"
	"os"

	ma "github.com/jbenet/go-multiaddr"
)

var usage = `multiaddr conversion
usage: multiaddr [fmt] <>`

var formats = []string{"string", "bytes", "hex", "raw"}
var format string

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s <multiaddr>\n\nFlags:\n", os.Args[0])
		flag.PrintDefaults()
	}

	usage := fmt.Sprintf("output format, one of: %v", formats)
	flag.StringVar(&format, "format", "string", usage)
	flag.StringVar(&format, "f", "string", usage+" (shorthand)")
}

func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) != 1 {
		die("error: can only take one argument")
	}

	m, err := ma.NewMultiaddr(args[0])
	if err != nil {
		die(err)
	}

	fmt.Println(outfmt(m))
}

func outfmt(m ma.Multiaddr) string {
	switch format {
	case "string":
		return m.String()
	case "bytes":
		return fmt.Sprintf("%v", m.Bytes())
	case "raw":
		return string(m.Bytes())
	case "hex":
		return "0x" + hex.EncodeToString(m.Bytes())
	}

	die("error: invalid format", format)
	return ""
}

func die(v ...interface{}) {
	fmt.Fprint(os.Stderr, v...)
	fmt.Fprint(os.Stderr, "\n")
	flag.Usage()
	os.Exit(-1)
}
<commit_msg>net: added InterfaceMultiaddrs<commit_after>package main

import (
	"encoding/hex"
	"flag"
	"fmt"
	"os"

	ma "github.com/jbenet/go-multiaddr"
	manet "github.com/jbenet/go-multiaddr/net"
)

var formats = []string{"string", "bytes", "hex", "slice"}
var format string

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s [<multiaddr>]\n\nFlags:\n", os.Args[0])
		flag.PrintDefaults()
	}

	usage := fmt.Sprintf("output format, one of: %v", formats)
	flag.StringVar(&format, "format", "string", usage)
	flag.StringVar(&format, "f", "string", usage+" (shorthand)")
}

func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) == 0 {
		maddrs, err := manet.InterfaceMultiaddrs()
		if err != nil {
			die(err)
		}

		output(maddrs...)
		return
	}

	m, err := ma.NewMultiaddr(args[0])
	if err != nil {
		die(err)
	}

	output(m)
}

func output(ms ...ma.Multiaddr) {
	for _, m := range ms {
		fmt.Println(outfmt(m))
	}
}

func outfmt(m ma.Multiaddr) string {
	switch format {
	case "string":
		return m.String()
	case "slice":
		return fmt.Sprintf("%v", m.Bytes())
	case "bytes":
		return string(m.Bytes())
	case "hex":
		return "0x" + hex.EncodeToString(m.Bytes())
	}

	die("error: invalid format", format)
	return ""
}

func die(v ...interface{}) {
	fmt.Fprint(os.Stderr, v...)
	fmt.Fprint(os.Stderr, "\n")
	flag.Usage()
	os.Exit(-1)
}
<|endoftext|>


<commit_before>// Package limiter provides data structure to configure rate-limiter.
package limiter

import (
	"net/http"
	"sync"
	"time"

	gocache "github.com/patrickmn/go-cache"
	"golang.org/x/time/rate"
)

// New is a constructor for Limiter.
func New(generalExpirableOptions *ExpirableOptions) *Limiter {
	lmt := &Limiter{}

	lmt.SetMessageContentType("text/plain; charset=utf-8").
		SetMessage("You have reached maximum request limit.").
		SetStatusCode(429).
		SetOnLimitReached(nil).
		SetIPLookups([]string{"RemoteAddr", "X-Forwarded-For", "X-Real-IP"}).
		SetForwardedForIndexFromBehind(0).
		SetHeaders(make(map[string][]string))

	if generalExpirableOptions != nil {
		lmt.generalExpirableOptions = generalExpirableOptions
	} else {
		lmt.generalExpirableOptions = &ExpirableOptions{}
	}

	// Default for ExpireJobInterval is every minute.
	if lmt.generalExpirableOptions.ExpireJobInterval <= 0 {
		lmt.generalExpirableOptions.ExpireJobInterval = time.Minute
	}

	// Default for DefaultExpirationTTL is 10 years.
	if lmt.generalExpirableOptions.DefaultExpirationTTL <= 0 {
		lmt.generalExpirableOptions.DefaultExpirationTTL = 87600 * time.Hour
	}

	lmt.tokenBuckets = gocache.New(
		lmt.generalExpirableOptions.DefaultExpirationTTL,
		lmt.generalExpirableOptions.ExpireJobInterval,
	)

	lmt.basicAuthUsers = gocache.New(
		lmt.generalExpirableOptions.DefaultExpirationTTL,
		lmt.generalExpirableOptions.ExpireJobInterval,
	)

	return lmt
}

// Limiter is a config struct to limit a particular request handler.
type Limiter struct {
	// Maximum number of requests to limit per second.
	max float64

	// Limiter burst size
	burst int

	// HTTP message when limit is reached.
	message string

	// Content-Type for Message
	messageContentType string

	// HTTP status code when limit is reached.
	statusCode int

	// A function to call when a request is rejected.
	onLimitReached func(w http.ResponseWriter, r *http.Request)

	// List of places to look up IP address.
	// Default is "RemoteAddr", "X-Forwarded-For", "X-Real-IP".
	// You can rearrange the order as you like.
	ipLookups []string

	forwardedForIndex int

	// List of HTTP Methods to limit (GET, POST, PUT, etc.).
	// Empty means limit all methods.
	methods []string

	// Able to configure token bucket expirations.
	generalExpirableOptions *ExpirableOptions

	// List of basic auth usernames to limit.
	basicAuthUsers *gocache.Cache

	// Map of HTTP headers to limit.
	// Empty means skip headers checking.
	headers map[string]*gocache.Cache

	// Map of limiters with TTL
	tokenBuckets *gocache.Cache

	tokenBucketExpirationTTL time.Duration
	basicAuthExpirationTTL   time.Duration
	headerEntryExpirationTTL time.Duration

	sync.RWMutex
}

// SetTokenBucketExpirationTTL is thread-safe way of setting custom token bucket expiration TTL.
func (l *Limiter) SetTokenBucketExpirationTTL(ttl time.Duration) *Limiter {
	l.Lock()
	l.tokenBucketExpirationTTL = ttl
	l.Unlock()

	return l
}

// GettokenBucketExpirationTTL is thread-safe way of getting custom token bucket expiration TTL.
func (l *Limiter) GetTokenBucketExpirationTTL() time.Duration {
	l.RLock()
	defer l.RUnlock()
	return l.tokenBucketExpirationTTL
}

// SetBasicAuthExpirationTTL is thread-safe way of setting custom basic auth expiration TTL.
func (l *Limiter) SetBasicAuthExpirationTTL(ttl time.Duration) *Limiter {
	l.Lock()
	l.basicAuthExpirationTTL = ttl
	l.Unlock()

	return l
}

// GetBasicAuthExpirationTTL is thread-safe way of getting custom basic auth expiration TTL.
func (l *Limiter) GetBasicAuthExpirationTTL() time.Duration {
	l.RLock()
	defer l.RUnlock()
	return l.basicAuthExpirationTTL
}

// SetHeaderEntryExpirationTTL is thread-safe way of setting custom basic auth expiration TTL.
func (l *Limiter) SetHeaderEntryExpirationTTL(ttl time.Duration) *Limiter {
	l.Lock()
	l.headerEntryExpirationTTL = ttl
	l.Unlock()

	return l
}

// GetHeaderEntryExpirationTTL is thread-safe way of getting custom basic auth expiration TTL.
func (l *Limiter) GetHeaderEntryExpirationTTL() time.Duration {
	l.RLock()
	defer l.RUnlock()
	return l.headerEntryExpirationTTL
}

// SetMax is thread-safe way of setting maximum number of requests to limit per duration.
func (l *Limiter) SetMax(max float64) *Limiter {
	l.Lock()
	l.max = max
	l.Unlock()

	return l
}

// GetMax is thread-safe way of getting maximum number of requests to limit per duration.
func (l *Limiter) GetMax() float64 {
	l.RLock()
	defer l.RUnlock()
	return l.max
}

// SetBurst is thread-safe way of setting maximum burst size.
func (l *Limiter) SetBurst(burst int) *Limiter {
	l.Lock()
	l.burst = burst
	l.Unlock()

	return l
}

// GetBurst is thread-safe way of setting maximum burst size.
func (l *Limiter) GetBurst() int {
	l.RLock()
	defer l.RUnlock()

	return l.burst
}

// SetMessage is thread-safe way of setting HTTP message when limit is reached.
func (l *Limiter) SetMessage(msg string) *Limiter {
	l.Lock()
	l.message = msg
	l.Unlock()

	return l
}

// GetMessage is thread-safe way of getting HTTP message when limit is reached.
func (l *Limiter) GetMessage() string {
	l.RLock()
	defer l.RUnlock()
	return l.message
}

// SetMessageContentType is thread-safe way of setting HTTP message Content-Type when limit is reached.
func (l *Limiter) SetMessageContentType(contentType string) *Limiter {
	l.Lock()
	l.messageContentType = contentType
	l.Unlock()

	return l
}

// GetMessageContentType is thread-safe way of getting HTTP message Content-Type when limit is reached.
func (l *Limiter) GetMessageContentType() string {
	l.RLock()
	defer l.RUnlock()
	return l.messageContentType
}

// SetStatusCode is thread-safe way of setting HTTP status code when limit is reached.
func (l *Limiter) SetStatusCode(statusCode int) *Limiter {
	l.Lock()
	l.statusCode = statusCode
	l.Unlock()

	return l
}

// GetStatusCode is thread-safe way of getting HTTP status code when limit is reached.
func (l *Limiter) GetStatusCode() int {
	l.RLock()
	defer l.RUnlock()
	return l.statusCode
}

// SetOnLimitReached is thread-safe way of setting after-rejection function when limit is reached.
func (l *Limiter) SetOnLimitReached(fn func(w http.ResponseWriter, r *http.Request)) *Limiter {
	l.Lock()
	l.onLimitReached = fn
	l.Unlock()

	return l
}

// ExecOnLimitReached is thread-safe way of executing after-rejection function when limit is reached.
func (l *Limiter) ExecOnLimitReached(w http.ResponseWriter, r *http.Request) {
	l.RLock()
	defer l.RUnlock()

	fn := l.onLimitReached
	if fn != nil {
		fn(w, r)
	}
}

// SetIPLookups is thread-safe way of setting list of places to look up IP address.
func (l *Limiter) SetIPLookups(ipLookups []string) *Limiter {
	l.Lock()
	l.ipLookups = ipLookups
	l.Unlock()

	return l
}

// GetIPLookups is thread-safe way of getting list of places to look up IP address.
func (l *Limiter) GetIPLookups() []string {
	l.RLock()
	defer l.RUnlock()
	return l.ipLookups
}

// SetForwardedForIndexFromBehind is thread-safe way of setting which X-Forwarded-For index to choose.
func (l *Limiter) SetForwardedForIndexFromBehind(forwardedForIndex int) *Limiter {
	l.Lock()
	l.forwardedForIndex = forwardedForIndex
	l.Unlock()

	return l
}

// GetForwardedForIndexFromBehind is thread-safe way of getting which X-Forwarded-For index to choose.
func (l *Limiter) GetForwardedForIndexFromBehind() int {
	l.RLock()
	defer l.RUnlock()
	return l.forwardedForIndex
}

// SetMethods is thread-safe way of setting list of HTTP Methods to limit (GET, POST, PUT, etc.).
func (l *Limiter) SetMethods(methods []string) *Limiter {
	l.Lock()
	l.methods = methods
	l.Unlock()

	return l
}

// GetMethods is thread-safe way of getting list of HTTP Methods to limit (GET, POST, PUT, etc.).
func (l *Limiter) GetMethods() []string {
	l.RLock()
	defer l.RUnlock()
	return l.methods
}

// SetBasicAuthUsers is thread-safe way of setting list of basic auth usernames to limit.
func (l *Limiter) SetBasicAuthUsers(basicAuthUsers []string) *Limiter {
	ttl := l.GetBasicAuthExpirationTTL()
	if ttl <= 0 {
		ttl = l.generalExpirableOptions.DefaultExpirationTTL
	}

	for _, basicAuthUser := range basicAuthUsers {
		l.basicAuthUsers.Set(basicAuthUser, true, ttl)
	}

	return l
}

// GetBasicAuthUsers is thread-safe way of getting list of basic auth usernames to limit.
func (l *Limiter) GetBasicAuthUsers() []string {
	asMap := l.basicAuthUsers.Items()

	var basicAuthUsers []string
	for basicAuthUser, _ := range asMap {
		basicAuthUsers = append(basicAuthUsers, basicAuthUser)
	}

	return basicAuthUsers
}

// RemoveBasicAuthUsers is thread-safe way of removing basic auth usernames from existing list.
func (l *Limiter) RemoveBasicAuthUsers(basicAuthUsers []string) *Limiter {
	for _, toBeRemoved := range basicAuthUsers {
		l.basicAuthUsers.Delete(toBeRemoved)
	}

	return l
}

// SetHeaders is thread-safe way of setting map of HTTP headers to limit.
func (l *Limiter) SetHeaders(headers map[string][]string) *Limiter {
	if l.headers == nil {
		l.headers = make(map[string]*gocache.Cache)
	}

	for header, entries := range headers {
		l.SetHeader(header, entries)
	}

	return l
}

// GetHeaders is thread-safe way of getting map of HTTP headers to limit.
func (l *Limiter) GetHeaders() map[string][]string {
	results := make(map[string][]string)

	l.RLock()
	defer l.RUnlock()

	for header, entriesAsGoCache := range l.headers {
		entries := make([]string, 0)

		for entry, _ := range entriesAsGoCache.Items() {
			entries = append(entries, entry)
		}

		results[header] = entries
	}

	return results
}

// SetHeader is thread-safe way of setting entries of 1 HTTP header.
func (l *Limiter) SetHeader(header string, entries []string) *Limiter {
	l.RLock()
	existing, found := l.headers[header]
	l.RUnlock()

	ttl := l.GetHeaderEntryExpirationTTL()
	if ttl <= 0 {
		ttl = l.generalExpirableOptions.DefaultExpirationTTL
	}

	if !found {
		existing = gocache.New(ttl, l.generalExpirableOptions.ExpireJobInterval)
	}

	for _, entry := range entries {
		existing.Set(entry, true, ttl)
	}

	l.Lock()
	l.headers[header] = existing
	l.Unlock()

	return l
}

// GetHeader is thread-safe way of getting entries of 1 HTTP header.
func (l *Limiter) GetHeader(header string) []string {
	l.RLock()
	entriesAsGoCache := l.headers[header]
	l.RUnlock()

	entriesAsMap := entriesAsGoCache.Items()
	entries := make([]string, 0)

	for entry, _ := range entriesAsMap {
		entries = append(entries, entry)
	}

	return entries
}

// RemoveHeader is thread-safe way of removing entries of 1 HTTP header.
func (l *Limiter) RemoveHeader(header string) *Limiter {
	ttl := l.GetHeaderEntryExpirationTTL()
	if ttl <= 0 {
		ttl = l.generalExpirableOptions.DefaultExpirationTTL
	}

	l.Lock()
	l.headers[header] = gocache.New(ttl, l.generalExpirableOptions.ExpireJobInterval)
	l.Unlock()

	return l
}

// RemoveHeaderEntries is thread-safe way of adding new entries to 1 HTTP header rule.
func (l *Limiter) RemoveHeaderEntries(header string, entriesForRemoval []string) *Limiter {
	l.RLock()
	entries, found := l.headers[header]
	l.RUnlock()

	if !found {
		return l
	}

	for _, toBeRemoved := range entriesForRemoval {
		entries.Delete(toBeRemoved)
	}

	return l
}

func (l *Limiter) limitReachedWithTokenBucketTTL(key string, tokenBucketTTL time.Duration) bool {
	lmtMax := l.GetMax()
	lmtBurst := l.GetBurst()
	l.Lock()
	defer l.Unlock()

	if _, found := l.tokenBuckets.Get(key); !found {
		l.tokenBuckets.Set(
			key,
			rate.NewLimiter(rate.Limit(lmtMax), lmtBurst),
			tokenBucketTTL,
		)
	}

	expiringMap, found := l.tokenBuckets.Get(key)
	if !found {
		return false
	}

	return !expiringMap.(*rate.Limiter).Allow()
}

// LimitReached returns a bool indicating if the Bucket identified by key ran out of tokens.
func (l *Limiter) LimitReached(key string) bool {
	ttl := l.GetTokenBucketExpirationTTL()

	if ttl <= 0 {
		ttl = l.generalExpirableOptions.DefaultExpirationTTL
	}

	return l.limitReachedWithTokenBucketTTL(key, ttl)
}
<commit_msg>Adding Expiration TTL for contextValues<commit_after>// Package limiter provides data structure to configure rate-limiter.
package limiter

import (
	"net/http"
	"sync"
	"time"

	gocache "github.com/patrickmn/go-cache"
	"golang.org/x/time/rate"
)

// New is a constructor for Limiter.
func New(generalExpirableOptions *ExpirableOptions) *Limiter {
	lmt := &Limiter{}

	lmt.SetMessageContentType("text/plain; charset=utf-8").
		SetMessage("You have reached maximum request limit.").
		SetStatusCode(429).
		SetOnLimitReached(nil).
		SetIPLookups([]string{"RemoteAddr", "X-Forwarded-For", "X-Real-IP"}).
		SetForwardedForIndexFromBehind(0).
		SetHeaders(make(map[string][]string))

	if generalExpirableOptions != nil {
		lmt.generalExpirableOptions = generalExpirableOptions
	} else {
		lmt.generalExpirableOptions = &ExpirableOptions{}
	}

	// Default for ExpireJobInterval is every minute.
	if lmt.generalExpirableOptions.ExpireJobInterval <= 0 {
		lmt.generalExpirableOptions.ExpireJobInterval = time.Minute
	}

	// Default for DefaultExpirationTTL is 10 years.
	if lmt.generalExpirableOptions.DefaultExpirationTTL <= 0 {
		lmt.generalExpirableOptions.DefaultExpirationTTL = 87600 * time.Hour
	}

	lmt.tokenBuckets = gocache.New(
		lmt.generalExpirableOptions.DefaultExpirationTTL,
		lmt.generalExpirableOptions.ExpireJobInterval,
	)

	lmt.basicAuthUsers = gocache.New(
		lmt.generalExpirableOptions.DefaultExpirationTTL,
		lmt.generalExpirableOptions.ExpireJobInterval,
	)

	return lmt
}

// Limiter is a config struct to limit a particular request handler.
type Limiter struct {
	// Maximum number of requests to limit per second.
	max float64

	// Limiter burst size
	burst int

	// HTTP message when limit is reached.
	message string

	// Content-Type for Message
	messageContentType string

	// HTTP status code when limit is reached.
	statusCode int

	// A function to call when a request is rejected.
	onLimitReached func(w http.ResponseWriter, r *http.Request)

	// List of places to look up IP address.
	// Default is "RemoteAddr", "X-Forwarded-For", "X-Real-IP".
	// You can rearrange the order as you like.
	ipLookups []string

	forwardedForIndex int

	// List of HTTP Methods to limit (GET, POST, PUT, etc.).
	// Empty means limit all methods.
	methods []string

	// Able to configure token bucket expirations.
	generalExpirableOptions *ExpirableOptions

	// List of basic auth usernames to limit.
	basicAuthUsers *gocache.Cache

	// Map of HTTP headers to limit.
	// Empty means skip headers checking.
	headers map[string]*gocache.Cache

	// Map of Context values to limit.
	contextValues map[string]*gocache.Cache

	// Map of limiters with TTL
	tokenBuckets *gocache.Cache

	tokenBucketExpirationTTL 	time.Duration
	basicAuthExpirationTTL   	time.Duration
	headerEntryExpirationTTL 	time.Duration
	contextEntryExpirationTTL 	time.Duration

	sync.RWMutex
}

// SetTokenBucketExpirationTTL is thread-safe way of setting custom token bucket expiration TTL.
func (l *Limiter) SetTokenBucketExpirationTTL(ttl time.Duration) *Limiter {
	l.Lock()
	l.tokenBucketExpirationTTL = ttl
	l.Unlock()

	return l
}

// GettokenBucketExpirationTTL is thread-safe way of getting custom token bucket expiration TTL.
func (l *Limiter) GetTokenBucketExpirationTTL() time.Duration {
	l.RLock()
	defer l.RUnlock()
	return l.tokenBucketExpirationTTL
}

// SetBasicAuthExpirationTTL is thread-safe way of setting custom basic auth expiration TTL.
func (l *Limiter) SetBasicAuthExpirationTTL(ttl time.Duration) *Limiter {
	l.Lock()
	l.basicAuthExpirationTTL = ttl
	l.Unlock()

	return l
}

// GetBasicAuthExpirationTTL is thread-safe way of getting custom basic auth expiration TTL.
func (l *Limiter) GetBasicAuthExpirationTTL() time.Duration {
	l.RLock()
	defer l.RUnlock()
	return l.basicAuthExpirationTTL
}

// SetHeaderEntryExpirationTTL is thread-safe way of setting custom basic auth expiration TTL.
func (l *Limiter) SetHeaderEntryExpirationTTL(ttl time.Duration) *Limiter {
	l.Lock()
	l.headerEntryExpirationTTL = ttl
	l.Unlock()

	return l
}

// GetHeaderEntryExpirationTTL is thread-safe way of getting custom basic auth expiration TTL.
func (l *Limiter) GetHeaderEntryExpirationTTL() time.Duration {
	l.RLock()
	defer l.RUnlock()
	return l.headerEntryExpirationTTL
}

// SetHeaderEntryExpirationTTL is thread-safe way of setting custom basic auth expiration TTL.
func (l *Limiter) SetContextEntryEntryExpirationTTL(ttl time.Duration) *Limiter {
	l.Lock()
	l.contextEntryExpirationTTL = ttl
	l.Unlock()

	return l
}

// GetHeaderEntryExpirationTTL is thread-safe way of getting custom basic auth expiration TTL.
func (l *Limiter) GetContextEntryExpirationTTL() time.Duration {
	l.RLock()
	defer l.RUnlock()
	return l.contextEntryExpirationTTL
}

// SetMax is thread-safe way of setting maximum number of requests to limit per duration.
func (l *Limiter) SetMax(max float64) *Limiter {
	l.Lock()
	l.max = max
	l.Unlock()

	return l
}

// GetMax is thread-safe way of getting maximum number of requests to limit per duration.
func (l *Limiter) GetMax() float64 {
	l.RLock()
	defer l.RUnlock()
	return l.max
}

// SetBurst is thread-safe way of setting maximum burst size.
func (l *Limiter) SetBurst(burst int) *Limiter {
	l.Lock()
	l.burst = burst
	l.Unlock()

	return l
}

// GetBurst is thread-safe way of setting maximum burst size.
func (l *Limiter) GetBurst() int {
	l.RLock()
	defer l.RUnlock()

	return l.burst
}

// SetMessage is thread-safe way of setting HTTP message when limit is reached.
func (l *Limiter) SetMessage(msg string) *Limiter {
	l.Lock()
	l.message = msg
	l.Unlock()

	return l
}

// GetMessage is thread-safe way of getting HTTP message when limit is reached.
func (l *Limiter) GetMessage() string {
	l.RLock()
	defer l.RUnlock()
	return l.message
}

// SetMessageContentType is thread-safe way of setting HTTP message Content-Type when limit is reached.
func (l *Limiter) SetMessageContentType(contentType string) *Limiter {
	l.Lock()
	l.messageContentType = contentType
	l.Unlock()

	return l
}

// GetMessageContentType is thread-safe way of getting HTTP message Content-Type when limit is reached.
func (l *Limiter) GetMessageContentType() string {
	l.RLock()
	defer l.RUnlock()
	return l.messageContentType
}

// SetStatusCode is thread-safe way of setting HTTP status code when limit is reached.
func (l *Limiter) SetStatusCode(statusCode int) *Limiter {
	l.Lock()
	l.statusCode = statusCode
	l.Unlock()

	return l
}

// GetStatusCode is thread-safe way of getting HTTP status code when limit is reached.
func (l *Limiter) GetStatusCode() int {
	l.RLock()
	defer l.RUnlock()
	return l.statusCode
}

// SetOnLimitReached is thread-safe way of setting after-rejection function when limit is reached.
func (l *Limiter) SetOnLimitReached(fn func(w http.ResponseWriter, r *http.Request)) *Limiter {
	l.Lock()
	l.onLimitReached = fn
	l.Unlock()

	return l
}

// ExecOnLimitReached is thread-safe way of executing after-rejection function when limit is reached.
func (l *Limiter) ExecOnLimitReached(w http.ResponseWriter, r *http.Request) {
	l.RLock()
	defer l.RUnlock()

	fn := l.onLimitReached
	if fn != nil {
		fn(w, r)
	}
}

// SetIPLookups is thread-safe way of setting list of places to look up IP address.
func (l *Limiter) SetIPLookups(ipLookups []string) *Limiter {
	l.Lock()
	l.ipLookups = ipLookups
	l.Unlock()

	return l
}

// GetIPLookups is thread-safe way of getting list of places to look up IP address.
func (l *Limiter) GetIPLookups() []string {
	l.RLock()
	defer l.RUnlock()
	return l.ipLookups
}

// SetForwardedForIndexFromBehind is thread-safe way of setting which X-Forwarded-For index to choose.
func (l *Limiter) SetForwardedForIndexFromBehind(forwardedForIndex int) *Limiter {
	l.Lock()
	l.forwardedForIndex = forwardedForIndex
	l.Unlock()

	return l
}

// GetForwardedForIndexFromBehind is thread-safe way of getting which X-Forwarded-For index to choose.
func (l *Limiter) GetForwardedForIndexFromBehind() int {
	l.RLock()
	defer l.RUnlock()
	return l.forwardedForIndex
}

// SetMethods is thread-safe way of setting list of HTTP Methods to limit (GET, POST, PUT, etc.).
func (l *Limiter) SetMethods(methods []string) *Limiter {
	l.Lock()
	l.methods = methods
	l.Unlock()

	return l
}

// GetMethods is thread-safe way of getting list of HTTP Methods to limit (GET, POST, PUT, etc.).
func (l *Limiter) GetMethods() []string {
	l.RLock()
	defer l.RUnlock()
	return l.methods
}

// SetBasicAuthUsers is thread-safe way of setting list of basic auth usernames to limit.
func (l *Limiter) SetBasicAuthUsers(basicAuthUsers []string) *Limiter {
	ttl := l.GetBasicAuthExpirationTTL()
	if ttl <= 0 {
		ttl = l.generalExpirableOptions.DefaultExpirationTTL
	}

	for _, basicAuthUser := range basicAuthUsers {
		l.basicAuthUsers.Set(basicAuthUser, true, ttl)
	}

	return l
}

// GetBasicAuthUsers is thread-safe way of getting list of basic auth usernames to limit.
func (l *Limiter) GetBasicAuthUsers() []string {
	asMap := l.basicAuthUsers.Items()

	var basicAuthUsers []string
	for basicAuthUser, _ := range asMap {
		basicAuthUsers = append(basicAuthUsers, basicAuthUser)
	}

	return basicAuthUsers
}

// RemoveBasicAuthUsers is thread-safe way of removing basic auth usernames from existing list.
func (l *Limiter) RemoveBasicAuthUsers(basicAuthUsers []string) *Limiter {
	for _, toBeRemoved := range basicAuthUsers {
		l.basicAuthUsers.Delete(toBeRemoved)
	}

	return l
}

// SetHeaders is thread-safe way of setting map of HTTP headers to limit.
func (l *Limiter) SetHeaders(headers map[string][]string) *Limiter {
	if l.headers == nil {
		l.headers = make(map[string]*gocache.Cache)
	}

	for header, entries := range headers {
		l.SetHeader(header, entries)
	}

	return l
}

// GetHeaders is thread-safe way of getting map of HTTP headers to limit.
func (l *Limiter) GetHeaders() map[string][]string {
	results := make(map[string][]string)

	l.RLock()
	defer l.RUnlock()

	for header, entriesAsGoCache := range l.headers {
		entries := make([]string, 0)

		for entry, _ := range entriesAsGoCache.Items() {
			entries = append(entries, entry)
		}

		results[header] = entries
	}

	return results
}

// SetHeader is thread-safe way of setting entries of 1 HTTP header.
func (l *Limiter) SetHeader(header string, entries []string) *Limiter {
	l.RLock()
	existing, found := l.headers[header]
	l.RUnlock()

	ttl := l.GetHeaderEntryExpirationTTL()
	if ttl <= 0 {
		ttl = l.generalExpirableOptions.DefaultExpirationTTL
	}

	if !found {
		existing = gocache.New(ttl, l.generalExpirableOptions.ExpireJobInterval)
	}

	for _, entry := range entries {
		existing.Set(entry, true, ttl)
	}

	l.Lock()
	l.headers[header] = existing
	l.Unlock()

	return l
}

// GetHeader is thread-safe way of getting entries of 1 HTTP header.
func (l *Limiter) GetHeader(header string) []string {
	l.RLock()
	entriesAsGoCache := l.headers[header]
	l.RUnlock()

	entriesAsMap := entriesAsGoCache.Items()
	entries := make([]string, 0)

	for entry, _ := range entriesAsMap {
		entries = append(entries, entry)
	}

	return entries
}

// RemoveHeader is thread-safe way of removing entries of 1 HTTP header.
func (l *Limiter) RemoveHeader(header string) *Limiter {
	ttl := l.GetHeaderEntryExpirationTTL()
	if ttl <= 0 {
		ttl = l.generalExpirableOptions.DefaultExpirationTTL
	}

	l.Lock()
	l.headers[header] = gocache.New(ttl, l.generalExpirableOptions.ExpireJobInterval)
	l.Unlock()

	return l
}

// RemoveHeaderEntries is thread-safe way of adding new entries to 1 HTTP header rule.
func (l *Limiter) RemoveHeaderEntries(header string, entriesForRemoval []string) *Limiter {
	l.RLock()
	entries, found := l.headers[header]
	l.RUnlock()

	if !found {
		return l
	}

	for _, toBeRemoved := range entriesForRemoval {
		entries.Delete(toBeRemoved)
	}

	return l
}

func (l *Limiter) limitReachedWithTokenBucketTTL(key string, tokenBucketTTL time.Duration) bool {
	lmtMax := l.GetMax()
	lmtBurst := l.GetBurst()
	l.Lock()
	defer l.Unlock()

	if _, found := l.tokenBuckets.Get(key); !found {
		l.tokenBuckets.Set(
			key,
			rate.NewLimiter(rate.Limit(lmtMax), lmtBurst),
			tokenBucketTTL,
		)
	}

	expiringMap, found := l.tokenBuckets.Get(key)
	if !found {
		return false
	}

	return !expiringMap.(*rate.Limiter).Allow()
}

// LimitReached returns a bool indicating if the Bucket identified by key ran out of tokens.
func (l *Limiter) LimitReached(key string) bool {
	ttl := l.GetTokenBucketExpirationTTL()

	if ttl <= 0 {
		ttl = l.generalExpirableOptions.DefaultExpirationTTL
	}

	return l.limitReachedWithTokenBucketTTL(key, ttl)
}
<|endoftext|>


<commit_before>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package thumbnail

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/logger"
	"github.com/andreaskoch/allmark2/common/route"
	"github.com/andreaskoch/allmark2/common/util/fsutil"
	"github.com/andreaskoch/allmark2/dataaccess"
	"github.com/andreaskoch/allmark2/services/imageconversion"
	"io"
	"path/filepath"
)

var (
	SizeSmall = ThumbDimension{
		MaxWidth:  200,
		MaxHeight: 0,
	}

	SizeMedium = ThumbDimension{
		MaxWidth:  400,
		MaxHeight: 0,
	}

	SizeLarge = ThumbDimension{
		MaxWidth:  800,
		MaxHeight: 0,
	}
)

func NewConversionService(logger logger.Logger, repository dataaccess.Repository, thumbnailIndex *Index) *ConversionService {

	// create a new conversion service
	conversionService := &ConversionService{
		logger:     logger,
		repository: repository,

		index:           thumbnailIndex,
		thumbnailFolder: thumbnailIndex.GetThumbnailFolder(),
	}

	// start the conversion
	conversionService.startConversion()

	return conversionService
}

type ConversionService struct {
	logger     logger.Logger
	repository dataaccess.Repository

	index           *Index
	thumbnailFolder string
}

// Start the conversion process.
func (conversion *ConversionService) startConversion() {

	// distinctive update
	conversion.repository.OnUpdate(func(route route.Route) {
		item := conversion.repository.Item(route)
		conversion.createThumbnailsForItem(item)
	})

	// full run
	go conversion.fullConversion()
}

// Process all items in the repository.
func (conversion *ConversionService) fullConversion() {
	for _, item := range conversion.repository.Items() {

		conversion.createThumbnailsForItem(item)

	}
}

// Create thumbnail for all image files found in the supplied item.
func (conversion *ConversionService) createThumbnailsForItem(item *dataaccess.Item) {

	if item == nil {
		return
	}

	for _, file := range item.Files() {

		// create the thumbnails
		conversion.createThumbnailsForFile(file)

	}

}

// Create thumbnail for all image files found in the supplied item.
func (conversion *ConversionService) createThumbnailsForFile(file *dataaccess.File) {

	// conversion.createThumbnail(file, SizeSmall)
	conversion.createThumbnail(file, SizeMedium)
	// conversion.createThumbnail(file, SizeLarge)

}

// Creates a thumbnail for the supplied file with the specified dimensions.
func (conversion *ConversionService) createThumbnail(file *dataaccess.File, dimensions ThumbDimension) {

	// get the mime type
	mimeType, err := file.MimeType()
	if err != nil {
		conversion.logger.Warn("Unable to detect mime type for file. Error: %s", err.Error())
		return
	}

	// check the mime type
	if !imageconversion.MimeTypeIsSupported(mimeType) {
		conversion.logger.Debug("The mime-type %q is currently not supported.", mimeType)
		return
	}

	// determine the file name
	fileExtension := imageconversion.GetFileExtensionFromMimeType(mimeType)
	filename := fmt.Sprintf("%s-%v-%v.%s", file.Id(), dimensions.MaxWidth, dimensions.MaxHeight, fileExtension)

	// assemble the full file route
	fullFileRoute, err := route.Combine(file.Parent(), file.Route())
	if err != nil {
		conversion.logger.Warn("Unable to combine routes %q and %q.", file.Parent(), file.Route())
		return
	}

	thumb := newThumb(fullFileRoute, conversion.thumbnailFolder, filename, dimensions)

	// check the index
	if conversion.isInIndex(thumb) {
		conversion.logger.Debug("Thumb %q already available in the index", thumb.String())
		return
	}

	// determine the file path
	filePath := filepath.Join(conversion.thumbnailFolder, filename)

	// open the target file
	target, fileError := fsutil.OpenFile(filePath)
	if fileError != nil {
		conversion.logger.Warn("Unable to detect mime type for file. Error: %s", fileError.Error())
		return
	}

	defer target.Close()

	// convert the image
	conversionError := file.Data(func(content io.ReadSeeker) error {
		return imageconversion.Resize(content, mimeType, dimensions.MaxWidth, dimensions.MaxHeight, target)
	})

	// handle errors
	if conversionError != nil {
		conversion.logger.Warn("Unable to create thumbnail for file %q. Error: %s", file, conversionError.Error())
		return
	}

	// add to index
	conversion.addToIndex(thumb)
	conversion.logger.Debug("Adding Thumb %q to index", thumb.String())
}

func (conversion *ConversionService) isInIndex(thumb Thumb) bool {

	// check if there are thumb for the route
	thumbs, entryExists := conversion.index.GetThumbs(thumb.Route)
	if !entryExists {
		return false
	}

	// check if there is a thumb with that dimensions
	if _, thumbExists := thumbs[thumb.Dimensions.String()]; thumbExists {
		// check if the file exists
		return fsutil.FileExists(filepath.Join(conversion.thumbnailFolder, thumb.Path))

	}

	return false
}

func (conversion *ConversionService) addToIndex(thumb Thumb) {
	thumbs, entryExists := conversion.index.GetThumbs(thumb.Route)
	if !entryExists {
		thumbs = make(Thumbs)
	}

	thumbs[thumb.Dimensions.String()] = thumb
	conversion.index.SetThumbs(thumb.Route, thumbs)
}
<commit_msg>Changed the thumbnail file check<commit_after>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package thumbnail

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/logger"
	"github.com/andreaskoch/allmark2/common/route"
	"github.com/andreaskoch/allmark2/common/util/fsutil"
	"github.com/andreaskoch/allmark2/dataaccess"
	"github.com/andreaskoch/allmark2/services/imageconversion"
	"io"
	"path/filepath"
)

var (
	SizeSmall = ThumbDimension{
		MaxWidth:  200,
		MaxHeight: 0,
	}

	SizeMedium = ThumbDimension{
		MaxWidth:  400,
		MaxHeight: 0,
	}

	SizeLarge = ThumbDimension{
		MaxWidth:  800,
		MaxHeight: 0,
	}
)

func NewConversionService(logger logger.Logger, repository dataaccess.Repository, thumbnailIndex *Index) *ConversionService {

	// create a new conversion service
	conversionService := &ConversionService{
		logger:     logger,
		repository: repository,

		index:           thumbnailIndex,
		thumbnailFolder: thumbnailIndex.GetThumbnailFolder(),
	}

	// start the conversion
	conversionService.startConversion()

	return conversionService
}

type ConversionService struct {
	logger     logger.Logger
	repository dataaccess.Repository

	index           *Index
	thumbnailFolder string
}

// Start the conversion process.
func (conversion *ConversionService) startConversion() {

	// distinctive update
	conversion.repository.OnUpdate(func(route route.Route) {
		item := conversion.repository.Item(route)
		conversion.createThumbnailsForItem(item)
	})

	// full run
	go conversion.fullConversion()
}

// Process all items in the repository.
func (conversion *ConversionService) fullConversion() {
	for _, item := range conversion.repository.Items() {

		conversion.createThumbnailsForItem(item)

	}
}

// Create thumbnail for all image files found in the supplied item.
func (conversion *ConversionService) createThumbnailsForItem(item *dataaccess.Item) {

	if item == nil {
		return
	}

	for _, file := range item.Files() {

		// create the thumbnails
		conversion.createThumbnailsForFile(file)

	}

}

// Create thumbnail for all image files found in the supplied item.
func (conversion *ConversionService) createThumbnailsForFile(file *dataaccess.File) {

	// conversion.createThumbnail(file, SizeSmall)
	conversion.createThumbnail(file, SizeMedium)
	// conversion.createThumbnail(file, SizeLarge)

}

// Creates a thumbnail for the supplied file with the specified dimensions.
func (conversion *ConversionService) createThumbnail(file *dataaccess.File, dimensions ThumbDimension) {

	// get the mime type
	mimeType, err := file.MimeType()
	if err != nil {
		conversion.logger.Warn("Unable to detect mime type for file. Error: %s", err.Error())
		return
	}

	// check the mime type
	if !imageconversion.MimeTypeIsSupported(mimeType) {
		conversion.logger.Debug("The mime-type %q is currently not supported.", mimeType)
		return
	}

	// determine the file name
	fileExtension := imageconversion.GetFileExtensionFromMimeType(mimeType)
	filename := fmt.Sprintf("%s-%v-%v.%s", file.Id(), dimensions.MaxWidth, dimensions.MaxHeight, fileExtension)

	// assemble the full file route
	fullFileRoute, err := route.Combine(file.Parent(), file.Route())
	if err != nil {
		conversion.logger.Warn("Unable to combine routes %q and %q.", file.Parent(), file.Route())
		return
	}

	thumb := newThumb(fullFileRoute, conversion.thumbnailFolder, filename, dimensions)

	// check the index
	if conversion.isInIndex(thumb) {
		conversion.logger.Debug("Thumb %q already available in the index", thumb.String())
		return
	}

	// determine the file path
	filePath := filepath.Join(conversion.thumbnailFolder, filename)

	// open the target file
	target, fileError := fsutil.OpenFile(filePath)
	if fileError != nil {
		conversion.logger.Warn("Unable to detect mime type for file. Error: %s", fileError.Error())
		return
	}

	defer target.Close()

	// convert the image
	conversionError := file.Data(func(content io.ReadSeeker) error {
		return imageconversion.Resize(content, mimeType, dimensions.MaxWidth, dimensions.MaxHeight, target)
	})

	// handle errors
	if conversionError != nil {
		conversion.logger.Warn("Unable to create thumbnail for file %q. Error: %s", file, conversionError.Error())
		return
	}

	// add to index
	conversion.addToIndex(thumb)
	conversion.logger.Debug("Adding Thumb %q to index", thumb.String())
}

func (conversion *ConversionService) isInIndex(thumb Thumb) bool {

	// check if there are thumb for the route
	thumbs, entryExists := conversion.index.GetThumbs(thumb.Route)
	if !entryExists {
		return false
	}

	// check if there is a thumb with that dimensions
	if _, thumbExists := thumbs[thumb.Dimensions.String()]; thumbExists {
		// check if the file exists
		thumbnailFilePath := conversion.index.GetThumbnailFilepath(thumb)
		return fsutil.FileExists(thumbnailFilePath)

	}

	return false
}

func (conversion *ConversionService) addToIndex(thumb Thumb) {
	thumbs, entryExists := conversion.index.GetThumbs(thumb.Route)
	if !entryExists {
		thumbs = make(Thumbs)
	}

	thumbs[thumb.Dimensions.String()] = thumb
	conversion.index.SetThumbs(thumb.Route, thumbs)
}
<|endoftext|>


<commit_before>// Package slack uses Norberto Lopes' slack library to implement the bot.Connector
// interface.
package slack

import (
	"fmt"
	"log"
	"sync"

	"github.com/lnxjedi/gopherbot/bot"
	"github.com/nlopes/slack"
)

type config struct {
	SlackToken      string // the 'bot token for connecting to Slack
	MaxMessageSplit int    // the maximum # of ~4000 byte messages to split a large message into
}

var lock sync.Mutex // package var lock
var started bool    // set when connector is started
// Map of bot IDs to unique names; note that for Slack, we have
// to store the botID in the name since bots aren't guaranteed to have
// unique names.
var bots = make(map[string]string)

func init() {
	bot.RegisterConnector("slack", Initialize)
}

// Initialize starts the connection, sets up and returns the connector object
func Initialize(robot bot.Handler, l *log.Logger) bot.Connector {
	lock.Lock()
	if started {
		lock.Unlock()
		return nil
	}
	started = true
	lock.Unlock()

	var c config

	err := robot.GetProtocolConfig(&c)
	if c.MaxMessageSplit == 0 {
		c.MaxMessageSplit = 1
	}
	if err != nil {
		robot.Log(bot.Fatal, fmt.Errorf("Unable to retrieve protocol configuration: %v", err))
	}

	api := slack.New(c.SlackToken)
	// This spits out a lot of extra stuff, so we only enable it when tracing
	if robot.GetLogLevel() == bot.Trace {
		api.SetDebug(true)
	}
	slack.SetLogger(l)

	sc := &slackConnector{api: api, conn: api.NewRTM(), maxMessageSplit: c.MaxMessageSplit}
	go sc.conn.ManageConnection()

	sc.Handler = robot

Loop:
	for {
		select {
		case msg := <-sc.conn.IncomingEvents:

			switch ev := msg.Data.(type) {

			case *slack.ConnectedEvent:
				sc.Log(bot.Debug, fmt.Sprintf("Infos: %T %v\n", ev, *ev.Info.User))
				sc.Log(bot.Debug, "Connection counter:", ev.ConnectionCount)
				sc.botName = ev.Info.User.Name
				sc.SetName(sc.botName)
				sc.Log(bot.Info, "Set bot name to", sc.botName)
				sc.botID = ev.Info.User.ID
				sc.Log(bot.Trace, "Set bot ID to", sc.botID)
				for _, b := range ev.Info.Bots {
					if b.Deleted {
						continue
					}
					name := fmt.Sprintf("bot:%s:%s", b.Name, b.ID)
					bots[b.ID] = name
				}
				break Loop

			case *slack.InvalidAuthEvent:
				sc.Log(bot.Fatal, "Invalid credentials")
			}
		}
	}

	sc.updateMaps()
	sc.botFullName = sc.userInfo[sc.botName].RealName
	sc.SetFullName(sc.botFullName)
	sc.Log(bot.Debug, "Set bot full name to", sc.botFullName)
	go sc.startSendLoop()

	return bot.Connector(sc)
}

func (sc *slackConnector) Run(stop chan struct{}) {
	sc.Lock()
	// This should never happen, just a bit of defensive coding
	if sc.running {
		sc.Unlock()
		return
	}
	sc.running = true
	sc.Unlock()
loop:
	for {
		select {
		case <-stop:
			sc.Log(bot.Debug, "Received stop in connector")
			break loop
		case msg := <-sc.conn.IncomingEvents:
			sc.Log(bot.Trace, fmt.Sprintf("Event Received (msg, data, type): %v; %v; %T", msg, msg.Data, msg.Data))
			switch ev := msg.Data.(type) {
			case *slack.HelloEvent:
				// Ignore hello
			case *slack.ChannelArchiveEvent, *slack.ChannelCreatedEvent, *slack.ChannelDeletedEvent, *slack.ChannelRenameEvent, *slack.TeamJoinEvent, *slack.GroupJoinedEvent, *slack.UserChangeEvent:
				sc.updateMaps()

			case *slack.MessageEvent:
				// Message processing is done concurrently
				go sc.processMessage(ev)

			case *slack.PresenceChangeEvent:
				sc.Log(bot.Debug, fmt.Sprintf("Presence Change: %v", ev))

			case *slack.LatencyReport:
				sc.Log(bot.Debug, fmt.Sprintf("Current latency: %v", ev.Value))

			case *slack.RTMError:
				sc.Log(bot.Debug, fmt.Sprintf("Error: %s\n", ev.Error()))

			default:

				// Ignore other events..
				// robot.Debug(fmt.Sprintf("Unexpected: %v\n", msg.Data)
			}
		}
	}
}
<commit_msg>Improve readability<commit_after>// Package slack uses Norberto Lopes' slack library to implement the bot.Connector
// interface.
package slack

import (
	"fmt"
	"log"
	"sync"

	"github.com/lnxjedi/gopherbot/bot"
	"github.com/nlopes/slack"
)

type config struct {
	SlackToken      string // the 'bot token for connecting to Slack
	MaxMessageSplit int    // the maximum # of ~4000 byte messages to split a large message into
}

var lock sync.Mutex // package var lock
var started bool    // set when connector is started
// Map of bot IDs to unique names; note that for Slack, we have
// to store the botID in the name since bots aren't guaranteed to have
// unique names.
var bots = make(map[string]string)

func init() {
	bot.RegisterConnector("slack", Initialize)
}

// Initialize starts the connection, sets up and returns the connector object
func Initialize(robot bot.Handler, l *log.Logger) bot.Connector {
	lock.Lock()
	if started {
		lock.Unlock()
		return nil
	}
	started = true
	lock.Unlock()

	var c config

	err := robot.GetProtocolConfig(&c)
	if err != nil {
		robot.Log(bot.Fatal, fmt.Errorf("Unable to retrieve protocol configuration: %v", err))
	}

	if c.MaxMessageSplit == 0 {
		c.MaxMessageSplit = 1
	}

	api := slack.New(c.SlackToken)
	// This spits out a lot of extra stuff, so we only enable it when tracing
	if robot.GetLogLevel() == bot.Trace {
		api.SetDebug(true)
	}
	slack.SetLogger(l)

	sc := &slackConnector{api: api, conn: api.NewRTM(), maxMessageSplit: c.MaxMessageSplit}
	go sc.conn.ManageConnection()

	sc.Handler = robot

Loop:
	for {
		select {
		case msg := <-sc.conn.IncomingEvents:

			switch ev := msg.Data.(type) {

			case *slack.ConnectedEvent:
				sc.Log(bot.Debug, fmt.Sprintf("Infos: %T %v\n", ev, *ev.Info.User))
				sc.Log(bot.Debug, "Connection counter:", ev.ConnectionCount)
				sc.botName = ev.Info.User.Name
				sc.SetName(sc.botName)
				sc.Log(bot.Info, "Set bot name to", sc.botName)
				sc.botID = ev.Info.User.ID
				sc.Log(bot.Trace, "Set bot ID to", sc.botID)
				for _, b := range ev.Info.Bots {
					if b.Deleted {
						continue
					}
					name := fmt.Sprintf("bot:%s:%s", b.Name, b.ID)
					bots[b.ID] = name
				}
				break Loop

			case *slack.InvalidAuthEvent:
				sc.Log(bot.Fatal, "Invalid credentials")
			}
		}
	}

	sc.updateMaps()
	sc.botFullName = sc.userInfo[sc.botName].RealName
	sc.SetFullName(sc.botFullName)
	sc.Log(bot.Debug, "Set bot full name to", sc.botFullName)
	go sc.startSendLoop()

	return bot.Connector(sc)
}

func (sc *slackConnector) Run(stop chan struct{}) {
	sc.Lock()
	// This should never happen, just a bit of defensive coding
	if sc.running {
		sc.Unlock()
		return
	}
	sc.running = true
	sc.Unlock()
loop:
	for {
		select {
		case <-stop:
			sc.Log(bot.Debug, "Received stop in connector")
			break loop
		case msg := <-sc.conn.IncomingEvents:
			sc.Log(bot.Trace, fmt.Sprintf("Event Received (msg, data, type): %v; %v; %T", msg, msg.Data, msg.Data))
			switch ev := msg.Data.(type) {
			case *slack.HelloEvent:
				// Ignore hello
			case *slack.ChannelArchiveEvent, *slack.ChannelCreatedEvent, *slack.ChannelDeletedEvent, *slack.ChannelRenameEvent, *slack.TeamJoinEvent, *slack.GroupJoinedEvent, *slack.UserChangeEvent:
				sc.updateMaps()

			case *slack.MessageEvent:
				// Message processing is done concurrently
				go sc.processMessage(ev)

			case *slack.PresenceChangeEvent:
				sc.Log(bot.Debug, fmt.Sprintf("Presence Change: %v", ev))

			case *slack.LatencyReport:
				sc.Log(bot.Debug, fmt.Sprintf("Current latency: %v", ev.Value))

			case *slack.RTMError:
				sc.Log(bot.Debug, fmt.Sprintf("Error: %s\n", ev.Error()))

			default:

				// Ignore other events..
				// robot.Debug(fmt.Sprintf("Unexpected: %v\n", msg.Data)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package syslog_test

import (
	"io/ioutil"
	"path/filepath"
	stdtesting "testing"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/agent"
	"launchpad.net/juju-core/log/syslog"
	syslogtesting "launchpad.net/juju-core/log/syslog/testing"
)

func Test(t *stdtesting.T) {
	gc.TestingT(t)
}

type SyslogConfigSuite struct {
	configDir string
}

var _ = gc.Suite(&SyslogConfigSuite{})

func (s *SyslogConfigSuite) SetUpTest(c *gc.C) {
	s.configDir = c.MkDir()
}

func (s *SyslogConfigSuite) assertRsyslogConfigPath(c *gc.C, slConfig *syslog.SyslogConfig) {
	slConfig.ConfigDir = s.configDir
	slConfig.ConfigFileName = "rsyslog.conf"
	c.Assert(slConfig.ConfigFilePath(), gc.Equals, filepath.Join(s.configDir, "rsyslog.conf"))
}

func (s *SyslogConfigSuite) assertRsyslogConfigContents(c *gc.C, slConfig *syslog.SyslogConfig,
	expectedConf string) {
	data, err := slConfig.Render()
	c.Assert(err, gc.IsNil)
	c.Assert(string(data), gc.Equals, expectedConf)
}

func (s *SyslogConfigSuite) TestAccumulateConfigRender(c *gc.C) {
	syslogConfigRenderer := syslog.NewAccumulateConfig("some-machine", agent.DefaultLogDir, 8888, "")
	s.assertRsyslogConfigContents(
		c,
		syslogConfigRenderer,
		syslogtesting.ExpectedAccumulateSyslogConf(c, "some-machine", "", 8888),
	)
}

func (s *SyslogConfigSuite) TestAccumulateConfigWrite(c *gc.C) {
	syslogConfigRenderer := syslog.NewAccumulateConfig("some-machine", agent.DefaultLogDir, 8888, "")
	syslogConfigRenderer.ConfigDir = s.configDir
	syslogConfigRenderer.ConfigFileName = "rsyslog.conf"
	s.assertRsyslogConfigPath(c, syslogConfigRenderer)
	err := syslogConfigRenderer.Write()
	c.Assert(err, gc.IsNil)
	syslogConfData, err := ioutil.ReadFile(syslogConfigRenderer.ConfigFilePath())
	c.Assert(err, gc.IsNil)
	c.Assert(
		string(syslogConfData),
		gc.Equals,
		syslogtesting.ExpectedAccumulateSyslogConf(c, "some-machine", "", 8888),
	)
}

func (s *SyslogConfigSuite) TestAccumulateConfigRenderWithNamespace(c *gc.C) {
	syslogConfigRenderer := syslog.NewAccumulateConfig("some-machine", agent.DefaultLogDir, 8888, "namespace")
	syslogConfigRenderer.LogDir += "-namespace"
	s.assertRsyslogConfigContents(
		c, syslogConfigRenderer, syslogtesting.ExpectedAccumulateSyslogConf(
			c, "some-machine", "namespace", 8888,
		),
	)
}

func (s *SyslogConfigSuite) TestForwardConfigRender(c *gc.C) {
	syslogConfigRenderer := syslog.NewForwardConfig(
		"some-machine", agent.DefaultLogDir, 999, "", []string{"server"},
	)
	s.assertRsyslogConfigContents(
		c, syslogConfigRenderer, syslogtesting.ExpectedForwardSyslogConf(
			c, "some-machine", agent.DefaultLogDir, "", "server", 999,
		),
	)
}

func (s *SyslogConfigSuite) TestForwardConfigRenderWithNamespace(c *gc.C) {
	syslogConfigRenderer := syslog.NewForwardConfig(
		"some-machine", agent.DefaultLogDir, 999, "namespace", []string{"server"},
	)
	s.assertRsyslogConfigContents(
		c, syslogConfigRenderer, syslogtesting.ExpectedForwardSyslogConf(
			c, "some-machine", agent.DefaultLogDir, "namespace", "server", 999,
		),
	)
}

func (s *SyslogConfigSuite) TestForwardConfigWrite(c *gc.C) {
	syslogConfigRenderer := syslog.NewForwardConfig(
		"some-machine", agent.DefaultLogDir, 999, "", []string{"server"},
	)
	syslogConfigRenderer.ConfigDir = s.configDir
	syslogConfigRenderer.ConfigFileName = "rsyslog.conf"
	s.assertRsyslogConfigPath(c, syslogConfigRenderer)
	err := syslogConfigRenderer.Write()
	c.Assert(err, gc.IsNil)
	syslogConfData, err := ioutil.ReadFile(syslogConfigRenderer.ConfigFilePath())
	c.Assert(err, gc.IsNil)
	c.Assert(
		string(syslogConfData),
		gc.Equals,
		syslogtesting.ExpectedForwardSyslogConf(
			c, "some-machine", agent.DefaultLogDir, "", "server", 999,
		),
	)
}
<commit_msg>Move the logged error func and remove the log dir.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package syslog_test

import (
	"io/ioutil"
	"path/filepath"
	stdtesting "testing"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/agent"
	"launchpad.net/juju-core/utils/syslog"
	syslogtesting "launchpad.net/juju-core/utils/syslog/testing"
)

func Test(t *stdtesting.T) {
	gc.TestingT(t)
}

type SyslogConfigSuite struct {
	configDir string
}

var _ = gc.Suite(&SyslogConfigSuite{})

func (s *SyslogConfigSuite) SetUpTest(c *gc.C) {
	s.configDir = c.MkDir()
}

func (s *SyslogConfigSuite) assertRsyslogConfigPath(c *gc.C, slConfig *syslog.SyslogConfig) {
	slConfig.ConfigDir = s.configDir
	slConfig.ConfigFileName = "rsyslog.conf"
	c.Assert(slConfig.ConfigFilePath(), gc.Equals, filepath.Join(s.configDir, "rsyslog.conf"))
}

func (s *SyslogConfigSuite) assertRsyslogConfigContents(c *gc.C, slConfig *syslog.SyslogConfig,
	expectedConf string) {
	data, err := slConfig.Render()
	c.Assert(err, gc.IsNil)
	c.Assert(string(data), gc.Equals, expectedConf)
}

func (s *SyslogConfigSuite) TestAccumulateConfigRender(c *gc.C) {
	syslogConfigRenderer := syslog.NewAccumulateConfig("some-machine", agent.DefaultLogDir, 8888, "")
	s.assertRsyslogConfigContents(
		c,
		syslogConfigRenderer,
		syslogtesting.ExpectedAccumulateSyslogConf(c, "some-machine", "", 8888),
	)
}

func (s *SyslogConfigSuite) TestAccumulateConfigWrite(c *gc.C) {
	syslogConfigRenderer := syslog.NewAccumulateConfig("some-machine", agent.DefaultLogDir, 8888, "")
	syslogConfigRenderer.ConfigDir = s.configDir
	syslogConfigRenderer.ConfigFileName = "rsyslog.conf"
	s.assertRsyslogConfigPath(c, syslogConfigRenderer)
	err := syslogConfigRenderer.Write()
	c.Assert(err, gc.IsNil)
	syslogConfData, err := ioutil.ReadFile(syslogConfigRenderer.ConfigFilePath())
	c.Assert(err, gc.IsNil)
	c.Assert(
		string(syslogConfData),
		gc.Equals,
		syslogtesting.ExpectedAccumulateSyslogConf(c, "some-machine", "", 8888),
	)
}

func (s *SyslogConfigSuite) TestAccumulateConfigRenderWithNamespace(c *gc.C) {
	syslogConfigRenderer := syslog.NewAccumulateConfig("some-machine", agent.DefaultLogDir, 8888, "namespace")
	syslogConfigRenderer.LogDir += "-namespace"
	s.assertRsyslogConfigContents(
		c, syslogConfigRenderer, syslogtesting.ExpectedAccumulateSyslogConf(
			c, "some-machine", "namespace", 8888,
		),
	)
}

func (s *SyslogConfigSuite) TestForwardConfigRender(c *gc.C) {
	syslogConfigRenderer := syslog.NewForwardConfig(
		"some-machine", agent.DefaultLogDir, 999, "", []string{"server"},
	)
	s.assertRsyslogConfigContents(
		c, syslogConfigRenderer, syslogtesting.ExpectedForwardSyslogConf(
			c, "some-machine", agent.DefaultLogDir, "", "server", 999,
		),
	)
}

func (s *SyslogConfigSuite) TestForwardConfigRenderWithNamespace(c *gc.C) {
	syslogConfigRenderer := syslog.NewForwardConfig(
		"some-machine", agent.DefaultLogDir, 999, "namespace", []string{"server"},
	)
	s.assertRsyslogConfigContents(
		c, syslogConfigRenderer, syslogtesting.ExpectedForwardSyslogConf(
			c, "some-machine", agent.DefaultLogDir, "namespace", "server", 999,
		),
	)
}

func (s *SyslogConfigSuite) TestForwardConfigWrite(c *gc.C) {
	syslogConfigRenderer := syslog.NewForwardConfig(
		"some-machine", agent.DefaultLogDir, 999, "", []string{"server"},
	)
	syslogConfigRenderer.ConfigDir = s.configDir
	syslogConfigRenderer.ConfigFileName = "rsyslog.conf"
	s.assertRsyslogConfigPath(c, syslogConfigRenderer)
	err := syslogConfigRenderer.Write()
	c.Assert(err, gc.IsNil)
	syslogConfData, err := ioutil.ReadFile(syslogConfigRenderer.ConfigFilePath())
	c.Assert(err, gc.IsNil)
	c.Assert(
		string(syslogConfData),
		gc.Equals,
		syslogtesting.ExpectedForwardSyslogConf(
			c, "some-machine", agent.DefaultLogDir, "", "server", 999,
		),
	)
}
<|endoftext|>


<commit_before>package nxcore

import (
	"encoding/json"
	"fmt"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/jaracil/ei"
	"github.com/jaracil/smartio"
	"golang.org/x/net/context"
)

const (
	ErrParse            = -32700
	ErrInvalidRequest   = -32600
	ErrMethodNotFound   = -32601
	ErrInvalidParams    = -32602
	ErrInternal         = -32603
	ErrTimeout          = -32000
	ErrCancel           = -32001
	ErrInvalidTask      = -32002
	ErrInvalidPipe      = -32003
	ErrInvalidUser      = -32004
	ErrUserExists       = -32005
	ErrPermissionDenied = -32010
	ErrTtlExpired       = -32011
)

var ErrStr = map[int]string{
	ErrParse:            "Parse error",
	ErrInvalidRequest:   "Invalid request",
	ErrMethodNotFound:   "Method not found",
	ErrInvalidParams:    "Invalid params",
	ErrInternal:         "Internal error",
	ErrTimeout:          "Timeout",
	ErrCancel:           "Cancel",
	ErrInvalidTask:      "Invalid task",
	ErrInvalidPipe:      "Invalid pipe",
	ErrInvalidUser:      "Invalid user",
	ErrUserExists:       "User already exists",
	ErrPermissionDenied: "Permission denied",
	ErrTtlExpired:       "TTL expired",
}

type JsonRpcErr struct {
	Cod  int         `json:"code"`
	Mess string      `json:"message"`
	Dat  interface{} `json:"data,omitempty"`
}

func (e *JsonRpcErr) Error() string {
	return fmt.Sprintf("[%d] %s", e.Cod, e.Mess)
}

func (e *JsonRpcErr) Code() int {
	return e.Cod
}

func (e *JsonRpcErr) Data() interface{} {
	return e.Dat
}

// NewJsonRpcErr creates new JSON-RPC error.
//
// code is the JSON-RPC error code.
// message is optional in case of well known error code (negative values).
// data is an optional extra info object.
func NewJsonRpcErr(code int, message string, data interface{}) error {
	if code < 0 {
		if message != "" {
			message = fmt.Sprintf("%s:[%s]", ErrStr[code], message)
		} else {
			message = ErrStr[code]
		}
	}

	return &JsonRpcErr{Cod: code, Mess: message, Dat: data}
}

type JsonRpcReq struct {
	Jsonrpc string      `json:"jsonrpc"`
	Id      uint64      `json:"id,omitempty"`
	Method  string      `json:"method"`
	Params  interface{} `json:"params"`
	nc      *NexusConn
}

type JsonRpcRes struct {
	Jsonrpc string      `json:"jsonrpc"`
	Id      uint64      `json:"id,omitempty"`
	Result  interface{} `json:"result,omitempty"`
	Error   *JsonRpcErr `json:"error,omitempty"`
}

// NexusConn represents the Nexus connection.
type NexusConn struct {
	connId       string
	conn         net.Conn
	connRx       *smartio.SmartReader
	connTx       *smartio.SmartWriter
	reqCount     uint64
	resTable     map[uint64]chan *JsonRpcRes
	resTableLock sync.Mutex
	chReq        chan *JsonRpcReq
	closed       int32 //Goroutine safe bool
	context      context.Context
	cancelFun    context.CancelFunc
	wdog         int64
	NexusVersion string
}

// Task represents a task pushed to Nexus.
type Task struct {
	nc           *NexusConn
	Id           string                 `json:"id"`
	Stat         string                 `json:"state""`
	Path         string                 `json:"path"`
	Prio         int                    `json:"priority"`
	Ttl          int                    `json:"ttl"`
	Detach       bool                   `json:"detached"`
	User         string                 `json:"user"`
	Method       string                 `json:"method"`
	Params       interface{}            `json:"params"`
	LocalId      interface{}            `json:"-"`
	Tses         string                 `json:"targetSession"`
	Result       interface{}            `json:"result"`
	ErrCode      *int                   `json:"errCode"`
	ErrStr       string                 `json:"errString"`
	ErrObj       interface{}            `json:"errObject"`
	Tags         map[string]interface{} `json:"tags"`
	CreationTime time.Time              `json:"creationTime"`
	DeadLine     time.Time              `json:"deadline"`
}

// TaskOpts represents task push options.
type TaskOpts struct {
	// Task priority default 0 (Set negative value for lower priority)
	Priority int
	// Task ttl default 5
	Ttl int
	// Task detach. If true, task is detached from creating session.
	// If task is detached and creating session deads, task is not removed from tasks queue.
	Detach bool
}

// Pipe represents a pipe.
// Pipes can only be read from the connection that created them and
// can be written from any conection.
type Pipe struct {
	nc        *NexusConn
	pipeId    string
	context   context.Context
	cancelFun context.CancelFunc
}

// Msg represents a pipe single message
type Msg struct {
	Count int64       // Message counter (unique and correlative)
	Msg   interface{} // Pipe message
}

// PipeData represents a pipe messages group obtained in read ops.
type PipeData struct {
	Msgs    []*Msg // Messages
	Waiting int    // Number of messages waiting in Nexus server since last read
	Drops   int    // Number of messages dropped (pipe overflows) since last read
}

// TopicMsg represents a single topic message
type TopicMsg struct {
	Topic string      // Topic the message was published to
	Count int64       // Message counter (unique and correlative)
	Msg   interface{} // The message itself
}

// TopicData represents a topic messages group obtained in read ops.
type TopicData struct {
	Msgs    []*TopicMsg // Messages
	Waiting int         // Number of messages waiting in Nexus server since last read
	Drops   int         // Number of messages dropped (pipe overflows) since last read
}

// PipeOpts represents pipe creation options
type PipeOpts struct {
	Length int // Pipe buffer capacity
}

// NewNexusConn creates new nexus connection from net.conn
func NewNexusConn(conn net.Conn) *NexusConn {
	nc := &NexusConn{
		conn:     conn,
		connRx:   smartio.NewSmartReader(conn),
		connTx:   smartio.NewSmartWriter(conn),
		resTable: make(map[uint64]chan *JsonRpcRes),
		chReq:    make(chan *JsonRpcReq, 16),
		wdog:     60,
	}
	nc.context, nc.cancelFun = context.WithCancel(context.Background())
	go nc.sendWorker()
	go nc.recvWorker()
	go nc.mainWorker()
	return nc
}

func (nc *NexusConn) pushReq(req *JsonRpcReq) (err error) {
	select {
	case nc.chReq <- req:
	case <-nc.context.Done():
		err = NewJsonRpcErr(ErrCancel, "", nil)
	}
	return
}

func (nc *NexusConn) pullReq() (req *JsonRpcReq, err error) {
	select {
	case req = <-nc.chReq:
	case <-nc.context.Done():
		err = NewJsonRpcErr(ErrCancel, "", nil)
	}
	return
}

func (nc *NexusConn) mainWorker() {
	defer nc.Close()
	tick := time.NewTicker(10 * time.Second)
	defer tick.Stop()
	for {
		select {
		case <-tick.C:
			now := time.Now().Unix()
			wdog := atomic.LoadInt64(&nc.wdog)
			max := wdog * 2
			if (now-nc.connRx.GetLast() > max) &&
				(now-nc.connTx.GetLast() > max) {
				return
			}
			if (now-nc.connRx.GetLast() >= wdog) &&
				(now-nc.connTx.GetLast() >= wdog) {
				if err := nc.Ping(10 * time.Second); err != nil {
					if time.Now().Unix()-nc.connTx.GetLast() > 5 {
						return
					}
				}
			}
		case <-nc.context.Done():
			return
		}
	}
}

func (nc *NexusConn) sendWorker() {
	defer nc.Close()
	for {
		req, err := nc.pullReq()
		if err != nil {
			break
		}
		req.Jsonrpc = "2.0"
		buf, err := json.Marshal(req)
		if err != nil {
			break
		}
		if !nc.Closed() {
			_, err = nc.connTx.Write(buf)
			if err != nil {
				break
			}
		} else {
			break
		}
	}
}

func (nc *NexusConn) recvWorker() {
	defer nc.Close()
	dec := json.NewDecoder(nc.connRx)
	for dec.More() {
		res := &JsonRpcRes{}
		err := dec.Decode(res)
		if err != nil {
			break
		}
		nc.resTableLock.Lock()
		ch := nc.resTable[res.Id]
		nc.resTableLock.Unlock()
		if ch != nil {
			select {
			case ch <- res:
			default:
			}
		}
	}
}

func (nc *NexusConn) newId() (uint64, chan *JsonRpcRes) {
	id := atomic.AddUint64(&nc.reqCount, 1)
	ch := make(chan *JsonRpcRes, 1)
	nc.resTableLock.Lock()
	defer nc.resTableLock.Unlock()
	nc.resTable[id] = ch
	return id, ch
}

func (nc *NexusConn) delId(id uint64) {
	nc.resTableLock.Lock()
	delete(nc.resTable, id)
	nc.resTableLock.Unlock()
}

// GetContext returns internal connection context.
func (nc *NexusConn) GetContext() context.Context {
	return nc.context
}

// Close closes nexus connection.
func (nc *NexusConn) Close() {
	if atomic.CompareAndSwapInt32(&nc.closed, 0, 1) {
		nc.cancelFun()
		nc.conn.Close()
	}
}

// Closed returns Nexus connection state.
func (nc *NexusConn) Closed() bool {
	return atomic.LoadInt32(&nc.closed) == 1
}

// ExecNoWait is a low level JSON-RPC call function, it don't wait response from server.
func (nc *NexusConn) ExecNoWait(method string, params interface{}) (id uint64, rch chan *JsonRpcRes, err error) {
	if nc.Closed() {
		err = NewJsonRpcErr(ErrCancel, "", nil)
		return
	}
	id, rch = nc.newId()
	req := &JsonRpcReq{
		Id:     id,
		Method: method,
		Params: params,
	}
	err = nc.pushReq(req)
	if err != nil {
		nc.delId(id)
		return 0, nil, err
	}
	return
}

// Exec is a low level JSON-RPC call function.
func (nc *NexusConn) Exec(method string, params interface{}) (result interface{}, err error) {
	id, rch, err := nc.ExecNoWait(method, params)
	if err != nil {
		return nil, err
	}
	defer nc.delId(id)
	select {
	case res := <-rch:
		if res.Error != nil {
			err = res.Error
		} else {
			result = res.Result
		}
	case <-nc.context.Done():
		err = NewJsonRpcErr(ErrCancel, "", nil)
	}
	return
}

// Ping pings Nexus server, timeout is the max time waiting for server response,
// after that ErrTimeout is returned.
func (nc *NexusConn) Ping(timeout time.Duration) (err error) {
	id, rch, err := nc.ExecNoWait("sys.ping", nil)
	if err != nil {
		return
	}
	defer nc.delId(id)
	select {
	case <-rch:
		break
	case <-time.After(timeout):
		err = NewJsonRpcErr(ErrTimeout, "", nil)
	case <-nc.context.Done():
		err = NewJsonRpcErr(ErrCancel, "", nil)
	}
	return
}

// Login attempts to login using user and pass.
// Returns the response object from Nexus or error.
func (nc *NexusConn) Login(user string, pass string) (interface{}, error) {
	par := map[string]interface{}{
		"user": user,
		"pass": pass,
	}
	res, err := nc.Exec("sys.login", par)
	if err != nil {
		return nil, err
	}
	nc.connId = ei.N(res).M("connId").StringZ()
	return res, nil
}

// Reload forces the node owner of the client connection to reload its info (tags)
// Returns the response object from Nexus or error.
func (nc *NexusConn) Reload() (interface{}, error) {
	return nc.Exec("sys.reload", nil)
}

// Id returns the connection id after a login.
func (nc *NexusConn) Id() string {
	return nc.connId
}<commit_msg>Remove Reload() and use SessionReload()<commit_after>package nxcore

import (
	"encoding/json"
	"fmt"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/jaracil/ei"
	"github.com/jaracil/smartio"
	"golang.org/x/net/context"
)

const (
	ErrParse            = -32700
	ErrInvalidRequest   = -32600
	ErrMethodNotFound   = -32601
	ErrInvalidParams    = -32602
	ErrInternal         = -32603
	ErrTimeout          = -32000
	ErrCancel           = -32001
	ErrInvalidTask      = -32002
	ErrInvalidPipe      = -32003
	ErrInvalidUser      = -32004
	ErrUserExists       = -32005
	ErrPermissionDenied = -32010
	ErrTtlExpired       = -32011
)

var ErrStr = map[int]string{
	ErrParse:            "Parse error",
	ErrInvalidRequest:   "Invalid request",
	ErrMethodNotFound:   "Method not found",
	ErrInvalidParams:    "Invalid params",
	ErrInternal:         "Internal error",
	ErrTimeout:          "Timeout",
	ErrCancel:           "Cancel",
	ErrInvalidTask:      "Invalid task",
	ErrInvalidPipe:      "Invalid pipe",
	ErrInvalidUser:      "Invalid user",
	ErrUserExists:       "User already exists",
	ErrPermissionDenied: "Permission denied",
	ErrTtlExpired:       "TTL expired",
}

type JsonRpcErr struct {
	Cod  int         `json:"code"`
	Mess string      `json:"message"`
	Dat  interface{} `json:"data,omitempty"`
}

func (e *JsonRpcErr) Error() string {
	return fmt.Sprintf("[%d] %s", e.Cod, e.Mess)
}

func (e *JsonRpcErr) Code() int {
	return e.Cod
}

func (e *JsonRpcErr) Data() interface{} {
	return e.Dat
}

// NewJsonRpcErr creates new JSON-RPC error.
//
// code is the JSON-RPC error code.
// message is optional in case of well known error code (negative values).
// data is an optional extra info object.
func NewJsonRpcErr(code int, message string, data interface{}) error {
	if code < 0 {
		if message != "" {
			message = fmt.Sprintf("%s:[%s]", ErrStr[code], message)
		} else {
			message = ErrStr[code]
		}
	}

	return &JsonRpcErr{Cod: code, Mess: message, Dat: data}
}

type JsonRpcReq struct {
	Jsonrpc string      `json:"jsonrpc"`
	Id      uint64      `json:"id,omitempty"`
	Method  string      `json:"method"`
	Params  interface{} `json:"params"`
	nc      *NexusConn
}

type JsonRpcRes struct {
	Jsonrpc string      `json:"jsonrpc"`
	Id      uint64      `json:"id,omitempty"`
	Result  interface{} `json:"result,omitempty"`
	Error   *JsonRpcErr `json:"error,omitempty"`
}

// NexusConn represents the Nexus connection.
type NexusConn struct {
	connId       string
	conn         net.Conn
	connRx       *smartio.SmartReader
	connTx       *smartio.SmartWriter
	reqCount     uint64
	resTable     map[uint64]chan *JsonRpcRes
	resTableLock sync.Mutex
	chReq        chan *JsonRpcReq
	closed       int32 //Goroutine safe bool
	context      context.Context
	cancelFun    context.CancelFunc
	wdog         int64
	NexusVersion string
}

// Task represents a task pushed to Nexus.
type Task struct {
	nc           *NexusConn
	Id           string                 `json:"id"`
	Stat         string                 `json:"state""`
	Path         string                 `json:"path"`
	Prio         int                    `json:"priority"`
	Ttl          int                    `json:"ttl"`
	Detach       bool                   `json:"detached"`
	User         string                 `json:"user"`
	Method       string                 `json:"method"`
	Params       interface{}            `json:"params"`
	LocalId      interface{}            `json:"-"`
	Tses         string                 `json:"targetSession"`
	Result       interface{}            `json:"result"`
	ErrCode      *int                   `json:"errCode"`
	ErrStr       string                 `json:"errString"`
	ErrObj       interface{}            `json:"errObject"`
	Tags         map[string]interface{} `json:"tags"`
	CreationTime time.Time              `json:"creationTime"`
	DeadLine     time.Time              `json:"deadline"`
}

// TaskOpts represents task push options.
type TaskOpts struct {
	// Task priority default 0 (Set negative value for lower priority)
	Priority int
	// Task ttl default 5
	Ttl int
	// Task detach. If true, task is detached from creating session.
	// If task is detached and creating session deads, task is not removed from tasks queue.
	Detach bool
}

// Pipe represents a pipe.
// Pipes can only be read from the connection that created them and
// can be written from any conection.
type Pipe struct {
	nc        *NexusConn
	pipeId    string
	context   context.Context
	cancelFun context.CancelFunc
}

// Msg represents a pipe single message
type Msg struct {
	Count int64       // Message counter (unique and correlative)
	Msg   interface{} // Pipe message
}

// PipeData represents a pipe messages group obtained in read ops.
type PipeData struct {
	Msgs    []*Msg // Messages
	Waiting int    // Number of messages waiting in Nexus server since last read
	Drops   int    // Number of messages dropped (pipe overflows) since last read
}

// TopicMsg represents a single topic message
type TopicMsg struct {
	Topic string      // Topic the message was published to
	Count int64       // Message counter (unique and correlative)
	Msg   interface{} // The message itself
}

// TopicData represents a topic messages group obtained in read ops.
type TopicData struct {
	Msgs    []*TopicMsg // Messages
	Waiting int         // Number of messages waiting in Nexus server since last read
	Drops   int         // Number of messages dropped (pipe overflows) since last read
}

// PipeOpts represents pipe creation options
type PipeOpts struct {
	Length int // Pipe buffer capacity
}

// NewNexusConn creates new nexus connection from net.conn
func NewNexusConn(conn net.Conn) *NexusConn {
	nc := &NexusConn{
		conn:     conn,
		connRx:   smartio.NewSmartReader(conn),
		connTx:   smartio.NewSmartWriter(conn),
		resTable: make(map[uint64]chan *JsonRpcRes),
		chReq:    make(chan *JsonRpcReq, 16),
		wdog:     60,
	}
	nc.context, nc.cancelFun = context.WithCancel(context.Background())
	go nc.sendWorker()
	go nc.recvWorker()
	go nc.mainWorker()
	return nc
}

func (nc *NexusConn) pushReq(req *JsonRpcReq) (err error) {
	select {
	case nc.chReq <- req:
	case <-nc.context.Done():
		err = NewJsonRpcErr(ErrCancel, "", nil)
	}
	return
}

func (nc *NexusConn) pullReq() (req *JsonRpcReq, err error) {
	select {
	case req = <-nc.chReq:
	case <-nc.context.Done():
		err = NewJsonRpcErr(ErrCancel, "", nil)
	}
	return
}

func (nc *NexusConn) mainWorker() {
	defer nc.Close()
	tick := time.NewTicker(10 * time.Second)
	defer tick.Stop()
	for {
		select {
		case <-tick.C:
			now := time.Now().Unix()
			wdog := atomic.LoadInt64(&nc.wdog)
			max := wdog * 2
			if (now-nc.connRx.GetLast() > max) &&
				(now-nc.connTx.GetLast() > max) {
				return
			}
			if (now-nc.connRx.GetLast() >= wdog) &&
				(now-nc.connTx.GetLast() >= wdog) {
				if err := nc.Ping(10 * time.Second); err != nil {
					if time.Now().Unix()-nc.connTx.GetLast() > 5 {
						return
					}
				}
			}
		case <-nc.context.Done():
			return
		}
	}
}

func (nc *NexusConn) sendWorker() {
	defer nc.Close()
	for {
		req, err := nc.pullReq()
		if err != nil {
			break
		}
		req.Jsonrpc = "2.0"
		buf, err := json.Marshal(req)
		if err != nil {
			break
		}
		if !nc.Closed() {
			_, err = nc.connTx.Write(buf)
			if err != nil {
				break
			}
		} else {
			break
		}
	}
}

func (nc *NexusConn) recvWorker() {
	defer nc.Close()
	dec := json.NewDecoder(nc.connRx)
	for dec.More() {
		res := &JsonRpcRes{}
		err := dec.Decode(res)
		if err != nil {
			break
		}
		nc.resTableLock.Lock()
		ch := nc.resTable[res.Id]
		nc.resTableLock.Unlock()
		if ch != nil {
			select {
			case ch <- res:
			default:
			}
		}
	}
}

func (nc *NexusConn) newId() (uint64, chan *JsonRpcRes) {
	id := atomic.AddUint64(&nc.reqCount, 1)
	ch := make(chan *JsonRpcRes, 1)
	nc.resTableLock.Lock()
	defer nc.resTableLock.Unlock()
	nc.resTable[id] = ch
	return id, ch
}

func (nc *NexusConn) delId(id uint64) {
	nc.resTableLock.Lock()
	delete(nc.resTable, id)
	nc.resTableLock.Unlock()
}

// GetContext returns internal connection context.
func (nc *NexusConn) GetContext() context.Context {
	return nc.context
}

// Close closes nexus connection.
func (nc *NexusConn) Close() {
	if atomic.CompareAndSwapInt32(&nc.closed, 0, 1) {
		nc.cancelFun()
		nc.conn.Close()
	}
}

// Closed returns Nexus connection state.
func (nc *NexusConn) Closed() bool {
	return atomic.LoadInt32(&nc.closed) == 1
}

// ExecNoWait is a low level JSON-RPC call function, it don't wait response from server.
func (nc *NexusConn) ExecNoWait(method string, params interface{}) (id uint64, rch chan *JsonRpcRes, err error) {
	if nc.Closed() {
		err = NewJsonRpcErr(ErrCancel, "", nil)
		return
	}
	id, rch = nc.newId()
	req := &JsonRpcReq{
		Id:     id,
		Method: method,
		Params: params,
	}
	err = nc.pushReq(req)
	if err != nil {
		nc.delId(id)
		return 0, nil, err
	}
	return
}

// Exec is a low level JSON-RPC call function.
func (nc *NexusConn) Exec(method string, params interface{}) (result interface{}, err error) {
	id, rch, err := nc.ExecNoWait(method, params)
	if err != nil {
		return nil, err
	}
	defer nc.delId(id)
	select {
	case res := <-rch:
		if res.Error != nil {
			err = res.Error
		} else {
			result = res.Result
		}
	case <-nc.context.Done():
		err = NewJsonRpcErr(ErrCancel, "", nil)
	}
	return
}

// Ping pings Nexus server, timeout is the max time waiting for server response,
// after that ErrTimeout is returned.
func (nc *NexusConn) Ping(timeout time.Duration) (err error) {
	id, rch, err := nc.ExecNoWait("sys.ping", nil)
	if err != nil {
		return
	}
	defer nc.delId(id)
	select {
	case <-rch:
		break
	case <-time.After(timeout):
		err = NewJsonRpcErr(ErrTimeout, "", nil)
	case <-nc.context.Done():
		err = NewJsonRpcErr(ErrCancel, "", nil)
	}
	return
}

// Login attempts to login using user and pass.
// Returns the response object from Nexus or error.
func (nc *NexusConn) Login(user string, pass string) (interface{}, error) {
	par := map[string]interface{}{
		"user": user,
		"pass": pass,
	}
	res, err := nc.Exec("sys.login", par)
	if err != nil {
		return nil, err
	}
	nc.connId = ei.N(res).M("connId").StringZ()
	return res, nil
}

// Id returns the connection id after a login.
func (nc *NexusConn) Id() string {
	return nc.connId
}<|endoftext|>


<commit_before>// Package jenkins implements some helper functions to use during
// tests. Many times certain facilities are not available, or tests
// must run differently.
package jenkins

import (
	"os"
	"strings"
)

// EnvVar is a type to use travis-only env var names with
// the type system.
type EnvVar string

// Environment variables that TravisCI uses.
const (
	VarBuildNumber    EnvVar = "BUILD_NUMBER"
	VarBuildId        EnvVar = "BUILD_ID"
	VarBuildUrl       EnvVar = "BUILD_URL"
	VarNodeName       EnvVar = "NODE_NAME"
	VarJobName        EnvVar = "JOB_NAME"
	VarBuildTag       EnvVar = "BUILD_TAG"
	VarJenkinsUrl     EnvVar = "JENKINS_URL"
	VarExecutorNumber EnvVar = "EXECUTOR_NUMBER"
	VarJavaHome       EnvVar = "JAVA_HOME"
	VarWorkspace      EnvVar = "WORKSPACE"
	VarSvnRevision    EnvVar = "SVN_REVISION"
	VarCvsBranch      EnvVar = "CVS_BRANCH"
	VarGitCommit      EnvVar = "GIT_COMMIT"
	VarGitUrl         EnvVar = "GIT_URL"
	VarGitBranch      EnvVar = "GIT_BRANCH"
)

// IsRunning attempts to determine whether this process is
// running on Jenkins CI. This is done by checking any of the
// following:
//
//  JENKINS_URL is set
//  BuildTag has prefix "jenkins-"
//
func IsRunning() bool {
	return len(Env(VarJenkinsUrl)) > 0 || strings.HasPrefix(Env(VarBuildTag), "jenkins-")
}

// Env returns the value of a travis env variable.
func Env(v EnvVar) string {
	return os.Getenv(string(v))
}

// JobName returns the jenkins JOB_NAME of this build.
func JobName() string {
	return Env(VarJobName)
}

// BuildTag returns the jenkins BUILD_TAG.
func BuildTag() string {
	return Env(VarBuildTag)
}
<commit_msg>Update jenkins.go<commit_after>// Package jenkins implements some helper functions to use during
// tests. Many times certain facilities are not available, or tests
// must run differently.
package jenkins

import (
	"os"
	"strings"
)

// EnvVar is a type to use travis-only env var names with
// the type system.
type EnvVar string

// Environment variables that Jenkins uses.
const (
	VarBuildNumber    EnvVar = "BUILD_NUMBER"
	VarBuildId        EnvVar = "BUILD_ID"
	VarBuildUrl       EnvVar = "BUILD_URL"
	VarNodeName       EnvVar = "NODE_NAME"
	VarJobName        EnvVar = "JOB_NAME"
	VarBuildTag       EnvVar = "BUILD_TAG"
	VarJenkinsUrl     EnvVar = "JENKINS_URL"
	VarExecutorNumber EnvVar = "EXECUTOR_NUMBER"
	VarJavaHome       EnvVar = "JAVA_HOME"
	VarWorkspace      EnvVar = "WORKSPACE"
	VarSvnRevision    EnvVar = "SVN_REVISION"
	VarCvsBranch      EnvVar = "CVS_BRANCH"
	VarGitCommit      EnvVar = "GIT_COMMIT"
	VarGitUrl         EnvVar = "GIT_URL"
	VarGitBranch      EnvVar = "GIT_BRANCH"
)

// IsRunning attempts to determine whether this process is
// running on Jenkins CI. This is done by checking any of the
// following:
//
//  JENKINS_URL is set
//  BuildTag has prefix "jenkins-"
//
func IsRunning() bool {
	return len(Env(VarJenkinsUrl)) > 0 || strings.HasPrefix(Env(VarBuildTag), "jenkins-")
}

// Env returns the value of a travis env variable.
func Env(v EnvVar) string {
	return os.Getenv(string(v))
}

// JobName returns the jenkins JOB_NAME of this build.
func JobName() string {
	return Env(VarJobName)
}

// BuildTag returns the jenkins BUILD_TAG.
func BuildTag() string {
	return Env(VarBuildTag)
}
<|endoftext|>


<commit_before>// Copyright 2014 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"math"
	"math/rand"
	"sort"
	"sync"
	"testing"
	"testing/quick"
	"time"

	dto "github.com/prometheus/client_model/go"
)

func benchmarkSummaryObserve(w int, b *testing.B) {
	b.StopTimer()

	wg := new(sync.WaitGroup)
	wg.Add(w)

	g := new(sync.WaitGroup)
	g.Add(1)

	s := NewSummary(SummaryOpts{})

	for i := 0; i < w; i++ {
		go func() {
			g.Wait()

			for i := 0; i < b.N; i++ {
				s.Observe(float64(i))
			}

			wg.Done()
		}()
	}

	b.StartTimer()
	g.Done()
	wg.Wait()
}

func BenchmarkSummaryObserve1(b *testing.B) {
	benchmarkSummaryObserve(1, b)
}

func BenchmarkSummaryObserve2(b *testing.B) {
	benchmarkSummaryObserve(2, b)
}

func BenchmarkSummaryObserve4(b *testing.B) {
	benchmarkSummaryObserve(4, b)
}

func BenchmarkSummaryObserve8(b *testing.B) {
	benchmarkSummaryObserve(8, b)
}

func benchmarkSummaryWrite(w int, b *testing.B) {
	b.StopTimer()

	wg := new(sync.WaitGroup)
	wg.Add(w)

	g := new(sync.WaitGroup)
	g.Add(1)

	s := NewSummary(SummaryOpts{})

	for i := 0; i < 1000000; i++ {
		s.Observe(float64(i))
	}

	for j := 0; j < w; j++ {
		outs := make([]dto.Metric, b.N)

		go func(o []dto.Metric) {
			g.Wait()

			for i := 0; i < b.N; i++ {
				s.Write(&o[i])
			}

			wg.Done()
		}(outs)
	}

	b.StartTimer()
	g.Done()
	wg.Wait()
}

func BenchmarkSummaryWrite1(b *testing.B) {
	benchmarkSummaryWrite(1, b)
}

func BenchmarkSummaryWrite2(b *testing.B) {
	benchmarkSummaryWrite(2, b)
}

func BenchmarkSummaryWrite4(b *testing.B) {
	benchmarkSummaryWrite(4, b)
}

func BenchmarkSummaryWrite8(b *testing.B) {
	benchmarkSummaryWrite(8, b)
}

func TestSummaryConcurrency(t *testing.T) {
	rand.Seed(42)

	it := func(n uint32) bool {
		mutations := int(n%10000 + 1e4)
		concLevel := int(n%5 + 1)
		total := mutations * concLevel

		var start, end sync.WaitGroup
		start.Add(1)
		end.Add(concLevel)

		sum := NewSummary(SummaryOpts{
			Name: "test_summary",
			Help: "helpless",
		})

		allVars := make([]float64, total)
		var sampleSum float64
		for i := 0; i < concLevel; i++ {
			vals := make([]float64, mutations)
			for j := 0; j < mutations; j++ {
				v := rand.NormFloat64()
				vals[j] = v
				allVars[i*mutations+j] = v
				sampleSum += v
			}

			go func(vals []float64) {
				start.Wait()
				for _, v := range vals {
					sum.Observe(v)
				}
				end.Done()
			}(vals)
		}
		sort.Float64s(allVars)
		start.Done()
		end.Wait()

		m := &dto.Metric{}
		sum.Write(m)
		if got, want := int(*m.Summary.SampleCount), total; got != want {
			t.Errorf("got sample count %d, want %d", got, want)
		}
		if got, want := *m.Summary.SampleSum, sampleSum; math.Abs((got-want)/want) > 0.001 {
			t.Errorf("got sample sum %f, want %f", got, want)
		}

		objectives := make([]float64, 0, len(DefObjectives))
		for qu := range DefObjectives {
			objectives = append(objectives, qu)
		}
		sort.Float64s(objectives)

		for i, wantQ := range objectives {
			ε := DefObjectives[wantQ]
			gotQ := *m.Summary.Quantile[i].Quantile
			gotV := *m.Summary.Quantile[i].Value
			min, max := getBounds(allVars, wantQ, ε)
			if gotQ != wantQ {
				t.Errorf("got quantile %f, want %f", gotQ, wantQ)
			}
			if gotV < min || gotV > max {
				t.Errorf("got %f for quantile %f, want [%f,%f]", gotV, gotQ, min, max)
			}
		}
		return true
	}

	if err := quick.Check(it, nil); err != nil {
		t.Error(err)
	}
}

func TestSummaryVecConcurrency(t *testing.T) {
	rand.Seed(42)

	objectives := make([]float64, 0, len(DefObjectives))
	for qu := range DefObjectives {

		objectives = append(objectives, qu)
	}
	sort.Float64s(objectives)

	it := func(n uint32) bool {
		mutations := int(n%10000 + 1e4)
		concLevel := int(n%7 + 1)
		vecLength := int(n%3 + 1)

		var start, end sync.WaitGroup
		start.Add(1)
		end.Add(concLevel)

		sum := NewSummaryVec(
			SummaryOpts{
				Name: "test_summary",
				Help: "helpless",
			},
			[]string{"label"},
		)

		allVars := make([][]float64, vecLength)
		sampleSums := make([]float64, vecLength)
		for i := 0; i < concLevel; i++ {
			vals := make([]float64, mutations)
			picks := make([]int, mutations)
			for j := 0; j < mutations; j++ {
				v := rand.NormFloat64()
				vals[j] = v
				pick := rand.Intn(vecLength)
				picks[j] = pick
				allVars[pick] = append(allVars[pick], v)
				sampleSums[pick] += v
			}

			go func(vals []float64) {
				start.Wait()
				for i, v := range vals {
					sum.WithLabelValues(string('A' + picks[i])).Observe(v)
				}
				end.Done()
			}(vals)
		}
		for _, vars := range allVars {
			sort.Float64s(vars)
		}
		start.Done()
		end.Wait()

		for i := 0; i < vecLength; i++ {
			m := &dto.Metric{}
			s := sum.WithLabelValues(string('A' + i))
			s.Write(m)
			if got, want := int(*m.Summary.SampleCount), len(allVars[i]); got != want {
				t.Errorf("got sample count %d for label %c, want %d", got, 'A'+i, want)
			}
			if got, want := *m.Summary.SampleSum, sampleSums[i]; math.Abs((got-want)/want) > 0.001 {
				t.Errorf("got sample sum %f for label %c, want %f", got, 'A'+i, want)
			}
			for j, wantQ := range objectives {
				ε := DefObjectives[wantQ]
				gotQ := *m.Summary.Quantile[j].Quantile
				gotV := *m.Summary.Quantile[j].Value
				min, max := getBounds(allVars[i], wantQ, ε)
				if gotQ != wantQ {
					t.Errorf("got quantile %f for label %c, want %f", gotQ, 'A'+i, wantQ)
				}
				if gotV < min || gotV > max {
					t.Errorf("got %f for quantile %f for label %c, want [%f,%f]", gotV, gotQ, 'A'+i, min, max)
				}
			}
		}
		return true
	}

	if err := quick.Check(it, nil); err != nil {
		t.Error(err)
	}
}

func TestSummaryDecay(t *testing.T) {
	sum := NewSummary(SummaryOpts{
		Name:       "test_summary",
		Help:       "helpless",
		MaxAge:     100 * time.Millisecond,
		Objectives: map[float64]float64{0.1: 0.001},
		AgeBuckets: 10,
	})

	m := &dto.Metric{}
	i := 0
	tick := time.NewTicker(time.Millisecond)
	for _ = range tick.C {
		i++
		sum.Observe(float64(i))
		if i%10 == 0 {
			sum.Write(m)
			if got, want := *m.Summary.Quantile[0].Value, math.Max(float64(i)/10, float64(i-90)); math.Abs(got-want) > 20 {
				t.Errorf("%d. got %f, want %f", i, got, want)
			}
			m.Reset()
		}
		if i >= 1000 {
			break
		}
	}
	tick.Stop()
}

func getBounds(vars []float64, q, ε float64) (min, max float64) {
	// TODO: This currently tolerates an error of up to 2*ε. The error must
	// be at most ε, but for some reason, it's sometimes slightly
	// higher. That's a bug.
	n := float64(len(vars))
	lower := int((q - 2*ε) * n)
	upper := int(math.Ceil((q + 2*ε) * n))
	min = vars[0]
	if lower > 1 {
		min = vars[lower-1]
	}
	max = vars[len(vars)-1]
	if upper < len(vars) {
		max = vars[upper-1]
	}
	return
}
<commit_msg>Make number notation consistent.<commit_after>// Copyright 2014 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"math"
	"math/rand"
	"sort"
	"sync"
	"testing"
	"testing/quick"
	"time"

	dto "github.com/prometheus/client_model/go"
)

func benchmarkSummaryObserve(w int, b *testing.B) {
	b.StopTimer()

	wg := new(sync.WaitGroup)
	wg.Add(w)

	g := new(sync.WaitGroup)
	g.Add(1)

	s := NewSummary(SummaryOpts{})

	for i := 0; i < w; i++ {
		go func() {
			g.Wait()

			for i := 0; i < b.N; i++ {
				s.Observe(float64(i))
			}

			wg.Done()
		}()
	}

	b.StartTimer()
	g.Done()
	wg.Wait()
}

func BenchmarkSummaryObserve1(b *testing.B) {
	benchmarkSummaryObserve(1, b)
}

func BenchmarkSummaryObserve2(b *testing.B) {
	benchmarkSummaryObserve(2, b)
}

func BenchmarkSummaryObserve4(b *testing.B) {
	benchmarkSummaryObserve(4, b)
}

func BenchmarkSummaryObserve8(b *testing.B) {
	benchmarkSummaryObserve(8, b)
}

func benchmarkSummaryWrite(w int, b *testing.B) {
	b.StopTimer()

	wg := new(sync.WaitGroup)
	wg.Add(w)

	g := new(sync.WaitGroup)
	g.Add(1)

	s := NewSummary(SummaryOpts{})

	for i := 0; i < 1000000; i++ {
		s.Observe(float64(i))
	}

	for j := 0; j < w; j++ {
		outs := make([]dto.Metric, b.N)

		go func(o []dto.Metric) {
			g.Wait()

			for i := 0; i < b.N; i++ {
				s.Write(&o[i])
			}

			wg.Done()
		}(outs)
	}

	b.StartTimer()
	g.Done()
	wg.Wait()
}

func BenchmarkSummaryWrite1(b *testing.B) {
	benchmarkSummaryWrite(1, b)
}

func BenchmarkSummaryWrite2(b *testing.B) {
	benchmarkSummaryWrite(2, b)
}

func BenchmarkSummaryWrite4(b *testing.B) {
	benchmarkSummaryWrite(4, b)
}

func BenchmarkSummaryWrite8(b *testing.B) {
	benchmarkSummaryWrite(8, b)
}

func TestSummaryConcurrency(t *testing.T) {
	rand.Seed(42)

	it := func(n uint32) bool {
		mutations := int(n%1e4 + 1e4)
		concLevel := int(n%5 + 1)
		total := mutations * concLevel

		var start, end sync.WaitGroup
		start.Add(1)
		end.Add(concLevel)

		sum := NewSummary(SummaryOpts{
			Name: "test_summary",
			Help: "helpless",
		})

		allVars := make([]float64, total)
		var sampleSum float64
		for i := 0; i < concLevel; i++ {
			vals := make([]float64, mutations)
			for j := 0; j < mutations; j++ {
				v := rand.NormFloat64()
				vals[j] = v
				allVars[i*mutations+j] = v
				sampleSum += v
			}

			go func(vals []float64) {
				start.Wait()
				for _, v := range vals {
					sum.Observe(v)
				}
				end.Done()
			}(vals)
		}
		sort.Float64s(allVars)
		start.Done()
		end.Wait()

		m := &dto.Metric{}
		sum.Write(m)
		if got, want := int(*m.Summary.SampleCount), total; got != want {
			t.Errorf("got sample count %d, want %d", got, want)
		}
		if got, want := *m.Summary.SampleSum, sampleSum; math.Abs((got-want)/want) > 0.001 {
			t.Errorf("got sample sum %f, want %f", got, want)
		}

		objectives := make([]float64, 0, len(DefObjectives))
		for qu := range DefObjectives {
			objectives = append(objectives, qu)
		}
		sort.Float64s(objectives)

		for i, wantQ := range objectives {
			ε := DefObjectives[wantQ]
			gotQ := *m.Summary.Quantile[i].Quantile
			gotV := *m.Summary.Quantile[i].Value
			min, max := getBounds(allVars, wantQ, ε)
			if gotQ != wantQ {
				t.Errorf("got quantile %f, want %f", gotQ, wantQ)
			}
			if gotV < min || gotV > max {
				t.Errorf("got %f for quantile %f, want [%f,%f]", gotV, gotQ, min, max)
			}
		}
		return true
	}

	if err := quick.Check(it, nil); err != nil {
		t.Error(err)
	}
}

func TestSummaryVecConcurrency(t *testing.T) {
	rand.Seed(42)

	objectives := make([]float64, 0, len(DefObjectives))
	for qu := range DefObjectives {

		objectives = append(objectives, qu)
	}
	sort.Float64s(objectives)

	it := func(n uint32) bool {
		mutations := int(n%1e4 + 1e4)
		concLevel := int(n%7 + 1)
		vecLength := int(n%3 + 1)

		var start, end sync.WaitGroup
		start.Add(1)
		end.Add(concLevel)

		sum := NewSummaryVec(
			SummaryOpts{
				Name: "test_summary",
				Help: "helpless",
			},
			[]string{"label"},
		)

		allVars := make([][]float64, vecLength)
		sampleSums := make([]float64, vecLength)
		for i := 0; i < concLevel; i++ {
			vals := make([]float64, mutations)
			picks := make([]int, mutations)
			for j := 0; j < mutations; j++ {
				v := rand.NormFloat64()
				vals[j] = v
				pick := rand.Intn(vecLength)
				picks[j] = pick
				allVars[pick] = append(allVars[pick], v)
				sampleSums[pick] += v
			}

			go func(vals []float64) {
				start.Wait()
				for i, v := range vals {
					sum.WithLabelValues(string('A' + picks[i])).Observe(v)
				}
				end.Done()
			}(vals)
		}
		for _, vars := range allVars {
			sort.Float64s(vars)
		}
		start.Done()
		end.Wait()

		for i := 0; i < vecLength; i++ {
			m := &dto.Metric{}
			s := sum.WithLabelValues(string('A' + i))
			s.Write(m)
			if got, want := int(*m.Summary.SampleCount), len(allVars[i]); got != want {
				t.Errorf("got sample count %d for label %c, want %d", got, 'A'+i, want)
			}
			if got, want := *m.Summary.SampleSum, sampleSums[i]; math.Abs((got-want)/want) > 0.001 {
				t.Errorf("got sample sum %f for label %c, want %f", got, 'A'+i, want)
			}
			for j, wantQ := range objectives {
				ε := DefObjectives[wantQ]
				gotQ := *m.Summary.Quantile[j].Quantile
				gotV := *m.Summary.Quantile[j].Value
				min, max := getBounds(allVars[i], wantQ, ε)
				if gotQ != wantQ {
					t.Errorf("got quantile %f for label %c, want %f", gotQ, 'A'+i, wantQ)
				}
				if gotV < min || gotV > max {
					t.Errorf("got %f for quantile %f for label %c, want [%f,%f]", gotV, gotQ, 'A'+i, min, max)
				}
			}
		}
		return true
	}

	if err := quick.Check(it, nil); err != nil {
		t.Error(err)
	}
}

func TestSummaryDecay(t *testing.T) {
	sum := NewSummary(SummaryOpts{
		Name:       "test_summary",
		Help:       "helpless",
		MaxAge:     100 * time.Millisecond,
		Objectives: map[float64]float64{0.1: 0.001},
		AgeBuckets: 10,
	})

	m := &dto.Metric{}
	i := 0
	tick := time.NewTicker(time.Millisecond)
	for _ = range tick.C {
		i++
		sum.Observe(float64(i))
		if i%10 == 0 {
			sum.Write(m)
			if got, want := *m.Summary.Quantile[0].Value, math.Max(float64(i)/10, float64(i-90)); math.Abs(got-want) > 20 {
				t.Errorf("%d. got %f, want %f", i, got, want)
			}
			m.Reset()
		}
		if i >= 1000 {
			break
		}
	}
	tick.Stop()
}

func getBounds(vars []float64, q, ε float64) (min, max float64) {
	// TODO: This currently tolerates an error of up to 2*ε. The error must
	// be at most ε, but for some reason, it's sometimes slightly
	// higher. That's a bug.
	n := float64(len(vars))
	lower := int((q - 2*ε) * n)
	upper := int(math.Ceil((q + 2*ε) * n))
	min = vars[0]
	if lower > 1 {
		min = vars[lower-1]
	}
	max = vars[len(vars)-1]
	if upper < len(vars) {
		max = vars[upper-1]
	}
	return
}
<|endoftext|>