<commit_before>package worker

import (
	"errors"
	"github.com/ipfs-search/ipfs-search/crawler"
	"github.com/ipfs-search/ipfs-search/indexer"
	"github.com/ipfs-search/ipfs-search/queue"
	"github.com/ipfs/go-ipfs-api"
	"time"
)

// Worker crawls hashes and files in parallel; it consumes hashes and files
// from respective queues and starts the configured number fo goroutines
// processing each.
type Worker struct {
	crawler      *crawler.Crawler
	config       *Config
	openChannels []*queue.TaskChannel // Channels to be closed later
	errc         chan<- error         // Channel the worker sends errors through
}

func newCrawler(config *Config, addCh *queue.TaskChannel) (*crawler.Crawler, error) {
	// Create tasks queue's
	// As there's potential failure, execute this first to allow quick fail
	hq, err := queue.NewTaskQueue(addCh, "hashes")
	if err != nil {
		addCh.Close()
		return nil, err
	}

	fq, err := queue.NewTaskQueue(addCh, "files")
	if err != nil {
		addCh.Close()
		return nil, err
	}

	// Create and configure Ipfs shell
	sh := shell.NewShell(config.IpfsAPI)
	sh.SetTimeout(config.IpfsTimeout)

	// Create elasticsearch indexer
	id := &indexer.Indexer{
		ElasticSearch: config.ElasticSearch,
	}

	c := &crawler.Crawler{
		Config:    config.CrawlerConfig,
		Shell:     sh,
		Indexer:   id,
		FileQueue: fq,
		HashQueue: hq,
	}

	return c, nil
}

// New returns an initialized worker
func New(config *Config, errc chan<- error) (*Worker, error) {
	// These is the channel the crawler uses to add newly crawled hashes
	addCh, err := queue.NewChannel()
	if err != nil {
		return nil, err
	}

	c, err := newCrawler(config, addCh)
	if err != nil {
		return nil, err
	}

	return &Worker{
		crawler:      c,
		config:       config,
		openChannels: []*queue.TaskChannel{addCh},
		errc:         errc,
	}, nil
}

// crawlWrapper wraps the actual crawl functions in a type assertion
// Essentially, it eats a function taking crawler.Args and poops out a
// function taking interface{}.
// Perhaps there's a better way to do this?
func (w *Worker) crawlWrapper(f func(*crawler.Args) error) queue.Func {
	return func(params interface{}) error {
		args, ok := params.(*crawler.Args)
		if !ok {
			return errors.New("could not assert params as crawler.Args")
		}

		return f(args)
	}
}

// workerQueue creates a channel and named queue for a worker to consume
func (w *Worker) workerQueue(name string) (*queue.TaskQueue, error) {
	ch, err := queue.NewChannel()
	if err != nil {
		return nil, err
	}
	w.openChannels = append(w.openChannels, ch)

	q, err := queue.NewTaskQueue(ch, name)
	if err != nil {
		return nil, err
	}

	return q, nil
}

// startWorkers starts count workers for q executing crawlFunc and waiting wait between starting them
func (w *Worker) startWorkers(count int, qName string, crawlFunc func(*crawler.Args) error, wait time.Duration) error {
	for i := 0; i < count; i++ {
		q, err := w.workerQueue(qName)
		if err != nil {
			return err
		}

		consumer := &queue.Consumer{
			Func:    w.crawlWrapper(crawlFunc),
			ErrChan: w.errc,
			Queue:   q,
			Params:  &crawler.Args{},
		}

		consumer.Start()

		// Start workers timeout/hash time apart
		time.Sleep(wait)
	}

	return nil
}

// Start initiates crawling of the worker
func (w *Worker) Start() error {
	// Start hash workers
	err := w.startWorkers(w.config.HashWorkers, "hashes", w.crawler.CrawlHash, w.config.HashWait)
	if err != nil {
		w.Close()
		return err
	}

	// Start file workers
	err = w.startWorkers(w.config.FileWorkers, "files", w.crawler.CrawlFile, w.config.FileWait)
	if err != nil {
		w.Close()
		return err
	}

	return nil
}

// Close destroy closes worker channels and frees resources
func (w *Worker) Close() error {
	for _, channel := range w.openChannels {
		err := channel.Close()
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Small comment fixups.<commit_after>package worker

import (
	"errors"
	"github.com/ipfs-search/ipfs-search/crawler"
	"github.com/ipfs-search/ipfs-search/indexer"
	"github.com/ipfs-search/ipfs-search/queue"
	"github.com/ipfs/go-ipfs-api"
	"time"
)

// Worker crawls hashes and files in parallel; it consumes hashes and files
// from respective queues and starts the configured number fo goroutines
// processing each.
type Worker struct {
	crawler      *crawler.Crawler
	config       *Config
	openChannels []*queue.TaskChannel // Channels to be closed later
	errc         chan<- error         // Channel the worker sends errors through
}

func newCrawler(config *Config, addCh *queue.TaskChannel) (*crawler.Crawler, error) {
	// Create tasks queue's
	// As there's potential failure, execute this first to allow quick fail
	hq, err := queue.NewTaskQueue(addCh, "hashes")
	if err != nil {
		addCh.Close()
		return nil, err
	}

	fq, err := queue.NewTaskQueue(addCh, "files")
	if err != nil {
		addCh.Close()
		return nil, err
	}

	// Create and configure Ipfs shell
	sh := shell.NewShell(config.IpfsAPI)
	sh.SetTimeout(config.IpfsTimeout)

	// Create elasticsearch indexer
	id := &indexer.Indexer{
		ElasticSearch: config.ElasticSearch,
	}

	c := &crawler.Crawler{
		Config:    config.CrawlerConfig,
		Shell:     sh,
		Indexer:   id,
		FileQueue: fq,
		HashQueue: hq,
	}

	return c, nil
}

// New returns an initialized worker
func New(config *Config, errc chan<- error) (*Worker, error) {
	// These is the channel the crawler uses to add newly crawled hashes
	addCh, err := queue.NewChannel()
	if err != nil {
		return nil, err
	}

	c, err := newCrawler(config, addCh)
	if err != nil {
		return nil, err
	}

	return &Worker{
		crawler:      c,
		config:       config,
		openChannels: []*queue.TaskChannel{addCh},
		errc:         errc,
	}, nil
}

// crawlWrapper wraps the actual crawl functions in a type assertion
// Essentially, it eats a function taking crawler.Args and poops out a
// function taking interface{}.
// Perhaps there's a better way to do this?
func (w *Worker) crawlWrapper(f func(*crawler.Args) error) queue.Func {
	return func(params interface{}) error {
		args, ok := params.(*crawler.Args)
		if !ok {
			return errors.New("could not assert params as crawler.Args")
		}

		return f(args)
	}
}

// workerQueue creates a channel and named queue for a worker to consume
func (w *Worker) workerQueue(name string) (*queue.TaskQueue, error) {
	ch, err := queue.NewChannel()
	if err != nil {
		return nil, err
	}
	w.openChannels = append(w.openChannels, ch)

	q, err := queue.NewTaskQueue(ch, name)
	if err != nil {
		return nil, err
	}

	return q, nil
}

// startWorkers starts count workers for q executing crawlFunc and waiting wait between starting them
func (w *Worker) startWorkers(count int, qName string, crawlFunc func(*crawler.Args) error, wait time.Duration) error {
	for i := 0; i < count; i++ {
		q, err := w.workerQueue(qName)
		if err != nil {
			return err
		}

		consumer := &queue.Consumer{
			Func:    w.crawlWrapper(crawlFunc),
			ErrChan: w.errc,
			Queue:   q,
			Params:  &crawler.Args{},
		}

		consumer.Start()

		// Start workers timeout/hash time apart
		time.Sleep(wait)
	}

	return nil
}

// Start initiates crawling of the worker (asynchronuously)
func (w *Worker) Start() error {
	// Start hash worker
	err := w.startWorkers(w.config.HashWorkers, "hashes", w.crawler.CrawlHash, w.config.HashWait)
	if err != nil {
		w.Close()
		return err
	}

	// Start file workers
	err = w.startWorkers(w.config.FileWorkers, "files", w.crawler.CrawlFile, w.config.FileWait)
	if err != nil {
		w.Close()
		return err
	}

	return nil
}

// Close destroy closes worker channels and frees resources
func (w *Worker) Close() error {
	for _, channel := range w.openChannels {
		err := channel.Close()
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

// go test *.go -bench=".*" -benchmem

package gcache_test

import (
	"fmt"
	"github.com/gogf/gf/os/glog"
	"github.com/gogf/gf/util/grand"
	"testing"
	"time"

	"github.com/gogf/gf/container/gset"
	"github.com/gogf/gf/frame/g"
	"github.com/gogf/gf/os/gcache"
	"github.com/gogf/gf/os/grpool"
	"github.com/gogf/gf/test/gtest"
)

func TestCache_GCache_Set(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		gcache.Set(1, 11, 0)
		defer gcache.Removes(g.Slice{1, 2, 3})
		t.Assert(gcache.Get(1), 11)
		t.Assert(gcache.Contains(1), true)
	})
}

func TestCache_Set(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		c := gcache.New()
		defer c.Close()
		c.Set(1, 11, 0)
		t.Assert(c.Get(1), 11)
		t.Assert(c.Contains(1), true)
	})
}

func TestCache_GetVar(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		c := gcache.New()
		defer c.Close()
		c.Set(1, 11, 0)
		t.Assert(c.Get(1), 11)
		t.Assert(c.Contains(1), true)
		t.Assert(c.GetVar(1).Int(), 11)
		t.Assert(c.GetVar(2).Int(), 0)
		t.Assert(c.GetVar(2).IsNil(), true)
		t.Assert(c.GetVar(2).IsEmpty(), true)
	})
}

func TestCache_Set_Expire(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.Set(2, 22, 100*time.Millisecond)
		t.Assert(cache.Get(2), 22)
		time.Sleep(200 * time.Millisecond)
		t.Assert(cache.Get(2), nil)
		time.Sleep(3 * time.Second)
		t.Assert(cache.Size(), 0)
		cache.Close()
	})

	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.Set(1, 11, 100*time.Millisecond)
		t.Assert(cache.Get(1), 11)
		time.Sleep(200 * time.Millisecond)
		t.Assert(cache.Get(1), nil)
	})
}

func TestCache_SetVar(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.Set(1, 11, 3*time.Second)
		cache.SetVar(1, 12)
		t.Assert(cache.GetVar(1), 12)
	})
}

func BenchmarkMemCache_GetSetExpire(b *testing.B) {
	cache := gcache.New()
	cache.Set(1, 11, 3*time.Second)
	if expire, ok := cache.GetExpire(1); ok {
		glog.Println(expire)
	}
	for i := 0; i < b.N; i++ {
		r := time.Duration(grand.N(5, 10))
		cache.SetExpire(1, r*time.Second)
		//cache.SetExpire(1, 7*time.Second)
		if _, ok := cache.GetExpire(1); !ok {
			panic(fmt.Sprintf("[ERROR] %s", "GetExpire error"))
		}
	}
}

func TestCache_Keys_Values(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		for i := 0; i < 10; i++ {
			cache.Set(i, i*10, 0)
		}
		t.Assert(len(cache.Keys()), 10)
		t.Assert(len(cache.Values()), 10)
		t.AssertIN(0, cache.Keys())
		t.AssertIN(90, cache.Values())
	})
}

func TestCache_LRU(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New(2)
		for i := 0; i < 10; i++ {
			cache.Set(i, i, 0)
		}
		t.Assert(cache.Size(), 10)
		t.Assert(cache.Get(6), 6)
		time.Sleep(4 * time.Second)
		t.Assert(cache.Size(), 2)
		t.Assert(cache.Get(6), 6)
		t.Assert(cache.Get(1), nil)
		cache.Close()
	})
}

func TestCache_LRU_expire(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New(2)
		cache.Set(1, nil, 1000)
		t.Assert(cache.Size(), 1)
		t.Assert(cache.Get(1), nil)
	})
}

func TestCache_SetIfNotExist(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.SetIfNotExist(1, 11, 0)
		t.Assert(cache.Get(1), 11)
		cache.SetIfNotExist(1, 22, 0)
		t.Assert(cache.Get(1), 11)
		cache.SetIfNotExist(2, 22, 0)
		t.Assert(cache.Get(2), 22)

		gcache.Removes(g.Slice{1, 2, 3})
		gcache.SetIfNotExist(1, 11, 0)
		t.Assert(gcache.Get(1), 11)
		gcache.SetIfNotExist(1, 22, 0)
		t.Assert(gcache.Get(1), 11)
	})
}

func TestCache_Sets(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.Sets(g.MapAnyAny{1: 11, 2: 22}, 0)
		t.Assert(cache.Get(1), 11)

		gcache.Removes(g.Slice{1, 2, 3})
		gcache.Sets(g.MapAnyAny{1: 11, 2: 22}, 0)
		t.Assert(gcache.Get(1), 11)
	})
}

func TestCache_GetOrSet(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.GetOrSet(1, 11, 0)
		t.Assert(cache.Get(1), 11)
		cache.GetOrSet(1, 111, 0)
		t.Assert(cache.Get(1), 11)

		gcache.Removes(g.Slice{1, 2, 3})
		gcache.GetOrSet(1, 11, 0)
		t.Assert(gcache.Get(1), 11)
		gcache.GetOrSet(1, 111, 0)
		t.Assert(gcache.Get(1), 11)
	})
}

func TestCache_GetOrSetFunc(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.GetOrSetFunc(1, func() interface{} {
			return 11
		}, 0)
		t.Assert(cache.Get(1), 11)
		cache.GetOrSetFunc(1, func() interface{} {
			return 111
		}, 0)
		t.Assert(cache.Get(1), 11)

		gcache.Removes(g.Slice{1, 2, 3})
		gcache.GetOrSetFunc(1, func() interface{} {
			return 11
		}, 0)
		t.Assert(gcache.Get(1), 11)
		gcache.GetOrSetFunc(1, func() interface{} {
			return 111
		}, 0)
		t.Assert(gcache.Get(1), 11)
	})
}

func TestCache_GetOrSetFuncLock(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.GetOrSetFuncLock(1, func() interface{} {
			return 11
		}, 0)
		t.Assert(cache.Get(1), 11)
		cache.GetOrSetFuncLock(1, func() interface{} {
			return 111
		}, 0)
		t.Assert(cache.Get(1), 11)

		gcache.Removes(g.Slice{1, 2, 3})
		gcache.GetOrSetFuncLock(1, func() interface{} {
			return 11
		}, 0)
		t.Assert(gcache.Get(1), 11)
		gcache.GetOrSetFuncLock(1, func() interface{} {
			return 111
		}, 0)
		t.Assert(gcache.Get(1), 11)
	})
}

func TestCache_Clear(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.Sets(g.MapAnyAny{1: 11, 2: 22}, 0)
		cache.Clear()
		t.Assert(cache.Size(), 0)
	})
}

func TestCache_SetConcurrency(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		pool := grpool.New(4)
		go func() {
			for {
				pool.Add(func() {
					cache.SetIfNotExist(1, 11, 10)
				})
			}
		}()
		select {
		case <-time.After(2 * time.Second):
			//t.Log("first part end")
		}

		go func() {
			for {
				pool.Add(func() {
					cache.SetIfNotExist(1, nil, 10)
				})
			}
		}()
		select {
		case <-time.After(2 * time.Second):
			//t.Log("second part end")
		}
	})
}

func TestCache_Basic(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		{
			cache := gcache.New()
			cache.Sets(g.MapAnyAny{1: 11, 2: 22}, 0)
			t.Assert(cache.Contains(1), true)
			t.Assert(cache.Get(1), 11)
			data := cache.Data()
			t.Assert(data[1], 11)
			t.Assert(data[2], 22)
			t.Assert(data[3], nil)
			t.Assert(cache.Size(), 2)
			keys := cache.Keys()
			t.Assert(gset.NewFrom(g.Slice{1, 2}).Equal(gset.NewFrom(keys)), true)
			keyStrs := cache.KeyStrings()
			t.Assert(gset.NewFrom(g.Slice{"1", "2"}).Equal(gset.NewFrom(keyStrs)), true)
			values := cache.Values()
			t.Assert(gset.NewFrom(g.Slice{11, 22}).Equal(gset.NewFrom(values)), true)
			removeData1 := cache.Remove(1)
			t.Assert(removeData1, 11)
			t.Assert(cache.Size(), 1)
			cache.Removes(g.Slice{2})
			t.Assert(cache.Size(), 0)
		}

		gcache.Removes(g.Slice{1, 2, 3})
		{
			gcache.Sets(g.MapAnyAny{1: 11, 2: 22}, 0)
			t.Assert(gcache.Contains(1), true)
			t.Assert(gcache.Get(1), 11)
			data := gcache.Data()
			t.Assert(data[1], 11)
			t.Assert(data[2], 22)
			t.Assert(data[3], nil)
			t.Assert(gcache.Size(), 2)
			keys := gcache.Keys()
			t.Assert(gset.NewFrom(g.Slice{1, 2}).Equal(gset.NewFrom(keys)), true)
			keyStrs := gcache.KeyStrings()
			t.Assert(gset.NewFrom(g.Slice{"1", "2"}).Equal(gset.NewFrom(keyStrs)), true)
			values := gcache.Values()
			t.Assert(gset.NewFrom(g.Slice{11, 22}).Equal(gset.NewFrom(values)), true)
			removeData1 := gcache.Remove(1)
			t.Assert(removeData1, 11)
			t.Assert(gcache.Size(), 1)
			gcache.Removes(g.Slice{2})
			t.Assert(gcache.Size(), 0)
		}
	})
}
<commit_msg>CI-glog失败重测<commit_after>// Copyright 2017 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

// go test *.go -bench=".*" -benchmem

package gcache_test

import (
	"fmt"
	"github.com/gogf/gf/os/glog"
	"github.com/gogf/gf/util/grand"
	"testing"
	"time"

	"github.com/gogf/gf/container/gset"
	"github.com/gogf/gf/frame/g"
	"github.com/gogf/gf/os/gcache"
	"github.com/gogf/gf/os/grpool"
	"github.com/gogf/gf/test/gtest"
)

func TestCache_GCache_Set(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		gcache.Set(1, 11, 0)
		defer gcache.Removes(g.Slice{1, 2, 3})
		t.Assert(gcache.Get(1), 11)
		t.Assert(gcache.Contains(1), true)
	})
}

func TestCache_Set(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		c := gcache.New()
		defer c.Close()
		c.Set(1, 11, 0)
		t.Assert(c.Get(1), 11)
		t.Assert(c.Contains(1), true)
	})
}

func TestCache_GetVar(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		c := gcache.New()
		defer c.Close()
		c.Set(1, 11, 0)
		t.Assert(c.Get(1), 11)
		t.Assert(c.Contains(1), true)
		t.Assert(c.GetVar(1).Int(), 11)
		t.Assert(c.GetVar(2).Int(), 0)
		t.Assert(c.GetVar(2).IsNil(), true)
		t.Assert(c.GetVar(2).IsEmpty(), true)
	})
}

func TestCache_Set_Expire(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.Set(2, 22, 100*time.Millisecond)
		t.Assert(cache.Get(2), 22)
		time.Sleep(200 * time.Millisecond)
		t.Assert(cache.Get(2), nil)
		time.Sleep(3 * time.Second)
		t.Assert(cache.Size(), 0)
		cache.Close()
	})

	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.Set(1, 11, 100*time.Millisecond)
		t.Assert(cache.Get(1), 11)
		time.Sleep(200 * time.Millisecond)
		t.Assert(cache.Get(1), nil)
	})
}

func TestCache_SetVar(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.Set(1, 11, 3*time.Second)
		expireBefore, _ := cache.GetExpire(1)
		cache.SetVar(1, 12)
		expireAfter, _ := cache.GetExpire(1)
		t.Assert(cache.GetVar(1), 12)
		t.Assert(expireBefore, expireAfter)
	})
}

func BenchmarkMemCache_GetSetExpire(b *testing.B) {
	cache := gcache.New()
	cache.Set(1, 11, 3*time.Second)
	if expire, ok := cache.GetExpire(1); ok {
		glog.Println(expire)
	}
	for i := 0; i < b.N; i++ {
		r := time.Duration(grand.N(5, 10))
		cache.SetExpire(1, r*time.Second)
		//cache.SetExpire(1, 7*time.Second)
		if _, ok := cache.GetExpire(1); !ok {
			panic(fmt.Sprintf("[ERROR] %s", "GetExpire error"))
		}
	}
}

func TestCache_Keys_Values(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		for i := 0; i < 10; i++ {
			cache.Set(i, i*10, 0)
		}
		t.Assert(len(cache.Keys()), 10)
		t.Assert(len(cache.Values()), 10)
		t.AssertIN(0, cache.Keys())
		t.AssertIN(90, cache.Values())
	})
}

func TestCache_LRU(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New(2)
		for i := 0; i < 10; i++ {
			cache.Set(i, i, 0)
		}
		t.Assert(cache.Size(), 10)
		t.Assert(cache.Get(6), 6)
		time.Sleep(4 * time.Second)
		t.Assert(cache.Size(), 2)
		t.Assert(cache.Get(6), 6)
		t.Assert(cache.Get(1), nil)
		cache.Close()
	})
}

func TestCache_LRU_expire(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New(2)
		cache.Set(1, nil, 1000)
		t.Assert(cache.Size(), 1)
		t.Assert(cache.Get(1), nil)
	})
}

func TestCache_SetIfNotExist(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.SetIfNotExist(1, 11, 0)
		t.Assert(cache.Get(1), 11)
		cache.SetIfNotExist(1, 22, 0)
		t.Assert(cache.Get(1), 11)
		cache.SetIfNotExist(2, 22, 0)
		t.Assert(cache.Get(2), 22)

		gcache.Removes(g.Slice{1, 2, 3})
		gcache.SetIfNotExist(1, 11, 0)
		t.Assert(gcache.Get(1), 11)
		gcache.SetIfNotExist(1, 22, 0)
		t.Assert(gcache.Get(1), 11)
	})
}

func TestCache_Sets(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.Sets(g.MapAnyAny{1: 11, 2: 22}, 0)
		t.Assert(cache.Get(1), 11)

		gcache.Removes(g.Slice{1, 2, 3})
		gcache.Sets(g.MapAnyAny{1: 11, 2: 22}, 0)
		t.Assert(gcache.Get(1), 11)
	})
}

func TestCache_GetOrSet(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.GetOrSet(1, 11, 0)
		t.Assert(cache.Get(1), 11)
		cache.GetOrSet(1, 111, 0)
		t.Assert(cache.Get(1), 11)

		gcache.Removes(g.Slice{1, 2, 3})
		gcache.GetOrSet(1, 11, 0)
		t.Assert(gcache.Get(1), 11)
		gcache.GetOrSet(1, 111, 0)
		t.Assert(gcache.Get(1), 11)
	})
}

func TestCache_GetOrSetFunc(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.GetOrSetFunc(1, func() interface{} {
			return 11
		}, 0)
		t.Assert(cache.Get(1), 11)
		cache.GetOrSetFunc(1, func() interface{} {
			return 111
		}, 0)
		t.Assert(cache.Get(1), 11)

		gcache.Removes(g.Slice{1, 2, 3})
		gcache.GetOrSetFunc(1, func() interface{} {
			return 11
		}, 0)
		t.Assert(gcache.Get(1), 11)
		gcache.GetOrSetFunc(1, func() interface{} {
			return 111
		}, 0)
		t.Assert(gcache.Get(1), 11)
	})
}

func TestCache_GetOrSetFuncLock(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.GetOrSetFuncLock(1, func() interface{} {
			return 11
		}, 0)
		t.Assert(cache.Get(1), 11)
		cache.GetOrSetFuncLock(1, func() interface{} {
			return 111
		}, 0)
		t.Assert(cache.Get(1), 11)

		gcache.Removes(g.Slice{1, 2, 3})
		gcache.GetOrSetFuncLock(1, func() interface{} {
			return 11
		}, 0)
		t.Assert(gcache.Get(1), 11)
		gcache.GetOrSetFuncLock(1, func() interface{} {
			return 111
		}, 0)
		t.Assert(gcache.Get(1), 11)
	})
}

func TestCache_Clear(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		cache.Sets(g.MapAnyAny{1: 11, 2: 22}, 0)
		cache.Clear()
		t.Assert(cache.Size(), 0)
	})
}

func TestCache_SetConcurrency(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		cache := gcache.New()
		pool := grpool.New(4)
		go func() {
			for {
				pool.Add(func() {
					cache.SetIfNotExist(1, 11, 10)
				})
			}
		}()
		select {
		case <-time.After(2 * time.Second):
			//t.Log("first part end")
		}

		go func() {
			for {
				pool.Add(func() {
					cache.SetIfNotExist(1, nil, 10)
				})
			}
		}()
		select {
		case <-time.After(2 * time.Second):
			//t.Log("second part end")
		}
	})
}

func TestCache_Basic(t *testing.T) {
	gtest.C(t, func(t *gtest.T) {
		{
			cache := gcache.New()
			cache.Sets(g.MapAnyAny{1: 11, 2: 22}, 0)
			t.Assert(cache.Contains(1), true)
			t.Assert(cache.Get(1), 11)
			data := cache.Data()
			t.Assert(data[1], 11)
			t.Assert(data[2], 22)
			t.Assert(data[3], nil)
			t.Assert(cache.Size(), 2)
			keys := cache.Keys()
			t.Assert(gset.NewFrom(g.Slice{1, 2}).Equal(gset.NewFrom(keys)), true)
			keyStrs := cache.KeyStrings()
			t.Assert(gset.NewFrom(g.Slice{"1", "2"}).Equal(gset.NewFrom(keyStrs)), true)
			values := cache.Values()
			t.Assert(gset.NewFrom(g.Slice{11, 22}).Equal(gset.NewFrom(values)), true)
			removeData1 := cache.Remove(1)
			t.Assert(removeData1, 11)
			t.Assert(cache.Size(), 1)
			cache.Removes(g.Slice{2})
			t.Assert(cache.Size(), 0)
		}

		gcache.Removes(g.Slice{1, 2, 3})
		{
			gcache.Sets(g.MapAnyAny{1: 11, 2: 22}, 0)
			t.Assert(gcache.Contains(1), true)
			t.Assert(gcache.Get(1), 11)
			data := gcache.Data()
			t.Assert(data[1], 11)
			t.Assert(data[2], 22)
			t.Assert(data[3], nil)
			t.Assert(gcache.Size(), 2)
			keys := gcache.Keys()
			t.Assert(gset.NewFrom(g.Slice{1, 2}).Equal(gset.NewFrom(keys)), true)
			keyStrs := gcache.KeyStrings()
			t.Assert(gset.NewFrom(g.Slice{"1", "2"}).Equal(gset.NewFrom(keyStrs)), true)
			values := gcache.Values()
			t.Assert(gset.NewFrom(g.Slice{11, 22}).Equal(gset.NewFrom(values)), true)
			removeData1 := gcache.Remove(1)
			t.Assert(removeData1, 11)
			t.Assert(gcache.Size(), 1)
			gcache.Removes(g.Slice{2})
			t.Assert(gcache.Size(), 0)
		}
	})
}
<|endoftext|>


<commit_before>package writer

import (
	"sync"
)

const bufferSize = 1024 * 8

type InsertBatch struct {
	Table string
	Rows  [][]interface{}
}

type InsertElement struct {
	Table string
	Row   []interface{}
}

type InsertBuffer struct {
	In  chan InsertElement
	Out chan InsertBatch
	wg  *sync.WaitGroup
}

func NewInsertBuffer() *InsertBuffer {
	ib := InsertBuffer{
		In:  make(chan InsertElement, 64),
		Out: make(chan InsertBatch, 1),
		wg:  &sync.WaitGroup{},
	}
	ib.wg.Add(1)
	go ib.loop()
	return &ib
}

func (ib *InsertBuffer) Close() {
	close(ib.In)
	ib.wg.Wait()
	close(ib.Out)
}

func (ib *InsertBuffer) Insert(table string, row []interface{}) {
	ib.In <- InsertElement{table, row}
}

func (ib *InsertBuffer) loop() {
	buffer := make(map[string]*InsertBatch)

	for elem := range ib.In {
		if batch, ok := buffer[elem.Table]; ok {
			batch.Rows = append(batch.Rows, elem.Row)
		} else {
			buffer[elem.Table] = &InsertBatch{elem.Table, [][]interface{}{elem.Row}}
		}
		if len(buffer[elem.Table].Rows) > bufferSize {
			batch := buffer[elem.Table]
			delete(buffer, elem.Table)
			ib.Out <- *batch
		}
	}
	for table, batch := range buffer {
		delete(buffer, table)
		ib.Out <- *batch
	}
	ib.wg.Done()
}
<commit_msg>remove unused InsertBuffer<commit_after><|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"time"

	"cloud.google.com/go/bigquery"
	"github.com/golang/protobuf/ptypes"
	"google.golang.org/api/iterator"

	evalpb "go.chromium.org/luci/rts/presubmit/eval/proto"
)

// rejections calls f for each found CQ rejection.
func (r *presubmitHistoryRun) rejections(ctx context.Context, f func(*evalpb.Rejection) error) error {
	q, err := r.bqQuery(ctx, rejectedPatchSetsSQL)
	if err != nil {
		return err
	}
	it, err := q.Read(ctx)
	if err != nil {
		return err
	}

	// Each result row represents a failed test variant.
	// The rows are sorted by change and patchset, and one rejection
	// may span multiple rows.
	// Accumulate failed tests to the current rejection until we encounter
	// a different patchset.
	var curRej *evalpb.Rejection
	for {
		var row rejectionRow
		switch err := it.Next(&row); {
		case err == iterator.Done:
			// Report the last one.
			if curRej != nil {
				return f(curRej)
			}
			return nil

		case err != nil:
			return err
		}

		// Report the current rejection if we have one and if `row` is a different
		// patchset.
		if curRej != nil && (curRej.Patchsets[0].Change.Number != int64(row.Change) || curRej.Patchsets[0].Patchset != int64(row.Patchset)) {
			if err := f(curRej); err != nil {
				return err
			}
			curRej = nil
		}

		// Initialize a new rejection if we don't have one.
		if curRej == nil {
			curRej = &evalpb.Rejection{
				Patchsets: []*evalpb.GerritPatchset{
					{
						Change: &evalpb.GerritChange{
							// Assume all Chromium source code is in
							// https://chromium.googlesource.com/chromium/src
							// TOOD(nodir): make it fail if it is not.
							Host:    "chromium-review.googlesource.com",
							Project: "chromium/src",
							Number:  int64(row.Change),
						},
						Patchset: int64(row.Patchset),
					},
				},
			}
			curRej.Timestamp, _ = ptypes.TimestampProto(row.Timestamp)
		}

		curRej.FailedTestVariants = append(curRej.FailedTestVariants, row.FailedTestVariant.proto())
	}
}

type rejectionRow struct {
	Change            int
	Patchset          int
	Timestamp         time.Time
	FailedTestVariant testVariantRow
}

type testVariantRow struct {
	ID       string
	FileName bigquery.NullString
	Variant  []struct {
		Key   string
		Value string
	}
}

func (t *testVariantRow) proto() *evalpb.TestVariant {
	ret := &evalpb.TestVariant{
		Id:       t.ID,
		FileName: t.FileName.StringVal,
		Variant:  make(map[string]string, len(t.Variant)),
	}
	for _, kv := range t.Variant {
		ret.Variant[kv.Key] = kv.Value
	}
	return ret
}

// rejectedPatchSetsSQL is a BigQuery query that returns patchsets with test
// failures. Ignores tests that don't have a test location.
//
// This query intentionally does not use ARRAY_AGG for failed tests because
// otherwise a result row size may exceed the API limitation of 10MB.
const rejectedPatchSetsSQL = `
	WITH
		tryjobs AS (
			SELECT
				b.id,
				ps,
				partition_time as ps_approx_timestamp,
			FROM commit-queue.chromium.attempts a, a.gerrit_changes ps, a.builds b
			-- Read next-day attempts too in case the CQ attempt finished soon after 12am.
			-- Note that for test results, the cutoff is still @endTime.
			WHERE partition_time BETWEEN @startTime AND TIMESTAMP_ADD(@endTime, INTERVAL 1 DAY)
		),
		failed_test_variants AS (
			SELECT
				CAST(REGEXP_EXTRACT(exported.id, r'build-(\d+)') as INT64) as build_id,
				ANY_VALUE(IFNULL(test_location.file_name, '')) as file_name,
				test_id,
				variant_hash,
				ANY_VALUE(variant) as variant,
			FROM luci-resultdb.chromium.try_test_results tr
			WHERE partition_time BETWEEN @startTime and @endTime
				AND (@test_id_regexp = '' OR REGEXP_CONTAINS(test_id, @test_id_regexp))
				AND (@builder_regexp = '' OR EXISTS (SELECT 0 FROM tr.variant WHERE key='builder' AND REGEXP_CONTAINS(value, @builder_regexp)))
				-- Exclude broken test locations.
				-- TODO(nodir): remove this after crbug.com/1130425 is fixed.
				AND REGEXP_CONTAINS(IFNULL(test_location.file_name, ''), r'(?i)^(|.*\.(cc|html|m|c|cpp))$')
				-- Exclude broken prefixes.
				-- TODO(nodir): remove after crbug.com/1017288 is fixed.
				AND (test_id NOT LIKE 'ninja://:blink_web_tests/%' OR test_location.file_name LIKE '//third_party/%')
			GROUP BY build_id, test_id, variant_hash
			-- TODO(crbug.com/1112125): consider doing this filtering after joining with patchsets.
			-- This will increase the query cost, but might improve
			-- data quality. Currenly the query is vulnerable to situations where
			-- a test failed in one CQ attempt for the patchset, and succeeded
			-- in another for the same patchset.
			HAVING LOGICAL_AND(NOT expected AND NOT exonerated)
		)
	SELECT
		ps.change as Change,
		ps.patchset as Patchset,
		MIN(ps_approx_timestamp) as Timestamp,
		STRUCT(
			test_id as ID,
			ANY_VALUE(variant) as Variant,
			ANY_VALUE(file_name) as FileName
		) as FailedTestVariant
	FROM tryjobs t
	JOIN failed_test_variants f ON t.id = f.build_id
	GROUP BY ps.change, ps.patchset, test_id, variant_hash
	ORDER BY ps.change, ps.patchset
`
<commit_msg>Revert "[rts] Use NullableString for FileName"<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"time"

	"github.com/golang/protobuf/ptypes"
	"google.golang.org/api/iterator"

	evalpb "go.chromium.org/luci/rts/presubmit/eval/proto"
)

// rejections calls f for each found CQ rejection.
func (r *presubmitHistoryRun) rejections(ctx context.Context, f func(*evalpb.Rejection) error) error {
	q, err := r.bqQuery(ctx, rejectedPatchSetsSQL)
	if err != nil {
		return err
	}
	it, err := q.Read(ctx)
	if err != nil {
		return err
	}

	// Each result row represents a failed test variant.
	// The rows are sorted by change and patchset, and one rejection
	// may span multiple rows.
	// Accumulate failed tests to the current rejection until we encounter
	// a different patchset.
	var curRej *evalpb.Rejection
	for {
		var row rejectionRow
		switch err := it.Next(&row); {
		case err == iterator.Done:
			// Report the last one.
			if curRej != nil {
				return f(curRej)
			}
			return nil

		case err != nil:
			return err
		}

		// Report the current rejection if we have one and if `row` is a different
		// patchset.
		if curRej != nil && (curRej.Patchsets[0].Change.Number != int64(row.Change) || curRej.Patchsets[0].Patchset != int64(row.Patchset)) {
			if err := f(curRej); err != nil {
				return err
			}
			curRej = nil
		}

		// Initialize a new rejection if we don't have one.
		if curRej == nil {
			curRej = &evalpb.Rejection{
				Patchsets: []*evalpb.GerritPatchset{
					{
						Change: &evalpb.GerritChange{
							// Assume all Chromium source code is in
							// https://chromium.googlesource.com/chromium/src
							// TOOD(nodir): make it fail if it is not.
							Host:    "chromium-review.googlesource.com",
							Project: "chromium/src",
							Number:  int64(row.Change),
						},
						Patchset: int64(row.Patchset),
					},
				},
			}
			curRej.Timestamp, _ = ptypes.TimestampProto(row.Timestamp)
		}

		curRej.FailedTestVariants = append(curRej.FailedTestVariants, row.FailedTestVariant.proto())
	}
}

type rejectionRow struct {
	Change            int
	Patchset          int
	Timestamp         time.Time
	FailedTestVariant testVariantRow
}

type testVariantRow struct {
	ID       string
	FileName string
	Variant  []struct {
		Key   string
		Value string
	}
}

func (t *testVariantRow) proto() *evalpb.TestVariant {
	ret := &evalpb.TestVariant{
		Id:       t.ID,
		FileName: t.FileName,
		Variant:  make(map[string]string, len(t.Variant)),
	}
	for _, kv := range t.Variant {
		ret.Variant[kv.Key] = kv.Value
	}
	return ret
}

// rejectedPatchSetsSQL is a BigQuery query that returns patchsets with test
// failures. Ignores tests that don't have a test location.
//
// This query intentionally does not use ARRAY_AGG for failed tests because
// otherwise a result row size may exceed the API limitation of 10MB.
const rejectedPatchSetsSQL = `
	WITH
		tryjobs AS (
			SELECT
				b.id,
				ps,
				partition_time as ps_approx_timestamp,
			FROM commit-queue.chromium.attempts a, a.gerrit_changes ps, a.builds b
			-- Read next-day attempts too in case the CQ attempt finished soon after 12am.
			-- Note that for test results, the cutoff is still @endTime.
			WHERE partition_time BETWEEN @startTime AND TIMESTAMP_ADD(@endTime, INTERVAL 1 DAY)
		),
		failed_test_variants AS (
			SELECT
				CAST(REGEXP_EXTRACT(exported.id, r'build-(\d+)') as INT64) as build_id,
				ANY_VALUE(IFNULL(test_location.file_name, '')) as file_name,
				test_id,
				variant_hash,
				ANY_VALUE(variant) as variant,
			FROM luci-resultdb.chromium.try_test_results tr
			WHERE partition_time BETWEEN @startTime and @endTime
				AND (@test_id_regexp = '' OR REGEXP_CONTAINS(test_id, @test_id_regexp))
				AND (@builder_regexp = '' OR EXISTS (SELECT 0 FROM tr.variant WHERE key='builder' AND REGEXP_CONTAINS(value, @builder_regexp)))
				-- Exclude broken test locations.
				-- TODO(nodir): remove this after crbug.com/1130425 is fixed.
				AND REGEXP_CONTAINS(IFNULL(test_location.file_name, ''), r'(?i)^(|.*\.(cc|html|m|c|cpp))$')
				-- Exclude broken prefixes.
				-- TODO(nodir): remove after crbug.com/1017288 is fixed.
				AND (test_id NOT LIKE 'ninja://:blink_web_tests/%' OR test_location.file_name LIKE '//third_party/%')
			GROUP BY build_id, test_id, variant_hash
			-- TODO(crbug.com/1112125): consider doing this filtering after joining with patchsets.
			-- This will increase the query cost, but might improve
			-- data quality. Currenly the query is vulnerable to situations where
			-- a test failed in one CQ attempt for the patchset, and succeeded
			-- in another for the same patchset.
			HAVING LOGICAL_AND(NOT expected AND NOT exonerated)
		)
	SELECT
		ps.change as Change,
		ps.patchset as Patchset,
		MIN(ps_approx_timestamp) as Timestamp,
		STRUCT(
			test_id as ID,
			ANY_VALUE(variant) as Variant,
			ANY_VALUE(file_name) as FileName
		) as FailedTestVariant
	FROM tryjobs t
	JOIN failed_test_variants f ON t.id = f.build_id
	GROUP BY ps.change, ps.patchset, test_id, variant_hash
	ORDER BY ps.change, ps.patchset
`
<|endoftext|>


<commit_before>package pongo2

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"regexp"
	"strings"
	"testing"
	"time"
)

var admin_list = []string{"user2"}

var time1 = time.Date(2014, 06, 10, 15, 30, 15, 0, time.UTC)
var time2 = time.Date(2011, 03, 21, 8, 37, 56, 12, time.UTC)

type post struct {
	Text    string
	Created time.Time
}

type user struct {
	Name      string
	Validated bool
}

type comment struct {
	Author *user
	Date   time.Time
	Text   string
}

func is_admin(in *Value) *Value {
	u, worked := in.Interface().(*user)
	if !worked {
		return AsValue(false)
	}
	for _, a := range admin_list {
		if a == u.Name {
			return AsValue(true)
		}
	}
	return AsValue(false)
}

func (u *user) Is_admin() *Value {
	return is_admin(AsValue(u))
}

func (p *post) String() string {
	return ":-)"
}

var tplContext = Context{
	"number": 11,
	"simple": map[string]interface{}{
		"number":        42,
		"name":          "john doe",
		"included_file": "INCLUDES.helper",
		"nil":           nil,
		"uint":          uint(8),
		"str":           "string",
		"bool_true":     true,
		"bool_false":    false,
		"newline_text": `this is a text
with a new line in it`,
	},
	"complex": map[string]interface{}{
		"is_admin": is_admin,
		"post": post{
			Text:    "
Hello!

Welcome to my new blog page. I'm using pongo2 which supports {{ variables }} and {% tags %}.
",
			Created: time2,
		},
		"comments": []*comment{
			&comment{
				Author: &user{
					Name:      "user1",
					Validated: true,
				},
				Date: time1,
				Text: "\"pongo2 is nice!\"",
			},
			&comment{
				Author: &user{
					Name:      "user2",
					Validated: true,
				},
				Date: time2,
				Text: "comment2 with  tags in it",
			},
			&comment{
				Author: &user{
					Name:      "user3",
					Validated: false,
				},
				Date: time1,
				Text: "hello! there",
			},
		},
	},
}

func TestTemplates(t *testing.T) {
	SetDebug(true) // activate pongo2's debugging output

	matches, err := filepath.Glob("./template_tests/*.tpl")
	if err != nil {
		t.Fatal(err)
	}
	for idx, match := range matches {
		t.Logf("[Template %3d] Testing '%s'", idx+1, match)
		tpl, err := FromFile(match)
		if err != nil {
			t.Fatalf("Error on FromFile('%s'): %s", match, err.Error())
		}
		test_filename := fmt.Sprintf("%s.out", match)
		test_out, err := ioutil.ReadFile(test_filename)
		if err != nil {
			t.Fatalf("Error on ReadFile('%s'): %s", test_filename, err.Error())
		}
		tpl_out, err := tpl.Execute(tplContext)
		if err != nil {
			t.Fatalf("Error on Execute('%s'): %s", match, err.Error())
		}
		if string(test_out) != tpl_out {
			t.Logf("Template (rendered) '%s': '%s'", match, tpl_out)
			err_filename := "tpl-error.out"
			ioutil.WriteFile(err_filename, []byte(tpl_out), 0700)
			t.Logf("get a complete diff with command: 'diff -ya %s %s'", test_filename, err_filename)
			t.Fatalf("Failed: test_out != tpl_out for %s", match)
		}
	}
}

func TestExecutionErrors(t *testing.T) {
	SetDebug(true) // activate pongo2's debugging output

	matches, err := filepath.Glob("./template_tests/*-execution.err")
	if err != nil {
		t.Fatal(err)
	}
	for idx, match := range matches {
		t.Logf("[Errors %3d] Testing '%s'", idx+1, match)

		test_data, err := ioutil.ReadFile(match)
		tests := strings.Split(string(test_data), "\n")

		check_filename := fmt.Sprintf("%s.out", match)
		check_data, err := ioutil.ReadFile(check_filename)
		if err != nil {
			t.Fatalf("Error on ReadFile('%s'): %s", check_filename, err.Error())
		}
		checks := strings.Split(string(check_data), "\n")

		if len(checks) != len(tests) {
			t.Fatal("Template lines != Checks lines")
		}

		for idx, test := range tests {
			if strings.TrimSpace(test) == "" {
				continue
			}
			if strings.TrimSpace(checks[idx]) == "" {
				t.Fatalf("[%s Line %d] Check is empty (must contain an regular expression).",
					match, idx+1)
			}

			tpl, err := FromString(test)
			if err != nil {
				t.Fatalf("Error on FromString('%s'): %s", test, err.Error())
			}

			_, err = tpl.Execute(tplContext)
			if err == nil {
				t.Fatalf("[%s Line %d] Expected error for (got none): %s",
					match, idx+1, tests[idx])
			}
			re := regexp.MustCompile(fmt.Sprintf("^%s$", checks[idx]))
			if !re.MatchString(test) {
				t.Fatalf("[%s Line %d] Error for '%s' does not match the (regexp-)check: %s",
					match, idx+1, test, checks[idx])
			}
		}
	}
}

func TestCompilationErrors(t *testing.T) {
	SetDebug(true) // activate pongo2's debugging output

	matches, err := filepath.Glob("./template_tests/*-compilation.err")
	if err != nil {
		t.Fatal(err)
	}
	for idx, match := range matches {
		t.Logf("[Errors %3d] Testing '%s'", idx+1, match)

		test_data, err := ioutil.ReadFile(match)
		tests := strings.Split(string(test_data), "\n")

		check_filename := fmt.Sprintf("%s.out", match)
		check_data, err := ioutil.ReadFile(check_filename)
		if err != nil {
			t.Fatalf("Error on ReadFile('%s'): %s", check_filename, err.Error())
		}
		checks := strings.Split(string(check_data), "\n")

		if len(checks) != len(tests) {
			t.Fatal("Template lines != Checks lines")
		}

		for idx, test := range tests {
			if strings.TrimSpace(test) == "" {
				continue
			}
			if strings.TrimSpace(checks[idx]) == "" {
				t.Fatalf("[%s Line %d] Check is empty (must contain an regular expression).",
					match, idx+1)
			}

			_, err = FromString(test)
			if err == nil {
				t.Fatalf("[%s | Line %d] Expected error for (got none): %s", match, idx+1, tests[idx])
			}
			re := regexp.MustCompile(fmt.Sprintf("^%s$", checks[idx]))
			if !re.MatchString(err.Error()) {
				t.Fatalf("[%s | Line %d] Error for '%s' (err = '%s') does not match the (regexp-)check: %s",
					match, idx+1, test, err.Error(), checks[idx])
			}
		}
	}
}

func BenchmarkExecuteComplex(b *testing.B) {
	tpl, err := FromFile("template_tests/complex.tpl")
	if err != nil {
		b.Fatal(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, err = tpl.Execute(tplContext)
		if err != nil {
			b.Fatal(err)
		}
	}
}

func BenchmarkCompileAndExecuteComplex(b *testing.B) {
	buf, err := ioutil.ReadFile("template_tests/complex.tpl")
	if err != nil {
		b.Fatal(err)
	}
	preloadedTpl := string(buf)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		tpl, err := FromString(preloadedTpl)
		if err != nil {
			b.Fatal(err)
		}

		_, err = tpl.Execute(tplContext)
		if err != nil {
			b.Fatal(err)
		}
	}
}

func BenchmarkParallelExecuteComplex(b *testing.B) {
	tpl, err := FromFile("template_tests/complex.tpl")
	if err != nil {
		b.Fatal(err)
	}
	b.ResetTimer()
	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			_, err = tpl.Execute(tplContext)
			if err != nil {
				b.Fatal(err)
			}
		}
	})
}
<commit_msg>Fixed bug in test suite.<commit_after>package pongo2

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"regexp"
	"strings"
	"testing"
	"time"
)

var admin_list = []string{"user2"}

var time1 = time.Date(2014, 06, 10, 15, 30, 15, 0, time.UTC)
var time2 = time.Date(2011, 03, 21, 8, 37, 56, 12, time.UTC)

type post struct {
	Text    string
	Created time.Time
}

type user struct {
	Name      string
	Validated bool
}

type comment struct {
	Author *user
	Date   time.Time
	Text   string
}

func is_admin(in *Value) *Value {
	u, worked := in.Interface().(*user)
	if !worked {
		return AsValue(false)
	}
	for _, a := range admin_list {
		if a == u.Name {
			return AsValue(true)
		}
	}
	return AsValue(false)
}

func (u *user) Is_admin() *Value {
	return is_admin(AsValue(u))
}

func (p *post) String() string {
	return ":-)"
}

var tplContext = Context{
	"number": 11,
	"simple": map[string]interface{}{
		"number":        42,
		"name":          "john doe",
		"included_file": "INCLUDES.helper",
		"nil":           nil,
		"uint":          uint(8),
		"float":         float64(3.1415),
		"str":           "string",
		"bool_true":     true,
		"bool_false":    false,
		"newline_text": `this is a text
with a new line in it`,
		"intmap": map[int]string{
			5: "five",
		},
	},
	"complex": map[string]interface{}{
		"is_admin": is_admin,
		"post": post{
			Text:    "
Hello!

Welcome to my new blog page. I'm using pongo2 which supports {{ variables }} and {% tags %}.
",
			Created: time2,
		},
		"comments": []*comment{
			&comment{
				Author: &user{
					Name:      "user1",
					Validated: true,
				},
				Date: time1,
				Text: "\"pongo2 is nice!\"",
			},
			&comment{
				Author: &user{
					Name:      "user2",
					Validated: true,
				},
				Date: time2,
				Text: "comment2 with  tags in it",
			},
			&comment{
				Author: &user{
					Name:      "user3",
					Validated: false,
				},
				Date: time1,
				Text: "hello! there",
			},
		},
	},
}

func TestTemplates(t *testing.T) {
	SetDebug(true) // activate pongo2's debugging output

	matches, err := filepath.Glob("./template_tests/*.tpl")
	if err != nil {
		t.Fatal(err)
	}
	for idx, match := range matches {
		t.Logf("[Template %3d] Testing '%s'", idx+1, match)
		tpl, err := FromFile(match)
		if err != nil {
			t.Fatalf("Error on FromFile('%s'): %s", match, err.Error())
		}
		test_filename := fmt.Sprintf("%s.out", match)
		test_out, err := ioutil.ReadFile(test_filename)
		if err != nil {
			t.Fatalf("Error on ReadFile('%s'): %s", test_filename, err.Error())
		}
		tpl_out, err := tpl.Execute(tplContext)
		if err != nil {
			t.Fatalf("Error on Execute('%s'): %s", match, err.Error())
		}
		if string(test_out) != tpl_out {
			t.Logf("Template (rendered) '%s': '%s'", match, tpl_out)
			err_filename := "tpl-error.out"
			ioutil.WriteFile(err_filename, []byte(tpl_out), 0700)
			t.Logf("get a complete diff with command: 'diff -ya %s %s'", test_filename, err_filename)
			t.Fatalf("Failed: test_out != tpl_out for %s", match)
		}
	}
}

func TestExecutionErrors(t *testing.T) {
	SetDebug(true) // activate pongo2's debugging output

	matches, err := filepath.Glob("./template_tests/*-execution.err")
	if err != nil {
		t.Fatal(err)
	}
	for idx, match := range matches {
		t.Logf("[Errors %3d] Testing '%s'", idx+1, match)

		test_data, err := ioutil.ReadFile(match)
		tests := strings.Split(string(test_data), "\n")

		check_filename := fmt.Sprintf("%s.out", match)
		check_data, err := ioutil.ReadFile(check_filename)
		if err != nil {
			t.Fatalf("Error on ReadFile('%s'): %s", check_filename, err.Error())
		}
		checks := strings.Split(string(check_data), "\n")

		if len(checks) != len(tests) {
			t.Fatal("Template lines != Checks lines")
		}

		for idx, test := range tests {
			if strings.TrimSpace(test) == "" {
				continue
			}
			if strings.TrimSpace(checks[idx]) == "" {
				t.Fatalf("[%s Line %d] Check is empty (must contain an regular expression).",
					match, idx+1)
			}

			tpl, err := FromString(test)
			if err != nil {
				t.Fatalf("Error on FromString('%s'): %s", test, err.Error())
			}

			_, err = tpl.Execute(tplContext)
			if err == nil {
				t.Fatalf("[%s Line %d] Expected error for (got none): %s",
					match, idx+1, tests[idx])
			}
			re := regexp.MustCompile(fmt.Sprintf("^%s$", checks[idx]))
			if !re.MatchString(err.Error()) {
				t.Fatalf("[%s Line %d] Error for '%s' (err = '%s') does not match the (regexp-)check: %s",
					match, idx+1, test, err.Error(), checks[idx])
			}
		}
	}
}

func TestCompilationErrors(t *testing.T) {
	SetDebug(true) // activate pongo2's debugging output

	matches, err := filepath.Glob("./template_tests/*-compilation.err")
	if err != nil {
		t.Fatal(err)
	}
	for idx, match := range matches {
		t.Logf("[Errors %3d] Testing '%s'", idx+1, match)

		test_data, err := ioutil.ReadFile(match)
		tests := strings.Split(string(test_data), "\n")

		check_filename := fmt.Sprintf("%s.out", match)
		check_data, err := ioutil.ReadFile(check_filename)
		if err != nil {
			t.Fatalf("Error on ReadFile('%s'): %s", check_filename, err.Error())
		}
		checks := strings.Split(string(check_data), "\n")

		if len(checks) != len(tests) {
			t.Fatal("Template lines != Checks lines")
		}

		for idx, test := range tests {
			if strings.TrimSpace(test) == "" {
				continue
			}
			if strings.TrimSpace(checks[idx]) == "" {
				t.Fatalf("[%s Line %d] Check is empty (must contain an regular expression).",
					match, idx+1)
			}

			_, err = FromString(test)
			if err == nil {
				t.Fatalf("[%s | Line %d] Expected error for (got none): %s", match, idx+1, tests[idx])
			}
			re := regexp.MustCompile(fmt.Sprintf("^%s$", checks[idx]))
			if !re.MatchString(err.Error()) {
				t.Fatalf("[%s | Line %d] Error for '%s' (err = '%s') does not match the (regexp-)check: %s",
					match, idx+1, test, err.Error(), checks[idx])
			}
		}
	}
}

func BenchmarkExecuteComplex(b *testing.B) {
	tpl, err := FromFile("template_tests/complex.tpl")
	if err != nil {
		b.Fatal(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, err = tpl.Execute(tplContext)
		if err != nil {
			b.Fatal(err)
		}
	}
}

func BenchmarkCompileAndExecuteComplex(b *testing.B) {
	buf, err := ioutil.ReadFile("template_tests/complex.tpl")
	if err != nil {
		b.Fatal(err)
	}
	preloadedTpl := string(buf)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		tpl, err := FromString(preloadedTpl)
		if err != nil {
			b.Fatal(err)
		}

		_, err = tpl.Execute(tplContext)
		if err != nil {
			b.Fatal(err)
		}
	}
}

func BenchmarkParallelExecuteComplex(b *testing.B) {
	tpl, err := FromFile("template_tests/complex.tpl")
	if err != nil {
		b.Fatal(err)
	}
	b.ResetTimer()
	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			_, err = tpl.Execute(tplContext)
			if err != nil {
				b.Fatal(err)
			}
		}
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package workqueue

import (
	"sync"
	"time"

	"k8s.io/apimachinery/pkg/util/clock"
)

type Interface interface {
	Add(item interface{})
	Len() int
	Get() (item interface{}, shutdown bool)
	Done(item interface{})
	ShutDown()
	ShutDownWithDrain()
	ShuttingDown() bool
}

// New constructs a new work queue (see the package comment).
func New() *Type {
	return NewNamed("")
}

func NewNamed(name string) *Type {
	rc := clock.RealClock{}
	return newQueue(
		rc,
		globalMetricsFactory.newQueueMetrics(name, rc),
		defaultUnfinishedWorkUpdatePeriod,
	)
}

func newQueue(c clock.Clock, metrics queueMetrics, updatePeriod time.Duration) *Type {
	t := &Type{
		clock:                      c,
		dirty:                      set{},
		processing:                 set{},
		cond:                       sync.NewCond(&sync.Mutex{}),
		metrics:                    metrics,
		unfinishedWorkUpdatePeriod: updatePeriod,
	}

	// Don't start the goroutine for a type of noMetrics so we don't consume
	// resources unnecessarily
	if _, ok := metrics.(noMetrics); !ok {
		go t.updateUnfinishedWorkLoop()
	}

	return t
}

const defaultUnfinishedWorkUpdatePeriod = 500 * time.Millisecond

// Type is a work queue (see the package comment).
type Type struct {
	// queue defines the order in which we will work on items. Every
	// element of queue should be in the dirty set and not in the
	// processing set.
	queue []t

	// dirty defines all of the items that need to be processed.
	dirty set

	// Things that are currently being processed are in the processing set.
	// These things may be simultaneously in the dirty set. When we finish
	// processing something and remove it from this set, we'll check if
	// it's in the dirty set, and if so, add it to the queue.
	processing set

	cond *sync.Cond

	shuttingDown bool
	drain        bool

	metrics queueMetrics

	unfinishedWorkUpdatePeriod time.Duration
	clock                      clock.Clock
}

type empty struct{}
type t interface{}
type set map[t]empty

func (s set) has(item t) bool {
	_, exists := s[item]
	return exists
}

func (s set) insert(item t) {
	s[item] = empty{}
}

func (s set) delete(item t) {
	delete(s, item)
}

func (s set) len() int {
	return len(s)
}

// Add marks item as needing processing.
func (q *Type) Add(item interface{}) {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	if q.shuttingDown {
		return
	}
	if q.dirty.has(item) {
		return
	}

	q.metrics.add(item)

	q.dirty.insert(item)
	if q.processing.has(item) {
		return
	}

	q.queue = append(q.queue, item)
	q.cond.Signal()
}

// Len returns the current queue length, for informational purposes only. You
// shouldn't e.g. gate a call to Add() or Get() on Len() being a particular
// value, that can't be synchronized properly.
func (q *Type) Len() int {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	return len(q.queue)
}

// Get blocks until it can return an item to be processed. If shutdown = true,
// the caller should end their goroutine. You must call Done with item when you
// have finished processing it.
func (q *Type) Get() (item interface{}, shutdown bool) {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	for len(q.queue) == 0 && !q.shuttingDown {
		q.cond.Wait()
	}
	if len(q.queue) == 0 {
		// We must be shutting down.
		return nil, true
	}

	item, q.queue = q.queue[0], q.queue[1:]

	q.metrics.get(item)

	q.processing.insert(item)
	q.dirty.delete(item)

	return item, false
}

// Done marks item as done processing, and if it has been marked as dirty again
// while it was being processed, it will be re-added to the queue for
// re-processing.
func (q *Type) Done(item interface{}) {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()

	q.metrics.done(item)

	q.processing.delete(item)
	if q.dirty.has(item) {
		q.queue = append(q.queue, item)
		q.cond.Signal()
	} else if q.processing.len() == 0 {
		q.cond.Signal()
	}
}

// ShutDown will cause q to ignore all new items added to it and
// immediately instruct the worker goroutines to exit.
func (q *Type) ShutDown() {
	q.setDrain(false)
	q.shutdown()
}

// ShutDownWithDrain will cause q to ignore all new items added to it. As soon
// as the worker goroutines have "drained", i.e: finished processing and called
// Done on all existing items in the queue; they will be instructed to exit and
// ShutDownWithDrain will return. Hence: a strict requirement for using this is;
// your workers must ensure that Done is called on all items in the queue once
// the shut down has been initiated, if that is not the case: this will block
// indefinitely. It is, however, safe to call ShutDown after having called
// ShutDownWithDrain, as to force the queue shut down to terminate immediately
// without waiting for the drainage.
func (q *Type) ShutDownWithDrain() {
	q.setDrain(true)
	q.shutdown()
	for q.isProcessing() && q.shouldDrain() {
		q.waitForProcessing()
	}
}

// isProcessing indicates if there are still items on the work queue being
// processed. It's used to drain the work queue on an eventual shutdown.
func (q *Type) isProcessing() bool {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	return q.processing.len() != 0
}

// waitForProcessing waits for the worker goroutines to finish processing items
// and call Done on them.
func (q *Type) waitForProcessing() {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	// Ensure that we do not wait on a queue which is already empty, as that
	// could result in waiting for Done to be called on items in an empty queue
	// which has already been shut down, which will result in waiting
	// indefinitely.
	if q.processing.len() == 0 {
		return
	}
	q.cond.Wait()
}

func (q *Type) setDrain(shouldDrain bool) {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	q.drain = shouldDrain
}

func (q *Type) shouldDrain() bool {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	return q.drain
}

func (q *Type) shutdown() {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	q.shuttingDown = true
	q.cond.Broadcast()
}

func (q *Type) ShuttingDown() bool {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()

	return q.shuttingDown
}

func (q *Type) updateUnfinishedWorkLoop() {
	t := q.clock.NewTicker(q.unfinishedWorkUpdatePeriod)
	defer t.Stop()
	for range t.C() {
		if !func() bool {
			q.cond.L.Lock()
			defer q.cond.L.Unlock()
			if !q.shuttingDown {
				q.metrics.updateUnfinishedWork()
				return true
			}
			return false

		}() {
			return
		}
	}
}
<commit_msg>Fix workqueue memory leak<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package workqueue

import (
	"sync"
	"time"

	"k8s.io/apimachinery/pkg/util/clock"
)

type Interface interface {
	Add(item interface{})
	Len() int
	Get() (item interface{}, shutdown bool)
	Done(item interface{})
	ShutDown()
	ShutDownWithDrain()
	ShuttingDown() bool
}

// New constructs a new work queue (see the package comment).
func New() *Type {
	return NewNamed("")
}

func NewNamed(name string) *Type {
	rc := clock.RealClock{}
	return newQueue(
		rc,
		globalMetricsFactory.newQueueMetrics(name, rc),
		defaultUnfinishedWorkUpdatePeriod,
	)
}

func newQueue(c clock.Clock, metrics queueMetrics, updatePeriod time.Duration) *Type {
	t := &Type{
		clock:                      c,
		dirty:                      set{},
		processing:                 set{},
		cond:                       sync.NewCond(&sync.Mutex{}),
		metrics:                    metrics,
		unfinishedWorkUpdatePeriod: updatePeriod,
	}

	// Don't start the goroutine for a type of noMetrics so we don't consume
	// resources unnecessarily
	if _, ok := metrics.(noMetrics); !ok {
		go t.updateUnfinishedWorkLoop()
	}

	return t
}

const defaultUnfinishedWorkUpdatePeriod = 500 * time.Millisecond

// Type is a work queue (see the package comment).
type Type struct {
	// queue defines the order in which we will work on items. Every
	// element of queue should be in the dirty set and not in the
	// processing set.
	queue []t

	// dirty defines all of the items that need to be processed.
	dirty set

	// Things that are currently being processed are in the processing set.
	// These things may be simultaneously in the dirty set. When we finish
	// processing something and remove it from this set, we'll check if
	// it's in the dirty set, and if so, add it to the queue.
	processing set

	cond *sync.Cond

	shuttingDown bool
	drain        bool

	metrics queueMetrics

	unfinishedWorkUpdatePeriod time.Duration
	clock                      clock.Clock
}

type empty struct{}
type t interface{}
type set map[t]empty

func (s set) has(item t) bool {
	_, exists := s[item]
	return exists
}

func (s set) insert(item t) {
	s[item] = empty{}
}

func (s set) delete(item t) {
	delete(s, item)
}

func (s set) len() int {
	return len(s)
}

// Add marks item as needing processing.
func (q *Type) Add(item interface{}) {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	if q.shuttingDown {
		return
	}
	if q.dirty.has(item) {
		return
	}

	q.metrics.add(item)

	q.dirty.insert(item)
	if q.processing.has(item) {
		return
	}

	q.queue = append(q.queue, item)
	q.cond.Signal()
}

// Len returns the current queue length, for informational purposes only. You
// shouldn't e.g. gate a call to Add() or Get() on Len() being a particular
// value, that can't be synchronized properly.
func (q *Type) Len() int {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	return len(q.queue)
}

// Get blocks until it can return an item to be processed. If shutdown = true,
// the caller should end their goroutine. You must call Done with item when you
// have finished processing it.
func (q *Type) Get() (item interface{}, shutdown bool) {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	for len(q.queue) == 0 && !q.shuttingDown {
		q.cond.Wait()
	}
	if len(q.queue) == 0 {
		// We must be shutting down.
		return nil, true
	}

	item = q.queue[0]
	// The underlying array still exists and reference this object, so the object will not be garbage collected.
	q.queue[0] = nil
	q.queue = q.queue[1:]

	q.metrics.get(item)

	q.processing.insert(item)
	q.dirty.delete(item)

	return item, false
}

// Done marks item as done processing, and if it has been marked as dirty again
// while it was being processed, it will be re-added to the queue for
// re-processing.
func (q *Type) Done(item interface{}) {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()

	q.metrics.done(item)

	q.processing.delete(item)
	if q.dirty.has(item) {
		q.queue = append(q.queue, item)
		q.cond.Signal()
	} else if q.processing.len() == 0 {
		q.cond.Signal()
	}
}

// ShutDown will cause q to ignore all new items added to it and
// immediately instruct the worker goroutines to exit.
func (q *Type) ShutDown() {
	q.setDrain(false)
	q.shutdown()
}

// ShutDownWithDrain will cause q to ignore all new items added to it. As soon
// as the worker goroutines have "drained", i.e: finished processing and called
// Done on all existing items in the queue; they will be instructed to exit and
// ShutDownWithDrain will return. Hence: a strict requirement for using this is;
// your workers must ensure that Done is called on all items in the queue once
// the shut down has been initiated, if that is not the case: this will block
// indefinitely. It is, however, safe to call ShutDown after having called
// ShutDownWithDrain, as to force the queue shut down to terminate immediately
// without waiting for the drainage.
func (q *Type) ShutDownWithDrain() {
	q.setDrain(true)
	q.shutdown()
	for q.isProcessing() && q.shouldDrain() {
		q.waitForProcessing()
	}
}

// isProcessing indicates if there are still items on the work queue being
// processed. It's used to drain the work queue on an eventual shutdown.
func (q *Type) isProcessing() bool {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	return q.processing.len() != 0
}

// waitForProcessing waits for the worker goroutines to finish processing items
// and call Done on them.
func (q *Type) waitForProcessing() {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	// Ensure that we do not wait on a queue which is already empty, as that
	// could result in waiting for Done to be called on items in an empty queue
	// which has already been shut down, which will result in waiting
	// indefinitely.
	if q.processing.len() == 0 {
		return
	}
	q.cond.Wait()
}

func (q *Type) setDrain(shouldDrain bool) {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	q.drain = shouldDrain
}

func (q *Type) shouldDrain() bool {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	return q.drain
}

func (q *Type) shutdown() {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()
	q.shuttingDown = true
	q.cond.Broadcast()
}

func (q *Type) ShuttingDown() bool {
	q.cond.L.Lock()
	defer q.cond.L.Unlock()

	return q.shuttingDown
}

func (q *Type) updateUnfinishedWorkLoop() {
	t := q.clock.NewTicker(q.unfinishedWorkUpdatePeriod)
	defer t.Stop()
	for range t.C() {
		if !func() bool {
			q.cond.L.Lock()
			defer q.cond.L.Unlock()
			if !q.shuttingDown {
				q.metrics.updateUnfinishedWork()
				return true
			}
			return false

		}() {
			return
		}
	}
}
<|endoftext|>


<commit_before>package openid

import "testing"

func Test_getsigningKey_WhenKeyIsCached(t *testing.T) {
	_, keyCache := createSigningKeyMapCache(t)

	iss := "issuer"
	kid := "kid1"
	key := "signingKey"
	keyCache.jwksMap[iss] = []signingKey{{keyID: kid, key: []byte(key)}}

	sk, re := keyCache.getSigningKey(iss, kid)

	if re != nil {
		t.Error("An error was returned but not expected.")
	}

	if sk == nil {
		t.Fatal("The returned signing key should not be nil.")
	}

	keyStr := string(sk)

	if keyStr != key {
		t.Error("Expected key", key, "but got", keyStr)
	}
}

func createSigningKeyMapCache(t *testing.T) (*signingKeyGetterMock, *signingKeyMapCache) {
	mock := newSigningKeyGetterMock(t)
	return mock, newSigningKeyMapCache(mock)
}
<commit_msg>Adding more tests for the signingkeycache<commit_after>package openid

import (
	"net/http"
	"testing"
)

func Test_getsigningKey_WhenKeyIsCached(t *testing.T) {
	_, keyCache := createSigningKeyMapCache(t)

	iss := "issuer"
	kid := "kid1"
	key := "signingKey"
	keyCache.jwksMap[iss] = []signingKey{{keyID: kid, key: []byte(key)}}

	expectKey(t, keyCache, iss, kid, key)
}

func Test_getsigningKey_WhenKeyIsNotCached_WhenProviderReturnsKey(t *testing.T) {
	keyGetter, keyCache := createSigningKeyMapCache(t)

	iss := "issuer"
	kid := "kid1"
	key := "signingKey"

	go func() {
		keyGetter.assertGetSigningKeys(iss, []signingKey{{keyID: kid, key: []byte(key)}}, nil)
		keyGetter.close()
	}()

	expectKey(t, keyCache, iss, kid, key)

	// Validate that the key is cached
	expectCachedKid(t, keyCache, iss, kid, key)

	keyGetter.assertDone()
}

func Test_getsigningKey_WhenProviderReturnsError(t *testing.T) {
	keyGetter, keyCache := createSigningKeyMapCache(t)

	iss := "issuer"
	kid := "kid1"
	ee := &ValidationError{Code: ValidationErrorGetJwksFailure, HTTPStatus: http.StatusUnauthorized}

	go func() {
		keyGetter.assertGetSigningKeys(iss, nil, ee)
		keyGetter.close()
	}()

	rk, re := keyCache.getSigningKey(iss, kid)

	expectValidationError(t, re, ee.Code, ee.HTTPStatus, nil)

	if rk != nil {
		t.Error("A key was returned but not expected")
	}

	cachedKeys := keyCache.jwksMap[iss]
	if len(cachedKeys) != 0 {
		t.Fatal("There shouldnt be cached keys for the targeted issuer.")
	}

	keyGetter.assertDone()
}

func Test_getsigningKey_WhenKeyIsNotFound(t *testing.T) {
	keyGetter, keyCache := createSigningKeyMapCache(t)

	iss := "issuer"
	kid := "kid1"
	tkid := "kid2"
	key := "signingKey"

	go func() {
		keyGetter.assertGetSigningKeys(iss, []signingKey{{keyID: kid, key: []byte(key)}}, nil)
		keyGetter.close()
	}()

	rk, re := keyCache.getSigningKey(iss, tkid)

	expectValidationError(t, re, ValidationErrorKidNotFound, http.StatusUnauthorized, nil)

	if rk != nil {
		t.Error("A key was returned but not expected")
	}

	expectCachedKid(t, keyCache, iss, kid, key)

	keyGetter.assertDone()
}

func Test_flushSigningKeys_FlushedKeysAreDeleted(t *testing.T) {
	_, keyCache := createSigningKeyMapCache(t)

	iss := "issuer"
	iss2 := "issuer2"
	kid := "kid1"
	key := "signingKey"
	keyCache.jwksMap[iss] = []signingKey{{keyID: kid, key: []byte(key)}}
	keyCache.jwksMap[iss2] = []signingKey{{keyID: kid, key: []byte(key)}}

	keyCache.flushSigningKeys(iss2)

	dk := keyCache.jwksMap[iss2]

	if dk != nil {
		t.Error("Flushed keys should not be in the cache.")
	}

	expectCachedKid(t, keyCache, iss, kid, key)
}

func Test_flushsigningKey_RetrieveFlushedKey(t *testing.T) {
	keyGetter, keyCache := createSigningKeyMapCache(t)

	iss := "issuer"
	kid := "kid1"
	key := "signingKey"

	go func() {
		keyGetter.assertGetSigningKeys(iss, []signingKey{{keyID: kid, key: []byte(key)}}, nil)
		keyGetter.assertGetSigningKeys(iss, []signingKey{{keyID: kid, key: []byte(key)}}, nil)

		keyGetter.close()
	}()

	// Get the signing key not yet cached will cache it.
	expectKey(t, keyCache, iss, kid, key)

	// Flush the signing keys for the given provider.
	keyCache.flushSigningKeys(iss)

	// Get the signing key will once again call the provider and cache the keys.

	expectKey(t, keyCache, iss, kid, key)

	// Validate that the key is cached
	expectCachedKid(t, keyCache, iss, kid, key)

	keyGetter.assertDone()
}

func expectCachedKid(t *testing.T, keyCache *signingKeyMapCache, iss string, kid string, key string) {

	cachedKeys := keyCache.jwksMap[iss]
	if len(cachedKeys) == 0 {
		t.Fatal("The keys were not cached as expected.")
	}

	foundKid := false
	for _, cachedKey := range cachedKeys {
		if cachedKey.keyID == kid {
			foundKid = true
			if keyStr := string(cachedKey.key); keyStr != key {
				t.Error("Expected key", key, "but got", keyStr)
			}

			continue
		}
	}

	if !foundKid {
		t.Error("A key with key id", kid, "was not found.")
	}
}

func expectKey(t *testing.T, c signingKeyCache, iss string, kid string, key string) {

	sk, re := c.getSigningKey(iss, kid)

	if re != nil {
		t.Error("An error was returned but not expected.")
	}

	if sk == nil {
		t.Fatal("The returned signing key should not be nil.")
	}

	keyStr := string(sk)

	if keyStr != key {
		t.Error("Expected key", key, "but got", keyStr)
	}
}

func createSigningKeyMapCache(t *testing.T) (*signingKeyGetterMock, *signingKeyMapCache) {
	mock := newSigningKeyGetterMock(t)
	return mock, newSigningKeyMapCache(mock)
}
<|endoftext|>


<commit_before>package programs

import (
	"github.com/pufferpanel/pufferd/environments"
	"github.com/pufferpanel/apufferi/logging"
	"github.com/pufferpanel/pufferd/programs/operations"
	"github.com/pufferpanel/apufferi/common"
	"os"
	"io/ioutil"
	"errors"
	"github.com/pufferpanel/apufferi/config"
	"encoding/json"
)

type ProgramData struct {
	Data            map[string]DataObject  `json:"data"`
	Display         string                 `json:"display"`
	EnvironmentData map[string]interface{} `json:"environment"`
	InstallData     InstallSection         `json:"install"`
	Type            string                 `json:"type"`
	Identifier      string                 `json:"id"`
	RunData         RunObject              `json:"run"`

	Environment     environments.Environment `json:"-"`
	CrashCounter    int 				   `json:"-"`
}

type DataObject struct {
	Description string      `json:"desc"`
	Display     string      `json:"display"`
	Internal    bool        `json:"internal"`
	Required    bool        `json:"required"`
	Value       interface{} `json:"value"`
}

type RunObject struct {
	Arguments               []string                 `json:"arguments"`
	Program                 string                   `json:"program"`
	Stop                    string                   `json:"stop"`
	Enabled                 bool                     `json:"enabled"`
	AutoStart               bool                     `json:"autostart"`
	AutoRestartFromCrash    bool                     `json:"autorecover"`
	AutoRestartFromGraceful bool                     `json:"autorestart"`
	Pre                     []map[string]interface{} `json:"pre"`
}

type InstallSection struct {
	Operations []map[string]interface{} `json:"commands"`
}

func (p ProgramData) DataToMap() map[string]interface{} {
	var result = make(map[string]interface{}, len(p.Data))

	for k, v := range p.Data {
		result[k] = v.Value
	}

	return result
}

func CreateProgram() ProgramData{
	return ProgramData{
		RunData: RunObject{
			Enabled: true,
			AutoStart: false,
			Pre: make([]map[string]interface{}, 0),
		},
		Type: "standard",
		Data: make(map[string]DataObject, 0),
		Display: "Unknown server",
		InstallData: InstallSection{
			Operations: make([]map[string]interface{}, 0),
		},
	}
}

//Starts the program.
//This includes starting the environment if it is not running.
func (p *ProgramData) Start() (err error) {
	if !p.IsEnabled() {
		logging.Errorf("Server %s is not enabled, cannot start", p.Id())
		return errors.New("server not enabled")
	}
	logging.Debugf("Starting server %s", p.Id())
	p.Environment.DisplayToConsole("Starting server\n")
	data := make(map[string]interface{})
	for k, v := range p.Data {
		data[k] = v.Value
	}

	process := operations.GenerateProcess(p.RunData.Pre, p.Environment, p.DataToMap())
	err = process.Run()
	if err != nil {
		p.Environment.DisplayToConsole("Error running pre execute, check daemon logs")
		return
	}

	err = p.Environment.ExecuteAsync(p.RunData.Program, common.ReplaceTokensInArr(p.RunData.Arguments, data), func(graceful bool) {
		if graceful {
			p.CrashCounter = 0
		}

		if graceful && p.RunData.AutoRestartFromGraceful {
			StartViaService(p)
		} else if !graceful && p.RunData.AutoRestartFromCrash && p.CrashCounter < config.GetIntOrDefault("crashlimit", 3) {
			p.CrashCounter++
			StartViaService(p)
		}
	})
	if err != nil {
		logging.Error("Error starting server", err)
		p.Environment.DisplayToConsole("Failed to start server\n")
	}

	return
}

//Stops the program.
//This will also stop the environment it is ran in.
func (p *ProgramData) Stop() (err error) {
	logging.Debugf("Stopping server %s", p.Id())
	err = p.Environment.ExecuteInMainProcess(p.RunData.Stop)
	if err != nil {
		p.Environment.DisplayToConsole("Failed to stop server\n")
	} else {
		p.Environment.DisplayToConsole("Server stopped\n")
	}
	return
}

//Kills the program.
//This will also stop the environment it is ran in.
func (p *ProgramData) Kill() (err error) {
	logging.Debugf("Killing server %s", p.Id())
	err = p.Environment.Kill()
	if err != nil {
		p.Environment.DisplayToConsole("Failed to kill server\n")
	} else {
		p.Environment.DisplayToConsole("Server killed\n")
	}
	return
}

//Creates any files needed for the program.
//This includes creating the environment.
func (p *ProgramData) Create() (err error) {
	logging.Debugf("Creating server %s", p.Id())
	p.Environment.DisplayToConsole("Allocating server\n")
	err = p.Environment.Create()
	p.Environment.DisplayToConsole("Server allocated\n")
	p.Environment.DisplayToConsole("Ready to be installed\n")
	return
}

//Destroys the server.
//This will delete the server, environment, and any files related to it.
func (p *ProgramData) Destroy() (err error) {
	logging.Debugf("Destroying server %s", p.Id())
	err = p.Environment.Delete()
	return
}

func (p *ProgramData) Install() (err error) {
	if !p.IsEnabled() {
		logging.Errorf("Server %s is not enabled, cannot install", p.Id())
		return errors.New("server not enabled")
	}

	logging.Debugf("Installing server %s", p.Id())
	running, err := p.IsRunning()
	if err != nil {
		logging.Error("Error stopping server to install: ", err)
		p.Environment.DisplayToConsole("Error stopping server\n")
		return
	}

	if running {
		err = p.Stop()
	}

	if err != nil {
		logging.Error("Error stopping server to install: ", err)
		p.Environment.DisplayToConsole("Error stopping server\n")
		return
	}

	p.Environment.DisplayToConsole("Installing server\n")

	os.MkdirAll(p.Environment.GetRootDirectory(), 0755)

	process := operations.GenerateProcess(p.InstallData.Operations, p.GetEnvironment(), p.DataToMap())
	err = process.Run()
	if err != nil {
		p.Environment.DisplayToConsole("Error running installer, check daemon logs")
	} else {
		p.Environment.DisplayToConsole("Server installed\n")
	}
	return
}

//Determines if the server is running.
func (p *ProgramData) IsRunning() (isRunning bool, err error) {
	isRunning, err = p.Environment.IsRunning()
	return
}

//Sends a command to the process
//If the program supports input, this will send the arguments to that.
func (p *ProgramData) Execute(command string) (err error) {
	err = p.Environment.ExecuteInMainProcess(command)
	return
}

func (p *ProgramData) SetEnabled(isEnabled bool) (err error) {
	p.RunData.Enabled = isEnabled
	return
}

func (p *ProgramData) IsEnabled() (isEnabled bool) {
	isEnabled = p.RunData.Enabled
	return
}

func (p *ProgramData) SetEnvironment(environment environments.Environment) (err error) {
	p.Environment = environment
	return
}

func (p *ProgramData) Id() string {
	return p.Identifier
}

func (p *ProgramData) GetEnvironment() environments.Environment {
	return p.Environment
}

func (p *ProgramData) SetAutoStart(isAutoStart bool) (err error) {
	p.RunData.AutoStart = isAutoStart
	return
}

func (p *ProgramData) IsAutoStart() (isAutoStart bool) {
	isAutoStart = p.RunData.AutoStart
	return
}

func (p *ProgramData) Save(file string) (err error) {
	logging.Debugf("Saving server %s", p.Id())

	endResult := make(map[string]interface{})
	endResult["pufferd"] = p

	data, err := json.MarshalIndent(endResult, "", "  ")
	if err != nil {
		return
	}

	err = ioutil.WriteFile(file, data, 0664)
	return
}

func (p *ProgramData) Edit(data map[string]interface{}) (err error) {
	for k, v := range data {
		if v == nil || v == "" {
			delete(p.Data, k)
		}

		var elem DataObject

		if _, ok := p.Data[k]; ok {
			elem = p.Data[k]
		} else {
			elem = DataObject{}
		}
		elem.Value = v

		p.Data[k] = elem
	}
	err = Save(p.Id())
	return
}

func (p *ProgramData) GetData() map[string]DataObject {
	return p.Data
}

func (p *ProgramData) GetNetwork() string {
	data := p.GetData()
	ip := "0.0.0.0"
	port := "0"

	if ipData, ok := data["ip"]; ok {
		ip = ipData.Value.(string)
	}

	if portData, ok := data["port"]; ok {
		port = portData.Value.(string)
	}

	return ip + ":" + port
}

func (p *ProgramData) CopyFrom(s *ProgramData) {
	p.RunData = s.RunData
	p.Display = s.Display
	p.EnvironmentData = s.EnvironmentData
	p.InstallData = s.InstallData
	p.Type = s.Type
}<commit_msg>Also reload data<commit_after>package programs

import (
	"github.com/pufferpanel/pufferd/environments"
	"github.com/pufferpanel/apufferi/logging"
	"github.com/pufferpanel/pufferd/programs/operations"
	"github.com/pufferpanel/apufferi/common"
	"os"
	"io/ioutil"
	"errors"
	"github.com/pufferpanel/apufferi/config"
	"encoding/json"
)

type ProgramData struct {
	Data            map[string]DataObject  `json:"data"`
	Display         string                 `json:"display"`
	EnvironmentData map[string]interface{} `json:"environment"`
	InstallData     InstallSection         `json:"install"`
	Type            string                 `json:"type"`
	Identifier      string                 `json:"id"`
	RunData         RunObject              `json:"run"`

	Environment     environments.Environment `json:"-"`
	CrashCounter    int 				   `json:"-"`
}

type DataObject struct {
	Description string      `json:"desc"`
	Display     string      `json:"display"`
	Internal    bool        `json:"internal"`
	Required    bool        `json:"required"`
	Value       interface{} `json:"value"`
}

type RunObject struct {
	Arguments               []string                 `json:"arguments"`
	Program                 string                   `json:"program"`
	Stop                    string                   `json:"stop"`
	Enabled                 bool                     `json:"enabled"`
	AutoStart               bool                     `json:"autostart"`
	AutoRestartFromCrash    bool                     `json:"autorecover"`
	AutoRestartFromGraceful bool                     `json:"autorestart"`
	Pre                     []map[string]interface{} `json:"pre"`
}

type InstallSection struct {
	Operations []map[string]interface{} `json:"commands"`
}

func (p ProgramData) DataToMap() map[string]interface{} {
	var result = make(map[string]interface{}, len(p.Data))

	for k, v := range p.Data {
		result[k] = v.Value
	}

	return result
}

func CreateProgram() ProgramData{
	return ProgramData{
		RunData: RunObject{
			Enabled: true,
			AutoStart: false,
			Pre: make([]map[string]interface{}, 0),
		},
		Type: "standard",
		Data: make(map[string]DataObject, 0),
		Display: "Unknown server",
		InstallData: InstallSection{
			Operations: make([]map[string]interface{}, 0),
		},
	}
}

//Starts the program.
//This includes starting the environment if it is not running.
func (p *ProgramData) Start() (err error) {
	if !p.IsEnabled() {
		logging.Errorf("Server %s is not enabled, cannot start", p.Id())
		return errors.New("server not enabled")
	}
	logging.Debugf("Starting server %s", p.Id())
	p.Environment.DisplayToConsole("Starting server\n")
	data := make(map[string]interface{})
	for k, v := range p.Data {
		data[k] = v.Value
	}

	process := operations.GenerateProcess(p.RunData.Pre, p.Environment, p.DataToMap())
	err = process.Run()
	if err != nil {
		p.Environment.DisplayToConsole("Error running pre execute, check daemon logs")
		return
	}

	err = p.Environment.ExecuteAsync(p.RunData.Program, common.ReplaceTokensInArr(p.RunData.Arguments, data), func(graceful bool) {
		if graceful {
			p.CrashCounter = 0
		}

		if graceful && p.RunData.AutoRestartFromGraceful {
			StartViaService(p)
		} else if !graceful && p.RunData.AutoRestartFromCrash && p.CrashCounter < config.GetIntOrDefault("crashlimit", 3) {
			p.CrashCounter++
			StartViaService(p)
		}
	})
	if err != nil {
		logging.Error("Error starting server", err)
		p.Environment.DisplayToConsole("Failed to start server\n")
	}

	return
}

//Stops the program.
//This will also stop the environment it is ran in.
func (p *ProgramData) Stop() (err error) {
	logging.Debugf("Stopping server %s", p.Id())
	err = p.Environment.ExecuteInMainProcess(p.RunData.Stop)
	if err != nil {
		p.Environment.DisplayToConsole("Failed to stop server\n")
	} else {
		p.Environment.DisplayToConsole("Server stopped\n")
	}
	return
}

//Kills the program.
//This will also stop the environment it is ran in.
func (p *ProgramData) Kill() (err error) {
	logging.Debugf("Killing server %s", p.Id())
	err = p.Environment.Kill()
	if err != nil {
		p.Environment.DisplayToConsole("Failed to kill server\n")
	} else {
		p.Environment.DisplayToConsole("Server killed\n")
	}
	return
}

//Creates any files needed for the program.
//This includes creating the environment.
func (p *ProgramData) Create() (err error) {
	logging.Debugf("Creating server %s", p.Id())
	p.Environment.DisplayToConsole("Allocating server\n")
	err = p.Environment.Create()
	p.Environment.DisplayToConsole("Server allocated\n")
	p.Environment.DisplayToConsole("Ready to be installed\n")
	return
}

//Destroys the server.
//This will delete the server, environment, and any files related to it.
func (p *ProgramData) Destroy() (err error) {
	logging.Debugf("Destroying server %s", p.Id())
	err = p.Environment.Delete()
	return
}

func (p *ProgramData) Install() (err error) {
	if !p.IsEnabled() {
		logging.Errorf("Server %s is not enabled, cannot install", p.Id())
		return errors.New("server not enabled")
	}

	logging.Debugf("Installing server %s", p.Id())
	running, err := p.IsRunning()
	if err != nil {
		logging.Error("Error stopping server to install: ", err)
		p.Environment.DisplayToConsole("Error stopping server\n")
		return
	}

	if running {
		err = p.Stop()
	}

	if err != nil {
		logging.Error("Error stopping server to install: ", err)
		p.Environment.DisplayToConsole("Error stopping server\n")
		return
	}

	p.Environment.DisplayToConsole("Installing server\n")

	os.MkdirAll(p.Environment.GetRootDirectory(), 0755)

	process := operations.GenerateProcess(p.InstallData.Operations, p.GetEnvironment(), p.DataToMap())
	err = process.Run()
	if err != nil {
		p.Environment.DisplayToConsole("Error running installer, check daemon logs")
	} else {
		p.Environment.DisplayToConsole("Server installed\n")
	}
	return
}

//Determines if the server is running.
func (p *ProgramData) IsRunning() (isRunning bool, err error) {
	isRunning, err = p.Environment.IsRunning()
	return
}

//Sends a command to the process
//If the program supports input, this will send the arguments to that.
func (p *ProgramData) Execute(command string) (err error) {
	err = p.Environment.ExecuteInMainProcess(command)
	return
}

func (p *ProgramData) SetEnabled(isEnabled bool) (err error) {
	p.RunData.Enabled = isEnabled
	return
}

func (p *ProgramData) IsEnabled() (isEnabled bool) {
	isEnabled = p.RunData.Enabled
	return
}

func (p *ProgramData) SetEnvironment(environment environments.Environment) (err error) {
	p.Environment = environment
	return
}

func (p *ProgramData) Id() string {
	return p.Identifier
}

func (p *ProgramData) GetEnvironment() environments.Environment {
	return p.Environment
}

func (p *ProgramData) SetAutoStart(isAutoStart bool) (err error) {
	p.RunData.AutoStart = isAutoStart
	return
}

func (p *ProgramData) IsAutoStart() (isAutoStart bool) {
	isAutoStart = p.RunData.AutoStart
	return
}

func (p *ProgramData) Save(file string) (err error) {
	logging.Debugf("Saving server %s", p.Id())

	endResult := make(map[string]interface{})
	endResult["pufferd"] = p

	data, err := json.MarshalIndent(endResult, "", "  ")
	if err != nil {
		return
	}

	err = ioutil.WriteFile(file, data, 0664)
	return
}

func (p *ProgramData) Edit(data map[string]interface{}) (err error) {
	for k, v := range data {
		if v == nil || v == "" {
			delete(p.Data, k)
		}

		var elem DataObject

		if _, ok := p.Data[k]; ok {
			elem = p.Data[k]
		} else {
			elem = DataObject{}
		}
		elem.Value = v

		p.Data[k] = elem
	}
	err = Save(p.Id())
	return
}

func (p *ProgramData) GetData() map[string]DataObject {
	return p.Data
}

func (p *ProgramData) GetNetwork() string {
	data := p.GetData()
	ip := "0.0.0.0"
	port := "0"

	if ipData, ok := data["ip"]; ok {
		ip = ipData.Value.(string)
	}

	if portData, ok := data["port"]; ok {
		port = portData.Value.(string)
	}

	return ip + ":" + port
}

func (p *ProgramData) CopyFrom(s *ProgramData) {
	p.Data = s.Data
	p.RunData = s.RunData
	p.Display = s.Display
	p.EnvironmentData = s.EnvironmentData
	p.InstallData = s.InstallData
	p.Type = s.Type
}<|endoftext|>


<commit_before>package webhooks

import "github.com/nkcraddock/rabbit-hole"

type rabbitFarm struct {
	conn *rabbithole.Client
}

func newRabbitFarm(r *rabbithole.Client) *rabbitFarm {
	return &rabbitFarm{conn: r}
}

func (r *rabbitFarm) SaveUrlQueue(id string) {
	queue := rabbithole.QueueSettings{
		Durable:    false,
		AutoDelete: true,
	}

	exch := rabbithole.ExchangeSettings{
		Type:       "topic",
		Durable:    false,
		AutoDelete: true,
	}

	r.conn.DeclareQueue("/", id, queue)
	r.conn.DeclareExchange("/", id, exch)
}
<commit_msg>Creating a new webhook now creates the queue, exchange, and binds them together<commit_after>package webhooks

import "github.com/nkcraddock/rabbit-hole"

type rabbitFarm struct {
	conn *rabbithole.Client
}

func newRabbitFarm(r *rabbithole.Client) *rabbitFarm {
	return &rabbitFarm{conn: r}
}

func (r *rabbitFarm) SaveUrlQueue(id string) {
	queue := rabbithole.QueueSettings{
		Durable:    false,
		AutoDelete: true,
	}

	exch := rabbithole.ExchangeSettings{
		Type:       "topic",
		Durable:    false,
		AutoDelete: true,
	}

	bind := rabbithole.BindingInfo{
		RoutingKey: "#",
	}

	r.conn.DeclareQueue("/", id, queue)
	r.conn.DeclareExchange("/", id, exch)
	r.conn.DeclareBinding("/", id, "q", id, bind)
}
<|endoftext|>