<commit_before>// Copyright 2020, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// nolint:errcheck
package main

import (
	"context"
	"log"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/exporters/prometheus"
	"go.opentelemetry.io/otel/metric/global"
	"go.opentelemetry.io/otel/sdk/metric/aggregator/histogram"
	controller "go.opentelemetry.io/otel/sdk/metric/controller/basic"
	"go.opentelemetry.io/otel/sdk/metric/export/aggregation"
	processor "go.opentelemetry.io/otel/sdk/metric/processor/basic"
	selector "go.opentelemetry.io/otel/sdk/metric/selector/simple"
	"go.uber.org/zap"
)

func initMeter() {
	config := prometheus.Config{}
	c := controller.New(
		processor.NewFactory(
			selector.NewWithHistogramDistribution(
				histogram.WithExplicitBoundaries(config.DefaultHistogramBoundaries),
			),
			aggregation.CumulativeTemporalitySelector(),
			processor.WithMemory(true),
		),
	)
	exporter, err := prometheus.New(config, c)

	if err != nil {
		log.Panicf("failed to initialize prometheus exporter %v", err)
	}
	http.HandleFunc("/", exporter.ServeHTTP)
	go func() {
		_ = http.ListenAndServe(":8080", nil)
	}()
}

func main() {
	// set up prometheus
	initMeter()
	// logging
	logger, _ := zap.NewProduction()
	defer logger.Sync()
	logger.Info("Start Prometheus metrics app")
	meter := global.MeterProvider().Meter("federation/prom-counter")
	valueRecorder, err := meter.SyncInt64().Histogram("prom_counter")
	if err != nil {
		log.Panicf("failed to initialize histogram %v", err)
	}
	ctx := context.Background()
	valueRecorder.Record(ctx, 0)
	commonLabels := []attribute.KeyValue{attribute.String("A", "1"), attribute.String("B", "2"), attribute.String("C", "3")}
	counter := int64(0)
	valueRecorder.Record(ctx, counter, commonLabels...)
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
	ticker := time.NewTicker(1 * time.Second)
	for {
		select {
		case <-ticker.C:
			time.Sleep(1 * time.Second)
			counter++
			valueRecorder.Record(ctx, counter, commonLabels...)
			break
		case <-c:
			ticker.Stop()
			logger.Info("Stop Prometheus metrics app")
			return
		}
	}
}
<commit_msg>[receiver/simpleprometheus] update example to use Meter directly (#9364)<commit_after>// Copyright 2020, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// nolint:errcheck
package main

import (
	"context"
	"log"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/exporters/prometheus"
	"go.opentelemetry.io/otel/metric"
	"go.opentelemetry.io/otel/sdk/metric/aggregator/histogram"
	controller "go.opentelemetry.io/otel/sdk/metric/controller/basic"
	"go.opentelemetry.io/otel/sdk/metric/export/aggregation"
	processor "go.opentelemetry.io/otel/sdk/metric/processor/basic"
	selector "go.opentelemetry.io/otel/sdk/metric/selector/simple"
	"go.uber.org/zap"
)

func initMeter() metric.Meter {
	config := prometheus.Config{}
	c := controller.New(
		processor.NewFactory(
			selector.NewWithHistogramDistribution(
				histogram.WithExplicitBoundaries(config.DefaultHistogramBoundaries),
			),
			aggregation.CumulativeTemporalitySelector(),
			processor.WithMemory(true),
		),
	)
	exporter, err := prometheus.New(config, c)

	if err != nil {
		log.Panicf("failed to initialize prometheus exporter %v", err)
	}
	http.HandleFunc("/", exporter.ServeHTTP)
	go func() {
		_ = http.ListenAndServe(":8080", nil)
	}()
	return exporter.MeterProvider().Meter("federation/prom-counter")
}

func main() {
	// set up prometheus
	meter := initMeter()
	// logging
	logger, _ := zap.NewProduction()
	defer logger.Sync()
	logger.Info("Start Prometheus metrics app")
	valueRecorder, err := meter.SyncInt64().Histogram("prom_counter")
	if err != nil {
		log.Panicf("failed to initialize histogram %v", err)
	}
	ctx := context.Background()
	valueRecorder.Record(ctx, 0)
	commonLabels := []attribute.KeyValue{attribute.String("A", "1"), attribute.String("B", "2"), attribute.String("C", "3")}
	counter := int64(0)
	valueRecorder.Record(ctx, counter, commonLabels...)
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
	ticker := time.NewTicker(1 * time.Second)
	for {
		select {
		case <-ticker.C:
			time.Sleep(1 * time.Second)
			counter++
			valueRecorder.Record(ctx, counter, commonLabels...)
			break
		case <-c:
			ticker.Stop()
			logger.Info("Stop Prometheus metrics app")
			return
		}
	}
}
<|endoftext|>


<commit_before>package cobbles

import (
	// "fmt"
	. "github.com/ricallinson/simplebdd"
	"testing"
)

type SimpleConfig struct {
	Settings  map[string]string
	Title_key string
	Data_url  string
	Logo      string
	Links     map[string]string
}

func TestBundle(t *testing.T) {

	Describe("flattenDimensions", func() {
		It("should return a flattened dimensions map", func() {
			b := New("./fixtures")
			index, dims := b.flattenDimensions(b.dimensions)
			// fmt.Printf("%s", toYaml(b.dimensions))
			AssertEqual(index["environment"], 0)
			AssertEqual(index["lang"], 1)
			AssertEqual(index["region"], 2)
			AssertEqual(index["flavor"], 3)
			// fmt.Printf("%s", toYaml(dims))
			AssertEqual(dims["lang"]["en"], "en")
			AssertEqual(dims["lang"]["en/en_CA"], "en_CA")
			AssertEqual(dims["lang"]["fr"], "fr")
			AssertEqual(dims["lang"]["fr/fr_FR/fr_CA"], "fr_CA")
		})
	})

	Describe("makeOrderedLookupList", func() {
		It("should return an ordered lookup list", func() {
			b := New("./fixtures")
			list := b.makeOrderedLookupList("lang=fr_CA,region=ir,environment=staging")
			// fmt.Printf("%s", toYaml(list))
			AssertEqual(list["environment"][0], "staging")
			AssertEqual(list["environment"][1], "*")
			AssertEqual(list["flavor"][0], "*")
			AssertEqual(list["lang"][0], "fr_CA")
			AssertEqual(list["lang"][1], "fr_FR")
			AssertEqual(list["lang"][2], "fr")
			AssertEqual(list["lang"][3], "*")
			AssertEqual(list["region"][0], "ir")
			AssertEqual(list["region"][1], "gb")
			AssertEqual(list["region"][2], "europe")
			AssertEqual(list["region"][3], "*")
		})
	})

	Describe("makeLookupPath", func() {
		It("should return a", func() {
			b := &Bundle{}
			p := b.makeLookupPath("lang=fr_CA,region=ir,environment=staging")
			AssertEqual(p, "fr_CA")
		})
	})

	Report(t)
}
<commit_msg>Added test for makeOrderedLookupList<commit_after>package cobbles

import (
	// "fmt"
	. "github.com/ricallinson/simplebdd"
	"testing"
)

type SimpleConfig struct {
	Settings  map[string]string
	Title_key string
	Data_url  string
	Logo      string
	Links     map[string]string
}

func TestBundle(t *testing.T) {

	Describe("flattenDimensions", func() {
		It("should return a flattened dimensions map", func() {
			b := New("./fixtures")
			index, dims := b.flattenDimensions(b.dimensions)
			// fmt.Printf("%s", toYaml(b.dimensions))
			AssertEqual(index["environment"], 0)
			AssertEqual(index["lang"], 1)
			AssertEqual(index["region"], 2)
			AssertEqual(index["flavor"], 3)
			// fmt.Printf("%s", toYaml(dims))
			AssertEqual(dims["lang"]["en"], "en")
			AssertEqual(dims["lang"]["en/en_CA"], "en_CA")
			AssertEqual(dims["lang"]["fr"], "fr")
			AssertEqual(dims["lang"]["fr/fr_FR/fr_CA"], "fr_CA")
		})
	})

	Describe("makeOrderedLookupList", func() {
		It("should return an ordered lookup list", func() {
			b := New("./fixtures")
			list := b.makeOrderedLookupList("lang=fr_CA,region=ir,environment=staging")
			// fmt.Printf("%s", toYaml(list))
			AssertEqual(list["environment"][0], "staging")
			AssertEqual(list["environment"][1], "*")
			AssertEqual(list["flavor"][0], "*")
			AssertEqual(list["lang"][0], "fr_CA")
			AssertEqual(list["lang"][1], "fr_FR")
			AssertEqual(list["lang"][2], "fr")
			AssertEqual(list["lang"][3], "*")
			AssertEqual(list["region"][0], "ir")
			AssertEqual(list["region"][1], "gb")
			AssertEqual(list["region"][2], "europe")
			AssertEqual(list["region"][3], "*")
		})
	})

	Describe("makeLookupPath", func() {
		It("should return a", func() {
			b := &Bundle{}
			p := b.makeLookupPath("lang=fr_CA,region=ir,environment=staging")
			AssertEqual(p, "xxx")
		})
	})

	Report(t)
}
<|endoftext|>


<commit_before>package cache

import (
	"encoding/gob"
	"golang.org/x/net/context"
	"log"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/memcache"
)

//http://www.funcmain.com/gob_encoding_an_interface
//http://stackoverflow.com/questions/13264555/store-an-object-in-memcache-of-gae-in-go

type SlowRetrieve func(ctx context.Context) (interface{}, error)

//Options:
//param 1 (bool): disable caching. (Usually for debugging)
//Param 2 (bool): Obtain fresh copy. Ignore content in cache but store fresh copy in cache.
//NB: In order for Param 2 to be activated, param 1 must be false.
func Remember(ctx context.Context, key string, expiration time.Duration, p SlowRetrieve, options ...bool) (interface{}, error) {

	disableCache := false
	fresh := false

	if len(options) != 0 {
		disableCache = options[0]
		if len(options) >= 2 {
			fresh = options[1]
		}
	}

	//For debugging, you can disable cache
	if disableCache {
		return p(ctx)
	}

	var v interface{}

	if fresh {
		if appengine.IsDevAppServer() {
			log.Println("\x1b[31mGrabbing (fresh) from SlowRetrieve key:", key, "\x1b[39;49m")
		}
		goto fresh
	}

	//Check if item exists
	if _, err := memcache.Gob.Get(ctx, key, &v); err == nil {
		//Item exists in cache
		if appengine.IsDevAppServer() {
			log.Println("\x1b[36mFound in Cache key:", key, "\x1b[39;49m")
		}
		return v, nil
	} else {
		if appengine.IsDevAppServer() {
			log.Println("\x1b[31mGrabbing from SlowRetrieve key:", key, err, "\x1b[39;49m")
		}
	}

fresh:
	//Item does not exist in cache so grab it from the persistent store
	itemToStore, err := p(ctx)
	func(itemToStore interface{}) {
		defer func() {
			recover()
		}()
		gob.Register(itemToStore)
	}(itemToStore)
	if err != nil {
		return nil, err
	}

	//Store item in Cache
	item := &memcache.Item{
		Key:        key,
		Object:     &itemToStore,
		Expiration: expiration,
	}

	err = memcache.Gob.Set(ctx, item)
	if err != nil {
		//Memcache storage failed
		if appengine.IsDevAppServer() {
			log.Println("\x1b[31mCould not store item to memcache key:", key, err, itemToStore, "\x1b[39;49m")
		}
	}
	return itemToStore, nil
}

//Delete key from memcache
func Delete(ctx context.Context, key string) error {
	return memcache.Delete(ctx, key)
}

func DeleteMulti(ctx context.Context, keys []string) error {
	return memcache.DeleteMulti(ctx, keys)
}
<commit_msg>Small change<commit_after>package cache

import (
	"encoding/gob"
	"golang.org/x/net/context"
	"log"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/memcache"
)

//http://www.funcmain.com/gob_encoding_an_interface
//http://stackoverflow.com/questions/13264555/store-an-object-in-memcache-of-gae-in-go

type SlowRetrieve func(ctx context.Context) (interface{}, error)

//Options:
//param 1 (bool): disable caching. (Usually for debugging)
//Param 2 (bool): Obtain fresh copy. Ignore content in cache but store fresh copy in cache.
//NB: In order for Param 2 to be activated, param 1 must be false.
func Remember(ctx context.Context, key string, expiration time.Duration, p SlowRetrieve, options ...bool) (interface{}, error) {

	disableCache := false
	fresh := false

	if len(options) != 0 {
		disableCache = options[0]
		if len(options) >= 2 {
			fresh = options[1]
		}
	}

	//For debugging, you can disable cache
	if disableCache {
		return p(ctx)
	}

	var v interface{}

	if fresh {
		if appengine.IsDevAppServer() {
			log.Println("\x1b[31mGrabbing (fresh) from SlowRetrieve key:", key, "\x1b[39;49m")
		}
		goto fresh
	}

	//Check if item exists
	if _, err := memcache.Gob.Get(ctx, key, &v); err == nil {
		//Item exists in cache
		if appengine.IsDevAppServer() {
			log.Println("\x1b[36mFound in Cache key:", key, "\x1b[39;49m")
		}
		return v, nil
	} else {
		if appengine.IsDevAppServer() {
			log.Println("\x1b[31mGrabbing from SlowRetrieve key:", key, err, "\x1b[39;49m")
		}
	}

fresh:
	//Item does not exist in cache so grab it from the persistent store
	itemToStore, err := p(ctx)
	if err != nil {
		return nil, err
	}
	func(itemToStore interface{}) {
		defer func() {
			recover()
		}()
		gob.Register(itemToStore)
	}(itemToStore)

	//Store item in Cache
	item := &memcache.Item{
		Key:        key,
		Object:     &itemToStore,
		Expiration: expiration,
	}

	err = memcache.Gob.Set(ctx, item)
	if err != nil {
		//Memcache storage failed
		if appengine.IsDevAppServer() {
			log.Println("\x1b[31mCould not store item to memcache key:", key, err, itemToStore, "\x1b[39;49m")
		}
	}
	return itemToStore, nil
}

//Delete key from memcache
func Delete(ctx context.Context, key string) error {
	return memcache.Delete(ctx, key)
}

func DeleteMulti(ctx context.Context, keys []string) error {
	return memcache.DeleteMulti(ctx, keys)
}
<|endoftext|>


<commit_before>package cache

import (
	"fmt"
	"io"
	"sort"
	"sync/atomic"
	"time"

	"github.com/lomik/go-carbon/helper"
	"github.com/lomik/go-carbon/points"
)

type queue []*points.Points

type byLength queue
type byTimestamp queue

func (v byLength) Len() int           { return len(v) }
func (v byLength) Swap(i, j int)      { v[i], v[j] = v[j], v[i] }
func (v byLength) Less(i, j int) bool { return len(v[i].Data) < len(v[j].Data) }

func (v byTimestamp) Len() int           { return len(v) }
func (v byTimestamp) Swap(i, j int)      { v[i], v[j] = v[j], v[i] }
func (v byTimestamp) Less(i, j int) bool { return v[i].Data[0].Timestamp > v[j].Data[0].Timestamp }

type WriteStrategy int

const (
	MaximumLength WriteStrategy = iota
	TimestampOrder
	Noop
)

// Cache stores and aggregate metrics in memory
type Cache struct {
	helper.Stoppable
	data                          map[string]*points.Points
	size                          uint32 // local read-only copy of sizeShared for worker goroutine
	sizeShared                    uint32 // changing via atomic
	metricCount                   uint32 // metrics count, changing via atomic
	maxSize                       uint32
	inputChan                     chan *points.Points // from receivers
	inputCapacity                 int                 // buffer size of inputChan
	outputChan                    chan *points.Points // to persisters
	queryChan                     chan *Query         // from carbonlink
	confirmChan                   chan *points.Points // for persisted confirmation
	queryCnt                      uint32
	overflowCnt                   uint32 // drop packages if cache full
	writeStrategy                 WriteStrategy
	queue                         queue
	queueBuildCnt                 uint32 // number of times writeout queue was built
	queueBuildTime                uint32 // time spent building writeout queue in milliseconds
	maxInputLenBeforeQueueRebuild int
	maxInputLenAfterQueueRebuild  int
	queueWriteoutStart            time.Time
	queueWriteoutTime             time.Duration
}

// New create Cache instance and run in/out goroutine
func New() *Cache {
	cache := &Cache{
		data:               make(map[string]*points.Points, 0),
		size:               0,
		maxSize:            1000000,
		queryChan:          make(chan *Query, 16),
		queue:              make(queue, 0),
		confirmChan:        make(chan *points.Points, 2048),
		inputCapacity:      51200,
		writeStrategy:      MaximumLength,
		queueWriteoutStart: time.Time{},
		// inputChan:   make(chan *points.Points, 51200), create in In() getter
	}
	return cache
}

// SetWriteStrategy ...
func (c *Cache) SetWriteStrategy(s string) (err error) {
	switch s {
	case "max":
		c.writeStrategy = MaximumLength
	case "sort":
		c.writeStrategy = TimestampOrder
	case "noop":
		c.writeStrategy = Noop
	default:
		return fmt.Errorf("Unknown write strategy '%s', should be one of: max, sort, noop", s)
	}
	return nil
}

// SetInputCapacity set buffer size of input channel. Call before In() getter
func (c *Cache) SetInputCapacity(size int) {
	c.inputCapacity = size
}

func (c *Cache) getNext() *points.Points {
	size := len(c.queue)
	if size == 0 {
		return nil
	}
	values := c.queue[size-1]
	c.queue = c.queue[:size-1]
	return values
}

// Get any key/values pair from Cache
func (c *Cache) Get() *points.Points {
	if values := c.getNext(); values != nil {
		return values
	}

	if (c.queueWriteoutStart != time.Time{}) {
		c.queueWriteoutTime += time.Now().Sub(c.queueWriteoutStart)
	}

	c.updateQueue()

	value := c.getNext()

	if value != nil {
		c.queueWriteoutStart = time.Now()
	} else {
		c.queueWriteoutStart = time.Time{}
	}

	return value
}

// Remove key from cache
func (c *Cache) Remove(key string, size int) {
	delete(c.data, key)
	c.size = atomic.AddUint32(&c.sizeShared, -uint32(size))
	atomic.AddUint32(&c.metricCount, ^uint32(0))
}

// Pop return and remove next for save point from cache
func (c *Cache) Pop() *points.Points {
	if c.size == 0 {
		return nil
	}
	v := c.Get()
	if v != nil {
		c.Remove(v.Metric, len(v.Data))
	}
	return v
}

// Add points to cache
func (c *Cache) Add(p *points.Points) {
	if values, exists := c.data[p.Metric]; exists {
		values.Data = append(values.Data, p.Data...)
	} else {
		atomic.AddUint32(&c.metricCount, 1)
		c.data[p.Metric] = p
	}
	c.size = atomic.AddUint32(&c.sizeShared, uint32(len(p.Data)))
}

// SetMaxSize of cache
func (c *Cache) SetMaxSize(maxSize uint32) {
	c.maxSize = maxSize
}

// Size returns size
func (c *Cache) Size() uint32 {
	return atomic.LoadUint32(&c.sizeShared)
}

func (c *Cache) updateQueue() {
	start := time.Now()
	inputLenBeforeQueueRebuild := len(c.inputChan)

	newQueue := c.queue[:0]

	for _, values := range c.data {
		newQueue = append(newQueue, values)
	}

	switch c.writeStrategy {
	case MaximumLength:
		sort.Sort(byLength(newQueue))
	case TimestampOrder:
		sort.Sort(byTimestamp(newQueue))
	case Noop:
	}
	c.queue = newQueue

	atomic.AddUint32(&c.queueBuildTime, uint32(time.Now().Sub(start).Nanoseconds()/10000000))
	atomic.AddUint32(&c.queueBuildCnt, 1)

	inputLenAfterQueueRebuild := len(c.inputChan)

	if inputLenAfterQueueRebuild > c.maxInputLenAfterQueueRebuild {
		c.maxInputLenAfterQueueRebuild = inputLenAfterQueueRebuild
		c.maxInputLenBeforeQueueRebuild = inputLenBeforeQueueRebuild
	}
}

// Collect cache metrics
func (c *Cache) Stat(send func(metric string, value float64)) {
	send("size", float64(c.Size()))
	send("metrics", float64(atomic.LoadUint32(&c.metricCount)))

	queryCnt := atomic.LoadUint32(&c.queryCnt)
	atomic.AddUint32(&c.queryCnt, -queryCnt)
	send("queries", float64(queryCnt))

	overflowCnt := atomic.LoadUint32(&c.overflowCnt)
	atomic.AddUint32(&c.overflowCnt, -overflowCnt)
	send("overflow", float64(overflowCnt))

	queueBuildCnt := atomic.LoadUint32(&c.queueBuildCnt)
	atomic.AddUint32(&c.queueBuildCnt, -queueBuildCnt)
	send("queueBuildCount", float64(queueBuildCnt))

	queueBuildTime := atomic.LoadUint32(&c.queueBuildTime)
	atomic.AddUint32(&c.queueBuildTime, -queueBuildTime)
	send("queueBuildTime", float64(queueBuildTime/1000))

	// c.stat("maxInputLenBeforeQueueRebuild", float64(c.maxInputLenBeforeQueueRebuild))
	// c.stat("maxInputLenAfterQueueRebuild", float64(c.maxInputLenAfterQueueRebuild))
	// c.stat("queueWriteoutTime", float64(c.queueWriteoutTime.Seconds()))
}

func (c *Cache) worker(exitChan chan bool) {
	var values *points.Points
	var sendTo chan *points.Points
	var forceReceive bool

	toConfirmTracker := make(chan *points.Points)

	confirmTracker := &notConfirmed{
		data:      make(map[string][]*points.Points),
		queryChan: make(chan *Query, 16),
		in:        toConfirmTracker,
		out:       c.outputChan,
		confirmed: c.confirmChan,
		cacheIn:   c.inputChan,
	}

	c.Go(func(exit chan bool) {
		confirmTracker.worker(exit)
	})

	forceReceiveThreshold := cap(c.inputChan) / 10

MAIN_LOOP:
	for {

		if len(c.inputChan) > forceReceiveThreshold {
			forceReceive = true
		} else {
			forceReceive = false
		}

		if values == nil && !forceReceive {
			values = c.Pop()
		}

		if values != nil {
			sendTo = toConfirmTracker
		} else {
			sendTo = nil
		}

		select {
		case query := <-c.queryChan: // carbonlink
			atomic.AddUint32(&c.queryCnt, 1)

			if values != nil && values.Metric == query.Metric {
				query.CacheData = values
			} else if v, ok := c.data[query.Metric]; ok {
				query.CacheData = v.Copy()
			}

			confirmTracker.queryChan <- query
		case sendTo <- values: // to persister
			values = nil
		case msg := <-c.inputChan: // from receiver
			if c.maxSize == 0 || c.size < c.maxSize {
				c.Add(msg)
			} else {
				atomic.AddUint32(&c.overflowCnt, 1)
			}
		case <-exitChan: // exit
			break MAIN_LOOP
		}
	}

}

// In returns input channel
func (c *Cache) In() chan *points.Points {
	if c.inputChan == nil {
		c.inputChan = make(chan *points.Points, c.inputCapacity)
	}
	return c.inputChan
}

// Out returns output channel
func (c *Cache) Out() chan *points.Points {
	if c.outputChan == nil {
		c.outputChan = make(chan *points.Points, 1024)
	}
	return c.outputChan
}

// Query returns carbonlink query channel
func (c *Cache) Query() chan *Query {
	return c.queryChan
}

// Confirm returns confirmation channel for persister
func (c *Cache) Confirm() chan *points.Points {
	return c.confirmChan
}

// SetOutputChanSize ...
func (c *Cache) SetOutputChanSize(size int) {
	c.outputChan = make(chan *points.Points, size)
}

// Start worker
func (c *Cache) Start() error {
	return c.StartFunc(func() error {
		if c.inputChan == nil {
			c.inputChan = make(chan *points.Points, c.inputCapacity)
		}
		if c.outputChan == nil {
			c.outputChan = make(chan *points.Points, 1024)
		}

		c.Go(func(exit chan bool) {
			c.worker(exit)
		})

		return nil
	})
}

// Dump all cache to writer
func (c *Cache) Dump(w io.Writer) error {

	for _, p := range c.data { // every metric
		for _, d := range p.Data { // every metric point
			_, err := w.Write([]byte(fmt.Sprintf("%s %v %v\n", p.Metric, d.Value, d.Timestamp)))
			if err != nil {
				return err
			}
		}
	}

	return nil
}
<commit_msg>queueRebuild metrics in new collector metric system<commit_after>package cache

import (
	"fmt"
	"io"
	"sort"
	"sync/atomic"
	"time"

	"github.com/lomik/go-carbon/helper"
	"github.com/lomik/go-carbon/points"
)

type queue []*points.Points

type byLength queue
type byTimestamp queue

func (v byLength) Len() int           { return len(v) }
func (v byLength) Swap(i, j int)      { v[i], v[j] = v[j], v[i] }
func (v byLength) Less(i, j int) bool { return len(v[i].Data) < len(v[j].Data) }

func (v byTimestamp) Len() int           { return len(v) }
func (v byTimestamp) Swap(i, j int)      { v[i], v[j] = v[j], v[i] }
func (v byTimestamp) Less(i, j int) bool { return v[i].Data[0].Timestamp > v[j].Data[0].Timestamp }

type WriteStrategy int

const (
	MaximumLength WriteStrategy = iota
	TimestampOrder
	Noop
)

// Cache stores and aggregate metrics in memory
type Cache struct {
	helper.Stoppable
	data                          map[string]*points.Points
	size                          uint32 // local read-only copy of sizeShared for worker goroutine
	sizeShared                    uint32 // changing via atomic
	metricCount                   uint32 // metrics count, changing via atomic
	maxSize                       uint32
	inputChan                     chan *points.Points // from receivers
	inputCapacity                 int                 // buffer size of inputChan
	outputChan                    chan *points.Points // to persisters
	queryChan                     chan *Query         // from carbonlink
	confirmChan                   chan *points.Points // for persisted confirmation
	queryCnt                      uint32
	overflowCnt                   uint32 // drop packages if cache full
	writeStrategy                 WriteStrategy
	queue                         queue
	queueBuildCnt                 uint32 // number of times writeout queue was built
	queueBuildTime                uint32 // time spent building writeout queue in milliseconds
	maxInputLenBeforeQueueRebuild uint32
	maxInputLenAfterQueueRebuild  uint32
	queueWriteoutStart            time.Time
	queueWriteoutTime             uint32 // in milliseconds
}

// New create Cache instance and run in/out goroutine
func New() *Cache {
	cache := &Cache{
		data:               make(map[string]*points.Points, 0),
		size:               0,
		maxSize:            1000000,
		queryChan:          make(chan *Query, 16),
		queue:              make(queue, 0),
		confirmChan:        make(chan *points.Points, 2048),
		inputCapacity:      51200,
		writeStrategy:      MaximumLength,
		queueWriteoutStart: time.Time{},
		// inputChan:   make(chan *points.Points, 51200), create in In() getter
	}
	return cache
}

// SetWriteStrategy ...
func (c *Cache) SetWriteStrategy(s string) (err error) {
	switch s {
	case "max":
		c.writeStrategy = MaximumLength
	case "sort":
		c.writeStrategy = TimestampOrder
	case "noop":
		c.writeStrategy = Noop
	default:
		return fmt.Errorf("Unknown write strategy '%s', should be one of: max, sort, noop", s)
	}
	return nil
}

// SetInputCapacity set buffer size of input channel. Call before In() getter
func (c *Cache) SetInputCapacity(size int) {
	c.inputCapacity = size
}

func (c *Cache) getNext() *points.Points {
	size := len(c.queue)
	if size == 0 {
		return nil
	}
	values := c.queue[size-1]
	c.queue = c.queue[:size-1]
	return values
}

// Get any key/values pair from Cache
func (c *Cache) Get() *points.Points {
	if values := c.getNext(); values != nil {
		return values
	}

	if (c.queueWriteoutStart != time.Time{}) {
		atomic.AddUint32(&c.queueWriteoutTime, uint32(time.Now().Sub(c.queueWriteoutStart).Nanoseconds()/1000000))
	}

	c.updateQueue()

	value := c.getNext()

	if value != nil {
		c.queueWriteoutStart = time.Now()
	} else {
		c.queueWriteoutStart = time.Time{}
	}

	return value
}

// Remove key from cache
func (c *Cache) Remove(key string, size int) {
	delete(c.data, key)
	c.size = atomic.AddUint32(&c.sizeShared, -uint32(size))
	atomic.AddUint32(&c.metricCount, ^uint32(0))
}

// Pop return and remove next for save point from cache
func (c *Cache) Pop() *points.Points {
	if c.size == 0 {
		return nil
	}
	v := c.Get()
	if v != nil {
		c.Remove(v.Metric, len(v.Data))
	}
	return v
}

// Add points to cache
func (c *Cache) Add(p *points.Points) {
	if values, exists := c.data[p.Metric]; exists {
		values.Data = append(values.Data, p.Data...)
	} else {
		atomic.AddUint32(&c.metricCount, 1)
		c.data[p.Metric] = p
	}
	c.size = atomic.AddUint32(&c.sizeShared, uint32(len(p.Data)))
}

// SetMaxSize of cache
func (c *Cache) SetMaxSize(maxSize uint32) {
	c.maxSize = maxSize
}

// Size returns size
func (c *Cache) Size() uint32 {
	return atomic.LoadUint32(&c.sizeShared)
}

func (c *Cache) updateQueue() {
	start := time.Now()
	inputLenBeforeQueueRebuild := len(c.inputChan)

	newQueue := c.queue[:0]

	for _, values := range c.data {
		newQueue = append(newQueue, values)
	}

	switch c.writeStrategy {
	case MaximumLength:
		sort.Sort(byLength(newQueue))
	case TimestampOrder:
		sort.Sort(byTimestamp(newQueue))
	case Noop:
	}
	c.queue = newQueue

	atomic.AddUint32(&c.queueBuildTime, uint32(time.Now().Sub(start).Nanoseconds()/10000000))
	atomic.AddUint32(&c.queueBuildCnt, 1)

	inputLenAfterQueueRebuild := len(c.inputChan)

	if uint32(inputLenAfterQueueRebuild) > atomic.LoadUint32(&c.maxInputLenAfterQueueRebuild) {
		atomic.StoreUint32(&c.maxInputLenAfterQueueRebuild, uint32(inputLenAfterQueueRebuild))
		atomic.StoreUint32(&c.maxInputLenBeforeQueueRebuild, uint32(inputLenBeforeQueueRebuild))
	}
}

// Collect cache metrics
func (c *Cache) Stat(send func(metric string, value float64)) {
	send("size", float64(c.Size()))
	send("metrics", float64(atomic.LoadUint32(&c.metricCount)))

	queryCnt := atomic.LoadUint32(&c.queryCnt)
	atomic.AddUint32(&c.queryCnt, -queryCnt)
	send("queries", float64(queryCnt))

	overflowCnt := atomic.LoadUint32(&c.overflowCnt)
	atomic.AddUint32(&c.overflowCnt, -overflowCnt)
	send("overflow", float64(overflowCnt))

	queueBuildCnt := atomic.LoadUint32(&c.queueBuildCnt)
	atomic.AddUint32(&c.queueBuildCnt, -queueBuildCnt)
	send("queueBuildCount", float64(queueBuildCnt))

	queueBuildTime := atomic.LoadUint32(&c.queueBuildTime)
	atomic.AddUint32(&c.queueBuildTime, -queueBuildTime)
	send("queueBuildTime", float64(queueBuildTime)/1000.0)

	queueWriteoutTime := atomic.LoadUint32(&c.queueWriteoutTime)
	atomic.AddUint32(&c.queueWriteoutTime, -queueWriteoutTime)
	send("queueWriteoutTime", float64(queueWriteoutTime)/1000.0)

	maxInputLenBeforeQueueRebuild := atomic.LoadUint32(&c.maxInputLenBeforeQueueRebuild)
	atomic.CompareAndSwapUint32(&c.maxInputLenAfterQueueRebuild, maxInputLenBeforeQueueRebuild, 0)
	send("maxInputLenBeforeQueueRebuild", float64(maxInputLenBeforeQueueRebuild))

	maxInputLenAfterQueueRebuild := atomic.LoadUint32(&c.maxInputLenAfterQueueRebuild)
	atomic.CompareAndSwapUint32(&c.maxInputLenAfterQueueRebuild, maxInputLenAfterQueueRebuild, 0)
	send("maxInputLenAfterQueueRebuild", float64(maxInputLenAfterQueueRebuild))
}

func (c *Cache) worker(exitChan chan bool) {
	var values *points.Points
	var sendTo chan *points.Points
	var forceReceive bool

	toConfirmTracker := make(chan *points.Points)

	confirmTracker := &notConfirmed{
		data:      make(map[string][]*points.Points),
		queryChan: make(chan *Query, 16),
		in:        toConfirmTracker,
		out:       c.outputChan,
		confirmed: c.confirmChan,
		cacheIn:   c.inputChan,
	}

	c.Go(func(exit chan bool) {
		confirmTracker.worker(exit)
	})

	forceReceiveThreshold := cap(c.inputChan) / 10

MAIN_LOOP:
	for {

		if len(c.inputChan) > forceReceiveThreshold {
			forceReceive = true
		} else {
			forceReceive = false
		}

		if values == nil && !forceReceive {
			values = c.Pop()
		}

		if values != nil {
			sendTo = toConfirmTracker
		} else {
			sendTo = nil
		}

		select {
		case query := <-c.queryChan: // carbonlink
			atomic.AddUint32(&c.queryCnt, 1)

			if values != nil && values.Metric == query.Metric {
				query.CacheData = values
			} else if v, ok := c.data[query.Metric]; ok {
				query.CacheData = v.Copy()
			}

			confirmTracker.queryChan <- query
		case sendTo <- values: // to persister
			values = nil
		case msg := <-c.inputChan: // from receiver
			if c.maxSize == 0 || c.size < c.maxSize {
				c.Add(msg)
			} else {
				atomic.AddUint32(&c.overflowCnt, 1)
			}
		case <-exitChan: // exit
			break MAIN_LOOP
		}
	}

}

// In returns input channel
func (c *Cache) In() chan *points.Points {
	if c.inputChan == nil {
		c.inputChan = make(chan *points.Points, c.inputCapacity)
	}
	return c.inputChan
}

// Out returns output channel
func (c *Cache) Out() chan *points.Points {
	if c.outputChan == nil {
		c.outputChan = make(chan *points.Points, 1024)
	}
	return c.outputChan
}

// Query returns carbonlink query channel
func (c *Cache) Query() chan *Query {
	return c.queryChan
}

// Confirm returns confirmation channel for persister
func (c *Cache) Confirm() chan *points.Points {
	return c.confirmChan
}

// SetOutputChanSize ...
func (c *Cache) SetOutputChanSize(size int) {
	c.outputChan = make(chan *points.Points, size)
}

// Start worker
func (c *Cache) Start() error {
	return c.StartFunc(func() error {
		if c.inputChan == nil {
			c.inputChan = make(chan *points.Points, c.inputCapacity)
		}
		if c.outputChan == nil {
			c.outputChan = make(chan *points.Points, 1024)
		}

		c.Go(func(exit chan bool) {
			c.worker(exit)
		})

		return nil
	})
}

// Dump all cache to writer
func (c *Cache) Dump(w io.Writer) error {

	for _, p := range c.data { // every metric
		for _, d := range p.Data { // every metric point
			_, err := w.Write([]byte(fmt.Sprintf("%s %v %v\n", p.Metric, d.Value, d.Timestamp)))
			if err != nil {
				return err
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package cache

import (
	"container/list"
	"goposm/cache/binary"
	"goposm/element"
	"sort"
	"sync"
)

type Nodes []element.Node

func (s Nodes) Len() int           { return len(s) }
func (s Nodes) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s Nodes) Less(i, j int) bool { return s[i].Id < s[j].Id }

func packNodes(nodes []element.Node) *binary.DeltaCoords {
	var lastLon, lastLat int64
	var lon, lat int64
	var lastId int64
	ids := make([]int64, len(nodes))
	lons := make([]int64, len(nodes))
	lats := make([]int64, len(nodes))

	i := 0
	for _, nd := range nodes {
		lon = int64(binary.CoordToInt(nd.Long))
		lat = int64(binary.CoordToInt(nd.Lat))
		ids[i] = nd.Id - lastId
		lons[i] = lon - lastLon
		lats[i] = lat - lastLat

		lastId = nd.Id
		lastLon = lon
		lastLat = lat
		i++
	}
	return &binary.DeltaCoords{Ids: ids, Lats: lats, Lons: lons}
}

func unpackNodes(deltaCoords *binary.DeltaCoords, nodes []element.Node) []element.Node {
	if len(deltaCoords.Ids) > cap(nodes) {
		nodes = make([]element.Node, len(deltaCoords.Ids))
	} else {
		nodes = nodes[:len(deltaCoords.Ids)]
	}

	var lastLon, lastLat int64
	var lon, lat int64
	var lastId, id int64

	for i := 0; i < len(deltaCoords.Ids); i++ {
		id = lastId + deltaCoords.Ids[i]
		lon = lastLon + deltaCoords.Lons[i]
		lat = lastLat + deltaCoords.Lats[i]
		nodes[i] = element.Node{
			OSMElem: element.OSMElem{Id: int64(id)},
			Long:    binary.IntToCoord(uint32(lon)),
			Lat:     binary.IntToCoord(uint32(lat)),
		}

		lastId = id
		lastLon = lon
		lastLat = lat
	}
	return nodes
}

type CoordsBunch struct {
	sync.Mutex
	id         int64
	coords     []element.Node
	elem       *list.Element
	needsWrite bool
}

func (b *CoordsBunch) GetCoord(id int64) (*element.Node, error) {
	idx := sort.Search(len(b.coords), func(i int) bool {
		return b.coords[i].Id >= id
	})
	if idx < len(b.coords) && b.coords[idx].Id == id {
		return &b.coords[idx], nil
	}
	return nil, NotFound
}

type DeltaCoordsCache struct {
	Cache
	lruList      *list.List
	table        map[int64]*CoordsBunch
	capacity     int64
	linearImport bool
	mu           sync.Mutex
}

func NewDeltaCoordsCache(path string) (*DeltaCoordsCache, error) {
	coordsCache := DeltaCoordsCache{}
	err := coordsCache.open(path)
	if err != nil {
		return nil, err
	}
	coordsCache.lruList = list.New()
	coordsCache.table = make(map[int64]*CoordsBunch)
	// mem req for cache approx. capacity*deltaCacheBunchSize*40
	coordsCache.capacity = 1024
	return &coordsCache, nil
}

func (self *DeltaCoordsCache) SetLinearImport(v bool) {
	self.linearImport = v
}

func (self *DeltaCoordsCache) Flush() {
	for bunchId, bunch := range self.table {
		if bunch.needsWrite {
			self.putCoordsPacked(bunchId, bunch.coords)
		}
	}
}
func (self *DeltaCoordsCache) Close() {
	self.Flush()
	self.Cache.Close()
}

func (self *DeltaCoordsCache) GetCoord(id int64) (*element.Node, error) {
	bunchId := getBunchId(id)
	bunch, err := self.getBunch(bunchId)
	if err != nil {
		return nil, err
	}
	defer bunch.Unlock()
	return bunch.GetCoord(id)
}

func (self *DeltaCoordsCache) FillWay(way *element.Way) error {
	if way == nil {
		return nil
	}
	way.Nodes = make([]element.Node, len(way.Refs))

	var err error
	var bunch *CoordsBunch
	var bunchId, lastBunchId int64
	lastBunchId = -1

	for i, id := range way.Refs {
		bunchId = getBunchId(id)
		// re-use bunches
		if bunchId != lastBunchId {
			if bunch != nil {
				bunch.Unlock()
			}
			bunch, err = self.getBunch(bunchId)
			if err != nil {
				return err
			}
		}
		lastBunchId = bunchId

		nd, err := bunch.GetCoord(id)
		if err != nil {
			bunch.Unlock()
			return err
		}
		way.Nodes[i] = *nd
	}
	if bunch != nil {
		bunch.Unlock()
	}
	return nil
}

// PutCoords puts nodes into cache.
// nodes need to be sorted by Id.
func (self *DeltaCoordsCache) PutCoords(nodes []element.Node) error {
	var start, currentBunchId int64
	currentBunchId = getBunchId(nodes[0].Id)
	start = 0
	totalNodes := len(nodes)
	for i, node := range nodes {
		bunchId := getBunchId(node.Id)
		if bunchId != currentBunchId {
			if self.linearImport && int64(i) > deltaCacheBunchSize && int64(i) < int64(totalNodes)-deltaCacheBunchSize {
				// no need to handle concurrent updates to the same
				// bunch if we are not at the boundary of a deltaCacheBunchSize
				self.putCoordsPacked(currentBunchId, nodes[start:i])
			} else {
				bunch, err := self.getBunch(currentBunchId)
				if err != nil {
					return err
				}
				bunch.coords = append(bunch.coords, nodes[start:i]...)
				sort.Sort(Nodes(bunch.coords))
				bunch.needsWrite = true
				bunch.Unlock()
			}
			currentBunchId = bunchId
			start = int64(i)
		}
	}
	bunch, err := self.getBunch(currentBunchId)
	if err != nil {
		return err
	}
	bunch.coords = append(bunch.coords, nodes[start:]...)
	sort.Sort(Nodes(bunch.coords))
	bunch.needsWrite = true
	bunch.Unlock()
	return nil
}

var (
	freeBuffer = make(chan []byte, 4)
)

func (p *DeltaCoordsCache) putCoordsPacked(bunchId int64, nodes []element.Node) error {
	if len(nodes) == 0 {
		return nil
	}
	keyBuf := idToKeyBuf(bunchId)

	var data []byte
	select {
	case data = <-freeBuffer:
	default:
	}

	data = binary.MarshalDeltaNodes(nodes, data)

	err := p.db.Put(p.wo, keyBuf, data)
	if err != nil {
		return err
	}

	select {
	case freeBuffer <- data:
	default:
	}

	return nil
}

func (p *DeltaCoordsCache) getCoordsPacked(bunchId int64, nodes []element.Node) ([]element.Node, error) {
	keyBuf := idToKeyBuf(bunchId)

	data, err := p.db.Get(p.ro, keyBuf)
	if err != nil {
		return nil, err
	}
	if data == nil {
		// clear before returning
		return nodes[:0], nil
	}
	nodes, err = binary.UnmarshalDeltaNodes(data, nodes)
	if err != nil {
		return nil, err
	}

	return nodes, nil
}

func getBunchId(nodeId int64) int64 {
	return nodeId / deltaCacheBunchSize
}

var (
	freeNodes = make(chan []element.Node, 4)
)

func (self *DeltaCoordsCache) getBunch(bunchId int64) (*CoordsBunch, error) {
	self.mu.Lock()
	defer self.mu.Unlock()
	bunch, ok := self.table[bunchId]
	var nodes []element.Node
	if !ok {
		elem := self.lruList.PushFront(bunchId)
		select {
		case nodes = <-freeNodes:
		default:
			nodes = make([]element.Node, 0, deltaCacheBunchSize)
		}
		nodes, err := self.getCoordsPacked(bunchId, nodes)
		if err != nil {
			return nil, err
		}
		bunch = &CoordsBunch{id: bunchId, coords: nodes, elem: elem}
		self.table[bunchId] = bunch
	} else {
		self.lruList.MoveToFront(bunch.elem)
	}
	bunch.Lock()
	self.CheckCapacity()
	return bunch, nil
}

func (self *DeltaCoordsCache) CheckCapacity() {
	for int64(len(self.table)) > self.capacity {
		elem := self.lruList.Back()
		bunchId := self.lruList.Remove(elem).(int64)
		bunch := self.table[bunchId]
		bunch.elem = nil
		if bunch.needsWrite {
			self.putCoordsPacked(bunchId, bunch.coords)
		}
		select {
		case freeNodes <- bunch.coords:
		default:
		}
		delete(self.table, bunchId)
	}
}
<commit_msg>release getBunch lock before loading missing coords from cache<commit_after>package cache

import (
	"container/list"
	"goposm/cache/binary"
	"goposm/element"
	"sort"
	"sync"
)

type Nodes []element.Node

func (s Nodes) Len() int           { return len(s) }
func (s Nodes) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s Nodes) Less(i, j int) bool { return s[i].Id < s[j].Id }

func packNodes(nodes []element.Node) *binary.DeltaCoords {
	var lastLon, lastLat int64
	var lon, lat int64
	var lastId int64
	ids := make([]int64, len(nodes))
	lons := make([]int64, len(nodes))
	lats := make([]int64, len(nodes))

	i := 0
	for _, nd := range nodes {
		lon = int64(binary.CoordToInt(nd.Long))
		lat = int64(binary.CoordToInt(nd.Lat))
		ids[i] = nd.Id - lastId
		lons[i] = lon - lastLon
		lats[i] = lat - lastLat

		lastId = nd.Id
		lastLon = lon
		lastLat = lat
		i++
	}
	return &binary.DeltaCoords{Ids: ids, Lats: lats, Lons: lons}
}

func unpackNodes(deltaCoords *binary.DeltaCoords, nodes []element.Node) []element.Node {
	if len(deltaCoords.Ids) > cap(nodes) {
		nodes = make([]element.Node, len(deltaCoords.Ids))
	} else {
		nodes = nodes[:len(deltaCoords.Ids)]
	}

	var lastLon, lastLat int64
	var lon, lat int64
	var lastId, id int64

	for i := 0; i < len(deltaCoords.Ids); i++ {
		id = lastId + deltaCoords.Ids[i]
		lon = lastLon + deltaCoords.Lons[i]
		lat = lastLat + deltaCoords.Lats[i]
		nodes[i] = element.Node{
			OSMElem: element.OSMElem{Id: int64(id)},
			Long:    binary.IntToCoord(uint32(lon)),
			Lat:     binary.IntToCoord(uint32(lat)),
		}

		lastId = id
		lastLon = lon
		lastLat = lat
	}
	return nodes
}

type CoordsBunch struct {
	sync.Mutex
	id         int64
	coords     []element.Node
	elem       *list.Element
	needsWrite bool
}

func (b *CoordsBunch) GetCoord(id int64) (*element.Node, error) {
	idx := sort.Search(len(b.coords), func(i int) bool {
		return b.coords[i].Id >= id
	})
	if idx < len(b.coords) && b.coords[idx].Id == id {
		return &b.coords[idx], nil
	}
	return nil, NotFound
}

type DeltaCoordsCache struct {
	Cache
	lruList      *list.List
	table        map[int64]*CoordsBunch
	capacity     int64
	linearImport bool
	mu           sync.Mutex
}

func NewDeltaCoordsCache(path string) (*DeltaCoordsCache, error) {
	coordsCache := DeltaCoordsCache{}
	err := coordsCache.open(path)
	if err != nil {
		return nil, err
	}
	coordsCache.lruList = list.New()
	coordsCache.table = make(map[int64]*CoordsBunch)
	// mem req for cache approx. capacity*deltaCacheBunchSize*40
	coordsCache.capacity = 1024
	return &coordsCache, nil
}

func (self *DeltaCoordsCache) SetLinearImport(v bool) {
	self.linearImport = v
}

func (self *DeltaCoordsCache) Flush() {
	for bunchId, bunch := range self.table {
		if bunch.needsWrite {
			self.putCoordsPacked(bunchId, bunch.coords)
		}
	}
}
func (self *DeltaCoordsCache) Close() {
	self.Flush()
	self.Cache.Close()
}

func (self *DeltaCoordsCache) GetCoord(id int64) (*element.Node, error) {
	bunchId := getBunchId(id)
	bunch, err := self.getBunch(bunchId)
	if err != nil {
		return nil, err
	}
	defer bunch.Unlock()
	return bunch.GetCoord(id)
}

func (self *DeltaCoordsCache) FillWay(way *element.Way) error {
	if way == nil {
		return nil
	}
	way.Nodes = make([]element.Node, len(way.Refs))

	var err error
	var bunch *CoordsBunch
	var bunchId, lastBunchId int64
	lastBunchId = -1

	for i, id := range way.Refs {
		bunchId = getBunchId(id)
		// re-use bunches
		if bunchId != lastBunchId {
			if bunch != nil {
				bunch.Unlock()
			}
			bunch, err = self.getBunch(bunchId)
			if err != nil {
				return err
			}
		}
		lastBunchId = bunchId

		nd, err := bunch.GetCoord(id)
		if err != nil {
			bunch.Unlock()
			return err
		}
		way.Nodes[i] = *nd
	}
	if bunch != nil {
		bunch.Unlock()
	}
	return nil
}

// PutCoords puts nodes into cache.
// nodes need to be sorted by Id.
func (self *DeltaCoordsCache) PutCoords(nodes []element.Node) error {
	var start, currentBunchId int64
	currentBunchId = getBunchId(nodes[0].Id)
	start = 0
	totalNodes := len(nodes)
	for i, node := range nodes {
		bunchId := getBunchId(node.Id)
		if bunchId != currentBunchId {
			if self.linearImport && int64(i) > deltaCacheBunchSize && int64(i) < int64(totalNodes)-deltaCacheBunchSize {
				// no need to handle concurrent updates to the same
				// bunch if we are not at the boundary of a deltaCacheBunchSize
				self.putCoordsPacked(currentBunchId, nodes[start:i])
			} else {
				bunch, err := self.getBunch(currentBunchId)
				if err != nil {
					return err
				}
				bunch.coords = append(bunch.coords, nodes[start:i]...)
				sort.Sort(Nodes(bunch.coords))
				bunch.needsWrite = true
				bunch.Unlock()
			}
			currentBunchId = bunchId
			start = int64(i)
		}
	}
	bunch, err := self.getBunch(currentBunchId)
	if err != nil {
		return err
	}
	bunch.coords = append(bunch.coords, nodes[start:]...)
	sort.Sort(Nodes(bunch.coords))
	bunch.needsWrite = true
	bunch.Unlock()
	return nil
}

var (
	freeBuffer = make(chan []byte, 4)
)

func (p *DeltaCoordsCache) putCoordsPacked(bunchId int64, nodes []element.Node) error {
	if len(nodes) == 0 {
		return nil
	}
	keyBuf := idToKeyBuf(bunchId)

	var data []byte
	select {
	case data = <-freeBuffer:
	default:
	}

	data = binary.MarshalDeltaNodes(nodes, data)

	err := p.db.Put(p.wo, keyBuf, data)
	if err != nil {
		return err
	}

	select {
	case freeBuffer <- data:
	default:
	}

	return nil
}

func (p *DeltaCoordsCache) getCoordsPacked(bunchId int64, nodes []element.Node) ([]element.Node, error) {
	keyBuf := idToKeyBuf(bunchId)

	data, err := p.db.Get(p.ro, keyBuf)
	if err != nil {
		return nil, err
	}
	if data == nil {
		// clear before returning
		return nodes[:0], nil
	}
	nodes, err = binary.UnmarshalDeltaNodes(data, nodes)
	if err != nil {
		return nil, err
	}

	return nodes, nil
}

func getBunchId(nodeId int64) int64 {
	return nodeId / deltaCacheBunchSize
}

var (
	freeNodes = make(chan []element.Node, 4)
)

func (self *DeltaCoordsCache) getBunch(bunchId int64) (*CoordsBunch, error) {
	self.mu.Lock()
	bunch, ok := self.table[bunchId]
	var nodes []element.Node
	needsGet := false
	if !ok {
		elem := self.lruList.PushFront(bunchId)
		select {
		case nodes = <-freeNodes:
		default:
			nodes = make([]element.Node, 0, deltaCacheBunchSize)
		}
		bunch = &CoordsBunch{id: bunchId, coords: nil, elem: elem}
		needsGet = true
		self.table[bunchId] = bunch
	} else {
		self.lruList.MoveToFront(bunch.elem)
	}
	bunch.Lock()
	self.CheckCapacity()
	defer self.mu.Unlock()

	if needsGet {
		nodes, err := self.getCoordsPacked(bunchId, nodes)
		if err != nil {
			return nil, err
		}
		bunch.coords = nodes
	}

	return bunch, nil
}

func (self *DeltaCoordsCache) CheckCapacity() {
	for int64(len(self.table)) > self.capacity {
		elem := self.lruList.Back()
		bunchId := self.lruList.Remove(elem).(int64)
		bunch := self.table[bunchId]
		bunch.elem = nil
		if bunch.needsWrite {
			self.putCoordsPacked(bunchId, bunch.coords)
		}
		select {
		case freeNodes <- bunch.coords:
		default:
		}
		delete(self.table, bunchId)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"strconv"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

type SimpleChaincode struct {
}

func main() {
	fmt.Println("started logging in main()")
	err := shim.Start(new(SimpleChaincode))
	if err != nil {
		fmt.Println("failed in function main()")
	}
}

var projectName string
var projectRate int
var projectPeriod int
var projectGoal int
var projectTimes int
var projectBenifary string
var projectState string
var currentPrice float64
var projectSummary int
var userList []string
var shareList map[string]int
var availableList map[string]int

//Init comment
func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Init()")

	//started to initialize the projectState
	//step1: check whether args == 5
	if len(args) != 5 {
		return nil, errors.New("failed in args")
	}

	//step 2, intialize the project properties
	projectName = args[0]
	stub.PutState("projectName", []byte(projectName))

	projectRate, _ = strconv.Atoi(args[1])
	if projectRate <= 0 {
		return nil, errors.New("errors in args[1], it cannot be negative")
	}
	stub.PutState("projectRate", []byte(args[1]))

	projectPeriod, _ = strconv.Atoi(args[2])
	if projectPeriod <= 0 {
		return nil, errors.New("errors in args[2], it cannot be negative")
	}
	stub.PutState("projectPeriod", []byte(args[2]))

	projectGoal, _ = strconv.Atoi(args[3])
	if projectGoal <= 0 {
		return nil, errors.New("errors in args[3], it cannot be negative")
	}
	stub.PutState("projectGoal", []byte(args[3]))

	projectTimes = 1
	stub.PutState("projectTimes", []byte(strconv.Itoa(projectTimes)))

	projectBenifary = args[4]
	stub.PutState("projectBenifary", []byte(args[4]))

	projectState = "draft"
	stub.PutState("projectState", []byte(projectState))

	currentPrice = 1.0
	projectSummary = 0

	return nil, nil
} //end of Init()

// Invoke comment
func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Invoke()")
	switch function {
	case "goPublic":
		fmt.Println("started logging in goPublic")
		if len(args) != 0 {
			return nil, errors.New("failed in args")
		}
		projectState = "public"
		return nil, nil //end of goPublic
	case "pay":
		fmt.Println("started logging in pay()")

		if len(args) != 2 {
			return nil, errors.New("failed in args")
		}
		user := args[0]
		amount := args[1]
		fmt.Printf("user is %v and amount is %v\n", user, amount)
		amountInt, _ := strconv.Atoi(amount)
		if amountInt <= 0 {
			return nil, errors.New("amount is negative")
		}

		// if shareList[user] == 0 {
		// 	userList = append(userList, user)
		// }
		//
		shareList[user] += amountInt
		projectSummary += amountInt
		return nil, nil
	case "checkGoalReached":
		fmt.Println("started logging in checkGoalReached()")
		// TODO: write code for checkGoalReached
		return nil, nil
	case "checkDaoqi":
		fmt.Println("started logging in checkDaoqi()")
		// TODO: write code for checkDaoqi
		return nil, nil
	case "calculatePrice":
		fmt.Println("started logging in calculatePrice")
		return nil, nil
	case "calculateResult":
		fmt.Println("started logging in calculateResult")
		return nil, nil
	}
	return nil, nil
}

// Query comment
func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Query()")
	switch function {
	case "getProjectName":
		result, err := stub.GetState("projectName")
		if err != nil {
			return nil, errors.New("error in getProjectName")
		}
		return result, nil
	case "getProjectGoal":
		result, err := stub.GetState("projectGoal")
		if err != nil {
			return nil, errors.New("error in getProjectGoal")
		}
		return result, nil
	case "getProjectSummary":
		result := strconv.Itoa(projectSummary)
		return []byte(result), nil
	case "getUserList":
		fmt.Println("started logging in getUserList")
		var result string
		for i, value := range userList {
			result += fmt.Sprintf("userList[%v] is %v ****", strconv.Itoa(i), value)
		}
		return []byte(result), nil
	case "getShareList":
		fmt.Println("started logging in getShareList")
		return nil, nil
	case "getAvailableList":
		fmt.Println("started logging in getAvailableList")
		return nil, nil
	}
	return nil, nil
}
<commit_msg>demo7.go<commit_after>package main

import (
	"errors"
	"fmt"
	"strconv"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

type SimpleChaincode struct {
}

func main() {
	fmt.Println("started logging in main()")
	err := shim.Start(new(SimpleChaincode))
	if err != nil {
		fmt.Println("failed in function main()")
	}
}

var projectName string
var projectRate int
var projectPeriod int
var projectGoal int
var projectTimes int
var projectBenifary string
var projectState string
var currentPrice float64
var projectSummary int
var userList []string
var shareList map[string]int
var availableList map[string]int

//Init comment
func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Init()")

	//started to initialize the projectState
	//step1: check whether args == 5
	if len(args) != 5 {
		return nil, errors.New("failed in args")
	}

	//step 2, intialize the project properties
	projectName = args[0]
	stub.PutState("projectName", []byte(projectName))

	projectRate, _ = strconv.Atoi(args[1])
	if projectRate <= 0 {
		return nil, errors.New("errors in args[1], it cannot be negative")
	}
	stub.PutState("projectRate", []byte(args[1]))

	projectPeriod, _ = strconv.Atoi(args[2])
	if projectPeriod <= 0 {
		return nil, errors.New("errors in args[2], it cannot be negative")
	}
	stub.PutState("projectPeriod", []byte(args[2]))

	projectGoal, _ = strconv.Atoi(args[3])
	if projectGoal <= 0 {
		return nil, errors.New("errors in args[3], it cannot be negative")
	}
	stub.PutState("projectGoal", []byte(args[3]))

	projectTimes = 1
	stub.PutState("projectTimes", []byte(strconv.Itoa(projectTimes)))

	projectBenifary = args[4]
	stub.PutState("projectBenifary", []byte(args[4]))

	projectState = "draft"
	stub.PutState("projectState", []byte(projectState))

	currentPrice = 1.0
	projectSummary = 0

	return nil, nil
} //end of Init()

// Invoke comment
func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Invoke()")
	switch function {
	case "goPublic":
		fmt.Println("started logging in goPublic")
		if len(args) != 0 {
			return nil, errors.New("failed in args")
		}
		projectState = "public"
		return nil, nil //end of goPublic
	case "pay":
		fmt.Println("started logging in pay()")

		if len(args) != 2 {
			return nil, errors.New("failed in args")
		}
		user := args[0]
		amount := args[1]
		fmt.Printf("user is %v and amount is %v\n", user, amount)
		amountInt, _ := strconv.Atoi(amount)
		if amountInt <= 0 {
			return nil, errors.New("amount is negative")
		}

		// if shareList[user] == 0 {
		// 	userList = append(userList, user)
		// }
		//
		value := shareList[user]
		value += amountInt
		shareList[user] = value

		projectSummary += amountInt
		return nil, nil
	case "checkGoalReached":
		fmt.Println("started logging in checkGoalReached()")
		// TODO: write code for checkGoalReached
		return nil, nil
	case "checkDaoqi":
		fmt.Println("started logging in checkDaoqi()")
		// TODO: write code for checkDaoqi
		return nil, nil
	case "calculatePrice":
		fmt.Println("started logging in calculatePrice")
		return nil, nil
	case "calculateResult":
		fmt.Println("started logging in calculateResult")
		return nil, nil
	}
	return nil, nil
}

// Query comment
func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("started logging in Query()")
	switch function {
	case "getProjectName":
		result, err := stub.GetState("projectName")
		if err != nil {
			return nil, errors.New("error in getProjectName")
		}
		return result, nil
	case "getProjectGoal":
		result, err := stub.GetState("projectGoal")
		if err != nil {
			return nil, errors.New("error in getProjectGoal")
		}
		return result, nil
	case "getProjectSummary":
		result := strconv.Itoa(projectSummary)
		return []byte(result), nil
	case "getUserList":
		fmt.Println("started logging in getUserList")
		var result string
		for i, value := range userList {
			result += fmt.Sprintf("userList[%v] is %v ****", strconv.Itoa(i), value)
		}
		return []byte(result), nil
	case "getShareList":
		fmt.Println("started logging in getShareList")
		return nil, nil
	case "getAvailableList":
		fmt.Println("started logging in getAvailableList")
		return nil, nil
	}
	return nil, nil
}
<|endoftext|>


<commit_before>package dht

import (
	"context"
	"fmt"
	"testing"
	"time"

	tu "github.com/libp2p/go-testutil"
)

func TestNotifieeMultipleConn(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	d1 := setupDHT(ctx, t, false)
	d2 := setupDHT(ctx, t, false)

	nn1 := (*netNotifiee)(d1)
	nn2 := (*netNotifiee)(d2)

	connect(t, ctx, d1, d2)
	c12 := d1.host.Network().ConnsToPeer(d2.self)[0]
	c21 := d2.host.Network().ConnsToPeer(d1.self)[0]

	// Pretend to reestablish/re-kill connection
	nn1.Connected(d1.host.Network(), c12)
	nn2.Connected(d2.host.Network(), c21)

	if !checkRoutingTable(d1, d2) {
		t.Fatal("no routes")
	}
	nn1.Disconnected(d1.host.Network(), c12)
	nn2.Disconnected(d2.host.Network(), c21)

	if !checkRoutingTable(d1, d2) {
		t.Fatal("no routes")
	}

	for _, conn := range d1.host.Network().ConnsToPeer(d2.self) {
		conn.Close()
	}
	for _, conn := range d2.host.Network().ConnsToPeer(d1.self) {
		conn.Close()
	}

	tu.WaitFor(ctx, func() error {
		if checkRoutingTable(d1, d2) {
			return fmt.Errorf("should not have routes")
		}
		return nil
	})
}

func TestNotifieeFuzz(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Second*2)
	defer cancel()

	d1 := setupDHT(ctx, t, false)
	d2 := setupDHT(ctx, t, false)

	for i := 0; i < 100; i++ {
		connectNoSync(t, ctx, d1, d2)
		for _, conn := range d1.host.Network().ConnsToPeer(d2.self) {
			conn.Close()
		}
	}
	tu.WaitFor(ctx, func() error {
		if checkRoutingTable(d1, d2) {
			return fmt.Errorf("should not have routes")
		}
		return nil
	})
	connect(t, ctx, d1, d2)
}

func checkRoutingTable(a, b *IpfsDHT) bool {
	// loop until connection notification has been received.
	// under high load, this may not happen as immediately as we would like.
	return a.routingTable.Find(b.self) != "" && b.routingTable.Find(a.self) != ""
}
<commit_msg>Reduce aggressive test<commit_after>package dht

import (
	"context"
	"fmt"
	"testing"
	"time"

	tu "github.com/libp2p/go-testutil"
)

func TestNotifieeMultipleConn(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	d1 := setupDHT(ctx, t, false)
	d2 := setupDHT(ctx, t, false)

	nn1 := (*netNotifiee)(d1)
	nn2 := (*netNotifiee)(d2)

	connect(t, ctx, d1, d2)
	c12 := d1.host.Network().ConnsToPeer(d2.self)[0]
	c21 := d2.host.Network().ConnsToPeer(d1.self)[0]

	// Pretend to reestablish/re-kill connection
	nn1.Connected(d1.host.Network(), c12)
	nn2.Connected(d2.host.Network(), c21)

	if !checkRoutingTable(d1, d2) {
		t.Fatal("no routes")
	}
	nn1.Disconnected(d1.host.Network(), c12)
	nn2.Disconnected(d2.host.Network(), c21)

	if !checkRoutingTable(d1, d2) {
		t.Fatal("no routes")
	}

	for _, conn := range d1.host.Network().ConnsToPeer(d2.self) {
		conn.Close()
	}
	for _, conn := range d2.host.Network().ConnsToPeer(d1.self) {
		conn.Close()
	}

	tu.WaitFor(ctx, func() error {
		if checkRoutingTable(d1, d2) {
			return fmt.Errorf("should not have routes")
		}
		return nil
	})
}

func TestNotifieeFuzz(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Second*2)
	defer cancel()

	d1 := setupDHT(ctx, t, false)
	d2 := setupDHT(ctx, t, false)

	for i := 0; i < 10; i++ {
		connectNoSync(t, ctx, d1, d2)
		for _, conn := range d1.host.Network().ConnsToPeer(d2.self) {
			conn.Close()
		}
	}
	tu.WaitFor(ctx, func() error {
		if checkRoutingTable(d1, d2) {
			return fmt.Errorf("should not have routes")
		}
		return nil
	})
	connect(t, ctx, d1, d2)
}

func checkRoutingTable(a, b *IpfsDHT) bool {
	// loop until connection notification has been received.
	// under high load, this may not happen as immediately as we would like.
	return a.routingTable.Find(b.self) != "" && b.routingTable.Find(a.self) != ""
}
<|endoftext|>


<commit_before>package models

import (
	"database/sql"
	"github.com/jinzhu/gorm"
	"github.com/qb0C80aE/clay/extensions"
)

// TemplatePersistentParameter is the model class what represents model-independent parameters used in templates
type TemplatePersistentParameter struct {
	ID          int             `json:"id" gorm:"primary_key;AUTO_INCREMENT"`
	TemplateID  int             `json:"template_id" gorm:"unique_index:template_id_name" sql:"type:integer references templates(id)"`
	Name        string          `json:"name" gorm:"unique_index:template_id_name"`
	ValueInt    sql.NullInt64   `json:"value_int"`
	ValueFloat  sql.NullFloat64 `json:"value_float"`
	ValueBool   sql.NullBool    `json:"value_bool"`
	ValueString sql.NullString  `json:"value_string"`
}

// NewTemplatePersistentParameterModel creates a template persistent parameter model instance
func NewTemplatePersistentParameterModel() *TemplatePersistentParameter {
	return &TemplatePersistentParameter{}
}

var sharedTemplatePersistentParameterModel = NewTemplatePersistentParameterModel()

// SharedTemplatePersistentParameterModel returns the template persistent parameter model instance used as a model prototype and type analysis
func SharedTemplatePersistentParameterModel() *TemplatePersistentParameter {
	return sharedTemplatePersistentParameterModel
}

// SetupInitialData setups the initial data
func (templatePersistentParameter *TemplatePersistentParameter) SetupInitialData(db *gorm.DB) error {
	return db.Exec(`
		create trigger if not exists DeleteTemplatePersistentParameterAfterTemplateDeletion after delete on templates
		begin
			delete from template_persistent_parameters where template_id = old.id;
		end;
	`).Error
}

func init() {
	extensions.RegisterInitialDataLoader(sharedTemplatePersistentParameterModel)
	extensions.RegisterModel(sharedTemplatePersistentParameterModel, true)
}
<commit_msg>Fix trigger conditions to delete children before parent deletion<commit_after>package models

import (
	"database/sql"
	"github.com/jinzhu/gorm"
	"github.com/qb0C80aE/clay/extensions"
)

// TemplatePersistentParameter is the model class what represents model-independent parameters used in templates
type TemplatePersistentParameter struct {
	ID          int             `json:"id" gorm:"primary_key;AUTO_INCREMENT"`
	TemplateID  int             `json:"template_id" gorm:"unique_index:template_id_name" sql:"type:integer references templates(id)"`
	Name        string          `json:"name" gorm:"unique_index:template_id_name"`
	ValueInt    sql.NullInt64   `json:"value_int"`
	ValueFloat  sql.NullFloat64 `json:"value_float"`
	ValueBool   sql.NullBool    `json:"value_bool"`
	ValueString sql.NullString  `json:"value_string"`
}

// NewTemplatePersistentParameterModel creates a template persistent parameter model instance
func NewTemplatePersistentParameterModel() *TemplatePersistentParameter {
	return &TemplatePersistentParameter{}
}

var sharedTemplatePersistentParameterModel = NewTemplatePersistentParameterModel()

// SharedTemplatePersistentParameterModel returns the template persistent parameter model instance used as a model prototype and type analysis
func SharedTemplatePersistentParameterModel() *TemplatePersistentParameter {
	return sharedTemplatePersistentParameterModel
}

// SetupInitialData setups the initial data
func (templatePersistentParameter *TemplatePersistentParameter) SetupInitialData(db *gorm.DB) error {
	return db.Exec(`
		create trigger if not exists DeleteTemplatePersistentParameterBeforeTemplateDeletion before delete on templates
		begin
			delete from template_persistent_parameters where template_id = old.id;
		end;
	`).Error
}

func init() {
	extensions.RegisterInitialDataLoader(sharedTemplatePersistentParameterModel)
	extensions.RegisterModel(sharedTemplatePersistentParameterModel, true)
}
<|endoftext|>


<commit_before>package webhook

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/stripe/stripe-go"
)

const (
	// DefaultTolerance indicates that signatures older than this will be rejected by ConstructEvent.
	DefaultTolerance time.Duration = 300 * time.Second
	// signingVersion represents the version of the signature we currently use.
	signingVersion string = "v1"
)

// This block represents the list of errors that could be raised when using the webhook package.
var (
	ErrNotSigned        = errors.New("webhook has no Stripe-Signature header")
	ErrInvalidHeader    = errors.New("webhook has invalid Stripe-Signature header")
	ErrTooOld           = errors.New("timestamp wasn't within tolerance")
	ErrNoValidSignature = errors.New("webhook had no valid signature")
)

// ComputeSignature computes a webhook signature using Stripe's v1 signing
// method.
//
// See https://stripe.com/docs/webhooks#signatures for more information.
func ComputeSignature(t time.Time, payload []byte, secret string) []byte {
	mac := hmac.New(sha256.New, []byte(secret))
	mac.Write([]byte(fmt.Sprintf("%d", t.Unix())))
	mac.Write([]byte("."))
	mac.Write(payload)
	return mac.Sum(nil)
}

type signedHeader struct {
	timestamp  time.Time
	signatures [][]byte
}

func parseSignatureHeader(header string) (*signedHeader, error) {
	sh := &signedHeader{}

	if header == "" {
		return sh, ErrNotSigned
	}

	// Signed header looks like "t=1495999758,v1=ABC,v1=DEF,v0=GHI"
	pairs := strings.Split(header, ",")
	for _, pair := range pairs {
		parts := strings.Split(pair, "=")
		if len(parts) != 2 {
			return sh, ErrInvalidHeader
		}

		switch parts[0] {
		case "t":
			timestamp, err := strconv.ParseInt(parts[1], 10, 64)
			if err != nil {
				return sh, ErrInvalidHeader
			}
			sh.timestamp = time.Unix(timestamp, 0)

		case signingVersion:
			sig, err := hex.DecodeString(parts[1])
			if err != nil {
				continue // Ignore invalid signatures
			}

			sh.signatures = append(sh.signatures, sig)

		default:
			continue // Ignore unknown parts of the header
		}
	}

	if len(sh.signatures) == 0 {
		return sh, ErrNoValidSignature
	}

	return sh, nil
}

// ValidatePayload validates the payload against the Stripe-Signature header
// using the specified signing secret. Returns an error if the body or
// Stripe-Signature header provided are unreadable, if the signature doesn't
// match, or if the timestamp for the signature is older than DefaultTolerance.
//
// NOTE: Stripe will only send Webhook signing headers after you have retrieved
// your signing secret from the Stripe dashboard:
// https://dashboard.stripe.com/webhooks
//
func ValidatePayload(payload []byte, header string, secret string) error {
	return ValidatePayloadWithTolerance(payload, header, secret, DefaultTolerance)
}

// ValidatePayloadWithTolerance validates the payload against the Stripe-Signature header
// using the specified signing secret and tolerance window. Returns an error if the body
// or Stripe-Signature header provided are unreadable, if the signature doesn't match, or
// if the timestamp for the signature is older than the specified tolerance.
//
// NOTE: Stripe will only send Webhook signing headers after you have retrieved
// your signing secret from the Stripe dashboard:
// https://dashboard.stripe.com/webhooks
//
func ValidatePayloadWithTolerance(payload []byte, header string, secret string, tolerance time.Duration) error {
	return validatePayload(payload, header, secret, tolerance, true)
}

// ValidatePayloadIgnoringTolerance validates the payload against the Stripe-Signature header
// header using the specified signing secret. Returns an error if the body or
// Stripe-Signature header provided are unreadable or if the signature doesn't match.
// Does not check the signature's timestamp.
//
// NOTE: Stripe will only send Webhook signing headers after you have retrieved
// your signing secret from the Stripe dashboard:
// https://dashboard.stripe.com/webhooks
//
func ValidatePayloadIgnoringTolerance(payload []byte, header string, secret string) error {
	return validatePayload(payload, header, secret, 0*time.Second, false)
}

func validatePayload(payload []byte, sigHeader string, secret string, tolerance time.Duration, enforceTolerance bool) error {

	header, err := parseSignatureHeader(sigHeader)
	if err != nil {
		return err
	}

	expectedSignature := ComputeSignature(header.timestamp, payload, secret)
	expiredTimestamp := time.Since(header.timestamp) > tolerance
	if enforceTolerance && expiredTimestamp {
		return ErrTooOld
	}

	// Check all given v1 signatures, multiple signatures will be sent temporarily in the case of a rolled signature secret
	for _, sig := range header.signatures {
		if hmac.Equal(expectedSignature, sig) {
			return nil
		}
	}

	return ErrNoValidSignature
}

// ConstructEvent initializes an Event object from a JSON webhook payload, validating
// the Stripe-Signature header using the specified signing secret. Returns an error
// if the body or Stripe-Signature header provided are unreadable, if the
// signature doesn't match, or if the timestamp for the signature is older than
// DefaultTolerance.
//
// NOTE: Stripe will only send Webhook signing headers after you have retrieved
// your signing secret from the Stripe dashboard:
// https://dashboard.stripe.com/webhooks
//
func ConstructEvent(payload []byte, header string, secret string) (stripe.Event, error) {
	return ConstructEventWithTolerance(payload, header, secret, DefaultTolerance)
}

// ConstructEventWithTolerance initializes an Event object from a JSON webhook payload,
// validating the signature in the Stripe-Signature header using the specified signing
// secret and tolerance window. Returns an error if the body or Stripe-Signature header
// provided are unreadable, if the signature doesn't match, or if the timestamp
// for the signature is older than the specified tolerance.
//
// NOTE: Stripe will only send Webhook signing headers after you have retrieved
// your signing secret from the Stripe dashboard:
// https://dashboard.stripe.com/webhooks
//
func ConstructEventWithTolerance(payload []byte, header string, secret string, tolerance time.Duration) (stripe.Event, error) {
	return constructEvent(payload, header, secret, tolerance, true)
}

// ConstructEventIgnoringTolerance initializes an Event object from a JSON webhook
// payload, validating the Stripe-Signature header using the specified signing secret.
// Returns an error if the body or Stripe-Signature header provided are unreadable or
// if the signature doesn't match. Does not check the signature's timestamp.
//
// NOTE: Stripe will only send Webhook signing headers after you have retrieved
// your signing secret from the Stripe dashboard:
// https://dashboard.stripe.com/webhooks
//
func ConstructEventIgnoringTolerance(payload []byte, header string, secret string) (stripe.Event, error) {
	return constructEvent(payload, header, secret, 0*time.Second, false)
}

func constructEvent(payload []byte, sigHeader string, secret string, tolerance time.Duration, enforceTolerance bool) (stripe.Event, error) {
	e := stripe.Event{}

	if err := validatePayload(payload, sigHeader, secret, tolerance, enforceTolerance); err != nil {
		return e, err
	}

	if err := json.Unmarshal(payload, &e); err != nil {
		return e, fmt.Errorf("Failed to parse webhook body json: %s", err.Error())
	}

	return e, nil

}
<commit_msg>Organize alphabetically and into sections<commit_after>package webhook

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/stripe/stripe-go"
)

//
// Public constants
//

const (
	// DefaultTolerance indicates that signatures older than this will be rejected by ConstructEvent.
	DefaultTolerance time.Duration = 300 * time.Second
	// signingVersion represents the version of the signature we currently use.
	signingVersion string = "v1"
)

//
// Public variables
//

// This block represents the list of errors that could be raised when using the webhook package.
var (
	ErrInvalidHeader    = errors.New("webhook has invalid Stripe-Signature header")
	ErrNoValidSignature = errors.New("webhook had no valid signature")
	ErrNotSigned        = errors.New("webhook has no Stripe-Signature header")
	ErrTooOld           = errors.New("timestamp wasn't within tolerance")
)

//
// Public functions
//

// ComputeSignature computes a webhook signature using Stripe's v1 signing
// method.
//
// See https://stripe.com/docs/webhooks#signatures for more information.
func ComputeSignature(t time.Time, payload []byte, secret string) []byte {
	mac := hmac.New(sha256.New, []byte(secret))
	mac.Write([]byte(fmt.Sprintf("%d", t.Unix())))
	mac.Write([]byte("."))
	mac.Write(payload)
	return mac.Sum(nil)
}

// ConstructEvent initializes an Event object from a JSON webhook payload, validating
// the Stripe-Signature header using the specified signing secret. Returns an error
// if the body or Stripe-Signature header provided are unreadable, if the
// signature doesn't match, or if the timestamp for the signature is older than
// DefaultTolerance.
//
// NOTE: Stripe will only send Webhook signing headers after you have retrieved
// your signing secret from the Stripe dashboard:
// https://dashboard.stripe.com/webhooks
//
func ConstructEvent(payload []byte, header string, secret string) (stripe.Event, error) {
	return ConstructEventWithTolerance(payload, header, secret, DefaultTolerance)
}

// ConstructEventIgnoringTolerance initializes an Event object from a JSON webhook
// payload, validating the Stripe-Signature header using the specified signing secret.
// Returns an error if the body or Stripe-Signature header provided are unreadable or
// if the signature doesn't match. Does not check the signature's timestamp.
//
// NOTE: Stripe will only send Webhook signing headers after you have retrieved
// your signing secret from the Stripe dashboard:
// https://dashboard.stripe.com/webhooks
//
func ConstructEventIgnoringTolerance(payload []byte, header string, secret string) (stripe.Event, error) {
	return constructEvent(payload, header, secret, 0*time.Second, false)
}

// ConstructEventWithTolerance initializes an Event object from a JSON webhook payload,
// validating the signature in the Stripe-Signature header using the specified signing
// secret and tolerance window. Returns an error if the body or Stripe-Signature header
// provided are unreadable, if the signature doesn't match, or if the timestamp
// for the signature is older than the specified tolerance.
//
// NOTE: Stripe will only send Webhook signing headers after you have retrieved
// your signing secret from the Stripe dashboard:
// https://dashboard.stripe.com/webhooks
//
func ConstructEventWithTolerance(payload []byte, header string, secret string, tolerance time.Duration) (stripe.Event, error) {
	return constructEvent(payload, header, secret, tolerance, true)
}

// ValidatePayload validates the payload against the Stripe-Signature header
// using the specified signing secret. Returns an error if the body or
// Stripe-Signature header provided are unreadable, if the signature doesn't
// match, or if the timestamp for the signature is older than DefaultTolerance.
//
// NOTE: Stripe will only send Webhook signing headers after you have retrieved
// your signing secret from the Stripe dashboard:
// https://dashboard.stripe.com/webhooks
//
func ValidatePayload(payload []byte, header string, secret string) error {
	return ValidatePayloadWithTolerance(payload, header, secret, DefaultTolerance)
}

// ValidatePayloadIgnoringTolerance validates the payload against the Stripe-Signature header
// header using the specified signing secret. Returns an error if the body or
// Stripe-Signature header provided are unreadable or if the signature doesn't match.
// Does not check the signature's timestamp.
//
// NOTE: Stripe will only send Webhook signing headers after you have retrieved
// your signing secret from the Stripe dashboard:
// https://dashboard.stripe.com/webhooks
//
func ValidatePayloadIgnoringTolerance(payload []byte, header string, secret string) error {
	return validatePayload(payload, header, secret, 0*time.Second, false)
}

// ValidatePayloadWithTolerance validates the payload against the Stripe-Signature header
// using the specified signing secret and tolerance window. Returns an error if the body
// or Stripe-Signature header provided are unreadable, if the signature doesn't match, or
// if the timestamp for the signature is older than the specified tolerance.
//
// NOTE: Stripe will only send Webhook signing headers after you have retrieved
// your signing secret from the Stripe dashboard:
// https://dashboard.stripe.com/webhooks
//
func ValidatePayloadWithTolerance(payload []byte, header string, secret string, tolerance time.Duration) error {
	return validatePayload(payload, header, secret, tolerance, true)
}

//
// Private types
//

type signedHeader struct {
	timestamp  time.Time
	signatures [][]byte
}

//
// Private functions
//

func constructEvent(payload []byte, sigHeader string, secret string, tolerance time.Duration, enforceTolerance bool) (stripe.Event, error) {
	e := stripe.Event{}

	if err := validatePayload(payload, sigHeader, secret, tolerance, enforceTolerance); err != nil {
		return e, err
	}

	if err := json.Unmarshal(payload, &e); err != nil {
		return e, fmt.Errorf("Failed to parse webhook body json: %s", err.Error())
	}

	return e, nil

}

func parseSignatureHeader(header string) (*signedHeader, error) {
	sh := &signedHeader{}

	if header == "" {
		return sh, ErrNotSigned
	}

	// Signed header looks like "t=1495999758,v1=ABC,v1=DEF,v0=GHI"
	pairs := strings.Split(header, ",")
	for _, pair := range pairs {
		parts := strings.Split(pair, "=")
		if len(parts) != 2 {
			return sh, ErrInvalidHeader
		}

		switch parts[0] {
		case "t":
			timestamp, err := strconv.ParseInt(parts[1], 10, 64)
			if err != nil {
				return sh, ErrInvalidHeader
			}
			sh.timestamp = time.Unix(timestamp, 0)

		case signingVersion:
			sig, err := hex.DecodeString(parts[1])
			if err != nil {
				continue // Ignore invalid signatures
			}

			sh.signatures = append(sh.signatures, sig)

		default:
			continue // Ignore unknown parts of the header
		}
	}

	if len(sh.signatures) == 0 {
		return sh, ErrNoValidSignature
	}

	return sh, nil
}

func validatePayload(payload []byte, sigHeader string, secret string, tolerance time.Duration, enforceTolerance bool) error {

	header, err := parseSignatureHeader(sigHeader)
	if err != nil {
		return err
	}

	expectedSignature := ComputeSignature(header.timestamp, payload, secret)
	expiredTimestamp := time.Since(header.timestamp) > tolerance
	if enforceTolerance && expiredTimestamp {
		return ErrTooOld
	}

	// Check all given v1 signatures, multiple signatures will be sent temporarily in the case of a rolled signature secret
	for _, sig := range header.signatures {
		if hmac.Equal(expectedSignature, sig) {
			return nil
		}
	}

	return ErrNoValidSignature
}
<|endoftext|>


<commit_before>package nsq

import (
	"bytes"
	"encoding/binary"
	"io"
	"io/ioutil"
	"time"
)

// The number of bytes for a Message.Id
const MsgIdLength = 16

type MessageID [MsgIdLength]byte

// Message is the fundamental data type containing
// the id, body, and metadata
type Message struct {
	Id        MessageID
	Body      []byte
	Timestamp int64
	Attempts  uint16
}

// NewMessage creates a Message, initializes some metadata,
// and returns a pointer
func NewMessage(id MessageID, body []byte) *Message {
	return &Message{
		Id:        id,
		Body:      body,
		Timestamp: time.Now().Unix(),
	}
}

// EncodeBytes serializes the message into a new, returned, []byte
func (m *Message) EncodeBytes() ([]byte, error) {
	var buf bytes.Buffer
	err := m.Write(&buf)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// Write serializes the message into the supplied writer.
//
// It is suggested that the target Writer is buffered to avoid performing many system calls.
func (m *Message) Write(w io.Writer) error {
	err := binary.Write(w, binary.BigEndian, &m.Timestamp)
	if err != nil {
		return err
	}

	err = binary.Write(w, binary.BigEndian, &m.Attempts)
	if err != nil {
		return err
	}

	_, err = w.Write(m.Id[:])
	if err != nil {
		return err
	}

	_, err = w.Write(m.Body)
	if err != nil {
		return err
	}

	return nil
}

// DecodeMessage deseralizes data (as []byte) and creates a new Message
func DecodeMessage(byteBuf []byte) (*Message, error) {
	var timestamp int64
	var attempts uint16
	var msg Message

	buf := bytes.NewBuffer(byteBuf)

	err := binary.Read(buf, binary.BigEndian, &timestamp)
	if err != nil {
		return nil, err
	}

	err = binary.Read(buf, binary.BigEndian, &attempts)
	if err != nil {
		return nil, err
	}

	_, err = io.ReadFull(buf, msg.Id[:])
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(buf)
	if err != nil {
		return nil, err
	}

	msg.Body = body
	msg.Timestamp = timestamp
	msg.Attempts = attempts

	return &msg, nil
}
<commit_msg>nsqd: message timestamps should be nano<commit_after>package nsq

import (
	"bytes"
	"encoding/binary"
	"io"
	"io/ioutil"
	"time"
)

// The number of bytes for a Message.Id
const MsgIdLength = 16

type MessageID [MsgIdLength]byte

// Message is the fundamental data type containing
// the id, body, and metadata
type Message struct {
	Id        MessageID
	Body      []byte
	Timestamp int64
	Attempts  uint16
}

// NewMessage creates a Message, initializes some metadata,
// and returns a pointer
func NewMessage(id MessageID, body []byte) *Message {
	return &Message{
		Id:        id,
		Body:      body,
		Timestamp: time.Now().UnixNano(),
	}
}

// EncodeBytes serializes the message into a new, returned, []byte
func (m *Message) EncodeBytes() ([]byte, error) {
	var buf bytes.Buffer
	err := m.Write(&buf)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// Write serializes the message into the supplied writer.
//
// It is suggested that the target Writer is buffered to avoid performing many system calls.
func (m *Message) Write(w io.Writer) error {
	err := binary.Write(w, binary.BigEndian, &m.Timestamp)
	if err != nil {
		return err
	}

	err = binary.Write(w, binary.BigEndian, &m.Attempts)
	if err != nil {
		return err
	}

	_, err = w.Write(m.Id[:])
	if err != nil {
		return err
	}

	_, err = w.Write(m.Body)
	if err != nil {
		return err
	}

	return nil
}

// DecodeMessage deseralizes data (as []byte) and creates a new Message
func DecodeMessage(byteBuf []byte) (*Message, error) {
	var timestamp int64
	var attempts uint16
	var msg Message

	buf := bytes.NewBuffer(byteBuf)

	err := binary.Read(buf, binary.BigEndian, &timestamp)
	if err != nil {
		return nil, err
	}

	err = binary.Read(buf, binary.BigEndian, &attempts)
	if err != nil {
		return nil, err
	}

	_, err = io.ReadFull(buf, msg.Id[:])
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(buf)
	if err != nil {
		return nil, err
	}

	msg.Body = body
	msg.Timestamp = timestamp
	msg.Attempts = attempts

	return &msg, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 bee authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package dockerize

import (
	"flag"
	"os"
	"path"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/beego/bee/cmd/commands"
	"github.com/beego/bee/cmd/commands/version"
	beeLogger "github.com/beego/bee/logger"
	"github.com/beego/bee/utils"
)

const dockerBuildTemplate = `FROM {{.BaseImage}}

# Godep for vendoring
RUN go get github.com/tools/godep

# Recompile the standard library without CGO
RUN CGO_ENABLED=0 go install -a std

ENV APP_DIR $GOPATH{{.Appdir}}
RUN mkdir -p $APP_DIR

# Set the entrypoint
ENTRYPOINT $APP_DIR/{{.Entrypoint}}
ADD . $APP_DIR

# Compile the binary and statically link
RUN cd $APP_DIR && CGO_ENABLED=0 godep go build -ldflags '-d -w -s'

EXPOSE {{.Expose}}
`

// Dockerfile holds the information about the Docker container.
type Dockerfile struct {
	BaseImage  string
	Appdir     string
	Entrypoint string
	Expose     string
}

var CmdDockerize = &commands.Command{
	CustomFlags: true,
	UsageLine:   "dockerize",
	Short:       "Generates a Dockerfile for your Beego application",
	Long: `Dockerize generates a Dockerfile for your Beego Web Application.
  The Dockerfile will compile, get the dependencies with {{"godep"|bold}}, and set the entrypoint.

  {{"Example:"|bold}}
    $ bee dockerize -expose="3000,80,25"
  `,
	PreRun: func(cmd *commands.Command, args []string) { version.ShowShortVersionBanner() },
	Run:    dockerizeApp,
}

var (
	expose    string
	baseImage string
)

func init() {
	fs := flag.NewFlagSet("dockerize", flag.ContinueOnError)
	fs.StringVar(&baseImage, "image", "library/golang", "Set the base image of the Docker container.")
	fs.StringVar(&expose, "expose", "8080", "Port(s) to expose in the Docker container.")
	CmdDockerize.Flag = *fs
	commands.AvailableCommands = append(commands.AvailableCommands, CmdDockerize)
}

func dockerizeApp(cmd *commands.Command, args []string) int {
	if err := cmd.Flag.Parse(args); err != nil {
		beeLogger.Log.Fatalf("Error parsing flags: %v", err.Error())
	}

	beeLogger.Log.Info("Generating Dockerfile...")

	gopath := os.Getenv("GOPATH")
	dir, err := filepath.Abs(".")
	if err != nil {
		beeLogger.Log.Error(err.Error())
	}

	appdir := strings.Replace(dir, gopath, "", 1)

	// In case of multiple ports to expose inside the container,
	// replace all the commas with whitespaces.
	// See the verb EXPOSE in the Docker documentation.
	if strings.Contains(expose, ",") {
		expose = strings.Replace(expose, ",", " ", -1)
	}

	_, entrypoint := path.Split(appdir)
	dockerfile := Dockerfile{
		BaseImage:  baseImage,
		Appdir:     appdir,
		Entrypoint: entrypoint,
		Expose:     expose,
	}

	generateDockerfile(dockerfile)
	return 0
}

func generateDockerfile(df Dockerfile) {
	t := template.Must(template.New("dockerBuildTemplate").Parse(dockerBuildTemplate)).Funcs(utils.BeeFuncMap())

	f, err := os.Create("Dockerfile")
	if err != nil {
		beeLogger.Log.Fatalf("Error writing Dockerfile: %v", err.Error())
	}
	defer utils.CloseFile(f)

	t.Execute(f, df)

	beeLogger.Log.Success("Dockerfile generated.")
}
<commit_msg>Fix mistake of execution directory by dockerBuldTemplate<commit_after>// Copyright 2016 bee authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package dockerize

import (
	"flag"
	"os"
	"path"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/beego/bee/cmd/commands"
	"github.com/beego/bee/cmd/commands/version"
	beeLogger "github.com/beego/bee/logger"
	"github.com/beego/bee/utils"
)

const dockerBuildTemplate = `FROM {{.BaseImage}}

# Godep for vendoring
RUN go get github.com/tools/godep

# Recompile the standard library without CGO
RUN CGO_ENABLED=0 go install -a std

ENV APP_DIR $GOPATH{{.Appdir}}
RUN mkdir -p $APP_DIR

# Set the entrypoint
ENTRYPOINT (cd $APP_DIR && ./{{.Entrypoint}})
ADD . $APP_DIR

# Compile the binary and statically link
RUN cd $APP_DIR && CGO_ENABLED=0 godep go build -ldflags '-d -w -s'

EXPOSE {{.Expose}}
`

// Dockerfile holds the information about the Docker container.
type Dockerfile struct {
	BaseImage  string
	Appdir     string
	Entrypoint string
	Expose     string
}

var CmdDockerize = &commands.Command{
	CustomFlags: true,
	UsageLine:   "dockerize",
	Short:       "Generates a Dockerfile for your Beego application",
	Long: `Dockerize generates a Dockerfile for your Beego Web Application.
  The Dockerfile will compile, get the dependencies with {{"godep"|bold}}, and set the entrypoint.

  {{"Example:"|bold}}
    $ bee dockerize -expose="3000,80,25"
  `,
	PreRun: func(cmd *commands.Command, args []string) { version.ShowShortVersionBanner() },
	Run:    dockerizeApp,
}

var (
	expose    string
	baseImage string
)

func init() {
	fs := flag.NewFlagSet("dockerize", flag.ContinueOnError)
	fs.StringVar(&baseImage, "image", "library/golang", "Set the base image of the Docker container.")
	fs.StringVar(&expose, "expose", "8080", "Port(s) to expose in the Docker container.")
	CmdDockerize.Flag = *fs
	commands.AvailableCommands = append(commands.AvailableCommands, CmdDockerize)
}

func dockerizeApp(cmd *commands.Command, args []string) int {
	if err := cmd.Flag.Parse(args); err != nil {
		beeLogger.Log.Fatalf("Error parsing flags: %v", err.Error())
	}

	beeLogger.Log.Info("Generating Dockerfile...")

	gopath := os.Getenv("GOPATH")
	dir, err := filepath.Abs(".")
	if err != nil {
		beeLogger.Log.Error(err.Error())
	}

	appdir := strings.Replace(dir, gopath, "", 1)

	// In case of multiple ports to expose inside the container,
	// replace all the commas with whitespaces.
	// See the verb EXPOSE in the Docker documentation.
	if strings.Contains(expose, ",") {
		expose = strings.Replace(expose, ",", " ", -1)
	}

	_, entrypoint := path.Split(appdir)
	dockerfile := Dockerfile{
		BaseImage:  baseImage,
		Appdir:     appdir,
		Entrypoint: entrypoint,
		Expose:     expose,
	}

	generateDockerfile(dockerfile)
	return 0
}

func generateDockerfile(df Dockerfile) {
	t := template.Must(template.New("dockerBuildTemplate").Parse(dockerBuildTemplate)).Funcs(utils.BeeFuncMap())

	f, err := os.Create("Dockerfile")
	if err != nil {
		beeLogger.Log.Fatalf("Error writing Dockerfile: %v", err.Error())
	}
	defer utils.CloseFile(f)

	t.Execute(f, df)

	beeLogger.Log.Success("Dockerfile generated.")
}
<|endoftext|>


<commit_before>package kafka

import (
	l "log"
	"os"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/golib/color"
	log "github.com/funkygao/log4go"
)

type pubStore struct {
	shutdownCh chan struct{}

	wg       sync.WaitGroup
	hostname string // used as kafka client id
	dryRun   bool
	compress bool

	pubPools        map[string]*pubPool // key is cluster, each cluster maintains a conn pool
	pubPoolsCapcity int
	pubPoolsLock    sync.RWMutex
	idleTimeout     time.Duration

	// to avoid too frequent refresh
	// TODO refresh by cluster: current implementation will refresh zone
	lastRefreshedAt time.Time
}

func NewPubStore(poolCapcity int, idleTimeout time.Duration, compress bool,
	debug bool, dryRun bool) *pubStore {
	if debug {
		sarama.Logger = l.New(os.Stdout, color.Green("[Sarama]"), l.LstdFlags|l.Lshortfile)
	}

	return &pubStore{
		hostname:        ctx.Hostname(),
		compress:        compress,
		idleTimeout:     idleTimeout,
		pubPoolsCapcity: poolCapcity,
		pubPools:        make(map[string]*pubPool),
		dryRun:          dryRun,
		shutdownCh:      make(chan struct{}),
	}
}

func (this *pubStore) Name() string {
	return "kafka"
}

func (this *pubStore) Start() (err error) {
	// warmup: create pools according the current kafka topology
	for _, cluster := range meta.Default.ClusterNames() {
		this.pubPools[cluster] = newPubPool(this, cluster,
			meta.Default.BrokerList(cluster), this.pubPoolsCapcity)
	}

	this.wg.Add(1)
	go func() {
		defer this.wg.Done()

		for {
			select {
			case <-meta.Default.RefreshEvent():
				this.doRefresh()

			case <-this.shutdownCh:
				log.Trace("pub store[%s] stopped", this.Name())
				return
			}
		}
	}()

	return
}

func (this *pubStore) Stop() {
	this.pubPoolsLock.Lock()
	defer this.pubPoolsLock.Unlock()

	// close all kafka connections
	for _, pool := range this.pubPools {
		pool.Close()
	}

	close(this.shutdownCh)
	this.wg.Wait()
}

func (this *pubStore) doRefresh() {
	if time.Since(this.lastRefreshedAt) <= time.Second*5 {
		log.Warn("ignored too frequent refresh: %s", time.Since(this.lastRefreshedAt))
		return
	}

	this.pubPoolsLock.Lock()
	defer this.pubPoolsLock.Unlock()

	// pub pool
	activeClusters := make(map[string]struct{})
	for _, cluster := range meta.Default.ClusterNames() {
		activeClusters[cluster] = struct{}{}
		if _, present := this.pubPools[cluster]; !present {
			// found a new cluster
			this.pubPools[cluster] = newPubPool(this, cluster,
				meta.Default.BrokerList(cluster), this.pubPoolsCapcity)
		} else {
			this.pubPools[cluster].RefreshBrokerList(meta.Default.BrokerList(cluster))
		}
	}

	// shutdown the dead clusters
	for cluster, pool := range this.pubPools {
		if _, present := activeClusters[cluster]; !present {
			// this cluster is dead or removed forever
			pool.Close()
			delete(this.pubPools, cluster)
		}
	}

	this.lastRefreshedAt = time.Now()
}

func (this *pubStore) markPartitionsDead(topic string, deadPartitionIds map[int32]struct{}) {
	excludedPartitionsLock.Lock()
	if deadPartitionIds == nil {
		// this topic comes alive
		delete(excludedPartitions, topic)
	} else {
		excludedPartitions[topic] = deadPartitionIds
	}
	excludedPartitionsLock.Unlock()
}

func (this *pubStore) watchDeadPartitions() {
	ticker := time.NewTicker(time.Minute * 2) // TODO
	defer ticker.Stop()

	var lastTopics = make(map[string]struct{})
	for {
		select {
		case <-ticker.C:
			deadPartitions := manager.Default.DeadPartitions()
			for topic, dp := range deadPartitions {
				this.markPartitionsDead(topic, dp)

				lastTopics[topic] = struct{}{}
			}

			for lastDeadTopic := range lastTopics {
				if _, present := deadPartitions[lastDeadTopic]; !present {
					// this topic was marked dead last round, but this round it comes alive

					log.Trace("%s come alive again", lastDeadTopic)

					this.markPartitionsDead(lastDeadTopic, nil)
					delete(lastTopics, lastDeadTopic)
				}
			}

		case <-this.shutdownCh:
			return
		}
	}

}
<commit_msg>ensure kafka installed before startup<commit_after>package kafka

import (
	"fmt"
	l "log"
	"os"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/golib/color"
	gio "github.com/funkygao/golib/io"
	log "github.com/funkygao/log4go"
)

type pubStore struct {
	shutdownCh chan struct{}

	wg       sync.WaitGroup
	hostname string // used as kafka client id
	dryRun   bool
	compress bool

	pubPools        map[string]*pubPool // key is cluster, each cluster maintains a conn pool
	pubPoolsCapcity int
	pubPoolsLock    sync.RWMutex
	idleTimeout     time.Duration

	// to avoid too frequent refresh
	// TODO refresh by cluster: current implementation will refresh zone
	lastRefreshedAt time.Time
}

func NewPubStore(poolCapcity int, idleTimeout time.Duration, compress bool,
	debug bool, dryRun bool) *pubStore {
	if debug {
		sarama.Logger = l.New(os.Stdout, color.Green("[Sarama]"), l.LstdFlags|l.Lshortfile)
	}

	return &pubStore{
		hostname:        ctx.Hostname(),
		compress:        compress,
		idleTimeout:     idleTimeout,
		pubPoolsCapcity: poolCapcity,
		pubPools:        make(map[string]*pubPool),
		dryRun:          dryRun,
		shutdownCh:      make(chan struct{}),
	}
}

func (this *pubStore) Name() string {
	return "kafka"
}

func (this *pubStore) Start() (err error) {
	if ctx.KafkaHome() == "" {
		return fmt.Errorf("empty kafka_home in ~/.gafka.cf")
	}
	if !gio.DirExists(ctx.KafkaHome()) {
		return fmt.Errorf("kafka not installed in %s, run 'gk deploy -kfkonly'", ctx.KafkaHome())
	}

	// warmup: create pools according the current kafka topology
	for _, cluster := range meta.Default.ClusterNames() {
		this.pubPools[cluster] = newPubPool(this, cluster,
			meta.Default.BrokerList(cluster), this.pubPoolsCapcity)
	}

	this.wg.Add(1)
	go func() {
		defer this.wg.Done()

		for {
			select {
			case <-meta.Default.RefreshEvent():
				this.doRefresh()

			case <-this.shutdownCh:
				log.Trace("pub store[%s] stopped", this.Name())
				return
			}
		}
	}()

	return
}

func (this *pubStore) Stop() {
	this.pubPoolsLock.Lock()
	defer this.pubPoolsLock.Unlock()

	// close all kafka connections
	for _, pool := range this.pubPools {
		pool.Close()
	}

	close(this.shutdownCh)
	this.wg.Wait()
}

func (this *pubStore) doRefresh() {
	if time.Since(this.lastRefreshedAt) <= time.Second*5 {
		log.Warn("ignored too frequent refresh: %s", time.Since(this.lastRefreshedAt))
		return
	}

	this.pubPoolsLock.Lock()
	defer this.pubPoolsLock.Unlock()

	// pub pool
	activeClusters := make(map[string]struct{})
	for _, cluster := range meta.Default.ClusterNames() {
		activeClusters[cluster] = struct{}{}
		if _, present := this.pubPools[cluster]; !present {
			// found a new cluster
			this.pubPools[cluster] = newPubPool(this, cluster,
				meta.Default.BrokerList(cluster), this.pubPoolsCapcity)
		} else {
			this.pubPools[cluster].RefreshBrokerList(meta.Default.BrokerList(cluster))
		}
	}

	// shutdown the dead clusters
	for cluster, pool := range this.pubPools {
		if _, present := activeClusters[cluster]; !present {
			// this cluster is dead or removed forever
			pool.Close()
			delete(this.pubPools, cluster)
		}
	}

	this.lastRefreshedAt = time.Now()
}

func (this *pubStore) markPartitionsDead(topic string, deadPartitionIds map[int32]struct{}) {
	excludedPartitionsLock.Lock()
	if deadPartitionIds == nil {
		// this topic comes alive
		delete(excludedPartitions, topic)
	} else {
		excludedPartitions[topic] = deadPartitionIds
	}
	excludedPartitionsLock.Unlock()
}

func (this *pubStore) watchDeadPartitions() {
	ticker := time.NewTicker(time.Minute * 2) // TODO
	defer ticker.Stop()

	var lastTopics = make(map[string]struct{})
	for {
		select {
		case <-ticker.C:
			deadPartitions := manager.Default.DeadPartitions()
			for topic, dp := range deadPartitions {
				this.markPartitionsDead(topic, dp)

				lastTopics[topic] = struct{}{}
			}

			for lastDeadTopic := range lastTopics {
				if _, present := deadPartitions[lastDeadTopic]; !present {
					// this topic was marked dead last round, but this round it comes alive

					log.Trace("%s come alive again", lastDeadTopic)

					this.markPartitionsDead(lastDeadTopic, nil)
					delete(lastTopics, lastDeadTopic)
				}
			}

		case <-this.shutdownCh:
			return
		}
	}

}
<|endoftext|>