<commit_before>// Copyright 2015 Pikkpoiss
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"../lib/twodee"
	"io/ioutil"
	"time"
)

const (
	PxPerUnit = 32
)

type GameLayer struct {
	gameRenderer   *GameRenderer
	spriteSheet    *twodee.Spritesheet
	spriteTexture  *twodee.Texture
	app            *Application
	level          *Level
	uiState        UiState
	mouseX, mouseY float32
}

func NewGameLayer(app *Application) (layer *GameLayer, err error) {
	var (
		level   *Level
		uiState UiState
	)
	if level, err = NewLevel(); err != nil {
		return
	}
	uiState = NewNormalUiState()
	uiState.Register(level)
	layer = &GameLayer{
		app:     app,
		level:   level,
		uiState: uiState,
	}
	err = layer.Reset()
	return
}

func (l *GameLayer) Delete() {
	l.gameRenderer.Delete()
}

func (l *GameLayer) Render() {
	l.spriteTexture.Bind()
	l.gameRenderer.Draw(l.level)
	l.spriteTexture.Unbind()
}

func (l *GameLayer) HandleEvent(evt twodee.Event) bool {
	if newState := l.uiState.HandleEvent(l.level, evt); newState != nil {
		l.uiState.Unregister(l.level)
		l.uiState = newState
		l.uiState.Register(l.level)
	}

	switch event := evt.(type) {
	case *twodee.KeyEvent:
		if event.Type == twodee.Release {
			break
		}
		switch event.Code {
		case twodee.KeyM:
			if twodee.MusicIsPaused() {
				l.app.GameEventHandler.Enqueue(twodee.NewBasicGameEvent(ResumeMusic))
			} else {
				l.app.GameEventHandler.Enqueue(twodee.NewBasicGameEvent(PauseMusic))
			}
		}
	}
	return true
}

func (l *GameLayer) Reset() (err error) {
	if err = l.loadSpritesheet(); err != nil {
		return
	}
	if l.gameRenderer, err = NewGameRenderer(l.level, l.spriteSheet); err != nil {
		return
	}
	l.app.GameEventHandler.Enqueue(twodee.NewBasicGameEvent(PlayBackgroundMusic))
	return
}

func (l *GameLayer) Update(elapsed time.Duration) {
	l.level.Update(elapsed)
}

func (l *GameLayer) loadSpritesheet() (err error) {
	var (
		data []byte
	)
	if data, err = ioutil.ReadFile("resources/spriteSheet.json"); err != nil {
		return
	}
	if l.spriteSheet, err = twodee.ParseTexturePackerJSONArrayString(
		string(data),
		PxPerUnit,
	); err != nil {
		return
	}
	if l.spriteTexture, err = twodee.LoadTexture(
		"resources/"+l.spriteSheet.TexturePath,
		twodee.Nearest,
	); err != nil {
		return
	}
	return
}
<commit_msg>Remove mouse coords from gamelayer.<commit_after>// Copyright 2015 Pikkpoiss
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"../lib/twodee"
	"io/ioutil"
	"time"
)

const (
	PxPerUnit = 32
)

type GameLayer struct {
	gameRenderer  *GameRenderer
	spriteSheet   *twodee.Spritesheet
	spriteTexture *twodee.Texture
	app           *Application
	level         *Level
	uiState       UiState
}

func NewGameLayer(app *Application) (layer *GameLayer, err error) {
	var (
		level   *Level
		uiState UiState
	)
	if level, err = NewLevel(); err != nil {
		return
	}
	uiState = NewNormalUiState()
	uiState.Register(level)
	layer = &GameLayer{
		app:     app,
		level:   level,
		uiState: uiState,
	}
	err = layer.Reset()
	return
}

func (l *GameLayer) Delete() {
	l.gameRenderer.Delete()
}

func (l *GameLayer) Render() {
	l.spriteTexture.Bind()
	l.gameRenderer.Draw(l.level)
	l.spriteTexture.Unbind()
}

func (l *GameLayer) HandleEvent(evt twodee.Event) bool {
	if newState := l.uiState.HandleEvent(l.level, evt); newState != nil {
		l.uiState.Unregister(l.level)
		l.uiState = newState
		l.uiState.Register(l.level)
	}

	switch event := evt.(type) {
	case *twodee.KeyEvent:
		if event.Type == twodee.Release {
			break
		}
		switch event.Code {
		case twodee.KeyM:
			if twodee.MusicIsPaused() {
				l.app.GameEventHandler.Enqueue(twodee.NewBasicGameEvent(ResumeMusic))
			} else {
				l.app.GameEventHandler.Enqueue(twodee.NewBasicGameEvent(PauseMusic))
			}
		}
	}
	return true
}

func (l *GameLayer) Reset() (err error) {
	if err = l.loadSpritesheet(); err != nil {
		return
	}
	if l.gameRenderer, err = NewGameRenderer(l.level, l.spriteSheet); err != nil {
		return
	}
	l.app.GameEventHandler.Enqueue(twodee.NewBasicGameEvent(PlayBackgroundMusic))
	return
}

func (l *GameLayer) Update(elapsed time.Duration) {
	l.level.Update(elapsed)
}

func (l *GameLayer) loadSpritesheet() (err error) {
	var (
		data []byte
	)
	if data, err = ioutil.ReadFile("resources/spriteSheet.json"); err != nil {
		return
	}
	if l.spriteSheet, err = twodee.ParseTexturePackerJSONArrayString(
		string(data),
		PxPerUnit,
	); err != nil {
		return
	}
	if l.spriteTexture, err = twodee.LoadTexture(
		"resources/"+l.spriteSheet.TexturePath,
		twodee.Nearest,
	); err != nil {
		return
	}
	return
}
<|endoftext|>


<commit_before>package filecache

import (
	"errors"
	"log"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"

	"github.com/anacrolix/missinggo"
)

type Cache struct {
	mu       sync.Mutex
	capacity int64
	filled   int64
	items    *lruItems
	paths    map[string]ItemInfo
	root     string
}

type CacheInfo struct {
	Capacity int64
	Filled   int64
	NumItems int
}

type ItemInfo struct {
	Accessed time.Time
	Size     int64
	Path     string
}

// Calls the function for every item known to the cache. The ItemInfo should
// not be modified.
func (me *Cache) WalkItems(cb func(ItemInfo)) {
	me.mu.Lock()
	defer me.mu.Unlock()
	for e := me.items.Front(); e != nil; e = e.Next() {
		cb(e.Value().(ItemInfo))
	}
}

func (me *Cache) Info() (ret CacheInfo) {
	me.mu.Lock()
	defer me.mu.Unlock()
	ret.Capacity = me.capacity
	ret.Filled = me.filled
	ret.NumItems = len(me.paths)
	return
}

func (me *Cache) SetCapacity(capacity int64) {
	me.mu.Lock()
	defer me.mu.Unlock()
	me.capacity = capacity
}

func NewCache(root string) (ret *Cache, err error) {
	if !filepath.IsAbs(root) {
		err = errors.New("root is not an absolute filepath")
		return
	}
	ret = &Cache{
		root:     root,
		capacity: -1,
	}
	ret.mu.Lock()
	go func() {
		defer ret.mu.Unlock()
		ret.rescan()
	}()
	return
}

// An empty return path is an error.
func sanitizePath(p string) (ret string) {
	if p == "" {
		return
	}
	ret = path.Clean("/" + p)
	if ret[0] == '/' {
		ret = ret[1:]
	}
	return
}

// Leaf is a descendent of root.
func pruneEmptyDirs(root string, leaf string) (err error) {
	rootInfo, err := os.Stat(root)
	if err != nil {
		return
	}
	for {
		var leafInfo os.FileInfo
		leafInfo, err = os.Stat(leaf)
		if os.IsNotExist(err) {
			goto parent
		}
		if err != nil {
			return
		}
		if !leafInfo.IsDir() {
			return
		}
		if os.SameFile(rootInfo, leafInfo) {
			return
		}
		if os.Remove(leaf) != nil {
			return
		}
	parent:
		leaf = filepath.Dir(leaf)
	}
}

func (me *Cache) Remove(path string) (err error) {
	path = sanitizePath(path)
	me.mu.Lock()
	defer me.mu.Unlock()
	err = me.remove(path)
	return
}

var (
	ErrBadPath = errors.New("bad path")
	ErrIsDir   = errors.New("is directory")
)

func (me *Cache) OpenFile(path string, flag int) (ret *File, err error) {
	path = sanitizePath(path)
	if path == "" {
		err = ErrIsDir
		return
	}
	f, err := os.OpenFile(me.realpath(path), flag, 0644)
	if flag&os.O_CREATE != 0 && os.IsNotExist(err) {
		os.MkdirAll(me.root, 0755)
		os.MkdirAll(filepath.Dir(me.realpath(path)), 0755)
		f, err = os.OpenFile(me.realpath(path), flag, 0644)
	}
	if err != nil {
		me.pruneEmptyDirs(path)
		return
	}
	ret = &File{
		c:    me,
		path: path,
		f:    f,
	}
	me.mu.Lock()
	go func() {
		defer me.mu.Unlock()
		me.statItem(path, time.Now())
	}()
	return
}

func (me *Cache) rescan() {
	me.filled = 0
	me.items = newLRUItems()
	me.paths = make(map[string]ItemInfo)
	err := filepath.Walk(me.root, func(path string, info os.FileInfo, err error) error {
		if os.IsNotExist(err) {
			return nil
		}
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		path, err = filepath.Rel(me.root, path)
		if err != nil {
			log.Print(err)
			return nil
		}
		me.statItem(path, time.Time{})
		return nil
	})
	if err != nil {
		panic(err)
	}
}

func (me *Cache) insertItem(i ItemInfo) {
	me.items.Insert(i)
}

func (me *Cache) removeInfo(path string) (ret ItemInfo, ok bool) {
	ret, ok = me.paths[path]
	if !ok {
		return
	}
	if !me.items.Remove(ret) {
		panic(ret)
	}
	me.filled -= ret.Size
	delete(me.paths, path)
	return
}

// Triggers the item for path to be updated. If access is non-zero, set the
// item's access time to that value, otherwise deduce it appropriately.
func (me *Cache) statItem(path string, access time.Time) {
	info, ok := me.removeInfo(path)
	fi, err := os.Stat(me.realpath(path))
	if os.IsNotExist(err) {
		return
	}
	if err != nil {
		panic(err)
	}
	if !ok {
		info.Path = path
	}
	if !access.IsZero() {
		info.Accessed = access
	}
	if info.Accessed.IsZero() {
		info.Accessed = missinggo.FileInfoAccessTime(fi)
	}
	info.Size = fi.Size()
	me.filled += info.Size
	me.insertItem(info)
	me.paths[path] = info
}

func (me *Cache) realpath(path string) string {
	return filepath.Join(me.root, filepath.FromSlash(path))
}

func (me *Cache) TrimToCapacity() {
	me.mu.Lock()
	defer me.mu.Unlock()
	me.trimToCapacity()
}

func (me *Cache) pruneEmptyDirs(path string) {
	pruneEmptyDirs(me.root, me.realpath(path))
}

func (me *Cache) remove(path string) (err error) {
	err = os.Remove(me.realpath(path))
	if os.IsNotExist(err) {
		err = nil
	}
	me.pruneEmptyDirs(path)
	me.removeInfo(path)
	return
}

func (me *Cache) trimToCapacity() {
	if me.capacity < 0 {
		return
	}
	for me.filled > me.capacity {
		item := me.items.LRU()
		me.remove(item.Path)
	}
}

func (me *Cache) pathInfo(p string) ItemInfo {
	return me.paths[p]
}
<commit_msg>Code comment<commit_after>package filecache

import (
	"errors"
	"log"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"

	"github.com/anacrolix/missinggo"
)

type Cache struct {
	mu       sync.Mutex
	capacity int64
	filled   int64
	items    *lruItems
	paths    map[string]ItemInfo
	root     string
}

type CacheInfo struct {
	Capacity int64
	Filled   int64
	NumItems int
}

type ItemInfo struct {
	Accessed time.Time
	Size     int64
	Path     string
}

// Calls the function for every item known to the cache. The ItemInfo should
// not be modified.
func (me *Cache) WalkItems(cb func(ItemInfo)) {
	me.mu.Lock()
	defer me.mu.Unlock()
	for e := me.items.Front(); e != nil; e = e.Next() {
		cb(e.Value().(ItemInfo))
	}
}

func (me *Cache) Info() (ret CacheInfo) {
	me.mu.Lock()
	defer me.mu.Unlock()
	ret.Capacity = me.capacity
	ret.Filled = me.filled
	ret.NumItems = len(me.paths)
	return
}

func (me *Cache) SetCapacity(capacity int64) {
	me.mu.Lock()
	defer me.mu.Unlock()
	me.capacity = capacity
}

func NewCache(root string) (ret *Cache, err error) {
	if !filepath.IsAbs(root) {
		err = errors.New("root is not an absolute filepath")
		return
	}
	ret = &Cache{
		root:     root,
		capacity: -1, // unlimited
	}
	ret.mu.Lock()
	go func() {
		defer ret.mu.Unlock()
		ret.rescan()
	}()
	return
}

// An empty return path is an error.
func sanitizePath(p string) (ret string) {
	if p == "" {
		return
	}
	ret = path.Clean("/" + p)
	if ret[0] == '/' {
		ret = ret[1:]
	}
	return
}

// Leaf is a descendent of root.
func pruneEmptyDirs(root string, leaf string) (err error) {
	rootInfo, err := os.Stat(root)
	if err != nil {
		return
	}
	for {
		var leafInfo os.FileInfo
		leafInfo, err = os.Stat(leaf)
		if os.IsNotExist(err) {
			goto parent
		}
		if err != nil {
			return
		}
		if !leafInfo.IsDir() {
			return
		}
		if os.SameFile(rootInfo, leafInfo) {
			return
		}
		if os.Remove(leaf) != nil {
			return
		}
	parent:
		leaf = filepath.Dir(leaf)
	}
}

func (me *Cache) Remove(path string) (err error) {
	path = sanitizePath(path)
	me.mu.Lock()
	defer me.mu.Unlock()
	err = me.remove(path)
	return
}

var (
	ErrBadPath = errors.New("bad path")
	ErrIsDir   = errors.New("is directory")
)

func (me *Cache) OpenFile(path string, flag int) (ret *File, err error) {
	path = sanitizePath(path)
	if path == "" {
		err = ErrIsDir
		return
	}
	f, err := os.OpenFile(me.realpath(path), flag, 0644)
	if flag&os.O_CREATE != 0 && os.IsNotExist(err) {
		os.MkdirAll(me.root, 0755)
		os.MkdirAll(filepath.Dir(me.realpath(path)), 0755)
		f, err = os.OpenFile(me.realpath(path), flag, 0644)
	}
	if err != nil {
		me.pruneEmptyDirs(path)
		return
	}
	ret = &File{
		c:    me,
		path: path,
		f:    f,
	}
	me.mu.Lock()
	go func() {
		defer me.mu.Unlock()
		me.statItem(path, time.Now())
	}()
	return
}

func (me *Cache) rescan() {
	me.filled = 0
	me.items = newLRUItems()
	me.paths = make(map[string]ItemInfo)
	err := filepath.Walk(me.root, func(path string, info os.FileInfo, err error) error {
		if os.IsNotExist(err) {
			return nil
		}
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		path, err = filepath.Rel(me.root, path)
		if err != nil {
			log.Print(err)
			return nil
		}
		me.statItem(path, time.Time{})
		return nil
	})
	if err != nil {
		panic(err)
	}
}

func (me *Cache) insertItem(i ItemInfo) {
	me.items.Insert(i)
}

func (me *Cache) removeInfo(path string) (ret ItemInfo, ok bool) {
	ret, ok = me.paths[path]
	if !ok {
		return
	}
	if !me.items.Remove(ret) {
		panic(ret)
	}
	me.filled -= ret.Size
	delete(me.paths, path)
	return
}

// Triggers the item for path to be updated. If access is non-zero, set the
// item's access time to that value, otherwise deduce it appropriately.
func (me *Cache) statItem(path string, access time.Time) {
	info, ok := me.removeInfo(path)
	fi, err := os.Stat(me.realpath(path))
	if os.IsNotExist(err) {
		return
	}
	if err != nil {
		panic(err)
	}
	if !ok {
		info.Path = path
	}
	if !access.IsZero() {
		info.Accessed = access
	}
	if info.Accessed.IsZero() {
		info.Accessed = missinggo.FileInfoAccessTime(fi)
	}
	info.Size = fi.Size()
	me.filled += info.Size
	me.insertItem(info)
	me.paths[path] = info
}

func (me *Cache) realpath(path string) string {
	return filepath.Join(me.root, filepath.FromSlash(path))
}

func (me *Cache) TrimToCapacity() {
	me.mu.Lock()
	defer me.mu.Unlock()
	me.trimToCapacity()
}

func (me *Cache) pruneEmptyDirs(path string) {
	pruneEmptyDirs(me.root, me.realpath(path))
}

func (me *Cache) remove(path string) (err error) {
	err = os.Remove(me.realpath(path))
	if os.IsNotExist(err) {
		err = nil
	}
	me.pruneEmptyDirs(path)
	me.removeInfo(path)
	return
}

func (me *Cache) trimToCapacity() {
	if me.capacity < 0 {
		return
	}
	for me.filled > me.capacity {
		item := me.items.LRU()
		me.remove(item.Path)
	}
}

func (me *Cache) pathInfo(p string) ItemInfo {
	return me.paths[p]
}
<|endoftext|>


<commit_before>package qmd

import (
	"fmt"

	"github.com/bitly/go-nsq"
)

type QueueConfig struct {
	HostNSQDAddr string   `toml:"host_nsqd_address"`
	NSQDAddrs    []string `toml:"nsqd_addresses"`
	LookupdAddrs []string `toml:"lookupd_addresses"`
}

func (qc *QueueConfig) Clean() error {
	if len(qc.LookupdAddrs) == 0 {
		if len(qc.NSQDAddrs) == 0 {
			return fmt.Errorf("Both LookupdAddresses and NSQDAddresses are missing")
		}
	}
	return nil
}

func ConnectConsumer(qc *QueueConfig, consumer *nsq.Consumer) error {
	var err error

	// Connect consumers to NSQLookupd
	if qc.LookupdAddrs != nil && len(qc.LookupdAddrs) != 0 {
		log.Info("Connecting Consumer to the following NSQLookupds %s", qc.LookupdAddrs)
		err = consumer.ConnectToNSQLookupds(qc.LookupdAddrs)
		if err != nil {
			return err
		}
		return nil
	}
	// Connect consumers to NSQD
	if qc.NSQDAddrs != nil && len(qc.NSQDAddrs) != 0 {
		log.Info("Connecting Consumer to the following NSQDs %s", qc.NSQDAddrs)
		err = consumer.ConnectToNSQDs(qc.NSQDAddrs)
		if err != nil {
			return err
		}
		return nil
	}
	return fmt.Errorf("Couldn't connect to either NSQDs or NSQLookupds")
}
<commit_msg>Let both NSQD and LOOKUPD connect<commit_after>package qmd

import (
	"fmt"

	"github.com/bitly/go-nsq"
)

type QueueConfig struct {
	HostNSQDAddr string   `toml:"host_nsqd_address"`
	NSQDAddrs    []string `toml:"nsqd_addresses"`
	LookupdAddrs []string `toml:"lookupd_addresses"`
}

func (qc *QueueConfig) Clean() error {
	if len(qc.LookupdAddrs) == 0 {
		if len(qc.NSQDAddrs) == 0 {
			return fmt.Errorf("Both LookupdAddresses and NSQDAddresses are missing")
		}
	}
	return nil
}

func ConnectConsumer(qc *QueueConfig, consumer *nsq.Consumer) error {
	var err error

	// Connect consumers to NSQLookupd
	if qc.LookupdAddrs != nil || len(qc.LookupdAddrs) != 0 {
		log.Info("Connecting Consumer to the following NSQLookupds %s", qc.LookupdAddrs)
		err = consumer.ConnectToNSQLookupds(qc.LookupdAddrs)
		if err != nil {
			return err
		}
	}
	// Connect consumers to NSQD
	if qc.NSQDAddrs != nil || len(qc.NSQDAddrs) != 0 {
		log.Info("Connecting Consumer to the following NSQDs %s", qc.NSQDAddrs)
		err = consumer.ConnectToNSQDs(qc.NSQDAddrs)
		if err != nil {
			return err
		}
	}
	return fmt.Errorf("Couldn't connect to either NSQDs or NSQLookupds")
}
<|endoftext|>


<commit_before>package sequence

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/watermint/toolbox/app"
	"github.com/watermint/toolbox/app/app_report"
	"github.com/watermint/toolbox/domain/infra/api_auth_impl"
	"github.com/watermint/toolbox/domain/infra/api_test"
	"github.com/watermint/toolbox/domain/sequence/sq_group"
	"github.com/watermint/toolbox/domain/sequence/sq_sharedfolder"
	"github.com/watermint/toolbox/domain/service"
	"go.uber.org/zap"
	"os"
	"path/filepath"
	"time"
)

type Task interface {
}

type TeamTask interface {
	Task
	Do(svc service.Business) error
}

type InterTeamTask interface {
	Task
	Do(src, dst service.Business) error
}

type Sequence interface {
	Load(path string) error
	Run(opts ...RunOpt) error
}

type RunOpt func(opts *runOpts) *runOpts
type runOpts struct {
	retryable bool
	maxRetry  int
}

func Retryable() RunOpt {
	return func(opts *runOpts) *runOpts {
		opts.retryable = true
		return opts
	}
}
func MaxRetry(max int) RunOpt {
	return func(opts *runOpts) *runOpts {
		opts.maxRetry = max
		return opts
	}
}

func New(ec *app.ExecContext) Sequence {
	return &sequenceImpl{
		ec: ec,
	}
}

type sequenceImpl struct {
	ec      *app.ExecContext
	seqPath string
	seqName string
	runId   int
}

func (z *sequenceImpl) backlogPath(runId int) string {
	return filepath.Join(z.seqPath, fmt.Sprintf("%03d.json", runId))
}

func (z *sequenceImpl) Load(path string) error {
	l := z.ec.Log().With(zap.String("filepath", path))
	z.seqName = fmt.Sprintf("%x", time.Now().Unix())
	z.seqPath = filepath.Join(z.ec.JobsPath(), "sequence", z.seqName)
	z.runId = 0
	if err := os.MkdirAll(z.seqPath, 0755); err != nil {
		l.Error("Unable to create sequence folder", zap.Error(err))
		return err
	}

	l.Debug("Opening sequence", zap.String("path", z.seqPath))
	f, err := os.Open(path)
	if err != nil {
		l.Error("Unable to open file", zap.Error(err))
		return err
	}
	defer func() {
		if err := f.Close(); err != nil {
			l.Error("Unable to close", zap.Error(err))
		}
	}()

	backlog, err := os.Create(z.backlogPath(z.runId))
	if err != nil {
		l.Error("Unable to open backlog file", zap.Error(err))
		return err
	}
	defer func() {
		if err := backlog.Close(); err != nil {
			l.Error("Unable to close backlog", zap.Error(err))
		}
	}()

	scanner := bufio.NewScanner(f)
	var lastErr error = nil
	for scanner.Scan() {
		t := scanner.Text()
		_, _, _, err := Parse(t)
		if err != nil {
			l.Error("Unable to parse line", zap.Error(err), zap.String("line", t))
			lastErr = err
		}
		_, err = backlog.WriteString(t + "\n")
		if err != nil {
			l.Error("Unable to write into backlog", zap.Error(err))
			lastErr = err
		}
	}
	if lastErr != nil {
		return lastErr
	}

	return nil
}
func (z *sequenceImpl) runWithRunId(runId int) (numBacklog int, err error) {
	l := z.ec.Log()
	l.Info("Run", zap.Int("runId", runId))

	rep := app_report.Factory{
		Path: filepath.Join(z.seqPath, fmt.Sprintf("%03d", z.runId)),
	}
	if err := rep.Init(z.ec); err != nil {
		l.Error("Unable to prepare report", zap.Error(err))
		return 0, err
	}
	defer rep.Close()

	l.Debug("Opening backlog of runId", zap.String("backlog", z.backlogPath(z.runId)))
	f, err := os.Open(z.backlogPath(z.runId))
	if err != nil {
		l.Error("Unable to open backlog file", zap.Error(err))
		return 0, err
	}
	defer func() {
		if err := f.Close(); err != nil {
			l.Error("Unable to close", zap.Error(err))
		}
	}()

	backlog, err := os.Create(z.backlogPath(runId + 1))
	if err != nil {
		l.Error("Unable to open next backlog file", zap.Error(err))
		return 0, err
	}
	defer func() {
		if err := backlog.Close(); err != nil {
			l.Error("Unable to close backlog", zap.Error(err))
		}
	}()

	// Poison message queue
	poison, err := os.Create(filepath.Join(z.seqPath, "poison.json"))
	if err != nil {
		l.Error("Unable to open next poison file", zap.Error(err))
		return 0, err
	}
	defer func() {
		if err := poison.Close(); err != nil {
			l.Error("Unable to close poison file", zap.Error(err))
		}
	}()

	scanner := bufio.NewScanner(f)
	numBacklog = 0
	numPoison := 0
	numSuccess := 0

	defer func() {
		l.Debug("RunId result",
			zap.Int("poison", numPoison),
			zap.Int("backlog", numBacklog),
			zap.Int("success", numSuccess),
		)
	}()

	type RunReport struct {
		TaskResult string      `json:"result"`
		TaskType   string      `json:"task"`
		Peer       interface{} `json:"peer"`
		TaskParam  interface{} `json:"task_param"`
		Reason     string      `json:"reason"`
	}

	enqueuePoison := func(line string) {
		numPoison++
		if _, err := poison.WriteString(line + "\n"); err != nil {
			l.Warn("Unable to write line into poison message queue", zap.Error(err))
		}
	}
	enqueueBacklog := func(line string, name string, peer, task interface{}, err error) {
		numBacklog++
		if _, err := backlog.WriteString(line + "\n"); err != nil {
			l.Warn("Unable to write line into poison message queue", zap.Error(err))
		}
		rr := RunReport{
			TaskResult: "failure",
			TaskType:   name,
			Peer:       peer,
			TaskParam:  task,
			Reason:     err.Error(),
		}
		if err := rep.Report(rr); err != nil {
			l.Warn("Unable to write report", zap.Error(err))
		}
	}
	reportSuccess := func(line string, name string, peer, task interface{}) {
		numSuccess++
		rr := RunReport{
			TaskResult: "success",
			TaskType:   name,
			Peer:       peer,
			TaskParam:  task,
		}
		if err := rep.Report(rr); err != nil {
			l.Warn("Unable to write report", zap.Error(err))
		}
	}

	for scanner.Scan() {
		line := scanner.Text()
		peer, name, task, err := Parse(line)
		if err != nil {
			l.Error("Unable to parse line", zap.Error(err), zap.String("line", line))
		}

		switch t := task.(type) {
		case TeamTask:
			p, ok := peer.(*PeerTeam)
			if !ok {
				l.Error("Invalid peer type")
				enqueuePoison(line)
				continue
			}
			l.Debug("Authentication with peer name", zap.String("peerName", p.PeerName))
			ctxMgmt, err := api_auth_impl.Auth(z.ec, api_auth_impl.PeerName(p.PeerName), api_auth_impl.BusinessManagement())
			if err != nil {
				enqueuePoison(line)
				continue
			}
			ctxFile, err := api_auth_impl.Auth(z.ec, api_auth_impl.PeerName(p.PeerName), api_auth_impl.BusinessFile())
			if err != nil {
				enqueuePoison(line)
				continue
			}
			biz, err := service.New(ctxMgmt.NoRetryOnError(), ctxFile.NoRetryOnError())
			if err != nil {
				enqueuePoison(line)
				continue
			}
			if err := t.Do(biz); err != nil {
				enqueueBacklog(line, name, peer, t, err)
				continue
			}
			reportSuccess(line, name, peer, t)
		}
	}

	return numBacklog, nil
}

func (z *sequenceImpl) Run(opts ...RunOpt) error {
	ro := &runOpts{}
	for _, o := range opts {
		o(ro)
	}

	if !ro.retryable {
		ro.maxRetry = 1
	}
	l := z.ec.Log().With(zap.Int("maxRetry", ro.maxRetry))

	for runId := z.runId; runId < ro.maxRetry; runId++ {
		ll := l.With(zap.Int("runId", runId))
		ll.Debug("Run")
		if num, err := z.runWithRunId(runId); err != nil {
			ll.Debug("Abort retry", zap.Error(err))
			return err
		} else {
			if num == 0 {
				ll.Debug("No more backlogs")
				return nil
			}
		}
	}
	return errors.New("one or more backlogs exists")
}

type Metadata struct {
	TaskName string          `json:"task_name"`
	TaskData json.RawMessage `json:"task_data"`
	Peer     json.RawMessage `json:"peer"`
}

type PeerTeam struct {
	PeerName string `json:"peer_name"`
}
type PeerInterTeam struct {
	PeerSource      string `json:"peer_src"`
	PeerDestination string `json:"peer_dst"`
}

func Parse(jsonString string) (peer interface{}, name string, t Task, err error) {
	l := app.Root().Log()

	meta := &Metadata{}
	if err := json.Unmarshal([]byte(jsonString), meta); err != nil {
		l.Debug("Unable to unmarshal data", zap.Error(err))
		return nil, "", nil, err
	}

	switch meta.TaskName {
	case "group/add_member":
		t = &sq_group.AddMember{}
		peer = &PeerTeam{}
	case "shared_folder/add_group":
		t = &sq_sharedfolder.AddGroup{}
		peer = &PeerTeam{}
	case "shared_folder/add_user":
		t = &sq_sharedfolder.AddUser{}
		peer = &PeerTeam{}
	case "shared_folder/mount":
		t = &sq_sharedfolder.Mount{}
		peer = &PeerTeam{}
	default:
		l.Debug("Unknown task name", zap.String("taskName", meta.TaskName))
		return nil, "", nil, errors.New("unknown task name (" + meta.TaskName + ")")
	}

	if err := json.Unmarshal(meta.Peer, peer); err != nil {
		l.Debug("Unable to unmarshal peer", zap.Error(err))
		return nil, "", nil, err
	}
	if err := json.Unmarshal(meta.TaskData, t); err != nil {
		l.Debug("Unable to unmarshal task data", zap.Error(err))
		return nil, "", nil, err
	}

	return peer, meta.TaskName, t, nil
}

func DoTestTeamTask(test func(biz service.Business)) {
	peerName := api_test.TestPeerName
	ec := app.NewExecContextForTest()
	defer ec.Shutdown()
	if !api_auth_impl.IsCacheAvailable(ec, peerName) {
		return
	}

	ctxMgmt, err := api_auth_impl.Auth(ec, api_auth_impl.PeerName(peerName), api_auth_impl.BusinessManagement())
	if err != nil {
		return
	}
	ctxFile, err := api_auth_impl.Auth(ec, api_auth_impl.PeerName(peerName), api_auth_impl.BusinessFile())
	if err != nil {
		return
	}
	biz, err := service.New(ctxMgmt, ctxFile)
	if err != nil {
		return
	}
	test(biz)
}
<commit_msg>#104 : fix NPE on exit<commit_after>package sequence

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/watermint/toolbox/app"
	"github.com/watermint/toolbox/app/app_report"
	"github.com/watermint/toolbox/domain/infra/api_auth_impl"
	"github.com/watermint/toolbox/domain/infra/api_test"
	"github.com/watermint/toolbox/domain/sequence/sq_group"
	"github.com/watermint/toolbox/domain/sequence/sq_sharedfolder"
	"github.com/watermint/toolbox/domain/service"
	"go.uber.org/zap"
	"os"
	"path/filepath"
	"time"
)

type Task interface {
}

type TeamTask interface {
	Task
	Do(svc service.Business) error
}

type InterTeamTask interface {
	Task
	Do(src, dst service.Business) error
}

type Sequence interface {
	Load(path string) error
	Run(opts ...RunOpt) error
}

type RunOpt func(opts *runOpts) *runOpts
type runOpts struct {
	retryable bool
	maxRetry  int
}

func Retryable() RunOpt {
	return func(opts *runOpts) *runOpts {
		opts.retryable = true
		return opts
	}
}
func MaxRetry(max int) RunOpt {
	return func(opts *runOpts) *runOpts {
		opts.maxRetry = max
		return opts
	}
}

func New(ec *app.ExecContext) Sequence {
	return &sequenceImpl{
		ec: ec,
	}
}

type sequenceImpl struct {
	ec      *app.ExecContext
	seqPath string
	seqName string
	runId   int
}

func (z *sequenceImpl) backlogPath(runId int) string {
	return filepath.Join(z.seqPath, fmt.Sprintf("%03d.json", runId))
}

func (z *sequenceImpl) Load(path string) error {
	l := z.ec.Log().With(zap.String("filepath", path))
	z.seqName = fmt.Sprintf("%x", time.Now().Unix())
	z.seqPath = filepath.Join(z.ec.JobsPath(), "sequence", z.seqName)
	z.runId = 0
	if err := os.MkdirAll(z.seqPath, 0755); err != nil {
		l.Error("Unable to create sequence folder", zap.Error(err))
		return err
	}

	l.Debug("Opening sequence", zap.String("path", z.seqPath))
	f, err := os.Open(path)
	if err != nil {
		l.Error("Unable to open file", zap.Error(err))
		return err
	}
	defer func() {
		if err := f.Close(); err != nil {
			l.Error("Unable to close", zap.Error(err))
		}
	}()

	backlog, err := os.Create(z.backlogPath(z.runId))
	if err != nil {
		l.Error("Unable to open backlog file", zap.Error(err))
		return err
	}
	defer func() {
		if err := backlog.Close(); err != nil {
			l.Error("Unable to close backlog", zap.Error(err))
		}
	}()

	scanner := bufio.NewScanner(f)
	var lastErr error = nil
	for scanner.Scan() {
		t := scanner.Text()
		_, _, _, err := Parse(t)
		if err != nil {
			l.Error("Unable to parse line", zap.Error(err), zap.String("line", t))
			lastErr = err
		}
		_, err = backlog.WriteString(t + "\n")
		if err != nil {
			l.Error("Unable to write into backlog", zap.Error(err))
			lastErr = err
		}
	}
	if lastErr != nil {
		return lastErr
	}

	return nil
}
func (z *sequenceImpl) runWithRunId(runId int) (numBacklog int, err error) {
	l := z.ec.Log()
	l.Info("Run", zap.Int("runId", runId))

	rep := app_report.Factory{
		ExecContext: z.ec,
		Path:        filepath.Join(z.seqPath, fmt.Sprintf("%03d", z.runId)),
	}
	if err := rep.Init(z.ec); err != nil {
		l.Error("Unable to prepare report", zap.Error(err))
		return 0, err
	}
	defer rep.Close()

	l.Debug("Opening backlog of runId", zap.String("backlog", z.backlogPath(z.runId)))
	f, err := os.Open(z.backlogPath(z.runId))
	if err != nil {
		l.Error("Unable to open backlog file", zap.Error(err))
		return 0, err
	}
	defer func() {
		if err := f.Close(); err != nil {
			l.Error("Unable to close", zap.Error(err))
		}
	}()

	backlog, err := os.Create(z.backlogPath(runId + 1))
	if err != nil {
		l.Error("Unable to open next backlog file", zap.Error(err))
		return 0, err
	}
	defer func() {
		if err := backlog.Close(); err != nil {
			l.Error("Unable to close backlog", zap.Error(err))
		}
	}()

	// Poison message queue
	poison, err := os.Create(filepath.Join(z.seqPath, "poison.json"))
	if err != nil {
		l.Error("Unable to open next poison file", zap.Error(err))
		return 0, err
	}
	defer func() {
		if err := poison.Close(); err != nil {
			l.Error("Unable to close poison file", zap.Error(err))
		}
	}()

	scanner := bufio.NewScanner(f)
	numBacklog = 0
	numPoison := 0
	numSuccess := 0

	defer func() {
		l.Debug("RunId result",
			zap.Int("poison", numPoison),
			zap.Int("backlog", numBacklog),
			zap.Int("success", numSuccess),
		)
	}()

	type RunReport struct {
		TaskResult string      `json:"result"`
		TaskType   string      `json:"task"`
		Peer       interface{} `json:"peer"`
		TaskParam  interface{} `json:"task_param"`
		Reason     string      `json:"reason"`
	}

	enqueuePoison := func(line string) {
		numPoison++
		if _, err := poison.WriteString(line + "\n"); err != nil {
			l.Warn("Unable to write line into poison message queue", zap.Error(err))
		}
	}
	enqueueBacklog := func(line string, name string, peer, task interface{}, err error) {
		numBacklog++
		if _, err := backlog.WriteString(line + "\n"); err != nil {
			l.Warn("Unable to write line into poison message queue", zap.Error(err))
		}
		rr := RunReport{
			TaskResult: "failure",
			TaskType:   name,
			Peer:       peer,
			TaskParam:  task,
			Reason:     err.Error(),
		}
		if err := rep.Report(rr); err != nil {
			l.Warn("Unable to write report", zap.Error(err))
		}
	}
	reportSuccess := func(line string, name string, peer, task interface{}) {
		numSuccess++
		rr := RunReport{
			TaskResult: "success",
			TaskType:   name,
			Peer:       peer,
			TaskParam:  task,
		}
		if err := rep.Report(rr); err != nil {
			l.Warn("Unable to write report", zap.Error(err))
		}
	}

	for scanner.Scan() {
		line := scanner.Text()
		peer, name, task, err := Parse(line)
		if err != nil {
			l.Error("Unable to parse line", zap.Error(err), zap.String("line", line))
		}

		switch t := task.(type) {
		case TeamTask:
			p, ok := peer.(*PeerTeam)
			if !ok {
				l.Error("Invalid peer type")
				enqueuePoison(line)
				continue
			}
			l.Debug("Authentication with peer name", zap.String("peerName", p.PeerName))
			ctxMgmt, err := api_auth_impl.Auth(z.ec, api_auth_impl.PeerName(p.PeerName), api_auth_impl.BusinessManagement())
			if err != nil {
				enqueuePoison(line)
				continue
			}
			ctxFile, err := api_auth_impl.Auth(z.ec, api_auth_impl.PeerName(p.PeerName), api_auth_impl.BusinessFile())
			if err != nil {
				enqueuePoison(line)
				continue
			}
			biz, err := service.New(ctxMgmt.NoRetryOnError(), ctxFile.NoRetryOnError())
			if err != nil {
				enqueuePoison(line)
				continue
			}
			if err := t.Do(biz); err != nil {
				enqueueBacklog(line, name, peer, t, err)
				continue
			}
			reportSuccess(line, name, peer, t)
		}
	}

	return numBacklog, nil
}

func (z *sequenceImpl) Run(opts ...RunOpt) error {
	ro := &runOpts{}
	for _, o := range opts {
		o(ro)
	}

	if !ro.retryable {
		ro.maxRetry = 1
	}
	l := z.ec.Log().With(zap.Int("maxRetry", ro.maxRetry))

	for runId := z.runId; runId < ro.maxRetry; runId++ {
		ll := l.With(zap.Int("runId", runId))
		ll.Debug("Run")
		if num, err := z.runWithRunId(runId); err != nil {
			ll.Debug("Abort retry", zap.Error(err))
			return err
		} else {
			if num == 0 {
				ll.Debug("No more backlogs")
				return nil
			}
		}
	}
	return errors.New("one or more backlogs exists")
}

type Metadata struct {
	TaskName string          `json:"task_name"`
	TaskData json.RawMessage `json:"task_data"`
	Peer     json.RawMessage `json:"peer"`
}

type PeerTeam struct {
	PeerName string `json:"peer_name"`
}
type PeerInterTeam struct {
	PeerSource      string `json:"peer_src"`
	PeerDestination string `json:"peer_dst"`
}

func Parse(jsonString string) (peer interface{}, name string, t Task, err error) {
	l := app.Root().Log()

	meta := &Metadata{}
	if err := json.Unmarshal([]byte(jsonString), meta); err != nil {
		l.Debug("Unable to unmarshal data", zap.Error(err))
		return nil, "", nil, err
	}

	switch meta.TaskName {
	case "group/add_member":
		t = &sq_group.AddMember{}
		peer = &PeerTeam{}
	case "shared_folder/add_group":
		t = &sq_sharedfolder.AddGroup{}
		peer = &PeerTeam{}
	case "shared_folder/add_user":
		t = &sq_sharedfolder.AddUser{}
		peer = &PeerTeam{}
	case "shared_folder/mount":
		t = &sq_sharedfolder.Mount{}
		peer = &PeerTeam{}
	default:
		l.Debug("Unknown task name", zap.String("taskName", meta.TaskName))
		return nil, "", nil, errors.New("unknown task name (" + meta.TaskName + ")")
	}

	if err := json.Unmarshal(meta.Peer, peer); err != nil {
		l.Debug("Unable to unmarshal peer", zap.Error(err))
		return nil, "", nil, err
	}
	if err := json.Unmarshal(meta.TaskData, t); err != nil {
		l.Debug("Unable to unmarshal task data", zap.Error(err))
		return nil, "", nil, err
	}

	return peer, meta.TaskName, t, nil
}

func DoTestTeamTask(test func(biz service.Business)) {
	peerName := api_test.TestPeerName
	ec := app.NewExecContextForTest()
	defer ec.Shutdown()
	if !api_auth_impl.IsCacheAvailable(ec, peerName) {
		return
	}

	ctxMgmt, err := api_auth_impl.Auth(ec, api_auth_impl.PeerName(peerName), api_auth_impl.BusinessManagement())
	if err != nil {
		return
	}
	ctxFile, err := api_auth_impl.Auth(ec, api_auth_impl.PeerName(peerName), api_auth_impl.BusinessFile())
	if err != nil {
		return
	}
	biz, err := service.New(ctxMgmt, ctxFile)
	if err != nil {
		return
	}
	test(biz)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ktmpl

import "text/template"

// V1Beta2 is kubeadm config template for Kubernetes v1.17+
var V1Beta2 = template.Must(template.New("configTmpl-v1beta2").Funcs(template.FuncMap{
	"printMapInOrder": printMapInOrder,
}).Parse(`apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: {{.AdvertiseAddress}}
  bindPort: {{.APIServerPort}}
bootstrapTokens:
  - groups:
      - system:bootstrappers:kubeadm:default-node-token
    ttl: 24h0m0s
    usages:
      - signing
      - authentication
nodeRegistration:
  criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}}
  name: {{.NodeName}}
  taints: []
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
{{ if .ImageRepository}}imageRepository: {{.ImageRepository}}
{{end}}{{range .ExtraArgs}}{{.Component}}:
  extraArgs:
{{- range $i, $val := printMapInOrder .Options ": " }}
    {{$val}}
{{- end}}
{{end -}}
{{if .FeatureArgs}}featureGates:
{{range $i, $val := .FeatureArgs}}{{$i}}: {{$val}}
{{end -}}{{end -}}
certificatesDir: {{.CertDir}}
clusterName: kubernetes
apiServer:
  certSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"]
controlPlaneEndpoint: localhost:{{.APIServerPort}}
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: {{.EtcdDataDir}}
kubernetesVersion: {{.KubernetesVersion}}
networking:
  dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}}
  podSubnet: "{{.PodSubnet }}"
  serviceSubnet: {{.ServiceCIDR}}
---
apiVersion: kubelet.config.k8s.io/v1beta2
kind: KubeletConfiguration
imageGCHighThresholdPercent: 100
evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
`))<commit_msg>fix code format<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ktmpl

import "text/template"

// V1Beta2 is kubeadm config template for Kubernetes v1.17+
var V1Beta2 = template.Must(template.New("configTmpl-v1beta2").Funcs(template.FuncMap{
	"printMapInOrder": printMapInOrder,
}).Parse(`apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: {{.AdvertiseAddress}}
  bindPort: {{.APIServerPort}}
bootstrapTokens:
  - groups:
      - system:bootstrappers:kubeadm:default-node-token
    ttl: 24h0m0s
    usages:
      - signing
      - authentication
nodeRegistration:
  criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}}
  name: {{.NodeName}}
  taints: []
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
{{ if .ImageRepository}}imageRepository: {{.ImageRepository}}
{{end}}{{range .ExtraArgs}}{{.Component}}:
  extraArgs:
{{- range $i, $val := printMapInOrder .Options ": " }}
    {{$val}}
{{- end}}
{{end -}}
{{if .FeatureArgs}}featureGates:
{{range $i, $val := .FeatureArgs}}{{$i}}: {{$val}}
{{end -}}{{end -}}
certificatesDir: {{.CertDir}}
clusterName: kubernetes
apiServer:
  certSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"]
controlPlaneEndpoint: localhost:{{.APIServerPort}}
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: {{.EtcdDataDir}}
kubernetesVersion: {{.KubernetesVersion}}
networking:
  dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}}
  podSubnet: "{{.PodSubnet }}"
  serviceSubnet: {{.ServiceCIDR}}
---
apiVersion: kubelet.config.k8s.io/v1beta2
kind: KubeletConfiguration
imageGCHighThresholdPercent: 100
evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
`))
<|endoftext|>


<commit_before>package model

import (
	"fmt"
	"github.com/viant/endly"
	"github.com/viant/endly/model/criteria"
	"github.com/viant/endly/util"
	"github.com/viant/toolbox"
)

//SliceKey represents slice key
const SliceKey = "data"

//Repeater represent repeated execution
type Repeater struct {
	Extract     Extracts  //textual regexp based data extraction
	Variables   Variables //structure data based data extraction
	Repeat      int       //how many time send this request
	SleepTimeMs int       //Sleep time after request send, this only makes sense with repeat option
	Exit        string    //Exit criteria, it uses expected variable to determine repeat termination
}

//Get returns non empty instance of default instance
func (r *Repeater) Init() *Repeater {
	if r == nil {
		repeater := NewRepeater()
		r = repeater
	}
	if r.Repeat == 0 {
		r.Repeat = 1
	}
	return r
}

//EvaluateExitCriteria check is exit criteria is met.
func (r *Repeater) EvaluateExitCriteria(callerInfo string, context *endly.Context, extracted map[string]interface{}) (bool, error) {
	var state = context.State()
	var extractedState = state.Clone()
	for k, v := range extracted {
		extractedState[k] = v
	}
	canBreak, err := criteria.Evaluate(context, extractedState, r.Exit, callerInfo, false)
	if err != nil {
		return true, fmt.Errorf("failed to check %v exit criteia: %v", callerInfo, err)
	}
	if canBreak {
		return true, nil
	}
	return false, nil

}

func (r *Repeater) runOnce(service *endly.AbstractService, callerInfo string, context *endly.Context, handler func() (interface{}, error), extracted map[string]interface{}) (bool, error) {
	defer service.Sleep(context, r.SleepTimeMs)
	out, err := handler()
	if err != nil {
		return false, err
	}
	if out == nil {
		return true, nil
	}
	extractableOutput, structuredOutput := util.AsExtractable(out)

	if len(structuredOutput) > 0 {
		if len(r.Variables) > 0 {
			err = r.Variables.Apply(structuredOutput, extracted)
		}
		if extractableOutput == "" {
			extractableOutput, _ = toolbox.AsJSONText(structuredOutput)
		}
	} else {
		err = r.Variables.Apply(extracted, extracted)
	}

	err = r.Extract.Extract(context, extracted, extractableOutput)
	if err != nil {
		return false, err
	}
	if extractableOutput != "" {
		//TODO rename to output
		extracted["value"] = extractableOutput //string output is published as $value
	}
	if r.Exit != "" {
		context.Publish(NewExtractEvent(extractableOutput, structuredOutput, extracted))
		if shouldBreak, err := r.EvaluateExitCriteria(callerInfo+"ExitEvaluation", context, extracted); shouldBreak || err != nil {
			return !shouldBreak, err
		}
	}
	return true, nil
}

//Run repeats x times supplied handler
func (r *Repeater) Run(service *endly.AbstractService, callerInfo string, context *endly.Context, handler func() (interface{}, error), extracted map[string]interface{}) error {
	for i := 0; i < r.Repeat; i++ {
		shouldContinue, err := r.runOnce(service, callerInfo, context, handler, extracted)
		if err != nil || !shouldContinue {
			return err
		}
	}
	return nil
}

//NewRepeater creates a new repeatable struct
func NewRepeater() *Repeater {
	return &Repeater{
		Repeat: 1,
	}
}
<commit_msg>moved sleep time<commit_after>package model

import (
	"fmt"
	"github.com/viant/endly"
	"github.com/viant/endly/model/criteria"
	"github.com/viant/endly/util"
	"github.com/viant/toolbox"
)

//SliceKey represents slice key
const SliceKey = "data"

//Repeater represent repeated execution
type Repeater struct {
	Extract     Extracts  //textual regexp based data extraction
	Variables   Variables //structure data based data extraction
	Repeat      int       //how many time send this request
	SleepTimeMs int       //Sleep time after request send, this only makes sense with repeat option
	Exit        string    //Exit criteria, it uses expected variable to determine repeat termination
}

//Get returns non empty instance of default instance
func (r *Repeater) Init() *Repeater {
	if r == nil {
		repeater := NewRepeater()
		r = repeater
	}
	if r.Repeat == 0 {
		r.Repeat = 1
	}
	return r
}

//EvaluateExitCriteria check is exit criteria is met.
func (r *Repeater) EvaluateExitCriteria(callerInfo string, context *endly.Context, extracted map[string]interface{}) (bool, error) {
	var state = context.State()
	var extractedState = state.Clone()
	for k, v := range extracted {
		extractedState[k] = v
	}
	canBreak, err := criteria.Evaluate(context, extractedState, r.Exit, callerInfo, false)
	if err != nil {
		return true, fmt.Errorf("failed to check %v exit criteia: %v", callerInfo, err)
	}
	if canBreak {
		return true, nil
	}
	return false, nil

}

func (r *Repeater) runOnce(service *endly.AbstractService, callerInfo string, context *endly.Context, handler func() (interface{}, error), extracted map[string]interface{}) (bool, error) {
	out, err := handler()
	if err != nil {
		return false, err
	}
	if out == nil {
		return true, nil
	}
	extractableOutput, structuredOutput := util.AsExtractable(out)

	if len(structuredOutput) > 0 {
		if len(r.Variables) > 0 {
			err = r.Variables.Apply(structuredOutput, extracted)
		}
		if extractableOutput == "" {
			extractableOutput, _ = toolbox.AsJSONText(structuredOutput)
		}
	} else {
		err = r.Variables.Apply(extracted, extracted)
	}

	err = r.Extract.Extract(context, extracted, extractableOutput)
	if err != nil {
		return false, err
	}
	if extractableOutput != "" {
		//TODO rename to output
		extracted["value"] = extractableOutput //string output is published as $value
	}
	if r.Exit != "" {
		context.Publish(NewExtractEvent(extractableOutput, structuredOutput, extracted))
		if shouldBreak, err := r.EvaluateExitCriteria(callerInfo+"ExitEvaluation", context, extracted); shouldBreak || err != nil {
			return !shouldBreak, err
		}
	}
	return true, nil
}

//Run repeats x times supplied handler
func (r *Repeater) Run(service *endly.AbstractService, callerInfo string, context *endly.Context, handler func() (interface{}, error), extracted map[string]interface{}) error {
	for i := 0; i < r.Repeat; i++ {
		shouldContinue, err := r.runOnce(service, callerInfo, context, handler, extracted)
		if err != nil || !shouldContinue {
			return err
		}
		service.Sleep(context, r.SleepTimeMs)
	}
	return nil
}

//NewRepeater creates a new repeatable struct
func NewRepeater() *Repeater {
	return &Repeater{
		Repeat: 1,
	}
}
<|endoftext|>


<commit_before>package ble

import (
	"github.com/paypal/gatt"
	"log"
	"sync"
	"time"
)

const (
	pwmService  = "000015231212efde1523785feabcd123"
	pwmLedChar  = "000015251212efde1523785feabcd123"
	pwmTempChar = "000015261212efde1523785feabcd123"
	pwmFanChar  = "000015241212efde1523785feabcd123"
)

type bleChannel struct {
	device           gatt.Device
	connectedPeriph  map[string]blePeriph
	knownPeriph      map[string]bool
	ignoredPeriph    map[string]bool
	connectingPeriph map[string]gatt.Peripheral
	idleTicker       *time.Ticker

	lock sync.Mutex
}

type blePeriph struct {
	gp       gatt.Peripheral
	ledChar  *gatt.Characteristic
	fanChar  *gatt.Characteristic
	tempChar *gatt.Characteristic
}

type BLEChannel interface {
}

var DefaultClientOptions = []gatt.Option{
	gatt.LnxMaxConnections(10),
	gatt.LnxDeviceID(-1, true),
}

func NewBLEChannel() BLEChannel {
	d, err := gatt.NewDevice(DefaultClientOptions...)
	if err != nil {
		log.Fatalf("Failed to open the bluetooth HCI device: %s\n", err)
		return nil
	}

	ble := &bleChannel{device: d,
		connectedPeriph:  make(map[string]blePeriph),
		knownPeriph:      make(map[string]bool),
		ignoredPeriph:    make(map[string]bool),
		connectingPeriph: make(map[string]gatt.Peripheral),
		idleTicker:       time.NewTicker(500 * time.Millisecond),
	}

	d.Handle(
		gatt.PeripheralDiscovered(ble.onPeriphDiscovered),
		gatt.PeripheralConnected(ble.onPeriphConnected),
		gatt.PeripheralDisconnected(ble.onPeriphDisconnected),
	)

	d.Init(ble.onStateChanged)

	go func() {
		var i = 0.0

		for _ = range ble.idleTicker.C {
			for _, p := range ble.connectedPeriph {
				i += 0.1
				ble.lock.Lock()
				for c := 0; c <= 8; c++ {
					value := int(i) % 0x80
					err := p.gp.WriteCharacteristic(p.ledChar, []byte{byte(c), byte(value)}, true)
					if err != nil {
						log.Println(err)
					}

				}
				ble.lock.Unlock()
			}
		}
	}()

	return &bleChannel{device: d}
}

// Force Gatt to enter scanning mode
func (ble *bleChannel) onStateChanged(d gatt.Device, s gatt.State) {
	log.Println("State:", s)
	switch s {
	case gatt.StatePoweredOn:
		log.Println("Scanning...")
		d.Scan([]gatt.UUID{}, true)
		return
	default:
		log.Println("Stop scanning")
		d.StopScanning()
	}
}

func (ble *bleChannel) onPeriphConnected(p gatt.Peripheral, err error) {
	ble.lock.Lock()
	defer ble.lock.Unlock()

	log.Println("Connected ", p.ID())
	// Remove from the connecting pool
	delete(ble.connectingPeriph, p.ID())

	bp := blePeriph{gp: p}

	// Discovery services
	ss, err := p.DiscoverServices(nil)
	if err != nil {
		log.Printf("Failed to discover services, err: %s\n", err)
		return
	}

	for _, s := range ss {
		msg := "Service: " + s.UUID().String()
		if len(s.Name()) > 0 {
			msg += " (" + s.Name() + ")"
		}
		log.Println(msg)

		// Discovery characteristics
		cs, err := p.DiscoverCharacteristics(nil, s)
		if err != nil {
			log.Printf("Failed to discover characteristics, err: %s\n", err)
			continue
		}

		for _, c := range cs {
			msg := "  Characteristic  " + c.UUID().String()

			// Grab and store the three characteristics we
			// case about by matching by UUID
			switch c.UUID().String() {
			case pwmLedChar:
				bp.ledChar = c
			case pwmTempChar:
				bp.tempChar = c
			case pwmFanChar:
				bp.fanChar = c
			}

			if len(c.Name()) > 0 {
				msg += " (" + c.Name() + ")"
			}
			msg += "\n    properties    " + c.Properties().String()
			log.Println(msg)

			// Read the characteristic, if possible.
			if (c.Properties() & gatt.CharRead) != 0 {
				b, err := p.ReadCharacteristic(c)
				if err != nil {
					log.Printf("Failed to read characteristic, err: %s\n", err)
					continue
				}
				log.Printf("    value         %x | %q\n", b, b)
			}

			// Discovery descriptors
			ds, err := p.DiscoverDescriptors(nil, c)
			if err != nil {
				log.Printf("Failed to discover descriptors, err: %s\n", err)
				continue
			}

			for _, d := range ds {
				msg := "  Descriptor      " + d.UUID().String()
				if len(d.Name()) > 0 {
					msg += " (" + d.Name() + ")"
				}
				log.Println(msg)

				// Read descriptor (could fail, if it's not readable)
				b, err := p.ReadDescriptor(d)
				if err != nil {
					log.Printf("Failed to read descriptor, err: %s\n", err)
					continue
				}
				log.Printf("    value         %x | %q\n", b, b)
			}

			// Subscribe the characteristic, if possible.
			if (c.Properties() & (gatt.CharNotify | gatt.CharIndicate)) != 0 {
				f := func(c *gatt.Characteristic, b []byte, err error) {
					log.Printf("notified: % X | %q\n", b, b)
				}
				if err := p.SetNotifyValue(c, f); err != nil {
					log.Printf("Failed to subscribe characteristic, err: %s\n", err)
					continue
				}
			}

		}
	}

	ble.connectedPeriph[p.ID()] = bp
}

func (ble *bleChannel) onPeriphDiscovered(p gatt.Peripheral, a *gatt.Advertisement, rssi int) {
	ble.lock.Lock()
	defer ble.lock.Unlock()

	if _, ok := ble.ignoredPeriph[p.ID()]; ok {
		return
	}

	log.Printf("Peripheral ID:%s, NAME:(%s)\n", p.ID(), p.Name())
	log.Println("  Local Name        =", a.LocalName)
	log.Println("  TX Power Level    =", a.TxPowerLevel)
	log.Println("  Manufacturer Data =", a.ManufacturerData)
	log.Println("  Service Data      =", a.ServiceData)
	log.Println("")

	if p.Name() != "LEDBrick-PWM" {
		ble.ignoredPeriph[p.ID()] = true
		log.Println("Ignoring this device.")
		return
	}

	ble.knownPeriph[p.ID()] = true
	if _, ok := ble.connectingPeriph[p.ID()]; ok {
		log.Printf("Peripheral is in connecting state: %s", p.ID())
	}

	log.Printf("Connecting to %s", p.ID())
	ble.connectingPeriph[p.ID()] = p
	go func() {
		time.Sleep(30 * time.Second)
		if _, ok := ble.connectedPeriph[p.ID()]; ok {
			return
		} else {
			delete(ble.connectingPeriph, p.ID())
			log.Printf("Haven't heard back about connection to  %s", p.ID())
		}
	}()
	p.Device().Connect(p)
}

func (ble *bleChannel) onPeriphDisconnected(p gatt.Peripheral, err error) {
	ble.lock.Lock()
	defer ble.lock.Unlock()

	log.Println("Disconnected ", p.ID())
	delete(ble.connectedPeriph, p.ID())
	p.Device().CancelConnection(p)
}
<commit_msg>Capture temp and fan speed, log these sensibly<commit_after>package ble

import (
	"github.com/paypal/gatt"
	"log"
	"sync"
	"time"
)

const (
	pwmService  = "000015231212efde1523785feabcd123"
	pwmLedChar  = "000015251212efde1523785feabcd123"
	pwmTempChar = "000015261212efde1523785feabcd123"
	pwmFanChar  = "000015241212efde1523785feabcd123"
)

type bleChannel struct {
	device           gatt.Device
	connectedPeriph  map[string]blePeriph
	knownPeriph      map[string]bool
	ignoredPeriph    map[string]bool
	connectingPeriph map[string]gatt.Peripheral
	idleTicker       *time.Ticker

	lock sync.Mutex
}

type blePeriph struct {
	gp       gatt.Peripheral
	ledChar  *gatt.Characteristic
	fanChar  *gatt.Characteristic
	tempChar *gatt.Characteristic

	temperature int
	fanRpm int
}

type BLEChannel interface {
}

var DefaultClientOptions = []gatt.Option{
	gatt.LnxMaxConnections(10),
	gatt.LnxDeviceID(-1, true),
}

func NewBLEChannel() BLEChannel {
	d, err := gatt.NewDevice(DefaultClientOptions...)
	if err != nil {
		log.Fatalf("Failed to open the bluetooth HCI device: %s\n", err)
		return nil
	}

	ble := &bleChannel{device: d,
		connectedPeriph:  make(map[string]blePeriph),
		knownPeriph:      make(map[string]bool),
		ignoredPeriph:    make(map[string]bool),
		connectingPeriph: make(map[string]gatt.Peripheral),
		idleTicker:       time.NewTicker(500 * time.Millisecond),
	}

	d.Handle(
		gatt.PeripheralDiscovered(ble.onPeriphDiscovered),
		gatt.PeripheralConnected(ble.onPeriphConnected),
		gatt.PeripheralDisconnected(ble.onPeriphDisconnected),
	)

	d.Init(ble.onStateChanged)

	go func() {
		var i = 0.0

		for _ = range ble.idleTicker.C {
			for _, p := range ble.connectedPeriph {
				i += 0.1
				ble.lock.Lock()
				for c := 0; c <= 8; c++ {
					value := int(i) % 0x80
					err := p.gp.WriteCharacteristic(p.ledChar, []byte{byte(c), byte(value)}, true)
					if err != nil {
						log.Println(err)
					}

				}
				ble.lock.Unlock()
			}
		}
	}()

	return &bleChannel{device: d}
}

// Force Gatt to enter scanning mode
func (ble *bleChannel) onStateChanged(d gatt.Device, s gatt.State) {
	log.Println("State:", s)
	switch s {
	case gatt.StatePoweredOn:
		log.Println("Scanning...")
		d.Scan([]gatt.UUID{}, true)
		return
	default:
		log.Println("Stop scanning")
		d.StopScanning()
	}
}

func (ble *bleChannel) onPeriphConnected(p gatt.Peripheral, err error) {
	ble.lock.Lock()
	defer ble.lock.Unlock()

	log.Println("Connected ", p.ID())
	// Remove from the connecting pool
	delete(ble.connectingPeriph, p.ID())

	bp := blePeriph{gp: p}

	// Discovery services
	ss, err := p.DiscoverServices(nil)
	if err != nil {
		log.Printf("Failed to discover services, err: %s\n", err)
		return
	}

	for _, s := range ss {
		msg := "Service: " + s.UUID().String()
		if len(s.Name()) > 0 {
			msg += " (" + s.Name() + ")"
		}
		log.Println(msg)

		// Discovery characteristics
		cs, err := p.DiscoverCharacteristics(nil, s)
		if err != nil {
			log.Printf("Failed to discover characteristics, err: %s\n", err)
			continue
		}

		for _, c := range cs {
			msg := "  Characteristic  " + c.UUID().String()

			// Grab and store the three characteristics we
			// case about by matching by UUID
			switch c.UUID().String() {
			case pwmLedChar:
				bp.ledChar = c
			case pwmTempChar:
				bp.tempChar = c
			case pwmFanChar:
				bp.fanChar = c
			}

			if len(c.Name()) > 0 {
				msg += " (" + c.Name() + ")"
			}
			msg += "\n    properties    " + c.Properties().String()
			log.Println(msg)

			// Read the characteristic, if possible.
			if (c.Properties() & gatt.CharRead) != 0 {
				b, err := p.ReadCharacteristic(c)
				if err != nil {
					log.Printf("Failed to read characteristic, err: %s\n", err)
					continue
				}
				log.Printf("    value         %x | %q\n", b, b)
			}

			// Discovery descriptors
			ds, err := p.DiscoverDescriptors(nil, c)
			if err != nil {
				log.Printf("Failed to discover descriptors, err: %s\n", err)
				continue
			}

			for _, d := range ds {
				msg := "  Descriptor      " + d.UUID().String()
				if len(d.Name()) > 0 {
					msg += " (" + d.Name() + ")"
				}
				log.Println(msg)

				// Read descriptor (could fail, if it's not readable)
				b, err := p.ReadDescriptor(d)
				if err != nil {
					log.Printf("Failed to read descriptor, err: %s\n", err)
					continue
				}
				log.Printf("    value         %x | %q\n", b, b)
			}

			// Subscribe the characteristic, if possible.
			if (c.Properties() & (gatt.CharNotify | gatt.CharIndicate)) != 0 {
				f := func(c *gatt.Characteristic, b []byte, err error) {
					//log.Printf("%s: % X | %q\n", p.ID(), b, b)
					switch c.UUID().String() {
					case pwmTempChar:
						bp.temperature = int(b[0])
						log.Printf("%s: temperature: %d C", p.ID(), bp.temperature)
					case pwmFanChar:
						bp.fanRpm = int(b[0]) | (int(b[1]) << 8)
						log.Printf("%s: fan speed: %d rpm", p.ID(), bp.fanRpm)
					default:
						log.Printf("unknown notification from %s", p.ID())
					}
				}
				if err := p.SetNotifyValue(c, f); err != nil {
					log.Printf("Failed to subscribe characteristic, err: %s\n", err)
					continue
				}
			}

		}
	}

	ble.connectedPeriph[p.ID()] = bp
}

func (ble *bleChannel) onPeriphDiscovered(p gatt.Peripheral, a *gatt.Advertisement, rssi int) {
	ble.lock.Lock()
	defer ble.lock.Unlock()

	if _, ok := ble.ignoredPeriph[p.ID()]; ok {
		return
	}

	log.Printf("Peripheral ID:%s, NAME:(%s)\n", p.ID(), p.Name())
	log.Println("  Local Name        =", a.LocalName)
	log.Println("  TX Power Level    =", a.TxPowerLevel)
	log.Println("  Manufacturer Data =", a.ManufacturerData)
	log.Println("  Service Data      =", a.ServiceData)
	log.Println("")

	if p.Name() != "LEDBrick-PWM" {
		ble.ignoredPeriph[p.ID()] = true
		log.Println("Ignoring this device.")
		return
	}

	ble.knownPeriph[p.ID()] = true
	if _, ok := ble.connectingPeriph[p.ID()]; ok {
		log.Printf("Peripheral is in connecting state: %s", p.ID())
	}

	log.Printf("Connecting to %s", p.ID())
	ble.connectingPeriph[p.ID()] = p
	go func() {
		time.Sleep(30 * time.Second)
		if _, ok := ble.connectedPeriph[p.ID()]; ok {
			return
		} else {
			delete(ble.connectingPeriph, p.ID())
			log.Printf("Haven't heard back about connection to  %s", p.ID())
		}
	}()
	p.Device().Connect(p)
}

func (ble *bleChannel) onPeriphDisconnected(p gatt.Peripheral, err error) {
	ble.lock.Lock()
	defer ble.lock.Unlock()

	log.Println("Disconnected ", p.ID())
	delete(ble.connectedPeriph, p.ID())
	p.Device().CancelConnection(p)
}
<|endoftext|>


<commit_before>// Copyright 2015 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dispatch

import (
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/prometheus/common/model"

	"github.com/prometheus/alertmanager/config"
	"github.com/prometheus/alertmanager/pkg/labels"
)

// DefaultRouteOpts are the defaulting routing options which apply
// to the root route of a routing tree.
var DefaultRouteOpts = RouteOpts{
	GroupWait:      30 * time.Second,
	GroupInterval:  5 * time.Minute,
	RepeatInterval: 4 * time.Hour,
	GroupBy:        map[model.LabelName]struct{}{},
	GroupByAll:     false,
}

// A Route is a node that contains definitions of how to handle alerts.
type Route struct {
	parent *Route

	// The configuration parameters for matches of this route.
	RouteOpts RouteOpts

	// Matchers an alert has to fulfill to match
	// this route.
	Matchers labels.Matchers

	// If true, an alert matches further routes on the same level.
	Continue bool

	// Children routes of this route.
	Routes []*Route
}

// NewRoute returns a new route.
func NewRoute(cr *config.Route, parent *Route) *Route {
	// Create default and overwrite with configured settings.
	opts := DefaultRouteOpts
	if parent != nil {
		opts = parent.RouteOpts
	}

	if cr.Receiver != "" {
		opts.Receiver = cr.Receiver
	}
	if cr.GroupBy != nil {
		opts.GroupBy = map[model.LabelName]struct{}{}
		for _, ln := range cr.GroupBy {
			opts.GroupBy[ln] = struct{}{}
		}
		opts.GroupByAll = false
	} else {
		if cr.GroupByAll {
			opts.GroupByAll = cr.GroupByAll
		}
	}

	if cr.GroupWait != nil {
		opts.GroupWait = time.Duration(*cr.GroupWait)
	}
	if cr.GroupInterval != nil {
		opts.GroupInterval = time.Duration(*cr.GroupInterval)
	}
	if cr.RepeatInterval != nil {
		opts.RepeatInterval = time.Duration(*cr.RepeatInterval)
	}

	// Build matchers.
	var matchers labels.Matchers

	// cr.Match will be deprecated. This for loop appends matchers.
	for ln, lv := range cr.Match {
		matcher, err := labels.NewMatcher(labels.MatchEqual, ln, lv)
		if err != nil {
			// This error must not happen because the config already validates the yaml.
			panic(err)
		}
		matchers = append(matchers, matcher)
	}

	// cr.MatchRE will be deprecated. This for loop appends regex matchers.
	for ln, lv := range cr.MatchRE {
		matcher, err := labels.NewMatcher(labels.MatchRegexp, ln, lv.String())
		if err != nil {
			// This error must not happen because the config already validates the yaml.
			panic(err)
		}
		matchers = append(matchers, matcher)
	}

	// We append the new-style matchers. This can be simplified once the deprecated matcher syntax is removed.
	matchers = append(matchers, cr.Matchers...)

	sort.Sort(matchers)

	route := &Route{
		parent:    parent,
		RouteOpts: opts,
		Matchers:  matchers,
		Continue:  cr.Continue,
	}

	route.Routes = NewRoutes(cr.Routes, route)

	return route
}

// NewRoutes returns a slice of routes.
func NewRoutes(croutes []*config.Route, parent *Route) []*Route {
	res := []*Route{}
	for _, cr := range croutes {
		res = append(res, NewRoute(cr, parent))
	}
	return res
}

// Match does a depth-first left-to-right search through the route tree
// and returns the matching routing nodes.
func (r *Route) Match(lset model.LabelSet) []*Route {
	if !r.Matchers.Matches(lset) {
		return nil
	}

	var all []*Route

	for _, cr := range r.Routes {
		matches := cr.Match(lset)

		all = append(all, matches...)

		if matches != nil && !cr.Continue {
			break
		}
	}

	// If no child nodes were matches, the current node itself is a match.
	if len(all) == 0 {
		all = append(all, r)
	}

	return all
}

// Key returns a key for the route. It does not uniquely identify the route in general.
func (r *Route) Key() string {
	b := strings.Builder{}

	if r.parent != nil {
		b.WriteString(r.parent.Key())
		b.WriteRune('/')
	}
	b.WriteString(r.Matchers.String())
	return b.String()
}

// Walk traverses the route tree in depth-first order.
func (r *Route) Walk(visit func(*Route)) {
	visit(r)
	if r.Routes == nil {
		return
	}
	for i := range r.Routes {
		r.Routes[i].Walk(visit)
	}
}

// RouteOpts holds various routing options necessary for processing alerts
// that match a given route.
type RouteOpts struct {
	// The identifier of the associated notification configuration.
	Receiver string

	// What labels to group alerts by for notifications.
	GroupBy map[model.LabelName]struct{}

	// Use all alert labels to group.
	GroupByAll bool

	// How long to wait to group matching alerts before sending
	// a notification.
	GroupWait      time.Duration
	GroupInterval  time.Duration
	RepeatInterval time.Duration
}

func (ro *RouteOpts) String() string {
	var labels []model.LabelName
	for ln := range ro.GroupBy {
		labels = append(labels, ln)
	}
	return fmt.Sprintf("<RouteOpts send_to:%q group_by:%q group_by_all:%t timers:%q|%q>",
		ro.Receiver, labels, ro.GroupByAll, ro.GroupWait, ro.GroupInterval)
}

// MarshalJSON returns a JSON representation of the routing options.
func (ro *RouteOpts) MarshalJSON() ([]byte, error) {
	v := struct {
		Receiver       string           `json:"receiver"`
		GroupBy        model.LabelNames `json:"groupBy"`
		GroupByAll     bool             `json:"groupByAll"`
		GroupWait      time.Duration    `json:"groupWait"`
		GroupInterval  time.Duration    `json:"groupInterval"`
		RepeatInterval time.Duration    `json:"repeatInterval"`
	}{
		Receiver:       ro.Receiver,
		GroupByAll:     ro.GroupByAll,
		GroupWait:      ro.GroupWait,
		GroupInterval:  ro.GroupInterval,
		RepeatInterval: ro.RepeatInterval,
	}
	for ln := range ro.GroupBy {
		v.GroupBy = append(v.GroupBy, ln)
	}

	return json.Marshal(&v)
}
<commit_msg>Allow routes to reference time intervals<commit_after>// Copyright 2015 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dispatch

import (
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/prometheus/common/model"

	"github.com/prometheus/alertmanager/config"
	"github.com/prometheus/alertmanager/pkg/labels"
)

// DefaultRouteOpts are the defaulting routing options which apply
// to the root route of a routing tree.
var DefaultRouteOpts = RouteOpts{
	GroupWait:      30 * time.Second,
	GroupInterval:  5 * time.Minute,
	RepeatInterval: 4 * time.Hour,
	GroupBy:        map[model.LabelName]struct{}{},
	GroupByAll:     false,
	MuteTimes:      []string{},
}

// A Route is a node that contains definitions of how to handle alerts.
type Route struct {
	parent *Route

	// The configuration parameters for matches of this route.
	RouteOpts RouteOpts

	// Matchers an alert has to fulfill to match
	// this route.
	Matchers labels.Matchers

	// If true, an alert matches further routes on the same level.
	Continue bool

	// Children routes of this route.
	Routes []*Route
}

// NewRoute returns a new route.
func NewRoute(cr *config.Route, parent *Route) *Route {
	// Create default and overwrite with configured settings.
	opts := DefaultRouteOpts
	if parent != nil {
		opts = parent.RouteOpts
	}

	if cr.Receiver != "" {
		opts.Receiver = cr.Receiver
	}

	if cr.GroupBy != nil {
		opts.GroupBy = map[model.LabelName]struct{}{}
		for _, ln := range cr.GroupBy {
			opts.GroupBy[ln] = struct{}{}
		}
		opts.GroupByAll = false
	} else {
		if cr.GroupByAll {
			opts.GroupByAll = cr.GroupByAll
		}
	}

	if cr.GroupWait != nil {
		opts.GroupWait = time.Duration(*cr.GroupWait)
	}
	if cr.GroupInterval != nil {
		opts.GroupInterval = time.Duration(*cr.GroupInterval)
	}
	if cr.RepeatInterval != nil {
		opts.RepeatInterval = time.Duration(*cr.RepeatInterval)
	}

	// Build matchers.
	var matchers labels.Matchers

	// cr.Match will be deprecated. This for loop appends matchers.
	for ln, lv := range cr.Match {
		matcher, err := labels.NewMatcher(labels.MatchEqual, ln, lv)
		if err != nil {
			// This error must not happen because the config already validates the yaml.
			panic(err)
		}
		matchers = append(matchers, matcher)
	}

	// cr.MatchRE will be deprecated. This for loop appends regex matchers.
	for ln, lv := range cr.MatchRE {
		matcher, err := labels.NewMatcher(labels.MatchRegexp, ln, lv.String())
		if err != nil {
			// This error must not happen because the config already validates the yaml.
			panic(err)
		}
		matchers = append(matchers, matcher)
	}

	// We append the new-style matchers. This can be simplified once the deprecated matcher syntax is removed.
	matchers = append(matchers, cr.Matchers...)

	sort.Sort(matchers)

	opts.MuteTimes = cr.MuteTimes

	route := &Route{
		parent:    parent,
		RouteOpts: opts,
		Matchers:  matchers,
		Continue:  cr.Continue,
	}

	route.Routes = NewRoutes(cr.Routes, route)

	return route
}

// NewRoutes returns a slice of routes.
func NewRoutes(croutes []*config.Route, parent *Route) []*Route {
	res := []*Route{}
	for _, cr := range croutes {
		res = append(res, NewRoute(cr, parent))
	}
	return res
}

// Match does a depth-first left-to-right search through the route tree
// and returns the matching routing nodes.
func (r *Route) Match(lset model.LabelSet) []*Route {
	if !r.Matchers.Matches(lset) {
		return nil
	}

	var all []*Route

	for _, cr := range r.Routes {
		matches := cr.Match(lset)

		all = append(all, matches...)

		if matches != nil && !cr.Continue {
			break
		}
	}

	// If no child nodes were matches, the current node itself is a match.
	if len(all) == 0 {
		all = append(all, r)
	}

	return all
}

// Key returns a key for the route. It does not uniquely identify the route in general.
func (r *Route) Key() string {
	b := strings.Builder{}

	if r.parent != nil {
		b.WriteString(r.parent.Key())
		b.WriteRune('/')
	}
	b.WriteString(r.Matchers.String())
	return b.String()
}

// Walk traverses the route tree in depth-first order.
func (r *Route) Walk(visit func(*Route)) {
	visit(r)
	if r.Routes == nil {
		return
	}
	for i := range r.Routes {
		r.Routes[i].Walk(visit)
	}
}

// RouteOpts holds various routing options necessary for processing alerts
// that match a given route.
type RouteOpts struct {
	// The identifier of the associated notification configuration.
	Receiver string

	// What labels to group alerts by for notifications.
	GroupBy map[model.LabelName]struct{}

	// Use all alert labels to group.
	GroupByAll bool

	// How long to wait to group matching alerts before sending
	// a notification.
	GroupWait      time.Duration
	GroupInterval  time.Duration
	RepeatInterval time.Duration

	// A list of time intervals for which the route is muted
	MuteTimes []string
}

func (ro *RouteOpts) String() string {
	var labels []model.LabelName
	for ln := range ro.GroupBy {
		labels = append(labels, ln)
	}
	return fmt.Sprintf("<RouteOpts send_to:%q group_by:%q group_by_all:%t timers:%q|%q>",
		ro.Receiver, labels, ro.GroupByAll, ro.GroupWait, ro.GroupInterval)
}

// MarshalJSON returns a JSON representation of the routing options.
func (ro *RouteOpts) MarshalJSON() ([]byte, error) {
	v := struct {
		Receiver       string           `json:"receiver"`
		GroupBy        model.LabelNames `json:"groupBy"`
		GroupByAll     bool             `json:"groupByAll"`
		GroupWait      time.Duration    `json:"groupWait"`
		GroupInterval  time.Duration    `json:"groupInterval"`
		RepeatInterval time.Duration    `json:"repeatInterval"`
	}{
		Receiver:       ro.Receiver,
		GroupByAll:     ro.GroupByAll,
		GroupWait:      ro.GroupWait,
		GroupInterval:  ro.GroupInterval,
		RepeatInterval: ro.RepeatInterval,
	}
	for ln := range ro.GroupBy {
		v.GroupBy = append(v.GroupBy, ln)
	}

	return json.Marshal(&v)
}
<|endoftext|>


<commit_before>package udp

import (
	"testing"
	"time"

	"github.com/hsheth2/gonet/ipv4"
)

const rwport = 20102

func TestReadWriteLocal(t *testing.T) {
	readWriteTest(t, ipv4.LoopbackIPAddress, 0)
}

func TestReadWriteLocalFragmentation(t *testing.T) {
	readWriteTest(t, ipv4.LoopbackIPAddress, 10)
}

func TestReadWriteExternal(t *testing.T) {
	t.Skip("External tests actually don't work")
	readWriteTest(t, ipv4.ExternalIPAddress, 0)
}

func readWriteTest(t *testing.T, ip *ipv4.Address, exp int) {
	success := make(chan bool, 1)
	wrote := make(chan bool, 1)

	r, err := NewReader(rwport, ip)
	if err != nil {
		t.Fatal(err)
	}
	defer r.Close()

	data := []byte{'h', 'e', 'l', 'l', 'o'}
	for i := 0; i < exp; i++ {
		data = append(data, data...)
	}

	go func(data []byte) {
		w, err := NewWriter(20000, rwport, ip)
		if err != nil {
			t.Fatal(err)
		}

		_, err = w.Write(data)
		if err != nil {
			t.Fatal(err)
		} else {
			t.Log("Wrote the data:", data)
		}

		w.Close()

		wrote <- true
	}(data)

	go func(data []byte) {
		//time.Sleep(10*time.Second)
		p, err := r.Read(maxUDPPacketLength)
		if err != nil {
			t.Fatal(err)
		}
		t.Log("Output:", string(p))

		if string(p) == string(data) {
			t.Log("Got correct output:", p)
			success <- true
		} else {
			t.Error("Got Wrong Output:", p)
		}
	}(data)

	select {
	case <-success:
		<- wrote
		t.Log("Success")
	case <-time.After(5 * time.Second):
		t.Error("Timed out")
	}
}
<commit_msg>format the code<commit_after>package udp

import (
	"testing"
	"time"

	"github.com/hsheth2/gonet/ipv4"
)

const rwport = 20102

func TestReadWriteLocal(t *testing.T) {
	readWriteTest(t, ipv4.LoopbackIPAddress, 0)
}

func TestReadWriteLocalFragmentation(t *testing.T) {
	readWriteTest(t, ipv4.LoopbackIPAddress, 10)
}

func TestReadWriteExternal(t *testing.T) {
	t.Skip("External tests actually don't work")
	readWriteTest(t, ipv4.ExternalIPAddress, 0)
}

func readWriteTest(t *testing.T, ip *ipv4.Address, exp int) {
	success := make(chan bool, 1)
	wrote := make(chan bool, 1)

	r, err := NewReader(rwport, ip)
	if err != nil {
		t.Fatal(err)
	}
	defer r.Close()

	data := []byte{'h', 'e', 'l', 'l', 'o'}
	for i := 0; i < exp; i++ {
		data = append(data, data...)
	}

	go func(data []byte) {
		w, err := NewWriter(20000, rwport, ip)
		if err != nil {
			t.Fatal(err)
		}

		_, err = w.Write(data)
		if err != nil {
			t.Fatal(err)
		} else {
			t.Log("Wrote the data:", data)
		}

		w.Close()

		wrote <- true
	}(data)

	go func(data []byte) {
		//time.Sleep(10*time.Second)
		p, err := r.Read(maxUDPPacketLength)
		if err != nil {
			t.Fatal(err)
		}
		t.Log("Output:", string(p))

		if string(p) == string(data) {
			t.Log("Got correct output:", p)
			success <- true
		} else {
			t.Error("Got Wrong Output:", p)
		}
	}(data)

	select {
	case <-success:
		<-wrote
		t.Log("Success")
	case <-time.After(5 * time.Second):
		t.Error("Timed out")
	}
}
<|endoftext|>


<commit_before>package start

import (
	"github.com/golang/glog"
	"github.com/spf13/pflag"

	kerrors "k8s.io/apimachinery/pkg/util/errors"
	schedulerapp "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app"
	_ "k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider"

	cmdflags "github.com/openshift/origin/pkg/cmd/util/flags"
)

func newScheduler(kubeconfigFile, schedulerConfigFile string, schedulerArgs map[string][]string) (*schedulerapp.Options, error) {
	cmdLineArgs := map[string][]string{}
	// deep-copy the input args to avoid mutation conflict.
	for k, v := range schedulerArgs {
		cmdLineArgs[k] = append([]string{}, v...)
	}
	if len(cmdLineArgs["kubeconfig"]) == 0 {
		cmdLineArgs["kubeconfig"] = []string{kubeconfigFile}
	}
	if len(cmdLineArgs["policy-config-file"]) == 0 {
		cmdLineArgs["policy-config-file"] = []string{schedulerConfigFile}
	}
	if len(cmdLineArgs["leader-elect-resource-lock"]) == 0 {
		cmdLineArgs["leader-elect-resource-lock"] = []string{"configmaps"}
	}

	// disable serving http since we didn't used to expose it
	if len(cmdLineArgs["port"]) == 0 {
		cmdLineArgs["port"] = []string{"-1"}
	}

	// resolve arguments
	schedulerOptions, err := schedulerapp.NewOptions()
	if err != nil {
		return nil, err
	}
	if err := schedulerOptions.ReallyApplyDefaults(); err != nil {
		return nil, err
	}
	if err := cmdflags.Resolve(cmdLineArgs, func(fs *pflag.FlagSet) {
		schedulerapp.AddFlags(schedulerOptions, fs)
	}); len(err) > 0 {
		return nil, kerrors.NewAggregate(err)
	}
	if err := schedulerOptions.Complete(); err != nil {
		return nil, err
	}

	return schedulerOptions, nil
}

func runEmbeddedScheduler(kubeconfigFile, schedulerConfigFile string, cmdLineArgs map[string][]string) {
	for {
		// TODO we need a real identity for this.  Right now it's just using the loopback connection like it used to.
		schedulerOptions, err := newScheduler(kubeconfigFile, schedulerConfigFile, cmdLineArgs)
		if err != nil {
			glog.Error(err)
			continue
		}
		// this does a second leader election, but doing the second leader election will allow us to move out process in
		// 3.8 if we so choose.
		if err := schedulerOptions.Run(); err != nil {
			glog.Error(err)
			continue
		}
	}
}
<commit_msg>add scheduler lease<commit_after>package start

import (
	"github.com/golang/glog"
	"github.com/spf13/pflag"

	kerrors "k8s.io/apimachinery/pkg/util/errors"
	schedulerapp "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app"
	_ "k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider"

	cmdflags "github.com/openshift/origin/pkg/cmd/util/flags"
)

func newScheduler(kubeconfigFile, schedulerConfigFile string, schedulerArgs map[string][]string) (*schedulerapp.Options, error) {
	cmdLineArgs := map[string][]string{}
	// deep-copy the input args to avoid mutation conflict.
	for k, v := range schedulerArgs {
		cmdLineArgs[k] = append([]string{}, v...)
	}
	if len(cmdLineArgs["kubeconfig"]) == 0 {
		cmdLineArgs["kubeconfig"] = []string{kubeconfigFile}
	}
	if len(cmdLineArgs["policy-config-file"]) == 0 {
		cmdLineArgs["policy-config-file"] = []string{schedulerConfigFile}
	}
	if _, ok := cmdLineArgs["leader-elect"]; !ok {
		cmdLineArgs["leader-elect"] = []string{"true"}
	}
	if len(cmdLineArgs["leader-elect-resource-lock"]) == 0 {
		cmdLineArgs["leader-elect-resource-lock"] = []string{"configmaps"}
	}

	// disable serving http since we didn't used to expose it
	if len(cmdLineArgs["port"]) == 0 {
		cmdLineArgs["port"] = []string{"-1"}
	}

	// resolve arguments
	schedulerOptions, err := schedulerapp.NewOptions()
	if err != nil {
		return nil, err
	}
	if err := schedulerOptions.ReallyApplyDefaults(); err != nil {
		return nil, err
	}
	if err := cmdflags.Resolve(cmdLineArgs, func(fs *pflag.FlagSet) {
		schedulerapp.AddFlags(schedulerOptions, fs)
	}); len(err) > 0 {
		return nil, kerrors.NewAggregate(err)
	}
	if err := schedulerOptions.Complete(); err != nil {
		return nil, err
	}

	return schedulerOptions, nil
}

func runEmbeddedScheduler(kubeconfigFile, schedulerConfigFile string, cmdLineArgs map[string][]string) {
	// TODO we need a real identity for this.  Right now it's just using the loopback connection like it used to.
	schedulerOptions, err := newScheduler(kubeconfigFile, schedulerConfigFile, cmdLineArgs)
	if err != nil {
		glog.Fatal(err)
	}
	// this does a second leader election, but doing the second leader election will allow us to move out process in
	// 3.8 if we so choose.
	if err := schedulerOptions.Run(); err != nil {
		glog.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Package in provides interfaces, concrete implementations, and utilities
// to ingest data into metrictank
package input

import (
	"fmt"

	"github.com/raintank/schema"
	"github.com/raintank/schema/msg"

	"github.com/grafana/metrictank/idx"
	"github.com/grafana/metrictank/mdata"
	"github.com/grafana/metrictank/stats"
	log "github.com/sirupsen/logrus"
)

type Handler interface {
	ProcessMetricData(md *schema.MetricData, partition int32)
	ProcessMetricPoint(point schema.MetricPoint, format msg.Format, partition int32)
}

// TODO: clever way to document all metrics for all different inputs

// Default is a base handler for a metrics packet, aimed to be embedded by concrete implementations
type DefaultHandler struct {
	receivedMD   *stats.Counter32
	receivedMP   *stats.Counter32
	receivedMPNO *stats.Counter32
	invalidMD    *stats.CounterRate32
	invalidMP    *stats.CounterRate32
	unknownMP    *stats.Counter32

	metrics     mdata.Metrics
	metricIndex idx.MetricIndex
}

func NewDefaultHandler(metrics mdata.Metrics, metricIndex idx.MetricIndex, input string) DefaultHandler {
	return DefaultHandler{
		// metric input.%s.metricdata.received is the count of metricdata datapoints received by input plugin
		receivedMD: stats.NewCounter32(fmt.Sprintf("input.%s.metricdata.received", input)),
		// metric input.%s.metricpoint.received is the count of metricpoint datapoints received by input plugin
		receivedMP: stats.NewCounter32(fmt.Sprintf("input.%s.metricpoint.received", input)),
		// metric input.%s.metricpoint_no_org.received is the count of metricpoint_no_org datapoints received by input plugin
		receivedMPNO: stats.NewCounter32(fmt.Sprintf("input.%s.metricpoint_no_org.received", input)),
		// metric input.%s.metricdata.invalid is a count of times a metricdata was invalid by input plugin
		invalidMD: stats.NewCounterRate32(fmt.Sprintf("input.%s.metricdata.invalid", input)),
		// metric input.%s.metricpoint.invalid is a count of times a metricpoint was invalid by input plugin
		invalidMP: stats.NewCounterRate32(fmt.Sprintf("input.%s.metricpoint.invalid", input)),
		// metric input.%s.metricpoint.unknown is the count of times the ID of a received metricpoint was not in the index, by input plugin
		unknownMP: stats.NewCounter32(fmt.Sprintf("input.%s.metricpoint.unknown", input)),

		metrics:     metrics,
		metricIndex: metricIndex,
	}
}

// ProcessMetricPoint updates the index if possible, and stores the data if we have an index entry
// concurrency-safe.
func (in DefaultHandler) ProcessMetricPoint(point schema.MetricPoint, format msg.Format, partition int32) {
	if format == msg.FormatMetricPoint {
		in.receivedMP.Inc()
	} else {
		in.receivedMPNO.Inc()
	}
	if !point.Valid() {
		in.invalidMP.Inc()
		log.Debugf("in: Invalid metric %v", point)
		return
	}

	archive, _, ok := in.metricIndex.Update(point, partition)

	if !ok {
		in.unknownMP.Inc()
		return
	}

	m := in.metrics.GetOrCreate(point.MKey, archive.SchemaId, archive.AggId)
	m.Add(point.Time, point.Value)
}

// ProcessMetricData assures the data is stored and the metadata is in the index
// concurrency-safe.
func (in DefaultHandler) ProcessMetricData(md *schema.MetricData, partition int32) {
	in.receivedMD.Inc()
	err := md.Validate()
	if err != nil {
		in.invalidMD.Inc()
		log.Debugf("in: Invalid metric %v: %s", md, err)
		return
	}
	if md.Time == 0 {
		in.invalidMD.Inc()
		log.Warnf("in: invalid metric. metric.Time is 0. %s", md.Id)
		return
	}

	mkey, err := schema.MKeyFromString(md.Id)
	if err != nil {
		log.Errorf("in: Invalid metric %v: could not parse ID: %s", md, err)
		return
	}

	archive, _, _ := in.metricIndex.AddOrUpdate(mkey, md, partition)

	m := in.metrics.GetOrCreate(mkey, archive.SchemaId, archive.AggId)
	m.Add(uint32(md.Time), md.Value)
}
<commit_msg>prevent integer values overflowing our index datatypes<commit_after>// Package in provides interfaces, concrete implementations, and utilities
// to ingest data into metrictank
package input

import (
	"fmt"
	"math"

	"github.com/raintank/schema"
	"github.com/raintank/schema/msg"

	"github.com/grafana/metrictank/idx"
	"github.com/grafana/metrictank/mdata"
	"github.com/grafana/metrictank/stats"
	log "github.com/sirupsen/logrus"
)

type Handler interface {
	ProcessMetricData(md *schema.MetricData, partition int32)
	ProcessMetricPoint(point schema.MetricPoint, format msg.Format, partition int32)
}

// TODO: clever way to document all metrics for all different inputs

// Default is a base handler for a metrics packet, aimed to be embedded by concrete implementations
type DefaultHandler struct {
	receivedMD   *stats.Counter32
	receivedMP   *stats.Counter32
	receivedMPNO *stats.Counter32
	invalidMD    *stats.CounterRate32
	invalidMP    *stats.CounterRate32
	unknownMP    *stats.Counter32

	metrics     mdata.Metrics
	metricIndex idx.MetricIndex
}

func NewDefaultHandler(metrics mdata.Metrics, metricIndex idx.MetricIndex, input string) DefaultHandler {
	return DefaultHandler{
		// metric input.%s.metricdata.received is the count of metricdata datapoints received by input plugin
		receivedMD: stats.NewCounter32(fmt.Sprintf("input.%s.metricdata.received", input)),
		// metric input.%s.metricpoint.received is the count of metricpoint datapoints received by input plugin
		receivedMP: stats.NewCounter32(fmt.Sprintf("input.%s.metricpoint.received", input)),
		// metric input.%s.metricpoint_no_org.received is the count of metricpoint_no_org datapoints received by input plugin
		receivedMPNO: stats.NewCounter32(fmt.Sprintf("input.%s.metricpoint_no_org.received", input)),
		// metric input.%s.metricdata.invalid is a count of times a metricdata was invalid by input plugin
		invalidMD: stats.NewCounterRate32(fmt.Sprintf("input.%s.metricdata.invalid", input)),
		// metric input.%s.metricpoint.invalid is a count of times a metricpoint was invalid by input plugin
		invalidMP: stats.NewCounterRate32(fmt.Sprintf("input.%s.metricpoint.invalid", input)),
		// metric input.%s.metricpoint.unknown is the count of times the ID of a received metricpoint was not in the index, by input plugin
		unknownMP: stats.NewCounter32(fmt.Sprintf("input.%s.metricpoint.unknown", input)),

		metrics:     metrics,
		metricIndex: metricIndex,
	}
}

// ProcessMetricPoint updates the index if possible, and stores the data if we have an index entry
// concurrency-safe.
func (in DefaultHandler) ProcessMetricPoint(point schema.MetricPoint, format msg.Format, partition int32) {
	if format == msg.FormatMetricPoint {
		in.receivedMP.Inc()
	} else {
		in.receivedMPNO.Inc()
	}
	// in cassandra we store timestamps as 32bit signed integers.
	// math.MaxInt32 = Jan 19 03:14:07 UTC 2038
	if !point.Valid() || point.Time >= math.MaxInt32 {
		in.invalidMP.Inc()
		log.Debugf("in: Invalid metric %v", point)
		return
	}

	archive, _, ok := in.metricIndex.Update(point, partition)

	if !ok {
		in.unknownMP.Inc()
		return
	}

	m := in.metrics.GetOrCreate(point.MKey, archive.SchemaId, archive.AggId)
	m.Add(point.Time, point.Value)
}

// ProcessMetricData assures the data is stored and the metadata is in the index
// concurrency-safe.
func (in DefaultHandler) ProcessMetricData(md *schema.MetricData, partition int32) {
	in.receivedMD.Inc()
	err := md.Validate()
	if err != nil {
		in.invalidMD.Inc()
		log.Debugf("in: Invalid metric %v: %s", md, err)
		return
	}
	// in cassandra we store timestamps and interval as 32bit signed integers.
	// math.MaxInt32 = Jan 19 03:14:07 UTC 2038
	if md.Time == 0 || md.Time >= math.MaxInt32 || md.Interval >= math.MaxInt32 {
		in.invalidMD.Inc()
		log.Warnf("in: invalid metric %q. Time/Interval out of range", md.Id)
		return
	}

	mkey, err := schema.MKeyFromString(md.Id)
	if err != nil {
		log.Errorf("in: Invalid metric %v: could not parse ID: %s", md, err)
		return
	}

	archive, _, _ := in.metricIndex.AddOrUpdate(mkey, md, partition)

	m := in.metrics.GetOrCreate(mkey, archive.SchemaId, archive.AggId)
	m.Add(uint32(md.Time), md.Value)
}
<|endoftext|>