<commit_before>package b2

import (
	"context"
	"io"
	"net/http"
	"path"
	"strings"

	"github.com/restic/restic/internal/backend"
	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"

	"github.com/kurin/blazer/b2"
)

// b2Backend is a backend which stores its data on Backblaze B2.
type b2Backend struct {
	client       *b2.Client
	bucket       *b2.Bucket
	cfg          Config
	listMaxItems int
	backend.Layout
	sem *backend.Semaphore
}

const defaultListMaxItems = 1000

// ensure statically that *b2Backend implements restic.Backend.
var _ restic.Backend = &b2Backend{}

func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Client, error) {
	opts := []b2.ClientOption{b2.Transport(rt)}

	c, err := b2.NewClient(ctx, cfg.AccountID, cfg.Key, opts...)
	if err != nil {
		return nil, errors.Wrap(err, "b2.NewClient")
	}
	return c, nil
}

// Open opens a connection to the B2 service.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
	debug.Log("cfg %#v", cfg)

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	client, err := newClient(ctx, cfg, rt)
	if err != nil {
		return nil, err
	}

	bucket, err := client.Bucket(ctx, cfg.Bucket)
	if err != nil {
		return nil, errors.Wrap(err, "Bucket")
	}

	sem, err := backend.NewSemaphore(cfg.Connections)
	if err != nil {
		return nil, err
	}

	be := &b2Backend{
		client: client,
		bucket: bucket,
		cfg:    cfg,
		Layout: &backend.DefaultLayout{
			Join: path.Join,
			Path: cfg.Prefix,
		},
		listMaxItems: defaultListMaxItems,
		sem:          sem,
	}

	return be, nil
}

// Create opens a connection to the B2 service. If the bucket does not exist yet,
// it is created.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
	debug.Log("cfg %#v", cfg)

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	client, err := newClient(ctx, cfg, rt)
	if err != nil {
		return nil, err
	}

	attr := b2.BucketAttrs{
		Type: b2.Private,
	}
	bucket, err := client.NewBucket(ctx, cfg.Bucket, &attr)
	if err != nil {
		return nil, errors.Wrap(err, "NewBucket")
	}

	sem, err := backend.NewSemaphore(cfg.Connections)
	if err != nil {
		return nil, err
	}

	be := &b2Backend{
		client: client,
		bucket: bucket,
		cfg:    cfg,
		Layout: &backend.DefaultLayout{
			Join: path.Join,
			Path: cfg.Prefix,
		},
		listMaxItems: defaultListMaxItems,
		sem:          sem,
	}

	present, err := be.Test(ctx, restic.Handle{Type: restic.ConfigFile})
	if err != nil {
		return nil, err
	}

	if present {
		return nil, errors.New("config already exists")
	}

	return be, nil
}

// SetListMaxItems sets the number of list items to load per request.
func (be *b2Backend) SetListMaxItems(i int) {
	be.listMaxItems = i
}

// Location returns the location for the backend.
func (be *b2Backend) Location() string {
	return be.cfg.Bucket
}

// IsNotExist returns true if the error is caused by a non-existing file.
func (be *b2Backend) IsNotExist(err error) bool {
	return b2.IsNotExist(errors.Cause(err))
}

// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
	return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}

func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
	if err := h.Valid(); err != nil {
		return nil, err
	}

	if offset < 0 {
		return nil, errors.New("offset is negative")
	}

	if length < 0 {
		return nil, errors.Errorf("invalid length %d", length)
	}

	ctx, cancel := context.WithCancel(ctx)

	be.sem.GetToken()

	name := be.Layout.Filename(h)
	obj := be.bucket.Object(name)

	if offset == 0 && length == 0 {
		rd := obj.NewReader(ctx)
		return be.sem.ReleaseTokenOnClose(rd, cancel), nil
	}

	// pass a negative length to NewRangeReader so that the remainder of the
	// file is read.
	if length == 0 {
		length = -1
	}

	rd := obj.NewRangeReader(ctx, offset, int64(length))
	return be.sem.ReleaseTokenOnClose(rd, cancel), nil
}

// Save stores data in the backend at the handle.
func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	if err := h.Valid(); err != nil {
		return err
	}

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	name := be.Filename(h)
	debug.Log("Save %v, name %v", h, name)
	obj := be.bucket.Object(name)

	w := obj.NewWriter(ctx)
	n, err := io.Copy(w, rd)
	debug.Log("  saved %d bytes, err %v", n, err)

	if err != nil {
		_ = w.Close()
		return errors.Wrap(err, "Copy")
	}

	return errors.Wrap(w.Close(), "Close")
}

// Stat returns information about a blob.
func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
	debug.Log("Stat %v", h)

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	name := be.Filename(h)
	obj := be.bucket.Object(name)
	info, err := obj.Attrs(ctx)
	if err != nil {
		debug.Log("Attrs() err %v", err)
		return restic.FileInfo{}, errors.Wrap(err, "Stat")
	}
	return restic.FileInfo{Size: info.Size, Name: h.Name}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (be *b2Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
	debug.Log("Test %v", h)

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	found := false
	name := be.Filename(h)
	obj := be.bucket.Object(name)
	info, err := obj.Attrs(ctx)
	if err == nil && info != nil && info.Status == b2.Uploaded {
		found = true
	}
	return found, nil
}

// Remove removes the blob with the given name and type.
func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
	debug.Log("Remove %v", h)

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	obj := be.bucket.Object(be.Filename(h))
	return errors.Wrap(obj.Delete(ctx), "Delete")
}

// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
	debug.Log("List %v", t)

	prefix, _ := be.Basedir(t)
	cur := &b2.Cursor{Prefix: prefix}

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	for {
		be.sem.GetToken()
		objs, c, err := be.bucket.ListCurrentObjects(ctx, be.listMaxItems, cur)
		be.sem.ReleaseToken()

		if err != nil && err != io.EOF {
			debug.Log("List: %v", err)
			return err
		}

		debug.Log("returned %v items", len(objs))
		for _, obj := range objs {
			// Skip objects returned that do not have the specified prefix.
			if !strings.HasPrefix(obj.Name(), prefix) {
				continue
			}

			m := path.Base(obj.Name())
			if m == "" {
				continue
			}

			if ctx.Err() != nil {
				return ctx.Err()
			}

			attrs, err := obj.Attrs(ctx)
			if err != nil {
				return err
			}

			fi := restic.FileInfo{
				Name: m,
				Size: attrs.Size,
			}

			err = fn(fi)
			if err != nil {
				return err
			}

			if ctx.Err() != nil {
				return ctx.Err()
			}
		}

		if err == io.EOF {
			return ctx.Err()
		}
		cur = c
	}
}

// Remove keys for a specified backend type.
func (be *b2Backend) removeKeys(ctx context.Context, t restic.FileType) error {
	debug.Log("removeKeys %v", t)
	return be.List(ctx, t, func(fi restic.FileInfo) error {
		return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name})
	})
}

// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *b2Backend) Delete(ctx context.Context) error {
	alltypes := []restic.FileType{
		restic.DataFile,
		restic.KeyFile,
		restic.LockFile,
		restic.SnapshotFile,
		restic.IndexFile}

	for _, t := range alltypes {
		err := be.removeKeys(ctx, t)
		if err != nil {
			return nil
		}
	}
	err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
	if err != nil && b2.IsNotExist(errors.Cause(err)) {
		err = nil
	}

	return err
}

// Close does nothing
func (be *b2Backend) Close() error { return nil }
<commit_msg>b2: simplify object iteration<commit_after>package b2

import (
	"context"
	"io"
	"net/http"
	"path"

	"github.com/restic/restic/internal/backend"
	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"

	"github.com/kurin/blazer/b2"
)

// b2Backend is a backend which stores its data on Backblaze B2.
type b2Backend struct {
	client       *b2.Client
	bucket       *b2.Bucket
	cfg          Config
	listMaxItems int
	backend.Layout
	sem *backend.Semaphore
}

const defaultListMaxItems = 1000

// ensure statically that *b2Backend implements restic.Backend.
var _ restic.Backend = &b2Backend{}

func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Client, error) {
	opts := []b2.ClientOption{b2.Transport(rt)}

	c, err := b2.NewClient(ctx, cfg.AccountID, cfg.Key, opts...)
	if err != nil {
		return nil, errors.Wrap(err, "b2.NewClient")
	}
	return c, nil
}

// Open opens a connection to the B2 service.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
	debug.Log("cfg %#v", cfg)

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	client, err := newClient(ctx, cfg, rt)
	if err != nil {
		return nil, err
	}

	bucket, err := client.Bucket(ctx, cfg.Bucket)
	if err != nil {
		return nil, errors.Wrap(err, "Bucket")
	}

	sem, err := backend.NewSemaphore(cfg.Connections)
	if err != nil {
		return nil, err
	}

	be := &b2Backend{
		client: client,
		bucket: bucket,
		cfg:    cfg,
		Layout: &backend.DefaultLayout{
			Join: path.Join,
			Path: cfg.Prefix,
		},
		listMaxItems: defaultListMaxItems,
		sem:          sem,
	}

	return be, nil
}

// Create opens a connection to the B2 service. If the bucket does not exist yet,
// it is created.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
	debug.Log("cfg %#v", cfg)

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	client, err := newClient(ctx, cfg, rt)
	if err != nil {
		return nil, err
	}

	attr := b2.BucketAttrs{
		Type: b2.Private,
	}
	bucket, err := client.NewBucket(ctx, cfg.Bucket, &attr)
	if err != nil {
		return nil, errors.Wrap(err, "NewBucket")
	}

	sem, err := backend.NewSemaphore(cfg.Connections)
	if err != nil {
		return nil, err
	}

	be := &b2Backend{
		client: client,
		bucket: bucket,
		cfg:    cfg,
		Layout: &backend.DefaultLayout{
			Join: path.Join,
			Path: cfg.Prefix,
		},
		listMaxItems: defaultListMaxItems,
		sem:          sem,
	}

	present, err := be.Test(ctx, restic.Handle{Type: restic.ConfigFile})
	if err != nil {
		return nil, err
	}

	if present {
		return nil, errors.New("config already exists")
	}

	return be, nil
}

// SetListMaxItems sets the number of list items to load per request.
func (be *b2Backend) SetListMaxItems(i int) {
	be.listMaxItems = i
}

// Location returns the location for the backend.
func (be *b2Backend) Location() string {
	return be.cfg.Bucket
}

// IsNotExist returns true if the error is caused by a non-existing file.
func (be *b2Backend) IsNotExist(err error) bool {
	return b2.IsNotExist(errors.Cause(err))
}

// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
	return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}

func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
	if err := h.Valid(); err != nil {
		return nil, err
	}

	if offset < 0 {
		return nil, errors.New("offset is negative")
	}

	if length < 0 {
		return nil, errors.Errorf("invalid length %d", length)
	}

	ctx, cancel := context.WithCancel(ctx)

	be.sem.GetToken()

	name := be.Layout.Filename(h)
	obj := be.bucket.Object(name)

	if offset == 0 && length == 0 {
		rd := obj.NewReader(ctx)
		return be.sem.ReleaseTokenOnClose(rd, cancel), nil
	}

	// pass a negative length to NewRangeReader so that the remainder of the
	// file is read.
	if length == 0 {
		length = -1
	}

	rd := obj.NewRangeReader(ctx, offset, int64(length))
	return be.sem.ReleaseTokenOnClose(rd, cancel), nil
}

// Save stores data in the backend at the handle.
func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	if err := h.Valid(); err != nil {
		return err
	}

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	name := be.Filename(h)
	debug.Log("Save %v, name %v", h, name)
	obj := be.bucket.Object(name)

	w := obj.NewWriter(ctx)
	n, err := io.Copy(w, rd)
	debug.Log("  saved %d bytes, err %v", n, err)

	if err != nil {
		_ = w.Close()
		return errors.Wrap(err, "Copy")
	}

	return errors.Wrap(w.Close(), "Close")
}

// Stat returns information about a blob.
func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
	debug.Log("Stat %v", h)

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	name := be.Filename(h)
	obj := be.bucket.Object(name)
	info, err := obj.Attrs(ctx)
	if err != nil {
		debug.Log("Attrs() err %v", err)
		return restic.FileInfo{}, errors.Wrap(err, "Stat")
	}
	return restic.FileInfo{Size: info.Size, Name: h.Name}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (be *b2Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
	debug.Log("Test %v", h)

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	found := false
	name := be.Filename(h)
	obj := be.bucket.Object(name)
	info, err := obj.Attrs(ctx)
	if err == nil && info != nil && info.Status == b2.Uploaded {
		found = true
	}
	return found, nil
}

// Remove removes the blob with the given name and type.
func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
	debug.Log("Remove %v", h)

	be.sem.GetToken()
	defer be.sem.ReleaseToken()

	obj := be.bucket.Object(be.Filename(h))
	return errors.Wrap(obj.Delete(ctx), "Delete")
}

type semLocker struct {
	*backend.Semaphore
}

func (sm semLocker) Lock()   { sm.GetToken() }
func (sm semLocker) Unlock() { sm.ReleaseToken() }

// List returns a channel that yields all names of blobs of type t.
func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
	debug.Log("List %v", t)

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	prefix, _ := be.Basedir(t)
	iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems), b2.ListLocker(semLocker{be.sem}))

	for iter.Next() {
		obj := iter.Object()

		attrs, err := obj.Attrs(ctx)
		if err != nil {
			return err
		}

		fi := restic.FileInfo{
			Name: path.Base(obj.Name()),
			Size: attrs.Size,
		}

		if err := fn(fi); err != nil {
			return err
		}
	}
	if err := iter.Err(); err != nil {
		debug.Log("List: %v", err)
		return err
	}
	return nil
}

// Remove keys for a specified backend type.
func (be *b2Backend) removeKeys(ctx context.Context, t restic.FileType) error {
	debug.Log("removeKeys %v", t)
	return be.List(ctx, t, func(fi restic.FileInfo) error {
		return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name})
	})
}

// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *b2Backend) Delete(ctx context.Context) error {
	alltypes := []restic.FileType{
		restic.DataFile,
		restic.KeyFile,
		restic.LockFile,
		restic.SnapshotFile,
		restic.IndexFile}

	for _, t := range alltypes {
		err := be.removeKeys(ctx, t)
		if err != nil {
			return nil
		}
	}
	err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
	if err != nil && b2.IsNotExist(errors.Cause(err)) {
		err = nil
	}

	return err
}

// Close does nothing
func (be *b2Backend) Close() error { return nil }
<|endoftext|>


<commit_before>package bloblru

import (
	"sync"

	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/restic"

	"github.com/hashicorp/golang-lru/simplelru"
)

// Crude estimate of the overhead per blob: a SHA-256, a linked list node
// and some pointers. See comment in Cache.add.
const overhead = len(restic.ID{}) + 64

// A Cache is a fixed-size LRU cache of blob contents.
// It is safe for concurrent access.
type Cache struct {
	mu sync.Mutex
	c  *simplelru.LRU

	free, size int // Current and max capacity, in bytes.
}

// Construct a blob cache that stores at most size bytes worth of blobs.
func New(size int) *Cache {
	c := &Cache{
		free: size,
		size: size,
	}

	// NewLRU wants us to specify some max. number of entries, else it errors.
	// The actual maximum will be smaller than size/overhead, because we
	// evict entries (RemoveOldest in add) to maintain our size bound.
	maxEntries := size / overhead
	lru, err := simplelru.NewLRU(maxEntries, c.evict)
	if err != nil {
		panic(err) // Can only be maxEntries <= 0.
	}
	c.c = lru

	return c
}

// Add adds key id with value blob to c.
// It may return an evicted buffer for reuse.
func (c *Cache) Add(id restic.ID, blob []byte) (old []byte) {
	debug.Log("bloblru.Cache: add %v", id)

	size := cap(blob) + overhead
	if size > c.size {
		return
	}

	c.mu.Lock()
	defer c.mu.Unlock()

	var key interface{} = id

	if c.c.Contains(key) { // Doesn't update the recency list.
		return
	}

	// This loop takes at most min(maxEntries, maxchunksize/overhead)
	// iterations.
	for size > c.free {
		_, val, _ := c.c.RemoveOldest()
		b := val.([]byte)
		if cap(b) > cap(old) {
			// We can only return one buffer, so pick the largest.
			old = b
		}
	}

	c.c.Add(key, blob)
	c.free -= size

	return old
}

func (c *Cache) Get(id restic.ID) ([]byte, bool) {
	c.mu.Lock()
	value, ok := c.c.Get(id)
	c.mu.Unlock()

	debug.Log("bloblru.Cache: get %v, hit %v", id, ok)

	blob, ok := value.([]byte)
	return blob, ok
}

func (c *Cache) evict(key, value interface{}) {
	blob := value.([]byte)
	debug.Log("bloblru.Cache: evict %v, %d bytes", key, cap(blob))
	c.free += cap(blob) + overhead
}
<commit_msg>bloblru: Fix comment for New function<commit_after>package bloblru

import (
	"sync"

	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/restic"

	"github.com/hashicorp/golang-lru/simplelru"
)

// Crude estimate of the overhead per blob: a SHA-256, a linked list node
// and some pointers. See comment in Cache.add.
const overhead = len(restic.ID{}) + 64

// A Cache is a fixed-size LRU cache of blob contents.
// It is safe for concurrent access.
type Cache struct {
	mu sync.Mutex
	c  *simplelru.LRU

	free, size int // Current and max capacity, in bytes.
}

// New constructs a blob cache that stores at most size bytes worth of blobs.
func New(size int) *Cache {
	c := &Cache{
		free: size,
		size: size,
	}

	// NewLRU wants us to specify some max. number of entries, else it errors.
	// The actual maximum will be smaller than size/overhead, because we
	// evict entries (RemoveOldest in add) to maintain our size bound.
	maxEntries := size / overhead
	lru, err := simplelru.NewLRU(maxEntries, c.evict)
	if err != nil {
		panic(err) // Can only be maxEntries <= 0.
	}
	c.c = lru

	return c
}

// Add adds key id with value blob to c.
// It may return an evicted buffer for reuse.
func (c *Cache) Add(id restic.ID, blob []byte) (old []byte) {
	debug.Log("bloblru.Cache: add %v", id)

	size := cap(blob) + overhead
	if size > c.size {
		return
	}

	c.mu.Lock()
	defer c.mu.Unlock()

	var key interface{} = id

	if c.c.Contains(key) { // Doesn't update the recency list.
		return
	}

	// This loop takes at most min(maxEntries, maxchunksize/overhead)
	// iterations.
	for size > c.free {
		_, val, _ := c.c.RemoveOldest()
		b := val.([]byte)
		if cap(b) > cap(old) {
			// We can only return one buffer, so pick the largest.
			old = b
		}
	}

	c.c.Add(key, blob)
	c.free -= size

	return old
}

func (c *Cache) Get(id restic.ID) ([]byte, bool) {
	c.mu.Lock()
	value, ok := c.c.Get(id)
	c.mu.Unlock()

	debug.Log("bloblru.Cache: get %v, hit %v", id, ok)

	blob, ok := value.([]byte)
	return blob, ok
}

func (c *Cache) evict(key, value interface{}) {
	blob := value.([]byte)
	debug.Log("bloblru.Cache: evict %v, %d bytes", key, cap(blob))
	c.free += cap(blob) + overhead
}
<|endoftext|>


<commit_before>package config

import (
	"bytes"
	"flag"
	"fmt"
	"net/url"
	"os"
	"reflect"
	"strings"
	"time"
	"unicode"

	"github.com/kelseyhightower/envconfig"

	log "github.com/sirupsen/logrus"
)

type spaceSeparatedList []string

func (mvd *spaceSeparatedList) Decode(value string) error {
	*mvd = spaceSeparatedList(strings.Split(value, " "))
	return nil
}

type configEnvs struct {
	AlertmanagerTimeout time.Duration      `envconfig:"ALERTMANAGER_TIMEOUT" default:"40s" help:"Timeout for all request send to Alertmanager"`
	AlertmanagerTTL     time.Duration      `envconfig:"ALERTMANAGER_TTL" default:"1m" help:"TTL for Alertmanager alerts and silences"`
	AlertmanagerURIs    spaceSeparatedList `envconfig:"ALERTMANAGER_URIS" required:"true" help:"List of Alertmanager URIs (name:uri)"`
	ColorLabelsStatic   spaceSeparatedList `envconfig:"COLOR_LABELS_STATIC" help:"List of label names that should have the same (but distinct) color"`
	ColorLabelsUnique   spaceSeparatedList `envconfig:"COLOR_LABELS_UNIQUE" help:"List of label names that should have unique color"`
	Debug               bool               `envconfig:"DEBUG" default:"false" help:"Enable debug mode"`
	FilterDefault       string             `envconfig:"FILTER_DEFAULT" help:"Default filter string"`
	JiraRegexp          spaceSeparatedList `envconfig:"JIRA_REGEX" help:"List of JIRA regex rules"`
	Port                int                `envconfig:"PORT" default:"8080" help:"HTTP port to listen on"`
	SentryDSN           string             `envconfig:"SENTRY_DSN" help:"Sentry DSN for Go exceptions"`
	SentryPublicDSN     string             `envconfig:"SENTRY_PUBLIC_DSN" help:"Sentry DSN for javascript exceptions"`
	StripLabels         spaceSeparatedList `envconfig:"STRIP_LABELS" help:"List of labels to ignore"`
	KeepLabels          spaceSeparatedList `envconfig:"KEEP_LABELS" help:"List of labels to keep, all other labels will be stripped"`
	WebPrefix           string             `envconfig:"WEB_PREFIX" default:"/" help:"URL prefix"`
}

// Config exposes all options required to run
var Config configEnvs

// generate flag name from the option name, a dot will be injected between
// <lower case char><upper case char>
func makeFlagName(s string) string {
	var buffer bytes.Buffer
	prevUpper := true
	for _, rune := range s {
		if unicode.IsUpper(rune) && !prevUpper {
			buffer.WriteRune('.')
		}
		prevUpper = unicode.IsUpper(rune)
		buffer.WriteRune(unicode.ToLower(rune))
	}
	return buffer.String()
}

// Iterate all defined envconfig variables and generate a flag for each key.
// Next parse those flags and for each set flag inject env variable which will
// be read by envconfig later on.
type flagMapper struct {
	isBool    bool
	stringVal *string
	boolVal   *bool
}

func mapEnvConfigToFlags() {
	flags := make(map[string]flagMapper)
	s := reflect.ValueOf(Config)
	typeOfSpec := s.Type()
	for i := 0; i < s.NumField(); i++ {
		f := typeOfSpec.Field(i)

		flagName := makeFlagName(f.Name)
		// check if flag was already set, this usually happens only during testing
		if flag.Lookup(flagName) != nil {
			continue
		}

		envName := f.Tag.Get("envconfig")

		helpMsg := fmt.Sprintf("%s. This flag can also be set via %s environment variable.", f.Tag.Get("help"), f.Tag.Get("envconfig"))
		if f.Tag.Get("required") == "true" {
			helpMsg = fmt.Sprintf("%s This option is required.", helpMsg)
		}

		mapper := flagMapper{}
		if s.Field(i).Kind() == reflect.Bool {
			mapper.isBool = true
			mapper.boolVal = flag.Bool(flagName, false, helpMsg)
		} else {
			mapper.stringVal = flag.String(flagName, "", helpMsg)
		}
		flags[envName] = mapper
	}
	flag.Parse()
	for envName, mapper := range flags {
		if mapper.isBool {
			if *mapper.boolVal == true {
				err := os.Setenv(envName, "true")
				if err != nil {
					log.Fatal(err)
				}
			}
		} else {
			if *mapper.stringVal != "" {
				err := os.Setenv(envName, *mapper.stringVal)
				if err != nil {
					log.Fatal(err)
				}
			}
		}
	}
}

func (config *configEnvs) Read() {
	mapEnvConfigToFlags()

	err := envconfig.Process("", config)
	if err != nil {
		log.Fatal(err)
	}
}

func hideURLPassword(s string) string {
	u, err := url.Parse(s)
	if err != nil {
		return s
	}
	if u.User != nil {
		if _, pwdSet := u.User.Password(); pwdSet {
			u.User = url.UserPassword(u.User.Username(), "xxx")
		}
		return u.String()
	}
	return s
}

func (config *configEnvs) LogValues() {
	s := reflect.ValueOf(config).Elem()
	typeOfT := s.Type()
	for i := 0; i < s.NumField(); i++ {
		env := typeOfT.Field(i).Tag.Get("envconfig")
		val := fmt.Sprintf("%v", s.Field(i).Interface())
		log.Infof("%20s => %v", env, hideURLPassword(val))
	}

}
<commit_msg>Add config options for controlling annotations visibility<commit_after>package config

import (
	"bytes"
	"flag"
	"fmt"
	"net/url"
	"os"
	"reflect"
	"strings"
	"time"
	"unicode"

	"github.com/kelseyhightower/envconfig"

	log "github.com/sirupsen/logrus"
)

type spaceSeparatedList []string

func (mvd *spaceSeparatedList) Decode(value string) error {
	*mvd = spaceSeparatedList(strings.Split(value, " "))
	return nil
}

type configEnvs struct {
	AlertmanagerTimeout time.Duration      `envconfig:"ALERTMANAGER_TIMEOUT" default:"40s" help:"Timeout for all request send to Alertmanager"`
	AlertmanagerTTL     time.Duration      `envconfig:"ALERTMANAGER_TTL" default:"1m" help:"TTL for Alertmanager alerts and silences"`
	AlertmanagerURIs    spaceSeparatedList `envconfig:"ALERTMANAGER_URIS" required:"true" help:"List of Alertmanager URIs (name:uri)"`
	AnnotationsHidden   spaceSeparatedList `envconfig:"ANNOTATIONS_HIDDEN" help:"List of annotations that are hidden by default"`
	AnnotationsVisible  spaceSeparatedList `envconfig:"ANNOTATIONS_VISIBLE" help:"List of annotations that are visible by default"`
	ColorLabelsStatic   spaceSeparatedList `envconfig:"COLOR_LABELS_STATIC" help:"List of label names that should have the same (but distinct) color"`
	ColorLabelsUnique   spaceSeparatedList `envconfig:"COLOR_LABELS_UNIQUE" help:"List of label names that should have unique color"`
	Debug               bool               `envconfig:"DEBUG" default:"false" help:"Enable debug mode"`
	FilterDefault       string             `envconfig:"FILTER_DEFAULT" help:"Default filter string"`
	JiraRegexp          spaceSeparatedList `envconfig:"JIRA_REGEX" help:"List of JIRA regex rules"`
	Port                int                `envconfig:"PORT" default:"8080" help:"HTTP port to listen on"`
	SentryDSN           string             `envconfig:"SENTRY_DSN" help:"Sentry DSN for Go exceptions"`
	SentryPublicDSN     string             `envconfig:"SENTRY_PUBLIC_DSN" help:"Sentry DSN for javascript exceptions"`
	StripLabels         spaceSeparatedList `envconfig:"STRIP_LABELS" help:"List of labels to ignore"`
	KeepLabels          spaceSeparatedList `envconfig:"KEEP_LABELS" help:"List of labels to keep, all other labels will be stripped"`
	WebPrefix           string             `envconfig:"WEB_PREFIX" default:"/" help:"URL prefix"`
}

// Config exposes all options required to run
var Config configEnvs

// generate flag name from the option name, a dot will be injected between
// <lower case char><upper case char>
func makeFlagName(s string) string {
	var buffer bytes.Buffer
	prevUpper := true
	for _, rune := range s {
		if unicode.IsUpper(rune) && !prevUpper {
			buffer.WriteRune('.')
		}
		prevUpper = unicode.IsUpper(rune)
		buffer.WriteRune(unicode.ToLower(rune))
	}
	return buffer.String()
}

// Iterate all defined envconfig variables and generate a flag for each key.
// Next parse those flags and for each set flag inject env variable which will
// be read by envconfig later on.
type flagMapper struct {
	isBool    bool
	stringVal *string
	boolVal   *bool
}

func mapEnvConfigToFlags() {
	flags := make(map[string]flagMapper)
	s := reflect.ValueOf(Config)
	typeOfSpec := s.Type()
	for i := 0; i < s.NumField(); i++ {
		f := typeOfSpec.Field(i)

		flagName := makeFlagName(f.Name)
		// check if flag was already set, this usually happens only during testing
		if flag.Lookup(flagName) != nil {
			continue
		}

		envName := f.Tag.Get("envconfig")

		helpMsg := fmt.Sprintf("%s. This flag can also be set via %s environment variable.", f.Tag.Get("help"), f.Tag.Get("envconfig"))
		if f.Tag.Get("required") == "true" {
			helpMsg = fmt.Sprintf("%s This option is required.", helpMsg)
		}

		mapper := flagMapper{}
		if s.Field(i).Kind() == reflect.Bool {
			mapper.isBool = true
			mapper.boolVal = flag.Bool(flagName, false, helpMsg)
		} else {
			mapper.stringVal = flag.String(flagName, "", helpMsg)
		}
		flags[envName] = mapper
	}
	flag.Parse()
	for envName, mapper := range flags {
		if mapper.isBool {
			if *mapper.boolVal == true {
				err := os.Setenv(envName, "true")
				if err != nil {
					log.Fatal(err)
				}
			}
		} else {
			if *mapper.stringVal != "" {
				err := os.Setenv(envName, *mapper.stringVal)
				if err != nil {
					log.Fatal(err)
				}
			}
		}
	}
}

func (config *configEnvs) Read() {
	mapEnvConfigToFlags()

	err := envconfig.Process("", config)
	if err != nil {
		log.Fatal(err)
	}
}

func hideURLPassword(s string) string {
	u, err := url.Parse(s)
	if err != nil {
		return s
	}
	if u.User != nil {
		if _, pwdSet := u.User.Password(); pwdSet {
			u.User = url.UserPassword(u.User.Username(), "xxx")
		}
		return u.String()
	}
	return s
}

func (config *configEnvs) LogValues() {
	s := reflect.ValueOf(config).Elem()
	typeOfT := s.Type()
	for i := 0; i < s.NumField(); i++ {
		env := typeOfT.Field(i).Tag.Get("envconfig")
		val := fmt.Sprintf("%v", s.Field(i).Interface())
		log.Infof("%20s => %v", env, hideURLPassword(val))
	}

}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package interpreter

import (
	"errors"
	"fmt"
	"sync"

	"github.com/google/cel-go/common/types/ref"
)

// Activation used to resolve identifiers by name and references by id.
//
// An Activation is the primary mechanism by which a caller supplies input into a CEL program.
type Activation interface {
	// ResolveName returns a value from the activation by qualified name, or false if the name
	// could not be found.
	ResolveName(name string) (interface{}, bool)

	// Parent returns the parent of the current activation, may be nil.
	// If non-nil, the parent will be searched during resolve calls.
	Parent() Activation
}

// EmptyActivation returns a variable free activation.
func EmptyActivation() Activation {
	// This call cannot fail.
	a, _ := NewActivation(map[string]interface{}{})
	return a
}

// NewActivation returns an activation based on a map-based binding where the map keys are
// expected to be qualified names used with ResolveName calls.
//
// The input `bindings` may either be of type `Activation` or `map[string]interface{}`.
//
// Lazy bindings may be supplied within the map-based input in either of the following forms:
// - func() interface{}
// - func() ref.Val
//
// The output of the lazy binding will overwrite the variable reference in the internal map.
//
// Values which are not represented as ref.Val types on input may be adapted to a ref.Val using
// the ref.TypeAdapter configured in the environment.
func NewActivation(bindings interface{}) (Activation, error) {
	if bindings == nil {
		return nil, errors.New("bindings must be non-nil")
	}
	a, isActivation := bindings.(Activation)
	if isActivation {
		return a, nil
	}
	m, isMap := bindings.(map[string]interface{})
	if !isMap {
		return nil, fmt.Errorf(
			"activation input must be an activation or map[string]interface: got %T",
			bindings)
	}
	return &mapActivation{bindings: m}, nil
}

// mapActivation which implements Activation and maps of named values.
//
// Named bindings may lazily supply values by providing a function which accepts no arguments and
// produces an interface value.
type mapActivation struct {
	bindings map[string]interface{}
}

// Parent implements the Activation interface method.
func (a *mapActivation) Parent() Activation {
	return nil
}

// ResolveName implements the Activation interface method.
func (a *mapActivation) ResolveName(name string) (interface{}, bool) {
	obj, found := a.bindings[name]
	if !found {
		return nil, false
	}
	fn, isLazy := obj.(func() ref.Val)
	if isLazy {
		obj = fn()
		a.bindings[name] = obj
	}
	fnRaw, isLazy := obj.(func() interface{})
	if isLazy {
		obj = fnRaw()
		a.bindings[name] = obj
	}
	return obj, found
}

// hierarchicalActivation which implements Activation and contains a parent and
// child activation.
type hierarchicalActivation struct {
	parent Activation
	child  Activation
}

// Parent implements the Activation interface method.
func (a *hierarchicalActivation) Parent() Activation {
	return a.parent
}

// ResolveName implements the Activation interface method.
func (a *hierarchicalActivation) ResolveName(name string) (interface{}, bool) {
	if object, found := a.child.ResolveName(name); found {
		return object, found
	}
	return a.parent.ResolveName(name)
}

// NewHierarchicalActivation takes two activations and produces a new one which prioritizes
// resolution in the child first and parent(s) second.
func NewHierarchicalActivation(parent Activation, child Activation) Activation {
	return &hierarchicalActivation{parent, child}
}

// NewPartialActivation returns an Activation which contains a list of AttributePattern values
// representing field and index operations that should result in a 'types.Unknown' result.
//
// The `bindings` value may be any value type supported by the interpreter.NewActivation call,
// but is typically either an existing Activation or map[string]interface{}.
func NewPartialActivation(bindings interface{},
	unknowns ...*AttributePattern) (PartialActivation, error) {
	a, err := NewActivation(bindings)
	if err != nil {
		return nil, err
	}
	return &partActivation{Activation: a, unknowns: unknowns}, nil
}

// PartialActivation extends the Activation interface with a set of UnknownAttributePatterns.
type PartialActivation interface {
	Activation

	// UnknownAttributePaths returns a set of AttributePattern values which match Attribute
	// expressions for data accesses whose values are not yet known.
	UnknownAttributePatterns() []*AttributePattern
}

// partActivation is the default implementations of the PartialActivation interface.
type partActivation struct {
	Activation
	unknowns []*AttributePattern
}

// UnknownAttributePatterns implements the PartialActivation interface method.
func (a *partActivation) UnknownAttributePatterns() []*AttributePattern {
	return a.unknowns
}

// varActivation represents a single mutable variable binding.
//
// This activation type should only be used within folds as the fold loop controls the object
// life-cycle.
type varActivation struct {
	parent Activation
	name   string
	val    ref.Val
}

// Parent implements the Activation interface method.
func (v *varActivation) Parent() Activation {
	return v.parent
}

// ResolveName implements the Activation interface method.
func (v *varActivation) ResolveName(name string) (interface{}, bool) {
	if name == v.name {
		return v.val, true
	}
	return v.parent.ResolveName(name)
}

var (
	// pool of var activations to reduce allocations during folds.
	varActivationPool = &sync.Pool{
		New: func() interface{} {
			return &varActivation{}
		},
	}
)
<commit_msg>Use a zero-size type for variable-free activations (#513)<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package interpreter

import (
	"errors"
	"fmt"
	"sync"

	"github.com/google/cel-go/common/types/ref"
)

// Activation used to resolve identifiers by name and references by id.
//
// An Activation is the primary mechanism by which a caller supplies input into a CEL program.
type Activation interface {
	// ResolveName returns a value from the activation by qualified name, or false if the name
	// could not be found.
	ResolveName(name string) (interface{}, bool)

	// Parent returns the parent of the current activation, may be nil.
	// If non-nil, the parent will be searched during resolve calls.
	Parent() Activation
}

// EmptyActivation returns a variable-free activation.
func EmptyActivation() Activation {
	return emptyActivation{}
}

// emptyActivation is a variable-free activation.
type emptyActivation struct{}

func (emptyActivation) ResolveName(string) (interface{}, bool) { return nil, false }
func (emptyActivation) Parent() Activation                     { return nil }

// NewActivation returns an activation based on a map-based binding where the map keys are
// expected to be qualified names used with ResolveName calls.
//
// The input `bindings` may either be of type `Activation` or `map[string]interface{}`.
//
// Lazy bindings may be supplied within the map-based input in either of the following forms:
// - func() interface{}
// - func() ref.Val
//
// The output of the lazy binding will overwrite the variable reference in the internal map.
//
// Values which are not represented as ref.Val types on input may be adapted to a ref.Val using
// the ref.TypeAdapter configured in the environment.
func NewActivation(bindings interface{}) (Activation, error) {
	if bindings == nil {
		return nil, errors.New("bindings must be non-nil")
	}
	a, isActivation := bindings.(Activation)
	if isActivation {
		return a, nil
	}
	m, isMap := bindings.(map[string]interface{})
	if !isMap {
		return nil, fmt.Errorf(
			"activation input must be an activation or map[string]interface: got %T",
			bindings)
	}
	return &mapActivation{bindings: m}, nil
}

// mapActivation which implements Activation and maps of named values.
//
// Named bindings may lazily supply values by providing a function which accepts no arguments and
// produces an interface value.
type mapActivation struct {
	bindings map[string]interface{}
}

// Parent implements the Activation interface method.
func (a *mapActivation) Parent() Activation {
	return nil
}

// ResolveName implements the Activation interface method.
func (a *mapActivation) ResolveName(name string) (interface{}, bool) {
	obj, found := a.bindings[name]
	if !found {
		return nil, false
	}
	fn, isLazy := obj.(func() ref.Val)
	if isLazy {
		obj = fn()
		a.bindings[name] = obj
	}
	fnRaw, isLazy := obj.(func() interface{})
	if isLazy {
		obj = fnRaw()
		a.bindings[name] = obj
	}
	return obj, found
}

// hierarchicalActivation which implements Activation and contains a parent and
// child activation.
type hierarchicalActivation struct {
	parent Activation
	child  Activation
}

// Parent implements the Activation interface method.
func (a *hierarchicalActivation) Parent() Activation {
	return a.parent
}

// ResolveName implements the Activation interface method.
func (a *hierarchicalActivation) ResolveName(name string) (interface{}, bool) {
	if object, found := a.child.ResolveName(name); found {
		return object, found
	}
	return a.parent.ResolveName(name)
}

// NewHierarchicalActivation takes two activations and produces a new one which prioritizes
// resolution in the child first and parent(s) second.
func NewHierarchicalActivation(parent Activation, child Activation) Activation {
	return &hierarchicalActivation{parent, child}
}

// NewPartialActivation returns an Activation which contains a list of AttributePattern values
// representing field and index operations that should result in a 'types.Unknown' result.
//
// The `bindings` value may be any value type supported by the interpreter.NewActivation call,
// but is typically either an existing Activation or map[string]interface{}.
func NewPartialActivation(bindings interface{},
	unknowns ...*AttributePattern) (PartialActivation, error) {
	a, err := NewActivation(bindings)
	if err != nil {
		return nil, err
	}
	return &partActivation{Activation: a, unknowns: unknowns}, nil
}

// PartialActivation extends the Activation interface with a set of UnknownAttributePatterns.
type PartialActivation interface {
	Activation

	// UnknownAttributePaths returns a set of AttributePattern values which match Attribute
	// expressions for data accesses whose values are not yet known.
	UnknownAttributePatterns() []*AttributePattern
}

// partActivation is the default implementations of the PartialActivation interface.
type partActivation struct {
	Activation
	unknowns []*AttributePattern
}

// UnknownAttributePatterns implements the PartialActivation interface method.
func (a *partActivation) UnknownAttributePatterns() []*AttributePattern {
	return a.unknowns
}

// varActivation represents a single mutable variable binding.
//
// This activation type should only be used within folds as the fold loop controls the object
// life-cycle.
type varActivation struct {
	parent Activation
	name   string
	val    ref.Val
}

// Parent implements the Activation interface method.
func (v *varActivation) Parent() Activation {
	return v.parent
}

// ResolveName implements the Activation interface method.
func (v *varActivation) ResolveName(name string) (interface{}, bool) {
	if name == v.name {
		return v.val, true
	}
	return v.parent.ResolveName(name)
}

var (
	// pool of var activations to reduce allocations during folds.
	varActivationPool = &sync.Pool{
		New: func() interface{} {
			return &varActivation{}
		},
	}
)
<|endoftext|>


<commit_before>package controllers

import (
	"encoding/json"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/unrolled/render"

	"github.com/spesnova/iruka/registry"
	"github.com/spesnova/iruka/schema"
)

type AppController struct {
	*registry.Registry
	*render.Render
}

func NewAppController(reg *registry.Registry, ren *render.Render) AppController {
	return AppController{reg, ren}
}

func (c *AppController) Create(rw http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	var opts schema.AppCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		// TODO (spesnova): response better error
		c.JSON(rw, http.StatusInternalServerError, "error")
		return
	}

	app, err := c.CreateApp(opts)
	if err != nil {
		// TODO (spesnova): if the reqeust is invalid, server should returns 400 instead of 500
		//c.JSON(rw, http.StatusBadRequest, "error")

		// TODO (spesnova): response better error
		c.JSON(rw, http.StatusInternalServerError, "error")
		return
	}

	c.JSON(rw, http.StatusCreated, app)
}

func (c *AppController) Delete(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	idOrName := vars["idOrName"]

	app, err := c.DestroyApp(idOrName)
	if err != nil {
		// TODO (spesnova): separate 404 and 500 error
		c.JSON(rw, http.StatusInternalServerError, err.Error())
		return
	}

	c.JSON(rw, http.StatusOK, app)
}

func (c *AppController) Info(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	idOrName := vars["idOrName"]

	app, err := c.App(idOrName)
	if err != nil {
		// TODO (spesnova): separate 404 and 500 error
		c.JSON(rw, http.StatusInternalServerError, err.Error())
		return
	}

	c.JSON(rw, http.StatusOK, app)
}

func (c *AppController) List(rw http.ResponseWriter, r *http.Request) {
	apps, err := c.Apps()
	if err != nil {
		c.JSON(rw, http.StatusInternalServerError, "error")
		return
	}

	c.JSON(rw, http.StatusOK, apps)
}

func (c *AppController) Update(rw http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	vars := mux.Vars(r)
	idOrName := vars["idOrName"]

	var opts schema.AppUpdateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		// TODO (spesnova): response better error
		c.JSON(rw, http.StatusInternalServerError, "error")
		return
	}

	app, err := c.UpdateApp(idOrName, opts)
	if err != nil {
		// TODO (spesnova): if the reqeust is invalid, server should returns 400 instead of 500
		//c.JSON(rw, http.StatusBadRequest, "error")

		// TODO (spesnova): response better error
		c.JSON(rw, http.StatusInternalServerError, err.Error())
		return
	}

	c.JSON(rw, http.StatusCreated, app)
}
<commit_msg>api: Return appropriate status codes<commit_after>package controllers

import (
	"encoding/json"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/unrolled/render"

	"github.com/spesnova/iruka/registry"
	"github.com/spesnova/iruka/schema"
)

type AppController struct {
	*registry.Registry
	*render.Render
}

func NewAppController(reg *registry.Registry, ren *render.Render) AppController {
	return AppController{reg, ren}
}

func (c *AppController) Create(rw http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	var opts schema.AppCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		// TODO (spesnova): response better error
		c.JSON(rw, http.StatusInternalServerError, "error")
		return
	}

	app, err := c.CreateApp(opts)
	if err != nil {
		// TODO (spesnova): if the reqeust is invalid, server should returns 400 instead of 500
		//c.JSON(rw, http.StatusBadRequest, "error")

		// TODO (spesnova): response better error
		c.JSON(rw, http.StatusInternalServerError, "error")
		return
	}

	c.JSON(rw, http.StatusCreated, app)
}

func (c *AppController) Delete(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	idOrName := vars["idOrName"]

	app, err := c.DestroyApp(idOrName)
	if err != nil {
		// TODO (spesnova): separate 404 and 500 error
		c.JSON(rw, http.StatusInternalServerError, err.Error())
		return
	}

	c.JSON(rw, http.StatusAccepted, app)
}

func (c *AppController) Info(rw http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	idOrName := vars["idOrName"]

	app, err := c.App(idOrName)
	if err != nil {
		// TODO (spesnova): separate 404 and 500 error
		c.JSON(rw, http.StatusInternalServerError, err.Error())
		return
	}

	c.JSON(rw, http.StatusOK, app)
}

func (c *AppController) List(rw http.ResponseWriter, r *http.Request) {
	apps, err := c.Apps()
	if err != nil {
		c.JSON(rw, http.StatusInternalServerError, "error")
		return
	}

	c.JSON(rw, http.StatusOK, apps)
}

func (c *AppController) Update(rw http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	vars := mux.Vars(r)
	idOrName := vars["idOrName"]

	var opts schema.AppUpdateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		// TODO (spesnova): response better error
		c.JSON(rw, http.StatusInternalServerError, "error")
		return
	}

	app, err := c.UpdateApp(idOrName, opts)
	if err != nil {
		// TODO (spesnova): if the reqeust is invalid, server should returns 400 instead of 500
		//c.JSON(rw, http.StatusBadRequest, "error")

		// TODO (spesnova): response better error
		c.JSON(rw, http.StatusInternalServerError, err.Error())
		return
	}

	c.JSON(rw, http.StatusAccepted, app)
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package lxdclient

import (
	"github.com/juju/errors"
)

const (
	remoteLocalName   = "local"
	remoteDefaultName = remoteLocalName

	// TODO(ericsnow) This may be changing to "local"
	remoteIDForLocal = ""
)

// Local is LXD's default "remote". Essentially it is an unencrypted,
// unauthenticated connection to localhost over a unix socket.
var Local = Remote{
	Name: remoteLocalName,
	Host: "", // The LXD API turns this into the local unix socket.
	Cert: nil,
}

// Remote describes a LXD "remote" server for a client. In
// particular it holds the information needed for the client
// to connect to the remote.
type Remote struct {
	// Name is a label for this remote.
	Name string

	// Host identifies the host to which the client should connect.
	// An empty string is interpreted as:
	//   "localhost over a unix socket (unencrypted)".
	Host string

	// Cert holds the TLS certificate data for the client to use.
	Cert *Cert
}

// isLocal determines if the remote is the implicit "local" remote,
// an unencrypted, unauthenticated unix socket to a locally running LXD.
func (r Remote) isLocal() bool {
	if Local.Host != "" {
		logger.Errorf("%#v", Local)
	}
	return r.Host == Local.Host
}

// ID identifies the remote to the raw LXD client code. For the
// non-local case this is Remote.Name. For the local case it is the
// remote name that LXD special-cases for the local unix socket.
func (r Remote) ID() string {
	if r.isLocal() {
		return remoteIDForLocal
	}
	return r.Name
}

// SetDefaults updates a copy of the remote with default values
// where needed.
func (r Remote) SetDefaults() (Remote, error) {
	if r.isLocal() {
		return r.setLocalDefaults(), nil
	}

	if r.Cert == nil {
		certPEM, keyPEM, err := genCertAndKey()
		if err != nil {
			return r, errors.Trace(err)
		}
		r.Cert = NewCert(certPEM, keyPEM)
	}

	return r, nil
}

func (r Remote) setLocalDefaults() Remote {
	if r.Name == "" {
		r.Name = remoteLocalName
	}

	// TODO(ericsnow) Set r.Cert to nil?

	return r
}

// Validate checks the Remote fields for invalid values.
func (r Remote) Validate() error {
	if r.Name == "" {
		return errors.NotValidf("remote missing name,")
	}

	if r.isLocal() {
		if err := r.validateLocal(); err != nil {
			return errors.Trace(err)
		}
		return nil
	}

	// TODO(ericsnow) Ensure the host is a valid hostname or address?

	if r.Cert == nil {
		return errors.NotValidf("remote without cert")
	} else if err := r.Cert.Validate(); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (r Remote) validateLocal() error {
	if r.Cert != nil {
		return errors.NotValidf("hostless remote with cert")
	}

	return nil
}

// TODO(ericsnow) Add a "Connect(Config)" method that connects
// to the remote and returns the corresponding Client.

// TODO(ericsnow) Add a "Register" method to Client that adds the remote
// to the client's remote?
<commit_msg>Add a doc comment for remoteLocalName (and drop remoteDefaultName).<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package lxdclient

import (
	"github.com/juju/errors"
)

const (
	// remoteLocalName is a specific remote name in the default LXD config.
	// See https://github.com/lxc/lxd/blob/master/config.go:defaultRemote.
	remoteLocalName = "local"

	// TODO(ericsnow) This may be changing to "local"
	remoteIDForLocal = ""
)

// Local is LXD's default "remote". Essentially it is an unencrypted,
// unauthenticated connection to localhost over a unix socket.
var Local = Remote{
	Name: remoteLocalName,
	Host: "", // The LXD API turns this into the local unix socket.
	Cert: nil,
}

// Remote describes a LXD "remote" server for a client. In
// particular it holds the information needed for the client
// to connect to the remote.
type Remote struct {
	// Name is a label for this remote.
	Name string

	// Host identifies the host to which the client should connect.
	// An empty string is interpreted as:
	//   "localhost over a unix socket (unencrypted)".
	Host string

	// Cert holds the TLS certificate data for the client to use.
	Cert *Cert
}

// isLocal determines if the remote is the implicit "local" remote,
// an unencrypted, unauthenticated unix socket to a locally running LXD.
func (r Remote) isLocal() bool {
	if Local.Host != "" {
		logger.Errorf("%#v", Local)
	}
	return r.Host == Local.Host
}

// ID identifies the remote to the raw LXD client code. For the
// non-local case this is Remote.Name. For the local case it is the
// remote name that LXD special-cases for the local unix socket.
func (r Remote) ID() string {
	if r.isLocal() {
		return remoteIDForLocal
	}
	return r.Name
}

// SetDefaults updates a copy of the remote with default values
// where needed.
func (r Remote) SetDefaults() (Remote, error) {
	if r.isLocal() {
		return r.setLocalDefaults(), nil
	}

	if r.Cert == nil {
		certPEM, keyPEM, err := genCertAndKey()
		if err != nil {
			return r, errors.Trace(err)
		}
		r.Cert = NewCert(certPEM, keyPEM)
	}

	return r, nil
}

func (r Remote) setLocalDefaults() Remote {
	if r.Name == "" {
		r.Name = remoteLocalName
	}

	// TODO(ericsnow) Set r.Cert to nil?

	return r
}

// Validate checks the Remote fields for invalid values.
func (r Remote) Validate() error {
	if r.Name == "" {
		return errors.NotValidf("remote missing name,")
	}

	if r.isLocal() {
		if err := r.validateLocal(); err != nil {
			return errors.Trace(err)
		}
		return nil
	}

	// TODO(ericsnow) Ensure the host is a valid hostname or address?

	if r.Cert == nil {
		return errors.NotValidf("remote without cert")
	} else if err := r.Cert.Validate(); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (r Remote) validateLocal() error {
	if r.Cert != nil {
		return errors.NotValidf("hostless remote with cert")
	}

	return nil
}

// TODO(ericsnow) Add a "Connect(Config)" method that connects
// to the remote and returns the corresponding Client.

// TODO(ericsnow) Add a "Register" method to Client that adds the remote
// to the client's remote?
<|endoftext|>


<commit_before>package db

/*
stores the functions related to file IO and category
*/
import (
	"log"

	"github.com/thewhitetulip/Tasks/types"
)

// AddFile is used to add the md5 of a file name which is uploaded to our application
// this will enable us to randomize the URL without worrying about the file names
func AddFile(fileName, token string) error {
	SQL := database.prepare("insert into files values(?,?)")
	tx := database.begin()
	_, err = tx.Stmt(SQL).Exec(fileName, token)
	if err != nil {
		log.Println(err)
		tx.Rollback()
	} else {
		log.Println(tx.Commit())
	}
	return err
}

// GetFileName is used to fetch the name according to the md5 checksum from the db
func GetFileName(token string) (string, error) {
	sql := "select name from files where autoName=?"
	var fileName string
	rows := database.query(sql, fileName)
	if rows.Next() {
		err := rows.Scan(&fileName)
		if err != nil {
			log.Println(err)
			return "", err
		}
	}
	if err != nil {
		return "", err
	}

	return fileName, nil
}

//GetCategories will return the list of categories to be
//rendered in the template
func GetCategories() []types.CategoryCount {
	stmt := "select c.name, count(*) from  category c left outer join task t  where c.id = t.cat_id and t.is_deleted='N' and t.finish_date is null   group by name    union     select name, 0  from category where name not in (select distinct name from task t join category c on t.cat_id = c.id)"
	rows := database.query(stmt)
	var categories []types.CategoryCount
	var category types.CategoryCount

	for rows.Next() {
		err := rows.Scan(&category.Name, &category.Count)
		if err != nil {
			log.Println(err)
		}
		categories = append(categories, category)
	}
	return categories
}

//AddCategory is used to add the task in the database
func AddCategory(category string) error {
	err := taskQuery("insert into category(name) values(?)", category)
	return err
}

// GetCategoryByName will return the ID of that category passed as args
// used while inserting tasks into the table
func GetCategoryByName(category string) int {
	stmt := "select id from category where name=?"
	rows := database.query(stmt, category)
	var categoryID int

	for rows.Next() {
		err := rows.Scan(&categoryID)
		if err != nil {
			log.Println(err)
		}
	}
	return categoryID
}

//DeleteCategoryByName will be used to delete a category from the category page
func DeleteCategoryByName(category string) error {
	//first we delete entries from task and then from category
	categoryID := GetCategoryByName(category)
	query := "update task set cat_id = null where id =?"
	err := taskQuery(query, categoryID)
	if err == nil {
		err = taskQuery("delete from category where id=?", categoryID)
		if err != nil {
			return err
		}
	}
	return err
}

//UpdateCategoryByName will be used to delete a category from the category page
func UpdateCategoryByName(oldName, newName string) error {
	query := "update category set name = ? where name=?"
	log.Println(query)
	err := taskQuery(query, newName, oldName)

	return err

}
<commit_msg>show 0 count for category with no tasks in navigation drawer<commit_after>package db

/*
stores the functions related to file IO and category
*/
import (
	"log"

	"github.com/thewhitetulip/Tasks/types"
)

// AddFile is used to add the md5 of a file name which is uploaded to our application
// this will enable us to randomize the URL without worrying about the file names
func AddFile(fileName, token string) error {
	SQL := database.prepare("insert into files values(?,?)")
	tx := database.begin()
	_, err = tx.Stmt(SQL).Exec(fileName, token)
	if err != nil {
		log.Println(err)
		tx.Rollback()
	} else {
		log.Println(tx.Commit())
	}
	return err
}

// GetFileName is used to fetch the name according to the md5 checksum from the db
func GetFileName(token string) (string, error) {
	sql := "select name from files where autoName=?"
	var fileName string
	rows := database.query(sql, fileName)
	if rows.Next() {
		err := rows.Scan(&fileName)
		if err != nil {
			log.Println(err)
			return "", err
		}
	}
	if err != nil {
		return "", err
	}

	return fileName, nil
}

//GetCategories will return the list of categories to be
//rendered in the template
func GetCategories() []types.CategoryCount {
	stmt := "select c.name, count(*) from  category c left outer join task t  where c.id = t.cat_id and t.is_deleted='N' and t.finish_date is null   group by name    union     select name, 0  from category where name not in (select distinct name from task t join category c on t.cat_id = c.id and is_deleted!='Y')"
	rows := database.query(stmt)
	var categories []types.CategoryCount
	var category types.CategoryCount

	for rows.Next() {
		err := rows.Scan(&category.Name, &category.Count)
		if err != nil {
			log.Println(err)
		}
		categories = append(categories, category)
	}
	return categories
}

//AddCategory is used to add the task in the database
func AddCategory(category string) error {
	err := taskQuery("insert into category(name) values(?)", category)
	return err
}

// GetCategoryByName will return the ID of that category passed as args
// used while inserting tasks into the table
func GetCategoryByName(category string) int {
	stmt := "select id from category where name=?"
	rows := database.query(stmt, category)
	var categoryID int

	for rows.Next() {
		err := rows.Scan(&categoryID)
		if err != nil {
			log.Println(err)
		}
	}
	return categoryID
}

//DeleteCategoryByName will be used to delete a category from the category page
func DeleteCategoryByName(category string) error {
	//first we delete entries from task and then from category
	categoryID := GetCategoryByName(category)
	query := "update task set cat_id = null where id =?"
	err := taskQuery(query, categoryID)
	if err == nil {
		err = taskQuery("delete from category where id=?", categoryID)
		if err != nil {
			return err
		}
	}
	return err
}

//UpdateCategoryByName will be used to delete a category from the category page
func UpdateCategoryByName(oldName, newName string) error {
	query := "update category set name = ? where name=?"
	log.Println(query)
	err := taskQuery(query, newName, oldName)

	return err

}
<|endoftext|>


<commit_before>// +build !lambdabinary

package sparta

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"regexp"
	"strings"
	"text/template"

	"github.com/Sirupsen/logrus"
)

const (
	nodeColorService     = "#8C1B20"
	nodeColorEventSource = "#FBBB06"
	nodeColorLambda      = "#F58206"
	nodeColorAPIGateway  = "#D9A741"
	nodeNameAPIGateway   = "API Gateway"
)

// RE for sanitizing golang/JS layer
var reSanitizeMermaidNodeName = regexp.MustCompile("[\\W\\s]+")
var reSanitizeMermaidLabelValue = regexp.MustCompile("[\\{\\}\"\\[\\]']+")

func mermaidNodeName(sourceName string) string {
	return reSanitizeMermaidNodeName.ReplaceAllString(sourceName, "x")
}

func mermaidLabelValue(labelText string) string {
	return reSanitizeMermaidLabelValue.ReplaceAllString(labelText, "")
}

func writeNode(writer io.Writer, nodeName string, nodeColor string, extraStyles string) {
	if "" != extraStyles {
		extraStyles = fmt.Sprintf(",%s", extraStyles)
	}
	sanitizedName := mermaidNodeName(nodeName)
	fmt.Fprintf(writer, "style %s fill:%s,stroke:#000,stroke-width:1px%s;\n", sanitizedName, nodeColor, extraStyles)
	fmt.Fprintf(writer, "%s[%s]\n", sanitizedName, mermaidLabelValue(nodeName))
}

func writeLink(writer io.Writer, fromNode string, toNode string, label string) {
	sanitizedFrom := mermaidNodeName(fromNode)
	sanitizedTo := mermaidNodeName(toNode)

	if "" != label {
		fmt.Fprintf(writer, "%s-- \"%s\" -->%s\n", sanitizedFrom, mermaidLabelValue(label), sanitizedTo)
	} else {
		fmt.Fprintf(writer, "%s-->%s\n", sanitizedFrom, sanitizedTo)
	}
}

// Describe produces a graphical representation of a service's Lambda and data sources.  Typically
// automatically called as part of a compiled golang binary via the `describe` command
// line option.
func Describe(serviceName string,
	serviceDescription string,
	lambdaAWSInfos []*LambdaAWSInfo,
	api *API,
	s3Site *S3Site,
	outputWriter io.Writer,
	workflowHooks *WorkflowHooks,
	logger *logrus.Logger) error {

	validationErr := validateSpartaPreconditions(lambdaAWSInfos, logger)
	if validationErr != nil {
		return validationErr
	}

	var cloudFormationTemplate bytes.Buffer
	err := Provision(true,
		serviceName,
		serviceDescription,
		lambdaAWSInfos,
		api,
		s3Site,
		"S3Bucket",
		"N/A",
		&cloudFormationTemplate,
		workflowHooks,
		logger)
	if nil != err {
		return err
	}

	tmpl, err := template.New("description").Parse(_escFSMustString(false, "/resources/describe/template.html"))
	if err != nil {
		return errors.New(err.Error())
	}

	var b bytes.Buffer

	// Setup the root object
	writeNode(&b, serviceName, nodeColorService, "color:white,font-weight:bold,stroke-width:4px")

	for _, eachLambda := range lambdaAWSInfos {
		// Create the node...
		writeNode(&b, eachLambda.lambdaFnName, nodeColorLambda, "")
		writeLink(&b, eachLambda.lambdaFnName, serviceName, "")

		// Create permission & event mappings
		// functions declared in this
		for _, eachPermission := range eachLambda.Permissions {
			nodes, err := eachPermission.descriptionInfo()
			if nil != err {
				return err
			}

			for _, eachNode := range nodes {
				name := strings.TrimSpace(eachNode.Name)
				link := strings.TrimSpace(eachNode.Relation)
				// Style it to have the Amazon color
				nodeColor := eachNode.Color
				if "" == nodeColor {
					nodeColor = nodeColorEventSource
				}
				writeNode(&b, name, nodeColor, "border-style:dotted")
				writeLink(&b, name, eachLambda.lambdaFnName, strings.Replace(link, "\n", "

", -1))
			}
		}

		for _, eachEventSourceMapping := range eachLambda.EventSourceMappings {
			writeNode(&b, eachEventSourceMapping.EventSourceArn, nodeColorEventSource, "border-style:dotted")
			writeLink(&b, eachEventSourceMapping.EventSourceArn, eachLambda.lambdaFnName, "")
		}
	}

	// API?
	if nil != api {
		// Create the APIGateway virtual node && connect it to the application
		writeNode(&b, nodeNameAPIGateway, nodeColorAPIGateway, "")

		for _, eachResource := range api.resources {
			for eachMethod, _ := range eachResource.Methods {
				// Create the PATH node
				var nodeName = fmt.Sprintf("%s - %s", eachMethod, eachResource.pathPart)
				writeNode(&b, nodeName, nodeColorAPIGateway, "")
				writeLink(&b, nodeNameAPIGateway, nodeName, "")
				writeLink(&b, nodeName, eachResource.parentLambda.lambdaFnName, "")
			}
		}
	}

	params := struct {
		SpartaVersion          string
		ServiceName            string
		ServiceDescription     string
		CloudFormationTemplate string
		BootstrapCSS           string
		MermaidCSS             string
		HighlightsCSS          string
		JQueryJS               string
		BootstrapJS            string
		MermaidJS              string
		HighlightsJS           string
		MermaidData            string
	}{
		SpartaVersion,
		serviceName,
		serviceDescription,
		cloudFormationTemplate.String(),
		_escFSMustString(false, "/resources/bootstrap/lumen/bootstrap.min.css"),
		_escFSMustString(false, "/resources/mermaid/mermaid.css"),
		_escFSMustString(false, "/resources/highlights/styles/vs.css"),
		_escFSMustString(false, "/resources/jquery/jquery-2.1.4.min.js"),
		_escFSMustString(false, "/resources/bootstrap/js/bootstrap.min.js"),
		_escFSMustString(false, "/resources/mermaid/mermaid.min.js"),
		_escFSMustString(false, "/resources/highlights/highlight.pack.js"),
		b.String(),
	}

	return tmpl.Execute(outputWriter, params)
}
<commit_msg>Minor updates to node colors<commit_after>// +build !lambdabinary

package sparta

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"regexp"
	"strings"
	"text/template"

	"github.com/Sirupsen/logrus"
)

const (
	nodeColorService     = "#EFEFEF"
	nodeColorEventSource = "#FBBB06"
	nodeColorLambda      = "#F58206"
	nodeColorAPIGateway  = "#06B5F5"
	nodeNameAPIGateway   = "API Gateway"
)

// RE for sanitizing golang/JS layer
var reSanitizeMermaidNodeName = regexp.MustCompile("[\\W\\s]+")
var reSanitizeMermaidLabelValue = regexp.MustCompile("[\\{\\}\"\\[\\]']+")

func mermaidNodeName(sourceName string) string {
	return reSanitizeMermaidNodeName.ReplaceAllString(sourceName, "x")
}

func mermaidLabelValue(labelText string) string {
	return reSanitizeMermaidLabelValue.ReplaceAllString(labelText, "")
}

func writeNode(writer io.Writer, nodeName string, nodeColor string, extraStyles string) {
	if "" != extraStyles {
		extraStyles = fmt.Sprintf(",%s", extraStyles)
	}
	sanitizedName := mermaidNodeName(nodeName)
	fmt.Fprintf(writer, "style %s fill:%s,stroke:#000,stroke-width:1px%s;\n", sanitizedName, nodeColor, extraStyles)
	fmt.Fprintf(writer, "%s[%s]\n", sanitizedName, mermaidLabelValue(nodeName))
}

func writeLink(writer io.Writer, fromNode string, toNode string, label string) {
	sanitizedFrom := mermaidNodeName(fromNode)
	sanitizedTo := mermaidNodeName(toNode)

	if "" != label {
		fmt.Fprintf(writer, "%s-- \"%s\" -->%s\n", sanitizedFrom, mermaidLabelValue(label), sanitizedTo)
	} else {
		fmt.Fprintf(writer, "%s-->%s\n", sanitizedFrom, sanitizedTo)
	}
}

// Describe produces a graphical representation of a service's Lambda and data sources.  Typically
// automatically called as part of a compiled golang binary via the `describe` command
// line option.
func Describe(serviceName string,
	serviceDescription string,
	lambdaAWSInfos []*LambdaAWSInfo,
	api *API,
	s3Site *S3Site,
	outputWriter io.Writer,
	workflowHooks *WorkflowHooks,
	logger *logrus.Logger) error {

	validationErr := validateSpartaPreconditions(lambdaAWSInfos, logger)
	if validationErr != nil {
		return validationErr
	}

	var cloudFormationTemplate bytes.Buffer
	err := Provision(true,
		serviceName,
		serviceDescription,
		lambdaAWSInfos,
		api,
		s3Site,
		"S3Bucket",
		"N/A",
		&cloudFormationTemplate,
		workflowHooks,
		logger)
	if nil != err {
		return err
	}

	tmpl, err := template.New("description").Parse(_escFSMustString(false, "/resources/describe/template.html"))
	if err != nil {
		return errors.New(err.Error())
	}

	var b bytes.Buffer

	// Setup the root object
	writeNode(&b, serviceName, nodeColorService, "color:white,font-weight:bold,stroke-width:4px")

	for _, eachLambda := range lambdaAWSInfos {
		// Create the node...
		writeNode(&b, eachLambda.lambdaFnName, nodeColorLambda, "")
		writeLink(&b, eachLambda.lambdaFnName, serviceName, "")

		// Create permission & event mappings
		// functions declared in this
		for _, eachPermission := range eachLambda.Permissions {
			nodes, err := eachPermission.descriptionInfo()
			if nil != err {
				return err
			}

			for _, eachNode := range nodes {
				name := strings.TrimSpace(eachNode.Name)
				link := strings.TrimSpace(eachNode.Relation)
				// Style it to have the Amazon color
				nodeColor := eachNode.Color
				if "" == nodeColor {
					nodeColor = nodeColorEventSource
				}
				writeNode(&b, name, nodeColor, "border-style:dotted")
				writeLink(&b, name, eachLambda.lambdaFnName, strings.Replace(link, "\n", "

", -1))
			}
		}

		for _, eachEventSourceMapping := range eachLambda.EventSourceMappings {
			writeNode(&b, eachEventSourceMapping.EventSourceArn, nodeColorEventSource, "border-style:dotted")
			writeLink(&b, eachEventSourceMapping.EventSourceArn, eachLambda.lambdaFnName, "")
		}
	}

	// API?
	if nil != api {
		// Create the APIGateway virtual node && connect it to the application
		writeNode(&b, nodeNameAPIGateway, nodeColorAPIGateway, "")

		for _, eachResource := range api.resources {
			for eachMethod, _ := range eachResource.Methods {
				// Create the PATH node
				var nodeName = fmt.Sprintf("%s - %s", eachMethod, eachResource.pathPart)
				writeNode(&b, nodeName, nodeColorAPIGateway, "")
				writeLink(&b, nodeNameAPIGateway, nodeName, "")
				writeLink(&b, nodeName, eachResource.parentLambda.lambdaFnName, "")
			}
		}
	}

	params := struct {
		SpartaVersion          string
		ServiceName            string
		ServiceDescription     string
		CloudFormationTemplate string
		BootstrapCSS           string
		MermaidCSS             string
		HighlightsCSS          string
		JQueryJS               string
		BootstrapJS            string
		MermaidJS              string
		HighlightsJS           string
		MermaidData            string
	}{
		SpartaVersion,
		serviceName,
		serviceDescription,
		cloudFormationTemplate.String(),
		_escFSMustString(false, "/resources/bootstrap/lumen/bootstrap.min.css"),
		_escFSMustString(false, "/resources/mermaid/mermaid.css"),
		_escFSMustString(false, "/resources/highlights/styles/vs.css"),
		_escFSMustString(false, "/resources/jquery/jquery-2.1.4.min.js"),
		_escFSMustString(false, "/resources/bootstrap/js/bootstrap.min.js"),
		_escFSMustString(false, "/resources/mermaid/mermaid.min.js"),
		_escFSMustString(false, "/resources/highlights/highlight.pack.js"),
		b.String(),
	}

	return tmpl.Execute(outputWriter, params)
}
<|endoftext|>


<commit_before>package kafka

import (
        "github.com/Shopify/sarama"
        "encoding/json"
        "reflect"
        "bytes"
        "log"
)

const (
  bufferSize     = 256
  initialOffset  = sarama.OffsetOldest // always start listening for the latest event.
)

type Consumer struct {
  consumer           sarama.Consumer
  partitionConsumers []sarama.PartitionConsumer
  messages           chan *sarama.ConsumerMessage
}

func NewConsumer(brokers []string, topic string) *Consumer {
  config := sarama.NewConfig()
  config.Consumer.Return.Errors = true

  consumer, err := sarama.NewConsumer(brokers, config)
  if err != nil {
    log.Fatalln(err)
  }

  partitions, err := consumer.Partitions(topic)
  if err != nil {
    log.Printf("Failed to get the list of partitions: %v", err)
  }

  log.Printf("%v partitions found for topic %v", len(partitions), topic)

  partitionConsumers := make([]sarama.PartitionConsumer, len(partitions))
  messages := make(chan *sarama.ConsumerMessage, bufferSize)

  for _, partition := range partitions {

    partitionConsumer, err := consumer.ConsumePartition(topic, partition, initialOffset)

    if err != nil {
      log.Fatalf("Failed to start consumer for partition %v: %v", partition, err)
    }

    go func(partitionConsumer sarama.PartitionConsumer) {
      for message := range partitionConsumer.Messages() {
        messages <- message
      }
    }(partitionConsumer)

  }

  return &Consumer{
    consumer           : consumer,
    partitionConsumers : partitionConsumers,
    messages           : messages,
  }
}

// Consume messages and process them through the method pass in parameter
func (this *Consumer) Consume(eventType reflect.Type, factory func() interface{}, processEvent func(interface{})) {

  go func() {
      log.Println("Start consuming messages ...")

      for message := range this.messages {
        log.Printf("Received message with offset %v", message.Offset)

        b := bytes.SplitAfterN(message.Value[:], []byte {44}, 1)

        eventTypeFromMessage  := string(b[0][:])
        if eventType.Name() != eventTypeFromMessage {
          log.Printf("Message with type %v is ignored", string(b[0][:]))
          continue
        }

        event := factory()
        if err := json.Unmarshal(b[1], event) ; err != nil {
          log.Println("Cannot read event : ", err)
          continue
        }

        log.Printf("Process message with offset %v", message.Offset)

        processEvent(event)
      }
    }()
}

// Close stops processing messages and releases the corresponding resources
func (this *Consumer) Close() {

  log.Println("Done consuming messages")

  for _, partitionConsumer := range this.partitionConsumers {
    if err := partitionConsumer.Close(); err != nil {
      log.Printf("Failed to close partition consumer: ", err)
    }
  }

  if err := this.consumer.Close(); err != nil {
    log.Printf("Failed to shutdown kafka consumer cleanly: %v", err)
  }

  close(this.messages)

}<commit_msg>Fix Consumer<commit_after>package kafka

import (
        "github.com/Shopify/sarama"
        "encoding/json"
        "reflect"
        "bytes"
        "log"
)

const (
  bufferSize     = 256
  initialOffset  = sarama.OffsetOldest // always start listening for the latest event.
)

type Consumer struct {
  consumer           sarama.Consumer
  partitionConsumers []sarama.PartitionConsumer
  messages           chan *sarama.ConsumerMessage
}

func NewConsumer(brokers []string, topic string) *Consumer {
  config := sarama.NewConfig()
  config.Consumer.Return.Errors = true

  consumer, err := sarama.NewConsumer(brokers, config)
  if err != nil {
    log.Fatalln(err)
  }

  partitions, err := consumer.Partitions(topic)
  if err != nil {
    log.Printf("Failed to get the list of partitions: %v", err)
  }

  log.Printf("%v partitions found for topic %v", len(partitions), topic)

  partitionConsumers := make([]sarama.PartitionConsumer, len(partitions))
  messages := make(chan *sarama.ConsumerMessage, bufferSize)

  for _, partition := range partitions {

    partitionConsumer, err := consumer.ConsumePartition(topic, partition, initialOffset)

    if err != nil {
      log.Fatalf("Failed to start consumer for partition %v: %v", partition, err)
    }

    go func(partitionConsumer sarama.PartitionConsumer) {
      for message := range partitionConsumer.Messages() {
        messages <- message
      }
    }(partitionConsumer)

  }

  return &Consumer{
    consumer           : consumer,
    partitionConsumers : partitionConsumers,
    messages           : messages,
  }
}

// Consume messages and process them through the method pass in parameter
func (this *Consumer) Consume(eventType reflect.Type, factory func() interface{}, processEvent func(interface{})) {

  go func() {
      log.Println("Start consuming messages ...")

      for message := range this.messages {
        log.Printf("Received message with offset %v", message.Offset)

        b := bytes.SplitAfterN(message.Value[:], []byte{','}, 1)

        eventTypeFromMessage  := string(b[0])
        if eventType.Name() != eventTypeFromMessage {
          log.Printf("Message with type %v is ignored", string(b[0]))
          continue
        }

        event := factory()
        if err := json.Unmarshal(b[1], event) ; err != nil {
          log.Println("Cannot read event : ", err)
          continue
        }

        log.Printf("Process message with offset %v", message.Offset)

        processEvent(event)
      }
    }()
}

// Close stops processing messages and releases the corresponding resources
func (this *Consumer) Close() {

  log.Println("Done consuming messages")

  for _, partitionConsumer := range this.partitionConsumers {
    if err := partitionConsumer.Close(); err != nil {
      log.Printf("Failed to close partition consumer: ", err)
    }
  }

  if err := this.consumer.Close(); err != nil {
    log.Printf("Failed to shutdown kafka consumer cleanly: %v", err)
  }

  close(this.messages)

}<|endoftext|>


<commit_before>package docker

import (
	"sort"

	"github.com/fsouza/go-dockerclient"
)

//Allowed
const (
	NoSort = iota
	SortByContainerID
	SortByImage
	SortByStatus
	SortByName
)

//SortMode represents allowed modes to sort a container slice
type SortMode uint16

type byContainerID []docker.APIContainers
type byImage []docker.APIContainers
type byStatus []docker.APIContainers
type byName []docker.APIContainers

func (a byContainerID) Len() int           { return len(a) }
func (a byContainerID) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byContainerID) Less(i, j int) bool { return a[i].ID < a[j].ID }

func (a byImage) Len() int           { return len(a) }
func (a byImage) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byImage) Less(i, j int) bool { return a[i].Image < a[j].Image }

func (a byStatus) Len() int           { return len(a) }
func (a byStatus) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byStatus) Less(i, j int) bool { return a[i].Status < a[j].Status }

func (a byName) Len() int      { return len(a) }
func (a byName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byName) Less(i, j int) bool {
	if len(a[i].Names) > 0 {
		if len(a[j].Names) > 0 {
			return a[i].Names[0] < a[j].Names[0]
		}
		return true
	}
	return false
}

//Sort the given containers slice using the given mode
func Sort(containers []docker.APIContainers, mode SortMode) {
	switch mode {
	case SortByContainerID:
		sort.Sort(byContainerID(containers))
	case SortByImage:
		sort.Sort(byImage(containers))
	case SortByStatus:
		sort.Sort(byStatus(containers))
	case SortByName:
		sort.Sort(byName(containers))
	}
}
<commit_msg>Minor<commit_after>package docker

import (
	"sort"

	"github.com/fsouza/go-dockerclient"
)

//Allowed sort methods
const (
	NoSort = iota
	SortByContainerID
	SortByImage
	SortByStatus
	SortByName
)

//SortMode represents allowed modes to sort a container slice
type SortMode uint16

//TODO figure out how to avoid so much duplicated code
type byContainerID []docker.APIContainers
type byImage []docker.APIContainers
type byStatus []docker.APIContainers
type byName []docker.APIContainers

func (a byContainerID) Len() int           { return len(a) }
func (a byContainerID) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byContainerID) Less(i, j int) bool { return a[i].ID < a[j].ID }

func (a byImage) Len() int           { return len(a) }
func (a byImage) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byImage) Less(i, j int) bool { return a[i].Image < a[j].Image }

func (a byStatus) Len() int           { return len(a) }
func (a byStatus) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byStatus) Less(i, j int) bool { return a[i].Status < a[j].Status }

func (a byName) Len() int      { return len(a) }
func (a byName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byName) Less(i, j int) bool {
	if len(a[i].Names) > 0 {
		if len(a[j].Names) > 0 {
			return a[i].Names[0] < a[j].Names[0]
		}
		return true
	}
	return false
}

//Sort the given containers slice using the given mode
func Sort(containers []docker.APIContainers, mode SortMode) {
	switch mode {
	case SortByContainerID:
		sort.Sort(byContainerID(containers))
	case SortByImage:
		sort.Sort(byImage(containers))
	case SortByStatus:
		sort.Sort(byStatus(containers))
	case SortByName:
		sort.Sort(byName(containers))
	}
}
<|endoftext|>


<commit_before>package kinesis

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/AdRoll/goamz/aws"
	kin "github.com/AdRoll/goamz/kinesis"
	"github.com/mozilla-services/heka/pipeline"
	"time"
)

type KinesisOutput struct {
	auth   aws.Auth
	config *KinesisOutputConfig
	Client *kin.Kinesis
}

type KinesisOutputConfig struct {
	Region          string `toml:"region"`
	Stream          string `toml:"stream"`
	AccessKeyID     string `toml:"access_key_id"`
	SecretAccessKey string `toml:"secret_access_key"`
	Token           string `toml:"token"`
	PayloadOnly     bool   `toml:"payload_only"`
}

func (k *KinesisOutput) ConfigStruct() interface{} {
	return &KinesisOutputConfig{
		Region:          "us-east-1",
		Stream:          "",
		AccessKeyID:     "",
		SecretAccessKey: "",
		Token:           "",
	}
}

func (k *KinesisOutput) Init(config interface{}) error {
	k.config = config.(*KinesisOutputConfig)
	a, err := aws.GetAuth(k.config.AccessKeyID, k.config.SecretAccessKey, k.config.Token, time.Now())
	if err != nil {
		return fmt.Errorf("Error authenticating: %s", err)
	}
	k.auth = a

	region, ok := aws.Regions[k.config.Region]
	if !ok {
		return fmt.Errorf("Region does not exist: %s", k.config.Region)
	}

	k.Client = kin.New(k.auth, region)

	return nil
}

func (k *KinesisOutput) Run(or pipeline.OutputRunner, helper pipeline.PluginHelper) error {
	var (
		pack     *pipeline.PipelinePack
		contents []byte
		msg      []byte
		err      error
	)

	for pack = range or.InChan() {
		msg, err = or.Encode(pack)
		if err != nil {
			or.LogError(fmt.Errorf("Error encoding message: %s", err))
			pack.Recycle()
			continue
		}
		msg, err = base64.StdEncoding.DecodeString(string(msg))
		if err != nil {
			or.LogError(fmt.Errorf("Error decoding: %s", err))
			pack.Recycle()
			continue
		}
		if contents, err = json.Marshal(msg); err != nil {
			or.LogError(fmt.Errorf("Error marshalling: %s", err))
			pack.Recycle()
			continue
		} else {
			pk := fmt.Sprintf("%d-%s", pack.Message.Timestamp, pack.Message.Hostname)
			_, err = k.Client.PutRecord(k.config.Stream, pk, contents, "", "")
			if err != nil {
				or.LogError(fmt.Errorf("Error pushing message to Kinesis: %s", err))
				pack.Recycle()
				continue
			}
		}
		pack.Recycle()
	}

	return nil
}

func init() {
	pipeline.RegisterPlugin("KinesisOutput", func() interface{} { return new(KinesisOutput) })
}
<commit_msg>decode in the right place<commit_after>package kinesis

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/AdRoll/goamz/aws"
	kin "github.com/AdRoll/goamz/kinesis"
	"github.com/mozilla-services/heka/pipeline"
	"time"
)

type KinesisOutput struct {
	auth   aws.Auth
	config *KinesisOutputConfig
	Client *kin.Kinesis
}

type KinesisOutputConfig struct {
	Region          string `toml:"region"`
	Stream          string `toml:"stream"`
	AccessKeyID     string `toml:"access_key_id"`
	SecretAccessKey string `toml:"secret_access_key"`
	Token           string `toml:"token"`
	PayloadOnly     bool   `toml:"payload_only"`
}

func (k *KinesisOutput) ConfigStruct() interface{} {
	return &KinesisOutputConfig{
		Region:          "us-east-1",
		Stream:          "",
		AccessKeyID:     "",
		SecretAccessKey: "",
		Token:           "",
	}
}

func (k *KinesisOutput) Init(config interface{}) error {
	k.config = config.(*KinesisOutputConfig)
	a, err := aws.GetAuth(k.config.AccessKeyID, k.config.SecretAccessKey, k.config.Token, time.Now())
	if err != nil {
		return fmt.Errorf("Error authenticating: %s", err)
	}
	k.auth = a

	region, ok := aws.Regions[k.config.Region]
	if !ok {
		return fmt.Errorf("Region does not exist: %s", k.config.Region)
	}

	k.Client = kin.New(k.auth, region)

	return nil
}

func (k *KinesisOutput) Run(or pipeline.OutputRunner, helper pipeline.PluginHelper) error {
	var (
		pack     *pipeline.PipelinePack
		contents []byte
		msg      []byte
		err      error
	)

	for pack = range or.InChan() {
		msg, err = or.Encode(pack)
		if err != nil {
			or.LogError(fmt.Errorf("Error encoding message: %s", err))
			pack.Recycle()
			continue
		}
		if contents, err = json.Marshal(msg); err != nil {
			or.LogError(fmt.Errorf("Error marshalling: %s", err))
			pack.Recycle()
			continue
		} else {
			contents, err = base64.StdEncoding.DecodeString(string(contents))
			if err != nil {
				or.LogError(fmt.Errorf("Error decoding: %s", err))
				pack.Recycle()
				continue
			}
			pk := fmt.Sprintf("%d-%s", pack.Message.Timestamp, pack.Message.Hostname)
			_, err = k.Client.PutRecord(k.config.Stream, pk, contents, "", "")
			if err != nil {
				or.LogError(fmt.Errorf("Error pushing message to Kinesis: %s", err))
				pack.Recycle()
				continue
			}
		}
		pack.Recycle()
	}

	return nil
}

func init() {
	pipeline.RegisterPlugin("KinesisOutput", func() interface{} { return new(KinesisOutput) })
}
<|endoftext|>


<commit_before>package taskgraph

import (
	"log"

	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
)

// This interface is used by application during taskgraph configuration phase.
type Bootstrap interface {
	// These allow application developer to set the task configuration so framework
	// implementation knows which task to invoke at each node.
	SetTaskBuilder(taskBuilder TaskBuilder)

	// This allow the application to specify how tasks are connection at each epoch
	SetTopology(topology Topology)

	// After all the configure is done, driver need to call start so that all
	// nodes will get into the event loop to run the application.
	Start()
}

// Framework hides distributed system complexity and provides users convenience of
// high level features.
type Framework interface {
	// This allow the task implementation query its neighbors.
	GetTopology() Topology

	// Kill the framework itself.
	// As epoch changes, some nodes isn't needed anymore
	Kill()

	// Some task can inform all participating tasks to shutdown.
	// If successful, all tasks will be gracefully shutdown.
	// TODO: @param status
	ShutdownJob()

	GetLogger() *log.Logger

	// This is used to figure out taskid for current node
	GetTaskID() uint64

	// This is useful for task to inform the framework their status change.
	// metaData has to be really small, since it might be stored in etcd.
	// Set meta flag to notify meta to all nodes of linkType to this node.
	FlagMeta(ctx context.Context, linkType, meta string)

	// Some task can inform all participating tasks to new epoch
	IncEpoch(ctx context.Context)

	// Request data from task toID with specified linkType and meta.
	DataRequest(ctx context.Context, toID uint64, method string, input proto.Message)
	CheckGRPCContext(ctx context.Context) error
}

// Note that framework can decide how update can be done, and how to serve the updatelog.
type BackedUpFramework interface {
	// Ask framework to do update on this update on this task, which consists
	// of one primary and some backup copies.
	Update(taskID uint64, log UpdateLog)
}

type MasterBoot interface {
	SetTask(MasterTask)
	// Only master knows the global topology and makes decisions.
	SetTopology(Topo)
	// Blocking call to run the task until it finishes.
	Start()
}

type WorkerBoot interface {
	SetTask(WorkerTask)
	Start()
}

type GRPCHandlerInterceptor interface {
	// Currently grpc doesn't support interceptor functionality. We need to rely on user
	// to call this at handler implementation.
	// The workflow would be
	//   C:Notify -> S:Intercept -> S:OnNotify
	Intercept(ctx context.Context, method string, input proto.Message) (proto.Message, error)
}

// Master-worker paradigm:
// There're usually a master (we can make it fault tolerance) and a bunch of workers.
// Master is responsible for making global decision and assign work to individual workers.
// Startup:
// 1. master should start first.
// 2. workers start with a unique worker ID, and ask/notify the master for assignment.
//
// Why master and what should be done on master?
//   Only master can make global decisions. Master should store the states (initial, updated,
// completed, etc.) of each workers and make decisions when state changes. This is important
// when task restart happens and reset state to "initial".
//   The framework keep tracks of physical addresses of connected workers assuming that
// workers always talk to master first. Basically, the first time a worker do notify on
// master, framework should carry the address of grpc server in hidden and set it
// automatically.
//
// What about worker?
//   Workers do the actual computation and data flow.
//   The framework keeps track of master address(es) in etcd.

type MasterFrame interface {
	// User can use this interface to simplify sending the messages to worker. By keeping
	// track of workers' states, user can make decisions on logical worker and communicate it
	// using proto messages.
	NotifyWorker(ctx context.Context, workerID uint64, method string, input proto.Message) (proto.Message, error)
	GetWorkerAddr(workerID uint64) string
	IncEpoch(ctx context.Context)
	GRPCHandlerInterceptor
}

type WorkerFrame interface {
	// It usually send states, etc. information to master in order to get further decision.
	NotifyMaster(ctx context.Context, input proto.Message) (proto.Message, error)
	// This is to help user do data transfer.
	// The "addr" is a physical one instead of logical worker ID.
	// Addr is usually known from the master. When maser tell a worker to talk to another
	// worker, the place to go to should be a physical one. Any failure later should be
	// handled by master to give a new address.
	DataRequest(ctx context.Context, addr string, method string, input proto.Message) (proto.Message, error)
	GRPCHandlerInterceptor
}
<commit_msg>add epoch docs<commit_after>package taskgraph

import (
	"log"

	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
)

// This interface is used by application during taskgraph configuration phase.
type Bootstrap interface {
	// These allow application developer to set the task configuration so framework
	// implementation knows which task to invoke at each node.
	SetTaskBuilder(taskBuilder TaskBuilder)

	// This allow the application to specify how tasks are connection at each epoch
	SetTopology(topology Topology)

	// After all the configure is done, driver need to call start so that all
	// nodes will get into the event loop to run the application.
	Start()
}

// Framework hides distributed system complexity and provides users convenience of
// high level features.
type Framework interface {
	// This allow the task implementation query its neighbors.
	GetTopology() Topology

	// Kill the framework itself.
	// As epoch changes, some nodes isn't needed anymore
	Kill()

	// Some task can inform all participating tasks to shutdown.
	// If successful, all tasks will be gracefully shutdown.
	// TODO: @param status
	ShutdownJob()

	GetLogger() *log.Logger

	// This is used to figure out taskid for current node
	GetTaskID() uint64

	// This is useful for task to inform the framework their status change.
	// metaData has to be really small, since it might be stored in etcd.
	// Set meta flag to notify meta to all nodes of linkType to this node.
	FlagMeta(ctx context.Context, linkType, meta string)

	// Some task can inform all participating tasks to new epoch
	IncEpoch(ctx context.Context)

	// Request data from task toID with specified linkType and meta.
	DataRequest(ctx context.Context, toID uint64, method string, input proto.Message)
	CheckGRPCContext(ctx context.Context) error
}

// Note that framework can decide how update can be done, and how to serve the updatelog.
type BackedUpFramework interface {
	// Ask framework to do update on this update on this task, which consists
	// of one primary and some backup copies.
	Update(taskID uint64, log UpdateLog)
}

type MasterBoot interface {
	SetTask(MasterTask)
	// Only master knows the global topology and makes decisions.
	SetTopology(Topo)
	// Blocking call to run the task until it finishes.
	Start()
}

type WorkerBoot interface {
	SetTask(WorkerTask)
	Start()
}

type GRPCHandlerInterceptor interface {
	// Currently grpc doesn't support interceptor functionality. We need to rely on user
	// to call this at handler implementation.
	// The workflow would be
	//   C:Notify -> S:Intercept -> S:OnNotify
	Intercept(ctx context.Context, method string, input proto.Message) (proto.Message, error)
}

// Master-worker paradigm:
// There're usually a master (we can make it fault tolerance) and a bunch of workers.
// Master is responsible for making global decision and assign work to individual workers.
// Startup:
// 1. master should start first.
// 2. workers start with a unique worker ID, and ask/notify the master for assignment.
//
// Why master and what should be done on master?
//   Only master can make global decisions. Master should store the states (initial, updated,
// completed, etc.) of each workers and make decisions when state changes. This is important
// when task restart happens and reset state to "initial".
//   The framework keep tracks of physical addresses of connected workers assuming that
// workers always talk to master first. Basically, the first time a worker do notify on
// master, framework should carry the address of grpc server in hidden and set it
// automatically.
//
// What about worker?
//   Workers do the actual computation and data flow.
//   The framework keeps track of master address(es) in etcd.
//
// About epoch.
//   TaskGraph helps user synchronize epoch changes. When a task restarted and lagged off,
//   actions (usually the first one, NotifyMaster) will get a framework-specified error.
//   User can use EnterEpoch() callback to manage lifecycle. Epoch changes and synchronization
//   are handled by framework.

type MasterFrame interface {
	// User can use this interface to simplify sending the messages to worker. By keeping
	// track of workers' states, user can make decisions on logical worker and communicate it
	// using proto messages.
	NotifyWorker(ctx context.Context, workerID uint64, method string, input proto.Message) (proto.Message, error)
	GetWorkerAddr(workerID uint64) string
	IncEpoch(ctx context.Context)
	GRPCHandlerInterceptor
}

type WorkerFrame interface {
	// It usually send states, etc. information to master in order to get further decision.
	NotifyMaster(ctx context.Context, input proto.Message) (proto.Message, error)
	// This is to help user do data transfer.
	// The "addr" is a physical one instead of logical worker ID.
	// Addr is usually known from the master. When maser tell a worker to talk to another
	// worker, the place to go to should be a physical one. Any failure later should be
	// handled by master to give a new address.
	DataRequest(ctx context.Context, addr string, method string, input proto.Message) (proto.Message, error)
	GRPCHandlerInterceptor
}
<|endoftext|>