<commit_before>package libkbfs

import (
	"sync"

	"golang.org/x/net/context"
)

// favReq represents a request to access the logged-in user's
// favorites list.  A single request can do one or more of the
// following: refresh the current cached list, add a favorite, remove
// a favorite, and get all the favorites.  When the request is done,
// the resulting error (or nil) is sent over the done channel.  The
// given ctx is used for all network operations.
type favReq struct {
	// Request types
	refresh bool
	toAdd   []Favorite
	toDel   []Favorite
	favs    chan<- []Favorite

	// Closed when the request is done.
	done chan struct{}
	// Set before done is closed
	err error

	// Context
	ctx context.Context
}

// Favorites manages a user's favorite list.
type Favorites struct {
	config Config

	// Channels for interacting with the favorites cache
	reqChan chan *favReq

	// cache tracks the favorites for this user, that we know about.
	// It may not be consistent with the server's view of the user's
	// favorites list, if other devices have modified the list since
	// the last refresh.
	cache map[Favorite]bool

	inFlightLock sync.Mutex
	inFlightAdds map[Favorite]*favReq
}

func newFavoritesWithChan(config Config, reqChan chan *favReq) *Favorites {
	f := &Favorites{
		config:       config,
		reqChan:      reqChan,
		inFlightAdds: make(map[Favorite]*favReq),
	}
	go f.loop()
	return f
}

// NewFavorites constructs a new Favorites instance.
func NewFavorites(config Config) *Favorites {
	return newFavoritesWithChan(config, make(chan *favReq, 100))
}

func (f *Favorites) handleReq(req *favReq) (err error) {
	defer func() {
		f.inFlightLock.Lock()
		defer f.inFlightLock.Unlock()
		req.err = err
		close(req.done)
		for _, fav := range req.toAdd {
			delete(f.inFlightAdds, fav)
		}
	}()

	kbpki := f.config.KBPKI()
	// Fetch a new list if:
	//  * The user asked us to refresh
	//  * We haven't fetched it before
	//  * The user wants the list of favorites.  TODO: use the cached list
	//    once we have proper invalidation from the server.
	if req.refresh || f.cache == nil || req.favs != nil {
		folders, err := kbpki.FavoriteList(req.ctx)
		if err != nil {
			return err
		}

		f.cache = make(map[Favorite]bool)
		for _, folder := range folders {
			f.cache[*NewFavoriteFromFolder(folder)] = true
		}
		username, _, err := f.config.KBPKI().GetCurrentUserInfo(req.ctx)
		if err == nil {
			// Add favorites for the current user, that cannot be deleted.
			f.cache[Favorite{string(username), true}] = true
			f.cache[Favorite{string(username), false}] = true
		}
	}

	for _, fav := range req.toAdd {
		if f.cache[fav] {
			continue
		}
		err := kbpki.FavoriteAdd(req.ctx, fav.toKBFolder())
		if err != nil {
			f.config.MakeLogger("").CDebugf(req.ctx,
				"Failure adding favorite %v: %v", fav, err)
			return err
		}
		f.cache[fav] = true
	}

	for _, fav := range req.toDel {
		// Since our cache isn't necessarily up-to-date, always delete
		// the favorite.
		folder := fav.toKBFolder()
		err := kbpki.FavoriteDelete(req.ctx, folder)
		if err != nil {
			return err
		}
		if !folder.Private {
			// Public folders may be stored under a different name,
			// pending CORE-2695.  TODO: remove me!
			folder.Name = folder.Name + ReaderSep + "public"
			err := kbpki.FavoriteDelete(req.ctx, folder)
			if err != nil {
				return err
			}
		}
		delete(f.cache, fav)
	}

	if req.favs != nil {
		favorites := make([]Favorite, 0, len(f.cache))
		for fav := range f.cache {
			favorites = append(favorites, fav)
		}
		req.favs <- favorites
	}

	return nil
}

func (f *Favorites) loop() {
	for req := range f.reqChan {
		f.handleReq(req)
	}
}

// Shutdown shuts down this Favorites instance.
func (f *Favorites) Shutdown() {
	close(f.reqChan)
}

func (f *Favorites) waitOnReq(ctx context.Context,
	req *favReq) (retry bool, err error) {
	select {
	case <-ctx.Done():
		return false, ctx.Err()
	case <-req.done:
		err = req.err
		// If the request was canceled due to a context timeout that
		// wasn't our own, try it again.
		if err == context.Canceled || err == context.DeadlineExceeded {
			select {
			case <-ctx.Done():
				return false, err
			default:
				return true, nil
			}
		}
		return false, err
	}
}

func (f *Favorites) sendReq(ctx context.Context, req *favReq) error {
	select {
	case f.reqChan <- req:
	case <-ctx.Done():
		return ctx.Err()
	}
	// With a direct sendReq call, we'll never have a shared request,
	// so no need to check the retry status.
	_, err := f.waitOnReq(ctx, req)
	return err
}

func (f *Favorites) startOrJoinAddReq(
	ctx context.Context, fav Favorite) (req *favReq, doSend bool) {
	f.inFlightLock.Lock()
	defer f.inFlightLock.Unlock()
	req, ok := f.inFlightAdds[fav]
	if !ok {
		req = &favReq{
			ctx:   ctx,
			toAdd: []Favorite{fav},
			done:  make(chan struct{}),
		}
		f.inFlightAdds[fav] = req
		doSend = true
	}
	return req, doSend
}

// Add adds a favorite to your favorites list.
func (f *Favorites) Add(ctx context.Context, fav Favorite) error {
	doAdd := true
	var err error
	// Retry until we get an error that wasn't related to someone
	// else's context being canceled.
	for doAdd {
		req, doSend := f.startOrJoinAddReq(ctx, fav)
		if doSend {
			return f.sendReq(ctx, req)
		}
		doAdd, err = f.waitOnReq(ctx, req)
	}
	return err
}

// AddAsync initiates a request to add this favorite to your favorites
// list, if one is not already in flight, but it doesn't wait for the
// result.  (It could block while kicking off the request, if lots of
// different favorite operations are in flight.)
func (f *Favorites) AddAsync(ctx context.Context, fav Favorite) {
	// Use a fresh context, since we want the request to succeed even
	// if the original context is canceled.
	req, doSend := f.startOrJoinAddReq(context.Background(), fav)
	if doSend {
		select {
		case f.reqChan <- req:
		case <-ctx.Done():
			return
		}
	}
}

// Delete deletes a favorite from the favorites list.  It is
// idempotent.
func (f *Favorites) Delete(ctx context.Context, fav Favorite) error {
	return f.sendReq(ctx, &favReq{
		ctx:   ctx,
		toDel: []Favorite{fav},
		done:  make(chan struct{}),
	})
}

// RefreshCache refreshes the cached list of favorites.
func (f *Favorites) RefreshCache(ctx context.Context) {
	// This request is non-blocking, so use a throw-away done channel
	// and context.
	req := &favReq{
		refresh: true,
		done:    make(chan struct{}),
		ctx:     context.Background(),
	}
	select {
	case f.reqChan <- req:
	case <-ctx.Done():
		return
	}
}

// Get returns the logged-in users list of favorites. It
// doesn't use the cache.
func (f *Favorites) Get(ctx context.Context) ([]Favorite, error) {
	favChan := make(chan []Favorite, 1)
	req := &favReq{
		ctx:  ctx,
		favs: favChan,
		done: make(chan struct{}),
	}
	err := f.sendReq(ctx, req)
	if err != nil {
		return nil, err
	}
	return <-favChan, nil
}
<commit_msg>favorites: clarify context for AddAsync in comment<commit_after>package libkbfs

import (
	"sync"

	"golang.org/x/net/context"
)

// favReq represents a request to access the logged-in user's
// favorites list.  A single request can do one or more of the
// following: refresh the current cached list, add a favorite, remove
// a favorite, and get all the favorites.  When the request is done,
// the resulting error (or nil) is sent over the done channel.  The
// given ctx is used for all network operations.
type favReq struct {
	// Request types
	refresh bool
	toAdd   []Favorite
	toDel   []Favorite
	favs    chan<- []Favorite

	// Closed when the request is done.
	done chan struct{}
	// Set before done is closed
	err error

	// Context
	ctx context.Context
}

// Favorites manages a user's favorite list.
type Favorites struct {
	config Config

	// Channels for interacting with the favorites cache
	reqChan chan *favReq

	// cache tracks the favorites for this user, that we know about.
	// It may not be consistent with the server's view of the user's
	// favorites list, if other devices have modified the list since
	// the last refresh.
	cache map[Favorite]bool

	inFlightLock sync.Mutex
	inFlightAdds map[Favorite]*favReq
}

func newFavoritesWithChan(config Config, reqChan chan *favReq) *Favorites {
	f := &Favorites{
		config:       config,
		reqChan:      reqChan,
		inFlightAdds: make(map[Favorite]*favReq),
	}
	go f.loop()
	return f
}

// NewFavorites constructs a new Favorites instance.
func NewFavorites(config Config) *Favorites {
	return newFavoritesWithChan(config, make(chan *favReq, 100))
}

func (f *Favorites) handleReq(req *favReq) (err error) {
	defer func() {
		f.inFlightLock.Lock()
		defer f.inFlightLock.Unlock()
		req.err = err
		close(req.done)
		for _, fav := range req.toAdd {
			delete(f.inFlightAdds, fav)
		}
	}()

	kbpki := f.config.KBPKI()
	// Fetch a new list if:
	//  * The user asked us to refresh
	//  * We haven't fetched it before
	//  * The user wants the list of favorites.  TODO: use the cached list
	//    once we have proper invalidation from the server.
	if req.refresh || f.cache == nil || req.favs != nil {
		folders, err := kbpki.FavoriteList(req.ctx)
		if err != nil {
			return err
		}

		f.cache = make(map[Favorite]bool)
		for _, folder := range folders {
			f.cache[*NewFavoriteFromFolder(folder)] = true
		}
		username, _, err := f.config.KBPKI().GetCurrentUserInfo(req.ctx)
		if err == nil {
			// Add favorites for the current user, that cannot be deleted.
			f.cache[Favorite{string(username), true}] = true
			f.cache[Favorite{string(username), false}] = true
		}
	}

	for _, fav := range req.toAdd {
		if f.cache[fav] {
			continue
		}
		err := kbpki.FavoriteAdd(req.ctx, fav.toKBFolder())
		if err != nil {
			f.config.MakeLogger("").CDebugf(req.ctx,
				"Failure adding favorite %v: %v", fav, err)
			return err
		}
		f.cache[fav] = true
	}

	for _, fav := range req.toDel {
		// Since our cache isn't necessarily up-to-date, always delete
		// the favorite.
		folder := fav.toKBFolder()
		err := kbpki.FavoriteDelete(req.ctx, folder)
		if err != nil {
			return err
		}
		if !folder.Private {
			// Public folders may be stored under a different name,
			// pending CORE-2695.  TODO: remove me!
			folder.Name = folder.Name + ReaderSep + "public"
			err := kbpki.FavoriteDelete(req.ctx, folder)
			if err != nil {
				return err
			}
		}
		delete(f.cache, fav)
	}

	if req.favs != nil {
		favorites := make([]Favorite, 0, len(f.cache))
		for fav := range f.cache {
			favorites = append(favorites, fav)
		}
		req.favs <- favorites
	}

	return nil
}

func (f *Favorites) loop() {
	for req := range f.reqChan {
		f.handleReq(req)
	}
}

// Shutdown shuts down this Favorites instance.
func (f *Favorites) Shutdown() {
	close(f.reqChan)
}

func (f *Favorites) waitOnReq(ctx context.Context,
	req *favReq) (retry bool, err error) {
	select {
	case <-ctx.Done():
		return false, ctx.Err()
	case <-req.done:
		err = req.err
		// If the request was canceled due to a context timeout that
		// wasn't our own, try it again.
		if err == context.Canceled || err == context.DeadlineExceeded {
			select {
			case <-ctx.Done():
				return false, err
			default:
				return true, nil
			}
		}
		return false, err
	}
}

func (f *Favorites) sendReq(ctx context.Context, req *favReq) error {
	select {
	case f.reqChan <- req:
	case <-ctx.Done():
		return ctx.Err()
	}
	// With a direct sendReq call, we'll never have a shared request,
	// so no need to check the retry status.
	_, err := f.waitOnReq(ctx, req)
	return err
}

func (f *Favorites) startOrJoinAddReq(
	ctx context.Context, fav Favorite) (req *favReq, doSend bool) {
	f.inFlightLock.Lock()
	defer f.inFlightLock.Unlock()
	req, ok := f.inFlightAdds[fav]
	if !ok {
		req = &favReq{
			ctx:   ctx,
			toAdd: []Favorite{fav},
			done:  make(chan struct{}),
		}
		f.inFlightAdds[fav] = req
		doSend = true
	}
	return req, doSend
}

// Add adds a favorite to your favorites list.
func (f *Favorites) Add(ctx context.Context, fav Favorite) error {
	doAdd := true
	var err error
	// Retry until we get an error that wasn't related to someone
	// else's context being canceled.
	for doAdd {
		req, doSend := f.startOrJoinAddReq(ctx, fav)
		if doSend {
			return f.sendReq(ctx, req)
		}
		doAdd, err = f.waitOnReq(ctx, req)
	}
	return err
}

// AddAsync initiates a request to add this favorite to your favorites
// list, if one is not already in flight, but it doesn't wait for the
// result.  (It could block while kicking off the request, if lots of
// different favorite operations are in flight.)  The given context is
// used only for enqueuing the request on an internal queue, not for
// any resulting I/O.
func (f *Favorites) AddAsync(ctx context.Context, fav Favorite) {
	// Use a fresh context, since we want the request to succeed even
	// if the original context is canceled.
	req, doSend := f.startOrJoinAddReq(context.Background(), fav)
	if doSend {
		select {
		case f.reqChan <- req:
		case <-ctx.Done():
			return
		}
	}
}

// Delete deletes a favorite from the favorites list.  It is
// idempotent.
func (f *Favorites) Delete(ctx context.Context, fav Favorite) error {
	return f.sendReq(ctx, &favReq{
		ctx:   ctx,
		toDel: []Favorite{fav},
		done:  make(chan struct{}),
	})
}

// RefreshCache refreshes the cached list of favorites.
func (f *Favorites) RefreshCache(ctx context.Context) {
	// This request is non-blocking, so use a throw-away done channel
	// and context.
	req := &favReq{
		refresh: true,
		done:    make(chan struct{}),
		ctx:     context.Background(),
	}
	select {
	case f.reqChan <- req:
	case <-ctx.Done():
		return
	}
}

// Get returns the logged-in users list of favorites. It
// doesn't use the cache.
func (f *Favorites) Get(ctx context.Context) ([]Favorite, error) {
	favChan := make(chan []Favorite, 1)
	req := &favReq{
		ctx:  ctx,
		favs: favChan,
		done: make(chan struct{}),
	}
	err := f.sendReq(ctx, req)
	if err != nil {
		return nil, err
	}
	return <-favChan, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package bundler supports bundling (batching) of items. Bundling amortizes an
// action with fixed costs over multiple items. For example, if an API provides
// an RPC that accepts a list of items as input, but clients would prefer
// adding items one at a time, then a Bundler can accept individual items from
// the client and bundle many of them into a single RPC.
//
// This package is experimental and subject to change without notice.
package bundler

import (
	"errors"
	"reflect"
	"sync"
	"time"

	"golang.org/x/net/context"
)

const (
	DefaultDelayThreshold       = time.Second
	DefaultBundleCountThreshold = 10
	DefaultBundleByteThreshold  = 1e6 // 1M
	DefaultBufferedByteLimit    = 1e9 // 1G
)

var (
	// ErrOverflow indicates that Bundler's stored bytes exceeds its BufferedByteLimit.
	ErrOverflow = errors.New("bundler reached buffered byte limit")

	// ErrOversizedItem indicates that an item's size exceeds the maximum bundle size.
	ErrOversizedItem = errors.New("item size exceeds bundle byte limit")
)

// A Bundler collects items added to it into a bundle until the bundle
// exceeds a given size, then calls a user-provided function to handle the bundle.
type Bundler struct {
	// Starting from the time that the first message is added to a bundle, once
	// this delay has passed, handle the bundle. The default is DefaultDelayThreshold.
	DelayThreshold time.Duration

	// Once a bundle has this many items, handle the bundle. Since only one
	// item at a time is added to a bundle, no bundle will exceed this
	// threshold, so it also serves as a limit. The default is
	// DefaultBundleCountThreshold.
	BundleCountThreshold int

	// Once the number of bytes in current bundle reaches this threshold, handle
	// the bundle. The default is DefaultBundleByteThreshold. This triggers handling,
	// but does not cap the total size of a bundle.
	BundleByteThreshold int

	// The maximum size of a bundle, in bytes. Zero means unlimited.
	BundleByteLimit int

	// The maximum number of bytes that the Bundler will keep in memory before
	// returning ErrOverflow. The default is DefaultBufferedByteLimit.
	BufferedByteLimit int

	handler       func(interface{}) // called to handle a bundle
	itemSliceZero reflect.Value     // nil (zero value) for slice of items
	flushTimer    *time.Timer       // implements DelayThreshold

	mu             sync.Mutex
	spaceAvailable chan struct{}   // closed and replaced when space is available
	bufferedSize   int             // total bytes buffered
	curBundle      bundle          // incoming items added to this bundle
	handlingc      <-chan struct{} // set to non-nil while a handler is running; closed when it returns
}

type bundle struct {
	items reflect.Value // slice of item type
	size  int           // size in bytes of all items
}

// NewBundler creates a new Bundler.
//
// itemExample is a value of the type that will be bundled. For example, if you
// want to create bundles of *Entry, you could pass &Entry{} for itemExample.
//
// handler is a function that will be called on each bundle. If itemExample is
// of type T, the argument to handler is of type []T. handler is always called
// sequentially for each bundle, and never in parallel.
func NewBundler(itemExample interface{}, handler func(interface{})) *Bundler {
	b := &Bundler{
		DelayThreshold:       DefaultDelayThreshold,
		BundleCountThreshold: DefaultBundleCountThreshold,
		BundleByteThreshold:  DefaultBundleByteThreshold,
		BufferedByteLimit:    DefaultBufferedByteLimit,

		handler:       handler,
		itemSliceZero: reflect.Zero(reflect.SliceOf(reflect.TypeOf(itemExample))),
	}
	b.curBundle.items = b.itemSliceZero
	return b
}

// Add adds item to the current bundle. It marks the bundle for handling and
// starts a new one if any of the thresholds or limits are exceeded.
//
// If the item's size exceeds the maximum bundle size (Bundler.BundleByteLimit), then
// the item can never be handled. Add returns ErrOversizedItem in this case.
//
// If adding the item would exceed the maximum memory allowed (Bundler.BufferedByteLimit),
// Add returns ErrOverflow.
//
// Add never blocks.
func (b *Bundler) Add(item interface{}, size int) error {
	// If this item exceeds the maximum size of a bundle,
	// we can never send it.
	if b.BundleByteLimit > 0 && size > b.BundleByteLimit {
		return ErrOversizedItem
	}
	b.mu.Lock()
	defer b.mu.Unlock()
	// If adding this item would exceed our allotted memory
	// footprint, we can't accept it.
	if b.bufferedSize+size > b.BufferedByteLimit {
		return ErrOverflow
	}
	b.addLocked(item, size)
	return nil
}

// addLocked adds item to the current bundle. It marks the bundle for handling and
// starts a new one if any of the thresholds or limits are exceeded.
//
// addLocked is called with the lock held.
func (b *Bundler) addLocked(item interface{}, size int) {
	// If adding this item to the current bundle would cause it to exceed the
	// maximum bundle size, close the current bundle and start a new one.
	if b.BundleByteLimit > 0 && b.curBundle.size+size > b.BundleByteLimit {
		b.startFlushLocked()
	}
	// Add the item.
	b.curBundle.items = reflect.Append(b.curBundle.items, reflect.ValueOf(item))
	b.curBundle.size += size
	b.bufferedSize += size

	// Start a timer to flush the item if one isn't already running.
	// startFlushLocked clears the timer and closes the bundle at the same time,
	// so we only allocate a new timer for the first item in each bundle.
	// (We could try to call Reset on the timer instead, but that would add a lot
	// of complexity to the code just to save one small allocation.)
	if b.flushTimer == nil {
		b.flushTimer = time.AfterFunc(b.DelayThreshold, b.Flush)
	}

	// If the current bundle equals the count threshold, close it.
	if b.curBundle.items.Len() == b.BundleCountThreshold {
		b.startFlushLocked()
	}
	// If the current bundle equals or exceeds the byte threshold, close it.
	if b.curBundle.size >= b.BundleByteThreshold {
		b.startFlushLocked()
	}
}

// AddWait adds item to the current bundle. It marks the bundle for handling and
// starts a new one if any of the thresholds or limits are exceeded.
//
// If the item's size exceeds the maximum bundle size (Bundler.BundleByteLimit), then
// the item can never be handled. AddWait returns ErrOversizedItem in this case.
//
// If adding the item would exceed the maximum memory allowed (Bundler.BufferedByteLimit),
// AddWait blocks until space is available or ctx is done.
func (b *Bundler) AddWait(ctx context.Context, item interface{}, size int) error {
	// If this item exceeds the maximum size of a bundle,
	// we can never send it.
	if b.BundleByteLimit > 0 && size > b.BundleByteLimit {
		return ErrOversizedItem
	}
	b.mu.Lock()
	// If adding this item would exceed our allotted memory
	// footprint, block until space is available.
	// TODO(jba): avoid starvation of large items.
	for b.bufferedSize+size > b.BufferedByteLimit {
		if b.spaceAvailable == nil {
			b.spaceAvailable = make(chan struct{})
		}
		avail := b.spaceAvailable
		b.mu.Unlock()
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-avail:
			b.mu.Lock()
		}
	}
	b.addLocked(item, size)
	b.mu.Unlock()
	return nil
}

// Flush invokes the handler for all remaining items in the Bundler and waits
// for it to return.
func (b *Bundler) Flush() {
	b.mu.Lock()
	b.startFlushLocked()
	done := b.handlingc
	b.mu.Unlock()

	if done != nil {
		<-done
	}
}

func (b *Bundler) startFlushLocked() {
	if b.flushTimer != nil {
		b.flushTimer.Stop()
		b.flushTimer = nil
	}

	if b.curBundle.items.Len() == 0 {
		return
	}
	bun := b.curBundle
	b.curBundle = bundle{items: b.itemSliceZero}

	done := make(chan struct{})
	var running <-chan struct{}
	running, b.handlingc = b.handlingc, done

	go func() {
		defer func() {
			b.mu.Lock()
			b.bufferedSize -= bun.size
			avail := b.spaceAvailable
			b.spaceAvailable = nil
			b.mu.Unlock()

			if avail != nil {
				close(avail)
			}
			close(done)
		}()

		if running != nil {
			// Wait for our turn to call the handler.
			<-running
		}

		b.handler(bun.items.Interface())
	}()
}

// Stop is deprecated.  Use Flush instead.
func (b *Bundler) Stop() {
	b.Flush()
}
<commit_msg>bundler: remove deprecated Stop method<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package bundler supports bundling (batching) of items. Bundling amortizes an
// action with fixed costs over multiple items. For example, if an API provides
// an RPC that accepts a list of items as input, but clients would prefer
// adding items one at a time, then a Bundler can accept individual items from
// the client and bundle many of them into a single RPC.
//
// This package is experimental and subject to change without notice.
package bundler

import (
	"errors"
	"reflect"
	"sync"
	"time"

	"golang.org/x/net/context"
)

const (
	DefaultDelayThreshold       = time.Second
	DefaultBundleCountThreshold = 10
	DefaultBundleByteThreshold  = 1e6 // 1M
	DefaultBufferedByteLimit    = 1e9 // 1G
)

var (
	// ErrOverflow indicates that Bundler's stored bytes exceeds its BufferedByteLimit.
	ErrOverflow = errors.New("bundler reached buffered byte limit")

	// ErrOversizedItem indicates that an item's size exceeds the maximum bundle size.
	ErrOversizedItem = errors.New("item size exceeds bundle byte limit")
)

// A Bundler collects items added to it into a bundle until the bundle
// exceeds a given size, then calls a user-provided function to handle the bundle.
type Bundler struct {
	// Starting from the time that the first message is added to a bundle, once
	// this delay has passed, handle the bundle. The default is DefaultDelayThreshold.
	DelayThreshold time.Duration

	// Once a bundle has this many items, handle the bundle. Since only one
	// item at a time is added to a bundle, no bundle will exceed this
	// threshold, so it also serves as a limit. The default is
	// DefaultBundleCountThreshold.
	BundleCountThreshold int

	// Once the number of bytes in current bundle reaches this threshold, handle
	// the bundle. The default is DefaultBundleByteThreshold. This triggers handling,
	// but does not cap the total size of a bundle.
	BundleByteThreshold int

	// The maximum size of a bundle, in bytes. Zero means unlimited.
	BundleByteLimit int

	// The maximum number of bytes that the Bundler will keep in memory before
	// returning ErrOverflow. The default is DefaultBufferedByteLimit.
	BufferedByteLimit int

	handler       func(interface{}) // called to handle a bundle
	itemSliceZero reflect.Value     // nil (zero value) for slice of items
	flushTimer    *time.Timer       // implements DelayThreshold

	mu             sync.Mutex
	spaceAvailable chan struct{}   // closed and replaced when space is available
	bufferedSize   int             // total bytes buffered
	curBundle      bundle          // incoming items added to this bundle
	handlingc      <-chan struct{} // set to non-nil while a handler is running; closed when it returns
}

type bundle struct {
	items reflect.Value // slice of item type
	size  int           // size in bytes of all items
}

// NewBundler creates a new Bundler.
//
// itemExample is a value of the type that will be bundled. For example, if you
// want to create bundles of *Entry, you could pass &Entry{} for itemExample.
//
// handler is a function that will be called on each bundle. If itemExample is
// of type T, the argument to handler is of type []T. handler is always called
// sequentially for each bundle, and never in parallel.
func NewBundler(itemExample interface{}, handler func(interface{})) *Bundler {
	b := &Bundler{
		DelayThreshold:       DefaultDelayThreshold,
		BundleCountThreshold: DefaultBundleCountThreshold,
		BundleByteThreshold:  DefaultBundleByteThreshold,
		BufferedByteLimit:    DefaultBufferedByteLimit,

		handler:       handler,
		itemSliceZero: reflect.Zero(reflect.SliceOf(reflect.TypeOf(itemExample))),
	}
	b.curBundle.items = b.itemSliceZero
	return b
}

// Add adds item to the current bundle. It marks the bundle for handling and
// starts a new one if any of the thresholds or limits are exceeded.
//
// If the item's size exceeds the maximum bundle size (Bundler.BundleByteLimit), then
// the item can never be handled. Add returns ErrOversizedItem in this case.
//
// If adding the item would exceed the maximum memory allowed (Bundler.BufferedByteLimit),
// Add returns ErrOverflow.
//
// Add never blocks.
func (b *Bundler) Add(item interface{}, size int) error {
	// If this item exceeds the maximum size of a bundle,
	// we can never send it.
	if b.BundleByteLimit > 0 && size > b.BundleByteLimit {
		return ErrOversizedItem
	}
	b.mu.Lock()
	defer b.mu.Unlock()
	// If adding this item would exceed our allotted memory
	// footprint, we can't accept it.
	if b.bufferedSize+size > b.BufferedByteLimit {
		return ErrOverflow
	}
	b.addLocked(item, size)
	return nil
}

// addLocked adds item to the current bundle. It marks the bundle for handling and
// starts a new one if any of the thresholds or limits are exceeded.
//
// addLocked is called with the lock held.
func (b *Bundler) addLocked(item interface{}, size int) {
	// If adding this item to the current bundle would cause it to exceed the
	// maximum bundle size, close the current bundle and start a new one.
	if b.BundleByteLimit > 0 && b.curBundle.size+size > b.BundleByteLimit {
		b.startFlushLocked()
	}
	// Add the item.
	b.curBundle.items = reflect.Append(b.curBundle.items, reflect.ValueOf(item))
	b.curBundle.size += size
	b.bufferedSize += size

	// Start a timer to flush the item if one isn't already running.
	// startFlushLocked clears the timer and closes the bundle at the same time,
	// so we only allocate a new timer for the first item in each bundle.
	// (We could try to call Reset on the timer instead, but that would add a lot
	// of complexity to the code just to save one small allocation.)
	if b.flushTimer == nil {
		b.flushTimer = time.AfterFunc(b.DelayThreshold, b.Flush)
	}

	// If the current bundle equals the count threshold, close it.
	if b.curBundle.items.Len() == b.BundleCountThreshold {
		b.startFlushLocked()
	}
	// If the current bundle equals or exceeds the byte threshold, close it.
	if b.curBundle.size >= b.BundleByteThreshold {
		b.startFlushLocked()
	}
}

// AddWait adds item to the current bundle. It marks the bundle for handling and
// starts a new one if any of the thresholds or limits are exceeded.
//
// If the item's size exceeds the maximum bundle size (Bundler.BundleByteLimit), then
// the item can never be handled. AddWait returns ErrOversizedItem in this case.
//
// If adding the item would exceed the maximum memory allowed (Bundler.BufferedByteLimit),
// AddWait blocks until space is available or ctx is done.
func (b *Bundler) AddWait(ctx context.Context, item interface{}, size int) error {
	// If this item exceeds the maximum size of a bundle,
	// we can never send it.
	if b.BundleByteLimit > 0 && size > b.BundleByteLimit {
		return ErrOversizedItem
	}
	b.mu.Lock()
	// If adding this item would exceed our allotted memory
	// footprint, block until space is available.
	// TODO(jba): avoid starvation of large items.
	for b.bufferedSize+size > b.BufferedByteLimit {
		if b.spaceAvailable == nil {
			b.spaceAvailable = make(chan struct{})
		}
		avail := b.spaceAvailable
		b.mu.Unlock()
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-avail:
			b.mu.Lock()
		}
	}
	b.addLocked(item, size)
	b.mu.Unlock()
	return nil
}

// Flush invokes the handler for all remaining items in the Bundler and waits
// for it to return.
func (b *Bundler) Flush() {
	b.mu.Lock()
	b.startFlushLocked()
	done := b.handlingc
	b.mu.Unlock()

	if done != nil {
		<-done
	}
}

func (b *Bundler) startFlushLocked() {
	if b.flushTimer != nil {
		b.flushTimer.Stop()
		b.flushTimer = nil
	}

	if b.curBundle.items.Len() == 0 {
		return
	}
	bun := b.curBundle
	b.curBundle = bundle{items: b.itemSliceZero}

	done := make(chan struct{})
	var running <-chan struct{}
	running, b.handlingc = b.handlingc, done

	go func() {
		defer func() {
			b.mu.Lock()
			b.bufferedSize -= bun.size
			avail := b.spaceAvailable
			b.spaceAvailable = nil
			b.mu.Unlock()

			if avail != nil {
				close(avail)
			}
			close(done)
		}()

		if running != nil {
			// Wait for our turn to call the handler.
			<-running
		}

		b.handler(bun.items.Interface())
	}()
}
<|endoftext|>


<commit_before>package scheduler

import (
	"io"

	"github.com/faststackco/machinestack/config"
	"github.com/faststackco/machinestack/driver"
)

// LocalScheduler runs all machine on localhost
type LocalScheduler struct {
	driverOptions *config.DriverOptions
}

// NewLocalScheduler creates a new LocalScheduler
func NewLocalScheduler(options *config.DriverOptions) (Scheduler, error) {
	return &LocalScheduler{
		driverOptions: options,
	}, nil
}

// Create creates a new machine
func (c *LocalScheduler) Create(name, image, driverName string, attrs driver.MachineAttributes) (string, error) {
	driver, err := driver.NewDriver(name, *c.driverOptions)
	if err != nil {
		return "", err
	}

	if err := driver.Create(name, image, attrs); err != nil {
		return "", err
	}

	return "", nil
}

// Delete deletes a machine
func (c *LocalScheduler) Delete(name, driverName, node string) error {

	driver, err := driver.NewDriver(driverName, *c.driverOptions)
	if err != nil {
		return err
	}

	if err := driver.Delete(name); err != nil {
		return err
	}

	return nil
}

// Exec creates an new exec session
func (c *LocalScheduler) Exec(name, driverName, node string, stdin io.ReadCloser, stdout io.WriteCloser, control chan driver.ControlMessage) error {
	driver, err := driver.NewDriver(driverName, *c.driverOptions)
	if err != nil {
		return err
	}

	return driver.Exec(name, stdin, stdout, control)
}
<commit_msg>fix typo<commit_after>package scheduler

import (
	"io"

	"github.com/faststackco/machinestack/config"
	"github.com/faststackco/machinestack/driver"
)

// LocalScheduler runs all machine on localhost
type LocalScheduler struct {
	driverOptions *config.DriverOptions
}

// NewLocalScheduler creates a new LocalScheduler
func NewLocalScheduler(options *config.DriverOptions) (Scheduler, error) {
	return &LocalScheduler{
		driverOptions: options,
	}, nil
}

// Create creates a new machine
func (c *LocalScheduler) Create(name, image, driverName string, attrs driver.MachineAttributes) (string, error) {
	driver, err := driver.NewDriver(driverName, *c.driverOptions)
	if err != nil {
		return "", err
	}

	if err := driver.Create(name, image, attrs); err != nil {
		return "", err
	}

	return "", nil
}

// Delete deletes a machine
func (c *LocalScheduler) Delete(name, driverName, node string) error {

	driver, err := driver.NewDriver(driverName, *c.driverOptions)
	if err != nil {
		return err
	}

	if err := driver.Delete(name); err != nil {
		return err
	}

	return nil
}

// Exec creates an new exec session
func (c *LocalScheduler) Exec(name, driverName, node string, stdin io.ReadCloser, stdout io.WriteCloser, control chan driver.ControlMessage) error {
	driver, err := driver.NewDriver(driverName, *c.driverOptions)
	if err != nil {
		return err
	}

	return driver.Exec(name, stdin, stdout, control)
}
<|endoftext|>


<commit_before>package scheduler

import (
	//"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/Dataman-Cloud/swan/mesosproto/mesos"
	"github.com/Dataman-Cloud/swan/mesosproto/sched"
	"github.com/Dataman-Cloud/swan/types"

	"github.com/Sirupsen/logrus"
	"github.com/golang/protobuf/proto"
)

func (s *Scheduler) BuildTask(offer *mesos.Offer, version *types.Version, name string) (*types.Task, error) {
	var task types.Task

	task.Name = name
	if task.Name == "" {
		app, err := s.registry.FetchApplication(version.ID)
		if err != nil {
			return nil, err
		}

		task.Name = fmt.Sprintf("%d.%s.%s.%s", app.Instances, app.ID, app.UserId, app.ClusterId)

		if err := s.registry.IncreaseApplicationInstances(app.ID); err != nil {
			return nil, err
		}
	}

	task.AppId = version.ID
	task.ID = fmt.Sprintf("%d-%s", time.Now().UnixNano(), task.Name)

	task.Image = version.Container.Docker.Image
	task.Network = version.Container.Docker.Network

	if version.Container.Docker.Parameters != nil {
		for _, parameter := range *version.Container.Docker.Parameters {
			task.Parameters = append(task.Parameters, &types.Parameter{
				Key:   parameter.Key,
				Value: parameter.Value,
			})
		}
	}

	if version.Container.Docker.PortMappings != nil {
		for _, portMapping := range *version.Container.Docker.PortMappings {
			task.PortMappings = append(task.PortMappings, &types.PortMappings{
				Port:     uint32(portMapping.ContainerPort),
				Protocol: portMapping.Protocol,
			})
		}
	}

	if version.Container.Docker.Privileged != nil {
		task.Privileged = version.Container.Docker.Privileged
	}

	if version.Container.Docker.ForcePullImage != nil {
		task.ForcePullImage = version.Container.Docker.ForcePullImage
	}

	task.Env = version.Env

	task.Volumes = version.Container.Volumes

	if version.Labels != nil {
		task.Labels = version.Labels
	}

	task.Cpus = version.Cpus
	task.Mem = version.Mem
	task.Disk = version.Disk

	task.OfferId = offer.GetId().Value
	task.AgentId = offer.AgentId.Value
	task.AgentHostname = offer.Hostname

	if version.KillPolicy != nil {
		task.KillPolicy = version.KillPolicy
	}

	if version.HealthChecks != nil {
		task.HealthChecks = version.HealthChecks
	}

	return &task, nil
}

func (s *Scheduler) BuildTaskInfo(offer *mesos.Offer, resources []*mesos.Resource, task *types.Task) *mesos.TaskInfo {
	logrus.Infof("Prepared task for launch with offer %s", *offer.GetId().Value)
	taskInfo := mesos.TaskInfo{
		Name: proto.String(task.Name),
		TaskId: &mesos.TaskID{
			Value: proto.String(task.ID),
		},
		AgentId:   offer.AgentId,
		Resources: resources,
		Command: &mesos.CommandInfo{
			Shell: proto.Bool(false),
			Value: nil,
		},
		Container: &mesos.ContainerInfo{
			Type: mesos.ContainerInfo_DOCKER.Enum(),
			Docker: &mesos.ContainerInfo_DockerInfo{
				Image: task.Image,
			},
		},
	}

	if task.Privileged != nil {
		taskInfo.Container.Docker.Privileged = task.Privileged
	}

	if task.ForcePullImage != nil {
		taskInfo.Container.Docker.ForcePullImage = task.ForcePullImage
	}

	for _, parameter := range task.Parameters {
		taskInfo.Container.Docker.Parameters = append(taskInfo.Container.Docker.Parameters, &mesos.Parameter{
			Key:   proto.String(parameter.Key),
			Value: proto.String(parameter.Value),
		})
	}

	for _, volume := range task.Volumes {
		mode := mesos.Volume_RO
		if volume.Mode == "RW" {
			mode = mesos.Volume_RW
		}
		taskInfo.Container.Volumes = append(taskInfo.Container.Volumes, &mesos.Volume{
			ContainerPath: proto.String(volume.ContainerPath),
			HostPath:      proto.String(volume.HostPath),
			Mode:          &mode,
		})
	}

	vars := make([]*mesos.Environment_Variable, 0)
	for k, v := range task.Env {
		vars = append(vars, &mesos.Environment_Variable{
			Name:  proto.String(k),
			Value: proto.String(v),
		})
	}

	taskInfo.Command.Environment = &mesos.Environment{
		Variables: vars,
	}

	if task.Labels != nil {
		labels := make([]*mesos.Label, 0)
		for k, v := range *task.Labels {
			labels = append(labels, &mesos.Label{
				Key:   proto.String(k),
				Value: proto.String(v),
			})
		}

		taskInfo.Labels = &mesos.Labels{
			Labels: labels,
		}
	}

	switch task.Network {
	case "NONE":
		taskInfo.Container.Docker.Network = mesos.ContainerInfo_DockerInfo_NONE.Enum()
	case "HOST":
		taskInfo.Container.Docker.Network = mesos.ContainerInfo_DockerInfo_HOST.Enum()
	case "BRIDGE":
		ports := GetPorts(offer)
		if len(ports) == 0 {
			logrus.Errorf("No ports resource defined")
			break
		}
		for _, m := range task.PortMappings {
			hostPort := ports[s.TaskLaunched]
			taskInfo.Container.Docker.PortMappings = append(taskInfo.Container.Docker.PortMappings,
				&mesos.ContainerInfo_DockerInfo_PortMapping{
					HostPort:      proto.Uint32(uint32(hostPort)),
					ContainerPort: proto.Uint32(m.Port),
					Protocol:      proto.String(m.Protocol),
				},
			)
			taskInfo.Resources = append(taskInfo.Resources, &mesos.Resource{
				Name: proto.String("ports"),
				Type: mesos.Value_RANGES.Enum(),
				Ranges: &mesos.Value_Ranges{
					Range: []*mesos.Value_Range{
						{
							Begin: proto.Uint64(uint64(hostPort)),
							End:   proto.Uint64(uint64(hostPort)),
						},
					},
				},
			})
		}
		taskInfo.Container.Docker.Network = mesos.ContainerInfo_DockerInfo_BRIDGE.Enum()
	default:
		taskInfo.Container.Docker.Network = mesos.ContainerInfo_DockerInfo_NONE.Enum()
	}

	return &taskInfo
}

// LaunchTasks lauch multiple tasks with specified offer.
func (s *Scheduler) LaunchTasks(offer *mesos.Offer, tasks []*mesos.TaskInfo) (*http.Response, error) {
	logrus.Infof("Launch %d tasks with offer %s", len(tasks), *offer.GetId().Value)
	call := &sched.Call{
		FrameworkId: s.framework.GetId(),
		Type:        sched.Call_ACCEPT.Enum(),
		Accept: &sched.Call_Accept{
			OfferIds: []*mesos.OfferID{
				offer.GetId(),
			},
			Operations: []*mesos.Offer_Operation{
				&mesos.Offer_Operation{
					Type: mesos.Offer_Operation_LAUNCH.Enum(),
					Launch: &mesos.Offer_Operation_Launch{
						TaskInfos: tasks,
					},
				},
			},
			Filters: &mesos.Filters{RefuseSeconds: proto.Float64(1)},
		},
	}

	return s.send(call)
}

func (s *Scheduler) KillTask(task *types.Task) (*http.Response, error) {
	logrus.Infof("Kill task %s", task.Name)
	call := &sched.Call{
		FrameworkId: s.framework.GetId(),
		Type:        sched.Call_KILL.Enum(),
		Kill: &sched.Call_Kill{
			TaskId: &mesos.TaskID{
				Value: proto.String(task.ID),
			},
			AgentId: &mesos.AgentID{
				Value: task.AgentId,
			},
		},
	}

	if task.KillPolicy != nil {
		if task.KillPolicy.Duration != 0 {
			call.Kill.KillPolicy = &mesos.KillPolicy{
				GracePeriod: &mesos.DurationInfo{
					Nanoseconds: proto.Int64(task.KillPolicy.Duration * 1000 * 1000),
				},
			}
		}
	}

	return s.send(call)
}

// ReschedulerTask process task re-scheduler if needed.
func (s *Scheduler) ReschedulerTask() {
	for {
		select {
		case msg := <-s.ReschedQueue:
			task, err := s.registry.FetchApplicationTask(msg.AppID, msg.TaskID)
			if err != nil {
				msg.Err <- fmt.Errorf("Rescheduling task failed: %s", err.Error())
				return
			}

			if task == nil {
				msg.Err <- fmt.Errorf("Task %s does not exists", msg.TaskID)
				return
			}

			if _, err := s.KillTask(task); err != nil {
				msg.Err <- fmt.Errorf("Kill task failed: %s for rescheduling", err.Error())
				return
			}

			s.Status = "busy"

			resources := s.BuildResources(task.Cpus, task.Mem, task.Disk)
			offers, err := s.RequestOffers(resources)
			if err != nil {
				msg.Err <- fmt.Errorf("Request offers failed: %s for rescheduling", err.Error())
				return
			}

			var choosedOffer *mesos.Offer
			for _, offer := range offers {
				cpus, mem, disk := s.OfferedResources(offer)
				if cpus >= task.Cpus && mem >= task.Mem && disk >= task.Disk {
					choosedOffer = offer
					break
				}
			}

			var taskInfos []*mesos.TaskInfo
			taskInfo := s.BuildTaskInfo(choosedOffer, resources, task)
			taskInfos = append(taskInfos, taskInfo)

			resp, err := s.LaunchTasks(choosedOffer, taskInfos)
			if err != nil {
				msg.Err <- fmt.Errorf("Launchs task failed: %s for rescheduling", err.Error())
				return
			}

			if resp != nil && resp.StatusCode != http.StatusAccepted {
				msg.Err <- fmt.Errorf("Launchs task failed: status code %d for rescheduling", resp.StatusCode)
				return
			}

			logrus.Infof("Remove health check for task %s", msg.TaskID)
			if err := s.registry.DeleteCheck(msg.TaskID); err != nil {
				msg.Err <- fmt.Errorf("Remove health check for %s failed: %s", msg.TaskID, err.Error())
				return
			}

			if len(task.HealthChecks) != 0 {
				if err := s.registry.RegisterCheck(task,
					*taskInfo.Container.Docker.PortMappings[0].HostPort,
					msg.AppID); err != nil {
				}
				for _, healthCheck := range task.HealthChecks {
					check := types.Check{
						ID:       task.Name,
						Address:  *task.AgentHostname,
						Port:     int(*taskInfo.Container.Docker.PortMappings[0].HostPort),
						TaskID:   task.Name,
						AppID:    msg.AppID,
						Protocol: healthCheck.Protocol,
						Interval: int(healthCheck.IntervalSeconds),
						Timeout:  int(healthCheck.TimeoutSeconds),
					}
					if healthCheck.Command != nil {
						check.Command = healthCheck.Command
					}

					if healthCheck.Path != nil {
						check.Path = *healthCheck.Path
					}

					if healthCheck.MaxConsecutiveFailures != nil {
						check.MaxFailures = *healthCheck.MaxConsecutiveFailures
					}

					s.HealthCheckManager.Add(&check)
				}
			}

			msg.Err <- nil

			s.Status = "idle"

		case <-s.doneChan:
			return
		}
	}
}
<commit_msg>fixed bug application not found when build task<commit_after>package scheduler

import (
	//"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/Dataman-Cloud/swan/mesosproto/mesos"
	"github.com/Dataman-Cloud/swan/mesosproto/sched"
	"github.com/Dataman-Cloud/swan/types"

	"github.com/Sirupsen/logrus"
	"github.com/golang/protobuf/proto"
)

func (s *Scheduler) BuildTask(offer *mesos.Offer, version *types.Version, name string) (*types.Task, error) {
	var task types.Task

	task.Name = name
	if task.Name == "" {
		app, err := s.registry.FetchApplication(version.ID)
		if err != nil {
			return nil, err
		}

		if app == nil {
			return nil, fmt.Errorf("Application %s not found.", version.ID)
		}

		task.Name = fmt.Sprintf("%d.%s.%s.%s", app.Instances, app.ID, app.UserId, app.ClusterId)

		if err := s.registry.IncreaseApplicationInstances(app.ID); err != nil {
			return nil, err
		}
	}

	task.AppId = version.ID
	task.ID = fmt.Sprintf("%d-%s", time.Now().UnixNano(), task.Name)

	task.Image = version.Container.Docker.Image
	task.Network = version.Container.Docker.Network

	if version.Container.Docker.Parameters != nil {
		for _, parameter := range *version.Container.Docker.Parameters {
			task.Parameters = append(task.Parameters, &types.Parameter{
				Key:   parameter.Key,
				Value: parameter.Value,
			})
		}
	}

	if version.Container.Docker.PortMappings != nil {
		for _, portMapping := range *version.Container.Docker.PortMappings {
			task.PortMappings = append(task.PortMappings, &types.PortMappings{
				Port:     uint32(portMapping.ContainerPort),
				Protocol: portMapping.Protocol,
			})
		}
	}

	if version.Container.Docker.Privileged != nil {
		task.Privileged = version.Container.Docker.Privileged
	}

	if version.Container.Docker.ForcePullImage != nil {
		task.ForcePullImage = version.Container.Docker.ForcePullImage
	}

	task.Env = version.Env

	task.Volumes = version.Container.Volumes

	if version.Labels != nil {
		task.Labels = version.Labels
	}

	task.Cpus = version.Cpus
	task.Mem = version.Mem
	task.Disk = version.Disk

	task.OfferId = offer.GetId().Value
	task.AgentId = offer.AgentId.Value
	task.AgentHostname = offer.Hostname

	if version.KillPolicy != nil {
		task.KillPolicy = version.KillPolicy
	}

	if version.HealthChecks != nil {
		task.HealthChecks = version.HealthChecks
	}

	return &task, nil
}

func (s *Scheduler) BuildTaskInfo(offer *mesos.Offer, resources []*mesos.Resource, task *types.Task) *mesos.TaskInfo {
	logrus.Infof("Prepared task for launch with offer %s", *offer.GetId().Value)
	taskInfo := mesos.TaskInfo{
		Name: proto.String(task.Name),
		TaskId: &mesos.TaskID{
			Value: proto.String(task.ID),
		},
		AgentId:   offer.AgentId,
		Resources: resources,
		Command: &mesos.CommandInfo{
			Shell: proto.Bool(false),
			Value: nil,
		},
		Container: &mesos.ContainerInfo{
			Type: mesos.ContainerInfo_DOCKER.Enum(),
			Docker: &mesos.ContainerInfo_DockerInfo{
				Image: task.Image,
			},
		},
	}

	if task.Privileged != nil {
		taskInfo.Container.Docker.Privileged = task.Privileged
	}

	if task.ForcePullImage != nil {
		taskInfo.Container.Docker.ForcePullImage = task.ForcePullImage
	}

	for _, parameter := range task.Parameters {
		taskInfo.Container.Docker.Parameters = append(taskInfo.Container.Docker.Parameters, &mesos.Parameter{
			Key:   proto.String(parameter.Key),
			Value: proto.String(parameter.Value),
		})
	}

	for _, volume := range task.Volumes {
		mode := mesos.Volume_RO
		if volume.Mode == "RW" {
			mode = mesos.Volume_RW
		}
		taskInfo.Container.Volumes = append(taskInfo.Container.Volumes, &mesos.Volume{
			ContainerPath: proto.String(volume.ContainerPath),
			HostPath:      proto.String(volume.HostPath),
			Mode:          &mode,
		})
	}

	vars := make([]*mesos.Environment_Variable, 0)
	for k, v := range task.Env {
		vars = append(vars, &mesos.Environment_Variable{
			Name:  proto.String(k),
			Value: proto.String(v),
		})
	}

	taskInfo.Command.Environment = &mesos.Environment{
		Variables: vars,
	}

	if task.Labels != nil {
		labels := make([]*mesos.Label, 0)
		for k, v := range *task.Labels {
			labels = append(labels, &mesos.Label{
				Key:   proto.String(k),
				Value: proto.String(v),
			})
		}

		taskInfo.Labels = &mesos.Labels{
			Labels: labels,
		}
	}

	switch task.Network {
	case "NONE":
		taskInfo.Container.Docker.Network = mesos.ContainerInfo_DockerInfo_NONE.Enum()
	case "HOST":
		taskInfo.Container.Docker.Network = mesos.ContainerInfo_DockerInfo_HOST.Enum()
	case "BRIDGE":
		ports := GetPorts(offer)
		if len(ports) == 0 {
			logrus.Errorf("No ports resource defined")
			break
		}
		for _, m := range task.PortMappings {
			hostPort := ports[s.TaskLaunched]
			taskInfo.Container.Docker.PortMappings = append(taskInfo.Container.Docker.PortMappings,
				&mesos.ContainerInfo_DockerInfo_PortMapping{
					HostPort:      proto.Uint32(uint32(hostPort)),
					ContainerPort: proto.Uint32(m.Port),
					Protocol:      proto.String(m.Protocol),
				},
			)
			taskInfo.Resources = append(taskInfo.Resources, &mesos.Resource{
				Name: proto.String("ports"),
				Type: mesos.Value_RANGES.Enum(),
				Ranges: &mesos.Value_Ranges{
					Range: []*mesos.Value_Range{
						{
							Begin: proto.Uint64(uint64(hostPort)),
							End:   proto.Uint64(uint64(hostPort)),
						},
					},
				},
			})
		}
		taskInfo.Container.Docker.Network = mesos.ContainerInfo_DockerInfo_BRIDGE.Enum()
	default:
		taskInfo.Container.Docker.Network = mesos.ContainerInfo_DockerInfo_NONE.Enum()
	}

	return &taskInfo
}

// LaunchTasks lauch multiple tasks with specified offer.
func (s *Scheduler) LaunchTasks(offer *mesos.Offer, tasks []*mesos.TaskInfo) (*http.Response, error) {
	logrus.Infof("Launch %d tasks with offer %s", len(tasks), *offer.GetId().Value)
	call := &sched.Call{
		FrameworkId: s.framework.GetId(),
		Type:        sched.Call_ACCEPT.Enum(),
		Accept: &sched.Call_Accept{
			OfferIds: []*mesos.OfferID{
				offer.GetId(),
			},
			Operations: []*mesos.Offer_Operation{
				&mesos.Offer_Operation{
					Type: mesos.Offer_Operation_LAUNCH.Enum(),
					Launch: &mesos.Offer_Operation_Launch{
						TaskInfos: tasks,
					},
				},
			},
			Filters: &mesos.Filters{RefuseSeconds: proto.Float64(1)},
		},
	}

	return s.send(call)
}

func (s *Scheduler) KillTask(task *types.Task) (*http.Response, error) {
	logrus.Infof("Kill task %s", task.Name)
	call := &sched.Call{
		FrameworkId: s.framework.GetId(),
		Type:        sched.Call_KILL.Enum(),
		Kill: &sched.Call_Kill{
			TaskId: &mesos.TaskID{
				Value: proto.String(task.ID),
			},
			AgentId: &mesos.AgentID{
				Value: task.AgentId,
			},
		},
	}

	if task.KillPolicy != nil {
		if task.KillPolicy.Duration != 0 {
			call.Kill.KillPolicy = &mesos.KillPolicy{
				GracePeriod: &mesos.DurationInfo{
					Nanoseconds: proto.Int64(task.KillPolicy.Duration * 1000 * 1000),
				},
			}
		}
	}

	return s.send(call)
}

// ReschedulerTask process task re-scheduler if needed.
func (s *Scheduler) ReschedulerTask() {
	for {
		select {
		case msg := <-s.ReschedQueue:
			task, err := s.registry.FetchApplicationTask(msg.AppID, msg.TaskID)
			if err != nil {
				msg.Err <- fmt.Errorf("Rescheduling task failed: %s", err.Error())
				return
			}

			if task == nil {
				msg.Err <- fmt.Errorf("Task %s does not exists", msg.TaskID)
				return
			}

			if _, err := s.KillTask(task); err != nil {
				msg.Err <- fmt.Errorf("Kill task failed: %s for rescheduling", err.Error())
				return
			}

			s.Status = "busy"

			resources := s.BuildResources(task.Cpus, task.Mem, task.Disk)
			offers, err := s.RequestOffers(resources)
			if err != nil {
				msg.Err <- fmt.Errorf("Request offers failed: %s for rescheduling", err.Error())
				return
			}

			var choosedOffer *mesos.Offer
			for _, offer := range offers {
				cpus, mem, disk := s.OfferedResources(offer)
				if cpus >= task.Cpus && mem >= task.Mem && disk >= task.Disk {
					choosedOffer = offer
					break
				}
			}

			var taskInfos []*mesos.TaskInfo
			taskInfo := s.BuildTaskInfo(choosedOffer, resources, task)
			taskInfos = append(taskInfos, taskInfo)

			resp, err := s.LaunchTasks(choosedOffer, taskInfos)
			if err != nil {
				msg.Err <- fmt.Errorf("Launchs task failed: %s for rescheduling", err.Error())
				return
			}

			if resp != nil && resp.StatusCode != http.StatusAccepted {
				msg.Err <- fmt.Errorf("Launchs task failed: status code %d for rescheduling", resp.StatusCode)
				return
			}

			logrus.Infof("Remove health check for task %s", msg.TaskID)
			if err := s.registry.DeleteCheck(msg.TaskID); err != nil {
				msg.Err <- fmt.Errorf("Remove health check for %s failed: %s", msg.TaskID, err.Error())
				return
			}

			if len(task.HealthChecks) != 0 {
				if err := s.registry.RegisterCheck(task,
					*taskInfo.Container.Docker.PortMappings[0].HostPort,
					msg.AppID); err != nil {
				}
				for _, healthCheck := range task.HealthChecks {
					check := types.Check{
						ID:       task.Name,
						Address:  *task.AgentHostname,
						Port:     int(*taskInfo.Container.Docker.PortMappings[0].HostPort),
						TaskID:   task.Name,
						AppID:    msg.AppID,
						Protocol: healthCheck.Protocol,
						Interval: int(healthCheck.IntervalSeconds),
						Timeout:  int(healthCheck.TimeoutSeconds),
					}
					if healthCheck.Command != nil {
						check.Command = healthCheck.Command
					}

					if healthCheck.Path != nil {
						check.Path = *healthCheck.Path
					}

					if healthCheck.MaxConsecutiveFailures != nil {
						check.MaxFailures = *healthCheck.MaxConsecutiveFailures
					}

					s.HealthCheckManager.Add(&check)
				}
			}

			msg.Err <- nil

			s.Status = "idle"

		case <-s.doneChan:
			return
		}
	}
}
<|endoftext|>


<commit_before>package titan

import (
    "strings"
    "fmt"
    "encoding/json"
    "errors"
    "github.com/dghubble/sling"
)

type ImagesApi struct {
    basePath  string
}

func NewImagesApi() *ImagesApi{
    return &ImagesApi {
        basePath:   "https://localhost:8080/v1",
    }
}

func NewImagesApiWithBasePath(basePath string) *ImagesApi{
    return &ImagesApi {
        basePath:   basePath,
    }
}

/**
 * Get images by name.
 * TODO: Using images for lack of a better term. See https://github.com/iron-io/titan/issues/43 for discussion.
 * @param name Name of the image.
 * @return ImagesWrapper
 */
//func (a ImagesApi) ImagesNameGet (name string) (ImagesWrapper, error) {
func (a ImagesApi) ImagesNameGet (name string) (ImagesWrapper, error) {

    _sling := sling.New().Get(a.basePath)

    // create path and map variables
    path := "/v1/images/{name}"
    path = strings.Replace(path, "{" + "name" + "}", fmt.Sprintf("%v", name), -1)

    _sling = _sling.Path(path)

    // accept header
    accepts := []string { "application/json" }
    for key := range accepts {
        _sling = _sling.Set("Accept", accepts[key])
        break // only use the first Accept
    }

  var successPayload = new(ImagesWrapper)

  // We use this map (below) so that any arbitrary error JSON can be handled.
  // FIXME: This is in the absence of this Go generator honoring the non-2xx
  // response (error) models, which needs to be implemented at some point.
  var failurePayload map[string]interface{}

  httpResponse, err := _sling.Receive(successPayload, &failurePayload)

  if err == nil {
    // err == nil only means that there wasn't a sub-application-layer error (e.g. no network error)
    if failurePayload != nil {
      // If the failurePayload is present, there likely was some kind of non-2xx status
      // returned (and a JSON payload error present)
      var str []byte
      str, err = json.Marshal(failurePayload)
      if err == nil { // For safety, check for an error marshalling... probably superfluous
        // This will return the JSON error body as a string
        err = errors.New(string(str))
      }
  } else {
    // So, there was no network-type error, and nothing in the failure payload,
    // but we should still check the status code
    if httpResponse == nil {
      // This should never happen...
      err = errors.New("No HTTP Response received.")
    } else if code := httpResponse.StatusCode; 200 > code || code > 299 {
        err = errors.New("HTTP Error: " + string(httpResponse.StatusCode))
      }
    }
  }

  return *successPayload, err
}
<commit_msg>Updated to api version 0.1.0<commit_after>package titan

import (
    "strings"
    "fmt"
    "encoding/json"
    "errors"
    "github.com/dghubble/sling"
)

type ImagesApi struct {
    basePath  string
}

func NewImagesApi() *ImagesApi{
    return &ImagesApi {
        basePath:   "https://localhost:8080/v1",
    }
}

func NewImagesApiWithBasePath(basePath string) *ImagesApi{
    return &ImagesApi {
        basePath:   basePath,
    }
}

/**
 * Get all image names.
 * TODO: Using images for lack of a better term. See https://github.com/iron-io/titan/issues/43 for discussion.
 * @return ImagesWrapper
 */
//func (a ImagesApi) ImagesGet () (ImagesWrapper, error) {
func (a ImagesApi) ImagesGet () (ImagesWrapper, error) {

    _sling := sling.New().Get(a.basePath)

    // create path and map variables
    path := "/v1/images"

    _sling = _sling.Path(path)

    // accept header
    accepts := []string { "application/json" }
    for key := range accepts {
        _sling = _sling.Set("Accept", accepts[key])
        break // only use the first Accept
    }

  var successPayload = new(ImagesWrapper)

  // We use this map (below) so that any arbitrary error JSON can be handled.
  // FIXME: This is in the absence of this Go generator honoring the non-2xx
  // response (error) models, which needs to be implemented at some point.
  var failurePayload map[string]interface{}

  httpResponse, err := _sling.Receive(successPayload, &failurePayload)

  if err == nil {
    // err == nil only means that there wasn't a sub-application-layer error (e.g. no network error)
    if failurePayload != nil {
      // If the failurePayload is present, there likely was some kind of non-2xx status
      // returned (and a JSON payload error present)
      var str []byte
      str, err = json.Marshal(failurePayload)
      if err == nil { // For safety, check for an error marshalling... probably superfluous
        // This will return the JSON error body as a string
        err = errors.New(string(str))
      }
  } else {
    // So, there was no network-type error, and nothing in the failure payload,
    // but we should still check the status code
    if httpResponse == nil {
      // This should never happen...
      err = errors.New("No HTTP Response received.")
    } else if code := httpResponse.StatusCode; 200 > code || code > 299 {
        err = errors.New("HTTP Error: " + string(httpResponse.StatusCode))
      }
    }
  }

  return *successPayload, err
}
/**
 * Get image by name.
 * NOT IMPLEMENTED YET. This gives more details about on image, such as statistics and what not.
 * @param name Name of the image.
 * @return ImageWrapper
 */
//func (a ImagesApi) ImagesNameGet (name string) (ImageWrapper, error) {
func (a ImagesApi) ImagesNameGet (name string) (ImageWrapper, error) {

    _sling := sling.New().Get(a.basePath)

    // create path and map variables
    path := "/v1/images/{name}"
    path = strings.Replace(path, "{" + "name" + "}", fmt.Sprintf("%v", name), -1)

    _sling = _sling.Path(path)

    // accept header
    accepts := []string { "application/json" }
    for key := range accepts {
        _sling = _sling.Set("Accept", accepts[key])
        break // only use the first Accept
    }

  var successPayload = new(ImageWrapper)

  // We use this map (below) so that any arbitrary error JSON can be handled.
  // FIXME: This is in the absence of this Go generator honoring the non-2xx
  // response (error) models, which needs to be implemented at some point.
  var failurePayload map[string]interface{}

  httpResponse, err := _sling.Receive(successPayload, &failurePayload)

  if err == nil {
    // err == nil only means that there wasn't a sub-application-layer error (e.g. no network error)
    if failurePayload != nil {
      // If the failurePayload is present, there likely was some kind of non-2xx status
      // returned (and a JSON payload error present)
      var str []byte
      str, err = json.Marshal(failurePayload)
      if err == nil { // For safety, check for an error marshalling... probably superfluous
        // This will return the JSON error body as a string
        err = errors.New(string(str))
      }
  } else {
    // So, there was no network-type error, and nothing in the failure payload,
    // but we should still check the status code
    if httpResponse == nil {
      // This should never happen...
      err = errors.New("No HTTP Response received.")
    } else if code := httpResponse.StatusCode; 200 > code || code > 299 {
        err = errors.New("HTTP Error: " + string(httpResponse.StatusCode))
      }
    }
  }

  return *successPayload, err
}
<|endoftext|>


<commit_before>package stripe

import "encoding/json"

// TransferStatus is the list of allowed values for the transfer's status.
// Allowed values are "paid", "pending", "in_transit",  "failed", "canceled".
type TransferStatus string

// TransferType is the list of allowed values for the transfer's type.
// Allowed values are "bank_account", "card", "stripe_account".
type TransferType string

// TransferSourceType is the list of allowed values for the transfer's source_type field.
// Allowed values are "alipay_account", bank_account", "bitcoin_receiver", "card".
type TransferSourceType string

// TransferFailCode is the list of allowed values for the transfer's failure code.
// Allowed values are "insufficient_funds", "account_closed", "no_account",
// "invalid_account_number", "debit_not_authorized", "bank_ownership_changed",
// "account_frozen", "could_not_process", "bank_account_restricted", "invalid_currency".
type TransferFailCode string

// TransferParams is the set of parameters that can be used when creating or updating a transfer.
// For more details see https://stripe.com/docs/api#create_transfer and https://stripe.com/docs/api#update_transfer.
type TransferParams struct {
	Params
	Amount                                                 int64
	Fee                                                    uint64
	Currency                                               Currency
	Recipient, Desc, Statement, Bank, Card, SourceTx, Dest string
	SourceType                                             TransferSourceType
}

// TransferListParams is the set of parameters that can be used when listing transfers.
// For more details see https://stripe.com/docs/api#list_transfers.
type TransferListParams struct {
	ListParams
	Created, Date int64
	Recipient     string
	Status        TransferStatus
}

// Transfer is the resource representing a Stripe transfer.
// For more details see https://stripe.com/docs/api#transfers.
type Transfer struct {
	ID         string             `json:"id"`
	Live       bool               `json:"livemode"`
	Amount     int64              `json:"amount"`
	Currency   Currency           `json:"currency"`
	Created    int64              `json:"created"`
	Date       int64              `json:"date"`
	Desc       string             `json:"description"`
	FailCode   TransferFailCode   `json:"failure_code"`
	FailMsg    string             `json:"failure_message"`
	Status     TransferStatus     `json:"status"`
	Type       TransferType       `json:"type"`
	Tx         *Transaction       `json:"balance_transaction"`
	Meta       map[string]string  `json:"metadata"`
	Bank       *BankAccount       `json:"bank_account"`
	Card       *Card              `json:"card"`
	Recipient  *Recipient         `json:"recipient"`
	Statement  string             `json:"statement_descriptor"`
	Reversals  *ReversalList      `json:"reversals"`
	SourceTx   *Transaction       `json:"source_transaction"`
	SourceType TransferSourceType `json:"source_type"`
}

// TransferList is a list of transfers as retrieved from a list endpoint.
type TransferList struct {
	ListMeta
	Values []*Transfer `json:"data"`
}

// UnmarshalJSON handles deserialization of a Transfer.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (t *Transfer) UnmarshalJSON(data []byte) error {
	type transfer Transfer
	var tb transfer
	err := json.Unmarshal(data, &tb)
	if err == nil {
		*t = Transfer(tb)
	} else {
		// the id is surrounded by "\" characters, so strip them
		t.ID = string(data[1 : len(data)-1])
	}

	return nil
}
<commit_msg>Add missing `Reversed` and `AmountReversed` fields to `Transfer`<commit_after>package stripe

import "encoding/json"

// TransferStatus is the list of allowed values for the transfer's status.
// Allowed values are "paid", "pending", "in_transit",  "failed", "canceled".
type TransferStatus string

// TransferType is the list of allowed values for the transfer's type.
// Allowed values are "bank_account", "card", "stripe_account".
type TransferType string

// TransferSourceType is the list of allowed values for the transfer's source_type field.
// Allowed values are "alipay_account", bank_account", "bitcoin_receiver", "card".
type TransferSourceType string

// TransferFailCode is the list of allowed values for the transfer's failure code.
// Allowed values are "insufficient_funds", "account_closed", "no_account",
// "invalid_account_number", "debit_not_authorized", "bank_ownership_changed",
// "account_frozen", "could_not_process", "bank_account_restricted", "invalid_currency".
type TransferFailCode string

// TransferParams is the set of parameters that can be used when creating or updating a transfer.
// For more details see https://stripe.com/docs/api#create_transfer and https://stripe.com/docs/api#update_transfer.
type TransferParams struct {
	Params
	Amount                                                 int64
	Fee                                                    uint64
	Currency                                               Currency
	Recipient, Desc, Statement, Bank, Card, SourceTx, Dest string
	SourceType                                             TransferSourceType
}

// TransferListParams is the set of parameters that can be used when listing transfers.
// For more details see https://stripe.com/docs/api#list_transfers.
type TransferListParams struct {
	ListParams
	Created, Date int64
	Recipient     string
	Status        TransferStatus
}

// Transfer is the resource representing a Stripe transfer.
// For more details see https://stripe.com/docs/api#transfers.
type Transfer struct {
	ID             string             `json:"id"`
	Live           bool               `json:"livemode"`
	Amount         int64              `json:"amount"`
	AmountReversed int64              `json:"amount_reversed"`
	Currency       Currency           `json:"currency"`
	Created        int64              `json:"created"`
	Date           int64              `json:"date"`
	Desc           string             `json:"description"`
	FailCode       TransferFailCode   `json:"failure_code"`
	FailMsg        string             `json:"failure_message"`
	Status         TransferStatus     `json:"status"`
	Type           TransferType       `json:"type"`
	Tx             *Transaction       `json:"balance_transaction"`
	Meta           map[string]string  `json:"metadata"`
	Bank           *BankAccount       `json:"bank_account"`
	Card           *Card              `json:"card"`
	Recipient      *Recipient         `json:"recipient"`
	Statement      string             `json:"statement_descriptor"`
	Reversals      *ReversalList      `json:"reversals"`
	Reversed       bool               `json:"reversed"`
	SourceTx       *Transaction       `json:"source_transaction"`
	SourceType     TransferSourceType `json:"source_type"`
}

// TransferList is a list of transfers as retrieved from a list endpoint.
type TransferList struct {
	ListMeta
	Values []*Transfer `json:"data"`
}

// UnmarshalJSON handles deserialization of a Transfer.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (t *Transfer) UnmarshalJSON(data []byte) error {
	type transfer Transfer
	var tb transfer
	err := json.Unmarshal(data, &tb)
	if err == nil {
		*t = Transfer(tb)
	} else {
		// the id is surrounded by "\" characters, so strip them
		t.ID = string(data[1 : len(data)-1])
	}

	return nil
}
<|endoftext|>


<commit_before>package api

import (
	ocispecs "github.com/opencontainers/runtime-spec/specs-go"
)

func (v *VolumeDescription) IsDir() bool {
	return v.Format == "vfs"
}

func (v *VolumeDescription) IsNas() bool {
	return v.Format == "nas"
}

func SandboxInfoFromOCF(s *ocispecs.Spec) *SandboxConfig {
	return &SandboxConfig{
		Hostname: s.Hostname,
	}
}

func ContainerDescriptionFromOCF(id string, s *ocispecs.Spec) *ContainerDescription {
	container := &ContainerDescription{
		Id:         id,
		Name:       s.Hostname,
		Image:      "",
		Labels:     make(map[string]string),
		RootVolume: nil,
		RootPath:   "rootfs",
		OciSpec:    *s,
	}

	if container.OciSpec.Linux.Sysctl == nil {
		container.OciSpec.Linux.Sysctl = map[string]string{}
	}
	if _, ok := container.OciSpec.Linux.Sysctl["vm.overcommit_memory"]; !ok {
		container.OciSpec.Linux.Sysctl["vm.overcommit_memory"] = "1"
	}

	// the mounts need to be filtered and add it back after ContainerDescriptionFromOCF()
	container.OciSpec.Mounts = []ocispecs.Mount{}
	container.OciSpec.Root.Path = "" // already mounted on filepath.Join(rootfs.Source, container.RootPath)

	rootfs := &VolumeDescription{
		Name:     id,
		Source:   id,
		Fstype:   "dir",
		Format:   "vfs",
		ReadOnly: s.Root.Readonly,
	}
	container.RootVolume = rootfs

	return container
}
<commit_msg>oci: keep pseudo filesystems when conversion<commit_after>package api

import (
	ocispecs "github.com/opencontainers/runtime-spec/specs-go"
)

func (v *VolumeDescription) IsDir() bool {
	return v.Format == "vfs"
}

func (v *VolumeDescription) IsNas() bool {
	return v.Format == "nas"
}

func SandboxInfoFromOCF(s *ocispecs.Spec) *SandboxConfig {
	return &SandboxConfig{
		Hostname: s.Hostname,
	}
}

func ContainerDescriptionFromOCF(id string, s *ocispecs.Spec) *ContainerDescription {
	container := &ContainerDescription{
		Id:         id,
		Name:       s.Hostname,
		Image:      "",
		Labels:     make(map[string]string),
		RootVolume: nil,
		RootPath:   "rootfs",
		OciSpec:    *s,
	}

	if container.OciSpec.Linux.Sysctl == nil {
		container.OciSpec.Linux.Sysctl = map[string]string{}
	}
	if _, ok := container.OciSpec.Linux.Sysctl["vm.overcommit_memory"]; !ok {
		container.OciSpec.Linux.Sysctl["vm.overcommit_memory"] = "1"
	}

	// the mounts need to be filtered and add it back after ContainerDescriptionFromOCF()
	// virtual mounts are added back here.
	container.OciSpec.Mounts = []ocispecs.Mount{}
	for _, m := range s.Mounts {
		switch m.Type {
		case "proc", "sysfs", "mqueue", "tmpfs", "cgroup", "devpts":
			container.OciSpec.Mounts = append(container.OciSpec.Mounts, m)
		}
	}
	container.OciSpec.Root.Path = "" // already mounted on filepath.Join(rootfs.Source, container.RootPath)

	rootfs := &VolumeDescription{
		Name:     id,
		Source:   id,
		Fstype:   "dir",
		Format:   "vfs",
		ReadOnly: s.Root.Readonly,
	}
	container.RootVolume = rootfs

	return container
}
<|endoftext|>


<commit_before>package api

// Logger used in mqtt package
type Logger interface {
	Debug(msg string)
	Info(msg string)
	Warn(msg string)
	Error(msg string)
	Fatal(msg string)
	Debugf(msg string, v ...interface{})
	Infof(msg string, v ...interface{})
	Warnf(msg string, v ...interface{})
	Errorf(msg string, v ...interface{})
	Fatalf(msg string, v ...interface{})
}

var logger Logger = noopLogger{}

// GetLogger returns the API Logger
func GetLogger() Logger {
	return logger
}

// SetLogger returns the API Logger
func SetLogger(log Logger) {
	logger = log
}

// noopLogger just does nothing
type noopLogger struct{}

func (l noopLogger) Debug(msg string)                    {}
func (l noopLogger) Info(msg string)                     {}
func (l noopLogger) Warn(msg string)                     {}
func (l noopLogger) Error(msg string)                    {}
func (l noopLogger) Fatal(msg string)                    {}
func (l noopLogger) Debugf(msg string, v ...interface{}) {}
func (l noopLogger) Infof(msg string, v ...interface{})  {}
func (l noopLogger) Warnf(msg string, v ...interface{})  {}
func (l noopLogger) Errorf(msg string, v ...interface{}) {}
func (l noopLogger) Fatalf(msg string, v ...interface{}) {}
<commit_msg>Wrap apex/log in api/logging.go<commit_after>package api

import "github.com/apex/log"

// Logger used in mqtt package
type Logger interface {
	Debug(msg string)
	Info(msg string)
	Warn(msg string)
	Error(msg string)
	Fatal(msg string)
	Debugf(msg string, v ...interface{})
	Infof(msg string, v ...interface{})
	Warnf(msg string, v ...interface{})
	Errorf(msg string, v ...interface{})
	Fatalf(msg string, v ...interface{})
	WithField(string, interface{}) Logger
	WithFields(log.Fielder) Logger
	WithError(error) Logger
}

// Apex wraps apex/log
func Apex(ctx log.Interface) Logger {
	return &apexInterfaceWrapper{ctx}
}

type apexInterfaceWrapper struct {
	log.Interface
}

func (w *apexInterfaceWrapper) WithField(k string, v interface{}) Logger {
	return &apexEntryWrapper{w.Interface.WithField(k, v)}
}

func (w *apexInterfaceWrapper) WithFields(fields log.Fielder) Logger {
	return &apexEntryWrapper{w.Interface.WithFields(fields)}
}

func (w *apexInterfaceWrapper) WithError(err error) Logger {
	return &apexEntryWrapper{w.Interface.WithError(err)}
}

type apexEntryWrapper struct {
	*log.Entry
}

func (w *apexEntryWrapper) WithField(k string, v interface{}) Logger {
	return &apexEntryWrapper{w.Entry.WithField(k, v)}
}

func (w *apexEntryWrapper) WithFields(fields log.Fielder) Logger {
	return &apexEntryWrapper{w.Entry.WithFields(fields)}
}

func (w *apexEntryWrapper) WithError(err error) Logger {
	return &apexEntryWrapper{w.Entry.WithError(err)}
}

var logger Logger = noopLogger{}

// GetLogger returns the API Logger
func GetLogger() Logger {
	return logger
}

// SetLogger returns the API Logger
func SetLogger(log Logger) {
	logger = log
}

// noopLogger just does nothing
type noopLogger struct{}

func (l noopLogger) Debug(msg string)                     {}
func (l noopLogger) Info(msg string)                      {}
func (l noopLogger) Warn(msg string)                      {}
func (l noopLogger) Error(msg string)                     {}
func (l noopLogger) Fatal(msg string)                     {}
func (l noopLogger) Debugf(msg string, v ...interface{})  {}
func (l noopLogger) Infof(msg string, v ...interface{})   {}
func (l noopLogger) Warnf(msg string, v ...interface{})   {}
func (l noopLogger) Errorf(msg string, v ...interface{})  {}
func (l noopLogger) Fatalf(msg string, v ...interface{})  {}
func (l noopLogger) WithField(string, interface{}) Logger { return l }
func (l noopLogger) WithFields(log.Fielder) Logger        { return l }
func (l noopLogger) WithError(error) Logger               { return l }
<|endoftext|>


<commit_before>// khan
// https://github.com/topfreegames/khan
//
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2016 Top Free Games <backend@tfgco.com>

package api

// VERSION identifies Khan's current version
var VERSION = "0.3.0"
<commit_msg>Release 0.3.1<commit_after>// khan
// https://github.com/topfreegames/khan
//
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2016 Top Free Games <backend@tfgco.com>

package api

// VERSION identifies Khan's current version
var VERSION = "0.3.1"
<|endoftext|>


<commit_before>package apidApigeeSync

import (
	"bytes"
	"encoding/json"
	"errors"
	"github.com/30x/apid"
	"github.com/apigee-labs/transicator/common"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"time"
)

var token string
var downloadDataSnapshot, downloadBootSnapshot, changeFinished bool
var lastSequence string

func addHeaders(req *http.Request) {
	req.Header.Add("Authorization", "Bearer " + token)
	req.Header.Set("apid_instance_id", apidInfo.InstanceID)
	req.Header.Set("apid_cluster_Id", apidInfo.ClusterID)
	req.Header.Set("updated_at_apid", time.Now().Format(time.RFC3339))
}

func postPluginDataDelivery(e apid.Event) {

	if ede, ok := e.(apid.EventDeliveryEvent); ok {

		if ev, ok := ede.Event.(*common.ChangeList); ok {
			lastSequence = ev.LastSequence
			err := persistChange(lastSequence)
			if err != nil {
				log.Panic("Unable to update Sequence in DB")
			}
			changeFinished = true

		} else if _, ok := ede.Event.(*common.Snapshot); ok {
			if downloadBootSnapshot == false {
				downloadBootSnapshot = true
				log.Debug("Updated bootstrap SnapshotInfo")
			} else {
				downloadDataSnapshot = true
				log.Debug("Updated data SnapshotInfo")
			}
		}
	}
}

/*
 * Helper function that sleeps for N seconds if comm with change agent
 * fails. The retry interval gradually is incremented each time it fails
 * till it reaches the Polling Int time, and after which it constantly
 * retries at the polling time interval
 */
func updatePeriodicChanges() {

	times := 1
	pollInterval := config.GetInt(configPollInterval)
	for {
		startTime := time.Second
		err := pollChangeAgent()
		if err != nil {
			log.Debugf("Error connecting to changeserver: %v", err)
		}
		endTime := time.Second
		// Gradually increase retry interval, and max at some level
		if endTime - startTime <= 1 {
			if times < pollInterval {
				times++
			} else {
				times = pollInterval
			}
			log.Debugf("Connecting to changeserver...")
			time.Sleep(time.Duration(times) * 100 * time.Millisecond)
		} else {
			// Reset sleep interval
			times = 1
		}

	}
}

/*
 * Long polls every 45 seconds the change agent. Parses the response from
 * change agent and raises an event.
 */
func pollChangeAgent() error {

	if downloadDataSnapshot != true {
		log.Warn("Waiting for snapshot download to complete")
		return errors.New("Snapshot download in progress...")
	}
	changesUri, err := url.Parse(config.GetString(configChangeServerBaseURI))
	if err != nil {
		log.Errorf("bad url value for config %s: %s", changesUri, err)
		return err
	}
	changesUri.Path = path.Join(changesUri.Path, "/changes")

	/*
	 * Check to see if we have lastSequence already saved in the DB,
	 * in which case, it has to be used to prevent re-reading same data
	 */
	lastSequence = findApidConfigInfo(lastSequence)
	for {
		log.Debug("polling...")
		if token == "" {
			/* token not valid?, get a new token */
			status := getBearerToken()
			if status == false {
				return errors.New("Unable to get new token")
			}
		}

		/* Find the scopes associated with the config id */
		scopes := findScopesForId(apidInfo.ClusterID)
		v := url.Values{}

		/* Sequence added to the query if available */
		if lastSequence != "" {
			v.Add("since", lastSequence)
		}
		v.Add("block", "45")

		/*
		 * Include all the scopes associated with the config Id
		 * The Config Id is included as well, as it acts as the
		 * Bootstrap scope
		 */
		for _, scope := range scopes {
			v.Add("scope", scope)
		}
		v.Add("scope", apidInfo.ClusterID)
		v.Add("snapshot", apidInfo.LastSnapshot)
		changesUri.RawQuery = v.Encode()
		uri := changesUri.String()
		log.Debugf("Fetching changes: %s", uri)

		/* If error, break the loop, and retry after interval */
		client := &http.Client{}
		req, err := http.NewRequest("GET", uri, nil)
		addHeaders(req)
		r, err := client.Do(req)
		if err != nil {
			log.Errorf("change agent comm error: %s", err)
			return err
		}

		/* If the call is not Authorized, update flag */
		if r.StatusCode != http.StatusOK {
			if r.StatusCode == http.StatusUnauthorized {
				token = ""
				log.Errorf("Token expired? Unauthorized request.")
			}
			r.Body.Close()
			log.Errorf("Get Changes request failed with Resp err: %d", r.StatusCode)
			return err
		}

		var resp common.ChangeList
		err = json.NewDecoder(r.Body).Decode(&resp)
		r.Body.Close()
		if err != nil {
			log.Errorf("JSON Response Data not parsable: %v", err)
			return err
		}

		/* If valid data present, Emit to plugins */
		if len(resp.Changes) > 0 {
			changeFinished = false
			events.ListenFunc(apid.EventDeliveredSelector, postPluginDataDelivery)
			events.Emit(ApigeeSyncEventSelector, &resp)
			/*
			 * The plugins should have finished what they are doing.
			 * Wait till they are done.
			 * If they take longer than expected - abort apid(?)
			 * (Should there be a configurable Fudge factor?) FIXME
			 */
			for count := 0; count < 1000; count++ {
				if changeFinished == false {
					log.Debug("Waiting for plugins to complete...")
					time.Sleep(time.Duration(count) * 100 * time.Millisecond)
				} else {
					break
				}
			}
			if changeFinished == false {
				log.Panic("Never got ack from plugins. Investigate.")
			}
		} else {
			log.Debugf("No Changes detected for Scopes: %s", scopes)
		}
	}
}

/*
 * This function will (for now) use the Access Key/Secret Key/ApidConfig Id
 * to get the bearer token, and the scopes (as comma separated scope)
 */
func getBearerToken() bool {

	log.Debug("Getting a Bearer token.")
	uri, err := url.Parse(config.GetString(configProxyServerBaseURI))
	if err != nil {
		log.Error(err)
		return false
	}
	uri.Path = path.Join(uri.Path, "/accesstoken")

	token = ""
	form := url.Values{}
	form.Set("grant_type", "client_credentials")
	form.Add("client_id", config.GetString(configConsumerKey))
	form.Add("client_secret", config.GetString(configConsumerSecret))
	req, err := http.NewRequest("POST", uri.String(), bytes.NewBufferString(form.Encode()))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
	req.Header.Set("display_name", apidInfo.InstanceName)
	req.Header.Set("apid_instance_id", apidInfo.InstanceID)
	req.Header.Set("apid_cluster_Id", apidInfo.ClusterID)
	req.Header.Set("status", "ONLINE")
	req.Header.Set("created_at_apid", time.Now().Format(time.RFC3339))
	req.Header.Set("plugin_details", apidPluginDetails)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Error("Unable to Connect to Edge Proxy Server ", err)
		return false
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		log.Error("Oauth Request Failed with Resp Code ", resp.StatusCode)
		return false
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Error("Unable to read EdgeProxy Sever response ", err)
		return false
	}

	var oauthResp oauthTokenResp
	err = json.Unmarshal(body, &oauthResp)
	if err != nil {
		log.Error(err)
		return false
	}
	token = oauthResp.AccessToken
	log.Debug("Got a new Bearer token.")
	return true
}

type oauthTokenResp struct {
	IssuedAt       int64    `json:"issuedAt"`
	AppName        string   `json:"applicationName"`
	Scope          string   `json:"scope"`
	Status         string   `json:"status"`
	ApiProdList    []string `json:"apiProductList"`
	ExpiresIn      int64    `json:"expiresIn"`
	DeveloperEmail string   `json:"developerEmail"`
	TokenType      string   `json:"tokenType"`
	ClientId       string   `json:"clientId"`
	AccessToken    string   `json:"accessToken"`
	TokenExpIn     int64    `json:"refreshTokenExpiresIn"`
	RefreshCount   int64    `json:"refreshCount"`
}

func Redirect(req *http.Request, via []*http.Request) error {
	req.Header.Add("Authorization", "Bearer "+token)
	req.Header.Add("org", apidInfo.ClusterID)
	return nil
}

/*
 * Method downloads the snapshot in a two phased manner.
 * Phase 1: Use the apidConfigId as the bootstrap scope, and
 * get the apid_config and apid_config_scope from the snapshot
 * server.
 * Phase 2: Get all the scopes fetches from phase 1, and issue
 * the second call to the snapshot server to get all the data
 * associated with the scope(s).
 * Emit the data for the necessary plugins to process.
 * If there is already previous data in sqlite, don't fetch
 * again from snapshot server.
 */
func bootstrap() {

	// Skip Downloading snapshot if there is already a snapshot available from previous run of APID
	if apidInfo.LastSnapshot != "" {

		downloadDataSnapshot = true
		downloadBootSnapshot = true

		log.Infof("Starting on downloaded snapshot: %s", apidInfo.LastSnapshot)

		// ensure DB version will be accessible on behalf of dependant plugins
		_, err := data.DBVersion(apidInfo.LastSnapshot)
		if err != nil {
			log.Panicf("Database inaccessible: %v", err)
		}

		// allow plugins (including this one) to start immediately on existing database
		snap := &common.Snapshot{
			SnapshotInfo: apidInfo.LastSnapshot,
		}
		events.Emit(ApigeeSyncEventSelector, snap)

		return
	}

	/* Phase 1 */
	downloadSnapshot()

	/*
	 * Give some time for all the plugins to process the Downloaded
	 * Snapshot
	 */
	for count := 0; count < 60; count++ {
		if !downloadBootSnapshot {
			log.Debug("Waiting for bootscope snapshot download...")
			time.Sleep(time.Duration(count) * 100 * time.Millisecond)
		} else {
			break
		}
	}

	/* Phase 2 */
	if downloadBootSnapshot && downloadDataSnapshot {
		log.Debug("Proceeding with existing Sqlite data")
	} else if downloadBootSnapshot == true {
		log.Debug("Proceed to download Snapshot for data scopes")
		downloadSnapshot()
	} else {
		log.Panic("Snapshot for bootscope failed")
	}
}

func downloadSnapshot() {

	log.Debugf("downloadSnapshot")

	var scopes []string

	/* Get the bearer token */
	status := getBearerToken()
	if status == false {
		log.Panic("Unable to get Bearer token or is Invalid")
	}
	snapshotUri, err := url.Parse(config.GetString(configSnapServerBaseURI))
	if err != nil {
		log.Fatalf("bad url value for config %s: %s", snapshotUri, err)
	}

	if downloadBootSnapshot == false {
		scopes = append(scopes, apidInfo.ClusterID)
	} else {
		scopes = findScopesForId(apidInfo.ClusterID)
	}
	if scopes == nil {
		log.Panic("Scope cannot be found to download snapshot")
	}
	/* Frame and send the snapshot request */
	snapshotUri.Path = path.Join(snapshotUri.Path, "/snapshots")

	v := url.Values{}
	for _, scope := range scopes {
		v.Add("scope", scope)
	}
	snapshotUri.RawQuery = v.Encode()
	uri := snapshotUri.String()
	log.Info("Snapshot Download: ", uri)

	client := &http.Client{
		CheckRedirect: Redirect,
	}
	req, err := http.NewRequest("GET", uri, nil)
	addHeaders(req)

	/* Set the transport protocol type based on conf file input */
	if config.GetString(configSnapshotProtocol) == "json" {
		req.Header.Set("Accept", "application/json")
	} else {
		req.Header.Set("Accept", "application/proto")
	}

	/* Issue the request to the snapshot server */
	r, err := client.Do(req)
	if err != nil {
		log.Fatalf("Snapshotserver comm error: [%s] ", err)
	}
	defer r.Body.Close()

	/* Decode the Snapshot server response */
	var resp common.Snapshot
	err = json.NewDecoder(r.Body).Decode(&resp)
	if err != nil {

		if downloadBootSnapshot == false {
			log.Fatalf("JSON Response Data not parsable: %v", err)
		} else {
			/*
			 * If the data set is empty, allow it to proceed, as change server
			 * will feed data. Since Bootstrapping has passed, it has the
			 * Bootstrap config id to function.
			 */
			downloadDataSnapshot = true
			return
		}
	}

	if r.StatusCode == 200 {
		log.Info("Emit Snapshot response to plugins")
		events.ListenFunc(apid.EventDeliveredSelector, postPluginDataDelivery)
		events.Emit(ApigeeSyncEventSelector, &resp)

	} else {
		log.Fatalf("Snapshot server conn failed. HTTP Resp code %d", r.StatusCode)
	}

}
<commit_msg>Change lastSequence on every change<commit_after>package apidApigeeSync

import (
	"bytes"
	"encoding/json"
	"errors"
	"github.com/30x/apid"
	"github.com/apigee-labs/transicator/common"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"time"
)

var token string
var downloadDataSnapshot, downloadBootSnapshot, changeFinished bool
var lastSequence string

func addHeaders(req *http.Request) {
	req.Header.Add("Authorization", "Bearer " + token)
	req.Header.Set("apid_instance_id", apidInfo.InstanceID)
	req.Header.Set("apid_cluster_Id", apidInfo.ClusterID)
	req.Header.Set("updated_at_apid", time.Now().Format(time.RFC3339))
}

func postPluginDataDelivery(e apid.Event) {

	if ede, ok := e.(apid.EventDeliveryEvent); ok {

		if ev, ok := ede.Event.(*common.ChangeList); ok {
			if lastSequence != ev.LastSequence {
				lastSequence = ev.LastSequence
				err := persistChange(lastSequence)
				if err != nil {
					log.Panic("Unable to update Sequence in DB")
				}
			}
			changeFinished = true

		} else if _, ok := ede.Event.(*common.Snapshot); ok {
			if downloadBootSnapshot == false {
				downloadBootSnapshot = true
				log.Debug("Updated bootstrap SnapshotInfo")
			} else {
				downloadDataSnapshot = true
				log.Debug("Updated data SnapshotInfo")
			}
		}
	}
}

/*
 * Helper function that sleeps for N seconds if comm with change agent
 * fails. The retry interval gradually is incremented each time it fails
 * till it reaches the Polling Int time, and after which it constantly
 * retries at the polling time interval
 */
func updatePeriodicChanges() {

	times := 1
	pollInterval := config.GetInt(configPollInterval)
	for {
		startTime := time.Second
		err := pollChangeAgent()
		if err != nil {
			log.Debugf("Error connecting to changeserver: %v", err)
		}
		endTime := time.Second
		// Gradually increase retry interval, and max at some level
		if endTime - startTime <= 1 {
			if times < pollInterval {
				times++
			} else {
				times = pollInterval
			}
			log.Debugf("Connecting to changeserver...")
			time.Sleep(time.Duration(times) * 100 * time.Millisecond)
		} else {
			// Reset sleep interval
			times = 1
		}

	}
}

/*
 * Long polls every 45 seconds the change agent. Parses the response from
 * change agent and raises an event.
 */
func pollChangeAgent() error {

	if downloadDataSnapshot != true {
		log.Warn("Waiting for snapshot download to complete")
		return errors.New("Snapshot download in progress...")
	}
	changesUri, err := url.Parse(config.GetString(configChangeServerBaseURI))
	if err != nil {
		log.Errorf("bad url value for config %s: %s", changesUri, err)
		return err
	}
	changesUri.Path = path.Join(changesUri.Path, "/changes")

	/*
	 * Check to see if we have lastSequence already saved in the DB,
	 * in which case, it has to be used to prevent re-reading same data
	 */
	lastSequence = findApidConfigInfo(lastSequence)
	for {
		log.Debug("polling...")
		if token == "" {
			/* token not valid?, get a new token */
			status := getBearerToken()
			if status == false {
				return errors.New("Unable to get new token")
			}
		}

		/* Find the scopes associated with the config id */
		scopes := findScopesForId(apidInfo.ClusterID)
		v := url.Values{}

		/* Sequence added to the query if available */
		if lastSequence != "" {
			v.Add("since", lastSequence)
		}
		v.Add("block", "45")

		/*
		 * Include all the scopes associated with the config Id
		 * The Config Id is included as well, as it acts as the
		 * Bootstrap scope
		 */
		for _, scope := range scopes {
			v.Add("scope", scope)
		}
		v.Add("scope", apidInfo.ClusterID)
		v.Add("snapshot", apidInfo.LastSnapshot)
		changesUri.RawQuery = v.Encode()
		uri := changesUri.String()
		log.Debugf("Fetching changes: %s", uri)

		/* If error, break the loop, and retry after interval */
		client := &http.Client{}
		req, err := http.NewRequest("GET", uri, nil)
		addHeaders(req)
		r, err := client.Do(req)
		if err != nil {
			log.Errorf("change agent comm error: %s", err)
			return err
		}

		/* If the call is not Authorized, update flag */
		if r.StatusCode != http.StatusOK {
			if r.StatusCode == http.StatusUnauthorized {
				token = ""
				log.Errorf("Token expired? Unauthorized request.")
			}
			r.Body.Close()
			log.Errorf("Get Changes request failed with Resp err: %d", r.StatusCode)
			return err
		}

		var resp common.ChangeList
		err = json.NewDecoder(r.Body).Decode(&resp)
		r.Body.Close()
		if err != nil {
			log.Errorf("JSON Response Data not parsable: %v", err)
			return err
		}

		if lastSequence != resp.LastSequence {
			lastSequence = resp.LastSequence
			err := persistChange(lastSequence)
			if err != nil {
				log.Panic("Unable to update Sequence in DB")
			}
		}

		/* If valid data present, Emit to plugins */
		if len(resp.Changes) > 0 {
			changeFinished = false
			events.ListenFunc(apid.EventDeliveredSelector, postPluginDataDelivery)
			events.Emit(ApigeeSyncEventSelector, &resp)
			/*
			 * The plugins should have finished what they are doing.
			 * Wait till they are done.
			 * If they take longer than expected - abort apid(?)
			 * (Should there be a configurable Fudge factor?) FIXME
			 */
			for count := 0; count < 1000; count++ {
				if changeFinished == false {
					log.Debug("Waiting for plugins to complete...")
					time.Sleep(time.Duration(count) * 100 * time.Millisecond)
				} else {
					break
				}
			}
			if changeFinished == false {
				log.Panic("Never got ack from plugins. Investigate.")
			}
		} else {
			log.Debugf("No Changes detected for Scopes: %s", scopes)

			if lastSequence != resp.LastSequence {
				lastSequence = resp.LastSequence
				err := persistChange(lastSequence)
				if err != nil {
					log.Panic("Unable to update Sequence in DB")
				}
			}
		}
	}
}

/*
 * This function will (for now) use the Access Key/Secret Key/ApidConfig Id
 * to get the bearer token, and the scopes (as comma separated scope)
 */
func getBearerToken() bool {

	log.Debug("Getting a Bearer token.")
	uri, err := url.Parse(config.GetString(configProxyServerBaseURI))
	if err != nil {
		log.Error(err)
		return false
	}
	uri.Path = path.Join(uri.Path, "/accesstoken")

	token = ""
	form := url.Values{}
	form.Set("grant_type", "client_credentials")
	form.Add("client_id", config.GetString(configConsumerKey))
	form.Add("client_secret", config.GetString(configConsumerSecret))
	req, err := http.NewRequest("POST", uri.String(), bytes.NewBufferString(form.Encode()))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
	req.Header.Set("display_name", apidInfo.InstanceName)
	req.Header.Set("apid_instance_id", apidInfo.InstanceID)
	req.Header.Set("apid_cluster_Id", apidInfo.ClusterID)
	req.Header.Set("status", "ONLINE")
	req.Header.Set("created_at_apid", time.Now().Format(time.RFC3339))
	req.Header.Set("plugin_details", apidPluginDetails)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Error("Unable to Connect to Edge Proxy Server ", err)
		return false
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		log.Error("Oauth Request Failed with Resp Code ", resp.StatusCode)
		return false
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Error("Unable to read EdgeProxy Sever response ", err)
		return false
	}

	var oauthResp oauthTokenResp
	err = json.Unmarshal(body, &oauthResp)
	if err != nil {
		log.Error(err)
		return false
	}
	token = oauthResp.AccessToken
	log.Debug("Got a new Bearer token.")
	return true
}

type oauthTokenResp struct {
	IssuedAt       int64    `json:"issuedAt"`
	AppName        string   `json:"applicationName"`
	Scope          string   `json:"scope"`
	Status         string   `json:"status"`
	ApiProdList    []string `json:"apiProductList"`
	ExpiresIn      int64    `json:"expiresIn"`
	DeveloperEmail string   `json:"developerEmail"`
	TokenType      string   `json:"tokenType"`
	ClientId       string   `json:"clientId"`
	AccessToken    string   `json:"accessToken"`
	TokenExpIn     int64    `json:"refreshTokenExpiresIn"`
	RefreshCount   int64    `json:"refreshCount"`
}

func Redirect(req *http.Request, via []*http.Request) error {
	req.Header.Add("Authorization", "Bearer "+token)
	req.Header.Add("org", apidInfo.ClusterID)
	return nil
}

/*
 * Method downloads the snapshot in a two phased manner.
 * Phase 1: Use the apidConfigId as the bootstrap scope, and
 * get the apid_config and apid_config_scope from the snapshot
 * server.
 * Phase 2: Get all the scopes fetches from phase 1, and issue
 * the second call to the snapshot server to get all the data
 * associated with the scope(s).
 * Emit the data for the necessary plugins to process.
 * If there is already previous data in sqlite, don't fetch
 * again from snapshot server.
 */
func bootstrap() {

	// Skip Downloading snapshot if there is already a snapshot available from previous run of APID
	if apidInfo.LastSnapshot != "" {

		downloadDataSnapshot = true
		downloadBootSnapshot = true

		log.Infof("Starting on downloaded snapshot: %s", apidInfo.LastSnapshot)

		// ensure DB version will be accessible on behalf of dependant plugins
		_, err := data.DBVersion(apidInfo.LastSnapshot)
		if err != nil {
			log.Panicf("Database inaccessible: %v", err)
		}

		// allow plugins (including this one) to start immediately on existing database
		snap := &common.Snapshot{
			SnapshotInfo: apidInfo.LastSnapshot,
		}
		events.Emit(ApigeeSyncEventSelector, snap)

		return
	}

	/* Phase 1 */
	downloadSnapshot()

	/*
	 * Give some time for all the plugins to process the Downloaded
	 * Snapshot
	 */
	for count := 0; count < 60; count++ {
		if !downloadBootSnapshot {
			log.Debug("Waiting for bootscope snapshot download...")
			time.Sleep(time.Duration(count) * 100 * time.Millisecond)
		} else {
			break
		}
	}

	/* Phase 2 */
	if downloadBootSnapshot && downloadDataSnapshot {
		log.Debug("Proceeding with existing Sqlite data")
	} else if downloadBootSnapshot == true {
		log.Debug("Proceed to download Snapshot for data scopes")
		downloadSnapshot()
	} else {
		log.Panic("Snapshot for bootscope failed")
	}
}

func downloadSnapshot() {

	log.Debugf("downloadSnapshot")

	var scopes []string

	/* Get the bearer token */
	status := getBearerToken()
	if status == false {
		log.Panic("Unable to get Bearer token or is Invalid")
	}
	snapshotUri, err := url.Parse(config.GetString(configSnapServerBaseURI))
	if err != nil {
		log.Fatalf("bad url value for config %s: %s", snapshotUri, err)
	}

	if downloadBootSnapshot == false {
		scopes = append(scopes, apidInfo.ClusterID)
	} else {
		scopes = findScopesForId(apidInfo.ClusterID)
	}
	if scopes == nil {
		log.Panic("Scope cannot be found to download snapshot")
	}
	/* Frame and send the snapshot request */
	snapshotUri.Path = path.Join(snapshotUri.Path, "/snapshots")

	v := url.Values{}
	for _, scope := range scopes {
		v.Add("scope", scope)
	}
	snapshotUri.RawQuery = v.Encode()
	uri := snapshotUri.String()
	log.Info("Snapshot Download: ", uri)

	client := &http.Client{
		CheckRedirect: Redirect,
	}
	req, err := http.NewRequest("GET", uri, nil)
	addHeaders(req)

	/* Set the transport protocol type based on conf file input */
	if config.GetString(configSnapshotProtocol) == "json" {
		req.Header.Set("Accept", "application/json")
	} else {
		req.Header.Set("Accept", "application/proto")
	}

	/* Issue the request to the snapshot server */
	r, err := client.Do(req)
	if err != nil {
		log.Fatalf("Snapshotserver comm error: [%s] ", err)
	}
	defer r.Body.Close()

	/* Decode the Snapshot server response */
	var resp common.Snapshot
	err = json.NewDecoder(r.Body).Decode(&resp)
	if err != nil {

		if downloadBootSnapshot == false {
			log.Fatalf("JSON Response Data not parsable: %v", err)
		} else {
			/*
			 * If the data set is empty, allow it to proceed, as change server
			 * will feed data. Since Bootstrapping has passed, it has the
			 * Bootstrap config id to function.
			 */
			downloadDataSnapshot = true
			return
		}
	}

	if r.StatusCode == 200 {
		log.Info("Emit Snapshot response to plugins")
		events.ListenFunc(apid.EventDeliveredSelector, postPluginDataDelivery)
		events.Emit(ApigeeSyncEventSelector, &resp)

	} else {
		log.Fatalf("Snapshot server conn failed. HTTP Resp code %d", r.StatusCode)
	}

}
<|endoftext|>