<commit_before>package cluster

import (
	"github.com/appcelerator/amp/cli"
	"github.com/spf13/cobra"
)

// NewCreateCommand returns a new instance of the create command for bootstrapping an cluster.
func NewCreateCommand(c cli.Interface) *cobra.Command {
	cmd := &cobra.Command{
		Use:     "create [OPTIONS]",
		Short:   "Create an amp cluster",
		PreRunE: cli.NoArgs,
		RunE: func(cmd *cobra.Command, args []string) error {
			return create(c, cmd)
		},
	}

	flags := cmd.Flags()
	flags.IntVarP(&opts.workers, "workers", "w", 2, "Initial number of worker nodes")
	flags.IntVarP(&opts.managers, "managers", "m", 3, "Intial number of manager nodes")
	flags.StringVar(&opts.provider, "provider", "local", "Cluster provider")
	flags.StringVar(&opts.name, "name", "", "Cluster Label")
	flags.StringVar(&opts.tag, "tag", "latest", "Specify tag for cluster images (default is 'latest', use 'local' for development)")
	return cmd
}

// Map cli cluster flags to target bootstrap cluster command flags and update the cluster
func create(c cli.Interface, cmd *cobra.Command) error {
	// This is a map from cli cluster flag name to bootstrap script flag name
	m := map[string]string{
		"workers":  "-w",
		"managers": "-m",
		"provider": "-t",
		"name":     "-l",
	}

	// TODO: only supporting local cluster management for this release
	args := []string{"hack/deploy", DefaultLocalClusterID}
	return queryCluster(c, reflag(cmd, m, args))
}
<commit_msg>supply tag option to bootstrap -T<commit_after>package cluster

import (
	"github.com/appcelerator/amp/cli"
	"github.com/spf13/cobra"
)

// NewCreateCommand returns a new instance of the create command for bootstrapping an cluster.
func NewCreateCommand(c cli.Interface) *cobra.Command {
	cmd := &cobra.Command{
		Use:     "create [OPTIONS]",
		Short:   "Create an amp cluster",
		PreRunE: cli.NoArgs,
		RunE: func(cmd *cobra.Command, args []string) error {
			return create(c, cmd)
		},
	}

	flags := cmd.Flags()
	flags.IntVarP(&opts.workers, "workers", "w", 2, "Initial number of worker nodes")
	flags.IntVarP(&opts.managers, "managers", "m", 3, "Intial number of manager nodes")
	flags.StringVar(&opts.provider, "provider", "local", "Cluster provider")
	flags.StringVar(&opts.name, "name", "", "Cluster Label")
	flags.StringVar(&opts.tag, "tag", "latest", "Specify tag for cluster images (default is 'latest', use 'local' for development)")
	return cmd
}

// Map cli cluster flags to target bootstrap cluster command flags and update the cluster
func create(c cli.Interface, cmd *cobra.Command) error {
	// This is a map from cli cluster flag name to bootstrap script flag name
	m := map[string]string{
		"workers":  "-w",
		"managers": "-m",
		"provider": "-t",
		"name":     "-l",
		"tag":      "-T",
	}

	// TODO: only supporting local cluster management for this release
	// the following ensures that flags are added before the final command arg
	// TODO: refactor reflag to handle this
	args := []string{"hack/deploy"}
	args = reflag(cmd, m, args)
	args = append(args, DefaultLocalClusterID)
	return queryCluster(c, args)
}
<|endoftext|>


<commit_before>package getter

import (
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"

	urlhelper "github.com/hashicorp/go-getter/helper/url"
)

// HgGetter is a Getter implementation that will download a module from
// a Mercurial repository.
type HgGetter struct{}

func (g *HgGetter) Get(dst string, u *url.URL) error {
	if _, err := exec.LookPath("hg"); err != nil {
		return fmt.Errorf("hg must be available and on the PATH")
	}

	newURL, err := urlhelper.Parse(u.String())
	if err != nil {
		return err
	}
	if fixWindowsDrivePath(newURL) {
		// See valid file path form on http://www.selenic.com/hg/help/urls
		newURL.Path = fmt.Sprintf("/%s", newURL.Path)
	}

	// Extract some query parameters we use
	var rev string
	q := newURL.Query()
	if len(q) > 0 {
		rev = q.Get("rev")
		q.Del("rev")

		newURL.RawQuery = q.Encode()
	}

	_, err = os.Stat(dst)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if err != nil {
		if err := g.clone(dst, newURL); err != nil {
			return err
		}
	}

	if err := g.pull(dst, newURL); err != nil {
		return err
	}

	return g.update(dst, newURL, rev)
}

// GetFile for Hg doesn't support updating at this time. It will download
// the file every time.
func (g *HgGetter) GetFile(dst string, u *url.URL) error {
	td, err := ioutil.TempDir("", "getter-hg")
	if err != nil {
		return err
	}
	if err := os.RemoveAll(td); err != nil {
		return err
	}

	// Get the filename, and strip the filename from the URL so we can
	// just get the repository directly.
	filename := filepath.Base(u.Path)
	u.Path = filepath.Dir(u.Path)

	// Get the full repository
	if err := g.Get(td, u); err != nil {
		return err
	}

	// Copy the single file
	u, err = urlhelper.Parse(fmtFileURL(filepath.Join(td, filename)))
	if err != nil {
		return err
	}

	fg := &FileGetter{Copy: true}
	return fg.GetFile(dst, u)
}

func (g *HgGetter) clone(dst string, u *url.URL) error {
	cmd := exec.Command("hg", "clone", "-U", u.String(), dst)
	return getRunCommand(cmd)
}

func (g *HgGetter) pull(dst string, u *url.URL) error {
	cmd := exec.Command("hg", "pull")
	cmd.Dir = dst
	return getRunCommand(cmd)
}

func (g *HgGetter) update(dst string, u *url.URL, rev string) error {
	args := []string{"update"}
	if rev != "" {
		args = append(args, rev)
	}

	cmd := exec.Command("hg", args...)
	cmd.Dir = dst
	return getRunCommand(cmd)
}

func fixWindowsDrivePath(u *url.URL) bool {
	// hg assumes a file:/// prefix for Windows drive letter file paths.
	// (e.g. file:///c:/foo/bar)
	// If the URL Path does not begin with a '/' character, the resulting URL
	// path will have a file:// prefix. (e.g. file://c:/foo/bar)
	// See http://www.selenic.com/hg/help/urls and the examples listed in
	// http://selenic.com/repo/hg-stable/file/1265a3a71d75/mercurial/util.py#l1936
	return runtime.GOOS == "windows" && u.Scheme == "file" &&
		len(u.Path) > 1 && u.Path[0] != '/' && u.Path[1] == ':'
}
<commit_msg>hg/windows: Fix GetFile<commit_after>package getter

import (
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"

	urlhelper "github.com/hashicorp/go-getter/helper/url"
)

// HgGetter is a Getter implementation that will download a module from
// a Mercurial repository.
type HgGetter struct{}

func (g *HgGetter) Get(dst string, u *url.URL) error {
	if _, err := exec.LookPath("hg"); err != nil {
		return fmt.Errorf("hg must be available and on the PATH")
	}

	newURL, err := urlhelper.Parse(u.String())
	if err != nil {
		return err
	}
	if fixWindowsDrivePath(newURL) {
		// See valid file path form on http://www.selenic.com/hg/help/urls
		newURL.Path = fmt.Sprintf("/%s", newURL.Path)
	}

	// Extract some query parameters we use
	var rev string
	q := newURL.Query()
	if len(q) > 0 {
		rev = q.Get("rev")
		q.Del("rev")

		newURL.RawQuery = q.Encode()
	}

	_, err = os.Stat(dst)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if err != nil {
		if err := g.clone(dst, newURL); err != nil {
			return err
		}
	}

	if err := g.pull(dst, newURL); err != nil {
		return err
	}

	return g.update(dst, newURL, rev)
}

// GetFile for Hg doesn't support updating at this time. It will download
// the file every time.
func (g *HgGetter) GetFile(dst string, u *url.URL) error {
	td, err := ioutil.TempDir("", "getter-hg")
	if err != nil {
		return err
	}
	if err := os.RemoveAll(td); err != nil {
		return err
	}

	// Get the filename, and strip the filename from the URL so we can
	// just get the repository directly.
	filename := filepath.Base(u.Path)
	u.Path = filepath.ToSlash(filepath.Dir(u.Path))

	// If we're on Windows, we need to set the host to "localhost" for hg
	if runtime.GOOS == "windows" {
		u.Host = "localhost"
	}

	// Get the full repository
	if err := g.Get(td, u); err != nil {
		return err
	}

	// Copy the single file
	u, err = urlhelper.Parse(fmtFileURL(filepath.Join(td, filename)))
	if err != nil {
		return err
	}

	fg := &FileGetter{Copy: true}
	return fg.GetFile(dst, u)
}

func (g *HgGetter) clone(dst string, u *url.URL) error {
	cmd := exec.Command("hg", "clone", "-U", u.String(), dst)
	return getRunCommand(cmd)
}

func (g *HgGetter) pull(dst string, u *url.URL) error {
	cmd := exec.Command("hg", "pull")
	cmd.Dir = dst
	return getRunCommand(cmd)
}

func (g *HgGetter) update(dst string, u *url.URL, rev string) error {
	args := []string{"update"}
	if rev != "" {
		args = append(args, rev)
	}

	cmd := exec.Command("hg", args...)
	cmd.Dir = dst
	return getRunCommand(cmd)
}

func fixWindowsDrivePath(u *url.URL) bool {
	// hg assumes a file:/// prefix for Windows drive letter file paths.
	// (e.g. file:///c:/foo/bar)
	// If the URL Path does not begin with a '/' character, the resulting URL
	// path will have a file:// prefix. (e.g. file://c:/foo/bar)
	// See http://www.selenic.com/hg/help/urls and the examples listed in
	// http://selenic.com/repo/hg-stable/file/1265a3a71d75/mercurial/util.py#l1936
	return runtime.GOOS == "windows" && u.Scheme == "file" &&
		len(u.Path) > 1 && u.Path[0] != '/' && u.Path[1] == ':'
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/coreos/etcd/storage/storagepb"
)

const (
	// chanBufLen is the length of the buffered chan
	// for sending out watched events.
	// TODO: find a good buf value. 1024 is just a random one that
	// seems to be reasonable.
	chanBufLen = 1024
)

type watchable interface {
	watch(key []byte, prefix bool, startRev, id int64, ch chan<- storagepb.Event) (*watching, CancelFunc)
}

type watchableStore struct {
	mu sync.Mutex

	*store

	// contains all unsynced watching that needs to sync events that have happened
	unsynced map[*watching]struct{}

	// contains all synced watching that are tracking the events that will happen
	// The key of the map is the key that the watching is watching on.
	synced map[string]map[*watching]struct{}
	tx     *ongoingTx

	stopc chan struct{}
	wg    sync.WaitGroup
}

func newWatchableStore(path string) *watchableStore {
	s := &watchableStore{
		store:    newStore(path),
		unsynced: make(map[*watching]struct{}),
		synced:   make(map[string]map[*watching]struct{}),
		stopc:    make(chan struct{}),
	}
	s.wg.Add(1)
	go s.syncWatchingsLoop()
	return s
}

func (s *watchableStore) Put(key, value []byte) (rev int64) {
	s.mu.Lock()
	defer s.mu.Unlock()

	rev = s.store.Put(key, value)
	// TODO: avoid this range
	kvs, _, err := s.store.Range(key, nil, 0, rev)
	if err != nil {
		log.Panicf("unexpected range error (%v)", err)
	}
	s.handle(rev, storagepb.Event{
		Type: storagepb.PUT,
		Kv:   &kvs[0],
	})
	return rev
}

func (s *watchableStore) DeleteRange(key, end []byte) (n, rev int64) {
	s.mu.Lock()
	defer s.mu.Unlock()

	// TODO: avoid this range
	kvs, _, err := s.store.Range(key, end, 0, 0)
	if err != nil {
		log.Panicf("unexpected range error (%v)", err)
	}
	n, rev = s.store.DeleteRange(key, end)
	for _, kv := range kvs {
		s.handle(rev, storagepb.Event{
			Type: storagepb.DELETE,
			Kv: &storagepb.KeyValue{
				Key: kv.Key,
			},
		})
	}
	return n, rev
}

func (s *watchableStore) TxnBegin() int64 {
	s.mu.Lock()
	s.tx = newOngoingTx()
	return s.store.TxnBegin()
}

func (s *watchableStore) TxnPut(txnID int64, key, value []byte) (rev int64, err error) {
	rev, err = s.store.TxnPut(txnID, key, value)
	if err == nil {
		s.tx.put(string(key))
	}
	return rev, err
}

func (s *watchableStore) TxnDeleteRange(txnID int64, key, end []byte) (n, rev int64, err error) {
	kvs, _, err := s.store.TxnRange(txnID, key, end, 0, 0)
	if err != nil {
		log.Panicf("unexpected range error (%v)", err)
	}
	n, rev, err = s.store.TxnDeleteRange(txnID, key, end)
	if err == nil {
		for _, kv := range kvs {
			s.tx.del(string(kv.Key))
		}
	}
	return n, rev, err
}

func (s *watchableStore) TxnEnd(txnID int64) error {
	err := s.store.TxnEnd(txnID)
	if err != nil {
		return err
	}

	_, rev, _ := s.store.Range(nil, nil, 0, 0)
	for k := range s.tx.putm {
		kvs, _, err := s.store.Range([]byte(k), nil, 0, 0)
		if err != nil {
			log.Panicf("unexpected range error (%v)", err)
		}
		s.handle(rev, storagepb.Event{
			Type: storagepb.PUT,
			Kv:   &kvs[0],
		})
	}
	for k := range s.tx.delm {
		s.handle(rev, storagepb.Event{
			Type: storagepb.DELETE,
			Kv: &storagepb.KeyValue{
				Key: []byte(k),
			},
		})
	}
	s.mu.Unlock()
	return nil
}

func (s *watchableStore) Close() error {
	close(s.stopc)
	s.wg.Wait()
	return s.store.Close()
}

func (s *watchableStore) NewWatcher() Watcher {
	watcherGauge.Inc()
	return &watcher{
		watchable: s,
		ch:        make(chan storagepb.Event, chanBufLen),
	}
}

func (s *watchableStore) watch(key []byte, prefix bool, startRev, id int64, ch chan<- storagepb.Event) (*watching, CancelFunc) {
	s.mu.Lock()
	defer s.mu.Unlock()

	wa := &watching{
		key:    key,
		prefix: prefix,
		cur:    startRev,
		id:     id,
		ch:     ch,
	}

	k := string(key)
	if startRev == 0 {
		if err := unsafeAddWatching(&s.synced, k, wa); err != nil {
			log.Panicf("error unsafeAddWatching (%v) for key %s", err, k)
		}
	} else {
		slowWatchingGauge.Inc()
		s.unsynced[wa] = struct{}{}
	}
	watchingGauge.Inc()

	cancel := CancelFunc(func() {
		s.mu.Lock()
		defer s.mu.Unlock()
		// remove global references of the watching
		if _, ok := s.unsynced[wa]; ok {
			delete(s.unsynced, wa)
			slowWatchingGauge.Dec()
			watchingGauge.Dec()
			return
		}

		if v, ok := s.synced[k]; ok {
			if _, ok := v[wa]; ok {
				delete(v, wa)
				// if there is nothing in s.synced[k],
				// remove the key from the synced
				if len(v) == 0 {
					delete(s.synced, k)
				}
				watchingGauge.Dec()
			}
		}
		// If we cannot find it, it should have finished watch.
	})

	return wa, cancel
}

// syncWatchingsLoop syncs the watching in the unsyncd map every 100ms.
func (s *watchableStore) syncWatchingsLoop() {
	defer s.wg.Done()

	for {
		s.mu.Lock()
		s.syncWatchings()
		s.mu.Unlock()

		select {
		case <-time.After(100 * time.Millisecond):
		case <-s.stopc:
			return
		}
	}
}

// syncWatchings syncs the watchings in the unsyncd map.
func (s *watchableStore) syncWatchings() {
	_, curRev, _ := s.store.Range(nil, nil, 0, 0)
	for w := range s.unsynced {
		var end []byte
		if w.prefix {
			end = make([]byte, len(w.key))
			copy(end, w.key)
			end[len(w.key)-1]++
		}
		limit := cap(w.ch) - len(w.ch)
		// the channel is full, try it in the next round
		if limit == 0 {
			continue
		}
		revbs, kvs, nextRev, err := s.store.RangeHistory(w.key, end, int64(limit), w.cur)
		if err != nil {
			// TODO: send error event to watching
			delete(s.unsynced, w)
			continue
		}

		// push events to the channel
		for i, kv := range kvs {
			var evt storagepb.Event_EventType
			switch {
			case isTombstone(revbs[i]):
				evt = storagepb.DELETE
			default:
				evt = storagepb.PUT
			}

			w.ch <- storagepb.Event{
				Type:    evt,
				Kv:      &kv,
				WatchID: w.id,
			}
			pendingEventsGauge.Inc()
		}
		// switch to tracking future events if needed
		if nextRev > curRev {
			k := string(w.key)
			if err := unsafeAddWatching(&s.synced, k, w); err != nil {
				log.Panicf("error unsafeAddWatching (%v) for key %s", err, k)
			}
			delete(s.unsynced, w)
			continue
		}
		// put it back to try it in the next round
		w.cur = nextRev
	}
	slowWatchingGauge.Set(float64(len(s.unsynced)))
}

// handle handles the change of the happening event on all watchings.
func (s *watchableStore) handle(rev int64, ev storagepb.Event) {
	s.notify(rev, ev)
}

// notify notifies the fact that given event at the given rev just happened to
// watchings that watch on the key of the event.
func (s *watchableStore) notify(rev int64, ev storagepb.Event) {
	// check all prefixes of the key to notify all corresponded watchings
	for i := 0; i <= len(ev.Kv.Key); i++ {
		k := string(ev.Kv.Key[:i])
		if wm, ok := s.synced[k]; ok {
			for w := range wm {
				// the watching needs to be notified when either it watches prefix or
				// the key is exactly matched.
				if !w.prefix && i != len(ev.Kv.Key) {
					continue
				}
				ev.WatchID = w.id
				select {
				case w.ch <- ev:
					pendingEventsGauge.Inc()
				default:
					w.cur = rev
					s.unsynced[w] = struct{}{}
					delete(wm, w)
					slowWatchingGauge.Inc()
				}
			}
		}
	}
}

type ongoingTx struct {
	// keys put/deleted in the ongoing txn
	putm map[string]struct{}
	delm map[string]struct{}
}

func newOngoingTx() *ongoingTx {
	return &ongoingTx{
		putm: make(map[string]struct{}),
		delm: make(map[string]struct{}),
	}
}

func (tx *ongoingTx) put(k string) {
	tx.putm[k] = struct{}{}
	if _, ok := tx.delm[k]; ok {
		delete(tx.delm, k)
	}
}

func (tx *ongoingTx) del(k string) {
	tx.delm[k] = struct{}{}
	if _, ok := tx.putm[k]; ok {
		delete(tx.putm, k)
	}
}

type watching struct {
	// the watching key
	key []byte
	// prefix indicates if watching is on a key or a prefix.
	// If prefix is true, the watching is on a prefix.
	prefix bool
	// cur is the current watching revision.
	// If cur is behind the current revision of the KV,
	// watching is unsynced and needs to catch up.
	cur int64
	id  int64

	// a chan to send out the watched events.
	// The chan might be shared with other watchings.
	ch chan<- storagepb.Event
}

// unsafeAddWatching puts watching with key k into watchableStore's synced.
// Make sure to this is thread-safe using mutex before and after.
func unsafeAddWatching(synced *map[string]map[*watching]struct{}, k string, wa *watching) error {
	if wa == nil {
		return fmt.Errorf("nil watching received")
	}
	mp := *synced
	if v, ok := mp[k]; ok {
		if _, ok := v[wa]; ok {
			return fmt.Errorf("put the same watch twice: %+v", wa)
		} else {
			v[wa] = struct{}{}
		}
		return nil
	}

	mp[k] = make(map[*watching]struct{})
	mp[k][wa] = struct{}{}
	return nil
}
<commit_msg>storage: range all unsynced at once<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"fmt"
	"log"
	"math"
	"sync"
	"time"

	"github.com/coreos/etcd/storage/storagepb"
)

const (
	// chanBufLen is the length of the buffered chan
	// for sending out watched events.
	// TODO: find a good buf value. 1024 is just a random one that
	// seems to be reasonable.
	chanBufLen = 1024
)

type watchable interface {
	watch(key []byte, prefix bool, startRev, id int64, ch chan<- storagepb.Event) (*watching, CancelFunc)
}

type watchableStore struct {
	mu sync.Mutex

	*store

	// contains all unsynced watching that needs to sync events that have happened
	unsynced map[*watching]struct{}

	// contains all synced watching that are tracking the events that will happen
	// The key of the map is the key that the watching is watching on.
	synced map[string]map[*watching]struct{}
	tx     *ongoingTx

	stopc chan struct{}
	wg    sync.WaitGroup
}

func newWatchableStore(path string) *watchableStore {
	s := &watchableStore{
		store:    newStore(path),
		unsynced: make(map[*watching]struct{}),
		synced:   make(map[string]map[*watching]struct{}),
		stopc:    make(chan struct{}),
	}
	s.wg.Add(1)
	go s.syncWatchingsLoop()
	return s
}

func (s *watchableStore) Put(key, value []byte) (rev int64) {
	s.mu.Lock()
	defer s.mu.Unlock()

	rev = s.store.Put(key, value)
	// TODO: avoid this range
	kvs, _, err := s.store.Range(key, nil, 0, rev)
	if err != nil {
		log.Panicf("unexpected range error (%v)", err)
	}
	s.handle(rev, storagepb.Event{
		Type: storagepb.PUT,
		Kv:   &kvs[0],
	})
	return rev
}

func (s *watchableStore) DeleteRange(key, end []byte) (n, rev int64) {
	s.mu.Lock()
	defer s.mu.Unlock()

	// TODO: avoid this range
	kvs, _, err := s.store.Range(key, end, 0, 0)
	if err != nil {
		log.Panicf("unexpected range error (%v)", err)
	}
	n, rev = s.store.DeleteRange(key, end)
	for _, kv := range kvs {
		s.handle(rev, storagepb.Event{
			Type: storagepb.DELETE,
			Kv: &storagepb.KeyValue{
				Key: kv.Key,
			},
		})
	}
	return n, rev
}

func (s *watchableStore) TxnBegin() int64 {
	s.mu.Lock()
	s.tx = newOngoingTx()
	return s.store.TxnBegin()
}

func (s *watchableStore) TxnPut(txnID int64, key, value []byte) (rev int64, err error) {
	rev, err = s.store.TxnPut(txnID, key, value)
	if err == nil {
		s.tx.put(string(key))
	}
	return rev, err
}

func (s *watchableStore) TxnDeleteRange(txnID int64, key, end []byte) (n, rev int64, err error) {
	kvs, _, err := s.store.TxnRange(txnID, key, end, 0, 0)
	if err != nil {
		log.Panicf("unexpected range error (%v)", err)
	}
	n, rev, err = s.store.TxnDeleteRange(txnID, key, end)
	if err == nil {
		for _, kv := range kvs {
			s.tx.del(string(kv.Key))
		}
	}
	return n, rev, err
}

func (s *watchableStore) TxnEnd(txnID int64) error {
	err := s.store.TxnEnd(txnID)
	if err != nil {
		return err
	}

	_, rev, _ := s.store.Range(nil, nil, 0, 0)
	for k := range s.tx.putm {
		kvs, _, err := s.store.Range([]byte(k), nil, 0, 0)
		if err != nil {
			log.Panicf("unexpected range error (%v)", err)
		}
		s.handle(rev, storagepb.Event{
			Type: storagepb.PUT,
			Kv:   &kvs[0],
		})
	}
	for k := range s.tx.delm {
		s.handle(rev, storagepb.Event{
			Type: storagepb.DELETE,
			Kv: &storagepb.KeyValue{
				Key: []byte(k),
			},
		})
	}
	s.mu.Unlock()
	return nil
}

func (s *watchableStore) Close() error {
	close(s.stopc)
	s.wg.Wait()
	return s.store.Close()
}

func (s *watchableStore) NewWatcher() Watcher {
	watcherGauge.Inc()
	return &watcher{
		watchable: s,
		ch:        make(chan storagepb.Event, chanBufLen),
	}
}

func (s *watchableStore) watch(key []byte, prefix bool, startRev, id int64, ch chan<- storagepb.Event) (*watching, CancelFunc) {
	s.mu.Lock()
	defer s.mu.Unlock()

	wa := &watching{
		key:    key,
		prefix: prefix,
		cur:    startRev,
		id:     id,
		ch:     ch,
	}

	k := string(key)
	if startRev == 0 {
		if err := unsafeAddWatching(&s.synced, k, wa); err != nil {
			log.Panicf("error unsafeAddWatching (%v) for key %s", err, k)
		}
	} else {
		slowWatchingGauge.Inc()
		s.unsynced[wa] = struct{}{}
	}
	watchingGauge.Inc()

	cancel := CancelFunc(func() {
		s.mu.Lock()
		defer s.mu.Unlock()
		// remove global references of the watching
		if _, ok := s.unsynced[wa]; ok {
			delete(s.unsynced, wa)
			slowWatchingGauge.Dec()
			watchingGauge.Dec()
			return
		}

		if v, ok := s.synced[k]; ok {
			if _, ok := v[wa]; ok {
				delete(v, wa)
				// if there is nothing in s.synced[k],
				// remove the key from the synced
				if len(v) == 0 {
					delete(s.synced, k)
				}
				watchingGauge.Dec()
			}
		}
		// If we cannot find it, it should have finished watch.
	})

	return wa, cancel
}

// syncWatchingsLoop syncs the watching in the unsyncd map every 100ms.
func (s *watchableStore) syncWatchingsLoop() {
	defer s.wg.Done()

	for {
		s.mu.Lock()
		s.syncWatchings()
		s.mu.Unlock()

		select {
		case <-time.After(100 * time.Millisecond):
		case <-s.stopc:
			return
		}
	}
}

// syncWatchings periodically syncs unsynced watchings by: Iterate all unsynced
// watchings to get the minimum revision within its range, skipping the
// watching if its current revision is behind the compact revision of the
// store. And use this minimum revision to get all key-value pairs. Then send
// those events to watchings.
func (s *watchableStore) syncWatchings() {
	s.store.mu.Lock()
	defer s.store.mu.Unlock()

	if len(s.unsynced) == 0 {
		return
	}

	// in order to find key-value pairs from unsynced watchings, we need to
	// find min revision index, and these revisions can be used to
	// query the backend store of key-value pairs
	minRev := int64(math.MaxInt64)

	curRev := s.store.currentRev.main
	compactionRev := s.store.compactMainRev

	// TODO: change unsynced struct type same to this
	keyToUnsynced := make(map[string]map[*watching]struct{})

	for w := range s.unsynced {
		k := string(w.key)

		if w.cur > curRev {
			panic("watching current revision should not exceed current revision")
		}

		if w.cur < compactionRev {
			// TODO: return error compacted to that watching instead of
			// just removing it sliently from unsynced.
			delete(s.unsynced, w)
			continue
		}

		if minRev >= w.cur {
			minRev = w.cur
		}

		if _, ok := keyToUnsynced[k]; !ok {
			keyToUnsynced[k] = make(map[*watching]struct{})
		}
		keyToUnsynced[k][w] = struct{}{}
	}

	minBytes, maxBytes := newRevBytes(), newRevBytes()
	revToBytes(revision{main: minRev}, minBytes)
	revToBytes(revision{main: curRev + 1}, maxBytes)

	// UnsafeRange returns keys and values. And in boltdb, keys are revisions.
	// values are actual key-value pairs in backend.
	tx := s.store.b.BatchTx()
	tx.Lock()
	ks, vs := tx.UnsafeRange(keyBucketName, minBytes, maxBytes, 0)
	tx.Unlock()

	for i, v := range vs {
		var kv storagepb.KeyValue
		if err := kv.Unmarshal(v); err != nil {
			log.Panicf("storage: cannot unmarshal event: %v", err)
		}

		k := string(kv.Key)
		wm, ok := keyToUnsynced[k]
		if !ok {
			continue
		}

		var ev storagepb.Event
		switch {
		case isTombstone(ks[i]):
			ev.Type = storagepb.DELETE
		default:
			ev.Type = storagepb.PUT
		}
		ev.Kv = &kv

		for w := range wm {
			ev.WatchID = w.id

			select {
			case w.ch <- ev:
				pendingEventsGauge.Inc()
			default:
				// TODO: handle the full unsynced watchings.
				// continue to process other watchings for now, the full ones
				// will be processed next time and hopefully it will not be full.
				continue
			}
			if err := unsafeAddWatching(&s.synced, k, w); err != nil {
				log.Panicf("error unsafeAddWatching (%v) for key %s", err, k)
			}
			delete(s.unsynced, w)
		}
	}

	slowWatchingGauge.Set(float64(len(s.unsynced)))
}

// handle handles the change of the happening event on all watchings.
func (s *watchableStore) handle(rev int64, ev storagepb.Event) {
	s.notify(rev, ev)
}

// notify notifies the fact that given event at the given rev just happened to
// watchings that watch on the key of the event.
func (s *watchableStore) notify(rev int64, ev storagepb.Event) {
	// check all prefixes of the key to notify all corresponded watchings
	for i := 0; i <= len(ev.Kv.Key); i++ {
		k := string(ev.Kv.Key[:i])
		if wm, ok := s.synced[k]; ok {
			for w := range wm {
				// the watching needs to be notified when either it watches prefix or
				// the key is exactly matched.
				if !w.prefix && i != len(ev.Kv.Key) {
					continue
				}
				ev.WatchID = w.id
				select {
				case w.ch <- ev:
					pendingEventsGauge.Inc()
				default:
					w.cur = rev
					s.unsynced[w] = struct{}{}
					delete(wm, w)
					slowWatchingGauge.Inc()
				}
			}
		}
	}
}

type ongoingTx struct {
	// keys put/deleted in the ongoing txn
	putm map[string]struct{}
	delm map[string]struct{}
}

func newOngoingTx() *ongoingTx {
	return &ongoingTx{
		putm: make(map[string]struct{}),
		delm: make(map[string]struct{}),
	}
}

func (tx *ongoingTx) put(k string) {
	tx.putm[k] = struct{}{}
	if _, ok := tx.delm[k]; ok {
		delete(tx.delm, k)
	}
}

func (tx *ongoingTx) del(k string) {
	tx.delm[k] = struct{}{}
	if _, ok := tx.putm[k]; ok {
		delete(tx.putm, k)
	}
}

type watching struct {
	// the watching key
	key []byte
	// prefix indicates if watching is on a key or a prefix.
	// If prefix is true, the watching is on a prefix.
	prefix bool
	// cur is the current watching revision.
	// If cur is behind the current revision of the KV,
	// watching is unsynced and needs to catch up.
	cur int64
	id  int64

	// a chan to send out the watched events.
	// The chan might be shared with other watchings.
	ch chan<- storagepb.Event
}

// unsafeAddWatching puts watching with key k into watchableStore's synced.
// Make sure to this is thread-safe using mutex before and after.
func unsafeAddWatching(synced *map[string]map[*watching]struct{}, k string, wa *watching) error {
	if wa == nil {
		return fmt.Errorf("nil watching received")
	}
	mp := *synced
	if v, ok := mp[k]; ok {
		if _, ok := v[wa]; ok {
			return fmt.Errorf("put the same watch twice: %+v", wa)
		} else {
			v[wa] = struct{}{}
		}
		return nil
	}

	mp[k] = make(map[*watching]struct{})
	mp[k][wa] = struct{}{}
	return nil
}
<|endoftext|>


<commit_before>package ircbot

import (
	"fmt"
	"math/rand"
	"strings"
)

type ActionFunc func(b *IrcBot, m *IrcMsg)

type Action struct {
	command string
	do      ActionFunc
}

func Pong(b *IrcBot, m *IrcMsg) {

	s := fmt.Sprintf("PONG %s", strings.Join(m.args, " "))
	fmt.Println("irc >> ", s)
	b.writer.PrintfLine(s)
}

func Join(b *IrcBot, m *IrcMsg) {
	if m.nick == b.Nick {
		b.joined = true
		return
	}

	s := fmt.Sprintf("%s :Salut %s", m.channel, m.nick)
	b.Out <- &IrcMsg{
		command: "PRIVMSG",
		args:    []string{s},
	}
}

func Respond(b *IrcBot, m *IrcMsg) {
	response := []string{
		"oui ?",
		"on parle de moi ?",
		"Je suis pas là",
	}

	s := strings.Join(m.args, " ")

	if strings.Contains(s, b.Nick) {
		nbr := rand.Intn(len(response))
		line := fmt.Sprintf("%s :%s", m.channel, response[nbr])
		b.Out <- &IrcMsg{
			command: "PRIVMSG",
			args:    []string{line},
		}
	}
}
<commit_msg>validation connection<commit_after>package ircbot

import (
	"fmt"
	"math/rand"
	"strings"
)

type ActionFunc func(b *IrcBot, m *IrcMsg)

type Action struct {
	command string
	do      ActionFunc
}

func Pong(b *IrcBot, m *IrcMsg) {

	s := fmt.Sprintf("PONG %s", strings.Join(m.args, " "))
	fmt.Println("irc >> ", s)
	b.writer.PrintfLine(s)
}

func Join(b *IrcBot, m *IrcMsg) {
	if m.nick == b.Nick {
		b.joined = true
		return
	}

	s := fmt.Sprintf("%s :Salut %s", m.channel, m.nick)
	b.Out <- &IrcMsg{
		command: "PRIVMSG",
		args:    []string{s},
	}
}

func Respond(b *IrcBot, m *IrcMsg) {
	response := []string{
		"oui ?",
		"on parle de moi ?",
		"Je suis pas là",
	}

	s := strings.Join(m.args, " ")

	if strings.Contains(s, b.Nick) {
		nbr := rand.Intn(len(response))
		line := fmt.Sprintf("%s :%s", m.channel, response[nbr])
		b.Out <- &IrcMsg{
			command: "PRIVMSG",
			args:    []string{line},
		}
	}
}

func ValidConnect(b *IrcBot, m *IrcMsg) {
	s := strings.Join(m.args, " ")
	if strings.Contains(s, b.Nick) {
		fmt.Println("Info : connection terminated")
		b.joined = true
	}
}
<|endoftext|>


<commit_before>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

package service

import (
	"bytes"
	"encoding/json"
	"github.com/zenoss/glog"
	"math"
	"text/template"
)

func parent(gs GetService) func(s Service) (Service, error) {
	return func(svc Service) (Service, error) {
		return gs(svc.ParentServiceID)
	}
}
func context() func(s Service) (map[string]interface{}, error) {
	return func(s Service) (map[string]interface{}, error) {
		ctx := make(map[string]interface{})
		err := json.Unmarshal([]byte(s.Context), &ctx)
		if err != nil {
			glog.Errorf("Error unmarshal service context Id=%s: %s -> %s", s.Id, s.Context, err)
		}
		return ctx, err
	}
}

// EvaluateActionsTemplate parses and evaluates the Actions string of a service.
func (service *Service) EvaluateActionsTemplate(gs GetService) (err error) {
	for key, value := range service.Actions {
		result := service.evaluateTemplate(gs, value)
		if result != "" {
			service.Actions[key] = result
		}
	}
	return
}

// EvaluateStartupTemplate parses and evaluates the StartUp string of a service.
func (service *Service) EvaluateStartupTemplate(gs GetService) (err error) {

	result := service.evaluateTemplate(gs, service.Startup)
	if result != "" {
		service.Startup = result
	}

	return
}

// EvaluateRunsTemplate parses and evaluates the Runs string of a service.
func (service *Service) EvaluateRunsTemplate(gs GetService) (err error) {
	for key, value := range service.Runs {
		result := service.evaluateTemplate(gs, value)
		if result != "" {
			service.Runs[key] = result
		}
	}
	return
}

// evaluateTemplate takes a control plane client and template string and evaluates
// the template using the service as the context. If the template is invalid or there is an error
// then an empty string is returned.
func (service *Service) evaluateTemplate(gs GetService, serviceTemplate string) string {
	functions := template.FuncMap{
		"parent":       parent(gs),
		"context":      context(),
		"percentScale": percentScale,
		"bytesToMB":    bytesToMB,
	}

	glog.V(3).Infof("Evaluating template string %v", serviceTemplate)
	// parse the template
	t := template.Must(template.New("ServiceDefinitionTemplate").Funcs(functions).Parse(serviceTemplate))

	// evaluate it
	var buffer bytes.Buffer
	err := t.Execute(&buffer, service)
	if err == nil {
		return buffer.String()
	}

	// something went wrong, warn them
	glog.Warningf("Evaluating template %v produced the following error %v ", serviceTemplate, err)
	return ""
}

// EvaluateLogConfigTemplate parses and evals the Path, Type and all the values for the tags of the log
// configs. This happens for each LogConfig on the service.
func (service *Service) EvaluateLogConfigTemplate(gs GetService) (err error) {
	// evaluate the template for the LogConfig as well as the tags
	for i, logConfig := range service.LogConfigs {
		// Path
		result := service.evaluateTemplate(gs, logConfig.Path)
		if result != "" {
			service.LogConfigs[i].Path = result
		}
		// Type
		result = service.evaluateTemplate(gs, logConfig.Type)
		if result != "" {
			service.LogConfigs[i].Type = result
		}

		// Tags
		for j, tag := range logConfig.LogTags {
			result = service.evaluateTemplate(gs, tag.Value)
			if result != "" {
				service.LogConfigs[i].LogTags[j].Value = result
			}
		}
	}
	return
}

// EvaluateConfigFilesTemplate parses and evals the Filename and Content. This happens for each
// ConfigFile on the service.
func (service *Service) EvaluateConfigFilesTemplate(gs GetService) (err error) {
	glog.V(3).Infof("Evaluating Config Files for %s", service.Id)
	for key, configFile := range service.ConfigFiles {
		glog.V(3).Infof("Evaluating Config File: %v", key)
		// Filename
		result := service.evaluateTemplate(gs, configFile.Filename)
		if result != "" {
			configFile.Filename = result
		}
		// Content
		result = service.evaluateTemplate(gs, configFile.Content)
		if result != "" {
			configFile.Content = result
		}
		service.ConfigFiles[key] = configFile
	}
	return
}

func percentScale(x uint64, percentage float64) uint64 {
	return uint64(math.Floor(float64(x) * percentage))
}

func bytesToMB(x uint64) uint64 {
	return x / (1024 * 1024)
}

// EvaluateEndpointTemplates parses and evaluates the "ApplicationTemplate" property
// of each of the service endpoints for this service.
func (service *Service) EvaluateEndpointTemplates(gs GetService) (err error) {
	functions := template.FuncMap{
		"parent":       parent(gs),
		"context":      context(),
		"percentScale": percentScale,
		"bytesToMB":    bytesToMB,
	}

	for i, ep := range service.Endpoints {
		if ep.Application != "" && ep.ApplicationTemplate == "" {
			ep.ApplicationTemplate = ep.Application
			service.Endpoints[i].ApplicationTemplate = ep.Application
		}
		if ep.ApplicationTemplate != "" {
			t := template.Must(template.New(service.Name).Funcs(functions).Parse(ep.ApplicationTemplate))
			var buffer bytes.Buffer
			if err = t.Execute(&buffer, service); err == nil {
				service.Endpoints[i].Application = buffer.String()
			} else {
				return
			}
		}
	}
	return
}

// Fill in all the templates in the ServiceDefinitions
func (service *Service) Evaluate(getSvc GetService) error {
	if err := service.EvaluateEndpointTemplates(getSvc); err != nil {
		return err
	}
	if err := service.EvaluateLogConfigTemplate(getSvc); err != nil {
		return err
	}
	if err := service.EvaluateConfigFilesTemplate(getSvc); err != nil {
		return err
	}
	if err := service.EvaluateStartupTemplate(getSvc); err != nil {
		return err
	}
	if err := service.EvaluateRunsTemplate(getSvc); err != nil {
		return err
	}
	if err := service.EvaluateActionsTemplate(getSvc); err != nil {
		return err
	}

	return nil
}
<commit_msg>gofmt<commit_after>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

package service

import (
	"github.com/zenoss/glog"

	"bytes"
	"encoding/json"
	"math"
	"text/template"
)

func parent(gs GetService) func(s Service) (Service, error) {
	return func(svc Service) (Service, error) {
		return gs(svc.ParentServiceID)
	}
}
func context() func(s Service) (map[string]interface{}, error) {
	return func(s Service) (map[string]interface{}, error) {
		ctx := make(map[string]interface{})
		err := json.Unmarshal([]byte(s.Context), &ctx)
		if err != nil {
			glog.Errorf("Error unmarshal service context Id=%s: %s -> %s", s.Id, s.Context, err)
		}
		return ctx, err
	}
}

// EvaluateActionsTemplate parses and evaluates the Actions string of a service.
func (service *Service) EvaluateActionsTemplate(gs GetService) (err error) {
	for key, value := range service.Actions {
		result := service.evaluateTemplate(gs, value)
		if result != "" {
			service.Actions[key] = result
		}
	}
	return
}

// EvaluateStartupTemplate parses and evaluates the StartUp string of a service.
func (service *Service) EvaluateStartupTemplate(gs GetService) (err error) {

	result := service.evaluateTemplate(gs, service.Startup)
	if result != "" {
		service.Startup = result
	}

	return
}

// EvaluateRunsTemplate parses and evaluates the Runs string of a service.
func (service *Service) EvaluateRunsTemplate(gs GetService) (err error) {
	for key, value := range service.Runs {
		result := service.evaluateTemplate(gs, value)
		if result != "" {
			service.Runs[key] = result
		}
	}
	return
}

// evaluateTemplate takes a control plane client and template string and evaluates
// the template using the service as the context. If the template is invalid or there is an error
// then an empty string is returned.
func (service *Service) evaluateTemplate(gs GetService, serviceTemplate string) string {
	functions := template.FuncMap{
		"parent":       parent(gs),
		"context":      context(),
		"percentScale": percentScale,
		"bytesToMB":    bytesToMB,
	}

	glog.V(3).Infof("Evaluating template string %v", serviceTemplate)
	// parse the template
	t := template.Must(template.New("ServiceDefinitionTemplate").Funcs(functions).Parse(serviceTemplate))

	// evaluate it
	var buffer bytes.Buffer
	err := t.Execute(&buffer, service)
	if err == nil {
		return buffer.String()
	}

	// something went wrong, warn them
	glog.Warningf("Evaluating template %v produced the following error %v ", serviceTemplate, err)
	return ""
}

// EvaluateLogConfigTemplate parses and evals the Path, Type and all the values for the tags of the log
// configs. This happens for each LogConfig on the service.
func (service *Service) EvaluateLogConfigTemplate(gs GetService) (err error) {
	// evaluate the template for the LogConfig as well as the tags
	for i, logConfig := range service.LogConfigs {
		// Path
		result := service.evaluateTemplate(gs, logConfig.Path)
		if result != "" {
			service.LogConfigs[i].Path = result
		}
		// Type
		result = service.evaluateTemplate(gs, logConfig.Type)
		if result != "" {
			service.LogConfigs[i].Type = result
		}

		// Tags
		for j, tag := range logConfig.LogTags {
			result = service.evaluateTemplate(gs, tag.Value)
			if result != "" {
				service.LogConfigs[i].LogTags[j].Value = result
			}
		}
	}
	return
}

// EvaluateConfigFilesTemplate parses and evals the Filename and Content. This happens for each
// ConfigFile on the service.
func (service *Service) EvaluateConfigFilesTemplate(gs GetService) (err error) {
	glog.V(3).Infof("Evaluating Config Files for %s", service.Id)
	for key, configFile := range service.ConfigFiles {
		glog.V(3).Infof("Evaluating Config File: %v", key)
		// Filename
		result := service.evaluateTemplate(gs, configFile.Filename)
		if result != "" {
			configFile.Filename = result
		}
		// Content
		result = service.evaluateTemplate(gs, configFile.Content)
		if result != "" {
			configFile.Content = result
		}
		service.ConfigFiles[key] = configFile
	}
	return
}

func percentScale(x uint64, percentage float64) uint64 {
	return uint64(math.Floor(float64(x) * percentage))
}

func bytesToMB(x uint64) uint64 {
	return x / (1024 * 1024)
}

// EvaluateEndpointTemplates parses and evaluates the "ApplicationTemplate" property
// of each of the service endpoints for this service.
func (service *Service) EvaluateEndpointTemplates(gs GetService) (err error) {
	functions := template.FuncMap{
		"parent":       parent(gs),
		"context":      context(),
		"percentScale": percentScale,
		"bytesToMB":    bytesToMB,
	}

	for i, ep := range service.Endpoints {
		if ep.Application != "" && ep.ApplicationTemplate == "" {
			ep.ApplicationTemplate = ep.Application
			service.Endpoints[i].ApplicationTemplate = ep.Application
		}
		if ep.ApplicationTemplate != "" {
			t := template.Must(template.New(service.Name).Funcs(functions).Parse(ep.ApplicationTemplate))
			var buffer bytes.Buffer
			if err = t.Execute(&buffer, service); err == nil {
				service.Endpoints[i].Application = buffer.String()
			} else {
				return
			}
		}
	}
	return
}

// Fill in all the templates in the ServiceDefinitions
func (service *Service) Evaluate(getSvc GetService) error {
	if err := service.EvaluateEndpointTemplates(getSvc); err != nil {
		return err
	}
	if err := service.EvaluateLogConfigTemplate(getSvc); err != nil {
		return err
	}
	if err := service.EvaluateConfigFilesTemplate(getSvc); err != nil {
		return err
	}
	if err := service.EvaluateStartupTemplate(getSvc); err != nil {
		return err
	}
	if err := service.EvaluateRunsTemplate(getSvc); err != nil {
		return err
	}
	if err := service.EvaluateActionsTemplate(getSvc); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package queue

import (
	"fmt"

	"gopkg.in/redis.v2"

	"github.com/dolaterio/dolaterio/core"
)

type redisQueue struct {
	client   *redis.Client
	queueKey string
}

// NewRedisQueue returns a redis-backed queue
func NewRedisQueue() (Queue, error) {
	client := redis.NewClient(&redis.Options{
		Network: "tcp",
		Addr: fmt.Sprintf(
			"%v:%v", core.Config.RedisIP, core.Config.RedisPort),
	})

	cmd := client.Ping()
	if cmd.Err() != nil {
		return nil, cmd.Err()
	}

	return &redisQueue{
		client:   client,
		queueKey: "jobs",
	}, nil
}

func (q *redisQueue) Empty() error {
	cmd := q.client.Del(q.queueKey)
	if cmd.Err() != nil {
		return cmd.Err()
	}
	return nil
}

func (q *redisQueue) Enqueue(message *Message) error {
	cmd := q.client.RPush(q.queueKey, message.JobID)
	if cmd.Err() != nil {
		return cmd.Err()
	}
	return nil
}

func (q *redisQueue) Dequeue() (*Message, error) {
	cmd := q.client.BLPop(0, q.queueKey)
	if cmd.Err() != nil {
		return nil, cmd.Err()
	}

	return &Message{
		JobID: cmd.Val()[1],
	}, nil
}

func (q *redisQueue) Close() error {
	return q.client.Close()
}
<commit_msg>Added logs to redis<commit_after>package queue

import (
	"fmt"

	"gopkg.in/redis.v2"

	"github.com/Sirupsen/logrus"
	"github.com/dolaterio/dolaterio/core"
)

type redisQueue struct {
	client   *redis.Client
	queueKey string
}

var (
	log = logrus.WithFields(logrus.Fields{
		"package": "queue",
	})
)

// NewRedisQueue returns a redis-backed queue
func NewRedisQueue() (Queue, error) {
	client := redis.NewClient(&redis.Options{
		Network: "tcp",
		Addr: fmt.Sprintf(
			"%v:%v", core.Config.RedisIP, core.Config.RedisPort),
	})

	cmd := client.Ping()
	if cmd.Err() != nil {
		return nil, cmd.Err()
	}

	return &redisQueue{
		client:   client,
		queueKey: "jobs",
	}, nil
}

func (q *redisQueue) Empty() error {
	cmd := q.client.Del(q.queueKey)
	if cmd.Err() != nil {
		return cmd.Err()
	}
	return nil
}

func (q *redisQueue) Enqueue(message *Message) error {
	log.WithField("jobID", message.JobID).Info("Queuing message")
	cmd := q.client.RPush(q.queueKey, message.JobID)
	if cmd.Err() != nil {
		log.WithField("jobID", message.JobID).
			WithField("err", cmd.Err()).
			Error("Error queuing message")
		return cmd.Err()
	}
	return nil
}

func (q *redisQueue) Dequeue() (*Message, error) {
	log.Info("Dequeuing message")
	cmd := q.client.BLPop(0, q.queueKey)
	if cmd.Err() != nil {
		log.WithField("err", cmd.Err()).
			Error("Error dequeuing message")
		return nil, cmd.Err()
	}

	return &Message{
		JobID: cmd.Val()[1],
	}, nil
}

func (q *redisQueue) Close() error {
	return q.client.Close()
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/libopenstorage/systemutils"
)

// Strings for VolumeSpec
const (
	SpecEphemeral        = "ephemeral"
	SpecSize             = "size"
	SpecFilesystem       = "fs"
	SpecBlockSize        = "bs"
	SpecHaLevel          = "ha"
	SpecCos              = "cos"
	SpecSnapshotInterval = "snap"
	SpecDedupe           = "dedupe"
)

// OptionKey specifies a set of recognized query params
const (
	// OptName query parameter used to lookup volume by name
	OptName = "Name"
	// OptVolumeID query parameter used to lookup volume by ID.
	OptVolumeID = "VolumeID"
	// OptLabel query parameter used to lookup volume by set of labels.
	OptLabel = "Label"
	// OptConfigLabel query parameter used to lookup volume by set of labels.
	OptConfigLabel = "ConfigLabel"
)

// Node describes the state of a node.
// It includes the current physical state (CPU, memory, storage, network usage) as
// well as the containers running on the system.
type Node struct {
	Id        string
	Cpu       float64 // percentage.
	MemTotal  uint64
	MemUsed   uint64
	MemFree   uint64
	Avgload   int
	Status    Status
	GenNumber uint64
	Luns      map[string]systemutils.Lun
	Disks     map[string]StorageResource
	MgmtIp    string
	DataIp    string
	Timestamp time.Time
	StartTime time.Time
	Hostname  string
	NodeData  map[string]interface{}
	// User defined labels for node. Key Value pairs
	NodeLabels map[string]string
}

// Cluster represents the state of the cluster.
type Cluster struct {
	Status Status

	// Id is the ID of the cluster.
	Id string

	// Nodes is an array of all the nodes in the cluster.
	Nodes []Node
}

func StatusSimpleValueOf(s string) (Status, error) {
	obj, err := simpleValueOf("status", Status_value, s)
	return Status(obj), err
}

func (x Status) SimpleString() string {
	return simpleString("status", Status_name, int32(x))
}

func DriverTypeSimpleValueOf(s string) (DriverType, error) {
	obj, err := simpleValueOf("driver_type", DriverType_value, s)
	return DriverType(obj), err
}

func (x DriverType) SimpleString() string {
	return simpleString("driver_type", DriverType_name, int32(x))
}

func FSTypeSimpleValueOf(s string) (FSType, error) {
	obj, err := simpleValueOf("fs_type", FSType_value, s)
	return FSType(obj), err
}

func (x FSType) SimpleString() string {
	return simpleString("fs_type", FSType_name, int32(x))
}

func GraphDriverChangeTypeSimpleValueOf(s string) (GraphDriverChangeType, error) {
	obj, err := simpleValueOf("graph_driver_change_type", GraphDriverChangeType_value, s)
	return GraphDriverChangeType(obj), err
}

func (x GraphDriverChangeType) SimpleString() string {
	return simpleString("graph_driver_change_type", GraphDriverChangeType_name, int32(x))
}

func VolumeActionParamSimpleValueOf(s string) (VolumeActionParam, error) {
	obj, err := simpleValueOf("volume_action_param", VolumeActionParam_value, s)
	return VolumeActionParam(obj), err
}

func (x VolumeActionParam) SimpleString() string {
	return simpleString("volume_action_param", VolumeActionParam_name, int32(x))
}

func VolumeStateSimpleValueOf(s string) (VolumeState, error) {
	obj, err := simpleValueOf("volume_state", VolumeState_value, s)
	return VolumeState(obj), err
}

func (x VolumeState) SimpleString() string {
	return simpleString("volume_state", VolumeState_name, int32(x))
}

func VolumeStatusSimpleValueOf(s string) (VolumeStatus, error) {
	obj, err := simpleValueOf("volume_status", VolumeStatus_value, s)
	return VolumeStatus(obj), err
}

func (x VolumeStatus) SimpleString() string {
	return simpleString("volume_status", VolumeStatus_name, int32(x))
}

func simpleValueOf(typeString string, valueMap map[string]int32, s string) (int32, error) {
	obj, ok := valueMap[strings.ToUpper(fmt.Sprintf("%s_%s", typeString, s))]
	if !ok {
		return 0, fmt.Errorf("no openstorage.%s for %s", strings.ToUpper(typeString), s)
	}
	return obj, nil
}

func simpleString(typeString string, nameMap map[int32]string, v int32) string {
	s, ok := nameMap[v]
	if !ok {
		return strconv.Itoa(int(v))
	}
	return strings.TrimPrefix(strings.ToLower(s), fmt.Sprintf("%s_", strings.ToLower(typeString)))
}
<commit_msg>Introduce StatPoint object in OSD<commit_after>package api

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/libopenstorage/systemutils"
)

// Strings for VolumeSpec
const (
	SpecEphemeral        = "ephemeral"
	SpecSize             = "size"
	SpecFilesystem       = "fs"
	SpecBlockSize        = "bs"
	SpecHaLevel          = "ha"
	SpecCos              = "cos"
	SpecSnapshotInterval = "snap"
	SpecDedupe           = "dedupe"
)

// OptionKey specifies a set of recognized query params
const (
	// OptName query parameter used to lookup volume by name
	OptName = "Name"
	// OptVolumeID query parameter used to lookup volume by ID.
	OptVolumeID = "VolumeID"
	// OptLabel query parameter used to lookup volume by set of labels.
	OptLabel = "Label"
	// OptConfigLabel query parameter used to lookup volume by set of labels.
	OptConfigLabel = "ConfigLabel"
)

// Node describes the state of a node.
// It includes the current physical state (CPU, memory, storage, network usage) as
// well as the containers running on the system.
type Node struct {
	Id        string
	Cpu       float64 // percentage.
	MemTotal  uint64
	MemUsed   uint64
	MemFree   uint64
	Avgload   int
	Status    Status
	GenNumber uint64
	Luns      map[string]systemutils.Lun
	Disks     map[string]StorageResource
	MgmtIp    string
	DataIp    string
	Timestamp time.Time
	StartTime time.Time
	Hostname  string
	NodeData  map[string]interface{}
	// User defined labels for node. Key Value pairs
	NodeLabels map[string]string
}

// Cluster represents the state of the cluster.
type Cluster struct {
	Status Status

	// Id is the ID of the cluster.
	Id string

	// Nodes is an array of all the nodes in the cluster.
	Nodes []Node
}

// StatPoint represents the basic structure of a single Stat reported
// TODO: This is the first step to introduce stats in openstorage.
//       Follow up task is to introduce an API for logging stats
type StatPoint struct {
	// Name of the Stat
	Name      string
	// Tags for the Stat
	Tags      map[string]string
	// Fields and values of the stat
	Fields    map[string]interface{}
	// Unix Timestamp
	Timestamp int64
}

func StatusSimpleValueOf(s string) (Status, error) {
	obj, err := simpleValueOf("status", Status_value, s)
	return Status(obj), err
}

func (x Status) SimpleString() string {
	return simpleString("status", Status_name, int32(x))
}

func DriverTypeSimpleValueOf(s string) (DriverType, error) {
	obj, err := simpleValueOf("driver_type", DriverType_value, s)
	return DriverType(obj), err
}

func (x DriverType) SimpleString() string {
	return simpleString("driver_type", DriverType_name, int32(x))
}

func FSTypeSimpleValueOf(s string) (FSType, error) {
	obj, err := simpleValueOf("fs_type", FSType_value, s)
	return FSType(obj), err
}

func (x FSType) SimpleString() string {
	return simpleString("fs_type", FSType_name, int32(x))
}

func GraphDriverChangeTypeSimpleValueOf(s string) (GraphDriverChangeType, error) {
	obj, err := simpleValueOf("graph_driver_change_type", GraphDriverChangeType_value, s)
	return GraphDriverChangeType(obj), err
}

func (x GraphDriverChangeType) SimpleString() string {
	return simpleString("graph_driver_change_type", GraphDriverChangeType_name, int32(x))
}

func VolumeActionParamSimpleValueOf(s string) (VolumeActionParam, error) {
	obj, err := simpleValueOf("volume_action_param", VolumeActionParam_value, s)
	return VolumeActionParam(obj), err
}

func (x VolumeActionParam) SimpleString() string {
	return simpleString("volume_action_param", VolumeActionParam_name, int32(x))
}

func VolumeStateSimpleValueOf(s string) (VolumeState, error) {
	obj, err := simpleValueOf("volume_state", VolumeState_value, s)
	return VolumeState(obj), err
}

func (x VolumeState) SimpleString() string {
	return simpleString("volume_state", VolumeState_name, int32(x))
}

func VolumeStatusSimpleValueOf(s string) (VolumeStatus, error) {
	obj, err := simpleValueOf("volume_status", VolumeStatus_value, s)
	return VolumeStatus(obj), err
}

func (x VolumeStatus) SimpleString() string {
	return simpleString("volume_status", VolumeStatus_name, int32(x))
}

func simpleValueOf(typeString string, valueMap map[string]int32, s string) (int32, error) {
	obj, ok := valueMap[strings.ToUpper(fmt.Sprintf("%s_%s", typeString, s))]
	if !ok {
		return 0, fmt.Errorf("no openstorage.%s for %s", strings.ToUpper(typeString), s)
	}
	return obj, nil
}

func simpleString(typeString string, nameMap map[int32]string, v int32) string {
	s, ok := nameMap[v]
	if !ok {
		return strconv.Itoa(int(v))
	}
	return strings.TrimPrefix(strings.ToLower(s), fmt.Sprintf("%s_", strings.ToLower(typeString)))
}
<|endoftext|>


<commit_before>package filters

import (
	"errors"
	"reflect"
	"strconv"
	"strings"
)

func floatToString(f float64) string {
	return strconv.FormatFloat(f, 'f', 8, 64)
}

func intToString(i int) string {
	return strconv.Itoa(i)
}

func returnArrayString(valueBox interface{}) (string, error) {

	jsonType, err := checkTypesInArrayAreStringOrNumeric(valueBox)
	if err != nil {
		return "", err
	}
	values := valueBox.([]interface{})
	toArray := "["
	for _, s := range values {
		if jsonType == "string" {
			toArray += "\""
		}

		if jsonType == "string" {
			toArray += s.(string)
		} else if jsonType == "float64" {
			toArray += floatToString(s.(float64))
		} else if jsonType == "int" {
			toArray += intToString(s.(int))
		} else {
			return "", errors.New("Type is neither String or Numeric(int or float64)")
		}

		if jsonType == "string" {
			toArray += "\""
		}
		toArray += ","
	}
	toArray = strings.TrimRight(toArray, ",")
	toArray += "]"
	return toArray, nil
}

func checkTypesInArrayAreStringOrNumeric(valueBox interface{}) (string, error) {
	//Get the first type of the first value
	values := valueBox.([]interface{})
	firstValueType := reflect.TypeOf(values[0]).String()

	for _, value := range values {
		if firstValueType != reflect.TypeOf(value).String() {
			return "", errors.New("Types in array are inconsistent")
		}

	}

	if firstValueType == "string" || firstValueType == "int" || firstValueType == "float64" {
		return firstValueType, nil
	} else {
		return "", errors.New("Type is neither String or Numeric(int or float64)")
	}

}

func returnJsonString(keyword, operator string, value interface{}) (string, error) {

	switch v := value.(type) {
	case int:
		return "{" + "\"" + keyword + ":" + "{\"" + operator + "\":" + intToString(v) + "}}", nil
	case float64:
		return "{" + "\"" + keyword + ":" + "{\"" + operator + "\":" + floatToString(v) + "}}", nil
	case string:
		return "{" + "\"" + keyword + ":" + "{\"" + operator + "\":" + v + "\"}}", nil
	case []string:
		unboxedValue, err := returnArrayString(value)
		if err != nil {
			return "", err
		}
		return "{" + "\"" + keyword + ":" + "{\"" + operator + "\":" + unboxedValue + "\"}}", nil
	case []int:
		unboxedValue, err := returnArrayString(value)
		if err != nil {
			return "", err
		}
		return "{" + "\"" + keyword + ":" + "{\"" + operator + "\":" + unboxedValue + "\"}}", nil
	case []float64:
		unboxedValue, err := returnArrayString(value)
		if err != nil {
			return "", err
		}
		return "{" + "\"" + keyword + ":" + "{\"" + operator + "\":" + unboxedValue + "\"}}", nil
	default:
		return "", errors.New("Error: Accepts only Text or Numeric Types")

	}

}

func Blank(keyword string, b bool) string {
	if b == true {
		return keyword + ":" + "{\"$blank\":true}"
	} else {
		return keyword + ":" + "{\"$blank\":false}"
	}
}

func BeginsWith(keyword string, value string) (string, error) {
	jsonString, err := returnJsonString(keyword, "bw", value)
	if err != nil {
		return "", err
	}
	return jsonString, nil
}

func BeginsWithAny(keyword string, values ...string) (string, error) {

	jsonString, err := returnJsonString(keyword, "$bwin", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func EqualTo(keyword string, value interface{}) (string, error) {
	jsonString, err := returnJsonString(keyword, "$eq", value)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func Excludes(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$excludes", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func ExcludesAny(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$excludes_any", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func GreaterThan(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$gt", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func GreaterThanEqual(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$gte", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func EqualsAnyOf(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$in", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func Includes(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$includes", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func IncludesAny(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$includes", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func LessThan(keyword string, values interface{}) (string, error) {

	valueType := reflect.TypeOf(values).String()

	jsonString, err := returnJsonString(keyword, "$lt", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func LessThanEqual(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$lte", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func NotBeginWith(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$nbw", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func NotBeginWithAny(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$nbwin", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func NotEqualTo(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$nbwin", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func NotEqualAnyOf(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$nbwin", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func Search(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$search", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}
<commit_msg>added checkTypes and CheckTypesInArray.  Added more unit tests.  There is quite a few unit tests to still do.  Also added way to type check values at runtime<commit_after>/*
TODO: Need to consider making external functions more strongly typed.  Right now we infer the types at runtime, might be better
to change arguments to specifc types instead of interface{}.   This will catch certain errors at compile time instead of waiting for
runtime.

*/
package filters

import (
	"errors"
	"reflect"
	"strconv"
	"strings"
)

const (
	floatPrecision = 2
	floatBits      = 64
)

type jsType int

const (
	stringT           = iota // 1
	numericT                 // = iota
	stringAndNumericT        // = iota
)

func floatToString(f float64) string {
	return strconv.FormatFloat(f, 'f', floatPrecision, floatBits)
}

func intToString(i int) string {
	return strconv.Itoa(i)
}

func returnJsonArray(valueBox interface{}) (string, error) {

	var values []interface{}

	var jsonType string

	switch v := valueBox.(type) {
	case []string:

		values = make([]interface{}, len(v))
		jsonType = "string"
		for i, j := range v {

			values[i] = j
		}
	case []int:
		values = make([]interface{}, len(v))
		jsonType = "int"
		for i, j := range v {
			values[i] = j
		}
	case []float64:
		values = make([]interface{}, len(v))
		jsonType = "float64"
		for i, j := range v {
			values[i] = j
		}
	case []interface{}:
		var err error
		jsonType, err = checkTypesInArray(v, stringAndNumericT)
		if err != nil {
			return "", err
		}
		values = v

	default:
		return "", errors.New("valueBox is not []string,[]int, or []float64")
	}

	toArray := "["
	for _, s := range values {
		if jsonType == "string" {
			toArray += "\""
		}

		if jsonType == "string" {
			toArray += s.(string)
		} else if jsonType == "float64" {
			toArray += floatToString(s.(float64))
		} else if jsonType == "int" {
			toArray += intToString(s.(int))
		} else {
			return "", errors.New("Type is neither String or Numeric(int or float64)")
		}

		if jsonType == "string" {
			toArray += "\""
		}
		toArray += ","
	}
	toArray = strings.TrimRight(toArray, ",")
	toArray += "]"
	return toArray, nil
}

func verifyType(valueType string, acceptedTypes jsType) error {
	if acceptedTypes == stringT {

		if valueType == "string" {
			return nil
		} else {
			return errors.New("Error: Expected Type was a string, but instead got a " + valueType)
		}
	}

	if acceptedTypes == numericT {

		if valueType == "int" || valueType == "float64" {
			return nil
		} else {
			return errors.New("Error: Expected Type was a numeric type (float64 or int), but instead got a " + valueType)
		}
	}

	if acceptedTypes == stringAndNumericT {

		if valueType == "int" || valueType == "float64" || valueType == "string" {
			return nil
		} else {
			return errors.New("Error: Expected Type was a numeric type (float64 or int), or string, but instead got a " + valueType)
		}
	}

	return errors.New("Unknown valueType given or unknown jsType given")

}

func checkTypes(values interface{}, acceptedTypes jsType) (string, error) {
	//Get the first type of the first value
	valueType := reflect.TypeOf(values).String()
	err := verifyType(valueType, acceptedTypes)

	if err != nil {
		return "", err
	}

	return valueType, nil

}

func checkTypesInArray(values []interface{}, acceptedTypes jsType) (string, error) {
	//Get the first type of the first value
	valueType := reflect.TypeOf(values[0]).String()

	for _, value := range values {
		if valueType != reflect.TypeOf(value).String() {
			return "", errors.New("Types in array are inconsistent")
		}

	}

	err := verifyType(valueType, acceptedTypes)

	if err != nil {
		return "", err
	}

	return valueType, nil

}

func returnJsonString(keyword, operator string, value interface{}) (string, error) {

	switch v := value.(type) {
	case int:
		return "{" + "\"" + keyword + "\":" + "{\"" + operator + "\":" + intToString(v) + "}}", nil
	case float64:
		return "{" + "\"" + keyword + "\":" + "{\"" + operator + "\":" + floatToString(v) + "}}", nil
	case string:
		return "{" + "\"" + keyword + "\":" + "{\"" + operator + "\":\"" + v + "\"}}", nil
	case []string:

		unboxedValue, err := returnJsonArray(value)

		if err != nil {
			return "", err
		}
		return "{" + "\"" + keyword + "\":" + "{\"" + operator + "\":" + unboxedValue + "}}", nil
	case []int:
		unboxedValue, err := returnJsonArray(value)
		if err != nil {
			return "", err
		}
		return "{" + "\"" + keyword + "\":" + "{\"" + operator + "\":" + unboxedValue + "\"}}", nil
	case []float64:
		unboxedValue, err := returnJsonArray(value)
		if err != nil {
			return "", err
		}
		return "{" + "\"" + keyword + "\":" + "{\"" + operator + "\":" + unboxedValue + "\"}}", nil
	case []interface{}:
		unboxedValue, err := returnJsonArray(value)
		if err != nil {
			return "", err
		}
		return "{" + "\"" + keyword + "\":" + "{\"" + operator + "\":" + unboxedValue + "}}", nil

	default:
		return "", errors.New("Error: Accepts only Text or Numeric Types")

	}

}

func Blank(keyword string, b bool) string {
	if b == true {
		return "{\"" + keyword + "\":" + "{\"$blank\":true}}"
	} else {
		return "{\"" + keyword + "\":" + "{\"$blank\":false}}"
	}
}

func BeginsWith(keyword string, value string) (string, error) {
	jsonString, err := returnJsonString(keyword, "$bw", value)
	if err != nil {
		return "", err
	}
	return jsonString, nil
}

func BeginsWithAny(keyword string, values ...string) (string, error) {
	jsonString, err := returnJsonString(keyword, "$bwin", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func EqualTo(keyword string, value interface{}) (string, error) {
	jsonString, err := returnJsonString(keyword, "$eq", value)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func Excludes(keyword string, values interface{}) (string, error) {
	jsonString, err := returnJsonString(keyword, "$excludes", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func ExcludesAny(keyword string, values ...interface{}) (string, error) {
	jsonString, err := returnJsonString(keyword, "$excludes_any", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func GreaterThan(keyword string, values interface{}) (string, error) {
	jsonType, err := checkTypes(values, numericT)
	if err != nil {
		return "", err
	}
	if jsonType == "string" {
		return "", errors.New("Can only accept numeric types")
	}

	jsonString, err := returnJsonString(keyword, "$gt", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func GreaterThanEqual(keyword string, values interface{}) (string, error) {
	jsonString, err := returnJsonString(keyword, "$gte", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func EqualsAnyOf(keyword string, values interface{}) (string, error) {
	jsonString, err := returnJsonString(keyword, "$in", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func Includes(keyword string, values interface{}) (string, error) {
	jsonString, err := returnJsonString(keyword, "$includes", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func IncludesAny(keyword string, values interface{}) (string, error) {
	jsonString, err := returnJsonString(keyword, "$includes", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func LessThan(keyword string, values interface{}) (string, error) {
	jsonString, err := returnJsonString(keyword, "$lt", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func LessThanEqual(keyword string, values interface{}) (string, error) {
	jsonString, err := returnJsonString(keyword, "$lte", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func NotBeginWith(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$nbw", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func NotBeginWithAny(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$nbwin", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func NotEqualTo(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$nbwin", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func NotEqualAnyOf(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$nbwin", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}

func Search(keyword string, values interface{}) (string, error) {

	jsonString, err := returnJsonString(keyword, "$search", values)
	if err != nil {
		return "", err
	}
	return jsonString, nil

}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/mesos/mesos-go/executor"
	mesos "github.com/mesos/mesos-go/mesosproto"
	"github.com/Nitro/sidecar-executor/container"
	"github.com/relistan/go-director"
)

// Callbacks from the Mesos driver. These are required to implement
// the Executor interface.

func (exec *sidecarExecutor) Registered(driver executor.ExecutorDriver,
	execInfo *mesos.ExecutorInfo, fwinfo *mesos.FrameworkInfo, slaveInfo *mesos.SlaveInfo) {
	log.Info("Registered Executor on slave ", slaveInfo.GetHostname())
}

func (exec *sidecarExecutor) Reregistered(driver executor.ExecutorDriver, slaveInfo *mesos.SlaveInfo) {
	log.Info("Re-registered Executor on slave ", slaveInfo.GetHostname())
}

func (exec *sidecarExecutor) Disconnected(driver executor.ExecutorDriver) {
	log.Info("Executor disconnected.")
}

// Copy the Docker container logs to stdout and stderr so we can capture some
// failure information in the Mesos logs. Then tooling can fetch crash info
// from the Mesos API.
func (exec *sidecarExecutor) copyLogs(taskId string) {
	startTimeEpoch := time.Now().UTC().Add(0 - config.LogsSince).Unix()

	container.GetLogs(
		exec.client, taskId, startTimeEpoch, os.Stdout, os.Stderr,
	)
}

// monitorTask runs in a goroutine and hangs out, waiting for the watchLooper to
// complete. When it completes, it handles the Docker and Mesos interactions.
func (exec *sidecarExecutor) monitorTask(cntnrId string, taskInfo *mesos.TaskInfo) {
	log.Infof("Monitoring container %s for Mesos task %s",
		cntnrId,
		*taskInfo.TaskId.Value,
	)

	// Wait on the watchLooper to return a status
	err := exec.watchLooper.Wait()
	if err != nil {
		log.Errorf("Error! %s", err.Error())
		// Something went wrong, we better take this thing out!
		err := container.StopContainer(exec.client, *taskInfo.TaskId.Value, config.KillTaskTimeout)
		if err != nil {
			log.Errorf("Error stopping container %s! %s", *taskInfo.TaskId.Value, err.Error())
		}
		// Copy the failure logs (hopefully) to stdout/stderr so we can get them
		exec.copyLogs(*taskInfo.TaskId.Value)
		// Notify Mesos
		exec.failTask(taskInfo)
		return
	}

	exec.finishTask(taskInfo)
	log.Info("Task completed: ", taskInfo.GetName())
	return
}

// Callback from Mesos driver to launch a new task in this executor
func (exec *sidecarExecutor) LaunchTask(driver executor.ExecutorDriver, taskInfo *mesos.TaskInfo) {
	log.Infof("Launching task %s with command '%s'", taskInfo.GetName(), taskInfo.Command.GetValue())
	log.Info("Task ID ", taskInfo.GetTaskId().GetValue())

	logTaskEnv(taskInfo)

	// We need to tell the scheduler that we started the task
	exec.sendStatus(TaskRunning, taskInfo.GetTaskId())

	log.Infof("Using image '%s'", *taskInfo.Container.Docker.Image)

	// TODO implement configurable pull timeout?
	if !container.CheckImage(exec.client, taskInfo) || *taskInfo.Container.Docker.ForcePullImage {
		err := container.PullImage(exec.client, taskInfo, exec.dockerAuth)
		if err != nil {
			log.Errorf("Failed to pull image: %s", err.Error())
			exec.failTask(taskInfo)
			return
		}
	} else {
		log.Info("Re-using existing image... already present")
	}

	// Configure and create the container
	containerConfig := container.ConfigForTask(taskInfo)
	cntnr, err := exec.client.CreateContainer(*containerConfig)
	if err != nil {
		log.Error("Failed to create Docker container: %s", err.Error())
		exec.failTask(taskInfo)
		return
	}

	// Start the container
	log.Info("Starting container with ID " + cntnr.ID[:12])
	err = exec.client.StartContainer(cntnr.ID, containerConfig.HostConfig)
	if err != nil {
		log.Error("Failed to create Docker container: %s", err.Error())
		exec.failTask(taskInfo)
		return
	}

	// For debugging, set process title to contain container ID & image
	SetProcessName("sidecar-executor " + cntnr.ID[:12] + " (" + *taskInfo.Container.Docker.Image + ")")

	exec.watchLooper =
		director.NewImmediateTimedLooper(director.FOREVER, config.SidecarPollInterval, make(chan error))

	// We have to do this in a different goroutine or the scheduler
	// can't send us any further updates.
	go exec.watchContainer(cntnr.ID)
	go exec.monitorTask(cntnr.ID[:12], taskInfo)
}

func (exec *sidecarExecutor) KillTask(driver executor.ExecutorDriver, taskID *mesos.TaskID) {
	log.Infof("Killing task: %s", *taskID.Value)

	// Stop watching the container so we don't send the wrong task status
	go func() { exec.watchLooper.Quit() }()

	err := container.StopContainer(exec.client, *taskID.Value, config.KillTaskTimeout)
	if err != nil {
		log.Errorf("Error stopping container %s! %s", *taskID.Value, err.Error())
	}

	// Have to force this to be an int64
	var status int64 = TaskKilled // Default status is that we shot it in the head

	// Now we need to sort out whether the task finished nicely or not.
	// This driver callback is used both to shoot a task in the head, and when
	// a task is being replaced. The Mesos task status needs to reflect the
	// resulting container State.ExitCode.
	container, err := exec.client.InspectContainer(*taskID.Value)
	if err == nil {
		if container.State.ExitCode == 0 {
			status = TaskFinished // We exited cleanly when asked
		}
	} else {
		log.Errorf("Error inspecting container %s! %s", *taskID.Value, err.Error())
	}

	// Copy the failure logs (hopefully) to stdout/stderr so we can get them
	exec.copyLogs(*taskID.Value)
	// Notify Mesos
	exec.sendStatus(status, taskID)

	// We have to give the driver time to send the message
	time.Sleep(1 * time.Second)
	exec.driver.Stop()
}

func (exec *sidecarExecutor) FrameworkMessage(driver executor.ExecutorDriver, msg string) {
	log.Info("Got framework message: ", msg)
}

func (exec *sidecarExecutor) Shutdown(driver executor.ExecutorDriver) {
	log.Info("Shutting down the executor")
}

func (exec *sidecarExecutor) Error(driver executor.ExecutorDriver, err string) {
	log.Info("Got error message:", err)
}
<commit_msg>These should call Errorf()<commit_after>package main

import (
	"os"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/mesos/mesos-go/executor"
	mesos "github.com/mesos/mesos-go/mesosproto"
	"github.com/Nitro/sidecar-executor/container"
	"github.com/relistan/go-director"
)

// Callbacks from the Mesos driver. These are required to implement
// the Executor interface.

func (exec *sidecarExecutor) Registered(driver executor.ExecutorDriver,
	execInfo *mesos.ExecutorInfo, fwinfo *mesos.FrameworkInfo, slaveInfo *mesos.SlaveInfo) {
	log.Info("Registered Executor on slave ", slaveInfo.GetHostname())
}

func (exec *sidecarExecutor) Reregistered(driver executor.ExecutorDriver, slaveInfo *mesos.SlaveInfo) {
	log.Info("Re-registered Executor on slave ", slaveInfo.GetHostname())
}

func (exec *sidecarExecutor) Disconnected(driver executor.ExecutorDriver) {
	log.Info("Executor disconnected.")
}

// Copy the Docker container logs to stdout and stderr so we can capture some
// failure information in the Mesos logs. Then tooling can fetch crash info
// from the Mesos API.
func (exec *sidecarExecutor) copyLogs(taskId string) {
	startTimeEpoch := time.Now().UTC().Add(0 - config.LogsSince).Unix()

	container.GetLogs(
		exec.client, taskId, startTimeEpoch, os.Stdout, os.Stderr,
	)
}

// monitorTask runs in a goroutine and hangs out, waiting for the watchLooper to
// complete. When it completes, it handles the Docker and Mesos interactions.
func (exec *sidecarExecutor) monitorTask(cntnrId string, taskInfo *mesos.TaskInfo) {
	log.Infof("Monitoring container %s for Mesos task %s",
		cntnrId,
		*taskInfo.TaskId.Value,
	)

	// Wait on the watchLooper to return a status
	err := exec.watchLooper.Wait()
	if err != nil {
		log.Errorf("Error! %s", err.Error())
		// Something went wrong, we better take this thing out!
		err := container.StopContainer(exec.client, *taskInfo.TaskId.Value, config.KillTaskTimeout)
		if err != nil {
			log.Errorf("Error stopping container %s! %s", *taskInfo.TaskId.Value, err.Error())
		}
		// Copy the failure logs (hopefully) to stdout/stderr so we can get them
		exec.copyLogs(*taskInfo.TaskId.Value)
		// Notify Mesos
		exec.failTask(taskInfo)
		return
	}

	exec.finishTask(taskInfo)
	log.Info("Task completed: ", taskInfo.GetName())
	return
}

// Callback from Mesos driver to launch a new task in this executor
func (exec *sidecarExecutor) LaunchTask(driver executor.ExecutorDriver, taskInfo *mesos.TaskInfo) {
	log.Infof("Launching task %s with command '%s'", taskInfo.GetName(), taskInfo.Command.GetValue())
	log.Info("Task ID ", taskInfo.GetTaskId().GetValue())

	logTaskEnv(taskInfo)

	// We need to tell the scheduler that we started the task
	exec.sendStatus(TaskRunning, taskInfo.GetTaskId())

	log.Infof("Using image '%s'", *taskInfo.Container.Docker.Image)

	// TODO implement configurable pull timeout?
	if !container.CheckImage(exec.client, taskInfo) || *taskInfo.Container.Docker.ForcePullImage {
		err := container.PullImage(exec.client, taskInfo, exec.dockerAuth)
		if err != nil {
			log.Errorf("Failed to pull image: %s", err.Error())
			exec.failTask(taskInfo)
			return
		}
	} else {
		log.Info("Re-using existing image... already present")
	}

	// Configure and create the container
	containerConfig := container.ConfigForTask(taskInfo)
	cntnr, err := exec.client.CreateContainer(*containerConfig)
	if err != nil {
		log.Errorf("Failed to create Docker container: %s", err.Error())
		exec.failTask(taskInfo)
		return
	}

	// Start the container
	log.Info("Starting container with ID " + cntnr.ID[:12])
	err = exec.client.StartContainer(cntnr.ID, containerConfig.HostConfig)
	if err != nil {
		log.Errorf("Failed to create Docker container: %s", err.Error())
		exec.failTask(taskInfo)
		return
	}

	// For debugging, set process title to contain container ID & image
	SetProcessName("sidecar-executor " + cntnr.ID[:12] + " (" + *taskInfo.Container.Docker.Image + ")")

	exec.watchLooper =
		director.NewImmediateTimedLooper(director.FOREVER, config.SidecarPollInterval, make(chan error))

	// We have to do this in a different goroutine or the scheduler
	// can't send us any further updates.
	go exec.watchContainer(cntnr.ID)
	go exec.monitorTask(cntnr.ID[:12], taskInfo)
}

func (exec *sidecarExecutor) KillTask(driver executor.ExecutorDriver, taskID *mesos.TaskID) {
	log.Infof("Killing task: %s", *taskID.Value)

	// Stop watching the container so we don't send the wrong task status
	go func() { exec.watchLooper.Quit() }()

	err := container.StopContainer(exec.client, *taskID.Value, config.KillTaskTimeout)
	if err != nil {
		log.Errorf("Error stopping container %s! %s", *taskID.Value, err.Error())
	}

	// Have to force this to be an int64
	var status int64 = TaskKilled // Default status is that we shot it in the head

	// Now we need to sort out whether the task finished nicely or not.
	// This driver callback is used both to shoot a task in the head, and when
	// a task is being replaced. The Mesos task status needs to reflect the
	// resulting container State.ExitCode.
	container, err := exec.client.InspectContainer(*taskID.Value)
	if err == nil {
		if container.State.ExitCode == 0 {
			status = TaskFinished // We exited cleanly when asked
		}
	} else {
		log.Errorf("Error inspecting container %s! %s", *taskID.Value, err.Error())
	}

	// Copy the failure logs (hopefully) to stdout/stderr so we can get them
	exec.copyLogs(*taskID.Value)
	// Notify Mesos
	exec.sendStatus(status, taskID)

	// We have to give the driver time to send the message
	time.Sleep(1 * time.Second)
	exec.driver.Stop()
}

func (exec *sidecarExecutor) FrameworkMessage(driver executor.ExecutorDriver, msg string) {
	log.Info("Got framework message: ", msg)
}

func (exec *sidecarExecutor) Shutdown(driver executor.ExecutorDriver) {
	log.Info("Shutting down the executor")
}

func (exec *sidecarExecutor) Error(driver executor.ExecutorDriver, err string) {
	log.Info("Got error message:", err)
}
<|endoftext|>


<commit_before>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

package service

import (
	"github.com/zenoss/glog"
	"bytes"
	"encoding/json"
	"text/template"
)

func parent(gs GetService) func(s Service) (Service, error) {
	return func(svc Service) (Service, error) {
		return gs(svc.ParentServiceID)
	}
}
func context() func(s Service) (map[string]interface{}, error) {
	return func(s Service) (map[string]interface{}, error) {
		ctx := make(map[string]interface{})
		err := json.Unmarshal([]byte(s.Context), &ctx)
		if err != nil {
			glog.Errorf("Error unmarshal service context Id=%s: %s -> %s", s.Id, s.Context, err)
		}
		return ctx, err
	}
}

// EvaluateActionsTemplate parses and evaluates the Actions string of a service.
func (service *Service) EvaluateActionsTemplate(gs GetService) (err error) {
	for key, value := range service.Actions {
		result := service.evaluateTemplate(gs, value)
		if result != "" {
			service.Actions[key] = result
		}
	}
	return
}

// EvaluateStartupTemplate parses and evaluates the StartUp string of a service.
func (service *Service) EvaluateStartupTemplate(gs GetService) (err error) {

	result := service.evaluateTemplate(gs, service.Startup)
	if result != "" {
		service.Startup = result
	}

	return
}

// EvaluateRunsTemplate parses and evaluates the Runs string of a service.
func (service *Service) EvaluateRunsTemplate(gs GetService) (err error) {
	for key, value := range service.Runs {
		result := service.evaluateTemplate(gs, value)
		if result != "" {
			service.Runs[key] = result
		}
	}
	return
}

// evaluateTemplate takes a control plane client and template string and evaluates
// the template using the service as the context. If the template is invalid or there is an error
// then an empty string is returned.
func (service *Service) evaluateTemplate(gs GetService, serviceTemplate string) string {
	functions := template.FuncMap{
		"parent":  parent(gs),
		"context": context(),
	}

	glog.V(3).Infof("Evaluating template string %v", serviceTemplate)
	// parse the template
	t := template.Must(template.New("ServiceDefinitionTemplate").Funcs(functions).Parse(serviceTemplate))

	// evaluate it
	var buffer bytes.Buffer
	err := t.Execute(&buffer, service)
	if err == nil {
		return buffer.String()
	}

	// something went wrong, warn them
	glog.Warningf("Evaluating template %v produced the following error %v ", serviceTemplate, err)
	return ""
}

// EvaluateLogConfigTemplate parses and evals the Path, Type and all the values for the tags of the log
// configs. This happens for each LogConfig on the service.
func (service *Service) EvaluateLogConfigTemplate(gs GetService) (err error) {
	// evaluate the template for the LogConfig as well as the tags
	for i, logConfig := range service.LogConfigs {
		// Path
		result := service.evaluateTemplate(gs, logConfig.Path)
		if result != "" {
			service.LogConfigs[i].Path = result
		}
		// Type
		result = service.evaluateTemplate(gs, logConfig.Type)
		if result != "" {
			service.LogConfigs[i].Type = result
		}

		// Tags
		for j, tag := range logConfig.LogTags {
			result = service.evaluateTemplate(gs, tag.Value)
			if result != "" {
				service.LogConfigs[i].LogTags[j].Value = result
			}
		}
	}
	return
}

// EvaluateConfigFilesTemplate parses and evals the Filename and Content. This happens for each
// ConfigFile on the service.
func (service *Service) EvaluateConfigFilesTemplate(gs GetService) (err error) {
	glog.V(3).Infof("Evaluating Config Files for %s", service.Id)
	for key, configFile := range service.ConfigFiles {
		glog.V(3).Infof("Evaluating Config File: %v", key)
		// Filename
		result := service.evaluateTemplate(gs, configFile.Filename)
		if result != "" {
			configFile.Filename = result
		}
		// Content
		result = service.evaluateTemplate(gs, configFile.Content)
		if result != "" {
			configFile.Content = result
		}
		service.ConfigFiles[key] = configFile
	}
	return
}

// EvaluateEndpointTemplates parses and evaluates the "ApplicationTemplate" property
// of each of the service endpoints for this service.
func (service *Service) EvaluateEndpointTemplates(gs GetService) (err error) {
	functions := template.FuncMap{
		"parent":  parent(gs),
		"context": context(),
	}

	for i, ep := range service.Endpoints {
		if ep.Application != "" && ep.ApplicationTemplate == "" {
			ep.ApplicationTemplate = ep.Application
			service.Endpoints[i].ApplicationTemplate = ep.Application
		}
		if ep.ApplicationTemplate != "" {
			t := template.Must(template.New(service.Name).Funcs(functions).Parse(ep.ApplicationTemplate))
			var buffer bytes.Buffer
			if err = t.Execute(&buffer, service); err == nil {
				service.Endpoints[i].Application = buffer.String()
			} else {
				return
			}
		}
	}
	return
}
<commit_msg>Add a scale function for templates to use<commit_after>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

package service

import (
	"github.com/zenoss/glog"
	"math"
	"bytes"
	"encoding/json"
	"text/template"
)

func parent(gs GetService) func(s Service) (Service, error) {
	return func(svc Service) (Service, error) {
		return gs(svc.ParentServiceID)
	}
}
func context() func(s Service) (map[string]interface{}, error) {
	return func(s Service) (map[string]interface{}, error) {
		ctx := make(map[string]interface{})
		err := json.Unmarshal([]byte(s.Context), &ctx)
		if err != nil {
			glog.Errorf("Error unmarshal service context Id=%s: %s -> %s", s.Id, s.Context, err)
		}
		return ctx, err
	}
}

// EvaluateActionsTemplate parses and evaluates the Actions string of a service.
func (service *Service) EvaluateActionsTemplate(gs GetService) (err error) {
	for key, value := range service.Actions {
		result := service.evaluateTemplate(gs, value)
		if result != "" {
			service.Actions[key] = result
		}
	}
	return
}

// EvaluateStartupTemplate parses and evaluates the StartUp string of a service.
func (service *Service) EvaluateStartupTemplate(gs GetService) (err error) {

	result := service.evaluateTemplate(gs, service.Startup)
	if result != "" {
		service.Startup = result
	}

	return
}

// EvaluateRunsTemplate parses and evaluates the Runs string of a service.
func (service *Service) EvaluateRunsTemplate(gs GetService) (err error) {
	for key, value := range service.Runs {
		result := service.evaluateTemplate(gs, value)
		if result != "" {
			service.Runs[key] = result
		}
	}
	return
}

// evaluateTemplate takes a control plane client and template string and evaluates
// the template using the service as the context. If the template is invalid or there is an error
// then an empty string is returned.
func (service *Service) evaluateTemplate(gs GetService, serviceTemplate string) string {
	functions := template.FuncMap{
		"parent":  parent(gs),
		"context": context(),
	}

	glog.V(3).Infof("Evaluating template string %v", serviceTemplate)
	// parse the template
	t := template.Must(template.New("ServiceDefinitionTemplate").Funcs(functions).Parse(serviceTemplate))

	// evaluate it
	var buffer bytes.Buffer
	err := t.Execute(&buffer, service)
	if err == nil {
		return buffer.String()
	}

	// something went wrong, warn them
	glog.Warningf("Evaluating template %v produced the following error %v ", serviceTemplate, err)
	return ""
}

// EvaluateLogConfigTemplate parses and evals the Path, Type and all the values for the tags of the log
// configs. This happens for each LogConfig on the service.
func (service *Service) EvaluateLogConfigTemplate(gs GetService) (err error) {
	// evaluate the template for the LogConfig as well as the tags
	for i, logConfig := range service.LogConfigs {
		// Path
		result := service.evaluateTemplate(gs, logConfig.Path)
		if result != "" {
			service.LogConfigs[i].Path = result
		}
		// Type
		result = service.evaluateTemplate(gs, logConfig.Type)
		if result != "" {
			service.LogConfigs[i].Type = result
		}

		// Tags
		for j, tag := range logConfig.LogTags {
			result = service.evaluateTemplate(gs, tag.Value)
			if result != "" {
				service.LogConfigs[i].LogTags[j].Value = result
			}
		}
	}
	return
}

// EvaluateConfigFilesTemplate parses and evals the Filename and Content. This happens for each
// ConfigFile on the service.
func (service *Service) EvaluateConfigFilesTemplate(gs GetService) (err error) {
	glog.V(3).Infof("Evaluating Config Files for %s", service.Id)
	for key, configFile := range service.ConfigFiles {
		glog.V(3).Infof("Evaluating Config File: %v", key)
		// Filename
		result := service.evaluateTemplate(gs, configFile.Filename)
		if result != "" {
			configFile.Filename = result
		}
		// Content
		result = service.evaluateTemplate(gs, configFile.Content)
		if result != "" {
			configFile.Content = result
		}
		service.ConfigFiles[key] = configFile
	}
	return
}

func percentScale(x uint64, percentage float64) uint64 {
	return uint64(math.Floor(float64(x) * percentage))
}

// EvaluateEndpointTemplates parses and evaluates the "ApplicationTemplate" property
// of each of the service endpoints for this service.
func (service *Service) EvaluateEndpointTemplates(gs GetService) (err error) {
	functions := template.FuncMap{
		"parent":  parent(gs),
		"context": context(),
		"percentScale": percentScale,
	}

	for i, ep := range service.Endpoints {
		if ep.Application != "" && ep.ApplicationTemplate == "" {
			ep.ApplicationTemplate = ep.Application
			service.Endpoints[i].ApplicationTemplate = ep.Application
		}
		if ep.ApplicationTemplate != "" {
			t := template.Must(template.New(service.Name).Funcs(functions).Parse(ep.ApplicationTemplate))
			var buffer bytes.Buffer
			if err = t.Execute(&buffer, service); err == nil {
				service.Endpoints[i].Application = buffer.String()
			} else {
				return
			}
		}
	}
	return
}
<|endoftext|>