<commit_before>//  Copyright (c) 2012 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package db

import (
	"encoding/json"
	"math"

	"github.com/couchbaselabs/go-couchbase"

	"github.com/couchbaselabs/sync_gateway/base"
	"github.com/couchbaselabs/sync_gateway/channels"
)

// Options for Database.getChanges
type ChangesOptions struct {
	Since          uint64
	Limit          int
	Conflicts      bool
	IncludeDocs    bool
	includeDocMeta bool
	Wait           bool
}

// A changes entry; Database.getChanges returns an array of these.
// Marshals into the standard CouchDB _changes format.
type ChangeEntry struct {
	Seq     uint64       `json:"seq"`
	ID      string       `json:"id"`
	Deleted bool         `json:"deleted,omitempty"`
	Removed channels.Set `json:"removed,omitempty"`
	Doc     Body         `json:"doc,omitempty"`
	Changes []ChangeRev  `json:"changes"`
	docMeta *document
}

type ChangeRev map[string]string

type ViewDoc struct {
	Json json.RawMessage // should be type 'document', but that fails to unmarshal correctly
}

type ViewRow struct {
	ID    string
	Key   interface{}
	Value interface{}
	Doc   *ViewDoc
}

type ViewResult struct {
	TotalRows int `json:"total_rows"`
	Rows      []ViewRow
	Errors    []couchbase.ViewError
}

const kChangesPageSize = 200

// Returns a list of all the changes made on a channel. Does NOT check authorization.
func (db *Database) ChangesFeed(channel string, options ChangesOptions) (<-chan *ChangeEntry, error) {
	lastSequence := options.Since
	endkey := []interface{}{channel, map[string]interface{}{}}
	totalLimit := options.Limit
	usingDocs := options.Conflicts || options.IncludeDocs || options.includeDocMeta
	opts := Body{"stale": false, "update_seq": true,
		"endkey":       endkey,
		"include_docs": usingDocs}

	feed := make(chan *ChangeEntry, kChangesPageSize)

	lastSeq, err := db.LastSequence()
	if err == nil && options.Since >= lastSeq && !options.Wait {
		close(feed)
		return feed, nil
	}

	// Generate the output in a new goroutine, writing to 'feed':
	go func() {
		defer close(feed)
		for {
			// Query the 'channels' view:
			opts["startkey"] = []interface{}{channel, lastSequence + 1}
			limit := totalLimit
			if limit == 0 || limit > kChangesPageSize {
				limit = kChangesPageSize
			}
			opts["limit"] = limit

			var vres ViewResult
			var err error
			for len(vres.Rows) == 0 {
				vres = ViewResult{}
				err = db.Bucket.ViewCustom("sync_gateway", "channels", opts, &vres)
				if err != nil {
					base.Log("Error from 'channels' view: %v", err)
					return
				}
				if len(vres.Rows) == 0 {
					if !options.Wait || !db.WaitForRevision() {
						return
					}
				}
			}

			for _, row := range vres.Rows {
				key := row.Key.([]interface{})
				lastSequence = uint64(key[1].(float64))
				value := row.Value.([]interface{})
				docID := value[0].(string)
				revID := value[1].(string)
				entry := &ChangeEntry{
					Seq:     lastSequence,
					ID:      docID,
					Changes: []ChangeRev{{"rev": revID}},
					Deleted: (len(value) >= 3 && value[2].(bool)),
				}
				if len(value) >= 3 && !value[2].(bool) {
					entry.Removed = channels.SetOf(channel)
				}
				if usingDocs {
					doc, err := unmarshalDocument(docID, row.Doc.Json)
					if err == nil {
						if options.Conflicts {
							for _, leafID := range doc.History.getLeaves() {
								if leafID != revID {
									entry.Changes = append(entry.Changes, ChangeRev{"rev": leafID})
								}
							}
						}
						if options.IncludeDocs {
							entry.Doc, _ = db.getRevFromDoc(doc, revID, false)
						}
						if options.includeDocMeta {
							entry.docMeta = doc
						}
					}
				}
				feed <- entry
			}

			// Step to the next page of results:
			nRows := len(vres.Rows)
			if nRows < kChangesPageSize || options.Wait {
				break
			}
			if totalLimit > 0 {
				totalLimit -= nRows
				if totalLimit <= 0 {
					break
				}
			}
			delete(opts, "stale") // we only need to update the index once
		}
	}()
	return feed, nil
}

// Returns of all the changes made to multiple channels. Does NOT check authorization.
func (db *Database) MultiChangesFeed(channels channels.Set, options ChangesOptions) (<-chan *ChangeEntry, error) {
	if len(channels) == 0 {
		return nil, nil
	} else if len(channels) == 1 {
		for channel, _ := range channels {
			return db.ChangesFeed(channel, options)
		}
	}

	waitMode := options.Wait
	options.Wait = false

	output := make(chan *ChangeEntry, kChangesPageSize)
	go func() {
		defer close(output)

		for {
			feeds := make([]<-chan *ChangeEntry, 0, len(channels))
			current := make([]*ChangeEntry, len(channels))
			i := 0
			for name, _ := range channels {
				var err error
				feeds[i], err = db.ChangesFeed(name, options)
				if err != nil {
					return
				}
				i++
			}

			wroteAnything := false
			for {
				//FIX: This assumes Reverse or Limit aren't set in the options
				// Read more entries to fill up the current[] array:
				for i, cur := range current {
					if cur == nil && feeds[i] != nil {
						var ok bool
						current[i], ok = <-feeds[i]
						if !ok {
							feeds[i] = nil
						}
					}
				}

				// Find the current entry with the minimum sequence:
				var minSeq uint64 = math.MaxUint64
				var minEntry *ChangeEntry
				for _, cur := range current {
					if cur != nil && cur.Seq < minSeq {
						minSeq = cur.Seq
						minEntry = cur
					}
				}
				if minEntry == nil {
					break
				}

				// Clear the current entries for the sequence just sent:
				for i, cur := range current {
					if cur != nil && cur.Seq == minEntry.Seq {
						current[i] = nil
						// Also concatenate the matching entries' Removed arrays:
						if cur != minEntry && cur.Removed != nil {
							if minEntry.Removed == nil {
								minEntry.Removed = cur.Removed
							} else {
								minEntry.Removed = minEntry.Removed.Union(cur.Removed)
							}
						}
					}
				}

				// Send the entry:
				output <- minEntry
				wroteAnything = true
			}

			// In wait mode, wait for the db to change, then run again
			if wroteAnything || !waitMode || !db.WaitForRevision() {
				break
			}
		}
	}()

	return output, nil
}

// Synchronous convenience function that returns all changes as a simple array.
// Does NOT check authorization.
func (db *Database) GetChanges(channels channels.Set, options ChangesOptions) ([]*ChangeEntry, error) {
	var changes = make([]*ChangeEntry, 0, 50)
	feed, err := db.MultiChangesFeed(channels, options)
	if err == nil && feed != nil {
		for entry := range feed {
			changes = append(changes, entry)
		}
	}
	return changes, err
}

func (db *Database) WaitForRevision() bool {
	base.Log("\twaiting for a revision...")
	waitFor("")
	base.Log("\t...done waiting")
	return true
}

func (db *Database) NotifyRevision() {
	notify("")
}

func (db *Database) LastSequence() (uint64, error) {
	return db.sequences.lastSequence()
}

func (db *Database) ReserveSequences(numToReserve uint64) error {
	return db.sequences.reserveSequences(numToReserve)
}
<commit_msg>Fixed out-of-range error regression in MultiChangesFeed()<commit_after>//  Copyright (c) 2012 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package db

import (
	"encoding/json"
	"math"

	"github.com/couchbaselabs/go-couchbase"

	"github.com/couchbaselabs/sync_gateway/base"
	"github.com/couchbaselabs/sync_gateway/channels"
)

// Options for Database.getChanges
type ChangesOptions struct {
	Since          uint64
	Limit          int
	Conflicts      bool
	IncludeDocs    bool
	includeDocMeta bool
	Wait           bool
}

// A changes entry; Database.getChanges returns an array of these.
// Marshals into the standard CouchDB _changes format.
type ChangeEntry struct {
	Seq     uint64       `json:"seq"`
	ID      string       `json:"id"`
	Deleted bool         `json:"deleted,omitempty"`
	Removed channels.Set `json:"removed,omitempty"`
	Doc     Body         `json:"doc,omitempty"`
	Changes []ChangeRev  `json:"changes"`
	docMeta *document
}

type ChangeRev map[string]string

type ViewDoc struct {
	Json json.RawMessage // should be type 'document', but that fails to unmarshal correctly
}

type ViewRow struct {
	ID    string
	Key   interface{}
	Value interface{}
	Doc   *ViewDoc
}

type ViewResult struct {
	TotalRows int `json:"total_rows"`
	Rows      []ViewRow
	Errors    []couchbase.ViewError
}

const kChangesPageSize = 200

// Returns a list of all the changes made on a channel. Does NOT check authorization.
func (db *Database) ChangesFeed(channel string, options ChangesOptions) (<-chan *ChangeEntry, error) {
	lastSequence := options.Since
	endkey := []interface{}{channel, map[string]interface{}{}}
	totalLimit := options.Limit
	usingDocs := options.Conflicts || options.IncludeDocs || options.includeDocMeta
	opts := Body{"stale": false, "update_seq": true,
		"endkey":       endkey,
		"include_docs": usingDocs}

	feed := make(chan *ChangeEntry, kChangesPageSize)

	lastSeq, err := db.LastSequence()
	if err == nil && options.Since >= lastSeq && !options.Wait {
		close(feed)
		return feed, nil
	}

	// Generate the output in a new goroutine, writing to 'feed':
	go func() {
		defer close(feed)
		for {
			// Query the 'channels' view:
			opts["startkey"] = []interface{}{channel, lastSequence + 1}
			limit := totalLimit
			if limit == 0 || limit > kChangesPageSize {
				limit = kChangesPageSize
			}
			opts["limit"] = limit

			var vres ViewResult
			var err error
			for len(vres.Rows) == 0 {
				vres = ViewResult{}
				err = db.Bucket.ViewCustom("sync_gateway", "channels", opts, &vres)
				if err != nil {
					base.Log("Error from 'channels' view: %v", err)
					return
				}
				if len(vres.Rows) == 0 {
					if !options.Wait || !db.WaitForRevision() {
						return
					}
				}
			}

			for _, row := range vres.Rows {
				key := row.Key.([]interface{})
				lastSequence = uint64(key[1].(float64))
				value := row.Value.([]interface{})
				docID := value[0].(string)
				revID := value[1].(string)
				entry := &ChangeEntry{
					Seq:     lastSequence,
					ID:      docID,
					Changes: []ChangeRev{{"rev": revID}},
					Deleted: (len(value) >= 3 && value[2].(bool)),
				}
				if len(value) >= 3 && !value[2].(bool) {
					entry.Removed = channels.SetOf(channel)
				}
				if usingDocs {
					doc, err := unmarshalDocument(docID, row.Doc.Json)
					if err == nil {
						if options.Conflicts {
							for _, leafID := range doc.History.getLeaves() {
								if leafID != revID {
									entry.Changes = append(entry.Changes, ChangeRev{"rev": leafID})
								}
							}
						}
						if options.IncludeDocs {
							entry.Doc, _ = db.getRevFromDoc(doc, revID, false)
						}
						if options.includeDocMeta {
							entry.docMeta = doc
						}
					}
				}
				feed <- entry
			}

			// Step to the next page of results:
			nRows := len(vres.Rows)
			if nRows < kChangesPageSize || options.Wait {
				break
			}
			if totalLimit > 0 {
				totalLimit -= nRows
				if totalLimit <= 0 {
					break
				}
			}
			delete(opts, "stale") // we only need to update the index once
		}
	}()
	return feed, nil
}

// Returns of all the changes made to multiple channels. Does NOT check authorization.
func (db *Database) MultiChangesFeed(channels channels.Set, options ChangesOptions) (<-chan *ChangeEntry, error) {
	if len(channels) == 0 {
		return nil, nil
	} else if len(channels) == 1 {
		for channel, _ := range channels {
			return db.ChangesFeed(channel, options)
		}
	}

	waitMode := options.Wait
	options.Wait = false

	output := make(chan *ChangeEntry, kChangesPageSize)
	go func() {
		defer close(output)

		for {
			feeds := make([]<-chan *ChangeEntry, 0, len(channels))
			for name, _ := range channels {
				feed, err := db.ChangesFeed(name, options)
				if err != nil {
					base.Warn("Error reading changes feed %q: %v", name, err)
					return
				}
				feeds = append(feeds, feed)
			}
			current := make([]*ChangeEntry, len(feeds))

			wroteAnything := false
			for {
				//FIX: This assumes Reverse or Limit aren't set in the options
				// Read more entries to fill up the current[] array:
				for i, cur := range current {
					if cur == nil && feeds[i] != nil {
						var ok bool
						current[i], ok = <-feeds[i]
						if !ok {
							feeds[i] = nil
						}
					}
				}

				// Find the current entry with the minimum sequence:
				var minSeq uint64 = math.MaxUint64
				var minEntry *ChangeEntry
				for _, cur := range current {
					if cur != nil && cur.Seq < minSeq {
						minSeq = cur.Seq
						minEntry = cur
					}
				}
				if minEntry == nil {
					break
				}

				// Clear the current entries for the sequence just sent:
				for i, cur := range current {
					if cur != nil && cur.Seq == minEntry.Seq {
						current[i] = nil
						// Also concatenate the matching entries' Removed arrays:
						if cur != minEntry && cur.Removed != nil {
							if minEntry.Removed == nil {
								minEntry.Removed = cur.Removed
							} else {
								minEntry.Removed = minEntry.Removed.Union(cur.Removed)
							}
						}
					}
				}

				// Send the entry:
				output <- minEntry
				wroteAnything = true
			}

			// In wait mode, wait for the db to change, then run again
			if wroteAnything || !waitMode || !db.WaitForRevision() {
				break
			}
		}
	}()

	return output, nil
}

// Synchronous convenience function that returns all changes as a simple array.
// Does NOT check authorization.
func (db *Database) GetChanges(channels channels.Set, options ChangesOptions) ([]*ChangeEntry, error) {
	var changes = make([]*ChangeEntry, 0, 50)
	feed, err := db.MultiChangesFeed(channels, options)
	if err == nil && feed != nil {
		for entry := range feed {
			changes = append(changes, entry)
		}
	}
	return changes, err
}

func (db *Database) WaitForRevision() bool {
	base.Log("\twaiting for a revision...")
	waitFor("")
	base.Log("\t...done waiting")
	return true
}

func (db *Database) NotifyRevision() {
	notify("")
}

func (db *Database) LastSequence() (uint64, error) {
	return db.sequences.lastSequence()
}

func (db *Database) ReserveSequences(numToReserve uint64) error {
	return db.sequences.reserveSequences(numToReserve)
}
<|endoftext|>


<commit_before>	Signature string     `json:"signature"`
<commit_msg>Remove block<commit_after><|endoftext|>


<commit_before>/*
 * (C) Copyright 2014 Nuxeo SA (http://nuxeo.com/) and contributors.
 *
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Apache License Version 2.0
 * which accompanies this distribution, and is available at
 * http://www.apache.org/licenses/
 *
 * See the Apache Licence for more details.
 *
 * Contributors:
 *     nuxeo.io Team
 */

package main

import (
	"github.com/coreos/go-etcd/etcd"
	"github.com/golang/glog"
	"time"
	"os/exec"
	"strconv"
	"os"
)

// A watcher loads and watch the etcd hierarchy for services.
type watcher struct {
	client   *etcd.Client
	config   *Config
}

// Constructor for a new watcher
func NewEtcdWatcher(config *Config) (*watcher, error) {
	client, err := config.getEtcdClient()

	if err != nil {
		return nil, err
	}
	return &watcher{client, config}, nil
}

//Init services watcher
func (w *watcher) init() {
	w.loadAndWatch(w.config.servicePrefix, w.checkServiceAccess)
}

// Loads and watch an etcd directory to register objects like services
// The register function is passed the etcd Node that has been loaded.
func (w *watcher) loadAndWatch(etcdDir string, registerFunc func(*etcd.Node, string)) {
	w.loadPrefix(etcdDir, registerFunc)
	updateChannel := make(chan *etcd.Response, 10)
	go w.watch(updateChannel, registerFunc)
	w.client.Watch(etcdDir, (uint64)(0), true, updateChannel, nil)
}

func (w *watcher) loadPrefix(etcDir string, registerFunc func(*etcd.Node, string)) {
	response, err := w.client.Get(etcDir, true, true)
	if err == nil {
		for _, serviceNode := range response.Node.Nodes {
			registerFunc(serviceNode, response.Action)
		}
	}
}

func (w *watcher) watch(updateChannel chan *etcd.Response, registerFunc func(*etcd.Node, string)) {
	for {
		response := <-updateChannel
		if response != nil {
			registerFunc(response.Node, response.Action)
		}
	}
}

func (w *watcher) checkServiceAccess(node *etcd.Node, action string) {
	serviceName := w.config.getServiceForNode(node, w.config)

	// Get service's root node instead of changed node.
	serviceNode, _ := w.client.Get(w.config.servicePrefix+"/"+serviceName, true, true)

	for _, indexNode := range serviceNode.Node.Nodes {

		serviceIndex := w.config.getServiceIndexForNode(indexNode, w.config)
		serviceKey := w.config.servicePrefix + "/" + serviceName + "/" + serviceIndex
		lastAccessKey := serviceKey + "/lastAccess"
		statusKey := serviceKey + "/status"

		response, err := w.client.Get(serviceKey, true, true)

		if err == nil {

			service := &Service{}
			service.index = serviceIndex
			service.nodeKey = serviceKey
			service.name = "nxio."+serviceName+"."+serviceIndex+".service"

			for _, node := range response.Node.Nodes {
				switch node.Key {
				case statusKey:
					service.status = &Status{}
				for _, subNode := range node.Nodes {
					switch subNode.Key {
					case statusKey + "/alive":
						service.status.alive = subNode.Value
					case statusKey + "/current":
						service.status.current = subNode.Value
					case statusKey + "/expected":
						service.status.expected = subNode.Value
					}
				}
				case lastAccessKey:
					lastAccess := node.Value
					lastAccessTime, err := time.Parse(TIME_FORMAT, lastAccess)
					if err != nil {
						glog.Errorf("Error parsing last access date with service %s: %s", service.name, err)
						break
					}
					service.lastAccess = &lastAccessTime
				}
			}

			parameter, _ := strconv.Atoi(w.config.passiveLimitDuration)
			passiveLimitDuration := time.Duration(parameter) * time.Hour

			// Checking if the service should be re-activated or not
			if service.lastAccess != nil && service.status != nil {
				if w.hasToBeActivated(service, passiveLimitDuration) {
					response, error := w.client.Set(statusKey+"/expected", STARTED_STATUS, 0)
					if error != nil && response == nil {
						glog.Errorf("Setting expected status to 'started' has failed for Service "+service.name+": %s", err)
					}
					cmd := exec.Command("/usr/bin/fleetctl","--endpoint=" + w.config.etcdAddress, "start", service.name)
					cmd.Stdin = os.Stdin
					cmd.Stdout = os.Stdout
					cmd.Stderr = os.Stderr
					err := cmd.Run()
					if err != nil {
						glog.Errorf("Service "+service.name+" restart has failed: %s", err)
						break
					}
					glog.Infof("Service %s restarted", service.name)
				}
			}
		}
	}
}

func (watcher *watcher) hasToBeActivated(service *Service, passiveLimitDuration time.Duration) bool {
	return !time.Now().After(service.lastAccess.Add(passiveLimitDuration)) && service.status.expected == PASSIVATED_STATUS
}

<commit_msg>NXIO-158: Update re-activation condition<commit_after>/*
 * (C) Copyright 2014 Nuxeo SA (http://nuxeo.com/) and contributors.
 *
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Apache License Version 2.0
 * which accompanies this distribution, and is available at
 * http://www.apache.org/licenses/
 *
 * See the Apache Licence for more details.
 *
 * Contributors:
 *     nuxeo.io Team
 */

package main

import (
	"github.com/coreos/go-etcd/etcd"
	"github.com/golang/glog"
	"time"
	"os/exec"
	"os"
)

// A watcher loads and watch the etcd hierarchy for services.
type watcher struct {
	client   *etcd.Client
	config   *Config
}

// Constructor for a new watcher
func NewEtcdWatcher(config *Config) (*watcher, error) {
	client, err := config.getEtcdClient()

	if err != nil {
		return nil, err
	}
	return &watcher{client, config}, nil
}

//Init services watcher
func (w *watcher) init() {
	w.loadAndWatch(w.config.servicePrefix, w.checkServiceAccess)
}

// Loads and watch an etcd directory to register objects like services
// The register function is passed the etcd Node that has been loaded.
func (w *watcher) loadAndWatch(etcdDir string, registerFunc func(*etcd.Node, string)) {
	w.loadPrefix(etcdDir, registerFunc)
	updateChannel := make(chan *etcd.Response, 10)
	go w.watch(updateChannel, registerFunc)
	w.client.Watch(etcdDir, (uint64)(0), true, updateChannel, nil)
}

func (w *watcher) loadPrefix(etcDir string, registerFunc func(*etcd.Node, string)) {
	response, err := w.client.Get(etcDir, true, true)
	if err == nil {
		for _, serviceNode := range response.Node.Nodes {
			registerFunc(serviceNode, response.Action)
		}
	}
}

func (w *watcher) watch(updateChannel chan *etcd.Response, registerFunc func(*etcd.Node, string)) {
	for {
		response := <-updateChannel
		if response != nil {
			registerFunc(response.Node, response.Action)
		}
	}
}

func (w *watcher) checkServiceAccess(node *etcd.Node, action string) {
	serviceName := w.config.getServiceForNode(node, w.config)

	// Get service's root node instead of changed node.
	serviceNode, _ := w.client.Get(w.config.servicePrefix+"/"+serviceName, true, true)

	for _, indexNode := range serviceNode.Node.Nodes {

		serviceIndex := w.config.getServiceIndexForNode(indexNode, w.config)
		serviceKey := w.config.servicePrefix + "/" + serviceName + "/" + serviceIndex
		lastAccessKey := serviceKey + "/lastAccess"
		statusKey := serviceKey + "/status"

		response, err := w.client.Get(serviceKey, true, true)

		if err == nil {

			service := &Service{}
			service.index = serviceIndex
			service.nodeKey = serviceKey
			service.name = "nxio."+serviceName+"."+serviceIndex+".service"

			for _, node := range response.Node.Nodes {
				switch node.Key {
				case statusKey:
					service.status = &Status{}
				for _, subNode := range node.Nodes {
					switch subNode.Key {
					case statusKey + "/alive":
						service.status.alive = subNode.Value
					case statusKey + "/current":
						service.status.current = subNode.Value
					case statusKey + "/expected":
						service.status.expected = subNode.Value
					}
				}
				case lastAccessKey:
					lastAccess := node.Value
					lastAccessTime, err := time.Parse(TIME_FORMAT, lastAccess)
					if err != nil {
						glog.Errorf("Error parsing last access date with service %s: %s", service.name, err)
						break
					}
					service.lastAccess = &lastAccessTime
				}
			}

			// Checking if the service should be re-activated or not
			if service.lastAccess != nil && service.status != nil {
				if w.hasToBeActivated(service) {
					cmd := exec.Command("/usr/bin/fleetctl", "--endpoint="+w.config.etcdAddress, "start", service.name)
					cmd.Stdin = os.Stdin
					cmd.Stdout = os.Stdout
					cmd.Stderr = os.Stderr
					err := cmd.Run()
					if err != nil {
						glog.Errorf("Service "+service.name+" restart has failed: %s", err)
						break
					}
					glog.Infof("Service %s restarted", service.name)
				}
			}
		}
	}
}

func (watcher *watcher) hasToBeActivated(service *Service) bool {
	return service.status.expected == STARTED_STATUS && service.status.current == STOPPED_STATUS
}

<|endoftext|>


<commit_before>package etl

import (
	"errors"
	"net"
	"os"
	"regexp"
	"strconv"
)

// YYYYMMDD is a regexp string for identifying dense dates.
const YYYYMMDD = `\d{4}[01]\d[0123]\d`

// MlabDomain is the DNS domain for all mlab servers.
const MlabDomain = `measurement-lab.org`

const start = `^gs://(?P<prefix>.*)/(?P<exp>[^/]*)/`
const datePath = `(?P<datepath>\d{4}/[01]\d/[0123]\d)/`
const dateTime = `(\d{4}[01]\d[0123]\d)T\d{6}Z`
const mlabN_podNN = `-(mlab\d)-([[:alpha:]]{3}\d[0-9t])-`
const exp_NNNN = `(.*)-(\d{4})`
const suffix = `(?:\.tar|\.tar.gz|\.tgz)$`

// These are here to facilitate use across queue-pusher and parsing components.
var (
	// This matches any valid test file name, and some invalid ones.
	TaskPattern = regexp.MustCompile(start + // #1 #2
		datePath + // #3 - YYYY/MM/DD
		dateTime + // #4 - YYYYMMDDTHHMMSSZ
		mlabN_podNN + // #5 #6 - e.g. -mlab1-lax04-
		exp_NNNN + // #7 #8 e.g. ndt-0001
		suffix) // #9 typically .tgz

	startPattern = regexp.MustCompile(start)
	endPattern   = regexp.MustCompile(suffix)
	podPattern   = regexp.MustCompile(mlabN_podNN)
)

// DataPath breaks out the components of a task filename.
type DataPath struct {
	// TODO(dev) Delete unused fields.
	// They are comprehensive now in anticipation of using them to populate
	// new fields in the BQ tables.
	Exp1       string // #2
	DatePath   string // #3
	PackedDate string // #4
	Host       string // #5
	Pod        string // #6
	Experiment string // #7
	FileNumber string // #8
}

// ValidateTestPath validates a task filename.
func ValidateTestPath(path string) (*DataPath, error) {
	fields := TaskPattern.FindStringSubmatch(path)

	if fields == nil {
		if !startPattern.MatchString(path) {
			return nil, errors.New("Path should begin with gs://.../.../: " + path)
		}
		if !endPattern.MatchString(path) {
			return nil, errors.New("Path should end in .tar, .tgz, or .tar.gz: " + path)
		}
		if !podPattern.MatchString(path) {
			return nil, errors.New("Path should contain -mlabN-podNN: " + path)
		}
		return nil, errors.New("Invalid test path: " + path)
	}
	return &DataPath{
			fields[2], fields[3], fields[4], fields[5],
			fields[6], fields[7], fields[8]},
		nil
}

// GetDataType finds the type of data stored in a file from its complete filename
func (fn *DataPath) GetDataType() DataType {
	dt, ok := DirToDataType[fn.Exp1]
	if !ok {
		return INVALID
	}
	return dt
}

// IsBatchService return true if this is a NDT batch service.
func IsBatchService() bool {
	isBatch, _ := strconv.ParseBool(os.Getenv("NDT_BATCH"))
	return isBatch
}

// GetMetroName extracts metro name like "acc" from file name like
// 20170501T000000Z-mlab1-acc02-paris-traceroute-0000.tgz
func GetMetroName(rawFilename string) string {
	podName := podPattern.FindString(rawFilename)
	if podName != "" {
		return podName[7:10]
	}
	return ""
}

// GetIntFromIPv4 converts an IPv4 address to equivalent uint32.
func GetIntFromIPv4(p4 net.IP) uint {
	return uint(p4[0])<<24 + uint(p4[1])<<16 + uint(p4[2])<<8 + uint(p4[3])
}

// GetIntFromIPv6Upper converts the upper 64 bits of an IPv6 address into uint64.
func GetIntFromIPv6Upper(p6 net.IP) uint64 {
	return uint64(p6[0])<<56 + uint64(p6[1])<<48 + uint64(p6[2])<<40 + uint64(p6[3])<<32 + uint64(p6[4])<<24 + uint64(p6[5])<<16 + uint64(p6[6])<<8 + uint64(p6[7])
}

// NumberBitsDifferent computes how many trailing bits differ between two IP addresses.
// The second returned number is 4 for IP_v4, 6 for IP_v6, and 0 for invalid input.
func NumberBitsDifferent(first string, second string) (int, int) {
	ip1 := net.ParseIP(first)
	ip2 := net.ParseIP(second)
	if ip1.To4() != nil && ip2.To4() != nil {
		dist := uint(GetIntFromIPv4(ip1.To4()) ^ uint(GetIntFromIPv4(ip2.To4())))
		n := 0
		for ; dist != 0; dist >>= 1 {
			n++
		}
		return n, 4
	}
	if ip1.To16() != nil && ip2.To16() != nil {
		// We will only compare the upper 64 bits.
		dist := uint64(GetIntFromIPv6Upper(ip1.To16())) ^ uint64(GetIntFromIPv6Upper(ip2.To16()))
		n := 0
		for ; dist != 0; dist >>= 1 {
			n++
		}
		return n, 6
	}
	return -1, 0
}

//=====================================================================

// DataType identifies the type of data handled by a parser.
type DataType string

// BQBufferSize returns the appropriate BQ insert buffer size.
func (dt DataType) BQBufferSize() int {
	// Special case for NDT when omitting deltas.
	if dt == NDT {
		omitDeltas, _ := strconv.ParseBool(os.Getenv("NDT_OMIT_DELTAS"))
		if omitDeltas {
			return dataTypeToBQBufferSize[NDT_OMIT_DELTAS]
		}
	}
	return dataTypeToBQBufferSize[dt]
}

const (
	NDT             = DataType("ndt")
	NDT_OMIT_DELTAS = DataType("ndt_nodelta") // to support larger buffer size.
	SS              = DataType("sidestream")
	PT              = DataType("traceroute")
	SW              = DataType("disco")
	INVALID         = DataType("invalid")
)

var (
	// DirToDataType maps from gs:// subdirectory to data type.
	// TODO - this should be loaded from a config.
	DirToDataType = map[string]DataType{
		"ndt":              NDT,
		"sidestream":       SS,
		"paris-traceroute": PT,
		"switch":           SW,
	}

	// DataTypeToTable maps from data type to BigQuery table name.
	// TODO - this should be loaded from a config.
	DataTypeToTable = map[DataType]string{
		NDT:     "ndt",
		SS:      "sidestream",
		PT:      "traceroute",
		SW:      "disco_test",
		INVALID: "invalid",
	}

	// Map from data type to number of buffer size for BQ insertion.
	// TODO - this should be loaded from a config.
	dataTypeToBQBufferSize = map[DataType]int{
		NDT:             10,
		NDT_OMIT_DELTAS: 50,
		SS:              100,
		PT:              300,
		SW:              100,
		INVALID:         0,
	}
	// There is also a mapping of data types to queue names in
	// queue_pusher.go
)
<commit_msg>Add TODO<commit_after>package etl

import (
	"errors"
	"net"
	"os"
	"regexp"
	"strconv"
)

// YYYYMMDD is a regexp string for identifying dense dates.
const YYYYMMDD = `\d{4}[01]\d[0123]\d`

// MlabDomain is the DNS domain for all mlab servers.
const MlabDomain = `measurement-lab.org`

const start = `^gs://(?P<prefix>.*)/(?P<exp>[^/]*)/`
const datePath = `(?P<datepath>\d{4}/[01]\d/[0123]\d)/`
const dateTime = `(\d{4}[01]\d[0123]\d)T\d{6}Z`
const mlabN_podNN = `-(mlab\d)-([[:alpha:]]{3}\d[0-9t])-`
const exp_NNNN = `(.*)-(\d{4})`
const suffix = `(?:\.tar|\.tar.gz|\.tgz)$`

// These are here to facilitate use across queue-pusher and parsing components.
var (
	// This matches any valid test file name, and some invalid ones.
	TaskPattern = regexp.MustCompile(start + // #1 #2
		datePath + // #3 - YYYY/MM/DD
		dateTime + // #4 - YYYYMMDDTHHMMSSZ
		mlabN_podNN + // #5 #6 - e.g. -mlab1-lax04-
		exp_NNNN + // #7 #8 e.g. ndt-0001
		suffix) // #9 typically .tgz

	startPattern = regexp.MustCompile(start)
	endPattern   = regexp.MustCompile(suffix)
	podPattern   = regexp.MustCompile(mlabN_podNN)
)

// DataPath breaks out the components of a task filename.
type DataPath struct {
	// TODO(dev) Delete unused fields.
	// They are comprehensive now in anticipation of using them to populate
	// new fields in the BQ tables.
	Exp1       string // #2
	DatePath   string // #3
	PackedDate string // #4
	Host       string // #5
	Pod        string // #6
	Experiment string // #7
	FileNumber string // #8
}

// ValidateTestPath validates a task filename.
func ValidateTestPath(path string) (*DataPath, error) {
	fields := TaskPattern.FindStringSubmatch(path)

	if fields == nil {
		if !startPattern.MatchString(path) {
			return nil, errors.New("Path should begin with gs://.../.../: " + path)
		}
		if !endPattern.MatchString(path) {
			return nil, errors.New("Path should end in .tar, .tgz, or .tar.gz: " + path)
		}
		if !podPattern.MatchString(path) {
			return nil, errors.New("Path should contain -mlabN-podNN: " + path)
		}
		return nil, errors.New("Invalid test path: " + path)
	}
	return &DataPath{
			fields[2], fields[3], fields[4], fields[5],
			fields[6], fields[7], fields[8]},
		nil
}

// GetDataType finds the type of data stored in a file from its complete filename
func (fn *DataPath) GetDataType() DataType {
	dt, ok := DirToDataType[fn.Exp1]
	if !ok {
		return INVALID
	}
	return dt
}

// IsBatchService return true if this is a NDT batch service.
// TODO - update this to BATCH_SERVICE, so it makes sense for other pipelines.
func IsBatchService() bool {
	isBatch, _ := strconv.ParseBool(os.Getenv("NDT_BATCH"))
	return isBatch
}

// GetMetroName extracts metro name like "acc" from file name like
// 20170501T000000Z-mlab1-acc02-paris-traceroute-0000.tgz
func GetMetroName(rawFilename string) string {
	podName := podPattern.FindString(rawFilename)
	if podName != "" {
		return podName[7:10]
	}
	return ""
}

// GetIntFromIPv4 converts an IPv4 address to equivalent uint32.
func GetIntFromIPv4(p4 net.IP) uint {
	return uint(p4[0])<<24 + uint(p4[1])<<16 + uint(p4[2])<<8 + uint(p4[3])
}

// GetIntFromIPv6Upper converts the upper 64 bits of an IPv6 address into uint64.
func GetIntFromIPv6Upper(p6 net.IP) uint64 {
	return uint64(p6[0])<<56 + uint64(p6[1])<<48 + uint64(p6[2])<<40 + uint64(p6[3])<<32 + uint64(p6[4])<<24 + uint64(p6[5])<<16 + uint64(p6[6])<<8 + uint64(p6[7])
}

// NumberBitsDifferent computes how many trailing bits differ between two IP addresses.
// The second returned number is 4 for IP_v4, 6 for IP_v6, and 0 for invalid input.
func NumberBitsDifferent(first string, second string) (int, int) {
	ip1 := net.ParseIP(first)
	ip2 := net.ParseIP(second)
	if ip1.To4() != nil && ip2.To4() != nil {
		dist := uint(GetIntFromIPv4(ip1.To4()) ^ uint(GetIntFromIPv4(ip2.To4())))
		n := 0
		for ; dist != 0; dist >>= 1 {
			n++
		}
		return n, 4
	}
	if ip1.To16() != nil && ip2.To16() != nil {
		// We will only compare the upper 64 bits.
		dist := uint64(GetIntFromIPv6Upper(ip1.To16())) ^ uint64(GetIntFromIPv6Upper(ip2.To16()))
		n := 0
		for ; dist != 0; dist >>= 1 {
			n++
		}
		return n, 6
	}
	return -1, 0
}

//=====================================================================

// DataType identifies the type of data handled by a parser.
type DataType string

// BQBufferSize returns the appropriate BQ insert buffer size.
func (dt DataType) BQBufferSize() int {
	// Special case for NDT when omitting deltas.
	if dt == NDT {
		omitDeltas, _ := strconv.ParseBool(os.Getenv("NDT_OMIT_DELTAS"))
		if omitDeltas {
			return dataTypeToBQBufferSize[NDT_OMIT_DELTAS]
		}
	}
	return dataTypeToBQBufferSize[dt]
}

const (
	NDT             = DataType("ndt")
	NDT_OMIT_DELTAS = DataType("ndt_nodelta") // to support larger buffer size.
	SS              = DataType("sidestream")
	PT              = DataType("traceroute")
	SW              = DataType("disco")
	INVALID         = DataType("invalid")
)

var (
	// DirToDataType maps from gs:// subdirectory to data type.
	// TODO - this should be loaded from a config.
	DirToDataType = map[string]DataType{
		"ndt":              NDT,
		"sidestream":       SS,
		"paris-traceroute": PT,
		"switch":           SW,
	}

	// DataTypeToTable maps from data type to BigQuery table name.
	// TODO - this should be loaded from a config.
	DataTypeToTable = map[DataType]string{
		NDT:     "ndt",
		SS:      "sidestream",
		PT:      "traceroute",
		SW:      "disco_test",
		INVALID: "invalid",
	}

	// Map from data type to number of buffer size for BQ insertion.
	// TODO - this should be loaded from a config.
	dataTypeToBQBufferSize = map[DataType]int{
		NDT:             10,
		NDT_OMIT_DELTAS: 50,
		SS:              100,
		PT:              300,
		SW:              100,
		INVALID:         0,
	}
	// There is also a mapping of data types to queue names in
	// queue_pusher.go
)
<|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/arachnid/evmdis"
)

func main() {
    hexdata, err := ioutil.ReadAll(os.Stdin)
    if err != nil {
        log.Fatalf("Could not read from stdin: %v", err)
    }

    bytecode := make([]byte, hex.DecodedLen(len(hexdata)))
    hex.Decode(bytecode, hexdata)

    program := evmdis.NewProgram(bytecode)
    if err := evmdis.PerformReachingAnalysis(program); err != nil {
    	log.Fatalf("Error performing reaching analysis: %v", err)
    }
    evmdis.PerformReachesAnalysis(program)
    evmdis.CreateLabels(program)
    evmdis.BuildExpressions(program)

    for _, block := range program.Blocks {
    	offset := block.Offset

    	var label *evmdis.JumpLabel
    	block.Annotations.Get(&label)
    	if label != nil {
    		fmt.Printf("%v\n", label)
    	}

    	for _, instruction := range block.Instructions {
    		var reaching evmdis.ReachingDefinition
    		instruction.Annotations.Get(&reaching)

    		var reaches evmdis.ReachesDefinition
    		instruction.Annotations.Get(&reaches)

    		var expression evmdis.Expression
    		instruction.Annotations.Get(&expression)

    		if expression != nil {
    			//fmt.Printf("0x%X\t%v\t%v\t%v\n", offset, expression, reaching, reaches)
    			if instruction.Op.StackWrites() == 1 && !instruction.Op.IsDup() {
    				fmt.Printf("0x%X\tPUSH(%v)\t%v\n", offset, expression, reaches)
    			} else {
	    			fmt.Printf("0x%X\t%v\n", offset, expression)
	    		}
    		}
    		offset += instruction.Op.OperandSize() + 1
    	}
    	fmt.Printf("\n")
    }
}
<commit_msg>Remove reaches defs from output<commit_after>package main

import (
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/arachnid/evmdis"
)

func main() {
    hexdata, err := ioutil.ReadAll(os.Stdin)
    if err != nil {
        log.Fatalf("Could not read from stdin: %v", err)
    }

    bytecode := make([]byte, hex.DecodedLen(len(hexdata)))
    hex.Decode(bytecode, hexdata)

    program := evmdis.NewProgram(bytecode)
    if err := evmdis.PerformReachingAnalysis(program); err != nil {
    	log.Fatalf("Error performing reaching analysis: %v", err)
    }
    evmdis.PerformReachesAnalysis(program)
    evmdis.CreateLabels(program)
    evmdis.BuildExpressions(program)

    for _, block := range program.Blocks {
    	offset := block.Offset

    	var label *evmdis.JumpLabel
    	block.Annotations.Get(&label)
    	if label != nil {
    		fmt.Printf("%v\n", label)
    	}

    	for _, instruction := range block.Instructions {
    		var reaching evmdis.ReachingDefinition
    		instruction.Annotations.Get(&reaching)

    		var reaches evmdis.ReachesDefinition
    		instruction.Annotations.Get(&reaches)

    		var expression evmdis.Expression
    		instruction.Annotations.Get(&expression)

    		if expression != nil {
    			//fmt.Printf("0x%X\t%v\t%v\t%v\n", offset, expression, reaching, reaches)
    			if instruction.Op.StackWrites() == 1 && !instruction.Op.IsDup() {
    				fmt.Printf("0x%X\tPUSH(%v)\n", offset, expression)
    			} else {
	    			fmt.Printf("0x%X\t%v\n", offset, expression)
	    		}
    		}
    		offset += instruction.Op.OperandSize() + 1
    	}
    	fmt.Printf("\n")
    }
}
<|endoftext|>


<commit_before>// Copyright 2012-2015 Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

import "testing"

func TestExists(t *testing.T) {
	client := setupTestClientAndCreateIndexAndAddDocs(t) //, SetTraceLog(log.New(os.Stdout, "", 0)))

	exists, err := client.Exists().Index(testIndexName).Type("comment").Id("1").Parent("tweet").Do()
	if err != nil {
		t.Fatal(err)
	}
	if !exists {
		t.Fatal("expected document to exist")
	}
}
<commit_msg>Exists test<commit_after>// Copyright 2012-2015 Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

import (
	"testing"
)

func TestExists(t *testing.T) {
	client := setupTestClientAndCreateIndexAndAddDocs(t) //, SetTraceLog(log.New(os.Stdout, "", 0)))

	exists, err := client.Exists().Index(testIndexName).Type("comment").Id("1").Parent("tweet").Do()
	if err != nil {
		t.Fatal(err)
	}
	if !exists {
		t.Fatal("expected document to exist")
	}
}

func TestExistsValidate(t *testing.T) {
	client := setupTestClient(t)

	// No index -> fail with error
	res, err := NewExistsService(client).Type("tweet").Id("1").Do()
	if err == nil {
		t.Fatalf("expected Delete to fail without index name")
	}
	if res != false {
		t.Fatalf("expected result to be false; got: %v", res)
	}

	// No type -> fail with error
	res, err = NewExistsService(client).Index(testIndexName).Id("1").Do()
	if err == nil {
		t.Fatalf("expected Delete to fail without index name")
	}
	if res != false {
		t.Fatalf("expected result to be false; got: %v", res)
	}

	// No id -> fail with error
	res, err = NewExistsService(client).Index(testIndexName).Type("tweet").Do()
	if err == nil {
		t.Fatalf("expected Delete to fail without index name")
	}
	if res != false {
		t.Fatalf("expected result to be false; got: %v", res)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package db encapsulates tsuru connection with MongoDB.
//
// The function Open dials to MongoDB and returns a connection (represented by
// the Storage type). It manages an internal pool of connections, and
// reconnects in case of failures. That means that you should not store
// references to the connection, but always call Open.
package db

import (
	"github.com/globocom/config"
	"labix.org/v2/mgo"
	"sync"
	"time"
)

var (
	conn   = make(map[string]*session) // pool of connections
	mut    sync.RWMutex                // for pool thread safety
	ticker *time.Ticker                // for garbage collection
)

const period time.Duration = 7 * 24 * time.Hour

type session struct {
	s    *mgo.Session
	used time.Time
}

// Storage holds the connection with the database.
type Storage struct {
	session *mgo.Session
	dbname  string
}

// Collection represents a database collection. It embeds mgo.Collection for
// operations, and holds a session to MongoDB. The user may close the session
// using the method close.
type Collection struct {
	*mgo.Collection
}

// Close closes the session with the database.
func (c *Collection) Close() {
	c.Collection.Database.Session.Close()
}

func open(addr, dbname string) (*Storage, error) {
	sess, err := mgo.Dial(addr)
	if err != nil {
		return nil, err
	}
	storage := &Storage{session: sess, dbname: dbname}
	mut.Lock()
	conn[addr] = &session{s: sess, used: time.Now()}
	mut.Unlock()
	return storage, nil
}

// Open dials to the MongoDB database, and return the connection (represented
// by the type Storage).
//
// addr is a MongoDB connection URI, and dbname is the name of the database.
//
// This function returns a pointer to a Storage, or a non-nil error in case of
// any failure.
func Open(addr, dbname string) (storage *Storage, err error) {
	defer func() {
		if r := recover(); r != nil {
			storage, err = open(addr, dbname)
		}
	}()
	mut.RLock()
	if session, ok := conn[addr]; ok {
		mut.RUnlock()
		if err = session.s.Ping(); err == nil {
			mut.Lock()
			session.used = time.Now()
			conn[addr] = session
			mut.Unlock()
			copy := session.s.Clone()
			return &Storage{copy, dbname}, nil
		}
		return open(addr, dbname)
	}
	mut.RUnlock()
	return open(addr, dbname)
}

// Conn reads the tsuru config and calls Open to get a database connection.
//
// Most tsuru packages should probably use this function. Open is intended for
// use when supporting more than one database.
func Conn() (*Storage, error) {
	url, _ := config.GetString("database:url")
	if url == "" {
		url = "127.0.0.1:27017"
	}
	dbname, _ := config.GetString("database:name")
	if dbname == "" {
		dbname = "tsuru"
	}
	return Open(url, dbname)
}

// Close closes the storage, releasing the connection.
func (s *Storage) Close() {
	s.session.Close()
}

// Collection returns a collection by its name.
//
// If the collection does not exist, MongoDB will create it.
func (s *Storage) Collection(name string) *Collection {
	return &Collection{s.session.DB(s.dbname).C(name)}
}

// Apps returns the apps collection from MongoDB.
func (s *Storage) Apps() *Collection {
	nameIndex := mgo.Index{Key: []string{"name"}, Unique: true}
	c := s.Collection("apps")
	c.EnsureIndex(nameIndex)
	return c
}

// Platforms returns the platforms collection from MongoDB.
func (s *Storage) Platforms() *Collection {
	return s.Collection("platforms")
}

// Logs returns the logs collection from MongoDB.
func (s *Storage) Logs() *Collection {
	appNameIndex := mgo.Index{Key: []string{"appname"}}
	sourceIndex := mgo.Index{Key: []string{"source"}}
	c := s.Collection("logs")
	c.EnsureIndex(appNameIndex)
	c.EnsureIndex(sourceIndex)
	return c
}

// Services returns the services collection from MongoDB.
func (s *Storage) Services() *Collection {
	c := s.Collection("services")
	return c
}

// ServiceInstances returns the services_instances collection from MongoDB.
func (s *Storage) ServiceInstances() *Collection {
	return s.Collection("service_instances")
}

// Users returns the users collection from MongoDB.
func (s *Storage) Users() *Collection {
	emailIndex := mgo.Index{Key: []string{"email"}, Unique: true}
	c := s.Collection("users")
	c.EnsureIndex(emailIndex)
	return c
}

func (s *Storage) Tokens() *Collection {
	return s.Collection("tokens")
}

func (s *Storage) PasswordTokens() *Collection {
	return s.Collection("password_tokens")
}

func (s *Storage) UserActions() *Collection {
	return s.Collection("user_actions")
}

// Teams returns the teams collection from MongoDB.
func (s *Storage) Teams() *Collection {
	return s.Collection("teams")
}

// Quota returns the quota collection from MongoDB.
func (s *Storage) Quota() *Collection {
	userIndex := mgo.Index{Key: []string{"owner"}, Unique: true}
	c := s.Collection("quota")
	c.EnsureIndex(userIndex)
	return c
}

func init() {
	ticker = time.NewTicker(time.Hour)
	go retire(ticker)
}

// retire retires old connections
func retire(t *time.Ticker) {
	for _ = range t.C {
		now := time.Now()
		var old []string
		mut.RLock()
		for k, v := range conn {
			if now.Sub(v.used) >= period {
				old = append(old, k)
			}
		}
		mut.RUnlock()
		mut.Lock()
		for _, c := range old {
			conn[c].s.Close()
			delete(conn, c)
		}
		mut.Unlock()
	}
}
<commit_msg>db: fixed race problems.<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package db encapsulates tsuru connection with MongoDB.
//
// The function Open dials to MongoDB and returns a connection (represented by
// the Storage type). It manages an internal pool of connections, and
// reconnects in case of failures. That means that you should not store
// references to the connection, but always call Open.
package db

import (
	"github.com/globocom/config"
	"labix.org/v2/mgo"
	"sync"
	"time"
)

var (
	conn   = make(map[string]*session) // pool of connections
	mut    sync.RWMutex                // for pool thread safety
	ticker *time.Ticker                // for garbage collection
)

const period time.Duration = 7 * 24 * time.Hour

type session struct {
	s    *mgo.Session
	used time.Time
}

// Storage holds the connection with the database.
type Storage struct {
	session *mgo.Session
	dbname  string
}

// Collection represents a database collection. It embeds mgo.Collection for
// operations, and holds a session to MongoDB. The user may close the session
// using the method close.
type Collection struct {
	*mgo.Collection
}

// Close closes the session with the database.
func (c *Collection) Close() {
	c.Collection.Database.Session.Close()
}

func open(addr, dbname string) (*Storage, error) {
	sess, err := mgo.Dial(addr)
	if err != nil {
		return nil, err
	}
	copy := sess.Clone()
	storage := &Storage{session: copy, dbname: dbname}
	mut.Lock()
	conn[addr] = &session{s: sess, used: time.Now()}
	mut.Unlock()
	return storage, nil
}

// Open dials to the MongoDB database, and return the connection (represented
// by the type Storage).
//
// addr is a MongoDB connection URI, and dbname is the name of the database.
//
// This function returns a pointer to a Storage, or a non-nil error in case of
// any failure.
func Open(addr, dbname string) (storage *Storage, err error) {
	defer func() {
		if r := recover(); r != nil {
			storage, err = open(addr, dbname)
		}
	}()
	mut.RLock()
	if session, ok := conn[addr]; ok {
		mut.RUnlock()
		if err = session.s.Ping(); err == nil {
			mut.Lock()
			session.used = time.Now()
			conn[addr] = session
			mut.Unlock()
			copy := session.s.Clone()
			return &Storage{copy, dbname}, nil
		}
		return open(addr, dbname)
	}
	mut.RUnlock()
	return open(addr, dbname)
}

// Conn reads the tsuru config and calls Open to get a database connection.
//
// Most tsuru packages should probably use this function. Open is intended for
// use when supporting more than one database.
func Conn() (*Storage, error) {
	url, _ := config.GetString("database:url")
	if url == "" {
		url = "127.0.0.1:27017"
	}
	dbname, _ := config.GetString("database:name")
	if dbname == "" {
		dbname = "tsuru"
	}
	return Open(url, dbname)
}

// Close closes the storage, releasing the connection.
func (s *Storage) Close() {
	s.session.Close()
}

// Collection returns a collection by its name.
//
// If the collection does not exist, MongoDB will create it.
func (s *Storage) Collection(name string) *Collection {
	return &Collection{s.session.DB(s.dbname).C(name)}
}

// Apps returns the apps collection from MongoDB.
func (s *Storage) Apps() *Collection {
	nameIndex := mgo.Index{Key: []string{"name"}, Unique: true}
	c := s.Collection("apps")
	c.EnsureIndex(nameIndex)
	return c
}

// Platforms returns the platforms collection from MongoDB.
func (s *Storage) Platforms() *Collection {
	return s.Collection("platforms")
}

// Logs returns the logs collection from MongoDB.
func (s *Storage) Logs() *Collection {
	appNameIndex := mgo.Index{Key: []string{"appname"}}
	sourceIndex := mgo.Index{Key: []string{"source"}}
	c := s.Collection("logs")
	c.EnsureIndex(appNameIndex)
	c.EnsureIndex(sourceIndex)
	return c
}

// Services returns the services collection from MongoDB.
func (s *Storage) Services() *Collection {
	c := s.Collection("services")
	return c
}

// ServiceInstances returns the services_instances collection from MongoDB.
func (s *Storage) ServiceInstances() *Collection {
	return s.Collection("service_instances")
}

// Users returns the users collection from MongoDB.
func (s *Storage) Users() *Collection {
	emailIndex := mgo.Index{Key: []string{"email"}, Unique: true}
	c := s.Collection("users")
	c.EnsureIndex(emailIndex)
	return c
}

func (s *Storage) Tokens() *Collection {
	return s.Collection("tokens")
}

func (s *Storage) PasswordTokens() *Collection {
	return s.Collection("password_tokens")
}

func (s *Storage) UserActions() *Collection {
	return s.Collection("user_actions")
}

// Teams returns the teams collection from MongoDB.
func (s *Storage) Teams() *Collection {
	return s.Collection("teams")
}

// Quota returns the quota collection from MongoDB.
func (s *Storage) Quota() *Collection {
	userIndex := mgo.Index{Key: []string{"owner"}, Unique: true}
	c := s.Collection("quota")
	c.EnsureIndex(userIndex)
	return c
}

func init() {
	ticker = time.NewTicker(time.Hour)
	go retire(ticker)
}

// retire retires old connections
func retire(t *time.Ticker) {
	for _ = range t.C {
		now := time.Now()
		var old []string
		mut.RLock()
		for k, v := range conn {
			if now.Sub(v.used) >= period {
				old = append(old, k)
			}
		}
		mut.RUnlock()
		mut.Lock()
		for _, c := range old {
			conn[c].s.Close()
			delete(conn, c)
		}
		mut.Unlock()
	}
}
<|endoftext|>


<commit_before>package isolated

import (
	"code.cloudfoundry.org/cli/integration/helpers"
	"fmt"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = FDescribe("stack command", func() {
	var (
		orgName          string
		spaceName        string
		stackName        string
		stackDescription string
	)

	BeforeEach(func() {
		orgName = helpers.NewOrgName()
		spaceName = helpers.NewSpaceName()
		stackName = helpers.PrefixedRandomName("stack")
		stackDescription = "this is a test stack"
	})

	Describe("help", func() {
		When("--help flag is set", func() {
			It("Displays command usage to output", func() {
				session := helpers.CF("stack", "--help")

				Eventually(session).Should(Say(`NAME:`))
				Eventually(session).Should(Say(`stack - Show information for a stack \(a stack is a pre-built file system, including an operating system, that can run apps\)`))
				Eventually(session).Should(Say("USAGE:"))
				Eventually(session).Should(Say("cf stack STACK_NAME"))

				Eventually(session).Should(Exit(0))
			})
		})
	})

	When("the stack name is not provided", func() {
		It("tells the user that the stack name is required, prints help text, and exits 1", func() {
			session := helpers.CF("stack")

			Eventually(session.Err).Should(Say("Incorrect Usage: the required argument `STACK_NAME` was not provided"))
			Eventually(session).Should(Say("NAME:"))
			Eventually(session).Should(Exit(1))
		})
	})

	When("the environment is not setup correctly", func() {
		It("fails with the appropriate errors", func() {
			helpers.CheckEnvironmentTargetedCorrectly(true, true, ReadOnlyOrg, "stack", stackName)
		})
	})

	When("the environment is set up correctly", func() {
		var username string

		BeforeEach(func() {
			helpers.SetupCF(orgName, spaceName)
			username, _ = helpers.GetCredentials()
		})

		AfterEach(func() {
			helpers.QuickDeleteOrg(orgName)
		})

		When("the input is invalid", func() {
			When("there are not enough arguments", func() {
				It("outputs the usage and exits 1", func() {
					session := helpers.CF("stack")

					Eventually(session.Err).Should(Say("Incorrect Usage:"))
					Eventually(session).Should(Say("NAME:"))
					Eventually(session).Should(Exit(1))
				})
			})

			When("there too many arguments", func() {
				It("ignores the extra arguments", func() {
					session := helpers.CF("stack", stackName, "extra")

					Eventually(session).Should(Say(`Getting stack %s in org %s / space %s as %s\.\.\.`, stackName, orgName, spaceName, username))
					Eventually(session.Err).Should(Say("Stack %s not found", stackName))
					Eventually(session).Should(Say("FAILED"))
					Eventually(session).Should(Exit(1))
				})
			})
		})

		When("the stack does not exist", func() {
			It("Fails", func() {
				session := helpers.CF("stack", stackName)

				Eventually(session).Should(Say(`Getting stack %s in org %s / space %s as %s\.\.\.`, stackName, orgName, spaceName, username))
				Eventually(session.Err).Should(Say("Stack %s not found", stackName))
				Eventually(session).Should(Say("FAILED"))
				Eventually(session).Should(Exit(1))
			})
		})

		When("the stack exists", func() {
			BeforeEach(func() {
				jsonBody := fmt.Sprintf(`{"name": "%s", "description": "%s"}`, stackName, stackDescription)
				session := helpers.CF("curl", "-d", jsonBody, "-X", "POST", "/v3/stacks")
				Eventually(session).Should(Exit(0))
			})
			It("Shows the details for the stack", func() {
				session := helpers.CF("stack", stackName)

				Eventually(session).Should(Say(`Getting stack %s in org %s / space %s as %s\.\.\.`, stackName, orgName, spaceName, username))
				Eventually(session).Should(Say(`name:\s+%s`, stackName))
				Eventually(session).Should(Say(`description:\s+%s`, stackDescription))
				Eventually(session).Should(Exit(0))
			})

			When("the stack exists and the --guid flag is passed", func() {
				It("prints nothing but the guid", func() {
					session := helpers.CF("stack", stackName, "--guid")

					Consistently(session).ShouldNot(Say(`Getting stack %s in org %s / space %s as %s\.\.\.`, stackName, orgName, spaceName, username))
					Consistently(session).ShouldNot(Say(`name:\s+%s`, stackName))
					Consistently(session).ShouldNot(Say(`description:\s+%s`, stackDescription))
					Eventually(session).Should(Say(`^[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}`))
					Eventually(session).Should(Exit(0))
				})
			})
		})
	})
})
<commit_msg>Remove the FDescribe I left in<commit_after>package isolated

import (
	"code.cloudfoundry.org/cli/integration/helpers"
	"fmt"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("stack command", func() {
	var (
		orgName          string
		spaceName        string
		stackName        string
		stackDescription string
	)

	BeforeEach(func() {
		orgName = helpers.NewOrgName()
		spaceName = helpers.NewSpaceName()
		stackName = helpers.PrefixedRandomName("stack")
		stackDescription = "this is a test stack"
	})

	Describe("help", func() {
		When("--help flag is set", func() {
			It("Displays command usage to output", func() {
				session := helpers.CF("stack", "--help")

				Eventually(session).Should(Say(`NAME:`))
				Eventually(session).Should(Say(`stack - Show information for a stack \(a stack is a pre-built file system, including an operating system, that can run apps\)`))
				Eventually(session).Should(Say("USAGE:"))
				Eventually(session).Should(Say("cf stack STACK_NAME"))

				Eventually(session).Should(Exit(0))
			})
		})
	})

	When("the stack name is not provided", func() {
		It("tells the user that the stack name is required, prints help text, and exits 1", func() {
			session := helpers.CF("stack")

			Eventually(session.Err).Should(Say("Incorrect Usage: the required argument `STACK_NAME` was not provided"))
			Eventually(session).Should(Say("NAME:"))
			Eventually(session).Should(Exit(1))
		})
	})

	When("the environment is not setup correctly", func() {
		It("fails with the appropriate errors", func() {
			helpers.CheckEnvironmentTargetedCorrectly(true, true, ReadOnlyOrg, "stack", stackName)
		})
	})

	When("the environment is set up correctly", func() {
		var username string

		BeforeEach(func() {
			helpers.SetupCF(orgName, spaceName)
			username, _ = helpers.GetCredentials()
		})

		AfterEach(func() {
			helpers.QuickDeleteOrg(orgName)
		})

		When("the input is invalid", func() {
			When("there are not enough arguments", func() {
				It("outputs the usage and exits 1", func() {
					session := helpers.CF("stack")

					Eventually(session.Err).Should(Say("Incorrect Usage:"))
					Eventually(session).Should(Say("NAME:"))
					Eventually(session).Should(Exit(1))
				})
			})

			When("there too many arguments", func() {
				It("ignores the extra arguments", func() {
					session := helpers.CF("stack", stackName, "extra")

					Eventually(session).Should(Say(`Getting stack %s in org %s / space %s as %s\.\.\.`, stackName, orgName, spaceName, username))
					Eventually(session.Err).Should(Say("Stack %s not found", stackName))
					Eventually(session).Should(Say("FAILED"))
					Eventually(session).Should(Exit(1))
				})
			})
		})

		When("the stack does not exist", func() {
			It("Fails", func() {
				session := helpers.CF("stack", stackName)

				Eventually(session).Should(Say(`Getting stack %s in org %s / space %s as %s\.\.\.`, stackName, orgName, spaceName, username))
				Eventually(session.Err).Should(Say("Stack %s not found", stackName))
				Eventually(session).Should(Say("FAILED"))
				Eventually(session).Should(Exit(1))
			})
		})

		When("the stack exists", func() {
			BeforeEach(func() {
				jsonBody := fmt.Sprintf(`{"name": "%s", "description": "%s"}`, stackName, stackDescription)
				session := helpers.CF("curl", "-d", jsonBody, "-X", "POST", "/v3/stacks")
				Eventually(session).Should(Exit(0))
			})
			It("Shows the details for the stack", func() {
				session := helpers.CF("stack", stackName)

				Eventually(session).Should(Say(`Getting stack %s in org %s / space %s as %s\.\.\.`, stackName, orgName, spaceName, username))
				Eventually(session).Should(Say(`name:\s+%s`, stackName))
				Eventually(session).Should(Say(`description:\s+%s`, stackDescription))
				Eventually(session).Should(Exit(0))
			})

			When("the stack exists and the --guid flag is passed", func() {
				It("prints nothing but the guid", func() {
					session := helpers.CF("stack", stackName, "--guid")

					Consistently(session).ShouldNot(Say(`Getting stack %s in org %s / space %s as %s\.\.\.`, stackName, orgName, spaceName, username))
					Consistently(session).ShouldNot(Say(`name:\s+%s`, stackName))
					Consistently(session).ShouldNot(Say(`description:\s+%s`, stackDescription))
					Eventually(session).Should(Say(`^[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}`))
					Eventually(session).Should(Exit(0))
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>package acceptance_test

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"code.cloudfoundry.org/cf-networking-helpers/testsupport"

	pusherConfig "cf-pusher/config"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	. "github.com/onsi/ginkgo"
	ginkgoConfig "github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

const Timeout_Push = 2 * time.Minute

var (
	appsDir    string
	config     helpers.Config
	testConfig pusherConfig.Config
)

func TestAcceptance(t *testing.T) {
	RegisterFailHandler(Fail)

	BeforeSuite(func() {
		config = helpers.LoadConfig()

		configPath := helpers.ConfigPath()
		configBytes, err := ioutil.ReadFile(configPath)
		Expect(err).NotTo(HaveOccurred())

		err = json.Unmarshal(configBytes, &testConfig)
		Expect(err).NotTo(HaveOccurred())

		if testConfig.Applications <= 0 {
			Fail("Applications count needs to be greater than 0")
		}

		if testConfig.AppInstances <= 0 {
			Fail("AppInstances count needs to be greater than 0")
		}

		if testConfig.ProxyApplications <= 0 {
			Fail("ProxyApplications count needs to be greater than 0")
		}

		if testConfig.ProxyInstances <= 0 {
			Fail("ProxyInstances count needs to be greater than 0")
		}

		Expect(cf.Cf("api", "--skip-ssl-validation", config.ApiEndpoint).Wait(Timeout_Short)).To(gexec.Exit(0))
		AuthAsAdmin()

		appsDir = os.Getenv("APPS_DIR")
		Expect(appsDir).NotTo(BeEmpty())

		rand.Seed(ginkgoConfig.GinkgoConfig.RandomSeed + int64(GinkgoParallelNode()))
	})

	RunSpecs(t, "Acceptance Suite")
}

func Auth(username, password string) {
	By("authenticating as " + username)
	cmd := exec.Command("cf", "auth", username, password)
	sess, err := gexec.Start(cmd, nil, nil)
	Expect(err).NotTo(HaveOccurred())
	Eventually(sess.Wait(Timeout_Short)).Should(gexec.Exit(0))
}

func getUAABaseURL() string {
	sess := cf.Cf("curl", "/v2/info")
	Eventually(sess.Wait(Timeout_Short)).Should(gexec.Exit(0))
	var response struct {
		TokenEndpoint string `json:"token_endpoint"`
	}
	err := json.Unmarshal(sess.Out.Contents(), &response)
	Expect(err).NotTo(HaveOccurred())

	uaaBaseURL := response.TokenEndpoint
	Expect(uaaBaseURL).To(HavePrefix("https://uaa."))
	return uaaBaseURL
}

func AuthAsAdmin() {
	Auth(config.AdminUser, config.AdminPassword)
}

func appDir(appType string) string {
	return filepath.Join(appsDir, appType)
}

func pushProxy(appName string) {
	Expect(cf.Cf(
		"push", appName,
		"-p", appDir("proxy"),
		"-f", defaultManifest("proxy"),
	).Wait(Timeout_Push)).To(gexec.Exit(0))
}

func defaultManifest(appType string) string {
	return filepath.Join(appDir(appType), "manifest.yml")
}

func scaleApps(apps []string, instances int) {
	parallelRunner := &testsupport.ParallelRunner{
		NumWorkers: 16,
	}
	parallelRunner.RunOnSliceStrings(apps, func(app string) {
		scaleApp(app, instances)
	})
}

func scaleApp(appName string, instances int) {
	Expect(cf.Cf(
		"scale", appName,
		"-i", fmt.Sprintf("%d", instances),
	).Wait(Timeout_Short)).To(gexec.Exit(0))
}

func pushAppWithInstanceCount(appName string, appCount int) {
	Expect(cf.Cf(
		"push", appName,
		"-p", appDir("proxy"),
		"-i", fmt.Sprintf("%d", appCount),
		"-f", defaultManifest("proxy"),
	).Wait(Timeout_Push)).To(gexec.Exit(0))
}

func restage(appName string) {
	Expect(cf.Cf(
		"restage", appName,
	).Wait(Timeout_Push)).To(gexec.Exit(0))
}

type AppInstance struct {
	hostIdentifier string
	index          string
	internalIP     string
}

func getAppInstances(appName string, instances int) []AppInstance {
	apps := make([]AppInstance, instances)
	for i := 0; i < instances; i++ {
		session := cf.Cf("ssh", appName, "-i", fmt.Sprintf("%d", i), "-c", "env | grep CF_INSTANCE")
		Expect(session.Wait(Timeout_Push)).To(gexec.Exit(0))

		env := strings.Split(string(session.Out.Contents()), "\n")
		var app AppInstance
		for _, envVar := range env {
			kv := strings.Split(envVar, "=")
			switch kv[0] {
			case "CF_INSTANCE_IP":
				app.hostIdentifier = kv[1]
			case "CF_INSTANCE_INDEX":
				app.index = kv[1]
			case "CF_INSTANCE_INTERNAL_IP":
				app.internalIP = kv[1]
			}
		}
		apps[i] = app
	}
	return apps
}

func findTwoInstancesOnTheSameHost(apps []AppInstance) (AppInstance, AppInstance) {
	hostsToApps := map[string]AppInstance{}

	for _, app := range apps {
		foundApp, ok := hostsToApps[app.hostIdentifier]
		if ok {
			return foundApp, app
		}
		hostsToApps[app.hostIdentifier] = app
	}

	Expect(errors.New("failed to find two instances on the same host")).ToNot(HaveOccurred())
	return AppInstance{}, AppInstance{}
}

func findTwoInstancesOnDifferentHosts(apps []AppInstance) (AppInstance, AppInstance) {
	for _, app := range apps[1:] {
		if apps[0].hostIdentifier != app.hostIdentifier {
			return apps[0], app
		}
	}

	Expect(errors.New("failed to find two instances on different hosts")).ToNot(HaveOccurred())
	return AppInstance{}, AppInstance{}
}
<commit_msg>Check if all instances running after push<commit_after>package acceptance_test

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"code.cloudfoundry.org/cf-networking-helpers/testsupport"

	pusherConfig "cf-pusher/config"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	. "github.com/onsi/ginkgo"
	ginkgoConfig "github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

const Timeout_Push = 2 * time.Minute

var (
	appsDir    string
	config     helpers.Config
	testConfig pusherConfig.Config
)

func TestAcceptance(t *testing.T) {
	RegisterFailHandler(Fail)

	BeforeSuite(func() {
		config = helpers.LoadConfig()

		configPath := helpers.ConfigPath()
		configBytes, err := ioutil.ReadFile(configPath)
		Expect(err).NotTo(HaveOccurred())

		err = json.Unmarshal(configBytes, &testConfig)
		Expect(err).NotTo(HaveOccurred())

		if testConfig.Applications <= 0 {
			Fail("Applications count needs to be greater than 0")
		}

		if testConfig.AppInstances <= 0 {
			Fail("AppInstances count needs to be greater than 0")
		}

		if testConfig.ProxyApplications <= 0 {
			Fail("ProxyApplications count needs to be greater than 0")
		}

		if testConfig.ProxyInstances <= 0 {
			Fail("ProxyInstances count needs to be greater than 0")
		}

		Expect(cf.Cf("api", "--skip-ssl-validation", config.ApiEndpoint).Wait(Timeout_Short)).To(gexec.Exit(0))
		AuthAsAdmin()

		appsDir = os.Getenv("APPS_DIR")
		Expect(appsDir).NotTo(BeEmpty())

		rand.Seed(ginkgoConfig.GinkgoConfig.RandomSeed + int64(GinkgoParallelNode()))
	})

	RunSpecs(t, "Acceptance Suite")
}

func Auth(username, password string) {
	By("authenticating as " + username)
	cmd := exec.Command("cf", "auth", username, password)
	sess, err := gexec.Start(cmd, nil, nil)
	Expect(err).NotTo(HaveOccurred())
	Eventually(sess.Wait(Timeout_Short)).Should(gexec.Exit(0))
}

func getUAABaseURL() string {
	sess := cf.Cf("curl", "/v2/info")
	Eventually(sess.Wait(Timeout_Short)).Should(gexec.Exit(0))
	var response struct {
		TokenEndpoint string `json:"token_endpoint"`
	}
	err := json.Unmarshal(sess.Out.Contents(), &response)
	Expect(err).NotTo(HaveOccurred())

	uaaBaseURL := response.TokenEndpoint
	Expect(uaaBaseURL).To(HavePrefix("https://uaa."))
	return uaaBaseURL
}

func AuthAsAdmin() {
	Auth(config.AdminUser, config.AdminPassword)
}

func appDir(appType string) string {
	return filepath.Join(appsDir, appType)
}

func pushProxy(appName string) {
	Expect(cf.Cf(
		"push", appName,
		"-p", appDir("proxy"),
		"-f", defaultManifest("proxy"),
	).Wait(Timeout_Push)).To(gexec.Exit(0))
}

func defaultManifest(appType string) string {
	return filepath.Join(appDir(appType), "manifest.yml")
}

func scaleApps(apps []string, instances int) {
	parallelRunner := &testsupport.ParallelRunner{
		NumWorkers: 16,
	}
	parallelRunner.RunOnSliceStrings(apps, func(app string) {
		scaleApp(app, instances)
	})
}

func scaleApp(appName string, instances int) {
	Expect(cf.Cf(
		"scale", appName,
		"-i", fmt.Sprintf("%d", instances),
	).Wait(Timeout_Short)).To(gexec.Exit(0))
}

func pushAppWithInstanceCount(appName string, appCount int) {
	Expect(cf.Cf(
		"push", appName,
		"-p", appDir("proxy"),
		"-i", fmt.Sprintf("%d", appCount),
		"-f", defaultManifest("proxy"),
	).Wait(Timeout_Push)).To(gexec.Exit(0))

	appGuidSession := cf.Cf("app", appName, "--guid")
	Expect(appGuidSession.Wait(Timeout_Short)).To(gexec.Exit(0))

	capiURL := fmt.Sprintf("v2/apps/%s/instances", strings.TrimSpace(string(appGuidSession.Out.Contents())))

	type instanceInfo struct {
		State string `json:"state"`
	}

	instances := make(map[string]instanceInfo, appCount)

	allInstancesRunning := func() bool {
		session := cf.Cf("curl", capiURL)
		Expect(session.Wait(Timeout_Short)).To(gexec.Exit(0))

		json.Unmarshal(session.Out.Contents(), &instances)
		Expect(len(instances)).To(Equal(appCount))

		for _, instance := range instances {
			if instance.State != "RUNNING" {
				return false
			}
		}
		return true
	}

	Eventually(allInstancesRunning).Should(Equal(true), "not all instances running")

}

func restage(appName string) {
	Expect(cf.Cf(
		"restage", appName,
	).Wait(Timeout_Push)).To(gexec.Exit(0))
}

type AppInstance struct {
	hostIdentifier string
	index          string
	internalIP     string
}

func getAppInstances(appName string, instances int) []AppInstance {
	apps := make([]AppInstance, instances)
	for i := 0; i < instances; i++ {
		session := cf.Cf("ssh", appName, "-i", fmt.Sprintf("%d", i), "-c", "env | grep CF_INSTANCE")
		Expect(session.Wait(Timeout_Push)).To(gexec.Exit(0))

		env := strings.Split(string(session.Out.Contents()), "\n")
		var app AppInstance
		for _, envVar := range env {
			kv := strings.Split(envVar, "=")
			switch kv[0] {
			case "CF_INSTANCE_IP":
				app.hostIdentifier = kv[1]
			case "CF_INSTANCE_INDEX":
				app.index = kv[1]
			case "CF_INSTANCE_INTERNAL_IP":
				app.internalIP = kv[1]
			}
		}
		apps[i] = app
	}
	return apps
}

func findTwoInstancesOnTheSameHost(apps []AppInstance) (AppInstance, AppInstance) {
	hostsToApps := map[string]AppInstance{}

	for _, app := range apps {
		foundApp, ok := hostsToApps[app.hostIdentifier]
		if ok {
			return foundApp, app
		}
		hostsToApps[app.hostIdentifier] = app
	}

	Expect(errors.New("failed to find two instances on the same host")).ToNot(HaveOccurred())
	return AppInstance{}, AppInstance{}
}

func findTwoInstancesOnDifferentHosts(apps []AppInstance) (AppInstance, AppInstance) {
	for _, app := range apps[1:] {
		if apps[0].hostIdentifier != app.hostIdentifier {
			return apps[0], app
		}
	}

	Expect(errors.New("failed to find two instances on different hosts")).ToNot(HaveOccurred())
	return AppInstance{}, AppInstance{}
}
<|endoftext|>


<commit_before>// Package play implements a simple mp3 player.
package play

/*
#include <stdlib.h>
#include "play.h"

#cgo LDFLAGS: -lmpg123 -lao -lasound -lid3
*/
import "C"

import (
	"errors"
	"fmt"
	"strings"
	"time"
	"unsafe"
)

// For development
var _ = fmt.Println

// Package init function
func init() {
	C.play_init()
}

//Play the specified file. Return when playback is complete.
func Play(filename string) {
	n := C.CString(filename)
	C.play_play(n)
	C.free(unsafe.Pointer(n))
}

// Set the volume as a percentage between 0 and 100 inclusive.
func SetVolume(pct byte) {
	C.play_setvolume(C.uchar(pct))
}

// Get the volume as a percentage between 0 and 100 inclusive.
func GetVolume() byte {
	return byte(C.play_getvolume())
}

// Metadata is information about an mp3.
type Metadata struct {
	Title  string
	Artist string
	Album  string
}

// GetMetadata extracts the id3 information from the mp3 file `filename`.
func GetMetadata(filename string) Metadata {
	meta := C.play_meta(C.CString(filename))
	r := Metadata{
		Title:  strings.Trim(C.GoString(meta.title), " "),
		Artist: strings.Trim(C.GoString(meta.artist), " "),
		Album:  strings.Trim(C.GoString(meta.album), " "),
	}
	C.play_delete_meta(meta)
	return r
}

// Print debugging information about the metadata in `filename` to stdout.
func DebugMetadata(filename string) {
	C.play_debug_meta(C.CString(filename))
}

const (
	// The player has no mp3 loaded
	Empty = iota
	// The player has an mp3 loaded and is playing it
	Playing
	// The player has an mp3 loaded and it is paused
	Paused
)

// PlayerState represents the current state of a Player. Must be one of Empty, Playing, or Paused.
type PlayerState int

// Player is an mp3 player.
type Player chan command

const (
	cmdLoad = iota
	cmdPlay
	cmdPause
	cmdStop
	cmdSeek
)

type command struct {
	id     int
	path   string
	offset int
	err    chan error
	// The player writes offsets to this channel periodically, if it is not nil
	offchan chan int
	// on load, this channel is written the size of the mp3 in samples
	size chan int
}

// Create a new Player.
func NewPlayer() Player {

	p := make(chan command)

	go func() {
		var state PlayerState = Empty
		var reader *C.play_reader_t
		var writer *C.ao_device
		var offchan chan int
		var lastofftime time.Time
		lastoff := -1

		stop := func() {
			if state != Empty {
				C.play_delete_reader(reader)
				reader = nil
				C.play_delete_writer(writer)
				writer = nil

				if offchan != nil {
					close(offchan)
				}

				state = Empty
			}
		}

		load := func(cmd command) {
			if state != Empty {
				stop()
			}

			lastoff = -1

			reader = C.play_new_reader(C.CString(cmd.path))
			if reader == nil {
				close(cmd.size)
				cmd.err <- errors.New("Creating reader failed")
				return
			}

			writer = C.play_new_writer(reader)
			if writer == nil {
				close(cmd.size)
				cmd.err <- errors.New("Creating writer failed")
				C.play_delete_reader(reader)
				reader = nil
				return
			}

			cmd.size <- int(C.play_length(reader))
			offchan = cmd.offchan

			cmd.err <- nil
			state = Paused
		}

		play := func() {
			if state != Empty {
				state = Playing
			}
		}

		pause := func() {
			if state != Empty {
				state = Paused
			}
		}

		seek := func(cmd command) {
			if state == Empty {
				return
			}

			C.play_seek(reader, C.int(cmd.offset))
			// Zero out lastofftime
			var zero time.Time
			lastofftime = zero
		}

		for {
			// Check for commands
			select {
			case cmd := <-p:
				switch cmd.id {
				case cmdLoad:
					load(cmd)
				case cmdPlay:
					play()
				case cmdPause:
					pause()
				case cmdStop:
					stop()
				case cmdSeek:
					seek(cmd)
				}
			default:
			}

			// If we are playing, copy a buffer of data to the output device
			if state == Playing {
				n, err := C.play_read(reader)
				if err != nil {
					// We're done
					C.play_delete_reader(reader)
					C.play_delete_writer(writer)
					if offchan != nil {
						close(offchan)
					}
					state = Empty
					continue
				}

				C.play_write(writer, reader.buffer, n)

				if offchan != nil {
					if lastofftime.IsZero() || time.Now().Sub(lastofftime) > time.Millisecond*100 {
						if o := int(C.play_offset(reader)); o != lastoff {
							select {
							case offchan <- o:
								lastofftime = time.Now()
							default:
							}
						}
					}
				}
			}

		}
	}()

	return p
}

// Load the specified file into the Player. Call Play to play the file.
// Returns the size of the file in samples. On failure, err is non nil.
// On success, the player will write the current playing offset (in samples) to offchan
// periodically.
func (p Player) Load(filename string, offchan chan int) (size int, err error) {
	ch := make(chan error)
	sizech := make(chan int)

	p <- command{id: cmdLoad, path: filename, err: ch, offchan: offchan, size: sizech}

	size = <-sizech
	err = <-ch

	return
}

// Play the currently loaded file. The player must have an mp3 loaded and not currently playing.
func (p Player) Play() {
	p <- command{id: cmdPlay}
}

// Pause the currently playing file.
func (p Player) Pause() {
	p <- command{id: cmdPause}
}

// Stop the currently playing file if it's playing, and unload the file from the mp3 player.
func (p Player) Stop() {
	p <- command{id: cmdStop}
}

// Seek to the specified sample in the loaded mp3 file.
func (p Player) Seek(offset int) {
	p <- command{id: cmdSeek, offset: offset}
}
<commit_msg>Refactored Player goroutine into clear state machine<commit_after>// Package play implements a simple mp3 player.
package play

/*
#include <stdlib.h>
#include "play.h"

#cgo LDFLAGS: -lmpg123 -lao -lasound -lid3
*/
import "C"

import (
	"errors"
	"fmt"
	"strings"
	"time"
	"unsafe"
)

// For development
var _ = fmt.Println

// Package init function
func init() {
	C.play_init()
}

// Play the specified file. Return when playback is complete.
func Play(filename string) {
	n := C.CString(filename)
	C.play_play(n)
	C.free(unsafe.Pointer(n))
}

// Set the volume as a percentage between 0 and 100 inclusive.
func SetVolume(pct byte) {
	C.play_setvolume(C.uchar(pct))
}

// Get the volume as a percentage between 0 and 100 inclusive.
func GetVolume() byte {
	return byte(C.play_getvolume())
}

// Metadata is information about an mp3.
type Metadata struct {
	Title  string
	Artist string
	Album  string
}

// GetMetadata extracts the id3 information from the mp3 file `filename`.
func GetMetadata(filename string) Metadata {
	meta := C.play_meta(C.CString(filename))
	r := Metadata{
		Title:  strings.Trim(C.GoString(meta.title), " "),
		Artist: strings.Trim(C.GoString(meta.artist), " "),
		Album:  strings.Trim(C.GoString(meta.album), " "),
	}
	C.play_delete_meta(meta)
	return r
}

// Print debugging information about the metadata in `filename` to stdout.
func DebugMetadata(filename string) {
	C.play_debug_meta(C.CString(filename))
}

const (
	// The player has no mp3 loaded
	Empty = iota
	// The player has an mp3 loaded and is playing it
	Playing
	// The player has an mp3 loaded and it is paused
	Paused
)

// PlayerState represents the current state of a Player. Must be one of Empty, Playing, or Paused.
type PlayerState int

// Player is an mp3 player.
type Player chan command

const (
	cmdLoad = iota
	cmdPlay
	cmdPause
	cmdStop
	cmdSeek
)

type command struct {
	id     int
	path   string
	offset int
	err    chan error
	// The player writes offsets to this channel periodically, if it is not nil
	offchan chan int
	// on load, this channel is written the size of the mp3 in samples
	size chan int
}

// Create a new Player.
func NewPlayer() Player {

	p := make(chan command)

  // This goroutine implements the player.
	go func() {
		var reader *C.play_reader_t
		var writer *C.ao_device
		var offchan chan int
		var lastofftime time.Time
		lastoff := -1

		var state PlayerState = Empty
    var states [3]func()

    // Stop the currently playing mp3 if it's playing, and unload it.
		stop := func() {
			if state != Empty {
				C.play_delete_reader(reader)
				reader = nil
				C.play_delete_writer(writer)
				writer = nil

				if offchan != nil {
					close(offchan)
				}

				state = Empty
			}
		}

    // Load a new mp3. Returns true if the state has changed.
		load := func(cmd command) {
			if state != Empty {
				stop()
			}

			lastoff = -1

			reader = C.play_new_reader(C.CString(cmd.path))
			if reader == nil {
				close(cmd.size)
				cmd.err <- errors.New("Creating reader failed")
				return
			}

			writer = C.play_new_writer(reader)
			if writer == nil {
				close(cmd.size)
				cmd.err <- errors.New("Creating writer failed")
				C.play_delete_reader(reader)
				reader = nil
				return
			}

			cmd.size <- int(C.play_length(reader))
			offchan = cmd.offchan

			cmd.err <- nil
			state = Paused
		}

    // Play the loaded mp3.
		play := func() {
			if state != Empty {
				state = Playing
			}
		}

    // Pause the playing mp3.
		pause := func() {
			if state != Empty {
				state = Paused
			}
		}

    // Seek to a position in the loaded mp3.
		seek := func(cmd command) {
			if state == Empty {
				return
			}

			C.play_seek(reader, C.int(cmd.offset))
			// Zero out lastofftime
			var zero time.Time
			lastofftime = zero
		}

    states[Empty] = func(){
      // Only the load command is not ignored.
      for {
			  select {
        case cmd := <-p:
          switch cmd.id {
          case cmdLoad:
					  load(cmd)
          }
        }

        if state != Empty {
          break
        }
      }
    }

    states[Paused] = func(){
      for {
			  select {
        case cmd := <-p:
          switch cmd.id {
          case cmdLoad:
					  load(cmd)
          case cmdPlay:
            play()
          case cmdStop:
            stop()
          case cmdSeek:
            seek(cmd)
          }
        }

        if state != Paused {
          break
        }
      }
    }

    states[Playing] = func(){
      for {
			  select {
        case cmd := <-p:
          switch cmd.id {
          case cmdLoad:
					  load(cmd)
				  case cmdPause:
					  pause()
          case cmdStop:
            stop()
          case cmdSeek:
            seek(cmd)
          }
          default:
        }

        if state != Playing {
          break
        }

        // Copy a buffer of data to the output device
        n, err := C.play_read(reader)
        if err != nil {
          // We're done
          stop()
          break
        }

        C.play_write(writer, reader.buffer, n)

        if offchan != nil {
          if lastofftime.IsZero() || time.Now().Sub(lastofftime) > time.Millisecond*100 {
            if o := int(C.play_offset(reader)); o != lastoff {
              select {
              case offchan <- o:
                lastofftime = time.Now()
              default:
              }
            }
          }
        }
      }
    }

    // Player state machine:
		for {
      states[state]()
		}

	}()

	return p
}

// Load the specified file into the Player. Call Play to play the file.
// Returns the size of the file in samples. On failure, err is non nil.
// On success, the player will write the current playing offset (in samples) to offchan
// periodically.
func (p Player) Load(filename string, offchan chan int) (size int, err error) {
	ch := make(chan error)
	sizech := make(chan int)

	p <- command{id: cmdLoad, path: filename, err: ch, offchan: offchan, size: sizech}

	size = <-sizech
	err = <-ch

	return
}

// Play the currently loaded file. The player must have an mp3 loaded and not currently playing.
func (p Player) Play() {
	p <- command{id: cmdPlay}
}

// Pause the currently playing file.
func (p Player) Pause() {
	p <- command{id: cmdPause}
}

// Stop the currently playing file if it's playing, and unload the file from the mp3 player.
func (p Player) Stop() {
	p <- command{id: cmdStop}
}

// Seek to the specified sample in the loaded mp3 file.
func (p Player) Seek(offset int) {
	p <- command{id: cmdSeek, offset: offset}
}
<|endoftext|>