<commit_before>// Copyright (c) 2016 ECS Team, Inc. - All Rights Reserved
// https://github.com/ECSTeam/cloudfoundry-top-plugin
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dataCommon

import (
	"strconv"
	"time"

	"github.com/ecsteam/cloudfoundry-top-plugin/config"
	"github.com/ecsteam/cloudfoundry-top-plugin/eventrouting"
	"github.com/ecsteam/cloudfoundry-top-plugin/metadata/app"
	"github.com/ecsteam/cloudfoundry-top-plugin/metadata/crashData"
)

type CommonData struct {
	//masterUI       masterUIInterface.MasterUIInterface
	router *eventrouting.EventRouter
	//eventProcessor *eventdata.EventProcessor
	appMdMgr *app.AppMetadataManager

	displayAppStatsMap map[string]*DisplayAppStats
	monitoredAppGuids  map[string]bool

	isWarmupComplete bool
	// This is a count of the number of apps that do not have
	// the correct number of containers running based on app
	// instance setting
	appsNotInDesiredState int
	totalCrash1hCount     int
	totalCrash24hCount    int
}

// TODO:  Create a common data struct -- which needs access to masterUI
// to get GetDisplayedEventData and GetAppMdMgr.  Also will allow
// appView to access this data through  masterUI so we don't process
// the same data twice

func NewCommonData(router *eventrouting.EventRouter, monitoredAppGuids map[string]bool) *CommonData {
	cd := &CommonData{router: router}

	cd.appMdMgr = router.GetProcessor().GetMetadataManager().GetAppMdManager()
	cd.monitoredAppGuids = monitoredAppGuids
	return cd
}

func (cd *CommonData) GetDisplayAppStatsMap() map[string]*DisplayAppStats {
	return cd.displayAppStatsMap
}

func (cd *CommonData) IsWarmupComplete() bool {
	return cd.isWarmupComplete
}

func (cd *CommonData) AppsNotInDesiredState() int {
	return cd.appsNotInDesiredState
}

func (cd *CommonData) TotalCrash1hCount() int {
	return cd.totalCrash1hCount
}

func (cd *CommonData) TotalCrash24hCount() int {
	return cd.totalCrash24hCount
}

func (cd *CommonData) SetMonitoredAppGuids(monitoredAppGuids map[string]bool) {
	cd.monitoredAppGuids = monitoredAppGuids
}

func (cd *CommonData) GetMonitoredAppGuids() map[string]bool {
	return cd.monitoredAppGuids
}

func (cd *CommonData) IsMonitoredAppGuid(appGuid string) bool {
	// Check if we're only monitoring a subset of apps and if this app is one of them
	if cd.monitoredAppGuids != nil {
		if cd.monitoredAppGuids[appGuid] {
			return true
		}
		return false
	}
	return true
}

func (cd *CommonData) PostProcessData() map[string]*DisplayAppStats {

	eventData := cd.router.GetProcessor().GetDisplayedEventData()
	statsTime := eventData.StatsTime
	runtimeSeconds := statsTime.Sub(cd.router.GetStartTime())
	cd.isWarmupComplete = runtimeSeconds > time.Second*config.WarmUpSeconds

	mdMgr := cd.router.GetProcessor().GetMetadataManager()
	displayStatsMap := make(map[string]*DisplayAppStats)

	appMap := cd.router.GetProcessor().GetDisplayedEventData().AppMap
	appsNotInDesiredState := 0
	totalCrash1hCount := 0
	totalCrash24hCount := 0

	for appId, appStats := range appMap {

		displayAppStats := NewDisplayAppStats(appStats)
		displayAppStats.Monitored = cd.IsMonitoredAppGuid(appId)

		displayStatsMap[appId] = displayAppStats
		appMetadata := cd.appMdMgr.FindItem(appStats.AppId)

		displayAppStats.AppNameForSort = appMetadata.Name
		displayAppStats.AppName = appMetadata.Name

		if appMetadata.PackageState == "PENDING" {
			displayAppStats.IsPackageStatePending = true
			displayAppStats.AppName = "[" + displayAppStats.AppName + "]"
		}
		if mdMgr.GetAppMdManager().IsPendingDeleteFromCache(appId) {
			displayAppStats.IsDeleted = true
			displayAppStats.AppName = "(" + displayAppStats.AppName + ")"
		}
		if mdMgr.IsMonitorAppDetails(appId) {
			displayAppStats.AppName += "*"
		}
		displayAppStats.SpaceId = appMetadata.SpaceGuid
		spaceMetadata := mdMgr.GetSpaceMdManager().FindItem(appMetadata.SpaceGuid)
		displayAppStats.SpaceName = spaceMetadata.Name

		orgMd := mdMgr.GetOrgMdManager().FindItem(spaceMetadata.OrgGuid)
		displayAppStats.OrgId = orgMd.GetGuid()
		displayAppStats.OrgName = orgMd.GetName()

		totalCpuPercentage := 0.0
		totalMemoryUsed := int64(0)
		totalDiskUsed := int64(0)
		totalReportingContainers := 0

		if appMetadata.State == "STARTED" {
			displayAppStats.DesiredContainers = int(appMetadata.Instances)
		}

		stack := mdMgr.GetStackMdManager().FindItem(appMetadata.StackGuid)
		displayAppStats.StackId = appMetadata.StackGuid
		displayAppStats.StackName = stack.Name

		isoSeg := mdMgr.GetIsoSegMdManager().FindItem(spaceMetadata.IsolationSegmentGuid)
		displayAppStats.IsolationSegmentGuid = isoSeg.Guid
		displayAppStats.IsolationSegmentName = isoSeg.Name

		// Crash count in last 1 hour (from call to /v2/events)
		crash1hCount := crashData.FindCountSinceByApp(appId, -1*time.Hour)
		crash1hCount = crash1hCount + appStats.Crash1hCount()

		// Crash count in last 24 hours (from call to /v2/events)
		crash24hCount := crashData.FindCountSinceByApp(appId, -24*time.Hour)
		crash24hCount = crash24hCount + appStats.Crash24hCount()

		for _, containerTraffic := range appStats.ContainerTrafficMap {
			for _, httpStatusCodeMap := range containerTraffic.HttpInfoMap {
				for statusCode, httpCountInfo := range httpStatusCodeMap {
					if httpCountInfo != nil {
						displayAppStats.HttpAllCount += httpCountInfo.HttpCount
						switch {
						case statusCode >= 200 && statusCode < 300:
							displayAppStats.Http2xxCount += httpCountInfo.HttpCount
						case statusCode >= 300 && statusCode < 400:
							displayAppStats.Http3xxCount += httpCountInfo.HttpCount
						case statusCode >= 400 && statusCode < 500:
							displayAppStats.Http4xxCount += httpCountInfo.HttpCount
						case statusCode >= 500 && statusCode < 600:
							displayAppStats.Http5xxCount += httpCountInfo.HttpCount
						}
					}
				}
			}
		}

		for _, cs := range appStats.ContainerArray {
			if cs != nil {
				appInsts := mdMgr.GetAppInstMdManager().FindItem(appId)
				if appInsts != nil {

					// If we have app instance metadata, lets check if the app is in a good state
					appInst := appInsts.Data[strconv.Itoa(cs.ContainerIndex)]
					if appInst == nil || appInst.State == "DOWN" || appInst.State == "CRASHED" {
						continue
					}
				}
				if cs.ContainerMetric != nil {
					totalReportingContainers++

					totalCpuPercentage = totalCpuPercentage + *cs.ContainerMetric.CpuPercentage
					totalMemoryUsed = totalMemoryUsed + int64(*cs.ContainerMetric.MemoryBytes)
					totalDiskUsed = totalDiskUsed + int64(*cs.ContainerMetric.DiskBytes)
				}
			}
		}

		now := time.Now()
		if displayAppStats.Monitored && appMetadata.State == "STARTED" && appMetadata.PackageState == "STAGED" {
			cacheTime := appMetadata.GetCacheTime()
			startedDuration := now.Sub(*cacheTime)
			if startedDuration > (config.AppNotInDesiredStateWaitTimeSeconds*time.Second) && totalReportingContainers < displayAppStats.DesiredContainers {
				appsNotInDesiredState = appsNotInDesiredState + 1
				displayAppStats.AppNotInDesiredState = true
			}
		}

		if totalReportingContainers > 0 {
			displayAppStats.TotalCpuPercentage = totalCpuPercentage
		} else {
			// In PCF 1.9 running containers can report 0.00 CPU percent usage
			// To help distiquish between a container with 0 CPU and no container
			// at all we set this to a very small negative number to help sort
			// no-container apps to the bottom when sorting by CPU%
			displayAppStats.TotalCpuPercentage = -0.0001
		}
		displayAppStats.TotalMemoryUsed = totalMemoryUsed
		displayAppStats.TotalDiskUsed = totalDiskUsed
		displayAppStats.TotalReportingContainers = totalReportingContainers
		displayAppStats.Crash1hCount = crash1hCount
		displayAppStats.Crash24hCount = crash24hCount
		totalCrash1hCount = totalCrash1hCount + crash1hCount
		totalCrash24hCount = totalCrash24hCount + crash24hCount
		/*
			logStdoutCount := int64(0)
			logStderrCount := int64(0)
			for _, cs := range appStats.ContainerArray {
				if cs != nil {
					logStdoutCount = logStdoutCount + cs.OutCount
					logStderrCount = logStderrCount + cs.ErrCount
				}
			}
			displayAppStats.TotalLogStdout = logStdoutCount + appStats.NonContainerStdout
			displayAppStats.TotalLogStderr = logStderrCount + appStats.NonContainerStderr
		*/
	}

	cd.displayAppStatsMap = displayStatsMap
	cd.appsNotInDesiredState = appsNotInDesiredState
	cd.totalCrash1hCount = totalCrash1hCount
	cd.totalCrash24hCount = totalCrash24hCount
	return displayStatsMap
}
<commit_msg>Fixed issue with negative CPU percent<commit_after>// Copyright (c) 2016 ECS Team, Inc. - All Rights Reserved
// https://github.com/ECSTeam/cloudfoundry-top-plugin
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dataCommon

import (
	"strconv"
	"time"

	"github.com/ecsteam/cloudfoundry-top-plugin/config"
	"github.com/ecsteam/cloudfoundry-top-plugin/eventrouting"
	"github.com/ecsteam/cloudfoundry-top-plugin/metadata/app"
	"github.com/ecsteam/cloudfoundry-top-plugin/metadata/crashData"
)

type CommonData struct {
	//masterUI       masterUIInterface.MasterUIInterface
	router *eventrouting.EventRouter
	//eventProcessor *eventdata.EventProcessor
	appMdMgr *app.AppMetadataManager

	displayAppStatsMap map[string]*DisplayAppStats
	monitoredAppGuids  map[string]bool

	isWarmupComplete bool
	// This is a count of the number of apps that do not have
	// the correct number of containers running based on app
	// instance setting
	appsNotInDesiredState int
	totalCrash1hCount     int
	totalCrash24hCount    int
}

// TODO:  Create a common data struct -- which needs access to masterUI
// to get GetDisplayedEventData and GetAppMdMgr.  Also will allow
// appView to access this data through  masterUI so we don't process
// the same data twice

func NewCommonData(router *eventrouting.EventRouter, monitoredAppGuids map[string]bool) *CommonData {
	cd := &CommonData{router: router}

	cd.appMdMgr = router.GetProcessor().GetMetadataManager().GetAppMdManager()
	cd.monitoredAppGuids = monitoredAppGuids
	return cd
}

func (cd *CommonData) GetDisplayAppStatsMap() map[string]*DisplayAppStats {
	return cd.displayAppStatsMap
}

func (cd *CommonData) IsWarmupComplete() bool {
	return cd.isWarmupComplete
}

func (cd *CommonData) AppsNotInDesiredState() int {
	return cd.appsNotInDesiredState
}

func (cd *CommonData) TotalCrash1hCount() int {
	return cd.totalCrash1hCount
}

func (cd *CommonData) TotalCrash24hCount() int {
	return cd.totalCrash24hCount
}

func (cd *CommonData) SetMonitoredAppGuids(monitoredAppGuids map[string]bool) {
	cd.monitoredAppGuids = monitoredAppGuids
}

func (cd *CommonData) GetMonitoredAppGuids() map[string]bool {
	return cd.monitoredAppGuids
}

func (cd *CommonData) IsMonitoredAppGuid(appGuid string) bool {
	// Check if we're only monitoring a subset of apps and if this app is one of them
	if cd.monitoredAppGuids != nil {
		if cd.monitoredAppGuids[appGuid] {
			return true
		}
		return false
	}
	return true
}

func (cd *CommonData) PostProcessData() map[string]*DisplayAppStats {

	eventData := cd.router.GetProcessor().GetDisplayedEventData()
	statsTime := eventData.StatsTime
	runtimeSeconds := statsTime.Sub(cd.router.GetStartTime())
	cd.isWarmupComplete = runtimeSeconds > time.Second*config.WarmUpSeconds

	mdMgr := cd.router.GetProcessor().GetMetadataManager()
	displayStatsMap := make(map[string]*DisplayAppStats)

	appMap := cd.router.GetProcessor().GetDisplayedEventData().AppMap
	appsNotInDesiredState := 0
	totalCrash1hCount := 0
	totalCrash24hCount := 0

	for appId, appStats := range appMap {

		displayAppStats := NewDisplayAppStats(appStats)
		displayAppStats.Monitored = cd.IsMonitoredAppGuid(appId)

		displayStatsMap[appId] = displayAppStats
		appMetadata := cd.appMdMgr.FindItem(appStats.AppId)

		displayAppStats.AppNameForSort = appMetadata.Name
		displayAppStats.AppName = appMetadata.Name

		if appMetadata.PackageState == "PENDING" {
			displayAppStats.IsPackageStatePending = true
			displayAppStats.AppName = "[" + displayAppStats.AppName + "]"
		}
		if mdMgr.GetAppMdManager().IsPendingDeleteFromCache(appId) {
			displayAppStats.IsDeleted = true
			displayAppStats.AppName = "(" + displayAppStats.AppName + ")"
		}
		if mdMgr.IsMonitorAppDetails(appId) {
			displayAppStats.AppName += "*"
		}
		displayAppStats.SpaceId = appMetadata.SpaceGuid
		spaceMetadata := mdMgr.GetSpaceMdManager().FindItem(appMetadata.SpaceGuid)
		displayAppStats.SpaceName = spaceMetadata.Name

		orgMd := mdMgr.GetOrgMdManager().FindItem(spaceMetadata.OrgGuid)
		displayAppStats.OrgId = orgMd.GetGuid()
		displayAppStats.OrgName = orgMd.GetName()

		totalCpuPercentage := 0.0
		totalMemoryUsed := int64(0)
		totalDiskUsed := int64(0)
		totalReportingContainers := 0

		if appMetadata.State == "STARTED" {
			displayAppStats.DesiredContainers = int(appMetadata.Instances)
		}

		stack := mdMgr.GetStackMdManager().FindItem(appMetadata.StackGuid)
		displayAppStats.StackId = appMetadata.StackGuid
		displayAppStats.StackName = stack.Name

		isoSeg := mdMgr.GetIsoSegMdManager().FindItem(spaceMetadata.IsolationSegmentGuid)
		displayAppStats.IsolationSegmentGuid = isoSeg.Guid
		displayAppStats.IsolationSegmentName = isoSeg.Name

		// Crash count in last 1 hour (from call to /v2/events)
		crash1hCount := crashData.FindCountSinceByApp(appId, -1*time.Hour)
		crash1hCount = crash1hCount + appStats.Crash1hCount()

		// Crash count in last 24 hours (from call to /v2/events)
		crash24hCount := crashData.FindCountSinceByApp(appId, -24*time.Hour)
		crash24hCount = crash24hCount + appStats.Crash24hCount()

		for _, containerTraffic := range appStats.ContainerTrafficMap {
			for _, httpStatusCodeMap := range containerTraffic.HttpInfoMap {
				for statusCode, httpCountInfo := range httpStatusCodeMap {
					if httpCountInfo != nil {
						displayAppStats.HttpAllCount += httpCountInfo.HttpCount
						switch {
						case statusCode >= 200 && statusCode < 300:
							displayAppStats.Http2xxCount += httpCountInfo.HttpCount
						case statusCode >= 300 && statusCode < 400:
							displayAppStats.Http3xxCount += httpCountInfo.HttpCount
						case statusCode >= 400 && statusCode < 500:
							displayAppStats.Http4xxCount += httpCountInfo.HttpCount
						case statusCode >= 500 && statusCode < 600:
							displayAppStats.Http5xxCount += httpCountInfo.HttpCount
						}
					}
				}
			}
		}

		for _, cs := range appStats.ContainerArray {
			if cs != nil {
				appInsts := mdMgr.GetAppInstMdManager().FindItem(appId)
				if appInsts != nil {

					// If we have app instance metadata, lets check if the app is in a good state
					appInst := appInsts.Data[strconv.Itoa(cs.ContainerIndex)]
					if appInst == nil || appInst.State == "DOWN" || appInst.State == "CRASHED" {
						continue
					}
				}
				if cs.ContainerMetric != nil {
					totalReportingContainers++

					totalCpuPercentage = totalCpuPercentage + *cs.ContainerMetric.CpuPercentage
					totalMemoryUsed = totalMemoryUsed + int64(*cs.ContainerMetric.MemoryBytes)
					totalDiskUsed = totalDiskUsed + int64(*cs.ContainerMetric.DiskBytes)
				}
			}
		}

		now := time.Now()
		if displayAppStats.Monitored && appMetadata.State == "STARTED" && appMetadata.PackageState == "STAGED" {
			cacheTime := appMetadata.GetCacheTime()
			startedDuration := now.Sub(*cacheTime)
			if startedDuration > (config.AppNotInDesiredStateWaitTimeSeconds*time.Second) && totalReportingContainers < displayAppStats.DesiredContainers {
				appsNotInDesiredState = appsNotInDesiredState + 1
				displayAppStats.AppNotInDesiredState = true
			}
		}

		if totalReportingContainers > 0 {
			// In PCF 1.9 running containers can report 0.00 CPU percent usage
			// To help distiquish between a container with 0 CPU and no container
			// at all we set this to a very small number to help sort
			// no-container apps to the bottom when sorting by CPU%
			if totalCpuPercentage == 0 {
				totalCpuPercentage = 0.00001
			}
			displayAppStats.TotalCpuPercentage = totalCpuPercentage
		}
		displayAppStats.TotalMemoryUsed = totalMemoryUsed
		displayAppStats.TotalDiskUsed = totalDiskUsed
		displayAppStats.TotalReportingContainers = totalReportingContainers
		displayAppStats.Crash1hCount = crash1hCount
		displayAppStats.Crash24hCount = crash24hCount
		totalCrash1hCount = totalCrash1hCount + crash1hCount
		totalCrash24hCount = totalCrash24hCount + crash24hCount
		/*
			logStdoutCount := int64(0)
			logStderrCount := int64(0)
			for _, cs := range appStats.ContainerArray {
				if cs != nil {
					logStdoutCount = logStdoutCount + cs.OutCount
					logStderrCount = logStderrCount + cs.ErrCount
				}
			}
			displayAppStats.TotalLogStdout = logStdoutCount + appStats.NonContainerStdout
			displayAppStats.TotalLogStderr = logStderrCount + appStats.NonContainerStderr
		*/
	}

	cd.displayAppStatsMap = displayStatsMap
	cd.appsNotInDesiredState = appsNotInDesiredState
	cd.totalCrash1hCount = totalCrash1hCount
	cd.totalCrash24hCount = totalCrash24hCount
	return displayStatsMap
}
<|endoftext|>


<commit_before>// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package treecache

import (
	"bytes"
	"fmt"
	"strings"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/samuel/go-zookeeper/zk"
)

var (
	failureCounter = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "prometheus",
		Subsystem: "treecache",
		Name:      "zookeeper_failures_total",
		Help:      "The total number of ZooKeeper failures.",
	})
	numWatchers = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "prometheus",
		Subsystem: "treecache",
		Name:      "watcher_goroutines",
		Help:      "The current number of watcher goroutines.",
	})
)

func init() {
	prometheus.MustRegister(failureCounter)
	prometheus.MustRegister(numWatchers)
}

type ZookeeperLogger struct {
}

// Implements zk.Logger
func (zl ZookeeperLogger) Printf(s string, i ...interface{}) {
	log.Infof(s, i...)
}

type ZookeeperTreeCache struct {
	conn     *zk.Conn
	prefix   string
	events   chan ZookeeperTreeCacheEvent
	zkEvents chan zk.Event
	stop     chan struct{}
	head     *zookeeperTreeCacheNode
}

type ZookeeperTreeCacheEvent struct {
	Path string
	Data *[]byte
}

type zookeeperTreeCacheNode struct {
	data     *[]byte
	events   chan zk.Event
	done     chan struct{}
	stopped  bool
	children map[string]*zookeeperTreeCacheNode
}

func NewZookeeperTreeCache(conn *zk.Conn, path string, events chan ZookeeperTreeCacheEvent) *ZookeeperTreeCache {
	tc := &ZookeeperTreeCache{
		conn:   conn,
		prefix: path,
		events: events,
		stop:   make(chan struct{}),
	}
	tc.head = &zookeeperTreeCacheNode{
		events:   make(chan zk.Event),
		children: map[string]*zookeeperTreeCacheNode{},
		stopped:  true,
	}
	err := tc.recursiveNodeUpdate(path, tc.head)
	if err != nil {
		log.Errorf("Error during initial read of Zookeeper: %s", err)
	}
	go tc.loop(err != nil)
	return tc
}

func (tc *ZookeeperTreeCache) Stop() {
	tc.stop <- struct{}{}
}

func (tc *ZookeeperTreeCache) loop(failureMode bool) {
	retryChan := make(chan struct{})

	failure := func() {
		failureCounter.Inc()
		failureMode = true
		time.AfterFunc(time.Second*10, func() {
			retryChan <- struct{}{}
		})
	}
	if failureMode {
		failure()
	}

	for {
		select {
		case ev := <-tc.head.events:
			log.Debugf("Received Zookeeper event: %s", ev)
			if failureMode {
				continue
			}

			if ev.Type == zk.EventNotWatching {
				log.Infof("Lost connection to Zookeeper.")
				failure()
			} else {
				path := strings.TrimPrefix(ev.Path, tc.prefix)
				parts := strings.Split(path, "/")
				node := tc.head
				for _, part := range parts[1:] {
					childNode := node.children[part]
					if childNode == nil {
						childNode = &zookeeperTreeCacheNode{
							events:   tc.head.events,
							children: map[string]*zookeeperTreeCacheNode{},
							done:     make(chan struct{}, 1),
						}
						node.children[part] = childNode
					}
					node = childNode
				}

				err := tc.recursiveNodeUpdate(ev.Path, node)
				if err != nil {
					log.Errorf("Error during processing of Zookeeper event: %s", err)
					failure()
				} else if tc.head.data == nil {
					log.Errorf("Error during processing of Zookeeper event: path %s no longer exists", tc.prefix)
					failure()
				}
			}
		case <-retryChan:
			log.Infof("Attempting to resync state with Zookeeper")
			previousState := &zookeeperTreeCacheNode{
				children: tc.head.children,
			}
			// Reset root child nodes before traversing the Zookeeper path.
			tc.head.children = make(map[string]*zookeeperTreeCacheNode)

			if err := tc.recursiveNodeUpdate(tc.prefix, tc.head); err != nil {
				log.Errorf("Error during Zookeeper resync: %s", err)
				// Revert to our previous state.
				tc.head.children = previousState.children
				failure()
			} else {
				tc.resyncState(tc.prefix, tc.head, previousState)
				log.Infof("Zookeeper resync successful")
				failureMode = false
			}
		case <-tc.stop:
			close(tc.events)
			return
		}
	}
}

func (tc *ZookeeperTreeCache) recursiveNodeUpdate(path string, node *zookeeperTreeCacheNode) error {
	data, _, dataWatcher, err := tc.conn.GetW(path)
	if err == zk.ErrNoNode {
		tc.recursiveDelete(path, node)
		if node == tc.head {
			return fmt.Errorf("path %s does not exist", path)
		}
		return nil
	} else if err != nil {
		return err
	}

	if node.data == nil || !bytes.Equal(*node.data, data) {
		node.data = &data
		tc.events <- ZookeeperTreeCacheEvent{Path: path, Data: node.data}
	}

	children, _, childWatcher, err := tc.conn.ChildrenW(path)
	if err == zk.ErrNoNode {
		tc.recursiveDelete(path, node)
		return nil
	} else if err != nil {
		return err
	}

	currentChildren := map[string]struct{}{}
	for _, child := range children {
		currentChildren[child] = struct{}{}
		childNode := node.children[child]
		// Does not already exists or we previous had a watch that
		// triggered.
		if childNode == nil || childNode.stopped {
			node.children[child] = &zookeeperTreeCacheNode{
				events:   node.events,
				children: map[string]*zookeeperTreeCacheNode{},
				done:     make(chan struct{}, 1),
			}
			err = tc.recursiveNodeUpdate(path+"/"+child, node.children[child])
			if err != nil {
				return err
			}
		}
	}

	// Remove nodes that no longer exist
	for name, childNode := range node.children {
		if _, ok := currentChildren[name]; !ok || node.data == nil {
			tc.recursiveDelete(path+"/"+name, childNode)
			delete(node.children, name)
		}
	}

	go func() {
		numWatchers.Inc()
		// Pass up zookeeper events, until the node is deleted.
		select {
		case event := <-dataWatcher:
			node.events <- event
		case event := <-childWatcher:
			node.events <- event
		case <-node.done:
		}
		numWatchers.Dec()
	}()
	return nil
}

func (tc *ZookeeperTreeCache) resyncState(path string, currentState, previousState *zookeeperTreeCacheNode) {
	for child, previousNode := range previousState.children {
		if currentNode, present := currentState.children[child]; present {
			tc.resyncState(path+"/"+child, currentNode, previousNode)
		} else {
			tc.recursiveDelete(path+"/"+child, previousNode)
		}
	}
}

func (tc *ZookeeperTreeCache) recursiveDelete(path string, node *zookeeperTreeCacheNode) {
	if !node.stopped {
		node.done <- struct{}{}
		node.stopped = true
	}
	if node.data != nil {
		tc.events <- ZookeeperTreeCacheEvent{Path: path, Data: nil}
		node.data = nil
	}
	for name, childNode := range node.children {
		tc.recursiveDelete(path+"/"+name, childNode)
	}
}
<commit_msg>Prevent deadlock in ZK TreeCache constructor by deferring the initial sync.<commit_after>// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package treecache

import (
	"bytes"
	"fmt"
	"strings"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"github.com/samuel/go-zookeeper/zk"
)

var (
	failureCounter = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "prometheus",
		Subsystem: "treecache",
		Name:      "zookeeper_failures_total",
		Help:      "The total number of ZooKeeper failures.",
	})
	numWatchers = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "prometheus",
		Subsystem: "treecache",
		Name:      "watcher_goroutines",
		Help:      "The current number of watcher goroutines.",
	})
)

func init() {
	prometheus.MustRegister(failureCounter)
	prometheus.MustRegister(numWatchers)
}

type ZookeeperLogger struct {
}

// Implements zk.Logger
func (zl ZookeeperLogger) Printf(s string, i ...interface{}) {
	log.Infof(s, i...)
}

type ZookeeperTreeCache struct {
	conn     *zk.Conn
	prefix   string
	events   chan ZookeeperTreeCacheEvent
	zkEvents chan zk.Event
	stop     chan struct{}
	head     *zookeeperTreeCacheNode
}

type ZookeeperTreeCacheEvent struct {
	Path string
	Data *[]byte
}

type zookeeperTreeCacheNode struct {
	data     *[]byte
	events   chan zk.Event
	done     chan struct{}
	stopped  bool
	children map[string]*zookeeperTreeCacheNode
}

func NewZookeeperTreeCache(conn *zk.Conn, path string, events chan ZookeeperTreeCacheEvent) *ZookeeperTreeCache {
	tc := &ZookeeperTreeCache{
		conn:   conn,
		prefix: path,
		events: events,
		stop:   make(chan struct{}),
	}
	tc.head = &zookeeperTreeCacheNode{
		events:   make(chan zk.Event),
		children: map[string]*zookeeperTreeCacheNode{},
		stopped:  true,
	}
	go tc.loop(path)
	return tc
}

func (tc *ZookeeperTreeCache) Stop() {
	tc.stop <- struct{}{}
}

func (tc *ZookeeperTreeCache) loop(path string) {
	failureMode := false
	retryChan := make(chan struct{})

	failure := func() {
		failureCounter.Inc()
		failureMode = true
		time.AfterFunc(time.Second*10, func() {
			retryChan <- struct{}{}
		})
	}

	err := tc.recursiveNodeUpdate(path, tc.head)
	if err != nil {
		log.Errorf("Error during initial read of Zookeeper: %s", err)
		failure()
	}

	for {
		select {
		case ev := <-tc.head.events:
			log.Debugf("Received Zookeeper event: %s", ev)
			if failureMode {
				continue
			}

			if ev.Type == zk.EventNotWatching {
				log.Infof("Lost connection to Zookeeper.")
				failure()
			} else {
				path := strings.TrimPrefix(ev.Path, tc.prefix)
				parts := strings.Split(path, "/")
				node := tc.head
				for _, part := range parts[1:] {
					childNode := node.children[part]
					if childNode == nil {
						childNode = &zookeeperTreeCacheNode{
							events:   tc.head.events,
							children: map[string]*zookeeperTreeCacheNode{},
							done:     make(chan struct{}, 1),
						}
						node.children[part] = childNode
					}
					node = childNode
				}

				err := tc.recursiveNodeUpdate(ev.Path, node)
				if err != nil {
					log.Errorf("Error during processing of Zookeeper event: %s", err)
					failure()
				} else if tc.head.data == nil {
					log.Errorf("Error during processing of Zookeeper event: path %s no longer exists", tc.prefix)
					failure()
				}
			}
		case <-retryChan:
			log.Infof("Attempting to resync state with Zookeeper")
			previousState := &zookeeperTreeCacheNode{
				children: tc.head.children,
			}
			// Reset root child nodes before traversing the Zookeeper path.
			tc.head.children = make(map[string]*zookeeperTreeCacheNode)

			if err := tc.recursiveNodeUpdate(tc.prefix, tc.head); err != nil {
				log.Errorf("Error during Zookeeper resync: %s", err)
				// Revert to our previous state.
				tc.head.children = previousState.children
				failure()
			} else {
				tc.resyncState(tc.prefix, tc.head, previousState)
				log.Infof("Zookeeper resync successful")
				failureMode = false
			}
		case <-tc.stop:
			close(tc.events)
			return
		}
	}
}

func (tc *ZookeeperTreeCache) recursiveNodeUpdate(path string, node *zookeeperTreeCacheNode) error {
	data, _, dataWatcher, err := tc.conn.GetW(path)
	if err == zk.ErrNoNode {
		tc.recursiveDelete(path, node)
		if node == tc.head {
			return fmt.Errorf("path %s does not exist", path)
		}
		return nil
	} else if err != nil {
		return err
	}

	if node.data == nil || !bytes.Equal(*node.data, data) {
		node.data = &data
		tc.events <- ZookeeperTreeCacheEvent{Path: path, Data: node.data}
	}

	children, _, childWatcher, err := tc.conn.ChildrenW(path)
	if err == zk.ErrNoNode {
		tc.recursiveDelete(path, node)
		return nil
	} else if err != nil {
		return err
	}

	currentChildren := map[string]struct{}{}
	for _, child := range children {
		currentChildren[child] = struct{}{}
		childNode := node.children[child]
		// Does not already exists or we previous had a watch that
		// triggered.
		if childNode == nil || childNode.stopped {
			node.children[child] = &zookeeperTreeCacheNode{
				events:   node.events,
				children: map[string]*zookeeperTreeCacheNode{},
				done:     make(chan struct{}, 1),
			}
			err = tc.recursiveNodeUpdate(path+"/"+child, node.children[child])
			if err != nil {
				return err
			}
		}
	}

	// Remove nodes that no longer exist
	for name, childNode := range node.children {
		if _, ok := currentChildren[name]; !ok || node.data == nil {
			tc.recursiveDelete(path+"/"+name, childNode)
			delete(node.children, name)
		}
	}

	go func() {
		numWatchers.Inc()
		// Pass up zookeeper events, until the node is deleted.
		select {
		case event := <-dataWatcher:
			node.events <- event
		case event := <-childWatcher:
			node.events <- event
		case <-node.done:
		}
		numWatchers.Dec()
	}()
	return nil
}

func (tc *ZookeeperTreeCache) resyncState(path string, currentState, previousState *zookeeperTreeCacheNode) {
	for child, previousNode := range previousState.children {
		if currentNode, present := currentState.children[child]; present {
			tc.resyncState(path+"/"+child, currentNode, previousNode)
		} else {
			tc.recursiveDelete(path+"/"+child, previousNode)
		}
	}
}

func (tc *ZookeeperTreeCache) recursiveDelete(path string, node *zookeeperTreeCacheNode) {
	if !node.stopped {
		node.done <- struct{}{}
		node.stopped = true
	}
	if node.data != nil {
		tc.events <- ZookeeperTreeCacheEvent{Path: path, Data: nil}
		node.data = nil
	}
	for name, childNode := range node.children {
		tc.recursiveDelete(path+"/"+name, childNode)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package test

import (
	"bytes"
	"os"
	"path/filepath"
	"strings"

	"v.io/x/devtools/internal/collect"
	"v.io/x/devtools/internal/gitutil"
	"v.io/x/devtools/internal/test"
	"v.io/x/devtools/internal/tool"
	"v.io/x/devtools/internal/util"
)

func vanadiumSignupProxy(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	return vanadiumSignupProxyHelper(ctx, "old_schema.go", testName)
}

func vanadiumSignupProxyNew(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	return vanadiumSignupProxyHelper(ctx, "new_schema.go", testName)
}

func vanadiumSignupProxyHelper(ctx *tool.Context, schema, testName string) (_ *test.Result, e error) {
	root, err := util.V23Root()
	if err != nil {
		return nil, internalTestError{err, "VanadiumRoot"}
	}

	// Fetch emails addresses.
	credentials := os.Getenv("CREDENTIALS")
	sheetID := os.Getenv("SHEET_ID")

	data, err := fetchFieldValues(ctx, credentials, "email", schema, sheetID)
	if err != nil {
		return nil, internalTestError{err, "fetch"}
	}

	// Create a feature branch in the infrastructure project.
	infraDir := tool.RootDirOpt(filepath.Join(root, "infrastructure"))
	if err := ctx.Git(infraDir).CreateAndCheckoutBranch("update"); err != nil {
		return nil, internalTestError{err, "create"}
	}
	defer collect.Error(func() error {
		if err := ctx.Git(infraDir).CheckoutBranch("master", gitutil.ForceOpt(true)); err != nil {
			return internalTestError{err, "checkout"}
		}
		if err := ctx.Git(infraDir).DeleteBranch("update", gitutil.ForceOpt(true)); err != nil {
			return internalTestError{err, "delete"}
		}
		return nil
	}, &e)

	// Update emails address whitelists.
	{
		whitelists := strings.Split(os.Getenv("WHITELISTS"), string(filepath.ListSeparator))
		mergeSrc := filepath.Join(root, "infrastructure", "signup", "merge.go")
		for _, whitelist := range whitelists {
			opts := ctx.Run().Opts()
			opts.Stdin = bytes.NewReader(data)
			if err := ctx.Run().CommandWithOpts(opts, "v23", "go", "run", mergeSrc, "-whitelist="+whitelist); err != nil {
				return nil, internalTestError{err, "merge"}
			}
			if err := ctx.Git(infraDir).Add(whitelist); err != nil {
				return nil, internalTestError{err, "commit"}
			}
		}
	}

	// Push changes (if any exist) to master.
	changed, err := ctx.Git(infraDir).HasUncommittedChanges()
	if err != nil {
		return nil, internalTestError{err, "changes"}
	}
	if changed {
		if err := ctx.Git(infraDir).CommitWithMessage("updating list of emails"); err != nil {
			return nil, internalTestError{err, "commit"}
		}
		if err := ctx.Git(infraDir).Push("origin", "update:master", gitutil.VerifyOpt(false)); err != nil {
			return nil, internalTestError{err, "push"}
		}
	}

	return &test.Result{Status: test.Passed}, nil
}

func vanadiumSignupGithub(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	return vanadiumSignupGithubHelper(ctx, "old_schema.go", testName)
}

func vanadiumSignupGithubNew(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	return vanadiumSignupGithubHelper(ctx, "new_schema.go", testName)
}

func vanadiumSignupGithubHelper(ctx *tool.Context, schema, testName string) (_ *test.Result, e error) {
	root, err := util.V23Root()
	if err != nil {
		return nil, internalTestError{err, "VanadiumRoot"}
	}

	credentials := os.Getenv("CREDENTIALS")
	sheetID := os.Getenv("SHEET_ID")

	data, err := fetchFieldValues(ctx, credentials, "github", schema, sheetID)
	if err != nil {
		return nil, internalTestError{err, "fetch"}
	}

	// Add them to @vanadium/developers
	github := filepath.Join(root, "infrastructure", "signup", "github.go")
	githubOpts := ctx.Run().Opts()
	githubOpts.Stdin = bytes.NewReader(data)
	if err := ctx.Run().CommandWithOpts(githubOpts, "v23", "go", "run", github, "-credentials="+credentials); err != nil {
		return nil, internalTestError{err, "github"}
	}

	return &test.Result{Status: test.Passed}, nil
}

func vanadiumSignupGroup(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	return vanadiumSignupGroupHelper(ctx, "old_schema.go", testName)
}

func vanadiumSignupGroupNew(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	return vanadiumSignupGroupHelper(ctx, "new_schema.go", testName)
}

func vanadiumSignupGroupHelper(ctx *tool.Context, schema, testName string) (_ *test.Result, e error) {
	root, err := util.V23Root()
	if err != nil {
		return nil, internalTestError{err, "VanadiumRoot"}
	}

	// Fetch emails addresses.
	credentials := os.Getenv("CREDENTIALS")
	sheetID := os.Getenv("SHEET_ID")

	data, err := fetchFieldValues(ctx, credentials, "email", schema, sheetID)
	if err != nil {
		return nil, internalTestError{err, "fetch"}
	}

	// Add them to Google Group.
	keyFile := os.Getenv("KEYFILE")
	serviceAccount := os.Getenv("SERVICE_ACCOUNT")
	opts := ctx.Run().Opts()
	opts.Stdin = bytes.NewReader(data)
	groupSrc := filepath.Join(root, "infrastructure", "signup", "group.go")
	if err := ctx.Run().CommandWithOpts(opts, "v23", "go", "run", groupSrc, "-keyFile="+keyFile, "-account="+serviceAccount); err != nil {
		return nil, internalTestError{err, "group"}
	}

	return &test.Result{Status: test.Passed}, nil
}

func fetchFieldValues(ctx *tool.Context, credentials, field, schema, sheetID string) ([]byte, error) {
	root, err := util.V23Root()
	if err != nil {
		return nil, internalTestError{err, "VanadiumRoot"}
	}

	var buffer bytes.Buffer

	fetchSrc := filepath.Join(root, "infrastructure", "signup", "fetch.go")
	schemaSrc := filepath.Join(root, "infrastructure", "signup", schema)
	opts := ctx.Run().Opts()
	opts.Stdout = &buffer
	if err := ctx.Run().CommandWithOpts(opts, "v23", "go", "run", fetchSrc, schemaSrc, "-credentials="+credentials, "-field="+field, "-sheet-id="+sheetID); err != nil {
		return nil, err
	}
	return buffer.Bytes(), nil
}
<commit_msg>TBR: devtools/signup: adds $GITHUB_CREDENTIALS to vanadium-signup-github-*<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package test

import (
	"bytes"
	"os"
	"path/filepath"
	"strings"

	"v.io/x/devtools/internal/collect"
	"v.io/x/devtools/internal/gitutil"
	"v.io/x/devtools/internal/test"
	"v.io/x/devtools/internal/tool"
	"v.io/x/devtools/internal/util"
)

func vanadiumSignupProxy(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	return vanadiumSignupProxyHelper(ctx, "old_schema.go", testName)
}

func vanadiumSignupProxyNew(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	return vanadiumSignupProxyHelper(ctx, "new_schema.go", testName)
}

func vanadiumSignupProxyHelper(ctx *tool.Context, schema, testName string) (_ *test.Result, e error) {
	root, err := util.V23Root()
	if err != nil {
		return nil, internalTestError{err, "VanadiumRoot"}
	}

	// Fetch emails addresses.
	credentials := os.Getenv("CREDENTIALS")
	sheetID := os.Getenv("SHEET_ID")

	data, err := fetchFieldValues(ctx, credentials, "email", schema, sheetID)
	if err != nil {
		return nil, internalTestError{err, "fetch"}
	}

	// Create a feature branch in the infrastructure project.
	infraDir := tool.RootDirOpt(filepath.Join(root, "infrastructure"))
	if err := ctx.Git(infraDir).CreateAndCheckoutBranch("update"); err != nil {
		return nil, internalTestError{err, "create"}
	}
	defer collect.Error(func() error {
		if err := ctx.Git(infraDir).CheckoutBranch("master", gitutil.ForceOpt(true)); err != nil {
			return internalTestError{err, "checkout"}
		}
		if err := ctx.Git(infraDir).DeleteBranch("update", gitutil.ForceOpt(true)); err != nil {
			return internalTestError{err, "delete"}
		}
		return nil
	}, &e)

	// Update emails address whitelists.
	{
		whitelists := strings.Split(os.Getenv("WHITELISTS"), string(filepath.ListSeparator))
		mergeSrc := filepath.Join(root, "infrastructure", "signup", "merge.go")
		for _, whitelist := range whitelists {
			opts := ctx.Run().Opts()
			opts.Stdin = bytes.NewReader(data)
			if err := ctx.Run().CommandWithOpts(opts, "v23", "go", "run", mergeSrc, "-whitelist="+whitelist); err != nil {
				return nil, internalTestError{err, "merge"}
			}
			if err := ctx.Git(infraDir).Add(whitelist); err != nil {
				return nil, internalTestError{err, "commit"}
			}
		}
	}

	// Push changes (if any exist) to master.
	changed, err := ctx.Git(infraDir).HasUncommittedChanges()
	if err != nil {
		return nil, internalTestError{err, "changes"}
	}
	if changed {
		if err := ctx.Git(infraDir).CommitWithMessage("updating list of emails"); err != nil {
			return nil, internalTestError{err, "commit"}
		}
		if err := ctx.Git(infraDir).Push("origin", "update:master", gitutil.VerifyOpt(false)); err != nil {
			return nil, internalTestError{err, "push"}
		}
	}

	return &test.Result{Status: test.Passed}, nil
}

func vanadiumSignupGithub(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	return vanadiumSignupGithubHelper(ctx, "old_schema.go", testName)
}

func vanadiumSignupGithubNew(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	return vanadiumSignupGithubHelper(ctx, "new_schema.go", testName)
}

func vanadiumSignupGithubHelper(ctx *tool.Context, schema, testName string) (_ *test.Result, e error) {
	root, err := util.V23Root()
	if err != nil {
		return nil, internalTestError{err, "VanadiumRoot"}
	}

	credentials := os.Getenv("CREDENTIALS")
	sheetID := os.Getenv("SHEET_ID")

	data, err := fetchFieldValues(ctx, credentials, "github", schema, sheetID)
	if err != nil {
		return nil, internalTestError{err, "fetch"}
	}

	// Add them to @vanadium/developers
	githubCredentials := os.Getenv("GITHUB_CREDENTIALS")
	github := filepath.Join(root, "infrastructure", "signup", "github.go")
	githubOpts := ctx.Run().Opts()
	githubOpts.Stdin = bytes.NewReader(data)
	if err := ctx.Run().CommandWithOpts(githubOpts, "v23", "go", "run", github, "-credentials="+githubCredentials); err != nil {
		return nil, internalTestError{err, "github"}
	}

	return &test.Result{Status: test.Passed}, nil
}

func vanadiumSignupGroup(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	return vanadiumSignupGroupHelper(ctx, "old_schema.go", testName)
}

func vanadiumSignupGroupNew(ctx *tool.Context, testName string, _ ...Opt) (_ *test.Result, e error) {
	return vanadiumSignupGroupHelper(ctx, "new_schema.go", testName)
}

func vanadiumSignupGroupHelper(ctx *tool.Context, schema, testName string) (_ *test.Result, e error) {
	root, err := util.V23Root()
	if err != nil {
		return nil, internalTestError{err, "VanadiumRoot"}
	}

	// Fetch emails addresses.
	credentials := os.Getenv("CREDENTIALS")
	sheetID := os.Getenv("SHEET_ID")

	data, err := fetchFieldValues(ctx, credentials, "email", schema, sheetID)
	if err != nil {
		return nil, internalTestError{err, "fetch"}
	}

	// Add them to Google Group.
	keyFile := os.Getenv("KEYFILE")
	serviceAccount := os.Getenv("SERVICE_ACCOUNT")
	opts := ctx.Run().Opts()
	opts.Stdin = bytes.NewReader(data)
	groupSrc := filepath.Join(root, "infrastructure", "signup", "group.go")
	if err := ctx.Run().CommandWithOpts(opts, "v23", "go", "run", groupSrc, "-keyFile="+keyFile, "-account="+serviceAccount); err != nil {
		return nil, internalTestError{err, "group"}
	}

	return &test.Result{Status: test.Passed}, nil
}

func fetchFieldValues(ctx *tool.Context, credentials, field, schema, sheetID string) ([]byte, error) {
	root, err := util.V23Root()
	if err != nil {
		return nil, internalTestError{err, "VanadiumRoot"}
	}

	var buffer bytes.Buffer

	fetchSrc := filepath.Join(root, "infrastructure", "signup", "fetch.go")
	schemaSrc := filepath.Join(root, "infrastructure", "signup", schema)
	opts := ctx.Run().Opts()
	opts.Stdout = &buffer
	if err := ctx.Run().CommandWithOpts(opts, "v23", "go", "run", fetchSrc, schemaSrc, "-credentials="+credentials, "-field="+field, "-sheet-id="+sheetID); err != nil {
		return nil, err
	}
	return buffer.Bytes(), nil
}
<|endoftext|>


<commit_before>package volumeplugin

import (
	"encoding/json"
	"fmt"

	"github.com/Sirupsen/logrus"
	"github.com/docker/go-plugins-helpers/volume"
	"github.com/pkg/errors"
	"github.com/rancher/go-rancher-metadata/metadata"
	"github.com/rancher/go-rancher/v2"
)

const (
	metadataURL = "http://169.254.169.250/2015-12-19"
)

var goodStates = map[string]bool{
	"active":       true,
	"activating":   true,
	"deactivating": true,
	"detached":     true,
}

type RancherState struct {
	client   *client.RancherClient
	driver   string
	hostID   string
	driverID string
}

func NewRancherState(driver string, client *client.RancherClient) (*RancherState, error) {
	host, err := getHostID(client)
	if err != nil {
		return nil, errors.Wrap(err, "getting host ID")
	}
	driverID, err := getDriverID(driver, client)
	if err != nil {
		return nil, errors.Wrap(err, "getting host ID")
	}

	logrus.Infof("Running on host %s(%s) with driver %s(%s)", host.Hostname, host.Id, driver, driverID)
	return &RancherState{
		client:   client,
		driver:   driver,
		hostID:   host.Id,
		driverID: driverID,
	}, nil
}

func (r *RancherState) IsCreated(name string) (bool, error) {
	_, vol, err := r.Get(name)
	if err == errNoSuchVolume {
		return false, nil
	}
	if err != nil {
		return false, err
	}
	return isCreated(r.driver, *vol), nil
}

func (r *RancherState) Save(name string, options map[string]string) error {
	_, vol, err := r.Get(name)
	if err == errNoSuchVolume {
		_, err = r.client.Volume.Create(&client.Volume{
			Name:            name,
			StorageDriverId: r.driverID,
			DriverOpts:      toMapInterface(options),
			HostId:          r.hostID,
		})
		return err
	} else if err == nil {
		_, err = r.client.Volume.Update(vol, &client.Volume{
			Name:            name,
			Driver:          r.driver,
			StorageDriverId: r.driverID,
			DriverOpts:      toMapInterface(options),
			HostId:          r.hostID,
		})
		return err
	}
	return err
}

func (r *RancherState) List() ([]*volume.Volume, error) {
	vols, err := r.client.Volume.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"removed_null":    "true",
			"limit":           "-1",
			"storageDriverId": r.driverID,
		},
	})
	if err != nil {
		return nil, err
	}
	result := []*volume.Volume{}
	for _, vol := range vols.Data {
		if isCreated(r.driver, vol) {
			result = append(result, volToVol(vol))
		}
	}

	return result, nil
}

func isCreated(driver string, vol client.Volume) bool {
	return goodStates[vol.State]
}

func (r *RancherState) Get(name string) (*volume.Volume, *client.Volume, error) {
	vols, err := r.client.Volume.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"name":         name,
			"removed_null": "true",
		},
	})
	if err != nil {
		return nil, nil, err
	}

	for _, vol := range vols.Data {
		if isCreated(r.driver, vol) {
			return volToVol(vol), &vol, nil
		}
	}

	return nil, nil, errNoSuchVolume
}

func volToVol(vol client.Volume) *volume.Volume {
	result := &volume.Volume{
		Name:   vol.Name,
		Status: map[string]interface{}{},
	}
	bytes, err := json.Marshal(vol)
	if err == nil {
		json.Unmarshal(bytes, &result.Status)
	}
	return result
}

func toMapInterface(data map[string]string) map[string]interface{} {
	result := map[string]interface{}{}
	for k, v := range data {
		result[k] = v
	}
	return result
}

func getDriverID(driver string, c *client.RancherClient) (string, error) {
	drivers, err := c.StorageDriver.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"name":         driver,
			"removed_null": true,
		},
	})
	if err != nil {
		return "", err
	}
	if len(drivers.Data) != 1 {
		return "", fmt.Errorf("%s is not a driver registered with the current Rancher environment", driver)
	}
	return drivers.Data[0].Id, nil
}

func getHostID(c *client.RancherClient) (*client.Host, error) {
	m, err := metadata.NewClientAndWait(metadataURL)
	if err != nil {
		return nil, errors.Wrap(err, "initializing metadata")
	}
	mHost, err := m.GetSelfHost()
	if err != nil {
		return nil, err
	}
	hosts, err := c.Host.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"uuid": mHost.UUID,
		},
	})
	if len(hosts.Data) != 1 {
		return nil, fmt.Errorf("Failed to find current host %s, got %d host(s)", mHost.UUID, len(hosts.Data))
	}
	return &hosts.Data[0], nil
}
<commit_msg>Add removing state to good states<commit_after>package volumeplugin

import (
	"encoding/json"
	"fmt"

	"github.com/Sirupsen/logrus"
	"github.com/docker/go-plugins-helpers/volume"
	"github.com/pkg/errors"
	"github.com/rancher/go-rancher-metadata/metadata"
	"github.com/rancher/go-rancher/v2"
)

const (
	metadataURL = "http://169.254.169.250/2015-12-19"
)

var goodStates = map[string]bool{
	"active":       true,
	"activating":   true,
	"deactivating": true,
	"detached":     true,
	"removing":     true,
}

type RancherState struct {
	client   *client.RancherClient
	driver   string
	hostID   string
	driverID string
}

func NewRancherState(driver string, client *client.RancherClient) (*RancherState, error) {
	host, err := getHostID(client)
	if err != nil {
		return nil, errors.Wrap(err, "getting host ID")
	}
	driverID, err := getDriverID(driver, client)
	if err != nil {
		return nil, errors.Wrap(err, "getting host ID")
	}

	logrus.Infof("Running on host %s(%s) with driver %s(%s)", host.Hostname, host.Id, driver, driverID)
	return &RancherState{
		client:   client,
		driver:   driver,
		hostID:   host.Id,
		driverID: driverID,
	}, nil
}

func (r *RancherState) IsCreated(name string) (bool, error) {
	_, vol, err := r.Get(name)
	if err == errNoSuchVolume {
		return false, nil
	}
	if err != nil {
		return false, err
	}
	return isCreated(r.driver, *vol), nil
}

func (r *RancherState) Save(name string, options map[string]string) error {
	_, vol, err := r.Get(name)
	if err == errNoSuchVolume {
		_, err = r.client.Volume.Create(&client.Volume{
			Name:            name,
			StorageDriverId: r.driverID,
			DriverOpts:      toMapInterface(options),
			HostId:          r.hostID,
		})
		return err
	} else if err == nil {
		_, err = r.client.Volume.Update(vol, &client.Volume{
			Name:            name,
			Driver:          r.driver,
			StorageDriverId: r.driverID,
			DriverOpts:      toMapInterface(options),
			HostId:          r.hostID,
		})
		return err
	}
	return err
}

func (r *RancherState) List() ([]*volume.Volume, error) {
	vols, err := r.client.Volume.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"removed_null":    "true",
			"limit":           "-1",
			"storageDriverId": r.driverID,
		},
	})
	if err != nil {
		return nil, err
	}
	result := []*volume.Volume{}
	for _, vol := range vols.Data {
		if isCreated(r.driver, vol) {
			result = append(result, volToVol(vol))
		}
	}

	return result, nil
}

func isCreated(driver string, vol client.Volume) bool {
	return goodStates[vol.State]
}

func (r *RancherState) Get(name string) (*volume.Volume, *client.Volume, error) {
	vols, err := r.client.Volume.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"name":         name,
			"removed_null": "true",
		},
	})
	if err != nil {
		return nil, nil, err
	}

	for _, vol := range vols.Data {
		if isCreated(r.driver, vol) {
			return volToVol(vol), &vol, nil
		}
	}

	return nil, nil, errNoSuchVolume
}

func volToVol(vol client.Volume) *volume.Volume {
	result := &volume.Volume{
		Name:   vol.Name,
		Status: map[string]interface{}{},
	}
	bytes, err := json.Marshal(vol)
	if err == nil {
		json.Unmarshal(bytes, &result.Status)
	}
	return result
}

func toMapInterface(data map[string]string) map[string]interface{} {
	result := map[string]interface{}{}
	for k, v := range data {
		result[k] = v
	}
	return result
}

func getDriverID(driver string, c *client.RancherClient) (string, error) {
	drivers, err := c.StorageDriver.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"name":         driver,
			"removed_null": true,
		},
	})
	if err != nil {
		return "", err
	}
	if len(drivers.Data) != 1 {
		return "", fmt.Errorf("%s is not a driver registered with the current Rancher environment", driver)
	}
	return drivers.Data[0].Id, nil
}

func getHostID(c *client.RancherClient) (*client.Host, error) {
	m, err := metadata.NewClientAndWait(metadataURL)
	if err != nil {
		return nil, errors.Wrap(err, "initializing metadata")
	}
	mHost, err := m.GetSelfHost()
	if err != nil {
		return nil, err
	}
	hosts, err := c.Host.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"uuid": mHost.UUID,
		},
	})
	if len(hosts.Data) != 1 {
		return nil, fmt.Errorf("Failed to find current host %s, got %d host(s)", mHost.UUID, len(hosts.Data))
	}
	return &hosts.Data[0], nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"strings"
	"math/rand"
	"time"
)

const (
	width      = 10
	height     = 10
	piecesFile = "./pieces.txt"
)

func readLines(path string) (lines []string, err error) {
	var (
		file   *os.File
		part   []byte
		prefix bool
	)
	if file, err = os.Open(path); err != nil {
		return
	}
	defer file.Close()

	reader := bufio.NewReader(file)
	buffer := bytes.NewBuffer(make([]byte, 0))
	for {
		if part, prefix, err = reader.ReadLine(); err != nil {
			break
		}
		buffer.Write(part)
		if !prefix {
			lines = append(lines, buffer.String())
			buffer.Reset()
		}
	}
	if err == io.EOF {
		err = nil
	}
	return
}

type Direction int

var None struct{} = struct{}{}

type Vector struct {
	x, y int
}

func (first Vector) plus(second Vector) Vector {
	return Vector{first.x + second.x, first.y + second.y}
}
func (first Vector) minus(second Vector) Vector {
	return Vector{first.x - second.x, first.y - second.y}
}

type VectorSet map[Vector]struct{}

type Piece struct {
	blocks VectorSet
}

func (ps VectorSet) contains(p Vector) bool {
	_, ok := ps[p]
	return ok
}

const (
	up Direction = iota + 1
	down
	left
	right
)

type Board struct {
	cells           VectorSet
	currentPiece    *Piece
	currentPosition Vector
}

func NewBoard() *Board {
	board := new(Board)
	board.cells = make(VectorSet)
	return board
}

type Game struct {
	board     *Board
	nextPiece *Piece
	pieces    []Piece
}

func loadPieces() (pieces []Piece, err error) {
	lines, err := readLines(piecesFile)
	waiting := true
	blocks := make(VectorSet)
	pieces = make([]Piece, 0)
	x, y := 0, 0
	for _, line := range lines {
		trimmed := strings.TrimSpace(line)
		if trimmed == "" {
			if waiting {
				continue
			}
			pieces = append(pieces, Piece{blocks})
			blocks = make(VectorSet)
			x, y = 0, 0
			waiting = true
		} else {
			waiting = false
			for _, char := range line {
				if char != ' ' {
					blocks[Vector{x, y}] = None
				}
				x++
			}
			y++
			x = 0
		}
	}
	if !waiting {
		pieces = append(pieces, Piece{blocks})
	}
	return
}

func NewPiece(points []Vector) Piece {
	pointSet := make(VectorSet)
	for _, point := range points {
		pointSet[point] = None
	}
	return Piece{pointSet}
}

func NewGame() *Game {
	game := new(Game)
	pieces, _ := loadPieces()
	game.pieces = pieces
	game.board = NewBoard()
	game.board.currentPiece = game.GeneratePiece()
	game.nextPiece = game.GeneratePiece()
	return game
}

func (game *Game) Start() {
	game.board.Draw()
	for {
		game.board.Draw()
	}
}

func (game *Game) GeneratePiece() *Piece {
	return &game.pieces[rand.Intn(len(game.pieces))]
}

func (board *Board) Move(where Direction) bool {
	translation := Vector{0, 0}
	switch where {
	case up:
		translation = Vector{0, -1}
	case down:
		translation = Vector{0, 1}
	case left:
		translation = Vector{-1, 0}
	case right:
		translation = Vector{1, 0}
	}
	board.currentPosition = board.currentPosition.plus(translation)
	return true
}

func (board *Board) Rotate(Direction) bool {
	return true
}

func (board *Board) MergeCurrentPiece() {
	return
}

func (board *Board) Filled(position Vector) bool {
	if board.cells.contains(position) {
		return true
	}
	if board.currentPiece == nil {
		return false
	}
	return board.currentPiece.blocks.contains(position.minus(board.currentPosition))
}

func (board *Board) Draw() {
	fmt.Println("+" + strings.Repeat("-", width) + "+")
	for y := 0; y < height; y++ {
		fmt.Printf("|")
		for x := 0; x < height; x++ {
			if board.Filled(Vector{x, y}) {
				fmt.Printf("#")
			} else {
				fmt.Printf(" ")
			}
		}
		fmt.Println("|")
	}
	fmt.Println("+" + strings.Repeat("-", width) + "+")
}

func main() {
	rand.Seed(time.Now().UnixNano())
	game := NewGame()
	game.Start()
}
<commit_msg>Got a nice board printed using termbox.<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/nsf/termbox-go"
	"io"
	"math/rand"
	"os"
	"strings"
	"time"
)

const (
	width      = 10
	height     = 15
	piecesFile = "./pieces.txt"
)

func readLines(path string) (lines []string, err error) {
	var (
		file   *os.File
		part   []byte
		prefix bool
	)
	if file, err = os.Open(path); err != nil {
		return
	}
	defer file.Close()

	reader := bufio.NewReader(file)
	buffer := bytes.NewBuffer(make([]byte, 0))
	for {
		if part, prefix, err = reader.ReadLine(); err != nil {
			break
		}
		buffer.Write(part)
		if !prefix {
			lines = append(lines, buffer.String())
			buffer.Reset()
		}
	}
	if err == io.EOF {
		err = nil
	}
	return
}

type Vector struct {
	x, y int
}

func (first Vector) plus(second Vector) Vector {
	return Vector{first.x + second.x, first.y + second.y}
}
func (first Vector) minus(second Vector) Vector {
	return Vector{first.x - second.x, first.y - second.y}
}

// A VectorSet is a Set of Vectors -- the values of the map have the type struct{} so as to not use any space.
type VectorSet map[Vector]struct{}

// None is the element used as a value in a VectorSet to indicate the vector's (key's) presence in the set. It
// is an empty placeholder.
var None struct{} = struct{}{}

type Piece struct {
	blocks VectorSet
}

func (ps VectorSet) contains(p Vector) bool {
	_, ok := ps[p]
	return ok
}

type Direction int

const (
	Up Direction = iota + 1
	Down
	Left
	Right
)

type Board struct {
	cells           VectorSet
	currentPiece    *Piece
	currentPosition Vector
}

func NewBoard() *Board {
	board := new(Board)
	board.cells = make(VectorSet)
	return board
}

type Game struct {
	board     *Board
	nextPiece *Piece
	pieces    []Piece
}

// Load in the visual representations of pieces from the file containing the piece shapes and emit an array of
// all possible pieces.
func loadPieces() (pieces []Piece, err error) {
	lines, err := readLines(piecesFile)
	waiting := true
	blocks := make(VectorSet)
	pieces = make([]Piece, 0)
	x, y := 0, 0
	for _, line := range lines {
		trimmed := strings.TrimSpace(line)
		if trimmed == "" {
			if waiting {
				continue
			}
			pieces = append(pieces, Piece{blocks})
			blocks = make(VectorSet)
			x, y = 0, 0
			waiting = true
		} else {
			waiting = false
			for _, char := range line {
				if char != ' ' {
					blocks[Vector{x, y}] = None
				}
				x++
			}
			y++
			x = 0
		}
	}
	if !waiting {
		pieces = append(pieces, Piece{blocks})
	}
	return
}

func NewPiece(points []Vector) Piece {
	pointSet := make(VectorSet)
	for _, point := range points {
		pointSet[point] = None
	}
	return Piece{pointSet}
}

func NewGame() *Game {
	game := new(Game)
	pieces, _ := loadPieces()
	game.pieces = pieces
	game.board = NewBoard()
	game.board.currentPiece = game.GeneratePiece()
	game.nextPiece = game.GeneratePiece()
	return game
}

func (game *Game) Start() {
	game.board.Draw()
gameLoop:
	for {
		switch event := termbox.PollEvent(); event.Type {
		case termbox.EventKey:
			switch event.Key {
			case termbox.KeyCtrlC:
				break gameLoop
			case termbox.KeyArrowLeft:
				game.board.Move(Left)
			case termbox.KeyArrowUp:
				game.board.Rotate()
			case termbox.KeyArrowRight:
				game.board.Move(Right)
			case termbox.KeyArrowDown:
				game.board.Move(Down)
			}
		case termbox.EventError:
			panic(event.Err)
		}
		game.board.Draw()
	}
}

func (game *Game) GeneratePiece() *Piece {
	return &game.pieces[rand.Intn(len(game.pieces))]
}

func (board *Board) moveIfPossible(translation Vector) bool {
	attemptedPosition := board.currentPosition.plus(translation)
	for point, _ := range board.currentPiece.blocks {
		attemptedPoint := point.plus(attemptedPosition)
		if attemptedPoint.x < 0 || attemptedPoint.x >= width ||
			attemptedPoint.y < 0 || attemptedPoint.y >= height ||
			board.cells.contains(attemptedPoint) {
			return false
		}
	}
	board.currentPosition = attemptedPosition
	return true
}

func (board *Board) Move(where Direction) bool {
	translation := Vector{0, 0}
	switch where {
	case Down:
		translation = Vector{0, 1}
	case Left:
		translation = Vector{-1, 0}
	case Right:
		translation = Vector{1, 0}
	}
	return board.moveIfPossible(translation)
}

func (board *Board) Rotate() bool {
	return true
}

func (board *Board) MergeCurrentPiece() {
	return
}

func (board *Board) Filled(position Vector) bool {
	if board.cells.contains(position) {
		return true
	}
	if board.currentPiece == nil {
		return false
	}
	return board.currentPiece.blocks.contains(position.minus(board.currentPosition))
}

func (board *Board) Draw() {
	termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)

	// Print the borders. The internal cells (the board cells) are treated as pairs, so to keep them on even x
	// coordinates we'll put an empty column on the left side.
	termbox.SetCell(1, 0, 0x256D, termbox.ColorBlue, termbox.ColorDefault)
	termbox.SetCell(width*2+2, 0, 0x256E, termbox.ColorBlue, termbox.ColorDefault)
	termbox.SetCell(1, height+1, 0x2570, termbox.ColorBlue, termbox.ColorDefault)
	termbox.SetCell(width*2+2, height+1, 0x256F, termbox.ColorBlue, termbox.ColorDefault)
	for x := 2; x <= width*2+1; x++ {
		termbox.SetCell(x, 0, 0x2500, termbox.ColorBlue, termbox.ColorDefault)
		termbox.SetCell(x, height+1, 0x2500, termbox.ColorBlue, termbox.ColorDefault)
	}
	for y := 1; y <= height; y++ {
		termbox.SetCell(1, y, 0x2502, termbox.ColorBlue, termbox.ColorDefault)
		termbox.SetCell(width*2+2, y, 0x2502, termbox.ColorBlue, termbox.ColorDefault)
	}

	// Print the board contents. Each block will correspond to a side-by-side pair of cells in the termbox, so
	// that the visible blocks will be roughly square.
	for x := 1; x <= width; x++ {
		for y := 1; y <= height; y++ {
			if board.Filled(Vector{x - 1, y - 1}) {
				termbox.SetCell(x*2, y, ' ', termbox.ColorDefault, termbox.ColorGreen)
				termbox.SetCell(x*2+1, y, ' ', termbox.ColorDefault, termbox.ColorGreen)
			}
		}
	}

	termbox.Flush()
}

func main() {
	rand.Seed(time.Now().UnixNano())

	err := termbox.Init()
	if err != nil {
		panic(err)
	}

	game := NewGame()
	game.Start()

	termbox.Close()
	fmt.Println("Bye!")
}
<|endoftext|>


<commit_before>package configuration

type HAProxy struct {
	TemplatePath          string
	OutputPath            string
	ReloadCommand         string
	ReloadValidationCommand string
	ReloadCleanupCommand  string
}
<commit_msg>go fmt<commit_after>package configuration

type HAProxy struct {
	TemplatePath            string
	OutputPath              string
	ReloadCommand           string
	ReloadValidationCommand string
	ReloadCleanupCommand    string
}
<|endoftext|>


<commit_before>package datastore

import (
	"bytes"
	"encoding/binary"
	"io"
	"log"
)

const (
	tabletMagic    uint32 = 0x0b501e7e
	metaIndexMagic uint32 = 0x0ea7da7a
	dataIndexMagic uint32 = 0xda7aba5e

	msgUint32 byte = 0xce
	msgUint64 byte = 0xcf
	msgFixRaw byte = 0xa0
	msgRaw16  byte = 0xda
	msgRaw32  byte = 0xdb
)

type BlockHandle struct {
	offset uint64
	length uint64
}

type IndexRecord struct {
	offset uint64
	length uint32
	name   []byte
}

type TabletOptions struct {
	BlockSize uint32
}

func WriteTablet(w io.Writer, kvs Iterator, opts *TabletOptions) {
	headLen := uint64(writeHeader(w))
	dataBlocks := writeDataBlocks(w, kvs, headLen, opts)
	metaIndexLen := writeIndex(w, metaIndexMagic, nil)

	lastBlock := dataBlocks[len(dataBlocks)-1]
	dataLen := lastBlock.offset + uint64(lastBlock.length)

	// For the moment we only support one data index record, so
	// its length must be cast to uint32 to fit.
	dataIndexLen := writeIndex(w, dataIndexMagic, dataBlocks)

	metaIndexHandle := BlockHandle{dataLen, metaIndexLen}
	dataIndexHandle := BlockHandle{dataLen + metaIndexLen, dataIndexLen}

	writeFooter(w, metaIndexHandle, dataIndexHandle)
}

func writeDataBlocks(w io.Writer, kvs Iterator, pos uint64, opts *TabletOptions) []*IndexRecord {
	blocks := make([]*IndexRecord, 0)

	for kvs.Next() {
		var thisBlock *IndexRecord
		if len(blocks) > 0 {
			thisBlock = blocks[len(blocks)-1]
		}

		if thisBlock == nil || thisBlock.length > opts.BlockSize {
			thisBlock = &IndexRecord{pos, 0, kvs.Key()}
			blocks = append(blocks, thisBlock)
		}

		rec := writeKv(w, kvs.Key(), kvs.Value())
		thisBlock.length += rec
		pos += uint64(rec)
	}

	return blocks
}

func writeKv(w io.Writer, key []byte, value []byte) uint32 {
	keyCount := writeRaw(w, key)
	valCount := writeRaw(w, value)

	return keyCount + valCount
}

func writeKvs(w io.Writer, kvs []*KV) (uint64, error) {
	var count uint64

	for i, kv := range kvs {
		if i > 0 && bytes.Compare(kvs[i-1].Key, kv.Key) >= 0 {
			log.Printf("writing non-increasing keys: %s -> %s",
				kvs[i-1].Key, kv.Key)
		}

		keyCount := writeRaw(w, kv.Key)
		valCount := writeRaw(w, kv.Value)

		count += uint64(keyCount) + uint64(valCount)
	}

	return count, nil
}

func writeRaw(w io.Writer, raw []byte) uint32 {
	n1 := writeRawHeader(w, len(raw))
	n2, _ := w.Write(raw)

	return n1 + uint32(n2)
}

func writeRawHeader(w io.Writer, n int) uint32 {
	if n < 32 {
		binary.Write(w, binary.BigEndian, msgFixRaw|byte(n))
		return 1
	} else if n < 65536 {
		w.Write([]byte{msgRaw16})
		binary.Write(w, binary.BigEndian, uint16(n))
		return 3
	} else {
		w.Write([]byte{msgRaw32})
		binary.Write(w, binary.BigEndian, uint32(n))
		return 5
	}
}

func writeUint32(w io.Writer, n uint32) uint32 {
	n1, _ := w.Write([]byte{msgUint32})
	binary.Write(w, binary.BigEndian, n)

	return uint32(n1) + 4
}

func writeUint64(w io.Writer, n uint64) uint32 {
	n1, _ := w.Write([]byte{msgUint64})
	binary.Write(w, binary.BigEndian, n)

	return uint32(n1) + 8
}

func writeIndex(w io.Writer, magic uint32, recs []*IndexRecord) uint64 {
	binary.Write(w, binary.BigEndian, magic)

	var n uint64
	for _, rec := range recs {
		n += uint64(writeUint64(w, rec.offset))
		n += uint64(writeUint32(w, rec.length))
		n += uint64(writeRaw(w, rec.name))
	}

	// include 4 bytes for the magic number
	return n + 4
}

func writeHeader(w io.Writer) uint32 {
	binary.Write(w, binary.BigEndian, tabletMagic)
	binary.Write(w, binary.BigEndian, uint32(0))

	// bytes written
	return 8
}

func writeFooter(w io.Writer, meta BlockHandle, data BlockHandle) {
	writeUint64(w, meta.offset)
	writeUint64(w, meta.length)
	writeUint64(w, data.offset)
	writeUint64(w, data.length)
	binary.Write(w, binary.BigEndian, tabletMagic)
}
<commit_msg>Remove a no longer accurate comment<commit_after>package datastore

import (
	"bytes"
	"encoding/binary"
	"io"
	"log"
)

const (
	tabletMagic    uint32 = 0x0b501e7e
	metaIndexMagic uint32 = 0x0ea7da7a
	dataIndexMagic uint32 = 0xda7aba5e

	msgUint32 byte = 0xce
	msgUint64 byte = 0xcf
	msgFixRaw byte = 0xa0
	msgRaw16  byte = 0xda
	msgRaw32  byte = 0xdb
)

type BlockHandle struct {
	offset uint64
	length uint64
}

type IndexRecord struct {
	offset uint64
	length uint32
	name   []byte
}

type TabletOptions struct {
	BlockSize uint32
}

func WriteTablet(w io.Writer, kvs Iterator, opts *TabletOptions) {
	headLen := uint64(writeHeader(w))
	dataBlocks := writeDataBlocks(w, kvs, headLen, opts)
	metaIndexLen := writeIndex(w, metaIndexMagic, nil)

	dataIndexLen := writeIndex(w, dataIndexMagic, dataBlocks)

	lastBlock := dataBlocks[len(dataBlocks)-1]
	dataLen := lastBlock.offset + uint64(lastBlock.length)

	metaIndexHandle := BlockHandle{dataLen, metaIndexLen}
	dataIndexHandle := BlockHandle{dataLen + metaIndexLen, dataIndexLen}

	writeFooter(w, metaIndexHandle, dataIndexHandle)
}

func writeDataBlocks(w io.Writer, kvs Iterator, pos uint64, opts *TabletOptions) []*IndexRecord {
	blocks := make([]*IndexRecord, 0)

	for kvs.Next() {
		var thisBlock *IndexRecord
		if len(blocks) > 0 {
			thisBlock = blocks[len(blocks)-1]
		}

		if thisBlock == nil || thisBlock.length > opts.BlockSize {
			thisBlock = &IndexRecord{pos, 0, kvs.Key()}
			blocks = append(blocks, thisBlock)
		}

		rec := writeKv(w, kvs.Key(), kvs.Value())
		thisBlock.length += rec
		pos += uint64(rec)
	}

	return blocks
}

func writeKv(w io.Writer, key []byte, value []byte) uint32 {
	keyCount := writeRaw(w, key)
	valCount := writeRaw(w, value)

	return keyCount + valCount
}

func writeKvs(w io.Writer, kvs []*KV) (uint64, error) {
	var count uint64

	for i, kv := range kvs {
		if i > 0 && bytes.Compare(kvs[i-1].Key, kv.Key) >= 0 {
			log.Printf("writing non-increasing keys: %s -> %s",
				kvs[i-1].Key, kv.Key)
		}

		keyCount := writeRaw(w, kv.Key)
		valCount := writeRaw(w, kv.Value)

		count += uint64(keyCount) + uint64(valCount)
	}

	return count, nil
}

func writeRaw(w io.Writer, raw []byte) uint32 {
	n1 := writeRawHeader(w, len(raw))
	n2, _ := w.Write(raw)

	return n1 + uint32(n2)
}

func writeRawHeader(w io.Writer, n int) uint32 {
	if n < 32 {
		binary.Write(w, binary.BigEndian, msgFixRaw|byte(n))
		return 1
	} else if n < 65536 {
		w.Write([]byte{msgRaw16})
		binary.Write(w, binary.BigEndian, uint16(n))
		return 3
	} else {
		w.Write([]byte{msgRaw32})
		binary.Write(w, binary.BigEndian, uint32(n))
		return 5
	}
}

func writeUint32(w io.Writer, n uint32) uint32 {
	n1, _ := w.Write([]byte{msgUint32})
	binary.Write(w, binary.BigEndian, n)

	return uint32(n1) + 4
}

func writeUint64(w io.Writer, n uint64) uint32 {
	n1, _ := w.Write([]byte{msgUint64})
	binary.Write(w, binary.BigEndian, n)

	return uint32(n1) + 8
}

func writeIndex(w io.Writer, magic uint32, recs []*IndexRecord) uint64 {
	binary.Write(w, binary.BigEndian, magic)

	var n uint64
	for _, rec := range recs {
		n += uint64(writeUint64(w, rec.offset))
		n += uint64(writeUint32(w, rec.length))
		n += uint64(writeRaw(w, rec.name))
	}

	// include 4 bytes for the magic number
	return n + 4
}

func writeHeader(w io.Writer) uint32 {
	binary.Write(w, binary.BigEndian, tabletMagic)
	binary.Write(w, binary.BigEndian, uint32(0))

	// bytes written
	return 8
}

func writeFooter(w io.Writer, meta BlockHandle, data BlockHandle) {
	writeUint64(w, meta.offset)
	writeUint64(w, meta.length)
	writeUint64(w, data.offset)
	writeUint64(w, data.length)
	binary.Write(w, binary.BigEndian, tabletMagic)
}
<|endoftext|>


<commit_before>package consul

import (
	"math/rand"
	"sync/atomic"
	"time"

	"github.com/hashicorp/consul/lib"
)

type consulServerEventTypes int

const (
	// consulServersNodeJoin is used to notify of a new consulServer.
	// The primary effect of this is a reshuffling of consulServers and
	// finding a new preferredServer.
	consulServersNodeJoin = iota

	// consulServersRebalance is used to signal we should rebalance our
	// connection load across servers
	consulServersRebalance

	// consulServersRPCError is used to signal when a server has either
	// timed out or returned an error and we would like to have the
	// server manager find a new preferredServer.
	consulServersRPCError
)

// serverCfg is the thread-safe configuration structure that is used to
// maintain the list of consul servers in Client.
//
// NOTE(sean@): We are explicitly relying on the fact that this is copied.
// Please keep this structure light.
type serverConfig struct {
	// servers tracks the locally known servers
	servers []*serverParts

	// Timer used to control rebalancing of servers
	rebalanceTimer *time.Timer
}

// consulServersManager is used to automatically shuffle and rebalance the
// list of consulServers.  This maintenance happens either when a new server
// is added or when a duration has been exceed.
func (c *Client) consulServersManager() {
	defaultTimeout := 5 * time.Second // FIXME(sean@): This is a bullshit value
	var rebalanceTimer *time.Timer
	func(c *Client) {
		c.serverConfigMtx.Lock()
		defer c.serverConfigMtx.Unlock()

		serverCfgPtr := c.serverConfigValue.Load()
		if serverCfgPtr == nil {
			panic("server config has not been initialized")
		}
		var serverCfg serverConfig
		serverCfg = serverCfgPtr.(serverConfig)
		rebalanceTimer = time.NewTimer(defaultTimeout)
		serverCfg.rebalanceTimer = rebalanceTimer
	}(c)

	for {
		select {
		case e := <-c.consulServersCh:
			switch e {
			case consulServersNodeJoin:
				c.logger.Printf("[INFO] consul: new node joined cluster")
				c.RebalanceServers()
			case consulServersRebalance:
				c.logger.Printf("[INFO] consul: rebalancing servers by request")
				c.RebalanceServers()
			case consulServersRPCError:
				c.logger.Printf("[INFO] consul: need to find a new server to talk with")
				c.CycleFailedServers()
				// FIXME(sean@): wtb preemptive Status.Ping
				// of servers, ideally parallel fan-out of N
				// nodes, then settle on the first node which
				// responds successfully.
				//
				// Is there a distinction between slow and
				// offline?  Do we run the Status.Ping with a
				// fixed timeout (say 30s) that way we can
				// alert administrators that they've set
				// their RPC time too low even though the
				// Ping did return successfully?
			default:
				c.logger.Printf("[WARN] consul: unhandled LAN Serf Event: %#v", e)
			}
		case <-rebalanceTimer.C:
			c.logger.Printf("[INFO] consul: server rebalance timeout")
			c.RebalanceServers()

		case <-c.shutdownCh:
			return
		}
	}
}

func (c *Client) AddServer(server *serverParts) {
	c.serverConfigMtx.Lock()
	defer c.serverConfigMtx.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	// Check if this server is known
	found := false
	for idx, existing := range serverCfg.servers {
		if existing.Name == server.Name {
			// Overwrite the existing server parts in order to
			// possibly update metadata (i.e. server version)
			serverCfg.servers[idx] = server
			found = true
			break
		}
	}

	// Add to the list if not known
	if !found {
		serverCfg.servers = append(serverCfg.servers, server)

		// Notify the server maintenance task of a new server
		c.consulServersCh <- consulServersNodeJoin
	}

	c.serverConfigValue.Store(serverCfg)

}

func (c *Client) CycleFailedServers() {
	c.serverConfigMtx.Lock()
	defer c.serverConfigMtx.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	for i := range serverCfg.servers {
		failCount := atomic.LoadUint64(&(serverCfg.servers[i].Disabled))
		if failCount == 0 {
			break
		} else if failCount > 0 {
			serverCfg.servers = serverCfg.cycleServer()
		}
	}

	serverCfg.resetRebalanceTimer(c)
	c.serverConfigValue.Store(serverCfg)
}

func (sc *serverConfig) cycleServer() (servers []*serverParts) {
	numServers := len(servers)
	if numServers < 2 {
		// No action required for zero or one server situations
		return servers
	}

	var failedNode *serverParts
	failedNode, servers = servers[0], servers[1:]
	servers = append(servers, failedNode)
	return servers
}

func (c *Client) RebalanceServers() {
	c.serverConfigMtx.Lock()
	defer c.serverConfigMtx.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	// Shuffle the server list on server join.  Servers are selected from
	// the head of the list and are moved to the end of the list on
	// failure.
	for i := len(serverCfg.servers) - 1; i > 0; i-- {
		j := rand.Int31n(int32(i + 1))
		serverCfg.servers[i], serverCfg.servers[j] = serverCfg.servers[j], serverCfg.servers[i]
	}

	serverCfg.resetRebalanceTimer(c)
	c.serverConfigValue.Store(serverCfg)
}

func (c *Client) RemoveServer(server *serverParts) {
	c.serverConfigMtx.Lock()
	defer c.serverConfigMtx.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	// Remove the server if known
	n := len(serverCfg.servers)
	for i := 0; i < n; i++ {
		if serverCfg.servers[i].Name == server.Name {
			serverCfg.servers[i], serverCfg.servers[n-1] = serverCfg.servers[n-1], nil
			serverCfg.servers = serverCfg.servers[:n-1]
			break
		}
	}

	c.serverConfigValue.Store(serverCfg)

}

// resetRebalanceTimer assumes:
//
// 1) the serverConfigMtx is already held by the caller.
// 2) the caller will call serverConfigValue.Store()
func (sc *serverConfig) resetRebalanceTimer(c *Client) {
	numConsulServers := len(sc.servers)
	// Limit this connection's life based on the size (and health) of the
	// cluster.  Never rebalance a connection more frequently than
	// connReuseLowWatermarkDuration, and make sure we never exceed
	// clusterWideRebalanceConnsPerSec operations/s across numLANMembers.
	clusterWideRebalanceConnsPerSec := float64(numConsulServers * newRebalanceConnsPerSecPerServer)
	connReuseLowWatermarkDuration := clientRPCMinReuseDuration + lib.RandomStagger(clientRPCMinReuseDuration/clientRPCJitterFraction)
	numLANMembers := len(c.LANMembers())
	connRebalanceTimeout := lib.RateScaledInterval(clusterWideRebalanceConnsPerSec, connReuseLowWatermarkDuration, numLANMembers)
	c.logger.Printf("[DEBUG] consul: connection will be rebalanced in %v", connRebalanceTimeout)

	sc.rebalanceTimer.Reset(connRebalanceTimeout)
}
<commit_msg>Rename serverConfigMtx to serverConfigLock<commit_after>package consul

import (
	"math/rand"
	"sync/atomic"
	"time"

	"github.com/hashicorp/consul/lib"
)

type consulServerEventTypes int

const (
	// consulServersNodeJoin is used to notify of a new consulServer.
	// The primary effect of this is a reshuffling of consulServers and
	// finding a new preferredServer.
	consulServersNodeJoin = iota

	// consulServersRebalance is used to signal we should rebalance our
	// connection load across servers
	consulServersRebalance

	// consulServersRPCError is used to signal when a server has either
	// timed out or returned an error and we would like to have the
	// server manager find a new preferredServer.
	consulServersRPCError
)

// serverCfg is the thread-safe configuration structure that is used to
// maintain the list of consul servers in Client.
//
// NOTE(sean@): We are explicitly relying on the fact that this is copied.
// Please keep this structure light.
type serverConfig struct {
	// servers tracks the locally known servers
	servers []*serverParts

	// Timer used to control rebalancing of servers
	rebalanceTimer *time.Timer
}

// consulServersManager is used to automatically shuffle and rebalance the
// list of consulServers.  This maintenance happens either when a new server
// is added or when a duration has been exceed.
func (c *Client) consulServersManager() {
	defaultTimeout := 5 * time.Second // FIXME(sean@): This is a bullshit value
	var rebalanceTimer *time.Timer
	func(c *Client) {
		c.serverConfigLock.Lock()
		defer c.serverConfigLock.Unlock()

		serverCfgPtr := c.serverConfigValue.Load()
		if serverCfgPtr == nil {
			panic("server config has not been initialized")
		}
		var serverCfg serverConfig
		serverCfg = serverCfgPtr.(serverConfig)
		rebalanceTimer = time.NewTimer(defaultTimeout)
		serverCfg.rebalanceTimer = rebalanceTimer
	}(c)

	for {
		select {
		case e := <-c.consulServersCh:
			switch e {
			case consulServersNodeJoin:
				c.logger.Printf("[INFO] consul: new node joined cluster")
				c.RebalanceServers()
			case consulServersRebalance:
				c.logger.Printf("[INFO] consul: rebalancing servers by request")
				c.RebalanceServers()
			case consulServersRPCError:
				c.logger.Printf("[INFO] consul: need to find a new server to talk with")
				c.CycleFailedServers()
				// FIXME(sean@): wtb preemptive Status.Ping
				// of servers, ideally parallel fan-out of N
				// nodes, then settle on the first node which
				// responds successfully.
				//
				// Is there a distinction between slow and
				// offline?  Do we run the Status.Ping with a
				// fixed timeout (say 30s) that way we can
				// alert administrators that they've set
				// their RPC time too low even though the
				// Ping did return successfully?
			default:
				c.logger.Printf("[WARN] consul: unhandled LAN Serf Event: %#v", e)
			}
		case <-rebalanceTimer.C:
			c.logger.Printf("[INFO] consul: server rebalance timeout")
			c.RebalanceServers()

		case <-c.shutdownCh:
			return
		}
	}
}

func (c *Client) AddServer(server *serverParts) {
	c.serverConfigLock.Lock()
	defer c.serverConfigLock.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	// Check if this server is known
	found := false
	for idx, existing := range serverCfg.servers {
		if existing.Name == server.Name {
			// Overwrite the existing server parts in order to
			// possibly update metadata (i.e. server version)
			serverCfg.servers[idx] = server
			found = true
			break
		}
	}

	// Add to the list if not known
	if !found {
		serverCfg.servers = append(serverCfg.servers, server)

		// Notify the server maintenance task of a new server
		c.consulServersCh <- consulServersNodeJoin
	}

	c.serverConfigValue.Store(serverCfg)

}

func (c *Client) CycleFailedServers() {
	c.serverConfigLock.Lock()
	defer c.serverConfigLock.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	for i := range serverCfg.servers {
		failCount := atomic.LoadUint64(&(serverCfg.servers[i].Disabled))
		if failCount == 0 {
			break
		} else if failCount > 0 {
			serverCfg.servers = serverCfg.cycleServer()
		}
	}

	serverCfg.resetRebalanceTimer(c)
	c.serverConfigValue.Store(serverCfg)
}

func (sc *serverConfig) cycleServer() (servers []*serverParts) {
	numServers := len(servers)
	if numServers < 2 {
		// No action required for zero or one server situations
		return servers
	}

	var failedNode *serverParts
	failedNode, servers = servers[0], servers[1:]
	servers = append(servers, failedNode)
	return servers
}

func (c *Client) RebalanceServers() {
	c.serverConfigLock.Lock()
	defer c.serverConfigLock.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	// Shuffle the server list on server join.  Servers are selected from
	// the head of the list and are moved to the end of the list on
	// failure.
	for i := len(serverCfg.servers) - 1; i > 0; i-- {
		j := rand.Int31n(int32(i + 1))
		serverCfg.servers[i], serverCfg.servers[j] = serverCfg.servers[j], serverCfg.servers[i]
	}

	serverCfg.resetRebalanceTimer(c)
	c.serverConfigValue.Store(serverCfg)
}

func (c *Client) RemoveServer(server *serverParts) {
	c.serverConfigLock.Lock()
	defer c.serverConfigLock.Unlock()
	serverCfg := c.serverConfigValue.Load().(serverConfig)

	// Remove the server if known
	n := len(serverCfg.servers)
	for i := 0; i < n; i++ {
		if serverCfg.servers[i].Name == server.Name {
			serverCfg.servers[i], serverCfg.servers[n-1] = serverCfg.servers[n-1], nil
			serverCfg.servers = serverCfg.servers[:n-1]
			break
		}
	}

	c.serverConfigValue.Store(serverCfg)

}

// resetRebalanceTimer assumes:
//
// 1) the serverConfigLock is already held by the caller.
// 2) the caller will call serverConfigValue.Store()
func (sc *serverConfig) resetRebalanceTimer(c *Client) {
	numConsulServers := len(sc.servers)
	// Limit this connection's life based on the size (and health) of the
	// cluster.  Never rebalance a connection more frequently than
	// connReuseLowWatermarkDuration, and make sure we never exceed
	// clusterWideRebalanceConnsPerSec operations/s across numLANMembers.
	clusterWideRebalanceConnsPerSec := float64(numConsulServers * newRebalanceConnsPerSecPerServer)
	connReuseLowWatermarkDuration := clientRPCMinReuseDuration + lib.RandomStagger(clientRPCMinReuseDuration/clientRPCJitterFraction)
	numLANMembers := len(c.LANMembers())
	connRebalanceTimeout := lib.RateScaledInterval(clusterWideRebalanceConnsPerSec, connReuseLowWatermarkDuration, numLANMembers)
	c.logger.Printf("[DEBUG] consul: connection will be rebalanced in %v", connRebalanceTimeout)

	sc.rebalanceTimer.Reset(connRebalanceTimeout)
}
<|endoftext|>


<commit_before>package oskite

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"koding/db/models"
	"koding/tools/dnode"
	"koding/tools/kite"
	"koding/virt"
	"net/http"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

const (
	ErrTokenRequired    = "TOKEN_REQUIRED"
	ErrUsernameRequired = "USERNAME_REQUIRED"
	ErrGroupRequired    = "GROUPNAME_REQUIRED"
	ErrKiteNotFound     = "KITE_NOT_FOUND"
	ErrUserNotFound     = "USER_NOT_FOUND"
	ErrGroupNotFound    = "GROUP_NOT_FOUND"
	ErrInvalid          = "NOT_A_MEMBER_OF_GROUP"
	ErrKiteNoPlan       = "KITE_HAS_NO_PLAN"
	ErrNoSubscription   = "NO_SUBSCRIPTION"
)

type KiteStore struct {
	Id       bson.ObjectId `bson:"_id"`
	Name     string        `bson:"name"`
	KiteCode string        `bson:"kiteCode"`
	Manifest struct {
		Description string `bson:"description"`
		Name        string `bson:"name"`
		Readme      string `bson:"readme"`
		AuthorNick  string `bson:"authorNick"`
		Author      string `bson:"author"`
	}
}

type Plan struct {
	CPU         int `bson:"cpu"`
	RAM         int `bson:"ram"`  // Memory usage in MB
	Disk        int `bson:"disk"` // Disk in MB
	TotalVMs    int `bson:"totalVMs"`
	AlwaysOnVMs int `bson:"alwaysOnVMs"`
}

type PlanResponse struct {
	CPU         string `bson:"cpu"`
	RAM         string `bson:"ram"`  // Memory usage in MB
	Disk        string `bson:"disk"` // Disk in MB
	TotalVMs    string `bson:"totalVMs"`
	AlwaysOnVMs string `bson:"alwaysOnVMs"`
}

type subscriptionResp struct {
	Plan   string `json:"plan"`
	PlanId string `json:"planId"`
	Err    string `json:"err"`
}

var (
	plans = map[string]Plan{
		"free": {CPU: 1, RAM: 1000, Disk: 3000, TotalVMs: 1, AlwaysOnVMs: 0},
		"1x":   {CPU: 2, RAM: 2000, Disk: 10000, TotalVMs: 2, AlwaysOnVMs: 1},
		"2x":   {CPU: 4, RAM: 4000, Disk: 20000, TotalVMs: 4, AlwaysOnVMs: 2},
		"3x":   {CPU: 6, RAM: 6000, Disk: 40000, TotalVMs: 6, AlwaysOnVMs: 3},
		"4x":   {CPU: 8, RAM: 8000, Disk: 80000, TotalVMs: 8, AlwaysOnVMs: 4},
		"5x":   {CPU: 10, RAM: 10000, Disk: 100000, TotalVMs: 10, AlwaysOnVMs: 5},
	}

	okString      = "ok"
	quotaExceeded = "quota exceeded"
	kiteCode      string
)

func NewPlanResponse() *PlanResponse {
	return &PlanResponse{
		CPU:         okString,
		RAM:         okString,
		Disk:        okString,
		TotalVMs:    okString,
		AlwaysOnVMs: okString,
	}
}

func vmUsage(args *dnode.Partial, vos *virt.VOS, username string) (interface{}, error) {
	var params struct {
		GroupId string
	}

	if args == nil {
		return nil, &kite.ArgumentError{Expected: "empy argument passed"}
	}

	if args.Unmarshal(&params) != nil || params.GroupId == "" {
		return nil, &kite.ArgumentError{Expected: "{ groupId: [string] }"}
	}

	usage, err := NewUsage(vos, params.GroupId)
	if err != nil {
		log.Info("vm.usage [%s] err: %v", vos.VM.HostnameAlias, err)
		return nil, errors.New("vm.usage couldn't be retrieved. please consult to support.")
	}

	return usage.checkLimits(username, params.GroupId)
}

func NewUsage(vos *virt.VOS, groupId string) (*Plan, error) {
	if !bson.IsObjectIdHex(groupId) {
		return nil, fmt.Errorf("groupID %s is not valid hex representation", groupId)
	}

	vms := make([]*models.VM, 0)

	// db.jVMs.find({"webHome":"foo", "groups": {$in:[{"id":ObjectId("5196fcb2bc9bdb0000000027")}]}})
	query := func(c *mgo.Collection) error {
		return c.Find(bson.M{
			"webHome": vos.VM.WebHome,
			"groups":  bson.M{"$in": []bson.M{bson.M{"id": bson.ObjectIdHex(groupId)}}},
		}).Iter().All(&vms)
	}

	if err := mongodbConn.Run("jVMs", query); err != nil {
		return nil, fmt.Errorf("vm fetching err for user %s. err: %s", vos.VM.WebHome, err)
	}

	usage := new(Plan)
	usage.TotalVMs = len(vms)

	for _, vm := range vms {
		if vm.AlwaysOn {
			usage.AlwaysOnVMs++
		}

		usage.CPU += vm.NumCPUs
		usage.RAM += vm.MaxMemoryInMB
		usage.Disk += vm.DiskSizeInMB
	}

	fmt.Printf("usage %+v\n", usage)
	return usage, nil
}

func (p *Plan) checkLimits(username, groupname string) (*PlanResponse, error) {
	sub, err := getSubscription(username, groupname)
	if err != nil {
		log.Warning("oskite checkLimits err: %v", err)
		return nil, errors.New("couldn't fetch subscription")
	}

	if sub.PlanId == "" {
		return nil, errors.New("planID is empty")
	}

	plan, ok := plans[sub.PlanId]
	if !ok {
		return nil, errors.New("plan doesn't exist")
	}

	resp := NewPlanResponse()
	if p.AlwaysOnVMs >= plan.AlwaysOnVMs {
		resp.AlwaysOnVMs = quotaExceeded
	}

	if p.TotalVMs >= plan.TotalVMs {
		resp.TotalVMs = quotaExceeded
	}

	return resp, nil
}

// getKiteCode returns the API token to be used with Koding's subscription
// endpoint.
func getKiteCode() (string, error) {
	if kiteCode != "" {
		return kiteCode, nil
	}

	kiteStore := new(KiteStore)

	query := func(c *mgo.Collection) error {
		return c.Find(bson.M{"name": "OsKite"}).One(kiteStore)
	}

	err := mongodbConn.Run("jKites", query)
	if err != nil {
		return "", err
	}

	kiteCode = kiteStore.KiteCode

	return kiteStore.KiteCode, nil
}

func getSubscription(username, groupname string) (*subscriptionResp, error) {
	code, err := getKiteCode()
	if err != nil {
		return nil, err
	}

	if code == "" {
		return nil, errors.New("kite code is empty")
	}

	endpointURL := conf.SubscriptionEndpoint + code + "/" + username + "/" + groupname

	resp, err := http.Get(endpointURL)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var sub = new(subscriptionResp)
	if err := json.Unmarshal(body, sub); err != nil {
		return nil, errors.New("Subscription data is malformed")
	}

	if resp.StatusCode != 200 {
		if sub.Err != "" {
			return nil, errors.New(sub.Err)
		}
		return nil, errors.New("api not allowed")
	}

	return sub, nil
}
<commit_msg>oskite: update latest changes<commit_after>package oskite

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"koding/db/models"
	"koding/tools/dnode"
	"koding/tools/kite"
	"koding/virt"
	"net/http"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

const (
	ErrTokenRequired    = "TOKEN_REQUIRED"
	ErrUsernameRequired = "USERNAME_REQUIRED"
	ErrGroupRequired    = "GROUPNAME_REQUIRED"
	ErrKiteNotFound     = "KITE_NOT_FOUND"
	ErrUserNotFound     = "USER_NOT_FOUND"
	ErrGroupNotFound    = "GROUP_NOT_FOUND"
	ErrInvalid          = "NOT_A_MEMBER_OF_GROUP"
	ErrKiteNoPlan       = "KITE_HAS_NO_PLAN"
	ErrNoSubscription   = "NO_SUBSCRIPTION"
)

type KiteStore struct {
	Id       bson.ObjectId `bson:"_id"`
	Name     string        `bson:"name"`
	KiteCode string        `bson:"kiteCode"`
	Manifest struct {
		Description string `bson:"description"`
		Name        string `bson:"name"`
		Readme      string `bson:"readme"`
		AuthorNick  string `bson:"authorNick"`
		Author      string `bson:"author"`
	}
}

type Plan struct {
	CPU         int `bson:"cpu"`
	RAM         int `bson:"ram"`  // Memory usage in MB
	Disk        int `bson:"disk"` // Disk in MB
	TotalVMs    int `bson:"totalVMs"`
	AlwaysOnVMs int `bson:"alwaysOnVMs"`
}

type PlanResponse struct {
	CPU         string `bson:"cpu"`
	RAM         string `bson:"ram"`  // Memory usage in MB
	Disk        string `bson:"disk"` // Disk in MB
	TotalVMs    string `bson:"totalVMs"`
	AlwaysOnVMs string `bson:"alwaysOnVMs"`
}

type subscriptionResp struct {
	PlanName string `json:"planName"`
	PlanId   string `json:"planId"`
	Err      string `json:"err"`
}

var (
	plans = map[string]Plan{
		"free": {CPU: 1, RAM: 1000, Disk: 3000, TotalVMs: 1, AlwaysOnVMs: 0},
		"1x":   {CPU: 2, RAM: 2000, Disk: 10000, TotalVMs: 2, AlwaysOnVMs: 1},
		"2x":   {CPU: 4, RAM: 4000, Disk: 20000, TotalVMs: 4, AlwaysOnVMs: 2},
		"3x":   {CPU: 6, RAM: 6000, Disk: 40000, TotalVMs: 6, AlwaysOnVMs: 3},
		"4x":   {CPU: 8, RAM: 8000, Disk: 80000, TotalVMs: 8, AlwaysOnVMs: 4},
		"5x":   {CPU: 10, RAM: 10000, Disk: 100000, TotalVMs: 10, AlwaysOnVMs: 5},
	}

	okString      = "ok"
	quotaExceeded = "quota exceeded"
	kiteCode      string
)

func NewPlanResponse() *PlanResponse {
	return &PlanResponse{
		CPU:         okString,
		RAM:         okString,
		Disk:        okString,
		TotalVMs:    okString,
		AlwaysOnVMs: okString,
	}
}

func vmUsage(args *dnode.Partial, vos *virt.VOS, username string) (interface{}, error) {
	var params struct {
		GroupId string
	}

	if args == nil {
		return nil, &kite.ArgumentError{Expected: "empy argument passed"}
	}

	if args.Unmarshal(&params) != nil || params.GroupId == "" {
		return nil, &kite.ArgumentError{Expected: "{ groupId: [string] }"}
	}

	usage, err := NewUsage(vos, params.GroupId)
	if err != nil {
		log.Info("vm.usage [%s] err: %v", vos.VM.HostnameAlias, err)
		return nil, errors.New("vm.usage couldn't be retrieved. please consult to support.")
	}

	return usage.checkLimits(username, params.GroupId)
}

func NewUsage(vos *virt.VOS, groupId string) (*Plan, error) {
	if !bson.IsObjectIdHex(groupId) {
		return nil, fmt.Errorf("groupID %s is not valid hex representation", groupId)
	}

	vms := make([]*models.VM, 0)

	// db.jVMs.find({"webHome":"foo", "groups": {$in:[{"id":ObjectId("5196fcb2bc9bdb0000000027")}]}})
	query := func(c *mgo.Collection) error {
		return c.Find(bson.M{
			"webHome": vos.VM.WebHome,
			"groups":  bson.M{"$in": []bson.M{bson.M{"id": bson.ObjectIdHex(groupId)}}},
		}).Iter().All(&vms)
	}

	if err := mongodbConn.Run("jVMs", query); err != nil {
		return nil, fmt.Errorf("vm fetching err for user %s. err: %s", vos.VM.WebHome, err)
	}

	usage := new(Plan)
	usage.TotalVMs = len(vms)

	for _, vm := range vms {
		if vm.AlwaysOn {
			usage.AlwaysOnVMs++
		}

		usage.CPU += vm.NumCPUs
		usage.RAM += vm.MaxMemoryInMB
		usage.Disk += vm.DiskSizeInMB
	}

	fmt.Printf("usage %+v\n", usage)
	return usage, nil
}

func (p *Plan) checkLimits(username, groupId string) (*PlanResponse, error) {
	sub, err := getSubscription(username, groupId)
	if err != nil {
		log.Warning("oskite checkLimits err: %v", err)
		return nil, errors.New("couldn't fetch subscription")
	}

	if sub.PlanName == "" {
		return nil, errors.New("planName is empty")
	}
	fmt.Printf("sub %+v\n", sub)

	plan, ok := plans[sub.PlanName]
	if !ok {
		return nil, errors.New("plan doesn't exist")
	}

	resp := NewPlanResponse()
	if p.AlwaysOnVMs >= plan.AlwaysOnVMs {
		resp.AlwaysOnVMs = quotaExceeded
	}

	if p.TotalVMs > plan.TotalVMs {
		resp.TotalVMs = quotaExceeded
	}

	return resp, nil
}

// getKiteCode returns the API token to be used with Koding's subscription
// endpoint.
func getKiteCode() (string, error) {
	if kiteCode != "" {
		return kiteCode, nil
	}

	kiteStore := new(KiteStore)

	query := func(c *mgo.Collection) error {
		return c.Find(bson.M{"name": "OsKite"}).One(kiteStore)
	}

	err := mongodbConn.Run("jKites", query)
	if err != nil {
		return "", err
	}

	kiteCode = kiteStore.KiteCode

	return kiteStore.KiteCode, nil
}

func getSubscription(username, groupId string) (*subscriptionResp, error) {
	code, err := getKiteCode()
	if err != nil {
		return nil, err
	}

	if code == "" {
		return nil, errors.New("kite code is empty")
	}

	endpointURL := conf.SubscriptionEndpoint + code + "/" + username + "/" + groupId

	fmt.Printf("endpointURL %+v\n", endpointURL)

	resp, err := http.Get(endpointURL)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var sub = new(subscriptionResp)
	if err := json.Unmarshal(body, sub); err != nil {
		return nil, errors.New("Subscription data is malformed")
	}

	if resp.StatusCode != 200 {
		if sub.Err != "" {
			return nil, errors.New(sub.Err)
		}
		return nil, errors.New("api not allowed")
	}

	return sub, nil
}
<|endoftext|>