<commit_before>package heartbeat

import (
	"context"
	"sync"
	"time"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes/fake"

	virtv1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/kubevirt/pkg/testutils"
	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
	device_manager "kubevirt.io/kubevirt/pkg/virt-handler/device-manager"
)

const (
	cpu_manager_static_path = "testdata/cpu_manager_state_static"
	cpu_manager_none_path   = "testdata/cpu_manager_state_none"
)

var _ = Describe("Heartbeat", func() {

	var node *v1.Node
	var fakeClient *fake.Clientset

	BeforeEach(func() {
		node = &v1.Node{
			ObjectMeta: metav1.ObjectMeta{
				Name: "mynode",
			},
		}
		fakeClient = fake.NewSimpleClientset(node)
	})

	table.DescribeTable("with cpumanager featuregate should set the node to", func(deviceController device_manager.DeviceControllerInterface, cpuManagerPaths []string, schedulable string, cpumanager string) {
		heartbeat := NewHeartBeat(fakeClient.CoreV1(), deviceController, config(virtconfig.CPUManager), "mynode")
		heartbeat.cpuManagerPaths = cpuManagerPaths
		heartbeat.do()
		node, err := fakeClient.CoreV1().Nodes().Get(context.Background(), "mynode", metav1.GetOptions{})
		Expect(err).ToNot(HaveOccurred())
		Expect(node.Labels).To(HaveKeyWithValue(virtv1.NodeSchedulable, schedulable))
		Expect(node.Labels).To(HaveKeyWithValue(virtv1.CPUManager, cpumanager))
	},
		table.Entry("not schedulable and no cpu manager with no cpu manager file and device plugins are not initialized",
			deviceController(false),
			[]string{"non/existent/cpumanager/statefile"},
			"false",
			"false",
		),
		table.Entry("schedulable and no cpu manager with no cpu manager file and plugins are not initialized",
			deviceController(true),
			[]string{"non/existent/cpumanager/statefile"},
			"true",
			"false",
		),
		table.Entry("schedulable and cpu manager with static cpu manager policy configured and device plugins are not initialized",
			deviceController(true),
			[]string{"non/existent/cpumanager/statefile", cpu_manager_static_path},
			"true",
			"true",
		),
		table.Entry("schedulable and no cpu manager with no cpu manager policy configured and device plugins are not initialized",
			deviceController(true),
			[]string{cpu_manager_none_path, "non/existent/cpumanager/statefile"},
			"true",
			"false",
		),
	)

	table.DescribeTable("without cpumanager featuregate should set the node to", func(deviceController device_manager.DeviceControllerInterface, schedulable string) {
		heartbeat := NewHeartBeat(fakeClient.CoreV1(), deviceController, config(), "mynode")
		heartbeat.do()
		node, err := fakeClient.CoreV1().Nodes().Get(context.Background(), "mynode", metav1.GetOptions{})
		Expect(err).ToNot(HaveOccurred())
		Expect(node.Labels).To(HaveKeyWithValue(virtv1.NodeSchedulable, schedulable))
		Expect(node.Labels).ToNot(HaveKeyWithValue(virtv1.CPUManager, false))
	},
		table.Entry("not schedulable with no cpumanager label present",
			deviceController(false),
			"false",
		),
		table.Entry("schedulable with no cpumanger label present",
			deviceController(true),
			"true",
		),
	)

	table.DescribeTable("without deviceplugin and", func(deviceController device_manager.DeviceControllerInterface, initiallySchedulable string, finallySchedulable string) {
		heartbeat := NewHeartBeat(fakeClient.CoreV1(), deviceController, config(), "mynode")
		heartbeat.devicePluginWaitTimeout = 1 * time.Second
		heartbeat.devicePluginPollIntervall = 10 * time.Millisecond
		stopChan := make(chan struct{})
		done := heartbeat.Run(100*time.Second, stopChan)
		defer func() {
			close(stopChan)
			<-done
		}()
		Eventually(func() map[string]string {
			node, err := fakeClient.CoreV1().Nodes().Get(context.Background(), "mynode", metav1.GetOptions{})
			Expect(err).ToNot(HaveOccurred())
			return node.Labels
		}).Should(And(
			HaveKeyWithValue(virtv1.NodeSchedulable, initiallySchedulable),
		))
		Consistently(func() map[string]string {
			node, err := fakeClient.CoreV1().Nodes().Get(context.Background(), "mynode", metav1.GetOptions{})
			Expect(err).ToNot(HaveOccurred())
			return node.Labels
		}, 500*time.Millisecond, 10*time.Millisecond).Should(And(
			HaveKeyWithValue(virtv1.NodeSchedulable, initiallySchedulable),
		))
		Eventually(func() map[string]string {
			node, err := fakeClient.CoreV1().Nodes().Get(context.Background(), "mynode", metav1.GetOptions{})
			Expect(err).ToNot(HaveOccurred())
			return node.Labels
		}).Should(And(
			HaveKeyWithValue(virtv1.NodeSchedulable, finallySchedulable),
		))
		Consistently(func() map[string]string {
			node, err := fakeClient.CoreV1().Nodes().Get(context.Background(), "mynode", metav1.GetOptions{})
			Expect(err).ToNot(HaveOccurred())
			return node.Labels
		}, 500*time.Millisecond, 10*time.Millisecond).Should(And(
			HaveKeyWithValue(virtv1.NodeSchedulable, finallySchedulable),
		))
	},
		table.Entry("not becoming ready, node should be set to unschedulable immediately and stick to it",
			newProbeCountingDeviceController(probe{false, 1000}),
			"false",
			"false",
		),
		table.Entry("becoming ready after a few probes, node should be set to unschedulable immediately and switch earlier than one minute",
			newProbeCountingDeviceController(probe{false, 100}, probe{true, 100}),
			"false",
			"true",
		),
	)
})

type fakeDeviceController struct {
	initialized bool
}

func (f *fakeDeviceController) Initialized() bool {
	return f.initialized
}

func config(featuregates ...string) *virtconfig.ClusterConfig {
	cfg := &virtv1.KubeVirtConfiguration{
		DeveloperConfiguration: &virtv1.DeveloperConfiguration{
			FeatureGates: featuregates,
		},
	}
	clusterConfig, _, _, _ := testutils.NewFakeClusterConfigUsingKVConfig(cfg)
	return clusterConfig
}

func deviceController(initialized bool) device_manager.DeviceControllerInterface {
	return &fakeDeviceController{initialized: initialized}
}

type probeCountingDeviceController struct {
	probes []bool
	probed int
	lock   *sync.Mutex
}

func (f *probeCountingDeviceController) Initialized() bool {
	f.lock.Lock()
	defer f.lock.Unlock()
	f.probed++
	return f.probes[f.probed-1]
}

func newProbeCountingDeviceController(probes ...probe) device_manager.DeviceControllerInterface {
	var probeArray []bool
	for _, p := range probes {
		for x := 0; x < p.repetitions; x++ {
			probeArray = append(probeArray, p.value)
		}
	}
	return &probeCountingDeviceController{probes: probeArray, lock: &sync.Mutex{}}
}

type probe struct {
	value       bool
	repetitions int
}
<commit_msg>Increase device plugin wait timeout<commit_after>package heartbeat

import (
	"context"
	"sync"
	"time"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes/fake"

	virtv1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/kubevirt/pkg/testutils"
	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
	device_manager "kubevirt.io/kubevirt/pkg/virt-handler/device-manager"
)

const (
	cpu_manager_static_path = "testdata/cpu_manager_state_static"
	cpu_manager_none_path   = "testdata/cpu_manager_state_none"
)

var _ = Describe("Heartbeat", func() {

	var node *v1.Node
	var fakeClient *fake.Clientset

	BeforeEach(func() {
		node = &v1.Node{
			ObjectMeta: metav1.ObjectMeta{
				Name: "mynode",
			},
		}
		fakeClient = fake.NewSimpleClientset(node)
	})

	table.DescribeTable("with cpumanager featuregate should set the node to", func(deviceController device_manager.DeviceControllerInterface, cpuManagerPaths []string, schedulable string, cpumanager string) {
		heartbeat := NewHeartBeat(fakeClient.CoreV1(), deviceController, config(virtconfig.CPUManager), "mynode")
		heartbeat.cpuManagerPaths = cpuManagerPaths
		heartbeat.do()
		node, err := fakeClient.CoreV1().Nodes().Get(context.Background(), "mynode", metav1.GetOptions{})
		Expect(err).ToNot(HaveOccurred())
		Expect(node.Labels).To(HaveKeyWithValue(virtv1.NodeSchedulable, schedulable))
		Expect(node.Labels).To(HaveKeyWithValue(virtv1.CPUManager, cpumanager))
	},
		table.Entry("not schedulable and no cpu manager with no cpu manager file and device plugins are not initialized",
			deviceController(false),
			[]string{"non/existent/cpumanager/statefile"},
			"false",
			"false",
		),
		table.Entry("schedulable and no cpu manager with no cpu manager file and plugins are not initialized",
			deviceController(true),
			[]string{"non/existent/cpumanager/statefile"},
			"true",
			"false",
		),
		table.Entry("schedulable and cpu manager with static cpu manager policy configured and device plugins are not initialized",
			deviceController(true),
			[]string{"non/existent/cpumanager/statefile", cpu_manager_static_path},
			"true",
			"true",
		),
		table.Entry("schedulable and no cpu manager with no cpu manager policy configured and device plugins are not initialized",
			deviceController(true),
			[]string{cpu_manager_none_path, "non/existent/cpumanager/statefile"},
			"true",
			"false",
		),
	)

	table.DescribeTable("without cpumanager featuregate should set the node to", func(deviceController device_manager.DeviceControllerInterface, schedulable string) {
		heartbeat := NewHeartBeat(fakeClient.CoreV1(), deviceController, config(), "mynode")
		heartbeat.do()
		node, err := fakeClient.CoreV1().Nodes().Get(context.Background(), "mynode", metav1.GetOptions{})
		Expect(err).ToNot(HaveOccurred())
		Expect(node.Labels).To(HaveKeyWithValue(virtv1.NodeSchedulable, schedulable))
		Expect(node.Labels).ToNot(HaveKeyWithValue(virtv1.CPUManager, false))
	},
		table.Entry("not schedulable with no cpumanager label present",
			deviceController(false),
			"false",
		),
		table.Entry("schedulable with no cpumanger label present",
			deviceController(true),
			"true",
		),
	)

	table.DescribeTable("without deviceplugin and", func(deviceController device_manager.DeviceControllerInterface, initiallySchedulable string, finallySchedulable string) {
		heartbeat := NewHeartBeat(fakeClient.CoreV1(), deviceController, config(), "mynode")
		heartbeat.devicePluginWaitTimeout = 2 * time.Second
		heartbeat.devicePluginPollIntervall = 10 * time.Millisecond
		stopChan := make(chan struct{})
		done := heartbeat.Run(100*time.Second, stopChan)
		defer func() {
			close(stopChan)
			<-done
		}()
		Eventually(func() map[string]string {
			node, err := fakeClient.CoreV1().Nodes().Get(context.Background(), "mynode", metav1.GetOptions{})
			Expect(err).ToNot(HaveOccurred())
			return node.Labels
		}).Should(And(
			HaveKeyWithValue(virtv1.NodeSchedulable, initiallySchedulable),
		))
		Consistently(func() map[string]string {
			node, err := fakeClient.CoreV1().Nodes().Get(context.Background(), "mynode", metav1.GetOptions{})
			Expect(err).ToNot(HaveOccurred())
			return node.Labels
		}, 500*time.Millisecond, 10*time.Millisecond).Should(And(
			HaveKeyWithValue(virtv1.NodeSchedulable, initiallySchedulable),
		))
		Eventually(func() map[string]string {
			node, err := fakeClient.CoreV1().Nodes().Get(context.Background(), "mynode", metav1.GetOptions{})
			Expect(err).ToNot(HaveOccurred())
			return node.Labels
		}).Should(And(
			HaveKeyWithValue(virtv1.NodeSchedulable, finallySchedulable),
		))
		Consistently(func() map[string]string {
			node, err := fakeClient.CoreV1().Nodes().Get(context.Background(), "mynode", metav1.GetOptions{})
			Expect(err).ToNot(HaveOccurred())
			return node.Labels
		}, 500*time.Millisecond, 10*time.Millisecond).Should(And(
			HaveKeyWithValue(virtv1.NodeSchedulable, finallySchedulable),
		))
	},
		table.Entry("not becoming ready, node should be set to unschedulable immediately and stick to it",
			newProbeCountingDeviceController(probe{false, 1000}),
			"false",
			"false",
		),
		table.Entry("becoming ready after a few probes, node should be set to unschedulable immediately and switch earlier than one minute",
			newProbeCountingDeviceController(probe{false, 100}, probe{true, 100}),
			"false",
			"true",
		),
	)
})

type fakeDeviceController struct {
	initialized bool
}

func (f *fakeDeviceController) Initialized() bool {
	return f.initialized
}

func config(featuregates ...string) *virtconfig.ClusterConfig {
	cfg := &virtv1.KubeVirtConfiguration{
		DeveloperConfiguration: &virtv1.DeveloperConfiguration{
			FeatureGates: featuregates,
		},
	}
	clusterConfig, _, _, _ := testutils.NewFakeClusterConfigUsingKVConfig(cfg)
	return clusterConfig
}

func deviceController(initialized bool) device_manager.DeviceControllerInterface {
	return &fakeDeviceController{initialized: initialized}
}

type probeCountingDeviceController struct {
	probes []bool
	probed int
	lock   *sync.Mutex
}

func (f *probeCountingDeviceController) Initialized() bool {
	f.lock.Lock()
	defer f.lock.Unlock()
	f.probed++
	return f.probes[f.probed-1]
}

func newProbeCountingDeviceController(probes ...probe) device_manager.DeviceControllerInterface {
	var probeArray []bool
	for _, p := range probes {
		for x := 0; x < p.repetitions; x++ {
			probeArray = append(probeArray, p.value)
		}
	}
	return &probeCountingDeviceController{probes: probeArray, lock: &sync.Mutex{}}
}

type probe struct {
	value       bool
	repetitions int
}
<|endoftext|>


<commit_before>package sqldb

import (
	"database/sql"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	"code.cloudfoundry.org/auctioneer"
	"code.cloudfoundry.org/bbs/models"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/runtimeschema/metric"
	"github.com/cloudfoundry/gunk/workpool"
)

const (
	convergeLRPRunsCounter = metric.Counter("ConvergenceLRPRuns")
	convergeLRPDuration    = metric.Duration("ConvergenceLRPDuration")

	domainMetricPrefix = "Domain."

	instanceLRPs  = metric.Metric("LRPsDesired") // this is the number of desired instances
	claimedLRPs   = metric.Metric("LRPsClaimed")
	unclaimedLRPs = metric.Metric("LRPsUnclaimed")
	runningLRPs   = metric.Metric("LRPsRunning")

	missingLRPs = metric.Metric("LRPsMissing")
	extraLRPs   = metric.Metric("LRPsExtra")

	crashedActualLRPs   = metric.Metric("CrashedActualLRPs")
	crashingDesiredLRPs = metric.Metric("CrashingDesiredLRPs")
)

func (db *SQLDB) ConvergeLRPs(logger lager.Logger, cellSet models.CellSet) ([]*auctioneer.LRPStartRequest, []*models.ActualLRPKeyWithSchedulingInfo, []*models.ActualLRPKey) {
	convergeStart := db.clock.Now()
	convergeLRPRunsCounter.Increment()
	logger.Info("starting")
	defer logger.Info("completed")

	defer func() {
		err := convergeLRPDuration.Send(time.Since(convergeStart))
		if err != nil {
			logger.Error("failed-sending-converge-lrp-duration-metric", err)
		}
	}()

	now := db.clock.Now()

	db.pruneDomains(logger, now)
	db.pruneEvacuatingActualLRPs(logger, now)

	domainSet, err := db.domainSet(logger)
	if err != nil {
		return nil, nil, nil
	}

	db.emitDomainMetrics(logger, domainSet)

	converge := newConvergence(db)
	converge.staleUnclaimedActualLRPs(logger, now)
	converge.actualLRPsWithMissingCells(logger, cellSet)
	converge.lrpInstanceCounts(logger, domainSet)
	converge.orphanedActualLRPs(logger)
	converge.crashedActualLRPs(logger, now)

	return converge.result(logger)
}

type convergence struct {
	*SQLDB

	guidsToStartRequests map[string]*auctioneer.LRPStartRequest
	startRequestsMutex   sync.Mutex

	keysWithMissingCells []*models.ActualLRPKeyWithSchedulingInfo

	keysToRetire []*models.ActualLRPKey
	keysMutex    sync.Mutex

	pool   *workpool.WorkPool
	poolWg sync.WaitGroup
}

func newConvergence(db *SQLDB) *convergence {
	pool, err := workpool.NewWorkPool(db.convergenceWorkersSize)
	if err != nil {
		panic(fmt.Sprintf("failing to create workpool is irrecoverable %v", err))
	}

	return &convergence{
		SQLDB:                db,
		guidsToStartRequests: map[string]*auctioneer.LRPStartRequest{},
		keysToRetire:         []*models.ActualLRPKey{},
		pool:                 pool,
	}
}

// Adds stale UNCLAIMED Actual LRPs to the list of start requests.
func (c *convergence) staleUnclaimedActualLRPs(logger lager.Logger, now time.Time) {
	logger = logger.Session("stale-unclaimed-actual-lrps")

	rows, err := c.selectStaleUnclaimedLRPs(logger, c.db, now)
	if err != nil {
		logger.Error("failed-query", err)
		return
	}

	for rows.Next() {
		var index int
		schedulingInfo, err := c.fetchDesiredLRPSchedulingInfoAndMore(logger, rows, &index)
		if err == nil {
			c.addStartRequestFromSchedulingInfo(logger, schedulingInfo, index)
		}
	}

	if rows.Err() != nil {
		logger.Error("failed-getting-next-row", rows.Err())
	}

	return
}

// Adds CRASHED Actual LRPs that can be restarted to the list of start requests
// and transitions them to UNCLAIMED.
func (c *convergence) crashedActualLRPs(logger lager.Logger, now time.Time) {
	logger = logger.Session("crashed-actual-lrps")
	restartCalculator := models.NewDefaultRestartCalculator()

	rows, err := c.selectCrashedLRPs(logger, c.db)
	if err != nil {
		logger.Error("failed-query", err)
		return
	}

	for rows.Next() {
		var index int
		actual := &models.ActualLRP{}

		schedulingInfo, err := c.fetchDesiredLRPSchedulingInfoAndMore(logger, rows, &index, &actual.Since, &actual.CrashCount)
		if err != nil {
			continue
		}

		actual.ProcessGuid = schedulingInfo.ProcessGuid
		actual.Domain = schedulingInfo.Domain
		actual.State = models.ActualLRPStateCrashed

		if actual.ShouldRestartCrash(now, restartCalculator) {
			c.submit(func() {
				_, _, err = c.UnclaimActualLRP(logger, &actual.ActualLRPKey)
				if err != nil {
					logger.Error("failed-unclaiming-actual-lrp", err)
					return
				}

				c.addStartRequestFromSchedulingInfo(logger, schedulingInfo, index)
			})
		}
	}

	if rows.Err() != nil {
		logger.Error("failed-getting-next-row", rows.Err())
	}

	return
}

// Adds orphaned Actual LRPs (ones with no corresponding Desired LRP) to the
// list of keys to retire.
func (c *convergence) orphanedActualLRPs(logger lager.Logger) {
	logger = logger.Session("orphaned-actual-lrps")

	rows, err := c.selectOrphanedActualLRPs(logger, c.db)
	if err != nil {
		logger.Error("failed-query", err)
		return
	}

	for rows.Next() {
		actualLRPKey := &models.ActualLRPKey{}

		err := rows.Scan(
			&actualLRPKey.ProcessGuid,
			&actualLRPKey.Index,
			&actualLRPKey.Domain,
		)
		if err != nil {
			logger.Error("failed-scanning", err)
			continue
		}

		c.addKeyToRetire(logger, actualLRPKey)
	}

	if rows.Err() != nil {
		logger.Error("failed-getting-next-row", rows.Err())
	}
}

// Creates and adds missing Actual LRPs to the list of start requests.
// Adds extra Actual LRPs  to the list of keys to retire.
func (c *convergence) lrpInstanceCounts(logger lager.Logger, domainSet map[string]struct{}) {
	logger = logger.Session("lrp-instance-counts")

	rows, err := c.selectLRPInstanceCounts(logger, c.db)
	if err != nil {
		logger.Error("failed-query", err)
		return
	}

	missingLRPCount := 0
	for rows.Next() {
		var existingIndicesStr sql.NullString
		var actualInstances int

		schedulingInfo, err := c.fetchDesiredLRPSchedulingInfoAndMore(logger, rows, &actualInstances, &existingIndicesStr)
		if err != nil {
			continue
		}

		indices := []int{}
		existingIndices := strings.Split(existingIndicesStr.String, ",")

		for i := 0; i < int(schedulingInfo.Instances); i++ {
			found := false
			for _, indexStr := range existingIndices {
				if indexStr == strconv.Itoa(i) {
					found = true
					break
				}
			}
			if !found {
				missingLRPCount++
				indices = append(indices, i)
				index := int32(i)

				c.submit(func() {
					_, err := c.CreateUnclaimedActualLRP(logger, &models.ActualLRPKey{ProcessGuid: schedulingInfo.ProcessGuid, Domain: schedulingInfo.Domain, Index: index})
					if err != nil {
						logger.Error("failed-creating-missing-actual-lrp", err)
					}
				})
			}
		}

		c.addStartRequestFromSchedulingInfo(logger, schedulingInfo, indices...)

		if actualInstances > int(schedulingInfo.Instances) {
			for i := int(schedulingInfo.Instances); i < actualInstances; i++ {
				if _, ok := domainSet[schedulingInfo.Domain]; ok {
					c.addKeyToRetire(logger, &models.ActualLRPKey{
						ProcessGuid: schedulingInfo.ProcessGuid,
						Index:       int32(i),
						Domain:      schedulingInfo.Domain,
					})
				}
			}
		}
	}

	if rows.Err() != nil {
		logger.Error("failed-getting-next-row", rows.Err())
	}

	missingLRPs.Send(missingLRPCount)
}

// Unclaim Actual LRPs that have missing cells (not in the cell set passed to
// convergence) and add them to the list of start requests.
func (c *convergence) actualLRPsWithMissingCells(logger lager.Logger, cellSet models.CellSet) {
	// time.Sleep(1000 * time.Second)
	logger = logger.Session("actual-lrps-with-missing-cells")

	keysWithMissingCells := make([]*models.ActualLRPKeyWithSchedulingInfo, 0)

	rows, err := c.selectLRPsWithMissingCells(logger, c.db, cellSet)
	if err != nil {
		logger.Error("failed-query", err)
		return
	}

	for rows.Next() {
		var index int32
		schedulingInfo, err := c.fetchDesiredLRPSchedulingInfoAndMore(logger, rows, &index)
		if err == nil {
			keysWithMissingCells = append(keysWithMissingCells, &models.ActualLRPKeyWithSchedulingInfo{
				Key: &models.ActualLRPKey{
					ProcessGuid: schedulingInfo.ProcessGuid,
					Domain:      schedulingInfo.Domain,
					Index:       index,
				},
				SchedulingInfo: schedulingInfo,
			})
		}
	}

	if rows.Err() != nil {
		logger.Error("failed-getting-next-row", rows.Err())
	}

	c.keysWithMissingCells = keysWithMissingCells
}

func (c *convergence) addStartRequestFromSchedulingInfo(logger lager.Logger, schedulingInfo *models.DesiredLRPSchedulingInfo, indices ...int) {
	if len(indices) == 0 {
		return
	}

	c.startRequestsMutex.Lock()
	defer c.startRequestsMutex.Unlock()

	if startRequest, ok := c.guidsToStartRequests[schedulingInfo.ProcessGuid]; ok {
		startRequest.Indices = append(startRequest.Indices, indices...)
		return
	}

	startRequest := auctioneer.NewLRPStartRequestFromSchedulingInfo(schedulingInfo, indices...)
	c.guidsToStartRequests[schedulingInfo.ProcessGuid] = &startRequest
}

func (c *convergence) addKeyToRetire(logger lager.Logger, key *models.ActualLRPKey) {
	c.keysMutex.Lock()
	defer c.keysMutex.Unlock()

	c.keysToRetire = append(c.keysToRetire, key)
}

func (c *convergence) submit(work func()) {
	c.poolWg.Add(1)
	c.pool.Submit(func() {
		defer c.poolWg.Done()
		work()
	})
}

func (c *convergence) result(logger lager.Logger) ([]*auctioneer.LRPStartRequest, []*models.ActualLRPKeyWithSchedulingInfo, []*models.ActualLRPKey) {
	c.poolWg.Wait()
	c.startRequestsMutex.Lock()
	defer c.startRequestsMutex.Unlock()
	c.keysMutex.Lock()
	defer c.keysMutex.Unlock()

	startRequests := make([]*auctioneer.LRPStartRequest, 0, len(c.guidsToStartRequests))
	for _, startRequest := range c.guidsToStartRequests {
		startRequests = append(startRequests, startRequest)
	}

	extraLRPs.Send(len(c.keysToRetire))
	c.emitLRPMetrics(logger)

	return startRequests, c.keysWithMissingCells, c.keysToRetire
}

func (db *SQLDB) pruneDomains(logger lager.Logger, now time.Time) {
	logger = logger.Session("prune-domains")

	_, err := db.delete(logger, db.db, domainsTable, "expire_time <= ?", now.UnixNano())
	if err != nil {
		logger.Error("failed-query", err)
	}
}

func (db *SQLDB) pruneEvacuatingActualLRPs(logger lager.Logger, now time.Time) {
	logger = logger.Session("prune-evacuating-actual-lrps")

	_, err := db.delete(logger, db.db, actualLRPsTable, "evacuating = ? AND expire_time <= ?", true, now.UnixNano())
	if err != nil {
		logger.Error("failed-query", err)
	}
}

func (db *SQLDB) domainSet(logger lager.Logger) (map[string]struct{}, error) {
	logger.Debug("listing-domains")
	domains, err := db.Domains(logger)
	if err != nil {
		logger.Error("failed-listing-domains", err)
		return nil, err
	}
	logger.Debug("succeeded-listing-domains")
	m := make(map[string]struct{}, len(domains))
	for _, domain := range domains {
		m[domain] = struct{}{}
	}
	return m, nil
}

func (db *SQLDB) emitDomainMetrics(logger lager.Logger, domainSet map[string]struct{}) {
	for domain := range domainSet {
		metric.Metric("Domain." + domain).Send(1)
	}
}

func (db *SQLDB) emitLRPMetrics(logger lager.Logger) {
	var err error
	logger = logger.Session("emit-lrp-metrics")
	claimedInstances, unclaimedInstances, runningInstances, crashedInstances, crashingDesireds := db.countActualLRPsByState(logger, db.db)

	desiredInstances := db.countDesiredInstances(logger, db.db)

	err = unclaimedLRPs.Send(unclaimedInstances)
	if err != nil {
		logger.Error("failed-sending-unclaimed-lrps-metric", err)
	}

	err = claimedLRPs.Send(claimedInstances)
	if err != nil {
		logger.Error("failed-sending-claimed-lrps-metric", err)
	}

	err = runningLRPs.Send(runningInstances)
	if err != nil {
		logger.Error("failed-sending-running-lrps-metric", err)
	}

	err = crashedActualLRPs.Send(crashedInstances)
	if err != nil {
		logger.Error("failed-sending-crashed-actual-lrps-metric", err)
	}

	err = crashingDesiredLRPs.Send(crashingDesireds)
	if err != nil {
		logger.Error("failed-sending-crashing-desired-lrps-metric", err)
	}

	err = instanceLRPs.Send(desiredInstances)
	if err != nil {
		logger.Error("failed-sending-desired-lrps-metric", err)
	}
}

func (db *SQLDB) GatherAndPruneLRPs(logger lager.Logger, cellSet models.CellSet) (*models.ConvergenceInput, error) {
	panic("not implemented")
}
<commit_msg>stop convergence workpool after it is done executing<commit_after>package sqldb

import (
	"database/sql"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	"code.cloudfoundry.org/auctioneer"
	"code.cloudfoundry.org/bbs/models"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/runtimeschema/metric"
	"github.com/cloudfoundry/gunk/workpool"
)

const (
	convergeLRPRunsCounter = metric.Counter("ConvergenceLRPRuns")
	convergeLRPDuration    = metric.Duration("ConvergenceLRPDuration")

	domainMetricPrefix = "Domain."

	instanceLRPs  = metric.Metric("LRPsDesired") // this is the number of desired instances
	claimedLRPs   = metric.Metric("LRPsClaimed")
	unclaimedLRPs = metric.Metric("LRPsUnclaimed")
	runningLRPs   = metric.Metric("LRPsRunning")

	missingLRPs = metric.Metric("LRPsMissing")
	extraLRPs   = metric.Metric("LRPsExtra")

	crashedActualLRPs   = metric.Metric("CrashedActualLRPs")
	crashingDesiredLRPs = metric.Metric("CrashingDesiredLRPs")
)

func (db *SQLDB) ConvergeLRPs(logger lager.Logger, cellSet models.CellSet) ([]*auctioneer.LRPStartRequest, []*models.ActualLRPKeyWithSchedulingInfo, []*models.ActualLRPKey) {
	convergeStart := db.clock.Now()
	convergeLRPRunsCounter.Increment()
	logger.Info("starting")
	defer logger.Info("completed")

	defer func() {
		err := convergeLRPDuration.Send(time.Since(convergeStart))
		if err != nil {
			logger.Error("failed-sending-converge-lrp-duration-metric", err)
		}
	}()

	now := db.clock.Now()

	db.pruneDomains(logger, now)
	db.pruneEvacuatingActualLRPs(logger, now)

	domainSet, err := db.domainSet(logger)
	if err != nil {
		return nil, nil, nil
	}

	db.emitDomainMetrics(logger, domainSet)

	converge := newConvergence(db)
	converge.staleUnclaimedActualLRPs(logger, now)
	converge.actualLRPsWithMissingCells(logger, cellSet)
	converge.lrpInstanceCounts(logger, domainSet)
	converge.orphanedActualLRPs(logger)
	converge.crashedActualLRPs(logger, now)

	return converge.result(logger)
}

type convergence struct {
	*SQLDB

	guidsToStartRequests map[string]*auctioneer.LRPStartRequest
	startRequestsMutex   sync.Mutex

	keysWithMissingCells []*models.ActualLRPKeyWithSchedulingInfo

	keysToRetire []*models.ActualLRPKey
	keysMutex    sync.Mutex

	pool   *workpool.WorkPool
	poolWg sync.WaitGroup
}

func newConvergence(db *SQLDB) *convergence {
	pool, err := workpool.NewWorkPool(db.convergenceWorkersSize)
	if err != nil {
		panic(fmt.Sprintf("failing to create workpool is irrecoverable %v", err))
	}

	return &convergence{
		SQLDB:                db,
		guidsToStartRequests: map[string]*auctioneer.LRPStartRequest{},
		keysToRetire:         []*models.ActualLRPKey{},
		pool:                 pool,
	}
}

// Adds stale UNCLAIMED Actual LRPs to the list of start requests.
func (c *convergence) staleUnclaimedActualLRPs(logger lager.Logger, now time.Time) {
	logger = logger.Session("stale-unclaimed-actual-lrps")

	rows, err := c.selectStaleUnclaimedLRPs(logger, c.db, now)
	if err != nil {
		logger.Error("failed-query", err)
		return
	}

	for rows.Next() {
		var index int
		schedulingInfo, err := c.fetchDesiredLRPSchedulingInfoAndMore(logger, rows, &index)
		if err == nil {
			c.addStartRequestFromSchedulingInfo(logger, schedulingInfo, index)
		}
	}

	if rows.Err() != nil {
		logger.Error("failed-getting-next-row", rows.Err())
	}

	return
}

// Adds CRASHED Actual LRPs that can be restarted to the list of start requests
// and transitions them to UNCLAIMED.
func (c *convergence) crashedActualLRPs(logger lager.Logger, now time.Time) {
	logger = logger.Session("crashed-actual-lrps")
	restartCalculator := models.NewDefaultRestartCalculator()

	rows, err := c.selectCrashedLRPs(logger, c.db)
	if err != nil {
		logger.Error("failed-query", err)
		return
	}

	for rows.Next() {
		var index int
		actual := &models.ActualLRP{}

		schedulingInfo, err := c.fetchDesiredLRPSchedulingInfoAndMore(logger, rows, &index, &actual.Since, &actual.CrashCount)
		if err != nil {
			continue
		}

		actual.ProcessGuid = schedulingInfo.ProcessGuid
		actual.Domain = schedulingInfo.Domain
		actual.State = models.ActualLRPStateCrashed

		if actual.ShouldRestartCrash(now, restartCalculator) {
			c.submit(func() {
				_, _, err = c.UnclaimActualLRP(logger, &actual.ActualLRPKey)
				if err != nil {
					logger.Error("failed-unclaiming-actual-lrp", err)
					return
				}

				c.addStartRequestFromSchedulingInfo(logger, schedulingInfo, index)
			})
		}
	}

	if rows.Err() != nil {
		logger.Error("failed-getting-next-row", rows.Err())
	}

	return
}

// Adds orphaned Actual LRPs (ones with no corresponding Desired LRP) to the
// list of keys to retire.
func (c *convergence) orphanedActualLRPs(logger lager.Logger) {
	logger = logger.Session("orphaned-actual-lrps")

	rows, err := c.selectOrphanedActualLRPs(logger, c.db)
	if err != nil {
		logger.Error("failed-query", err)
		return
	}

	for rows.Next() {
		actualLRPKey := &models.ActualLRPKey{}

		err := rows.Scan(
			&actualLRPKey.ProcessGuid,
			&actualLRPKey.Index,
			&actualLRPKey.Domain,
		)
		if err != nil {
			logger.Error("failed-scanning", err)
			continue
		}

		c.addKeyToRetire(logger, actualLRPKey)
	}

	if rows.Err() != nil {
		logger.Error("failed-getting-next-row", rows.Err())
	}
}

// Creates and adds missing Actual LRPs to the list of start requests.
// Adds extra Actual LRPs  to the list of keys to retire.
func (c *convergence) lrpInstanceCounts(logger lager.Logger, domainSet map[string]struct{}) {
	logger = logger.Session("lrp-instance-counts")

	rows, err := c.selectLRPInstanceCounts(logger, c.db)
	if err != nil {
		logger.Error("failed-query", err)
		return
	}

	missingLRPCount := 0
	for rows.Next() {
		var existingIndicesStr sql.NullString
		var actualInstances int

		schedulingInfo, err := c.fetchDesiredLRPSchedulingInfoAndMore(logger, rows, &actualInstances, &existingIndicesStr)
		if err != nil {
			continue
		}

		indices := []int{}
		existingIndices := strings.Split(existingIndicesStr.String, ",")

		for i := 0; i < int(schedulingInfo.Instances); i++ {
			found := false
			for _, indexStr := range existingIndices {
				if indexStr == strconv.Itoa(i) {
					found = true
					break
				}
			}
			if !found {
				missingLRPCount++
				indices = append(indices, i)
				index := int32(i)

				c.submit(func() {
					_, err := c.CreateUnclaimedActualLRP(logger, &models.ActualLRPKey{ProcessGuid: schedulingInfo.ProcessGuid, Domain: schedulingInfo.Domain, Index: index})
					if err != nil {
						logger.Error("failed-creating-missing-actual-lrp", err)
					}
				})
			}
		}

		c.addStartRequestFromSchedulingInfo(logger, schedulingInfo, indices...)

		if actualInstances > int(schedulingInfo.Instances) {
			for i := int(schedulingInfo.Instances); i < actualInstances; i++ {
				if _, ok := domainSet[schedulingInfo.Domain]; ok {
					c.addKeyToRetire(logger, &models.ActualLRPKey{
						ProcessGuid: schedulingInfo.ProcessGuid,
						Index:       int32(i),
						Domain:      schedulingInfo.Domain,
					})
				}
			}
		}
	}

	if rows.Err() != nil {
		logger.Error("failed-getting-next-row", rows.Err())
	}

	missingLRPs.Send(missingLRPCount)
}

// Unclaim Actual LRPs that have missing cells (not in the cell set passed to
// convergence) and add them to the list of start requests.
func (c *convergence) actualLRPsWithMissingCells(logger lager.Logger, cellSet models.CellSet) {
	// time.Sleep(1000 * time.Second)
	logger = logger.Session("actual-lrps-with-missing-cells")

	keysWithMissingCells := make([]*models.ActualLRPKeyWithSchedulingInfo, 0)

	rows, err := c.selectLRPsWithMissingCells(logger, c.db, cellSet)
	if err != nil {
		logger.Error("failed-query", err)
		return
	}

	for rows.Next() {
		var index int32
		schedulingInfo, err := c.fetchDesiredLRPSchedulingInfoAndMore(logger, rows, &index)
		if err == nil {
			keysWithMissingCells = append(keysWithMissingCells, &models.ActualLRPKeyWithSchedulingInfo{
				Key: &models.ActualLRPKey{
					ProcessGuid: schedulingInfo.ProcessGuid,
					Domain:      schedulingInfo.Domain,
					Index:       index,
				},
				SchedulingInfo: schedulingInfo,
			})
		}
	}

	if rows.Err() != nil {
		logger.Error("failed-getting-next-row", rows.Err())
	}

	c.keysWithMissingCells = keysWithMissingCells
}

func (c *convergence) addStartRequestFromSchedulingInfo(logger lager.Logger, schedulingInfo *models.DesiredLRPSchedulingInfo, indices ...int) {
	if len(indices) == 0 {
		return
	}

	c.startRequestsMutex.Lock()
	defer c.startRequestsMutex.Unlock()

	if startRequest, ok := c.guidsToStartRequests[schedulingInfo.ProcessGuid]; ok {
		startRequest.Indices = append(startRequest.Indices, indices...)
		return
	}

	startRequest := auctioneer.NewLRPStartRequestFromSchedulingInfo(schedulingInfo, indices...)
	c.guidsToStartRequests[schedulingInfo.ProcessGuid] = &startRequest
}

func (c *convergence) addKeyToRetire(logger lager.Logger, key *models.ActualLRPKey) {
	c.keysMutex.Lock()
	defer c.keysMutex.Unlock()

	c.keysToRetire = append(c.keysToRetire, key)
}

func (c *convergence) submit(work func()) {
	c.poolWg.Add(1)
	c.pool.Submit(func() {
		defer c.poolWg.Done()
		work()
	})
}

func (c *convergence) result(logger lager.Logger) ([]*auctioneer.LRPStartRequest, []*models.ActualLRPKeyWithSchedulingInfo, []*models.ActualLRPKey) {
	c.poolWg.Wait()
	c.pool.Stop()

	c.startRequestsMutex.Lock()
	defer c.startRequestsMutex.Unlock()

	c.keysMutex.Lock()
	defer c.keysMutex.Unlock()

	startRequests := make([]*auctioneer.LRPStartRequest, 0, len(c.guidsToStartRequests))
	for _, startRequest := range c.guidsToStartRequests {
		startRequests = append(startRequests, startRequest)
	}

	extraLRPs.Send(len(c.keysToRetire))
	c.emitLRPMetrics(logger)

	return startRequests, c.keysWithMissingCells, c.keysToRetire
}

func (db *SQLDB) pruneDomains(logger lager.Logger, now time.Time) {
	logger = logger.Session("prune-domains")

	_, err := db.delete(logger, db.db, domainsTable, "expire_time <= ?", now.UnixNano())
	if err != nil {
		logger.Error("failed-query", err)
	}
}

func (db *SQLDB) pruneEvacuatingActualLRPs(logger lager.Logger, now time.Time) {
	logger = logger.Session("prune-evacuating-actual-lrps")

	_, err := db.delete(logger, db.db, actualLRPsTable, "evacuating = ? AND expire_time <= ?", true, now.UnixNano())
	if err != nil {
		logger.Error("failed-query", err)
	}
}

func (db *SQLDB) domainSet(logger lager.Logger) (map[string]struct{}, error) {
	logger.Debug("listing-domains")
	domains, err := db.Domains(logger)
	if err != nil {
		logger.Error("failed-listing-domains", err)
		return nil, err
	}
	logger.Debug("succeeded-listing-domains")
	m := make(map[string]struct{}, len(domains))
	for _, domain := range domains {
		m[domain] = struct{}{}
	}
	return m, nil
}

func (db *SQLDB) emitDomainMetrics(logger lager.Logger, domainSet map[string]struct{}) {
	for domain := range domainSet {
		metric.Metric("Domain." + domain).Send(1)
	}
}

func (db *SQLDB) emitLRPMetrics(logger lager.Logger) {
	var err error
	logger = logger.Session("emit-lrp-metrics")
	claimedInstances, unclaimedInstances, runningInstances, crashedInstances, crashingDesireds := db.countActualLRPsByState(logger, db.db)

	desiredInstances := db.countDesiredInstances(logger, db.db)

	err = unclaimedLRPs.Send(unclaimedInstances)
	if err != nil {
		logger.Error("failed-sending-unclaimed-lrps-metric", err)
	}

	err = claimedLRPs.Send(claimedInstances)
	if err != nil {
		logger.Error("failed-sending-claimed-lrps-metric", err)
	}

	err = runningLRPs.Send(runningInstances)
	if err != nil {
		logger.Error("failed-sending-running-lrps-metric", err)
	}

	err = crashedActualLRPs.Send(crashedInstances)
	if err != nil {
		logger.Error("failed-sending-crashed-actual-lrps-metric", err)
	}

	err = crashingDesiredLRPs.Send(crashingDesireds)
	if err != nil {
		logger.Error("failed-sending-crashing-desired-lrps-metric", err)
	}

	err = instanceLRPs.Send(desiredInstances)
	if err != nil {
		logger.Error("failed-sending-desired-lrps-metric", err)
	}
}

func (db *SQLDB) GatherAndPruneLRPs(logger lager.Logger, cellSet models.CellSet) (*models.ConvergenceInput, error) {
	panic("not implemented")
}
<|endoftext|>


<commit_before>package controller

import (
	"github.com/cuigh/auxo/errors"
	"github.com/cuigh/auxo/net/web"
	"github.com/cuigh/swirl/biz"
	"github.com/cuigh/swirl/biz/docker"
	"github.com/cuigh/swirl/model"
)

// HomeController is a basic controller of site
type HomeController struct {
	Index    web.HandlerFunc `path:"/" name:"index" authorize:"?" desc:"index page"`
	Login    web.HandlerFunc `path:"/login" name:"login" authorize:"*" desc:"sign in page"`
	InitGet  web.HandlerFunc `path:"/init" name:"init" authorize:"*" desc:"initialize page"`
	InitPost web.HandlerFunc `path:"/init" method:"post" name:"init" authorize:"*" desc:"initialize system"`
	Error403 web.HandlerFunc `path:"/403" name:"403" authorize:"?" desc:"403 page"`
	Error404 web.HandlerFunc `path:"/404" name:"404" authorize:"*" desc:"404 page"`
}

// Home creates an instance of HomeController
func Home() (c *HomeController) {
	return &HomeController{
		Index:    homeIndex,
		Login:    homeLogin,
		InitGet:  homeInitGet,
		InitPost: homeInitPost,
		Error403: homeError403,
		Error404: homeError404,
	}
}

func homeIndex(ctx web.Context) (err error) {
	var (
		count int
		m     = newModel(ctx)
	)

	if count, err = docker.NodeCount(); err != nil {
		return
	}
	m.Set("NodeCount", count)

	if count, err = docker.NetworkCount(); err != nil {
		return
	}
	m.Set("NetworkCount", count)

	if count, err = docker.ServiceCount(); err != nil {
		return
	}
	m.Set("ServiceCount", count)

	if count, err = docker.StackCount(); err != nil {
		return
	}
	m.Set("StackCount", count)

	return ctx.Render("index", m)
}

func homeLogin(ctx web.Context) error {
	count, err := biz.User.Count()
	if err != nil {
		return err
	} else if count == 0 {
		return ctx.Redirect("init")
	}
	if ctx.User() != nil {
		u := ctx.Q("from")
		if u == "" {
			u = "/"
		}
		return ctx.Redirect(u)
	}
	return ctx.Render("login", nil)
}

func homeInitGet(ctx web.Context) error {
	count, err := biz.User.Count()
	if err != nil {
		return err
	} else if count > 0 {
		return ctx.Redirect("login")
	}
	return ctx.Render("init", nil)
}

func homeInitPost(ctx web.Context) error {
	count, err := biz.User.Count()
	if err != nil {
		return err
	} else if count > 0 {
		return errors.New("Swirl was already initialized")
	}

	user := &model.User{}
	err = ctx.Bind(user)
	if err != nil {
		return err
	}

	user.Admin = true
	user.Type = model.UserTypeInternal
	err = biz.User.Create(user, nil)
	return ajaxResult(ctx, err)
}

func homeError403(ctx web.Context) error {
	return ctx.Render("403", nil)
}

func homeError404(ctx web.Context) error {
	return ctx.Render("404", nil)
}
<commit_msg>Rename handlers<commit_after>package controller

import (
	"github.com/cuigh/auxo/errors"
	"github.com/cuigh/auxo/net/web"
	"github.com/cuigh/swirl/biz"
	"github.com/cuigh/swirl/biz/docker"
	"github.com/cuigh/swirl/model"
)

// HomeController is a basic controller of site
type HomeController struct {
	Index    web.HandlerFunc `path:"/" name:"index" authorize:"?" desc:"index page"`
	Login    web.HandlerFunc `path:"/login" name:"login.view" authorize:"*" desc:"sign in page"`
	InitGet  web.HandlerFunc `path:"/init" name:"init.view" authorize:"*" desc:"initialize page"`
	InitPost web.HandlerFunc `path:"/init" name:"init" method:"post" authorize:"*" desc:"initialize system"`
	Error403 web.HandlerFunc `path:"/403" name:"403" authorize:"?" desc:"403 page"`
	Error404 web.HandlerFunc `path:"/404" name:"404" authorize:"*" desc:"404 page"`
}

// Home creates an instance of HomeController
func Home() (c *HomeController) {
	return &HomeController{
		Index:    homeIndex,
		Login:    homeLogin,
		InitGet:  homeInitGet,
		InitPost: homeInitPost,
		Error403: homeError403,
		Error404: homeError404,
	}
}

func homeIndex(ctx web.Context) (err error) {
	var (
		count int
		m     = newModel(ctx)
	)

	if count, err = docker.NodeCount(); err != nil {
		return
	}
	m.Set("NodeCount", count)

	if count, err = docker.NetworkCount(); err != nil {
		return
	}
	m.Set("NetworkCount", count)

	if count, err = docker.ServiceCount(); err != nil {
		return
	}
	m.Set("ServiceCount", count)

	if count, err = docker.StackCount(); err != nil {
		return
	}
	m.Set("StackCount", count)

	return ctx.Render("index", m)
}

func homeLogin(ctx web.Context) error {
	count, err := biz.User.Count()
	if err != nil {
		return err
	} else if count == 0 {
		return ctx.Redirect("init")
	}
	if ctx.User() != nil {
		u := ctx.Q("from")
		if u == "" {
			u = "/"
		}
		return ctx.Redirect(u)
	}
	return ctx.Render("login", nil)
}

func homeInitGet(ctx web.Context) error {
	count, err := biz.User.Count()
	if err != nil {
		return err
	} else if count > 0 {
		return ctx.Redirect("login")
	}
	return ctx.Render("init", nil)
}

func homeInitPost(ctx web.Context) error {
	count, err := biz.User.Count()
	if err != nil {
		return err
	} else if count > 0 {
		return errors.New("Swirl was already initialized")
	}

	user := &model.User{}
	err = ctx.Bind(user)
	if err != nil {
		return err
	}

	user.Admin = true
	user.Type = model.UserTypeInternal
	err = biz.User.Create(user, nil)
	return ajaxResult(ctx, err)
}

func homeError403(ctx web.Context) error {
	return ctx.Render("403", nil)
}

func homeError404(ctx web.Context) error {
	return ctx.Render("404", nil)
}
<|endoftext|>


<commit_before>package agent

import (
	"reflect"
	"sync"

	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/log"
	"golang.org/x/net/context"
)

// StatusReporter receives updates to task status. Method may be called
// concurrently, so implementations should be goroutine-safe.
type StatusReporter interface {
	UpdateTaskStatus(ctx context.Context, taskID string, status *api.TaskStatus) error
}

type statusReporterFunc func(ctx context.Context, taskID string, status *api.TaskStatus) error

func (fn statusReporterFunc) UpdateTaskStatus(ctx context.Context, taskID string, status *api.TaskStatus) error {
	return fn(ctx, taskID, status)
}

// statusReporter creates a reliable StatusReporter that will always succeed.
// It handles several tasks at once, ensuring all statuses are reported.
//
// The reporter will continue reporting the current status until it succeeds.
type statusReporter struct {
	reporter StatusReporter
	statuses map[string]*api.TaskStatus
	mu       sync.Mutex
	cond     sync.Cond
	closed   bool
}

func newStatusReporter(ctx context.Context, upstream StatusReporter) *statusReporter {
	r := &statusReporter{
		reporter: upstream,
		statuses: make(map[string]*api.TaskStatus),
	}

	r.cond.L = &r.mu

	go r.run(ctx)
	return r
}

func (sr *statusReporter) UpdateTaskStatus(ctx context.Context, taskID string, status *api.TaskStatus) error {
	sr.mu.Lock()
	defer sr.mu.Unlock()

	current, ok := sr.statuses[taskID]
	if ok {
		if reflect.DeepEqual(current, status) {
			return nil
		}

		if current.State > status.State {
			return nil // ignore old updates
		}
	}
	sr.statuses[taskID] = status
	sr.cond.Signal()

	return nil
}

func (sr *statusReporter) Close() error {
	sr.mu.Lock()
	defer sr.mu.Unlock()

	sr.closed = true
	sr.cond.Signal()

	return nil
}

func (sr *statusReporter) run(ctx context.Context) {
	done := make(chan struct{})
	defer close(done)

	go func() {
		select {
		case <-ctx.Done():
			sr.Close()
		case <-done:
			return
		}
	}()

	sr.mu.Lock() // released during wait, below.
	defer sr.mu.Unlock()

	for {
		if len(sr.statuses) == 0 {
			sr.cond.Wait()
		}

		for taskID, status := range sr.statuses {
			if sr.closed {
				// TODO(stevvooe): Add support here for waiting until all
				// statuses are flushed before shutting down.
				return
			}

			delete(sr.statuses, taskID) // delete the entry, while trying to send.

			sr.mu.Unlock()
			err := sr.reporter.UpdateTaskStatus(ctx, taskID, status)
			sr.mu.Lock()

			if err != nil {
				log.G(ctx).WithError(err).Error("failed reporting status to agent")

				// place it back in the map, if not there, allowing us to pick
				// the value if a new one came in when we were sending the last
				// update.
				if _, ok := sr.statuses[taskID]; !ok {
					sr.statuses[taskID] = status
				}
			}
		}
	}
}
<commit_msg>agent: fix leak in reporter on stop<commit_after>package agent

import (
	"reflect"
	"sync"

	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/log"
	"golang.org/x/net/context"
)

// StatusReporter receives updates to task status. Method may be called
// concurrently, so implementations should be goroutine-safe.
type StatusReporter interface {
	UpdateTaskStatus(ctx context.Context, taskID string, status *api.TaskStatus) error
}

type statusReporterFunc func(ctx context.Context, taskID string, status *api.TaskStatus) error

func (fn statusReporterFunc) UpdateTaskStatus(ctx context.Context, taskID string, status *api.TaskStatus) error {
	return fn(ctx, taskID, status)
}

// statusReporter creates a reliable StatusReporter that will always succeed.
// It handles several tasks at once, ensuring all statuses are reported.
//
// The reporter will continue reporting the current status until it succeeds.
type statusReporter struct {
	reporter StatusReporter
	statuses map[string]*api.TaskStatus
	mu       sync.Mutex
	cond     sync.Cond
	closed   bool
}

func newStatusReporter(ctx context.Context, upstream StatusReporter) *statusReporter {
	r := &statusReporter{
		reporter: upstream,
		statuses: make(map[string]*api.TaskStatus),
	}

	r.cond.L = &r.mu

	go r.run(ctx)
	return r
}

func (sr *statusReporter) UpdateTaskStatus(ctx context.Context, taskID string, status *api.TaskStatus) error {
	sr.mu.Lock()
	defer sr.mu.Unlock()

	current, ok := sr.statuses[taskID]
	if ok {
		if reflect.DeepEqual(current, status) {
			return nil
		}

		if current.State > status.State {
			return nil // ignore old updates
		}
	}
	sr.statuses[taskID] = status
	sr.cond.Signal()

	return nil
}

func (sr *statusReporter) Close() error {
	sr.mu.Lock()
	defer sr.mu.Unlock()

	sr.closed = true
	sr.cond.Signal()

	return nil
}

func (sr *statusReporter) run(ctx context.Context) {
	done := make(chan struct{})
	defer close(done)

	sr.mu.Lock() // released during wait, below.
	defer sr.mu.Unlock()

	go func() {
		select {
		case <-ctx.Done():
			sr.Close()
		case <-done:
			return
		}
	}()

	for {
		if len(sr.statuses) == 0 {
			sr.cond.Wait()
		}

		if sr.closed {
			// TODO(stevvooe): Add support here for waiting until all
			// statuses are flushed before shutting down.
			return
		}

		for taskID, status := range sr.statuses {
			delete(sr.statuses, taskID) // delete the entry, while trying to send.

			sr.mu.Unlock()
			err := sr.reporter.UpdateTaskStatus(ctx, taskID, status)
			sr.mu.Lock()

			// reporter might be closed during UpdateTaskStatus call
			if sr.closed {
				return
			}

			if err != nil {
				log.G(ctx).WithError(err).Error("failed reporting status to agent")

				// place it back in the map, if not there, allowing us to pick
				// the value if a new one came in when we were sending the last
				// update.
				if _, ok := sr.statuses[taskID]; !ok {
					sr.statuses[taskID] = status
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package iot

import (
	"time"
	. "github.com/influxdata/influxdb-comparisons/bulk_data_gen/common"
)

var (
	DoorByteString      = []byte("door_state")       // heap optimization
)

var (
	// Field keys for 'air condition indoor' points.
	DoorFieldKeys = [][]byte{
		[]byte("state"),
		[]byte("battery_voltage"),

	}
)

type DoorMeasurement struct {
	sensorId 	[]byte
	timestamp     time.Time
	distributions []Distribution
}

func NewDoorMeasurement(start time.Time, id []byte) *DoorMeasurement {
	distributions := make([]Distribution, len(DoorFieldKeys))
	//state
	distributions[0] = TSD(0,1,0)
	//battery_voltage
	distributions[1] = MUDWD(ND(1,0.5), 1, 3.2, 3.2 )

	return &DoorMeasurement{
		timestamp:   start,
		distributions: distributions,
		sensorId: id,
	}
}

func (m *DoorMeasurement) Tick(d time.Duration) {
	m.timestamp = m.timestamp.Add(d)
	for i := range m.distributions {
		m.distributions[i].Advance()
	}
}

func (m *DoorMeasurement) ToPoint(p *Point) {
	p.SetMeasurementName(DoorByteString)
	p.SetTimestamp(&m.timestamp)

	for i := range m.distributions {
		p.AppendField(DoorFieldKeys[i], m.distributions[i].Get())
	}
}
<commit_msg>Improved variables<commit_after>package iot

import (
	. "github.com/influxdata/influxdb-comparisons/bulk_data_gen/common"
	"time"
)

var (
	DoorByteString = []byte("door_state") // heap optimization
	DoorTagKey     = []byte("door")
)

var (
	// Field keys for 'air condition indoor' points.
	DoorFieldKeys = [][]byte{
		[]byte("state"),
		[]byte("battery_voltage"),
	}
)

type DoorMeasurement struct {
	sensorId      []byte
	doorId        []byte
	timestamp     time.Time
	distributions []Distribution
}

func NewDoorMeasurement(start time.Time, doorId []byte, sendorId []byte) *DoorMeasurement {
	distributions := make([]Distribution, len(DoorFieldKeys))
	//state
	distributions[0] = TSD(0, 1, 0)
	//battery_voltage
	distributions[1] = MUDWD(ND(1, 0.5), 1, 3.2, 3.2)

	return &DoorMeasurement{
		timestamp:     start,
		distributions: distributions,
		sensorId:      sendorId,
		doorId:        doorId,
	}
}

func (m *DoorMeasurement) Tick(d time.Duration) {
	m.timestamp = m.timestamp.Add(d)
	for i := range m.distributions {
		m.distributions[i].Advance()
	}
}

func (m *DoorMeasurement) ToPoint(p *Point) {
	p.SetMeasurementName(DoorByteString)
	p.SetTimestamp(&m.timestamp)
	p.AppendTag(DoorTagKey, m.doorId)
	for i := range m.distributions {
		p.AppendField(DoorFieldKeys[i], m.distributions[i].Get())
	}
}
<|endoftext|>


<commit_before>package main

import "github.com/kormoc/unit/datarate"
import "github.com/ncw/directio"
import "io"
import "os"
import "sync"
import "time"

var workerIOJobs chan job
var workerIOJobswg sync.WaitGroup

func initWorkerIO() {
	workerIOJobs = make(chan job, workerCountIO*2)
	workerIOJobswg.Add(workerCountIO)
	for i := 0; i < workerCountIO; i++ {
		go workerIO()
	}
	go func() {
		workerIOJobswg.Wait()
		close(workerEndJobs)

	}()
}

func workerIO() {
	defer workerIOJobswg.Done()
	for currentJob := range workerIOJobs {
		err := func() error {
			time_start := time.Now()
			Trace.Printf("%v: IO Processing...\n", currentJob.path)
			// Try direct io. Fail back to normal IO if needed
			fp, err := directio.OpenFile(currentJob.path, os.O_RDONLY, 0000)
			if err != nil {
				fp, err = os.OpenFile(currentJob.path, os.O_RDONLY, 0000)
				if err != nil {
					return err
				}
			}

			defer fp.Close()

			buffer := directio.AlignedBlock(directio.BlockSize * bufferSize)
			totalRead := 0

			for {
				amountRead, err := io.ReadAtLeast(fp, buffer, 0)
				totalRead += amountRead
				if err == io.EOF {
					break
				}
				if err != nil {
					return err
				}

				for checksumAlgo := range currentJob.hashers {
					currentJob.hashers[checksumAlgo].Write(buffer[:amountRead])
				}
			}

			duration := time.Since(time_start)
			currentJob.duration += duration
			currentJob.dataRate = datarate.NewDatarateSIBytes(datarate.Datarate(totalRead)*datarate.Byte, duration)
			Trace.Printf("%v: IO Processing took %v at %v\n", currentJob.path, duration, currentJob.dataRate)
			workerEndJobs <- currentJob
			return nil
		}()

		if err != nil {
			Error.Printf("%v: IO Processing: %v\n", currentJob.path, err)
		}
	}
}
<commit_msg>Break on empty read<commit_after>package main

import "github.com/kormoc/unit/datarate"
import "github.com/ncw/directio"
import "io"
import "os"
import "sync"
import "time"

var workerIOJobs chan job
var workerIOJobswg sync.WaitGroup

func initWorkerIO() {
	workerIOJobs = make(chan job, workerCountIO*2)
	workerIOJobswg.Add(workerCountIO)
	for i := 0; i < workerCountIO; i++ {
		go workerIO()
	}
	go func() {
		workerIOJobswg.Wait()
		close(workerEndJobs)

	}()
}

func workerIO() {
	defer workerIOJobswg.Done()
	for currentJob := range workerIOJobs {
		err := func() error {
			time_start := time.Now()
			Trace.Printf("%v: IO Processing...\n", currentJob.path)
			// Try direct io. Fail back to normal IO if needed
			fp, err := directio.OpenFile(currentJob.path, os.O_RDONLY, 0000)
			if err != nil {
				fp, err = os.OpenFile(currentJob.path, os.O_RDONLY, 0000)
				if err != nil {
					return err
				}
			}

			defer fp.Close()

			buffer := directio.AlignedBlock(directio.BlockSize * bufferSize)
			totalRead := 0

			for {
				amountRead, err := io.ReadAtLeast(fp, buffer, 0)
				totalRead += amountRead
				if err == io.EOF || amountRead == 0 {
					break
				}
				if err != nil {
					return err
				}

				for checksumAlgo := range currentJob.hashers {
					currentJob.hashers[checksumAlgo].Write(buffer[:amountRead])
				}
			}

			duration := time.Since(time_start)
			currentJob.duration += duration
			currentJob.dataRate = datarate.NewDatarateSIBytes(datarate.Datarate(totalRead)*datarate.Byte, duration)
			Trace.Printf("%v: IO Processing took %v at %v\n", currentJob.path, duration, currentJob.dataRate)
			workerEndJobs <- currentJob
			return nil
		}()

		if err != nil {
			Error.Printf("%v: IO Processing: %v\n", currentJob.path, err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/stefanprodan/syros/models"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"strings"
	"time"
)

type Repository struct {
	Config  *Config
	Session *mgo.Session
}

func NewRepository(config *Config) (*Repository, error) {
	cluster := strings.Split(config.MongoDB, ",")
	dialInfo := &mgo.DialInfo{
		Addrs:    cluster,
		Database: config.Database,
		Timeout:  10 * time.Second,
		FailFast: true,
	}

	session, err := mgo.DialWithInfo(dialInfo)
	if err != nil {
		return nil, err
	}

	session.SetMode(mgo.Monotonic, true)

	repo := &Repository{
		Config:  config,
		Session: session,
	}

	return repo, nil
}

func (repo *Repository) Initialize() {
	repo.CreateIndex("hosts", "environment")
	repo.CreateIndex("hosts", "collected")
	repo.CreateIndex("containers", "host_id")
	repo.CreateIndex("containers", "environment")
	repo.CreateIndex("containers", "collected")
	repo.CreateIndex("checks", "host_id")
	repo.CreateIndex("checks", "environment")
	repo.CreateIndex("checks", "collected")
	repo.CreateIndex("checks_log", "check_id")
	repo.CreateIndex("checks_log", "begin")
	repo.CreateIndex("checks_log", "end")
	repo.CreateIndex("syros_services", "environment")
	repo.CreateIndex("syros_services", "collected")
	repo.CreateIndex("releases", "ticket_id")
	repo.CreateIndex("deployments", "release_id")
}

func (repo *Repository) CreateIndex(col string, index string) {
	c := repo.Session.DB(repo.Config.Database).C(col)
	err := c.EnsureIndexKey(index)

	if err != nil {
		log.Fatalf("MongoDB index %v init failed %v", index, err)
	}
}

func (repo *Repository) HostUpsert(host models.DockerHost) {
	s := repo.Session.Copy()
	defer s.Close()

	c := s.DB(repo.Config.Database).C("hosts")

	_, err := c.UpsertId(host.Id, &host)
	if err != nil {
		log.Errorf("Repository hosts upsert failed %v", err)
	}
}

func (repo *Repository) ContainerUpsert(container models.DockerContainer) {
	s := repo.Session.Copy()
	defer s.Close()

	c := s.DB(repo.Config.Database).C("containers")

	_, err := c.UpsertId(container.Id, &container)
	if err != nil {
		log.Errorf("Repository containers upsert failed %v", err)
	}
}

func (repo *Repository) ContainersUpsert(containers []models.DockerContainer) {
	s := repo.Session.Copy()
	defer s.Close()

	c := s.DB(repo.Config.Database).C("containers")

	for _, container := range containers {
		_, err := c.UpsertId(container.Id, &container)
		if err != nil {
			log.Errorf("Repository containers upsert failed %v", err)
		}
	}
}

func (repo *Repository) ChecksUpsert(checks []models.ConsulHealthCheck) {
	s := repo.Session.Copy()
	defer s.Close()

	c := s.DB(repo.Config.Database).C("checks")

	for _, check := range checks {
		res := models.ConsulHealthCheck{}
		err := c.FindId(check.Id).One(&res)
		if err != nil {
			// insert check
			if err.Error() == "not found" {
				check.Since = check.Collected
				err = c.Insert(&check)
				if err != nil {
					log.Errorf("Repository checks insert failed %v", err)
				}
			} else {
				log.Errorf("Repository checks find by id failed %v", err)
			}
			continue
		}

		// if status changed insert into logs and reset since
		if res.Status != check.Status {
			checkLog := models.NewConsulHealthCheckLog(res, res.Since, check.Collected)
			l := s.DB(repo.Config.Database).C("checks_log")
			err = l.Insert(&checkLog)
			if err != nil {
				log.Errorf("Repository checks_log insert failed %v", err)
			}
			check.Since = check.Collected
		} else {
			check.Since = res.Since
		}

		// update check
		_, err = c.UpsertId(check.Id, &check)
		if err != nil {
			log.Errorf("Repository checks upsert failed %v", err)
		}
	}
}

func (repo *Repository) SyrosServiceUpsert(service models.SyrosService) {
	s := repo.Session.Copy()
	defer s.Close()

	c := s.DB(repo.Config.Database).C("syros_services")

	_, err := c.UpsertId(service.Id, &service)
	if err != nil {
		log.Errorf("Repository syros_services upsert failed %v", err)
	}
}

// Removes stale records
func (repo *Repository) RunGarbageCollector(cols []string) {
	if repo.Config.DatabaseStale > 0 {
		log.Infof("Stating repository GC interval %v minutes", repo.Config.DatabaseStale)
		go func(stale int) {

			for true {
				s := repo.Session.Copy()
				for _, col := range cols {
					c := s.DB(repo.Config.Database).C(col)
					info, err := c.RemoveAll(
						bson.M{
							"collected": bson.M{
								"$lt": time.Now().Add(-time.Duration(stale) * time.Minute).UTC(),
							},
						})
					if err != nil {
						log.Errorf("Repository GC for col %v query failed %v", col, err)
					} else {
						if info.Removed > 0 {
							log.Infof("Repository GC removed %v from %v", info.Removed, col)
						}
					}
				}
				s.Close()
				time.Sleep(60 * time.Second)
			}

		}(repo.Config.DatabaseStale)
	}
}
<commit_msg>run GC with a ticker<commit_after>package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/stefanprodan/syros/models"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"strings"
	"time"
)

type Repository struct {
	Config  *Config
	Session *mgo.Session
}

func NewRepository(config *Config) (*Repository, error) {
	cluster := strings.Split(config.MongoDB, ",")
	dialInfo := &mgo.DialInfo{
		Addrs:    cluster,
		Database: config.Database,
		Timeout:  10 * time.Second,
		FailFast: true,
	}

	session, err := mgo.DialWithInfo(dialInfo)
	if err != nil {
		return nil, err
	}

	session.SetMode(mgo.Monotonic, true)

	repo := &Repository{
		Config:  config,
		Session: session,
	}

	return repo, nil
}

func (repo *Repository) Initialize() {
	repo.CreateIndex("hosts", "environment")
	repo.CreateIndex("hosts", "collected")
	repo.CreateIndex("containers", "host_id")
	repo.CreateIndex("containers", "environment")
	repo.CreateIndex("containers", "collected")
	repo.CreateIndex("checks", "host_id")
	repo.CreateIndex("checks", "environment")
	repo.CreateIndex("checks", "collected")
	repo.CreateIndex("checks_log", "check_id")
	repo.CreateIndex("checks_log", "begin")
	repo.CreateIndex("checks_log", "end")
	repo.CreateIndex("syros_services", "environment")
	repo.CreateIndex("syros_services", "collected")
	repo.CreateIndex("releases", "ticket_id")
	repo.CreateIndex("deployments", "release_id")
}

func (repo *Repository) CreateIndex(col string, index string) {
	c := repo.Session.DB(repo.Config.Database).C(col)
	err := c.EnsureIndexKey(index)

	if err != nil {
		log.Fatalf("MongoDB index %v init failed %v", index, err)
	}
}

func (repo *Repository) HostUpsert(host models.DockerHost) {
	s := repo.Session.Copy()
	defer s.Close()

	c := s.DB(repo.Config.Database).C("hosts")

	_, err := c.UpsertId(host.Id, &host)
	if err != nil {
		log.Errorf("Repository hosts upsert failed %v", err)
	}
}

func (repo *Repository) ContainerUpsert(container models.DockerContainer) {
	s := repo.Session.Copy()
	defer s.Close()

	c := s.DB(repo.Config.Database).C("containers")

	_, err := c.UpsertId(container.Id, &container)
	if err != nil {
		log.Errorf("Repository containers upsert failed %v", err)
	}
}

func (repo *Repository) ContainersUpsert(containers []models.DockerContainer) {
	s := repo.Session.Copy()
	defer s.Close()

	c := s.DB(repo.Config.Database).C("containers")

	for _, container := range containers {
		_, err := c.UpsertId(container.Id, &container)
		if err != nil {
			log.Errorf("Repository containers upsert failed %v", err)
		}
	}
}

func (repo *Repository) ChecksUpsert(checks []models.ConsulHealthCheck) {
	s := repo.Session.Copy()
	defer s.Close()

	c := s.DB(repo.Config.Database).C("checks")

	for _, check := range checks {
		res := models.ConsulHealthCheck{}
		err := c.FindId(check.Id).One(&res)
		if err != nil {
			// insert check
			if err.Error() == "not found" {
				check.Since = check.Collected
				err = c.Insert(&check)
				if err != nil {
					log.Errorf("Repository checks insert failed %v", err)
				}
			} else {
				log.Errorf("Repository checks find by id failed %v", err)
			}
			continue
		}

		// if status changed insert into logs and reset since
		if res.Status != check.Status {
			checkLog := models.NewConsulHealthCheckLog(res, res.Since, check.Collected)
			l := s.DB(repo.Config.Database).C("checks_log")
			err = l.Insert(&checkLog)
			if err != nil {
				log.Errorf("Repository checks_log insert failed %v", err)
			}
			check.Since = check.Collected
		} else {
			check.Since = res.Since
		}

		// update check
		_, err = c.UpsertId(check.Id, &check)
		if err != nil {
			log.Errorf("Repository checks upsert failed %v", err)
		}
	}
}

func (repo *Repository) SyrosServiceUpsert(service models.SyrosService) {
	s := repo.Session.Copy()
	defer s.Close()

	c := s.DB(repo.Config.Database).C("syros_services")

	_, err := c.UpsertId(service.Id, &service)
	if err != nil {
		log.Errorf("Repository syros_services upsert failed %v", err)
	}
}

// Removes stale records
func (repo *Repository) RunGarbageCollector(cols []string) {
	if repo.Config.DatabaseStale > 0 {
		ticker := time.NewTicker(60 * time.Second)
		log.Infof("Stating repository GC interval %v minutes", repo.Config.DatabaseStale)
		go func(stale int) {
			for {
				select {
				case <-ticker.C:
					s := repo.Session.Copy()
					for _, col := range cols {
						c := s.DB(repo.Config.Database).C(col)
						info, err := c.RemoveAll(
							bson.M{
								"collected": bson.M{
									"$lt": time.Now().Add(-time.Duration(stale) * time.Minute).UTC(),
								},
							})
						if err != nil {
							log.Errorf("Repository GC for col %v query failed %v", col, err)
						} else {
							if info.Removed > 0 {
								log.Infof("Repository GC removed %v from %v", info.Removed, col)
							}
						}
					}
					s.Close()
				}
			}
		}(repo.Config.DatabaseStale)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package api

import (
	"encoding/json"
	"errors"
)

// Implements Error & Retriable
type APIError struct {
	err       error
	Retriable bool
}

type jsonError struct {
	ErrType string `json:"__type"`
	Message string `json:"message"`
}

func NewAPIError(err error) *APIError {
	intermediate := &jsonError{}
	if err := json.Unmarshal([]byte(err.Error()), intermediate); err == nil {
		if intermediate.ErrType == "ClientException" {
			return &APIError{errors.New(intermediate.Message), false}
		}
	}

	return &APIError{err, true}
}

func (sce *APIError) Retry() bool {
	return sce.Retriable
}

func (sce *APIError) Error() string {
	return sce.err.Error()
}
<commit_msg>Rename receiver<commit_after>// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package api

import (
	"encoding/json"
	"errors"
)

// Implements Error & Retriable
type APIError struct {
	err       error
	Retriable bool
}

type jsonError struct {
	ErrType string `json:"__type"`
	Message string `json:"message"`
}

func NewAPIError(err error) *APIError {
	intermediate := &jsonError{}
	if err := json.Unmarshal([]byte(err.Error()), intermediate); err == nil {
		if intermediate.ErrType == "ClientException" {
			return &APIError{errors.New(intermediate.Message), false}
		}
	}

	return &APIError{err, true}
}

func (apiErr *APIError) Retry() bool {
	return apiErr.Retriable
}

func (apiErr *APIError) Error() string {
	return apiErr.err.Error()
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"github.com/networkservicemesh/networkservicemesh/controlplane/pkg/apis/local/connection"
	"github.com/networkservicemesh/networkservicemesh/controlplane/pkg/nsmd"
	"github.com/networkservicemesh/networkservicemesh/pkg/tools"
	"github.com/networkservicemesh/networkservicemesh/sdk/client"
	"github.com/opentracing/opentracing-go"
	"github.com/sirupsen/logrus"
	"os"
	"sync"
)

const (
	defaultVPPAgentEndpoint = "localhost:9113"
)

type nsClientBackend struct {
	workspace        string
	vppAgentEndpoint string
}

func (nscb *nsClientBackend) New() error {
	if err := Reset(nscb.vppAgentEndpoint); err != nil {
		logrus.Fatal(err)
	}
	logrus.Infof("workspace: %s", nscb.workspace)
	return nil
}

func (nscb *nsClientBackend) Connect(connection *connection.Connection) error {
	logrus.Infof("nsClientBackend received: %v", connection)
	err := CreateVppInterface(connection, nscb.workspace, nscb.vppAgentEndpoint)
	if err != nil {
		logrus.Errorf("VPPAgent failed creating the requested interface with: %v", err)
	}
	return err
}

func main() {
	// Capture signals to cleanup before exiting
	c := tools.NewOSSignalChannel()

	tracer, closer := tools.InitJaeger("nsc")
	opentracing.SetGlobalTracer(tracer)
	defer closer.Close()

	workspace, ok := os.LookupEnv(nsmd.WorkspaceEnv)
	if !ok {
		logrus.Fatalf("Failed gettign %s", nsmd.WorkspaceEnv)
	}

	backend := &nsClientBackend{
		workspace:        workspace,
		vppAgentEndpoint: defaultVPPAgentEndpoint,
	}

	client, err := client.NewNSMClient(nil, nil)
	if err != nil {
		logrus.Fatalf("Unable to create the NSM client %v", err)
	}

	err = backend.New()
	if err != nil {
		logrus.Fatalf("Unable to create the backend %v", err)
	}

	var outgoingConnection *connection.Connection
	outgoingConnection, err = client.Connect("if1", "mem", "Primary interface")
	if err != nil {
		logrus.Fatalf("Unable to connect %v", err)
	}

	err = backend.Connect(outgoingConnection)
	if err != nil {
		logrus.Fatalf("Unable to connect %v", err)
	}

	logrus.Info("nsm client: initialization is completed successfully, wait for Ctrl+C...")
	var wg sync.WaitGroup
	wg.Add(1)

	<-c
}
<commit_msg>Update nsc.go (#1052)<commit_after>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"github.com/networkservicemesh/networkservicemesh/controlplane/pkg/apis/local/connection"
	"github.com/networkservicemesh/networkservicemesh/controlplane/pkg/nsmd"
	"github.com/networkservicemesh/networkservicemesh/pkg/tools"
	"github.com/networkservicemesh/networkservicemesh/sdk/client"
	"github.com/opentracing/opentracing-go"
	"github.com/sirupsen/logrus"
	"os"
	"sync"
)

const (
	defaultVPPAgentEndpoint = "localhost:9113"
)

type nsClientBackend struct {
	workspace        string
	vppAgentEndpoint string
}

func (nscb *nsClientBackend) New() error {
	if err := Reset(nscb.vppAgentEndpoint); err != nil {
		logrus.Fatal(err)
	}
	logrus.Infof("workspace: %s", nscb.workspace)
	return nil
}

func (nscb *nsClientBackend) Connect(connection *connection.Connection) error {
	logrus.Infof("nsClientBackend received: %v", connection)
	err := CreateVppInterface(connection, nscb.workspace, nscb.vppAgentEndpoint)
	if err != nil {
		logrus.Errorf("VPPAgent failed creating the requested interface with: %v", err)
	}
	return err
}

func main() {
	// Capture signals to cleanup before exiting
	c := tools.NewOSSignalChannel()

	tracer, closer := tools.InitJaeger("nsc")
	opentracing.SetGlobalTracer(tracer)
	defer closer.Close()

	workspace, ok := os.LookupEnv(nsmd.WorkspaceEnv)
	if !ok {
		logrus.Fatalf("Failed getting %s", nsmd.WorkspaceEnv)
	}

	backend := &nsClientBackend{
		workspace:        workspace,
		vppAgentEndpoint: defaultVPPAgentEndpoint,
	}

	client, err := client.NewNSMClient(nil, nil)
	if err != nil {
		logrus.Fatalf("Unable to create the NSM client %v", err)
	}

	err = backend.New()
	if err != nil {
		logrus.Fatalf("Unable to create the backend %v", err)
	}

	var outgoingConnection *connection.Connection
	outgoingConnection, err = client.Connect("if1", "mem", "Primary interface")
	if err != nil {
		logrus.Fatalf("Unable to connect %v", err)
	}

	err = backend.Connect(outgoingConnection)
	if err != nil {
		logrus.Fatalf("Unable to connect %v", err)
	}

	logrus.Info("nsm client: initialization is completed successfully, wait for Ctrl+C...")
	var wg sync.WaitGroup
	wg.Add(1)

	<-c
}
<|endoftext|>


<commit_before>package control

import (
	"TOGY/util"
	"os/exec"
)

// Starts PowerPoint in presentation mode.
func StartPresentation(ppExe, path string) error {
	err := exec.Command(ppExe, "/s", path).Start()
	if err != nil {
		return err
	}
	return nil
}

//Sends Terminate signal to PowerPoint.
func KillPresentation() {
	exec.Command("taskkill", "/IM", "POWERPNT.exe").Run()
}

//Kills powerpoint and loads presentation at p.
func ReloadPresentation(ppExe, p string) {
	KillPresentation()
	util.Sleep(1)
	StartPresentation(ppExe, p)
}
<commit_msg>Reworked powerpoint API to use Broadcast interface<commit_after>package control

import (
	"TOGY/util"
	"os/exec"
)

type PowerPointBroadcast struct {
	//Path to the presentation
	path string
	//Path to PowerPoint executable
	powerPoint string
	//Current running PowerPoint instance
	cmd exec.Cmd
}

// Starts PowerPoint in presentation mode.
func (*PowerPointBroadcast b) Start() error {
	cmd := exec.Command(b.powerPoint, "/s", b.path)
	err = cmd.start()
	if err != nil {
		return err
	}
	b.cmd = cmd
	return nil
}

//Sends Terminate signal to PowerPoint.
func (*PowerPointBroadcast b) Kill() error {
	err = b.cmd.Process.Kill()
	if err != nil {
		return err
	}
	b.cmd = nil
	return
}

func (PowerPointBroadcast b) Status() bool {
	if b.cmd == nil {
		return false
	}
	return true
}

func NewPowerPoint(ppExe, presentation string) (*PowerPointBroadcast) {
	return &PowerPointBroadcast{path: presentation, powerPoint: ppExe}
}<|endoftext|>


<commit_before>package routing

import (
	"bytes"

	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcutil"
	"github.com/davecgh/go-spew/spew"
	"github.com/go-errors/errors"
	"github.com/lightningnetwork/lnd/lnwire"
)

// ValidateChannelAnn validates the channel announcement message and checks
// that node signatures covers the announcement message, and that the bitcoin
// signatures covers the node keys.
func ValidateChannelAnn(a *lnwire.ChannelAnnouncement) error {
	// First, we'll compute the digest (h) which is to be signed by each of
	// the keys included within the node announcement message. This hash
	// digest includes all the keys, so the (up to 4 signatures) will
	// attest to the validity of each of the keys.
	data, err := a.DataToSign()
	if err != nil {
		return err
	}
	dataHash := chainhash.DoubleHashB(data)

	// First we'll verify that the passed bitcoin key signature is indeed a
	// signature over the computed hash digest.
	bitcoinSig1, err := a.BitcoinSig1.ToSignature()
	if err != nil {
		return err
	}
	bitcoinKey1, err := btcec.ParsePubKey(a.BitcoinKey1[:], btcec.S256())
	if err != nil {
		return err
	}
	if !bitcoinSig1.Verify(dataHash, bitcoinKey1) {
		return errors.New("can't verify first bitcoin signature")
	}

	// If that checks out, then we'll verify that the second bitcoin
	// signature is a valid signature of the bitcoin public key over hash
	// digest as well.
	bitcoinSig2, err := a.BitcoinSig2.ToSignature()
	if err != nil {
		return err
	}
	bitcoinKey2, err := btcec.ParsePubKey(a.BitcoinKey2[:], btcec.S256())
	if err != nil {
		return err
	}
	if !bitcoinSig2.Verify(dataHash, bitcoinKey2) {
		return errors.New("can't verify second bitcoin signature")
	}

	// Both node signatures attached should indeed be a valid signature
	// over the selected digest of the channel announcement signature.
	nodeSig1, err := a.NodeSig1.ToSignature()
	if err != nil {
		return err
	}
	nodeKey1, err := btcec.ParsePubKey(a.NodeID1[:], btcec.S256())
	if err != nil {
		return err
	}
	if !nodeSig1.Verify(dataHash, nodeKey1) {
		return errors.New("can't verify data in first node signature")
	}

	nodeSig2, err := a.NodeSig2.ToSignature()
	if err != nil {
		return err
	}
	nodeKey2, err := btcec.ParsePubKey(a.NodeID2[:], btcec.S256())
	if err != nil {
		return err
	}
	if !nodeSig2.Verify(dataHash, nodeKey2) {
		return errors.New("can't verify data in second node signature")
	}

	return nil

}

// ValidateNodeAnn validates the node announcement by ensuring that the
// attached signature is needed a signature of the node announcement under the
// specified node public key.
func ValidateNodeAnn(a *lnwire.NodeAnnouncement) error {
	// Reconstruct the data of announcement which should be covered by the
	// signature so we can verify the signature shortly below
	data, err := a.DataToSign()
	if err != nil {
		return err
	}

	nodeSig, err := a.Signature.ToSignature()
	if err != nil {
		return err
	}
	nodeKey, err := btcec.ParsePubKey(a.NodeID[:], btcec.S256())
	if err != nil {
		return err
	}

	// Finally ensure that the passed signature is valid, if not we'll
	// return an error so this node announcement can be rejected.
	dataHash := chainhash.DoubleHashB(data)
	if !nodeSig.Verify(dataHash, nodeKey) {
		var msgBuf bytes.Buffer
		if _, err := lnwire.WriteMessage(&msgBuf, a, 0); err != nil {
			return err
		}

		return errors.Errorf("signature on NodeAnnouncement(%x) is "+
			"invalid: %x", nodeKey.SerializeCompressed(),
			msgBuf.Bytes())
	}

	return nil
}

// ValidateChannelUpdateAnn validates the channel update announcement by
// checking (1) that the included signature covers the announcement and has been
// signed by the node's private key, and (2) that the announcement's message
// flags and optional fields are sane.
func ValidateChannelUpdateAnn(pubKey *btcec.PublicKey, capacity btcutil.Amount,
	a *lnwire.ChannelUpdate) error {

	if err := validateOptionalFields(capacity, a); err != nil {
		return err
	}

	data, err := a.DataToSign()
	if err != nil {
		return errors.Errorf("unable to reconstruct message: %v", err)
	}
	dataHash := chainhash.DoubleHashB(data)

	nodeSig, err := a.Signature.ToSignature()
	if err != nil {
		return err
	}

	if !nodeSig.Verify(dataHash, pubKey) {
		return errors.Errorf("invalid signature for channel "+
			"update %v", spew.Sdump(a))
	}

	return nil
}

// validateOptionalFields validates a channel update's message flags and
// corresponding update fields.
func validateOptionalFields(capacity btcutil.Amount,
	msg *lnwire.ChannelUpdate) error {

	if msg.MessageFlags.HasMaxHtlc() {
		maxHtlc := msg.HtlcMaximumMsat
		if maxHtlc == 0 || maxHtlc < msg.HtlcMinimumMsat {
			return errors.Errorf("invalid max htlc for channel "+
				"update %v", spew.Sdump(msg))
		}
		cap := lnwire.NewMSatFromSatoshis(capacity)
		if maxHtlc > cap {
			return errors.Errorf("max_htlc(%v) for channel "+
				"update greater than capacity(%v)", maxHtlc,
				cap)
		}
	}

	return nil
}
<commit_msg>routing: expose VerifyChannelUpdateSignature function<commit_after>package routing

import (
	"bytes"
	"fmt"

	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcutil"
	"github.com/davecgh/go-spew/spew"
	"github.com/go-errors/errors"
	"github.com/lightningnetwork/lnd/lnwire"
)

// ValidateChannelAnn validates the channel announcement message and checks
// that node signatures covers the announcement message, and that the bitcoin
// signatures covers the node keys.
func ValidateChannelAnn(a *lnwire.ChannelAnnouncement) error {
	// First, we'll compute the digest (h) which is to be signed by each of
	// the keys included within the node announcement message. This hash
	// digest includes all the keys, so the (up to 4 signatures) will
	// attest to the validity of each of the keys.
	data, err := a.DataToSign()
	if err != nil {
		return err
	}
	dataHash := chainhash.DoubleHashB(data)

	// First we'll verify that the passed bitcoin key signature is indeed a
	// signature over the computed hash digest.
	bitcoinSig1, err := a.BitcoinSig1.ToSignature()
	if err != nil {
		return err
	}
	bitcoinKey1, err := btcec.ParsePubKey(a.BitcoinKey1[:], btcec.S256())
	if err != nil {
		return err
	}
	if !bitcoinSig1.Verify(dataHash, bitcoinKey1) {
		return errors.New("can't verify first bitcoin signature")
	}

	// If that checks out, then we'll verify that the second bitcoin
	// signature is a valid signature of the bitcoin public key over hash
	// digest as well.
	bitcoinSig2, err := a.BitcoinSig2.ToSignature()
	if err != nil {
		return err
	}
	bitcoinKey2, err := btcec.ParsePubKey(a.BitcoinKey2[:], btcec.S256())
	if err != nil {
		return err
	}
	if !bitcoinSig2.Verify(dataHash, bitcoinKey2) {
		return errors.New("can't verify second bitcoin signature")
	}

	// Both node signatures attached should indeed be a valid signature
	// over the selected digest of the channel announcement signature.
	nodeSig1, err := a.NodeSig1.ToSignature()
	if err != nil {
		return err
	}
	nodeKey1, err := btcec.ParsePubKey(a.NodeID1[:], btcec.S256())
	if err != nil {
		return err
	}
	if !nodeSig1.Verify(dataHash, nodeKey1) {
		return errors.New("can't verify data in first node signature")
	}

	nodeSig2, err := a.NodeSig2.ToSignature()
	if err != nil {
		return err
	}
	nodeKey2, err := btcec.ParsePubKey(a.NodeID2[:], btcec.S256())
	if err != nil {
		return err
	}
	if !nodeSig2.Verify(dataHash, nodeKey2) {
		return errors.New("can't verify data in second node signature")
	}

	return nil

}

// ValidateNodeAnn validates the node announcement by ensuring that the
// attached signature is needed a signature of the node announcement under the
// specified node public key.
func ValidateNodeAnn(a *lnwire.NodeAnnouncement) error {
	// Reconstruct the data of announcement which should be covered by the
	// signature so we can verify the signature shortly below
	data, err := a.DataToSign()
	if err != nil {
		return err
	}

	nodeSig, err := a.Signature.ToSignature()
	if err != nil {
		return err
	}
	nodeKey, err := btcec.ParsePubKey(a.NodeID[:], btcec.S256())
	if err != nil {
		return err
	}

	// Finally ensure that the passed signature is valid, if not we'll
	// return an error so this node announcement can be rejected.
	dataHash := chainhash.DoubleHashB(data)
	if !nodeSig.Verify(dataHash, nodeKey) {
		var msgBuf bytes.Buffer
		if _, err := lnwire.WriteMessage(&msgBuf, a, 0); err != nil {
			return err
		}

		return errors.Errorf("signature on NodeAnnouncement(%x) is "+
			"invalid: %x", nodeKey.SerializeCompressed(),
			msgBuf.Bytes())
	}

	return nil
}

// ValidateChannelUpdateAnn validates the channel update announcement by
// checking (1) that the included signature covers the announcement and has been
// signed by the node's private key, and (2) that the announcement's message
// flags and optional fields are sane.
func ValidateChannelUpdateAnn(pubKey *btcec.PublicKey, capacity btcutil.Amount,
	a *lnwire.ChannelUpdate) error {

	if err := validateOptionalFields(capacity, a); err != nil {
		return err
	}

	return VerifyChannelUpdateSignature(a, pubKey)
}

// VerifyChannelUpdateSignature verifies that the channel update message was
// signed by the party with the given node public key.
func VerifyChannelUpdateSignature(msg *lnwire.ChannelUpdate,
	pubKey *btcec.PublicKey) error {

	data, err := msg.DataToSign()
	if err != nil {
		return fmt.Errorf("unable to reconstruct message data: %v", err)
	}
	dataHash := chainhash.DoubleHashB(data)

	nodeSig, err := msg.Signature.ToSignature()
	if err != nil {
		return err
	}

	if !nodeSig.Verify(dataHash, pubKey) {
		return fmt.Errorf("invalid signature for channel update %v",
			spew.Sdump(msg))
	}

	return nil
}

// validateOptionalFields validates a channel update's message flags and
// corresponding update fields.
func validateOptionalFields(capacity btcutil.Amount,
	msg *lnwire.ChannelUpdate) error {

	if msg.MessageFlags.HasMaxHtlc() {
		maxHtlc := msg.HtlcMaximumMsat
		if maxHtlc == 0 || maxHtlc < msg.HtlcMinimumMsat {
			return errors.Errorf("invalid max htlc for channel "+
				"update %v", spew.Sdump(msg))
		}
		cap := lnwire.NewMSatFromSatoshis(capacity)
		if maxHtlc > cap {
			return errors.Errorf("max_htlc(%v) for channel "+
				"update greater than capacity(%v)", maxHtlc,
				cap)
		}
	}

	return nil
}
<|endoftext|>