<commit_before>package rollinghash_test

import (
	"hash"
	"io/ioutil"
	"math/rand"
	"testing"

	"github.com/chmduquesne/rollinghash"
	_adler32 "github.com/chmduquesne/rollinghash/adler32"
	"github.com/chmduquesne/rollinghash/bozo32"
	"github.com/chmduquesne/rollinghash/buzhash32"
	"github.com/chmduquesne/rollinghash/buzhash64"
	"github.com/chmduquesne/rollinghash/rabinkarp64"
)

var allHashes = []struct {
	name    string
	classic hash.Hash
	rolling rollinghash.Hash
}{
	{"adler32", _adler32.New(), _adler32.New()},
	{"buzhash32", buzhash32.New(), buzhash32.New()},
	{"buzhash64", buzhash64.New(), buzhash64.New()},
	{"bozo32", bozo32.New(), bozo32.New()},
	{"rabinkarp64", rabinkarp64.New(), rabinkarp64.New()},
}

// Gets the hash sum as a uint64
func sum64(h hash.Hash) (res uint64) {
	buf := make([]byte, 0, 8)
	s := h.Sum(buf)
	for _, b := range s {
		res <<= 8
		res |= uint64(b)
	}
	return
}

// Compute the hash by creating a byte slice with an additionnal '\0' at
// the beginning, writing the slice without the last byte, and then
// rolling the last byte.
func SumByWriteAndRoll(h rollinghash.Hash, b []byte) uint64 {
	q := []byte("\x00")
	q = append(q, b...)

	h.Reset()
	h.Write(q[:len(q)-1])
	h.Roll(q[len(q)-1])
	return sum64(h)
}

// Compute the hash the classic way
func SumByWriteOnly(h hash.Hash, b []byte) uint64 {
	h.Reset()
	h.Write(b)
	return sum64(h)
}

// Create some random slice (length betwen 0 and 8KB, random content)
func RandomBytes() (res []byte) {
	n := rand.Intn(8192)
	res = make([]byte, n)
	rand.Read(res)
	return res
}

// Verify that, on random inputs, the classic hash and the rollinghash
// return the same values
func blackBox(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	for i := 0; i < 1000; i++ {
		in := RandomBytes()
		if len(in) > 0 {
			sum := SumByWriteAndRoll(rolling, in)
			ref := SumByWriteOnly(classic, in)

			if ref != sum {
				t.Errorf("[%s] Expected 0x%x, got 0x%x", hashname, ref, sum)
			}
		}
	}
}

// Roll a window of 16 bytes with a classic hash and a rolling hash and
// compare the results
func foxDog(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	s := []byte("The quick brown fox jumps over the lazy dog")

	// Window len
	n := 16

	// Load the window into the rolling hash
	rolling.Write(s[:n])

	// Roll it and compare the result with full re-calculus every time
	for i := n; i < len(s); i++ {

		// Reset and write the window in classic
		classic.Reset()
		classic.Write(s[i-n+1 : i+1])

		// Roll the incoming byte in rolling
		rolling.Roll(s[i])

		// Compare the hashes
		sumc := sum64(classic)
		sumr := sum64(rolling)
		if sumc != sumr {
			t.Errorf("[%s] %v: expected %x, got %x",
				hashname, s[i-n+1:i+1], sumc, sumr)
		}
	}
}

func rollEmptyWindow(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	defer func() {
		if r := recover(); r == nil {
			t.Errorf("[%s] Rolling an empty window should cause a panic", hashname)
		}
	}()
	// This should panic
	rolling.Roll(byte('x'))
}

func writeTwice(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	rolling.Write([]byte("hello "))
	rolling.Write([]byte("world"))

	classic.Write([]byte("hello world"))

	if sum64(rolling) != sum64(classic) {
		t.Errorf("[%s] Expected same results on rolling and classic", hashname)
	}
}

func writeRollWrite(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	rolling.Write([]byte(" hello"))
	rolling.Roll(byte(' '))
	rolling.Write([]byte("world"))

	classic.Write([]byte("hello world"))

	if sum64(rolling) != sum64(classic) {
		t.Errorf("[%s] Expected same results on rolling and classic", hashname)
	}
}

func writeThenWriteNothing(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	rolling.Write([]byte("hello"))
	rolling.Write([]byte(""))

	classic.Write([]byte("hello"))

	if sum64(rolling) != sum64(classic) {
		t.Errorf("[%s] Expected same results on rolling and classic", hashname)
	}
}

func writeNothing(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	rolling.Write([]byte(""))

	if sum64(rolling) != sum64(classic) {
		t.Errorf("[%s] Expected same results on rolling and classic", hashname)
	}
}

func read(t *testing.T, hashname string, rolling rollinghash.Hash) {
	rolling.Write([]byte("hello "))
	rolling.Roll(byte('w'))

	window, _ := ioutil.ReadAll(rolling)

	if string(window) != "ello w" {
		t.Errorf("[%s] Unexpect read from the hash", hashname)
	}
}

func TestFoxDog(t *testing.T) {
	for _, h := range allHashes {
		h.classic.Reset()
		h.rolling.Reset()
		foxDog(t, h.name, h.classic, h.rolling)
	}
}

func TestBlackBox(t *testing.T) {
	for _, h := range allHashes {
		h.classic.Reset()
		h.rolling.Reset()
		blackBox(t, h.name, h.classic, h.rolling)
	}
}

func TestRollEmptyWindow(t *testing.T) {
	for _, h := range allHashes {
		h.classic.Reset()
		h.rolling.Reset()
		rollEmptyWindow(t, h.name, h.classic, h.rolling)
	}
}

func TestwriteTwice(t *testing.T) {
	for _, h := range allHashes {
		h.classic.Reset()
		h.rolling.Reset()
		writeTwice(t, h.name, h.classic, h.rolling)
	}
}

func TestwriteRollWrite(t *testing.T) {
	for _, h := range allHashes {
		h.classic.Reset()
		h.rolling.Reset()
		writeRollWrite(t, h.name, h.classic, h.rolling)
	}
}

func TestWriteThenWriteNothing(t *testing.T) {
	for _, h := range allHashes {
		h.classic.Reset()
		h.rolling.Reset()
		writeThenWriteNothing(t, h.name, h.classic, h.rolling)
	}
}

func TestWriteNothing(t *testing.T) {
	for _, h := range allHashes {
		h.classic.Reset()
		h.rolling.Reset()
		writeNothing(t, h.name, h.classic, h.rolling)
	}
}

func TestRead(t *testing.T) {
	for _, h := range allHashes {
		h.rolling.Reset()
		read(t, h.name, h.rolling)
	}
}
<commit_msg>Reducing the memory footprint of the test suite (rabinkarp64 likes a lot of cached data)<commit_after>package rollinghash_test

import (
	"hash"
	"io/ioutil"
	"math/rand"
	"testing"

	"github.com/chmduquesne/rollinghash"
	_adler32 "github.com/chmduquesne/rollinghash/adler32"
	"github.com/chmduquesne/rollinghash/bozo32"
	"github.com/chmduquesne/rollinghash/buzhash32"
	"github.com/chmduquesne/rollinghash/buzhash64"
	"github.com/chmduquesne/rollinghash/rabinkarp64"
)

var allHashes = []struct {
	name    string
	classic hash.Hash
	rolling rollinghash.Hash
}{
	{"adler32", _adler32.New(), _adler32.New()},
	{"buzhash32", buzhash32.New(), buzhash32.New()},
	{"buzhash64", buzhash64.New(), buzhash64.New()},
	{"bozo32", bozo32.New(), bozo32.New()},
	{"rabinkarp64", rabinkarp64.New(), rabinkarp64.New()},
}

// Gets the hash sum as a uint64
func sum64(h hash.Hash) (res uint64) {
	buf := make([]byte, 0, 8)
	s := h.Sum(buf)
	for _, b := range s {
		res <<= 8
		res |= uint64(b)
	}
	return
}

// Compute the hash by creating a byte slice with an additionnal '\0' at
// the beginning, writing the slice without the last byte, and then
// rolling the last byte.
func SumByWriteAndRoll(h rollinghash.Hash, b []byte) uint64 {
	q := []byte("\x00")
	q = append(q, b...)

	h.Reset()
	h.Write(q[:len(q)-1])
	h.Roll(q[len(q)-1])
	return sum64(h)
}

// Compute the hash the classic way
func SumByWriteOnly(h hash.Hash, b []byte) uint64 {
	h.Reset()
	h.Write(b)
	return sum64(h)
}

// Create some random slice (length betwen 0 and 1KB, random content)
func RandomBytes() (res []byte) {
	n := rand.Intn(1024)
	res = make([]byte, n)
	rand.Read(res)
	return res
}

// Verify that, on random inputs, the classic hash and the rollinghash
// return the same values
func blackBox(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	for i := 0; i < 100; i++ {
		in := RandomBytes()
		if len(in) > 0 {
			sum := SumByWriteAndRoll(rolling, in)
			ref := SumByWriteOnly(classic, in)

			if ref != sum {
				t.Errorf("[%s] Expected 0x%x, got 0x%x", hashname, ref, sum)
			}
		}
	}
}

// Roll a window of 16 bytes with a classic hash and a rolling hash and
// compare the results
func foxDog(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	s := []byte("The quick brown fox jumps over the lazy dog")

	// Window len
	n := 16

	// Load the window into the rolling hash
	rolling.Write(s[:n])

	// Roll it and compare the result with full re-calculus every time
	for i := n; i < len(s); i++ {

		// Reset and write the window in classic
		classic.Reset()
		classic.Write(s[i-n+1 : i+1])

		// Roll the incoming byte in rolling
		rolling.Roll(s[i])

		// Compare the hashes
		sumc := sum64(classic)
		sumr := sum64(rolling)
		if sumc != sumr {
			t.Errorf("[%s] %v: expected %x, got %x",
				hashname, s[i-n+1:i+1], sumc, sumr)
		}
	}
}

func rollEmptyWindow(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	defer func() {
		if r := recover(); r == nil {
			t.Errorf("[%s] Rolling an empty window should cause a panic", hashname)
		}
	}()
	// This should panic
	rolling.Roll(byte('x'))
}

func writeTwice(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	rolling.Write([]byte("hello "))
	rolling.Write([]byte("world"))

	classic.Write([]byte("hello world"))

	if sum64(rolling) != sum64(classic) {
		t.Errorf("[%s] Expected same results on rolling and classic", hashname)
	}
}

func writeRollWrite(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	rolling.Write([]byte(" hello"))
	rolling.Roll(byte(' '))
	rolling.Write([]byte("world"))

	classic.Write([]byte("hello world"))

	if sum64(rolling) != sum64(classic) {
		t.Errorf("[%s] Expected same results on rolling and classic", hashname)
	}
}

func writeThenWriteNothing(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	rolling.Write([]byte("hello"))
	rolling.Write([]byte(""))

	classic.Write([]byte("hello"))

	if sum64(rolling) != sum64(classic) {
		t.Errorf("[%s] Expected same results on rolling and classic", hashname)
	}
}

func writeNothing(t *testing.T, hashname string, classic hash.Hash, rolling rollinghash.Hash) {
	rolling.Write([]byte(""))

	if sum64(rolling) != sum64(classic) {
		t.Errorf("[%s] Expected same results on rolling and classic", hashname)
	}
}

func read(t *testing.T, hashname string, rolling rollinghash.Hash) {
	rolling.Write([]byte("hello "))
	rolling.Roll(byte('w'))

	window, _ := ioutil.ReadAll(rolling)

	if string(window) != "ello w" {
		t.Errorf("[%s] Unexpect read from the hash", hashname)
	}
}

func TestFoxDog(t *testing.T) {
	for _, h := range allHashes {
		h.classic.Reset()
		h.rolling.Reset()
		foxDog(t, h.name, h.classic, h.rolling)
	}
}

func TestBlackBox(t *testing.T) {
	for _, h := range allHashes {
		h.classic.Reset()
		h.rolling.Reset()
		blackBox(t, h.name, h.classic, h.rolling)
	}
}

func TestRollEmptyWindow(t *testing.T) {
	for _, h := range allHashes {
		h.classic.Reset()
		h.rolling.Reset()
		rollEmptyWindow(t, h.name, h.classic, h.rolling)
	}
}

func TestwriteTwice(t *testing.T) {
	for _, h := range allHashes {
		h.classic.Reset()
		h.rolling.Reset()
		writeTwice(t, h.name, h.classic, h.rolling)
	}
}

func TestwriteRollWrite(t *testing.T) {
	for _, h := range allHashes {
		h.classic.Reset()
		h.rolling.Reset()
		writeRollWrite(t, h.name, h.classic, h.rolling)
	}
}

func TestWriteThenWriteNothing(t *testing.T) {
	for _, h := range allHashes {
		h.classic.Reset()
		h.rolling.Reset()
		writeThenWriteNothing(t, h.name, h.classic, h.rolling)
	}
}

func TestWriteNothing(t *testing.T) {
	for _, h := range allHashes {
		h.classic.Reset()
		h.rolling.Reset()
		writeNothing(t, h.name, h.classic, h.rolling)
	}
}

func TestRead(t *testing.T) {
	for _, h := range allHashes {
		h.rolling.Reset()
		read(t, h.name, h.rolling)
	}
}
<|endoftext|>


<commit_before>package admin

import (
	"github.com/fuxiaohei/purine/route/base"
	"github.com/lunny/tango"
)

type Page struct {
	base.AdminRender
	base.BaseAuther
	tango.Ctx
}

func (p *Page) Get() {
	p.Title("Page")
	p.Render("page.tmpl")
}
<commit_msg>finish page-frond-end actions<commit_after><|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"math/rand"
	"os"
	"strconv"
	"sync"
	"time"

	"k8s.io/kubernetes/pkg/api"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/util/intstr"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	smallRCSize       = 5
	mediumRCSize      = 30
	bigRCSize         = 250
	smallRCGroupName  = "load-small-rc"
	mediumRCGroupName = "load-medium-rc"
	bigRCGroupName    = "load-big-rc"
	smallRCBatchSize  = 30
	mediumRCBatchSize = 5
	bigRCBatchSize    = 1
	// We start RCs/Services/pods/... in different namespace in this test.
	// nodeCountPerNamespace determines how many namespaces we will be using
	// depending on the number of nodes in the underlying cluster.
	nodeCountPerNamespace = 250
)

// This test suite can take a long time to run, so by default it is added to
// the ginkgo.skip list (see driver.go).
// To run this suite you must explicitly ask for it by setting the
// -t/--test flag or ginkgo.focus flag.
var _ = framework.KubeDescribe("Load capacity", func() {
	var c *client.Client
	var nodeCount int
	var ns string
	var configs []*framework.RCConfig
	var namespaces []*api.Namespace

	// Gathers metrics before teardown
	// TODO add flag that allows to skip cleanup on failure
	AfterEach(func() {
		// Verify latency metrics
		highLatencyRequests, err := framework.HighLatencyRequests(c)
		framework.ExpectNoError(err, "Too many instances metrics above the threshold")
		Expect(highLatencyRequests).NotTo(BeNumerically(">", 0))
	})

	// Explicitly put here, to delete namespace at the end of the test
	// (after measuring latency metrics, etc.).
	options := framework.FrameworkOptions{
		ClientQPS:   50,
		ClientBurst: 100,
	}
	f := framework.NewFramework("load", options, nil)
	f.NamespaceDeletionTimeout = time.Hour

	BeforeEach(func() {
		c = f.Client

		ns = f.Namespace.Name
		nodes := framework.ListSchedulableNodesOrDie(c)
		nodeCount = len(nodes.Items)
		Expect(nodeCount).NotTo(BeZero())

		// Terminating a namespace (deleting the remaining objects from it - which
		// generally means events) can affect the current run. Thus we wait for all
		// terminating namespace to be finally deleted before starting this test.
		err := framework.CheckTestingNSDeletedExcept(c, ns)
		framework.ExpectNoError(err)

		framework.ExpectNoError(framework.ResetMetrics(c))
	})

	type Load struct {
		podsPerNode int
		image       string
		command     []string
	}

	loadTests := []Load{
		// The container will consume 1 cpu and 512mb of memory.
		{podsPerNode: 3, image: "jess/stress", command: []string{"stress", "-c", "1", "-m", "2"}},
		{podsPerNode: 30, image: "gcr.io/google_containers/serve_hostname:v1.4"},
	}

	for _, testArg := range loadTests {
		name := fmt.Sprintf("should be able to handle %v pods per node", testArg.podsPerNode)
		if testArg.podsPerNode == 30 {
			name = "[Feature:Performance] " + name
		} else {
			name = "[Feature:ManualPerformance] " + name
		}
		itArg := testArg

		It(name, func() {
			// Create a number of namespaces.
			namespaces = createNamespaces(f, nodeCount, itArg.podsPerNode)

			totalPods := itArg.podsPerNode * nodeCount
			configs = generateRCConfigs(totalPods, itArg.image, itArg.command, c, namespaces)
			var services []*api.Service
			// Read the environment variable to see if we want to create services
			createServices := os.Getenv("CREATE_SERVICES")
			if createServices == "true" {
				framework.Logf("Creating services")
				services := generateServicesForConfigs(configs)
				for _, service := range services {
					_, err := c.Services(ns).Create(service)
					framework.ExpectNoError(err)
				}
			} else {
				framework.Logf("Skipping service creation")
			}

			// Simulate lifetime of RC:
			//  * create with initial size
			//  * scale RC to a random size and list all pods
			//  * scale RC to a random size and list all pods
			//  * delete it
			//
			// This will generate ~5 creations/deletions per second assuming:
			//  - X small RCs each 5 pods   [ 5 * X = totalPods / 2 ]
			//  - Y medium RCs each 30 pods [ 30 * Y = totalPods / 4 ]
			//  - Z big RCs each 250 pods   [ 250 * Z = totalPods / 4]

			// We would like to spread creating replication controllers over time
			// to make it possible to create/schedule them in the meantime.
			// Currently we assume 10 pods/second average throughput.
			// We may want to revisit it in the future.
			creatingTime := time.Duration(totalPods/10) * time.Second
			createAllRC(configs, creatingTime)
			By("============================================================================")

			// We would like to spread scaling replication controllers over time
			// to make it possible to create/schedule & delete them in the meantime.
			// Currently we assume that 10 pods/second average throughput.
			// The expected number of created/deleted pods is less than totalPods/3.
			scalingTime := time.Duration(totalPods/30) * time.Second
			scaleAllRC(configs, scalingTime)
			By("============================================================================")

			scaleAllRC(configs, scalingTime)
			By("============================================================================")

			// Cleanup all created replication controllers.
			// Currently we assume 5 pods/second average deletion throughput.
			// We may want to revisit it in the future.
			deletingTime := time.Duration(totalPods/5) * time.Second
			deleteAllRC(configs, deletingTime)
			if createServices == "true" {
				for _, service := range services {
					err := c.Services(ns).Delete(service.Name)
					framework.ExpectNoError(err)
				}
				framework.Logf("%v Services created.", len(services))
			}
		})
	}
})

func createNamespaces(f *framework.Framework, nodeCount, podsPerNode int) []*api.Namespace {
	namespaceCount := (nodeCount + nodeCountPerNamespace - 1) / nodeCountPerNamespace
	namespaces := []*api.Namespace{}
	for i := 1; i <= namespaceCount; i++ {
		namespace, err := f.CreateNamespace(fmt.Sprintf("load-%d-nodepods-%d", podsPerNode, i), nil)
		framework.ExpectNoError(err)
		namespaces = append(namespaces, namespace)
	}
	return namespaces
}

func computeRCCounts(total int) (int, int, int) {
	// Small RCs owns ~0.5 of total number of pods, medium and big RCs ~0.25 each.
	// For example for 3000 pods (100 nodes, 30 pods per node) there are:
	//  - 300 small RCs each 5 pods
	//  - 25 medium RCs each 30 pods
	//  - 3 big RCs each 250 pods
	bigRCCount := total / 4 / bigRCSize
	total -= bigRCCount * bigRCSize
	mediumRCCount := total / 3 / mediumRCSize
	total -= mediumRCCount * mediumRCSize
	smallRCCount := total / smallRCSize
	return smallRCCount, mediumRCCount, bigRCCount
}

func generateRCConfigs(totalPods int, image string, command []string, c *client.Client, nss []*api.Namespace) []*framework.RCConfig {
	configs := make([]*framework.RCConfig, 0)

	smallRCCount, mediumRCCount, bigRCCount := computeRCCounts(totalPods)
	configs = append(configs, generateRCConfigsForGroup(c, nss, smallRCGroupName, smallRCSize, smallRCCount, image, command)...)
	configs = append(configs, generateRCConfigsForGroup(c, nss, mediumRCGroupName, mediumRCSize, mediumRCCount, image, command)...)
	configs = append(configs, generateRCConfigsForGroup(c, nss, bigRCGroupName, bigRCSize, bigRCCount, image, command)...)

	return configs
}

func generateRCConfigsForGroup(c *client.Client, nss []*api.Namespace, groupName string, size, count int, image string, command []string) []*framework.RCConfig {
	configs := make([]*framework.RCConfig, 0, count)
	for i := 1; i <= count; i++ {
		config := &framework.RCConfig{
			Client:     c,
			Name:       groupName + "-" + strconv.Itoa(i),
			Namespace:  nss[i%len(nss)].Name,
			Timeout:    10 * time.Minute,
			Image:      image,
			Command:    command,
			Replicas:   size,
			CpuRequest: 10,       // 0.01 core
			MemRequest: 26214400, // 25MB
		}
		configs = append(configs, config)
	}
	return configs
}

func generateServicesForConfigs(configs []*framework.RCConfig) []*api.Service {
	services := make([]*api.Service, 0, len(configs))
	for _, config := range configs {
		serviceName := config.Name + "-svc"
		labels := map[string]string{"name": config.Name}
		service := &api.Service{
			ObjectMeta: api.ObjectMeta{
				Name:      serviceName,
				Namespace: config.Namespace,
			},
			Spec: api.ServiceSpec{
				Selector: labels,
				Ports: []api.ServicePort{{
					Port:       80,
					TargetPort: intstr.FromInt(80),
				}},
			},
		}
		services = append(services, service)
	}
	return services
}

func sleepUpTo(d time.Duration) {
	time.Sleep(time.Duration(rand.Int63n(d.Nanoseconds())))
}

func createAllRC(configs []*framework.RCConfig, creatingTime time.Duration) {
	var wg sync.WaitGroup
	wg.Add(len(configs))
	for _, config := range configs {
		go createRC(&wg, config, creatingTime)
	}
	wg.Wait()
}

func createRC(wg *sync.WaitGroup, config *framework.RCConfig, creatingTime time.Duration) {
	defer GinkgoRecover()
	defer wg.Done()

	sleepUpTo(creatingTime)
	framework.ExpectNoError(framework.RunRC(*config), fmt.Sprintf("creating rc %s", config.Name))
}

func scaleAllRC(configs []*framework.RCConfig, scalingTime time.Duration) {
	var wg sync.WaitGroup
	wg.Add(len(configs))
	for _, config := range configs {
		go scaleRC(&wg, config, scalingTime)
	}
	wg.Wait()
}

// Scales RC to a random size within [0.5*size, 1.5*size] and lists all the pods afterwards.
// Scaling happens always based on original size, not the current size.
func scaleRC(wg *sync.WaitGroup, config *framework.RCConfig, scalingTime time.Duration) {
	defer GinkgoRecover()
	defer wg.Done()

	sleepUpTo(scalingTime)
	newSize := uint(rand.Intn(config.Replicas) + config.Replicas/2)
	framework.ExpectNoError(framework.ScaleRC(config.Client, config.Namespace, config.Name, newSize, true),
		fmt.Sprintf("scaling rc %s for the first time", config.Name))
	selector := labels.SelectorFromSet(labels.Set(map[string]string{"name": config.Name}))
	options := api.ListOptions{
		LabelSelector:   selector,
		ResourceVersion: "0",
	}
	_, err := config.Client.Pods(config.Namespace).List(options)
	framework.ExpectNoError(err, fmt.Sprintf("listing pods from rc %v", config.Name))
}

func deleteAllRC(configs []*framework.RCConfig, deletingTime time.Duration) {
	var wg sync.WaitGroup
	wg.Add(len(configs))
	for _, config := range configs {
		go deleteRC(&wg, config, deletingTime)
	}
	wg.Wait()
}

func deleteRC(wg *sync.WaitGroup, config *framework.RCConfig, deletingTime time.Duration) {
	defer GinkgoRecover()
	defer wg.Done()

	sleepUpTo(deletingTime)
	framework.ExpectNoError(framework.DeleteRC(config.Client, config.Namespace, config.Name), fmt.Sprintf("deleting rc %s", config.Name))
}
<commit_msg>Fix services in load test<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"math/rand"
	"os"
	"strconv"
	"sync"
	"time"

	"k8s.io/kubernetes/pkg/api"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/util/intstr"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	smallRCSize       = 5
	mediumRCSize      = 30
	bigRCSize         = 250
	smallRCGroupName  = "load-small-rc"
	mediumRCGroupName = "load-medium-rc"
	bigRCGroupName    = "load-big-rc"
	smallRCBatchSize  = 30
	mediumRCBatchSize = 5
	bigRCBatchSize    = 1
	// We start RCs/Services/pods/... in different namespace in this test.
	// nodeCountPerNamespace determines how many namespaces we will be using
	// depending on the number of nodes in the underlying cluster.
	nodeCountPerNamespace = 250
)

// This test suite can take a long time to run, so by default it is added to
// the ginkgo.skip list (see driver.go).
// To run this suite you must explicitly ask for it by setting the
// -t/--test flag or ginkgo.focus flag.
var _ = framework.KubeDescribe("Load capacity", func() {
	var c *client.Client
	var nodeCount int
	var ns string
	var configs []*framework.RCConfig
	var namespaces []*api.Namespace

	// Gathers metrics before teardown
	// TODO add flag that allows to skip cleanup on failure
	AfterEach(func() {
		// Verify latency metrics
		highLatencyRequests, err := framework.HighLatencyRequests(c)
		framework.ExpectNoError(err, "Too many instances metrics above the threshold")
		Expect(highLatencyRequests).NotTo(BeNumerically(">", 0))
	})

	// Explicitly put here, to delete namespace at the end of the test
	// (after measuring latency metrics, etc.).
	options := framework.FrameworkOptions{
		ClientQPS:   50,
		ClientBurst: 100,
	}
	f := framework.NewFramework("load", options, nil)
	f.NamespaceDeletionTimeout = time.Hour

	BeforeEach(func() {
		c = f.Client

		ns = f.Namespace.Name
		nodes := framework.ListSchedulableNodesOrDie(c)
		nodeCount = len(nodes.Items)
		Expect(nodeCount).NotTo(BeZero())

		// Terminating a namespace (deleting the remaining objects from it - which
		// generally means events) can affect the current run. Thus we wait for all
		// terminating namespace to be finally deleted before starting this test.
		err := framework.CheckTestingNSDeletedExcept(c, ns)
		framework.ExpectNoError(err)

		framework.ExpectNoError(framework.ResetMetrics(c))
	})

	type Load struct {
		podsPerNode int
		image       string
		command     []string
	}

	loadTests := []Load{
		// The container will consume 1 cpu and 512mb of memory.
		{podsPerNode: 3, image: "jess/stress", command: []string{"stress", "-c", "1", "-m", "2"}},
		{podsPerNode: 30, image: "gcr.io/google_containers/serve_hostname:v1.4"},
	}

	for _, testArg := range loadTests {
		name := fmt.Sprintf("should be able to handle %v pods per node", testArg.podsPerNode)
		if testArg.podsPerNode == 30 {
			name = "[Feature:Performance] " + name
		} else {
			name = "[Feature:ManualPerformance] " + name
		}
		itArg := testArg

		It(name, func() {
			// Create a number of namespaces.
			namespaces = createNamespaces(f, nodeCount, itArg.podsPerNode)

			totalPods := itArg.podsPerNode * nodeCount
			configs = generateRCConfigs(totalPods, itArg.image, itArg.command, c, namespaces)
			var services []*api.Service
			// Read the environment variable to see if we want to create services
			createServices := os.Getenv("CREATE_SERVICES")
			if createServices == "true" {
				framework.Logf("Creating services")
				services := generateServicesForConfigs(configs)
				for _, service := range services {
					_, err := c.Services(service.Namespace).Create(service)
					framework.ExpectNoError(err)
				}
			} else {
				framework.Logf("Skipping service creation")
			}

			// Simulate lifetime of RC:
			//  * create with initial size
			//  * scale RC to a random size and list all pods
			//  * scale RC to a random size and list all pods
			//  * delete it
			//
			// This will generate ~5 creations/deletions per second assuming:
			//  - X small RCs each 5 pods   [ 5 * X = totalPods / 2 ]
			//  - Y medium RCs each 30 pods [ 30 * Y = totalPods / 4 ]
			//  - Z big RCs each 250 pods   [ 250 * Z = totalPods / 4]

			// We would like to spread creating replication controllers over time
			// to make it possible to create/schedule them in the meantime.
			// Currently we assume 10 pods/second average throughput.
			// We may want to revisit it in the future.
			creatingTime := time.Duration(totalPods/10) * time.Second
			createAllRC(configs, creatingTime)
			By("============================================================================")

			// We would like to spread scaling replication controllers over time
			// to make it possible to create/schedule & delete them in the meantime.
			// Currently we assume that 10 pods/second average throughput.
			// The expected number of created/deleted pods is less than totalPods/3.
			scalingTime := time.Duration(totalPods/30) * time.Second
			scaleAllRC(configs, scalingTime)
			By("============================================================================")

			scaleAllRC(configs, scalingTime)
			By("============================================================================")

			// Cleanup all created replication controllers.
			// Currently we assume 5 pods/second average deletion throughput.
			// We may want to revisit it in the future.
			deletingTime := time.Duration(totalPods/5) * time.Second
			deleteAllRC(configs, deletingTime)
			if createServices == "true" {
				for _, service := range services {
					err := c.Services(ns).Delete(service.Name)
					framework.ExpectNoError(err)
				}
				framework.Logf("%v Services created.", len(services))
			}
		})
	}
})

func createNamespaces(f *framework.Framework, nodeCount, podsPerNode int) []*api.Namespace {
	namespaceCount := (nodeCount + nodeCountPerNamespace - 1) / nodeCountPerNamespace
	namespaces := []*api.Namespace{}
	for i := 1; i <= namespaceCount; i++ {
		namespace, err := f.CreateNamespace(fmt.Sprintf("load-%d-nodepods-%d", podsPerNode, i), nil)
		framework.ExpectNoError(err)
		namespaces = append(namespaces, namespace)
	}
	return namespaces
}

func computeRCCounts(total int) (int, int, int) {
	// Small RCs owns ~0.5 of total number of pods, medium and big RCs ~0.25 each.
	// For example for 3000 pods (100 nodes, 30 pods per node) there are:
	//  - 300 small RCs each 5 pods
	//  - 25 medium RCs each 30 pods
	//  - 3 big RCs each 250 pods
	bigRCCount := total / 4 / bigRCSize
	total -= bigRCCount * bigRCSize
	mediumRCCount := total / 3 / mediumRCSize
	total -= mediumRCCount * mediumRCSize
	smallRCCount := total / smallRCSize
	return smallRCCount, mediumRCCount, bigRCCount
}

func generateRCConfigs(totalPods int, image string, command []string, c *client.Client, nss []*api.Namespace) []*framework.RCConfig {
	configs := make([]*framework.RCConfig, 0)

	smallRCCount, mediumRCCount, bigRCCount := computeRCCounts(totalPods)
	configs = append(configs, generateRCConfigsForGroup(c, nss, smallRCGroupName, smallRCSize, smallRCCount, image, command)...)
	configs = append(configs, generateRCConfigsForGroup(c, nss, mediumRCGroupName, mediumRCSize, mediumRCCount, image, command)...)
	configs = append(configs, generateRCConfigsForGroup(c, nss, bigRCGroupName, bigRCSize, bigRCCount, image, command)...)

	return configs
}

func generateRCConfigsForGroup(c *client.Client, nss []*api.Namespace, groupName string, size, count int, image string, command []string) []*framework.RCConfig {
	configs := make([]*framework.RCConfig, 0, count)
	for i := 1; i <= count; i++ {
		config := &framework.RCConfig{
			Client:     c,
			Name:       groupName + "-" + strconv.Itoa(i),
			Namespace:  nss[i%len(nss)].Name,
			Timeout:    10 * time.Minute,
			Image:      image,
			Command:    command,
			Replicas:   size,
			CpuRequest: 10,       // 0.01 core
			MemRequest: 26214400, // 25MB
		}
		configs = append(configs, config)
	}
	return configs
}

func generateServicesForConfigs(configs []*framework.RCConfig) []*api.Service {
	services := make([]*api.Service, 0, len(configs))
	for _, config := range configs {
		serviceName := config.Name + "-svc"
		labels := map[string]string{"name": config.Name}
		service := &api.Service{
			ObjectMeta: api.ObjectMeta{
				Name:      serviceName,
				Namespace: config.Namespace,
			},
			Spec: api.ServiceSpec{
				Selector: labels,
				Ports: []api.ServicePort{{
					Port:       80,
					TargetPort: intstr.FromInt(80),
				}},
			},
		}
		services = append(services, service)
	}
	return services
}

func sleepUpTo(d time.Duration) {
	time.Sleep(time.Duration(rand.Int63n(d.Nanoseconds())))
}

func createAllRC(configs []*framework.RCConfig, creatingTime time.Duration) {
	var wg sync.WaitGroup
	wg.Add(len(configs))
	for _, config := range configs {
		go createRC(&wg, config, creatingTime)
	}
	wg.Wait()
}

func createRC(wg *sync.WaitGroup, config *framework.RCConfig, creatingTime time.Duration) {
	defer GinkgoRecover()
	defer wg.Done()

	sleepUpTo(creatingTime)
	framework.ExpectNoError(framework.RunRC(*config), fmt.Sprintf("creating rc %s", config.Name))
}

func scaleAllRC(configs []*framework.RCConfig, scalingTime time.Duration) {
	var wg sync.WaitGroup
	wg.Add(len(configs))
	for _, config := range configs {
		go scaleRC(&wg, config, scalingTime)
	}
	wg.Wait()
}

// Scales RC to a random size within [0.5*size, 1.5*size] and lists all the pods afterwards.
// Scaling happens always based on original size, not the current size.
func scaleRC(wg *sync.WaitGroup, config *framework.RCConfig, scalingTime time.Duration) {
	defer GinkgoRecover()
	defer wg.Done()

	sleepUpTo(scalingTime)
	newSize := uint(rand.Intn(config.Replicas) + config.Replicas/2)
	framework.ExpectNoError(framework.ScaleRC(config.Client, config.Namespace, config.Name, newSize, true),
		fmt.Sprintf("scaling rc %s for the first time", config.Name))
	selector := labels.SelectorFromSet(labels.Set(map[string]string{"name": config.Name}))
	options := api.ListOptions{
		LabelSelector:   selector,
		ResourceVersion: "0",
	}
	_, err := config.Client.Pods(config.Namespace).List(options)
	framework.ExpectNoError(err, fmt.Sprintf("listing pods from rc %v", config.Name))
}

func deleteAllRC(configs []*framework.RCConfig, deletingTime time.Duration) {
	var wg sync.WaitGroup
	wg.Add(len(configs))
	for _, config := range configs {
		go deleteRC(&wg, config, deletingTime)
	}
	wg.Wait()
}

func deleteRC(wg *sync.WaitGroup, config *framework.RCConfig, deletingTime time.Duration) {
	defer GinkgoRecover()
	defer wg.Done()

	sleepUpTo(deletingTime)
	framework.ExpectNoError(framework.DeleteRC(config.Client, config.Namespace, config.Name), fmt.Sprintf("deleting rc %s", config.Name))
}
<|endoftext|>


<commit_before>package backoff

import (
	"context"
	"sync"
	"time"
)

// Ticker holds a channel that delivers `ticks' of a clock at times reported by a BackOff.
//
// Ticks will continue to arrive when the previous operation is still running,
// so operations that take a while to fail could run in quick succession.
type Ticker struct {
	C        <-chan time.Time
	c        chan time.Time
	b        BackOff
	ctx      context.Context
	stop     chan struct{}
	stopOnce sync.Once
}

// NewTicker returns a new Ticker containing a channel that will send
// the time at times specified by the BackOff argument. Ticker is
// guaranteed to tick at least once.  The channel is closed when Stop
// method is called or BackOff stops. It is not safe to manipulate the
// provided backoff policy (notably calling NextBackOff or Reset)
// while the ticker is running.
func NewTicker(b BackOff) *Ticker {
	c := make(chan time.Time)
	t := &Ticker{
		C:    c,
		c:    c,
		b:    b,
		ctx:  getContext(b),
		stop: make(chan struct{}),
	}
	t.b.Reset()
	go t.run()
	return t
}

// Stop turns off a ticker. After Stop, no more ticks will be sent.
func (t *Ticker) Stop() {
	t.stopOnce.Do(func() { close(t.stop) })
}

func (t *Ticker) run() {
	c := t.c
	defer close(c)

	// Ticker is guaranteed to tick at least once.
	afterC := t.send(time.Now())

	for {
		if afterC == nil {
			return
		}

		select {
		case tick := <-afterC:
			afterC = t.send(tick)
		case <-t.stop:
			t.c = nil // Prevent future ticks from being sent to the channel.
			return
		case <-t.ctx.Done():
			return
		}
	}
}

func (t *Ticker) send(tick time.Time) <-chan time.Time {
	select {
	case t.c <- tick:
	case <-t.stop:
		return nil
	}

	next := t.b.NextBackOff()
	if next == Stop {
		t.Stop()
		return nil
	}

	return time.After(next)
}
<commit_msg>add NewTickerWithTimer function<commit_after>package backoff

import (
	"context"
	"sync"
	"time"
)

// Ticker holds a channel that delivers `ticks' of a clock at times reported by a BackOff.
//
// Ticks will continue to arrive when the previous operation is still running,
// so operations that take a while to fail could run in quick succession.
type Ticker struct {
	C        <-chan time.Time
	c        chan time.Time
	b        BackOff
	ctx      context.Context
	timer    Timer
	stop     chan struct{}
	stopOnce sync.Once
}

// NewTicker returns a new Ticker containing a channel that will send
// the time at times specified by the BackOff argument. Ticker is
// guaranteed to tick at least once.  The channel is closed when Stop
// method is called or BackOff stops. It is not safe to manipulate the
// provided backoff policy (notably calling NextBackOff or Reset)
// while the ticker is running.
func NewTicker(b BackOff) *Ticker {
	return NewTickerWithTimer(b, &DefaultTimer{})
}

func NewTickerWithTimer(b BackOff, timer Timer) *Ticker {
	c := make(chan time.Time)
	t := &Ticker{
		C:     c,
		c:     c,
		b:     b,
		ctx:   getContext(b),
		timer: timer,
		stop:  make(chan struct{}),
	}
	t.b.Reset()
	go t.run()
	return t
}

// Stop turns off a ticker. After Stop, no more ticks will be sent.
func (t *Ticker) Stop() {
	t.stopOnce.Do(func() { close(t.stop) })
}

func (t *Ticker) run() {
	c := t.c
	defer close(c)

	// Ticker is guaranteed to tick at least once.
	afterC := t.send(time.Now())

	for {
		if afterC == nil {
			return
		}

		select {
		case tick := <-afterC:
			afterC = t.send(tick)
		case <-t.stop:
			t.c = nil // Prevent future ticks from being sent to the channel.
			return
		case <-t.ctx.Done():
			return
		}
	}
}

func (t *Ticker) send(tick time.Time) <-chan time.Time {
	select {
	case t.c <- tick:
	case <-t.stop:
		return nil
	}

	next := t.b.NextBackOff()
	if next == Stop {
		t.Stop()
		return nil
	}

	t.timer.Start(next)
	return t.timer.C()
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package platformvm

import (
	"fmt"
	"time"

	"github.com/ava-labs/avalanchego/codec"
	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/snow"
	"github.com/ava-labs/avalanchego/utils/crypto"
	"github.com/ava-labs/avalanchego/vms/components/avax"
	"github.com/ava-labs/avalanchego/vms/components/verify"
	"github.com/ava-labs/avalanchego/vms/secp256k1fx"
)

var _ UnsignedDecisionTx = &UnsignedCreateSubnetTx{}

// UnsignedCreateSubnetTx is an unsigned proposal to create a new subnet
type UnsignedCreateSubnetTx struct {
	// Metadata, inputs and outputs
	BaseTx `serialize:"true"`
	// Who is authorized to manage this subnet
	Owner verify.Verifiable `serialize:"true" json:"owner"`
}

// Verify this transaction is well-formed
func (tx *UnsignedCreateSubnetTx) Verify(
	ctx *snow.Context,
	c codec.Manager,
	feeAmount uint64,
	feeAssetID ids.ID,
) error {
	switch {
	case tx == nil:
		return errNilTx
	case tx.syntacticallyVerified: // already passed syntactic verification
		return nil
	}

	if err := tx.BaseTx.Verify(ctx, c); err != nil {
		return err
	}
	if err := tx.Owner.Verify(); err != nil {
		return err
	}

	tx.syntacticallyVerified = true
	return nil
}

// SemanticVerify returns nil if [tx] is valid given the state in [db]
func (tx *UnsignedCreateSubnetTx) SemanticVerify(
	vm *VM,
	vs VersionedState,
	stx *Tx,
) (
	func() error,
	TxError,
) {
	timestamp := vm.internalState.GetTimestamp()
	createSubnetTxFee := vm.getCreateSubnetTxFee(timestamp)
	// Make sure this transaction is well formed.
	if err := tx.Verify(vm.ctx, vm.codec, createSubnetTxFee, vm.ctx.AVAXAssetID); err != nil {
		return nil, permError{err}
	}

	// Verify the flowcheck
	if err := vm.semanticVerifySpend(vs, tx, tx.Ins, tx.Outs, stx.Creds, createSubnetTxFee, vm.ctx.AVAXAssetID); err != nil {
		return nil, err
	}

	// Consume the UTXOS
	consumeInputs(vs, tx.Ins)
	// Produce the UTXOS
	txID := tx.ID()
	produceOutputs(vs, txID, vm.ctx.AVAXAssetID, tx.Outs)
	// Attempt to the new chain to the database
	vs.AddSubnet(stx)

	return nil, nil
}

// [controlKeys] must be unique. They will be sorted by this method.
// If [controlKeys] is nil, [tx.Controlkeys] will be an empty list.
func (vm *VM) newCreateSubnetTx(
	threshold uint32, // [threshold] of [ownerAddrs] needed to manage this subnet
	ownerAddrs []ids.ShortID, // control addresses for the new subnet
	keys []*crypto.PrivateKeySECP256K1R, // pay the fee
	changeAddr ids.ShortID, // Address to send change to, if there is any
) (*Tx, error) {
	timestamp := vm.internalState.GetTimestamp()
	createSubnetTxFee := vm.getCreateSubnetTxFee(timestamp)
	ins, outs, _, signers, err := vm.stake(keys, 0, createSubnetTxFee, changeAddr)
	if err != nil {
		return nil, fmt.Errorf("couldn't generate tx inputs/outputs: %w", err)
	}

	// Sort control addresses
	ids.SortShortIDs(ownerAddrs)

	// Create the tx
	utx := &UnsignedCreateSubnetTx{
		BaseTx: BaseTx{BaseTx: avax.BaseTx{
			NetworkID:    vm.ctx.NetworkID,
			BlockchainID: vm.ctx.ChainID,
			Ins:          ins,
			Outs:         outs,
		}},
		Owner: &secp256k1fx.OutputOwners{
			Threshold: threshold,
			Addrs:     ownerAddrs,
		},
	}
	tx := &Tx{UnsignedTx: utx}
	if err := tx.Sign(vm.codec, signers); err != nil {
		return nil, err
	}
	return tx, utx.Verify(vm.ctx, vm.codec, createSubnetTxFee, vm.ctx.AVAXAssetID)
}

func (vm *VM) getCreateSubnetTxFee(t time.Time) uint64 {
	if t.Before(vm.ApricotPhase3Time) {
		return vm.CreateAssetTxFee
	}
	return vm.CreateSubnetTxFee
}
<commit_msg>fix timestamp in createSubnetTx<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package platformvm

import (
	"fmt"
	"time"

	"github.com/ava-labs/avalanchego/codec"
	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/snow"
	"github.com/ava-labs/avalanchego/utils/crypto"
	"github.com/ava-labs/avalanchego/vms/components/avax"
	"github.com/ava-labs/avalanchego/vms/components/verify"
	"github.com/ava-labs/avalanchego/vms/secp256k1fx"
)

var _ UnsignedDecisionTx = &UnsignedCreateSubnetTx{}

// UnsignedCreateSubnetTx is an unsigned proposal to create a new subnet
type UnsignedCreateSubnetTx struct {
	// Metadata, inputs and outputs
	BaseTx `serialize:"true"`
	// Who is authorized to manage this subnet
	Owner verify.Verifiable `serialize:"true" json:"owner"`
}

// Verify this transaction is well-formed
func (tx *UnsignedCreateSubnetTx) Verify(
	ctx *snow.Context,
	c codec.Manager,
	feeAmount uint64,
	feeAssetID ids.ID,
) error {
	switch {
	case tx == nil:
		return errNilTx
	case tx.syntacticallyVerified: // already passed syntactic verification
		return nil
	}

	if err := tx.BaseTx.Verify(ctx, c); err != nil {
		return err
	}
	if err := tx.Owner.Verify(); err != nil {
		return err
	}

	tx.syntacticallyVerified = true
	return nil
}

// SemanticVerify returns nil if [tx] is valid given the state in [db]
func (tx *UnsignedCreateSubnetTx) SemanticVerify(
	vm *VM,
	vs VersionedState,
	stx *Tx,
) (
	func() error,
	TxError,
) {
	timestamp := vs.GetTimestamp()
	createSubnetTxFee := vm.getCreateSubnetTxFee(timestamp)
	// Make sure this transaction is well formed.
	if err := tx.Verify(vm.ctx, vm.codec, createSubnetTxFee, vm.ctx.AVAXAssetID); err != nil {
		return nil, permError{err}
	}

	// Verify the flowcheck
	if err := vm.semanticVerifySpend(vs, tx, tx.Ins, tx.Outs, stx.Creds, createSubnetTxFee, vm.ctx.AVAXAssetID); err != nil {
		return nil, err
	}

	// Consume the UTXOS
	consumeInputs(vs, tx.Ins)
	// Produce the UTXOS
	txID := tx.ID()
	produceOutputs(vs, txID, vm.ctx.AVAXAssetID, tx.Outs)
	// Attempt to the new chain to the database
	vs.AddSubnet(stx)

	return nil, nil
}

// [controlKeys] must be unique. They will be sorted by this method.
// If [controlKeys] is nil, [tx.Controlkeys] will be an empty list.
func (vm *VM) newCreateSubnetTx(
	threshold uint32, // [threshold] of [ownerAddrs] needed to manage this subnet
	ownerAddrs []ids.ShortID, // control addresses for the new subnet
	keys []*crypto.PrivateKeySECP256K1R, // pay the fee
	changeAddr ids.ShortID, // Address to send change to, if there is any
) (*Tx, error) {
	timestamp := vm.internalState.GetTimestamp()
	createSubnetTxFee := vm.getCreateSubnetTxFee(timestamp)
	ins, outs, _, signers, err := vm.stake(keys, 0, createSubnetTxFee, changeAddr)
	if err != nil {
		return nil, fmt.Errorf("couldn't generate tx inputs/outputs: %w", err)
	}

	// Sort control addresses
	ids.SortShortIDs(ownerAddrs)

	// Create the tx
	utx := &UnsignedCreateSubnetTx{
		BaseTx: BaseTx{BaseTx: avax.BaseTx{
			NetworkID:    vm.ctx.NetworkID,
			BlockchainID: vm.ctx.ChainID,
			Ins:          ins,
			Outs:         outs,
		}},
		Owner: &secp256k1fx.OutputOwners{
			Threshold: threshold,
			Addrs:     ownerAddrs,
		},
	}
	tx := &Tx{UnsignedTx: utx}
	if err := tx.Sign(vm.codec, signers); err != nil {
		return nil, err
	}
	return tx, utx.Verify(vm.ctx, vm.codec, createSubnetTxFee, vm.ctx.AVAXAssetID)
}

func (vm *VM) getCreateSubnetTxFee(t time.Time) uint64 {
	if t.Before(vm.ApricotPhase3Time) {
		return vm.CreateAssetTxFee
	}
	return vm.CreateSubnetTxFee
}
<|endoftext|>


<commit_before>package positions

import (
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/MKuranowski/WarsawGTFS/realtime/util"
	"github.com/cenkalti/backoff/v4"
)

// CalculateLastStopTime sets the LastStopTime object on a brigadeEntry
func (b brigadeEntry) SetLastStopTime() (err error) {
	b.LastStopTime, err = newCompareTimeFromGtfs(b.LastStopTimepoint)
	return
}

// Options represent options for creating positions GTFS-Realtime
type Options struct {
	GtfsRtTarget  string
	JSONTarget    string
	HumanReadable bool
	Apikey        string
	Brigades      string
}

// Create auto-magically creates realtime feeds with position data.
// Create is designed to run a loop, hence it doesn't conatin logic to load
// brigades JSON and assumes
func Create(api VehicleAPI, brigadeMap map[string][]*brigadeEntry, prevVehicles map[string]*Vehicle, opts Options) (map[string]*Vehicle, error) {
	// 0. Prepare a container
	container := &VehicleContainer{}
	container.SyncTime = time.Now()
	container.CompareSyncTime = newCompareTimeFromTime(container.SyncTime)

	// 1. Get data from the api
	apiEntries, err := api.GetAll()
	if err != nil {
		return nil, err
	}

	// 2. Create Vehicle objects
	err = container.Prepare(apiEntries)
	if err != nil {
		return nil, err
	}

	// 3. Match all Vehicles to trips
	err = container.MatchAll(brigadeMap, prevVehicles)
	if err != nil {
		return nil, err
	}

	// 4. Export to JSON
	if opts.JSONTarget != "" {
		err = container.SaveJSON(opts.JSONTarget)
		if err != nil {
			return container.Vehicles, err
		}
	}

	// 5. Export to GTFS-RT
	if opts.JSONTarget != "" {
		err = container.SavePB(opts.GtfsRtTarget, opts.HumanReadable)
	}

	return container.Vehicles, err
}

// Main auto-magically creates vehicle position data
func Main(client *http.Client, opts Options) (err error) {
	// Load brigades.json
	brigadeMap, err := loadBrigades(opts.Brigades, client)
	if err != nil {
		return
	}

	// Create other objects required by the Create function
	var prevVehicles map[string]*Vehicle
	api := VehicleAPI{Key: opts.Apikey, Client: client}

	// Call Create
	_, err = Create(api, brigadeMap, prevVehicles, opts)
	return
}

// routesResource is a pair of resource pointing to a GTFS file and a routeMap
type brigadesResource struct {
	Resource   util.Resource
	BrigadeMap map[string][]*brigadeEntry
}

// Update automatically updates the RouteMap if the Resource has changed
func (rr *brigadesResource) Update() error {
	// Check for GTFS updates
	shouldUpdate, err := rr.Resource.Check()
	if err != nil {
		return err
	} else if shouldUpdate || rr.BrigadeMap == nil {
		log.Println("brigades.json changed, reloading")

		// Try to fetch updated brigades.json
		newData, err := rr.Resource.Fetch()
		if err != nil {
			return err
		}
		defer newData.Close()

		// Read brigades.json
		rawData, err := ioutil.ReadAll(newData)
		if err != nil {
			return err
		}

		// Re-load brigades
		rr.BrigadeMap, err = makeBrigades(rawData)
		if err != nil {
			return err
		}
	}
	return nil
}

// Loop automatically updates the GTFS-RT Positions files
func Loop(client *http.Client, jsonResource util.Resource, sleepTime time.Duration, opts Options) (err error) {
	// Automatic wrapper around the resource
	var prevPositions map[string]*Vehicle
	api := VehicleAPI{Key: opts.Apikey, Client: client}
	br := brigadesResource{Resource: jsonResource}

	// Backoff shit
	exponantialBackoff := backoff.NewExponentialBackOff()
	exponantialBackoff.Multiplier = 2
	loopBackoff := backoff.WithMaxRetries(exponantialBackoff, 12)

	for {
		// Try to update brigades.json
		err = br.Update()
		if err != nil {
			return
		}

		// Try updating the GTFS-RT
		loopBackoff.Reset()
		for sleep := time.Duration(0); sleep != backoff.Stop; sleep = loopBackoff.NextBackOff() {
			// Print error when backing off
			if sleep != 0 {
				// Log when backingoff
				sleepUntil := time.Now().Add(sleep).Format("15:04:05")
				log.Printf(
					"Updating the GTFS-RT Positions failed. Backoff until %s. Error: %q.\n",
					sleepUntil, err.Error(),
				)

				// Sleep for the backoff
				time.Sleep(sleep)
			}

			// Try to update the GTFS-RT
			var newPositions map[string]*Vehicle
			newPositions, err = Create(api, br.BrigadeMap, prevPositions, opts)
			if err != nil {
				return
			}

			// Replace positions
			prevPositions, newPositions = newPositions, nil

			// If no errors were encountered, break out of the backoff loop
			if err == nil {
				log.Println("GTFS-RT Positions updated successfully.")
				break
			}
		}
		if err != nil {
			return
		}

		// Sleep until next try
		time.Sleep(sleepTime)
	}

}
<commit_msg>positions: fix no backoff<commit_after>package positions

import (
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/MKuranowski/WarsawGTFS/realtime/util"
	"github.com/cenkalti/backoff/v4"
)

// CalculateLastStopTime sets the LastStopTime object on a brigadeEntry
func (b brigadeEntry) SetLastStopTime() (err error) {
	b.LastStopTime, err = newCompareTimeFromGtfs(b.LastStopTimepoint)
	return
}

// Options represent options for creating positions GTFS-Realtime
type Options struct {
	GtfsRtTarget  string
	JSONTarget    string
	HumanReadable bool
	Apikey        string
	Brigades      string
}

// Create auto-magically creates realtime feeds with position data.
// Create is designed to run a loop, hence it doesn't conatin logic to load
// brigades JSON and assumes
func Create(api VehicleAPI, brigadeMap map[string][]*brigadeEntry, prevVehicles map[string]*Vehicle, opts Options) (map[string]*Vehicle, error) {
	// 0. Prepare a container
	container := &VehicleContainer{}
	container.SyncTime = time.Now()
	container.CompareSyncTime = newCompareTimeFromTime(container.SyncTime)

	// 1. Get data from the api
	apiEntries, err := api.GetAll()
	if err != nil {
		return nil, err
	}

	// 2. Create Vehicle objects
	err = container.Prepare(apiEntries)
	if err != nil {
		return nil, err
	}

	// 3. Match all Vehicles to trips
	err = container.MatchAll(brigadeMap, prevVehicles)
	if err != nil {
		return nil, err
	}

	// 4. Export to JSON
	if opts.JSONTarget != "" {
		err = container.SaveJSON(opts.JSONTarget)
		if err != nil {
			return container.Vehicles, err
		}
	}

	// 5. Export to GTFS-RT
	if opts.JSONTarget != "" {
		err = container.SavePB(opts.GtfsRtTarget, opts.HumanReadable)
	}

	return container.Vehicles, err
}

// Main auto-magically creates vehicle position data
func Main(client *http.Client, opts Options) (err error) {
	// Load brigades.json
	brigadeMap, err := loadBrigades(opts.Brigades, client)
	if err != nil {
		return
	}

	// Create other objects required by the Create function
	var prevVehicles map[string]*Vehicle
	api := VehicleAPI{Key: opts.Apikey, Client: client}

	// Call Create
	_, err = Create(api, brigadeMap, prevVehicles, opts)
	return
}

// routesResource is a pair of resource pointing to a GTFS file and a routeMap
type brigadesResource struct {
	Resource   util.Resource
	BrigadeMap map[string][]*brigadeEntry
}

// Update automatically updates the RouteMap if the Resource has changed
func (rr *brigadesResource) Update() error {
	// Check for GTFS updates
	shouldUpdate, err := rr.Resource.Check()
	if err != nil {
		return err
	} else if shouldUpdate || rr.BrigadeMap == nil {
		log.Println("brigades.json changed, reloading")

		// Try to fetch updated brigades.json
		newData, err := rr.Resource.Fetch()
		if err != nil {
			return err
		}
		defer newData.Close()

		// Read brigades.json
		rawData, err := ioutil.ReadAll(newData)
		if err != nil {
			return err
		}

		// Re-load brigades
		rr.BrigadeMap, err = makeBrigades(rawData)
		if err != nil {
			return err
		}
	}
	return nil
}

// Loop automatically updates the GTFS-RT Positions files
func Loop(client *http.Client, jsonResource util.Resource, sleepTime time.Duration, opts Options) (err error) {
	// Automatic wrapper around the resource
	var prevPositions map[string]*Vehicle
	api := VehicleAPI{Key: opts.Apikey, Client: client}
	br := brigadesResource{Resource: jsonResource}

	// Backoff shit
	exponantialBackoff := backoff.NewExponentialBackOff()
	exponantialBackoff.Multiplier = 2
	loopBackoff := backoff.WithMaxRetries(exponantialBackoff, 12)

	for {
		// Try to update brigades.json
		err = br.Update()
		if err != nil {
			return
		}

		// Try updating the GTFS-RT
		loopBackoff.Reset()
		for sleep := time.Duration(0); sleep != backoff.Stop; sleep = loopBackoff.NextBackOff() {
			// Print error when backing off
			if sleep != 0 {
				// Log when backingoff
				sleepUntil := time.Now().Add(sleep).Format("15:04:05")
				log.Printf(
					"Updating the GTFS-RT Positions failed. Backoff until %s. Error: %q.\n",
					sleepUntil, err.Error(),
				)

				// Sleep for the backoff
				time.Sleep(sleep)
			}

			// Try to update the GTFS-RT
			var newPositions map[string]*Vehicle
			newPositions, err = Create(api, br.BrigadeMap, prevPositions, opts)

			// If no errors were encountered, break out of the backoff loop
			if err == nil {
				prevPositions, newPositions = newPositions, nil
				log.Println("GTFS-RT Positions updated successfully.")
				break
			}
		}
		if err != nil {
			return
		}

		// Sleep until next try
		time.Sleep(sleepTime)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"sync"
	"testing"
	"time"

	. "gopkg.in/check.v1"
)

func init() {
	//log.SetFlags(log.LstdFlags | log.Lshortfile)
	log.SetOutput(ioutil.Discard)
}

// something that can wrap a gocheck.C testing.T or testing.B
// Just add more methods as we need them.
type Tester interface {
	Fatal(args ...interface{})
	Fatalf(format string, args ...interface{})
	Log(args ...interface{})
	Logf(format string, args ...interface{})
}

func Test(t *testing.T) { TestingT(t) }

type BasicSuite struct {
	servers []*testServer
	service *Service
}

var _ = Suite(&BasicSuite{})

// Make Setup and TearDown more generic, so we can bypass the gocheck Suite if
// needed.
func mySetup(s *BasicSuite, t Tester) {
	// start 4 possible backend servers
	ports := []string{"2001", "2002", "2003", "2004"}
	for _, p := range ports {
		server, err := NewTestServer("127.0.0.1:"+p, t)
		if err != nil {
			t.Fatal(err)
		}
		s.servers = append(s.servers, server)
	}

	svcCfg := ServiceConfig{
		Name: "testService",
		Addr: "127.0.0.1:2222",
	}

	if err := Registry.AddService(svcCfg); err != nil {
		t.Fatal(err)
	}

	s.service = Registry.GetService(svcCfg.Name)
}

// shutdown our backend servers
func myTearDown(s *BasicSuite, t Tester) {
	for _, s := range s.servers {
		s.Stop()
	}

	// get rid of the servers refs too!
	s.servers = nil

	err := Registry.RemoveService(s.service.Name)
	if err != nil {
		t.Fatalf("could not remove service '%s': %s", s.service.Name, err)
	}
}

func (s *BasicSuite) SetUpTest(c *C) {
	mySetup(s, c)
}

func (s *BasicSuite) TearDownTest(c *C) {
	myTearDown(s, c)
}

// Add a default backend for the next server we have running
func (s *BasicSuite) AddBackend(c Tester) {
	// get the backends via Config to use the Service's locking.
	svcCfg := s.service.Config()
	next := len(svcCfg.Backends)
	if next >= len(s.servers) {
		c.Fatal("no more servers")
	}

	name := fmt.Sprintf("backend_%d", next)
	cfg := BackendConfig{
		Name:      name,
		Addr:      s.servers[next].addr,
		CheckAddr: s.servers[next].addr,
	}

	s.service.add(NewBackend(cfg))
}

// Connect to address, and check response after write.
func checkResp(addr, expected string, c Tester) {
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		c.Fatal(err)
	}
	defer conn.Close()

	if _, err := io.WriteString(conn, "testing\n"); err != nil {
		c.Fatal(err)
	}

	buff := make([]byte, 1024)
	n, err := conn.Read(buff)
	if err != nil {
		c.Fatal(err)
	}

	resp := string(buff[:n])
	if resp == "" {
		c.Fatal("No response")
	}

	if expected != "" && resp != expected {
		c.Fatal("Expected", expected, ", got", resp)
	}
}

func (s *BasicSuite) TestSingleBackend(c *C) {
	s.AddBackend(c)

	checkResp(s.service.Addr, s.servers[0].addr, c)
}

func (s *BasicSuite) TestRoundRobin(c *C) {
	s.AddBackend(c)
	s.AddBackend(c)

	checkResp(s.service.Addr, s.servers[0].addr, c)
	checkResp(s.service.Addr, s.servers[1].addr, c)
	checkResp(s.service.Addr, s.servers[0].addr, c)
	checkResp(s.service.Addr, s.servers[1].addr, c)
}

func (s *BasicSuite) TestWeightedRoundRobin(c *C) {
	s.AddBackend(c)
	s.AddBackend(c)
	s.AddBackend(c)

	s.service.Backends[0].Weight = 1
	s.service.Backends[1].Weight = 2
	s.service.Backends[2].Weight = 3

	// we already checked that we connect to the correct backends,
	// so skip the tcp connection this time.

	// one from the first server
	c.Assert(s.service.next().Name, Equals, "backend_0")
	// A weight of 2 should return twice
	c.Assert(s.service.next().Name, Equals, "backend_1")
	c.Assert(s.service.next().Name, Equals, "backend_1")
	// And a weight of 3 should return thrice
	c.Assert(s.service.next().Name, Equals, "backend_2")
	c.Assert(s.service.next().Name, Equals, "backend_2")
	c.Assert(s.service.next().Name, Equals, "backend_2")
	// and once around or good measure
	c.Assert(s.service.next().Name, Equals, "backend_0")
}

func (s *BasicSuite) TestLeastConn(c *C) {
	// replace out default service with one using LeastConn balancing
	Registry.RemoveService("testService")
	svcCfg := ServiceConfig{
		Name:    "testService",
		Addr:    "127.0.0.1:2223",
		Balance: "LC",
	}

	if err := Registry.AddService(svcCfg); err != nil {
		c.Fatal(err)
	}
	s.service = Registry.GetService("testService")

	s.AddBackend(c)
	s.AddBackend(c)

	// tie up 4 connections to the backends
	for i := 0; i < 4; i++ {
		conn, e := net.Dial("tcp", s.service.Addr)
		if e != nil {
			c.Fatal(e)
		}
		defer conn.Close()
	}

	s.AddBackend(c)

	checkResp(s.service.Addr, s.servers[2].addr, c)
	checkResp(s.service.Addr, s.servers[2].addr, c)
}

// Test health check by taking down a server from a configured backend
func (s *BasicSuite) TestFailedCheck(c *C) {
	s.service.CheckInterval = 500
	s.service.Fall = 1
	s.AddBackend(c)

	stats := s.service.Stats()
	c.Assert(stats.Backends[0].Up, Equals, true)

	// Stop the server, and see if the backend shows Down after our check
	// interval.
	s.servers[0].Stop()
	time.Sleep(800 * time.Millisecond)

	stats = s.service.Stats()
	c.Assert(stats.Backends[0].Up, Equals, false)
	c.Assert(stats.Backends[0].CheckFail, Equals, 1)

	// now try and connect to the service
	conn, err := net.Dial("tcp", s.service.Addr)
	if err != nil {
		// we should still get an initial connection
		c.Fatal(err)
	}

	b := make([]byte, 1024)
	n, err := conn.Read(b)
	if n != 0 || err != io.EOF {
		c.Fatal("connection should have been closed")
	}

	// now bring that server back up
	server, err := NewTestServer(s.servers[0].addr, c)
	if err != nil {
		c.Fatal(err)
	}
	s.servers[0] = server

	time.Sleep(800 * time.Millisecond)
	stats = s.service.Stats()
	c.Assert(stats.Backends[0].Up, Equals, true)
}

// Update a backend in place
func (s *BasicSuite) TestUpdateBackend(c *C) {
	s.service.CheckInterval = 500
	s.service.Fall = 1
	s.AddBackend(c)

	cfg := s.service.Config()
	backendCfg := cfg.Backends[0]

	c.Assert(backendCfg.CheckAddr, Equals, backendCfg.Addr)

	backendCfg.CheckAddr = ""
	s.service.add(NewBackend(backendCfg))

	// see if the config reflects the new backend
	cfg = s.service.Config()
	c.Assert(len(cfg.Backends), Equals, 1)
	c.Assert(cfg.Backends[0].CheckAddr, Equals, "")

	// Stopping the server should not take down the backend
	// since there is no longer a Check address.
	s.servers[0].Stop()
	time.Sleep(800 * time.Millisecond)

	stats := s.service.Stats()
	c.Assert(stats.Backends[0].Up, Equals, true)
	// should have been no check failures
	c.Assert(stats.Backends[0].CheckFail, Equals, 0)
}

// Test removal of a single Backend from a service with multiple.
func (s *BasicSuite) TestRemoveBackend(c *C) {
	s.AddBackend(c)
	s.AddBackend(c)

	stats, err := Registry.ServiceStats("testService")
	if err != nil {
		c.Fatal(err)
	}

	c.Assert(len(stats.Backends), Equals, 2)

	backend1 := stats.Backends[0].Name

	err = Registry.RemoveBackend("testService", backend1)
	if err != nil {
		c.Fatal(err)
	}

	stats, err = Registry.ServiceStats("testService")
	if err != nil {
		c.Fatal(err)
	}

	c.Assert(len(stats.Backends), Equals, 1)

	_, err = Registry.BackendStats("testService", backend1)
	c.Assert(err, Equals, ErrNoBackend)
}

func (s *BasicSuite) TestUpdateService(c *C) {
	svcCfg := ServiceConfig{
		Name: "Update",
		Addr: "127.0.0.1:9324",
	}

	if err := Registry.AddService(svcCfg); err != nil {
		c.Fatal(err)
	}

	svc := Registry.GetService("Update")
	if svc == nil {
		c.Fatal(ErrNoService)
	}

	svcCfg = ServiceConfig{
		Name: "Update",
		Addr: "127.0.0.1:9425",
	}

	// Make sure we can't add it through AddService
	if err := Registry.AddService(svcCfg); err == nil {
		c.Fatal(err)
	}

	// Now update the service
	if err := Registry.UpdateService(svcCfg, false); err != nil {
		c.Fatal(err)
	}

	svc = Registry.GetService("Update")
	if svc == nil {
		c.Fatal(ErrNoService)
	}
	c.Assert(svc.Addr, Equals, "127.0.0.1:9425")

	if err := Registry.RemoveService("Update"); err != nil {
		c.Fatal(err)
	}
}

// Add backends and run response tests in parallel
func (s *BasicSuite) TestParallel(c *C) {
	var wg sync.WaitGroup

	client := func(i int) {
		s.AddBackend(c)
		// do a bunch of new connections in unison
		for i := 0; i < 100; i++ {
			checkResp(s.service.Addr, "", c)
		}

		conn, err := net.Dial("tcp", s.service.Addr)
		if err != nil {
			// we should still get an initial connection
			c.Fatal(err)
		}
		defer conn.Close()

		// now do some more continuous ping-pongs with the server
		buff := make([]byte, 1024)

		for i := 0; i < 1000; i++ {
			n, err := io.WriteString(conn, "Testing testing\n")
			if err != nil || n == 0 {
				c.Fatal("couldn't write:", err)
			}

			n, err = conn.Read(buff)
			if err != nil || n == 0 {
				c.Fatal("no response:", err)
			}
		}
		wg.Done()
	}

	for i := 0; i < 4; i++ {
		wg.Add(1)
		go client(i)
	}

	wg.Wait()
}
<commit_msg>Store exposed ports w/ ServiceConfig<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"sync"
	"testing"
	"time"

	. "gopkg.in/check.v1"
)

func init() {
	//log.SetFlags(log.LstdFlags | log.Lshortfile)
	log.SetOutput(ioutil.Discard)
}

// something that can wrap a gocheck.C testing.T or testing.B
// Just add more methods as we need them.
type Tester interface {
	Fatal(args ...interface{})
	Fatalf(format string, args ...interface{})
	Log(args ...interface{})
	Logf(format string, args ...interface{})
}

func Test(t *testing.T) { TestingT(t) }

type BasicSuite struct {
	servers []*testServer
	service *Service
}

var _ = Suite(&BasicSuite{})

// Make Setup and TearDown more generic, so we can bypass the gocheck Suite if
// needed.
func mySetup(s *BasicSuite, t Tester) {
	// start 4 possible backend servers
	ports := []string{"2001", "2002", "2003", "2004"}
	for _, p := range ports {
		server, err := NewTestServer("127.0.0.1:"+p, t)
		if err != nil {
			t.Fatal(err)
		}
		s.servers = append(s.servers, server)
	}

	svcCfg := ServiceConfig{
		Name: "testService",
		Addr: "127.0.0.1:2000",
	}

	if err := Registry.AddService(svcCfg); err != nil {
		t.Fatal(err)
	}

	s.service = Registry.GetService(svcCfg.Name)
}

// shutdown our backend servers
func myTearDown(s *BasicSuite, t Tester) {
	for _, s := range s.servers {
		s.Stop()
	}

	// get rid of the servers refs too!
	s.servers = nil

	err := Registry.RemoveService(s.service.Name)
	if err != nil {
		t.Fatalf("could not remove service '%s': %s", s.service.Name, err)
	}
}

func (s *BasicSuite) SetUpTest(c *C) {
	mySetup(s, c)
}

func (s *BasicSuite) TearDownTest(c *C) {
	myTearDown(s, c)
}

// Add a default backend for the next server we have running
func (s *BasicSuite) AddBackend(c Tester) {
	// get the backends via Config to use the Service's locking.
	svcCfg := s.service.Config()
	next := len(svcCfg.Backends)
	if next >= len(s.servers) {
		c.Fatal("no more servers")
	}

	name := fmt.Sprintf("backend_%d", next)
	cfg := BackendConfig{
		Name:      name,
		Addr:      s.servers[next].addr,
		CheckAddr: s.servers[next].addr,
	}

	s.service.add(NewBackend(cfg))
}

// Connect to address, and check response after write.
func checkResp(addr, expected string, c Tester) {
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		c.Fatal(err)
	}
	defer conn.Close()

	if _, err := io.WriteString(conn, "testing\n"); err != nil {
		c.Fatal(err)
	}

	buff := make([]byte, 1024)
	n, err := conn.Read(buff)
	if err != nil {
		c.Fatal(err)
	}

	resp := string(buff[:n])
	if resp == "" {
		c.Fatal("No response")
	}

	if expected != "" && resp != expected {
		c.Fatal("Expected", expected, ", got", resp)
	}
}

func (s *BasicSuite) TestSingleBackend(c *C) {
	s.AddBackend(c)

	checkResp(s.service.Addr, s.servers[0].addr, c)
}

func (s *BasicSuite) TestRoundRobin(c *C) {
	s.AddBackend(c)
	s.AddBackend(c)

	checkResp(s.service.Addr, s.servers[0].addr, c)
	checkResp(s.service.Addr, s.servers[1].addr, c)
	checkResp(s.service.Addr, s.servers[0].addr, c)
	checkResp(s.service.Addr, s.servers[1].addr, c)
}

func (s *BasicSuite) TestWeightedRoundRobin(c *C) {
	s.AddBackend(c)
	s.AddBackend(c)
	s.AddBackend(c)

	s.service.Backends[0].Weight = 1
	s.service.Backends[1].Weight = 2
	s.service.Backends[2].Weight = 3

	// we already checked that we connect to the correct backends,
	// so skip the tcp connection this time.

	// one from the first server
	c.Assert(s.service.next().Name, Equals, "backend_0")
	// A weight of 2 should return twice
	c.Assert(s.service.next().Name, Equals, "backend_1")
	c.Assert(s.service.next().Name, Equals, "backend_1")
	// And a weight of 3 should return thrice
	c.Assert(s.service.next().Name, Equals, "backend_2")
	c.Assert(s.service.next().Name, Equals, "backend_2")
	c.Assert(s.service.next().Name, Equals, "backend_2")
	// and once around or good measure
	c.Assert(s.service.next().Name, Equals, "backend_0")
}

func (s *BasicSuite) TestLeastConn(c *C) {
	// replace out default service with one using LeastConn balancing
	Registry.RemoveService("testService")
	svcCfg := ServiceConfig{
		Name:    "testService",
		Addr:    "127.0.0.1:2223",
		Balance: "LC",
	}

	if err := Registry.AddService(svcCfg); err != nil {
		c.Fatal(err)
	}
	s.service = Registry.GetService("testService")

	s.AddBackend(c)
	s.AddBackend(c)

	// tie up 4 connections to the backends
	for i := 0; i < 4; i++ {
		conn, e := net.Dial("tcp", s.service.Addr)
		if e != nil {
			c.Fatal(e)
		}
		defer conn.Close()
	}

	s.AddBackend(c)

	checkResp(s.service.Addr, s.servers[2].addr, c)
	checkResp(s.service.Addr, s.servers[2].addr, c)
}

// Test health check by taking down a server from a configured backend
func (s *BasicSuite) TestFailedCheck(c *C) {
	s.service.CheckInterval = 500
	s.service.Fall = 1
	s.AddBackend(c)

	stats := s.service.Stats()
	c.Assert(stats.Backends[0].Up, Equals, true)

	// Stop the server, and see if the backend shows Down after our check
	// interval.
	s.servers[0].Stop()
	time.Sleep(800 * time.Millisecond)

	stats = s.service.Stats()
	c.Assert(stats.Backends[0].Up, Equals, false)
	c.Assert(stats.Backends[0].CheckFail, Equals, 1)

	// now try and connect to the service
	conn, err := net.Dial("tcp", s.service.Addr)
	if err != nil {
		// we should still get an initial connection
		c.Fatal(err)
	}

	b := make([]byte, 1024)
	n, err := conn.Read(b)
	if n != 0 || err != io.EOF {
		c.Fatal("connection should have been closed")
	}

	// now bring that server back up
	server, err := NewTestServer(s.servers[0].addr, c)
	if err != nil {
		c.Fatal(err)
	}
	s.servers[0] = server

	time.Sleep(800 * time.Millisecond)
	stats = s.service.Stats()
	c.Assert(stats.Backends[0].Up, Equals, true)
}

// Update a backend in place
func (s *BasicSuite) TestUpdateBackend(c *C) {
	s.service.CheckInterval = 500
	s.service.Fall = 1
	s.AddBackend(c)

	cfg := s.service.Config()
	backendCfg := cfg.Backends[0]

	c.Assert(backendCfg.CheckAddr, Equals, backendCfg.Addr)

	backendCfg.CheckAddr = ""
	s.service.add(NewBackend(backendCfg))

	// see if the config reflects the new backend
	cfg = s.service.Config()
	c.Assert(len(cfg.Backends), Equals, 1)
	c.Assert(cfg.Backends[0].CheckAddr, Equals, "")

	// Stopping the server should not take down the backend
	// since there is no longer a Check address.
	s.servers[0].Stop()
	time.Sleep(800 * time.Millisecond)

	stats := s.service.Stats()
	c.Assert(stats.Backends[0].Up, Equals, true)
	// should have been no check failures
	c.Assert(stats.Backends[0].CheckFail, Equals, 0)
}

// Test removal of a single Backend from a service with multiple.
func (s *BasicSuite) TestRemoveBackend(c *C) {
	s.AddBackend(c)
	s.AddBackend(c)

	stats, err := Registry.ServiceStats("testService")
	if err != nil {
		c.Fatal(err)
	}

	c.Assert(len(stats.Backends), Equals, 2)

	backend1 := stats.Backends[0].Name

	err = Registry.RemoveBackend("testService", backend1)
	if err != nil {
		c.Fatal(err)
	}

	stats, err = Registry.ServiceStats("testService")
	if err != nil {
		c.Fatal(err)
	}

	c.Assert(len(stats.Backends), Equals, 1)

	_, err = Registry.BackendStats("testService", backend1)
	c.Assert(err, Equals, ErrNoBackend)
}

func (s *BasicSuite) TestUpdateService(c *C) {
	svcCfg := ServiceConfig{
		Name: "Update",
		Addr: "127.0.0.1:9324",
	}

	if err := Registry.AddService(svcCfg); err != nil {
		c.Fatal(err)
	}

	svc := Registry.GetService("Update")
	if svc == nil {
		c.Fatal(ErrNoService)
	}

	svcCfg = ServiceConfig{
		Name: "Update",
		Addr: "127.0.0.1:9425",
	}

	// Make sure we can't add it through AddService
	if err := Registry.AddService(svcCfg); err == nil {
		c.Fatal(err)
	}

	// Now update the service
	if err := Registry.UpdateService(svcCfg, false); err != nil {
		c.Fatal(err)
	}

	svc = Registry.GetService("Update")
	if svc == nil {
		c.Fatal(ErrNoService)
	}
	c.Assert(svc.Addr, Equals, "127.0.0.1:9425")

	if err := Registry.RemoveService("Update"); err != nil {
		c.Fatal(err)
	}
}

// Add backends and run response tests in parallel
func (s *BasicSuite) TestParallel(c *C) {
	var wg sync.WaitGroup

	client := func(i int) {
		s.AddBackend(c)
		// do a bunch of new connections in unison
		for i := 0; i < 100; i++ {
			checkResp(s.service.Addr, "", c)
		}

		conn, err := net.Dial("tcp", s.service.Addr)
		if err != nil {
			// we should still get an initial connection
			c.Fatal(err)
		}
		defer conn.Close()

		// now do some more continuous ping-pongs with the server
		buff := make([]byte, 1024)

		for i := 0; i < 1000; i++ {
			n, err := io.WriteString(conn, "Testing testing\n")
			if err != nil || n == 0 {
				c.Fatal("couldn't write:", err)
			}

			n, err = conn.Read(buff)
			if err != nil || n == 0 {
				c.Fatal("no response:", err)
			}
		}
		wg.Done()
	}

	for i := 0; i < 4; i++ {
		wg.Add(1)
		go client(i)
	}

	wg.Wait()
}
<|endoftext|>


<commit_before>package sys

/*
#include <termios.h>
*/
import "C"
import

// Termios represents terminal attributes.
"syscall"

type Termios C.struct_termios

// NewTermiosFromFd extracts the terminal attribute of the given file
// descriptor.
func NewTermiosFromFd(fd int) (*Termios, error) {
	term := new(Termios)
	err := term.FromFd(fd)
	if err != nil {
		return nil, err
	}
	return term, nil
}

func (term *Termios) c() *C.struct_termios {
	return (*C.struct_termios)(term)
}

// FromFd fills term with the terminal attribute of the given file descriptor.
func (term *Termios) FromFd(fd int) error {
	_, err := C.tcgetattr((C.int)(fd), term.c())
	return err
}

// ApplyToFd applies term to the given file descriptor.
func (term *Termios) ApplyToFd(fd int) error {
	_, err := C.tcsetattr((C.int)(fd), 0, term.c())
	return err
}

// Copy returns a copy of term.
func (term *Termios) Copy() *Termios {
	v := *term
	return &v
}

// SetVTime sets the timeout in deciseconds for noncanonical read.
func (term *Termios) SetVTime(v uint8) {
	term.c_cc[C.VTIME] = C.cc_t(v)
}

// SetVMin sets the minimal number of characters for noncanonical read.
func (term *Termios) SetVMin(v uint8) {
	term.c_cc[C.VMIN] = C.cc_t(v)
}

func setFlag(flag *C.tcflag_t, mask C.tcflag_t, v bool) {
	if v {
		*flag |= mask
	} else {
		*flag &= ^mask
	}
}

// SetICanon sets the canonical flag.
func (term *Termios) SetICanon(v bool) {
	setFlag(&term.c_lflag, C.ICANON, v)
}

// SetEcho sets the echo flag.
func (term *Termios) SetEcho(v bool) {
	setFlag(&term.c_lflag, C.ECHO, v)
}

// FlushInput discards data written to a file descriptor but not read.
func FlushInput(fd int) error {
	_, err := C.tcflush((C.int)(fd), syscall.TCIFLUSH)
	return err
}
<commit_msg>fix a weird formatting issue<commit_after>package sys

/*
#include <termios.h>
*/
import "C"
import "syscall"

// Termios represents terminal attributes.
type Termios C.struct_termios

// NewTermiosFromFd extracts the terminal attribute of the given file
// descriptor.
func NewTermiosFromFd(fd int) (*Termios, error) {
	term := new(Termios)
	err := term.FromFd(fd)
	if err != nil {
		return nil, err
	}
	return term, nil
}

func (term *Termios) c() *C.struct_termios {
	return (*C.struct_termios)(term)
}

// FromFd fills term with the terminal attribute of the given file descriptor.
func (term *Termios) FromFd(fd int) error {
	_, err := C.tcgetattr((C.int)(fd), term.c())
	return err
}

// ApplyToFd applies term to the given file descriptor.
func (term *Termios) ApplyToFd(fd int) error {
	_, err := C.tcsetattr((C.int)(fd), 0, term.c())
	return err
}

// Copy returns a copy of term.
func (term *Termios) Copy() *Termios {
	v := *term
	return &v
}

// SetVTime sets the timeout in deciseconds for noncanonical read.
func (term *Termios) SetVTime(v uint8) {
	term.c_cc[C.VTIME] = C.cc_t(v)
}

// SetVMin sets the minimal number of characters for noncanonical read.
func (term *Termios) SetVMin(v uint8) {
	term.c_cc[C.VMIN] = C.cc_t(v)
}

func setFlag(flag *C.tcflag_t, mask C.tcflag_t, v bool) {
	if v {
		*flag |= mask
	} else {
		*flag &= ^mask
	}
}

// SetICanon sets the canonical flag.
func (term *Termios) SetICanon(v bool) {
	setFlag(&term.c_lflag, C.ICANON, v)
}

// SetEcho sets the echo flag.
func (term *Termios) SetEcho(v bool) {
	setFlag(&term.c_lflag, C.ECHO, v)
}

// FlushInput discards data written to a file descriptor but not read.
func FlushInput(fd int) error {
	_, err := C.tcflush((C.int)(fd), syscall.TCIFLUSH)
	return err
}
<|endoftext|>


<commit_before>package remote

import (
	"sync"
	"sync/atomic"

	"github.com/AsynkronIT/protoactor-go/actor"
	"github.com/AsynkronIT/protoactor-go/eventstream"
)

var endpointManager *endpointManagerValue

type endpointLazy struct {
	value    *endpoint
	loadOnce sync.Once
	unloaded uint32
}

type endpoint struct {
	writer  *actor.PID
	watcher *actor.PID
}

type endpointManagerValue struct {
	connections        *sync.Map
	config             *remoteConfig
	endpointSupervisor *actor.PID
	endpointSub        *eventstream.Subscription
}

func startEndpointManager(config *remoteConfig) {
	plog.Debug("Started EndpointManager")

	props := actor.FromProducer(newEndpointSupervisor).WithSupervisor(actor.RestartingSupervisorStrategy())
	endpointSupervisor := actor.Spawn(props)

	endpointManager = &endpointManagerValue{
		connections:        &sync.Map{},
		config:             config,
		endpointSupervisor: endpointSupervisor,
	}

	endpointManager.endpointSub = eventstream.
		Subscribe(endpointManager.endpointEvent).
		WithPredicate(func(m interface{}) bool {
			switch m.(type) {
			case *EndpointTerminatedEvent, *EndpointConnectedEvent:
				return true
			}
			return false
		})
}

func stopEndpointManager() {
	eventstream.Unsubscribe(endpointManager.endpointSub)
	endpointManager.endpointSupervisor.GracefulStop()
	endpointManager.endpointSub = nil
	endpointManager.connections = nil
	plog.Debug("Stopped EndpointManager")
}

func (em *endpointManagerValue) endpointEvent(evn interface{}) {
	switch msg := evn.(type) {
	case *EndpointTerminatedEvent:
		em.removeEndpoint(msg)
	case *EndpointConnectedEvent:
		endpoint := em.ensureConnected(msg.Address)
		endpoint.watcher.Tell(msg)
	}
}

func (em *endpointManagerValue) remoteTerminate(msg *remoteTerminate) {
	address := msg.Watchee.Address
	endpoint := em.ensureConnected(address)
	endpoint.watcher.Tell(msg)
}

func (em *endpointManagerValue) remoteWatch(msg *remoteWatch) {
	address := msg.Watchee.Address
	endpoint := em.ensureConnected(address)
	endpoint.watcher.Tell(msg)
}

func (em *endpointManagerValue) remoteUnwatch(msg *remoteUnwatch) {
	address := msg.Watchee.Address
	endpoint := em.ensureConnected(address)
	endpoint.watcher.Tell(msg)
}

func (em *endpointManagerValue) remoteDeliver(msg *remoteDeliver) {
	address := msg.target.Address
	endpoint := em.ensureConnected(address)
	endpoint.writer.Tell(msg)
}

func (em *endpointManagerValue) ensureConnected(address string) *endpoint {
	e, ok := em.connections.Load(address)
	if !ok {
		el := &endpointLazy{}
		e, _ = em.connections.LoadOrStore(address, el)
	}

	el := e.(*endpointLazy)
	el.loadOnce.Do(func() {
		rst, _ := em.endpointSupervisor.RequestFuture(address, 0).Result()
		el.value = rst.(*endpoint)
	})

	return el.value
}

func (em *endpointManagerValue) removeEndpoint(msg *EndpointTerminatedEvent) {
	v, ok := em.connections.Load(msg.Address)
	if ok {
		le := v.(*endpointLazy)
		if atomic.CompareAndSwapUint32(&le.unloaded, 0, 1) {
			em.connections.Delete(msg.Address)
			le.value.watcher.Tell(msg)
			le.value.watcher.Stop()
			le.value.writer.Stop()
		}
	}
}

type endpointSupervisor struct{}

func newEndpointSupervisor() actor.Actor {
	return &endpointSupervisor{}
}

func (state *endpointSupervisor) Receive(ctx actor.Context) {
	if address, ok := ctx.Message().(string); ok {
		e := &endpoint{
			writer:  state.spawnEndpointWriter(address, ctx),
			watcher: state.spawnEndpointWatcher(address, ctx),
		}
		ctx.Respond(e)
	}
}

func (state *endpointSupervisor) HandleFailure(supervisor actor.Supervisor, child *actor.PID, rs *actor.RestartStatistics, reason interface{}, message interface{}) {
	supervisor.RestartChildren(child)
}

func (state *endpointSupervisor) spawnEndpointWriter(address string, ctx actor.Context) *actor.PID {
	props := actor.
		FromProducer(newEndpointWriter(address, endpointManager.config)).
		WithMailbox(newEndpointWriterMailbox(endpointManager.config.endpointWriterBatchSize, endpointManager.config.endpointWriterQueueSize))
	pid := ctx.Spawn(props)
	return pid
}

func (state *endpointSupervisor) spawnEndpointWatcher(address string, ctx actor.Context) *actor.PID {
	props := actor.
		FromProducer(newEndpointWatcher(address))
	pid := ctx.Spawn(props)
	return pid
}
<commit_msg>Revise endpointLazy.<commit_after>package remote

import (
	"sync"
	"sync/atomic"

	"github.com/AsynkronIT/protoactor-go/actor"
	"github.com/AsynkronIT/protoactor-go/eventstream"
)

var endpointManager *endpointManagerValue

type endpointLazy struct {
	valueFunc func() *endpoint
	unloaded  uint32
}

type endpoint struct {
	writer  *actor.PID
	watcher *actor.PID
}

type endpointManagerValue struct {
	connections        *sync.Map
	config             *remoteConfig
	endpointSupervisor *actor.PID
	endpointSub        *eventstream.Subscription
}

func startEndpointManager(config *remoteConfig) {
	plog.Debug("Started EndpointManager")

	props := actor.FromProducer(newEndpointSupervisor).WithSupervisor(actor.RestartingSupervisorStrategy())
	endpointSupervisor := actor.Spawn(props)

	endpointManager = &endpointManagerValue{
		connections:        &sync.Map{},
		config:             config,
		endpointSupervisor: endpointSupervisor,
	}

	endpointManager.endpointSub = eventstream.
		Subscribe(endpointManager.endpointEvent).
		WithPredicate(func(m interface{}) bool {
			switch m.(type) {
			case *EndpointTerminatedEvent, *EndpointConnectedEvent:
				return true
			}
			return false
		})
}

func stopEndpointManager() {
	eventstream.Unsubscribe(endpointManager.endpointSub)
	endpointManager.endpointSupervisor.GracefulStop()
	endpointManager.endpointSub = nil
	endpointManager.connections = nil
	plog.Debug("Stopped EndpointManager")
}

func (em *endpointManagerValue) endpointEvent(evn interface{}) {
	switch msg := evn.(type) {
	case *EndpointTerminatedEvent:
		em.removeEndpoint(msg)
	case *EndpointConnectedEvent:
		endpoint := em.ensureConnected(msg.Address)
		endpoint.watcher.Tell(msg)
	}
}

func (em *endpointManagerValue) remoteTerminate(msg *remoteTerminate) {
	address := msg.Watchee.Address
	endpoint := em.ensureConnected(address)
	endpoint.watcher.Tell(msg)
}

func (em *endpointManagerValue) remoteWatch(msg *remoteWatch) {
	address := msg.Watchee.Address
	endpoint := em.ensureConnected(address)
	endpoint.watcher.Tell(msg)
}

func (em *endpointManagerValue) remoteUnwatch(msg *remoteUnwatch) {
	address := msg.Watchee.Address
	endpoint := em.ensureConnected(address)
	endpoint.watcher.Tell(msg)
}

func (em *endpointManagerValue) remoteDeliver(msg *remoteDeliver) {
	address := msg.target.Address
	endpoint := em.ensureConnected(address)
	endpoint.writer.Tell(msg)
}

func (em *endpointManagerValue) ensureConnected(address string) *endpoint {
	e, ok := em.connections.Load(address)
	if !ok {
		el := &endpointLazy{}
		var once sync.Once
		el.valueFunc = func() *endpoint {
			once.Do(func() {
				rst, _ := em.endpointSupervisor.RequestFuture(address, -1).Result()
				ep := rst.(*endpoint)
				el.valueFunc = func() *endpoint {
					return ep
				}
			})
			return el.valueFunc()
		}
		e, _ = em.connections.LoadOrStore(address, el)
	}

	el := e.(*endpointLazy)
	return el.valueFunc()
}

func (em *endpointManagerValue) removeEndpoint(msg *EndpointTerminatedEvent) {
	v, ok := em.connections.Load(msg.Address)
	if ok {
		le := v.(*endpointLazy)
		if atomic.CompareAndSwapUint32(&le.unloaded, 0, 1) {
			em.connections.Delete(msg.Address)
			ep := le.valueFunc()
			ep.watcher.Tell(msg)
			ep.watcher.Stop()
			ep.writer.Stop()
		}
	}
}

type endpointSupervisor struct{}

func newEndpointSupervisor() actor.Actor {
	return &endpointSupervisor{}
}

func (state *endpointSupervisor) Receive(ctx actor.Context) {
	if address, ok := ctx.Message().(string); ok {
		e := &endpoint{
			writer:  state.spawnEndpointWriter(address, ctx),
			watcher: state.spawnEndpointWatcher(address, ctx),
		}
		ctx.Respond(e)
	}
}

func (state *endpointSupervisor) HandleFailure(supervisor actor.Supervisor, child *actor.PID, rs *actor.RestartStatistics, reason interface{}, message interface{}) {
	supervisor.RestartChildren(child)
}

func (state *endpointSupervisor) spawnEndpointWriter(address string, ctx actor.Context) *actor.PID {
	props := actor.
		FromProducer(newEndpointWriter(address, endpointManager.config)).
		WithMailbox(newEndpointWriterMailbox(endpointManager.config.endpointWriterBatchSize, endpointManager.config.endpointWriterQueueSize))
	pid := ctx.Spawn(props)
	return pid
}

func (state *endpointSupervisor) spawnEndpointWatcher(address string, ctx actor.Context) *actor.PID {
	props := actor.
		FromProducer(newEndpointWatcher(address))
	pid := ctx.Spawn(props)
	return pid
}
<|endoftext|>