<commit_before>package chaos

import (
	"context"
	"fmt"
	"log"
	"math"
	"math/rand"
	"os"
	"os/exec"
	"reflect"
	"testing"
	"time"

	"github.com/davecgh/go-spew/spew"
	"github.com/grafana/metrictank/chaos/out/kafkamdm"
	"github.com/raintank/met/helper"
	"gopkg.in/raintank/schema.v1"
)

// TODO: cleanup when ctrl-C go test (teardown all containers)

const numPartitions = 12

var tracker *Tracker
var metrics []*schema.MetricData

func init() {
	for i := 0; i < numPartitions; i++ {
		name := fmt.Sprintf("some.id.of.a.metric.%d", i)
		m := &schema.MetricData{
			OrgId:    1,
			Name:     name,
			Metric:   name,
			Interval: 1,
			Value:    1,
			Unit:     "s",
			Mtype:    "gauge",
		}
		m.SetId()
		metrics = append(metrics, m)
	}
}

func TestMain(m *testing.M) {
	ctx, cancelFunc := context.WithCancel(context.Background())
	cmd := exec.CommandContext(ctx, path("docker/launch.sh"), "docker-chaos")
	cmd.Env = append(cmd.Env, "MT_CLUSTER_MIN_AVAILABLE_SHARDS=12")

	var err error
	tracker, err = NewTracker(cmd, false, false, "launch-stdout", "launch-stderr")
	if err != nil {
		log.Fatal(err)
	}

	err = cmd.Start()
	if err != nil {
		log.Fatal(err)
	}

	retcode := m.Run()

	fmt.Println("stopping the docker-compose stack...")
	cancelFunc()
	if err := cmd.Wait(); err != nil {
		log.Printf("ERROR: could not cleanly shutdown running docker-compose command: %s", err)
		retcode = 1
	}

	os.Exit(retcode)
}

func TestClusterStartup(t *testing.T) {
	// wait until MT's are up and connected to kafka and cassandra
	matchers := []Matcher{
		{
			Str: "metrictank0_1.*metricIndex initialized.*starting data consumption$",
		},
		{
			Str: "metrictank1_1.*metricIndex initialized.*starting data consumption$",
		},
		{
			Str: "metrictank2_1.*metricIndex initialized.*starting data consumption$",
		},
		{
			Str: "metrictank3_1.*metricIndex initialized.*starting data consumption$",
		},
		{
			Str: "metrictank4_1.*metricIndex initialized.*starting data consumption$",
		},
		{
			Str: "metrictank5_1.*metricIndex initialized.*starting data consumption$",
		},
	}
	ch := tracker.Match(matchers)
	select {
	case <-ch:
		return
	case <-time.After(time.Second * 40):
		t.Fatal("timed out while waiting for all metrictank instances to come up")
	}
}

// 1 metric to each of 12 partitions, each partition replicated twice = expect total workload across cluster of 24Hz
func TestClusterBaseWorkload(t *testing.T) {

	//	tracker.LogStdout(true)
	//	tracker.LogStderr(true)

	go func() {
		t.Log("Starting kafka publishing")
		stats, _ := helper.New(false, "", "standard", "", "")
		out, err := kafkamdm.New("mdm", []string{"localhost:9092"}, "none", stats, "lastNum")
		if err != nil {
			log.Fatal(4, "failed to create kafka-mdm output. %s", err)
		}
		ticker := time.NewTicker(time.Second)

		for tick := range ticker.C {
			unix := tick.Unix()
			for i := range metrics {
				metrics[i].Time = unix
			}
			err := out.Flush(metrics)
			if err != nil {
				t.Fatalf("failed to send data to kafka: %s", err)
			}
		}
	}()

	suc6, resp := retryGraphite("perSecond(metrictank.stats.docker-cluster.*.input.kafka-mdm.metrics_received.counter32)", "-5s", 15, func(resp response) bool {
		exp := []string{
			"perSecond(metrictank.stats.docker-cluster.metrictank0.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank1.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank2.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank3.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank4.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank5.input.kafka-mdm.metrics_received.counter32)",
		}
		if !validateTargets(exp)(resp) {
			return false
		}
		for _, series := range resp.r {
			var sum float64
			if len(series.Datapoints) != 5 {
				return false
			}
			// skip the first point. it always seems to be null for some reason
			for _, p := range series.Datapoints[1:] {
				if math.IsNaN(p.Val) {
					return false
				}
				sum += p.Val
			}
			// avg of all (4) datapoints must be 4 (metrics ingested per second by each instance)
			if sum/4 != 4 {
				return false
			}
		}
		return true
	})
	if !suc6 {
		t.Fatalf("cluster did not reach a state where each MT instance receives 4 points per second. last response was: %s", spew.Sdump(resp))
	}

	suc6, resp = retryMT("sum(some.id.of.a.metric.*)", "-5s", 10, validateCorrect(12))
	if !suc6 {
		t.Fatalf("could not query correct result set. sum of 12 series, each valued 1, should result in 12.  last response was: %s", spew.Sdump(resp))
	}
}

// TestIsolateOneInstance tests what happens during the isolation of one instance, when min-available-shards is 12
// this should happen:
// at all times, all queries to all of the remaining nodes should be successful
// since they have at least 1 instance running for each shard.
// the isolated shard should either return correct replies, or errors (in two cases: when it marks any shards as down,
// but also before it does, but fails to get data via clustered requests from peers)
//. TODO: in production do we stop querying isolated peers?

func TestIsolateOneInstance(t *testing.T) {
	t.Log("Starting TestIsolateOneInstance)")
	tracker.LogStdout(true)
	tracker.LogStderr(true)
	pre := time.Now()
	rand.Seed(pre.Unix())

	mt4ResultsChan := make(chan checkResults, 1)
	otherResultsChan := make(chan checkResults, 1)

	go func() {
		mt4ResultsChan <- checkMT([]int{6064}, "some.id.of.a.*", "-10s", time.Minute, 6000, validateCorrect(12), validateCode(503))
	}()
	go func() {
		otherResultsChan <- checkMT([]int{6060, 6061, 6062, 6063, 6065}, "some.id.of.a.*", "-10s", time.Minute, 6000, validateCorrect(12))
	}()

	// now go ahead and isolate for 30s
	isolate("metrictank4", "30s", "metrictank0", "metrictank1", "metrictank2", "metrictank3", "metrictank5")

	// collect results of the minute long experiment
	mt4Results := <-mt4ResultsChan
	otherResults := <-otherResultsChan

	// validate results of isolated node
	if mt4Results.valid[0]+mt4Results.valid[1] != 6000 {
		t.Fatalf("expected mt4 to return either correct or erroring responses. got %+v", mt4Results)
	}
	if mt4Results.valid[1] < 30*6000/100 {
		// the instance is completely down for 30s of the 60s experiment run, but we allow some slack
		t.Fatalf("expected at least 30%% of all mt4 results to succeed. got %+v", mt4Results)
	}

	// validate results of other cluster nodes
	exp := checkResults{
		valid:      []int{6000},
		empty:      0,
		timeout:    0,
		other:      0,
		firstOther: response{},
	}
	if !reflect.DeepEqual(exp, otherResults) {
		t.Fatalf("expected only correct results for all cluster nodes. got %+v", otherResults)
	}
}

func TestHang(t *testing.T) {
	t.Log("whatever happens, keep hanging for now, so that we can query grafana dashboards still")
	var ch chan struct{}
	<-ch
}

// maybe useful in the future, test also clean exit and rejoin like so:
//stop("metrictank4")
//time.AfterFunc(30*time.Second, func() {
//	start("metrictank4")
//})
<commit_msg>fix query bug<commit_after>package chaos

import (
	"context"
	"fmt"
	"log"
	"math"
	"math/rand"
	"os"
	"os/exec"
	"reflect"
	"testing"
	"time"

	"github.com/davecgh/go-spew/spew"
	"github.com/grafana/metrictank/chaos/out/kafkamdm"
	"github.com/raintank/met/helper"
	"gopkg.in/raintank/schema.v1"
)

// TODO: cleanup when ctrl-C go test (teardown all containers)

const numPartitions = 12

var tracker *Tracker
var metrics []*schema.MetricData

func init() {
	for i := 0; i < numPartitions; i++ {
		name := fmt.Sprintf("some.id.of.a.metric.%d", i)
		m := &schema.MetricData{
			OrgId:    1,
			Name:     name,
			Metric:   name,
			Interval: 1,
			Value:    1,
			Unit:     "s",
			Mtype:    "gauge",
		}
		m.SetId()
		metrics = append(metrics, m)
	}
}

func TestMain(m *testing.M) {
	ctx, cancelFunc := context.WithCancel(context.Background())
	cmd := exec.CommandContext(ctx, path("docker/launch.sh"), "docker-chaos")
	cmd.Env = append(cmd.Env, "MT_CLUSTER_MIN_AVAILABLE_SHARDS=12")

	var err error
	tracker, err = NewTracker(cmd, false, false, "launch-stdout", "launch-stderr")
	if err != nil {
		log.Fatal(err)
	}

	err = cmd.Start()
	if err != nil {
		log.Fatal(err)
	}

	retcode := m.Run()

	fmt.Println("stopping the docker-compose stack...")
	cancelFunc()
	if err := cmd.Wait(); err != nil {
		log.Printf("ERROR: could not cleanly shutdown running docker-compose command: %s", err)
		retcode = 1
	}

	os.Exit(retcode)
}

func TestClusterStartup(t *testing.T) {
	// wait until MT's are up and connected to kafka and cassandra
	matchers := []Matcher{
		{
			Str: "metrictank0_1.*metricIndex initialized.*starting data consumption$",
		},
		{
			Str: "metrictank1_1.*metricIndex initialized.*starting data consumption$",
		},
		{
			Str: "metrictank2_1.*metricIndex initialized.*starting data consumption$",
		},
		{
			Str: "metrictank3_1.*metricIndex initialized.*starting data consumption$",
		},
		{
			Str: "metrictank4_1.*metricIndex initialized.*starting data consumption$",
		},
		{
			Str: "metrictank5_1.*metricIndex initialized.*starting data consumption$",
		},
	}
	ch := tracker.Match(matchers)
	select {
	case <-ch:
		return
	case <-time.After(time.Second * 40):
		t.Fatal("timed out while waiting for all metrictank instances to come up")
	}
}

// 1 metric to each of 12 partitions, each partition replicated twice = expect total workload across cluster of 24Hz
func TestClusterBaseWorkload(t *testing.T) {

	//	tracker.LogStdout(true)
	//	tracker.LogStderr(true)

	go func() {
		t.Log("Starting kafka publishing")
		stats, _ := helper.New(false, "", "standard", "", "")
		out, err := kafkamdm.New("mdm", []string{"localhost:9092"}, "none", stats, "lastNum")
		if err != nil {
			log.Fatal(4, "failed to create kafka-mdm output. %s", err)
		}
		ticker := time.NewTicker(time.Second)

		for tick := range ticker.C {
			unix := tick.Unix()
			for i := range metrics {
				metrics[i].Time = unix
			}
			err := out.Flush(metrics)
			if err != nil {
				t.Fatalf("failed to send data to kafka: %s", err)
			}
		}
	}()

	suc6, resp := retryGraphite("perSecond(metrictank.stats.docker-cluster.*.input.kafka-mdm.metrics_received.counter32)", "-5s", 15, func(resp response) bool {
		exp := []string{
			"perSecond(metrictank.stats.docker-cluster.metrictank0.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank1.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank2.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank3.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank4.input.kafka-mdm.metrics_received.counter32)",
			"perSecond(metrictank.stats.docker-cluster.metrictank5.input.kafka-mdm.metrics_received.counter32)",
		}
		if !validateTargets(exp)(resp) {
			return false
		}
		for _, series := range resp.r {
			var sum float64
			if len(series.Datapoints) != 5 {
				return false
			}
			// skip the first point. it always seems to be null for some reason
			for _, p := range series.Datapoints[1:] {
				if math.IsNaN(p.Val) {
					return false
				}
				sum += p.Val
			}
			// avg of all (4) datapoints must be 4 (metrics ingested per second by each instance)
			if sum/4 != 4 {
				return false
			}
		}
		return true
	})
	if !suc6 {
		t.Fatalf("cluster did not reach a state where each MT instance receives 4 points per second. last response was: %s", spew.Sdump(resp))
	}

	suc6, resp = retryMT("sum(some.id.of.a.metric.*)", "-5s", 10, validateCorrect(12))
	if !suc6 {
		t.Fatalf("could not query correct result set. sum of 12 series, each valued 1, should result in 12.  last response was: %s", spew.Sdump(resp))
	}
}

// TestIsolateOneInstance tests what happens during the isolation of one instance, when min-available-shards is 12
// this should happen:
// at all times, all queries to all of the remaining nodes should be successful
// since they have at least 1 instance running for each shard.
// the isolated shard should either return correct replies, or errors (in two cases: when it marks any shards as down,
// but also before it does, but fails to get data via clustered requests from peers)
//. TODO: in production do we stop querying isolated peers?

func TestIsolateOneInstance(t *testing.T) {
	t.Log("Starting TestIsolateOneInstance)")
	tracker.LogStdout(true)
	tracker.LogStderr(true)
	pre := time.Now()
	rand.Seed(pre.Unix())

	mt4ResultsChan := make(chan checkResults, 1)
	otherResultsChan := make(chan checkResults, 1)

	go func() {
		mt4ResultsChan <- checkMT([]int{6064}, "some.id.of.a.metric.*", "-10s", time.Minute, 6000, validateCorrect(12), validateCode(503))
	}()
	go func() {
		otherResultsChan <- checkMT([]int{6060, 6061, 6062, 6063, 6065}, "some.id.of.a.metric.*", "-10s", time.Minute, 6000, validateCorrect(12))
	}()

	// now go ahead and isolate for 30s
	isolate("metrictank4", "30s", "metrictank0", "metrictank1", "metrictank2", "metrictank3", "metrictank5")

	// collect results of the minute long experiment
	mt4Results := <-mt4ResultsChan
	otherResults := <-otherResultsChan

	// validate results of isolated node
	if mt4Results.valid[0]+mt4Results.valid[1] != 6000 {
		t.Fatalf("expected mt4 to return either correct or erroring responses. got %+v", mt4Results)
	}
	if mt4Results.valid[1] < 30*6000/100 {
		// the instance is completely down for 30s of the 60s experiment run, but we allow some slack
		t.Fatalf("expected at least 30%% of all mt4 results to succeed. got %+v", mt4Results)
	}

	// validate results of other cluster nodes
	exp := checkResults{
		valid:      []int{6000},
		empty:      0,
		timeout:    0,
		other:      0,
		firstOther: response{},
	}
	if !reflect.DeepEqual(exp, otherResults) {
		t.Fatalf("expected only correct results for all cluster nodes. got %+v", otherResults)
	}
}

func TestHang(t *testing.T) {
	t.Log("whatever happens, keep hanging for now, so that we can query grafana dashboards still")
	var ch chan struct{}
	<-ch
}

// maybe useful in the future, test also clean exit and rejoin like so:
//stop("metrictank4")
//time.AfterFunc(30*time.Second, func() {
//	start("metrictank4")
//})
<|endoftext|>


<commit_before>package vxlan

import (
	//"fmt"
	//"strings"
	//"time"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/go-plugins-helpers/network"
	//"github.com/samalba/dockerclient"
	"github.com/vishvananda/netlink"
)

type Driver struct {
	network.Driver
	networks map[string]*NetworkState
}

// NetworkState is filled in at network creation time
// it contains state that we wish to keep for each network
type NetworkState struct {
	Bridge *netlink.Bridge
	VXLan  *netlink.Vxlan
}

func NewDriver() (*Driver, error) {
	d := &Driver{
		networks: make(map[string]*NetworkState),
	}
	return d, nil
}

func (d *Driver) CreateNetwork(r *network.CreateNetworkRequest) error {
	log.Debugf("Create network request: %+v", r)

	name := r.NetworkID

	//if r.Options == nil {
	//	return "", fmt.Errorf("No options provided")
	//}

	vxlanName := "vx_" + name
	vxlanID := 42
	//if r.Options["vxlanID"] != nil {
	//	vxlanID = r.Options["vxlanID"]
	//}

	bridgeName := "br_" + name

	bridge := &netlink.Bridge{
		LinkAttrs: netlink.LinkAttrs{Name: bridgeName},
	}
	err := netlink.LinkAdd(bridge)
	if err != nil {
		return err
	}

	vxlan := &netlink.Vxlan{
		LinkAttrs: netlink.LinkAttrs{Name: vxlanName},
		VxlanId:   vxlanID,
	}
	err = netlink.LinkAdd(vxlan)
	if err != nil {
		return err
	}

	ns := &NetworkState{
		VXLan:  vxlan,
		Bridge: bridge,
	}
	d.networks[name] = ns

	err = netlink.LinkSetMaster(vxlan, bridge)
	if err != nil {
		return err
	}

	return nil
}

func (d *Driver) DeleteNetwork(r *network.DeleteNetworkRequest) error {
	name := r.NetworkID

	err := netlink.LinkDel(d.networks[name].VXLan)
	if err != nil {
		return err
	}
	err = netlink.LinkDel(d.networks[name].Bridge)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>parse options<commit_after>package vxlan

import (
	//"fmt"
	//"strings"
	//"time"
	"net"
	"strconv"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/go-plugins-helpers/network"
	//"github.com/samalba/dockerclient"
	"github.com/davecgh/go-spew/spew"
	"github.com/vishvananda/netlink"
)

type Driver struct {
	network.Driver
	networks map[string]*NetworkState
}

// NetworkState is filled in at network creation time
// it contains state that we wish to keep for each network
type NetworkState struct {
	Bridge *netlink.Bridge
	VXLan  *netlink.Vxlan
}

func NewDriver() (*Driver, error) {
	d := &Driver{
		networks: make(map[string]*NetworkState),
	}
	return d, nil
}

func (d *Driver) CreateNetwork(r *network.CreateNetworkRequest) error {
	log.Debugf("Create network request: %+v", r)
	spew.Dump(r)

	name := r.NetworkID[0:12]

	vxlanName := "vx_" + name
	bridgeName := "br_" + name

	if r.Options != nil {
		if r.Options["vxlanName"] != nil {
			vxlanName = r.Options["vxlanName"]
		}
		if r.Options["bridgeName"] != nil {
			bridgeName = r.Options["bridgeName"]
		}
	}

	bridge := &netlink.Bridge{
		LinkAttrs: netlink.LinkAttrs{Name: bridgeName},
	}
	vxlan := &netlink.Vxlan{
		LinkAttrs: netlink.LinkAttrs{Name: vxlanName},
	}

	if r.Options != nil {
		if r.Options["VxlanID"] != nil {
			vxlan.VxlanId = strconf.ParseUInt(r.Options["VxlanID"])
		}
		if r.Options["VtepDev"] != nil {
			vtepDev = netlink.LinkByName(r.Options["VtepDev"]
			vxlan.VtepDevIndex = vtepDev.Attrs().Index
		}
		if r.Options["SrcAddr"] != nil {
			vxlan.SrcAddr = net.ParseIP(r.Options["SrcAddr"])
		}
		if r.Options["Group"] != nil {
			vxlan.Group = net.ParseIP(r.Options["Group"])
		}
		if r.Options["TTL"] != nil {
			vxlan.TTL = strconf.ParseUInt(r.Options["TTL"])
		}
		if r.Options["TOS"] != nil {
			vxlan.TOS = strconf.ParseUInt(r.Options["TOS"])
		}
		if r.Options["Learning"] != nil {
			vxlan.Learning = strconf.ParseBool(r.Options["Learning"])
		}
		if r.Options["Proxy"] != nil {
			vxlan.Proxy = strconf.ParseBool(r.Options["Proxy"])
		}
		if r.Options["RSC"] != nil {
			vxlan.RSC = strconf.ParseBool(r.Options["RSC"])
		}
		if r.Options["L2miss"] != nil {
			vxlan.L2miss = strconf.ParseBool(r.Options["L2miss"])
		}
		if r.Options["L3miss"] != nil {
			vxlan.L3miss = strconf.ParseBool(r.Options["L2miss"])
		}
		if r.Options["NoAge"] != nil {
			vxlan.NoAge = strconf.ParseBool(r.Options["NoAge"])
		}
		if r.Options["BGP"] != nil {
			vxlan.BGP = strconf.ParseBool(r.Options["BGP"])
		}
		if r.Options["Age"] != nil {
			vxlan.Age = strconf.ParseUInt(r.Options["Age"])
		}
		if r.Options["Limit"] != nil {
			vxlan.Limit = strconf.ParseUInt(r.Options["Limit"])
		}
		if r.Options["Port"] != nil {
			vxlan.Port = strconf.ParseUInt(r.Options["Port"])
		}
		if r.Options["PortLow"] != nil {
			vxlan.PortLow = strconf.ParseUInt(r.Options["PortLow"])
		}
		if r.Options["PortHigh"] != nil {
			vxlan.PortHigh = strconf.ParseUInt(r.Options["PortHigh"])
		}
	}

	err := netlink.LinkAdd(bridge)
	if err != nil {
		return err
	}

	err = netlink.LinkAdd(vxlan)
	if err != nil {
		return err
	}

	ns := &NetworkState{
		VXLan:  vxlan,
		Bridge: bridge,
	}
	d.networks[name] = ns

	err = netlink.LinkSetMaster(vxlan, bridge)
	if err != nil {
		return err
	}

	return nil
}

func (d *Driver) DeleteNetwork(r *network.DeleteNetworkRequest) error {
	name := r.NetworkID[0:12]

	err := netlink.LinkDel(d.networks[name].VXLan)
	if err != nil {
		return err
	}
	err = netlink.LinkDel(d.networks[name].Bridge)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package test

import (
	"github.com/gruntwork-io/terratest"
	"testing"
	"os"
	terralog "github.com/gruntwork-io/terratest/log"
	"log"
	"github.com/gruntwork-io/terratest/util"
	"time"
	"fmt"
	"path/filepath"
	"net/http"
	"io/ioutil"
	"encoding/json"
)

const REPO_ROOT = "../"

const VAR_AWS_REGION = "aws_region"
const VAR_AMI_ID = "ami_id"

const CLUSTER_COLOCATED_EXAMPLE_PATH = "examples/nomad-consul-colocated-cluster"
const CLUSTER_COLOCATED_EXAMPLE_VAR_CLUSTER_NAME = "cluster_name"
const CLUSTER_COLOCATED_EXAMPLE_VAR_NUM_SERVERS = "num_servers"
const CLUSTER_COLOCATED_EXAMPLE_VAR_NUM_CLIENTS = "num_clients"
const CLUSTER_COLOCATED_EXAMPLE_OUTPUT_SERVER_ASG_NAME = "asg_name_servers"

const CLUSTER_SEPARATE_EXAMPLE_PATH = "examples/nomad-consul-separate-cluster"
const CLUSTER_SEPARATE_EXAMPLE_VAR_NOMAD_CLUSTER_NAME = "nomad_cluster_name"
const CLUSTER_SEPARATE_EXAMPLE_VAR_CONSUL_CLUSTER_NAME = "consul_cluster_name"
const CLUSTER_SEPARATE_EXAMPLE_VAR_NUM_NOMAD_SERVERS = "num_nomad_servers"
const CLUSTER_SEPARATE_EXAMPLE_VAR_NUM_CONSUL_SERVERS = "num_consul_servers"
const CLUSTER_SEPARATE_EXAMPLE_VAR_NUM_NOMAD_CLIENTS = "num_nomad_clients"
const CLUSTER_SEPARATE_EXAMPLE_OUTPUT_NOMAD_SERVER_ASG_NAME = "asg_name_nomad_servers"

const DEFAULT_NUM_SERVERS = 3
const DEFAULT_NUM_CLIENTS = 6

const AMI_EXAMPLE_PATH = "../examples/nomad-consul-ami/nomad-consul.json"

// Test the Nomad/Consul colocated cluster example by:
//
// 1. Copying the code in this repo to a temp folder so tests on the Terraform code can run in parallel without the
//    state files overwriting each other.
// 2. Building the AMI in the nomad-consul-ami example with the given build name
// 3. Deploying that AMI using the example Terraform code
// 4. Checking that the Nomad cluster comes up within a reasonable time period and can respond to requests
func runNomadClusterColocatedTest(t *testing.T, testName string, packerBuildName string) {
	rootTempPath := copyRepoToTempFolder(t, REPO_ROOT)
	defer os.RemoveAll(rootTempPath)

	resourceCollection := createBaseRandomResourceCollection(t)
	terratestOptions := createBaseTerratestOptions(t, testName, filepath.Join(rootTempPath, CLUSTER_COLOCATED_EXAMPLE_PATH), resourceCollection)
	defer terratest.Destroy(terratestOptions, resourceCollection)

	logger := terralog.NewLogger(testName)
	amiId := buildAmi(t, AMI_EXAMPLE_PATH, packerBuildName, resourceCollection, logger)

	terratestOptions.Vars = map[string]interface{} {
		VAR_AWS_REGION: resourceCollection.AwsRegion,
		CLUSTER_COLOCATED_EXAMPLE_VAR_CLUSTER_NAME: testName + resourceCollection.UniqueId,
		CLUSTER_COLOCATED_EXAMPLE_VAR_NUM_SERVERS: DEFAULT_NUM_SERVERS,
		CLUSTER_COLOCATED_EXAMPLE_VAR_NUM_CLIENTS: DEFAULT_NUM_CLIENTS,
		VAR_AMI_ID: amiId,
	}

	deploy(t, terratestOptions)
	checkNomadClusterIsWorking(t, CLUSTER_COLOCATED_EXAMPLE_OUTPUT_SERVER_ASG_NAME, terratestOptions, resourceCollection, logger)
}

// Test the Nomad/Consul separate clusters example by:
//
// 1. Copying the code in this repo to a temp folder so tests on the Terraform code can run in parallel without the
//    state files overwriting each other.
// 2. Building the AMI in the nomad-consul-ami example with the given build name
// 3. Deploying that AMI using the example Terraform code
// 4. Checking that the Nomad cluster comes up within a reasonable time period and can respond to requests
func runNomadClusterSeparateTest(t *testing.T, testName string, packerBuildName string) {
	rootTempPath := copyRepoToTempFolder(t, REPO_ROOT)
	defer os.RemoveAll(rootTempPath)

	resourceCollection := createBaseRandomResourceCollection(t)
	terratestOptions := createBaseTerratestOptions(t, testName, filepath.Join(rootTempPath, CLUSTER_SEPARATE_EXAMPLE_PATH), resourceCollection)
	defer terratest.Destroy(terratestOptions, resourceCollection)

	logger := terralog.NewLogger(testName)
	amiId := buildAmi(t, AMI_EXAMPLE_PATH, packerBuildName, resourceCollection, logger)

	terratestOptions.Vars = map[string]interface{} {
		VAR_AWS_REGION: resourceCollection.AwsRegion,
		CLUSTER_SEPARATE_EXAMPLE_VAR_NOMAD_CLUSTER_NAME: "nomad-" + testName + resourceCollection.UniqueId,
		CLUSTER_SEPARATE_EXAMPLE_VAR_CONSUL_CLUSTER_NAME: "consul-" + testName + resourceCollection.UniqueId,
		CLUSTER_SEPARATE_EXAMPLE_VAR_NUM_NOMAD_SERVERS: DEFAULT_NUM_SERVERS,
		CLUSTER_SEPARATE_EXAMPLE_VAR_NUM_CONSUL_SERVERS: DEFAULT_NUM_SERVERS,
		CLUSTER_SEPARATE_EXAMPLE_VAR_NUM_NOMAD_CLIENTS: DEFAULT_NUM_CLIENTS,
		VAR_AMI_ID: amiId,
	}

	deploy(t, terratestOptions)
	checkNomadClusterIsWorking(t, CLUSTER_SEPARATE_EXAMPLE_OUTPUT_NOMAD_SERVER_ASG_NAME, terratestOptions, resourceCollection, logger)
}

// Check that the Nomad cluster comes up within a reasonable time period and can respond to requests
func checkNomadClusterIsWorking(t *testing.T, asgNameOutputVar string, terratestOptions *terratest.TerratestOptions, resourceCollection *terratest.RandomResourceCollection, logger *log.Logger) {
	asgName, err := terratest.Output(terratestOptions, asgNameOutputVar)
	if err != nil {
		t.Fatalf("Could not read output %s due to error: %v", asgNameOutputVar, err)
	}

	nodeIpAddress := getIpAddressOfAsgInstance(t, asgName, resourceCollection.AwsRegion)
	testNomadCluster(t, nodeIpAddress, logger)
}

// Use a Nomad client to connect to the given node and use it to verify that:
//
// 1. The Nomad cluster has deployed
// 2. The cluster has the expected number of server nodes
// 2. The cluster has the expected number of client nodes
func testNomadCluster(t *testing.T, nodeIpAddress string, logger *log.Logger) {
	maxRetries := 60
	sleepBetweenRetries := 10 * time.Second

	response, err := util.DoWithRetry("Check Nomad members", maxRetries, sleepBetweenRetries, logger, func() (string, error) {
		clients, err := callNomadApi(nodeIpAddress, "v1/nodes", logger)
		if err != nil {
			return "", err
		}

		if len(clients) != DEFAULT_NUM_CLIENTS {
			return "", fmt.Errorf("Expected the cluster to have %d clients, but found %d", DEFAULT_NUM_CLIENTS, len(clients))
		}

		servers, err := callNomadApi(nodeIpAddress, "v1/status/peers", logger)
		if err != nil {
			return "", err
		}

		if len(servers) != DEFAULT_NUM_SERVERS {
			return "", fmt.Errorf("Expected the cluster to have %d servers, but found %d", DEFAULT_NUM_SERVERS, len(servers))
		}

		return fmt.Sprintf("Got back expected number of clients (%d) and servers (%d)", len(clients), len(servers)), nil
	})

	if err != nil {
		t.Fatalf("Could not verify Nomad node at %s was working: %v", nodeIpAddress, err)
	}

	logger.Printf("Nomad cluster is properly deployed: %s", response)
}

// A quick, hacky way to call the Nomad HTTP API: https://www.nomadproject.io/docs/http/index.html
func callNomadApi(nodeIpAddress string, path string, logger *log.Logger) ([]interface{}, error) {
	resp, err := http.Get(fmt.Sprintf("http://%s:4646/%s", nodeIpAddress, path))
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	logger.Printf("Response from Nomad: %s", string(body))

	result := []interface{}{}
	if err := json.Unmarshal(body, &result); err != nil {
		return nil, err
	}

	return result, nil
}
<commit_msg>Add more logging to tests<commit_after>package test

import (
	"github.com/gruntwork-io/terratest"
	"testing"
	"os"
	terralog "github.com/gruntwork-io/terratest/log"
	"log"
	"github.com/gruntwork-io/terratest/util"
	"time"
	"fmt"
	"path/filepath"
	"net/http"
	"io/ioutil"
	"encoding/json"
)

const REPO_ROOT = "../"

const VAR_AWS_REGION = "aws_region"
const VAR_AMI_ID = "ami_id"

const CLUSTER_COLOCATED_EXAMPLE_PATH = "examples/nomad-consul-colocated-cluster"
const CLUSTER_COLOCATED_EXAMPLE_VAR_CLUSTER_NAME = "cluster_name"
const CLUSTER_COLOCATED_EXAMPLE_VAR_NUM_SERVERS = "num_servers"
const CLUSTER_COLOCATED_EXAMPLE_VAR_NUM_CLIENTS = "num_clients"
const CLUSTER_COLOCATED_EXAMPLE_OUTPUT_SERVER_ASG_NAME = "asg_name_servers"

const CLUSTER_SEPARATE_EXAMPLE_PATH = "examples/nomad-consul-separate-cluster"
const CLUSTER_SEPARATE_EXAMPLE_VAR_NOMAD_CLUSTER_NAME = "nomad_cluster_name"
const CLUSTER_SEPARATE_EXAMPLE_VAR_CONSUL_CLUSTER_NAME = "consul_cluster_name"
const CLUSTER_SEPARATE_EXAMPLE_VAR_NUM_NOMAD_SERVERS = "num_nomad_servers"
const CLUSTER_SEPARATE_EXAMPLE_VAR_NUM_CONSUL_SERVERS = "num_consul_servers"
const CLUSTER_SEPARATE_EXAMPLE_VAR_NUM_NOMAD_CLIENTS = "num_nomad_clients"
const CLUSTER_SEPARATE_EXAMPLE_OUTPUT_NOMAD_SERVER_ASG_NAME = "asg_name_nomad_servers"

const DEFAULT_NUM_SERVERS = 3
const DEFAULT_NUM_CLIENTS = 6

const AMI_EXAMPLE_PATH = "../examples/nomad-consul-ami/nomad-consul.json"

// Test the Nomad/Consul colocated cluster example by:
//
// 1. Copying the code in this repo to a temp folder so tests on the Terraform code can run in parallel without the
//    state files overwriting each other.
// 2. Building the AMI in the nomad-consul-ami example with the given build name
// 3. Deploying that AMI using the example Terraform code
// 4. Checking that the Nomad cluster comes up within a reasonable time period and can respond to requests
func runNomadClusterColocatedTest(t *testing.T, testName string, packerBuildName string) {
	rootTempPath := copyRepoToTempFolder(t, REPO_ROOT)
	defer os.RemoveAll(rootTempPath)

	resourceCollection := createBaseRandomResourceCollection(t)
	terratestOptions := createBaseTerratestOptions(t, testName, filepath.Join(rootTempPath, CLUSTER_COLOCATED_EXAMPLE_PATH), resourceCollection)
	defer terratest.Destroy(terratestOptions, resourceCollection)

	logger := terralog.NewLogger(testName)
	amiId := buildAmi(t, AMI_EXAMPLE_PATH, packerBuildName, resourceCollection, logger)

	terratestOptions.Vars = map[string]interface{} {
		VAR_AWS_REGION: resourceCollection.AwsRegion,
		CLUSTER_COLOCATED_EXAMPLE_VAR_CLUSTER_NAME: testName + resourceCollection.UniqueId,
		CLUSTER_COLOCATED_EXAMPLE_VAR_NUM_SERVERS: DEFAULT_NUM_SERVERS,
		CLUSTER_COLOCATED_EXAMPLE_VAR_NUM_CLIENTS: DEFAULT_NUM_CLIENTS,
		VAR_AMI_ID: amiId,
	}

	deploy(t, terratestOptions)
	checkNomadClusterIsWorking(t, CLUSTER_COLOCATED_EXAMPLE_OUTPUT_SERVER_ASG_NAME, terratestOptions, resourceCollection, logger)
}

// Test the Nomad/Consul separate clusters example by:
//
// 1. Copying the code in this repo to a temp folder so tests on the Terraform code can run in parallel without the
//    state files overwriting each other.
// 2. Building the AMI in the nomad-consul-ami example with the given build name
// 3. Deploying that AMI using the example Terraform code
// 4. Checking that the Nomad cluster comes up within a reasonable time period and can respond to requests
func runNomadClusterSeparateTest(t *testing.T, testName string, packerBuildName string) {
	rootTempPath := copyRepoToTempFolder(t, REPO_ROOT)
	defer os.RemoveAll(rootTempPath)

	resourceCollection := createBaseRandomResourceCollection(t)
	terratestOptions := createBaseTerratestOptions(t, testName, filepath.Join(rootTempPath, CLUSTER_SEPARATE_EXAMPLE_PATH), resourceCollection)
	defer terratest.Destroy(terratestOptions, resourceCollection)

	logger := terralog.NewLogger(testName)
	amiId := buildAmi(t, AMI_EXAMPLE_PATH, packerBuildName, resourceCollection, logger)

	terratestOptions.Vars = map[string]interface{} {
		VAR_AWS_REGION: resourceCollection.AwsRegion,
		CLUSTER_SEPARATE_EXAMPLE_VAR_NOMAD_CLUSTER_NAME: "nomad-" + testName + resourceCollection.UniqueId,
		CLUSTER_SEPARATE_EXAMPLE_VAR_CONSUL_CLUSTER_NAME: "consul-" + testName + resourceCollection.UniqueId,
		CLUSTER_SEPARATE_EXAMPLE_VAR_NUM_NOMAD_SERVERS: DEFAULT_NUM_SERVERS,
		CLUSTER_SEPARATE_EXAMPLE_VAR_NUM_CONSUL_SERVERS: DEFAULT_NUM_SERVERS,
		CLUSTER_SEPARATE_EXAMPLE_VAR_NUM_NOMAD_CLIENTS: DEFAULT_NUM_CLIENTS,
		VAR_AMI_ID: amiId,
	}

	deploy(t, terratestOptions)
	checkNomadClusterIsWorking(t, CLUSTER_SEPARATE_EXAMPLE_OUTPUT_NOMAD_SERVER_ASG_NAME, terratestOptions, resourceCollection, logger)
}

// Check that the Nomad cluster comes up within a reasonable time period and can respond to requests
func checkNomadClusterIsWorking(t *testing.T, asgNameOutputVar string, terratestOptions *terratest.TerratestOptions, resourceCollection *terratest.RandomResourceCollection, logger *log.Logger) {
	asgName, err := terratest.Output(terratestOptions, asgNameOutputVar)
	if err != nil {
		t.Fatalf("Could not read output %s due to error: %v", asgNameOutputVar, err)
	}

	nodeIpAddress := getIpAddressOfAsgInstance(t, asgName, resourceCollection.AwsRegion)
	testNomadCluster(t, nodeIpAddress, logger)
}

// Use a Nomad client to connect to the given node and use it to verify that:
//
// 1. The Nomad cluster has deployed
// 2. The cluster has the expected number of server nodes
// 2. The cluster has the expected number of client nodes
func testNomadCluster(t *testing.T, nodeIpAddress string, logger *log.Logger) {
	maxRetries := 60
	sleepBetweenRetries := 10 * time.Second

	response, err := util.DoWithRetry("Check Nomad members", maxRetries, sleepBetweenRetries, logger, func() (string, error) {
		clients, err := callNomadApi(nodeIpAddress, "v1/nodes", logger)
		if err != nil {
			return "", err
		}

		if len(clients) != DEFAULT_NUM_CLIENTS {
			return "", fmt.Errorf("Expected the cluster to have %d clients, but found %d", DEFAULT_NUM_CLIENTS, len(clients))
		}

		servers, err := callNomadApi(nodeIpAddress, "v1/status/peers", logger)
		if err != nil {
			return "", err
		}

		if len(servers) != DEFAULT_NUM_SERVERS {
			return "", fmt.Errorf("Expected the cluster to have %d servers, but found %d", DEFAULT_NUM_SERVERS, len(servers))
		}

		return fmt.Sprintf("Got back expected number of clients (%d) and servers (%d)", len(clients), len(servers)), nil
	})

	if err != nil {
		t.Fatalf("Could not verify Nomad node at %s was working: %v", nodeIpAddress, err)
	}

	logger.Printf("Nomad cluster is properly deployed: %s", response)
}

// A quick, hacky way to call the Nomad HTTP API: https://www.nomadproject.io/docs/http/index.html
func callNomadApi(nodeIpAddress string, path string, logger *log.Logger) ([]interface{}, error) {
	url := fmt.Sprintf("http://%s:4646/%s", nodeIpAddress, path)
	logger.Printf("Making an HTTP GET to URL %s", url)

	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	logger.Printf("Response from Nomad for URL %s: %s", url, string(body))

	result := []interface{}{}
	if err := json.Unmarshal(body, &result); err != nil {
		return nil, err
	}

	return result, nil
}
<|endoftext|>


<commit_before>package web

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"golang.org/x/net/context"
)

type IncrHandler struct {
	before int
	after  int
}

func (i *IncrHandler) ServeHTTPCN(ctx context.Context, rw http.ResponseWriter, r *http.Request, next ContextHandler) {
	i.before++
	next.ServeHTTPC(ctx, rw, r)
	i.after++
}

func (i *IncrHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.Handler) {
	i.before++
	next.ServeHTTP(rw, r)
	i.after++
}

func (i *IncrHandler) makeHTTP(next http.Handler) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		i.ServeHTTP(rw, r, next)
	})
}

func TestHandler(t *testing.T) {
	i := IncrHandler{}
	expectAfter := 0
	expectBefore := 2
	destination := HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
		assert.Equal(t, "value", ctx.Value("key"))
		assert.Equal(t, expectAfter, i.after)
		assert.Equal(t, expectBefore, i.before)
	})
	ctx := context.Background()
	h := NewHandler(ctx, destination)
	v1 := VarAdder{
		Key:   "key",
		Value: "value",
	}

	h.Add(ConstructorFunc(v1.Generate), HTTPConstructor(i.makeHTTP), NextHTTP(i.ServeHTTP))
	rw := httptest.NewRecorder()
	req, _ := http.NewRequest("POST", "/", nil)
	h.ServeHTTP(rw, req)
	assert.Equal(t, 2, i.after)

	expectAfter = 2
	expectBefore = 4
	h.ServeHTTPC(ctx, rw, req)
	assert.Equal(t, 4, i.after)

	bodyTest := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		rw.Write([]byte("test"))
	})

	ToHTTP(ctx, FromHTTP(bodyTest)).ServeHTTP(rw, req)
	assert.Equal(t, "test", rw.Body.String())
}

func addTowrite(ctx context.Context, rw http.ResponseWriter, r *http.Request, next ContextHandler) {
	next.ServeHTTPC(context.WithValue(ctx, "towrite", []byte(r.Header.Get("towrite"))), rw, r)
}

func TestMany(t *testing.T) {
	incrHandler := IncrHandler{}

	destination := HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
		rw.Write(ctx.Value("towrite").([]byte))
	})

	ctx := context.Background()
	h := NewHandler(ctx, HandlerFunc(destination)).Add(NextConstructor(addTowrite), HTTPConstructor(incrHandler.makeHTTP))

	wg := sync.WaitGroup{}
	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for j := 0; j < 100; j++ {
				if j%11 == 0 {
					time.Sleep(time.Nanosecond)
				}
				rw := httptest.NewRecorder()
				req, _ := http.NewRequest("POST", "/", nil)
				req.Header.Add("towrite", fmt.Sprintf("%d", j))
				h.ServeHTTP(rw, req)
				assert.Equal(t, fmt.Sprintf("%d", j), rw.Body.String())
			}
		}()
	}
	wg.Wait()
}

func TestNoMiddleware(t *testing.T) {
	destination := HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
		rw.Write([]byte("Hello"))
	})

	ctx := context.Background()
	h := NewHandler(ctx, HandlerFunc(destination))
	rw := httptest.NewRecorder()
	req, _ := http.NewRequest("POST", "/", nil)
	h.ServeHTTP(rw, req)
	assert.Equal(t, "Hello", rw.Body.String())
}

func TestPanicCheck(t *testing.T) {
	var hand http.Handler
	destination := HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
		hand.ServeHTTP(rw, r)
	})
	hcreate := HTTPConstructor(func(next http.Handler) http.Handler {
		return next
	})
	middle := hcreate.CreateMiddleware(destination)
	hand = ToHTTP(nil, middle)
	ctx := context.Background()
	req, _ := http.NewRequest("POST", "/", nil)
	rw := httptest.NewRecorder()
	assert.Panics(t, func() {
		middle.ServeHTTPC(ctx, rw, req)
	})
}

func BenchmarkSendWithContext(b *testing.B) {
	incrHandler := IncrHandler{}

	destination := HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
		b.StopTimer()
		rw.Write(ctx.Value("towrite").([]byte))
		b.StartTimer()
	})

	ctx := context.Background()
	h := NewHandler(ctx, HandlerFunc(destination)).Add(NextConstructor(addTowrite), HTTPConstructor(incrHandler.makeHTTP))

	b.ReportAllocs()
	b.ResetTimer()
	b.StopTimer()
	for j := 0; j < b.N; j++ {
		rw := httptest.NewRecorder()
		req, _ := http.NewRequest("POST", "/", nil)
		req.Header.Add("towrite", fmt.Sprintf("%d", j))
		b.StartTimer()
		h.ServeHTTP(rw, req)
		b.StopTimer()
		assert.Equal(b, fmt.Sprintf("%d", j), rw.Body.String())
	}
}

func BenchmarkMinimal(b *testing.B) {
	destination := HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
	})

	ctx := context.Background()
	h := NewHandler(ctx, HandlerFunc(destination))

	b.ReportAllocs()
	b.ResetTimer()
	b.StopTimer()
	for j := 0; j < b.N; j++ {
		rw := httptest.NewRecorder()
		req, _ := http.NewRequest("POST", "/", nil)
		b.StartTimer()
		h.ServeHTTP(rw, req)
		b.StopTimer()
	}
}

func BenchmarkSingle(b *testing.B) {
	incrHandler := IncrHandler{}
	destination := HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
	})

	ctx := context.Background()
	h := NewHandler(ctx, HandlerFunc(destination)).Add(NextConstructor(incrHandler.ServeHTTPCN))

	b.ReportAllocs()
	b.ResetTimer()
	b.StopTimer()
	for j := 0; j < b.N; j++ {
		rw := httptest.NewRecorder()
		req, _ := http.NewRequest("POST", "/", nil)
		b.StartTimer()
		h.ServeHTTP(rw, req)
		b.StopTimer()
	}
}
<commit_msg>Resolve race condition in test<commit_after>package web

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"golang.org/x/net/context"
	"sync/atomic"
)

type IncrHandler struct {
	before int64
	after  int64
}

func (i *IncrHandler) ServeHTTPCN(ctx context.Context, rw http.ResponseWriter, r *http.Request, next ContextHandler) {
	atomic.AddInt64(&i.before, 1)
	next.ServeHTTPC(ctx, rw, r)
	atomic.AddInt64(&i.after, 1)
}

func (i *IncrHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.Handler) {
	atomic.AddInt64(&i.before, 1)
	next.ServeHTTP(rw, r)
	atomic.AddInt64(&i.after, 1)
}

func (i *IncrHandler) makeHTTP(next http.Handler) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		i.ServeHTTP(rw, r, next)
	})
}

func TestHandler(t *testing.T) {
	i := IncrHandler{}
	expectAfter := 0
	expectBefore := 2
	destination := HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
		assert.Equal(t, "value", ctx.Value("key"))
		assert.Equal(t, expectAfter, i.after)
		assert.Equal(t, expectBefore, i.before)
	})
	ctx := context.Background()
	h := NewHandler(ctx, destination)
	v1 := VarAdder{
		Key:   "key",
		Value: "value",
	}

	h.Add(ConstructorFunc(v1.Generate), HTTPConstructor(i.makeHTTP), NextHTTP(i.ServeHTTP))
	rw := httptest.NewRecorder()
	req, _ := http.NewRequest("POST", "/", nil)
	h.ServeHTTP(rw, req)
	assert.Equal(t, 2, i.after)

	expectAfter = 2
	expectBefore = 4
	h.ServeHTTPC(ctx, rw, req)
	assert.Equal(t, 4, i.after)

	bodyTest := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
		rw.Write([]byte("test"))
	})

	ToHTTP(ctx, FromHTTP(bodyTest)).ServeHTTP(rw, req)
	assert.Equal(t, "test", rw.Body.String())
}

func addTowrite(ctx context.Context, rw http.ResponseWriter, r *http.Request, next ContextHandler) {
	next.ServeHTTPC(context.WithValue(ctx, "towrite", []byte(r.Header.Get("towrite"))), rw, r)
}

func TestMany(t *testing.T) {
	incrHandler := IncrHandler{}

	destination := HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
		rw.Write(ctx.Value("towrite").([]byte))
	})

	ctx := context.Background()
	h := NewHandler(ctx, HandlerFunc(destination)).Add(NextConstructor(addTowrite), HTTPConstructor(incrHandler.makeHTTP))

	wg := sync.WaitGroup{}
	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for j := 0; j < 100; j++ {
				if j%11 == 0 {
					time.Sleep(time.Nanosecond)
				}
				rw := httptest.NewRecorder()
				req, _ := http.NewRequest("POST", "/", nil)
				req.Header.Add("towrite", fmt.Sprintf("%d", j))
				h.ServeHTTP(rw, req)
				assert.Equal(t, fmt.Sprintf("%d", j), rw.Body.String())
			}
		}()
	}
	wg.Wait()
}

func TestNoMiddleware(t *testing.T) {
	destination := HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
		rw.Write([]byte("Hello"))
	})

	ctx := context.Background()
	h := NewHandler(ctx, HandlerFunc(destination))
	rw := httptest.NewRecorder()
	req, _ := http.NewRequest("POST", "/", nil)
	h.ServeHTTP(rw, req)
	assert.Equal(t, "Hello", rw.Body.String())
}

func TestPanicCheck(t *testing.T) {
	var hand http.Handler
	destination := HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
		hand.ServeHTTP(rw, r)
	})
	hcreate := HTTPConstructor(func(next http.Handler) http.Handler {
		return next
	})
	middle := hcreate.CreateMiddleware(destination)
	hand = ToHTTP(nil, middle)
	ctx := context.Background()
	req, _ := http.NewRequest("POST", "/", nil)
	rw := httptest.NewRecorder()
	assert.Panics(t, func() {
		middle.ServeHTTPC(ctx, rw, req)
	})
}

func BenchmarkSendWithContext(b *testing.B) {
	incrHandler := IncrHandler{}

	destination := HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
		b.StopTimer()
		rw.Write(ctx.Value("towrite").([]byte))
		b.StartTimer()
	})

	ctx := context.Background()
	h := NewHandler(ctx, HandlerFunc(destination)).Add(NextConstructor(addTowrite), HTTPConstructor(incrHandler.makeHTTP))

	b.ReportAllocs()
	b.ResetTimer()
	b.StopTimer()
	for j := 0; j < b.N; j++ {
		rw := httptest.NewRecorder()
		req, _ := http.NewRequest("POST", "/", nil)
		req.Header.Add("towrite", fmt.Sprintf("%d", j))
		b.StartTimer()
		h.ServeHTTP(rw, req)
		b.StopTimer()
		assert.Equal(b, fmt.Sprintf("%d", j), rw.Body.String())
	}
}

func BenchmarkMinimal(b *testing.B) {
	destination := HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
	})

	ctx := context.Background()
	h := NewHandler(ctx, HandlerFunc(destination))

	b.ReportAllocs()
	b.ResetTimer()
	b.StopTimer()
	for j := 0; j < b.N; j++ {
		rw := httptest.NewRecorder()
		req, _ := http.NewRequest("POST", "/", nil)
		b.StartTimer()
		h.ServeHTTP(rw, req)
		b.StopTimer()
	}
}

func BenchmarkSingle(b *testing.B) {
	incrHandler := IncrHandler{}
	destination := HandlerFunc(func(ctx context.Context, rw http.ResponseWriter, r *http.Request) {
	})

	ctx := context.Background()
	h := NewHandler(ctx, HandlerFunc(destination)).Add(NextConstructor(incrHandler.ServeHTTPCN))

	b.ReportAllocs()
	b.ResetTimer()
	b.StopTimer()
	for j := 0; j < b.N; j++ {
		rw := httptest.NewRecorder()
		req, _ := http.NewRequest("POST", "/", nil)
		b.StartTimer()
		h.ServeHTTP(rw, req)
		b.StopTimer()
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/******************************************************************************
Template Functions are what map functions in the models, to internal logic in
kops. This is the point where we connect static YAML configuration to dynamic
runtime values in memory.

When defining a new function:
	- Build the new function here
	- Define the new function in AddTo()
		dest["MyNewFunction"] = MyNewFunction // <-- Function Pointer
******************************************************************************/

package cloudup

import (
	"encoding/base64"
	"fmt"
	"k8s.io/apimachinery/pkg/util/sets"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/dns"
	"k8s.io/kops/pkg/model"
	"k8s.io/kops/pkg/model/components"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
	"os"
	"strings"
	"text/template"
)

type TemplateFunctions struct {
	cluster        *api.Cluster
	instanceGroups []*api.InstanceGroup

	tags   sets.String
	region string

	modelContext *model.KopsModelContext
}

// This will define the available functions we can use in our YAML models
// If we are trying to get a new function implemented it MUST
// be defined here.
func (tf *TemplateFunctions) AddTo(dest template.FuncMap) {
	dest["SharedVPC"] = tf.SharedVPC

	// Remember that we may be on a different arch from the target.  Hard-code for now.
	dest["Arch"] = func() string { return "amd64" }

	dest["Base64Encode"] = func(s string) string {
		return base64.StdEncoding.EncodeToString([]byte(s))
	}
	dest["replace"] = func(s, find, replace string) string {
		return strings.Replace(s, find, replace, -1)
	}
	dest["join"] = func(a []string, sep string) string {
		return strings.Join(a, sep)
	}

	dest["ClusterName"] = tf.modelContext.ClusterName

	dest["HasTag"] = tf.HasTag

	dest["Image"] = tf.Image

	dest["WithDefaultBool"] = func(v *bool, defaultValue bool) bool {
		if v != nil {
			return *v
		}
		return defaultValue
	}

	dest["GetInstanceGroup"] = tf.GetInstanceGroup

	dest["CloudTags"] = tf.modelContext.CloudTagsForInstanceGroup

	dest["KubeDNS"] = func() *api.KubeDNSConfig {
		return tf.cluster.Spec.KubeDNS
	}

	dest["DnsControllerArgv"] = tf.DnsControllerArgv

	// TODO: Only for GCE?
	dest["EncodeGCELabel"] = gce.EncodeGCELabel

	dest["DnsControllerImage"] = tf.DnsControllerImage
}

// SharedVPC is a simple helper function which makes the templates for a shared VPC clearer
func (tf *TemplateFunctions) SharedVPC() bool {
	return tf.cluster.SharedVPC()
}

// Image returns the docker image name for the specified component
func (tf *TemplateFunctions) Image(component string) (string, error) {
	return components.Image(component, &tf.cluster.Spec)
}

// HasTag returns true if the specified tag is set
func (tf *TemplateFunctions) HasTag(tag string) bool {
	_, found := tf.tags[tag]
	return found
}

// GetInstanceGroup returns the instance group with the specified name
func (tf *TemplateFunctions) GetInstanceGroup(name string) (*api.InstanceGroup, error) {
	for _, ig := range tf.instanceGroups {
		if ig.ObjectMeta.Name == name {
			return ig, nil
		}
	}
	return nil, fmt.Errorf("InstanceGroup %q not found", name)
}

func (tf *TemplateFunctions) DnsControllerArgv() ([]string, error) {
	var argv []string

	argv = append(argv, "/usr/bin/dns-controller")

	argv = append(argv, "--watch-ingress=false")

	switch fi.CloudProviderID(tf.cluster.Spec.CloudProvider) {
	case fi.CloudProviderAWS:
		argv = append(argv, "--dns=aws-route53")
	case fi.CloudProviderGCE:
		argv = append(argv, "--dns=google-clouddns")
	case fi.CloudProviderVSphere:
		argv = append(argv, "--dns=coredns")
		argv = append(argv, "--dns-server="+*tf.cluster.Spec.CloudConfig.VSphereCoreDNSServer)

	default:
		return nil, fmt.Errorf("unhandled cloudprovider %q", tf.cluster.Spec.CloudProvider)
	}

	if dns.IsGossipHostname(tf.cluster.Spec.MasterInternalName) {
		argv = append(argv, "--gossip-seed=127.0.0.1:3999")
	}

	zone := tf.cluster.Spec.DNSZone
	if zone != "" {
		if strings.Contains(zone, ".") {
			// match by name
			argv = append(argv, "--zone="+zone)
		} else {
			// match by id
			argv = append(argv, "--zone=*/"+zone)
		}
	}
	// permit wildcard updates
	argv = append(argv, "--zone=*/*")

	// Verbose, but not crazy logging
	argv = append(argv, "-v=2")

	return argv, nil
}

// To use user-defined DNS Controller:
// 1. DOCKER_REGISTRY=[your docker hub repo] make dns-controller-push
// 2. export DNSCONTROLLER_IMAGE=[your docker hub repo]
// 3. make kops and create/apply cluster
func (tf *TemplateFunctions) DnsControllerImage() (string, error) {
	image := os.Getenv("DNSCONTROLLER_IMAGE")
	if image == "" {
		return "kope/dns-controller", nil
	} else {
		return image, nil
	}
}
<commit_msg>Deploys dns-controller with default ingress setting (--watch-ingress=true)<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/******************************************************************************
Template Functions are what map functions in the models, to internal logic in
kops. This is the point where we connect static YAML configuration to dynamic
runtime values in memory.

When defining a new function:
	- Build the new function here
	- Define the new function in AddTo()
		dest["MyNewFunction"] = MyNewFunction // <-- Function Pointer
******************************************************************************/

package cloudup

import (
	"encoding/base64"
	"fmt"
	"k8s.io/apimachinery/pkg/util/sets"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/dns"
	"k8s.io/kops/pkg/model"
	"k8s.io/kops/pkg/model/components"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
	"os"
	"strings"
	"text/template"
)

type TemplateFunctions struct {
	cluster        *api.Cluster
	instanceGroups []*api.InstanceGroup

	tags   sets.String
	region string

	modelContext *model.KopsModelContext
}

// This will define the available functions we can use in our YAML models
// If we are trying to get a new function implemented it MUST
// be defined here.
func (tf *TemplateFunctions) AddTo(dest template.FuncMap) {
	dest["SharedVPC"] = tf.SharedVPC

	// Remember that we may be on a different arch from the target.  Hard-code for now.
	dest["Arch"] = func() string { return "amd64" }

	dest["Base64Encode"] = func(s string) string {
		return base64.StdEncoding.EncodeToString([]byte(s))
	}
	dest["replace"] = func(s, find, replace string) string {
		return strings.Replace(s, find, replace, -1)
	}
	dest["join"] = func(a []string, sep string) string {
		return strings.Join(a, sep)
	}

	dest["ClusterName"] = tf.modelContext.ClusterName

	dest["HasTag"] = tf.HasTag

	dest["Image"] = tf.Image

	dest["WithDefaultBool"] = func(v *bool, defaultValue bool) bool {
		if v != nil {
			return *v
		}
		return defaultValue
	}

	dest["GetInstanceGroup"] = tf.GetInstanceGroup

	dest["CloudTags"] = tf.modelContext.CloudTagsForInstanceGroup

	dest["KubeDNS"] = func() *api.KubeDNSConfig {
		return tf.cluster.Spec.KubeDNS
	}

	dest["DnsControllerArgv"] = tf.DnsControllerArgv

	// TODO: Only for GCE?
	dest["EncodeGCELabel"] = gce.EncodeGCELabel

	dest["DnsControllerImage"] = tf.DnsControllerImage
}

// SharedVPC is a simple helper function which makes the templates for a shared VPC clearer
func (tf *TemplateFunctions) SharedVPC() bool {
	return tf.cluster.SharedVPC()
}

// Image returns the docker image name for the specified component
func (tf *TemplateFunctions) Image(component string) (string, error) {
	return components.Image(component, &tf.cluster.Spec)
}

// HasTag returns true if the specified tag is set
func (tf *TemplateFunctions) HasTag(tag string) bool {
	_, found := tf.tags[tag]
	return found
}

// GetInstanceGroup returns the instance group with the specified name
func (tf *TemplateFunctions) GetInstanceGroup(name string) (*api.InstanceGroup, error) {
	for _, ig := range tf.instanceGroups {
		if ig.ObjectMeta.Name == name {
			return ig, nil
		}
	}
	return nil, fmt.Errorf("InstanceGroup %q not found", name)
}

func (tf *TemplateFunctions) DnsControllerArgv() ([]string, error) {
	var argv []string

	argv = append(argv, "/usr/bin/dns-controller")

	// Default dns-controller behavior --watch-ingress=true
	// Turning on as per: https://github.com/kubernetes/kops/issues/551#issuecomment-275981949
	// argv = append(argv, "--watch-ingress=false")

	switch fi.CloudProviderID(tf.cluster.Spec.CloudProvider) {
	case fi.CloudProviderAWS:
		argv = append(argv, "--dns=aws-route53")
	case fi.CloudProviderGCE:
		argv = append(argv, "--dns=google-clouddns")
	case fi.CloudProviderVSphere:
		argv = append(argv, "--dns=coredns")
		argv = append(argv, "--dns-server="+*tf.cluster.Spec.CloudConfig.VSphereCoreDNSServer)

	default:
		return nil, fmt.Errorf("unhandled cloudprovider %q", tf.cluster.Spec.CloudProvider)
	}

	if dns.IsGossipHostname(tf.cluster.Spec.MasterInternalName) {
		argv = append(argv, "--gossip-seed=127.0.0.1:3999")
	}

	zone := tf.cluster.Spec.DNSZone
	if zone != "" {
		if strings.Contains(zone, ".") {
			// match by name
			argv = append(argv, "--zone="+zone)
		} else {
			// match by id
			argv = append(argv, "--zone=*/"+zone)
		}
	}
	// permit wildcard updates
	argv = append(argv, "--zone=*/*")

	// Verbose, but not crazy logging
	argv = append(argv, "-v=2")

	return argv, nil
}

// To use user-defined DNS Controller:
// 1. DOCKER_REGISTRY=[your docker hub repo] make dns-controller-push
// 2. export DNSCONTROLLER_IMAGE=[your docker hub repo]
// 3. make kops and create/apply cluster
func (tf *TemplateFunctions) DnsControllerImage() (string, error) {
	image := os.Getenv("DNSCONTROLLER_IMAGE")
	if image == "" {
		return "kope/dns-controller", nil
	} else {
		return image, nil
	}
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "math"
    "net"
    "net/http"
    "os"
    "strconv"
    "strings"
    "sync/atomic"
    "time"

    "amproxy/envparse"
    "amproxy/message"
)

var cServerAddr *net.TCPAddr
var skew float64
var authMap map[string]Creds
var authMapLoadTime time.Time
var c counters

func main() {
    var err error

    c.init()

    // Read config from the environment
    var bInterface = envparse.GetSettingStr("BIND_INTERFACE", "127.0.0.1")
    var bPort      = envparse.GetSettingInt("BIND_PORT", 2005)
    var cServer    = envparse.GetSettingStr("CARBON_SERVER", "localhost")
    var cPort      = envparse.GetSettingInt("CARBON_PORT", 2003)
    var authFile   = envparse.GetSettingStr("AUTH_FILE", "")
    skew           = float64(envparse.GetSettingInt("SKEW", 300))

    if(authFile == "") {
        println("No auth file passed")
        os.Exit(1)
    }

    authMap, authMapLoadTime = loadUserConfigFile(authFile)

    cServerAddr, err = net.ResolveTCPAddr("tcp", cServer + ":" + strconv.Itoa(cPort))
    if err != nil {
        println("Unable to resolve carbon server: ", err.Error())
        os.Exit(1)
    }
    fmt.Printf("Carbon server: %s:%d\n", cServer, cPort)

    // Set up the metrics http server
    go func() {
        http.HandleFunc("/", metrics_http_handler)
        http.ListenAndServe(":8080", nil)
    }()

    go shipMetrics(cServerAddr, &c)

    go reloadAuth(authFile)

    // Listen for incoming connections.
    l, err := net.Listen("tcp", bInterface + ":" + strconv.Itoa(bPort))
    if err != nil {
        fmt.Println("Error listening:", err.Error())
        os.Exit(1)
    }
    // Close the listener when the application closes.
    defer l.Close()
    fmt.Println("Listening on " + bInterface + ":" + strconv.Itoa(bPort))

    for {
        // Listen for an incoming connection.
        conn, err := l.Accept()
        if err != nil {
            fmt.Println("Error accepting: ", err.Error())
            os.Exit(1)
        }
        // Handle connections in a new goroutine.
        atomic.AddUint64(&c.Connections, 1)
        go handleRequest(conn)
    }
}

func reloadAuth(authFile string) {
    ticker := time.NewTicker(time.Second * 60)
    for _ = range ticker.C {
        info, err := os.Stat(authFile)
        if err != nil {
             fmt.Println("Error stating authFile:", err.Error())
             continue
        }

        ts := info.ModTime()
        if ts != authMapLoadTime {
            fmt.Println("Reloading auth file configuration")
            authMap, authMapLoadTime = loadUserConfigFile(authFile)
        }
    }
}

func handleRequest(conn net.Conn) {
    defer conn.Close()

    fmt.Println("Connection from: ", conn.RemoteAddr())

    // connect to carbon server
    carbon_conn, err := net.DialTCP("tcp", nil, cServerAddr)
    if err != nil {
        atomic.AddUint64(&c.BadCarbonconn, 1)
        println("Connection to carbon server failed:", err.Error())
        return
    }
    defer carbon_conn.Close()

    var buf [1024]byte
    buffer := ""
    for {
        n, err := conn.Read(buf[0:])
        if err != nil {
            return
        }
        buffer = buffer + string(buf[:n])

        // If the buffer ends in a newline, process the metrics
        if string(buf[n - 1]) == "\n" {
            lines := strings.Split(buffer, "\n")

            for i := 0; i < len(lines) ; i++ {
                if len(strings.TrimSpace(lines[i])) > 0 {
                    processMessage(carbon_conn, lines[i])
                }
            }

            buffer = ""
        }
    }
}

func processMessage(conn *net.TCPConn, line string) {

    msg := new(message.Message)
    e := msg.Decompose(line)
    if e != nil {
        atomic.AddUint64(&c.BadDecompose, 1)
        fmt.Printf("Error decomposing message %q - %s\n", line, e.Error())
        return
    }

    creds, ok := authMap[msg.Public_key]

    if !ok {
        atomic.AddUint64(&c.BadKeyundef, 1)
        fmt.Printf("key not defined for %s\n", msg.Public_key)
        return
    }

    sig := msg.ComputeSignature(creds.SecretKey)

    if sig != msg.Signature {
        atomic.AddUint64(&c.BadSig, 1)
        fmt.Printf("Computed signature %s doesn't match provided signature %s\n", sig, msg.Signature)
        return
    }

    delta := math.Abs(float64(time.Now().Unix() - int64(msg.Timestamp)))
    if delta > skew {
        atomic.AddUint64(&c.BadSkew, 1)
        fmt.Printf("delta = %.0f, max skew set to %.0f\n", delta, skew)
        return
    }

    // validate the metric is on the approved list
    _, ok = creds.Metrics[msg.Name]
    if !ok {
        atomic.AddUint64(&c.BadMetric, 1)
        fmt.Printf("not an approved metric: %s\n", msg.Name)
        return
    }

    _, err := conn.Write([]byte(msg.MetricStr() + "\n"))
    if err != nil {
        atomic.AddUint64(&c.BadCarbonwrite, 1)
        println("Write to carbon server failed:", err.Error())
        return
    }
    atomic.AddUint64(&c.GoodMetric, 1)

    // write the n bytes read
    _, err2 := conn.Write([]byte(line))
    if err2 != nil {
        return
    }
}
<commit_msg>don't echo the input anymore (especially not to the wrong connection)<commit_after>package main

import (
    "fmt"
    "math"
    "net"
    "net/http"
    "os"
    "strconv"
    "strings"
    "sync/atomic"
    "time"

    "amproxy/envparse"
    "amproxy/message"
)

var cServerAddr *net.TCPAddr
var skew float64
var authMap map[string]Creds
var authMapLoadTime time.Time
var c counters

func main() {
    var err error

    c.init()

    // Read config from the environment
    var bInterface = envparse.GetSettingStr("BIND_INTERFACE", "127.0.0.1")
    var bPort      = envparse.GetSettingInt("BIND_PORT", 2005)
    var cServer    = envparse.GetSettingStr("CARBON_SERVER", "localhost")
    var cPort      = envparse.GetSettingInt("CARBON_PORT", 2003)
    var authFile   = envparse.GetSettingStr("AUTH_FILE", "")
    skew           = float64(envparse.GetSettingInt("SKEW", 300))

    if(authFile == "") {
        println("No auth file passed")
        os.Exit(1)
    }

    authMap, authMapLoadTime = loadUserConfigFile(authFile)

    cServerAddr, err = net.ResolveTCPAddr("tcp", cServer + ":" + strconv.Itoa(cPort))
    if err != nil {
        println("Unable to resolve carbon server: ", err.Error())
        os.Exit(1)
    }
    fmt.Printf("Carbon server: %s:%d\n", cServer, cPort)

    // Set up the metrics http server
    go func() {
        http.HandleFunc("/", metrics_http_handler)
        http.ListenAndServe(":8080", nil)
    }()

    go shipMetrics(cServerAddr, &c)

    go reloadAuth(authFile)

    // Listen for incoming connections.
    l, err := net.Listen("tcp", bInterface + ":" + strconv.Itoa(bPort))
    if err != nil {
        fmt.Println("Error listening:", err.Error())
        os.Exit(1)
    }
    // Close the listener when the application closes.
    defer l.Close()
    fmt.Println("Listening on " + bInterface + ":" + strconv.Itoa(bPort))

    for {
        // Listen for an incoming connection.
        conn, err := l.Accept()
        if err != nil {
            fmt.Println("Error accepting: ", err.Error())
            os.Exit(1)
        }
        // Handle connections in a new goroutine.
        atomic.AddUint64(&c.Connections, 1)
        go handleRequest(conn)
    }
}

func reloadAuth(authFile string) {
    ticker := time.NewTicker(time.Second * 60)
    for _ = range ticker.C {
        info, err := os.Stat(authFile)
        if err != nil {
             fmt.Println("Error stating authFile:", err.Error())
             continue
        }

        ts := info.ModTime()
        if ts != authMapLoadTime {
            fmt.Println("Reloading auth file configuration")
            authMap, authMapLoadTime = loadUserConfigFile(authFile)
        }
    }
}

func handleRequest(conn net.Conn) {
    defer conn.Close()

    fmt.Println("Connection from: ", conn.RemoteAddr())

    // connect to carbon server
    carbon_conn, err := net.DialTCP("tcp", nil, cServerAddr)
    if err != nil {
        atomic.AddUint64(&c.BadCarbonconn, 1)
        println("Connection to carbon server failed:", err.Error())
        return
    }
    defer carbon_conn.Close()

    var buf [1024]byte
    buffer := ""
    for {
        n, err := conn.Read(buf[0:])
        if err != nil {
            return
        }
        buffer = buffer + string(buf[:n])

        // If the buffer ends in a newline, process the metrics
        if string(buf[n - 1]) == "\n" {
            lines := strings.Split(buffer, "\n")

            for i := 0; i < len(lines) ; i++ {
                if len(strings.TrimSpace(lines[i])) > 0 {
                    processMessage(carbon_conn, lines[i])
                }
            }

            buffer = ""
        }
    }
}

func processMessage(conn *net.TCPConn, line string) {

    msg := new(message.Message)
    e := msg.Decompose(line)
    if e != nil {
        atomic.AddUint64(&c.BadDecompose, 1)
        fmt.Printf("Error decomposing message %q - %s\n", line, e.Error())
        return
    }

    creds, ok := authMap[msg.Public_key]

    if !ok {
        atomic.AddUint64(&c.BadKeyundef, 1)
        fmt.Printf("key not defined for %s\n", msg.Public_key)
        return
    }

    sig := msg.ComputeSignature(creds.SecretKey)

    if sig != msg.Signature {
        atomic.AddUint64(&c.BadSig, 1)
        fmt.Printf("Computed signature %s doesn't match provided signature %s\n", sig, msg.Signature)
        return
    }

    delta := math.Abs(float64(time.Now().Unix() - int64(msg.Timestamp)))
    if delta > skew {
        atomic.AddUint64(&c.BadSkew, 1)
        fmt.Printf("delta = %.0f, max skew set to %.0f\n", delta, skew)
        return
    }

    // validate the metric is on the approved list
    _, ok = creds.Metrics[msg.Name]
    if !ok {
        atomic.AddUint64(&c.BadMetric, 1)
        fmt.Printf("not an approved metric: %s\n", msg.Name)
        return
    }

    fmt.Printf("writing: |" + msg.MetricStr() + "|\n")
    _, err := conn.Write([]byte(msg.MetricStr() + "\n"))
    if err != nil {
        atomic.AddUint64(&c.BadCarbonwrite, 1)
        println("Write to carbon server failed:", err.Error())
        return
    }
    atomic.AddUint64(&c.GoodMetric, 1)
}
<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package RuntimeTest

import (
	"context"
	"fmt"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"
	"github.com/cilium/cilium/test/helpers/constants"

	. "github.com/onsi/gomega"
)

var _ = Describe("RuntimeKafka", func() {

	var (
		vm          *helpers.SSHMeta
		monitorStop = func() error { return nil }

		allowedTopic  = "allowedTopic"
		disallowTopic = "disallowTopic"
		topicTest     = "test-topic"
		listTopicsCmd = "/opt/kafka/bin/kafka-topics.sh --list --zookeeper zook:2181"
		MaxMessages   = 5
		client        = "client"
	)

	containers := func(mode string) {

		images := map[string]string{
			"zook":   constants.ZookeeperImage,
			"client": constants.KafkaClientImage,
		}

		switch mode {
		case "create":
			for k, v := range images {
				vm.ContainerCreate(k, v, helpers.CiliumDockerNetwork, fmt.Sprintf("-l id.%s", k))
			}
			zook, err := vm.ContainerInspectNet("zook")
			Expect(err).Should(BeNil())

			vm.ContainerCreate("kafka", constants.KafkaImage, helpers.CiliumDockerNetwork, fmt.Sprintf(
				"-l id.kafka -e KAFKA_ZOOKEEPER_CONNECT=%s:2181 -e KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS=60000 -e KAFKA_LISTENERS=PLAINTEXT://:9092 -e KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS=60000", zook["IPv4"]))

		case "delete":
			for k := range images {
				vm.ContainerRm(k)
			}
			vm.ContainerRm("kafka")
		}
	}

	createTopicCmd := func(topic string) string {
		return fmt.Sprintf("/opt/kafka/bin/kafka-topics.sh --create --zookeeper zook:2181 "+
			"--replication-factor 1 --partitions 1 --topic %s", topic)
	}

	createTopic := func(topic string) {
		logger.Infof("Creating new kafka topic %s", topic)
		res := vm.ContainerExec(client, createTopicCmd(topic))
		res.ExpectSuccess("Unable to create topic  %s", topic)
	}

	consumerCmd := func(topic string, maxMsg int) string {
		return fmt.Sprintf("/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server "+
			"kafka:9092 --topic %s --max-messages %d --timeout-ms 300000 --from-beginning",
			topic, maxMsg)
	}

	consumer := func(topic string, maxMsg int) *helpers.CmdRes {
		return vm.ContainerExec(client, consumerCmd(topic, maxMsg))
	}

	producer := func(topic string, message string) {
		cmd := fmt.Sprintf(
			"echo %s | docker exec -i %s /opt/kafka/bin/kafka-console-producer.sh "+
				"--broker-list kafka:9092 --topic %s",
			message, client, topic)
		vm.Exec(cmd)
	}

	// WaitKafkaBroker waits for the broker to be ready, by executing
	// a command repeatedly until it succeeds, or a timeout occurs
	waitForKafkaBroker := func(pod string, cmd string) error {
		body := func() bool {
			res := vm.ContainerExec(pod, cmd)
			return res.WasSuccessful()
		}
		err := helpers.WithTimeout(body, "Kafka Broker not ready", &helpers.TimeoutConfig{Timeout: helpers.HelperTimeout})
		return err
	}

	BeforeAll(func() {
		vm = helpers.InitRuntimeHelper(helpers.Runtime, logger)
		ExpectCiliumReady(vm)

		containers("create")
		epsReady := vm.WaitEndpointsReady()
		Expect(epsReady).Should(BeTrue(), "Endpoints are not ready after timeout")

		err := waitForKafkaBroker(client, createTopicCmd(topicTest))
		Expect(err).To(BeNil(), "Kafka broker failed to come up")

		By("Creating kafka topics")
		createTopic(allowedTopic)
		createTopic(disallowTopic)

		By("Listing created Kafka topics")
		res := vm.ContainerExec(client, listTopicsCmd)
		res.ExpectSuccess("Cannot list kafka topics")
	})

	AfterEach(func() {
		vm.PolicyDelAll()

	})

	AfterAll(func() {
		containers("delete")
	})

	JustBeforeEach(func() {
		monitorStop = vm.MonitorStart()
	})

	JustAfterEach(func() {
		vm.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
		Expect(monitorStop()).To(BeNil(), "cannot stop monitor command")
	})

	AfterFailed(func() {
		vm.ReportFailed("cilium policy get")
	})

	It("Kafka Policy Ingress", func() {
		_, err := vm.PolicyImportAndWait(vm.GetFullPath("Policies-kafka.json"), helpers.HelperTimeout)
		Expect(err).Should(BeNil())

		endPoints, err := vm.PolicyEndpointsSummary()
		Expect(err).Should(BeNil(), "Cannot get endpoint list")
		Expect(endPoints[helpers.Enabled]).To(Equal(1),
			"Check number of endpoints with policy enforcement enabled")
		Expect(endPoints[helpers.Disabled]).To(Equal(2),
			"Check number of endpoints with policy enforcement disabled")

		By("Allowed topic")

		By("Sending produce request on kafka topic `allowedTopic`")
		for i := 1; i <= MaxMessages; i++ {
			producer(allowedTopic, fmt.Sprintf("Message %d", i))
		}

		By("Sending consume request on kafka topic `allowedTopic`")
		res := consumer(allowedTopic, MaxMessages)
		res.ExpectSuccess("Failed to consume messages from kafka topic `allowedTopic`")
		Expect(res.CombineOutput().String()).
			Should(ContainSubstring("Processed a total of %d messages", MaxMessages),
				"Kafka did not process the expected number of messages")

		By("Disable topic")
		res = consumer(disallowTopic, MaxMessages)
		res.ExpectFail("Kafka consumer can access to disallowTopic")
	})

	It("Kafka Policy Role Ingress", func() {
		_, err := vm.PolicyImportAndWait(vm.GetFullPath("Policies-kafka-Role.json"), helpers.HelperTimeout)
		Expect(err).Should(BeNil(), "Expected nil got %s while importing policy Policies-kafka-Role.json", err)

		endPoints, err := vm.PolicyEndpointsSummary()
		Expect(err).Should(BeNil(), "Expect nil. Failed to apply policy on all endpoints with error :%s", err)
		Expect(endPoints[helpers.Enabled]).To(Equal(1), "Expected 1 endpoint to be policy enabled. Policy enforcement failed")
		Expect(endPoints[helpers.Disabled]).To(Equal(2), "Expected 2 endpoint to be policy disabled. Policy enforcement failed")

		By("Sending produce request on kafka topic `allowedTopic`")
		for i := 1; i <= MaxMessages; i++ {
			producer(allowedTopic, fmt.Sprintf("Message %d", i))
		}

		By("Sending consume request on kafka topic `allowedTopic`")
		res := consumer(allowedTopic, MaxMessages)
		res.ExpectSuccess("Failed to consume messages from kafka topic `allowedTopic`")
		Expect(res.CombineOutput().String()).
			Should(ContainSubstring("Processed a total of %d messages", MaxMessages),
				"Kafka did not process the expected number of messages")

		By("Disable topic")
		// Consumer timeout didn't work correctly, so make sure that AUTH is present in the reply
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		res = vm.ExecInBackground(ctx, fmt.Sprintf(
			"docker exec -i %s %s", client, consumerCmd(disallowTopic, MaxMessages)))
		err = res.WaitUntilMatch("{disallowTopic=TOPIC_AUTHORIZATION_FAILED}")
		Expect(err).To(BeNil(), "Traffic in disallowTopic is allowed")
	})
})
<commit_msg>CI: Change Kafka runtime tests to use local conntrack maps.<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package RuntimeTest

import (
	"context"
	"fmt"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"
	"github.com/cilium/cilium/test/helpers/constants"

	. "github.com/onsi/gomega"
)

var _ = Describe("RuntimeKafka", func() {

	var (
		vm          *helpers.SSHMeta
		monitorStop = func() error { return nil }

		allowedTopic  = "allowedTopic"
		disallowTopic = "disallowTopic"
		topicTest     = "test-topic"
		listTopicsCmd = "/opt/kafka/bin/kafka-topics.sh --list --zookeeper zook:2181"
		MaxMessages   = 5
		client        = "client"
	)

	containers := func(mode string) {

		images := map[string]string{
			"zook":   constants.ZookeeperImage,
			"client": constants.KafkaClientImage,
		}

		switch mode {
		case "create":
			for k, v := range images {
				vm.ContainerCreate(k, v, helpers.CiliumDockerNetwork, fmt.Sprintf("-l id.%s", k))
			}
			zook, err := vm.ContainerInspectNet("zook")
			Expect(err).Should(BeNil())

			vm.ContainerCreate("kafka", constants.KafkaImage, helpers.CiliumDockerNetwork, fmt.Sprintf(
				"-l id.kafka -e KAFKA_ZOOKEEPER_CONNECT=%s:2181 -e KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS=60000 -e KAFKA_LISTENERS=PLAINTEXT://:9092 -e KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS=60000", zook["IPv4"]))

		case "delete":
			for k := range images {
				vm.ContainerRm(k)
			}
			vm.ContainerRm("kafka")
		}
	}

	createTopicCmd := func(topic string) string {
		return fmt.Sprintf("/opt/kafka/bin/kafka-topics.sh --create --zookeeper zook:2181 "+
			"--replication-factor 1 --partitions 1 --topic %s", topic)
	}

	createTopic := func(topic string) {
		logger.Infof("Creating new kafka topic %s", topic)
		res := vm.ContainerExec(client, createTopicCmd(topic))
		res.ExpectSuccess("Unable to create topic  %s", topic)
	}

	consumerCmd := func(topic string, maxMsg int) string {
		return fmt.Sprintf("/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server "+
			"kafka:9092 --topic %s --max-messages %d --timeout-ms 300000 --from-beginning",
			topic, maxMsg)
	}

	consumer := func(topic string, maxMsg int) *helpers.CmdRes {
		return vm.ContainerExec(client, consumerCmd(topic, maxMsg))
	}

	producer := func(topic string, message string) {
		cmd := fmt.Sprintf(
			"echo %s | docker exec -i %s /opt/kafka/bin/kafka-console-producer.sh "+
				"--broker-list kafka:9092 --topic %s",
			message, client, topic)
		vm.Exec(cmd)
	}

	// WaitKafkaBroker waits for the broker to be ready, by executing
	// a command repeatedly until it succeeds, or a timeout occurs
	waitForKafkaBroker := func(pod string, cmd string) error {
		body := func() bool {
			res := vm.ContainerExec(pod, cmd)
			return res.WasSuccessful()
		}
		err := helpers.WithTimeout(body, "Kafka Broker not ready", &helpers.TimeoutConfig{Timeout: helpers.HelperTimeout})
		return err
	}

	BeforeAll(func() {
		vm = helpers.InitRuntimeHelper(helpers.Runtime, logger)
		ExpectCiliumReady(vm)

		status := vm.ExecCilium(fmt.Sprintf("config %s=true",
			helpers.OptionConntrackLocal))
		status.ExpectSuccess()

		containers("create")
		epsReady := vm.WaitEndpointsReady()
		Expect(epsReady).Should(BeTrue(), "Endpoints are not ready after timeout")

		err := waitForKafkaBroker(client, createTopicCmd(topicTest))
		Expect(err).To(BeNil(), "Kafka broker failed to come up")

		By("Creating kafka topics")
		createTopic(allowedTopic)
		createTopic(disallowTopic)

		By("Listing created Kafka topics")
		res := vm.ContainerExec(client, listTopicsCmd)
		res.ExpectSuccess("Cannot list kafka topics")
	})

	AfterEach(func() {
		vm.PolicyDelAll()

	})

	AfterAll(func() {
		containers("delete")

		status := vm.ExecCilium(fmt.Sprintf("config %s=false",
			helpers.OptionConntrackLocal))
		status.ExpectSuccess()
	})

	JustBeforeEach(func() {
		monitorStop = vm.MonitorStart()
	})

	JustAfterEach(func() {
		vm.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
		Expect(monitorStop()).To(BeNil(), "cannot stop monitor command")
	})

	AfterFailed(func() {
		vm.ReportFailed("cilium policy get")
	})

	It("Kafka Policy Ingress", func() {
		_, err := vm.PolicyImportAndWait(vm.GetFullPath("Policies-kafka.json"), helpers.HelperTimeout)
		Expect(err).Should(BeNil())

		endPoints, err := vm.PolicyEndpointsSummary()
		Expect(err).Should(BeNil(), "Cannot get endpoint list")
		Expect(endPoints[helpers.Enabled]).To(Equal(1),
			"Check number of endpoints with policy enforcement enabled")
		Expect(endPoints[helpers.Disabled]).To(Equal(2),
			"Check number of endpoints with policy enforcement disabled")

		By("Allowed topic")

		By("Sending produce request on kafka topic `allowedTopic`")
		for i := 1; i <= MaxMessages; i++ {
			producer(allowedTopic, fmt.Sprintf("Message %d", i))
		}

		By("Sending consume request on kafka topic `allowedTopic`")
		res := consumer(allowedTopic, MaxMessages)
		res.ExpectSuccess("Failed to consume messages from kafka topic `allowedTopic`")
		Expect(res.CombineOutput().String()).
			Should(ContainSubstring("Processed a total of %d messages", MaxMessages),
				"Kafka did not process the expected number of messages")

		By("Disable topic")
		res = consumer(disallowTopic, MaxMessages)
		res.ExpectFail("Kafka consumer can access to disallowTopic")
	})

	It("Kafka Policy Role Ingress", func() {
		_, err := vm.PolicyImportAndWait(vm.GetFullPath("Policies-kafka-Role.json"), helpers.HelperTimeout)
		Expect(err).Should(BeNil(), "Expected nil got %s while importing policy Policies-kafka-Role.json", err)

		endPoints, err := vm.PolicyEndpointsSummary()
		Expect(err).Should(BeNil(), "Expect nil. Failed to apply policy on all endpoints with error :%s", err)
		Expect(endPoints[helpers.Enabled]).To(Equal(1), "Expected 1 endpoint to be policy enabled. Policy enforcement failed")
		Expect(endPoints[helpers.Disabled]).To(Equal(2), "Expected 2 endpoint to be policy disabled. Policy enforcement failed")

		By("Sending produce request on kafka topic `allowedTopic`")
		for i := 1; i <= MaxMessages; i++ {
			producer(allowedTopic, fmt.Sprintf("Message %d", i))
		}

		By("Sending consume request on kafka topic `allowedTopic`")
		res := consumer(allowedTopic, MaxMessages)
		res.ExpectSuccess("Failed to consume messages from kafka topic `allowedTopic`")
		Expect(res.CombineOutput().String()).
			Should(ContainSubstring("Processed a total of %d messages", MaxMessages),
				"Kafka did not process the expected number of messages")

		By("Disable topic")
		// Consumer timeout didn't work correctly, so make sure that AUTH is present in the reply
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		res = vm.ExecInBackground(ctx, fmt.Sprintf(
			"docker exec -i %s %s", client, consumerCmd(disallowTopic, MaxMessages)))
		err = res.WaitUntilMatch("{disallowTopic=TOPIC_AUTHORIZATION_FAILED}")
		Expect(err).To(BeNil(), "Traffic in disallowTopic is allowed")
	})
})
<|endoftext|>


<commit_before>package backend

import (
	"net/http"

	"github.com/Dataman-Cloud/swan/mesosproto/mesos"
	"github.com/Dataman-Cloud/swan/types"
	"github.com/Sirupsen/logrus"
)

func (b *Backend) LaunchApplication(version *types.Version) error {
	b.sched.TaskLaunched = 0

	// Set scheduler's status to busy for accepting resource.
	b.sched.Status = "busy"

	go func() {
		resources := b.sched.BuildResources(version.Cpus, version.Mem, version.Disk)
		offers, err := b.sched.RequestOffers(resources)
		if err != nil {
			logrus.Errorf("Request offers failed: %s", err.Error())
		}

		for _, offer := range offers {
			cpus, mem, disk := b.sched.OfferedResources(offer)
			var tasks []*mesos.TaskInfo
			for b.sched.TaskLaunched < version.Instances &&
				cpus >= version.Cpus &&
				mem >= version.Mem &&
				disk >= version.Disk {
				task, err := b.sched.BuildTask(offer, version, "")
				if err != nil {
					logrus.Errorf("Build task failed: %s", err.Error())
					return
				}

				if err := b.store.SaveTask(task); err != nil {
					return
				}

				taskInfo := b.sched.BuildTaskInfo(offer, resources, task)
				tasks = append(tasks, taskInfo)

				if len(task.HealthChecks) != 0 {
					if err := b.store.SaveCheck(task,
						*taskInfo.Container.Docker.PortMappings[0].HostPort,
						version.ID); err != nil {
					}
					for _, healthCheck := range task.HealthChecks {
						check := types.Check{
							ID:       task.Name,
							Address:  *task.AgentHostname,
							Port:     int(*taskInfo.Container.Docker.PortMappings[0].HostPort),
							TaskID:   task.Name,
							AppID:    version.ID,
							Protocol: healthCheck.Protocol,
							Interval: int(healthCheck.IntervalSeconds),
							Timeout:  int(healthCheck.TimeoutSeconds),
						}
						if healthCheck.Command != nil {
							check.Command = healthCheck.Command
						}

						if healthCheck.Path != nil {
							check.Path = *healthCheck.Path
						}

						if healthCheck.MaxConsecutiveFailures != nil {
							check.MaxFailures = *healthCheck.MaxConsecutiveFailures
						}

						b.sched.HealthCheckManager.Add(&check)
					}
				}

				b.sched.TaskLaunched++
				cpus -= version.Cpus
				mem -= version.Mem
				disk -= version.Disk
			}

			resp, err := b.sched.LaunchTasks(offer, tasks)
			if err != nil {
				logrus.Errorf("Launchs task failed: %s", err.Error())
			}

			if resp != nil && resp.StatusCode != http.StatusAccepted {
				logrus.Errorf("status code %d received", resp.StatusCode)
			}
		}

		// Set scheduler's status back to idle after launch applicaiton.
		b.sched.Status = "idle"
	}()

	return nil
}
<commit_msg>add process for no offer satisfied<commit_after>package backend

import (
	"fmt"
	"github.com/Dataman-Cloud/swan/mesosproto/mesos"
	"github.com/Dataman-Cloud/swan/types"
	"github.com/Sirupsen/logrus"
	"net/http"
)

func (b *Backend) LaunchApplication(version *types.Version) error {
	b.sched.TaskLaunched = 0

	// Set scheduler's status to busy for accepting resource.
	b.sched.Status = "busy"

	resources := b.sched.BuildResources(version.Cpus, version.Mem, version.Disk)
	offers, err := b.sched.RequestOffers(resources)
	if err != nil {
		logrus.Errorf("Request offers failed: %s", err.Error())
	}

	for _, offer := range offers {
		cpus, mem, disk := b.sched.OfferedResources(offer)
		var tasks []*mesos.TaskInfo
		for b.sched.TaskLaunched < version.Instances &&
			cpus >= version.Cpus &&
			mem >= version.Mem &&
			disk >= version.Disk {
			task, err := b.sched.BuildTask(offer, version, "")
			if err != nil {
				return fmt.Errorf("Build task failed: %s", err.Error())
			}

			if err := b.store.SaveTask(task); err != nil {
				return fmt.Errorf("Save task failed: %s", err.Error())
			}

			taskInfo := b.sched.BuildTaskInfo(offer, resources, task)
			tasks = append(tasks, taskInfo)

			if len(task.HealthChecks) != 0 {
				if err := b.store.SaveCheck(task,
					*taskInfo.Container.Docker.PortMappings[0].HostPort,
					version.ID); err != nil {
				}
				for _, healthCheck := range task.HealthChecks {
					check := types.Check{
						ID:       task.Name,
						Address:  *task.AgentHostname,
						Port:     int(*taskInfo.Container.Docker.PortMappings[0].HostPort),
						TaskID:   task.Name,
						AppID:    version.ID,
						Protocol: healthCheck.Protocol,
						Interval: int(healthCheck.IntervalSeconds),
						Timeout:  int(healthCheck.TimeoutSeconds),
					}
					if healthCheck.Command != nil {
						check.Command = healthCheck.Command
					}

					if healthCheck.Path != nil {
						check.Path = *healthCheck.Path
					}

					if healthCheck.MaxConsecutiveFailures != nil {
						check.MaxFailures = *healthCheck.MaxConsecutiveFailures
					}

					b.sched.HealthCheckManager.Add(&check)
				}
			}

			b.sched.TaskLaunched++
			cpus -= version.Cpus
			mem -= version.Mem
			disk -= version.Disk
		}

		if len(tasks) == 0 {
			return fmt.Errorf("Not enough resource")
		}

		resp, err := b.sched.LaunchTasks(offer, tasks)
		if err != nil {
			return fmt.Errorf("Launchs task failed: %s", err.Error())
		}

		if resp != nil && resp.StatusCode != http.StatusAccepted {
			return fmt.Errorf("status code %d received", resp.StatusCode)
		}
	}

	// Set scheduler's status back to idle after launch applicaiton.
	b.sched.Status = "idle"

	return nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package beacon

import (
	"errors"
	"fmt"
	"net"
	"time"

	"github.com/thejerf/suture"
	"golang.org/x/net/ipv6"
)

type Multicast struct {
	*suture.Supervisor
	addr   *net.UDPAddr
	inbox  chan []byte
	outbox chan recv
	mr     *multicastReader
	mw     *multicastWriter
}

func NewMulticast(addr string) *Multicast {
	m := &Multicast{
		Supervisor: suture.New("multicastBeacon", suture.Spec{
			// Don't retry too frenetically: an error to open a socket or
			// whatever is usually something that is either permanent or takes
			// a while to get solved...
			FailureThreshold: 2,
			FailureBackoff:   60 * time.Second,
			// Only log restarts in debug mode.
			Log: func(line string) {
				l.Debugln(line)
			},
		}),
		inbox:  make(chan []byte),
		outbox: make(chan recv, 16),
	}

	m.mr = &multicastReader{
		addr:   addr,
		outbox: m.outbox,
		stop:   make(chan struct{}),
	}
	m.Add(m.mr)

	m.mw = &multicastWriter{
		addr:  addr,
		inbox: m.inbox,
		stop:  make(chan struct{}),
	}
	m.Add(m.mw)

	return m
}

func (m *Multicast) Send(data []byte) {
	m.inbox <- data
}

func (m *Multicast) Recv() ([]byte, net.Addr) {
	recv := <-m.outbox
	return recv.data, recv.src
}

func (m *Multicast) Error() error {
	if err := m.mr.Error(); err != nil {
		return err
	}
	return m.mw.Error()
}

type multicastWriter struct {
	addr  string
	inbox <-chan []byte
	errorHolder
	stop chan struct{}
}

func (w *multicastWriter) Serve() {
	l.Debugln(w, "starting")
	defer l.Debugln(w, "stopping")

	gaddr, err := net.ResolveUDPAddr("udp6", w.addr)
	if err != nil {
		l.Debugln(err)
		w.setError(err)
		return
	}

	conn, err := net.ListenPacket("udp6", ":0")
	if err != nil {
		l.Debugln(err)
		w.setError(err)
		return
	}

	pconn := ipv6.NewPacketConn(conn)

	wcm := &ipv6.ControlMessage{
		HopLimit: 1,
	}

	for bs := range w.inbox {
		intfs, err := net.Interfaces()
		if err != nil {
			l.Debugln(err)
			w.setError(err)
			return
		}

		success := 0
		for _, intf := range intfs {
			wcm.IfIndex = intf.Index
			pconn.SetWriteDeadline(time.Now().Add(time.Second))
			_, err = pconn.WriteTo(bs, wcm, gaddr)
			pconn.SetWriteDeadline(time.Time{})

			if err != nil {
				l.Debugln(err, "on write to", gaddr, intf.Name)
				w.setError(err)
				continue
			}

			l.Debugf("sent %d bytes to %v on %s", len(bs), gaddr, intf.Name)

			success++
		}

		if success > 0 {
			w.setError(nil)
		} else {
			l.Debugln(err)
			w.setError(err)
		}
	}
}

func (w *multicastWriter) Stop() {
	close(w.stop)
}

func (w *multicastWriter) String() string {
	return fmt.Sprintf("multicastWriter@%p", w)
}

type multicastReader struct {
	addr   string
	outbox chan<- recv
	errorHolder
	stop chan struct{}
}

func (r *multicastReader) Serve() {
	l.Debugln(r, "starting")
	defer l.Debugln(r, "stopping")

	gaddr, err := net.ResolveUDPAddr("udp6", r.addr)
	if err != nil {
		l.Debugln(err)
		r.setError(err)
		return
	}

	conn, err := net.ListenPacket("udp6", r.addr)
	if err != nil {
		l.Debugln(err)
		r.setError(err)
		return
	}

	intfs, err := net.Interfaces()
	if err != nil {
		l.Debugln(err)
		r.setError(err)
		return
	}

	pconn := ipv6.NewPacketConn(conn)
	joined := 0
	for _, intf := range intfs {
		err := pconn.JoinGroup(&intf, &net.UDPAddr{IP: gaddr.IP})
		if err != nil {
			l.Debugln("IPv6 join", intf.Name, "failed:", err)
		} else {
			l.Debugln("IPv6 join", intf.Name, "success")
		}
		joined++
	}

	if joined == 0 {
		l.Debugln("no multicast interfaces available")
		r.setError(errors.New("no multicast interfaces available"))
		return
	}

	bs := make([]byte, 65536)
	for {
		n, _, addr, err := pconn.ReadFrom(bs)
		if err != nil {
			l.Debugln(err)
			r.setError(err)
			continue
		}
		l.Debugf("recv %d bytes from %s", n, addr)

		c := make([]byte, n)
		copy(c, bs)
		select {
		case r.outbox <- recv{c, addr}:
		default:
			l.Debugln("dropping message")
		}
	}
}

func (r *multicastReader) Stop() {
	close(r.stop)
}

func (r *multicastReader) String() string {
	return fmt.Sprintf("multicastReader@%p", r)
}
<commit_msg>Remove unused struct field<commit_after>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package beacon

import (
	"errors"
	"fmt"
	"net"
	"time"

	"github.com/thejerf/suture"
	"golang.org/x/net/ipv6"
)

type Multicast struct {
	*suture.Supervisor
	inbox  chan []byte
	outbox chan recv
	mr     *multicastReader
	mw     *multicastWriter
}

func NewMulticast(addr string) *Multicast {
	m := &Multicast{
		Supervisor: suture.New("multicastBeacon", suture.Spec{
			// Don't retry too frenetically: an error to open a socket or
			// whatever is usually something that is either permanent or takes
			// a while to get solved...
			FailureThreshold: 2,
			FailureBackoff:   60 * time.Second,
			// Only log restarts in debug mode.
			Log: func(line string) {
				l.Debugln(line)
			},
		}),
		inbox:  make(chan []byte),
		outbox: make(chan recv, 16),
	}

	m.mr = &multicastReader{
		addr:   addr,
		outbox: m.outbox,
		stop:   make(chan struct{}),
	}
	m.Add(m.mr)

	m.mw = &multicastWriter{
		addr:  addr,
		inbox: m.inbox,
		stop:  make(chan struct{}),
	}
	m.Add(m.mw)

	return m
}

func (m *Multicast) Send(data []byte) {
	m.inbox <- data
}

func (m *Multicast) Recv() ([]byte, net.Addr) {
	recv := <-m.outbox
	return recv.data, recv.src
}

func (m *Multicast) Error() error {
	if err := m.mr.Error(); err != nil {
		return err
	}
	return m.mw.Error()
}

type multicastWriter struct {
	addr  string
	inbox <-chan []byte
	errorHolder
	stop chan struct{}
}

func (w *multicastWriter) Serve() {
	l.Debugln(w, "starting")
	defer l.Debugln(w, "stopping")

	gaddr, err := net.ResolveUDPAddr("udp6", w.addr)
	if err != nil {
		l.Debugln(err)
		w.setError(err)
		return
	}

	conn, err := net.ListenPacket("udp6", ":0")
	if err != nil {
		l.Debugln(err)
		w.setError(err)
		return
	}

	pconn := ipv6.NewPacketConn(conn)

	wcm := &ipv6.ControlMessage{
		HopLimit: 1,
	}

	for bs := range w.inbox {
		intfs, err := net.Interfaces()
		if err != nil {
			l.Debugln(err)
			w.setError(err)
			return
		}

		success := 0
		for _, intf := range intfs {
			wcm.IfIndex = intf.Index
			pconn.SetWriteDeadline(time.Now().Add(time.Second))
			_, err = pconn.WriteTo(bs, wcm, gaddr)
			pconn.SetWriteDeadline(time.Time{})

			if err != nil {
				l.Debugln(err, "on write to", gaddr, intf.Name)
				w.setError(err)
				continue
			}

			l.Debugf("sent %d bytes to %v on %s", len(bs), gaddr, intf.Name)

			success++
		}

		if success > 0 {
			w.setError(nil)
		} else {
			l.Debugln(err)
			w.setError(err)
		}
	}
}

func (w *multicastWriter) Stop() {
	close(w.stop)
}

func (w *multicastWriter) String() string {
	return fmt.Sprintf("multicastWriter@%p", w)
}

type multicastReader struct {
	addr   string
	outbox chan<- recv
	errorHolder
	stop chan struct{}
}

func (r *multicastReader) Serve() {
	l.Debugln(r, "starting")
	defer l.Debugln(r, "stopping")

	gaddr, err := net.ResolveUDPAddr("udp6", r.addr)
	if err != nil {
		l.Debugln(err)
		r.setError(err)
		return
	}

	conn, err := net.ListenPacket("udp6", r.addr)
	if err != nil {
		l.Debugln(err)
		r.setError(err)
		return
	}

	intfs, err := net.Interfaces()
	if err != nil {
		l.Debugln(err)
		r.setError(err)
		return
	}

	pconn := ipv6.NewPacketConn(conn)
	joined := 0
	for _, intf := range intfs {
		err := pconn.JoinGroup(&intf, &net.UDPAddr{IP: gaddr.IP})
		if err != nil {
			l.Debugln("IPv6 join", intf.Name, "failed:", err)
		} else {
			l.Debugln("IPv6 join", intf.Name, "success")
		}
		joined++
	}

	if joined == 0 {
		l.Debugln("no multicast interfaces available")
		r.setError(errors.New("no multicast interfaces available"))
		return
	}

	bs := make([]byte, 65536)
	for {
		n, _, addr, err := pconn.ReadFrom(bs)
		if err != nil {
			l.Debugln(err)
			r.setError(err)
			continue
		}
		l.Debugf("recv %d bytes from %s", n, addr)

		c := make([]byte, n)
		copy(c, bs)
		select {
		case r.outbox <- recv{c, addr}:
		default:
			l.Debugln("dropping message")
		}
	}
}

func (r *multicastReader) Stop() {
	close(r.stop)
}

func (r *multicastReader) String() string {
	return fmt.Sprintf("multicastReader@%p", r)
}
<|endoftext|>