<commit_before>// bulk_query_gen generates queries for various use cases. Its output will
// be consumed by query_benchmarker.
package main

import (
	"bufio"
	"encoding/gob"
	"flag"
	"fmt"
	"github.com/influxdata/influxdb-comparisons/bulk_data_gen/common"
	bulkQueryGen "github.com/influxdata/influxdb-comparisons/bulk_query_gen"
	"github.com/influxdata/influxdb-comparisons/bulk_query_gen/cassandra"
	"github.com/influxdata/influxdb-comparisons/bulk_query_gen/elasticsearch"
	"github.com/influxdata/influxdb-comparisons/bulk_query_gen/influxdb"
	"github.com/influxdata/influxdb-comparisons/bulk_query_gen/mongodb"
	"github.com/influxdata/influxdb-comparisons/bulk_query_gen/opentsdb"
	"github.com/influxdata/influxdb-comparisons/bulk_query_gen/timescaledb"
	"log"
	"math/rand"
	"os"
	"sort"
	"time"
)

const (
	DevOps                          = "devops"
	DevOpsOneHostOneHour            = "1-host-1-hr"
	DevOpsOneHostTwelveHours        = "1-host-12-hr"
	DevOpsEightHostsOneHour         = "8-host-1-hr"
	DevOpsGroupBy                   = "groupby"
	Iot                             = "iot"
	IotOneHomeTwelveHours           = "1-home-12-hours"
	Dashboard                       = "dashboard"
	DashboardAll                    = "dashboard-all"
	DashboardAvailability           = "availability"
	DashboardCpuNum                 = "cpu-num"
	DashboardCpuUtilization         = "cpu-utilization"
	DashboardDiskAllocated          = "disk-allocated"
	DashboardDiskUsage              = "disk-usage"
	DashboardDiskUtilization        = "disk-utilization"
	DashboardHttpRequestDuration    = "http-request-duration"
	DashboardHttpRequests           = "http-requests"
	DashboardKapaCpu                = "kapa-cpu"
	DashboardKapaLoad               = "kapa-load"
	DashboardKapaRam                = "kapa-ram"
	DashboardMemoryTotal            = "memory-total"
	DashboardMemoryUtilization      = "memory-utilization"
	DashboardNginxRequests          = "nginx-requests"
	DashboardQueueBytes             = "queue-bytes"
	DashboardRedisMemoryUtilization = "redis-memory-utilization"
	DashboardSystemLoad             = "system-load"
	DashboardThroughput             = "throughput"
)

// query generator choices {use-case, query-type, format}
// (This object is shown to the user when flag.Usage is called.)
var useCaseMatrix = map[string]map[string]map[string]bulkQueryGen.QueryGeneratorMaker{
	DevOps: {
		DevOpsOneHostOneHour: {
			"cassandra":        cassandra.NewCassandraDevopsSingleHost,
			"es-http":          elasticsearch.NewElasticSearchDevopsSingleHost,
			"influx-flux-http": influxdb.NewFluxDevopsSingleHost,
			"influx-http":      influxdb.NewInfluxQLDevopsSingleHost,
			"mongo":            mongodb.NewMongoDevopsSingleHost,
			"opentsdb":         opentsdb.NewOpenTSDBDevopsSingleHost,
			"timescaledb":      timescaledb.NewTimescaleDevopsSingleHost,
		},
		DevOpsOneHostTwelveHours: {
			"cassandra":        cassandra.NewCassandraDevopsSingleHost12hr,
			"es-http":          elasticsearch.NewElasticSearchDevopsSingleHost12hr,
			"influx-flux-http": influxdb.NewFluxDevopsSingleHost12hr,
			"influx-http":      influxdb.NewInfluxQLDevopsSingleHost12hr,
			"mongo":            mongodb.NewMongoDevopsSingleHost12hr,
			"opentsdb":         opentsdb.NewOpenTSDBDevopsSingleHost12hr,
			"timescaledb":      timescaledb.NewTimescaleDevopsSingleHost12hr,
		},
		DevOpsEightHostsOneHour: {
			"cassandra":        cassandra.NewCassandraDevops8Hosts,
			"es-http":          elasticsearch.NewElasticSearchDevops8Hosts,
			"influx-flux-http": influxdb.NewFluxDevops8Hosts,
			"influx-http":      influxdb.NewInfluxQLDevops8Hosts,
			"mongo":            mongodb.NewMongoDevops8Hosts1Hr,
			"opentsdb":         opentsdb.NewOpenTSDBDevops8Hosts,
			"timescaledb":      timescaledb.NewTimescaleDevops8Hosts1Hr,
		},
		DevOpsGroupBy: {
			"cassandra":        cassandra.NewCassandraDevopsGroupBy,
			"es-http":          elasticsearch.NewElasticSearchDevopsGroupBy,
			"influx-flux-http": influxdb.NewFluxDevopsGroupBy,
			"influx-http":      influxdb.NewInfluxQLDevopsGroupBy,
			"timescaledb":      timescaledb.NewTimescaleDevopsGroupby,
		},
	},
	Iot: {
		IotOneHomeTwelveHours: {
			"influx-flux-http": influxdb.NewFluxIotSingleHost,
			"influx-http":      influxdb.NewInfluxQLIotSingleHost,
			"timescaledb":      timescaledb.NewTimescaleIotSingleHost,
			"cassandra":        cassandra.NewCassandraIotSingleHost,
			"mongo":            mongodb.NewMongoIotSingleHost,
		},
	},
	Dashboard: {
		DashboardAll: {
			"influx-http": influxdb.NewInfluxQLDashboardAll,
		},
		DashboardCpuNum: {
			"influx-http": influxdb.NewInfluxQLDashboardCpuNum,
		},
		DashboardAvailability: {
			"influx-http": influxdb.NewInfluxQLDashboardAvailability,
		},
		DashboardCpuUtilization:         {"influx-http": influxdb.NewInfluxQLDashboardCpuUtilization},
		DashboardDiskAllocated:          {"influx-http": influxdb.NewInfluxQLDashboardDiskAllocated},
		DashboardDiskUsage:              {"influx-http": influxdb.NewInfluxQLDashboardDiskUsage},
		DashboardDiskUtilization:        {"influx-http": influxdb.NewInfluxQLDashboardDiskUtilization},
		DashboardHttpRequestDuration:    {"influx-http": influxdb.NewInfluxQLDashboardHttpRequestDuration},
		DashboardHttpRequests:           {"influx-http": influxdb.NewInfluxQLDashboardHttpRequests},
		DashboardKapaCpu:                {"influx-http": influxdb.NewInfluxQLDashboardKapaCpu},
		DashboardKapaLoad:               {"influx-http": influxdb.NewInfluxQLDashboardKapaLoad},
		DashboardKapaRam:                {"influx-http": influxdb.NewInfluxQLDashboardKapaRam},
		DashboardMemoryTotal:            {"influx-http": influxdb.NewInfluxQLDashboardMemoryTotal},
		DashboardMemoryUtilization:      {"influx-http": influxdb.NewInfluxQLDashboardMemoryUtilization},
		DashboardNginxRequests:          {"influx-http": influxdb.NewInfluxQLDashboardNginxRequests},
		DashboardQueueBytes:             {"influx-http": influxdb.NewInfluxQLDashboardQueueBytes},
		DashboardRedisMemoryUtilization: {"influx-http": influxdb.NewInfluxQLDashboardRedisMemoryUtilization},
		DashboardSystemLoad:             {"influx-http": influxdb.NewInfluxQLDashboardSystemLoad},
		DashboardThroughput:             {"influx-http": influxdb.NewInfluxQLDashboardThroughput},
	},
}

// Program option vars:
var (
	useCase   string
	queryType string
	format    string

	scaleVar   int
	queryCount int

	dbName string // TODO(rw): make this a map[string]string -> DatabaseConfig

	timestampStartStr string
	timestampEndStr   string

	timestampStart time.Time
	timestampEnd   time.Time
	queryInterval  time.Duration
	timeWindowShift time.Duration

	seed  int64
	debug int

	interleavedGenerationGroupID uint
	interleavedGenerationGroups  uint
)

// Parse args:
func init() {
	// Change the Usage function to print the use case matrix of choices:
	oldUsage := flag.Usage
	flag.Usage = func() {
		oldUsage()

		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "The use case matrix of choices is:\n")
		for uc, queryTypes := range useCaseMatrix {
			for qt, formats := range queryTypes {
				for f := range formats {
					fmt.Fprintf(os.Stderr, "  use case: %s, query type: %s, format: %s\n", uc, qt, f)
				}
			}
		}
	}

	flag.StringVar(&format, "format", "influx-http", "Format to emit. (Choices are in the use case matrix.)")
	flag.StringVar(&useCase, "use-case", "devops", "Use case to model. (Choices are in the use case matrix.)")
	flag.StringVar(&queryType, "query-type", "", "Query type. (Choices are in the use case matrix.)")

	flag.IntVar(&scaleVar, "scale-var", 1, "Scaling variable (must be the equal to the scalevar used for data generation).")
	flag.IntVar(&queryCount, "queries", 1000, "Number of queries to generate.")
	flag.StringVar(&dbName, "db", "benchmark_db", "Database for influx to use (ignored for ElasticSearch).")

	flag.StringVar(&timestampStartStr, "timestamp-start", common.DefaultDateTimeStart, "Beginning timestamp (RFC3339).")
	flag.StringVar(&timestampEndStr, "timestamp-end", common.DefaultDateTimeEnd, "Ending timestamp (RFC3339).")
	flag.DurationVar(&queryInterval, "query-interval", bulkQueryGen.DefaultQueryInterval, "Time interval query should ask for.")
	flag.DurationVar(&timeWindowShift, "time-window-shift", -1, "Sliding time window shift. (When set to > 0s, queries option is ignored - number of queries is calculated.")

	flag.Int64Var(&seed, "seed", 0, "PRNG seed (default, or 0, uses the current timestamp).")
	flag.IntVar(&debug, "debug", 0, "Debug printing (choices: 0, 1) (default 0).")

	flag.UintVar(&interleavedGenerationGroupID, "interleaved-generation-group-id", 0, "Group (0-indexed) to perform round-robin serialization within. Use this to scale up data generation to multiple processes.")
	flag.UintVar(&interleavedGenerationGroups, "interleaved-generation-groups", 1, "The number of round-robin serialization groups. Use this to scale up data generation to multiple processes.")

	flag.Parse()

	if queryType == DevOpsEightHostsOneHour && scaleVar < 8 {
		log.Fatal("\"scale-var\" must be greater than the hosts grouping number")
	}

	if !(interleavedGenerationGroupID < interleavedGenerationGroups) {
		log.Fatal("incorrect interleaved groups configuration")
	}

	if _, ok := useCaseMatrix[useCase]; !ok {
		log.Fatal("invalid use case specifier")
	}

	if _, ok := useCaseMatrix[useCase][queryType]; !ok {
		log.Fatal("invalid query type specifier")
	}

	if _, ok := useCaseMatrix[useCase][queryType][format]; !ok {
		log.Fatal("invalid format specifier")
	}

	hourGroupInterval := 1

	if queryType == DevOpsOneHostTwelveHours {
		hourGroupInterval = 12
	}

	// Parse timestamps:
	var err error
	timestampStart, err = time.Parse(time.RFC3339, timestampStartStr)
	if err != nil {
		log.Fatal(err)
	}
	timestampStart = timestampStart.UTC()
	timestampEnd, err = time.Parse(time.RFC3339, timestampEndStr)
	if err != nil {
		log.Fatal(err)
	}
	timestampEnd = timestampEnd.UTC()

	duration := timestampEnd.Sub(timestampStart)

	if duration.Nanoseconds() < 0 {
		log.Fatal("\"timestamp-end\" must be grater than \"timestamp-start\"")
	}

	if duration.Nanoseconds()/time.Hour.Nanoseconds() < int64(hourGroupInterval) {
		log.Fatal("Time interval must be greater than the grouping interval")
	}
	if duration.Nanoseconds() < queryInterval.Nanoseconds() {
		log.Fatal("Query interval must be greater than the grouping interval")
	}

	// TODO temporary for benchmarks
	if useCase == Dashboard && timeWindowShift == -1 { // when not set for dashboard, always use 5s default
		timeWindowShift = 5 * time.Second
	}

	if timeWindowShift > 0 {
		bulkQueryGen.TimeWindowShift = timeWindowShift // global
		queryCount = int(timestampEnd.Sub(timestampStart).Seconds() / timeWindowShift.Seconds())
		if (queryType == DashboardAll) {
			queryCount *= 18
		}
		log.Printf("%v queries will be generated to cover time interval using %v shift", queryCount,  timeWindowShift)
	}

	// the default seed is the current timestamp:
	if seed == 0 {
		seed = int64(time.Now().Nanosecond())
	}
	fmt.Fprintf(os.Stderr, "using random seed %d\n", seed)
}

func main() {
	rand.Seed(seed)

	dbConfig := bulkQueryGen.DatabaseConfig{
		bulkQueryGen.DatabaseName: dbName,
	}

	// Make the query generator:
	maker := useCaseMatrix[useCase][queryType][format]
	interval := bulkQueryGen.NewTimeInterval(timestampStart, timestampEnd)
	var generator = maker(dbConfig, interval, queryInterval, scaleVar)

	// Set up bookkeeping:
	stats := make(map[string]int64)

	// Set up output buffering:
	out := bufio.NewWriter(os.Stdout)
	defer out.Flush()

	// Create request instances, serializing them to stdout and collecting
	// counts for each kind. If applicable, only prints queries that
	// belong to this interleaved group id:
	var currentInterleavedGroup uint = 0

	enc := gob.NewEncoder(out)
	for i := 0; i < queryCount; i++ {
		q := generator.Dispatch(i)

		if currentInterleavedGroup == interleavedGenerationGroupID {
			err := enc.Encode(q)
			if err != nil {
				log.Fatal("encoder ", err)
			}
			stats[string(q.HumanLabelName())]++

			if debug == 1 {
				_, err := fmt.Fprintf(os.Stderr, "%s\n", q.HumanLabelName())
				if err != nil {
					log.Fatal(err)
				}
			} else if debug == 2 {
				_, err := fmt.Fprintf(os.Stderr, "%s\n", q.HumanDescriptionName())
				if err != nil {
					log.Fatal(err)
				}
			} else if debug >= 3 {
				_, err := fmt.Fprintf(os.Stderr, "%s\n", q.String())
				if err != nil {
					log.Fatal(err)
				}
			}
		}
		q.Release()

		currentInterleavedGroup++
		if currentInterleavedGroup == interleavedGenerationGroups {
			currentInterleavedGroup = 0
		}
	}

	// Print stats:
	keys := []string{}
	for k, _ := range stats {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		_, err := fmt.Fprintf(os.Stderr, "%s: %d points\n", k, stats[k])
		if err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>Condition respects also default of 0<commit_after>// bulk_query_gen generates queries for various use cases. Its output will
// be consumed by query_benchmarker.
package main

import (
	"bufio"
	"encoding/gob"
	"flag"
	"fmt"
	"github.com/influxdata/influxdb-comparisons/bulk_data_gen/common"
	bulkQueryGen "github.com/influxdata/influxdb-comparisons/bulk_query_gen"
	"github.com/influxdata/influxdb-comparisons/bulk_query_gen/cassandra"
	"github.com/influxdata/influxdb-comparisons/bulk_query_gen/elasticsearch"
	"github.com/influxdata/influxdb-comparisons/bulk_query_gen/influxdb"
	"github.com/influxdata/influxdb-comparisons/bulk_query_gen/mongodb"
	"github.com/influxdata/influxdb-comparisons/bulk_query_gen/opentsdb"
	"github.com/influxdata/influxdb-comparisons/bulk_query_gen/timescaledb"
	"log"
	"math/rand"
	"os"
	"sort"
	"time"
)

const (
	DevOps                          = "devops"
	DevOpsOneHostOneHour            = "1-host-1-hr"
	DevOpsOneHostTwelveHours        = "1-host-12-hr"
	DevOpsEightHostsOneHour         = "8-host-1-hr"
	DevOpsGroupBy                   = "groupby"
	Iot                             = "iot"
	IotOneHomeTwelveHours           = "1-home-12-hours"
	Dashboard                       = "dashboard"
	DashboardAll                    = "dashboard-all"
	DashboardAvailability           = "availability"
	DashboardCpuNum                 = "cpu-num"
	DashboardCpuUtilization         = "cpu-utilization"
	DashboardDiskAllocated          = "disk-allocated"
	DashboardDiskUsage              = "disk-usage"
	DashboardDiskUtilization        = "disk-utilization"
	DashboardHttpRequestDuration    = "http-request-duration"
	DashboardHttpRequests           = "http-requests"
	DashboardKapaCpu                = "kapa-cpu"
	DashboardKapaLoad               = "kapa-load"
	DashboardKapaRam                = "kapa-ram"
	DashboardMemoryTotal            = "memory-total"
	DashboardMemoryUtilization      = "memory-utilization"
	DashboardNginxRequests          = "nginx-requests"
	DashboardQueueBytes             = "queue-bytes"
	DashboardRedisMemoryUtilization = "redis-memory-utilization"
	DashboardSystemLoad             = "system-load"
	DashboardThroughput             = "throughput"
)

// query generator choices {use-case, query-type, format}
// (This object is shown to the user when flag.Usage is called.)
var useCaseMatrix = map[string]map[string]map[string]bulkQueryGen.QueryGeneratorMaker{
	DevOps: {
		DevOpsOneHostOneHour: {
			"cassandra":        cassandra.NewCassandraDevopsSingleHost,
			"es-http":          elasticsearch.NewElasticSearchDevopsSingleHost,
			"influx-flux-http": influxdb.NewFluxDevopsSingleHost,
			"influx-http":      influxdb.NewInfluxQLDevopsSingleHost,
			"mongo":            mongodb.NewMongoDevopsSingleHost,
			"opentsdb":         opentsdb.NewOpenTSDBDevopsSingleHost,
			"timescaledb":      timescaledb.NewTimescaleDevopsSingleHost,
		},
		DevOpsOneHostTwelveHours: {
			"cassandra":        cassandra.NewCassandraDevopsSingleHost12hr,
			"es-http":          elasticsearch.NewElasticSearchDevopsSingleHost12hr,
			"influx-flux-http": influxdb.NewFluxDevopsSingleHost12hr,
			"influx-http":      influxdb.NewInfluxQLDevopsSingleHost12hr,
			"mongo":            mongodb.NewMongoDevopsSingleHost12hr,
			"opentsdb":         opentsdb.NewOpenTSDBDevopsSingleHost12hr,
			"timescaledb":      timescaledb.NewTimescaleDevopsSingleHost12hr,
		},
		DevOpsEightHostsOneHour: {
			"cassandra":        cassandra.NewCassandraDevops8Hosts,
			"es-http":          elasticsearch.NewElasticSearchDevops8Hosts,
			"influx-flux-http": influxdb.NewFluxDevops8Hosts,
			"influx-http":      influxdb.NewInfluxQLDevops8Hosts,
			"mongo":            mongodb.NewMongoDevops8Hosts1Hr,
			"opentsdb":         opentsdb.NewOpenTSDBDevops8Hosts,
			"timescaledb":      timescaledb.NewTimescaleDevops8Hosts1Hr,
		},
		DevOpsGroupBy: {
			"cassandra":        cassandra.NewCassandraDevopsGroupBy,
			"es-http":          elasticsearch.NewElasticSearchDevopsGroupBy,
			"influx-flux-http": influxdb.NewFluxDevopsGroupBy,
			"influx-http":      influxdb.NewInfluxQLDevopsGroupBy,
			"timescaledb":      timescaledb.NewTimescaleDevopsGroupby,
		},
	},
	Iot: {
		IotOneHomeTwelveHours: {
			"influx-flux-http": influxdb.NewFluxIotSingleHost,
			"influx-http":      influxdb.NewInfluxQLIotSingleHost,
			"timescaledb":      timescaledb.NewTimescaleIotSingleHost,
			"cassandra":        cassandra.NewCassandraIotSingleHost,
			"mongo":            mongodb.NewMongoIotSingleHost,
		},
	},
	Dashboard: {
		DashboardAll: {
			"influx-http": influxdb.NewInfluxQLDashboardAll,
		},
		DashboardCpuNum: {
			"influx-http": influxdb.NewInfluxQLDashboardCpuNum,
		},
		DashboardAvailability: {
			"influx-http": influxdb.NewInfluxQLDashboardAvailability,
		},
		DashboardCpuUtilization:         {"influx-http": influxdb.NewInfluxQLDashboardCpuUtilization},
		DashboardDiskAllocated:          {"influx-http": influxdb.NewInfluxQLDashboardDiskAllocated},
		DashboardDiskUsage:              {"influx-http": influxdb.NewInfluxQLDashboardDiskUsage},
		DashboardDiskUtilization:        {"influx-http": influxdb.NewInfluxQLDashboardDiskUtilization},
		DashboardHttpRequestDuration:    {"influx-http": influxdb.NewInfluxQLDashboardHttpRequestDuration},
		DashboardHttpRequests:           {"influx-http": influxdb.NewInfluxQLDashboardHttpRequests},
		DashboardKapaCpu:                {"influx-http": influxdb.NewInfluxQLDashboardKapaCpu},
		DashboardKapaLoad:               {"influx-http": influxdb.NewInfluxQLDashboardKapaLoad},
		DashboardKapaRam:                {"influx-http": influxdb.NewInfluxQLDashboardKapaRam},
		DashboardMemoryTotal:            {"influx-http": influxdb.NewInfluxQLDashboardMemoryTotal},
		DashboardMemoryUtilization:      {"influx-http": influxdb.NewInfluxQLDashboardMemoryUtilization},
		DashboardNginxRequests:          {"influx-http": influxdb.NewInfluxQLDashboardNginxRequests},
		DashboardQueueBytes:             {"influx-http": influxdb.NewInfluxQLDashboardQueueBytes},
		DashboardRedisMemoryUtilization: {"influx-http": influxdb.NewInfluxQLDashboardRedisMemoryUtilization},
		DashboardSystemLoad:             {"influx-http": influxdb.NewInfluxQLDashboardSystemLoad},
		DashboardThroughput:             {"influx-http": influxdb.NewInfluxQLDashboardThroughput},
	},
}

// Program option vars:
var (
	useCase   string
	queryType string
	format    string

	scaleVar   int
	queryCount int

	dbName string // TODO(rw): make this a map[string]string -> DatabaseConfig

	timestampStartStr string
	timestampEndStr   string

	timestampStart  time.Time
	timestampEnd    time.Time
	queryInterval   time.Duration
	timeWindowShift time.Duration

	seed  int64
	debug int

	interleavedGenerationGroupID uint
	interleavedGenerationGroups  uint
)

// Parse args:
func init() {
	// Change the Usage function to print the use case matrix of choices:
	oldUsage := flag.Usage
	flag.Usage = func() {
		oldUsage()

		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "The use case matrix of choices is:\n")
		for uc, queryTypes := range useCaseMatrix {
			for qt, formats := range queryTypes {
				for f := range formats {
					fmt.Fprintf(os.Stderr, "  use case: %s, query type: %s, format: %s\n", uc, qt, f)
				}
			}
		}
	}

	flag.StringVar(&format, "format", "influx-http", "Format to emit. (Choices are in the use case matrix.)")
	flag.StringVar(&useCase, "use-case", "devops", "Use case to model. (Choices are in the use case matrix.)")
	flag.StringVar(&queryType, "query-type", "", "Query type. (Choices are in the use case matrix.)")

	flag.IntVar(&scaleVar, "scale-var", 1, "Scaling variable (must be the equal to the scalevar used for data generation).")
	flag.IntVar(&queryCount, "queries", 1000, "Number of queries to generate.")
	flag.StringVar(&dbName, "db", "benchmark_db", "Database for influx to use (ignored for ElasticSearch).")

	flag.StringVar(&timestampStartStr, "timestamp-start", common.DefaultDateTimeStart, "Beginning timestamp (RFC3339).")
	flag.StringVar(&timestampEndStr, "timestamp-end", common.DefaultDateTimeEnd, "Ending timestamp (RFC3339).")
	flag.DurationVar(&queryInterval, "query-interval", bulkQueryGen.DefaultQueryInterval, "Time interval query should ask for.")
	flag.DurationVar(&timeWindowShift, "time-window-shift", -1, "Sliding time window shift. (When set to > 0s, queries option is ignored - number of queries is calculated.")

	flag.Int64Var(&seed, "seed", 0, "PRNG seed (default, or 0, uses the current timestamp).")
	flag.IntVar(&debug, "debug", 0, "Debug printing (choices: 0, 1) (default 0).")

	flag.UintVar(&interleavedGenerationGroupID, "interleaved-generation-group-id", 0, "Group (0-indexed) to perform round-robin serialization within. Use this to scale up data generation to multiple processes.")
	flag.UintVar(&interleavedGenerationGroups, "interleaved-generation-groups", 1, "The number of round-robin serialization groups. Use this to scale up data generation to multiple processes.")

	flag.Parse()

	if queryType == DevOpsEightHostsOneHour && scaleVar < 8 {
		log.Fatal("\"scale-var\" must be greater than the hosts grouping number")
	}

	if !(interleavedGenerationGroupID < interleavedGenerationGroups) {
		log.Fatal("incorrect interleaved groups configuration")
	}

	if _, ok := useCaseMatrix[useCase]; !ok {
		log.Fatal("invalid use case specifier")
	}

	if _, ok := useCaseMatrix[useCase][queryType]; !ok {
		log.Fatal("invalid query type specifier")
	}

	if _, ok := useCaseMatrix[useCase][queryType][format]; !ok {
		log.Fatal("invalid format specifier")
	}

	hourGroupInterval := 1

	if queryType == DevOpsOneHostTwelveHours {
		hourGroupInterval = 12
	}

	// Parse timestamps:
	var err error
	timestampStart, err = time.Parse(time.RFC3339, timestampStartStr)
	if err != nil {
		log.Fatal(err)
	}
	timestampStart = timestampStart.UTC()
	timestampEnd, err = time.Parse(time.RFC3339, timestampEndStr)
	if err != nil {
		log.Fatal(err)
	}
	timestampEnd = timestampEnd.UTC()

	duration := timestampEnd.Sub(timestampStart)

	if duration.Nanoseconds() < 0 {
		log.Fatal("\"timestamp-end\" must be grater than \"timestamp-start\"")
	}

	if duration.Nanoseconds()/time.Hour.Nanoseconds() < int64(hourGroupInterval) {
		log.Fatal("Time interval must be greater than the grouping interval")
	}
	if duration.Nanoseconds() < queryInterval.Nanoseconds() {
		log.Fatal("Query interval must be greater than the grouping interval")
	}

	// TODO temporary for benchmarks
	if useCase == Dashboard && timeWindowShift <= 0 { // when not set for dashboard, always use 5s default
		timeWindowShift = 5 * time.Second
	}

	if timeWindowShift > 0 {
		bulkQueryGen.TimeWindowShift = timeWindowShift // global
		queryCount = int(timestampEnd.Sub(timestampStart).Seconds() / timeWindowShift.Seconds())
		if queryType == DashboardAll {
			queryCount *= 18
		}
		log.Printf("%v queries will be generated to cover time interval using %v shift", queryCount, timeWindowShift)
	}

	// the default seed is the current timestamp:
	if seed == 0 {
		seed = int64(time.Now().Nanosecond())
	}
	fmt.Fprintf(os.Stderr, "using random seed %d\n", seed)
}

func main() {
	rand.Seed(seed)

	dbConfig := bulkQueryGen.DatabaseConfig{
		bulkQueryGen.DatabaseName: dbName,
	}

	// Make the query generator:
	maker := useCaseMatrix[useCase][queryType][format]
	interval := bulkQueryGen.NewTimeInterval(timestampStart, timestampEnd)
	var generator = maker(dbConfig, interval, queryInterval, scaleVar)

	// Set up bookkeeping:
	stats := make(map[string]int64)

	// Set up output buffering:
	out := bufio.NewWriter(os.Stdout)
	defer out.Flush()

	// Create request instances, serializing them to stdout and collecting
	// counts for each kind. If applicable, only prints queries that
	// belong to this interleaved group id:
	var currentInterleavedGroup uint = 0

	enc := gob.NewEncoder(out)
	for i := 0; i < queryCount; i++ {
		q := generator.Dispatch(i)

		if currentInterleavedGroup == interleavedGenerationGroupID {
			err := enc.Encode(q)
			if err != nil {
				log.Fatal("encoder ", err)
			}
			stats[string(q.HumanLabelName())]++

			if debug == 1 {
				_, err := fmt.Fprintf(os.Stderr, "%s\n", q.HumanLabelName())
				if err != nil {
					log.Fatal(err)
				}
			} else if debug == 2 {
				_, err := fmt.Fprintf(os.Stderr, "%s\n", q.HumanDescriptionName())
				if err != nil {
					log.Fatal(err)
				}
			} else if debug >= 3 {
				_, err := fmt.Fprintf(os.Stderr, "%s\n", q.String())
				if err != nil {
					log.Fatal(err)
				}
			}
		}
		q.Release()

		currentInterleavedGroup++
		if currentInterleavedGroup == interleavedGenerationGroups {
			currentInterleavedGroup = 0
		}
	}

	// Print stats:
	keys := []string{}
	for k, _ := range stats {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		_, err := fmt.Fprintf(os.Stderr, "%s: %d points\n", k, stats[k])
		if err != nil {
			log.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"go/build"
	"reflect"
	"sort"
	"strings"
	"testing"
)

func TestCallgraph(t *testing.T) {
	ctxt := build.Default // copy
	ctxt.GOPATH = "testdata"

	const format = "{{.Caller}} --> {{.Callee}}"

	for _, test := range []struct {
		algo, format string
		tests        bool
		want         []string
	}{
		{"rta", format, false, []string{
			// rta imprecisely shows cross product of {main,main2} x {C,D}
			`pkg.main --> (pkg.C).f`,
			`pkg.main --> (pkg.D).f`,
			`pkg.main --> pkg.main2`,
			`pkg.main2 --> (pkg.C).f`,
			`pkg.main2 --> (pkg.D).f`,
		}},
		{"pta", format, false, []string{
			// pta distinguishes main->C, main2->D.  Also has a root node.
			`<root> --> pkg.init`,
			`<root> --> pkg.main`,
			`pkg.main --> (pkg.C).f`,
			`pkg.main --> pkg.main2`,
			`pkg.main2 --> (pkg.D).f`,
		}},
		// tests: main is not called.
		{"rta", format, true, []string{
			`pkg.Example --> (pkg.C).f`,
			`test$main.init --> pkg.init`,
		}},
		{"pta", format, true, []string{
			`<root> --> pkg.Example`,
			`<root> --> test$main.init`,
			`pkg.Example --> (pkg.C).f`,
			`test$main.init --> pkg.init`,
		}},
	} {
		stdout = new(bytes.Buffer)
		if err := doCallgraph(&ctxt, test.algo, test.format, test.tests, []string{"pkg"}); err != nil {
			t.Error(err)
			continue
		}

		got := sortedLines(fmt.Sprint(stdout))
		if !reflect.DeepEqual(got, test.want) {
			t.Errorf("callgraph(%q, %q, %t):\ngot:\n%s\nwant:\n%s",
				test.algo, test.format, test.tests,
				strings.Join(got, "\n"),
				strings.Join(test.want, "\n"))
		}
	}
}

func sortedLines(s string) []string {
	s = strings.TrimSpace(s)
	lines := strings.Split(s, "\n")
	sort.Strings(lines)
	return lines
}
<commit_msg>cmd/callgraph: fix build<commit_after>package main

import (
	"bytes"
	"fmt"
	"go/build"
	"reflect"
	"sort"
	"strings"
	"testing"
)

func TestCallgraph(t *testing.T) {
	ctxt := build.Default // copy
	ctxt.GOPATH = "testdata"

	const format = "{{.Caller}} --> {{.Callee}}"

	for _, test := range []struct {
		algo, format string
		tests        bool
		want         []string
	}{
		{"rta", format, false, []string{
			// rta imprecisely shows cross product of {main,main2} x {C,D}
			`pkg.main --> (pkg.C).f`,
			`pkg.main --> (pkg.D).f`,
			`pkg.main --> pkg.main2`,
			`pkg.main2 --> (pkg.C).f`,
			`pkg.main2 --> (pkg.D).f`,
		}},
		{"pta", format, false, []string{
			// pta distinguishes main->C, main2->D.  Also has a root node.
			`<root> --> pkg.init`,
			`<root> --> pkg.main`,
			`pkg.main --> (pkg.C).f`,
			`pkg.main --> pkg.main2`,
			`pkg.main2 --> (pkg.D).f`,
		}},
		// tests: main is not called.
		{"rta", format, true, []string{
			`pkg.Example --> (pkg.C).f`,
			`testmain.init --> pkg.init`,
		}},
		{"pta", format, true, []string{
			`<root> --> pkg.Example`,
			`<root> --> testmain.init`,
			`pkg.Example --> (pkg.C).f`,
			`testmain.init --> pkg.init`,
		}},
	} {
		stdout = new(bytes.Buffer)
		if err := doCallgraph(&ctxt, test.algo, test.format, test.tests, []string{"pkg"}); err != nil {
			t.Error(err)
			continue
		}

		got := sortedLines(fmt.Sprint(stdout))
		if !reflect.DeepEqual(got, test.want) {
			t.Errorf("callgraph(%q, %q, %t):\ngot:\n%s\nwant:\n%s",
				test.algo, test.format, test.tests,
				strings.Join(got, "\n"),
				strings.Join(test.want, "\n"))
		}
	}
}

func sortedLines(s string) []string {
	s = strings.TrimSpace(s)
	lines := strings.Split(s, "\n")
	sort.Strings(lines)
	return lines
}
<|endoftext|>


<commit_before>package main_test

import (
	"fmt"
	"syscall"
	"time"

	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"

	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/converger/cmd/converger/testrunner"
	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
)

var _ = Describe("Converger", func() {
	const (
		exitDuration = 4 * time.Second
	)

	var (
		etcdRunner *etcdstorerunner.ETCDClusterRunner
		bbs        *Bbs.BBS
		runner     *testrunner.ConvergerRunner

		consulRunner  *consuladapter.ClusterRunner
		consulSession *consuladapter.Session

		convergeRepeatInterval      time.Duration
		taskKickInterval            time.Duration
		expireCompletedTaskDuration time.Duration

		etcdClient storeadapter.StoreAdapter

		logger lager.Logger
	)

	SynchronizedBeforeSuite(func() []byte {
		convergerBinPath, err := Build("github.com/cloudfoundry-incubator/converger/cmd/converger", "-race")
		Ω(err).ShouldNot(HaveOccurred())
		return []byte(convergerBinPath)
	}, func(convergerBinPath []byte) {
		etcdPort := 5001 + config.GinkgoConfig.ParallelNode
		etcdCluster := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
		etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1)

		etcdClient = etcdRunner.Adapter()

		consulRunner = consuladapter.NewClusterRunner(
			9001+config.GinkgoConfig.ParallelNode*consuladapter.PortOffsetLength,
			1,
			"http",
		)

		logger = lagertest.NewTestLogger("test")

		runner = testrunner.New(string(convergerBinPath), etcdCluster, consulRunner.ConsulCluster(), "info")
	})

	SynchronizedAfterSuite(func() {
		etcdRunner.Stop()
	}, func() {
		CleanupBuildArtifacts()
	})

	BeforeEach(func() {
		etcdRunner.Start()
		consulRunner.Start()
		consulRunner.WaitUntilReady()

		consulSession = consulRunner.NewSession("a-session")
		bbs = Bbs.NewBBS(etcdClient, consulSession, "http://receptor.bogus.com", clock.NewClock(), logger)

		capacity := models.NewCellCapacity(512, 1024, 124)
		cellPresence := models.NewCellPresence("the-cell-id", "1.2.3.4", "the-zone", capacity)

		value, err := models.ToJSON(cellPresence)
		Ω(err).ShouldNot(HaveOccurred())

		_, err = consulSession.SetPresence(shared.CellSchemaPath(cellPresence.CellID), value)
		Ω(err).ShouldNot(HaveOccurred())

		convergeRepeatInterval = 500 * time.Millisecond
		taskKickInterval = convergeRepeatInterval
		expireCompletedTaskDuration = 3 * convergeRepeatInterval
	})

	AfterEach(func() {
		runner.KillWithFire()
		consulRunner.Stop()
		etcdRunner.Stop()
	})

	startConverger := func() {
		runner.Start(convergeRepeatInterval, taskKickInterval, 30*time.Minute, expireCompletedTaskDuration)
		time.Sleep(convergeRepeatInterval)
	}

	createRunningTaskWithDeadCell := func() {
		task := models.Task{
			Domain: "tests",

			TaskGuid: "task-guid",
			RootFS:   "some:rootfs",
			Action: &models.RunAction{
				Path: "cat",
				Args: []string{"/tmp/file"},
			},
		}

		err := bbs.DesireTask(logger, task)
		Ω(err).ShouldNot(HaveOccurred())

		_, err = bbs.StartTask(logger, task.TaskGuid, "dead-cell")
		Ω(err).ShouldNot(HaveOccurred())
	}

	itIsInactive := func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("does not converge the task", func() {
				Consistently(func() ([]models.Task, error) {
					return bbs.CompletedTasks(logger)
				}, 10*convergeRepeatInterval).Should(BeEmpty())
			})
		})
	}

	Context("when the converger has the lock", func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("marks the task as completed and failed", func() {
				Consistently(func() ([]models.Task, error) {
					return bbs.CompletedTasks(logger)
				}, 0.5).Should(BeEmpty())

				startConverger()

				Eventually(func() ([]models.Task, error) {
					return bbs.CompletedTasks(logger)
				}, 10*convergeRepeatInterval).Should(HaveLen(1))
			})
		})
	})

	Context("when the converger loses the lock", func() {
		BeforeEach(func() {
			startConverger()
			Eventually(runner.Session, 5*time.Second).Should(gbytes.Say("acquire-lock-succeeded"))

			consulRunner.Reset()
		})

		It("exits with an error", func() {
			Eventually(runner.Session, exitDuration).Should(Exit(1))
		})
	})

	Context("when the converger initially does not have the lock", func() {
		var otherSession *consuladapter.Session

		BeforeEach(func() {
			otherSession = consulRunner.NewSession("other-session")
			err := otherSession.AcquireLock(shared.LockSchemaPath("converge_lock"), []byte("something-else"))
			Ω(err).ShouldNot(HaveOccurred())

			startConverger()
		})

		itIsInactive()

		Describe("when the lock becomes available", func() {
			BeforeEach(func() {
				otherSession.Destroy()
				time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
			})

			Describe("when a running task with a dead cell is present", func() {
				JustBeforeEach(createRunningTaskWithDeadCell)

				It("eventually marks the task as failed", func() {
					Eventually(func() ([]models.Task, error) {
						return bbs.FailedTasks(logger)
					}, 10*convergeRepeatInterval).Should(HaveLen(1))
				})
			})
		})
	})

	Describe("signal handling", func() {
		BeforeEach(func() {
			startConverger()
		})

		Describe("when it receives SIGINT", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGINT)
				Eventually(runner.Session, exitDuration).Should(Exit(0))
			})
		})

		Describe("when it receives SIGTERM", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGTERM)
				Eventually(runner.Session, exitDuration).Should(Exit(0))
			})
		})
	})

	Context("when etcd is down", func() {
		BeforeEach(func() {
			etcdRunner.Stop()
			startConverger()
		})

		AfterEach(func() {
			etcdRunner.Start()
		})

		It("starts", func() {
			Consistently(runner.Session).ShouldNot(Exit())
		})
	})
})
<commit_msg>Remove redundant/unecessary etcd starts and stops<commit_after>package main_test

import (
	"fmt"
	"syscall"
	"time"

	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"

	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/converger/cmd/converger/testrunner"
	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
)

var _ = Describe("Converger", func() {
	const (
		exitDuration = 4 * time.Second
	)

	var (
		etcdRunner *etcdstorerunner.ETCDClusterRunner
		bbs        *Bbs.BBS
		runner     *testrunner.ConvergerRunner

		consulRunner  *consuladapter.ClusterRunner
		consulSession *consuladapter.Session

		convergeRepeatInterval      time.Duration
		taskKickInterval            time.Duration
		expireCompletedTaskDuration time.Duration

		etcdClient storeadapter.StoreAdapter

		logger lager.Logger
	)

	SynchronizedBeforeSuite(func() []byte {
		convergerBinPath, err := Build("github.com/cloudfoundry-incubator/converger/cmd/converger", "-race")
		Ω(err).ShouldNot(HaveOccurred())
		return []byte(convergerBinPath)
	}, func(convergerBinPath []byte) {
		etcdPort := 5001 + config.GinkgoConfig.ParallelNode
		etcdCluster := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
		etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1)

		etcdClient = etcdRunner.Adapter()

		consulRunner = consuladapter.NewClusterRunner(
			9001+config.GinkgoConfig.ParallelNode*consuladapter.PortOffsetLength,
			1,
			"http",
		)

		logger = lagertest.NewTestLogger("test")

		runner = testrunner.New(string(convergerBinPath), etcdCluster, consulRunner.ConsulCluster(), "info")
	})

	SynchronizedAfterSuite(func() {
	}, func() {
		CleanupBuildArtifacts()
	})

	BeforeEach(func() {
		etcdRunner.Start()
		consulRunner.Start()
		consulRunner.WaitUntilReady()

		consulSession = consulRunner.NewSession("a-session")
		bbs = Bbs.NewBBS(etcdClient, consulSession, "http://receptor.bogus.com", clock.NewClock(), logger)

		capacity := models.NewCellCapacity(512, 1024, 124)
		cellPresence := models.NewCellPresence("the-cell-id", "1.2.3.4", "the-zone", capacity)

		value, err := models.ToJSON(cellPresence)
		Ω(err).ShouldNot(HaveOccurred())

		_, err = consulSession.SetPresence(shared.CellSchemaPath(cellPresence.CellID), value)
		Ω(err).ShouldNot(HaveOccurred())

		convergeRepeatInterval = 500 * time.Millisecond
		taskKickInterval = convergeRepeatInterval
		expireCompletedTaskDuration = 3 * convergeRepeatInterval
	})

	AfterEach(func() {
		runner.KillWithFire()
		consulRunner.Stop()
		etcdRunner.Stop()
	})

	startConverger := func() {
		runner.Start(convergeRepeatInterval, taskKickInterval, 30*time.Minute, expireCompletedTaskDuration)
		time.Sleep(convergeRepeatInterval)
	}

	createRunningTaskWithDeadCell := func() {
		task := models.Task{
			Domain: "tests",

			TaskGuid: "task-guid",
			RootFS:   "some:rootfs",
			Action: &models.RunAction{
				Path: "cat",
				Args: []string{"/tmp/file"},
			},
		}

		err := bbs.DesireTask(logger, task)
		Ω(err).ShouldNot(HaveOccurred())

		_, err = bbs.StartTask(logger, task.TaskGuid, "dead-cell")
		Ω(err).ShouldNot(HaveOccurred())
	}

	itIsInactive := func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("does not converge the task", func() {
				Consistently(func() ([]models.Task, error) {
					return bbs.CompletedTasks(logger)
				}, 10*convergeRepeatInterval).Should(BeEmpty())
			})
		})
	}

	Context("when the converger has the lock", func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("marks the task as completed and failed", func() {
				Consistently(func() ([]models.Task, error) {
					return bbs.CompletedTasks(logger)
				}, 0.5).Should(BeEmpty())

				startConverger()

				Eventually(func() ([]models.Task, error) {
					return bbs.CompletedTasks(logger)
				}, 10*convergeRepeatInterval).Should(HaveLen(1))
			})
		})
	})

	Context("when the converger loses the lock", func() {
		BeforeEach(func() {
			startConverger()
			Eventually(runner.Session, 5*time.Second).Should(gbytes.Say("acquire-lock-succeeded"))

			consulRunner.Reset()
		})

		It("exits with an error", func() {
			Eventually(runner.Session, exitDuration).Should(Exit(1))
		})
	})

	Context("when the converger initially does not have the lock", func() {
		var otherSession *consuladapter.Session

		BeforeEach(func() {
			otherSession = consulRunner.NewSession("other-session")
			err := otherSession.AcquireLock(shared.LockSchemaPath("converge_lock"), []byte("something-else"))
			Ω(err).ShouldNot(HaveOccurred())

			startConverger()
		})

		itIsInactive()

		Describe("when the lock becomes available", func() {
			BeforeEach(func() {
				otherSession.Destroy()
				time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
			})

			Describe("when a running task with a dead cell is present", func() {
				JustBeforeEach(createRunningTaskWithDeadCell)

				It("eventually marks the task as failed", func() {
					Eventually(func() ([]models.Task, error) {
						return bbs.FailedTasks(logger)
					}, 10*convergeRepeatInterval).Should(HaveLen(1))
				})
			})
		})
	})

	Describe("signal handling", func() {
		BeforeEach(func() {
			startConverger()
		})

		Describe("when it receives SIGINT", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGINT)
				Eventually(runner.Session, exitDuration).Should(Exit(0))
			})
		})

		Describe("when it receives SIGTERM", func() {
			It("exits successfully", func() {
				runner.Session.Command.Process.Signal(syscall.SIGTERM)
				Eventually(runner.Session, exitDuration).Should(Exit(0))
			})
		})
	})

	Context("when etcd is down", func() {
		BeforeEach(func() {
			etcdRunner.Stop()
			startConverger()
		})

		It("starts", func() {
			Consistently(runner.Session).ShouldNot(Exit())
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"net/http/pprof"
	"os"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	n = flag.Int(
		"port-count", 5,
		"Number of sequential ports to serve metrics on, starting at 8080.",
	)
	registerProcessMetrics = flag.Bool(
		"enable-process-metrics", true,
		"Include (potentially expensive) process_* metrics.",
	)
	registerGoMetrics = flag.Bool(
		"enable-go-metrics", true,
		"Include (potentially expensive) go_* metrics.",
	)
	allowCompression = flag.Bool(
		"allow-metrics-compression", true,
		"Allow gzip compression of metrics.",
	)

	start = time.Now()
)

func main() {
	flag.Parse()

	if *registerProcessMetrics {
		registry.MustRegister(prometheus.NewProcessCollector(os.Getpid(), ""))
	}
	if *registerGoMetrics {
		registry.MustRegister(prometheus.NewGoCollector())
	}

	for i := 0; i < *n; i++ {
		mux := http.NewServeMux()
		mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
		mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
		mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
		mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
		mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
		mux.Handle("/metrics", promhttp.HandlerFor(
			registry,
			promhttp.HandlerOpts{
				DisableCompression: !*allowCompression,
			},
		))
		go http.ListenAndServe(fmt.Sprintf(":%d", 8080+i), mux)
	}

	runClient()
}
<commit_msg>Update use of prometheus.NewProcessCollector (#150)<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"net/http/pprof"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	n = flag.Int(
		"port-count", 5,
		"Number of sequential ports to serve metrics on, starting at 8080.",
	)
	registerProcessMetrics = flag.Bool(
		"enable-process-metrics", true,
		"Include (potentially expensive) process_* metrics.",
	)
	registerGoMetrics = flag.Bool(
		"enable-go-metrics", true,
		"Include (potentially expensive) go_* metrics.",
	)
	allowCompression = flag.Bool(
		"allow-metrics-compression", true,
		"Allow gzip compression of metrics.",
	)

	start = time.Now()
)

func main() {
	flag.Parse()

	if *registerProcessMetrics {
		registry.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
	}
	if *registerGoMetrics {
		registry.MustRegister(prometheus.NewGoCollector())
	}

	for i := 0; i < *n; i++ {
		mux := http.NewServeMux()
		mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
		mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
		mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
		mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
		mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
		mux.Handle("/metrics", promhttp.HandlerFor(
			registry,
			promhttp.HandlerOpts{
				DisableCompression: !*allowCompression,
			},
		))
		go http.ListenAndServe(fmt.Sprintf(":%d", 8080+i), mux)
	}

	runClient()
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"bytes"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/constraints"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/dummy"
	envtesting "launchpad.net/juju-core/environs/testing"
	"launchpad.net/juju-core/environs/tools"
	"launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
	"strings"
)

type BootstrapSuite struct {
	testing.LoggingSuite
	testing.MgoSuite
}

var _ = Suite(&BootstrapSuite{})

func (s *BootstrapSuite) SetUpSuite(c *C) {
	s.LoggingSuite.SetUpSuite(c)
	s.MgoSuite.SetUpSuite(c)
}

func (s *BootstrapSuite) SetUpTest(c *C) {
	s.LoggingSuite.SetUpTest(c)
	s.MgoSuite.SetUpTest(c)
}

func (s *BootstrapSuite) TearDownSuite(c *C) {
	s.MgoSuite.TearDownSuite(c)
	s.LoggingSuite.TearDownSuite(c)
}

func (s *BootstrapSuite) TearDownTest(c *C) {
	s.MgoSuite.TearDownTest(c)
	s.LoggingSuite.TearDownTest(c)
	dummy.Reset()
}

func (*BootstrapSuite) TestMissingEnvironment(c *C) {
	defer testing.MakeFakeHomeNoEnvironments(c, "empty").Restore()
	ctx := testing.Context(c)
	code := cmd.Main(&BootstrapCommand{}, ctx, nil)
	c.Check(code, Equals, 1)
	errStr := ctx.Stderr.(*bytes.Buffer).String()
	strippedErr := strings.Replace(errStr, "\n", "", -1)
	c.Assert(strippedErr, Matches, ".*No juju environment configuration file exists.*")
}

func (s *BootstrapSuite) TestBootstrapState(c *C) {
	uploadTools = mockUploadTools
	defer func() { uploadTools = tools.Upload }()
	for i, test := range bootstrapTests {
		c.Logf("\ntest %d: %s", i, test.info)
		test.run(c)
	}
}

type bootstrapTest struct {
	info string
	// binary version string used to set version.Current
	version string
	args    []string
	err     string
	// binary version strings for expected tools; if set, no default tools
	// will be uploaded before running the test.
	uploads     []string
	constraints constraints.Value
}

func (test bootstrapTest) run(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	dummy.Reset()
	env, err := environs.NewFromName("peckham")
	c.Assert(err, IsNil)
	envtesting.RemoveAllTools(c, env)

	if test.version != "" {
		origVersion := version.Current
		version.Current = version.MustParseBinary(test.version)
		defer func() { version.Current = origVersion }()
	}
	uploadCount := len(test.uploads)
	if uploadCount == 0 {
		usefulVersion := version.Current
		usefulVersion.Series = env.Config().DefaultSeries()
		envtesting.UploadFakeToolsVersion(c, env.Storage(), usefulVersion)
	}

	// Run command and check for uploads.
	opc, errc := runCommand(new(BootstrapCommand), test.args...)
	if uploadCount > 0 {
		for i := 0; i < uploadCount; i++ {
			c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham")
		}
		list, err := environs.FindAvailableTools(env, version.Current.Major)
		c.Check(err, IsNil)
		c.Logf("found: " + list.String())
		urls := list.URLs()
		c.Check(urls, HasLen, len(test.uploads))
		for _, v := range test.uploads {
			c.Logf("seeking: " + v)
			vers := version.MustParseBinary(v)
			_, found := urls[vers]
			c.Check(found, Equals, true)
		}
	}

	// Check for remaining operations/errors.
	if test.err != "" {
		c.Check(<-errc, ErrorMatches, test.err)
		return
	}
	if !c.Check(<-errc, IsNil) {
		return
	}
	opBootstrap := (<-opc).(dummy.OpBootstrap)
	c.Check(opBootstrap.Env, Equals, "peckham")
	c.Check(opBootstrap.Constraints, DeepEquals, test.constraints)

	// Check a CA cert/key was generated by reloading the environment.
	env, err = environs.NewFromName("peckham")
	c.Assert(err, IsNil)
	_, hasCert := env.Config().CACert()
	c.Check(hasCert, Equals, true)
	_, hasKey := env.Config().CAPrivateKey()
	c.Check(hasKey, Equals, true)
}

var bootstrapTests = []bootstrapTest{{
	info: "no args, no error, no uploads, no constraints",
}, {
	info: "bad arg",
	args: []string{"twiddle"},
	err:  `unrecognized args: \["twiddle"\]`,
}, {
	info: "bad --constraints",
	args: []string{"--constraints", "bad=wrong"},
	err:  `invalid value "bad=wrong" for flag --constraints: unknown constraint "bad"`,
}, {
	info: "bad --series",
	args: []string{"--series", "bad1"},
	err:  `invalid value "bad1" for flag --series: invalid series name "bad1"`,
}, {
	info: "lonely --series",
	args: []string{"--series", "fine"},
	err:  `--series requires --upload-tools`,
}, {
	info: "bad environment",
	args: []string{"-e", "brokenenv"},
	err:  `environment configuration has no admin-secret`,
}, {
	info:        "constraints",
	args:        []string{"--constraints", "mem=4G cpu-cores=4"},
	constraints: constraints.MustParse("mem=4G cpu-cores=4"),
}, {
	info:    "--upload-tools picks all reasonable series",
	version: "1.2.3-hostseries-hostarch",
	args:    []string{"--upload-tools"},
	uploads: []string{
		"1.2.3.1-hostseries-hostarch",    // from version.Current
		"1.2.3.1-defaultseries-hostarch", // from env.Config().DefaultSeries()
		"1.2.3.1-precise-hostarch",       // from environs/config.DefaultSeries
	},
}, {
	info:    "--upload-tools only uploads each file once",
	version: "1.2.3-precise-hostarch",
	args:    []string{"--upload-tools"},
	uploads: []string{
		"1.2.3.1-defaultseries-hostarch",
		"1.2.3.1-precise-hostarch",
	},
}, {
	info:    "--upload-tools accepts specific series even if they're crazy",
	version: "1.2.3-hostseries-hostarch",
	args:    []string{"--upload-tools", "--series", "ping,ping,pong"},
	uploads: []string{
		"1.2.3.1-hostseries-hostarch",
		"1.2.3.1-ping-hostarch",
		"1.2.3.1-pong-hostarch",
	},
	err: "no matching tools available",
}, {
	info:    "--upload-tools always bumps build number",
	version: "1.2.3.4-defaultseries-hostarch",
	args:    []string{"--upload-tools"},
	uploads: []string{
		"1.2.3.5-defaultseries-hostarch",
		"1.2.3.5-precise-hostarch",
	},
}}
<commit_msg>cmd/juju: revert bogus change<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"bytes"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/constraints"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/dummy"
	envtesting "launchpad.net/juju-core/environs/testing"
	"launchpad.net/juju-core/environs/tools"
	"launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
	"strings"
)

type BootstrapSuite struct {
	testing.LoggingSuite
	testing.MgoSuite
}

var _ = Suite(&BootstrapSuite{})

func (s *BootstrapSuite) SetUpSuite(c *C) {
	s.LoggingSuite.SetUpSuite(c)
	s.MgoSuite.SetUpSuite(c)
}

func (s *BootstrapSuite) SetUpTest(c *C) {
	s.LoggingSuite.SetUpTest(c)
	s.MgoSuite.SetUpTest(c)
}

func (s *BootstrapSuite) TearDownSuite(c *C) {
	s.MgoSuite.TearDownSuite(c)
	s.LoggingSuite.TearDownSuite(c)
}

func (s *BootstrapSuite) TearDownTest(c *C) {
	s.MgoSuite.TearDownTest(c)
	s.LoggingSuite.TearDownTest(c)
	dummy.Reset()
}

func (*BootstrapSuite) TestMissingEnvironment(c *C) {
	defer testing.MakeFakeHomeNoEnvironments(c, "empty").Restore()
	ctx := testing.Context(c)
	code := cmd.Main(&BootstrapCommand{}, ctx, nil)
	c.Check(code, Equals, 1)
	errStr := ctx.Stderr.(*bytes.Buffer).String()
	strippedErr := strings.Replace(errStr, "\n", "", -1)
	c.Assert(strippedErr, Matches, ".*No juju environment configuration file exists.*")
}

func (s *BootstrapSuite) TestTest(c *C) {
	uploadTools = mockUploadTools
	defer func() { uploadTools = tools.Upload }()
	for i, test := range bootstrapTests {
		c.Logf("\ntest %d: %s", i, test.info)
		test.run(c)
	}
}

type bootstrapTest struct {
	info string
	// binary version string used to set version.Current
	version string
	args    []string
	err     string
	// binary version strings for expected tools; if set, no default tools
	// will be uploaded before running the test.
	uploads     []string
	constraints constraints.Value
}

func (test bootstrapTest) run(c *C) {
	defer testing.MakeFakeHome(c, envConfig).Restore()
	dummy.Reset()
	env, err := environs.NewFromName("peckham")
	c.Assert(err, IsNil)
	envtesting.RemoveAllTools(c, env)

	if test.version != "" {
		origVersion := version.Current
		version.Current = version.MustParseBinary(test.version)
		defer func() { version.Current = origVersion }()
	}
	uploadCount := len(test.uploads)
	if uploadCount == 0 {
		usefulVersion := version.Current
		usefulVersion.Series = env.Config().DefaultSeries()
		envtesting.UploadFakeToolsVersion(c, env.Storage(), usefulVersion)
	}

	// Run command and check for uploads.
	opc, errc := runCommand(new(BootstrapCommand), test.args...)
	if uploadCount > 0 {
		for i := 0; i < uploadCount; i++ {
			c.Check((<-opc).(dummy.OpPutFile).Env, Equals, "peckham")
		}
		list, err := environs.FindAvailableTools(env, version.Current.Major)
		c.Check(err, IsNil)
		c.Logf("found: " + list.String())
		urls := list.URLs()
		c.Check(urls, HasLen, len(test.uploads))
		for _, v := range test.uploads {
			c.Logf("seeking: " + v)
			vers := version.MustParseBinary(v)
			_, found := urls[vers]
			c.Check(found, Equals, true)
		}
	}

	// Check for remaining operations/errors.
	if test.err != "" {
		c.Check(<-errc, ErrorMatches, test.err)
		return
	}
	if !c.Check(<-errc, IsNil) {
		return
	}
	opBootstrap := (<-opc).(dummy.OpBootstrap)
	c.Check(opBootstrap.Env, Equals, "peckham")
	c.Check(opBootstrap.Constraints, DeepEquals, test.constraints)

	// Check a CA cert/key was generated by reloading the environment.
	env, err = environs.NewFromName("peckham")
	c.Assert(err, IsNil)
	_, hasCert := env.Config().CACert()
	c.Check(hasCert, Equals, true)
	_, hasKey := env.Config().CAPrivateKey()
	c.Check(hasKey, Equals, true)
}

var bootstrapTests = []bootstrapTest{{
	info: "no args, no error, no uploads, no constraints",
}, {
	info: "bad arg",
	args: []string{"twiddle"},
	err:  `unrecognized args: \["twiddle"\]`,
}, {
	info: "bad --constraints",
	args: []string{"--constraints", "bad=wrong"},
	err:  `invalid value "bad=wrong" for flag --constraints: unknown constraint "bad"`,
}, {
	info: "bad --series",
	args: []string{"--series", "bad1"},
	err:  `invalid value "bad1" for flag --series: invalid series name "bad1"`,
}, {
	info: "lonely --series",
	args: []string{"--series", "fine"},
	err:  `--series requires --upload-tools`,
}, {
	info: "bad environment",
	args: []string{"-e", "brokenenv"},
	err:  `environment configuration has no admin-secret`,
}, {
	info:        "constraints",
	args:        []string{"--constraints", "mem=4G cpu-cores=4"},
	constraints: constraints.MustParse("mem=4G cpu-cores=4"),
}, {
	info:    "--upload-tools picks all reasonable series",
	version: "1.2.3-hostseries-hostarch",
	args:    []string{"--upload-tools"},
	uploads: []string{
		"1.2.3.1-hostseries-hostarch",    // from version.Current
		"1.2.3.1-defaultseries-hostarch", // from env.Config().DefaultSeries()
		"1.2.3.1-precise-hostarch",       // from environs/config.DefaultSeries
	},
}, {
	info:    "--upload-tools only uploads each file once",
	version: "1.2.3-precise-hostarch",
	args:    []string{"--upload-tools"},
	uploads: []string{
		"1.2.3.1-defaultseries-hostarch",
		"1.2.3.1-precise-hostarch",
	},
}, {
	info:    "--upload-tools accepts specific series even if they're crazy",
	version: "1.2.3-hostseries-hostarch",
	args:    []string{"--upload-tools", "--series", "ping,ping,pong"},
	uploads: []string{
		"1.2.3.1-hostseries-hostarch",
		"1.2.3.1-ping-hostarch",
		"1.2.3.1-pong-hostarch",
	},
	err: "no matching tools available",
}, {
	info:    "--upload-tools always bumps build number",
	version: "1.2.3.4-defaultseries-hostarch",
	args:    []string{"--upload-tools"},
	uploads: []string{
		"1.2.3.5-defaultseries-hostarch",
		"1.2.3.5-precise-hostarch",
	},
}}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"strconv"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/inflights"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/sla"
	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

// /topics/:appid/:topic/:ver?group=xx&&reset=<newest|oldest>&ack=1&use=<dead|retry>
func (this *Gateway) subHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	var (
		topic      string
		ver        string
		myAppid    string
		hisAppid   string
		reset      string
		group      string
		shadow     string
		partition  string
		partitionN int = -1
		offset     string
		offsetN    int64 = -1
		ack        string
		delayedAck bool
		err        error
	)

	if options.EnableClientStats {
		this.clientStates.RegisterSubClient(r)
	}

	query := r.URL.Query()
	group = query.Get("group")
	reset = query.Get("reset")
	if !validateGroupName(group) {
		this.writeBadRequest(w, "illegal group")
		return
	}

	ver = params.ByName(UrlParamVersion)
	topic = params.ByName(UrlParamTopic)
	hisAppid = params.ByName(UrlParamAppid)
	shadow = query.Get("use")
	ack = query.Get("ack")
	myAppid = r.Header.Get(HttpHeaderAppid)
	if r.Header.Get("Connection") == "close" {
		// sub should use keep-alive
		log.Warn("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} not keep-alive",
			myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group)
	}

	if err = manager.Default.AuthSub(myAppid, r.Header.Get(HttpHeaderSubkey),
		hisAppid, topic); err != nil {
		log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
			myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)

		this.writeAuthFailure(w, err)
		return
	}

	delayedAck = ack == "1"
	if delayedAck {
		partition = r.Header.Get(HttpHeaderPartition)
		offset = r.Header.Get(HttpHeaderOffset)
		if partition != "" && offset != "" {
			// convert partition and offset to int

			offsetN, err = strconv.ParseInt(offset, 10, 64)
			if err != nil || offsetN < 0 {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} offset:%s",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, offset)

				this.writeBadRequest(w, "ack with bad offset")
				return
			}
			partitionN, err = strconv.Atoi(partition)
			if err != nil || partitionN < 0 {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} partition:%s",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, partition)

				this.writeBadRequest(w, "ack with bad partition")
				return
			}
		}
	}

	log.Debug("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s ack:%s, partition:%s, offset:%s}",
		myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver,
		group, ack, partition, offset)

	var rawTopic string
	if shadow != "" {
		if !sla.ValidateShadowName(shadow) {
			log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s use:%s} invalid shadow name",
				myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, shadow)

			this.writeBadRequest(w, "invalid shadow name")
			return
		}

		if !manager.Default.IsShadowedTopic(hisAppid, topic, ver, group) {
			log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s use:%s} not a shadowed topic",
				myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, shadow)

			this.writeBadRequest(w, "register shadow first")
			return
		}

		rawTopic = meta.ShadowTopic(shadow, myAppid, hisAppid, topic, ver, group)
	} else {
		rawTopic = meta.KafkaTopic(hisAppid, topic, ver)
	}

	// pick a consumer from the consumer group
	cluster, found := manager.Default.LookupCluster(hisAppid)
	if !found {
		log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} cluster not found",
			myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group)

		this.writeBadRequest(w, "invalid appid")
		return
	}

	fetcher, err := store.DefaultSubStore.Fetch(cluster, rawTopic,
		myAppid+"."+group, r.RemoteAddr, reset)
	if err != nil {
		log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
			myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)

		this.writeBadRequest(w, err.Error())
		return
	}

	if delayedAck && partitionN >= 0 && offsetN >= 0 {
		if bury := r.Header.Get(HttpHeaderMsgBury); bury != "" {
			// bury message to shadow topic and pump next message
			if !sla.ValidateShadowName(bury) {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} illegal bury: %s",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err, bury)

				this.writeBadRequest(w, "illegal bury")
				return
			}

			msg, err := inflights.Default.LandX(cluster, rawTopic, group, partition, offsetN)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)

				// will deadloop? FIXME
				this.writeBadRequest(w, err.Error())
				return
			}

			shadowTopic := meta.ShadowTopic(bury, myAppid, hisAppid, topic, ver, group)
			_, _, err = store.DefaultPubStore.SyncPub(cluster, shadowTopic, nil, msg)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)

				this.writeErrorResponse(w, err.Error(), http.StatusInternalServerError)
				return
			}

			// skip this message in the master topic
			if err = fetcher.CommitUpto(&sarama.ConsumerMessage{
				Topic:     rawTopic,
				Partition: int32(partitionN),
				Offset:    offsetN,
			}); err != nil {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)
			}
		} else {
			// what if shutdown kateway now?
			// the commit will be ok, and when pumpMessages, the conn will get http.StatusNoContent
			if err = fetcher.CommitUpto(&sarama.ConsumerMessage{
				Topic:     rawTopic,
				Partition: int32(partitionN),
				Offset:    offsetN,
			}); err != nil {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)
			}

			log.Debug("land {G:%s, T:%s, P:%s, O:%s}", group, rawTopic, partition, offset)
			if err = inflights.Default.Land(cluster, rawTopic, group, partition, offsetN); err != nil {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)
			}
		}

	}

	err = this.pumpMessages(w, fetcher, myAppid, hisAppid, cluster, rawTopic, ver, group, delayedAck)
	if err != nil {
		// e,g. broken pipe, io timeout, client gone
		log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
			myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)

		this.writeErrorResponse(w, err.Error(), http.StatusInternalServerError)

		if err = fetcher.Close(); err != nil {
			log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
				myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)
		}
	}

	return
}

func (this *Gateway) pumpMessages(w http.ResponseWriter, fetcher store.Fetcher,
	myAppid, hisAppid, cluster, topic, ver, group string, delayedAck bool) (err error) {
	clientGoneCh := w.(http.CloseNotifier).CloseNotify()

	select {
	case <-clientGoneCh:
		// FIXME access log will not be able to record this behavior
		err = ErrClientGone

	case <-this.shutdownCh:
		w.WriteHeader(http.StatusNoContent)
		w.Write([]byte{})

	case <-this.timer.After(options.SubTimeout):
		w.WriteHeader(http.StatusNoContent)
		w.Write([]byte{}) // without this, client cant get response

	case msg := <-fetcher.Messages():
		partition := strconv.FormatInt(int64(msg.Partition), 10)

		if delayedAck {
			log.Debug("take off {G:%s, T:%s, P:%d, O:%d}", group, msg.Topic, msg.Partition, msg.Offset)
			if err = inflights.Default.TakeOff(cluster, topic, group,
				partition, msg.Offset, msg.Value); err != nil {
				// keep consuming the same message, offset never move ahead
				return
			}
		}

		w.Header().Set(HttpHeaderMsgKey, string(msg.Key))
		w.Header().Set(HttpHeaderPartition, partition)
		w.Header().Set(HttpHeaderOffset, strconv.FormatInt(msg.Offset, 10))

		// TODO when remote close silently, the write still ok
		// which will lead to msg lost for sub
		if _, err = w.Write(msg.Value); err != nil {
			return
		}

		if !delayedAck {
			log.Debug("commit offset {G:%s, T:%s, P:%d, O:%d}", group, msg.Topic, msg.Partition, msg.Offset)
			if err = fetcher.CommitUpto(msg); err != nil {
				log.Error("commit offset {T:%s, P:%d, O:%d}: %v", msg.Topic, msg.Partition, msg.Offset, err)
			}
		}

		this.subMetrics.ConsumeOk(myAppid, topic, ver)
		this.subMetrics.ConsumedOk(hisAppid, topic, ver)

	case err = <-fetcher.Errors():
		// e,g. consume a non-existent topic
		// e,g. conn with broker is broken
	}

	return

}
<commit_msg>add comment<commit_after>package main

import (
	"net/http"
	"strconv"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/inflights"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/sla"
	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

// /topics/:appid/:topic/:ver?group=xx&&reset=<newest|oldest>&ack=1&use=<dead|retry>
func (this *Gateway) subHandler(w http.ResponseWriter, r *http.Request,
	params httprouter.Params) {
	var (
		topic      string
		ver        string
		myAppid    string
		hisAppid   string
		reset      string
		group      string
		shadow     string
		partition  string
		partitionN int = -1
		offset     string
		offsetN    int64 = -1
		ack        string
		delayedAck bool
		err        error
	)

	if options.EnableClientStats {
		this.clientStates.RegisterSubClient(r)
	}

	query := r.URL.Query()
	group = query.Get("group")
	reset = query.Get("reset")
	if !validateGroupName(group) {
		this.writeBadRequest(w, "illegal group")
		return
	}

	ver = params.ByName(UrlParamVersion)
	topic = params.ByName(UrlParamTopic)
	hisAppid = params.ByName(UrlParamAppid)
	shadow = query.Get("use")
	ack = query.Get("ack")
	myAppid = r.Header.Get(HttpHeaderAppid)
	if r.Header.Get("Connection") == "close" {
		// sub should use keep-alive
		log.Warn("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} not keep-alive",
			myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group)
	}

	if err = manager.Default.AuthSub(myAppid, r.Header.Get(HttpHeaderSubkey),
		hisAppid, topic); err != nil {
		log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
			myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)

		this.writeAuthFailure(w, err)
		return
	}

	delayedAck = ack == "1"
	if delayedAck {
		// get the partitionN and offsetN from client header
		partition = r.Header.Get(HttpHeaderPartition)
		offset = r.Header.Get(HttpHeaderOffset)
		if partition != "" && offset != "" {
			// convert partition and offset to int

			offsetN, err = strconv.ParseInt(offset, 10, 64)
			if err != nil || offsetN < 0 {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} offset:%s",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, offset)

				this.writeBadRequest(w, "ack with bad offset")
				return
			}
			partitionN, err = strconv.Atoi(partition)
			if err != nil || partitionN < 0 {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} partition:%s",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, partition)

				this.writeBadRequest(w, "ack with bad partition")
				return
			}
		}
	}

	log.Debug("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s ack:%s, partition:%s, offset:%s}",
		myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver,
		group, ack, partition, offset)

	var rawTopic string
	if shadow != "" {
		if !sla.ValidateShadowName(shadow) {
			log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s use:%s} invalid shadow name",
				myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, shadow)

			this.writeBadRequest(w, "invalid shadow name")
			return
		}

		if !manager.Default.IsShadowedTopic(hisAppid, topic, ver, group) {
			log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s use:%s} not a shadowed topic",
				myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, shadow)

			this.writeBadRequest(w, "register shadow first")
			return
		}

		rawTopic = meta.ShadowTopic(shadow, myAppid, hisAppid, topic, ver, group)
	} else {
		rawTopic = meta.KafkaTopic(hisAppid, topic, ver)
	}

	// pick a consumer from the consumer group
	cluster, found := manager.Default.LookupCluster(hisAppid)
	if !found {
		log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} cluster not found",
			myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group)

		this.writeBadRequest(w, "invalid appid")
		return
	}

	fetcher, err := store.DefaultSubStore.Fetch(cluster, rawTopic,
		myAppid+"."+group, r.RemoteAddr, reset)
	if err != nil {
		log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
			myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)

		this.writeBadRequest(w, err.Error())
		return
	}

	if delayedAck && partitionN >= 0 && offsetN >= 0 {
		if bury := r.Header.Get(HttpHeaderMsgBury); bury != "" {
			// bury message to shadow topic and pump next message
			if !sla.ValidateShadowName(bury) {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} illegal bury: %s",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err, bury)

				this.writeBadRequest(w, "illegal bury")
				return
			}

			msg, err := inflights.Default.LandX(cluster, rawTopic, group, partition, offsetN)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)

				// will deadloop? FIXME
				this.writeBadRequest(w, err.Error())
				return
			}

			shadowTopic := meta.ShadowTopic(bury, myAppid, hisAppid, topic, ver, group)
			_, _, err = store.DefaultPubStore.SyncPub(cluster, shadowTopic, nil, msg)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)

				this.writeErrorResponse(w, err.Error(), http.StatusInternalServerError)
				return
			}

			// skip this message in the master topic
			if err = fetcher.CommitUpto(&sarama.ConsumerMessage{
				Topic:     rawTopic,
				Partition: int32(partitionN),
				Offset:    offsetN,
			}); err != nil {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)
			}
		} else {
			// what if shutdown kateway now?
			// the commit will be ok, and when pumpMessages, the conn will get http.StatusNoContent
			if err = fetcher.CommitUpto(&sarama.ConsumerMessage{
				Topic:     rawTopic,
				Partition: int32(partitionN),
				Offset:    offsetN,
			}); err != nil {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)
			}

			log.Debug("land {G:%s, T:%s, P:%s, O:%s}", group, rawTopic, partition, offset)
			if err = inflights.Default.Land(cluster, rawTopic, group, partition, offsetN); err != nil {
				log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
					myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)
			}
		}

	}

	err = this.pumpMessages(w, fetcher, myAppid, hisAppid, cluster, rawTopic, ver, group, delayedAck)
	if err != nil {
		// e,g. broken pipe, io timeout, client gone
		log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
			myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)

		this.writeErrorResponse(w, err.Error(), http.StatusInternalServerError)

		if err = fetcher.Close(); err != nil {
			log.Error("sub[%s] %s(%s): {app:%s, topic:%s, ver:%s, group:%s} %v",
				myAppid, r.RemoteAddr, getHttpRemoteIp(r), hisAppid, topic, ver, group, err)
		}
	}

	return
}

func (this *Gateway) pumpMessages(w http.ResponseWriter, fetcher store.Fetcher,
	myAppid, hisAppid, cluster, topic, ver, group string, delayedAck bool) (err error) {
	clientGoneCh := w.(http.CloseNotifier).CloseNotify()

	select {
	case <-clientGoneCh:
		// FIXME access log will not be able to record this behavior
		err = ErrClientGone

	case <-this.shutdownCh:
		w.WriteHeader(http.StatusNoContent)
		w.Write([]byte{})

	case <-this.timer.After(options.SubTimeout):
		w.WriteHeader(http.StatusNoContent)
		w.Write([]byte{}) // without this, client cant get response

	case msg := <-fetcher.Messages():
		partition := strconv.FormatInt(int64(msg.Partition), 10)

		if delayedAck {
			log.Debug("take off {G:%s, T:%s, P:%d, O:%d}", group, msg.Topic, msg.Partition, msg.Offset)
			if err = inflights.Default.TakeOff(cluster, topic, group,
				partition, msg.Offset, msg.Value); err != nil {
				// keep consuming the same message, offset never move ahead
				return
			}
		}

		w.Header().Set(HttpHeaderMsgKey, string(msg.Key))
		w.Header().Set(HttpHeaderPartition, partition)
		w.Header().Set(HttpHeaderOffset, strconv.FormatInt(msg.Offset, 10))

		// TODO when remote close silently, the write still ok
		// which will lead to msg lost for sub
		if _, err = w.Write(msg.Value); err != nil {
			return
		}

		if !delayedAck {
			log.Debug("commit offset {G:%s, T:%s, P:%d, O:%d}", group, msg.Topic, msg.Partition, msg.Offset)
			if err = fetcher.CommitUpto(msg); err != nil {
				log.Error("commit offset {T:%s, P:%d, O:%d}: %v", msg.Topic, msg.Partition, msg.Offset, err)
			}
		}

		this.subMetrics.ConsumeOk(myAppid, topic, ver)
		this.subMetrics.ConsumedOk(hisAppid, topic, ver)

	case err = <-fetcher.Errors():
		// e,g. consume a non-existent topic
		// e,g. conn with broker is broken
	}

	return

}
<|endoftext|>


<commit_before>// Copyright © 2017 Pennock Tech, LLC.
// All rights reserved, except as granted under license.
// Licensed per file LICENSE.txt

package main // import "go.pennock.tech/ocsprenewer/cmd/ocsprenewer"

import (
	"runtime"
)

var ProjectName = "OCSP Renewer"
var Version = "0.1.7"

func showVersion() {
	stdout("%s version %s\n", ProjectName, Version)
	stdout("%s: Golang: Runtime: %s\n", ProjectName, runtime.Version())
}

// We expect Version to be overridable at the linker, perhaps with git
// information, so it might be more than just a tuple of digits joined with
// dots.
// In HTTP, any "token" can be used as the "product-version"
func httpVersion(ver string) string {
	for i := 0; i < len(ver); i++ {
		// see comment after func with relevant grammar
		// the RFC7230/RFC2616 approaches should be identical in result, the 2616 is simpler to code here
		if ver[i] > 126 {
			// DEL or bit 7 set, so UTF-8 sequence, non-CHAR
			return ver[:i]
		}
		switch ver[i] {
		case 0, '(', ')', '<', '>', '@', ',', ';', ':', '\\', '"', '/', '[', ']', '?', '=', '{', '}', ' ', '\t':
			return ver[:i]
		}
	}
	return ver
}

/* RFC 2616 or 7230/7231

2616:
       token          = 1*<any CHAR except CTLs or separators>
       separators     = "(" | ")" | "<" | ">" | "@"
                      | "," | ";" | ":" | "\" | <">
                      | "/" | "[" | "]" | "?" | "="
                      | "{" | "}" | SP | HT
       CTL            = <any US-ASCII control character
                        (octets 0 - 31) and DEL (127)>
       CHAR           = <any US-ASCII character (octets 0 - 127)>

7230:
   Most HTTP header field values are defined using common syntax
   components (token, quoted-string, and comment) separated by
   whitespace or specific delimiting characters.  Delimiters are chosen
   from the set of US-ASCII visual characters not allowed in a token
   (DQUOTE and "(),/:;<=>?@[\]{}").

     token          = 1*tchar

     tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
                    / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
                    / DIGIT / ALPHA
                    ; any VCHAR, except delimiters
*/
<commit_msg>Version bump: 0.1.8<commit_after>// Copyright © 2017 Pennock Tech, LLC.
// All rights reserved, except as granted under license.
// Licensed per file LICENSE.txt

package main // import "go.pennock.tech/ocsprenewer/cmd/ocsprenewer"

import (
	"runtime"
)

var ProjectName = "OCSP Renewer"
var Version = "0.1.8"

func showVersion() {
	stdout("%s version %s\n", ProjectName, Version)
	stdout("%s: Golang: Runtime: %s\n", ProjectName, runtime.Version())
}

// We expect Version to be overridable at the linker, perhaps with git
// information, so it might be more than just a tuple of digits joined with
// dots.
// In HTTP, any "token" can be used as the "product-version"
func httpVersion(ver string) string {
	for i := 0; i < len(ver); i++ {
		// see comment after func with relevant grammar
		// the RFC7230/RFC2616 approaches should be identical in result, the 2616 is simpler to code here
		if ver[i] > 126 {
			// DEL or bit 7 set, so UTF-8 sequence, non-CHAR
			return ver[:i]
		}
		switch ver[i] {
		case 0, '(', ')', '<', '>', '@', ',', ';', ':', '\\', '"', '/', '[', ']', '?', '=', '{', '}', ' ', '\t':
			return ver[:i]
		}
	}
	return ver
}

/* RFC 2616 or 7230/7231

2616:
       token          = 1*<any CHAR except CTLs or separators>
       separators     = "(" | ")" | "<" | ">" | "@"
                      | "," | ";" | ":" | "\" | <">
                      | "/" | "[" | "]" | "?" | "="
                      | "{" | "}" | SP | HT
       CTL            = <any US-ASCII control character
                        (octets 0 - 31) and DEL (127)>
       CHAR           = <any US-ASCII character (octets 0 - 127)>

7230:
   Most HTTP header field values are defined using common syntax
   components (token, quoted-string, and comment) separated by
   whitespace or specific delimiting characters.  Delimiters are chosen
   from the set of US-ASCII visual characters not allowed in a token
   (DQUOTE and "(),/:;<=>?@[\]{}").

     token          = 1*tchar

     tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
                    / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
                    / DIGIT / ALPHA
                    ; any VCHAR, except delimiters
*/
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/nuclio/nuclio/cmd/processor/app"
)

func run() error {

	processor, err := app.NewProcessor("test/e2e/config/nuclio.yaml")
	if err != nil {
		return err
	}

	return processor.Start()
}

func main() {

	if err := run(); err != nil {
		fmt.Printf("Failed to run processor: %s", err)

		os.Exit(1)
	}

	os.Exit(0)
}
<commit_msg>Get config path in args (#23)<commit_after>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/nuclio/nuclio/cmd/processor/app"
)

func run() error {
	configPath := flag.String("config", "", "Path of configuration file")
	flag.Parse()

	processor, err := app.NewProcessor(*configPath)
	if err != nil {
		return err
	}

	return processor.Start()
}

func main() {

	if err := run(); err != nil {
		fmt.Printf("Failed to run processor: %s", err)

		os.Exit(1)
	}

	os.Exit(0)
}
<|endoftext|>


<commit_before>// Copyright 2014 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

// This command does a one-time setup of the RabbitMQ exchange and the Activity
// Monitor queue, suitable for setting up a dev environment or Travis.

import (
	"flag"
	"fmt"

	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/streadway/amqp"

	"github.com/letsencrypt/boulder/cmd"
)

var server = flag.String("server", "", "RabbitMQ Server URL")

func init() {
	flag.Parse()
}

// Constants for AMQP
const (
	monitorQueueName = "Monitor"
	amqpExchange     = "boulder"
	amqpExchangeType = "topic"
	amqpInternal     = false
	amqpDurable      = false
	amqpDeleteUnused = false
	amqpExclusive    = false
	amqpNoWait       = false
)

func main() {
	server := *server
	conn, err := amqp.Dial(server)
	cmd.FailOnError(err, "Could not connect to AMQP")
	ch, err := conn.Channel()
	cmd.FailOnError(err, "Could not connect to AMQP")

	err = ch.ExchangeDeclare(
		amqpExchange,
		amqpExchangeType,
		amqpDurable,
		amqpDeleteUnused,
		amqpInternal,
		amqpNoWait,
		nil)
	cmd.FailOnError(err, "Declaring exchange")

	_, err = ch.QueueDeclare(
		monitorQueueName,
		amqpDurable,
		amqpDeleteUnused,
		amqpExclusive,
		amqpNoWait,
		nil)
	if err != nil {
		cmd.FailOnError(err, "Could not declare queue")
	}

	routingKey := "#" //wildcard

	err = ch.QueueBind(
		monitorQueueName,
		routingKey,
		amqpExchange,
		false,
		nil)
	if err != nil {
		txt := fmt.Sprintf("Could not bind to queue [%s]. NOTE: You may need to delete %s to re-trigger the bind attempt after fixing permissions, or manually bind the queue to %s.", monitorQueueName, monitorQueueName, routingKey)
		cmd.FailOnError(err, txt)
	}
}
<commit_msg>Declare exchange durably<commit_after>// Copyright 2014 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

// This command does a one-time setup of the RabbitMQ exchange and the Activity
// Monitor queue, suitable for setting up a dev environment or Travis.

import (
	"flag"
	"fmt"

	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/streadway/amqp"

	"github.com/letsencrypt/boulder/cmd"
)

var server = flag.String("server", "", "RabbitMQ Server URL")

func init() {
	flag.Parse()
}

// Constants for AMQP
const (
	monitorQueueName    = "Monitor"
	amqpExchange        = "boulder"
	amqpExchangeType    = "topic"
	amqpInternal        = false
	amqpExchangeDurable = true
	amqpQueueDurable    = false
	amqpDeleteUnused    = false
	amqpExclusive       = false
	amqpNoWait          = false
)

func main() {
	server := *server
	conn, err := amqp.Dial(server)
	cmd.FailOnError(err, "Could not connect to AMQP")
	ch, err := conn.Channel()
	cmd.FailOnError(err, "Could not connect to AMQP")

	err = ch.ExchangeDeclare(
		amqpExchange,
		amqpExchangeType,
		amqpExchangeDurable,
		amqpDeleteUnused,
		amqpInternal,
		amqpNoWait,
		nil)
	cmd.FailOnError(err, "Declaring exchange")

	_, err = ch.QueueDeclare(
		monitorQueueName,
		amqpQueueDurable,
		amqpDeleteUnused,
		amqpExclusive,
		amqpNoWait,
		nil)
	if err != nil {
		cmd.FailOnError(err, "Could not declare queue")
	}

	routingKey := "#" //wildcard

	err = ch.QueueBind(
		monitorQueueName,
		routingKey,
		amqpExchange,
		false,
		nil)
	if err != nil {
		txt := fmt.Sprintf("Could not bind to queue [%s]. NOTE: You may need to delete %s to re-trigger the bind attempt after fixing permissions, or manually bind the queue to %s.", monitorQueueName, monitorQueueName, routingKey)
		cmd.FailOnError(err, txt)
	}
}
<|endoftext|>