<commit_before>// Copyright 2012-present Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

// TermsAggregation is a multi-bucket value source based aggregation
// where buckets are dynamically built - one per unique value.
//
// See: http://www.elasticsearch.org/guide/en/elasticsearch/reference/7.0/search-aggregations-bucket-terms-aggregation.html
type TermsAggregation struct {
	field           string
	script          *Script
	missing         interface{}
	subAggregations map[string]Aggregation
	meta            map[string]interface{}

	size                  *int
	shardSize             *int
	requiredSize          *int
	minDocCount           *int
	shardMinDocCount      *int
	valueType             string
	includeExclude        *TermsAggregationIncludeExclude
	executionHint         string
	collectionMode        string
	showTermDocCountError *bool
	order                 []TermsOrder
}

func NewTermsAggregation() *TermsAggregation {
	return &TermsAggregation{
		subAggregations: make(map[string]Aggregation, 0),
	}
}

func (a *TermsAggregation) Field(field string) *TermsAggregation {
	a.field = field
	return a
}

func (a *TermsAggregation) Script(script *Script) *TermsAggregation {
	a.script = script
	return a
}

// Missing configures the value to use when documents miss a value.
func (a *TermsAggregation) Missing(missing interface{}) *TermsAggregation {
	a.missing = missing
	return a
}

func (a *TermsAggregation) SubAggregation(name string, subAggregation Aggregation) *TermsAggregation {
	a.subAggregations[name] = subAggregation
	return a
}

// Meta sets the meta data to be included in the aggregation response.
func (a *TermsAggregation) Meta(metaData map[string]interface{}) *TermsAggregation {
	a.meta = metaData
	return a
}

func (a *TermsAggregation) Size(size int) *TermsAggregation {
	a.size = &size
	return a
}

func (a *TermsAggregation) RequiredSize(requiredSize int) *TermsAggregation {
	a.requiredSize = &requiredSize
	return a
}

func (a *TermsAggregation) ShardSize(shardSize int) *TermsAggregation {
	a.shardSize = &shardSize
	return a
}

func (a *TermsAggregation) MinDocCount(minDocCount int) *TermsAggregation {
	a.minDocCount = &minDocCount
	return a
}

func (a *TermsAggregation) ShardMinDocCount(shardMinDocCount int) *TermsAggregation {
	a.shardMinDocCount = &shardMinDocCount
	return a
}

func (a *TermsAggregation) Include(regexp string) *TermsAggregation {
	if a.includeExclude == nil {
		a.includeExclude = &TermsAggregationIncludeExclude{}
	}
	a.includeExclude.Include = regexp
	return a
}

func (a *TermsAggregation) IncludeValues(values ...interface{}) *TermsAggregation {
	if a.includeExclude == nil {
		a.includeExclude = &TermsAggregationIncludeExclude{}
	}
	a.includeExclude.IncludeValues = append(a.includeExclude.IncludeValues, values...)
	return a
}

func (a *TermsAggregation) Exclude(regexp string) *TermsAggregation {
	if a.includeExclude == nil {
		a.includeExclude = &TermsAggregationIncludeExclude{}
	}
	a.includeExclude.Exclude = regexp
	return a
}

func (a *TermsAggregation) ExcludeValues(values ...interface{}) *TermsAggregation {
	if a.includeExclude == nil {
		a.includeExclude = &TermsAggregationIncludeExclude{}
	}
	a.includeExclude.ExcludeValues = append(a.includeExclude.ExcludeValues, values...)
	return a
}

func (a *TermsAggregation) Partition(p int) *TermsAggregation {
	if a.includeExclude == nil {
		a.includeExclude = &TermsAggregationIncludeExclude{}
	}
	a.includeExclude.Partition = p
	return a
}

func (a *TermsAggregation) NumPartitions(n int) *TermsAggregation {
	if a.includeExclude == nil {
		a.includeExclude = &TermsAggregationIncludeExclude{}
	}
	a.includeExclude.NumPartitions = n
	return a
}

// ValueType can be string, long, or double.
func (a *TermsAggregation) ValueType(valueType string) *TermsAggregation {
	a.valueType = valueType
	return a
}

func (a *TermsAggregation) Order(order string, asc bool) *TermsAggregation {
	a.order = append(a.order, TermsOrder{Field: order, Ascending: asc})
	return a
}

func (a *TermsAggregation) OrderByCount(asc bool) *TermsAggregation {
	// "order" : { "_count" : "asc" }
	a.order = append(a.order, TermsOrder{Field: "_count", Ascending: asc})
	return a
}

func (a *TermsAggregation) OrderByCountAsc() *TermsAggregation {
	return a.OrderByCount(true)
}

func (a *TermsAggregation) OrderByCountDesc() *TermsAggregation {
	return a.OrderByCount(false)
}

// Deprecated: Use OrderByKey instead.
func (a *TermsAggregation) OrderByTerm(asc bool) *TermsAggregation {
	// "order" : { "_term" : "asc" }
	a.order = append(a.order, TermsOrder{Field: "_term", Ascending: asc})
	return a
}

// Deprecated: Use OrderByKeyAsc instead.
func (a *TermsAggregation) OrderByTermAsc() *TermsAggregation {
	return a.OrderByTerm(true)
}

// Deprecated: Use OrderByKeyDesc instead.
func (a *TermsAggregation) OrderByTermDesc() *TermsAggregation {
	return a.OrderByTerm(false)
}

func (a *TermsAggregation) OrderByKey(asc bool) *TermsAggregation {
	// "order" : { "_term" : "asc" }
	a.order = append(a.order, TermsOrder{Field: "_key", Ascending: asc})
	return a
}

func (a *TermsAggregation) OrderByKeyAsc() *TermsAggregation {
	return a.OrderByKey(true)
}

func (a *TermsAggregation) OrderByKeyDesc() *TermsAggregation {
	return a.OrderByKey(false)
}

// OrderByAggregation creates a bucket ordering strategy which sorts buckets
// based on a single-valued calc get.
func (a *TermsAggregation) OrderByAggregation(aggName string, asc bool) *TermsAggregation {
	// {
	//     "aggs" : {
	//         "genders" : {
	//             "terms" : {
	//                 "field" : "gender",
	//                 "order" : { "avg_height" : "desc" }
	//             },
	//             "aggs" : {
	//                 "avg_height" : { "avg" : { "field" : "height" } }
	//             }
	//         }
	//     }
	// }
	a.order = append(a.order, TermsOrder{Field: aggName, Ascending: asc})
	return a
}

// OrderByAggregationAndMetric creates a bucket ordering strategy which
// sorts buckets based on a multi-valued calc get.
func (a *TermsAggregation) OrderByAggregationAndMetric(aggName, metric string, asc bool) *TermsAggregation {
	// {
	//     "aggs" : {
	//         "genders" : {
	//             "terms" : {
	//                 "field" : "gender",
	//                 "order" : { "height_stats.avg" : "desc" }
	//             },
	//             "aggs" : {
	//                 "height_stats" : { "stats" : { "field" : "height" } }
	//             }
	//         }
	//     }
	// }
	a.order = append(a.order, TermsOrder{Field: aggName + "." + metric, Ascending: asc})
	return a
}

func (a *TermsAggregation) ExecutionHint(hint string) *TermsAggregation {
	a.executionHint = hint
	return a
}

// Collection mode can be depth_first or breadth_first as of 1.4.0.
func (a *TermsAggregation) CollectionMode(collectionMode string) *TermsAggregation {
	a.collectionMode = collectionMode
	return a
}

func (a *TermsAggregation) ShowTermDocCountError(showTermDocCountError bool) *TermsAggregation {
	a.showTermDocCountError = &showTermDocCountError
	return a
}

func (a *TermsAggregation) Source() (interface{}, error) {
	// Example:
	//	{
	//    "aggs" : {
	//      "genders" : {
	//        "terms" : { "field" : "gender" }
	//      }
	//    }
	//	}
	// This method returns only the { "terms" : { "field" : "gender" } } part.

	source := make(map[string]interface{})
	opts := make(map[string]interface{})
	source["terms"] = opts

	// ValuesSourceAggregationBuilder
	if a.field != "" {
		opts["field"] = a.field
	}
	if a.script != nil {
		src, err := a.script.Source()
		if err != nil {
			return nil, err
		}
		opts["script"] = src
	}
	if a.missing != nil {
		opts["missing"] = a.missing
	}

	// TermsBuilder
	if a.size != nil && *a.size >= 0 {
		opts["size"] = *a.size
	}
	if a.shardSize != nil && *a.shardSize >= 0 {
		opts["shard_size"] = *a.shardSize
	}
	if a.requiredSize != nil && *a.requiredSize >= 0 {
		opts["required_size"] = *a.requiredSize
	}
	if a.minDocCount != nil && *a.minDocCount >= 0 {
		opts["min_doc_count"] = *a.minDocCount
	}
	if a.shardMinDocCount != nil && *a.shardMinDocCount >= 0 {
		opts["shard_min_doc_count"] = *a.shardMinDocCount
	}
	if a.showTermDocCountError != nil {
		opts["show_term_doc_count_error"] = *a.showTermDocCountError
	}
	if a.collectionMode != "" {
		opts["collect_mode"] = a.collectionMode
	}
	if a.valueType != "" {
		opts["value_type"] = a.valueType
	}
	if len(a.order) > 0 {
		var orderSlice []interface{}
		for _, order := range a.order {
			src, err := order.Source()
			if err != nil {
				return nil, err
			}
			orderSlice = append(orderSlice, src)
		}
		opts["order"] = orderSlice
	}
	// Include/Exclude
	if ie := a.includeExclude; ie != nil {
		// Include
		if ie.Include != "" {
			opts["include"] = ie.Include
		} else if len(ie.IncludeValues) > 0 {
			opts["include"] = ie.IncludeValues
		} else if ie.NumPartitions > 0 {
			inc := make(map[string]interface{})
			inc["partition"] = ie.Partition
			inc["num_partitions"] = ie.NumPartitions
			opts["include"] = inc
		}
		// Exclude
		if ie.Exclude != "" {
			opts["exclude"] = ie.Exclude
		} else if len(ie.ExcludeValues) > 0 {
			opts["exclude"] = ie.ExcludeValues
		}
	}

	if a.executionHint != "" {
		opts["execution_hint"] = a.executionHint
	}

	// AggregationBuilder (SubAggregations)
	if len(a.subAggregations) > 0 {
		aggsMap := make(map[string]interface{})
		source["aggregations"] = aggsMap
		for name, aggregate := range a.subAggregations {
			src, err := aggregate.Source()
			if err != nil {
				return nil, err
			}
			aggsMap[name] = src
		}
	}

	// Add Meta data if available
	if len(a.meta) > 0 {
		source["meta"] = a.meta
	}

	return source, nil
}

// TermsAggregationIncludeExclude allows for include/exclude in a TermsAggregation.
type TermsAggregationIncludeExclude struct {
	Include       string
	Exclude       string
	IncludeValues []interface{}
	ExcludeValues []interface{}
	Partition     int
	NumPartitions int
}

// TermsOrder specifies a single order field for a terms aggregation.
type TermsOrder struct {
	Field     string
	Ascending bool
}

// Source returns serializable JSON of the TermsOrder.
func (order *TermsOrder) Source() (interface{}, error) {
	source := make(map[string]string)
	if order.Ascending {
		source[order.Field] = "asc"
	} else {
		source[order.Field] = "desc"
	}
	return source, nil
}
<commit_msg>Fix URL<commit_after>// Copyright 2012-present Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

// TermsAggregation is a multi-bucket value source based aggregation
// where buckets are dynamically built - one per unique value.
//
// See: http://www.elastic.co/guide/en/elasticsearch/reference/7.0/search-aggregations-bucket-terms-aggregation.html
type TermsAggregation struct {
	field           string
	script          *Script
	missing         interface{}
	subAggregations map[string]Aggregation
	meta            map[string]interface{}

	size                  *int
	shardSize             *int
	requiredSize          *int
	minDocCount           *int
	shardMinDocCount      *int
	valueType             string
	includeExclude        *TermsAggregationIncludeExclude
	executionHint         string
	collectionMode        string
	showTermDocCountError *bool
	order                 []TermsOrder
}

func NewTermsAggregation() *TermsAggregation {
	return &TermsAggregation{
		subAggregations: make(map[string]Aggregation, 0),
	}
}

func (a *TermsAggregation) Field(field string) *TermsAggregation {
	a.field = field
	return a
}

func (a *TermsAggregation) Script(script *Script) *TermsAggregation {
	a.script = script
	return a
}

// Missing configures the value to use when documents miss a value.
func (a *TermsAggregation) Missing(missing interface{}) *TermsAggregation {
	a.missing = missing
	return a
}

func (a *TermsAggregation) SubAggregation(name string, subAggregation Aggregation) *TermsAggregation {
	a.subAggregations[name] = subAggregation
	return a
}

// Meta sets the meta data to be included in the aggregation response.
func (a *TermsAggregation) Meta(metaData map[string]interface{}) *TermsAggregation {
	a.meta = metaData
	return a
}

func (a *TermsAggregation) Size(size int) *TermsAggregation {
	a.size = &size
	return a
}

func (a *TermsAggregation) RequiredSize(requiredSize int) *TermsAggregation {
	a.requiredSize = &requiredSize
	return a
}

func (a *TermsAggregation) ShardSize(shardSize int) *TermsAggregation {
	a.shardSize = &shardSize
	return a
}

func (a *TermsAggregation) MinDocCount(minDocCount int) *TermsAggregation {
	a.minDocCount = &minDocCount
	return a
}

func (a *TermsAggregation) ShardMinDocCount(shardMinDocCount int) *TermsAggregation {
	a.shardMinDocCount = &shardMinDocCount
	return a
}

func (a *TermsAggregation) Include(regexp string) *TermsAggregation {
	if a.includeExclude == nil {
		a.includeExclude = &TermsAggregationIncludeExclude{}
	}
	a.includeExclude.Include = regexp
	return a
}

func (a *TermsAggregation) IncludeValues(values ...interface{}) *TermsAggregation {
	if a.includeExclude == nil {
		a.includeExclude = &TermsAggregationIncludeExclude{}
	}
	a.includeExclude.IncludeValues = append(a.includeExclude.IncludeValues, values...)
	return a
}

func (a *TermsAggregation) Exclude(regexp string) *TermsAggregation {
	if a.includeExclude == nil {
		a.includeExclude = &TermsAggregationIncludeExclude{}
	}
	a.includeExclude.Exclude = regexp
	return a
}

func (a *TermsAggregation) ExcludeValues(values ...interface{}) *TermsAggregation {
	if a.includeExclude == nil {
		a.includeExclude = &TermsAggregationIncludeExclude{}
	}
	a.includeExclude.ExcludeValues = append(a.includeExclude.ExcludeValues, values...)
	return a
}

func (a *TermsAggregation) Partition(p int) *TermsAggregation {
	if a.includeExclude == nil {
		a.includeExclude = &TermsAggregationIncludeExclude{}
	}
	a.includeExclude.Partition = p
	return a
}

func (a *TermsAggregation) NumPartitions(n int) *TermsAggregation {
	if a.includeExclude == nil {
		a.includeExclude = &TermsAggregationIncludeExclude{}
	}
	a.includeExclude.NumPartitions = n
	return a
}

// ValueType can be string, long, or double.
func (a *TermsAggregation) ValueType(valueType string) *TermsAggregation {
	a.valueType = valueType
	return a
}

func (a *TermsAggregation) Order(order string, asc bool) *TermsAggregation {
	a.order = append(a.order, TermsOrder{Field: order, Ascending: asc})
	return a
}

func (a *TermsAggregation) OrderByCount(asc bool) *TermsAggregation {
	// "order" : { "_count" : "asc" }
	a.order = append(a.order, TermsOrder{Field: "_count", Ascending: asc})
	return a
}

func (a *TermsAggregation) OrderByCountAsc() *TermsAggregation {
	return a.OrderByCount(true)
}

func (a *TermsAggregation) OrderByCountDesc() *TermsAggregation {
	return a.OrderByCount(false)
}

// Deprecated: Use OrderByKey instead.
func (a *TermsAggregation) OrderByTerm(asc bool) *TermsAggregation {
	// "order" : { "_term" : "asc" }
	a.order = append(a.order, TermsOrder{Field: "_term", Ascending: asc})
	return a
}

// Deprecated: Use OrderByKeyAsc instead.
func (a *TermsAggregation) OrderByTermAsc() *TermsAggregation {
	return a.OrderByTerm(true)
}

// Deprecated: Use OrderByKeyDesc instead.
func (a *TermsAggregation) OrderByTermDesc() *TermsAggregation {
	return a.OrderByTerm(false)
}

func (a *TermsAggregation) OrderByKey(asc bool) *TermsAggregation {
	// "order" : { "_term" : "asc" }
	a.order = append(a.order, TermsOrder{Field: "_key", Ascending: asc})
	return a
}

func (a *TermsAggregation) OrderByKeyAsc() *TermsAggregation {
	return a.OrderByKey(true)
}

func (a *TermsAggregation) OrderByKeyDesc() *TermsAggregation {
	return a.OrderByKey(false)
}

// OrderByAggregation creates a bucket ordering strategy which sorts buckets
// based on a single-valued calc get.
func (a *TermsAggregation) OrderByAggregation(aggName string, asc bool) *TermsAggregation {
	// {
	//     "aggs" : {
	//         "genders" : {
	//             "terms" : {
	//                 "field" : "gender",
	//                 "order" : { "avg_height" : "desc" }
	//             },
	//             "aggs" : {
	//                 "avg_height" : { "avg" : { "field" : "height" } }
	//             }
	//         }
	//     }
	// }
	a.order = append(a.order, TermsOrder{Field: aggName, Ascending: asc})
	return a
}

// OrderByAggregationAndMetric creates a bucket ordering strategy which
// sorts buckets based on a multi-valued calc get.
func (a *TermsAggregation) OrderByAggregationAndMetric(aggName, metric string, asc bool) *TermsAggregation {
	// {
	//     "aggs" : {
	//         "genders" : {
	//             "terms" : {
	//                 "field" : "gender",
	//                 "order" : { "height_stats.avg" : "desc" }
	//             },
	//             "aggs" : {
	//                 "height_stats" : { "stats" : { "field" : "height" } }
	//             }
	//         }
	//     }
	// }
	a.order = append(a.order, TermsOrder{Field: aggName + "." + metric, Ascending: asc})
	return a
}

func (a *TermsAggregation) ExecutionHint(hint string) *TermsAggregation {
	a.executionHint = hint
	return a
}

// Collection mode can be depth_first or breadth_first as of 1.4.0.
func (a *TermsAggregation) CollectionMode(collectionMode string) *TermsAggregation {
	a.collectionMode = collectionMode
	return a
}

func (a *TermsAggregation) ShowTermDocCountError(showTermDocCountError bool) *TermsAggregation {
	a.showTermDocCountError = &showTermDocCountError
	return a
}

func (a *TermsAggregation) Source() (interface{}, error) {
	// Example:
	//	{
	//    "aggs" : {
	//      "genders" : {
	//        "terms" : { "field" : "gender" }
	//      }
	//    }
	//	}
	// This method returns only the { "terms" : { "field" : "gender" } } part.

	source := make(map[string]interface{})
	opts := make(map[string]interface{})
	source["terms"] = opts

	// ValuesSourceAggregationBuilder
	if a.field != "" {
		opts["field"] = a.field
	}
	if a.script != nil {
		src, err := a.script.Source()
		if err != nil {
			return nil, err
		}
		opts["script"] = src
	}
	if a.missing != nil {
		opts["missing"] = a.missing
	}

	// TermsBuilder
	if a.size != nil && *a.size >= 0 {
		opts["size"] = *a.size
	}
	if a.shardSize != nil && *a.shardSize >= 0 {
		opts["shard_size"] = *a.shardSize
	}
	if a.requiredSize != nil && *a.requiredSize >= 0 {
		opts["required_size"] = *a.requiredSize
	}
	if a.minDocCount != nil && *a.minDocCount >= 0 {
		opts["min_doc_count"] = *a.minDocCount
	}
	if a.shardMinDocCount != nil && *a.shardMinDocCount >= 0 {
		opts["shard_min_doc_count"] = *a.shardMinDocCount
	}
	if a.showTermDocCountError != nil {
		opts["show_term_doc_count_error"] = *a.showTermDocCountError
	}
	if a.collectionMode != "" {
		opts["collect_mode"] = a.collectionMode
	}
	if a.valueType != "" {
		opts["value_type"] = a.valueType
	}
	if len(a.order) > 0 {
		var orderSlice []interface{}
		for _, order := range a.order {
			src, err := order.Source()
			if err != nil {
				return nil, err
			}
			orderSlice = append(orderSlice, src)
		}
		opts["order"] = orderSlice
	}
	// Include/Exclude
	if ie := a.includeExclude; ie != nil {
		// Include
		if ie.Include != "" {
			opts["include"] = ie.Include
		} else if len(ie.IncludeValues) > 0 {
			opts["include"] = ie.IncludeValues
		} else if ie.NumPartitions > 0 {
			inc := make(map[string]interface{})
			inc["partition"] = ie.Partition
			inc["num_partitions"] = ie.NumPartitions
			opts["include"] = inc
		}
		// Exclude
		if ie.Exclude != "" {
			opts["exclude"] = ie.Exclude
		} else if len(ie.ExcludeValues) > 0 {
			opts["exclude"] = ie.ExcludeValues
		}
	}

	if a.executionHint != "" {
		opts["execution_hint"] = a.executionHint
	}

	// AggregationBuilder (SubAggregations)
	if len(a.subAggregations) > 0 {
		aggsMap := make(map[string]interface{})
		source["aggregations"] = aggsMap
		for name, aggregate := range a.subAggregations {
			src, err := aggregate.Source()
			if err != nil {
				return nil, err
			}
			aggsMap[name] = src
		}
	}

	// Add Meta data if available
	if len(a.meta) > 0 {
		source["meta"] = a.meta
	}

	return source, nil
}

// TermsAggregationIncludeExclude allows for include/exclude in a TermsAggregation.
type TermsAggregationIncludeExclude struct {
	Include       string
	Exclude       string
	IncludeValues []interface{}
	ExcludeValues []interface{}
	Partition     int
	NumPartitions int
}

// TermsOrder specifies a single order field for a terms aggregation.
type TermsOrder struct {
	Field     string
	Ascending bool
}

// Source returns serializable JSON of the TermsOrder.
func (order *TermsOrder) Source() (interface{}, error) {
	source := make(map[string]string)
	if order.Ascending {
		source[order.Field] = "asc"
	} else {
		source[order.Field] = "desc"
	}
	return source, nil
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/goprotobuf/proto"
	ap "github.com/couchbase/indexing/secondary/adminport"
	c "github.com/couchbase/indexing/secondary/common"
	"github.com/couchbase/indexing/secondary/indexer"
	"github.com/couchbase/indexing/secondary/projector"
	"github.com/couchbase/indexing/secondary/protobuf"
	"log"
	"time"
)

var cluster = "localhost:9000"
var pooln = "default"
var bucketn = "beer-sample"
var kvaddrs1 = []string{"127.0.0.1:12000"}
var kvaddrs2 = []string{"127.0.0.1:12002"}
var adminport1 = "localhost:9010"
var adminport2 = "localhost:9011"
var endpoint = "localhost:9020"
var coordEndpoint = "localhost:9021"
var vbMax = 1024
var vbnos = vbucketsList()

var done = make(chan bool)

var instances = []uint64{0x11, 0x12}

func main() {
	c.SetLogLevel(c.LogLevelInfo)
	go endpointServer(endpoint)
	go endpointServer(coordEndpoint)

	time.Sleep(100 * time.Millisecond)

	doProjector(cluster, kvaddrs1, adminport1)
	doProjector(cluster, kvaddrs2, adminport2)

	<-done
	<-done
}

func doProjector(cluster string, kvaddrs []string, adminport string) {
	projector.NewProjector(cluster, kvaddrs, adminport)
	time.Sleep(100 * time.Millisecond)
	aport := ap.NewHTTPClient("http://"+adminport, "/adminport/")
	fReq := protobuf.FailoverLogRequest{
		Pool:   proto.String(pooln),
		Bucket: proto.String(bucketn),
		Vbnos:  c.Vbno16to32(vbnos),
	}
	fRes := protobuf.FailoverLogResponse{}
	if err := aport.Request(&fReq, &fRes); err != nil {
		log.Fatal(err)
	}
	vbuuids := make(map[uint16]uint64)
	for _, flog := range fRes.GetLogs() {
		vbno := uint16(flog.GetVbno())
		vbuuid := flog.Vbuuids[len(flog.Vbuuids)-1]
		vbuuids[vbno] = vbuuid
	}

	time.Sleep(100 * time.Millisecond)

	mReq := makeStartRequest(vbuuids)
	mRes := protobuf.MutationStreamResponse{}
	if err := aport.Request(mReq, &mRes); err != nil {
		log.Fatal(err)
	}
}

func makeStartRequest(vbuuids map[uint16]uint64) *protobuf.MutationStreamRequest {
	bTs := makeBranchTimestamp(vbuuids)
	req := protobuf.MutationStreamRequest{
		Topic:             proto.String("maintanence"),
		Pools:             []string{pooln},
		Buckets:           []string{bucketn},
		RestartTimestamps: []*protobuf.BranchTimestamp{bTs},
		Instances:         makeIndexInstances(),
	}
	req.SetStartFlag()
	return &req
}

func makeIndexInstances() []*protobuf.IndexInst {
	sExprs := []string{`{"type":"property","path":"name"}`,
		`{"type":"property","path":"abv"}`}
	defn1 := &protobuf.IndexDefn{
		DefnID:          proto.Uint64(instances[0]),
		Bucket:          proto.String(bucketn),
		IsPrimary:       proto.Bool(false),
		Name:            proto.String("index1"),
		Using:           protobuf.StorageType_View.Enum(),
		ExprType:        protobuf.ExprType_N1QL.Enum(),
		SecExpressions:  sExprs,
		PartitionScheme: protobuf.PartitionScheme_TEST.Enum(),
		PartnExpression: proto.String(`{"type":"property","path":"type"}`),
	}
	instance1 := &protobuf.IndexInst{
		InstId:     proto.Uint64(0x1),
		State:      protobuf.IndexState_IndexInitial.Enum(),
		Definition: defn1,
		Tp: &protobuf.TestPartition{
			CoordEndpoint: proto.String(coordEndpoint),
			Endpoints:     []string{endpoint},
		},
	}

	defn2 := &protobuf.IndexDefn{
		DefnID:          proto.Uint64(instances[1]),
		Bucket:          proto.String(bucketn),
		IsPrimary:       proto.Bool(false),
		Name:            proto.String("index2"),
		Using:           protobuf.StorageType_View.Enum(),
		ExprType:        protobuf.ExprType_N1QL.Enum(),
		SecExpressions:  []string{`{"type":"property","path":"city"}`},
		PartitionScheme: protobuf.PartitionScheme_TEST.Enum(),
		PartnExpression: proto.String(`{"type":"property","path":"type"}`),
	}
	instance2 := &protobuf.IndexInst{
		InstId:     proto.Uint64(0x2),
		State:      protobuf.IndexState_IndexInitial.Enum(),
		Definition: defn2,
		Tp: &protobuf.TestPartition{
			CoordEndpoint: proto.String(coordEndpoint),
			Endpoints:     []string{endpoint},
		},
	}
	return []*protobuf.IndexInst{instance1, instance2}
}

func endpointServer(addr string) {
	mutChanSize := 100
	mutch := make(chan []*protobuf.VbKeyVersions, mutChanSize)
	sbch := make(chan interface{}, 100)
	_, err := indexer.NewMutationStream(addr, mutch, sbch)
	if err != nil {
		log.Fatal(err)
	}
	mutations, messages := 0, 0
	commandWise := make(map[byte]int)
	keys := map[uint64]map[string][]string{
		0x11: make(map[string][]string),
		0x12: make(map[string][]string),
	}

	printTm := time.Tick(100 * time.Millisecond)

loop:
	for {
		select {
		case vbs, ok := <-mutch:
			if ok {
				mutations += gatherKeys(vbs, commandWise, keys)
			} else {
				break loop
			}
		case s, ok := <-sbch:
			if ok {
				switch v := s.(type) {
				case []*indexer.RestartVbuckets:
					printRestartVbuckets(addr, v)
				}
				messages++
			} else {
				break loop
			}
		case <-time.After(4 * time.Second):
			break loop
		case <-printTm:
			log.Println(addr, "-- mutations", mutations)
			log.Println(addr, "-- commandWise", commandWise)
		}
	}

	log.Println(addr, "-- mutations", mutations, "-- messages", messages)
	log.Println(addr, "-- commandWise", commandWise)
	ks, ds := countKeysAndDocs(keys[0x11])
	log.Printf("%v -- for instance 0x11, %v unique keys found in %v docs\n", addr, ks, ds)
	ks, ds = countKeysAndDocs(keys[0x12])
	log.Printf("%v -- for instance 0x12, %v unique keys found in %v docs\n", addr, ks, ds)
	done <- true
}

func printRestartVbuckets(addr string, rs []*indexer.RestartVbuckets) {
	for _, r := range rs {
		log.Printf("restart: %s, %v %v\n", addr, r.Bucket, r.Vbuckets)
	}
}

func gatherKeys(
	vbs []*protobuf.VbKeyVersions,
	commandWise map[byte]int,
	keys map[uint64]map[string][]string,
) int {

	mutations := 0
	for _, vb := range vbs {
		kvs := vb.GetKvs()
		for _, kv := range kvs {
			mutations++
			docid := string(kv.GetDocid())
			uuids, seckeys := kv.GetUuids(), kv.GetKeys()
			for i, command := range kv.GetCommands() {
				cmd := byte(command)
				if _, ok := commandWise[cmd]; !ok {
					commandWise[cmd] = 0
				}
				commandWise[cmd]++

				if cmd == c.StreamBegin {
					continue
				}

				if cmd == c.Upsert {
					uuid := uuids[i]
					key := string(seckeys[i])
					if _, ok := keys[uuid][key]; !ok {
						keys[uuid][key] = make([]string, 0)
					}
					keys[uuid][key] = append(keys[uuid][key], docid)
				}
			}
		}
	}
	return mutations
}

func countKeysAndDocs(keys map[string][]string) (int, int) {
	countKs, countDs := 0, 0
	for _, docs := range keys {
		countKs++
		countDs += len(docs)
	}
	return countKs, countDs
}

func vbucketsList() []uint16 {
	vbnos := make([]uint16, 0, vbMax)
	for i := 0; i < vbMax; i++ {
		vbnos = append(vbnos, uint16(i))
	}
	return vbnos
}

func makeBranchTimestamp(vbuuids map[uint16]uint64) *protobuf.BranchTimestamp {
	vbuuidsSorted := make([]uint64, 0, vbMax)
	for vbno := range vbnos {
		vbuuidsSorted = append(vbuuidsSorted, vbuuids[uint16(vbno)])
	}
	bTs := &protobuf.BranchTimestamp{
		Bucket:  proto.String(bucketn),
		Vbnos:   c.Vbno16to32(vbnos),
		Seqnos:  make([]uint64, vbMax),
		Vbuuids: vbuuidsSorted,
	}
	return bTs
}
<commit_msg>datapath.go goes multi-bucket, multi-kvnode.<commit_after>package main

import (
	"code.google.com/p/goprotobuf/proto"
	ap "github.com/couchbase/indexing/secondary/adminport"
	c "github.com/couchbase/indexing/secondary/common"
	"github.com/couchbase/indexing/secondary/indexer"
	"github.com/couchbase/indexing/secondary/projector"
	"github.com/couchbase/indexing/secondary/protobuf"
	"log"
	"time"
)

var cluster = "localhost:9000"
var pooln = "default"
var bucketn = "beer-sample"
var kvaddrs1 = []string{"127.0.0.1:12000"}
var kvaddrs2 = []string{"127.0.0.1:12002"}
var adminport1 = "localhost:9010"
var adminport2 = "localhost:9011"
var endpoint = "localhost:9020"
var coordEndpoint = "localhost:9021"
var vbMax = 1024
var vbnos = vbucketsList()

var done = make(chan bool)

var instances = []uint64{0x11, 0x12, 0x13}

func main() {
	c.SetLogLevel(c.LogLevelInfo)
	go endpointServer(endpoint)
	go endpointServer(coordEndpoint)

	time.Sleep(100 * time.Millisecond)

	doProjector(cluster, kvaddrs1, adminport1)
	doProjector(cluster, kvaddrs2, adminport2)

	<-done
	<-done
}

func doProjector(cluster string, kvaddrs []string, adminport string) {
	projector.NewProjector(cluster, kvaddrs, adminport)
	time.Sleep(100 * time.Millisecond)
	aport := ap.NewHTTPClient("http://"+adminport, "/adminport/")

	// failover log for "default"
	fReq := protobuf.FailoverLogRequest{
		Pool:   proto.String(pooln),
		Bucket: proto.String("default"),
		Vbnos:  c.Vbno16to32(vbnos),
	}
	fRes := protobuf.FailoverLogResponse{}
	if err := aport.Request(&fReq, &fRes); err != nil {
		log.Fatal(err)
	}
	vbuuids1 := make(map[uint16]uint64)
	for _, flog := range fRes.GetLogs() {
		vbno := uint16(flog.GetVbno())
		vbuuid := flog.Vbuuids[len(flog.Vbuuids)-1]
		vbuuids1[vbno] = vbuuid
	}

	// failover log for "beer-sample"
	fReq = protobuf.FailoverLogRequest{
		Pool:   proto.String(pooln),
		Bucket: proto.String("beer-sample"),
		Vbnos:  c.Vbno16to32(vbnos),
	}
	fRes = protobuf.FailoverLogResponse{}
	if err := aport.Request(&fReq, &fRes); err != nil {
		log.Fatal(err)
	}
	vbuuids2 := make(map[uint16]uint64)
	for _, flog := range fRes.GetLogs() {
		vbno := uint16(flog.GetVbno())
		vbuuid := flog.Vbuuids[len(flog.Vbuuids)-1]
		vbuuids2[vbno] = vbuuid
	}

	time.Sleep(100 * time.Millisecond)

	mReq := makeStartRequest(vbuuids1, vbuuids2)
	mRes := protobuf.MutationStreamResponse{}
	if err := aport.Request(mReq, &mRes); err != nil {
		log.Fatal(err)
	}
}

func makeStartRequest(vbuuids1, vbuuids2 map[uint16]uint64) *protobuf.MutationStreamRequest {
	b1Ts := makeBranchTimestamp(vbuuids1)
	b2Ts := makeBranchTimestamp(vbuuids2)
	req := protobuf.MutationStreamRequest{
		Topic:             proto.String("maintanence"),
		Pools:             []string{pooln, pooln},
		Buckets:           []string{"default", "beer-sample"},
		RestartTimestamps: []*protobuf.BranchTimestamp{b1Ts, b2Ts},
		Instances:         makeIndexInstances(),
	}
	req.SetStartFlag()
	return &req
}

func makeIndexInstances() []*protobuf.IndexInst {
	sExprs := []string{`{"type":"property","path":"name"}`,
		`{"type":"property","path":"abv"}`}
	defn1 := &protobuf.IndexDefn{
		DefnID:          proto.Uint64(instances[0]),
		Bucket:          proto.String("beer-sample"),
		IsPrimary:       proto.Bool(false),
		Name:            proto.String("index1"),
		Using:           protobuf.StorageType_View.Enum(),
		ExprType:        protobuf.ExprType_N1QL.Enum(),
		SecExpressions:  sExprs,
		PartitionScheme: protobuf.PartitionScheme_TEST.Enum(),
		PartnExpression: proto.String(`{"type":"property","path":"type"}`),
	}
	instance1 := &protobuf.IndexInst{
		InstId:     proto.Uint64(0x1),
		State:      protobuf.IndexState_IndexInitial.Enum(),
		Definition: defn1,
		Tp: &protobuf.TestPartition{
			CoordEndpoint: proto.String(coordEndpoint),
			Endpoints:     []string{endpoint},
		},
	}

	defn2 := &protobuf.IndexDefn{
		DefnID:          proto.Uint64(instances[1]),
		Bucket:          proto.String("beer-sample"),
		IsPrimary:       proto.Bool(false),
		Name:            proto.String("index2"),
		Using:           protobuf.StorageType_View.Enum(),
		ExprType:        protobuf.ExprType_N1QL.Enum(),
		SecExpressions:  []string{`{"type":"property","path":"city"}`},
		PartitionScheme: protobuf.PartitionScheme_TEST.Enum(),
		PartnExpression: proto.String(`{"type":"property","path":"type"}`),
	}
	instance2 := &protobuf.IndexInst{
		InstId:     proto.Uint64(0x2),
		State:      protobuf.IndexState_IndexInitial.Enum(),
		Definition: defn2,
		Tp: &protobuf.TestPartition{
			CoordEndpoint: proto.String(coordEndpoint),
			Endpoints:     []string{endpoint},
		},
	}

	defn3 := &protobuf.IndexDefn{
		DefnID:          proto.Uint64(instances[2]),
		Bucket:          proto.String("default"),
		IsPrimary:       proto.Bool(false),
		Name:            proto.String("index3"),
		Using:           protobuf.StorageType_View.Enum(),
		ExprType:        protobuf.ExprType_N1QL.Enum(),
		SecExpressions:  []string{`{"type":"property","path":"age"}`},
		PartitionScheme: protobuf.PartitionScheme_TEST.Enum(),
		PartnExpression: proto.String(`{"type":"property","path":"index"}`),
	}
	instance3 := &protobuf.IndexInst{
		InstId:     proto.Uint64(0x3),
		State:      protobuf.IndexState_IndexInitial.Enum(),
		Definition: defn3,
		Tp: &protobuf.TestPartition{
			CoordEndpoint: proto.String(coordEndpoint),
			Endpoints:     []string{endpoint},
		},
	}
	return []*protobuf.IndexInst{instance1, instance2, instance3}
}

func endpointServer(addr string) {
	mutChanSize := 100
	mutch := make(chan []*protobuf.VbKeyVersions, mutChanSize)
	sbch := make(chan interface{}, 100)
	_, err := indexer.NewMutationStream(addr, mutch, sbch)
	if err != nil {
		log.Fatal(err)
	}
	mutations, messages := 0, 0
	commandWise := make(map[byte]int)
	keys := map[uint64]map[string][]string{
		0x11: make(map[string][]string),
		0x12: make(map[string][]string),
		0x13: make(map[string][]string),
	}

	printTm := time.Tick(1000 * time.Millisecond)

loop:
	for {
		select {
		case vbs, ok := <-mutch:
			if ok {
				mutations += gatherKeys(vbs, commandWise, keys)
			} else {
				break loop
			}
		case s, ok := <-sbch:
			if ok {
				switch v := s.(type) {
				case []*indexer.RestartVbuckets:
					printRestartVbuckets(addr, v)
				}
				messages++
			} else {
				break loop
			}
		case <-time.After(4 * time.Second):
			break loop
		case <-printTm:
			log.Println(addr, "-- mutations", mutations)
			log.Println(addr, "-- commandWise", commandWise)
		}
	}

	log.Println(addr, "-- mutations", mutations, "-- messages", messages)
	log.Println(addr, "-- commandWise", commandWise)
	ks, ds := countKeysAndDocs(keys[0x11])
	log.Printf("%v -- for instance 0x11, %v unique keys found in %v docs\n", addr, ks, ds)
	ks, ds = countKeysAndDocs(keys[0x12])
	log.Printf("%v -- for instance 0x12, %v unique keys found in %v docs\n", addr, ks, ds)
	done <- true
}

func printRestartVbuckets(addr string, rs []*indexer.RestartVbuckets) {
	for _, r := range rs {
		log.Printf("restart: %s, %v %v\n", addr, r.Bucket, r.Vbuckets)
	}
}

func gatherKeys(
	vbs []*protobuf.VbKeyVersions,
	commandWise map[byte]int,
	keys map[uint64]map[string][]string,
) int {

	mutations := 0
	for _, vb := range vbs {
		kvs := vb.GetKvs()
		for _, kv := range kvs {
			mutations++
			docid := string(kv.GetDocid())
			uuids, seckeys := kv.GetUuids(), kv.GetKeys()
			for i, command := range kv.GetCommands() {
				cmd := byte(command)
				if _, ok := commandWise[cmd]; !ok {
					commandWise[cmd] = 0
				}
				commandWise[cmd]++

				if cmd == c.StreamBegin {
					continue
				}

				if cmd == c.Upsert {
					uuid := uuids[i]
					key := string(seckeys[i])
					if _, ok := keys[uuid][key]; !ok {
						keys[uuid][key] = make([]string, 0)
					}
					keys[uuid][key] = append(keys[uuid][key], docid)
				}
			}
		}
	}
	return mutations
}

func countKeysAndDocs(keys map[string][]string) (int, int) {
	countKs, countDs := 0, 0
	for _, docs := range keys {
		countKs++
		countDs += len(docs)
	}
	return countKs, countDs
}

func vbucketsList() []uint16 {
	vbnos := make([]uint16, 0, vbMax)
	for i := 0; i < vbMax; i++ {
		vbnos = append(vbnos, uint16(i))
	}
	return vbnos
}

func makeBranchTimestamp(vbuuids map[uint16]uint64) *protobuf.BranchTimestamp {
	vbuuidsSorted := make([]uint64, 0, vbMax)
	for vbno := range vbnos {
		vbuuidsSorted = append(vbuuidsSorted, vbuuids[uint16(vbno)])
	}
	bTs := &protobuf.BranchTimestamp{
		Bucket:  proto.String(bucketn),
		Vbnos:   c.Vbno16to32(vbnos),
		Seqnos:  make([]uint64, vbMax),
		Vbuuids: vbuuidsSorted,
	}
	return bTs
}
<|endoftext|>


<commit_before>package cwl

import (
	"fmt"
	"github.com/davecgh/go-spew/spew"
	"reflect"
)

// http://www.commonwl.org/draft-3/CommandLineTool.html#CWLType
const (
	CWL_null      = "null"      //no value
	CWL_boolean   = "boolean"   //a binary value
	CWL_int       = "int"       //32-bit signed integer
	CWL_long      = "long"      //64-bit signed integer
	CWL_float     = "float"     //single precision (32-bit) IEEE 754 floating-point number
	CWL_double    = "double"    //double precision (64-bit) IEEE 754 floating-point number
	CWL_string    = "string"    //Unicode character sequence
	CWL_File      = "File"      //A File object
	CWL_Directory = "Directory" //A Directory object

	CWL_array  = "array"
	CWL_record = "record"
	CWL_enum   = "enum"

	CWL_stdout = "stdout"
	CWL_stderr = "stderr"

	CWL_boolean_array   = "boolean_array"   //a binary value
	CWL_int_array       = "int_array"       //32-bit signed integer
	CWL_long_array      = "long_array"      //64-bit signed integer
	CWL_float_array     = "float_array"     //single precision (32-bit) IEEE 754 floating-point number
	CWL_double_array    = "double_array"    //double precision (64-bit) IEEE 754 floating-point number
	CWL_string_array    = "string_array"    //Unicode character sequence
	CWL_File_array      = "File_array"      //A File object
	CWL_Directory_array = "Directory_array" //A Directory object
)

var Valid_cwltypes = map[string]bool{
	CWL_null:            true,
	CWL_boolean:         true,
	CWL_int:             true,
	CWL_long:            true,
	CWL_float:           true,
	CWL_double:          true,
	CWL_string:          true,
	CWL_File:            true,
	CWL_Directory:       true,
	CWL_stdout:          true,
	CWL_stderr:          true,
	CWL_boolean_array:   true,
	CWL_int_array:       true,
	CWL_long_array:      true,
	CWL_float_array:     true,
	CWL_double_array:    true,
	CWL_string_array:    true,
	CWL_File_array:      true,
	CWL_Directory_array: true,
}

var Native2Array = map[string]string{
	CWL_boolean:   CWL_boolean_array,
	CWL_int:       CWL_int_array,
	CWL_long:      CWL_long_array,
	CWL_float:     CWL_float_array,
	CWL_double:    CWL_double_array,
	CWL_string:    CWL_string_array,
	CWL_File:      CWL_File_array,
	CWL_Directory: CWL_Directory_array,
}

type CWL_array_type interface {
	Is_CWL_array_type()
	Get_Array() *[]CWLType
}

type CWL_minimal_interface interface {
	Is_CWL_minimal()
}

type CWL_minimal struct{}

func (c *CWL_minimal) Is_CWL_minimal() {}

// generic class to represent Files and Directories
type CWL_location interface {
	GetLocation() string
}

// CWLType - CWL basic types: int, string, boolean, .. etc
// http://www.commonwl.org/v1.0/CommandLineTool.html#CWLType
// null, boolean, int, long, float, double, string, File, Directory
type CWLType interface {
	CWL_object // is an interface
	Is_CommandInputParameterType()
	Is_CommandOutputParameterType()
	Is_CWLType()

	//Is_Array() bool
	//Is_CWL_minimal()
}

type CWLType_Impl struct{}

func (c *CWLType_Impl) Is_CWL_minimal()                {}
func (c *CWLType_Impl) Is_CWLType()                    {}
func (c *CWLType_Impl) Is_CommandInputParameterType()  {}
func (c *CWLType_Impl) Is_CommandOutputParameterType() {}

//func (c *CWLType_Impl) Is_Array() bool                 { return false }

func NewCWLType(id string, native interface{}) (cwl_type CWLType, err error) {

	//var cwl_type CWLType

	switch native.(type) {
	case []interface{}:

		native_array, _ := native.([]interface{})

		array, xerr := NewArray(id, native_array)
		if xerr != nil {
			err = xerr
			return
		}
		cwl_type = array
		return

	case int:
		native_int := native.(int)

		cwl_type = &Int{Value: native_int}

		//cwl_type = int_type.(*CWLType)

		return
		//temp = &Int{Value: native_int}

	case string:
		native_str := native.(string)

		cwl_type = &String{Value: native_str}
	case bool:
		native_bool := native.(bool)

		cwl_type = &Boolean{Value: native_bool}

	case map[interface{}]interface{}:
		empty, xerr := NewEmpty(native)
		if xerr != nil {
			err = xerr
			return
		}
		switch empty.GetClass() {
		case "File":
			file, yerr := NewFile(native)
			cwl_type = &file
			if yerr != nil {
				err = yerr
				return
			}
		default:
			// Map type unknown, maybe a record
			spew.Dump(native)

			record, xerr := NewRecord(native)
			if xerr != nil {
				err = xerr
				return
			}
			cwl_type = record
			return
		}
	case map[string]interface{}:
		//empty, xerr := NewEmpty(native)
		//if xerr != nil {
		//	err = xerr
		//	return
		//}
		class, xerr := GetClass(native)
		if xerr != nil {
			err = xerr
			return
		}
		switch class {
		case "File":
			file, yerr := NewFile(native)
			cwl_type = &file
			if yerr != nil {
				err = yerr
				return
			}
		default:
			spew.Dump(native)
			err = fmt.Errorf("(NewCWLType) Map type unknown")
			return
		}

	default:
		spew.Dump(native)
		err = fmt.Errorf("(NewCWLType) Type unknown: \"%s\"", reflect.TypeOf(native))
		return
	}
	//cwl_type_ptr = &cwl_type

	return

}

func NewCWLTypeArray_deprecated(native interface{}) (cwl_array_ptr *[]CWLType, err error) {

	switch native.(type) {
	case []interface{}:

		native_array, ok := native.([]interface{})
		if !ok {
			err = fmt.Errorf("(NewCWLTypeArray) could not parse []interface{}")
			return
		}

		cwl_array := []CWLType{}

		for _, value := range native_array {
			value_cwl, xerr := NewCWLType("", value)
			if xerr != nil {
				err = xerr
				return
			}
			cwl_array = append(cwl_array, value_cwl)
		}
		cwl_array_ptr = &cwl_array
	default:

		ct, xerr := NewCWLType("", native)
		if xerr != nil {
			err = xerr
			return
		}

		cwl_array_ptr = &[]CWLType{ct}
	}

	return

}
<commit_msg>added NewCWLTypeByClass<commit_after>package cwl

import (
	"fmt"
	"github.com/davecgh/go-spew/spew"
	"reflect"
)

// http://www.commonwl.org/draft-3/CommandLineTool.html#CWLType
const (
	CWL_null      = "null"      //no value
	CWL_boolean   = "boolean"   //a binary value
	CWL_int       = "int"       //32-bit signed integer
	CWL_long      = "long"      //64-bit signed integer
	CWL_float     = "float"     //single precision (32-bit) IEEE 754 floating-point number
	CWL_double    = "double"    //double precision (64-bit) IEEE 754 floating-point number
	CWL_string    = "string"    //Unicode character sequence
	CWL_File      = "File"      //A File object
	CWL_Directory = "Directory" //A Directory object

	CWL_array  = "array"
	CWL_record = "record"
	CWL_enum   = "enum"

	CWL_stdout = "stdout"
	CWL_stderr = "stderr"

	CWL_boolean_array   = "boolean_array"   //a binary value
	CWL_int_array       = "int_array"       //32-bit signed integer
	CWL_long_array      = "long_array"      //64-bit signed integer
	CWL_float_array     = "float_array"     //single precision (32-bit) IEEE 754 floating-point number
	CWL_double_array    = "double_array"    //double precision (64-bit) IEEE 754 floating-point number
	CWL_string_array    = "string_array"    //Unicode character sequence
	CWL_File_array      = "File_array"      //A File object
	CWL_Directory_array = "Directory_array" //A Directory object
)

var Valid_cwltypes = map[string]bool{
	CWL_null:            true,
	CWL_boolean:         true,
	CWL_int:             true,
	CWL_long:            true,
	CWL_float:           true,
	CWL_double:          true,
	CWL_string:          true,
	CWL_File:            true,
	CWL_Directory:       true,
	CWL_stdout:          true,
	CWL_stderr:          true,
	CWL_boolean_array:   true,
	CWL_int_array:       true,
	CWL_long_array:      true,
	CWL_float_array:     true,
	CWL_double_array:    true,
	CWL_string_array:    true,
	CWL_File_array:      true,
	CWL_Directory_array: true,
}

var Native2Array = map[string]string{
	CWL_boolean:   CWL_boolean_array,
	CWL_int:       CWL_int_array,
	CWL_long:      CWL_long_array,
	CWL_float:     CWL_float_array,
	CWL_double:    CWL_double_array,
	CWL_string:    CWL_string_array,
	CWL_File:      CWL_File_array,
	CWL_Directory: CWL_Directory_array,
}

type CWL_array_type interface {
	Is_CWL_array_type()
	Get_Array() *[]CWLType
}

type CWL_minimal_interface interface {
	Is_CWL_minimal()
}

type CWL_minimal struct{}

func (c *CWL_minimal) Is_CWL_minimal() {}

// generic class to represent Files and Directories
type CWL_location interface {
	GetLocation() string
}

// CWLType - CWL basic types: int, string, boolean, .. etc
// http://www.commonwl.org/v1.0/CommandLineTool.html#CWLType
// null, boolean, int, long, float, double, string, File, Directory
type CWLType interface {
	CWL_object // is an interface
	Is_CommandInputParameterType()
	Is_CommandOutputParameterType()
	Is_CWLType()

	//Is_Array() bool
	//Is_CWL_minimal()
}

type CWLType_Impl struct{}

func (c *CWLType_Impl) Is_CWL_minimal()                {}
func (c *CWLType_Impl) Is_CWLType()                    {}
func (c *CWLType_Impl) Is_CommandInputParameterType()  {}
func (c *CWLType_Impl) Is_CommandOutputParameterType() {}

//func (c *CWLType_Impl) Is_Array() bool                 { return false }

func NewCWLType(id string, native interface{}) (cwl_type CWLType, err error) {

	//var cwl_type CWLType

	switch native.(type) {
	case []interface{}:

		native_array, _ := native.([]interface{})

		array, xerr := NewArray(id, native_array)
		if xerr != nil {
			err = xerr
			return
		}
		cwl_type = array
		return

	case int:
		native_int := native.(int)

		cwl_type = &Int{Value: native_int}

		//cwl_type = int_type.(*CWLType)

		return
		//temp = &Int{Value: native_int}

	case string:
		native_str := native.(string)

		cwl_type = NewString("", native_str)
	case bool:
		native_bool := native.(bool)

		cwl_type = &Boolean{Value: native_bool}

	case map[interface{}]interface{}:

		class, xerr := GetClass(native)
		if xerr != nil {
			err = xerr
			return
		}
		cwl_type, err = NewCWLTypeByClass(class, native)
		return

	case map[string]interface{}:
		//empty, xerr := NewEmpty(native)
		//if xerr != nil {
		//	err = xerr
		//	return
		//}
		class, xerr := GetClass(native)
		if xerr != nil {
			err = xerr
			return
		}
		cwl_type, err = NewCWLTypeByClass(class, native)
		return

	default:
		spew.Dump(native)
		err = fmt.Errorf("(NewCWLType) Type unknown: \"%s\"", reflect.TypeOf(native))
		return
	}
	//cwl_type_ptr = &cwl_type

	return

}

func NewCWLTypeByClass(class string, native interface{}) (cwl_type CWLType, err error) {
	switch class {
	case CWL_File:
		file, yerr := NewFile(native)
		cwl_type = &file
		if yerr != nil {
			err = yerr
			return
		}
	case CWL_string:
		mystring, yerr := NewStringFromInterface(native)
		cwl_type = mystring
		if yerr != nil {
			err = yerr
			return
		}
	default:
		// Map type unknown, maybe a record
		spew.Dump(native)

		record, xerr := NewRecord(native)
		if xerr != nil {
			err = xerr
			return
		}
		cwl_type = record
		return
	}
	return
}

func NewCWLTypeArray_deprecated(native interface{}) (cwl_array_ptr *[]CWLType, err error) {

	switch native.(type) {
	case []interface{}:

		native_array, ok := native.([]interface{})
		if !ok {
			err = fmt.Errorf("(NewCWLTypeArray) could not parse []interface{}")
			return
		}

		cwl_array := []CWLType{}

		for _, value := range native_array {
			value_cwl, xerr := NewCWLType("", value)
			if xerr != nil {
				err = xerr
				return
			}
			cwl_array = append(cwl_array, value_cwl)
		}
		cwl_array_ptr = &cwl_array
	default:

		ct, xerr := NewCWLType("", native)
		if xerr != nil {
			err = xerr
			return
		}

		cwl_array_ptr = &[]CWLType{ct}
	}

	return

}
<|endoftext|>


<commit_before>package runner

import (
	"fmt"
	"os"
	"os/exec"
	"time"

	"github.com/onsi/ginkgo"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

type Config struct {
	Name              string
	BinPath           string
	AnsiColorCode     string
	StartCheck        string
	StartCheckTimeout time.Duration
	Args              []string
}

type Runner struct {
	config Config
}

func New(config Config) *GexecRunner {
	return &Runner{
		config: Config,
	}
}

func (r *Runner) Run(sigChan <-chan os.Signal, ready chan<- struct{}) error {
	session, err := gexec.Start(
		exec.Command(
			r.BinPath,
			r.Args...,
		),
		gexec.NewPrefixedWriter("\x1b[32m[o]\x1b[%s[%s]\x1b[0m ", r.AnsiColorCode, r.Name, ginkgo.GinkgoWriter),
		gexec.NewPrefixedWriter("\x1b[91m[e]\x1b[%s[%s]\x1b[0m ", r.AnsiColorCode, r.Name, ginkgo.GinkgoWriter),
	)

	Ω(err).ShouldNot(HaveOccurred())

	if r.StartCheck != "" {
		Eventually(r.Session, r.StartCheckTimeout).Should(gbytes.Say(r.StartCheck))
	}

	close(ready)

	var signal os.Signal

	for {
		select {

		case signal = <-sigChan:
			session.Signal(signal)

		case <-session.Exited:
			if session.ExitCode() == 0 {
				return nil
			}
			return fmt.Errorf("exit status %d", session.ExitCode())
		}
	}
}
<commit_msg>fix up ginkgomon<commit_after>package ginkgomon

import (
	"fmt"
	"os"
	"os/exec"
	"time"

	"github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

type Runner struct {
	Name              string
	BinPath           string
	AnsiColorCode     string
	StartCheck        string
	StartCheckTimeout time.Duration
	Args              []string
}

func (r *Runner) Run(sigChan <-chan os.Signal, ready chan<- struct{}) error {
	session, err := gexec.Start(
		exec.Command(
			r.BinPath,
			r.Args...,
		),
		gexec.NewPrefixedWriter(fmt.Sprintf("\x1b[32m[o]\x1b[%s[%s]\x1b[0m ", r.AnsiColorCode, r.Name), ginkgo.GinkgoWriter),
		gexec.NewPrefixedWriter(fmt.Sprintf("\x1b[91m[e]\x1b[%s[%s]\x1b[0m ", r.AnsiColorCode, r.Name), ginkgo.GinkgoWriter),
	)

	Ω(err).ShouldNot(HaveOccurred())

	if r.StartCheck != "" {
		Eventually(session, r.StartCheckTimeout).Should(gbytes.Say(r.StartCheck))
	}

	close(ready)

	var signal os.Signal

	for {
		select {

		case signal = <-sigChan:
			session.Signal(signal)

		case <-session.Exited:
			if session.ExitCode() == 0 {
				return nil
			}
			return fmt.Errorf("exit status %d", session.ExitCode())
		}
	}
}
<|endoftext|>


<commit_before>// The Fakio Client write by go
// thank github.com/shadowsocks/shadowsocks-go

package main

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/sha256"
	//"encoding/binary"
	"errors"
	//"flag"
	"fmt"
	"io"
	//"io/ioutil"
	"log"
	"net"
	//"net/http"
	//"net/url"
	//"strconv"
	//"strings"
	//"time"
)

type Fclient struct {
	username string
	password string

	server string
	local  string
}

var fclient Fclient
var localReply []byte

// The Cipher
func stringToKey(str string) (key []byte) {
	hash := sha256.New()
	hash.Write([]byte(str))
	k := hash.Sum(nil)
	return k
}

type Cipher struct {
	enc cipher.Stream
	dec cipher.Stream
}

func NewCipher(bytes []byte) (c *Cipher, err error) {
	block, err := aes.NewCipher(bytes[32:])
	if err != nil {
		return c, err
	}

	enc := cipher.NewCFBEncrypter(block, bytes[0:16])
	dec := cipher.NewCFBDecrypter(block, bytes[16:32])

	return &Cipher{enc, dec}, nil
}

func (c *Cipher) Encrypt(dst, src []byte) {
	c.enc.XORKeyStream(dst, src)
}

func (c *Cipher) Decrypt(dst, src []byte) {
	c.dec.XORKeyStream(dst, src)
}

// The Fakio connection
type FakioConn struct {
	net.Conn
	*Cipher
}

func FakioDial(server string, req []byte) (c *FakioConn, err error) {
	conn, err := net.Dial("tcp", server)
	if err != nil {
		return nil, err
	}

	//handshake
	key := stringToKey(fclient.password)

	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	enc := cipher.NewCFBEncrypter(block, req[0:16])
	// 22 = iv + username
	enc.XORKeyStream(req[22:], req[22:])
	if _, err := conn.Write(req); err != nil {
		return nil, errors.New("handshake to server error")
	}

	hand := make([]byte, 64)
	if _, err := conn.Read(hand); err != nil {
		return nil, errors.New("handshake to server error")
	}

	dec := cipher.NewCFBDecrypter(block, hand[0:16])
	dec.XORKeyStream(hand[16:], hand[16:])

	cipher, err := NewCipher(hand[16:])

	return &FakioConn{conn, cipher}, nil
}

func (c *FakioConn) Read(b []byte) (n int, err error) {

	n, err = c.Conn.Read(b)
	c.Decrypt(b[0:n], b[0:n])

	return n, err
}

func (c *FakioConn) Write(b []byte) (n int, err error) {

	c.Encrypt(b, b)
	n, err = c.Conn.Write(b)

	return n, err
}

func buildFakioReq(buf []byte) (req []byte, err error) {
	req = make([]byte, 1024)

	iv := make([]byte, 16)
	_, err = rand.Read(iv)
	if err != nil {
		log.Println("rand iv error:", err)
		return
	}
	copy(req, iv)

	nameLen := len(fclient.username)
	index := 16 + 1 + nameLen
	req[16] = byte(nameLen)
	copy(req[17:index], fclient.username)
	req[index] = 0x5
	copy(req[index+1:], buf[3:])

	return req, nil
}

// SOCKS5 Handshake
// http://tools.ietf.org/rfc/rfc1928.txt
func socks5Handshake(conn net.Conn) (req []byte, err error) {

	var n int

	// 264 is max version identifier/method selection message len
	buf := make([]byte, 264)
	n, err = io.ReadAtLeast(conn, buf, 2)
	if err != nil {
		fmt.Println("155")
		return
	}

	// SOCKS VER
	if buf[0] != 0x05 {
		return req, errors.New("only support SOCKS5")
	}
	nmethod := int(buf[1])
	msgLen := nmethod + 2
	if n < msgLen {
		if _, err = io.ReadFull(conn, buf[n:msgLen]); err != nil {
			fmt.Println("167")
			return
		}
	}
	// send confirmation: version 5, no authentication required
	if _, err = conn.Write([]byte{0x05, 0}); err != nil {
		fmt.Println("173")
		return
	}

	// client request
	n, err = io.ReadAtLeast(conn, buf, 5)
	//fmt.Println(n)
	if err != nil {
		fmt.Println("180")
		return
	}
	// check version and cmd
	if buf[0] != 0x05 || buf[1] != 0x01 {
		return req, errors.New("only support socks5 tcp connect")
	}

	//address type: now not support ipv6
	var reqLen = 0
	if buf[3] == 0x01 {
		reqLen = 3 + 1 + net.IPv4len + 2
	} else if buf[3] == 0x03 {
		reqLen = 3 + 1 + 1 + 2 + int(buf[4])
	} else {
		return req, errors.New("address type: only support ipv4 and domain name")
	}

	if n < reqLen {
		if _, err = io.ReadFull(conn, buf[n:reqLen]); err != nil {
			fmt.Println("200")
			return
		}
	}

	req, err = buildFakioReq(buf)
	if err != nil {
		fmt.Println("207")
		return
	}

	//SOCKS5 Replies
	_, err = conn.Write(localReply)
	if err != nil {
		log.Println("send connection confirmation:", err)
		return
	}

	return req, nil
}

// Server
func handleConnection(client net.Conn) {
	log.Println("new connection....")

	defer func() {
		client.Close()
	}()

	req, err := socks5Handshake(client)
	if err != nil {
		log.Println("socks handshake:", err)
		return
	}

	remote, err := FakioDial(fclient.server, req)
	if err != nil {
		log.Println("Failed connect to fakio server: ", err)
		return
	}
	defer func() {
		remote.Close()
	}()

	go func(dst, src net.Conn) {
		defer func() {
			remote.Close()
		}()
		buf := make([]byte, 1024)

		for {
			n, err := src.Read(buf)

			if n > 0 {
				if _, err = dst.Write(buf[0:n]); err != nil {
					log.Println("write to remote:", err)
					break
				}
			}
			if err != nil {
				break
			}
		}
	}(remote, client)

	buf := make([]byte, 1024)

	for {
		n, err := remote.Read(buf)

		if n > 0 {
			if _, err = client.Write(buf[0:n]); err != nil {
				log.Println("write to client:", err)
				break
			}
		}
		if err != nil {
			fmt.Println("259:", err, " ", n)
			break
		}
	}

	log.Println("closed connection")

}

func run(listenAddr string) {
	ln, err := net.Listen("tcp", listenAddr)
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("starting fakio client(socks5 server) at %v ...\n", listenAddr)
	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Println("accept:", err)
			continue
		}
		go handleConnection(conn)
	}
}

func main() {
	fclient = Fclient{"serho", "123456", "localhost:8888", "127.0.0.1:1080"}
	localReply = []byte{0x05, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x7f, 0x08, 0x43}
	run(fclient.local)
}
<commit_msg>add config file<commit_after>// The Fakio Client write by go
// thank github.com/shadowsocks/shadowsocks-go

package main

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/sha256"
	"encoding/binary"
	"encoding/json"
	"errors"
	//"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	//"net/http"
	//"net/url"
	"strconv"
	//"strings"
	//"time"
)

type Fclient struct {
	UserName string
	PassWord string

	Server string
	Local  string
}

var fclient Fclient
var localReply []byte

// The Cipher
func stringToKey(str string) (key []byte) {
	hash := sha256.New()
	hash.Write([]byte(str))
	k := hash.Sum(nil)
	return k
}

type Cipher struct {
	enc cipher.Stream
	dec cipher.Stream
}

func NewCipher(bytes []byte) (c *Cipher, err error) {
	block, err := aes.NewCipher(bytes[32:])
	if err != nil {
		return c, err
	}

	enc := cipher.NewCFBEncrypter(block, bytes[0:16])
	dec := cipher.NewCFBDecrypter(block, bytes[16:32])

	return &Cipher{enc, dec}, nil
}

func (c *Cipher) Encrypt(dst, src []byte) {
	c.enc.XORKeyStream(dst, src)
}

func (c *Cipher) Decrypt(dst, src []byte) {
	c.dec.XORKeyStream(dst, src)
}

// The Fakio connection
type FakioConn struct {
	net.Conn
	*Cipher
}

func FakioDial(server string, req []byte) (c *FakioConn, err error) {
	conn, err := net.Dial("tcp", server)
	if err != nil {
		return nil, err
	}

	//handshake
	key := stringToKey(fclient.PassWord)

	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	enc := cipher.NewCFBEncrypter(block, req[0:16])
	// 22 = iv + username
	enc.XORKeyStream(req[22:], req[22:])
	if _, err := conn.Write(req); err != nil {
		return nil, errors.New("handshake to server error")
	}

	hand := make([]byte, 64)
	if _, err := conn.Read(hand); err != nil {
		return nil, errors.New("handshake to server error")
	}

	dec := cipher.NewCFBDecrypter(block, hand[0:16])
	dec.XORKeyStream(hand[16:], hand[16:])

	cipher, err := NewCipher(hand[16:])

	return &FakioConn{conn, cipher}, nil
}

func (c *FakioConn) Read(b []byte) (n int, err error) {

	n, err = c.Conn.Read(b)
	c.Decrypt(b[0:n], b[0:n])

	return n, err
}

func (c *FakioConn) Write(b []byte) (n int, err error) {

	c.Encrypt(b, b)
	n, err = c.Conn.Write(b)

	return n, err
}

func buildFakioReq(buf []byte) (req []byte, err error) {
	req = make([]byte, 1024)

	iv := make([]byte, 16)
	_, err = rand.Read(iv)
	if err != nil {
		log.Println("rand iv error:", err)
		return
	}
	copy(req, iv)

	nameLen := len(fclient.UserName)
	index := 16 + 1 + nameLen
	req[16] = byte(nameLen)
	copy(req[17:index], fclient.UserName)
	req[index] = 0x5
	copy(req[index+1:], buf[3:])

	return req, nil
}

// SOCKS5 Handshake
// http://tools.ietf.org/rfc/rfc1928.txt
func socks5Handshake(conn net.Conn) (req []byte, err error) {

	var n int

	// 264 is max version identifier/method selection message len
	buf := make([]byte, 264)
	n, err = io.ReadAtLeast(conn, buf, 2)
	if err != nil {
		fmt.Println("155")
		return
	}

	// SOCKS VER
	if buf[0] != 0x05 {
		return req, errors.New("only support SOCKS5")
	}
	nmethod := int(buf[1])
	msgLen := nmethod + 2
	if n < msgLen {
		if _, err = io.ReadFull(conn, buf[n:msgLen]); err != nil {
			fmt.Println("167")
			return
		}
	}
	// send confirmation: version 5, no authentication required
	if _, err = conn.Write([]byte{0x05, 0}); err != nil {
		fmt.Println("173")
		return
	}

	// client request
	n, err = io.ReadAtLeast(conn, buf, 5)
	//fmt.Println(n)
	if err != nil {
		fmt.Println("180")
		return
	}
	// check version and cmd
	if buf[0] != 0x05 || buf[1] != 0x01 {
		return req, errors.New("only support socks5 tcp connect")
	}

	//address type: now not support ipv6
	var reqLen = 0
	if buf[3] == 0x01 {
		reqLen = 3 + 1 + net.IPv4len + 2
	} else if buf[3] == 0x03 {
		reqLen = 3 + 1 + 1 + 2 + int(buf[4])
	} else {
		return req, errors.New("address type: only support ipv4 and domain name")
	}

	if n < reqLen {
		if _, err = io.ReadFull(conn, buf[n:reqLen]); err != nil {
			fmt.Println("200")
			return
		}
	}

	req, err = buildFakioReq(buf)
	if err != nil {
		return
	}

	//log request
	var host string
	port := binary.BigEndian.Uint16(buf[reqLen-2 : reqLen])
	if buf[3] == 0x01 {
		host = net.IP(buf[4:8]).String()
	}
	if buf[3] == 0x03 {
		host = string(buf[5 : 5+int(buf[4])])
	}

	addr := net.JoinHostPort(host, strconv.Itoa(int(port)))
	log.Printf("connect %s", addr)

	//SOCKS5 Replies
	_, err = conn.Write(localReply)
	if err != nil {
		log.Println("send connection confirmation:", err)
		return
	}

	return req, nil
}

// Server
func handleConnection(client net.Conn) {
	log.Println("new connection....")

	defer func() {
		client.Close()
	}()

	req, err := socks5Handshake(client)
	if err != nil {
		log.Println("socks handshake:", err)
		return
	}

	remote, err := FakioDial(fclient.Server, req)
	if err != nil {
		log.Println("Failed connect to fakio server: ", err)
		return
	}
	defer func() {
		remote.Close()
	}()

	go func(dst, src net.Conn) {
		defer func() {
			client.Close()
			remote.Close()
		}()
		buf := make([]byte, 1024)

		for {
			n, err := src.Read(buf)

			if n > 0 {
				if _, err = dst.Write(buf[0:n]); err != nil {
					log.Println("write to remote:", err)
					break
				}
			}
			if err != nil {
				break
			}
		}
	}(remote, client)

	buf := make([]byte, 1024)

	for {
		n, err := remote.Read(buf)

		if n > 0 {
			if _, err = client.Write(buf[0:n]); err != nil {
				log.Println("write to client:", err)
				break
			}
		}
		if err != nil {
			fmt.Println("259:", err, " ", n)
			break
		}
	}

	log.Println("closed connection")

}

func run(listenAddr string) {
	ln, err := net.Listen("tcp", listenAddr)
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("starting fakio client(socks5 server) at %v ...\n", listenAddr)
	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Println("accept:", err)
			continue
		}
		go handleConnection(conn)
	}
}

func getConfig(path string, fclient *Fclient) (err error) {
	file, err := os.Open(path) // For read access.
	if err != nil {
		return
	}
	defer file.Close()

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return
	}

	err = json.Unmarshal(data, fclient)
	return
}

func buildReply(addr string) (buf []byte, err error) {

	buf = make([]byte, 264)

	host, portStr, err := net.SplitHostPort(addr)
	if err != nil {
		return nil, errors.New(
			fmt.Sprintf("fakio: address error %s %v", addr, err))
	}
	port, err := strconv.Atoi(portStr)
	if err != nil {
		return nil, errors.New(
			fmt.Sprintf("fakio: invalid port %s", addr))
	}

	buf[0] = 0x05
	buf[1] = 0x00
	buf[2] = 0x00

	ip := net.ParseIP(host).To4()
	if ip != nil {
		buf[3] = 0x01
		buf[4] = ip[3]
		buf[5] = ip[2]
		buf[6] = ip[1]
		buf[7] = ip[0]
		binary.BigEndian.PutUint16(buf[8:], uint16(port))
		return buf[0:10], nil
	}

	hostLen := len(host)
	buf[3] = 0x03
	buf[4] = byte(hostLen)

	copy(buf[5:], host)
	binary.BigEndian.PutUint16(buf[5+hostLen:], uint16(port))

	return buf[0 : 7+hostLen], nil
}

func main() {
	if err := getConfig("config.json", &fclient); err != nil {
		log.Fatalf("get config error: %s", err)
	}
	log.Printf("use config: %s", fclient)

	var err error
	localReply, err = buildReply(fclient.Local)
	if err != nil {
		log.Fatalf("fakio start error: %s", err)
	}

	run(fclient.Local)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/helper/config"
	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/packer/plugin"
	"github.com/mitchellh/packer/template/interpolate"
	"os"
	"strings"
)

const TeamcityVersionEnvVar = "TEAMCITY_VERSION"

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	OutputPath string `mapstructure:"output"`
	StripPath  bool   `mapstructure:"strip_path"`
	ctx        interpolate.Context
}

type PostProcessor struct {
	config Config
}

type ManifestFile struct {
	Builds      []Artifact `json:"builds"`
	LastRunUUID string     `json:"last_run_uuid"`
}

func (p *PostProcessor) Configure(raws ...interface{}) error {
	err := config.Decode(&p.config, &config.DecodeOpts{
		Interpolate:        true,
		InterpolateContext: &p.config.ctx,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{},
		},
	}, raws...)
	if err != nil {
		return err
	}

	if p.config.OutputPath == "" {
		p.config.OutputPath = "packer-manifest.json"
	}

	if err = interpolate.Validate(p.config.OutputPath, &p.config.ctx); err != nil {
		return fmt.Errorf("Error parsing target template: %s", err)
	}

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, source packer.Artifact) (packer.Artifact, bool, error) {
	if os.Getenv(TeamcityVersionEnvVar) != "" {
		if source.BuilderId() == "aws" {
			s := strings.Split(source.Id(), ":")
			region, ami := s[0], s[1] // TODO: several AMIs
			ui.Message(fmt.Sprintf("##teamcity[setParameter name='packer.artifact.aws.region' value='%v']", region))
			ui.Message(fmt.Sprintf("##teamcity[setParameter name='packer.artifact.aws.ami' value='%v']", ami))
		} else {
			ui.Message(fmt.Sprintf("##teamcity[setParameter name='packer.artifact.id' value='%v']", source.Id()))
		}
	}
	return source, true, nil
}

func main() {
	server, err := plugin.Server()
	if err != nil {
		panic(err)
	}
	server.RegisterPostProcessor(new(PostProcessor))
	server.Serve()
}
<commit_msg>Updated references; added support for actual amazon id's<commit_after>package main

import (
	"fmt"

	"github.com/hashicorp/packer/common"
	"github.com/hashicorp/packer/helper/config"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/packer/plugin"
	"github.com/hashicorp/packer/template/interpolate"
	"os"
	"strings"
)

const TeamcityVersionEnvVar = "TEAMCITY_VERSION"
var AmazonBuilderIds = []string {
	"mitchellh.amazonebs",
	"mitchellh.amazon.ebssurrogate",
	"mitchellh.amazon.instance",
	"mitchellh.amazon.chroot",
}

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	OutputPath string `mapstructure:"output"`
	StripPath  bool   `mapstructure:"strip_path"`
	ctx        interpolate.Context
}

type PostProcessor struct {
	config Config
}

type ManifestFile struct {
	Builds      []Artifact `json:"builds"`
	LastRunUUID string     `json:"last_run_uuid"`
}

func Contains(vs []string, t string) bool {
	for _, v := range vs {
		if v == t {
			return true
		}
	}
	return false
}

func (p *PostProcessor) Configure(raws ...interface{}) error {
	err := config.Decode(&p.config, &config.DecodeOpts{
		Interpolate:        true,
		InterpolateContext: &p.config.ctx,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{},
		},
	}, raws...)
	if err != nil {
		return err
	}

	if p.config.OutputPath == "" {
		p.config.OutputPath = "packer-manifest.json"
	}

	if err = interpolate.Validate(p.config.OutputPath, &p.config.ctx); err != nil {
		return fmt.Errorf("Error parsing target template: %s", err)
	}

	return nil
}

func (p *PostProcessor) PostProcess(ui packer.Ui, source packer.Artifact) (packer.Artifact, bool, error) {
	if os.Getenv(TeamcityVersionEnvVar) != "" {
		if Contains(AmazonBuilderIds, source.BuilderId())  {
			s := strings.Split(source.Id(), ":")
			region, ami := s[0], s[1] // TODO: several AMIs
			ui.Message(fmt.Sprintf("##teamcity[setParameter name='packer.artifact.aws.region' value='%v']", region))
			ui.Message(fmt.Sprintf("##teamcity[setParameter name='packer.artifact.aws.ami' value='%v']", ami))
		} else {
			ui.Message(fmt.Sprintf("##teamcity[setParameter name='packer.artifact.id' value='%v']", source.Id()))
		}
	}
	return source, true, nil
}

func main() {
	server, err := plugin.Server()
	if err != nil {
		panic(err)
	}
	server.RegisterPostProcessor(new(PostProcessor))
	server.Serve()
}
<|endoftext|>


<commit_before>/*
 * MinIO Client (C) 2019 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"bytes"
	"fmt"
	"hash/fnv"
	"net/http"
	"strings"
	"time"

	humanize "github.com/dustin/go-humanize"
	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/mc/pkg/colorjson"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/minio/pkg/madmin"
)

var adminTraceFlags = []cli.Flag{
	cli.BoolFlag{
		Name:  "verbose, v",
		Usage: "print verbose trace",
	},
	cli.BoolFlag{
		Name:  "all, a",
		Usage: "trace all traffic (including internode traffic between MinIO servers)",
	},
	cli.BoolFlag{
		Name:  "errors, e",
		Usage: "trace failed requests only",
	},
}

var adminTraceCmd = cli.Command{
	Name:            "trace",
	Usage:           "show http trace for minio server",
	Action:          mainAdminTrace,
	Before:          setGlobalsFromContext,
	Flags:           append(adminTraceFlags, globalFlags...),
	HideHelpCommand: true,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [FLAGS] TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Show console trace for a Minio server with alias 'play'
		$ {{.HelpName}} play -v -a

  2. Show trace only for failed requests for a Minio server with alias 'myminio'
		$ {{.HelpName}} myminio -v -e
 `,
}

const timeFormat = "15:04:05.000"

var (
	colors = []color.Attribute{color.FgCyan, color.FgWhite, color.FgYellow, color.FgGreen}
)

func checkAdminTraceSyntax(ctx *cli.Context) {
	if len(ctx.Args()) != 1 {
		cli.ShowCommandHelpAndExit(ctx, "trace", 1) // last argument is exit code
	}
}

// mainAdminTrace - the entry function of trace command
func mainAdminTrace(ctx *cli.Context) error {
	// Check for command syntax
	checkAdminTraceSyntax(ctx)
	verbose := ctx.Bool("verbose")
	all := ctx.Bool("all")
	errfltr := ctx.Bool("errors")
	aliasedURL := ctx.Args().Get(0)
	console.SetColor("Stat", color.New(color.FgYellow))

	console.SetColor("Request", color.New(color.FgCyan))
	console.SetColor("Method", color.New(color.Bold, color.FgWhite))
	console.SetColor("Host", color.New(color.Bold, color.FgGreen))
	console.SetColor("FuncName", color.New(color.Bold, color.FgGreen))

	console.SetColor("ReqHeaderKey", color.New(color.Bold, color.FgWhite))
	console.SetColor("RespHeaderKey", color.New(color.Bold, color.FgCyan))
	console.SetColor("HeaderValue", color.New(color.FgWhite))
	console.SetColor("RespStatus", color.New(color.Bold, color.FgYellow))
	console.SetColor("ErrStatus", color.New(color.Bold, color.FgRed))

	console.SetColor("Response", color.New(color.FgGreen))
	console.SetColor("Body", color.New(color.FgYellow))
	for _, c := range colors {
		console.SetColor(fmt.Sprintf("Node%d", c), color.New(c))
	}
	// Create a new Minio Admin Client
	client, err := newAdminClient(aliasedURL)
	if err != nil {
		fatalIf(err.Trace(aliasedURL), "Cannot initialize admin client.")
		return nil
	}
	doneCh := make(chan struct{})
	defer close(doneCh)

	// Start listening on all trace activity.
	traceCh := client.Trace(all, errfltr, doneCh)
	for traceInfo := range traceCh {
		if traceInfo.Err != nil {
			fatalIf(probe.NewError(traceInfo.Err), "Cannot listen to http trace")
		}
		if verbose {
			printMsg(traceMessage{traceInfo})
			continue
		}
		printMsg(shortTrace(traceInfo))
	}
	return nil
}

// Short trace record
type shortTraceMsg struct {
	Host       string    `json:"host"`
	Time       time.Time `json:"time"`
	Client     string    `json:"client"`
	CallStats  callStats `json:"callstats"`
	FuncName   string    `json:"api"`
	Path       string    `json:"path"`
	Query      string    `json:"query"`
	StatusCode int       `json:"statuscode"`
	StatusMsg  string    `json:"statusmsg"`
}

type traceMessage struct {
	madmin.TraceInfo
}

type requestInfo struct {
	Time     time.Time         `json:"time"`
	Method   string            `json:"method"`
	Path     string            `json:"path,omitempty"`
	RawQuery string            `json:"rawquery,omitempty"`
	Headers  map[string]string `json:"headers,omitempty"`
	Body     string            `json:"body,omitempty"`
}

type responseInfo struct {
	Time       time.Time         `json:"time"`
	Headers    map[string]string `json:"headers,omitempty"`
	Body       string            `json:"body,omitempty"`
	StatusCode int               `json:"statuscode,omitempty"`
}
type callStats struct {
	Rx       int           `json:"rx"`
	Tx       int           `json:"tx"`
	Duration time.Duration `json:"duration"`
}

type trace struct {
	NodeName     string       `json:"host"`
	FuncName     string       `json:"api"`
	RequestInfo  requestInfo  `json:"request"`
	ResponseInfo responseInfo `json:"response"`
	CallStats    callStats    `json:"callstats"`
}

// return a struct with minimal trace info.
func shortTrace(ti madmin.TraceInfo) shortTraceMsg {
	s := shortTraceMsg{}
	t := ti.Trace
	s.Time = t.ReqInfo.Time
	if host, ok := t.ReqInfo.Headers["Host"]; ok {
		s.Host = strings.Join(host, "")
	}
	s.Path = t.ReqInfo.Path
	s.Query = t.ReqInfo.RawQuery
	s.FuncName = t.FuncName
	s.StatusCode = t.RespInfo.StatusCode
	s.StatusMsg = http.StatusText(t.RespInfo.StatusCode)
	cSlice := strings.Split(t.ReqInfo.Client, ":")
	s.Client = cSlice[0]
	s.CallStats.Duration = t.CallStats.Latency
	s.CallStats.Rx = t.CallStats.InputBytes
	s.CallStats.Tx = t.CallStats.OutputBytes
	return s
}

func (s shortTraceMsg) JSON() string {
	buf := &bytes.Buffer{}
	enc := json.NewEncoder(buf)
	enc.SetIndent("", " ")
	// Disable escaping special chars to display XML tags correctly
	enc.SetEscapeHTML(false)

	fatalIf(probe.NewError(enc.Encode(s)), "Unable to marshal into JSON.")
	return buf.String()
}

func (s shortTraceMsg) String() string {
	var hostStr string
	var b = &strings.Builder{}

	if s.Host != "" {
		hostStr = colorizedNodeName(s.Host)
	}
	fmt.Fprintf(b, "%s ", s.Time.Format(timeFormat))
	statusStr := console.Colorize("RespStatus", fmt.Sprintf("%d %s", s.StatusCode, s.StatusMsg))
	if s.StatusCode != http.StatusOK {
		statusStr = console.Colorize("ErrStatus", fmt.Sprintf("%d %s", s.StatusCode, s.StatusMsg))
	}
	fmt.Fprintf(b, "[%s] %s ", statusStr, console.Colorize("FuncName", s.FuncName))
	fmt.Fprintf(b, "%s%s", hostStr, s.Path)

	if s.Query != "" {
		fmt.Fprintf(b, "?%s ", s.Query)
	}
	fmt.Fprintf(b, " %s ", s.Client)
	fmt.Fprint(b, console.Colorize("HeaderValue", fmt.Sprintf("%2s ", s.CallStats.Duration.Round(time.Microsecond))))
	fmt.Fprint(b, console.Colorize("Stat", fmt.Sprintf(" 🠉 ")))
	fmt.Fprint(b, console.Colorize("HeaderValue", humanize.IBytes(uint64(s.CallStats.Rx))))
	fmt.Fprint(b, console.Colorize("Stat", fmt.Sprintf("  🠋 ")))
	fmt.Fprint(b, console.Colorize("HeaderValue", humanize.IBytes(uint64(s.CallStats.Tx))))

	return b.String()
}

// colorize node name
func colorizedNodeName(nodeName string) string {
	nodeHash := fnv.New32a()
	nodeHash.Write([]byte(nodeName))
	nHashSum := nodeHash.Sum32()
	idx := uint32(nHashSum) % uint32(len(colors))
	return console.Colorize(fmt.Sprintf("Node%d", colors[idx]), nodeName)
}

func (t traceMessage) JSON() string {
	rqHdrs := make(map[string]string)
	rspHdrs := make(map[string]string)
	rq := t.Trace.ReqInfo
	rs := t.Trace.RespInfo
	for k, v := range rq.Headers {
		rqHdrs[k] = strings.Join(v, " ")
	}
	for k, v := range rs.Headers {
		rspHdrs[k] = strings.Join(v, " ")
	}
	trc := trace{
		NodeName: t.Trace.NodeName,
		FuncName: t.Trace.FuncName,
		RequestInfo: requestInfo{
			Time:     rq.Time,
			Method:   rq.Method,
			Path:     rq.Path,
			RawQuery: rq.RawQuery,
			Body:     string(rq.Body),
			Headers:  rqHdrs,
		},
		ResponseInfo: responseInfo{
			Time:       rs.Time,
			Body:       string(rs.Body),
			Headers:    rspHdrs,
			StatusCode: rs.StatusCode,
		},
		CallStats: callStats{
			Duration: t.Trace.CallStats.Latency,
			Rx:       t.Trace.CallStats.InputBytes,
			Tx:       t.Trace.CallStats.OutputBytes,
		},
	}
	buf := &bytes.Buffer{}
	enc := json.NewEncoder(buf)
	enc.SetIndent("", " ")
	// Disable escaping special chars to display XML tags correctly
	enc.SetEscapeHTML(false)
	fatalIf(probe.NewError(enc.Encode(trc)), "Unable to marshal into JSON.")

	// strip off extra newline added by json encoder
	return strings.TrimSuffix(buf.String(), "\n")
}

func (t traceMessage) String() string {
	var nodeNameStr string
	var b = &strings.Builder{}

	trc := t.Trace
	if trc.NodeName != "" {
		nodeNameStr = fmt.Sprintf("%s ", colorizedNodeName(trc.NodeName))
	}

	ri := trc.ReqInfo
	rs := trc.RespInfo
	fmt.Fprintf(b, "%s%s", nodeNameStr, console.Colorize("Request", fmt.Sprintf("[REQUEST %s] ", trc.FuncName)))
	fmt.Fprintf(b, "%s\n", ri.Time.Format(timeFormat))
	fmt.Fprintf(b, "%s%s", nodeNameStr, console.Colorize("Method", fmt.Sprintf("%s %s", ri.Method, ri.Path)))
	if ri.RawQuery != "" {
		fmt.Fprintf(b, "?%s", ri.RawQuery)
	}
	fmt.Fprint(b, "\n")
	host, ok := ri.Headers["Host"]
	if ok {
		delete(ri.Headers, "Host")
	}
	hostStr := strings.Join(host, "")
	fmt.Fprintf(b, "%s%s", nodeNameStr, console.Colorize("Host", fmt.Sprintf("Host: %s\n", hostStr)))
	for k, v := range ri.Headers {
		fmt.Fprintf(b, "%s%s", nodeNameStr, console.Colorize("ReqHeaderKey",
			fmt.Sprintf("%s: ", k))+console.Colorize("HeaderValue", fmt.Sprintf("%s\n", strings.Join(v, ""))))
	}

	fmt.Fprintf(b, "%s%s", nodeNameStr, console.Colorize("Body", fmt.Sprintf("%s\n", string(ri.Body))))
	fmt.Fprintf(b, "%s%s", nodeNameStr, console.Colorize("Response", fmt.Sprintf("[RESPONSE] ")))
	fmt.Fprintf(b, "[%s] ", rs.Time.Format(timeFormat))
	fmt.Fprint(b, console.Colorize("Stat", fmt.Sprintf("[ Duration %2s  🠉 %s  🠋 %s ]\n", trc.CallStats.Latency.Round(time.Microsecond), humanize.IBytes(uint64(trc.CallStats.InputBytes)), humanize.IBytes(uint64(trc.CallStats.OutputBytes)))))

	statusStr := console.Colorize("RespStatus", fmt.Sprintf("%d %s", rs.StatusCode, http.StatusText(rs.StatusCode)))
	if rs.StatusCode != http.StatusOK {
		statusStr = console.Colorize("ErrStatus", fmt.Sprintf("%d %s", rs.StatusCode, http.StatusText(rs.StatusCode)))
	}
	fmt.Fprintf(b, "%s%s\n", nodeNameStr, statusStr)

	for k, v := range rs.Headers {
		fmt.Fprintf(b, "%s%s", nodeNameStr, console.Colorize("RespHeaderKey",
			fmt.Sprintf("%s: ", k))+console.Colorize("HeaderValue", fmt.Sprintf("%s\n", strings.Join(v, ""))))
	}
	fmt.Fprintf(b, "%s%s\n", nodeNameStr, console.Colorize("Body", string(rs.Body)))
	fmt.Fprint(b, nodeNameStr)
	return b.String()
}
<commit_msg>fix formatting for admin trace (#2854)<commit_after>/*
 * MinIO Client (C) 2019 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"bytes"
	"fmt"
	"hash/fnv"
	"net/http"
	"strings"
	"time"

	humanize "github.com/dustin/go-humanize"
	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/mc/pkg/colorjson"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/minio/pkg/madmin"
)

var adminTraceFlags = []cli.Flag{
	cli.BoolFlag{
		Name:  "verbose, v",
		Usage: "print verbose trace",
	},
	cli.BoolFlag{
		Name:  "all, a",
		Usage: "trace all traffic (including internode traffic between MinIO servers)",
	},
	cli.BoolFlag{
		Name:  "errors, e",
		Usage: "trace failed requests only",
	},
}

var adminTraceCmd = cli.Command{
	Name:            "trace",
	Usage:           "show http trace for minio server",
	Action:          mainAdminTrace,
	Before:          setGlobalsFromContext,
	Flags:           append(adminTraceFlags, globalFlags...),
	HideHelpCommand: true,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [FLAGS] TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Show console trace for a Minio server with alias 'play'
		$ {{.HelpName}} play -v -a

  2. Show trace only for failed requests for a Minio server with alias 'myminio'
		$ {{.HelpName}} myminio -v -e
 `,
}

const timeFormat = "15:04:05.000"

var (
	colors = []color.Attribute{color.FgCyan, color.FgWhite, color.FgYellow, color.FgGreen}
)

func checkAdminTraceSyntax(ctx *cli.Context) {
	if len(ctx.Args()) != 1 {
		cli.ShowCommandHelpAndExit(ctx, "trace", 1) // last argument is exit code
	}
}

// mainAdminTrace - the entry function of trace command
func mainAdminTrace(ctx *cli.Context) error {
	// Check for command syntax
	checkAdminTraceSyntax(ctx)
	verbose := ctx.Bool("verbose")
	all := ctx.Bool("all")
	errfltr := ctx.Bool("errors")
	aliasedURL := ctx.Args().Get(0)
	console.SetColor("Stat", color.New(color.FgYellow))

	console.SetColor("Request", color.New(color.FgCyan))
	console.SetColor("Method", color.New(color.Bold, color.FgWhite))
	console.SetColor("Host", color.New(color.Bold, color.FgGreen))
	console.SetColor("FuncName", color.New(color.Bold, color.FgGreen))

	console.SetColor("ReqHeaderKey", color.New(color.Bold, color.FgWhite))
	console.SetColor("RespHeaderKey", color.New(color.Bold, color.FgCyan))
	console.SetColor("HeaderValue", color.New(color.FgWhite))
	console.SetColor("RespStatus", color.New(color.Bold, color.FgYellow))
	console.SetColor("ErrStatus", color.New(color.Bold, color.FgRed))

	console.SetColor("Response", color.New(color.FgGreen))
	console.SetColor("Body", color.New(color.FgYellow))
	for _, c := range colors {
		console.SetColor(fmt.Sprintf("Node%d", c), color.New(c))
	}
	// Create a new Minio Admin Client
	client, err := newAdminClient(aliasedURL)
	if err != nil {
		fatalIf(err.Trace(aliasedURL), "Cannot initialize admin client.")
		return nil
	}
	doneCh := make(chan struct{})
	defer close(doneCh)

	// Start listening on all trace activity.
	traceCh := client.Trace(all, errfltr, doneCh)
	for traceInfo := range traceCh {
		if traceInfo.Err != nil {
			fatalIf(probe.NewError(traceInfo.Err), "Cannot listen to http trace")
		}
		if verbose {
			printMsg(traceMessage{traceInfo})
			continue
		}
		printMsg(shortTrace(traceInfo))
	}
	return nil
}

// Short trace record
type shortTraceMsg struct {
	Host       string    `json:"host"`
	Time       time.Time `json:"time"`
	Client     string    `json:"client"`
	CallStats  callStats `json:"callstats"`
	FuncName   string    `json:"api"`
	Path       string    `json:"path"`
	Query      string    `json:"query"`
	StatusCode int       `json:"statuscode"`
	StatusMsg  string    `json:"statusmsg"`
}

type traceMessage struct {
	madmin.TraceInfo
}

type requestInfo struct {
	Time     time.Time         `json:"time"`
	Method   string            `json:"method"`
	Path     string            `json:"path,omitempty"`
	RawQuery string            `json:"rawquery,omitempty"`
	Headers  map[string]string `json:"headers,omitempty"`
	Body     string            `json:"body,omitempty"`
}

type responseInfo struct {
	Time       time.Time         `json:"time"`
	Headers    map[string]string `json:"headers,omitempty"`
	Body       string            `json:"body,omitempty"`
	StatusCode int               `json:"statuscode,omitempty"`
}
type callStats struct {
	Rx       int           `json:"rx"`
	Tx       int           `json:"tx"`
	Duration time.Duration `json:"duration"`
}

type trace struct {
	NodeName     string       `json:"host"`
	FuncName     string       `json:"api"`
	RequestInfo  requestInfo  `json:"request"`
	ResponseInfo responseInfo `json:"response"`
	CallStats    callStats    `json:"callstats"`
}

// return a struct with minimal trace info.
func shortTrace(ti madmin.TraceInfo) shortTraceMsg {
	s := shortTraceMsg{}
	t := ti.Trace
	s.Time = t.ReqInfo.Time
	if host, ok := t.ReqInfo.Headers["Host"]; ok {
		s.Host = strings.Join(host, "")
	}
	s.Path = t.ReqInfo.Path
	s.Query = t.ReqInfo.RawQuery
	s.FuncName = t.FuncName
	s.StatusCode = t.RespInfo.StatusCode
	s.StatusMsg = http.StatusText(t.RespInfo.StatusCode)
	cSlice := strings.Split(t.ReqInfo.Client, ":")
	s.Client = cSlice[0]
	s.CallStats.Duration = t.CallStats.Latency
	s.CallStats.Rx = t.CallStats.InputBytes
	s.CallStats.Tx = t.CallStats.OutputBytes
	return s
}

func (s shortTraceMsg) JSON() string {
	buf := &bytes.Buffer{}
	enc := json.NewEncoder(buf)
	enc.SetIndent("", " ")
	// Disable escaping special chars to display XML tags correctly
	enc.SetEscapeHTML(false)

	fatalIf(probe.NewError(enc.Encode(s)), "Unable to marshal into JSON.")
	return buf.String()
}

func (s shortTraceMsg) String() string {
	var hostStr string
	var b = &strings.Builder{}

	if s.Host != "" {
		hostStr = colorizedNodeName(s.Host)
	}
	fmt.Fprintf(b, "%s ", s.Time.Format(timeFormat))
	statusStr := console.Colorize("RespStatus", fmt.Sprintf("%d %s", s.StatusCode, s.StatusMsg))
	if s.StatusCode >= http.StatusBadRequest {
		statusStr = console.Colorize("ErrStatus", fmt.Sprintf("%d %s", s.StatusCode, s.StatusMsg))
	}
	fmt.Fprintf(b, "[%s] %s ", statusStr, console.Colorize("FuncName", s.FuncName))
	fmt.Fprintf(b, "%s%s", hostStr, s.Path)

	if s.Query != "" {
		fmt.Fprintf(b, "?%s ", s.Query)
	}
	fmt.Fprintf(b, " %s ", s.Client)
	spaces := 15 - len(s.Client)
	fmt.Fprintf(b, "%*s", spaces, " ")
	fmt.Fprintf(b, console.Colorize("HeaderValue", fmt.Sprintf("  %2s", s.CallStats.Duration.Round(time.Microsecond))))
	spaces = 12 - len(fmt.Sprintf("%2s", s.CallStats.Duration.Round(time.Microsecond)))
	fmt.Fprintf(b, "%*s", spaces, " ")
	fmt.Fprintf(b, console.Colorize("Stat", fmt.Sprintf(" 🠉 ")))
	fmt.Fprintf(b, console.Colorize("HeaderValue", humanize.IBytes(uint64(s.CallStats.Rx))))
	fmt.Fprintf(b, console.Colorize("Stat", fmt.Sprintf("  🠋 ")))
	fmt.Fprintf(b, console.Colorize("HeaderValue", humanize.IBytes(uint64(s.CallStats.Tx))))

	return b.String()
}

// colorize node name
func colorizedNodeName(nodeName string) string {
	nodeHash := fnv.New32a()
	nodeHash.Write([]byte(nodeName))
	nHashSum := nodeHash.Sum32()
	idx := uint32(nHashSum) % uint32(len(colors))
	return console.Colorize(fmt.Sprintf("Node%d", colors[idx]), nodeName)
}

func (t traceMessage) JSON() string {
	rqHdrs := make(map[string]string)
	rspHdrs := make(map[string]string)
	rq := t.Trace.ReqInfo
	rs := t.Trace.RespInfo
	for k, v := range rq.Headers {
		rqHdrs[k] = strings.Join(v, " ")
	}
	for k, v := range rs.Headers {
		rspHdrs[k] = strings.Join(v, " ")
	}
	trc := trace{
		NodeName: t.Trace.NodeName,
		FuncName: t.Trace.FuncName,
		RequestInfo: requestInfo{
			Time:     rq.Time,
			Method:   rq.Method,
			Path:     rq.Path,
			RawQuery: rq.RawQuery,
			Body:     string(rq.Body),
			Headers:  rqHdrs,
		},
		ResponseInfo: responseInfo{
			Time:       rs.Time,
			Body:       string(rs.Body),
			Headers:    rspHdrs,
			StatusCode: rs.StatusCode,
		},
		CallStats: callStats{
			Duration: t.Trace.CallStats.Latency,
			Rx:       t.Trace.CallStats.InputBytes,
			Tx:       t.Trace.CallStats.OutputBytes,
		},
	}
	buf := &bytes.Buffer{}
	enc := json.NewEncoder(buf)
	enc.SetIndent("", " ")
	// Disable escaping special chars to display XML tags correctly
	enc.SetEscapeHTML(false)
	fatalIf(probe.NewError(enc.Encode(trc)), "Unable to marshal into JSON.")

	// strip off extra newline added by json encoder
	return strings.TrimSuffix(buf.String(), "\n")
}

func (t traceMessage) String() string {
	var nodeNameStr string
	var b = &strings.Builder{}

	trc := t.Trace
	if trc.NodeName != "" {
		nodeNameStr = fmt.Sprintf("%s ", colorizedNodeName(trc.NodeName))
	}

	ri := trc.ReqInfo
	rs := trc.RespInfo
	fmt.Fprintf(b, "%s%s", nodeNameStr, console.Colorize("Request", fmt.Sprintf("[REQUEST %s] ", trc.FuncName)))
	fmt.Fprintf(b, "%s\n", ri.Time.Format(timeFormat))
	fmt.Fprintf(b, "%s%s", nodeNameStr, console.Colorize("Method", fmt.Sprintf("%s %s", ri.Method, ri.Path)))
	if ri.RawQuery != "" {
		fmt.Fprintf(b, "?%s", ri.RawQuery)
	}
	fmt.Fprint(b, "\n")
	host, ok := ri.Headers["Host"]
	if ok {
		delete(ri.Headers, "Host")
	}
	hostStr := strings.Join(host, "")
	fmt.Fprintf(b, "%s%s", nodeNameStr, console.Colorize("Host", fmt.Sprintf("Host: %s\n", hostStr)))
	for k, v := range ri.Headers {
		fmt.Fprintf(b, "%s%s", nodeNameStr, console.Colorize("ReqHeaderKey",
			fmt.Sprintf("%s: ", k))+console.Colorize("HeaderValue", fmt.Sprintf("%s\n", strings.Join(v, ""))))
	}

	fmt.Fprintf(b, "%s%s", nodeNameStr, console.Colorize("Body", fmt.Sprintf("%s\n", string(ri.Body))))
	fmt.Fprintf(b, "%s%s", nodeNameStr, console.Colorize("Response", fmt.Sprintf("[RESPONSE] ")))
	fmt.Fprintf(b, "[%s] ", rs.Time.Format(timeFormat))
	fmt.Fprint(b, console.Colorize("Stat", fmt.Sprintf("[ Duration %2s  🠉 %s  🠋 %s ]\n", trc.CallStats.Latency.Round(time.Microsecond), humanize.IBytes(uint64(trc.CallStats.InputBytes)), humanize.IBytes(uint64(trc.CallStats.OutputBytes)))))

	statusStr := console.Colorize("RespStatus", fmt.Sprintf("%d %s", rs.StatusCode, http.StatusText(rs.StatusCode)))
	if rs.StatusCode != http.StatusOK {
		statusStr = console.Colorize("ErrStatus", fmt.Sprintf("%d %s", rs.StatusCode, http.StatusText(rs.StatusCode)))
	}
	fmt.Fprintf(b, "%s%s\n", nodeNameStr, statusStr)

	for k, v := range rs.Headers {
		fmt.Fprintf(b, "%s%s", nodeNameStr, console.Colorize("RespHeaderKey",
			fmt.Sprintf("%s: ", k))+console.Colorize("HeaderValue", fmt.Sprintf("%s\n", strings.Join(v, ""))))
	}
	fmt.Fprintf(b, "%s%s\n", nodeNameStr, console.Colorize("Body", string(rs.Body)))
	fmt.Fprint(b, nodeNameStr)
	return b.String()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"os"

	"github.com/kusabashira/alita"
)

func usage() {
	os.Stderr.WriteString(`
Usage: alita [OPTION]... [FILE]...
Align FILE(s), or standard input.

Delimiter control:
  -r, --regexp             DELIM is a regular expression(default: off)
  -d, --delimiter=DELIM    delimit line by DELIM (default: /\s+/)

Output control:
  -m, --margin=FORMAT      join cells by FORMAT (default: 1:1)
  -j, --justfy=SEQUENCE    justfy cells by SEQUENCE (default: l)

Miscellaneous:
  -h, --help               show this help message
  -v, --version            print the version
`[1:])
}

func version() {
	os.Stderr.WriteString(`
v0.3.0
`[1:])
}

func do(a *alita.Aligner, r io.Reader) error {
	if err := a.ReadAll(r); err != nil {
		return err
	}
	return a.Flush()
}

func _main() error {
	var isHelp, isVersion bool
	flag.BoolVar(&isHelp, "h", false, "")
	flag.BoolVar(&isHelp, "help", false, "")
	flag.BoolVar(&isVersion, "v", false, "")
	flag.BoolVar(&isVersion, "version", false, "")

	a := alita.NewAligner(os.Stdout)
	flag.Var(a.Margin, "m", "")
	flag.Var(a.Margin, "margin", "")
	flag.Var(a.Padding, "j", "")
	flag.Var(a.Padding, "justfy", "")
	flag.Var(a.Delimiter, "d", "")
	flag.Var(a.Delimiter, "delimiter", "")
	flag.BoolVar(&a.Delimiter.UseRegexp, "r", false, "")
	flag.BoolVar(&a.Delimiter.UseRegexp, "regexp", false, "")

	flag.Usage = usage
	flag.Parse()
	switch {
	case isHelp:
		usage()
		return nil
	case isVersion:
		version()
		return nil
	}

	if flag.NArg() < 1 {
		return do(a, os.Stdin)
	}

	var input []io.Reader
	for _, fname := range flag.Args() {
		f, err := os.Open(fname)
		if err != nil {
			return err
		}
		defer f.Close()
		input = append(input, f)
	}
	return do(a, io.MultiReader(input...))
}

func main() {
	if err := _main(); err != nil {
		fmt.Fprintln(os.Stderr, "alita:", err)
		os.Exit(1)
	}
}
<commit_msg>Remove -v flag<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"

	"github.com/kusabashira/alita"
)

func usage() {
	os.Stderr.WriteString(`
Usage: alita [OPTION]... [FILE]...
Align FILE(s), or standard input.

Delimiter control:
  -r, --regexp             DELIM is a regular expression(default: off)
  -d, --delimiter=DELIM    delimit line by DELIM (default: /\s+/)

Output control:
  -m, --margin=FORMAT      join cells by FORMAT (default: 1:1)
  -j, --justfy=SEQUENCE    justfy cells by SEQUENCE (default: l)

Miscellaneous:
  -h, --help               show this help message
      --version            print the version
`[1:])
}

func version() {
	os.Stderr.WriteString(`
v0.3.0
`[1:])
}

func do(a *alita.Aligner, r io.Reader) error {
	if err := a.ReadAll(r); err != nil {
		return err
	}
	return a.Flush()
}

func _main() error {
	var isHelp, isVersion bool
	flag.BoolVar(&isHelp, "h", false, "")
	flag.BoolVar(&isHelp, "help", false, "")
	flag.BoolVar(&isVersion, "version", false, "")

	a := alita.NewAligner(os.Stdout)
	flag.Var(a.Margin, "m", "")
	flag.Var(a.Margin, "margin", "")
	flag.Var(a.Padding, "j", "")
	flag.Var(a.Padding, "justfy", "")
	flag.Var(a.Delimiter, "d", "")
	flag.Var(a.Delimiter, "delimiter", "")
	flag.BoolVar(&a.Delimiter.UseRegexp, "r", false, "")
	flag.BoolVar(&a.Delimiter.UseRegexp, "regexp", false, "")

	flag.Usage = usage
	flag.Parse()
	switch {
	case isHelp:
		usage()
		return nil
	case isVersion:
		version()
		return nil
	}

	if flag.NArg() < 1 {
		return do(a, os.Stdin)
	}

	var input []io.Reader
	for _, fname := range flag.Args() {
		f, err := os.Open(fname)
		if err != nil {
			return err
		}
		defer f.Close()
		input = append(input, f)
	}
	return do(a, io.MultiReader(input...))
}

func main() {
	if err := _main(); err != nil {
		fmt.Fprintln(os.Stderr, "alita:", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"

	"os"

	"github.com/mpolden/echoip/http"
	"github.com/mpolden/echoip/iputil"
	"github.com/mpolden/echoip/iputil/geo"
)

type multiValueFlag []string

func (f *multiValueFlag) String() string {
	vs := ""
	for i, v := range *f {
		vs += v
		if i < len(*f)-1 {
			vs += ", "
		}
	}
	return vs
}

func (f *multiValueFlag) Set(v string) error {
	*f = append(*f, v)
	return nil
}

func main() {
	countryFile := flag.String("f", "", "Path to GeoIP country database")
	cityFile := flag.String("c", "", "Path to GeoIP city database")
	asnFile := flag.String("a", "", "Path to GeoIP ASN database")
	listen := flag.String("l", ":8080", "Listening address")
	reverseLookup := flag.Bool("r", false, "Perform reverse hostname lookups")
	portLookup := flag.Bool("p", false, "Enable port lookup")
	template := flag.String("t", "index.html", "Path to template")
	cacheSize := flag.Int("C", 0, "Size of response cache. Set to 0 to disable")
	profile := flag.Bool("P", false, "Enables profiling handlers")
	var headers multiValueFlag
	flag.Var(&headers, "H", "Header to trust for remote IP, if present (e.g. X-Real-IP)")
	flag.Parse()

	log := log.New(os.Stderr, "echoip: ", 0)
	r, err := geo.Open(*countryFile, *cityFile, *asnFile)
	if err != nil {
		log.Fatal(err)
	}
	cache := http.NewCache(*cacheSize)
	server := http.New(r, cache, *profile)
	server.IPHeaders = headers
	if _, err := os.Stat(*template); err == nil {
		server.Template = *template
	} else {
		log.Printf("Not configuring default handler: Template not found: %s", *template)
	}
	if *reverseLookup {
		log.Println("Enabling reverse lookup")
		server.LookupAddr = iputil.LookupAddr
	}
	if *portLookup {
		log.Println("Enabling port lookup")
		server.LookupPort = iputil.LookupPort
	}
	if len(headers) > 0 {
		log.Printf("Trusting remote IP from header(s): %s", headers.String())
	}
	if *cacheSize > 0 {
		log.Printf("Cache capacity set to %d", *cacheSize)
	}
	if *profile {
		log.Printf("Enabling profiling handlers")
	}
	log.Printf("Listening on http://%s", *listen)
	if err := server.ListenAndServe(*listen); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>cmd: Reject non-option arguments<commit_after>package main

import (
	"flag"
	"log"

	"os"

	"github.com/mpolden/echoip/http"
	"github.com/mpolden/echoip/iputil"
	"github.com/mpolden/echoip/iputil/geo"
)

type multiValueFlag []string

func (f *multiValueFlag) String() string {
	vs := ""
	for i, v := range *f {
		vs += v
		if i < len(*f)-1 {
			vs += ", "
		}
	}
	return vs
}

func (f *multiValueFlag) Set(v string) error {
	*f = append(*f, v)
	return nil
}

func main() {
	countryFile := flag.String("f", "", "Path to GeoIP country database")
	cityFile := flag.String("c", "", "Path to GeoIP city database")
	asnFile := flag.String("a", "", "Path to GeoIP ASN database")
	listen := flag.String("l", ":8080", "Listening address")
	reverseLookup := flag.Bool("r", false, "Perform reverse hostname lookups")
	portLookup := flag.Bool("p", false, "Enable port lookup")
	template := flag.String("t", "index.html", "Path to template")
	cacheSize := flag.Int("C", 0, "Size of response cache. Set to 0 to disable")
	profile := flag.Bool("P", false, "Enables profiling handlers")
	var headers multiValueFlag
	flag.Var(&headers, "H", "Header to trust for remote IP, if present (e.g. X-Real-IP)")
	flag.Parse()
	if len(flag.Args()) != 0 {
		flag.Usage()
		return
	}

	log := log.New(os.Stderr, "echoip: ", 0)
	r, err := geo.Open(*countryFile, *cityFile, *asnFile)
	if err != nil {
		log.Fatal(err)
	}
	cache := http.NewCache(*cacheSize)
	server := http.New(r, cache, *profile)
	server.IPHeaders = headers
	if _, err := os.Stat(*template); err == nil {
		server.Template = *template
	} else {
		log.Printf("Not configuring default handler: Template not found: %s", *template)
	}
	if *reverseLookup {
		log.Println("Enabling reverse lookup")
		server.LookupAddr = iputil.LookupAddr
	}
	if *portLookup {
		log.Println("Enabling port lookup")
		server.LookupPort = iputil.LookupPort
	}
	if len(headers) > 0 {
		log.Printf("Trusting remote IP from header(s): %s", headers.String())
	}
	if *cacheSize > 0 {
		log.Printf("Cache capacity set to %d", *cacheSize)
	}
	if *profile {
		log.Printf("Enabling profiling handlers")
	}
	log.Printf("Listening on http://%s", *listen)
	if err := server.ListenAndServe(*listen); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>