<commit_before>package parser

import (
	"fmt"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/influxdb/influxdb/common"
)

// this file provides the high level api of the query object

func uniq(slice []string) []string {
	// TODO: optimize this, maybe ?
	uniqueMap := map[string]bool{}
	for _, name := range slice {
		uniqueMap[name] = true
	}
	slice = []string{}
	for name := range uniqueMap {
		slice = append(slice, name)
	}
	sort.Strings(slice)
	return slice
}

func (self *SelectDeleteCommonQuery) WillReturnSingleSeries() bool {
	fromClause := self.GetFromClause()
	if fromClause.Type != FromClauseArray {
		return false
	}

	if len(fromClause.Names) > 1 {
		return false
	}

	if _, ok := fromClause.Names[0].Name.GetCompiledRegex(); ok {
		return false
	}

	return true
}

func (self *SelectDeleteCommonQuery) GetTableAliases(name string) []string {
	names := self.GetFromClause().Names
	if len(names) == 1 && names[0].Name.Type == ValueRegex {
		return []string{name}
	}

	aliases := []string{}

	for _, fromName := range names {
		if fromName.Name.Name != name {
			continue
		}

		if fromName.Alias == "" {
			aliases = append(aliases, name)
			continue
		}

		aliases = append(aliases, fromName.Alias)
	}
	return aliases
}

func (self *SelectQuery) revertAlias(mapping map[string][]string) {
	fromClause := self.GetFromClause()
	if fromClause.Type != FromClauseInnerJoin {
		return
	}

	columns := make(map[string]map[string]bool)

	for _, table := range fromClause.Names {
		name := table.Name.Name
		alias := name
		if table.Alias != "" {
			alias = table.Alias
		}

		for _, column := range mapping[alias] {
			tableColumns := columns[name]
			if tableColumns == nil {
				tableColumns = make(map[string]bool)
				columns[name] = tableColumns
			}
			tableColumns[column] = true
		}

		delete(mapping, alias)
	}

	for table, tableColumns := range columns {
		mapping[table] = []string{}
		for column := range tableColumns {
			mapping[table] = append(mapping[table], column)
		}
	}
}

// Returns true if the query has aggregate functions applied to the
// columns
func (self *SelectQuery) HasAggregates() bool {
	for _, column := range self.GetColumnNames() {
		if column.IsFunctionCall() {
			return true
		}
	}
	return false
}

// Returns a mapping from the time series names (or regex) to the
// column names that are references
func (self *SelectQuery) GetReferencedColumns() map[*Value][]string {
	return self.getColumns(true)
}

func (self *SelectQuery) GetResultColumns() map[*Value][]string {
	return self.getColumns(false)
}

func (self *SelectQuery) getColumns(includeWhereClause bool) map[*Value][]string {
	mapping := make(map[string][]string)

	notPrefixedColumns := []string{}
	for _, value := range self.GetColumnNames() {
		if value.Name == "time" || value.Name == "sequence_number" {
			continue
		}
		notPrefixedColumns = append(notPrefixedColumns, getReferencedColumnsFromValue(value, mapping)...)
	}

	if !self.IsSinglePointQuery() {
		if condition := self.GetWhereCondition(); condition != nil && includeWhereClause {
			notPrefixedColumns = append(notPrefixedColumns, getReferencedColumnsFromCondition(condition, mapping)...)
		}

		for _, groupBy := range self.groupByClause.Elems {
			notPrefixedColumns = append(notPrefixedColumns, getReferencedColumnsFromValue(groupBy, mapping)...)
		}
	}

	notPrefixedColumns = uniq(notPrefixedColumns)

	self.revertAlias(mapping)

	addedTables := make(map[string]bool)

	returnedMapping := make(map[*Value][]string)
	for _, tableName := range self.GetFromClause().Names {
		value := tableName.Name
		if _, ok := value.GetCompiledRegex(); ok {
			// this is a regex table, cannot be referenced, only unreferenced
			// columns will be attached to regex table names
			returnedMapping[value] = notPrefixedColumns
			continue
		}

		name := value.Name
		if addedTables[name] {
			continue
		}
		addedTables[name] = true
		returnedMapping[value] = uniq(append(mapping[name], notPrefixedColumns...))
		if len(returnedMapping[value]) > 1 && returnedMapping[value][0] == "*" {
			returnedMapping[value] = returnedMapping[value][:1]
		}

		delete(mapping, name)
	}

	if len(mapping) == 0 {
		return returnedMapping
	}

	// if `mapping` still have some mappings, then we have mistaken a
	// column name with dots with a prefix.column, see issue #240
	for prefix, columnNames := range mapping {
		for _, columnName := range columnNames {
			for table, columns := range returnedMapping {
				if len(returnedMapping[table]) > 1 && returnedMapping[table][0] == "*" {
					continue
				}
				returnedMapping[table] = append(columns, prefix+"."+columnName)
			}
		}
		delete(mapping, prefix)
	}

	return returnedMapping
}

// Returns the start time of the query. Queries can only have
// one condition of the form time > start_time
func (self *BasicQuery) GetStartTime() time.Time {
	return self.startTime
}

// Returns the start time of the query. Queries can only have
// one condition of the form time > start_time
func (self *BasicQuery) GetEndTime() time.Time {
	return self.endTime
}

// parse time that matches the following format:
//   2006-01-02 [15[:04[:05[.000]]]]
// notice, hour, minute and seconds are optional
var time_regex *regexp.Regexp

func init() {
	var err error
	time_regex, err = regexp.Compile(
		"^([0-9]{4}|[0-9]{2})-[0-9]{1,2}-[0-9]{1,2}( [0-9]{1,2}(:[0-9]{1,2}(:[0-9]{1,2}?(\\.[0-9]+)?)?)?)?$")
	if err != nil {
		panic(err)
	}
}

func parseTimeString(t string) (*time.Time, error) {
	submatches := time_regex.FindStringSubmatch(t)
	if len(submatches) == 0 {
		return nil, fmt.Errorf("%s isn't a valid time string", t)
	}

	if submatches[5] != "" || submatches[4] != "" {
		t, err := time.Parse("2006-01-02 15:04:05", t)
		return &t, err
	}

	if submatches[3] != "" {
		t, err := time.Parse("2006-01-02 15:04", t)
		return &t, err
	}

	if submatches[2] != "" {
		t, err := time.Parse("2006-01-02 15", t)
		return &t, err
	}

	_t, err := time.Parse("2006-01-02", t)
	return &_t, err
}

func parseTimeWithoutSuffix(value string) (int64, error) {
	var err error
	var f float64
	var i int64
	if strings.Contains(value, ".") {
		f, err = strconv.ParseFloat(value, 64)
		i = int64(f)
	} else {
		i, err = strconv.ParseInt(value, 10, 64)
	}
	if err != nil {
		return 0, err
	}
	return i, nil
}

// parse time expressions, e.g. now() - 1d
func parseTime(value *Value) (int64, error) {
	if value.Type != ValueExpression {
		if value.IsFunctionCall() && strings.ToLower(value.Name) == "now" {
			return time.Now().UTC().UnixNano(), nil
		}

		if value.IsFunctionCall() {
			return 0, fmt.Errorf("Invalid use of function %s", value.Name)
		}

		if value.Type == ValueString {
			t, err := parseTimeString(value.Name)
			if err != nil {
				return 0, err
			}
			return t.UnixNano(), err
		}

		return common.ParseTimeDuration(value.Name)
	}

	leftValue, err := parseTime(value.Elems[0])
	if err != nil {
		return 0, err
	}
	rightValue, err := parseTime(value.Elems[1])
	if err != nil {
		return 0, err
	}
	switch value.Name {
	case "+":
		return leftValue + rightValue, nil
	case "-":
		return leftValue - rightValue, nil
	default:
		return 0, fmt.Errorf("Cannot use '%s' in a time expression", value.Name)
	}
}

func getReferencedColumnsFromValue(v *Value, mapping map[string][]string) (notAssigned []string) {
	switch v.Type {
	case ValueSimpleName, ValueTableName:
		if idx := strings.LastIndex(v.Name, "."); idx != -1 {
			tableName := v.Name[:idx]
			columnName := v.Name[idx+1:]
			mapping[tableName] = append(mapping[tableName], columnName)
			return
		}
		notAssigned = append(notAssigned, v.Name)
	case ValueWildcard:
		notAssigned = append(notAssigned, "*")
	case ValueExpression, ValueFunctionCall:
		for _, value := range v.Elems {
			newNotAssignedColumns := getReferencedColumnsFromValue(value, mapping)
			if len(newNotAssignedColumns) > 0 && newNotAssignedColumns[0] == "*" {
				newNotAssignedColumns = newNotAssignedColumns[1:]
			}

			notAssigned = append(notAssigned, newNotAssignedColumns...)
		}
	}
	return
}

func getReferencedColumnsFromCondition(condition *WhereCondition, mapping map[string][]string) (notPrefixed []string) {
	if left, ok := condition.GetLeftWhereCondition(); ok {
		notPrefixed = append(notPrefixed, getReferencedColumnsFromCondition(left, mapping)...)
		notPrefixed = append(notPrefixed, getReferencedColumnsFromCondition(condition.Right, mapping)...)
		return
	}

	expr, _ := condition.GetBoolExpression()
	notPrefixed = append(notPrefixed, getReferencedColumnsFromValue(expr, mapping)...)
	return
}

func isNumericValue(value *Value) bool {
	switch value.Type {
	case ValueDuration, ValueFloat, ValueInt, ValueString:
		return true
	default:
		return false
	}
}

// parse the start time or end time from the where conditions and return the new condition
// without the time clauses, or nil if there are no where conditions left
func getTime(condition *WhereCondition, isParsingStartTime bool) (*WhereCondition, *time.Time, error) {
	if condition == nil {
		return nil, nil, nil
	}

	if expr, ok := condition.GetBoolExpression(); ok {
		switch expr.Type {
		case ValueDuration, ValueFloat, ValueInt, ValueString, ValueWildcard:
			return nil, nil, fmt.Errorf("Invalid where expression: %v", expr)
		}

		if expr.Type == ValueFunctionCall {
			return condition, nil, nil
		}

		leftValue := expr.Elems[0]
		isTimeOnLeft := leftValue.Type != ValueExpression && leftValue.Type != ValueFunctionCall
		rightValue := expr.Elems[1]
		isTimeOnRight := rightValue.Type != ValueExpression && rightValue.Type != ValueFunctionCall

		// this can only be the case if the where condition
		// is of the form `"time" > 123456789`, so let's see
		// which side is a float value
		if isTimeOnLeft && isTimeOnRight {
			if isNumericValue(rightValue) {
				isTimeOnRight = false
			} else {
				isTimeOnLeft = false
			}
		}

		// if this expression isn't "time > xxx" or "xxx < time" then return
		// TODO: we should do a check to make sure "time" doesn't show up in
		// either expressions
		if !isTimeOnLeft && !isTimeOnRight {
			return condition, nil, nil
		}

		var timeExpression *Value
		if !isTimeOnRight {
			if leftValue.Name != "time" {
				return condition, nil, nil
			}
			timeExpression = rightValue
		} else if !isTimeOnLeft {
			if rightValue.Name != "time" {
				return condition, nil, nil
			}
			timeExpression = leftValue
		} else {
			return nil, nil, fmt.Errorf("Invalid time condition %v", condition)
		}

		switch expr.Name {
		case ">":
			if isParsingStartTime && !isTimeOnLeft || !isParsingStartTime && !isTimeOnRight {
				return condition, nil, nil
			}
		case "<":
			if !isParsingStartTime && !isTimeOnLeft || isParsingStartTime && !isTimeOnRight {
				return condition, nil, nil
			}
		case "=":
			nanoseconds, err := parseTime(timeExpression)
			if err != nil {
				return nil, nil, err
			}
			t := time.Unix(nanoseconds/int64(time.Second), nanoseconds%int64(time.Second)).UTC()
			return condition, &t, nil
		default:
			return nil, nil, fmt.Errorf("Cannot use time with '%s'", expr.Name)
		}

		nanoseconds, err := parseTime(timeExpression)
		if err != nil {
			return nil, nil, err
		}
		t := time.Unix(nanoseconds/int64(time.Second), nanoseconds%int64(time.Second)).UTC()
		return nil, &t, nil
	}

	leftCondition, _ := condition.GetLeftWhereCondition()
	newLeftCondition, timeLeft, err := getTime(leftCondition, isParsingStartTime)
	if err != nil {
		return nil, nil, err
	}
	newRightCondition, timeRight, err := getTime(condition.Right, isParsingStartTime)
	if err != nil {
		return nil, nil, err
	}

	if condition.Operation == "OR" && (timeLeft != nil || timeRight != nil) {
		// we can't have two start times or'd together
		return nil, nil, fmt.Errorf("Invalid where clause, time must appear twice to specify start and end time")
	}

	newCondition := condition
	if newLeftCondition == nil {
		newCondition = newRightCondition
	} else if newRightCondition == nil {
		newCondition = newLeftCondition
	} else {
		newCondition.Left = newLeftCondition
		newCondition.Right = newRightCondition
	}

	if timeLeft == nil {
		return newCondition, timeRight, nil
	}
	if timeRight == nil {
		return newCondition, timeLeft, nil
	}
	if isParsingStartTime && timeLeft.Unix() < timeRight.Unix() {
		return newCondition, timeLeft, nil
	}
	if !isParsingStartTime && timeLeft.Unix() > timeRight.Unix() {
		return newCondition, timeLeft, nil
	}
	return newCondition, timeRight, nil
}
<commit_msg>use camel casing instead of underscore<commit_after>package parser

import (
	"fmt"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/influxdb/influxdb/common"
)

// this file provides the high level api of the query object

func uniq(slice []string) []string {
	// TODO: optimize this, maybe ?
	uniqueMap := map[string]bool{}
	for _, name := range slice {
		uniqueMap[name] = true
	}
	slice = []string{}
	for name := range uniqueMap {
		slice = append(slice, name)
	}
	sort.Strings(slice)
	return slice
}

func (self *SelectDeleteCommonQuery) WillReturnSingleSeries() bool {
	fromClause := self.GetFromClause()
	if fromClause.Type != FromClauseArray {
		return false
	}

	if len(fromClause.Names) > 1 {
		return false
	}

	if _, ok := fromClause.Names[0].Name.GetCompiledRegex(); ok {
		return false
	}

	return true
}

func (self *SelectDeleteCommonQuery) GetTableAliases(name string) []string {
	names := self.GetFromClause().Names
	if len(names) == 1 && names[0].Name.Type == ValueRegex {
		return []string{name}
	}

	aliases := []string{}

	for _, fromName := range names {
		if fromName.Name.Name != name {
			continue
		}

		if fromName.Alias == "" {
			aliases = append(aliases, name)
			continue
		}

		aliases = append(aliases, fromName.Alias)
	}
	return aliases
}

func (self *SelectQuery) revertAlias(mapping map[string][]string) {
	fromClause := self.GetFromClause()
	if fromClause.Type != FromClauseInnerJoin {
		return
	}

	columns := make(map[string]map[string]bool)

	for _, table := range fromClause.Names {
		name := table.Name.Name
		alias := name
		if table.Alias != "" {
			alias = table.Alias
		}

		for _, column := range mapping[alias] {
			tableColumns := columns[name]
			if tableColumns == nil {
				tableColumns = make(map[string]bool)
				columns[name] = tableColumns
			}
			tableColumns[column] = true
		}

		delete(mapping, alias)
	}

	for table, tableColumns := range columns {
		mapping[table] = []string{}
		for column := range tableColumns {
			mapping[table] = append(mapping[table], column)
		}
	}
}

// Returns true if the query has aggregate functions applied to the
// columns
func (self *SelectQuery) HasAggregates() bool {
	for _, column := range self.GetColumnNames() {
		if column.IsFunctionCall() {
			return true
		}
	}
	return false
}

// Returns a mapping from the time series names (or regex) to the
// column names that are references
func (self *SelectQuery) GetReferencedColumns() map[*Value][]string {
	return self.getColumns(true)
}

func (self *SelectQuery) GetResultColumns() map[*Value][]string {
	return self.getColumns(false)
}

func (self *SelectQuery) getColumns(includeWhereClause bool) map[*Value][]string {
	mapping := make(map[string][]string)

	notPrefixedColumns := []string{}
	for _, value := range self.GetColumnNames() {
		if value.Name == "time" || value.Name == "sequence_number" {
			continue
		}
		notPrefixedColumns = append(notPrefixedColumns, getReferencedColumnsFromValue(value, mapping)...)
	}

	if !self.IsSinglePointQuery() {
		if condition := self.GetWhereCondition(); condition != nil && includeWhereClause {
			notPrefixedColumns = append(notPrefixedColumns, getReferencedColumnsFromCondition(condition, mapping)...)
		}

		for _, groupBy := range self.groupByClause.Elems {
			notPrefixedColumns = append(notPrefixedColumns, getReferencedColumnsFromValue(groupBy, mapping)...)
		}
	}

	notPrefixedColumns = uniq(notPrefixedColumns)

	self.revertAlias(mapping)

	addedTables := make(map[string]bool)

	returnedMapping := make(map[*Value][]string)
	for _, tableName := range self.GetFromClause().Names {
		value := tableName.Name
		if _, ok := value.GetCompiledRegex(); ok {
			// this is a regex table, cannot be referenced, only unreferenced
			// columns will be attached to regex table names
			returnedMapping[value] = notPrefixedColumns
			continue
		}

		name := value.Name
		if addedTables[name] {
			continue
		}
		addedTables[name] = true
		returnedMapping[value] = uniq(append(mapping[name], notPrefixedColumns...))
		if len(returnedMapping[value]) > 1 && returnedMapping[value][0] == "*" {
			returnedMapping[value] = returnedMapping[value][:1]
		}

		delete(mapping, name)
	}

	if len(mapping) == 0 {
		return returnedMapping
	}

	// if `mapping` still have some mappings, then we have mistaken a
	// column name with dots with a prefix.column, see issue #240
	for prefix, columnNames := range mapping {
		for _, columnName := range columnNames {
			for table, columns := range returnedMapping {
				if len(returnedMapping[table]) > 1 && returnedMapping[table][0] == "*" {
					continue
				}
				returnedMapping[table] = append(columns, prefix+"."+columnName)
			}
		}
		delete(mapping, prefix)
	}

	return returnedMapping
}

// Returns the start time of the query. Queries can only have
// one condition of the form time > start_time
func (self *BasicQuery) GetStartTime() time.Time {
	return self.startTime
}

// Returns the start time of the query. Queries can only have
// one condition of the form time > start_time
func (self *BasicQuery) GetEndTime() time.Time {
	return self.endTime
}

// parse time that matches the following format:
//   2006-01-02 [15[:04[:05[.000]]]]
// notice, hour, minute and seconds are optional
var timeRegex *regexp.Regexp

func init() {
	var err error
	timeRegex, err = regexp.Compile(
		"^([0-9]{4}|[0-9]{2})-[0-9]{1,2}-[0-9]{1,2}( [0-9]{1,2}(:[0-9]{1,2}(:[0-9]{1,2}?(\\.[0-9]+)?)?)?)?$")
	if err != nil {
		panic(err)
	}
}

func parseTimeString(t string) (*time.Time, error) {
	submatches := timeRegex.FindStringSubmatch(t)
	if len(submatches) == 0 {
		return nil, fmt.Errorf("%s isn't a valid time string", t)
	}

	if submatches[5] != "" || submatches[4] != "" {
		t, err := time.Parse("2006-01-02 15:04:05", t)
		return &t, err
	}

	if submatches[3] != "" {
		t, err := time.Parse("2006-01-02 15:04", t)
		return &t, err
	}

	if submatches[2] != "" {
		t, err := time.Parse("2006-01-02 15", t)
		return &t, err
	}

	_t, err := time.Parse("2006-01-02", t)
	return &_t, err
}

func parseTimeWithoutSuffix(value string) (int64, error) {
	var err error
	var f float64
	var i int64
	if strings.Contains(value, ".") {
		f, err = strconv.ParseFloat(value, 64)
		i = int64(f)
	} else {
		i, err = strconv.ParseInt(value, 10, 64)
	}
	if err != nil {
		return 0, err
	}
	return i, nil
}

// parse time expressions, e.g. now() - 1d
func parseTime(value *Value) (int64, error) {
	if value.Type != ValueExpression {
		if value.IsFunctionCall() && strings.ToLower(value.Name) == "now" {
			return time.Now().UTC().UnixNano(), nil
		}

		if value.IsFunctionCall() {
			return 0, fmt.Errorf("Invalid use of function %s", value.Name)
		}

		if value.Type == ValueString {
			t, err := parseTimeString(value.Name)
			if err != nil {
				return 0, err
			}
			return t.UnixNano(), err
		}

		return common.ParseTimeDuration(value.Name)
	}

	leftValue, err := parseTime(value.Elems[0])
	if err != nil {
		return 0, err
	}
	rightValue, err := parseTime(value.Elems[1])
	if err != nil {
		return 0, err
	}
	switch value.Name {
	case "+":
		return leftValue + rightValue, nil
	case "-":
		return leftValue - rightValue, nil
	default:
		return 0, fmt.Errorf("Cannot use '%s' in a time expression", value.Name)
	}
}

func getReferencedColumnsFromValue(v *Value, mapping map[string][]string) (notAssigned []string) {
	switch v.Type {
	case ValueSimpleName, ValueTableName:
		if idx := strings.LastIndex(v.Name, "."); idx != -1 {
			tableName := v.Name[:idx]
			columnName := v.Name[idx+1:]
			mapping[tableName] = append(mapping[tableName], columnName)
			return
		}
		notAssigned = append(notAssigned, v.Name)
	case ValueWildcard:
		notAssigned = append(notAssigned, "*")
	case ValueExpression, ValueFunctionCall:
		for _, value := range v.Elems {
			newNotAssignedColumns := getReferencedColumnsFromValue(value, mapping)
			if len(newNotAssignedColumns) > 0 && newNotAssignedColumns[0] == "*" {
				newNotAssignedColumns = newNotAssignedColumns[1:]
			}

			notAssigned = append(notAssigned, newNotAssignedColumns...)
		}
	}
	return
}

func getReferencedColumnsFromCondition(condition *WhereCondition, mapping map[string][]string) (notPrefixed []string) {
	if left, ok := condition.GetLeftWhereCondition(); ok {
		notPrefixed = append(notPrefixed, getReferencedColumnsFromCondition(left, mapping)...)
		notPrefixed = append(notPrefixed, getReferencedColumnsFromCondition(condition.Right, mapping)...)
		return
	}

	expr, _ := condition.GetBoolExpression()
	notPrefixed = append(notPrefixed, getReferencedColumnsFromValue(expr, mapping)...)
	return
}

func isNumericValue(value *Value) bool {
	switch value.Type {
	case ValueDuration, ValueFloat, ValueInt, ValueString:
		return true
	default:
		return false
	}
}

// parse the start time or end time from the where conditions and return the new condition
// without the time clauses, or nil if there are no where conditions left
func getTime(condition *WhereCondition, isParsingStartTime bool) (*WhereCondition, *time.Time, error) {
	if condition == nil {
		return nil, nil, nil
	}

	if expr, ok := condition.GetBoolExpression(); ok {
		switch expr.Type {
		case ValueDuration, ValueFloat, ValueInt, ValueString, ValueWildcard:
			return nil, nil, fmt.Errorf("Invalid where expression: %v", expr)
		}

		if expr.Type == ValueFunctionCall {
			return condition, nil, nil
		}

		leftValue := expr.Elems[0]
		isTimeOnLeft := leftValue.Type != ValueExpression && leftValue.Type != ValueFunctionCall
		rightValue := expr.Elems[1]
		isTimeOnRight := rightValue.Type != ValueExpression && rightValue.Type != ValueFunctionCall

		// this can only be the case if the where condition
		// is of the form `"time" > 123456789`, so let's see
		// which side is a float value
		if isTimeOnLeft && isTimeOnRight {
			if isNumericValue(rightValue) {
				isTimeOnRight = false
			} else {
				isTimeOnLeft = false
			}
		}

		// if this expression isn't "time > xxx" or "xxx < time" then return
		// TODO: we should do a check to make sure "time" doesn't show up in
		// either expressions
		if !isTimeOnLeft && !isTimeOnRight {
			return condition, nil, nil
		}

		var timeExpression *Value
		if !isTimeOnRight {
			if leftValue.Name != "time" {
				return condition, nil, nil
			}
			timeExpression = rightValue
		} else if !isTimeOnLeft {
			if rightValue.Name != "time" {
				return condition, nil, nil
			}
			timeExpression = leftValue
		} else {
			return nil, nil, fmt.Errorf("Invalid time condition %v", condition)
		}

		switch expr.Name {
		case ">":
			if isParsingStartTime && !isTimeOnLeft || !isParsingStartTime && !isTimeOnRight {
				return condition, nil, nil
			}
		case "<":
			if !isParsingStartTime && !isTimeOnLeft || isParsingStartTime && !isTimeOnRight {
				return condition, nil, nil
			}
		case "=":
			nanoseconds, err := parseTime(timeExpression)
			if err != nil {
				return nil, nil, err
			}
			t := time.Unix(nanoseconds/int64(time.Second), nanoseconds%int64(time.Second)).UTC()
			return condition, &t, nil
		default:
			return nil, nil, fmt.Errorf("Cannot use time with '%s'", expr.Name)
		}

		nanoseconds, err := parseTime(timeExpression)
		if err != nil {
			return nil, nil, err
		}
		t := time.Unix(nanoseconds/int64(time.Second), nanoseconds%int64(time.Second)).UTC()
		return nil, &t, nil
	}

	leftCondition, _ := condition.GetLeftWhereCondition()
	newLeftCondition, timeLeft, err := getTime(leftCondition, isParsingStartTime)
	if err != nil {
		return nil, nil, err
	}
	newRightCondition, timeRight, err := getTime(condition.Right, isParsingStartTime)
	if err != nil {
		return nil, nil, err
	}

	if condition.Operation == "OR" && (timeLeft != nil || timeRight != nil) {
		// we can't have two start times or'd together
		return nil, nil, fmt.Errorf("Invalid where clause, time must appear twice to specify start and end time")
	}

	newCondition := condition
	if newLeftCondition == nil {
		newCondition = newRightCondition
	} else if newRightCondition == nil {
		newCondition = newLeftCondition
	} else {
		newCondition.Left = newLeftCondition
		newCondition.Right = newRightCondition
	}

	if timeLeft == nil {
		return newCondition, timeRight, nil
	}
	if timeRight == nil {
		return newCondition, timeLeft, nil
	}
	if isParsingStartTime && timeLeft.Unix() < timeRight.Unix() {
		return newCondition, timeLeft, nil
	}
	if !isParsingStartTime && timeLeft.Unix() > timeRight.Unix() {
		return newCondition, timeLeft, nil
	}
	return newCondition, timeRight, nil
}
<|endoftext|>


<commit_before>package pilosactl

import (
	"context"
	"encoding/csv"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/pilosa/pilosa"
)

// ImportCommand represents a command for bulk importing data.
type ImportCommand struct {
	// Destination host and port.
	Host string `json:"host"`

	// Name of the database & frame to import into.
	Database string `json:"db"`
	Frame    string `json:"frame"`

	// Filenames to import from.
	Paths []string `json:"paths"`

	// Size of buffer used to chunk import.
	BufferSize int `json:"buffer-size"`

	// Reusable client.
	Client *pilosa.Client `json:"-"`

	// Standard input/output
	Stdin  io.Reader `json:"-"`
	Stdout io.Writer `json:"-"`
	Stderr io.Writer `json:"-"`
}

// NewImportCommand returns a new instance of ImportCommand.
func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *ImportCommand {
	return &ImportCommand{
		Stdin:  stdin,
		Stdout: stdout,
		Stderr: stderr,

		BufferSize: 10000000,
	}
}

func (cmd *ImportCommand) String() string {
	return fmt.Sprint(*cmd)
}

// ParseFlags parses command line flags from args.
func (cmd *ImportCommand) ParseFlags(args []string) error {
	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
	fs.SetOutput(ioutil.Discard)
	fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port")
	fs.StringVar(&cmd.Database, "d", "", "database")
	fs.StringVar(&cmd.Frame, "f", "", "frame")
	fs.IntVar(&cmd.BufferSize, "buffer-size", cmd.BufferSize, "buffer size")
	if err := fs.Parse(args); err != nil {
		return err
	}

	// Extract the import paths.
	cmd.Paths = fs.Args()

	return nil
}

// Usage returns the usage message to be printed.
func (cmd *ImportCommand) Usage() string {
	return strings.TrimSpace(`
usage: pilosactl import -host HOST -d database -f frame paths

Bulk imports one or more CSV files to a host's database and frame. The bits
of the CSV file are grouped by slice for the most efficient import.

The format of the CSV file is:

	BITMAPID,PROFILEID,[TIME]

The file should contain no headers. The TIME column is optional and can be
omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
`)
}

// Run executes the main program execution.
func (cmd *ImportCommand) Run(ctx context.Context) error {
	logger := log.New(cmd.Stderr, "", log.LstdFlags)

	// Validate arguments.
	// Database and frame are validated early before the files are parsed.
	if cmd.Database == "" {
		return pilosa.ErrDatabaseRequired
	} else if cmd.Frame == "" {
		return pilosa.ErrFrameRequired
	} else if len(cmd.Paths) == 0 {
		return errors.New("path required")
	}

	// Create a client to the server.
	client, err := pilosa.NewClient(cmd.Host)
	if err != nil {
		return err
	}
	cmd.Client = client

	// Import each path and import by slice.
	for _, path := range cmd.Paths {
		// Parse path into bits.
		logger.Printf("parsing: %s", path)
		if err := cmd.importPath(ctx, path); err != nil {
			return err
		}
	}

	return nil
}

// importPath parses a path into bits and imports it to the server.
func (cmd *ImportCommand) importPath(ctx context.Context, path string) error {
	a := make([]pilosa.Bit, 0, cmd.BufferSize)

	// Open file for reading.
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	// Read rows as bits.
	r := csv.NewReader(f)
	r.FieldsPerRecord = -1
	rnum := 0
	for {
		rnum++

		// Read CSV row.
		record, err := r.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		// Ignore blank rows.
		if record[0] == "" {
			continue
		} else if len(record) < 2 {
			return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record))
		}

		var bit pilosa.Bit

		// Parse bitmap id.
		bitmapID, err := strconv.ParseUint(record[0], 10, 64)
		if err != nil {
			return fmt.Errorf("invalid bitmap id on row %d: %q", rnum, record[0])
		}
		bit.BitmapID = bitmapID

		// Parse bitmap id.
		profileID, err := strconv.ParseUint(record[1], 10, 64)
		if err != nil {
			return fmt.Errorf("invalid profile id on row %d: %q", rnum, record[1])
		}
		bit.ProfileID = profileID

		// Parse time, if exists.
		if len(record) > 2 && record[2] != "" {
			t, err := time.Parse(pilosa.TimeFormat, record[2])
			if err != nil {
				return fmt.Errorf("invalid timestamp on row %d: %q", rnum, record[2])
			}
			bit.Timestamp = t.UnixNano()
		}

		a = append(a, bit)

		// If we've reached the buffer size then import bits.
		if len(a) == cmd.BufferSize {
			if err := cmd.importBits(ctx, a); err != nil {
				return err
			}
			a = a[:0]
		}
	}

	// If there are still bits in the buffer then flush them.
	if err := cmd.importBits(ctx, a); err != nil {
		return err
	}

	return nil
}

// importPath parses a path into bits and imports it to the server.
func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) error {
	logger := log.New(cmd.Stderr, "", log.LstdFlags)

	// Group bits by slice.
	logger.Printf("grouping %d bits", len(bits))
	bitsBySlice := pilosa.Bits(bits).GroupBySlice()

	// Parse path into bits.
	for slice, bits := range bitsBySlice {
		logger.Printf("importing slice: %d, n=%d", slice, len(bits))
		if err := cmd.Client.Import(ctx, cmd.Database, cmd.Frame, slice, bits); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>modify pilosactl import to allow it to read from stdin<commit_after>package pilosactl

import (
	"context"
	"encoding/csv"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/pilosa/pilosa"
)

// ImportCommand represents a command for bulk importing data.
type ImportCommand struct {
	// Destination host and port.
	Host string `json:"host"`

	// Name of the database & frame to import into.
	Database string `json:"db"`
	Frame    string `json:"frame"`

	// Filenames to import from.
	Paths []string `json:"paths"`

	// Size of buffer used to chunk import.
	BufferSize int `json:"buffer-size"`

	// Reusable client.
	Client *pilosa.Client `json:"-"`

	// Standard input/output
	Stdin  io.Reader `json:"-"`
	Stdout io.Writer `json:"-"`
	Stderr io.Writer `json:"-"`
}

// NewImportCommand returns a new instance of ImportCommand.
func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *ImportCommand {
	return &ImportCommand{
		Stdin:  stdin,
		Stdout: stdout,
		Stderr: stderr,

		BufferSize: 10000000,
	}
}

func (cmd *ImportCommand) String() string {
	return fmt.Sprint(*cmd)
}

// ParseFlags parses command line flags from args.
func (cmd *ImportCommand) ParseFlags(args []string) error {
	fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
	fs.SetOutput(ioutil.Discard)
	fs.StringVar(&cmd.Host, "host", "localhost:15000", "host:port")
	fs.StringVar(&cmd.Database, "d", "", "database")
	fs.StringVar(&cmd.Frame, "f", "", "frame")
	fs.IntVar(&cmd.BufferSize, "buffer-size", cmd.BufferSize, "buffer size")
	if err := fs.Parse(args); err != nil {
		return err
	}

	// Extract the import paths.
	cmd.Paths = fs.Args()

	return nil
}

// Usage returns the usage message to be printed.
func (cmd *ImportCommand) Usage() string {
	return strings.TrimSpace(`
usage: pilosactl import -host HOST -d database -f frame paths

Bulk imports one or more CSV files to a host's database and frame. The bits
of the CSV file are grouped by slice for the most efficient import.

The format of the CSV file is:

	BITMAPID,PROFILEID,[TIME]

The file should contain no headers. The TIME column is optional and can be
omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
`)
}

// Run executes the main program execution.
func (cmd *ImportCommand) Run(ctx context.Context) error {
	logger := log.New(cmd.Stderr, "", log.LstdFlags)

	// Validate arguments.
	// Database and frame are validated early before the files are parsed.
	if cmd.Database == "" {
		return pilosa.ErrDatabaseRequired
	} else if cmd.Frame == "" {
		return pilosa.ErrFrameRequired
	} else if len(cmd.Paths) == 0 {
		return errors.New("path required")
	}

	// Create a client to the server.
	client, err := pilosa.NewClient(cmd.Host)
	if err != nil {
		return err
	}
	cmd.Client = client

	// Import each path and import by slice.
	for _, path := range cmd.Paths {
		// Parse path into bits.
		logger.Printf("parsing: %s", path)
		if err := cmd.importPath(ctx, path); err != nil {
			return err
		}
	}

	return nil
}

// importPath parses a path into bits and imports it to the server.
func (cmd *ImportCommand) importPath(ctx context.Context, path string) error {
	a := make([]pilosa.Bit, 0, cmd.BufferSize)

	var r *csv.Reader

	if path != "-" {
		// Open file for reading.
		f, err := os.Open(path)
		if err != nil {
			return err
		}
		defer f.Close()

		// Read rows as bits.
		r = csv.NewReader(f)
	} else {
		r = csv.NewReader(cmd.Stdin)
	}

	r.FieldsPerRecord = -1
	rnum := 0
	for {
		rnum++

		// Read CSV row.
		record, err := r.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		// Ignore blank rows.
		if record[0] == "" {
			continue
		} else if len(record) < 2 {
			return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record))
		}

		var bit pilosa.Bit

		// Parse bitmap id.
		bitmapID, err := strconv.ParseUint(record[0], 10, 64)
		if err != nil {
			return fmt.Errorf("invalid bitmap id on row %d: %q", rnum, record[0])
		}
		bit.BitmapID = bitmapID

		// Parse bitmap id.
		profileID, err := strconv.ParseUint(record[1], 10, 64)
		if err != nil {
			return fmt.Errorf("invalid profile id on row %d: %q", rnum, record[1])
		}
		bit.ProfileID = profileID

		// Parse time, if exists.
		if len(record) > 2 && record[2] != "" {
			t, err := time.Parse(pilosa.TimeFormat, record[2])
			if err != nil {
				return fmt.Errorf("invalid timestamp on row %d: %q", rnum, record[2])
			}
			bit.Timestamp = t.UnixNano()
		}

		a = append(a, bit)

		// If we've reached the buffer size then import bits.
		if len(a) == cmd.BufferSize {
			if err := cmd.importBits(ctx, a); err != nil {
				return err
			}
			a = a[:0]
		}
	}

	// If there are still bits in the buffer then flush them.
	if err := cmd.importBits(ctx, a); err != nil {
		return err
	}

	return nil
}

// importPath parses a path into bits and imports it to the server.
func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) error {
	logger := log.New(cmd.Stderr, "", log.LstdFlags)

	// Group bits by slice.
	logger.Printf("grouping %d bits", len(bits))
	bitsBySlice := pilosa.Bits(bits).GroupBySlice()

	// Parse path into bits.
	for slice, bits := range bitsBySlice {
		logger.Printf("importing slice: %d, n=%d", slice, len(bits))
		if err := cmd.Client.Import(ctx, cmd.Database, cmd.Frame, slice, bits); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package util

import (
	"errors"
	"fmt"
	"io"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/spf13/cobra"

	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

var commaSepVarsPattern = regexp.MustCompile(".*=.*,.*=.*")

// ReplaceCommandName recursively processes the examples in a given command to change a hardcoded
// command name (like 'kubectl' to the appropriate target name). It returns c.
func ReplaceCommandName(from, to string, c *cobra.Command) *cobra.Command {
	c.Example = strings.Replace(c.Example, from, to, -1)
	for _, sub := range c.Commands() {
		ReplaceCommandName(from, to, sub)
	}
	return c
}

// GetDisplayFilename returns the absolute path of the filename as long as there was no error, otherwise it returns the filename as-is
func GetDisplayFilename(filename string) string {
	if absName, err := filepath.Abs(filename); err == nil {
		return absName
	}

	return filename
}

// ResolveResource returns the resource type and name of the resourceString.
// If the resource string has no specified type, defaultResource will be returned.
func ResolveResource(defaultResource schema.GroupResource, resourceString string, mapper meta.RESTMapper) (schema.GroupResource, string, error) {
	if mapper == nil {
		return schema.GroupResource{}, "", errors.New("mapper cannot be nil")
	}

	var name string
	parts := strings.Split(resourceString, "/")
	switch len(parts) {
	case 1:
		name = parts[0]
	case 2:
		name = parts[1]

		// Allow specifying the group the same way kubectl does, as "resource.group.name"
		groupResource := schema.ParseGroupResource(parts[0])
		// normalize resource case
		groupResource.Resource = strings.ToLower(groupResource.Resource)

		gvr, err := mapper.ResourceFor(groupResource.WithVersion(""))
		if err != nil {
			return schema.GroupResource{}, "", err
		}
		return gvr.GroupResource(), name, nil
	default:
		return schema.GroupResource{}, "", fmt.Errorf("invalid resource format: %s", resourceString)
	}

	return defaultResource, name, nil
}

func WarnAboutCommaSeparation(errout io.Writer, values []string, flag string) {
	if errout == nil {
		return
	}
	for _, value := range values {
		if commaSepVarsPattern.MatchString(value) {
			fmt.Fprintf(errout, "warning: %s no longer accepts comma-separated lists of values. %q will be treated as a single key-value pair.\n", flag, value)
		}
	}
}
<commit_msg>Bug 1558935 - Replace kubectl name in long description of oc commands<commit_after>package util

import (
	"errors"
	"fmt"
	"io"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/spf13/cobra"

	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

var commaSepVarsPattern = regexp.MustCompile(".*=.*,.*=.*")

// ReplaceCommandName recursively processes the examples in a given command to change a hardcoded
// command name (like 'kubectl' to the appropriate target name). It returns c.
func ReplaceCommandName(from, to string, c *cobra.Command) *cobra.Command {
	c.Example = strings.Replace(c.Example, from, to, -1)
	c.Long = strings.Replace(c.Long, from, to, -1)
	for _, sub := range c.Commands() {
		ReplaceCommandName(from, to, sub)
	}
	return c
}

// GetDisplayFilename returns the absolute path of the filename as long as there was no error, otherwise it returns the filename as-is
func GetDisplayFilename(filename string) string {
	if absName, err := filepath.Abs(filename); err == nil {
		return absName
	}

	return filename
}

// ResolveResource returns the resource type and name of the resourceString.
// If the resource string has no specified type, defaultResource will be returned.
func ResolveResource(defaultResource schema.GroupResource, resourceString string, mapper meta.RESTMapper) (schema.GroupResource, string, error) {
	if mapper == nil {
		return schema.GroupResource{}, "", errors.New("mapper cannot be nil")
	}

	var name string
	parts := strings.Split(resourceString, "/")
	switch len(parts) {
	case 1:
		name = parts[0]
	case 2:
		name = parts[1]

		// Allow specifying the group the same way kubectl does, as "resource.group.name"
		groupResource := schema.ParseGroupResource(parts[0])
		// normalize resource case
		groupResource.Resource = strings.ToLower(groupResource.Resource)

		gvr, err := mapper.ResourceFor(groupResource.WithVersion(""))
		if err != nil {
			return schema.GroupResource{}, "", err
		}
		return gvr.GroupResource(), name, nil
	default:
		return schema.GroupResource{}, "", fmt.Errorf("invalid resource format: %s", resourceString)
	}

	return defaultResource, name, nil
}

func WarnAboutCommaSeparation(errout io.Writer, values []string, flag string) {
	if errout == nil {
		return
	}
	for _, value := range values {
		if commaSepVarsPattern.MatchString(value) {
			fmt.Fprintf(errout, "warning: %s no longer accepts comma-separated lists of values. %q will be treated as a single key-value pair.\n", flag, value)
		}
	}
}
<|endoftext|>


<commit_before>package dgroup

import (
	"context"
	"os"
	"os/signal"
	"runtime/pprof"
	"sort"
	"strings"
	"syscall"

	"github.com/pkg/errors"

	"github.com/datawire/ambassador/pkg/dcontext"
	"github.com/datawire/ambassador/pkg/derrgroup"
	"github.com/datawire/ambassador/pkg/dlog"
)

// Group is a wrapper around
// github.com/datawire/ambassador/pkg/derrgroup.Group that:
//  - (optionally) handles SIGINT and SIGTERM
//  - (configurable) manages Context for you
//  - (optionally) adds hard/soft cancellation
//  - (optionally) does some minimal logging
type Group struct {
	cfg     GroupConfig
	baseCtx context.Context
	inner   *derrgroup.Group
}

func logGoroutineStatuses(ctx context.Context, printf func(ctx context.Context, format string, args ...interface{}), list map[string]derrgroup.GoroutineState) {
	printf(ctx, "  goroutine shutdown status:")
	names := make([]string, 0, len(list))
	nameWidth := 0
	for name := range list {
		names = append(names, name)
		if len(name) > nameWidth {
			nameWidth = len(name)
		}
	}
	sort.Strings(names)
	for _, name := range names {
		printf(ctx, "    %-*s: %s", nameWidth, name, list[name])
	}
}

func logGoroutineTraces(ctx context.Context, printf func(ctx context.Context, format string, args ...interface{})) {
	p := pprof.Lookup("goroutine")
	if p == nil {
		return
	}
	stacktrace := new(strings.Builder)
	if err := p.WriteTo(stacktrace, 2); err != nil {
		return
	}
	printf(ctx, "  goroutine stack traces:")
	for _, line := range strings.Split(strings.TrimSpace(stacktrace.String()), "\n") {
		printf(ctx, "    %s", line)
	}
}

// GroupConfig is a readable way of setting the configuration options
// for NewGroup.
type GroupConfig struct {
	// EnableWithSoftness says whether it should call
	// dcontext.WithSoftness() on the Context passed to NewGroup.
	// This should probably NOT be set for a Context that is
	// already soft.  However, this must be set for features that
	// require separate hard/soft cancellation, such as signal
	// handling.  If any of those features are enabled, then it
	// will force EnableWithSoftness to be set.
	EnableWithSoftness   bool
	EnableSignalHandling bool // implies EnableWithSoftness

	DisableLogging bool

	WorkerContext func(ctx context.Context, name string) context.Context
}

// NewGroup returns a new Group.
func NewGroup(ctx context.Context, cfg GroupConfig) *Group {
	cfg.EnableWithSoftness = cfg.EnableWithSoftness || cfg.EnableSignalHandling

	ctx, hardCancel := context.WithCancel(ctx)
	var softCancel context.CancelFunc
	if cfg.EnableWithSoftness {
		ctx = dcontext.WithSoftness(ctx)
		ctx, softCancel = context.WithCancel(ctx)
	} else {
		softCancel = hardCancel
	}

	g := &Group{
		cfg:     cfg,
		baseCtx: ctx,
		inner:   derrgroup.NewGroup(softCancel),
	}

	if !g.cfg.DisableLogging {
		g.Go("supervisor", func(ctx context.Context) error {
			<-ctx.Done()
			// log that a shutdown has been triggered
			// be as specific with the logging as possible
			if dcontext.HardContext(ctx) == ctx {
				dlog.Infoln(ctx, "shutting down...")
			} else {
				select {
				case <-dcontext.HardContext(ctx).Done():
					dlog.Infoln(ctx, "shutting down (not-so-gracefully)...")
				default:
					dlog.Infoln(ctx, "shutting down (gracefully)...")
					<-dcontext.HardContext(ctx).Done()
					dlog.Infoln(ctx, "shutting down (not-so-gracefully)...")
				}
			}
			return nil
		})
	}

	if g.cfg.EnableSignalHandling {
		g.Go("signal_handler", func(ctx context.Context) error {
			sigs := make(chan os.Signal, 1)
			signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

			defer func() {
				// If we receive another signal after
				// graceful-shutdown, we should trigger a
				// not-so-graceful shutdown.
				go func() {
					select {
					case sig := <-sigs:
						if !g.cfg.DisableLogging {
							dlog.Errorln(ctx, errors.Errorf("received signal %v (graceful shutdown already triggered; triggering not-so-graceful shutdown)", sig))
							logGoroutineStatuses(ctx, dlog.Errorf, g.List())
						}
						hardCancel()
					case <-dcontext.HardContext(ctx).Done():
					}
					// keep logging signals and draining 'sigs'--don't let 'sigs' block
					for sig := range sigs {
						if !g.cfg.DisableLogging {
							dlog.Errorln(ctx, errors.Errorf("received signal %v (not-so-graceful shutdown already triggered)", sig))
							logGoroutineStatuses(ctx, dlog.Errorf, g.List())
							logGoroutineTraces(ctx, dlog.Errorf)
						}
					}
				}()
			}()

			select {
			case sig := <-sigs:
				return errors.Errorf("received signal %v (first signal; triggering graceful shutdown)", sig)
			case <-ctx.Done():
				return nil
			}
		})
	}

	return g
}

// Go wraps derrgroup.Group.Go().
//
// Cancellation of the Context should trigger a graceful shutdown.
// Cancellation of the dcontext.HardContext(ctx) of it should trigger
// a not-so-graceful shutdown.
func (g *Group) Go(name string, fn func(ctx context.Context) error) {
	g.inner.Go(name, func() error {
		ctx := g.baseCtx
		ctx = WithGoroutineName(ctx, "/"+name)
		if g.cfg.WorkerContext != nil {
			ctx = g.cfg.WorkerContext(ctx, name)
		}
		err := fn(ctx)
		if !g.cfg.DisableLogging {
			if err == nil {
				dlog.Debugln(ctx, "goroutine exited without error")
			} else {
				dlog.Errorln(ctx, "goroutine exited with error:", err)
			}
		}
		return err
	})
}

// Wait wraps derrgroup.Group.Wait().
func (g *Group) Wait() error {
	ret := g.inner.Wait()
	if ret != nil && !g.cfg.DisableLogging {
		ctx := WithGoroutineName(g.baseCtx, ":shutdown_status")
		logGoroutineStatuses(ctx, dlog.Infof, g.List())
	}
	return ret
}

// List wraps derrgroup.Group.List().
func (g *Group) List() map[string]derrgroup.GoroutineState {
	return g.inner.List()
}
<commit_msg>(from AES) dgroup: Add panic recovery<commit_after>package dgroup

import (
	"context"
	"os"
	"os/signal"
	"runtime/pprof"
	"sort"
	"strings"
	"syscall"

	"github.com/pkg/errors"

	"github.com/datawire/ambassador/pkg/dcontext"
	"github.com/datawire/ambassador/pkg/derrgroup"
	"github.com/datawire/ambassador/pkg/dlog"
	"github.com/datawire/ambassador/pkg/errutil"
)

// Group is a wrapper around
// github.com/datawire/ambassador/pkg/derrgroup.Group that:
//  - (optionally) handles SIGINT and SIGTERM
//  - (configurable) manages Context for you
//  - (optionally) adds hard/soft cancellation
//  - (optionally) does panic recovery
//  - (optionally) does some minimal logging
type Group struct {
	cfg     GroupConfig
	baseCtx context.Context
	inner   *derrgroup.Group
}

func logGoroutineStatuses(ctx context.Context, printf func(ctx context.Context, format string, args ...interface{}), list map[string]derrgroup.GoroutineState) {
	printf(ctx, "  goroutine shutdown status:")
	names := make([]string, 0, len(list))
	nameWidth := 0
	for name := range list {
		names = append(names, name)
		if len(name) > nameWidth {
			nameWidth = len(name)
		}
	}
	sort.Strings(names)
	for _, name := range names {
		printf(ctx, "    %-*s: %s", nameWidth, name, list[name])
	}
}

func logGoroutineTraces(ctx context.Context, printf func(ctx context.Context, format string, args ...interface{})) {
	p := pprof.Lookup("goroutine")
	if p == nil {
		return
	}
	stacktrace := new(strings.Builder)
	if err := p.WriteTo(stacktrace, 2); err != nil {
		return
	}
	printf(ctx, "  goroutine stack traces:")
	for _, line := range strings.Split(strings.TrimSpace(stacktrace.String()), "\n") {
		printf(ctx, "    %s", line)
	}
}

// GroupConfig is a readable way of setting the configuration options
// for NewGroup.
type GroupConfig struct {
	// EnableWithSoftness says whether it should call
	// dcontext.WithSoftness() on the Context passed to NewGroup.
	// This should probably NOT be set for a Context that is
	// already soft.  However, this must be set for features that
	// require separate hard/soft cancellation, such as signal
	// handling.  If any of those features are enabled, then it
	// will force EnableWithSoftness to be set.
	EnableWithSoftness   bool
	EnableSignalHandling bool // implies EnableWithSoftness

	DisablePanicRecovery bool
	DisableLogging       bool

	WorkerContext func(ctx context.Context, name string) context.Context
}

// NewGroup returns a new Group.
func NewGroup(ctx context.Context, cfg GroupConfig) *Group {
	cfg.EnableWithSoftness = cfg.EnableWithSoftness || cfg.EnableSignalHandling

	ctx, hardCancel := context.WithCancel(ctx)
	var softCancel context.CancelFunc
	if cfg.EnableWithSoftness {
		ctx = dcontext.WithSoftness(ctx)
		ctx, softCancel = context.WithCancel(ctx)
	} else {
		softCancel = hardCancel
	}

	g := &Group{
		cfg:     cfg,
		baseCtx: ctx,
		inner:   derrgroup.NewGroup(softCancel),
	}

	if !g.cfg.DisableLogging {
		g.Go("supervisor", func(ctx context.Context) error {
			<-ctx.Done()
			// log that a shutdown has been triggered
			// be as specific with the logging as possible
			if dcontext.HardContext(ctx) == ctx {
				dlog.Infoln(ctx, "shutting down...")
			} else {
				select {
				case <-dcontext.HardContext(ctx).Done():
					dlog.Infoln(ctx, "shutting down (not-so-gracefully)...")
				default:
					dlog.Infoln(ctx, "shutting down (gracefully)...")
					<-dcontext.HardContext(ctx).Done()
					dlog.Infoln(ctx, "shutting down (not-so-gracefully)...")
				}
			}
			return nil
		})
	}

	if g.cfg.EnableSignalHandling {
		g.Go("signal_handler", func(ctx context.Context) error {
			sigs := make(chan os.Signal, 1)
			signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

			defer func() {
				// If we receive another signal after
				// graceful-shutdown, we should trigger a
				// not-so-graceful shutdown.
				go func() {
					select {
					case sig := <-sigs:
						if !g.cfg.DisableLogging {
							dlog.Errorln(ctx, errors.Errorf("received signal %v (graceful shutdown already triggered; triggering not-so-graceful shutdown)", sig))
							logGoroutineStatuses(ctx, dlog.Errorf, g.List())
						}
						hardCancel()
					case <-dcontext.HardContext(ctx).Done():
					}
					// keep logging signals and draining 'sigs'--don't let 'sigs' block
					for sig := range sigs {
						if !g.cfg.DisableLogging {
							dlog.Errorln(ctx, errors.Errorf("received signal %v (not-so-graceful shutdown already triggered)", sig))
							logGoroutineStatuses(ctx, dlog.Errorf, g.List())
							logGoroutineTraces(ctx, dlog.Errorf)
						}
					}
				}()
			}()

			select {
			case sig := <-sigs:
				return errors.Errorf("received signal %v (first signal; triggering graceful shutdown)", sig)
			case <-ctx.Done():
				return nil
			}
		})
	}

	return g
}

// Go wraps derrgroup.Group.Go().
//
// Cancellation of the Context should trigger a graceful shutdown.
// Cancellation of the dcontext.HardContext(ctx) of it should trigger
// a not-so-graceful shutdown.
func (g *Group) Go(name string, fn func(ctx context.Context) error) {
	g.inner.Go(name, func() (err error) {
		ctx := g.baseCtx
		ctx = WithGoroutineName(ctx, "/"+name)
		if g.cfg.WorkerContext != nil {
			ctx = g.cfg.WorkerContext(ctx, name)
		}

		defer func() {
			if !g.cfg.DisablePanicRecovery {
				if _err := errutil.PanicToError(recover()); _err != nil {
					err = _err
				}
			}
			if !g.cfg.DisableLogging {
				if err == nil {
					dlog.Debugln(ctx, "goroutine exited without error")
				} else {
					dlog.Errorln(ctx, "goroutine exited with error:", err)
				}
			}
		}()

		return fn(ctx)
	})
}

// Wait wraps derrgroup.Group.Wait().
func (g *Group) Wait() error {
	ret := g.inner.Wait()
	if ret != nil && !g.cfg.DisableLogging {
		ctx := WithGoroutineName(g.baseCtx, ":shutdown_status")
		logGoroutineStatuses(ctx, dlog.Infof, g.List())
	}
	return ret
}

// List wraps derrgroup.Group.List().
func (g *Group) List() map[string]derrgroup.GoroutineState {
	return g.inner.List()
}
<|endoftext|>


<commit_before>package docker

import (
	"strings"

	"github.com/docker/cli/cli/command/image"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/client"
	"github.com/flant/logboek"
	"golang.org/x/net/context"
)

func Images(options types.ImageListOptions) ([]types.ImageSummary, error) {
	ctx := context.Background()
	images, err := apiClient.ImageList(ctx, options)
	if err != nil {
		return nil, err
	}

	return images, nil
}

func ImageExist(ref string) (bool, error) {
	if _, err := ImageInspect(ref); err != nil {
		if client.IsErrNotFound(err) {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

func ImageInspect(ref string) (*types.ImageInspect, error) {
	ctx := context.Background()
	inspect, _, err := apiClient.ImageInspectWithRaw(ctx, ref)
	if err != nil {
		return nil, err
	}

	return &inspect, nil
}

const cliPullMaxAttempts = 5

func CliPullWithRetries(args ...string) error {
	var attempt int

tryPull:
	if err := CliPull(args...); err != nil {
		if attempt < cliPullMaxAttempts {
			specificErrors := []string{
				"Client.Timeout exceeded while awaiting headers",
				"TLS handshake timeout",
				"i/o timeout",
			}

			for _, specificError := range specificErrors {
				if strings.Index(err.Error(), specificError) != -1 {
					attempt += 1

					logboek.LogInfoF("Retrying (%d/%d) ...\n", attempt, cliPullMaxAttempts)
					goto tryPull
				}
			}
		}

		return err
	}

	return nil
}

func CliPull(args ...string) error {
	cmd := image.NewPullCommand(cli)
	cmd.SilenceErrors = true
	cmd.SilenceUsage = true
	cmd.SetArgs(args)

	err := cmd.Execute()
	if err != nil {
		return err
	}

	return nil
}

func CliPush(args ...string) error {
	cmd := image.NewPushCommand(cli)
	cmd.SilenceErrors = true
	cmd.SilenceUsage = true
	cmd.SetArgs(args)

	err := cmd.Execute()
	if err != nil {
		return err
	}

	return nil
}

const cliPushMaxAttempts = 5

func CliPushWithRetries(args ...string) error {
	var attempt int

tryPush:
	if err := CliPush(args...); err != nil {
		if attempt < cliPushMaxAttempts {
			specificErrors := []string{
				"TLS handshake timeout",
				"i/o timeout",
			}

			for _, specificError := range specificErrors {
				if strings.Index(err.Error(), specificError) != -1 {
					attempt += 1

					logboek.LogInfoF("Retrying (%d/%d) ...\n", attempt, cliPushMaxAttempts)
					goto tryPush
				}
			}
		}

		return err
	}

	return nil
}

func CliTag(args ...string) error {
	cmd := image.NewTagCommand(cli)
	cmd.SilenceErrors = true
	cmd.SilenceUsage = true
	cmd.SetArgs(args)

	err := cmd.Execute()
	if err != nil {
		return err
	}

	return nil
}

func CliRmi(args ...string) error {
	cmd := image.NewRemoveCommand(cli)
	cmd.SilenceErrors = true
	cmd.SilenceUsage = true
	cmd.SetArgs(args)

	err := cmd.Execute()
	if err != nil {
		return err
	}

	return nil
}

func CliBuild(args ...string) error {
	cmd := image.NewBuildCommand(cli)
	cmd.SilenceErrors = true
	cmd.SilenceUsage = true
	cmd.SetArgs(args)

	err := cmd.Execute()
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Retry docker push when `Client.Timeout exceeded while awaiting headers` error occurred<commit_after>package docker

import (
	"strings"

	"github.com/docker/cli/cli/command/image"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/client"
	"github.com/flant/logboek"
	"golang.org/x/net/context"
)

func Images(options types.ImageListOptions) ([]types.ImageSummary, error) {
	ctx := context.Background()
	images, err := apiClient.ImageList(ctx, options)
	if err != nil {
		return nil, err
	}

	return images, nil
}

func ImageExist(ref string) (bool, error) {
	if _, err := ImageInspect(ref); err != nil {
		if client.IsErrNotFound(err) {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

func ImageInspect(ref string) (*types.ImageInspect, error) {
	ctx := context.Background()
	inspect, _, err := apiClient.ImageInspectWithRaw(ctx, ref)
	if err != nil {
		return nil, err
	}

	return &inspect, nil
}

const cliPullMaxAttempts = 5

func CliPullWithRetries(args ...string) error {
	var attempt int

tryPull:
	if err := CliPull(args...); err != nil {
		if attempt < cliPullMaxAttempts {
			specificErrors := []string{
				"Client.Timeout exceeded while awaiting headers",
				"TLS handshake timeout",
				"i/o timeout",
			}

			for _, specificError := range specificErrors {
				if strings.Index(err.Error(), specificError) != -1 {
					attempt += 1

					logboek.LogInfoF("Retrying (%d/%d) ...\n", attempt, cliPullMaxAttempts)
					goto tryPull
				}
			}
		}

		return err
	}

	return nil
}

func CliPull(args ...string) error {
	cmd := image.NewPullCommand(cli)
	cmd.SilenceErrors = true
	cmd.SilenceUsage = true
	cmd.SetArgs(args)

	err := cmd.Execute()
	if err != nil {
		return err
	}

	return nil
}

func CliPush(args ...string) error {
	cmd := image.NewPushCommand(cli)
	cmd.SilenceErrors = true
	cmd.SilenceUsage = true
	cmd.SetArgs(args)

	err := cmd.Execute()
	if err != nil {
		return err
	}

	return nil
}

const cliPushMaxAttempts = 5

func CliPushWithRetries(args ...string) error {
	var attempt int

tryPush:
	if err := CliPush(args...); err != nil {
		if attempt < cliPushMaxAttempts {
			specificErrors := []string{
				"Client.Timeout exceeded while awaiting headers",
				"TLS handshake timeout",
				"i/o timeout",
			}

			for _, specificError := range specificErrors {
				if strings.Index(err.Error(), specificError) != -1 {
					attempt += 1

					logboek.LogInfoF("Retrying (%d/%d) ...\n", attempt, cliPushMaxAttempts)
					goto tryPush
				}
			}
		}

		return err
	}

	return nil
}

func CliTag(args ...string) error {
	cmd := image.NewTagCommand(cli)
	cmd.SilenceErrors = true
	cmd.SilenceUsage = true
	cmd.SetArgs(args)

	err := cmd.Execute()
	if err != nil {
		return err
	}

	return nil
}

func CliRmi(args ...string) error {
	cmd := image.NewRemoveCommand(cli)
	cmd.SilenceErrors = true
	cmd.SilenceUsage = true
	cmd.SetArgs(args)

	err := cmd.Execute()
	if err != nil {
		return err
	}

	return nil
}

func CliBuild(args ...string) error {
	cmd := image.NewBuildCommand(cli)
	cmd.SilenceErrors = true
	cmd.SilenceUsage = true
	cmd.SetArgs(args)

	err := cmd.Execute()
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package hyper

import (
	"strings"

	"github.com/golang/glog"
	kubeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
)

// ListImages lists existing images.
func (h *Runtime) ListImages(filter *kubeapi.ImageFilter) ([]*kubeapi.Image, error) {
	images, err := h.client.GetImages()
	if err != nil {
		glog.Errorf("Get image list failed: %v", err)
		return nil, err
	}

	var results []*kubeapi.Image
	for _, img := range images {
		if filter != nil && filter.GetImage().Image != "" {
			filter := filter.GetImage().Image
			// Use 'latest' tag if not specified explicitly
			if !strings.Contains(filter, ":") {
				filter = filter + ":latest"
			}

			if !inList(filter, img.RepoTags) && !inList(filter, img.RepoDigests) {
				continue
			}
		}

		imageSize := uint64(img.VirtualSize)
		results = append(results, &kubeapi.Image{
			Id:          img.Id,
			RepoTags:    img.RepoTags,
			RepoDigests: img.RepoDigests,
			Size_:       imageSize,
		})
	}

	glog.V(4).Infof("Got imageList: %q", results)
	return results, nil
}

// PullImage pulls the image with authentication config.
func (h *Runtime) PullImage(image *kubeapi.ImageSpec, authConfig *kubeapi.AuthConfig) (string, error) {
	repo, tag := parseRepositoryTag(image.Image)
	auth := getHyperAuthConfig(authConfig)
	err := h.client.PullImage(repo, tag, auth, nil)
	if err != nil {
		glog.Errorf("Pull image %q failed: %v", image.Image, err)
		return "", err
	}

	imageInfo, err := h.client.GetImageInfo(repo, tag)
	if err != nil {
		glog.Errorf("Get image info for %q failed: %v", image.Image, err)
		return "", err
	}

	return imageInfo.Id, nil
}

// RemoveImage removes the image.
func (h *Runtime) RemoveImage(image *kubeapi.ImageSpec) error {
	repo, tag := parseRepositoryTag(image.Image)
	err := h.client.RemoveImage(repo, tag)
	if err != nil {
		glog.Errorf("Remove image %q failed: %v", image.Image, err)
		return err
	}

	return nil
}

// ImageStatus returns the status of the image.
func (h *Runtime) ImageStatus(image *kubeapi.ImageSpec) (*kubeapi.Image, error) {
	repo, tag := parseRepositoryTag(image.Image)
	imageInfo, err := h.client.GetImageInfo(repo, tag)
	if err != nil {
		if strings.Contains(err.Error(), "not found") {
			return nil, nil
		}
		glog.Errorf("Get image info for %q failed: %v", image.Image, err)
		return nil, err
	}

	imageSize := uint64(imageInfo.VirtualSize)
	return &kubeapi.Image{
		Id:          imageInfo.Id,
		RepoTags:    imageInfo.RepoTags,
		RepoDigests: imageInfo.RepoDigests,
		Size_:       imageSize,
	}, nil
}
<commit_msg>Avoid panic when image spec in filter is nil<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package hyper

import (
	"strings"

	"github.com/golang/glog"
	kubeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
)

// ListImages lists existing images.
func (h *Runtime) ListImages(filter *kubeapi.ImageFilter) ([]*kubeapi.Image, error) {
	images, err := h.client.GetImages()
	if err != nil {
		glog.Errorf("Get image list failed: %v", err)
		return nil, err
	}

	var results []*kubeapi.Image
	for _, img := range images {
		if filter != nil && filter.GetImage() != nil && filter.GetImage().Image != "" {
			filter := filter.GetImage().Image
			// Use 'latest' tag if not specified explicitly
			if !strings.Contains(filter, ":") {
				filter = filter + ":latest"
			}

			if !inList(filter, img.RepoTags) && !inList(filter, img.RepoDigests) {
				continue
			}
		}

		imageSize := uint64(img.VirtualSize)
		results = append(results, &kubeapi.Image{
			Id:          img.Id,
			RepoTags:    img.RepoTags,
			RepoDigests: img.RepoDigests,
			Size_:       imageSize,
		})
	}

	glog.V(4).Infof("Got imageList: %q", results)
	return results, nil
}

// PullImage pulls the image with authentication config.
func (h *Runtime) PullImage(image *kubeapi.ImageSpec, authConfig *kubeapi.AuthConfig) (string, error) {
	repo, tag := parseRepositoryTag(image.Image)
	auth := getHyperAuthConfig(authConfig)
	err := h.client.PullImage(repo, tag, auth, nil)
	if err != nil {
		glog.Errorf("Pull image %q failed: %v", image.Image, err)
		return "", err
	}

	imageInfo, err := h.client.GetImageInfo(repo, tag)
	if err != nil {
		glog.Errorf("Get image info for %q failed: %v", image.Image, err)
		return "", err
	}

	return imageInfo.Id, nil
}

// RemoveImage removes the image.
func (h *Runtime) RemoveImage(image *kubeapi.ImageSpec) error {
	repo, tag := parseRepositoryTag(image.Image)
	err := h.client.RemoveImage(repo, tag)
	if err != nil {
		glog.Errorf("Remove image %q failed: %v", image.Image, err)
		return err
	}

	return nil
}

// ImageStatus returns the status of the image.
func (h *Runtime) ImageStatus(image *kubeapi.ImageSpec) (*kubeapi.Image, error) {
	repo, tag := parseRepositoryTag(image.Image)
	imageInfo, err := h.client.GetImageInfo(repo, tag)
	if err != nil {
		if strings.Contains(err.Error(), "not found") {
			return nil, nil
		}
		glog.Errorf("Get image info for %q failed: %v", image.Image, err)
		return nil, err
	}

	imageSize := uint64(imageInfo.VirtualSize)
	return &kubeapi.Image{
		Id:          imageInfo.Id,
		RepoTags:    imageInfo.RepoTags,
		RepoDigests: imageInfo.RepoDigests,
		Size_:       imageSize,
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The kubecfg authors
//
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package kubecfg

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"regexp"
	"sort"

	isatty "github.com/mattn/go-isatty"
	"github.com/sergi/go-diff/diffmatchpatch"
	log "github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/client-go/dynamic"

	"github.com/bitnami/kubecfg/utils"
)

var ErrDiffFound = fmt.Errorf("Differences found.")

// Matches all the line starts on a diff text, which is where we put diff markers and indent
var DiffLineStart = regexp.MustCompile("(^|\n)(.)")

var DiffKeyValue = regexp.MustCompile(`"([[:alnum:]_-]+)":\s"([[:alnum:]=+]+)",?`)

// DiffCmd represents the diff subcommand
type DiffCmd struct {
	Client           dynamic.Interface
	Mapper           meta.RESTMapper
	DefaultNamespace string
	OmitSecrets      bool

	DiffStrategy string
}

func (c DiffCmd) Run(apiObjects []*unstructured.Unstructured, out io.Writer) error {
	sort.Sort(utils.AlphabeticalOrder(apiObjects))

	dmp := diffmatchpatch.New()
	diffFound := false
	for _, obj := range apiObjects {
		desc := fmt.Sprintf("%s %s", utils.ResourceNameFor(c.Mapper, obj), utils.FqName(obj))
		log.Debug("Fetching ", desc)

		client, err := utils.ClientForResource(c.Client, c.Mapper, obj, c.DefaultNamespace)
		if err != nil {
			return err
		}

		liveObj, err := client.Get(obj.GetName(), metav1.GetOptions{})
		if err != nil && errors.IsNotFound(err) {
			log.Debugf("%s doesn't exist on the server", desc)
			liveObj = nil
		} else if err != nil {
			return fmt.Errorf("Error fetching %s: %v", desc, err)
		}

		fmt.Fprintln(out, "---")
		fmt.Fprintf(out, "- live %s\n+ config %s\n", desc, desc)
		if liveObj == nil {
			fmt.Fprintf(out, "%s doesn't exist on server\n", desc)
			diffFound = true
			continue
		}

		liveObjObject := liveObj.Object
		if c.DiffStrategy == "subset" {
			liveObjObject = removeMapFields(obj.Object, liveObjObject)
		}

		liveObjText, _ := json.MarshalIndent(liveObjObject, "", "  ")
		objText, _ := json.MarshalIndent(obj.Object, "", "  ")

		liveObjTextLines, objTextLines, lines := dmp.DiffLinesToChars(string(liveObjText), string(objText))

		diff := dmp.DiffMain(
			string(liveObjTextLines),
			string(objTextLines),
			false)

		diff = dmp.DiffCharsToLines(diff, lines)
		if (len(diff) == 1) && (diff[0].Type == diffmatchpatch.DiffEqual) {
			fmt.Fprintf(out, "%s unchanged\n", desc)
		} else {
			diffFound = true
			text := c.formatDiff(diff, isatty.IsTerminal(os.Stdout.Fd()), c.OmitSecrets && obj.GetKind() == "Secret")
			fmt.Fprintf(out, "%s\n", text)
		}
	}

	if diffFound {
		return ErrDiffFound
	}
	return nil
}

// Formats the supplied Diff as a unified-diff-like text with infinite context and optionally colorizes it.
func (c DiffCmd) formatDiff(diffs []diffmatchpatch.Diff, color bool, omitchanges bool) string {
	var buff bytes.Buffer

	for _, diff := range diffs {
		text := diff.Text

		if omitchanges {
			text = DiffKeyValue.ReplaceAllString(text, "$1: <omitted>")
		}
		switch diff.Type {
		case diffmatchpatch.DiffInsert:
			if color {
				_, _ = buff.WriteString("\x1b[32m")
			}
			_, _ = buff.WriteString(DiffLineStart.ReplaceAllString(text, "$1+ $2"))
			if color {
				_, _ = buff.WriteString("\x1b[0m")
			}
		case diffmatchpatch.DiffDelete:
			if color {
				_, _ = buff.WriteString("\x1b[31m")
			}
			_, _ = buff.WriteString(DiffLineStart.ReplaceAllString(text, "$1- $2"))
			if color {
				_, _ = buff.WriteString("\x1b[0m")
			}
		case diffmatchpatch.DiffEqual:
			if !omitchanges {
				_, _ = buff.WriteString(DiffLineStart.ReplaceAllString(text, "$1  $2"))
			}
		}
	}

	return buff.String()
}

// See also feature request for golang reflect pkg at
func isEmptyValue(i interface{}) bool {
	switch v := i.(type) {
	case []interface{}:
		return len(v) == 0
	case []string:
		return len(v) == 0
	case map[string]interface{}:
		return len(v) == 0
	case bool:
		return !v
	case float64:
		return v == 0
	case int64:
		return v == 0
	case string:
		return v == ""
	case nil:
		return true
	default:
		panic(fmt.Sprintf("Found unexpected type %T in json unmarshal (value=%v)", i, i))
	}
}

func removeFields(config, live interface{}) interface{} {
	switch c := config.(type) {
	case map[string]interface{}:
		return removeMapFields(c, live.(map[string]interface{}))
	case []interface{}:
		return removeListFields(c, live.([]interface{}))
	default:
		return live
	}
}

func removeMapFields(config, live map[string]interface{}) map[string]interface{} {
	result := map[string]interface{}{}
	for k, v1 := range config {
		v2, ok := live[k]
		if !ok {
			// Copy empty value from config, as API won't return them,
			// see https://github.com/bitnami/kubecfg/issues/179
			if isEmptyValue(v1) {
				result[k] = v1
			}
			continue
		}
		result[k] = removeFields(v1, v2)
	}
	return result
}

func removeListFields(config, live []interface{}) []interface{} {
	// If live is longer than config, then the extra elements at the end of the
	// list will be returned as is so they appear in the diff.
	result := make([]interface{}, 0, len(live))
	for i, v2 := range live {
		if len(config) > i {
			result = append(result, removeFields(config[i], v2))
		} else {
			result = append(result, v2)
		}
	}
	return result
}

func istty(w io.Writer) bool {
	if f, ok := w.(*os.File); ok {
		return isatty.IsTerminal(f.Fd())
	}
	return false
}
<commit_msg>change regex to the one used for internal kubernetes validation<commit_after>// Copyright 2017 The kubecfg authors
//
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package kubecfg

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"regexp"
	"sort"

	isatty "github.com/mattn/go-isatty"
	"github.com/sergi/go-diff/diffmatchpatch"
	log "github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/client-go/dynamic"

	"github.com/bitnami/kubecfg/utils"
)

var ErrDiffFound = fmt.Errorf("Differences found.")

// Matches all the line starts on a diff text, which is where we put diff markers and indent
var DiffLineStart = regexp.MustCompile("(^|\n)(.)")

var DiffKeyValue = regexp.MustCompile(`"([-._a-zA-Z0-9]+)":\s"([[:alnum:]=+]+)",?`)

// DiffCmd represents the diff subcommand
type DiffCmd struct {
	Client           dynamic.Interface
	Mapper           meta.RESTMapper
	DefaultNamespace string
	OmitSecrets      bool

	DiffStrategy string
}

func (c DiffCmd) Run(apiObjects []*unstructured.Unstructured, out io.Writer) error {
	sort.Sort(utils.AlphabeticalOrder(apiObjects))

	dmp := diffmatchpatch.New()
	diffFound := false
	for _, obj := range apiObjects {
		desc := fmt.Sprintf("%s %s", utils.ResourceNameFor(c.Mapper, obj), utils.FqName(obj))
		log.Debug("Fetching ", desc)

		client, err := utils.ClientForResource(c.Client, c.Mapper, obj, c.DefaultNamespace)
		if err != nil {
			return err
		}

		liveObj, err := client.Get(obj.GetName(), metav1.GetOptions{})
		if err != nil && errors.IsNotFound(err) {
			log.Debugf("%s doesn't exist on the server", desc)
			liveObj = nil
		} else if err != nil {
			return fmt.Errorf("Error fetching %s: %v", desc, err)
		}

		fmt.Fprintln(out, "---")
		fmt.Fprintf(out, "- live %s\n+ config %s\n", desc, desc)
		if liveObj == nil {
			fmt.Fprintf(out, "%s doesn't exist on server\n", desc)
			diffFound = true
			continue
		}

		liveObjObject := liveObj.Object
		if c.DiffStrategy == "subset" {
			liveObjObject = removeMapFields(obj.Object, liveObjObject)
		}

		liveObjText, _ := json.MarshalIndent(liveObjObject, "", "  ")
		objText, _ := json.MarshalIndent(obj.Object, "", "  ")

		liveObjTextLines, objTextLines, lines := dmp.DiffLinesToChars(string(liveObjText), string(objText))

		diff := dmp.DiffMain(
			string(liveObjTextLines),
			string(objTextLines),
			false)

		diff = dmp.DiffCharsToLines(diff, lines)
		if (len(diff) == 1) && (diff[0].Type == diffmatchpatch.DiffEqual) {
			fmt.Fprintf(out, "%s unchanged\n", desc)
		} else {
			diffFound = true
			text := c.formatDiff(diff, isatty.IsTerminal(os.Stdout.Fd()), c.OmitSecrets && obj.GetKind() == "Secret")
			fmt.Fprintf(out, "%s\n", text)
		}
	}

	if diffFound {
		return ErrDiffFound
	}
	return nil
}

// Formats the supplied Diff as a unified-diff-like text with infinite context and optionally colorizes it.
func (c DiffCmd) formatDiff(diffs []diffmatchpatch.Diff, color bool, omitchanges bool) string {
	var buff bytes.Buffer

	for _, diff := range diffs {
		text := diff.Text

		if omitchanges {
			text = DiffKeyValue.ReplaceAllString(text, "$1: <omitted>")
		}
		switch diff.Type {
		case diffmatchpatch.DiffInsert:
			if color {
				_, _ = buff.WriteString("\x1b[32m")
			}
			_, _ = buff.WriteString(DiffLineStart.ReplaceAllString(text, "$1+ $2"))
			if color {
				_, _ = buff.WriteString("\x1b[0m")
			}
		case diffmatchpatch.DiffDelete:
			if color {
				_, _ = buff.WriteString("\x1b[31m")
			}
			_, _ = buff.WriteString(DiffLineStart.ReplaceAllString(text, "$1- $2"))
			if color {
				_, _ = buff.WriteString("\x1b[0m")
			}
		case diffmatchpatch.DiffEqual:
			if !omitchanges {
				_, _ = buff.WriteString(DiffLineStart.ReplaceAllString(text, "$1  $2"))
			}
		}
	}

	return buff.String()
}

// See also feature request for golang reflect pkg at
func isEmptyValue(i interface{}) bool {
	switch v := i.(type) {
	case []interface{}:
		return len(v) == 0
	case []string:
		return len(v) == 0
	case map[string]interface{}:
		return len(v) == 0
	case bool:
		return !v
	case float64:
		return v == 0
	case int64:
		return v == 0
	case string:
		return v == ""
	case nil:
		return true
	default:
		panic(fmt.Sprintf("Found unexpected type %T in json unmarshal (value=%v)", i, i))
	}
}

func removeFields(config, live interface{}) interface{} {
	switch c := config.(type) {
	case map[string]interface{}:
		return removeMapFields(c, live.(map[string]interface{}))
	case []interface{}:
		return removeListFields(c, live.([]interface{}))
	default:
		return live
	}
}

func removeMapFields(config, live map[string]interface{}) map[string]interface{} {
	result := map[string]interface{}{}
	for k, v1 := range config {
		v2, ok := live[k]
		if !ok {
			// Copy empty value from config, as API won't return them,
			// see https://github.com/bitnami/kubecfg/issues/179
			if isEmptyValue(v1) {
				result[k] = v1
			}
			continue
		}
		result[k] = removeFields(v1, v2)
	}
	return result
}

func removeListFields(config, live []interface{}) []interface{} {
	// If live is longer than config, then the extra elements at the end of the
	// list will be returned as is so they appear in the diff.
	result := make([]interface{}, 0, len(live))
	for i, v2 := range live {
		if len(config) > i {
			result = append(result, removeFields(config[i], v2))
		} else {
			result = append(result, v2)
		}
	}
	return result
}

func istty(w io.Writer) bool {
	if f, ok := w.(*os.File); ok {
		return isatty.IsTerminal(f.Fd())
	}
	return false
}
<|endoftext|>


<commit_before>package model

type Config struct {
	ConfigPath string
	TokenFile  string
	ConfigFile string
}
<commit_msg>add api addr field<commit_after>package model

type Config struct {
	ConfigPath string
	TokenFile  string
	ConfigFile string
	APIaddr    string
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package osutil

import (
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

func HomeDir() string {
	if runtime.GOOS == "windows" {
		return os.Getenv("HOMEPATH")
	}
	return os.Getenv("HOME")
}

func CacheDir() string {
	switch runtime.GOOS {
	case "darwin":
		return filepath.Join(HomeDir(), "Library", "Caches", "Camlistore")
	case "windows":
		panic("CacheDir not implemented on OS == " + runtime.GOOS)
	}
	return filepath.Join(HomeDir(), ".cache")
}

func CamliBlobRoot() string {
	switch runtime.GOOS {
	case "windows":
		return filepath.Join(os.Getenv("APPDATA"), "Camlistore", "blobs")
	case "darwin":
		return filepath.Join(HomeDir(), "Library", "Camlistore", "blobs")
	}
	return filepath.Join(HomeDir(), "var", "camlistore", "blobs")
}

func CamliConfigDir() string {
	if p := os.Getenv("CAMLI_CONFIG_DIR"); p != "" {
		return p
	}
	if runtime.GOOS == "windows" {
		return filepath.Join(os.Getenv("APPDATA"), "Camlistore")
	}
	return filepath.Join(HomeDir(), ".camlistore")
}

func UserServerConfigPath() string {
	return filepath.Join(CamliConfigDir(), "server-config.json")
}

func IdentitySecretRing() string {
	return filepath.Join(CamliConfigDir(), "identity-secring.gpg")
}

// Find the correct absolute path corresponding to a relative path,
// searching the following sequence of directories:
// 1. Working Directory
// 2. CAMLI_CONFIG_DIR (deprecated, will complain if this is on env)
// 3. (windows only) APPDATA/camli
// 4. All directories in CAMLI_INCLUDE_PATH (standard PATH form for OS)
func FindCamliInclude(configFile string) (absPath string, err error) {
	// Try to open as absolute / relative to CWD
	_, err = os.Stat(configFile)
	if err == nil {
		return configFile, nil
	}
	if filepath.IsAbs(configFile) {
		// End of the line for absolute path
		return "", err
	}

	// Try the config dir
	configDir := CamliConfigDir()
	if _, err = os.Stat(filepath.Join(configDir, configFile)); err == nil {
		return filepath.Join(configDir, configFile), nil
	}

	// Finally, search CAMLI_INCLUDE_PATH
	p := os.Getenv("CAMLI_INCLUDE_PATH")
	for _, d := range strings.Split(p, string(filepath.ListSeparator)) {
		if _, err = os.Stat(filepath.Join(d, configFile)); err == nil {
			return filepath.Join(d, configFile), nil
		}
	}

	return "", os.ErrNotExist
}
<commit_msg>osutil: create cache directory on first access<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package osutil

import (
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
)

func HomeDir() string {
	if runtime.GOOS == "windows" {
		return os.Getenv("HOMEPATH")
	}
	return os.Getenv("HOME")
}

var cacheDirOnce sync.Once

func CacheDir() string {
	cacheDirOnce.Do(makeCacheDir)
	return cacheDir()
}

func cacheDir() string {
	switch runtime.GOOS {
	case "darwin":
		return filepath.Join(HomeDir(), "Library", "Caches", "Camlistore")
	case "windows":
		panic("CacheDir not implemented on OS == " + runtime.GOOS)
	}
	return filepath.Join(HomeDir(), ".cache", "camlistore")
}

func makeCacheDir() {
	os.Mkdir(cacheDir(), 0700)
}

func CamliBlobRoot() string {
	switch runtime.GOOS {
	case "windows":
		return filepath.Join(os.Getenv("APPDATA"), "Camlistore", "blobs")
	case "darwin":
		return filepath.Join(HomeDir(), "Library", "Camlistore", "blobs")
	}
	return filepath.Join(HomeDir(), "var", "camlistore", "blobs")
}

func CamliConfigDir() string {
	if p := os.Getenv("CAMLI_CONFIG_DIR"); p != "" {
		return p
	}
	if runtime.GOOS == "windows" {
		return filepath.Join(os.Getenv("APPDATA"), "Camlistore")
	}
	return filepath.Join(HomeDir(), ".camlistore")
}

func UserServerConfigPath() string {
	return filepath.Join(CamliConfigDir(), "server-config.json")
}

func IdentitySecretRing() string {
	return filepath.Join(CamliConfigDir(), "identity-secring.gpg")
}

// Find the correct absolute path corresponding to a relative path,
// searching the following sequence of directories:
// 1. Working Directory
// 2. CAMLI_CONFIG_DIR (deprecated, will complain if this is on env)
// 3. (windows only) APPDATA/camli
// 4. All directories in CAMLI_INCLUDE_PATH (standard PATH form for OS)
func FindCamliInclude(configFile string) (absPath string, err error) {
	// Try to open as absolute / relative to CWD
	_, err = os.Stat(configFile)
	if err == nil {
		return configFile, nil
	}
	if filepath.IsAbs(configFile) {
		// End of the line for absolute path
		return "", err
	}

	// Try the config dir
	configDir := CamliConfigDir()
	if _, err = os.Stat(filepath.Join(configDir, configFile)); err == nil {
		return filepath.Join(configDir, configFile), nil
	}

	// Finally, search CAMLI_INCLUDE_PATH
	p := os.Getenv("CAMLI_INCLUDE_PATH")
	for _, d := range strings.Split(p, string(filepath.ListSeparator)) {
		if _, err = os.Stat(filepath.Join(d, configFile)); err == nil {
			return filepath.Join(d, configFile), nil
		}
	}

	return "", os.ErrNotExist
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/docker/libnetwork/ipamapi"
	go_docker "github.com/fsouza/go-dockerclient"
	. "github.com/weaveworks/weave/common"
	"github.com/weaveworks/weave/common/docker"
	ipamplugin "github.com/weaveworks/weave/plugin/ipam"
	netplugin "github.com/weaveworks/weave/plugin/net"
	"github.com/weaveworks/weave/plugin/skel"
)

var version = "(unreleased version)"

func main() {
	var (
		justVersion      bool
		address          string
		nameserver       string
		meshAddress      string
		logLevel         string
		meshNetworkName  string
		noMulticastRoute bool
		removeNetwork    bool
	)

	flag.BoolVar(&justVersion, "version", false, "print version and exit")
	flag.StringVar(&logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
	flag.StringVar(&address, "socket", "/run/docker/plugins/weave.sock", "socket on which to listen")
	flag.StringVar(&nameserver, "nameserver", "", "nameserver to provide to containers")
	flag.StringVar(&meshAddress, "meshsocket", "/run/docker/plugins/weavemesh.sock", "socket on which to listen in mesh mode")
	flag.StringVar(&meshNetworkName, "mesh-network-name", "weave", "network name to create in mesh mode")
	flag.BoolVar(&noMulticastRoute, "no-multicast-route", false, "do not add a multicast route to network endpoints")
	flag.BoolVar(&removeNetwork, "remove-network", false, "remove mesh network and exit")

	flag.Parse()

	if justVersion {
		fmt.Printf("weave plugin %s\n", version)
		os.Exit(0)
	}

	SetLogLevel(logLevel)

	// API 1.21 is the first version that supports docker network commands
	dockerClient, err := docker.NewVersionedClientFromEnv("1.21")
	if err != nil {
		Log.Fatalf("unable to connect to docker: %s", err)
	}

	if removeNetwork {
		if _, err = dockerClient.Client.NetworkInfo(meshNetworkName); err == nil {
			err = dockerClient.Client.RemoveNetwork(meshNetworkName)
			if err != nil {
				Log.Fatalf("unable to remove network: %s", err)
			}
		}
		os.Exit(0)
	}

	Log.Println("Weave plugin", version, "Command line options:", os.Args[1:])
	Log.Info(dockerClient.Info())

	var globalListener, meshListener net.Listener
	endChan := make(chan error, 1)
	if address != "" {
		globalListener, err := listenAndServe(dockerClient, address, nameserver, noMulticastRoute, endChan, "global", false)
		if err != nil {
			Log.Fatalf("unable to create driver: %s", err)
		}
		defer globalListener.Close()
	}
	if meshAddress != "" {
		meshListener, err := listenAndServe(dockerClient, meshAddress, nameserver, noMulticastRoute, endChan, "local", true)
		if err != nil {
			Log.Fatalf("unable to create driver: %s", err)
		}
		defer meshListener.Close()
	}

	if meshNetworkName != "" {
		createNetwork(dockerClient, meshNetworkName, meshAddress)
	}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, os.Kill, syscall.SIGTERM)

	select {
	case sig := <-sigChan:
		Log.Debugf("Caught signal %s; shutting down", sig)
		if err := dockerClient.Client.RemoveNetwork(meshNetworkName); err != nil {
			Log.Fatal(err)
		}
	case err := <-endChan:
		if err != nil {
			Log.Errorf("Error from listener: %s", err)
			globalListener.Close()
			meshListener.Close()
			os.Exit(1)
		}
	}
}

func listenAndServe(dockerClient *docker.Client, address, nameserver string, noMulticastRoute bool, endChan chan<- error, scope string, withIpam bool) (net.Listener, error) {
	d, err := netplugin.New(dockerClient, version, nameserver, scope, noMulticastRoute)
	if err != nil {
		return nil, err
	}

	var i ipamapi.Ipam
	if withIpam {
		if i, err = ipamplugin.NewIpam(dockerClient, version); err != nil {
			return nil, err
		}
	}

	var listener net.Listener

	// remove sockets from last invocation
	if err := os.Remove(address); err != nil && !os.IsNotExist(err) {
		return nil, err
	}
	listener, err = net.Listen("unix", address)
	if err != nil {
		return nil, err
	}
	Log.Printf("Listening on %s for %s scope", address, scope)

	go func() {
		endChan <- skel.Listen(listener, d, i)
	}()

	return listener, nil
}

func createNetwork(dockerClient *docker.Client, networkName, address string) {
	if _, err := dockerClient.Client.NetworkInfo(networkName); err == nil {
		Log.Printf("Docker network '%s' already exists", networkName)
	} else if _, ok := err.(*go_docker.NoSuchNetwork); ok {
		driverName := strings.TrimSuffix(address, ".sock")
		if i := strings.LastIndex(driverName, "/"); i >= 0 {
			driverName = driverName[i+1:]
		}
		options := go_docker.CreateNetworkOptions{
			Name:           networkName,
			CheckDuplicate: true,
			Driver:         driverName,
			IPAM:           go_docker.IPAMOptions{Driver: driverName},
		}
		_, err := dockerClient.Client.CreateNetwork(options)
		if err != nil {
			Log.Fatalf("Error creating network: %s", err)
		}
	}
}
<commit_msg>Remove Unix domain sockets on shutdown<commit_after>package main

import (
	"flag"
	"fmt"
	"net"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/docker/libnetwork/ipamapi"
	go_docker "github.com/fsouza/go-dockerclient"
	. "github.com/weaveworks/weave/common"
	"github.com/weaveworks/weave/common/docker"
	ipamplugin "github.com/weaveworks/weave/plugin/ipam"
	netplugin "github.com/weaveworks/weave/plugin/net"
	"github.com/weaveworks/weave/plugin/skel"
)

var version = "(unreleased version)"

func main() {
	var (
		justVersion      bool
		address          string
		nameserver       string
		meshAddress      string
		logLevel         string
		meshNetworkName  string
		noMulticastRoute bool
		removeNetwork    bool
	)

	flag.BoolVar(&justVersion, "version", false, "print version and exit")
	flag.StringVar(&logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
	flag.StringVar(&address, "socket", "/run/docker/plugins/weave.sock", "socket on which to listen")
	flag.StringVar(&nameserver, "nameserver", "", "nameserver to provide to containers")
	flag.StringVar(&meshAddress, "meshsocket", "/run/docker/plugins/weavemesh.sock", "socket on which to listen in mesh mode")
	flag.StringVar(&meshNetworkName, "mesh-network-name", "weave", "network name to create in mesh mode")
	flag.BoolVar(&noMulticastRoute, "no-multicast-route", false, "do not add a multicast route to network endpoints")
	flag.BoolVar(&removeNetwork, "remove-network", false, "remove mesh network and exit")

	flag.Parse()

	if justVersion {
		fmt.Printf("weave plugin %s\n", version)
		os.Exit(0)
	}

	SetLogLevel(logLevel)

	// API 1.21 is the first version that supports docker network commands
	dockerClient, err := docker.NewVersionedClientFromEnv("1.21")
	if err != nil {
		Log.Fatalf("unable to connect to docker: %s", err)
	}

	if removeNetwork {
		if _, err = dockerClient.Client.NetworkInfo(meshNetworkName); err == nil {
			err = dockerClient.Client.RemoveNetwork(meshNetworkName)
			if err != nil {
				Log.Fatalf("unable to remove network: %s", err)
			}
		}
		os.Exit(0)
	}

	Log.Println("Weave plugin", version, "Command line options:", os.Args[1:])
	Log.Info(dockerClient.Info())

	var globalListener, meshListener net.Listener
	endChan := make(chan error, 1)
	if address != "" {
		globalListener, err := listenAndServe(dockerClient, address, nameserver, noMulticastRoute, endChan, "global", false)
		if err != nil {
			Log.Fatalf("unable to create driver: %s", err)
		}
		defer globalListener.Close()
	}
	if meshAddress != "" {
		meshListener, err := listenAndServe(dockerClient, meshAddress, nameserver, noMulticastRoute, endChan, "local", true)
		if err != nil {
			Log.Fatalf("unable to create driver: %s", err)
		}
		defer meshListener.Close()
	}

	if meshNetworkName != "" {
		createNetwork(dockerClient, meshNetworkName, meshAddress)
	}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt, os.Kill, syscall.SIGTERM)

	cleanup := func() {
		globalListener.Close()
		meshListener.Close()
		_ = os.Remove(address)
		_ = os.Remove(meshAddress)
	}
	select {
	case sig := <-sigChan:
		Log.Debugf("Caught signal %s; shutting down", sig)
		if err := dockerClient.Client.RemoveNetwork(meshNetworkName); err != nil {
			Log.Fatal(err)
		}
		cleanup()
	case err := <-endChan:
		cleanup()
		if err != nil {
			Log.Errorf("Error from listener: %s", err)
			os.Exit(1)
		}
	}
}

func listenAndServe(dockerClient *docker.Client, address, nameserver string, noMulticastRoute bool, endChan chan<- error, scope string, withIpam bool) (net.Listener, error) {
	d, err := netplugin.New(dockerClient, version, nameserver, scope, noMulticastRoute)
	if err != nil {
		return nil, err
	}

	var i ipamapi.Ipam
	if withIpam {
		if i, err = ipamplugin.NewIpam(dockerClient, version); err != nil {
			return nil, err
		}
	}

	var listener net.Listener

	// remove sockets from last invocation
	if err := os.Remove(address); err != nil && !os.IsNotExist(err) {
		return nil, err
	}
	listener, err = net.Listen("unix", address)
	if err != nil {
		return nil, err
	}
	Log.Printf("Listening on %s for %s scope", address, scope)

	go func() {
		endChan <- skel.Listen(listener, d, i)
	}()

	return listener, nil
}

func createNetwork(dockerClient *docker.Client, networkName, address string) {
	if _, err := dockerClient.Client.NetworkInfo(networkName); err == nil {
		Log.Printf("Docker network '%s' already exists", networkName)
	} else if _, ok := err.(*go_docker.NoSuchNetwork); ok {
		driverName := strings.TrimSuffix(address, ".sock")
		if i := strings.LastIndex(driverName, "/"); i >= 0 {
			driverName = driverName[i+1:]
		}
		options := go_docker.CreateNetworkOptions{
			Name:           networkName,
			CheckDuplicate: true,
			Driver:         driverName,
			IPAM:           go_docker.IPAMOptions{Driver: driverName},
		}
		_, err := dockerClient.Client.CreateNetwork(options)
		if err != nil {
			Log.Fatalf("Error creating network: %s", err)
		}
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2012 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package proto

import (
	"testing"
	"fmt"
)

func testSendingCommands(t *testing.T, from, to *commandIO, cmds ...*command) {
	errCh := make(chan error)
	go func() {
		for i, cmd := range cmds {
			recved, err := to.ReadCommand()
			if err != nil {
				errCh <- err
				return
			}
			if !cmd.eq(recved) {
				errCh <- fmt.Errorf("%vth command does not equal", i)
			}
		}
		close(errCh)
	}()

	for _, cmd := range cmds {
		err := from.WriteCommand(cmd)
		if err != nil {
			t.Errorf("Error on write: %v", err)
		}
	}

	for err := range errCh {
		if err != nil {
			t.Errorf("Error on write: %v", err)
		}
	}
}

func testExchangingCommands(t *testing.T, compress, encrypt bool, cmds ...*command) {
	sks, cks, s2c, c2s := exchangeKeysOrReport(t)
	if sks == nil || cks == nil || s2c == nil || c2s == nil {
		return
	}

	scmdio := sks.getServerCommandIO(s2c)
	if !compress {
		scmdio.ReadCompressOff()
		scmdio.WriteCompressOff()
	}
	if !encrypt {
		scmdio.ReadEncryptOff()
		scmdio.WriteEncryptOff()
	}

	ccmdio := cks.getServerCommandIO(c2s)
	if !compress {
		ccmdio.ReadCompressOff()
		ccmdio.WriteCompressOff()
	}
	if !encrypt {
		ccmdio.ReadEncryptOff()
		ccmdio.WriteEncryptOff()
	}
	testSendingCommands(t, scmdio, ccmdio, cmds...)
	testSendingCommands(t, ccmdio, scmdio, cmds...)
}

func TestExchangingFullCommandNoCompressNoEncrypt(t *testing.T) {
	cmd := new(command)
	cmd.Body = []byte{1,2,3}
	cmd.Type = 1
	cmd.Params = make([][]byte, 2)
	cmd.Params[0] = []byte{1,2,3}
	cmd.Params[1] = []byte{2,2,3}
	cmd.Header = make(map[string]string, 2)
	cmd.Header["a"] = "hello"
	cmd.Header["b"] = "hell"
	testExchangingCommands(t, false, false, cmd)
}

<commit_msg>Command IO: encryption failed<commit_after>/*
 * Copyright 2012 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package proto

import (
	"testing"
	"fmt"
)

func testSendingCommands(t *testing.T, from, to *commandIO, cmds ...*command) {
	errCh := make(chan error)
	go func() {
		for i, cmd := range cmds {
			recved, err := to.ReadCommand()
			if err != nil {
				errCh <- err
				return
			}
			if !cmd.eq(recved) {
				errCh <- fmt.Errorf("%vth command does not equal", i)
			}
		}
		close(errCh)
	}()

	for _, cmd := range cmds {
		err := from.WriteCommand(cmd)
		if err != nil {
			t.Errorf("Error on write: %v", err)
		}
	}

	for err := range errCh {
		if err != nil {
			t.Errorf("Error on write: %v", err)
		}
	}
}

func testExchangingCommands(t *testing.T, compress, encrypt bool, cmds ...*command) {
	sks, cks, s2c, c2s := exchangeKeysOrReport(t)
	if sks == nil || cks == nil || s2c == nil || c2s == nil {
		return
	}

	scmdio := sks.getServerCommandIO(s2c)
	if !compress {
		scmdio.ReadCompressOff()
		scmdio.WriteCompressOff()
	}
	if !encrypt {
		scmdio.ReadEncryptOff()
		scmdio.WriteEncryptOff()
	}

	ccmdio := cks.getServerCommandIO(c2s)
	if !compress {
		ccmdio.ReadCompressOff()
		ccmdio.WriteCompressOff()
	}
	if !encrypt {
		ccmdio.ReadEncryptOff()
		ccmdio.WriteEncryptOff()
	}
	testSendingCommands(t, scmdio, ccmdio, cmds...)
	testSendingCommands(t, ccmdio, scmdio, cmds...)
}

func TestExchangingFullCommandNoCompressNoEncrypt(t *testing.T) {
	cmd := new(command)
	cmd.Body = []byte{1,2,3}
	cmd.Type = 1
	cmd.Params = make([][]byte, 2)
	cmd.Params[0] = []byte{1,2,3}
	cmd.Params[1] = []byte{2,2,3}
	cmd.Header = make(map[string]string, 2)
	cmd.Header["a"] = "hello"
	cmd.Header["b"] = "hell"
	testExchangingCommands(t, false, false, cmd)
}

func TestExchangingFullCommandNoCompress(t *testing.T) {
	cmd := new(command)
	cmd.Body = []byte{1,2,3}
	cmd.Type = 1
	cmd.Params = make([][]byte, 2)
	cmd.Params[0] = []byte{1,2,3}
	cmd.Params[1] = []byte{2,2,3}
	cmd.Header = make(map[string]string, 2)
	cmd.Header["a"] = "hello"
	cmd.Header["b"] = "hell"
	testExchangingCommands(t, true, false, cmd)
}

func TestExchangingFullCommand(t *testing.T) {
	cmd := new(command)
	cmd.Body = []byte{1,2,3}
	cmd.Type = 1
	cmd.Params = make([][]byte, 2)
	cmd.Params[0] = []byte{1,2,3}
	cmd.Params[1] = []byte{2,2,3}
	cmd.Header = make(map[string]string, 2)
	cmd.Header["a"] = "hello"
	cmd.Header["b"] = "hell"
	testExchangingCommands(t, true, true, cmd)
}

<|endoftext|>