<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Peter Mattis (peter@cockroachlabs.com)

package parser

import (
	"fmt"
	"strings"
)

type normalizableExpr interface {
	Expr
	normalize(*normalizeVisitor) TypedExpr
}

func (expr *AndExpr) normalize(v *normalizeVisitor) TypedExpr {
	left := expr.TypedLeft()
	right := expr.TypedRight()

	// Use short-circuit evaluation to simplify AND expressions.
	if v.isConst(left) {
		left, v.err = left.Eval(v.ctx)
		if v.err != nil {
			return expr
		}
		if left != DNull {
			if d, err := GetBool(left.(Datum)); err == nil {
				if !d {
					return left
				}
				return right
			}
			return DNull
		}
		return NewTypedAndExpr(
			left,
			right,
		)
	}
	if v.isConst(right) {
		right, v.err = right.Eval(v.ctx)
		if v.err != nil {
			return expr
		}
		if right != DNull {
			if d, err := GetBool(expr.Right.(Datum)); err == nil {
				if !d {
					return right
				}
				return left
			}
			return DNull
		}
		return NewTypedAndExpr(
			left,
			right,
		)
	}
	return expr
}

func (expr *ComparisonExpr) normalize(v *normalizeVisitor) TypedExpr {

	switch expr.Operator {
	case EQ, GE, GT, LE, LT:
		// We want var nodes (VariableExpr, QualifiedName, etc) to be immediate
		// children of the comparison expression and not second or third
		// children. That is, we want trees that look like:
		//
		//    cmp            cmp
		//   /   \          /   \
		//  a    op        op    a
		//      /  \      /  \
		//     1    2    1    2
		//
		// Not trees that look like:
		//
		//      cmp          cmp        cmp          cmp
		//     /   \        /   \      /   \        /   \
		//    op    2      op    2    1    op      1    op
		//   /  \         /  \            /  \         /  \
		//  a    1       1    a          a    2       2    a
		//
		// We loop attempting to simplify the comparison expression. As a
		// pre-condition, we know there is at least one variable in the expression
		// tree or we would not have entered this code path.
		exprCopied := false
		for {
			if v.isConst(expr.Left) {
				switch expr.Right.(type) {
				case *BinaryExpr, VariableExpr:
					break
				default:
					return expr
				}
				// The left side is const and the right side is a binary expression or a
				// variable. Flip the comparison op so that the right side is const and
				// the left side is a binary expression or variable.
				// Create a new ComparisonExpr so the function cache isn't reused.
				if !exprCopied {
					exprCopy := *expr
					expr = &exprCopy
					exprCopied = true
				}
				*expr = ComparisonExpr{
					Operator: invertComparisonOp(expr.Operator),
					Left:     expr.Right,
					Right:    expr.Left,
					typeAnnotation: typeAnnotation{
						typ: TypeBool,
					},
				}
			} else if !v.isConst(expr.Right) {
				return expr
			}

			left, ok := expr.Left.(*BinaryExpr)
			if !ok {
				return expr
			}

			// The right is const and the left side is a binary expression. Rotate the
			// comparison combining portions that are const.

			switch {
			case v.isConst(left.Right) &&
				(left.Operator == Plus || left.Operator == Minus || left.Operator == Div):

				//        cmp          cmp
				//       /   \        /   \
				//    [+-/]   2  ->  a   [-+*]
				//   /     \            /     \
				//  a       1          2       1

				if !exprCopied {
					exprCopy := *expr
					expr = &exprCopy
					exprCopied = true
				}

				leftCopy := *left
				left = &leftCopy

				switch left.Operator {
				case Plus:
					left.Operator = Minus
				case Minus:
					left.Operator = Plus
				case Div:
					left.Operator = Mult
				}

				// Clear the function caches since we're rotating.
				left.fn.fn = nil
				expr.fn.fn = nil

				expr.Left = left.Left
				left.Left = expr.Right
				expr.Right, v.err = left.Eval(v.ctx)
				if v.err != nil {
					return nil
				}
				if !isVar(expr.Left) {
					// Continue as long as the left side of the comparison is not a
					// variable.
					continue
				}

			case v.isConst(left.Left) && (left.Operator == Plus || left.Operator == Minus):
				//       cmp              cmp
				//      /   \            /   \
				//    [+-]   2  ->     [+-]   a
				//   /    \           /    \
				//  1      a         1      2

				if !exprCopied {
					exprCopy := *expr
					expr = &exprCopy
					exprCopied = true
				}

				leftCopy := *left
				left = &leftCopy

				// Clear the function caches; we're about to change stuff.
				expr.fn.fn = nil
				left.fn.fn = nil
				left.Right, expr.Right = expr.Right, left.Right
				if left.Operator == Plus {
					left.Operator = Minus
					left.Left, left.Right = left.Right, left.Left
				} else {
					expr.Operator = invertComparisonOp(expr.Operator)
				}
				expr.Left, v.err = left.Eval(v.ctx)
				if v.err != nil {
					return nil
				}
				expr.Left, expr.Right = expr.Right, expr.Left
				if !isVar(expr.Left) {
					// Continue as long as the left side of the comparison is not a
					// variable.
					continue
				}
			}

			// We've run out of work to do.
			break
		}
	case In, NotIn:
		// If the right tuple in an In or NotIn comparison expression is constant, it can
		// be normalized.
		tuple, ok := expr.Right.(*DTuple)
		if ok {
			tupleCopy := *tuple
			tupleCopy.Normalize()
			exprCopy := *expr
			expr = &exprCopy
			expr.Right = &tupleCopy
		}
	}

	return expr
}

func (expr *OrExpr) normalize(v *normalizeVisitor) TypedExpr {
	left := expr.TypedLeft()
	right := expr.TypedRight()

	// Use short-circuit evaluation to simplify OR expressions.
	if v.isConst(left) {
		left, v.err = left.Eval(v.ctx)
		if v.err != nil {
			return expr
		}
		if left != DNull {
			if d, err := GetBool(left.(Datum)); err == nil {
				if d {
					return left
				}
				return right
			}
			return DNull
		}
		return NewTypedOrExpr(
			left,
			right,
		)
	}
	if v.isConst(right) {
		right, v.err = right.Eval(v.ctx)
		if v.err != nil {
			return expr
		}
		if right != DNull {
			if d, err := GetBool(right.(Datum)); err == nil {
				if d {
					return right
				}
				return left
			}
			return DNull
		}
		return NewTypedOrExpr(
			left,
			right,
		)
	}
	return expr
}

func (expr *ParenExpr) normalize(v *normalizeVisitor) TypedExpr {
	return expr.TypedInnerExpr()
}

func (expr *RangeCond) normalize(v *normalizeVisitor) TypedExpr {
	if expr.Not {
		// "a NOT BETWEEN b AND c" -> "a < b OR a > c"
		return NewTypedOrExpr(
			NewTypedComparisonExpr(
				LT,
				expr.TypedLeft(),
				expr.TypedFrom(),
			),
			NewTypedComparisonExpr(
				GT,
				expr.TypedLeft(),
				expr.TypedTo(),
			),
		)
	}

	// "a BETWEEN b AND c" -> "a >= b AND a <= c"
	return NewTypedAndExpr(
		NewTypedComparisonExpr(
			GE,
			expr.TypedLeft(),
			expr.TypedFrom(),
		),
		NewTypedComparisonExpr(
			LE,
			expr.TypedLeft(),
			expr.TypedTo(),
		),
	)
}

// NormalizeExpr normalizes a typed expression, simplifying where possible,
// but guaranteeing that the result of evaluating the expression is
// unchanged and that resulting expression tree is still well-typed.
// Example normalizations:
//
//   (a)                   -> a
//   a = 1 + 1             -> a = 2
//   a + 1 = 2             -> a = 1
//   a BETWEEN b AND c     -> (a >= b) AND (a <= c)
//   a NOT BETWEEN b AND c -> (a < b) OR (a > c)
func (ctx EvalContext) NormalizeExpr(typedExpr TypedExpr) (TypedExpr, error) {
	v := normalizeVisitor{ctx: ctx}
	expr, _ := WalkExpr(&v, typedExpr)
	if v.err != nil {
		return nil, v.err
	}
	return expr.(TypedExpr), nil
}

type normalizeVisitor struct {
	ctx EvalContext
	err error

	isConstVisitor isConstVisitor
}

var _ Visitor = &normalizeVisitor{}

func (v *normalizeVisitor) VisitPre(expr Expr) (recurse bool, newExpr Expr) {
	if v.err != nil {
		return false, expr
	}

	// Normalize expressions that know how to normalize themselves.
	if normalizeable, ok := expr.(normalizableExpr); ok {
		expr = normalizeable.normalize(v)
		if v.err != nil {
			return false, expr
		}
	}

	switch expr.(type) {
	case *CaseExpr, *IfExpr, *NullIfExpr, *CoalesceExpr:
		// Conditional expressions need to be evaluated during the downward
		// traversal in order to avoid evaluating sub-expressions which should
		// not be evaluated due to the case/conditional.
		if v.isConst(expr) {
			expr, v.err = expr.(TypedExpr).Eval(v.ctx)
			if v.err != nil {
				return false, expr
			}
		}
	case *Subquery:
		// Avoid normalizing subqueries. We need the subquery to be expanded in
		// order to do so properly.
		// TODO(knz) This should happen when the prepare and execute phases are
		//     separated for SelectClause.
		return false, expr
	}

	return true, expr
}

func (v *normalizeVisitor) VisitPost(expr Expr) Expr {
	if v.err != nil {
		return expr
	}

	// Normalize expressions that know how to normalize themselves.
	if normalizeable, ok := expr.(normalizableExpr); ok {
		expr = normalizeable.normalize(v)
		if v.err != nil {
			return expr
		}
	}

	// Evaluate all constant expressions.
	if v.isConst(expr) {
		expr, v.err = expr.(TypedExpr).Eval(v.ctx)
	}
	return expr
}

func (v *normalizeVisitor) isConst(expr Expr) bool {
	return v.isConstVisitor.run(expr)
}

func invertComparisonOp(op ComparisonOperator) ComparisonOperator {
	switch op {
	case EQ:
		return EQ
	case GE:
		return LE
	case GT:
		return LT
	case LE:
		return GE
	case LT:
		return GT
	default:
		panic(fmt.Sprintf("unable to invert: %s", op))
	}
}

type isConstVisitor struct {
	isConst bool
}

var _ Visitor = &isConstVisitor{}

func (v *isConstVisitor) VisitPre(expr Expr) (recurse bool, newExpr Expr) {
	if v.isConst {
		if isVar(expr) {
			v.isConst = false
			return false, expr
		}

		switch t := expr.(type) {
		case *Subquery:
			v.isConst = false
			return false, expr
		case *FuncExpr:
			// TODO(nvanbenschoten) This could be cleaned up a bit.
			if t.fn.fn == nil {
				name := string(t.Name.Base)
				candidates, _ := Builtins[strings.ToLower(name)]
				args := make(DTuple, 0, len(t.Exprs))
				for _, e := range t.Exprs {
					args = append(args, e.(TypedExpr).ReturnType())
				}
				for _, fn := range candidates {
					if fn.Types.match(ArgTypes(args)) {
						t.fn = fn
						break
					}
				}
			}
			if t.fn.fn == nil || t.fn.impure {
				v.isConst = false
				return false, expr
			}
		}
	}
	return true, expr
}

func (*isConstVisitor) VisitPost(expr Expr) Expr { return expr }

func (v *isConstVisitor) run(expr Expr) bool {
	v.isConst = true
	WalkExprConst(v, expr)
	return v.isConst
}

func isVar(expr Expr) bool {
	_, ok := expr.(VariableExpr)
	return ok
}

type containsVarsVisitor struct {
	containsVars bool
}

var _ Visitor = &containsVarsVisitor{}

func (v *containsVarsVisitor) VisitPre(expr Expr) (recurse bool, newExpr Expr) {
	if !v.containsVars && isVar(expr) {
		v.containsVars = true
	}
	if v.containsVars {
		return false, expr
	}
	return true, expr
}

func (*containsVarsVisitor) VisitPost(expr Expr) Expr { return expr }

// ContainsVars returns true if the expression contains any variables.
func ContainsVars(expr Expr) bool {
	v := containsVarsVisitor{containsVars: false}
	WalkExprConst(&v, expr)
	return v.containsVars
}
<commit_msg>sql/parser: Simplify FuncExpr handling in isConstVisitor<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Peter Mattis (peter@cockroachlabs.com)

package parser

import "fmt"

type normalizableExpr interface {
	Expr
	normalize(*normalizeVisitor) TypedExpr
}

func (expr *AndExpr) normalize(v *normalizeVisitor) TypedExpr {
	left := expr.TypedLeft()
	right := expr.TypedRight()

	// Use short-circuit evaluation to simplify AND expressions.
	if v.isConst(left) {
		left, v.err = left.Eval(v.ctx)
		if v.err != nil {
			return expr
		}
		if left != DNull {
			if d, err := GetBool(left.(Datum)); err == nil {
				if !d {
					return left
				}
				return right
			}
			return DNull
		}
		return NewTypedAndExpr(
			left,
			right,
		)
	}
	if v.isConst(right) {
		right, v.err = right.Eval(v.ctx)
		if v.err != nil {
			return expr
		}
		if right != DNull {
			if d, err := GetBool(expr.Right.(Datum)); err == nil {
				if !d {
					return right
				}
				return left
			}
			return DNull
		}
		return NewTypedAndExpr(
			left,
			right,
		)
	}
	return expr
}

func (expr *ComparisonExpr) normalize(v *normalizeVisitor) TypedExpr {

	switch expr.Operator {
	case EQ, GE, GT, LE, LT:
		// We want var nodes (VariableExpr, QualifiedName, etc) to be immediate
		// children of the comparison expression and not second or third
		// children. That is, we want trees that look like:
		//
		//    cmp            cmp
		//   /   \          /   \
		//  a    op        op    a
		//      /  \      /  \
		//     1    2    1    2
		//
		// Not trees that look like:
		//
		//      cmp          cmp        cmp          cmp
		//     /   \        /   \      /   \        /   \
		//    op    2      op    2    1    op      1    op
		//   /  \         /  \            /  \         /  \
		//  a    1       1    a          a    2       2    a
		//
		// We loop attempting to simplify the comparison expression. As a
		// pre-condition, we know there is at least one variable in the expression
		// tree or we would not have entered this code path.
		exprCopied := false
		for {
			if v.isConst(expr.Left) {
				switch expr.Right.(type) {
				case *BinaryExpr, VariableExpr:
					break
				default:
					return expr
				}
				// The left side is const and the right side is a binary expression or a
				// variable. Flip the comparison op so that the right side is const and
				// the left side is a binary expression or variable.
				// Create a new ComparisonExpr so the function cache isn't reused.
				if !exprCopied {
					exprCopy := *expr
					expr = &exprCopy
					exprCopied = true
				}
				*expr = ComparisonExpr{
					Operator: invertComparisonOp(expr.Operator),
					Left:     expr.Right,
					Right:    expr.Left,
					typeAnnotation: typeAnnotation{
						typ: TypeBool,
					},
				}
			} else if !v.isConst(expr.Right) {
				return expr
			}

			left, ok := expr.Left.(*BinaryExpr)
			if !ok {
				return expr
			}

			// The right is const and the left side is a binary expression. Rotate the
			// comparison combining portions that are const.

			switch {
			case v.isConst(left.Right) &&
				(left.Operator == Plus || left.Operator == Minus || left.Operator == Div):

				//        cmp          cmp
				//       /   \        /   \
				//    [+-/]   2  ->  a   [-+*]
				//   /     \            /     \
				//  a       1          2       1

				if !exprCopied {
					exprCopy := *expr
					expr = &exprCopy
					exprCopied = true
				}

				leftCopy := *left
				left = &leftCopy

				switch left.Operator {
				case Plus:
					left.Operator = Minus
				case Minus:
					left.Operator = Plus
				case Div:
					left.Operator = Mult
				}

				// Clear the function caches since we're rotating.
				left.fn.fn = nil
				expr.fn.fn = nil

				expr.Left = left.Left
				left.Left = expr.Right
				expr.Right, v.err = left.Eval(v.ctx)
				if v.err != nil {
					return nil
				}
				if !isVar(expr.Left) {
					// Continue as long as the left side of the comparison is not a
					// variable.
					continue
				}

			case v.isConst(left.Left) && (left.Operator == Plus || left.Operator == Minus):
				//       cmp              cmp
				//      /   \            /   \
				//    [+-]   2  ->     [+-]   a
				//   /    \           /    \
				//  1      a         1      2

				if !exprCopied {
					exprCopy := *expr
					expr = &exprCopy
					exprCopied = true
				}

				leftCopy := *left
				left = &leftCopy

				// Clear the function caches; we're about to change stuff.
				expr.fn.fn = nil
				left.fn.fn = nil
				left.Right, expr.Right = expr.Right, left.Right
				if left.Operator == Plus {
					left.Operator = Minus
					left.Left, left.Right = left.Right, left.Left
				} else {
					expr.Operator = invertComparisonOp(expr.Operator)
				}
				expr.Left, v.err = left.Eval(v.ctx)
				if v.err != nil {
					return nil
				}
				expr.Left, expr.Right = expr.Right, expr.Left
				if !isVar(expr.Left) {
					// Continue as long as the left side of the comparison is not a
					// variable.
					continue
				}
			}

			// We've run out of work to do.
			break
		}
	case In, NotIn:
		// If the right tuple in an In or NotIn comparison expression is constant, it can
		// be normalized.
		tuple, ok := expr.Right.(*DTuple)
		if ok {
			tupleCopy := *tuple
			tupleCopy.Normalize()
			exprCopy := *expr
			expr = &exprCopy
			expr.Right = &tupleCopy
		}
	}

	return expr
}

func (expr *OrExpr) normalize(v *normalizeVisitor) TypedExpr {
	left := expr.TypedLeft()
	right := expr.TypedRight()

	// Use short-circuit evaluation to simplify OR expressions.
	if v.isConst(left) {
		left, v.err = left.Eval(v.ctx)
		if v.err != nil {
			return expr
		}
		if left != DNull {
			if d, err := GetBool(left.(Datum)); err == nil {
				if d {
					return left
				}
				return right
			}
			return DNull
		}
		return NewTypedOrExpr(
			left,
			right,
		)
	}
	if v.isConst(right) {
		right, v.err = right.Eval(v.ctx)
		if v.err != nil {
			return expr
		}
		if right != DNull {
			if d, err := GetBool(right.(Datum)); err == nil {
				if d {
					return right
				}
				return left
			}
			return DNull
		}
		return NewTypedOrExpr(
			left,
			right,
		)
	}
	return expr
}

func (expr *ParenExpr) normalize(v *normalizeVisitor) TypedExpr {
	return expr.TypedInnerExpr()
}

func (expr *RangeCond) normalize(v *normalizeVisitor) TypedExpr {
	if expr.Not {
		// "a NOT BETWEEN b AND c" -> "a < b OR a > c"
		return NewTypedOrExpr(
			NewTypedComparisonExpr(
				LT,
				expr.TypedLeft(),
				expr.TypedFrom(),
			),
			NewTypedComparisonExpr(
				GT,
				expr.TypedLeft(),
				expr.TypedTo(),
			),
		)
	}

	// "a BETWEEN b AND c" -> "a >= b AND a <= c"
	return NewTypedAndExpr(
		NewTypedComparisonExpr(
			GE,
			expr.TypedLeft(),
			expr.TypedFrom(),
		),
		NewTypedComparisonExpr(
			LE,
			expr.TypedLeft(),
			expr.TypedTo(),
		),
	)
}

// NormalizeExpr normalizes a typed expression, simplifying where possible,
// but guaranteeing that the result of evaluating the expression is
// unchanged and that resulting expression tree is still well-typed.
// Example normalizations:
//
//   (a)                   -> a
//   a = 1 + 1             -> a = 2
//   a + 1 = 2             -> a = 1
//   a BETWEEN b AND c     -> (a >= b) AND (a <= c)
//   a NOT BETWEEN b AND c -> (a < b) OR (a > c)
func (ctx EvalContext) NormalizeExpr(typedExpr TypedExpr) (TypedExpr, error) {
	v := normalizeVisitor{ctx: ctx}
	expr, _ := WalkExpr(&v, typedExpr)
	if v.err != nil {
		return nil, v.err
	}
	return expr.(TypedExpr), nil
}

type normalizeVisitor struct {
	ctx EvalContext
	err error

	isConstVisitor isConstVisitor
}

var _ Visitor = &normalizeVisitor{}

func (v *normalizeVisitor) VisitPre(expr Expr) (recurse bool, newExpr Expr) {
	if v.err != nil {
		return false, expr
	}

	// Normalize expressions that know how to normalize themselves.
	if normalizeable, ok := expr.(normalizableExpr); ok {
		expr = normalizeable.normalize(v)
		if v.err != nil {
			return false, expr
		}
	}

	switch expr.(type) {
	case *CaseExpr, *IfExpr, *NullIfExpr, *CoalesceExpr:
		// Conditional expressions need to be evaluated during the downward
		// traversal in order to avoid evaluating sub-expressions which should
		// not be evaluated due to the case/conditional.
		if v.isConst(expr) {
			expr, v.err = expr.(TypedExpr).Eval(v.ctx)
			if v.err != nil {
				return false, expr
			}
		}
	case *Subquery:
		// Avoid normalizing subqueries. We need the subquery to be expanded in
		// order to do so properly.
		// TODO(knz) This should happen when the prepare and execute phases are
		//     separated for SelectClause.
		return false, expr
	}

	return true, expr
}

func (v *normalizeVisitor) VisitPost(expr Expr) Expr {
	if v.err != nil {
		return expr
	}

	// Normalize expressions that know how to normalize themselves.
	if normalizeable, ok := expr.(normalizableExpr); ok {
		expr = normalizeable.normalize(v)
		if v.err != nil {
			return expr
		}
	}

	// Evaluate all constant expressions.
	if v.isConst(expr) {
		expr, v.err = expr.(TypedExpr).Eval(v.ctx)
	}
	return expr
}

func (v *normalizeVisitor) isConst(expr Expr) bool {
	return v.isConstVisitor.run(expr)
}

func invertComparisonOp(op ComparisonOperator) ComparisonOperator {
	switch op {
	case EQ:
		return EQ
	case GE:
		return LE
	case GT:
		return LT
	case LE:
		return GE
	case LT:
		return GT
	default:
		panic(fmt.Sprintf("unable to invert: %s", op))
	}
}

type isConstVisitor struct {
	isConst bool
}

var _ Visitor = &isConstVisitor{}

func (v *isConstVisitor) VisitPre(expr Expr) (recurse bool, newExpr Expr) {
	if v.isConst {
		if isVar(expr) {
			v.isConst = false
			return false, expr
		}

		switch t := expr.(type) {
		case *Subquery:
			v.isConst = false
			return false, expr
		case *FuncExpr:
			if t.fn.fn == nil || t.fn.impure {
				v.isConst = false
				return false, expr
			}
		}
	}
	return true, expr
}

func (*isConstVisitor) VisitPost(expr Expr) Expr { return expr }

func (v *isConstVisitor) run(expr Expr) bool {
	v.isConst = true
	WalkExprConst(v, expr)
	return v.isConst
}

func isVar(expr Expr) bool {
	_, ok := expr.(VariableExpr)
	return ok
}

type containsVarsVisitor struct {
	containsVars bool
}

var _ Visitor = &containsVarsVisitor{}

func (v *containsVarsVisitor) VisitPre(expr Expr) (recurse bool, newExpr Expr) {
	if !v.containsVars && isVar(expr) {
		v.containsVars = true
	}
	if v.containsVars {
		return false, expr
	}
	return true, expr
}

func (*containsVarsVisitor) VisitPost(expr Expr) Expr { return expr }

// ContainsVars returns true if the expression contains any variables.
func ContainsVars(expr Expr) bool {
	v := containsVarsVisitor{containsVars: false}
	WalkExprConst(&v, expr)
	return v.containsVars
}
<|endoftext|>


<commit_before>package decor

import (
	"fmt"
)

const (
	_ = iota
	UnitKiB
	UnitKB
)

// CountersNoUnit is a wrapper around Counters with no unit param.
func CountersNoUnit(pairFmt string, wcc ...WC) Decorator {
	return Counters(0, pairFmt, wcc...)
}

// CountersKibiByte is a wrapper around Counters with predefined unit
// UnitKiB (bytes/1024).
func CountersKibiByte(pairFmt string, wcc ...WC) Decorator {
	return Counters(UnitKiB, pairFmt, wcc...)
}

// CountersKiloByte is a wrapper around Counters with predefined unit
// UnitKB (bytes/1000).
func CountersKiloByte(pairFmt string, wcc ...WC) Decorator {
	return Counters(UnitKB, pairFmt, wcc...)
}

// Counters decorator with dynamic unit measure adjustment.
//
//	`unit` one of [0|UnitKiB|UnitKB] zero for no unit
//
//	`pairFmt` printf compatible verbs for current and total, like "%f" or "%d"
//
//	`wcc` optional WC config
//
// pairFmt example if unit=UnitKB:
//
//	pairFmt="%.1f / %.1f"   output: "1.0MB / 12.0MB"
//	pairFmt="% .1f / % .1f" output: "1.0 MB / 12.0 MB"
//	pairFmt="%d / %d"       output: "1MB / 12MB"
//	pairFmt="% d / % d"     output: "1 MB / 12 MB"
//
func Counters(unit int, pairFmt string, wcc ...WC) Decorator {
	var wc WC
	for _, widthConf := range wcc {
		wc = widthConf
	}
	if pairFmt == "" {
		pairFmt = "%d / %d"
	}
	d := &countersDecorator{
		WC:      wc.Init(),
		unit:    unit,
		pairFmt: pairFmt,
	}
	return d
}

type countersDecorator struct {
	WC
	unit    int
	pairFmt string
}

func (d *countersDecorator) Decor(st *Statistics) string {
	var res string
	switch d.unit {
	case UnitKiB:
		res = fmt.Sprintf(d.pairFmt, SizeB1024(st.Current), SizeB1024(st.Total))
	case UnitKB:
		res = fmt.Sprintf(d.pairFmt, SizeB1000(st.Current), SizeB1000(st.Total))
	default:
		res = fmt.Sprintf(d.pairFmt, st.Current, st.Total)
	}

	return d.FormatMsg(res)
}
<commit_msg>counters producer<commit_after>package decor

import (
	"fmt"
)

const (
	_ = iota
	UnitKiB
	UnitKB
)

// CountersNoUnit is a wrapper around Counters with no unit param.
func CountersNoUnit(pairFmt string, wcc ...WC) Decorator {
	return Counters(0, pairFmt, wcc...)
}

// CountersKibiByte is a wrapper around Counters with predefined unit
// UnitKiB (bytes/1024).
func CountersKibiByte(pairFmt string, wcc ...WC) Decorator {
	return Counters(UnitKiB, pairFmt, wcc...)
}

// CountersKiloByte is a wrapper around Counters with predefined unit
// UnitKB (bytes/1000).
func CountersKiloByte(pairFmt string, wcc ...WC) Decorator {
	return Counters(UnitKB, pairFmt, wcc...)
}

// Counters decorator with dynamic unit measure adjustment.
//
//	`unit` one of [0|UnitKiB|UnitKB] zero for no unit
//
//	`pairFmt` printf compatible verbs for current and total, like "%f" or "%d"
//
//	`wcc` optional WC config
//
// pairFmt example if unit=UnitKB:
//
//	pairFmt="%.1f / %.1f"   output: "1.0MB / 12.0MB"
//	pairFmt="% .1f / % .1f" output: "1.0 MB / 12.0 MB"
//	pairFmt="%d / %d"       output: "1MB / 12MB"
//	pairFmt="% d / % d"     output: "1 MB / 12 MB"
//
func Counters(unit int, pairFmt string, wcc ...WC) Decorator {
	var wc WC
	for _, widthConf := range wcc {
		wc = widthConf
	}
	d := &countersDecorator{
		WC:       wc.Init(),
		producer: chooseSizeProducer(unit, pairFmt),
	}
	return d
}

type countersDecorator struct {
	WC
	producer func(*Statistics) string
}

func (d *countersDecorator) Decor(st *Statistics) string {
	return d.producer(st)
}

func chooseSizeProducer(unit int, format string) func(*Statistics) string {
	if format == "" {
		format = "%d / %d"
	}
	switch unit {
	case UnitKiB:
		return func(st *Statistics) string {
			return fmt.Sprintf(format, SizeB1024(st.Current), SizeB1024(st.Total))
		}
	case UnitKB:
		return func(st *Statistics) string {
			return fmt.Sprintf(format, SizeB1000(st.Current), SizeB1000(st.Total))
		}
	default:
		return func(st *Statistics) string {
			return fmt.Sprintf(format, st.Current, st.Total)
		}
	}
}
<|endoftext|>


<commit_before>package version

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
)

// Release is the minimal set of release data carina needs from the GitHub API
type Release struct {
	TagName string `json:"tag_name"`
}

func githubGet(uri string, rel *Release) error {
	resp, err := http.Get("https://api.github.com" + uri)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return fmt.Errorf("could not fetch releases, %v", err)
	}

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("github responded with a non 200 OK: %v", resp.Status)
	}

	var r io.Reader = resp.Body

	if err = json.NewDecoder(r).Decode(rel); err != nil {
		return fmt.Errorf("could not unmarshal JSON into Release struct, %v", err)
	}

	return nil
}

const latestReleaseURI = "/repos/%s/%s/releases/latest%s"

func latestReleaseAPI(user, repo, token string) (*Release, error) {
	if token != "" {
		token = "?access_token=" + token
	}
	var release Release
	return &release, githubGet(fmt.Sprintf(latestReleaseURI, user, repo, token), &release)
}

const ownerName = "getcarina"
const repoName = "carina"

// LatestRelease returns the most recent release of carina
func LatestRelease() (*Release, error) {
	return latestRelease(ownerName, repoName)
}

// Get the latest release currently
func latestRelease(owner, repo string) (*Release, error) {
	rel, err := latestReleaseAPI(owner, repo, "")

	if err != nil {
		return nil, err
	}

	return rel, nil
}
<commit_msg>Use GITHUB_TOKEN if set.<commit_after>package version

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
)

// Release is the minimal set of release data carina needs from the GitHub API
type Release struct {
	TagName string `json:"tag_name"`
}

func githubGet(uri string, rel *Release) error {
	resp, err := http.Get("https://api.github.com" + uri)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return fmt.Errorf("could not fetch releases, %v", err)
	}

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("github responded with a non 200 OK: %v", resp.Status)
	}

	var r io.Reader = resp.Body

	if err = json.NewDecoder(r).Decode(rel); err != nil {
		return fmt.Errorf("could not unmarshal JSON into Release struct, %v", err)
	}

	return nil
}

const latestReleaseURI = "/repos/%s/%s/releases/latest%s"

func latestReleaseAPI(user, repo, token string) (*Release, error) {
	if token != "" {
		token = "?access_token=" + token
	}
	var release Release
	return &release, githubGet(fmt.Sprintf(latestReleaseURI, user, repo, token), &release)
}

const ownerName = "getcarina"
const repoName = "carina"

// LatestRelease returns the most recent release of carina
func LatestRelease() (*Release, error) {
	return latestRelease(ownerName, repoName)
}

// Get the latest release currently
func latestRelease(owner, repo string) (*Release, error) {
	rel, err := latestReleaseAPI(owner, repo, os.Getenv("GITHUB_TOKEN"))

	if err != nil {
		return nil, err
	}

	return rel, nil
}
<|endoftext|>


<commit_before>package main

import (
	"time"
	"flag"

	"github.com/mitchellh/cli"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
//"os"
	"strconv"
)

type DeleteCommand struct {
	Ui 			cli.Ui
	InstanceId 	string
	OlderThan 	string
	DryRun		bool
}

// descriptions for args
var deleteDscrInstanceId = "The EC2 instance from which the AMIs to be deleted were originally created"
var deleteOlderThan = "Delete AMIs older than the specified time; accepts formats like '30d' or '4h'"
var deleteDscrDryRun = "Execute a simulated run. Lists AMIs to be deleted, but does not actually delete them."

func (c *DeleteCommand) Help() string {
	return `ec2-snapper create <args> [--help]

Create an AMI of the given EC2 instance.

Available args are:
--instance      ` + deleteDscrInstanceId + `
--older-than    ` + deleteOlderThan + `
--dry-run       ` + deleteDscrDryRun
}

func (c *DeleteCommand) Synopsis() string {
	return "Delete the specified AMIs"
}

func (c *DeleteCommand) Run(args []string) int {

	// Handle the command-line args
	cmdFlags := flag.NewFlagSet("delete", flag.ExitOnError)
	cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }

	cmdFlags.StringVar(&c.InstanceId, "instance", "", deleteDscrInstanceId)
	cmdFlags.StringVar(&c.OlderThan, "older-than", "", deleteOlderThan)
	cmdFlags.BoolVar(&c.DryRun, "dry-run", false, deleteDscrDryRun)

	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	// Check for required command-line args
	if c.InstanceId == "" {
		c.Ui.Error("ERROR: The argument '--instance' is required.")
		return 1
	}

	if c.OlderThan == "" {
		c.Ui.Error("ERROR: The argument '--older-than' is required.")
		return 1
	}

	// Create an EC2 service object; AWS region is picked up from the "AWS_REGION" env var.
	svc := ec2.New(nil)

	// Get a list of the existing AMIs that meet our criteria
	resp, err := svc.DescribeImages(&ec2.DescribeImagesInput{
		Filters: []*ec2.Filter{
			&ec2.Filter{
				Name: aws.String("tag:ec2-snapper-instance-id"),
				Values: []*string{&c.InstanceId},
			},
		},
	})
	if err != nil {
		panic(err)
	}

	if len(resp.Images) == 0 {
		c.Ui.Error("No results!")
	} else {
		c.Ui.Info("Found " + strconv.Itoa(len(resp.Images)) + " results.")
		c.Ui.Info(*resp.Images[0].ImageID)
	}

	// Get a list of all snapshots

	// For each AMI
	// - De-register the AMI
	// - Delete the corresponding AMI snapshot

	// Generate a nicely formatted timestamp for right now
//	const dateLayoutForAmiName = "2006-01-02 at 15_04_05 (MST)"
	time.Now()
	//t := time.Now()
//
//	// Create the AMI Snapshot
//	name := c.Name + " " + t.Format(dateLayoutForAmiName)
//	instanceId := c.InstanceId
//	dryRun := c.DryRun
//	noReboot := c.NoReboot
//
//	c.Ui.Output("==> Creating AMI for " + instanceId + "...")
//
//	resp, err := svc.CreateImage(&ec2.CreateImageInput{
//		Name: &name,
//		InstanceID: &instanceId,
//		DryRun: &dryRun,
//		NoReboot: &noReboot })
//	if err != nil {
//		panic(err)
//	}
//
//	// Assign tags to this AMI.  We'll use these when it comes time to delete the AMI
//	c.Ui.Output("==> Adding tags to AMI " + *resp.ImageID + "...")
//
//	//const dateLayoutForTags = "2006-01-02 at 15:04:05 (UTC)"
//	tagName1 := "ec2-snapper-instance-id"
//	tagName2 := "ec2-snapper-snapshot-date"
//	tagValue2 := time.Now().Format(time.RFC3339)
//
//	svc.CreateTags(&ec2.CreateTagsInput{
//		Resources: []*string{resp.ImageID},
//		Tags: []*ec2.Tag{
//			&ec2.Tag{ Key: &tagName1, Value: &c.InstanceId },
//			&ec2.Tag{ Key: &tagName2, Value: &tagValue2 },
//		},
//	})

	c.Ui.Info("==> Success! Deleted a bunch of things.")
	return 0
}

<commit_msg>Delete subcommand now able to delete all AMIs<commit_after>package main

import (
	"time"
	"flag"

	"github.com/mitchellh/cli"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"strconv"
	"strings"
	"github.com/aws/aws-sdk-go/service/iam"
)

type DeleteCommand struct {
	Ui 			cli.Ui
	InstanceId 	string
	OlderThan 	string
	DryRun		bool
}

// descriptions for args
var deleteDscrInstanceId = "The EC2 instance from which the AMIs to be deleted were originally created"
var deleteOlderThan = "Delete AMIs older than the specified time; accepts formats like '30d' or '4h'"
var deleteDscrDryRun = "Execute a simulated run. Lists AMIs to be deleted, but does not actually delete them."

func (c *DeleteCommand) Help() string {
	return `ec2-snapper create <args> [--help]

Create an AMI of the given EC2 instance.

Available args are:
--instance      ` + deleteDscrInstanceId + `
--older-than    ` + deleteOlderThan + `
--dry-run       ` + deleteDscrDryRun
}

func (c *DeleteCommand) Synopsis() string {
	return "Delete the specified AMIs"
}

func (c *DeleteCommand) Run(args []string) int {

	// Handle the command-line args
	cmdFlags := flag.NewFlagSet("delete", flag.ExitOnError)
	cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }

	cmdFlags.StringVar(&c.InstanceId, "instance", "", deleteDscrInstanceId)
	cmdFlags.StringVar(&c.OlderThan, "older-than", "", deleteOlderThan)
	cmdFlags.BoolVar(&c.DryRun, "dry-run", false, deleteDscrDryRun)

	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	// Check for required command-line args
	if c.InstanceId == "" {
		c.Ui.Error("ERROR: The argument '--instance' is required.")
		return 1
	}

	if c.OlderThan == "" {
		c.Ui.Error("ERROR: The argument '--older-than' is required.")
		return 1
	}

	// Warn the user that this is a dry run
	if c.DryRun {
		c.Ui.Warn("WARNING: This is a dry run, and no actions will be taken, despite what any output may say!")
	}

	// Create an EC2 service object; AWS region is picked up from the "AWS_REGION" env var.
	svc := ec2.New(nil)

	// Get a list of the existing AMIs that meet our criteria
	resp, err := svc.DescribeImages(&ec2.DescribeImagesInput{
		Filters: []*ec2.Filter{
			&ec2.Filter{
				Name: aws.String("tag:ec2-snapper-instance-id"),
				Values: []*string{&c.InstanceId},
			},
		},
	})
	if err != nil {
		panic(err)
	}

	// Get the AWS Account ID of the current AWS account
	// We need this to do a more efficient lookup on the snapshot volumes
	// - Per http://docs.aws.amazon.com/general/latest/gr/acct-identifiers.html, we assume the Account Id is always 12 digits
	// - Per http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arn-syntax-iam, we assume the current user's ARN
	//   is always of the form arn:aws:iam::account-id:user/user-name
	svcIam := iam.New(nil)

	respIam, err := svcIam.GetUser(&iam.GetUserInput{})
	awsAccountId := strings.Split(*respIam.User.ARN, ":")[4]
	c.Ui.Output("==> Identified current AWS Account Id as " + awsAccountId)

	// Get a list of every single snapshot in our account
	// (I wasn't able to find a better way to filter these, but suggestions welcome!)
	respDscrSnapshots, err := svc.DescribeSnapshots(&ec2.DescribeSnapshotsInput{
		OwnerIDs: []*string{&awsAccountId},
	})
	if err != nil {
		panic(err)
	}
	c.Ui.Output("==> Found " + strconv.Itoa(len(respDscrSnapshots.Snapshots)) + " snapshots in our account to search through.")

	// Begin deleting AMIs...
	if len(resp.Images) == 0 {
		c.Ui.Error("No AMIs were found for EC2 instance \"" + c.InstanceId + "\"")
	} else {
		for i := 0; i < len(resp.Images); i++ {
			// Step 1: De-register the AMI
			c.Ui.Output(*resp.Images[i].ImageID + ": De-registering...")
			_, err := svc.DeregisterImage(&ec2.DeregisterImageInput{
				DryRun: &c.DryRun,
				ImageID: resp.Images[i].ImageID,
			})
			if err != nil {
				if ! strings.Contains(err.Error(), "DryRunOperation") {
					panic(err)
				}
			}

			// Step 2: Delete the corresponding AMI snapshot
			// Look at the "description" for each Snapshot to see if it contains our AMI id
			snapshotId := ""
			for j := 0; j < len(respDscrSnapshots.Snapshots); j++ {
				if strings.Contains(*respDscrSnapshots.Snapshots[j].Description, *resp.Images[i].ImageID) {
					snapshotId = *respDscrSnapshots.Snapshots[j].SnapshotID
					break
				}
			}

			c.Ui.Output(*resp.Images[i].ImageID + ": Deleting snapshot " + snapshotId + "...")
			svc.DeleteSnapshot(&ec2.DeleteSnapshotInput{
				DryRun: &c.DryRun,
				SnapshotID: &snapshotId,
			})

			c.Ui.Output(*resp.Images[i].ImageID + ": Done!")
			c.Ui.Output("")
		}
	}

	// Generate a nicely formatted timestamp for right now
	//	const dateLayoutForAmiName = "2006-01-02 at 15_04_05 (MST)"
	time.Now()
	//t := time.Now()

	if c.DryRun {
		c.Ui.Info("==> DRY RUN. Had this not been a dry run, " + strconv.Itoa(len(resp.Images)) + " AMI's and their corresponding snapshots would have been deleted.")
	} else {
		c.Ui.Info("==> Success! Deleted " + strconv.Itoa(len(resp.Images)) + " AMI's and their corresponding snapshots.")
	}
	return 0
}

<|endoftext|>


<commit_before>package backend

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/juju/arrar"
)

const (
	dirMode         = 0700
	blobPath        = "blobs"
	snapshotPath    = "snapshots"
	treePath        = "trees"
	lockPath        = "locks"
	keyPath         = "keys"
	tempPath        = "tmp"
	versionFileName = "version"
)

type Local struct {
	p   string
	ver uint
}

// OpenLocal opens the local backend at dir.
func OpenLocal(dir string) (*Local, error) {
	items := []string{
		dir,
		filepath.Join(dir, blobPath),
		filepath.Join(dir, snapshotPath),
		filepath.Join(dir, treePath),
		filepath.Join(dir, lockPath),
		filepath.Join(dir, keyPath),
		filepath.Join(dir, tempPath),
	}

	// test if all necessary dirs and files are there
	for _, d := range items {
		if _, err := os.Stat(d); err != nil {
			return nil, fmt.Errorf("%s does not exist", d)
		}
	}

	// read version file
	f, err := os.Open(filepath.Join(dir, versionFileName))
	if err != nil {
		return nil, fmt.Errorf("unable to read version file: %v\n", err)
	}

	buf := make([]byte, 100)
	n, err := f.Read(buf)
	if err != nil {
		return nil, err
	}

	err = f.Close()
	if err != nil {
		return nil, err
	}

	version, err := strconv.Atoi(strings.TrimSpace(string(buf[:n])))
	if err != nil {
		return nil, fmt.Errorf("unable to convert version to integer: %v\n", err)
	}

	// check version
	if version != BackendVersion {
		return nil, fmt.Errorf("wrong version %d", version)
	}

	return &Local{p: dir, ver: uint(version)}, nil
}

// CreateLocal creates all the necessary files and directories for a new local
// backend at dir.
func CreateLocal(dir string) (*Local, error) {
	versionFile := filepath.Join(dir, versionFileName)
	dirs := []string{
		dir,
		filepath.Join(dir, blobPath),
		filepath.Join(dir, snapshotPath),
		filepath.Join(dir, treePath),
		filepath.Join(dir, lockPath),
		filepath.Join(dir, keyPath),
		filepath.Join(dir, tempPath),
	}

	// test if version file already exists
	_, err := os.Lstat(versionFile)
	if err == nil {
		return nil, errors.New("version file already exists")
	}

	// test if directories already exist
	for _, d := range dirs[1:] {
		if _, err := os.Stat(d); err == nil {
			return nil, fmt.Errorf("dir %s already exists", d)
		}
	}

	// create paths for blobs, refs and temp
	for _, d := range dirs {
		err := os.MkdirAll(d, dirMode)
		if err != nil {
			return nil, err
		}
	}

	// create version file
	f, err := os.Create(versionFile)
	if err != nil {
		return nil, err
	}

	_, err = f.Write([]byte(strconv.Itoa(BackendVersion)))
	if err != nil {
		return nil, err
	}

	err = f.Close()
	if err != nil {
		return nil, err
	}

	// open backend
	return OpenLocal(dir)
}

// Location returns this backend's location (the directory name).
func (b *Local) Location() string {
	return b.p
}

// Return temp directory in correct directory for this backend.
func (b *Local) tempFile() (*os.File, error) {
	return ioutil.TempFile(filepath.Join(b.p, tempPath), "temp-")
}

// Rename temp file to final name according to type and ID.
func (b *Local) renameFile(file *os.File, t Type, id ID) error {
	filename := filepath.Join(b.dir(t), id.String())
	return os.Rename(file.Name(), filename)
}

// Construct directory for given Type.
func (b *Local) dir(t Type) string {
	var n string
	switch t {
	case Blob:
		n = blobPath
	case Snapshot:
		n = snapshotPath
	case Tree:
		n = treePath
	case Lock:
		n = lockPath
	case Key:
		n = keyPath
	}
	return filepath.Join(b.p, n)
}

// Create stores new content of type t and data and returns the ID. If the blob
// is already present, returns ErrAlreadyPresent and the blob's ID.
func (b *Local) Create(t Type, data []byte) (ID, error) {
	// TODO: make sure that tempfile is removed upon error

	// check if blob is already present in backend
	id := IDFromData(data)
	res, err := b.Test(t, id)
	if err != nil {
		return nil, arrar.Annotate(err, "test for presence")
	}

	if res {
		return id, ErrAlreadyPresent
	}

	// create tempfile in backend
	file, err := b.tempFile()
	if err != nil {
		return nil, err
	}

	// write data to tempfile
	_, err = file.Write(data)
	if err != nil {
		return nil, err
	}

	err = file.Close()
	if err != nil {
		return nil, err
	}

	// return id
	err = b.renameFile(file, t, id)
	if err != nil {
		return nil, err
	}

	return id, nil
}

// Construct path for given Type and ID.
func (b *Local) filename(t Type, id ID) string {
	return filepath.Join(b.dir(t), id.String())
}

// Get returns the content stored under the given ID.
func (b *Local) Get(t Type, id ID) ([]byte, error) {
	// try to open file
	file, err := os.Open(b.filename(t, id))
	defer file.Close()
	if err != nil {
		return nil, err
	}

	// read all
	buf, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}

	return buf, nil
}

// Test returns true if a blob of the given type and ID exists in the backend.
func (b *Local) Test(t Type, id ID) (bool, error) {
	// try to open file
	file, err := os.Open(b.filename(t, id))
	defer func() {
		file.Close()
	}()

	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}

	return true, nil
}

// Remove removes the content stored at ID.
func (b *Local) Remove(t Type, id ID) error {
	return os.Remove(b.filename(t, id))
}

// List lists all objects of a given type.
func (b *Local) List(t Type) (IDs, error) {
	// TODO: use os.Open() and d.Readdirnames() instead of Glob()
	pattern := filepath.Join(b.dir(t), "*")

	matches, err := filepath.Glob(pattern)
	if err != nil {
		return nil, err
	}

	ids := make(IDs, 0, len(matches))

	for _, m := range matches {
		base := filepath.Base(m)

		if base == "" {
			continue
		}
		id, err := ParseID(base)

		if err != nil {
			continue
		}

		ids = append(ids, id)
	}

	return ids, nil
}

// Version returns the version of this local backend.
func (b *Local) Version() uint {
	return b.ver
}

// Close closes the backend
func (b *Local) Close() error {
	return nil
}
<commit_msg>Add newlink to version file<commit_after>package backend

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/juju/arrar"
)

const (
	dirMode         = 0700
	blobPath        = "blobs"
	snapshotPath    = "snapshots"
	treePath        = "trees"
	lockPath        = "locks"
	keyPath         = "keys"
	tempPath        = "tmp"
	versionFileName = "version"
)

type Local struct {
	p   string
	ver uint
}

// OpenLocal opens the local backend at dir.
func OpenLocal(dir string) (*Local, error) {
	items := []string{
		dir,
		filepath.Join(dir, blobPath),
		filepath.Join(dir, snapshotPath),
		filepath.Join(dir, treePath),
		filepath.Join(dir, lockPath),
		filepath.Join(dir, keyPath),
		filepath.Join(dir, tempPath),
	}

	// test if all necessary dirs and files are there
	for _, d := range items {
		if _, err := os.Stat(d); err != nil {
			return nil, fmt.Errorf("%s does not exist", d)
		}
	}

	// read version file
	f, err := os.Open(filepath.Join(dir, versionFileName))
	if err != nil {
		return nil, fmt.Errorf("unable to read version file: %v\n", err)
	}

	buf := make([]byte, 100)
	n, err := f.Read(buf)
	if err != nil {
		return nil, err
	}

	err = f.Close()
	if err != nil {
		return nil, err
	}

	version, err := strconv.Atoi(strings.TrimSpace(string(buf[:n])))
	if err != nil {
		return nil, fmt.Errorf("unable to convert version to integer: %v\n", err)
	}

	// check version
	if version != BackendVersion {
		return nil, fmt.Errorf("wrong version %d", version)
	}

	return &Local{p: dir, ver: uint(version)}, nil
}

// CreateLocal creates all the necessary files and directories for a new local
// backend at dir.
func CreateLocal(dir string) (*Local, error) {
	versionFile := filepath.Join(dir, versionFileName)
	dirs := []string{
		dir,
		filepath.Join(dir, blobPath),
		filepath.Join(dir, snapshotPath),
		filepath.Join(dir, treePath),
		filepath.Join(dir, lockPath),
		filepath.Join(dir, keyPath),
		filepath.Join(dir, tempPath),
	}

	// test if version file already exists
	_, err := os.Lstat(versionFile)
	if err == nil {
		return nil, errors.New("version file already exists")
	}

	// test if directories already exist
	for _, d := range dirs[1:] {
		if _, err := os.Stat(d); err == nil {
			return nil, fmt.Errorf("dir %s already exists", d)
		}
	}

	// create paths for blobs, refs and temp
	for _, d := range dirs {
		err := os.MkdirAll(d, dirMode)
		if err != nil {
			return nil, err
		}
	}

	// create version file
	f, err := os.Create(versionFile)
	if err != nil {
		return nil, err
	}

	_, err = f.Write([]byte(fmt.Sprintf("%d\n", BackendVersion)))
	if err != nil {
		return nil, err
	}

	err = f.Close()
	if err != nil {
		return nil, err
	}

	// open backend
	return OpenLocal(dir)
}

// Location returns this backend's location (the directory name).
func (b *Local) Location() string {
	return b.p
}

// Return temp directory in correct directory for this backend.
func (b *Local) tempFile() (*os.File, error) {
	return ioutil.TempFile(filepath.Join(b.p, tempPath), "temp-")
}

// Rename temp file to final name according to type and ID.
func (b *Local) renameFile(file *os.File, t Type, id ID) error {
	filename := filepath.Join(b.dir(t), id.String())
	return os.Rename(file.Name(), filename)
}

// Construct directory for given Type.
func (b *Local) dir(t Type) string {
	var n string
	switch t {
	case Blob:
		n = blobPath
	case Snapshot:
		n = snapshotPath
	case Tree:
		n = treePath
	case Lock:
		n = lockPath
	case Key:
		n = keyPath
	}
	return filepath.Join(b.p, n)
}

// Create stores new content of type t and data and returns the ID. If the blob
// is already present, returns ErrAlreadyPresent and the blob's ID.
func (b *Local) Create(t Type, data []byte) (ID, error) {
	// TODO: make sure that tempfile is removed upon error

	// check if blob is already present in backend
	id := IDFromData(data)
	res, err := b.Test(t, id)
	if err != nil {
		return nil, arrar.Annotate(err, "test for presence")
	}

	if res {
		return id, ErrAlreadyPresent
	}

	// create tempfile in backend
	file, err := b.tempFile()
	if err != nil {
		return nil, err
	}

	// write data to tempfile
	_, err = file.Write(data)
	if err != nil {
		return nil, err
	}

	err = file.Close()
	if err != nil {
		return nil, err
	}

	// return id
	err = b.renameFile(file, t, id)
	if err != nil {
		return nil, err
	}

	return id, nil
}

// Construct path for given Type and ID.
func (b *Local) filename(t Type, id ID) string {
	return filepath.Join(b.dir(t), id.String())
}

// Get returns the content stored under the given ID.
func (b *Local) Get(t Type, id ID) ([]byte, error) {
	// try to open file
	file, err := os.Open(b.filename(t, id))
	defer file.Close()
	if err != nil {
		return nil, err
	}

	// read all
	buf, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}

	return buf, nil
}

// Test returns true if a blob of the given type and ID exists in the backend.
func (b *Local) Test(t Type, id ID) (bool, error) {
	// try to open file
	file, err := os.Open(b.filename(t, id))
	defer func() {
		file.Close()
	}()

	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}

	return true, nil
}

// Remove removes the content stored at ID.
func (b *Local) Remove(t Type, id ID) error {
	return os.Remove(b.filename(t, id))
}

// List lists all objects of a given type.
func (b *Local) List(t Type) (IDs, error) {
	// TODO: use os.Open() and d.Readdirnames() instead of Glob()
	pattern := filepath.Join(b.dir(t), "*")

	matches, err := filepath.Glob(pattern)
	if err != nil {
		return nil, err
	}

	ids := make(IDs, 0, len(matches))

	for _, m := range matches {
		base := filepath.Base(m)

		if base == "" {
			continue
		}
		id, err := ParseID(base)

		if err != nil {
			continue
		}

		ids = append(ids, id)
	}

	return ids, nil
}

// Version returns the version of this local backend.
func (b *Local) Version() uint {
	return b.ver
}

// Close closes the backend
func (b *Local) Close() error {
	return nil
}
<|endoftext|>


<commit_before>package identify

import (
	"strings"
	"sync"

	semver "github.com/coreos/go-semver/semver"
	ggio "github.com/gogo/protobuf/io"
	pstore "github.com/ipfs/go-libp2p-peerstore"
	ma "github.com/jbenet/go-multiaddr"
	host "github.com/libp2p/go-libp2p/p2p/host"
	mstream "github.com/libp2p/go-libp2p/p2p/metrics/stream"
	inet "github.com/libp2p/go-libp2p/p2p/net"
	pb "github.com/libp2p/go-libp2p/p2p/protocol/identify/pb"
	msmux "github.com/whyrusleeping/go-multistream"
	context "golang.org/x/net/context"

	lgbl "github.com/ipfs/go-libp2p-loggables"
	logging "github.com/ipfs/go-log"
)

var log = logging.Logger("net/identify")

// ID is the protocol.ID of the Identify Service.
const ID = "/ipfs/id/1.0.0"

// LibP2PVersion holds the current protocol version for a client running this code
// TODO(jbenet): fix the versioning mess.
const LibP2PVersion = "ipfs/0.1.0"
const ClientVersion = "go-libp2p/3.3.4"

// IDService is a structure that implements ProtocolIdentify.
// It is a trivial service that gives the other peer some
// useful information about the local peer. A sort of hello.
//
// The IDService sends:
//  * Our IPFS Protocol Version
//  * Our IPFS Agent Version
//  * Our public Listen Addresses
type IDService struct {
	Host host.Host

	// connections undergoing identification
	// for wait purposes
	currid map[inet.Conn]chan struct{}
	currmu sync.RWMutex

	// our own observed addresses.
	// TODO: instead of expiring, remove these when we disconnect
	observedAddrs ObservedAddrSet
}

func NewIDService(h host.Host) *IDService {
	s := &IDService{
		Host:   h,
		currid: make(map[inet.Conn]chan struct{}),
	}
	h.SetStreamHandler(ID, s.RequestHandler)
	return s
}

// OwnObservedAddrs returns the addresses peers have reported we've dialed from
func (ids *IDService) OwnObservedAddrs() []ma.Multiaddr {
	return ids.observedAddrs.Addrs()
}

func (ids *IDService) IdentifyConn(c inet.Conn) {
	ids.currmu.Lock()
	if wait, found := ids.currid[c]; found {
		ids.currmu.Unlock()
		log.Debugf("IdentifyConn called twice on: %s", c)
		<-wait // already identifying it. wait for it.
		return
	}
	ch := make(chan struct{})
	ids.currid[c] = ch
	ids.currmu.Unlock()

	defer close(ch)

	s, err := c.NewStream()
	if err != nil {
		log.Debugf("error opening initial stream for %s: %s", ID, err)
		log.Event(context.TODO(), "IdentifyOpenFailed", c.RemotePeer())
		c.Close()
		return
	}

	bwc := ids.Host.GetBandwidthReporter()
	s = mstream.WrapStream(s, ID, bwc)

	// ok give the response to our handler.
	if err := msmux.SelectProtoOrFail(ID, s); err != nil {
		log.Debugf("error writing stream header for %s", ID)
		log.Event(context.TODO(), "IdentifyOpenFailed", c.RemotePeer())
		s.Close()
		return
	}

	ids.ResponseHandler(s)

	ids.currmu.Lock()
	_, found := ids.currid[c]
	delete(ids.currid, c)
	ids.currmu.Unlock()

	if !found {
		log.Debugf("IdentifyConn failed to find channel (programmer error) for %s", c)
		return
	}
}

func (ids *IDService) RequestHandler(s inet.Stream) {
	defer s.Close()
	c := s.Conn()

	bwc := ids.Host.GetBandwidthReporter()
	s = mstream.WrapStream(s, ID, bwc)

	w := ggio.NewDelimitedWriter(s)
	mes := pb.Identify{}
	ids.populateMessage(&mes, s.Conn())
	w.WriteMsg(&mes)

	log.Debugf("%s sent message to %s %s", ID,
		c.RemotePeer(), c.RemoteMultiaddr())
}

func (ids *IDService) ResponseHandler(s inet.Stream) {
	defer s.Close()
	c := s.Conn()

	r := ggio.NewDelimitedReader(s, 2048)
	mes := pb.Identify{}
	if err := r.ReadMsg(&mes); err != nil {
		return
	}
	ids.consumeMessage(&mes, c)

	log.Debugf("%s received message from %s %s", ID,
		c.RemotePeer(), c.RemoteMultiaddr())
}

func (ids *IDService) populateMessage(mes *pb.Identify, c inet.Conn) {

	// set protocols this node is currently handling
	protos := ids.Host.Mux().Protocols()
	mes.Protocols = make([]string, len(protos))
	for i, p := range protos {
		mes.Protocols[i] = string(p)
	}

	// observed address so other side is informed of their
	// "public" address, at least in relation to us.
	mes.ObservedAddr = c.RemoteMultiaddr().Bytes()

	// set listen addrs, get our latest addrs from Host.
	laddrs := ids.Host.Addrs()
	mes.ListenAddrs = make([][]byte, len(laddrs))
	for i, addr := range laddrs {
		mes.ListenAddrs[i] = addr.Bytes()
	}
	log.Debugf("%s sent listen addrs to %s: %s", c.LocalPeer(), c.RemotePeer(), laddrs)

	// set protocol versions
	pv := LibP2PVersion
	av := ClientVersion
	mes.ProtocolVersion = &pv
	mes.AgentVersion = &av
}

func (ids *IDService) consumeMessage(mes *pb.Identify, c inet.Conn) {
	p := c.RemotePeer()

	// mes.Protocols

	// mes.ObservedAddr
	ids.consumeObservedAddress(mes.GetObservedAddr(), c)

	// mes.ListenAddrs
	laddrs := mes.GetListenAddrs()
	lmaddrs := make([]ma.Multiaddr, 0, len(laddrs))
	for _, addr := range laddrs {
		maddr, err := ma.NewMultiaddrBytes(addr)
		if err != nil {
			log.Debugf("%s failed to parse multiaddr from %s %s", ID,
				p, c.RemoteMultiaddr())
			continue
		}
		lmaddrs = append(lmaddrs, maddr)
	}

	// if the address reported by the connection roughly matches their annoucned
	// listener addresses, its likely to be an external NAT address
	if HasConsistentTransport(c.RemoteMultiaddr(), lmaddrs) {
		lmaddrs = append(lmaddrs, c.RemoteMultiaddr())
	}

	// update our peerstore with the addresses. here, we SET the addresses, clearing old ones.
	// We are receiving from the peer itself. this is current address ground truth.
	ids.Host.Peerstore().SetAddrs(p, lmaddrs, pstore.ConnectedAddrTTL)
	log.Debugf("%s received listen addrs for %s: %s", c.LocalPeer(), c.RemotePeer(), lmaddrs)

	// get protocol versions
	pv := mes.GetProtocolVersion()
	av := mes.GetAgentVersion()

	// version check. if we shouldn't talk, bail.
	// TODO: at this point, we've already exchanged information.
	// move this into a first handshake before the connection can open streams.
	if !protocolVersionsAreCompatible(pv, LibP2PVersion) {
		logProtocolMismatchDisconnect(c, pv, av)
		c.Close()
		return
	}

	ids.Host.Peerstore().Put(p, "ProtocolVersion", pv)
	ids.Host.Peerstore().Put(p, "AgentVersion", av)
}

// HasConsistentTransport returns true if the address 'a' shares a
// protocol set with any address in the green set. This is used
// to check if a given address might be one of the addresses a peer is
// listening on.
func HasConsistentTransport(a ma.Multiaddr, green []ma.Multiaddr) bool {
	protosMatch := func(a, b []ma.Protocol) bool {
		if len(a) != len(b) {
			return false
		}

		for i, p := range a {
			if b[i].Code != p.Code {
				return false
			}
		}
		return true
	}

	protos := a.Protocols()

	for _, ga := range green {
		if protosMatch(protos, ga.Protocols()) {
			return true
		}
	}

	return false
}

// IdentifyWait returns a channel which will be closed once
// "ProtocolIdentify" (handshake3) finishes on given conn.
// This happens async so the connection can start to be used
// even if handshake3 knowledge is not necesary.
// Users **MUST** call IdentifyWait _after_ IdentifyConn
func (ids *IDService) IdentifyWait(c inet.Conn) <-chan struct{} {
	ids.currmu.Lock()
	ch, found := ids.currid[c]
	ids.currmu.Unlock()
	if found {
		return ch
	}

	// if not found, it means we are already done identifying it, or
	// haven't even started. either way, return a new channel closed.
	ch = make(chan struct{})
	close(ch)
	return ch
}

func (ids *IDService) consumeObservedAddress(observed []byte, c inet.Conn) {
	if observed == nil {
		return
	}

	maddr, err := ma.NewMultiaddrBytes(observed)
	if err != nil {
		log.Debugf("error parsing received observed addr for %s: %s", c, err)
		return
	}

	// we should only use ObservedAddr when our connection's LocalAddr is one
	// of our ListenAddrs. If we Dial out using an ephemeral addr, knowing that
	// address's external mapping is not very useful because the port will not be
	// the same as the listen addr.
	ifaceaddrs, err := ids.Host.Network().InterfaceListenAddresses()
	if err != nil {
		log.Infof("failed to get interface listen addrs", err)
		return
	}

	log.Debugf("identify identifying observed multiaddr: %s %s", c.LocalMultiaddr(), ifaceaddrs)
	if !addrInAddrs(c.LocalMultiaddr(), ifaceaddrs) {
		// not in our list
		return
	}

	// ok! we have the observed version of one of our ListenAddresses!
	log.Debugf("added own observed listen addr: %s --> %s", c.LocalMultiaddr(), maddr)
	ids.observedAddrs.Add(maddr, c.RemoteMultiaddr())
}

func addrInAddrs(a ma.Multiaddr, as []ma.Multiaddr) bool {
	for _, b := range as {
		if a.Equal(b) {
			return true
		}
	}
	return false
}

// protocolVersionsAreCompatible checks that the two implementations
// can talk to each other. It will use semver, but for now while
// we're in tight development, we will return false for minor version
// changes too.
func protocolVersionsAreCompatible(v1, v2 string) bool {
	if strings.HasPrefix(v1, "ipfs/") {
		v1 = v1[5:]
	}
	if strings.HasPrefix(v2, "ipfs/") {
		v2 = v2[5:]
	}

	v1s, err := semver.NewVersion(v1)
	if err != nil {
		return false
	}

	v2s, err := semver.NewVersion(v2)
	if err != nil {
		return false
	}

	return v1s.Major == v2s.Major && v1s.Minor == v2s.Minor
}

// netNotifiee defines methods to be used with the IpfsDHT
type netNotifiee IDService

func (nn *netNotifiee) IDService() *IDService {
	return (*IDService)(nn)
}

func (nn *netNotifiee) Connected(n inet.Network, v inet.Conn) {
	// TODO: deprecate the setConnHandler hook, and kick off
	// identification here.
}

func (nn *netNotifiee) Disconnected(n inet.Network, v inet.Conn) {
	// undo the setting of addresses to peer.ConnectedAddrTTL we did
	ids := nn.IDService()
	ps := ids.Host.Peerstore()
	addrs := ps.Addrs(v.RemotePeer())
	ps.SetAddrs(v.RemotePeer(), addrs, pstore.RecentlyConnectedAddrTTL)
}

func (nn *netNotifiee) OpenedStream(n inet.Network, v inet.Stream) {}
func (nn *netNotifiee) ClosedStream(n inet.Network, v inet.Stream) {}
func (nn *netNotifiee) Listen(n inet.Network, a ma.Multiaddr)      {}
func (nn *netNotifiee) ListenClose(n inet.Network, a ma.Multiaddr) {}

func logProtocolMismatchDisconnect(c inet.Conn, protocol, agent string) {
	lm := make(lgbl.DeferredMap)
	lm["remotePeer"] = func() interface{} { return c.RemotePeer().Pretty() }
	lm["remoteAddr"] = func() interface{} { return c.RemoteMultiaddr().String() }
	lm["protocolVersion"] = protocol
	lm["agentVersion"] = agent
	log.Event(context.TODO(), "IdentifyProtocolMismatch", lm)
	log.Debugf("IdentifyProtocolMismatch %s %s %s (disconnected)", c.RemotePeer(), protocol, agent)
}
<commit_msg>identify: log protocols from remote peers in peerstore<commit_after>package identify

import (
	"strings"
	"sync"

	semver "github.com/coreos/go-semver/semver"
	ggio "github.com/gogo/protobuf/io"
	pstore "github.com/ipfs/go-libp2p-peerstore"
	ma "github.com/jbenet/go-multiaddr"
	host "github.com/libp2p/go-libp2p/p2p/host"
	mstream "github.com/libp2p/go-libp2p/p2p/metrics/stream"
	inet "github.com/libp2p/go-libp2p/p2p/net"
	pb "github.com/libp2p/go-libp2p/p2p/protocol/identify/pb"
	msmux "github.com/whyrusleeping/go-multistream"
	context "golang.org/x/net/context"

	lgbl "github.com/ipfs/go-libp2p-loggables"
	logging "github.com/ipfs/go-log"
)

var log = logging.Logger("net/identify")

// ID is the protocol.ID of the Identify Service.
const ID = "/ipfs/id/1.0.0"

// LibP2PVersion holds the current protocol version for a client running this code
// TODO(jbenet): fix the versioning mess.
const LibP2PVersion = "ipfs/0.1.0"
const ClientVersion = "go-libp2p/3.3.4"

// IDService is a structure that implements ProtocolIdentify.
// It is a trivial service that gives the other peer some
// useful information about the local peer. A sort of hello.
//
// The IDService sends:
//  * Our IPFS Protocol Version
//  * Our IPFS Agent Version
//  * Our public Listen Addresses
type IDService struct {
	Host host.Host

	// connections undergoing identification
	// for wait purposes
	currid map[inet.Conn]chan struct{}
	currmu sync.RWMutex

	// our own observed addresses.
	// TODO: instead of expiring, remove these when we disconnect
	observedAddrs ObservedAddrSet
}

func NewIDService(h host.Host) *IDService {
	s := &IDService{
		Host:   h,
		currid: make(map[inet.Conn]chan struct{}),
	}
	h.SetStreamHandler(ID, s.RequestHandler)
	return s
}

// OwnObservedAddrs returns the addresses peers have reported we've dialed from
func (ids *IDService) OwnObservedAddrs() []ma.Multiaddr {
	return ids.observedAddrs.Addrs()
}

func (ids *IDService) IdentifyConn(c inet.Conn) {
	ids.currmu.Lock()
	if wait, found := ids.currid[c]; found {
		ids.currmu.Unlock()
		log.Debugf("IdentifyConn called twice on: %s", c)
		<-wait // already identifying it. wait for it.
		return
	}
	ch := make(chan struct{})
	ids.currid[c] = ch
	ids.currmu.Unlock()

	defer close(ch)

	s, err := c.NewStream()
	if err != nil {
		log.Debugf("error opening initial stream for %s: %s", ID, err)
		log.Event(context.TODO(), "IdentifyOpenFailed", c.RemotePeer())
		c.Close()
		return
	}

	bwc := ids.Host.GetBandwidthReporter()
	s = mstream.WrapStream(s, ID, bwc)

	// ok give the response to our handler.
	if err := msmux.SelectProtoOrFail(ID, s); err != nil {
		log.Debugf("error writing stream header for %s", ID)
		log.Event(context.TODO(), "IdentifyOpenFailed", c.RemotePeer())
		s.Close()
		return
	}

	ids.ResponseHandler(s)

	ids.currmu.Lock()
	_, found := ids.currid[c]
	delete(ids.currid, c)
	ids.currmu.Unlock()

	if !found {
		log.Debugf("IdentifyConn failed to find channel (programmer error) for %s", c)
		return
	}
}

func (ids *IDService) RequestHandler(s inet.Stream) {
	defer s.Close()
	c := s.Conn()

	bwc := ids.Host.GetBandwidthReporter()
	s = mstream.WrapStream(s, ID, bwc)

	w := ggio.NewDelimitedWriter(s)
	mes := pb.Identify{}
	ids.populateMessage(&mes, s.Conn())
	w.WriteMsg(&mes)

	log.Debugf("%s sent message to %s %s", ID,
		c.RemotePeer(), c.RemoteMultiaddr())
}

func (ids *IDService) ResponseHandler(s inet.Stream) {
	defer s.Close()
	c := s.Conn()

	r := ggio.NewDelimitedReader(s, 2048)
	mes := pb.Identify{}
	if err := r.ReadMsg(&mes); err != nil {
		return
	}
	ids.consumeMessage(&mes, c)

	log.Debugf("%s received message from %s %s", ID,
		c.RemotePeer(), c.RemoteMultiaddr())
}

func (ids *IDService) populateMessage(mes *pb.Identify, c inet.Conn) {

	// set protocols this node is currently handling
	protos := ids.Host.Mux().Protocols()
	mes.Protocols = make([]string, len(protos))
	for i, p := range protos {
		mes.Protocols[i] = string(p)
	}

	// observed address so other side is informed of their
	// "public" address, at least in relation to us.
	mes.ObservedAddr = c.RemoteMultiaddr().Bytes()

	// set listen addrs, get our latest addrs from Host.
	laddrs := ids.Host.Addrs()
	mes.ListenAddrs = make([][]byte, len(laddrs))
	for i, addr := range laddrs {
		mes.ListenAddrs[i] = addr.Bytes()
	}
	log.Debugf("%s sent listen addrs to %s: %s", c.LocalPeer(), c.RemotePeer(), laddrs)

	// set protocol versions
	pv := LibP2PVersion
	av := ClientVersion
	mes.ProtocolVersion = &pv
	mes.AgentVersion = &av
}

func (ids *IDService) consumeMessage(mes *pb.Identify, c inet.Conn) {
	p := c.RemotePeer()

	// mes.Protocols
	ids.Host.Peerstore().SetProtocols(p, mes.Protocols)

	// mes.ObservedAddr
	ids.consumeObservedAddress(mes.GetObservedAddr(), c)

	// mes.ListenAddrs
	laddrs := mes.GetListenAddrs()
	lmaddrs := make([]ma.Multiaddr, 0, len(laddrs))
	for _, addr := range laddrs {
		maddr, err := ma.NewMultiaddrBytes(addr)
		if err != nil {
			log.Debugf("%s failed to parse multiaddr from %s %s", ID,
				p, c.RemoteMultiaddr())
			continue
		}
		lmaddrs = append(lmaddrs, maddr)
	}

	// if the address reported by the connection roughly matches their annoucned
	// listener addresses, its likely to be an external NAT address
	if HasConsistentTransport(c.RemoteMultiaddr(), lmaddrs) {
		lmaddrs = append(lmaddrs, c.RemoteMultiaddr())
	}

	// update our peerstore with the addresses. here, we SET the addresses, clearing old ones.
	// We are receiving from the peer itself. this is current address ground truth.
	ids.Host.Peerstore().SetAddrs(p, lmaddrs, pstore.ConnectedAddrTTL)
	log.Debugf("%s received listen addrs for %s: %s", c.LocalPeer(), c.RemotePeer(), lmaddrs)

	// get protocol versions
	pv := mes.GetProtocolVersion()
	av := mes.GetAgentVersion()

	// version check. if we shouldn't talk, bail.
	// TODO: at this point, we've already exchanged information.
	// move this into a first handshake before the connection can open streams.
	if !protocolVersionsAreCompatible(pv, LibP2PVersion) {
		logProtocolMismatchDisconnect(c, pv, av)
		c.Close()
		return
	}

	ids.Host.Peerstore().Put(p, "ProtocolVersion", pv)
	ids.Host.Peerstore().Put(p, "AgentVersion", av)
}

// HasConsistentTransport returns true if the address 'a' shares a
// protocol set with any address in the green set. This is used
// to check if a given address might be one of the addresses a peer is
// listening on.
func HasConsistentTransport(a ma.Multiaddr, green []ma.Multiaddr) bool {
	protosMatch := func(a, b []ma.Protocol) bool {
		if len(a) != len(b) {
			return false
		}

		for i, p := range a {
			if b[i].Code != p.Code {
				return false
			}
		}
		return true
	}

	protos := a.Protocols()

	for _, ga := range green {
		if protosMatch(protos, ga.Protocols()) {
			return true
		}
	}

	return false
}

// IdentifyWait returns a channel which will be closed once
// "ProtocolIdentify" (handshake3) finishes on given conn.
// This happens async so the connection can start to be used
// even if handshake3 knowledge is not necesary.
// Users **MUST** call IdentifyWait _after_ IdentifyConn
func (ids *IDService) IdentifyWait(c inet.Conn) <-chan struct{} {
	ids.currmu.Lock()
	ch, found := ids.currid[c]
	ids.currmu.Unlock()
	if found {
		return ch
	}

	// if not found, it means we are already done identifying it, or
	// haven't even started. either way, return a new channel closed.
	ch = make(chan struct{})
	close(ch)
	return ch
}

func (ids *IDService) consumeObservedAddress(observed []byte, c inet.Conn) {
	if observed == nil {
		return
	}

	maddr, err := ma.NewMultiaddrBytes(observed)
	if err != nil {
		log.Debugf("error parsing received observed addr for %s: %s", c, err)
		return
	}

	// we should only use ObservedAddr when our connection's LocalAddr is one
	// of our ListenAddrs. If we Dial out using an ephemeral addr, knowing that
	// address's external mapping is not very useful because the port will not be
	// the same as the listen addr.
	ifaceaddrs, err := ids.Host.Network().InterfaceListenAddresses()
	if err != nil {
		log.Infof("failed to get interface listen addrs", err)
		return
	}

	log.Debugf("identify identifying observed multiaddr: %s %s", c.LocalMultiaddr(), ifaceaddrs)
	if !addrInAddrs(c.LocalMultiaddr(), ifaceaddrs) {
		// not in our list
		return
	}

	// ok! we have the observed version of one of our ListenAddresses!
	log.Debugf("added own observed listen addr: %s --> %s", c.LocalMultiaddr(), maddr)
	ids.observedAddrs.Add(maddr, c.RemoteMultiaddr())
}

func addrInAddrs(a ma.Multiaddr, as []ma.Multiaddr) bool {
	for _, b := range as {
		if a.Equal(b) {
			return true
		}
	}
	return false
}

// protocolVersionsAreCompatible checks that the two implementations
// can talk to each other. It will use semver, but for now while
// we're in tight development, we will return false for minor version
// changes too.
func protocolVersionsAreCompatible(v1, v2 string) bool {
	if strings.HasPrefix(v1, "ipfs/") {
		v1 = v1[5:]
	}
	if strings.HasPrefix(v2, "ipfs/") {
		v2 = v2[5:]
	}

	v1s, err := semver.NewVersion(v1)
	if err != nil {
		return false
	}

	v2s, err := semver.NewVersion(v2)
	if err != nil {
		return false
	}

	return v1s.Major == v2s.Major && v1s.Minor == v2s.Minor
}

// netNotifiee defines methods to be used with the IpfsDHT
type netNotifiee IDService

func (nn *netNotifiee) IDService() *IDService {
	return (*IDService)(nn)
}

func (nn *netNotifiee) Connected(n inet.Network, v inet.Conn) {
	// TODO: deprecate the setConnHandler hook, and kick off
	// identification here.
}

func (nn *netNotifiee) Disconnected(n inet.Network, v inet.Conn) {
	// undo the setting of addresses to peer.ConnectedAddrTTL we did
	ids := nn.IDService()
	ps := ids.Host.Peerstore()
	addrs := ps.Addrs(v.RemotePeer())
	ps.SetAddrs(v.RemotePeer(), addrs, pstore.RecentlyConnectedAddrTTL)
}

func (nn *netNotifiee) OpenedStream(n inet.Network, v inet.Stream) {}
func (nn *netNotifiee) ClosedStream(n inet.Network, v inet.Stream) {}
func (nn *netNotifiee) Listen(n inet.Network, a ma.Multiaddr)      {}
func (nn *netNotifiee) ListenClose(n inet.Network, a ma.Multiaddr) {}

func logProtocolMismatchDisconnect(c inet.Conn, protocol, agent string) {
	lm := make(lgbl.DeferredMap)
	lm["remotePeer"] = func() interface{} { return c.RemotePeer().Pretty() }
	lm["remoteAddr"] = func() interface{} { return c.RemoteMultiaddr().String() }
	lm["protocolVersion"] = protocol
	lm["agentVersion"] = agent
	log.Event(context.TODO(), "IdentifyProtocolMismatch", lm)
	log.Debugf("IdentifyProtocolMismatch %s %s %s (disconnected)", c.RemotePeer(), protocol, agent)
}
<|endoftext|>


<commit_before>// Copyright 2016 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package web

import (
	"encoding/json"
	"strconv"
	"strings"
	"sync/atomic"
	"time"

	"github.com/control-center/serviced/dao"
	"github.com/control-center/serviced/domain"
	"github.com/control-center/serviced/health"
	"github.com/control-center/serviced/isvcs"
	"github.com/control-center/serviced/metrics"
	"github.com/control-center/serviced/node"
	"github.com/control-center/serviced/utils"
	"github.com/zenoss/glog"
	"github.com/zenoss/go-json-rest"
)

var (
	cachedvalue     atomic.Value
	cachetimeout    = time.Duration(5) * time.Second
	emptyvalue      = []byte{}
	isvcsRootHealth = map[string]health.HealthStatus{
		"alive": health.HealthStatus{
			Status:    health.OK,
			StartedAt: time.Now(),
		},
	}
)

// SetServiceStatsCacheTimeout sets the time in seconds for stats on
// running instances to be cached for the UI.
func SetServiceStatsCacheTimeout(seconds int) {
	cachetimeout = time.Duration(seconds) * time.Second
}

// ConciseServiceStatus contains the information necessary to display the
// status of a service in the UI. It joins running instances with health checks
// with memory stats. It does not include fields unnecessary for status
// display, such as config files or monitoring profiles.
type ConciseServiceStatus struct {
	DockerID        string
	HostID          string
	ID              string
	InSync          bool
	InstanceID      int
	Instances       int
	Name            string
	ParentServiceID string
	PoolID          string
	ServiceID       string
	StartedAt       time.Time
	RAMMax          int64
	RAMLast         int64
	RAMAverage      int64
	RAMCommitment   utils.EngNotation
	HealthChecks    map[string]health.HealthStatus
}

type memorykey struct {
	ServiceID  string
	InstanceID int
}

// convertInstancesToMetric converts dao.RunningServices into the structure the
// metrics API wants
func convertInstancesToMetric(instances []dao.RunningService) []metrics.ServiceInstance {
	svcInstances := make([]metrics.ServiceInstance, len(instances))
	for i, inst := range instances {
		svcInstances[i] = metrics.ServiceInstance{
			inst.ServiceID,
			inst.InstanceID,
		}
	}
	return svcInstances
}

func getAllServiceStatuses(client *node.ControlClient) (statuses []*ConciseServiceStatus, err error) {
	// Get all running service instances
	glog.V(2).Info("Retrieving statuses, memory and health checks for running services")
	var instances []dao.RunningService
	if err := client.GetRunningServices(&empty, &instances); err != nil {
		return nil, err
	}
	if instances == nil {
		glog.V(3).Info("Nil service list returned")
		instances = []dao.RunningService{}
	}

	// Look up memory stats for the last hour
	// Memory stats for isvcs don't work the same way, so we don't append
	// those services to this list yet
	memoryStats := make(map[memorykey]metrics.MemoryUsageStats)
	if len(instances) > 0 {
		response := []metrics.MemoryUsageStats{}
		query := dao.MetricRequest{
			StartTime: time.Now().Add(-time.Hour * 24),
			Instances: convertInstancesToMetric(instances),
		}
		if err := client.GetInstanceMemoryStats(query, &response); err != nil {
			glog.Errorf("Unable to look up instance memory stats (%s)", err)
		}
		for _, mus := range response {
			instID, err := strconv.Atoi(mus.InstanceID)
			if err != nil {
				glog.Errorf("Could not convert instance id %s", mus.InstanceID)
				continue
			}
			memoryStats[memorykey{ServiceID: mus.ServiceID, InstanceID: instID}] = mus
		}
	}

	// Look up health check statuses
	healthChecks := make(map[memorykey]map[string]health.HealthStatus)
	if len(instances) > 0 {
		results := make(map[string]map[int]map[string]health.HealthStatus)
		if err := client.GetServicesHealth(0, &results); err != nil {
			glog.Errorf("Unable to look up health check results (%s)", err)
		}
		for svcid, insts := range results {
			for instid, checks := range insts {
				healthChecks[memorykey{ServiceID: svcid, InstanceID: instid}] = checks
			}
		}
	}

	// Add isvcs to the mix
	isvcInstances := getIRS()
	for _, isvc := range isvcInstances {
		var checks map[string]health.HealthStatus
		instances = append(instances, isvc)
		if isvc.ServiceID == "isvc-internalservices" {
			checks = isvcsRootHealth
		} else {
			results, err := isvcs.Mgr.GetHealthStatus(strings.TrimPrefix(isvc.ServiceID, "isvc-"))
			if err != nil {
				glog.Warningf("Error acquiring health status for %s (%s)", isvc.ServiceID, err)
				continue
			}
			checks = convertDomainHealthToNewHealth(results.HealthStatuses)
		}
		healthChecks[memorykey{ServiceID: isvc.ServiceID, InstanceID: isvc.InstanceID}] = checks
	}
	// Create the concise service statuses
	for _, instance := range instances {
		stat := &ConciseServiceStatus{
			DockerID:        instance.DockerID,
			HostID:          instance.HostID,
			ID:              instance.ID,
			InSync:          instance.InSync,
			InstanceID:      instance.InstanceID,
			Instances:       instance.Instances,
			Name:            instance.Name,
			ParentServiceID: instance.ParentServiceID,
			PoolID:          instance.PoolID,
			ServiceID:       instance.ServiceID,
			StartedAt:       instance.StartedAt,
			RAMCommitment:   instance.RAMCommitment,
		}
		key := memorykey{ServiceID: instance.ServiceID, InstanceID: instance.InstanceID}
		if mem, ok := memoryStats[key]; ok {
			stat.RAMMax = mem.Max
			stat.RAMLast = mem.Last
			stat.RAMAverage = mem.Average
		}
		if hc, ok := healthChecks[key]; ok {
			stat.HealthChecks = hc
		}
		statuses = append(statuses, stat)
	}
	return
}

func restGetConciseServiceStatus(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
	f := func() ([]byte, error) {
		statuses, err := getAllServiceStatuses(client)
		if err != nil {
			glog.V(2).Infof("Error retrieving service statuses: (%s)", err)
			return nil, err
		}
		bytes, err := json.Marshal(statuses)
		if err != nil {
			glog.V(2).Infof("Error serializing service statuses: (%s)", err)
			return nil, err
		}
		return bytes, nil
	}
	w.Header().Set("content-type", "application/json")
	bytes, err := getCached(f)
	if err != nil {
		glog.Errorf("Error retrieving service statuses: %s", err)
		restServerError(w, err)
	}
	w.Write(bytes)
}

func getCached(f func() ([]byte, error)) ([]byte, error) {
	var err error
	val := cachedvalue.Load()
	if val == nil || len(val.([]byte)) == 0 {
		val, err = f()
		if err != nil {
			return nil, err
		}
		cachedvalue.Store(val)
		go func() {
			time.Sleep(cachetimeout)
			cachedvalue.Store(emptyvalue)
		}()
	}
	return val.([]byte), nil
}

// ConvertDomainHealthToNewHealth changes the domain health check status into
// a new-style health status for consumption by the UI or other interface.
// Sort of deprecated, in that it will be rendered useless once
// domain.HealthCheckStatus is no longer returned by isvcs.Mgr.
func convertDomainHealthToNewHealth(sources []domain.HealthCheckStatus) map[string]health.HealthStatus {
	var status health.Status
	result := make(map[string]health.HealthStatus)
	for _, hcs := range sources {
		switch hcs.Status {
		case "passed":
			status = health.OK
		case "failed":
			status = health.Failed
		default:
			status = health.Unknown
		}
		// Not going to bother populating Duration, since it doesn't have an
		// analogue in domain.HealthCheckStatus.
		result[hcs.Name] = health.HealthStatus{
			Status:    status,
			StartedAt: time.Unix(hcs.Timestamp, 0),
		}
	}
	return result
}
<commit_msg>WIP: trying to find where bad healthchecks are getting in<commit_after>// Copyright 2016 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package web

import (
	"encoding/json"
	"strconv"
	"strings"
	"sync/atomic"
	"time"

	"github.com/control-center/serviced/dao"
	"github.com/control-center/serviced/domain"
	"github.com/control-center/serviced/health"
	"github.com/control-center/serviced/isvcs"
	"github.com/control-center/serviced/metrics"
	"github.com/control-center/serviced/node"
	"github.com/control-center/serviced/utils"
	"github.com/zenoss/glog"
	"github.com/zenoss/go-json-rest"
	"fmt"
)

var (
	cachedvalue     atomic.Value
	cachetimeout    = time.Duration(5) * time.Second
	emptyvalue      = []byte{}
	isvcsRootHealth = map[string]health.HealthStatus{
		"alive": health.HealthStatus{
			Status:    health.OK,
			StartedAt: time.Now(),
		},
	}
)

// SetServiceStatsCacheTimeout sets the time in seconds for stats on
// running instances to be cached for the UI.
func SetServiceStatsCacheTimeout(seconds int) {
	cachetimeout = time.Duration(seconds) * time.Second
}

// ConciseServiceStatus contains the information necessary to display the
// status of a service in the UI. It joins running instances with health checks
// with memory stats. It does not include fields unnecessary for status
// display, such as config files or monitoring profiles.
type ConciseServiceStatus struct {
	DockerID        string
	HostID          string
	ID              string
	InSync          bool
	InstanceID      int
	Instances       int
	Name            string
	ParentServiceID string
	PoolID          string
	ServiceID       string
	StartedAt       time.Time
	RAMMax          int64
	RAMLast         int64
	RAMAverage      int64
	RAMCommitment   utils.EngNotation
	HealthChecks    map[string]health.HealthStatus
}

type memorykey struct {
	ServiceID  string
	InstanceID int
}

// convertInstancesToMetric converts dao.RunningServices into the structure the
// metrics API wants
func convertInstancesToMetric(instances []dao.RunningService) []metrics.ServiceInstance {
	svcInstances := make([]metrics.ServiceInstance, len(instances))
	for i, inst := range instances {
		svcInstances[i] = metrics.ServiceInstance{
			inst.ServiceID,
			inst.InstanceID,
		}
	}
	return svcInstances
}

func getAllServiceStatuses(client *node.ControlClient) (statuses []*ConciseServiceStatus, err error) {
	// Get all running service instances
	glog.V(2).Info("Retrieving statuses, memory and health checks for running services")
	var instances []dao.RunningService
	if err := client.GetRunningServices(&empty, &instances); err != nil {
		return nil, err
	}
	if instances == nil {
		glog.V(3).Info("Nil service list returned")
		instances = []dao.RunningService{}
	}

	// Look up memory stats for the last hour
	// Memory stats for isvcs don't work the same way, so we don't append
	// those services to this list yet
	memoryStats := make(map[memorykey]metrics.MemoryUsageStats)
	if len(instances) > 0 {
		response := []metrics.MemoryUsageStats{}
		query := dao.MetricRequest{
			StartTime: time.Now().Add(-time.Hour * 24),
			Instances: convertInstancesToMetric(instances),
		}
		if err := client.GetInstanceMemoryStats(query, &response); err != nil {
			glog.Errorf("Unable to look up instance memory stats (%s)", err)
		}
		for _, mus := range response {
			instID, err := strconv.Atoi(mus.InstanceID)
			if err != nil {
				glog.Errorf("Could not convert instance id %s", mus.InstanceID)
				continue
			}
			memoryStats[memorykey{ServiceID: mus.ServiceID, InstanceID: instID}] = mus
		}
	}

	// Look up health check statuses
	healthChecks := make(map[memorykey]map[string]health.HealthStatus)
	if len(instances) > 0 {
		results := make(map[string]map[int]map[string]health.HealthStatus)
		if err := client.GetServicesHealth(0, &results); err != nil {
			glog.Errorf("Unable to look up health check results (%s)", err)
		}
		for svcid, insts := range results {
			for instid, checks := range insts {
				healthChecks[memorykey{ServiceID: svcid, InstanceID: instid}] = checks
			}
		}
	}

	// Add isvcs to the mix
	isvcInstances := getIRS()
	for _, isvc := range isvcInstances {
		var checks map[string]health.HealthStatus
		instances = append(instances, isvc)
		if isvc.ServiceID == "isvc-internalservices" {
			checks = isvcsRootHealth
		} else {
			results, err := isvcs.Mgr.GetHealthStatus(strings.TrimPrefix(isvc.ServiceID, "isvc-"))
			if err != nil {
				glog.Warningf("Error acquiring health status for %s (%s)", isvc.ServiceID, err)
				continue
			}
			checks = convertDomainHealthToNewHealth(results.HealthStatuses)
		}
		healthChecks[memorykey{ServiceID: isvc.ServiceID, InstanceID: isvc.InstanceID}] = checks
	}
	// Create the concise service statuses
	for _, instance := range instances {
		stat := &ConciseServiceStatus{
			DockerID:        instance.DockerID,
			HostID:          instance.HostID,
			ID:              instance.ID,
			InSync:          instance.InSync,
			InstanceID:      instance.InstanceID,
			Instances:       instance.Instances,
			Name:            instance.Name,
			ParentServiceID: instance.ParentServiceID,
			PoolID:          instance.PoolID,
			ServiceID:       instance.ServiceID,
			StartedAt:       instance.StartedAt,
			RAMCommitment:   instance.RAMCommitment,
		}
		key := memorykey{ServiceID: instance.ServiceID, InstanceID: instance.InstanceID}
		if mem, ok := memoryStats[key]; ok {
			stat.RAMMax = mem.Max
			stat.RAMLast = mem.Last
			stat.RAMAverage = mem.Average
		}
		if hc, ok := healthChecks[key]; ok {
			stat.HealthChecks = hc
		}
		statuses = append(statuses, stat)
	}
	//fmt.Println("Statues: ", statuses)
	fmt.Println("Name\t ServiceID\t InstanceID\t DockerID\t HostID")
	zopes := []*ConciseServiceStatus{}
	for _, s := range statuses {
		if strings.Contains(s.Name, "Zope") {
			zopes = append(zopes, s)
			if s.InstanceID == 0 {
				fmt.Printf("%s\t %s\t %s\t %d\t %s\t %s\n", s.Name, s.ParentServiceID, s.ServiceID, s.InstanceID, s.DockerID, s.HostID)
				if s.HealthChecks != nil {
					//k := memorykey{ServiceID: s.ServiceID, InstanceID:s.InstanceID}
					fmt.Println("")
					for hc_key, hc_val := range s.HealthChecks {
						fmt.Print(hc_key, hc_val)
					}
					fmt.Println("")
				}
			}
		}

	}
	fmt.Println(zopes)
	return
}

func restGetConciseServiceStatus(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
	f := func() ([]byte, error) {
		statuses, err := getAllServiceStatuses(client)
		if err != nil {
			glog.V(2).Infof("Error retrieving service statuses: (%s)", err)
			return nil, err
		}
		bytes, err := json.Marshal(statuses)
		if err != nil {
			glog.V(2).Infof("Error serializing service statuses: (%s)", err)
			return nil, err
		}
		return bytes, nil
	}
	w.Header().Set("content-type", "application/json")
	bytes, err := getCached(f)
	if err != nil {
		glog.Errorf("Error retrieving service statuses: %s", err)
		restServerError(w, err)
	}
	w.Write(bytes)
}

func getCached(f func() ([]byte, error)) ([]byte, error) {
	var err error
	val := cachedvalue.Load()
	if val == nil || len(val.([]byte)) == 0 {
		val, err = f()
		if err != nil {
			return nil, err
		}
		cachedvalue.Store(val)
		go func() {
			time.Sleep(cachetimeout)
			cachedvalue.Store(emptyvalue)
		}()
	}
	return val.([]byte), nil
}

// ConvertDomainHealthToNewHealth changes the domain health check status into
// a new-style health status for consumption by the UI or other interface.
// Sort of deprecated, in that it will be rendered useless once
// domain.HealthCheckStatus is no longer returned by isvcs.Mgr.
func convertDomainHealthToNewHealth(sources []domain.HealthCheckStatus) map[string]health.HealthStatus {
	var status health.Status
	result := make(map[string]health.HealthStatus)
	for _, hcs := range sources {
		switch hcs.Status {
		case "passed":
			status = health.OK
		case "failed":
			status = health.Failed
		default:
			status = health.Unknown
		}
		// Not going to bother populating Duration, since it doesn't have an
		// analogue in domain.HealthCheckStatus.
		result[hcs.Name] = health.HealthStatus{
			Status:    status,
			StartedAt: time.Unix(hcs.Timestamp, 0),
		}
	}
	return result
}
<|endoftext|>


<commit_before>package consul

import (
	"fmt"
	"strings"

	"github.com/hashicorp/consul/api"
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/db/keyval"
	"github.com/ligato/cn-infra/logging/logrus"
)

type ConsulStore struct {
	client *api.Client
}

func NewConsulStore(addr string) (store *ConsulStore, err error) {
	cfg := api.DefaultConfig()
	cfg.Address = addr

	var c *api.Client
	if c, err = api.NewClient(cfg); err != nil {
		return nil, fmt.Errorf("failed to create Consul client %s", err)
	}

	peers, err := c.Status().Peers()
	if err != nil {
		return nil, err
	}
	logrus.DefaultLogger().Debugf("consul peers: %v", peers)

	return &ConsulStore{
		client: c,
	}, nil

}

func transformKey(key string) string {
	return strings.TrimPrefix(key, "/")
}

func (c *ConsulStore) Put(key string, data []byte, opts ...datasync.PutOption) error {
	fmt.Printf("put: %q\n", key)
	p := &api.KVPair{Key: transformKey(key), Value: data}
	_, err := c.client.KV().Put(p, nil)
	if err != nil {
		return err
	}

	return nil
}

func (c *ConsulStore) NewTxn() keyval.BytesTxn {
	panic("implement me")
}

func (c *ConsulStore) GetValue(key string) (data []byte, found bool, revision int64, err error) {
	fmt.Printf("get value: %q\n", key)
	pair, _, err := c.client.KV().Get(transformKey(key), nil)
	if err != nil {
		return nil, false, 0, err
	} else if pair == nil {
		return nil, false, 0, nil
	}

	return pair.Value, true, int64(pair.ModifyIndex), nil
}

func (c *ConsulStore) ListValues(key string) (keyval.BytesKeyValIterator, error) {
	pairs, _, err := c.client.KV().List(transformKey(key), nil)
	if err != nil {
		return nil, err
	}

	return &bytesKeyValIterator{len: len(pairs), pairs: pairs}, nil
}

func (c *ConsulStore) ListKeys(prefix string) (keyval.BytesKeyIterator, error) {
	keys, _, err := c.client.KV().Keys(transformKey(prefix), "", nil)
	if err != nil {
		return nil, err
	}

	return &bytesKeyIterator{len: len(keys), keys: keys}, nil
}

func (c *ConsulStore) Delete(key string, opts ...datasync.DelOption) (existed bool, err error) {
	fmt.Printf("delete: %q\n", key)
	if _, err := c.client.KV().Delete(transformKey(key), nil); err != nil {
		return false, err
	}

	return true, nil
}

func (c *ConsulStore) Watch(respChan func(keyval.BytesWatchResp), closeChan chan string, keys ...string) error {
	panic("implement me")
}

func (c *ConsulStore) NewBroker(prefix string) keyval.BytesBroker {
	panic("implement me")
}

func (c *ConsulStore) NewWatcher(prefix string) keyval.BytesWatcher {
	panic("implement me")
}

func (c *ConsulStore) Close() error {
	return nil
}

// bytesKeyIterator is an iterator returned by ListKeys call.
type bytesKeyIterator struct {
	index int
	len   int
	keys  []string
}

// GetNext returns the following key (+ revision) from the result set.
// When there are no more keys to get, <stop> is returned as *true*
// and <key> and <rev> are default values.
func (ctx *bytesKeyIterator) GetNext() (key string, rev int64, stop bool) {
	if ctx.index >= ctx.len {
		return "", 0, true
	}

	key = string(ctx.keys[ctx.index])
	rev = 0 //ctx.keys[ctx.index].mod
	ctx.index++

	return key, rev, false
}

// Close does nothing since db cursors are not needed.
// The method is required by the code since it implements Iterator API.
func (ctx *bytesKeyIterator) Close() error {
	return nil
}

// bytesKeyValIterator is an iterator returned by ListValues call.
type bytesKeyValIterator struct {
	index int
	len   int
	pairs api.KVPairs
}

func (ctx *bytesKeyValIterator) GetNext() (val keyval.BytesKeyVal, stop bool) {
	if ctx.index >= ctx.len {
		return nil, true
	}

	key := string(ctx.pairs[ctx.index].Key)
	data := ctx.pairs[ctx.index].Value
	rev := int64(ctx.pairs[ctx.index].ModifyIndex)

	var prevValue []byte
	if len(ctx.pairs) > 0 && ctx.index > 0 {
		prevValue = ctx.pairs[ctx.index-1].Value
	}

	ctx.index++

	return &bytesKeyVal{key, data, prevValue, rev}, false
}

func (ctx *bytesKeyValIterator) Close() error {
	return nil
}

// bytesKeyVal represents a single key-value pair.
type bytesKeyVal struct {
	key       string
	value     []byte
	prevValue []byte
	revision  int64
}

// Close does nothing since db cursors are not needed.
// The method is required by the code since it implements Iterator API.
func (kv *bytesKeyVal) Close() error {
	return nil
}

// GetValue returns the value of the pair.
func (kv *bytesKeyVal) GetValue() []byte {
	return kv.value
}

// GetPrevValue returns the previous value of the pair.
func (kv *bytesKeyVal) GetPrevValue() []byte {
	return kv.prevValue
}

// GetKey returns the key of the pair.
func (kv *bytesKeyVal) GetKey() string {
	return kv.key
}

// GetRevision returns the revision associated with the pair.
func (kv *bytesKeyVal) GetRevision() int64 {
	return kv.revision
}
<commit_msg>Satisfy linter<commit_after>package consul

import (
	"fmt"
	"strings"

	"github.com/hashicorp/consul/api"
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/db/keyval"
	"github.com/ligato/cn-infra/logging/logrus"
)

// Store serves as a client for Consul KV storage and implements keyval.CoreBrokerWatcher interface.
type Store struct {
	client *api.Client
}

// NewConsulStore creates new client for Consul using given address.
func NewConsulStore(addr string) (store *Store, err error) {
	cfg := api.DefaultConfig()
	cfg.Address = addr

	var c *api.Client
	if c, err = api.NewClient(cfg); err != nil {
		return nil, fmt.Errorf("failed to create Consul client %s", err)
	}

	peers, err := c.Status().Peers()
	if err != nil {
		return nil, err
	}
	logrus.DefaultLogger().Debugf("consul peers: %v", peers)

	return &Store{
		client: c,
	}, nil

}

func transformKey(key string) string {
	return strings.TrimPrefix(key, "/")
}

// Put stores given data for the key.
func (c *Store) Put(key string, data []byte, opts ...datasync.PutOption) error {
	fmt.Printf("put: %q\n", key)
	p := &api.KVPair{Key: transformKey(key), Value: data}
	_, err := c.client.KV().Put(p, nil)
	if err != nil {
		return err
	}

	return nil
}

// NewTxn creates new transaction.
func (c *Store) NewTxn() keyval.BytesTxn {
	panic("implement me")
}

// GetValue returns data for the given key.
func (c *Store) GetValue(key string) (data []byte, found bool, revision int64, err error) {
	fmt.Printf("get value: %q\n", key)
	pair, _, err := c.client.KV().Get(transformKey(key), nil)
	if err != nil {
		return nil, false, 0, err
	} else if pair == nil {
		return nil, false, 0, nil
	}

	return pair.Value, true, int64(pair.ModifyIndex), nil
}

// ListValues returns interator with key-value pairs for given key prefix.
func (c *Store) ListValues(key string) (keyval.BytesKeyValIterator, error) {
	pairs, _, err := c.client.KV().List(transformKey(key), nil)
	if err != nil {
		return nil, err
	}

	return &bytesKeyValIterator{len: len(pairs), pairs: pairs}, nil
}

// ListKeys returns interator with keys for given key prefix.
func (c *Store) ListKeys(prefix string) (keyval.BytesKeyIterator, error) {
	keys, _, err := c.client.KV().Keys(transformKey(prefix), "", nil)
	if err != nil {
		return nil, err
	}

	return &bytesKeyIterator{len: len(keys), keys: keys}, nil
}

// Delete deletes given key.
func (c *Store) Delete(key string, opts ...datasync.DelOption) (existed bool, err error) {
	fmt.Printf("delete: %q\n", key)
	if _, err := c.client.KV().Delete(transformKey(key), nil); err != nil {
		return false, err
	}

	return true, nil
}

// Watch watches given list of key prefixes.
func (c *Store) Watch(respChan func(keyval.BytesWatchResp), closeChan chan string, keys ...string) error {
	panic("implement me")
}

// NewBroker returns new broker.
func (c *Store) NewBroker(prefix string) keyval.BytesBroker {
	panic("implement me")
}

// NewWatcher returns new watcher.
func (c *Store) NewWatcher(prefix string) keyval.BytesWatcher {
	panic("implement me")
}

// Close returns nil.
func (c *Store) Close() error {
	return nil
}

// bytesKeyIterator is an iterator returned by ListKeys call.
type bytesKeyIterator struct {
	index int
	len   int
	keys  []string
}

// GetNext returns the following key (+ revision) from the result set.
// When there are no more keys to get, <stop> is returned as *true*
// and <key> and <rev> are default values.
func (ctx *bytesKeyIterator) GetNext() (key string, rev int64, stop bool) {
	if ctx.index >= ctx.len {
		return "", 0, true
	}

	key = string(ctx.keys[ctx.index])
	rev = 0 //ctx.keys[ctx.index].mod
	ctx.index++

	return key, rev, false
}

// Close does nothing since db cursors are not needed.
// The method is required by the code since it implements Iterator API.
func (ctx *bytesKeyIterator) Close() error {
	return nil
}

// bytesKeyValIterator is an iterator returned by ListValues call.
type bytesKeyValIterator struct {
	index int
	len   int
	pairs api.KVPairs
}

// GetNext returns the following item from the result set.
// When there are no more items to get, <stop> is returned as *true* and <val>
// is simply *nil*.
func (ctx *bytesKeyValIterator) GetNext() (val keyval.BytesKeyVal, stop bool) {
	if ctx.index >= ctx.len {
		return nil, true
	}

	key := string(ctx.pairs[ctx.index].Key)
	data := ctx.pairs[ctx.index].Value
	rev := int64(ctx.pairs[ctx.index].ModifyIndex)

	var prevValue []byte
	if len(ctx.pairs) > 0 && ctx.index > 0 {
		prevValue = ctx.pairs[ctx.index-1].Value
	}

	ctx.index++

	return &bytesKeyVal{key, data, prevValue, rev}, false
}

// Close does nothing since db cursors are not needed.
// The method is required by the code since it implements Iterator API.
func (ctx *bytesKeyValIterator) Close() error {
	return nil
}

// bytesKeyVal represents a single key-value pair.
type bytesKeyVal struct {
	key       string
	value     []byte
	prevValue []byte
	revision  int64
}

// Close does nothing since db cursors are not needed.
// The method is required by the code since it implements Iterator API.
func (kv *bytesKeyVal) Close() error {
	return nil
}

// GetValue returns the value of the pair.
func (kv *bytesKeyVal) GetValue() []byte {
	return kv.value
}

// GetPrevValue returns the previous value of the pair.
func (kv *bytesKeyVal) GetPrevValue() []byte {
	return kv.prevValue
}

// GetKey returns the key of the pair.
func (kv *bytesKeyVal) GetKey() string {
	return kv.key
}

// GetRevision returns the revision associated with the pair.
func (kv *bytesKeyVal) GetRevision() int64 {
	return kv.revision
}
<|endoftext|>