<commit_before>package php

import (
	"fmt"

	"stephensearles.com/php/ast"
)

/*

Valid Expression Patterns
Expr [Binary Op] Expr
[Unary Op] Expr
Expr [Unary Op]
Expr [Tertiary Op 1] Expr [Tertiary Op 2] Expr
Identifier
Literal
Function Call

Parentesis always triggers sub-expression

non-associative clone new clone and new
left  [ array()
right ++ -- ~ (int) (float) (string) (array) (object) (bool) @  types and increment/decrement
non-associative instanceof  types
right ! logical
left  * / % arithmetic
left  + - . arithmetic and string
left  << >> bitwise
non-associative < <= > >= comparison
non-associative == != === !== <>  comparison
left  & bitwise and references
left  ^ bitwise
left  | bitwise
left  &&  logical
left  ||  logical
left  ? : ternary
right = += -= *= /= .= %= &= |= ^= <<= >>= => assignment
left  and logical
left  xor logical
left  or  logical
left  , many uses

*/

var operatorPrecedence = map[ItemType]int{
	itemArrayLookupOperatorLeft: 19,
	itemUnaryOperator:           18,
	itemCastOperator:            18,
	itemInstanceofOperator:      17,
	itemNegationOperator:        16,
	itemMultOperator:            15,
	itemAdditionOperator:        14,
	itemSubtractionOperator:     14,
	itemConcatenationOperator:   14,

	itemBitwiseShiftOperator: 13,
	itemComparisonOperator:   12,
	itemEqualityOperator:     11,

	itemAmpersandOperator:  10,
	itemBitwiseXorOperator: 9,
	itemBitwiseOrOperator:  8,
	itemAndOperator:        7,
	itemOrOperator:         6,
	itemTernaryOperator1:   5,
	itemTernaryOperator2:   5,
	itemAssignmentOperator: 4,
	itemWrittenAndOperator: 3,
	itemWrittenXorOperator: 2,
	itemWrittenOrOperator:  1,
}

func (p *parser) parseExpression() (expr ast.Expression) {
	// consume expression
	originalParenLev := p.parenLevel
	switch p.current.typ {
	case itemIgnoreErrorOperator:
		p.next()
		return p.parseExpression()
	case itemFunction:
		return p.parseAnonymousFunction()
	case itemNewOperator:
		expr = p.parseInstantiation()
		p.next()
		expr = p.parseOperation(originalParenLev, expr)
		return
	case itemVariableOperator:
		if p.peek().typ == itemAssignmentOperator {
			assignee, ok := p.expressionize().(ast.Assignable)
			if !ok {
				p.errorf("%s is not assignable", assignee)
			}
			p.next()
			return ast.AssignmentExpression{
				Assignee: assignee,
				Operator: p.current.val,
				Value:    p.parseNextExpression(),
			}
		}
		fallthrough
	case itemArray:
		fallthrough
	case itemUnaryOperator, itemNegationOperator, itemAmpersandOperator, itemCastOperator, itemSubtractionOperator:
		fallthrough
	case itemIdentifier, itemStringLiteral, itemNumberLiteral, itemBooleanLiteral, itemInclude, itemNull, itemSelf, itemStatic, itemParent:
		expr = p.parseOperation(originalParenLev, p.expressionize())
	case itemOpenParen:
		p.parenLevel += 1
		p.next()
		expr = p.parseExpression()
		p.expect(itemCloseParen)
		p.parenLevel -= 1
		expr = p.parseOperation(originalParenLev, expr)
	default:
		p.errorf("Expected expression. Found %s", p.current)
		return
	}
	if p.parenLevel != originalParenLev {
		p.errorf("unbalanced parens: %d prev: %d", p.parenLevel, originalParenLev)
		return
	}
	return
}

func (p *parser) parseOperation(originalParenLevel int, lhs ast.Expression) (expr ast.Expression) {
	p.next()
	switch p.current.typ {
	case itemIgnoreErrorOperator:
		return p.parseOperation(originalParenLevel, lhs)
	case itemUnaryOperator:
		expr = p.parseUnaryExpressionLeft(lhs, p.current)
	case itemAdditionOperator, itemSubtractionOperator, itemConcatenationOperator, itemComparisonOperator, itemMultOperator, itemAndOperator, itemOrOperator, itemAmpersandOperator, itemBitwiseXorOperator, itemBitwiseOrOperator, itemBitwiseShiftOperator, itemWrittenAndOperator, itemWrittenXorOperator, itemWrittenOrOperator, itemInstanceofOperator:
		expr = p.parseBinaryOperation(lhs, p.current, originalParenLevel)
	case itemTernaryOperator1:
		expr = p.parseTernaryOperation(lhs)
	case itemCloseParen:
		if p.parenLevel <= originalParenLevel {
			p.backup()
			return lhs
		}
		p.parenLevel -= 1
		return p.parseOperation(originalParenLevel, lhs)
	case itemScopeResolutionOperator:
		p.next()
		expr = &ast.ClassExpression{Receiver: lhs, Expression: p.expressionize()}
	case itemArrayLookupOperatorLeft:
		expr = p.parseArrayLookup(lhs)
	case itemObjectOperator:
		expr = p.parseObjectLookup(lhs)
	case itemAssignmentOperator:
		assignee, ok := lhs.(ast.Assignable)
		if !ok {
			p.errorf("%s is not assignable", assignee)
		}
		expr = ast.AssignmentExpression{
			Assignee: assignee,
			Operator: p.current.val,
			Value:    p.parseNextExpression(),
		}
		fmt.Println(p.current)
	default:
		p.backup()
		return lhs
	}
	return p.parseOperation(originalParenLevel, expr)
}

func newUnaryOperation(operator Item, expr ast.Expression) ast.OperatorExpression {
	t := ast.Numeric
	if operator.val == "!" {
		t = ast.Boolean
	}
	return ast.OperatorExpression{
		Type:     t,
		Operand1: expr,
		Operator: operator.val,
	}
}

func newBinaryOperation(operator Item, expr1, expr2 ast.Expression) ast.OperatorExpression {
	t := ast.Numeric
	switch operator.typ {
	case itemComparisonOperator, itemAndOperator, itemOrOperator, itemWrittenAndOperator, itemWrittenOrOperator, itemWrittenXorOperator:
		t = ast.Boolean
	case itemConcatenationOperator:
		t = ast.String
	case itemAmpersandOperator, itemBitwiseXorOperator, itemBitwiseOrOperator, itemBitwiseShiftOperator:
		t = ast.AnyType
	}
	return ast.OperatorExpression{
		Type:     t,
		Operand1: expr1,
		Operand2: expr2,
		Operator: operator.val,
	}
}

func (p *parser) parseBinaryOperation(lhs ast.Expression, operator Item, originalParenLevel int) ast.Expression {
	p.next()
	rhs := p.expressionize()
	for {
		nextOperator := p.peek()
		nextOperatorPrecedence, ok := operatorPrecedence[nextOperator.typ]
		if ok && nextOperatorPrecedence > operatorPrecedence[operator.typ] {
			rhs = p.parseOperation(originalParenLevel, rhs)
		} else {
			break
		}
	}
	return newBinaryOperation(operator, lhs, rhs)
}

func (p *parser) parseTernaryOperation(lhs ast.Expression) ast.Expression {
	truthy := p.parseNextExpression()
	p.expect(itemTernaryOperator2)
	falsy := p.parseNextExpression()
	return &ast.OperatorExpression{
		Operand1: lhs,
		Operand2: truthy,
		Operand3: falsy,
		Type:     truthy.EvaluatesTo() | falsy.EvaluatesTo(),
		Operator: "?:",
	}
}

func (p *parser) parseUnaryExpressionRight(operand ast.Expression, operator Item) ast.Expression {
	return newUnaryOperation(operator, operand)
}

func (p *parser) parseUnaryExpressionLeft(operand ast.Expression, operator Item) ast.Expression {
	return newUnaryOperation(operator, operand)
}

// expressionize takes the current token and returns it as the simplest
// expression for that token. That means an expression with no operators
// except for the object operator.
func (p *parser) expressionize() (expr ast.Expression) {
	for {
		switch p.current.typ {
		case itemStringLiteral, itemBooleanLiteral, itemNumberLiteral, itemNull:
			return p.parseLiteral()
		case itemUnaryOperator, itemNegationOperator, itemCastOperator, itemSubtractionOperator, itemAmpersandOperator:
			op := p.current
			p.next()
			return p.parseUnaryExpressionRight(p.expressionize(), op)

		case itemArray:
			expr = p.parseArrayDeclaration()
		case itemVariableOperator:
			expr = p.parseVariable()
			p.next()
		case itemIdentifier:
			if p.peek().typ == itemOpenParen {
				expr = p.parseFunctionCall(ast.Identifier{Value: p.current.val})
				continue
			}
			fallthrough
		case itemSelf, itemStatic, itemParent:
			if p.peek().typ == itemScopeResolutionOperator {
				r := p.current.val
				p.expect(itemScopeResolutionOperator)
				expr = ast.NewClassExpression(r, p.parseNextExpression())
				return
			}
			expr = ast.ConstantExpression{
				Variable: ast.NewVariable(p.current.val),
			}
			p.next()
		case itemOpenParen:
			expr = p.parseExpression()
		default:
			p.backup()
			return
		}
	}
}

func (p *parser) parseLiteral() *ast.Literal {
	switch p.current.typ {
	case itemStringLiteral:
		return &ast.Literal{Type: ast.String}
	case itemBooleanLiteral:
		return &ast.Literal{Type: ast.Boolean}
	case itemNumberLiteral:
		return &ast.Literal{Type: ast.Float}
	case itemNull:
		return &ast.Literal{Type: ast.Null}
	}
	p.errorf("Unknown literal type")
	return nil
}

func (p *parser) parseVariable() ast.Expression {
	p.expectCurrent(itemVariableOperator)
	switch p.next(); {
	case isKeyword(p.current.typ):
		// keywords are all valid variable names
		fallthrough
	case p.current.typ == itemIdentifier:
		expr := ast.NewVariable(p.current.val)
		return expr
	default:
		return p.parseExpression()
	}
}

func (p *parser) parseAnonymousFunction() ast.Expression {
	f := &ast.AnonymousFunction{}
	f.Arguments = make([]ast.FunctionArgument, 0)
	p.expect(itemOpenParen)
	if p.peek().typ == itemCloseParen {
		p.expect(itemCloseParen)
		return f
	}
	f.Arguments = append(f.Arguments, p.parseFunctionArgument())
	for {
		switch p.peek().typ {
		case itemComma:
			p.expect(itemComma)
			f.Arguments = append(f.Arguments, p.parseFunctionArgument())
		case itemCloseParen:
			p.expect(itemCloseParen)
			return f
		default:
			p.errorf("unexpected argument separator:", p.current)
			return f
		}
	}
	f.Body = p.parseBlock()
	return f
}
<commit_msg>Now two tests failing for real<commit_after>package php

import "stephensearles.com/php/ast"

/*

Valid Expression Patterns
Expr [Binary Op] Expr
[Unary Op] Expr
Expr [Unary Op]
Expr [Tertiary Op 1] Expr [Tertiary Op 2] Expr
Identifier
Literal
Function Call

Parentesis always triggers sub-expression

non-associative clone new clone and new
left  [ array()
right ++ -- ~ (int) (float) (string) (array) (object) (bool) @  types and increment/decrement
non-associative instanceof  types
right ! logical
left  * / % arithmetic
left  + - . arithmetic and string
left  << >> bitwise
non-associative < <= > >= comparison
non-associative == != === !== <>  comparison
left  & bitwise and references
left  ^ bitwise
left  | bitwise
left  &&  logical
left  ||  logical
left  ? : ternary
right = += -= *= /= .= %= &= |= ^= <<= >>= => assignment
left  and logical
left  xor logical
left  or  logical
left  , many uses

*/

var operatorPrecedence = map[ItemType]int{
	itemArrayLookupOperatorLeft: 19,
	itemUnaryOperator:           18,
	itemCastOperator:            18,
	itemInstanceofOperator:      17,
	itemNegationOperator:        16,
	itemMultOperator:            15,
	itemAdditionOperator:        14,
	itemSubtractionOperator:     14,
	itemConcatenationOperator:   14,

	itemBitwiseShiftOperator: 13,
	itemComparisonOperator:   12,
	itemEqualityOperator:     11,

	itemAmpersandOperator:  10,
	itemBitwiseXorOperator: 9,
	itemBitwiseOrOperator:  8,
	itemAndOperator:        7,
	itemOrOperator:         6,
	itemTernaryOperator1:   5,
	itemTernaryOperator2:   5,
	itemAssignmentOperator: 4,
	itemWrittenAndOperator: 3,
	itemWrittenXorOperator: 2,
	itemWrittenOrOperator:  1,
}

func (p *parser) parseExpression() (expr ast.Expression) {
	// consume expression
	originalParenLev := p.parenLevel
	switch p.current.typ {
	case itemIgnoreErrorOperator:
		p.next()
		return p.parseExpression()
	case itemFunction:
		return p.parseAnonymousFunction()
	case itemNewOperator:
		expr = p.parseInstantiation()
		p.next()
		expr = p.parseOperation(originalParenLev, expr)
		return
	case itemVariableOperator:
		if p.peek().typ == itemAssignmentOperator {
			assignee, ok := p.expressionize().(ast.Assignable)
			if !ok {
				p.errorf("%s is not assignable", assignee)
			}
			p.next()
			return ast.AssignmentExpression{
				Assignee: assignee,
				Operator: p.current.val,
				Value:    p.parseNextExpression(),
			}
		}
		fallthrough
	case itemArray:
		fallthrough
	case itemUnaryOperator, itemNegationOperator, itemAmpersandOperator, itemCastOperator, itemSubtractionOperator:
		fallthrough
	case itemIdentifier, itemStringLiteral, itemNumberLiteral, itemBooleanLiteral, itemInclude, itemNull, itemSelf, itemStatic, itemParent:
		expr = p.parseOperation(originalParenLev, p.expressionize())
	case itemOpenParen:
		p.parenLevel += 1
		p.next()
		expr = p.parseExpression()
		p.expect(itemCloseParen)
		p.parenLevel -= 1
		expr = p.parseOperation(originalParenLev, expr)
	default:
		p.errorf("Expected expression. Found %s", p.current)
		return
	}
	if p.parenLevel != originalParenLev {
		p.errorf("unbalanced parens: %d prev: %d", p.parenLevel, originalParenLev)
		return
	}
	return
}

func (p *parser) parseOperation(originalParenLevel int, lhs ast.Expression) (expr ast.Expression) {
	p.next()
	switch p.current.typ {
	case itemIgnoreErrorOperator:
		return p.parseOperation(originalParenLevel, lhs)
	case itemUnaryOperator:
		expr = p.parseUnaryExpressionLeft(lhs, p.current)
	case itemAdditionOperator, itemSubtractionOperator, itemConcatenationOperator, itemComparisonOperator, itemMultOperator, itemAndOperator, itemOrOperator, itemAmpersandOperator, itemBitwiseXorOperator, itemBitwiseOrOperator, itemBitwiseShiftOperator, itemWrittenAndOperator, itemWrittenXorOperator, itemWrittenOrOperator, itemInstanceofOperator:
		expr = p.parseBinaryOperation(lhs, p.current, originalParenLevel)
	case itemTernaryOperator1:
		expr = p.parseTernaryOperation(lhs)
	case itemCloseParen:
		if p.parenLevel <= originalParenLevel {
			p.backup()
			return lhs
		}
		p.parenLevel -= 1
		return p.parseOperation(originalParenLevel, lhs)
	case itemScopeResolutionOperator:
		p.next()
		expr = &ast.ClassExpression{Receiver: lhs, Expression: p.expressionize()}
	case itemArrayLookupOperatorLeft:
		expr = p.parseArrayLookup(lhs)
	case itemObjectOperator:
		expr = p.parseObjectLookup(lhs)
	case itemAssignmentOperator:
		assignee, ok := lhs.(ast.Assignable)
		if !ok {
			p.errorf("%s is not assignable", assignee)
		}
		expr = ast.AssignmentExpression{
			Assignee: assignee,
			Operator: p.current.val,
			Value:    p.parseNextExpression(),
		}
	default:
		p.backup()
		return lhs
	}
	return p.parseOperation(originalParenLevel, expr)
}

func newUnaryOperation(operator Item, expr ast.Expression) ast.OperatorExpression {
	t := ast.Numeric
	if operator.val == "!" {
		t = ast.Boolean
	}
	return ast.OperatorExpression{
		Type:     t,
		Operand1: expr,
		Operator: operator.val,
	}
}

func newBinaryOperation(operator Item, expr1, expr2 ast.Expression) ast.OperatorExpression {
	t := ast.Numeric
	switch operator.typ {
	case itemComparisonOperator, itemAndOperator, itemOrOperator, itemWrittenAndOperator, itemWrittenOrOperator, itemWrittenXorOperator:
		t = ast.Boolean
	case itemConcatenationOperator:
		t = ast.String
	case itemAmpersandOperator, itemBitwiseXorOperator, itemBitwiseOrOperator, itemBitwiseShiftOperator:
		t = ast.AnyType
	}
	return ast.OperatorExpression{
		Type:     t,
		Operand1: expr1,
		Operand2: expr2,
		Operator: operator.val,
	}
}

func (p *parser) parseBinaryOperation(lhs ast.Expression, operator Item, originalParenLevel int) ast.Expression {
	p.next()
	rhs := p.expressionize()
	for {
		nextOperator := p.peek()
		nextOperatorPrecedence, ok := operatorPrecedence[nextOperator.typ]
		if ok && nextOperatorPrecedence > operatorPrecedence[operator.typ] {
			rhs = p.parseOperation(originalParenLevel, rhs)
		} else {
			break
		}
	}
	return newBinaryOperation(operator, lhs, rhs)
}

func (p *parser) parseTernaryOperation(lhs ast.Expression) ast.Expression {
	truthy := p.parseNextExpression()
	p.expect(itemTernaryOperator2)
	falsy := p.parseNextExpression()
	return &ast.OperatorExpression{
		Operand1: lhs,
		Operand2: truthy,
		Operand3: falsy,
		Type:     truthy.EvaluatesTo() | falsy.EvaluatesTo(),
		Operator: "?:",
	}
}

func (p *parser) parseUnaryExpressionRight(operand ast.Expression, operator Item) ast.Expression {
	return newUnaryOperation(operator, operand)
}

func (p *parser) parseUnaryExpressionLeft(operand ast.Expression, operator Item) ast.Expression {
	return newUnaryOperation(operator, operand)
}

// expressionize takes the current token and returns it as the simplest
// expression for that token. That means an expression with no operators
// except for the object operator.
func (p *parser) expressionize() (expr ast.Expression) {
	// This case must come first and not repeat
	switch p.current.typ {
	case itemUnaryOperator, itemNegationOperator, itemCastOperator, itemSubtractionOperator, itemAmpersandOperator:
		op := p.current
		p.next()
		return p.parseUnaryExpressionRight(p.expressionize(), op)
	}
	for {
		switch p.current.typ {
		case itemStringLiteral, itemBooleanLiteral, itemNumberLiteral, itemNull:
			return p.parseLiteral()
		case itemUnaryOperator:
			expr = newUnaryOperation(p.current, expr)
			p.next()
		case itemArray:
			expr = p.parseArrayDeclaration()
		case itemVariableOperator:
			expr = p.parseVariable()
			p.next()
		case itemIdentifier:
			if p.peek().typ == itemOpenParen {
				expr = p.parseFunctionCall(ast.Identifier{Value: p.current.val})
				continue
			}
			fallthrough
		case itemSelf, itemStatic, itemParent:
			if p.peek().typ == itemScopeResolutionOperator {
				r := p.current.val
				p.expect(itemScopeResolutionOperator)
				expr = ast.NewClassExpression(r, p.parseNextExpression())
				return
			}
			expr = ast.ConstantExpression{
				Variable: ast.NewVariable(p.current.val),
			}
			p.next()
		case itemOpenParen:
			expr = p.parseExpression()
		default:
			p.backup()
			return
		}
	}
}

func (p *parser) parseLiteral() *ast.Literal {
	switch p.current.typ {
	case itemStringLiteral:
		return &ast.Literal{Type: ast.String}
	case itemBooleanLiteral:
		return &ast.Literal{Type: ast.Boolean}
	case itemNumberLiteral:
		return &ast.Literal{Type: ast.Float}
	case itemNull:
		return &ast.Literal{Type: ast.Null}
	}
	p.errorf("Unknown literal type")
	return nil
}

func (p *parser) parseVariable() ast.Expression {
	p.expectCurrent(itemVariableOperator)
	switch p.next(); {
	case isKeyword(p.current.typ):
		// keywords are all valid variable names
		fallthrough
	case p.current.typ == itemIdentifier:
		expr := ast.NewVariable(p.current.val)
		return expr
	default:
		return p.parseExpression()
	}
}

func (p *parser) parseAnonymousFunction() ast.Expression {
	f := &ast.AnonymousFunction{}
	f.Arguments = make([]ast.FunctionArgument, 0)
	p.expect(itemOpenParen)
	if p.peek().typ == itemCloseParen {
		p.expect(itemCloseParen)
		return f
	}
	f.Arguments = append(f.Arguments, p.parseFunctionArgument())
	for {
		switch p.peek().typ {
		case itemComma:
			p.expect(itemComma)
			f.Arguments = append(f.Arguments, p.parseFunctionArgument())
		case itemCloseParen:
			p.expect(itemCloseParen)
			return f
		default:
			p.errorf("unexpected argument separator:", p.current)
			return f
		}
	}
	f.Body = p.parseBlock()
	return f
}
<|endoftext|>


<commit_before>package mem

import (
	"encoding/json"
	"fmt"
	"io/ioutil"

	"github.com/funkygao/gafka/cmd/kateway/inflights"
	"github.com/funkygao/golib/cmap"
	log "github.com/funkygao/log4go"
)

type dumpRecord struct {
	Key string
	Val message
}

type message struct {
	Offset int64
	Value  []byte
}

type memInflights struct {
	offsets cmap.ConcurrentMap

	snapshotFile string
	debug        bool
}

func New(fn string, debug bool) *memInflights {
	return &memInflights{
		offsets:      cmap.New(),
		snapshotFile: fn,
		debug:        debug,
	}
}

// FIXME bad perf
func (this *memInflights) key(cluster, topic, group, partition string) string {
	return fmt.Sprintf("%s:%s:%s:%s", cluster, topic, group, partition)
}

func (this *memInflights) Land(cluster, topic, group, partition string, offset int64) error {
	key := this.key(cluster, topic, group, partition)
	if this.debug {
		log.Info("Land %s", key)
	}
	o, found := this.offsets.Get(key)
	if !found || o.(message).Offset != offset {
		if this.debug {
			log.Info("out of order: %s", this)
		}
		return inflights.ErrOutOfOrder
	}

	this.offsets.Remove(key)
	return nil
}

func (this *memInflights) LandX(cluster, topic, group, partition string, offset int64) ([]byte, error) {
	key := this.key(cluster, topic, group, partition)
	if this.debug {
		log.Info("LandX %s", key)
	}
	o, found := this.offsets.Get(key)
	if !found || o.(message).Offset != offset {
		if this.debug {
			log.Info("out of order: %s", this)
		}
		return nil, inflights.ErrOutOfOrder
	}

	msg := o.(message).Value

	this.offsets.Remove(key)
	return msg, nil
}

// FIXME not atomic, add CAS
func (this *memInflights) TakeOff(cluster, topic, group, partition string, offset int64, msg []byte) error {
	key := this.key(cluster, topic, group, partition)
	if this.debug {
		log.Info("TakeOff %s", key)
	}
	o, found := this.offsets.Get(key)
	if found && o.(message).Offset != offset {
		if this.debug {
			log.Info("out of order: %s", this)
		}
		return inflights.ErrOutOfOrder
	}

	this.offsets.Set(key, message{
		Offset: offset,
		Value:  msg,
	})
	return nil
}

func (this *memInflights) TakenOff(cluster, topic, group, partition string, offset int64) bool {
	key := this.key(cluster, topic, group, partition)
	o, found := this.offsets.Get(key)
	if this.debug {
		log.Info("TakenOff %s", key)
	}
	if found && o.(message).Offset == offset {
		return true
	}

	return false
}

func (this *memInflights) String() string {
	dumps := make([]dumpRecord, 0, this.offsets.Count())
	for item := range this.offsets.Iter() {
		dumps = append(dumps, dumpRecord{
			Key: item.Key,
			Val: item.Val.(message),
		})
	}
	data, _ := json.Marshal(dumps)
	return string(data)
}

func (this *memInflights) Init() error {
	if this.snapshotFile == "" {
		return nil
	}

	data, err := ioutil.ReadFile(this.snapshotFile)
	if err != nil {
		return err
	}

	dumps := make([]dumpRecord, 0)
	if err = json.Unmarshal(data, &dumps); err != nil {
		return err
	}
	for _, record := range dumps {
		this.offsets.Set(record.Key, record.Val)
	}
	return nil
}

func (this *memInflights) Stop() error {
	if this.snapshotFile == "" {
		return nil
	}

	dumps := make([]dumpRecord, 0, this.offsets.Count())
	for item := range this.offsets.Iter() {
		dumps = append(dumps, dumpRecord{
			Key: item.Key,
			Val: item.Val.(message),
		})
	}
	data, err := json.Marshal(dumps)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(this.snapshotFile, data, 0644)
}
<commit_msg>debug info add offset<commit_after>package mem

import (
	"encoding/json"
	"fmt"
	"io/ioutil"

	"github.com/funkygao/gafka/cmd/kateway/inflights"
	"github.com/funkygao/golib/cmap"
	log "github.com/funkygao/log4go"
)

type dumpRecord struct {
	Key string
	Val message
}

type message struct {
	Offset int64
	Value  []byte
}

type memInflights struct {
	offsets cmap.ConcurrentMap

	snapshotFile string
	debug        bool
}

func New(fn string, debug bool) *memInflights {
	return &memInflights{
		offsets:      cmap.New(),
		snapshotFile: fn,
		debug:        debug,
	}
}

// FIXME bad perf
func (this *memInflights) key(cluster, topic, group, partition string) string {
	return fmt.Sprintf("%s:%s:%s:%s", cluster, topic, group, partition)
}

func (this *memInflights) Land(cluster, topic, group, partition string, offset int64) error {
	key := this.key(cluster, topic, group, partition)
	if this.debug {
		log.Info("Land %s => %d", key, offset)
	}
	o, found := this.offsets.Get(key)
	if !found || o.(message).Offset != offset {
		if this.debug {
			log.Info("out of order: %s", this)
		}
		return inflights.ErrOutOfOrder
	}

	this.offsets.Remove(key)
	return nil
}

func (this *memInflights) LandX(cluster, topic, group, partition string, offset int64) ([]byte, error) {
	key := this.key(cluster, topic, group, partition)
	if this.debug {
		log.Info("LandX %s => %d", key, offset)
	}
	o, found := this.offsets.Get(key)
	if !found || o.(message).Offset != offset {
		if this.debug {
			log.Info("out of order: %s", this)
		}
		return nil, inflights.ErrOutOfOrder
	}

	msg := o.(message).Value

	this.offsets.Remove(key)
	return msg, nil
}

// FIXME not atomic, add CAS
func (this *memInflights) TakeOff(cluster, topic, group, partition string, offset int64, msg []byte) error {
	key := this.key(cluster, topic, group, partition)
	if this.debug {
		log.Info("TakeOff %s => %d", key, offset)
	}
	o, found := this.offsets.Get(key)
	if found && o.(message).Offset != offset {
		if this.debug {
			log.Info("out of order: %s", this)
		}
		return inflights.ErrOutOfOrder
	}

	this.offsets.Set(key, message{
		Offset: offset,
		Value:  msg,
	})
	return nil
}

func (this *memInflights) TakenOff(cluster, topic, group, partition string, offset int64) bool {
	key := this.key(cluster, topic, group, partition)
	o, found := this.offsets.Get(key)
	if this.debug {
		log.Info("TakenOff %s => %d", key, offset)
	}
	if found && o.(message).Offset == offset {
		return true
	}

	return false
}

func (this *memInflights) String() string {
	dumps := make([]dumpRecord, 0, this.offsets.Count())
	for item := range this.offsets.Iter() {
		dumps = append(dumps, dumpRecord{
			Key: item.Key,
			Val: item.Val.(message),
		})
	}
	data, _ := json.Marshal(dumps)
	return string(data)
}

func (this *memInflights) Init() error {
	if this.snapshotFile == "" {
		return nil
	}

	data, err := ioutil.ReadFile(this.snapshotFile)
	if err != nil {
		return err
	}

	dumps := make([]dumpRecord, 0)
	if err = json.Unmarshal(data, &dumps); err != nil {
		return err
	}
	for _, record := range dumps {
		this.offsets.Set(record.Key, record.Val)
	}
	return nil
}

func (this *memInflights) Stop() error {
	if this.snapshotFile == "" {
		return nil
	}

	dumps := make([]dumpRecord, 0, this.offsets.Count())
	for item := range this.offsets.Iter() {
		dumps = append(dumps, dumpRecord{
			Key: item.Key,
			Val: item.Val.(message),
		})
	}
	data, err := json.Marshal(dumps)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(this.snapshotFile, data, 0644)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"
	"os"
	"strconv"
	"time"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/kops/cmd/kops/util"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/featureflag"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kops/upup/pkg/kutil"
	"k8s.io/kops/util/pkg/tables"
)

// Command Object for a Rolling Update.
type RollingUpdateOptions struct {
	Yes       bool
	Force     bool
	CloudOnly bool

	// The following two variables are when kops is validating a cluster
	// during a rolling update.

	// FailOnDrainError fail rolling-update if drain errors.
	FailOnDrainError bool

	// FailOnValidate fail the cluster rolling-update when the cluster
	// does not validate, after a validation period.
	FailOnValidate bool

	DrainInterval time.Duration

	ValidateRetries int

	MasterInterval  time.Duration
	NodeInterval    time.Duration
	BastionInterval time.Duration

	ClusterName string

	// InstanceGroups is the list of instance groups to rolling-update;
	// if not specified all instance groups will be updated
	InstanceGroups []string
}

func (o *RollingUpdateOptions) InitDefaults() {
	o.Yes = false
	o.Force = false
	o.CloudOnly = false
	o.FailOnDrainError = false
	o.FailOnValidate = true

	o.MasterInterval = 5 * time.Minute
	o.NodeInterval = 2 * time.Minute
	o.BastionInterval = 5 * time.Minute

	o.ValidateRetries = 8

	o.DrainInterval = 90 * time.Second

}

func NewCmdRollingUpdateCluster(f *util.Factory, out io.Writer) *cobra.Command {

	var options RollingUpdateOptions
	options.InitDefaults()

	cmd := &cobra.Command{
		Use:   "cluster",
		Short: "Rolling update a cluster",
		Long: `Rolling update a cluster instance groups.

This command updates a kubernetes cluseter to match the cloud, and kops specifications.

To perform rolling update, you need to update the cloud resources first with "kops update cluster"

Use KOPS_FEATURE_FLAGS="+DrainAndValidateRollingUpdate" to use beta code that drains the nodes
and validates the cluser.  New flags for Drain and Validation operations will be shown when
the environment variable is set.`,
	}

	cmd.Flags().BoolVar(&options.Yes, "yes", options.Yes, "perform rolling update without confirmation")
	cmd.Flags().BoolVar(&options.Force, "force", options.Force, "Force rolling update, even if no changes")
	cmd.Flags().BoolVar(&options.CloudOnly, "cloudonly", options.CloudOnly, "Perform rolling update without confirming progress with k8s")

	cmd.Flags().DurationVar(&options.MasterInterval, "master-interval", options.MasterInterval, "Time to wait between restarting masters")
	cmd.Flags().DurationVar(&options.NodeInterval, "node-interval", options.NodeInterval, "Time to wait between restarting nodes")
	cmd.Flags().DurationVar(&options.BastionInterval, "bastion-interval", options.BastionInterval, "Time to wait between restarting bastions")
	cmd.Flags().StringSliceVar(&options.InstanceGroups, "instance-group", options.InstanceGroups, "List of instance groups to update (defaults to all if not specified)")

	if featureflag.DrainAndValidateRollingUpdate.Enabled() {
		cmd.Flags().BoolVar(&options.FailOnDrainError, "fail-on-drain-error", true, "The rolling-update will fail if draining a node fails.")
		cmd.Flags().BoolVar(&options.FailOnValidate, "fail-on-validate-error", true, "The rolling-update will fail if the cluster fails to validate.")
		cmd.Flags().IntVar(&options.ValidateRetries, "validate-retries", options.ValidateRetries, "The number of times that a node will be validated.  Between validation kops sleeps the master-interval/2 or node-interval/2 duration.")
		cmd.Flags().DurationVar(&options.DrainInterval, "drain-interval", options.DrainInterval, "The duration that a rolling-update will wait after the node is drained.")
	}

	cmd.Run = func(cmd *cobra.Command, args []string) {
		err := rootCommand.ProcessArgs(args)
		if err != nil {
			exitWithError(err)
			return
		}

		clusterName := rootCommand.ClusterName()
		if clusterName == "" {
			exitWithError(fmt.Errorf("--name is required"))
			return
		}

		options.ClusterName = clusterName

		err = RunRollingUpdateCluster(f, os.Stdout, &options)
		if err != nil {
			exitWithError(err)
			return
		}

	}

	return cmd
}

func RunRollingUpdateCluster(f *util.Factory, out io.Writer, options *RollingUpdateOptions) error {

	clientset, err := f.Clientset()
	if err != nil {
		return err
	}

	cluster, err := GetCluster(f, options.ClusterName)
	if err != nil {
		return err
	}

	contextName := cluster.ObjectMeta.Name
	config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		clientcmd.NewDefaultClientConfigLoadingRules(),
		&clientcmd.ConfigOverrides{CurrentContext: contextName}).ClientConfig()
	if err != nil {
		return fmt.Errorf("cannot load kubecfg settings for %q: %v", contextName, err)
	}

	if options.ValidateRetries <= 0 {
		return fmt.Errorf("validate-retries flag cannot be 0 or smaller")
	}

	var nodes []v1.Node
	var k8sClient kubernetes.Interface
	if !options.CloudOnly {
		k8sClient, err = kubernetes.NewForConfig(config)
		if err != nil {
			return fmt.Errorf("cannot build kube client for %q: %v", contextName, err)
		}

		nodeList, err := k8sClient.CoreV1().Nodes().List(metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(os.Stderr, "Unable to reach the kubernetes API.\n")
			fmt.Fprintf(os.Stderr, "Use --cloudonly to do a rolling-update without confirming progress with the k8s API\n\n")
			return fmt.Errorf("error listing nodes in cluster: %v", err)
		}

		if nodeList != nil {
			nodes = nodeList.Items
		}
	}

	list, err := clientset.InstanceGroups(cluster.ObjectMeta.Name).List(metav1.ListOptions{})
	if err != nil {
		return err
	}

	var instanceGroups []*api.InstanceGroup
	for i := range list.Items {
		instanceGroups = append(instanceGroups, &list.Items[i])
	}

	if len(options.InstanceGroups) != 0 {
		var filtered []*api.InstanceGroup

		for _, instanceGroupName := range options.InstanceGroups {
			var found *api.InstanceGroup
			for _, ig := range instanceGroups {
				if ig.ObjectMeta.Name == instanceGroupName {
					found = ig
					break
				}
			}
			if found == nil {
				return fmt.Errorf("InstanceGroup %q not found", instanceGroupName)
			}

			filtered = append(filtered, found)
		}

		instanceGroups = filtered
	}

	cloud, err := cloudup.BuildCloud(cluster)
	if err != nil {
		return err
	}

	warnUnmatched := true
	groups, err := kutil.FindCloudInstanceGroups(cloud, cluster, instanceGroups, warnUnmatched, nodes)
	if err != nil {
		return err
	}

	{
		t := &tables.Table{}
		t.AddColumn("NAME", func(r *kutil.CloudInstanceGroup) string {
			return r.InstanceGroup.ObjectMeta.Name
		})
		t.AddColumn("STATUS", func(r *kutil.CloudInstanceGroup) string {
			return r.Status
		})
		t.AddColumn("NEEDUPDATE", func(r *kutil.CloudInstanceGroup) string {
			return strconv.Itoa(len(r.NeedUpdate))
		})
		t.AddColumn("READY", func(r *kutil.CloudInstanceGroup) string {
			return strconv.Itoa(len(r.Ready))
		})
		t.AddColumn("MIN", func(r *kutil.CloudInstanceGroup) string {
			return strconv.Itoa(r.MinSize())
		})
		t.AddColumn("MAX", func(r *kutil.CloudInstanceGroup) string {
			return strconv.Itoa(r.MaxSize())
		})
		t.AddColumn("NODES", func(r *kutil.CloudInstanceGroup) string {
			var nodes []*v1.Node
			for _, i := range r.Ready {
				if i.Node != nil {
					nodes = append(nodes, i.Node)
				}
			}
			for _, i := range r.NeedUpdate {
				if i.Node != nil {
					nodes = append(nodes, i.Node)
				}
			}
			return strconv.Itoa(len(nodes))
		})
		var l []*kutil.CloudInstanceGroup
		for _, v := range groups {
			l = append(l, v)
		}

		columns := []string{"NAME", "STATUS", "NEEDUPDATE", "READY", "MIN", "MAX"}
		if !options.CloudOnly {
			columns = append(columns, "NODES")
		}
		err := t.Render(l, out, columns...)
		if err != nil {
			return err
		}
	}

	needUpdate := false
	for _, group := range groups {
		if len(group.NeedUpdate) != 0 {
			needUpdate = true
		}
	}

	if !needUpdate && !options.Force {
		fmt.Printf("\nNo rolling-update required.\n")
		return nil
	}

	if !options.Yes {
		fmt.Printf("\nMust specify --yes to rolling-update.\n")
		return nil
	}

	if featureflag.DrainAndValidateRollingUpdate.Enabled() {
		glog.V(2).Infof("New rolling update with drain and validate enabled.")
	}
	d := &kutil.RollingUpdateCluster{
		MasterInterval:   options.MasterInterval,
		NodeInterval:     options.NodeInterval,
		Force:            options.Force,
		Cloud:            cloud,
		K8sClient:        k8sClient,
		ClientConfig:     kutil.NewClientConfig(config, "kube-system"),
		FailOnDrainError: options.FailOnDrainError,
		FailOnValidate:   options.FailOnValidate,
		CloudOnly:        options.CloudOnly,
		ClusterName:      options.ClusterName,
		ValidateRetries:  options.ValidateRetries,
		DrainInterval:    options.DrainInterval,
	}
	return d.RollingUpdate(groups, list)
}
<commit_msg>typo on rollingupdatecluster<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"
	"os"
	"strconv"
	"time"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/kops/cmd/kops/util"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/featureflag"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kops/upup/pkg/kutil"
	"k8s.io/kops/util/pkg/tables"
)

// Command Object for a Rolling Update.
type RollingUpdateOptions struct {
	Yes       bool
	Force     bool
	CloudOnly bool

	// The following two variables are when kops is validating a cluster
	// during a rolling update.

	// FailOnDrainError fail rolling-update if drain errors.
	FailOnDrainError bool

	// FailOnValidate fail the cluster rolling-update when the cluster
	// does not validate, after a validation period.
	FailOnValidate bool

	DrainInterval time.Duration

	ValidateRetries int

	MasterInterval  time.Duration
	NodeInterval    time.Duration
	BastionInterval time.Duration

	ClusterName string

	// InstanceGroups is the list of instance groups to rolling-update;
	// if not specified all instance groups will be updated
	InstanceGroups []string
}

func (o *RollingUpdateOptions) InitDefaults() {
	o.Yes = false
	o.Force = false
	o.CloudOnly = false
	o.FailOnDrainError = false
	o.FailOnValidate = true

	o.MasterInterval = 5 * time.Minute
	o.NodeInterval = 2 * time.Minute
	o.BastionInterval = 5 * time.Minute

	o.ValidateRetries = 8

	o.DrainInterval = 90 * time.Second

}

func NewCmdRollingUpdateCluster(f *util.Factory, out io.Writer) *cobra.Command {

	var options RollingUpdateOptions
	options.InitDefaults()

	cmd := &cobra.Command{
		Use:   "cluster",
		Short: "Rolling update a cluster",
		Long: `Rolling update a cluster instance groups.

This command updates a kubernetes cluster to match the cloud, and kops specifications.

To perform rolling update, you need to update the cloud resources first with "kops update cluster"

Use KOPS_FEATURE_FLAGS="+DrainAndValidateRollingUpdate" to use beta code that drains the nodes
and validates the cluser.  New flags for Drain and Validation operations will be shown when
the environment variable is set.`,
	}

	cmd.Flags().BoolVar(&options.Yes, "yes", options.Yes, "perform rolling update without confirmation")
	cmd.Flags().BoolVar(&options.Force, "force", options.Force, "Force rolling update, even if no changes")
	cmd.Flags().BoolVar(&options.CloudOnly, "cloudonly", options.CloudOnly, "Perform rolling update without confirming progress with k8s")

	cmd.Flags().DurationVar(&options.MasterInterval, "master-interval", options.MasterInterval, "Time to wait between restarting masters")
	cmd.Flags().DurationVar(&options.NodeInterval, "node-interval", options.NodeInterval, "Time to wait between restarting nodes")
	cmd.Flags().DurationVar(&options.BastionInterval, "bastion-interval", options.BastionInterval, "Time to wait between restarting bastions")
	cmd.Flags().StringSliceVar(&options.InstanceGroups, "instance-group", options.InstanceGroups, "List of instance groups to update (defaults to all if not specified)")

	if featureflag.DrainAndValidateRollingUpdate.Enabled() {
		cmd.Flags().BoolVar(&options.FailOnDrainError, "fail-on-drain-error", true, "The rolling-update will fail if draining a node fails.")
		cmd.Flags().BoolVar(&options.FailOnValidate, "fail-on-validate-error", true, "The rolling-update will fail if the cluster fails to validate.")
		cmd.Flags().IntVar(&options.ValidateRetries, "validate-retries", options.ValidateRetries, "The number of times that a node will be validated.  Between validation kops sleeps the master-interval/2 or node-interval/2 duration.")
		cmd.Flags().DurationVar(&options.DrainInterval, "drain-interval", options.DrainInterval, "The duration that a rolling-update will wait after the node is drained.")
	}

	cmd.Run = func(cmd *cobra.Command, args []string) {
		err := rootCommand.ProcessArgs(args)
		if err != nil {
			exitWithError(err)
			return
		}

		clusterName := rootCommand.ClusterName()
		if clusterName == "" {
			exitWithError(fmt.Errorf("--name is required"))
			return
		}

		options.ClusterName = clusterName

		err = RunRollingUpdateCluster(f, os.Stdout, &options)
		if err != nil {
			exitWithError(err)
			return
		}

	}

	return cmd
}

func RunRollingUpdateCluster(f *util.Factory, out io.Writer, options *RollingUpdateOptions) error {

	clientset, err := f.Clientset()
	if err != nil {
		return err
	}

	cluster, err := GetCluster(f, options.ClusterName)
	if err != nil {
		return err
	}

	contextName := cluster.ObjectMeta.Name
	config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
		clientcmd.NewDefaultClientConfigLoadingRules(),
		&clientcmd.ConfigOverrides{CurrentContext: contextName}).ClientConfig()
	if err != nil {
		return fmt.Errorf("cannot load kubecfg settings for %q: %v", contextName, err)
	}

	if options.ValidateRetries <= 0 {
		return fmt.Errorf("validate-retries flag cannot be 0 or smaller")
	}

	var nodes []v1.Node
	var k8sClient kubernetes.Interface
	if !options.CloudOnly {
		k8sClient, err = kubernetes.NewForConfig(config)
		if err != nil {
			return fmt.Errorf("cannot build kube client for %q: %v", contextName, err)
		}

		nodeList, err := k8sClient.CoreV1().Nodes().List(metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(os.Stderr, "Unable to reach the kubernetes API.\n")
			fmt.Fprintf(os.Stderr, "Use --cloudonly to do a rolling-update without confirming progress with the k8s API\n\n")
			return fmt.Errorf("error listing nodes in cluster: %v", err)
		}

		if nodeList != nil {
			nodes = nodeList.Items
		}
	}

	list, err := clientset.InstanceGroups(cluster.ObjectMeta.Name).List(metav1.ListOptions{})
	if err != nil {
		return err
	}

	var instanceGroups []*api.InstanceGroup
	for i := range list.Items {
		instanceGroups = append(instanceGroups, &list.Items[i])
	}

	if len(options.InstanceGroups) != 0 {
		var filtered []*api.InstanceGroup

		for _, instanceGroupName := range options.InstanceGroups {
			var found *api.InstanceGroup
			for _, ig := range instanceGroups {
				if ig.ObjectMeta.Name == instanceGroupName {
					found = ig
					break
				}
			}
			if found == nil {
				return fmt.Errorf("InstanceGroup %q not found", instanceGroupName)
			}

			filtered = append(filtered, found)
		}

		instanceGroups = filtered
	}

	cloud, err := cloudup.BuildCloud(cluster)
	if err != nil {
		return err
	}

	warnUnmatched := true
	groups, err := kutil.FindCloudInstanceGroups(cloud, cluster, instanceGroups, warnUnmatched, nodes)
	if err != nil {
		return err
	}

	{
		t := &tables.Table{}
		t.AddColumn("NAME", func(r *kutil.CloudInstanceGroup) string {
			return r.InstanceGroup.ObjectMeta.Name
		})
		t.AddColumn("STATUS", func(r *kutil.CloudInstanceGroup) string {
			return r.Status
		})
		t.AddColumn("NEEDUPDATE", func(r *kutil.CloudInstanceGroup) string {
			return strconv.Itoa(len(r.NeedUpdate))
		})
		t.AddColumn("READY", func(r *kutil.CloudInstanceGroup) string {
			return strconv.Itoa(len(r.Ready))
		})
		t.AddColumn("MIN", func(r *kutil.CloudInstanceGroup) string {
			return strconv.Itoa(r.MinSize())
		})
		t.AddColumn("MAX", func(r *kutil.CloudInstanceGroup) string {
			return strconv.Itoa(r.MaxSize())
		})
		t.AddColumn("NODES", func(r *kutil.CloudInstanceGroup) string {
			var nodes []*v1.Node
			for _, i := range r.Ready {
				if i.Node != nil {
					nodes = append(nodes, i.Node)
				}
			}
			for _, i := range r.NeedUpdate {
				if i.Node != nil {
					nodes = append(nodes, i.Node)
				}
			}
			return strconv.Itoa(len(nodes))
		})
		var l []*kutil.CloudInstanceGroup
		for _, v := range groups {
			l = append(l, v)
		}

		columns := []string{"NAME", "STATUS", "NEEDUPDATE", "READY", "MIN", "MAX"}
		if !options.CloudOnly {
			columns = append(columns, "NODES")
		}
		err := t.Render(l, out, columns...)
		if err != nil {
			return err
		}
	}

	needUpdate := false
	for _, group := range groups {
		if len(group.NeedUpdate) != 0 {
			needUpdate = true
		}
	}

	if !needUpdate && !options.Force {
		fmt.Printf("\nNo rolling-update required.\n")
		return nil
	}

	if !options.Yes {
		fmt.Printf("\nMust specify --yes to rolling-update.\n")
		return nil
	}

	if featureflag.DrainAndValidateRollingUpdate.Enabled() {
		glog.V(2).Infof("New rolling update with drain and validate enabled.")
	}
	d := &kutil.RollingUpdateCluster{
		MasterInterval:   options.MasterInterval,
		NodeInterval:     options.NodeInterval,
		Force:            options.Force,
		Cloud:            cloud,
		K8sClient:        k8sClient,
		ClientConfig:     kutil.NewClientConfig(config, "kube-system"),
		FailOnDrainError: options.FailOnDrainError,
		FailOnValidate:   options.FailOnValidate,
		CloudOnly:        options.CloudOnly,
		ClusterName:      options.ClusterName,
		ValidateRetries:  options.ValidateRetries,
		DrainInterval:    options.DrainInterval,
	}
	return d.RollingUpdate(groups, list)
}
<|endoftext|>


<commit_before>// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package cmd

import (
	"fmt"
	"io"
	"time"

	"sigs.k8s.io/kustomize/kstatus/status"
	"sigs.k8s.io/kustomize/kstatus/wait"
)

const (
	typeColumn      = "type"
	namespaceColumn = "namespace"
	nameColumn      = "name"
	statusColumn    = "status"
	messageColumn   = "message"

	ESC          = 27
	RED    color = 31
	GREEN  color = 32
	YELLOW color = 33
	WHITE  color = 37
)

type color int

func moveUp(w io.Writer, lineCount int) {
	printOrDie(w, "%c[%dA", ESC, lineCount)
}

func eraseCurrentLine(w io.Writer) {
	printOrDie(w, "%c[2K\r", ESC)
}

func setColor(w io.Writer, color color) {
	printOrDie(w, "%c[%dm", ESC, color)
}

type colorFunc func(s status.Status) color
type contentFunc func(resource ResourceStatusData) string

type tableColumnInfo struct {
	header      string
	width       int
	colorFunc   colorFunc
	contentFunc contentFunc
}

func defaultColorFunc(_ status.Status) color {
	return WHITE
}

var (
	tableColumns = map[string]tableColumnInfo{
		typeColumn: {
			header:    "TYPE",
			width:     25,
			colorFunc: defaultColorFunc,
			contentFunc: func(data ResourceStatusData) string {
				return fmt.Sprintf("%s/%s", data.Identifier.GroupKind.Group, data.Identifier.GroupKind.Kind)
			},
		},
		namespaceColumn: {
			header:    "NAMESPACE",
			width:     15,
			colorFunc: defaultColorFunc,
			contentFunc: func(data ResourceStatusData) string {
				return data.Identifier.Namespace
			},
		},
		nameColumn: {
			header:    "NAME",
			width:     20,
			colorFunc: defaultColorFunc,
			contentFunc: func(data ResourceStatusData) string {
				return data.Identifier.Name
			},
		},
		statusColumn: {
			header:    "STATUS",
			width:     10,
			colorFunc: colorForStatus,
			contentFunc: func(data ResourceStatusData) string {
				return data.Status.String()
			},
		},
		messageColumn: {
			header:    "MESSAGE",
			width:     40,
			colorFunc: defaultColorFunc,
			contentFunc: func(data ResourceStatusData) string {
				return data.Message
			},
		},
	}
	tableColumnOrder = []string{typeColumn, namespaceColumn, nameColumn, statusColumn, messageColumn}
)

type StatusInfo interface {
	CurrentStatus() StatusData
}

type StatusData struct {
	AggregateStatus  status.Status
	ResourceStatuses []ResourceStatusData
}

type ResourceStatusData struct {
	Identifier wait.ResourceIdentifier
	Status     status.Status
	Message    string
}

type TablePrinter struct {
	statusInfo    StatusInfo
	out           io.Writer
	err           io.Writer
	showAggStatus bool
}

func newTablePrinter(statusInfo StatusInfo, out io.Writer, err io.Writer, showAggStatus bool) *TablePrinter {
	return &TablePrinter{
		statusInfo:    statusInfo,
		out:           out,
		err:           err,
		showAggStatus: showAggStatus,
	}
}

func (s *TablePrinter) Print() {
	s.printTable(s.statusInfo.CurrentStatus(), false)
}

func (s *TablePrinter) PrintUntil(stop <-chan struct{}, interval time.Duration) <-chan struct{} {
	completed := make(chan struct{})
	setColor(s.out, WHITE)
	s.printTable(s.statusInfo.CurrentStatus(), false)
	go func() {
		defer close(completed)
		ticker := time.NewTicker(interval)
		for {
			select {
			case <-stop:
				ticker.Stop()
				s.printTable(s.statusInfo.CurrentStatus(), true)
				return
			case <-ticker.C:
				s.printTable(s.statusInfo.CurrentStatus(), true)
			}
		}
	}()
	return completed
}

func (s *TablePrinter) printTable(data StatusData, deleteUp bool) {
	if deleteUp {
		if s.showAggStatus {
			moveUp(s.out, 1)
		}
		moveUp(s.out, 1)
		moveUp(s.out, len(data.ResourceStatuses))
	}
	eraseCurrentLine(s.out)
	if s.showAggStatus {
		printOrDie(s.out, "AggregateStatus: ")
		setColor(s.out, colorForStatus(data.AggregateStatus))
		printOrDie(s.out, "%s\n", data.AggregateStatus)
		setColor(s.out, WHITE)
	}
	s.printTableRow(headers())
	for _, resource := range data.ResourceStatuses {
		s.printTableRow(row(resource))
	}
}

func (s *TablePrinter) printTableRow(rowData []RowData) {
	for i, row := range rowData {
		setColor(s.out, row.color)
		format := fmt.Sprintf("%%-%ds", row.width)
		printOrDie(s.out, format, trimString(row.content, row.width))
		if i != len(rowData)-1 {
			printOrDie(s.out, "  ")
		}
		setColor(s.out, WHITE)
	}
	printOrDie(s.out, "\n")
}

type RowData struct {
	content string
	color   color
	width   int
}

func headers() []RowData {
	var headers []RowData
	for _, columnName := range tableColumnOrder {
		column := tableColumns[columnName]
		headers = append(headers, RowData{
			content: column.header,
			color:   WHITE,
			width:   column.width,
		})
	}
	return headers
}

func row(resource ResourceStatusData) []RowData {
	var row []RowData
	for _, columnName := range tableColumnOrder {
		column := tableColumns[columnName]
		row = append(row, RowData{
			content: column.contentFunc(resource),
			color:   column.colorFunc(resource.Status),
			width:   column.width,
		})
	}
	return row
}

type eventContentFunc func(wait.Event) string

type eventColumnInfo struct {
	header                     string
	width                      int
	requireResourceUpdateEvent bool
	contentFunc                eventContentFunc
}

var (
	eventColumns = []eventColumnInfo{
		{
			header:                     "EVENT TYPE",
			width:                      15,
			requireResourceUpdateEvent: false,
			contentFunc: func(event wait.Event) string {
				return string(event.Type)
			},
		},
		{
			header:                     "AGG STATUS",
			width:                      10,
			requireResourceUpdateEvent: false,
			contentFunc: func(event wait.Event) string {
				return event.AggregateStatus.String()
			},
		},
		{
			header:                     "TYPE",
			width:                      20,
			requireResourceUpdateEvent: true,
			contentFunc: func(event wait.Event) string {
				return fmt.Sprintf("%s/%s", event.EventResource.ResourceIdentifier.GroupKind.Group,
					event.EventResource.ResourceIdentifier.GroupKind.Kind)
			},
		},
		{
			header:                     "NAMESPACE",
			width:                      15,
			requireResourceUpdateEvent: true,
			contentFunc: func(event wait.Event) string {
				return event.EventResource.ResourceIdentifier.Namespace
			},
		},
		{
			header:                     "NAME",
			width:                      20,
			requireResourceUpdateEvent: true,
			contentFunc: func(event wait.Event) string {
				return event.EventResource.ResourceIdentifier.Name
			},
		},
		{
			header:                     "STATUS",
			width:                      10,
			requireResourceUpdateEvent: true,
			contentFunc: func(event wait.Event) string {
				return event.EventResource.Status.String()
			},
		},
		{
			header:                     "MESSAGE",
			width:                      50,
			requireResourceUpdateEvent: true,
			contentFunc: func(event wait.Event) string {
				if event.EventResource.Error != nil {
					return event.EventResource.Error.Error()
				}
				return event.EventResource.Message
			},
		},
	}
)

type EventPrinter struct {
	out io.Writer
	err io.Writer
}

func newEventPrinter(out io.Writer, err io.Writer) *EventPrinter {
	for _, column := range eventColumns {
		format := fmt.Sprintf("%%-%ds  ", column.width)
		printOrDie(out, format, column.header)
	}
	printOrDie(out, "\n")
	return &EventPrinter{
		out: out,
		err: err,
	}
}

func (e *EventPrinter) printEvent(event wait.Event) {
	for _, column := range eventColumns {
		if event.Type != wait.ResourceUpdate && column.requireResourceUpdateEvent {
			continue
		}
		format := fmt.Sprintf("%%-%ds  ", column.width)
		printOrDie(e.out, format, trimString(column.contentFunc(event), column.width))
	}
	printOrDie(e.out, "\n")
}

func printOrDie(w io.Writer, format string, a ...interface{}) {
	_, err := fmt.Fprintf(w, format, a...)
	if err != nil {
		panic(err)
	}
}

func colorForStatus(s status.Status) color {
	switch s {
	case status.CurrentStatus:
		return GREEN
	case status.UnknownStatus:
		return WHITE
	case status.InProgressStatus:
		return YELLOW
	case status.FailedStatus:
		return RED
	}
	return WHITE
}

func trimString(str string, maxLength int) string {
	if len(str) <= maxLength {
		return str
	}
	return str[:maxLength]
}
<commit_msg>Fix printing to make sure always match setting colors with a reset<commit_after>// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package cmd

import (
	"fmt"
	"io"
	"time"

	"sigs.k8s.io/kustomize/kstatus/status"
	"sigs.k8s.io/kustomize/kstatus/wait"
)

const (
	typeColumn      = "type"
	namespaceColumn = "namespace"
	nameColumn      = "name"
	statusColumn    = "status"
	messageColumn   = "message"

	RESET         = 0
	ESC           = 27
	RED     color = 31
	GREEN   color = 32
	YELLOW  color = 33
	DEFAULT color = -1 // This is not a valid ANSI escape code. It is used here to mean that no color should be set.
)

type color int

func moveUp(w io.Writer, lineCount int) {
	printOrDie(w, "%c[%dA", ESC, lineCount)
}

func eraseCurrentLine(w io.Writer) {
	printOrDie(w, "%c[2K\r", ESC)
}

type colorFunc func(s status.Status) color
type contentFunc func(resource ResourceStatusData) string

type tableColumnInfo struct {
	header      string
	width       int
	colorFunc   colorFunc
	contentFunc contentFunc
}

func defaultColorFunc(_ status.Status) color {
	return DEFAULT
}

var (
	tableColumns = map[string]tableColumnInfo{
		typeColumn: {
			header:    "TYPE",
			width:     25,
			colorFunc: defaultColorFunc,
			contentFunc: func(data ResourceStatusData) string {
				return fmt.Sprintf("%s/%s", data.Identifier.GroupKind.Group, data.Identifier.GroupKind.Kind)
			},
		},
		namespaceColumn: {
			header:    "NAMESPACE",
			width:     15,
			colorFunc: defaultColorFunc,
			contentFunc: func(data ResourceStatusData) string {
				return data.Identifier.Namespace
			},
		},
		nameColumn: {
			header:    "NAME",
			width:     20,
			colorFunc: defaultColorFunc,
			contentFunc: func(data ResourceStatusData) string {
				return data.Identifier.Name
			},
		},
		statusColumn: {
			header:    "STATUS",
			width:     10,
			colorFunc: colorForStatus,
			contentFunc: func(data ResourceStatusData) string {
				return data.Status.String()
			},
		},
		messageColumn: {
			header:    "MESSAGE",
			width:     40,
			colorFunc: defaultColorFunc,
			contentFunc: func(data ResourceStatusData) string {
				return data.Message
			},
		},
	}
	tableColumnOrder = []string{typeColumn, namespaceColumn, nameColumn, statusColumn, messageColumn}
)

type StatusInfo interface {
	CurrentStatus() StatusData
}

type StatusData struct {
	AggregateStatus  status.Status
	ResourceStatuses []ResourceStatusData
}

type ResourceStatusData struct {
	Identifier wait.ResourceIdentifier
	Status     status.Status
	Message    string
}

type TablePrinter struct {
	statusInfo    StatusInfo
	out           io.Writer
	err           io.Writer
	showAggStatus bool
}

func newTablePrinter(statusInfo StatusInfo, out io.Writer, err io.Writer, showAggStatus bool) *TablePrinter {
	return &TablePrinter{
		statusInfo:    statusInfo,
		out:           out,
		err:           err,
		showAggStatus: showAggStatus,
	}
}

func (s *TablePrinter) Print() {
	s.printTable(s.statusInfo.CurrentStatus(), false)
}

func (s *TablePrinter) PrintUntil(stop <-chan struct{}, interval time.Duration) <-chan struct{} {
	completed := make(chan struct{})
	s.printTable(s.statusInfo.CurrentStatus(), false)
	go func() {
		defer close(completed)
		ticker := time.NewTicker(interval)
		for {
			select {
			case <-stop:
				ticker.Stop()
				s.printTable(s.statusInfo.CurrentStatus(), true)
				return
			case <-ticker.C:
				s.printTable(s.statusInfo.CurrentStatus(), true)
			}
		}
	}()
	return completed
}

func (s *TablePrinter) printTable(data StatusData, deleteUp bool) {
	if deleteUp {
		if s.showAggStatus {
			moveUp(s.out, 1)
		}
		moveUp(s.out, 1)
		moveUp(s.out, len(data.ResourceStatuses))
	}
	eraseCurrentLine(s.out)
	if s.showAggStatus {
		printOrDie(s.out, "AggregateStatus: ")
		printWithColorOrDie(s.out, colorForStatus(data.AggregateStatus), "%s\n", data.AggregateStatus)
	}
	s.printTableRow(headers())
	for _, resource := range data.ResourceStatuses {
		s.printTableRow(row(resource))
	}
}

func (s *TablePrinter) printTableRow(rowData []RowData) {
	for i, row := range rowData {

		format := fmt.Sprintf("%%-%ds", row.width)
		printWithColorOrDie(s.out, row.color, format, trimString(row.content, row.width))
		if i != len(rowData)-1 {
			printOrDie(s.out, "  ")
		}
	}
	printOrDie(s.out, "\n")
}

type RowData struct {
	content string
	color   color
	width   int
}

func headers() []RowData {
	var headers []RowData
	for _, columnName := range tableColumnOrder {
		column := tableColumns[columnName]
		headers = append(headers, RowData{
			content: column.header,
			color:   DEFAULT,
			width:   column.width,
		})
	}
	return headers
}

func row(resource ResourceStatusData) []RowData {
	var row []RowData
	for _, columnName := range tableColumnOrder {
		column := tableColumns[columnName]
		row = append(row, RowData{
			content: column.contentFunc(resource),
			color:   column.colorFunc(resource.Status),
			width:   column.width,
		})
	}
	return row
}

type eventContentFunc func(wait.Event) string

type eventColumnInfo struct {
	header                     string
	width                      int
	requireResourceUpdateEvent bool
	contentFunc                eventContentFunc
}

var (
	eventColumns = []eventColumnInfo{
		{
			header:                     "EVENT TYPE",
			width:                      15,
			requireResourceUpdateEvent: false,
			contentFunc: func(event wait.Event) string {
				return string(event.Type)
			},
		},
		{
			header:                     "AGG STATUS",
			width:                      10,
			requireResourceUpdateEvent: false,
			contentFunc: func(event wait.Event) string {
				return event.AggregateStatus.String()
			},
		},
		{
			header:                     "TYPE",
			width:                      20,
			requireResourceUpdateEvent: true,
			contentFunc: func(event wait.Event) string {
				return fmt.Sprintf("%s/%s", event.EventResource.ResourceIdentifier.GroupKind.Group,
					event.EventResource.ResourceIdentifier.GroupKind.Kind)
			},
		},
		{
			header:                     "NAMESPACE",
			width:                      15,
			requireResourceUpdateEvent: true,
			contentFunc: func(event wait.Event) string {
				return event.EventResource.ResourceIdentifier.Namespace
			},
		},
		{
			header:                     "NAME",
			width:                      20,
			requireResourceUpdateEvent: true,
			contentFunc: func(event wait.Event) string {
				return event.EventResource.ResourceIdentifier.Name
			},
		},
		{
			header:                     "STATUS",
			width:                      10,
			requireResourceUpdateEvent: true,
			contentFunc: func(event wait.Event) string {
				return event.EventResource.Status.String()
			},
		},
		{
			header:                     "MESSAGE",
			width:                      50,
			requireResourceUpdateEvent: true,
			contentFunc: func(event wait.Event) string {
				if event.EventResource.Error != nil {
					return event.EventResource.Error.Error()
				}
				return event.EventResource.Message
			},
		},
	}
)

type EventPrinter struct {
	out io.Writer
	err io.Writer
}

func newEventPrinter(out io.Writer, err io.Writer) *EventPrinter {
	for _, column := range eventColumns {
		format := fmt.Sprintf("%%-%ds  ", column.width)
		printOrDie(out, format, column.header)
	}
	printOrDie(out, "\n")
	return &EventPrinter{
		out: out,
		err: err,
	}
}

func (e *EventPrinter) printEvent(event wait.Event) {
	for _, column := range eventColumns {
		if event.Type != wait.ResourceUpdate && column.requireResourceUpdateEvent {
			continue
		}
		format := fmt.Sprintf("%%-%ds  ", column.width)
		printOrDie(e.out, format, trimString(column.contentFunc(event), column.width))
	}
	printOrDie(e.out, "\n")
}

func printOrDie(w io.Writer, format string, a ...interface{}) {
	_, err := fmt.Fprintf(w, format, a...)
	if err != nil {
		panic(err)
	}
}

func printWithColorOrDie(w io.Writer, color color, format string, a ...interface{}) {
	if color == DEFAULT {
		printOrDie(w, format, a...)
	} else {
		printOrDie(w, "%c[%dm", ESC, color)
		printOrDie(w, format, a...)
		printOrDie(w, "%c[%dm", ESC, RESET)
	}
}

func colorForStatus(s status.Status) color {
	switch s {
	case status.CurrentStatus:
		return GREEN
	case status.InProgressStatus:
		return YELLOW
	case status.FailedStatus:
		return RED
	}
	return DEFAULT
}

func trimString(str string, maxLength int) string {
	if len(str) <= maxLength {
		return str
	}
	return str[:maxLength]
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"reflect"
	"time"

	"github.com/jessevdk/go-flags"
	"github.com/lestrrat/go-server-starter"
)

type options struct {
	OptArgs         []string
	OptCommand      string
	OptDir          string
	OptInterval     time.Duration // Time between checks for liveness
	OptPidFile      string
	OptPorts        []int     `long:"port" description:"TCP port to listen to (if omitted, will not bind to any ports)"`
	OptPaths        []string  // Paths (UNIX domain socket) to bind to
	OptSignalOnHUP  os.Signal // Signal to send when HUP is received
	OptSignalOnTERM os.Signal // Signal to send when TERM is received
	OptStatusFile   string
}

func (o options) Args() []string          { return o.OptArgs }
func (o options) Command() string         { return o.OptCommand }
func (o options) Dir() string             { return o.OptDir }
func (o options) Interval() time.Duration { return o.OptInterval }
func (o options) PidFile() string         { return o.OptPidFile }
func (o options) Ports() []int            { return o.OptPorts }
func (o options) Paths() []string         { return o.OptPaths }
func (o options) SignalOnHUP() os.Signal  { return o.OptSignalOnHUP }
func (o options) SignalOnTERM() os.Signal { return o.OptSignalOnTERM }
func (o options) StatusFile() string      { return o.OptStatusFile }

func showHelp() {
	// The ONLY reason we're not using go-flags' help option is
	// because I wanted to tweak the format just a bit... but
	// there wasn't an easy way to do so
	os.Stderr.WriteString(`
Usage: start_server [options] cmd [args...]

Options:
`)

	t := reflect.TypeOf(options{})
	for i := 0; i < t.NumField(); i++ {
		tag := t.Field(i).Tag
		if tag == "" {
			continue
		}
		var o string
		if s := tag.Get("short"); s != "" {
			o = fmt.Sprintf("-%s, --%s", tag.Get("short"), tag.Get("long"))
		} else {
			o = fmt.Sprintf("--%s", tag.Get("long"))
		}

		fmt.Fprintf(
			os.Stderr,
			"  %-21s %s\n",
			o,
			tag.Get("description"),
		)
	}
}

func main() {
	os.Exit(_main())
}

func _main() (st int) {
	st = 1

	opts := &options{OptInterval: -1}
	p := flags.NewParser(opts, flags.PrintErrors|flags.PassDoubleDash)
	args, err := p.Parse()
	if err != nil {
		showHelp()
		return
	}
	if opts.OptInterval < 0 {
		opts.OptInterval = time.Second
	}
	opts.OptCommand = args[0]
	if len(args) > 1 {
		opts.OptArgs = args[1:]
	}

	s, err := starter.NewStarter(opts)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err)
		return
	}
	s.Run()
	st = 0
	return
}<commit_msg>More parameter stuff<commit_after>package main

import (
	"fmt"
	"os"
	"reflect"
	"strings"
	"time"

	"github.com/jessevdk/go-flags"
	"github.com/lestrrat/go-server-starter"
)

type options struct {
	OptArgs         []string
	OptCommand      string
	OptDir          string    `long:"dir" arg:"path" description:"working directory, start_server do chdir to before exec (optional)"`
	OptInterval     int       `long:"interval" arg:"seconds" description:"minimum interval (in seconds) to respawn the server program (default: 1)"`
	OptPidFile      string    `long:"pid-file" arg:"filename" description:"if set, writes the process id of the start_server process to the file"`
	OptPorts        []int     `long:"port" arg:"(port|host:port)" description:"TCP port to listen to (if omitted, will not bind to any ports)"`
	OptPaths        []string  `long:"path" arg:"path" description:"path at where to listen using unix socket (optional)"`
	OptSignalOnHUP  os.Signal `long:"signal-on-hup" arg:"Signal" description:"name of the signal to be sent to the server process when start_server\nreceives a SIGHUP (default: SIGTERM). If you use this option, be sure to\nalso use '--signal-on-term' below."`
	OptSignalOnTERM os.Signal `long:"signal-on-term" arg:"Signal" description:"name of the signal to be sent to the server process when start_server\nreceives a SIGTERM (default: SIGTERM)"`
	OptStatusFile   string
}

func (o options) Args() []string          { return o.OptArgs }
func (o options) Command() string         { return o.OptCommand }
func (o options) Dir() string             { return o.OptDir }
func (o options) Interval() time.Duration { return time.Duration(o.OptInterval) * time.Second }
func (o options) PidFile() string         { return o.OptPidFile }
func (o options) Ports() []int            { return o.OptPorts }
func (o options) Paths() []string         { return o.OptPaths }
func (o options) SignalOnHUP() os.Signal  { return o.OptSignalOnHUP }
func (o options) SignalOnTERM() os.Signal { return o.OptSignalOnTERM }
func (o options) StatusFile() string      { return o.OptStatusFile }

func showHelp() {
	// The ONLY reason we're not using go-flags' help option is
	// because I wanted to tweak the format just a bit... but
	// there wasn't an easy way to do so
	os.Stderr.WriteString(`
Usage:
      start_server [options] -- server-prog server-arg1 server-arg2 ...

      # start Plack using Starlet listening at TCP port 8000
      start_server --port=8000 -- plackup -s Starlet --max-workers=100 index.psgi

Options:
`)

	t := reflect.TypeOf(options{})
	for i := 0; i < t.NumField(); i++ {
		tag := t.Field(i).Tag
		if tag == "" {
			continue
		}
		if s := tag.Get("long"); s != "" {
			fmt.Fprintf(os.Stderr, "  --%s", s)
			if a := tag.Get("arg"); a != "" {
				fmt.Fprintf(os.Stderr, "=%s", a)
			}
			fmt.Fprintf(os.Stderr, ":\n")
		}
		for _, l := range strings.Split(tag.Get("description"), "\n") {
			fmt.Fprintf(os.Stderr, "    %s\n", l)
		}
		fmt.Fprintf(os.Stderr, "\n")
	}
}

func main() {
	os.Exit(_main())
}

func _main() (st int) {
	st = 1

	opts := &options{OptInterval: -1}
	p := flags.NewParser(opts, flags.PrintErrors|flags.PassDoubleDash)
	args, err := p.Parse()
	if err != nil {
		showHelp()
		return
	}
	if opts.OptInterval < 0 {
		opts.OptInterval = 1
	}
	opts.OptCommand = args[0]
	if len(args) > 1 {
		opts.OptArgs = args[1:]
	}

	s, err := starter.NewStarter(opts)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err)
		return
	}
	s.Run()
	st = 0
	return
}<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"github.com/texttheater/golang-levenshtein/levenshtein"
	"gopkg.in/jdkato/prose.v2"
	"log"
	"math"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

// TEI file mappings
type Entry struct {
	Original   string   `xml:"form>orth"`
	Translated []string `xml:"sense>cit>quote"`
}

var LangFrom string
var LangTo string
var Mode string

var dictMap = make(map[string][]string)

var translateFunc func(langFrom, langTo, text string, maxAlt int) string = TranslateText

func InitDictionary(langFrom, langTo, path string) {
	LangFrom = langFrom
	LangTo = langTo

	filePath, error := filepath.Abs(path)
	if error != nil {
		fmt.Println(error)
		os.Exit(1)
	}

	xmlFile, error := os.Open(filePath)
	if error != nil {
		fmt.Println(error)
		os.Exit(1)
	}
	defer xmlFile.Close()

	decoder := xml.NewDecoder(xmlFile)

	for {
		token, _ := decoder.Token()
		if token == nil {
			break
		}

		switch startElement := token.(type) {
		case xml.StartElement:
			if startElement.Name.Local == "entry" {
				var e Entry
				decoder.DecodeElement(&e, &startElement)
				// put pair into the map
				dictMap[strings.ToLower(e.Original)] = e.Translated
			}
		}
	}
}

func IsSupported(from, to string) bool {
	return LangFrom == from && LangTo == to
}

func TranslateTextWithParse(langFrom, langTo, text string, maxAlt int) string {
	doc, err := prose.NewDocument(text)
	if err != nil {
		log.Fatal(err)
		return "Sorry, cannot build a new document from text " + text
	}

	var res bytes.Buffer
	for _, tok := range doc.Tokens() {
		nonPunctuation := regexp.MustCompile("^[^A-Z]+$")
		if !nonPunctuation.MatchString(tok.Tag) {
			var word, _ = findByMinDist(strings.ToLower(tok.Text))

			if dictMap[word] != nil {
				res.WriteString(translationWords(dictMap[word], maxAlt) + " ")
				continue
			}
		}

		res.WriteString(tok.Text + " ")
	}

	return res.String()
}

// TODO take punctuation into account
func TranslateText(langFrom, langTo, text string, maxAlt int) string {
	var res bytes.Buffer

	// split text into words
	words := strings.Fields(text)

	for i := 0; i < len(words); i++ {
		word := strings.ToLower(words[i])
		foundWord, distWord := findByMinDist(word)

		if i+1 < len(words) {
			nextWord := strings.ToLower(words[i+1])
			foundPair, distPair := findByMinDist(word + " " + nextWord)

			// pair has better result than a single word?
			if distPair < distWord {
				res.WriteString(translationWords(dictMap[foundPair], maxAlt) + " ")
				i += 2
				continue
			}
		}

		if foundWord == "" {
			res.WriteString(word + " ")
		} else {
			res.WriteString(translationWords(dictMap[foundWord], maxAlt) + " ")
		}

	}

	return res.String()
}

func translationWords(val []string, maxAlt int) string {
	var buf bytes.Buffer

	for i, _ := range val {
		if i > maxAlt {
			break
		}

		if i == 0 {
			buf.WriteString(val[i])
		} else {
			buf.WriteString("[alt:" + val[i] + "]")
		}
	}

	return buf.String()
}

func findByMinDist(word string) (string, int) {
	minDist := math.MaxInt64
	result := ""

	for k := range dictMap {
		// limit key set in order to limit number of distance calculations
		if k[0] != word[0] {
			continue
		}

		options := levenshtein.Options{
			InsCost: 1,
			DelCost: 1,
			SubCost: 1,
			Matches: func(sourceCharacter rune, targetCharacter rune) bool {
				return sourceCharacter == targetCharacter
			},
		}
		dist := levenshtein.DistanceForStrings([]rune(word), []rune(k), options)

		if dist < minDist {
			minDist = dist
			result = k
		}

	}

	return result, minDist
}
<commit_msg>Mod: try to match sequences instead of single words in prose mode.<commit_after>package main

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"github.com/texttheater/golang-levenshtein/levenshtein"
	"gopkg.in/jdkato/prose.v2"
	"log"
	"math"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

// TEI file mappings
type Entry struct {
	Original   string   `xml:"form>orth"`
	Translated []string `xml:"sense>cit>quote"`
}

var LangFrom string
var LangTo string
var Mode string

var dictMap = make(map[string][]string)

var translateFunc = TranslateText

func InitDictionary(langFrom, langTo, path string) {
	LangFrom = langFrom
	LangTo = langTo

	filePath, error := filepath.Abs(path)
	if error != nil {
		fmt.Println(error)
		os.Exit(1)
	}

	xmlFile, error := os.Open(filePath)
	if error != nil {
		fmt.Println(error)
		os.Exit(1)
	}
	defer xmlFile.Close()

	decoder := xml.NewDecoder(xmlFile)

	for {
		token, _ := decoder.Token()
		if token == nil {
			break
		}

		switch startElement := token.(type) {
		case xml.StartElement:
			if startElement.Name.Local == "entry" {
				var e Entry
				decoder.DecodeElement(&e, &startElement)
				// put pair into the map
				dictMap[strings.ToLower(e.Original)] = e.Translated
			}
		}
	}
}

func IsSupported(from, to string) bool {
	return LangFrom == from && LangTo == to
}

func TranslateTextWithParse(langFrom, langTo, text string, maxAlt int) string {
	doc, err := prose.NewDocument(text)
	if err != nil {
		log.Fatal(err)
		return "Sorry, cannot build a new document from text " + text
	}

	var res bytes.Buffer

	for _, seq := range splitToSequences(doc.Tokens()) {
		if dictMap[seq] != nil {
			res.WriteString(translationWords(dictMap[seq], maxAlt) + " ")
			continue
		}

		res.WriteString(seq)
	}

	return res.String()
}

func splitToSequences(tokens []prose.Token) []string {
	var result []string

	seqSoFar := ""
	distSoFar := math.MaxInt64

	resetSeq := func() {
		seqSoFar = ""
		distSoFar = math.MaxInt64
	}

	setSeq := func(newDictSeq string, newDistSoFar int) {
		seqSoFar = newDictSeq
		distSoFar = newDistSoFar
	}

	for _, tok := range tokens {
		nonPunctuation := regexp.MustCompile("^[A-Z]+$")
		if !nonPunctuation.MatchString(tok.Tag) {
			// non empty seqSoFar?
			if seqSoFar != "" {
				result = append(result, seqSoFar)
				resetSeq()
			}

			result = append(result, tok.Text+" ")
			continue
		}

		// sequence extension and flush related logic
		newSeq := strings.ToLower(tok.Text)
		if seqSoFar != "" {
			newSeq = seqSoFar + " " + newSeq
		}

		dictSeq, dist := findByMinDist(newSeq)

		if dictSeq != "" {

			// better sequence? increase
			if dist < distSoFar {
				seqSoFar = dictSeq
				distSoFar = dist
				continue
			}

			// flush previous sequence
			result = append(result, seqSoFar)

			// start with building of a new sequence
			setSeq(findByMinDist(strings.ToLower(tok.Text)))
		}

	}

	// final flush
	if seqSoFar != "" {
		result = append(result, seqSoFar)
	}

	return result
}

// TODO take punctuation into account
func TranslateText(langFrom, langTo, text string, maxAlt int) string {
	var res bytes.Buffer

	// split text into words
	words := strings.Fields(text)

	for i := 0; i < len(words); i++ {
		word := strings.ToLower(words[i])
		foundWord, distWord := findByMinDist(word)

		if i+1 < len(words) {
			nextWord := strings.ToLower(words[i+1])
			foundPair, distPair := findByMinDist(word + " " + nextWord)

			// pair has better result than a single word?
			if distPair < distWord {
				res.WriteString(translationWords(dictMap[foundPair], maxAlt) + " ")
				i += 2
				continue
			}
		}

		if foundWord == "" {
			res.WriteString(word + " ")
		} else {
			res.WriteString(translationWords(dictMap[foundWord], maxAlt) + " ")
		}

	}

	return res.String()
}

func translationWords(val []string, maxAlt int) string {
	var buf bytes.Buffer

	for i := range val {
		if i > maxAlt {
			break
		}

		if i == 0 {
			buf.WriteString(val[i])
		} else {
			buf.WriteString("[alt:" + val[i] + "]")
		}
	}

	return buf.String()
}

func findByMinDist(word string) (string, int) {
	minDist := math.MaxInt64
	result := ""

	for k := range dictMap {
		// limit key set in order to limit number of distance calculations
		if k[0] != word[0] {
			continue
		}

		options := levenshtein.Options{
			InsCost: 1,
			DelCost: 1,
			SubCost: 1,
			Matches: func(sourceCharacter rune, targetCharacter rune) bool {
				return sourceCharacter == targetCharacter
			},
		}
		dist := levenshtein.DistanceForStrings([]rune(word), []rune(k), options)

		if dist < minDist {
			minDist = dist
			result = k
		}

	}

	return result, minDist
}
<|endoftext|>


<commit_before>package saf

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/fatih/color"
)

// Public http endpoints for getting required data
const (
	bundlesURL = "http://www.safaricom.com/bundles/GetSubDetails"
)

// Bundles defines the structure for bundles values returned
type Bundles struct {
	AccType       string
	Bundles       string
	BundlesExpiry string
	Airtime       float64
	BongaSMS      string
	BongaBalance  string
}

func (b Bundles) String() string {
	var (
		head   = color.New(color.FgGreen, color.Bold).Add(color.Underline).SprintFunc()
		red    = color.New(color.FgRed).SprintFunc()
		green  = color.New(color.FgGreen).SprintFunc()
		yellow = color.New(color.FgYellow).SprintFunc()
	)
	return fmt.Sprintf(`%s
%s
%s
%s
%s
%s`,
		fmt.Sprintf("%s", head("SAFARICOM BALANCE")),
		fmt.Sprintf("Airtime Balance: %.2f/=", (b.Airtime)),
		fmt.Sprintf("Data Bundles: %s", green(b.Bundles)),
		fmt.Sprintf("Bundles Expiry: %s", red(b.BundlesExpiry)),
		fmt.Sprintf("Bonga SMS: %s", yellow(b.BongaSMS)),
		fmt.Sprintf("Bonga Balance: %s", yellow(b.BongaBalance)),
	)

}

// GetBundles returns the bundles for the line in use
func GetBundles() (*Bundles, error) {
	resp, err := http.Get(bundlesURL)
	if err != nil {
		return nil, errors.New("no internet connection")
	}
	defer func() { _ = resp.Body.Close() }()
	dat, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	_ = resp.Body.Close()
	return ParseBundles(string(dat))
}

/*
ParseBundles parses raw bundles html to Bundles value.
Raw bundles html are as received from the Saf GetSubDetails http endpoint.
Especially useful in processing data sent from clients (e.g. xhr)
*/
func ParseBundles(rawBundles string) (*Bundles, error) {
	if strings.Contains(rawBundles, "not able to capture") {
		return nil, errors.New("you aren't connected through Safaricom")
	}
	rows := strings.Split(rawBundles, "
")

	res := map[string]string{
		"Account Types":           "",
		"Data Bundle":      "",
		"Data Bundle Expiry Date": "",
		"Airtime Balances":        "",
		"Bonga SMS</":             "",
		"Bonga Balance":           "",
	}

	for _, val := range rows {
		if !strings.Contains(val, "
") {
			continue
		}
		var title string
		for key := range res {
			if strings.Contains(val, key) {
				title = key
			}
		}
		if title == "" {
			continue
		}

		if strings.Count(val, "") < 2 {
			continue
		}
		val2 := strings.Split(val, "")[2]
		val2 = strings.Replace(val2, "", "", 1)
		val2 = strings.Replace(val2, "
", "", 1)
		res[title] = val2
	}
	var ok bool
	for _, val := range res {
		if val != "" {
			ok = true // some data found
			break
		}
	}
	if !ok {
		return nil, errors.New("Unable to get bundles")
	}

	airtime, err := strconv.ParseFloat(res["Airtime Balances"], 64)
	if err != nil {
		return nil, errors.New("failed to parse bundles: unable to get airtime")
	}
	return &Bundles{
		AccType:       res["Account Types"],
		Bundles:       res["Data Bundle"],
		Airtime:       airtime,
		BundlesExpiry: res["Data Bundle Expiry Date"],
		BongaSMS:      res["Bonga SMS</"],
		BongaBalance:  res["Bonga Balance"],
	}, nil

}
<commit_msg>fix bundles balance printing<commit_after>package saf

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/fatih/color"
)

// Public http endpoints for getting required data
const (
	bundlesURL = "http://www.safaricom.com/bundles/GetSubDetails"
)

// Bundles defines the structure for bundles values returned
type Bundles struct {
	AccType       string
	Bundles       string
	BundlesExpiry string
	Airtime       float64
	BongaSMS      string
	BongaBalance  string
}

func (b Bundles) String() string {
	var (
		head   = color.New(color.FgGreen, color.Bold).Add(color.Underline).SprintFunc()
		red    = color.New(color.FgRed).SprintFunc()
		green  = color.New(color.FgGreen).SprintFunc()
		yellow = color.New(color.FgYellow).SprintFunc()
	)

	bal := head("SAFARICOM BALANCE")
	if b.Airtime != 0 {
		bal += fmt.Sprintf("\nAirtime Balance: %.2f/=", (b.Airtime))
	}
	if b.Bundles != "" {
		bal += fmt.Sprintf("\nData Bundles: %s", green(b.Bundles))
	}
	if b.BundlesExpiry != "" {
		bal += fmt.Sprintf("\nBundles Expiry: %s", red(b.BundlesExpiry))
	}
	if b.BongaSMS != "" {
		bal += fmt.Sprintf("\nBonga SMS: %s", yellow(b.BongaSMS))
	}
	if b.BongaBalance != "" {
		bal += fmt.Sprintf("\nBonga Balance: %s", yellow(b.BongaBalance))
	}
	return bal

}

// GetBundles returns the bundles for the line in use
func GetBundles() (*Bundles, error) {
	resp, err := http.Get(bundlesURL)
	if err != nil {
		return nil, errors.New("no internet connection")
	}
	defer func() { _ = resp.Body.Close() }()
	dat, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	_ = resp.Body.Close()
	return ParseBundles(string(dat))
}

/*
ParseBundles parses raw bundles html to Bundles value.
Raw bundles html are as received from the Saf GetSubDetails http endpoint.
Especially useful in processing data sent from clients (e.g. xhr)
*/
func ParseBundles(rawBundles string) (*Bundles, error) {
	if strings.Contains(rawBundles, "not able to capture") {
		return nil, errors.New("you aren't connected through Safaricom")
	}
	rows := strings.Split(rawBundles, "
")

	res := map[string]string{
		"Account Types":           "",
		"Data Bundle":      "",
		"Data Bundle Expiry Date": "",
		"Airtime Balances":        "",
		"Bonga SMS</":             "",
		"Bonga Balance":           "",
	}

	for _, val := range rows {
		if !strings.Contains(val, "
") {
			continue
		}
		var title string
		for key := range res {
			if strings.Contains(val, key) {
				title = key
			}
		}
		if title == "" {
			continue
		}

		if strings.Count(val, "") < 2 {
			continue
		}
		val2 := strings.Split(val, "")[2]
		val2 = strings.Replace(val2, "", "", 1)
		val2 = strings.Replace(val2, "
", "", 1)
		res[title] = val2
	}
	var ok bool
	for _, val := range res {
		if val != "" {
			ok = true // some data found
			break
		}
	}
	if !ok {
		return nil, errors.New("Unable to get bundles")
	}

	airtime, err := strconv.ParseFloat(res["Airtime Balances"], 64)
	if err != nil {
		return nil, errors.New("failed to parse bundles: unable to get airtime")
	}
	return &Bundles{
		AccType:       res["Account Types"],
		Bundles:       res["Data Bundle"],
		Airtime:       airtime,
		BundlesExpiry: res["Data Bundle Expiry Date"],
		BongaSMS:      res["Bonga SMS</"],
		BongaBalance:  res["Bonga Balance"],
	}, nil

}
<|endoftext|>