<commit_before>package php

import (
	"fmt"

	"stephensearles.com/php/ast"
)

type parser struct {
	lexer *lexer

	previous   []Item
	idx        int
	current    Item
	errors     []error
	parenLevel int
	errorMap   map[int]bool
	errorCount int

	Debug       bool
	PrintTokens bool
	MaxErrors   int
}

func NewParser(input string) *parser {
	p := &parser{
		idx:       -1,
		MaxErrors: 10,
		lexer:     newLexer(input),
		errorMap:  make(map[int]bool),
	}
	return p
}

func (p *parser) Parse() (nodes []ast.Node, errors []error) {
	defer func() {
		if r := recover(); r != nil {
			errors = append([]error{fmt.Errorf("%s", r)}, p.errors...)
			if p.Debug {
				for _, err := range p.errors {
					fmt.Println(err)
				}
				panic(r)
			}
		}
	}()
	// expecting either itemHTML or itemPHPBegin
	nodes = make([]ast.Node, 0, 1)
TokenLoop:
	for {
		p.next()
		switch p.current.typ {
		case itemEOF:
			break TokenLoop
		default:
			n := p.parseNode()
			if n != nil {
				nodes = append(nodes, n)
			}
		}
	}
	errors = p.errors
	return nodes, errors
}

func (p *parser) parseNode() ast.Node {
	switch p.current.typ {
	case itemHTML:
		return ast.Echo(ast.Literal{Type: ast.String})
	case itemPHPBegin:
		return nil
	case itemPHPEnd:
		return nil
	}
	return p.parseStmt()
}

func (p *parser) next() {
	p.idx += 1
	if len(p.previous) <= p.idx {
		p.current = p.lexer.nextItem()
		if p.PrintTokens {
			fmt.Println(p.current)
		}
		p.previous = append(p.previous, p.current)
	} else {
		p.current = p.previous[p.idx]
	}
}

func (p *parser) backup() {
	p.idx -= 1
	p.current = p.previous[p.idx]
}

func (p *parser) peek() (i Item) {
	p.next()
	i = p.current
	p.backup()
	return
}

func (p *parser) expectCurrent(i ...ItemType) {
	for _, typ := range i {
		if p.current.typ == typ {
			return
		}
	}
	p.expected(i...)
}

func (p *parser) expectAndNext(i ...ItemType) {
	defer p.next()
	for _, typ := range i {
		if p.current.typ == typ {
			return
		}
	}
	p.expected(i...)
}

func (p *parser) expect(i ...ItemType) {
	p.next()
	p.expectCurrent(i...)
}

func (p *parser) expected(i ...ItemType) {
	p.errorf("Found %s, expected %s", p.current, i)
}

func (p *parser) errorf(str string, args ...interface{}) {
	if p.errorCount > p.MaxErrors {
		panic("too many errors")
	}
	if _, ok := p.errorMap[p.current.pos.Line]; ok {
		return
	}
	errString := fmt.Sprintf(str, args...)
	p.errorCount += 1
	p.errors = append(p.errors, fmt.Errorf("%s: %s", p.errorPrefix(), errString))
	p.errorMap[p.current.pos.Line] = true
}

func (p *parser) errorPrefix() string {
	return fmt.Sprintf("%s %d", p.lexer.file, p.current.pos.Line)
}

func (p *parser) parseNextExpression() ast.Expression {
	p.next()
	return p.parseExpression()
}

func (p *parser) parseFunctionCall(callable ast.Expression) *ast.FunctionCallExpression {
	expr := &ast.FunctionCallExpression{}
	expr.FunctionName = callable
	return p.parseFunctionArguments(expr)
}

func (p *parser) parseFunctionArguments(expr *ast.FunctionCallExpression) *ast.FunctionCallExpression {
	expr.Arguments = make([]ast.Expression, 0)
	p.expect(itemOpenParen)
	if p.peek().typ == itemCloseParen {
		p.expect(itemCloseParen)
		return expr
	}
	expr.Arguments = append(expr.Arguments, p.parseNextExpression())
	for p.peek().typ != itemCloseParen {
		p.expect(itemComma)
		arg := p.parseNextExpression()
		if arg == nil {
			break
		}
		expr.Arguments = append(expr.Arguments, arg)
	}
	p.expect(itemCloseParen)
	return expr

}

func (p *parser) parseStmt() ast.Statement {
	switch p.current.typ {
	case itemBlockBegin:
		p.backup()
		return p.parseBlock()
	case itemGlobal:
		p.next()
		g := &ast.GlobalDeclaration{
			Identifiers: make([]*ast.Variable, 0, 1),
		}
		for p.current.typ == itemVariableOperator {
			variable, ok := p.parseVariable().(*ast.Variable)
			if !ok {
				p.errorf("global declarations must be of standard variables")
				break
			}
			g.Identifiers = append(g.Identifiers, variable)
			if p.peek().typ != itemComma {
				break
			}
			p.expect(itemComma)
			p.next()
		}
		p.expectStmtEnd()
		return g
	case itemNamespace:
		p.expect(itemIdentifier)
		p.expectStmtEnd()
		// We are ignoring this for now
		return nil
	case itemUse:
		p.expect(itemIdentifier)
		if p.peek().typ == itemAsOperator {
			p.expect(itemAsOperator)
			p.expect(itemIdentifier)
		}
		p.expectStmtEnd()
		// We are ignoring this for now
		return nil
	case itemVariableOperator:
		ident := p.expressionize()
		switch p.peek().typ {
		case itemUnaryOperator:
			expr := ast.ExpressionStmt{p.parseOperation(p.parenLevel, ident)}
			p.expectStmtEnd()
			return expr
		case itemOpenParen:
			var expr ast.Expression
			expr = p.parseFunctionArguments(&ast.FunctionCallExpression{
				FunctionName: ident,
			})
			if p.peek().typ == itemObjectOperator {
				expr = p.parseObjectLookup(expr)
			}
			p.expectStmtEnd()
			return expr
		default:
			stmt := ast.ExpressionStmt{p.parseOperation(p.parenLevel, ident)}
			p.expectStmtEnd()
			return stmt
		}
	case itemUnaryOperator:
		expr := ast.ExpressionStmt{p.parseExpression()}
		p.expectStmtEnd()
		return expr
	case itemFunction:
		return p.parseFunctionStmt()
	case itemPHPEnd:
		if p.peek().typ == itemEOF {
			return nil
		}
		p.expect(itemHTML)
		expr := ast.Echo(&ast.Literal{Type: ast.String})
		p.next()
		if p.current.typ != itemEOF {
			p.expectCurrent(itemPHPBegin)
		}
		return expr
	case itemEcho:
		exprs := []ast.Expression{
			p.parseNextExpression(),
		}
		for p.peek().typ == itemComma {
			p.expect(itemComma)
			exprs = append(exprs, p.parseNextExpression())
		}
		p.expectStmtEnd()
		return ast.Echo(exprs...)
	case itemIf:
		return p.parseIf()
	case itemWhile:
		return p.parseWhile()
	case itemDo:
		return p.parseDo()
	case itemFor:
		return p.parseFor()
	case itemForeach:
		return p.parseForeach()
	case itemSwitch:
		return p.parseSwitch()
	case itemAbstract:
		fallthrough
	case itemClass:
		return p.parseClass()
	case itemInterface:
		return p.parseInterface()
	case itemReturn:
		p.next()
		stmt := ast.ReturnStmt{}
		if p.current.typ != itemStatementEnd {
			stmt.Expression = p.parseExpression()
			p.expectStmtEnd()
		}
		return stmt
	case itemBreak:
		p.next()
		stmt := ast.BreakStmt{}
		if p.current.typ != itemStatementEnd {
			stmt.Expression = p.parseExpression()
			p.expectStmtEnd()
		}
		return stmt
	case itemContinue:
		p.next()
		stmt := ast.ContinueStmt{}
		if p.current.typ != itemStatementEnd {
			stmt.Expression = p.parseExpression()
			p.expectStmtEnd()
		}
		return stmt
	case itemThrow:
		stmt := ast.ThrowStmt{Expression: p.parseNextExpression()}
		p.expectStmtEnd()
		return stmt
	case itemExit:
		stmt := ast.ExitStmt{}
		if p.peek().typ == itemOpenParen {
			p.expect(itemOpenParen)
			if p.peek().typ != itemCloseParen {
				stmt.Expression = p.parseNextExpression()
			}
			p.expect(itemCloseParen)
		}
		p.expectStmtEnd()
		return stmt
	case itemTry:
		stmt := &ast.TryStmt{}
		stmt.TryBlock = p.parseBlock()
		for p.expect(itemCatch); p.current.typ == itemCatch; p.next() {
			caught := &ast.CatchStmt{}
			p.expect(itemOpenParen)
			p.expect(itemIdentifier)
			caught.CatchType = p.current.val
			p.expect(itemVariableOperator)
			p.expect(itemIdentifier)
			caught.CatchVar = ast.NewVariable(p.current.val)
			p.expect(itemCloseParen)
			caught.CatchBlock = p.parseBlock()
			stmt.CatchStmts = append(stmt.CatchStmts, caught)
		}
		p.backup()
		return stmt
	case itemIgnoreErrorOperator:
		// Ignore this operator
		p.next()
		return p.parseStmt()
	default:
		expr := p.parseExpression()
		if expr != nil {
			p.expectStmtEnd()
			return ast.ExpressionStmt{expr}
		}
		p.errorf("Found %s, statement or expression", p.current)
		return nil
	}
}

func (p *parser) expectStmtEnd() {
	if p.peek().typ != itemPHPEnd {
		p.expect(itemStatementEnd)
	}
}
func (p *parser) parseFunctionStmt() *ast.FunctionStmt {
	stmt := &ast.FunctionStmt{}
	stmt.FunctionDefinition = p.parseFunctionDefinition()
	stmt.Body = p.parseBlock()
	return stmt
}

func (p *parser) parseFunctionDefinition() *ast.FunctionDefinition {
	def := &ast.FunctionDefinition{}
	p.expect(itemIdentifier)
	def.Name = p.current.val
	def.Arguments = make([]ast.FunctionArgument, 0)
	p.expect(itemOpenParen)
	if p.peek().typ == itemCloseParen {
		p.expect(itemCloseParen)
		return def
	}
	def.Arguments = append(def.Arguments, p.parseFunctionArgument())
	for {
		switch p.peek().typ {
		case itemComma:
			p.expect(itemComma)
			def.Arguments = append(def.Arguments, p.parseFunctionArgument())
		case itemCloseParen:
			p.expect(itemCloseParen)
			return def
		default:
			p.errorf("unexpected argument separator:", p.current)
			return def
		}
	}
}

func (p *parser) parseFunctionArgument() ast.FunctionArgument {
	arg := ast.FunctionArgument{}
	switch p.peek().typ {
	case itemIdentifier, itemArray:
		p.next()
		arg.TypeHint = p.current.val
	}
	if p.peek().typ == itemAmpersandOperator {
		p.next()
	}
	p.expect(itemVariableOperator)
	p.next()
	arg.Variable = ast.NewVariable(p.current.val)
	if p.peek().typ == itemAssignmentOperator {
		p.expect(itemAssignmentOperator)
		p.next()
		arg.Default = p.parseExpression()
	}
	return arg
}

func (p *parser) parseBlock() *ast.Block {
	p.expect(itemBlockBegin)
	b := p.parseStatementsUntil(itemBlockEnd)
	p.expectCurrent(itemBlockEnd)
	return b
}

func (p *parser) parseStatementsUntil(endTokens ...ItemType) *ast.Block {
	block := &ast.Block{}
	breakTypes := map[ItemType]bool{}
	for _, typ := range endTokens {
		breakTypes[typ] = true
	}
	for {
		p.next()
		if _, ok := breakTypes[p.current.typ]; ok {
			break
		}
		stmt := p.parseStmt()
		if stmt == nil {
			return block
		}
		block.Statements = append(block.Statements, stmt)
	}
	return block
}
<commit_msg>Accepting an ampersand operator at the start of a function identifier to indicate returning a reference<commit_after>package php

import (
	"fmt"

	"stephensearles.com/php/ast"
)

type parser struct {
	lexer *lexer

	previous   []Item
	idx        int
	current    Item
	errors     []error
	parenLevel int
	errorMap   map[int]bool
	errorCount int

	Debug       bool
	PrintTokens bool
	MaxErrors   int
}

func NewParser(input string) *parser {
	p := &parser{
		idx:       -1,
		MaxErrors: 10,
		lexer:     newLexer(input),
		errorMap:  make(map[int]bool),
	}
	return p
}

func (p *parser) Parse() (nodes []ast.Node, errors []error) {
	defer func() {
		if r := recover(); r != nil {
			errors = append([]error{fmt.Errorf("%s", r)}, p.errors...)
			if p.Debug {
				for _, err := range p.errors {
					fmt.Println(err)
				}
				panic(r)
			}
		}
	}()
	// expecting either itemHTML or itemPHPBegin
	nodes = make([]ast.Node, 0, 1)
TokenLoop:
	for {
		p.next()
		switch p.current.typ {
		case itemEOF:
			break TokenLoop
		default:
			n := p.parseNode()
			if n != nil {
				nodes = append(nodes, n)
			}
		}
	}
	errors = p.errors
	return nodes, errors
}

func (p *parser) parseNode() ast.Node {
	switch p.current.typ {
	case itemHTML:
		return ast.Echo(ast.Literal{Type: ast.String})
	case itemPHPBegin:
		return nil
	case itemPHPEnd:
		return nil
	}
	return p.parseStmt()
}

func (p *parser) next() {
	p.idx += 1
	if len(p.previous) <= p.idx {
		p.current = p.lexer.nextItem()
		if p.PrintTokens {
			fmt.Println(p.current)
		}
		p.previous = append(p.previous, p.current)
	} else {
		p.current = p.previous[p.idx]
	}
}

func (p *parser) backup() {
	p.idx -= 1
	p.current = p.previous[p.idx]
}

func (p *parser) peek() (i Item) {
	p.next()
	i = p.current
	p.backup()
	return
}

func (p *parser) expectCurrent(i ...ItemType) {
	for _, typ := range i {
		if p.current.typ == typ {
			return
		}
	}
	p.expected(i...)
}

func (p *parser) expectAndNext(i ...ItemType) {
	defer p.next()
	for _, typ := range i {
		if p.current.typ == typ {
			return
		}
	}
	p.expected(i...)
}

func (p *parser) expect(i ...ItemType) {
	p.next()
	p.expectCurrent(i...)
}

func (p *parser) expected(i ...ItemType) {
	p.errorf("Found %s, expected %s", p.current, i)
}

func (p *parser) errorf(str string, args ...interface{}) {
	if p.errorCount > p.MaxErrors {
		panic("too many errors")
	}
	if _, ok := p.errorMap[p.current.pos.Line]; ok {
		return
	}
	errString := fmt.Sprintf(str, args...)
	p.errorCount += 1
	p.errors = append(p.errors, fmt.Errorf("%s: %s", p.errorPrefix(), errString))
	p.errorMap[p.current.pos.Line] = true
}

func (p *parser) errorPrefix() string {
	return fmt.Sprintf("%s %d", p.lexer.file, p.current.pos.Line)
}

func (p *parser) parseNextExpression() ast.Expression {
	p.next()
	return p.parseExpression()
}

func (p *parser) parseFunctionCall(callable ast.Expression) *ast.FunctionCallExpression {
	expr := &ast.FunctionCallExpression{}
	expr.FunctionName = callable
	return p.parseFunctionArguments(expr)
}

func (p *parser) parseFunctionArguments(expr *ast.FunctionCallExpression) *ast.FunctionCallExpression {
	expr.Arguments = make([]ast.Expression, 0)
	p.expect(itemOpenParen)
	if p.peek().typ == itemCloseParen {
		p.expect(itemCloseParen)
		return expr
	}
	expr.Arguments = append(expr.Arguments, p.parseNextExpression())
	for p.peek().typ != itemCloseParen {
		p.expect(itemComma)
		arg := p.parseNextExpression()
		if arg == nil {
			break
		}
		expr.Arguments = append(expr.Arguments, arg)
	}
	p.expect(itemCloseParen)
	return expr

}

func (p *parser) parseStmt() ast.Statement {
	switch p.current.typ {
	case itemBlockBegin:
		p.backup()
		return p.parseBlock()
	case itemGlobal:
		p.next()
		g := &ast.GlobalDeclaration{
			Identifiers: make([]*ast.Variable, 0, 1),
		}
		for p.current.typ == itemVariableOperator {
			variable, ok := p.parseVariable().(*ast.Variable)
			if !ok {
				p.errorf("global declarations must be of standard variables")
				break
			}
			g.Identifiers = append(g.Identifiers, variable)
			if p.peek().typ != itemComma {
				break
			}
			p.expect(itemComma)
			p.next()
		}
		p.expectStmtEnd()
		return g
	case itemNamespace:
		p.expect(itemIdentifier)
		p.expectStmtEnd()
		// We are ignoring this for now
		return nil
	case itemUse:
		p.expect(itemIdentifier)
		if p.peek().typ == itemAsOperator {
			p.expect(itemAsOperator)
			p.expect(itemIdentifier)
		}
		p.expectStmtEnd()
		// We are ignoring this for now
		return nil
	case itemVariableOperator:
		ident := p.expressionize()
		switch p.peek().typ {
		case itemUnaryOperator:
			expr := ast.ExpressionStmt{p.parseOperation(p.parenLevel, ident)}
			p.expectStmtEnd()
			return expr
		case itemOpenParen:
			var expr ast.Expression
			expr = p.parseFunctionArguments(&ast.FunctionCallExpression{
				FunctionName: ident,
			})
			if p.peek().typ == itemObjectOperator {
				expr = p.parseObjectLookup(expr)
			}
			p.expectStmtEnd()
			return expr
		default:
			stmt := ast.ExpressionStmt{p.parseOperation(p.parenLevel, ident)}
			p.expectStmtEnd()
			return stmt
		}
	case itemUnaryOperator:
		expr := ast.ExpressionStmt{p.parseExpression()}
		p.expectStmtEnd()
		return expr
	case itemFunction:
		return p.parseFunctionStmt()
	case itemPHPEnd:
		if p.peek().typ == itemEOF {
			return nil
		}
		p.expect(itemHTML)
		expr := ast.Echo(&ast.Literal{Type: ast.String})
		p.next()
		if p.current.typ != itemEOF {
			p.expectCurrent(itemPHPBegin)
		}
		return expr
	case itemEcho:
		exprs := []ast.Expression{
			p.parseNextExpression(),
		}
		for p.peek().typ == itemComma {
			p.expect(itemComma)
			exprs = append(exprs, p.parseNextExpression())
		}
		p.expectStmtEnd()
		return ast.Echo(exprs...)
	case itemIf:
		return p.parseIf()
	case itemWhile:
		return p.parseWhile()
	case itemDo:
		return p.parseDo()
	case itemFor:
		return p.parseFor()
	case itemForeach:
		return p.parseForeach()
	case itemSwitch:
		return p.parseSwitch()
	case itemAbstract:
		fallthrough
	case itemClass:
		return p.parseClass()
	case itemInterface:
		return p.parseInterface()
	case itemReturn:
		p.next()
		stmt := ast.ReturnStmt{}
		if p.current.typ != itemStatementEnd {
			stmt.Expression = p.parseExpression()
			p.expectStmtEnd()
		}
		return stmt
	case itemBreak:
		p.next()
		stmt := ast.BreakStmt{}
		if p.current.typ != itemStatementEnd {
			stmt.Expression = p.parseExpression()
			p.expectStmtEnd()
		}
		return stmt
	case itemContinue:
		p.next()
		stmt := ast.ContinueStmt{}
		if p.current.typ != itemStatementEnd {
			stmt.Expression = p.parseExpression()
			p.expectStmtEnd()
		}
		return stmt
	case itemThrow:
		stmt := ast.ThrowStmt{Expression: p.parseNextExpression()}
		p.expectStmtEnd()
		return stmt
	case itemExit:
		stmt := ast.ExitStmt{}
		if p.peek().typ == itemOpenParen {
			p.expect(itemOpenParen)
			if p.peek().typ != itemCloseParen {
				stmt.Expression = p.parseNextExpression()
			}
			p.expect(itemCloseParen)
		}
		p.expectStmtEnd()
		return stmt
	case itemTry:
		stmt := &ast.TryStmt{}
		stmt.TryBlock = p.parseBlock()
		for p.expect(itemCatch); p.current.typ == itemCatch; p.next() {
			caught := &ast.CatchStmt{}
			p.expect(itemOpenParen)
			p.expect(itemIdentifier)
			caught.CatchType = p.current.val
			p.expect(itemVariableOperator)
			p.expect(itemIdentifier)
			caught.CatchVar = ast.NewVariable(p.current.val)
			p.expect(itemCloseParen)
			caught.CatchBlock = p.parseBlock()
			stmt.CatchStmts = append(stmt.CatchStmts, caught)
		}
		p.backup()
		return stmt
	case itemIgnoreErrorOperator:
		// Ignore this operator
		p.next()
		return p.parseStmt()
	default:
		expr := p.parseExpression()
		if expr != nil {
			p.expectStmtEnd()
			return ast.ExpressionStmt{expr}
		}
		p.errorf("Found %s, statement or expression", p.current)
		return nil
	}
}

func (p *parser) expectStmtEnd() {
	if p.peek().typ != itemPHPEnd {
		p.expect(itemStatementEnd)
	}
}
func (p *parser) parseFunctionStmt() *ast.FunctionStmt {
	stmt := &ast.FunctionStmt{}
	stmt.FunctionDefinition = p.parseFunctionDefinition()
	stmt.Body = p.parseBlock()
	return stmt
}

func (p *parser) parseFunctionDefinition() *ast.FunctionDefinition {
	def := &ast.FunctionDefinition{}
	if p.peek().typ == itemAmpersandOperator {
		// This is a function returning a reference ... ignore this for now
		p.next()
	}
	p.expect(itemIdentifier)
	def.Name = p.current.val
	def.Arguments = make([]ast.FunctionArgument, 0)
	p.expect(itemOpenParen)
	if p.peek().typ == itemCloseParen {
		p.expect(itemCloseParen)
		return def
	}
	def.Arguments = append(def.Arguments, p.parseFunctionArgument())
	for {
		switch p.peek().typ {
		case itemComma:
			p.expect(itemComma)
			def.Arguments = append(def.Arguments, p.parseFunctionArgument())
		case itemCloseParen:
			p.expect(itemCloseParen)
			return def
		default:
			p.errorf("unexpected argument separator:", p.current)
			return def
		}
	}
}

func (p *parser) parseFunctionArgument() ast.FunctionArgument {
	arg := ast.FunctionArgument{}
	switch p.peek().typ {
	case itemIdentifier, itemArray:
		p.next()
		arg.TypeHint = p.current.val
	}
	if p.peek().typ == itemAmpersandOperator {
		p.next()
	}
	p.expect(itemVariableOperator)
	p.next()
	arg.Variable = ast.NewVariable(p.current.val)
	if p.peek().typ == itemAssignmentOperator {
		p.expect(itemAssignmentOperator)
		p.next()
		arg.Default = p.parseExpression()
	}
	return arg
}

func (p *parser) parseBlock() *ast.Block {
	p.expect(itemBlockBegin)
	b := p.parseStatementsUntil(itemBlockEnd)
	p.expectCurrent(itemBlockEnd)
	return b
}

func (p *parser) parseStatementsUntil(endTokens ...ItemType) *ast.Block {
	block := &ast.Block{}
	breakTypes := map[ItemType]bool{}
	for _, typ := range endTokens {
		breakTypes[typ] = true
	}
	for {
		p.next()
		if _, ok := breakTypes[p.current.typ]; ok {
			break
		}
		stmt := p.parseStmt()
		if stmt == nil {
			return block
		}
		block.Statements = append(block.Statements, stmt)
	}
	return block
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"unicode/utf8"
)

type Node struct {
	tag string
	attrs map[string]string
	text string
}

type Parser struct {

	debug bool

	Scanner Scanner
	output string

	// node heirarchy
	stack []*Node

	// states as the Parser object figures out how to output the tokens

	// new line has started
	isNewLine bool

	isIndent bool
	isDedent bool
	dedentCount int

	// flag indicates we are still checking for attribute assignments
	isAttr bool

	// attrName and attrValue are buffers to determine if TokWord for a tag are
	// potentially for an attribute, or if they are just normal text
	attrName string
	attrValue string
	attrAssigned bool
	// dumb appended string while attribute assignment is still being determined
	// this is used if it turns out tokens are NOT part of an attribute assignment
	// and the string (with whitespace if any) gets appended to the node text
	attrString string

	node *Node
}

func (p *Parser) Output() string {
	var tokens []rune
	var text string

	p.stack = make([]*Node, 0)
	p.output = ""
	p.isNewLine = true
	p.isIndent = false
	p.isDedent = false
	p.isAttr = false
	p.attrName = ""
	p.attrValue = ""
	p.attrAssigned = false
	p.attrString = ""
	p.dedentCount = 0
	p.node = p.newNode()

	for {
		tokens = p.Scanner.Scan()
		for _, token := range tokens {
			text = p.Scanner.TokenText()
			if p.debug {
				fmt.Printf("[debug] token: [%s]\n", TokenString(token))
				fmt.Printf("[debug] text:  [%s]\n", text)
			}
			p.processToken(token, text)
			if p.debug {
				fmt.Printf("[debug] node:  [%s]\n\n", p.node)
			}
		}
		if tokens[0] == TokEOF {
			break
		}
	}
	return p.output
}

func (p *Parser) closeLastNode() {
	if len(p.stack) > 0 {
		oldNode := p.stack[len(p.stack) - 1]
		p.stack = p.stack[0: len(p.stack) - 1]
		if r, l := utf8.DecodeLastRuneInString(p.output); r == ' ' {
			p.output = p.output[0: len(p.output) - l]
		}
		p.output += "</" + oldNode.tag + ">"
	}
}

func (p *Parser) lastNode() *Node {
	return p.stack[len(p.stack) - 1]
}

func (p *Parser) outputNode(node *Node) {
	if node.tag != "" {
		p.output += "<" + node.tag
		if len(node.attrs) > 0 {
			for k, v := range node.attrs {
				p.output += fmt.Sprintf(" %s='%s'", k, v)
			}
		}
		p.output += ">"
		p.output += node.text
	}
}
func (p *Parser) pushNode(node *Node) {
	if p.node.tag != "" {
		p.stack = append(p.stack, p.node)
	}
}

func (p *Parser) processToken(tok rune, text string) {
	switch tok {
	case TokWord, TokComma:
		if p.node.tag == "" {
			p.node.tag = text
			// found the tag, now check for attributes
			p.isAttr = true
		} else if p.isAttr {
			if p.attrAssigned {
				if _, found := p.node.attrs[p.attrName]; found {
					p.node.attrs[p.attrName] += " "
					p.node.attrs[p.attrName] += text
				} else {
					p.node.attrs[p.attrName] = text
				}

				// reset to look for a new attribute assignment
				p.attrName = ""
				p.attrValue = ""
				p.attrAssigned = false
			} else if p.attrName == "" {
				p.attrString += text
				p.attrName = text
			} else {
				p.node.text += p.attrString
				p.node.text += text
				p.isAttr = false
				p.attrName = ""
				p.attrValue = ""
				p.attrAssigned = false
			}
		} else {
			p.node.text += text
		}
	case TokAssign:
		if p.isAttr {
			if p.attrName != "" && p.attrValue == "" {
				p.attrAssigned = true
				p.attrString += text
			}
			if p.attrName == "" || p.attrValue != "" {
				p.attrAssigned = false
				p.isAttr = false
				p.node.text += p.attrString
			}
		} else {
			p.node.text += text
		}
	case TokWhitespace:
		if p.isAttr && p.attrString != "" {
			p.attrString += text

		// skip initial whitespace for text
		} else if p.node.text != "" {
			p.node.text += text
		}
	case TokIndent:
		p.isIndent = true
		p.isDedent = false
	case TokDedent:
		p.isIndent = false
		p.isDedent = true
		p.dedentCount += 1
	case TokNewLine:
		// up to this point a node has been constructed but not appended to the
		// output
		if p.isDedent {
			// for dedents, pop off the stack and close the node
			for p.dedentCount > 0 {
				p.closeLastNode()
				p.dedentCount -= 1
			}
		}
		if p.isDedent || !p.isIndent {
			// closing one more time because this new node replaces the top node in the
			// stack for dedents or siblings (non indent)
			p.closeLastNode()
		}
		p.outputNode(p.node)
		p.pushNode(p.node)

		// reset
		p.node = p.newNode()
		p.isIndent = false
		p.isDedent = false
		p.isAttr = false
		p.attrName = ""
		p.attrValue = ""
		p.attrString = ""
	case TokEOF:
		// since p.node is never on the stack until TokNewLine we push to the stack
		// in this scenario
		p.pushNode(p.node)
		p.outputNode(p.node)
		// close the remaining nodes in the stack
		for len(p.stack) > 0 {
			p.closeLastNode()
		}
	default:
		if !p.isAttr {
			p.node.text += text
		}
	}
}

func (p *Parser) newNode() *Node {
	n := new(Node)
	n.attrs = make(map[string]string)
	return n
}
<commit_msg>parser correctly knows when to output attrString or text<commit_after>package main

import (
	"fmt"
	"unicode/utf8"
)

type Node struct {
	tag string
	attrs map[string]string
	text string

	// dumb appended string while attribute assignment is still being determined
	// this is used if it turns out tokens are NOT part of an attribute assignment
	// and the string (with whitespace if any) gets used as the node's text
	attrString string
}

type Parser struct {

	debug bool

	Scanner Scanner
	output string

	// node heirarchy
	stack []*Node

	// states as the Parser object figures out how to output the tokens

	// new line has started
	isNewLine bool

	isIndent bool
	isDedent bool
	dedentCount int

	// flag indicates we are still checking for attribute assignments
	isAttr bool

	// attrName and attrValue are buffers to determine if TokWord for a tag are
	// potentially for an attribute, or if they are just normal text
	attrName string
	attrValue string
	attrAssigned bool

	node *Node
}

func (p *Parser) Output() string {
	var tokens []rune
	var text string

	p.stack = make([]*Node, 0)
	p.output = ""
	p.isNewLine = true
	p.isIndent = false
	p.isDedent = false
	p.isAttr = false
	p.attrName = ""
	p.attrValue = ""
	p.attrAssigned = false
	p.dedentCount = 0
	p.node = p.newNode()

	for {
		tokens = p.Scanner.Scan()
		for _, token := range tokens {
			text = p.Scanner.TokenText()
			if p.debug {
				fmt.Printf("[debug] token: [%s]\n", TokenString(token))
				fmt.Printf("[debug] text:  [%s]\n", text)
			}
			p.processToken(token, text)
			if p.debug {
				fmt.Printf("[debug] node:  [%s]\n\n", p.node)
			}
		}
		if tokens[0] == TokEOF {
			break
		}
	}
	return p.output
}

func (p *Parser) closeLastNode() {
	if len(p.stack) > 0 {
		oldNode := p.stack[len(p.stack) - 1]
		p.stack = p.stack[0: len(p.stack) - 1]
		if r, l := utf8.DecodeLastRuneInString(p.output); r == ' ' {
			p.output = p.output[0: len(p.output) - l]
		}
		p.output += "</" + oldNode.tag + ">"
	}
}

func (p *Parser) lastNode() *Node {
	return p.stack[len(p.stack) - 1]
}

func (p *Parser) outputNode(node *Node) {
	if node.tag != "" {
		p.output += "<" + node.tag
		if len(node.attrs) > 0 {
			for k, v := range node.attrs {
				p.output += fmt.Sprintf(" %s='%s'", k, v)
			}
		}
		p.output += ">"
		if node.text != "" {
			p.output += node.text
		} else {
			p.output += node.attrString
		}
	}
}
func (p *Parser) pushNode(node *Node) {
	if p.node.tag != "" {
		p.stack = append(p.stack, p.node)
	}
}

func (p *Parser) processToken(tok rune, text string) {
	switch tok {
	case TokWord, TokComma:
		if p.node.tag == "" {
			p.node.tag = text
			// found the tag, now check for attributes
			p.isAttr = true
		} else if p.isAttr {
			if p.attrAssigned {
				if _, found := p.node.attrs[p.attrName]; found {
					p.node.attrs[p.attrName] += " "
					p.node.attrs[p.attrName] += text
				} else {
					p.node.attrs[p.attrName] = text
				}

				// reset to look for a new attribute assignment
				p.attrName = ""
				p.attrValue = ""
				p.attrAssigned = false
			} else if p.attrName == "" {
				p.node.attrString += text
				p.attrName = text
			} else {
				p.node.text += p.node.attrString
				p.node.text += text
				p.node.attrString = ""
				p.isAttr = false
				p.attrName = ""
				p.attrValue = ""
				p.attrAssigned = false
			}
		} else {
			p.node.text += text
		}
	case TokAssign:
		if p.isAttr {
			if p.attrName != "" && p.attrValue == "" {
				p.attrAssigned = true
				p.node.attrString += text
			}
			if p.attrName == "" || p.attrValue != "" {
				p.attrAssigned = false
				p.isAttr = false
				p.node.text += p.node.attrString
				p.node.attrString = ""
			}
		} else {
			p.node.text += text
		}
	case TokWhitespace:
		if p.isAttr && p.node.attrString != "" {
			p.node.attrString += text

		// skip initial whitespace for text
		} else if p.node.text != "" {
			p.node.text += text
		}
	case TokIndent:
		p.isIndent = true
		p.isDedent = false
	case TokDedent:
		p.isIndent = false
		p.isDedent = true
		p.dedentCount += 1
	case TokNewLine:
		// up to this point a node has been constructed but not appended to the
		// output
		if p.isDedent {
			// for dedents, pop off the stack and close the node
			for p.dedentCount > 0 {
				p.closeLastNode()
				p.dedentCount -= 1
			}
		}
		if p.isDedent || !p.isIndent {
			// closing one more time because this new node replaces the top node in the
			// stack for dedents or siblings (non indent)
			p.closeLastNode()
		}
		p.outputNode(p.node)
		p.pushNode(p.node)

		// reset
		p.node = p.newNode()
		p.isIndent = false
		p.isDedent = false
		p.isAttr = false
		p.attrName = ""
		p.attrValue = ""
	case TokEOF:
		// since p.node is never on the stack until TokNewLine we push to the stack
		// in this scenario
		p.pushNode(p.node)
		p.outputNode(p.node)
		// close the remaining nodes in the stack
		for len(p.stack) > 0 {
			p.closeLastNode()
		}
	default:
		if !p.isAttr {
			p.node.text += text
		}
	}
}

func (p *Parser) newNode() *Node {
	n := new(Node)
	n.attrs = make(map[string]string)
	return n
}
<|endoftext|>


<commit_before>package nmea

import (
	"fmt"
	"strconv"
)

// parser provides a simple way of accessing and parsing
// sentence fields
type parser struct {
	BaseSentence
	prefix string
	err    error
}

// newParser constructor
func newParser(s BaseSentence, prefix string) *parser {
	p := &parser{BaseSentence: s, prefix: prefix}
	if p.Type != prefix {
		p.SetErr("prefix", p.Type)
	}
	return p
}

// Err returns the first error encounterd during the parser's usage.
func (p *parser) Err() error {
	return p.err
}

// SetErr assigns an error. Calling this method has no
// effect if there is already an error.
func (p *parser) SetErr(context, value string) {
	if p.err == nil {
		p.err = fmt.Errorf("nmea: %s invalid %s: %s", p.prefix, context, value)
	}
}

// String returns the field value at the specified index.
func (p *parser) String(i int, context string) string {
	if p.err != nil {
		return ""
	}
	if i < 0 || i >= len(p.Fields) {
		p.SetErr(context, "index out of range")
		return ""
	}
	return p.Fields[i]
}

// EnumString returns the field value at the specified index.
// An error occurs if the value is not one of the options and not empty.
func (p *parser) EnumString(i int, context string, options ...string) string {
	s := p.String(i, context)
	if p.err != nil || s == "" {
		return ""
	}
	for _, o := range options {
		if o == s {
			return s
		}
	}
	p.SetErr(context, s)
	return ""
}

// Int64 returns the int64 value at the specified index.
// If the value is an emtpy string, 0 is returned.
func (p *parser) Int64(i int, context string) int64 {
	s := p.String(i, context)
	if p.err != nil {
		return 0
	}
	if s == "" {
		return 0
	}
	v, err := strconv.ParseInt(s, 10, 64)
	if err != nil {
		p.SetErr(context, s)
	}
	return v
}

// Float64 returns the float64 value at the specified index.
// If the value is an empty string, 0 is returned.
func (p *parser) Float64(i int, context string) float64 {
	s := p.String(i, context)
	if p.err != nil {
		return 0
	}
	if s == "" {
		return 0
	}
	v, err := strconv.ParseFloat(s, 64)
	if err != nil {
		p.SetErr(context, s)
	}
	return v
}

// Time returns the Time value at the specified index.
// If the value is empty, the Time is marked as invalid.
func (p *parser) Time(i int, context string) Time {
	s := p.String(i, context)
	if p.err != nil {
		return Time{}
	}
	v, err := ParseTime(s)
	if err != nil {
		p.SetErr(context, s)
	}
	return v
}

// Date returns the Date value at the specified index.
// If the value is empty, the Date is marked as invalid.
func (p *parser) Date(i int, context string) Date {
	s := p.String(i, context)
	if p.err != nil {
		return Date{}
	}
	v, err := ParseDate(s)
	if err != nil {
		p.SetErr(context, s)
	}
	return v
}

// LatLong returns the coordinate value of the specified fields.
func (p *parser) LatLong(i, j int, context string) LatLong {
	a := p.String(i, context)
	b := p.String(j, context)
	if p.err != nil {
		return 0
	}
	s := fmt.Sprintf("%s %s", a, b)
	v, err := ParseLatLong(s)
	if err != nil {
		p.SetErr(context, err.Error())
	}
	return v
}
<commit_msg>fix spelling mistakes in parser<commit_after>package nmea

import (
	"fmt"
	"strconv"
)

// parser provides a simple way of accessing and parsing
// sentence fields
type parser struct {
	BaseSentence
	prefix string
	err    error
}

// newParser constructor
func newParser(s BaseSentence, prefix string) *parser {
	p := &parser{BaseSentence: s, prefix: prefix}
	if p.Type != prefix {
		p.SetErr("prefix", p.Type)
	}
	return p
}

// Err returns the first error encountered during the parser's usage.
func (p *parser) Err() error {
	return p.err
}

// SetErr assigns an error. Calling this method has no
// effect if there is already an error.
func (p *parser) SetErr(context, value string) {
	if p.err == nil {
		p.err = fmt.Errorf("nmea: %s invalid %s: %s", p.prefix, context, value)
	}
}

// String returns the field value at the specified index.
func (p *parser) String(i int, context string) string {
	if p.err != nil {
		return ""
	}
	if i < 0 || i >= len(p.Fields) {
		p.SetErr(context, "index out of range")
		return ""
	}
	return p.Fields[i]
}

// EnumString returns the field value at the specified index.
// An error occurs if the value is not one of the options and not empty.
func (p *parser) EnumString(i int, context string, options ...string) string {
	s := p.String(i, context)
	if p.err != nil || s == "" {
		return ""
	}
	for _, o := range options {
		if o == s {
			return s
		}
	}
	p.SetErr(context, s)
	return ""
}

// Int64 returns the int64 value at the specified index.
// If the value is an empty string, 0 is returned.
func (p *parser) Int64(i int, context string) int64 {
	s := p.String(i, context)
	if p.err != nil {
		return 0
	}
	if s == "" {
		return 0
	}
	v, err := strconv.ParseInt(s, 10, 64)
	if err != nil {
		p.SetErr(context, s)
	}
	return v
}

// Float64 returns the float64 value at the specified index.
// If the value is an empty string, 0 is returned.
func (p *parser) Float64(i int, context string) float64 {
	s := p.String(i, context)
	if p.err != nil {
		return 0
	}
	if s == "" {
		return 0
	}
	v, err := strconv.ParseFloat(s, 64)
	if err != nil {
		p.SetErr(context, s)
	}
	return v
}

// Time returns the Time value at the specified index.
// If the value is empty, the Time is marked as invalid.
func (p *parser) Time(i int, context string) Time {
	s := p.String(i, context)
	if p.err != nil {
		return Time{}
	}
	v, err := ParseTime(s)
	if err != nil {
		p.SetErr(context, s)
	}
	return v
}

// Date returns the Date value at the specified index.
// If the value is empty, the Date is marked as invalid.
func (p *parser) Date(i int, context string) Date {
	s := p.String(i, context)
	if p.err != nil {
		return Date{}
	}
	v, err := ParseDate(s)
	if err != nil {
		p.SetErr(context, s)
	}
	return v
}

// LatLong returns the coordinate value of the specified fields.
func (p *parser) LatLong(i, j int, context string) LatLong {
	a := p.String(i, context)
	b := p.String(j, context)
	if p.err != nil {
		return 0
	}
	s := fmt.Sprintf("%s %s", a, b)
	v, err := ParseLatLong(s)
	if err != nil {
		p.SetErr(context, err.Error())
	}
	return v
}
<|endoftext|>


<commit_before>package astgen

import (
	"strconv"
	"fmt"
)

type parser struct {
	file []byte
	line int
}

func Load(data []byte) (def *LangDef, e error) {
	var p parser
	p.line = 0
	p.file = data

	defer func() {
		err := recover()
		if err != nil {
			def = nil
			e = fmt.Errorf("Error on line %d: %s\n", p.line, err)
		}
	}()

	def = new(LangDef)
	def.Types = make(map[string]Type)

	for !p.finished() {
		t := p.parse_type()
		def.Types[t.Common().Name] = t
	}

	return def, nil
}

func is_nl(b byte) bool {
	return b == '\n' || b == '\r'
}

func is_space(b byte) bool {
	return b == ' ' || b == '\t' || b == '\n' || b == '\r'
}

func is_letter(b byte) bool {
	return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_'
}

func (p *parser) skip_space() {
	for len(p.file) > 0 && is_space(p.file[0]) {
		if is_nl(p.file[0]) {
			p.line++
		}

		p.file = p.file[1:]
	}
}

func (p *parser) accept_token(token string) bool {
	p.skip_space()

	tok := []byte(token)

	if len(p.file) < len(tok) {
		return false
	}

	for i := range tok {
		if p.file[i] != tok[i] {
			return false
		}
	}

	if len(p.file) != len(tok) {
		next := p.file[len(tok)]
		if !is_space(next) && is_letter(p.file[len(tok)-1]) == is_letter(next) {
			return false
		}
	}

	p.file = p.file[len(tok):]

	return true
}

func (p *parser) match_token(token string) {
	if !p.accept_token(token) {
		panic("can't match token \"" + token + "\". Current: \"" + p.current_token() + "\"")
	}
}

func (p *parser) consume_line() string {
	p.skip_space()

	for i := range p.file {
		if p.file[i] == '\n' {
			ret := string(p.file[:i])
			p.file = p.file[i+1:]
			p.line++
			return ret
		}
	}

	ret := string(p.file)
	p.file = nil
	return ret
}

func (p *parser) consume_token() string {
	p.skip_space()

	if len(p.file) == 0 {
		panic("bad file")
	}

	ident := is_letter(p.file[0])

	for i := range p.file {
		if ident != is_letter(p.file[i]) || is_space(p.file[i]) {
			ret := string(p.file[:i])
			p.file = p.file[i:]
			return ret
		}
	}

	ret := string(p.file)
	p.file = nil
	return ret
}

func (p *parser) current_token() string {
	p.skip_space()

	if len(p.file) == 0 {
		panic("bad file")
	}

	ident := is_letter(p.file[0])

	for i := range p.file {
		if ident != is_letter(p.file[i]) || is_space(p.file[i]) {
			return string(p.file[:i])
		}
	}

	return string(p.file)
}

func (p *parser) finished() bool {
	p.skip_space()
	return len(p.file) == 0
}

func (p *parser) parse_production() Production {
	var line []byte

	for i := range p.file {
		if p.file[i] == '\n' {
			p.line++
			line = p.file[:i]
			p.file = p.file[i:]
			break
		}
	}

	if line == nil {
		panic("bad file")
	}

	tokens := []Token{}

	for len(line) != 0 {
		for len(line) != 0 && line[0] == ' ' {
			line = line[1:]
		}

		token_len := len(line)

		for i := range line {
			if line[i] == ' ' {
				token_len = i
				break
			}
		}

		var token Token
		if line[0] == '$' {
			token = Token{string(line[1:token_len]), ""}
		} else {
			token = Token{"", string(line[:token_len])}
		}
		tokens = append(tokens, token)
		line = line[token_len:]
	}

	return Production{tokens}
}

func (p *parser) parse_member() (m StructMember) {
	for p.accept_token("//") {
		if p.accept_token("terminator") {
			pr := p.parse_production()
			m.ArrayTerminator = &pr
		} else if p.accept_token("separator") {
			pr := p.parse_production()
			m.ArraySeparator = &pr
		} else if p.accept_token("min_length") {
			var err error
			m.ArrayMinLength, err = strconv.Atoi(string(p.consume_line()))
			if err != nil {
				panic(err)
			}
		} else {
			panic("bad file")
		}
	}

	m.Name = p.consume_token()
	p.match_token(":")

	if p.accept_token("?") {
		m.Nullable = true
	}
	if p.accept_token("[]") {
		m.Array = true
	}

	m.Type = p.consume_token()
	return
}

func (p *parser) parse_struct_type() *StructType {
	typ := new(StructType)

	for p.accept_token("//") {
		typ.Productions = append(typ.Productions, p.parse_production())
	}

	p.match_token("type")

	typ.Name = p.consume_token()

	p.match_token("=")
	p.match_token("struct")
	p.match_token("{")

	for !p.accept_token("}") {
		typ.Members = append(typ.Members, p.parse_member())
	}

	return typ
}

func (p *parser) parse_struct_type2(name string) *StructType {
	typ := new(StructType)
	typ.Name = name

	p.match_token("struct")
	p.match_token("{")

	for !p.accept_token("}") {
		typ.Members = append(typ.Members, p.parse_member())
	}

	return typ
}

func (p *parser) parse_type() Type {
	if p.current_token() == "//" {
		return p.parse_struct_type()
	}

	p.match_token("type")

	name := p.consume_token()

	if !p.accept_token("=") {
		return &LexicalType{TypeBase{name, false}}
	}

	if p.current_token() == "struct" {
		return p.parse_struct_type2(name)
	}

	typ := new(OptionType)
	typ.Name = name
	typ.Options = []string{p.consume_token()}

	for p.accept_token("|") {
		typ.Options = append(typ.Options, p.consume_token())
	}

	return typ
}
<commit_msg>Improve error reporting.<commit_after>package astgen

import (
	"strconv"
	"fmt"
)

type parser struct {
	file []byte
	line int
}

func Load(data []byte) (def *LangDef, e error) {
	var p parser
	p.line = 0
	p.file = data

	defer func() {
		err := recover()
		if err != nil {
			def = nil
			e = fmt.Errorf("Error on line %d: %s\nNext 20 bytes: %s\n", p.line, err, string(p.file[:20]))
		}
	}()

	def = new(LangDef)
	def.Types = make(map[string]Type)

	for !p.finished() {
		t := p.parse_type()
		def.Types[t.Common().Name] = t
	}

	return def, nil
}

func is_nl(b byte) bool {
	return b == '\n' || b == '\r'
}

func is_space(b byte) bool {
	return b == ' ' || b == '\t' || b == '\n' || b == '\r'
}

func is_letter(b byte) bool {
	return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_'
}

func (p *parser) skip_space() {
	for len(p.file) > 0 && is_space(p.file[0]) {
		if is_nl(p.file[0]) {
			p.line++
		}

		p.file = p.file[1:]
	}
}

func (p *parser) accept_token(token string) bool {
	p.skip_space()

	tok := []byte(token)

	if len(p.file) < len(tok) {
		return false
	}

	for i := range tok {
		if p.file[i] != tok[i] {
			return false
		}
	}

	if len(p.file) != len(tok) {
		next := p.file[len(tok)]
		if !is_space(next) && is_letter(p.file[len(tok)-1]) == is_letter(next) {
			return false
		}
	}

	p.file = p.file[len(tok):]

	return true
}

func (p *parser) match_token(token string) {
	if !p.accept_token(token) {
		panic("can't match token \"" + token + "\". Current: \"" + p.current_token() + "\"")
	}
}

func (p *parser) consume_line() string {
	p.skip_space()

	for i := range p.file {
		if p.file[i] == '\n' {
			ret := string(p.file[:i])
			p.file = p.file[i+1:]
			p.line++
			return ret
		}
	}

	ret := string(p.file)
	p.file = nil
	return ret
}

func (p *parser) consume_token() string {
	p.skip_space()

	if len(p.file) == 0 {
		panic("bad file")
	}

	ident := is_letter(p.file[0])

	for i := range p.file {
		if ident != is_letter(p.file[i]) || is_space(p.file[i]) {
			ret := string(p.file[:i])
			p.file = p.file[i:]
			return ret
		}
	}

	ret := string(p.file)
	p.file = nil
	return ret
}

func (p *parser) current_token() string {
	p.skip_space()

	if len(p.file) == 0 {
		panic("bad file")
	}

	ident := is_letter(p.file[0])

	for i := range p.file {
		if ident != is_letter(p.file[i]) || is_space(p.file[i]) {
			return string(p.file[:i])
		}
	}

	return string(p.file)
}

func (p *parser) finished() bool {
	p.skip_space()
	return len(p.file) == 0
}

func (p *parser) parse_production() Production {
	var line []byte

	for i := range p.file {
		if p.file[i] == '\n' {
			p.line++
			line = p.file[:i]
			p.file = p.file[i:]
			break
		}
	}

	if line == nil {
		panic("bad file")
	}

	tokens := []Token{}

	for len(line) != 0 {
		for len(line) != 0 && line[0] == ' ' {
			line = line[1:]
		}

		token_len := len(line)

		for i := range line {
			if line[i] == ' ' {
				token_len = i
				break
			}
		}

		var token Token
		if line[0] == '$' {
			token = Token{string(line[1:token_len]), ""}
		} else {
			token = Token{"", string(line[:token_len])}
		}
		tokens = append(tokens, token)
		line = line[token_len:]
	}

	return Production{tokens}
}

func (p *parser) parse_member() (m StructMember) {
	for p.accept_token("//") {
		if p.accept_token("terminator") {
			pr := p.parse_production()
			m.ArrayTerminator = &pr
		} else if p.accept_token("separator") {
			pr := p.parse_production()
			m.ArraySeparator = &pr
		} else if p.accept_token("min_length") {
			var err error
			m.ArrayMinLength, err = strconv.Atoi(string(p.consume_line()))
			if err != nil {
				panic(err)
			}
		} else {
			panic("bad file")
		}
	}

	m.Name = p.consume_token()
	p.match_token(":")

	if p.accept_token("?") {
		m.Nullable = true
	}
	if p.accept_token("[]") {
		m.Array = true
	}

	m.Type = p.consume_token()
	return
}

func (p *parser) parse_struct_type() *StructType {
	typ := new(StructType)

	for p.accept_token("//") {
		typ.Productions = append(typ.Productions, p.parse_production())
	}

	p.match_token("type")

	typ.Name = p.consume_token()

	p.match_token("=")
	p.match_token("struct")
	p.match_token("{")

	for !p.accept_token("}") {
		typ.Members = append(typ.Members, p.parse_member())
	}

	return typ
}

func (p *parser) parse_struct_type2(name string) *StructType {
	typ := new(StructType)
	typ.Name = name

	p.match_token("struct")
	p.match_token("{")

	for !p.accept_token("}") {
		typ.Members = append(typ.Members, p.parse_member())
	}

	return typ
}

func (p *parser) parse_type() Type {
	if p.current_token() == "//" {
		return p.parse_struct_type()
	}

	p.match_token("type")

	name := p.consume_token()

	if !p.accept_token("=") {
		return &LexicalType{TypeBase{name, false}}
	}

	if p.current_token() == "struct" {
		return p.parse_struct_type2(name)
	}

	typ := new(OptionType)
	typ.Name = name
	typ.Options = []string{p.consume_token()}

	for p.accept_token("|") {
		typ.Options = append(typ.Options, p.consume_token())
	}

	return typ
}
<|endoftext|>


<commit_before>package sqlread

import (
	"log"
)

type parser struct {
	items chan LexItem
}

func Parse(l chan LexItem) *parser {
	return &parser{
		items: l,
	}
}

func (p *parser) Run() {
	for {
		c, ok := <-p.items
		if !ok {
			break
		}

		if c.Type == TIllegal {
			log.Println("read", c.Type.String(), c.Pos, c.Val)
		}
	}
}
<commit_msg>Very rough start to the parser<commit_after>package sqlread

import (
	"fmt"
	"log"
)

type parser struct {
	items chan LexItem
	err   error
}

func Parse(l chan LexItem) *parser {
	return &parser{
		items: l,
	}
}

func (p *parser) scan() (c LexItem, ok bool) {
	c, ok = <-p.items
	return
}

func (p *parser) scanUntil(l ...lexItemType) (c LexItem, ok bool) {
	for {
		c, ok = p.scan()
		if !ok {
			return
		}

		if isOfAny(c, l...) {
			return
		}
	}
}

func (p *parser) Run() error {
	sp := SummaryParser{
		tree: make(SummaryTree),
	}

	for state := sp.parseStart; state != nil; {
		state = state(p)
	}

	return p.err
}

func (p *parser) errorUnexpectedLex(f LexItem, e lexItemType) {
	p.err = fmt.Errorf("found '%s'; expected '%s' at byte: %d", f.Type.String(), e.String(), f.Pos)
}

func (p *parser) errorUnexpectedEof() {
	p.err = fmt.Errorf("unexpected eof")
}

func isOfAny(c LexItem, l ...lexItemType) bool {
	for _, lt := range l {
		if c.Type == lt {
			return true
		}
	}

	return false
}

type SummaryColumn struct {
	Name string
	Type string
}

type SummaryTree map[string]SummaryTable

type SummaryTable struct {
	create LexItem
	cols   []*SummaryColumn
}

type SummaryParser struct {
	tree SummaryTree
}

func (t *SummaryParser) parseStart(p *parser) parseState {
	for {
		c, ok := p.scan()
		if !ok {
			break
		}

		log.Println(c, string(c.Type))

		if isOfAny(c, TComment, TSemi, TDropTableFullStmt, TLockTableFullStmt, TUnlockTablesFullStmt, TSetFullStmt) {
			continue
		}

		if isOfAny(c, TCreateTable) {
			return t.parseCreate
		}

		// spew.Dump(t.tree)
		log.Fatal("dead", c, string(c.Type))
	}
	return nil
}

func (t *SummaryParser) parseCreate(p *parser) parseState {
	c, ok := p.scan()
	if !ok {
		p.errorUnexpectedEof()
		return nil
	}
	if c.Type != TIdentifier {
		p.errorUnexpectedLex(c, TIdentifier)
		return nil
	}

	if _, ok := t.tree[c.Val]; !ok {
		t.tree[c.Val] = SummaryTable{
			create: c,
			cols:   make([]*SummaryColumn, 0),
		}
	}

	for {
		x, ok := p.scanUntil(TIdentifier, TSemi)
		if !ok {
			p.errorUnexpectedEof()
			return nil
		}

		if x.Type == TSemi {
			return t.parseStart
		}

		y, ok := p.scan()
		if !ok {
			p.errorUnexpectedEof()
			return nil
		}
		if y.Type != TColumnType {
			p.errorUnexpectedLex(y, TColumnType)
			return nil
		}

		v := t.tree[c.Val]

		v.cols = append(v.cols, &SummaryColumn{
			Name: x.Val,
			Type: y.Val,
		})

		t.tree[c.Val] = v
	}
}

type parseState func(*parser) parseState
<|endoftext|>


<commit_before>package jsonparser

import (
	"bytes"
	"errors"
	"fmt"
	"strconv"
)

// Errors
var (
	KeyPathNotFoundError       = errors.New("Key path not found")
	UnknownValueTypeError      = errors.New("Unknown value type")
	MalformedJsonError         = errors.New("Malformed JSON error")
	MalformedStringError       = errors.New("Value is string, but can't find closing '\"' symbol")
	MalformedArrayError        = errors.New("Value is array, but can't find closing ']' symbol")
	MalformedObjectError       = errors.New("Value looks like object, but can't find closing '}' symbol")
	MalformedValueError        = errors.New("Value looks like Number/Boolean/None, but can't find its end: ',' or '}' symbol")
	MalformedStringEscapeError = errors.New("Encountered an invalid escape sequence in a string")
)

// How much stack space to allocate for unescaping JSON strings; if a string longer
// than this needs to be escaped, it will result in a heap allocation
const unescapeStackBufSize = 64

func tokenEnd(data []byte) int {
	for i, c := range data {
		switch c {
		case ' ', '\n', '\r', '\t', ',', '}', ']':
			return i
		}
	}

	return -1
}

// Find position of next character which is not whitespace
func nextToken(data []byte) int {
	for i, c := range data {
		switch c {
		case ' ', '\n', '\r', '\t':
			continue
		default:
			return i
		}
	}

	return -1
}

// Tries to find the end of string
// Support if string contains escaped quote symbols.
func stringEnd(data []byte) int {
	for i, c := range data {
		if c == '"' {
			j := i - 1
			for {
				if j < 0 || data[j] != '\\' {
					return i + 1 // even number of backslashes
				}
				j--
				if j < 0 || data[j] != '\\' {
					break // odd number of backslashes
				}
				j--
			}
		}
	}

	return -1
}

// Find end of the data structure, array or object.
// For array openSym and closeSym will be '[' and ']', for object '{' and '}'
func blockEnd(data []byte, openSym byte, closeSym byte) int {
	level := 0
	i := 0
	ln := len(data)

	for i < ln {
		switch data[i] {
		case '"': // If inside string, skip it
			se := stringEnd(data[i+1:])
			if se == -1 {
				return -1
			}
			i += se
		case openSym: // If open symbol, increase level
			level++
		case closeSym: // If close symbol, increase level
			level--

			// If we have returned to the original level, we're done
			if level == 0 {
				return i + 1
			}
		}
		i++
	}

	return -1
}

func searchKeys(data []byte, keys ...string) int {
	keyLevel := 0
	level := 0
	i := 0
	ln := len(data)
	lk := len(keys)

	var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings

	for i < ln {
		switch data[i] {
		case '"':
			i++
			keyBegin := i

			strEnd := stringEnd(data[i:])
			if strEnd == -1 {
				return -1
			}
			i += strEnd
			keyEnd := i - 1

			valueOffset := nextToken(data[i:])
			if valueOffset == -1 {
				return -1
			}

			i += valueOffset

			// if string is a Key, and key level match
			if data[i] == ':' {
				key := data[keyBegin:keyEnd]

				// for unescape: if there are no escape sequences, this is cheap; if there are, it is a
				// bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize
				if keyUnesc, err := Unescape(key, stackbuf[:]); err != nil {
					return -1
				} else {
					if keyLevel == level-1 && // If key nesting level match current object nested level
						BytesEqualStr(&keyUnesc, keys[level-1]) {
						keyLevel++
						// If we found all keys in path
						if keyLevel == lk {
							return i + 1
						}
					}
				}
			} else {
				i--
			}
		case '{':
			level++
		case '}':
			level--
		case '[':
			// Do not search for keys inside arrays
			arraySkip := blockEnd(data[i:], '[', ']')
			i += arraySkip - 1
		}

		i++
	}

	return -1
}

// Data types available in valid JSON data.
type ValueType int

const (
	NotExist = ValueType(iota)
	String
	Number
	Object
	Array
	Boolean
	Null
	Unknown
)

var (
	trueLiteral  = []byte("true")
	falseLiteral = []byte("false")
	nullLiteral  = []byte("null")
)

/*
Get - Receives data structure, and key path to extract value from.

Returns:
`value` - Pointer to original data structure containing key value, or just empty slice if nothing found or error
`dataType` -    Can be: `NotExist`, `String`, `Number`, `Object`, `Array`, `Boolean` or `Null`
`offset` - Offset from provided data structure where key value ends. Used mostly internally, for example for `ArrayEach` helper.
`err` - If key not found or any other parsing issue it should return error. If key not found it also sets `dataType` to `NotExist`

Accept multiple keys to specify path to JSON value (in case of quering nested structures).
If no keys provided it will try to extract closest JSON value (simple ones or object/array), useful for reading streams or arrays, see `ArrayEach` implementation.
*/
func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) {
	if len(keys) > 0 {
		if offset = searchKeys(data, keys...); offset == -1 {
			return []byte{}, NotExist, -1, KeyPathNotFoundError
		}
	}

	// Go to closest value
	nO := nextToken(data[offset:])
	if nO == -1 {
		return []byte{}, NotExist, -1, MalformedJsonError
	}

	offset += nO

	endOffset := offset
	// if string value
	if data[offset] == '"' {
		dataType = String
		if idx := stringEnd(data[offset+1:]); idx != -1 {
			endOffset += idx + 1
		} else {
			return []byte{}, dataType, offset, MalformedStringError
		}
	} else if data[offset] == '[' { // if array value
		dataType = Array
		// break label, for stopping nested loops
		endOffset = blockEnd(data[offset:], '[', ']')

		if endOffset == -1 {
			return []byte{}, dataType, offset, MalformedArrayError
		}

		endOffset += offset
	} else if data[offset] == '{' { // if object value
		dataType = Object
		// break label, for stopping nested loops
		endOffset = blockEnd(data[offset:], '{', '}')

		if endOffset == -1 {
			return []byte{}, dataType, offset, MalformedObjectError
		}

		endOffset += offset
	} else {
		// Number, Boolean or None
		end := tokenEnd(data[endOffset:])

		if end == -1 {
			return nil, dataType, offset, MalformedValueError
		}

		value := data[offset : endOffset+end]

		switch data[offset] {
		case 't', 'f': // true or false
			if bytes.Equal(value, trueLiteral) || bytes.Equal(value, falseLiteral) {
				dataType = Boolean
			} else {
				return nil, Unknown, offset, UnknownValueTypeError
			}
		case 'u', 'n': // undefined or null
			if bytes.Equal(value, nullLiteral) {
				dataType = Null
			} else {
				return nil, Unknown, offset, UnknownValueTypeError
			}
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-':
			dataType = Number
		default:
			return nil, Unknown, offset, UnknownValueTypeError
		}

		endOffset += end
	}

	value = data[offset:endOffset]

	// Strip quotes from string values
	if dataType == String {
		value = value[1 : len(value)-1]
	}

	if dataType == Null {
		value = []byte{}
	}

	return value, dataType, endOffset, nil
}

// ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`.
func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int, err error), keys ...string) (err error) {
	if len(data) == 0 {
		return MalformedObjectError
	}

	offset := 1

	if len(keys) > 0 {
		if offset = searchKeys(data, keys...); offset == -1 {
			return KeyPathNotFoundError
		}

		// Go to closest value
		nO := nextToken(data[offset:])
		if nO == -1 {
			return MalformedJsonError
		}

		offset += nO

		if data[offset] != '[' {
			return MalformedArrayError
		}

		offset++
	}

	for true {
		v, t, o, e := Get(data[offset:])

		if o == 0 {
			break
		}

		if t != NotExist {
			cb(v, t, o, e)
		}

		if e != nil {
			break
		}

		offset += o

		skipToToken := nextToken(data[offset:])
		if skipToToken == -1 {
			return MalformedArrayError
		}
		offset += skipToToken

		if data[offset] == ']' {
			break
		}

		if data[offset] != ',' {
			return MalformedArrayError
		}

		offset++
	}

	return nil
}

// GetString returns the value retrieved by `Get`, cast to a string if possible, trying to properly handle escape and utf8 symbols
// If key data type do not match, it will return an error.
func GetString(data []byte, keys ...string) (val string, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return "", e
	}

	if t != String {
		return "", fmt.Errorf("Value is not a number: %s", string(v))
	}

	// If no escapes return raw conten
	if bytes.IndexByte(v, '\\') == -1 {
		return string(v), nil
	}

	var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings
	out, err := Unescape(v, stackbuf[:])

	return string(out), err
}

// GetFloat returns the value retrieved by `Get`, cast to a float64 if possible.
// The offset is the same as in `Get`.
// If key data type do not match, it will return an error.
func GetFloat(data []byte, keys ...string) (val float64, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return 0, e
	}

	if t != Number {
		return 0, fmt.Errorf("Value is not a number: %s", string(v))
	}

	val, err = BytesParseFloat(&v, 64)
	return
}

// GetInt returns the value retrieved by `Get`, cast to a float64 if possible.
// If key data type do not match, it will return an error.
func GetInt(data []byte, keys ...string) (val int64, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return 0, e
	}

	if t != Number {
		return 0, fmt.Errorf("Value is not a number: %s", string(v))
	}

	if val, ok := BytesParseInt(v); !ok {
		return 0, MalformedValueError
	} else {
		return val, nil
	}
}

// GetBoolean returns the value retrieved by `Get`, cast to a bool if possible.
// The offset is the same as in `Get`.
// If key data type do not match, it will return error.
func GetBoolean(data []byte, keys ...string) (val bool, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return false, e
	}

	if t != Boolean {
		return false, fmt.Errorf("Value is not a boolean: %s", string(v))
	}

	if v[0] == 't' {
		val = true
	} else {
		val = false
	}

	return
}

// ParseBoolean parses a Boolean ValueType into a Go bool (not particularly useful, but here for completeness)
func ParseBoolean(vbytes []byte) bool {
	return (vbytes[0] == 't') // assumes value is already validated by Get(), etc. as signaled by jtype == Boolean
}

// ParseString parses a String ValueType into a Go []byte (the main parsing work is unescaping the JSON string)
func ParseStringAsBytes(vbytes []byte) ([]byte, error) {
	var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings (hopefully; the Go compiler might just always kick stackbuf[:] into the heap)
	return Unescape(vbytes, stackbuf[:])
}

// ParseString parses a String ValueType into a Go string (the main parsing work is unescaping the JSON string)
func ParseString(vbytes []byte) (string, error) {
	if vbytesUnesc, err := ParseStringAsBytes(vbytes); err != nil {
		return "", nil
	} else {
		return string(vbytesUnesc), nil
	}
}

// ParseNumber parses a Number ValueType into a Go float64
func ParseNumber(vbytes []byte) (float64, error) {
	if v, err := strconv.ParseFloat(unsafeBytesToString(vbytes), 64); err != nil { // TODO: use better BytesParseFloat in PR #25
		return 0, MalformedValueError
	} else {
		return v, nil
	}
}
<commit_msg>Updated ParseNumber to use new BytesParseFloat<commit_after>package jsonparser

import (
	"bytes"
	"errors"
	"fmt"
)

// Errors
var (
	KeyPathNotFoundError       = errors.New("Key path not found")
	UnknownValueTypeError      = errors.New("Unknown value type")
	MalformedJsonError         = errors.New("Malformed JSON error")
	MalformedStringError       = errors.New("Value is string, but can't find closing '\"' symbol")
	MalformedArrayError        = errors.New("Value is array, but can't find closing ']' symbol")
	MalformedObjectError       = errors.New("Value looks like object, but can't find closing '}' symbol")
	MalformedValueError        = errors.New("Value looks like Number/Boolean/None, but can't find its end: ',' or '}' symbol")
	MalformedStringEscapeError = errors.New("Encountered an invalid escape sequence in a string")
)

// How much stack space to allocate for unescaping JSON strings; if a string longer
// than this needs to be escaped, it will result in a heap allocation
const unescapeStackBufSize = 64

func tokenEnd(data []byte) int {
	for i, c := range data {
		switch c {
		case ' ', '\n', '\r', '\t', ',', '}', ']':
			return i
		}
	}

	return -1
}

// Find position of next character which is not whitespace
func nextToken(data []byte) int {
	for i, c := range data {
		switch c {
		case ' ', '\n', '\r', '\t':
			continue
		default:
			return i
		}
	}

	return -1
}

// Tries to find the end of string
// Support if string contains escaped quote symbols.
func stringEnd(data []byte) int {
	for i, c := range data {
		if c == '"' {
			j := i - 1
			for {
				if j < 0 || data[j] != '\\' {
					return i + 1 // even number of backslashes
				}
				j--
				if j < 0 || data[j] != '\\' {
					break // odd number of backslashes
				}
				j--
			}
		}
	}

	return -1
}

// Find end of the data structure, array or object.
// For array openSym and closeSym will be '[' and ']', for object '{' and '}'
func blockEnd(data []byte, openSym byte, closeSym byte) int {
	level := 0
	i := 0
	ln := len(data)

	for i < ln {
		switch data[i] {
		case '"': // If inside string, skip it
			se := stringEnd(data[i+1:])
			if se == -1 {
				return -1
			}
			i += se
		case openSym: // If open symbol, increase level
			level++
		case closeSym: // If close symbol, increase level
			level--

			// If we have returned to the original level, we're done
			if level == 0 {
				return i + 1
			}
		}
		i++
	}

	return -1
}

func searchKeys(data []byte, keys ...string) int {
	keyLevel := 0
	level := 0
	i := 0
	ln := len(data)
	lk := len(keys)

	var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings

	for i < ln {
		switch data[i] {
		case '"':
			i++
			keyBegin := i

			strEnd := stringEnd(data[i:])
			if strEnd == -1 {
				return -1
			}
			i += strEnd
			keyEnd := i - 1

			valueOffset := nextToken(data[i:])
			if valueOffset == -1 {
				return -1
			}

			i += valueOffset

			// if string is a Key, and key level match
			if data[i] == ':' {
				key := data[keyBegin:keyEnd]

				// for unescape: if there are no escape sequences, this is cheap; if there are, it is a
				// bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize
				if keyUnesc, err := Unescape(key, stackbuf[:]); err != nil {
					return -1
				} else {
					if keyLevel == level-1 && // If key nesting level match current object nested level
						BytesEqualStr(&keyUnesc, keys[level-1]) {
						keyLevel++
						// If we found all keys in path
						if keyLevel == lk {
							return i + 1
						}
					}
				}
			} else {
				i--
			}
		case '{':
			level++
		case '}':
			level--
		case '[':
			// Do not search for keys inside arrays
			arraySkip := blockEnd(data[i:], '[', ']')
			i += arraySkip - 1
		}

		i++
	}

	return -1
}

// Data types available in valid JSON data.
type ValueType int

const (
	NotExist = ValueType(iota)
	String
	Number
	Object
	Array
	Boolean
	Null
	Unknown
)

var (
	trueLiteral  = []byte("true")
	falseLiteral = []byte("false")
	nullLiteral  = []byte("null")
)

/*
Get - Receives data structure, and key path to extract value from.

Returns:
`value` - Pointer to original data structure containing key value, or just empty slice if nothing found or error
`dataType` -    Can be: `NotExist`, `String`, `Number`, `Object`, `Array`, `Boolean` or `Null`
`offset` - Offset from provided data structure where key value ends. Used mostly internally, for example for `ArrayEach` helper.
`err` - If key not found or any other parsing issue it should return error. If key not found it also sets `dataType` to `NotExist`

Accept multiple keys to specify path to JSON value (in case of quering nested structures).
If no keys provided it will try to extract closest JSON value (simple ones or object/array), useful for reading streams or arrays, see `ArrayEach` implementation.
*/
func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) {
	if len(keys) > 0 {
		if offset = searchKeys(data, keys...); offset == -1 {
			return []byte{}, NotExist, -1, KeyPathNotFoundError
		}
	}

	// Go to closest value
	nO := nextToken(data[offset:])
	if nO == -1 {
		return []byte{}, NotExist, -1, MalformedJsonError
	}

	offset += nO

	endOffset := offset
	// if string value
	if data[offset] == '"' {
		dataType = String
		if idx := stringEnd(data[offset+1:]); idx != -1 {
			endOffset += idx + 1
		} else {
			return []byte{}, dataType, offset, MalformedStringError
		}
	} else if data[offset] == '[' { // if array value
		dataType = Array
		// break label, for stopping nested loops
		endOffset = blockEnd(data[offset:], '[', ']')

		if endOffset == -1 {
			return []byte{}, dataType, offset, MalformedArrayError
		}

		endOffset += offset
	} else if data[offset] == '{' { // if object value
		dataType = Object
		// break label, for stopping nested loops
		endOffset = blockEnd(data[offset:], '{', '}')

		if endOffset == -1 {
			return []byte{}, dataType, offset, MalformedObjectError
		}

		endOffset += offset
	} else {
		// Number, Boolean or None
		end := tokenEnd(data[endOffset:])

		if end == -1 {
			return nil, dataType, offset, MalformedValueError
		}

		value := data[offset : endOffset+end]

		switch data[offset] {
		case 't', 'f': // true or false
			if bytes.Equal(value, trueLiteral) || bytes.Equal(value, falseLiteral) {
				dataType = Boolean
			} else {
				return nil, Unknown, offset, UnknownValueTypeError
			}
		case 'u', 'n': // undefined or null
			if bytes.Equal(value, nullLiteral) {
				dataType = Null
			} else {
				return nil, Unknown, offset, UnknownValueTypeError
			}
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-':
			dataType = Number
		default:
			return nil, Unknown, offset, UnknownValueTypeError
		}

		endOffset += end
	}

	value = data[offset:endOffset]

	// Strip quotes from string values
	if dataType == String {
		value = value[1 : len(value)-1]
	}

	if dataType == Null {
		value = []byte{}
	}

	return value, dataType, endOffset, nil
}

// ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`.
func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int, err error), keys ...string) (err error) {
	if len(data) == 0 {
		return MalformedObjectError
	}

	offset := 1

	if len(keys) > 0 {
		if offset = searchKeys(data, keys...); offset == -1 {
			return KeyPathNotFoundError
		}

		// Go to closest value
		nO := nextToken(data[offset:])
		if nO == -1 {
			return MalformedJsonError
		}

		offset += nO

		if data[offset] != '[' {
			return MalformedArrayError
		}

		offset++
	}

	for true {
		v, t, o, e := Get(data[offset:])

		if o == 0 {
			break
		}

		if t != NotExist {
			cb(v, t, o, e)
		}

		if e != nil {
			break
		}

		offset += o

		skipToToken := nextToken(data[offset:])
		if skipToToken == -1 {
			return MalformedArrayError
		}
		offset += skipToToken

		if data[offset] == ']' {
			break
		}

		if data[offset] != ',' {
			return MalformedArrayError
		}

		offset++
	}

	return nil
}

// GetString returns the value retrieved by `Get`, cast to a string if possible, trying to properly handle escape and utf8 symbols
// If key data type do not match, it will return an error.
func GetString(data []byte, keys ...string) (val string, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return "", e
	}

	if t != String {
		return "", fmt.Errorf("Value is not a number: %s", string(v))
	}

	// If no escapes return raw conten
	if bytes.IndexByte(v, '\\') == -1 {
		return string(v), nil
	}

	var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings
	out, err := Unescape(v, stackbuf[:])

	return string(out), err
}

// GetFloat returns the value retrieved by `Get`, cast to a float64 if possible.
// The offset is the same as in `Get`.
// If key data type do not match, it will return an error.
func GetFloat(data []byte, keys ...string) (val float64, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return 0, e
	}

	if t != Number {
		return 0, fmt.Errorf("Value is not a number: %s", string(v))
	}

	val, err = BytesParseFloat(&v, 64)
	return
}

// GetInt returns the value retrieved by `Get`, cast to a float64 if possible.
// If key data type do not match, it will return an error.
func GetInt(data []byte, keys ...string) (val int64, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return 0, e
	}

	if t != Number {
		return 0, fmt.Errorf("Value is not a number: %s", string(v))
	}

	if val, ok := BytesParseInt(v); !ok {
		return 0, MalformedValueError
	} else {
		return val, nil
	}
}

// GetBoolean returns the value retrieved by `Get`, cast to a bool if possible.
// The offset is the same as in `Get`.
// If key data type do not match, it will return error.
func GetBoolean(data []byte, keys ...string) (val bool, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return false, e
	}

	if t != Boolean {
		return false, fmt.Errorf("Value is not a boolean: %s", string(v))
	}

	if v[0] == 't' {
		val = true
	} else {
		val = false
	}

	return
}

// ParseBoolean parses a Boolean ValueType into a Go bool (not particularly useful, but here for completeness)
func ParseBoolean(vbytes []byte) bool {
	return (vbytes[0] == 't') // assumes value is already validated by Get(), etc. as signaled by jtype == Boolean
}

// ParseString parses a String ValueType into a Go []byte (the main parsing work is unescaping the JSON string)
func ParseStringAsBytes(vbytes []byte) ([]byte, error) {
	var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings (hopefully; the Go compiler might just always kick stackbuf[:] into the heap)
	return Unescape(vbytes, stackbuf[:])
}

// ParseString parses a String ValueType into a Go string (the main parsing work is unescaping the JSON string)
func ParseString(vbytes []byte) (string, error) {
	if vbytesUnesc, err := ParseStringAsBytes(vbytes); err != nil {
		return "", nil
	} else {
		return string(vbytesUnesc), nil
	}
}

// ParseNumber parses a Number ValueType into a Go float64
func ParseNumber(vbytes []byte) (float64, error) {
	if v, err := BytesParseFloat(&vbytes, 64); err != nil {
		return 0, MalformedValueError
	} else {
		return v, nil
	}
}
<|endoftext|>


<commit_before>package display

import (
	"fmt"
	"io"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/nanobox-io/nanobox/util"
)

// ...
var (
	EscSeqRegex   = regexp.MustCompile("\\x1b[[][?0123456789]*;?[?0123456789]*[ABCDEFGHJKRSTfminsulhp]")
	LogStripRegex = regexp.MustCompile("^[ \t-]*")
	termWidth     = 0
)

type (

	// Summarizer ...
	Summarizer struct {
		Label  string    // the task label to print as the header
		Prefix string    // the prefix to prepend to the summary
		Out    io.Writer // writer to send output to

		// internal
		chEvent     chan *sEventOp // channel to receive stop/error/tick events
		chLog       chan string    // channel to receive logs
		spinIdx     int            // track the current spinner index
		ticker      *time.Ticker   // timer to send tick events at an interval
		detail      string         // the current line of detail to print
		shutdown    bool           // toggle to inform the run loop to exit
		windowWidth int
		leftover    string
	}

	// Sending events to the summarizer needs to block the caller until
	// the event handler has completed. To do this we need to create
	// a stateful event with an ad-hoc channel to receive the response
	sEventOp struct {
		action string    // the requested action (stop/error/tick/etc)
		res    chan bool // the response channel to syncronize state
	}
)

// NewSummarizer returns a new Summarizer
func NewSummarizer(label string, prefix string) *Summarizer {
	if termWidth == 0 {
		_, termWidth = util.GetTerminalSize()
	}

	return &Summarizer{
		Label:  label,
		Prefix: prefix,
		Out:    os.Stdout,

		chEvent:     make(chan *sEventOp),   // no buffering, block the sending process
		chLog:       make(chan string, 100), // buffer up to 100 log messages before blocking
		windowWidth: termWidth,
	}
}

// Start starts the summary process in a goroutine
func (s *Summarizer) Start() {
	go s.run()
}

// Pause ...
func (s *Summarizer) Pause() {
	// create the 'stop' event
	event := &sEventOp{
		action: "pause",
		res:    make(chan bool),
	}

	// send the event
	s.chEvent <- event

	// now wait until we get a response back
	<-event.res
}

// Resume prints the "complete" label and toggles shutdown
func (s *Summarizer) Resume() {
	// generate and print the complete header
	s.Label = s.Label

	// turn the ticker back on
	s.shutdown = false
	go s.run()
}

// Stop stops the summary process
func (s *Summarizer) Stop() {
	// create the 'stop' event
	event := &sEventOp{
		action: "stop",
		res:    make(chan bool),
	}

	// send the event
	s.chEvent <- event

	// now wait until we get a response back
	<-event.res
}

// Error will stop the summary process and print an error header
func (s *Summarizer) Error() {
	// create the 'error' event
	event := &sEventOp{
		action: "error",
		res:    make(chan bool),
	}

	// send the event
	s.chEvent <- event

	// now wait until we get a response back
	<-event.res
}

// Log sends a log message to the summary process
func (s *Summarizer) Log(msg string) {
	// send the message to the log channel
	s.chLog <- msg
}

// run runs the main loop to wait for events or timers
func (s *Summarizer) run() {

	// to kick things off, we need to print the first summary
	s.print()

	// start a ticker to send a "tick" event every 80 milliseconds
	s.startTicker()

	for !s.shutdown {
		select {
		case event := <-s.chEvent:
			s.handleEvent(event)
		case msg := <-s.chLog:
			s.handleLog(msg)
		}
	}

	// stop the ticker
	s.stopTicker()
}

// startTicker starts a ticker to send a tick event
func (s *Summarizer) startTicker() {
	s.ticker = time.NewTicker(time.Millisecond * 80)

	// send a "tick" event at each interval
	go func() {
		for range s.ticker.C {
			// create the 'tick' event
			event := &sEventOp{
				action: "tick",
				res:    make(chan bool),
			}

			// send the event
			s.chEvent <- event

			// now wait until we get a response back
			<-event.res
		}
	}()
}

// stopTicker stops the tick event ticker
func (s *Summarizer) stopTicker() {
	s.ticker.Stop()
}

// handleEvent dispatches actions for events received on the event channel
func (s *Summarizer) handleEvent(event *sEventOp) {

	switch event.action {
	case "tick":
		s.tick()
	case "stop":
		s.stop()
	case "pause":
		s.pause()
	case "error":
		s.error()
	}

	// send the response so the caller can continue on
	event.res <- true
}

// handleLog sets the detail line and refreshes the summary
func (s *Summarizer) handleLog(data string) {

	msg := s.leftover
	s.leftover = ""

	// a function to decide where to split
	f := func(c rune) bool {
		return c == '\n' || c == '\r'
	}

	// iterate through the lines, we'll keep the last line that has data
	lines := strings.FieldsFunc(msg+data, f)
	for i, line := range lines {
		// detect if this is the last line and its unfinished.
		if len(lines) == i+1 && (!strings.HasSuffix(line, "\n") || !strings.HasSuffix(line, "\r")) {
			continue
		}

		// first we need to remove escape sequences
		line = EscSeqRegex.ReplaceAllString(line, "")

		// then we need to remove any leading whitespace
		line = LogStripRegex.ReplaceAllString(line, "")

		if len(line) == 0 {
			continue
		}

		availableLen := s.windowWidth - (len(s.Label) + len(s.Prefix) + 5)
		if s.windowWidth > 0 && len(msg) > availableLen {
			msg = msg[:availableLen] + "..."
		}

		s.detail = msg
		s.reset()
		s.print()
	}

	// if there is any data and it doesnt end with a newline
	if len(lines) > 0 && (!strings.HasSuffix(data, "\n") || !strings.HasSuffix(data, "\r")) {
		// place it in the leftovers
		s.leftover = lines[len(lines)-1]
	}

}

// tick updates the spinner index and refreshes the summary
func (s *Summarizer) tick() {
	// update the spinner index
	s.spinIdx++

	// reset the index back to 0 if we've reached the top
	if s.spinIdx == len(TaskSpinner) {
		s.spinIdx = 0
	}

	// reset and print the screen
	s.reset()
	s.print()
}

// stop prints the "complete" label and toggles shutdown
func (s *Summarizer) stop() {
	// reset the screen
	s.reset()

	// generate and print the complete header
	header := fmt.Sprintf("%s%s %s\n", s.Prefix, TaskComplete, s.Label)
	io.WriteString(s.Out, header)

	// set the shutdown flag to ensure the loop ends
	s.shutdown = true
}

// stop prints the "complete" label and toggles shutdown
func (s *Summarizer) pause() {
	// reset the screen
	s.reset()

	// generate and print the complete header
	header := fmt.Sprintf("%s%s %s\n", s.Prefix, TaskPause, s.Label)
	io.WriteString(s.Out, header)

	// set the shutdown flag to ensure the loop ends
	s.shutdown = true
}

// error prints the "error" label and toggles shutdown
func (s *Summarizer) error() {
	// reset the screen
	s.reset()

	// generate and print the complete header
	header := fmt.Sprintf("%s! %s\n", s.Prefix, s.Label)
	io.WriteString(s.Out, header)

	// set the shutdown flag to ensure the loop ends
	s.shutdown = true
}

// reset will reset the screen using escape sequences
func (s *Summarizer) reset() {
	// http://bluesock.org/~willg/dev/ansi.html

	// todo: make this conditional on the progress estimator
	lines := 2

	// create escape sequence to move up a line and clear the line for each line
	reset := strings.Repeat("\x1b[1A\x1b[K", lines)

	io.WriteString(s.Out, reset)
}

// print prints the current summary
func (s *Summarizer) print() {

	header := fmt.Sprintf("%s%s %s :\n", s.Prefix, TaskSpinner[s.spinIdx], s.Label)
	detail := fmt.Sprintf("%s  %s\n", s.Prefix, s.detail)

	// todo: add progress estimator

	io.WriteString(s.Out, header)
	io.WriteString(s.Out, detail)
}
<commit_msg>detected a potential typo<commit_after>package display

import (
	"fmt"
	"io"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/nanobox-io/nanobox/util"
)

// ...
var (
	EscSeqRegex   = regexp.MustCompile("\\x1b[[][?0123456789]*;?[?0123456789]*[ABCDEFGHJKRSTfminsulhp]")
	LogStripRegex = regexp.MustCompile("^[ \t-]*")
	termWidth     = 0
)

type (

	// Summarizer ...
	Summarizer struct {
		Label  string    // the task label to print as the header
		Prefix string    // the prefix to prepend to the summary
		Out    io.Writer // writer to send output to

		// internal
		chEvent     chan *sEventOp // channel to receive stop/error/tick events
		chLog       chan string    // channel to receive logs
		spinIdx     int            // track the current spinner index
		ticker      *time.Ticker   // timer to send tick events at an interval
		detail      string         // the current line of detail to print
		shutdown    bool           // toggle to inform the run loop to exit
		windowWidth int
		leftover    string
	}

	// Sending events to the summarizer needs to block the caller until
	// the event handler has completed. To do this we need to create
	// a stateful event with an ad-hoc channel to receive the response
	sEventOp struct {
		action string    // the requested action (stop/error/tick/etc)
		res    chan bool // the response channel to syncronize state
	}
)

// NewSummarizer returns a new Summarizer
func NewSummarizer(label string, prefix string) *Summarizer {
	if termWidth == 0 {
		_, termWidth = util.GetTerminalSize()
	}

	return &Summarizer{
		Label:  label,
		Prefix: prefix,
		Out:    os.Stdout,

		chEvent:     make(chan *sEventOp),   // no buffering, block the sending process
		chLog:       make(chan string, 100), // buffer up to 100 log messages before blocking
		windowWidth: termWidth,
	}
}

// Start starts the summary process in a goroutine
func (s *Summarizer) Start() {
	go s.run()
}

// Pause ...
func (s *Summarizer) Pause() {
	// create the 'stop' event
	event := &sEventOp{
		action: "pause",
		res:    make(chan bool),
	}

	// send the event
	s.chEvent <- event

	// now wait until we get a response back
	<-event.res
}

// Resume prints the "complete" label and toggles shutdown
func (s *Summarizer) Resume() {
	// generate and print the complete header
	s.Label = s.Label

	// turn the ticker back on
	s.shutdown = false
	go s.run()
}

// Stop stops the summary process
func (s *Summarizer) Stop() {
	// create the 'stop' event
	event := &sEventOp{
		action: "stop",
		res:    make(chan bool),
	}

	// send the event
	s.chEvent <- event

	// now wait until we get a response back
	<-event.res
}

// Error will stop the summary process and print an error header
func (s *Summarizer) Error() {
	// create the 'error' event
	event := &sEventOp{
		action: "error",
		res:    make(chan bool),
	}

	// send the event
	s.chEvent <- event

	// now wait until we get a response back
	<-event.res
}

// Log sends a log message to the summary process
func (s *Summarizer) Log(msg string) {
	// send the message to the log channel
	s.chLog <- msg
}

// run runs the main loop to wait for events or timers
func (s *Summarizer) run() {

	// to kick things off, we need to print the first summary
	s.print()

	// start a ticker to send a "tick" event every 80 milliseconds
	s.startTicker()

	for !s.shutdown {
		select {
		case event := <-s.chEvent:
			s.handleEvent(event)
		case msg := <-s.chLog:
			s.handleLog(msg)
		}
	}

	// stop the ticker
	s.stopTicker()
}

// startTicker starts a ticker to send a tick event
func (s *Summarizer) startTicker() {
	s.ticker = time.NewTicker(time.Millisecond * 80)

	// send a "tick" event at each interval
	go func() {
		for range s.ticker.C {
			// create the 'tick' event
			event := &sEventOp{
				action: "tick",
				res:    make(chan bool),
			}

			// send the event
			s.chEvent <- event

			// now wait until we get a response back
			<-event.res
		}
	}()
}

// stopTicker stops the tick event ticker
func (s *Summarizer) stopTicker() {
	s.ticker.Stop()
}

// handleEvent dispatches actions for events received on the event channel
func (s *Summarizer) handleEvent(event *sEventOp) {

	switch event.action {
	case "tick":
		s.tick()
	case "stop":
		s.stop()
	case "pause":
		s.pause()
	case "error":
		s.error()
	}

	// send the response so the caller can continue on
	event.res <- true
}

// handleLog sets the detail line and refreshes the summary
func (s *Summarizer) handleLog(data string) {

	msg := s.leftover
	s.leftover = ""

	// a function to decide where to split
	f := func(c rune) bool {
		return c == '\n' || c == '\r'
	}

	// iterate through the lines, we'll keep the last line that has data
	lines := strings.FieldsFunc(msg+data, f)
	for i, line := range lines {
		// detect if this is the last line and its unfinished.
		if len(lines) == i+1 && (!strings.HasSuffix(line, "\n") || !strings.HasSuffix(line, "\r")) {
			continue
		}

		// first we need to remove escape sequences
		line = EscSeqRegex.ReplaceAllString(line, "")

		// then we need to remove any leading whitespace
		line = LogStripRegex.ReplaceAllString(line, "")

		if len(line) == 0 {
			continue
		}

		availableLen := s.windowWidth - (len(s.Label) + len(s.Prefix) + 5)
		if s.windowWidth > 0 && len(line) > availableLen {
			line = line[:availableLen] + "..."
		}

		s.detail = line
		s.reset()
		s.print()
	}

	// if there is any data and it doesnt end with a newline
	if len(lines) > 0 && (!strings.HasSuffix(data, "\n") || !strings.HasSuffix(data, "\r")) {
		// place it in the leftovers
		s.leftover = lines[len(lines)-1]
	}

}

// tick updates the spinner index and refreshes the summary
func (s *Summarizer) tick() {
	// update the spinner index
	s.spinIdx++

	// reset the index back to 0 if we've reached the top
	if s.spinIdx == len(TaskSpinner) {
		s.spinIdx = 0
	}

	// reset and print the screen
	s.reset()
	s.print()
}

// stop prints the "complete" label and toggles shutdown
func (s *Summarizer) stop() {
	// reset the screen
	s.reset()

	// generate and print the complete header
	header := fmt.Sprintf("%s%s %s\n", s.Prefix, TaskComplete, s.Label)
	io.WriteString(s.Out, header)

	// set the shutdown flag to ensure the loop ends
	s.shutdown = true
}

// stop prints the "complete" label and toggles shutdown
func (s *Summarizer) pause() {
	// reset the screen
	s.reset()

	// generate and print the complete header
	header := fmt.Sprintf("%s%s %s\n", s.Prefix, TaskPause, s.Label)
	io.WriteString(s.Out, header)

	// set the shutdown flag to ensure the loop ends
	s.shutdown = true
}

// error prints the "error" label and toggles shutdown
func (s *Summarizer) error() {
	// reset the screen
	s.reset()

	// generate and print the complete header
	header := fmt.Sprintf("%s! %s\n", s.Prefix, s.Label)
	io.WriteString(s.Out, header)

	// set the shutdown flag to ensure the loop ends
	s.shutdown = true
}

// reset will reset the screen using escape sequences
func (s *Summarizer) reset() {
	// http://bluesock.org/~willg/dev/ansi.html

	// todo: make this conditional on the progress estimator
	lines := 2

	// create escape sequence to move up a line and clear the line for each line
	reset := strings.Repeat("\x1b[1A\x1b[K", lines)

	io.WriteString(s.Out, reset)
}

// print prints the current summary
func (s *Summarizer) print() {

	header := fmt.Sprintf("%s%s %s :\n", s.Prefix, TaskSpinner[s.spinIdx], s.Label)
	detail := fmt.Sprintf("%s  %s\n", s.Prefix, s.detail)

	// todo: add progress estimator

	io.WriteString(s.Out, header)
	io.WriteString(s.Out, detail)
}
<|endoftext|>


<commit_before>package convo

import (
	"github.com/synapse-garden/sg-proto/store"
	"github.com/synapse-garden/sg-proto/stream"
)

// Connected is a Resourcer which can notify that a user has connected.
type Connected stream.ConnectionNotif

// Resource implements Resourcer.Resource on Connected.
func (Connected) Resource() store.Resource { return "convo-connected" }

// Disconnected is a Resourcer which can notify that a user has
// disconnected.
type Disconnected stream.ConnectionNotif

// Resource implements Resourcer.Resource on Disconnected.
func (Disconnected) Resource() store.Resource { return "convo-disconnected" }

// Connected returns a store.Resourcer which can notify that a user
// has connected.
func (c *Convo) Connected(user string) store.Resourcer {
	return Connected{
		UserID:   user,
		StreamID: c.ID,
	}
}

// Disconnected returns a store.Resourcer which can notify that a user
// has disconnected.
func (c *Convo) Disconnected(user string) store.Resourcer {
	return Disconnected{
		UserID:   user,
		StreamID: c.ID,
	}
}

// Deleteed is a Resourcer which can notify that the convo has been
// deleted.
type Deleted string

// Resource implements Resourcer.Resource on Deleted.
func (Deleted) Resource() store.Resource { return "convo-deleted" }
<commit_msg>Use ConvoID instead of StreamID for convo notifs<commit_after>package convo

import "github.com/synapse-garden/sg-proto/store"

// ConnectionNotif is a base for convo Resourcers to create notifs.
// Implement store.Resourcer as a method on an alias of ConnectionNotif.
type ConnectionNotif struct {
	UserID  string `json:"userID"`
	ConvoID string `json:"convoID"`
}

// Connected is a ConnectionNotif for convo connection events
type Connected ConnectionNotif

// Resource implements Resourcer.Resource on Connected.
func (Connected) Resource() store.Resource { return "convo-connected" }

// Disconnected is a Resourcer which can notify that a user has
// disconnected.
type Disconnected ConnectionNotif

// Resource implements Resourcer.Resource on Disconnected.
func (Disconnected) Resource() store.Resource { return "convo-disconnected" }

// Connected returns a store.Resourcer which can notify that a user
// has connected.
func (c *Convo) Connected(user string) store.Resourcer {
	return Connected{
		UserID:   user,
		ConvoID: c.ID,
	}
}

// Disconnected returns a store.Resourcer which can notify that a user
// has disconnected.
func (c *Convo) Disconnected(user string) store.Resourcer {
	return Disconnected{
		UserID:   user,
		ConvoID: c.ID,
	}
}

// Deleted is a Resourcer which can notify that the convo has been
// deleted.
type Deleted string

// Resource implements Resourcer.Resource on Deleted.
func (Deleted) Resource() store.Resource { return "convo-deleted" }
<|endoftext|>


<commit_before>// Application that builds and isolates telemetry.
package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/ct/go/worker_scripts/worker_common"
	"go.skia.org/infra/go/cas/rbe"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/isolate"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
)

var (
	runID          = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	chromiumHash   = flag.String("chromium_hash", "", "Chromium repo will be synced to this hash if specified. Optional.")
	patches        = flag.String("patches", "", "Comma separated names of patches to apply to the specified repo. Optional.")
	targetPlatform = flag.String("target_platform", util.PLATFORM_LINUX, "The platform we want to build for.")
	outDir         = flag.String("out", "", "The out directory where the isolate hash will be stored.")
)

func buildRepo() error {
	ctx := context.Background()
	httpClient, err := worker_common.Init(ctx, true /* useDepotTools */)
	if err != nil {
		return skerr.Wrap(err)
	}
	defer util.TimeTrack(time.Now(), "Isolating Telemetry")
	defer sklog.Flush()

	// Validate required arguments.
	if *runID == "" {
		return errors.New("Must specify --run_id")
	}
	if *outDir == "" {
		return errors.New("Must specify --out")
	}

	// Find git exec.
	gitExec, err := git.Executable(ctx)
	if err != nil {
		return skerr.Wrap(err)
	}

	// Instantiate GcsUtil object.
	gs, err := util.NewGcsUtil(httpClient)
	if err != nil {
		return err
	}

	// Apply patches if specified and create the telemetry isolates.
	applyPatches := false
	if *patches != "" {
		applyPatches = true
		for _, patch := range strings.Split(*patches, ",") {
			patchName := path.Base(patch)
			patchLocalPath := filepath.Join(os.TempDir(), patchName)
			if _, err := util.DownloadPatch(patchLocalPath, patch, gs); err != nil {
				return err
			}
		}
	}
	pathToPyFiles, err := util.GetPathToPyFiles(*worker_common.Local)
	if err != nil {
		return fmt.Errorf("Could not get path to py files: %s", err)
	}
	if err = util.CreateTelemetryIsolates(ctx, *runID, *targetPlatform, *chromiumHash, pathToPyFiles, gitExec, applyPatches); err != nil {
		return fmt.Errorf("Could not create telemetry isolates: %s", err)
	}

	buildOutDir := filepath.Join(util.ChromiumSrcDir, util.TELEMETRY_ISOLATES_OUT_DIR)
	isolateFile := filepath.Join(buildOutDir, fmt.Sprintf("%s.isolate", util.TELEMETRY_ISOLATES_TARGET))

	// Isolate telemetry artifacts.
	hash, err := isolate.Upload(ctx, rbe.InstanceChromeSwarming, buildOutDir, isolateFile)
	if err != nil {
		return fmt.Errorf("Could not isolate telemetry task: %s", err)
	}

	// Record the isolate hash in the output file.
	hashOutputFile := filepath.Join(*outDir, util.ISOLATE_TELEMETRY_FILENAME)
	if err := ioutil.WriteFile(hashOutputFile, []byte(hash), os.ModePerm); err != nil {
		return fmt.Errorf("Could not write to %s: %s", hashOutputFile, err)
	}

	return nil
}

func main() {
	retCode := 0
	if err := buildRepo(); err != nil {
		sklog.Errorf("Error while building repo: %s", err)
		retCode = 255
	}
	os.Exit(retCode)
}
<commit_msg>[CT] Fix isolate_telemetry on windows<commit_after>// Application that builds and isolates telemetry.
package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/ct/go/worker_scripts/worker_common"
	"go.skia.org/infra/go/cas/rbe"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/isolate"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
)

var (
	runID          = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	chromiumHash   = flag.String("chromium_hash", "", "Chromium repo will be synced to this hash if specified. Optional.")
	patches        = flag.String("patches", "", "Comma separated names of patches to apply to the specified repo. Optional.")
	targetPlatform = flag.String("target_platform", util.PLATFORM_LINUX, "The platform we want to build for.")
	outDir         = flag.String("out", "", "The out directory where the isolate hash will be stored.")
)

func buildRepo() error {
	ctx := context.Background()
	httpClient, err := worker_common.Init(ctx, true /* useDepotTools */)
	if err != nil {
		return skerr.Wrap(err)
	}
	defer util.TimeTrack(time.Now(), "Isolating Telemetry")
	defer sklog.Flush()

	// Validate required arguments.
	if *runID == "" {
		return errors.New("Must specify --run_id")
	}
	if *outDir == "" {
		return errors.New("Must specify --out")
	}

	// Find git exec.
	gitExec, err := git.Executable(ctx)
	if err != nil {
		return skerr.Wrap(err)
	}

	// Instantiate GcsUtil object.
	gs, err := util.NewGcsUtil(httpClient)
	if err != nil {
		return err
	}

	// Apply patches if specified and create the telemetry isolates.
	applyPatches := false
	if *patches != "" {
		applyPatches = true
		for _, patch := range strings.Split(*patches, ",") {
			patchName := path.Base(patch)
			patchLocalPath := filepath.Join(os.TempDir(), patchName)
			if _, err := util.DownloadPatch(patchLocalPath, patch, gs); err != nil {
				return err
			}
		}
	}
	pathToPyFiles, err := util.GetPathToPyFiles(*worker_common.Local)
	if err != nil {
		return fmt.Errorf("Could not get path to py files: %s", err)
	}
	if err = util.CreateTelemetryIsolates(ctx, *runID, *targetPlatform, *chromiumHash, pathToPyFiles, gitExec, applyPatches); err != nil {
		return fmt.Errorf("Could not create telemetry isolates: %s", err)
	}

	buildOutDir := filepath.Join(util.ChromiumSrcDir, util.TELEMETRY_ISOLATES_OUT_DIR)
	isolateFile := fmt.Sprintf("%s.isolate", util.TELEMETRY_ISOLATES_TARGET)

	// Isolate telemetry artifacts.
	hash, err := isolate.Upload(ctx, rbe.InstanceChromeSwarming, buildOutDir, isolateFile)
	if err != nil {
		return fmt.Errorf("Could not isolate telemetry task: %s", err)
	}

	// Record the isolate hash in the output file.
	hashOutputFile := filepath.Join(*outDir, util.ISOLATE_TELEMETRY_FILENAME)
	if err := ioutil.WriteFile(hashOutputFile, []byte(hash), os.ModePerm); err != nil {
		return fmt.Errorf("Could not write to %s: %s", hashOutputFile, err)
	}

	return nil
}

func main() {
	retCode := 0
	if err := buildRepo(); err != nil {
		sklog.Errorf("Error while building repo: %s", err)
		retCode = 255
	}
	os.Exit(retCode)
}
<|endoftext|>