<commit_before>package mark

import (
	fmt "github.com/k0kubun/pp"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

type Tree struct {
	text  string
	lex   *lexer
	Nodes []Node
	// Parsing only
	token     [3]item // three-token lookahead for parser
	peekCount int
	output    string
}

// Parse convert the raw text to NodeTree.
func (t *Tree) parse() {
Loop:
	for {
		var n Node
		switch p := t.peek(); p.typ {
		case eof, itemError:
			break Loop
		case itemNewLine:
			n = t.newLine(t.next().pos)
		case itemBr:
			n = t.newBr(t.next().pos)
		case itemHr:
			n = t.newHr(t.next().pos)
		case itemText, itemStrong, itemItalic, itemStrike, itemCode,
			itemLink, itemAutoLink, itemGfmLink, itemImage:
			tmp := t.newParagraph(p.pos)
			tmp.Nodes = t.parseText(t.collectTextItems())
			n = tmp
		case itemHeading, itemLHeading:
			n = t.parseHeading()
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		case itemList:
			// 0 for the depth
			n = t.parseList(0)
		case itemTable, itemLpTable:
			n = t.parseTable()
		default:
			t.next()
		}
		if n != nil {
			t.append(n)
		}
	}
}

// Collect all items for paragraph creation
func (t *Tree) collectTextItems() (items []item) {
Loop:
	for {
		switch tkn := t.next(); tkn.typ {
		case eof, itemError, itemHeading, itemList, itemIndent:
			t.backup()
			break Loop
		case itemNewLine:
			if typ := t.peek().typ; typ == itemNewLine || isBlock(typ) {
				t.backup2(tkn)
				break Loop
			}
			fallthrough
		default:
			items = append(items, tkn)
		}
	}
	return
}

// Render parse nodes to the wanted output
func (t *Tree) render() {
	var last Node
	last = t.newLine(0)
	for _, node := range t.Nodes {
		if last.Type() != NodeNewLine || node.Type() != last.Type() {
			t.output += node.Render()
		}
		last = node
	}
}

// append new node to nodes-list
func (t *Tree) append(n Node) {
	t.Nodes = append(t.Nodes, n)
}

// next returns the next token
func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]
	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

// backup backs the input stream tp one token
func (t *Tree) backup() {
	t.peekCount++
}

// backup2 backs the input stream up two tokens.
// The zeroth token is already there.
func (t *Tree) backup2(t1 item) {
	t.token[1] = t1
	t.peekCount = 2
}

// parseParagraph scan until itemBr occur.
func (t *Tree) parseText(tokens []item) []Node {
	var nodes []Node
	for _, token := range tokens {
		var node Node
		switch token.typ {
		case itemNewLine:
			node = t.newLine(token.pos)
		case itemBr:
			node = t.newBr(token.pos)
		case itemStrong, itemItalic, itemStrike, itemCode:
			node = t.parseEmphasis(token.typ, token.pos, token.val)
		case itemLink, itemAutoLink, itemGfmLink:
			var title, text, href string
			match := span[token.typ].FindStringSubmatch(token.val)
			if token.typ == itemLink {
				text, href, title = match[1], match[2], match[3]
			} else {
				text, href = match[1], match[1]
			}
			node = t.newLink(token.pos, title, href, text)
		case itemImage:
			match := span[token.typ].FindStringSubmatch(token.val)
			node = t.newImage(token.pos, match[3], match[2], match[1])
		case itemText:
			node = t.newText(token.pos, token.val)
		default:
			fmt.Println("Matching not found for this token:", token)
		}
		nodes = append(nodes, node)
	}
	return nodes
}

// Parse inline emphasis
func (t *Tree) parseEmphasis(typ itemType, pos Pos, val string) *EmphasisNode {
	node := t.newEmphasis(pos, typ)
	match := span[typ].FindStringSubmatch(val)
	text := match[len(match)-1]
	if text == "" {
		text = match[1]
	}
	// sub node
	var c Node
	switch {
	case isWrap(text, "**", "__"):
		c = t.parseEmphasis(itemStrong, pos, text)
	case isWrap(text, "*", "_"):
		c = t.parseEmphasis(itemItalic, pos, text)
	case isWrap(text, "~~"):
		c = t.parseEmphasis(itemStrike, pos, text)
	default:
		c = t.newText(pos, text)
	}
	node.append(c)
	return node
}

// parse heading block
func (t *Tree) parseHeading() (node *HeadingNode) {
	token := t.next()
	match := block[token.typ].FindStringSubmatch(token.val)
	if token.typ == itemHeading {
		node = t.newHeading(token.pos, len(match[1]), match[2])
	} else {
		// itemLHeading will always be in level 1.
		node = t.newHeading(token.pos, 1, match[1])
	}
	return
}

// parse codeBlock
func (t *Tree) parseCodeBlock() *CodeNode {
	var lang, text string
	token := t.next()
	if token.typ == itemGfmCodeBlock {
		match := block[itemGfmCodeBlock].FindStringSubmatch(token.val)
		if text = match[2]; text == "" {
			text = match[4]
		}
		if lang = match[1]; lang == "" {
			lang = match[3]
		}
	} else {
		text = regexp.MustCompile("(?m)( {4}|\t)").ReplaceAllLiteralString(token.val, "")
	}
	return t.newCode(token.pos, lang, text)
}

// parse list
func (t *Tree) parseList(depth int) *ListNode {
	token := t.next()
	list := t.newList(token.pos, depth, isDigit(token.val))
	item := new(ListItemNode)
Loop:
	for {
		switch token = t.next(); token.typ {
		case eof, itemError:
			break Loop
		// It's actually a listItem
		case itemList:
			// List, but not the same type
			if list.Ordered != isDigit(token.val) || depth > 0 {
				t.backup()
				break Loop
			}
			item = t.parseListItem(token.pos, list)
		case itemNewLine:
			t.backup()
			break Loop
		case itemIndent:
			if depth == len(token.val) {
				item = t.parseListItem(t.next().pos, list)
			} else {
				t.backup()
				break Loop
			}
		default:
			t.backup()
			item = t.parseListItem(token.pos, list)
		}
		list.append(item)
	}
	return list
}

// parse listItem
func (t *Tree) parseListItem(pos Pos, list *ListNode) *ListItemNode {
	item := t.newListItem(pos, list)
	var n Node
Loop:
	for {
		switch token := t.next(); token.typ {
		case eof, itemError:
			break Loop
		case itemList:
			t.backup()
			break Loop
		case itemNewLine:
			switch typ := t.peek().typ; typ {
			case itemNewLine, eof, itemError:
				t.backup2(token)
				break Loop
			case itemList, itemIndent:
				continue
			default:
				n = t.newLine(token.pos)
			}
		case itemIndent:
			if t.peek().typ == itemList {
				depth := len(token.val)
				// If it's in the same depth - sibling
				// or if it's less-than - exit
				if depth <= item.List.Depth {
					t.backup2(token)
					break Loop
				}
				n = t.parseList(depth)
			} else {
				n = t.newText(token.pos, token.val)
			}
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		default:
			t.backup()
			// DRY
			for _, n := range t.parseText(t.collectTextItems()) {
				// TODO: Remove this condition
				if n.Type() != NodeNewLine {
					item.append(n)
				}
			}
			continue
		}
		item.append(n)
	}
	return item
}

// parse table
func (t *Tree) parseTable() *TableNode {
	// End of table
	done := t.lex.eot
	token := t.next()
	// ignore the first and last one...
	//lp := token.val == "|"
	table := t.newTable(token.pos)
	// Align	[ None, Left, Right, ... ]
	// Header	[ Cels: [token, token, ... ] ]
	// Data:	[ Row: [Cells: [token, ... ] ] ]
	rows := struct {
		Align  []AlignType
		Header [][]item
		Data   [][][]item
	}{}
	var cell []item
	var row [][]item
	// Collect items
Loop:
	for i := 0; ; {
		switch token := t.next(); token.typ {
		case eof, itemError:
			break Loop
		case itemNewLine:
			// If we done with this table
			if t.peek().pos >= done {
				break Loop
			}
			fallthrough
		case itemPipe:
			// Test if cell non-empty before appending to current row
			if len(cell) > 0 {
				// Header
				if i == 0 {
					rows.Header = append(rows.Header, cell)
					// Alignment
				} else if i == 1 {
					align := cell[0].val
					if len(cell) > 1 {
						for i := 1; i < len(cell); i++ {
							align += cell[i].val
						}
					}
					// Trim spaces
					rows.Align = append(rows.Align, parseAlign(align))
					// Data
				} else {
					row = append(row, cell)
				}
			}
			if token.typ == itemNewLine {
				i++
				// test if there's an elemnts to append to tbody.
				// we want to avoid situations like `appending empty rows`, etc..
				if i > 2 && len(row) > 0 {
					rows.Data = append(rows.Data, row)
					row = [][]item{}
				}
			}
			cell = []item{}
		default:
			cell = append(cell, token)
		}
	}
	// Drain cell/row
	if len(cell) > 0 {
		row = append(row, cell)
	}
	if len(row) > 0 {
		rows.Data = append(rows.Data, row)
	}
	// Tranform to nodes
	// Add an average mechanisem, that ignore empty(or " ") in end of cell
	//	rowLen := len(rows.Align)
	// Table head
	table.append(t.parseCells(Header, rows.Header, rows.Align))
	// Table body
	for _, row := range rows.Data {
		table.append(t.parseCells(Data, row, rows.Align))
	}
	return table
}

// Should return typ []CellNode
func (t *Tree) parseCells(kind int, items [][]item, align []AlignType) *RowNode {
	// TODO(Ariel): Add real position
	row := t.newRow(1)
	for i, item := range items {
		// Cell contain nodes
		cell := t.newCell(item[0].pos, kind, align[i])
		// Map: Trim all items
		for i, _ := range item {
			item[i].val = strings.Trim(item[i].val, " ")
		}
		cell.Nodes = t.parseText(item)
		row.append(cell)
	}
	return row
}

// get align-string and return the align type of it
// e.g: ":---", "---:", ":---:", "---"
func parseAlign(s string) (typ AlignType) {
	// Trim spaces before
	s = strings.Trim(s, " ")
	sfx, pfx := strings.HasSuffix(s, ":"), strings.HasPrefix(s, ":")
	switch {
	case sfx && pfx:
		typ = Center
	case sfx:
		typ = Right
	case pfx:
		typ = Left
	}
	return
}

// test if given string is digit
func isDigit(s string) bool {
	r, _ := utf8.DecodeRuneInString(s)
	return unicode.IsDigit(r)
}

// test if given token is type block
func isBlock(item itemType) (b bool) {
	switch item {
	case itemHeading, itemLHeading, itemCodeBlock, itemBlockQuote,
		itemList, itemTable, itemGfmCodeBlock, itemHr:
		b = true
	}
	return
}

// Test if strings start and end with specific string
func isWrap(text string, args ...string) bool {
	for _, s := range args {
		if strings.HasPrefix(text, s) && strings.HasSuffix(text, s) {
			return true
		}
	}
	return false
}
<commit_msg>fix(parser): fix table space-triming<commit_after>package mark

import (
	fmt "github.com/k0kubun/pp"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

type Tree struct {
	text  string
	lex   *lexer
	Nodes []Node
	// Parsing only
	token     [3]item // three-token lookahead for parser
	peekCount int
	output    string
}

// Parse convert the raw text to NodeTree.
func (t *Tree) parse() {
Loop:
	for {
		var n Node
		switch p := t.peek(); p.typ {
		case eof, itemError:
			break Loop
		case itemNewLine:
			n = t.newLine(t.next().pos)
		case itemBr:
			n = t.newBr(t.next().pos)
		case itemHr:
			n = t.newHr(t.next().pos)
		case itemText, itemStrong, itemItalic, itemStrike, itemCode,
			itemLink, itemAutoLink, itemGfmLink, itemImage:
			tmp := t.newParagraph(p.pos)
			tmp.Nodes = t.parseText(t.collectTextItems())
			n = tmp
		case itemHeading, itemLHeading:
			n = t.parseHeading()
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		case itemList:
			// 0 for the depth
			n = t.parseList(0)
		case itemTable, itemLpTable:
			n = t.parseTable()
		default:
			t.next()
		}
		if n != nil {
			t.append(n)
		}
	}
}

// Collect all items for paragraph creation
func (t *Tree) collectTextItems() (items []item) {
Loop:
	for {
		switch tkn := t.next(); tkn.typ {
		case eof, itemError, itemHeading, itemList, itemIndent:
			t.backup()
			break Loop
		case itemNewLine:
			if typ := t.peek().typ; typ == itemNewLine || isBlock(typ) {
				t.backup2(tkn)
				break Loop
			}
			fallthrough
		default:
			items = append(items, tkn)
		}
	}
	return
}

// Render parse nodes to the wanted output
func (t *Tree) render() {
	var last Node
	last = t.newLine(0)
	for _, node := range t.Nodes {
		if last.Type() != NodeNewLine || node.Type() != last.Type() {
			t.output += node.Render()
		}
		last = node
	}
}

// append new node to nodes-list
func (t *Tree) append(n Node) {
	t.Nodes = append(t.Nodes, n)
}

// next returns the next token
func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]
	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

// backup backs the input stream tp one token
func (t *Tree) backup() {
	t.peekCount++
}

// backup2 backs the input stream up two tokens.
// The zeroth token is already there.
func (t *Tree) backup2(t1 item) {
	t.token[1] = t1
	t.peekCount = 2
}

// parseParagraph scan until itemBr occur.
func (t *Tree) parseText(tokens []item) []Node {
	var nodes []Node
	for _, token := range tokens {
		var node Node
		switch token.typ {
		case itemNewLine:
			node = t.newLine(token.pos)
		case itemBr:
			node = t.newBr(token.pos)
		case itemStrong, itemItalic, itemStrike, itemCode:
			node = t.parseEmphasis(token.typ, token.pos, token.val)
		case itemLink, itemAutoLink, itemGfmLink:
			var title, text, href string
			match := span[token.typ].FindStringSubmatch(token.val)
			if token.typ == itemLink {
				text, href, title = match[1], match[2], match[3]
			} else {
				text, href = match[1], match[1]
			}
			node = t.newLink(token.pos, title, href, text)
		case itemImage:
			match := span[token.typ].FindStringSubmatch(token.val)
			node = t.newImage(token.pos, match[3], match[2], match[1])
		case itemText:
			node = t.newText(token.pos, token.val)
		default:
			fmt.Println("Matching not found for this token:", token)
		}
		nodes = append(nodes, node)
	}
	return nodes
}

// Parse inline emphasis
func (t *Tree) parseEmphasis(typ itemType, pos Pos, val string) *EmphasisNode {
	node := t.newEmphasis(pos, typ)
	match := span[typ].FindStringSubmatch(val)
	text := match[len(match)-1]
	if text == "" {
		text = match[1]
	}
	// sub node
	var c Node
	switch {
	case isWrap(text, "**", "__"):
		c = t.parseEmphasis(itemStrong, pos, text)
	case isWrap(text, "*", "_"):
		c = t.parseEmphasis(itemItalic, pos, text)
	case isWrap(text, "~~"):
		c = t.parseEmphasis(itemStrike, pos, text)
	default:
		c = t.newText(pos, text)
	}
	node.append(c)
	return node
}

// parse heading block
func (t *Tree) parseHeading() (node *HeadingNode) {
	token := t.next()
	match := block[token.typ].FindStringSubmatch(token.val)
	if token.typ == itemHeading {
		node = t.newHeading(token.pos, len(match[1]), match[2])
	} else {
		// itemLHeading will always be in level 1.
		node = t.newHeading(token.pos, 1, match[1])
	}
	return
}

// parse codeBlock
func (t *Tree) parseCodeBlock() *CodeNode {
	var lang, text string
	token := t.next()
	if token.typ == itemGfmCodeBlock {
		match := block[itemGfmCodeBlock].FindStringSubmatch(token.val)
		if text = match[2]; text == "" {
			text = match[4]
		}
		if lang = match[1]; lang == "" {
			lang = match[3]
		}
	} else {
		text = regexp.MustCompile("(?m)( {4}|\t)").ReplaceAllLiteralString(token.val, "")
	}
	return t.newCode(token.pos, lang, text)
}

// parse list
func (t *Tree) parseList(depth int) *ListNode {
	token := t.next()
	list := t.newList(token.pos, depth, isDigit(token.val))
	item := new(ListItemNode)
Loop:
	for {
		switch token = t.next(); token.typ {
		case eof, itemError:
			break Loop
		// It's actually a listItem
		case itemList:
			// List, but not the same type
			if list.Ordered != isDigit(token.val) || depth > 0 {
				t.backup()
				break Loop
			}
			item = t.parseListItem(token.pos, list)
		case itemNewLine:
			t.backup()
			break Loop
		case itemIndent:
			if depth == len(token.val) {
				item = t.parseListItem(t.next().pos, list)
			} else {
				t.backup()
				break Loop
			}
		default:
			t.backup()
			item = t.parseListItem(token.pos, list)
		}
		list.append(item)
	}
	return list
}

// parse listItem
func (t *Tree) parseListItem(pos Pos, list *ListNode) *ListItemNode {
	item := t.newListItem(pos, list)
	var n Node
Loop:
	for {
		switch token := t.next(); token.typ {
		case eof, itemError:
			break Loop
		case itemList:
			t.backup()
			break Loop
		case itemNewLine:
			switch typ := t.peek().typ; typ {
			case itemNewLine, eof, itemError:
				t.backup2(token)
				break Loop
			case itemList, itemIndent:
				continue
			default:
				n = t.newLine(token.pos)
			}
		case itemIndent:
			if t.peek().typ == itemList {
				depth := len(token.val)
				// If it's in the same depth - sibling
				// or if it's less-than - exit
				if depth <= item.List.Depth {
					t.backup2(token)
					break Loop
				}
				n = t.parseList(depth)
			} else {
				n = t.newText(token.pos, token.val)
			}
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		default:
			t.backup()
			// DRY
			for _, n := range t.parseText(t.collectTextItems()) {
				// TODO: Remove this condition
				if n.Type() != NodeNewLine {
					item.append(n)
				}
			}
			continue
		}
		item.append(n)
	}
	return item
}

// parse table
func (t *Tree) parseTable() *TableNode {
	// End of table
	done := t.lex.eot
	token := t.next()
	// ignore the first and last one...
	//lp := token.val == "|"
	table := t.newTable(token.pos)
	// Align	[ None, Left, Right, ... ]
	// Header	[ Cels: [token, token, ... ] ]
	// Data:	[ Row: [Cells: [token, ... ] ] ]
	rows := struct {
		Align  []AlignType
		Header [][]item
		Data   [][][]item
	}{}
	var cell []item
	var row [][]item
	// Collect items
Loop:
	for i := 0; ; {
		switch token := t.next(); token.typ {
		case eof, itemError:
			break Loop
		case itemNewLine:
			// If we done with this table
			if t.peek().pos >= done {
				break Loop
			}
			fallthrough
		case itemPipe:
			// Test if cell non-empty before appending to current row
			if len(cell) > 0 {
				// Header
				if i == 0 {
					rows.Header = append(rows.Header, cell)
					// Alignment
				} else if i == 1 {
					align := cell[0].val
					if len(cell) > 1 {
						for i := 1; i < len(cell); i++ {
							align += cell[i].val
						}
					}
					// Trim spaces
					rows.Align = append(rows.Align, parseAlign(align))
					// Data
				} else {
					row = append(row, cell)
				}
			}
			if token.typ == itemNewLine {
				i++
				// test if there's an elemnts to append to tbody.
				// we want to avoid situations like `appending empty rows`, etc..
				if i > 2 && len(row) > 0 {
					rows.Data = append(rows.Data, row)
					row = [][]item{}
				}
			}
			cell = []item{}
		default:
			cell = append(cell, token)
		}
	}
	// Drain cell/row
	if len(cell) > 0 {
		row = append(row, cell)
	}
	if len(row) > 0 {
		rows.Data = append(rows.Data, row)
	}
	// Tranform to nodes
	// Add an average mechanisem, that ignore empty(or " ") in end of cell
	//	rowLen := len(rows.Align)
	// Table head
	table.append(t.parseCells(Header, rows.Header, rows.Align))
	// Table body
	for _, row := range rows.Data {
		table.append(t.parseCells(Data, row, rows.Align))
	}
	return table
}

// Should return typ []CellNode
func (t *Tree) parseCells(kind int, items [][]item, align []AlignType) *RowNode {
	// TODO(Ariel): Add real position
	row := t.newRow(1)
	for i, item := range items {
		// Cell contain nodes
		cell := t.newCell(item[0].pos, kind, align[i])
		// Map: Trim all start and end spaces.
		// TODO(Ariel): it's just a patch right now
		for i, _ := range item {
			if i == 0 || i == len(item)-1 {
				item[i].val = strings.Trim(item[i].val, " ")
			}
		}
		cell.Nodes = t.parseText(item)
		row.append(cell)
	}
	return row
}

// get align-string and return the align type of it
// e.g: ":---", "---:", ":---:", "---"
func parseAlign(s string) (typ AlignType) {
	// Trim spaces before
	s = strings.Trim(s, " ")
	sfx, pfx := strings.HasSuffix(s, ":"), strings.HasPrefix(s, ":")
	switch {
	case sfx && pfx:
		typ = Center
	case sfx:
		typ = Right
	case pfx:
		typ = Left
	}
	return
}

// test if given string is digit
func isDigit(s string) bool {
	r, _ := utf8.DecodeRuneInString(s)
	return unicode.IsDigit(r)
}

// test if given token is type block
func isBlock(item itemType) (b bool) {
	switch item {
	case itemHeading, itemLHeading, itemCodeBlock, itemBlockQuote,
		itemList, itemTable, itemGfmCodeBlock, itemHr:
		b = true
	}
	return
}

// Test if strings start and end with specific string
func isWrap(text string, args ...string) bool {
	for _, s := range args {
		if strings.HasPrefix(text, s) && strings.HasSuffix(text, s) {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>/*
parser inspired by Rob Pikes lexer
*/

package parser

import (
	"errors"
	"fmt"
	"strings"
	"unicode/utf8"
)

// last State must return ErrEOF
type State func(p *Parser) (next State, err error)

var ErrEOF = errors.New("End of File")

var EOF = rune('∎')

type ASTNode interface {
	AddChild(ASTNode)
}

type Parser struct {
	astQueue    []ASTNode
	input       string // the string being scanned
	start       int    // start position of this item
	pos         int    // current position in the input
	width       int    // width of the last rune read
	line        int
	linepos     int
	linePrev    int
	lineposPrev int
}

func New(input string, root ASTNode) *Parser {
	return &Parser{
		astQueue: []ASTNode{root},
		input:    input,
	}
}

func (p *Parser) Root() ASTNode {
	return p.astQueue[0]
}

func (p *Parser) currentNode() ASTNode {
	return p.astQueue[len(p.astQueue)-1]
}

func (p *Parser) AddNode(n ASTNode) {
	p.currentNode().AddChild(n)
	p.astQueue = append(p.astQueue, n)
}

func (p *Parser) PopNode() {
	if len(p.astQueue) < 2 {
		return
	}
	p.astQueue = p.astQueue[:len(p.astQueue)-1]
}

func (p *Parser) Next() (rune_ rune) {
	if p.pos >= len(p.input) {
		p.width = 0
		return EOF
	}
	rune_, p.width = utf8.DecodeRuneInString(p.input[p.pos:])
	p.pos += p.width
	p.linePrev = p.line
	p.lineposPrev = p.linepos
	if rune_ == '\n' {
		p.line++
		p.linepos = 0
	} else {
		p.linepos++
	}

	return
}

// emit passes an item back to the client
func (p *Parser) Emit() string {
	s := p.input[p.start:p.pos]
	p.start = p.pos
	return s
}

func (p *Parser) Ignore() {
	p.start = p.pos
}

// backup steps back one rune
// can be called only once per call of next
func (p *Parser) Backup() {
	rune_, _ := utf8.DecodeRuneInString(p.input[p.pos:])
	if rune_ == '\n' {
		p.line--
	}
	p.linepos = p.lineposPrev
	p.pos -= p.width
}

func (p *Parser) Peek() rune {
	r := p.Next()
	p.Backup()
	return r
}

func (p *Parser) Accept(valid string) bool {
	if strings.IndexRune(valid, p.Next()) >= 0 {
		return true
	}
	p.Backup()
	return false
}

func (p *Parser) AcceptRun(valid string) {
	for strings.IndexRune(valid, p.Next()) >= 0 {
	}
	p.Backup()
}

func (p *Parser) Errorf(format string, args ...interface{}) error {
	start := p.pos - 5
	if start < 0 {
		start = 0
	}

	end := p.pos + 5

	if end > len(p.input) {
		end = len(p.input)
	}

	return errors.New(fmt.Sprintf(
		"Error in line %d at position %d: %s\ncontext:\n%s\n",
		p.line+1,
		p.linepos+1,
		fmt.Sprintf(format, args...),
		p.input[start:end],
	))
}

func (p *Parser) Run(fn State) (err error) {
	for err == nil {
		fn, err = fn(p)
	}
	if err == ErrEOF {
		return nil
	}

	return err
}
<commit_msg>add ForwardUntil<commit_after>/*
parser inspired by Rob Pikes lexer
*/

package parser

import (
	"errors"
	"fmt"
	"strings"
	"unicode/utf8"
)

// last State must return ErrEOF
type State func(p *Parser) (next State, err error)

var ErrEOF = errors.New("End of File")

var EOF = rune('∎')

type ASTNode interface {
	AddChild(ASTNode)
}

type Parser struct {
	astQueue    []ASTNode
	input       string // the string being scanned
	start       int    // start position of this item
	pos         int    // current position in the input
	width       int    // width of the last rune read
	line        int
	linepos     int
	linePrev    int
	lineposPrev int
}

func New(input string, root ASTNode) *Parser {
	return &Parser{
		astQueue: []ASTNode{root},
		input:    input,
	}
}

func (p *Parser) Root() ASTNode {
	return p.astQueue[0]
}

func (p *Parser) currentNode() ASTNode {
	return p.astQueue[len(p.astQueue)-1]
}

func (p *Parser) AddNode(n ASTNode) {
	p.currentNode().AddChild(n)
	p.astQueue = append(p.astQueue, n)
}

func (p *Parser) PopNode() {
	if len(p.astQueue) < 2 {
		return
	}
	p.astQueue = p.astQueue[:len(p.astQueue)-1]
}

func (p *Parser) Next() (rune_ rune) {
	if p.pos >= len(p.input) {
		p.width = 0
		return EOF
	}
	rune_, p.width = utf8.DecodeRuneInString(p.input[p.pos:])
	p.pos += p.width
	p.linePrev = p.line
	p.lineposPrev = p.linepos
	if rune_ == '\n' {
		p.line++
		p.linepos = 0
	} else {
		p.linepos++
	}

	return
}

// emit passes an item back to the client
func (p *Parser) Emit() string {
	s := p.input[p.start:p.pos]
	p.start = p.pos
	return s
}

func (p *Parser) Ignore() {
	p.start = p.pos
}

// backup steps back one rune
// can be called only once per call of next
func (p *Parser) Backup() {
	rune_, _ := utf8.DecodeRuneInString(p.input[p.pos:])
	if rune_ == '\n' {
		p.line--
	}
	p.linepos = p.lineposPrev
	p.pos -= p.width
}

func (p *Parser) Peek() rune {
	r := p.Next()
	p.Backup()
	return r
}

func (p *Parser) Accept(valid string) bool {
	if strings.IndexRune(valid, p.Next()) >= 0 {
		return true
	}
	p.Backup()
	return false
}

func (p *Parser) AcceptRun(valid string) {
	for strings.IndexRune(valid, p.Next()) >= 0 {
	}
	p.Backup()
}

// runs forward until one of the stopper
func (p *Parser) ForwardUntil(stopper string) {
	for strings.IndexRune(stopper, p.Next()) == -1 {
	}
	p.Backup()
}

func (p *Parser) Errorf(format string, args ...interface{}) error {
	start := p.pos - 5
	if start < 0 {
		start = 0
	}

	end := p.pos + 5

	if end > len(p.input) {
		end = len(p.input)
	}

	return errors.New(fmt.Sprintf(
		"Error in line %d at position %d: %s\ncontext:\n%s\n",
		p.line+1,
		p.linepos+1,
		fmt.Sprintf(format, args...),
		p.input[start:end],
	))
}

func (p *Parser) Run(fn State) (err error) {
	for err == nil {
		fn, err = fn(p)
	}
	if err == ErrEOF {
		return nil
	}

	return err
}
<|endoftext|>


<commit_before>package svg

import (
	"fmt"
	"strconv"

	mt "github.com/rustyoz/Mtransform"
	gl "github.com/rustyoz/genericlexer"
)

func parseNumber(i gl.Item) (float64, error) {
	var n float64
	var ok error
	if i.Type == gl.ItemNumber {
		n, ok = strconv.ParseFloat(i.Value, 64)
		if ok != nil {
			return n, fmt.Errorf("Error passing number %s", ok)
		}
	}
	return n, nil
}

func parseTuple(l *gl.Lexer) (Tuple, error) {
	t := Tuple{}

	l.ConsumeWhiteSpace()

	ni := l.NextItem()
	if ni.Type == gl.ItemNumber {
		n, ok := strconv.ParseFloat(ni.Value, 64)
		if ok != nil {
			return t, fmt.Errorf("Error parsing number %s", ok)
		}
		t[0] = n
	} else {
		return t, fmt.Errorf("Error parsing Tuple expected Number got: %s", ni.Value)
	}

	if l.PeekItem().Type == gl.ItemWSP || l.PeekItem().Type == gl.ItemComma {
		l.NextItem()
	}
	ni = l.NextItem()
	if ni.Type == gl.ItemNumber {
		n, ok := strconv.ParseFloat(ni.Value, 64)
		if ok != nil {
			return t, fmt.Errorf("Error passing Number %s", ok)
		}
		t[1] = n
	} else {
		return t, fmt.Errorf("Error passing Tuple expected Number got: %v", ni)
	}

	return t, nil
}

func parseTransform(tstring string) (mt.Transform, error) {
	lexer, _ := gl.Lex("tlexer", tstring)
	for {
		i := lexer.NextItem()
		switch i.Type {
		case gl.ItemEOS:
			return mt.Identity(),
				fmt.Errorf("transform parse failed")
		case gl.ItemWord:
			switch i.Value {
			case "matrix":
				return parseMatrix(lexer)
				// case "scale":
				// case "rotate":
			case "translate":
				return parseTranslate(lexer)
			}
		}
	}
}

func parseMatrix(l *gl.Lexer) (mt.Transform, error) {
	nums, err := parseParenNumList(l, 6)
	if err != nil {
		return mt.Identity(),
			fmt.Errorf("Error Parsing Transform Matrix: %v", err)
	}
	var tm mt.Transform
	for i := 0; i < 6; i++ {
		tm[i%3][i/3] = nums[i]
	}
	return tm, nil
}

func parseTranslate(l *gl.Lexer) (mt.Transform, error) {
	nums, err := parseParenNumList(l, 2)
	if err != nil {
		return mt.Identity(), fmt.Errorf("Error Parsing Translate: %v", err)
	}
	tm := mt.Identity()
	tm[0][2] = nums[0]
	tm[1][2] = nums[1]
	return tm, nil
}

// Parse a parenthesized list of ncount numbers.
func parseParenNumList(l *gl.Lexer, ncount int) ([]float64, error) {
	i := l.NextItem()
	if i.Type != gl.ItemParan {
		return nil, fmt.Errorf("Expected Opening Parantheses")
	}
	var nums []float64
	for {
		if len(nums) > 0 {
			for l.PeekItem().Type == gl.ItemComma || l.PeekItem().Type == gl.ItemWSP {
				l.NextItem()
			}
		}
		if l.PeekItem().Type != gl.ItemNumber {
			return nil, fmt.Errorf("Expected Number got %v", l.PeekItem().String())
		}
		n, err := parseNumber(l.NextItem())
		if err != nil {
			return nil, err
		}
		nums = append(nums, n)
		if len(nums) >= ncount {
			i = l.PeekItem()
			if i.Type != gl.ItemParan {
				return nil, fmt.Errorf("Expected Closing Parantheses")
			}
			l.NextItem() // consume Parantheses
			return nums, nil
		}
	}
}
<commit_msg>Fix matrix transform parsing<commit_after>package svg

import (
	"fmt"
	"strconv"

	mt "github.com/rustyoz/Mtransform"
	gl "github.com/rustyoz/genericlexer"
)

func parseNumber(i gl.Item) (float64, error) {
	var n float64
	var ok error
	if i.Type == gl.ItemNumber {
		n, ok = strconv.ParseFloat(i.Value, 64)
		if ok != nil {
			return n, fmt.Errorf("Error passing number %s", ok)
		}
	}
	return n, nil
}

func parseTuple(l *gl.Lexer) (Tuple, error) {
	t := Tuple{}

	l.ConsumeWhiteSpace()

	ni := l.NextItem()
	if ni.Type == gl.ItemNumber {
		n, ok := strconv.ParseFloat(ni.Value, 64)
		if ok != nil {
			return t, fmt.Errorf("Error parsing number %s", ok)
		}
		t[0] = n
	} else {
		return t, fmt.Errorf("Error parsing Tuple expected Number got: %s", ni.Value)
	}

	if l.PeekItem().Type == gl.ItemWSP || l.PeekItem().Type == gl.ItemComma {
		l.NextItem()
	}
	ni = l.NextItem()
	if ni.Type == gl.ItemNumber {
		n, ok := strconv.ParseFloat(ni.Value, 64)
		if ok != nil {
			return t, fmt.Errorf("Error passing Number %s", ok)
		}
		t[1] = n
	} else {
		return t, fmt.Errorf("Error passing Tuple expected Number got: %v", ni)
	}

	return t, nil
}

func parseTransform(tstring string) (mt.Transform, error) {
	lexer, _ := gl.Lex("tlexer", tstring)
	for {
		i := lexer.NextItem()
		switch i.Type {
		case gl.ItemEOS:
			return mt.Identity(),
				fmt.Errorf("transform parse failed")
		case gl.ItemWord:
			switch i.Value {
			case "matrix":
				return parseMatrix(lexer)
				// case "scale":
				// case "rotate":
			case "translate":
				return parseTranslate(lexer)
			}
		}
	}
}

func parseMatrix(l *gl.Lexer) (mt.Transform, error) {
	nums, err := parseParenNumList(l, 6)
	if err != nil {
		return mt.Identity(),
			fmt.Errorf("Error Parsing Transform Matrix: %v", err)
	}
	var tm mt.Transform
	tm[0][0] = nums[0]
	tm[0][1] = nums[2]
	tm[0][2] = nums[4]
	tm[1][0] = nums[1]
	tm[1][1] = nums[3]
	tm[1][2] = nums[5]
	tm[2][0] = 0
	tm[2][1] = 0
	tm[2][2] = 1

	return tm, nil
}

func parseTranslate(l *gl.Lexer) (mt.Transform, error) {
	nums, err := parseParenNumList(l, 2)
	if err != nil {
		return mt.Identity(), fmt.Errorf("Error Parsing Translate: %v", err)
	}
	tm := mt.Identity()
	tm[0][2] = nums[0]
	tm[1][2] = nums[1]
	return tm, nil
}

// Parse a parenthesized list of ncount numbers.
func parseParenNumList(l *gl.Lexer, ncount int) ([]float64, error) {
	i := l.NextItem()
	if i.Type != gl.ItemParan {
		return nil, fmt.Errorf("Expected Opening Parantheses")
	}
	var nums []float64
	for {
		if len(nums) > 0 {
			for l.PeekItem().Type == gl.ItemComma || l.PeekItem().Type == gl.ItemWSP {
				l.NextItem()
			}
		}
		if l.PeekItem().Type != gl.ItemNumber {
			return nil, fmt.Errorf("Expected Number got %v", l.PeekItem().String())
		}
		n, err := parseNumber(l.NextItem())
		if err != nil {
			return nil, err
		}
		nums = append(nums, n)
		if len(nums) >= ncount {
			i = l.PeekItem()
			if i.Type != gl.ItemParan {
				return nil, fmt.Errorf("Expected Closing Parantheses")
			}
			l.NextItem() // consume Parantheses
			return nums, nil
		}
	}
}
<|endoftext|>


<commit_before>// Package protoparser contains a protobuf parser.
// nolint: govet, golint
package protoparser

import (
	"fmt"
	"io"
	"math/big"
	"strings"

	"github.com/alecthomas/participle/v2"
	"github.com/alecthomas/participle/v2/lexer"
)

type Proto struct {
	Pos lexer.Position

	Entries []*Entry `{ @@ { ";" } }`
}

type Entry struct {
	Pos lexer.Position

	Syntax  string   `  "syntax" "=" @String`
	Package string   `| "package" @(Ident { "." Ident })`
	Import  *Import  `| @@`
	Message *Message `| @@`
	Service *Service `| @@`
	Enum    *Enum    `| @@`
	Option  *Option  `| "option" @@`
	Extend  *Extend  `| @@`
}

type Import struct {
	Public bool   `"import" @("public")?`
	Name   string `@String`
}

type Option struct {
	Pos lexer.Position

	Name  string  `( "(" @("."? Ident { "." Ident }) ")" | @("."? Ident { "." Ident }) )`
	Attr  *string `[ @("."? Ident { "." Ident }) ]`
	Value *Value  `"=" @@`
}

type Value struct {
	Pos lexer.Position

	String    *string    `  @String`
	Number    *big.Float `| ("-" | "+")? (@Float | @Int)`
	Bool      *bool      `| (@"true" | "false")`
	Reference *string    `| @("."? Ident { "." Ident })`
	ProtoText *ProtoText `| "{" @@ "}"`
	Array     *Array     `| @@`
}

type ProtoText struct {
	Pos lexer.Position

	Fields []ProtoTextField `( @@ ( "," | ";" )? )*`
}

type ProtoTextField struct {
	Pos lexer.Position

	Name  string `(@Ident | ( "[" @("."? Ident { "." Ident }) "]" ))`
	Value *Value `( ":"? @@ )`
}

type Array struct {
	Pos lexer.Position

	Elements []*Value `"[" [ @@ { [ "," ] @@ } ] "]"`
}

type Extensions struct {
	Pos lexer.Position

	Extensions []Range `"extensions" @@ { "," @@ }`
}

type Reserved struct {
	Pos lexer.Position

	Ranges     []Range  `( @@ { "," @@ }`
	FieldNames []string `  | @String { "," @String } )`
}

type Range struct {
	Start int  `( @Int`
	End   *int `  [ "to" ( @Int`
	Max   bool `           | @"max" ) ] )`
}

type Extend struct {
	Pos lexer.Position

	Reference string   `"extend" @("."? Ident { "." Ident })`
	Fields    []*Field `"{" { @@ [ ";" ] } "}"`
}

type Service struct {
	Pos lexer.Position

	Name  string          `"service" @Ident`
	Entry []*ServiceEntry `"{" { @@ [ ";" ] } "}"`
}

type ServiceEntry struct {
	Pos lexer.Position

	Option *Option `  "option" @@`
	Method *Method `| @@`
}

type Method struct {
	Pos lexer.Position

	Name              string    `"rpc" @Ident`
	StreamingRequest  bool      `"(" [ @"stream" ]`
	Request           *Type     `    @@ ")"`
	StreamingResponse bool      `"returns" "(" [ @"stream" ]`
	Response          *Type     `              @@ ")"`
	Options           []*Option `[ "{" { "option" @@ ";" } "}" ]`
}

type Enum struct {
	Pos lexer.Position

	Name   string       `"enum" @Ident`
	Values []*EnumEntry `"{" { @@ { ";" } } "}"`
}

type EnumEntry struct {
	Pos lexer.Position

	Value    *EnumValue `  @@`
	Option   *Option    `| "option" @@`
	Reserved *Reserved  `| "reserved" @@`
}

type EnumValue struct {
	Pos lexer.Position

	Key   string `@Ident`
	Value int    `"=" @( [ "-" ] Int )`

	Options []*Option `[ "[" @@ { "," @@ } "]" ]`
}

type Message struct {
	Pos lexer.Position

	Name    string          `"message" @Ident`
	Entries []*MessageEntry `"{" { @@ } "}"`
}

type MessageEntry struct {
	Pos lexer.Position

	Enum       *Enum       `( @@`
	Option     *Option     ` | "option" @@`
	Message    *Message    ` | @@`
	Oneof      *Oneof      ` | @@`
	Extend     *Extend     ` | @@`
	Reserved   *Reserved   ` | "reserved" @@`
	Extensions *Extensions ` | @@`
	Field      *Field      ` | @@ ) { ";" }`
}

type Oneof struct {
	Pos lexer.Position

	Name    string        `"oneof" @Ident`
	Entries []*OneofEntry `"{" { @@ { ";" } } "}"`
}

type OneofEntry struct {
	Pos lexer.Position

	Field  *Field  `  @@`
	Option *Option `| "option" @@`
}

type Field struct {
	Pos lexer.Position

	Optional bool `[   @"optional"`
	Required bool `  | @"required"`
	Repeated bool `  | @"repeated" ]`

	Group  *Group  `( @@`
	Direct *Direct `| @@ )`
}

type Direct struct {
	Pos lexer.Position

	Type *Type  `@@`
	Name string `@Ident`
	Tag  int    `"=" @Int`

	Options []*Option `[ "[" @@ { "," @@ } "]" ]`
}

type Group struct {
	Pos lexer.Position

	Name    string          `"group" @Ident`
	Tag     int             `"=" @Int`
	Entries []*MessageEntry `"{" { @@ [ ";" ] } "}"`
}

type Scalar int

const (
	None Scalar = iota
	Double
	Float
	Int32
	Int64
	Uint32
	Uint64
	Sint32
	Sint64
	Fixed32
	Fixed64
	SFixed32
	SFixed64
	Bool
	String
	Bytes
)

var scalarToString = map[Scalar]string{
	None: "None", Double: "Double", Float: "Float", Int32: "Int32", Int64: "Int64", Uint32: "Uint32",
	Uint64: "Uint64", Sint32: "Sint32", Sint64: "Sint64", Fixed32: "Fixed32", Fixed64: "Fixed64",
	SFixed32: "SFixed32", SFixed64: "SFixed64", Bool: "Bool", String: "String", Bytes: "Bytes",
}

func (s Scalar) GoString() string { return scalarToString[s] }

var stringToScalar = map[string]Scalar{
	"double": Double, "float": Float, "int32": Int32, "int64": Int64, "uint32": Uint32, "uint64": Uint64,
	"sint32": Sint32, "sint64": Sint64, "fixed32": Fixed32, "fixed64": Fixed64, "sfixed32": SFixed32,
	"sfixed64": SFixed64, "bool": Bool, "string": String, "bytes": Bytes,
}

func (s *Scalar) Parse(lex *lexer.PeekingLexer) error {
	token, err := lex.Peek(0)
	if err != nil {
		return fmt.Errorf("failed to peek next token: %w", err)
	}
	scalar, ok := stringToScalar[token.Value]
	if !ok {
		return participle.NextMatch
	}
	_, err = lex.Next()
	if err != nil {
		return fmt.Errorf("failed to read next token: %w", err)
	}
	*s = scalar
	return nil
}

type Type struct {
	Pos lexer.Position

	Scalar    Scalar   `  @@`
	Map       *MapType `| @@`
	Reference *string  `| @("."? Ident { "." Ident })`
}

type MapType struct {
	Pos lexer.Position

	Key   *Type `"map" "<" @@`
	Value *Type `"," @@ ">"`
}

// Parse protobuf.
func Parse(filename string, r io.Reader) (*Proto, error) {
	p := &Proto{}

	l := lexer.MustSimple([]lexer.Rule{
		{"String", `"(\\"|[^"])*"|'(\\'|[^'])*'`, nil},
		{"Float", `[-+]?\d*\.\d+`, nil},
		{"Int", `[-+]?\d+`, nil},
		{"Ident", `[a-zA-Z_]([a-zA-Z_0-9])*`, nil},
		{"Whitespace", `[ \t\n\r]+`, nil},
		{"BlockComment", `/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/`, nil},
		{"LineComment", `//(.*)[^\n]*\n`, nil},
		{".", `.`, nil},
	})

	parser := participle.MustBuild(
		&Proto{},
		participle.UseLookahead(2),
		participle.Unquote("String"),
		participle.Lexer(l),
		participle.Elide("Whitespace", "LineComment", "BlockComment"),
	)
	err := parser.Parse(filename, r, p)
	if err != nil {
		return p, err
	}
	return p, nil
}

func ParseString(filename string, source string) (*Proto, error) {
	return Parse(filename, strings.NewReader(source))
}
<commit_msg>Support hex ints<commit_after>// Package protoparser contains a protobuf parser.
// nolint: govet, golint
package protoparser

import (
	"fmt"
	"io"
	"math/big"
	"strings"

	"github.com/alecthomas/participle/v2"
	"github.com/alecthomas/participle/v2/lexer"
)

type Proto struct {
	Pos lexer.Position

	Entries []*Entry `{ @@ { ";" } }`
}

type Entry struct {
	Pos lexer.Position

	Syntax  string   `  "syntax" "=" @String`
	Package string   `| "package" @(Ident { "." Ident })`
	Import  *Import  `| @@`
	Message *Message `| @@`
	Service *Service `| @@`
	Enum    *Enum    `| @@`
	Option  *Option  `| "option" @@`
	Extend  *Extend  `| @@`
}

type Import struct {
	Public bool   `"import" @("public")?`
	Name   string `@String`
}

type Option struct {
	Pos lexer.Position

	Name  string  `( "(" @("."? Ident { "." Ident }) ")" | @("."? Ident { "." Ident }) )`
	Attr  *string `[ @("."? Ident { "." Ident }) ]`
	Value *Value  `"=" @@`
}

type Value struct {
	Pos lexer.Position

	String    *string    `  @String`
	Number    *big.Float `| ("-" | "+")? (@Float | @Int)`
	Bool      *bool      `| (@"true" | "false")`
	Reference *string    `| @("."? Ident { "." Ident })`
	ProtoText *ProtoText `| "{" @@ "}"`
	Array     *Array     `| @@`
}

type ProtoText struct {
	Pos lexer.Position

	Fields []ProtoTextField `( @@ ( "," | ";" )? )*`
}

type ProtoTextField struct {
	Pos lexer.Position

	Name  string `(@Ident | ( "[" @("."? Ident { "." Ident }) "]" ))`
	Value *Value `( ":"? @@ )`
}

type Array struct {
	Pos lexer.Position

	Elements []*Value `"[" [ @@ { [ "," ] @@ } ] "]"`
}

type Extensions struct {
	Pos lexer.Position

	Extensions []Range `"extensions" @@ { "," @@ }`
}

type Reserved struct {
	Pos lexer.Position

	Ranges     []Range  `( @@ { "," @@ }`
	FieldNames []string `  | @String { "," @String } )`
}

type Range struct {
	Start int  `( @Int`
	End   *int `  [ "to" ( @Int`
	Max   bool `           | @"max" ) ] )`
}

type Extend struct {
	Pos lexer.Position

	Reference string   `"extend" @("."? Ident { "." Ident })`
	Fields    []*Field `"{" { @@ [ ";" ] } "}"`
}

type Service struct {
	Pos lexer.Position

	Name  string          `"service" @Ident`
	Entry []*ServiceEntry `"{" { @@ [ ";" ] } "}"`
}

type ServiceEntry struct {
	Pos lexer.Position

	Option *Option `  "option" @@`
	Method *Method `| @@`
}

type Method struct {
	Pos lexer.Position

	Name              string    `"rpc" @Ident`
	StreamingRequest  bool      `"(" [ @"stream" ]`
	Request           *Type     `    @@ ")"`
	StreamingResponse bool      `"returns" "(" [ @"stream" ]`
	Response          *Type     `              @@ ")"`
	Options           []*Option `[ "{" { "option" @@ ";" } "}" ]`
}

type Enum struct {
	Pos lexer.Position

	Name   string       `"enum" @Ident`
	Values []*EnumEntry `"{" { @@ { ";" } } "}"`
}

type EnumEntry struct {
	Pos lexer.Position

	Value    *EnumValue `  @@`
	Option   *Option    `| "option" @@`
	Reserved *Reserved  `| "reserved" @@`
}

type EnumValue struct {
	Pos lexer.Position

	Key   string `@Ident`
	Value int    `"=" @( [ "-" ] Int )`

	Options []*Option `[ "[" @@ { "," @@ } "]" ]`
}

type Message struct {
	Pos lexer.Position

	Name    string          `"message" @Ident`
	Entries []*MessageEntry `"{" { @@ } "}"`
}

type MessageEntry struct {
	Pos lexer.Position

	Enum       *Enum       `( @@`
	Option     *Option     ` | "option" @@`
	Message    *Message    ` | @@`
	Oneof      *Oneof      ` | @@`
	Extend     *Extend     ` | @@`
	Reserved   *Reserved   ` | "reserved" @@`
	Extensions *Extensions ` | @@`
	Field      *Field      ` | @@ ) { ";" }`
}

type Oneof struct {
	Pos lexer.Position

	Name    string        `"oneof" @Ident`
	Entries []*OneofEntry `"{" { @@ { ";" } } "}"`
}

type OneofEntry struct {
	Pos lexer.Position

	Field  *Field  `  @@`
	Option *Option `| "option" @@`
}

type Field struct {
	Pos lexer.Position

	Optional bool `[   @"optional"`
	Required bool `  | @"required"`
	Repeated bool `  | @"repeated" ]`

	Group  *Group  `( @@`
	Direct *Direct `| @@ )`
}

type Direct struct {
	Pos lexer.Position

	Type *Type  `@@`
	Name string `@Ident`
	Tag  int    `"=" @Int`

	Options []*Option `[ "[" @@ { "," @@ } "]" ]`
}

type Group struct {
	Pos lexer.Position

	Name    string          `"group" @Ident`
	Tag     int             `"=" @Int`
	Entries []*MessageEntry `"{" { @@ [ ";" ] } "}"`
}

type Scalar int

const (
	None Scalar = iota
	Double
	Float
	Int32
	Int64
	Uint32
	Uint64
	Sint32
	Sint64
	Fixed32
	Fixed64
	SFixed32
	SFixed64
	Bool
	String
	Bytes
)

var scalarToString = map[Scalar]string{
	None: "None", Double: "Double", Float: "Float", Int32: "Int32", Int64: "Int64", Uint32: "Uint32",
	Uint64: "Uint64", Sint32: "Sint32", Sint64: "Sint64", Fixed32: "Fixed32", Fixed64: "Fixed64",
	SFixed32: "SFixed32", SFixed64: "SFixed64", Bool: "Bool", String: "String", Bytes: "Bytes",
}

func (s Scalar) GoString() string { return scalarToString[s] }

var stringToScalar = map[string]Scalar{
	"double": Double, "float": Float, "int32": Int32, "int64": Int64, "uint32": Uint32, "uint64": Uint64,
	"sint32": Sint32, "sint64": Sint64, "fixed32": Fixed32, "fixed64": Fixed64, "sfixed32": SFixed32,
	"sfixed64": SFixed64, "bool": Bool, "string": String, "bytes": Bytes,
}

func (s *Scalar) Parse(lex *lexer.PeekingLexer) error {
	token, err := lex.Peek(0)
	if err != nil {
		return fmt.Errorf("failed to peek next token: %w", err)
	}
	scalar, ok := stringToScalar[token.Value]
	if !ok {
		return participle.NextMatch
	}
	_, err = lex.Next()
	if err != nil {
		return fmt.Errorf("failed to read next token: %w", err)
	}
	*s = scalar
	return nil
}

type Type struct {
	Pos lexer.Position

	Scalar    Scalar   `  @@`
	Map       *MapType `| @@`
	Reference *string  `| @("."? Ident { "." Ident })`
}

type MapType struct {
	Pos lexer.Position

	Key   *Type `"map" "<" @@`
	Value *Type `"," @@ ">"`
}

// Parse protobuf.
func Parse(filename string, r io.Reader) (*Proto, error) {
	p := &Proto{}

	l := lexer.MustSimple([]lexer.Rule{
		{"String", `"(\\"|[^"])*"|'(\\'|[^'])*'`, nil},
		{"Float", `[-+]?\d*\.\d+`, nil},
		{"Int", `(0x[0-9A-Fa-f]+)|([-+]?\d+)`, nil},
		{"Ident", `[a-zA-Z_]([a-zA-Z_0-9])*`, nil},
		{"Whitespace", `[ \t\n\r]+`, nil},
		{"BlockComment", `/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/`, nil},
		{"LineComment", `//(.*)[^\n]*\n`, nil},
		{".", `.`, nil},
	})

	parser := participle.MustBuild(
		&Proto{},
		participle.UseLookahead(2),
		participle.Unquote("String"),
		participle.Lexer(l),
		participle.Elide("Whitespace", "LineComment", "BlockComment"),
	)
	err := parser.Parse(filename, r, p)
	if err != nil {
		return p, err
	}
	return p, nil
}

func ParseString(filename string, source string) (*Proto, error) {
	return Parse(filename, strings.NewReader(source))
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"io/ioutil"
)

type argument struct {
	Description string      `json:"description"`
	Default     interface{} `json:"default"`
	EnvName     string      `json:"env_name"`
	FlagName    string      `json:"flag_name"`
	Type        string      `json:"type"`
	Required    bool        `json:"require"`
}

func parseJSON(filename string) (map[string]argument, error) {
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	var c map[string]argument
	if err := json.Unmarshal(b, &c); err != nil {
		return nil, err
	}

	return c, nil
}
<commit_msg>fixing json tag for required fields<commit_after>package config

import (
	"encoding/json"
	"io/ioutil"
)

type argument struct {
	Description string      `json:"description"`
	Default     interface{} `json:"default"`
	EnvName     string      `json:"env_name"`
	FlagName    string      `json:"flag_name"`
	Type        string      `json:"type"`
	Required    bool        `json:"required"`
}

func parseJSON(filename string) (map[string]argument, error) {
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	var c map[string]argument
	if err := json.Unmarshal(b, &c); err != nil {
		return nil, err
	}

	return c, nil
}
<|endoftext|>


<commit_before>package switching

import (
	bh "github.com/kandoo/beehive"
	"github.com/kandoo/beehive-netctrl/nom"
	"github.com/kandoo/beehive/Godeps/_workspace/src/github.com/golang/glog"
)

type LearningSwitch struct {
	Hub
}

func (h LearningSwitch) Rcv(msg bh.Msg, ctx bh.RcvContext) error {
	in := msg.Data().(nom.PacketIn)
	src := in.Packet.SrcMAC()
	dst := in.Packet.DstMAC()
	glog.V(2).Infof("received packet in from %v to %v", src, dst)
	if dst.IsMulticast() {
		glog.Infof("dropped multi-cast packet to %v", dst)
		return nil
	}

	if dst.IsBroadcast() {
		return h.Hub.Rcv(msg, ctx)
	}

	d := ctx.Dict("mac2port")
	srck := src.Key()
	var p nom.UID
	if err := d.GetGob(srck, &p); err != nil || p != in.InPort {
		if err == nil {
			// TODO(soheil): maybe add support for multi ports.
			glog.Infof("%v is moved from port %v to port %v", src, p, in.InPort)
		}

		if err = d.PutGob(srck, &in.InPort); err != nil {
			glog.Fatalf("cannot serialize port: %v", err)
		}
	}

	dstk := dst.Key()
	err := d.GetGob(dstk, &p)
	if err != nil {
		return h.Hub.Rcv(msg, ctx)
	}

	add := nom.AddFlowEntry{
		Flow: nom.FlowEntry{
			Node: in.Node,
			Match: nom.Match{
				Fields: []nom.Field{
					nom.EthDst{Addr: dst},
				},
			},
			Actions: []nom.Action{
				nom.ActionForward{
					Ports: []nom.UID{p},
				},
			},
		},
	}
	ctx.ReplyTo(msg, add)

	out := nom.PacketOut{
		Node:     in.Node,
		InPort:   in.InPort,
		BufferID: in.BufferID,
		Packet:   in.Packet,
		Actions: []nom.Action{
			nom.ActionForward{
				Ports: []nom.UID{p},
			},
		},
	}
	ctx.ReplyTo(msg, out)
	return nil
}
<commit_msg>Add a TODO<commit_after>package switching

import (
	bh "github.com/kandoo/beehive"
	"github.com/kandoo/beehive-netctrl/nom"
	"github.com/kandoo/beehive/Godeps/_workspace/src/github.com/golang/glog"
)

type LearningSwitch struct {
	Hub
}

func (h LearningSwitch) Rcv(msg bh.Msg, ctx bh.RcvContext) error {
	in := msg.Data().(nom.PacketIn)
	src := in.Packet.SrcMAC()
	dst := in.Packet.DstMAC()
	glog.V(2).Infof("received packet in from %v to %v", src, dst)
	if dst.IsMulticast() {
		// TODO(soheil): just drop LLDP.
		glog.Infof("dropped multi-cast packet to %v", dst)
		return nil
	}

	if dst.IsBroadcast() {
		return h.Hub.Rcv(msg, ctx)
	}

	d := ctx.Dict("mac2port")
	srck := src.Key()
	var p nom.UID
	if err := d.GetGob(srck, &p); err != nil || p != in.InPort {
		if err == nil {
			// TODO(soheil): maybe add support for multi ports.
			glog.Infof("%v is moved from port %v to port %v", src, p, in.InPort)
		}

		if err = d.PutGob(srck, &in.InPort); err != nil {
			glog.Fatalf("cannot serialize port: %v", err)
		}
	}

	dstk := dst.Key()
	err := d.GetGob(dstk, &p)
	if err != nil {
		return h.Hub.Rcv(msg, ctx)
	}

	add := nom.AddFlowEntry{
		Flow: nom.FlowEntry{
			Node: in.Node,
			Match: nom.Match{
				Fields: []nom.Field{
					nom.EthDst{Addr: dst},
				},
			},
			Actions: []nom.Action{
				nom.ActionForward{
					Ports: []nom.UID{p},
				},
			},
		},
	}
	ctx.ReplyTo(msg, add)

	out := nom.PacketOut{
		Node:     in.Node,
		InPort:   in.InPort,
		BufferID: in.BufferID,
		Packet:   in.Packet,
		Actions: []nom.Action{
			nom.ActionForward{
				Ports: []nom.UID{p},
			},
		},
	}
	ctx.ReplyTo(msg, out)
	return nil
}
<|endoftext|>


<commit_before>package main

// pcstat.go - page cache stat
// uses the mincore(2) syscall to find out which pages (almost always 4k)
// of a file are currently cached in memory
//
// Copyright 2014 Albert P. Tobey <atobey@datastax.com> @AlTobey
// License: Apache 2.0

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"strings"
	"syscall"
	"unsafe"
)

// pcStat: page cache status
// Bytes: size of the file (from os.File.Stat())
// Pages: array of booleans: true if cached, false otherwise
type pcStat struct {
	Name     string `json:"filename"` // file name as specified on command line
	Size     int64  `json:"size"`     // file size in bytes
	Pages    int    `json:"pages"`    // total memory pages
	Cached   int    `json:"cached"`   // number of pages that are cached
	Uncached int    `json:"uncached"` // number of pages that are not cached
	PPStat   []bool `json:"status"`   // per-page status, true if cached, false otherwise
}

type pcStatList []pcStat

var (
	terseFlag, nohdrFlag, jsonFlag, ppsFlag, bnameFlag bool
)

func init() {
	// TODO: error on useless/broken combinations
	flag.BoolVar(&terseFlag, "terse", false, "show terse output")
	flag.BoolVar(&nohdrFlag, "nohdr", false, "omit the header from terse & text output")
	flag.BoolVar(&jsonFlag, "json", false, "return data in JSON format")
	flag.BoolVar(&ppsFlag, "pps", false, "include the per-page status in JSON output")
	flag.BoolVar(&bnameFlag, "bname", false, "convert paths to basename to narrow the output")
}

func main() {
	flag.Parse()

	// all non-flag arguments are considered to be filenames
	// this works well with shell globbing
	// file order is preserved throughout this program
	stats := make(pcStatList, len(flag.Args()))

	for i, fname := range flag.Args() {
		stats[i] = getMincore(fname)
	}

	if jsonFlag {
		stats.formatJson()
	} else if terseFlag {
		stats.formatTerse()
	} else {
		stats.formatText()
	}
}

func (stats pcStatList) formatText() {
	// find the longest filename in the data for calculating whitespace padding
	maxName := 8
	for _, pcs := range stats {
		if len(pcs.Name) > maxName {
			maxName = len(pcs.Name)
		}
	}

	// create horizontal grid line
	pad := strings.Repeat("-", maxName+2)
	hr := fmt.Sprintf("|%s+----------------+------------+-----------+---------|", pad)

	fmt.Println(hr)

	// -nohdr may be chosen to save 2 lines of precious vertical space
	if !nohdrFlag {
		pad = strings.Repeat(" ", maxName-4)
		fmt.Printf("| Name%s | Size           | Pages      | Cached    | Percent |\n", pad)
		fmt.Println(hr)
	}

	for _, pcs := range stats {
		// convert to float for the occasional sparsely-cached file
		// see the README.md for how to produce one
		percent := (float64(pcs.Cached) / float64(pcs.Pages)) * 100.00
		pad = strings.Repeat(" ", maxName-len(pcs.Name))

		// %07.3f was chosen to make it easy to scan the percentages vertically
		// I tried a few different formats only this one kept the decimals aligned
		fmt.Printf("| %s%s | %-15d| %-11d| %-10d| %07.3f |\n",
			pcs.Name, pad, pcs.Size, pcs.Pages, pcs.Cached, percent)
	}

	fmt.Println(hr)
}

func (stats pcStatList) formatTerse() {
	if !nohdrFlag {
		fmt.Println("name,size,pages,cached,percent")
	}
	for _, pcs := range stats {
		percent := (pcs.Cached / pcs.Pages) * 100
		fmt.Printf("%s,%d,%d,%d,%d\n", pcs.Name, pcs.Size, pcs.Pages, pcs.Cached, percent)
	}
}

func (stats pcStatList) formatJson() {
	b, err := json.Marshal(stats)
	if err != nil {
		log.Fatalf("JSON formatting failed: %s\n", err)
	}
	os.Stdout.Write(b)
	fmt.Println("")
}

func getMincore(fname string) pcStat {
	f, err := os.Open(fname)
	if err != nil {
		log.Fatalf("Could not open file '%s' for read: %s\n", fname, err)
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		log.Fatalf("Could not stat file %s: %s\n", fname, err)
	}
	if fi.Size() == 0 {
		log.Fatalf("%s appears to be 0 bytes in length\n", fname)
	}

	// []byte slice
	mmap, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_NONE, syscall.MAP_SHARED)
	if err != nil {
		log.Fatalf("Could not mmap file '%s': %s\n", fname, err)
	}
	// TODO: check for MAP_FAILED which is ((void *) -1)
	// but maybe unnecessary since it looks like errno is always set when MAP_FAILED

	// one byte per page, only LSB is used, remainder is reserved and clear
	vecsz := (fi.Size() + int64(os.Getpagesize()) - 1) / int64(os.Getpagesize())
	vec := make([]byte, vecsz)

	// get all of the arguments to the mincore syscall converted to uintptr
	mmap_ptr := uintptr(unsafe.Pointer(&mmap[0]))
	size_ptr := uintptr(fi.Size())
	vec_ptr := uintptr(unsafe.Pointer(&vec[0]))

	// use Go's ASM to submit directly to the kernel, no C wrapper needed
	// mincore(2): int mincore(void *addr, size_t length, unsigned char *vec);
	// 0 on success, takes the pointer to the mmap, a size, which is the
	// size that came from f.Stat(), and the vector, which is a pointer
	// to the memory behind an []byte
	// this writes a snapshot of the data into vec which a list of 8-bit flags
	// with the LSB set if the page in that position is currently in VFS cache
	ret, _, err := syscall.RawSyscall(syscall.SYS_MINCORE, mmap_ptr, size_ptr, vec_ptr)
	if ret != 0 {
		log.Fatalf("syscall SYS_MINCORE failed: %s", err)
	}
	defer syscall.Munmap(mmap)

	pcs := pcStat{fname, fi.Size(), int(vecsz), 0, 0, []bool{}}

	// only export the per-page cache mapping if it's explicitly enabled
	// an empty "status": [] field, but NBD.
	if ppsFlag {
		pcs.PPStat = make([]bool, vecsz)

		// there is no bitshift only bool
		for i, b := range vec {
			if b%2 == 1 {
				pcs.PPStat[i] = true
			} else {
				pcs.PPStat[i] = false
			}
		}

	}

	// convert long paths to their basename with the -bname flag
	// this overwrites the original filename in pcs but it doesn't matter since
	// it's not used to access the file again -- and should not be!
	if bnameFlag {
		pcs.Name = path.Base(fname)
	}

	for i, b := range vec {
		if b%2 == 1 {
			pcs.Cached++
		} else {
			pcs.Uncached++
		}
	}

	return pcs
}
<commit_msg>Oops. Syntax error. Time to sleep.<commit_after>package main

// pcstat.go - page cache stat
// uses the mincore(2) syscall to find out which pages (almost always 4k)
// of a file are currently cached in memory
//
// Copyright 2014 Albert P. Tobey <atobey@datastax.com> @AlTobey
// License: Apache 2.0

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"strings"
	"syscall"
	"unsafe"
)

// pcStat: page cache status
// Bytes: size of the file (from os.File.Stat())
// Pages: array of booleans: true if cached, false otherwise
type pcStat struct {
	Name     string `json:"filename"` // file name as specified on command line
	Size     int64  `json:"size"`     // file size in bytes
	Pages    int    `json:"pages"`    // total memory pages
	Cached   int    `json:"cached"`   // number of pages that are cached
	Uncached int    `json:"uncached"` // number of pages that are not cached
	PPStat   []bool `json:"status"`   // per-page status, true if cached, false otherwise
}

type pcStatList []pcStat

var (
	terseFlag, nohdrFlag, jsonFlag, ppsFlag, bnameFlag bool
)

func init() {
	// TODO: error on useless/broken combinations
	flag.BoolVar(&terseFlag, "terse", false, "show terse output")
	flag.BoolVar(&nohdrFlag, "nohdr", false, "omit the header from terse & text output")
	flag.BoolVar(&jsonFlag, "json", false, "return data in JSON format")
	flag.BoolVar(&ppsFlag, "pps", false, "include the per-page status in JSON output")
	flag.BoolVar(&bnameFlag, "bname", false, "convert paths to basename to narrow the output")
}

func main() {
	flag.Parse()

	// all non-flag arguments are considered to be filenames
	// this works well with shell globbing
	// file order is preserved throughout this program
	stats := make(pcStatList, len(flag.Args()))

	for i, fname := range flag.Args() {
		stats[i] = getMincore(fname)
	}

	if jsonFlag {
		stats.formatJson()
	} else if terseFlag {
		stats.formatTerse()
	} else {
		stats.formatText()
	}
}

func (stats pcStatList) formatText() {
	// find the longest filename in the data for calculating whitespace padding
	maxName := 8
	for _, pcs := range stats {
		if len(pcs.Name) > maxName {
			maxName = len(pcs.Name)
		}
	}

	// create horizontal grid line
	pad := strings.Repeat("-", maxName+2)
	hr := fmt.Sprintf("|%s+----------------+------------+-----------+---------|", pad)

	fmt.Println(hr)

	// -nohdr may be chosen to save 2 lines of precious vertical space
	if !nohdrFlag {
		pad = strings.Repeat(" ", maxName-4)
		fmt.Printf("| Name%s | Size           | Pages      | Cached    | Percent |\n", pad)
		fmt.Println(hr)
	}

	for _, pcs := range stats {
		// convert to float for the occasional sparsely-cached file
		// see the README.md for how to produce one
		percent := (float64(pcs.Cached) / float64(pcs.Pages)) * 100.00
		pad = strings.Repeat(" ", maxName-len(pcs.Name))

		// %07.3f was chosen to make it easy to scan the percentages vertically
		// I tried a few different formats only this one kept the decimals aligned
		fmt.Printf("| %s%s | %-15d| %-11d| %-10d| %07.3f |\n",
			pcs.Name, pad, pcs.Size, pcs.Pages, pcs.Cached, percent)
	}

	fmt.Println(hr)
}

func (stats pcStatList) formatTerse() {
	if !nohdrFlag {
		fmt.Println("name,size,pages,cached,percent")
	}
	for _, pcs := range stats {
		percent := (pcs.Cached / pcs.Pages) * 100
		fmt.Printf("%s,%d,%d,%d,%d\n", pcs.Name, pcs.Size, pcs.Pages, pcs.Cached, percent)
	}
}

func (stats pcStatList) formatJson() {
	b, err := json.Marshal(stats)
	if err != nil {
		log.Fatalf("JSON formatting failed: %s\n", err)
	}
	os.Stdout.Write(b)
	fmt.Println("")
}

func getMincore(fname string) pcStat {
	f, err := os.Open(fname)
	if err != nil {
		log.Fatalf("Could not open file '%s' for read: %s\n", fname, err)
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		log.Fatalf("Could not stat file %s: %s\n", fname, err)
	}
	if fi.Size() == 0 {
		log.Fatalf("%s appears to be 0 bytes in length\n", fname)
	}

	// []byte slice
	mmap, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_NONE, syscall.MAP_SHARED)
	if err != nil {
		log.Fatalf("Could not mmap file '%s': %s\n", fname, err)
	}
	// TODO: check for MAP_FAILED which is ((void *) -1)
	// but maybe unnecessary since it looks like errno is always set when MAP_FAILED

	// one byte per page, only LSB is used, remainder is reserved and clear
	vecsz := (fi.Size() + int64(os.Getpagesize()) - 1) / int64(os.Getpagesize())
	vec := make([]byte, vecsz)

	// get all of the arguments to the mincore syscall converted to uintptr
	mmap_ptr := uintptr(unsafe.Pointer(&mmap[0]))
	size_ptr := uintptr(fi.Size())
	vec_ptr := uintptr(unsafe.Pointer(&vec[0]))

	// use Go's ASM to submit directly to the kernel, no C wrapper needed
	// mincore(2): int mincore(void *addr, size_t length, unsigned char *vec);
	// 0 on success, takes the pointer to the mmap, a size, which is the
	// size that came from f.Stat(), and the vector, which is a pointer
	// to the memory behind an []byte
	// this writes a snapshot of the data into vec which a list of 8-bit flags
	// with the LSB set if the page in that position is currently in VFS cache
	ret, _, err := syscall.RawSyscall(syscall.SYS_MINCORE, mmap_ptr, size_ptr, vec_ptr)
	if ret != 0 {
		log.Fatalf("syscall SYS_MINCORE failed: %s", err)
	}
	defer syscall.Munmap(mmap)

	pcs := pcStat{fname, fi.Size(), int(vecsz), 0, 0, []bool{}}

	// only export the per-page cache mapping if it's explicitly enabled
	// an empty "status": [] field, but NBD.
	if ppsFlag {
		pcs.PPStat = make([]bool, vecsz)

		// there is no bitshift only bool
		for i, b := range vec {
			if b%2 == 1 {
				pcs.PPStat[i] = true
			} else {
				pcs.PPStat[i] = false
			}
		}

	}

	// convert long paths to their basename with the -bname flag
	// this overwrites the original filename in pcs but it doesn't matter since
	// it's not used to access the file again -- and should not be!
	if bnameFlag {
		pcs.Name = path.Base(fname)
	}

	for _, b := range vec {
		if b%2 == 1 {
			pcs.Cached++
		} else {
			pcs.Uncached++
		}
	}

	return pcs
}
<|endoftext|>


<commit_before>// Copyright 2019 xgfone
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package strings2

import (
	"encoding"
	"encoding/json"
	"fmt"
	"io"
	"reflect"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

// Builder is a thin wrapper around a byte slice. It's intended to be pooled, so
// the only way to construct one is via a Pool.
type Builder struct {
	buf []byte
}

// NewBuilder returns a new Builder with a initial capacity n.
func NewBuilder(n int) *Builder {
	return NewBuilderBytes(make([]byte, 0, n))
}

// NewBuilderBytes returns a new Builder with a initial data.
func NewBuilderBytes(buf []byte) *Builder {
	return &Builder{buf: buf}
}

// NewBuilderString returns a new Builder with a initial string.
func NewBuilderString(s string) *Builder {
	b := NewBuilderBytes(make([]byte, 0, len(s)*2))
	b.WriteString(s)
	return b
}

// Len returns the length of the underlying byte slice.
func (b *Builder) Len() int {
	return len(b.buf)
}

// Cap returns the capacity of the underlying byte slice.
func (b *Builder) Cap() int {
	return cap(b.buf)
}

// Bytes returns a mutable reference to the underlying byte slice.
func (b *Builder) Bytes() []byte {
	return b.buf
}

// String returns a string copy of the underlying byte slice.
func (b *Builder) String() string {
	return string(b.buf)
}

// Reset resets the underlying byte slice.
//
// Subsequent writes will re-use the slice's backing array.
func (b *Builder) Reset() {
	b.buf = b.buf[:0]
}

// ResetBytes resets the underlying byte slice to bs.
func (b *Builder) ResetBytes(bs []byte) {
	b.buf = bs
}

// TruncateBefore truncates and discards first n bytes.
//
// It will is equal to reset if n is greater than the length of the underlying
// byte slice,
func (b *Builder) TruncateBefore(n int) {
	if n = len(b.buf) - n; n > 0 {
		copy(b.buf, b.buf[n-1:])
		b.buf = b.buf[:n]
	} else {
		b.buf = b.buf[:0]
	}
}

// TruncateAfter truncates and discards last n bytes.
//
// It will is equal to reset if n is greater than the length of the underlying
// byte slice,
func (b *Builder) TruncateAfter(n int) {
	if n = len(b.buf) - n; n < 0 {
		n = 0
	}
	b.buf = b.buf[:n]
}

// AppendBool appends a bool to the underlying buffer.
func (b *Builder) AppendBool(v bool) {
	b.buf = strconv.AppendBool(b.buf, v)
}

// AppendByte is the same as WriteByte, but no return.
func (b *Builder) AppendByte(c byte) {
	b.WriteByte(c)
}

// AppendString is the same as WriteString, but no return.
func (b *Builder) AppendString(s string) {
	b.WriteString(s)
}

// AppendInt appends an integer to the underlying buffer (assuming base 10).
func (b *Builder) AppendInt(i int64) {
	b.buf = strconv.AppendInt(b.buf, i, 10)
}

// AppendUint appends an unsigned integer to the underlying buffer (assuming
// base 10).
func (b *Builder) AppendUint(i uint64) {
	b.buf = strconv.AppendUint(b.buf, i, 10)
}

// AppendFloat appends a float to the underlying buffer. It doesn't quote NaN
// or +/- Inf.
func (b *Builder) AppendFloat(f float64, bitSize int) {
	b.buf = strconv.AppendFloat(b.buf, f, 'f', -1, bitSize)
}

// AppendTime appends a time to the underlying buffer.
func (b *Builder) AppendTime(t time.Time, layout string) {
	b.buf = t.AppendFormat(b.buf, layout)
}

// AppendAny appends any value to the underlying buffer.
//
// It supports the type:
//    nil     ==> "<nil>"
//    bool    ==> "true" or "false"
//    []byte
//    string
//    float32
//    float64
//    int
//    int8
//    int16
//    int32
//    int64
//    uint
//    uint8
//    uint16
//    uint32
//    uint64
//    time.Time ==> time.RFC3339Nano
//    Slice
//    Map
//    interface error
//    interface fmt.Stringer
//    interface encoding.TextMarshaler
//
// For the unknown type, it does not append it and return false, or return true.
func (b *Builder) AppendAny(any interface{}) (ok bool, err error) {
	switch v := any.(type) {
	case nil:
		b.WriteString("<nil>")
	case bool:
		b.AppendBool(v)
	case []byte:
		b.Write(v)
	case string:
		b.WriteString(v)
	case float32:
		b.AppendFloat(float64(v), 32)
	case float64:
		b.AppendFloat(v, 64)
	case int:
		b.AppendInt(int64(v))
	case int8:
		b.AppendInt(int64(v))
	case int16:
		b.AppendInt(int64(v))
	case int32:
		b.AppendInt(int64(v))
	case int64:
		b.AppendInt(v)
	case uint:
		b.AppendUint(uint64(v))
	case uint8:
		b.AppendUint(uint64(v))
	case uint16:
		b.AppendUint(uint64(v))
	case uint32:
		b.AppendUint(uint64(v))
	case uint64:
		b.AppendUint(v)
	case time.Time:
		b.AppendTime(v, time.RFC3339Nano)
	case fmt.Stringer:
		b.WriteString(v.String())
	case error:
		b.WriteString(v.Error())
	case encoding.TextMarshaler:
		data, err := v.MarshalText()
		if err != nil {
			return true, err
		}
		b.Write(data)
	case []interface{}:
		b.WriteByte('[')
		for i, _v := range v {
			if i > 0 {
				b.WriteByte(' ')
			}
			if ok, err = b.AppendAny(_v); !ok || err != nil {
				return
			}
		}
		b.WriteByte(']')
	case []string:
		b.WriteByte('[')
		for i, _v := range v {
			if i > 0 {
				b.WriteByte(' ')
			}
			b.WriteString(_v)
		}
		b.WriteByte(']')
	case []int:
		b.WriteByte('[')
		for i, _v := range v {
			if i > 0 {
				b.WriteByte(' ')
			}
			b.AppendInt(int64(_v))
		}
		b.WriteByte(']')
	default:
		kind := reflect.ValueOf(v).Kind()
		if kind != reflect.Map && kind != reflect.Slice && kind != reflect.Array {
			return false, nil
		}
		fmt.Fprintf(b, "%v", v)
	}
	return true, nil
}

// AppendAnyFmt is the same as AppendAny(any), but it will use
// `fmt.Sprintf("%+v", any)` to format the unknown type `any`.
func (b *Builder) AppendAnyFmt(any interface{}) error {
	switch any.(type) {
	case nil:
	case bool:
	case []byte:
	case string:
	case float32:
	case float64:
	case int:
	case int8:
	case int16:
	case int32:
	case int64:
	case uint:
	case uint8:
	case uint16:
	case uint32:
	case uint64:
	case time.Time:
	case fmt.Stringer:
	case error:
	case encoding.TextMarshaler:
	case []interface{}:
	case []string:
	case []int:
	default:
		fmt.Fprintf(b, "%+v", any)
	}
	_, err := b.AppendAny(any)
	return err
}

// AppendJSONString appends a string as JSON string, which will escape
// the double quotation(") and enclose it with a pair of the double quotation.
func (b *Builder) AppendJSONString(s string) {
	if strings.IndexByte(s, '"') > -1 {
		b.buf = strconv.AppendQuote(b.buf, s)
	} else {
		b.WriteByte('"')
		b.WriteString(s)
		b.WriteByte('"')
	}
}

// AppendJSON appends the value as the JSON value, that's, the value will
// be encoded to JSON and appended into the underlying byte slice.
func (b *Builder) AppendJSON(value interface{}) error {
	switch v := value.(type) {
	case nil:
		b.WriteString(`null`)
	case bool:
		if v {
			b.WriteString(`true`)
		} else {
			b.WriteString(`false`)
		}
	case int:
		b.AppendInt(int64(v))
	case int8:
		b.AppendInt(int64(v))
	case int16:
		b.AppendInt(int64(v))
	case int32:
		b.AppendInt(int64(v))
	case int64:
		b.AppendInt(v)
	case uint:
		b.AppendUint(uint64(v))
	case uint8:
		b.AppendUint(uint64(v))
	case uint16:
		b.AppendUint(uint64(v))
	case uint32:
		b.AppendUint(uint64(v))
	case uint64:
		b.AppendUint(v)
	case float32:
		b.AppendFloat(float64(v), 32)
	case float64:
		b.AppendFloat(v, 64)
	case time.Time:
		b.WriteByte('"')
		b.AppendTime(v, time.RFC3339Nano)
		b.WriteByte('"')
	case error:
		b.AppendJSONString(v.Error())
	case string:
		b.AppendJSONString(v)
	case fmt.Stringer:
		b.AppendJSONString(v.String())
	case json.Marshaler:
		data, err := v.MarshalJSON()
		if err != nil {
			return err
		}
		b.Write(data)
	case []interface{}: // Optimize []interface{}
		b.WriteByte('[')
		for i, _v := range v {
			if i > 0 {
				b.WriteByte(',')
			}
			if err := b.AppendJSON(_v); err != nil {
				return err
			}
		}
		b.WriteByte(']')
	case []string: // Optimize []string
		b.WriteByte('[')
		for i, _v := range v {
			if i > 0 {
				b.WriteByte(',')
			}
			b.AppendJSONString(_v)
		}
		b.WriteByte(']')
	case []int: // Optimize []int
		b.WriteByte('[')
		for i, _v := range v {
			if i > 0 {
				b.WriteByte(',')
			}
			b.AppendInt(int64(_v))
		}
		b.WriteByte(']')
	case map[string]interface{}: // Optimize map[string]interface{}
		b.WriteByte('{')
		var i int
		for key, value := range v {
			if i > 0 {
				b.WriteByte(',')
			}
			i++
			b.AppendJSONString(key)
			b.WriteByte(':')
			if err := b.AppendJSON(value); err != nil {
				return err
			}
		}
		b.WriteByte('}')
	case map[string]string: // Optimize map[string]string
		b.WriteByte('{')
		var i int
		for key, value := range v {
			if i > 0 {
				b.WriteByte(',')
			}
			i++
			b.AppendJSONString(key)
			b.WriteByte(':')
			b.AppendJSONString(value)
		}
		b.WriteByte('}')
	default:
		data, err := json.Marshal(v)
		if err != nil {
			return err
		}
		b.Write(data)
	}
	return nil
}

// Write implements io.Writer.
func (b *Builder) Write(bs []byte) (int, error) {
	b.buf = append(b.buf, bs...)
	return len(bs), nil
}

// WriteByte writes a byte into the builder.
func (b *Builder) WriteByte(c byte) error {
	b.buf = append(b.buf, c)
	return nil
}

// WriteRune writes a rune into the builder.
func (b *Builder) WriteRune(r rune) (int, error) {
	if r < utf8.RuneSelf {
		b.WriteByte(byte(r))
		return 1, nil
	}

	var buf [utf8.UTFMax]byte
	n := utf8.EncodeRune(buf[:], r)
	b.buf = append(b.buf, buf[:n]...)
	return n, nil
}

// WriteString writes a string into the builder.
func (b *Builder) WriteString(s string) (int, error) {
	b.buf = append(b.buf, s...)
	return len(s), nil
}

// WriteTo implements io.WriterTo.
func (b *Builder) WriteTo(w io.Writer) (int64, error) {
	n, err := w.Write(b.buf)
	return int64(n), err
}

// TrimNewline trims any final "\n" byte from the end of the buffer.
func (b *Builder) TrimNewline() {
	maxIndex := len(b.buf) - 1
	for i := maxIndex; i >= 0; i-- {
		if b.buf[i] != '\n' {
			if i < maxIndex {
				b.buf = b.buf[:i+1]
			}
			return
		}
	}
}
<commit_msg>fix: AppendAnyFmt returns nil for default<commit_after>// Copyright 2019 xgfone
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package strings2

import (
	"encoding"
	"encoding/json"
	"fmt"
	"io"
	"reflect"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

// Builder is a thin wrapper around a byte slice. It's intended to be pooled, so
// the only way to construct one is via a Pool.
type Builder struct {
	buf []byte
}

// NewBuilder returns a new Builder with a initial capacity n.
func NewBuilder(n int) *Builder {
	return NewBuilderBytes(make([]byte, 0, n))
}

// NewBuilderBytes returns a new Builder with a initial data.
func NewBuilderBytes(buf []byte) *Builder {
	return &Builder{buf: buf}
}

// NewBuilderString returns a new Builder with a initial string.
func NewBuilderString(s string) *Builder {
	b := NewBuilderBytes(make([]byte, 0, len(s)*2))
	b.WriteString(s)
	return b
}

// Len returns the length of the underlying byte slice.
func (b *Builder) Len() int {
	return len(b.buf)
}

// Cap returns the capacity of the underlying byte slice.
func (b *Builder) Cap() int {
	return cap(b.buf)
}

// Bytes returns a mutable reference to the underlying byte slice.
func (b *Builder) Bytes() []byte {
	return b.buf
}

// String returns a string copy of the underlying byte slice.
func (b *Builder) String() string {
	return string(b.buf)
}

// Reset resets the underlying byte slice.
//
// Subsequent writes will re-use the slice's backing array.
func (b *Builder) Reset() {
	b.buf = b.buf[:0]
}

// ResetBytes resets the underlying byte slice to bs.
func (b *Builder) ResetBytes(bs []byte) {
	b.buf = bs
}

// TruncateBefore truncates and discards first n bytes.
//
// It will is equal to reset if n is greater than the length of the underlying
// byte slice,
func (b *Builder) TruncateBefore(n int) {
	if n = len(b.buf) - n; n > 0 {
		copy(b.buf, b.buf[n-1:])
		b.buf = b.buf[:n]
	} else {
		b.buf = b.buf[:0]
	}
}

// TruncateAfter truncates and discards last n bytes.
//
// It will is equal to reset if n is greater than the length of the underlying
// byte slice,
func (b *Builder) TruncateAfter(n int) {
	if n = len(b.buf) - n; n < 0 {
		n = 0
	}
	b.buf = b.buf[:n]
}

// AppendBool appends a bool to the underlying buffer.
func (b *Builder) AppendBool(v bool) {
	b.buf = strconv.AppendBool(b.buf, v)
}

// AppendByte is the same as WriteByte, but no return.
func (b *Builder) AppendByte(c byte) {
	b.WriteByte(c)
}

// AppendString is the same as WriteString, but no return.
func (b *Builder) AppendString(s string) {
	b.WriteString(s)
}

// AppendInt appends an integer to the underlying buffer (assuming base 10).
func (b *Builder) AppendInt(i int64) {
	b.buf = strconv.AppendInt(b.buf, i, 10)
}

// AppendUint appends an unsigned integer to the underlying buffer (assuming
// base 10).
func (b *Builder) AppendUint(i uint64) {
	b.buf = strconv.AppendUint(b.buf, i, 10)
}

// AppendFloat appends a float to the underlying buffer. It doesn't quote NaN
// or +/- Inf.
func (b *Builder) AppendFloat(f float64, bitSize int) {
	b.buf = strconv.AppendFloat(b.buf, f, 'f', -1, bitSize)
}

// AppendTime appends a time to the underlying buffer.
func (b *Builder) AppendTime(t time.Time, layout string) {
	b.buf = t.AppendFormat(b.buf, layout)
}

// AppendAny appends any value to the underlying buffer.
//
// It supports the type:
//    nil     ==> "<nil>"
//    bool    ==> "true" or "false"
//    []byte
//    string
//    float32
//    float64
//    int
//    int8
//    int16
//    int32
//    int64
//    uint
//    uint8
//    uint16
//    uint32
//    uint64
//    time.Time ==> time.RFC3339Nano
//    Slice
//    Map
//    interface error
//    interface fmt.Stringer
//    interface encoding.TextMarshaler
//
// For the unknown type, it does not append it and return false, or return true.
func (b *Builder) AppendAny(any interface{}) (ok bool, err error) {
	switch v := any.(type) {
	case nil:
		b.WriteString("<nil>")
	case bool:
		b.AppendBool(v)
	case []byte:
		b.Write(v)
	case string:
		b.WriteString(v)
	case float32:
		b.AppendFloat(float64(v), 32)
	case float64:
		b.AppendFloat(v, 64)
	case int:
		b.AppendInt(int64(v))
	case int8:
		b.AppendInt(int64(v))
	case int16:
		b.AppendInt(int64(v))
	case int32:
		b.AppendInt(int64(v))
	case int64:
		b.AppendInt(v)
	case uint:
		b.AppendUint(uint64(v))
	case uint8:
		b.AppendUint(uint64(v))
	case uint16:
		b.AppendUint(uint64(v))
	case uint32:
		b.AppendUint(uint64(v))
	case uint64:
		b.AppendUint(v)
	case time.Time:
		b.AppendTime(v, time.RFC3339Nano)
	case fmt.Stringer:
		b.WriteString(v.String())
	case error:
		b.WriteString(v.Error())
	case encoding.TextMarshaler:
		data, err := v.MarshalText()
		if err != nil {
			return true, err
		}
		b.Write(data)
	case []interface{}:
		b.WriteByte('[')
		for i, _v := range v {
			if i > 0 {
				b.WriteByte(' ')
			}
			if ok, err = b.AppendAny(_v); !ok || err != nil {
				return
			}
		}
		b.WriteByte(']')
	case []string:
		b.WriteByte('[')
		for i, _v := range v {
			if i > 0 {
				b.WriteByte(' ')
			}
			b.WriteString(_v)
		}
		b.WriteByte(']')
	case []int:
		b.WriteByte('[')
		for i, _v := range v {
			if i > 0 {
				b.WriteByte(' ')
			}
			b.AppendInt(int64(_v))
		}
		b.WriteByte(']')
	default:
		kind := reflect.ValueOf(v).Kind()
		if kind != reflect.Map && kind != reflect.Slice && kind != reflect.Array {
			return false, nil
		}
		fmt.Fprintf(b, "%v", v)
	}
	return true, nil
}

// AppendAnyFmt is the same as AppendAny(any), but it will use
// `fmt.Sprintf("%+v", any)` to format the unknown type `any`.
func (b *Builder) AppendAnyFmt(any interface{}) error {
	switch any.(type) {
	case nil:
	case bool:
	case []byte:
	case string:
	case float32:
	case float64:
	case int:
	case int8:
	case int16:
	case int32:
	case int64:
	case uint:
	case uint8:
	case uint16:
	case uint32:
	case uint64:
	case time.Time:
	case fmt.Stringer:
	case error:
	case encoding.TextMarshaler:
	case []interface{}:
	case []string:
	case []int:
	default:
		fmt.Fprintf(b, "%+v", any)
		return nil
	}
	_, err := b.AppendAny(any)
	return err
}

// AppendJSONString appends a string as JSON string, which will escape
// the double quotation(") and enclose it with a pair of the double quotation.
func (b *Builder) AppendJSONString(s string) {
	if strings.IndexByte(s, '"') > -1 {
		b.buf = strconv.AppendQuote(b.buf, s)
	} else {
		b.WriteByte('"')
		b.WriteString(s)
		b.WriteByte('"')
	}
}

// AppendJSON appends the value as the JSON value, that's, the value will
// be encoded to JSON and appended into the underlying byte slice.
func (b *Builder) AppendJSON(value interface{}) error {
	switch v := value.(type) {
	case nil:
		b.WriteString(`null`)
	case bool:
		if v {
			b.WriteString(`true`)
		} else {
			b.WriteString(`false`)
		}
	case int:
		b.AppendInt(int64(v))
	case int8:
		b.AppendInt(int64(v))
	case int16:
		b.AppendInt(int64(v))
	case int32:
		b.AppendInt(int64(v))
	case int64:
		b.AppendInt(v)
	case uint:
		b.AppendUint(uint64(v))
	case uint8:
		b.AppendUint(uint64(v))
	case uint16:
		b.AppendUint(uint64(v))
	case uint32:
		b.AppendUint(uint64(v))
	case uint64:
		b.AppendUint(v)
	case float32:
		b.AppendFloat(float64(v), 32)
	case float64:
		b.AppendFloat(v, 64)
	case time.Time:
		b.WriteByte('"')
		b.AppendTime(v, time.RFC3339Nano)
		b.WriteByte('"')
	case error:
		b.AppendJSONString(v.Error())
	case string:
		b.AppendJSONString(v)
	case fmt.Stringer:
		b.AppendJSONString(v.String())
	case json.Marshaler:
		data, err := v.MarshalJSON()
		if err != nil {
			return err
		}
		b.Write(data)
	case []interface{}: // Optimize []interface{}
		b.WriteByte('[')
		for i, _v := range v {
			if i > 0 {
				b.WriteByte(',')
			}
			if err := b.AppendJSON(_v); err != nil {
				return err
			}
		}
		b.WriteByte(']')
	case []string: // Optimize []string
		b.WriteByte('[')
		for i, _v := range v {
			if i > 0 {
				b.WriteByte(',')
			}
			b.AppendJSONString(_v)
		}
		b.WriteByte(']')
	case []int: // Optimize []int
		b.WriteByte('[')
		for i, _v := range v {
			if i > 0 {
				b.WriteByte(',')
			}
			b.AppendInt(int64(_v))
		}
		b.WriteByte(']')
	case map[string]interface{}: // Optimize map[string]interface{}
		b.WriteByte('{')
		var i int
		for key, value := range v {
			if i > 0 {
				b.WriteByte(',')
			}
			i++
			b.AppendJSONString(key)
			b.WriteByte(':')
			if err := b.AppendJSON(value); err != nil {
				return err
			}
		}
		b.WriteByte('}')
	case map[string]string: // Optimize map[string]string
		b.WriteByte('{')
		var i int
		for key, value := range v {
			if i > 0 {
				b.WriteByte(',')
			}
			i++
			b.AppendJSONString(key)
			b.WriteByte(':')
			b.AppendJSONString(value)
		}
		b.WriteByte('}')
	default:
		data, err := json.Marshal(v)
		if err != nil {
			return err
		}
		b.Write(data)
	}
	return nil
}

// Write implements io.Writer.
func (b *Builder) Write(bs []byte) (int, error) {
	b.buf = append(b.buf, bs...)
	return len(bs), nil
}

// WriteByte writes a byte into the builder.
func (b *Builder) WriteByte(c byte) error {
	b.buf = append(b.buf, c)
	return nil
}

// WriteRune writes a rune into the builder.
func (b *Builder) WriteRune(r rune) (int, error) {
	if r < utf8.RuneSelf {
		b.WriteByte(byte(r))
		return 1, nil
	}

	var buf [utf8.UTFMax]byte
	n := utf8.EncodeRune(buf[:], r)
	b.buf = append(b.buf, buf[:n]...)
	return n, nil
}

// WriteString writes a string into the builder.
func (b *Builder) WriteString(s string) (int, error) {
	b.buf = append(b.buf, s...)
	return len(s), nil
}

// WriteTo implements io.WriterTo.
func (b *Builder) WriteTo(w io.Writer) (int64, error) {
	n, err := w.Write(b.buf)
	return int64(n), err
}

// TrimNewline trims any final "\n" byte from the end of the buffer.
func (b *Builder) TrimNewline() {
	maxIndex := len(b.buf) - 1
	for i := maxIndex; i >= 0; i-- {
		if b.buf[i] != '\n' {
			if i < maxIndex {
				b.buf = b.buf[:i+1]
			}
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/client"
)

type estafettePipelineRunResult struct {
	Pipeline           estafettePipeline
	DockerPullDuration time.Duration
	DockerPullError    error
	DockerRunDuration  time.Duration
	DockerRunError     error
	Status             string
	Detail             string
}

func runDockerPull(p estafettePipeline) (err error) {

	fmt.Printf("[estafette] Pulling docker container '%v'\n", p.ContainerImage)

	cli, err := client.NewEnvClient()
	if err != nil {
		return err
	}

	rc, err := cli.ImagePull(context.Background(), p.ContainerImage, types.ImagePullOptions{})
	defer rc.Close()
	if err != nil {
		return err
	}

	// wait for image pull to finish
	_, err = ioutil.ReadAll(rc)
	if err != nil {
		return err
	}

	return
}

func runDockerRun(dir string, envvars map[string]string, p estafettePipeline) (err error) {

	// run docker with image and commands from yaml
	ctx := context.Background()
	cli, err := client.NewEnvClient()
	if err != nil {
		return err
	}

	// define commands
	cmdSlice := make([]string, 0)
	cmdSlice = append(cmdSlice, p.Shell)
	cmdSlice = append(cmdSlice, "-c")
	cmdSlice = append(cmdSlice, "set -e;"+os.ExpandEnv(strings.Join(p.Commands, ";")))

	// define envvars
	envVars := make([]string, 0)
	if envvars != nil && len(envvars) > 0 {
		for k, v := range envvars {
			envVars = append(envVars, fmt.Sprintf("\"%v=%v\"", k, v))
		}
	}

	// define entrypoint
	entrypoint := make([]string, 0)
	entrypoint = append(entrypoint, "")

	// define binds
	binds := make([]string, 0)
	binds = append(binds, fmt.Sprintf("%v:%v", dir, os.ExpandEnv(p.WorkingDirectory)))
	binds = append(binds, "/var/run/docker.sock:/var/run/docker.sock")
	binds = append(binds, "/var/run/secrets/kubernetes.io/serviceaccount:/var/run/secrets/kubernetes.io/serviceaccount")

	// create container
	resp, err := cli.ContainerCreate(ctx, &container.Config{
		AttachStdout: true,
		AttachStderr: true,
		Env:          envVars,
		Cmd:          cmdSlice,
		Image:        p.ContainerImage,
		WorkingDir:   os.ExpandEnv(p.WorkingDirectory),
		Entrypoint:   entrypoint,
	}, &container.HostConfig{
		Binds:      binds,
		AutoRemove: true,
		Privileged: true,
	}, &network.NetworkingConfig{}, "")
	if err != nil {
		return err
	}

	// start container
	if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
		return err
	}

	// follow logs
	rc, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{
		ShowStdout: true,
		ShowStderr: true,
		Follow:     true,
	})
	defer rc.Close()
	if err != nil {
		return err
	}

	// stream logs to stdout with buffering
	in := bufio.NewScanner(rc)
	for in.Scan() {

		// strip first 8 bytes, they contain docker control characters (https://github.com/docker/docker/issues/7375)
		logLine := in.Text()
		output := os.Stdout
		if len(logLine) > 8 {

			headers := []byte(logLine[0:8])

			// first byte contains the streamType
			// -   0: stdin (will be written on stdout)
			// -   1: stdout
			// -   2: stderr
			streamType := headers[0]

			if streamType == 2 {
				output = os.Stderr
			}

			logLine = logLine[8:]
		}

		fmt.Fprintf(output, "[estafette] [%v] %v\n", p.Name, logLine)
	}
	if err := in.Err(); err != nil {
		fmt.Fprintf(os.Stderr, "[estafette] [%v] Error: %v\n", p.Name, err)
		return err
	}

	// wait for container to stop run
	if _, err = cli.ContainerWait(ctx, resp.ID); err != nil {
		return err
	}

	return
}

func runPipeline(dir string, envvars map[string]string, p estafettePipeline) (result estafettePipelineRunResult, err error) {

	result.Pipeline = p

	fmt.Printf("[estafette] Starting pipeline '%v'\n", p.Name)

	// pull docker image
	dockerPullStart := time.Now()
	result.DockerPullError = runDockerPull(p)
	result.DockerPullDuration = time.Since(dockerPullStart)
	if err != nil {
		return
	}

	// run commands in docker container
	dockerRunStart := time.Now()
	result.DockerRunError = runDockerRun(dir, envvars, p)
	result.DockerRunDuration = time.Since(dockerRunStart)
	if err != nil {
		return
	}

	fmt.Printf("[estafette] Finished pipeline '%v' successfully\n", p.Name)

	return
}

type estafetteRunPipelinesResult struct {
	PipelineResults []estafettePipelineRunResult
}

func runPipelines(manifest estafetteManifest, dir string, envvars map[string]string) (result estafetteRunPipelinesResult, firstErr error) {

	for _, p := range manifest.Pipelines {

		if firstErr != nil {

			// if an error has happened in one of the previous steps we still want to render the following steps in the result table
			r := estafettePipelineRunResult{
				Pipeline: *p,
				Status:   "SKIPPED",
			}
			result.PipelineResults = append(result.PipelineResults, r)

			continue
		}

		r, err := runPipeline(dir, envvars, *p)
		if err != nil {
			if firstErr == nil {
				firstErr = err
			}

			r.Status = "FAILED"
			if r.DockerPullError != nil {
				r.Detail = r.DockerPullError.Error()
			} else if r.DockerRunError != nil {
				r.Detail = r.DockerRunError.Error()
			} else {
				r.Detail = err.Error()
			}

			result.PipelineResults = append(result.PipelineResults, r)

			continue
		}

		r.Status = "SUCCEEDED"
		result.PipelineResults = append(result.PipelineResults, r)
	}

	return
}
<commit_msg>return docker pull or run error as error from runPipeline<commit_after>package main

import (
	"bufio"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/client"
)

type estafettePipelineRunResult struct {
	Pipeline           estafettePipeline
	DockerPullDuration time.Duration
	DockerPullError    error
	DockerRunDuration  time.Duration
	DockerRunError     error
	Status             string
	Detail             string
}

func runDockerPull(p estafettePipeline) (err error) {

	fmt.Printf("[estafette] Pulling docker container '%v'\n", p.ContainerImage)

	cli, err := client.NewEnvClient()
	if err != nil {
		return err
	}

	rc, err := cli.ImagePull(context.Background(), p.ContainerImage, types.ImagePullOptions{})
	defer rc.Close()
	if err != nil {
		return err
	}

	// wait for image pull to finish
	_, err = ioutil.ReadAll(rc)
	if err != nil {
		return err
	}

	return
}

func runDockerRun(dir string, envvars map[string]string, p estafettePipeline) (err error) {

	// run docker with image and commands from yaml
	ctx := context.Background()
	cli, err := client.NewEnvClient()
	if err != nil {
		return err
	}

	// define commands
	cmdSlice := make([]string, 0)
	cmdSlice = append(cmdSlice, p.Shell)
	cmdSlice = append(cmdSlice, "-c")
	cmdSlice = append(cmdSlice, "set -e;"+os.ExpandEnv(strings.Join(p.Commands, ";")))

	// define envvars
	envVars := make([]string, 0)
	if envvars != nil && len(envvars) > 0 {
		for k, v := range envvars {
			envVars = append(envVars, fmt.Sprintf("\"%v=%v\"", k, v))
		}
	}

	// define entrypoint
	entrypoint := make([]string, 0)
	entrypoint = append(entrypoint, "")

	// define binds
	binds := make([]string, 0)
	binds = append(binds, fmt.Sprintf("%v:%v", dir, os.ExpandEnv(p.WorkingDirectory)))
	binds = append(binds, "/var/run/docker.sock:/var/run/docker.sock")
	binds = append(binds, "/var/run/secrets/kubernetes.io/serviceaccount:/var/run/secrets/kubernetes.io/serviceaccount")

	// create container
	resp, err := cli.ContainerCreate(ctx, &container.Config{
		AttachStdout: true,
		AttachStderr: true,
		Env:          envVars,
		Cmd:          cmdSlice,
		Image:        p.ContainerImage,
		WorkingDir:   os.ExpandEnv(p.WorkingDirectory),
		Entrypoint:   entrypoint,
	}, &container.HostConfig{
		Binds:      binds,
		AutoRemove: true,
		Privileged: true,
	}, &network.NetworkingConfig{}, "")
	if err != nil {
		return err
	}

	// start container
	if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
		return err
	}

	// follow logs
	rc, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{
		ShowStdout: true,
		ShowStderr: true,
		Follow:     true,
	})
	defer rc.Close()
	if err != nil {
		return err
	}

	// stream logs to stdout with buffering
	in := bufio.NewScanner(rc)
	for in.Scan() {

		// strip first 8 bytes, they contain docker control characters (https://github.com/docker/docker/issues/7375)
		logLine := in.Text()
		output := os.Stdout
		if len(logLine) > 8 {

			headers := []byte(logLine[0:8])

			// first byte contains the streamType
			// -   0: stdin (will be written on stdout)
			// -   1: stdout
			// -   2: stderr
			streamType := headers[0]

			if streamType == 2 {
				output = os.Stderr
			}

			logLine = logLine[8:]
		}

		fmt.Fprintf(output, "[estafette] [%v] %v\n", p.Name, logLine)
	}
	if err := in.Err(); err != nil {
		fmt.Fprintf(os.Stderr, "[estafette] [%v] Error: %v\n", p.Name, err)
		return err
	}

	// wait for container to stop run
	if _, err = cli.ContainerWait(ctx, resp.ID); err != nil {
		return err
	}

	return
}

func runPipeline(dir string, envvars map[string]string, p estafettePipeline) (result estafettePipelineRunResult, err error) {

	result.Pipeline = p

	fmt.Printf("[estafette] Starting pipeline '%v'\n", p.Name)

	// pull docker image
	dockerPullStart := time.Now()
	result.DockerPullError = runDockerPull(p)
	result.DockerPullDuration = time.Since(dockerPullStart)
	if result.DockerPullError != nil {
		return result, result.DockerPullError
	}

	// run commands in docker container
	dockerRunStart := time.Now()
	result.DockerRunError = runDockerRun(dir, envvars, p)
	result.DockerRunDuration = time.Since(dockerRunStart)
	if result.DockerRunError != nil {
		return result, result.DockerRunError
	}

	fmt.Printf("[estafette] Finished pipeline '%v' successfully\n", p.Name)

	return
}

type estafetteRunPipelinesResult struct {
	PipelineResults []estafettePipelineRunResult
	Errors          []error
}

func runPipelines(manifest estafetteManifest, dir string, envvars map[string]string) (result estafetteRunPipelinesResult, firstErr error) {

	for _, p := range manifest.Pipelines {

		if firstErr != nil {

			// if an error has happened in one of the previous steps we still want to render the following steps in the result table
			r := estafettePipelineRunResult{
				Pipeline: *p,
				Status:   "SKIPPED",
			}
			result.PipelineResults = append(result.PipelineResults, r)

			continue
		}

		r, err := runPipeline(dir, envvars, *p)
		if err != nil {
			if firstErr == nil {
				firstErr = err
			}

			r.Status = "FAILED"
			if r.DockerPullError != nil {
				r.Detail = r.DockerPullError.Error()
			} else if r.DockerRunError != nil {
				r.Detail = r.DockerRunError.Error()
			} else {
				r.Detail = err.Error()
			}

			result.PipelineResults = append(result.PipelineResults, r)

			continue
		}

		r.Status = "SUCCEEDED"
		result.PipelineResults = append(result.PipelineResults, r)
	}

	return
}
<|endoftext|>