<commit_before>/*
Package css is a CSS3 tokenizer and parser written in Go. Both are implemented using the specifications at http://www.w3.org/TR/css-syntax-3/
Tokenizer using example:
	package main
	import (
		"fmt"
		"io"
		"os"
		"github.com/tdewolff/css"
	)
	// Tokenize CSS3 from stdin.
	func main() {
		z := css.NewTokenizer(os.Stdin)
		for {
			tt, data := z.Next()
			switch tt {
			case css.ErrorToken:
				if z.Err() != io.EOF {
					fmt.Println("Error on line", z.Line(), ":", z.Err())
				}
				return
			case css.IdentToken:
				fmt.Println("Identifier", data)
			case css.NumberToken:
				fmt.Println("Number", data)
			// ...
			}
		}
	}
Parser using example:
	package main
	import (
		"fmt"
		"os"
		"github.com/tdewolff/css"
	)
	// Parse CSS3 from stdin.
	func main() {
		stylesheet, err := css.Parse(os.Stdin)
		if err != nil {
			fmt.Println("Error:", err)
			return
		}
		for _, node := range stylesheet.Nodes {
			switch m := node.(type) {
			case *css.TokenNode:
				fmt.Println("Token", string(m.Data))
			case *css.DeclarationNode:
				fmt.Println("Declaration for property", string(m.Prop.Data))
			case *css.RulesetNode:
				fmt.Println("Ruleset with", len(m.Decls), "declarations")
			case *css.AtRuleNode:
				fmt.Println("AtRule", string(m.At.Data))
			}
		}
	}
*/
package css // import "github.com/tdewolff/parse/css"

import (
	"io"
	"strconv"

	"github.com/tdewolff/parse"
)

////////////////////////////////////////////////////////////////

// GrammarType determines the type of grammar.
type GrammarType uint32

// GrammarType values.
const (
	ErrorGrammar GrammarType = iota // extra token when errors occur
	AtRuleGrammar
	StartAtRuleGrammar
	EndAtRuleGrammar
	StartRulesetGrammar
	EndRulesetGrammar
	DeclarationGrammar
	TokenGrammar
)

// String returns the string representation of a GrammarType.
func (tt GrammarType) String() string {
	switch tt {
	case ErrorGrammar:
		return "Error"
	case AtRuleGrammar:
		return "AtRule"
	case StartAtRuleGrammar:
		return "StartAtRule"
	case EndAtRuleGrammar:
		return "EndAtRule"
	case StartRulesetGrammar:
		return "StartRuleset"
	case EndRulesetGrammar:
		return "EndRuleset"
	case DeclarationGrammar:
		return "Declaration"
	case TokenGrammar:
		return "Token"
	}
	return "Invalid(" + strconv.Itoa(int(tt)) + ")"
}

// ParserState denotes the state of the parser.
type ParserState uint32

// ParserState values.
const (
	StylesheetState ParserState = iota
	AtRuleState
	RulesetState
)

// String returns the string representation of a ParserState.
func (tt ParserState) String() string {
	switch tt {
	case StylesheetState:
		return "Stylesheet"
	case AtRuleState:
		return "AtRule"
	case RulesetState:
		return "Ruleset"
	}
	return "Invalid(" + strconv.Itoa(int(tt)) + ")"
}

////////////////////////////////////////////////////////////////

// Parser is the state for the parser.
type Parser struct {
	z     *Tokenizer
	state []ParserState

	tb   *tokenBuffer
	copy bool
}

// NewParser returns a new Parser for a io.Reader.
func NewParser(r io.Reader) *Parser {
	z := NewTokenizer(r)
	return &Parser{
		z:     z,
		state: []ParserState{StylesheetState},
		tb:    newTokenBuffer(z),
	}
}

// Parse parses the entire CSS file and returns the root StylesheetNode.
func Parse(r io.Reader) (*StylesheetNode, error) {
	p := NewParser(r)
	p.tb.EnableLookback()
	var err error
	stylesheet := &StylesheetNode{}
	for {
		gt, n := p.Next()
		if gt == ErrorGrammar {
			err = p.z.Err()
			break
		}
		stylesheet.Nodes = append(stylesheet.Nodes, n)
		if err = p.parseRecursively(gt, n); err != nil {
			break
		}
	}
	if err != io.EOF {
		return stylesheet, err
	}
	return stylesheet, nil
}

func (p *Parser) parseRecursively(rootGt GrammarType, n Node) error {
	if rootGt == StartAtRuleGrammar {
		atRule := n.(*AtRuleNode)
		for {
			gt, m := p.Next()
			if gt == ErrorGrammar {
				return p.z.Err()
			} else if gt == EndAtRuleGrammar {
				break
			}
			atRule.Rules = append(atRule.Rules, m)
			if err := p.parseRecursively(gt, m); err != nil {
				return err
			}
		}
	} else if rootGt == StartRulesetGrammar {
		ruleset := n.(*RulesetNode)
		for {
			gt, m := p.Next()
			if gt == ErrorGrammar {
				return p.z.Err()
			} else if gt == EndRulesetGrammar {
				break
			}
			if decl, ok := m.(*DeclarationNode); ok {
				ruleset.Decls = append(ruleset.Decls, decl)
			}
		}
	}
	return nil
}

// Err returns the error encountered during parsing, this is often io.EOF but also other errors can be returned.
func (p *Parser) Err() error {
	return p.z.Err()
}

// Next returns the next grammar unit from the CSS file.
// This is a lower-level function than Parse and is used to stream parse CSS instead of loading it fully into memory.
// Returned nodes of AtRule and Ruleset do not contain entries for Decls and Rules respectively. These are returned consecutively with calls to Next.
func (p *Parser) Next() (GrammarType, Node) {
	if p.at(ErrorToken) {
		return ErrorGrammar, nil
	}
	for p.at(WhitespaceToken) || p.at(SemicolonToken) {
		p.tb.Shift()
	}

	state := p.state[len(p.state)-1]
	if p.at(RightBraceToken) && (state == AtRuleState || state == RulesetState) {
		// return End types
		token := p.tb.Shift()
		p.state = p.state[:len(p.state)-1]
		if state == AtRuleState {
			return EndAtRuleGrammar, token
		} else {
			return EndRulesetGrammar, token
		}
	} else if p.at(CDOToken) || p.at(CDCToken) {
		return TokenGrammar, p.tb.Shift()
	}

	// find out whether this is a declaration or ruleset, because we don't know if we have a stylesheet or a style attribute
	// second objective is to visit each TokenNodes before taking pointers to them, so that buffer reallocations won't invalidate those pointers
	i := 0
	hasSemicolon := false
	for p.tb.Peek(i).TokenType != LeftBraceToken {
		if p.tb.Peek(i).TokenType == SemicolonToken || p.tb.Peek(i).TokenType == ErrorToken {
			hasSemicolon = true
			break
		}
		i++
	}

	if grammar, atrule := p.parseAtRule(); atrule != nil {
		return grammar, atrule
	} else if hasSemicolon {
		if decl := p.parseDeclaration(); decl != nil {
			return DeclarationGrammar, decl
		}
	} else if ruleset := p.parseRuleset(); ruleset != nil {
		return StartRulesetGrammar, ruleset
	}
	return TokenGrammar, p.tb.Shift()
}

////////////////////////////////////////////////////////////////

func (p *Parser) parseAtRule() (GrammarType, *AtRuleNode) {
	if !p.at(AtKeywordToken) {
		return ErrorGrammar, nil
	}
	atrule := &AtRuleNode{}
	atrule.Name = p.tb.Shift()
	p.skipWhitespace()
	for !p.at(SemicolonToken) && !p.at(LeftBraceToken) && !p.at(ErrorToken) {
		atrule.Nodes = append(atrule.Nodes, p.shiftComponent())
		p.skipWhitespace()
	}
	if p.at(LeftBraceToken) {
		p.tb.Shift()
		p.state = append(p.state, AtRuleState)
		return StartAtRuleGrammar, atrule
	} else if p.at(SemicolonToken) {
		p.tb.Shift()
	}
	return AtRuleGrammar, atrule
}

func (p *Parser) parseRuleset() *RulesetNode {
	ruleset := &RulesetNode{}
	for !p.at(LeftBraceToken) && !p.at(ErrorToken) {
		if p.at(CommaToken) {
			p.tb.Shift()
			p.skipWhitespace()
			continue
		}
		if sel, ok := p.parseSelector(); ok {
			ruleset.Selectors = append(ruleset.Selectors, sel)
		}
		p.skipWhitespace()
	}
	if p.at(ErrorToken) {
		return nil
	}
	p.tb.Shift()
	p.state = append(p.state, RulesetState)
	return ruleset
}

func (p *Parser) parseSelector() (SelectorNode, bool) {
	sel := SelectorNode{}
	var ws *TokenNode
	for !p.at(CommaToken) && !p.at(LeftBraceToken) && !p.at(ErrorToken) {
		if p.at(DelimToken) && (p.data()[0] == '>' || p.data()[0] == '+' || p.data()[0] == '~') {
			sel.Elems = append(sel.Elems, p.tb.Shift())
			p.skipWhitespace()
		} else {
			if ws != nil {
				sel.Elems = append(sel.Elems, ws)
			}
			if p.at(LeftBracketToken) {
				for !p.at(RightBracketToken) && !p.at(ErrorToken) {
					sel.Elems = append(sel.Elems, p.tb.Shift())
					p.skipWhitespace()
				}
				if p.at(RightBracketToken) {
					sel.Elems = append(sel.Elems, p.tb.Shift())
				}
			} else {
				sel.Elems = append(sel.Elems, p.tb.Shift())
			}
		}
		if p.at(WhitespaceToken) {
			ws = p.tb.Shift()
		} else {
			ws = nil
		}
	}
	if len(sel.Elems) == 0 {
		return sel, false
	}
	return sel, true
}

func (p *Parser) parseDeclaration() *DeclarationNode {
	if !p.at(IdentToken) {
		return nil
	}
	decl := &DeclarationNode{}
	decl.Prop = p.tb.Shift()
	parse.ToLower(decl.Prop.Data)
	p.skipWhitespace()
	if !p.at(ColonToken) {
		return nil
	}
	p.tb.Shift() // colon
	p.skipWhitespace()
	for !p.at(SemicolonToken) && !p.at(RightBraceToken) && !p.at(ErrorToken) {
		if fun := p.parseFunction(); fun != nil {
			decl.Vals = append(decl.Vals, fun)
		} else {
			decl.Vals = append(decl.Vals, p.tb.Shift())
		}
		p.skipWhitespace()
	}
	if p.at(SemicolonToken) {
		p.tb.Shift()
	}
	return decl
}

func (p *Parser) parseFunction() *FunctionNode {
	fun := &FunctionNode{}
	if !p.at(FunctionToken) {
		return nil
	}
	fun.Name = p.tb.Shift()
	fun.Name.Data = fun.Name.Data[:len(fun.Name.Data)-1]
	p.skipWhitespace()
	for !p.at(RightParenthesisToken) && !p.at(ErrorToken) {
		if p.at(CommaToken) {
			p.tb.Shift()
			p.skipWhitespace()
			continue
		}
		fun.Args = append(fun.Args, p.parseArgument())
	}
	if p.at(ErrorToken) {
		return nil
	}
	p.tb.Shift() // right parenthesis
	return fun
}

func (p *Parser) parseArgument() ArgumentNode {
	arg := ArgumentNode{}
	for !p.at(CommaToken) && !p.at(RightParenthesisToken) && !p.at(ErrorToken) {
		arg.Vals = append(arg.Vals, p.shiftComponent())
		p.skipWhitespace()
	}
	return arg
}

func (p *Parser) parseBlock() *BlockNode {
	if !p.at(LeftParenthesisToken) && !p.at(LeftBraceToken) && !p.at(LeftBracketToken) {
		return nil
	}
	block := &BlockNode{}
	block.Open = p.tb.Shift()
	p.skipWhitespace()
	for {
		if p.at(RightBraceToken) || p.at(RightParenthesisToken) || p.at(RightBracketToken) || p.at(ErrorToken) {
			break
		}
		block.Nodes = append(block.Nodes, p.shiftComponent())
		p.skipWhitespace()
	}
	if !p.at(ErrorToken) {
		block.Close = p.tb.Shift()
	}
	return block
}

func (p *Parser) shiftComponent() Node {
	if block := p.parseBlock(); block != nil {
		return block
	} else if fun := p.parseFunction(); fun != nil {
		return fun
	} else {
		return p.tb.Shift()
	}
}

func (p *Parser) skipWhitespace() {
	if p.at(WhitespaceToken) {
		p.tb.Shift()
	}
}

////////////////////////////////////////////////////////////////

func (p *Parser) at(tt TokenType) bool {
	return p.tb.Peek(0).TokenType == tt
}

func (p *Parser) data() []byte {
	return p.tb.Peek(0).Data
}
<commit_msg>Reuse selector elements buffer<commit_after>/*
Package css is a CSS3 tokenizer and parser written in Go. Both are implemented using the specifications at http://www.w3.org/TR/css-syntax-3/
Tokenizer using example:
	package main
	import (
		"fmt"
		"io"
		"os"
		"github.com/tdewolff/css"
	)
	// Tokenize CSS3 from stdin.
	func main() {
		z := css.NewTokenizer(os.Stdin)
		for {
			tt, data := z.Next()
			switch tt {
			case css.ErrorToken:
				if z.Err() != io.EOF {
					fmt.Println("Error on line", z.Line(), ":", z.Err())
				}
				return
			case css.IdentToken:
				fmt.Println("Identifier", data)
			case css.NumberToken:
				fmt.Println("Number", data)
			// ...
			}
		}
	}
Parser using example:
	package main
	import (
		"fmt"
		"os"
		"github.com/tdewolff/css"
	)
	// Parse CSS3 from stdin.
	func main() {
		stylesheet, err := css.Parse(os.Stdin)
		if err != nil {
			fmt.Println("Error:", err)
			return
		}
		for _, node := range stylesheet.Nodes {
			switch m := node.(type) {
			case *css.TokenNode:
				fmt.Println("Token", string(m.Data))
			case *css.DeclarationNode:
				fmt.Println("Declaration for property", string(m.Prop.Data))
			case *css.RulesetNode:
				fmt.Println("Ruleset with", len(m.Decls), "declarations")
			case *css.AtRuleNode:
				fmt.Println("AtRule", string(m.At.Data))
			}
		}
	}
*/
package css // import "github.com/tdewolff/parse/css"

import (
	"io"
	"strconv"

	"github.com/tdewolff/parse"
)

////////////////////////////////////////////////////////////////

// GrammarType determines the type of grammar.
type GrammarType uint32

// GrammarType values.
const (
	ErrorGrammar GrammarType = iota // extra token when errors occur
	AtRuleGrammar
	StartAtRuleGrammar
	EndAtRuleGrammar
	StartRulesetGrammar
	EndRulesetGrammar
	DeclarationGrammar
	TokenGrammar
)

// String returns the string representation of a GrammarType.
func (tt GrammarType) String() string {
	switch tt {
	case ErrorGrammar:
		return "Error"
	case AtRuleGrammar:
		return "AtRule"
	case StartAtRuleGrammar:
		return "StartAtRule"
	case EndAtRuleGrammar:
		return "EndAtRule"
	case StartRulesetGrammar:
		return "StartRuleset"
	case EndRulesetGrammar:
		return "EndRuleset"
	case DeclarationGrammar:
		return "Declaration"
	case TokenGrammar:
		return "Token"
	}
	return "Invalid(" + strconv.Itoa(int(tt)) + ")"
}

// ParserState denotes the state of the parser.
type ParserState uint32

// ParserState values.
const (
	StylesheetState ParserState = iota
	AtRuleState
	RulesetState
)

// String returns the string representation of a ParserState.
func (tt ParserState) String() string {
	switch tt {
	case StylesheetState:
		return "Stylesheet"
	case AtRuleState:
		return "AtRule"
	case RulesetState:
		return "Ruleset"
	}
	return "Invalid(" + strconv.Itoa(int(tt)) + ")"
}

////////////////////////////////////////////////////////////////

// Parser is the state for the parser.
type Parser struct {
	z     *Tokenizer
	state []ParserState

	tb   *tokenBuffer
	copy bool
	tBuf []*TokenNode
}

// NewParser returns a new Parser for a io.Reader.
func NewParser(r io.Reader) *Parser {
	z := NewTokenizer(r)
	return &Parser{
		z:     z,
		state: []ParserState{StylesheetState},
		tb:    newTokenBuffer(z),
	}
}

// Parse parses the entire CSS file and returns the root StylesheetNode.
func Parse(r io.Reader) (*StylesheetNode, error) {
	p := NewParser(r)
	p.copy = true
	p.tb.EnableLookback()

	var err error
	stylesheet := &StylesheetNode{}
	for {
		gt, n := p.Next()
		if gt == ErrorGrammar {
			err = p.z.Err()
			break
		}
		stylesheet.Nodes = append(stylesheet.Nodes, n)
		if err = p.parseRecursively(gt, n); err != nil {
			break
		}
	}
	if err != io.EOF {
		return stylesheet, err
	}
	return stylesheet, nil
}

func (p *Parser) parseRecursively(rootGt GrammarType, n Node) error {
	if rootGt == StartAtRuleGrammar {
		atRule := n.(*AtRuleNode)
		for {
			gt, m := p.Next()
			if gt == ErrorGrammar {
				return p.z.Err()
			} else if gt == EndAtRuleGrammar {
				break
			}
			atRule.Rules = append(atRule.Rules, m)
			if err := p.parseRecursively(gt, m); err != nil {
				return err
			}
		}
	} else if rootGt == StartRulesetGrammar {
		ruleset := n.(*RulesetNode)
		for {
			gt, m := p.Next()
			if gt == ErrorGrammar {
				return p.z.Err()
			} else if gt == EndRulesetGrammar {
				break
			}
			if decl, ok := m.(*DeclarationNode); ok {
				ruleset.Decls = append(ruleset.Decls, decl)
			}
		}
	}
	return nil
}

// Err returns the error encountered during parsing, this is often io.EOF but also other errors can be returned.
func (p *Parser) Err() error {
	return p.z.Err()
}

// Next returns the next grammar unit from the CSS file.
// This is a lower-level function than Parse and is used to stream parse CSS instead of loading it fully into memory.
// Returned nodes of AtRule and Ruleset do not contain entries for Decls and Rules respectively. These are returned consecutively with calls to Next.
func (p *Parser) Next() (GrammarType, Node) {
	if p.at(ErrorToken) {
		return ErrorGrammar, nil
	}
	for p.at(WhitespaceToken) || p.at(SemicolonToken) {
		p.tb.Shift()
	}

	state := p.state[len(p.state)-1]
	if p.at(RightBraceToken) && (state == AtRuleState || state == RulesetState) {
		// return End types
		token := p.tb.Shift()
		p.state = p.state[:len(p.state)-1]
		if state == AtRuleState {
			return EndAtRuleGrammar, token
		} else {
			return EndRulesetGrammar, token
		}
	} else if p.at(CDOToken) || p.at(CDCToken) {
		return TokenGrammar, p.tb.Shift()
	}

	// find out whether this is a declaration or ruleset, because we don't know if we have a stylesheet or a style attribute
	// second objective is to visit each TokenNodes before taking pointers to them, so that buffer reallocations won't invalidate those pointers
	i := 0
	hasSemicolon := false
	for p.tb.Peek(i).TokenType != LeftBraceToken {
		if p.tb.Peek(i).TokenType == SemicolonToken || p.tb.Peek(i).TokenType == ErrorToken {
			hasSemicolon = true
			break
		}
		i++
	}

	if grammar, atrule := p.parseAtRule(); atrule != nil {
		return grammar, atrule
	} else if hasSemicolon {
		if decl := p.parseDeclaration(); decl != nil {
			return DeclarationGrammar, decl
		}
	} else if ruleset := p.parseRuleset(); ruleset != nil {
		return StartRulesetGrammar, ruleset
	}
	return TokenGrammar, p.tb.Shift()
}

////////////////////////////////////////////////////////////////

func (p *Parser) parseAtRule() (GrammarType, *AtRuleNode) {
	if !p.at(AtKeywordToken) {
		return ErrorGrammar, nil
	}
	atrule := &AtRuleNode{}
	atrule.Name = p.tb.Shift()
	p.skipWhitespace()
	for !p.at(SemicolonToken) && !p.at(LeftBraceToken) && !p.at(ErrorToken) {
		atrule.Nodes = append(atrule.Nodes, p.shiftComponent())
		p.skipWhitespace()
	}
	if p.at(LeftBraceToken) {
		p.tb.Shift()
		p.state = append(p.state, AtRuleState)
		return StartAtRuleGrammar, atrule
	} else if p.at(SemicolonToken) {
		p.tb.Shift()
	}
	return AtRuleGrammar, atrule
}

func (p *Parser) parseRuleset() *RulesetNode {
	p.tBuf = p.tBuf[:0]

	ruleset := &RulesetNode{}
	for !p.at(LeftBraceToken) && !p.at(ErrorToken) {
		if p.at(CommaToken) {
			p.tb.Shift()
			p.skipWhitespace()
			continue
		}
		if sel, ok := p.parseSelector(); ok {
			ruleset.Selectors = append(ruleset.Selectors, sel)
		}
		p.skipWhitespace()
	}
	if p.at(ErrorToken) {
		return nil
	}
	p.tb.Shift()
	p.state = append(p.state, RulesetState)
	return ruleset
}

func (p *Parser) parseSelector() (SelectorNode, bool) {
	nElems := len(p.tBuf)
	var ws *TokenNode
	for !p.at(CommaToken) && !p.at(LeftBraceToken) && !p.at(ErrorToken) {
		if p.at(DelimToken) && (p.data()[0] == '>' || p.data()[0] == '+' || p.data()[0] == '~') {
			p.tBuf = append(p.tBuf, p.tb.Shift())
			p.skipWhitespace()
		} else {
			if ws != nil {
				p.tBuf = append(p.tBuf, ws)
			}
			if p.at(LeftBracketToken) {
				for !p.at(RightBracketToken) && !p.at(ErrorToken) {
					p.tBuf = append(p.tBuf, p.tb.Shift())
					p.skipWhitespace()
				}
				if p.at(RightBracketToken) {
					p.tBuf = append(p.tBuf, p.tb.Shift())
				}
			} else {
				p.tBuf = append(p.tBuf, p.tb.Shift())
			}
		}
		if p.at(WhitespaceToken) {
			ws = p.tb.Shift()
		} else {
			ws = nil
		}
	}
	if len(p.tBuf) == nElems {
		return SelectorNode{}, false
	}
	elems := p.tBuf[nElems:]
	if p.copy {
		elems = make([]*TokenNode, len(p.tBuf)-nElems)
		copy(elems, p.tBuf[nElems:])
	}
	return SelectorNode{elems}, true
}

func (p *Parser) parseDeclaration() *DeclarationNode {
	if !p.at(IdentToken) {
		return nil
	}
	decl := &DeclarationNode{}
	decl.Prop = p.tb.Shift()
	parse.ToLower(decl.Prop.Data)
	p.skipWhitespace()
	if !p.at(ColonToken) {
		return nil
	}
	p.tb.Shift() // colon
	p.skipWhitespace()
	for !p.at(SemicolonToken) && !p.at(RightBraceToken) && !p.at(ErrorToken) {
		if fun := p.parseFunction(); fun != nil {
			decl.Vals = append(decl.Vals, fun)
		} else {
			decl.Vals = append(decl.Vals, p.tb.Shift())
		}
		p.skipWhitespace()
	}
	if p.at(SemicolonToken) {
		p.tb.Shift()
	}
	return decl
}

func (p *Parser) parseFunction() *FunctionNode {
	fun := &FunctionNode{}
	if !p.at(FunctionToken) {
		return nil
	}
	fun.Name = p.tb.Shift()
	fun.Name.Data = fun.Name.Data[:len(fun.Name.Data)-1]
	p.skipWhitespace()
	for !p.at(RightParenthesisToken) && !p.at(ErrorToken) {
		if p.at(CommaToken) {
			p.tb.Shift()
			p.skipWhitespace()
			continue
		}
		fun.Args = append(fun.Args, p.parseArgument())
	}
	if p.at(ErrorToken) {
		return nil
	}
	p.tb.Shift() // right parenthesis
	return fun
}

func (p *Parser) parseArgument() ArgumentNode {
	arg := ArgumentNode{}
	for !p.at(CommaToken) && !p.at(RightParenthesisToken) && !p.at(ErrorToken) {
		arg.Vals = append(arg.Vals, p.shiftComponent())
		p.skipWhitespace()
	}
	return arg
}

func (p *Parser) parseBlock() *BlockNode {
	if !p.at(LeftParenthesisToken) && !p.at(LeftBraceToken) && !p.at(LeftBracketToken) {
		return nil
	}
	block := &BlockNode{}
	block.Open = p.tb.Shift()
	p.skipWhitespace()
	for {
		if p.at(RightBraceToken) || p.at(RightParenthesisToken) || p.at(RightBracketToken) || p.at(ErrorToken) {
			break
		}
		block.Nodes = append(block.Nodes, p.shiftComponent())
		p.skipWhitespace()
	}
	if !p.at(ErrorToken) {
		block.Close = p.tb.Shift()
	}
	return block
}

func (p *Parser) shiftComponent() Node {
	if block := p.parseBlock(); block != nil {
		return block
	} else if fun := p.parseFunction(); fun != nil {
		return fun
	} else {
		return p.tb.Shift()
	}
}

func (p *Parser) skipWhitespace() {
	if p.at(WhitespaceToken) {
		p.tb.Shift()
	}
}

////////////////////////////////////////////////////////////////

func (p *Parser) at(tt TokenType) bool {
	return p.tb.Peek(0).TokenType == tt
}

func (p *Parser) data() []byte {
	return p.tb.Peek(0).Data
}
<|endoftext|>


<commit_before>/*  Go program to decode an audio stream of morse code into a stream of ascii.

*/

package main

import (
	"fmt"
	"math"
	"rand"
)

// ------- Stage 1:  Detect tones in the stream. ------------------

// Use Root Mean Square (RMS) method to return 'average' value of an
// array of audio samples.
func rms(audiovals []int) int {
	sum := 0
	squaresum := 0
	for i := 0; i < len(audiovals); i++ {
		v := audiovals[i]
		sum = sum + v
		squaresum = squaresum + (v*v)
	}
	mean := sum / len(audiovals)
	mean_of_squares := squaresum / len(audiovals)
	return int(math.Sqrt(float64(mean_of_squares - (mean * mean))))
}

// Read audiosample chunks from 'chunks' channel, and push simple RMS
// amplitudes into the 'amplitudes' channel.
func amplituder(chunks chan []int, amplitudes chan int) {
	for chunk := range chunks {
		amplitudes <- rms(chunk)
	}
	close(amplitudes)
}

// Read amplitudes from 'amplitudes' channel, and push quantized
// on/off values to 'quants' channel.
func quantizer(amplitudes chan int, quants chan bool) {
	var group [100]int
	seen := 0
	max := 0
	min := 0
	for amp := range amplitudes {
		// Suck 100 amplitudes at a time from input channel,
		// figure out 'middle' amplitude for the group, and
		// use that value to quantize each amplitude.
		group[seen] = amp
		seen += 1
		if amp > max { max = amp }
		if amp < min { min = amp }
		if seen == 100 {
			middle := (max - min) / 2
			for i := 0; i < 100; i++ {
				quants <- (group[i] >= middle)
			}
			max = 0
			min = 0
			seen = 0
		}
	}
	close(quants)
}

// Main stage 1 pipeline: reads audiochunks from input channel;
// returns a boolean channel to which it pushes quantized on/off
// values.
func get_quantize_pipe(audiochunks chan []int) chan bool {
	amplitudes := make(chan int)
	quants := make(chan bool)
	go amplituder(audiochunks, amplitudes)
	go quantizer(amplitudes, quants)
	return quants
}

// ------ Put all the pipes together. --------------

func main () {
	chunks := make(chan []int)
	quants := get_quantize_pipe(chunks)

	// Push a crapload of random data into the pipeline
	go func() {
		for i :=0 ; i < 5000; i++ {
			chunk := make([]int, 10)
			for j := 0; j < 10; j++ { chunk[j] = rand.Int() }
			chunks <- chunk
		}
		close(chunks)
	}()

	// Pull quantized booleans from the pipeline's output
	for quant := range quants {
		if quant { fmt.Printf("X ")
		} else { fmt.Printf("O ") }
	}
}

<commit_msg>Phase 2: run-length encode the on/off stream.<commit_after>/*  Go program to decode an audio stream of morse code into a stream of ascii.

*/

package main

import (
	"fmt"
	"math"
	"rand"
)

// ------- Stage 1:  Detect tones in the stream. ------------------

// Use Root Mean Square (RMS) method to return 'average' value of an
// array of audio samples.
func rms(audiovals []int) int {
	sum := 0
	squaresum := 0
	for i := 0; i < len(audiovals); i++ {
		v := audiovals[i]
		sum = sum + v
		squaresum = squaresum + (v*v)
	}
	mean := sum / len(audiovals)
	meanOfSquares := squaresum / len(audiovals)
	return int(math.Sqrt(float64(meanOfSquares - (mean * mean))))
}

// Read audiosample chunks from 'chunks' channel, and push simple RMS
// amplitudes into the 'amplitudes' channel.
func amplituder(chunks chan []int, amplitudes chan int) {
	for chunk := range chunks {
		amplitudes <- rms(chunk)
	}
	close(amplitudes)
}

// Read amplitudes from 'amplitudes' channel, and push quantized
// on/off values to 'quants' channel.
func quantizer(amplitudes chan int, quants chan bool) {
	var group [100]int
	seen := 0
	max := 0
	min := 0
	for amp := range amplitudes {
		// Suck 100 amplitudes at a time from input channel,
		// figure out 'middle' amplitude for the group, and
		// use that value to quantize each amplitude.
		group[seen] = amp
		seen += 1
		if amp > max { max = amp }
		if amp < min { min = amp }
		if seen == 100 {
			middle := (max - min) / 2
			for i := 0; i < 100; i++ {
				quants <- (group[i] >= middle)
			}
			max = 0
			min = 0
			seen = 0
		}
	}
	close(quants)
}

// Main stage 1 pipeline: reads audiochunks from input channel;
// returns a boolean channel to which it pushes quantized on/off
// values.
func getQuantizePipe(audiochunks chan []int) chan bool {
	amplitudes := make(chan int)
	quants := make(chan bool)
	go amplituder(audiochunks, amplitudes)
	go quantizer(amplitudes, quants)
	return quants
}

// ------- Stage 2:  Run-length encode the on/off states. ----------
//
// That is, if the input stream is 0001100111100, we want to output
// the list [3, 2, 2, 4, 2], which can be seen as the "rhythm" of the
// coded message.

func getRlePipe(quants chan bool) chan int {
	lengths := make(chan int)
	go func() {
		currentState := false
		tally := 0
		for quant := range quants {
			if quant == currentState {
				tally += 1
			} else {
				lengths <- tally
				currentState = quant
				tally = 1
			}
		}
		close(lengths)
	}()
	return lengths
}

// ------ Put all the pipes together. --------------

func main () {
	chunks := make(chan []int)  // main input pipe
	output := getRlePipe(getQuantizePipe(chunks))  // main output pipe

	// Start pushing random data into the pipeline in the background
	go func() {
		for i :=0 ; i < 5000; i++ {
			chunk := make([]int, 10)
			for j := 0; j < 10; j++ { chunk[j] = rand.Int() }
			chunks <- chunk
		}
		close(chunks)
	}()

	// Pull quantized booleans from the pipeline's output
	for val := range output {
		fmt.Printf("%d ", val)
	}
}

<|endoftext|>


<commit_before>package grafana

import (
	"encoding/json"
	"time"
)

type DashboardID uint64

type Dashboard struct {
	ID           DashboardID `json:"id,omitempty"`
	Editable     bool        `json:"editable"`
	GraphTooltip uint8       `json:"graphTooltip"`
	HideControls bool        `json:"hideControls"`
	Rows         []*Row      `json:"rows"`
	Style        string      `json:"style"`
	Timezone     string      `json:"timezone"`
	Title        string      `json:"title"`
	tags         []string
	Meta         *DashboardMeta
}

type DashboardMeta struct {
	Slug    string `json:"slug"`
	Type    string `json:"type"`
	Version int    `json:"version"`

	CanEdit bool `json:"canEdit"`
	CanSave bool `json:"canSave"`
	CanStar bool `json:"canStar"`

	Created   time.Time `json:"created"`
	CreatedBy string    `json:"createdBy"`
	Expires   time.Time `json:"expires"`
	Updated   time.Time `json:"updated"`
	UpdatedBy string    `json:"updatedBy"`
}

// Tags is a getter for Dashboard tags field
func (d *Dashboard) Tags() []string {
	return d.tags
}

// SetTags sets new tags to dashboard
func (d *Dashboard) SetTags(tags ...string) {
	newTags := []string{}
	uniqTags := make(map[string]bool)
	for _, tag := range tags {
		if _, ok := uniqTags[tag]; ok {
			continue
		}

		uniqTags[tag] = true
		newTags = append(newTags, tag)
	}

	d.tags = newTags
}

// AddTags adds given tags to dashboard. This method keeps uniqueness of tags.
func (d *Dashboard) AddTags(tags ...string) {
	tagFound := make(map[string]bool, len(d.tags))
	for _, tag := range d.tags {
		tagFound[tag] = true
	}

	for _, tag := range tags {
		if _, ok := tagFound[tag]; ok {
			continue
		}
		d.tags = append(d.tags, tag)
	}
}

// RemoveTags removes given tags from dashboard. Does nothing if tag is not found.
func (d *Dashboard) RemoveTags(tags ...string) {
	tagIndex := make(map[string]int, len(d.tags))
	for i, tag := range d.tags {
		tagIndex[tag] = i
	}

	for _, tag := range tags {
		if i, ok := tagIndex[tag]; ok {
			d.tags = append(d.tags[:i], d.tags[i+1:]...)
		}
	}
}

func (d *Dashboard) MarshalJSON() ([]byte, error) {
	type JSONDashboard Dashboard
	return json.Marshal(&struct {
		*JSONDashboard
		Tags []string `json:"tags"`
	}{
		JSONDashboard: (*JSONDashboard)(d),
		Tags:          d.Tags(),
	})
}

type Row struct {
	Collapse bool     `json:"collapse"`
	Editable bool     `json:"editable"`
	Height   string   `json:"height"`
	Title    string   `json:"title"`
	Panels   []*Panel `json:"panels"`
}

type Panel struct {
}
<commit_msg>Don't json marshall Meta field of Dashboard<commit_after>package grafana

import (
	"encoding/json"
	"time"
)

type DashboardID uint64

type Dashboard struct {
	ID           DashboardID `json:"id,omitempty"`
	Editable     bool        `json:"editable"`
	GraphTooltip uint8       `json:"graphTooltip"`
	HideControls bool        `json:"hideControls"`
	Rows         []*Row      `json:"rows"`
	Style        string      `json:"style"`
	Timezone     string      `json:"timezone"`
	Title        string      `json:"title"`
	tags         []string
	Meta         *DashboardMeta `json:"meta,omitempty"`
}

type DashboardMeta struct {
	Slug    string `json:"slug"`
	Type    string `json:"type"`
	Version int    `json:"version"`

	CanEdit bool `json:"canEdit"`
	CanSave bool `json:"canSave"`
	CanStar bool `json:"canStar"`

	Created   time.Time `json:"created"`
	CreatedBy string    `json:"createdBy"`
	Expires   time.Time `json:"expires"`
	Updated   time.Time `json:"updated"`
	UpdatedBy string    `json:"updatedBy"`
}

// Tags is a getter for Dashboard tags field
func (d *Dashboard) Tags() []string {
	return d.tags
}

// SetTags sets new tags to dashboard
func (d *Dashboard) SetTags(tags ...string) {
	newTags := []string{}
	uniqTags := make(map[string]bool)
	for _, tag := range tags {
		if _, ok := uniqTags[tag]; ok {
			continue
		}

		uniqTags[tag] = true
		newTags = append(newTags, tag)
	}

	d.tags = newTags
}

// AddTags adds given tags to dashboard. This method keeps uniqueness of tags.
func (d *Dashboard) AddTags(tags ...string) {
	tagFound := make(map[string]bool, len(d.tags))
	for _, tag := range d.tags {
		tagFound[tag] = true
	}

	for _, tag := range tags {
		if _, ok := tagFound[tag]; ok {
			continue
		}
		d.tags = append(d.tags, tag)
	}
}

// RemoveTags removes given tags from dashboard. Does nothing if tag is not found.
func (d *Dashboard) RemoveTags(tags ...string) {
	tagIndex := make(map[string]int, len(d.tags))
	for i, tag := range d.tags {
		tagIndex[tag] = i
	}

	for _, tag := range tags {
		if i, ok := tagIndex[tag]; ok {
			d.tags = append(d.tags[:i], d.tags[i+1:]...)
		}
	}
}

func (d *Dashboard) MarshalJSON() ([]byte, error) {
	type JSONDashboard Dashboard
	dd := (*JSONDashboard)(d)
	dd.Meta = nil

	return json.Marshal(&struct {
		*JSONDashboard
		Tags []string `json:"tags,omitempty"`
	}{
		JSONDashboard: dd,
		Tags:          d.Tags(),
	})
}

type Row struct {
	Collapse bool     `json:"collapse"`
	Editable bool     `json:"editable"`
	Height   string   `json:"height"`
	Title    string   `json:"title"`
	Panels   []*Panel `json:"panels"`
}

type Panel struct {
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2015-present Adam Hanna <ahanna@alumni.mines.edu>
 * Copyright (C) 2015-present Jonathan Barronville <jonathan@belairlabs.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package data

import (
	"encoding/csv"
	"gopkg.in/mgo.v2/bson"
	"io"
	"log"
	"os"
	"strconv"
)

type ConfigJsonDescriptor struct {
	Header      []interface{} `json:"header"`
	IndexField  string        `json:"index_field"`
	StartAtLine uint          `json:"start_at_line"`
}

var DataSet map[string][]byte

func LoadAndTransformCsvData(csvFileHandle *os.File, configJsonDescriptor *ConfigJsonDescriptor) {
	// Get ready to start reading the CSV file.
	csvFileReader := csv.NewReader(csvFileHandle)
	var csvFileLineCount uint = 1
	DataSet = make(map[string][]byte)
	for {
		dataRecord, err := csvFileReader.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Print(err)
		}
		if configJsonDescriptor.StartAtLine > csvFileLineCount {
			csvFileLineCount += 1
			continue
		}
		// NOTE(@jonathanmarvens):
		// The code you're about to see is probably super inefficient.
		// You write bad code every now and then, too, so don't judge me.
		// TODO(@jonathanmarvens): Fucking fix this shit ASAP.
		bsonDataRecordMap := make(bson.M)
		for idx, dataRecordFieldVal := range dataRecord {
			var dataRecordFieldName string
			var dataRecordFieldTypeString string
			// There should always be only one iteration of this loop.
			for key, val := range configJsonDescriptor.Header[idx].(map[string]interface{}) {
				dataRecordFieldName = key
				dataRecordFieldTypeString = val.(string)
			}
			switch dataRecordFieldTypeString {
			case "string":
				bsonDataRecordMap[dataRecordFieldName] = dataRecordFieldVal
			case "bool":
				dataRecordFieldTypeBoolVal, err := strconv.ParseBool(dataRecordFieldVal)
				if err != nil {
					// NOTE(@jonathanmarvens): Should be Fatal?
					log.Print(err)
				}
				bsonDataRecordMap[dataRecordFieldName] = dataRecordFieldTypeBoolVal
			case "double":
				dataRecordFieldTypeDoubleVal, err := strconv.ParseFloat(dataRecordFieldVal, 64)
				if err != nil {
					// NOTE(@jonathanmarvens): Should be Fatal?
					log.Print(err)
				}
				bsonDataRecordMap[dataRecordFieldName] = dataRecordFieldTypeDoubleVal
			case "int32":
				dataRecordFieldTypeInt32ValTmp, err := strconv.ParseInt(dataRecordFieldVal, 10, 32)
				if err != nil {
					// NOTE(@jonathanmarvens): Should be Fatal?
					log.Print(err)
				}
				dataRecordFieldTypeInt32Val := int32(dataRecordFieldTypeInt32ValTmp)
				bsonDataRecordMap[dataRecordFieldName] = dataRecordFieldTypeInt32Val
			case "int64":
				dataRecordFieldTypeInt64Val, err := strconv.ParseInt(dataRecordFieldVal, 10, 64)
				if err != nil {
					// NOTE(@jonathanmarvens): Should be Fatal?
					log.Print(err)
				}
				bsonDataRecordMap[dataRecordFieldName] = dataRecordFieldTypeInt64Val
			}
		}
		// fmt.Println(bsonDataRecordMap)
		// for key, val := range bsonDataRecordMap {
		// 	fmt.Printf("%s: %v\n", key, val)
		// 	fmt.Print("\n\n")
		// }
		bsonDataRecordBytes, err := bson.Marshal(bsonDataRecordMap)
		if err != nil {
			// NOTE(@jonathanmarvens): Should be Fatal?
			log.Print(err)
		}
		// Assumes the data set's key is always a string.
		DataSet[bsonDataRecordMap[configJsonDescriptor.IndexField].(string)] = bsonDataRecordBytes
		// fmt.Printf("%s: %v\n", bsonDataRecordMap[configJsonDescriptor.IndexField].(string), data.DataSet[bsonDataRecordMap[configJsonDescriptor.IndexField].(string)])
		csvFileLineCount += 1
	}
	// fmt.Print(data.DataSet)
}
<commit_msg>Stop hatin', bruh ...<commit_after>/*
 * Copyright (C) 2015-present Adam Hanna <ahanna@alumni.mines.edu>
 * Copyright (C) 2015-present Jonathan Barronville <jonathan@belairlabs.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package data

import (
	"encoding/csv"
	// "encoding/json"
	// "fmt"
	"gopkg.in/mgo.v2/bson"
	"io"
	"log"
	"os"
	"strconv"
)

type ConfigJsonDescriptor struct {
	Header      []interface{} `json:"header"`
	IndexField  string        `json:"index_field"`
	StartAtLine uint          `json:"start_at_line"`
}

var DataSet map[string][]byte

func LoadAndTransformCsvData(csvFileHandle *os.File, configJsonDescriptor *ConfigJsonDescriptor) {
	// Get ready to start reading the CSV file.
	csvFileReader := csv.NewReader(csvFileHandle)
	var csvFileLineCount uint = 1
	DataSet = make(map[string][]byte)
	for {
		dataRecord, err := csvFileReader.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Print(err)
		}
		if configJsonDescriptor.StartAtLine > csvFileLineCount {
			csvFileLineCount += 1
			continue
		}
		// NOTE(@jonathanmarvens):
		// The code you're about to see is probably super inefficient.
		// You write bad code every now and then, too, so don't judge me.
		// TODO(@jonathanmarvens): Fucking fix this shit ASAP.
		bsonDataRecordMap := make(bson.M)
		for idx, dataRecordFieldVal := range dataRecord {
			var dataRecordFieldName string
			var dataRecordFieldTypeString string
			// There should always be only one iteration of this loop.
			for key, val := range configJsonDescriptor.Header[idx].(map[string]interface{}) {
				dataRecordFieldName = key
				dataRecordFieldTypeString = val.(string)
			}
			switch dataRecordFieldTypeString {
			case "string":
				bsonDataRecordMap[dataRecordFieldName] = dataRecordFieldVal
			case "bool":
				dataRecordFieldTypeBoolVal, err := strconv.ParseBool(dataRecordFieldVal)
				if err != nil {
					// NOTE(@jonathanmarvens): Should be Fatal?
					log.Print(err)
				}
				bsonDataRecordMap[dataRecordFieldName] = dataRecordFieldTypeBoolVal
			case "double":
				dataRecordFieldTypeDoubleVal, err := strconv.ParseFloat(dataRecordFieldVal, 64)
				if err != nil {
					// NOTE(@jonathanmarvens): Should be Fatal?
					log.Print(err)
				}
				bsonDataRecordMap[dataRecordFieldName] = dataRecordFieldTypeDoubleVal
			case "int32":
				dataRecordFieldTypeInt32ValTmp, err := strconv.ParseInt(dataRecordFieldVal, 10, 32)
				if err != nil {
					// NOTE(@jonathanmarvens): Should be Fatal?
					log.Print(err)
				}
				dataRecordFieldTypeInt32Val := int32(dataRecordFieldTypeInt32ValTmp)
				bsonDataRecordMap[dataRecordFieldName] = dataRecordFieldTypeInt32Val
			case "int64":
				dataRecordFieldTypeInt64Val, err := strconv.ParseInt(dataRecordFieldVal, 10, 64)
				if err != nil {
					// NOTE(@jonathanmarvens): Should be Fatal?
					log.Print(err)
				}
				bsonDataRecordMap[dataRecordFieldName] = dataRecordFieldTypeInt64Val
			}
		}
		// fmt.Println(bsonDataRecordMap)
		// for key, val := range bsonDataRecordMap {
		// 	fmt.Printf("%s: %v\n", key, val)
		// 	fmt.Print("\n\n")
		// }
		bsonDataRecordBytes, err := bson.Marshal(bsonDataRecordMap)
		if err != nil {
			// NOTE(@jonathanmarvens): Should be Fatal?
			log.Print(err)
		}

		// // NOTE(@jonathanmarvens):
		// // Listen, I know there are better ways to test.
		// // This works for now, so stop judging me!!!
		// jsonEncodedBytesFromBson, err := json.Marshal(&bsonDataRecordMap)
		// if err != nil {
		// 	log.Print(err)
		// }
		// os.Stdout.Write(jsonEncodedBytesFromBson)
		// fmt.Print("\n")

		// Assumes the data set's key is always a string.
		DataSet[bsonDataRecordMap[configJsonDescriptor.IndexField].(string)] = bsonDataRecordBytes
		// fmt.Printf("%s: %v\n", bsonDataRecordMap[configJsonDescriptor.IndexField].(string), data.DataSet[bsonDataRecordMap[configJsonDescriptor.IndexField].(string)])
		csvFileLineCount += 1
	}
	// fmt.Print(data.DataSet)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	Pagelimit = 40
)

/*
 * Helper function to easily read input from console
 */
func readFromConsole() (stringstore string) {
	scanner := bufio.NewScanner(os.Stdin)
	if scanner.Scan() {
		stringstore = scanner.Text()
	}
	return
}

// Given an array of statuses, returns an array of their ids
func getStatusIds(vs []Status) []uint64 {
	vsm := make([]uint64, len(vs))
	for i, v := range vs {
		vsm[i] = v.ID
	}
	return vsm
}

// Returns true if `a` is an element of the array
func idInSlice(a uint64, list []uint64) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

/*
 * Various data structs
 */

type MastoPurgeSettings struct {
	Server       string `json:"server"`
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
	AccessToken  string `json:"access_token"`
}

type AccountInfo struct {
	ID       int    `json:"id,string"`
	Username string `json:"username"`
}

type RespAppRegister struct {
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
}

type RespAccessToken struct {
	AccessToken string `json:"access_token"`
}

type Status struct {
	ID uint64 `json:"id,string"`
	//Content     string  `json:"content"`
	CreatedAt time.Time `json:"created_at"`
}

var (
	noninteractiveMode = flag.Bool("noninteractive", false, "Run in non-interactive mode, suitable for eg. cron jobs. (When run with a missing settings file, the config process will run interactively.)")
	maxAgeArgument     = flag.String("maxage", "", "Max age of posts you want to keep. Required when running in non-interactive mode. Allowed units: hours, days, weeks, months, years. Example: \"6 months\".")
	configFile         = flag.String("config", ".mastopurgesettings", "Path + filename for the settings file.")
	printVersion       = flag.Bool("version", false, "Print version, and exit.")
	quietMode          = flag.Bool("quiet", false, "Reduce output to the most important messages only.")
	dryRun             = flag.Bool("dryrun", false, "Run MastoPurge to preview its results, but without actually deleting any statuses.")
)

var versionString string = "0.0.0"

func main() {
	flag.Parse()

	if *printVersion {
		fmt.Printf("MastoPurge version %s\n", versionString)
		os.Exit(0)
	}

	interactiveMode := !(*noninteractiveMode)

	if interactiveMode && !*quietMode {
		/*
		 * Add fancyness!
		 */
		fmt.Println("\nWelcome to ...")
		fmt.Println("                     _                                   ")
		fmt.Println(" _ __ ___   __ _ ___| |_ ___  _ __  _   _ _ __ __ _  ___ ")
		fmt.Println("| '_ ` _ \\ / _` / __| __/ _ \\| '_ \\| | | | '__/ _` |/ _ \\")
		fmt.Println("| | | | | | (_| \\__ \\ || (_) | |_) | |_| | | | (_| |  __/")
		fmt.Println("|_| |_| |_|\\__,_|___/\\__\\___/| .__/ \\__,_|_|  \\__, |\\___|")
		fmt.Println("                             |_|              |___/      ")
		fmt.Println("    ... add German Datenhygiene to your Mastodon-Account!")
		fmt.Print("\n\n")
		fmt.Printf("Version %s\n\n", versionString)
	}

	/*
	 * Set up settings and Httpclient
	 */
	settings := MastoPurgeSettings{}
	hc := &APIClient{}
	hc.Timeout = time.Second * 5
	hc.Init()

	/*
	 * Check if configuration file (*configFile, default .mastopurgesettings) exists
	 * - Create if it does not exist
	 * - Use if exists
	 */
	configRaw, readerr := ioutil.ReadFile(*configFile)
	if readerr != nil {
		log.Printf("MastoPurge configuration file %s does not exist or is not accessible.\n", *configFile)
		fmt.Println("\nFirst we need to connect MastoPurge to your Mastodon account.")
		fmt.Println("Enter the domain of your Mastodon home instance: (e.g. \"metalhead.club\")")
		fmt.Print("[Mastodon home instance]: ")
		settings.Server = readFromConsole()
		hc.Server = "https://" + settings.Server

		// Register application for user on server
		if !*quietMode {
			log.Println(">>>>>> Registering MastoPurge App on " + settings.Server)
		}
		params := url.Values{}
		params.Add("client_name", "MastoPurge")
		params.Add("redirect_uris", "urn:ietf:wg:oauth:2.0:oob")
		params.Add("scopes", "read write")
		body, registerErr := hc.Request(http.MethodPost, "/api/v1/apps", params)
		if registerErr != nil {
			log.Fatal(registerErr)
		}

		// Parse response JSON
		respAppRegister := RespAppRegister{}
		err := json.Unmarshal(body, &respAppRegister)
		if err != nil {
			log.Fatal(err)
		}
		settings.ClientID = respAppRegister.ClientID
		settings.ClientSecret = respAppRegister.ClientSecret

		// User must manually authenticate app
		authurl := "https://" + settings.Server + "/oauth/authorize?scope=read%20write&response_type=code&redirect_uri=urn:ietf:wg:oauth:2.0:oob&client_id=" + settings.ClientID
		fmt.Println("\nPlease visit this URL in your web browser:")
		fmt.Println(authurl)
		fmt.Println("\n... and enter the code here:")
		fmt.Print("[Auth code]: ")
		code := readFromConsole()

		// Request auth token via auth code ...
		params = url.Values{}
		params.Add("client_id", settings.ClientID)
		params.Add("client_secret", settings.ClientSecret)
		params.Add("grant_type", "authorization_code")
		params.Add("redirect_uri", "urn:ietf:wg:oauth:2.0:oob")
		params.Add("code", code)
		body, err = hc.Request(http.MethodPost, "/oauth/token", params)
		if err != nil {
			log.Fatal(err)
		}

		respAccessToken := RespAccessToken{}
		err = json.Unmarshal(body, &respAccessToken)
		if err != nil {
			log.Fatal(err)
		}
		settings.AccessToken = respAccessToken.AccessToken
		hc.AccessToken = settings.AccessToken

		// Write settings to config file
		configRaw, _ = json.Marshal(settings)
		err = ioutil.WriteFile(*configFile, configRaw, 0600)
		if err != nil {
			log.Fatal(err)
		}

	} else {
		if !*quietMode {
			log.Println("MastoPurge configuration found! Reading config.")
		}

		/*
		 * Load settings
		 */
		err := json.Unmarshal(configRaw, &settings)
		if err != nil {
			log.Fatalf("Config file is malformed :(\nPlease consider deleting '%s' from your file system and starting MastoPurge again.", *configFile)
		}

		hc.Server = "https://" + settings.Server
		hc.AccessToken = settings.AccessToken
	}

	/*
	 * Check if account access is okay
	 */

	if !*quietMode {
		log.Println("Requesting access to Mastodon account")
	}
	body, accessErr := hc.Request(http.MethodGet, "/api/v1/accounts/verify_credentials", nil)
	if accessErr != nil {
		log.Fatal(accessErr)
	} else {
		accountinfo := AccountInfo{}
		err := json.Unmarshal(body, &accountinfo)

		if accountinfo.ID == 0 {
			if !*quietMode {
				log.Println("Access DENIED :-(")
			}
			log.Fatalf("Unfortunately API access was not granted. Consider deleting '%s' and starting MastoPurge again!", *configFile)
		} else {
			if !*quietMode {
				log.Println("Access GRANTED :-)")
				log.Println(">>> Account ID:", accountinfo.ID)
				log.Println(">>> Username:", accountinfo.Username)
			}

			// Do some date calculations ...
			var maxage time.Duration
			var maxtime time.Time

			for {
				maxagestring := *maxAgeArgument
				if maxagestring == "" {
					if interactiveMode {
						fmt.Println("\nEnter the maximum age of the posts you want to KEEP, e.g. \"30 days\". Older posts will be deleted. Allowed units: hours, days, weeks, months, years.")
						fmt.Print("[Maximum post age]: ")
						maxagestring = readFromConsole()
					} else {
						log.Println("missing required argument --maxage")
						flag.PrintDefaults()
						os.Exit(1)
					}
				}

				parts := strings.Split(maxagestring, " ")

				if len(parts) == 2 {
					maxagenum, converr := strconv.Atoi(parts[0])
					if converr == nil {
						var factor time.Duration

						switch parts[1] {
						case "hours", "hour":
							factor = time.Hour
						case "days", "day":
							factor = time.Hour * time.Duration(24)
						case "weeks", "week":
							factor = time.Hour * time.Duration(24*7)
						case "months", "month":
							factor = time.Hour * time.Duration(24*30)
						case "years", "year":
							factor = time.Hour * time.Duration(24*365)
						default:
							factor = 0
						}

						if factor != 0 {
							maxage = factor * time.Duration(maxagenum)
							break
						}
					}
				}

				if interactiveMode {
					fmt.Println("Error: Invalid age format.")
				} else {
					log.Fatalf("Invalid maximum age \"%s\".", maxagestring)
				}
			}

			maxtime = time.Now().Add(-maxage)
			if interactiveMode {
				fmt.Println("Okay, let's do it! ")
				fmt.Println("Posts older than", maxtime, "will be deleted!")
				fmt.Print("Loading gun ")
				for i := 0; i < 40; i++ {
					fmt.Print(".")
					time.Sleep(time.Duration(50) * time.Millisecond)
				}
				time.Sleep(time.Duration(2) * time.Second)
				fmt.Print("\n\n")
			} else {
				log.Println("Posts older than", maxtime.Format("Jan 2, 2006 at 3:04:05 PM MST"), "will be deleted!")
			}

			// Get IDs of pinned posts (these won't be deleted)
			if !*quietMode {
				log.Printf("========== Fetching pinned statuses ==========\n")
			}
			params := url.Values{}
			params.Add("pinned", "true")
			resp, fetchErr := hc.Request(http.MethodGet, "/api/v1/accounts/"+strconv.Itoa(accountinfo.ID)+"/statuses", params)
			if fetchErr != nil {
				log.Fatal(fetchErr)
			}
			var pinnedStatuses []Status
			err = json.Unmarshal(resp, &pinnedStatuses)
			if err != nil {
				// Maybe server response is an error message?
				log.Println(string(resp))
				log.Fatal(err)
			}
			pinnedStatusIds := getStatusIds(pinnedStatuses)
			log.Printf("Found %d pinned statuses, which will not be deleted.", len(pinnedStatusIds))

			var maxid uint64 = 0
			var deletedcount uint16

			// Fetch new pages until there are no more pages
			for {
				if !*quietMode {
					log.Printf("========== Fetching new statuses until status %d ==========\n", maxid)
				}

				nodeletions := true

				// Fetch posts
				params := url.Values{}
				params.Add("limit", strconv.Itoa(Pagelimit))
				if maxid != 0 {
					params.Add("max_id", fmt.Sprint(maxid))
				}
				resp, fetchErr := hc.Request(http.MethodGet, "/api/v1/accounts/"+strconv.Itoa(accountinfo.ID)+"/statuses", params)
				if fetchErr != nil {
					log.Fatal(fetchErr)
				}

				var statuses []Status
				err = json.Unmarshal(resp, &statuses)
				if err != nil {
					// Maybe server response is an error message?
					log.Println(string(resp))
					log.Fatal(err)
				}

				// Exit killer loop if there are no more statuses
				if len(statuses) == 0 {
					break
				}

				for _, status := range statuses {
					// Parse time
					if status.CreatedAt.Before(maxtime) {
						if idInSlice(status.ID, pinnedStatusIds) {
							if !*quietMode {
								log.Println("Status " + fmt.Sprint(status.ID) + " is pinned; not deleting.")
							}
							continue
						}

						// Delete post
						nodeletions = false

						if !*dryRun {
							delResp, delErr := hc.Request(http.MethodDelete, "/api/v1/statuses/"+fmt.Sprint(status.ID), nil)
							if delErr != nil {
								log.Println("!!! Could not delete status " + fmt.Sprint(status.ID) + " !!!")
							}

							var delStatus Status
							err = json.Unmarshal(delResp, &delStatus)
							if err != nil {
								log.Println(string(delResp))
								log.Fatal(err)
							}

							if delStatus.ID == status.ID {
								deletedcount++
							} else {
								log.Println("Status " + fmt.Sprint(status.ID) + " could not be deleted :( \nResponse: " + string(delResp))
							}
						}
					}

					if status.ID < maxid || maxid == 0 {
						maxid = status.ID - 1
					}
				}

				if nodeletions {
					if !*quietMode {
						log.Println("No posts to be deleted on this page. Trying next page ...")
					}
				} else {
					if deletedcount == 0 && *dryRun {
						if !*quietMode {
							log.Println("0 statuses deleted, because -dryRun was passed.")
						}
					} else {
						if !*quietMode {
							log.Println(deletedcount, "statuses deleted.")
						}
					}
					// Wait before fetching a new page. Give server time to re-assemble pages.
					time.Sleep(time.Duration(1) * time.Second)
				}
			}

			if deletedcount == 0 && *dryRun {
				log.Println("[dryRun] 0 statuses deleted in total, because -dryRun was passed.")
			}

			// No more pages, done! :-)
			if interactiveMode {
				fmt.Println(">>>>>> ", deletedcount, "statuses were successfully deleted.")
			} else {
				log.Println(deletedcount, "statuses were successfully deleted.")
			}
		}
	}
}
<commit_msg>Fix error: Infinite page grabbing loop if oldest post is pinned.<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	Pagelimit = 40
)

/*
 * Helper function to easily read input from console
 */
func readFromConsole() (stringstore string) {
	scanner := bufio.NewScanner(os.Stdin)
	if scanner.Scan() {
		stringstore = scanner.Text()
	}
	return
}

// Given an array of statuses, returns an array of their ids
func getStatusIds(vs []Status) []uint64 {
	vsm := make([]uint64, len(vs))
	for i, v := range vs {
		vsm[i] = v.ID
	}
	return vsm
}

// Returns true if `a` is an element of the array
func idInSlice(a uint64, list []uint64) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

/*
 * Various data structs
 */

type MastoPurgeSettings struct {
	Server       string `json:"server"`
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
	AccessToken  string `json:"access_token"`
}

type AccountInfo struct {
	ID       int    `json:"id,string"`
	Username string `json:"username"`
}

type RespAppRegister struct {
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
}

type RespAccessToken struct {
	AccessToken string `json:"access_token"`
}

type Status struct {
	ID uint64 `json:"id,string"`
	//Content     string  `json:"content"`
	CreatedAt time.Time `json:"created_at"`
}

var (
	noninteractiveMode = flag.Bool("noninteractive", false, "Run in non-interactive mode, suitable for eg. cron jobs. (When run with a missing settings file, the config process will run interactively.)")
	maxAgeArgument     = flag.String("maxage", "", "Max age of posts you want to keep. Required when running in non-interactive mode. Allowed units: hours, days, weeks, months, years. Example: \"6 months\".")
	configFile         = flag.String("config", ".mastopurgesettings", "Path + filename for the settings file.")
	printVersion       = flag.Bool("version", false, "Print version, and exit.")
	quietMode          = flag.Bool("quiet", false, "Reduce output to the most important messages only.")
	dryRun             = flag.Bool("dryrun", false, "Run MastoPurge to preview its results, but without actually deleting any statuses.")
)

var versionString string = "0.0.0"

func main() {
	flag.Parse()

	if *printVersion {
		fmt.Printf("MastoPurge version %s\n", versionString)
		os.Exit(0)
	}

	interactiveMode := !(*noninteractiveMode)

	if interactiveMode && !*quietMode {
		/*
		 * Add fancyness!
		 */
		fmt.Println("\nWelcome to ...")
		fmt.Println("                     _                                   ")
		fmt.Println(" _ __ ___   __ _ ___| |_ ___  _ __  _   _ _ __ __ _  ___ ")
		fmt.Println("| '_ ` _ \\ / _` / __| __/ _ \\| '_ \\| | | | '__/ _` |/ _ \\")
		fmt.Println("| | | | | | (_| \\__ \\ || (_) | |_) | |_| | | | (_| |  __/")
		fmt.Println("|_| |_| |_|\\__,_|___/\\__\\___/| .__/ \\__,_|_|  \\__, |\\___|")
		fmt.Println("                             |_|              |___/      ")
		fmt.Println("    ... add German Datenhygiene to your Mastodon-Account!")
		fmt.Print("\n\n")
		fmt.Printf("Version %s\n\n", versionString)
	}

	/*
	 * Set up settings and Httpclient
	 */
	settings := MastoPurgeSettings{}
	hc := &APIClient{}
	hc.Timeout = time.Second * 5
	hc.Init()

	/*
	 * Check if configuration file (*configFile, default .mastopurgesettings) exists
	 * - Create if it does not exist
	 * - Use if exists
	 */
	configRaw, readerr := ioutil.ReadFile(*configFile)
	if readerr != nil {
		log.Printf("MastoPurge configuration file %s does not exist or is not accessible.\n", *configFile)
		fmt.Println("\nFirst we need to connect MastoPurge to your Mastodon account.")
		fmt.Println("Enter the domain of your Mastodon home instance: (e.g. \"metalhead.club\")")
		fmt.Print("[Mastodon home instance]: ")
		settings.Server = readFromConsole()
		hc.Server = "https://" + settings.Server

		// Register application for user on server
		if !*quietMode {
			log.Println(">>>>>> Registering MastoPurge App on " + settings.Server)
		}
		params := url.Values{}
		params.Add("client_name", "MastoPurge")
		params.Add("redirect_uris", "urn:ietf:wg:oauth:2.0:oob")
		params.Add("scopes", "read write")
		body, registerErr := hc.Request(http.MethodPost, "/api/v1/apps", params)
		if registerErr != nil {
			log.Fatal(registerErr)
		}

		// Parse response JSON
		respAppRegister := RespAppRegister{}
		err := json.Unmarshal(body, &respAppRegister)
		if err != nil {
			log.Fatal(err)
		}
		settings.ClientID = respAppRegister.ClientID
		settings.ClientSecret = respAppRegister.ClientSecret

		// User must manually authenticate app
		authurl := "https://" + settings.Server + "/oauth/authorize?scope=read%20write&response_type=code&redirect_uri=urn:ietf:wg:oauth:2.0:oob&client_id=" + settings.ClientID
		fmt.Println("\nPlease visit this URL in your web browser:")
		fmt.Println(authurl)
		fmt.Println("\n... and enter the code here:")
		fmt.Print("[Auth code]: ")
		code := readFromConsole()

		// Request auth token via auth code ...
		params = url.Values{}
		params.Add("client_id", settings.ClientID)
		params.Add("client_secret", settings.ClientSecret)
		params.Add("grant_type", "authorization_code")
		params.Add("redirect_uri", "urn:ietf:wg:oauth:2.0:oob")
		params.Add("code", code)
		body, err = hc.Request(http.MethodPost, "/oauth/token", params)
		if err != nil {
			log.Fatal(err)
		}

		respAccessToken := RespAccessToken{}
		err = json.Unmarshal(body, &respAccessToken)
		if err != nil {
			log.Fatal(err)
		}
		settings.AccessToken = respAccessToken.AccessToken
		hc.AccessToken = settings.AccessToken

		// Write settings to config file
		configRaw, _ = json.Marshal(settings)
		err = ioutil.WriteFile(*configFile, configRaw, 0600)
		if err != nil {
			log.Fatal(err)
		}

	} else {
		if !*quietMode {
			log.Println("MastoPurge configuration found! Reading config.")
		}

		/*
		 * Load settings
		 */
		err := json.Unmarshal(configRaw, &settings)
		if err != nil {
			log.Fatalf("Config file is malformed :(\nPlease consider deleting '%s' from your file system and starting MastoPurge again.", *configFile)
		}

		hc.Server = "https://" + settings.Server
		hc.AccessToken = settings.AccessToken
	}

	/*
	 * Check if account access is okay
	 */

	if !*quietMode {
		log.Println("Requesting access to Mastodon account")
	}
	body, accessErr := hc.Request(http.MethodGet, "/api/v1/accounts/verify_credentials", nil)
	if accessErr != nil {
		log.Fatal(accessErr)
	} else {
		accountinfo := AccountInfo{}
		err := json.Unmarshal(body, &accountinfo)

		if accountinfo.ID == 0 {
			if !*quietMode {
				log.Println("Access DENIED :-(")
			}
			log.Fatalf("Unfortunately API access was not granted. Consider deleting '%s' and starting MastoPurge again!", *configFile)
		} else {
			if !*quietMode {
				log.Println("Access GRANTED :-)")
				log.Println(">>> Account ID:", accountinfo.ID)
				log.Println(">>> Username:", accountinfo.Username)
			}

			// Do some date calculations ...
			var maxage time.Duration
			var maxtime time.Time

			for {
				maxagestring := *maxAgeArgument
				if maxagestring == "" {
					if interactiveMode {
						fmt.Println("\nEnter the maximum age of the posts you want to KEEP, e.g. \"30 days\". Older posts will be deleted. Allowed units: hours, days, weeks, months, years.")
						fmt.Print("[Maximum post age]: ")
						maxagestring = readFromConsole()
					} else {
						log.Println("missing required argument --maxage")
						flag.PrintDefaults()
						os.Exit(1)
					}
				}

				parts := strings.Split(maxagestring, " ")

				if len(parts) == 2 {
					maxagenum, converr := strconv.Atoi(parts[0])
					if converr == nil {
						var factor time.Duration

						switch parts[1] {
						case "hours", "hour":
							factor = time.Hour
						case "days", "day":
							factor = time.Hour * time.Duration(24)
						case "weeks", "week":
							factor = time.Hour * time.Duration(24*7)
						case "months", "month":
							factor = time.Hour * time.Duration(24*30)
						case "years", "year":
							factor = time.Hour * time.Duration(24*365)
						default:
							factor = 0
						}

						if factor != 0 {
							maxage = factor * time.Duration(maxagenum)
							break
						}
					}
				}

				if interactiveMode {
					fmt.Println("Error: Invalid age format.")
				} else {
					log.Fatalf("Invalid maximum age \"%s\".", maxagestring)
				}
			}

			maxtime = time.Now().Add(-maxage)
			if interactiveMode {
				fmt.Println("Okay, let's do it! ")
				fmt.Println("Posts older than", maxtime, "will be deleted!")
				fmt.Print("Loading gun ")
				for i := 0; i < 40; i++ {
					fmt.Print(".")
					time.Sleep(time.Duration(50) * time.Millisecond)
				}
				time.Sleep(time.Duration(2) * time.Second)
				fmt.Print("\n\n")
			} else {
				log.Println("Posts older than", maxtime.Format("Jan 2, 2006 at 3:04:05 PM MST"), "will be deleted!")
			}

			// Get IDs of pinned posts (these won't be deleted)
			if !*quietMode {
				log.Printf("========== Fetching pinned statuses ==========\n")
			}
			params := url.Values{}
			params.Add("pinned", "true")
			resp, fetchErr := hc.Request(http.MethodGet, "/api/v1/accounts/"+strconv.Itoa(accountinfo.ID)+"/statuses", params)
			if fetchErr != nil {
				log.Fatal(fetchErr)
			}
			var pinnedStatuses []Status
			err = json.Unmarshal(resp, &pinnedStatuses)
			if err != nil {
				// Maybe server response is an error message?
				log.Println(string(resp))
				log.Fatal(err)
			}
			pinnedStatusIds := getStatusIds(pinnedStatuses)
			log.Printf("Found %d pinned statuses, which will not be deleted.", len(pinnedStatusIds))

			var maxid uint64 = 0
			var prevmaxid uint64 = 1
			var deletedcount uint16

			// Fetch new pages until there are no more pages
			for {
				if !*quietMode {
					log.Printf("========== Fetching new statuses until status %d ==========\n", maxid)
				}

				nodeletions := true

				// Fetch posts
				params := url.Values{}
				params.Add("limit", strconv.Itoa(Pagelimit))
				if maxid != 0 {
					params.Add("max_id", fmt.Sprint(maxid))
				}
				resp, fetchErr := hc.Request(http.MethodGet, "/api/v1/accounts/"+strconv.Itoa(accountinfo.ID)+"/statuses", params)
				if fetchErr != nil {
					log.Fatal(fetchErr)
				}

				var statuses []Status
				err = json.Unmarshal(resp, &statuses)
				if err != nil {
					// Maybe server response is an error message?
					log.Println(string(resp))
					log.Fatal(err)
				}

				// Exit killer loop if there are no more statuses or if we are in a loop (maxid == prevmaxid)
				if (len(statuses) == 0) || (maxid == prevmaxid) {
					break
				}

				for _, status := range statuses {
					// Parse time
					if status.CreatedAt.Before(maxtime) {
						if idInSlice(status.ID, pinnedStatusIds) {
							if !*quietMode {
								log.Println("Status " + fmt.Sprint(status.ID) + " is pinned; not deleting.")
							}
							continue
						}

						// Delete post
						nodeletions = false

						if !*dryRun {
							delResp, delErr := hc.Request(http.MethodDelete, "/api/v1/statuses/"+fmt.Sprint(status.ID), nil)
							if delErr != nil {
								log.Println("!!! Could not delete status " + fmt.Sprint(status.ID) + " !!!")
							}

							var delStatus Status
							err = json.Unmarshal(delResp, &delStatus)
							if err != nil {
								log.Println(string(delResp))
								log.Fatal(err)
							}

							if delStatus.ID == status.ID {
								deletedcount++
							} else {
								log.Println("Status " + fmt.Sprint(status.ID) + " could not be deleted :( \nResponse: " + string(delResp))
							}
						}
					}

					if status.ID < maxid || maxid == 0 {
						maxid = status.ID - 1
						prevmaxid = maxid
					}
				}

				if nodeletions {
					if !*quietMode {
						log.Println("No posts to be deleted on this page. Trying next page ...")
					}
				} else {
					if deletedcount == 0 && *dryRun {
						if !*quietMode {
							log.Println("0 statuses deleted, because -dryRun was passed.")
						}
					} else {
						if !*quietMode {
							log.Println(deletedcount, "statuses deleted.")
						}
					}
					// Wait before fetching a new page. Give server time to re-assemble pages.
					time.Sleep(time.Duration(1) * time.Second)
				}
			}

			if deletedcount == 0 && *dryRun {
				log.Println("[dryRun] 0 statuses deleted in total, because -dryRun was passed.")
			}

			// No more pages, done! :-)
			if interactiveMode {
				fmt.Println(">>>>>> ", deletedcount, "statuses were successfully deleted.")
			} else {
				log.Println(deletedcount, "statuses were successfully deleted.")
			}
		}
	}
}
<|endoftext|>


<commit_before>package sarama

import (
	"fmt"
	"reflect"
	"strings"
)

// TopicAndBroker contains several topic for possible several brokers
type TopicAndBroker map[string]string

// String returns string representation
func (cb *TopicAndBroker) String() string {
	return fmt.Sprint(*cb)
}

// Type
func (cb *TopicAndBroker) Type() string {
	return reflect.TypeOf(map[string]string{}).String()
}

// Set the value
func (cb *TopicAndBroker) Set(value string) error {
	TopicAndBroker := strings.SplitN(value, ":", 2)
	if len(TopicAndBroker) < 2 {
		return fmt.Errorf("wrong input: %s", value)
	}
	(*cb)[TopicAndBroker[0]] = TopicAndBroker[1]
	return nil
}

// FirstBrokers return first broker or empty string
func (cb *TopicAndBroker) FirstBrokers() string {
	if cb.IsEmpty() {
		return ""
	}
	for _, v := range *cb {
		return v
	}
	return ""
}

// FirstTopic return first topic or empty string
func (cb *TopicAndBroker) FirstTopic() string {
	if cb.IsEmpty() {
		return ""
	}
	for k := range *cb {
		return k
	}
	return ""
}

// Available Check if there are any topic avaliable
func (cb *TopicAndBroker) Available() bool {
	return cb != nil && len(*cb) > 0
}

// IsEmpty return true if empty
func (cb *TopicAndBroker) IsEmpty() bool {
	return cb == nil || len(*cb) == 0
}
<commit_msg>add missing funcitons to topic_and_broker<commit_after>package sarama

import (
	"fmt"
	"reflect"
	"strings"
)

// TopicAndBroker contains several topic for possible several brokers
type TopicAndBroker map[string]string

// String returns string representation
func (cb *TopicAndBroker) String() string {
	return fmt.Sprint(*cb)
}

// Type
func (cb *TopicAndBroker) Type() string {
	return reflect.TypeOf(map[string]string{}).String()
}

// Set the value
func (cb *TopicAndBroker) Set(value string) error {
	TopicAndBroker := strings.SplitN(value, ":", 2)
	if len(TopicAndBroker) < 2 {
		return fmt.Errorf("wrong input: %s", value)
	}
	(*cb)[TopicAndBroker[0]] = TopicAndBroker[1]
	return nil
}

// First return the first topic and brokers pair
func (cb *TopicAndBroker) First() (string, string) {
	if cb.IsEmpty() {
		return "", ""
	}
	for k, v := range *cb {
		return k, v
	}
	return "", ""
}

// FirstBrokers return first broker(s) as one string or an empty string
func (cb *TopicAndBroker) FirstBrokers() string {
	if cb.IsEmpty() {
		return ""
	}
	for _, v := range *cb {
		return v
	}
	return ""
}

// FirstBrokers return first broker(s) split into a slice of strings
func (cb *TopicAndBroker) FirstBrokersSplit() []string {
	return strings.Split(cb.FirstBrokers(), ",")
}

// FirstTopic return first topic or empty string
func (cb *TopicAndBroker) FirstTopic() string {
	if cb.IsEmpty() {
		return ""
	}
	for k := range *cb {
		return k
	}
	return ""
}

// Available Check if there are any topic avaliable
func (cb *TopicAndBroker) Available() bool {
	return cb != nil && len(*cb) > 0
}

// IsEmpty return true if empty
func (cb *TopicAndBroker) IsEmpty() bool {
	return cb == nil || len(*cb) == 0
}
<|endoftext|>


<commit_before>package trace

import (
	"net/http"
	"testing"

	opentracing "github.com/opentracing/opentracing-go"
	mocktracer "github.com/opentracing/opentracing-go/mocktracer"
	zipkin "github.com/openzipkin/zipkin-go-opentracing"
	zipkintypes "github.com/openzipkin/zipkin-go-opentracing/types"
)

const testServiceName = "TEST-SERVICE"

func TestCreateSpanNoGlobalTracer(t *testing.T) {
	opentracing.SetGlobalTracer(nil)
	req, _ := http.NewRequest("GET", "http://example.com", nil)

	if CreateSpan(req, testServiceName) != nil {
		t.Error("CreateSpan returned a non-nil result using a nil global tracer.")
		t.Fail()
	}
}

func TestCreateSpanWithNoParent(t *testing.T) {
	tracer, _ := zipkin.NewTracer(nil)
	opentracing.SetGlobalTracer(tracer)

	req, _ := http.NewRequest("GET", "http://example.com", nil)
	if CreateSpan(req, testServiceName) == nil {
		t.Error("Received nil span while a global tracer was set.")
		t.FailNow()
	}
}

func TestCreateSpanWithParent(t *testing.T) {
	mt := mocktracer.New()
	opentracing.SetGlobalTracer(mt)
	globalTracer := opentracing.GlobalTracer()
	parentSpan := globalTracer.StartSpan(testServiceName)

	requestIn, _ := http.NewRequest("GET", "http://example.com", nil)
	mt.Inject(parentSpan.Context(),
		opentracing.HTTPHeaders,
		opentracing.HTTPHeadersCarrier(requestIn.Header),
	)

	if CreateSpan(requestIn, testServiceName+"-child") == nil {
		t.Error("Received a nil span while a global tracer was set.")
		t.FailNow()
	}
}

func TestInjectHeaders(t *testing.T) {
	mt := mocktracer.New()
	opentracing.SetGlobalTracer(mt)
	globalTracer := opentracing.GlobalTracer()
	span := globalTracer.StartSpan(testServiceName)

	req, _ := http.NewRequest("GET", "http://example.com", nil)
	InjectHeaders(span, req)

	if req.Header.Get("Mockpfx-Ids-Traceid") == "" {
		t.Error("Inject did not set the Traceid in the request.")
		t.Fail()
	}
	if req.Header.Get("Mockpfx-Ids-Spanid") == "" {
		t.Error("Inject did not set the Spanid in the request.")
		t.Fail()
	}
}

func TestInjectHeadersWithParentSpan(t *testing.T) {
	parentSpanId := uint64(12345)
	parentSpanContext := zipkin.SpanContext{
		SpanID:  parentSpanId,
		TraceID: zipkintypes.TraceID{High: uint64(1234), Low: uint64(4321)},
	}

	tracer, _ := zipkin.NewTracer(nil)
	opentracing.SetGlobalTracer(tracer)
	globalTracer := opentracing.GlobalTracer()
	childSpan := globalTracer.StartSpan(testServiceName+"-CHILD", opentracing.ChildOf(parentSpanContext))

	req, _ := http.NewRequest("GET", "http://example.com", nil)
	InjectHeaders(childSpan, req)

	if req.Header.Get("X-B3-Traceid") == "" {
		t.Error("Inject did not set the Traceid in the request.")
		t.Fail()
	}
	if req.Header.Get("X-B3-Spanid") == "" {
		t.Error("Inject did not set the Spanid in the request.")
		t.Fail()
	}
	if req.Header.Get("X-B3-Parentspanid") != "0000000000003039" {
		t.Error("Inject did not set the correct Parentspanid in the request.")
		t.Fail()
	}
	if req.Header.Get("x-B3-Traceid") != "00000000000004d200000000000010e1" {
		t.Error("Inject did not reuse the Traceid from the parent span")
		t.Fail()
	}
}
<commit_msg>Added tests for trace.InitializeTracer.<commit_after>package trace

import (
	"net/http"
	"testing"

	opentracing "github.com/opentracing/opentracing-go"
	mocktracer "github.com/opentracing/opentracing-go/mocktracer"
	zipkin "github.com/openzipkin/zipkin-go-opentracing"
	zipkintypes "github.com/openzipkin/zipkin-go-opentracing/types"
)

const testServiceName = "TEST-SERVICE"

func TestCreateSpanNoGlobalTracer(t *testing.T) {
	opentracing.SetGlobalTracer(nil)
	req, _ := http.NewRequest("GET", "http://example.com", nil)

	if CreateSpan(req, testServiceName) != nil {
		t.Error("CreateSpan returned a non-nil result using a nil global tracer.")
		t.Fail()
	}
}

func TestCreateSpanWithNoParent(t *testing.T) {
	tracer, _ := zipkin.NewTracer(nil)
	opentracing.SetGlobalTracer(tracer)

	req, _ := http.NewRequest("GET", "http://example.com", nil)
	if CreateSpan(req, testServiceName) == nil {
		t.Error("Received nil span while a global tracer was set.")
		t.FailNow()
	}
}

func TestCreateSpanWithParent(t *testing.T) {
	mt := mocktracer.New()
	opentracing.SetGlobalTracer(mt)
	globalTracer := opentracing.GlobalTracer()
	parentSpan := globalTracer.StartSpan(testServiceName)

	requestIn, _ := http.NewRequest("GET", "http://example.com", nil)
	mt.Inject(parentSpan.Context(),
		opentracing.HTTPHeaders,
		opentracing.HTTPHeadersCarrier(requestIn.Header),
	)

	if CreateSpan(requestIn, testServiceName+"-child") == nil {
		t.Error("Received a nil span while a global tracer was set.")
		t.FailNow()
	}
}

func TestInitializeTracer(t *testing.T) {
	opentracing.SetGlobalTracer(nil)
	InitializeTracer("", "", "", "", 0.0, "")
	if opentracing.GlobalTracer() == nil {
		t.Error("InitializeTracer set a nil tracer.")
		t.FailNow()
	}
}

func TestInjectHeaders(t *testing.T) {
	mt := mocktracer.New()
	opentracing.SetGlobalTracer(mt)
	globalTracer := opentracing.GlobalTracer()
	span := globalTracer.StartSpan(testServiceName)

	req, _ := http.NewRequest("GET", "http://example.com", nil)
	InjectHeaders(span, req)

	if req.Header.Get("Mockpfx-Ids-Traceid") == "" {
		t.Error("Inject did not set the Traceid in the request.")
		t.Fail()
	}
	if req.Header.Get("Mockpfx-Ids-Spanid") == "" {
		t.Error("Inject did not set the Spanid in the request.")
		t.Fail()
	}
}

func TestInjectHeadersWithParentSpan(t *testing.T) {
	parentSpanId := uint64(12345)
	parentSpanContext := zipkin.SpanContext{
		SpanID:  parentSpanId,
		TraceID: zipkintypes.TraceID{High: uint64(1234), Low: uint64(4321)},
	}

	tracer, _ := zipkin.NewTracer(nil)
	opentracing.SetGlobalTracer(tracer)
	globalTracer := opentracing.GlobalTracer()
	childSpan := globalTracer.StartSpan(testServiceName+"-CHILD", opentracing.ChildOf(parentSpanContext))

	req, _ := http.NewRequest("GET", "http://example.com", nil)
	InjectHeaders(childSpan, req)

	if req.Header.Get("X-B3-Traceid") == "" {
		t.Error("Inject did not set the Traceid in the request.")
		t.Fail()
	}
	if req.Header.Get("X-B3-Spanid") == "" {
		t.Error("Inject did not set the Spanid in the request.")
		t.Fail()
	}
	if req.Header.Get("X-B3-Parentspanid") != "0000000000003039" {
		t.Error("Inject did not set the correct Parentspanid in the request.")
		t.Fail()
	}
	if req.Header.Get("x-B3-Traceid") != "00000000000004d200000000000010e1" {
		t.Error("Inject did not reuse the Traceid from the parent span")
		t.Fail()
	}
}
<|endoftext|>


<commit_before>/* gorcon/track version 14.1.12 (lee8oi)

This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.

gorcon/track package contains the PlayerList types and the Tracker methods
needed to track player connections & stats.

*/
package track

import (
	"fmt"
	"github.com/lee8oi/gorcon"
	"strconv"
	"strings"
	"time"
)

type Player struct {
	Pid, Name, Profileid, Team, Level, Kit, Score,
	Kills, Deaths, Alive, Connected, Vip, Nucleus,
	Ping, Suicides string
	Joined time.Time
}

//PlayerList contains a maximum of 16 player 'slots' as per game server limits.
type PlayerList [16]Player

//Tracker uses an Rcon connection to monitor player connection changes and keeps
//current player list updated. Uses 'bf2cc pl' rcon command to request player data.
func (pl *PlayerList) Tracker(r *gorcon.Rcon) {
	for {
		str, err := r.Send("bf2cc pl")
		if err != nil {
			fmt.Println("main 36 error: ", err)
			break
		}
		list := pl.new(str)
		pl.track(&list)
		pl.updateall(list)
		time.Sleep(1 * time.Second)
	}
}

//track compares current PlayerList to new list, slot by slot, to track player connection changes.
func (pl *PlayerList) track(list *PlayerList) {
	var base time.Time
	for i := 0; i < 16; i++ {
		switch {
		case pl[i].Name == list[i].Name: //connecting existing
			if pl[i].Connected == "0" && list[i].Connected == "1" {
				if pl[i].Joined == base {
					fmt.Printf("%s: connected\n", list[i].Name)
					t := time.Now()
					pl[i].Joined = t
				}
			}
		case len(pl[i].Name) == 0 && len(list[i].Name) > 0: //connecting new
			pl.update(i, list[i])
			if pl[i].Connected == "1" && pl[i].Joined == base {
				pl[i].Joined = time.Now()
				fmt.Printf("%s - connection exists\n", list[i].Name)
			}
			if list[i].Connected == "0" {
				fmt.Printf("%s - connecting\n", list[i].Name)
			}
		case len(pl[i].Name) > 0 && len(list[i].Name) == 0: //disconnecting
			if pl[i].Joined != base {
				dur := time.Since(pl[i].Joined)
				fmt.Printf("%s - disconnected (playtime: %s)\n", pl[i].Name, dur.String())
			} else {
				fmt.Printf("%s - disconnected (interrupted)\n", pl[i].Name)
			}

		}
	}
}

//updateall parses new list and updates all player slots.
func (pl *PlayerList) updateall(l PlayerList) {
	var base Player
	for i := 0; i < 16; i++ {
		if pl[i] == base && l[i] == base { //skip if current & new are empty
			continue
		}
		pl.update(i, l[i])
	}
}

//update uses the p.Pid as a key to update the data in the corresponding player slot.
//Item by item assignment allows tracking elements like '.Joined' to retain existing values.
func (pl *PlayerList) update(key int, p Player) {
	if len(p.Pid) > 0 && pl[key].Pid == p.Pid {
		pl[key].Alive = p.Alive
		pl[key].Connected = p.Connected
		pl[key].Deaths = p.Deaths
		pl[key].Kills = p.Kills
		pl[key].Kit = p.Kit
		pl[key].Level = p.Level
		pl[key].Name = p.Name
		pl[key].Nucleus = p.Nucleus
		pl[key].Pid = p.Pid
		pl[key].Ping = p.Ping
		pl[key].Profileid = p.Profileid
		pl[key].Score = p.Score
		pl[key].Suicides = p.Suicides
		pl[key].Team = p.Team
		pl[key].Vip = p.Vip
		return
	}
	pl[key] = p
}

//new returns a new PlayerList generated from given 'bf2cc pl' data string
func (pl *PlayerList) new(data string) (plist PlayerList) {
	if len(data) > 1 {
		split := strings.Split(data, "\r")
		for _, value := range split {
			var p Player
			splitLine := strings.Split(strings.TrimSpace(value), "\t")
			if len(splitLine) < 48 {
				continue
			}
			kit := "none"
			if splitLine[34] != "none" {
				kit = strings.Split(splitLine[34], "_")[1]
			}
			p = Player{
				Pid:       splitLine[0],
				Name:      splitLine[1],
				Profileid: splitLine[10],
				Team:      splitLine[2],
				Level:     splitLine[39],
				Kit:       kit,
				Score:     splitLine[37],
				Kills:     splitLine[31],
				Deaths:    splitLine[36],
				Alive:     splitLine[8],
				Connected: splitLine[4],
				Vip:       splitLine[46],
				Nucleus:   splitLine[47],
				Ping:      splitLine[3],
				Suicides:  strings.TrimSpace(splitLine[30]),
			}
			key, _ := strconv.Atoi(p.Pid)
			plist[key] = p
		}
		return
	}
	return
}
<commit_msg>Improved player data handling efficiency with more pointers. PlayerList.update() has been tweaked.<commit_after>/* gorcon/track version 14.1.12 (lee8oi)

This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.

gorcon/track package contains the PlayerList types and the Tracker methods
needed to track player connections & stats.

*/
package track

import (
	"fmt"
	"github.com/lee8oi/gorcon"
	"strconv"
	"strings"
	"time"
)

type Player struct {
	Pid, Name, Profileid, Team, Level, Kit, Score,
	Kills, Deaths, Alive, Connected, Vip, Nucleus,
	Ping, Suicides string
	Joined time.Time
}

//PlayerList contains a maximum of 16 player 'slots' as per game server limits.
type PlayerList [16]Player

//Tracker uses an Rcon connection to monitor player connection changes and keeps
//current player list updated. Uses 'bf2cc pl' rcon command to request player data.
func (pl *PlayerList) Tracker(r *gorcon.Rcon) {
	for {
		str, err := r.Send("bf2cc pl")
		if err != nil {
			fmt.Println("main 36 error: ", err)
			break
		}
		list := pl.new(str)
		pl.track(&list)
		pl.updateall(&list)
		time.Sleep(1 * time.Second)
	}
}

//track compares current PlayerList to new list, slot by slot, to track player connection changes.
func (pl *PlayerList) track(list *PlayerList) {
	var base time.Time
	for i := 0; i < 16; i++ {
		switch {
		case pl[i].Name == list[i].Name: //connecting existing
			if pl[i].Connected == "0" && list[i].Connected == "1" {
				if pl[i].Joined == base {
					fmt.Printf("%s: connected\n", list[i].Name)
					t := time.Now()
					pl[i].Joined = t
				}
			}
		case len(pl[i].Name) == 0 && len(list[i].Name) > 0: //connecting
			pl.update(i, &list[i])
			if pl[i].Connected == "1" && pl[i].Joined == base {
				pl[i].Joined = time.Now()
				fmt.Printf("%s - connection exists\n", list[i].Name)
			}
			if list[i].Connected == "0" {
				fmt.Printf("%s - connecting\n", list[i].Name)
			}
		case len(pl[i].Name) > 0 && len(list[i].Name) == 0: //disconnecting
			if pl[i].Joined != base {
				dur := time.Since(pl[i].Joined)
				fmt.Printf("%s - disconnected (playtime: %s)\n", pl[i].Name, dur.String())
			} else {
				fmt.Printf("%s - disconnected (interrupted)\n", pl[i].Name)
			}

		}
	}
}

//updateall parses a given PlayerList and updates all existing player slots.
func (pl *PlayerList) updateall(l *PlayerList) {
	var base Player
	for i := 0; i < 16; i++ {
		if pl[i] == base && l[i] == base { //skip if current & new are empty
			continue
		}
		pl.update(i, &l[i])
	}
}

//update the player slot at the index specifed by key.
//After initial update: only elements that can potentially change during playtime
//are updated.
func (pl *PlayerList) update(key int, p *Player) {
	if len(p.Name) > 0 && pl[key].Name == p.Name {
		pl[key].Alive = p.Alive
		pl[key].Connected = p.Connected
		pl[key].Deaths = p.Deaths
		pl[key].Kills = p.Kills
		pl[key].Level = p.Level
		pl[key].Ping = p.Ping
		pl[key].Score = p.Score
		pl[key].Suicides = p.Suicides
		pl[key].Vip = p.Vip
		return
	}
	pl[key] = *p
}

//new takes a 'bf2cc pl' result string and returns a new PlayerList.
func (pl *PlayerList) new(data string) (plist PlayerList) {
	if len(data) > 1 {
		split := strings.Split(data, "\r")
		for _, value := range split {
			var p Player
			splitLine := strings.Split(strings.TrimSpace(value), "\t")
			if len(splitLine) < 48 {
				continue
			}
			kit := "none"
			if splitLine[34] != "none" {
				kit = strings.Split(splitLine[34], "_")[1]
			}
			p = Player{
				Pid:       splitLine[0],
				Name:      splitLine[1],
				Profileid: splitLine[10],
				Team:      splitLine[2],
				Level:     splitLine[39],
				Kit:       kit,
				Score:     splitLine[37],
				Kills:     splitLine[31],
				Deaths:    splitLine[36],
				Alive:     splitLine[8],
				Connected: splitLine[4],
				Vip:       splitLine[46],
				Nucleus:   splitLine[47],
				Ping:      splitLine[3],
				Suicides:  strings.TrimSpace(splitLine[30]),
			}
			key, _ := strconv.Atoi(p.Pid)
			plist[key] = p
		}
		return
	}
	return
}
<|endoftext|>


<commit_before>package tsparser

import (
	"io"
)

const (
	PacketSize          = 188
	BufferedPacketCount = 5
	BufferSize          = PacketSize * BufferedPacketCount

	SyncByte byte = 0x47
)

type Scanner struct {
	r      io.Reader
	buffer [PacketSize * BufferedPacketCount]byte
	seek   int
	eof    bool
	err    error
}

func NewScanner(r io.Reader) *Scanner {
	return &Scanner{
		r:    r,
		seek: 0,
		eof:  false,
		err:  nil,
	}
}

func (s *Scanner) Err() error {
	return s.err
}

func (s *Scanner) leftShift(n int) {
	copy(s.buffer[:BufferSize-n], s.buffer[n:])
	s.seek -= n
}

func (s *Scanner) rightShift(n int) {
	copy(s.buffer[n:], s.buffer[:BufferSize-n])
	s.seek += n
}

func (s *Scanner) fillBuffer(n int) bool {
	switch m, err := io.ReadFull(s.r, s.buffer[BufferSize-n:]); err {
	case io.ErrUnexpectedEOF, io.EOF:
		s.eof = true

		restBytes := (BufferSize - n + m) / PacketSize * PacketSize
		shiftBytes := BufferSize - restBytes
		s.rightShift(shiftBytes)
		return s.seek < BufferSize
	case nil:
		return true
	default:
		panic(err)
	}
}

func (s *Scanner) isSynced() bool {
	synced := true
	for i := s.seek; i < BufferSize; i += PacketSize {
		if s.buffer[i] != SyncByte {
			synced = false
			break
		}
	}

	return synced
}

func (s *Scanner) sync() bool {
	for i := 0; i < PacketSize; i++ {
		if !s.isSynced() {
			s.seek++
			continue
		}

		if i == 0 {
			return true
		}

		s.leftShift(i)
		return s.fillBuffer(i)
	}

	return false
}

func (s *Scanner) Scan() bool {
	if s.seek >= BufferSize {
		return false
	}

	s.seek += PacketSize
	if s.seek < BufferSize && (s.eof || s.isSynced()) {
		return true
	} else if s.eof {
		return false
	}

	s.seek = 0
	if !s.fillBuffer(BufferSize) {
		return false
	}

	for !s.sync() {
		s.leftShift(PacketSize)
		if !s.fillBuffer(PacketSize) {
			return false
		}
	}

	return true
}

func (s *Scanner) Bytes() []byte {
	if s.seek < BufferSize {
		return s.buffer[s.seek : s.seek+PacketSize]
	}

	return nil
}

func (s *Scanner) Packet() Packet {
	return Packet(s.Bytes())
}
<commit_msg>s/Scanner/PacketScanner/g, impl TableScanner<commit_after>package tsparser

import (
	"io"
)

const (
	PacketSize          = 188
	BufferedPacketCount = 5
	BufferSize          = PacketSize * BufferedPacketCount

	SyncByte byte = 0x47
)

type PacketScanner struct {
	r      io.Reader
	buffer [PacketSize * BufferedPacketCount]byte
	seek   int
	eof    bool
	err    error
}

func NewPacketScanner(r io.Reader) *PacketScanner {
	return &PacketScanner{
		r:    r,
		seek: 0,
		eof:  false,
		err:  nil,
	}
}

func (s *PacketScanner) Err() error {
	return s.err
}

func (s *PacketScanner) leftShift(n int) {
	copy(s.buffer[:BufferSize-n], s.buffer[n:])
	s.seek -= n
}

func (s *PacketScanner) rightShift(n int) {
	copy(s.buffer[n:], s.buffer[:BufferSize-n])
	s.seek += n
}

func (s *PacketScanner) fillBuffer(n int) bool {
	switch m, err := io.ReadFull(s.r, s.buffer[BufferSize-n:]); err {
	case io.ErrUnexpectedEOF, io.EOF:
		s.eof = true

		restBytes := (BufferSize - n + m) / PacketSize * PacketSize
		shiftBytes := BufferSize - restBytes
		s.rightShift(shiftBytes)
		return s.seek < BufferSize
	case nil:
		return true
	default:
		panic(err)
	}
}

func (s *PacketScanner) isSynced() bool {
	synced := true
	for i := s.seek; i < BufferSize; i += PacketSize {
		if s.buffer[i] != SyncByte {
			synced = false
			break
		}
	}

	return synced
}

func (s *PacketScanner) sync() bool {
	for i := 0; i < PacketSize; i++ {
		if !s.isSynced() {
			s.seek++
			continue
		}

		if i == 0 {
			return true
		}

		s.leftShift(i)
		return s.fillBuffer(i)
	}

	return false
}

func (s *PacketScanner) Scan() bool {
	if s.seek >= BufferSize {
		return false
	}

	s.seek += PacketSize
	if s.seek < BufferSize && (s.eof || s.isSynced()) {
		return true
	} else if s.eof {
		return false
	}

	s.seek = 0
	if !s.fillBuffer(BufferSize) {
		return false
	}

	for !s.sync() {
		s.leftShift(PacketSize)
		if !s.fillBuffer(PacketSize) {
			return false
		}
	}

	return true
}

func (s *PacketScanner) Bytes() []byte {
	if s.seek < BufferSize {
		return s.buffer[s.seek : s.seek+PacketSize]
	}

	return nil
}

func (s *PacketScanner) Packet() Packet {
	return Packet(s.Bytes())
}

type tableScannerBuffer struct {
	ok     bool
	data   []byte
	length int
}

func (b *tableScannerBuffer) extendData(right []byte) {
	left := b.data
	b.data = make([]byte, len(left)+len(right))
	copy(b.data, left)
	copy(b.data[len(left):], right)
}

func (b *tableScannerBuffer) Extend(packet Packet) {
	isFirst := len(b.data) == 0

	payload := packet.Payload()
	if !packet.payloadUnitStartIndicator() {
		if !isFirst {
			b.extendData(payload)
		}
		b.ok = false
		return
	}

	if !isFirst {
		b.data = b.data[b.length:]
		b.length = len(b.data)
		b.ok = true
	}
	packetStartCodePrefix := payload[0]<<16 | payload[1]<<8 | payload[2]
	if packetStartCodePrefix == 0x000001 {
		// Packetized Elementary Stream
		b.extendData(payload)
	} else {
		// Program Specific Information
		pointerField := int(payload[0])
		if isFirst {
			b.extendData(payload[1+pointerField:])
		} else {
			b.length += pointerField
			b.extendData(payload[1:])
		}
	}

	return
}

func (b *tableScannerBuffer) OK() bool {
	return b.ok
}

func (b *tableScannerBuffer) Bytes() []byte {
	if b.OK() {
		return b.data[b.length:]
	}

	return nil
}

type TableScanner struct {
	s       PacketStream
	pid     PID
	buffers map[PID]*tableScannerBuffer
}

func NewTableScanner(s PacketStream) *TableScanner {
	buffers := make(map[PID]*tableScannerBuffer)
	return &TableScanner{
		s:       s,
		buffers: buffers,
	}
}

func (f *TableScanner) Scan() bool {
	for f.s.Scan() {
		packet := f.s.Packet()

		buffer, ok := f.buffers[packet.PID()]
		if !ok {
			f.buffers[packet.PID()] = new(tableScannerBuffer)
			buffer = f.buffers[packet.PID()]
		}

		buffer.Extend(packet)
		if buffer.OK() {
			f.pid = packet.PID()
			return true
		}
	}

	return false
}

func (f *TableScanner) Bytes() []byte {
	buffer := f.buffers[f.pid]
	if buffer.OK() {
		return buffer.Bytes()
	}

	return nil
}

func (f *TableScanner) Table() Table {
	return Table(f.Bytes())
}
<|endoftext|>


<commit_before>package goat

import (
	"fmt"
	"io"
)

// ASCIItoSVG reads in a newline-delimited ASCII diagram and writes a
// corresponding SVG diagram.
func ASCIItoSVG(in io.Reader, out io.Writer) {
	canvas := NewCanvas(in)

	// Preamble
	writeBytes(&out,
		"<svg class='%s' xmlns='%s' version='%s' height='%d' width='%d'>\n",
		"diagram",
		"http://www.w3.org/2000/svg",
		"1.1",
		canvas.Height*16+8, (canvas.Width+1)*8,
	)

	writeBytes(&out, "<g transform='translate(8,16)'>\n")

	for _, l := range canvas.Lines() {
		l.Draw(out)
	}

	for _, t := range canvas.Triangles() {
		t.Draw(out)
	}

	for _, c := range canvas.Circles() {
		c.Draw(out)
	}

	for _, c := range canvas.RoundedCorners() {
		c.Draw(out)
	}

	for _, b := range canvas.Bridges() {
		b.Draw(out)
	}

	for _, t := range canvas.Text() {
		t.Draw(out)
	}

	writeBytes(&out, "</g>\n")
	writeBytes(&out, "</svg>\n")
}

func writeBytes(out *io.Writer, format string, args ...interface{}) {
	bytesOut := fmt.Sprintf(format, args...)

	_, err := (*out).Write([]byte(bytesOut))

	if err != nil {
		panic(nil)
	}
}

// Draw a straight line as an SVG path.
func (l *Line) Draw(out io.Writer) {

	start := l.start.asPixel()
	stop := l.stop.asPixel()

	// For cases when a vertical line hits a perpendicular like this:
	//
	//   |          |
	//   |    or    v
	//  ---        ---
	//
	// We need to nudge the vertical line half a vertical cell in the
	// appropriate direction in order to meet up cleanly with the midline of
	// the cell next to it.

	if l.needsNudgingUp {
		start.y -= 8
	}

	if l.needsNudgingDown {
		stop.y += 8
	}

	writeBytes(&out,
		"<path d='M %d,%d L %d,%d' style='fill:none;stroke:#000;'></path>\n",
		start.x, start.y,
		stop.x, stop.y,
	)
}

// Draw a solid triable as an SVG polygon element.
func (t *Triangle) Draw(out io.Writer) {
	// https://www.w3.org/TR/SVG/shapes.html#PolygonElement

	/*
		   	+-----+-----+
		    |    /|\    |
		    |   / | \   |
		  x +- / -+- \ -+
			| /   |   \ |
			|/    |    \|
		    +-----+-----+
		          y
	*/

	x, y := float32(t.start.asPixel().x), float32(t.start.asPixel().y)
	r := 0.0

	x0 := x + 8
	y0 := y
	x1 := x - 4
	y1 := y - 0.35*16
	x2 := x - 4
	y2 := y + 0.35*16

	switch t.orientation {
	case N:
		r = 270
		if t.needsNudging {
			x0 += 8
			x1 += 8
			x2 += 8
		}
	case W:
		r = 180
		if t.needsNudging {
			x0 -= 8
			x1 -= 8
			x2 -= 8
		}
	case E:
		r = 0
		if t.needsNudging {
			x0 -= 8
			x1 -= 8
			x2 -= 8
		}
	case S:
		r = 90
		if t.needsNudging {
			x0 += 8
			x1 += 8
			x2 += 8
		}
	}

	writeBytes(&out,
		"<polygon points='%f,%f %f,%f %f,%f' style='fill:#000' transform='rotate(%f, %f, %f)'></polygon>\n",
		x0, y0,
		x1, y1,
		x2, y2,
		r,
		x, y,
	)
}

// Draw a solid circle as an SVG circle element.
func (c *Circle) Draw(out io.Writer) {
	fill := "#fff"

	if c.bold {
		fill = "#000"
	}

	pixel := c.start.asPixel()

	writeBytes(&out,
		"<circle cx='%d' cy='%d' r='6' style='fill:%s;stroke:#000;'></circle>\n",
		pixel.x,
		pixel.y,
		fill,
	)
}

// Draw a single text character as an SVG text element.
func (t *Text) Draw(out io.Writer) {
	p := t.start.asPixel()
	c := t.contents

	opacity := 0

	// Markdeep special-cases these character and treats them like a
	// checkerboard.
	switch c {
	case "▉":
		opacity = -64
	case "▓":
		opacity = 64
	case "▒":
		opacity = 128
	case "░":
		opacity = 191
	}

	if opacity != 0 {
		writeBytes(&out,
			"<rect x='%d' y='%d' width='8' height='16' fill='rgb(%d,%d,%d)'></rect>",
			p.x-4, p.y-8,
			opacity, opacity, opacity,
		)
		return
	}

	// Escape for XML
	switch c {
	case "&":
		c = "&"
	case ">":
		c = ">"
	case "<":
		c = "<"
	}

	writeBytes(&out,
		"<text text-anchor='middle' font-family='Menlo,Lucida Console,monospace' x='%d' y='%d' style='fill:#000;font-size:1em'>%s</text>\n",
		p.x, p.y+4, c,
	)
}

// Draw a rounded corner as an SVG elliptical arc element.
func (c *RoundedCorner) Draw(out io.Writer) {
	// https://www.w3.org/TR/SVG/paths.html#PathDataEllipticalArcCommands

	x, y := c.start.asPixelXY()
	startX, startY, endX, endY, sweepFlag := 0, 0, 0, 0, 0

	switch c.orientation {
	case NW:
		startX = x + 8
		startY = y
		endX = x - 8
		endY = y + 16
	case NE:
		sweepFlag = 1
		startX = x - 8
		startY = y
		endX = x + 8
		endY = y + 16
	case SE:
		sweepFlag = 1
		startX = x + 8
		startY = y - 16
		endX = x - 8
		endY = y
	case SW:
		startX = x - 8
		startY = y - 16
		endX = x + 8
		endY = y
	}

	writeBytes(&out,
		"<path d='M %d,%d A 16,16 0 0,%d %d,%d' style='fill:none;stroke:#000;'></path>\n",
		startX,
		startY,
		sweepFlag,
		endX,
		endY,
	)
}

// Draw a bridge as an SVG elliptical arc element.
func (b *Bridge) Draw(out io.Writer) {
	x, y := b.start.asPixelXY()
	sweepFlag := 1

	if b.orientation == W {
		sweepFlag = 0
	}

	writeBytes(&out,
		"<path d='M %d,%d A 9,9 0 0,%d %d,%d' style='fill:none;stroke:#000;'></path>\n",
		x, y-8,
		sweepFlag,
		x, y+8,
	)
}
<commit_msg>Draw circles on top of corners.<commit_after>package goat

import (
	"fmt"
	"io"
)

// ASCIItoSVG reads in a newline-delimited ASCII diagram and writes a
// corresponding SVG diagram.
func ASCIItoSVG(in io.Reader, out io.Writer) {
	canvas := NewCanvas(in)

	// Preamble
	writeBytes(&out,
		"<svg class='%s' xmlns='%s' version='%s' height='%d' width='%d'>\n",
		"diagram",
		"http://www.w3.org/2000/svg",
		"1.1",
		canvas.Height*16+8, (canvas.Width+1)*8,
	)

	writeBytes(&out, "<g transform='translate(8,16)'>\n")

	for _, l := range canvas.Lines() {
		l.Draw(out)
	}

	for _, t := range canvas.Triangles() {
		t.Draw(out)
	}

	for _, c := range canvas.RoundedCorners() {
		c.Draw(out)
	}

	for _, c := range canvas.Circles() {
		c.Draw(out)
	}

	for _, b := range canvas.Bridges() {
		b.Draw(out)
	}

	for _, t := range canvas.Text() {
		t.Draw(out)
	}

	writeBytes(&out, "</g>\n")
	writeBytes(&out, "</svg>\n")
}

func writeBytes(out *io.Writer, format string, args ...interface{}) {
	bytesOut := fmt.Sprintf(format, args...)

	_, err := (*out).Write([]byte(bytesOut))

	if err != nil {
		panic(nil)
	}
}

// Draw a straight line as an SVG path.
func (l *Line) Draw(out io.Writer) {

	start := l.start.asPixel()
	stop := l.stop.asPixel()

	// For cases when a vertical line hits a perpendicular like this:
	//
	//   |          |
	//   |    or    v
	//  ---        ---
	//
	// We need to nudge the vertical line half a vertical cell in the
	// appropriate direction in order to meet up cleanly with the midline of
	// the cell next to it.

	if l.needsNudgingUp {
		start.y -= 8
	}

	if l.needsNudgingDown {
		stop.y += 8
	}

	writeBytes(&out,
		"<path d='M %d,%d L %d,%d' style='fill:none;stroke:#000;'></path>\n",
		start.x, start.y,
		stop.x, stop.y,
	)
}

// Draw a solid triable as an SVG polygon element.
func (t *Triangle) Draw(out io.Writer) {
	// https://www.w3.org/TR/SVG/shapes.html#PolygonElement

	/*
		   	+-----+-----+
		    |    /|\    |
		    |   / | \   |
		  x +- / -+- \ -+
			| /   |   \ |
			|/    |    \|
		    +-----+-----+
		          y
	*/

	x, y := float32(t.start.asPixel().x), float32(t.start.asPixel().y)
	r := 0.0

	x0 := x + 8
	y0 := y
	x1 := x - 4
	y1 := y - 0.35*16
	x2 := x - 4
	y2 := y + 0.35*16

	switch t.orientation {
	case N:
		r = 270
		if t.needsNudging {
			x0 += 8
			x1 += 8
			x2 += 8
		}
	case W:
		r = 180
		if t.needsNudging {
			x0 -= 8
			x1 -= 8
			x2 -= 8
		}
	case E:
		r = 0
		if t.needsNudging {
			x0 -= 8
			x1 -= 8
			x2 -= 8
		}
	case S:
		r = 90
		if t.needsNudging {
			x0 += 8
			x1 += 8
			x2 += 8
		}
	}

	writeBytes(&out,
		"<polygon points='%f,%f %f,%f %f,%f' style='fill:#000' transform='rotate(%f, %f, %f)'></polygon>\n",
		x0, y0,
		x1, y1,
		x2, y2,
		r,
		x, y,
	)
}

// Draw a solid circle as an SVG circle element.
func (c *Circle) Draw(out io.Writer) {
	fill := "#fff"

	if c.bold {
		fill = "#000"
	}

	pixel := c.start.asPixel()

	writeBytes(&out,
		"<circle cx='%d' cy='%d' r='6' style='fill:%s;stroke:#000;'></circle>\n",
		pixel.x,
		pixel.y,
		fill,
	)
}

// Draw a single text character as an SVG text element.
func (t *Text) Draw(out io.Writer) {
	p := t.start.asPixel()
	c := t.contents

	opacity := 0

	// Markdeep special-cases these character and treats them like a
	// checkerboard.
	switch c {
	case "▉":
		opacity = -64
	case "▓":
		opacity = 64
	case "▒":
		opacity = 128
	case "░":
		opacity = 191
	}

	if opacity != 0 {
		writeBytes(&out,
			"<rect x='%d' y='%d' width='8' height='16' fill='rgb(%d,%d,%d)'></rect>",
			p.x-4, p.y-8,
			opacity, opacity, opacity,
		)
		return
	}

	// Escape for XML
	switch c {
	case "&":
		c = "&"
	case ">":
		c = ">"
	case "<":
		c = "<"
	}

	writeBytes(&out,
		"<text text-anchor='middle' font-family='Menlo,Lucida Console,monospace' x='%d' y='%d' style='fill:#000;font-size:1em'>%s</text>\n",
		p.x, p.y+4, c,
	)
}

// Draw a rounded corner as an SVG elliptical arc element.
func (c *RoundedCorner) Draw(out io.Writer) {
	// https://www.w3.org/TR/SVG/paths.html#PathDataEllipticalArcCommands

	x, y := c.start.asPixelXY()
	startX, startY, endX, endY, sweepFlag := 0, 0, 0, 0, 0

	switch c.orientation {
	case NW:
		startX = x + 8
		startY = y
		endX = x - 8
		endY = y + 16
	case NE:
		sweepFlag = 1
		startX = x - 8
		startY = y
		endX = x + 8
		endY = y + 16
	case SE:
		sweepFlag = 1
		startX = x + 8
		startY = y - 16
		endX = x - 8
		endY = y
	case SW:
		startX = x - 8
		startY = y - 16
		endX = x + 8
		endY = y
	}

	writeBytes(&out,
		"<path d='M %d,%d A 16,16 0 0,%d %d,%d' style='fill:none;stroke:#000;'></path>\n",
		startX,
		startY,
		sweepFlag,
		endX,
		endY,
	)
}

// Draw a bridge as an SVG elliptical arc element.
func (b *Bridge) Draw(out io.Writer) {
	x, y := b.start.asPixelXY()
	sweepFlag := 1

	if b.orientation == W {
		sweepFlag = 0
	}

	writeBytes(&out,
		"<path d='M %d,%d A 9,9 0 0,%d %d,%d' style='fill:none;stroke:#000;'></path>\n",
		x, y-8,
		sweepFlag,
		x, y+8,
	)
}
<|endoftext|>