<commit_before>package jsonparser

import (
	"bytes"
	"errors"
	"fmt"
)

// Errors
var (
	KeyPathNotFoundError       = errors.New("Key path not found")
	UnknownValueTypeError      = errors.New("Unknown value type")
	MalformedJsonError         = errors.New("Malformed JSON error")
	MalformedStringError       = errors.New("Value is string, but can't find closing '\"' symbol")
	MalformedArrayError        = errors.New("Value is array, but can't find closing ']' symbol")
	MalformedObjectError       = errors.New("Value looks like object, but can't find closing '}' symbol")
	MalformedValueError        = errors.New("Value looks like Number/Boolean/None, but can't find its end: ',' or '}' symbol")
	MalformedStringEscapeError = errors.New("Encountered an invalid escape sequence in a string")
)

// How much stack space to allocate for unescaping JSON strings; if a string longer
// than this needs to be escaped, it will result in a heap allocation
const unescapeStackBufSize = 64

func tokenEnd(data []byte) int {
	for i, c := range data {
		switch c {
		case ' ', '\n', '\r', '\t', ',', '}', ']':
			return i
		}
	}

	return -1
}

// Find position of next character which is not whitespace
func nextToken(data []byte) int {
	for i, c := range data {
		switch c {
		case ' ', '\n', '\r', '\t':
			continue
		default:
			return i
		}
	}

	return -1
}

// Tries to find the end of string
// Support if string contains escaped quote symbols.
func stringEnd(data []byte) (int, bool) {
	escaped := false
	for i, c := range data {
		if c == '"' {
			if !escaped {
				return i + 1, false
			} else {
				j := i - 1
				for {
					if j < 0 || data[j] != '\\' {
						return i + 1, true // even number of backslashes
					}
					j--
					if j < 0 || data[j] != '\\' {
						break // odd number of backslashes
					}
					j--

				}
			}
		} else if c == '\\' {
			escaped = true
		}
	}

	return -1, escaped
}

// Find end of the data structure, array or object.
// For array openSym and closeSym will be '[' and ']', for object '{' and '}'
func blockEnd(data []byte, openSym byte, closeSym byte) int {
	level := 0
	i := 0
	ln := len(data)

	for i < ln {
		switch data[i] {
		case '"': // If inside string, skip it
			se, _ := stringEnd(data[i+1:])
			if se == -1 {
				return -1
			}
			i += se
		case openSym: // If open symbol, increase level
			level++
		case closeSym: // If close symbol, increase level
			level--

			// If we have returned to the original level, we're done
			if level == 0 {
				return i + 1
			}
		}
		i++
	}

	return -1
}

func searchKeys(data []byte, keys ...string) int {
	keyLevel := 0
	level := 0
	i := 0
	ln := len(data)
	lk := len(keys)

	var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings

	for i < ln {
		switch data[i] {
		case '"':
			i++
			keyBegin := i

			strEnd, keyEscaped := stringEnd(data[i:])
			if strEnd == -1 {
				return -1
			}
			i += strEnd
			keyEnd := i - 1

			valueOffset := nextToken(data[i:])
			if valueOffset == -1 {
				return -1
			}

			i += valueOffset

			// if string is a key, and key level match
			if data[i] == ':' && keyLevel == level-1 {
				key := data[keyBegin:keyEnd]

				// for unescape: if there are no escape sequences, this is cheap; if there are, it is a
				// bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize
				var keyUnesc []byte
				if !keyEscaped {
					keyUnesc = key
				} else if ku, err := Unescape(key, stackbuf[:]); err != nil {
					return -1
				} else {
					keyUnesc = ku
				}

				if equalStr(&keyUnesc, keys[level-1]) {
					keyLevel++
					// If we found all keys in path
					if keyLevel == lk {
						return i + 1
					}
				}
			} else {
				i--
			}
		case '{':
			level++
		case '}':
			level--
		case '[':
			// Do not search for keys inside arrays
			arraySkip := blockEnd(data[i:], '[', ']')
			i += arraySkip - 1
		}

		i++
	}

	return -1
}

// Data types available in valid JSON data.
type ValueType int

const (
	NotExist = ValueType(iota)
	String
	Number
	Object
	Array
	Boolean
	Null
	Unknown
)

var (
	trueLiteral  = []byte("true")
	falseLiteral = []byte("false")
	nullLiteral  = []byte("null")
)

/*
Get - Receives data structure, and key path to extract value from.

Returns:
`value` - Pointer to original data structure containing key value, or just empty slice if nothing found or error
`dataType` -    Can be: `NotExist`, `String`, `Number`, `Object`, `Array`, `Boolean` or `Null`
`offset` - Offset from provided data structure where key value ends. Used mostly internally, for example for `ArrayEach` helper.
`err` - If key not found or any other parsing issue it should return error. If key not found it also sets `dataType` to `NotExist`

Accept multiple keys to specify path to JSON value (in case of quering nested structures).
If no keys provided it will try to extract closest JSON value (simple ones or object/array), useful for reading streams or arrays, see `ArrayEach` implementation.
*/
func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) {
	if len(keys) > 0 {
		if offset = searchKeys(data, keys...); offset == -1 {
			return []byte{}, NotExist, -1, KeyPathNotFoundError
		}
	}

	// Go to closest value
	nO := nextToken(data[offset:])
	if nO == -1 {
		return []byte{}, NotExist, -1, MalformedJsonError
	}

	offset += nO

	endOffset := offset
	// if string value
	if data[offset] == '"' {
		dataType = String
		if idx, _ := stringEnd(data[offset+1:]); idx != -1 {
			endOffset += idx + 1
		} else {
			return []byte{}, dataType, offset, MalformedStringError
		}
	} else if data[offset] == '[' { // if array value
		dataType = Array
		// break label, for stopping nested loops
		endOffset = blockEnd(data[offset:], '[', ']')

		if endOffset == -1 {
			return []byte{}, dataType, offset, MalformedArrayError
		}

		endOffset += offset
	} else if data[offset] == '{' { // if object value
		dataType = Object
		// break label, for stopping nested loops
		endOffset = blockEnd(data[offset:], '{', '}')

		if endOffset == -1 {
			return []byte{}, dataType, offset, MalformedObjectError
		}

		endOffset += offset
	} else {
		// Number, Boolean or None
		end := tokenEnd(data[endOffset:])

		if end == -1 {
			return nil, dataType, offset, MalformedValueError
		}

		value := data[offset : endOffset+end]

		switch data[offset] {
		case 't', 'f': // true or false
			if bytes.Equal(value, trueLiteral) || bytes.Equal(value, falseLiteral) {
				dataType = Boolean
			} else {
				return nil, Unknown, offset, UnknownValueTypeError
			}
		case 'u', 'n': // undefined or null
			if bytes.Equal(value, nullLiteral) {
				dataType = Null
			} else {
				return nil, Unknown, offset, UnknownValueTypeError
			}
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-':
			dataType = Number
		default:
			return nil, Unknown, offset, UnknownValueTypeError
		}

		endOffset += end
	}

	value = data[offset:endOffset]

	// Strip quotes from string values
	if dataType == String {
		value = value[1 : len(value)-1]
	}

	if dataType == Null {
		value = []byte{}
	}

	return value, dataType, endOffset, nil
}

// ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`.
func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int, err error), keys ...string) (err error) {
	if len(data) == 0 {
		return MalformedObjectError
	}

	offset := 1

	if len(keys) > 0 {
		if offset = searchKeys(data, keys...); offset == -1 {
			return KeyPathNotFoundError
		}

		// Go to closest value
		nO := nextToken(data[offset:])
		if nO == -1 {
			return MalformedJsonError
		}

		offset += nO

		if data[offset] != '[' {
			return MalformedArrayError
		}

		offset++
	}

	for true {
		v, t, o, e := Get(data[offset:])

		if o == 0 {
			break
		}

		if t != NotExist {
			cb(v, t, o, e)
		}

		if e != nil {
			break
		}

		offset += o

		skipToToken := nextToken(data[offset:])
		if skipToToken == -1 {
			return MalformedArrayError
		}
		offset += skipToToken

		if data[offset] == ']' {
			break
		}

		if data[offset] != ',' {
			return MalformedArrayError
		}

		offset++
	}

	return nil
}

// GetUnsafeString returns the value retrieved by `Get`, use creates string without memory allocation by mapping string to slice memory. It does not handle escape symbols.
func GetUnsafeString(data []byte, keys ...string) (val string, err error) {
	v, _, _, e := Get(data, keys...)

	if e != nil {
		return "", e
	}

	return bytesToString(&v), nil
}

// GetString returns the value retrieved by `Get`, cast to a string if possible, trying to properly handle escape and utf8 symbols
// If key data type do not match, it will return an error.
func GetString(data []byte, keys ...string) (val string, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return "", e
	}

	if t != String {
		return "", fmt.Errorf("Value is not a number: %s", string(v))
	}

	// If no escapes return raw conten
	if bytes.IndexByte(v, '\\') == -1 {
		return string(v), nil
	}

	return ParseString(v)
}

// GetFloat returns the value retrieved by `Get`, cast to a float64 if possible.
// The offset is the same as in `Get`.
// If key data type do not match, it will return an error.
func GetFloat(data []byte, keys ...string) (val float64, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return 0, e
	}

	if t != Number {
		return 0, fmt.Errorf("Value is not a number: %s", string(v))
	}

	return ParseFloat(v)
}

// GetInt returns the value retrieved by `Get`, cast to a float64 if possible.
// If key data type do not match, it will return an error.
func GetInt(data []byte, keys ...string) (val int64, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return 0, e
	}

	if t != Number {
		return 0, fmt.Errorf("Value is not a number: %s", string(v))
	}

	return ParseInt(v)
}

// GetBoolean returns the value retrieved by `Get`, cast to a bool if possible.
// The offset is the same as in `Get`.
// If key data type do not match, it will return error.
func GetBoolean(data []byte, keys ...string) (val bool, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return false, e
	}

	if t != Boolean {
		return false, fmt.Errorf("Value is not a boolean: %s", string(v))
	}

	return ParseBoolean(v)
}

// ParseBoolean parses a Boolean ValueType into a Go bool (not particularly useful, but here for completeness)
func ParseBoolean(b []byte) (bool, error) {
	switch {
	case bytes.Equal(b, trueLiteral):
		return true, nil
	case bytes.Equal(b, falseLiteral):
		return false, nil
	default:
		return false, MalformedValueError
	}
}

// ParseString parses a String ValueType into a Go string (the main parsing work is unescaping the JSON string)
func ParseString(b []byte) (string, error) {
	var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings
	if bU, err := Unescape(b, stackbuf[:]); err != nil {
		return "", nil
	} else {
		return string(bU), nil
	}
}

// ParseNumber parses a Number ValueType into a Go float64
func ParseFloat(b []byte) (float64, error) {
	if v, err := parseFloat(&b); err != nil {
		return 0, MalformedValueError
	} else {
		return v, nil
	}
}

// ParseInt parses a Number ValueType into a Go int64
func ParseInt(b []byte) (int64, error) {
	if v, ok := parseInt(b); !ok {
		return 0, MalformedValueError
	} else {
		return v, nil
	}
}
<commit_msg>Update parser.go<commit_after>package jsonparser

import (
	"bytes"
	"errors"
	"fmt"
)

// Errors
var (
	KeyPathNotFoundError       = errors.New("Key path not found")
	UnknownValueTypeError      = errors.New("Unknown value type")
	MalformedJsonError         = errors.New("Malformed JSON error")
	MalformedStringError       = errors.New("Value is string, but can't find closing '\"' symbol")
	MalformedArrayError        = errors.New("Value is array, but can't find closing ']' symbol")
	MalformedObjectError       = errors.New("Value looks like object, but can't find closing '}' symbol")
	MalformedValueError        = errors.New("Value looks like Number/Boolean/None, but can't find its end: ',' or '}' symbol")
	MalformedStringEscapeError = errors.New("Encountered an invalid escape sequence in a string")
)

// How much stack space to allocate for unescaping JSON strings; if a string longer
// than this needs to be escaped, it will result in a heap allocation
const unescapeStackBufSize = 64

func tokenEnd(data []byte) int {
	for i, c := range data {
		switch c {
		case ' ', '\n', '\r', '\t', ',', '}', ']':
			return i
		}
	}

	return -1
}

// Find position of next character which is not whitespace
func nextToken(data []byte) int {
	for i, c := range data {
		switch c {
		case ' ', '\n', '\r', '\t':
			continue
		default:
			return i
		}
	}

	return -1
}

// Tries to find the end of string
// Support if string contains escaped quote symbols.
func stringEnd(data []byte) (int, bool) {
	escaped := false
	for i, c := range data {
		if c == '"' {
			if !escaped {
				return i + 1, false
			} else {
				j := i - 1
				for {
					if j < 0 || data[j] != '\\' {
						return i + 1, true // even number of backslashes
					}
					j--
					if j < 0 || data[j] != '\\' {
						break // odd number of backslashes
					}
					j--

				}
			}
		} else if c == '\\' {
			escaped = true
		}
	}

	return -1, escaped
}

// Find end of the data structure, array or object.
// For array openSym and closeSym will be '[' and ']', for object '{' and '}'
func blockEnd(data []byte, openSym byte, closeSym byte) int {
	level := 0
	i := 0
	ln := len(data)

	for i < ln {
		switch data[i] {
		case '"': // If inside string, skip it
			se, _ := stringEnd(data[i+1:])
			if se == -1 {
				return -1
			}
			i += se
		case openSym: // If open symbol, increase level
			level++
		case closeSym: // If close symbol, increase level
			level--

			// If we have returned to the original level, we're done
			if level == 0 {
				return i + 1
			}
		}
		i++
	}

	return -1
}

func searchKeys(data []byte, keys ...string) int {
	keyLevel := 0
	level := 0
	i := 0
	ln := len(data)
	lk := len(keys)

	var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings

	for i < ln {
		switch data[i] {
		case '"':
			i++
			keyBegin := i

			strEnd, keyEscaped := stringEnd(data[i:])
			if strEnd == -1 {
				return -1
			}
			i += strEnd
			keyEnd := i - 1

			valueOffset := nextToken(data[i:])
			if valueOffset == -1 {
				return -1
			}

			i += valueOffset

			// if string is a key, and key level match
			if data[i] == ':' && keyLevel == level-1 {
				key := data[keyBegin:keyEnd]

				// for unescape: if there are no escape sequences, this is cheap; if there are, it is a
				// bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize
				var keyUnesc []byte
				if !keyEscaped {
					keyUnesc = key
				} else if ku, err := Unescape(key, stackbuf[:]); err != nil {
					return -1
				} else {
					keyUnesc = ku
				}

				if equalStr(&keyUnesc, keys[level-1]) {
					keyLevel++
					// If we found all keys in path
					if keyLevel == lk {
						return i + 1
					}
				}
			} else {
				i--
			}
		case '{':
			level++
		case '}':
			level--
		case '[':
			// Do not search for keys inside arrays
			arraySkip := blockEnd(data[i:], '[', ']')
			i += arraySkip - 1
		}

		i++
	}

	return -1
}

// Data types available in valid JSON data.
type ValueType int

const (
	NotExist = ValueType(iota)
	String
	Number
	Object
	Array
	Boolean
	Null
	Unknown
)

var (
	trueLiteral  = []byte("true")
	falseLiteral = []byte("false")
	nullLiteral  = []byte("null")
)

/*
Get - Receives data structure, and key path to extract value from.

Returns:
`value` - Pointer to original data structure containing key value, or just empty slice if nothing found or error
`dataType` -    Can be: `NotExist`, `String`, `Number`, `Object`, `Array`, `Boolean` or `Null`
`offset` - Offset from provided data structure where key value ends. Used mostly internally, for example for `ArrayEach` helper.
`err` - If key not found or any other parsing issue it should return error. If key not found it also sets `dataType` to `NotExist`

Accept multiple keys to specify path to JSON value (in case of quering nested structures).
If no keys provided it will try to extract closest JSON value (simple ones or object/array), useful for reading streams or arrays, see `ArrayEach` implementation.
*/
func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) {
	if len(keys) > 0 {
		if offset = searchKeys(data, keys...); offset == -1 {
			return []byte{}, NotExist, -1, KeyPathNotFoundError
		}
	}

	// Go to closest value
	nO := nextToken(data[offset:])
	if nO == -1 {
		return []byte{}, NotExist, -1, MalformedJsonError
	}

	offset += nO

	endOffset := offset
	// if string value
	if data[offset] == '"' {
		dataType = String
		if idx, _ := stringEnd(data[offset+1:]); idx != -1 {
			endOffset += idx + 1
		} else {
			return []byte{}, dataType, offset, MalformedStringError
		}
	} else if data[offset] == '[' { // if array value
		dataType = Array
		// break label, for stopping nested loops
		endOffset = blockEnd(data[offset:], '[', ']')

		if endOffset == -1 {
			return []byte{}, dataType, offset, MalformedArrayError
		}

		endOffset += offset
	} else if data[offset] == '{' { // if object value
		dataType = Object
		// break label, for stopping nested loops
		endOffset = blockEnd(data[offset:], '{', '}')

		if endOffset == -1 {
			return []byte{}, dataType, offset, MalformedObjectError
		}

		endOffset += offset
	} else {
		// Number, Boolean or None
		end := tokenEnd(data[endOffset:])

		if end == -1 {
			return nil, dataType, offset, MalformedValueError
		}

		value := data[offset : endOffset+end]

		switch data[offset] {
		case 't', 'f': // true or false
			if bytes.Equal(value, trueLiteral) || bytes.Equal(value, falseLiteral) {
				dataType = Boolean
			} else {
				return nil, Unknown, offset, UnknownValueTypeError
			}
		case 'u', 'n': // undefined or null
			if bytes.Equal(value, nullLiteral) {
				dataType = Null
			} else {
				return nil, Unknown, offset, UnknownValueTypeError
			}
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-':
			dataType = Number
		default:
			return nil, Unknown, offset, UnknownValueTypeError
		}

		endOffset += end
	}

	value = data[offset:endOffset]

	// Strip quotes from string values
	if dataType == String {
		value = value[1 : len(value)-1]
	}

	if dataType == Null {
		value = []byte{}
	}

	return value, dataType, endOffset, nil
}

// ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`.
func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int, err error), keys ...string) (err error) {
	if len(data) == 0 {
		return MalformedObjectError
	}

	offset := 1

	if len(keys) > 0 {
		if offset = searchKeys(data, keys...); offset == -1 {
			return KeyPathNotFoundError
		}

		// Go to closest value
		nO := nextToken(data[offset:])
		if nO == -1 {
			return MalformedJsonError
		}

		offset += nO

		if data[offset] != '[' {
			return MalformedArrayError
		}

		offset++
	}

	for true {
		v, t, o, e := Get(data[offset:])

		if o == 0 {
			break
		}

		if t != NotExist {
			cb(v, t, o, e)
		}

		if e != nil {
			break
		}

		offset += o

		skipToToken := nextToken(data[offset:])
		if skipToToken == -1 {
			return MalformedArrayError
		}
		offset += skipToToken

		if data[offset] == ']' {
			break
		}

		if data[offset] != ',' {
			return MalformedArrayError
		}

		offset++
	}

	return nil
}

// GetUnsafeString returns the value retrieved by `Get`, use creates string without memory allocation by mapping string to slice memory. It does not handle escape symbols.
func GetUnsafeString(data []byte, keys ...string) (val string, err error) {
	v, _, _, e := Get(data, keys...)

	if e != nil {
		return "", e
	}

	return bytesToString(&v), nil
}

// GetString returns the value retrieved by `Get`, cast to a string if possible, trying to properly handle escape and utf8 symbols
// If key data type do not match, it will return an error.
func GetString(data []byte, keys ...string) (val string, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return "", e
	}

	if t != String {
		return "", fmt.Errorf("Value is not a number: %s", string(v))
	}

	// If no escapes return raw conten
	if bytes.IndexByte(v, '\\') == -1 {
		return string(v), nil
	}

	return ParseString(v)
}

// GetFloat returns the value retrieved by `Get`, cast to a float64 if possible.
// The offset is the same as in `Get`.
// If key data type do not match, it will return an error.
func GetFloat(data []byte, keys ...string) (val float64, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return 0, e
	}

	if t != Number {
		return 0, fmt.Errorf("Value is not a number: %s", string(v))
	}

	return ParseFloat(v)
}

// GetInt returns the value retrieved by `Get`, cast to a int64 if possible.
// If key data type do not match, it will return an error.
func GetInt(data []byte, keys ...string) (val int64, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return 0, e
	}

	if t != Number {
		return 0, fmt.Errorf("Value is not a number: %s", string(v))
	}

	return ParseInt(v)
}

// GetBoolean returns the value retrieved by `Get`, cast to a bool if possible.
// The offset is the same as in `Get`.
// If key data type do not match, it will return error.
func GetBoolean(data []byte, keys ...string) (val bool, err error) {
	v, t, _, e := Get(data, keys...)

	if e != nil {
		return false, e
	}

	if t != Boolean {
		return false, fmt.Errorf("Value is not a boolean: %s", string(v))
	}

	return ParseBoolean(v)
}

// ParseBoolean parses a Boolean ValueType into a Go bool (not particularly useful, but here for completeness)
func ParseBoolean(b []byte) (bool, error) {
	switch {
	case bytes.Equal(b, trueLiteral):
		return true, nil
	case bytes.Equal(b, falseLiteral):
		return false, nil
	default:
		return false, MalformedValueError
	}
}

// ParseString parses a String ValueType into a Go string (the main parsing work is unescaping the JSON string)
func ParseString(b []byte) (string, error) {
	var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings
	if bU, err := Unescape(b, stackbuf[:]); err != nil {
		return "", nil
	} else {
		return string(bU), nil
	}
}

// ParseNumber parses a Number ValueType into a Go float64
func ParseFloat(b []byte) (float64, error) {
	if v, err := parseFloat(&b); err != nil {
		return 0, MalformedValueError
	} else {
		return v, nil
	}
}

// ParseInt parses a Number ValueType into a Go int64
func ParseInt(b []byte) (int64, error) {
	if v, ok := parseInt(b); !ok {
		return 0, MalformedValueError
	} else {
		return v, nil
	}
}
<|endoftext|>


<commit_before>package php

import (
	"fmt"

	"stephensearles.com/php/ast"
)

type parser struct {
	lexer *lexer

	previous   []Item
	idx        int
	current    Item
	errors     []error
	parenLevel int
	errorMap   map[int]bool

	debug     bool
	MaxErrors int
}

func NewParser(input string) *parser {
	return newParser(input)
}

func newParser(input string) *parser {
	p := &parser{
		idx:       -1,
		MaxErrors: 10,
		lexer:     newLexer(input),
		errorMap:  make(map[int]bool),
	}
	return p
}

func (p *parser) Parse() []ast.Node {
	return p.parse()
}

func (p *parser) parse() []ast.Node {
	defer func() {
		if len(p.errors) > 0 {
			for _, err := range p.errors {
				fmt.Println(err)
			}
		}
		if r := recover(); r != nil {
			if p.debug {
				panic(r)
			} else {
				fmt.Println(r)
			}
		}
	}()
	// expecting either itemHTML or itemPHPBegin
	nodes := make([]ast.Node, 0, 1)
TokenLoop:
	for {
		p.next()
		switch p.current.typ {
		case itemEOF:
			break TokenLoop
		default:
			n := p.parseNode()
			if n != nil {
				nodes = append(nodes, n)
			}
		}
	}
	return nodes
}

func (p *parser) parseNode() ast.Node {
	switch p.current.typ {
	case itemHTML:
		return ast.Echo(ast.Literal{Type: ast.String})
	case itemPHPBegin:
		return nil
	case itemPHPEnd:
		return nil
	}
	return p.parseStmt()
}

func (p *parser) next() {
	p.idx += 1
	if len(p.previous) <= p.idx {
		p.current = p.lexer.nextItem()
		p.previous = append(p.previous, p.current)
	} else {
		p.current = p.previous[p.idx]
	}
}

func (p *parser) backup() {
	p.idx -= 1
	p.current = p.previous[p.idx]
}

func (p *parser) peek() (i Item) {
	p.next()
	i = p.current
	p.backup()
	return
}

func (p *parser) expect(i ItemType) {
	p.next()
	if p.current.typ != i {
		p.expected(i)
	}
}

func (p *parser) expected(i ItemType) {
	p.errorf("Found %s, expected %s", p.current, i)
}

func (p *parser) errorf(str string, args ...interface{}) {
	if _, ok := p.errorMap[p.current.pos.Line]; ok {
		return
	}
	errString := fmt.Sprintf(str, args...)
	p.errorMap[p.current.pos.Line] = true
	p.errors = append(p.errors, fmt.Errorf("%s: %s", p.errorPrefix(), errString))
	if len(p.errors) > p.MaxErrors {
		panic("too many errors")
	}
}

func (p *parser) errorPrefix() string {
	return fmt.Sprintf("%s %d", p.lexer.file, p.current.pos.Line)
}

func (p *parser) parseIf() *ast.IfStmt {
	p.expect(itemOpenParen)
	p.parenLevel += 1
	n := &ast.IfStmt{}
	p.next()
	n.Condition = p.parseExpression()
	p.expect(itemCloseParen)
	p.next()
	n.TrueBlock = p.parseStmt()
	p.next()
	if p.current.typ == itemElse {
		p.next()
		n.FalseBlock = p.parseStmt()
	} else {
		n.FalseBlock = ast.Block{}
		p.backup()
	}
	return n
}

func (p *parser) parseNextExpression() ast.Expression {
	p.next()
	return p.parseExpression()
}

func newUnaryOperation(operator Item, expr ast.Expression) ast.OperatorExpression {
	t := ast.Numeric
	if operator.val == "!" {
		t = ast.Boolean
	}
	return ast.OperatorExpression{
		Type:     t,
		Operand1: expr,
	}
}

func newBinaryOperation(operator Item, expr1, expr2 ast.Expression) ast.OperatorExpression {
	t := ast.Numeric
	switch operator.typ {
	case itemComparisonOperator, itemAndOperator, itemOrOperator, itemWrittenAndOperator, itemWrittenOrOperator, itemWrittenXorOperator:
		t = ast.Boolean
	case itemConcatenationOperator:
		t = ast.String
	case itemAmpersandOperator, itemBitwiseXorOperator, itemBitwiseOrOperator, itemBitwiseShiftOperator:
		t = ast.AnyType
	}
	return ast.OperatorExpression{
		Type:     t,
		Operand1: expr1,
		Operand2: expr2,
		Operator: operator.val,
	}
}

func (p *parser) parseFunctionCall() *ast.FunctionCallExpression {
	expr := &ast.FunctionCallExpression{}
	if p.current.typ != itemNonVariableIdentifier {
		p.expected(itemNonVariableIdentifier)
	}
	expr.FunctionName = p.current.val
	expr.Arguments = make([]ast.Expression, 0)
	p.expect(itemOpenParen)
	first := true
	p.next()
	for {
		if p.current.typ == itemCloseParen {
			break
		}
		if !first {
			p.backup()
			p.expect(itemArgumentSeparator)
			p.next()
		} else {
			first = false
		}
		arg := p.parseExpression()
		if arg == nil {
			break
		}
		expr.Arguments = append(expr.Arguments, arg)
		p.next()
	}
	return expr
}

func (p *parser) parseStmt() ast.Statement {
	switch p.current.typ {
	case itemBlockBegin:
		p.backup()
		return p.parseBlock()
	case itemIdentifier:
		switch p.peek().typ {
		case itemUnaryOperator:
			expr := ast.ExpressionStmt{p.parseExpression()}
			p.expect(itemStatementEnd)
			return expr
		default:
			n := ast.AssignmentStmt{}
			n.Assignee = p.parseIdentifier().(ast.Assignable)
			p.expect(itemAssignmentOperator)
			n.Operator = p.current.val
			p.next()
			n.Value = p.parseExpression()
			p.expect(itemStatementEnd)
			return n
		}
	case itemUnaryOperator:
		expr := ast.ExpressionStmt{p.parseExpression()}
		p.expect(itemStatementEnd)
		return expr
	case itemFunction:
		return p.parseFunctionStmt()
	case itemEcho:
		p.next()
		expr := p.parseExpression()
		p.expect(itemStatementEnd)
		return ast.Echo(expr)
	case itemIf:
		return p.parseIf()
	case itemWhile:
		return p.parseWhile()
	case itemDo:
		return p.parseDo()
	case itemNonVariableIdentifier:
		stmt := p.parseFunctionCall()
		p.expect(itemStatementEnd)
		return stmt
	case itemClass:
		return p.parseClass()
	case itemReturn:
		p.next()
		stmt := ast.ReturnStmt{Expression: p.parseExpression()}
		p.expect(itemStatementEnd)
		return stmt
	case itemThrow:
		stmt := ast.ThrowStmt{Expression: p.parseNextExpression()}
		p.expect(itemStatementEnd)
		return stmt
	case itemExit:
		stmt := ast.ExitStmt{}
		if p.peek().typ == itemOpenParen {
			p.expect(itemOpenParen)
			stmt.Expression = p.parseNextExpression()
			p.expect(itemCloseParen)
		}
		p.expect(itemStatementEnd)
		return stmt
	case itemInclude:
		stmt := ast.IncludeStmt{Expression: p.parseNextExpression()}
		p.expect(itemStatementEnd)
		return stmt
	case itemIgnoreErrorOperator:
		// Ignore this operator
		return p.parseStmt()
	default:
		p.errorf("Found %s, expected html or php begin", p.current)
		return nil
	}
}

func (p *parser) parseWhile() ast.Statement {
	p.expect(itemOpenParen)
	term := p.parseNextExpression()
	p.expect(itemCloseParen)
	block := p.parseBlock()
	return &ast.WhileStmt{
		Termination: term,
		LoopBlock:   block,
	}
}

func (p *parser) parseDo() ast.Statement {
	block := p.parseBlock()
	p.expect(itemWhile)
	p.expect(itemOpenParen)
	term := p.parseNextExpression()
	p.expect(itemCloseParen)
	p.expect(itemStatementEnd)
	return &ast.DoWhileStmt{
		Termination: term,
		LoopBlock:   block,
	}
}

func (p *parser) parseFunctionStmt() *ast.FunctionStmt {
	stmt := &ast.FunctionStmt{}
	p.expect(itemNonVariableIdentifier)
	stmt.Name = p.current.val
	stmt.Arguments = make([]ast.FunctionArgument, 0)
	p.expect(itemOpenParen)
	first := true
	for {
		p.next()
		if p.current.typ == itemCloseParen {
			break
		}
		p.backup()
		if !first {
			p.expect(itemArgumentSeparator)
		} else {
			first = false
		}
		p.next()
		arg := ast.FunctionArgument{}
		if p.current.typ == itemNonVariableIdentifier {
			arg.TypeHint = p.current.val
		} else {
			p.backup()
		}
		p.expect(itemIdentifier)
		arg.Identifier = ast.NewIdentifier(p.current.val)
		stmt.Arguments = append(stmt.Arguments, arg)
	}
	stmt.Body = p.parseBlock()
	return stmt
}

func (p *parser) parseBlock() ast.Block {
	block := ast.Block{}
	p.expect(itemBlockBegin)
	for {
		p.next()
		block.Statements = append(block.Statements, p.parseStmt())
		if p.next(); p.current.typ == itemBlockEnd {
			break
		}
		p.backup()
	}
	return block
}

func (p *parser) parseClass() ast.Class {
	p.expect(itemNonVariableIdentifier)
	name := p.current.val
	p.next()
	if p.current.typ == itemExtends {
		p.expect(itemNonVariableIdentifier)
	} else {
		p.backup()
	}
	p.expect(itemBlockBegin)
	return p.parseClassFields(ast.Class{Name: name})
}

func (p *parser) parseClassFields(c ast.Class) ast.Class {
	c.Methods = make([]ast.Method, 0)
	c.Properties = make([]ast.Property, 0)
	p.next()
	var vis ast.Visibility
	for p.current.typ != itemBlockEnd {
		switch p.current.typ {
		case itemPrivate:
			vis = ast.Private
		case itemProtected:
			vis = ast.Protected
		case itemPublic:
			vis = ast.Public
		default:
			vis = ast.Public
			p.backup()
		}
		p.next()
		switch p.current.typ {
		case itemFunction:
			c.Methods = append(c.Methods, ast.Method{
				Visibility:   vis,
				FunctionStmt: p.parseFunctionStmt(),
			})
		case itemIdentifier:
			c.Properties = append(c.Properties, ast.Property{
				Visibility: vis,
				Name:       p.current.val,
			})
		default:
		}
		p.next()
	}
	return c
}
<commit_msg>Supporting for statement<commit_after>package php

import (
	"fmt"

	"stephensearles.com/php/ast"
)

type parser struct {
	lexer *lexer

	previous   []Item
	idx        int
	current    Item
	errors     []error
	parenLevel int
	errorMap   map[int]bool

	debug     bool
	MaxErrors int
}

func NewParser(input string) *parser {
	return newParser(input)
}

func newParser(input string) *parser {
	p := &parser{
		idx:       -1,
		MaxErrors: 10,
		lexer:     newLexer(input),
		errorMap:  make(map[int]bool),
	}
	return p
}

func (p *parser) Parse() []ast.Node {
	return p.parse()
}

func (p *parser) parse() []ast.Node {
	defer func() {
		if len(p.errors) > 0 {
			for _, err := range p.errors {
				fmt.Println(err)
			}
		}
		if r := recover(); r != nil {
			if p.debug {
				panic(r)
			} else {
				fmt.Println(r)
			}
		}
	}()
	// expecting either itemHTML or itemPHPBegin
	nodes := make([]ast.Node, 0, 1)
TokenLoop:
	for {
		p.next()
		switch p.current.typ {
		case itemEOF:
			break TokenLoop
		default:
			n := p.parseNode()
			if n != nil {
				nodes = append(nodes, n)
			}
		}
	}
	return nodes
}

func (p *parser) parseNode() ast.Node {
	switch p.current.typ {
	case itemHTML:
		return ast.Echo(ast.Literal{Type: ast.String})
	case itemPHPBegin:
		return nil
	case itemPHPEnd:
		return nil
	}
	return p.parseStmt()
}

func (p *parser) next() {
	p.idx += 1
	if len(p.previous) <= p.idx {
		p.current = p.lexer.nextItem()
		p.previous = append(p.previous, p.current)
	} else {
		p.current = p.previous[p.idx]
	}
}

func (p *parser) backup() {
	p.idx -= 1
	p.current = p.previous[p.idx]
}

func (p *parser) peek() (i Item) {
	p.next()
	i = p.current
	p.backup()
	return
}

func (p *parser) expect(i ItemType) {
	p.next()
	if p.current.typ != i {
		p.expected(i)
	}
}

func (p *parser) expected(i ItemType) {
	p.errorf("Found %s, expected %s", p.current, i)
}

func (p *parser) errorf(str string, args ...interface{}) {
	if _, ok := p.errorMap[p.current.pos.Line]; ok {
		return
	}
	errString := fmt.Sprintf(str, args...)
	p.errorMap[p.current.pos.Line] = true
	p.errors = append(p.errors, fmt.Errorf("%s: %s", p.errorPrefix(), errString))
	if len(p.errors) > p.MaxErrors {
		panic("too many errors")
	}
}

func (p *parser) errorPrefix() string {
	return fmt.Sprintf("%s %d", p.lexer.file, p.current.pos.Line)
}

func (p *parser) parseIf() *ast.IfStmt {
	p.expect(itemOpenParen)
	p.parenLevel += 1
	n := &ast.IfStmt{}
	p.next()
	n.Condition = p.parseExpression()
	p.expect(itemCloseParen)
	p.next()
	n.TrueBlock = p.parseStmt()
	p.next()
	if p.current.typ == itemElse {
		p.next()
		n.FalseBlock = p.parseStmt()
	} else {
		n.FalseBlock = ast.Block{}
		p.backup()
	}
	return n
}

func (p *parser) parseNextExpression() ast.Expression {
	p.next()
	return p.parseExpression()
}

func newUnaryOperation(operator Item, expr ast.Expression) ast.OperatorExpression {
	t := ast.Numeric
	if operator.val == "!" {
		t = ast.Boolean
	}
	return ast.OperatorExpression{
		Type:     t,
		Operand1: expr,
	}
}

func newBinaryOperation(operator Item, expr1, expr2 ast.Expression) ast.OperatorExpression {
	t := ast.Numeric
	switch operator.typ {
	case itemComparisonOperator, itemAndOperator, itemOrOperator, itemWrittenAndOperator, itemWrittenOrOperator, itemWrittenXorOperator:
		t = ast.Boolean
	case itemConcatenationOperator:
		t = ast.String
	case itemAmpersandOperator, itemBitwiseXorOperator, itemBitwiseOrOperator, itemBitwiseShiftOperator:
		t = ast.AnyType
	}
	return ast.OperatorExpression{
		Type:     t,
		Operand1: expr1,
		Operand2: expr2,
		Operator: operator.val,
	}
}

func (p *parser) parseFunctionCall() *ast.FunctionCallExpression {
	expr := &ast.FunctionCallExpression{}
	if p.current.typ != itemNonVariableIdentifier {
		p.expected(itemNonVariableIdentifier)
	}
	expr.FunctionName = p.current.val
	expr.Arguments = make([]ast.Expression, 0)
	p.expect(itemOpenParen)
	first := true
	p.next()
	for {
		if p.current.typ == itemCloseParen {
			break
		}
		if !first {
			p.backup()
			p.expect(itemArgumentSeparator)
			p.next()
		} else {
			first = false
		}
		arg := p.parseExpression()
		if arg == nil {
			break
		}
		expr.Arguments = append(expr.Arguments, arg)
		p.next()
	}
	return expr
}

func (p *parser) parseStmt() ast.Statement {
	switch p.current.typ {
	case itemBlockBegin:
		p.backup()
		return p.parseBlock()
	case itemIdentifier:
		switch p.peek().typ {
		case itemUnaryOperator:
			expr := ast.ExpressionStmt{p.parseExpression()}
			p.expect(itemStatementEnd)
			return expr
		default:
			n := ast.AssignmentStmt{}
			n.Assignee = p.parseIdentifier().(ast.Assignable)
			p.expect(itemAssignmentOperator)
			n.Operator = p.current.val
			p.next()
			n.Value = p.parseExpression()
			p.expect(itemStatementEnd)
			return n
		}
	case itemUnaryOperator:
		expr := ast.ExpressionStmt{p.parseExpression()}
		p.expect(itemStatementEnd)
		return expr
	case itemFunction:
		return p.parseFunctionStmt()
	case itemEcho:
		p.next()
		expr := p.parseExpression()
		p.expect(itemStatementEnd)
		return ast.Echo(expr)
	case itemIf:
		return p.parseIf()
	case itemWhile:
		return p.parseWhile()
	case itemDo:
		return p.parseDo()
	case itemFor:
		return p.parseFor()
	case itemNonVariableIdentifier:
		stmt := p.parseFunctionCall()
		p.expect(itemStatementEnd)
		return stmt
	case itemClass:
		return p.parseClass()
	case itemReturn:
		p.next()
		stmt := ast.ReturnStmt{Expression: p.parseExpression()}
		p.expect(itemStatementEnd)
		return stmt
	case itemThrow:
		stmt := ast.ThrowStmt{Expression: p.parseNextExpression()}
		p.expect(itemStatementEnd)
		return stmt
	case itemExit:
		stmt := ast.ExitStmt{}
		if p.peek().typ == itemOpenParen {
			p.expect(itemOpenParen)
			stmt.Expression = p.parseNextExpression()
			p.expect(itemCloseParen)
		}
		p.expect(itemStatementEnd)
		return stmt
	case itemInclude:
		stmt := ast.IncludeStmt{Expression: p.parseNextExpression()}
		p.expect(itemStatementEnd)
		return stmt
	case itemIgnoreErrorOperator:
		// Ignore this operator
		return p.parseStmt()
	default:
		p.errorf("Found %s, expected html or php begin", p.current)
		return nil
	}
}

func (p *parser) parseWhile() ast.Statement {
	p.expect(itemOpenParen)
	term := p.parseNextExpression()
	p.expect(itemCloseParen)
	block := p.parseBlock()
	return &ast.WhileStmt{
		Termination: term,
		LoopBlock:   block,
	}
}

func (p *parser) parseFor() ast.Statement {
	stmt := &ast.ForStmt{}
	p.expect(itemOpenParen)
	stmt.Initialization = p.parseNextExpression()
	p.expect(itemStatementEnd)
	stmt.Termination = p.parseNextExpression()
	p.expect(itemStatementEnd)
	stmt.Iteration = p.parseNextExpression()
	p.expect(itemCloseParen)
	stmt.LoopBlock = p.parseBlock()
	return stmt
}

func (p *parser) parseDo() ast.Statement {
	block := p.parseBlock()
	p.expect(itemWhile)
	p.expect(itemOpenParen)
	term := p.parseNextExpression()
	p.expect(itemCloseParen)
	p.expect(itemStatementEnd)
	return &ast.DoWhileStmt{
		Termination: term,
		LoopBlock:   block,
	}
}

func (p *parser) parseFunctionStmt() *ast.FunctionStmt {
	stmt := &ast.FunctionStmt{}
	p.expect(itemNonVariableIdentifier)
	stmt.Name = p.current.val
	stmt.Arguments = make([]ast.FunctionArgument, 0)
	p.expect(itemOpenParen)
	first := true
	for {
		p.next()
		if p.current.typ == itemCloseParen {
			break
		}
		p.backup()
		if !first {
			p.expect(itemArgumentSeparator)
		} else {
			first = false
		}
		p.next()
		arg := ast.FunctionArgument{}
		if p.current.typ == itemNonVariableIdentifier {
			arg.TypeHint = p.current.val
		} else {
			p.backup()
		}
		p.expect(itemIdentifier)
		arg.Identifier = ast.NewIdentifier(p.current.val)
		stmt.Arguments = append(stmt.Arguments, arg)
	}
	stmt.Body = p.parseBlock()
	return stmt
}

func (p *parser) parseBlock() ast.Block {
	block := ast.Block{}
	p.expect(itemBlockBegin)
	for {
		p.next()
		block.Statements = append(block.Statements, p.parseStmt())
		if p.next(); p.current.typ == itemBlockEnd {
			break
		}
		p.backup()
	}
	return block
}

func (p *parser) parseClass() ast.Class {
	p.expect(itemNonVariableIdentifier)
	name := p.current.val
	p.next()
	if p.current.typ == itemExtends {
		p.expect(itemNonVariableIdentifier)
	} else {
		p.backup()
	}
	p.expect(itemBlockBegin)
	return p.parseClassFields(ast.Class{Name: name})
}

func (p *parser) parseClassFields(c ast.Class) ast.Class {
	c.Methods = make([]ast.Method, 0)
	c.Properties = make([]ast.Property, 0)
	p.next()
	var vis ast.Visibility
	for p.current.typ != itemBlockEnd {
		switch p.current.typ {
		case itemPrivate:
			vis = ast.Private
		case itemProtected:
			vis = ast.Protected
		case itemPublic:
			vis = ast.Public
		default:
			vis = ast.Public
			p.backup()
		}
		p.next()
		switch p.current.typ {
		case itemFunction:
			c.Methods = append(c.Methods, ast.Method{
				Visibility:   vis,
				FunctionStmt: p.parseFunctionStmt(),
			})
		case itemIdentifier:
			c.Properties = append(c.Properties, ast.Property{
				Visibility: vis,
				Name:       p.current.val,
			})
		default:
		}
		p.next()
	}
	return c
}
<|endoftext|>


<commit_before>package ics

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"strings"
)

const (
	ianaTokenChars  = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-"
	invSafeChars    = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\",:;\x7f"
	invQSafeChars   = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\"\x7f"
	invValueChars   = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x7f"
	paramDelim      = ";"
	paramValueDelim = "="
	nameValueDelim  = ":"
)

type token struct {
	typ  tokenType
	data string
}

type tokenType uint8

const (
	TokenError tokenType = iota
	TokenName
	TokenParamName
	TokenParamValue
	TokenParamQValue
	TokenValue
)

type stateFn func() (token, stateFn)

type parser struct {
	br                 *bufio.Reader
	buf                bytes.Buffer
	state              stateFn
	pos, lineNo, colNo int
	lastWidth          int
	wasNewLine         bool
	err                error
}

func newParser(r io.Reader) *parser {
	p := &parser{
		br: bufio.NewReader(r),
	}
	p.state = p.parseName
	return p
}

func (p *parser) GetToken() (token, error) {
	var t token
	p.buf.Reset()
	t, p.state = p.state()
	if p.err == io.EOF {
		p.state = p.errorFn
		if t.typ == TokenError {
			p.err = io.ErrUnexpectedEOF
		} else {
			return t, nil
		}
	}
	return t, p.err
}

func (p *parser) next() rune {
	if p.err != nil {
		return -1
	}
	r, s, err := p.br.ReadRune()
	if err != nil {
		p.lastWidth = 0
		p.err = err
		return -1
	}
	p.buf.WriteRune(r)
	p.pos += s
	p.lastWidth = s
	if s > 0 {
		if r == '\n' {
			p.lineNo++
			p.wasNewLine = true
			p.colNo = 0
		} else {
			p.colNo++
			p.wasNewLine = false
		}
	}
	return r
}

func (p *parser) backup() {
	if p.lastWidth > 0 {
		p.pos -= p.lastWidth
		if p.wasNewLine {
			p.lineNo--
		}
		p.colNo--
		p.br.UnreadRune()
		p.buf.Truncate(p.buf.Len() - p.lastWidth)
		p.lastWidth = 0
	}
}

func (p *parser) accept(valid string) bool {
	if strings.ContainsRune(valid, p.next()) {
		return true
	}
	p.backup()
	return false
}

func (p *parser) acceptRun(valid string) {
	for {
		r := p.next()
		if r == -1 {
			return
		}
		if !strings.ContainsRune(valid, r) {
			p.backup()
			return
		}
	}
}

func (p *parser) except(invalid string) bool {
	r := p.next()
	if r == -1 {
		return false
	}
	if !strings.ContainsRune(invalid, r) {
		return true
	}
	p.backup()
	return false
}

func (p *parser) exceptRun(invalid string) {
	for {
		r := p.next()
		if r == -1 {
			return
		}
		if strings.ContainsRune(invalid, r) {
			p.backup()
			return
		}
	}
}

func (p *parser) parseName() (token, stateFn) {
	p.acceptRun(ianaTokenChars)
	t := token{
		TokenName,
		p.buf.String(),
	}
	if p.buf.Len() == 0 {
		p.err = ErrNoName
	} else if p.accept(paramDelim) {
		return t, p.parseParamName
	} else if p.accept(nameValueDelim) {
		return t, p.parseValue
	} else if p.err == nil {
		p.err = ErrInvalidChar
	}
	return p.errorFn()
}

func (p *parser) parseParamName() (token, stateFn) {
	p.acceptRun(ianaTokenChars)
	t := token{
		TokenName,
		p.buf.String(),
	}
	if p.buf.Len() == 0 {
		p.err = ErrNoParamName
	} else if p.accept(paramValueDelim) {
		return t, p.parseParamValue
	} else if p.err == nil {
		p.err = ErrInvalidChar
	}
	return p.errorFn()
}

func (p *parser) parseParamValue() (token, stateFn) {
	var t token
	if p.accept("\"") {
		p.exceptRun(invQSafeChars)
		if !p.accept("\"") {
			p.err = ErrInvalidChar
			return p.errorFn()
		}
		t.typ = TokenParamQValue
		t.data = p.buf.String()
	} else {
		p.exceptRun(invSafeChars)
		t.typ = TokenParamQValue
		t.data = p.buf.String()
	}
	if p.accept(paramDelim) {
		return t, p.parseParamName
	} else if p.accept(nameValueDelim) {
		return t, p.parseValue
	} else if p.err == nil {
		p.err = ErrInvalidChar
	}
	return p.errorFn()
}

func (p *parser) parseValue() (token, stateFn) {
	var toRet []byte
	for {
		p.exceptRun(invValueChars)
		if !p.accept("\r") || !p.accept("\n") {
			if p.err == nil {
				p.err = ErrInvalidChar
			}
			return p.errorFn()
		}
		toAdd := p.buf.Bytes()
		toRet = append(toRet, toAdd[:len(toAdd)-2]...)
		if !p.accept(" ") {
			break
		}
		p.buf.Reset()
	}
	return token{
		TokenValue,
		string(toRet),
	}, p.parseName
}

func (p *parser) errorFn() (token, stateFn) {
	return token{
		TokenError,
		p.err.Error(),
	}, p.errorFn
}

// Errors

var (
	ErrInvalidChar = errors.New("invalid character")
	ErrNoName      = errors.New("zero length name")
	ErrNoParamName = errors.New("zero length param name")
)
<commit_msg>corrected incorrect token types<commit_after>package ics

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"strings"
)

const (
	ianaTokenChars  = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-"
	invSafeChars    = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\",:;\x7f"
	invQSafeChars   = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\"\x7f"
	invValueChars   = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x7f"
	paramDelim      = ";"
	paramValueDelim = "="
	nameValueDelim  = ":"
)

type token struct {
	typ  tokenType
	data string
}

type tokenType uint8

const (
	TokenError tokenType = iota
	TokenName
	TokenParamName
	TokenParamValue
	TokenParamQValue
	TokenValue
)

type stateFn func() (token, stateFn)

type parser struct {
	br                 *bufio.Reader
	buf                bytes.Buffer
	state              stateFn
	pos, lineNo, colNo int
	lastWidth          int
	wasNewLine         bool
	err                error
}

func newParser(r io.Reader) *parser {
	p := &parser{
		br: bufio.NewReader(r),
	}
	p.state = p.parseName
	return p
}

func (p *parser) GetToken() (token, error) {
	var t token
	p.buf.Reset()
	t, p.state = p.state()
	if p.err == io.EOF {
		p.state = p.errorFn
		if t.typ == TokenError {
			p.err = io.ErrUnexpectedEOF
		} else {
			return t, nil
		}
	}
	return t, p.err
}

func (p *parser) next() rune {
	if p.err != nil {
		return -1
	}
	r, s, err := p.br.ReadRune()
	if err != nil {
		p.lastWidth = 0
		p.err = err
		return -1
	}
	p.buf.WriteRune(r)
	p.pos += s
	p.lastWidth = s
	if s > 0 {
		if r == '\n' {
			p.lineNo++
			p.wasNewLine = true
			p.colNo = 0
		} else {
			p.colNo++
			p.wasNewLine = false
		}
	}
	return r
}

func (p *parser) backup() {
	if p.lastWidth > 0 {
		p.pos -= p.lastWidth
		if p.wasNewLine {
			p.lineNo--
		}
		p.colNo--
		p.br.UnreadRune()
		p.buf.Truncate(p.buf.Len() - p.lastWidth)
		p.lastWidth = 0
	}
}

func (p *parser) accept(valid string) bool {
	if strings.ContainsRune(valid, p.next()) {
		return true
	}
	p.backup()
	return false
}

func (p *parser) acceptRun(valid string) {
	for {
		r := p.next()
		if r == -1 {
			return
		}
		if !strings.ContainsRune(valid, r) {
			p.backup()
			return
		}
	}
}

func (p *parser) except(invalid string) bool {
	r := p.next()
	if r == -1 {
		return false
	}
	if !strings.ContainsRune(invalid, r) {
		return true
	}
	p.backup()
	return false
}

func (p *parser) exceptRun(invalid string) {
	for {
		r := p.next()
		if r == -1 {
			return
		}
		if strings.ContainsRune(invalid, r) {
			p.backup()
			return
		}
	}
}

func (p *parser) parseName() (token, stateFn) {
	p.acceptRun(ianaTokenChars)
	t := token{
		TokenName,
		p.buf.String(),
	}
	if p.buf.Len() == 0 {
		p.err = ErrNoName
	} else if p.accept(paramDelim) {
		return t, p.parseParamName
	} else if p.accept(nameValueDelim) {
		return t, p.parseValue
	} else if p.err == nil {
		p.err = ErrInvalidChar
	}
	return p.errorFn()
}

func (p *parser) parseParamName() (token, stateFn) {
	p.acceptRun(ianaTokenChars)
	t := token{
		TokenParamName,
		p.buf.String(),
	}
	if p.buf.Len() == 0 {
		p.err = ErrNoParamName
	} else if p.accept(paramValueDelim) {
		return t, p.parseParamValue
	} else if p.err == nil {
		p.err = ErrInvalidChar
	}
	return p.errorFn()
}

func (p *parser) parseParamValue() (token, stateFn) {
	var t token
	if p.accept("\"") {
		p.exceptRun(invQSafeChars)
		if !p.accept("\"") {
			p.err = ErrInvalidChar
			return p.errorFn()
		}
		t.typ = TokenParamQValue
		t.data = p.buf.String()
	} else {
		p.exceptRun(invSafeChars)
		t.typ = TokenParamValue
		t.data = p.buf.String()
	}
	if p.accept(paramDelim) {
		return t, p.parseParamName
	} else if p.accept(nameValueDelim) {
		return t, p.parseValue
	} else if p.err == nil {
		p.err = ErrInvalidChar
	}
	return p.errorFn()
}

func (p *parser) parseValue() (token, stateFn) {
	var toRet []byte
	for {
		p.exceptRun(invValueChars)
		if !p.accept("\r") || !p.accept("\n") {
			if p.err == nil {
				p.err = ErrInvalidChar
			}
			return p.errorFn()
		}
		toAdd := p.buf.Bytes()
		toRet = append(toRet, toAdd[:len(toAdd)-2]...)
		if !p.accept(" ") {
			break
		}
		p.buf.Reset()
	}
	return token{
		TokenValue,
		string(toRet),
	}, p.parseName
}

func (p *parser) errorFn() (token, stateFn) {
	return token{
		TokenError,
		p.err.Error(),
	}, p.errorFn
}

// Errors

var (
	ErrInvalidChar = errors.New("invalid character")
	ErrNoName      = errors.New("zero length name")
	ErrNoParamName = errors.New("zero length param name")
)
<|endoftext|>


<commit_before>package zipcode

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"
)

type entryParser interface {
	// Parseは、cから値を受信したあと必要な加工を行い、c1へ送信する。
	Parse(c <-chan interface{}, c1 chan<- interface{})
}

type entryHandlerFunc func(entry *Entry) *Entry

func (f entryHandlerFunc) Parse(c <-chan interface{}, c1 chan<- interface{}) {
	defer close(c1)
	for v := range c {
		if err, ok := v.(error); ok {
			c1 <- err
			break
		}
		c1 <- f(v.(*Entry))
	}
}

var (
	// 行連結が未解決のままとなった場合のエラー
	incompleteEntry = errors.New("incomplete entry")
)

// entryが完結している場合はtrue、次のエントリと連結する場合はfalse
type entryCollectorFunc func(entry *Entry) bool

func (f entryCollectorFunc) Parse(c <-chan interface{}, c1 chan<- interface{}) {
	defer close(c1)
	for v := range c {
		if err, ok := v.(error); ok {
			c1 <- err
			return
		}
		entry := v.(*Entry)
		for !f(entry) {
			v, ok := <-c
			if !ok {
				c1 <- incompleteEntry
				return
			}
			if err, ok := v.(error); ok {
				c1 <- err
				return
			}
			entry1 := v.(*Entry)
			entry.Town = entry.Town.combine(entry1.Town)
		}
		c1 <- entry
	}
}

type entryExpanderFunc func(entry *Entry) []*Entry

func (f entryExpanderFunc) Parse(c <-chan interface{}, c1 chan<- interface{}) {
	textRule := cmplxRule{
		TokenBegin: '(',
		TokenEnd:   ')',
		Delim:      '、',
		Range:      '〜',
	}
	rubyRule := cmplxRule{
		TokenBegin: '(',
		TokenEnd:   ')',
		Delim:      '､',
		Range:      '-',
	}
	defer close(c1)
	for v := range c {
		if err, ok := v.(error); ok {
			c1 <- err
			return
		}
		entry := v.(*Entry)
		a1, err := textRule.Eval(entry.Town.Text)
		if err != nil {
			c1 <- err
			return
		}
		a2, err := rubyRule.Eval(entry.Town.Ruby)
		if err != nil {
			c1 <- err
			return
		}

		// Town.Textには複数書式を持つが、Town.Rubyには複数部分を省略しているケースがある。
		if len(a1) > 1 && len(a2) == 1 {
			a3 := make([]string, len(a1))
			for i := 0; i < len(a1); i++ {
				a3[i] = a2[0]
			}
			a2 = a3
		}
		if len(a1) != len(a2) {
			// TODO
		}
		for i, _ := range a1 {
			entry1 := new(Entry)
			*entry1 = *entry
			entry1.Town = Name{a1[i], a2[i]}
			c1 <- entry1
		}
	}
}

// cmplxRuleはKEN_ALL.CSVで複数の要素をまとめた書式をあらわす。
type cmplxRule struct {
	// 範囲書式の開始文字。たとえば"（"など。
	TokenBegin rune
	// 複数書式の終了文字。たとえば"）"など。
	TokenEnd rune
	// 複数書式で要素を分割する文字。たとえば"、"など。
	Delim rune
	// 複数書式で範囲をあらわす文字。たとえば"〜"など。
	Range rune
}

// Evalは（）の中の文字列を展開して、一連の文字列を配列で返す。
// "あああ（ほげ、ふが）" => ["あああほげ", "あああふが"]
// "1〜3" => ["1", "2", "3"]
func (rule cmplxRule) Eval(s string) ([]string, error) {
	t := []rune(s)
	for i := 0; i < len(t); i++ {
		switch t[i] {
		case rule.TokenBegin:
			expr, err := rule.Expr(t[i+1:])
			if err != nil {
				return nil, err
			}
			tokens, err := rule.Tokens(expr)
			if err != nil {
				return nil, err
			}
			a := make([]string, len(tokens))
			for j, token := range tokens {
				// "その他"だけは特別扱い
				if token == "その他" || token == "ｿﾉﾀ" {
					token = ""
				}
				a[j] = string(t[0:i]) + token + string(t[i+len(expr)+2:])
			}
			return a, nil
		case rule.TokenEnd:
			return nil, fmt.Errorf("missing %q", rule.TokenBegin)
		}
	}
	return []string{s}, nil
}

// Expr returns string inside rule.TokenBegin and rule.TokenEnd.
func (rule cmplxRule) Expr(s []rune) ([]rune, error) {
	for i := 0; i < len(s); i++ {
		switch s[i] {
		case rule.TokenBegin:
			return nil, fmt.Errorf("missing %q", rule.TokenEnd)
		case rule.TokenEnd:
			return s[0:i], nil
		}
	}
	return nil, fmt.Errorf("missing %q", rule.TokenEnd)
}

// Tokensはrule.Delimとrule.Rangeを評価して展開後の文字列配列を返す。
func (rule cmplxRule) Tokens(expr []rune) (tokens []string, err error) {
	stage1 := strings.FieldsFunc(string(expr), func(c rune) bool {
		return c == rule.Delim
	})

	for _, s := range stage1 {
		stage2, err := rule.Expand(s)
		if err != nil {
			return nil, err
		}
		for _, token := range stage2 {
			tokens = append(tokens, token)
		}
	}
	return
}

// Expandはrule.Rangeを展開して複数の文字列を返す。
func (rule cmplxRule) Expand(token string) (tokens []string, err error) {
	re := regexp.MustCompile(`(\d+)` + string(rule.Range) + `(\d+)`)
	m := re.FindStringSubmatchIndex(token)
	if m == nil {
		tokens = append(tokens, token)
		return
	}
	prefix := token[0:m[2]]
	bp, err := strconv.Atoi(token[m[2]:m[3]])
	if err != nil {
		return
	}
	ep, err := strconv.Atoi(token[m[4]:m[5]])
	if err != nil {
		return
	}
	suffix := token[m[5]:]
	for i := bp; i <= ep; i++ {
		tokens = append(tokens, fmt.Sprintf("%s%d%s", prefix, i, suffix))
	}
	return
}

var parserFilters = []entryParser{
	entryHandlerFunc(func(entry *Entry) *Entry {
		if entry.Town.Text == "以下に掲載がない場合" {
			entry.Notice = entry.Town.Text
			entry.Town.Text = ""
			entry.Town.Ruby = ""
		}
		return entry
	}),
	entryHandlerFunc(func(entry *Entry) *Entry {
		const (
			textSuffix = "（次のビルを除く）"
			rubySuffix = "(ﾂｷﾞﾉﾋﾞﾙｦﾉｿﾞｸ)"
		)
		if strings.HasSuffix(entry.Town.Text, textSuffix) {
			entry.Town.Text = entry.Town.Text[0 : len(entry.Town.Text)-len(textSuffix)]
			entry.Town.Ruby = entry.Town.Ruby[0 : len(entry.Town.Ruby)-len(rubySuffix)]
		}
		return entry
	}),
	entryHandlerFunc(func(entry *Entry) *Entry {
		const (
			garbText = "（高層棟）"
			garbRuby = "(ｺｳｿｳﾄｳ)"
		)
		entry.Town.Text = strings.Replace(entry.Town.Text, garbText, "", -1)
		entry.Town.Ruby = strings.Replace(entry.Town.Ruby, garbRuby, "", -1)
		return entry
	}),
	entryHandlerFunc(func(entry *Entry) *Entry {
		const (
			textSuffix = "の次に番地がくる場合"
			rubySuffix = "ﾉﾂｷﾞﾆﾊﾞﾝﾁｶﾞｸﾙﾊﾞｱｲ"
		)
		if strings.HasSuffix(entry.Town.Text, textSuffix) {
			entry.Notice = entry.Town.Text
			town := Name{
				Text: entry.Town.Text[0 : len(entry.Town.Text)-len(textSuffix)],
				Ruby: entry.Town.Ruby[0 : len(entry.Town.Ruby)-len(rubySuffix)],
			}
			if strings.HasSuffix(entry.Region.Text, town.Text) {
				entry.Town.Text = ""
				entry.Town.Ruby = ""
			} else {
				entry.Town.Text = town.Text
				entry.Town.Ruby = town.Ruby
			}
		}
		return entry
	}),
	entryHandlerFunc(func(entry *Entry) *Entry {
		const (
			textSuffix = "一円"
			rubySuffix = "ｲﾁｴﾝ"
		)
		if strings.HasSuffix(entry.Town.Text, textSuffix) {
			if entry.Town.Text != textSuffix {
				entry.Notice = entry.Town.Text
				entry.Town.Text = ""
				entry.Town.Ruby = ""
			}
		}
		return entry
	}),
	entryHandlerFunc(func(entry *Entry) *Entry {
		entry.Town.Text = normalizeText(entry.Town.Text)
		return entry
	}),
	entryCollectorFunc(func(entry *Entry) bool {
		open := strings.Count(entry.Town.Text, "(")
		close := strings.Count(entry.Town.Text, ")")
		return open == close
	}),
	entryExpanderFunc(func(entry *Entry) []*Entry {
		return []*Entry{entry}
	}),
}

type Parser struct {
	Error error
}

// Parseは郵便番号データを流すチャネルを返す。
// エラーが途中で発生した場合、チャネルはclosedになりparser.Errorにエラーをセットする。
func (parser *Parser) Parse(r io.Reader) <-chan *Entry {
	c1 := make(chan interface{})
	go readFromCSVLoop(r, c1)
	for _, parser := range parserFilters {
		c2 := make(chan interface{})
		go parser.Parse(c1, c2)
		c1 = c2
	}

	c := make(chan *Entry)
	go func() {
		defer close(c)
		for v := range c1 {
			if err, ok := v.(error); ok {
				parser.Error = err
				return
			}
			c <- v.(*Entry)
		}
	}()
	return c
}

// rからCSVデータを読み、cにエントリを送信する。
// エラーが発生した場合はecへエラーを送信する。
func readFromCSVLoop(r io.Reader, c chan<- interface{}) {
	defer close(c)
	fin := csv.NewReader(r)
	for {
		record, err := fin.Read()
		if err == io.EOF {
			return
		}
		if err != nil {
			c <- err
			return
		}

		isPartialTown, err := strconv.ParseBool(record[9])
		if err != nil {
			c <- err
			return
		}
		isLargeTown, err := strconv.ParseBool(record[10])
		if err != nil {
			c <- err
			return
		}
		isBlockedScheme, err := strconv.ParseBool(record[11])
		if err != nil {
			c <- err
			return
		}
		isOverlappedZip, err := strconv.ParseBool(record[12])
		if err != nil {
			c <- err
			return
		}
		status, err := parseStatus(record[13])
		if err != nil {
			c <- err
			return
		}
		reason, err := parseReason(record[14])
		if err != nil {
			c <- err
			return
		}
		c <- &Entry{
			Code:            record[0],
			OldZip:          record[1],
			Zip:             record[2],
			Pref:            Name{record[6], record[3]},
			Region:          Name{record[7], record[4]},
			Town:            Name{record[8], record[5]},
			IsPartialTown:   isPartialTown,
			IsLargeTown:     isLargeTown,
			IsBlockedScheme: isBlockedScheme,
			IsOverlappedZip: isOverlappedZip,
			Status:          status,
			Reason:          reason,
		}
	}
}
<commit_msg>fix out of index error related to Town Name Ruby.<commit_after>package zipcode

import (
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"
)

type entryParser interface {
	// Parseは、cから値を受信したあと必要な加工を行い、c1へ送信する。
	Parse(c <-chan interface{}, c1 chan<- interface{})
}

type entryHandlerFunc func(entry *Entry) *Entry

func (f entryHandlerFunc) Parse(c <-chan interface{}, c1 chan<- interface{}) {
	defer close(c1)
	for v := range c {
		if err, ok := v.(error); ok {
			c1 <- err
			break
		}
		c1 <- f(v.(*Entry))
	}
}

var (
	// 行連結が未解決のままとなった場合のエラー
	incompleteEntry = errors.New("incomplete entry")
)

// entryが完結している場合はtrue、次のエントリと連結する場合はfalse
type entryCollectorFunc func(entry *Entry) bool

func (f entryCollectorFunc) Parse(c <-chan interface{}, c1 chan<- interface{}) {
	defer close(c1)
	for v := range c {
		if err, ok := v.(error); ok {
			c1 <- err
			return
		}
		entry := v.(*Entry)
		for !f(entry) {
			v, ok := <-c
			if !ok {
				c1 <- incompleteEntry
				return
			}
			if err, ok := v.(error); ok {
				c1 <- err
				return
			}
			entry1 := v.(*Entry)
			entry.Town = entry.Town.combine(entry1.Town)
		}
		c1 <- entry
	}
}

type entryExpanderFunc func(entry *Entry) []*Entry

func (f entryExpanderFunc) Parse(c <-chan interface{}, c1 chan<- interface{}) {
	textRule := cmplxRule{
		TokenBegin: '(',
		TokenEnd:   ')',
		Delim:      '、',
		Range:      '〜',
	}
	rubyRule := cmplxRule{
		TokenBegin: '(',
		TokenEnd:   ')',
		Delim:      '､',
		Range:      '-',
	}
	defer close(c1)
	for v := range c {
		if err, ok := v.(error); ok {
			c1 <- err
			return
		}
		entry := v.(*Entry)
		a1, err := textRule.Eval(entry.Town.Text)
		if err != nil {
			c1 <- err
			return
		}
		a2, err := rubyRule.Eval(entry.Town.Ruby)
		if err != nil {
			c1 <- err
			return
		}

		// Town.Textには複数書式を持つが、Town.Rubyには複数部分を省略しているケースがある。
		len_a1 := len(a1)
		len_a2 := len(a2)
		if len_a1 > 1 && len_a2 == 1 {
			a3 := make([]string, len_a1)
			for i := 0; i < len_a1; i++ {
				a3[i] = a2[0]
			}
			a2 = a3
		}
		if len_a1 != len_a2 {
			// TODO
		}
		for i, _ := range a1 {
			entry1 := new(Entry)
			*entry1 = *entry
			entry1.Town = Name{Text: a1[i]}
			if len_a2-1 < i {
				continue
			}
			if len_a2 > 0 {
				entry1.Town.Ruby = a2[i]
			}
			c1 <- entry1
		}
	}
}

// cmplxRuleはKEN_ALL.CSVで複数の要素をまとめた書式をあらわす。
type cmplxRule struct {
	// 範囲書式の開始文字。たとえば"（"など。
	TokenBegin rune
	// 複数書式の終了文字。たとえば"）"など。
	TokenEnd rune
	// 複数書式で要素を分割する文字。たとえば"、"など。
	Delim rune
	// 複数書式で範囲をあらわす文字。たとえば"〜"など。
	Range rune
}

// Evalは（）の中の文字列を展開して、一連の文字列を配列で返す。
// "あああ（ほげ、ふが）" => ["あああほげ", "あああふが"]
// "1〜3" => ["1", "2", "3"]
func (rule cmplxRule) Eval(s string) ([]string, error) {
	t := []rune(s)
	for i := 0; i < len(t); i++ {
		switch t[i] {
		case rule.TokenBegin:
			expr, err := rule.Expr(t[i+1:])
			if err != nil {
				return nil, err
			}
			tokens, err := rule.Tokens(expr)
			if err != nil {
				return nil, err
			}
			a := make([]string, len(tokens))
			for j, token := range tokens {
				// "その他"だけは特別扱い
				if token == "その他" || token == "ｿﾉﾀ" {
					token = ""
				}
				a[j] = string(t[0:i]) + token + string(t[i+len(expr)+2:])
			}
			return a, nil
		case rule.TokenEnd:
			return nil, fmt.Errorf("missing %q", rule.TokenBegin)
		}
	}
	return []string{s}, nil
}

// Expr returns string inside rule.TokenBegin and rule.TokenEnd.
func (rule cmplxRule) Expr(s []rune) ([]rune, error) {
	for i := 0; i < len(s); i++ {
		switch s[i] {
		case rule.TokenBegin:
			return nil, fmt.Errorf("missing %q", rule.TokenEnd)
		case rule.TokenEnd:
			return s[0:i], nil
		}
	}
	return nil, fmt.Errorf("missing %q", rule.TokenEnd)
}

// Tokensはrule.Delimとrule.Rangeを評価して展開後の文字列配列を返す。
func (rule cmplxRule) Tokens(expr []rune) (tokens []string, err error) {
	stage1 := strings.FieldsFunc(string(expr), func(c rune) bool {
		return c == rule.Delim
	})

	for _, s := range stage1 {
		stage2, err := rule.Expand(s)
		if err != nil {
			return nil, err
		}
		for _, token := range stage2 {
			tokens = append(tokens, token)
		}
	}
	return
}

// Expandはrule.Rangeを展開して複数の文字列を返す。
func (rule cmplxRule) Expand(token string) (tokens []string, err error) {
	re := regexp.MustCompile(`(\d+)` + string(rule.Range) + `(\d+)`)
	m := re.FindStringSubmatchIndex(token)
	if m == nil {
		tokens = append(tokens, token)
		return
	}
	prefix := token[0:m[2]]
	bp, err := strconv.Atoi(token[m[2]:m[3]])
	if err != nil {
		return
	}
	ep, err := strconv.Atoi(token[m[4]:m[5]])
	if err != nil {
		return
	}
	suffix := token[m[5]:]
	for i := bp; i <= ep; i++ {
		tokens = append(tokens, fmt.Sprintf("%s%d%s", prefix, i, suffix))
	}
	return
}

var parserFilters = []entryParser{
	entryHandlerFunc(func(entry *Entry) *Entry {
		if entry.Town.Text == "以下に掲載がない場合" {
			entry.Notice = entry.Town.Text
			entry.Town.Text = ""
			entry.Town.Ruby = ""
		}
		return entry
	}),
	entryHandlerFunc(func(entry *Entry) *Entry {
		const (
			textSuffix = "（次のビルを除く）"
			rubySuffix = "(ﾂｷﾞﾉﾋﾞﾙｦﾉｿﾞｸ)"
		)
		if strings.HasSuffix(entry.Town.Text, textSuffix) {
			entry.Town.Text = entry.Town.Text[0 : len(entry.Town.Text)-len(textSuffix)]
			entry.Town.Ruby = entry.Town.Ruby[0 : len(entry.Town.Ruby)-len(rubySuffix)]
		}
		return entry
	}),
	entryHandlerFunc(func(entry *Entry) *Entry {
		const (
			garbText = "（高層棟）"
			garbRuby = "(ｺｳｿｳﾄｳ)"
		)
		entry.Town.Text = strings.Replace(entry.Town.Text, garbText, "", -1)
		entry.Town.Ruby = strings.Replace(entry.Town.Ruby, garbRuby, "", -1)
		return entry
	}),
	entryHandlerFunc(func(entry *Entry) *Entry {
		const (
			textSuffix = "の次に番地がくる場合"
			rubySuffix = "ﾉﾂｷﾞﾆﾊﾞﾝﾁｶﾞｸﾙﾊﾞｱｲ"
		)
		if strings.HasSuffix(entry.Town.Text, textSuffix) {
			entry.Notice = entry.Town.Text
			town := Name{
				Text: entry.Town.Text[0 : len(entry.Town.Text)-len(textSuffix)],
				Ruby: entry.Town.Ruby[0 : len(entry.Town.Ruby)-len(rubySuffix)],
			}
			if strings.HasSuffix(entry.Region.Text, town.Text) {
				entry.Town.Text = ""
				entry.Town.Ruby = ""
			} else {
				entry.Town.Text = town.Text
				entry.Town.Ruby = town.Ruby
			}
		}
		return entry
	}),
	entryHandlerFunc(func(entry *Entry) *Entry {
		const (
			textSuffix = "一円"
			rubySuffix = "ｲﾁｴﾝ"
		)
		if strings.HasSuffix(entry.Town.Text, textSuffix) {
			if entry.Town.Text != textSuffix {
				entry.Notice = entry.Town.Text
				entry.Town.Text = ""
				entry.Town.Ruby = ""
			}
		}
		return entry
	}),
	entryHandlerFunc(func(entry *Entry) *Entry {
		entry.Town.Text = normalizeText(entry.Town.Text)
		return entry
	}),
	entryCollectorFunc(func(entry *Entry) bool {
		open := strings.Count(entry.Town.Text, "(")
		close := strings.Count(entry.Town.Text, ")")
		return open == close
	}),
	entryExpanderFunc(func(entry *Entry) []*Entry {
		return []*Entry{entry}
	}),
}

type Parser struct {
	Error error
}

// Parseは郵便番号データを流すチャネルを返す。
// エラーが途中で発生した場合、チャネルはclosedになりparser.Errorにエラーをセットする。
func (parser *Parser) Parse(r io.Reader) <-chan *Entry {
	c1 := make(chan interface{})
	go readFromCSVLoop(r, c1)
	for _, parser := range parserFilters {
		c2 := make(chan interface{})
		go parser.Parse(c1, c2)
		c1 = c2
	}

	c := make(chan *Entry)
	go func() {
		defer close(c)
		for v := range c1 {
			if err, ok := v.(error); ok {
				parser.Error = err
				return
			}
			c <- v.(*Entry)
		}
	}()
	return c
}

// rからCSVデータを読み、cにエントリを送信する。
// エラーが発生した場合はecへエラーを送信する。
func readFromCSVLoop(r io.Reader, c chan<- interface{}) {
	defer close(c)
	fin := csv.NewReader(r)
	for {
		record, err := fin.Read()
		if err == io.EOF {
			return
		}
		if err != nil {
			c <- err
			return
		}

		isPartialTown, err := strconv.ParseBool(record[9])
		if err != nil {
			c <- err
			return
		}
		isLargeTown, err := strconv.ParseBool(record[10])
		if err != nil {
			c <- err
			return
		}
		isBlockedScheme, err := strconv.ParseBool(record[11])
		if err != nil {
			c <- err
			return
		}
		isOverlappedZip, err := strconv.ParseBool(record[12])
		if err != nil {
			c <- err
			return
		}
		status, err := parseStatus(record[13])
		if err != nil {
			c <- err
			return
		}
		reason, err := parseReason(record[14])
		if err != nil {
			c <- err
			return
		}
		c <- &Entry{
			Code:            record[0],
			OldZip:          record[1],
			Zip:             record[2],
			Pref:            Name{record[6], record[3]},
			Region:          Name{record[7], record[4]},
			Town:            Name{record[8], record[5]},
			IsPartialTown:   isPartialTown,
			IsLargeTown:     isLargeTown,
			IsBlockedScheme: isBlockedScheme,
			IsOverlappedZip: isOverlappedZip,
			Status:          status,
			Reason:          reason,
		}
	}
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package handlers

import (
	"encoding/base64"
	"fmt"
	"reflect"
	"strings"
	"sync"
	"time"

	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/core/adapters/udp"
	"github.com/TheThingsNetwork/ttn/semtech"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/pointer"
	"github.com/TheThingsNetwork/ttn/utils/stats"
	"github.com/brocaar/lorawan"
)

// Semtech implements the Semtech protocol and make a bridge between gateways and routers
type Semtech struct{}

// Handle implements the udp.Handler interface
func (s Semtech) Handle(conn chan<- udp.MsgUDP, packets chan<- udp.MsgReq, msg udp.MsgUDP) error {
	pkt := new(semtech.Packet)
	err := pkt.UnmarshalBinary(msg.Data)
	if err != nil {
		return errors.New(errors.Structural, err)
	}

	switch pkt.Identifier {
	case semtech.PULL_DATA: // PULL_DATA -> Respond to the recipient with an ACK
		stats.MarkMeter("semtech_adapter.pull_data")
		stats.MarkMeter(fmt.Sprintf("semtech_adapter.gateways.%X.pull_data", pkt.GatewayId))
		stats.SetString(fmt.Sprintf("semtech_adapter.gateways.%X.last_pull_data", pkt.GatewayId), "date", time.Now().UTC().Format(time.RFC3339))

		data, err := semtech.Packet{
			Version:    semtech.VERSION,
			Token:      pkt.Token,
			Identifier: semtech.PULL_ACK,
		}.MarshalBinary()
		if err != nil {
			return errors.New(errors.Structural, err)
		}
		conn <- udp.MsgUDP{
			Addr: msg.Addr,
			Data: data,
		}
	case semtech.PUSH_DATA: // PUSH_DATA -> Transfer all RXPK to the component
		stats.MarkMeter("semtech_adapter.push_data")
		stats.MarkMeter(fmt.Sprintf("semtech_adapter.gateways.%X.push_data", pkt.GatewayId))
		stats.SetString(fmt.Sprintf("semtech_adapter.gateways.%X.last_push_data", pkt.GatewayId), "date", time.Now().UTC().Format(time.RFC3339))

		data, err := semtech.Packet{
			Version:    semtech.VERSION,
			Token:      pkt.Token,
			Identifier: semtech.PUSH_ACK,
		}.MarshalBinary()
		if err != nil {
			return errors.New(errors.Structural, err)
		}
		conn <- udp.MsgUDP{
			Addr: msg.Addr,
			Data: data,
		}

		if pkt.Payload == nil {
			return errors.New(errors.Structural, "Unable to process empty PUSH_DATA payload")
		}

		// Handle stat payload
		if pkt.Payload.Stat != nil {
			spacket, err := core.NewSPacket(pkt.GatewayId, extractMetadata(*pkt.Payload.Stat))
			if err == nil {
				data, err := spacket.MarshalBinary()
				if err == nil {
					go func() {
						packets <- udp.MsgReq{Data: data, Chresp: nil}
					}()
				}
			}
		}

		// Handle rxpks payloads
		cherr := make(chan error, len(pkt.Payload.RXPK))
		wait := sync.WaitGroup{}
		wait.Add(len(pkt.Payload.RXPK))
		for _, rxpk := range pkt.Payload.RXPK {
			go func(rxpk semtech.RXPK) {
				defer wait.Done()
				pktOut, err := rxpk2packet(rxpk, pkt.GatewayId)
				if err != nil {
					cherr <- errors.New(errors.Structural, err)
					return
				}
				data, err := pktOut.MarshalBinary()
				if err != nil {
					cherr <- errors.New(errors.Structural, err)
					return
				}
				chresp := make(chan udp.MsgRes)
				packets <- udp.MsgReq{Data: data, Chresp: chresp}
				select {
				case resp := <-chresp:
					itf, err := core.UnmarshalPacket(resp)
					if err != nil {
						cherr <- errors.New(errors.Structural, err)
						return
					}
					pkt, ok := itf.(core.RPacket) // NOTE Here we'll handle join-accept
					if !ok {
						cherr <- errors.New(errors.Structural, "Unhandled packet type")
						return
					}
					txpk, err := packet2txpk(pkt)
					if err != nil {
						cherr <- errors.New(errors.Structural, err)
						return
					}

					data, err := semtech.Packet{
						Version:    semtech.VERSION,
						Identifier: semtech.PULL_RESP,
						Payload:    &semtech.Payload{TXPK: &txpk},
					}.MarshalBinary()
					if err != nil {
						cherr <- errors.New(errors.Structural, err)
						return
					}
					conn <- udp.MsgUDP{Addr: msg.Addr, Data: data}
				case <-time.After(time.Second * 2):
				}
			}(rxpk)
		}
		wait.Wait()
		close(cherr)
		if err := <-cherr; err != nil {
			return err
		}
	default:
		return errors.New(errors.Implementation, "Unhandled packet type")
	}
	return nil
}

func rxpk2packet(p semtech.RXPK, gid []byte) (core.Packet, error) {
	// First, we have to get the physical payload which is encoded in the Data field
	if p.Data == nil {
		return nil, errors.New(errors.Structural, "There's no data in the packet")
	}

	// RXPK Data are base64 encoded, yet without the trailing "==" if any.....
	encoded := *p.Data
	switch len(encoded) % 4 {
	case 2:
		encoded += "=="
	case 3:
		encoded += "="
	}

	raw, err := base64.StdEncoding.DecodeString(encoded)
	if err != nil {
		return nil, errors.New(errors.Structural, err)
	}

	payload := lorawan.NewPHYPayload(true)
	if err = payload.UnmarshalBinary(raw); err != nil {
		return nil, errors.New(errors.Structural, err)
	}
	// At the end, our converted packet hold the same metadata than the RXPK packet but the Data
	// which as been completely transformed into a lorawan Physical Payload.
	return core.NewRPacket(payload, gid, extractMetadata(p))
}

func packet2txpk(p core.RPacket) (semtech.TXPK, error) {
	// Step 1, convert the physical payload to a base64 string (without the padding)
	raw, err := p.Payload().MarshalBinary()
	if err != nil {
		return semtech.TXPK{}, errors.New(errors.Structural, err)
	}

	data := strings.Trim(base64.StdEncoding.EncodeToString(raw), "=")
	txpk := semtech.TXPK{Data: pointer.String(data)}

	// Step 2, copy every compatible metadata from the packet to the TXPK packet.
	// We are possibly loosing information here.
	injectMetadata(&txpk, p.Metadata())

	return txpk, nil
}

func injectMetadata(ptr interface{}, metadata core.Metadata) {
	v := reflect.ValueOf(metadata)
	t := v.Type()
	d := reflect.ValueOf(ptr).Elem()

	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i).Name
		if d.FieldByName(field).CanSet() {
			d.FieldByName(field).Set(v.Field(i))
		}
	}
}

func extractMetadata(xpk interface{}) core.Metadata {
	metadata := core.Metadata{}
	v := reflect.ValueOf(xpk)
	t := v.Type()
	m := reflect.ValueOf(&metadata).Elem()

	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i).Name
		if m.FieldByName(field).CanSet() {
			m.FieldByName(field).Set(v.Field(i))
		}
	}

	return metadata
}
<commit_msg>[fixes] Remove error from udp semtech when no downlink is sent<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package handlers

import (
	"encoding/base64"
	"fmt"
	"reflect"
	"strings"
	"sync"
	"time"

	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/core/adapters/udp"
	"github.com/TheThingsNetwork/ttn/semtech"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/pointer"
	"github.com/TheThingsNetwork/ttn/utils/stats"
	"github.com/brocaar/lorawan"
)

// Semtech implements the Semtech protocol and make a bridge between gateways and routers
type Semtech struct{}

// Handle implements the udp.Handler interface
func (s Semtech) Handle(conn chan<- udp.MsgUDP, packets chan<- udp.MsgReq, msg udp.MsgUDP) error {
	pkt := new(semtech.Packet)
	err := pkt.UnmarshalBinary(msg.Data)
	if err != nil {
		return errors.New(errors.Structural, err)
	}

	switch pkt.Identifier {
	case semtech.PULL_DATA: // PULL_DATA -> Respond to the recipient with an ACK
		stats.MarkMeter("semtech_adapter.pull_data")
		stats.MarkMeter(fmt.Sprintf("semtech_adapter.gateways.%X.pull_data", pkt.GatewayId))
		stats.SetString(fmt.Sprintf("semtech_adapter.gateways.%X.last_pull_data", pkt.GatewayId), "date", time.Now().UTC().Format(time.RFC3339))

		data, err := semtech.Packet{
			Version:    semtech.VERSION,
			Token:      pkt.Token,
			Identifier: semtech.PULL_ACK,
		}.MarshalBinary()
		if err != nil {
			return errors.New(errors.Structural, err)
		}
		conn <- udp.MsgUDP{
			Addr: msg.Addr,
			Data: data,
		}
	case semtech.PUSH_DATA: // PUSH_DATA -> Transfer all RXPK to the component
		stats.MarkMeter("semtech_adapter.push_data")
		stats.MarkMeter(fmt.Sprintf("semtech_adapter.gateways.%X.push_data", pkt.GatewayId))
		stats.SetString(fmt.Sprintf("semtech_adapter.gateways.%X.last_push_data", pkt.GatewayId), "date", time.Now().UTC().Format(time.RFC3339))

		data, err := semtech.Packet{
			Version:    semtech.VERSION,
			Token:      pkt.Token,
			Identifier: semtech.PUSH_ACK,
		}.MarshalBinary()
		if err != nil {
			return errors.New(errors.Structural, err)
		}
		conn <- udp.MsgUDP{
			Addr: msg.Addr,
			Data: data,
		}

		if pkt.Payload == nil {
			return errors.New(errors.Structural, "Unable to process empty PUSH_DATA payload")
		}

		// Handle stat payload
		if pkt.Payload.Stat != nil {
			spacket, err := core.NewSPacket(pkt.GatewayId, extractMetadata(*pkt.Payload.Stat))
			if err == nil {
				data, err := spacket.MarshalBinary()
				if err == nil {
					go func() {
						packets <- udp.MsgReq{Data: data, Chresp: nil}
					}()
				}
			}
		}

		// Handle rxpks payloads
		cherr := make(chan error, len(pkt.Payload.RXPK))
		wait := sync.WaitGroup{}
		wait.Add(len(pkt.Payload.RXPK))
		for _, rxpk := range pkt.Payload.RXPK {
			go func(rxpk semtech.RXPK) {
				defer wait.Done()
				pktOut, err := rxpk2packet(rxpk, pkt.GatewayId)
				if err != nil {
					cherr <- errors.New(errors.Structural, err)
					return
				}
				data, err := pktOut.MarshalBinary()
				if err != nil {
					cherr <- errors.New(errors.Structural, err)
					return
				}
				chresp := make(chan udp.MsgRes)
				packets <- udp.MsgReq{Data: data, Chresp: chresp}
				select {
				case resp, ok := <-chresp:
					if !ok {
						// No response
						return
					}

					itf, err := core.UnmarshalPacket(resp)
					if err != nil {
						cherr <- errors.New(errors.Structural, err)
						return
					}
					pkt, ok := itf.(core.RPacket) // NOTE Here we'll handle join-accept
					if !ok {
						cherr <- errors.New(errors.Structural, "Unhandled packet type")
						return
					}
					txpk, err := packet2txpk(pkt)
					if err != nil {
						cherr <- errors.New(errors.Structural, err)
						return
					}

					data, err := semtech.Packet{
						Version:    semtech.VERSION,
						Identifier: semtech.PULL_RESP,
						Payload:    &semtech.Payload{TXPK: &txpk},
					}.MarshalBinary()
					if err != nil {
						cherr <- errors.New(errors.Structural, err)
						return
					}
					conn <- udp.MsgUDP{Addr: msg.Addr, Data: data}
				case <-time.After(time.Second * 2):
				}
			}(rxpk)
		}
		wait.Wait()
		close(cherr)
		if err := <-cherr; err != nil {
			return err
		}
	default:
		return errors.New(errors.Implementation, "Unhandled packet type")
	}
	return nil
}

func rxpk2packet(p semtech.RXPK, gid []byte) (core.Packet, error) {
	// First, we have to get the physical payload which is encoded in the Data field
	if p.Data == nil {
		return nil, errors.New(errors.Structural, "There's no data in the packet")
	}

	// RXPK Data are base64 encoded, yet without the trailing "==" if any.....
	encoded := *p.Data
	switch len(encoded) % 4 {
	case 2:
		encoded += "=="
	case 3:
		encoded += "="
	}

	raw, err := base64.StdEncoding.DecodeString(encoded)
	if err != nil {
		return nil, errors.New(errors.Structural, err)
	}

	payload := lorawan.NewPHYPayload(true)
	if err = payload.UnmarshalBinary(raw); err != nil {
		return nil, errors.New(errors.Structural, err)
	}
	// At the end, our converted packet hold the same metadata than the RXPK packet but the Data
	// which as been completely transformed into a lorawan Physical Payload.
	return core.NewRPacket(payload, gid, extractMetadata(p))
}

func packet2txpk(p core.RPacket) (semtech.TXPK, error) {
	// Step 1, convert the physical payload to a base64 string (without the padding)
	raw, err := p.Payload().MarshalBinary()
	if err != nil {
		return semtech.TXPK{}, errors.New(errors.Structural, err)
	}

	data := strings.Trim(base64.StdEncoding.EncodeToString(raw), "=")
	txpk := semtech.TXPK{Data: pointer.String(data)}

	// Step 2, copy every compatible metadata from the packet to the TXPK packet.
	// We are possibly loosing information here.
	injectMetadata(&txpk, p.Metadata())

	return txpk, nil
}

func injectMetadata(ptr interface{}, metadata core.Metadata) {
	v := reflect.ValueOf(metadata)
	t := v.Type()
	d := reflect.ValueOf(ptr).Elem()

	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i).Name
		if d.FieldByName(field).CanSet() {
			d.FieldByName(field).Set(v.Field(i))
		}
	}
}

func extractMetadata(xpk interface{}) core.Metadata {
	metadata := core.Metadata{}
	v := reflect.ValueOf(xpk)
	t := v.Type()
	m := reflect.ValueOf(&metadata).Elem()

	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i).Name
		if m.FieldByName(field).CanSet() {
			m.FieldByName(field).Set(v.Field(i))
		}
	}

	return metadata
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/ejcx/passgo/edit"
	"github.com/ejcx/passgo/generate"
	"github.com/ejcx/passgo/initialize"
	"github.com/ejcx/passgo/insert"
	"github.com/ejcx/passgo/pc"
	"github.com/ejcx/passgo/pio"
	"github.com/ejcx/passgo/show"
	"github.com/ejcx/passgo/sync"
)

const (
	ALLARGS = -1
)

var (
	// copyPass indicates that the shown password should be copied to the clipboard.
	copyPass = flag.Bool("copy", false, "If true, copy password to clipboard instead of displaying it")

	version = `======================================
= passgo: v1.0                       =
= The simple golang password and     =
= file manager                       =
=                                    =
= Twiinsen Security                  =
= evan@twiinsen.com                  =
= https://twiinsen.com/passgo        =
======================================`
	usage = `Usage:
	passgo
		Print the contents of the vault.
	passgo show site-path
		Print the password of a passgo entry.
	passgo init
		Initialize the .passgo directory, and generate your secret keys.
	passgo insert site-path
		Add a site to your password store. This site can optionally be a part
		of a group by prepending a group name and slash to the site name.
		Will prompt for confirmation when a site path is not unique.
		passgo
	passgo rename site-path
		Rename an entry in the password vault.
	passgo edit site-path
		Change the password of a site in the vault.
	passgo generate length=24
		Prints a randomly generated password. The length of this password defaults
		to 24. If a password length is specified as greater than 2048 then generate
    will fail.
	passgo find site-path
		Prints all sites that contain the site-path. Used to print just one group
		or all sites that contain a certain word in the group or name.
	passgo ls site-path
		An alias for the find subcommand.
	passgo remove site-path
		Remove a site from the password vault by specifying the entire site-path.
	passgo removefile site-path
		Remove a file from the vault by specifying the entire file-path.
	passgo rm site-path
		An alias for remove.
	passgo rmfile site-path
		An alias for removefile.
	passgo pull
		Pull will perform a git pull and sync the changes in the remote git
		repository with your local repo.
	passgo push
		Push will perform a git push to sync your changes with your remote
		git repository.
	passgo remote remote-url
		Remote is used to set the remote repository url. This is the repository
		that your sites will be pushed to and pulled from.
	passgo clone remote-url
		Clone will copy the remote url in to the .passgo directory in your
		home directory. It will fail if the directory already exists.
	passgo integrity
		Update the integrity hash of your password store if you are planning
		to manually push to the server.
	passgo usage
		Print this message!
	passgo version
		Print version information
`
)

func main() {
	// Check to see if this user is under attack.
	pio.CheckAttackFile()

	flag.Parse()

	// Default behavior of just running the command is listing all sites.
	if len(flag.Args()) < 1 {
		show.ListAll()
		return
	}

	// subArgs is used by subcommands to retreive only their args.
	subArgs := flag.Args()[1:]
	switch flag.Args()[0] {
	case "edit":
		path := getSubArguments(subArgs, ALLARGS)
		edit.Edit(path)
	case "ls", "find":
		path := getSubArguments(subArgs, ALLARGS)
		show.Find(path)
	case "generate":
		pwlenStr := getSubArguments(subArgs, 0)
		pwlen, err := strconv.Atoi(pwlenStr)
		if err != nil {
			pwlen = -1
		}
		pass := generate.Generate(pwlen)
		fmt.Println(pass)
	case "init":
		initialize.Init()
	case "insert":
		pathName := getSubArguments(subArgs, ALLARGS)
		insert.Password(pathName)
	case "integrity":
		pc.GetSitesIntegrity()
		sync.Commit(sync.IntegrityCommit)
	case "rm", "remove":
		path := getSubArguments(subArgs, ALLARGS)
		edit.RemovePassword(path)
	case "rename":
		path := getSubArguments(subArgs, ALLARGS)
		edit.Rename(path)
	case "help", "usage":
		printUsage()
	case "version":
		printVersion()
	case "pull":
		sync.Pull()
	case "push":
		sync.Push()
	case "remote":
		remote := getSubArguments(subArgs, 0)
		sync.Remote(remote)
	case "clone":
		repo := getSubArguments(subArgs, 0)
		sync.Clone(repo)
	case "show":
		path := getSubArguments(flag.Args(), 1)
		show.Site(path, *copyPass)
	case "insertfile":
		allArgs := getSubArguments(subArgs, ALLARGS)
		argList := strings.Split(allArgs, " ")
		if len(argList) != 2 {
			printUsage()
			log.Fatalln("Incorrect args.")
		}
		path := argList[0]
		filename := argList[1]
		insert.File(path, filename)
	case "rmfile", "removefile":
		path := getSubArguments(subArgs, ALLARGS)
		edit.RemoveFile(path)
	default:
		log.Fatalf("%s\nInvalid Command %s", usage, os.Args[1])
	}
}

func printUsage() {
	fmt.Println(usage)
}
func printVersion() {
	fmt.Println(version)
}

// getSubArguments requires the list of subarguments and the
// argument number that you want returned. Non existent args
// will return an empty string. A negative arg index will
// return all arguments concatenated as one.
func getSubArguments(args []string, arg int) string {
	if len(args) == 0 {
		return ""
	}
	if arg < 0 {
		return strings.Join(args, " ")
	}
	if len(args) < arg+1 {
		log.Fatalf("Not enough args")
	}
	return args[arg]
}
<commit_msg>Begin v2<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/ejcx/passgo/edit"
	"github.com/ejcx/passgo/generate"
	"github.com/ejcx/passgo/initialize"
	"github.com/ejcx/passgo/insert"
	"github.com/ejcx/passgo/pc"
	"github.com/ejcx/passgo/pio"
	"github.com/ejcx/passgo/show"
	"github.com/ejcx/passgo/sync"
	"github.com/spf13/cobra"
)

const (
	ALLARGS = -1
	version = `v2.0`
)

var (
	RootCmd = &cobra.Command{
		Use:   "passgo",
		Short: "Print the contents of the vault.",
		Run: func(cmd *cobra.Command, args []string) {
			show.ListAll()
		},
	}
	versionCmd = &cobra.Command{
		Use:   "version",
		Short: "Print the version of your passgo binary.",
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Println(version)
		},
	}
	// copyPass indicates that the shown password should be copied to the clipboard.
	copyPass = flag.Bool("copy", false, "If true, copy password to clipboard instead of displaying it")

	usage = `Usage:
	passgo
		Print the contents of the vault.
	passgo show site-path
		Print the password of a passgo entry.
	passgo init
		Initialize the .passgo directory, and generate your secret keys.
	passgo insert site-path
		Add a site to your password store. This site can optionally be a part
		of a group by prepending a group name and slash to the site name.
		Will prompt for confirmation when a site path is not unique.
		passgo
	passgo rename site-path
		Rename an entry in the password vault.
	passgo edit site-path
		Change the password of a site in the vault.
	passgo generate length=24
		Prints a randomly generated password. The length of this password defaults
		to 24. If a password length is specified as greater than 2048 then generate
    will fail.
	passgo find site-path
		Prints all sites that contain the site-path. Used to print just one group
		or all sites that contain a certain word in the group or name.
	passgo ls site-path
		An alias for the find subcommand.
	passgo remove site-path
		Remove a site from the password vault by specifying the entire site-path.
	passgo removefile site-path
		Remove a file from the vault by specifying the entire file-path.
	passgo rm site-path
		An alias for remove.
	passgo rmfile site-path
		An alias for removefile.
	passgo pull
		Pull will perform a git pull and sync the changes in the remote git
		repository with your local repo.
	passgo integrity
		Update the integrity hash of your password store if you are planning
		to manually push to the server.
	passgo usage
		Print this message!
`
)

func init() {
	RootCmd.AddCommand(versionCmd)
}
func main() {
	RootCmd.Execute()
}

func run() {
	// Check to see if this user is under attack.
	pio.CheckAttackFile()

	flag.Parse()

	// Default behavior of just running the command is listing all sites.
	if len(flag.Args()) < 1 {
		show.ListAll()
		return
	}

	// subArgs is used by subcommands to retreive only their args.
	subArgs := flag.Args()[1:]
	switch flag.Args()[0] {
	case "edit":
		path := getSubArguments(subArgs, ALLARGS)
		edit.Edit(path)
	case "ls", "find":
		path := getSubArguments(subArgs, ALLARGS)
		show.Find(path)
	case "generate":
		pwlenStr := getSubArguments(subArgs, 0)
		pwlen, err := strconv.Atoi(pwlenStr)
		if err != nil {
			pwlen = -1
		}
		pass := generate.Generate(pwlen)
		fmt.Println(pass)
	case "init":
		initialize.Init()
	case "insert":
		pathName := getSubArguments(subArgs, ALLARGS)
		insert.Password(pathName)
	case "integrity":
		pc.GetSitesIntegrity()
		sync.Commit(sync.IntegrityCommit)
	case "rm", "remove":
		path := getSubArguments(subArgs, ALLARGS)
		edit.RemovePassword(path)
	case "rename":
		path := getSubArguments(subArgs, ALLARGS)
		edit.Rename(path)
	case "pull":
		sync.Pull()
	case "push":
		sync.Push()
	case "remote":
		remote := getSubArguments(subArgs, 0)
		sync.Remote(remote)
	case "clone":
		repo := getSubArguments(subArgs, 0)
		sync.Clone(repo)
	case "show":
		path := getSubArguments(flag.Args(), 1)
		show.Site(path, *copyPass)
	case "insertfile":
		allArgs := getSubArguments(subArgs, ALLARGS)
		argList := strings.Split(allArgs, " ")
		if len(argList) != 2 {
			log.Fatalln("Incorrect args.")
		}
		path := argList[0]
		filename := argList[1]
		insert.File(path, filename)
	case "rmfile", "removefile":
		path := getSubArguments(subArgs, ALLARGS)
		edit.RemoveFile(path)
	}
}

// getSubArguments requires the list of subarguments and the
// argument number that you want returned. Non existent args
// will return an empty string. A negative arg index will
// return all arguments concatenated as one.
func getSubArguments(args []string, arg int) string {
	if len(args) == 0 {
		return ""
	}
	if arg < 0 {
		return strings.Join(args, " ")
	}
	if len(args) < arg+1 {
		log.Fatalf("Not enough args")
	}
	return args[arg]
}
<|endoftext|>


<commit_before>package piglow

import (
	"bitbucket.org/gmcbay/i2c"
)

var i2cbus *i2c.I2CBus

const (
	i2c_bus_num byte = 1
	i2c_addr    byte = 0x54 // fixed i2c address of SN3218 ic

	CMD_ENABLE_OUTPUT  byte = 0x00
	CMD_ENABLE_LEDS    byte = 0x13
	CMD_SET_PWM_VALUES byte = 0x01
	CMD_UPDATE         byte = 0x16
)

var white_leds = [3]int8{12, 9, 10}
var blue_leds = [3]int8{14, 4, 11}
var green_leds = [3]int8{3, 5, 13}
var yellow_leds = [3]int8{2, 8, 15}
var orange_leds = [3]int8{1, 7, 16}
var red_leds = [3]int8{0, 6, 17}

type Piglow struct {
	values [18]byte
}

func NewPiglow() (piglow *Piglow, err error) {
	piglow = new(Piglow)
	piglow.values = [18]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}

	i2cbus, err = i2c.Bus(i2c_bus_num)
	if err != nil {
		return nil, err
	}

	write_err := i2cbus.WriteByte(i2c_addr, CMD_ENABLE_OUTPUT, 0x01)
	if write_err != nil {
		return nil, write_err
	}

	enable_err := i2cbus.WriteByteBlock(i2c_addr, CMD_ENABLE_LEDS, []byte{0xFF, 0xFF, 0xFF})
	if enable_err != nil {
		return nil, enable_err
	}

	return
}

func (p *Piglow) Apply() (err error) {
	err = i2cbus.WriteByteBlock(i2c_addr, CMD_SET_PWM_VALUES, p.values[0:18])
	if err != nil {
		return
	}

	// Tell the i2c device to apply the changes
	err = i2cbus.WriteByte(i2c_addr, CMD_UPDATE, 0xFF)
	if err != nil {
		return
	}

	return err
}

// Set LED n to brightness
// n must be 0 through 17
// brightness must be 0 through 255
func (p *Piglow) SetLED(n int8, brightness uint8) {
	p.values[n] = brightness
}

// Set all LEDs to brightness
func (p *Piglow) SetAll(brightness uint8) {
	for i := 0; i < 18; i++ {
		p.values[i] = brightness
	}
}

/*

	Set colour rings to the given brightness

*/

func (p *Piglow) SetWhite(brightness uint8) {
	for i := 0; i < 3; i++ {
		p.values[white_leds[i]] = brightness
	}
}

func (p *Piglow) SetBlue(brightness uint8) {
	for i := 0; i < 3; i++ {
		p.values[blue_leds[i]] = brightness
	}
}

func (p *Piglow) SetGreen(brightness uint8) {
	for i := 0; i < 3; i++ {
		p.values[green_leds[i]] = brightness
	}
}

func (p *Piglow) SetYellow(brightness uint8) {
	for i := 0; i < 3; i++ {
		p.values[yellow_leds[i]] = brightness
	}
}

func (p *Piglow) SetOrange(brightness uint8) {
	for i := 0; i < 3; i++ {
		p.values[orange_leds[i]] = brightness
	}
}

func (p *Piglow) SetRed(brightness uint8) {
	for i := 0; i < 3; i++ {
		p.values[red_leds[i]] = brightness
	}
}
<commit_msg>Allow a whole tentacle to be set to a brightness, or a value to be displayed along one tentacle<commit_after>package piglow

import (
	"bitbucket.org/gmcbay/i2c"
	"math"
)

var i2cbus *i2c.I2CBus

const (
	i2c_bus_num byte = 1
	i2c_addr    byte = 0x54 // fixed i2c address of SN3218 ic

	CMD_ENABLE_OUTPUT  byte = 0x00
	CMD_ENABLE_LEDS    byte = 0x13
	CMD_SET_PWM_VALUES byte = 0x01
	CMD_UPDATE         byte = 0x16
)

var white_leds = [3]int8{12, 9, 10}
var blue_leds = [3]int8{14, 4, 11}
var green_leds = [3]int8{3, 5, 13}
var yellow_leds = [3]int8{2, 8, 15}
var orange_leds = [3]int8{1, 7, 16}
var red_leds = [3]int8{0, 6, 17}

var tentacle_0_leds = [6]int8{12, 14, 3, 2, 1, 0}
var tentacle_1_leds = [6]int8{9, 4, 5, 8, 7, 6}
var tentacle_2_leds = [6]int8{10, 11, 13, 15, 16, 17}

type Piglow struct {
	values [18]byte
}

func NewPiglow() (piglow *Piglow, err error) {
	piglow = new(Piglow)
	piglow.values = [18]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}

	i2cbus, err = i2c.Bus(i2c_bus_num)
	if err != nil {
		return nil, err
	}

	write_err := i2cbus.WriteByte(i2c_addr, CMD_ENABLE_OUTPUT, 0x01)
	if write_err != nil {
		return nil, write_err
	}

	enable_err := i2cbus.WriteByteBlock(i2c_addr, CMD_ENABLE_LEDS, []byte{0xFF, 0xFF, 0xFF})
	if enable_err != nil {
		return nil, enable_err
	}

	return
}

func (p *Piglow) Apply() (err error) {
	err = i2cbus.WriteByteBlock(i2c_addr, CMD_SET_PWM_VALUES, p.values[0:18])
	if err != nil {
		return
	}

	// Tell the i2c device to apply the changes
	err = i2cbus.WriteByte(i2c_addr, CMD_UPDATE, 0xFF)
	if err != nil {
		return
	}

	return err
}

// Set LED n to brightness
// n must be 0 through 17
// brightness must be 0 through 255
func (p *Piglow) SetLED(n int8, brightness uint8) {
	p.values[n] = brightness
}

// Set all LEDs to brightness
func (p *Piglow) SetAll(brightness uint8) {
	for i := 0; i < 18; i++ {
		p.values[i] = brightness
	}
}

/*

	Set colour rings to the given brightness

*/

func (p *Piglow) SetWhite(brightness uint8) {
	for i := 0; i < 3; i++ {
		p.values[white_leds[i]] = brightness
	}
}

func (p *Piglow) SetBlue(brightness uint8) {
	for i := 0; i < 3; i++ {
		p.values[blue_leds[i]] = brightness
	}
}

func (p *Piglow) SetGreen(brightness uint8) {
	for i := 0; i < 3; i++ {
		p.values[green_leds[i]] = brightness
	}
}

func (p *Piglow) SetYellow(brightness uint8) {
	for i := 0; i < 3; i++ {
		p.values[yellow_leds[i]] = brightness
	}
}

func (p *Piglow) SetOrange(brightness uint8) {
	for i := 0; i < 3; i++ {
		p.values[orange_leds[i]] = brightness
	}
}

func (p *Piglow) SetRed(brightness uint8) {
	for i := 0; i < 3; i++ {
		p.values[red_leds[i]] = brightness
	}
}

// Set all LEDs along the whole of a tentacle to brightness
func (p *Piglow) SetTentacle(tentacle int, brightness uint8) {
	leds := selectTentacle(tentacle)
	for i := 0; i < 6; i++ {
		p.SetLED(leds[i], brightness)
	}
}

// Display a value on a tentacle.
// tentacle: 0-2, the tentacle to display the value
// val: the value to display
// max_val: the range in which to display the value. If val equals, or goes above this all LEDs will be lit.
// brightness: What the max brightness should be for fully list LEDs
// direction: start the display from the centre out, or the outside in
func (p *Piglow) DisplayValueOnTentacle(tentacle int, val float64, max_val float64, brightness uint8, direction bool) {
	leds := selectTentacle(tentacle)

	num_leds := len(leds)
	values_per_led := max_val / float64(num_leds)
	proportion_lit := val / values_per_led
	full, partial := math.Modf(proportion_lit)
	if int(full) >= len(leds) {
		p.SetTentacle(tentacle, brightness)
		return
	}

	p.zeroLEDs(leds[0:len(leds)])

	for i := 0; i < int(full); i++ {
		if direction {
			p.SetLED(leds[i], brightness)
		} else {
			p.SetLED(leds[len(leds)-i-1], brightness)
		}
	}

	if partial == 0 {
		return
	}

	partial_brightness := math.Floor((float64(brightness) * partial) + 0.5)
	if direction {
		p.SetLED(leds[int(full)], uint8(partial_brightness))
	} else {
		p.SetLED(leds[len(leds)-int(full)-1], uint8(partial_brightness))
	}

	return
}

func selectTentacle(tentacle int) (leds [6]int8) {
	switch {
	case tentacle == 0:
		leds = tentacle_0_leds
	case tentacle == 1:
		leds = tentacle_1_leds
	case tentacle == 2:
		leds = tentacle_2_leds
	}
	return
}

func (p *Piglow) zeroLEDs(leds []int8) {
	for i := 0; i < len(leds); i++ {
		p.SetLED(leds[i], 0)
	}
}
<|endoftext|>