<commit_before>package msgpack

import (
	"math"
	"reflect"
	"sort"

	"github.com/vmihailenco/msgpack/v5/msgpcode"
)

func encodeMapValue(e *Encoder, v reflect.Value) error {
	if v.IsNil() {
		return e.EncodeNil()
	}

	if err := e.EncodeMapLen(v.Len()); err != nil {
		return err
	}

	for _, key := range v.MapKeys() {
		if err := e.EncodeValue(key); err != nil {
			return err
		}
		if err := e.EncodeValue(v.MapIndex(key)); err != nil {
			return err
		}
	}

	return nil
}

func encodeMapStringStringValue(e *Encoder, v reflect.Value) error {
	if v.IsNil() {
		return e.EncodeNil()
	}

	if err := e.EncodeMapLen(v.Len()); err != nil {
		return err
	}

	m := v.Convert(mapStringStringType).Interface().(map[string]string)
	if e.flags&sortMapKeysFlag != 0 {
		return e.encodeSortedMapStringString(m)
	}

	for mk, mv := range m {
		if err := e.EncodeString(mk); err != nil {
			return err
		}
		if err := e.EncodeString(mv); err != nil {
			return err
		}
	}

	return nil
}

func encodeMapStringInterfaceValue(e *Encoder, v reflect.Value) error {
	if v.IsNil() {
		return e.EncodeNil()
	}
	m := v.Convert(mapStringInterfaceType).Interface().(map[string]interface{})
	if e.flags&sortMapKeysFlag != 0 {
		return e.EncodeMapSorted(m)
	}
	return e.EncodeMap(m)
}

func (e *Encoder) EncodeMap(m map[string]interface{}) error {
	if m == nil {
		return e.EncodeNil()
	}
	if err := e.EncodeMapLen(len(m)); err != nil {
		return err
	}
	for mk, mv := range m {
		if err := e.EncodeString(mk); err != nil {
			return err
		}
		if err := e.Encode(mv); err != nil {
			return err
		}
	}
	return nil
}

func (e *Encoder) EncodeMapSorted(m map[string]interface{}) error {
	if m == nil {
		return e.EncodeNil()
	}
	if err := e.EncodeMapLen(len(m)); err != nil {
		return err
	}

	keys := make([]string, 0, len(m))

	for k := range m {
		keys = append(keys, k)
	}

	sort.Strings(keys)

	for _, k := range keys {
		if err := e.EncodeString(k); err != nil {
			return err
		}
		if err := e.Encode(m[k]); err != nil {
			return err
		}
	}

	return nil
}

func (e *Encoder) encodeSortedMapStringString(m map[string]string) error {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, k := range keys {
		err := e.EncodeString(k)
		if err != nil {
			return err
		}
		if err = e.EncodeString(m[k]); err != nil {
			return err
		}
	}

	return nil
}

func (e *Encoder) EncodeMapLen(l int) error {
	if l < 16 {
		return e.writeCode(msgpcode.FixedMapLow | byte(l))
	}
	if l <= math.MaxUint16 {
		return e.write2(msgpcode.Map16, uint16(l))
	}
	return e.write4(msgpcode.Map32, uint32(l))
}

func encodeStructValue(e *Encoder, strct reflect.Value) error {
	structFields := structs.Fields(strct.Type(), e.structTag)
	if e.flags&arrayEncodedStructsFlag != 0 || structFields.AsArray {
		return encodeStructValueAsArray(e, strct, structFields.List)
	}
	fields := structFields.OmitEmpty(strct, e.flags&omitEmptyFlag != 0)

	if err := e.EncodeMapLen(len(fields)); err != nil {
		return err
	}

	for _, f := range fields {
		if err := e.EncodeString(f.name); err != nil {
			return err
		}
		if err := f.EncodeValue(e, strct); err != nil {
			return err
		}
	}

	return nil
}

func encodeStructValueAsArray(e *Encoder, strct reflect.Value, fields []*field) error {
	if err := e.EncodeArrayLen(len(fields)); err != nil {
		return err
	}
	for _, f := range fields {
		if err := f.EncodeValue(e, strct); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Use MapRange for map iteration<commit_after>package msgpack

import (
	"math"
	"reflect"
	"sort"

	"github.com/vmihailenco/msgpack/v5/msgpcode"
)

func encodeMapValue(e *Encoder, v reflect.Value) error {
	if v.IsNil() {
		return e.EncodeNil()
	}

	if err := e.EncodeMapLen(v.Len()); err != nil {
		return err
	}

	iter := v.MapRange()
	for iter.Next() {
		if err := e.EncodeValue(iter.Key()); err != nil {
			return err
		}
		if err := e.EncodeValue(iter.Value()); err != nil {
			return err
		}
	}

	return nil
}

func encodeMapStringStringValue(e *Encoder, v reflect.Value) error {
	if v.IsNil() {
		return e.EncodeNil()
	}

	if err := e.EncodeMapLen(v.Len()); err != nil {
		return err
	}

	m := v.Convert(mapStringStringType).Interface().(map[string]string)
	if e.flags&sortMapKeysFlag != 0 {
		return e.encodeSortedMapStringString(m)
	}

	for mk, mv := range m {
		if err := e.EncodeString(mk); err != nil {
			return err
		}
		if err := e.EncodeString(mv); err != nil {
			return err
		}
	}

	return nil
}

func encodeMapStringInterfaceValue(e *Encoder, v reflect.Value) error {
	if v.IsNil() {
		return e.EncodeNil()
	}
	m := v.Convert(mapStringInterfaceType).Interface().(map[string]interface{})
	if e.flags&sortMapKeysFlag != 0 {
		return e.EncodeMapSorted(m)
	}
	return e.EncodeMap(m)
}

func (e *Encoder) EncodeMap(m map[string]interface{}) error {
	if m == nil {
		return e.EncodeNil()
	}
	if err := e.EncodeMapLen(len(m)); err != nil {
		return err
	}
	for mk, mv := range m {
		if err := e.EncodeString(mk); err != nil {
			return err
		}
		if err := e.Encode(mv); err != nil {
			return err
		}
	}
	return nil
}

func (e *Encoder) EncodeMapSorted(m map[string]interface{}) error {
	if m == nil {
		return e.EncodeNil()
	}
	if err := e.EncodeMapLen(len(m)); err != nil {
		return err
	}

	keys := make([]string, 0, len(m))

	for k := range m {
		keys = append(keys, k)
	}

	sort.Strings(keys)

	for _, k := range keys {
		if err := e.EncodeString(k); err != nil {
			return err
		}
		if err := e.Encode(m[k]); err != nil {
			return err
		}
	}

	return nil
}

func (e *Encoder) encodeSortedMapStringString(m map[string]string) error {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, k := range keys {
		err := e.EncodeString(k)
		if err != nil {
			return err
		}
		if err = e.EncodeString(m[k]); err != nil {
			return err
		}
	}

	return nil
}

func (e *Encoder) EncodeMapLen(l int) error {
	if l < 16 {
		return e.writeCode(msgpcode.FixedMapLow | byte(l))
	}
	if l <= math.MaxUint16 {
		return e.write2(msgpcode.Map16, uint16(l))
	}
	return e.write4(msgpcode.Map32, uint32(l))
}

func encodeStructValue(e *Encoder, strct reflect.Value) error {
	structFields := structs.Fields(strct.Type(), e.structTag)
	if e.flags&arrayEncodedStructsFlag != 0 || structFields.AsArray {
		return encodeStructValueAsArray(e, strct, structFields.List)
	}
	fields := structFields.OmitEmpty(strct, e.flags&omitEmptyFlag != 0)

	if err := e.EncodeMapLen(len(fields)); err != nil {
		return err
	}

	for _, f := range fields {
		if err := e.EncodeString(f.name); err != nil {
			return err
		}
		if err := f.EncodeValue(e, strct); err != nil {
			return err
		}
	}

	return nil
}

func encodeStructValueAsArray(e *Encoder, strct reflect.Value, fields []*field) error {
	if err := e.EncodeArrayLen(len(fields)); err != nil {
		return err
	}
	for _, f := range fields {
		if err := f.EncodeValue(e, strct); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package emil

import (
	"errors"
	"fmt"
	"time"
)

var errNowNewAnalysis = errors.New("errNowNewAnalysis")

type Analysis struct {
	dtm int // Depth to mate

	board *Board
	move  *Move
}

func (a *Analysis) Move() *Move {
	return a.move
}
func (a *Analysis) Board() *Board {
	return a.board
}

// EndGameDb to query for mate in 1,2, etc.
type EndGameDb struct {
	positionDb map[string]*Analysis

	dtmDb []map[string]bool

	searchedPositions int
}

func (db *EndGameDb) Find(board *Board) (bestMove *Move) {
	if DEBUG {
		fmt.Printf("Find:\n%s\n", board.String())
	}
	a := db.positionDb[board.String()]
	if DEBUG {
		fmt.Printf("Found: positionDb with dtm %d\n", a.dtm)
	}
	return a.move
}
func (db *EndGameDb) FindMatesIn(dtm int) (as []*Analysis) {
	if dtm == -1 {
		for _, a := range db.positionDb {
			if a.dtm == -1 {
				as = append(as, a)
			}
		}
	} else {
		for str := range db.dtmDb[dtm] {
			as = append(as, db.positionDb[str])
		}
	}
	return as
}

func (db *EndGameDb) FindMates() (as []*Analysis) {
	return db.FindMatesIn(0)
}

func (db *EndGameDb) FindMate(piece, square int) (boards []*Board) {
	for str := range db.dtmDb[0] {
		a := db.positionDb[str]
		if a.board.squares[square] == piece {
			boards = append(boards, a.board)
		}
	}
	return boards
}

func (db *EndGameDb) addPosition(board *Board) {
	db.addAnalysis(board, -1, nil)
}

func (db *EndGameDb) addAnalysis(board *Board, dtm int, move *Move) {
	a := &Analysis{
		dtm:   dtm,
		board: board}
	if move != nil {
		a.move = move.reverse()
	}
	if dtm >= 0 {
		if move != nil {
			playerForStep := playerForStepN(dtm)
			if playerForStep != move.player {
				panic("playerForStep != move.player")
			}
		}

		db.dtmDb[dtm][a.board.String()] = true
	}

	db.positionDb[a.board.String()] = a
}

func (db *EndGameDb) positions() int {
	return len(db.positionDb)
}

// find positions where black is checkmate
func (db *EndGameDb) retrogradeAnalysisStep1() {
	db.dtmDb = append(db.dtmDb, make(map[string]bool))

	start := time.Now()

	player := BLACK
	for boardStr, a := range db.positionDb {
		// mate only on border square
		blackKingSquare := BoardSquares[a.board.blackKing]
		if !blackKingSquare.isBorder {
			continue
		}
		// mate only with help from king
		if squaresDistances[a.board.blackKing][a.board.whiteKing] > 2 {
			continue
		}

		p := NewPosition(a.board, player)

		move := Search(p)
		db.searchedPositions++
		if move == nil {
			if isKingInCheck(p) {
				a.dtm = 0
				db.addAnalysis(a.board, 0, nil)
				if DEBUG {
					fmt.Printf("mate:\n%s\n", boardStr)
				}
			}
		}
	}
	end := time.Now()
	if DEBUG {
		fmt.Printf("searchedPositions %d\n", db.searchedPositions)
		fmt.Printf("db.dtmDb[0] %d\n", len(db.dtmDb[0]))
		fmt.Printf("duration %v\n\n\n", end.Sub(start))
	}
}
func playerForStepN(dtm int) (player int) {
	if dtm%2 == 0 {
		return BLACK
	}
	return WHITE
}

func (db *EndGameDb) retrogradeAnalysisStepN(dtm int) (noError error) {
	start := time.Now()
	db.dtmDb = append(db.dtmDb, make(map[string]bool))

	player := playerForStepN(dtm)

	positions := 0
	if player == WHITE {
		for _, a := range db.positionDb {
			if db.isMateIn1357(a.board, dtm) >= 0 {
				positions++
			}
		}
		if DEBUG {
			fmt.Printf("WHITE Start positions %d\n", len(db.positionDb)-positions)
		}
		for _, a := range db.positionDb {
			if db.isMateIn1357(a.board, dtm) >= 0 {
				continue
			}
			if IsTheKingInCheck(NewPosition(a.board, WHITE)) {
				continue
			}
			p := NewPosition(a.board, player)
			moves := generateMoves(p)

			for _, m := range moves {
				newBoard := a.board.doMove(m)
				newDtm := db.isMateIn0246(newBoard, dtm)
				if newDtm == dtm-1 {
					db.addAnalysis(a.board, dtm, m)
				}
			}
		}
	} else {
		// Suche alle Stellungen, bei denen Schwarz am Zug ist und
		// **jeder** Zug von ihm zu einer Stellung unter 2. führt.
		// Schwarz kann hier Matt in einem Zug nicht verhindern.
		// Markiere diese Stellungen in der Datei.
		for _, a := range db.positionDb {
			if db.isMateIn0246(a.board, dtm) >= 0 {
				positions++
			}
		}
		if DEBUG {
			fmt.Printf("BLACK Start positions %d\n", len(db.positionDb)-positions)
		}
		for _, a := range db.positionDb {
			if db.isMateIn0246(a.board, dtm) >= 0 {
				continue
			}
			p := NewPosition(a.board, player)
			moves := GenerateMoves(p)

			found := 0
			maxDTM := -1
			for _, m := range moves {
				newBoard := a.board.doMove(m)
				newDtm := db.isMateIn1357(newBoard, dtm)
				if newDtm > maxDTM {
					maxDTM = newDtm
				}
				if db.isMateIn1357(newBoard, dtm) >= 0 {
					found++
				}
			}

			if found == len(moves) {
				for _, m := range moves {
					db.addAnalysis(a.board, maxDTM+1, m)
				}
			}
		}
	}
	end := time.Now()

	if DEBUG {
		fmt.Printf("db.dtmDb[%d] %d\n", dtm, len(db.dtmDb[dtm]))
		fmt.Printf("duration %v\n\n\n", end.Sub(start))
	}

	if len(db.dtmDb[dtm]) == 0 {
		return errNowNewAnalysis
	}
	return noError
}
func (db *EndGameDb) isMateIn0246(board *Board, maxDtm int) int {
	for dtm := 0; dtm < maxDtm; dtm += 2 {
		_, ok := db.dtmDb[dtm][board.String()]
		if ok {
			return dtm
		}
	}
	return -1
}
func (db *EndGameDb) isMateIn1357(board *Board, maxDtm int) int {
	for dtm := 1; dtm < maxDtm; dtm += 2 {
		_, ok := db.dtmDb[dtm][board.String()]
		if ok {
			return dtm
		}
	}
	return -1
}

func (db *EndGameDb) MaxDtm() int {
	return len(db.dtmDb)
}

func (db *EndGameDb) retrogradeAnalysis() {
	// find positions where black is checkmate
	db.retrogradeAnalysisStep1()
	dtm := 1
	for {
		err := db.retrogradeAnalysisStepN(dtm)
		if err != nil {
			break
		}
		dtm++
	}
}
func GenerateMoves(p *position) (list []*Move) {
	for _, m := range generateMoves(p) {
		b := p.board.DoMove(m)
		if !IsTheKingInCheck(NewPosition(b, WHITE)) {
			list = append(list, m)
		}
	}
	return list
}
func generateMoves(p *position) (list []*Move) {
	for src, piece := range p.board.squares {
		if isOwnPiece(p.player, piece) {
			switch abs(piece) {
			case kingValue:
				for _, dst := range kingDestinationsFrom(src) {
					capture := p.board.squares[dst]
					if isOtherKing(p.player, capture) {
						continue
					}
					if capture == Empty {
						list = append(list, newSilentMove(p.player, piece, src, dst))
					} else if !isOwnPiece(p.player, capture) {
						list = append(list, newCaptureMove(p.player, piece, capture, src, dst))
					}
				}
			case rockValue:
				for _, dsts := range rockDestinationsFrom(src) {
					for _, dst := range dsts {
						capture := p.board.squares[dst]
						if isOtherKing(p.player, capture) {
							break
						}
						if capture == Empty {
							list = append(list, newSilentMove(p.player, piece, src, dst))
						} else if !isOwnPiece(p.player, capture) {
							list = append(list, newCaptureMove(p.player, piece, capture, src, dst))
							break
						} else {
							break // onOwnPiece
						}
					}
				}
			}
		}
	}
	return list
}

// NewEndGameDb generates an end game DB for KRK
func NewEndGameDb() *EndGameDb {
	var err error
	start := time.Now()

	endGames := &EndGameDb{
		positionDb: make(map[string]*Analysis),
		dtmDb:      make([]map[string]bool, 0)}

	for wk := A1; wk <= H8; wk++ {
		//for wk := E3; wk <= E3; wk++ {
		if DEBUG {
			fmt.Printf("White king on %s\n", BoardSquares[wk])
		}
		for wr := A1; wr <= H8; wr++ {
			for bk := A1; bk <= H8; bk++ {

				board := NewBoard()

				err = board.Setup(WhiteKing, wk)
				if err != nil {
					continue
				}

				err = board.Setup(WhiteRock, wr)
				if err != nil {
					continue
				}

				err = board.Setup(BlackKing, bk)
				if err != nil {
					continue
				}

				err = board.kingsToClose()
				if err != nil {
					continue
				}

				endGames.addPosition(board)
			}
		}
	}
	end := time.Now()
	if DEBUG {
		fmt.Printf("all positions %d\n", 64*63*62)
		fmt.Printf("endGames.positions() %d\n", endGames.positions())
		fmt.Printf("difference %d\n", 64*63*62-endGames.positions())
		fmt.Printf("duration %v\n", end.Sub(start))
	}
	endGames.retrogradeAnalysis()

	return endGames
}
<commit_msg>refactor retrogradeAnalysisStepN<commit_after>package emil

import (
	"errors"
	"fmt"
	"time"
)

var errNowNewAnalysis = errors.New("errNowNewAnalysis")

type Analysis struct {
	dtm int // Depth to mate

	board *Board
	move  *Move
}

func (a *Analysis) Move() *Move {
	return a.move
}
func (a *Analysis) Board() *Board {
	return a.board
}

// EndGameDb to query for mate in 1,2, etc.
type EndGameDb struct {
	positionDb map[string]*Analysis

	dtmDb []map[string]bool

	searchedPositions int
}

func (db *EndGameDb) Find(board *Board) (bestMove *Move) {
	if DEBUG {
		fmt.Printf("Find:\n%s\n", board.String())
	}
	a := db.positionDb[board.String()]
	if DEBUG {
		fmt.Printf("Found: positionDb with dtm %d\n", a.dtm)
	}
	return a.move
}
func (db *EndGameDb) FindMatesIn(dtm int) (as []*Analysis) {
	if dtm == -1 {
		for _, a := range db.positionDb {
			if a.dtm == -1 {
				as = append(as, a)
			}
		}
	} else {
		for str := range db.dtmDb[dtm] {
			as = append(as, db.positionDb[str])
		}
	}
	return as
}

func (db *EndGameDb) FindMates() (as []*Analysis) {
	return db.FindMatesIn(0)
}

func (db *EndGameDb) FindMate(piece, square int) (boards []*Board) {
	for str := range db.dtmDb[0] {
		a := db.positionDb[str]
		if a.board.squares[square] == piece {
			boards = append(boards, a.board)
		}
	}
	return boards
}

func (db *EndGameDb) addPosition(board *Board) {
	db.addAnalysis(board, -1, nil)
}

func (db *EndGameDb) addAnalysis(board *Board, dtm int, move *Move) {
	a := &Analysis{
		dtm:   dtm,
		board: board}
	if move != nil {
		a.move = move.reverse()
	}
	if dtm >= 0 {
		if move != nil {
			playerForStep := playerForStepN(dtm)
			if playerForStep != move.player {
				panic("playerForStep != move.player")
			}
		}

		db.dtmDb[dtm][a.board.String()] = true
	}

	db.positionDb[a.board.String()] = a
}

func (db *EndGameDb) positions() int {
	return len(db.positionDb)
}

// find positions where black is checkmate
func (db *EndGameDb) retrogradeAnalysisStep1() {
	db.dtmDb = append(db.dtmDb, make(map[string]bool))

	start := time.Now()

	player := BLACK
	for boardStr, a := range db.positionDb {
		// mate only on border square
		blackKingSquare := BoardSquares[a.board.blackKing]
		if !blackKingSquare.isBorder {
			continue
		}
		// mate only with help from king
		if squaresDistances[a.board.blackKing][a.board.whiteKing] > 2 {
			continue
		}

		p := NewPosition(a.board, player)

		move := Search(p)
		db.searchedPositions++
		if move == nil {
			if isKingInCheck(p) {
				a.dtm = 0
				db.addAnalysis(a.board, 0, nil)
				if DEBUG {
					fmt.Printf("mate:\n%s\n", boardStr)
				}
			}
		}
	}
	end := time.Now()
	if DEBUG {
		fmt.Printf("searchedPositions %d\n", db.searchedPositions)
		fmt.Printf("db.dtmDb[0] %d\n", len(db.dtmDb[0]))
		fmt.Printf("duration %v\n\n\n", end.Sub(start))
	}
}
func playerForStepN(dtm int) (player int) {
	if dtm%2 == 0 {
		return BLACK
	}
	return WHITE
}

func (db *EndGameDb) retrogradeAnalysisStepN(dtm int) (noError error) {
	start := time.Now()
	db.dtmDb = append(db.dtmDb, make(map[string]bool))

	player := playerForStepN(dtm)

	positions := 0
	if player == WHITE {
		for _, a := range db.positionDb {
			if db.isMateIn1357(a.board, dtm) >= 0 {
				positions++
			}
		}
		if DEBUG {
			fmt.Printf("WHITE Start positions %d\n", len(db.positionDb)-positions)
		}
		for _, a := range db.positionDb {
			if db.isMateIn1357(a.board, dtm) >= 0 {
				continue
			}
			p := NewPosition(a.board, player)
			if IsTheKingInCheck(p) {
				continue
			}
			moves := generateMoves(p)

			for _, m := range moves {
				newBoard := a.board.doMove(m)
				newDtm := db.isMateIn0246(newBoard, dtm)
				if newDtm == dtm-1 {
					db.addAnalysis(a.board, dtm, m)
				}
			}
		}
	} else {
		// Suche alle Stellungen, bei denen Schwarz am Zug ist und
		// **jeder** Zug von ihm zu einer Stellung unter 2. führt.
		// Schwarz kann hier Matt in einem Zug nicht verhindern.
		// Markiere diese Stellungen in der Datei.
		for _, a := range db.positionDb {
			if db.isMateIn0246(a.board, dtm) >= 0 {
				positions++
			}
		}
		if DEBUG {
			fmt.Printf("BLACK Start positions %d\n", len(db.positionDb)-positions)
		}
		for _, a := range db.positionDb {
			if db.isMateIn0246(a.board, dtm) >= 0 {
				continue
			}
			p := NewPosition(a.board, player)
			moves := GenerateMoves(p)

			found := 0
			maxDTM := -1
			for _, m := range moves {
				newBoard := a.board.doMove(m)
				newDtm := db.isMateIn1357(newBoard, dtm)
				if newDtm > maxDTM {
					maxDTM = newDtm
				}
				if db.isMateIn1357(newBoard, dtm) >= 0 {
					found++
				}
			}

			if found == len(moves) {
				for _, m := range moves {
					db.addAnalysis(a.board, maxDTM+1, m)
				}
			}
		}
	}
	end := time.Now()

	if DEBUG {
		fmt.Printf("db.dtmDb[%d] %d\n", dtm, len(db.dtmDb[dtm]))
		fmt.Printf("duration %v\n\n\n", end.Sub(start))
	}

	if len(db.dtmDb[dtm]) == 0 {
		return errNowNewAnalysis
	}
	return noError
}
func (db *EndGameDb) isMateIn0246(board *Board, maxDtm int) int {
	for dtm := 0; dtm < maxDtm; dtm += 2 {
		_, ok := db.dtmDb[dtm][board.String()]
		if ok {
			return dtm
		}
	}
	return -1
}
func (db *EndGameDb) isMateIn1357(board *Board, maxDtm int) int {
	for dtm := 1; dtm < maxDtm; dtm += 2 {
		_, ok := db.dtmDb[dtm][board.String()]
		if ok {
			return dtm
		}
	}
	return -1
}

func (db *EndGameDb) MaxDtm() int {
	return len(db.dtmDb)
}

func (db *EndGameDb) retrogradeAnalysis() {
	// find positions where black is checkmate
	db.retrogradeAnalysisStep1()
	dtm := 1
	for {
		err := db.retrogradeAnalysisStepN(dtm)
		if err != nil {
			break
		}
		dtm++
	}
}
func GenerateMoves(p *position) (list []*Move) {
	for _, m := range generateMoves(p) {
		b := p.board.DoMove(m)
		if !IsTheKingInCheck(NewPosition(b, WHITE)) {
			list = append(list, m)
		}
	}
	return list
}
func generateMoves(p *position) (list []*Move) {
	for src, piece := range p.board.squares {
		if isOwnPiece(p.player, piece) {
			switch abs(piece) {
			case kingValue:
				for _, dst := range kingDestinationsFrom(src) {
					capture := p.board.squares[dst]
					if isOtherKing(p.player, capture) {
						continue
					}
					if capture == Empty {
						list = append(list, newSilentMove(p.player, piece, src, dst))
					} else if !isOwnPiece(p.player, capture) {
						list = append(list, newCaptureMove(p.player, piece, capture, src, dst))
					}
				}
			case rockValue:
				for _, dsts := range rockDestinationsFrom(src) {
					for _, dst := range dsts {
						capture := p.board.squares[dst]
						if isOtherKing(p.player, capture) {
							break
						}
						if capture == Empty {
							list = append(list, newSilentMove(p.player, piece, src, dst))
						} else if !isOwnPiece(p.player, capture) {
							list = append(list, newCaptureMove(p.player, piece, capture, src, dst))
							break
						} else {
							break // onOwnPiece
						}
					}
				}
			}
		}
	}
	return list
}

// NewEndGameDb generates an end game DB for KRK
func NewEndGameDb() *EndGameDb {
	var err error
	start := time.Now()

	endGames := &EndGameDb{
		positionDb: make(map[string]*Analysis),
		dtmDb:      make([]map[string]bool, 0)}

	for wk := A1; wk <= H8; wk++ {
		//for wk := E3; wk <= E3; wk++ {
		if DEBUG {
			fmt.Printf("White king on %s\n", BoardSquares[wk])
		}
		for wr := A1; wr <= H8; wr++ {
			for bk := A1; bk <= H8; bk++ {

				board := NewBoard()

				err = board.Setup(WhiteKing, wk)
				if err != nil {
					continue
				}

				err = board.Setup(WhiteRock, wr)
				if err != nil {
					continue
				}

				err = board.Setup(BlackKing, bk)
				if err != nil {
					continue
				}

				err = board.kingsToClose()
				if err != nil {
					continue
				}

				endGames.addPosition(board)
			}
		}
	}
	end := time.Now()
	if DEBUG {
		fmt.Printf("all positions %d\n", 64*63*62)
		fmt.Printf("endGames.positions() %d\n", endGames.positions())
		fmt.Printf("difference %d\n", 64*63*62-endGames.positions())
		fmt.Printf("duration %v\n", end.Sub(start))
	}
	endGames.retrogradeAnalysis()

	return endGames
}
<|endoftext|>


<commit_before>package controllers

import (

	// "encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/looyun/feedall/models"
	"github.com/mmcdole/gofeed"

	"gopkg.in/macaron.v1"
	"gopkg.in/mgo.v2/bson"
)

var TelegramBotToken string = "123456"

func InsertFeedAndUpdateItems(feedurl string) error {
	// parse feed url
	fp := gofeed.NewParser()
	parsedFeed, err := fp.ParseURL(feedurl)
	if err != nil {
		return err
	}

	bsonFeed, err := bson.Marshal(parsedFeed)
	if err != nil {
		return err
	}

	// Insert feed
	feed := &models.Feed{}
	err = bson.Unmarshal(bsonFeed, &feed)
	if err != nil {
		return err
	}
	feed.ID = bson.NewObjectId()
	err = models.Insert(models.Feeds, feed)
	if err != nil {
		return err
	}

	// Update items
	var data struct {
		Items []models.Item `bson:"items"`
	}
	err = bson.Unmarshal(bsonFeed, &data)
	if err != nil {
		return err
	}
	for _, item := range data.Items {
		item.FeedID = feed.ID
		_, err := models.Upsert(models.Items,
			bson.M{
				"$and": []bson.M{
					bson.M{"feedID": feed.ID}, bson.M{"link": item.Link}}},
			item)
		if err != nil {
			return err
		}
	}

	return nil
}

func UpdateItems(feed models.Feed) error {

	feedurl := feed.FeedURL

	// parse feed url
	fp := gofeed.NewParser()
	parsedFeed, err := fp.ParseURL(feedurl)
	if err != nil {
		return err
	}

	bsonFeed, err := bson.Marshal(parsedFeed)
	if err != nil {
		return err
	}

	var data struct {
		Items []models.Item `bson:"items"`
	}
	err = bson.Unmarshal(bsonFeed, &data)
	if err != nil {
		return err
	}

	for _, item := range data.Items {
		item.FeedID = feed.ID
		_, err := models.Upsert(models.Items,
			bson.M{
				"$and": []bson.M{
					bson.M{"feedID": feed.ID}, bson.M{"link": item.Link}}},
			item)
		if err != nil {
			return err
		}
	}
	return nil
}

// GetFeeds get feeds sort by subscribeCount.
func GetFeeds(c *macaron.Context) interface{} {

	page := c.QueryInt("page")
	if page > 0 {
		page--
	}
	perPage := c.QueryInt("per_page")
	if perPage == 0 {
		perPage = 30
	}
	if perPage > 100 {
		perPage = 100
	}
	feeds := []bson.M{}
	err := models.Feeds.Find(bson.M{}).Sort("-subscribeCount").Skip(page * perPage).Limit(perPage).All(&feeds)

	if err != nil {
		fmt.Println(err)
	}
	return feeds
}

// GetFeeds get feeds sort by subscribeCount.
func GetFeedItems(c *macaron.Context) interface{} {

	page := c.QueryInt("page")
	if page > 0 {
		page--
	}
	perPage := c.QueryInt("per_page")
	if perPage == 0 {
		perPage = 30
	}
	if perPage > 100 {
		perPage = 100
	}

	feedID := c.Params(":id")

	items := []bson.M{}
	err := models.Items.Find(bson.M{"feedID": feedID}).Sort("-publishedParsed").Skip(perPage * page).Limit(perPage).All(&items)

	if err != nil {
		fmt.Println(err)
	}
	return items
}
func GetFeed(c *macaron.Context) interface{} {
	id := c.Params(":id")
	feed := bson.M{}
	err := models.FindOne(models.Feeds,
		bson.M{"_id": id},
		&feed)
	if err != nil {
		fmt.Println(err)
	}

	return feed
}

func GetItems(c *macaron.Context, n int) interface{} {
	if n > 100 {
		return nil
	}
	items := []bson.M{}
	err := models.FindSortLimit(models.Items,
		bson.M{}, "-starCount", n,
		&items)
	if err != nil {
		fmt.Println(err)
	}
	return items
}
func GetItem(c *macaron.Context) interface{} {
	id := c.Params(":id")
	item := bson.M{}
	err := models.FindOne(models.Items,
		bson.M{"_id": id},
		&item)
	if err != nil {
		fmt.Println(err)
	}
	return item

}

func GetRandomItem(c *macaron.Context, n int) interface{} {
	if n > 100 {
		return nil
	}
	items := []bson.M{}
	err := models.PipeAll(models.Items, []bson.M{{"$sample": bson.M{"size": n}}},
		&items)
	if err != nil {
		fmt.Println(err)
	}
	return items

}
func GetLatestItem(c *macaron.Context, n int) interface{} {
	if n > 100 {
		return nil
	}
	items := []bson.M{}
	err := models.FindSortLimit(models.Items,
		bson.M{}, "-publishedParsed", n,
		&items)
	if err != nil {
		fmt.Println(err)
	}
	return items

}

func StandarURL(s string) string {
	if !strings.HasSuffix(s, "/") {
		s = s + "/"
	}
	return s
}

func DecodeImg(str string, link string) string {
	str = strings.Replace(str, """, "\"", -1)
	str = strings.Replace(str, "src=\"/", "src=\""+link+"/", -1)
	return str
}

func ParseDate(t string) (then time.Time) {

	if len(t) >= 25 {
		if strings.HasSuffix(t, "0000") {
			then, _ = time.Parse("Mon, 02 Jan 2006 15:04:05 +0000", t)
		} else if strings.HasSuffix(t, "GMT") {
			then, _ = time.Parse("Mon, 02 Jan 2006 15:04:05 GMT", t)
		} else if strings.HasSuffix(t, "UTC") {
			then, _ = time.Parse("Mon, 02 Jan 2006 15:04:05 UTC", t)
		} else if strings.HasSuffix(t, "CST") {
			then, _ = time.Parse("Mon, 02 Jan 2006 15:04:05 CST", t)
		} else if strings.HasSuffix(t, "0400") {
			then, _ = time.Parse("Mon, 02 Jan 2006 15:04:05 -0400", t)
		} else if strings.HasSuffix(t, "Z") {
			then, _ = time.Parse(time.RFC3339, t)
		} else if strings.HasSuffix(t, "0800") {
			then, _ = time.Parse("Mon, 02 Jan 2006 15:04:05 +0800", t)
		}
	} else {
		if strings.HasSuffix(t, "0000") {
			then, _ = time.Parse("02 Jan 06 15:04 +0000", t)
		} else if strings.HasSuffix(t, "GMT") {
			then, _ = time.Parse("02 Jan 06 15:04 GMT", t)
		} else if strings.HasSuffix(t, "UTC") {
			then, _ = time.Parse("02 Jan 06 15:04 UTC", t)
		} else if strings.HasSuffix(t, "CST") {
			then, _ = time.Parse("02 Jan 06 15:04 CST", t)
		} else if strings.HasSuffix(t, "0400") {
			then, _ = time.Parse("02 Jan 06 15:04 -0400", t)
		} else if strings.HasSuffix(t, "Z") {
			then, _ = time.Parse(time.RFC3339, t)
		} else if strings.HasSuffix(t, "0800") {
			then, _ = time.Parse("02 Jan 06 15:04 +0800", t)
		}
	}
	return then
}
<commit_msg>Upsert feed with feedurl<commit_after>package controllers

import (

	// "encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/looyun/feedall/models"
	"github.com/mmcdole/gofeed"

	"gopkg.in/macaron.v1"
	"gopkg.in/mgo.v2/bson"
)

var TelegramBotToken string = "123456"

func InsertFeedAndUpdateItems(feedurl string) error {
	// parse feed url
	fp := gofeed.NewParser()
	parsedFeed, err := fp.ParseURL(feedurl)
	if err != nil {
		return err
	}

	bsonFeed, err := bson.Marshal(parsedFeed)
	if err != nil {
		return err
	}

	// Insert feed
	feed := &models.Feed{}
	err = bson.Unmarshal(bsonFeed, &feed)
	if err != nil {
		return err
	}
	feed.FeedURL = feedurl
	feed.ID = bson.NewObjectId()
	_, err = models.Upsert(models.Feeds,
		bson.M{"feedURL": feedurl},
		feed)
	if err != nil {
		return err
	}

	// Update items
	var data struct {
		Items []models.Item `bson:"items"`
	}
	err = bson.Unmarshal(bsonFeed, &data)
	if err != nil {
		return err
	}
	for _, item := range data.Items {
		item.FeedID = feed.ID
		_, err := models.Upsert(models.Items,
			bson.M{
				"$and": []bson.M{
					bson.M{"feedID": feed.ID}, bson.M{"link": item.Link}}},
			item)
		if err != nil {
			return err
		}
	}

	return nil
}

func UpdateItems(feed models.Feed) error {

	feedurl := feed.FeedURL

	// parse feed url
	fp := gofeed.NewParser()
	parsedFeed, err := fp.ParseURL(feedurl)
	if err != nil {
		return err
	}

	bsonFeed, err := bson.Marshal(parsedFeed)
	if err != nil {
		return err
	}

	var data struct {
		Items []models.Item `bson:"items"`
	}
	err = bson.Unmarshal(bsonFeed, &data)
	if err != nil {
		return err
	}

	for _, item := range data.Items {
		item.FeedID = feed.ID
		_, err := models.Upsert(models.Items,
			bson.M{
				"$and": []bson.M{
					bson.M{"feedID": feed.ID}, bson.M{"link": item.Link}}},
			item)
		if err != nil {
			return err
		}
	}
	return nil
}

// GetFeeds get feeds sort by subscribeCount.
func GetFeeds(c *macaron.Context) interface{} {

	page := c.QueryInt("page")
	if page > 0 {
		page--
	}
	perPage := c.QueryInt("per_page")
	if perPage == 0 {
		perPage = 30
	}
	if perPage > 100 {
		perPage = 100
	}
	feeds := []bson.M{}
	err := models.Feeds.Find(bson.M{}).Sort("-subscribeCount").Skip(page * perPage).Limit(perPage).All(&feeds)

	if err != nil {
		fmt.Println(err)
	}
	return feeds
}

// GetFeeds get feeds sort by subscribeCount.
func GetFeedItems(c *macaron.Context) interface{} {

	page := c.QueryInt("page")
	if page > 0 {
		page--
	}
	perPage := c.QueryInt("per_page")
	if perPage == 0 {
		perPage = 30
	}
	if perPage > 100 {
		perPage = 100
	}

	feedID := c.Params(":id")

	items := []bson.M{}
	err := models.Items.Find(bson.M{"feedID": feedID}).Sort("-publishedParsed").Skip(perPage * page).Limit(perPage).All(&items)

	if err != nil {
		fmt.Println(err)
	}
	return items
}
func GetFeed(c *macaron.Context) interface{} {
	id := c.Params(":id")
	feed := bson.M{}
	err := models.FindOne(models.Feeds,
		bson.M{"_id": id},
		&feed)
	if err != nil {
		fmt.Println(err)
	}

	return feed
}

func GetItems(c *macaron.Context, n int) interface{} {
	if n > 100 {
		return nil
	}
	items := []bson.M{}
	err := models.FindSortLimit(models.Items,
		bson.M{}, "-starCount", n,
		&items)
	if err != nil {
		fmt.Println(err)
	}
	return items
}
func GetItem(c *macaron.Context) interface{} {
	id := c.Params(":id")
	item := bson.M{}
	err := models.FindOne(models.Items,
		bson.M{"_id": id},
		&item)
	if err != nil {
		fmt.Println(err)
	}
	return item

}

func GetRandomItem(c *macaron.Context, n int) interface{} {
	if n > 100 {
		return nil
	}
	items := []bson.M{}
	err := models.PipeAll(models.Items, []bson.M{{"$sample": bson.M{"size": n}}},
		&items)
	if err != nil {
		fmt.Println(err)
	}
	return items

}
func GetLatestItem(c *macaron.Context, n int) interface{} {
	if n > 100 {
		return nil
	}
	items := []bson.M{}
	err := models.FindSortLimit(models.Items,
		bson.M{}, "-publishedParsed", n,
		&items)
	if err != nil {
		fmt.Println(err)
	}
	return items

}

func StandarURL(s string) string {
	if !strings.HasSuffix(s, "/") {
		s = s + "/"
	}
	return s
}

func DecodeImg(str string, link string) string {
	str = strings.Replace(str, """, "\"", -1)
	str = strings.Replace(str, "src=\"/", "src=\""+link+"/", -1)
	return str
}

func ParseDate(t string) (then time.Time) {

	if len(t) >= 25 {
		if strings.HasSuffix(t, "0000") {
			then, _ = time.Parse("Mon, 02 Jan 2006 15:04:05 +0000", t)
		} else if strings.HasSuffix(t, "GMT") {
			then, _ = time.Parse("Mon, 02 Jan 2006 15:04:05 GMT", t)
		} else if strings.HasSuffix(t, "UTC") {
			then, _ = time.Parse("Mon, 02 Jan 2006 15:04:05 UTC", t)
		} else if strings.HasSuffix(t, "CST") {
			then, _ = time.Parse("Mon, 02 Jan 2006 15:04:05 CST", t)
		} else if strings.HasSuffix(t, "0400") {
			then, _ = time.Parse("Mon, 02 Jan 2006 15:04:05 -0400", t)
		} else if strings.HasSuffix(t, "Z") {
			then, _ = time.Parse(time.RFC3339, t)
		} else if strings.HasSuffix(t, "0800") {
			then, _ = time.Parse("Mon, 02 Jan 2006 15:04:05 +0800", t)
		}
	} else {
		if strings.HasSuffix(t, "0000") {
			then, _ = time.Parse("02 Jan 06 15:04 +0000", t)
		} else if strings.HasSuffix(t, "GMT") {
			then, _ = time.Parse("02 Jan 06 15:04 GMT", t)
		} else if strings.HasSuffix(t, "UTC") {
			then, _ = time.Parse("02 Jan 06 15:04 UTC", t)
		} else if strings.HasSuffix(t, "CST") {
			then, _ = time.Parse("02 Jan 06 15:04 CST", t)
		} else if strings.HasSuffix(t, "0400") {
			then, _ = time.Parse("02 Jan 06 15:04 -0400", t)
		} else if strings.HasSuffix(t, "Z") {
			then, _ = time.Parse(time.RFC3339, t)
		} else if strings.HasSuffix(t, "0800") {
			then, _ = time.Parse("02 Jan 06 15:04 +0800", t)
		}
	}
	return then
}
<|endoftext|>


<commit_before>package controllers

import (
	"log"
	"net/http"
	"regexp"

	"github.com/UniversityRadioYork/2016-site/models"
	"github.com/UniversityRadioYork/2016-site/structs"
	"github.com/UniversityRadioYork/2016-site/utils"
	"github.com/UniversityRadioYork/myradio-go"
)

// SignUpController is the controller for processing signup requests.
type SignUpController struct {
	Controller
}

// NewSignUpController returns a new SignUpController with the MyRadio
// session s and configuration context c.
func NewSignUpController(s *myradio.Session, c *structs.Config) *SignUpController {
	return &SignUpController{Controller{session: s, config: c}}
}

// Post handles the HTTP POST request r for the get involved, writing to w.
func (gic *SignUpController) Post(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	formParams := r.Form
	var feedback []string

	//Validate that necessary params are present and correct(enough)
	_, ok := formParams["fname"]
	if !ok || formParams["fname"][0] == "" {
		feedback = append(feedback, "You need to provide your First Name")
	}
	_, ok = formParams["sname"]
	if !ok || formParams["sname"][0] == "" {
		feedback = append(feedback, "You need to provide your Last Name")
	}
	// Check an eduroam value is submitted
	// If not then the user is signing up using a personal email
	if _, ok := formParams["eduroam"]; ok {
		if formParams["eduroam"][0] == "" {
			feedback = append(feedback, "You need to provide your York Email")
		} else {
			match, _ := regexp.MatchString("^[a-z]{1,6}[0-9]{1,6}$", formParams["eduroam"][0])
			if !match {
				feedback = append(feedback, "The @york.ac.uk email you provided seems invalid")
			}
		}
	} else {
		if _, ok = formParams["email"]; !ok {
			feedback = append(feedback, "You need to provide your email address")
		}
	}
	_, ok = formParams["phone"]
	if !ok || formParams["phone"][0] == "" {
		delete(formParams, "phone")
	}

	//If they are then post them off to the API
	if len(feedback) == 0 {
		sm := models.NewSignUpModel(gic.session)
		err := sm.Post(formParams)
		if err != nil {
			log.Println(err)
			feedback = append(feedback, "Oops. Something went wrong on our end.")
			feedback = append(feedback, "Please try again later")
		}
	}

	data := struct {
		Feedback []string
	}{
		Feedback: feedback,
	}

	err := utils.RenderTemplate(w, gic.config.PageContext, data, "signedup.tmpl")

	if err != nil {
		log.Println(err)
		return
	}
}
<commit_msg>Allow @york.ac.uk to be entered even though we assume it<commit_after>package controllers

import (
	"log"
	"net/http"
	"regexp"
	"strings"

	"github.com/UniversityRadioYork/2016-site/models"
	"github.com/UniversityRadioYork/2016-site/structs"
	"github.com/UniversityRadioYork/2016-site/utils"
	"github.com/UniversityRadioYork/myradio-go"
)

// SignUpController is the controller for processing signup requests.
type SignUpController struct {
	Controller
}

// NewSignUpController returns a new SignUpController with the MyRadio
// session s and configuration context c.
func NewSignUpController(s *myradio.Session, c *structs.Config) *SignUpController {
	return &SignUpController{Controller{session: s, config: c}}
}

// Post handles the HTTP POST request r for the get involved, writing to w.
func (gic *SignUpController) Post(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	formParams := r.Form
	var feedback []string

	//Validate that necessary params are present and correct(enough)
	_, ok := formParams["fname"]
	if !ok || formParams["fname"][0] == "" {
		feedback = append(feedback, "You need to provide your First Name")
	}
	_, ok = formParams["sname"]
	if !ok || formParams["sname"][0] == "" {
		feedback = append(feedback, "You need to provide your Last Name")
	}
	// Check an eduroam value is submitted
	// If not then the user is signing up using a personal email
	if _, ok := formParams["eduroam"]; ok {
		eduroam := formParams["eduroam"][0]
		if eduroam == "" {
			feedback = append(feedback, "You need to provide your York Email")
		} else {
			// Ignore an added @york.ac.uk (since we assume it)
			if strings.HasSuffix(eduroam, "@york.ac.uk"){
				eduroam = eduroam[:len(eduroam)-11]
			}
			log.Println(eduroam)
			match, _ := regexp.MatchString("^[a-z]{1,6}[0-9]{1,6}$", eduroam)
			if !match {
				feedback = append(feedback, "The @york.ac.uk email you provided seems invalid")
			}
			formParams["eduroam"][0] = eduroam
		}
	} else {
		if _, ok = formParams["email"]; !ok {
			feedback = append(feedback, "You need to provide your email address")
		}
	}
	_, ok = formParams["phone"]
	if !ok || formParams["phone"][0] == "" {
		delete(formParams, "phone")
	}

	//If they are then post them off to the API
	if len(feedback) == 0 {
		sm := models.NewSignUpModel(gic.session)
		err := sm.Post(formParams)
		if err != nil {
			log.Println(err)
			feedback = append(feedback, "Oops. Something went wrong on our end.")
			feedback = append(feedback, "Please try again later")
		}
	}

	data := struct {
		Feedback []string
	}{
		Feedback: feedback,
	}

	err := utils.RenderTemplate(w, gic.config.PageContext, data, "signedup.tmpl")

	if err != nil {
		log.Println(err)
		return
	}
}
<|endoftext|>


<commit_before>// Only files including IPv4, IPv6, and Location (in english)
// will be read and parsed into lists.
package parser

import (
	"encoding/binary"
	"encoding/csv"
	"errors"
	"io"
	"log"
	"math"
	"net"
	"reflect"
	"regexp"
	"strconv"
	"strings"
)

const (
	ipNumColumnsGlite2       = 10
	locationNumColumnsGlite2 = 13
	gLite2Prefix             = "GeoLite2-City"

	ipNumColumnsGlite1       = 3
	locationNumColumnsGlite1 = 9
	gLite1Prefix             = "GeoLiteCity"

	mapMax = 200000
)

// IPNode defines IPv4 and IPv6 databases
type IPNode struct {
	IPAddressLow  net.IP
	IPAddressHigh net.IP
	LocationIndex int // Index to slice of locations
	PostalCode    string
	Latitude      float64
	Longitude     float64
}

// LocationNode defines Location databases
type LocationNode struct {
	GeonameID     int
	ContinentCode string
	CountryCode   string
	CountryName   string
	MetroCode     int64
	CityName      string
}

// Creates a List of IPNodes
func CreateIPList(reader io.Reader, idMap map[int]int, file string) ([]IPNode, error) {
	g1IP := []string{"startIpNum", "endIpNum", "locId"}
	list := []IPNode{}
	r := csv.NewReader(reader)
	// Skip first line
	title, err := r.Read()
	if err == io.EOF {
		log.Println("Empty input data")
		return nil, errors.New("Empty input data")
	}
	switch {
	case strings.HasPrefix(file, gLite1Prefix):
		// Skip 2nd line
		title, err = r.Read()
		if err == io.EOF {
			log.Println("Empty input data")
			return nil, errors.New("Empty input data")
		}
		if !reflect.DeepEqual(g1IP, title) {
			log.Println("Improper data format got: ", title, " wanted: ", g1IP)
			return nil, errors.New("Improper data format")
		}
		for {
			// Example:
			// GLite1 : record = [16777216,16777471,17]
			record, err := r.Read()
			if err == io.EOF {
				break
			}
			err = checkColumnLength(record, ipNumColumnsGlite1)
			if err != nil {
				return nil, err
			}
			var newNode IPNode
			newNode.IPAddressLow, err = Int2ip(record[0])
			if err != nil {
				return nil, err
			}
			newNode.IPAddressHigh, err = Int2ip(record[1])
			if err != nil {
				return nil, err
			}
			// Look for GeoId within idMap and return index
			index, err := lookupGeoId(record[2], idMap)
			if err != nil {
				return nil, err
			}
			newNode.LocationIndex = index
			list = append(list, newNode)
		}
	case strings.HasPrefix(file, gLite2Prefix):
		for {
			// Example:
			// GLite2 : record = [2a04:97c0::/29,2658434,2658434,0,0,47,8,100]
			record, err := r.Read()
			if err == io.EOF {
				break
			}
			var newNode IPNode
			err = checkColumnLength(record, ipNumColumnsGlite2)
			if err != nil {
				return nil, err
			}
			lowIp, highIp, err := RangeCIDR(record[0])
			if err != nil {
				return nil, err
			}
			newNode.IPAddressLow = lowIp
			newNode.IPAddressHigh = highIp
			// Look for GeoId within idMap and return index
			index, err := lookupGeoId(record[1], idMap)
			if err != nil {
				index, err = lookupGeoId(record[2], idMap)
			}
			newNode.LocationIndex = index
			newNode.PostalCode = record[6]
			newNode.Latitude, err = stringToFloat(record[7], "Latitude")
			if err != nil {
				return nil, err
			}
			newNode.Longitude, err = stringToFloat(record[8], "Longitude")
			if err != nil {
				return nil, err
			}
			list = append(list, newNode)
		}
	default:
		log.Println("Unaccepted csv file provided: ", file)
		return list, errors.New("Unaccepted csv file provided")
	}
	return list, nil
}

// Verify column length
func checkColumnLength(record []string, size int) error {
	if len(record) != size {
		log.Println("Incorrect number of columns in IP list", size, " got: ", len(record), record)
		return errors.New("Corrupted Data: wrong number of columns")
	}
	return nil
}

// Converts integer to net.IPv4
func Int2ip(str string) (net.IP, error) {
	num, err := strconv.Atoi(str)
	if err != nil {
		log.Println("Provided IP should be a number")
		return nil, errors.New("Inputed string cannot be converted to a number")
	}
	ft := float64(num)
	if ft > math.Pow(2, 32) || num < 1 {
		log.Println("Provided IP should be in the range of 0.0.0.1 and 255.255.255.255 ", str)
	}
	ip := make(net.IP, 4)
	binary.BigEndian.PutUint32(ip, uint32(num))
	ip = net.IPv4(ip[0], ip[1], ip[2], ip[3])
	return ip, nil
}

// Finds the smallest and largest net.IP from a CIDR range
// Example: "1.0.0.0/24" -> 1.0.0.0 , 1.0.0.255
func RangeCIDR(cidr string) (net.IP, net.IP, error) {
	ip, ipnet, err := net.ParseCIDR(cidr)
	if err != nil {
		return nil, nil, errors.New("Invalid CIDR IP range")
	}
	lowIp := make(net.IP, len(ip))
	copy(lowIp, ip)
	mask := ipnet.Mask
	for x, _ := range ip {
		if len(mask) == 4 {
			if x < 12 {
				ip[x] |= 0
			} else {
				ip[x] |= ^mask[x-12]
			}
		} else {
			ip[x] |= ^mask[x]
		}
	}
	return lowIp, ip, nil
}

// Finds provided geonameID within idMap and returns the index in idMap
// locationIdMap := map[int]int{
//	609013: 0,
//	104084: 4,
//	17:     4,
// }
// lookupGeoId("17",locationIdMap) would return (2,nil).
// TODO: Add error metrics
func lookupGeoId(gnid string, idMap map[int]int) (int, error) {
	geonameId, err := strconv.Atoi(gnid)
	if err != nil {
		log.Println("geonameID should be a number")
		return 0, errors.New("Corrupted Data: geonameID should be a number")
	}
	loadIndex, ok := idMap[geonameId]
	if !ok {
		log.Println("geonameID not found ", geonameId)
		return 0, errors.New("Corrupted Data: geonameId not found")
	}
	return loadIndex, nil
}

func stringToFloat(str, field string) (float64, error) {
	flt, err := strconv.ParseFloat(str, 64)
	if err != nil {
		if len(str) > 0 {
			log.Println(field, " was not a number")
			output := strings.Join([]string{"Corrupted Data: ", field, " should be an int"}, "")
			return 0, errors.New(output)
		}
	}
	return flt, nil
}

func checkAllCaps(str, field string) (string, error) {
	match, _ := regexp.MatchString("^[A-Z]*$", str)
	if match {
		return str, nil
	} else {
		log.Println(field, "should be all capitals and no numbers")
		output := strings.Join([]string{"Corrupted Data: ", field, " should be all caps"}, "")
		return "", errors.New(output)

	}
}

// Creates list for location databases
// returns list with location data and a hashmap with index to geonameId
func CreateLocationList(reader io.Reader) ([]LocationNode, map[int]int, error) {
	idMap := make(map[int]int, mapMax)
	list := []LocationNode{}
	r := csv.NewReader(reader)
	r.TrimLeadingSpace = true
	// Skip the first line
	_, err := r.Read()
	if err == io.EOF {
		log.Println("Empty input data")
		return nil, nil, errors.New("Empty input data")
	}
	if err != nil {
		log.Println("Error reading file")
		return nil, nil, errors.New("Error reading file")
	}
	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		}
		if len(record) != locationNumColumnsGlite2 {
			log.Println("Incorrect number of columns in Location list\n\twanted: ", locationNumColumnsGlite2, " got: ", len(record), record)
			return nil, nil, errors.New("Corrupted Data: wrong number of columns")
		}
		var newNode LocationNode
		newNode.GeonameID, err = strconv.Atoi(record[0])
		if err != nil {
			if len(record[0]) > 0 {
				log.Println("GeonameID should be a number ", record[0])
				return nil, nil, errors.New("Corrupted Data: GeonameID should be a number")
			}
		}
		newNode.ContinentCode, err = checkAllCaps(record[2], "Continent code")
		if err != nil {
			return nil, nil, err
		}
		newNode.CountryCode, err = checkAllCaps(record[4], "Country code")
		if err != nil {
			return nil, nil, err
		}
		match, _ := regexp.MatchString(`^[^0-9]*$`, record[5])
		if match {
			newNode.CountryName = record[5]
		} else {
			log.Println("Country name should be letters only : ", record[5])
			return nil, nil, errors.New("Corrupted Data: country name should be letters")
		}
		newNode.MetroCode, err = strconv.ParseInt(record[11], 10, 64)
		if err != nil {
			if len(record[11]) > 0 {
				log.Println("MetroCode should be a number")
				return nil, nil, errors.New("Corrupted Data: metrocode should be a number")
			}
		}
		newNode.CityName = record[10]
		list = append(list, newNode)
		idMap[newNode.GeonameID] = len(list) - 1
	}
	return list, idMap, nil
}

// Returns nil if two nodes are equal
// Used by the search package
func IsEqualIPNodes(expected, node IPNode) error {
	if !((node.IPAddressLow).Equal(expected.IPAddressLow)) {
		output := strings.Join([]string{"IPAddress Low inconsistent\ngot:", node.IPAddressLow.String(), " \nwanted:", expected.IPAddressLow.String()}, "")
		log.Println(output)
		return errors.New(output)
	}
	if !((node.IPAddressHigh).Equal(expected.IPAddressHigh)) {
		output := strings.Join([]string{"IPAddressHigh inconsistent\ngot:", node.IPAddressHigh.String(), " \nwanted:", expected.IPAddressHigh.String()}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.LocationIndex != expected.LocationIndex {
		output := strings.Join([]string{"LocationIndex inconsistent\ngot:", strconv.Itoa(node.LocationIndex), " \nwanted:", strconv.Itoa(expected.LocationIndex)}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.PostalCode != expected.PostalCode {
		output := strings.Join([]string{"PostalCode inconsistent\ngot:", node.PostalCode, " \nwanted:", expected.PostalCode}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.Latitude != expected.Latitude {
		output := strings.Join([]string{"Latitude inconsistent\ngot:", floatToString(node.Latitude), " \nwanted:", floatToString(expected.Latitude)}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.Longitude != expected.Longitude {
		output := strings.Join([]string{"Longitude inconsistent\ngot:", floatToString(node.Longitude), " \nwanted:", floatToString(expected.Longitude)}, "")
		log.Println(output)
		return errors.New(output)
	}
	return nil
}
func floatToString(num float64) string {
	return strconv.FormatFloat(num, 'f', 6, 64)
}
<commit_msg>clarify comment<commit_after>// Only files including IPv4, IPv6, and Location (in english)
// will be read and parsed into lists.
package parser

import (
	"encoding/binary"
	"encoding/csv"
	"errors"
	"io"
	"log"
	"math"
	"net"
	"reflect"
	"regexp"
	"strconv"
	"strings"
)

const (
	ipNumColumnsGlite2       = 10
	locationNumColumnsGlite2 = 13
	gLite2Prefix             = "GeoLite2-City"

	ipNumColumnsGlite1       = 3
	locationNumColumnsGlite1 = 9
	gLite1Prefix             = "GeoLiteCity"

	mapMax = 200000
)

// IPNode defines IPv4 and IPv6 databases
type IPNode struct {
	IPAddressLow  net.IP
	IPAddressHigh net.IP
	LocationIndex int // Index to slice of locations
	PostalCode    string
	Latitude      float64
	Longitude     float64
}

// LocationNode defines Location databases
type LocationNode struct {
	GeonameID     int
	ContinentCode string
	CountryCode   string
	CountryName   string
	MetroCode     int64
	CityName      string
}

// Creates a List of IPNodes
func CreateIPList(reader io.Reader, idMap map[int]int, file string) ([]IPNode, error) {
	g1IP := []string{"startIpNum", "endIpNum", "locId"}
	list := []IPNode{}
	r := csv.NewReader(reader)
	// Skip first line
	title, err := r.Read()
	if err == io.EOF {
		log.Println("Empty input data")
		return nil, errors.New("Empty input data")
	}
	switch {
	case strings.HasPrefix(file, gLite1Prefix):
		// Skip 2nd line, which contains column labels
		title, err = r.Read()
		if err == io.EOF {
			log.Println("Empty input data")
			return nil, errors.New("Empty input data")
		}
		if !reflect.DeepEqual(g1IP, title) {
			log.Println("Improper data format got: ", title, " wanted: ", g1IP)
			return nil, errors.New("Improper data format")
		}
		for {
			// Example:
			// GLite1 : record = [16777216,16777471,17]
			record, err := r.Read()
			if err == io.EOF {
				break
			}
			err = checkColumnLength(record, ipNumColumnsGlite1)
			if err != nil {
				return nil, err
			}
			var newNode IPNode
			newNode.IPAddressLow, err = Int2ip(record[0])
			if err != nil {
				return nil, err
			}
			newNode.IPAddressHigh, err = Int2ip(record[1])
			if err != nil {
				return nil, err
			}
			// Look for GeoId within idMap and return index
			index, err := lookupGeoId(record[2], idMap)
			if err != nil {
				return nil, err
			}
			newNode.LocationIndex = index
			list = append(list, newNode)
		}
	case strings.HasPrefix(file, gLite2Prefix):
		for {
			// Example:
			// GLite2 : record = [2a04:97c0::/29,2658434,2658434,0,0,47,8,100]
			record, err := r.Read()
			if err == io.EOF {
				break
			}
			var newNode IPNode
			err = checkColumnLength(record, ipNumColumnsGlite2)
			if err != nil {
				return nil, err
			}
			lowIp, highIp, err := RangeCIDR(record[0])
			if err != nil {
				return nil, err
			}
			newNode.IPAddressLow = lowIp
			newNode.IPAddressHigh = highIp
			// Look for GeoId within idMap and return index
			index, err := lookupGeoId(record[1], idMap)
			if err != nil {
				index, err = lookupGeoId(record[2], idMap)
			}
			newNode.LocationIndex = index
			newNode.PostalCode = record[6]
			newNode.Latitude, err = stringToFloat(record[7], "Latitude")
			if err != nil {
				return nil, err
			}
			newNode.Longitude, err = stringToFloat(record[8], "Longitude")
			if err != nil {
				return nil, err
			}
			list = append(list, newNode)
		}
	default:
		log.Println("Unaccepted csv file provided: ", file)
		return list, errors.New("Unaccepted csv file provided")
	}
	return list, nil
}

// Verify column length
func checkColumnLength(record []string, size int) error {
	if len(record) != size {
		log.Println("Incorrect number of columns in IP list", size, " got: ", len(record), record)
		return errors.New("Corrupted Data: wrong number of columns")
	}
	return nil
}

// Converts integer to net.IPv4
func Int2ip(str string) (net.IP, error) {
	num, err := strconv.Atoi(str)
	if err != nil {
		log.Println("Provided IP should be a number")
		return nil, errors.New("Inputed string cannot be converted to a number")
	}
	ft := float64(num)
	if ft > math.Pow(2, 32) || num < 1 {
		log.Println("Provided IP should be in the range of 0.0.0.1 and 255.255.255.255 ", str)
	}
	ip := make(net.IP, 4)
	binary.BigEndian.PutUint32(ip, uint32(num))
	ip = net.IPv4(ip[0], ip[1], ip[2], ip[3])
	return ip, nil
}

// Finds the smallest and largest net.IP from a CIDR range
// Example: "1.0.0.0/24" -> 1.0.0.0 , 1.0.0.255
func RangeCIDR(cidr string) (net.IP, net.IP, error) {
	ip, ipnet, err := net.ParseCIDR(cidr)
	if err != nil {
		return nil, nil, errors.New("Invalid CIDR IP range")
	}
	lowIp := make(net.IP, len(ip))
	copy(lowIp, ip)
	mask := ipnet.Mask
	for x, _ := range ip {
		if len(mask) == 4 {
			if x < 12 {
				ip[x] |= 0
			} else {
				ip[x] |= ^mask[x-12]
			}
		} else {
			ip[x] |= ^mask[x]
		}
	}
	return lowIp, ip, nil
}

// Finds provided geonameID within idMap and returns the index in idMap
// locationIdMap := map[int]int{
//	609013: 0,
//	104084: 4,
//	17:     4,
// }
// lookupGeoId("17",locationIdMap) would return (2,nil).
// TODO: Add error metrics
func lookupGeoId(gnid string, idMap map[int]int) (int, error) {
	geonameId, err := strconv.Atoi(gnid)
	if err != nil {
		log.Println("geonameID should be a number")
		return 0, errors.New("Corrupted Data: geonameID should be a number")
	}
	loadIndex, ok := idMap[geonameId]
	if !ok {
		log.Println("geonameID not found ", geonameId)
		return 0, errors.New("Corrupted Data: geonameId not found")
	}
	return loadIndex, nil
}

func stringToFloat(str, field string) (float64, error) {
	flt, err := strconv.ParseFloat(str, 64)
	if err != nil {
		if len(str) > 0 {
			log.Println(field, " was not a number")
			output := strings.Join([]string{"Corrupted Data: ", field, " should be an int"}, "")
			return 0, errors.New(output)
		}
	}
	return flt, nil
}

func checkAllCaps(str, field string) (string, error) {
	match, _ := regexp.MatchString("^[A-Z]*$", str)
	if match {
		return str, nil
	} else {
		log.Println(field, "should be all capitals and no numbers")
		output := strings.Join([]string{"Corrupted Data: ", field, " should be all caps"}, "")
		return "", errors.New(output)

	}
}

// Creates list for location databases
// returns list with location data and a hashmap with index to geonameId
func CreateLocationList(reader io.Reader) ([]LocationNode, map[int]int, error) {
	idMap := make(map[int]int, mapMax)
	list := []LocationNode{}
	r := csv.NewReader(reader)
	r.TrimLeadingSpace = true
	// Skip the first line
	_, err := r.Read()
	if err == io.EOF {
		log.Println("Empty input data")
		return nil, nil, errors.New("Empty input data")
	}
	if err != nil {
		log.Println("Error reading file")
		return nil, nil, errors.New("Error reading file")
	}
	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		}
		if len(record) != locationNumColumnsGlite2 {
			log.Println("Incorrect number of columns in Location list\n\twanted: ", locationNumColumnsGlite2, " got: ", len(record), record)
			return nil, nil, errors.New("Corrupted Data: wrong number of columns")
		}
		var newNode LocationNode
		newNode.GeonameID, err = strconv.Atoi(record[0])
		if err != nil {
			if len(record[0]) > 0 {
				log.Println("GeonameID should be a number ", record[0])
				return nil, nil, errors.New("Corrupted Data: GeonameID should be a number")
			}
		}
		newNode.ContinentCode, err = checkAllCaps(record[2], "Continent code")
		if err != nil {
			return nil, nil, err
		}
		newNode.CountryCode, err = checkAllCaps(record[4], "Country code")
		if err != nil {
			return nil, nil, err
		}
		match, _ := regexp.MatchString(`^[^0-9]*$`, record[5])
		if match {
			newNode.CountryName = record[5]
		} else {
			log.Println("Country name should be letters only : ", record[5])
			return nil, nil, errors.New("Corrupted Data: country name should be letters")
		}
		newNode.MetroCode, err = strconv.ParseInt(record[11], 10, 64)
		if err != nil {
			if len(record[11]) > 0 {
				log.Println("MetroCode should be a number")
				return nil, nil, errors.New("Corrupted Data: metrocode should be a number")
			}
		}
		newNode.CityName = record[10]
		list = append(list, newNode)
		idMap[newNode.GeonameID] = len(list) - 1
	}
	return list, idMap, nil
}

// Returns nil if two nodes are equal
// Used by the search package
func IsEqualIPNodes(expected, node IPNode) error {
	if !((node.IPAddressLow).Equal(expected.IPAddressLow)) {
		output := strings.Join([]string{"IPAddress Low inconsistent\ngot:", node.IPAddressLow.String(), " \nwanted:", expected.IPAddressLow.String()}, "")
		log.Println(output)
		return errors.New(output)
	}
	if !((node.IPAddressHigh).Equal(expected.IPAddressHigh)) {
		output := strings.Join([]string{"IPAddressHigh inconsistent\ngot:", node.IPAddressHigh.String(), " \nwanted:", expected.IPAddressHigh.String()}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.LocationIndex != expected.LocationIndex {
		output := strings.Join([]string{"LocationIndex inconsistent\ngot:", strconv.Itoa(node.LocationIndex), " \nwanted:", strconv.Itoa(expected.LocationIndex)}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.PostalCode != expected.PostalCode {
		output := strings.Join([]string{"PostalCode inconsistent\ngot:", node.PostalCode, " \nwanted:", expected.PostalCode}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.Latitude != expected.Latitude {
		output := strings.Join([]string{"Latitude inconsistent\ngot:", floatToString(node.Latitude), " \nwanted:", floatToString(expected.Latitude)}, "")
		log.Println(output)
		return errors.New(output)
	}
	if node.Longitude != expected.Longitude {
		output := strings.Join([]string{"Longitude inconsistent\ngot:", floatToString(node.Longitude), " \nwanted:", floatToString(expected.Longitude)}, "")
		log.Println(output)
		return errors.New(output)
	}
	return nil
}
func floatToString(num float64) string {
	return strconv.FormatFloat(num, 'f', 6, 64)
}
<|endoftext|>


<commit_before>// Copyright 2016 The clang-server Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package parser

import (
	"encoding/binary"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/go-clang/v3.9/clang"
	"github.com/pkg/errors"
	"github.com/pkgutil/osutil"
	"github.com/pkgutil/stringsutil"
	"github.com/zchee/clang-server/compilationdatabase"
	"github.com/zchee/clang-server/indexdb"
	"github.com/zchee/clang-server/internal/log"
	"github.com/zchee/clang-server/internal/pathutil"
	"github.com/zchee/clang-server/parser/builtinheader"
	"github.com/zchee/clang-server/symbol"
)

// defaultClangOption defalut global clang options.
// clang.TranslationUnit_DetailedPreprocessingRecord = 0x01
// clang.TranslationUnit_Incomplete = 0x02
// clang.TranslationUnit_PrecompiledPreamble = 0x04
// clang.TranslationUnit_CacheCompletionResults = 0x08
// clang.TranslationUnit_ForSerialization = 0x10
// clang.TranslationUnit_CXXChainedPCH = 0x20
// clang.TranslationUnit_SkipFunctionBodies = 0x40
// clang.TranslationUnit_IncludeBriefCommentsInCodeCompletion = 0x80
// clang.TranslationUnit_CreatePreambleOnFirstParse = 0x100
// clang.TranslationUnit_KeepGoing = 0x200
// const defaultClangOption uint32 = 0x445 // Use all flags for now
var defaultClangOption = clang.DefaultEditingTranslationUnitOptions() | uint32(clang.TranslationUnit_KeepGoing)

// Parser represents a C/C++ AST parser.
type Parser struct {
	root        string
	clangOption uint32

	idx clang.Index
	cd  *compilationdatabase.CompilationDatabase
	db  *indexdb.IndexDB

	dispatcher *dispatcher

	debugUncatched bool                     // for debug
	uncachedKind   map[clang.CursorKind]int // for debug
}

// Config represents a parser config.
type Config struct {
	JSONName    string
	PathRange   []string
	ClangOption uint32

	Debug bool
}

// NewParser return the new Parser.
func NewParser(path string, config Config) *Parser {
	root, err := pathutil.FindProjectRoot(path)
	if err != nil {
		log.Fatal(err)
	}

	cd := compilationdatabase.NewCompilationDatabase(root)
	if err := cd.Parse(config.JSONName, config.PathRange); err != nil {
		log.Fatal(err)
	}

	db, err := indexdb.NewIndexDB(root)
	if err != nil {
		log.Fatal(err)
	}

	clangOption := config.ClangOption
	if clangOption == 0 {
		clangOption = defaultClangOption
	}

	p := &Parser{
		root:        root,
		clangOption: clangOption,
		idx:         clang.NewIndex(0, 1), // disable excludeDeclarationsFromPCH, enable displayDiagnostics
		cd:          cd,
		db:          db,
	}

	if config.Debug {
		p.debugUncatched = true
		p.uncachedKind = make(map[clang.CursorKind]int)
	}

	if err := CreateBulitinHeaders(); err != nil {
		log.Fatal(err)
	}

	return p
}

// CreateBulitinHeaders creates(dumps) a clang builtin header to cache directory.
func CreateBulitinHeaders() error {
	builtinHdrDir := filepath.Join(pathutil.CacheDir(), "clang", "include")
	if !osutil.IsExist(builtinHdrDir) {
		if err := os.MkdirAll(builtinHdrDir, 0700); err != nil {
			return errors.WithStack(err)
		}
	}

	for _, fname := range builtinheader.AssetNames() {
		data, err := builtinheader.AssetInfo(fname)
		if err != nil {
			return errors.WithStack(err)
		}

		if strings.Contains(data.Name(), string(filepath.Separator)) {
			dir, _ := filepath.Split(data.Name())
			if err := os.MkdirAll(filepath.Join(builtinHdrDir, dir), 0700); err != nil {
				return errors.WithStack(err)
			}
		}

		buf, err := builtinheader.Asset(data.Name())
		if err != nil {
			return errors.WithStack(err)
		}

		if err := ioutil.WriteFile(filepath.Join(builtinHdrDir, data.Name()), buf, 0600); err != nil {
			return errors.WithStack(err)
		}
	}

	return nil
}

// Parse parses the project directories.
func (p *Parser) Parse() {
	defer func() {
		p.db.Close()
		symbol.Serve()
	}()

	ccs := p.cd.CompileCommands()
	if len(ccs) == 0 {
		log.Fatal("not walk")
	}

	compilerConfig := p.cd.CompilerConfig
	flags := append(compilerConfig.SystemCIncludeDir, compilerConfig.SystemFrameworkDir...)

	// TODO(zchee): needs include stdint.h?
	if i := stringsutil.IndexContainsSlice(ccs[0].Arguments, "-std="); i > 0 {
		std := ccs[0].Arguments[i][5:]
		switch {
		case strings.HasPrefix(std, "c"), strings.HasPrefix(std, "gnu"):
			if std[len(std)-2] == '8' || std[len(std)-2] == '9' {
				flags = append(flags, "-include", "/usr/include/stdint.h")
			}
		}
	} else {
		flags = append(flags, "-include", "/usr/include/stdint.h")
	}
	if !(filepath.Ext(ccs[0].File) == ".c") {
		flags = append(flags, compilerConfig.SystemCXXIncludeDir...)
	}

	builtinHdrDir := filepath.Join(pathutil.CacheDir(), "clang", "include")
	flags = append(flags, "-I"+builtinHdrDir)

	p.dispatcher = newDispatcher(p.ParseFile)
	p.dispatcher.Start()
	for i := 0; i < len(ccs); i++ {
		args := ccs[i].Arguments
		args = append(flags, args...)
		p.dispatcher.Add(parseArg{ccs[i].File, args})
	}
	p.dispatcher.Wait()
}

type parseArg struct {
	filename string
	flag     []string
}

// ParseFile parses the C/C++ file.
func (p *Parser) ParseFile(arg parseArg) error {
	var tu clang.TranslationUnit

	if p.db.Has(arg.filename) {
		buf, err := p.db.Get(arg.filename)
		if err != nil {
			return err
		}

		tu, err = deserializeTranslationUnit(p.idx, buf)
		if err != nil {
			return err
		}
		defer tu.Dispose()

		log.Debugf("tu.Spelling(): %T => %+v\n", tu.Spelling(), tu.Spelling()) // for debug

		return nil
	}

	if cErr := p.idx.ParseTranslationUnit2(arg.filename, arg.flag, nil, p.clangOption, &tu); clang.ErrorCode(cErr) != clang.Error_Success {
		return errors.New(clang.ErrorCode(cErr).Spelling())
	}
	defer tu.Dispose()

	tuch := make(chan []byte)
	go func() {
		tuch <- serializeTranslationUnit(arg.filename, tu)
	}()

	printDiagnostics(tu.Diagnostics())

	rootCursor := tu.TranslationUnitCursor()
	file := symbol.NewFile(arg.filename)
	visitNode := func(cursor, parent clang.Cursor) clang.ChildVisitResult {
		if cursor.IsNull() {
			log.Debug("cursor: <none>")
			return clang.ChildVisit_Continue
		}

		cursorLoc := symbol.FromCursor(cursor)
		if cursorLoc.FileName() == "" || cursorLoc.FileName() == "." {
			// TODO(zchee): Ignore system header(?)
			return clang.ChildVisit_Continue
		}

		kind := cursor.Kind()
		switch kind {
		case clang.Cursor_FunctionDecl, clang.Cursor_StructDecl, clang.Cursor_FieldDecl, clang.Cursor_TypedefDecl, clang.Cursor_EnumDecl, clang.Cursor_EnumConstantDecl:
			defCursor := cursor.Definition()
			if defCursor.IsNull() {
				file.AddDecl(cursorLoc)
			} else {
				defLoc := symbol.FromCursor(defCursor)
				file.AddDefinition(cursorLoc, defLoc)
			}
		case clang.Cursor_MacroDefinition:
			file.AddDefinition(cursorLoc, cursorLoc)
		case clang.Cursor_VarDecl:
			file.AddDecl(cursorLoc)
		case clang.Cursor_ParmDecl:
			if cursor.Spelling() != "" {
				file.AddDecl(cursorLoc)
			}
		case clang.Cursor_CallExpr:
			refCursor := cursor.Referenced()
			refLoc := symbol.FromCursor(refCursor)
			file.AddCaller(cursorLoc, refLoc, true)
		case clang.Cursor_DeclRefExpr, clang.Cursor_TypeRef, clang.Cursor_MemberRefExpr, clang.Cursor_MacroExpansion:
			refCursor := cursor.Referenced()
			refLoc := symbol.FromCursor(refCursor)
			file.AddCaller(cursorLoc, refLoc, false)
		case clang.Cursor_InclusionDirective:
			incFile := cursor.IncludedFile()
			file.AddHeader(cursor.Spelling(), incFile)
		default:
			if p.debugUncatched {
				p.uncachedKind[kind]++
			}
		}

		return clang.ChildVisit_Recurse
	}

	rootCursor.Visit(visitNode)
	file.AddTranslationUnit(<-tuch)
	buf := file.Serialize()

	out := symbol.GetRootAsFile(buf.FinishedBytes(), 0)
	printFile(out) // for debug

	log.Debugf("Goroutine:%d", runtime.NumGoroutine())
	log.Debugf("\n================== DONE: filename: %+v ==================\n\n\n", arg.filename)

	return p.db.Put(arg.filename, buf.FinishedBytes())
}

// serializeTranslationUnit selialize the TranslationUnit to Clang serialized representation.
// TODO(zchee): Avoid ioutil.TempFile if possible.
func serializeTranslationUnit(filename string, tu clang.TranslationUnit) []byte {
	tmpFile, err := ioutil.TempFile(os.TempDir(), filepath.Base(filename))
	if err != nil {
		log.Fatal(err)
	}

	saveOptions := uint32(clang.TranslationUnit_KeepGoing)
	if cErr := tu.SaveTranslationUnit(tmpFile.Name(), saveOptions); clang.SaveError(cErr) != clang.SaveError_None {
		log.Fatal(clang.SaveError(cErr))
	}

	buf, err := ioutil.ReadFile(tmpFile.Name())
	if err != nil {
		log.Fatal(err)
	}
	os.Remove(tmpFile.Name())

	return buf
}

func deserializeTranslationUnit(idx clang.Index, buf []byte) (clang.TranslationUnit, error) {
	var tu clang.TranslationUnit

	tmpfile, err := ioutil.TempFile(os.TempDir(), "clang-server")
	if err != nil {
		return tu, err
	}
	binary.Write(tmpfile, binary.LittleEndian, buf)

	if err := idx.TranslationUnit2(tmpfile.Name(), &tu); clang.ErrorCode(err) != clang.Error_Success {
		return tu, errors.New(err.Spelling())
	}
	// finished create a translation unit from an AST file, remove tmpfile
	os.Remove(tmpfile.Name())

	return tu, nil
}

// ClangVersion return the current clang version.
func ClangVersion() string {
	return clang.GetClangVersion()
}
<commit_msg>parser: fix TranslationUnit chan(tuch) size to 1<commit_after>// Copyright 2016 The clang-server Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package parser

import (
	"encoding/binary"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/go-clang/v3.9/clang"
	"github.com/pkg/errors"
	"github.com/pkgutil/osutil"
	"github.com/pkgutil/stringsutil"
	"github.com/zchee/clang-server/compilationdatabase"
	"github.com/zchee/clang-server/indexdb"
	"github.com/zchee/clang-server/internal/log"
	"github.com/zchee/clang-server/internal/pathutil"
	"github.com/zchee/clang-server/parser/builtinheader"
	"github.com/zchee/clang-server/symbol"
)

// defaultClangOption defalut global clang options.
// clang.TranslationUnit_DetailedPreprocessingRecord = 0x01
// clang.TranslationUnit_Incomplete = 0x02
// clang.TranslationUnit_PrecompiledPreamble = 0x04
// clang.TranslationUnit_CacheCompletionResults = 0x08
// clang.TranslationUnit_ForSerialization = 0x10
// clang.TranslationUnit_CXXChainedPCH = 0x20
// clang.TranslationUnit_SkipFunctionBodies = 0x40
// clang.TranslationUnit_IncludeBriefCommentsInCodeCompletion = 0x80
// clang.TranslationUnit_CreatePreambleOnFirstParse = 0x100
// clang.TranslationUnit_KeepGoing = 0x200
// const defaultClangOption uint32 = 0x445 // Use all flags for now
var defaultClangOption = clang.DefaultEditingTranslationUnitOptions() | uint32(clang.TranslationUnit_KeepGoing)

// Parser represents a C/C++ AST parser.
type Parser struct {
	root        string
	clangOption uint32

	idx clang.Index
	cd  *compilationdatabase.CompilationDatabase
	db  *indexdb.IndexDB

	dispatcher *dispatcher

	debugUncatched bool                     // for debug
	uncachedKind   map[clang.CursorKind]int // for debug
}

// Config represents a parser config.
type Config struct {
	JSONName    string
	PathRange   []string
	ClangOption uint32

	Debug bool
}

// NewParser return the new Parser.
func NewParser(path string, config Config) *Parser {
	root, err := pathutil.FindProjectRoot(path)
	if err != nil {
		log.Fatal(err)
	}

	cd := compilationdatabase.NewCompilationDatabase(root)
	if err := cd.Parse(config.JSONName, config.PathRange); err != nil {
		log.Fatal(err)
	}

	db, err := indexdb.NewIndexDB(root)
	if err != nil {
		log.Fatal(err)
	}

	clangOption := config.ClangOption
	if clangOption == 0 {
		clangOption = defaultClangOption
	}

	p := &Parser{
		root:        root,
		clangOption: clangOption,
		idx:         clang.NewIndex(0, 1), // disable excludeDeclarationsFromPCH, enable displayDiagnostics
		cd:          cd,
		db:          db,
	}

	if config.Debug {
		p.debugUncatched = true
		p.uncachedKind = make(map[clang.CursorKind]int)
	}

	if err := CreateBulitinHeaders(); err != nil {
		log.Fatal(err)
	}

	return p
}

// CreateBulitinHeaders creates(dumps) a clang builtin header to cache directory.
func CreateBulitinHeaders() error {
	builtinHdrDir := filepath.Join(pathutil.CacheDir(), "clang", "include")
	if !osutil.IsExist(builtinHdrDir) {
		if err := os.MkdirAll(builtinHdrDir, 0700); err != nil {
			return errors.WithStack(err)
		}
	}

	for _, fname := range builtinheader.AssetNames() {
		data, err := builtinheader.AssetInfo(fname)
		if err != nil {
			return errors.WithStack(err)
		}

		if strings.Contains(data.Name(), string(filepath.Separator)) {
			dir, _ := filepath.Split(data.Name())
			if err := os.MkdirAll(filepath.Join(builtinHdrDir, dir), 0700); err != nil {
				return errors.WithStack(err)
			}
		}

		buf, err := builtinheader.Asset(data.Name())
		if err != nil {
			return errors.WithStack(err)
		}

		if err := ioutil.WriteFile(filepath.Join(builtinHdrDir, data.Name()), buf, 0600); err != nil {
			return errors.WithStack(err)
		}
	}

	return nil
}

// Parse parses the project directories.
func (p *Parser) Parse() {
	defer func() {
		p.db.Close()
		symbol.Serve()
	}()

	ccs := p.cd.CompileCommands()
	if len(ccs) == 0 {
		log.Fatal("not walk")
	}

	compilerConfig := p.cd.CompilerConfig
	flags := append(compilerConfig.SystemCIncludeDir, compilerConfig.SystemFrameworkDir...)

	// TODO(zchee): needs include stdint.h?
	if i := stringsutil.IndexContainsSlice(ccs[0].Arguments, "-std="); i > 0 {
		std := ccs[0].Arguments[i][5:]
		switch {
		case strings.HasPrefix(std, "c"), strings.HasPrefix(std, "gnu"):
			if std[len(std)-2] == '8' || std[len(std)-2] == '9' {
				flags = append(flags, "-include", "/usr/include/stdint.h")
			}
		}
	} else {
		flags = append(flags, "-include", "/usr/include/stdint.h")
	}
	if !(filepath.Ext(ccs[0].File) == ".c") {
		flags = append(flags, compilerConfig.SystemCXXIncludeDir...)
	}

	builtinHdrDir := filepath.Join(pathutil.CacheDir(), "clang", "include")
	flags = append(flags, "-I"+builtinHdrDir)

	p.dispatcher = newDispatcher(p.ParseFile)
	p.dispatcher.Start()
	for i := 0; i < len(ccs); i++ {
		args := ccs[i].Arguments
		args = append(flags, args...)
		p.dispatcher.Add(parseArg{ccs[i].File, args})
	}
	p.dispatcher.Wait()
}

type parseArg struct {
	filename string
	flag     []string
}

// ParseFile parses the C/C++ file.
func (p *Parser) ParseFile(arg parseArg) error {
	var tu clang.TranslationUnit

	if p.db.Has(arg.filename) {
		buf, err := p.db.Get(arg.filename)
		if err != nil {
			return err
		}

		tu, err = deserializeTranslationUnit(p.idx, buf)
		if err != nil {
			return err
		}
		defer tu.Dispose()

		log.Debugf("tu.Spelling(): %T => %+v\n", tu.Spelling(), tu.Spelling()) // for debug

		return nil
	}

	if cErr := p.idx.ParseTranslationUnit2(arg.filename, arg.flag, nil, p.clangOption, &tu); clang.ErrorCode(cErr) != clang.Error_Success {
		return errors.New(clang.ErrorCode(cErr).Spelling())
	}
	defer tu.Dispose()

	tuch := make(chan []byte, 1)
	go func() {
		tuch <- serializeTranslationUnit(arg.filename, tu)
	}()

	printDiagnostics(tu.Diagnostics())

	rootCursor := tu.TranslationUnitCursor()
	file := symbol.NewFile(arg.filename)
	visitNode := func(cursor, parent clang.Cursor) clang.ChildVisitResult {
		if cursor.IsNull() {
			log.Debug("cursor: <none>")
			return clang.ChildVisit_Continue
		}

		cursorLoc := symbol.FromCursor(cursor)
		if cursorLoc.FileName() == "" || cursorLoc.FileName() == "." {
			// TODO(zchee): Ignore system header(?)
			return clang.ChildVisit_Continue
		}

		kind := cursor.Kind()
		switch kind {
		case clang.Cursor_FunctionDecl, clang.Cursor_StructDecl, clang.Cursor_FieldDecl, clang.Cursor_TypedefDecl, clang.Cursor_EnumDecl, clang.Cursor_EnumConstantDecl:
			defCursor := cursor.Definition()
			if defCursor.IsNull() {
				file.AddDecl(cursorLoc)
			} else {
				defLoc := symbol.FromCursor(defCursor)
				file.AddDefinition(cursorLoc, defLoc)
			}
		case clang.Cursor_MacroDefinition:
			file.AddDefinition(cursorLoc, cursorLoc)
		case clang.Cursor_VarDecl:
			file.AddDecl(cursorLoc)
		case clang.Cursor_ParmDecl:
			if cursor.Spelling() != "" {
				file.AddDecl(cursorLoc)
			}
		case clang.Cursor_CallExpr:
			refCursor := cursor.Referenced()
			refLoc := symbol.FromCursor(refCursor)
			file.AddCaller(cursorLoc, refLoc, true)
		case clang.Cursor_DeclRefExpr, clang.Cursor_TypeRef, clang.Cursor_MemberRefExpr, clang.Cursor_MacroExpansion:
			refCursor := cursor.Referenced()
			refLoc := symbol.FromCursor(refCursor)
			file.AddCaller(cursorLoc, refLoc, false)
		case clang.Cursor_InclusionDirective:
			incFile := cursor.IncludedFile()
			file.AddHeader(cursor.Spelling(), incFile)
		default:
			if p.debugUncatched {
				p.uncachedKind[kind]++
			}
		}

		return clang.ChildVisit_Recurse
	}

	rootCursor.Visit(visitNode)
	file.AddTranslationUnit(<-tuch)
	buf := file.Serialize()

	out := symbol.GetRootAsFile(buf.FinishedBytes(), 0)
	printFile(out) // for debug

	log.Debugf("Goroutine:%d", runtime.NumGoroutine())
	log.Debugf("\n================== DONE: filename: %+v ==================\n\n\n", arg.filename)

	return p.db.Put(arg.filename, buf.FinishedBytes())
}

// serializeTranslationUnit selialize the TranslationUnit to Clang serialized representation.
// TODO(zchee): Avoid ioutil.TempFile if possible.
func serializeTranslationUnit(filename string, tu clang.TranslationUnit) []byte {
	tmpFile, err := ioutil.TempFile(os.TempDir(), filepath.Base(filename))
	if err != nil {
		log.Fatal(err)
	}

	saveOptions := uint32(clang.TranslationUnit_KeepGoing)
	if cErr := tu.SaveTranslationUnit(tmpFile.Name(), saveOptions); clang.SaveError(cErr) != clang.SaveError_None {
		log.Fatal(clang.SaveError(cErr))
	}

	buf, err := ioutil.ReadFile(tmpFile.Name())
	if err != nil {
		log.Fatal(err)
	}
	os.Remove(tmpFile.Name())

	return buf
}

func deserializeTranslationUnit(idx clang.Index, buf []byte) (clang.TranslationUnit, error) {
	var tu clang.TranslationUnit

	tmpfile, err := ioutil.TempFile(os.TempDir(), "clang-server")
	if err != nil {
		return tu, err
	}
	binary.Write(tmpfile, binary.LittleEndian, buf)

	if err := idx.TranslationUnit2(tmpfile.Name(), &tu); clang.ErrorCode(err) != clang.Error_Success {
		return tu, errors.New(err.Spelling())
	}
	// finished create a translation unit from an AST file, remove tmpfile
	os.Remove(tmpfile.Name())

	return tu, nil
}

// ClangVersion return the current clang version.
func ClangVersion() string {
	return clang.GetClangVersion()
}
<|endoftext|>


<commit_before>// Copyright 2015-2016 Sevki <s@sevki.org>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package parser // import "sevki.org/build/parser"

import (
	"errors"
	"fmt"
	"io"

	"sevki.org/build/token"

	"sevki.org/build/ast"
	"sevki.org/build/lexer"
)

var (
	ErrConsumption = errors.New("consumption error")
)

type Parser struct {
	name    string
	Path    string
	lexer   *lexer.Lexer
	Decls   chan ast.Decl
	state   stateFn
	peekTok token.Token
	curTok  token.Token
	Error   error
}

func (p *Parser) peek() token.Token {
	return p.peekTok
}

func (p *Parser) next() token.Token {
IGNORETOKEN:
	t := <-p.lexer.Tokens

	switch t.Type {
	case token.Error:
		p.errorf("%q", t)
	case token.Newline:
		goto IGNORETOKEN
	}

	tok := p.peekTok
	p.peekTok = t
	p.curTok = tok

	return tok
}

func (p *Parser) errorf(format string, args ...interface{}) error {
	p.curTok = token.Token{Type: token.Error}
	p.peekTok = token.Token{Type: token.EOF}
	p.Error = fmt.Errorf(format, args...)
	return p.Error
}

func New(name, path string, r io.Reader) *Parser {
	p := &Parser{
		name:  name,
		Path:  path,
		lexer: lexer.New(name, r),
		Decls: make(chan ast.Decl),
	}
	return p
}

func (p *Parser) Run() {
	p.next()
	for p.state = parseDecl; p.state != nil; {
		p.state = p.state(p)
	}
	p.Decls <- nil
	close(p.Decls)

}

type stateFn func(*Parser) stateFn

func parseDecl(p *Parser) stateFn {
	switch p.peek().Type {
	case token.Func:
		return parseFunc
	case token.String:
		return parseVar
	}
	return nil
}

func parseFunc(p *Parser) stateFn {
	if f, err := p.consumeFunc(); err != nil {
		p.Error = err
		return nil
	} else {
		p.Decls <- f
	}
	return parseDecl
}

func parseVar(p *Parser) stateFn {
	t := p.next()

	if err := p.expects(t, token.String); err != nil {
		p.Error = err
		return nil
	}
	if err := p.expects(p.next(), token.Equal); err != nil {
		p.Error = err
		return nil
	}

	switch p.peek().Type {
	case token.LeftBrac, token.LeftCurly, token.String, token.Quote, token.True, token.False, token.Func:
		if n, err := p.consumeNode(); err != nil {
			return nil
		} else {
			a := ast.Assignment{
				Key:   t.String(),
				Value: n,
			}
			a.SetEnd(t)
			a.SetStart(p.curTok)
			p.Decls <- &a
		}
	}

	return parseDecl
}

func (p *Parser) consumeNode() (interface{}, error) {
	var r interface{}
	var err error

	switch p.peek().Type {
	case token.Quote:
		r, err = ast.NewBasicLit(p.next()), nil
	case token.True:
		r, err = ast.NewBasicLit(p.next()), nil
	case token.False:
		r, err = ast.NewBasicLit(p.next()), nil
	case token.String:
		r, err = ast.Variable{Key: p.next().String()}, nil
	case token.LeftBrac:
		r, err = p.consumeSlice()
	case token.LeftCurly:
		r, err = p.consumeMap()
	case token.Func:
		r, err = p.consumeFunc()
	default:
		return nil, ErrConsumption
	}
REPROCESS:
	switch p.peek().Type {
	case token.Plus:
		r, err = p.consumeAddFunc(r)
		goto REPROCESS
	case token.LeftBrac:
		r, err = p.consumeSliceFunc(r)
		goto REPROCESS
	}
	return r, err
}

func (p *Parser) consumeAddFunc(v interface{}) (*ast.Func, error) {
	f := &ast.Func{
		Name: "addition",
	}

	f.File = p.name
	f.SetStart(p.curTok)

	f.AnonParams = []interface{}{v}

	for p.peek().Type == token.Plus {

		p.next()
		switch p.peek().Type {
		case token.String:
			f.AnonParams = append(
				f.AnonParams,
				ast.Variable{Key: p.next().String()},
			)
		case token.Quote:
			f.AnonParams = append(
				f.AnonParams,
				p.next().String(),
			)
		}
	}

	f.SetEnd(p.curTok)
	return f, nil
}

func (p *Parser) consumeSliceFunc(v interface{}) (*ast.Func, error) {

	f := &ast.Func{
		Params: make(map[string]interface{}),
	}
	f.File = p.name
	f.SetStart(p.curTok)

	// advance [
	p.next()

	f.Params["var"] = v
	if p.peek().Type == token.Colon {
		// advance :
		p.next()
		f.Name = "slice"
		f.Params["start"] = 0
		node, err := p.consumeNode()
		if err != nil {
			return nil, err
		}
		f.Params["end"] = node
		goto END
	} else if p.peek().Type == token.Int {
		node, err := p.consumeNode()
		if err != nil {
			return nil, err
		}

		if p.peek().Type == token.RightBrac {
			f.Name = "index"
			f.Params["index"] = node
			goto END
		} else if p.peek().Type == token.Colon {
			// advance :
			p.next()
		} else {
			return nil, fmt.Errorf("this is a malformed slice")
		}
		f.Name = "slice"

		node, err = p.consumeNode()
		if err != nil {
			return nil, err
		}

		f.Params["start"] = node
		if p.peek().Type == token.Int {
			node, err = p.consumeNode()
			if err != nil {
				return nil, err
			}
			f.Params["end"] = node
		} else if p.peek().Type == token.RightBrac {
			goto END
		} else {
			return nil, fmt.Errorf("this is a malformed slice")
		}

	}
END:
	// advance ]
	f.SetEnd(p.next())
	return f, nil
}
func (p *Parser) consumeParams(f *ast.Func) error {
	for {
		switch p.peek().Type {
		case token.Quote, token.LeftBrac, token.Func:
			if n, err := p.consumeNode(); err != nil {
				return err
			} else {
				f.AnonParams = append(f.AnonParams, n)
			}
		case token.String:
			t := p.next()
			if f.Params == nil {
				f.Params = make(map[string]interface{})
			}

			if err := p.expects(p.peek(), token.Colon, token.Equal); err == nil {
				switch p.next().Type {
				case token.Colon:
				case token.Equal:
					if n, err := p.consumeNode(); err != nil {
						return err
					} else {
						f.Params[t.String()] = n
					}
				}
			} else {
				return err
			}
		default:
			return ErrConsumption
		}

		if err := p.expects(p.peek(), token.RightParen, token.Comma); err == nil {
		DANGLING_COMMA:
			switch p.peek().Type {
			case token.RightParen:
				p.next()
				return nil
			case token.Comma:
				p.next()
				if p.peek().Type == token.RightParen {
					goto DANGLING_COMMA
				}
				continue
			}
		} else {
			return err
		}

	}
}
func (p *Parser) consumeMap() (*ast.Map, error) {
	t := p.next()
	_map := ast.Map{
		Value: make(map[string]interface{}),
	}
	_map.SetStart(t)

	for p.peek().Type != token.RightCurly {
		t := p.next()
		if err := p.expects(t, token.Quote); err != nil {
			return nil, err
		}
		if err := p.expects(p.next(), token.Colon); err != nil {
			return nil, err
		}

		if n, err := p.consumeNode(); err != nil {
			return nil, err
		} else {
			_map.Value[t.String()] = n
		}
		if p.peek().Type == token.Comma {
			p.next()
		} else if err := p.expects(p.peek(), token.RightCurly); err != nil {
			return nil, err
		}
	}

	// advance }

	_map.SetEnd(p.next())
	return &_map, nil
}
func (p *Parser) consumeFunc() (*ast.Func, error) {
	t := p.next()
	if err := p.expects(t, token.Func); err != nil {
		return nil, err
	}
	f := ast.Func{
		Name: t.String(),
	}

	f.File = p.name
	f.Start = ast.Position{
		Line:  t.Line,
		Index: t.Start,
	}

	t = p.next()
	if err := p.expects(t, token.LeftParen); err != nil {
		return nil, err
	}
	p.consumeParams(&f)
	return &f, nil
}

func (p *Parser) consumeSlice() (ast.Slice, error) {
	var _slice ast.Slice

	if err := p.expects(p.peek(), token.LeftBrac); err != nil {
		return _slice, err
	} else {
		_slice.SetStart(p.next())
	}

	for p.peek().Type != token.RightBrac {
		_slice.Slice = append(_slice.Slice, p.next().String())
		if p.peek().Type == token.Comma {
			p.next()
		} else if err := p.expects(p.peek(), token.RightBrac); err != nil {
			return _slice, err
		}
	}

	// advance ]
	_slice.SetEnd(p.next())

	return _slice, nil
}
<commit_msg>parser: slice consumes nodes<commit_after>// Copyright 2015-2016 Sevki <s@sevki.org>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package parser // import "sevki.org/build/parser"

import (
	"errors"
	"fmt"
	"io"

	"sevki.org/build/token"

	"sevki.org/build/ast"
	"sevki.org/build/lexer"
)

var (
	ErrConsumption = errors.New("consumption error")
)

type Parser struct {
	name    string
	Path    string
	lexer   *lexer.Lexer
	Decls   chan ast.Decl
	state   stateFn
	peekTok token.Token
	curTok  token.Token
	Error   error
}

func (p *Parser) peek() token.Token {
	return p.peekTok
}

func (p *Parser) next() token.Token {
IGNORETOKEN:
	t := <-p.lexer.Tokens

	switch t.Type {
	case token.Error:
		p.errorf("%q", t)
	case token.Newline:
		goto IGNORETOKEN
	}

	tok := p.peekTok
	p.peekTok = t
	p.curTok = tok

	return tok
}

func (p *Parser) errorf(format string, args ...interface{}) error {
	p.curTok = token.Token{Type: token.Error}
	p.peekTok = token.Token{Type: token.EOF}
	p.Error = fmt.Errorf(format, args...)
	return p.Error
}

func New(name, path string, r io.Reader) *Parser {
	p := &Parser{
		name:  name,
		Path:  path,
		lexer: lexer.New(name, r),
		Decls: make(chan ast.Decl),
	}
	return p
}

func (p *Parser) Run() {
	p.next()
	for p.state = parseDecl; p.state != nil; {
		p.state = p.state(p)
	}
	p.Decls <- nil
	close(p.Decls)

}

type stateFn func(*Parser) stateFn

func parseDecl(p *Parser) stateFn {
	switch p.peek().Type {
	case token.Func:
		return parseFunc
	case token.String:
		return parseVar
	}
	return nil
}

func parseFunc(p *Parser) stateFn {
	if f, err := p.consumeFunc(); err != nil {
		p.Error = err
		return nil
	} else {
		p.Decls <- f
	}
	return parseDecl
}

func parseVar(p *Parser) stateFn {
	t := p.next()

	if err := p.expects(t, token.String); err != nil {
		p.Error = err
		return nil
	}
	if err := p.expects(p.next(), token.Equal); err != nil {
		p.Error = err
		return nil
	}

	switch p.peek().Type {
	case token.LeftBrac, token.LeftCurly, token.String, token.Quote, token.True, token.False, token.Func:
		if n, err := p.consumeNode(); err != nil {
			return nil
		} else {
			a := ast.Assignment{
				Key:   t.String(),
				Value: n,
			}
			a.SetEnd(t)
			a.SetStart(p.curTok)
			p.Decls <- &a
		}
	}

	return parseDecl
}

func (p *Parser) consumeNode() (interface{}, error) {
	var r interface{}
	var err error

	switch p.peek().Type {
	case token.Quote:
		r, err = ast.NewBasicLit(p.next()), nil
	case token.True:
		r, err = ast.NewBasicLit(p.next()), nil
	case token.False:
		r, err = ast.NewBasicLit(p.next()), nil
	case token.String:
		r, err = ast.Variable{Key: p.next().String()}, nil
	case token.LeftBrac:
		r, err = p.consumeSlice()
	case token.LeftCurly:
		r, err = p.consumeMap()
	case token.Func:
		r, err = p.consumeFunc()
	default:
		return nil, ErrConsumption
	}
REPROCESS:
	switch p.peek().Type {
	case token.Plus:
		r, err = p.consumeAddFunc(r)
		goto REPROCESS
	case token.LeftBrac:
		r, err = p.consumeSliceFunc(r)
		goto REPROCESS
	}
	return r, err
}

func (p *Parser) consumeAddFunc(v interface{}) (*ast.Func, error) {
	f := &ast.Func{
		Name: "addition",
	}

	f.File = p.name
	f.SetStart(p.curTok)

	f.AnonParams = []interface{}{v}

	for p.peek().Type == token.Plus {

		p.next()
		switch p.peek().Type {
		case token.String:
			f.AnonParams = append(
				f.AnonParams,
				ast.Variable{Key: p.next().String()},
			)
		case token.Quote:
			f.AnonParams = append(
				f.AnonParams,
				p.next().String(),
			)
		}
	}

	f.SetEnd(p.curTok)
	return f, nil
}

func (p *Parser) consumeSliceFunc(v interface{}) (*ast.Func, error) {

	f := &ast.Func{
		Params: make(map[string]interface{}),
	}
	f.File = p.name
	f.SetStart(p.curTok)

	// advance [
	p.next()

	f.Params["var"] = v
	if p.peek().Type == token.Colon {
		// advance :
		p.next()
		f.Name = "slice"
		f.Params["start"] = 0
		node, err := p.consumeNode()
		if err != nil {
			return nil, err
		}
		f.Params["end"] = node
		goto END
	} else if p.peek().Type == token.Int {
		node, err := p.consumeNode()
		if err != nil {
			return nil, err
		}

		if p.peek().Type == token.RightBrac {
			f.Name = "index"
			f.Params["index"] = node
			goto END
		} else if p.peek().Type == token.Colon {
			// advance :
			p.next()
		} else {
			return nil, fmt.Errorf("this is a malformed slice")
		}
		f.Name = "slice"

		node, err = p.consumeNode()
		if err != nil {
			return nil, err
		}

		f.Params["start"] = node
		if p.peek().Type == token.Int {
			node, err = p.consumeNode()
			if err != nil {
				return nil, err
			}
			f.Params["end"] = node
		} else if p.peek().Type == token.RightBrac {
			goto END
		} else {
			return nil, fmt.Errorf("this is a malformed slice")
		}

	}
END:
	// advance ]
	f.SetEnd(p.next())
	return f, nil
}
func (p *Parser) consumeParams(f *ast.Func) error {
	for {
		switch p.peek().Type {
		case token.Quote, token.LeftBrac, token.Func:
			if n, err := p.consumeNode(); err != nil {
				return err
			} else {
				f.AnonParams = append(f.AnonParams, n)
			}
		case token.String:
			t := p.next()
			if f.Params == nil {
				f.Params = make(map[string]interface{})
			}

			if err := p.expects(p.peek(), token.Colon, token.Equal); err == nil {
				switch p.next().Type {
				case token.Colon:
				case token.Equal:
					if n, err := p.consumeNode(); err != nil {
						return err
					} else {
						f.Params[t.String()] = n
					}
				}
			} else {
				return err
			}
		default:
			return ErrConsumption
		}

		if err := p.expects(p.peek(), token.RightParen, token.Comma); err == nil {
		DANGLING_COMMA:
			switch p.peek().Type {
			case token.RightParen:
				p.next()
				return nil
			case token.Comma:
				p.next()
				if p.peek().Type == token.RightParen {
					goto DANGLING_COMMA
				}
				continue
			}
		} else {
			return err
		}

	}
}
func (p *Parser) consumeMap() (*ast.Map, error) {
	t := p.next()
	_map := ast.Map{
		Value: make(map[string]interface{}),
	}
	_map.SetStart(t)

	for p.peek().Type != token.RightCurly {
		t := p.next()
		if err := p.expects(t, token.Quote); err != nil {
			return nil, err
		}
		if err := p.expects(p.next(), token.Colon); err != nil {
			return nil, err
		}

		if n, err := p.consumeNode(); err != nil {
			return nil, err
		} else {
			_map.Value[t.String()] = n
		}
		if p.peek().Type == token.Comma {
			p.next()
		} else if err := p.expects(p.peek(), token.RightCurly); err != nil {
			return nil, err
		}
	}

	// advance }

	_map.SetEnd(p.next())
	return &_map, nil
}
func (p *Parser) consumeFunc() (*ast.Func, error) {
	t := p.next()
	if err := p.expects(t, token.Func); err != nil {
		return nil, err
	}
	f := ast.Func{
		Name: t.String(),
	}

	f.File = p.name
	f.Start = ast.Position{
		Line:  t.Line,
		Index: t.Start,
	}

	t = p.next()
	if err := p.expects(t, token.LeftParen); err != nil {
		return nil, err
	}
	p.consumeParams(&f)
	return &f, nil
}

func (p *Parser) consumeSlice() (ast.Slice, error) {
	var _slice ast.Slice

	if err := p.expects(p.peek(), token.LeftBrac); err != nil {
		return _slice, err
	} else {
		_slice.SetStart(p.next())
	}

	for p.peek().Type != token.RightBrac {
		node, err := p.consumeNode()
		if err != nil {
			return _slice, err
		}
		_slice.Slice = append(_slice.Slice, node)
		if p.peek().Type == token.Comma {
			p.next()
		} else if err := p.expects(p.peek(), token.RightBrac); err != nil {
			return _slice, err
		}
	}

	// advance ]
	_slice.SetEnd(p.next())

	return _slice, nil
}
<|endoftext|>