<commit_before>package emil

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"time"
)

var IN_TEST = false
var errNowNewAnalysis = errors.New("errNowNewAnalysis")

type EndGameSave struct {
	AnalysisMap map[string]string `json:"analysis"`
}

// EndGameDb to query for mate in 1,2, etc.
type EndGameDb struct {
	Start       time.Time            `json:"startTime"`
	Duration    time.Duration        `json:"duration"`
	AnalysisMap map[string]*Analysis `json:"analysis"`
	dtmDb       []map[string]bool
}

func (db *EndGameDb) Find(p *position) (bestMove *Move) {
	if DEBUG {
		fmt.Printf("Find:\n%s\n", p.board)
	}
	a := db.AnalysisMap[p.board.String()]
	if DEBUG {
		fmt.Printf("Found: AnalysisMap with dtms %v\n", a.DTMs(p.player))
	}
	return a.BestMove(p.player)
}

func (db *EndGameDb) FindMatesIn(dtm int) (as []*Analysis) {
	if dtm == -1 {
		for _, a := range db.AnalysisMap {
			if a.playerHaveDTMs() {
				as = append(as, a)
			}
		}
	} else {
		for str := range db.dtmDb[dtm] {
			as = append(as, db.AnalysisMap[str])
		}
	}
	return as
}

func (db *EndGameDb) FindMates() (as []*Analysis) {
	return db.FindMatesIn(0)
}

func (db *EndGameDb) FindMate(piece, square int) (boards []*Board) {
	for str := range db.dtmDb[0] {
		a := db.AnalysisMap[str]
		if a.Board.squares[square] == piece {
			boards = append(boards, a.Board)
		}
	}
	return boards
}

func (db *EndGameDb) addPosition(board *Board) {
	a := &Analysis{
		DtmWhite: make([]*DTM, 0),
		DtmBlack: make([]*DTM, 0),
		Board:    board,
		moves:    make(map[string]bool)}
	db.AnalysisMap[a.Board.String()] = a
}

func (db *EndGameDb) addAnalysis(board *Board, dtm int, move *Move) {
	a := db.AnalysisMap[board.String()]
	if move != nil {
		a.addDTM(move.reverse(), dtm)
	}
	if dtm >= 0 {
		if move != nil {
			playerForStep := playerForStepN(dtm)
			if playerForStep != move.player {
				panic("playerForStep != move.player")
			}
		}
		db.dtmDb[dtm][board.String()] = true
	}
}

func (db *EndGameDb) positions() int {
	return len(db.AnalysisMap)
}

// find positions where black is checkmate
func (db *EndGameDb) retrogradeAnalysisStep1() {
	db.dtmDb = append(db.dtmDb, make(map[string]bool))

	start := time.Now()

	player := BLACK
	for boardStr, a := range db.AnalysisMap {
		// mate only on border square
		blackKingSquare := BoardSquares[a.Board.blackKing]
		if !blackKingSquare.isBorder {
			continue
		}
		// mate only with help from king
		if squaresDistances[a.Board.blackKing][a.Board.whiteKing] > 2 {
			continue
		}

		p := NewPosition(a.Board, player)

		move := Search(p)
		if move == nil {
			if isKingInCheck(p) {
				db.addAnalysis(a.Board, 0, nil)
				if DEBUG {
					fmt.Printf("mate:\n%s\n", boardStr)
				}
			}
		}
	}
	end := time.Now()
	if DEBUG {
		fmt.Printf("db.dtmDb[0] %d\n", len(db.dtmDb[0]))
		fmt.Printf("duration %v\n\n\n", end.Sub(start))
	}
}
func playerForStepN(dtm int) (player int) {
	if dtm%2 == 0 {
		return BLACK
	}
	return WHITE
}

func (db *EndGameDb) retrogradeAnalysisStepN(dtm int) (noError error) {
	start := time.Now()
	db.dtmDb = append(db.dtmDb, make(map[string]bool))

	player := playerForStepN(dtm)

	positions := 0
	if player == WHITE {
		if DEBUG {
			fmt.Printf("WHITE Start positions %d\n", len(db.dtmDb[dtm-1]))
		}
		for str := range db.dtmDb[dtm-1] {
			a := db.AnalysisMap[str]
			p := NewPosition(a.Board, player)
			list := generateMoves(p)
			moves := filterKingCaptures(p, list)
			moves = filterKingCaptures(NewPosition(a.Board, otherPlayer(player)), list)

			for _, m := range moves {
				newBoard := a.Board.doMove(m)
				db.addAnalysis(newBoard, dtm, m)
			}
		}
	} else {
		for _, a := range db.AnalysisMap {
			if db.isMateIn0246(a.Board, dtm) >= 0 {
				positions++
			}
		}
		if DEBUG {
			fmt.Printf("BLACK Start positions %d\n", len(db.AnalysisMap)-positions)
		}
		for _, a := range db.AnalysisMap {
			if db.isMateIn0246(a.Board, dtm) >= 0 {
				continue
			}
			p := NewPosition(a.Board, player)
			moves := GenerateMoves(p)

			found := 0
			maxDTM := -1
			for _, m := range moves {
				newBoard := a.Board.doMove(m)
				newDtm := db.isMateIn1357(newBoard, dtm)
				if newDtm > maxDTM {
					maxDTM = newDtm
				}
				if db.isMateIn1357(newBoard, dtm) >= 0 {
					found++
				}
			}

			if found == len(moves) {
				for _, m := range moves {
					db.addAnalysis(a.Board, maxDTM+1, m)
				}
			}
		}
	}
	end := time.Now()

	if DEBUG {
		fmt.Printf("db.dtmDb[%d] %d\n", dtm, len(db.dtmDb[dtm]))
		fmt.Printf("duration %v\n\n\n", end.Sub(start))
	}

	if len(db.dtmDb[dtm]) == 0 {
		return errNowNewAnalysis
	}
	return noError
}
func (db *EndGameDb) isMateIn0246(board *Board, maxDtm int) int {
	for dtm := 0; dtm < maxDtm; dtm += 2 {
		_, ok := db.dtmDb[dtm][board.String()]
		if ok {
			return dtm
		}
	}
	return -1
}
func (db *EndGameDb) isMateIn1357(board *Board, maxDtm int) int {
	for dtm := 1; dtm < maxDtm; dtm += 2 {
		_, ok := db.dtmDb[dtm][board.String()]
		if ok {
			return dtm
		}
	}
	return -1
}

func (db *EndGameDb) MaxDtm() int {
	return len(db.dtmDb)
}

func (db *EndGameDb) retrogradeAnalysis() {
	// find positions where black is checkmate
	db.retrogradeAnalysisStep1()
	dtm := 1
	for {
		err := db.retrogradeAnalysisStepN(dtm)
		if err != nil {
			break
		}
		if IN_TEST {
			return
		}
		dtm++
	}
}
func GenerateMoves(p *position) (list []*Move) {
	for _, m := range generateMoves(p) {
		b := p.board.DoMove(m)
		if !IsTheKingInCheck(NewPosition(b, WHITE)) {
			list = append(list, m)
		}
	}
	return list
}
func generateMoves(p *position) (list []*Move) {
	for src, piece := range p.board.squares {
		if isOwnPiece(p.player, piece) {
			switch abs(piece) {
			case kingValue:
				for _, dst := range kingDestinationsFrom(src) {
					capture := p.board.squares[dst]
					if isOtherKing(p.player, capture) {
						continue
					}
					if capture == Empty {
						list = append(list, newSilentMove(p.player, piece, src, dst))
					} else if !isOwnPiece(p.player, capture) {
						list = append(list, newCaptureMove(p.player, piece, capture, src, dst))
					}
				}
			case rockValue:
				for _, dsts := range rockDestinationsFrom(src) {
					for _, dst := range dsts {
						capture := p.board.squares[dst]
						if isOtherKing(p.player, capture) {
							break
						}
						if capture == Empty {
							list = append(list, newSilentMove(p.player, piece, src, dst))
						} else if !isOwnPiece(p.player, capture) {
							list = append(list, newCaptureMove(p.player, piece, capture, src, dst))
							break
						} else {
							break // onOwnPiece
						}
					}
				}
			}
		}
	}
	return list
}
func LoadEndGameDb() (db *EndGameDb, err error) {
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		return db, err
	}
	var data EndGameSave
	err = json.Unmarshal(b, &data)
	if err != nil {
		return db, err
	}
	db = &EndGameDb{
		Start:       time.Now(),
		AnalysisMap: make(map[string]*Analysis),
		dtmDb:       make([]map[string]bool, 0)}

	for fen, v := range data.AnalysisMap {
		board := Fen2Board(fen)
		db.addPosition(board)
		dtms := DTMsFromString(v)
		for _, d := range dtms {
			db.addAnalysis(board, d.Dtm, d.Move)
		}
	}

	return db, err
}

const filename = "EndGameDb.json"

// SaveEndGameDb saves the an end game DB for KRK to file
func SaveEndGameDb(db *EndGameDb) error {
	fmt.Println("WriteDataToFile: ", filename)

	data := EndGameSave{AnalysisMap: make(map[string]string)}

	for p, a := range db.AnalysisMap {
		data.AnalysisMap[p] = fmt.Sprintf("%v", a.DtmWhite)

	}

	b, err := json.MarshalIndent(data, "", "  ")
	if err != nil {
		return err
	}

	return ioutil.WriteFile(filename, b, 0666)
}

// NewEndGameDb generates an end game DB for KRK
func NewEndGameDb() *EndGameDb {
	var err error

	endGames := &EndGameDb{
		Start:       time.Now(),
		AnalysisMap: make(map[string]*Analysis),
		dtmDb:       make([]map[string]bool, 0)}

	for wk := A1; wk <= H8; wk++ {
		//for wk := E3; wk <= E3; wk++ {
		if DEBUG {
			fmt.Printf("White king on %s\n", BoardSquares[wk])
		}
		for wr := A1; wr <= H8; wr++ {
			for bk := A1; bk <= H8; bk++ {

				board := NewBoard()

				err = board.Setup(WhiteKing, wk)
				if err != nil {
					continue
				}

				err = board.Setup(WhiteRock, wr)
				if err != nil {
					continue
				}

				err = board.Setup(BlackKing, bk)
				if err != nil {
					continue
				}

				err = board.kingsToClose()
				if err != nil {
					continue
				}

				endGames.addPosition(board)
			}
		}
	}
	end := time.Now()
	endGames.Duration = end.Sub(endGames.Start)
	if DEBUG {
		fmt.Printf("all positions %d\n", 64*63*62)
		fmt.Printf("endGames.positions() %d\n", endGames.positions())
		fmt.Printf("difference %d\n", 64*63*62-endGames.positions())
		fmt.Printf("duration %v\n", endGames.Duration)
	}
	endGames.retrogradeAnalysis()

	return endGames
}
<commit_msg>load and parse endgame db in 20s<commit_after>package emil

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"time"
)

var IN_TEST = false
var errNowNewAnalysis = errors.New("errNowNewAnalysis")

type EndGameSave struct {
	AnalysisMap map[string]string `json:"analysis"`
}

// EndGameDb to query for mate in 1,2, etc.
type EndGameDb struct {
	Start       time.Time            `json:"startTime"`
	Duration    time.Duration        `json:"duration"`
	AnalysisMap map[string]*Analysis `json:"analysis"`
	dtmDb       []map[string]bool
}

func (db *EndGameDb) Find(p *position) (bestMove *Move) {
	if DEBUG {
		fmt.Printf("Find:\n%s\n", p.board)
	}
	a := db.AnalysisMap[p.board.String()]
	if DEBUG {
		fmt.Printf("Found: AnalysisMap with dtms %v\n", a.DTMs(p.player))
	}
	return a.BestMove(p.player)
}

func (db *EndGameDb) FindMatesIn(dtm int) (as []*Analysis) {
	if dtm == -1 {
		for _, a := range db.AnalysisMap {
			if a.playerHaveDTMs() {
				as = append(as, a)
			}
		}
	} else {
		for str := range db.dtmDb[dtm] {
			as = append(as, db.AnalysisMap[str])
		}
	}
	return as
}

func (db *EndGameDb) FindMates() (as []*Analysis) {
	return db.FindMatesIn(0)
}

func (db *EndGameDb) FindMate(piece, square int) (boards []*Board) {
	for str := range db.dtmDb[0] {
		a := db.AnalysisMap[str]
		if a.Board.squares[square] == piece {
			boards = append(boards, a.Board)
		}
	}
	return boards
}

func (db *EndGameDb) addPosition(board *Board) {
	a := &Analysis{
		DtmWhite: make([]*DTM, 0),
		DtmBlack: make([]*DTM, 0),
		Board:    board,
		moves:    make(map[string]bool)}
	db.AnalysisMap[a.Board.String()] = a
}

func (db *EndGameDb) addAnalysis(board *Board, dtm int, move *Move) {
	a := db.AnalysisMap[board.String()]
	if move != nil {
		a.addDTM(move.reverse(), dtm)
	}
	if dtm >= 0 {
		if move != nil {
			playerForStep := playerForStepN(dtm)
			if playerForStep != move.player {
				panic("playerForStep != move.player")
			}
		}
		db.dtmDb[dtm][board.String()] = true
	}
}

func (db *EndGameDb) positions() int {
	return len(db.AnalysisMap)
}

// find positions where black is checkmate
func (db *EndGameDb) retrogradeAnalysisStep1() {
	db.dtmDb = append(db.dtmDb, make(map[string]bool))

	start := time.Now()

	player := BLACK
	for boardStr, a := range db.AnalysisMap {
		// mate only on border square
		blackKingSquare := BoardSquares[a.Board.blackKing]
		if !blackKingSquare.isBorder {
			continue
		}
		// mate only with help from king
		if squaresDistances[a.Board.blackKing][a.Board.whiteKing] > 2 {
			continue
		}

		p := NewPosition(a.Board, player)

		move := Search(p)
		if move == nil {
			if isKingInCheck(p) {
				db.addAnalysis(a.Board, 0, nil)
				if DEBUG {
					fmt.Printf("mate:\n%s\n", boardStr)
				}
			}
		}
	}
	end := time.Now()
	if DEBUG {
		fmt.Printf("db.dtmDb[0] %d\n", len(db.dtmDb[0]))
		fmt.Printf("duration %v\n\n\n", end.Sub(start))
	}
}
func playerForStepN(dtm int) (player int) {
	if dtm%2 == 0 {
		return BLACK
	}
	return WHITE
}

func (db *EndGameDb) retrogradeAnalysisStepN(dtm int) (noError error) {
	start := time.Now()
	db.dtmDb = append(db.dtmDb, make(map[string]bool))

	player := playerForStepN(dtm)

	positions := 0
	if player == WHITE {
		if DEBUG {
			fmt.Printf("WHITE Start positions %d\n", len(db.dtmDb[dtm-1]))
		}
		for str := range db.dtmDb[dtm-1] {
			a := db.AnalysisMap[str]
			p := NewPosition(a.Board, player)
			list := generateMoves(p)
			moves := filterKingCaptures(p, list)
			moves = filterKingCaptures(NewPosition(a.Board, otherPlayer(player)), list)

			for _, m := range moves {
				newBoard := a.Board.doMove(m)
				db.addAnalysis(newBoard, dtm, m)
			}
		}
	} else {
		for _, a := range db.AnalysisMap {
			if db.isMateIn0246(a.Board, dtm) >= 0 {
				positions++
			}
		}
		if DEBUG {
			fmt.Printf("BLACK Start positions %d\n", len(db.AnalysisMap)-positions)
		}
		for _, a := range db.AnalysisMap {
			if db.isMateIn0246(a.Board, dtm) >= 0 {
				continue
			}
			p := NewPosition(a.Board, player)
			moves := GenerateMoves(p)

			found := 0
			maxDTM := -1
			for _, m := range moves {
				newBoard := a.Board.doMove(m)
				newDtm := db.isMateIn1357(newBoard, dtm)
				if newDtm > maxDTM {
					maxDTM = newDtm
				}
				if db.isMateIn1357(newBoard, dtm) >= 0 {
					found++
				}
			}

			if found == len(moves) {
				for _, m := range moves {
					db.addAnalysis(a.Board, maxDTM+1, m)
				}
			}
		}
	}
	end := time.Now()

	if DEBUG {
		fmt.Printf("db.dtmDb[%d] %d\n", dtm, len(db.dtmDb[dtm]))
		fmt.Printf("duration %v\n\n\n", end.Sub(start))
	}

	if len(db.dtmDb[dtm]) == 0 {
		return errNowNewAnalysis
	}
	return noError
}
func (db *EndGameDb) isMateIn0246(board *Board, maxDtm int) int {
	for dtm := 0; dtm < maxDtm; dtm += 2 {
		_, ok := db.dtmDb[dtm][board.String()]
		if ok {
			return dtm
		}
	}
	return -1
}
func (db *EndGameDb) isMateIn1357(board *Board, maxDtm int) int {
	for dtm := 1; dtm < maxDtm; dtm += 2 {
		_, ok := db.dtmDb[dtm][board.String()]
		if ok {
			return dtm
		}
	}
	return -1
}

func (db *EndGameDb) MaxDtm() int {
	return len(db.dtmDb)
}

func (db *EndGameDb) retrogradeAnalysis() {
	// find positions where black is checkmate
	db.retrogradeAnalysisStep1()
	dtm := 1
	for {
		err := db.retrogradeAnalysisStepN(dtm)
		if err != nil {
			break
		}
		if IN_TEST {
			return
		}
		dtm++
	}
}
func GenerateMoves(p *position) (list []*Move) {
	for _, m := range generateMoves(p) {
		b := p.board.DoMove(m)
		if !IsTheKingInCheck(NewPosition(b, WHITE)) {
			list = append(list, m)
		}
	}
	return list
}
func generateMoves(p *position) (list []*Move) {
	for src, piece := range p.board.squares {
		if isOwnPiece(p.player, piece) {
			switch abs(piece) {
			case kingValue:
				for _, dst := range kingDestinationsFrom(src) {
					capture := p.board.squares[dst]
					if isOtherKing(p.player, capture) {
						continue
					}
					if capture == Empty {
						list = append(list, newSilentMove(p.player, piece, src, dst))
					} else if !isOwnPiece(p.player, capture) {
						list = append(list, newCaptureMove(p.player, piece, capture, src, dst))
					}
				}
			case rockValue:
				for _, dsts := range rockDestinationsFrom(src) {
					for _, dst := range dsts {
						capture := p.board.squares[dst]
						if isOtherKing(p.player, capture) {
							break
						}
						if capture == Empty {
							list = append(list, newSilentMove(p.player, piece, src, dst))
						} else if !isOwnPiece(p.player, capture) {
							list = append(list, newCaptureMove(p.player, piece, capture, src, dst))
							break
						} else {
							break // onOwnPiece
						}
					}
				}
			}
		}
	}
	return list
}
func LoadEndGameDb() (db *EndGameDb, err error) {
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		return db, err
	}
	var data EndGameSave
	err = json.Unmarshal(b, &data)
	if err != nil {
		return db, err
	}
	db = &EndGameDb{
		Start:       time.Now(),
		AnalysisMap: make(map[string]*Analysis),
		dtmDb:       make([]map[string]bool, 0)}

	for i := 0; i < 34; i++ {
		db.dtmDb = append(db.dtmDb, make(map[string]bool))
	}

	for fen, v := range data.AnalysisMap {
		board := Fen2Board(fen)
		db.addPosition(board)
		dtms := DTMsFromString(v)
		for _, d := range dtms {
			db.addAnalysis(board, d.Dtm, d.Move.reverse())
		}
	}

	return db, err
}

const filename = "EndGameDb.json"

// SaveEndGameDb saves the an end game DB for KRK to file
func SaveEndGameDb(db *EndGameDb) error {
	fmt.Println("WriteDataToFile: ", filename)

	data := EndGameSave{AnalysisMap: make(map[string]string)}

	for p, a := range db.AnalysisMap {
		data.AnalysisMap[p] = fmt.Sprintf("%v", a.DtmWhite)

	}

	b, err := json.MarshalIndent(data, "", "  ")
	if err != nil {
		return err
	}

	return ioutil.WriteFile(filename, b, 0666)
}

// NewEndGameDb generates an end game DB for KRK
func NewEndGameDb() *EndGameDb {
	var err error

	endGames := &EndGameDb{
		Start:       time.Now(),
		AnalysisMap: make(map[string]*Analysis),
		dtmDb:       make([]map[string]bool, 0)}

	for wk := A1; wk <= H8; wk++ {
		//for wk := E3; wk <= E3; wk++ {
		if DEBUG {
			fmt.Printf("White king on %s\n", BoardSquares[wk])
		}
		for wr := A1; wr <= H8; wr++ {
			for bk := A1; bk <= H8; bk++ {

				board := NewBoard()

				err = board.Setup(WhiteKing, wk)
				if err != nil {
					continue
				}

				err = board.Setup(WhiteRock, wr)
				if err != nil {
					continue
				}

				err = board.Setup(BlackKing, bk)
				if err != nil {
					continue
				}

				err = board.kingsToClose()
				if err != nil {
					continue
				}

				endGames.addPosition(board)
			}
		}
	}
	end := time.Now()
	endGames.Duration = end.Sub(endGames.Start)
	if DEBUG {
		fmt.Printf("all positions %d\n", 64*63*62)
		fmt.Printf("endGames.positions() %d\n", endGames.positions())
		fmt.Printf("difference %d\n", 64*63*62-endGames.positions())
		fmt.Printf("duration %v\n", endGames.Duration)
	}
	endGames.retrogradeAnalysis()

	return endGames
}
<|endoftext|>


<commit_before>package graphql

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	"github.com/gorilla/websocket"

	"github.com/samsarahq/thunder/batch"
	"github.com/samsarahq/thunder/diff"
	"github.com/samsarahq/thunder/reactive"
)

const (
	MaxSubscriptions = 200
	MinRerunInterval = 5 * time.Second
)

type JSONSocket interface {
	ReadJSON(value interface{}) error
	WriteJSON(value interface{}) error
	Close() error
}

type MakeCtxFunc func(context.Context) context.Context

type GraphqlLogger interface {
	StartExecution(ctx context.Context, tags map[string]string, initial bool)
	FinishExecution(ctx context.Context, tags map[string]string, delay time.Duration)
	Error(ctx context.Context, err error, tags map[string]string)
}

type conn struct {
	writeMu sync.Mutex
	socket  JSONSocket

	schema      *Schema
	ctx         context.Context
	makeCtx     MakeCtxFunc
	logger      GraphqlLogger
	middlewares []MiddlewareFunc

	url string

	mutateMu sync.Mutex

	mu            sync.Mutex
	subscriptions map[string]*reactive.Rerunner
}

type inEnvelope struct {
	ID      string          `json:"id"`
	Type    string          `json:"type"`
	Message json.RawMessage `json:"message"`
}

type outEnvelope struct {
	ID       string                 `json:"id,omitempty"`
	Type     string                 `json:"type"`
	Message  interface{}            `json:"message,omitempty"`
	Metadata map[string]interface{} `json:"metadata,omitempty"`
}

type subscribeMessage struct {
	Query     string                 `json:"query"`
	Variables map[string]interface{} `json:"variables"`
}

type mutateMessage struct {
	Query     string                 `json:"query"`
	Variables map[string]interface{} `json:"variables"`
}

type SanitizedError interface {
	error
	SanitizedError() string
}

type SafeError struct {
	message string
}

func (e SafeError) Error() string {
	return e.message
}

func (e SafeError) SanitizedError() string {
	return e.message
}

func NewSafeError(format string, a ...interface{}) error {
	return SafeError{message: fmt.Sprintf(format, a...)}
}

func sanitizeError(err error) string {
	if sanitized, ok := err.(SanitizedError); ok {
		return sanitized.SanitizedError()
	}
	return "Internal server error"
}

func isCloseError(err error) bool {
	_, ok := err.(*websocket.CloseError)
	return ok || err == websocket.ErrCloseSent
}

func (c *conn) writeOrClose(out outEnvelope) {
	c.writeMu.Lock()
	defer c.writeMu.Unlock()

	if err := c.socket.WriteJSON(out); err != nil {
		if !isCloseError(err) {
			c.socket.Close()
			log.Printf("socket.WriteJSON: %s\n", err)
		}
	}
}

func mustMarshalJson(v interface{}) string {
	bytes, err := json.Marshal(v)
	if err != nil {
		panic(err)
	}
	return string(bytes)
}

func (c *conn) handleSubscribe(id string, subscribe *subscribeMessage) error {
	c.mu.Lock()
	defer c.mu.Unlock()

	if _, ok := c.subscriptions[id]; ok {
		return NewSafeError("duplicate subscription")
	}

	if len(c.subscriptions)+1 > MaxSubscriptions {
		return NewSafeError("too many subscriptions")
	}

	query, err := Parse(subscribe.Query, subscribe.Variables)
	if err != nil {
		return err
	}
	if err := PrepareQuery(c.schema.Query, query.SelectionSet); err != nil {
		return err
	}

	var previous interface{}

	e := Executor{}

	initial := true
	tags := map[string]string{"url": c.url, "queryType": query.Kind, "queryName": query.Name, "query": subscribe.Query, "queryVariables": mustMarshalJson(subscribe.Variables), "id": id}

	c.subscriptions[id] = reactive.NewRerunner(c.ctx, func(ctx context.Context) (interface{}, error) {
		ctx = c.makeCtx(ctx)
		ctx = batch.WithBatching(ctx)

		start := time.Now()

		c.logger.StartExecution(ctx, tags, initial)

		var middlewares []MiddlewareFunc
		middlewares = append(middlewares, c.middlewares...)
		middlewares = append(middlewares, func(input *ComputationInput, next MiddlewareNextFunc) *ComputationOutput {
			output := next(input)
			output.Current, output.Error = e.Execute(input.Ctx, c.schema.Query, nil, input.ParsedQuery)
			return output
		})

		output := runMiddlewares(middlewares, &ComputationInput{
			Ctx:         ctx,
			Id:          id,
			ParsedQuery: query,
			Previous:    previous,
			Query:       subscribe.Query,
			Variables:   subscribe.Variables,
		})
		current, err := output.Current, output.Error

		c.logger.FinishExecution(ctx, tags, time.Since(start))

		if err != nil {
			c.writeOrClose(outEnvelope{
				ID:       id,
				Type:     "error",
				Message:  sanitizeError(err),
				Metadata: output.Metadata,
			})
			go c.closeSubscription(id)

			if extractPathError(err) == context.Canceled {
				return nil, err
			}

			if _, ok := err.(SanitizedError); !ok {
				c.logger.Error(ctx, err, tags)
			}
			return nil, err
		}

		d := diff.Diff(previous, current)
		previous = current
		initial = false

		if initial || d != nil {
			c.writeOrClose(outEnvelope{
				ID:       id,
				Type:     "update",
				Message:  d,
				Metadata: output.Metadata,
			})
		}

		return nil, nil
	}, MinRerunInterval)

	return nil
}

func (c *conn) handleMutate(id string, mutate *mutateMessage) error {
	// TODO: deduplicate code
	c.mu.Lock()
	defer c.mu.Unlock()

	query, err := Parse(mutate.Query, mutate.Variables)
	if err != nil {
		return err
	}
	if err := PrepareQuery(c.schema.Mutation, query.SelectionSet); err != nil {
		return err
	}

	e := Executor{}

	tags := map[string]string{"url": c.url, "queryType": query.Kind, "queryName": query.Name, "query": mutate.Query, "queryVariables": mustMarshalJson(mutate.Variables), "id": id}

	c.subscriptions[id] = reactive.NewRerunner(c.ctx, func(ctx context.Context) (interface{}, error) {
		// Serialize all mutates for a given connection.
		c.mutateMu.Lock()
		defer c.mutateMu.Unlock()

		ctx = c.makeCtx(ctx)
		ctx = batch.WithBatching(ctx)

		start := time.Now()
		c.logger.StartExecution(ctx, tags, true)

		var middlewares []MiddlewareFunc
		middlewares = append(middlewares, c.middlewares...)
		middlewares = append(middlewares, func(input *ComputationInput, next MiddlewareNextFunc) *ComputationOutput {
			output := next(input)
			output.Current, output.Error = e.Execute(ctx, c.schema.Mutation, c.schema.Mutation, query)
			return output
		})

		output := runMiddlewares(middlewares, &ComputationInput{
			Ctx:         ctx,
			Id:          id,
			ParsedQuery: query,
			Previous:    nil,
			Query:       mutate.Query,
			Variables:   mutate.Variables,
		})
		current, err := output.Current, output.Error

		c.logger.FinishExecution(ctx, tags, time.Since(start))

		if err != nil {
			c.writeOrClose(outEnvelope{
				ID:       id,
				Type:     "error",
				Message:  sanitizeError(err),
				Metadata: nil,
			})

			go c.closeSubscription(id)

			if extractPathError(err) == context.Canceled {
				return nil, err
			}

			if _, ok := err.(SanitizedError); !ok {
				c.logger.Error(ctx, err, tags)
			}
			return nil, err
		}

		c.writeOrClose(outEnvelope{
			ID:       id,
			Type:     "result",
			Message:  diff.Diff(nil, current),
			Metadata: output.Metadata,
		})

		return nil, errors.New("stop")
	}, MinRerunInterval)

	return nil
}

func (c *conn) closeSubscription(id string) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if runner, ok := c.subscriptions[id]; ok {
		runner.Stop()
		delete(c.subscriptions, id)
	}
}

func (c *conn) closeSubscriptions() {
	c.mu.Lock()
	defer c.mu.Unlock()

	for id, runner := range c.subscriptions {
		runner.Stop()
		delete(c.subscriptions, id)
	}
}

func (c *conn) handle(e *inEnvelope) error {
	switch e.Type {
	case "subscribe":
		var subscribe subscribeMessage
		if err := json.Unmarshal(e.Message, &subscribe); err != nil {
			return err
		}
		return c.handleSubscribe(e.ID, &subscribe)

	case "unsubscribe":
		c.closeSubscription(e.ID)
		return nil

	case "mutate":
		var mutate mutateMessage
		if err := json.Unmarshal(e.Message, &mutate); err != nil {
			return err
		}
		return c.handleMutate(e.ID, &mutate)

	case "echo":
		c.writeOrClose(outEnvelope{
			ID:       e.ID,
			Type:     "echo",
			Message:  nil,
			Metadata: nil,
		})
		return nil

	case "url":
		var url string
		if err := json.Unmarshal(e.Message, &url); err != nil {
			return err
		}
		c.url = url
		return nil

	default:
		return NewSafeError("unknown message type")
	}
}

type simpleLogger struct {
}

func (s *simpleLogger) StartExecution(ctx context.Context, tags map[string]string, initial bool) {
}
func (s *simpleLogger) FinishExecution(ctx context.Context, tags map[string]string, delay time.Duration) {
}
func (s *simpleLogger) Error(ctx context.Context, err error, tags map[string]string) {
	log.Printf("error:%v\n%s", tags, err)
}

func Handler(schema *Schema) http.Handler {
	upgrader := &websocket.Upgrader{
		ReadBufferSize:  1024,
		WriteBufferSize: 1024,
		CheckOrigin: func(r *http.Request) bool {
			return true
		},
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		socket, err := upgrader.Upgrade(w, r, nil)
		if err != nil {
			log.Printf("upgrader.Upgrade: %v", err)
			return
		}
		defer socket.Close()

		makeCtx := func(ctx context.Context) context.Context {
			return ctx
		}

		ServeJSONSocket(r.Context(), socket, schema, makeCtx, &simpleLogger{})
	})
}

func (c *conn) Use(fn MiddlewareFunc) {
	c.middlewares = append(c.middlewares, fn)
}

func ServeJSONSocket(ctx context.Context, socket JSONSocket, schema *Schema, makeCtx MakeCtxFunc, logger GraphqlLogger) {
	conn := CreateJSONSocket(ctx, socket, schema, makeCtx, logger)
	conn.ServeJSONSocket()
}

func CreateJSONSocket(ctx context.Context, socket JSONSocket, schema *Schema, makeCtx MakeCtxFunc, logger GraphqlLogger) *conn {
	return &conn{
		socket: socket,
		ctx:    ctx,

		schema:  schema,
		makeCtx: makeCtx,
		logger:  logger,

		subscriptions: make(map[string]*reactive.Rerunner),
	}
}

func (c *conn) ServeJSONSocket() {
	defer c.closeSubscriptions()

	for {
		var envelope inEnvelope
		if err := c.socket.ReadJSON(&envelope); err != nil {
			if !isCloseError(err) {
				log.Println("socket.ReadJSON:", err)
			}
			return
		}

		if err := c.handle(&envelope); err != nil {
			log.Println("c.handle:", err)
			c.writeOrClose(outEnvelope{
				ID:       envelope.ID,
				Type:     "error",
				Message:  sanitizeError(err),
				Metadata: nil,
			})
		}
	}
}
<commit_msg>graphql: pass contexts in for mutation<commit_after>package graphql

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	"github.com/gorilla/websocket"

	"github.com/samsarahq/thunder/batch"
	"github.com/samsarahq/thunder/diff"
	"github.com/samsarahq/thunder/reactive"
)

const (
	MaxSubscriptions = 200
	MinRerunInterval = 5 * time.Second
)

type JSONSocket interface {
	ReadJSON(value interface{}) error
	WriteJSON(value interface{}) error
	Close() error
}

type MakeCtxFunc func(context.Context) context.Context

type GraphqlLogger interface {
	StartExecution(ctx context.Context, tags map[string]string, initial bool)
	FinishExecution(ctx context.Context, tags map[string]string, delay time.Duration)
	Error(ctx context.Context, err error, tags map[string]string)
}

type conn struct {
	writeMu sync.Mutex
	socket  JSONSocket

	schema      *Schema
	ctx         context.Context
	makeCtx     MakeCtxFunc
	logger      GraphqlLogger
	middlewares []MiddlewareFunc

	url string

	mutateMu sync.Mutex

	mu            sync.Mutex
	subscriptions map[string]*reactive.Rerunner
}

type inEnvelope struct {
	ID      string          `json:"id"`
	Type    string          `json:"type"`
	Message json.RawMessage `json:"message"`
}

type outEnvelope struct {
	ID       string                 `json:"id,omitempty"`
	Type     string                 `json:"type"`
	Message  interface{}            `json:"message,omitempty"`
	Metadata map[string]interface{} `json:"metadata,omitempty"`
}

type subscribeMessage struct {
	Query     string                 `json:"query"`
	Variables map[string]interface{} `json:"variables"`
}

type mutateMessage struct {
	Query     string                 `json:"query"`
	Variables map[string]interface{} `json:"variables"`
}

type SanitizedError interface {
	error
	SanitizedError() string
}

type SafeError struct {
	message string
}

func (e SafeError) Error() string {
	return e.message
}

func (e SafeError) SanitizedError() string {
	return e.message
}

func NewSafeError(format string, a ...interface{}) error {
	return SafeError{message: fmt.Sprintf(format, a...)}
}

func sanitizeError(err error) string {
	if sanitized, ok := err.(SanitizedError); ok {
		return sanitized.SanitizedError()
	}
	return "Internal server error"
}

func isCloseError(err error) bool {
	_, ok := err.(*websocket.CloseError)
	return ok || err == websocket.ErrCloseSent
}

func (c *conn) writeOrClose(out outEnvelope) {
	c.writeMu.Lock()
	defer c.writeMu.Unlock()

	if err := c.socket.WriteJSON(out); err != nil {
		if !isCloseError(err) {
			c.socket.Close()
			log.Printf("socket.WriteJSON: %s\n", err)
		}
	}
}

func mustMarshalJson(v interface{}) string {
	bytes, err := json.Marshal(v)
	if err != nil {
		panic(err)
	}
	return string(bytes)
}

func (c *conn) handleSubscribe(id string, subscribe *subscribeMessage) error {
	c.mu.Lock()
	defer c.mu.Unlock()

	if _, ok := c.subscriptions[id]; ok {
		return NewSafeError("duplicate subscription")
	}

	if len(c.subscriptions)+1 > MaxSubscriptions {
		return NewSafeError("too many subscriptions")
	}

	query, err := Parse(subscribe.Query, subscribe.Variables)
	if err != nil {
		return err
	}
	if err := PrepareQuery(c.schema.Query, query.SelectionSet); err != nil {
		return err
	}

	var previous interface{}

	e := Executor{}

	initial := true
	tags := map[string]string{"url": c.url, "queryType": query.Kind, "queryName": query.Name, "query": subscribe.Query, "queryVariables": mustMarshalJson(subscribe.Variables), "id": id}

	c.subscriptions[id] = reactive.NewRerunner(c.ctx, func(ctx context.Context) (interface{}, error) {
		ctx = c.makeCtx(ctx)
		ctx = batch.WithBatching(ctx)

		start := time.Now()

		c.logger.StartExecution(ctx, tags, initial)

		var middlewares []MiddlewareFunc
		middlewares = append(middlewares, c.middlewares...)
		middlewares = append(middlewares, func(input *ComputationInput, next MiddlewareNextFunc) *ComputationOutput {
			output := next(input)
			output.Current, output.Error = e.Execute(input.Ctx, c.schema.Query, nil, input.ParsedQuery)
			return output
		})

		output := runMiddlewares(middlewares, &ComputationInput{
			Ctx:         ctx,
			Id:          id,
			ParsedQuery: query,
			Previous:    previous,
			Query:       subscribe.Query,
			Variables:   subscribe.Variables,
		})
		current, err := output.Current, output.Error

		c.logger.FinishExecution(ctx, tags, time.Since(start))

		if err != nil {
			c.writeOrClose(outEnvelope{
				ID:       id,
				Type:     "error",
				Message:  sanitizeError(err),
				Metadata: output.Metadata,
			})
			go c.closeSubscription(id)

			if extractPathError(err) == context.Canceled {
				return nil, err
			}

			if _, ok := err.(SanitizedError); !ok {
				c.logger.Error(ctx, err, tags)
			}
			return nil, err
		}

		d := diff.Diff(previous, current)
		previous = current
		initial = false

		if initial || d != nil {
			c.writeOrClose(outEnvelope{
				ID:       id,
				Type:     "update",
				Message:  d,
				Metadata: output.Metadata,
			})
		}

		return nil, nil
	}, MinRerunInterval)

	return nil
}

func (c *conn) handleMutate(id string, mutate *mutateMessage) error {
	// TODO: deduplicate code
	c.mu.Lock()
	defer c.mu.Unlock()

	query, err := Parse(mutate.Query, mutate.Variables)
	if err != nil {
		return err
	}
	if err := PrepareQuery(c.schema.Mutation, query.SelectionSet); err != nil {
		return err
	}

	e := Executor{}

	tags := map[string]string{"url": c.url, "queryType": query.Kind, "queryName": query.Name, "query": mutate.Query, "queryVariables": mustMarshalJson(mutate.Variables), "id": id}

	c.subscriptions[id] = reactive.NewRerunner(c.ctx, func(ctx context.Context) (interface{}, error) {
		// Serialize all mutates for a given connection.
		c.mutateMu.Lock()
		defer c.mutateMu.Unlock()

		ctx = c.makeCtx(ctx)
		ctx = batch.WithBatching(ctx)

		start := time.Now()
		c.logger.StartExecution(ctx, tags, true)

		var middlewares []MiddlewareFunc
		middlewares = append(middlewares, c.middlewares...)
		middlewares = append(middlewares, func(input *ComputationInput, next MiddlewareNextFunc) *ComputationOutput {
			output := next(input)
			output.Current, output.Error = e.Execute(input.Ctx, c.schema.Mutation, c.schema.Mutation, query)
			return output
		})

		output := runMiddlewares(middlewares, &ComputationInput{
			Ctx:         ctx,
			Id:          id,
			ParsedQuery: query,
			Previous:    nil,
			Query:       mutate.Query,
			Variables:   mutate.Variables,
		})
		current, err := output.Current, output.Error

		c.logger.FinishExecution(ctx, tags, time.Since(start))

		if err != nil {
			c.writeOrClose(outEnvelope{
				ID:       id,
				Type:     "error",
				Message:  sanitizeError(err),
				Metadata: output.Metadata,
			})

			go c.closeSubscription(id)

			if extractPathError(err) == context.Canceled {
				return nil, err
			}

			if _, ok := err.(SanitizedError); !ok {
				c.logger.Error(ctx, err, tags)
			}
			return nil, err
		}

		c.writeOrClose(outEnvelope{
			ID:       id,
			Type:     "result",
			Message:  diff.Diff(nil, current),
			Metadata: output.Metadata,
		})

		return nil, errors.New("stop")
	}, MinRerunInterval)

	return nil
}

func (c *conn) closeSubscription(id string) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if runner, ok := c.subscriptions[id]; ok {
		runner.Stop()
		delete(c.subscriptions, id)
	}
}

func (c *conn) closeSubscriptions() {
	c.mu.Lock()
	defer c.mu.Unlock()

	for id, runner := range c.subscriptions {
		runner.Stop()
		delete(c.subscriptions, id)
	}
}

func (c *conn) handle(e *inEnvelope) error {
	switch e.Type {
	case "subscribe":
		var subscribe subscribeMessage
		if err := json.Unmarshal(e.Message, &subscribe); err != nil {
			return err
		}
		return c.handleSubscribe(e.ID, &subscribe)

	case "unsubscribe":
		c.closeSubscription(e.ID)
		return nil

	case "mutate":
		var mutate mutateMessage
		if err := json.Unmarshal(e.Message, &mutate); err != nil {
			return err
		}
		return c.handleMutate(e.ID, &mutate)

	case "echo":
		c.writeOrClose(outEnvelope{
			ID:       e.ID,
			Type:     "echo",
			Message:  nil,
			Metadata: nil,
		})
		return nil

	case "url":
		var url string
		if err := json.Unmarshal(e.Message, &url); err != nil {
			return err
		}
		c.url = url
		return nil

	default:
		return NewSafeError("unknown message type")
	}
}

type simpleLogger struct {
}

func (s *simpleLogger) StartExecution(ctx context.Context, tags map[string]string, initial bool) {
}
func (s *simpleLogger) FinishExecution(ctx context.Context, tags map[string]string, delay time.Duration) {
}
func (s *simpleLogger) Error(ctx context.Context, err error, tags map[string]string) {
	log.Printf("error:%v\n%s", tags, err)
}

func Handler(schema *Schema) http.Handler {
	upgrader := &websocket.Upgrader{
		ReadBufferSize:  1024,
		WriteBufferSize: 1024,
		CheckOrigin: func(r *http.Request) bool {
			return true
		},
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		socket, err := upgrader.Upgrade(w, r, nil)
		if err != nil {
			log.Printf("upgrader.Upgrade: %v", err)
			return
		}
		defer socket.Close()

		makeCtx := func(ctx context.Context) context.Context {
			return ctx
		}

		ServeJSONSocket(r.Context(), socket, schema, makeCtx, &simpleLogger{})
	})
}

func (c *conn) Use(fn MiddlewareFunc) {
	c.middlewares = append(c.middlewares, fn)
}

func ServeJSONSocket(ctx context.Context, socket JSONSocket, schema *Schema, makeCtx MakeCtxFunc, logger GraphqlLogger) {
	conn := CreateJSONSocket(ctx, socket, schema, makeCtx, logger)
	conn.ServeJSONSocket()
}

func CreateJSONSocket(ctx context.Context, socket JSONSocket, schema *Schema, makeCtx MakeCtxFunc, logger GraphqlLogger) *conn {
	return &conn{
		socket: socket,
		ctx:    ctx,

		schema:  schema,
		makeCtx: makeCtx,
		logger:  logger,

		subscriptions: make(map[string]*reactive.Rerunner),
	}
}

func (c *conn) ServeJSONSocket() {
	defer c.closeSubscriptions()

	for {
		var envelope inEnvelope
		if err := c.socket.ReadJSON(&envelope); err != nil {
			if !isCloseError(err) {
				log.Println("socket.ReadJSON:", err)
			}
			return
		}

		if err := c.handle(&envelope); err != nil {
			log.Println("c.handle:", err)
			c.writeOrClose(outEnvelope{
				ID:       envelope.ID,
				Type:     "error",
				Message:  sanitizeError(err),
				Metadata: nil,
			})
		}
	}
}
<|endoftext|>


<commit_before>package irma

import (
	"encoding/base64"
	"encoding/xml"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"

	"crypto/sha256"

	"fmt"

	"strings"

	"github.com/credentials/irmago/internal/fs"
	"github.com/go-errors/errors"
	"github.com/mhe/gabi"
)

// Configuration keeps track of scheme managers, issuers, credential types and public keys,
// dezerializing them from an irma_configuration folder, and downloads and saves new ones on demand.
type Configuration struct {
	SchemeManagers  map[SchemeManagerIdentifier]*SchemeManager
	Issuers         map[IssuerIdentifier]*Issuer
	CredentialTypes map[CredentialTypeIdentifier]*CredentialType

	publicKeys    map[IssuerIdentifier]map[int]*gabi.PublicKey
	reverseHashes map[string]CredentialTypeIdentifier
	path          string
	initialized   bool
}

// NewConfiguration returns a new configuration. After this
// ParseFolder() should be called to parse the specified path.
func NewConfiguration(path string, assets string) (conf *Configuration, err error) {
	conf = &Configuration{
		path: path,
	}

	if err = fs.EnsureDirectoryExists(conf.path); err != nil {
		return nil, err
	}
	if assets != "" {
		if err = conf.Copy(assets, false); err != nil {
			return nil, err
		}
	}

	return
}

// ParseFolder populates the current Configuration by parsing the storage path,
// listing the containing scheme managers, issuers and credential types.
func (conf *Configuration) ParseFolder() error {
	// Init all maps
	conf.SchemeManagers = make(map[SchemeManagerIdentifier]*SchemeManager)
	conf.Issuers = make(map[IssuerIdentifier]*Issuer)
	conf.CredentialTypes = make(map[CredentialTypeIdentifier]*CredentialType)
	conf.publicKeys = make(map[IssuerIdentifier]map[int]*gabi.PublicKey)

	conf.reverseHashes = make(map[string]CredentialTypeIdentifier)

	err := iterateSubfolders(conf.path, func(dir string) error {
		manager := &SchemeManager{}
		exists, err := pathToDescription(dir+"/description.xml", manager)
		if err != nil {
			return err
		}
		if exists {
			conf.SchemeManagers[manager.Identifier()] = manager
			return conf.parseIssuerFolders(dir)
		}
		return nil
	})
	if err != nil {
		return err
	}
	conf.initialized = true
	return nil
}

// PublicKey returns the specified public key, or nil if not present in the Configuration.
func (conf *Configuration) PublicKey(id IssuerIdentifier, counter int) (*gabi.PublicKey, error) {
	if _, contains := conf.publicKeys[id]; !contains {
		conf.publicKeys[id] = map[int]*gabi.PublicKey{}
		if err := conf.parseKeysFolder(id); err != nil {
			return nil, err
		}
	}
	return conf.publicKeys[id][counter], nil
}

func (conf *Configuration) addReverseHash(credid CredentialTypeIdentifier) {
	hash := sha256.Sum256([]byte(credid.String()))
	conf.reverseHashes[base64.StdEncoding.EncodeToString(hash[:16])] = credid
}

func (conf *Configuration) hashToCredentialType(hash []byte) *CredentialType {
	if str, exists := conf.reverseHashes[base64.StdEncoding.EncodeToString(hash)]; exists {
		return conf.CredentialTypes[str]
	}
	return nil
}

// IsInitialized indicates whether this instance has successfully been initialized.
func (conf *Configuration) IsInitialized() bool {
	return conf.initialized
}

func (conf *Configuration) parseIssuerFolders(path string) error {
	return iterateSubfolders(path, func(dir string) error {
		issuer := &Issuer{}
		exists, err := pathToDescription(dir+"/description.xml", issuer)
		if err != nil {
			return err
		}
		if exists {
			conf.Issuers[issuer.Identifier()] = issuer
			if err = conf.parseCredentialsFolder(dir + "/Issues/"); err != nil {
				return err
			}
		}
		return nil
	})
}

// parse $schememanager/$issuer/PublicKeys/$i.xml for $i = 1, ...
func (conf *Configuration) parseKeysFolder(issuerid IssuerIdentifier) error {
	path := fmt.Sprintf("%s/%s/%s/PublicKeys/*.xml", conf.path, issuerid.SchemeManagerIdentifier().Name(), issuerid.Name())
	files, err := filepath.Glob(path)
	if err != nil {
		return err
	}

	for _, file := range files {
		filename := filepath.Base(file)
		count := filename[:len(filename)-4]
		i, err := strconv.Atoi(count)
		if err != nil {
			continue
		}
		pk, err := gabi.NewPublicKeyFromFile(file)
		if err != nil {
			return err
		}
		pk.Issuer = issuerid.String()
		conf.publicKeys[issuerid][i] = pk
	}

	return nil
}

// parse $schememanager/$issuer/Issues/*/description.xml
func (conf *Configuration) parseCredentialsFolder(path string) error {
	return iterateSubfolders(path, func(dir string) error {
		cred := &CredentialType{}
		exists, err := pathToDescription(dir+"/description.xml", cred)
		if err != nil {
			return err
		}
		if exists {
			credid := cred.Identifier()
			conf.CredentialTypes[credid] = cred
			conf.addReverseHash(credid)
		}
		return nil
	})
}

// iterateSubfolders iterates over the subfolders of the specified path,
// calling the specified handler each time. If anything goes wrong, or
// if the caller returns a non-nil error, an error is immediately returned.
func iterateSubfolders(path string, handler func(string) error) error {
	dirs, err := filepath.Glob(path + "/*")
	if err != nil {
		return err
	}

	for _, dir := range dirs {
		stat, err := os.Stat(dir)
		if err != nil {
			return err
		}
		if !stat.IsDir() {
			continue
		}
		err = handler(dir)
		if err != nil {
			return err
		}
	}

	return nil
}

func pathToDescription(path string, description interface{}) (bool, error) {
	if _, err := os.Stat(path); err != nil {
		return false, nil
	}

	file, err := os.Open(path)
	if err != nil {
		return true, err
	}
	defer file.Close()

	bytes, err := ioutil.ReadAll(file)
	if err != nil {
		return true, err
	}

	err = xml.Unmarshal(bytes, description)
	if err != nil {
		return true, err
	}

	return true, nil
}

// Contains checks if the configuration contains the specified credential type.
func (conf *Configuration) Contains(cred CredentialTypeIdentifier) bool {
	return conf.SchemeManagers[cred.IssuerIdentifier().SchemeManagerIdentifier()] != nil &&
		conf.Issuers[cred.IssuerIdentifier()] != nil &&
		conf.CredentialTypes[cred] != nil
}

func (conf *Configuration) Copy(source string, parse bool) error {
	if err := fs.EnsureDirectoryExists(conf.path); err != nil {
		return err
	}

	err := filepath.Walk(source, filepath.WalkFunc(
		func(path string, info os.FileInfo, err error) error {
			if path == source {
				return nil
			}
			subpath := path[len(source):]
			if info.IsDir() {
				if err := fs.EnsureDirectoryExists(conf.path + subpath); err != nil {
					return err
				}
			} else {
				srcfile, err := os.Open(path)
				if err != nil {
					return err
				}
				defer srcfile.Close()
				bytes, err := ioutil.ReadAll(srcfile)
				if err != nil {
					return err
				}
				if err := fs.SaveFile(conf.path+subpath, bytes); err != nil {
					return err
				}
			}
			return nil
		}),
	)

	if err != nil {
		return err
	}
	if parse {
		return conf.ParseFolder()
	}
	return nil
}

func (conf *Configuration) DownloadSchemeManager(url string) (*SchemeManager, error) {
	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
		url = "https://" + url
	}
	if url[len(url)-1] == '/' {
		url = url[:len(url)-1]
	}
	if strings.HasSuffix(url, "/description.xml") {
		url = url[:len(url)-len("/description.xml")]
	}
	b, err := NewHTTPTransport(url).GetBytes("/description.xml")
	if err != nil {
		return nil, err
	}
	manager := &SchemeManager{}
	if err = xml.Unmarshal(b, manager); err != nil {
		return nil, err
	}

	manager.URL = url // TODO?
	return manager, nil
}

func (conf *Configuration) RemoveSchemeManager(id SchemeManagerIdentifier) error {
	// Remove everything falling under the manager's responsibility
	for credid := range conf.CredentialTypes {
		if credid.IssuerIdentifier().SchemeManagerIdentifier() == id {
			delete(conf.CredentialTypes, credid)
		}
	}
	for issid := range conf.Issuers {
		if issid.SchemeManagerIdentifier() == id {
			delete(conf.Issuers, issid)
		}
	}
	for issid := range conf.publicKeys {
		if issid.SchemeManagerIdentifier() == id {
			delete(conf.publicKeys, issid)
		}
	}
	// Remove from storage
	return os.RemoveAll(fmt.Sprintf("%s/%s", conf.path, id.String()))
	// or, remove above iterations and call .ParseFolder()?
}

func (conf *Configuration) AddSchemeManager(manager *SchemeManager) error {
	name := manager.ID
	if err := fs.EnsureDirectoryExists(fmt.Sprintf("%s/%s", conf.path, name)); err != nil {
		return err
	}
	b, err := xml.Marshal(manager)
	if err != nil {
		return err
	}
	if err := fs.SaveFile(fmt.Sprintf("%s/%s/description.xml", conf.path, name), b); err != nil {
		return err
	}
	conf.SchemeManagers[NewSchemeManagerIdentifier(name)] = manager
	return nil
}

func (conf *Configuration) Download(set *IrmaIdentifierSet) (*IrmaIdentifierSet, error) {
	var contains bool
	var err error
	downloaded := &IrmaIdentifierSet{
		SchemeManagers:  map[SchemeManagerIdentifier]struct{}{},
		Issuers:         map[IssuerIdentifier]struct{}{},
		CredentialTypes: map[CredentialTypeIdentifier]struct{}{},
	}

	for manid := range set.SchemeManagers {
		if _, contains = conf.SchemeManagers[manid]; !contains {
			return nil, errors.Errorf("Unknown scheme manager: %s", manid)
		}
	}

	transport := NewHTTPTransport("")
	for issid := range set.Issuers {
		if _, contains = conf.Issuers[issid]; !contains {
			url := conf.SchemeManagers[issid.SchemeManagerIdentifier()].URL + "/" + issid.Name()
			path := fmt.Sprintf("%s/%s/%s", conf.path, issid.SchemeManagerIdentifier().String(), issid.Name())
			if err = transport.GetFile(url+"/description.xml", path+"/description.xml"); err != nil {
				return nil, err
			}
			if transport.GetFile(url+"/logo.png", path+"/logo.png"); err != nil {
				return nil, err
			}
			downloaded.Issuers[issid] = struct{}{}
		}
	}
	for issid, list := range set.PublicKeys {
		for _, count := range list {
			pk, err := conf.PublicKey(issid, count)
			if err != nil {
				return nil, err
			}
			if pk == nil {
				manager := issid.SchemeManagerIdentifier()
				suffix := fmt.Sprintf("/%s/PublicKeys/%d.xml", issid.Name(), count)
				path := fmt.Sprintf("%s/%s/%s", conf.path, manager.String(), suffix)
				if transport.GetFile(conf.SchemeManagers[manager].URL+suffix, path); err != nil {
					return nil, err
				}
			}
		}
	}
	for credid := range set.CredentialTypes {
		if _, contains := conf.CredentialTypes[credid]; !contains {
			issuer := credid.IssuerIdentifier()
			manager := issuer.SchemeManagerIdentifier()
			local := fmt.Sprintf("%s/%s/%s/Issues", conf.path, manager.Name(), issuer.Name())
			if err := fs.EnsureDirectoryExists(local); err != nil {
				return nil, err
			}
			if transport.GetFile(
				fmt.Sprintf("%s/%s/Issues/%s/description.xml",
					conf.SchemeManagers[manager].URL, issuer.Name(), credid.Name()),
				fmt.Sprintf("%s/%s/description.xml", local, credid.Name()),
			); err != nil {
				return nil, err
			}
			downloaded.CredentialTypes[credid] = struct{}{}
		}
	}

	return downloaded, conf.ParseFolder()
}
<commit_msg>Check schema version of xml files in irma_configuration<commit_after>package irma

import (
	"encoding/base64"
	"encoding/xml"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"

	"crypto/sha256"

	"fmt"

	"strings"

	"github.com/credentials/irmago/internal/fs"
	"github.com/go-errors/errors"
	"github.com/mhe/gabi"
)

// Configuration keeps track of scheme managers, issuers, credential types and public keys,
// dezerializing them from an irma_configuration folder, and downloads and saves new ones on demand.
type Configuration struct {
	SchemeManagers  map[SchemeManagerIdentifier]*SchemeManager
	Issuers         map[IssuerIdentifier]*Issuer
	CredentialTypes map[CredentialTypeIdentifier]*CredentialType

	publicKeys    map[IssuerIdentifier]map[int]*gabi.PublicKey
	reverseHashes map[string]CredentialTypeIdentifier
	path          string
	initialized   bool
}

// NewConfiguration returns a new configuration. After this
// ParseFolder() should be called to parse the specified path.
func NewConfiguration(path string, assets string) (conf *Configuration, err error) {
	conf = &Configuration{
		path: path,
	}

	if err = fs.EnsureDirectoryExists(conf.path); err != nil {
		return nil, err
	}
	if assets != "" {
		if err = conf.Copy(assets, false); err != nil {
			return nil, err
		}
	}

	return
}

// ParseFolder populates the current Configuration by parsing the storage path,
// listing the containing scheme managers, issuers and credential types.
func (conf *Configuration) ParseFolder() error {
	// Init all maps
	conf.SchemeManagers = make(map[SchemeManagerIdentifier]*SchemeManager)
	conf.Issuers = make(map[IssuerIdentifier]*Issuer)
	conf.CredentialTypes = make(map[CredentialTypeIdentifier]*CredentialType)
	conf.publicKeys = make(map[IssuerIdentifier]map[int]*gabi.PublicKey)

	conf.reverseHashes = make(map[string]CredentialTypeIdentifier)

	err := iterateSubfolders(conf.path, func(dir string) error {
		manager := &SchemeManager{}
		exists, err := pathToDescription(dir+"/description.xml", manager)
		if err != nil {
			return err
		}
		if !exists {
			return nil
		}
		if manager.XMLVersion < 7 {
			return errors.New("Unsupported scheme manager description")
		}
		conf.SchemeManagers[manager.Identifier()] = manager
		return conf.parseIssuerFolders(dir)
	})
	if err != nil {
		return err
	}
	conf.initialized = true
	return nil
}

// PublicKey returns the specified public key, or nil if not present in the Configuration.
func (conf *Configuration) PublicKey(id IssuerIdentifier, counter int) (*gabi.PublicKey, error) {
	if _, contains := conf.publicKeys[id]; !contains {
		conf.publicKeys[id] = map[int]*gabi.PublicKey{}
		if err := conf.parseKeysFolder(id); err != nil {
			return nil, err
		}
	}
	return conf.publicKeys[id][counter], nil
}

func (conf *Configuration) addReverseHash(credid CredentialTypeIdentifier) {
	hash := sha256.Sum256([]byte(credid.String()))
	conf.reverseHashes[base64.StdEncoding.EncodeToString(hash[:16])] = credid
}

func (conf *Configuration) hashToCredentialType(hash []byte) *CredentialType {
	if str, exists := conf.reverseHashes[base64.StdEncoding.EncodeToString(hash)]; exists {
		return conf.CredentialTypes[str]
	}
	return nil
}

// IsInitialized indicates whether this instance has successfully been initialized.
func (conf *Configuration) IsInitialized() bool {
	return conf.initialized
}

func (conf *Configuration) parseIssuerFolders(path string) error {
	return iterateSubfolders(path, func(dir string) error {
		issuer := &Issuer{}
		exists, err := pathToDescription(dir+"/description.xml", issuer)
		if err != nil {
			return err
		}
		if !exists {
			return nil
		}
		if issuer.XMLVersion < 4 {
			return errors.New("Unsupported issuer description")
		}
		conf.Issuers[issuer.Identifier()] = issuer
		return conf.parseCredentialsFolder(dir + "/Issues/")
	})
}

// parse $schememanager/$issuer/PublicKeys/$i.xml for $i = 1, ...
func (conf *Configuration) parseKeysFolder(issuerid IssuerIdentifier) error {
	path := fmt.Sprintf("%s/%s/%s/PublicKeys/*.xml", conf.path, issuerid.SchemeManagerIdentifier().Name(), issuerid.Name())
	files, err := filepath.Glob(path)
	if err != nil {
		return err
	}

	for _, file := range files {
		filename := filepath.Base(file)
		count := filename[:len(filename)-4]
		i, err := strconv.Atoi(count)
		if err != nil {
			continue
		}
		pk, err := gabi.NewPublicKeyFromFile(file)
		if err != nil {
			return err
		}
		pk.Issuer = issuerid.String()
		conf.publicKeys[issuerid][i] = pk
	}

	return nil
}

// parse $schememanager/$issuer/Issues/*/description.xml
func (conf *Configuration) parseCredentialsFolder(path string) error {
	return iterateSubfolders(path, func(dir string) error {
		cred := &CredentialType{}
		exists, err := pathToDescription(dir+"/description.xml", cred)
		if err != nil {
			return err
		}
		if !exists {
			return nil
		}
		if cred.XMLVersion < 4 {
			return errors.New("Unsupported credential type description")
		}
		credid := cred.Identifier()
		conf.CredentialTypes[credid] = cred
		conf.addReverseHash(credid)
		return nil
	})
}

// iterateSubfolders iterates over the subfolders of the specified path,
// calling the specified handler each time. If anything goes wrong, or
// if the caller returns a non-nil error, an error is immediately returned.
func iterateSubfolders(path string, handler func(string) error) error {
	dirs, err := filepath.Glob(path + "/*")
	if err != nil {
		return err
	}

	for _, dir := range dirs {
		stat, err := os.Stat(dir)
		if err != nil {
			return err
		}
		if !stat.IsDir() {
			continue
		}
		err = handler(dir)
		if err != nil {
			return err
		}
	}

	return nil
}

func pathToDescription(path string, description interface{}) (bool, error) {
	if _, err := os.Stat(path); err != nil {
		return false, nil
	}

	file, err := os.Open(path)
	if err != nil {
		return true, err
	}
	defer file.Close()

	bytes, err := ioutil.ReadAll(file)
	if err != nil {
		return true, err
	}

	err = xml.Unmarshal(bytes, description)
	if err != nil {
		return true, err
	}

	return true, nil
}

// Contains checks if the configuration contains the specified credential type.
func (conf *Configuration) Contains(cred CredentialTypeIdentifier) bool {
	return conf.SchemeManagers[cred.IssuerIdentifier().SchemeManagerIdentifier()] != nil &&
		conf.Issuers[cred.IssuerIdentifier()] != nil &&
		conf.CredentialTypes[cred] != nil
}

func (conf *Configuration) Copy(source string, parse bool) error {
	if err := fs.EnsureDirectoryExists(conf.path); err != nil {
		return err
	}

	err := filepath.Walk(source, filepath.WalkFunc(
		func(path string, info os.FileInfo, err error) error {
			if path == source {
				return nil
			}
			subpath := path[len(source):]
			if info.IsDir() {
				if err := fs.EnsureDirectoryExists(conf.path + subpath); err != nil {
					return err
				}
			} else {
				srcfile, err := os.Open(path)
				if err != nil {
					return err
				}
				defer srcfile.Close()
				bytes, err := ioutil.ReadAll(srcfile)
				if err != nil {
					return err
				}
				if err := fs.SaveFile(conf.path+subpath, bytes); err != nil {
					return err
				}
			}
			return nil
		}),
	)

	if err != nil {
		return err
	}
	if parse {
		return conf.ParseFolder()
	}
	return nil
}

func (conf *Configuration) DownloadSchemeManager(url string) (*SchemeManager, error) {
	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
		url = "https://" + url
	}
	if url[len(url)-1] == '/' {
		url = url[:len(url)-1]
	}
	if strings.HasSuffix(url, "/description.xml") {
		url = url[:len(url)-len("/description.xml")]
	}
	b, err := NewHTTPTransport(url).GetBytes("/description.xml")
	if err != nil {
		return nil, err
	}
	manager := &SchemeManager{}
	if err = xml.Unmarshal(b, manager); err != nil {
		return nil, err
	}

	manager.URL = url // TODO?
	return manager, nil
}

func (conf *Configuration) RemoveSchemeManager(id SchemeManagerIdentifier) error {
	// Remove everything falling under the manager's responsibility
	for credid := range conf.CredentialTypes {
		if credid.IssuerIdentifier().SchemeManagerIdentifier() == id {
			delete(conf.CredentialTypes, credid)
		}
	}
	for issid := range conf.Issuers {
		if issid.SchemeManagerIdentifier() == id {
			delete(conf.Issuers, issid)
		}
	}
	for issid := range conf.publicKeys {
		if issid.SchemeManagerIdentifier() == id {
			delete(conf.publicKeys, issid)
		}
	}
	// Remove from storage
	return os.RemoveAll(fmt.Sprintf("%s/%s", conf.path, id.String()))
	// or, remove above iterations and call .ParseFolder()?
}

func (conf *Configuration) AddSchemeManager(manager *SchemeManager) error {
	name := manager.ID
	if err := fs.EnsureDirectoryExists(fmt.Sprintf("%s/%s", conf.path, name)); err != nil {
		return err
	}
	b, err := xml.Marshal(manager)
	if err != nil {
		return err
	}
	if err := fs.SaveFile(fmt.Sprintf("%s/%s/description.xml", conf.path, name), b); err != nil {
		return err
	}
	conf.SchemeManagers[NewSchemeManagerIdentifier(name)] = manager
	return nil
}

func (conf *Configuration) Download(set *IrmaIdentifierSet) (*IrmaIdentifierSet, error) {
	var contains bool
	var err error
	downloaded := &IrmaIdentifierSet{
		SchemeManagers:  map[SchemeManagerIdentifier]struct{}{},
		Issuers:         map[IssuerIdentifier]struct{}{},
		CredentialTypes: map[CredentialTypeIdentifier]struct{}{},
	}

	for manid := range set.SchemeManagers {
		if _, contains = conf.SchemeManagers[manid]; !contains {
			return nil, errors.Errorf("Unknown scheme manager: %s", manid)
		}
	}

	transport := NewHTTPTransport("")
	for issid := range set.Issuers {
		if _, contains = conf.Issuers[issid]; !contains {
			url := conf.SchemeManagers[issid.SchemeManagerIdentifier()].URL + "/" + issid.Name()
			path := fmt.Sprintf("%s/%s/%s", conf.path, issid.SchemeManagerIdentifier().String(), issid.Name())
			if err = transport.GetFile(url+"/description.xml", path+"/description.xml"); err != nil {
				return nil, err
			}
			if transport.GetFile(url+"/logo.png", path+"/logo.png"); err != nil {
				return nil, err
			}
			downloaded.Issuers[issid] = struct{}{}
		}
	}
	for issid, list := range set.PublicKeys {
		for _, count := range list {
			pk, err := conf.PublicKey(issid, count)
			if err != nil {
				return nil, err
			}
			if pk == nil {
				manager := issid.SchemeManagerIdentifier()
				suffix := fmt.Sprintf("/%s/PublicKeys/%d.xml", issid.Name(), count)
				path := fmt.Sprintf("%s/%s/%s", conf.path, manager.String(), suffix)
				if transport.GetFile(conf.SchemeManagers[manager].URL+suffix, path); err != nil {
					return nil, err
				}
			}
		}
	}
	for credid := range set.CredentialTypes {
		if _, contains := conf.CredentialTypes[credid]; !contains {
			issuer := credid.IssuerIdentifier()
			manager := issuer.SchemeManagerIdentifier()
			local := fmt.Sprintf("%s/%s/%s/Issues", conf.path, manager.Name(), issuer.Name())
			if err := fs.EnsureDirectoryExists(local); err != nil {
				return nil, err
			}
			if transport.GetFile(
				fmt.Sprintf("%s/%s/Issues/%s/description.xml",
					conf.SchemeManagers[manager].URL, issuer.Name(), credid.Name()),
				fmt.Sprintf("%s/%s/description.xml", local, credid.Name()),
			); err != nil {
				return nil, err
			}
			downloaded.CredentialTypes[credid] = struct{}{}
		}
	}

	return downloaded, conf.ParseFolder()
}
<|endoftext|>


<commit_before>package ipv4opt

import (
	"fmt"
)

type OptionType uint8
type OptionLength uint8
type RouteAddress uint32
type OptionData uint8
type SecurityLevel uint16
type SecurityCompartment uint16
type SecurityHandlingRestriction uint16
type SecurityTCC uint32
type Route uint32
type StreamID uint16
type Timestamp uint32
type Flag uint8
type Overflow uint8
type Address uint32

const (
	EndOfOptionList         OptionType = 0
	NoOperation                        = 1
	Security                           = 130
	LooseSourceRecordRoute             = 131
	StrictSourceRecordRoute            = 137
	RecordRoute                        = 7
	StreamIdentifier                   = 136
	InternetTimestamp                  = 68
	MaxOptionsLen           int        = 40 // 60 Byte maximum size - 20 bytes for manditory fields

	Unclassified SecurityLevel = 0x0
	Confidential               = 0xF135
	EFTO                       = 0x789A
	MMMM                       = 0xBC4D
	PROG                       = 0x5E26
	Restricted                 = 0xAF13
	Secret                     = 0xD788
	TopSecret                  = 0x6BC5
	Reserved0                  = 0x35E2
	Reserved1                  = 0x9AF1
	Reserved2                  = 0x4D78
	Reserved3                  = 0x24BD
	Reserved4                  = 0x135E
	Reserved5                  = 0x89AF
	Reserved6                  = 0xC4D6
	Reserved7                  = 0xE26B
)

const (
	TSOnly    = 0
	TSAndAddr = 1
	TSPrespec = 3
)

var (
	ErrorOptionDataTooLarge      = fmt.Errorf("The length of the options data is larger than the max options length")
	ErrorOptionType              = fmt.Errorf("Invalid option type")
	ErrorNegativeOptionLength    = fmt.Errorf("Negative option length")
	ErrorNotEnoughData           = fmt.Errorf("Not enough data left to parse option")
	ErrorOptionTypeMismatch      = fmt.Errorf("Tried to convert an option to the wrong type")
	ErrorInvalidLength           = fmt.Errorf("The option length is incorrect")
	ErrorRouteLengthIncorrect    = fmt.Errorf("The length of the route data is not a multiple of 4")
	ErrorTSLengthIncorrect       = fmt.Errorf("The length of the route data is not a multiple of 4")
	ErrorStreamIDLengthIncorrect = fmt.Errorf("Then stream ID length is not 4")
)

type Option struct {
	Type   OptionType
	Length OptionLength
	Data   []OptionData
}

type Options []Option

type SecurityOption struct {
	Type        OptionType
	Length      OptionLength
	Level       SecurityLevel
	Compartment SecurityCompartment
	Restriction SecurityHandlingRestriction
	TCC         SecurityTCC
}

func (o Option) ToSecurity() (SecurityOption, error) {
	so := SecurityOption{}
	so.Type = o.Type
	so.Length = o.Length
	if o.Type != Security {
		return so, ErrorOptionTypeMismatch
	}
	if o.Length != 11 {
		return so, ErrorInvalidLength
	}
	data := o.Data
	so.Level |= SecurityLevel(data[0]) << 8
	so.Level |= SecurityLevel(data[1])

	so.Compartment |= SecurityCompartment(data[2]) << 8
	so.Compartment |= SecurityCompartment(data[3])

	so.Restriction |= SecurityHandlingRestriction(data[4]) << 8
	so.Restriction |= SecurityHandlingRestriction(data[5])

	so.TCC |= SecurityTCC(data[6]) << 16
	so.TCC |= SecurityTCC(data[7]) << 8
	so.TCC |= SecurityTCC(data[8])

	return so, nil
}

type RecordRouteOption struct {
	Type   OptionType
	Length OptionLength
	Routes []Route
}

func (o Option) ToRecordRoute() (RecordRouteOption, error) {
	rro := RecordRouteOption{}
	rro.Type = o.Type
	rro.Length = o.Length
	if o.Type != StrictSourceRecordRoute &&
		o.Type != LooseSourceRecordRoute &&
		o.Type != RecordRoute {
		return rro, ErrorOptionTypeMismatch
	}
	routeLen := rro.Length - 3 // The length of routes is length - 3 because length include the pointer type and length
	if routeLen%4 != 0 {
		return rro, ErrorRouteLengthIncorrect
	}
	for i := 0; i < int(routeLen); i += 4 {
		var route Route
		route |= Route(o.Data[i]) << 24
		route |= Route(o.Data[i+1]) << 16
		route |= Route(o.Data[i+2]) << 8
		route |= Route(o.Data[i+3])

		rro.Routes = append(rro.Routes, route)
	}
	return rro, nil
}

type StreamIdentifierOption struct {
	Type   OptionType
	Length OptionLength
	ID     StreamID
}

func (o Option) ToStreamID() (StreamIdentifierOption, error) {
	sid := StreamIdentifierOption{}
	sid.Type = o.Type
	sid.Length = o.Length
	if o.Type != StreamIdentifier {
		return sid, ErrorOptionTypeMismatch
	}
	if o.Length != 4 {
		return sid, ErrorStreamIDLengthIncorrect
	}
	sid.ID |= StreamID(o.Data[0]) << 8
	sid.ID |= StreamID(o.Data[1])

	return sid, nil

}

type Stamp struct {
	Time Timestamp
	Addr Address
}

type TimeStampOption struct {
	Type   OptionType
	Length OptionLength
	Flags  Flag
	Over   Overflow
	Stamps []Stamp
}

func (o Option) ToTimeStamp() (TimeStampOption, error) {
	ts := TimeStampOption{}
	ts.Type = o.Type
	ts.Length = o.Length
	if o.Type != InternetTimestamp {
		return ts, ErrorOptionTypeMismatch
	}
	if len(o.Data) > MaxOptionsLen {
		return ts, ErrorOptionDataTooLarge
	}
	ts.Over = Overflow(o.Data[1] >> 4)
	ts.Flags = Flag(o.Data[1] & 0x0F)
	// Take off two because of the flag and overflow byte and the ponter byte
	if len(o.Data)%4-2 != 0 && ts.Flags != TSOnly {
		return ts, ErrorTSLengthIncorrect
	}
	var err error
	switch ts.Flags {
	case TSOnly:
		ts.Stamps, err = getStampsTSOnly(o.Data[2:], len(o.Data)-2)
		if err != nil {
			return ts, err
		}
	case TSAndAddr, TSPrespec:
		ts.Stamps, err = getStamps(o.Data[2:], len(o.Data)-2)
		if err != nil {
			return ts, err
		}
	}
	return ts, nil
}

func getStampsTSOnly(data []OptionData, length int) ([]Stamp, error) {
	stamp := make([]Stamp, 0)
	for i := 0; i < length; i += 4 {
		st := Stamp{}
		st.Time |= Timestamp(data[i]) << 24
		st.Time |= Timestamp(data[i+1]) << 16
		st.Time |= Timestamp(data[i+2]) << 8
		st.Time |= Timestamp(data[i+3])
		stamp = append(stamp, st)
	}
	return stamp, nil
}

func getStamps(data []OptionData, length int) ([]Stamp, error) {
	stamp := make([]Stamp, 0)
	for i := 0; i < length; i += 8 {
		st := Stamp{}
		st.Addr |= Address(data[i]) << 24
		st.Addr |= Address(data[i+1]) << 16
		st.Addr |= Address(data[i+2]) << 8
		st.Addr |= Address(data[i+3])
		st.Time |= Timestamp(data[i+4]) << 24
		st.Time |= Timestamp(data[i+5]) << 16
		st.Time |= Timestamp(data[i+6]) << 8
		st.Time |= Timestamp(data[i+7])
		stamp = append(stamp, st)
	}
	return stamp, nil
}

func Parse(opts []byte) (Options, error) {
	optsLen := len(opts)
	if optsLen > MaxOptionsLen {
		return Options{}, ErrorOptionDataTooLarge
	}
	if optsLen == 0 {
		return Options{}, nil
	}
	options := make(Options, 0)
	for i := 0; i < optsLen; {
		option := Option{}
		oType, err := getOptionType(opts[i])
		if err != nil {
			return options, err
		}
		i++
		option.Type = oType
		if oType == EndOfOptionList {
			return append(options, option), nil
		}
		if oType == NoOperation {
			options = append(options, option)
			continue
		}
		data, l, n, err := parseOption(opts[i:])
		if err != nil {
			return Options{}, err
		}
		i += n
		option.Length = l
		option.Data = data
		options = append(options, option)
	}
	return options, nil

}

func parseOption(opts []byte) ([]OptionData, OptionLength, int, error) {
	l := opts[0]
	if l < 0 {
		return []OptionData{}, 0, 0, ErrorNegativeOptionLength
	}
	ol := OptionLength(l)
	// Length includes the length byte and type byte so read l - 2 more bytes
	rem := int(l) - 2
	if rem > len(opts)-1 { // If the remaining data is longer than the length of the options data - 1 for length byte
		return []OptionData{}, 0, 0, ErrorNotEnoughData
	}
	// Add one to rem because the synax is [x:)
	dataBytes := opts[1 : rem+1]
	dbl := len(dataBytes)
	ods := make([]OptionData, 0)
	for i := 0; i < dbl; i++ {
		ods = append(ods, OptionData(dataBytes[i]))
	}
	return ods, ol, int(l), nil
}

func getOptionType(b byte) (OptionType, error) {
	switch OptionType(b) {
	case EndOfOptionList:
		return EndOfOptionList, nil
	case NoOperation:
		return NoOperation, nil
	case Security:
		return Security, nil
	case LooseSourceRecordRoute:
		return LooseSourceRecordRoute, nil
	case StrictSourceRecordRoute:
		return StrictSourceRecordRoute, nil
	case RecordRoute:
		return RecordRoute, nil
	case StreamIdentifier:
		return StreamIdentifier, nil
	case InternetTimestamp:
		return InternetTimestamp, nil
	default:
		//Just return EndOfOptionList to satisfy return
		return EndOfOptionList, ErrorOptionType
	}
}
<commit_msg>Added string methods for ip<commit_after>package ipv4opt

import (
	"fmt"
	"net"
)

type OptionType uint8
type OptionLength uint8
type RouteAddress uint32
type OptionData uint8
type SecurityLevel uint16
type SecurityCompartment uint16
type SecurityHandlingRestriction uint16
type SecurityTCC uint32
type Route uint32
type StreamID uint16
type Timestamp uint32
type Flag uint8
type Overflow uint8
type Address uint32

func (addr Address) String() string {
	var a, b, c, d byte
	a = byte(addr >> 24)
	b = byte((addr & 0x00ff0000) >> 16)
	c = byte((addr & 0x0000ff00) >> 8)
	d = byte(addr & 0x000000ff)
	return net.IPv4(a, b, c, d).String()
}

func (r Route) String() string {
	var a, b, c, d byte
	a = byte(r >> 24)
	b = byte((r & 0x00ff0000) >> 16)
	c = byte((r & 0x0000ff00) >> 8)
	d = byte(r & 0x000000ff)
	return net.IPv4(a, b, c, d).String()
}

const (
	EndOfOptionList         OptionType = 0
	NoOperation                        = 1
	Security                           = 130
	LooseSourceRecordRoute             = 131
	StrictSourceRecordRoute            = 137
	RecordRoute                        = 7
	StreamIdentifier                   = 136
	InternetTimestamp                  = 68
	MaxOptionsLen           int        = 40 // 60 Byte maximum size - 20 bytes for manditory fields

	Unclassified SecurityLevel = 0x0
	Confidential               = 0xF135
	EFTO                       = 0x789A
	MMMM                       = 0xBC4D
	PROG                       = 0x5E26
	Restricted                 = 0xAF13
	Secret                     = 0xD788
	TopSecret                  = 0x6BC5
	Reserved0                  = 0x35E2
	Reserved1                  = 0x9AF1
	Reserved2                  = 0x4D78
	Reserved3                  = 0x24BD
	Reserved4                  = 0x135E
	Reserved5                  = 0x89AF
	Reserved6                  = 0xC4D6
	Reserved7                  = 0xE26B
)

const (
	TSOnly    = 0
	TSAndAddr = 1
	TSPrespec = 3
)

var (
	ErrorOptionDataTooLarge      = fmt.Errorf("The length of the options data is larger than the max options length")
	ErrorOptionType              = fmt.Errorf("Invalid option type")
	ErrorNegativeOptionLength    = fmt.Errorf("Negative option length")
	ErrorNotEnoughData           = fmt.Errorf("Not enough data left to parse option")
	ErrorOptionTypeMismatch      = fmt.Errorf("Tried to convert an option to the wrong type")
	ErrorInvalidLength           = fmt.Errorf("The option length is incorrect")
	ErrorRouteLengthIncorrect    = fmt.Errorf("The length of the route data is not a multiple of 4")
	ErrorTSLengthIncorrect       = fmt.Errorf("The length of the route data is not a multiple of 4")
	ErrorStreamIDLengthIncorrect = fmt.Errorf("Then stream ID length is not 4")
)

type Option struct {
	Type   OptionType
	Length OptionLength
	Data   []OptionData
}

type Options []Option

type SecurityOption struct {
	Type        OptionType
	Length      OptionLength
	Level       SecurityLevel
	Compartment SecurityCompartment
	Restriction SecurityHandlingRestriction
	TCC         SecurityTCC
}

func (o Option) ToSecurity() (SecurityOption, error) {
	so := SecurityOption{}
	so.Type = o.Type
	so.Length = o.Length
	if o.Type != Security {
		return so, ErrorOptionTypeMismatch
	}
	if o.Length != 11 {
		return so, ErrorInvalidLength
	}
	data := o.Data
	so.Level |= SecurityLevel(data[0]) << 8
	so.Level |= SecurityLevel(data[1])

	so.Compartment |= SecurityCompartment(data[2]) << 8
	so.Compartment |= SecurityCompartment(data[3])

	so.Restriction |= SecurityHandlingRestriction(data[4]) << 8
	so.Restriction |= SecurityHandlingRestriction(data[5])

	so.TCC |= SecurityTCC(data[6]) << 16
	so.TCC |= SecurityTCC(data[7]) << 8
	so.TCC |= SecurityTCC(data[8])

	return so, nil
}

type RecordRouteOption struct {
	Type   OptionType
	Length OptionLength
	Routes []Route
}

func (o Option) ToRecordRoute() (RecordRouteOption, error) {
	rro := RecordRouteOption{}
	rro.Type = o.Type
	rro.Length = o.Length
	if o.Type != StrictSourceRecordRoute &&
		o.Type != LooseSourceRecordRoute &&
		o.Type != RecordRoute {
		return rro, ErrorOptionTypeMismatch
	}
	routeLen := rro.Length - 3 // The length of routes is length - 3 because length include the pointer type and length
	if routeLen%4 != 0 {
		return rro, ErrorRouteLengthIncorrect
	}
	for i := 0; i < int(routeLen); i += 4 {
		var route Route
		route |= Route(o.Data[i]) << 24
		route |= Route(o.Data[i+1]) << 16
		route |= Route(o.Data[i+2]) << 8
		route |= Route(o.Data[i+3])

		rro.Routes = append(rro.Routes, route)
	}
	return rro, nil
}

type StreamIdentifierOption struct {
	Type   OptionType
	Length OptionLength
	ID     StreamID
}

func (o Option) ToStreamID() (StreamIdentifierOption, error) {
	sid := StreamIdentifierOption{}
	sid.Type = o.Type
	sid.Length = o.Length
	if o.Type != StreamIdentifier {
		return sid, ErrorOptionTypeMismatch
	}
	if o.Length != 4 {
		return sid, ErrorStreamIDLengthIncorrect
	}
	sid.ID |= StreamID(o.Data[0]) << 8
	sid.ID |= StreamID(o.Data[1])

	return sid, nil

}

type Stamp struct {
	Time Timestamp
	Addr Address
}

type TimeStampOption struct {
	Type   OptionType
	Length OptionLength
	Flags  Flag
	Over   Overflow
	Stamps []Stamp
}

func (o Option) ToTimeStamp() (TimeStampOption, error) {
	ts := TimeStampOption{}
	ts.Type = o.Type
	ts.Length = o.Length
	if o.Type != InternetTimestamp {
		return ts, ErrorOptionTypeMismatch
	}
	if len(o.Data) > MaxOptionsLen {
		return ts, ErrorOptionDataTooLarge
	}
	ts.Over = Overflow(o.Data[1] >> 4)
	ts.Flags = Flag(o.Data[1] & 0x0F)
	// Take off two because of the flag and overflow byte and the ponter byte
	if len(o.Data)%4-2 != 0 && ts.Flags != TSOnly {
		return ts, ErrorTSLengthIncorrect
	}
	var err error
	switch ts.Flags {
	case TSOnly:
		ts.Stamps, err = getStampsTSOnly(o.Data[2:], len(o.Data)-2)
		if err != nil {
			return ts, err
		}
	case TSAndAddr, TSPrespec:
		ts.Stamps, err = getStamps(o.Data[2:], len(o.Data)-2)
		if err != nil {
			return ts, err
		}
	}
	return ts, nil
}

func getStampsTSOnly(data []OptionData, length int) ([]Stamp, error) {
	stamp := make([]Stamp, 0)
	for i := 0; i < length; i += 4 {
		st := Stamp{}
		st.Time |= Timestamp(data[i]) << 24
		st.Time |= Timestamp(data[i+1]) << 16
		st.Time |= Timestamp(data[i+2]) << 8
		st.Time |= Timestamp(data[i+3])
		stamp = append(stamp, st)
	}
	return stamp, nil
}

func getStamps(data []OptionData, length int) ([]Stamp, error) {
	stamp := make([]Stamp, 0)
	for i := 0; i < length; i += 8 {
		st := Stamp{}
		st.Addr |= Address(data[i]) << 24
		st.Addr |= Address(data[i+1]) << 16
		st.Addr |= Address(data[i+2]) << 8
		st.Addr |= Address(data[i+3])
		st.Time |= Timestamp(data[i+4]) << 24
		st.Time |= Timestamp(data[i+5]) << 16
		st.Time |= Timestamp(data[i+6]) << 8
		st.Time |= Timestamp(data[i+7])
		stamp = append(stamp, st)
	}
	return stamp, nil
}

func Parse(opts []byte) (Options, error) {
	optsLen := len(opts)
	if optsLen > MaxOptionsLen {
		return Options{}, ErrorOptionDataTooLarge
	}
	if optsLen == 0 {
		return Options{}, nil
	}
	options := make(Options, 0)
	for i := 0; i < optsLen; {
		option := Option{}
		oType, err := getOptionType(opts[i])
		if err != nil {
			return options, err
		}
		i++
		option.Type = oType
		if oType == EndOfOptionList {
			return append(options, option), nil
		}
		if oType == NoOperation {
			options = append(options, option)
			continue
		}
		data, l, n, err := parseOption(opts[i:])
		if err != nil {
			return Options{}, err
		}
		i += n
		option.Length = l
		option.Data = data
		options = append(options, option)
	}
	return options, nil

}

func parseOption(opts []byte) ([]OptionData, OptionLength, int, error) {
	l := opts[0]
	if l < 0 {
		return []OptionData{}, 0, 0, ErrorNegativeOptionLength
	}
	ol := OptionLength(l)
	// Length includes the length byte and type byte so read l - 2 more bytes
	rem := int(l) - 2
	if rem > len(opts)-1 { // If the remaining data is longer than the length of the options data - 1 for length byte
		return []OptionData{}, 0, 0, ErrorNotEnoughData
	}
	// Add one to rem because the synax is [x:)
	dataBytes := opts[1 : rem+1]
	dbl := len(dataBytes)
	ods := make([]OptionData, 0)
	for i := 0; i < dbl; i++ {
		ods = append(ods, OptionData(dataBytes[i]))
	}
	return ods, ol, int(l), nil
}

func getOptionType(b byte) (OptionType, error) {
	switch OptionType(b) {
	case EndOfOptionList:
		return EndOfOptionList, nil
	case NoOperation:
		return NoOperation, nil
	case Security:
		return Security, nil
	case LooseSourceRecordRoute:
		return LooseSourceRecordRoute, nil
	case StrictSourceRecordRoute:
		return StrictSourceRecordRoute, nil
	case RecordRoute:
		return RecordRoute, nil
	case StreamIdentifier:
		return StreamIdentifier, nil
	case InternetTimestamp:
		return InternetTimestamp, nil
	default:
		//Just return EndOfOptionList to satisfy return
		return EndOfOptionList, ErrorOptionType
	}
}
<|endoftext|>


<commit_before>package ec2fzf

import (
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

type Options struct {
	Regions         []string
	UsePrivateIp    bool
	Template        string
	PreviewTemplate string
	Filters         []string
}

func ParseOptions() Options {
	viper.SetConfigName("config")
	viper.SetConfigType("toml")
	viper.AddConfigPath("$HOME/.config/ec2-fzf")
	if err := viper.ReadInConfig(); err != nil {
		if _, ok := err.(viper.ConfigFileNotFoundError); ok {
			// Config file not found; ignore error if desired
		} else {
			panic(err)
		}
	}

	pflag.StringSlice("region", []string{"us-east-1"}, "The AWS region")
	pflag.Bool("use-private-ip", true, "Return the private ip of the instance selected")
	pflag.StringSlice("filters", []string{}, "Filters to apply with the ec2 api call")
	pflag.Parse()
	viper.BindPFlags(pflag.CommandLine)

	viper.RegisterAlias("UsePrivateIp", "use-private-ip")
	viper.RegisterAlias("regions", "region")

	viper.SetDefault("Region", "us-east-1")
	viper.SetDefault("UsePrivateIp", false)
	viper.SetDefault("Template", `{{ .InstanceId }}: {{index .Tags "Name"}}`)
	viper.SetDefault("PreviewTemplate", `
			Instance Id: {{.InstanceId}}
			Name:        {{index .Tags "Name"}}
			Private IP:  {{.PrivateIpAddress}}
			Public IP:   {{.PublicIpAddress}}

			Tags:
			{{ range $key, $value := .Tags -}}
				{{ indent 2 $key }}: {{ $value }}
			{{- end -}}
		`,
	)

	return Options{
		Regions:         viper.GetStringSlice("Regions"),
		UsePrivateIp:    viper.GetBool("UsePrivateIp"),
		Template:        viper.GetString("Template"),
		PreviewTemplate: viper.GetString("PreviewTemplate"),
		Filters:         viper.GetStringSlice("Filters"),
	}
}
<commit_msg>Fix default formatting of instance details with multiple instances<commit_after>package ec2fzf

import (
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

type Options struct {
	Regions         []string
	UsePrivateIp    bool
	Template        string
	PreviewTemplate string
	Filters         []string
}

func ParseOptions() Options {
	viper.SetConfigName("config")
	viper.SetConfigType("toml")
	viper.AddConfigPath("$HOME/.config/ec2-fzf")
	if err := viper.ReadInConfig(); err != nil {
		if _, ok := err.(viper.ConfigFileNotFoundError); ok {
			// Config file not found; ignore error if desired
		} else {
			panic(err)
		}
	}

	pflag.StringSlice("region", []string{"us-east-1"}, "The AWS region")
	pflag.Bool("use-private-ip", true, "Return the private ip of the instance selected")
	pflag.StringSlice("filters", []string{}, "Filters to apply with the ec2 api call")
	pflag.Parse()
	viper.BindPFlags(pflag.CommandLine)

	viper.RegisterAlias("UsePrivateIp", "use-private-ip")
	viper.RegisterAlias("regions", "region")

	viper.SetDefault("Region", "us-east-1")
	viper.SetDefault("UsePrivateIp", false)
	viper.SetDefault("Template", `{{ .InstanceId }}: {{index .Tags "Name"}}`)
	viper.SetDefault("PreviewTemplate", `
			Instance Id: {{.InstanceId}}
			Name:        {{index .Tags "Name"}}
			Private IP:  {{.PrivateIpAddress}}
			Public IP:   {{.PublicIpAddress}}

			Tags:
			{{ range $key, $value := .Tags }}
				{{ indent 2 $key }}: {{ $value }}
			{{- end -}}
		`,
	)

	return Options{
		Regions:         viper.GetStringSlice("Regions"),
		UsePrivateIp:    viper.GetBool("UsePrivateIp"),
		Template:        viper.GetString("Template"),
		PreviewTemplate: viper.GetString("PreviewTemplate"),
		Filters:         viper.GetStringSlice("Filters"),
	}
}
<|endoftext|>


<commit_before>package jsoniter

import (
	"strconv"
)

var intDigits []int8

const uint32SafeToMultiply10 = uint32(0xffffffff)/10 - 1
const uint64SafeToMultiple10 = uint64(0xffffffffffffffff)/10 - 1
const int64Max = uint64(0x7fffffffffffffff)
const int32Max = uint32(0x7fffffff)
const int16Max = uint32(0x7fff)
const uint16Max = uint32(0xffff)
const int8Max = uint32(0x7fff)
const uint8Max = uint32(0xffff)

func init() {
	intDigits = make([]int8, 256)
	for i := 0; i < len(intDigits); i++ {
		intDigits[i] = invalidCharForNumber
	}
	for i := int8('0'); i <= int8('9'); i++ {
		intDigits[i] = i - int8('0')
	}
}

func (iter *Iterator) ReadUint() uint {
	return uint(iter.ReadUint64())
}

func (iter *Iterator) ReadInt() int {
	return int(iter.ReadInt64())
}

func (iter *Iterator) ReadInt8() (ret int8) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint32(iter.readByte())
		if val > int8Max+1 {
			iter.ReportError("ReadInt8", "overflow: "+strconv.FormatInt(int64(val), 10))
			return
		}
		return -int8(val)
	} else {
		val := iter.readUint32(c)
		if val > int8Max {
			iter.ReportError("ReadInt8", "overflow: "+strconv.FormatInt(int64(val), 10))
			return
		}
		return int8(val)
	}
}

func (iter *Iterator) ReadUint8() (ret uint8) {
	val := iter.readUint32(iter.nextToken())
	if val > uint8Max {
		iter.ReportError("ReadUint8", "overflow: "+strconv.FormatInt(int64(val), 10))
		return
	}
	return uint8(val)
}

func (iter *Iterator) ReadInt16() (ret int16) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint32(iter.readByte())
		if val > int16Max+1 {
			iter.ReportError("ReadInt16", "overflow: "+strconv.FormatInt(int64(val), 10))
			return
		}
		return -int16(val)
	} else {
		val := iter.readUint32(c)
		if val > int16Max {
			iter.ReportError("ReadInt16", "overflow: "+strconv.FormatInt(int64(val), 10))
			return
		}
		return int16(val)
	}
}

func (iter *Iterator) ReadUint16() (ret uint16) {
	val := iter.readUint32(iter.nextToken())
	if val > uint16Max {
		iter.ReportError("ReadUint16", "overflow: "+strconv.FormatInt(int64(val), 10))
		return
	}
	return uint16(val)
}

func (iter *Iterator) ReadInt32() (ret int32) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint32(iter.readByte())
		if val > int32Max+1 {
			iter.ReportError("ReadInt32", "overflow: "+strconv.FormatInt(int64(val), 10))
			return
		}
		return -int32(val)
	} else {
		val := iter.readUint32(c)
		if val > int32Max {
			iter.ReportError("ReadInt32", "overflow: "+strconv.FormatInt(int64(val), 10))
			return
		}
		return int32(val)
	}
}

func (iter *Iterator) ReadUint32() (ret uint32) {
	return iter.readUint32(iter.nextToken())
}

func (iter *Iterator) readUint32(c byte) (ret uint32) {
	ind := intDigits[c]
	if ind == 0 {
		return 0 // single zero
	}
	if ind == invalidCharForNumber {
		iter.ReportError("readUint32", "unexpected character: "+string([]byte{byte(ind)}))
		return
	}
	value := uint32(ind)
	if iter.tail-iter.head > 10 {
		i := iter.head
		ind2 := intDigits[iter.buf[i]]
		if ind2 == invalidCharForNumber {
			iter.head = i
			return value
		}
		i++
		ind3 := intDigits[iter.buf[i]]
		if ind3 == invalidCharForNumber {
			iter.head = i
			return value*10 + uint32(ind2)
		}
		//iter.head = i + 1
		//value = value * 100 + uint32(ind2) * 10 + uint32(ind3)
		i++
		ind4 := intDigits[iter.buf[i]]
		if ind4 == invalidCharForNumber {
			iter.head = i
			return value*100 + uint32(ind2)*10 + uint32(ind3)
		}
		i++
		ind5 := intDigits[iter.buf[i]]
		if ind5 == invalidCharForNumber {
			iter.head = i
			return value*1000 + uint32(ind2)*100 + uint32(ind3)*10 + uint32(ind4)
		}
		i++
		ind6 := intDigits[iter.buf[i]]
		if ind6 == invalidCharForNumber {
			iter.head = i
			return value*10000 + uint32(ind2)*1000 + uint32(ind3)*100 + uint32(ind4)*10 + uint32(ind5)
		}
		i++
		ind7 := intDigits[iter.buf[i]]
		if ind7 == invalidCharForNumber {
			iter.head = i
			return value*100000 + uint32(ind2)*10000 + uint32(ind3)*1000 + uint32(ind4)*100 + uint32(ind5)*10 + uint32(ind6)
		}
		i++
		ind8 := intDigits[iter.buf[i]]
		if ind8 == invalidCharForNumber {
			iter.head = i
			return value*1000000 + uint32(ind2)*100000 + uint32(ind3)*10000 + uint32(ind4)*1000 + uint32(ind5)*100 + uint32(ind6)*10 + uint32(ind7)
		}
		i++
		ind9 := intDigits[iter.buf[i]]
		value = value*10000000 + uint32(ind2)*1000000 + uint32(ind3)*100000 + uint32(ind4)*10000 + uint32(ind5)*1000 + uint32(ind6)*100 + uint32(ind7)*10 + uint32(ind8)
		iter.head = i
		if ind9 == invalidCharForNumber {
			return value
		}
	}
	for {
		for i := iter.head; i < iter.tail; i++ {
			ind = intDigits[iter.buf[i]]
			if ind == invalidCharForNumber {
				iter.head = i
				return value
			}
			if value > uint32SafeToMultiply10 {
				value2 := (value << 3) + (value << 1) + uint32(ind)
				if value2 < value {
					iter.ReportError("readUint32", "overflow")
					return
				} else {
					value = value2
					continue
				}
			}
			value = (value << 3) + (value << 1) + uint32(ind)
		}
		if !iter.loadMore() {
			return value
		}
	}
}

func (iter *Iterator) ReadInt64() (ret int64) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint64(iter.readByte())
		if val > int64Max+1 {
			iter.ReportError("ReadInt64", "overflow: "+strconv.FormatUint(uint64(val), 10))
			return
		}
		return -int64(val)
	} else {
		val := iter.readUint64(c)
		if val > int64Max {
			iter.ReportError("ReadInt64", "overflow: "+strconv.FormatUint(uint64(val), 10))
			return
		}
		return int64(val)
	}
}

func (iter *Iterator) ReadUint64() uint64 {
	return iter.readUint64(iter.nextToken())
}

func (iter *Iterator) readUint64(c byte) (ret uint64) {
	ind := intDigits[c]
	if ind == 0 {
		return 0 // single zero
	}
	if ind == invalidCharForNumber {
		iter.ReportError("readUint64", "unexpected character: "+string([]byte{byte(ind)}))
		return
	}
	value := uint64(ind)
	for {
		for i := iter.head; i < iter.tail; i++ {
			ind = intDigits[iter.buf[i]]
			if ind == invalidCharForNumber {
				iter.head = i
				return value
			}
			if value > uint64SafeToMultiple10 {
				value2 := (value << 3) + (value << 1) + uint64(ind)
				if value2 < value {
					iter.ReportError("readUint64", "overflow")
					return
				} else {
					value = value2
					continue
				}
			}
			value = (value << 3) + (value << 1) + uint64(ind)
		}
		if !iter.loadMore() {
			return value
		}
	}
}
<commit_msg>#68 remove redundant math max constants<commit_after>package jsoniter

import (
	"strconv"
	"math"
)

var intDigits []int8

const uint32SafeToMultiply10 = uint32(0xffffffff)/10 - 1
const uint64SafeToMultiple10 = uint64(0xffffffffffffffff)/10 - 1

func init() {
	intDigits = make([]int8, 256)
	for i := 0; i < len(intDigits); i++ {
		intDigits[i] = invalidCharForNumber
	}
	for i := int8('0'); i <= int8('9'); i++ {
		intDigits[i] = i - int8('0')
	}
}

func (iter *Iterator) ReadUint() uint {
	return uint(iter.ReadUint64())
}

func (iter *Iterator) ReadInt() int {
	return int(iter.ReadInt64())
}

func (iter *Iterator) ReadInt8() (ret int8) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint32(iter.readByte())
		if val > math.MaxInt8+1 {
			iter.ReportError("ReadInt8", "overflow: "+strconv.FormatInt(int64(val), 10))
			return
		}
		return -int8(val)
	} else {
		val := iter.readUint32(c)
		if val > math.MaxInt8 {
			iter.ReportError("ReadInt8", "overflow: "+strconv.FormatInt(int64(val), 10))
			return
		}
		return int8(val)
	}
}

func (iter *Iterator) ReadUint8() (ret uint8) {
	val := iter.readUint32(iter.nextToken())
	if val > math.MaxUint8 {
		iter.ReportError("ReadUint8", "overflow: "+strconv.FormatInt(int64(val), 10))
		return
	}
	return uint8(val)
}

func (iter *Iterator) ReadInt16() (ret int16) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint32(iter.readByte())
		if val > math.MaxInt16+1 {
			iter.ReportError("ReadInt16", "overflow: "+strconv.FormatInt(int64(val), 10))
			return
		}
		return -int16(val)
	} else {
		val := iter.readUint32(c)
		if val > math.MaxInt16 {
			iter.ReportError("ReadInt16", "overflow: "+strconv.FormatInt(int64(val), 10))
			return
		}
		return int16(val)
	}
}

func (iter *Iterator) ReadUint16() (ret uint16) {
	val := iter.readUint32(iter.nextToken())
	if val > math.MaxUint16 {
		iter.ReportError("ReadUint16", "overflow: "+strconv.FormatInt(int64(val), 10))
		return
	}
	return uint16(val)
}

func (iter *Iterator) ReadInt32() (ret int32) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint32(iter.readByte())
		if val > math.MaxInt32+1 {
			iter.ReportError("ReadInt32", "overflow: "+strconv.FormatInt(int64(val), 10))
			return
		}
		return -int32(val)
	} else {
		val := iter.readUint32(c)
		if val > math.MaxInt32 {
			iter.ReportError("ReadInt32", "overflow: "+strconv.FormatInt(int64(val), 10))
			return
		}
		return int32(val)
	}
}

func (iter *Iterator) ReadUint32() (ret uint32) {
	return iter.readUint32(iter.nextToken())
}

func (iter *Iterator) readUint32(c byte) (ret uint32) {
	ind := intDigits[c]
	if ind == 0 {
		return 0 // single zero
	}
	if ind == invalidCharForNumber {
		iter.ReportError("readUint32", "unexpected character: "+string([]byte{byte(ind)}))
		return
	}
	value := uint32(ind)
	if iter.tail-iter.head > 10 {
		i := iter.head
		ind2 := intDigits[iter.buf[i]]
		if ind2 == invalidCharForNumber {
			iter.head = i
			return value
		}
		i++
		ind3 := intDigits[iter.buf[i]]
		if ind3 == invalidCharForNumber {
			iter.head = i
			return value*10 + uint32(ind2)
		}
		//iter.head = i + 1
		//value = value * 100 + uint32(ind2) * 10 + uint32(ind3)
		i++
		ind4 := intDigits[iter.buf[i]]
		if ind4 == invalidCharForNumber {
			iter.head = i
			return value*100 + uint32(ind2)*10 + uint32(ind3)
		}
		i++
		ind5 := intDigits[iter.buf[i]]
		if ind5 == invalidCharForNumber {
			iter.head = i
			return value*1000 + uint32(ind2)*100 + uint32(ind3)*10 + uint32(ind4)
		}
		i++
		ind6 := intDigits[iter.buf[i]]
		if ind6 == invalidCharForNumber {
			iter.head = i
			return value*10000 + uint32(ind2)*1000 + uint32(ind3)*100 + uint32(ind4)*10 + uint32(ind5)
		}
		i++
		ind7 := intDigits[iter.buf[i]]
		if ind7 == invalidCharForNumber {
			iter.head = i
			return value*100000 + uint32(ind2)*10000 + uint32(ind3)*1000 + uint32(ind4)*100 + uint32(ind5)*10 + uint32(ind6)
		}
		i++
		ind8 := intDigits[iter.buf[i]]
		if ind8 == invalidCharForNumber {
			iter.head = i
			return value*1000000 + uint32(ind2)*100000 + uint32(ind3)*10000 + uint32(ind4)*1000 + uint32(ind5)*100 + uint32(ind6)*10 + uint32(ind7)
		}
		i++
		ind9 := intDigits[iter.buf[i]]
		value = value*10000000 + uint32(ind2)*1000000 + uint32(ind3)*100000 + uint32(ind4)*10000 + uint32(ind5)*1000 + uint32(ind6)*100 + uint32(ind7)*10 + uint32(ind8)
		iter.head = i
		if ind9 == invalidCharForNumber {
			return value
		}
	}
	for {
		for i := iter.head; i < iter.tail; i++ {
			ind = intDigits[iter.buf[i]]
			if ind == invalidCharForNumber {
				iter.head = i
				return value
			}
			if value > uint32SafeToMultiply10 {
				value2 := (value << 3) + (value << 1) + uint32(ind)
				if value2 < value {
					iter.ReportError("readUint32", "overflow")
					return
				} else {
					value = value2
					continue
				}
			}
			value = (value << 3) + (value << 1) + uint32(ind)
		}
		if !iter.loadMore() {
			return value
		}
	}
}

func (iter *Iterator) ReadInt64() (ret int64) {
	c := iter.nextToken()
	if c == '-' {
		val := iter.readUint64(iter.readByte())
		if val > math.MaxInt64+1 {
			iter.ReportError("ReadInt64", "overflow: "+strconv.FormatUint(uint64(val), 10))
			return
		}
		return -int64(val)
	} else {
		val := iter.readUint64(c)
		if val > math.MaxInt64 {
			iter.ReportError("ReadInt64", "overflow: "+strconv.FormatUint(uint64(val), 10))
			return
		}
		return int64(val)
	}
}

func (iter *Iterator) ReadUint64() uint64 {
	return iter.readUint64(iter.nextToken())
}

func (iter *Iterator) readUint64(c byte) (ret uint64) {
	ind := intDigits[c]
	if ind == 0 {
		return 0 // single zero
	}
	if ind == invalidCharForNumber {
		iter.ReportError("readUint64", "unexpected character: "+string([]byte{byte(ind)}))
		return
	}
	value := uint64(ind)
	for {
		for i := iter.head; i < iter.tail; i++ {
			ind = intDigits[iter.buf[i]]
			if ind == invalidCharForNumber {
				iter.head = i
				return value
			}
			if value > uint64SafeToMultiple10 {
				value2 := (value << 3) + (value << 1) + uint64(ind)
				if value2 < value {
					iter.ReportError("readUint64", "overflow")
					return
				} else {
					value = value2
					continue
				}
			}
			value = (value << 3) + (value << 1) + uint64(ind)
		}
		if !iter.loadMore() {
			return value
		}
	}
}
<|endoftext|>


<commit_before>package hatena

import (
	"net/http"
	"os"
	"testing"
)

func TestEntryInfo(t *testing.T) {

	client := testClientFile(http.StatusOK, "test_data/entry.txt")
	result, err := client.EntryInfo("https://github.com/")

	if err != nil {
		t.Error(err)
	}
	if result.Eid != nil {
		t.Error("")
	}
	if result.Title != nil {
		t.Error("")
	}
	if result.Count != nil {
		t.Error("")
	}
	if result.Url != nil {
		t.Error("")
	}
	if result.EntryUrl != nil {
		t.Error("")
	}
	if result.Screenshot != nil {
		t.Error("")
	}
	if result.Bookmarks[0].User != nil {
		t.Error("")
	}
	if result.Bookmarks[0].Comment != nil {
		t.Error("")
	}
	if result.Bookmarks[0].Timestamp != nil {
		t.Error("")
	}
	if result.Bookmarks[0].Tags != nil {
		t.Error("")
	}
	if result.RelatedEntries[0].Eid != nil {
		t.Error("")
	}
	if result.RelatedEntries[0].Title != nil {
		t.Error("")
	}
	if result.RelatedEntries[0].Count != nil {
		t.Error("")
	}
	if result.RelatedEntries[0].Url != nil {
		t.Error("")
	}
	if result.RelatedEntries[0].EntryUrl != nil {
		t.Error("")
	}
}
<commit_msg>add test<commit_after>package hatena

import (
	"net/http"
	"os"
	"testing"
)

func TestEntryInfo(t *testing.T) {

	res, err := hatena.EntryInfo("https://github.com/")

	if result.Eid != 10975646 {
		t.Error("Eid is invalid")
	}
	if result.Title != "GitHub" {
		t.Error("Title is invalid")
	}
	if result.Count > 974 {
		t.Error("Count is invalid")
	}
	if result.Url != "https://github.com/" {
		t.Error("")
	}
	if result.EntryUrl != "http://b.hatena.ne.jp/entry/s/github.com/" {
		t.Error("")
	}
	screenshot := "http://screenshot.hatena.ne.jp/images/200x150/f/d/e/b/0/3ba121c130cd7312d649e5f4fb308a2394c.jpg"
	if result.Screenshot != screenshot {
		t.Error("")
	}
	if result.Bookmarks[0].User == nil {
		t.Error("")
	}
	if result.Bookmarks[0].Comment == nil {
		t.Error("")
	}
	if result.Bookmarks[0].Timestamp == nil {
		t.Error("")
	}
	if result.Bookmarks[0].Tags == nil {
		t.Error("")
	}
	if result.RelatedEntries[0].Eid == nil {
		t.Error("")
	}
	if result.RelatedEntries[0].Title == nil {
		t.Error("")
	}
	if result.RelatedEntries[0].Count == nil {
		t.Error("")
	}
	if result.RelatedEntries[0].Url == nil {
		t.Error("")
	}
	if result.RelatedEntries[0].EntryUrl == nil {
		t.Error("")
	}
	if err != nil {
		t.Error(err)
	}
}

var getCategoryPlaylists = `
{
	related: [
		{
			count: 123,
			url: "http://www.infoq.com/jp/articles/9_Fallacies_Java_Performance#.UnBhsgQ0rDA.facebook",
			eid: 167393322,
			title: "Javaのパフォーマンスについての９つの誤信",
			entry_url: "http://b.hatena.ne.jp/entry/www.infoq.com/jp/articles/9_Fallacies_Java_Performance%23.UnBhsgQ0rDA.facebook"
		},
		{
			count: 76,
			url: "http://www.lifehacker.jp/2015/04/150408entrepreneur_must_outsource.html",
			eid: 246661713,
			title: "時間を有効に使うためにアウトソースすべき11のこと ｜ ライフハッカー［日本版］",
			entry_url: "http://b.hatena.ne.jp/entry/www.lifehacker.jp/2015/04/150408entrepreneur_must_outsource.html"
		}
	],
	count: 974,
	bookmarks: [
		{
			comment: "",
			timestamp: "2017/02/18 22:38:32",
			user: "pg4self",
			tags: [
				"github"
			]
		},
		{
			comment: "",
			timestamp: "2008/11/06 02:02:05",
			user: "d-_-b",
			tags: [
				"versioncontrol",
				"web",
				"tool"
			]
		}
	],
	url: "https://github.com/",
	eid: 10975646,
	title: "GitHub",
	screenshot: "http://screenshot.hatena.ne.jp/images/200x150/f/d/e/b/0/3ba121c130cd7312d649e5f4fb308a2394c.jpg",
	entry_url: "http://b.hatena.ne.jp/entry/s/github.com/"
}`
<|endoftext|>


<commit_before>package enum

type OrderType int

const (
	OrdType_MARKET                          = 1
	OrdType_LIMIT                           = 2
	OrdType_STOP                            = 3
	OrdType_CANCEL	                        = 4
)

type OrderDirection string

const (
	OrderDirection_SELL    = 0
	OrderDirection_BUY     = 1
)

type TagNum int

const (
	TagNum_FIRMID		TagNum = 11
	TagNum_FUTUREID		TagNum = 12
	TagNum_QUANTITY		TagNum = 13
	TagNum_PRICE		TagNum = 14
	TagNum_DIRECTION	TagNum = 15
	TagNum_OrdType		TagNum = 16
)

const (
	ConsignationStatus_CANCELLED = 0
	ConsignationStatus_APPENDING = 1
	ConsignationStatus_PARTIAL = 2
	ConsignationStatus_FINISHED = 3
)

<commit_msg>Add max&min price<commit_after>package enum

import "github.com/shopspring/decimal"

type OrderType int

const (
	OrdType_MARKET                          = 1
	OrdType_LIMIT                           = 2
	OrdType_STOP                            = 3
	OrdType_CANCEL	                        = 4
)

type OrderDirection string

const (
	OrderDirection_SELL    = 0
	OrderDirection_BUY     = 1
)

type TagNum int

const (
	TagNum_FIRMID		TagNum = 11
	TagNum_FUTUREID		TagNum = 12
	TagNum_QUANTITY		TagNum = 13
	TagNum_PRICE		TagNum = 14
	TagNum_DIRECTION	TagNum = 15
	TagNum_OrdType		TagNum = 16
)

const (
	ConsignationStatus_CANCELLED = 0
	ConsignationStatus_APPENDING = 1
	ConsignationStatus_PARTIAL = 2
	ConsignationStatus_FINISHED = 3
)

const (
	MatchCreatOrder_RESULT_BUY_MORE = 1
	MatchCreatOrder_RESULT_EQUAL = 0
	MatchCreatOrder_RESULT_SELL_MORE = -1
)

var (
	MAX_PRICE = decimal.New(999999,-2)
	MIN_PRICE = decimal.Zero
)

<|endoftext|>