<commit_before>package toscalib

import (
	"io"
	"io/ioutil"
	"regexp"

	"github.com/gonum/matrix/mat64"
	"gopkg.in/yaml.v2"
)

// NodeGap is the gap between each node see @FillAdjacencyMatrix for explanation
const nodeGap int = 10

// GetInitialIndex return the index of the initial state of the node in the AdjacencyMatrix
func (nodeTemplate *NodeTemplate) GetInitialIndex() int { return nodeTemplate.Id }

// GetCreateIndex return the index of the Create state of the node in the AdjacencyMatrix
func (nodeTemplate *NodeTemplate) GetCreateIndex() int { return nodeTemplate.Id + 1 }

// GetPreConfigureSourceIndex return the index of the pre_configure_source state of the node in the AdjacencyMatrix
func (nodeTemplate *NodeTemplate) GetPreConfigureSourceIndex() int  { return nodeTemplate.Id + 2 }
func (nodeTemplate *NodeTemplate) GetPreConfigureTargetIndex() int  { return nodeTemplate.Id + 3 }
func (nodeTemplate *NodeTemplate) GetConfigureIndex() int           { return nodeTemplate.Id + 4 }
func (nodeTemplate *NodeTemplate) GetPostConfigureSourceIndex() int { return nodeTemplate.Id + 5 }
func (nodeTemplate *NodeTemplate) GetPostConfigureTargetIndex() int { return nodeTemplate.Id + 6 }
func (nodeTemplate *NodeTemplate) GetStartIndex() int               { return nodeTemplate.Id + 7 }
func (nodeTemplate *NodeTemplate) GetStopIndex() int                { return nodeTemplate.Id + 8 }
func (nodeTemplate *NodeTemplate) GetDeleteIndex() int              { return nodeTemplate.Id + 9 }

// GetNodeTemplate returns a pointer to a node template given its name
// its returns nil if not found
func (toscaStructure *ToscaDefinition) GetNodeTemplate(nodeName string) *NodeTemplate {
	for name, nodeTemplate := range toscaStructure.TopologyTemplate.NodeTemplates {
		if name == nodeName {
			return &nodeTemplate
		}
	}
	return nil
}

// GetNodeTemplate returns a pointer to a node template given its id
// the ID may be the initial index or whatever index of the lifecycle operation
// its returns nil if not found
func (toscaStructure *ToscaDefinition) GetNodeTemplateFromId(nodeId int) *NodeTemplate {
	modulo := 10
	for _, nodeTemplate := range toscaStructure.TopologyTemplate.NodeTemplates {
		if nodeTemplate.Id == nodeId-(nodeId%modulo) {
			return &nodeTemplate
		}
	}
	return nil
}

// FillAdjacencyMatrix fills the adjacency matrix AdjacencyMatrix in the current ToscaDefinition structure
// for more information, see doc/node_instanciation_lifecycle.md
func (toscaStructure *ToscaDefinition) FillAdjacencyMatrix() error {
	// Get the number of nodes
	numberOfNodes := len(toscaStructure.TopologyTemplate.NodeTemplates)
	// Initialize the AdjacencyMatrix
	adjacencyMatrix := mat64.NewDense(numberOfNodes*nodeGap, numberOfNodes*nodeGap, nil)
	index := 1
	for i, nodeDetail := range toscaStructure.TopologyTemplate.NodeTemplates {
		// Set the Id of the node
		nodeDetail.Id = index
		toscaStructure.TopologyTemplate.NodeTemplates[i] = nodeDetail
		index = index + nodeGap
	}
	// Then set the matrix
	for nodeAName, nodeDetail := range toscaStructure.TopologyTemplate.NodeTemplates {
		// Check if the current node has at least one requirement with an interface of type tosca.interfaces.relationship.Configure
		var res1 bool
		var res2 bool
		if nodeDetail.Requirements != nil {
			for _, requirementAssignements := range nodeDetail.Requirements {
				for _, requirementAssignement := range requirementAssignements {
					nodeBName := requirementAssignement.Node
					// Check if we have a requirement type that is .*Configure of if we have an Interface key that is .*Configure
					res1, _ = regexp.MatchString(".*Configure", requirementAssignement.Relationship.Type)
					for inter := range requirementAssignement.Relationship.Interfaces {
						res2, _ = regexp.MatchString(".*Configure", inter)
						if res2 == true {
							break
						}
					}
					// We have a Configure relationship
					if res1 == true || res2 == true {
						//log.Printf("%v Special workflow with %v", nodeAName, nodeBName)
						nodeA := toscaStructure.GetNodeTemplate(nodeAName)
						nodeB := toscaStructure.GetNodeTemplate(nodeBName)
						//nodeB:Create() -> nodeA:Create()
						adjacencyMatrix.Set(nodeB.GetCreateIndex(), nodeA.GetCreateIndex(), 1)
						//nodeA:Create() -> nodeA:PreConfigureSource()
						adjacencyMatrix.Set(nodeA.GetCreateIndex(), nodeA.GetPreConfigureSourceIndex(), 1)
						//nodeA:PreConfigureSource -> nodeB:PreConfigureTarget()
						adjacencyMatrix.Set(nodeA.GetPreConfigureSourceIndex(), nodeB.GetPreConfigureTargetIndex(), 1)
						//nodeB:PreConfigureTarget -> nodeA:Configure()
						adjacencyMatrix.Set(nodeB.GetPreConfigureTargetIndex(), nodeA.GetConfigureIndex(), 1)
						//nodeB:PreConfigureTarget -> nodeB:Configure()
						adjacencyMatrix.Set(nodeB.GetPreConfigureTargetIndex(), nodeB.GetConfigureIndex(), 1)
						//nodeA:Configure() -> nodeA:PostConfigureSource()
						adjacencyMatrix.Set(nodeA.GetConfigureIndex(), nodeA.GetPostConfigureSourceIndex(), 1)
						//nodeB:Configure() -> nodeB:PostConfigureTarget()
						adjacencyMatrix.Set(nodeB.GetConfigureIndex(), nodeB.GetPostConfigureTargetIndex(), 1)
						//nodeA:PostConfigureSource() -> nodeA:Start()
						adjacencyMatrix.Set(nodeA.GetPostConfigureSourceIndex(), nodeA.GetStartIndex(), 1)
						//nodeB:PostConfigureTarget() -> nodeB:Start()
						adjacencyMatrix.Set(nodeB.GetPostConfigureTargetIndex(), nodeB.GetStartIndex(), 1)
						//nodeB:Start() -> nodeA:Start()
						adjacencyMatrix.Set(nodeA.GetStartIndex(), nodeA.GetStartIndex(), 1)
					} else {
						//log.Printf("%v normal workflow with %v", nodeAName, nodeBName)
						nodeA := toscaStructure.GetNodeTemplate(nodeAName)
						nodeB := toscaStructure.GetNodeTemplate(nodeBName)
						// nodeB:Create() -> nodeB:Configure() -> nodeB:Start() -> nodeA:Create() -> nodeA:Configure() -> nodeA:Start()
						adjacencyMatrix.Set(nodeB.GetCreateIndex(), nodeB.GetConfigureIndex(), 1)
						adjacencyMatrix.Set(nodeB.GetConfigureIndex(), nodeB.GetStartIndex(), 1)
						adjacencyMatrix.Set(nodeB.GetStartIndex(), nodeA.GetCreateIndex(), 1)
						adjacencyMatrix.Set(nodeA.GetCreateIndex(), nodeA.GetConfigureIndex(), 1)
						adjacencyMatrix.Set(nodeA.GetConfigureIndex(), nodeA.GetStartIndex(), 1)
					}
				}

			}
		}
		index = index + nodeGap
	}
	toscaStructure.AdjacencyMatrix = *adjacencyMatrix
	return nil
}

// Parse a TOSCA document and fill in the structure
func (toscaStructure *ToscaDefinition) Parse(r io.Reader) error {
	var tempStruct ToscaDefinition
	tempStruct.NodeTypes = make(map[string]NodeType)
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}

	// Unmarshal the data in an interface
	err = yaml.Unmarshal(data, &tempStruct)
	if err != nil {
		return err
	}
	// Sets the initial state
	for _, nodeTemplate := range tempStruct.TopologyTemplate.NodeTemplates {
		nodeTemplate.RunChan = make(chan int)
		nodeTemplate.State = StateInitial
	}
	/*
		// for each node, add its corresponding notetype definition to the structure
		// if not present yet

		// index is the node name and nodeTemplate is the corresponding NodeTemplate
		for _, nodeTemplate := range tempStruct.TopologyTemplate.NodeTemplates {
			// nodeType is he node type of the current NodeTemplate
			nodeType := nodeTemplate.Type
			if _, typeIsPresent := tempStruct.NodeTypes[nodeType]; typeIsPresent == false {
				// Get the corresponding asset and add it to the global structure
				data, err := Asset(nodeType)
				if err != nil {
					//  For debuging purpode
					log.Printf("Cannot find the NodeType definition for %v", nodeType)
				}
				var nt map[string]NodeType
				// Unmarshal the data in an interface
				err = yaml.Unmarshal(data, &nt)
				if err != nil {
					return errors.New(fmt.Sprintf("cannot unmarshal %v (%v)", nodeType, err))
				}
				tempStruct.NodeTypes[nodeType] = nt[nodeType]
			}
		}
	*/
	// TODO: deal with the import files
	*toscaStructure = tempStruct
	err = toscaStructure.FillAdjacencyMatrix()
	if err != nil {
		return err
	}
	return nil

}
<commit_msg>Error in the id<commit_after>package toscalib

import (
	"io"
	"io/ioutil"
	"regexp"

	"github.com/gonum/matrix/mat64"
	"gopkg.in/yaml.v2"
)

// NodeGap is the gap between each node see @FillAdjacencyMatrix for explanation
const nodeGap int = 10

// GetInitialIndex return the index of the initial state of the node in the AdjacencyMatrix
func (nodeTemplate *NodeTemplate) GetInitialIndex() int { return nodeTemplate.Id }

// GetCreateIndex return the index of the Create state of the node in the AdjacencyMatrix
func (nodeTemplate *NodeTemplate) GetCreateIndex() int { return nodeTemplate.Id + 1 }

// GetPreConfigureSourceIndex return the index of the pre_configure_source state of the node in the AdjacencyMatrix
func (nodeTemplate *NodeTemplate) GetPreConfigureSourceIndex() int  { return nodeTemplate.Id + 2 }
func (nodeTemplate *NodeTemplate) GetPreConfigureTargetIndex() int  { return nodeTemplate.Id + 3 }
func (nodeTemplate *NodeTemplate) GetConfigureIndex() int           { return nodeTemplate.Id + 4 }
func (nodeTemplate *NodeTemplate) GetPostConfigureSourceIndex() int { return nodeTemplate.Id + 5 }
func (nodeTemplate *NodeTemplate) GetPostConfigureTargetIndex() int { return nodeTemplate.Id + 6 }
func (nodeTemplate *NodeTemplate) GetStartIndex() int               { return nodeTemplate.Id + 7 }
func (nodeTemplate *NodeTemplate) GetStopIndex() int                { return nodeTemplate.Id + 8 }
func (nodeTemplate *NodeTemplate) GetDeleteIndex() int              { return nodeTemplate.Id + 9 }

// GetNodeTemplate returns a pointer to a node template given its name
// its returns nil if not found
func (toscaStructure *ToscaDefinition) GetNodeTemplate(nodeName string) *NodeTemplate {
	for name, nodeTemplate := range toscaStructure.TopologyTemplate.NodeTemplates {
		if name == nodeName {
			return &nodeTemplate
		}
	}
	return nil
}

// GetNodeTemplate returns a pointer to a node template given its id
// the ID may be the initial index or whatever index of the lifecycle operation
// its returns nil if not found
func (toscaStructure *ToscaDefinition) GetNodeTemplateFromId(nodeId int) *NodeTemplate {
	modulo := 10
	for _, nodeTemplate := range toscaStructure.TopologyTemplate.NodeTemplates {
		if nodeTemplate.Id == nodeId-(nodeId%modulo)+1 {
			return &nodeTemplate
		}
	}
	return nil
}

// FillAdjacencyMatrix fills the adjacency matrix AdjacencyMatrix in the current ToscaDefinition structure
// for more information, see doc/node_instanciation_lifecycle.md
func (toscaStructure *ToscaDefinition) FillAdjacencyMatrix() error {
	// Get the number of nodes
	numberOfNodes := len(toscaStructure.TopologyTemplate.NodeTemplates)
	// Initialize the AdjacencyMatrix
	adjacencyMatrix := mat64.NewDense(numberOfNodes*nodeGap, numberOfNodes*nodeGap, nil)
	index := 1
	for i, nodeDetail := range toscaStructure.TopologyTemplate.NodeTemplates {
		// Set the Id of the node
		nodeDetail.Id = index
		toscaStructure.TopologyTemplate.NodeTemplates[i] = nodeDetail
		index = index + nodeGap
	}
	// Then set the matrix
	for nodeAName, nodeDetail := range toscaStructure.TopologyTemplate.NodeTemplates {
		// Check if the current node has at least one requirement with an interface of type tosca.interfaces.relationship.Configure
		var res1 bool
		var res2 bool
		if nodeDetail.Requirements != nil {
			for _, requirementAssignements := range nodeDetail.Requirements {
				for _, requirementAssignement := range requirementAssignements {
					nodeBName := requirementAssignement.Node
					// Check if we have a requirement type that is .*Configure of if we have an Interface key that is .*Configure
					res1, _ = regexp.MatchString(".*Configure", requirementAssignement.Relationship.Type)
					for inter := range requirementAssignement.Relationship.Interfaces {
						res2, _ = regexp.MatchString(".*Configure", inter)
						if res2 == true {
							break
						}
					}
					// We have a Configure relationship
					if res1 == true || res2 == true {
						//log.Printf("%v Special workflow with %v", nodeAName, nodeBName)
						nodeA := toscaStructure.GetNodeTemplate(nodeAName)
						nodeB := toscaStructure.GetNodeTemplate(nodeBName)
						//nodeB:Create() -> nodeA:Create()
						adjacencyMatrix.Set(nodeB.GetCreateIndex(), nodeA.GetCreateIndex(), 1)
						//nodeA:Create() -> nodeA:PreConfigureSource()
						adjacencyMatrix.Set(nodeA.GetCreateIndex(), nodeA.GetPreConfigureSourceIndex(), 1)
						//nodeA:PreConfigureSource -> nodeB:PreConfigureTarget()
						adjacencyMatrix.Set(nodeA.GetPreConfigureSourceIndex(), nodeB.GetPreConfigureTargetIndex(), 1)
						//nodeB:PreConfigureTarget -> nodeA:Configure()
						adjacencyMatrix.Set(nodeB.GetPreConfigureTargetIndex(), nodeA.GetConfigureIndex(), 1)
						//nodeB:PreConfigureTarget -> nodeB:Configure()
						adjacencyMatrix.Set(nodeB.GetPreConfigureTargetIndex(), nodeB.GetConfigureIndex(), 1)
						//nodeA:Configure() -> nodeA:PostConfigureSource()
						adjacencyMatrix.Set(nodeA.GetConfigureIndex(), nodeA.GetPostConfigureSourceIndex(), 1)
						//nodeB:Configure() -> nodeB:PostConfigureTarget()
						adjacencyMatrix.Set(nodeB.GetConfigureIndex(), nodeB.GetPostConfigureTargetIndex(), 1)
						//nodeA:PostConfigureSource() -> nodeA:Start()
						adjacencyMatrix.Set(nodeA.GetPostConfigureSourceIndex(), nodeA.GetStartIndex(), 1)
						//nodeB:PostConfigureTarget() -> nodeB:Start()
						adjacencyMatrix.Set(nodeB.GetPostConfigureTargetIndex(), nodeB.GetStartIndex(), 1)
						//nodeB:Start() -> nodeA:Start()
						adjacencyMatrix.Set(nodeA.GetStartIndex(), nodeA.GetStartIndex(), 1)
					} else {
						//log.Printf("%v normal workflow with %v", nodeAName, nodeBName)
						nodeA := toscaStructure.GetNodeTemplate(nodeAName)
						nodeB := toscaStructure.GetNodeTemplate(nodeBName)
						// nodeB:Create() -> nodeB:Configure() -> nodeB:Start() -> nodeA:Create() -> nodeA:Configure() -> nodeA:Start()
						adjacencyMatrix.Set(nodeB.GetCreateIndex(), nodeB.GetConfigureIndex(), 1)
						adjacencyMatrix.Set(nodeB.GetConfigureIndex(), nodeB.GetStartIndex(), 1)
						adjacencyMatrix.Set(nodeB.GetStartIndex(), nodeA.GetCreateIndex(), 1)
						adjacencyMatrix.Set(nodeA.GetCreateIndex(), nodeA.GetConfigureIndex(), 1)
						adjacencyMatrix.Set(nodeA.GetConfigureIndex(), nodeA.GetStartIndex(), 1)
					}
				}

			}
		}
		index = index + nodeGap
	}
	toscaStructure.AdjacencyMatrix = *adjacencyMatrix
	return nil
}

// Parse a TOSCA document and fill in the structure
func (toscaStructure *ToscaDefinition) Parse(r io.Reader) error {
	var tempStruct ToscaDefinition
	tempStruct.NodeTypes = make(map[string]NodeType)
	data, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}

	// Unmarshal the data in an interface
	err = yaml.Unmarshal(data, &tempStruct)
	if err != nil {
		return err
	}
	// Sets the initial state
	for _, nodeTemplate := range tempStruct.TopologyTemplate.NodeTemplates {
		nodeTemplate.RunChan = make(chan int)
		nodeTemplate.State = StateInitial
	}
	/*
		// for each node, add its corresponding notetype definition to the structure
		// if not present yet

		// index is the node name and nodeTemplate is the corresponding NodeTemplate
		for _, nodeTemplate := range tempStruct.TopologyTemplate.NodeTemplates {
			// nodeType is he node type of the current NodeTemplate
			nodeType := nodeTemplate.Type
			if _, typeIsPresent := tempStruct.NodeTypes[nodeType]; typeIsPresent == false {
				// Get the corresponding asset and add it to the global structure
				data, err := Asset(nodeType)
				if err != nil {
					//  For debuging purpode
					log.Printf("Cannot find the NodeType definition for %v", nodeType)
				}
				var nt map[string]NodeType
				// Unmarshal the data in an interface
				err = yaml.Unmarshal(data, &nt)
				if err != nil {
					return errors.New(fmt.Sprintf("cannot unmarshal %v (%v)", nodeType, err))
				}
				tempStruct.NodeTypes[nodeType] = nt[nodeType]
			}
		}
	*/
	// TODO: deal with the import files
	*toscaStructure = tempStruct
	err = toscaStructure.FillAdjacencyMatrix()
	if err != nil {
		return err
	}
	return nil

}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fi

type Lifecycle string

const (
	// LifecycleSync should do the normal synchronization
	LifecycleSync Lifecycle = "Sync"

	// LifecycleIgnore will skip the task
	LifecycleIgnore Lifecycle = "Ignore"

	// LifecycleValidateMustExist will check that the task exists and is the same
	LifecycleExistsAndValidates Lifecycle = "ExistsAndValidates"

	// LifecycleExistsAndWarnIfChanges will check that the task exists and will warn on changes, but then ignore them
	LifecycleExistsAndWarnIfChanges Lifecycle = "ExistsAndWarnIfChanges"
)

// HasLifecycle indicates that the task has a Lifecycle
type HasLifecycle interface {
	GetLifecycle() *Lifecycle
}
<commit_msg>Fix comment on Lifecycle constant<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fi

type Lifecycle string

const (
	// LifecycleSync should do the normal synchronization
	LifecycleSync Lifecycle = "Sync"

	// LifecycleIgnore will skip the task
	LifecycleIgnore Lifecycle = "Ignore"

	// LifecycleExistsAndValidates will check that the task exists and is the same
	LifecycleExistsAndValidates Lifecycle = "ExistsAndValidates"

	// LifecycleExistsAndWarnIfChanges will check that the task exists and will warn on changes, but then ignore them
	LifecycleExistsAndWarnIfChanges Lifecycle = "ExistsAndWarnIfChanges"
)

// HasLifecycle indicates that the task has a Lifecycle
type HasLifecycle interface {
	GetLifecycle() *Lifecycle
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
)

// Parser represents a parser.
type Parser struct {
	s   *Scanner
	buf struct {
		tok TokenType // last read token
		lit string    // last read literal
		n   int       // buffer size (max=1)
	}
}

// NewParser returns a new instance of Parser
func NewParser(s *Scanner) *Parser {
	return &Parser{
		s: s,
	}
}

func (p *Parser) scan() (tok TokenType, lit string) {
	// If we have a token on the buffer, then return it.
	if p.buf.n != 0 {
		p.buf.n = 0
		return p.buf.tok, p.buf.lit
	}

	// Otherwise read the next token from the scanner.
	tok, lit = p.s.Scan()

	// Save it to the buffer in case we unscan later.
	p.buf.tok, p.buf.lit = tok, lit

	return
}

func (p *Parser) unscan() {
	//this is trash
	p.buf.n = 1
}

func (p *Parser) scanIgnoreWhitespace() (tok TokenType, lit string) {
	tok, lit = p.scan()
	if tok == TWhitespace {
		tok, lit = p.scan()
	}
	return
}

func tokenArrayContains(t TokenType, types []TokenType) bool {
	for _, i := range types {
		if i == t {
			return true
		}
	}

	return false
}

// Parse parses scanner result
func (p *Parser) Parse() (*Schema, error) {
	sch := &Schema{}

TableLoop:
	for {
		var tok TokenType
		var lit string

		tok, lit = p.scanIgnoreWhitespace()
		if tok == TEof {
			break
		}

		if !tokenArrayContains(tok, HeadingTokens) {
			return nil, fmt.Errorf("found %q %s, expected %s", lit, tok, HeadingTokens)
		}

		tbl := &CreateTable{
			UniqueKeys: make(map[int]TableKeyColumns),
			Keys:       make(map[int]TableKeyColumns),
		}
		if tok == TAtSignHeadingLine {
			tbl.IsPsuedo = true
		}

		if tok, lit = p.scanIgnoreWhitespace(); tok != TString {
			return nil, fmt.Errorf("found %q %s, expected TableName", lit, tok)
		}

		tbl.TableName = lit
		sch.Tables = append(sch.Tables, tbl)

	TblCommentLoop:
		for {
			comtok, comlit := p.scanIgnoreWhitespace()
			if comtok == TColonLine {
				tbl.TableComment = append(tbl.TableComment, strings.TrimSpace(comlit))
			} else {
				// I don't believe this is nessessary anymore
				// tbl.TableComment = strings.TrimSpace(tbl.TableComment)
				p.unscan()
				break TblCommentLoop
			}
		}

		// ColumnLoop:
		for {
			var coltok TokenType
			var collit string

			if coltok, collit = p.scanIgnoreWhitespace(); !tokenArrayContains(coltok, ColumnTokens) {
				if tokenArrayContains(coltok, HeadingTokens) {
					p.unscan()
					continue TableLoop
				}

				if coltok == TEof {
					p.unscan()
					continue TableLoop
				}

				return nil, fmt.Errorf("found %q %s, expected %s", collit, coltok, ColumnTokens)
			}

			col := &TableColumn{}

			switch coltok {
			case TExclaimLine:
				col.ColumnReferenceType = ColumnForeignKeyRegister
			case TQuestionLine:
				col.ColumnReferenceType = ColumnForeignKeyReference
			case TDashLine:
				col.ColumnReferenceType = ColumnRegular
			default:
				return nil, fmt.Errorf("unexpected token: %s", coltok)
			}

			var colntok TokenType
			var colnlit string
			if colntok, colnlit = p.scanIgnoreWhitespace(); colntok != TString {
				return nil, fmt.Errorf("found %q %s, expected Column Name", colnlit, colntok)
			}

			col.ColumnName = colnlit

		ModLoop:
			for {
				modtok, _ := p.scanIgnoreWhitespace()
				switch modtok {
				case TAstrisk:
					col.Nullable = true
				case TSigned:
					col.Signed = true
				case TString:
					p.unscan()
					break ModLoop
				default:
					return nil, fmt.Errorf("unexpected token: %s", modtok)
				}
			} //todo limit to one of each

			_, ctype := p.scan()
			ctype, csize, err := strIntSuffixSplit(ctype)
			if err != nil {
				return nil, err
			}

			cctype, err := ColumnTypes.getColumnType(ctype)
			if err != nil {
				return nil, err
			}

			col.ColumnType = cctype
			col.ColumnSize = csize
			tbl.TableColumns = append(tbl.TableColumns, col)

			for {
				tok, lit := p.scanIgnoreWhitespace()

				fmt.Println(tok, lit)

				if tok != TString && tok != TAstrisk {
					fmt.Println("continuing", tok, lit)
					p.unscan()
					break
				}

				// autoIncr := false
				if tok == TAstrisk {
					// autoIncr = true
					if tbl.AutoIncrColumn == nil {
						tbl.AutoIncrColumn = col
					} else {
						return nil, fmt.Errorf("auto increment column already declared")
					}

					tok, lit := p.scanIgnoreWhitespace()
					if tok != TString {
						return nil, fmt.Errorf("found %q %s, expected %s", lit, tok, TString)
					}
				}

				if lit == "pk" {
				} else if lit[0:1] == "k" {
				} else if lit[0:1] == "u" {
				}
			}

		ColCommentLoop:
			for {
				comtok, comlit := p.scanIgnoreWhitespace()
				if comtok == TColonLine {
					col.ColumnComment = append(col.ColumnComment, strings.TrimSpace(comlit))
				} else {
					// This shouldn't be nessessary
					// col.ColumnComment = strings.TrimSpace(col.ColumnComment)
					p.unscan()
					break ColCommentLoop
				}
			}
			// break
		}

		// break
	}

	return sch, nil
}
<commit_msg>Implements key parsing<commit_after>package main

import (
	"fmt"
	"log"
	"strings"
)

// Parser represents a parser.
type Parser struct {
	s   *Scanner
	buf struct {
		tok TokenType // last read token
		lit string    // last read literal
		n   int       // buffer size (max=1)
	}
}

// NewParser returns a new instance of Parser
func NewParser(s *Scanner) *Parser {
	return &Parser{
		s: s,
	}
}

func (p *Parser) scan() (tok TokenType, lit string) {
	// If we have a token on the buffer, then return it.
	if p.buf.n != 0 {
		p.buf.n = 0
		return p.buf.tok, p.buf.lit
	}

	// Otherwise read the next token from the scanner.
	tok, lit = p.s.Scan()

	// Save it to the buffer in case we unscan later.
	p.buf.tok, p.buf.lit = tok, lit

	return
}

func (p *Parser) unscan() {
	//this is trash
	p.buf.n = 1
}

func (p *Parser) scanIgnoreWhitespace() (tok TokenType, lit string) {
	tok, lit = p.scan()
	if tok == TWhitespace {
		tok, lit = p.scan()
	}
	return
}

func tokenArrayContains(t TokenType, types []TokenType) bool {
	for _, i := range types {
		if i == t {
			return true
		}
	}

	return false
}

// Parse parses scanner result
func (p *Parser) Parse() (*Schema, error) {
	sch := &Schema{}

TableLoop:
	for {
		var tok TokenType
		var lit string

		tok, lit = p.scanIgnoreWhitespace()
		if tok == TEof {
			break
		}

		if !tokenArrayContains(tok, HeadingTokens) {
			return nil, fmt.Errorf("found %q %s, expected %s", lit, tok, HeadingTokens)
		}

		tbl := &CreateTable{
			UniqueKeys: make(map[int]TableKeyColumns),
			Keys:       make(map[int]TableKeyColumns),
		}
		if tok == TAtSignHeadingLine {
			tbl.IsPsuedo = true
		}

		if tok, lit = p.scanIgnoreWhitespace(); tok != TString {
			return nil, fmt.Errorf("found %q %s, expected TableName", lit, tok)
		}

		tbl.TableName = lit
		sch.Tables = append(sch.Tables, tbl)

	TblCommentLoop:
		for {
			comtok, comlit := p.scanIgnoreWhitespace()
			if comtok == TColonLine {
				tbl.TableComment = append(tbl.TableComment, strings.TrimSpace(comlit))
			} else {
				// I don't believe this is nessessary anymore
				// tbl.TableComment = strings.TrimSpace(tbl.TableComment)
				p.unscan()
				break TblCommentLoop
			}
		}

		// ColumnLoop:
		for {
			var coltok TokenType
			var collit string

			if coltok, collit = p.scanIgnoreWhitespace(); !tokenArrayContains(coltok, ColumnTokens) {
				if tokenArrayContains(coltok, HeadingTokens) {
					p.unscan()
					continue TableLoop
				}

				if coltok == TEof {
					p.unscan()
					continue TableLoop
				}

				return nil, fmt.Errorf("found %q %s, expected %s", collit, coltok, ColumnTokens)
			}

			col := &TableColumn{}

			switch coltok {
			case TExclaimLine:
				col.ColumnReferenceType = ColumnForeignKeyRegister
			case TQuestionLine:
				col.ColumnReferenceType = ColumnForeignKeyReference
			case TDashLine:
				col.ColumnReferenceType = ColumnRegular
			default:
				return nil, fmt.Errorf("unexpected token: %s", coltok)
			}

			var colntok TokenType
			var colnlit string
			if colntok, colnlit = p.scanIgnoreWhitespace(); colntok != TString {
				return nil, fmt.Errorf("found %q %s, expected Column Name", colnlit, colntok)
			}

			col.ColumnName = colnlit

		ModLoop:
			for {
				modtok, _ := p.scanIgnoreWhitespace()
				switch modtok {
				case TAstrisk:
					col.Nullable = true
				case TSigned:
					col.Signed = true
				case TString:
					p.unscan()
					break ModLoop
				default:
					return nil, fmt.Errorf("unexpected token: %s", modtok)
				}
			} //todo limit to one of each

			_, ctype := p.scan()
			ctype, csize, err := strIntSuffixSplit(ctype)
			if err != nil {
				return nil, err
			}

			cctype, err := ColumnTypes.getColumnType(ctype)
			if err != nil {
				return nil, err
			}

			col.ColumnType = cctype
			col.ColumnSize = csize
			tbl.TableColumns = append(tbl.TableColumns, col)

			// KEYLOOP:
			for {
				tok, lit := p.scanIgnoreWhitespace()
				// log.Println(lit)
				log.Println("xxx", tok, lit)

				if tok != TString && tok != TAstrisk {
					log.Println("keyloop continuing", tok, lit)
					p.unscan()
					break
				}

				// autoIncr := false
				if tok == TAstrisk {
					// autoIncr = true
					if tbl.AutoIncrColumn == nil {
						tbl.AutoIncrColumn = col
					} else {
						return nil, fmt.Errorf("auto increment column already declared")
					}

					tok, lit = p.scanIgnoreWhitespace()
					if tok != TString {
						return nil, fmt.Errorf("found %q %s, expected %s", lit, tok, TString)
					}

					// continue KEYLOOP
				}

				log.Println("AAAAAAAAAAA", lit, tok)
				if lit == "pk" {
					log.Println("PRIMARY KEY", lit, tok)
					tbl.PrimaryKey = append(tbl.PrimaryKey, col)
				} else if lit[0:1] == "k" || lit[0:1] == "u" {
					log.Println("KEY", lit, tok)
					sPart, kIndex, err := strIntSuffixSplit(lit)
					if err != nil || kIndex <= 0 {
						return nil, fmt.Errorf("found '%s'; expected key name - %s", lit, err)
					}

					switch sPart {
					case "k":
						tbl.Keys[kIndex] = append(tbl.Keys[kIndex], col)
					case "u":
						tbl.UniqueKeys[kIndex] = append(tbl.UniqueKeys[kIndex], col)
					default:
						return nil, fmt.Errorf("found '%s'; expected key name - %s", lit, err)
					}

					// strIntSuffixSplit
				} else {
					return nil, fmt.Errorf("found '%s'; expected key name - %s", lit, err)
				}
			}

		ColCommentLoop:
			for {
				comtok, comlit := p.scanIgnoreWhitespace()
				if comtok == TColonLine {
					col.ColumnComment = append(col.ColumnComment, strings.TrimSpace(comlit))
				} else {
					// This shouldn't be nessessary
					// col.ColumnComment = strings.TrimSpace(col.ColumnComment)
					p.unscan()
					break ColCommentLoop
				}
			}
			// break
		}

		// break
	}

	return sch, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Steven Oud. All rights reserved.
// Use of this source code is governed by a MIT-style license that can be found
// in the LICENSE file.

package mathcat

import (
	"errors"
	"fmt"
	"math"
	"strconv"
	"strings"
)

// Parser holds the lexed tokens, token position and declared variables. By
// default, variables always contains the constants defined below. These can
// however be overwritten.
type Parser struct {
	tokens    []*Token
	pos       int
	Variables map[string]float64
	tok       *Token
}

var (
	errDivionByZero         = errors.New("Divison by zero")
	errUnmatchedParentheses = errors.New("Unmatched parentheses")
	errInvalidSyntax        = errors.New("Invalid syntax")
)

// Some useful predefined variables that can be used in expressions. These
// can be overwritten.
var constants = map[string]float64{
	"pi":  math.Pi,
	"tau": math.Pi / 2,
	"phi": math.Phi,
	"e":   math.E,
}

// New initializes a new Parser instance, useful when you want to run multiple
// expression and/or use variables.
func New() *Parser {
	return &Parser{
		pos:       0,
		Variables: constants,
	}
}

// Eval evaluates an expression and returns its result and any errors found.
//
// Example:
//     res, err := mathcat.Eval("2 * 2 * 2") // 8
func Eval(expr string) (float64, error) {
	tokens, err := Lex(expr)

	// If a lexer error occured don't parse
	if err != nil {
		return -1, err
	}

	p := &Parser{
		tokens:    tokens,
		pos:       0,
		Variables: constants,
	}

	return p.parse()
}

// Run executes an expression on an existing parser instance. Useful for
// variable assignment.
//
// Example:
//     p.Run("a = 555")
//     p.Run("a += 45")
//     res, err := p.Run("a + a") // 1200
func (p *Parser) Run(expr string) (float64, error) {
	tokens, err := Lex(expr)

	if err != nil {
		return -1, err
	}

	p.reset()
	p.tokens = tokens

	return p.parse()
}

// Exec executes an expression with a given map of variables.
//
// Example:
//     res, err := mathcat.Exec("a + b * b", map[string]float64{
//         "a": 1,
//         "b": 3,
//     }) // 10
func Exec(expr string, vars map[string]float64) (float64, error) {
	tokens, err := Lex(expr)

	if err != nil {
		return -1, err
	}

	p := &Parser{
		tokens:    tokens,
		pos:       0,
		Variables: constants,
	}

	isValidIdent := func(c rune) bool { return isIdent(c) || isNumber(c) }

	for k, v := range vars {
		if !isIdent(rune(k[0])) || strings.IndexFunc(k, isValidIdent) == -1 {
			return -1, fmt.Errorf("Invalid variable name: '%s'", k)
		}
		p.Variables[k] = v
	}

	p.tokens = tokens

	return p.parse()
}

// GetVar gets an existing variable.
//
// Example:
//     p.Run("酷 = -33")
//     if val, err := p.GetVar("酷"); !err {
//         fmt.Printf("%f\n", val) // -33
//     }
func (p *Parser) GetVar(index string) (float64, error) {
	if val, ok := p.Variables[index]; ok {
		return val, nil
	}

	return -1, fmt.Errorf("Undefined variable '%s'", index)
}

func (p *Parser) parse() (float64, error) {
	var (
		operands, operators stack
		o1, o2              *operator
	)

	p.tok = p.tokens[0]

	for p.eat().Type != EOL {
		switch {
		case p.tok.IsLiteral():
			if p.peek().Type == LPAREN {
				// It's a function call, push to operators stack instead
				operators.Push(p.tok)
				break
			}
			operands.Push(p.tok)
		case p.tok.Type == LPAREN:
			operators.Push(p.tok)
		case p.tok.Type == COMMA:
			for {
				if operators.Empty() {
					return -1, errors.New("Misplaced ','")
				}

				if operators.Top().(*Token).Type == LPAREN {
					break
				}

				val, err := p.evaluate(operators.Pop().(*Token), &operands)
				if err != nil {
					return -1, err
				}

				operands.Push(val)
			}
		case p.tok.IsOperator():
			o1 = ops[p.tok.Type]

			if !operators.Empty() {
				var ok bool
				if o2, ok = ops[operators.Top().(*Token).Type]; !ok {
					operators.Push(p.tok)
					break
				}

				if o2.hasHigherPrecThan(o1) {
					operator := operators.Pop().(*Token)
					val, err := p.evaluateOp(operator, &operands)
					if err != nil {
						return -1, err
					}
					operands.Push(val)
				}
			}
			operators.Push(p.tok)
		case p.tok.Type == RPAREN:
			for {
				if operators.Empty() {
					return -1, errUnmatchedParentheses
				}

				top := operators.Pop().(*Token)
				if top.Type == LPAREN {
					break
				}

				val, err := p.evaluate(top, &operands)
				if err != nil {
					return -1, err
				}

				operands.Push(val)
			}
		}
	}

	// Evaluate remaining operators
	for !operators.Empty() {
		top := operators.Pop().(*Token)

		if top.Type == LPAREN {
			return -1, errUnmatchedParentheses
		}

		val, err := p.evaluate(top, &operands)
		if err != nil {
			return -1, err
		}

		operands.Push(val)
	}

	// If there are no operands, the expression is useless and doesn't do
	// anything, for example `()` or an empty string
	if operands.Empty() {
		return 0, nil
	}

	// Single operand left means the expression was evaluated successful
	if len(operands) == 1 {
		return p.lookup(operands[0])
	}

	// Leftover token on operand stack indicates invalid syntax
	return -1, errInvalidSyntax
}

// Evaluate gets called when an operator or function call has to be evaluated
// for a result. In case of a function, evaluateFunc is called and in case of
// an operator evaluateOp is called.
func (p *Parser) evaluate(tok *Token, operands *stack) (float64, error) {
	var err error
	var val float64

	if tok.Type == IDENT {
		// Function call
		val, err = p.evaluateFunc(tok, operands)
	} else {
		// Operator
		val, err = p.evaluateOp(tok, operands)
	}

	if err != nil {
		return -1, err
	}

	return val, nil
}

func (p *Parser) evaluateFunc(tok *Token, operands *stack) (float64, error) {
	var (
		function *function
		ok       bool
		i        int
	)

	// Arguments received counter
	count := 0

	if function, ok = functions[tok.Value]; !ok {
		return -1, fmt.Errorf("Undefined function '%s'", tok.Value)
	}

	// Start popping off arguments for the function call
	args := make([]float64, function.nargs)
	for i = function.nargs - 1; i >= 0; i, count = i-1, count+1 {
		if operands.Empty() {
			return -1, fmt.Errorf("Invalid argument count for '%s' (expected %d, got %d)", tok.Value, function.nargs, count)
		}

		arg, err := p.lookup(operands.Pop())
		if err != nil {
			return -1, err
		}

		args[i] = arg
	}

	return function.operation(args), nil
}

func (p *Parser) evaluateOp(operator *Token, operands *stack) (float64, error) {
	var (
		result      float64
		left, right float64
		err         error
		lhsToken    interface{}
	)

	if operands.Empty() {
		return -1, fmt.Errorf("Unexpected '%s'", operator.Type)
	}

	if right, err = p.lookup(operands.Pop()); err != nil {
		return -1, err
	}

	// Unary operators have no left hand side
	if op := ops[operator.Type]; !op.unary {
		if operands.Empty() {
			return -1, errInvalidSyntax
		}
		// Save the token in case of a assignment variable is used and we need to
		// save the result in a variable
		lhsToken = operands.Pop()

		// Don't lookup the left hand side if = is used so we can do initial
		// assignment
		if operator.Type != EQ {
			left, err = p.lookup(lhsToken)
			if err != nil {
				return -1, err
			}
		}
	}

	result, err = execute(operator, left, right)
	if err != nil {
		return -1, err
	}

	if operator.IsAssignment() {
		// Save result in variable
		if lhsToken.(*Token).Type != IDENT {
			return -1, errors.New("Can't assign to literal")
		}
		p.Variables[lhsToken.(*Token).Value] = result
	}

	return result, nil
}

func execute(operator *Token, lhs, rhs float64) (float64, error) {
	var result float64

	// Both lhs and rhs have to be whole numbers for bitwise operations
	if operator.IsBitwise() && (!IsWholeNumber(lhs) || !IsWholeNumber(rhs)) {
		return -1, fmt.Errorf("Unsupported type (float) for '%s'", operator.Type)
	}

	switch operator.Type {
	case ADD, ADD_EQ:
		result = lhs + rhs
	case SUB, SUB_EQ:
		result = lhs - rhs
	case UNARY_MIN:
		result = -rhs
	case DIV, DIV_EQ:
		if rhs == 0 {
			return -1, errDivionByZero
		}
		result = lhs / rhs
	case MUL, MUL_EQ:
		result = lhs * rhs
	case POW, POW_EQ:
		result = math.Pow(lhs, rhs)
	case REM, REM_EQ:
		if rhs == 0 {
			return -1, errDivionByZero
		}
		result = math.Mod(lhs, rhs)
	case AND, AND_EQ:
		result = float64(int64(lhs) & int64(rhs))
	case OR, OR_EQ:
		result = float64(int64(lhs) | int64(rhs))
	case XOR, XOR_EQ:
		result = float64(int64(lhs) ^ int64(rhs))
	case LSH, LSH_EQ:
		result = float64(uint64(lhs) << uint64(rhs))
	case RSH, RSH_EQ:
		result = float64(uint64(lhs) >> uint64(rhs))
	case NOT:
		result = float64(^int64(rhs))
	case EQ:
		result = rhs
	case EQ_EQ:
		result = bool2float(lhs == rhs)
	case BANG_EQ:
		result = bool2float(lhs != rhs)
	case GT:
		result = bool2float(lhs > rhs)
	case GT_EQ:
		result = bool2float(lhs >= rhs)
	case LT:
		result = bool2float(lhs < rhs)
	case LT_EQ:
		result = bool2float(lhs <= rhs)
	default:
		return -1, fmt.Errorf("Invalid operator '%s'", operator.Type)
	}

	return result, nil
}

// Look up a literal. If it's an identifier, check the parser's variables map,
// otherwise convert the tokenized string to a float64.
func (p *Parser) lookup(val interface{}) (float64, error) {
	// val can be a token or a float64, if it's a float64 it has been already
	// evaluated and we don't need to do anything
	if v, ok := val.(float64); ok {
		return v, nil
	}

	tok := val.(*Token)
	switch tok.Type {
	case NUMBER:
		res, err := strconv.ParseFloat(tok.Value, 64)
		if err != nil {
			return -1, fmt.Errorf("Error parsing '%s': invalid syntax", tok.Value)
		}

		return res, nil
	case HEX:
		// Remove 0x part of hex literal and convert to uint first
		res, err := strconv.ParseUint(tok.Value[2:], 16, 64)
		if err != nil {
			return -1, fmt.Errorf("Error parsing '%s': invalid syntax", tok.Value)
		}

		// Then convert to float
		return float64(res), nil
	case BINARY:
		// Remove 0b part of binary literal and convert to uint first
		res, err := strconv.ParseUint(tok.Value[2:], 2, 64)
		if err != nil {
			return -1, fmt.Errorf("Error parsing '%s': invalid syntax", tok.Value)
		}

		// Then convert to float
		return float64(res), nil
	case IDENT:
		res, err := p.GetVar(tok.Value)
		if err != nil {
			return -1, err
		}

		return res, nil
	}

	return -1, fmt.Errorf("Invalid lookup type: %s", tok.Type)
}

func (p *Parser) reset() {
	p.tokens = nil
	p.pos = 0
}

func (p *Parser) peek() *Token {
	return p.tokens[p.pos]
}

func (p *Parser) eat() *Token {
	p.tok = p.peek()
	p.pos++
	return p.tok
}

// IsWholeNumber checks if a float is a whole number
func IsWholeNumber(n float64) bool {
	epsilon := 1e-9
	_, frac := math.Modf(math.Abs(n))

	return frac < epsilon || frac > 1.0-epsilon
}

func bool2float(b bool) float64 {
	if b {
		return 1
	}
	return 0
}
<commit_msg>Increase accuracy of IsWholeNumber<commit_after>// Copyright 2016 Steven Oud. All rights reserved.
// Use of this source code is governed by a MIT-style license that can be found
// in the LICENSE file.

package mathcat

import (
	"errors"
	"fmt"
	"math"
	"strconv"
	"strings"
)

// Parser holds the lexed tokens, token position and declared variables. By
// default, variables always contains the constants defined below. These can
// however be overwritten.
type Parser struct {
	tokens    []*Token
	pos       int
	Variables map[string]float64
	tok       *Token
}

var (
	errDivionByZero         = errors.New("Divison by zero")
	errUnmatchedParentheses = errors.New("Unmatched parentheses")
	errInvalidSyntax        = errors.New("Invalid syntax")
)

// Some useful predefined variables that can be used in expressions. These
// can be overwritten.
var constants = map[string]float64{
	"pi":  math.Pi,
	"tau": math.Pi / 2,
	"phi": math.Phi,
	"e":   math.E,
}

// New initializes a new Parser instance, useful when you want to run multiple
// expression and/or use variables.
func New() *Parser {
	return &Parser{
		pos:       0,
		Variables: constants,
	}
}

// Eval evaluates an expression and returns its result and any errors found.
//
// Example:
//     res, err := mathcat.Eval("2 * 2 * 2") // 8
func Eval(expr string) (float64, error) {
	tokens, err := Lex(expr)

	// If a lexer error occured don't parse
	if err != nil {
		return -1, err
	}

	p := &Parser{
		tokens:    tokens,
		pos:       0,
		Variables: constants,
	}

	return p.parse()
}

// Run executes an expression on an existing parser instance. Useful for
// variable assignment.
//
// Example:
//     p.Run("a = 555")
//     p.Run("a += 45")
//     res, err := p.Run("a + a") // 1200
func (p *Parser) Run(expr string) (float64, error) {
	tokens, err := Lex(expr)

	if err != nil {
		return -1, err
	}

	p.reset()
	p.tokens = tokens

	return p.parse()
}

// Exec executes an expression with a given map of variables.
//
// Example:
//     res, err := mathcat.Exec("a + b * b", map[string]float64{
//         "a": 1,
//         "b": 3,
//     }) // 10
func Exec(expr string, vars map[string]float64) (float64, error) {
	tokens, err := Lex(expr)

	if err != nil {
		return -1, err
	}

	p := &Parser{
		tokens:    tokens,
		pos:       0,
		Variables: constants,
	}

	isValidIdent := func(c rune) bool { return isIdent(c) || isNumber(c) }

	for k, v := range vars {
		if !isIdent(rune(k[0])) || strings.IndexFunc(k, isValidIdent) == -1 {
			return -1, fmt.Errorf("Invalid variable name: '%s'", k)
		}
		p.Variables[k] = v
	}

	p.tokens = tokens

	return p.parse()
}

// GetVar gets an existing variable.
//
// Example:
//     p.Run("酷 = -33")
//     if val, err := p.GetVar("酷"); !err {
//         fmt.Printf("%f\n", val) // -33
//     }
func (p *Parser) GetVar(index string) (float64, error) {
	if val, ok := p.Variables[index]; ok {
		return val, nil
	}

	return -1, fmt.Errorf("Undefined variable '%s'", index)
}

func (p *Parser) parse() (float64, error) {
	var (
		operands, operators stack
		o1, o2              *operator
	)

	p.tok = p.tokens[0]

	for p.eat().Type != EOL {
		switch {
		case p.tok.IsLiteral():
			if p.peek().Type == LPAREN {
				// It's a function call, push to operators stack instead
				operators.Push(p.tok)
				break
			}
			operands.Push(p.tok)
		case p.tok.Type == LPAREN:
			operators.Push(p.tok)
		case p.tok.Type == COMMA:
			for {
				if operators.Empty() {
					return -1, errors.New("Misplaced ','")
				}

				if operators.Top().(*Token).Type == LPAREN {
					break
				}

				val, err := p.evaluate(operators.Pop().(*Token), &operands)
				if err != nil {
					return -1, err
				}

				operands.Push(val)
			}
		case p.tok.IsOperator():
			o1 = ops[p.tok.Type]

			if !operators.Empty() {
				var ok bool
				if o2, ok = ops[operators.Top().(*Token).Type]; !ok {
					operators.Push(p.tok)
					break
				}

				if o2.hasHigherPrecThan(o1) {
					operator := operators.Pop().(*Token)
					val, err := p.evaluateOp(operator, &operands)
					if err != nil {
						return -1, err
					}
					operands.Push(val)
				}
			}
			operators.Push(p.tok)
		case p.tok.Type == RPAREN:
			for {
				if operators.Empty() {
					return -1, errUnmatchedParentheses
				}

				top := operators.Pop().(*Token)
				if top.Type == LPAREN {
					break
				}

				val, err := p.evaluate(top, &operands)
				if err != nil {
					return -1, err
				}

				operands.Push(val)
			}
		}
	}

	// Evaluate remaining operators
	for !operators.Empty() {
		top := operators.Pop().(*Token)

		if top.Type == LPAREN {
			return -1, errUnmatchedParentheses
		}

		val, err := p.evaluate(top, &operands)
		if err != nil {
			return -1, err
		}

		operands.Push(val)
	}

	// If there are no operands, the expression is useless and doesn't do
	// anything, for example `()` or an empty string
	if operands.Empty() {
		return 0, nil
	}

	// Single operand left means the expression was evaluated successful
	if len(operands) == 1 {
		return p.lookup(operands[0])
	}

	// Leftover token on operand stack indicates invalid syntax
	return -1, errInvalidSyntax
}

// Evaluate gets called when an operator or function call has to be evaluated
// for a result. In case of a function, evaluateFunc is called and in case of
// an operator evaluateOp is called.
func (p *Parser) evaluate(tok *Token, operands *stack) (float64, error) {
	var err error
	var val float64

	if tok.Type == IDENT {
		// Function call
		val, err = p.evaluateFunc(tok, operands)
	} else {
		// Operator
		val, err = p.evaluateOp(tok, operands)
	}

	if err != nil {
		return -1, err
	}

	return val, nil
}

func (p *Parser) evaluateFunc(tok *Token, operands *stack) (float64, error) {
	var (
		function *function
		ok       bool
		i        int
	)

	// Arguments received counter
	count := 0

	if function, ok = functions[tok.Value]; !ok {
		return -1, fmt.Errorf("Undefined function '%s'", tok.Value)
	}

	// Start popping off arguments for the function call
	args := make([]float64, function.nargs)
	for i = function.nargs - 1; i >= 0; i, count = i-1, count+1 {
		if operands.Empty() {
			return -1, fmt.Errorf("Invalid argument count for '%s' (expected %d, got %d)", tok.Value, function.nargs, count)
		}

		arg, err := p.lookup(operands.Pop())
		if err != nil {
			return -1, err
		}

		args[i] = arg
	}

	return function.operation(args), nil
}

func (p *Parser) evaluateOp(operator *Token, operands *stack) (float64, error) {
	var (
		result      float64
		left, right float64
		err         error
		lhsToken    interface{}
	)

	if operands.Empty() {
		return -1, fmt.Errorf("Unexpected '%s'", operator.Type)
	}

	if right, err = p.lookup(operands.Pop()); err != nil {
		return -1, err
	}

	// Unary operators have no left hand side
	if op := ops[operator.Type]; !op.unary {
		if operands.Empty() {
			return -1, errInvalidSyntax
		}
		// Save the token in case of a assignment variable is used and we need to
		// save the result in a variable
		lhsToken = operands.Pop()

		// Don't lookup the left hand side if = is used so we can do initial
		// assignment
		if operator.Type != EQ {
			left, err = p.lookup(lhsToken)
			if err != nil {
				return -1, err
			}
		}
	}

	result, err = execute(operator, left, right)
	if err != nil {
		return -1, err
	}

	if operator.IsAssignment() {
		// Save result in variable
		if lhsToken.(*Token).Type != IDENT {
			return -1, errors.New("Can't assign to literal")
		}
		p.Variables[lhsToken.(*Token).Value] = result
	}

	return result, nil
}

func execute(operator *Token, lhs, rhs float64) (float64, error) {
	var result float64

	// Both lhs and rhs have to be whole numbers for bitwise operations
	if operator.IsBitwise() && (!IsWholeNumber(lhs) || !IsWholeNumber(rhs)) {
		return -1, fmt.Errorf("Unsupported type (float) for '%s'", operator.Type)
	}

	switch operator.Type {
	case ADD, ADD_EQ:
		result = lhs + rhs
	case SUB, SUB_EQ:
		result = lhs - rhs
	case UNARY_MIN:
		result = -rhs
	case DIV, DIV_EQ:
		if rhs == 0 {
			return -1, errDivionByZero
		}
		result = lhs / rhs
	case MUL, MUL_EQ:
		result = lhs * rhs
	case POW, POW_EQ:
		result = math.Pow(lhs, rhs)
	case REM, REM_EQ:
		if rhs == 0 {
			return -1, errDivionByZero
		}
		result = math.Mod(lhs, rhs)
	case AND, AND_EQ:
		result = float64(int64(lhs) & int64(rhs))
	case OR, OR_EQ:
		result = float64(int64(lhs) | int64(rhs))
	case XOR, XOR_EQ:
		result = float64(int64(lhs) ^ int64(rhs))
	case LSH, LSH_EQ:
		result = float64(uint64(lhs) << uint64(rhs))
	case RSH, RSH_EQ:
		result = float64(uint64(lhs) >> uint64(rhs))
	case NOT:
		result = float64(^int64(rhs))
	case EQ:
		result = rhs
	case EQ_EQ:
		result = bool2float(lhs == rhs)
	case BANG_EQ:
		result = bool2float(lhs != rhs)
	case GT:
		result = bool2float(lhs > rhs)
	case GT_EQ:
		result = bool2float(lhs >= rhs)
	case LT:
		result = bool2float(lhs < rhs)
	case LT_EQ:
		result = bool2float(lhs <= rhs)
	default:
		return -1, fmt.Errorf("Invalid operator '%s'", operator.Type)
	}

	return result, nil
}

// Look up a literal. If it's an identifier, check the parser's variables map,
// otherwise convert the tokenized string to a float64.
func (p *Parser) lookup(val interface{}) (float64, error) {
	// val can be a token or a float64, if it's a float64 it has been already
	// evaluated and we don't need to do anything
	if v, ok := val.(float64); ok {
		return v, nil
	}

	tok := val.(*Token)
	switch tok.Type {
	case NUMBER:
		res, err := strconv.ParseFloat(tok.Value, 64)
		if err != nil {
			return -1, fmt.Errorf("Error parsing '%s': invalid syntax", tok.Value)
		}

		return res, nil
	case HEX:
		// Remove 0x part of hex literal and convert to uint first
		res, err := strconv.ParseUint(tok.Value[2:], 16, 64)
		if err != nil {
			return -1, fmt.Errorf("Error parsing '%s': invalid syntax", tok.Value)
		}

		// Then convert to float
		return float64(res), nil
	case BINARY:
		// Remove 0b part of binary literal and convert to uint first
		res, err := strconv.ParseUint(tok.Value[2:], 2, 64)
		if err != nil {
			return -1, fmt.Errorf("Error parsing '%s': invalid syntax", tok.Value)
		}

		// Then convert to float
		return float64(res), nil
	case IDENT:
		res, err := p.GetVar(tok.Value)
		if err != nil {
			return -1, err
		}

		return res, nil
	}

	return -1, fmt.Errorf("Invalid lookup type: %s", tok.Type)
}

func (p *Parser) reset() {
	p.tokens = nil
	p.pos = 0
}

func (p *Parser) peek() *Token {
	return p.tokens[p.pos]
}

func (p *Parser) eat() *Token {
	p.tok = p.peek()
	p.pos++
	return p.tok
}

// IsWholeNumber checks if a float is a whole number
func IsWholeNumber(n float64) bool {
	epsilon := 1e-15
	_, frac := math.Modf(math.Abs(n))

	return frac < epsilon || frac > 1.0-epsilon
}

func bool2float(b bool) float64 {
	if b {
		return 1
	}
	return 0
}
<|endoftext|>


<commit_before>package mark

import (
	fmt "github.com/k0kubun/pp"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

type Tree struct {
	text  string
	lex   *lexer
	Nodes []Node
	// Parsing only
	token     [3]item // three-token lookahead for parser
	peekCount int
	output    string
	links     map[string]*DefLinkNode
}

// Parse convert the raw text to NodeTree.
func (t *Tree) parse() {
Loop:
	for {
		var n Node
		switch p := t.peek(); p.typ {
		case eof, itemError:
			break Loop
		case itemNewLine:
			n = t.newLine(t.next().pos)
		case itemBr:
			n = t.newBr(t.next().pos)
		case itemHr:
			n = t.newHr(t.next().pos)
		case itemText:
			tmp := t.newParagraph(p.pos)
			tmp.Nodes = t.parseText(t.next().val)
			n = tmp
		case itemDefLink:
			n = t.parseDefLink()
		case itemHeading, itemLHeading:
			n = t.parseHeading()
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		case itemList:
			// 0 for the depth
			n = t.parseList(0)
		case itemTable, itemLpTable:
			n = t.parseTable()
		case itemBlockQuote:
			n = t.parseBlockQuote()
		default:
			t.next()
		}
		if n != nil {
			t.append(n)
		}
	}
}

// Render parse nodes to the wanted output
func (t *Tree) render() {
	var last Node
	last = t.newLine(0)
	for _, node := range t.Nodes {
		if last.Type() != NodeNewLine || node.Type() != last.Type() {
			t.output += node.Render()
		}
		last = node
	}
}

// append new node to nodes-list
func (t *Tree) append(n Node) {
	t.Nodes = append(t.Nodes, n)
}

// next returns the next token
func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]
	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

// backup backs the input stream tp one token
func (t *Tree) backup() {
	t.peekCount++
}

// backup2 backs the input stream up two tokens.
// The zeroth token is already there.
func (t *Tree) backup2(t1 item) {
	t.token[1] = t1
	t.peekCount = 2
}

// parseText scan until itemBr occur.
func (t *Tree) parseText(input string) (nodes []Node) {
	// HACK: if there's more 'itemText' in the way, make it one.
	for {
		tkn := t.next()
		if tkn.typ == itemText {
			input += tkn.val
		} else if tkn.typ == itemNewLine {
			if t.peek().typ != itemText {
				t.backup2(tkn)
				break
			}
			input += tkn.val
		} else {
			t.backup()
			break
		}
	}
	l := lexInline(input)
	for token := range l.items {
		var node Node
		switch token.typ {
		case itemNewLine:
			node = t.newLine(token.pos)
		case itemBr:
			node = t.newBr(token.pos)
		case itemStrong, itemItalic, itemStrike, itemCode:
			node = t.parseEmphasis(token.typ, token.pos, token.val)
		case itemLink, itemAutoLink, itemGfmLink:
			var title, text, href string
			match := span[token.typ].FindStringSubmatch(token.val)
			if token.typ == itemLink {
				text, href, title = match[1], match[2], match[3]
			} else {
				text, href = match[1], match[1]
			}
			node = t.newLink(token.pos, title, href, text)
		case itemImage:
			match := span[token.typ].FindStringSubmatch(token.val)
			node = t.newImage(token.pos, match[3], match[2], match[1])
		case itemText:
			node = t.newText(token.pos, token.val)
		default:
			fmt.Println("Matching not found for this token:", token)
		}
		nodes = append(nodes, node)
	}
	return nodes
}

// Parse inline emphasis
func (t *Tree) parseEmphasis(typ itemType, pos Pos, val string) *EmphasisNode {
	node := t.newEmphasis(pos, typ)
	match := span[typ].FindStringSubmatch(val)
	text := match[len(match)-1]
	if text == "" {
		text = match[1]
	}
	node.Nodes = t.parseText(text)
	return node
}

// parse heading block
func (t *Tree) parseHeading() (node *HeadingNode) {
	token := t.next()
	match := block[token.typ].FindStringSubmatch(token.val)
	if token.typ == itemHeading {
		node = t.newHeading(token.pos, len(match[1]), match[2])
	} else {
		// using equal signs for first-level, and dashes for second-level.
		level := 1
		if match[2] == "-" {
			level = 2
		}
		node = t.newHeading(token.pos, level, match[1])
	}
	return
}

func (t *Tree) parseDefLink() *DefLinkNode {
	token := t.next()
	match := block[itemDefLink].FindStringSubmatch(token.val)
	name := strings.ToLower(match[1])
	// name(lowercase), href, title
	n := t.newDefLink(token.pos, name, match[2], match[3])
	// store in links
	t.links[name] = n
	return n
}

// parse codeBlock
func (t *Tree) parseCodeBlock() *CodeNode {
	var lang, text string
	token := t.next()
	if token.typ == itemGfmCodeBlock {
		match := block[itemGfmCodeBlock].FindStringSubmatch(token.val)
		if text = match[2]; text == "" {
			text = match[4]
		}
		if lang = match[1]; lang == "" {
			lang = match[3]
		}
	} else {
		text = regexp.MustCompile("(?m)( {4}|\t)").ReplaceAllLiteralString(token.val, "")
	}
	return t.newCode(token.pos, lang, text)
}

func (t *Tree) parseBlockQuote() (n *BlockQuoteNode) {
	token := t.next()
	// replacer
	re := regexp.MustCompile(`(?m)^> ?`)
	raw := re.ReplaceAllString(token.val, "")
	// TODO(Ariel): not work right now with defLink(inside the blockQuote)
	tr := &Tree{lex: lex(raw, raw)}
	tr.parse()
	n = t.newBlockQuote(token.pos)
	n.Nodes = tr.Nodes
	return
}

// parse list
func (t *Tree) parseList(depth int) *ListNode {
	token := t.next()
	list := t.newList(token.pos, depth, isDigit(token.val))
	item := new(ListItemNode)
Loop:
	for {
		switch token = t.peek(); token.typ {
		case eof, itemError, itemNewLine:
			break Loop
		// It's actually a listItem
		case itemList:
			// List, but not the same type
			if list.Ordered != isDigit(token.val) || depth > 0 {
				break Loop
			}
			item = t.parseListItem(t.next().pos, list)
		case itemIndent:
			t.next()
			if depth == len(token.val) {
				item = t.parseListItem(t.next().pos, list)
			} else {
				t.backup()
				break Loop
			}
		default:
			item = t.parseListItem(token.pos, list)
		}
		list.append(item)
	}
	return list
}

// parse listItem
func (t *Tree) parseListItem(pos Pos, list *ListNode) *ListItemNode {
	item := t.newListItem(pos, list)
	var n Node
Loop:
	for {
		switch token := t.next(); token.typ {
		case eof, itemError:
			break Loop
		case itemList:
			t.backup()
			break Loop
		case itemNewLine:
			switch typ := t.peek().typ; typ {
			case itemNewLine, eof, itemError:
				t.backup2(token)
				break Loop
			case itemList, itemIndent:
				continue
			default:
				n = t.newLine(token.pos)
			}
		case itemIndent:
			if t.peek().typ == itemList {
				depth := len(token.val)
				// If it's in the same depth - sibling
				// or if it's less-than - exit
				if depth <= item.List.Depth {
					t.backup2(token)
					break Loop
				}
				n = t.parseList(depth)
			} else {
				n = t.newText(token.pos, token.val)
			}
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		default:
			// DRY
			for _, n := range t.parseText(token.val) {
				// TODO: Remove this condition
				if n.Type() != NodeNewLine {
					item.append(n)
				}
			}
			continue
		}
		item.append(n)
	}
	return item
}

// parse table
func (t *Tree) parseTable() *TableNode {
	// End of table
	done := t.lex.eot
	token := t.next()
	// ignore the first and last one...
	//lp := token.val == "|"
	table := t.newTable(token.pos)
	// Align	[ None, Left, Right, ... ]
	// Header	[ Cels: [token, token, ... ] ]
	// Data:	[ Row: [Cells: [token, ... ] ] ]
	rows := struct {
		Align  []AlignType
		Header [][]item
		Data   [][][]item
	}{}
	var cell []item
	var row [][]item
	// Collect items
Loop:
	for i := 0; ; {
		switch token := t.next(); token.typ {
		case eof, itemError:
			break Loop
		case itemNewLine:
			// If we done with this table
			if t.peek().pos >= done {
				break Loop
			}
			fallthrough
		case itemPipe:
			// Test if cell non-empty before appending to current row
			if len(cell) > 0 {
				// Header
				if i == 0 {
					rows.Header = append(rows.Header, cell)
					// Alignment
				} else if i == 1 {
					align := cell[0].val
					if len(cell) > 1 {
						for i := 1; i < len(cell); i++ {
							align += cell[i].val
						}
					}
					// Trim spaces
					rows.Align = append(rows.Align, parseAlign(align))
					// Data
				} else {
					row = append(row, cell)
				}
			}
			if token.typ == itemNewLine {
				i++
				// test if there's an elemnts to append to tbody.
				// we want to avoid situations like `appending empty rows`, etc..
				if i > 2 && len(row) > 0 {
					rows.Data = append(rows.Data, row)
					row = [][]item{}
				}
			}
			cell = []item{}
		default:
			cell = append(cell, token)
		}
	}
	// Drain cell/row
	if len(cell) > 0 {
		row = append(row, cell)
	}
	if len(row) > 0 {
		rows.Data = append(rows.Data, row)
	}
	// Tranform to nodes
	// Add an average mechanisem, that ignore empty(or " ") in end of cell
	//	rowLen := len(rows.Align)
	// Table head
	table.append(t.parseCells(Header, rows.Header, rows.Align))
	// Table body
	for _, row := range rows.Data {
		table.append(t.parseCells(Data, row, rows.Align))
	}
	return table
}

// Should return typ []CellNode
func (t *Tree) parseCells(kind int, items [][]item, align []AlignType) *RowNode {
	// TODO(Ariel): Add real position
	row := t.newRow(1)
	for i, item := range items {
		// Cell contain nodes
		var s string
		cell := t.newCell(item[0].pos, kind, align[i])
		for _, tkn := range item {
			s += tkn.val
		}
		// Used before: `^[ |\t]*|[ |\t]*$`
		cell.Nodes = t.parseText(strings.TrimSpace(s))
		row.append(cell)
	}
	return row
}

// get align-string and return the align type of it
// e.g: ":---", "---:", ":---:", "---"
func parseAlign(s string) (typ AlignType) {
	// Trim spaces before
	s = strings.TrimSpace(s)
	sfx, pfx := strings.HasSuffix(s, ":"), strings.HasPrefix(s, ":")
	switch {
	case sfx && pfx:
		typ = Center
	case sfx:
		typ = Right
	case pfx:
		typ = Left
	}
	return
}

// test if given string is digit
func isDigit(s string) bool {
	r, _ := utf8.DecodeRuneInString(s)
	return unicode.IsDigit(r)
}

// test if given token is type block
func isBlock(item itemType) (b bool) {
	switch item {
	case itemHeading, itemLHeading, itemCodeBlock, itemBlockQuote,
		itemList, itemTable, itemGfmCodeBlock, itemHr:
		b = true
	}
	return
}
<commit_msg>refac(parser): parseList/ListItem<commit_after>package mark

import (
	fmt "github.com/k0kubun/pp"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

type Tree struct {
	text  string
	lex   *lexer
	Nodes []Node
	// Parsing only
	token     [3]item // three-token lookahead for parser
	peekCount int
	output    string
	links     map[string]*DefLinkNode
}

// Parse convert the raw text to NodeTree.
func (t *Tree) parse() {
Loop:
	for {
		var n Node
		switch p := t.peek(); p.typ {
		case eof, itemError:
			break Loop
		case itemNewLine:
			n = t.newLine(t.next().pos)
		case itemBr:
			n = t.newBr(t.next().pos)
		case itemHr:
			n = t.newHr(t.next().pos)
		case itemText:
			tmp := t.newParagraph(p.pos)
			tmp.Nodes = t.parseText(t.next().val)
			n = tmp
		case itemDefLink:
			n = t.parseDefLink()
		case itemHeading, itemLHeading:
			n = t.parseHeading()
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		case itemList:
			n = t.parseList()
		case itemTable, itemLpTable:
			n = t.parseTable()
		case itemBlockQuote:
			n = t.parseBlockQuote()
		default:
			t.next()
		}
		if n != nil {
			t.append(n)
		}
	}
}

// Render parse nodes to the wanted output
func (t *Tree) render() {
	var last Node
	last = t.newLine(0)
	for _, node := range t.Nodes {
		if last.Type() != NodeNewLine || node.Type() != last.Type() {
			t.output += node.Render()
		}
		last = node
	}
}

// append new node to nodes-list
func (t *Tree) append(n Node) {
	t.Nodes = append(t.Nodes, n)
}

// next returns the next token
func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]
	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

// backup backs the input stream tp one token
func (t *Tree) backup() {
	t.peekCount++
}

// backup2 backs the input stream up two tokens.
// The zeroth token is already there.
func (t *Tree) backup2(t1 item) {
	t.token[1] = t1
	t.peekCount = 2
}

// parseText scan until itemBr occur.
func (t *Tree) parseText(input string) (nodes []Node) {
	// HACK: if there's more 'itemText' in the way, make it one.
	for {
		tkn := t.next()
		if tkn.typ == itemText {
			input += tkn.val
		} else if tkn.typ == itemNewLine {
			if t.peek().typ != itemText {
				t.backup2(tkn)
				break
			}
			input += tkn.val
		} else {
			t.backup()
			break
		}
	}
	l := lexInline(input)
	for token := range l.items {
		var node Node
		switch token.typ {
		case itemNewLine:
			node = t.newLine(token.pos)
		case itemBr:
			node = t.newBr(token.pos)
		case itemStrong, itemItalic, itemStrike, itemCode:
			node = t.parseEmphasis(token.typ, token.pos, token.val)
		case itemLink, itemAutoLink, itemGfmLink:
			var title, text, href string
			match := span[token.typ].FindStringSubmatch(token.val)
			if token.typ == itemLink {
				text, href, title = match[1], match[2], match[3]
			} else {
				text, href = match[1], match[1]
			}
			node = t.newLink(token.pos, title, href, text)
		case itemImage:
			match := span[token.typ].FindStringSubmatch(token.val)
			node = t.newImage(token.pos, match[3], match[2], match[1])
		case itemText:
			node = t.newText(token.pos, token.val)
		default:
			fmt.Println("Matching not found for this token:", token)
		}
		nodes = append(nodes, node)
	}
	return nodes
}

// Parse inline emphasis
func (t *Tree) parseEmphasis(typ itemType, pos Pos, val string) *EmphasisNode {
	node := t.newEmphasis(pos, typ)
	match := span[typ].FindStringSubmatch(val)
	text := match[len(match)-1]
	if text == "" {
		text = match[1]
	}
	node.Nodes = t.parseText(text)
	return node
}

// parse heading block
func (t *Tree) parseHeading() (node *HeadingNode) {
	token := t.next()
	match := block[token.typ].FindStringSubmatch(token.val)
	if token.typ == itemHeading {
		node = t.newHeading(token.pos, len(match[1]), match[2])
	} else {
		// using equal signs for first-level, and dashes for second-level.
		level := 1
		if match[2] == "-" {
			level = 2
		}
		node = t.newHeading(token.pos, level, match[1])
	}
	return
}

func (t *Tree) parseDefLink() *DefLinkNode {
	token := t.next()
	match := block[itemDefLink].FindStringSubmatch(token.val)
	name := strings.ToLower(match[1])
	// name(lowercase), href, title
	n := t.newDefLink(token.pos, name, match[2], match[3])
	// store in links
	t.links[name] = n
	return n
}

// parse codeBlock
func (t *Tree) parseCodeBlock() *CodeNode {
	var lang, text string
	token := t.next()
	if token.typ == itemGfmCodeBlock {
		match := block[itemGfmCodeBlock].FindStringSubmatch(token.val)
		if text = match[2]; text == "" {
			text = match[4]
		}
		if lang = match[1]; lang == "" {
			lang = match[3]
		}
	} else {
		text = regexp.MustCompile("(?m)( {4}|\t)").ReplaceAllLiteralString(token.val, "")
	}
	return t.newCode(token.pos, lang, text)
}

func (t *Tree) parseBlockQuote() (n *BlockQuoteNode) {
	token := t.next()
	// replacer
	re := regexp.MustCompile(`(?m)^> ?`)
	raw := re.ReplaceAllString(token.val, "")
	// TODO(Ariel): not work right now with defLink(inside the blockQuote)
	tr := &Tree{lex: lex(raw)}
	tr.parse()
	n = t.newBlockQuote(token.pos)
	n.Nodes = tr.Nodes
	return
}

// parse list
func (t *Tree) parseList() *ListNode {
	token := t.next()
	list := t.newList(token.pos, isDigit(token.val))
Loop:
	for {
		switch token = t.peek(); token.typ {
		case itemLooseItem, itemListItem:
			list.append(t.parseListItem())
		default:
			break Loop
		}
	}
	return list
}

// parse listItem
// Add ignore list(e.g: table should parse as a text)
func (t *Tree) parseListItem() *ListItemNode {
	token := t.next()
	item := t.newListItem(token.pos)
	tr := &Tree{lex: lex(strings.TrimSpace(token.val))}
	tr.parse()
	for _, node := range tr.Nodes {
		// wrap with paragraph only when it's loose item
		if n, ok := node.(*ParagraphNode); ok && token.typ == itemListItem {
			item.Nodes = append(item.Nodes, n.Nodes...)
		} else {
			item.append(node)
		}
	}
	return item
}

// parse table
func (t *Tree) parseTable() *TableNode {
	// End of table
	done := t.lex.eot
	token := t.next()
	// ignore the first and last one...
	//lp := token.val == "|"
	table := t.newTable(token.pos)
	// Align	[ None, Left, Right, ... ]
	// Header	[ Cels: [token, token, ... ] ]
	// Data:	[ Row: [Cells: [token, ... ] ] ]
	rows := struct {
		Align  []AlignType
		Header [][]item
		Data   [][][]item
	}{}
	var cell []item
	var row [][]item
	// Collect items
Loop:
	for i := 0; ; {
		switch token := t.next(); token.typ {
		case eof, itemError:
			break Loop
		case itemNewLine:
			// If we done with this table
			if t.peek().pos >= done {
				break Loop
			}
			fallthrough
		case itemPipe:
			// Test if cell non-empty before appending to current row
			if len(cell) > 0 {
				// Header
				if i == 0 {
					rows.Header = append(rows.Header, cell)
					// Alignment
				} else if i == 1 {
					align := cell[0].val
					if len(cell) > 1 {
						for i := 1; i < len(cell); i++ {
							align += cell[i].val
						}
					}
					// Trim spaces
					rows.Align = append(rows.Align, parseAlign(align))
					// Data
				} else {
					row = append(row, cell)
				}
			}
			if token.typ == itemNewLine {
				i++
				// test if there's an elemnts to append to tbody.
				// we want to avoid situations like `appending empty rows`, etc..
				if i > 2 && len(row) > 0 {
					rows.Data = append(rows.Data, row)
					row = [][]item{}
				}
			}
			cell = []item{}
		default:
			cell = append(cell, token)
		}
	}
	// Drain cell/row
	if len(cell) > 0 {
		row = append(row, cell)
	}
	if len(row) > 0 {
		rows.Data = append(rows.Data, row)
	}
	// Tranform to nodes
	// Add an average mechanisem, that ignore empty(or " ") in end of cell
	//	rowLen := len(rows.Align)
	// Table head
	table.append(t.parseCells(Header, rows.Header, rows.Align))
	// Table body
	for _, row := range rows.Data {
		table.append(t.parseCells(Data, row, rows.Align))
	}
	return table
}

// Should return typ []CellNode
func (t *Tree) parseCells(kind int, items [][]item, align []AlignType) *RowNode {
	// TODO(Ariel): Add real position
	row := t.newRow(1)
	for i, item := range items {
		// Cell contain nodes
		var s string
		cell := t.newCell(item[0].pos, kind, align[i])
		for _, tkn := range item {
			s += tkn.val
		}
		// Used before: `^[ |\t]*|[ |\t]*$`
		cell.Nodes = t.parseText(strings.TrimSpace(s))
		row.append(cell)
	}
	return row
}

// get align-string and return the align type of it
// e.g: ":---", "---:", ":---:", "---"
func parseAlign(s string) (typ AlignType) {
	// Trim spaces before
	s = strings.TrimSpace(s)
	sfx, pfx := strings.HasSuffix(s, ":"), strings.HasPrefix(s, ":")
	switch {
	case sfx && pfx:
		typ = Center
	case sfx:
		typ = Right
	case pfx:
		typ = Left
	}
	return
}

// test if given string is digit
func isDigit(s string) bool {
	r, _ := utf8.DecodeRuneInString(s)
	return unicode.IsDigit(r)
}

// test if given token is type block
func isBlock(item itemType) (b bool) {
	switch item {
	case itemHeading, itemLHeading, itemCodeBlock, itemBlockQuote,
		itemList, itemTable, itemGfmCodeBlock, itemHr:
		b = true
	}
	return
}
<|endoftext|>


<commit_before>/*
parser inspired by Rob Pikes lexer
*/

package parser

import (
	"errors"
	"fmt"
	"strings"
	"unicode/utf8"
)

// last State must return ErrEOF
type State func(p *Parser) (next State)

var ErrEOF = errors.New("End of File")

var EOF = rune('∎')

type ASTNode interface {
	AddChild(ASTNode)
}

type Parser struct {
	astQueue    []ASTNode
	input       string // the string being scanned
	start       int    // start position of this item
	pos         int    // current position in the input
	width       int    // width of the last rune read
	line        int
	linepos     int
	linePrev    int
	lineposPrev int
	err         error
}

func New(input string, root ASTNode) *Parser {
	return &Parser{
		astQueue: []ASTNode{root},
		input:    input,
	}
}

func (p *Parser) Root() ASTNode {
	return p.astQueue[0]
}

func (p *Parser) currentNode() ASTNode {
	return p.astQueue[len(p.astQueue)-1]
}

func (p *Parser) AddNode(n ASTNode) {
	p.currentNode().AddChild(n)
	p.astQueue = append(p.astQueue, n)
}

func (p *Parser) PopNode() {
	if len(p.astQueue) < 2 {
		return
	}
	p.astQueue = p.astQueue[:len(p.astQueue)-1]
}

func (p *Parser) HasError() bool {
	return p.err == nil || p.err == ErrEOF
}

func (p *Parser) IsEOF() bool {
	return p.err == ErrEOF
}

func (p *Parser) Next() (rune_ rune) {
	if p.pos >= len(p.input) {
		p.width = 0
		p.err = ErrEOF
		return EOF
	}
	rune_, p.width = utf8.DecodeRuneInString(p.input[p.pos:])
	p.pos += p.width
	p.linePrev = p.line
	p.lineposPrev = p.linepos
	if rune_ == '\n' {
		p.line++
		p.linepos = 0
	} else {
		p.linepos++
	}

	return
}

// emit passes an item back to the client
func (p *Parser) Emit() string {
	s := p.input[p.start:p.pos]
	p.start = p.pos
	return s
}

func (p *Parser) Ignore() {
	p.start = p.pos
}

// backup steps back one rune
// can be called only once per call of next
func (p *Parser) Backup() {
	rune_, _ := utf8.DecodeRuneInString(p.input[p.pos:])
	if rune_ == '\n' {
		p.line--
	}
	p.linepos = p.lineposPrev
	p.pos -= p.width
}

func (p *Parser) Peek() rune {
	r := p.Next()
	p.Backup()
	return r
}

func (p *Parser) Accept(valid string) bool {
	if strings.IndexRune(valid, p.Next()) >= 0 {
		return true
	}
	p.Backup()
	return false
}

func (p *Parser) AcceptRun(valid string) {
	for strings.IndexRune(valid, p.Next()) >= 0 {
	}
	p.Backup()
}

// runs forward until one of the stopper
func (p *Parser) ForwardUntil(stopper string) {
	for strings.IndexRune(stopper, p.Next()) == -1 {
	}
	p.Backup()
}

func (p *Parser) Errorf(format string, args ...interface{}) {
	start := p.pos - 5
	if start < 0 {
		start = 0
	}

	end := p.pos + 5

	if end > len(p.input) {
		end = len(p.input)
	}

	p.err = errors.New(fmt.Sprintf(
		"Error in line %d at position %d: %s\ncontext:\n%s\n",
		p.line+1,
		p.linepos+1,
		fmt.Sprintf(format, args...),
		p.input[start:end],
	))
}

func (p *Parser) Run(fn State) (err error) {
	for p.err == nil {
		fn = fn(p)
	}
	if p.err == ErrEOF {
		return nil
	}

	return err
}
<commit_msg>break if returned function is nil<commit_after>/*
parser inspired by Rob Pikes lexer
*/

package parser

import (
	"errors"
	"fmt"
	"strings"
	"unicode/utf8"
)

// last State must return ErrEOF
type State func(p *Parser) (next State)

var ErrEOF = errors.New("End of File")

var EOF = rune('∎')

type ASTNode interface {
	AddChild(ASTNode)
}

type Parser struct {
	astQueue    []ASTNode
	input       string // the string being scanned
	start       int    // start position of this item
	pos         int    // current position in the input
	width       int    // width of the last rune read
	line        int
	linepos     int
	linePrev    int
	lineposPrev int
	err         error
}

func New(input string, root ASTNode) *Parser {
	return &Parser{
		astQueue: []ASTNode{root},
		input:    input,
	}
}

func (p *Parser) Root() ASTNode {
	return p.astQueue[0]
}

func (p *Parser) currentNode() ASTNode {
	return p.astQueue[len(p.astQueue)-1]
}

func (p *Parser) AddNode(n ASTNode) {
	p.currentNode().AddChild(n)
	p.astQueue = append(p.astQueue, n)
}

func (p *Parser) PopNode() {
	if len(p.astQueue) < 2 {
		return
	}
	p.astQueue = p.astQueue[:len(p.astQueue)-1]
}

func (p *Parser) HasError() bool {
	return p.err == nil || p.err == ErrEOF
}

func (p *Parser) IsEOF() bool {
	return p.err == ErrEOF
}

func (p *Parser) Next() (rune_ rune) {
	if p.pos >= len(p.input) {
		p.width = 0
		p.err = ErrEOF
		return EOF
	}
	rune_, p.width = utf8.DecodeRuneInString(p.input[p.pos:])
	p.pos += p.width
	p.linePrev = p.line
	p.lineposPrev = p.linepos
	if rune_ == '\n' {
		p.line++
		p.linepos = 0
	} else {
		p.linepos++
	}

	return
}

// emit passes an item back to the client
func (p *Parser) Emit() string {
	s := p.input[p.start:p.pos]
	p.start = p.pos
	return s
}

func (p *Parser) Ignore() {
	p.start = p.pos
}

// backup steps back one rune
// can be called only once per call of next
func (p *Parser) Backup() {
	rune_, _ := utf8.DecodeRuneInString(p.input[p.pos:])
	if rune_ == '\n' {
		p.line--
	}
	p.linepos = p.lineposPrev
	p.pos -= p.width
}

func (p *Parser) Peek() rune {
	r := p.Next()
	p.Backup()
	return r
}

func (p *Parser) Accept(valid string) bool {
	if strings.IndexRune(valid, p.Next()) >= 0 {
		return true
	}
	p.Backup()
	return false
}

func (p *Parser) AcceptRun(valid string) {
	for strings.IndexRune(valid, p.Next()) >= 0 {
	}
	p.Backup()
}

// runs forward until one of the stopper
func (p *Parser) ForwardUntil(stopper string) {
	for strings.IndexRune(stopper, p.Next()) == -1 {
	}
	p.Backup()
}

func (p *Parser) Errorf(format string, args ...interface{}) {
	start := p.pos - 5
	if start < 0 {
		start = 0
	}

	end := p.pos + 5

	if end > len(p.input) {
		end = len(p.input)
	}

	p.err = errors.New(fmt.Sprintf(
		"Error in line %d at position %d: %s\ncontext:\n%s\n",
		p.line+1,
		p.linepos+1,
		fmt.Sprintf(format, args...),
		p.input[start:end],
	))
}

func (p *Parser) Run(fn State) (err error) {
	for p.err == nil {
		fn = fn(p)
		if fn == nil {
			break
		}
	}
	if p.err == ErrEOF {
		return nil
	}

	return err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"

	"./Gobang"
	"github.com/bitly/go-simplejson"
)

func main() {
	fmt.Println("listening: 80 ...")
	http.HandleFunc("/AI", handlerAI)
	http.ListenAndServe(":80", nil)
}

func handlerAI(w http.ResponseWriter, r *http.Request) {
	r_data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return
	}
	defer r.Body.Close()

	r_json, err := simplejson.NewJson(r_data)
	if err != nil {
		w.Write([]byte("parse request failed"))
		return
	}
	w_json := r_json

	// 0 - 控制消息    1 - 下棋消息
	msg_type := r_json.GetPath("head", "type").MustString()
	fmt.Println("---------------------------------------------------")
	fmt.Println("request type:", msg_type, string(r_data))
	fmt.Println("---------------------------------------------------")

	if msg_type == "0" {
		player_black := r_json.GetPath("body", "player_black").MustMap()

		// 请求和回复的数据结构都一样，修改部分字段回复即可
		// 回复下棋的URL
		w_json.SetPath([]string{"head", "result"}, 0)
		w_json.SetPath([]string{"head", "err_msg"}, "success")
		w_json.SetPath([]string{"body", "url"}, "http://120.76.53.165/AI")
		if player_black == nil || len(player_black) == 0 {
			w_json.SetPath([]string{"body", "player_white", "name"}, "alpha_white")
		} else {
			w_json.SetPath([]string{"body", "player_black", "name"}, "alpha_black")
		}

	} else {
		// 读取棋盘信息
		chessMap := make([][]byte, 0)
		for i := 0; i < 15; i++ {
			sliceByte := make([]byte, 0)
			for j := 0; j < 15; j++ {
				sliceByte = append(sliceByte, ' ')
			}
			chessMap = append(chessMap, sliceByte)
		}

		m := r_json.GetPath("body", "steps").MustArray()
		for _, v := range m {
			var m2 map[string]interface{} = v.(map[string]interface{})
			x := m2["x"].(string)
			y := m2["y"].(string)
			side := m2["side"].(string)

			xi, _ := strconv.ParseInt(x, 10, 8)
			yi, _ := strconv.ParseInt(y, 10, 8)

			if xi >= 1 && xi <= 15 && yi >= 1 && yi <= 15 {
				if side == "w" {
					chessMap[xi-1][yi-1] = 'W'
				} else {
					chessMap[xi-1][yi-1] = 'B'
				}
			}
		}

		// AI
		row, col, flag := Gobang.AI(chessMap)

		fmt.Printf("******* row: %v  col: %v  len(map): %v\n", row+1, col+1, len(m))

		// response
		steps_json := simplejson.New()
		if flag == 1 {
			steps_json.Set("side", "b")
		} else {
			steps_json.Set("side", "w")
		}

		steps_json.Set("x", strconv.FormatInt(int64(row+1), 10))
		steps_json.Set("y", strconv.FormatInt(int64(col+1), 10))
		steps_json.Set("time", getTime())

		w_json := r_json
		w_json.SetPath([]string{"body", "steps"}, []*simplejson.Json{steps_json})
	}

	w_json_bytes, _ := w_json.Encode()
	w.Write(w_json_bytes)

	fmt.Println("---------------------------------------------------")
	fmt.Println("reponse type:", msg_type, string(w_json_bytes))
	fmt.Println("---------------------------------------------------")
}

func getTime() string {
	now := time.Now()
	year, mon, day := now.Date()
	hour, min, sec := now.Clock()
	return fmt.Sprintf("%d%d%d%02d%02d%02d", year, mon, day, hour, min, sec)
}
<commit_msg>fixtypo<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"

	"./Gobang"
	"github.com/bitly/go-simplejson"
)

func main() {
	fmt.Println("listening: 80 ...")
	http.HandleFunc("/AI", handlerAI)
	http.ListenAndServe(":80", nil)
}

func handlerAI(w http.ResponseWriter, r *http.Request) {
	r_data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return
	}
	defer r.Body.Close()

	r_json, err := simplejson.NewJson(r_data)
	if err != nil {
		w.Write([]byte("parse request failed"))
		return
	}
	w_json := r_json

	// 0 - 控制消息    1 - 下棋消息
	msg_type := r_json.GetPath("head", "type").MustString()
	fmt.Println("---------------------------------------------------")
	fmt.Println("request type:", msg_type, string(r_data))
	fmt.Println("---------------------------------------------------")

	if msg_type == "0" {
		player_black := r_json.GetPath("body", "player_black").MustMap()

		// 请求和回复的数据结构都一样，修改部分字段回复即可
		// 回复下棋的URL
		w_json.SetPath([]string{"head", "result"}, 0)
		w_json.SetPath([]string{"head", "err_msg"}, "success")
		w_json.SetPath([]string{"body", "url"}, "http://120.76.53.165/AI")
		if player_black == nil || len(player_black) == 0 {
			w_json.SetPath([]string{"body", "player_white", "name"}, "alpha_white")
		} else {
			w_json.SetPath([]string{"body", "player_black", "name"}, "alpha_black")
		}

	} else {
		// 读取棋盘信息
		chessMap := make([][]byte, 0)
		for i := 0; i < 15; i++ {
			sliceByte := make([]byte, 0)
			for j := 0; j < 15; j++ {
				sliceByte = append(sliceByte, ' ')
			}
			chessMap = append(chessMap, sliceByte)
		}

		m := r_json.GetPath("body", "steps").MustArray()
		for _, v := range m {
			var m2 map[string]interface{} = v.(map[string]interface{})
			x := m2["x"].(string)
			y := m2["y"].(string)
			side := m2["side"].(string)

			xi, _ := strconv.ParseInt(x, 10, 8)
			yi, _ := strconv.ParseInt(y, 10, 8)

			if xi >= 1 && xi <= 15 && yi >= 1 && yi <= 15 {
				if side == "w" {
					chessMap[xi-1][yi-1] = 'W'
				} else {
					chessMap[xi-1][yi-1] = 'B'
				}
			}
		}

		// AI
		row, col, flag := Gobang.AI(chessMap)

		fmt.Printf("******* row: %v  col: %v  len(map): %v\n", row+1, col+1, len(m))

		// response
		steps_json := simplejson.New()
		if flag == 1 {
			steps_json.Set("side", "b")
		} else {
			steps_json.Set("side", "w")
		}

		steps_json.Set("x", strconv.FormatInt(int64(row+1), 10))
		steps_json.Set("y", strconv.FormatInt(int64(col+1), 10))
		steps_json.Set("time", getTime())

		w_json := r_json

		stepArray := r_json.GetPath("body", "steps").MustArray()
		stepArray = append(stepArray, steps_json)
		w_json.SetPath([]string{"body", "steps"}, stepArray)
	}

	w_json_bytes, _ := w_json.Encode()
	w.Write(w_json_bytes)

	fmt.Println("---------------------------------------------------")
	fmt.Println("reponse type:", msg_type, string(w_json_bytes))
	fmt.Println("---------------------------------------------------")
}

func getTime() string {
	now := time.Now()
	year, mon, day := now.Date()
	hour, min, sec := now.Clock()
	return fmt.Sprintf("%d%d%d%02d%02d%02d", year, mon, day, hour, min, sec)
}
<|endoftext|>


<commit_before>// RTLAMR - An rtl-sdr receiver for smart meters operating in the 900MHz ISM band.
// Copyright (C) 2015 Douglas Hall
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package decode

import (
	"fmt"
	"log"
	"math"
)

// PacketConfig specifies packet-specific radio configuration.
type PacketConfig struct {
	DataRate int

	BlockSize, BlockSize2    int
	ChipLength, SymbolLength int
	SampleRate               int

	PreambleSymbols, PacketSymbols int
	PreambleLength, PacketLength   int
	Preamble                       string

	BufferLength int

	CenterFreq uint32
}

func (d Decoder) Log() {
	log.Println("CenterFreq:", d.Cfg.CenterFreq)
	log.Println("SampleRate:", d.Cfg.SampleRate)
	log.Println("DataRate:", d.Cfg.DataRate)
	log.Println("ChipLength:", d.Cfg.ChipLength)
	log.Println("PreambleSymbols:", d.Cfg.PreambleSymbols)
	log.Println("PreambleLength:", d.Cfg.PreambleLength)
	log.Println("PacketSymbols:", d.Cfg.PacketSymbols)
	log.Println("PacketLength:", d.Cfg.PacketLength)
	log.Println("Preamble:", d.Cfg.Preamble)
}

// Decoder contains buffers and radio configuration.
type Decoder struct {
	Cfg PacketConfig

	Signal    []float64
	Filtered  []float64
	Quantized []byte

	csum  []float64
	demod Demodulator

	preamble []byte

	pkt []byte

	packed       []byte
	sIdxA, sIdxB []int
}

// Create a new decoder with the given packet configuration.
func NewDecoder(cfg PacketConfig) (d Decoder) {
	d.Cfg = cfg

	d.Cfg.SymbolLength = d.Cfg.ChipLength << 1
	d.Cfg.SampleRate = d.Cfg.DataRate * d.Cfg.ChipLength

	d.Cfg.PreambleLength = d.Cfg.PreambleSymbols * d.Cfg.SymbolLength
	d.Cfg.PacketLength = d.Cfg.PacketSymbols * d.Cfg.SymbolLength

	d.Cfg.BlockSize = NextPowerOf2(d.Cfg.PreambleLength)
	d.Cfg.BlockSize2 = d.Cfg.BlockSize << 1

	d.Cfg.BufferLength = d.Cfg.PacketLength + d.Cfg.BlockSize

	// Allocate necessary buffers.
	d.Signal = make([]float64, d.Cfg.BlockSize+d.Cfg.SymbolLength)
	d.Filtered = make([]float64, d.Cfg.BlockSize)
	d.Quantized = make([]byte, d.Cfg.BufferLength)

	d.csum = make([]float64, len(d.Signal)+1)

	// Calculate magnitude lookup table specified by -fastmag flag.
	d.demod = NewMagLUT()

	// Pre-calculate a byte-slice version of the preamble for searching.
	d.preamble = make([]byte, d.Cfg.PreambleSymbols)
	for idx := range d.Cfg.Preamble {
		if d.Cfg.Preamble[idx] == '1' {
			d.preamble[idx] = 1
		}
	}

	// Signal up to the final stage is 1-bit per byte. Allocate a buffer to
	// store packed version 8-bits per byte.
	d.pkt = make([]byte, (d.Cfg.PacketSymbols+7)>>3)

	d.sIdxA = make([]int, 0, d.Cfg.BlockSize)
	d.sIdxB = make([]int, 0, d.Cfg.BlockSize)

	d.packed = make([]byte, (d.Cfg.BlockSize+d.Cfg.PreambleLength+7)>>3)

	return
}

// Decode accepts a sample block and performs various DSP techniques to extract a packet.
func (d Decoder) Decode(input []byte) []int {
	// Shift buffers to append new block.
	copy(d.Signal, d.Signal[d.Cfg.BlockSize:])
	copy(d.Quantized, d.Quantized[d.Cfg.BlockSize:])

	// Compute the magnitude of the new block.
	d.demod.Execute(input, d.Signal[d.Cfg.SymbolLength:])

	// Perform matched filter on new block.
	d.Filter(d.Signal, d.Filtered)

	// Perform bit-decision on new block.
	Quantize(d.Filtered, d.Quantized[d.Cfg.PacketLength:])

	// Return a list of indices the preamble exists at.
	return d.Search()
}

// A Demodulator knows how to demodulate an array of uint8 IQ samples into an
// array of float64 samples.
type Demodulator interface {
	Execute([]byte, []float64)
}

// Default Magnitude Lookup Table
type MagLUT []float64

// Pre-computes normalized squares with most common DC offset for rtl-sdr dongles.
func NewMagLUT() (lut MagLUT) {
	lut = make([]float64, 0x100)
	for idx := range lut {
		lut[idx] = (127.5 - float64(idx)) / 127.5
		lut[idx] *= lut[idx]
	}
	return
}

// Calculates complex magnitude on given IQ stream writing result to output.
func (lut MagLUT) Execute(input []byte, output []float64) {
	i := 0
	for idx := range output {
		output[idx] = lut[input[i]] + lut[input[i+1]]
		i += 2
	}
}

// Matched filter for Manchester coded signals. Output signal's sign at each
// sample determines the bit-value due to Manchester symbol odd symmetry.
func (d Decoder) Filter(input, output []float64) {
	// Computing the cumulative summation over the signal simplifies
	// filtering to the difference of a pair of subtractions.
	var sum float64
	for idx, v := range input {
		sum += v
		d.csum[idx+1] = sum
	}

	// Filter result is difference of summation of lower and upper chips.
	lower := d.csum[d.Cfg.ChipLength:]
	upper := d.csum[d.Cfg.SymbolLength:]
	for idx, l := range lower[:len(output)] {
		output[idx] = (l - d.csum[idx]) - (upper[idx] - l)
	}

	return
}

// Bit-value is determined by the sign of each sample after filtering.
func Quantize(input []float64, output []byte) {
	// There's really not much we can optimize here as it depends pretty
	// heavily on the content of the input, which is unbiased on average.
	for idx, val := range input {
		output[idx] = 1 - byte(math.Float64bits(val)>>63)
	}

	return
}

// Return a list of indices into the quantized signal at which a valid preamble exists.
func (d *Decoder) Search() []int {
	symLenByte := d.Cfg.SymbolLength >> 3

	// Pack the bit-wise quantized signal into bytes.
	for bIdx := range d.packed {
		var b byte
		for _, qBit := range d.Quantized[bIdx<<3 : (bIdx+1)<<3] {
			b = (b << 1) | qBit
		}
		d.packed[bIdx] = b
	}

	// Filter out indices at which the preamble cannot exist.
	for pIdx, pBit := range d.preamble {
		pBit = (pBit ^ 1) * 0xFF
		offset := pIdx * symLenByte
		if pIdx == 0 {
			d.sIdxA = d.sIdxA[:0]
			for qIdx, b := range d.packed[:d.Cfg.BlockSize>>3] {
				if b != pBit {
					d.sIdxA = append(d.sIdxA, qIdx)
				}
			}
		} else {
			d.sIdxB, d.sIdxA = searchPassByte(pBit, d.packed[offset:], d.sIdxA, d.sIdxB[:0])

			if len(d.sIdxA) == 0 {
				return nil
			}
		}
	}

	symLen := d.Cfg.SymbolLength

	// Unpack the indices from bytes to bits.
	d.sIdxB = d.sIdxB[:0]
	for _, qIdx := range d.sIdxA {
		for idx := 0; idx < 8; idx++ {
			d.sIdxB = append(d.sIdxB, (qIdx<<3)+idx)
		}
	}
	d.sIdxA, d.sIdxB = d.sIdxB, d.sIdxA

	// Filter out indices at which the preamble does not exist.
	for pIdx, pBit := range d.preamble {
		offset := pIdx * symLen
		offsetQuantized := d.Quantized[offset : offset+d.Cfg.BlockSize]
		d.sIdxB, d.sIdxA = searchPass(pBit, offsetQuantized, d.sIdxA, d.sIdxB[:0])

		if len(d.sIdxA) == 0 {
			return nil
		}
	}

	return d.sIdxA
}

func searchPassByte(pBit byte, sig []byte, a, b []int) ([]int, []int) {
	for _, qIdx := range a {
		if sig[qIdx] != pBit {
			b = append(b, qIdx)
		}
	}

	return a, b
}

func searchPass(pBit byte, sig []byte, a, b []int) ([]int, []int) {
	for _, qIdx := range a {
		if sig[qIdx] == pBit {
			b = append(b, qIdx)
		}
	}

	return a, b
}

// Given a list of indices the preamble exists at, sample the appropriate bits
// of the signal's bit-decision. Pack bits of each index into an array of byte
// arrays and return.
func (d Decoder) Slice(indices []int) (pkts [][]byte) {
	// It is likely that a message will be successfully decoded at multiple indices,
	// only keep track of unique instances.
	seen := make(map[string]bool)

	// For each of the indices the preamble exists at.
	for _, qIdx := range indices {
		// Check that we're still within the first sample block. We'll catch
		// the message on the next sample block otherwise.
		if qIdx > d.Cfg.BlockSize {
			continue
		}

		// Packet is 1 bit per byte, pack to 8-bits per byte.
		for pIdx := 0; pIdx < d.Cfg.PacketSymbols; pIdx++ {
			d.pkt[pIdx>>3] <<= 1
			d.pkt[pIdx>>3] |= d.Quantized[qIdx+(pIdx*d.Cfg.SymbolLength)]
		}

		// Store the packet in the seen map and append to the packet list.
		pktStr := fmt.Sprintf("%02X", d.pkt)
		if !seen[pktStr] {
			seen[pktStr] = true
			pkts = append(pkts, make([]byte, len(d.pkt)))
			copy(pkts[len(pkts)-1], d.pkt)
		}
	}

	return
}

func NextPowerOf2(v int) int {
	return 1 << uint(math.Ceil(math.Log2(float64(v))))
}
<commit_msg>Move quantization into filter.<commit_after>// RTLAMR - An rtl-sdr receiver for smart meters operating in the 900MHz ISM band.
// Copyright (C) 2015 Douglas Hall
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package decode

import (
	"fmt"
	"log"
	"math"
)

// PacketConfig specifies packet-specific radio configuration.
type PacketConfig struct {
	DataRate int

	BlockSize, BlockSize2    int
	ChipLength, SymbolLength int
	SampleRate               int

	PreambleSymbols, PacketSymbols int
	PreambleLength, PacketLength   int
	Preamble                       string

	BufferLength int

	CenterFreq uint32
}

func (d Decoder) Log() {
	log.Println("CenterFreq:", d.Cfg.CenterFreq)
	log.Println("SampleRate:", d.Cfg.SampleRate)
	log.Println("DataRate:", d.Cfg.DataRate)
	log.Println("ChipLength:", d.Cfg.ChipLength)
	log.Println("PreambleSymbols:", d.Cfg.PreambleSymbols)
	log.Println("PreambleLength:", d.Cfg.PreambleLength)
	log.Println("PacketSymbols:", d.Cfg.PacketSymbols)
	log.Println("PacketLength:", d.Cfg.PacketLength)
	log.Println("Preamble:", d.Cfg.Preamble)
}

// Decoder contains buffers and radio configuration.
type Decoder struct {
	Cfg PacketConfig

	Signal    []float64
	Quantized []byte

	csum  []float64
	demod Demodulator

	preamble []byte

	pkt []byte

	packed       []byte
	sIdxA, sIdxB []int
}

// Create a new decoder with the given packet configuration.
func NewDecoder(cfg PacketConfig) (d Decoder) {
	d.Cfg = cfg

	d.Cfg.SymbolLength = d.Cfg.ChipLength << 1
	d.Cfg.SampleRate = d.Cfg.DataRate * d.Cfg.ChipLength

	d.Cfg.PreambleLength = d.Cfg.PreambleSymbols * d.Cfg.SymbolLength
	d.Cfg.PacketLength = d.Cfg.PacketSymbols * d.Cfg.SymbolLength

	d.Cfg.BlockSize = NextPowerOf2(d.Cfg.PreambleLength)
	d.Cfg.BlockSize2 = d.Cfg.BlockSize << 1

	d.Cfg.BufferLength = d.Cfg.PacketLength + d.Cfg.BlockSize

	// Allocate necessary buffers.
	d.Signal = make([]float64, d.Cfg.BlockSize+d.Cfg.SymbolLength)
	d.Quantized = make([]byte, d.Cfg.BufferLength)

	d.csum = make([]float64, len(d.Signal)+1)

	// Calculate magnitude lookup table specified by -fastmag flag.
	d.demod = NewMagLUT()

	// Pre-calculate a byte-slice version of the preamble for searching.
	d.preamble = make([]byte, d.Cfg.PreambleSymbols)
	for idx := range d.Cfg.Preamble {
		if d.Cfg.Preamble[idx] == '1' {
			d.preamble[idx] = 1
		}
	}

	// Signal up to the final stage is 1-bit per byte. Allocate a buffer to
	// store packed version 8-bits per byte.
	d.pkt = make([]byte, (d.Cfg.PacketSymbols+7)>>3)

	d.sIdxA = make([]int, 0, d.Cfg.BlockSize)
	d.sIdxB = make([]int, 0, d.Cfg.BlockSize)

	d.packed = make([]byte, (d.Cfg.BlockSize+d.Cfg.PreambleLength+7)>>3)

	return
}

// Decode accepts a sample block and performs various DSP techniques to extract a packet.
func (d Decoder) Decode(input []byte) []int {
	// Shift buffers to append new block.
	copy(d.Signal, d.Signal[d.Cfg.BlockSize:])
	copy(d.Quantized, d.Quantized[d.Cfg.BlockSize:])

	// Compute the magnitude of the new block.
	d.demod.Execute(input, d.Signal[d.Cfg.SymbolLength:])

	// Perform matched filter on new block.
	d.Filter(d.Signal, d.Quantized[d.Cfg.PacketLength:])

	// Return a list of indices the preamble exists at.
	return d.Search()
}

// A Demodulator knows how to demodulate an array of uint8 IQ samples into an
// array of float64 samples.
type Demodulator interface {
	Execute([]byte, []float64)
}

// Default Magnitude Lookup Table
type MagLUT []float64

// Pre-computes normalized squares with most common DC offset for rtl-sdr dongles.
func NewMagLUT() (lut MagLUT) {
	lut = make([]float64, 0x100)
	for idx := range lut {
		lut[idx] = (127.5 - float64(idx)) / 127.5
		lut[idx] *= lut[idx]
	}
	return
}

// Calculates complex magnitude on given IQ stream writing result to output.
func (lut MagLUT) Execute(input []byte, output []float64) {
	i := 0
	for idx := range output {
		output[idx] = lut[input[i]] + lut[input[i+1]]
		i += 2
	}
}

// Matched filter for Manchester coded signals. Output signal's sign at each
// sample determines the bit-value due to Manchester symbol odd symmetry.
func (d Decoder) Filter(input []float64, output []byte) {
	// Computing the cumulative summation over the signal simplifies
	// filtering to the difference of a pair of subtractions.
	var sum float64
	for idx, v := range input {
		sum += v
		d.csum[idx+1] = sum
	}

	// Filter result is difference of summation of lower and upper chips.
	lower := d.csum[d.Cfg.ChipLength:]
	upper := d.csum[d.Cfg.SymbolLength:]
	for idx, l := range lower[:len(output)] {
		f := (l - d.csum[idx]) - (upper[idx] - l)
		output[idx] = 1 - byte(math.Float64bits(f)>>63)
	}

	return
}

// Return a list of indices into the quantized signal at which a valid preamble exists.
func (d *Decoder) Search() []int {
	symLenByte := d.Cfg.SymbolLength >> 3

	// Pack the bit-wise quantized signal into bytes.
	for bIdx := range d.packed {
		var b byte
		for _, qBit := range d.Quantized[bIdx<<3 : (bIdx+1)<<3] {
			b = (b << 1) | qBit
		}
		d.packed[bIdx] = b
	}

	// Filter out indices at which the preamble cannot exist.
	for pIdx, pBit := range d.preamble {
		pBit = (pBit ^ 1) * 0xFF
		offset := pIdx * symLenByte
		if pIdx == 0 {
			d.sIdxA = d.sIdxA[:0]
			for qIdx, b := range d.packed[:d.Cfg.BlockSize>>3] {
				if b != pBit {
					d.sIdxA = append(d.sIdxA, qIdx)
				}
			}
		} else {
			d.sIdxB, d.sIdxA = searchPassByte(pBit, d.packed[offset:], d.sIdxA, d.sIdxB[:0])

			if len(d.sIdxA) == 0 {
				return nil
			}
		}
	}

	symLen := d.Cfg.SymbolLength

	// Unpack the indices from bytes to bits.
	d.sIdxB = d.sIdxB[:0]
	for _, qIdx := range d.sIdxA {
		for idx := 0; idx < 8; idx++ {
			d.sIdxB = append(d.sIdxB, (qIdx<<3)+idx)
		}
	}
	d.sIdxA, d.sIdxB = d.sIdxB, d.sIdxA

	// Filter out indices at which the preamble does not exist.
	for pIdx, pBit := range d.preamble {
		offset := pIdx * symLen
		offsetQuantized := d.Quantized[offset : offset+d.Cfg.BlockSize]
		d.sIdxB, d.sIdxA = searchPass(pBit, offsetQuantized, d.sIdxA, d.sIdxB[:0])

		if len(d.sIdxA) == 0 {
			return nil
		}
	}

	return d.sIdxA
}

func searchPassByte(pBit byte, sig []byte, a, b []int) ([]int, []int) {
	for _, qIdx := range a {
		if sig[qIdx] != pBit {
			b = append(b, qIdx)
		}
	}

	return a, b
}

func searchPass(pBit byte, sig []byte, a, b []int) ([]int, []int) {
	for _, qIdx := range a {
		if sig[qIdx] == pBit {
			b = append(b, qIdx)
		}
	}

	return a, b
}

// Given a list of indices the preamble exists at, sample the appropriate bits
// of the signal's bit-decision. Pack bits of each index into an array of byte
// arrays and return.
func (d Decoder) Slice(indices []int) (pkts [][]byte) {
	// It is likely that a message will be successfully decoded at multiple indices,
	// only keep track of unique instances.
	seen := make(map[string]bool)

	// For each of the indices the preamble exists at.
	for _, qIdx := range indices {
		// Check that we're still within the first sample block. We'll catch
		// the message on the next sample block otherwise.
		if qIdx > d.Cfg.BlockSize {
			continue
		}

		// Packet is 1 bit per byte, pack to 8-bits per byte.
		for pIdx := 0; pIdx < d.Cfg.PacketSymbols; pIdx++ {
			d.pkt[pIdx>>3] <<= 1
			d.pkt[pIdx>>3] |= d.Quantized[qIdx+(pIdx*d.Cfg.SymbolLength)]
		}

		// Store the packet in the seen map and append to the packet list.
		pktStr := fmt.Sprintf("%02X", d.pkt)
		if !seen[pktStr] {
			seen[pktStr] = true
			pkts = append(pkts, make([]byte, len(d.pkt)))
			copy(pkts[len(pkts)-1], d.pkt)
		}
	}

	return
}

func NextPowerOf2(v int) int {
	return 1 << uint(math.Ceil(math.Log2(float64(v))))
}
<|endoftext|>