PropertySort.Alphabetical does not always work

git-svn-id: c3b8989232654601f1acea7403f771adaaa44a81@1237 2c1201b4-01cd-e047-a400-b836ae1fbc61

#region license
// Copyright (c) 2004, Rodrigo B. de Oliveira (rbo@acm.org)
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//     * Neither the name of Rodrigo B. de Oliveira nor the names of its
//     contributors may be used to endorse or promote products derived from this
//     software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endregion

"""
Interactive Forms-based Console
"""

namespace booish.gui

import System
import booish
import System.Windows.Forms
import System.Drawing

class PromptBox(TextBox):

	static Enter = chr(13)

	enum InputState:
		SingleLine = 0
		Block = 1

	_state = InputState.SingleLine

	_block = System.IO.StringWriter()

	[getter(Interpreter)]
	_interpreter = InteractiveInterpreter(
								RememberLastValue: true,
								Print: print)

	def constructor():
		self.Dock = DockStyle.Fill
		self.Multiline = true
		self.AcceptsTab = true
		self.ScrollBars = ScrollBars.Vertical
		_interpreter.References.Add(typeof(TextBox).Assembly)
		_interpreter.References.Add(typeof(Font).Assembly)
		_interpreter.SetValue("inspect", inspect)

		prompt()

	def GetCurrentLine():
		line = Lines[-1][4:]
		print("")
		return line

	def SingleLineInputState():
		code = GetCurrentLine()

		if code[-1:] in ":", "\\":
			_state = InputState.Block
			_block.GetStringBuilder().Length = 0
			_block.WriteLine(code)
		else:
			_interpreter.LoopEval(code)

	def BlockInputState():
		code = GetCurrentLine()
		if 0 == len(code):
			_interpreter.LoopEval(_block.ToString())
			_state = InputState.SingleLine
		else:
			_block.WriteLine(code)

	override def OnKeyPress(args as KeyPressEventArgs):
		if Enter == args.KeyChar:
			try:
				(SingleLineInputState, BlockInputState)[_state]()
			except x:
				print(x)
			prompt()
			args.Handled = true
		super(args)

	def print(msg):
		AppendText("${msg}\r\n")

	def inspect([required] obj):
		f = Form(Text: "Object Inspector [${obj}]")
		f.Controls.Add(PropertyGrid(
							Dock: DockStyle.Fill,
							SelectedObject: obj,
							Font: Font))
		f.Show()
		return f

	def prompt():
		AppendText((">>> ", "... ")[_state])

	static def chr(value as int):
		return cast(IConvertible, value).ToChar(null)

#region license
// Copyright (c) 2004, Rodrigo B. de Oliveira (rbo@acm.org)
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//     * Neither the name of Rodrigo B. de Oliveira nor the names of its
//     contributors may be used to endorse or promote products derived from this
//     software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endregion

"""
Interactive Forms-based Console
"""

namespace booish.gui

import System
import booish
import System.Windows.Forms
import System.Drawing

class PromptBox(TextBox):

	static Enter = chr(13)

	enum InputState:
		SingleLine = 0
		Block = 1

	_state = InputState.SingleLine

	_block = System.IO.StringWriter()

	[getter(Interpreter)]
	_interpreter = InteractiveInterpreter(
								RememberLastValue: true,
								Print: print)

	def constructor():
		self.Dock = DockStyle.Fill
		self.Multiline = true
		self.AcceptsTab = true
		self.ScrollBars = ScrollBars.Vertical
		_interpreter.References.Add(typeof(TextBox).Assembly)
		_interpreter.References.Add(typeof(Font).Assembly)
		_interpreter.SetValue("inspect", inspect)

		prompt()

	def GetCurrentLine():
		line = Lines[-1][4:]
		print("")
		return line

	def SingleLineInputState():
		code = GetCurrentLine()

		if code[-1:] in ":", "\\":
			_state = InputState.Block
			_block.GetStringBuilder().Length = 0
			_block.WriteLine(code)
		else:
			_interpreter.LoopEval(code)

	def BlockInputState():
		code = GetCurrentLine()
		if 0 == len(code):
			_interpreter.LoopEval(_block.ToString())
			_state = InputState.SingleLine
		else:
			_block.WriteLine(code)

	override def OnKeyPress(args as KeyPressEventArgs):
		if Enter == args.KeyChar:
			try:
				(SingleLineInputState, BlockInputState)[_state]()
			except x:
				print(x)
			prompt()
			args.Handled = true
		super(args)

	def print(msg):
		AppendText("${msg}\r\n")

	def inspect([required] obj):
		f = Form(Text: "Object Inspector [${obj}]")
		f.Controls.Add(PropertyGrid(
							Dock: DockStyle.Fill,
							SelectedObject: obj,
							Font: Font,
							PropertySort: PropertySort.Alphabetical))
		f.Show()
		return f

	def prompt():
		AppendText((">>> ", "... ")[_state])

	static def chr(value as int):
		return cast(IConvertible, value).ToChar(null)


booish: read/eval/loop

booish: read/eval/loop

git-svn-id: c3b8989232654601f1acea7403f771adaaa44a81@1085 2c1201b4-01cd-e047-a400-b836ae1fbc61

namespace booish

import Boo.Lang
import Boo.Lang.Compiler
import Boo.Lang.Compiler.Ast
import Boo.Lang.Compiler.TypeSystem
import Boo.Lang.Compiler.IO

class InterpreterEntity(ITypedEntity):

	[getter(Name)]
	[getter(FullName)]
	_name as string

	[getter(Type)]
	_type as IType

	def constructor(name, type):
		_name = name
		_type = type

	EntityType:
		get:
			return TypeSystem.EntityType.Custom

	static def IsInterpreterEntity(node as Node):
		return node.Entity is not null and TypeSystem.EntityType.Custom == node.Entity.EntityType

class InterpreterNamespace(INamespace):

	[getter(ParentNamespace)]
	_parent as INamespace

	_tss as TypeSystemServices

	_interpreter as InteractiveInterpreter

	_declarations = {}

	def constructor(interpreter, tss, parent):
		_interpreter = interpreter
		_tss = tss
		_parent = parent

	def Declare(name as string, type as IType):
		_declarations.Add(name, entity=InterpreterEntity(name, type))
		return entity

	def Resolve(targetList as List, name as string, flags as EntityType) as bool:
		return false unless flags == EntityType.Any

		entity as IEntity = _declarations[name]
		if entity is null:
			value = _interpreter.GetValue(name)
			if value is not null:
				entity = Declare(name, _tss.Map(value.GetType()))

		if entity is not null:
			targetList.Add(entity)
			return true

		return false

	def GetMembers():
		return array(IEntity, 0)

class ProcessVariableDeclarations(Steps.ProcessMethodBodiesWithDuckTyping):

	_namespace as InterpreterNamespace
	_interpreter as InteractiveInterpreter

	def constructor(interpreter):
		_interpreter = interpreter

	override def Initialize(context as CompilerContext):
		super(context)

		_namespace = InterpreterNamespace(
							_interpreter,
							TypeSystemServices,
							NameResolutionService.GlobalNamespace)
		NameResolutionService.GlobalNamespace = _namespace

	override def ProcessAssignment(node as BinaryExpression):
		# prevent 'Expression can't be assigned to' error
		super(node) unless InterpreterEntity.IsInterpreterEntity(node.Left)

	override def DeclareLocal(name as string, type as IType, privateScope as bool):
		return super(name, type, privateScope) if privateScope
		return _namespace.Declare(name, type)

class ProcessInterpreterReferences(Steps.AbstractTransformerCompilerStep):

	static InteractiveInterpreter_GetValue = typeof(InteractiveInterpreter).GetMethod("GetValue")
	static InteractiveInterpreter_SetValue = typeof(InteractiveInterpreter).GetMethod("SetValue")

	_interpreterField as Field
	_namespace as InterpreterNamespace
	_interpreter as InteractiveInterpreter

	def constructor(interpreter):
		_interpreter = interpreter

	override def Run():
		Visit(CompileUnit) if 0 == len(Errors)

	override def OnModule(node as Module):

		module = cast(ModuleEntity, node.Entity).ModuleClass
		return unless module

		_interpreterField = CodeBuilder.CreateField("ParentInterpreter", TypeSystemServices.Map(InteractiveInterpreter))
		_interpreterField.Modifiers = TypeMemberModifiers.Public | TypeMemberModifiers.Static
		module.Members.Add(_interpreterField)

		super(node)

	override def OnReferenceExpression(node as ReferenceExpression):
		super(node)
		return unless InterpreterEntity.IsInterpreterEntity(node) and not AstUtil.IsLhsOfAssignment(node)

		ReplaceCurrentNode(CreateGetValue(node))

	override def LeaveBinaryExpression(node as BinaryExpression):
		if InterpreterEntity.IsInterpreterEntity(node.Left):
			ReplaceCurrentNode(CreateSetValue(node))

	def CreateGetValue(node as ReferenceExpression):
		return CodeBuilder.CreateCast(
				node.ExpressionType,
				CodeBuilder.CreateMethodInvocation(
					CodeBuilder.CreateReference(_interpreterField),
					TypeSystemServices.Map(InteractiveInterpreter_GetValue),
					CodeBuilder.CreateStringLiteral(node.Name)))

	def CreateSetValue(node as BinaryExpression):
		return CodeBuilder.CreateMethodInvocation(
					CodeBuilder.CreateReference(_interpreterField),
					TypeSystemServices.Map(InteractiveInterpreter_SetValue),
					CodeBuilder.CreateStringLiteral(cast(ReferenceExpression, node.Left).Name),
					node.Right)

class InteractiveInterpreter:

	_compiler = BooCompiler()

	_values = {}

	def constructor():
		pipeline = Pipelines.CompileToMemory()
		index = pipeline.Find(Steps.ProcessMethodBodiesWithDuckTyping)
		pipeline[index] = ProcessVariableDeclarations(self)
		pipeline.InsertBefore(Steps.EmitAssembly, ProcessInterpreterReferences(self))
		//pipeline.Add(Steps.PrintBoo())

		_compiler.Parameters.Pipeline = pipeline

	def Eval(code as string):
		_compiler.Parameters.Input.Clear()
		_compiler.Parameters.Input.Add(StringInput("src", code))

		result = _compiler.Run()
		if 0 == len(result.Errors):
			entry = result.GeneratedAssemblyEntryPoint
			entry.DeclaringType.GetField("ParentInterpreter").SetValue(null, self)
			entry.Invoke(null, (null,))
		return result

	def SetValue(name as string, value):
		_values[name] = value

	def GetValue(name as string):
		return _values[name]
/*
interpreter = InteractiveInterpreter()
interpreter.SetValue("name", "boo")
interpreter.SetValue("age", 3)

interpreter.Eval("""
print(name)
print(age)
age += 1
""")

assert 4 == interpreter.GetValue("age")

interpreter.Eval("age = 42")
assert 42 == interpreter.GetValue("age")

interpreter.Eval("""
value = 3
print(value*2)
""")

assert 3 == interpreter.GetValue("value")

interpreter.Eval("x2 = { v as int | return v*2 }")

x2 as callable = interpreter.GetValue("x2")
assert 4 == x2(2)

interpreter.Eval("e = i*2 for i in range(value)")
assert array(interpreter.GetValue("e")) == (0, 2, 4)

# let's loop
interpreter.Eval("""
for i in range(3):
	print(i*2)
""")
print("i: ${interpreter.GetValue('i')}")
*/

interpreter = InteractiveInterpreter()
while line=prompt(">>> "):
	result = interpreter.Eval(line)
	for error in result.Errors:
		print("ERROR: ${error.Message}")

namespace booish

import Boo.Lang
import Boo.Lang.Compiler
import Boo.Lang.Compiler.Ast
import Boo.Lang.Compiler.TypeSystem
import Boo.Lang.Compiler.IO

class InterpreterEntity(ITypedEntity):

	[getter(Name)]
	[getter(FullName)]
	_name as string

	[getter(Type)]
	_type as IType

	def constructor(name, type):
		_name = name
		_type = type

	EntityType:
		get:
			return TypeSystem.EntityType.Custom

	static def IsInterpreterEntity(node as Node):
		return node.Entity is not null and TypeSystem.EntityType.Custom == node.Entity.EntityType

class InterpreterNamespace(INamespace):

	[getter(ParentNamespace)]
	_parent as INamespace

	_tss as TypeSystemServices

	_interpreter as InteractiveInterpreter

	_declarations = {}

	def constructor(interpreter, tss, parent):
		_interpreter = interpreter
		_tss = tss
		_parent = parent

	def Declare(name as string, type as IType):
		_declarations.Add(name, entity=InterpreterEntity(name, type))
		return entity

	def Resolve(targetList as List, name as string, flags as EntityType) as bool:
		return false unless flags == EntityType.Any

		entity as IEntity = _declarations[name]
		if entity is null:
			value = _interpreter.GetValue(name)
			if value is not null:
				entity = Declare(name, _tss.Map(value.GetType()))

		if entity is not null:
			targetList.Add(entity)
			return true

		return false

	def GetMembers():
		return array(IEntity, 0)

class ProcessVariableDeclarations(Steps.ProcessMethodBodiesWithDuckTyping):

	_namespace as InterpreterNamespace
	_interpreter as InteractiveInterpreter

	def constructor(interpreter):
		_interpreter = interpreter

	override def Initialize(context as CompilerContext):
		super(context)

		_namespace = InterpreterNamespace(
							_interpreter,
							TypeSystemServices,
							NameResolutionService.GlobalNamespace)
		NameResolutionService.GlobalNamespace = _namespace

	override def ProcessAssignment(node as BinaryExpression):
		# prevent 'Expression can't be assigned to' error
		super(node) unless InterpreterEntity.IsInterpreterEntity(node.Left)

	override def DeclareLocal(name as string, type as IType, privateScope as bool):
		return super(name, type, privateScope) if privateScope
		return _namespace.Declare(name, type)

class ProcessInterpreterReferences(Steps.AbstractTransformerCompilerStep):

	static InteractiveInterpreter_GetValue = typeof(InteractiveInterpreter).GetMethod("GetValue")
	static InteractiveInterpreter_SetValue = typeof(InteractiveInterpreter).GetMethod("SetValue")

	_interpreterField as Field
	_namespace as InterpreterNamespace
	_interpreter as InteractiveInterpreter

	def constructor(interpreter):
		_interpreter = interpreter

	override def Run():
		Visit(CompileUnit) if 0 == len(Errors)

	override def OnModule(node as Module):

		module = cast(ModuleEntity, node.Entity).ModuleClass
		return unless module

		_interpreterField = CodeBuilder.CreateField("ParentInterpreter", TypeSystemServices.Map(InteractiveInterpreter))
		_interpreterField.Modifiers = TypeMemberModifiers.Public | TypeMemberModifiers.Static
		module.Members.Add(_interpreterField)

		super(node)

	override def OnReferenceExpression(node as ReferenceExpression):
		super(node)
		return unless InterpreterEntity.IsInterpreterEntity(node) and not AstUtil.IsLhsOfAssignment(node)

		ReplaceCurrentNode(CreateGetValue(node))

	override def LeaveBinaryExpression(node as BinaryExpression):
		if InterpreterEntity.IsInterpreterEntity(node.Left):
			ReplaceCurrentNode(CreateSetValue(node))

	def CreateGetValue(node as ReferenceExpression):
		return CodeBuilder.CreateCast(
				node.ExpressionType,
				CodeBuilder.CreateMethodInvocation(
					CodeBuilder.CreateReference(_interpreterField),
					TypeSystemServices.Map(InteractiveInterpreter_GetValue),
					CodeBuilder.CreateStringLiteral(node.Name)))

	def CreateSetValue(node as BinaryExpression):
		return CodeBuilder.CreateMethodInvocation(
					CodeBuilder.CreateReference(_interpreterField),
					TypeSystemServices.Map(InteractiveInterpreter_SetValue),
					CodeBuilder.CreateStringLiteral(cast(ReferenceExpression, node.Left).Name),
					node.Right)

class InteractiveInterpreter:

	_compiler = BooCompiler()

	_values = {}

	def constructor():
		pipeline = Pipelines.CompileToMemory()
		index = pipeline.Find(Steps.ProcessMethodBodiesWithDuckTyping)
		pipeline[index] = ProcessVariableDeclarations(self)
		pipeline.InsertBefore(Steps.EmitAssembly, ProcessInterpreterReferences(self))
		//pipeline.Add(Steps.PrintBoo())

		_compiler.Parameters.Pipeline = pipeline

	def Eval(code as string):
		_compiler.Parameters.Input.Clear()
		_compiler.Parameters.Input.Add(StringInput("src", code))

		result = _compiler.Run()
		assert 0 == len(result.Errors), "\n" + result.Errors.ToString(true)

		entry = result.GeneratedAssemblyEntryPoint
		entry.DeclaringType.GetField("ParentInterpreter").SetValue(null, self)
		entry.Invoke(null, (null,))

	def SetValue(name as string, value):
		_values[name] = value

	def GetValue(name as string):
		return _values[name]

interpreter = InteractiveInterpreter()
interpreter.SetValue("name", "boo")
interpreter.SetValue("age", 3)

interpreter.Eval("""
print(name)
print(age)
age += 1
""")

assert 4 == interpreter.GetValue("age")

interpreter.Eval("age = 42")
assert 42 == interpreter.GetValue("age")

interpreter.Eval("""
value = 3
print(value*2)
""")

assert 3 == interpreter.GetValue("value")

interpreter.Eval("x2 = { v as int | return v*2 }")

x2 as callable = interpreter.GetValue("x2")
assert 4 == x2(2)

interpreter.Eval("e = i*2 for i in range(value)")
assert array(interpreter.GetValue("e")) == (0, 2, 4)

# let's loop
interpreter.Eval("""
for i in range(3):
	print(i*2)
""")
print("i: ${interpreter.GetValue('i')}")


Improved the generator a bit by just checking for uninitialized generators at state 0

namespace BooJs.Compiler.Steps

import System
import Boo.Lang.Compiler.Ast
import Boo.Lang.Compiler.Steps
import Boo.Lang.Compiler.TypeSystem.Internal

class ProcessGenerators(AbstractTransformerCompilerStep):
"""
    Specialized step to process generators to replace Boo's original one.

    TODO: State to run just before this one that converts all loops to while ones
          if the method is a generator
"""
    class TransformGenerator(FastDepthFirstVisitor):
        [getter(States)]
        _states = StatementCollection()

        [property(State)]
        _state as int = 0

        Current as Block:
            get: return States[State]

        _statemachinelabel = ReferenceExpression('statemachine')
        _gotostatemachine = GotoStatement(Label: _statemachinelabel)

        def constructor():
            CreateState()

        def CreateState():
            block = Block()
            States.Add(block)
            return len(States)-1

        def OnExpressionStatement(node as ExpressionStatement):
            Current.Add(node)

        def OnYieldStatement(node as YieldStatement):
            # Create a re-entry state
            nextstate = CreateState()

            # Setup jump to next state and return value
            Current.Add([| __state = $(nextstate) |])
            Current.Add([| return $(node.Expression) |])

            # Continue in the re-entry state
            State = nextstate

        def OnWhileStatement(node as WhileStatement):
            # Loop always starts in a new step
            State = loopstate = CreateState()

            # Create new step for exiting the loop but don't use it yet
            exitstate = CreateState()

            # Check if we should skip the loop by negating the condition
            ifs = [|
                if not $(node.Condition):
                    __state = $(exitstate)
                    $(_gotostatemachine)
            |]
            Current.Add(ifs)

            # Process the loop body
            Visit node.Block

            # Go back to the start of the loop to check if we have more iterations left
            Current.Add([| __state = $(loopstate) |])
            Current.Add(_gotostatemachine)

            # Continue in the exit step previously created
            State = exitstate

        def OnTryStatement(node as TryStatement):
            # A try block always initiates an state
            State = trystate = CreateState()

            # Register ensure block
            if node.EnsureBlock:
                Current.Add([| __final.push( $(BlockExpression(Body: node.EnsureBlock)) ) |])

            Visit node.ProtectedBlock

            # Execute ensure block after we reach the end of the protected block
            if node.EnsureBlock:
                Current.Add( [| __final.pop()() |] )

            # Create a new state for the exception handler
            exceptstate = CreateState()

            # Register the states covered by this protected block
            tryblock = States[trystate] as Block
            for state in range(trystate, exceptstate):
                tryblock.Insert(0, [| __catch[$state] = $exceptstate |] )

            # Create a new state for exiting the try/except block
            afterstate = CreateState()

            # Skip the exception handler state
            Current.Add( [| __state = $afterstate |] )
            Current.Add(_gotostatemachine)

            # Produce the exception handler
            State = exceptstate
            Visit node.ExceptionHandlers

            # Continue in the after state
            State = afterstate

        def Terminate():
            # Create an exit point
            exitstate = CreateState()

            # Direct the current state to the exit point
            Current.Add([| __state = $(exitstate) |])
            Current.Add(_gotostatemachine)

            # Define the exit strategy
            State = exitstate
            Current.Add( [| raise Boo.STOP |] )

    override def Run():
        if len(Errors) > 0:
            return
        Visit CompileUnit

    def HasTryStatement(node as Block) as bool:
        for st in node.Statements:
            return true if st.NodeType == NodeType.TryStatement
            if wst = st as WhileStatement:
                return true if HasTryStatement(wst.Block)
            elif ist = st as IfStatement:
                return true if HasTryStatement(ist.TrueBlock) or HasTryStatement(ist.FalseBlock)
            elif bst = st as Block:
                return true if HasTryStatement(bst)

        return false

    override def EnterMethod(node as Method):
        # Types should be already resolved so we can just check if it was flagged as a generator
        entity = node.Entity as InternalMethod
        return entity and entity.IsGenerator

    override def LeaveMethod(node as Method):

        has_try = HasTryStatement(node.Body)

        # Transform the method body into a list of states
        transformer = TransformGenerator()
        transformer.OnBlock(node.Body)
        transformer.Terminate()

        # If the method is a closure we look for the original method node to declare locals in it
        method = node
        if node.IsSynthetic:
            for member in (node.ParentNode as TypeDefinition).Members:
                if member.LexicalInfo.Equals(node.LexicalInfo):
                    method = member
                    break

        # Remove the original method contents
        node.Body.Clear()
        # Create a new local to keep the current step of the generator
        CodeBuilder.DeclareLocal(method, '__state', TypeSystemServices.IntType)
        if has_try:
            CodeBuilder.DeclareLocal(method, '__catch', TypeSystemServices.HashType)
            CodeBuilder.DeclareLocal(method, '__final', TypeSystemServices.ArrayType)
            node.Body.Add( CodeBuilder.CreateAssignment(ReferenceExpression('__catch'), HashLiteralExpression()) )
            node.Body.Add( CodeBuilder.CreateAssignment(ReferenceExpression('__final'), ListLiteralExpression()) )

        # Add value check to first state
        valuecheck = [|
            if __value is not Boo.UNDEF or __error is not Boo.UNDEF:
                raise TypeError('Generator not started yet, unable to process sent value/error')
        |]
        (transformer.States[0] as Block).Insert(0, valuecheck)

        # Wrap the steps into a switch/case construct
        switch = MacroStatement(Name: 'switch')
        switch.Arguments.Add(ReferenceExpression('__state'))
        for idx as int, step as Block in enumerate(transformer.States):
            case = MacroStatement()
            case.Name = 'case'
            case.Arguments.Add(IntegerLiteralExpression(idx))
            case.Body = step
            switch.Body.Add(case)

        statemachine as Statement
        statemachine = [|
            if __error:
                raise __error

            $switch
        |]

        # Wrap the state machine to support exception handling
        if has_try:
            statemachine = [|
                try:
                    $statemachine
                except:
                    if __e is not Boo.STOP and __state in __catch:
                        __error = null  # Make sure any reported error is removed
                        __state = __catch[__state]
                        goto statemachine

                    # Position the generator to its last state
                    __state = $(len(transformer.States)-1)

                    # Ensure all finally blocks are executed
                    while __final.length:
                        __final.pop()()

                    # Re-raise the exception
                    raise __e
            |]

        # Make the method return a generator
        generator = [|
            return Boo.generator do(__value, __error):
                :statemachine
                $statemachine
        |]
        node.Body.Add(generator)

namespace BooJs.Compiler.Steps

import System
import Boo.Lang.Compiler.Ast
import Boo.Lang.Compiler.Steps
import Boo.Lang.Compiler.TypeSystem.Internal

class ProcessGenerators(AbstractTransformerCompilerStep):
"""
    Specialized step to process generators to replace Boo's original one.

    TODO: State to run just before this one that converts all loops to while ones
          if the method is a generator
"""
    class TransformGenerator(FastDepthFirstVisitor):
        [getter(States)]
        _states = StatementCollection()

        [property(State)]
        _state as int = 0

        Current as Block:
            get: return States[State]

        _statemachinelabel = ReferenceExpression('statemachine')
        _gotostatemachine = GotoStatement(Label: _statemachinelabel)

        def constructor():
            CreateState()

        def CreateState():
            block = Block()
            States.Add(block)
            return len(States)-1

        def OnExpressionStatement(node as ExpressionStatement):
            Current.Add(node)

        def OnYieldStatement(node as YieldStatement):
            # Create a re-entry state
            nextstate = CreateState()

            # Setup jump to next state and return value
            Current.Add([| __state = $(nextstate) |])
            Current.Add([| return $(node.Expression) |])

            # Continue in the re-entry state
            State = nextstate

        def OnWhileStatement(node as WhileStatement):
            # Loop always starts in a new step
            State = loopstate = CreateState()

            # Create new step for exiting the loop but don't use it yet
            exitstate = CreateState()

            # Check if we should skip the loop by negating the condition
            ifs = [|
                if not $(node.Condition):
                    __state = $(exitstate)
                    $(_gotostatemachine)
            |]
            Current.Add(ifs)

            # Process the loop body
            Visit node.Block

            # Go back to the start of the loop to check if we have more iterations left
            Current.Add([| __state = $(loopstate) |])
            Current.Add(_gotostatemachine)

            # Continue in the exit step previously created
            State = exitstate

        def OnTryStatement(node as TryStatement):
            # A try block always initiates an state
            State = trystate = CreateState()

            # Register ensure block
            if node.EnsureBlock:
                Current.Add([| __final.push( $(BlockExpression(Body: node.EnsureBlock)) ) |])

            Visit node.ProtectedBlock

            # Execute ensure block after we reach the end of the protected block
            if node.EnsureBlock:
                Current.Add( [| __final.pop()() |] )

            # Create a new state for the exception handler
            exceptstate = CreateState()

            # Register the states covered by this protected block
            tryblock = States[trystate] as Block
            for state in range(trystate, exceptstate):
                tryblock.Insert(0, [| __catch[$state] = $exceptstate |] )

            # Create a new state for exiting the try/except block
            afterstate = CreateState()

            # Skip the exception handler state
            Current.Add( [| __state = $afterstate |] )
            Current.Add(_gotostatemachine)

            # Produce the exception handler
            State = exceptstate
            Visit node.ExceptionHandlers

            # Continue in the after state
            State = afterstate

        def Terminate():
            # Create an exit point
            exitstate = CreateState()

            # Direct the current state to the exit point
            Current.Add([| __state = $(exitstate) |])
            Current.Add(_gotostatemachine)

            # Define the exit strategy
            State = exitstate
            Current.Add( [| raise Boo.STOP |] )

    override def Run():
        if len(Errors) > 0:
            return
        Visit CompileUnit

    def HasTryStatement(node as Block) as bool:
        for st in node.Statements:
            return true if st.NodeType == NodeType.TryStatement
            if wst = st as WhileStatement:
                return true if HasTryStatement(wst.Block)
            elif ist = st as IfStatement:
                return true if HasTryStatement(ist.TrueBlock) or HasTryStatement(ist.FalseBlock)
            elif bst = st as Block:
                return true if HasTryStatement(bst)

        return false

    override def EnterMethod(node as Method):
        # Types should be already resolved so we can just check if it was flagged as a generator
        entity = node.Entity as InternalMethod
        return entity and entity.IsGenerator

    override def LeaveMethod(node as Method):

        has_try = HasTryStatement(node.Body)

        # Transform the method body into a list of states
        transformer = TransformGenerator()
        transformer.OnBlock(node.Body)
        transformer.Terminate()

        # If the method is a closure we look for the original method node to declare locals in it
        method = node
        if node.IsSynthetic:
            for member in (node.ParentNode as TypeDefinition).Members:
                if member.LexicalInfo.Equals(node.LexicalInfo):
                    method = member
                    break

        # Remove the original method contents
        node.Body.Clear()
        # Create a new local to keep the current step of the generator
        CodeBuilder.DeclareLocal(method, '__state', TypeSystemServices.IntType)
        if has_try:
            CodeBuilder.DeclareLocal(method, '__catch', TypeSystemServices.HashType)
            CodeBuilder.DeclareLocal(method, '__final', TypeSystemServices.ArrayType)
            node.Body.Add( CodeBuilder.CreateAssignment(ReferenceExpression('__catch'), HashLiteralExpression()) )
            node.Body.Add( CodeBuilder.CreateAssignment(ReferenceExpression('__final'), ListLiteralExpression()) )

        # Wrap the steps into a switch/case construct
        switch = MacroStatement(Name: 'switch')
        switch.Arguments.Add(ReferenceExpression('__state'))
        for idx as int, step as Block in enumerate(transformer.States):
            case = MacroStatement()
            case.Name = 'case'
            case.Arguments.Add(IntegerLiteralExpression(idx))
            case.Body = step
            switch.Body.Add(case)

        statemachine as Statement
        statemachine = [|
            #console.log('<STATE: ' + __state + '>')
            if __state == 0 and (__value is not Boo.UNDEF or __error is not Boo.UNDEF):
                raise TypeError('Generator not started yet, unable to process sent value/error')
            if __error:
                raise __error

            $switch
        |]

        # Wrap the state machine to support exception handling
        if has_try:
            statemachine = [|
                try:
                    $statemachine
                except:
                    if __e is not Boo.STOP and __state in __catch:
                        __error = null  # Make sure any reported error is removed
                        __state = __catch[__state]
                        goto statemachine

                    # Position the generator to its last state
                    __state = $(len(transformer.States)-1)

                    # Ensure all finally blocks are executed
                    while __final.length:
                        __final.pop()()

                    # Re-raise the exception
                    raise __e
            |]

        # Make the method return a generator
        generator = [|
            return Boo.generator do(__value, __error):
                :statemachine
                $statemachine
        |]
        node.Body.Add(generator)


fixed tests

fixed tests

"""
1, 3
1, 2
1, 1
2, 3
2, 2
2, 1
"""

items = (x, y) for x in range(1, 3) for y in range(3, 0)
for item in items:
	print join(item, ", ")

"""
1, 3
1, 2
1, 1
2, 3
2, 2
2, 1
"""
items = (x, y) for x in range(1, 3) for y in range(3, 0)
#for item in items:
#	print join(item, ", ")


misleading comments removed

misleading comments removed

git-svn-id: fcce7d6ce0ac15bf24f68871b5145d46fc068279@1908 2c1201b4-01cd-e047-a400-b836ae1fbc61

"""
started
executing
done
"""
import System

def run(message):
	print(message)
	return 42

print("started")

result = run.BeginInvoke("executing", null, null)
assert 42 == run.EndInvoke(result)

print("done")

"""
started
executing
done
"""
import System

def run(message):
	print(message)
	return 42

print("started")

# BeginInvoke is overloaded
# in this example we call the version
# that takes only the original method parameters
result = run.BeginInvoke("executing", null, null)
assert 42 == run.EndInvoke(result)

print("done")


typed array literals; allow assignments in ifs and other places

namespace Boo.OMeta.Parser

import Boo.OMeta
import Boo.PatternMatching

import Boo.Lang.Compiler
import Boo.Lang.Compiler.Ast

import System.Globalization

macro infix:

	l, op, r = infix.Arguments

	return ExpressionStatement([| $l = ((($l >> l, $op >> op, $r >> r) ^ newInfixExpression(op, l, r)) | $r) |])

macro infixr:

	l, op, r = infixr.Arguments

	return ExpressionStatement([| $l = ((($r >> l, $op >> op, $l >> r) ^ newInfixExpression(op, l, r)) | $r) |])

macro prefix:

	rule, op, next = prefix.Arguments

	return ExpressionStatement([| $rule = ($op >> op, $rule >> e) ^ newPrefixExpression(op, e) | $next |])

macro list_of:
"""
Generates rules for lists of the given expression.

	list_of expression

Expands to something that matches:

	expression (COMMA expression)+
"""

	rule, = list_of.Arguments

	block as Block = list_of.ParentNode

	listRuleName = ReferenceExpression(Name: "${rule}_list")
	listRule = [| $listRuleName = ((($rule >> first), ++((COMMA, $rule >> e) ^ e) >> rest) ^ prepend(first, rest)) | ($rule >> v ^ [v]) |]
	block.Add(listRule)

	optionalRuleName = ReferenceExpression(Name: "optional_${rule}_list")
	optionalListRule = [| $optionalRuleName = $listRuleName | ("" ^ []) |]
	block.Add(optionalListRule)

ometa BooParser < WhitespaceSensitiveTokenizer:

	tokens:
		equality = "=="
		inequality = "!="
		assign = "="
		assign_inplace = "+=" | "-=" | "*=" | "/=" | "%=" | "^=" | "&=" | "|="
		xor = "^"
		increment = "++"
		decrement = "--"
		plus = "+"
		minus = "-"
		exponentiation = "**"
		star = "*"
		division = "/"
		modulus = "%"
		ones_complement = "~"
		bitwise_shift_left = "<<"
		bitwise_shift_right = ">>"
		greater_than_eq = ">="
		greater_than = ">"
		less_than_eq = "<="
		less_than = "<"
		bitwise_and = "&"
		bitwise_or = "|"
		hexnum = ("0x", ++(hex_digit | digit) >> ds) ^ makeString(ds)
		num = ++digit
		colon = ":"
		dot = "."
		comma = ","
		lparen = "(", enterWhitespaceAgnosticRegion
		rparen = ")", leaveWhitespaceAgnosticRegion
		lbrack = "[", enterWhitespaceAgnosticRegion
		rbrack = "]", leaveWhitespaceAgnosticRegion
		lbrace = "{", enterWhitespaceAgnosticRegion
		rbrace = "}", leaveWhitespaceAgnosticRegion

		kw = (keywords >> value, ~(letter | digit | '_')) ^ value
		tdq = '"""'
		dq = '"'
		sqs = ("'", ++(~"'", _) >> s, "'") ^ s
		id = ((letter | '_') >> p, --(letter | digit | '_') >> s) ^ makeString(p, s)

	hex_digit = _ >> c as char and ((c >= char('a') and c <= char('f')) or (c >= char('A') and c <= char('Z')))

	keywords "class", "def", "import", "pass", "return", "true", \
		"false", "and", "or", "as", "not", "if", "is", "null", \
		"for", "in", "yield", "self", "super", "of"

	keyword[expected] = ((KW >> t) and (expected is tokenValue(t))) ^ t

	module = (
		--EOL,
		(docstring >> s | ""),
		--import_declaration >> ids,
		--module_member >> members,
		--stmt >> stmts,
		--whitespace
	) ^ newModule(s, ids, members, stmts)

	docstring = (TDQ, ++(~tdq, string_char) >> s, TDQ, eol) ^ makeString(s)

	import_declaration = (IMPORT, qualified_name >> qn, eol) ^ newImport(qn)

	qualified_name = (ID >> qualifier, --((DOT, ID >> n) ^ n) >> suffix)^ buildQName(qualifier, suffix)

	module_member = class_def | method

	class_def = (
		CLASS, ID >> className, begin_block, class_body >> body, end_block
	) ^ newClass(className, body)

	begin_block = COLON, INDENT

	end_block = DEDENT

	class_body = ((PASS, eol) ^ null) | (++class_member >> members ^ members)

	class_member = method | class_def

	method = (
		DEF, ID >> name, LPAREN, optional_parameter_list >> parameters, RPAREN, block >> body
	) ^ newMethod(name, parameters, body)

	list_of parameter

	parameter = ((ID >> name, AS, type_reference >> type) | (ID >> name)) ^ ParameterDeclaration(Name: tokenValue(name), Type: type)

	block = empty_block | non_empty_block

	empty_block = (begin_block, (PASS, eol), end_block) ^ Block()

	non_empty_block = (begin_block, ++stmt >> stmts, end_block)  ^ newBlock(stmts)

	stmt = stmt_block | stmt_line

	stmt_line = (~~(ID, AS), stmt_declaration) \
		| stmt_expression \
		| stmt_macro \
		| stmt_return \
		| stmt_yield

	stmt_macro = (ID >> name, assignment_list >> args, ((block >> b) | (stmt_modifier >> m))) ^ newMacro(name, args, b, m)

	stmt_yield = (YIELD, assignment >> e, stmt_modifier >> m) ^ YieldStatement(Expression: e, Modifier: m)

	stmt_modifier = ((IF, assignment >> e, eol) ^ StatementModifier(Type: StatementModifierType.If, Condition: e)) | (eol ^ null)

	stmt_declaration = (declaration >> d, ((ASSIGN, expression >> e) | ""), eol) ^ newDeclarationStatement(d, e)

	declaration = (ID >> name, ((AS, type_reference >> typeRef) | "")) ^ newDeclaration(name, typeRef)

	stmt_block = stmt_if | stmt_for

	stmt_for = (FOR, declaration_list >> dl, IN, rvalue >> e, block >> body) ^ newForStatement(dl, e, body)

	stmt_if = (IF, assignment >> e, block >> trueBlock) ^ newIfStatement(e, trueBlock)

	stmt_return = (RETURN, optional_assignment >> e, stmt_modifier >> m) ^ ReturnStatement(Expression: e, Modifier: m)

	optional_assignment = assignment | ""

	stmt_expression = ((multi_assignment | assignment) >> e, stmt_modifier >> m) ^ ExpressionStatement(Expression: e, Modifier: m)

	multi_assignment = (expression >> l, ASSIGN >> op, rvalue >> r) ^ newInfixExpression(op, l, r)

	rvalue = assignment_list >> items ^ newRValue(items)

	list_of assignment

	infixr assignment, (ASSIGN | ASSIGN_INPLACE), expression

	expression = or_expression

	infix or_expression, OR, and_expression

	infix and_expression, AND, not_expression

	prefix not_expression, NOT, membership_expression

	infix membership_expression, (IN | ((NOT, IN) ^ makeToken("not in"))), identity_test_expression

	// need to fix memoization for rule with arguments
	infix identity_test_expression, (IS, ((NOT ^ makeToken("is not")) | ("" ^ makeToken("is")))), comparison

	infix comparison, (EQUALITY | INEQUALITY | GREATER_THAN | GREATER_THAN_EQ | LESS_THAN | LESS_THAN_EQ), bitwise_or_expression

	infix bitwise_or_expression, BITWISE_OR, bitwise_xor_expression

	infix bitwise_xor_expression, XOR, bitwise_and_expression

	infix bitwise_and_expression, BITWISE_AND, bitwise_shift_expression

	infix bitwise_shift_expression, (BITWISE_SHIFT_LEFT | BITWISE_SHIFT_RIGHT), term

	infix term, (PLUS | MINUS), factor

	infix factor, (STAR | DIVISION | MODULUS), signalled_expression

	prefix signalled_expression, (MINUS | INCREMENT | DECREMENT), ones_complement_expression

	prefix ones_complement_expression, ONES_COMPLEMENT, exponentiation_expression

	infix exponentiation_expression, EXPONENTIATION, try_cast

	try_cast = ((try_cast >> e, AS, type_reference >> typeRef) ^ TryCastExpression(Target: e, Type: typeRef)) | member_reference

	member_reference = ((member_reference >> e, DOT, ID >> name) ^ newMemberReference(e, name)) | slicing

	slicing = ((member_reference >> e, LBRACK, slice_list >> indices, RBRACK) ^ newSlicing(e, indices)) | invocation

	slice = (
			(
				(COLON ^ OmittedExpression.Default) >> begin,
				(expression | ("" ^ OmittedExpression.Default)) >> end,
				(omitted_expression | "") >> step
			)
			|
			(
				expression >> begin,
				((omitted_expression >> end,
					((omitted_expression >> step) | ""))
				| "")
			)
		) ^ newSlice(begin, end, step)

	list_of slice

	omitted_expression = (COLON, expression) | (COLON ^ OmittedExpression.Default)

	invocation = ((member_reference >> target, LPAREN, optional_assignment_list >> args, RPAREN) ^ newInvocation(target, args)) \
		| atom

	type_reference = type_reference_simple | type_reference_array

	type_reference_array = (LPAREN, ranked_type_reference >> tr, RPAREN) ^ tr

	type_reference_simple = (qualified_name >> qname) ^ SimpleTypeReference(Name: qname)

	atom = integer | boolean | reference | array_literal | list_literal \
		| string_interpolation | string_literal | null_literal | parenthesized_expression  \
		| self_literal | super_literal

	parenthesized_expression = (LPAREN, assignment >> e, RPAREN) ^ e

	null_literal = NULL ^ [| null |]

	super_literal = SUPER ^ [| super |]

	self_literal = SELF ^ [| self |]

	string_literal = (SQS >> s) ^ newStringLiteral(s)

	string_interpolation = (
		DQ,
		++(
			((++(~('"' | '$'), string_char) >> s) ^ StringLiteralExpression(makeString(s)))
			| (('${', expression >> v, --whitespace, '}') ^ v)
			| ('$', atom)
			) >> items,
		DQ) ^ newStringInterpolation(items)

	string_char = ('\\', ('\\' | '$')) | (~'\\', _)

	array_literal = array_literal_empty | array_literal_single | array_literal_multi

	array_literal_empty = (LPAREN, array_literal_type >> type, COMMA, RPAREN) ^ newArrayLiteral(type, [])

	array_literal_single = (LPAREN, array_literal_type >> type, assignment >> e, COMMA, RPAREN) ^ newArrayLiteral(type, [e])

	array_literal_multi = (LPAREN, array_literal_type >> type, assignment >> e, ++(COMMA, assignment) >> tail, (COMMA | ""), RPAREN) ^ newArrayLiteral(type, prepend(e, tail))

	array_literal_type = ((OF, ranked_type_reference >> type, COLON) | "") ^ type

	ranked_type_reference = ((type_reference >> type), ((COMMA,  integer >> rank) | "")) ^ ArrayTypeReference(ElementType: type, Rank: rank)

	list_literal = (LBRACK, optional_expression_list >> items, RBRACK) ^ newListLiteral(items)

	list_of expression

	list_of declaration

	reference = ID >> r ^ newReference(r)

	integer = (NUM >> n ^ newInteger(n, NumberStyles.None)) \
		| (HEXNUM >> n ^ newInteger(n, NumberStyles.HexNumber))

	boolean = true_literal | false_literal

	true_literal = TRUE ^ [| true |]

	false_literal = FALSE ^ [| false |]

	eol = ++EOL | ~_

	def newMacro(name, args, body, m):
		node = MacroStatement(Name: tokenValue(name), Block: body, Modifier: m)
		for arg in args: node.Arguments.Add(arg)
		return node

	def newSlicing(target as Expression, slices):
		node = SlicingExpression(Target: target)
		for slice in slices: node.Indices.Add(slice)
		return node

	def newSlice(begin as Expression, end as Expression, step as Expression):
		return Slice(begin, end, step)

	def newRValue(items as List):
		if len(items) > 1: return newArrayLiteral(items)
		return items[0]

	def newForStatement(declarations, e as Expression, body as Block):
		node = ForStatement(Iterator: e, Block: body)
		for d in declarations: node.Declarations.Add(d)
		return node

	def newDeclaration(name, type as TypeReference):
		return Declaration(Name: tokenValue(name), Type: type)

	def newDeclarationStatement(d as Declaration,  initializer as Expression):
		return DeclarationStatement(Declaration: d, Initializer: initializer)

	def newIfStatement(condition as Expression, trueBlock as Block):
		return IfStatement(Condition: condition, TrueBlock: trueBlock)

	def newModule(doc, imports, members, stmts):
		m = Module(Documentation: doc)
		for item in imports: m.Imports.Add(item)
		for member in members: m.Members.Add(member)
		for stmt as Statement in stmts: m.Globals.Add(stmt)
		return m

	def newImport(qname as string):
		return Import(Namespace: qname)

	def newInteger(t, style as NumberStyles):
		value = int.Parse(tokenValue(t), style)
		return IntegerLiteralExpression(Value: value)

	def newMethod(name, parameters, body as Block):
		node = Method(Name: tokenValue(name), Body: body)
		for p in parameters: node.Parameters.Add(p)
		return node

	def newClass(name, members):
		klass = ClassDefinition(Name: tokenValue(name))
		if members is not null:
			for member in members: klass.Members.Add(member)
		return klass

	def newInvocation(target as Expression, args as List):
		mie = MethodInvocationExpression(Target: target)
		for arg in args: mie.Arguments.Add(arg)
		return mie

	def newReference(t):
		return ReferenceExpression(Name: tokenValue(t))

	def newMemberReference(target as Expression, name):
		return MemberReferenceExpression(Target: target, Name: tokenValue(name))

	def newArrayLiteral(type, items):
		node = newArrayLiteral(items)
		node.Type = type
		return node

	def newArrayLiteral(items):
		literal = ArrayLiteralExpression()
		for item in items:
			literal.Items.Add(item)
		return literal

	def newListLiteral(items):
		literal = ListLiteralExpression()
		for item in items: literal.Items.Add(item)
		return literal

	def newStringLiteral(s):
		return StringLiteralExpression(Value: tokenValue(s))

	def newStringInterpolation(items as List):
		if len(items) == 1 and items[0] isa StringLiteralExpression:
			return items[0]
		node = ExpressionInterpolationExpression()
		for item in items: node.Expressions.Add(item)
		return node

	def newInfixExpression(op, l as Expression, r as Expression):
		return BinaryExpression(Operator: binaryOperatorFor(op), Left: l, Right: r)

	def newPrefixExpression(op, e as Expression):
		return UnaryExpression(Operator: unaryOperatorFor(op), Operand: e)

	def unaryOperatorFor(op):
		match tokenValue(op):
			case "not": return UnaryOperatorType.LogicalNot
			case "-": return UnaryOperatorType.UnaryNegation
			case "~": return UnaryOperatorType.OnesComplement
			case "++": return UnaryOperatorType.Increment
			case "--": return UnaryOperatorType.Decrement

	def binaryOperatorFor(op):
		match tokenValue(op):
			case "is": return BinaryOperatorType.ReferenceEquality
			case "is not": return BinaryOperatorType.ReferenceInequality
			case "in": return BinaryOperatorType.Member
			case "not in": return BinaryOperatorType.NotMember
			case "and": return BinaryOperatorType.And
			case "or": return BinaryOperatorType.Or
			case "|": return BinaryOperatorType.BitwiseOr
			case "&": return BinaryOperatorType.BitwiseAnd
			case "^": return BinaryOperatorType.ExclusiveOr
			case "+": return BinaryOperatorType.Addition
			case "-": return BinaryOperatorType.Subtraction
			case "*": return BinaryOperatorType.Multiply
			case "**": return BinaryOperatorType.Exponentiation
			case "/": return BinaryOperatorType.Division
			case "%": return BinaryOperatorType.Modulus
			case "=": return BinaryOperatorType.Assign
			case "==": return BinaryOperatorType.Equality
			case "!=": return BinaryOperatorType.Inequality
			case "+=": return BinaryOperatorType.InPlaceAddition
			case "-=": return BinaryOperatorType.InPlaceSubtraction
			case "/=": return BinaryOperatorType.InPlaceDivision
			case "*=": return BinaryOperatorType.InPlaceMultiply
			case "^=": return BinaryOperatorType.InPlaceExclusiveOr
			case "&=": return BinaryOperatorType.InPlaceBitwiseAnd
			case "|=": return BinaryOperatorType.InPlaceBitwiseOr
			case ">>": return BinaryOperatorType.ShiftRight
			case "<<": return BinaryOperatorType.ShiftLeft
			case "<": return BinaryOperatorType.LessThan
			case "<=": return BinaryOperatorType.LessThanOrEqual
			case ">": return BinaryOperatorType.GreaterThan
			case ">=": return BinaryOperatorType.GreaterThanOrEqual

	def newAssignment(l as Expression, r as Expression):
		return [| $l = $r |]

	def newBlock(stmts):
		b = Block()
		for item in stmts:
			b.Statements.Add(item)
		return b

	def prepend(first, tail as List):
		if first is null: return tail
		return [first] + tail

	def buildQName(q, rest):
		return join(tokenValue(t) for t in prepend(q, rest), '.')

namespace Boo.OMeta.Parser

import Boo.OMeta
import Boo.PatternMatching

import Boo.Lang.Compiler
import Boo.Lang.Compiler.Ast

import System.Globalization

macro infix:

	l, op, r = infix.Arguments

	return ExpressionStatement([| $l = ((($l >> l, $op >> op, $r >> r) ^ newInfixExpression(op, l, r)) | $r) |])

macro infixr:

	l, op, r = infixr.Arguments

	return ExpressionStatement([| $l = ((($r >> l, $op >> op, $l >> r) ^ newInfixExpression(op, l, r)) | $r) |])

macro prefix:

	rule, op, next = prefix.Arguments

	return ExpressionStatement([| $rule = ($op >> op, $rule >> e) ^ newPrefixExpression(op, e) | $next |])

macro list_of:
"""
Generates rules for lists of the given expression.

	list_of expression

Expands to something that matches:

	expression (COMMA expression)+
"""

	rule, = list_of.Arguments

	block as Block = list_of.ParentNode

	listRuleName = ReferenceExpression(Name: "${rule}_list")
	listRule = [| $listRuleName = ((($rule >> first), ++((COMMA, $rule >> e) ^ e) >> rest) ^ prepend(first, rest)) | ($rule >> v ^ [v]) |]
	block.Add(listRule)

	optionalRuleName = ReferenceExpression(Name: "optional_${rule}_list")
	optionalListRule = [| $optionalRuleName = $listRuleName | ("" ^ []) |]
	block.Add(optionalListRule)

ometa BooParser < WhitespaceSensitiveTokenizer:

	tokens:
		equality = "=="
		inequality = "!="
		assign = "="
		assign_inplace = "+=" | "-=" | "*=" | "/=" | "%=" | "^=" | "&=" | "|="
		xor = "^"
		increment = "++"
		decrement = "--"
		plus = "+"
		minus = "-"
		exponentiation = "**"
		star = "*"
		division = "/"
		modulus = "%"
		ones_complement = "~"
		bitwise_shift_left = "<<"
		bitwise_shift_right = ">>"
		greater_than_eq = ">="
		greater_than = ">"
		less_than_eq = "<="
		less_than = "<"
		bitwise_and = "&"
		bitwise_or = "|"
		hexnum = ("0x", ++(hex_digit | digit) >> ds) ^ makeString(ds)
		num = ++digit
		colon = ":"
		dot = "."
		comma = ","
		lparen = "(", enterWhitespaceAgnosticRegion
		rparen = ")", leaveWhitespaceAgnosticRegion
		lbrack = "[", enterWhitespaceAgnosticRegion
		rbrack = "]", leaveWhitespaceAgnosticRegion
		lbrace = "{", enterWhitespaceAgnosticRegion
		rbrace = "}", leaveWhitespaceAgnosticRegion

		kw = (keywords >> value, ~(letter | digit | '_')) ^ value
		tdq = '"""'
		dq = '"'
		sqs = ("'", ++(~"'", _) >> s, "'") ^ s
		id = ((letter | '_') >> p, --(letter | digit | '_') >> s) ^ makeString(p, s)

	hex_digit = _ >> c as char and ((c >= char('a') and c <= char('f')) or (c >= char('A') and c <= char('Z')))

	keywords "class", "def", "import", "pass", "return", "true", \
		"false", "and", "or", "as", "not", "if", "is", "null", \
		"for", "in", "yield", "self", "super"

	keyword[expected] = ((KW >> t) and (expected is tokenValue(t))) ^ t

	module = (
		--EOL,
		(docstring >> s | ""),
		--import_declaration >> ids,
		--module_member >> members,
		--stmt >> stmts,
		--whitespace
	) ^ newModule(s, ids, members, stmts)

	docstring = (TDQ, ++(~tdq, string_char) >> s, TDQ, eol) ^ makeString(s)

	import_declaration = (IMPORT, qualified_name >> qn, eol) ^ newImport(qn)

	qualified_name = (ID >> qualifier, --((DOT, ID >> n) ^ n) >> suffix)^ buildQName(qualifier, suffix)

	module_member = class_def | method

	class_def = (
		CLASS, ID >> className, begin_block, class_body >> body, end_block
	) ^ newClass(className, body)

	begin_block = COLON, INDENT

	end_block = DEDENT

	class_body = ((PASS, eol) ^ null) | (++class_member >> members ^ members)

	class_member = method | class_def

	method = (
		DEF, ID >> name, LPAREN, optional_parameter_list >> parameters, RPAREN, block >> body
	) ^ newMethod(name, parameters, body)

	list_of parameter

	parameter = ((ID >> name, AS, type_reference >> type) | (ID >> name)) ^ ParameterDeclaration(Name: tokenValue(name), Type: type)

	block = empty_block | non_empty_block

	empty_block = (begin_block, (PASS, eol), end_block) ^ Block()

	non_empty_block = (begin_block, ++stmt >> stmts, end_block)  ^ newBlock(stmts)

	stmt = stmt_block | stmt_line

	stmt_line = (~~(ID, AS), stmt_declaration) \
		| stmt_expression \
		| stmt_macro \
		| stmt_return \
		| stmt_yield

	stmt_macro = (ID >> name, assignment_list >> args, ((block >> b) | (stmt_modifier >> m))) ^ newMacro(name, args, b, m)

	stmt_yield = (YIELD, assignment >> e, stmt_modifier >> m) ^ YieldStatement(Expression: e, Modifier: m)

	stmt_modifier = ((IF, assignment >> e, eol) ^ StatementModifier(Type: StatementModifierType.If, Condition: e)) | (eol ^ null)

	stmt_declaration = (declaration >> d, ((ASSIGN, expression >> e) | ""), eol) ^ newDeclarationStatement(d, e)

	declaration = (ID >> name, ((AS, type_reference >> typeRef) | "")) ^ newDeclaration(name, typeRef)

	stmt_block = stmt_if | stmt_for

	stmt_for = (FOR, declaration_list >> dl, IN, expression >> e, block >> body) ^ newForStatement(dl, e, body)

	stmt_if = (IF, expression >> e, block >> trueBlock) ^ newIfStatement(e, trueBlock)

	stmt_return = (RETURN, optional_expression >> e, stmt_modifier >> m) ^ ReturnStatement(Expression: e, Modifier: m)

	optional_expression = expression | ""

	stmt_expression = ((multi_assignment | assignment) >> e, stmt_modifier >> m) ^ ExpressionStatement(Expression: e, Modifier: m)

	multi_assignment = (expression >> l, ASSIGN >> op, assignment_list >> items) ^ newInfixExpression(op, l, newRValue(items))

	list_of assignment

	infixr assignment, (ASSIGN | ASSIGN_INPLACE), expression

	expression = or_expression

	infix or_expression, OR, and_expression

	infix and_expression, AND, not_expression

	prefix not_expression, NOT, membership_expression

	infix membership_expression, (IN | ((NOT, IN) ^ makeToken("not in"))), identity_test_expression

	// need to fix memoization for rule with arguments
	infix identity_test_expression, (IS, ((NOT ^ makeToken("is not")) | ("" ^ makeToken("is")))), comparison

	infix comparison, (EQUALITY | INEQUALITY | GREATER_THAN | GREATER_THAN_EQ | LESS_THAN | LESS_THAN_EQ), bitwise_or_expression

	infix bitwise_or_expression, BITWISE_OR, bitwise_xor_expression

	infix bitwise_xor_expression, XOR, bitwise_and_expression

	infix bitwise_and_expression, BITWISE_AND, bitwise_shift_expression

	infix bitwise_shift_expression, (BITWISE_SHIFT_LEFT | BITWISE_SHIFT_RIGHT), term

	infix term, (PLUS | MINUS), factor

	infix factor, (STAR | DIVISION | MODULUS), signalled_expression

	prefix signalled_expression, (MINUS | INCREMENT | DECREMENT), ones_complement_expression

	prefix ones_complement_expression, ONES_COMPLEMENT, exponentiation_expression

	infix exponentiation_expression, EXPONENTIATION, try_cast

	try_cast = ((try_cast >> e, AS, type_reference >> typeRef) ^ TryCastExpression(Target: e, Type: typeRef)) | member_reference

	member_reference = ((member_reference >> e, DOT, ID >> name) ^ newMemberReference(e, name)) | slicing

	slicing = ((member_reference >> e, LBRACK, slice_list >> indices, RBRACK) ^ newSlicing(e, indices)) | invocation

	slice = (
			(
				(COLON ^ OmittedExpression.Default) >> begin,
				(expression | ("" ^ OmittedExpression.Default)) >> end,
				(omitted_expression | "") >> step
			)
			|
			(
				expression >> begin,
				((omitted_expression >> end,
					((omitted_expression >> step) | ""))
				| "")
			)
		) ^ newSlice(begin, end, step)

	list_of slice

	omitted_expression = (COLON, expression) | (COLON ^ OmittedExpression.Default)

	invocation = ((member_reference >> target, LPAREN, optional_expression_list >> args, RPAREN) ^ newInvocation(target, args)) \
		| atom

	type_reference = type_reference_simple

	type_reference_simple = (qualified_name >> qname) ^ SimpleTypeReference(Name: qname)

	atom = integer | boolean | reference | array_literal | list_literal \
		| string_interpolation | string_literal | null_literal | parenthesized_expression  \
		| self_literal | super_literal

	parenthesized_expression = (LPAREN, assignment >> e, RPAREN) ^ e

	null_literal = NULL ^ [| null |]

	super_literal = SUPER ^ [| super |]

	self_literal = SELF ^ [| self |]

	string_literal = (SQS >> s) ^ newStringLiteral(s)

	string_interpolation = (
		DQ,
		++(
			((++(~('"' | '$'), string_char) >> s) ^ StringLiteralExpression(makeString(s)))
			| (('${', expression >> v, --whitespace, '}') ^ v)
			| ('$', atom)
			) >> items,
		DQ) ^ newStringInterpolation(items)

	string_char = ('\\', ('\\' | '$')) | (~'\\', _)

	array_literal = array_literal_empty | array_literal_single | array_literal_multi

	array_literal_empty = (LPAREN, COMMA, RPAREN) ^ newArrayLiteral([])

	array_literal_single = (LPAREN, assignment >> e, COMMA, RPAREN) ^ newArrayLiteral([e])

	array_literal_multi = (LPAREN, (assignment >> e, ++(COMMA, assignment) >> tail), (COMMA | ""), RPAREN) ^ newArrayLiteral(prepend(e, tail))

	list_literal = (LBRACK, optional_expression_list >> items, RBRACK) ^ newListLiteral(items)

	list_of expression

	list_of declaration

	reference = ID >> r ^ newReference(r)

	integer = (NUM >> n ^ newInteger(n, NumberStyles.None)) \
		| (HEXNUM >> n ^ newInteger(n, NumberStyles.HexNumber))

	boolean = true_literal | false_literal

	true_literal = TRUE ^ [| true |]

	false_literal = FALSE ^ [| false |]

	eol = ++EOL | ~_

	def newMacro(name, args, body, m):
		node = MacroStatement(Name: tokenValue(name), Block: body, Modifier: m)
		for arg in args: node.Arguments.Add(arg)
		return node

	def newSlicing(target as Expression, slices):
		node = SlicingExpression(Target: target)
		for slice in slices: node.Indices.Add(slice)
		return node

	def newSlice(begin as Expression, end as Expression, step as Expression):
		return Slice(begin, end, step)

	def newRValue(items as List):
		if len(items) > 1: return newArrayLiteral(items)
		return items[0]

	def newForStatement(declarations, e as Expression, body as Block):
		node = ForStatement(Iterator: e, Block: body)
		for d in declarations: node.Declarations.Add(d)
		return node

	def newDeclaration(name, type as TypeReference):
		return Declaration(Name: tokenValue(name), Type: type)

	def newDeclarationStatement(d as Declaration,  initializer as Expression):
		return DeclarationStatement(Declaration: d, Initializer: initializer)

	def newIfStatement(condition as Expression, trueBlock as Block):
		return IfStatement(Condition: condition, TrueBlock: trueBlock)

	def newModule(doc, imports, members, stmts):
		m = Module(Documentation: doc)
		for item in imports: m.Imports.Add(item)
		for member in members: m.Members.Add(member)
		for stmt as Statement in stmts: m.Globals.Add(stmt)
		return m

	def newImport(qname as string):
		return Import(Namespace: qname)

	def newInteger(t, style as NumberStyles):
		value = int.Parse(tokenValue(t), style)
		return IntegerLiteralExpression(Value: value)

	def newMethod(name, parameters, body as Block):
		node = Method(Name: tokenValue(name), Body: body)
		for p in parameters: node.Parameters.Add(p)
		return node

	def newClass(name, members):
		klass = ClassDefinition(Name: tokenValue(name))
		if members is not null:
			for member in members: klass.Members.Add(member)
		return klass

	def newInvocation(target as Expression, args as List):
		mie = MethodInvocationExpression(Target: target)
		for arg in args: mie.Arguments.Add(arg)
		return mie

	def newReference(t):
		return ReferenceExpression(Name: tokenValue(t))

	def newMemberReference(target as Expression, name):
		return MemberReferenceExpression(Target: target, Name: tokenValue(name))

	def newArrayLiteral(items):
		literal = ArrayLiteralExpression()
		for item in items:
			literal.Items.Add(item)
		return literal

	def newListLiteral(items):
		literal = ListLiteralExpression()
		for item in items: literal.Items.Add(item)
		return literal

	def newStringLiteral(s):
		return StringLiteralExpression(Value: tokenValue(s))

	def newStringInterpolation(items as List):
		if len(items) == 1 and items[0] isa StringLiteralExpression:
			return items[0]
		node = ExpressionInterpolationExpression()
		for item in items: node.Expressions.Add(item)
		return node

	def newInfixExpression(op, l as Expression, r as Expression):
		return BinaryExpression(Operator: binaryOperatorFor(op), Left: l, Right: r)

	def newPrefixExpression(op, e as Expression):
		return UnaryExpression(Operator: unaryOperatorFor(op), Operand: e)

	def unaryOperatorFor(op):
		match tokenValue(op):
			case "not": return UnaryOperatorType.LogicalNot
			case "-": return UnaryOperatorType.UnaryNegation
			case "~": return UnaryOperatorType.OnesComplement
			case "++": return UnaryOperatorType.Increment
			case "--": return UnaryOperatorType.Decrement

	def binaryOperatorFor(op):
		match tokenValue(op):
			case "is": return BinaryOperatorType.ReferenceEquality
			case "is not": return BinaryOperatorType.ReferenceInequality
			case "in": return BinaryOperatorType.Member
			case "not in": return BinaryOperatorType.NotMember
			case "and": return BinaryOperatorType.And
			case "or": return BinaryOperatorType.Or
			case "|": return BinaryOperatorType.BitwiseOr
			case "&": return BinaryOperatorType.BitwiseAnd
			case "^": return BinaryOperatorType.ExclusiveOr
			case "+": return BinaryOperatorType.Addition
			case "-": return BinaryOperatorType.Subtraction
			case "*": return BinaryOperatorType.Multiply
			case "**": return BinaryOperatorType.Exponentiation
			case "/": return BinaryOperatorType.Division
			case "%": return BinaryOperatorType.Modulus
			case "=": return BinaryOperatorType.Assign
			case "==": return BinaryOperatorType.Equality
			case "!=": return BinaryOperatorType.Inequality
			case "+=": return BinaryOperatorType.InPlaceAddition
			case "-=": return BinaryOperatorType.InPlaceSubtraction
			case "/=": return BinaryOperatorType.InPlaceDivision
			case "*=": return BinaryOperatorType.InPlaceMultiply
			case "^=": return BinaryOperatorType.InPlaceExclusiveOr
			case "&=": return BinaryOperatorType.InPlaceBitwiseAnd
			case "|=": return BinaryOperatorType.InPlaceBitwiseOr
			case ">>": return BinaryOperatorType.ShiftRight
			case "<<": return BinaryOperatorType.ShiftLeft
			case "<": return BinaryOperatorType.LessThan
			case "<=": return BinaryOperatorType.LessThanOrEqual
			case ">": return BinaryOperatorType.GreaterThan
			case ">=": return BinaryOperatorType.GreaterThanOrEqual

	def newAssignment(l as Expression, r as Expression):
		return [| $l = $r |]

	def newBlock(stmts):
		b = Block()
		for item in stmts:
			b.Statements.Add(item)
		return b

	def prepend(first, tail as List):
		if first is null: return tail
		return [first] + tail

	def buildQName(q, rest):
		return join(tokenValue(t) for t in prepend(q, rest), '.')