New transformers near completion

Nearley tool still needs fixing

examples/calc.py

@@ -2,7 +2,7 @@
 # This example shows how to write a basic calculator with variables.
 #
 
-from lark import Lark, Transformer, children_args_inline
+from lark import Lark, Transformer, visitor_args
 
 try:
     input = raw_input   # For Python2 compatibility
@@ -34,7 +34,8 @@ calc_grammar = """
     %ignore WS_INLINE
 """
 
-class CalculateTree(SimpleTransformer):
+@visitor_args(inline=True)
+class CalculateTree(Transformer):
     from operator import add, sub, mul, truediv as div, neg
     number = float
 

examples/json_parser.py

@@ -7,7 +7,7 @@
 
 import sys
 
-from lark import Lark, inline_args, Transformer
+from lark import Lark, Transformer, visitor_args
 
 json_grammar = r"""
     ?start: value
@@ -34,14 +34,14 @@ json_grammar = r"""
 """
 
 class TreeToJson(Transformer):
-    @inline_args
+    @visitor_args(inline=True)
     def string(self, s):
         return s[1:-1].replace('\\"', '"')
 
     array = list
     pair = tuple
     object = dict
-    number = inline_args(float)
+    number = visitor_args(inline=True)(float)
 
     null = lambda self, _: None
     true = lambda self, _: True

lark/__init__.py

@@ -1,5 +1,6 @@
 from .tree import Tree
-from .visitors import Transformer, Visitor, children_args, children_args_inline
+from .visitors import Transformer, Visitor, visitor_args, Discard
+from .visitors import InlineTransformer, inline_args   # XXX Deprecated
 from .common import ParseError, GrammarError, UnexpectedToken
 from .lexer import UnexpectedInput, LexError
 from .lark import Lark

lark/load_grammar.py

@@ -16,7 +16,7 @@ from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol
 from .utils import classify, suppress
 
 from .tree import Tree, SlottedTree as ST
-from .visitors import Transformer, Visitor, children_args, children_args_inline
+from .visitors import Transformer, Visitor, visitor_args
 
 __path__ = os.path.dirname(__file__)
 IMPORT_PATHS = [os.path.join(__path__, 'grammars')]
@@ -138,7 +138,7 @@ RULES = {
 }
 
 
-@children_args_inline
+@visitor_args(inline=True)
 class EBNF_to_BNF(Transformer):
     def __init__(self):
         self.new_rules = []
@@ -232,7 +232,6 @@ class SimplifyRule_Visitor(Visitor):
         tree.children = list(set(tree.children))
 
 
-@children_args
 class RuleTreeToText(Transformer):
     def expansions(self, x):
         return x
@@ -244,7 +243,7 @@ class RuleTreeToText(Transformer):
         return expansion, alias.value
 
 
-@children_args_inline
+@visitor_args(inline=True)
 class CanonizeTree(Transformer):
     def maybe(self, expr):
         return ST('expr', [expr, Token('OP', '?', -1)])
@@ -265,7 +264,7 @@ class PrepareAnonTerminals(Transformer):
         self.i = 0
 
 
-    @children_args_inline
+    @visitor_args(inline=True)
     def pattern(self, p):
         value = p.value
         if p in self.token_reverse and p.flags != self.token_reverse[p].pattern.flags:
@@ -355,7 +354,7 @@ def _literal_to_pattern(literal):
              'REGEXP': PatternRE }[literal.type](s, flags)
 
 
-@children_args_inline
+@visitor_args(inline=True)
 class PrepareLiterals(Transformer):
     def literal(self, literal):
         return ST('pattern', [_literal_to_pattern(literal)])
@@ -369,7 +368,6 @@ class PrepareLiterals(Transformer):
         return ST('pattern', [PatternRE(regexp)])
 
 
-@children_args
 class TokenTreeToPattern(Transformer):
     def pattern(self, ps):
         p ,= ps
@@ -410,7 +408,6 @@ class TokenTreeToPattern(Transformer):
         return v[0]
 
 class PrepareSymbols(Transformer):
-    @children_args
     def value(self, v):
         v ,= v
         if isinstance(v, Tree):
@@ -535,7 +532,7 @@ def options_from_rule(name, *x):
 def symbols_from_strcase(expansion):
     return [Terminal(x, filter_out=x.startswith('_')) if is_terminal(x) else NonTerminal(x) for x in expansion]
 
-@children_args_inline
+@visitor_args(inline=True)
 class PrepareGrammar(Transformer):
     def terminal(self, name):
         return name

lark/parse_tree_builder.py

@@ -3,9 +3,10 @@ from .utils import suppress
 from .lexer import Token
 from .grammar import Rule
 from .tree import Tree
+from .visitors import InlineTransformer # XXX Deprecated
 
 ###{standalone
-from functools import partial
+from functools import partial, wraps
 
 
 class ExpandSingleChild:
@@ -95,6 +96,15 @@ def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous):
 class Callback(object):
     pass
 
+
+def inline_args(func):
+    @wraps(func)
+    def f(children):
+        return func(*children)
+    return f
+
+
+
 class ParseTreeBuilder:
     def __init__(self, rules, tree_class, propagate_positions=False, keep_all_tokens=False, ambiguous=False):
         self.tree_class = tree_class
@@ -130,6 +140,10 @@ class ParseTreeBuilder:
             user_callback_name = rule.alias or rule.origin.name
             try:
                 f = getattr(transformer, user_callback_name)
+                assert not getattr(f, 'meta', False), "Meta args not supported for internal transformer"
+                # XXX InlineTransformer is deprecated!
+                if getattr(f, 'inline', False) or isinstance(transformer, InlineTransformer):
+                    f = inline_args(f)
             except AttributeError:
                 f = partial(self.tree_class, user_callback_name)
 

lark/parsers/earley.py

@@ -14,7 +14,7 @@
 # Email : erezshin@gmail.com
 
 from ..tree import Tree
-from ..visitors import Transformer_InPlace
+from ..visitors import Transformer_InPlace, visitor_args
 from ..common import ParseError, UnexpectedToken
 from .grammar_analysis import GrammarAnalyzer
 from ..grammar import NonTerminal
@@ -114,9 +114,9 @@ class Column:
 
                     if old_tree.data != '_ambig':
                         new_tree = old_tree.copy()
-                        new_tree.rule = old_tree.rule
+                        new_tree.meta.rule = old_tree.meta.rule
                         old_tree.set('_ambig', [new_tree])
-                        old_tree.rule = None    # No longer a 'drv' node
+                        old_tree.meta.rule = None    # No longer a 'drv' node
 
                     if item.tree.children[0] is old_tree:   # XXX a little hacky!
                         raise ParseError("Infinite recursion in grammar! (Rule %s)" % item.rule)
@@ -234,5 +234,6 @@ class ApplyCallbacks(Transformer_InPlace):
     def __init__(self, postprocess):
         self.postprocess = postprocess
 
-    def drv(self, tree):
-        return self.postprocess[tree.meta.rule](tree.children)
+    @visitor_args(meta=True)
+    def drv(self, children, meta):
+        return self.postprocess[meta.rule](children)

lark/parsers/resolve_ambig.py

@@ -26,8 +26,15 @@ def _compare_priority(tree1, tree2):
     tree1.iter_subtrees()
 
 def _compare_drv(tree1, tree2):
-    rule1 = getattr(tree1.meta, 'rule', None)
-    rule2 = getattr(tree2.meta, 'rule', None)
+    try:
+        rule1 = tree1.meta.rule
+    except AttributeError:
+        rule1 = None
+
+    try:
+        rule2 = tree2.meta.rule
+    except AttributeError:
+        rule2 = None
 
     if None == rule1 == rule2:
         return compare(tree1, tree2)

lark/tree.py

@@ -10,10 +10,10 @@ class Meta:
 
 ###{standalone
 class Tree(object):
-    def __init__(self, data, children):
+    def __init__(self, data, children, meta=None):
         self.data = data
         self.children = children
-        self._meta = None
+        self._meta = meta
 
     @property
     def meta(self):

lark/visitors.py

@@ -8,39 +8,23 @@ class Discard(Exception):
     pass
 
 
-class Base:
-    def _call_userfunc(self, tree):
-        return getattr(self, tree.data, self.__default__)(tree)
+# Transformers
 
-    def __default__(self, tree):
-        "Default operation on tree (for override)"
-        return tree
-
-    @classmethod
-    def _apply_decorator(cls, decorator):
-        mro = getmro(cls)
-        assert mro[0] is cls
-        libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)}
-        for name, value in getmembers(cls):
-            if name.startswith('_') or name in libmembers:
-                continue
-
-            setattr(cls, name, decorator(value))
-        return cls
-
-
-class SimpleBase(Base):
-    def _call_userfunc(self, tree):
+class Transformer:
+    def _call_userfunc(self, data, children, meta):
         # Assumes tree is already transformed
         try:
-            f = getattr(self, tree.data)
+            f = getattr(self, data)
         except AttributeError:
-            return self.__default__(tree)
+            return self.__default__(data, children, meta)
         else:
-            return f(tree.children)
-
+            if getattr(f, 'meta', False):
+                return f(children, meta)
+            elif getattr(f, 'inline', False):
+                return f(*children)
+            else:
+                return f(children)
 
-class Transformer(Base):
     def _transform_children(self, children):
         for c in children:
             try:
@@ -49,8 +33,8 @@ class Transformer(Base):
                 pass
 
     def _transform_tree(self, tree):
-        tree = Tree(tree.data, list(self._transform_children(tree.children)))
-        return self._call_userfunc(tree)
+        children = list(self._transform_children(tree.children))
+        return self._call_userfunc(tree.data, children, tree.meta)
 
     def transform(self, tree):
         return self._transform_tree(tree)
@@ -58,6 +42,32 @@ class Transformer(Base):
     def __mul__(self, other):
         return TransformerChain(self, other)
 
+    def __default__(self, data, children, meta):
+        "Default operation on tree (for override)"
+        return Tree(data, children, meta)
+
+    @classmethod
+    def _apply_decorator(cls, decorator, **kwargs):
+        mro = getmro(cls)
+        assert mro[0] is cls
+        libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)}
+        for name, value in getmembers(cls):
+            if name.startswith('_') or name in libmembers:
+                continue
+
+            setattr(cls, name, decorator(value, **kwargs))
+        return cls
+
+
+class InlineTransformer(Transformer):   # XXX Deprecated
+    def _call_userfunc(self, data, children, meta):
+        # Assumes tree is already transformed
+        try:
+            f = getattr(self, data)
+        except AttributeError:
+            return self.__default__(data, children, meta)
+        else:
+            return f(*children)
 
 
 class TransformerChain(object):
@@ -75,7 +85,7 @@ class TransformerChain(object):
 
 class Transformer_InPlace(Transformer):
     def _transform_tree(self, tree):           # Cancel recursion
-        return self._call_userfunc(tree)
+        return self._call_userfunc(tree.data, tree.children, tree.meta)
 
     def transform(self, tree):
         for subtree in tree.iter_subtrees():
@@ -87,11 +97,22 @@ class Transformer_InPlace(Transformer):
 class Transformer_InPlaceRecursive(Transformer):
     def _transform_tree(self, tree):
         tree.children = list(self._transform_children(tree.children))
-        return self._call_userfunc(tree)
+        return self._call_userfunc(tree.data, tree.children, tree.meta)
+
 
 
+# Visitors
 
-class Visitor(Base):
+class VisitorBase:
+    def _call_userfunc(self, tree):
+        return getattr(self, tree.data, self.__default__)(tree)
+
+    def __default__(self, tree):
+        "Default operation on tree (for override)"
+        return tree
+
+
+class Visitor(VisitorBase):
     "Bottom-up visitor"
 
     def visit(self, tree):
@@ -99,7 +120,7 @@ class Visitor(Base):
             self._call_userfunc(subtree)
         return tree
 
-class Visitor_Recursive(Base):
+class Visitor_Recursive(VisitorBase):
     def visit(self, tree):
         for child in tree.children:
             if isinstance(child, Tree):
@@ -110,6 +131,7 @@ class Visitor_Recursive(Base):
         return tree
 
 
+
 def visit_children_decor(func):
     @wraps(func)
     def inner(cls, tree):
@@ -117,7 +139,8 @@ def visit_children_decor(func):
         return func(cls, values)
     return inner
 
-class Interpreter(object):
+
+class Interpreter:
     "Top-down visitor"
 
     def visit(self, tree):
@@ -136,56 +159,58 @@ class Interpreter(object):
 
 
 
+# Decorators
 
-def _apply_decorator(obj, decorator):
+def _apply_decorator(obj, decorator, **kwargs):
     try:
         _apply = obj._apply_decorator
     except AttributeError:
-        return decorator(obj)
+        return decorator(obj, **kwargs)
     else:
-        return _apply(decorator)
+        return _apply(decorator, **kwargs)
 
 
-def _children_args__func(func):
-    if getattr(func, '_children_args_decorated', False):
-        return func
 
+def _inline_args__func(func):
     @wraps(func)
     def create_decorator(_f, with_self):
         if with_self:
-            def f(self, tree):
-                return _f(self, tree.children)
+            def f(self, children):
+                return _f(self, *children)
         else:
-            def f(args):
-                return _f(tree.children)
-        f._children_args_decorated = True
+            def f(self, children):
+                return _f(*children)
         return f
 
     return smart_decorator(func, create_decorator)
 
-def children_args(obj):
-    return _apply_decorator(obj, _children_args__func)
 
+def inline_args(obj):   # XXX Deprecated
+    return _apply_decorator(obj, _inline_args__func)
 
 
-def _children_args_inline__func(func):
-    if getattr(func, '_children_args_decorated', False):
-        return func
 
-    @wraps(func)
+def _visitor_args_func_dec(func, inline=False, meta=False):
+    assert not (inline and meta)
     def create_decorator(_f, with_self):
         if with_self:
-            def f(self, tree):
-                return _f(self, *tree.children)
+            def f(self, *args, **kwargs):
+                return _f(self, *args, **kwargs)
         else:
-            def f(self, tree):
-                print ('##', _f, tree)
-                return _f(*tree.children)
-        f._children_args_decorated = True
+            def f(self, *args, **kwargs):
+                return _f(*args, **kwargs)
         return f
 
-    return smart_decorator(func, create_decorator)
+    f = smart_decorator(func, create_decorator)
+    f.inline = inline
+    f.meta = meta
+    return f
+
+def visitor_args(inline=False, meta=False):
+    if inline and meta:
+        raise ValueError("Visitor functions can either accept meta, or be inlined. Not both.")
+    def _visitor_args_dec(obj):
+        return _apply_decorator(obj, _visitor_args_func_dec, inline=inline, meta=meta)
+    return _visitor_args_dec
 
 
-def children_args_inline(obj):
-    return _apply_decorator(obj, _children_args_inline__func)

tests/test_parser.py

@@ -21,7 +21,7 @@ from lark.lark import Lark
 from lark.common import GrammarError, ParseError, UnexpectedToken
 from lark.lexer import LexError, UnexpectedInput
 from lark.tree import Tree
-from lark.visitors import Transformer, children_args
+from lark.visitors import Transformer
 
 __path__ = os.path.dirname(__file__)
 def _read(n, *args):
@@ -93,7 +93,6 @@ class TestParsers(unittest.TestCase):
         self.assertEqual( r.children[0].data, "c" )
 
     def test_embedded_transformer(self):
-        @children_args
         class T(Transformer):
             def a(self, children):
                 return "<a>"

tests/test_trees.py

@@ -6,7 +6,7 @@ import copy
 import pickle
 
 from lark.tree import Tree
-from lark.visitors import Transformer, Interpreter, visit_children_decor, children_args_inline, children_args
+from lark.visitors import Transformer, Interpreter, visit_children_decor, visitor_args
 
 
 class TestTrees(TestCase):
@@ -63,19 +63,18 @@ class TestTrees(TestCase):
         t = Tree('add', [Tree('sub', [Tree('i', ['3']), Tree('f', ['1.1'])]), Tree('i', ['1'])])
 
         class T(Transformer):
-            i = children_args_inline(int)
-            f = children_args_inline(float)
+            i = visitor_args(inline=True)(int)
+            f = visitor_args(inline=True)(float)
 
-            sub = lambda self, tree: tree.children[0] - tree.children[1]
-
-            def add(self, tree):
-                return sum(tree.children)
+            sub = lambda self, values: values[0] - values[1]
 
+            def add(self, values):
+                return sum(values)
 
         res = T().transform(t)
         self.assertEqual(res, 2.9)
 
-        @children_args_inline
+        @visitor_args(inline=True)
         class T(Transformer):
             i = int
             f = float
@@ -89,7 +88,7 @@ class TestTrees(TestCase):
         self.assertEqual(res, 2.9)
 
 
-        @children_args_inline
+        @visitor_args(inline=True)
         class T(Transformer):
             i = int
             f = float
@@ -99,19 +98,6 @@ class TestTrees(TestCase):
         self.assertEqual(res, 2.9)
 
 
-        @children_args
-        class T(Transformer):
-            i = children_args_inline(int)
-            f = children_args_inline(float)
-
-            sub = lambda self, values: values[0] - values[1]
-
-            def add(self, values):
-                return sum(values)
-
-        res = T().transform(t)
-        self.assertEqual(res, 2.9)
-
 
 if __name__ == '__main__':
     unittest.main()