Some more normalizing

lark/load_grammar.py

@@ -14,8 +14,8 @@ from .parsers.lalr_parser import UnexpectedToken
 from .common import is_terminal, GrammarError, LexerConf, ParserConf, PatternStr, PatternRE, TokenDef
 from .grammar import RuleOptions, Rule
 
-from .tree import Tree, Visitor, SlottedTree as ST
-from .transformers import Transformer_Children, Transformer_ChildrenInline
+from .tree import Tree, SlottedTree as ST
+from .transformers import Transformer, Transformer_Children, Transformer_ChildrenInline, Visitor
 
 __path__ = os.path.dirname(__file__)
 IMPORT_PATHS = [os.path.join(__path__, 'grammars')]
@@ -200,17 +200,14 @@ class SimplifyRule_Visitor(Visitor):
         #   -->
         # expansions( expansion(b, c, e), expansion(b, d, e) )
 
-        while True:
-            self._flatten(tree)
-
-            for i, child in enumerate(tree.children):
-                if isinstance(child, Tree) and child.data == 'expansions':
-                    tree.data = 'expansions'
-                    tree.children = [self.visit(ST('expansion', [option if i==j else other
-                                                                for j, other in enumerate(tree.children)]))
-                                     for option in set(child.children)]
-                    break
-            else:
+        self._flatten(tree)
+
+        for i, child in enumerate(tree.children):
+            if isinstance(child, Tree) and child.data == 'expansions':
+                tree.data = 'expansions'
+                tree.children = [self.visit(ST('expansion', [option if i==j else other
+                                                            for j, other in enumerate(tree.children)]))
+                                    for option in set(child.children)]
                 break
 
     def alias(self, tree):
@@ -234,7 +231,7 @@ class RuleTreeToText(Transformer_Children):
         return [sym.value for sym in symbols], None
     def alias(self, x):
         (expansion, _alias), alias = x
-        assert _alias is None, (alias, expansion, '-', _alias)
+        assert _alias is None, (alias, expansion, '-', _alias)  # Double alias not allowed
         return expansion, alias.value
 
 

lark/parsers/earley.py

@@ -14,7 +14,8 @@
 # Email : erezshin@gmail.com
 
 from ..common import ParseError, UnexpectedToken, is_terminal
-from ..tree import Tree, Transformer_NoRecurse
+from ..tree import Tree
+from ..transformers import InPlaceTransformer
 from .grammar_analysis import GrammarAnalyzer
 
 
@@ -229,7 +230,7 @@ class Parser:
         return ApplyCallbacks(self.postprocess).transform(tree)
 
 
-class ApplyCallbacks(Transformer_NoRecurse):
+class ApplyCallbacks(InPlaceTransformer):
     def __init__(self, postprocess):
         self.postprocess = postprocess
 

lark/parsers/resolve_ambig.py

@@ -1,7 +1,7 @@
 from ..utils import compare
 from functools import cmp_to_key
 
-from ..tree import Tree, Visitor_NoRecurse
+from ..tree import Tree
 
 
 # Standard ambiguity resolver (uses comparison)

lark/transformers.py

@@ -6,33 +6,25 @@ class Discard(Exception):
     pass
 
 
-class Transformer:
-    def _get_userfunc(self, name):
-        return getattr(self, name)
-
+class Base:
     def _call_userfunc(self, tree):
-        # Assumes tree is already transformed
-        try:
-            f = self._get_userfunc(tree.data)
-        except AttributeError:
-            return self.__default__(tree)
-        else:
-            return f(tree)
+        return getattr(self, tree.data, self.__default__)(tree)
 
-    def _transform(self, tree):
-        children = []
-        for c in tree.children:
+    def __default__(self, tree):
+        return tree
+
+class Transformer(Base):
+    def _transform_children(self, children):
+        for c in children:
             try:
-                children.append(self._transform(c) if isinstance(c, Tree) else c)
+                yield self._transform(c) if isinstance(c, Tree) else c
             except Discard:
                 pass
 
-        tree = Tree(tree.data, children)
+    def _transform(self, tree):
+        tree = Tree(tree.data, list(self._transform_children(tree.children)))
         return self._call_userfunc(tree)
 
-    def __default__(self, tree):
-        return tree
-
     def transform(self, tree):
         return self._transform(tree)
 
@@ -43,7 +35,7 @@ class Transformer_Children(Transformer):
     def _call_userfunc(self, tree):
         # Assumes tree is already transformed
         try:
-            f = self._get_userfunc(tree.data)
+            f = getattr(self, tree.data)
         except AttributeError:
             return self.__default__(tree)
         else:
@@ -53,7 +45,7 @@ class Transformer_ChildrenInline(Transformer):
     def _call_userfunc(self, tree):
         # Assumes tree is already transformed
         try:
-            f = self._get_userfunc(tree.data)
+            f = getattr(self, tree.data)
         except AttributeError:
             return self.__default__(tree)
         else:
@@ -72,6 +64,45 @@ class TransformerChain(object):
     def __mul__(self, other):
         return TransformerChain(*self.transformers + (other,))
 
+class Visitor(Base):
+    # def visit(self, tree):
+    #     for child in tree.children:
+    #         if isinstance(child, Tree):
+    #             self.visit(child)
+
+    #     f = getattr(self, tree.data, self.__default__)
+    #     f(tree)
+    #     return tree
+
+    def visit(self, tree):
+        for subtree in tree.iter_subtrees():
+            self._call_userfunc(subtree)
+        return tree
+
+    def __default__(self, tree):
+        pass
+
+
+class InPlaceTransformer(Transformer):
+    # def _transform(self, tree):
+    #     children = []
+    #     for c in tree.children:
+    #         try:
+    #             children.append(self._transform(c) if isinstance(c, Tree) else c)
+    #         except Discard:
+    #             pass
+
+    #     tree.children = children
+    #     return self._call_userfunc(tree)
+
+    def _transform(self, tree):
+        return self._call_userfunc(tree)
+
+    def transform(self, tree):
+        for subtree in tree.iter_subtrees():
+            subtree.children = list(self._transform_children(subtree.children))
+
+        return self._transform(tree)
 
 
 #### XXX PSEUDOCODE TODO

lark/tree.py

@@ -12,8 +12,6 @@ class Meta:
 
 ###{standalone
 class Tree(object):
-    __slots__ = ('data', 'children', '_meta', 'rule')
-
     def __init__(self, data, children):
         self.data = data
         self.children = children
@@ -141,77 +139,12 @@ class Transformer(object):
         return TransformerChain(self, other)
 
 
-class Discard(Exception):
-    pass
-
-class TransformerChain(object):
-    def __init__(self, *transformers):
-        self.transformers = transformers
-
-    def transform(self, tree):
-        for t in self.transformers:
-            tree = t.transform(tree)
-        return tree
-
-    def __mul__(self, other):
-        return TransformerChain(*self.transformers + (other,))
-
-
-
 class InlineTransformer(Transformer):
     def _get_func(self, name):  # use super()._get_func
         return inline_args(getattr(self, name)).__get__(self)
 
 
-class Visitor(object):
-    def visit(self, tree):
-        for child in tree.children:
-            if isinstance(child, Tree):
-                self.visit(child)
-
-        f = getattr(self, tree.data, self.__default__)
-        f(tree)
-        return tree
-
-    def __default__(self, tree):
-        pass
-
-
-class Visitor_NoRecurse(Visitor):
-    def visit(self, tree):
-        subtrees = list(tree.iter_subtrees())
-
-        for subtree in (subtrees):
-            getattr(self, subtree.data, self.__default__)(subtree)
-        return tree
-
-
-class Transformer_NoRecurse(Transformer):
-    def transform(self, tree):
-        subtrees = list(tree.iter_subtrees())
-
-        def _t(t):
-            # Assumes t is already transformed
-            try:
-                f = self._get_func(t.data)
-            except AttributeError:
-                return self.__default__(t)
-            else:
-                return f(t)
-
-        for subtree in subtrees:
-            children = []
-            for c in subtree.children:
-                try:
-                    children.append(_t(c) if isinstance(c, Tree) else c)
-                except Discard:
-                    pass
-            subtree.children = children
-
-        return _t(tree)
 
-    def __default__(self, t):
-        return t
 ###}