diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py index 977c371..b54b6e8 100644 --- a/lark/parse_tree_builder.py +++ b/lark/parse_tree_builder.py @@ -2,6 +2,7 @@ from .exceptions import GrammarError from .lexer import Token from .tree import Tree from .visitors import InlineTransformer # XXX Deprecated +from .visitors import Transformer_InPlace ###{standalone from functools import partial, wraps @@ -193,6 +194,14 @@ def ptb_inline_args(func): return func(*children) return f +def inplace_transformer(func): + @wraps(func) + def f(children): + # function name in a Transformer is a rule name. + tree = Tree(func.__name__, children) + return func(tree) + return f + class ParseTreeBuilder: def __init__(self, rules, tree_class, propagate_positions=False, keep_all_tokens=False, ambiguous=False, maybe_placeholders=False): self.tree_class = tree_class @@ -231,6 +240,8 @@ class ParseTreeBuilder: # XXX InlineTransformer is deprecated! if getattr(f, 'inline', False) or isinstance(transformer, InlineTransformer): f = ptb_inline_args(f) + elif hasattr(f, 'whole_tree') or isinstance(transformer, Transformer_InPlace): + f = inplace_transformer(f) except AttributeError: f = partial(self.tree_class, user_callback_name) diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py index ab69d01..0634814 100644 --- a/lark/parser_frontends.py +++ b/lark/parser_frontends.py @@ -107,7 +107,7 @@ class LALR_ContextualLexer(LALR_WithLexer): ###} class LALR_CustomLexer(LALR_WithLexer): - def __init__(self, lexer_cls, lexer_conf, parser_conf, *, options=None): + def __init__(self, lexer_cls, lexer_conf, parser_conf, options=None): self.lexer = lexer_cls(self.lexer_conf) debug = options.debug if options else False self.parser = LALR_Parser(parser_conf, debug=debug) diff --git a/lark/visitors.py b/lark/visitors.py index 53847f9..4a0f639 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -36,7 +36,7 @@ class Transformer: return f(*children) elif getattr(f, 'whole_tree', False): if new_children is not None: - raise NotImplementedError("Doesn't work with the base Transformer class") + tree.children = new_children return f(tree) else: return f(children) diff --git a/tests/test_parser.py b/tests/test_parser.py index ce8b7d6..1cf702d 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -20,7 +20,7 @@ logging.basicConfig(level=logging.INFO) from lark.lark import Lark from lark.exceptions import GrammarError, ParseError, UnexpectedToken, UnexpectedInput, UnexpectedCharacters from lark.tree import Tree -from lark.visitors import Transformer +from lark.visitors import Transformer, Transformer_InPlace, v_args from lark.grammar import Rule from lark.lexer import TerminalDef @@ -150,6 +150,51 @@ class TestParsers(unittest.TestCase): r = g.parse("xx") self.assertEqual( r.children, [""] ) + def test_embedded_transformer_inplace(self): + @v_args(tree=True) + class T1(Transformer_InPlace): + def a(self, tree): + assert isinstance(tree, Tree), tree + tree.children.append("tested") + return tree + + def b(self, tree): + return Tree(tree.data, tree.children + ['tested2']) + + @v_args(tree=True) + class T2(Transformer): + def a(self, tree): + assert isinstance(tree, Tree) + tree.children.append("tested") + return tree + + def b(self, tree): + return Tree(tree.data, tree.children + ['tested2']) + + class T3(Transformer): + @v_args(tree=True) + def a(self, tree): + assert isinstance(tree, Tree) + tree.children.append("tested") + return tree + + @v_args(tree=True) + def b(self, tree): + return Tree(tree.data, tree.children + ['tested2']) + + for t in [T1(), T2(), T3()]: + for internal in [False, True]: + g = Lark("""start: a b + a : "x" + b : "y" + """, parser='lalr', transformer=t if internal else None) + r = g.parse("xy") + if not internal: + r = t.transform(r) + + a, b = r.children + self.assertEqual(a.children, ["tested"]) + self.assertEqual(b.children, ["tested2"]) def test_alias(self): Lark("""start: ["a"] "b" ["c"] "e" ["f"] ["g"] ["h"] "x" -> d """)