Support for token visitation in internal transformers, as an alternative mechanism for lexer_callbacks

lark/exceptions.py

@@ -97,10 +97,10 @@ class UnexpectedToken(ParseError, UnexpectedInput):
         super(UnexpectedToken, self).__init__(message)
 
 class VisitError(LarkError):
-    def __init__(self, tree, orig_exc):
-        self.tree = tree
+    def __init__(self, rule, obj, orig_exc):
+        self.obj = obj
         self.orig_exc = orig_exc
 
-        message = 'Error trying to process rule "%s":\n\n%s' % (tree.data, orig_exc)
+        message = 'Error trying to process rule "%s":\n\n%s' % (rule, orig_exc)
         super(VisitError, self).__init__(message)
 ###}

lark/lark.py

@@ -225,7 +225,16 @@ class Lark(Serialize):
             for rule in self.rules:
                 if rule.options and rule.options.priority is not None:
                     rule.options.priority = None
-        self.lexer_conf = LexerConf(self.terminals, self.ignore_tokens, self.options.postlex, self.options.lexer_callbacks)
+
+        # TODO Deprecate lexer_callbacks?
+        lexer_callbacks = dict(self.options.lexer_callbacks)
+        if self.options.transformer:
+            t = self.options.transformer
+            for term in self.terminals:
+                if hasattr(t, term.name):
+                    lexer_callbacks[term.name] = getattr(t, term.name)
+
+        self.lexer_conf = LexerConf(self.terminals, self.ignore_tokens, self.options.postlex, lexer_callbacks)
 
         if self.options.parser:
             self.parser = self._build_parser()

lark/lexer.py

@@ -108,6 +108,13 @@ class Token(Str):
         self.end_column = end_column
         return self
 
+    def update(self, type_=None, value=None):
+        return Token.new_borrow_pos(
+            type_ if type_ is not None else self.type,
+            value if value is not None else self.value,
+            self
+        )
+
     @classmethod
     def new_borrow_pos(cls, type_, value, borrow_t):
         return cls(type_, value, borrow_t.pos_in_stream, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column)

lark/visitors.py

@@ -48,7 +48,7 @@ class Transformer:
             except (GrammarError, Discard):
                 raise
             except Exception as e:
-                raise VisitError(tree, e)
+                raise VisitError(tree.data, tree, e)
 
     def _call_userfunc_token(self, token):
         try:
@@ -61,7 +61,7 @@ class Transformer:
             except (GrammarError, Discard):
                 raise
             except Exception as e:
-                raise VisitError(token, e)
+                raise VisitError(token.type, token, e)
 
 
     def _transform_children(self, children):

tests/test_parser.py

@@ -99,16 +99,22 @@ class TestParsers(unittest.TestCase):
             def a(self, children):
                 return children[0] + "!"
             def A(self, tok):
-                return tok.upper()
+                return tok.update(value=tok.upper())
 
         # Test regular
-        g = Lark("""start: a
-                    a : A
-                    A: "x"
-                 """, parser='lalr')
-        r = T().transform(g.parse("x"))
+        g = """start: a
+            a : A
+            A: "x"
+            """
+        p = Lark(g, parser='lalr')
+        r = T(False).transform(p.parse("x"))
         self.assertEqual( r.children, ["x!"] )
-        r = T(True).transform(g.parse("x"))
+        r = T().transform(p.parse("x"))
+        self.assertEqual( r.children, ["X!"] )
+
+        # Test internal transformer
+        p = Lark(g, parser='lalr', transformer=T())
+        r = p.parse("x")
         self.assertEqual( r.children, ["X!"] )