Browse Source
Bugfix: Some tokens did not recieve and end_line (Issue #472)
tags/gm/2021-09-23T00Z/github.com--lark-parser-lark/0.8.0
Erez Sh
6 years ago
1 changed files with 8 additions and 8 deletions
Unified View
Diff Options
-
+8 -8lark/lexer.py
+ 8
- 8
lark/lexer.py
View File
| @@ -175,24 +175,24 @@ class _Lex: | |||||
| value, type_ = res | value, type_ = res | ||||
| t = None | |||||
| if type_ not in ignore_types: | if type_ not in ignore_types: | ||||
| t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column) | t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column) | ||||
| line_ctr.feed(value, type_ in newline_types) | |||||
| t.end_line = line_ctr.line | |||||
| t.end_column = line_ctr.column | |||||
| if t.type in lexer.callback: | if t.type in lexer.callback: | ||||
| t = lexer.callback[t.type](t) | t = lexer.callback[t.type](t) | ||||
| if not isinstance(t, Token): | if not isinstance(t, Token): | ||||
| raise ValueError("Callbacks must return a token (returned %r)" % t) | raise ValueError("Callbacks must return a token (returned %r)" % t) | ||||
| last_token = t | |||||
| yield t | yield t | ||||
| last_token = t | |||||
| else: | else: | ||||
| if type_ in lexer.callback: | if type_ in lexer.callback: | ||||
| t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column) | |||||
| lexer.callback[type_](t) | |||||
| t2 = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column) | |||||
| lexer.callback[type_](t2) | |||||
| line_ctr.feed(value, type_ in newline_types) | |||||
| line_ctr.feed(value, type_ in newline_types) | |||||
| if t: | |||||
| t.end_line = line_ctr.line | |||||
| t.end_column = line_ctr.column | |||||
| class UnlessCallback: | class UnlessCallback: | ||||