diff --git a/lark/common.py b/lark/common.py index 698a3ec..ec7012b 100644 --- a/lark/common.py +++ b/lark/common.py @@ -1,7 +1,7 @@ import re import sys -from .utils import get_regexp_width, STRING_TYPE +from .utils import get_regexp_width Py36 = (sys.version_info[:2] >= (3, 6)) @@ -40,6 +40,9 @@ class Pattern(object): def __eq__(self, other): return type(self) == type(other) and self.value == other.value and self.flags == other.flags + def to_regexp(self): + raise NotImplementedError() + if Py36: # Python 3.6 changed syntax for flags in regular expression def _get_flags(self, value): diff --git a/lark/exceptions.py b/lark/exceptions.py index 73b206a..ceca493 100644 --- a/lark/exceptions.py +++ b/lark/exceptions.py @@ -13,6 +13,8 @@ class LexError(LarkError): pass class UnexpectedInput(LarkError): + pos_in_stream = None + def get_context(self, text, span=40): pos = self.pos_in_stream start = max(pos - span, 0) diff --git a/lark/lexer.py b/lark/lexer.py index 3ab91b4..0950a00 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -113,7 +113,6 @@ class UnlessCallback: for mre, type_from_index in self.mres: m = mre.match(t.value) if m: - value = m.group(0) t.type = type_from_index[m.lastindex] break return t diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py index 7bd9378..7dd972d 100644 --- a/lark/parser_frontends.py +++ b/lark/parser_frontends.py @@ -5,16 +5,19 @@ from .utils import get_regexp_width from .parsers.grammar_analysis import GrammarAnalyzer from .lexer import TraditionalLexer, ContextualLexer, Lexer, Token -from .exceptions import GrammarError from .parsers import lalr_parser, earley, xearley, resolve_ambig, cyk from .tree import Tree class WithLexer: + lexer = None + parser = None + lexer_conf = None + def init_traditional_lexer(self, lexer_conf): self.lexer_conf = lexer_conf self.lexer = TraditionalLexer(lexer_conf.tokens, ignore=lexer_conf.ignore, user_callbacks=lexer_conf.callbacks) - def init_contextual_lexer(self, lexer_conf, parser_conf): + def init_contextual_lexer(self, lexer_conf): self.lexer_conf = lexer_conf states = {idx:list(t.keys()) for idx, t in self.parser._parse_table.states.items()} always_accept = lexer_conf.postlex.always_accept if lexer_conf.postlex else () @@ -27,8 +30,7 @@ class WithLexer: stream = self.lexer.lex(text) if self.lexer_conf.postlex: return self.lexer_conf.postlex.process(stream) - else: - return stream + return stream def parse(self, text): token_stream = self.lex(text) @@ -43,7 +45,7 @@ class LALR_TraditionalLexer(WithLexer): class LALR_ContextualLexer(WithLexer): def __init__(self, lexer_conf, parser_conf, options=None): self.parser = lalr_parser.Parser(parser_conf) - self.init_contextual_lexer(lexer_conf, parser_conf) + self.init_contextual_lexer(lexer_conf) class LALR_CustomLexer(WithLexer): def __init__(self, lexer_cls, lexer_conf, parser_conf, options=None): diff --git a/lark/utils.py b/lark/utils.py index 6f603bb..c86a66d 100644 --- a/lark/utils.py +++ b/lark/utils.py @@ -105,8 +105,7 @@ except NameError: return 0 elif a > b: return 1 - else: - return -1 + return -1 import sre_parse diff --git a/lark/visitors.py b/lark/visitors.py index cb7e32b..ec9bde6 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -1,4 +1,4 @@ -from inspect import isclass, getmembers, getmro +from inspect import getmembers, getmro from functools import wraps from .utils import smart_decorator