diff --git a/examples/python_parser.py b/examples/python_parser.py index f738a35..ddbd5c4 100644 --- a/examples/python_parser.py +++ b/examples/python_parser.py @@ -78,6 +78,6 @@ def test_earley_equals_lalr(): if __name__ == '__main__': test_python_lib() - # test_earley_equals_lalr() + test_earley_equals_lalr() # python_parser3.parse(_read(sys.argv[1]) + '\n') diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py index 54a1bac..e81569f 100644 --- a/lark/parse_tree_builder.py +++ b/lark/parse_tree_builder.py @@ -1,4 +1,4 @@ -from .common import is_terminal, GrammarError +from .common import GrammarError from .utils import suppress from .lexer import Token from .grammar import Rule diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py index a36252c..24c3622 100644 --- a/lark/parser_frontends.py +++ b/lark/parser_frontends.py @@ -4,9 +4,10 @@ from .utils import get_regexp_width from .parsers.grammar_analysis import GrammarAnalyzer from .lexer import Lexer, ContextualLexer, Token -from .common import is_terminal, GrammarError +from .common import GrammarError from .parsers import lalr_parser, earley, xearley, resolve_ambig, cyk from .tree import Tree +from .grammar import Terminal class WithLexer: def init_traditional_lexer(self, lexer_conf): @@ -96,7 +97,7 @@ class Earley(WithLexer): resolve_ambiguity=get_ambiguity_resolver(options)) def match(self, term, token): - return term == token.type + return term.name == token.type def parse(self, text): tokens = self.lex(text) @@ -117,7 +118,7 @@ class XEarley: ) def match(self, term, text, index=0): - return self.regexps[term].match(text, index) + return self.regexps[term.name].match(text, index) def _prepare_match(self, lexer_conf): self.regexps = {} diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index d119e41..f6397dd 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -13,9 +13,10 @@ # Author: Erez Shinan (2017) # Email : erezshin@gmail.com -from ..common import ParseError, UnexpectedToken, is_terminal +from ..common import ParseError, UnexpectedToken from ..tree import Tree, Transformer_NoRecurse from .grammar_analysis import GrammarAnalyzer +from ..grammar import NonTerminal class Derivation(Tree): @@ -127,7 +128,7 @@ class Column: self.completed[item_key] = item self.to_reduce.append(item) else: - if is_terminal(item.expect): + if item.expect.is_term: self.to_scan.append(item) else: k = item_key if self.predict_all else item @@ -161,13 +162,13 @@ class Parser: def parse(self, stream, start_symbol=None): # Define parser functions - start_symbol = start_symbol or self.parser_conf.start + start_symbol = NonTerminal(start_symbol or self.parser_conf.start) _Item = Item match = self.term_matcher def predict(nonterm, column): - assert not is_terminal(nonterm), nonterm + assert not nonterm.is_term, nonterm return [_Item(rule, 0, column, None) for rule in self.predictions[nonterm]] def complete(item): diff --git a/lark/parsers/xearley.py b/lark/parsers/xearley.py index d710f34..321b829 100644 --- a/lark/parsers/xearley.py +++ b/lark/parsers/xearley.py @@ -20,10 +20,11 @@ from collections import defaultdict -from ..common import ParseError, is_terminal +from ..common import ParseError from ..lexer import Token, UnexpectedInput from ..tree import Tree from .grammar_analysis import GrammarAnalyzer +from ..grammar import NonTerminal, Terminal from .earley import ApplyCallbacks, Item, Column @@ -32,7 +33,7 @@ class Parser: self.analysis = GrammarAnalyzer(parser_conf) self.parser_conf = parser_conf self.resolve_ambiguity = resolve_ambiguity - self.ignore = list(ignore) + self.ignore = [Terminal(t) for t in ignore] self.predict_all = predict_all self.FIRST = self.analysis.FIRST @@ -47,7 +48,7 @@ class Parser: def parse(self, stream, start_symbol=None): # Define parser functions - start_symbol = start_symbol or self.parser_conf.start + start_symbol = NonTerminal(start_symbol or self.parser_conf.start) delayed_matches = defaultdict(list) match = self.term_matcher @@ -55,7 +56,7 @@ class Parser: text_column = 0 def predict(nonterm, column): - assert not is_terminal(nonterm), nonterm + assert not nonterm.is_term, nonterm return [Item(rule, 0, column, None) for rule in self.predictions[nonterm]] def complete(item):