From 8001dc416f2e2c4a140dcee21c2cfb976f191ce3 Mon Sep 17 00:00:00 2001 From: Erez Sh Date: Thu, 26 Dec 2019 11:03:21 +0200 Subject: [PATCH] Implemented priority-decay (proposal to solve Issue #502) --- lark/lark.py | 2 +- lark/parser_frontends.py | 5 ++++- lark/parsers/earley.py | 6 +++--- lark/parsers/earley_forest.py | 22 +++++++++++++++------- lark/parsers/xearley.py | 4 ++-- 5 files changed, 25 insertions(+), 14 deletions(-) diff --git a/lark/lark.py b/lark/lark.py index 36cb4b6..d636a8a 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -167,7 +167,7 @@ class Lark(Serialize): elif self.options.priority in ('invert', 'normal'): assert self.options.parser in ('earley', 'cyk'), "priorities are not supported for LALR at this time" - assert self.options.priority in ('auto', None, 'normal', 'invert'), 'invalid priority option specified: {}. options are auto, none, normal, invert.'.format(self.options.priority) + assert self.options.priority in ('auto', None, 'normal', 'invert', 'decay'), 'invalid priority option specified: {}. options are auto, none, normal, invert.'.format(self.options.priority) assert self.options.ambiguity not in ('resolve__antiscore_sum', ), 'resolve__antiscore_sum has been replaced with the option priority="invert"' assert self.options.ambiguity in ('resolve', 'explicit', 'auto', ) diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py index 8b42772..d7c4805 100644 --- a/lark/parser_frontends.py +++ b/lark/parser_frontends.py @@ -148,7 +148,8 @@ class Earley(WithLexer): resolve_ambiguity = options.ambiguity == 'resolve' debug = options.debug if options else False - self.parser = earley.Parser(parser_conf, self.match, resolve_ambiguity=resolve_ambiguity, debug=debug) + priority_decay = 0.5 if options and options.priority == 'decay' else 1 + self.parser = earley.Parser(parser_conf, self.match, resolve_ambiguity=resolve_ambiguity, debug=debug, priority_decay=priority_decay) def match(self, term, token): return term.name == token.type @@ -162,11 +163,13 @@ class XEarley(_ParserFrontend): self._prepare_match(lexer_conf) resolve_ambiguity = options.ambiguity == 'resolve' debug = options.debug if options else False + priority_decay = 0.5 if options and options.priority == 'decay' else 1 self.parser = xearley.Parser(parser_conf, self.match, ignore=lexer_conf.ignore, resolve_ambiguity=resolve_ambiguity, debug=debug, + priority_decay=priority_decay, **kw ) diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index a4ffead..675d3b8 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -21,7 +21,7 @@ from .earley_common import Item, TransitiveItem from .earley_forest import ForestToTreeVisitor, ForestSumVisitor, SymbolNode, ForestToAmbiguousTreeVisitor class Parser: - def __init__(self, parser_conf, term_matcher, resolve_ambiguity=True, debug=False): + def __init__(self, parser_conf, term_matcher, resolve_ambiguity=True, debug=False, priority_decay=1): analysis = GrammarAnalyzer(parser_conf) self.parser_conf = parser_conf self.resolve_ambiguity = resolve_ambiguity @@ -46,7 +46,7 @@ class Parser: # skip the extra tree walk. We'll also skip this if the user just didn't specify priorities # on any rules. if self.forest_sum_visitor is None and rule.options.priority is not None: - self.forest_sum_visitor = ForestSumVisitor + self.forest_sum_visitor = ForestSumVisitor(priority_decay) self.term_matcher = term_matcher @@ -313,7 +313,7 @@ class Parser: # Perform our SPPF -> AST conversion using the right ForestVisitor. forest_tree_visitor_cls = ForestToTreeVisitor if self.resolve_ambiguity else ForestToAmbiguousTreeVisitor - forest_tree_visitor = forest_tree_visitor_cls(self.callbacks, self.forest_sum_visitor and self.forest_sum_visitor()) + forest_tree_visitor = forest_tree_visitor_cls(self.callbacks, self.forest_sum_visitor) return forest_tree_visitor.visit(solutions[0]) diff --git a/lark/parsers/earley_forest.py b/lark/parsers/earley_forest.py index c8b4f25..8cf077a 100644 --- a/lark/parsers/earley_forest.py +++ b/lark/parsers/earley_forest.py @@ -71,7 +71,10 @@ class SymbolNode(ForestNode): @property def children(self): if not self.paths_loaded: self.load_paths() - return sorted(self._children, key=attrgetter('sort_key')) + return self._children + + def sorted_children(self): + return sorted(self.children, key=attrgetter('sort_key')) def __iter__(self): return iter(self._children) @@ -243,6 +246,9 @@ class ForestSumVisitor(ForestVisitor): items created during parsing than there are SPPF nodes in the final tree. """ + def __init__(self, decay): + self._decay = decay + def visit_packed_node_in(self, node): return iter([node.left, node.right]) @@ -251,9 +257,9 @@ class ForestSumVisitor(ForestVisitor): def visit_packed_node_out(self, node): priority = node.rule.options.priority if not node.parent.is_intermediate and node.rule.options.priority else 0 - priority += getattr(node.right, 'priority', 0) - priority += getattr(node.left, 'priority', 0) - node.priority = priority + l = getattr(node.right, 'priority', 0) + r = getattr(node.left, 'priority', 0) + node.priority = priority + (l + r) * self._decay def visit_symbol_node_out(self, node): node.priority = max(child.priority for child in node.children) @@ -286,7 +292,7 @@ class ForestToTreeVisitor(ForestVisitor): def visit_symbol_node_in(self, node): if self.forest_sum_visitor and node.is_ambiguous and isinf(node.priority): self.forest_sum_visitor.visit(node) - return next(iter(node.children)) + return next(iter(node.sorted_children())) def visit_packed_node_in(self, node): if not node.parent.is_intermediate: @@ -321,7 +327,7 @@ class ForestToAmbiguousTreeVisitor(ForestToTreeVisitor): This is mainly used by the test framework, to make it simpler to write tests ensuring the SPPF contains the right results. """ - def __init__(self, callbacks, forest_sum_visitor = ForestSumVisitor): + def __init__(self, callbacks, forest_sum_visitor): super(ForestToAmbiguousTreeVisitor, self).__init__(callbacks, forest_sum_visitor) def visit_token_node(self, node): @@ -349,7 +355,9 @@ class ForestToAmbiguousTreeVisitor(ForestToTreeVisitor): def visit_packed_node_out(self, node): if not node.parent.is_intermediate: - result = self.callbacks[node.rule](self.output_stack.pop().children) + drv_node = self.output_stack.pop() + assert drv_node.data == 'drv' + result = self.callbacks[node.rule](drv_node.children) if self.output_stack: self.output_stack[-1].children.append(result) else: diff --git a/lark/parsers/xearley.py b/lark/parsers/xearley.py index f32d0d1..fb40d10 100644 --- a/lark/parsers/xearley.py +++ b/lark/parsers/xearley.py @@ -24,8 +24,8 @@ from .earley_forest import SymbolNode class Parser(BaseParser): - def __init__(self, parser_conf, term_matcher, resolve_ambiguity=True, ignore = (), complete_lex = False, debug=False): - BaseParser.__init__(self, parser_conf, term_matcher, resolve_ambiguity, debug) + def __init__(self, parser_conf, term_matcher, resolve_ambiguity=True, ignore = (), complete_lex = False, debug=False, priority_decay=1): + BaseParser.__init__(self, parser_conf, term_matcher, resolve_ambiguity, debug, priority_decay) self.ignore = [Terminal(t) for t in ignore] self.complete_lex = complete_lex