Implemented priority-decay (proposal to solve Issue #502)

lark/lark.py

@@ -167,7 +167,7 @@ class Lark(Serialize):
         elif self.options.priority in ('invert', 'normal'):
             assert self.options.parser in ('earley', 'cyk'), "priorities are not supported for LALR at this time"
 
-        assert self.options.priority in ('auto', None, 'normal', 'invert'), 'invalid priority option specified: {}. options are auto, none, normal, invert.'.format(self.options.priority)
+        assert self.options.priority in ('auto', None, 'normal', 'invert', 'decay'), 'invalid priority option specified: {}. options are auto, none, normal, invert.'.format(self.options.priority)
         assert self.options.ambiguity not in ('resolve__antiscore_sum', ), 'resolve__antiscore_sum has been replaced with the option priority="invert"'
         assert self.options.ambiguity in ('resolve', 'explicit', 'auto', )
 

lark/parser_frontends.py

@@ -148,7 +148,8 @@ class Earley(WithLexer):
 
         resolve_ambiguity = options.ambiguity == 'resolve'
         debug = options.debug if options else False
-        self.parser = earley.Parser(parser_conf, self.match, resolve_ambiguity=resolve_ambiguity, debug=debug)
+        priority_decay = 0.5 if options and options.priority == 'decay' else 1
+        self.parser = earley.Parser(parser_conf, self.match, resolve_ambiguity=resolve_ambiguity, debug=debug, priority_decay=priority_decay)
 
     def match(self, term, token):
         return term.name == token.type
@@ -162,11 +163,13 @@ class XEarley(_ParserFrontend):
         self._prepare_match(lexer_conf)
         resolve_ambiguity = options.ambiguity == 'resolve'
         debug = options.debug if options else False
+        priority_decay = 0.5 if options and options.priority == 'decay' else 1
         self.parser = xearley.Parser(parser_conf,
                                     self.match,
                                     ignore=lexer_conf.ignore,
                                     resolve_ambiguity=resolve_ambiguity,
                                     debug=debug,
+                                    priority_decay=priority_decay,
                                     **kw
                                     )
 

lark/parsers/earley.py

@@ -21,7 +21,7 @@ from .earley_common import Item, TransitiveItem
 from .earley_forest import ForestToTreeVisitor, ForestSumVisitor, SymbolNode, ForestToAmbiguousTreeVisitor
 
 class Parser:
-    def __init__(self, parser_conf, term_matcher, resolve_ambiguity=True, debug=False):
+    def __init__(self, parser_conf, term_matcher, resolve_ambiguity=True, debug=False, priority_decay=1):
         analysis = GrammarAnalyzer(parser_conf)
         self.parser_conf = parser_conf
         self.resolve_ambiguity = resolve_ambiguity
@@ -46,7 +46,7 @@ class Parser:
             #  skip the extra tree walk. We'll also skip this if the user just didn't specify priorities
             #  on any rules.
             if self.forest_sum_visitor is None and rule.options.priority is not None:
-                self.forest_sum_visitor = ForestSumVisitor
+                self.forest_sum_visitor = ForestSumVisitor(priority_decay)
 
         self.term_matcher = term_matcher
 
@@ -313,7 +313,7 @@ class Parser:
 
         # Perform our SPPF -> AST conversion using the right ForestVisitor.
         forest_tree_visitor_cls = ForestToTreeVisitor if self.resolve_ambiguity else ForestToAmbiguousTreeVisitor
-        forest_tree_visitor = forest_tree_visitor_cls(self.callbacks, self.forest_sum_visitor and self.forest_sum_visitor())
+        forest_tree_visitor = forest_tree_visitor_cls(self.callbacks, self.forest_sum_visitor)
 
         return forest_tree_visitor.visit(solutions[0])
 

lark/parsers/earley_forest.py

@@ -71,7 +71,10 @@ class SymbolNode(ForestNode):
     @property
     def children(self):
         if not self.paths_loaded: self.load_paths()
-        return sorted(self._children, key=attrgetter('sort_key'))
+        return self._children
+
+    def sorted_children(self):
+        return sorted(self.children, key=attrgetter('sort_key'))
 
     def __iter__(self):
         return iter(self._children)
@@ -243,6 +246,9 @@ class ForestSumVisitor(ForestVisitor):
     items created during parsing than there are SPPF nodes in the
     final tree.
     """
+    def __init__(self, decay):
+        self._decay = decay
+
     def visit_packed_node_in(self, node):
         return iter([node.left, node.right])
 
@@ -251,9 +257,9 @@ class ForestSumVisitor(ForestVisitor):
 
     def visit_packed_node_out(self, node):
         priority = node.rule.options.priority if not node.parent.is_intermediate and node.rule.options.priority else 0
-        priority += getattr(node.right, 'priority', 0)
-        priority += getattr(node.left, 'priority', 0)
-        node.priority = priority
+        l = getattr(node.right, 'priority', 0)
+        r = getattr(node.left, 'priority', 0)
+        node.priority = priority + (l + r) * self._decay
 
     def visit_symbol_node_out(self, node):
         node.priority = max(child.priority for child in node.children)
@@ -286,7 +292,7 @@ class ForestToTreeVisitor(ForestVisitor):
     def visit_symbol_node_in(self, node):
         if self.forest_sum_visitor and node.is_ambiguous and isinf(node.priority):
             self.forest_sum_visitor.visit(node)
-        return next(iter(node.children))
+        return next(iter(node.sorted_children()))
 
     def visit_packed_node_in(self, node):
         if not node.parent.is_intermediate:
@@ -321,7 +327,7 @@ class ForestToAmbiguousTreeVisitor(ForestToTreeVisitor):
     This is mainly used by the test framework, to make it simpler to write
     tests ensuring the SPPF contains the right results.
     """
-    def __init__(self, callbacks, forest_sum_visitor = ForestSumVisitor):
+    def __init__(self, callbacks, forest_sum_visitor):
         super(ForestToAmbiguousTreeVisitor, self).__init__(callbacks, forest_sum_visitor)
 
     def visit_token_node(self, node):
@@ -349,7 +355,9 @@ class ForestToAmbiguousTreeVisitor(ForestToTreeVisitor):
 
     def visit_packed_node_out(self, node):
         if not node.parent.is_intermediate:
-            result = self.callbacks[node.rule](self.output_stack.pop().children)
+            drv_node = self.output_stack.pop()
+            assert drv_node.data == 'drv'
+            result = self.callbacks[node.rule](drv_node.children)
             if self.output_stack:
                 self.output_stack[-1].children.append(result)
             else:

lark/parsers/xearley.py

@@ -24,8 +24,8 @@ from .earley_forest import SymbolNode
 
 
 class Parser(BaseParser):
-    def __init__(self,  parser_conf, term_matcher, resolve_ambiguity=True, ignore = (), complete_lex = False, debug=False):
-        BaseParser.__init__(self, parser_conf, term_matcher, resolve_ambiguity, debug)
+    def __init__(self,  parser_conf, term_matcher, resolve_ambiguity=True, ignore = (), complete_lex = False, debug=False, priority_decay=1):
+        BaseParser.__init__(self, parser_conf, term_matcher, resolve_ambiguity, debug, priority_decay)
         self.ignore = [Terminal(t) for t in ignore]
         self.complete_lex = complete_lex