Small fixes

lark/load_grammar.py

@@ -1,3 +1,5 @@
+"Parses and creates Grammar objects"
+
 import os.path
 from itertools import chain
 import re
@@ -205,12 +207,6 @@ class SimplifyRule_Visitor(Visitor):
 
     expansions = _flatten
 
-def dict_update_safe(d1, d2):
-    for k, v in d2.items():
-        assert k not in d1
-        d1[k] = v
-
-
 class RuleTreeToText(Transformer):
     def expansions(self, x):
         return x
@@ -233,6 +229,8 @@ class CanonizeTree(InlineTransformer):
         return tokenmods + [value]
 
 class ExtractAnonTokens(InlineTransformer):
+    "Create a unique list of anonymous tokens. Attempt to give meaningful names to them when we add them"
+
     def __init__(self, tokens):
         self.tokens = tokens
         self.token_set = {td.name for td in self.tokens}
@@ -244,6 +242,7 @@ class ExtractAnonTokens(InlineTransformer):
         value = p.value
         if p in self.token_reverse and p.flags != self.token_reverse[p].pattern.flags:
             raise GrammarError(u'Conflicting flags for the same terminal: %s' % p)
+
         if isinstance(p, PatternStr):
             try:
                 # If already defined, use the user-defined token name
@@ -353,6 +352,12 @@ def _interleave(l, item):
 def _choice_of_rules(rules):
     return T('expansions', [T('expansion', [Token('RULE', name)]) for name in rules])
 
+def dict_update_safe(d1, d2):
+    for k, v in d2.items():
+        assert k not in d1
+        d1[k] = v
+
+
 class Grammar:
     def __init__(self, rule_defs, token_defs, ignore):
         self.token_defs = token_defs

lark/parser_frontends.py

@@ -4,10 +4,7 @@ import sre_parse
 from .lexer import Lexer, ContextualLexer, Token
 
 from .common import is_terminal, GrammarError, ParserConf, Terminal_Regexp, Terminal_Token
-from .parsers import lalr_parser, earley
-from .tree import Transformer
-
-from .parsers import xearley
+from .parsers import lalr_parser, earley, xearley
 
 class WithLexer:
     def __init__(self, lexer_conf):
@@ -21,6 +18,7 @@ class WithLexer:
         else:
             return stream
 
+
 class LALR(WithLexer):
     def __init__(self, lexer_conf, parser_conf, options=None):
         WithLexer.__init__(self, lexer_conf)
@@ -32,6 +30,7 @@ class LALR(WithLexer):
         tokens = self.lex(text)
         return self.parser.parse(tokens)
 
+
 class LALR_ContextualLexer:
     def __init__(self, lexer_conf, parser_conf, options=None):
         self.lexer_conf = lexer_conf
@@ -49,6 +48,7 @@ class LALR_ContextualLexer:
             tokens = self.lexer_conf.postlex.process(tokens)
         return self.parser.parse(tokens, self.lexer.set_parser_state)
 
+
 def tokenize_text(text):
     new_text = []
     line = 1

lark/parsers/earley.py

@@ -224,6 +224,9 @@ class ApplyCallbacks(Transformer_NoRecurse):
             return Tree(rule.origin, children)
 
 def _compare_rules(rule1, rule2):
+    if rule1 == rule2:
+        return 0
+
     if rule1.options and rule2.options:
         if rule1.options.priority is not None and rule2.options.priority is not None:
             assert rule1.options.priority != rule2.options.priority, "Priority is the same between both rules: %s == %s" % (rule1, rule2)

lark/tree.py

@@ -15,7 +15,7 @@ class Tree(object):
 
     def _pretty(self, level, indent_str):
         if len(self.children) == 1 and not isinstance(self.children[0], Tree):
-            return [ indent_str*level, self.data, '\t', '%s' % self.children[0], '\n']
+            return [ indent_str*level, self._pretty_label(), '\t', '%s' % self.children[0], '\n']
 
         l = [ indent_str*level, self._pretty_label(), '\n' ]
         for n in self.children: