Fixed bug in lexer where unidentical tokens got the same name

README.md

@@ -63,6 +63,7 @@ Lark has no dependencies.
  - Automatic token collision resolution (unless both tokens are regexps)
  - Python 2 & 3 compatible
  - Unicode fully supported
+ - Extensive test suite
 
 ## Coming soon
 

lark/indenter.py

@@ -18,7 +18,7 @@ class Indenter:
 
         if indent > self.indent_level[-1]:
             self.indent_level.append(indent)
-            yield Token(self.INDENT_type, indent_str)
+            yield Token.new_borrow_pos(self.INDENT_type, indent_str, token)
         else:
             while indent < self.indent_level[-1]:
                 self.indent_level.pop()

lark/load_grammar.py

@@ -252,8 +252,8 @@ class ExtractAnonTokens(InlineTransformer):
                 try:
                     token_name = _TOKEN_NAMES[value]
                 except KeyError:
-                    if value.isalnum() and value[0].isalpha():
-                        token_name = value.upper()
+                    if value.isalnum() and value[0].isalpha() and ('__'+value.upper()) not in self.token_set:
+                        token_name = value.upper()  # This can create name duplications for unidentical tokens
                     else:
                         token_name = 'ANONSTR_%d' % self.i
                         self.i += 1

lark/parse_tree_builder.py

@@ -48,7 +48,7 @@ class ParseTreeBuilder:
         new_rules = []
         for origin, expansions in rules.items():
             expand1 = origin.startswith('?')
-            _origin = origin.lstrip('?*')
+            _origin = origin.lstrip('?')
 
             for expansion, alias in expansions:
                 if alias and origin.startswith('_'):

tests/test_parser.py

@@ -340,6 +340,17 @@ def _make_parser_test(PARSER):
             g = _Lark("""start: %s
                       %s""" % (' '.join(tokens), '\n'.join("%s: %s"%x for x in tokens.items())))
 
+        def test_float_without_lexer(self):
+            g = _Lark("""start: ["+"|"-"] float
+                         float: digit* "." digit+ exp?
+                              | digit+ exp
+                         exp: ("e"|"E") ["+"|"-"] digit+
+                         digit: "0"|"1"|"2"|"3"|"4"|"5"|"6"|"7"|"8"|"9"
+                      """)
+            g.parse("1.2")
+            g.parse("-.2e9")
+            g.parse("+2e-9")
+            self.assertRaises(ParseError, g.parse, "+2e-9e")
 
     _NAME = "Test" + PARSER.capitalize()
     _TestParser.__name__ = _NAME