Better error reporting in lexer and lalr parser

lark/lexer.py

@@ -61,7 +61,7 @@ class Lexer(object):
 
     def lex(self, stream):
         lex_pos = 0
-        line = 0
+        line = 1
         col_start_pos = 0
         while True:
             i = 0
@@ -87,7 +87,7 @@ class Lexer(object):
             else:
                 if lex_pos < len(stream):
                     context = stream[lex_pos:lex_pos+5]
-                    raise LexError("No token defined for: '%s' in %s" % (stream[lex_pos], context))
+                    raise LexError("No token defined for: '%s' in %s at line %d" % (stream[lex_pos], context, line))
                 break
 
 

lark/load_grammar.py

@@ -231,6 +231,7 @@ class ExtractAnonTokens(InlineTransformer):
         self.tokens = tokens
         self.token_set = token_set
         self.token_reverse = {value[1:-1]: name for name, value, _flags in tokens}
+        self.i = 0
 
     def anontoken(self, token):
         if token.type == 'STRING':

lark/parser.py

@@ -22,8 +22,11 @@ class Parser(object):
                 return states_idx[state][key]
             except KeyError:
                 expected = states_idx[state].keys()
-                context = ' '.join(['%s(%r)' % (t.type, t.value) for t in seq[i:i+5]])
-                raise ParseError("Unexpected input %r.\nExpected: %s\nContext: %s" % (key, expected, context))
+                context = ' '.join(['%r(%s)' % (t.value, t.type) for t in seq[i:i+5]])
+                token = seq[i]
+                raise ParseError("Unexpected input %r at line %d, column %d.\n"
+                                 "Expected: %s\n"
+                                 "Context: %s" % (token.value, token.line, token.column, expected, context))
 
         def reduce(rule):
             if rule.expansion:

lark/tests/test_parser.py

@@ -267,6 +267,8 @@ class TestLalr(unittest.TestCase):
                 """, parser='lalr')
         x = g.parse('Hello World')
         self.assertSequenceEqual(x.children, ['World'])
+        x = g.parse('HelloWorld')
+        self.assertSequenceEqual(x.children, ['HelloWorld'])
 
     def test_undefined_rule(self):
         self.assertRaises(GrammarError, Lark, """start: a""", parser='lalr')