| @@ -54,20 +54,6 @@ class TestParsers(unittest.TestCase): | |||||
| l = Lark(g, parser='earley', lexer='dynamic') | l = Lark(g, parser='earley', lexer='dynamic') | ||||
| self.assertRaises(ParseError, l.parse, 'a') | self.assertRaises(ParseError, l.parse, 'a') | ||||
| def test_utf8(self): | |||||
| g = u"""start: a | |||||
| a: "±a" | |||||
| """ | |||||
| l = Lark(g) | |||||
| l.parse(u'±a') | |||||
| l = Lark(g, parser='earley', lexer=None) | |||||
| l.parse(u'±a') | |||||
| l = Lark(g, parser='earley', lexer='dynamic') | |||||
| l.parse(u'±a') | |||||
| def _make_full_earley_test(LEXER): | def _make_full_earley_test(LEXER): | ||||
| class _TestFullEarley(unittest.TestCase): | class _TestFullEarley(unittest.TestCase): | ||||
| def test_anon_in_scanless(self): | def test_anon_in_scanless(self): | ||||
| @@ -797,6 +783,22 @@ def _make_parser_test(LEXER, PARSER): | |||||
| self.assertEqual(''.join(child.data for child in res.children), 'indirection') | self.assertEqual(''.join(child.data for child in res.children), 'indirection') | ||||
| def test_utf8(self): | |||||
| g = u"""start: a | |||||
| a: "±a" | |||||
| """ | |||||
| l = _Lark(g) | |||||
| self.assertEqual(l.parse(u'±a'), Tree('start', [Tree('a', [])])) | |||||
| g = u"""start: A | |||||
| A: "±a" | |||||
| """ | |||||
| l = _Lark(g) | |||||
| self.assertEqual(l.parse(u'±a'), Tree('start', [u'\xb1a'])) | |||||
| _NAME = "Test" + PARSER.capitalize() + (LEXER or 'Scanless').capitalize() | _NAME = "Test" + PARSER.capitalize() + (LEXER or 'Scanless').capitalize() | ||||
| _TestParser.__name__ = _NAME | _TestParser.__name__ = _NAME | ||||
| globals()[_NAME] = _TestParser | globals()[_NAME] = _TestParser | ||||