Merge branch 'Rogdham-import-rule-rename'

docs/grammar.md

@@ -133,20 +133,29 @@ When importing rules, all their dependencies will be imported into a namespace,
 **Syntax:**
 ```html
 %import <module>.<TERMINAL>
-%import <module> (<TERM1> <TERM2>)
+%import <module>.<rule>
+%import <module>.<TERMINAL> -> <NEWTERMINAL>
+%import <module>.<rule> -> <newrule>
+%import <module> (<TERM1> <TERM2> <rule1> <rule2>)
 ```
 
 If the module path is absolute, Lark will attempt to load it from the built-in directory (currently, only `common.lark` is available).
 
 If the module path is relative, such as `.path.to.file`, Lark will attempt to load it from the current working directory. Grammars must have the `.lark` extension.
 
+The rule or terminal can be imported under an other name with the `->` syntax.
+
 **Example:**
 ```perl
 %import common.NUMBER
 
 %import .terminals_file (A B C)
+
+%import .rules_file.rulea -> ruleb
 ```
 
+Note that `%ignore` directives cannot be imported. Imported rules will abide by the `%ignore` directives declared in the main grammar.
+
 ### %declare
 
 Declare a terminal without defining it. Useful for plugins.

examples/lark.lark

@@ -10,10 +10,10 @@ token: TOKEN priority? ":" expansions _NL
 priority: "." NUMBER
 
 statement: "%ignore" expansions _NL                -> ignore
-         | "%import" import_args ["->" TOKEN] _NL  -> import
+         | "%import" import_args ["->" name] _NL   -> import
          | "%declare" name+                        -> declare
 
-import_args: name ("." name)*
+import_args: "."? name ("." name)*
 
 ?expansions: alias (_VBAR alias)*
 

examples/lark_grammar.py

@@ -6,6 +6,9 @@ grammar_files = [
     'examples/python2.lark',
     'examples/python3.lark',
     'examples/lark.lark',
+    'examples/relative-imports/multiples.lark',
+    'examples/relative-imports/multiple2.lark',
+    'examples/relative-imports/multiple3.lark',
     'lark/grammars/common.lark',
 ]
 

examples/relative-imports/multiple2.lark

@@ -0,0 +1 @@
+start: ("0" | "1")* "0"

examples/relative-imports/multiple3.lark

@@ -0,0 +1,5 @@
+start: mod0mod0+
+
+mod0mod0: "0" | "1" mod1mod0
+mod1mod0: "1" | "0" mod2mod1 mod1mod0
+mod2mod1: "0" | "1" mod2mod1

examples/relative-imports/multiples.lark

@@ -0,0 +1,5 @@
+start: "2:" multiple2
+     | "3:" multiple3
+
+%import .multiple2.start -> multiple2
+%import .multiple3.start -> multiple3

examples/relative-imports/multiples.py

@@ -0,0 +1,28 @@
+#
+# This example demonstrates relative imports with rule rewrite
+# see multiples.lark
+#
+
+#
+# if b is a number written in binary, and m is either 2 or 3,
+# the grammar aims to recognise m:b iif b is a multiple of m
+#
+# for example, 3:1001 is recognised
+# because 9 (0b1001) is a multiple of 3
+#
+
+from lark import Lark, UnexpectedInput
+
+parser = Lark.open('multiples.lark', parser='lalr')
+
+def is_in_grammar(data):
+    try:
+        parser.parse(data)
+    except UnexpectedInput:
+        return False
+    return True
+
+for n_dec in range(100):
+    n_bin = bin(n_dec)[2:]
+    assert is_in_grammar('2:{}'.format(n_bin)) == (n_dec % 2 == 0)
+    assert is_in_grammar('3:{}'.format(n_bin)) == (n_dec % 3 == 0)

lark/load_grammar.py

@@ -139,7 +139,7 @@ RULES = {
     'declare': ['_DECLARE _declare_args _NL'],
     'import': ['_IMPORT _import_path _NL',
                '_IMPORT _import_path _LPAR name_list _RPAR _NL',
-               '_IMPORT _import_path _TO TERMINAL _NL'],
+               '_IMPORT _import_path _TO name _NL'],
 
     '_import_path': ['import_lib', 'import_rel'],
     'import_lib': ['_import_args'],
@@ -586,7 +586,7 @@ def import_from_grammar_into_namespace(grammar, namespace, aliases):
         try:
             return aliases[name].value
         except KeyError:
-            return '%s.%s' % (namespace, name)
+            return '%s__%s' % (namespace, name)
 
     to_import = list(bfs(aliases, rule_dependencies))
     for symbol in to_import:
@@ -745,7 +745,7 @@ class GrammarLoader:
                     g = import_grammar(grammar_path, base_paths=[base_path])
 
                 aliases_dict = dict(zip(names, aliases))
-                new_td, new_rd = import_from_grammar_into_namespace(g, '.'.join(dotted_path), aliases_dict)
+                new_td, new_rd = import_from_grammar_into_namespace(g, '__'.join(dotted_path), aliases_dict)
 
                 term_defs += new_td
                 rule_defs += new_rd

tests/grammars/ab.lark

@@ -0,0 +1,10 @@
+startab: expr
+
+expr: A B
+    | A expr B
+
+A: "a"
+B: "b"
+
+%import common.WS
+%ignore WS

tests/test_parser.py

@@ -998,11 +998,89 @@ def _make_parser_test(LEXER, PARSER):
             self.assertEqual(x.children, ['12', 'elephants'])
 
 
+        def test_import_rename(self):
+            grammar = """
+            start: N W
+
+            %import common.NUMBER -> N
+            %import common.WORD -> W
+            %import common.WS
+            %ignore WS
+
+            """
+            l = _Lark(grammar)
+            x = l.parse('12 elephants')
+            self.assertEqual(x.children, ['12', 'elephants'])
+
+
         def test_relative_import(self):
             l = _Lark_open('test_relative_import.lark', rel_to=__file__)
             x = l.parse('12 lions')
             self.assertEqual(x.children, ['12', 'lions'])
 
+
+        def test_relative_import_rename(self):
+            l = _Lark_open('test_relative_import_rename.lark', rel_to=__file__)
+            x = l.parse('12 lions')
+            self.assertEqual(x.children, ['12', 'lions'])
+
+
+        def test_relative_rule_import(self):
+            l = _Lark_open('test_relative_rule_import.lark', rel_to=__file__)
+            x = l.parse('xaabby')
+            self.assertEqual(x.children, [
+                'x',
+                Tree('expr', ['a', Tree('expr', ['a', 'b']), 'b']),
+                'y'])
+
+
+        def test_relative_rule_import_drop_ignore(self):
+            # %ignore rules are dropped on import
+            l = _Lark_open('test_relative_rule_import_drop_ignore.lark',
+                           rel_to=__file__)
+            self.assertRaises((ParseError, UnexpectedInput),
+                              l.parse, 'xa abby')
+
+
+        def test_relative_rule_import_subrule(self):
+            l = _Lark_open('test_relative_rule_import_subrule.lark',
+                           rel_to=__file__)
+            x = l.parse('xaabby')
+            self.assertEqual(x.children, [
+                'x',
+                Tree('startab', [
+                    Tree('grammars__ab__expr', [
+                        'a', Tree('grammars__ab__expr', ['a', 'b']), 'b',
+                    ]),
+                ]),
+                'y'])
+
+
+        def test_relative_rule_import_subrule_no_conflict(self):
+            l = _Lark_open(
+                'test_relative_rule_import_subrule_no_conflict.lark',
+                rel_to=__file__)
+            x = l.parse('xaby')
+            self.assertEqual(x.children, [Tree('expr', [
+                'x',
+                Tree('startab', [
+                    Tree('grammars__ab__expr', ['a', 'b']),
+                ]),
+                'y'])])
+            self.assertRaises((ParseError, UnexpectedInput),
+                              l.parse, 'xaxabyby')
+
+
+        def test_relative_rule_import_rename(self):
+            l = _Lark_open('test_relative_rule_import_rename.lark',
+                           rel_to=__file__)
+            x = l.parse('xaabby')
+            self.assertEqual(x.children, [
+                'x',
+                Tree('ab', ['a', Tree('ab', ['a', 'b']), 'b']),
+                'y'])
+
+
         def test_multi_import(self):
             grammar = """
             start: NUMBER WORD

tests/test_relative_import_rename.lark

@@ -0,0 +1,7 @@
+start: N WORD
+
+%import .grammars.test.NUMBER -> N
+%import common.WORD
+%import common.WS
+%ignore WS
+

tests/test_relative_rule_import.lark

@@ -0,0 +1,7 @@
+start: X expr Y
+
+X: "x"
+Y: "y"
+
+%import .grammars.ab.expr
+

tests/test_relative_rule_import_drop_ignore.lark

@@ -0,0 +1,7 @@
+start: X expr Y
+
+X: "x"
+Y: "y"
+
+%import .grammars.ab.expr
+

tests/test_relative_rule_import_rename.lark

@@ -0,0 +1,7 @@
+start: X ab Y
+
+X: "x"
+Y: "y"
+
+%import .grammars.ab.expr -> ab
+

tests/test_relative_rule_import_subrule.lark

@@ -0,0 +1,7 @@
+start: X startab Y
+
+X: "x"
+Y: "y"
+
+%import .grammars.ab.startab
+

tests/test_relative_rule_import_subrule_no_conflict.lark

@@ -0,0 +1,9 @@
+start: expr
+
+expr: X startab Y
+
+X: "x"
+Y: "y"
+
+%import .grammars.ab.startab
+