Continue merging .pyi files

load_grammar, reconstruct, and visitors

lark-stubs/load_grammar.pyi

@@ -1,31 +0,0 @@
-from typing import List, Tuple, Union, Callable, Dict, Optional
-
-from .tree import Tree
-from .grammar import RuleOptions
-from .exceptions import UnexpectedInput
-
-
-class Grammar:
-    rule_defs: List[Tuple[str, Tuple[str, ...], Tree, RuleOptions]]
-    term_defs: List[Tuple[str, Tuple[Tree, int]]]
-    ignore: List[str]
-
-
-class GrammarBuilder:
-    global_keep_all_tokens: bool
-    import_paths: List[Union[str, Callable]]
-    used_files: Dict[str, str]
-
-    def __init__(self, global_keep_all_tokens: bool = False, import_paths: List[Union[str, Callable]] = None, used_files: Dict[str, str]=None) -> None: ...
-
-    def load_grammar(self, grammar_text: str, grammar_name: str = ..., mangle: Callable[[str], str] = None) -> None: ...
-
-    def do_import(self, dotted_path: Tuple[str, ...], base_path: Optional[str], aliases: Dict[str, str],
-                  base_mangle: Callable[[str], str] = None) -> None:  ...
-
-    def validate(self) -> None: ...
-
-    def build(self) -> Grammar: ...
-
-
-def find_grammar_errors(text: str, start: str='start') -> List[Tuple[UnexpectedInput, str]]: ...

lark-stubs/reconstruct.pyi

@@ -1,39 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from typing import List, Dict, Union, Callable, Iterable
-
-from .grammar import Symbol
-from .lark import Lark
-from .tree import Tree
-from .visitors import Transformer_InPlace
-from .lexer import TerminalDef
-
-
-class WriteTokensTransformer(Transformer_InPlace):
-
-    def __init__(self, tokens: Dict[str, TerminalDef], term_subs: Dict[str, Callable[[Symbol], str]] = ...): ...
-
-
-class MatchTree(Tree):
-    pass
-
-
-class MakeMatchTree:
-    name: str
-    expansion: List[TerminalDef]
-
-    def __init__(self, name: str, expansion: List[TerminalDef]):
-        ...
-
-    def __call__(self, args: List[Union[str, Tree]]):
-        ...
-
-
-class Reconstructor:
-
-    def __init__(self, parser: Lark, term_subs: Dict[str, Callable[[Symbol], str]] = ...):
-        ...
-
-    def reconstruct(self, tree: Tree, postproc: Callable[[Iterable[str]], Iterable[str]]=None,
-                    insert_spaces: bool = True) -> str:
-        ...

lark-stubs/visitors.pyi

@@ -1,108 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union
-from abc import ABC
-from .tree import Tree
-
-_T = TypeVar('_T')
-_R = TypeVar('_R')
-_FUNC = Callable[..., _T]
-_DECORATED = Union[_FUNC, type]
-
-
-class Transformer(ABC, Generic[_T]):
-
-    def __init__(self, visit_tokens: bool = True) -> None:
-        ...
-
-    def transform(self, tree: Tree) -> _T:
-        ...
-
-    def __mul__(self, other: Transformer[_T]) -> TransformerChain[_T]:
-        ...
-
-
-class TransformerChain(Generic[_T]):
-    transformers: Tuple[Transformer[_T], ...]
-
-    def __init__(self, *transformers: Transformer[_T]) -> None:
-        ...
-
-    def transform(self, tree: Tree) -> _T:
-        ...
-
-    def __mul__(self, other: Transformer[_T]) -> TransformerChain[_T]:
-        ...
-
-
-class Transformer_InPlace(Transformer):
-    pass
-
-
-class Transformer_NonRecursive(Transformer):
-    pass
-
-
-class Transformer_InPlaceRecursive(Transformer):
-    pass
-
-
-class VisitorBase:
-    pass
-
-
-class Visitor(VisitorBase, ABC, Generic[_T]):
-
-    def visit(self, tree: Tree) -> Tree:
-        ...
-
-    def visit_topdown(self, tree: Tree) -> Tree:
-        ...
-
-
-class Visitor_Recursive(VisitorBase):
-
-    def visit(self, tree: Tree) -> Tree:
-        ...
-
-    def visit_topdown(self, tree: Tree) -> Tree:
-        ...
-
-
-class Interpreter(ABC, Generic[_T]):
-
-    def visit(self, tree: Tree) -> _T:
-        ...
-
-    def visit_children(self, tree: Tree) -> List[_T]:
-        ...
-
-
-_InterMethod = Callable[[Type[Interpreter], _T], _R]
-
-
-def v_args(
-        inline: bool = False,
-        meta: bool = False,
-        tree: bool = False,
-        wrapper: Callable = None
-) -> Callable[[_DECORATED], _DECORATED]:
-    ...
-
-
-def visit_children_decor(func: _InterMethod) -> _InterMethod:
-    ...
-
-
-class Discard(Exception):
-    pass
-
-
-# Deprecated
-class InlineTransformer:
-    pass
-
-
-# Deprecated
-def inline_args(obj: _FUNC) -> _FUNC:
-    ...

lark/indenter.py

@@ -1,13 +1,14 @@
 "Provides Indentation services for languages with indentation similar to Python"
 
 from abc import ABC, abstractmethod
-from typing import Tuple, List, Iterator, Optional
 
 from .exceptions import LarkError
 from .lark import PostLex
 from .lexer import Token
 
 ###{standalone
+from typing import Tuple, List, Iterator, Optional
+
 class DedentError(LarkError):
     pass
 

lark/load_grammar.py

@@ -8,6 +8,7 @@ import pkgutil
 from ast import literal_eval
 from numbers import Integral
 from contextlib import suppress
+from typing import List, Tuple, Union, Callable, Dict, Optional
 
 from .utils import bfs, Py36, logger, classify_bool, is_id_continue, is_id_start, bfs_all_unique
 from .lexer import Token, TerminalDef, PatternStr, PatternRE
@@ -17,7 +18,7 @@ from .parser_frontends import ParsingFrontend
 from .common import LexerConf, ParserConf
 from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol
 from .utils import classify, dedup_list
-from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken, ParseError
+from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken, ParseError, UnexpectedInput
 
 from .tree import Tree, SlottedTree as ST
 from .visitors import Transformer, Visitor, v_args, Transformer_InPlace, Transformer_NonRecursive
@@ -552,9 +553,9 @@ def nr_deepcopy_tree(t):
 
 class Grammar:
     def __init__(self, rule_defs, term_defs, ignore):
-        self.term_defs = term_defs
-        self.rule_defs = rule_defs
-        self.ignore = ignore
+        self.term_defs: List[Tuple[str, Tuple[Tree, int]]] = term_defs
+        self.rule_defs: List[Tuple[str, Tuple[str, ...], Tree, RuleOptions]] = rule_defs
+        self.ignore: List[str] = ignore
 
     def compile(self, start, terminals_to_keep):
         # We change the trees in-place (to support huge grammars)
@@ -874,7 +875,7 @@ def _search_interactive_parser(interactive_parser, predicate):
         if predicate(p):
             return path, p
 
-def find_grammar_errors(text, start='start'):
+def find_grammar_errors(text: str, start: str='start') -> List[Tuple[UnexpectedInput, str]]:
     errors = []
     def on_error(e):
         errors.append((e, _error_repr(e)))
@@ -923,10 +924,10 @@ def _mangle_exp(exp, mangle):
 
 
 class GrammarBuilder:
-    def __init__(self, global_keep_all_tokens=False, import_paths=None, used_files=None):
-        self.global_keep_all_tokens = global_keep_all_tokens
-        self.import_paths = import_paths or []
-        self.used_files = used_files or {}
+    def __init__(self, global_keep_all_tokens: bool=False, import_paths: List[Union[str, Callable]]=None, used_files: Dict[str, str]=None) -> None:
+        self.global_keep_all_tokens: bool = global_keep_all_tokens
+        self.import_paths: List[Union[str, Callable]] = import_paths or []
+        self.used_files: Dict[str, str] = used_files or {}
 
         self._definitions = {}
         self._ignore_names = []
@@ -1067,7 +1068,7 @@ class GrammarBuilder:
         return name, exp, params, opts
 
 
-    def load_grammar(self, grammar_text, grammar_name="<?>", mangle=None):
+    def load_grammar(self, grammar_text: str, grammar_name: str="<?>", mangle: Callable[[str], str]=None) -> None:
         tree = _parse_grammar(grammar_text, grammar_name)
 
         imports = {}
@@ -1130,7 +1131,7 @@ class GrammarBuilder:
         self._definitions = {k: v for k, v in self._definitions.items() if k in _used}
 
 
-    def do_import(self, dotted_path, base_path, aliases, base_mangle=None):
+    def do_import(self, dotted_path: Tuple[str, ...], base_path: Optional[str], aliases: Dict[str, str], base_mangle: Callable[[str], str]=None) -> None:
         assert dotted_path
         mangle = _get_mangle('__'.join(dotted_path), aliases, base_mangle)
         grammar_path = os.path.join(*dotted_path) + EXT
@@ -1166,7 +1167,7 @@ class GrammarBuilder:
             assert False, "Couldn't import grammar %s, but a corresponding file was found at a place where lark doesn't search for it" % (dotted_path,)
 
 
-    def validate(self):
+    def validate(self) -> None:
         for name, (params, exp, _options) in self._definitions.items():
             for i, p in enumerate(params):
                 if p in self._definitions:
@@ -1195,7 +1196,7 @@ class GrammarBuilder:
         if not set(self._definitions).issuperset(self._ignore_names):
             raise GrammarError("Terminals %s were marked to ignore but were not defined!" % (set(self._ignore_names) - set(self._definitions)))
 
-    def build(self):
+    def build(self) -> Grammar:
         self.validate()
         rule_defs = []
         term_defs = []

lark/reconstruct.py

@@ -1,11 +1,13 @@
 """Reconstruct text from a tree, based on Lark grammar"""
 
+from typing import List, Dict, Union, Callable, Iterable
 import unicodedata
 
+from .lark import Lark
 from .tree import Tree
 from .visitors import Transformer_InPlace
-from .lexer import Token, PatternStr
-from .grammar import Terminal, NonTerminal
+from .lexer import Token, PatternStr, TerminalDef
+from .grammar import Terminal, NonTerminal, Symbol
 
 from .tree_matcher import TreeMatcher, is_discarded_terminal
 from .utils import is_id_continue
@@ -21,7 +23,7 @@ def is_iter_empty(i):
 class WriteTokensTransformer(Transformer_InPlace):
     "Inserts discarded tokens into their correct place, according to the rules of grammar"
 
-    def __init__(self, tokens, term_subs):
+    def __init__(self, tokens: Dict[str, TerminalDef], term_subs: Dict[str, Callable[[Symbol], str]]) -> None:
         self.tokens = tokens
         self.term_subs = term_subs
 
@@ -70,7 +72,7 @@ class Reconstructor(TreeMatcher):
         term_subs: a dictionary of [Terminal name as str] to [output text as str]
     """
 
-    def __init__(self, parser, term_subs=None):
+    def __init__(self, parser: Lark, term_subs: Dict[str, Callable[[Symbol], str]]=None) -> None:
         TreeMatcher.__init__(self, parser)
 
         self.write_tokens = WriteTokensTransformer({t.name:t for t in self.tokens}, term_subs or {})
@@ -87,7 +89,7 @@ class Reconstructor(TreeMatcher):
             else:
                 yield item
 
-    def reconstruct(self, tree, postproc=None, insert_spaces=True):
+    def reconstruct(self, tree: Tree, postproc: Callable[[Iterable[str]], Iterable[str]]=None, insert_spaces: bool=True) -> str:
         x = self._reconstruct(tree)
         if postproc:
             x = postproc(x)

lark/visitors.py

@@ -1,3 +1,4 @@
+from abc import ABC
 from functools import wraps
 
 from .utils import smart_decorator, combine_alternatives
@@ -7,7 +8,12 @@ from .lexer import Token
 
 ###{standalone
 from inspect import getmembers, getmro
+from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union
 
+_T = TypeVar('_T')
+_R = TypeVar('_R')
+_FUNC = Callable[..., _T]
+_DECORATED = Union[_FUNC, type]
 
 class Discard(Exception):
     """When raising the Discard exception in a transformer callback,
@@ -46,7 +52,7 @@ class _Decoratable:
         return cls
 
 
-class Transformer(_Decoratable):
+class Transformer(_Decoratable, ABC, Generic[_T]):
     """Transformers visit each node of the tree, and run the appropriate method on it according to the node's data.
 
     Methods are provided by the user via inheritance, and called according to ``tree.data``.
@@ -74,7 +80,7 @@ class Transformer(_Decoratable):
     """
     __visit_tokens__ = True   # For backwards compatibility
 
-    def __init__(self,  visit_tokens=True):
+    def __init__(self,  visit_tokens: bool=True) -> None:
         self.__visit_tokens__ = visit_tokens
 
     def _call_userfunc(self, tree, new_children=None):
@@ -125,11 +131,11 @@ class Transformer(_Decoratable):
         children = list(self._transform_children(tree.children))
         return self._call_userfunc(tree, children)
 
-    def transform(self, tree):
+    def transform(self, tree: Tree) -> _T:
         "Transform the given tree, and return the final result"
         return self._transform_tree(tree)
 
-    def __mul__(self, other):
+    def __mul__(self, other: 'Transformer[_T]') -> 'TransformerChain[_T]':
         """Chain two transformers together, returning a new transformer.
         """
         return TransformerChain(self, other)
@@ -149,16 +155,16 @@ class Transformer(_Decoratable):
         return token
 
 
-class TransformerChain(object):
+class TransformerChain(Generic[_T]):
     def __init__(self, *transformers):
-        self.transformers = transformers
+        self.transformers: Tuple[Transformer[_T], ...] = transformers
 
-    def transform(self, tree):
+    def transform(self, tree: Tree) -> _T:
         for t in self.transformers:
             tree = t.transform(tree)
         return tree
 
-    def __mul__(self, other):
+    def __mul__(self, other: Transformer[_T]) -> 'TransformerChain[_T]':
         return TransformerChain(*self.transformers + (other,))
 
 
@@ -239,19 +245,19 @@ class VisitorBase:
         return cls
 
 
-class Visitor(VisitorBase):
+class Visitor(VisitorBase, ABC, Generic[_T]):
     """Tree visitor, non-recursive (can handle huge trees).
 
     Visiting a node calls its methods (provided by the user via inheritance) according to ``tree.data``
     """
 
-    def visit(self, tree):
+    def visit(self, tree: Tree) -> Tree:
         "Visits the tree, starting with the leaves and finally the root (bottom-up)"
         for subtree in tree.iter_subtrees():
             self._call_userfunc(subtree)
         return tree
 
-    def visit_topdown(self,tree):
+    def visit_topdown(self, tree: Tree) -> Tree:
         "Visit the tree, starting at the root, and ending at the leaves (top-down)"
         for subtree in tree.iter_subtrees_topdown():
             self._call_userfunc(subtree)
@@ -266,7 +272,7 @@ class Visitor_Recursive(VisitorBase):
     Slightly faster than the non-recursive version.
     """
 
-    def visit(self, tree):
+    def visit(self, tree: Tree) -> Tree:
         "Visits the tree, starting with the leaves and finally the root (bottom-up)"
         for child in tree.children:
             if isinstance(child, Tree):
@@ -275,7 +281,7 @@ class Visitor_Recursive(VisitorBase):
         self._call_userfunc(tree)
         return tree
 
-    def visit_topdown(self,tree):
+    def visit_topdown(self,tree: Tree) -> Tree:
         "Visit the tree, starting at the root, and ending at the leaves (top-down)"
         self._call_userfunc(tree)
 
@@ -286,16 +292,7 @@ class Visitor_Recursive(VisitorBase):
         return tree
 
 
-def visit_children_decor(func):
-    "See Interpreter"
-    @wraps(func)
-    def inner(cls, tree):
-        values = cls.visit_children(tree)
-        return func(cls, values)
-    return inner
-
-
-class Interpreter(_Decoratable):
+class Interpreter(_Decoratable, ABC, Generic[_T]):
     """Interpreter walks the tree starting at the root.
 
     Visits the tree, starting with the root and finally the leaves (top-down)
@@ -307,7 +304,7 @@ class Interpreter(_Decoratable):
     This allows the user to implement branching and loops.
     """
 
-    def visit(self, tree):
+    def visit(self, tree: Tree) -> _T:
         f = getattr(self, tree.data)
         wrapper = getattr(f, 'visit_wrapper', None)
         if wrapper is not None:
@@ -315,7 +312,7 @@ class Interpreter(_Decoratable):
         else:
             return f(tree)
 
-    def visit_children(self, tree):
+    def visit_children(self, tree: Tree) -> List[_T]:
         return [self.visit(child) if isinstance(child, Tree) else child
                 for child in tree.children]
 
@@ -326,6 +323,16 @@ class Interpreter(_Decoratable):
         return self.visit_children(tree)
 
 
+_InterMethod = Callable[[Type[Interpreter], _T], _R]
+
+def visit_children_decor(func: _InterMethod) -> _InterMethod:
+    "See Interpreter"
+    @wraps(func)
+    def inner(cls, tree):
+        values = cls.visit_children(tree)
+        return func(cls, values)
+    return inner
+
 # Decorators
 
 def _apply_decorator(obj, decorator, **kwargs):
@@ -380,7 +387,7 @@ def _vargs_tree(f, data, children, meta):
     return f(Tree(data, children, meta))
 
 
-def v_args(inline=False, meta=False, tree=False, wrapper=None):
+def v_args(inline: bool=False, meta: bool=False, tree: bool=False, wrapper: Callable[[_DECORATED], _DECORATED]=None) -> Callable[[_DECORATED], _DECORATED]:
     """A convenience decorator factory for modifying the behavior of user-supplied visitor methods.
 
     By default, callback methods of transformers/visitors accept one argument - a list of the node's children.