Repository: lark-parser/lark
Pull request #1150 (2022-05-24 16:15:39)

Fix for PR #1149


Patch:
diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml
index 970b869..d078850 100644
--- a/.github/workflows/codecov.yml
+++ b/.github/workflows/codecov.yml
@@ -10,16 +10,14 @@ jobs:
       OS: ${{ matrix.os }}
       PYTHON: '3.7'
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v3
     - name: Download submodules
-      run: |
-        git submodule update --init --recursive
-        git submodule sync -q
-        git submodule update --init
+      with:
+        submodules: recursive
     - name: Setup Python
-      uses: actions/setup-python@master
+      uses: actions/setup-python@v3
       with:
-        python-version: 3.7
+        python-version: "3.7"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml
index f1c667f..21450d6 100644
--- a/.github/workflows/mypy.yml
+++ b/.github/workflows/mypy.yml
@@ -4,16 +4,8 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
-      - name: Download submodules
-        run: git submodule update --init --recursive
-      - name: Set up Python
-        uses: actions/setup-python@v1
+      - uses: actions/checkout@v3
         with:
-          python-version: 3.8
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install mypy
+          submodules: recursive
       - name: Lint with mypy
-        run: mypy -p lark || true
+        run: pipx run tox -e type
diff --git a/lark/lark.py b/lark/lark.py
index afebdd1..40b74be 100644
--- a/lark/lark.py
+++ b/lark/lark.py
@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 import sys, os, pickle, hashlib
 import tempfile
+import types
 from typing import (
     TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional, Sequence,
     Tuple, Iterable, IO, Any, TYPE_CHECKING, Collection
@@ -27,9 +28,10 @@ from .grammar import Rule
 
 import re
 try:
-    import regex  # type: ignore
+    import regex
+    _has_regex = True
 except ImportError:
-    regex = None
+    _has_regex = False
 
 
 ###{standalone
@@ -253,11 +255,12 @@ class Lark(Serialize):
 
     def __init__(self, grammar: 'Union[Grammar, str, IO[str]]', **options) -> None:
         self.options = LarkOptions(options)
+        re_module: types.ModuleType
 
         # Set regex or re module
         use_regex = self.options.regex
         if use_regex:
-            if regex:
+            if _has_regex:
                 re_module = regex
             else:
                 raise ImportError('`regex` module must be installed if calling `Lark(regex=True)`.')
@@ -267,7 +270,7 @@ class Lark(Serialize):
         # Some, but not all file-like objects have a 'name' attribute
         if self.options.source_path is None:
             try:
-                self.source_path = grammar.name
+                self.source_path = grammar.name  # type: ignore[union-attr]
             except AttributeError:
                 self.source_path = '<string>'
         else:
@@ -275,7 +278,7 @@ class Lark(Serialize):
 
         # Drain file-like objects to get their contents
         try:
-            read = grammar.read
+            read = grammar.read  # type: ignore[union-attr]
         except AttributeError:
             pass
         else:
diff --git a/lark/tree.py b/lark/tree.py
index 9e223a8..7ad620f 100644
--- a/lark/tree.py
+++ b/lark/tree.py
@@ -5,6 +5,7 @@ from typing import List, Callable, Iterator, Union, Optional, Generic, TypeVar,
 
 if TYPE_CHECKING:
     from .lexer import TerminalDef, Token
+    import rich
     if sys.version_info >= (3, 8):
         from typing import Literal
     else:
diff --git a/lark/tree_templates.py b/lark/tree_templates.py
index 25b4d24..5da70cf 100644
--- a/lark/tree_templates.py
+++ b/lark/tree_templates.py
@@ -9,7 +9,9 @@ from typing import Union, Optional, Mapping, Dict, Tuple, Iterator
 from lark import Tree, Transformer
 from lark.exceptions import MissingVariableError
 
+Branch = Union[Tree[str], str]
 TreeOrCode = Union[Tree[str], str]
+MatchResult = Dict[str, Tree]
 _TEMPLATE_MARKER = '$'
 
 
@@ -17,6 +19,8 @@ class TemplateConf:
     """Template Configuration
 
     Allows customization for different uses of Template
+
+    parse() must return a Tree instance.
     """
 
     def __init__(self, parse=None):
@@ -49,15 +53,21 @@ class TemplateConf:
             assert self._parse
             template = self._parse(template)
 
-        assert isinstance(template, Tree)
+        if not isinstance(template, Tree):
+            raise TypeError("template parser must return a Tree instance")
+
         return template
 
     def __call__(self, template: Tree[str]) -> 'Template':
         return Template(template, conf=self)
 
-    def _match_tree_template(self, template: TreeOrCode, tree: TreeOrCode) -> Optional[Dict[str, TreeOrCode]]:
+    def _match_tree_template(self, template: TreeOrCode, tree: Branch) -> Optional[MatchResult]:
+        """Returns dict of {var: match} if found a match, else None
+        """
         template_var = self.test_var(template)
         if template_var:
+            if not isinstance(tree, Tree):
+                raise TypeError(f"Template variables can only match Tree instances. Not {tree!r}")
             return {template_var: tree}
 
         if isinstance(template, str):
@@ -100,7 +110,7 @@ class _ReplaceVars(Transformer[str, Tree[str]]):
 
 
 class Template:
-    """Represents a tree templates, tied to a specific configuration
+    """Represents a tree template, tied to a specific configuration
 
     A tree template is a tree that contains nodes that are template variables.
     Those variables will match any tree.
@@ -111,7 +121,7 @@ class Template:
         self.conf = conf
         self.tree = conf._get_tree(tree)
 
-    def match(self, tree: TreeOrCode) -> Optional[Dict[str, TreeOrCode]]:
+    def match(self, tree: TreeOrCode) -> Optional[MatchResult]:
         """Match a tree template to a tree.
 
         A tree template without variables will only match ``tree`` if it is equal to the template.
@@ -127,7 +137,7 @@ class Template:
         tree = self.conf._get_tree(tree)
         return self.conf._match_tree_template(self.tree, tree)
 
-    def search(self, tree: TreeOrCode) -> Iterator[Tuple[Tree[str], Dict[str, TreeOrCode]]]:
+    def search(self, tree: TreeOrCode) -> Iterator[Tuple[Tree[str], MatchResult]]:
         """Search for all occurances of the tree template inside ``tree``.
         """
         tree = self.conf._get_tree(tree)
diff --git a/lark/utils.py b/lark/utils.py
index 1ba72f4..f9c0fd0 100644
--- a/lark/utils.py
+++ b/lark/utils.py
@@ -109,9 +109,10 @@ class SerializeMemoizer(Serialize):
 
 
 try:
-    import regex  # type: ignore
+    import regex
+    _has_regex = True
 except ImportError:
-    regex = None
+    _has_regex = False
 
 if sys.version_info >= (3, 11):
     import re._parser as sre_parse
@@ -123,7 +124,7 @@ else:
 categ_pattern = re.compile(r'\\p{[A-Za-z_]+}')
 
 def get_regexp_width(expr):
-    if regex:
+    if _has_regex:
         # Since `sre_parse` cannot deal with Unicode categories of the form `\p{Mn}`, we replace these with
         # a simple letter, which makes no difference as we are only trying to get the possible lengths of the regex
         # match here below.
@@ -135,7 +136,7 @@ def get_regexp_width(expr):
     try:
         return [int(x) for x in sre_parse.parse(regexp_final).getwidth()]
     except sre_constants.error:
-        if not regex:
+        if not _has_regex:
             raise ValueError(expr)
         else:
             # sre_parse does not support the new features in regex. To not completely fail in that case,
@@ -223,15 +224,16 @@ def combine_alternatives(lists):
 
 try:
     import atomicwrites
+    _has_atomicwrites = True
 except ImportError:
-    atomicwrites = None  # type: ignore[assigment]
+    _has_atomicwrites = False
 
 class FS:
     exists = staticmethod(os.path.exists)
 
     @staticmethod
     def open(name, mode="r", **kwargs):
-        if atomicwrites and "w" in mode:
+        if _has_atomicwrites and "w" in mode:
             return atomicwrites.atomic_write(name, mode=mode, overwrite=True, **kwargs)
         else:
             return open(name, mode, **kwargs)
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..e73c651
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,18 @@
+[build-system]
+requires = ["setuptools>=42"]
+build-backend = "setuptools.build_meta"
+
+
+[tool.mypy]
+files = "lark"
+python_version = "3.6"
+show_error_codes = true
+enable_error_code = ["ignore-without-code"]
+exclude = [
+  "^lark/__pyinstaller",
+]
+
+# You can disable imports or control per-module/file settings here
+[[tool.mypy.overrides]]
+module = [ "js2py" ]
+ignore_missing_imports = true
diff --git a/tox.ini b/tox.ini
index 04840eb..ef89565 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,16 +1,30 @@
 [tox]
-envlist = py36, py37, py38, py39, pypy, pypy3
-skip_missing_interpreters=true
+envlist = py36, py37, py38, py39, py310, pypy3, type
+skip_missing_interpreters = true
 
 [testenv]
 whitelist_externals = git
 deps =
     -rtest-requirements.txt
+passenv =
+    TERM
 
 # to always force recreation and avoid unexpected side effects
-recreate=True
+recreate = True
 
-commands=
+commands =
     git submodule sync -q
     git submodule update --init
     python -m tests {posargs}
+
+[testenv:type]
+description = run type check on code base
+skip_install = true
+recreate = false
+deps =
+    mypy==0.950
+    types-atomicwrites
+    types-regex
+    rich
+commands =
+    mypy


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 0066845..f15e487 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -6,17 +6,14 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9, 3.10.0-rc - 3.10, 3.11-dev, pypy3]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11-dev", "pypy-3.7"]
 
     steps:
-      - uses: actions/checkout@v2
-      - name: Download submodules
-        run: |
-          git submodule update --init --recursive
-          git submodule sync -q
-          git submodule update --init
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -25,4 +22,4 @@ jobs:
           pip install -r test-requirements.txt
       - name: Run tests
         run: |
-          python -m tests
\ No newline at end of file
+          python -m tests
diff --git a/tests/test_nearley/nearley b/tests/test_nearley/nearley
index a46b374..3268316 160000
--- a/tests/test_nearley/nearley
+++ b/tests/test_nearley/nearley
@@ -1,1 +1,1 @@
-Subproject commit a46b37471db486db0f6e1ce6a2934fb238346b44
+Subproject commit 326831689826cb1b9a4d21d1ce0d5db9278e9636
diff --git a/tests/test_tree_templates.py b/tests/test_tree_templates.py
index fa9578e..ae3c3e0 100644
--- a/tests/test_tree_templates.py
+++ b/tests/test_tree_templates.py
@@ -87,6 +87,12 @@ class TestTreeTemplatesConf(unittest.TestCase):
 
         self.assertEqual({}, template.match(SOME_NON_TEMPLATE_TREE))
 
+    def test_template_match__only_tree(self):
+        "This test might become irrelevant in the future"
+        template_tree = Tree('bar', [Tree("var", children=["$foo"])])
+        template = Template(template_tree)
+        self.assertRaises(TypeError, template.match, Tree('bar', ['BAD']))
+
 
 class TestTreeTemplatesTemplate(unittest.TestCase):
     parser = Lark(SOME_TEMPLATING_GRAMMAR)