Repository: scipy/scipy

Issue:
Check for non-running tests because of test function name and fix them up
It looks like we have some issues in the test suite with non-running tests. @MarcoGorelli just wrote a script to check this, see `scripts/check_test_naming.py` in https://github.com/pandas-dev/pandas/pull/50397. 

I verified that we've got at least one such test (see https://github.com/rgommers/scipy/tree/fix-broken-test-name), so there are probably more. It'd be good if we could take over the test script from Pandas, put it in `tools/`, fix whatever issues that turns up, and then run the script in exactly one CI job (next to refguide check probably).


Discussion:
Should we not just rely on coverage of tests as @MarcoGorelli suggests instead of running this tool?
No one is going to look through all coverage results for PRs, that's way too much work. And it's not possible to rely on a coverage threshold either. So someone can look through coverage results once manually and fix what's not running, but that's about it.

If we judge that this script is too maintenance-intensive, then it's fine to reject the idea. But I think it's either that, or the status quo.
There's quite a few more:

https://github.com/scipy/scipy/blob/6a555164767a6624ba3721edd8c3f580f97a85f3/scipy/stats/tests/test_distributions.py#L6743-L6750

https://github.com/scipy/scipy/blob/6a555164767a6624ba3721edd8c3f580f97a85f3/scipy/stats/tests/test_distributions.py#L4402-L4415

https://github.com/scipy/scipy/blob/6a555164767a6624ba3721edd8c3f580f97a85f3/scipy/stats/tests/test_distributions.py#L1166-L1172

https://github.com/scipy/scipy/blob/4e738d77dd217304a564c4269c99f2f4332f02a8/scipy/stats/tests/test_continuous_basic.py#L638-L644

https://github.com/scipy/scipy/blob/4c53ae1f970cf8c3a8e5d624c8043af9fe465db7/scipy/special/tests/test_orthogonal.py#L211-L238

and many, many more...all the ones I checked were genuine, though if there are any false positives, you can prevent the check from flaggin them with a `# not a test` comment

---

@tupui yeah if you're close enough to 100% coverage that you can enable it, that would be a better solution. In meantime, I've made this a bit more generic and have put it in its own package: https://github.com/MarcoGorelli/madforhooks#check-test-naming

Resolving patch:
diff --git a/dev.py b/dev.py
--- a/dev.py
+++ b/dev.py
@@ -861,6 +861,14 @@ def run(cls, **kwargs):
 ###################
 # linters
 
+def emit_cmdstr(cmd):
+    """Print the command that's being run to stdout"""
+    console = Console(theme=console_theme)
+    # The [cmd] square brackets controls the font styling, typically in italics
+    # to differentiate it from other stdout content
+    console.print(f"{EMOJI.cmd} [cmd] {cmd}")
+
+
 @task_params([{'name': 'output_file', 'long': 'output-file', 'default': None,
                'help': 'Redirect report to a file'}])
 def task_flake8(output_file):
@@ -868,8 +876,11 @@ def task_flake8(output_file):
     opts = ''
     if output_file:
         opts += f'--output-file={output_file}'
+
+    cmd = f"flake8 {opts} scipy benchmarks/benchmarks"
+    emit_cmdstr(f"{cmd}")
     return {
-        'actions': [f"flake8 {opts} scipy benchmarks/benchmarks"],
+        'actions': [cmd],
         'doc': 'Lint scipy and benchmarks directory',
     }
 
@@ -877,6 +888,7 @@ def task_flake8(output_file):
 def task_pep8diff():
     # Lint just the diff since branching off of main using a
     # stricter configuration.
+    emit_cmdstr(os.path.join('tools', 'lint_diff.py'))
     return {
         'basename': 'pep8-diff',
         'actions': [str(Dirs().root / 'tools' / 'lint_diff.py')],
@@ -885,6 +897,7 @@ def task_pep8diff():
 
 
 def task_unicode_check():
+    emit_cmdstr(os.path.join('tools', 'unicode-check.py'))
     return {
         'basename': 'unicode-check',
         'actions': [str(Dirs().root / 'tools' / 'unicode-check.py')],
@@ -893,16 +906,30 @@ def task_unicode_check():
     }
 
 
+def task_check_test_name():
+    emit_cmdstr(os.path.join('tools', 'check_test_name.py'))
+    return {
+        "basename": "check-testname",
+        "actions": [str(Dirs().root / "tools" / "check_test_name.py")],
+        "doc": "Check tests are correctly named so that pytest runs them."
+    }
+
+
 @cli.cls_cmd('lint')
 class Lint():
     """:dash: Run flake8, check PEP 8 compliance on branch diff and check for
-    disallowed Unicode characters."""
+    disallowed Unicode characters and possibly-invalid test names."""
     output_file = Option(
         ['--output-file'], default=None, help='Redirect report to a file')
 
     def run(output_file):
         opts = {'output_file': output_file}
-        run_doit_task({'flake8': opts, 'pep8-diff': {}, 'unicode-check': {}})
+        run_doit_task({
+            'flake8': opts,
+            'pep8-diff': {},
+            'unicode-check': {},
+            'check-testname': {},
+        })
 
 
 @cli.cls_cmd('mypy')
@@ -938,6 +965,7 @@ def run(cls, **kwargs):
             os.environ['MYPY_FORCE_COLOR'] = '1'
             # Change to the site directory to make sure mypy doesn't pick
             # up any type stubs in the source tree.
+            emit_cmdstr(f"mypy.api.run --config-file {config} {check_path}")
             report, errors, status = mypy.api.run([
                 "--config-file",
                 str(config),
diff --git a/tools/lint_diff.py b/tools/lint_diff.py
--- a/tools/lint_diff.py
+++ b/tools/lint_diff.py
@@ -93,8 +93,7 @@ def main():
     rc, errors = run_flake8(diff)
     if errors:
         print(errors)
-    else:
-        print("No lint errors found.")
+
     sys.exit(rc)
 
 


Test patch:
diff --git a/scipy/optimize/tests/test__numdiff.py b/scipy/optimize/tests/test__numdiff.py
--- a/scipy/optimize/tests/test__numdiff.py
+++ b/scipy/optimize/tests/test__numdiff.py
@@ -215,9 +215,6 @@ def jac_zero_jacobian(self, x):
             [-x[1] * np.sin(x[0] * x[1]), -x[0] * np.sin(x[0] * x[1])]
         ])
 
-    def fun_non_numpy(self, x):
-        return math.exp(x)
-
     def jac_non_numpy(self, x):
         # x can be a scalar or an array [val].
         # Cast to true scalar before handing over to math.exp
diff --git a/scipy/optimize/tests/test__spectral.py b/scipy/optimize/tests/test__spectral.py
--- a/scipy/optimize/tests/test__spectral.py
+++ b/scipy/optimize/tests/test__spectral.py
@@ -149,7 +149,8 @@ def x0_2(n):
     x0.fill(1/n**2)
     return x0
 
-def F_4(x, n):
+
+def F_4(x, n):  # skip name check
     assert_equal(n % 3, 0)
     g = np.zeros([n])
     # Note: the first line is typoed in some of the references;
@@ -159,7 +160,8 @@ def F_4(x, n):
     g[2::3] = 1.25 * x[2::3] - 0.25*x[2::3]**3
     return g
 
-def x0_4(n):
+
+def x0_4(n):  # skip name check
     assert_equal(n % 3, 0)
     x0 = np.array([-1, 1/2, -1] * (n//3))
     return x0
diff --git a/scipy/optimize/tests/test_linesearch.py b/scipy/optimize/tests/test_linesearch.py
--- a/scipy/optimize/tests/test_linesearch.py
+++ b/scipy/optimize/tests/test_linesearch.py
@@ -53,19 +53,19 @@ def assert_fp_equal(x, y, err_msg="", nulp=50):
 
 class TestLineSearch:
     # -- scalar functions; must have dphi(0.) < 0
-    def _scalar_func_1(self, s):
+    def _scalar_func_1(self, s):  # skip name check
         self.fcount += 1
         p = -s - s**3 + s**4
         dp = -1 - 3*s**2 + 4*s**3
         return p, dp
 
-    def _scalar_func_2(self, s):
+    def _scalar_func_2(self, s):  # skip name check
         self.fcount += 1
         p = np.exp(-4*s) + s**2
         dp = -4*np.exp(-4*s) + 2*s
         return p, dp
 
-    def _scalar_func_3(self, s):
+    def _scalar_func_3(self, s):  # skip name check
         self.fcount += 1
         p = -np.sin(10*s)
         dp = -10*np.cos(10*s)
@@ -73,13 +73,13 @@ def _scalar_func_3(self, s):
 
     # -- n-d functions
 
-    def _line_func_1(self, x):
+    def _line_func_1(self, x):  # skip name check
         self.fcount += 1
         f = np.dot(x, x)
         df = 2*x
         return f, df
 
-    def _line_func_2(self, x):
+    def _line_func_2(self, x):  # skip name check
         self.fcount += 1
         f = np.dot(x, np.dot(self.A, x)) + 1
         df = np.dot(self.A + self.A.T, x)
diff --git a/scipy/sparse/tests/test_sparsetools.py b/scipy/sparse/tests/test_sparsetools.py
--- a/scipy/sparse/tests/test_sparsetools.py
+++ b/scipy/sparse/tests/test_sparsetools.py
@@ -220,7 +220,7 @@ def get_matrix():
         finally:
             gc.collect()
 
-    def _check_bsr_matvecs(self, m):
+    def _check_bsr_matvecs(self, m):  # skip name check
         m = m()
         n = self.n
 
@@ -228,7 +228,7 @@ def _check_bsr_matvecs(self, m):
         r = m.dot(np.ones((n, 2), dtype=np.int8))
         assert_equal(r[0, 0], int_to_int8(n))
 
-    def _check_bsr_matvec(self, m):
+    def _check_bsr_matvec(self, m):  # skip name check
         m = m()
         n = self.n
 
@@ -236,7 +236,7 @@ def _check_bsr_matvec(self, m):
         r = m.dot(np.ones((n,), dtype=np.int8))
         assert_equal(r[0], int_to_int8(n))
 
-    def _check_bsr_diagonal(self, m):
+    def _check_bsr_diagonal(self, m):  # skip name check
         m = m()
         n = self.n
 
@@ -244,17 +244,17 @@ def _check_bsr_diagonal(self, m):
         r = m.diagonal()
         assert_equal(r, np.ones(n))
 
-    def _check_bsr_sort_indices(self, m):
+    def _check_bsr_sort_indices(self, m):  # skip name check
         # _sort_indices
         m = m()
         m.sort_indices()
 
-    def _check_bsr_transpose(self, m):
+    def _check_bsr_transpose(self, m):  # skip name check
         # _transpose
         m = m()
         m.transpose()
 
-    def _check_bsr_matmat(self, m):
+    def _check_bsr_matmat(self, m):  # skip name check
         m = m()
         n = self.n
 
diff --git a/scipy/special/tests/test_basic.py b/scipy/special/tests/test_basic.py
--- a/scipy/special/tests/test_basic.py
+++ b/scipy/special/tests/test_basic.py
@@ -835,7 +835,7 @@ def test_pro_ang1_cv(self):
         assert_array_almost_equal(cephes.pro_ang1_cv(1,1,1,1,0),
                                   array((1.0,0.0)))
 
-    def _check_pro_cv(self):
+    def test_pro_cv(self):
         assert_equal(cephes.pro_cv(1,1,0),2.0)
 
     def test_pro_rad1(self):
diff --git a/scipy/special/tests/test_data.py b/scipy/special/tests/test_data.py
--- a/scipy/special/tests/test_data.py
+++ b/scipy/special/tests/test_data.py
@@ -10,7 +10,7 @@
     jn, jv, jvp, yn, yv, yvp, iv, ivp, kn, kv, kvp,
     gamma, gammaln, gammainc, gammaincc, gammaincinv, gammainccinv, digamma,
     beta, betainc, betaincinv, poch,
-    ellipe, ellipeinc, ellipk, ellipkm1, ellipkinc, ellipj,
+    ellipe, ellipeinc, ellipk, ellipkm1, ellipkinc,
     elliprc, elliprd, elliprf, elliprg, elliprj,
     erf, erfc, erfinv, erfcinv, exp1, expi, expn,
     bdtrik, btdtr, btdtri, btdtria, btdtrib, chndtr, gdtr, gdtrc, gdtrix, gdtrib,
@@ -64,10 +64,6 @@ def ellipeinc_(f, k):
     return ellipeinc(f, k*k)
 
 
-def ellipj_(k):
-    return ellipj(k*k)
-
-
 def zeta_(x):
     return zeta(x, 1.)
 
diff --git a/tools/check_test_name.py b/tools/check_test_name.py
new file mode 100755
--- /dev/null
+++ b/tools/check_test_name.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python
+
+"""
+MIT License
+
+Copyright (c) 2020 Marco Gorelli
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+Check that test names start with `test`, and that test classes start with
+`Test`.
+"""
+from __future__ import annotations
+
+import ast
+import os
+from pathlib import Path
+import sys
+from typing import Iterator, Sequence
+import itertools
+
+PRAGMA = "# skip name check"
+
+
+def _find_names(node: ast.Module) -> Iterator[str]:
+    for _node in ast.walk(node):
+        if isinstance(_node, ast.Name):
+            yield _node.id
+        elif isinstance(_node, ast.Attribute):
+            yield _node.attr
+
+
+def _is_fixture(node: ast.expr) -> bool:
+    if isinstance(node, ast.Call):
+        node = node.func
+    return (
+        isinstance(node, ast.Attribute)
+        and node.attr == "fixture"
+        and isinstance(node.value, ast.Name)
+        and node.value.id == "pytest"
+    )
+
+
+def is_misnamed_test_func(
+    node: ast.expr | ast.stmt, names: Sequence[str], line: str
+) -> bool:
+    return (
+        isinstance(node, ast.FunctionDef)
+        and not node.name.startswith("test")
+        and names.count(node.name) == 0
+        and not any(
+            _is_fixture(decorator) for decorator in node.decorator_list
+            )
+        and PRAGMA not in line
+        and node.name
+        not in ("teardown_method", "setup_method",
+                "teardown_class", "setup_class",
+                "setup_module", "teardown_module")
+    )
+
+
+def is_misnamed_test_class(
+    node: ast.expr | ast.stmt, names: Sequence[str], line: str
+) -> bool:
+    return (
+        isinstance(node, ast.ClassDef)
+        and not node.name.startswith("Test")
+        and names.count(node.name) == 0
+        and PRAGMA not in line
+        # Some of the KDTreeTest use a decorator to setup tests so these are
+        # actually fine
+        and "KDTreeTest" not in [
+            decorator.id for decorator in node.decorator_list
+            ]
+    )
+
+
+def main(content: str, file: str) -> int:
+    lines = content.splitlines()
+    tree = ast.parse(content)
+    names = list(_find_names(tree))
+    ret = 0
+    for node in tree.body:
+        if is_misnamed_test_func(node, names, lines[node.lineno - 1]):
+            print(
+                f"{file}:{node.lineno}:{node.col_offset} "
+                f"found test function '{node.name}' which does not start with"
+                " 'test'"
+            )
+            ret = 1
+        elif is_misnamed_test_class(node, names, lines[node.lineno - 1]):
+            print(
+                f"{file}:{node.lineno}:{node.col_offset} "
+                f"found test class '{node.name}' which does not start with"
+                " 'Test'"
+            )
+            ret = 1
+        if (
+            isinstance(node, ast.ClassDef)
+            and names.count(node.name) == 0
+            and PRAGMA not in lines[node.lineno - 1]
+        ):
+            for _node in node.body:
+                if is_misnamed_test_func(_node, names,
+                                         lines[_node.lineno - 1]):
+                    # It could be that this function is used somewhere by the
+                    # parent class. For example, there might be a base class
+                    # with
+                    #
+                    # class Foo:
+                    #     def foo(self):
+                    #          assert 1+1==2
+                    #     def test_foo(self):
+                    #         self.foo()
+                    #
+                    # and then some subclass overwrites `foo`. So, we check
+                    # that `self.foo` doesn't appear in any of the test
+                    # classes. Note some false negatives might get through,
+                    # but that's OK. This is good enough that has helped
+                    # identify several examples of tests not being run.
+                    should_continue = False
+                    for _file in itertools.chain(
+                        Path("scipy").rglob("**/tests/**/test*.py"),
+                        ["scipy/_lib/_testutils.py"],
+                    ):
+                        with open(os.path.join(_file)) as fd:
+                            _content = fd.read()
+                        if f"self.{_node.name}" in _content:
+                            should_continue = True
+                            break
+                    if should_continue:
+                        continue
+
+                    print(
+                        f"{file}:{_node.lineno}:{_node.col_offset} "
+                        f"found test function '{_node.name}' which does not "
+                        "start with 'test'"
+                    )
+                    ret = 1
+    return ret
+
+
+if __name__ == "__main__":
+    ret = 0
+    path = Path("scipy").rglob("**/tests/**/test*.py")
+
+    for file in path:
+        filename = os.path.basename(file)
+        with open(file, encoding="utf-8") as fd:
+            content = fd.read()
+        ret |= main(content, file)
+
+    sys.exit(ret)