Repository: scipy/scipy_doctest (language: python)

Issue:
Configure pytest to use the DTParser



Discussion / hints:

Fix patch:
diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml
index e7c2a62..926773e 100644
--- a/.github/workflows/pip.yml
+++ b/.github/workflows/pip.yml
@@ -66,4 +66,7 @@ jobs:
     - name: Test pytest plugin
       # This test will fail in a venv where scpdt has not been installed and the plugin has not been activated
       run: |
-        python -m pytest scpdt/tests/module_cases.py --doctest-modules
+        test_files=("scpdt/tests/module_cases.py" "scpdt/tests/stopwords_cases.py")
+          for file in "${test_files[@]}"; do
+            python -m pytest "${file}" --doctest-modules
+          done
diff --git a/scpdt/plugin.py b/scpdt/plugin.py
index 93ff2f8..3ad1614 100644
--- a/scpdt/plugin.py
+++ b/scpdt/plugin.py
@@ -3,21 +3,121 @@
 """
 
 from _pytest import doctest
+from _pytest.doctest import DoctestModule, DoctestTextfile
+from _pytest.pathlib import import_path
+from _pytest.outcomes import skip
 
-from scpdt.impl import DTChecker, DTConfig
+from scpdt.impl import DTChecker, DTConfig, DTParser, DTFinder
 
 
 def pytest_configure(config):
     """
     Allow plugins and conftest files to perform initial configuration.
     """
-    if config.pluginmanager.getplugin('doctest'):
-        config.pluginmanager.register(DTChecker())
 
     doctest._get_checker = _get_checker
+    doctest.DoctestModule = DTModule
+    doctest.DoctestTextfile = DTTextfile
+
 
 def _get_checker():
     """
     Override function to return an instance of DTChecker with default configurations
     """
     return DTChecker(config=DTConfig())
+
+
+class DTModule(DoctestModule):
+    """
+    This class extends the DoctestModule class provided by pytest. 
+    The purpose of DTModule is to override the behavior of the collect method, 
+    which is called by pytest to collect and generate test items for doctests in the 
+    specified module or file.
+    """
+    def collect(self):
+        # This code is copy-pasted from the `_pytest.doctest` module(pytest 7.4.0):
+        # https://github.com/pytest-dev/pytest/blob/448563caaac559b8a3195edc58e8806aca8d2c71/src/_pytest/doctest.py#L497
+        if self.path.name == "setup.py":
+            return
+        if self.path.name == "conftest.py":
+            module = self.config.pluginmanager._importconftest(
+                self.path,
+                self.config.getoption("importmode"),
+                rootpath=self.config.rootpath
+            )
+        else:
+            try:
+                module = import_path(
+                    self.path,
+                    root=self.config.rootpath,
+                    mode=self.config.getoption("importmode"),
+                )
+            except ImportError:
+                if self.config.getvalue("doctest_ignore_import_errors"):
+                    skip("unable to import module %r" % self.path)
+                else:
+                    raise
+
+        # The `_pytest.doctest` module uses the internal doctest parsing mechanism.
+        # We plugin scpdt's `DTFinder` that uses the `DTParser` which parses the doctest examples 
+        # from the python module or file and filters out stopwords and pseudocode.
+        finder = DTFinder(config=DTConfig())
+
+        # the rest remains unchanged
+        optionflags = doctest.get_optionflags(self)
+        runner = doctest._get_runner(
+            verbose=False,
+            optionflags=optionflags,
+            checker=_get_checker(),
+            continue_on_failure=doctest._get_continue_on_failure(self.config),
+        )
+
+        for test in finder.find(module, module.__name__):
+            if test.examples:  # skip empty doctests
+                yield doctest.DoctestItem.from_parent(
+                    self, name=test.name, runner=runner, dtest=test
+                )
+
+
+class DTTextfile(DoctestTextfile):
+    """
+    This class extends the DoctestTextfile class provided by pytest. 
+    The purpose of DTTextfile is to override the behavior of the collect method, 
+    which is called by pytest to collect and generate test items for doctests in 
+    the specified text files.
+    """
+    def collect(self):
+        # This code is copy-pasted from `_pytest.doctest` module(pytest 7.4.0):
+        # https://github.com/pytest-dev/pytest/blob/448563caaac559b8a3195edc58e8806aca8d2c71/src/_pytest/doctest.py#L417
+        encoding = self.config.getini("doctest_encoding")
+        text = self.path.read_text(encoding)
+        filename = str(self.path)
+        name = self.path.name
+        globs = {"__name__": "__main__"}
+
+        optionflags = doctest.get_optionflags(self)
+
+        runner = doctest._get_runner(
+            verbose=False,
+            optionflags=optionflags,
+            checker=_get_checker(),
+            continue_on_failure=doctest._get_continue_on_failure(self.config)
+        )
+
+        # We plug in an instance of `DTParser` which parses the doctest examples from the text file and
+        # filters out stopwords and pseudocode.
+        parser = _get_parser()
+
+        # the rest remains unchanged
+        test = parser.get_doctest(text, globs, name, filename, 0)
+        if test.examples:
+            yield doctest.DoctestItem.from_parent(
+                self, name=test.name, runner=runner, dtest=test
+            )
+
+
+def _get_parser():
+    """
+    Return instance of DTParser with default configuration
+    """
+    return DTParser(config=DTConfig())
\ No newline at end of file


Test patch:
diff --git a/scpdt/tests/test_pytest_configuration.py b/scpdt/tests/test_pytest_configuration.py
index 0a0682e..412533f 100644
--- a/scpdt/tests/test_pytest_configuration.py
+++ b/scpdt/tests/test_pytest_configuration.py
@@ -2,7 +2,7 @@
 from pathlib import PosixPath
 
 
-from . import module_cases, failure_cases, failure_cases_2
+from . import module_cases, failure_cases, failure_cases_2, stopwords_cases
 
 pytest_plugins = ['pytester']
 
@@ -23,3 +23,13 @@ def test_failure_cases(pytester):
         python_file = PosixPath(path_str)
         result = pytester.inline_run(python_file, "--doctest-modules")
     assert result.ret == pytest.ExitCode.TESTS_FAILED
+    
+
+"""
+Test that pytest uses the DTParser for doctests
+"""
+def test_stopword_cases(pytester):
+    path_str = stopwords_cases.__file__
+    python_file = PosixPath(path_str)
+    result = pytester.inline_run(python_file, "--doctest-modules")
+    assert result.ret == pytest.ExitCode.OK