Repository: scipy/scipy_doctest (language: python)

Issue:
Replicate the user context manager
https://github.com/scipy/scipy/blob/5b4b3ec030f4d27f6bb1b4c4ec53a92f83e648ef/tools/doctest_public_modules.py#L54


Discussion / hints:
@ev-br 
Should this context manager: [warnings_errors_and_rng](https://github.com/scipy/scipy/blob/5b4b3ec030f4d27f6bb1b4c4ec53a92f83e648ef/tools/doctest_public_modules.py#L54) be copied to scpdt's util.py module?
Not sure. This looks somewhat scipy-specific. If the tool itself is meant to be cross-project, why others need to know about scipy-specific names and exceptions? 

Off the cuff, this looks like a material for project-specific conftest.py (or however other means of project-specific configuration). WDYT? 
That is true. I think we will know for sure if we need to implement a project-agnostic version of this after the integration.
We do want a project-agnostic version + a clear way to add project specifics. 
Okay, I think this is my next task then.
@ev-br I've noticed that this [context manager](https://github.com/scipy/scipy/blob/5b4b3ec030f4d27f6bb1b4c4ec53a92f83e648ef/tools/doctest_public_modules.py#L54) is a combination of scpdt util's [scipy_rndm_state](https://github.com/ev-br/scpdt/blob/31d54d73c7079466292f4499e566f115d8001994/scpdt/util.py#L74) and [warnings_errors](https://github.com/ev-br/scpdt/blob/31d54d73c7079466292f4499e566f115d8001994/scpdt/util.py#L116) context managers but with a `known_warnings` dictionary for warnings that should be ignored. 

So my goal is to ensure projects can use this context manager by specifying their own list of functions whose warnings should be ignored?
The goal is that projects can specify the context where their doctests are run. This may or may not include warnings->errors, manipulations of the random state etc etc. This is up to a project.

What the cross-project tool needs to do is:
- restore the numpy error state in case a test manipulates it
- make MPL gui-less
- allow a user project to plug in their own context manager

cf https://github.com/ev-br/scpdt/blob/main/scpdt/frontend.py#L247 

EDIT: Maybe this logic needs to live in the runner which is responsible for actually running tests. Or an autofixture.

Fix patch:
diff --git a/scpdt/impl.py b/scpdt/impl.py
index 0bba2db..1ec5c42 100644
--- a/scpdt/impl.py
+++ b/scpdt/impl.py
@@ -375,7 +375,7 @@ class DebugDTRunner(DTRunner):
     Almost verbatim copy of `doctest.DebugRunner`.
     """
     def run(self, test, compileflags=None, out=None, clear_globs=True):
-        r = super().run(test, compileflags, out, False)
+        r = super().run(test, compileflags=compileflags, out=out, clear_globs=clear_globs)
         if clear_globs:
             test.globs.clear()
         return r
diff --git a/scpdt/plugin.py b/scpdt/plugin.py
index 3280b39..5038d3a 100644
--- a/scpdt/plugin.py
+++ b/scpdt/plugin.py
@@ -1,16 +1,19 @@
 """
 A pytest plugin that provides enhanced doctesting for Pydata libraries
 """
+import bdb
 import os
 import shutil
 
-from _pytest import doctest
+from _pytest import doctest, outcomes
 from _pytest.doctest import DoctestModule, DoctestTextfile
 from _pytest.pathlib import import_path
-from _pytest.outcomes import skip
+from _pytest.outcomes import skip, OutcomeException
 
-from scpdt.impl import DTChecker, DTParser, DTFinder
+from scpdt.impl import DTChecker, DTParser, DTFinder, DebugDTRunner
 from scpdt.conftest import dt_config
+from .util import np_errstate, matplotlib_make_nongui
+
 
 copied_files = []
 
@@ -19,6 +22,7 @@ def pytest_configure(config):
     Allow plugins and conftest files to perform initial configuration.
     """
 
+    doctest._get_runner = _get_runner
     doctest._get_checker = _get_checker
     doctest.DoctestModule = DTModule
     doctest.DoctestTextfile = DTTextfile
@@ -39,7 +43,7 @@ def pytest_unconfigure(config):
 
 def _get_checker():
     """
-    Override function to return an instance of DTChecker with default configurations
+    Override function to return an instance of DTChecker
     """
     return DTChecker(config=dt_config)
 
@@ -99,16 +103,16 @@ def collect(self):
         # We plugin scpdt's `DTFinder` that uses the `DTParser` which parses the doctest examples 
         # from the python module or file and filters out stopwords and pseudocode.
         finder = DTFinder(config=dt_config)
-
-        # the rest remains unchanged
         optionflags = doctest.get_optionflags(self)
-        runner = doctest._get_runner(
+
+        # We plug in `PytestDTRunner`
+        runner = _get_runner(
             verbose=False,
             optionflags=optionflags,
-            checker=_get_checker(),
-            continue_on_failure=doctest._get_continue_on_failure(self.config),
+            checker=_get_checker()
         )
 
+        # the rest remains unchanged
         for test in finder.find(module, module.__name__):
             if test.examples:  # skip empty doctests
                 yield doctest.DoctestItem.from_parent(
@@ -138,11 +142,11 @@ def collect(self):
         if dt_config.local_resources:
             copy_local_files(dt_config.local_resources, os.getcwd())
 
-        runner = doctest._get_runner(
+        # We plug in `PytestDTRunner`
+        runner = _get_runner(
             verbose=False,
             optionflags=optionflags,
-            checker=_get_checker(),
-            continue_on_failure=doctest._get_continue_on_failure(self.config)
+            checker=_get_checker()
         )
 
         # We plug in an instance of `DTParser` which parses the doctest examples from the text file and
@@ -159,6 +163,55 @@ def collect(self):
 
 def _get_parser():
     """
-    Return instance of DTParser with default configuration
+    Return instance of DTParser
+    Please refer to `testmod` source for a discussion of the order of context managers.
+    """
+    return DTParser(config=dt_config)
+
+
+def _get_runner(checker, verbose, optionflags):
+    import doctest
+    """
+    Override function to return instance of PytestDTRunner
     """
-    return DTParser(config=dt_config)
\ No newline at end of file
+    class PytestDTRunner(DebugDTRunner):
+        def run(self, test, compileflags=None, out=None, clear_globs=False):
+            """
+            Run tests in context managers.
+            Restore the errstate/print state after each docstring.
+            Also make MPL backend non-GUI and close the figures.
+            The order of context managers is actually relevant. Consider
+            user_context_mgr that turns warnings into errors.
+            Additionally, suppose that MPL deprecates something and plt.something
+            starts issuing warngings. Now all of those become errors
+            *unless* the `mpl()` context mgr has a chance to filter them out
+            *before* they become errors in `config.user_context_mgr()`.
+            """
+            with np_errstate():
+                with dt_config.user_context_mgr(test):
+                    with matplotlib_make_nongui():
+                        super().run(test, compileflags=compileflags, out=out, clear_globs=clear_globs)
+
+        """
+        Almost verbatim copy of `_pytest.doctest.PytestDoctestRunner` except we utilize
+        DTConfig's `nameerror_after_exception` attribute in place of doctest's `continue_on_failure`.
+        """
+        def report_failure(self, out, test, example, got):
+            failure = doctest.DocTestFailure(test, example, got)
+            if dt_config.nameerror_after_exception:
+                out.append(failure)
+            else:
+                raise failure
+
+        def report_unexpected_exception(self, out, test, example, exc_info):
+            if isinstance(exc_info[1], OutcomeException):
+                raise exc_info[1]
+            if isinstance(exc_info[1], bdb.BdbQuit):
+                outcomes.exit("Quitting debugger")
+            failure = doctest.UnexpectedException(test, example, exc_info)
+            if dt_config.nameerror_after_exception:
+                out.append(failure)
+            else:
+                raise failure
+            
+    return PytestDTRunner(checker=checker, verbose=verbose, optionflags=optionflags, config=dt_config)
\ No newline at end of file


Test patch:
diff --git a/scpdt/conftest.py b/scpdt/conftest.py
index d0ba022..ae51d2e 100644
--- a/scpdt/conftest.py
+++ b/scpdt/conftest.py
@@ -8,4 +8,4 @@
                                   ['scpdt/tests/local_file.txt'],
                             'scpdt.tests.local_file_cases.sio':
                                   ['scpdt/tests/octave_a.mat']   
-                                  }
\ No newline at end of file
+                                  }