Repository: certbot/certbot

Issue:
Single test failure with pytest 6.2.2
Hi,

I'm seeing an issue where the builds of certbot in nixpkgs (the package repository using the nix package manager) recently started failing on our CI.

Here's a link which shows the status over time, and from there you can navigate to the individual log files: https://hydra.nixos.org/job/nixpkgs/trunk/python38Packages.certbot.x86_64-linux

Here's one of the log files: https://nix-cache.s3.amazonaws.com/log/ih5zphxw85jzw1xg8h1ldjmqkc9sa18i-python3.8-certbot-1.12.0.drv

Nixpkgs generally stays extremely up-to-date with upstream package versions (although it looks like we were one version behind with certbot -- I've verified manually that the issue I'm reporting also seems to apply to the latest version). The certbot failures began to occur due to, I believe, our upgrade of pytest from 6.1.2. to 6.2.2.

The test error looks like this:
```
python3.8-certbot> ____________________________ MainTest.test_register ____________________________
python3.8-certbot> cls = <class '_pytest.runner.CallInfo'>
python3.8-certbot> func = <function call_runtest_hook.<locals>.<lambda> at 0x7ffff5493b80>
python3.8-certbot> when = 'call'
python3.8-certbot> reraise = (<class '_pytest.outcomes.Exit'>, <class 'KeyboardInterrupt'>)
python3.8-certbot> @classmethod
python3.8-certbot> def from_call(
python3.8-certbot> cls,
python3.8-certbot> func: "Callable[[], TResult]",
python3.8-certbot> when: "Literal['collect', 'setup', 'call', 'teardown']",
python3.8-certbot> reraise: Optional[
python3.8-certbot> Union[Type[BaseException], Tuple[Type[BaseException], ...]]
python3.8-certbot> ] = None,
python3.8-certbot> ) -> "CallInfo[TResult]":
python3.8-certbot> excinfo = None
python3.8-certbot> start = timing.time()
python3.8-certbot> precise_start = timing.perf_counter()
python3.8-certbot> try:
python3.8-certbot> >           result: Optional[TResult] = func()
python3.8-certbot> /nix/store/wjyqgb6545zj0xs2iyp8kplvlshb5li7-python3.8-pytest-6.2.2/lib/python3.8/site-packages/_pytest/runner.py:311:
python3.8-certbot> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
python3.8-certbot> /nix/store/wjyqgb6545zj0xs2iyp8kplvlshb5li7-python3.8-pytest-6.2.2/lib/python3.8/site-packages/_pytest/runner.py:255: in <lambda>
python3.8-certbot> lambda: ihook(item=item, **kwds), when=when, reraise=reraise
python3.8-certbot> /nix/store/d8gb6jh5y31zvi44mk6rqn2i7l6smwms-python3.8-pluggy-0.13.1/lib/python3.8/site-packages/pluggy/hooks.py:286: in __call__
python3.8-certbot> return self._hookexec(self, self.get_hookimpls(), kwargs)
python3.8-certbot> /nix/store/d8gb6jh5y31zvi44mk6rqn2i7l6smwms-python3.8-pluggy-0.13.1/lib/python3.8/site-packages/pluggy/manager.py:93: in _hookexec
python3.8-certbot> return self._inner_hookexec(hook, methods, kwargs)
python3.8-certbot> /nix/store/d8gb6jh5y31zvi44mk6rqn2i7l6smwms-python3.8-pluggy-0.13.1/lib/python3.8/site-packages/pluggy/manager.py:84: in <lambda>
python3.8-certbot> self._inner_hookexec = lambda hook, methods, kwargs: hook.multicall(
python3.8-certbot> /nix/store/wjyqgb6545zj0xs2iyp8kplvlshb5li7-python3.8-pytest-6.2.2/lib/python3.8/site-packages/_pytest/unraisableexception.py:88: in pytest_runtest_call
python3.8-certbot> yield from unraisable_exception_runtest_hook()
python3.8-certbot> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
python3.8-certbot> def unraisable_exception_runtest_hook() -> Generator[None, None, None]:
python3.8-certbot> with catch_unraisable_exception() as cm:
python3.8-certbot> yield
python3.8-certbot> if cm.unraisable:
python3.8-certbot> if cm.unraisable.err_msg is not None:
python3.8-certbot> err_msg = cm.unraisable.err_msg
python3.8-certbot> else:
python3.8-certbot> err_msg = "Exception ignored in"
python3.8-certbot> msg = f"{err_msg}: {cm.unraisable.object!r}\n\n"
python3.8-certbot> msg += "".join(
python3.8-certbot> traceback.format_exception(
python3.8-certbot> cm.unraisable.exc_type,
python3.8-certbot> cm.unraisable.exc_value,
python3.8-certbot> cm.unraisable.exc_traceback,
python3.8-certbot> )
python3.8-certbot> )
python3.8-certbot> >               warnings.warn(pytest.PytestUnraisableExceptionWarning(msg))
python3.8-certbot> E               pytest.PytestUnraisableExceptionWarning: Exception ignored in: <_io.FileIO [closed]>
python3.8-certbot> E               
python3.8-certbot> E               Traceback (most recent call last):
python3.8-certbot> E                 File "/build/source/certbot/certbot/_internal/cli/__init__.py", line 450, in prepare_and_parse_args
python3.8-certbot> E                   helpful_parser = helpful
python3.8-certbot> E               ResourceWarning: unclosed file <_io.TextIOWrapper name='/dev/null' mode='w' encoding='UTF-8'>
python3.8-certbot> /nix/store/wjyqgb6545zj0xs2iyp8kplvlshb5li7-python3.8-pytest-6.2.2/lib/python3.8/site-packages/_pytest/unraisableexception.py:78: PytestUnraisableExceptionWarning
```

I just wanted to give ya'll a heads up. On our end we'll probably disable the test or downgrade pytest or something like that, and no specific help from you is necessary. But in case other users were seeing this issue when running the tests (or maybe will in the future once they upgrade pytest), I just wanted to flag it for you.


Discussion:

Resolving patch:
diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py
--- a/certbot/certbot/_internal/main.py
+++ b/certbot/certbot/_internal/main.py
@@ -4,6 +4,9 @@
 import functools
 import logging.handlers
 import sys
+from contextlib import contextmanager
+from typing import Generator
+from typing import IO
 from typing import Iterable
 from typing import List
 from typing import Optional
@@ -1347,26 +1350,36 @@ def make_or_verify_needed_dirs(config):
         util.make_or_verify_dir(hook_dir, strict=config.strict_permissions)
 
 
-def set_displayer(config):
-    """Set the displayer
+@contextmanager
+def make_displayer(config: configuration.NamespaceConfig
+                   ) -> Generator[Union[display_util.NoninteractiveDisplay,
+                                        display_util.FileDisplay], None, None]:
+    """Creates a display object appropriate to the flags in the supplied config.
 
     :param config: Configuration object
-    :type config: interfaces.IConfig
 
-    :returns: `None`
-    :rtype: None
+    :returns: Display object implementing :class:`certbot.interfaces.IDisplay`
 
     """
+    displayer: Union[None, display_util.NoninteractiveDisplay,
+                     display_util.FileDisplay] = None
+    devnull: Optional[IO] = None
+
     if config.quiet:
         config.noninteractive_mode = True
-        displayer: Union[None, display_util.NoninteractiveDisplay, display_util.FileDisplay] =\
-            display_util.NoninteractiveDisplay(open(os.devnull, "w"))
+        devnull = open(os.devnull, "w")
+        displayer = display_util.NoninteractiveDisplay(devnull)
     elif config.noninteractive_mode:
         displayer = display_util.NoninteractiveDisplay(sys.stdout)
     else:
-        displayer = display_util.FileDisplay(sys.stdout,
-                                             config.force_interactive)
-    zope.component.provideUtility(displayer)
+        displayer = display_util.FileDisplay(
+            sys.stdout, config.force_interactive)
+
+    try:
+        yield displayer
+    finally:
+        if devnull:
+            devnull.close()
 
 
 def main(cli_args=None):
@@ -1411,11 +1424,12 @@ def main(cli_args=None):
         if config.func != plugins_cmd:  # pylint: disable=comparison-with-callable
             raise
 
-    set_displayer(config)
-
     # Reporter
     report = reporter.Reporter(config)
     zope.component.provideUtility(report)
     util.atexit_register(report.print_messages)
 
-    return config.func(config, plugins)
+    with make_displayer(config) as displayer:
+        zope.component.provideUtility(displayer)
+
+        return config.func(config, plugins)


Test patch:
diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py
--- a/certbot/tests/log_test.py
+++ b/certbot/tests/log_test.py
@@ -28,7 +28,8 @@ class PreArgParseSetupTest(unittest.TestCase):
     @classmethod
     def _call(cls, *args, **kwargs):  # pylint: disable=unused-argument
         from certbot._internal.log import pre_arg_parse_setup
-        return pre_arg_parse_setup()
+        with mock.patch('builtins.open', mock.mock_open()):
+            return pre_arg_parse_setup()
 
     @mock.patch('certbot._internal.log.sys')
     @mock.patch('certbot._internal.log.pre_arg_parse_except_hook')
diff --git a/certbot/tests/plugins/webroot_test.py b/certbot/tests/plugins/webroot_test.py
--- a/certbot/tests/plugins/webroot_test.py
+++ b/certbot/tests/plugins/webroot_test.py
@@ -141,7 +141,8 @@ def test_perform_reraises_other_errors(self):
             f.write("thingimy")
         filesystem.chmod(self.path, 0o000)
         try:
-            open(permission_canary, "r")
+            with open(permission_canary, "r"):
+                pass
             print("Warning, running tests as root skips permissions tests...")
         except IOError:
             # ok, permissions work, test away...