Repository: twisted/twisted

Issue:
twisted.trial.test.test_reporter.ErrorReportingTests.test_hiddenException is unnecessarily coupled to reactor implementation details
This test asserts a bunch of specific things about the frames in the traceback associated with a logged failure.  The call stack is not part of Twisted's compatibility policy so this is overly specific for any end-user facing behavior.  This test should limit itself to making assertions about the public interface of this functionality.  This makes refactoring the implementation of that public interface easier.

This is specifically motivated by work on #11673 which caused this test to fail because of reactor refactoring.




Discussion:

Resolving patch:
diff --git a/src/twisted/trial/_dist/workerreporter.py b/src/twisted/trial/_dist/workerreporter.py
--- a/src/twisted/trial/_dist/workerreporter.py
+++ b/src/twisted/trial/_dist/workerreporter.py
@@ -10,11 +10,11 @@
 """
 
 from types import TracebackType
-from typing import Callable, List, Optional, Sequence, Tuple, Type, TypeVar, Union
+from typing import Callable, List, Optional, Sequence, Type, TypeVar
 from unittest import TestCase as PyUnitTestCase
 
 from attrs import Factory, define
-from typing_extensions import Literal, TypeAlias
+from typing_extensions import Literal
 
 from twisted.internet.defer import Deferred, maybeDeferred
 from twisted.protocols.amp import AMP
@@ -22,12 +22,10 @@
 from twisted.python.reflect import qual
 from twisted.trial._dist import managercommands
 from twisted.trial.reporter import TestResult
+from ..reporter import TrialFailure
 from .stream import chunk, stream
 
 T = TypeVar("T")
-ExcInfo: TypeAlias = Tuple[Type[BaseException], BaseException, TracebackType]
-XUnitFailure = Union[ExcInfo, Tuple[None, None, None]]
-TrialFailure = Union[XUnitFailure, Failure]
 
 
 async def addError(
diff --git a/src/twisted/trial/reporter.py b/src/twisted/trial/reporter.py
--- a/src/twisted/trial/reporter.py
+++ b/src/twisted/trial/reporter.py
@@ -15,10 +15,13 @@
 import unittest as pyunit
 import warnings
 from collections import OrderedDict
-from typing import TYPE_CHECKING, List, Tuple
+from types import TracebackType
+from typing import TYPE_CHECKING, List, Tuple, Type, Union
 
 from zope.interface import implementer
 
+from typing_extensions import TypeAlias
+
 from twisted.python import log, reflect
 from twisted.python.components import proxyForInterface
 from twisted.python.failure import Failure
@@ -33,6 +36,10 @@
 except ImportError:
     TestProtocolClient = None
 
+ExcInfo: TypeAlias = Tuple[Type[BaseException], BaseException, TracebackType]
+XUnitFailure = Union[ExcInfo, Tuple[None, None, None]]
+TrialFailure = Union[XUnitFailure, Failure]
+
 
 def _makeTodo(value: str) -> "Todo":
     """


Test patch:
diff --git a/src/twisted/trial/test/test_reporter.py b/src/twisted/trial/test/test_reporter.py
--- a/src/twisted/trial/test/test_reporter.py
+++ b/src/twisted/trial/test/test_reporter.py
@@ -14,15 +14,22 @@
 from inspect import getmro
 from io import BytesIO, StringIO
 from typing import Type
-from unittest import TestCase as StdlibTestCase, expectedFailure
+from unittest import (
+    TestCase as StdlibTestCase,
+    TestSuite as PyUnitTestSuite,
+    expectedFailure,
+)
 
-from twisted.python import log, reflect
+from hamcrest import assert_that, equal_to, has_item, has_length
+
+from twisted.python import log
 from twisted.python.failure import Failure
-from twisted.python.reflect import qual
 from twisted.trial import itrial, reporter, runner, unittest, util
 from twisted.trial.reporter import UncleanWarningsReporterWrapper, _ExitWrapper
 from twisted.trial.test import erroneous, sample
 from twisted.trial.unittest import SkipTest, Todo, makeTodo
+from .._dist.test.matchers import isFailure, matches_result, similarFrame
+from .matchers import after
 
 
 class BrokenStream:
@@ -132,14 +139,14 @@ class ErrorReportingTests(StringTest):
     def setUp(self):
         self.loader = runner.TestLoader()
         self.output = StringIO()
-        self.result = reporter.Reporter(self.output)
+        self.result: reporter.Reporter = reporter.Reporter(self.output)
 
     def getOutput(self, suite):
         result = self.getResult(suite)
         result.done()
         return self.output.getvalue()
 
-    def getResult(self, suite):
+    def getResult(self, suite: PyUnitTestSuite) -> reporter.Reporter:
         suite.run(self.result)
         return self.result
 
@@ -207,51 +214,48 @@ def test_doctestError(self):
         expect = [self.doubleSeparator, re.compile(r"\[(ERROR|FAIL)\]")]
         self.stringComparison(expect, output.splitlines())
 
-    def test_hiddenException(self):
+    def test_hiddenException(self) -> None:
         """
-        Check that errors in C{DelayedCall}s get reported, even if the
-        test already has a failure.
+        When a function scheduled using L{IReactorTime.callLater} in a
+        test method raises an exception that exception is added to the test
+        result as an error.
+
+        This happens even if the test also fails and the test failure is also
+        added to the test result as a failure.
 
         Only really necessary for testing the deprecated style of tests that
         use iterate() directly. See
         L{erroneous.DelayedCall.testHiddenException} for more details.
         """
-        from twisted.internet import reactor
-
-        if reflect.qual(reactor).startswith("twisted.internet.asyncioreactor"):
-            raise self.skipTest(
-                "This test does not work on the asyncio reactor, as the "
-                "traceback comes from inside asyncio, not Twisted."
-            )
-
         test = erroneous.DelayedCall("testHiddenException")
-        output = self.getOutput(test).splitlines()
-        errorQual = qual(RuntimeError)
-        match = [
-            self.doubleSeparator,
-            "[FAIL]",
-            "Traceback (most recent call last):",
-            re.compile(
-                r"^\s+File .*erroneous\.py., line \d+, in " "testHiddenException$"
+
+        result = self.getResult(PyUnitTestSuite([test]))
+        assert_that(
+            result, matches_result(errors=has_length(1), failures=has_length(1))
+        )
+        [(actualCase, error)] = result.errors
+        assert_that(test, equal_to(actualCase))
+        assert_that(
+            error,
+            isFailure(
+                type=equal_to(RuntimeError),
+                value=after(str, equal_to("something blew up")),
+                frames=has_item(similarFrame("go", "erroneous.py")),  # type: ignore[arg-type]
             ),
-            re.compile(
-                r'^\s+self\.fail\("Deliberate failure to mask the '
-                r'hidden exception"\)$'
+        )
+
+        [(actualCase, failure)] = result.failures
+        assert_that(test, equal_to(actualCase))
+        assert_that(
+            failure,
+            isFailure(
+                type=equal_to(test.failureException),
+                value=after(
+                    str, equal_to("Deliberate failure to mask the hidden exception")
+                ),
+                frames=has_item(similarFrame("testHiddenException", "erroneous.py")),  # type: ignore[arg-type]
             ),
-            "twisted.trial.unittest.FailTest: "
-            "Deliberate failure to mask the hidden exception",
-            "twisted.trial.test.erroneous.DelayedCall.testHiddenException",
-            self.doubleSeparator,
-            "[ERROR]",
-            "Traceback (most recent call last):",
-            re.compile(r"^\s+File .* in runUntilCurrent"),
-            re.compile(r"^\s+.*"),
-            re.compile(r'^\s+File .*erroneous\.py", line \d+, in go'),
-            re.compile(r"^\s+raise RuntimeError\(self.hiddenExceptionMsg\)"),
-            errorQual + ": something blew up",
-            "twisted.trial.test.erroneous.DelayedCall.testHiddenException",
-        ]
-        self.stringComparison(match, output)
+        )
 
 
 class UncleanWarningWrapperErrorReportingTests(ErrorReportingTests):
@@ -263,7 +267,12 @@ class UncleanWarningWrapperErrorReportingTests(ErrorReportingTests):
     def setUp(self):
         self.loader = runner.TestLoader()
         self.output = StringIO()
-        self.result = UncleanWarningsReporterWrapper(reporter.Reporter(self.output))
+        self.reporter: reporter.Reporter = reporter.Reporter(self.output)
+        self.result = UncleanWarningsReporterWrapper(self.reporter)
+
+    def getResult(self, suite: PyUnitTestSuite) -> reporter.Reporter:
+        suite.run(self.result)
+        return self.reporter
 
 
 class TracebackHandlingTests(unittest.SynchronousTestCase):