Repository: adamchainz/django-rich (language: python)

Issue:
Style subtest results
### Python Version

3.12

### Django Version

5.1

### Package Version

1.9

### Description

Since adding the rich django test runner cPython has added a new method specially for subtests. 

I think broadly we need to copy this method over and add rich formatting.

https://github.com/python/cpython/blob/main/Lib%2Funittest%2Frunner.py#L64

I'm happy to work on this. However, we're off to London for a couple days this week. I can start looking at it at the weekend. 


Discussion / hints:
Sounds good! Let’s add a [diffing unit test](https://adamj.eu/tech/2024/04/26/python-diffing-unit-test/) to ensure our method stays in sync.

Fix patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 08e8cde..98b0d17 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -2,6 +2,10 @@
 Changelog
 =========
 
+* Extend test runner to style subtests, teardown errors, and teardown failures.
+
+  THanks to David Smith in `PR #224 <https://github.com/adamchainz/django-rich/pull/224>`__.
+
 1.10.0 (2024-08-12)
 -------------------
 


Test patch:
diff --git a/src/django_rich/test.py b/src/django_rich/test.py
index 5edd0c0..e1b6d23 100644
--- a/src/django_rich/test.py
+++ b/src/django_rich/test.py
@@ -12,8 +12,10 @@
 from typing import Union
 from typing import cast
 from unittest.case import TestCase
+from unittest.case import _SubTest  # type: ignore [attr-defined]
 from unittest.result import STDERR_LINE
 from unittest.result import STDOUT_LINE
+from unittest.result import TestResult
 from unittest.result import failfast
 
 from django.test import testcases
@@ -29,7 +31,6 @@
 
 _SysExcInfoType = Union[
     Tuple[Type[BaseException], BaseException, TracebackType],
-    Tuple[Type[BaseException], BaseException, None],
     Tuple[None, None, None],
 ]
 
@@ -44,6 +45,9 @@ class RichTextTestResult(unittest.TextTestResult):
     # https://github.com/python/typeshed/pull/7340
     showAll: bool
 
+    # Declaring attribute as _newline was added in Python 3.11.
+    _newline: bool
+
     def __init__(
         self,
         stream: TextIO,
@@ -61,10 +65,17 @@ def __init__(
         with self.console.capture() as cap:
             self.console.print(Rule(characters="━", style=DJANGO_GREEN))
         self.separator2 = cap.get().rstrip("\n")
+        if sys.version_info < (3, 11):
+            self._newline = True
+
+    def startTest(self, test: TestCase) -> None:
+        super().startTest(test)
+        if sys.version_info < (3, 11):
+            self._newline = False
 
     def addSuccess(self, test: TestCase) -> None:
         if self.showAll:
-            self.console.print("ok", style=DJANGO_GREEN)
+            self._write_status(test, "ok")
         elif self.dots:
             self.console.print(".", style=DJANGO_GREEN, end="")
 
@@ -73,7 +84,7 @@ def addError(self, test: TestCase, err: _SysExcInfoType) -> None:
         self.errors.append((test, self._exc_info_to_string(err, test)))
         self._mirrorOutput = True
         if self.showAll:
-            self.console.print("ERROR", style=RED)
+            self._write_status(test, "ERROR")
         elif self.dots:
             self.console.print("E", style=RED, end="")
 
@@ -82,14 +93,14 @@ def addFailure(self, test: TestCase, err: _SysExcInfoType) -> None:
         self.failures.append((test, self._exc_info_to_string(err, test)))
         self._mirrorOutput = True
         if self.showAll:
-            self.console.print("FAIL", style=RED)
+            self._write_status(test, "FAIL")
         elif self.dots:
             self.console.print("F", style=RED, end="")
 
     def addSkip(self, test: TestCase, reason: str) -> None:
         self.skipped.append((test, reason))
         if self.showAll:
-            self.console.print(f"skipped {reason!r}", style=YELLOW)
+            self._write_status(test, f"skipped {reason!r}")
         elif self.dots:
             self.console.print("s", style=YELLOW, end="")
 
@@ -118,6 +129,41 @@ def printErrorList(
             self.console.print(DJANGO_GREEN_RULE, title, DJANGO_GREEN_RULE)
             self.stream.write("%s\n" % err)
 
+    def _write_status(self, test: TestCase, status: str) -> None:
+        is_subtest = isinstance(test, _SubTest)
+        if is_subtest or self._newline:
+            if not self._newline:
+                self.console.print("\n", end="")
+            if is_subtest:
+                self.console.print("  ", end="")
+            self.console.print(self.getDescription(test), end="")
+            self.console.print(" ... ", end="")
+        if status == "FAIL":
+            self.console.print("FAIL", style=RED)
+        elif status == "ERROR":
+            self.console.print("ERROR", style=RED)
+        elif status.startswith("skipped"):
+            self.console.print(status, style=YELLOW)
+        else:
+            self.console.print(status, style=DJANGO_GREEN)
+        self._newline = True
+
+    def addSubTest(
+        self, test: TestCase, subtest: TestCase, err: _SysExcInfoType | None
+    ) -> None:
+        if err is not None:
+            if self.showAll:
+                if issubclass(err[0], subtest.failureException):  # type: ignore [arg-type]
+                    self._write_status(subtest, "FAIL")
+                else:
+                    self._write_status(subtest, "ERROR")
+            elif self.dots:
+                if issubclass(err[0], subtest.failureException):  # type: ignore [arg-type]
+                    self.console.print("F", style=RED, end="")
+                else:
+                    self.console.print("E", style=RED, end="")
+        TestResult.addSubTest(self, test, subtest, err)
+
     def _exc_info_to_string(self, err: _SysExcInfoType, test: TestCase) -> str:
         """Converts a sys.exc_info()-style tuple of values into a string."""
         exctype, value, tb = err
diff --git a/tests/test_test.py b/tests/test_test.py
index 1524608..d4264cf 100644
--- a/tests/test_test.py
+++ b/tests/test_test.py
@@ -75,6 +75,57 @@ def test_slow(self):
         # Ensure over the 0.001s threshold
         time.sleep(0.002)
 
+    def test_subtest(self):
+        for i in range(0, 2):
+            with self.subTest(i=i):
+                self.assertEqual(i, i)
+
+    def test_failure_subtest(self):
+        for i in range(0, 2):
+            with self.subTest(i=i):
+                # 1 is not even.
+                self.assertEqual(i % 2, 0)
+
+    def test_skip_subtest(self):
+        with self.subTest("success", a=1):
+            pass
+        with self.subTest("skip", b=2):
+            self.skipTest("skip")
+
+    def test_mixed_subtest(self):
+        with self.subTest("success", a=1):
+            pass
+        with self.subTest("skip", b=2):
+            self.skipTest("skip")
+        with self.subTest("error", c=3):  # type: ignore [unreachable]
+            self.fail("fail")
+        with self.subTest("error", d=4):
+            raise Exception("error")
+
+
+@pytest.mark.skip(reason="Run below via Django unittest subprocess.")
+class TearDownFailTests(TestCase):
+    def tearDown(self):
+        raise AssertionError("fail")
+
+    def test_tearDownError_success(self):
+        self.assertEqual(1, 1)
+
+    def test_tearDownError_fail(self):
+        self.assertEqual(1, 2)
+
+    def test_tearDownError_error(self):
+        raise ValueError("Woops")
+
+
+@pytest.mark.skip(reason="Run below via Django unittest subprocess.")
+class TearDownErrorTests(TestCase):
+    def tearDown(self):
+        raise Exception("error")
+
+    def test_tearDownError_skip(self):
+        self.skipTest("skip")
+
 
 PYPROJECT_PATH = Path(__file__).resolve().parent.parent / "pyproject.toml"
 
@@ -354,7 +405,6 @@ def test_unexpected_success_quiet(self):
         result = self.run_test(
             "-v", "0", f"{__name__}.ExampleTests.test_unexpected_success"
         )
-
         if django.VERSION >= (4, 1):
             assert result.returncode == 1
         else:
@@ -694,3 +744,271 @@ def test_durations_shown_hidden(self):
             "└──────────┴────────────────────────────────────────────────────┘",
             " Durations < 0.001s were hidden. Use -v to show these durations. ",
         ]
+
+    sub_test_test = pytest.mark.skipif(
+        sys.version_info < (3, 11),
+        reason="addSubTest added in Python 3.11.",
+    )
+
+    @sub_test_test
+    def test_subtest_upstream_source(self):
+        # RichTextTestResult completely replaces _addSubTest(), so check the
+        # overriden function for changes that may need copying in.
+        source = dedent(
+            inspect.getsource(
+                unittest.TextTestResult.addSubTest,
+            )
+        )
+        expected = dedent(
+            """\
+            def addSubTest(self, test, subtest, err):
+                if err is not None:
+                    if self.showAll:
+                        if issubclass(err[0], subtest.failureException):
+                            self._write_status(subtest, "FAIL")
+                        else:
+                            self._write_status(subtest, "ERROR")
+                    elif self.dots:
+                        if issubclass(err[0], subtest.failureException):
+                            self.stream.write('F')
+                        else:
+                            self.stream.write('E')
+                        self.stream.flush()
+                super(TextTestResult, self).addSubTest(test, subtest, err)
+            """
+        )
+        assert source == expected
+
+    @sub_test_test
+    def test_write_status_upstream_source(self):
+        # RichTextTestResult completely replaces _write_status(), so check the
+        # overriden function for changes that may need copying in.
+        source = dedent(
+            inspect.getsource(
+                unittest.TextTestResult._write_status,  # type: ignore[attr-defined]
+            )
+        )
+        expected = dedent(
+            """\
+            def _write_status(self, test, status):
+                is_subtest = isinstance(test, _SubTest)
+                if is_subtest or self._newline:
+                    if not self._newline:
+                        self.stream.writeln()
+                    if is_subtest:
+                        self.stream.write("  ")
+                    self.stream.write(self.getDescription(test))
+                    self.stream.write(" ... ")
+                self.stream.writeln(status)
+                self.stream.flush()
+                self._newline = True
+            """
+        )
+        assert source == expected
+
+    def test_subtest(self):
+        result = self.run_test(f"{__name__}.ExampleTests.test_subtest")
+        assert result.returncode == 0
+        lines = result.stderr.splitlines()
+        assert lines[1] == "."
+
+    def test_subtest_verbose(self):
+        result = self.run_test("-v", "2", f"{__name__}.ExampleTests.test_subtest")
+        assert result.returncode == 0
+        lines = result.stderr.splitlines()
+        if sys.version_info >= (3, 11):
+            assert (
+                lines[1]
+                == "test_subtest (tests.test_test.ExampleTests.test_subtest) ... ok"
+            )
+        else:
+            assert lines[1] == "test_subtest (tests.test_test.ExampleTests) ... ok"
+
+    def test_subtest_quiet(self):
+        result = self.run_test("-v", "0", f"{__name__}.ExampleTests.test_subtest")
+        assert result.returncode == 0
+        lines = result.stderr.splitlines()
+        assert lines[0] == "━" * 80
+
+    def test_subtest_quiet_fail(self):
+        result = self.run_test(
+            "-v", "0", f"{__name__}.ExampleTests.test_failure_subtest"
+        )
+        assert result.returncode == 1
+        lines = result.stderr.splitlines()
+        if sys.version_info >= (3, 11):
+            assert lines[:2] == [
+                "─" * 80,
+                "FAIL: test_failure_subtest (tests.test_test.ExampleTests.test_failure_subtest) ",
+            ]
+        else:
+            assert lines[:2] == [
+                "─" * 80,
+                "FAIL: test_failure_subtest (tests.test_test.ExampleTests) (i=1)",
+            ]
+
+    def test_failure_subtest(self):
+        result = self.run_test(f"{__name__}.ExampleTests.test_failure_subtest")
+        assert result.returncode == 1
+        lines = result.stderr.splitlines()
+        assert lines[1] == "F"
+
+    def test_failure_subtest_verbose(self):
+        result = self.run_test(
+            "-v", "2", f"{__name__}.ExampleTests.test_failure_subtest"
+        )
+        assert result.returncode == 1
+        lines = result.stderr.splitlines()
+        if sys.version_info >= (3, 11):
+            assert lines[1:4] == [
+                "test_failure_subtest (tests.test_test.ExampleTests.test_failure_subtest) ... ",
+                "  test_failure_subtest (tests.test_test.ExampleTests.test_failure_subtest) (i=1) ... FAIL",
+                "",
+            ]
+        else:
+            assert lines[1:4] == [
+                "test_failure_subtest (tests.test_test.ExampleTests) ... ",
+                "  test_failure_subtest (tests.test_test.ExampleTests) (i=1) ... FAIL",
+                "",
+            ]
+
+    def test_subtest_skip(self):
+        result = self.run_test(f"{__name__}.ExampleTests.test_skip_subtest")
+        assert result.returncode == 0
+        lines = result.stderr.splitlines()
+        assert lines[1] == "s"
+
+    def test_subtest_skip_verbose(self):
+        result = self.run_test("-v", "2", f"{__name__}.ExampleTests.test_skip_subtest")
+        assert result.returncode == 0
+        lines = result.stderr.splitlines()
+        if sys.version_info >= (3, 11):
+            assert lines[1:4] == [
+                "test_skip_subtest (tests.test_test.ExampleTests.test_skip_subtest) ... ",
+                "  test_skip_subtest (tests.test_test.ExampleTests.test_skip_subtest)  (b=2) ... skipped 'skip'",
+                "",
+            ]
+        else:
+            assert lines[1:4] == [
+                "test_skip_subtest (tests.test_test.ExampleTests) ... ",
+                "  test_skip_subtest (tests.test_test.ExampleTests)  (b=2) ... skipped 'skip'",
+                "",
+            ]
+
+    def test_subtest_mixed(self):
+        result = self.run_test(f"{__name__}.ExampleTests.test_mixed_subtest")
+        assert result.returncode == 1
+        lines = result.stderr.splitlines()
+        assert lines[1] == "sFE"
+
+    def test_subtest_mixed_verbose(self):
+        result = self.run_test("-v", "2", f"{__name__}.ExampleTests.test_mixed_subtest")
+        assert result.returncode == 1
+        lines = result.stderr.splitlines()
+        if sys.version_info >= (3, 11):
+            assert lines[1:6] == [
+                "test_mixed_subtest (tests.test_test.ExampleTests.test_mixed_subtest) ... ",
+                "  test_mixed_subtest (tests.test_test.ExampleTests.test_mixed_subtest)  (b=2) ... skipped 'skip'",
+                "  test_mixed_subtest (tests.test_test.ExampleTests.test_mixed_subtest)  (c=3) ... FAIL",
+                "  test_mixed_subtest (tests.test_test.ExampleTests.test_mixed_subtest)  (d=4) ... ERROR",
+                "",
+            ]
+        else:
+            assert lines[1:6] == [
+                "test_mixed_subtest (tests.test_test.ExampleTests) ... ",
+                "  test_mixed_subtest (tests.test_test.ExampleTests)  (b=2) ... skipped 'skip'",
+                "  test_mixed_subtest (tests.test_test.ExampleTests)  (c=3) ... FAIL",
+                "  test_mixed_subtest (tests.test_test.ExampleTests)  (d=4) ... ERROR",
+                "",
+            ]
+
+    def test_tearDown_fail(self):
+        result = self.run_test(
+            f"{__name__}.TearDownFailTests.test_tearDownError_success"
+        )
+        assert result.returncode == 1
+        lines = result.stderr.splitlines()
+        assert lines[1] == "F"
+
+        result = self.run_test(f"{__name__}.TearDownFailTests.test_tearDownError_fail")
+        assert result.returncode == 1
+        lines = result.stderr.splitlines()
+        assert lines[1] == "FF"
+
+        result = self.run_test(f"{__name__}.TearDownFailTests.test_tearDownError_error")
+        assert result.returncode == 1
+        lines = result.stderr.splitlines()
+        assert lines[1] == "EF"
+
+        result = self.run_test(f"{__name__}.TearDownErrorTests.test_tearDownError_skip")
+        assert result.returncode == 1
+        lines = result.stderr.splitlines()
+        assert lines[1] == "sE"
+
+    def test_tearDown_fail_verbose(self):
+        result = self.run_test(
+            "-v", "2", f"{__name__}.TearDownFailTests.test_tearDownError_success"
+        )
+        assert result.returncode == 1
+        lines = result.stderr.splitlines()
+        if sys.version_info >= (3, 11):
+            assert (
+                lines[1]
+                == "test_tearDownError_success (tests.test_test.TearDownFailTests.test_tearDownError_success) ... FAIL"
+            )
+        else:
+            assert (
+                lines[1]
+                == "test_tearDownError_success (tests.test_test.TearDownFailTests) ... FAIL"
+            )
+
+        result = self.run_test(
+            "-v", "2", f"{__name__}.TearDownFailTests.test_tearDownError_fail"
+        )
+        assert result.returncode == 1
+        lines = result.stderr.splitlines()
+        if sys.version_info >= (3, 11):
+            assert lines[1:4] == [
+                "test_tearDownError_fail (tests.test_test.TearDownFailTests.test_tearDownError_fail) ... FAIL",
+                "test_tearDownError_fail ",
+                "(tests.test_test.TearDownFailTests.test_tearDownError_fail) ... FAIL",
+            ]
+        else:
+            assert lines[1:3] == [
+                "test_tearDownError_fail (tests.test_test.TearDownFailTests) ... FAIL",
+                "test_tearDownError_fail (tests.test_test.TearDownFailTests) ... FAIL",
+            ]
+
+        result = self.run_test(
+            "-v", "2", f"{__name__}.TearDownFailTests.test_tearDownError_error"
+        )
+        assert result.returncode == 1
+        lines = result.stderr.splitlines()
+        if sys.version_info >= (3, 11):
+            assert lines[1:4] == [
+                "test_tearDownError_error (tests.test_test.TearDownFailTests.test_tearDownError_error) ... ERROR",
+                "test_tearDownError_error ",
+                "(tests.test_test.TearDownFailTests.test_tearDownError_error) ... FAIL",
+            ]
+        else:
+            assert lines[1:3] == [
+                "test_tearDownError_error (tests.test_test.TearDownFailTests) ... ERROR",
+                "test_tearDownError_error (tests.test_test.TearDownFailTests) ... FAIL",
+            ]
+
+        result = self.run_test(
+            "-v", "2", f"{__name__}.TearDownErrorTests.test_tearDownError_skip"
+        )
+        assert result.returncode == 1
+        lines = result.stderr.splitlines()
+        if sys.version_info >= (3, 11):
+            assert lines[1:4] == [
+                "test_tearDownError_skip (tests.test_test.TearDownErrorTests.test_tearDownError_skip) ... skipped 'skip'",
+                "test_tearDownError_skip ",
+                "(tests.test_test.TearDownErrorTests.test_tearDownError_skip) ... ERROR",
+            ]
+        else:
+            assert lines[1:3] == [
+                "test_tearDownError_skip (tests.test_test.TearDownErrorTests) ... skipped 'skip'",
+                "test_tearDownError_skip (tests.test_test.TearDownErrorTests) ... ERROR",
+            ]