Repository: dask/distributed

Issue:
Test trend report should cover timed out test suites
# Use case
When pytest_timeout kills a test in Windows or MacOS, it kills off the whole test suite and no junit test report is generated.
e.g. https://github.com/dask/distributed/runs/5157694410?check_suite_focus=true
As a consequence, the failure is invisible in https://dask.org/distributed/test_report.html.

# Proposed design
Write a wrapper/parser that fetches the name of the test that caused the timeout, e.g.

```
+++++++++++++++++++++++++++++++++++ Timeout ++++++++++++++++++++++++++++++++++++
distributed/cli/tests/test_dask_worker.py::test_nworkers_requires_nanny 
Error: Process completed with exit code 1.
```
and generates a junit report containing that single test as failed.


Discussion:

Resolving patch:
diff --git a/continuous_integration/scripts/parse_stdout.py b/continuous_integration/scripts/parse_stdout.py
new file mode 100644
--- /dev/null
+++ b/continuous_integration/scripts/parse_stdout.py
@@ -0,0 +1,111 @@
+"""On Windows, pytest-timeout kills off the whole test suite, leaving no junit report
+behind. Parse the stdout of pytest to generate one.
+"""
+from __future__ import annotations
+
+import re
+import sys
+from collections import Counter, defaultdict
+from collections.abc import Iterable
+from datetime import datetime
+
+OUTCOMES = {
+    "PASSED",
+    "FAILED",
+    # Test timeout. Marked as a variant of FAILED in the junit report
+    None,
+    # Setup failed or teardown failed.
+    # In the latter case, if the test also failed, show both a FAILED and an ERROR line.
+    "ERROR",
+    # @pytest.mark.skip, @pytest.mark.skipif, or raise pytest.skip()
+    "SKIPPED",
+    # Reported as a variant of SKIPPED in the junit report
+    "XFAIL",
+    # These appear respectively before and after another status. Ignore.
+    "RERUN",
+    "LEAKED",
+}
+
+
+def parse_rows(rows: Iterable[str]) -> list[tuple[str, str, set[str | None]]]:
+    match = re.compile(
+        r"(distributed/.*test.*)::([^ ]*)"
+        r"( (.*)(PASSED|FAILED|ERROR|SKIPPED|XFAIL|RERUN|LEAKED).*| )$"
+    )
+
+    out: defaultdict[tuple[str, str], set[str | None]] = defaultdict(set)
+
+    for row in rows:
+        m = match.match(row)
+        if not m:
+            continue
+
+        fname = m.group(1)
+        clsname = fname.replace("/", ".").replace(".py", "").replace("::", ".")
+
+        tname = m.group(2).strip()
+        if m.group(4) and "]" in m.group(4):
+            tname += " " + m.group(4).split("]")[0] + "]"
+
+        outcome = m.group(5)
+        assert outcome in OUTCOMES
+        if outcome not in {"RERUN", "LEAKED"}:
+            out[clsname, tname].add(outcome)
+
+    return [(clsname, tname, outcomes) for (clsname, tname), outcomes in out.items()]
+
+
+def build_xml(rows: list[tuple[str, str, set[str | None]]]) -> None:
+    cnt = Counter(outcome for _, _, outcomes in rows for outcome in outcomes)
+    timestamp = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f")
+
+    # We could have used ElementTree but it feels like overkill here
+    print('<?xml version="1.0" encoding="utf-8"?>')
+    print("<testsuites>")
+    print(
+        '<testsuite name="distributed" '
+        f'errors="{cnt["ERROR"]}" failures="{cnt["FAILED"] + cnt[None]}" '
+        f'skipped="{cnt["SKIPPED"] + cnt["XFAIL"]}" tests="{sum(cnt.values())}" '
+        f'time="0.0" timestamp="{timestamp}" hostname="">'
+    )
+
+    for clsname, tname, outcomes in rows:
+        print(f'<testcase classname="{clsname}" name="{tname}" time="0.0"', end="")
+        if outcomes == {"PASSED"}:
+            print(" />")
+        elif outcomes == {"FAILED"}:
+            print('><failure message=""></failure></testcase>')
+        elif outcomes == {None}:
+            print('><failure message="pytest-timeout exceeded"></failure></testcase>')
+        elif outcomes == {"ERROR"}:
+            print('><error message="failed on setup"></error></testcase>')
+        elif outcomes == {"PASSED", "ERROR"}:
+            print('><error message="failed on teardown"></error></testcase>')
+        elif outcomes == {"FAILED", "ERROR"}:
+            print(
+                '><failure message=""></failure></testcase>\n'
+                f'<testcase classname="{clsname}" name="{tname}" time="0.0">'
+                '<error message="failed on teardown"></error></testcase>'
+            )
+        elif outcomes == {"SKIPPED"}:
+            print('><skipped type="pytest.skip" message="skip"></skipped></testcase>')
+        elif outcomes == {"XFAIL"}:
+            print('><skipped type="pytest.xfail" message="xfail"></skipped></testcase>')
+        else:  # pragma: nocover
+            # This should be unreachable. We would normally raise ValueError, except
+            # that a crash in this script would be pretty much invisible.
+            print(
+                f' />\n<testcase classname="parse_stdout" name="build_xml" time="0.0">'
+                f'><failure message="Unexpected {outcomes=}"></failure></testcase>'
+            )
+
+    print("</testsuite>")
+    print("</testsuites>")
+
+
+def main() -> None:  # pragma: nocover
+    build_xml(parse_rows(sys.stdin))
+
+
+if __name__ == "__main__":
+    main()  # pragma: nocover


Test patch:
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -109,8 +109,8 @@ jobs:
 
       - name: Reconfigure pytest-timeout
         shell: bash -l {0}
-        # No SIGALRM available on Windows. On MacOS, it kills the whole test suite.
-        if: ${{ matrix.os == 'ubuntu-latest' }}
+        # No SIGALRM available on Windows
+        if: ${{ matrix.os != 'windows-latest' }}
         run: sed -i.bak 's/timeout_method = thread/timeout_method = signal/' setup.cfg
 
       - name: Test
@@ -124,13 +124,29 @@ jobs:
               # https://github.com/dask/distributed/issues/4514
               export DISABLE_IPV6=1
           fi
+
           source continuous_integration/scripts/set_ulimit.sh
+          set -o pipefail
+          mkdir reports
 
           pytest distributed \
             -m "not avoid_ci and ${{ matrix.partition }}" --runslow \
             --leaks=fds,processes,threads \
             --junitxml reports/pytest.xml -o junit_suite_name=$TEST_ID \
-            --cov=distributed --cov-report=xml
+            --cov=distributed --cov-report=xml \
+          | tee reports/stdout
+
+      - name: Generate junit XML report in case of pytest-timeout
+        if: ${{ failure() }}
+        shell: bash -l {0}
+        run: |
+          if [ ! -e reports/pytest.xml ]
+          then
+            # This should only ever happen on Windows.
+            # On Linux and MacOS, pytest-timeout kills off the individual tests
+            # See (reconfigure pytest-timeout above)
+            python continuous_integration/scripts/parse_stdout.py < reports/stdout > reports/pytest.xml
+          fi
 
       # - name: Debug with tmate on failure
       #   if: ${{ failure() }}
diff --git a/distributed/tests/test_parse_stdout.py b/distributed/tests/test_parse_stdout.py
new file mode 100644
--- /dev/null
+++ b/distributed/tests/test_parse_stdout.py
@@ -0,0 +1,95 @@
+import os
+import re
+import sys
+
+import pytest
+
+script_dir = os.path.join(
+    os.path.dirname(__file__), "..", "..", "continuous_integration", "scripts"
+)
+sys.path.append(script_dir)
+# Will fail if test suite was pip-installed
+parse_stdout = pytest.importorskip("parse_stdout")
+
+
+# Note: test_timeout below ends with a whitespace!
+# flake8: noqa: W291
+stdout = """
+Unrelated row, must ignore
+distributed/tests/test1.py::test_fail [31mFAILED[0m[31m                             [ 10%][0m
+distributed/tests/test1.py::test_error_in_setup [31mERROR[0m[31m                    [ 20%][0m
+distributed/tests/test1.py::test_pass_and_then_error_in_teardown [32mPASSED[0m[31m  [ 30%][0m
+distributed/tests/test1.py::test_pass_and_then_error_in_teardown [31mERROR[0m[31m   [ 30%][0m
+distributed/tests/test1.py::test_fail_and_then_error_in_teardown [31mFAILED[0m[31m  [ 40%][0m
+distributed/tests/test1.py::test_fail_and_then_error_in_teardown [31mERROR[0m[31m   [ 40%][0m
+distributed/tests/test1.py::test_skip [33mSKIPPED[0m (unconditional skip)[31m       [ 50%][0m
+distributed/tests/test1.py::test_xfail [33mXFAIL[0m[31m                             [ 60%][0m
+distributed/tests/test1.py::test_flaky [33mRERUN[0m[31m                             [ 70%][0m
+distributed/tests/test1.py::test_flaky [32mPASSED[0m[31m                            [ 70%][0m
+distributed/tests/test1.py::test_leaking [32mPASSED[0m[31m                          [ 72%][0m
+distributed/tests/test1.py::test_leaking [32mLEAKED[0m[31m                          [ 72%][0m
+distributed/tests/test1.py::test_pass [32mPASSED[0m[31m                             [ 80%][0m
+distributed/tests/test1.py::test_params[a a] [32mPASSED[0m[31m                      [ 78%][0m
+distributed/tests/test1.py::MyTest::test_unittest [32mPASSED[0m[31m                 [ 86%][0m
+distributed/tests/test1.py::test_timeout 
+"""
+
+
+def test_parse_rows():
+    actual = parse_stdout.parse_rows(stdout.splitlines())
+
+    expect = [
+        ("distributed.tests.test1", "test_fail", {"FAILED"}),
+        ("distributed.tests.test1", "test_error_in_setup", {"ERROR"}),
+        (
+            "distributed.tests.test1",
+            "test_pass_and_then_error_in_teardown",
+            {"PASSED", "ERROR"},
+        ),
+        (
+            "distributed.tests.test1",
+            "test_fail_and_then_error_in_teardown",
+            {"FAILED", "ERROR"},
+        ),
+        ("distributed.tests.test1", "test_skip", {"SKIPPED"}),
+        ("distributed.tests.test1", "test_xfail", {"XFAIL"}),
+        ("distributed.tests.test1", "test_flaky", {"PASSED"}),
+        ("distributed.tests.test1", "test_leaking", {"PASSED"}),
+        ("distributed.tests.test1", "test_pass", {"PASSED"}),
+        ("distributed.tests.test1", "test_params[a a]", {"PASSED"}),
+        ("distributed.tests.test1.MyTest", "test_unittest", {"PASSED"}),
+        ("distributed.tests.test1", "test_timeout", {None}),
+    ]
+
+    assert actual == expect
+
+
+def test_build_xml(capsys):
+    rows = parse_stdout.parse_rows(stdout.splitlines())
+    parse_stdout.build_xml(rows)
+    actual = capsys.readouterr().out.strip()
+    # Remove timestamp
+    actual = re.sub(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}", r"snip", actual)
+
+    expect = """
+<?xml version="1.0" encoding="utf-8"?>
+<testsuites>
+<testsuite name="distributed" errors="3" failures="3" skipped="2" tests="14" time="0.0" timestamp="snip" hostname="">
+<testcase classname="distributed.tests.test1" name="test_fail" time="0.0"><failure message=""></failure></testcase>
+<testcase classname="distributed.tests.test1" name="test_error_in_setup" time="0.0"><error message="failed on setup"></error></testcase>
+<testcase classname="distributed.tests.test1" name="test_pass_and_then_error_in_teardown" time="0.0"><error message="failed on teardown"></error></testcase>
+<testcase classname="distributed.tests.test1" name="test_fail_and_then_error_in_teardown" time="0.0"><failure message=""></failure></testcase>
+<testcase classname="distributed.tests.test1" name="test_fail_and_then_error_in_teardown" time="0.0"><error message="failed on teardown"></error></testcase>
+<testcase classname="distributed.tests.test1" name="test_skip" time="0.0"><skipped type="pytest.skip" message="skip"></skipped></testcase>
+<testcase classname="distributed.tests.test1" name="test_xfail" time="0.0"><skipped type="pytest.xfail" message="xfail"></skipped></testcase>
+<testcase classname="distributed.tests.test1" name="test_flaky" time="0.0" />
+<testcase classname="distributed.tests.test1" name="test_leaking" time="0.0" />
+<testcase classname="distributed.tests.test1" name="test_pass" time="0.0" />
+<testcase classname="distributed.tests.test1" name="test_params[a a]" time="0.0" />
+<testcase classname="distributed.tests.test1.MyTest" name="test_unittest" time="0.0" />
+<testcase classname="distributed.tests.test1" name="test_timeout" time="0.0"><failure message="pytest-timeout exceeded"></failure></testcase>
+</testsuite>
+</testsuites>
+    """.strip()
+
+    assert actual == expect