Repository: twisted/twisted

Issue:
twisted tests failing on main on github actions with Python 3.12.1
In the past few days, pyca/cryptography's downstream running on twisted's tests have begun failing with:

```
twisted.web.test.test_tap.ServiceTests.test_HTTPSFailureOnMissingSSL
===============================================================================
[FAIL]
Traceback (most recent call last):
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/_dist/test/test_worker.py", line 254, in test_runSkip
    result = self.workerRunTest(expectedCase)
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/_dist/test/test_worker.py", line 189, in workerRunTest
    self.assertEqual({"success": True}, self.successResultOf(d))
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/_synctest.py", line 732, in successResultOf
    self.fail(
twisted.trial.unittest.FailTest: Success result expected on <Deferred at 0x7fd17c3581a0 current result: None>, found failure result instead:
Traceback (most recent call last):
--- <exception caught here> ---
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/_dist/worker.py", line 305, in run
    return await self.callRemote(workercommands.Run, testCase=testCaseId)  # type: ignore[no-any-return]
twisted.protocols.amp.UnknownRemoteError: Code<UNKNOWN>: Unknown Error


twisted.trial._dist.test.test_worker.LocalWorkerAMPTests.test_runSkip
===============================================================================
[FAIL]
Traceback (most recent call last):
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/_dist/test/test_workerreporter.py", line 137, in test_addSkipPyunit
    self.assertTestRun(
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/_dist/test/test_workerreporter.py", line 48, in assertTestRun
    assert_that(run(self, target), matches_result(**expectations))
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/_dist/test/test_workerreporter.py", line 34, in run
    assert_that(case.successResultOf(d), equal_to({"success": True}))
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/_synctest.py", line 732, in successResultOf
    self.fail(
twisted.trial.unittest.FailTest: Success result expected on <Deferred at 0x7fd17c359760 current result: None>, found failure result instead:
Traceback (most recent call last):
--- <exception caught here> ---
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/_dist/worker.py", line 305, in run
    return await self.callRemote(workercommands.Run, testCase=testCaseId)  # type: ignore[no-any-return]
twisted.protocols.amp.UnknownRemoteError: Code<UNKNOWN>: Unknown Error


twisted.trial._dist.test.test_workerreporter.WorkerReporterTests.test_addSkipPyunit
===============================================================================
Error: 
Traceback (most recent call last):
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/protocols/amp.py", line 1141, in checkKnownErrors
    key = error.trap(*command.allErrors)
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/python/failure.py", line 481, in trap
    self.raiseException()
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/python/failure.py", line 505, in raiseException
    raise self.value.with_traceback(self.tb)
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/internet/defer.py", line 2000, in _inlineCallbacks
    result = context.run(gen.send, result)
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/_dist/worker.py", line 97, in run
    suite.run(self._result)
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/runner.py", line 269, in run
    TestSuite.run(self, result)
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/_asyncrunner.py", line 35, in run
    test(result)
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/unittest/suite.py", line 84, in __call__
    return self.run(*args, **kwds)
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/runner.py", line 231, in run
    super().run(result)
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/_asyncrunner.py", line 35, in run
    test(result)
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/unittest/suite.py", line 84, in __call__
    return self.run(*args, **kwds)
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/_asyncrunner.py", line 35, in run
    test(result)
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/unittest/case.py", line 692, in __call__
    return self.run(*args, **kwds)
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/unittest/case.py", line 662, in run
    result.stopTest(self)
  File "/opt/hostedtoolcache/Python/3.12.1/x64/lib/python3.12/site-packages/twisted/trial/reporter.py", line 149, in stopTest
    self._lastTime = self._getTime() - self._testStarted
builtins.AttributeError: 'WorkerReporter' object has no attribute '_testStarted'

twisted.trial._dist.test.test_worker.LocalWorkerAMPTests.test_runSkip
twisted.trial._dist.test.test_workerreporter.WorkerReporterTests.test_addSkipPyunit
-------------------------------------------------------------------------------
Ran 11068 tests in 114.943s

FAILED (skips=520, failures=2, errors=2, successes=10546)
```

Example CI run: https://github.com/pyca/cryptography/actions/runs/7194931288/job/19596527447

This appears to be correlated with a github actions CI image update: https://github.com/actions/runner-images/commit/e6f1047898c3923e2613a2e5da6ee1ca49cc77b3


Discussion:
@adiroiban ICYMI

I am not going to have time to look at this until next week at the earliest, and probably more like early January, so just wanted to poke someone else, as it seems annoyingly serious
Thanks Alex for the report.

I can see that this job run was first succesull, and after a retry it's now failing

https://github.com/twisted/twisted/actions/runs/7073299930/job/19658904212

-----

This looks Python 3.12 only... at least for Linux.

We don't run Python 3.12 tests on macOS and Windows.

I will try to trigger a Python 3.12 run for macOS and Windows to see where we stand.
Same error is present on Windows on Python 3.12.1 on Windows.

I can see the test pass on CPython 3.12.0 and fails on CPython 3.12.1

With 3.12.0 all tests pass, even Windows and MacOS.

I got 3.12.1 installed on my laptop and I will try to reproduce this and see if there is an easy fix

So far not much progress

```
$ python --version
Python 3.12.1
$ python -m twisted.trial -j 4 twisted.trial._dist

all ok
```
This issue is not resolved, see: https://github.com/pyca/cryptography/pull/10007
Thanks, we'll continue to investigate
Twisted `trunk` tests were "fixed" by pinning python 3.12.0.
Cryptography tests run with 3.12.1 on GitHub Action.

The ugly part is that I got 3.12.1 on my laptop from `deadsnakes ppa`, but I was not able to reproduce the error :(

I will try to get txmate enabled on our CI and try to run with GitHub Actions software
On 3.12.1 I was able to reproduce an error for this test that works on 3.11 ... so I got something

```
$ trial twisted.cred.test.test_strcred.CheckerOptionsTests.test_invalidAuthError
twisted.cred.test.test_strcred
  CheckerOptionsTests
    test_invalidAuthError ...                                           [ERROR]
                                          [ERROR]
                                          [ERROR]

===============================================================================
[ERROR]
Traceback (most recent call last):
  File "/home/adi/chevah/twisted/src/twisted/plugin.py", line 218, in getPlugins
    adapted = interface(plugin, None)
  File "/home/adi/chevah/twisted/venv-3.12/lib/python3.12/site-packages/zope/interface/interface.py", line 924, in _call_conform
    return conform(self)
  File "/home/adi/chevah/twisted/src/twisted/plugin.py", line 59, in __conform__
    return self.load()
  File "/home/adi/chevah/twisted/src/twisted/plugin.py", line 54, in load
    return namedAny(self.dropin.moduleName + "." + self.name)
  File "/home/adi/chevah/twisted/src/twisted/python/reflect.py", line 308, in namedAny
    obj = getattr(obj, n)
builtins.AttributeError: module 'twisted.plugins.cred_sshkeys' has no attribute 'theSSHKeyCheckerFactory'

twisted.cred.test.test_strcred.CheckerOptionsTests.test_invalidAuthError
twisted.cred.test.test_strcred.CheckerOptionsTests.test_invalidAuthError
twisted.cred.test.test_strcred.CheckerOptionsTests.test_invalidAuthError
-------------------------------------------------------------------------------
Ran 1 tests in 0.018s

FAILED (errors=3)
```

and a quick fix is

```diff
diff --git a/src/twisted/plugins/cred_sshkeys.py b/src/twisted/plugins/cred_sshkeys.py
index 8b7326b258..77dff769e5 100644
--- a/src/twisted/plugins/cred_sshkeys.py
+++ b/src/twisted/plugins/cred_sshkeys.py
@@ -18,6 +18,7 @@ This allows SSH public key authentication, based on public keys listed in
 authorized_keys and authorized_keys2 files in user .ssh/ directories.
 """
 
+theSSHKeyCheckerFactory = None
 
 try:
     from twisted.conch.checkers import SSHPublicKeyChecker, UNIXAuthorizedKeysFiles

```

Might be some kind of optimization 
Somehow `twisted.trial._dist.workerreporter.WorkerReporter` is not started on 3.12.1

```
# This is 3.11
$ trial twisted.trial._dist.test.test_worker.LocalWorkerAMPTests.test_runSkip
twisted.trial._dist.test.test_worker
  LocalWorkerAMPTests
    test_runSkip ... Start <twisted.trial.reporter.TreeReporter run=0 errors=0 failures=0 todos=0 dones=0 skips=0>
Start <twisted.trial.reporter.TestResult run=0 errors=0 failures=0 todos=0 dones=0 skips=0>
Start <twisted.trial._dist.workerreporter.WorkerReporter run=0 errors=0 failures=0 todos=0 dones=0 skips=0>
Stop <twisted.trial._dist.workerreporter.WorkerReporter run=1 errors=0 failures=0 todos=0 dones=1 skips=0>
Stop <twisted.trial.reporter.TestResult run=1 errors=0 failures=0 todos=0 dones=1 skips=0>
                                                      [OK]
Stop <twisted.trial.reporter.TreeReporter run=1 errors=0 failures=0 todos=0 dones=0 skips=0>

-------------------------------------------------------------------------------
Ran 1 tests in 0.004s

PASSED (successes=1)
$ deactivate 
$ . venv-3.12/bin/activate
# This is 3.12.1
$ trial twisted.trial._dist.test.test_worker.LocalWorkerAMPTests.test_runSkip
twisted.trial._dist.test.test_worker
  LocalWorkerAMPTests
    test_runSkip ... Start <twisted.trial.reporter.TreeReporter run=0 errors=0 failures=0 todos=0 dones=0 skips=0>
Start <twisted.trial.reporter.TestResult run=0 errors=0 failures=0 todos=0 dones=0 skips=0>
Stop <twisted.trial._dist.workerreporter.WorkerReporter run=0 errors=0 failures=0 todos=0 dones=1 skips=0>
Stop <twisted.trial.reporter.TestResult run=1 errors=0 failures=0 todos=0 dones=1 skips=0>
                                                    [FAIL]
                                                   [ERROR]
Stop <twisted.trial.reporter.TreeReporter run=1 errors=1 failures=1 todos=0 dones=0 skips=0>

===============================================================================
[FAIL]
Traceback (most recent call last):
  File "/home/adi/chevah/twisted/src/twisted/trial/_dist/test/test_worker.py", line 254, in test_runSkip
    result = self.workerRunTest(expectedCase)
  File "/home/adi/chevah/twisted/src/twisted/trial/_dist/test/test_worker.py", line 189, in workerRunTest
    self.assertEqual({"success": True}, self.successResultOf(d))
  File "/home/adi/chevah/twisted/src/twisted/trial/_synctest.py", line 732, in successResultOf
    self.fail(
twisted.trial.unittest.FailTest: Success result expected on <Deferred at 0x7f15f1d750d0 current result: None>, found failure result instead:
Traceback (most recent call last):
--- <exception caught here> ---
  File "/home/adi/chevah/twisted/src/twisted/trial/_dist/worker.py", line 305, in run
    return await self.callRemote(workercommands.Run, testCase=testCaseId)  # type: ignore[no-any-return]
twisted.protocols.amp.UnknownRemoteError: Code<UNKNOWN>: Unknown Error


twisted.trial._dist.test.test_worker.LocalWorkerAMPTests.test_runSkip
===============================================================================
[ERROR]
Traceback (most recent call last):
  File "/home/adi/chevah/twisted/src/twisted/protocols/amp.py", line 1141, in checkKnownErrors
    key = error.trap(*command.allErrors)
....SNIP...
  File "/home/adi/chevah/twisted/src/twisted/trial/reporter.py", line 151, in stopTest
    self._lastTime = self._getTime() - self._testStarted
builtins.AttributeError: 'WorkerReporter' object has no attribute '_testStarted'

twisted.trial._dist.test.test_worker.LocalWorkerAMPTests.test_runSkip
-------------------------------------------------------------------------------
Ran 1 tests in 0.009s

FAILED (failures=1, errors=1)

```

Resolving patch:
diff --git a/src/twisted/trial/reporter.py b/src/twisted/trial/reporter.py
--- a/src/twisted/trial/reporter.py
+++ b/src/twisted/trial/reporter.py
@@ -7,7 +7,7 @@
 """
 Defines classes that handle the results of tests.
 """
-
+from __future__ import annotations
 
 import os
 import sys
@@ -16,7 +16,7 @@
 import warnings
 from collections import OrderedDict
 from types import TracebackType
-from typing import TYPE_CHECKING, List, Tuple, Type, Union
+from typing import TYPE_CHECKING, List, Optional, Tuple, Type, Union
 
 from zope.interface import implementer
 
@@ -87,6 +87,11 @@ class TestResult(pyunit.TestResult):
 
     @ivar successes: count the number of successes achieved by the test run.
     @type successes: C{int}
+
+    @ivar _startTime: The time when the current test was started. It defaults to
+    L{None}, which means that the test was skipped.
+    @ivar _lastTime: The duration of the current test run. It defaults to
+    L{None}, which means that the test was skipped.
     """
 
     # Used when no todo provided to addExpectedFailure or addUnexpectedSuccess.
@@ -96,6 +101,9 @@ class TestResult(pyunit.TestResult):
     expectedFailures: List[Tuple[itrial.ITestCase, str, "Todo"]]  # type: ignore[assignment]
     unexpectedSuccesses: List[Tuple[itrial.ITestCase, str]]  # type: ignore[assignment]
     successes: int
+    _testStarted: Optional[int]
+    # The duration of the test. It is None until the test completes.
+    _lastTime: Optional[int]
 
     def __init__(self):
         super().__init__()
@@ -104,6 +112,8 @@ def __init__(self):
         self.unexpectedSuccesses = []
         self.successes = 0
         self._timings = []
+        self._testStarted = None
+        self._lastTime = None
 
     def __repr__(self) -> str:
         return "<%s run=%d errors=%d failures=%d todos=%d dones=%d skips=%d>" % (
@@ -146,7 +156,8 @@ def stopTest(self, test):
         @type test: L{pyunit.TestCase}
         """
         super().stopTest(test)
-        self._lastTime = self._getTime() - self._testStarted
+        if self._testStarted is not None:
+            self._lastTime = self._getTime() - self._testStarted
 
     def addFailure(self, test, fail):
         """


Test patch:
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -71,7 +71,7 @@ jobs:
         # The Python version on which the job is executed.
         # We need at least one value here, so we go with latest Python version
         # that we support..
-        python-version: ['3.12.0']
+        python-version: ['3.12']
         # Just use the default OS.
         runs-on: ['']
         # Human readable short description for this job.
@@ -141,7 +141,7 @@ jobs:
             tox-env: 'macos-withcov-alldeps'
 
           # Newest macOS and newest Python supported versions.
-          - python-version: '3.12.0'
+          - python-version: '3.12'
             runs-on: 'macos-12'
             job-name: 'macos-12-default-tests'
             tox-env: 'macos-withcov-alldeps'
@@ -157,7 +157,7 @@ jobs:
             trial-args: '--reactor=select'
 
           # Windows, newest Python supported version with iocp reactor.
-          - python-version: '3.12.0'
+          - python-version: '3.12'
             runs-on: 'windows-2022'
             tox-env: 'alldeps-withcov-windows'
             job-name: 'win-default-tests-iocp'