Repository: twisted/twisted

Issue:
Test CFReactor on CI
Twisted has a "CoreFoundation" reactor for doing interesting macOS platform integration.  And there are tests for this reactor in `twisted.internet.test`.  But!  These tests are not by the macOS CI job.  They should be because they don't tell us if something is broken when they're not run.



Discussion:

Resolving patch:
diff --git a/src/twisted/internet/cfreactor.py b/src/twisted/internet/cfreactor.py
--- a/src/twisted/internet/cfreactor.py
+++ b/src/twisted/internet/cfreactor.py
@@ -33,7 +33,7 @@
     CFAbsoluteTimeGetCurrent,
     CFRunLoopAddSource,
     CFRunLoopAddTimer,
-    CFRunLoopGetMain,
+    CFRunLoopGetCurrent,
     CFRunLoopRemoveSource,
     CFRunLoopRun,
     CFRunLoopStop,
@@ -132,7 +132,7 @@ def __init__(self, runLoop=None, runner=None):
         self._runner = runner
 
         if runLoop is None:
-            runLoop = CFRunLoopGetMain()
+            runLoop = CFRunLoopGetCurrent()
         self._cfrunloop = runLoop
         PosixReactorBase.__init__(self)
 
@@ -372,22 +372,121 @@ def _moveCallLaterSooner(self, tple):
         self._scheduleSimulate()
         return result
 
+    def startRunning(self, installSignalHandlers: bool = True) -> None:
+        """
+        Start running the reactor, then kick off the timer that advances
+        Twisted's clock to keep pace with CFRunLoop's.
+        """
+        super().startRunning(installSignalHandlers)
+
+        # Before 'startRunning' is called, the reactor is not attached to the
+        # CFRunLoop[1]; specifically, the CFTimer that runs all of Twisted's
+        # timers is not active and will not have been added to the loop by any
+        # application code.  Now that _running is probably[2] True, we need to
+        # ensure that timed calls will actually run on the main loop.  This
+        # call needs to be here, rather than at the top of mainLoop, because
+        # it's possible to use startRunning to *attach* a reactor to an
+        # already-running CFRunLoop, i.e. within a plugin for an application
+        # that doesn't otherwise use Twisted, rather than calling it via run().
+        self._scheduleSimulate(force=True)
+
+        # [1]: readers & writers are still active in the loop, but arguably
+        #      they should not be.
+
+        # [2]: application code within a 'startup' system event trigger *may*
+        #      have already crashed the reactor and thus set _started to False,
+        #      but that specific case is handled by mainLoop, since that case
+        #      is inherently irrelevant in an attach-to-application case and is
+        #      only necessary to handle mainLoop spuriously blocking.
+
     _inCFLoop = False
 
-    def mainLoop(self):
+    def mainLoop(self) -> None:
         """
         Run the runner (C{CFRunLoopRun} or something that calls it), which runs
         the run loop until C{crash()} is called.
         """
-        self._inCFLoop = True
+        if not self._started:
+            # If we arrive here, we were crashed by application code in a
+            # 'startup' system event trigger, (or crashed manually before the
+            # application calls 'mainLoop' directly for whatever reason; sigh,
+            # this method should not be public).  However, application code
+            # doing obscure things will expect an invocation of this loop to
+            # have at least *one* pass over ready readers, writers, and delayed
+            # calls.  iterate(), in particular, is emulated in exactly this way
+            # in this reactor implementation.  In order to ensure that we enter
+            # the real implementation of the mainloop and do all of those
+            # things, we need to set _started back to True so that callLater
+            # actually schedules itself against the CFRunLoop, but immediately
+            # crash once we are in the context of the loop where we've run
+            # ready I/O and timers.
+
+            def docrash() -> None:
+                self.crash()
+
+            self._started = True
+            self.callLater(0, docrash)
+        already = False
         try:
-            self._runner()
+            while self._started:
+                if already:
+                    # Sometimes CFRunLoopRun (or its equivalents) may exit
+                    # without CFRunLoopStop being called.
+
+                    # This is really only *supposed* to happen when it runs out
+                    # of sources & timers to process.  However, in full Twisted
+                    # test-suite runs we have observed, extremely rarely (once
+                    # in every 3000 tests or so) CFRunLoopRun exiting in cases
+                    # where it seems as though there *is* still some work to
+                    # do.  However, given the difficulty of reproducing the
+                    # race conditions necessary to make this happen, it's
+                    # possible that we have missed some nuance of when
+                    # CFRunLoop considers the list of work "empty" and various
+                    # callbacks and timers to be "invalidated".  Therefore we
+                    # are not fully confident that this is a platform bug, but
+                    # it is nevertheless unexpected behavior from our reading
+                    # of the documentation.
+
+                    # To accommodate this rare and slightly ambiguous stress
+                    # case, we make extra sure that our scheduled timer is
+                    # re-created on the loop as a CFRunLoopTimer, which
+                    # reliably gives the loop some work to do and 'fixes' it if
+                    # it exited due to having no active sources or timers.
+                    self._scheduleSimulate()
+
+                    # At this point, there may be a little more code that we
+                    # would need to put here for full correctness for a very
+                    # peculiar type of application: if you're writing a
+                    # command-line tool using CFReactor, adding *nothing* to
+                    # the reactor itself, disabling even the internal Waker
+                    # file descriptors, then there's a possibility that
+                    # CFRunLoopRun will exit early, and if we have no timers,
+                    # we might busy-loop here.  Because we cannot seem to force
+                    # this to happen under normal circumstances, we're leaving
+                    # that code out.
+
+                already = True
+                self._inCFLoop = True
+                try:
+                    self._runner()
+                finally:
+                    self._inCFLoop = False
         finally:
-            self._inCFLoop = False
+            self._stopSimulating()
 
-    _currentSimulator = None
+    _currentSimulator: object | None = None
 
-    def _scheduleSimulate(self, force=False):
+    def _stopSimulating(self) -> None:
+        """
+        If we have a CFRunLoopTimer registered with the CFRunLoop, invalidate
+        it and set it to None.
+        """
+        if self._currentSimulator is None:
+            return
+        CFRunLoopTimerInvalidate(self._currentSimulator)
+        self._currentSimulator = None
+
+    def _scheduleSimulate(self, force: bool = False) -> None:
         """
         Schedule a call to C{self.runUntilCurrent}.  This will cancel the
         currently scheduled call if it is already scheduled.
@@ -401,24 +500,28 @@ def _scheduleSimulate(self, force=False):
 
         @type force: C{bool}
         """
-        if self._currentSimulator is not None:
-            CFRunLoopTimerInvalidate(self._currentSimulator)
+        self._stopSimulating()
+        if not self._started:
+            # If the reactor is not running (e.g. we are scheduling callLater
+            # calls before starting the reactor) we should not be scheduling
+            # CFRunLoopTimers against the global CFRunLoop.
+            return
+
+        timeout = 0.0 if force else self.timeout()
+        if timeout is None:
+            return
+
+        fireDate = CFAbsoluteTimeGetCurrent() + timeout
+
+        def simulate(cftimer, extra):
             self._currentSimulator = None
-        timeout = self.timeout()
-        if force:
-            timeout = 0.0
-        if timeout is not None:
-            fireDate = CFAbsoluteTimeGetCurrent() + timeout
-
-            def simulate(cftimer, extra):
-                self._currentSimulator = None
-                self.runUntilCurrent()
-                self._scheduleSimulate()
-
-            c = self._currentSimulator = CFRunLoopTimerCreate(
-                kCFAllocatorDefault, fireDate, 0, 0, 0, simulate, None
-            )
-            CFRunLoopAddTimer(self._cfrunloop, c, kCFRunLoopCommonModes)
+            self.runUntilCurrent()
+            self._scheduleSimulate()
+
+        c = self._currentSimulator = CFRunLoopTimerCreate(
+            kCFAllocatorDefault, fireDate, 0, 0, 0, simulate, None
+        )
+        CFRunLoopAddTimer(self._cfrunloop, c, kCFRunLoopCommonModes)
 
     def callLater(self, _seconds, _f, *args, **kw):
         """
@@ -439,18 +542,9 @@ def crash(self):
         """
         Implement L{IReactorCore.crash}
         """
-        wasStarted = self._started
         PosixReactorBase.crash(self)
-        if self._inCFLoop:
-            self._stopNow()
-        else:
-            if wasStarted:
-                self.callLater(0, self._stopNow)
-
-    def _stopNow(self):
-        """
-        Immediately stop the CFRunLoop (which must be running!).
-        """
+        if not self._inCFLoop:
+            return
         CFRunLoopStop(self._cfrunloop)
 
     def iterate(self, delay=0):
@@ -459,7 +553,14 @@ def iterate(self, delay=0):
         by letting the loop run for a little while and then scheduling a timed
         call to exit it.
         """
-        self.callLater(delay, self._stopNow)
+        self._started = True
+        # Since the CoreFoundation loop doesn't have the concept of "iterate"
+        # we can't ask it to do this.  Instead we will make arrangements to
+        # crash it *very* soon and then make it run.  This is a rough
+        # approximation of "an iteration".  Using crash and mainLoop here
+        # means that it's safe (as safe as anything using "iterate" can be) to
+        # do this repeatedly.
+        self.callLater(0, self.crash)
         self.mainLoop()
 
 


Test patch:
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -129,6 +129,7 @@ jobs:
           - python-version: '3.11'
             runs-on: 'macos-12'
             job-name: 'macos-12-default-tests'
+            tox-env: 'macos-withcov-alldeps'
 
           # Windows, minimum Python version with select reactor.
           - python-version: '3.7'
diff --git a/src/twisted/internet/test/test_cfreactor.py b/src/twisted/internet/test/test_cfreactor.py
new file mode 100644
--- /dev/null
+++ b/src/twisted/internet/test/test_cfreactor.py
@@ -0,0 +1,101 @@
+from typing import TYPE_CHECKING, List
+
+from twisted.trial.unittest import SynchronousTestCase
+from .reactormixins import ReactorBuilder
+
+if TYPE_CHECKING:
+    fakeBase = SynchronousTestCase
+else:
+    fakeBase = object
+
+
+def noop() -> None:
+    """
+    Do-nothing callable. Stub for testing.
+    """
+
+
+noop()  # Exercise for coverage, since it will never be called below.
+
+
+class CoreFoundationSpecificTests(ReactorBuilder, fakeBase):
+    """
+    Tests for platform interactions of the CoreFoundation-based reactor.
+    """
+
+    _reactors = ["twisted.internet.cfreactor.CFReactor"]
+
+    def test_whiteboxStopSimulating(self) -> None:
+        """
+        CFReactor's simulation timer is None after CFReactor crashes.
+        """
+        r = self.buildReactor()
+        r.callLater(0, r.crash)
+        r.callLater(100, noop)
+        self.runReactor(r)
+        self.assertIs(r._currentSimulator, None)
+
+    def test_callLaterLeakage(self) -> None:
+        """
+        callLater should not leak global state into CoreFoundation which will
+        be invoked by a different reactor running the main loop.
+
+        @note: this test may actually be usable for other reactors as well, so
+            we may wish to promote it to ensure this invariant across other
+            foreign-main-loop reactors.
+        """
+        r = self.buildReactor()
+        delayed = r.callLater(0, noop)
+        r2 = self.buildReactor()
+
+        def stopBlocking() -> None:
+            r2.callLater(0, r2stop)
+
+        def r2stop() -> None:
+            r2.stop()
+
+        r2.callLater(0, stopBlocking)
+        self.runReactor(r2)
+        self.assertEqual(r.getDelayedCalls(), [delayed])
+
+    def test_whiteboxIterate(self) -> None:
+        """
+        C{.iterate()} should remove the CFTimer that will run Twisted's
+        callLaters from the loop, even if one is still pending.  We test this
+        state indirectly with a white-box assertion by verifying the
+        C{_currentSimulator} is set to C{None}, since CoreFoundation does not
+        allow us to enumerate all active timers or sources.
+        """
+        r = self.buildReactor()
+        x: List[int] = []
+        r.callLater(0, x.append, 1)
+        delayed = r.callLater(100, noop)
+        r.iterate()
+        self.assertIs(r._currentSimulator, None)
+        self.assertEqual(r.getDelayedCalls(), [delayed])
+        self.assertEqual(x, [1])
+
+    def test_noTimers(self) -> None:
+        """
+        The loop can wake up just fine even if there are no timers in it.
+        """
+        r = self.buildReactor()
+        stopped = []
+
+        def doStop() -> None:
+            r.stop()
+            stopped.append("yes")
+
+        def sleepThenStop() -> None:
+            r.callFromThread(doStop)
+
+        r.callLater(0, r.callInThread, sleepThenStop)
+        # Can't use runReactor here because it does a callLater.  This is
+        # therefore a somewhat risky test: inherently, this is the "no timed
+        # events anywhere in the reactor" test case and so we can't have a
+        # timeout for it.
+        r.run()
+        self.assertEqual(stopped, ["yes"])
+
+
+globals().update(CoreFoundationSpecificTests.makeTestCaseClasses())