Repository: twisted/twisted

Issue:
process tests are flaky
Tests which call spawnProcess, particularly recently `twisted.python.test.test_sendmsg`, have become flaky enough that they're interfering with the downstream CI of the Cryptography project.

This has recently become worse with the introduction of posix_spawn, although it hints at a similar bug underlying our own file-descriptor logic.

This is a fix for the former since I couldn't find anything as obviously wrong with the latter.

Comments:

Patch:
diff --git a/src/twisted/internet/process.py b/src/twisted/internet/process.py
index 51d4bcf00d..89e71b6fef 100644
--- a/src/twisted/internet/process.py
+++ b/src/twisted/internet/process.py
@@ -18,7 +18,8 @@ import signal
 import stat
 import sys
 import traceback
-from typing import TYPE_CHECKING, Callable, Dict, Optional
+from collections import defaultdict
+from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple
 
 _PS_CLOSE: int
 _PS_DUP2: int
@@ -629,6 +630,83 @@ def _listOpenFDs():
     return detector._listOpenFDs()
 
 
+def _getFileActions(
+    fdState: List[Tuple[int, bool]],
+    childToParentFD: Dict[int, int],
+    doClose: int,
+    doDup2: int,
+) -> List[Tuple[int, ...]]:
+    """
+    Get the C{file_actions} parameter for C{posix_spawn} based on the
+    parameters describing the current process state.
+
+    @param fdState: A list of 2-tuples of (file descriptor, close-on-exec
+        flag).
+
+    @param doClose: the integer to use for the 'close' instruction
+
+    @param doDup2: the integer to use for the 'dup2' instruction
+    """
+    fdStateDict = dict(fdState)
+    parentToChildren: Dict[int, List[int]] = defaultdict(list)
+    for inChild, inParent in childToParentFD.items():
+        parentToChildren[inParent].append(inChild)
+    allocated = set(fdStateDict)
+    allocated |= set(childToParentFD.values())
+    allocated |= set(childToParentFD.keys())
+    nextFD = 0
+
+    def allocateFD() -> int:
+        nonlocal nextFD
+        while nextFD in allocated:
+            nextFD += 1
+        allocated.add(nextFD)
+        return nextFD
+
+    result: List[Tuple[int, ...]] = []
+    relocations = {}
+    for inChild, inParent in sorted(childToParentFD.items()):
+        # The parent FD will later be reused by a child FD.
+        parentToChildren[inParent].remove(inChild)
+        if parentToChildren[inChild]:
+            new = relocations[inChild] = allocateFD()
+            result.append((doDup2, inChild, new))
+        if inParent in relocations:
+            result.append((doDup2, relocations[inParent], inChild))
+            if not parentToChildren[inParent]:
+                result.append((doClose, relocations[inParent]))
+        else:
+            if inParent == inChild:
+                if fdStateDict[inParent]:
+                    # If the child is attempting to inherit the parent as-is,
+                    # and it is not close-on-exec, the job is already done; we
+                    # can bail.  Otherwise...
+
+                    tempFD = allocateFD()
+                    # The child wants to inherit the parent as-is, so the
+                    # handle must be heritable.. dup2 makes the new descriptor
+                    # inheritable by default, *but*, per the man page, “if
+                    # fildes and fildes2 are equal, then dup2() just returns
+                    # fildes2; no other changes are made to the existing
+                    # descriptor”, so we need to dup it somewhere else and dup
+                    # it back before closing the temporary place we put it.
+                    result.extend(
+                        [
+                            (doDup2, inParent, tempFD),
+                            (doDup2, tempFD, inChild),
+                            (doClose, tempFD),
+                        ]
+                    )
+            else:
+                result.append((doDup2, inParent, inChild))
+
+    for eachFD, uninheritable in fdStateDict.items():
+        if eachFD not in childToParentFD and not uninheritable:
+            result.append((doClose, eachFD))
+
+    return result
+
+
 @implementer(IProcessTransport)
 class Process(_BaseProcess):
     """
@@ -791,34 +869,17 @@ class Process(_BaseProcess):
         ):
             return False
         fdmap = kwargs.get("fdmap")
-        dupSources = set(fdmap.values())
-        shouldEventuallyClose = _listOpenFDs()
-        closeBeforeDup = []
-        closeAfterDup = []
-        for eachFD in shouldEventuallyClose:
+        fdState = []
+        for eachFD in _listOpenFDs():
             try:
                 isCloseOnExec = fcntl.fcntl(eachFD, fcntl.F_GETFD, fcntl.FD_CLOEXEC)
             except OSError:
                 pass
             else:
-                if eachFD not in fdmap and not isCloseOnExec:
-                    (closeAfterDup if eachFD in dupSources else closeBeforeDup).append(
-                        (_PS_CLOSE, eachFD)
-                    )
-
+                fdState.append((eachFD, isCloseOnExec))
         if environment is None:
             environment = {}
 
-        fileActions = (
-            closeBeforeDup
-            + [
-                (_PS_DUP2, parentFD, childFD)
-                for (childFD, parentFD) in fdmap.items()
-                if childFD != parentFD
-            ]
-            + closeAfterDup
-        )
-
         setSigDef = [
             everySignal
             for everySignal in range(1, signal.NSIG)
@@ -829,7 +890,9 @@ class Process(_BaseProcess):
             executable,
             args,
             environment,
-            file_actions=fileActions,
+            file_actions=_getFileActions(
+                fdState, fdmap, doClose=_PS_CLOSE, doDup2=_PS_DUP2
+            ),
             setsigdef=setSigDef,
         )
         self.status = -1


Test patch:
diff --git a/src/twisted/internet/test/test_process.py b/src/twisted/internet/test/test_process.py
index 48bd9e0eb3..0b8cdee750 100644
--- a/src/twisted/internet/test/test_process.py
+++ b/src/twisted/internet/test/test_process.py
@@ -29,7 +29,7 @@ from twisted.python.compat import networkString
 from twisted.python.filepath import FilePath, _asFilesystemBytes
 from twisted.python.log import err, msg
 from twisted.python.runtime import platform
-from twisted.trial.unittest import TestCase
+from twisted.trial.unittest import SynchronousTestCase, TestCase
 
 # Get the current Python executable as a bytestring.
 pyExe = FilePath(sys.executable)._asBytesPath()
@@ -39,21 +39,35 @@ _uidgidSkipReason = ""
 properEnv = dict(os.environ)
 properEnv["PYTHONPATH"] = os.pathsep.join(sys.path)
 try:
-    import resource as _resource
-
     from twisted.internet import process as _process
 
     if os.getuid() != 0:
         _uidgidSkip = True
         _uidgidSkipReason = "Cannot change UID/GID except as root"
 except ImportError:
-    resource = None
     process = None
     _uidgidSkip = True
     _uidgidSkipReason = "Cannot change UID/GID on Windows"
 else:
-    resource = _resource
     process = _process
+    from twisted.internet.process import _getFileActions
+
+
+def _getRealMaxOpenFiles() -> int:
+    from resource import RLIMIT_NOFILE, getrlimit
+
+    potentialLimits = [getrlimit(RLIMIT_NOFILE)[0], os.sysconf("SC_OPEN_MAX")]
+    if platform.isMacOSX():
+        # The OPEN_MAX macro is still used on macOS.  Sometimes, you can open
+        # file descriptors that go all the way up to SC_OPEN_MAX or
+        # RLIMIT_NOFILE (which *should* be the same) but OPEN_MAX still trumps
+        # in some circumstances.  In particular, when using the posix_spawn
+        # family of functions, file_actions on files greater than OPEN_MAX
+        # return a EBADF errno.  Since this macro is deprecated on every other
+        # UNIX, it's not exposed by Python, since you're really supposed to get
+        # these values somewhere else...
+        potentialLimits.append(0x2800)
+    return min(potentialLimits)
 
 
 def onlyOnPOSIX(testMethod):
@@ -64,7 +78,7 @@ def onlyOnPOSIX(testMethod):
 
     @return: the C{testMethod} argument.
     """
-    if resource is None:
+    if os.name != "posix":
         testMethod.skip = "Test only applies to POSIX platforms."
     return testMethod
 
@@ -99,6 +113,7 @@ class ProcessTestsBuilderBase(ReactorBuilder):
     """
 
     requiredInterfaces = [IReactorProcess]
+    usePTY: bool
 
     def test_processTransportInterface(self):
         """
@@ -381,14 +396,6 @@ class ProcessTestsBuilderBase(ReactorBuilder):
         self.assertEqual(result, [b"Foo" + os.linesep.encode("ascii")])
 
     @skipIf(platform.isWindows(), "Test only applies to POSIX platforms.")
-    # If you see this comment and are running on macOS, try to see if this pass on your environment.
-    # Only run this test on Linux and macOS local tests and Linux CI platforms.
-    # This should be used for POSIX tests that are expected to pass on macOS but which fail due to lack of macOS developers.
-    # We still want to run it on local development macOS environments to help developers discover and fix this issue.
-    @skipIf(
-        platform.isMacOSX() and os.environ.get("CI", "").lower() == "true",
-        "Skipped on macOS CI env.",
-    )
     def test_openFileDescriptors(self):
         """
         Processes spawned with spawnProcess() close all extraneous file
@@ -431,7 +438,8 @@ sys.stdout.flush()"""
         # might at least hypothetically select.)
 
         fudgeFactor = 17
-        unlikelyFD = resource.getrlimit(resource.RLIMIT_NOFILE)[0] - fudgeFactor
+        hardResourceLimit = _getRealMaxOpenFiles()
+        unlikelyFD = hardResourceLimit - fudgeFactor
 
         os.dup2(w, unlikelyFD)
         self.addCleanup(os.close, unlikelyFD)
@@ -886,7 +894,7 @@ class ProcessTestsBuilder(ProcessTestsBuilderBase):
         """
         us = b"twisted.internet.test.process_cli"
 
-        args = [b"hello", b'"', b" \t|<>^&", br'"\\"hello\\"', br'"foo\ bar baz\""']
+        args = [b"hello", b'"', b" \t|<>^&", rb'"\\"hello\\"', rb'"foo\ bar baz\""']
         # Ensure that all non-NUL characters can be passed too.
         allChars = "".join(map(chr, range(1, 255)))
         if isinstance(allChars, str):
@@ -1102,3 +1110,97 @@ class ReapingNonePidsLogsProperly(TestCase):
             self.expected_message,
             "Wrong error message logged",
         )
+
+
+CLOSE = 9999
+DUP2 = 10101
+
+
+@onlyOnPOSIX
+class GetFileActionsTests(SynchronousTestCase):
+    """
+    Tests to make sure that the file actions computed for posix_spawn are
+    correct.
+    """
+
+    def test_nothing(self) -> None:
+        """
+        If there are no open FDs and no requested child FDs, there's nothing to
+        do.
+        """
+        self.assertEqual(_getFileActions([], {}, CLOSE, DUP2), [])
+
+    def test_closeNoCloexec(self) -> None:
+        """
+        If a file descriptor is not requested but it is not close-on-exec, it
+        should be closed.
+        """
+        self.assertEqual(_getFileActions([(0, False)], {}, CLOSE, DUP2), [(CLOSE, 0)])
+
+    def test_closeWithCloexec(self) -> None:
+        """
+        If a file descriptor is close-on-exec and it is not requested, no
+        action should be taken.
+        """
+        self.assertEqual(_getFileActions([(0, True)], {}, CLOSE, DUP2), [])
+
+    def test_moveWithCloexec(self) -> None:
+        """
+        If a file descriptor is close-on-exec and it is moved, then there should be a dup2 but no close.
+        """
+        self.assertEqual(
+            _getFileActions([(0, True)], {3: 0}, CLOSE, DUP2), [(DUP2, 0, 3)]
+        )
+
+    def test_moveNoCloexec(self) -> None:
+        """
+        If a file descriptor is not close-on-exec and it is moved, then there
+        should be a dup2 followed by a close.
+        """
+        self.assertEqual(
+            _getFileActions([(0, False)], {3: 0}, CLOSE, DUP2),
+            [(DUP2, 0, 3), (CLOSE, 0)],
+        )
+
+    def test_stayPut(self) -> None:
+        """
+        If a file descriptor is not close-on-exec and it's left in the same
+        place, then there should be no actions taken.
+        """
+        self.assertEqual(_getFileActions([(0, False)], {0: 0}, CLOSE, DUP2), [])
+
+    def test_cloexecStayPut(self) -> None:
+        """
+        If a file descriptor is close-on-exec and it's left in the same place,
+        then we need to DUP2 it elsewhere, close the original, then DUP2 it
+        back so it doesn't get closed by the implicit exec at the end of
+        posix_spawn's file actions.
+        """
+        self.assertEqual(
+            _getFileActions([(0, True)], {0: 0}, CLOSE, DUP2),
+            [(DUP2, 0, 1), (DUP2, 1, 0), (CLOSE, 1)],
+        )
+
+    def test_inheritableConflict(self) -> None:
+        """
+        If our file descriptor mapping requests that file descriptors change
+        places, we must DUP2 them to a new location before DUP2ing them back.
+        """
+        self.assertEqual(
+            _getFileActions(
+                [(0, False), (1, False)],
+                {
+                    0: 1,
+                    1: 0,
+                },
+                CLOSE,
+                DUP2,
+            ),
+            [
+                (DUP2, 0, 2),  # we're working on the desired fd 0 for the
+                # child, so we are about to overwrite 0.
+                (DUP2, 1, 0),  # move 1 to 0, also closing 0
+                (DUP2, 2, 1),  # move 2 to 1, closing previous 1
+                (CLOSE, 2),  # done with 2
+            ],
+        )