Repository: DataBiosphere/toil

Issue:
When killing single machine batch system, child processes outlive parent process
Since the daddy-thread refactor of the `SingleMachineBatchSystem`, when we ctrl+c Toil, Toil stops without waiting for its child processes to go away. Then they print errors because the leader cleaned up all its working directories out from under them, clobbering your prompt.

We should `kill -9` all the `_toil_worker` processes instead of just vanishing on them.

┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-543)
┆Issue Number: TOIL-543



Discussion:
~This seems to maybe only happen with Cactus; I think Cactus is getting its workers to create other child workers (instead of chaining?).~

I didn't test this with anything other than Cactus yet; I can't say that.
Yeah, when running Cactus at least, we definitely have workers forking off more workers.

Unless we put all the worker processes into their own process groups, there's no way to kill the whole group. We *might* be able to stop the process groups of our children from being re-used until we wait on them to avoid a race?

But we don't necessarily want the Toil workers to not be in Toil's own process group, because that causes trouble for people above us trying to kill Toil and all descendants.

I think Cactus should just stop doing that.
> I think Cactus should just stop doing that.

Sounds very tacky; how would Cactus do this??
It might not be intentionally..

I'm not sure; I don't see anywhere in Cactus where it tries to start its own worker from a worker, but I also don't see anywhere where the Toil worker itself will try to start another worker process. But I'm definitely observing worker processes that claim their parents are other worker processes.

Resolving patch:
diff --git a/src/toil/batchSystems/singleMachine.py b/src/toil/batchSystems/singleMachine.py
--- a/src/toil/batchSystems/singleMachine.py
+++ b/src/toil/batchSystems/singleMachine.py
@@ -14,6 +14,7 @@
 import logging
 import math
 import os
+import signal
 import subprocess
 import sys
 import time
@@ -21,8 +22,10 @@
 from contextlib import contextmanager
 from queue import Empty, Queue
 from threading import Condition, Event, Lock, Thread
+from typing import Dict, List, Optional, Sequence
 
 import toil
+import toil.job
 from toil import worker as toil_worker
 from toil.batchSystems.abstractBatchSystem import (EXIT_STATUS_UNAVAILABLE_VALUE,
                                                    BatchSystemSupport,
@@ -117,10 +120,7 @@ def __init__(self, config, maxCores, maxMemory, maxDisk):
         self.jobIndexLock = Lock()
 
         # A dictionary mapping IDs of submitted jobs to the command line
-        self.jobs = {}
-        """
-        :type: dict[str,toil.job.JobDescription]
-        """
+        self.jobs: Dict[str, toil.job.JobDescription] = {}
 
         # A queue of jobs waiting to be executed. Consumed by the daddy thread.
         self.inputQueue = Queue()
@@ -129,24 +129,15 @@ def __init__(self, config, maxCores, maxMemory, maxDisk):
         self.outputQueue = Queue()
 
         # A dictionary mapping IDs of currently running jobs to their Info objects
-        self.runningJobs = {}
-        """
-        :type: dict[str,Info]
-        """
+        self.runningJobs: Dict[str, Info] = {}
 
         # These next two are only used outside debug-worker mode
 
         # A dict mapping PIDs to Popen objects for running jobs.
         # Jobs that don't fork are executed one at a time in the main thread.
-        self.children = {}
-        """
-        :type: dict[int,subprocess.Popen]
-        """
+        self.children: Dict[int, subprocess.Popen] = {}
         # A dict mapping child PIDs to the Job IDs they are supposed to be running.
-        self.childToJob = {}
-        """
-        :type: dict[int,str]
-        """
+        self.childToJob: Dict[int, str] = {}
 
         # A pool representing available CPU in units of minCores
         self.coreFractions = ResourcePool(int(self.maxCores / self.minCores), 'cores')
@@ -165,14 +156,14 @@ def __init__(self, config, maxCores, maxMemory, maxDisk):
         # Also takes care of resource accounting.
         self.daddyThread = None
         # If it breaks it will fill this in
-        self.daddyException = None
+        self.daddyException: Optional[Exception] = None
 
         if self.debugWorker:
-            log.debug('Started in worker debug mode.')
+            log.debug('Started batch system %s in worker debug mode.', id(self))
         else:
             self.daddyThread = Thread(target=self.daddy, daemon=True)
             self.daddyThread.start()
-            log.debug('Started in normal mode.')
+            log.debug('Started batch system %s in normal mode.', id(self))
 
     def daddy(self):
         """
@@ -190,7 +181,7 @@ def daddy(self):
         """
 
         try:
-            log.debug('Started daddy thread.')
+            log.debug('Started daddy thread for batch system %s.', id(self))
 
             while not self.shuttingDown.is_set():
                 # Main loop
@@ -237,18 +228,16 @@ def daddy(self):
 
 
             # When we get here, we are shutting down.
+            log.debug('Daddy thread cleaning up %d remaining children for batch system %s...', len(self.children), id(self))
 
-            for popen in self.children.values():
-                # Kill all the children, going through popen to avoid signaling re-used PIDs.
-                popen.kill()
-            for popen in self.children.values():
-                # Reap all the children
-                popen.wait()
+            self._stop_and_wait(self.children.values())
+
+            log.debug('Daddy thread for batch system %s finishing because no children should now exist', id(self))
 
             # Then exit the thread.
             return
         except Exception as e:
-            log.critical('Unhandled exception in daddy thread: %s', traceback.format_exc())
+            log.critical('Unhandled exception in daddy thread for batch system %s: %s', id(self), traceback.format_exc())
             # Pass the exception back to the main thread so it can stop the next person who calls into us.
             self.daddyException = e
             raise
@@ -259,7 +248,90 @@ def _checkOnDaddy(self):
             log.critical('Propagating unhandled exception in daddy thread to main thread')
             exc = self.daddyException
             self.daddyException = None
-            raise exc
+            if isinstance(exc, Exception):
+                raise exc
+            else:
+                raise TypeError(f'Daddy thread failed with non-exception: {exc}')
+
+    def _stop_now(self, popens: Sequence[subprocess.Popen]) -> List[int]:
+        """
+        Stop the given child processes and all their children. Does not reap them.
+
+        Returns a list of PGIDs killed, where processes may exist that have not
+        yet received their kill signals.
+        """
+
+        # We will potentially need to poll these PGIDs to ensure that all
+        # processes in them are gone.
+        pgids = []
+
+        for popen in popens:
+            # Kill all the children
+
+            if popen.returncode is None:
+                # Process is not known to be dead. Try and grab its group.
+                try:
+                    pgid = os.getpgid(popen.pid)
+                except OSError:
+                    # It just died. Assume the pgid was its PID.
+                    pgid = popen.pid
+            else:
+                # It is dead. Try it's PID as a PGID and hope we didn't re-use it.
+                pgid = popen.pid
+
+            if pgid != os.getpgrp():
+                # The child process really is in its own group, and not ours.
+
+                # Kill the group, which hopefully hasn't been reused
+                log.debug('Send shutdown kill to process group %s known to batch system %s', pgid, id(self))
+                try:
+                    os.killpg(pgid, signal.SIGKILL)
+                    pgids.append(pgid)
+                except ProcessLookupError:
+                    # It is dead already
+                    pass
+                except PermissionError:
+                    # It isn't ours actually. Ours is dead.
+                    pass
+            else:
+                # Kill the subprocess again through popen in case it somehow
+                # never managed to make the group.
+                popen.kill()
+
+        return pgids
+
+    def _stop_and_wait(self, popens: Sequence[subprocess.Popen]) -> None:
+        """
+        Stop the given child processes and all their children. Blocks until the
+        processes are gone.
+        """
+
+        pgids = self._stop_now(popens)
+
+        for popen in popens:
+            # Wait on all the children
+            popen.wait()
+
+            log.debug('Process %s known to batch system %s is stopped; it returned %s', popen.pid, id(self), popen.returncode)
+
+        for pgid in pgids:
+            try:
+                while True:
+                    # Send a kill to the group again, to see if anything in it
+                    # is still alive. Our first kill might not have been
+                    # delivered yet.
+                    os.killpg(pgid, signal.SIGKILL)
+                    # If that worked it is still alive, so wait for the kernel
+                    # to stop fooling around and kill it.
+                    log.warning('Sent redundant shutdown kill to surviving process group %s known to batch system %s', pgid, id(self))
+                    time.sleep(0.1)
+            except ProcessLookupError:
+                # The group is actually gone now.
+                pass
+            except PermissionError:
+                # The group is not only gone but reused
+                pass
+
 
     def _pollForDoneChildrenIn(self, pid_to_popen):
         """
@@ -387,10 +459,18 @@ def _startChild(self, jobCommand, jobID, coreFractions, jobMemory, jobDisk, envi
                     # So it is important to not lose track of the child process.
 
                     try:
-                        # Launch the job
+                        # Launch the job.
+                        # Make sure it is in its own session (and thus its own
+                        # process group) so that, if the user signals the
+                        # workflow, Toil will be responsible for killing the
+                        # job. This also makes sure that we can signal the job
+                        # and all its children together. We assume that the
+                        # process group ID will equal the PID of the process we
+                        # are starting.
                         popen = subprocess.Popen(jobCommand,
                                                  shell=True,
-                                                 env=dict(os.environ, **environment))
+                                                 env=dict(os.environ, **environment),
+                                                 start_new_session=True)
                     except Exception:
                         # If the job can't start, make sure we release resources now
                         self.coreFractions.release(coreFractions)
@@ -441,7 +521,7 @@ def _startChild(self, jobCommand, jobID, coreFractions, jobMemory, jobDisk, envi
         # Report that.
         return None
 
-    def _handleChild(self, pid):
+    def _handleChild(self, pid: int) -> None:
         """
         Handle a child process PID that has finished.
         The PID must be for a child job we started.
@@ -463,6 +543,21 @@ def _handleChild(self, pid):
         self.childToJob.pop(pid)
         self.children.pop(pid)
 
+        if popen.returncode is None or not callable(getattr(os, 'waitid', None)):
+            # It isn't reaped yet, or we have to reap all children to see if thay're done.
+            # Before we reap it (if possible), kill its PID as a PGID to make sure
+            # it isn't leaving children behind.
+            # TODO: This is a PGID re-use risk on Mac because the process is
+            # reaped already and the PGID may have been reused.
+            try:
+                os.killpg(pid, signal.SIGKILL)
+            except ProcessLookupError:
+                # It is dead already
+                pass
+            except PermissionError:
+                # It isn't ours actually. Ours is dead.
+                pass
+
         # See how the child did, and reap it.
         statusCode = popen.wait()
         if statusCode != 0 and not info.killIntended:
@@ -473,6 +568,31 @@ def _handleChild(self, pid):
             # If we killed it then it shouldn't show up in the queue.
             self.outputQueue.put(UpdatedBatchJobInfo(jobID=jobID, exitStatus=statusCode, wallTime=time.time() - info.time, exitReason=None))
 
+        # Make absolutely sure all processes in the group have received their
+        # kill signals and been cleaned up.
+        # TODO: this opens a PGID reuse risk; we reaped the process and its
+        # PGID may have been re-used. But it probably hasn't been and we
+        # definitely want to make sure all its children died before saying the
+        # job is done. Some might not be dead yet if we don't do this.
+        # TODO: can we safely do this before reaping? Or would we sit forever
+        # signaling a dead but unreaped process?
+        try:
+            while True:
+                # Send a kill to the group again, to see if anything in it
+                # is still alive. Our first kill might not have been
+                # delivered yet.
+                os.killpg(pid, signal.SIGKILL)
+                # If that worked it is still alive, so wait for the kernel
+                # to stop fooling around and kill it.
+                log.warning('Sent redundant job completion kill to surviving process group %s known to batch system %s', pid, id(self))
+                time.sleep(0.1)
+        except ProcessLookupError:
+            # It is dead already
+            pass
+        except PermissionError:
+            # It isn't ours actually. Ours is dead.
+            pass
+
         # Free up the job's resources.
         self.coreFractions.release(coreFractions)
         self.memory.release(jobMemory)
@@ -513,26 +633,35 @@ def issueBatchJob(self, jobDesc):
 
         return jobID
 
-    def killBatchJobs(self, jobIDs):
+    def killBatchJobs(self, jobIDs: Sequence[str]) -> None:
         """Kills jobs by ID."""
 
         self._checkOnDaddy()
 
         log.debug('Killing jobs: {}'.format(jobIDs))
+
+        # Collect the popen handles for the jobs we have to stop
+        popens: List[subprocess.Popen] = []
+
         for jobID in jobIDs:
             if jobID in self.runningJobs:
                 info = self.runningJobs[jobID]
                 info.killIntended = True
                 if info.popen is not None:
-                    log.debug('Send kill to PID %s', info.popen.pid)
-                    info.popen.kill()
-                    log.debug('Sent kill to PID %s', info.popen.pid)
+                    popens.append(info.popen)
                 else:
                     # No popen if running in forkless mode currently
                     assert self.debugWorker
                     log.critical("Can't kill job: %s in debug mode" % jobID)
-                while jobID in self.runningJobs:
-                    pass
+
+        # Stop them all in a batch. Don't reap, because we need the daddy
+        # thread to reap them to mark the jobs as not running anymore.
+        self._stop_now(popens)
+
+        for jobID in jobIDs:
+            while jobID in self.runningJobs:
+                # Wait for the daddy thread to collect them.
+                time.sleep(0.01)
 
     def getIssuedBatchJobIDs(self):
         """Just returns all the jobs that have been run, but not yet returned as updated."""
diff --git a/src/toil/jobStores/aws/jobStore.py b/src/toil/jobStores/aws/jobStore.py
--- a/src/toil/jobStores/aws/jobStore.py
+++ b/src/toil/jobStores/aws/jobStore.py
@@ -71,6 +71,11 @@
 s3_boto3_client = boto3_session.client('s3')
 logger = logging.getLogger(__name__)
 
+# Sometimes we have to wait for multipart uploads to become real. How long
+# should we wait?
+CONSISTENCY_TICKS = 5
+CONSISTENCY_TIME = 1
+
 
 class ChecksumError(Exception):
     """Raised when a download from AWS does not contain the correct data."""
@@ -1195,12 +1200,25 @@ def readFrom(self, readable):
                                                                         **headerArgs)
                                 uploadId = upload['UploadId']
                                 parts = []
+                                logger.debug('Multipart upload started as %s', uploadId)
+
+                        for i in range(CONSISTENCY_TICKS):
+                            # Sometimes we can create a multipart upload and not see it. Wait around for it.
+                            response = client.list_multipart_uploads(Bucket=bucket_name,
+                                                                     MaxUploads=1,
+                                                                     Prefix=compat_bytes(info.fileID))
+                            if len(response['Uploads']) != 0 and response['Uploads'][0]['UploadId'] == uploadId:
+                                logger.debug('Multipart upload visible as %s', uploadId)
+                                break
+                            else:
+                                logger.debug('Multipart upload %s is not visible; we see %s', uploadId, response['Uploads'])
+                                time.sleep(CONSISTENCY_TIME * 2 ** i)
 
                         try:
                             for part_num in itertools.count():
                                 for attempt in retry_s3():
                                     with attempt:
-                                        logger.debug('Uploading part %d of %d bytes', part_num + 1, len(buf))
+                                        logger.debug('Uploading part %d of %d bytes to %s', part_num + 1, len(buf), uploadId)
                                         # TODO: include the Content-MD5 header:
                                         #  https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.complete_multipart_upload
                                         part = client.upload_part(Bucket=bucket_name,
diff --git a/src/toil/jobStores/fileJobStore.py b/src/toil/jobStores/fileJobStore.py
--- a/src/toil/jobStores/fileJobStore.py
+++ b/src/toil/jobStores/fileJobStore.py
@@ -225,10 +225,13 @@ def update(self, job):
         assert not isinstance(job.jobStoreID, TemporaryID), f"Tried to update job {job} without an assigned ID"
 
         # The job is serialised to a file suffixed by ".new"
+        # We insist on creating the file; an existing .new file indicates
+        # multiple simultaneous attempts to update the job, which will lose
+        # updates.
         # The file is then moved to its correct path.
         # Atomicity guarantees use the fact the underlying file systems "move"
         # function is atomic.
-        with open(self._getJobFileName(job.jobStoreID) + ".new", 'wb') as f:
+        with open(self._getJobFileName(job.jobStoreID) + ".new", 'xb') as f:
             pickle.dump(job, f)
         # This should be atomic for the file system
         os.rename(self._getJobFileName(job.jobStoreID) + ".new", self._getJobFileName(job.jobStoreID))


Test patch:
diff --git a/src/toil/test/batchSystems/batchSystemTest.py b/src/toil/test/batchSystems/batchSystemTest.py
--- a/src/toil/test/batchSystems/batchSystemTest.py
+++ b/src/toil/test/batchSystems/batchSystemTest.py
@@ -15,6 +15,7 @@
 import itertools
 import logging
 import os
+import stat
 import subprocess
 import sys
 import tempfile
@@ -23,9 +24,11 @@
 from fractions import Fraction
 from inspect import getsource
 from textwrap import dedent
+from typing import Callable
 from unittest import skipIf
 
-from toil.batchSystems.abstractBatchSystem import (BatchSystemSupport,
+from toil.batchSystems.abstractBatchSystem import (AbstractBatchSystem,
+                                                   BatchSystemSupport,
                                                    InsufficientSystemResources)
 # Don't import any batch systems here that depend on extras
 # in order to import properly. Import them later, in tests
@@ -79,10 +82,7 @@ class AbstractBatchSystemTest(ToilTest, metaclass=ABCMeta):
         """
 
         @abstractmethod
-        def createBatchSystem(self):
-            """
-            :rtype: AbstractBatchSystem
-            """
+        def createBatchSystem(self) -> AbstractBatchSystem:
             raise NotImplementedError
 
         def supportsWallTime(self):
@@ -209,10 +209,11 @@ def testRunJobs(self):
             self.batchSystem.killBatchJobs([10])
 
         def testSetEnv(self):
-            # Parasol disobeys shell rules and stupidly splits the command at
-            # the space character into arguments before exec'ing it, whether
-            # the space is quoted, escaped or not.
+            # Parasol disobeys shell rules and splits the command at the space
+            # character into arguments before exec'ing it, whether the space is
+            # quoted, escaped or not.
 
+            # Start with a relatively safe script
             script_shell = 'if [ "x${FOO}" == "xbar" ] ; then exit 23 ; else exit 42 ; fi'
 
             # Escape the semicolons
@@ -417,19 +418,145 @@ def testIgnoreNode(self):
         # Make sure job is NOT running
         self.assertEqual(set(runningJobIDs), set({}))
 
+def write_temp_file(s: str, temp_dir: str) -> str:
+    """
+    Dump a string into a temp file and return its path.
+    """
+    fd, path = tempfile.mkstemp(dir=temp_dir)
+    try:
+        encoded = s.encode('utf-8')
+        assert os.write(fd, encoded) == len(encoded)
+    except:
+        os.unlink(path)
+        raise
+    else:
+        return path
+    finally:
+        os.close(fd)
+
 @travis_test
 class SingleMachineBatchSystemTest(hidden.AbstractBatchSystemTest):
     """
     Tests against the single-machine batch system
     """
 
-    def supportsWallTime(self):
+    def supportsWallTime(self) -> None:
         return True
 
-    def createBatchSystem(self):
+    def createBatchSystem(self) -> AbstractBatchSystem:
         return SingleMachineBatchSystem(config=self.config,
                                         maxCores=numCores, maxMemory=1e9, maxDisk=2001)
 
+    def testProcessEscape(self, hide: bool = False) -> None:
+        """
+        Test to make sure that child processes and their descendants go away
+        when the Toil workflow stops.
+
+        If hide is true, will try and hide the child processes to make them
+        hard to stop.
+        """
+
+        def script() -> None:
+            #!/usr/bin/env python3
+            import fcntl
+            import os
+            import sys
+            import signal
+            import time
+            from typing import Any
+
+            def handle_signal(sig: Any, frame: Any) -> None:
+                sys.stderr.write(f'{os.getpid()} ignoring signal {sig}\n')
+
+            if hasattr(signal, 'valid_signals'):
+                # We can just ask about the signals
+                all_signals = signal.valid_signals()
+            else:
+                # Fish them out by name
+                all_signals = [getattr(signal, n) for n in dir(signal) if n.startswith('SIG') and not n.startswith('SIG_')]
+
+            for sig in all_signals:
+                # Set up to ignore all signals we can and generally be obstinate
+                if sig != signal.SIGKILL and sig != signal.SIGSTOP:
+                    signal.signal(sig, handle_signal)
+
+            if len(sys.argv) > 2:
+                # Instructed to hide
+                if os.fork():
+                    # Try and hide the first process immediately so getting its
+                    # pgid won't work.
+                    sys.exit(0)
+
+            for depth in range(3):
+                # Bush out into a tree of processes
+                os.fork()
+
+            if len(sys.argv) > 1:
+                fd = os.open(sys.argv[1], os.O_RDONLY)
+                fcntl.lockf(fd, fcntl.LOCK_SH)
+
+            sys.stderr.write(f'{os.getpid()} waiting...\n')
+
+            while True:
+                # Wait around forever
+                time.sleep(60)
+
+        # Get a directory where we can safely dump files.
+        temp_dir = self._createTempDir()
+
+        script_path = write_temp_file(self._getScriptSource(script), temp_dir)
+
+        # We will have all the job processes try and lock this file shared while they are alive.
+        lockable_path = write_temp_file('', temp_dir)
+
+        try:
+            command = f'{sys.executable} {script_path} {lockable_path}'
+            if hide:
+                # Tell the children to stop the first child and hide out in the
+                # process group it made.
+                command += ' hide'
+
+            # Start the job
+            self.batchSystem.issueBatchJob(self._mockJobDescription(command=command, jobName='fork',
+                                                                    jobStoreID='1', requirements=defaultRequirements))
+            # Wait
+            time.sleep(10)
+
+            lockfile = open(lockable_path, 'w')
+
+            if not hide:
+                # In hiding mode the job will finish, and the batch system will
+                # clean up after it promptly. In non-hiding mode the job will
+                # stick around until shutdown, so make sure we can see it.
+
+                # Try to lock the file and make sure it fails
+
+                try:
+                    fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                    assert False, "Should not be able to lock file while job is running"
+                except OSError:
+                    pass
+
+            # Shut down the batch system
+            self.batchSystem.shutdown()
+
+            # After the batch system shuts down, we should be able to get the
+            # lock immediately, because all the children should be gone.
+            fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            # Then we can release it
+            fcntl.lockf(lockfile, fcntl.LOCK_UN)
+        finally:
+            os.unlink(script_path)
+            os.unlink(lockable_path)
+
+    def testHidingProcessEscape(self):
+        """
+        Test to make sure that child processes and their descendants go away
+        when the Toil workflow stops, even if the job process stops and leaves children.
+        """
+
+        self.testProcessEscape(hide=True)
+
 
 @slow
 class MaxCoresSingleMachineBatchSystemTest(ToilTest):
@@ -439,36 +566,25 @@ class MaxCoresSingleMachineBatchSystemTest(ToilTest):
     """
 
     @classmethod
-    def setUpClass(cls):
+    def setUpClass(cls) -> None:
         super(MaxCoresSingleMachineBatchSystemTest, cls).setUpClass()
         logging.basicConfig(level=logging.DEBUG)
 
-    def setUp(self):
+    def setUp(self) -> None:
         super(MaxCoresSingleMachineBatchSystemTest, self).setUp()
 
-        def writeTempFile(s):
-            fd, path = tempfile.mkstemp()
-            try:
-                encoded = s.encode('utf-8')
-                assert os.write(fd, encoded) == len(encoded)
-            except:
-                os.unlink(path)
-                raise
-            else:
-                return path
-            finally:
-                os.close(fd)
+        temp_dir = self._createTempDir()
 
         # Write initial value of counter file containing a tuple of two integers (i, n) where i
         # is the number of currently executing tasks and n the maximum observed value of i
-        self.counterPath = writeTempFile('0,0')
+        self.counterPath = write_temp_file('0,0', temp_dir)
 
-        def script():
+        def script() -> None:
             import fcntl
             import os
             import sys
             import time
-            def count(delta):
+            def count(delta: int) -> None:
                 """
                 Adjust the first integer value in a file by the given amount. If the result
                 exceeds the second integer value, set the second one to the first.
@@ -500,13 +616,13 @@ def count(delta):
             else:
                 count(int(sys.argv[2]))
 
-        self.scriptPath = writeTempFile(dedent('\n'.join(getsource(script).split('\n')[1:])))
+        self.scriptPath = write_temp_file(self._getScriptSource(script), temp_dir)
 
-    def tearDown(self):
+    def tearDown(self) -> None:
         os.unlink(self.scriptPath)
         os.unlink(self.counterPath)
 
-    def scriptCommand(self):
+    def scriptCommand(self) -> str:
         return ' '.join([sys.executable, self.scriptPath, self.counterPath])
 
     @retry_flaky_test()
@@ -625,7 +741,7 @@ def _createConfig(self):
         config.jobStore = self._createTempDir('jobStore')
         return config
 
-    def createBatchSystem(self):
+    def createBatchSystem(self) -> AbstractBatchSystem:
         memory = int(3e9)
         self._startParasol(numCores=numCores, memory=memory)
 
@@ -690,7 +806,7 @@ class GridEngineBatchSystemTest(hidden.AbstractGridEngineBatchSystemTest):
     Tests against the GridEngine batch system
     """
 
-    def createBatchSystem(self):
+    def createBatchSystem(self) -> AbstractBatchSystem:
         from toil.batchSystems.gridengine import GridEngineBatchSystem
         return GridEngineBatchSystem(config=self.config, maxCores=numCores, maxMemory=1000e9,
                                      maxDisk=1e9)
@@ -710,7 +826,7 @@ class SlurmBatchSystemTest(hidden.AbstractGridEngineBatchSystemTest):
     Tests against the Slurm batch system
     """
 
-    def createBatchSystem(self):
+    def createBatchSystem(self) -> AbstractBatchSystem:
         from toil.batchSystems.slurm import SlurmBatchSystem
         return SlurmBatchSystem(config=self.config, maxCores=numCores, maxMemory=1000e9,
                                 maxDisk=1e9)
@@ -729,7 +845,7 @@ class LSFBatchSystemTest(hidden.AbstractGridEngineBatchSystemTest):
     """
     Tests against the LSF batch system
     """
-    def createBatchSystem(self):
+    def createBatchSystem(self) -> AbstractBatchSystem:
         from toil.batchSystems.lsf import LSFBatchSystem
         return LSFBatchSystem(config=self.config, maxCores=numCores,
                               maxMemory=1000e9, maxDisk=1e9)
@@ -748,7 +864,7 @@ def _createDummyConfig(self):
         config.jobStore = self._createTempDir('jobStore')
         return config
 
-    def createBatchSystem(self):
+    def createBatchSystem(self) -> AbstractBatchSystem:
         from toil.batchSystems.torque import TorqueBatchSystem
         return TorqueBatchSystem(config=self.config, maxCores=numCores, maxMemory=1000e9,
                                      maxDisk=1e9)
@@ -767,7 +883,7 @@ class HTCondorBatchSystemTest(hidden.AbstractGridEngineBatchSystemTest):
     Tests against the HTCondor batch system
     """
 
-    def createBatchSystem(self):
+    def createBatchSystem(self) -> AbstractBatchSystem:
         from toil.batchSystems.htcondor import HTCondorBatchSystem
         return HTCondorBatchSystem(config=self.config, maxCores=numCores, maxMemory=1000e9,
                                        maxDisk=1e9)