Repository: DataBiosphere/toil

Issue:
Toil fails with RecursionError when restarting big workflow
```
Traceback (most recent call last):
  File "//venv/bin/cactus-align", line 8, in <module>
    sys.exit(main())
  File "//venv/lib/python3.6/site-packages/cactus/setup/cactus_align.py", line 139, in main
    results_dict = toil.restart()
  File "/usr/local/lib/python3.6/dist-packages/toil/common.py", line 860, in restart
    rootJobGraph = self._jobStore.clean(jobCache=self._jobCache)
  File "/usr/local/lib/python3.6/dist-packages/toil/jobStores/abstractJobStore.py", line 526, in clean
    getConnectedJobs(self.loadRootJob())
  File "/usr/local/lib/python3.6/dist-packages/toil/jobStores/abstractJobStore.py", line 517, in getConnectedJobs
    getConnectedJobs(getJob(successorJobStoreID))
  File "/usr/local/lib/python3.6/dist-packages/toil/jobStores/abstractJobStore.py", line 517, in getConnectedJobs
    getConnectedJobs(getJob(successorJobStoreID))
  File "/usr/local/lib/python3.6/dist-packages/toil/jobStores/abstractJobStore.py", line 517, in getConnectedJobs
    getConnectedJobs(getJob(successorJobStoreID))
  [Previous line repeated 992 more times]
  File "/usr/local/lib/python3.6/dist-packages/toil/jobStores/abstractJobStore.py", line 514, in getConnectedJobs
    for successorJobStoreID in [x.jobStoreID for x in jobs]:
RecursionError: maximum recursion depth exceeded
```

┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-765)
┆Issue Number: TOIL-765



Discussion:

Resolving patch:
diff --git a/src/toil/jobStores/abstractJobStore.py b/src/toil/jobStores/abstractJobStore.py
--- a/src/toil/jobStores/abstractJobStore.py
+++ b/src/toil/jobStores/abstractJobStore.py
@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
 import logging
 import pickle
 import re
@@ -23,12 +22,13 @@
 from http.client import BadStatusLine
 from urllib.request import urlopen
 from uuid import uuid4
-
+from typing import Set, Union
 from requests.exceptions import HTTPError
 
 from toil.common import safeUnpickleFromStream
 from toil.fileStores import FileID
 from toil.job import (CheckpointJobDescription,
+                      TemporaryID,
                       JobException,
                       ServiceJobDescription)
 from toil.lib.memoize import memoize
@@ -483,8 +483,7 @@ def clean(self, jobCache=None):
         if jobCache is None:
             logger.warning("Cleaning jobStore recursively. This may be slow.")
 
-        # Functions to get and check the existence of jobs, using the jobCache
-        # if present
+        # Functions to get and check the existence of jobs, using the jobCache if present
         def getJobDescription(jobId):
             if jobCache is not None:
                 try:
@@ -520,47 +519,65 @@ def getJobDescriptions():
             else:
                 return self.jobs()
 
-        # Iterate from the root JobDescription and collate all jobs that are reachable from it
-        # All other jobs returned by self.jobs() are orphaned and can be removed
-        reachableFromRoot = set()
-
-        def getConnectedJobs(jobDescription):
-            if jobDescription.jobStoreID in reachableFromRoot:
-                return
-            reachableFromRoot.add(jobDescription.jobStoreID)
-            # Traverse jobs in stack
-            for jobs in jobDescription.stack:
-                for successorJobStoreID in jobs:
-                    if (successorJobStoreID not in reachableFromRoot
-                        and haveJob(successorJobStoreID)):
-                        getConnectedJobs(getJobDescription(successorJobStoreID))
-            # Traverse service jobs
-            for serviceJobStoreID in jobDescription.services:
-                if haveJob(serviceJobStoreID):
-                    assert serviceJobStoreID not in reachableFromRoot
-                    reachableFromRoot.add(serviceJobStoreID)
-
-        logger.debug("Checking job graph connectivity...")
-        getConnectedJobs(self.loadRootJob())
-        logger.debug("%d jobs reachable from root." % len(reachableFromRoot))
+        def get_jobs_reachable_from_root() -> Set[Union[TemporaryID, str]]:
+            """
+            Traverse the job graph from the root job and return a flattened set of all active jobstore IDs.
+
+            Note: Jobs returned by self.jobs(), but not this function, are orphaned, and can be removed as dead jobs.
+            """
+            # Iterate from the root JobDescription and collate all jobs that are reachable from it.
+            root_job_description = self.loadRootJob()
+            reachable_from_root = set()
+
+            # Add first root job outside of the loop below.
+            reachable_from_root.add(root_job_description.jobStoreID)
+            # add all of root's linked service jobs as well
+            for service_jobstore_id in root_job_description.services:
+                if haveJob(service_jobstore_id):
+                    reachable_from_root.add(service_jobstore_id)
+
+            # Unprocessed means it might have successor jobs we need to add.
+            unprocessed_job_descriptions = [root_job_description]
+
+            while unprocessed_job_descriptions:
+                new_job_descriptions_to_process = []  # Reset.
+                for job_description in unprocessed_job_descriptions:
+                    for jobs in job_description.stack:
+                        for successor_jobstore_id in jobs:
+                            if successor_jobstore_id not in reachable_from_root and haveJob(successor_jobstore_id):
+                                successor_job_description = getJobDescription(successor_jobstore_id)
+
+                                # Add each successor job.
+                                reachable_from_root.add(successor_job_description.jobStoreID)
+                                # Add all of the successor's linked service jobs as well.
+                                for service_jobstore_id in successor_job_description.services:
+                                    if haveJob(service_jobstore_id):
+                                        reachable_from_root.add(service_jobstore_id)
+
+                                new_job_descriptions_to_process.append(successor_job_description)
+                unprocessed_job_descriptions = new_job_descriptions_to_process
+
+            logger.debug(f"{len(reachable_from_root)} jobs reachable from root.")
+            return reachable_from_root
+
+        reachable_from_root = get_jobs_reachable_from_root()
 
         # Cleanup jobs that are not reachable from the root, and therefore orphaned
-        jobsToDelete = [x for x in getJobDescriptions() if x.jobStoreID not in reachableFromRoot]
+        # TODO: Avoid reiterating reachable_from_root (which may be very large)
+        jobsToDelete = [x for x in getJobDescriptions() if x.jobStoreID not in reachable_from_root]
         for jobDescription in jobsToDelete:
             # clean up any associated files before deletion
             for fileID in jobDescription.filesToDelete:
                 # Delete any files that should already be deleted
-                logger.warning("Deleting file '%s'. It is marked for deletion but has not yet been "
-                            "removed.", fileID)
+                logger.warning(f"Deleting file '{fileID}'. It is marked for deletion but has not yet been removed.")
                 self.deleteFile(fileID)
             # Delete the job from us and the cache
             deleteJob(jobDescription.jobStoreID)
 
-        jobDescriptionsReachableFromRoot = {id: getJobDescription(id) for id in reachableFromRoot}
+        jobDescriptionsReachableFromRoot = {id: getJobDescription(id) for id in reachable_from_root}
 
         # Clean up any checkpoint jobs -- delete any successors it
-        # may have launched, and restore the job to a pristine
-        # state
+        # may have launched, and restore the job to a pristine state
         jobsDeletedByCheckpoints = set()
         for jobDescription in [desc for desc in jobDescriptionsReachableFromRoot.values() if isinstance(desc, CheckpointJobDescription)]:
             if jobDescription.jobStoreID in jobsDeletedByCheckpoints:


Test patch: