Repository: DataBiosphere/toil

Issue:
Temporary CWL workdir files are not cleaned up
The CWL runner creates temporary dirs in `workdir` to use as temporary output dirs ([code](https://github.com/DataBiosphere/toil/blob/6a18b8a6a4ad885729ef9fa53542e0a6fd4b6c0a/src/toil/cwl/cwltoil.py#L586)). These are added to `openTempDirs` variable and are supposed to be cleaned up after the workflow is done through the [`cleanTempDirs`](https://github.com/DataBiosphere/toil/blob/6a18b8a6a4ad885729ef9fa53542e0a6fd4b6c0a/src/toil/cwl/cwltoil.py#L1031) function. However, this function is only added as a [followOn to the first CWLJob](https://github.com/DataBiosphere/toil/blob/6a18b8a6a4ad885729ef9fa53542e0a6fd4b6c0a/src/toil/cwl/cwltoil.py#L1273) and not for the nested workflows [*]. This results in intermediate files being left on the workers, which can take up a lot of space.

[*] It doesn't even seem to work for the first one as [`wf1 is CWLJob`](https://github.com/DataBiosphere/toil/blob/6a18b8a6a4ad885729ef9fa53542e0a6fd4b6c0a/src/toil/cwl/cwltoil.py#L1272) is always false. Should it be `isinstance(wf1, CWLJob)` instead?

┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-35)
┆Issue Number: TOIL-35



Discussion:
Actually, it looks like we can safely delete the temporary files in the `run` method itself just [after the output files are copied to the global filesystem](https://github.com/DataBiosphere/toil/blob/6a18b8a6a4ad885729ef9fa53542e0a6fd4b6c0a/src/toil/cwl/cwltoil.py#L615). I tested it on a sample workflow and it seems to work fine as those temporary files are not referenced anymore.
OK, I am able to replicate this issue using the workflow at https://www.commonwl.org/user_guide/05-stdout/index.html

Resolving patch:
diff --git a/src/toil/cwl/cwltoil.py b/src/toil/cwl/cwltoil.py
--- a/src/toil/cwl/cwltoil.py
+++ b/src/toil/cwl/cwltoil.py
@@ -469,40 +469,6 @@ def run(self, file_store):
         self.addChild(realjob)
         return realjob.rv()
 
-def _makeNestedTempDir(top, seed, levels=2):
-    """
-    Gets a temporary directory in the hierarchy of directories under a given
-    top directory.
-
-    This exists to avoid placing too many temporary directories under a single
-    top in a flat structure, which can slow down metadata updates such as
-    deletes on the local file system.
-
-    The seed parameter allows for deterministic placement of the created
-    directory. The seed is hashed into hex digest and the directory structure
-    is created from the initial letters of the digest.
-
-    :param top : string, top directory for the hierarchy
-    :param seed : string, the hierarchy will be generated from this seed string
-    :rtype : string, path to temporary directory - will be created when
-        necessary.
-    """
-    # Valid chars for the creation of temporary directories
-    validDirs = hashlib.md5(six.b(str(seed))).hexdigest()
-    tempDir = top
-    for i in range(max(min(levels, len(validDirs)), 1)):
-        tempDir = os.path.join(tempDir, validDirs[i])
-        if not os.path.exists(tempDir):
-            try:
-                os.makedirs(tempDir)
-            except os.error:
-                if not os.path.exists(tempDir):
-                    # In the case that a collision occurs and
-                    # it is created while we wait then we ignore
-                    raise
-    return tempDir
-
-
 class CWLJob(Job):
     """Execute a CWL tool using cwltool.executors.SingleJobExecutor"""
 
@@ -561,7 +527,6 @@ def __init__(self, tool, cwljob, runtime_context, step_inputs=None):
         self.runtime_context = runtime_context
         self.step_inputs = step_inputs or self.cwltool.tool["inputs"]
         self.workdir = runtime_context.workdir
-        self.openTempDirs = []
 
     def run(self, file_store):
         cwljob = resolve_indirect(self.cwljob)
@@ -582,14 +547,18 @@ def run(self, file_store):
             self.runtime_context.tmpdir or file_store.getLocalTempDir())
         outdir = os.path.join(file_store.getLocalTempDir(), "out")
         os.mkdir(outdir)
-        top_tmp_outdir = self.workdir or os.environ["TMPDIR"]
-        tmp_outdir_prefix = os.path.join(
-            _makeNestedTempDir(top=top_tmp_outdir, seed=outdir, levels=2),
-            "out_tmpdir")
-        self.openTempDirs.append(top_tmp_outdir)
+        # Just keep the temporary output prefix under the job's local temp dir,
+        # next to the outdir.
+        #
+        # If we maintain our own system of nested temp directories, we won't
+        # know when all the jobs using a higher-level directory are ready for
+        # it to be deleted. The local temp dir, under Toil's workDir, will be
+        # cleaned up by Toil.
+        tmp_outdir_prefix = os.path.join(file_store.getLocalTempDir(), "tmp-out")
 
         index = {}
         existing = {}
+        # Prepare the run instructions for cwltool
         runtime_context = self.runtime_context.copy()
         runtime_context.basedir = os.getcwd()
         runtime_context.outdir = outdir
@@ -1027,16 +996,6 @@ def visitSteps(t, op):
             op(s.tool)
             visitSteps(s.embedded_tool, op)
 
-
-def cleanTempDirs(job):
-    """Remove temporarly created directories."""
-    if job is CWLJob and job._succeeded:  # Only CWLJobs have this attribute.
-        for tempDir in job.openTempDirs:
-            if os.path.exists(tempDir):
-                shutil.rmtree(tempDir)
-        job.openTempDirs = []
-
-
 def main(args=None, stdout=sys.stdout):
     """Main method for toil-cwl-runner."""
     cwllogger.removeHandler(defaultStreamHandler)
@@ -1271,8 +1230,6 @@ def set_secondary(fileobj):
                 return 33
 
             wf1.cwljob = initialized_job_order
-            if wf1 is CWLJob:  # Clean up temporary directories only created with CWLJobs.
-                wf1.addFollowOnFn(cleanTempDirs, wf1)
             outobj = toil.start(wf1)
 
         outobj = resolve_indirect(outobj)


Test patch: