Repository: DataBiosphere/toil

Issue:
Name WDL output subdirectories with human-readable names instead of UUIDs
I ran a WDL workflow, and I got an output directory that looks like this:

```
.
├── 045a7e47-d56d-4fcc-809f-c710bb9dee26
│   ├── HG002.chr1.bam
│   ├── HG002.chr1.bam.bai
│   ├── HG002.chr10.bam
│   ├── HG002.chr10.bam.bai
│   ├── HG002.chr11.bam
│   ├── HG002.chr11.bam.bai
│   ├── HG002.chr12.bam
│   ├── HG002.chr12.bam.bai
│   ├── HG002.chr13.bam
│   ├── HG002.chr13.bam.bai
│   ├── HG002.chr14.bam
│   ├── HG002.chr14.bam.bai
│   ├── HG002.chr15.bam
│   ├── HG002.chr15.bam.bai
│   ├── HG002.chr16.bam
│   ├── HG002.chr16.bam.bai
│   ├── HG002.chr17.bam
│   ├── HG002.chr17.bam.bai
│   ├── HG002.chr18.bam
│   ├── HG002.chr18.bam.bai
│   ├── HG002.chr19.bam
│   ├── HG002.chr19.bam.bai
│   ├── HG002.chr2.bam
│   ├── HG002.chr2.bam.bai
│   ├── HG002.chr20.bam
│   ├── HG002.chr20.bam.bai
│   ├── HG002.chr21.bam
│   ├── HG002.chr21.bam.bai
│   ├── HG002.chr22.bam
│   ├── HG002.chr22.bam.bai
│   ├── HG002.chr3.bam
│   ├── HG002.chr3.bam.bai
│   ├── HG002.chr4.bam
│   ├── HG002.chr4.bam.bai
│   ├── HG002.chr5.bam
│   ├── HG002.chr5.bam.bai
│   ├── HG002.chr6.bam
│   ├── HG002.chr6.bam.bai
│   ├── HG002.chr7.bam
│   ├── HG002.chr7.bam.bai
│   ├── HG002.chr8.bam
│   ├── HG002.chr8.bam.bai
│   ├── HG002.chr9.bam
│   ├── HG002.chr9.bam.bai
│   ├── HG002.chrX.bam
│   ├── HG002.chrX.bam.bai
│   ├── HG002.chrY.bam
│   └── HG002.chrY.bam.bai
├── 1a542638-0756-4b85-8871-6d149206fcd9
│   └── HG002.gaf.gz
├── 56edb39f-cd4f-41d3-9311-cb614edbefc1
│   └── vcfeval_results.tar.gz
├── dc107a3f-258e-47a9-be95-b704900484a8
│   ├── HG002.vcf.gz
│   └── HG002.vcf.gz.tbi
├── de250502-7236-424e-8d09-8135dda1bd6a
│   └── happy_results.tar.gz
└── dffe3688-697d-42b8-b026-c8297efde40e
    ├── HG002.vcf.gz
    └── HG002.vcf.gz.tbi

6 directories, 55 files
```

The UUID directory names are impenetrable. Unless I have the WDL JSON output I can’t tell what anything is (which `HG002.vcf.gz` is which?). If I do have it it is asking me to navigate to UUID-named directories to get my files, which is rude of it.

We should organize the outputs in a more human-readable way, like maybe by task name as long as that is unambiguous.



┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-1612)
┆Issue Number: TOIL-1612


Discussion:

Patch:
diff --git a/src/toil/wdl/wdltoil.py b/src/toil/wdl/wdltoil.py
index 8d560719..72a8a372 100755
--- a/src/toil/wdl/wdltoil.py
+++ b/src/toil/wdl/wdltoil.py
@@ -593,17 +593,24 @@ TOIL_URI_SCHEME = 'toilfile:'
 # We also want to store the filename even if it does not exist, so use a sentinel URI scheme (can be useful in error messages)
 TOIL_NONEXISTENT_URI_SCHEME = 'nonexistent:'
 
-def pack_toil_uri(file_id: FileID, dir_id: uuid.UUID, file_basename: str) -> str:
+def pack_toil_uri(file_id: FileID, task_path: str, dir_id: uuid.UUID, file_basename: str) -> str:
     """
-    Encode a Toil file ID and its source path in a URI that starts with the scheme in TOIL_URI_SCHEME.
+    Encode a Toil file ID and metadata about who wrote it as a URI.
+
+    The URI will start with the scheme in TOIL_URI_SCHEME.
     """
 
     # We urlencode everything, including any slashes. We need to use a slash to
     # set off the actual filename, so the WDL standard library basename
     # function works correctly.
-    return f"{TOIL_URI_SCHEME}{quote(file_id.pack(), safe='')}/{quote(str(dir_id))}/{quote(file_basename, safe='')}"
+    return TOIL_URI_SCHEME + "/".join([
+        quote(file_id.pack(), safe=''),
+        quote(task_path, safe=''),
+        quote(str(dir_id)),
+        quote(file_basename, safe='')
+    ])
 
-def unpack_toil_uri(toil_uri: str) -> Tuple[FileID, str, str]:
+def unpack_toil_uri(toil_uri: str) -> Tuple[FileID, str, str, str]:
     """
     Unpack a URI made by make_toil_uri to retrieve the FileID and the basename
     (no path prefix) that the file is supposed to have.
@@ -617,13 +624,67 @@ def unpack_toil_uri(toil_uri: str) -> Tuple[FileID, str, str]:
         raise ValueError(f"URI doesn't start with {TOIL_URI_SCHEME} and should: {toil_uri}")
     # Split encoded file ID from filename
     parts = parts[1].split('/')
-    if len(parts) != 3:
+    if len(parts) != 4:
         raise ValueError(f"Wrong number of path segments in URI: {toil_uri}")
     file_id = FileID.unpack(unquote(parts[0]))
-    parent_id = unquote(parts[1])
-    file_basename = unquote(parts[2])
+    task_path = unquote(parts[1])
+    parent_id = unquote(parts[2])
+    file_basename = unquote(parts[3])
+
+    return file_id, task_path, parent_id, file_basename
+
+
+DirectoryNamingStateDict = Dict[str, Tuple[Dict[str, str], Set[str]]]
+def choose_human_readable_directory(root_dir: str, source_task_path: str, parent_id: str, state: DirectoryNamingStateDict) -> str:
+    """
+    Select a good directory to save files from a task and source directory in.
+
+    The directories involved may not exist.
+
+    :param root_dir: Directory that the path will be under
+    :param source_task_path: The dotted WDL name of whatever generated the
+        file. We assume this is an acceptable filename component.
+    :param parent_id: UUID of the directory that the file came from. All files
+        with the same parent ID will be placed as siblings files in a shared
+        parent directory.
+    :param state: A state dict that must be passed to repeated calls.
+    """
+
+    # We need to always put things as siblings if they come from the same UUID
+    # even if different tasks generated them. So the first task we download
+    # from will get to name the directory for a parent ID.
+
+    # Get the state info for this root directory.
+    #
+    # For each parent ID, we need the directory we are using for it (dict).
+    #
+    # For each local directory, we need to know if we used it for a parent ID already (set).
+    id_to_dir, used_dirs = state.setdefault(root_dir, ({}, set()))
+    logger.debug("Pick location for parent %s source %s root %s against id map %s and used set %s", parent_id, source_task_path, root_dir, id_to_dir, used_dirs)
+    if parent_id not in id_to_dir:
+        # Make a path for this parent named after this source task
+
+        # Problem: If we put any files right at the root of the source task
+        # directory, then we can't put any directories with guessable names in
+        # it, because we might later come across a file with that name that
+        # must be sibling to an existing file. So if a task uploads from
+        # multiple sources or otherwise manages to collide with our numbering,
+        # we will make multiple directories for it.
+
+        candidate = source_task_path
+        deduplicator = len(used_dirs)
+        while candidate in used_dirs:
+            # We use one run of deduplicating numbers across all the names.
+            candidate = f"{source_task_path}-{deduplicator}"
+            deduplicator += 1
+
+        id_to_dir[parent_id] = candidate
+        used_dirs.add(candidate)
+
+    result = os.path.join(root_dir, id_to_dir[parent_id])
+    logger.debug("Picked path %s", result)
+    return result
 
-    return file_id, parent_id, file_basename
 
 def evaluate_output_decls(output_decls: List[WDL.Tree.Decl], all_bindings: WDL.Env.Bindings[WDL.Value.Base], standard_library: ToilWDLStdLibBase) -> WDL.Env.Bindings[WDL.Value.Base]:
     """
@@ -672,7 +733,7 @@ class NonDownloadingSize(WDL.StdLib._Size):
                 if uri.startswith(TOIL_URI_SCHEME):
                     # This is a Toil File ID we encoded; we have the size
                     # available.
-                    file_id, _, _ = unpack_toil_uri(uri)
+                    file_id = unpack_toil_uri(uri)[0]
                     # Use the encoded size
                     total_size += file_id.size
                 else:
@@ -738,12 +799,23 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
     """
     Standard library implementation for WDL as run on Toil.
     """
-    def __init__(self, file_store: AbstractFileStore, execution_dir: Optional[str] = None, enforce_existence: bool = True):
+    def __init__(
+        self,
+        file_store: AbstractFileStore,
+        task_path: str,
+        execution_dir: Optional[str] = None,
+        enforce_existence: bool = True,
+        share_files_with: Optional["ToilWDLStdLibBase"] = None
+    ) -> None:
         """
         Set up the standard library.
-
+        
+        :param task_path: Dotted WDL name of the part of the workflow this library is working for.
         :param execution_dir: Directory to use as the working directory for workflow code.
-        :param enforce_existence: If true, then if a file is detected as nonexistent, raise an error. Else, let it pass through
+        :param enforce_existence: If true, then if a file is detected as
+            nonexistent, raise an error. Else, let it pass through
+        :param share_files_with: If set to an existing standard library
+            instance, use the same file upload and download paths as it.
         """
         # TODO: Just always be the 1.2 standard library.
         wdl_version = "1.2"
@@ -756,22 +828,35 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
         # to always download the file.
         self.size = NonDownloadingSize(self)
 
+        # Save the task path to tag uploads
+        self._task_path = task_path
+
         # Keep the file store around so we can access files.
         self._file_store = file_store
 
-        # UUID to differentiate which node files are virtualized from
-        self._parent_dir_to_ids: Dict[str, uuid.UUID] = dict()
-
-        # Map forward from virtualized files to absolute devirtualized ones.
-        self._virtualized_to_devirtualized: Dict[str, str] = {}
-        # Allow mapping back from absolute devirtualized files to virtualized
-        # paths, to save re-uploads.
-        self._devirtualized_to_virtualized: Dict[str, str] = {}
-
         self._execution_dir = execution_dir or gettempdir()
 
         self._enforce_existence = enforce_existence
 
+        if share_files_with is None:
+            # We get fresh file download/upload state
+
+            # Map forward from virtualized files to absolute devirtualized ones.
+            self._virtualized_to_devirtualized: Dict[str, str] = {}
+            # Allow mapping back from absolute devirtualized files to virtualized
+            # paths, to save re-uploads.
+            self._devirtualized_to_virtualized: Dict[str, str] = {}
+            # State we need for choosing good names for devirtualized files
+            self._devirtualization_state: DirectoryNamingStateDict = {}
+            # UUID to differentiate which node files are virtualized from
+            self._parent_dir_to_ids: Dict[str, uuid.UUID] = dict()
+        else:
+            # Share file download/upload state
+            self._virtualized_to_devirtualized = share_files_with._virtualized_to_devirtualized
+            self._devirtualized_to_virtualized = share_files_with._devirtualized_to_virtualized
+            self._devirtualization_state = share_files_with._devirtualization_state
+            self._parent_dir_to_ids = share_files_with._parent_dir_to_ids
+
     @property
     def execution_dir(self) -> str:
         return self._execution_dir
@@ -783,24 +868,6 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
 
         return list(self._virtualized_to_devirtualized.values())
 
-    def share_files(self, other: "ToilWDLStdLibBase") -> None:
-        """
-        Share caches for devirtualizing and virtualizing files with another instance.
-
-        Files devirtualized by one instance can be re-virtualized back to their
-        original virtualized filenames by the other.
-        """
-
-        if id(self._virtualized_to_devirtualized) != id(other._virtualized_to_devirtualized):
-            # Merge the virtualized to devirtualized mappings
-            self._virtualized_to_devirtualized.update(other._virtualized_to_devirtualized)
-            other._virtualized_to_devirtualized = self._virtualized_to_devirtualized
-
-        if id(self._devirtualized_to_virtualized) != id(other._devirtualized_to_virtualized):
-            # Merge the devirtualized to virtualized mappings
-            self._devirtualized_to_virtualized.update(other._devirtualized_to_virtualized)
-            other._devirtualized_to_virtualized = self._devirtualized_to_virtualized
-
     @memoize
     def _devirtualize_filename(self, filename: str) -> str:
         """
@@ -808,14 +875,29 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
         on the local host.
         """
         
-        result = self.devirtualize_to(filename, self._file_store.localTempDir, self._file_store, self._execution_dir,
-                                      self._devirtualized_to_virtualized, self._virtualized_to_devirtualized, self._enforce_existence)
+        result = self.devirtualize_to(
+            filename,
+            self._file_store.localTempDir,
+            self._file_store,
+            self._execution_dir,
+            self._devirtualization_state,
+            self._devirtualized_to_virtualized,
+            self._virtualized_to_devirtualized,
+            self._enforce_existence
+        )
         return result
 
     @staticmethod
-    def devirtualize_to(filename: str, dest_dir: str, file_source: Union[AbstractFileStore, Toil], execution_dir: Optional[str],
-                        devirtualized_to_virtualized: Optional[Dict[str, str]] = None, virtualized_to_devirtualized: Optional[Dict[str, str]] = None,
-                        enforce_existence: bool = True) -> str:
+    def devirtualize_to(
+        filename: str,
+        dest_dir: str,
+        file_source: Union[AbstractFileStore, Toil],
+        execution_dir: Optional[str],
+        state: DirectoryNamingStateDict,
+        devirtualized_to_virtualized: Optional[Dict[str, str]] = None,
+        virtualized_to_devirtualized: Optional[Dict[str, str]] = None,
+        enforce_existence: bool = True
+    ) -> str:
         """
         Download or export a WDL virtualized filename/URL to the given directory.
 
@@ -830,7 +912,8 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
 
         The input filename could already be devirtualized. In this case, the filename
         should not be added to the cache
-
+        
+        :param state: State dict which must be shared among successive calls into a dest_dir.
         :param enforce_existence: Raise an error if the file is nonexistent. Else, let it pass through.
         """
 
@@ -851,14 +934,10 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
             if filename.startswith(TOIL_URI_SCHEME):
                 # This is a reference to the Toil filestore.
                 # Deserialize the FileID
-                file_id, parent_id, file_basename = unpack_toil_uri(filename)
+                file_id, task_path, parent_id, file_basename = unpack_toil_uri(filename)
 
                 # Decide where it should be put.
-                # This is a URI with the "parent" UUID attached to the filename.
-                # Use UUID as folder name rather than a new temp folder to reduce internal clutter.
-                # Put the UUID in the destination path in order for tasks to
-                # see where to put files depending on their parents.
-                dir_path = os.path.join(dest_dir, parent_id)
+                dir_path = choose_human_readable_directory(dest_dir, task_path, parent_id, state)
             elif filename.startswith(TOIL_NONEXISTENT_URI_SCHEME):
                 if enforce_existence:
                     raise FileNotFoundError(f"File {filename[len(TOIL_NONEXISTENT_URI_SCHEME):]} was not available when virtualized!")
@@ -960,7 +1039,7 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
 
         file_dir = os.path.dirname(abs_filename)
         parent_id = self._parent_dir_to_ids.setdefault(file_dir, uuid.uuid4())
-        result = pack_toil_uri(file_id, parent_id, os.path.basename(abs_filename))
+        result = pack_toil_uri(file_id, self._task_path, parent_id, os.path.basename(abs_filename))
         logger.debug('Virtualized %s as WDL file %s', filename, result)
         # Remember the upload in case we share a cache
         self._devirtualized_to_virtualized[abs_filename] = result
@@ -977,13 +1056,13 @@ class ToilWDLStdLibTaskCommand(ToilWDLStdLibBase):
     are host-side paths.
     """
 
-    def __init__(self, file_store: AbstractFileStore, container: TaskContainer, execution_dir: Optional[str] = None):
+    def __init__(self, file_store: AbstractFileStore, task_path: str, container: TaskContainer, execution_dir: Optional[str] = None):
         """
         Set up the standard library for the task command section.
         """
 
         # TODO: Don't we want to make sure we don't actually use the file store?
-        super().__init__(file_store, execution_dir)
+        super().__init__(file_store, task_path, execution_dir=execution_dir)
         self.container = container
 
     @memoize
@@ -1032,19 +1111,35 @@ class ToilWDLStdLibTaskOutputs(ToilWDLStdLibBase, WDL.StdLib.TaskOutputs):
     functions only allowed in task output sections.
     """
 
-    def __init__(self, file_store: AbstractFileStore, stdout_path: str, stderr_path: str, file_to_mountpoint: Dict[str, str], current_directory_override: Optional[str] = None):
+    def __init__(
+        self,
+        file_store: AbstractFileStore,
+        task_path: str,
+        stdout_path: str,
+        stderr_path: str,
+        file_to_mountpoint: Dict[str, str],
+        current_directory_override: Optional[str] = None,
+        share_files_with: Optional[ToilWDLStdLibBase] = None
+    ):
         """
         Set up the standard library for a task output section. Needs to know
         where standard output and error from the task have been stored, and
         what local paths to pretend are where for resolving symlinks.
 
-        If current_directory_override is set, resolves relative paths and globs
-        from there instead of from the real current directory.
+        :param current_directory_override: If set, resolves relative paths and
+            globs from there instead of from the real current directory.
+        :param share_files_with: If set to an existing standard library
+            instance, use the same file upload and download paths as it.
         """
 
         # Just set up as ToilWDLStdLibBase, but it will call into
         # WDL.StdLib.TaskOutputs next.
-        super().__init__(file_store, current_directory_override)
+        super().__init__(
+            file_store,
+            task_path,
+            execution_dir=current_directory_override,
+            share_files_with=share_files_with
+        )
 
         # Remember task output files
         self._stdout_path = stdout_path
@@ -1362,11 +1457,14 @@ def add_paths(task_container: TaskContainer, host_paths: Iterable[str]) -> None:
             task_container.input_path_map[host_path] = container_path
             task_container.input_path_map_rev[container_path] = host_path
 
-def import_files(environment: WDLBindings, toil: Toil, path: Optional[List[str]] = None, skip_remote: bool = False) -> WDLBindings:
+def import_files(environment: WDLBindings, task_path: str, toil: Toil, path: Optional[List[str]] = None, skip_remote: bool = False) -> WDLBindings:
     """
     Make sure all File values embedded in the given bindings are imported,
     using the given Toil object.
 
+    :param task_path: Dotted WDL name of the user-level code doing the
+           importing (probably the workflow name).
+
     :param path: If set, try resolving input location relative to the URLs or
            directories in this list.
 
@@ -1445,7 +1543,7 @@ def import_files(environment: WDLBindings, toil: Toil, path: Optional[List[str]]
             # Pack a UUID of the parent directory
             dir_id = path_to_id.setdefault(parent_dir, uuid.uuid4())
 
-            return pack_toil_uri(imported, dir_id, file_basename)
+            return pack_toil_uri(imported, task_path, dir_id, file_basename)
 
         # If we get here we tried all the candidates
         raise RuntimeError(f"Could not find {uri} at any of: {tried}")
@@ -1772,7 +1870,7 @@ class WDLTaskWrapperJob(WDLBaseJob):
         bindings = combine_bindings(unwrap_all(self._prev_node_results))
         # Set up the WDL standard library
         # UUID to use for virtualizing files
-        standard_library = ToilWDLStdLibBase(file_store)
+        standard_library = ToilWDLStdLibBase(file_store, self._task_path)
         with monkeypatch_coerce(standard_library):
             if self._task.inputs:
                 logger.debug("Evaluating task code")
@@ -2111,7 +2209,7 @@ class WDLTaskJob(WDLBaseJob):
         # Set up the WDL standard library
         # UUID to use for virtualizing files
         # We process nonexistent files in WDLTaskWrapperJob as those must be run locally, so don't try to devirtualize them
-        standard_library = ToilWDLStdLibBase(file_store, enforce_existence=False)
+        standard_library = ToilWDLStdLibBase(file_store, self._task_path, enforce_existence=False)
 
         # Get the bindings from after the input section
         bindings = unwrap(self._task_internal_bindings)
@@ -2295,7 +2393,7 @@ class WDLTaskJob(WDLBaseJob):
             contained_bindings = map_over_files_in_bindings(bindings, get_path_in_container)
 
             # Make a new standard library for evaluating the command specifically, which only deals with in-container paths and out-of-container paths.
-            command_library = ToilWDLStdLibTaskCommand(file_store, task_container, workdir_in_container)
+            command_library = ToilWDLStdLibTaskCommand(file_store, self._task_path, task_container, workdir_in_container)
 
             # Work out the command string, and unwrap it
             command_string: str = evaluate_named_expression(self._task, "command", WDL.Type.String(), remove_common_leading_whitespace(self._task.command), contained_bindings, command_library).coerce(WDL.Type.String()).value
@@ -2378,9 +2476,15 @@ class WDLTaskJob(WDLBaseJob):
         # container-determined strings that are absolute paths to WDL File
         # objects, and like MiniWDL we can say we only support
         # working-directory-based relative paths for globs.
-        outputs_library = ToilWDLStdLibTaskOutputs(file_store, host_stdout_txt, host_stderr_txt, task_container.input_path_map, current_directory_override=workdir_in_container)
-        # Make sure files downloaded as inputs get re-used if we re-upload them.
-        outputs_library.share_files(standard_library)
+        outputs_library = ToilWDLStdLibTaskOutputs(
+            file_store,
+            self._task_path,
+            host_stdout_txt,
+            host_stderr_txt,
+            task_container.input_path_map,
+            current_directory_override=workdir_in_container,
+            share_files_with=standard_library
+        )
         with monkeypatch_coerce(outputs_library):
             output_bindings = evaluate_output_decls(self._task.outputs, bindings, outputs_library)
 
@@ -2452,7 +2556,7 @@ class WDLWorkflowNodeJob(WDLBaseJob):
         # Combine the bindings we get from previous jobs
         incoming_bindings = combine_bindings(unwrap_all(self._prev_node_results))
         # Set up the WDL standard library
-        standard_library = ToilWDLStdLibBase(file_store, execution_dir=self._wdl_options.get("execution_dir"))
+        standard_library = ToilWDLStdLibBase(file_store, self._task_path, execution_dir=self._wdl_options.get("execution_dir"))
         with monkeypatch_coerce(standard_library):
             if isinstance(self._node, WDL.Tree.Decl):
                 # This is a variable assignment
@@ -2518,7 +2622,7 @@ class WDLWorkflowNodeListJob(WDLBaseJob):
     workflows or tasks or sections.
     """
 
-    def __init__(self, nodes: List[WDL.Tree.WorkflowNode], prev_node_results: Sequence[Promised[WDLBindings]], namespace: str, wdl_options: Optional[Dict[str, str]] = None, **kwargs: Any) -> None:
+    def __init__(self, nodes: List[WDL.Tree.WorkflowNode], prev_node_results: Sequence[Promised[WDLBindings]], namespace: str, task_path: str, wdl_options: Optional[Dict[str, str]] = None, **kwargs: Any) -> None:
         """
         Make a new job to run a list of workflow nodes to completion.
         """
@@ -2527,6 +2631,7 @@ class WDLWorkflowNodeListJob(WDLBaseJob):
         self._nodes = nodes
         self._prev_node_results = prev_node_results
         self._namespace = namespace
+        self._task_path = task_path
 
         for n in self._nodes:
             if isinstance(n, (WDL.Tree.Call, WDL.Tree.Scatter, WDL.Tree.Conditional)):
@@ -2542,7 +2647,7 @@ class WDLWorkflowNodeListJob(WDLBaseJob):
         # Combine the bindings we get from previous jobs
         current_bindings = combine_bindings(unwrap_all(self._prev_node_results))
         # Set up the WDL standard library
-        standard_library = ToilWDLStdLibBase(file_store, execution_dir=self._wdl_options.get("execution_dir"))
+        standard_library = ToilWDLStdLibBase(file_store, self._task_path, execution_dir=self._wdl_options.get("execution_dir"))
 
         with monkeypatch_coerce(standard_library):
             for node in self._nodes:
@@ -2719,7 +2824,7 @@ class WDLWorkflowGraph:
 
 class WDLSectionJob(WDLBaseJob):
     """
-    Job that can create more graph for a section of the wrokflow.
+    Job that can create more graph for a section of the workflow.
     """
 
     def __init__(self, namespace: str, task_path: str, wdl_options: Optional[Dict[str, str]] = None, **kwargs: Any) -> None:
@@ -2885,7 +2990,7 @@ class WDLSectionJob(WDLBaseJob):
                 job: WDLBaseJob = WDLWorkflowNodeJob(section_graph.get(node_ids[0]), rvs, self._namespace, task_path, wdl_options=self._wdl_options)
             else:
                 # Make a multi-node job
-                job = WDLWorkflowNodeListJob([section_graph.get(node_id) for node_id in node_ids], rvs, self._namespace, wdl_options=self._wdl_options)
+                job = WDLWorkflowNodeListJob([section_graph.get(node_id) for node_id in node_ids], rvs, self._namespace, task_path, wdl_options=self._wdl_options)
             for prev_job in prev_jobs:
                 # Connect up the happens-after relationships to make sure the
                 # return values are available.
@@ -3013,7 +3118,7 @@ class WDLScatterJob(WDLSectionJob):
         # For a task we only see the inside-the-task namespace.
         bindings = combine_bindings(unwrap_all(self._prev_node_results))
         # Set up the WDL standard library
-        standard_library = ToilWDLStdLibBase(file_store)
+        standard_library = ToilWDLStdLibBase(file_store, self._task_path)
 
         # Get what to scatter over
         with monkeypatch_coerce(standard_library):
@@ -3152,7 +3257,7 @@ class WDLConditionalJob(WDLSectionJob):
         # For a task we only see the insode-the-task namespace.
         bindings = combine_bindings(unwrap_all(self._prev_node_results))
         # Set up the WDL standard library
-        standard_library = ToilWDLStdLibBase(file_store)
+        standard_library = ToilWDLStdLibBase(file_store, self._task_path)
 
         # Get the expression value. Fake a name.
         with monkeypatch_coerce(standard_library):
@@ -3219,7 +3324,7 @@ class WDLWorkflowJob(WDLSectionJob):
         # For a task we only see the insode-the-task namespace.
         bindings = combine_bindings(unwrap_all(self._prev_node_results))
         # Set up the WDL standard library
-        standard_library = ToilWDLStdLibBase(file_store, execution_dir=self._wdl_options.get("execution_dir"))
+        standard_library = ToilWDLStdLibBase(file_store, self._task_path, execution_dir=self._wdl_options.get("execution_dir"))
 
         if self._workflow.inputs:
             with monkeypatch_coerce(standard_library):
@@ -3237,7 +3342,7 @@ class WDLWorkflowJob(WDLSectionJob):
         if self._workflow.outputs != []:  # Compare against empty list as None means there should be outputs
             # Either the output section is declared and nonempty or it is not declared
             # Add evaluating the outputs after the sink
-            outputs_job = WDLOutputsJob(self._workflow, sink.rv(), wdl_options=self._wdl_options)
+            outputs_job = WDLOutputsJob(self._workflow, sink.rv(), self._task_path, wdl_options=self._wdl_options)
             sink.addFollowOn(outputs_job)
             # Caller is responsible for making sure namespaces are applied
             self.defer_postprocessing(outputs_job)
@@ -3252,7 +3357,7 @@ class WDLOutputsJob(WDLBaseJob):
 
     Returns an environment with just the outputs bound, in no namespace.
     """
-    def __init__(self, workflow: WDL.Tree.Workflow, bindings: Promised[WDLBindings], wdl_options: Optional[Dict[str, str]] = None, **kwargs: Any):
+    def __init__(self, workflow: WDL.Tree.Workflow, bindings: Promised[WDLBindings], task_path: str, wdl_options: Optional[Dict[str, str]] = None, **kwargs: Any):
         """
         Make a new WDLWorkflowOutputsJob for the given workflow, with the given set of bindings after its body runs.
         """
@@ -3260,6 +3365,7 @@ class WDLOutputsJob(WDLBaseJob):
 
         self._bindings = bindings
         self._workflow = workflow
+        self._task_path = task_path
 
     @report_wdl_errors("evaluate outputs")
     def run(self, file_store: AbstractFileStore) -> WDLBindings:
@@ -3269,7 +3375,7 @@ class WDLOutputsJob(WDLBaseJob):
         super().run(file_store)
 
         # Evaluate all output expressions in the normal, non-task-outputs library context
-        standard_library = ToilWDLStdLibBase(file_store, execution_dir=self._wdl_options.get("execution_dir"))
+        standard_library = ToilWDLStdLibBase(file_store, self._task_path, execution_dir=self._wdl_options.get("execution_dir"))
 
         try:
             if self._workflow.outputs is None:
@@ -3474,7 +3580,7 @@ def main() -> None:
                         inputs_search_path.append(match.group(0))
 
                 # Import any files in the bindings
-                input_bindings = import_files(input_bindings, toil, inputs_search_path, skip_remote=options.reference_inputs)
+                input_bindings = import_files(input_bindings, target.name, toil, inputs_search_path, skip_remote=options.reference_inputs)
 
                 # TODO: Automatically set a good MINIWDL__SINGULARITY__IMAGE_CACHE ?
 
@@ -3492,7 +3598,8 @@ def main() -> None:
                 output_bindings = toil.start(root_job)
             if not isinstance(output_bindings, WDL.Env.Bindings):
                 raise RuntimeError("The output of the WDL job is not a binding.")
-
+            
+            devirtualization_state: DirectoryNamingStateDict = {}
             devirtualized_to_virtualized: Dict[str, str] = dict()
             virtualized_to_devirtualized: Dict[str, str] = dict()
 
@@ -3505,7 +3612,15 @@ def main() -> None:
                 # Make sure the output directory exists if we have output files
                 # that might need to use it.
                 os.makedirs(output_directory, exist_ok=True)
-                return ToilWDLStdLibBase.devirtualize_to(filename, output_directory, toil, execution_dir, devirtualized_to_virtualized, virtualized_to_devirtualized)
+                return ToilWDLStdLibBase.devirtualize_to(
+                    filename,
+                    output_directory,
+                    toil,
+                    execution_dir,
+                    devirtualization_state,
+                    devirtualized_to_virtualized,
+                    virtualized_to_devirtualized
+                )
 
             # Make all the files local files
             output_bindings = map_over_files_in_bindings(output_bindings, devirtualize_output)


Test patch:
diff --git a/src/toil/test/wdl/wdltoil_test.py b/src/toil/test/wdl/wdltoil_test.py
index f92b405d..b0e749d3 100644
--- a/src/toil/test/wdl/wdltoil_test.py
+++ b/src/toil/test/wdl/wdltoil_test.py
@@ -2,6 +2,7 @@ import json
 import os
 import re
 import shutil
+import string
 import subprocess
 import unittest
 from uuid import uuid4
@@ -13,6 +14,7 @@ from typing import Any, Dict, List, Set
 import logging
 import pytest
 
+from toil.fileStores import FileID
 from toil.provisioners import cluster_factory
 from toil.test import (ToilTest,
                        needs_docker_cuda,
@@ -184,8 +186,22 @@ class WDLTests(BaseWDLTest):
         assert len(outputs['hello_caller.message_files']) == 2
         for item in outputs['hello_caller.message_files']:
             # All the files should be strings in the "out" directory
-            assert isinstance(item, str)
-            assert item.startswith(out_dir)
+            assert isinstance(item, str), "File output must be a string"
+            assert item.startswith(out_dir), "File output must be in the output directory"
+
+            # Look at the filename within that directory
+            name_in_out_dir = item[len(out_dir):]
+
+            # Ity should contain the job name of "hello", so they are human-readable.
+            assert "hello" in name_in_out_dir, f"File output {name_in_out_dir} should have the originating task name in it"
+
+            # And it should not contain non-human-readable content.
+            #
+            # We use a threshold number of digits as a proxy for this, but
+            # don't try and get around this by just rolling other random
+            # strings; we want these outputs to be human-readable!!!
+            digit_count = len([c for c in name_in_out_dir if c in string.digits])
+            assert digit_count < 3, f"File output {name_in_out_dir} has {digit_count} digits, which is too many to be plausibly human-readable"
 
         assert 'hello_caller.messages' in outputs
         assert outputs['hello_caller.messages'] == ["Hello, Alyssa P. Hacker!", "Hello, Ben Bitdiddle!"]
@@ -521,6 +537,60 @@ class WDLToilBenchTests(ToilTest):
         trimmed = remove_common_leading_whitespace(expr)
         assert trimmed.command == True
 
+    def test_choose_human_readable_directory(self):
+        """
+        Test to make sure that we pick sensible but non-colliding directories to put files in.
+        """
+
+        from toil.wdl.wdltoil import choose_human_readable_directory, DirectoryNamingStateDict
+
+        state: DirectoryNamingStateDict = {}
+
+        # The first time we should get  apath with the task name and without the ID
+        first_chosen = choose_human_readable_directory("root", "taskname", "111-222-333", state)
+        assert first_chosen.startswith("root")
+        assert "taskname" in first_chosen
+        assert "111-222-333" not in first_chosen
+
+        # If we use the same ID we should get the same result
+        same_id = choose_human_readable_directory("root", "taskname", "111-222-333", state)
+        self.assertEqual(same_id, first_chosen)
+
+        # If we use a different ID we shoudl get a different result still obeying the constraints
+        diff_id = choose_human_readable_directory("root", "taskname", "222-333-444", state)
+        self.assertNotEqual(diff_id, first_chosen)
+        assert diff_id.startswith("root")
+        assert "taskname" in diff_id
+        assert "222-333-444" not in diff_id
+
+    def test_uri_packing(self):
+        """
+        Test to make sure Toil URI packing brings through the required information.
+        """
+
+        from toil.wdl.wdltoil import pack_toil_uri, unpack_toil_uri
+
+        # Set up a file
+        file_id = FileID("fileXYZ", 123, True)
+        task_path = "the_wf.the_task"
+        dir_id = uuid4()
+        file_basename = "thefile.txt"
+
+        # Pack and unpack it
+        uri = pack_toil_uri(file_id, task_path, dir_id, file_basename)
+        unpacked = unpack_toil_uri(uri)
+
+        # Make sure we got what we put in
+        self.assertEqual(unpacked[0], file_id)
+        self.assertEqual(unpacked[0].size, file_id.size)
+        self.assertEqual(unpacked[0].executable, file_id.executable)
+
+        self.assertEqual(unpacked[1], task_path)
+
+        # TODO: We don't make the UUIDs back into UUID objects
+        self.assertEqual(unpacked[2], str(dir_id))
+
+        self.assertEqual(unpacked[3], file_basename)