Repository: DataBiosphere/toil

Issue:
WDL `File` inputs with empty paths can be imported but can't be devirtualized
Someone tried to run a WDL workflow with Toil and (I think) gave an empty string in the input JSON for a `File` type input.

Toil logged that it manage to import `file://<path to the direcotry the inputs file is in>/.`. The resulting WDL `File` can't be devirtualized, because we stored an empty `basename` for it in the encoded Toil URL, and when we join that onto the temp directory we want to put it in, we get the temp directory's path and not a path to a file we can write stuff in.

We should make sure that Toil URLs in the WDL interpreter can't exist with empty basenames.

And we should make sure that empty strings won't be imported as the whole `.` directory. They probably should fail to import as `File`s.

┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-1335)
┆Epic: Improved WDL support and more complex conformance tests Q5
┆Issue Number: TOIL-1335



Discussion:

Resolving patch:
diff --git a/src/toil/jobStores/abstractJobStore.py b/src/toil/jobStores/abstractJobStore.py
--- a/src/toil/jobStores/abstractJobStore.py
+++ b/src/toil/jobStores/abstractJobStore.py
@@ -421,7 +421,8 @@ def import_file(self,
                 e.g. gs://bucket/file
 
         :param str src_uri: URL that points to a file or object in the storage mechanism of a
-                supported URL scheme e.g. a blob in an AWS s3 bucket.
+                supported URL scheme e.g. a blob in an AWS s3 bucket. It must be a file, not a
+                directory or prefix.
 
         :param str shared_file_name: Optional name to assign to the imported file within the job store
 
diff --git a/src/toil/jobStores/fileJobStore.py b/src/toil/jobStores/fileJobStore.py
--- a/src/toil/jobStores/fileJobStore.py
+++ b/src/toil/jobStores/fileJobStore.py
@@ -314,6 +314,9 @@ def _copy_or_link(self, src_path, dst_path, symlink=False):
 
     def _import_file(self, otherCls, uri, shared_file_name=None, hardlink=False, symlink=False):
         if issubclass(otherCls, FileJobStore):
+            if os.path.isdir(uri.path):
+                # Don't allow directories (unless someone is racing us)
+                raise IsADirectoryError(f"URI {uri} points to a directory but a file was expected")
             if shared_file_name is None:
                 executable = os.stat(uri.path).st_mode & stat.S_IXUSR != 0
                 absPath = self._get_unique_file_path(uri.path)  # use this to get a valid path to write to in job store
diff --git a/src/toil/wdl/wdltoil.py b/src/toil/wdl/wdltoil.py
--- a/src/toil/wdl/wdltoil.py
+++ b/src/toil/wdl/wdltoil.py
@@ -63,6 +63,10 @@ def potential_absolute_uris(uri: str, path: List[str], importer: Optional[WDL.Tr
     document if any.
     """
 
+    if uri == "":
+        # Empty URIs can't come from anywhere.
+        return
+
     # We need to brute-force find this URI relative to:
     #
     # 1. Itself if a full URI.
@@ -723,6 +727,11 @@ def import_file_from_uri(uri: str) -> str:
             # Work out what the basename for the file was
             file_basename = os.path.basename(urlsplit(candidate_uri).path)
 
+            if file_basename == "":
+                # We can't have files with no basename because we need to
+                # download them at that basename later.
+                raise RuntimeError(f"File {candidate_uri} has no basename and so cannot be a WDL File")
+
             # Was actually found
             return pack_toil_uri(imported, file_basename)
 


Test patch:
diff --git a/src/toil/test/wdl/md5sum/empty_file.json b/src/toil/test/wdl/md5sum/empty_file.json
new file mode 100644
--- /dev/null
+++ b/src/toil/test/wdl/md5sum/empty_file.json
@@ -0,0 +1 @@
+{"ga4ghMd5.inputFile": ""}
diff --git a/src/toil/test/wdl/wdltoil_test.py b/src/toil/test/wdl/wdltoil_test.py
--- a/src/toil/test/wdl/wdltoil_test.py
+++ b/src/toil/test/wdl/wdltoil_test.py
@@ -25,7 +25,7 @@ def setUpClass(cls) -> None:
         cls.base_command = [exactPython, '-m', 'toil.wdl.wdltoil']
 
     # We inherit a testMD5sum but it is going to need Singularity and not
-    # Docker now. And also needs to have a WDL 1.0+ WDL file. So we replace it. 
+    # Docker now. And also needs to have a WDL 1.0+ WDL file. So we replace it.
     @needs_singularity
     def testMD5sum(self):
         """Test if toilwdl produces the same outputs as known good outputs for WDL's
@@ -41,6 +41,18 @@ def testMD5sum(self):
         assert os.path.exists(result['ga4ghMd5.value'])
         assert os.path.basename(result['ga4ghMd5.value']) == 'md5sum.txt'
 
+    def test_empty_file_path(self):
+        """Test if empty File type inputs are protected against"""
+        wdl = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.1.0.wdl')
+        json_file = os.path.abspath('src/toil/test/wdl/md5sum/empty_file.json')
+
+        p = subprocess.Popen(self.base_command + [wdl, json_file, '-o', self.output_dir, '--logDebug'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = p.communicate()
+        retval = p.wait()
+
+        assert retval != 0
+        assert b'Could not find' in stderr
+
     @needs_singularity
     def test_miniwdl_self_test(self):
         """Test if the MiniWDL self test runs and produces the expected output."""