Repository: DataBiosphere/toil

Issue:
CWL conformance test 236 fails


┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-682)
┆Epic: CWL v1.2 support
┆Issue Number: TOIL-682



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@ def runSetup():
     gcs = 'google-cloud-storage==1.6.0'
     gcs_oauth2_boto_plugin = 'gcs_oauth2_boto_plugin==1.14'
     apacheLibcloud = 'apache-libcloud==2.2.1'
-    cwltool = 'cwltool==3.0.20201116114821'
+    cwltool = 'cwltool==3.0.20201121085451'
     galaxyToolUtil = 'galaxy-tool-util'
     htcondor = 'htcondor>=8.6.0'
     kubernetes = 'kubernetes>=10, <11'
diff --git a/src/toil/cwl/cwltoil.py b/src/toil/cwl/cwltoil.py
--- a/src/toil/cwl/cwltoil.py
+++ b/src/toil/cwl/cwltoil.py
@@ -29,8 +29,22 @@
 import tempfile
 import urllib
 import uuid
-from typing import (Any, Dict, Iterator, List, Mapping, MutableMapping,
-                    MutableSequence, Text, TextIO, Tuple, TypeVar, Union, cast)
+import shutil
+from typing import (
+    Any,
+    Dict,
+    Iterator,
+    List,
+    Mapping,
+    MutableMapping,
+    MutableSequence,
+    Text,
+    TextIO,
+    Tuple,
+    TypeVar,
+    Union,
+    cast,
+)
 from urllib import parse as urlparse
 
 import cwltool.builder
@@ -48,14 +62,29 @@
 from cwltool.loghandler import defaultStreamHandler
 from cwltool.mutation import MutationManager
 from cwltool.pathmapper import MapperEnt, PathMapper, downloadHttpFile
-from cwltool.process import (Process, add_sizes, compute_checksums,
-                             fill_in_defaults, shortname)
+from cwltool.process import (
+    Process,
+    add_sizes,
+    compute_checksums,
+    fill_in_defaults,
+    shortname,
+)
 from cwltool.secrets import SecretStore
 from cwltool.software_requirements import (
-    DependenciesConfiguration, get_container_from_software_requirements)
-from cwltool.utils import (CWLObjectType, adjustDirObjs, adjustFileObjs,
-                           aslist, convert_pathsep_to_unix, get_listing,
-                           normalizeFilesDirs, visit_class)
+    DependenciesConfiguration,
+    get_container_from_software_requirements,
+)
+from cwltool.utils import (
+    CWLOutputAtomType,
+    CWLObjectType,
+    adjustDirObjs,
+    adjustFileObjs,
+    aslist,
+    convert_pathsep_to_unix,
+    get_listing,
+    normalizeFilesDirs,
+    visit_class,
+)
 from ruamel.yaml.comments import CommentedMap
 from schema_salad import validate
 from schema_salad.schema import Names
@@ -65,8 +94,7 @@
 from toil.fileStores import FileID
 from toil.fileStores.abstractFileStore import AbstractFileStore
 from toil.job import Job
-from toil.jobStores.abstractJobStore import (NoSuchFileException,
-                                             NoSuchJobStoreException)
+from toil.jobStores.abstractJobStore import NoSuchFileException, NoSuchJobStoreException
 from toil.version import baseVersion
 
 logger = logging.getLogger(__name__)
@@ -481,6 +509,17 @@ def resolve_dict_w_promises(
             result[k] = v.do_eval(inputs=first_pass_results)
         else:
             result[k] = first_pass_results[k]
+
+    # '_:' prefixed file paths are a signal to cwltool to create folders in place
+    # rather than copying them, so we make them here
+    for entry in result:
+        if isinstance(result[entry], dict):
+            location = result[entry].get("location")
+            if location:
+                if location.startswith("_:file://"):
+                    local_dir_path = location[len("_:file://") :]
+                    os.makedirs(local_dir_path, exist_ok=True)
+                    result[entry]["location"] = local_dir_path
     return result
 
 
@@ -658,6 +697,31 @@ def exists(self, path: str) -> bool:
         except NoSuchFileException:
             return False
 
+    def realpath(self, path: str) -> str:
+        if path.startswith("toilfs:"):
+            # import the file and make it available locally if it exists
+            path = self._abs(path)
+        elif path.startswith("_:"):
+            return path
+        return os.path.realpath(path)
+
+    def listdir(self, fn: str) -> List[str]:
+        directory = self._abs(fn)
+        if fn.startswith("_:file://"):
+            directory = fn[len("_:file://") :]
+            if os.path.isdir(directory):
+                return [
+                    cwltool.stdfsaccess.abspath(urllib.parse.quote(entry), fn)
+                    for entry in os.listdir(directory)
+                ]
+            else:
+                return []
+        else:
+            return [
+                cwltool.stdfsaccess.abspath(urllib.parse.quote(entry), fn)
+                for entry in os.listdir(self._abs(directory))
+            ]
+
     def _abs(self, path: str) -> str:
         """
         Return a local absolute path for a file (no schema).
@@ -672,11 +736,15 @@ def _abs(self, path: str) -> str:
             logger.debug("Need to download file to get a local absolute path.")
             destination = self.file_store.readGlobalFile(FileID.unpack(path[7:]))
             logger.debug("Downloaded %s to %s", path, destination)
-            assert os.path.exists(destination)
-            return destination
+            if not os.path.exists(destination):
+                raise RuntimeError(
+                    f"{destination} does not exist after filestore import."
+                )
+        elif path.startswith("_:file://"):
+            destination = path
         else:
-            result = super(ToilFsAccess, self)._abs(path)
-            return result
+            destination = super(ToilFsAccess, self)._abs(path)
+        return destination
 
 
 def toil_get_file(
@@ -814,6 +882,18 @@ def writeGlobalFileWrapper(file_store: AbstractFileStore, fileuri: str) -> str:
     return file_store.writeGlobalFile(schema_salad.ref_resolver.uri_file_path(fileuri))
 
 
+def remove_empty_listings(rec: CWLObjectType) -> None:
+    if rec.get("class") != "Directory":
+        finddirs = []  # type: List[CWLObjectType]
+        visit_class(rec, ("Directory",), finddirs.append)
+        for f in finddirs:
+            remove_empty_listings(f)
+        return
+    if "listing" in rec and rec["listing"] == []:
+        del rec["listing"]
+        return
+
+
 class ResolveIndirect(Job):
     """
     Helper Job.
@@ -1092,6 +1172,11 @@ def run(self, file_store: AbstractFileStore) -> Any:
             if not found:
                 cwljob.pop(inp_id)
 
+        adjustDirObjs(
+            cwljob,
+            functools.partial(remove_empty_listings),
+        )
+
         # Exports temporary directory for batch systems that reset TMPDIR
         os.environ["TMPDIR"] = os.path.realpath(file_store.getLocalTempDir())
         outdir = os.path.join(file_store.getLocalTempDir(), "out")
@@ -1695,7 +1780,13 @@ def determine_load_listing(tool: ToilCommandLineTool):
         if load_listing_req
         else "no_listing"
     )
-    load_listing = tool.tool.get("loadListing") or load_listing_tool_req
+    load_listing = tool.tool.get("loadListing", None) or load_listing_tool_req
+
+    listing_choices = ("no_listing", "shallow_listing", "deep_listing")
+    if load_listing not in listing_choices:
+        raise ValueError(
+            f'Unknown loadListing specified: "{load_listing}".  Valid choices: {listing_choices}'
+        )
     return load_listing
 
 
@@ -2104,6 +2195,7 @@ def set_secondary(fileobj):
             runtime_context.no_match_user = options.no_match_user
             runtime_context.no_read_only = options.no_read_only
             runtime_context.basedir = options.basedir
+            runtime_context.move_outputs = "move"
 
             # We instantiate an early builder object here to populate indirect
             # secondaryFile references using cwltool's library because we need


Test patch:
diff --git a/src/toil/test/__init__.py b/src/toil/test/__init__.py
--- a/src/toil/test/__init__.py
+++ b/src/toil/test/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2018 Regents of the University of California
+# Copyright (C) 2015-2020 Regents of the University of California
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -412,6 +412,7 @@ def needs_docker(test_item):
     else:
         return unittest.skip("Install docker to include this test.")(test_item)
 
+
 def needs_encryption(test_item):
     """
     Use as a decorator before test classes or methods to only run them if PyNaCl is installed
@@ -466,6 +467,7 @@ def needs_appliance(test_item):
     return unittest.skip(f"Cannot find appliance {image}. Use 'make test' target to automatically build appliance, or "
                          f"just run 'make push_docker' prior to running this test.")(test_item)
 
+
 def needs_fetchable_appliance(test_item):
     """
     Use as a decorator before test classes or methods to only run them if
@@ -483,8 +485,7 @@ def needs_fetchable_appliance(test_item):
                              f"just run 'make push_docker' prior to running this test.")(test_item)
     else:
         return test_item
-        
-    
+
 
 def integrative(test_item):
     """
diff --git a/src/toil/test/cwl/cwlTest.py b/src/toil/test/cwl/cwlTest.py
--- a/src/toil/test/cwl/cwlTest.py
+++ b/src/toil/test/cwl/cwlTest.py
@@ -32,7 +32,7 @@
 
 from toil.test import (ToilTest, needs_cwl, slow, needs_docker, needs_lsf,
                        needs_mesos, needs_parasol, needs_gridengine, needs_slurm,
-                       needs_torque)
+                       needs_torque, needs_aws_s3)
 
 
 log = logging.getLogger(__name__)
@@ -136,6 +136,7 @@ def download(self, inputs, tester_fn):
                   input_location,
                   self._expected_download_output(self.outDir))
 
+    @needs_aws_s3
     def test_s3_as_secondary_file(self):
         from toil.cwl import cwltoil
         st = StringIO()
@@ -159,6 +160,7 @@ def test_run_revsort2(self):
     def test_run_revsort_debug_worker(self):
         self.revsort('revsort.cwl', self._debug_worker_tester)
 
+    @needs_aws_s3
     def test_run_s3(self):
         self.download('download_s3.json', self._tester)
 
@@ -380,9 +382,7 @@ def test_run_conformance(self, batchSystem=None, caching=False):
         run_conformance_tests(workDir=self.cwlSpec,
                               yml=self.test_yaml,
                               caching=caching,
-                              batchSystem=batchSystem,
-                              # TODO: we do not currently pass test: 236
-                              selected_tests='1-235,237-253')
+                              batchSystem=batchSystem)
 
 @needs_cwl
 class CWLv12Test(ToilTest):
@@ -414,9 +414,9 @@ def test_run_conformance_with_caching(self):
     @slow
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
     def test_run_conformance(self, batchSystem=None, caching=False):
-        # TODO: we do not currently pass tests: 237 (offset from other versions), 307, 309, 310, 311, 330, 331, 332
+        # TODO: we do not currently pass tests: 307, 309, 310, 311, 330, 331, 332
         run_conformance_tests(workDir=self.cwlSpec,
                               yml=self.test_yaml,
                               caching=caching,
                               batchSystem=batchSystem,
-                              selected_tests='1-236,238-306,308,312-329,333-336')
+                              selected_tests='1-306,308,312-329,333-336')