Repository: DataBiosphere/toil

Issue:
CWL conformance test 213 fails


┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-681)
┆Epic: CWL v1.2 support
┆Issue Number: TOIL-681



Discussion:

Resolving patch:
diff --git a/src/toil/cwl/cwltoil.py b/src/toil/cwl/cwltoil.py
--- a/src/toil/cwl/cwltoil.py
+++ b/src/toil/cwl/cwltoil.py
@@ -40,7 +40,8 @@
                     Iterator,
                     TextIO,
                     Set,
-                    Tuple)
+                    Tuple,
+                    cast)
 
 # Python 3 compatibility imports
 from six import iteritems, string_types
@@ -77,6 +78,7 @@
     DependenciesConfiguration, get_container_from_software_requirements)
 from cwltool.utils import aslist, convert_pathsep_to_unix, get_listing, normalizeFilesDirs, CWLObjectType
 from cwltool.mutation import MutationManager
+from cwltool.builder import content_limit_respected_read
 
 from toil.job import Job, Promise
 from toil.common import Config, Toil, addOptions
@@ -362,6 +364,23 @@ def __init__(self, expr: str, source: Any, req: List[Dict[str, Any]]):
         self.context = None
         self.req = req
 
+    def eval_prep(self, step_inputs: dict, file_store: AbstractFileStore):
+        """
+        If loadContents is specified, this will resolve the contents of any file in
+        a set of inputs associated with the StepValueFrom object's self.source.
+
+        :param step_inputs: Workflow step inputs.
+        :param file_store: A toil file store, needed to resolve toil fs:// paths.
+        """
+        for k, v in step_inputs.items():
+            val = cast(CWLObjectType, v)
+            source_input = getattr(self.source, 'input', {})
+            if isinstance(val, dict) and isinstance(source_input, dict):
+                if val.get("contents") is None and source_input.get('loadContents') is True:
+                    fs_access = functools.partial(ToilFsAccess, file_store=file_store)
+                    with fs_access('').open(cast(str, val["location"]), "rb") as f:
+                        val["contents"] = content_limit_respected_read(f)
+
     def resolve(self) -> Any:
         """
         The valueFrom expression's context is the value for this input. Resolve the promise.
@@ -421,7 +440,7 @@ def resolve(self) -> Any:
         return self.val
 
 
-def resolve_dict_w_promises(dict_w_promises: dict) -> dict:
+def resolve_dict_w_promises(dict_w_promises: dict, file_store: AbstractFileStore = None) -> dict:
     """
     Resolve the contents of an unresolved dictionary (containing promises) and
     evaluate expressions to produce the dictionary of actual values.
@@ -437,6 +456,8 @@ def resolve_dict_w_promises(dict_w_promises: dict) -> dict:
     result = {}
     for k, v in dict_w_promises.items():
         if isinstance(v, StepValueFrom):
+            if file_store:
+                v.eval_prep(first_pass_results, file_store)
             result[k] = v.do_eval(inputs=first_pass_results)
         else:
             result[k] = first_pass_results[k]
@@ -713,6 +734,7 @@ def uploadFile(uploadfunc: Any,
 
 def writeGlobalFileWrapper(file_store: AbstractFileStore, fileuri: str) -> str:
     """Wrap writeGlobalFile to accept file:// URIs"""
+    fileuri = fileuri if ':/' in fileuri else f'file://{fileuri}'
     return file_store.writeGlobalFile(
         schema_salad.ref_resolver.uri_file_path(fileuri))
 
@@ -804,7 +826,7 @@ def __init__(self,
         self.conditional = conditional
 
     def run(self, file_store: AbstractFileStore) -> Any:
-        cwljob = resolve_dict_w_promises(self.cwljob)
+        cwljob = resolve_dict_w_promises(self.cwljob, file_store)
         fill_in_defaults(
             self.cwltool.tool['inputs'],
             cwljob,
@@ -933,7 +955,7 @@ def run(self, file_store: AbstractFileStore) -> Any:
         cwllogger.removeHandler(defaultStreamHandler)
         cwllogger.setLevel(logger.getEffectiveLevel())
 
-        cwljob = resolve_dict_w_promises(self.cwljob)
+        cwljob = resolve_dict_w_promises(self.cwljob, file_store)
 
         if self.conditional.is_false(cwljob):
             return self.conditional.skipped_outputs()
@@ -975,8 +997,7 @@ def run(self, file_store: AbstractFileStore) -> Any:
         runtime_context.outdir = outdir
         runtime_context.tmp_outdir_prefix = tmp_outdir_prefix
         runtime_context.tmpdir_prefix = file_store.getLocalTempDir()
-        runtime_context.make_fs_access = functools.partial(
-            ToilFsAccess, file_store=file_store)
+        runtime_context.make_fs_access = functools.partial(ToilFsAccess, file_store=file_store)
         runtime_context.preserve_environment = required_env_vars
 
         runtime_context.toil_get_file = functools.partial(
@@ -1108,7 +1129,7 @@ def nested_crossproduct_scatter(self,
         return outputs
 
     def run(self, file_store: AbstractFileStore) -> list:
-        cwljob = resolve_dict_w_promises(self.cwljob)
+        cwljob = resolve_dict_w_promises(self.cwljob, file_store)
 
         if isinstance(self.step.tool["scatter"], string_types):
             scatter = [self.step.tool["scatter"]]
@@ -1256,7 +1277,7 @@ def __init__(self,
         self.conditional = conditional or Conditional()
 
     def run(self, file_store: AbstractFileStore):
-        cwljob = resolve_dict_w_promises(self.cwljob)
+        cwljob = resolve_dict_w_promises(self.cwljob, file_store)
 
         if self.conditional.is_false(cwljob):
             return self.conditional.skipped_outputs()


Test patch:
diff --git a/src/toil/test/cwl/cwlTest.py b/src/toil/test/cwl/cwlTest.py
--- a/src/toil/test/cwl/cwlTest.py
+++ b/src/toil/test/cwl/cwlTest.py
@@ -349,8 +349,8 @@ def test_run_conformance_with_caching(self):
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
     def test_run_conformance(self, batchSystem=None, caching=False):
         try:
-            # TODO: we do not currently pass tests: 213, 236
-            selected_tests = '1-212,214-235,237-253'
+            # TODO: we do not currently pass test: 236
+            selected_tests = '1-235,237-253'
             cmd = [f'cwltest',
                    f'--tool=toil-cwl-runner',
                    f'--test={self.test_yaml}',
@@ -411,8 +411,8 @@ def test_run_conformance_with_caching(self):
     @pytest.mark.timeout(CONFORMANCE_TEST_TIMEOUT)
     def test_run_conformance(self, batchSystem=None, caching=False):
         try:
-            # TODO: we do not currently pass tests: 214, 237 (offset from other versions), 307, 309, 310, 311, 330, 331, 332
-            selected_tests = '1-213,215-236,238-306,308,312-329,333-336'
+            # TODO: we do not currently pass tests: 237 (offset from other versions), 307, 309, 310, 311, 330, 331, 332
+            selected_tests = '1-236,238-306,308,312-329,333-336'
             cmd = [f'cwltest',
                    f'--tool=toil-cwl-runner',
                    f'--test={self.test_yaml}',