Repository: DataBiosphere/toil

Issue:
Bring up not fully passing conformance tests for MiniWDL-based WDL interpreter


┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-1275)
┆Issue Number: TOIL-1275



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -115,7 +115,8 @@ def run_setup():
                 '_toil_worker = toil.worker:main',
                 'cwltoil = toil.cwl.cwltoil:cwltoil_was_removed [cwl]',
                 'toil-cwl-runner = toil.cwl.cwltoil:main [cwl]',
-                'toil-wdl-runner = toil.wdl.toilwdl:main',
+                'toil-wdl-runner = toil.wdl.wdltoil:main [wdl]',
+                'toil-wdl-runner-old = toil.wdl.toilwdl:main [wdl]',
                 'toil-wes-cwl-runner = toil.server.cli.wes_cwl_runner:main [server]',
                 '_toil_mesos_executor = toil.batchSystems.mesos.executor:main [mesos]',
                 '_toil_contained_executor = toil.batchSystems.contained_executor:executor']})
diff --git a/src/toil/wdl/wdltoil.py b/src/toil/wdl/wdltoil.py
--- a/src/toil/wdl/wdltoil.py
+++ b/src/toil/wdl/wdltoil.py
@@ -25,18 +25,20 @@
 import os
 import re
 import shlex
+import shutil
 import subprocess
 import sys
 import tempfile
 import uuid
 
 from contextlib import ExitStack
-from typing import cast, Any, Callable, Union, Dict, List, Optional, Set, Sequence, Tuple, TypeVar, Iterator
+from typing import cast, Any, Callable, Union, Dict, List, Optional, Set, Sequence, Tuple, Type, TypeVar, Iterator
 from urllib.parse import urlsplit, urljoin, quote, unquote
 
 import WDL
 from WDL.runtime.task_container import TaskContainer
 from WDL.runtime.backend.singularity import SingularityContainer
+from WDL.runtime.backend.docker_swarm import SwarmContainer
 import WDL.runtime.config
 
 from toil.common import Config, Toil, addOptions
@@ -988,20 +990,33 @@ def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]:
 
         # If we get here we have all the resources we need, so run the task
 
-        # Prepare to use Singularity. Make sure that we have plenty of space to
-        # download images.
-        if 'SINGULARITY_CACHEDIR' not in os.environ:
-            # Cache Singularity's layers somehwere known to have space, not in home
-            os.environ['SINGULARITY_CACHEDIR'] = os.path.join(file_store.workflow_dir, 'singularity_cache')
+        if shutil.which('singularity'):
 
-        if 'MINIWDL__SINGULARITY__IMAGE_CACHE' not in os.environ:
-            # Cache Singularity images for the workflow on this machine.
-            # Since MiniWDL does only within-process synchronization for pulls,
-            # we also will need to pre-pull one image into here at a time.
-            os.environ['MINIWDL__SINGULARITY__IMAGE_CACHE'] = os.path.join(file_store.workflow_dir, 'miniwdl_sif_cache')
+            # Prepare to use Singularity. Make sure that we have plenty of space to
+            # download images.
+            if 'SINGULARITY_CACHEDIR' not in os.environ:
+                # Cache Singularity's layers somehwere known to have space, not in home
+                os.environ['SINGULARITY_CACHEDIR'] = os.path.join(file_store.workflow_dir, 'singularity_cache')
+
+            if 'MINIWDL__SINGULARITY__IMAGE_CACHE' not in os.environ:
+                # Cache Singularity images for the workflow on this machine.
+                # Since MiniWDL does only within-process synchronization for pulls,
+                # we also will need to pre-pull one image into here at a time.
+                os.environ['MINIWDL__SINGULARITY__IMAGE_CACHE'] = os.path.join(file_store.workflow_dir, 'miniwdl_sif_cache')
+
+            # Run containers with Singularity
+            TaskContainerImplementation: Type[TaskContainer]  = SingularityContainer
+        else:
+            # Run containers with Docker
+            TaskContainerImplementation = SwarmContainer
+            if runtime_accelerators:
+                # Complain to the user that this is unlikely to work.
+                logger.warning("Running job that needs accelerators with Docker, because "
+                               "Singularity is not available. Accelerator and GPU support "
+                               "is not yet implemented in the MiniWDL Docker "
+                               "containerization implementation.")
 
         # Set up the MiniWDL container running stuff
-        TaskContainerImplementation = SingularityContainer
         miniwdl_logger = logging.getLogger("MiniWDLContainers")
         miniwdl_config = WDL.runtime.config.Loader(miniwdl_logger)
         if not getattr(TaskContainerImplementation, 'toil_initialized__', False):
@@ -1042,53 +1057,56 @@ def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]:
             workdir_in_container = os.path.join(host_dir, "work")
             task_container = TaskContainerImplementation(miniwdl_config, run_id, host_dir)
 
-            # We might need to send GPUs and the current miniwdl deosn't do
-            # that for Singularity. And we might need to *not* try and use
-            # --fakeroot if we lack sub-UIDs. So we sneakily monkey patch it
-            # here.
-            original_run_invocation = task_container._run_invocation
-            def patched_run_invocation(*args: Any, **kwargs: Any) -> List[str]:
-                """
-                Invoke the original _run_invocation to get a base Singularity
-                command line, and then adjust the result to pass GPUs and not
-                fake root if needed.
-                """
-                command_line: List[str] = original_run_invocation(*args, **kwargs)
-
-                logger.debug('MiniWDL wants to run command line: %s', command_line)
-
-                # "exec" can be at index 1 or 2 depending on if we have a --verbose.
-                subcommand_index = 2 if command_line[1] == "--verbose" else 1
-
-                if '--fakeroot' in command_line and not self.can_fake_root():
-                    # We can't fake root so don't try.
-                    command_line.remove('--fakeroot')
-
-                extra_flags: Set[str] = set()
-                accelerators_needed: Optional[List[AcceleratorRequirement]] = self.accelerators
-                if accelerators_needed is not None:
-                    for accelerator in accelerators_needed:
-                        if accelerator['kind'] == 'gpu':
-                            if accelerator['brand'] == 'nvidia':
-                                # Tell Singularity to expose nvidia GPUs
-                                extra_flags.add('--nv')
-                            elif accelerator['api'] == 'rocm':
-                                # Tell Singularity to expose ROCm GPUs
-                                extra_flags.add('--nv')
-                            else:
-                                raise RuntimeError('Cannot expose allocated accelerator %s to Singularity job', accelerator)
-
-                for flag in extra_flags:
-                    # Put in all those flags
-                    command_line.insert(subcommand_index + 1, flag)
-
-                logger.debug('Ammended command line to: %s', command_line)
-
-                # Return the modified command line
-                return command_line
-
-            # Apply the patch
-            task_container._run_invocation = patched_run_invocation #  type: ignore
+            if isinstance(task_container, SingularityContainer):
+                # We need to patch the Singularity container run invocation
+
+                # We might need to send GPUs and the current miniwdl doesn't do
+                # that for Singularity. And we might need to *not* try and use
+                # --fakeroot if we lack sub-UIDs. So we sneakily monkey patch it
+                # here.
+                original_run_invocation = task_container._run_invocation
+                def patched_run_invocation(*args: Any, **kwargs: Any) -> List[str]:
+                    """
+                    Invoke the original _run_invocation to get a base Singularity
+                    command line, and then adjust the result to pass GPUs and not
+                    fake root if needed.
+                    """
+                    command_line: List[str] = original_run_invocation(*args, **kwargs)
+
+                    logger.debug('MiniWDL wants to run command line: %s', command_line)
+
+                    # "exec" can be at index 1 or 2 depending on if we have a --verbose.
+                    subcommand_index = 2 if command_line[1] == "--verbose" else 1
+
+                    if '--fakeroot' in command_line and not self.can_fake_root():
+                        # We can't fake root so don't try.
+                        command_line.remove('--fakeroot')
+
+                    extra_flags: Set[str] = set()
+                    accelerators_needed: Optional[List[AcceleratorRequirement]] = self.accelerators
+                    if accelerators_needed is not None:
+                        for accelerator in accelerators_needed:
+                            if accelerator['kind'] == 'gpu':
+                                if accelerator['brand'] == 'nvidia':
+                                    # Tell Singularity to expose nvidia GPUs
+                                    extra_flags.add('--nv')
+                                elif accelerator['api'] == 'rocm':
+                                    # Tell Singularity to expose ROCm GPUs
+                                    extra_flags.add('--nv')
+                                else:
+                                    raise RuntimeError('Cannot expose allocated accelerator %s to Singularity job', accelerator)
+
+                    for flag in extra_flags:
+                        # Put in all those flags
+                        command_line.insert(subcommand_index + 1, flag)
+
+                    logger.debug('Amended command line to: %s', command_line)
+
+                    # Return the modified command line
+                    return command_line
+
+                # Apply the patch
+                task_container._run_invocation = patched_run_invocation #  type: ignore
 
             # Show the runtime info to the container
             task_container.process_runtime(miniwdl_logger, {binding.name: binding.value for binding in runtime_bindings})
@@ -1112,15 +1130,16 @@ def patched_run_invocation(*args: Any, **kwargs: Any) -> List[str]:
             # become typed.
             host_stdout_txt: str = task_container.host_stdout_txt() #  type: ignore
             host_stderr_txt: str = task_container.host_stderr_txt() #  type: ignore
-
-            # Before running the command, we need to make sure the container's
-            # image is already pulled, so MiniWDL doesn't try and pull it.
-            # MiniWDL only locks its cache directory within a process, and we
-            # need to coordinate with other processes sharing the cache.
-            # TODO: We assume that if we share the cache, we share the coordination directory!
-            with global_mutex(file_store.coordination_dir, 'miniwdl_sif_cache_mutex'):
-                with ExitStack() as cleanup:
-                    task_container._pull(miniwdl_logger, cleanup)
+            
+            if isinstance(task_container, SingularityContainer):
+                # Before running the command, we need to make sure the container's
+                # image is already pulled, so MiniWDL doesn't try and pull it.
+                # MiniWDL only locks its cache directory within a process, and we
+                # need to coordinate with other processes sharing the cache.
+                # TODO: We assume that if we share the cache, we share the coordination directory!
+                with global_mutex(file_store.coordination_dir, 'miniwdl_sif_cache_mutex'):
+                    with ExitStack() as cleanup:
+                        task_container._pull(miniwdl_logger, cleanup)
 
             # Run the command in the container
             logger.info('Executing command in %s: %s', task_container, command_string)
@@ -1813,14 +1832,20 @@ def main() -> None:
     parser = argparse.ArgumentParser(description='Runs WDL files with toil.')
     addOptions(parser, jobstore_as_flag=True)
 
-    parser.add_argument("wdl_uri", type=str, help="WDL document URI")
-    parser.add_argument("inputs_uri", type=str, help="WDL input JSON URI")
+    parser.add_argument("wdl_uri", type=str,
+                        help="WDL document URI")
+    parser.add_argument("inputs_uri", type=str, nargs='?',
+                        help="WDL input JSON URI")
+    parser.add_argument("--input", "-i", dest="inputs_uri", type=str,
+                        help="WDL input JSON URI")
     parser.add_argument("--outputDialect", dest="output_dialect", type=str, default='cromwell', choices=['cromwell', 'miniwdl'],
                         help=("JSON output format dialect. 'cromwell' just returns the workflow's output"
-                              "values as JSON, while 'miniwld' nests that under an 'outputs' key, and "
+                              "values as JSON, while 'miniwdl' nests that under an 'outputs' key, and "
                               "includes a 'dir' key where files are written."))
     parser.add_argument("--outputDirectory", "-o", dest="output_directory", type=str, default=None,
                         help=("Directory in which to save output files. By default a new directory is created in the current directory."))
+    parser.add_argument("--outputFile", "-m", dest="output_file", type=argparse.FileType('w'), default=sys.stdout,
+                        help="File to save output JSON to.")
 
     options = parser.parse_args(sys.argv[1:])
 
@@ -1937,8 +1962,8 @@ def devirtualize_output(filename: str) -> str:
         outputs = WDL.values_to_json(output_bindings)
         if options.output_dialect == 'miniwdl':
             outputs = {'dir': output_directory, 'outputs': outputs}
-        print(json.dumps(outputs))
-
+        options.output_file.write(json.dumps(outputs))
+        options.output_file.write('\n')
 
 
 if __name__ == "__main__":


Test patch:
diff --git a/src/toil/test/docs/scripts/tutorial_wdlexample.py b/src/toil/test/docs/scripts/tutorial_wdlexample.py
deleted file mode 100644
--- a/src/toil/test/docs/scripts/tutorial_wdlexample.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import os
-import subprocess
-import tempfile
-
-from toil.common import Toil
-from toil.job import Job
-
-
-
-def initialize_jobs(job):
-    job.fileStore.logToMaster("initialize_jobs")
-
-
-def runQC(job, wdl_file, wdl_filename, json_file, json_filename, outputs_dir, jar_loc,output_num):
-    job.fileStore.logToMaster("runQC")
-    tempDir = job.fileStore.getLocalTempDir()
-
-    wdl = job.fileStore.readGlobalFile(wdl_file, userPath=os.path.join(tempDir, wdl_filename))
-    json = job.fileStore.readGlobalFile(json_file, userPath=os.path.join(tempDir, json_filename))
-
-    subprocess.check_call(["java","-jar",jar_loc,"run",wdl,"--inputs",json])
-
-    output_filename = "output.txt"
-    output_file = job.fileStore.writeGlobalFile(outputs_dir + output_filename)
-    job.fileStore.readGlobalFile(output_file, userPath=os.path.join(outputs_dir, "sample_" + output_num + "_" + output_filename))
-    return output_file
-
-
-if __name__ == "__main__":
-    jobstore: str = tempfile.mkdtemp("tutorial_wdlexample")
-    os.rmdir(jobstore)
-    options = Job.Runner.getDefaultOptions(jobstore)
-    options.logLevel = "INFO"
-    options.clean = "always"
-
-    with Toil(options) as toil:
-
-        # specify the folder where the wdl and json files live
-        inputs_dir = "wdlExampleFiles/"
-        # specify where you wish the outputs to be written
-        outputs_dir = "wdlExampleFiles/"
-        # specify the location of your cromwell jar
-        jar_loc = os.path.abspath("wdlExampleFiles/cromwell-35.jar")
-
-        job0 = Job.wrapJobFn(initialize_jobs)
-
-        wdl_filename = "hello.wdl"
-        wdl_file = toil.importFile("file://" + os.path.abspath(os.path.join(inputs_dir, wdl_filename)))
-
-
-        # add list of yml config inputs here or import and construct from file
-        json_files = ["hello1.json", "hello2.json", "hello3.json"]
-        i = 0
-        for json in json_files:
-            i = i + 1
-            json_file = toil.importFile("file://" + os.path.join(inputs_dir, json))
-            json_filename = json
-            job = Job.wrapJobFn(runQC, wdl_file, wdl_filename, json_file, json_filename, outputs_dir, jar_loc, output_num=str(i))
-            job0.addChild(job)
-
-        toil.start(job0)
diff --git a/src/toil/test/docs/scriptsTest.py b/src/toil/test/docs/scriptsTest.py
--- a/src/toil/test/docs/scriptsTest.py
+++ b/src/toil/test/docs/scriptsTest.py
@@ -147,10 +147,5 @@ def testServices(self):
     def testStaging(self):
         self.checkExitCode("tutorial_staging.py")
 
-    @unittest.skip('Needs cromwell jar file to run.')
-    def testWdlexample(self):
-        self.checkExitCode("tutorial_wdlexample.py")
-
-
 if __name__ == "__main__":
     unittest.main()