Repository: DataBiosphere/toil

Issue:
Allow toil-wdl-runner to run on Mesos and Kubernetes batchsystems
These issues popped up when running on both mesos and kubernetes:
- Singularity tries to create user namespaces but doesn't have enough permissions
- /etc/localtime does not exist and singularity is unable to run
- `--containall` command in miniwdl results in operation not permitted

This is related to #4717

┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-1479)
┆Issue Number: TOIL-1479



Discussion:

Resolving patch:
diff --git a/docker/Dockerfile.py b/docker/Dockerfile.py
--- a/docker/Dockerfile.py
+++ b/docker/Dockerfile.py
@@ -61,7 +61,9 @@
                          'libcurl4-nss-dev',
                          'libapr1',
                          # Dependencies for singularity
-                         'containernetworking-plugins'])
+                         'containernetworking-plugins',
+                         # Dependencies for singularity on kubernetes
+                         'tzdata'])
 
 
 def heredoc(s):
diff --git a/src/toil/provisioners/abstractProvisioner.py b/src/toil/provisioners/abstractProvisioner.py
--- a/src/toil/provisioners/abstractProvisioner.py
+++ b/src/toil/provisioners/abstractProvisioner.py
@@ -812,6 +812,10 @@ def add_toil_service(self, config: InstanceConfiguration, role: str, keyPath: st
                 -v /opt:/opt \\
                 -v /etc/kubernetes:/etc/kubernetes \\
                 -v /etc/kubernetes/admin.conf:/root/.kube/config \\
+                # These rules are necessary in order to get user namespaces working
+                # https://github.com/apptainer/singularity/issues/5806
+                --security-opt seccomp=unconfined \\
+                --security-opt systempaths=unconfined\\
                 --name=toil_{role} \\
                 {applianceSelf()} \\
                 {entryPointArgs}
diff --git a/src/toil/wdl/wdltoil.py b/src/toil/wdl/wdltoil.py
--- a/src/toil/wdl/wdltoil.py
+++ b/src/toil/wdl/wdltoil.py
@@ -1341,7 +1341,7 @@ def __init__(self, task: WDL.Tree.Task, prev_node_results: Sequence[Promised[WDL
 
     def can_fake_root(self) -> bool:
         """
-        Determie if --fakeroot is likely to work for Singularity.
+        Determine if --fakeroot is likely to work for Singularity.
         """
 
         # We need to have an entry for our user in /etc/subuid to grant us a range of UIDs to use, for fakeroot to work.
@@ -1359,6 +1359,16 @@ def can_fake_root(self) -> bool:
         logger.warning('No subuids are assigned to %s; cannot fake root.', username)
         return False
 
+    def can_mount_proc(self) -> bool:
+        """
+        Determine if --containall will work for Singularity. On Kubernetes, this will result in operation not permitted
+        See: https://github.com/apptainer/singularity/issues/5857
+
+        So if Kubernetes is detected, return False
+        :return: bool
+        """
+        return "KUBERNETES_SERVICE_HOST" not in os.environ
+
     @report_wdl_errors("run task")
     def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]:
         """
@@ -1597,6 +1607,10 @@ def patched_run_invocation(*args: Any, **kwargs: Any) -> List[str]:
                         # We can't fake root so don't try.
                         command_line.remove('--fakeroot')
 
+                    # If on Kubernetes and proc cannot be mounted, get rid of --containall
+                    if '--containall' in command_line and not self.can_mount_proc():
+                        command_line.remove('--containall')
+
                     extra_flags: Set[str] = set()
                     accelerators_needed: Optional[List[AcceleratorRequirement]] = self.accelerators
                     if accelerators_needed is not None:


Test patch:
diff --git a/src/toil/test/wdl/wdltoil_test.py b/src/toil/test/wdl/wdltoil_test.py
--- a/src/toil/test/wdl/wdltoil_test.py
+++ b/src/toil/test/wdl/wdltoil_test.py
@@ -3,27 +3,31 @@
 import shutil
 import subprocess
 import unittest
-import uuid
-from typing import Any, Dict, List, Optional, Set
-from unittest.mock import patch
+from uuid import uuid4
+from typing import Optional
 
 from unittest.mock import patch
 from typing import Any, Dict, List, Set
 
+import pytest
+
+from toil.provisioners import cluster_factory
 from toil.test import (ToilTest,
                        needs_docker_cuda,
                        needs_google_storage,
                        needs_singularity_or_docker,
-                       slow)
+                       slow, integrative)
+from toil.test.provisioners.clusterTest import AbstractClusterTest
 from toil.version import exactPython
 from toil.wdl.wdltoil import WDLSectionJob, WDLWorkflowGraph
 
 
 class BaseWDLTest(ToilTest):
     """Base test class for WDL tests."""
+
     def setUp(self) -> None:
         """Runs anew before each test to create farm fresh temp dirs."""
-        self.output_dir = os.path.join('/tmp/', 'toil-wdl-test-' + str(uuid.uuid4()))
+        self.output_dir = os.path.join('/tmp/', 'toil-wdl-test-' + str(uuid4()))
         os.makedirs(self.output_dir)
 
     def tearDown(self) -> None:
@@ -36,6 +40,7 @@ class WDLConformanceTests(BaseWDLTest):
     WDL conformance tests for Toil.
     """
     wdl_dir = "wdl-conformance-tests"
+
     @classmethod
     def setUpClass(cls) -> None:
 
@@ -87,6 +92,7 @@ def tearDownClass(cls) -> None:
 
 class WDLTests(BaseWDLTest):
     """Tests for Toil's MiniWDL-based implementation."""
+
     @classmethod
     def setUpClass(cls) -> None:
         """Runs once for all tests."""
@@ -101,7 +107,8 @@ def test_MD5sum(self):
         wdl = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.1.0.wdl')
         json_file = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.json')
 
-        result_json = subprocess.check_output(self.base_command + [wdl, json_file, '-o', self.output_dir, '--logDebug', '--retryCount=0'])
+        result_json = subprocess.check_output(
+            self.base_command + [wdl, json_file, '-o', self.output_dir, '--logDebug', '--retryCount=0'])
         result = json.loads(result_json)
 
         assert 'ga4ghMd5.value' in result
@@ -115,7 +122,9 @@ def test_miniwdl_self_test(self, extra_args: Optional[List[str]] = None) -> None
         wdl_file = os.path.abspath('src/toil/test/wdl/miniwdl_self_test/self_test.wdl')
         json_file = os.path.abspath('src/toil/test/wdl/miniwdl_self_test/inputs.json')
 
-        result_json = subprocess.check_output(self.base_command + [wdl_file, json_file, '--logDebug', '-o', self.output_dir, '--outputDialect', 'miniwdl'] + (extra_args or []))
+        result_json = subprocess.check_output(
+            self.base_command + [wdl_file, json_file, '--logDebug', '-o', self.output_dir, '--outputDialect',
+                                 'miniwdl'] + (extra_args or []))
         result = json.loads(result_json)
 
         # Expect MiniWDL-style output with a designated "dir"
@@ -172,7 +181,8 @@ def test_giraffe_deepvariant(self):
                 "GiraffeDeepVariant.runDeepVariantCallVariants.in_dv_gpu_container": "google/deepvariant:1.3.0-gpu"
             })
 
-        result_json = subprocess.check_output(self.base_command + [wdl_file, json_file, '-o', self.output_dir, '--outputDialect', 'miniwdl'])
+        result_json = subprocess.check_output(
+            self.base_command + [wdl_file, json_file, '-o', self.output_dir, '--outputDialect', 'miniwdl'])
         result = json.loads(result_json)
 
         # Expect MiniWDL-style output with a designated "dir"
@@ -200,7 +210,9 @@ def test_giraffe(self):
         wdl_file = f"{base_uri}/workflows/giraffe.wdl"
         json_file = f"{base_uri}/params/giraffe.json"
 
-        result_json = subprocess.check_output(self.base_command + [wdl_file, json_file, '-o', self.output_dir, '--outputDialect', 'miniwdl', '--scale', '0.1'])
+        result_json = subprocess.check_output(
+            self.base_command + [wdl_file, json_file, '-o', self.output_dir, '--outputDialect', 'miniwdl', '--scale',
+                                 '0.1'])
         result = json.loads(result_json)
 
         # Expect MiniWDL-style output with a designated "dir"
@@ -265,7 +277,6 @@ def mock_get_transitive_dependencies(self: Any, node_id: str) -> Set[str]:
         # worth extracting a base type for this interface.
         with patch.object(WDLWorkflowGraph, 'is_decl', mock_is_decl):
             with patch.object(WDLWorkflowGraph, 'get_transitive_dependencies', mock_get_transitive_dependencies):
-
                 with self.subTest(msg="Two unrelated decls can coalesce"):
                     # Set up two unrelated decls
                     all_decls = {"decl1", "decl2"}
@@ -275,7 +286,7 @@ def mock_get_transitive_dependencies(self: Any, node_id: str) -> Set[str]:
                     }
 
                     result = WDLSectionJob.coalesce_nodes(["decl1", "decl2"], WDLWorkflowGraph([]))
-                    
+
                     # Make sure they coalesced
                     assert len(result) == 1
                     assert "decl1" in result[0]
@@ -289,12 +300,11 @@ def mock_get_transitive_dependencies(self: Any, node_id: str) -> Set[str]:
                     }
 
                     result = WDLSectionJob.coalesce_nodes(["decl", "nondecl"], WDLWorkflowGraph([]))
-                    
+
                     assert len(result) == 2
                     assert len(result[0]) == 1
                     assert len(result[1]) == 1
 
-
                 with self.subTest(msg="Two adjacent decls with a common dependency can coalesce"):
                     all_decls = {"decl1", "decl2"}
                     all_deps = {
@@ -304,7 +314,7 @@ def mock_get_transitive_dependencies(self: Any, node_id: str) -> Set[str]:
                     }
 
                     result = WDLSectionJob.coalesce_nodes(["base", "decl1", "decl2"], WDLWorkflowGraph([]))
-                    
+
                     assert len(result) == 2
                     assert "base" in result[0]
                     assert "decl1" in result[1]
@@ -319,7 +329,7 @@ def mock_get_transitive_dependencies(self: Any, node_id: str) -> Set[str]:
                     }
 
                     result = WDLSectionJob.coalesce_nodes(["base", "decl1", "decl2"], WDLWorkflowGraph([]))
-                    
+
                     assert len(result) == 3
                     assert "base" in result[0]
 
@@ -332,12 +342,77 @@ def mock_get_transitive_dependencies(self: Any, node_id: str) -> Set[str]:
                     }
 
                     result = WDLSectionJob.coalesce_nodes(["decl1", "decl2", "successor"], WDLWorkflowGraph([]))
-                    
+
                     assert len(result) == 2
                     assert "decl1" in result[0]
                     assert "decl2" in result[0]
                     assert "successor" in result[1]
 
 
+@integrative
+@slow
+@pytest.mark.timeout(600)
+class WDLKubernetesClusterTest(AbstractClusterTest):
+    """
+    Ensure WDL works on the Kubernetes batchsystem.
+    """
+
+    def __init__(self, name):
+        super().__init__(name)
+        self.clusterName = 'wdl-integration-test-' + str(uuid4())
+        # t2.medium is the minimum t2 instance that permits Kubernetes
+        self.leaderNodeType = "t2.medium"
+        self.instanceTypes = ["t2.medium"]
+        self.clusterType = "kubernetes"
+
+    def setUp(self) -> None:
+        super().setUp()
+        self.jobStore = f'aws:{self.awsRegion()}:wdl-test-{uuid4()}'
+
+    def launchCluster(self) -> None:
+        self.createClusterUtil(args=['--leaderStorage', str(self.requestedLeaderStorage),
+                                     '--nodeTypes', ",".join(self.instanceTypes),
+                                     '-w', ",".join(self.numWorkers),
+                                     '--nodeStorage', str(self.requestedLeaderStorage)])
+
+    def test_wdl_kubernetes_cluster(self):
+        """
+        Test that a wdl workflow works on a kubernetes cluster. Launches a cluster with 1 worker. This runs a wdl
+        workflow that performs an image pull on the worker.
+        :return:
+        """
+        self.numWorkers = "1"
+        self.requestedLeaderStorage = 30
+        # create the cluster
+        self.launchCluster()
+        # get leader
+        self.cluster = cluster_factory(
+            provisioner="aws", zone=self.zone, clusterName=self.clusterName
+        )
+        self.leader = self.cluster.getLeader()
+
+        url = "https://github.com/DataBiosphere/wdl-conformance-tests.git"
+        commit = "09b9659cd01473e836738a2e0dd205df0adb49c5"
+        wdl_dir = "wdl_conformance_tests"
+
+        # get the wdl-conformance-tests repo to get WDL tasks to run
+        self.sshUtil([
+            "bash",
+            "-c",
+            f"git clone {url} {wdl_dir} && cd {wdl_dir} && git checkout {commit}"
+        ])
+
+        # run on kubernetes batchsystem
+        toil_options = ['--batchSystem=kubernetes',
+                        f"--jobstore={self.jobStore}"]
+
+        # run WDL workflow that will run singularity
+        test_options = [f"tests/md5sum/md5sum.wdl", f"tests/md5sum/md5sum.json"]
+        self.sshUtil([
+            "bash",
+            "-c",
+            f"cd {wdl_dir} && toil-wdl-runner {' '.join(test_options)} {' '.join(toil_options)}"])
+
+
 if __name__ == "__main__":
     unittest.main()  # run all tests