Repository: common-workflow-language/cwltool

Issue:
Singularity volume mapping fails
## Expected Behavior
CWL v1.2 conformance test 338 (iwdr_dir_literal_real_file) should work on both Docker and Singularity

## Actual Behavior
Fails on singularity with:
```
Test 338 failed: /home/runner/work/cwltool/cwltool/cwl-v1.2-main/cwltool_with_cov3 --parallel --singularity --outdir=/tmp/tmp_jvxdboq --quiet tests/iwd/iwdr_dir_literal_real_file.cwl tests/loadContents/input.yml
883
Test directory literal containing a real file
884
Returned non-zero
885
INFO:    Converting OCI blobs to SIF format
886
INFO:    Starting build...
887
Getting image source signatures
888
Copying blob sha256:79d3b412d7261ce15487ba01f99e99609855e5ceb3578b4513a2a96d6184352e
889
Copying config sha256:cc397186c1b59edd2076c8986f259e8ef49c2108afe9691817a52cc8cc3fa000
890
Writing manifest to image destination
891
Storing signatures
892
2021/04/23 18:01:42  info unpack layer: sha256:79d3b412d7261ce15487ba01f99e99609855e5ceb3578b4513a2a96d6184352e
893
INFO:    Creating SIF file...
894
ERROR Got workflow error: [Errno 2] No such file or directory: '_:f9e78fa6-0296-4398-9dde-fb7b2580145a'
895
Traceback (most recent call last):
896
  File "/home/runner/work/cwltool/cwltool/cwltool/executors.py", line 332, in _runner
897
    job.run(runtime_context, TMPDIR_LOCK)
898
  File "/home/runner/work/cwltool/cwltool/cwltool/job.py", line 827, in run
899
    (runtime, cidfile) = self.create_runtime(env, runtimeContext)
900
  File "/home/runner/work/cwltool/cwltool/cwltool/singularity.py", line 427, in create_runtime
901
    self.add_volumes(
902
  File "/home/runner/work/cwltool/cwltool/cwltool/job.py", line 700, in add_volumes
903
    self.add_file_or_directory_volume(runtime, vol, host_outdir_tgt)
904
  File "/home/runner/work/cwltool/cwltool/cwltool/singularity.py", line 296, in add_file_or_directory_volume
905
    shutil.copytree(volume.resolved, host_outdir_tgt)
906
  File "/opt/hostedtoolcache/Python/3.9.4/x64/lib/python3.9/shutil.py", line 555, in copytree
907
    with os.scandir(src) as itr:
908
FileNotFoundError: [Errno 2] No such file or directory: '_:f9e78fa6-0296-4398-9dde-fb7b2580145a'
909
ERROR Workflow error, try again with --debug for more information:
910
[Errno 2] No such file or directory: '_:f9e78fa6-0296-4398-9dde-fb7b2580145a'
911
```

There's clearly a bug in `cwl.singularity` around volume mapping. After a quick look, it appears to be related to `cwltool.singularity.SingularityCommandLineJob.add_file_or_directory_volume` which says in a comment 
```
            # workaround for lack of overlapping mounts in Singularity
            # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
            # once https://github.com/sylabs/singularity/issues/1607
            # is fixed
```
(Introduced in 087594e8e22310830bc5b95a5fa01d869f6a1339)

That singularity issue appears to be fixed in more recent versions (3.4 or 3.5 according to S. contributors)
https://github.com/hpcng/singularity/issues/1607


Discussion:
Note that when this is resolved, we can remove the exclusion from `conformance-test.sh`.

Resolving patch:
diff --git a/cwltool/singularity.py b/cwltool/singularity.py
--- a/cwltool/singularity.py
+++ b/cwltool/singularity.py
@@ -74,6 +74,12 @@ def is_version_3_1_or_newer() -> bool:
     return int(version[0]) >= 4 or (int(version[0]) == 3 and int(version[1]) >= 1)
 
 
+def is_version_3_4_or_newer() -> bool:
+    """Detect if Singularity v3.4+ is available."""
+    version = get_version().split(".")
+    return int(version[0]) >= 4 or (int(version[0]) == 3 and int(version[1]) >= 4)
+
+
 def _normalize_image_id(string: str) -> str:
     return string.replace("/", "_") + ".img"
 
@@ -285,18 +291,17 @@ def append_volume(
     def add_file_or_directory_volume(
         self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str]
     ) -> None:
-        if host_outdir_tgt is not None:
-            # workaround for lack of overlapping mounts in Singularity
-            # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
-            # once https://github.com/sylabs/singularity/issues/1607
-            # is fixed
-            if volume.type == "File":
-                shutil.copy(volume.resolved, host_outdir_tgt)
+        if not volume.resolved.startswith("_:"):
+            if host_outdir_tgt is not None and not is_version_3_4_or_newer():
+                # workaround for lack of overlapping mounts in Singularity <3.4
+                if volume.type == "File":
+                    os.makedirs(os.path.dirname(host_outdir_tgt), exist_ok=True)
+                    shutil.copy(volume.resolved, host_outdir_tgt)
+                else:
+                    shutil.copytree(volume.resolved, host_outdir_tgt)
+                ensure_non_writable(host_outdir_tgt)
             else:
-                shutil.copytree(volume.resolved, host_outdir_tgt)
-            ensure_non_writable(host_outdir_tgt)
-        elif not volume.resolved.startswith("_:"):
-            self.append_volume(runtime, volume.resolved, volume.target)
+                self.append_volume(runtime, volume.resolved, volume.target)
 
     def add_writable_file_volume(
         self,
@@ -305,11 +310,8 @@ def add_writable_file_volume(
         host_outdir_tgt: Optional[str],
         tmpdir_prefix: str,
     ) -> None:
-        if host_outdir_tgt is not None:
-            # workaround for lack of overlapping mounts in Singularity
-            # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
-            # once https://github.com/sylabs/singularity/issues/1607
-            # is fixed
+        if host_outdir_tgt is not None and not is_version_3_4_or_newer():
+            # workaround for lack of overlapping mounts in Singularity <3.4
             if self.inplace_update:
                 try:
                     os.link(os.path.realpath(volume.resolved), host_outdir_tgt)
@@ -322,14 +324,21 @@ def add_writable_file_volume(
             self.append_volume(runtime, volume.resolved, volume.target, writable=True)
             ensure_writable(volume.resolved)
         else:
-            file_copy = os.path.join(
-                create_tmp_dir(tmpdir_prefix),
-                os.path.basename(volume.resolved),
-            )
-            shutil.copy(volume.resolved, file_copy)
-            # volume.resolved = file_copy
-            self.append_volume(runtime, file_copy, volume.target, writable=True)
-            ensure_writable(file_copy)
+            if host_outdir_tgt:
+                # shortcut, just copy to the output directory
+                # which is already going to be mounted
+                if not os.path.exists(os.path.dirname(host_outdir_tgt)):
+                    os.makedirs(os.path.dirname(host_outdir_tgt))
+                shutil.copy(volume.resolved, host_outdir_tgt)
+                ensure_writable(host_outdir_tgt)
+            else:
+                file_copy = os.path.join(
+                    create_tmp_dir(tmpdir_prefix),
+                    os.path.basename(volume.resolved),
+                )
+                shutil.copy(volume.resolved, file_copy)
+                self.append_volume(runtime, file_copy, volume.target, writable=True)
+                ensure_writable(file_copy)
 
     def add_writable_directory_volume(
         self,
@@ -339,31 +348,33 @@ def add_writable_directory_volume(
         tmpdir_prefix: str,
     ) -> None:
         if volume.resolved.startswith("_:"):
+            # Synthetic directory that needs creating first
             if host_outdir_tgt is not None:
                 new_dir = host_outdir_tgt
             else:
                 new_dir = os.path.join(
                     create_tmp_dir(tmpdir_prefix),
-                    os.path.basename(volume.resolved),
+                    os.path.basename(volume.target),
                 )
+                self.append_volume(runtime, new_dir, volume.target, writable=True)
             os.makedirs(new_dir)
         else:
-            if host_outdir_tgt is not None:
-                # workaround for lack of overlapping mounts in Singularity
-                # revert to daa923d5b0be3819b6ed0e6440e7193e65141052
-                # once https://github.com/sylabs/singularity/issues/1607
-                # is fixed
+            if host_outdir_tgt is not None and not is_version_3_4_or_newer():
+                # workaround for lack of overlapping mounts in Singularity < 3.4
                 shutil.copytree(volume.resolved, host_outdir_tgt)
                 ensure_writable(host_outdir_tgt)
             else:
                 if not self.inplace_update:
-                    dir_copy = os.path.join(
-                        create_tmp_dir(tmpdir_prefix),
-                        os.path.basename(volume.resolved),
-                    )
-                    shutil.copytree(volume.resolved, dir_copy)
-                    source = dir_copy
-                    # volume.resolved = dir_copy
+                    if not host_outdir_tgt:
+                        tmpdir = create_tmp_dir(tmpdir_prefix)
+                        new_dir = os.path.join(
+                            tmpdir, os.path.basename(volume.resolved)
+                        )
+                        shutil.copytree(volume.resolved, new_dir)
+                        source = tmpdir
+                    else:
+                        shutil.copytree(volume.resolved, host_outdir_tgt)
+                        source = host_outdir_tgt
                 else:
                     source = volume.resolved
                 self.append_volume(runtime, source, volume.target, writable=True)


Test patch:
diff --git a/tests/test_environment.py b/tests/test_environment.py
--- a/tests/test_environment.py
+++ b/tests/test_environment.py
@@ -41,6 +41,7 @@ def assert_env_matches(
     """
     e = dict(env)
     for k, check in checks.items():
+        assert k in e
         v = e.pop(k)
         assert_envvar_matches(check, k, env)
 
@@ -127,7 +128,6 @@ def PWD(v: str, env: Env) -> bool:
             "LANG": "C",
             "LD_LIBRARY_PATH": None,
             "PATH": None,
-            "PROMPT_COMMAND": None,
             "PS1": None,
             "PWD": PWD,
             "TMPDIR": "/tmp",
@@ -140,12 +140,16 @@ def PWD(v: str, env: Env) -> bool:
         vminor = int(version[1])
         sing_vars: EnvChecks = {
             "SINGULARITY_CONTAINER": None,
-            "SINGULARITY_ENVIRONMENT": None,
             "SINGULARITY_NAME": None,
         }
-        if vminor < 6:
+        if vminor < 5:
+            sing_vars["SINGULARITY_APPNAME"] = None
+        if vminor >= 5:
+            sing_vars["PROMPT_COMMAND"] = None
+            sing_vars["SINGULARITY_ENVIRONMENT"] = None
+        if vminor == 5:
             sing_vars["SINGULARITY_INIT"] = "1"
-        else:
+        elif vminor > 5:
             sing_vars["SINGULARITY_COMMAND"] = "exec"
             if vminor >= 7: