Repository: DataBiosphere/toil

Issue:
Fix Singularity caching with toil-wdl-runner
toil-wdl-runner's singularity image cache is currently set a new directory with the workflow UUID under the config's `workDir`. This results in images being pulled every workflow run.

┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-1484)
┆Issue Number: TOIL-1484



Discussion:

Resolving patch:
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -286,8 +286,6 @@ def set_option(option_name: str,
 
             1. options object under option_name
             2. options object under old_names
-            3. environment variables in env
-            4. provided default value
 
             Selected option value is run through parsing_funtion if it is set.
             Then the parsed value is run through check_function to check it for
@@ -403,6 +401,14 @@ def set_option(option_name: str,
         set_option("badWorkerFailInterval")
         set_option("logLevel")
 
+        # Apply overrides as highest priority
+        # Override workDir with value of TOIL_WORKDIR_OVERRIDE if it exists
+        if os.getenv('TOIL_WORKDIR_OVERRIDE') is not None:
+            self.workDir = os.getenv('TOIL_WORKDIR_OVERRIDE')
+        # Override workDir with value of TOIL_WORKDIR_OVERRIDE if it exists
+        if os.getenv('TOIL_COORDINATION_DIR_OVERRIDE') is not None:
+            self.workDir = os.getenv('TOIL_COORDINATION_DIR_OVERRIDE')
+
         self.check_configuration_consistency()
 
         logger.debug("Loaded configuration: %s", vars(options))
diff --git a/src/toil/provisioners/abstractProvisioner.py b/src/toil/provisioners/abstractProvisioner.py
--- a/src/toil/provisioners/abstractProvisioner.py
+++ b/src/toil/provisioners/abstractProvisioner.py
@@ -812,10 +812,14 @@ def add_toil_service(self, config: InstanceConfiguration, role: str, keyPath: st
                 -v /opt:/opt \\
                 -v /etc/kubernetes:/etc/kubernetes \\
                 -v /etc/kubernetes/admin.conf:/root/.kube/config \\
+                # Pass in a path to use for singularity image caching into the container
+                -e TOIL_KUBERNETES_HOST_PATH=/var/lib/toil \\
+                -e SINGULARITY_CACHEDIR=/var/lib/toil/singularity \\
+                -e MINIWDL__SINGULARITY__IMAGE_CACHE=/var/lib/toil/miniwdl \\
                 # These rules are necessary in order to get user namespaces working
                 # https://github.com/apptainer/singularity/issues/5806
                 --security-opt seccomp=unconfined \\
-                --security-opt systempaths=unconfined\\
+                --security-opt systempaths=unconfined \\
                 --name=toil_{role} \\
                 {applianceSelf()} \\
                 {entryPointArgs}
diff --git a/src/toil/wdl/wdltoil.py b/src/toil/wdl/wdltoil.py
--- a/src/toil/wdl/wdltoil.py
+++ b/src/toil/wdl/wdltoil.py
@@ -1539,17 +1539,25 @@ def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]:
 
             # Prepare to use Singularity. We will need plenty of space to
             # download images.
-            if 'SINGULARITY_CACHEDIR' not in os.environ:
-                # Cache Singularity's layers somehwere known to have space, not in home
-                os.environ['SINGULARITY_CACHEDIR'] = os.path.join(file_store.workflow_dir, 'singularity_cache')
+            # Default the Singularity and MiniWDL cache directories. This sets the cache to the same place as
+            # Singularity/MiniWDL's default cache directory
+            # With launch-cluster, the singularity and miniwdl cache is set to /var/lib/toil in abstractProvisioner.py
+            # A current limitation with the singularity/miniwdl cache is it cannot check for image updates if the
+            # filename is the same
+            singularity_cache = os.path.join(os.path.expanduser("~"), ".singularity")
+            miniwdl_cache = os.path.join(os.path.expanduser("~"), ".cache/miniwdl")
+
+            # Cache Singularity's layers somewhere known to have space
+            os.environ['SINGULARITY_CACHEDIR'] = os.environ.get("SINGULARITY_CACHEDIR", singularity_cache)
+
             # Make sure it exists.
             os.makedirs(os.environ['SINGULARITY_CACHEDIR'], exist_ok=True)
 
-            if 'MINIWDL__SINGULARITY__IMAGE_CACHE' not in os.environ:
-                # Cache Singularity images for the workflow on this machine.
-                # Since MiniWDL does only within-process synchronization for pulls,
-                # we also will need to pre-pull one image into here at a time.
-                os.environ['MINIWDL__SINGULARITY__IMAGE_CACHE'] = os.path.join(file_store.workflow_dir, 'miniwdl_sif_cache')
+            # Cache Singularity images for the workflow on this machine.
+            # Since MiniWDL does only within-process synchronization for pulls,
+            # we also will need to pre-pull one image into here at a time.
+            os.environ['MINIWDL__SINGULARITY__IMAGE_CACHE'] = os.environ.get("MINIWDL__SINGULARITY__IMAGE_CACHE", miniwdl_cache)
+
             # Make sure it exists.
             os.makedirs(os.environ['MINIWDL__SINGULARITY__IMAGE_CACHE'], exist_ok=True)
 


Test patch: