Repository: lanl/BEE (language: python)

Issue:
Checkpoint/Restart is broken
When testing the latest PR it seems that the checkpoint/restart was not working. 
We need to explore what is happening  and also create a test for it even if it is just to run manually.
Also we need to consider what to do with outputs stdout and stderr that may already exist for steps that are created due to checkpointing.

Builder broken with latest Charliecloud
I'm getting this error when trying to build with the latest charliecloud version: `ch-image build: error: argument --force: invalid choice: '/tmp' (choose from 'fakeroot', 'seccomp')`.


Discussion / hints:

Fix patch:
diff --git a/beeflow/cli.py b/beeflow/cli.py
index fad2e46c8..3d56ba0f2 100755
--- a/beeflow/cli.py
+++ b/beeflow/cli.py
@@ -223,7 +223,7 @@ def handle_terminate(signum, stack): # noqa
     sys.exit(1)
 
 
-MIN_CHARLIECLOUD_VERSION = (0, 27)
+MIN_CHARLIECLOUD_VERSION = (0, 32)
 
 
 def version_str(version):
diff --git a/beeflow/common/build/container_drivers.py b/beeflow/common/build/container_drivers.py
index a4a814898..06f53423d 100644
--- a/beeflow/common/build/container_drivers.py
+++ b/beeflow/common/build/container_drivers.py
@@ -37,12 +37,8 @@ def __init__(self, task):
         Parse hints and requirements to determine target build
         system. Harvest relevant BeeConfig params in kwargs.
 
-        :param requirements: the workflow requirements
-        :type requirements: set of Requirement instances
-        :param hints: the workflow hints (optional requirements)
-        :type hints: set of Requirement instances
-        :param kwargs: Dictionary of build system config options
-        :type kwargs: set of build system parameters
+        :param task: the task to build for
+        :type task: Task
         """
         # Store build container archive pased on config file or relative to bee_workdir if not set.
         container_archive = bc.get('builder', 'container_archive')
@@ -222,6 +218,9 @@ def process_docker_file(self, task_dockerfile=None, force=False):
             except FileNotFoundError:
                 pass
 
+        # Get the force type (uses seccomp by default)
+        force_type = self.task.get_requirement('DockerRequirement', 'beeflow:forceType', 'seccomp')
+
         # Out of excuses. Build the image.
         with tempfile.NamedTemporaryFile(mode='w+', encoding='utf-8') as tmp:
             # Write the dockerfile to the tempfile
@@ -230,7 +229,7 @@ def process_docker_file(self, task_dockerfile=None, force=False):
             dockerfile_path = tmp.name
             # Build and run the command
             log.info('Context directory configured. Beginning build.')
-            cmd = (f'ch-image build -t {self.container_name} --force '
+            cmd = (f'ch-image build -t {self.container_name} --force {force_type} '
                    f'-f {dockerfile_path} {context_dir}\n'
                    f'ch-convert -i ch-image -o tar {ch_build_addr} '
                    f'{self.container_archive}/{ch_build_addr}.tar.gz'
diff --git a/beeflow/common/config_driver.py b/beeflow/common/config_driver.py
index a39c4e48e..903dcab51 100644
--- a/beeflow/common/config_driver.py
+++ b/beeflow/common/config_driver.py
@@ -4,7 +4,6 @@
 import getpass
 import os
 import platform
-from pathlib import Path
 import random
 import shutil
 import sys
@@ -13,6 +12,7 @@
 
 from beeflow.common.config_validator import ConfigValidator
 from beeflow.common.cli import NaturalOrderGroup
+from beeflow.common import validation
 from beeflow.common.tab_completion import filepath_completion
 
 
@@ -164,59 +164,6 @@ def check_yes(msg):
     return res.lower() == 'y'
 
 
-# Specialized functions for validation and config initialization
-
-def validate_path(path):
-    """Check that the path exists."""
-    path = os.path.expanduser(path)
-    if not os.path.exists(path):
-        raise ValueError(f'path "{path}" does not exist')
-    return path
-
-
-def validate_dir(path):
-    """Check that the path exists and is a directory."""
-    path = validate_path(path)
-    if not os.path.isdir(path):
-        raise ValueError('path "{path}" is not a directory')
-    return path
-
-
-def validate_make_dir(path):
-    """Check if the dir exists and if not create it."""
-    Path(path).mkdir(parents=True, exist_ok=True)
-    return path
-
-
-def validate_parent_dir(path):
-    """Ensure that the parent dir of path exists, or create it."""
-    validate_make_dir(Path(path).parent)
-    return path
-
-
-def validate_file(path):
-    """Check that the path exists and is a file."""
-    path = validate_path(path)
-    if not os.path.isfile(path):
-        raise ValueError(f'path "{path}" is not a file')
-    return path
-
-
-def validate_nonnegative_int(value):
-    """Validate that the input is nonnegative."""
-    i = int(value)
-    if i < 0:
-        raise ValueError('the value must be nonnegative')
-    return i
-
-
-# NOTE: You must use validate_bool for all boolean values (since just using
-#       bool, as in bool('False'), gives True for any string of length > 0)
-def validate_bool(value):
-    """Validate a boolean value."""
-    return str(value).lower() == 'true'
-
-
 def check_choice(msg, opts):
     """Ask the user to pick from opts."""
     while True:
@@ -234,42 +181,6 @@ def join_path(*pargs):
     return path
 
 
-def job_template_init(path, cur_opts):
-    """Job template init function.
-
-    :param path: chosen path of Jinja job template file
-    :param cur_opts: current chosen options from the config generator
-    :return: initialized Jinja template path
-    """
-    path = os.path.expanduser(path)
-    if os.path.exists(path):
-        return path
-    print('Check that options (like accounts)for your particular system are satisfied.')
-    if not check_yes(f'Do you want to write a default job template to:\n "{path}"?'):
-        return path
-    # Check for workload_scheduler in cur_opts (NB: this will need to be updated
-    # if the option name changes later on)
-    if 'DEFAULT' in cur_opts and 'workload_scheduler' in cur_opts['DEFAULT']:
-        template = cur_opts['DEFAULT']['workload_scheduler']
-    else:
-        template = check_choice('What template should be generated?', ('Slurm', 'LSF', 'Simple'))
-    template_files = {
-        'Slurm': 'slurm-submit.jinja',
-        'LSF': 'lsf-submit.jinja',
-    }
-    if template not in template_files:
-        raise NotImplementedError(f'generation of template file "{template}" is not implemented')
-    fname = template_files[template]
-    # I don't like how this is done, but there seems to be some difficulties
-    # with using the pkgutil.get_data() method and the importlib.resources
-    # module.
-    common = os.path.dirname(__file__)
-    beeflow = os.path.dirname(common)
-    file_path = join_path(beeflow, 'data', 'job_templates', fname)
-    shutil.copy(file_path, path)
-    return path
-
-
 def bee_workdir_init(path, _cur_opts):
     """BEE workdir init function.
 
@@ -323,19 +234,19 @@ def filepath_completion_input(*pargs, **kwargs):
 VALIDATOR = ConfigValidator('BEE configuration file and validation information.')
 VALIDATOR.section('DEFAULT', info='Default bee.conf configuration section.')
 VALIDATOR.option('DEFAULT', 'bee_workdir', info='main BEE workdir',
-                 attrs={'default': DEFAULT_BEE_WORKDIR}, validator=validate_make_dir)
+                 attrs={'default': DEFAULT_BEE_WORKDIR}, validator=validation.make_dir)
 VALIDATOR.option('DEFAULT', 'workload_scheduler', choices=('Slurm', 'LSF', 'Simple'),
                  info='backend workload scheduler to interact with ')
-VALIDATOR.option('DEFAULT', 'use_archive', validator=validate_bool, attrs={'default': True},
+VALIDATOR.option('DEFAULT', 'use_archive', validator=validation.bool_, attrs={'default': True},
                  info='use the BEE archiving functinality')
-VALIDATOR.option('DEFAULT', 'bee_dep_image', validator=validate_file,
+VALIDATOR.option('DEFAULT', 'bee_dep_image', validator=validation.file_,
                  info='container image with BEE dependencies',
                  attrs={'input': filepath_completion_input})
 VALIDATOR.option('DEFAULT', 'beeflow_pidfile',
                  attrs={'default': join_path(DEFAULT_BEE_WORKDIR, 'beeflow.pid')},
                  info='location of beeflow pidfile')
 VALIDATOR.option('DEFAULT', 'beeflow_socket',
-                 validator=validate_parent_dir,
+                 validator=validation.parent_dir,
                  attrs={'default': DEFAULT_BEEFLOW_SOCKET},
                  info='location of beeflow socket')
 VALIDATOR.option('DEFAULT', 'max_restarts', validator=int,
@@ -343,7 +254,7 @@ def filepath_completion_input(*pargs, **kwargs):
                  info='max number of times beeflow will restart a component on failure')
 # Workflow Manager
 VALIDATOR.section('workflow_manager', info='Workflow manager section.')
-VALIDATOR.option('workflow_manager', 'socket', validator=validate_parent_dir,
+VALIDATOR.option('workflow_manager', 'socket', validator=validation.parent_dir,
                  attrs={'default': DEFAULT_WFM_SOCKET},
                  info='workflow manager port')
 # Task manager
@@ -351,25 +262,28 @@ def filepath_completion_input(*pargs, **kwargs):
                   info='Task manager configuration and config of container to use.')
 VALIDATOR.option('task_manager', 'socket',
                  attrs={'default': DEFAULT_TM_SOCKET},
-                 validator=validate_parent_dir, info='task manager listen port')
+                 validator=validation.parent_dir, info='task manager listen port')
 VALIDATOR.option('task_manager', 'container_runtime', attrs={'default': 'Charliecloud'},
                  choices=('Charliecloud', 'Singularity'),
                  info='container runtime to use for configuration')
 VALIDATOR.option('task_manager', 'runner_opts', attrs={'default': ''},
                  info='special runner options to pass to the runner opts')
-# Note: I've added a special attrs keyword which can include anything. In this
-# case it's being used to store a special 'init' function that can be used to
-# initialize the parameter when a user is generating a new configuration.
-VALIDATOR.option('task_manager', 'job_template',
-                 info='job template to use for generating submission scripts',
-                 validator=validate_file,
-                 attrs={'init': job_template_init, 'default': join_path(CONF_DIR,
-                        'slurm-submit.jinja')})
+
+# Note: The special attrs keyword can include anything. One use case is for
+# storing a special 'init' function that can be used to initialize the
+# parameter when a user is generating a new configuration.
+
 # Charliecloud (depends on task_manager::container_runtime == Charliecloud)
 VALIDATOR.section('charliecloud', info='Charliecloud configuration section.',
                   depends_on=('task_manager', 'container_runtime', 'Charliecloud'))
 VALIDATOR.option('charliecloud', 'image_mntdir', attrs={'default': join_path('/tmp', USER)},
-                 info='Charliecloud mount directory', validator=validate_make_dir)
+                 info='Charliecloud mount directory', validator=validation.make_dir)
+# General job requirements
+VALIDATOR.section('job', info='General job requirements')
+VALIDATOR.option('job', 'default_account', validator=lambda val: val.strip(),
+                 info='default account to launch jobs with (leave blank if none)')
+VALIDATOR.option('job', 'default_time_limit', validator=validation.time_limit,
+                 info='default account time limit (leave blank if none)')
 
 
 def validate_chrun_opts(opts):
@@ -382,7 +296,7 @@ def validate_chrun_opts(opts):
     return opts
 
 
-VALIDATOR.option('charliecloud', 'chrun_opts', attrs={'default': ''},
+VALIDATOR.option('charliecloud', 'chrun_opts', attrs={'default': '--home'},
                  validator=validate_chrun_opts,
                  info='extra options to pass to ch-run')
 VALIDATOR.option('charliecloud', 'setup', attrs={'default': ''},
@@ -399,16 +313,16 @@ def validate_chrun_opts(opts):
 VALIDATOR.option('graphdb', 'https_port', attrs={'default': DEFAULT_HTTPS_PORT},
                  info='HTTPS port used for the graph database')
 VALIDATOR.option('graphdb', 'gdb_image_mntdir', attrs={'default': join_path('/tmp', USER)},
-                 info='graph database image mount directory', validator=validate_make_dir)
+                 info='graph database image mount directory', validator=validation.make_dir)
 VALIDATOR.option('graphdb', 'sleep_time', validator=int, attrs={'default': 10},
                  info='how long to wait for the graph database to come up (this can take a while, '
                       'depending on the system)')
 # Builder
 VALIDATOR.section('builder', info='General builder configuration section.')
 VALIDATOR.option('builder', 'deployed_image_root', attrs={'default': '/tmp'},
-                 info='where to deploy container images', validator=validate_make_dir)
+                 info='where to deploy container images', validator=validation.make_dir)
 VALIDATOR.option('builder', 'container_output_path', attrs={'default': '/tmp'},
-                 info='container output path', validator=validate_make_dir)
+                 info='container output path', validator=validation.make_dir)
 VALIDATOR.option('builder', 'container_archive',
                  attrs={'default': join_path(DEFAULT_BEE_WORKDIR, 'container_archive')},
                  info='container archive location')
@@ -417,11 +331,11 @@ def validate_chrun_opts(opts):
 # Slurmrestd (depends on DEFAULT:workload_scheduler == Slurm)
 VALIDATOR.section('slurm', info='Configuration section for Slurm.',
                   depends_on=('DEFAULT', 'workload_scheduler', 'Slurm'))
-VALIDATOR.option('slurm', 'use_commands', validator=validate_bool,
+VALIDATOR.option('slurm', 'use_commands', validator=validation.bool_,
                  attrs={'default': shutil.which('slurmrestd') is None},
                  info='if set, use slurm cli commands instead of slurmrestd')
 DEFAULT_SLURMRESTD_SOCK = join_path('/tmp', f'slurm_{USER}_{random.randint(1, 10000)}.sock')
-VALIDATOR.option('slurm', 'slurmrestd_socket', validator=validate_parent_dir,
+VALIDATOR.option('slurm', 'slurmrestd_socket', validator=validation.parent_dir,
                  attrs={'default': DEFAULT_SLURMRESTD_SOCK},
                  info='socket location')
 VALIDATOR.option('slurm', 'openapi_version', attrs={'default': 'v0.0.37'},
@@ -431,7 +345,7 @@ def validate_chrun_opts(opts):
 VALIDATOR.option('scheduler', 'log',
                  attrs={'default': join_path(DEFAULT_BEE_WORKDIR, 'logs', 'scheduler.log')},
                  info='scheduler log file')
-VALIDATOR.option('scheduler', 'socket', validator=validate_parent_dir,
+VALIDATOR.option('scheduler', 'socket', validator=validation.parent_dir,
                  attrs={'default': DEFAULT_SCHED_SOCKET},
                  info='scheduler socket')
 VALIDATOR.option('scheduler', 'alloc_logfile',
@@ -565,8 +479,6 @@ def save(self):
         print('Before running BEE, check defaults in the configuration file:',
               f'\n\t{self.fname}',
               '\n ** See documentation for values you should refrain from editing! **',
-              '\n\nThe job template configured is:',
-              f'\n\t{self.sections["task_manager"]["job_template"]}',
               '\n ** Include job options (such as account) required for this system.**')
         print('\n(Try `beecfg info` to see more about each option)')
         print(70 * '#')
diff --git a/beeflow/common/validation.py b/beeflow/common/validation.py
new file mode 100644
index 000000000..d95404e67
--- /dev/null
+++ b/beeflow/common/validation.py
@@ -0,0 +1,63 @@
+"""Specialized functions for validation and config initialization."""
+import os
+from pathlib import Path
+
+
+def validate_path(path):
+    """Check that the path exists."""
+    path = os.path.expanduser(path)
+    if not os.path.exists(path):
+        raise ValueError(f'path "{path}" does not exist')
+    return path
+
+
+def dir_(path):
+    """Check that the path exists and is a directory."""
+    path = validate_path(path)
+    if not os.path.isdir(path):
+        raise ValueError('path "{path}" is not a directory')
+    return path
+
+
+def make_dir(path):
+    """Check if the dir exists and if not create it."""
+    Path(path).mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def parent_dir(path):
+    """Ensure that the parent dir of path exists, or create it."""
+    make_dir(Path(path).parent)
+    return path
+
+
+def file_(path):
+    """Check that the path exists and is a file."""
+    path = validate_path(path)
+    if not os.path.isfile(path):
+        raise ValueError(f'path "{path}" is not a file')
+    return path
+
+
+def nonnegative_int(value):
+    """Validate that the input is nonnegative."""
+    i = int(value)
+    if i < 0:
+        raise ValueError('the value must be nonnegative')
+    return i
+
+
+# NOTE: You must use validate_bool for all boolean values (since just using
+#       bool, as in bool('False'), gives True for any string of length > 0)
+def bool_(value):
+    """Validate a boolean value."""
+    return str(value).lower() == 'true'
+
+
+def time_limit(value):
+    """Validate a time limit entry."""
+    value = value.strip()
+    if not value:
+        return ''
+    _ = [int(part) for part in value.split(':')]
+    return value
diff --git a/beeflow/common/worker/slurm_worker.py b/beeflow/common/worker/slurm_worker.py
index 8457fa586..6855be4df 100644
--- a/beeflow/common/worker/slurm_worker.py
+++ b/beeflow/common/worker/slurm_worker.py
@@ -12,6 +12,7 @@
 
 from beeflow.common import log as bee_logging
 from beeflow.common.worker.worker import (Worker, WorkerError)
+from beeflow.common import validation
 
 log = bee_logging.setup(__name__)
 
@@ -19,11 +20,16 @@
 class BaseSlurmWorker(Worker):
     """Base slurm worker code."""
 
+    def __init__(self, default_account='', default_time_limit='', **kwargs):
+        """Initialize the base slurm worker."""
+        super().__init__(**kwargs)
+        self.default_account = default_account
+        self.default_time_limit = default_time_limit
+
     def build_text(self, task):
-        """Build text for task script; use template if it exists."""
+        """Build text for task script."""
         task_save_path = self.task_save_path(task)
         crt_res = self.crt.run_text(task)
-        requirements = dict(task.requirements)
         stdout_param = ['--output', task.stdout]
         stderr_param = ['--error', task.stderr]
         if task.stdout and task.stderr:
@@ -36,24 +42,54 @@ def build_text(self, task):
             main_command_srun_args = []
         nodes = task.get_requirement('beeflow:MPIRequirement', 'nodes', default=1)
         ntasks = task.get_requirement('beeflow:MPIRequirement', 'ntasks', default=1)
-
-        job_text = self.template.render(
-            task_save_path=task_save_path,
-            task_name=task.name,
-            task_id=task.id,
-            workflow_id=task.workflow_id,
-            env_code=crt_res.env_code,
-            pre_commands=crt_res.pre_commands,
-            main_command=crt_res.main_command,
-            post_commands=crt_res.post_commands,
-            requirements=requirements,
-            nodes=nodes,
-            ntasks=ntasks,
-            main_command_srun_args=main_command_srun_args,
-            # Default MPI version
-            mpi_version='pmi2',
-        )
-        return job_text
+        mpi_version = task.get_requirement('beeflow:MPIRequirement', 'mpiVersion', default='pmi2')
+        time_limit = task.get_requirement('beeflow:SchedulerRequirement', 'timeLimit',
+                                          default=self.default_time_limit)
+        time_limit = validation.time_limit(time_limit)
+        account = task.get_requirement('beeflow:SchedulerRequirement', 'account',
+                                       default=self.default_account)
+
+        # sbatch header
+        script = [
+            '#!/bin/bash',
+            f'#SBATCH --job-name={task.name}-{task.id}',
+            f'#SBATCH --output={task_save_path}/{task.name}-{task.id}.out',
+            f'#SBATCH --error={task_save_path}/{task.name}-{task.id}.err',
+            f'#SBATCH -N {nodes}',
+            f'#SBATCH -n {ntasks}',
+            '#SBATCH --open-mode=append',
+        ]
+        if time_limit:
+            script.append(f'#SBATCH --time={time_limit}')
+        if account:
+            script.append(f'#SBATCH -A {account}')
+
+        # Return immediately on error
+        script.append('set -e')
+        script.append(crt_res.env_code)
+
+        def srun(script_lines, script_cmd):
+            """Wrap a pre or post command with srun."""
+            cmd_args = ' '.join(script_cmd.args)
+            if script_cmd.type == 'one-per-node':
+                script.append(f'srun -N {nodes} -n {nodes} {cmd_args}')
+            else:
+                script_lines.append(f'srun {cmd_args}')
+
+        # Pre commands
+        for cmd in crt_res.pre_commands:
+            srun(script, cmd)
+
+        # Main command
+        srun_args = ' '.join(main_command_srun_args)
+        print(crt_res.main_command)
+        args = ' '.join(crt_res.main_command.args)
+        script.append(f'srun --mpi={mpi_version} {srun_args} {args}')
+
+        for cmd in crt_res.post_commands:
+            srun(script, cmd)
+
+        return '\n'.join(script)
 
     def write_script(self, task):
         """Build task script; returns filename of script."""
@@ -87,7 +123,7 @@ class SlurmrestdWorker(BaseSlurmWorker):
 
     def __init__(self, bee_workdir, openapi_version, **kwargs):
         """Create a new Slurmrestd Worker object."""
-        super().__init__(bee_workdir, **kwargs)
+        super().__init__(bee_workdir=bee_workdir, **kwargs)
         # Pull slurm socket configs from kwargs (Uses getpass.getuser() instead
         # of os.getlogin() because of an issue with using getlogin() without a
         # controlling terminal)
diff --git a/beeflow/common/worker/worker.py b/beeflow/common/worker/worker.py
index 0103f7319..aa89de7ca 100644
--- a/beeflow/common/worker/worker.py
+++ b/beeflow/common/worker/worker.py
@@ -2,7 +2,6 @@
 
 from abc import ABC, abstractmethod
 import os
-import jinja2
 from beeflow.common import log as bee_logging
 from beeflow.common.crt_interface import ContainerRuntimeInterface
 
@@ -51,29 +50,6 @@ def __init__(self, bee_workdir, **kwargs):
         # Get BEE workdir from config file
         self.workdir = bee_workdir
 
-        # Get template for job
-        self.job_template = kwargs['job_template']
-        if self.job_template:
-            # Make sure that the file exists and is readable
-            try:
-                with open(self.job_template, 'r', encoding='UTF-8') as template_file:
-                    template_file.read()
-                log.info(f'Jobs will use template: {self.job_template}')
-            except ValueError as error:
-                raise RuntimeError(f'Cannot open {self.job_template}, {error}') from error
-            except FileNotFoundError as exc:
-                raise RuntimeError(f'Cannot find job template {self.job_template}') from exc
-            except PermissionError as error:
-                raise RuntimeError(f'Permission error {self.job_template}') from error
-        else:
-            raise RuntimeError('No job_template found for the worker class')
-
-    @property
-    def template(self):
-        """Load the template file."""
-        with open(self.job_template, encoding='UTF-8') as fp:
-            return jinja2.Template(fp.read())
-
     def task_save_path(self, task):
         """Return the task save path used for storing submission scripts output logs."""
         return f'{self.workdir}/workflows/{task.workflow_id}/{task.name}-{task.id}'
@@ -85,7 +61,7 @@ def prepare(self, task):
 
     @abstractmethod
     def build_text(self, task):
-        """Build text for task script; use template if it exists.
+        """Build text for task script.
 
         :param task: task that we're building a script for
         :type task: Task
diff --git a/beeflow/data/cwl/bee_workflows/clamr-wf-checkpoint/Dockerfile.clamr-ffmpeg b/beeflow/data/cwl/bee_workflows/clamr-wf-checkpoint/Dockerfile.clamr-ffmpeg
new file mode 100644
index 000000000..be993ba18
--- /dev/null
+++ b/beeflow/data/cwl/bee_workflows/clamr-wf-checkpoint/Dockerfile.clamr-ffmpeg
@@ -0,0 +1,12 @@
+# Dockerfile.clamr-ffmpeg
+# Developed on Chicoma @lanl
+# Patricia Grubel <pagrubel@lanl.gov>
+
+FROM debian:11
+
+
+RUN apt-get update && \
+    apt-get install -y wget gnupg git cmake ffmpeg g++ make openmpi-bin libopenmpi-dev libpng-dev libpng16-16 libpng-tools imagemagick libmagickwand-6.q16-6 libmagickwand-6.q16-dev
+
+RUN git clone https://github.com/lanl/CLAMR.git
+RUN cd CLAMR && cmake . && make clamr_cpuonly
diff --git a/beeflow/data/cwl/bee_workflows/clamr-wf-checkpoint/clamr.cwl b/beeflow/data/cwl/bee_workflows/clamr-wf-checkpoint/clamr.cwl
index 0a816a477..e94aa4af6 100644
--- a/beeflow/data/cwl/bee_workflows/clamr-wf-checkpoint/clamr.cwl
+++ b/beeflow/data/cwl/bee_workflows/clamr-wf-checkpoint/clamr.cwl
@@ -3,7 +3,7 @@
 class: CommandLineTool
 cwlVersion: v1.0
 
-baseCommand: /clamr/CLAMR-master/clamr_cpuonly
+baseCommand: /CLAMR/clamr_cpuonly
 # This is the stdout field which makes all stdout be captured in this file
 # stderr is not currently implemented but it is also a thing
 stdout: clamr_stdout.txt
diff --git a/beeflow/data/cwl/bee_workflows/clamr-wf-checkpoint/clamr_wf.cwl b/beeflow/data/cwl/bee_workflows/clamr-wf-checkpoint/clamr_wf.cwl
index af8601424..100945b93 100644
--- a/beeflow/data/cwl/bee_workflows/clamr-wf-checkpoint/clamr_wf.cwl
+++ b/beeflow/data/cwl/bee_workflows/clamr-wf-checkpoint/clamr_wf.cwl
@@ -53,11 +53,6 @@ steps:
       checkpoint_disk_interval: checkpoint_disk_interval
     out: [clamr_stdout, outdir, checkpoint_dir, time_log]
     hints:
-        DockerRequirement:
-            # TODO Sort this out
-            #dockerImport: clamr_img.tar.gz
-            #beeflow:copyContainer: clamr
-            beeflow:copyContainer: "/usr/projects/beedev/clamr/clamr-toss.tar.gz"
         beeflow:CheckpointRequirement:
             enabled: true
             file_path: checkpoint_output
@@ -65,7 +60,11 @@ steps:
             file_regex: backup[0-9]*.crx 
             restart_parameters: -R
             num_tries: 3
-            
+        beeflow:SchedulerRequirement:
+            timeLimit: 00:00:10
+        DockerRequirement:
+            dockerFile: "Dockerfile.clamr-ffmpeg"
+            beeflow:containerName: "clamr-ffmpeg"
   ffmpeg:
     run: ffmpeg.cwl
     in:
@@ -82,3 +81,7 @@ steps:
     out: [movie]
     requirements:
         InlineJavascriptRequirement: {}
+    hints:
+        DockerRequirement:
+            dockerFile: "Dockerfile.clamr-ffmpeg"
+            beeflow:containerName: "clamr-ffmpeg"
diff --git a/beeflow/data/job_templates/README.md b/beeflow/data/job_templates/README.md
deleted file mode 100644
index 2ee59c3cd..000000000
--- a/beeflow/data/job_templates/README.md
+++ /dev/null
@@ -1,30 +0,0 @@
-# Job templates
-
-BEE uses HPC resource scheduler templates to execute tasks on HPC resources. This directory is a home for templates of supported system schedulers. Add the absolute path to your template to the bee.conf file in the TaskManager section.
-Example:
-```
-[task_manager]
-...
-job_template = <home dir>/.config/beeflow/submit.jinja
-```
-
-## Jinja2 Templating
-
-The BEE worker code should now work with Jinja2-based templates set with the
-`job_template` parameter. An example job template `slurm-submit.jinja` is included
-here which should work for Slurm.
-
-Currently the template is passed a number of variables which I'll try to
-document here:
-* `task_save_path` - path for stored task info as a string
-* `task_name` - task name sring
-* `task_id` - task ID string
-* `workflow_id` - workflow ID string
-* `env_code` - commands to set environment such as 'module load charliecloud'
-* `pre_commands` - initial commands to run before the main command
-* `command` - main command to run
-* `post_commands` - commands to run after the main command
-* `requirements` - requirements as a dict
-* `hints` - hints as a dict
-These should be used to determine exactly what options should be generated in
-the submission script.
diff --git a/beeflow/data/job_templates/lsf-submit.jinja b/beeflow/data/job_templates/lsf-submit.jinja
deleted file mode 100644
index d3adb40c1..000000000
--- a/beeflow/data/job_templates/lsf-submit.jinja
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/bash
-#BSUB -P <account> 
-#BSUB -W 0:30
-#BSUB -J {{task_name}}-{{task_id}}
-#BSUB -o {{task_save_path}}/{{task_name}}-{{task_id}}.out
-#BSUB -e {{task_save_path}}/{{task_name}}-{{task_id}}.err
-#BSUB -nnodes 1
-{{ env_code }}
-
-# pre commands
-{% for cmd in pre_commands %}
-{{ cmd.args|join(' ') }}
-{% endfor %}
-
-# main command
-{{ main_command.args|join(' ') }}
-
-# post commands
-{% for cmd in post_commands %}
-{{ cmd.args|join(' ') }}
-{% endfor %}
-
diff --git a/beeflow/data/job_templates/slurm-submit.jinja b/beeflow/data/job_templates/slurm-submit.jinja
deleted file mode 100644
index 6539f3d9f..000000000
--- a/beeflow/data/job_templates/slurm-submit.jinja
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name={{task_name}}-{{task_id}}
-#SBATCH --output={{task_save_path}}/{{task_name}}-{{task_id}}.out
-#SBATCH --error={{task_save_path}}/{{task_name}}-{{task_id}}.err
-#SBATCH -N {{ nodes }}
-#SBATCH -n {{ ntasks }}
-
-{# Macro around srun to make it easier to emit commands #}
-{% macro srun(cmd) %}
-{% if cmd.type == 'one-per-node' %}
-# Set both -N and -n to the same value to run one-per-node
-srun -N {{ nodes }} -n {{ nodes }} {{ cmd.args|join(' ') }}
-{% else %}
-srun {{ cmd.args|join(' ') }}
-{% endif %}
-{% endmacro %}
-
-{{ env_code }}
-
-# pre commands
-{% for cmd in pre_commands %}
-{{ srun(cmd) }}
-{% endfor %}
-
-# main command
-srun --mpi={{ mpi_version }} {{ main_command_srun_args|join(' ') }} {{ main_command.args|join(' ') }}
-
-# post commands
-{% for cmd in post_commands %}
-{{ srun(cmd) }}
-{% endfor %}
diff --git a/beeflow/task_manager.py b/beeflow/task_manager.py
index 0c943d62d..79945c1f9 100755
--- a/beeflow/task_manager.py
+++ b/beeflow/task_manager.py
@@ -209,9 +209,11 @@ def update_jobs():
         # If state changes update the WFM
         if job_state != new_job_state:
             db.job_queue.update_job_state(id_, new_job_state)
-            if job_state in ('FAILED', 'TIMELIMIT', 'TIMEOUT'):
+            if new_job_state in ('FAILED', 'TIMELIMIT', 'TIMEOUT'):
                 # Harvest lastest checkpoint file.
                 task_checkpoint = get_task_checkpoint(task)
+                log.info(f'state: {new_job_state}')
+                log.info(f'TIMELIMIT/TIMEOUT task_checkpoint: {task_checkpoint}')
                 if task_checkpoint:
                     checkpoint_file = get_restart_file(task_checkpoint, task.workdir)
                     task_info = {'checkpoint_file': checkpoint_file, 'restart': True}
@@ -307,9 +309,10 @@ def get_status():
 worker_kwargs = {
     'bee_workdir': bc.get('DEFAULT', 'bee_workdir'),
     'container_runtime': bc.get('task_manager', 'container_runtime'),
-    'job_template': bc.get('task_manager', 'job_template'),
     # extra options to be passed to the runner (i.e. srun [RUNNER_OPTS] ... for Slurm)
     'runner_opts': bc.get('task_manager', 'runner_opts'),
+    'default_account': bc.get('job', 'default_account'),
+    'default_time_limit': bc.get('job', 'default_time_limit'),
 }
 if WLS == 'Slurm':
     worker_kwargs['use_commands'] = bc.get('slurm', 'use_commands')
diff --git a/beeflow/wf_manager/resources/wf_update.py b/beeflow/wf_manager/resources/wf_update.py
index b86945766..6f0f9c5ee 100644
--- a/beeflow/wf_manager/resources/wf_update.py
+++ b/beeflow/wf_manager/resources/wf_update.py
@@ -96,6 +96,7 @@ def put(self):
             task_info = jsonpickle.decode(data['task_info'])
             checkpoint_file = task_info['checkpoint_file']
             new_task = wfi.restart_task(task, checkpoint_file)
+            db.workflows.add_task(new_task.id, wf_id, new_task.name, "WAITING")
             if new_task is None:
                 log.info('No more restarts')
                 state = wfi.get_task_state(task)
@@ -125,6 +126,6 @@ def put(self):
         resp = make_response(jsonify(status=(f'Task {task_id} belonging to WF {wf_id} set to'
                                              f'{job_state}')), 200)
         return resp
-# Ignoring C901: "'WFUPdate.put' is too complex" - this requires a refactor (or
-#                maybe the LOC limit is too low)
-# pylama:ignore=C901
+# Ignoring C901,R0915: "'WFUPdate.put' is too complex" - this requires a refactor
+#                      (or maybe the LOC limit is too low)
+# pylama:ignore=C901,R0915
diff --git a/ci/bee_install.sh b/ci/bee_install.sh
index 5b589c5cc..df7def5fd 100755
--- a/ci/bee_install.sh
+++ b/ci/bee_install.sh
@@ -28,8 +28,6 @@ poetry install -E cloud_extras || exit 1
 
 # BEE Configuration
 mkdir -p ~/.config/beeflow
-JOB_TEMPLATE=$HOME/.config/beeflow/submit.jinja
-cp beeflow/data/job_templates/slurm-submit.jinja $JOB_TEMPLATE
 cat >> ~/.config/beeflow/bee.conf <<EOF
 # BEE CONFIGURATION FILE #
 [DEFAULT]
@@ -44,14 +42,17 @@ max_restarts = 2
 [task_manager]
 socket = $HOME/tm.sock
 container_runtime = Charliecloud
-job_template = $JOB_TEMPLATE
 runner_opts =
 
 [charliecloud]
 image_mntdir = /tmp
-chrun_opts =
+chrun_opts = --home
 setup =
 
+[job]
+default_account =
+default_time_limit =
+
 [graphdb]
 hostname = localhost
 dbpass = password
diff --git a/ci/deps_install.sh b/ci/deps_install.sh
index 9f4dc7c54..26b657f72 100755
--- a/ci/deps_install.sh
+++ b/ci/deps_install.sh
@@ -9,9 +9,9 @@ sudo apt-get install -y slurmctld slurmd slurmrestd munge python3 python3-venv \
     libyaml-dev # needed for PyYAML
 
 # Install most recent Charliecloud
-curl -O -L https://github.com/hpc/charliecloud/releases/download/v0.27/charliecloud-0.27.tar.gz
-tar -xvf charliecloud-0.27.tar.gz
-(cd charliecloud-0.27
+curl -O -L https://github.com/hpc/charliecloud/releases/download/v0.32/charliecloud-0.32.tar.gz
+tar -xvf charliecloud-0.32.tar.gz
+(cd charliecloud-0.32
  ./configure --prefix=/usr
  make
  sudo make install)
diff --git a/docs/sphinx/advanced_usage.rst b/docs/sphinx/advanced_usage.rst
index 33809c2b2..b99158aa9 100644
--- a/docs/sphinx/advanced_usage.rst
+++ b/docs/sphinx/advanced_usage.rst
@@ -14,9 +14,10 @@ setup/installation issues). Note, to allow for better provenance, this output
 is also saved when archiving a workflow. This is also one reason why the
 specific path below should not be changed unless you know what you're doing.
 
-When using the default template (see `Jinja file`_), outputs from the batch
-script will go to files in
-``$bee_workdir/workflows/$workflow_id/$task_name-$task_id``:
+By default, output from job, if not captured by an ``stdout`` or an ``stderr``
+option within your workflow CWL will be directed to
+``$bee_workdir/workflows/$workflow_id/$task_name-$task_id``,
+with the meaning of each component listed below:
 
 ``$bee_workdir``
     BEE's workdir, which is set in the bee.conf file and by default is
@@ -36,93 +37,6 @@ to examine these output files. Sometimes the error may have to do with the
 environment or with a task's requirements. If you're unable to find the cause
 of the problem, then you should contact the BEE developers.
 
-.. _Jinja file:
-
-Jinja File Templating
----------------------
-
-When BEE launches a step of a workflow on an HPC system, the step and its
-metadata are given as input to a special template file. The generated output is
-used as the submission script for Slurm or LSF. While this allows a good deal
-of flexibility for different types of HPC jobs, it also makes it difficult to
-use. Currently, this section explains how it works and how you can configure
-the template file for the needs of your workflow, but please note that we're
-also exploring different ways to do this, so this functionality may change in
-future releases.
-
-These templates use the Jinja2_ templating library. While originally designed
-for templating HTML, it can also be used for generating any text file. It also
-has a simple Python-like syntax which should make it somewhat easy to work with
-for those already familiar with the language.
-
-.. _Jinja2: https://jinja.palletsprojects.com/en/3.1.x/
-
-See the `Jinja Template Documentation`_ for more information on the templating
-language and how to use it. There are also some example Jinja Files in
-``beeflow/data/job_templates``.
-
-
-.. _Jinja Template Documentation: https://jinja.palletsprojects.com/en/3.1.x/templates/
-
-Here is a small example of a Jinja submit file for Slurm::
-
-    #!/bin/bash
-    #SBATCH --job-name={{task_name}}-{{task_id}}
-    #SBATCH --output={{task_save_path}}/{{task_name}}-{{task_id}}.out
-    #SBATCH --error={{task_save_path}}/{{task_name}}-{{task_id}}.err
-    {% if 'beeflow:MPIRequirement' in hints and 'nodes' in hints['beeflow:MPIRequirement'] %}
-    #SBATCH -N {{ hints['beeflow:MPIRequirement']['nodes'] }}
-    {% endif %}
-    {% if 'beeflow:MPIRequirement' in hints and 'ntasks' in hints['beeflow:MPIRequirement'] %}
-    #SBATCH -n {{ hints['beeflow:MPIRequirement']['ntasks'] }}
-    {% endif %}
-
-    {{ env_code }}
-
-    # pre commands
-    {% for cmd in pre_commands %}
-    srun {{ cmd|join(' ') }}
-    {% endfor %}
-
-    # main command
-    srun {{ main_command|join(' ') }}
-
-    # post commands
-    {% for cmd in post_commands %}
-    srun {{ cmd|join(' ') }}
-    {% endfor %}
-
-By default when you run ``beecfg new``, a default template file will be
-generated for you, not unlike the one above.  The default template for Slurm
-accepts the number of nodes and the number of tasks and submits corresponding
-``#SBATCH`` directives for the job. You may also add other ``#SBATCH`` (or for
-LSF ``#BSUB``) directives to your jinja file, to use a specific partition or
-for accounting purposes.
-
-This default template should work fine for most workflows, so you really don't
-need to worry about editing it unless you need something extra. The important
-parts to note above, are where the template code is generating the ``#SBATCH``
-directives by checking the contents of the ``hints`` variable and the code
-under the ``# main command`` comment which is where the command for a step will
-be added. If you need to use a specific MPI type, then you may want to add
-``--mpi={type}`` on the line under the ``main command`` comment. Or if you need
-something extra from the scheduler, then you may want to add it to the
-directives, or you can add the option on the main command itself.
-
-If you only need to apply some particular option to one step of a workflow,
-then you'll need to use Jinja's if_ construct to handle the case for that
-particular step and then for the other steps. There are also some other
-constructs, such as for_ loops, which may be useful for more complicated
-workflows. The good thing is that most of these behave and act like normal
-Python code, except being delimited by ``{% .. %}``.
-
-.. _if: https://jinja.palletsprojects.com/en/3.1.x/templates/#if
-.. _for: https://jinja.palletsprojects.com/en/3.1.x/templates/#for
-
-Check the bee configuration file (bee.conf) or type ``beecfg show`` for the
-current location of your job_template. If you need to, edit it for your
-particular needs.
-
 .. _Multiple Workflows:
 
 Multiple Workflows
diff --git a/docs/sphinx/bee_cwl.rst b/docs/sphinx/bee_cwl.rst
index 0172130d0..98ee6c917 100644
--- a/docs/sphinx/bee_cwl.rst
+++ b/docs/sphinx/bee_cwl.rst
@@ -97,6 +97,9 @@ Suboption Name            Usage/Meaning/Requirements
 
 ``beeflow:containerName`` ``beeflow:containerName: "containerName"`` |vspace| |br|
                           Specifies the container name. Used in conjunction with dockerFile.
+``beeflow:forceType``     ``beeflow:forceType: "forceType"`` |vspace| |br|
+                          Charliecloud specific option that |vspace| |br|
+                          corresponds to ``ch-image``'s ``--force`` argument.
 ========================= =========================================================
 
 beeflow:MPIRequirement
@@ -114,14 +117,8 @@ option).
 .. _paper: https://ieeexplore.ieee.org/document/9308116
 
 BEE's ``beeflow:MPIRequirement`` attempts to be as configurable as possible for
-running HPC jobs with MPI. Currently, we use a special :ref:`Jinja file` that is
-used as a template for generating submission scripts. All options in the
-``beeflow:MPIRequirement`` get passed to the :ref:`Jinja file` template and can
-then be used to make step-level decisions for running MPI jobs. Note that we're
-also exploring other ways to implement this directly from the options within
-the CWL files, instead of using a template file.
-
-An example ``beeflow:MPIRequirement`` in BEE is shown below::
+running HPC jobs with MPI. An example ``beeflow:MPIRequirement`` in BEE is
+shown below::
 
     beeflow:MPIRequirement:
       nodes: 10
@@ -131,8 +128,6 @@ The values for ``nodes`` and  ``ntasks`` are then passed to the template and
 can be used to request the required resources from the underlying scheduler on
 submission.
 
-See our section on the :ref:`Jinja file` for more info on how this works currently.
-
 beeflow:CheckpointRequirement
 -----------------------------
 
@@ -158,3 +153,19 @@ For the above example ``file_path`` is the location of the checkpoint_file. The
 filenames, the ``restart parameter`` will be added to the run command followed
 by the path to the latest checkpoint file, and ``num_tries`` specifies the maximum
 number of times the task will be restarted.
+
+beeflow:SchedulerRequirement
+----------------------------
+
+This requirement is designed for specifying additional information that will be
+passed to a scheduler such as Slurm on job submission. It currently supports
+the ``timeLimit`` option and an optional ``account`` field which may be useful
+if running jobs with different accounts (if you want to run all workflows with
+the same account it's best to set this with the ``default_account`` option
+under the ``job`` section in the bee.conf file).
+
+An example is shown below::
+
+    beeflow:SchedulerRequirement:
+      timeLimit: 00:00:10
+      account: account12345
diff --git a/docs/sphinx/installation.rst b/docs/sphinx/installation.rst
index 011c13aa6..29ccf97e6 100644
--- a/docs/sphinx/installation.rst
+++ b/docs/sphinx/installation.rst
@@ -62,23 +62,20 @@ You will need to setup the bee configuration file that will be located in:
 Before creating a bee.conf file you will need to know the path to your **BEE
 dependency container** and the type of workload scheduler (Slurm or LSF). (On
 LANL systems you may use the BEE provided container:
-**/usr/projects/BEE/neo4j-3-5-17-ch.tar.gz**).
+**/usr/projects/BEE/neo4j-3-5-17-ch.tar.gz**). Depending on the system, you
+may also need to know an account name to use.
 
-Once you are ready type ``beecfg new``
+Once you are ready type ``beecfg new``.
 
 The bee.conf configuration file is a text file and you can edit it for your
-needs. You may show the contents by typing ``beecfg show``. You may also want
-to take a look at the :ref:`Jinja file`, which is how submission scripts are
-generated. The default file is listed with the ``job_template`` option under
-the ``[task_manager]`` section.
-
+needs.
 
 **Caution: The default for container_archive is in the home directory. Some
 systems have small quotas for home directories and containers can be large
 files.**
 
 **beecfg** has other options including a configuration validator. For more
-information or help run: ``beecfg info`` or ``beecfg --help``
+information or help run: ``beecfg info`` or ``beecfg --help``.
 
 Starting up the BEE components:
 -------------------------------


Test patch:
diff --git a/beeflow/tests/test_slurm_worker.py b/beeflow/tests/test_slurm_worker.py
index 79b17d2cf..0767442c4 100644
--- a/beeflow/tests/test_slurm_worker.py
+++ b/beeflow/tests/test_slurm_worker.py
@@ -54,7 +54,6 @@ def slurm_worker(request):
     worker_iface = WorkerInterface(worker=SlurmWorker, container_runtime='Charliecloud',
                                    slurm_socket=slurm_socket, bee_workdir=bee_workdir,
                                    openapi_version=OPENAPI_VERSION,
-                                   job_template=bc.get('task_manager', 'job_template'),
                                    use_commands=request.param)
     yield worker_iface
     time.sleep(1)
@@ -70,7 +69,6 @@ def slurmrestd_worker_no_daemon():
     yield WorkerInterface(worker=SlurmWorker, container_runtime='Charliecloud',
                           slurm_socket=slurm_socket, bee_workdir=bee_workdir,
                           openapi_version=OPENAPI_VERSION,
-                          job_template=bc.get('task_manager', 'job_template'),
                           use_commands=False)
 
 
diff --git a/ci/integration_test.py b/ci/integration_test.py
index a75f0cc1a..046a08376 100755
--- a/ci/integration_test.py
+++ b/ci/integration_test.py
@@ -18,6 +18,7 @@
 
 # Max time of each workflow
 TIMEOUT = 90
+INTEGRATION_TEST_DIR = os.path.expanduser('~/.beeflow-integration')
 
 
 class CIError(Exception):
@@ -43,7 +44,7 @@ def build(self):
         if not self.done:
             try:
                 subprocess.check_call(['ch-image', 'build', '-f', self.dockerfile, '-t', self.name,
-                                       '--force', os.path.dirname(self.dockerfile)])
+                                       '--force', 'seccomp', os.path.dirname(self.dockerfile)])
                 subprocess.check_call(['ch-convert', '-i', 'ch-image', '-o', 'tar', self.name,
                                        self.tarball])
             except subprocess.CalledProcessError as error:
@@ -106,52 +107,62 @@ class TestRunner:
     def __init__(self):
         """Build a new test runner class."""
         self.test_cases = []
+        self.timeout = TIMEOUT
 
-    def add(self, test_case):
+    def add(self, ignore=False):
         """Decorate a test case and add it to the runner instance."""
-        # First turn it into a contextmanager
-        test_case = contextmanager(test_case)
-        self.test_cases.append(test_case)
 
-        def wrap():
+        def wrap(test_case):
             """Wrap the function."""
-            return test_case()
+            # First turn it into a contextmanager
+            test_case = contextmanager(test_case)
+            self.test_cases.append((test_case, ignore))
+            return test_case
 
         return wrap
 
-    def test_names(self):
-        """Return a list of all the test names."""
-        return [test_case.__name__ for test_case in self.test_cases]
+    def test_details(self):
+        """Return a list of all the test details (test_name, ignore)."""
+        return [(test_case.__name__, ignore) for test_case, ignore in self.test_cases]
 
     def _run_workflows(self, workflows):
         """Run a list of workflows to completion."""
         # Start all workflows at once
         for wfl in workflows:
             wfl.run()
-        # Now run until all workflows are complete or until TIMEOUT is hit
+        # Now run until all workflows are complete or until self.timeout is hit
         t = 0
-        while t < TIMEOUT:
+        while t < self.timeout:
             if all(not wfl.running() for wfl in workflows):
                 # All workflows have completed
                 break
             time.sleep(2)
             t += 2
-        if t >= TIMEOUT:
+        if t >= self.timeout:
             raise CIError('workflow timeout')
 
     def run(self, test_names=None):
         """Test run all test cases."""
         results = {}
-        for test_case in self.test_cases:
-            # Skip tests that aren't required to be run
-            if test_names is not None and test_case.__name__ not in test_names:
+        for test_case, ignore in self.test_cases:
+            if test_names is not None:
+                # Skip tests that aren't required to be run
+                if test_case.__name__ not in test_names:
+                    continue
+            elif ignore:
+                # Skip ignored tests
                 continue
-            msg = f'Starting test {test_case.__name__}'
-            print('#' * len(msg))
-            print(msg)
-            print('-' * len(msg))
+            outer_workdir = os.path.join(INTEGRATION_TEST_DIR, uuid.uuid4().hex)
+            os.makedirs(outer_workdir)
+            msg0 = f'Starting test {test_case.__name__}'
+            msg1 = f'(running in "{outer_workdir}")'
+            count = max(len(msg0), len(msg1))
+            print('#' * count)
+            print(msg0)
+            print(msg1)
+            print('-' * count)
             try:
-                with test_case() as workflows:
+                with test_case(outer_workdir) as workflows:
                     self._run_workflows(workflows)
             except CIError as error:
                 traceback.print_exc()
@@ -164,6 +175,8 @@ def run(self, test_names=None):
                 print('------')
                 print('PASSED')
                 print('------')
+                # Only remove the outer_workdir if it passed
+                shutil.rmtree(outer_workdir)
 
         print('######## WORKFLOW RESULTS ########')
         fails = sum(1 if result is not None else 0 for name, result in results.items())
@@ -378,16 +391,16 @@ def generate_simple_workflow(output_path, fname):
     ch_image_delete(BASE_CONTAINER)
 
 
-@TEST_RUNNER.add
-def copy_container():
+@TEST_RUNNER.add()
+def copy_container(outer_workdir):
     """Prepare, check results of using `beeflow:copyContainer` then do cleanup."""
     # `beeflow:copyContainer` workflow
     container_path = f'/tmp/copy_container-{uuid.uuid4().hex}.tar.gz'
     container_name = 'copy-container'
-    workdir = os.path.expanduser(f'~/{uuid.uuid4().hex}')
+    workdir = os.path.join(outer_workdir, uuid.uuid4().hex)
     os.makedirs(workdir)
     container = Container(container_name, DOCKER_FILE_PATH, container_path)
-    workflow_path = os.path.join('/tmp', f'bee-cc-workflow-{uuid.uuid4().hex}')
+    workflow_path = os.path.join(outer_workdir, f'bee-cc-workflow-{uuid.uuid4().hex}')
     main_input = 'copy_container'
     docker_requirement = {
         'beeflow:copyContainer': container_path,
@@ -408,24 +421,19 @@ def copy_container():
     path = os.path.join(container_archive, basename)
     check_path_exists(path)
     os.remove(path)
-    ch_image_delete(BASE_CONTAINER)
     ch_image_delete(container_name)
-    # Delete the generated container
-    shutil.rmtree(workflow_path)
-    os.remove(container_path)
-    shutil.rmtree(workdir)
 
 
-@TEST_RUNNER.add
-def use_container():
+@TEST_RUNNER.add()
+def use_container(outer_workdir):
     """Prepare, check results of using `beeflow:useContainer` and clean up."""
     # `beeflow:useContainer` workflow
-    container_path = os.path.expanduser(f'~/use_container-{uuid.uuid4().hex}.tar.gz')
+    container_path = os.path.join(outer_workdir, f'use_container-{uuid.uuid4().hex}.tar.gz')
     container_name = 'use-container'
-    container_workdir = os.path.expanduser(f'~/{uuid.uuid4().hex}')
+    container_workdir = os.path.join(outer_workdir, uuid.uuid4().hex)
     os.makedirs(container_workdir)
     container = Container(container_name, DOCKER_FILE_PATH, container_path)
-    workflow_path = os.path.join('/tmp', f'bee-use-ctr-workflow-{uuid.uuid4().hex}')
+    workflow_path = os.path.join(outer_workdir, f'bee-use-ctr-workflow-{uuid.uuid4().hex}')
     main_input = 'use_ctr'
     docker_requirement = {
         'beeflow:useContainer': container_path,
@@ -441,7 +449,6 @@ def use_container():
     yield [workflow]
     path = os.path.join(container_workdir, main_input)
     check_path_exists(path)
-    os.remove(path)
     # This container should not have been copied into the container archive
     container_archive = bc.get('builder', 'container_archive')
     basename = os.path.basename(container_path)
@@ -450,24 +457,19 @@ def use_container():
         not path.exists(),
         f'the container "{basename}" was copied into the container archive'
     )
-    ch_image_delete(BASE_CONTAINER)
     ch_image_delete(container_name)
-    # Delete both the workflow path and the container generated
-    shutil.rmtree(workflow_path)
-    os.remove(container_path)
-    shutil.rmtree(container_workdir)
 
 
-@TEST_RUNNER.add
-def docker_file():
+@TEST_RUNNER.add()
+def docker_file(outer_workdir):
     """Ensure that the `dockerFile` example ran properly and then clean up."""
     # `dockerFile` workflow
-    workflow_path = os.path.join('/tmp', f'bee-df-workflow-{uuid.uuid4().hex}')
+    workflow_path = os.path.join(outer_workdir, f'bee-df-workflow-{uuid.uuid4().hex}')
     # Copy the Dockerfile to the workdir path
     os.makedirs(workflow_path)
-    shutil.copy(DOCKER_FILE_PATH, os.path.join(workflow_path, 'Dockerfile'))
-    workdir = os.path.expanduser(f'~/{uuid.uuid4().hex}')
+    workdir = os.path.join(outer_workdir, uuid.uuid4().hex)
     os.makedirs(workdir)
+    shutil.copy(DOCKER_FILE_PATH, os.path.join(workflow_path, 'Dockerfile'))
     container_name = 'docker_file_test'
     docker_requirement = {
         'dockerFile': 'Dockerfile',
@@ -485,7 +487,6 @@ def docker_file():
     yield [workflow]
     path = os.path.join(workdir, main_input)
     check_path_exists(path)
-    os.remove(path)
     # The container should have been copied into the archive
     container_archive = bc.get('builder', 'container_archive')
     tarball = f'{container_name}.tar.gz'
@@ -495,25 +496,21 @@ def docker_file():
     # Check that the container is listed
     images = ch_image_list()
     ci_assert(container_name in images, f'cannot find expected container "{container_name}"')
-    ch_image_delete(BASE_CONTAINER)
     ch_image_delete(container_name)
-    # Delete the generated workflow
-    shutil.rmtree(workflow_path)
-    shutil.rmtree(workdir)
 
 
-@TEST_RUNNER.add
-def docker_pull():
+@TEST_RUNNER.add()
+def docker_pull(outer_workdir):
     """Prepare, then check that the `dockerPull` option was successful."""
     # `dockerPull` workflow
-    workflow_path = os.path.join('/tmp', f'bee-dp-workflow-{uuid.uuid4().hex}')
+    workflow_path = os.path.join(outer_workdir, f'bee-dp-workflow-{uuid.uuid4().hex}')
+    workdir = os.path.join(outer_workdir, uuid.uuid4().hex)
+    os.makedirs(workdir)
     container_name = BASE_CONTAINER
     docker_requirement = {
         'dockerPull': container_name,
     }
     main_input = 'docker_pull'
-    workdir = os.path.expanduser(f'~/{uuid.uuid4().hex}')
-    os.makedirs(workdir)
     main_cwl, job_file = generate_builder_workflow(
         workflow_path,
         docker_requirement,
@@ -525,32 +522,29 @@ def docker_pull():
     yield [workflow]
     path = os.path.join(workdir, main_input)
     check_path_exists(path)
-    os.remove(path)
     # Check that the image tarball is in the archive
     container_archive = bc.get('builder', 'container_archive')
     path = os.path.join(container_archive, f'{container_name}.tar.gz')
     check_path_exists(path)
     os.remove(path)
+    # Commenting the below out for now; looks like Charliecloud 0.32 isn't
+    # showing base containers for some reason?
     # Check for the image with `ch-image list`
-    images = ch_image_list()
-    ci_assert(container_name in images, f'could not find expected container "{container_name}"')
-    ch_image_delete(container_name)
-    # Delete the workflow path
-    shutil.rmtree(workflow_path)
-    shutil.rmtree(workdir)
+    # images = ch_image_list()
+    # ci_assert(container_name in images, f'could not find expected container "{container_name}"')
+    # ch_image_delete(container_name)
 
 
-@TEST_RUNNER.add
-def multiple_workflows():
+@TEST_RUNNER.add()
+def multiple_workflows(outer_workdir):
     """Test running three different workflows at the same time."""
     output_file = 'output_file'
     workflow_data = []
     workflows = []
     for i in range(3):
-        random_dir = uuid.uuid4().hex
-        workdir = os.path.expanduser(f'~/{random_dir}')
+        workdir = os.path.join(outer_workdir, uuid.uuid4().hex)
         os.makedirs(workdir)
-        workflow_path = Path('/tmp', random_dir)
+        workflow_path = Path(outer_workdir, uuid.uuid4().hex)
         main_cwl, job_file = generate_simple_workflow(workflow_path, output_file)
         workflow = Workflow(f'multi-workflow-{i}', workflow_path,
                             main_cwl=main_cwl, job_file=job_file,
@@ -567,8 +561,21 @@ def multiple_workflows():
         # Each test should have touched this file
         path = Path(workdir, output_file)
         check_path_exists(path)
-        shutil.rmtree(workdir)
-        shutil.rmtree(workflow_path)
+
+
+@TEST_RUNNER.add(ignore=True)
+def checkpoint_restart(outer_workdir):
+    """Test the clamr-ffmpeg checkpoint restart workflow."""
+    workdir = os.path.join(outer_workdir, uuid.uuid4().hex)
+    os.makedirs(workdir)
+    workflow = Workflow('checkpoint-restart',
+                        'beeflow/data/cwl/bee_workflows/clamr-wf-checkpoint',
+                        main_cwl='clamr_wf.cwl', job_file='clamr_job.yml',
+                        workdir=workdir, containers=[])
+    yield [workflow]
+    # Check for the movie file
+    path = Path(workdir, 'CLAMR_movie.mp4')
+    check_path_exists(path)
 
 
 def test_input_callback(arg):
@@ -580,13 +587,15 @@ def main(tests = typer.Option(None, '--tests', '-t',  # noqa (conflict on '=' si
                               callback=test_input_callback,
                               help='tests run as comma-separated string'),
          show_tests: bool = typer.Option(False, '--show-tests', '-s',  # noqa (conflict on '=' sign)
-                                         help='show a list of all tests')):
+                                         help='show a list of all tests'),
+         timeout: int = typer.Option(TIMEOUT, '--timeout', help='workflow timeout in seconds')):
     """Launch the integration tests."""
     if show_tests:
         print('INTEGRATION TEST CASES:')
-        for test_name in TEST_RUNNER.test_names():
-            print('*', test_name)
-        sys.exit()
+        for test_name, ignore in TEST_RUNNER.test_details():
+            print('*', test_name, '(ignored)' if ignore else '')
+        return
+    TEST_RUNNER.timeout = timeout
     # Run the workflows and then show completion results
     ret = TEST_RUNNER.run(tests)
     # ret = test_workflows(WORKFLOWS)