Repository: reframe-hpc/reframe

Issue:
Support a dry-run mode in ReFrame
This mode can simply emit job scripts without executing them.

Discussion:

Patch:
diff --git a/docs/manpage.rst b/docs/manpage.rst
index 5b2147f0..13df9fa0 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -232,6 +232,20 @@ An action must always be specified.
    .. versionadded:: 3.10.0
 
 
+.. option:: --dry-run
+
+   Dry run the selected tests.
+
+   The dry-run mode will try to execute as much of the test pipeline as possible.
+   More specifically, the tests will not be submitted and will not be run for real,
+   but their stage directory will be prepared and the corresponding job script will be emitted.
+   Similarly, the sanity and performance functions will not be evaluated but all the preparation will happen.
+   Tests run in dry-run mode will not fail unless there is a programming error in the test or if the test tries to use a resource that is not produced in dry run mode (e.g., access the standard output or a resource produced by a dependency outside any sanity or performance function).
+   In this case, users can call the :func:`~reframe.core.pipeline.RegressionTest.is_dry_run` method in their test and take a specific action if the test is run in dry-run mode.
+
+   .. versionadded:: 4.1
+
+
 .. option:: -L, --list-detailed[=T|C]
 
    List selected tests providing more details for each test.
diff --git a/reframe/core/decorators.py b/reframe/core/decorators.py
index 0510ae16..0a41dfd8 100644
--- a/reframe/core/decorators.py
+++ b/reframe/core/decorators.py
@@ -16,6 +16,7 @@ import traceback
 
 import reframe.utility.osext as osext
 from reframe.core.exceptions import ReframeSyntaxError, SkipTestError, what
+from reframe.core.fields import make_convertible
 from reframe.core.fixtures import FixtureRegistry
 from reframe.core.logging import getlogger, time_function
 from reframe.core.pipeline import RegressionTest
@@ -25,6 +26,28 @@ from reframe.utility.versioning import VersionValidator
 # NOTE: we should consider renaming this module in 4.0; it practically takes
 # care of the registration and instantiation of the tests.
 
+def _setvars(registry, variables):
+    unset_vars = {}
+    for test in registry:
+        for name, val in variables.items():
+            if '.' in name:
+                testname, varname = name.split('.', maxsplit=1)
+            else:
+                testname, varname = test.__name__, name
+
+            if testname == test.__name__:
+                # Treat special values
+                if val == '@none':
+                    val = None
+                else:
+                    val = make_convertible(val)
+
+                if not test.setvar(varname, val):
+                    unset_vars.setdefault(test.__name__, [])
+                    unset_vars[test.__name__].append(varname)
+
+    return unset_vars
+
 
 class TestRegistry:
     '''Regression test registry.
@@ -52,8 +75,11 @@ class TestRegistry:
         self._tests.setdefault(test, [])
         self._tests[test].append((args, kwargs))
 
+    def setvars(self, variables):
+        return _setvars(self, variables)
+
     @time_function
-    def instantiate_all(self, reset_sysenv=0):
+    def instantiate_all(self, reset_sysenv=0, external_vars=None):
         '''Instantiate all the registered tests.
 
         :param reset_sysenv: Reset valid_systems and valid_prog_environs after
@@ -105,6 +131,9 @@ class TestRegistry:
 
             # Instantiate the new fixtures and update the registry
             new_fixtures = tmp_registry.difference(fixture_registry)
+            if external_vars:
+                _setvars(new_fixtures.uninst_tests(), external_vars)
+
             leaf_tests = new_fixtures.instantiate_all()
             fixture_registry.update(new_fixtures)
 
diff --git a/reframe/core/fixtures.py b/reframe/core/fixtures.py
index f02acbc6..247dfdbd 100644
--- a/reframe/core/fixtures.py
+++ b/reframe/core/fixtures.py
@@ -304,7 +304,6 @@ class FixtureRegistry:
                     'valid_systems': part,
                     **variables
                 }
-
                 try:
                     # Instantiate the fixture
                     inst = cls(variant_num=varnum, fixt_name=name,
@@ -322,6 +321,10 @@ class FixtureRegistry:
 
         return ret
 
+    def uninst_tests(self):
+        '''Get the uninstantiated tests of this registry'''
+        return self._registry.keys()
+
     def _filter_valid_partitions(self, candidate_parts):
         return [p for p in candidate_parts if p in self._env_by_part]
 
diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py
index a68fe7ff..f4c5fcd2 100644
--- a/reframe/core/pipeline.py
+++ b/reframe/core/pipeline.py
@@ -963,6 +963,11 @@ class RegressionTest(RegressionMixin, jsonext.JSONSerializable):
     #: :default: :class:`True`
     build_locally = variable(typ.Bool, value=True, loggable=True)
 
+    # Special variables
+
+    #: Dry-run mode
+    _rfm_dry_run = variable(typ.Bool, value=False)
+
     def __new__(cls, *args, **kwargs):
         obj = super().__new__(cls)
 
@@ -1514,6 +1519,14 @@ class RegressionTest(RegressionMixin, jsonext.JSONSerializable):
         '''Check if the test is a fixture.'''
         return getattr(self, '_rfm_is_fixture', False)
 
+    def is_dry_run(self):
+        '''Check if the test runs in dry-run mode.
+
+        .. versionadded:: 4.1
+
+        '''
+        return self._rfm_dry_run
+
     def _resolve_fixtures(self):
         '''Resolve the fixture dependencies and inject the fixture handle.
 
@@ -1814,7 +1827,8 @@ class RegressionTest(RegressionMixin, jsonext.JSONSerializable):
             except OSError as e:
                 raise PipelineError('failed to prepare build job') from e
 
-            self._build_job.submit()
+            if not self.is_dry_run():
+                self._build_job.submit()
 
     @final
     def compile_wait(self):
@@ -1836,6 +1850,9 @@ class RegressionTest(RegressionMixin, jsonext.JSONSerializable):
               more details.
 
         '''
+        if self.is_dry_run():
+            return
+
         self._build_job.wait()
 
         # We raise a BuildError when we an exit code and it is non zero
@@ -1955,9 +1972,9 @@ class RegressionTest(RegressionMixin, jsonext.JSONSerializable):
             except OSError as e:
                 raise PipelineError('failed to prepare run job') from e
 
-            self._job.submit()
-
-        self.logger.debug(f'Spawned run job (id={self.job.jobid})')
+            if not self.is_dry_run():
+                self._job.submit()
+                self.logger.debug(f'Spawned run job (id={self.job.jobid})')
 
         # Update num_tasks if test is flexible
         if self.job.sched_flex_alloc_nodes:
@@ -1982,7 +1999,7 @@ class RegressionTest(RegressionMixin, jsonext.JSONSerializable):
         :raises reframe.core.exceptions.ReframeError: In case of errors.
 
         '''
-        if not self._build_job:
+        if not self._build_job or self.is_dry_run():
             return True
 
         return self._build_job.finished()
@@ -2011,7 +2028,7 @@ class RegressionTest(RegressionMixin, jsonext.JSONSerializable):
               more details.
 
         '''
-        if not self._job:
+        if not self._job or self.is_dry_run():
             return True
 
         return self._job.finished()
@@ -2034,6 +2051,9 @@ class RegressionTest(RegressionMixin, jsonext.JSONSerializable):
               more details.
 
         '''
+        if self.is_dry_run():
+            return
+
         self._job.wait()
 
     @final
@@ -2095,6 +2115,9 @@ class RegressionTest(RegressionMixin, jsonext.JSONSerializable):
         elif not hasattr(self, 'sanity_patterns'):
             raise SanityError('sanity_patterns not set')
 
+        if self.is_dry_run():
+            return
+
         with osext.change_dir(self._stagedir):
             success = sn.evaluate(self.sanity_patterns)
             if not success:
@@ -2151,6 +2174,9 @@ class RegressionTest(RegressionMixin, jsonext.JSONSerializable):
                 self.perf_variables[var] = sn.make_performance_function(expr,
                                                                         unit)
 
+        if self.is_dry_run():
+            return
+
         # Evaluate the performance function and retrieve the metrics
         with osext.change_dir(self._stagedir):
             for tag, expr in self.perf_variables.items():
@@ -2272,6 +2298,9 @@ class RegressionTest(RegressionMixin, jsonext.JSONSerializable):
               more details.
 
         '''
+        if self.is_dry_run():
+            return
+
         aliased = os.path.samefile(self._stagedir, self._outputdir)
         if aliased:
             self.logger.debug(
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 54436755..2bb18628 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -380,6 +380,10 @@ def main():
         '-r', '--run', action='store_true',
         help='Run the selected checks'
     )
+    action_options.add_argument(
+        '--dry-run', action='store_true',
+        help='Dry run the tests without submitting them for execution'
+    )
 
     # Run options
     run_options.add_argument(
@@ -896,6 +900,9 @@ def main():
         else:
             external_vars[lhs] = rhs
 
+    if options.dry_run:
+        external_vars['_rfm_dry_run'] = '1'
+
     loader = RegressionCheckLoader(check_search_path,
                                    check_search_recursive,
                                    external_vars,
@@ -1166,10 +1173,11 @@ def main():
             )
             sys.exit(0)
 
-        if not options.run:
+        if not options.run and not options.dry_run:
             printer.error("No action option specified. Available options:\n"
                           "  - `-l'/`-L' for listing\n"
                           "  - `-r' for running\n"
+                          "  - `--dry-run' for dry running\n"
                           "  - `--list-tags' for listing unique test tags\n"
                           "  - `--ci-generate' for generating a CI pipeline\n"
                           f"Try `{argparser.prog} -h' for more options.")
@@ -1266,6 +1274,7 @@ def main():
         exec_policy.keep_stage_files = site_config.get(
             'general/0/keep_stage_files'
         )
+        exec_policy.dry_run_mode = options.dry_run
         try:
             errmsg = "invalid option for --flex-alloc-nodes: '{0}'"
             sched_flex_alloc_nodes = int(options.flex_alloc_nodes)
diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py
index dea65856..9409ce2f 100644
--- a/reframe/frontend/executors/__init__.py
+++ b/reframe/frontend/executors/__init__.py
@@ -587,6 +587,7 @@ class ExecutionPolicy(abc.ABC):
         self.skip_sanity_check = False
         self.skip_performance_check = False
         self.keep_stage_files = False
+        self.dry_run_mode = False
         self.only_environs = None
         self.printer = None
 
diff --git a/reframe/frontend/executors/policies.py b/reframe/frontend/executors/policies.py
index 1009a13c..00ddfe59 100644
--- a/reframe/frontend/executors/policies.py
+++ b/reframe/frontend/executors/policies.py
@@ -98,9 +98,13 @@ class SerialExecutionPolicy(ExecutionPolicy, TaskEventListener):
 
     def runcase(self, case):
         super().runcase(case)
-        check, partition, environ = case
+        check, partition, _ = case
         task = RegressionTask(case, self.task_listeners)
-        self.printer.status('RUN', task.info())
+        if check.is_dry_run():
+            self.printer.status('DRY', task.info())
+        else:
+            self.printer.status('RUN', task.info())
+
         self._task_index[case] = task
         self.stats.add_task(task)
         try:
@@ -138,7 +142,9 @@ class SerialExecutionPolicy(ExecutionPolicy, TaskEventListener):
 
             self._pollctl.reset_snooze_time()
             while True:
-                sched.poll(task.check.job)
+                if not self.dry_run_mode:
+                    sched.poll(task.check.job)
+
                 if task.run_complete():
                     break
 
@@ -362,6 +368,9 @@ class AsynchronousExecutionPolicy(ExecutionPolicy, TaskEventListener):
             self._dump_pipeline_progress('pipeline-progress.json')
 
     def _poll_tasks(self):
+        if self.dry_run_mode:
+            return
+
         for partname, sched in self._schedulers.items():
             jobs = []
             for t in self._partition_tasks[partname]:
@@ -431,7 +440,11 @@ class AsynchronousExecutionPolicy(ExecutionPolicy, TaskEventListener):
                 return 1
         elif self.deps_succeeded(task):
             try:
-                self.printer.status('RUN', task.info())
+                if task.check.is_dry_run():
+                    self.printer.status('DRY', task.info())
+                else:
+                    self.printer.status('RUN', task.info())
+
                 task.setup(task.testcase.partition,
                            task.testcase.environ,
                            sched_flex_alloc_nodes=self.sched_flex_alloc_nodes,
diff --git a/reframe/frontend/loader.py b/reframe/frontend/loader.py
index 39a4d48d..5d96e615 100644
--- a/reframe/frontend/loader.py
+++ b/reframe/frontend/loader.py
@@ -14,7 +14,6 @@ import os
 import sys
 import traceback
 
-import reframe.core.fields as fields
 import reframe.utility as util
 import reframe.utility.osext as osext
 from reframe.core.exceptions import NameConflictError, is_severe, what
@@ -133,29 +132,6 @@ class RegressionCheckLoader:
     def recurse(self):
         return self._recurse
 
-    def _set_defaults(self, test_registry):
-        if test_registry is None:
-            return
-
-        self._unset_vars = {}
-        for test in test_registry:
-            for name, val in self._external_vars.items():
-                if '.' in name:
-                    testname, varname = name.split('.', maxsplit=1)
-                else:
-                    testname, varname = test.__name__, name
-
-                if testname == test.__name__:
-                    # Treat special values
-                    if val == '@none':
-                        val = None
-                    else:
-                        val = fields.make_convertible(val)
-
-                    if not test.setvar(varname, val):
-                        self._unset_vars.setdefault(test.__name__, [])
-                        self._unset_vars[test.__name__].append(varname)
-
     def load_from_module(self, module):
         '''Load user checks from module.
 
@@ -164,10 +140,13 @@ class RegressionCheckLoader:
         are validated before return.
         '''
         registry = getattr(module, '_rfm_test_registry', None)
-        self._set_defaults(registry)
+        if registry:
+            self._unset_vars.update(registry.setvars(self._external_vars))
+
         reset_sysenv = self._skip_prgenv_check << 1 | self._skip_system_check
         if registry:
-            candidate_tests = registry.instantiate_all(reset_sysenv)
+            candidate_tests = registry.instantiate_all(reset_sysenv,
+                                                       self._external_vars)
         else:
             candidate_tests = []
 
diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py
index b6c18309..64647140 100644
--- a/reframe/frontend/printer.py
+++ b/reframe/frontend/printer.py
@@ -48,15 +48,15 @@ class PrettyPrinter:
 
         status_stripped = status.strip()
         if self.colorize:
-            if status_stripped == 'SKIP':
+            if status_stripped in ('DRY', 'SKIP'):
                 status = color.colorize(status, color.YELLOW)
-            elif status_stripped in ['FAIL', 'FAILED', 'ERROR']:
+            elif status_stripped in ('FAIL', 'FAILED', 'ERROR'):
                 status = color.colorize(status, color.RED)
             else:
                 status = color.colorize(status, color.GREEN)
 
         final_msg = f'[ {status} ] '
-        if status_stripped in ['OK', 'SKIP', 'FAIL']:
+        if status_stripped in ('OK', 'SKIP', 'FAIL'):
             self._progress_count += 1
             width = len(str(self._progress_total))
             padded_progress = str(self._progress_count).rjust(width)


Test patch:
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 81339a55..4f4f3c3e 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -86,6 +86,8 @@ def run_reframe(tmp_path, perflogdir, monkeypatch):
 
         if action == 'run':
             argv += ['-r']
+        elif action == 'dry_run':
+            argv += ['--dry-run']
         elif action == 'list':
             argv += ['-l']
         elif action == 'list_detailed':
@@ -129,12 +131,21 @@ def remote_exec_ctx(user_exec_ctx):
     return partition, partition.environs[0]
 
 
-def test_check_success(run_reframe, tmp_path):
-    returncode, stdout, _ = run_reframe(more_options=['--save-log-files'])
+@pytest.fixture(params=['run', 'dry_run'])
+def run_action(request):
+    return request.param
+
+
+def test_check_success(run_reframe, tmp_path, run_action):
+    returncode, stdout, _ = run_reframe(more_options=['--save-log-files'],
+                                        action=run_action)
     assert 'PASSED' in stdout
     assert 'FAILED' not in stdout
     assert returncode == 0
 
+    if run_action == 'dry_run':
+        assert 'DRY' in stdout
+
     logfile = logging.log_files()[0]
     assert os.path.exists(tmp_path / 'output' / logfile)
     assert os.path.exists(tmp_path / 'report.json')
@@ -142,7 +153,7 @@ def test_check_success(run_reframe, tmp_path):
 
 def test_check_restore_session_failed(run_reframe, tmp_path):
     run_reframe(
-        checkpath=['unittests/resources/checks_unlisted/deps_complex.py'],
+        checkpath=['unittests/resources/checks_unlisted/deps_complex.py']
     )
     returncode, stdout, _ = run_reframe(
         checkpath=[],
@@ -195,37 +206,40 @@ def test_check_restore_session_check_search_path(run_reframe, tmp_path):
     assert 'Found 0 check(s)' in stdout
 
 
-def test_check_success_force_local(run_reframe, tmp_path):
+def test_check_success_force_local(run_reframe, tmp_path, run_action):
     # We explicitly use a system here with a non-local scheduler and we run
     # with `-S local=1`
-    returncode, stdout, _ = run_reframe(system='testsys:gpu', local=True)
+    returncode, stdout, _ = run_reframe(system='testsys:gpu', local=True,
+                                        action=run_action)
     assert 'PASSED' in stdout
     assert 'FAILED' not in stdout
     assert returncode == 0
 
 
-def test_report_file_with_sessionid(run_reframe, tmp_path):
+def test_report_file_with_sessionid(run_reframe, tmp_path, run_action):
     returncode, *_ = run_reframe(
         more_options=[
             f'--report-file={tmp_path / "rfm-report-{sessionid}.json"}'
-        ]
+        ],
+        action=run_action
     )
     assert returncode == 0
     assert os.path.exists(tmp_path / 'rfm-report-0.json')
 
 
-def test_report_ends_with_newline(run_reframe, tmp_path):
+def test_report_ends_with_newline(run_reframe, tmp_path, run_action):
     returncode, stdout, _ = run_reframe(
         more_options=[
             f'--report-file={tmp_path / "rfm-report.json"}'
-        ]
+        ],
+        action=run_action
     )
     assert returncode == 0
     with open(tmp_path / 'rfm-report.json') as fp:
         assert fp.read()[-1] == '\n'
 
 
-def test_check_submit_success(run_reframe, remote_exec_ctx):
+def test_check_submit_success(run_reframe, remote_exec_ctx, run_action):
     # This test will run on the auto-detected system
     partition, environ = remote_exec_ctx
     returncode, stdout, _ = run_reframe(
@@ -234,7 +248,8 @@ def test_check_submit_success(run_reframe, remote_exec_ctx):
         system=partition.fullname,
         # Pick up the programming environment of the partition
         # Prepend ^ and append $ so as to much exactly the given name
-        environs=[f'^{environ.name}$']
+        environs=[f'^{environ.name}$'],
+        action=run_action
     )
 
     assert 'FAILED' not in stdout
@@ -281,22 +296,36 @@ def test_check_kbd_interrupt(run_reframe):
     assert returncode != 0
 
 
-def test_check_sanity_failure(run_reframe, tmp_path):
+def test_check_sanity_failure(run_reframe, tmp_path, run_action):
     returncode, stdout, stderr = run_reframe(
         checkpath=['unittests/resources/checks/frontend_checks.py'],
-        more_options=['-n', 'SanityFailureCheck']
+        more_options=['-n', 'SanityFailureCheck'],
+        action=run_action
     )
-    assert 'FAILED' in stdout
-
     # This is a normal failure, it should not raise any exception
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
-    assert returncode != 0
+
     assert os.path.exists(
         tmp_path / 'stage' / 'generic' / 'default' /
-        'builtin' / 'SanityFailureCheck'
+        'builtin' / 'SanityFailureCheck' / 'rfm_job.sh'
+    )
+    assert not os.path.exists(
+        tmp_path / 'output' / 'generic' / 'default' /
+        'builtin' / 'SanityFailureCheck' / 'rfm_job.sh'
     )
 
+    if run_action == 'dry_run':
+        assert 'PASSED' in stdout
+        assert returncode == 0
+        assert not os.path.exists(
+            tmp_path / 'stage' / 'generic' / 'default' /
+            'builtin' / 'SanityFailureCheck' / 'rfm_job.out'
+        )
+    else:
+        assert 'FAILED' in stdout
+        assert returncode != 0
+
 
 def test_dont_restage(run_reframe, tmp_path):
     run_reframe(
@@ -341,23 +370,30 @@ def test_checkpath_symlink(run_reframe, tmp_path):
     assert num_checks_in_checkdir == num_checks_default
 
 
-def test_performance_check_failure(run_reframe, tmp_path, perflogdir):
+def test_performance_check_failure(run_reframe, tmp_path,
+                                   perflogdir, run_action):
     returncode, stdout, stderr = run_reframe(
         checkpath=['unittests/resources/checks/frontend_checks.py'],
-        more_options=['-n', 'PerformanceFailureCheck']
+        more_options=['-n', 'PerformanceFailureCheck'],
+        action=run_action
     )
-    assert 'FAILED' in stdout
+    if run_action == 'dry_run':
+        assert returncode == 0
+        assert 'PASSED' in stdout
+    else:
+        assert returncode != 0
+        assert 'FAILED' in stdout
 
     # This is a normal failure, it should not raise any exception
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
-    assert returncode != 0
     assert os.path.exists(
         tmp_path / 'stage' / 'generic' / 'default' /
         'builtin' / 'PerformanceFailureCheck'
     )
-    assert os.path.exists(perflogdir / 'generic' /
-                          'default' / 'PerformanceFailureCheck.log')
+    if run_action == 'run':
+        assert os.path.exists(perflogdir / 'generic' /
+                              'default' / 'PerformanceFailureCheck.log')
 
 
 def test_perflogdir_from_env(run_reframe, tmp_path, monkeypatch):
@@ -374,28 +410,34 @@ def test_perflogdir_from_env(run_reframe, tmp_path, monkeypatch):
                           'default' / 'PerformanceFailureCheck.log')
 
 
-def test_performance_report(run_reframe):
+def test_performance_report(run_reframe, run_action):
     returncode, stdout, _ = run_reframe(
         checkpath=['unittests/resources/checks/frontend_checks.py'],
-        more_options=['-n', 'PerformanceFailureCheck', '--performance-report']
+        more_options=['-n', 'PerformanceFailureCheck', '--performance-report'],
+        action=run_action
     )
-    assert r'PERFORMANCE REPORT' in stdout
-    assert r'perf: 10 Gflop/s' in stdout
+    if run_action == 'run':
+        assert r'PERFORMANCE REPORT' in stdout
+        assert r'perf: 10 Gflop/s' in stdout
+    else:
+        assert r'PERFORMANCE REPORT' not in stdout
 
 
-def test_skip_system_check_option(run_reframe):
+def test_skip_system_check_option(run_reframe, run_action):
     returncode, stdout, _ = run_reframe(
         checkpath=['unittests/resources/checks/frontend_checks.py'],
-        more_options=['--skip-system-check', '-n', 'NoSystemCheck']
+        more_options=['--skip-system-check', '-n', 'NoSystemCheck'],
+        action=run_action
     )
     assert 'PASSED' in stdout
     assert returncode == 0
 
 
-def test_skip_prgenv_check_option(run_reframe):
+def test_skip_prgenv_check_option(run_reframe, run_action):
     returncode, stdout, _ = run_reframe(
         checkpath=['unittests/resources/checks/frontend_checks.py'],
-        more_options=['--skip-prgenv-check', '-n', 'NoPrgEnvCheck']
+        more_options=['--skip-prgenv-check', '-n', 'NoPrgEnvCheck'],
+        action=run_action
     )
     assert 'PASSED' in stdout
     assert returncode == 0
@@ -464,12 +506,13 @@ def test_same_output_stage_dir(run_reframe, tmp_path):
     assert os.path.exists(output_dir)
 
 
-def test_execution_modes(run_reframe):
+def test_execution_modes(run_reframe, run_action):
     returncode, stdout, stderr = run_reframe(
         checkpath=[],
         environs=[],
         local=False,
-        mode='unittest'
+        mode='unittest',
+        action=run_action
     )
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
@@ -744,7 +787,7 @@ def test_unload_module(run_reframe, user_exec_ctx):
     assert returncode == 0
 
 
-def test_unuse_module_path(run_reframe, user_exec_ctx):
+def test_unuse_module_path(run_reframe, user_exec_ctx, run_action):
     ms = rt.runtime().modules_system
     if not test_util.has_sane_modules_system():
         pytest.skip('no modules system found')
@@ -753,7 +796,7 @@ def test_unuse_module_path(run_reframe, user_exec_ctx):
     ms.searchpath_add(module_path)
     returncode, stdout, stderr = run_reframe(
         more_options=[f'--module-path=-{module_path}', '--module=testmod_foo'],
-        config_file=test_util.USER_CONFIG_FILE, action='run',
+        config_file=test_util.USER_CONFIG_FILE, action=run_action,
         system=rt.runtime().system.name
     )
     ms.searchpath_remove(module_path)
@@ -762,14 +805,14 @@ def test_unuse_module_path(run_reframe, user_exec_ctx):
     assert returncode == 1
 
 
-def test_use_module_path(run_reframe, user_exec_ctx):
+def test_use_module_path(run_reframe, user_exec_ctx, run_action):
     if not test_util.has_sane_modules_system():
         pytest.skip('no modules system found')
 
     module_path = 'unittests/modules'
     returncode, stdout, stderr = run_reframe(
         more_options=[f'--module-path=+{module_path}', '--module=testmod_foo'],
-        config_file=test_util.USER_CONFIG_FILE, action='run',
+        config_file=test_util.USER_CONFIG_FILE, action=run_action,
         system=rt.runtime().system.name
     )
     assert 'Traceback' not in stdout
@@ -778,7 +821,7 @@ def test_use_module_path(run_reframe, user_exec_ctx):
     assert returncode == 0
 
 
-def test_overwrite_module_path(run_reframe, user_exec_ctx):
+def test_overwrite_module_path(run_reframe, user_exec_ctx, run_action):
     if not test_util.has_sane_modules_system():
         pytest.skip('no modules system found')
 
@@ -788,7 +831,7 @@ def test_overwrite_module_path(run_reframe, user_exec_ctx):
 
     returncode, stdout, stderr = run_reframe(
         more_options=[f'--module-path={module_path}', '--module=testmod_foo'],
-        config_file=test_util.USER_CONFIG_FILE, action='run',
+        config_file=test_util.USER_CONFIG_FILE, action=run_action,
         system=rt.runtime().system.name
     )
     assert 'Traceback' not in stdout
@@ -797,16 +840,21 @@ def test_overwrite_module_path(run_reframe, user_exec_ctx):
     assert returncode == 0
 
 
-def test_failure_stats(run_reframe):
+def test_failure_stats(run_reframe, run_action):
     returncode, stdout, stderr = run_reframe(
         checkpath=['unittests/resources/checks/frontend_checks.py'],
-        more_options=['-n', 'SanityFailureCheck', '--failure-stats']
+        more_options=['-n', 'SanityFailureCheck', '--failure-stats'],
+        action=run_action
     )
-    assert r'FAILURE STATISTICS' in stdout
-    assert r'sanity        1     [SanityFailureCheck' in stdout
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
-    assert returncode != 0
+    if run_action == 'dry_run':
+        assert returncode == 0
+        assert r'FAILURE STATISTICS' not in stdout
+    else:
+        assert returncode != 0
+        assert r'FAILURE STATISTICS' in stdout
+        assert r'sanity        1     [SanityFailureCheck' in stdout
 
 
 def test_maxfail_option(run_reframe):
@@ -846,10 +894,11 @@ def test_maxfail_negative(run_reframe):
     assert returncode == 1
 
 
-def test_repeat_option(run_reframe):
+def test_repeat_option(run_reframe, run_action):
     returncode, stdout, stderr = run_reframe(
         more_options=['--repeat', '2', '-n', '^HelloTest'],
-        checkpath=['unittests/resources/checks/hellocheck.py']
+        checkpath=['unittests/resources/checks/hellocheck.py'],
+        action=run_action
     )
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
@@ -946,7 +995,7 @@ def test_detect_host_topology_file(run_reframe, tmp_path):
         assert json.load(fp) == cpuinfo()
 
 
-def test_external_vars(run_reframe):
+def test_external_vars(run_reframe, run_action):
     returncode, stdout, stderr = run_reframe(
         checkpath=['unittests/resources/checks_unlisted/externalvars.py'],
         more_options=['-S', 'external_x.foo=3',
@@ -955,7 +1004,8 @@ def test_external_vars(run_reframe):
                       '-S', 'external_y.foo=2',
                       '-S', 'external_y.baz=false',
                       '-S', 'foolist=3,4',
-                      '-S', 'bar=@none']
+                      '-S', 'bar=@none'],
+        action=run_action
     )
     assert 'PASSED' in stdout
     assert 'Ran 6/6 test case(s)' in stdout
@@ -964,9 +1014,9 @@ def test_external_vars(run_reframe):
     assert returncode == 0
 
 
-def test_external_vars_invalid_expr(run_reframe):
+def test_external_vars_invalid_expr(run_reframe, run_action):
     returncode, stdout, stderr = run_reframe(
-        more_options=['-S', 'foo']
+        more_options=['-S', 'foo'], action=run_action
     )
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
@@ -1025,36 +1075,36 @@ def test_fixture_registry_env_sys(run_reframe):
     assert 'sys1:p1' in stdout
 
 
-def test_fixture_resolution(run_reframe):
+def test_fixture_resolution(run_reframe, run_action):
     returncode, stdout, stderr = run_reframe(
         system='sys1',
         environs=[],
         checkpath=['unittests/resources/checks_unlisted/fixtures_complex.py'],
-        action='run'
+        action=run_action
     )
     assert returncode == 0
 
 
-def test_dynamic_tests(run_reframe, tmp_path):
+def test_dynamic_tests(run_reframe, tmp_path, run_action):
     returncode, stdout, _ = run_reframe(
         system='sys0',
         environs=[],
         checkpath=['unittests/resources/checks_unlisted/distribute.py'],
-        action='run',
-        more_options=['-n', '^Complex', '--distribute=idle']
+        more_options=['-n', '^Complex', '--distribute=idle'],
+        action=run_action
     )
     assert returncode == 0
     assert 'Ran 10/10 test case(s)' in stdout
     assert 'FAILED' not in stdout
 
 
-def test_dynamic_tests_filtering(run_reframe, tmp_path):
+def test_dynamic_tests_filtering(run_reframe, tmp_path, run_action):
     returncode, stdout, _ = run_reframe(
         system='sys1',
         environs=[],
         checkpath=['unittests/resources/checks_unlisted/distribute.py'],
-        action='run',
-        more_options=['-n', 'Complex@1', '--distribute=idle']
+        more_options=['-n', 'Complex@1', '--distribute=idle'],
+        action=run_action
     )
     assert returncode == 0
     assert 'Ran 7/7 test case(s)' in stdout