Repository: reframe-hpc/reframe

Issue:
RFE: randomize test execution order
I don't think this is possible right now. It would be useful to be able to add a CLI argument to randomize the execution order of all the tests. 

Ideally it would randomize the tests across all files passed to Reframe, e.g. with `-c test1.py -c test2.py -c test3.py` it would randomize and interleave tests between all 3 files, not just within each file. To replay particular sequence of execution, perhaps the argument could accept an optional seed for the RNG.

The rationale is that some test A might change the state of the system like making the system very hot and throttling clocks, that could cause a failure in a later test C only if you run A -> C -> B instead of always executing A -> B -> C, for example if test B is long but does not stress the system (giving the system the time to cool down).
Similarly, if the same Reframe test starts in parallel on 10 machines, they might all reach a network test at the same time, creating heavy congestion (which is useful in itself, but might not be the target of the test).

If possible, it would also be great if the performance report printed would always stay in the same order to make it easier to compare runs despite the randomized execution order.


Discussion:

Patch:
diff --git a/docs/manpage.rst b/docs/manpage.rst
index 90543a9f..8ee59874 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -418,6 +418,23 @@ Options controlling ReFrame execution
 
    .. versionadded:: 3.11.0
 
+
+.. option:: --exec-order=ORDER
+
+   Impose an execution order for the independent tests.
+   The ``ORDER`` argument can take one of the following values:
+
+   - ``name``: Order tests by their display name.
+   - ``rname``: Order tests by their display name in reverse order.
+   - ``uid``: Order tests by their unique name.
+   - ``ruid``: Order tests by their unique name in reverse order.
+   - ``random``: Randomize the order of execution.
+
+   If this option is not specified the order of execution of independent tests is implementation defined.
+   This option can be combined with any of the listing options (:option:`-l` or :option:`-L`) to list the tests in the order.
+
+   .. versionadded:: 4.0.0
+
 .. option:: --exec-policy=POLICY
 
    The execution policy to be used for running tests.
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 83bba0fb..0412fd61 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -7,6 +7,7 @@ import inspect
 import itertools
 import json
 import os
+import random
 import shlex
 import socket
 import sys
@@ -25,6 +26,7 @@ import reframe.frontend.ci as ci
 import reframe.frontend.dependencies as dependencies
 import reframe.frontend.filters as filters
 import reframe.frontend.runreport as runreport
+import reframe.utility as util
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
 import reframe.utility.typecheck as typ
@@ -386,6 +388,11 @@ def main():
         help=('Distribute the selected single-node jobs on every node that'
               'is in STATE (default: "idle"')
     )
+    run_options.add_argument(
+        '--exec-order', metavar='ORDER', action='store',
+        choices=['name', 'random', 'rname', 'ruid', 'uid'],
+        help='Impose an execution order for independent tests'
+    )
     run_options.add_argument(
         '--exec-policy', metavar='POLICY', action='store',
         choices=['async', 'serial'], default='async',
@@ -1054,8 +1061,8 @@ def main():
                     "a non-negative integer"
                 ) from None
 
-            testcases = repeat_tests(testcases, num_repeats)
-            testcases_all = testcases
+            testcases_all = repeat_tests(testcases, num_repeats)
+            testcases = testcases_all
 
         if options.distribute:
             node_map = getallnodes(options.distribute, parsed_job_options)
@@ -1070,15 +1077,31 @@ def main():
                 x for x in parsed_job_options
                 if (not x.startswith('-w') and not x.startswith('--nodelist'))
             ]
-            testcases = distribute_tests(testcases, node_map)
-            testcases_all = testcases
+            testcases_all = distribute_tests(testcases, node_map)
+            testcases = testcases_all
+
+        @logging.time_function
+        def _sort_testcases(testcases):
+            if options.exec_order in ('name', 'rname'):
+                testcases.sort(key=lambda c: c.check.display_name,
+                               reverse=(options.exec_order == 'rname'))
+            elif options.exec_order in ('uid', 'ruid'):
+                testcases.sort(key=lambda c: c.check.unique_name,
+                               reverse=(options.exec_order == 'ruid'))
+            elif options.exec_order == 'random':
+                random.shuffle(testcases)
+
+        _sort_testcases(testcases)
+        if testcases_all is not testcases:
+            _sort_testcases(testcases_all)
 
         # Prepare for running
         printer.debug('Building and validating the full test DAG')
         testgraph, skipped_cases = dependencies.build_deps(testcases_all)
         if skipped_cases:
             # Some cases were skipped, so adjust testcases
-            testcases = list(set(testcases) - set(skipped_cases))
+            testcases = list(util.OrderedSet(testcases) -
+                             util.OrderedSet(skipped_cases))
             printer.verbose(
                 f'Filtering test case(s) due to unresolved dependencies: '
                 f'{len(testcases)} remaining'


Test patch:
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 0136b66f..12ce3ad0 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -834,7 +834,7 @@ def test_repeat_invalid_option(run_reframe):
 
 def test_repeat_negative(run_reframe):
     returncode, stdout, stderr = run_reframe(
-        more_options=['--repeat', 'foo'],
+        more_options=['--repeat', '-1'],
         checkpath=['unittests/resources/checks/hellocheck.py']
     )
     errmsg = "argument to '--repeat' option must be a non-negative integer"
@@ -844,6 +844,44 @@ def test_repeat_negative(run_reframe):
     assert returncode == 1
 
 
+@pytest.fixture(params=['name', 'rname', 'uid', 'ruid', 'random'])
+def exec_order(request):
+    return request.param
+
+
+def test_exec_order(run_reframe, exec_order):
+    import reframe.utility.sanity as sn
+
+    returncode, stdout, stderr = run_reframe(
+        more_options=['--repeat', '11', '-n', 'HelloTest',
+                      f'--exec-order={exec_order}'],
+        checkpath=['unittests/resources/checks/hellocheck.py'],
+        action='list_detailed',
+    )
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    assert 'Found 11 check(s)' in stdout
+    assert returncode == 0
+
+    # Verify the order
+    if exec_order == 'name':
+        repeat_no = sn.extractsingle_s(r'- HelloTest.*repeat_no=(\d+)',
+                                       stdout, 1, int, 2).evaluate()
+        assert repeat_no == 10
+    elif exec_order == 'rname':
+        repeat_no = sn.extractsingle_s(r'- HelloTest.*repeat_no=(\d+)',
+                                       stdout, 1, int, -3).evaluate()
+        assert repeat_no == 10
+    elif exec_order == 'uid':
+        repeat_no = sn.extractsingle_s(r'- HelloTest.*repeat_no=(\d+)',
+                                       stdout, 1, int, -1).evaluate()
+        assert repeat_no == 10
+    elif exec_order == 'ruid':
+        repeat_no = sn.extractsingle_s(r'- HelloTest.*repeat_no=(\d+)',
+                                       stdout, 1, int, 0).evaluate()
+        assert repeat_no == 10
+
+
 def test_detect_host_topology(run_reframe):
     from reframe.utility.cpuinfo import cpuinfo