Repository: reframe-hpc/reframe

Issue:
When running with `--duration` do not exit with a non-zero exit code just because of the time out of the last run
Also, we should document what is the expected exit code in case of reruns, either through the `--duration` or the `--reruns` option. For example, for `--reruns` it seems that reframe exits with zero if the last rerun was successful, despite failures in previous reruns.

Discussion:

Patch:
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index f55bf989..2337546b 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -1638,9 +1638,12 @@ def main():
             if options.max_retries and runner.stats.failed(run=0):
                 printer.retry_report(report)
 
-            # Print a failure report if we had failures in the last run
+            # Print a failure report in case of failures.
+            # If `--duration` or `--reruns` is used then take into account
+            # all runs, else (i.e., `--max-retries`) only the last run.
             success = True
-            if runner.stats.failed():
+            runid = None if options.duration or options.reruns else -1
+            if runner.stats.failed(run=runid):
                 success = False
                 printer.failure_report(
                     report,
@@ -1734,6 +1737,12 @@ def main():
             sys.exit(1)
 
         sys.exit(0)
+    except errors.RunSessionTimeout as err:
+        printer.warning(f'run session stopped: {err}')
+        if not success:
+            sys.exit(1)
+        else:
+            sys.exit(0)
     except (Exception, KeyboardInterrupt, errors.ReframeFatalError):
         exc_info = sys.exc_info()
         tb = ''.join(traceback.format_exception(*exc_info))


Test patch:
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 1e176359..799b45f9 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -1076,6 +1076,13 @@ def test_reruns_with_duration(run_reframe):
     assert returncode == 1
 
 
+def test_exitcode_timeout(run_reframe):
+    assert_no_crash(*run_reframe(
+        more_options=['--duration=5s', '-n^HelloTest'],
+        checkpath=['unittests/resources/checks/hellocheck.py']
+    ))
+
+
 @pytest.fixture(params=['name', 'rname', 'uid', 'ruid', 'random'])
 def exec_order(request):
     return request.param