Repository: reframe-hpc/reframe

Issue:
Make ReFrame more resilient to `sacct` temporary errors
Sometimes, during long runs, Slurm is not completely healthy and sacct will fail with this error:
```
sacct: error: _open_persist_conn: failed to open persistent connection to host:daint-slurmctl.svc.cscs.ch:6819: Connection timed out
sacct: error: Sending PersistInit msg: Connection timed out
sacct: error: Problem talking to the database: Connection timed out
```

In this case reframe will crash and will not create the json report as expected:
```
ERROR: run session stopped: key error: 'fail_info'
ERROR: Traceback (most recent call last):
  File "/capstor/scratch/cscs/reframe/gitlab-runner/f7t/0/835515875116211/reframe-develop/reframe/frontend/cli.py", line 1621, in main
    runner.runall(testcases, restored_cases)
  File "/capstor/scratch/cscs/reframe/gitlab-runner/f7t/0/835515875116211/reframe-develop/reframe/core/logging.py", line 1051, in _fn
    return fn(*args, **kwargs)
  File "/capstor/scratch/cscs/reframe/gitlab-runner/f7t/0/835515875116211/reframe-develop/reframe/frontend/executors/__init__.py", line 643, in runall
    self._retry_failed(testcases + restored_cases)
  File "/capstor/scratch/cscs/reframe/gitlab-runner/f7t/0/835515875116211/reframe-develop/reframe/frontend/executors/__init__.py", line 704, in _retry_failed
    self._runall(failed_cases)
  File "/capstor/scratch/cscs/reframe/gitlab-runner/f7t/0/835515875116211/reframe-develop/reframe/frontend/executors/__init__.py", line 721, in _runall
    self._policy.exit()
  File "/capstor/scratch/cscs/reframe/gitlab-runner/f7t/0/835515875116211/reframe-develop/reframe/frontend/executors/policies.py", line 349, in exit
    self._poll_tasks()
  File "/capstor/scratch/cscs/reframe/gitlab-runner/f7t/0/835515875116211/reframe-develop/reframe/frontend/executors/policies.py", line 399, in _poll_tasks
    sched.poll(*jobs)
  File "/capstor/scratch/cscs/reframe/gitlab-runner/f7t/0/835515875116211/reframe-develop/reframe/core/schedulers/slurm.py", line 455, in poll
    f'sacct -S {t_start} -P '
  File "/capstor/scratch/cscs/reframe/gitlab-runner/f7t/0/835515875116211/reframe-develop/reframe/utility/osext.py", line 295, in run_command
    completed.returncode)
reframe.core.exceptions.SpawnedProcessError: command 'sacct -S 2025-03-04 -P -j 505003,505004,505005,505006,505007,505008,505009,505010,505011,505012,505013,505014,505015,505016,505017,505018,505019 -o jobid,state,exitcode,end,nodelist' failed with exit code 1:
--- stdout ---
--- stdout ---
--- stderr ---
sacct: error: _open_persist_conn: failed to open persistent connection to host:daint-slurmctl.svc.cscs.ch:6819: Connection timed out
sacct: error: Sending PersistInit msg: Connection timed out
sacct: error: Problem talking to the database: Connection timed out
--- stderr ---
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/capstor/scratch/cscs/reframe/gitlab-runner/f7t/0/835515875116211/reframe-develop/reframe/frontend/cli.py", line 1654, in main
    global_stats=options.duration or options.reruns
  File "/capstor/scratch/cscs/reframe/gitlab-runner/f7t/0/835515875116211/reframe-develop/reframe/frontend/printer.py", line 182, in failure_report
    _print_failure_info(r, run_no, len(report['runs']))
  File "/capstor/scratch/cscs/reframe/gitlab-runner/f7t/0/835515875116211/reframe-develop/reframe/frontend/printer.py", line 145, in _print_failure_info
    if isinstance(rec['fail_info']['exc_value'], BuildError):
KeyError: 'fail_info'
```

Even though it's understandable to crash, I think the framework should be a bit more resilient to this kind of (temporary) failures. In our case the error is not usually persistent but will happen in long runs, that have hundreds of tests. My suggestion would be to have a configuration parameter that will determine the number of retries for certain commands. I am not sure whether this should be only for sacct/squeue or more general to other commands. What do you think?

Discussion:

Patch:
diff --git a/docs/config_reference.rst b/docs/config_reference.rst
index 128e527e..48213c14 100644
--- a/docs/config_reference.rst
+++ b/docs/config_reference.rst
@@ -456,6 +456,22 @@ System Partition Configuration
       No other test would be able to proceed.
 
 
+.. py:attribute:: systems.partitions.sched_options.max_sacct_failures
+
+   :required: No
+   :default: ``3``
+
+   Maximum number of consecutive failures of `sacct` before ReFrame reports a failure.
+
+   Some times the ``sacct`` command may be temporarily unavailable.
+   In this case, ReFrame will continue polling until ``max_sacct_failures`` is reached.
+   Every time the command succeeds, the counter is reset.
+
+   This option is relevant for the ``slurm`` backend only.
+
+   .. versionadded:: 4.8
+
+
 .. py:attribute:: systems.partitions.sched_options.unqualified_hostnames
 
    :required: No
diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py
index 59a94ae7..e0fc28f7 100644
--- a/reframe/core/schedulers/slurm.py
+++ b/reframe/core/schedulers/slurm.py
@@ -12,6 +12,7 @@ import time
 from argparse import ArgumentParser
 from contextlib import suppress
 
+import reframe.core.logging as logging
 import reframe.core.runtime as rt
 import reframe.core.schedulers as sched
 import reframe.utility.osext as osext
@@ -140,6 +141,8 @@ class SlurmJobScheduler(sched.JobScheduler):
         self._submit_timeout = self.get_option('job_submit_timeout')
         self._use_nodes_opt = self.get_option('use_nodes_option')
         self._resubmit_on_errors = self.get_option('resubmit_on_errors')
+        self._max_sacct_failures = self.get_option('max_sacct_failures')
+        self._num_sacct_failures = 0
         self._sched_access_in_submit = self.get_option(
             'sched_access_in_submit'
         )
@@ -322,7 +325,7 @@ class SlurmJobScheduler(sched.JobScheduler):
             if partition_match:
                 return partition_match.group('partition')
 
-        except SpawnedProcessError as e:
+        except SpawnedProcessError:
             self.log('could not retrieve actual partition')
 
         return None
@@ -451,11 +454,25 @@ class SlurmJobScheduler(sched.JobScheduler):
             t_start = time.strftime(
                 '%F', time.localtime(min(job.submit_time for job in jobs))
             )
-            completed = _run_strict(
-                f'sacct -S {t_start} -P '
-                f'-j {",".join(job.jobid for job in jobs)} '
-                f'-o jobid,state,exitcode,end,nodelist'
-            )
+            try:
+                completed = _run_strict(
+                    f'sacct -S {t_start} -P '
+                    f'-j {",".join(job.jobid for job in jobs)} '
+                    f'-o jobid,state,exitcode,end,nodelist'
+                )
+                # Reset the retry counter if the command succeeds
+                self._num_sacct_failures = 0
+            except SpawnedProcessError as e:
+                self._num_sacct_failures += 1
+                if self._num_sacct_failures > self._max_sacct_failures:
+                    self.log(
+                        f'sacct failed ({self._num_sacct_failures}/'
+                        f'{self._max_sacct_failures}): {e.stderr}',
+                        level=logging.WARNING
+                    )
+                    return
+                else:
+                    raise e
 
         self._update_state_count += 1
 
@@ -508,7 +525,7 @@ class SlurmJobScheduler(sched.JobScheduler):
 
         t_pending = time.time() - job.submit_time
         if t_pending >= job.max_pending_time:
-            self.log(f'maximum pending time for job exceeded; cancelling it')
+            self.log('maximum pending time for job exceeded; cancelling it')
             self.cancel(job)
             job._exception = JobError('maximum pending time exceeded',
                                       job.jobid)
diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json
index 92a5c68d..6fd7c1bb 100644
--- a/reframe/schemas/config.json
+++ b/reframe/schemas/config.json
@@ -106,17 +106,18 @@
         "sched_options": {
             "type": "object",
             "properties": {
-                "sched_access_in_submit": {"type": "boolean"},
                 "hosts": {
                     "type": "array",
                     "items": {"type": "string"}
                 },
                 "ignore_reqnodenotavail": {"type": "boolean"},
                 "job_submit_timeout": {"type": "number"},
+                "max_sacct_failures": {"type": "number"},
                 "resubmit_on_errors": {
                     "type": "array",
                     "items": {"type": "string"}
                 },
+                "sched_access_in_submit": {"type": "boolean"},
                 "unqualified_hostnames": {"type": "boolean"},
                 "use_nodes_option": {"type": "boolean"}
             }
@@ -678,10 +679,11 @@
         "systems/partitions/time_limit": null,
         "systems/partitions/devices": [],
         "systems/partitions/extras": {},
-        "systems*/sched_options/sched_access_in_submit": false,
-        "systems*/sched_options/ssh_hosts": [],
         "systems*/sched_options/ignore_reqnodenotavail": false,
         "systems*/sched_options/job_submit_timeout": 60,
+        "systems*/sched_options/max_sacct_failures": 3,
+        "systems*/sched_options/sched_access_in_submit": false,
+        "systems*/sched_options/ssh_hosts": [],
         "systems*/sched_options/resubmit_on_errors": [],
         "systems*/sched_options/unqualified_hostnames": false,
         "systems*/sched_options/use_nodes_option": false


Test patch:
diff --git a/unittests/test_config.py b/unittests/test_config.py
index 755f8aec..265f4ac3 100644
--- a/unittests/test_config.py
+++ b/unittests/test_config.py
@@ -325,9 +325,15 @@ def test_select_subconfig(site_config):
     assert site_config.get(
         'systems/0/sched_options/resubmit_on_errors'
     ) == []
+    assert site_config.get(
+        'systems/0/sched_options/max_sacct_failures'
+    ) == 3
     assert site_config.get(
         'systems/0/partitions/@gpu/sched_options/resubmit_on_errors'
     ) == []
+    assert site_config.get(
+        'systems/0/partitions/@gpu/sched_options/max_sacct_failures'
+    ) == 3
     assert site_config.get('environments/@PrgEnv-gnu/cc') == 'cc'
     assert site_config.get('environments/1/cxx') == 'CC'
     assert site_config.get('general/0/check_search_path') == ['c:d']