Repository: Parsl/parsl

Issue:
JobStatus is missing relevant stdout/err summary in SlurmProvider
**Describe the bug**

The `JobStatus` object reported by the provider on a `provider.status` call should be reporting information relevant to the submit scipts failure for better debugging. While the `LocalProvider` does this appropriately most cluster providers do not.

The SlurmProvider reports no stdout/stderr path when a job reaches a terminal state, and this means that the `stdout_summary` and `stderr_summary` is not filled in leading to poor exception messages., for eg:

```
ERROR    parsl.executors.status_handling:status_handling.py:141 Setting bad state due to exception
parsl.jobs.errors.TooManyJobFailuresError: Error 1:

INFO     parsl.dataflow.dflow:dflow.py:305 Task 0 try 0 failed with exception of type BadStateException
ERROR    parsl.dataflow.dflow:dflow.py:351 Task 0 failed after 0 retry attempts
Traceback (most recent call last):
  File "/home1/02551/yadunand/parsl/parsl/dataflow/dflow.py", line 302, in handle_exec_update
    res = self._unwrap_remote_exception_wrapper(future)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home1/02551/yadunand/parsl/parsl/dataflow/dflow.py", line 570, in _unwrap_remote_exception_wrapper
    result = future.result()
             ^^^^^^^^^^^^^^^
  File "/home1/02551/yadunand/anaconda3/envs/parsl_py311/lib/python3.11/concurrent/futures/_base.py", line 449, in result
    return self.__get_result()
           ^^^^^^^^^^^^^^^^^^^
  File "/home1/02551/yadunand/anaconda3/envs/parsl_py311/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
parsl.executors.errors.BadStateException: Executor HTEX failed due to: Error 1:
```

The LocalProvider on the other hand reports much more useful error messages, like this:
```
1707244405.623917 2024-02-06 12:33:25 MainProcess-90952 JobStatusPoller-Timer-Thread-140483421832336-13085667328 parsl.executors.status_handling:141 set_bad_state_and_fail_all ERROR: Setting bad state due to exception
parsl.jobs.errors.TooManyJobFailuresError: Error 1:
    EXIT CODE: 0
    STDERR:
CommandNotFoundError: Your shell has not been properly configured to use 'conda deactivate'.
To initialize your shell, run

    $ conda init <SHELL_NAME>

Currently supported shells are:
  - bash
  - fish
  - tcsh
  - xonsh
  - zsh
  - powershell

See 'conda init --help' for more information and options.

IMPORTANT: You may need to close and restart your shell after running 'conda init'.


/tmp/test_htex/001/submit_scripts/parsl.HTEX.block-0.1707244400.5608902.sh: line 1: which: No such file or directory
```

**Expected behavior**

All Providers should be reporting the `stdout` and `stderr` from the submit scripts as part of the exception when a job fails.



Discussion:
crossref #2949 as a similar-looking providers-losing-error-flow issue

Resolving patch:
diff --git a/parsl/providers/slurm/slurm.py b/parsl/providers/slurm/slurm.py
--- a/parsl/providers/slurm/slurm.py
+++ b/parsl/providers/slurm/slurm.py
@@ -188,14 +188,18 @@ def _status(self):
                 logger.warning(f"Slurm status {slurm_state} is not recognized")
             status = translate_table.get(slurm_state, JobState.UNKNOWN)
             logger.debug("Updating job {} with slurm status {} to parsl state {!s}".format(job_id, slurm_state, status))
-            self.resources[job_id]['status'] = JobStatus(status)
+            self.resources[job_id]['status'] = JobStatus(status,
+                                                         stdout_path=self.resources[job_id]['job_stdout_path'],
+                                                         stderr_path=self.resources[job_id]['job_stderr_path'])
             jobs_missing.remove(job_id)
 
         # squeue does not report on jobs that are not running. So we are filling in the
         # blanks for missing jobs, we might lose some information about why the jobs failed.
         for missing_job in jobs_missing:
             logger.debug("Updating missing job {} to completed status".format(missing_job))
-            self.resources[missing_job]['status'] = JobStatus(JobState.COMPLETED)
+            self.resources[missing_job]['status'] = JobStatus(JobState.COMPLETED,
+                                                              stdout_path=self.resources[missing_job]['job_stdout_path'],
+                                                              stderr_path=self.resources[missing_job]['job_stderr_path'])
 
     def submit(self, command: str, tasks_per_node: int, job_name="parsl.slurm") -> str:
         """Submit the command as a slurm job.
@@ -226,8 +230,11 @@ def submit(self, command: str, tasks_per_node: int, job_name="parsl.slurm") -> s
 
         job_name = "{0}.{1}".format(job_name, time.time())
 
-        script_path = "{0}/{1}.submit".format(self.script_dir, job_name)
+        assert self.script_dir, "Expected script_dir to be set"
+        script_path = os.path.join(self.script_dir, job_name)
         script_path = os.path.abspath(script_path)
+        job_stdout_path = script_path + ".stdout"
+        job_stderr_path = script_path + ".stderr"
 
         logger.debug("Requesting one block with {} nodes".format(self.nodes_per_block))
 
@@ -239,6 +246,8 @@ def submit(self, command: str, tasks_per_node: int, job_name="parsl.slurm") -> s
         job_config["scheduler_options"] = scheduler_options
         job_config["worker_init"] = worker_init
         job_config["user_script"] = command
+        job_config["job_stdout_path"] = job_stdout_path
+        job_config["job_stderr_path"] = job_stderr_path
 
         # Wrap the command
         job_config["user_script"] = self.launcher(command,
@@ -262,7 +271,11 @@ def submit(self, command: str, tasks_per_node: int, job_name="parsl.slurm") -> s
                 match = re.match(self.regex_job_id, line)
                 if match:
                     job_id = match.group("id")
-                    self.resources[job_id] = {'job_id': job_id, 'status': JobStatus(JobState.PENDING)}
+                    self.resources[job_id] = {'job_id': job_id,
+                                              'status': JobStatus(JobState.PENDING),
+                                              'job_stdout_path': job_stdout_path,
+                                              'job_stderr_path': job_stderr_path,
+                                              }
                     return job_id
             else:
                 logger.error("Could not read job ID from submit command standard output.")
diff --git a/parsl/providers/slurm/template.py b/parsl/providers/slurm/template.py
--- a/parsl/providers/slurm/template.py
+++ b/parsl/providers/slurm/template.py
@@ -1,8 +1,8 @@
 template_string = '''#!/bin/bash
 
 #SBATCH --job-name=${jobname}
-#SBATCH --output=${submit_script_dir}/${jobname}.submit.stdout
-#SBATCH --error=${submit_script_dir}/${jobname}.submit.stderr
+#SBATCH --output=${job_stdout_path}
+#SBATCH --error=${job_stderr_path}
 #SBATCH --nodes=${nodes}
 #SBATCH --time=${walltime}
 #SBATCH --ntasks-per-node=${tasks_per_node}


Test patch:
diff --git a/parsl/tests/test_providers/test_slurm_template.py b/parsl/tests/test_providers/test_slurm_template.py
new file mode 100644
--- /dev/null
+++ b/parsl/tests/test_providers/test_slurm_template.py
@@ -0,0 +1,29 @@
+import logging
+import random
+
+from unittest import mock
+import pytest
+
+from parsl.channels import LocalChannel
+from parsl.providers import SlurmProvider
+
+
+@pytest.mark.local
+def test_submit_script_basic(tmp_path):
+    """Test slurm resources table"""
+
+    provider = SlurmProvider(
+        partition="debug", channel=LocalChannel(script_dir=tmp_path)
+    )
+    provider.script_dir = tmp_path
+    job_id = str(random.randint(55000, 59000))
+    provider.execute_wait = mock.MagicMock(spec=SlurmProvider.execute_wait)
+    provider.execute_wait.return_value = (0, f"Submitted batch job {job_id}", "")
+    result_job_id = provider.submit("test", tasks_per_node=1)
+    assert job_id == result_job_id
+    provider.execute_wait.assert_called()
+    assert job_id in provider.resources
+
+    job_info = provider.resources[job_id]
+    assert "job_stdout_path" in job_info
+    assert "job_stderr_path" in job_info