Repository: Parsl/parsl

Issue:
process_worker_pool can hang if a worker dies mid-task
I've encountered a situation where a process_worker worker process is getting killed during task execution/result serialisation.

That's pretty obscure; however process_worker_pool as a whole does not continue working properly - that task is never seen as failed by the main parsl process (until the whole pool dies for example by running out of batch job walltime)


Discussion:
Another case I think I have noticed this happening is when running inside `srun` on `cori`, when a bash app tries to run, and that app fails, then this sort of thing happens which I think might be killing the corresponding worker process without the manager noticing.

```
cat runinfo/025/task_logs/0000/task_0004_ingest.stderr
--> executable follows <--
ingestDriver.py /global/cscratch1/sd/bxc/parslTest/test0 /global/projecta/projectdirs/lsst/production/DC2_ImSim/Run2.1.1i/sim/agn-test/00385844to00445379/00425529/lsst_a_425529_R03_S21_y.fits --cores 1 --mode link --output /global/cscratch1/sd/bxc/parslTest/test0 -c clobber=True allowError=True register.ignore=True
--> end executable <--
/tmp/tmpb2spzmc5: line 618: warning: setlocale: LC_CTYPE: cannot change locale (C.UTF-8)
/tmp/tmpb2spzmc5: line 618: warning: setlocale: LC_CTYPE: cannot change locale (C.UTF-8)
[mpiexec@nid01343] HYDU_create_process (utils/launch/launch.c:75): execvp error on file srun (No such file or directory)
[mpiexec@nid01343] HYDU_sock_write (utils/sock/sock.c:294): write error (Bad file descriptor)
[mpiexec@nid01343] HYD_pmcd_pmiserv_send_signal (pm/pmiserv/pmiserv_cb.c:177): unable to write data to proxy
[mpiexec@nid01343] ui_cmd_cb (pm/pmiserv/pmiserv_pmci.c:79): unable to send signal downstream
[mpiexec@nid01343] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status
[mpiexec@nid01343] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:198): error waiting for event
[mpiexec@nid01343] main (ui/mpich/mpiexec.c:340): process manager error waiting for completion
```

I am also seeing this running Coffea tasks.
We need more machinery in the pool to deal with this properly-- we can see that a process died, but we have no way of knowing which task it was working on. One option is adding a shared dict via which the manager can track who is working on what.

Resolving patch:
diff --git a/parsl/executors/high_throughput/errors.py b/parsl/executors/high_throughput/errors.py
new file mode 100644
--- /dev/null
+++ b/parsl/executors/high_throughput/errors.py
@@ -0,0 +1,12 @@
+class WorkerLost(Exception):
+    """Exception raised when a worker is lost
+    """
+    def __init__(self, worker_id, hostname):
+        self.worker_id = worker_id
+        self.hostname = hostname
+
+    def __repr__(self):
+        return "Task failure due to loss of worker {} on host {}".format(self.worker_id, self.hostname)
+
+    def __str__(self):
+        return self.__repr__()
diff --git a/parsl/executors/high_throughput/process_worker_pool.py b/parsl/executors/high_throughput/process_worker_pool.py
--- a/parsl/executors/high_throughput/process_worker_pool.py
+++ b/parsl/executors/high_throughput/process_worker_pool.py
@@ -19,6 +19,7 @@
 
 from parsl.version import VERSION as PARSL_VERSION
 from parsl.app.errors import RemoteExceptionWrapper
+from parsl.executors.high_throughput.errors import WorkerLost
 import multiprocessing
 
 from ipyparallel.serialize import unpack_apply_message  # pack_apply_message,
@@ -302,6 +303,47 @@ def push_results(self, kill_event):
 
         logger.critical("[RESULT_PUSH_THREAD] Exiting")
 
+    def worker_watchdog(self, kill_event):
+        """ Listens on the pending_result_queue and sends out results via 0mq
+
+        Parameters:
+        -----------
+        kill_event : threading.Event
+              Event to let the thread know when it is time to die.
+        """
+
+        logger.debug("[WORKER_WATCHDOG_THREAD] Starting thread")
+
+        while not kill_event.is_set():
+            for worker_id, p in self.procs.items():
+                if not p.is_alive():
+                    logger.info("[WORKER_WATCHDOG_THREAD] Worker {} has died".format(worker_id))
+                    try:
+                        task = self._tasks_in_progress.pop(worker_id)
+                        logger.info("[WORKER_WATCHDOG_THREAD] Worker {} was busy when it died".format(worker_id))
+                        try:
+                            raise WorkerLost(worker_id, platform.node())
+                        except Exception:
+                            logger.info("[WORKER_WATCHDOG_THREAD] Putting exception for task {} in the pending result queue".format(task['task_id']))
+                            result_package = {'task_id': task['task_id'], 'exception': serialize_object(RemoteExceptionWrapper(*sys.exc_info()))}
+                            pkl_package = pickle.dumps(result_package)
+                            self.pending_result_queue.put(pkl_package)
+                    except KeyError:
+                        logger.info("[WORKER_WATCHDOG_THREAD] Worker {} was not busy when it died".format(worker_id))
+
+                    p = multiprocessing.Process(target=worker, args=(worker_id,
+                                                                     self.uid,
+                                                                     self.pending_task_queue,
+                                                                     self.pending_result_queue,
+                                                                     self.ready_worker_queue,
+                                                                     self._tasks_in_progress
+                                                                 ), name="HTEX-Worker-{}".format(worker_id))
+                    self.procs[worker_id] = p
+                    logger.info("[WORKER_WATCHDOG_THREAD] Worker {} has been restarted".format(worker_id))
+                time.sleep(self.poll_period)
+
+        logger.critical("[WORKER_WATCHDOG_THREAD] Exiting")
+
     def start(self):
         """ Start the worker processes.
 
@@ -309,6 +351,7 @@ def start(self):
         """
         start = time.time()
         self._kill_event = threading.Event()
+        self._tasks_in_progress = multiprocessing.Manager().dict()
 
         self.procs = {}
         for worker_id in range(self.worker_count):
@@ -317,6 +360,7 @@ def start(self):
                                                              self.pending_task_queue,
                                                              self.pending_result_queue,
                                                              self.ready_worker_queue,
+                                                             self._tasks_in_progress
                                                          ), name="HTEX-Worker-{}".format(worker_id))
             p.start()
             self.procs[worker_id] = p
@@ -329,8 +373,12 @@ def start(self):
         self._result_pusher_thread = threading.Thread(target=self.push_results,
                                                       args=(self._kill_event,),
                                                       name="Result-Pusher")
+        self._worker_watchdog_thread = threading.Thread(target=self.worker_watchdog,
+                                                        args=(self._kill_event,),
+                                                        name="worker-watchdog")
         self._task_puller_thread.start()
         self._result_pusher_thread.start()
+        self._worker_watchdog_thread.start()
 
         logger.info("Loop start")
 
@@ -341,6 +389,7 @@ def start(self):
 
         self._task_puller_thread.join()
         self._result_pusher_thread.join()
+        self._worker_watchdog_thread.join()
         for proc_id in self.procs:
             self.procs[proc_id].terminate()
             logger.critical("Terminating worker {}:{}".format(self.procs[proc_id],
@@ -394,7 +443,7 @@ def execute_task(bufs):
         return user_ns.get(resultname)
 
 
-def worker(worker_id, pool_id, task_queue, result_queue, worker_queue):
+def worker(worker_id, pool_id, task_queue, result_queue, worker_queue, tasks_in_progress):
     """
 
     Put request token into queue
@@ -417,6 +466,7 @@ def worker(worker_id, pool_id, task_queue, result_queue, worker_queue):
 
         # The worker will receive {'task_id':<tid>, 'buffer':<buf>}
         req = task_queue.get()
+        tasks_in_progress[worker_id] = req
         tid = req['task_id']
         logger.info("Received task {}".format(tid))
 
@@ -429,7 +479,8 @@ def worker(worker_id, pool_id, task_queue, result_queue, worker_queue):
         try:
             result = execute_task(req['buffer'])
             serialized_result = serialize_object(result)
-        except Exception:
+        except Exception as e:
+            logger.info('Caught an exception: {}'.format(e))
             result_package = {'task_id': tid, 'exception': serialize_object(RemoteExceptionWrapper(*sys.exc_info()))}
         else:
             result_package = {'task_id': tid, 'result': serialized_result}
@@ -439,6 +490,7 @@ def worker(worker_id, pool_id, task_queue, result_queue, worker_queue):
         pkl_package = pickle.dumps(result_package)
 
         result_queue.put(pkl_package)
+        tasks_in_progress.pop(worker_id)
 
 
 def start_file_logger(filename, rank, name='parsl', level=logging.DEBUG, format_string=None):


Test patch:
diff --git a/parsl/tests/configs/htex_local.py b/parsl/tests/configs/htex_local.py
--- a/parsl/tests/configs/htex_local.py
+++ b/parsl/tests/configs/htex_local.py
@@ -5,19 +5,24 @@
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 
-config = Config(
-    executors=[
-        HighThroughputExecutor(
-            label="htex_Local",
-            worker_debug=True,
-            cores_per_worker=1,
-            provider=LocalProvider(
-                channel=LocalChannel(),
-                init_blocks=1,
-                max_blocks=1,
-                launcher=SimpleLauncher(),
-            ),
-        )
-    ],
-    strategy=None,
-)
+
+def fresh_config():
+    return Config(
+        executors=[
+            HighThroughputExecutor(
+                label="htex_local",
+                worker_debug=True,
+                cores_per_worker=1,
+                provider=LocalProvider(
+                    channel=LocalChannel(),
+                    init_blocks=1,
+                    max_blocks=1,
+                    launcher=SimpleLauncher(),
+                ),
+            )
+        ],
+        strategy=None,
+    )
+
+
+config = fresh_config()
diff --git a/parsl/tests/test_error_handling/test_htex_worker_failure.py b/parsl/tests/test_error_handling/test_htex_worker_failure.py
new file mode 100644
--- /dev/null
+++ b/parsl/tests/test_error_handling/test_htex_worker_failure.py
@@ -0,0 +1,31 @@
+import pytest
+
+import parsl
+from parsl.app.app import python_app
+from parsl.tests.configs.htex_local import fresh_config
+
+from parsl.executors.high_throughput.errors import WorkerLost
+
+
+def local_setup():
+    config = fresh_config()
+    config.executors[0].poll_period = 1
+    config.executors[0].max_workers = 1
+    parsl.load(config)
+
+
+def local_teardown():
+    parsl.clear()
+
+
+@python_app
+def kill_worker():
+    import sys
+    sys.exit(2)
+
+
+@pytest.mark.local
+def test_htex_worker_failure():
+    with pytest.raises(WorkerLost):
+        f = kill_worker()
+        f.result()