Repository: pulp/pulpcore

Issue:
All workers idle while a task is pending
**Version**
At least 3.14.14 used in katello

**Describe the bug**
Workers can all enter sleep state and miss the wakeup notification leading to pending tasks while all workers idle. Dispatching a new tasks unfreezes the system again.

**To Reproduce**
https://bugzilla.redhat.com/show_bug.cgi?id=2050440

**Expected behavior**
At least one worker is expected to pick up the next pending task.

**Additional context**
https://bugzilla.redhat.com/show_bug.cgi?id=2050440



Discussion:

Resolving patch:
diff --git a/pulpcore/tasking/pulpcore_worker.py b/pulpcore/tasking/pulpcore_worker.py
--- a/pulpcore/tasking/pulpcore_worker.py
+++ b/pulpcore/tasking/pulpcore_worker.py
@@ -102,26 +102,39 @@ def beat(self):
     def notify_workers(self):
         self.cursor.execute("NOTIFY pulp_worker_wakeup")
 
-    def cancel_abandoned_task(self, task):
+    def cancel_abandoned_task(self, task, final_state, reason=None):
         """Cancel and clean up an abandoned task.
 
-        This function must only be called while holding the lock for that task.
+        This function must only be called while holding the lock for that task. It is a no-op if
+        the task is neither in "running" nor "canceling" state.
 
         Return ``True`` if the task was actually canceled, ``False`` otherwise.
         """
         # A task is considered abandoned when in running state, but no worker holds its lock
-        _logger.info(_("Cleaning up and canceling Task %s"), task.pk)
         Task.objects.filter(pk=task.pk, state=TASK_STATES.RUNNING).update(
             state=TASK_STATES.CANCELING
         )
         task.refresh_from_db()
         if task.state == TASK_STATES.CANCELING:
+            if reason:
+                _logger.info(
+                    _("Cleaning up task %s and marking as %s. Reason: %s"),
+                    task.pk,
+                    final_state,
+                    reason,
+                )
+            else:
+                _logger.info(_("Cleaning up task %s and marking as %s."), task.pk, final_state)
             _delete_incomplete_resources(task)
             if task.reserved_resources_record:
                 self.notify_workers()
-            Task.objects.filter(pk=task.pk, state=TASK_STATES.CANCELING).update(
-                state=TASK_STATES.CANCELED
-            )
+            task_data = {
+                "state": final_state,
+                "finished_at": timezone.now(),
+            }
+            if reason:
+                task_data["error"] = {"reason": reason}
+            Task.objects.filter(pk=task.pk, state=TASK_STATES.CANCELING).update(**task_data)
             return True
         return False
 
@@ -160,7 +173,9 @@ def iter_tasks(self):
                             continue
                     if task.state in [TASK_STATES.RUNNING, TASK_STATES.CANCELING]:
                         # A running task without a lock must be abandoned
-                        if self.cancel_abandoned_task(task):
+                        if self.cancel_abandoned_task(
+                            task, TASK_STATES.FAILED, "Worker has gone missing."
+                        ):
                             # Continue looking for the next task
                             # without considering this tasks resources
                             # as we just released them
@@ -205,26 +220,26 @@ def sleep(self):
         """Wait for signals on the wakeup channel while heart beating."""
 
         _logger.debug(_("Worker %s entering sleep state."), self.name)
-        # Subscribe to "pulp_worker_wakeup"
-        self.cursor.execute("LISTEN pulp_worker_wakeup")
+        wakeup = False
         while not self.shutdown_requested:
+            # Handle all notifications before sleeping in `select`
+            while connection.connection.notifies:
+                item = connection.connection.notifies.pop(0)
+                if item.channel == "pulp_worker_wakeup":
+                    _logger.debug(_("Worker %s received wakeup call."), self.name)
+                    wakeup = True
+                # ignore all other notifications
+            if wakeup:
+                break
+
             r, w, x = select.select(
                 [self.sentinel, connection.connection], [], [], self.heartbeat_period
             )
             self.beat()
             if connection.connection in r:
                 connection.connection.poll()
-                if any(
-                    (
-                        item.channel == "pulp_worker_wakeup"
-                        for item in connection.connection.notifies
-                    )
-                ):
-                    connection.connection.notifies.clear()
-                    break
             if self.sentinel in r:
                 os.read(self.sentinel, 256)
-        self.cursor.execute("UNLISTEN pulp_worker_wakeup")
 
     def supervise_task(self, task):
         """Call and supervise the task process while heart beating.
@@ -232,13 +247,25 @@ def supervise_task(self, task):
         This function must only be called while holding the lock for that task."""
 
         self.task_grace_timeout = TASK_GRACE_INTERVAL
-        self.cursor.execute("LISTEN pulp_worker_cancel")
         task.worker = self.worker
         task.save(update_fields=["worker"])
+        cancel_state = None
+        cancel_reason = None
         with TemporaryDirectory(dir=".") as task_working_dir_rel_path:
             task_process = Process(target=_perform_task, args=(task.pk, task_working_dir_rel_path))
             task_process.start()
             while True:
+                # Handle all notifications before sleeping in `select`
+                while connection.connection.notifies:
+                    item = connection.connection.notifies.pop(0)
+                    if item.channel == "pulp_worker_cancel" and item.payload == str(task.pk):
+                        _logger.info(_("Received signal to cancel current task %s."), task.pk)
+                        os.kill(task_process.pid, signal.SIGUSR1)
+                        cancel_state = TASK_STATES.CANCELED
+                    # ignore all other notifications
+                if cancel_state:
+                    break
+
                 r, w, x = select.select(
                     [self.sentinel, connection.connection, task_process.sentinel],
                     [],
@@ -248,16 +275,6 @@ def supervise_task(self, task):
                 self.beat()
                 if connection.connection in r:
                     connection.connection.poll()
-                    if any(
-                        (
-                            item.channel == "pulp_worker_cancel" and item.payload == str(task.pk)
-                            for item in connection.connection.notifies
-                        )
-                    ):
-                        connection.connection.notifies.clear()
-                        _logger.info(_("Received signal to cancel current task %s."), task.pk)
-                        os.kill(task_process.pid, signal.SIGUSR1)
-                        break
                 if task_process.sentinel in r:
                     if not task_process.is_alive():
                         break
@@ -271,23 +288,29 @@ def supervise_task(self, task):
                     else:
                         _logger.info(_("Aborting current task %s due to worker shutdown."), task.pk)
                         os.kill(task_process.pid, signal.SIGUSR1)
-                        task_process.join()
-                        self.cancel_abandoned_task(task)
+                        cancel_state = TASK_STATES.FAILED
+                        cancel_reason = "Aborted during worker shutdown."
                         break
             task_process.join()
+            if cancel_state:
+                self.cancel_abandoned_task(task, cancel_state, cancel_reason)
         if task.reserved_resources_record:
             self.notify_workers()
-        self.cursor.execute("UNLISTEN pulp_worker_cancel")
 
     def run_forever(self):
         with WorkerDirectory(self.name):
             signal.signal(signal.SIGINT, self._signal_handler)
             signal.signal(signal.SIGTERM, self._signal_handler)
+            # Subscribe to pgsql channels
+            self.cursor.execute("LISTEN pulp_worker_wakeup")
+            self.cursor.execute("LISTEN pulp_worker_cancel")
             while not self.shutdown_requested:
                 for task in self.iter_tasks():
                     self.supervise_task(task)
                 if not self.shutdown_requested:
                     self.sleep()
+            self.cursor.execute("UNLISTEN pulp_worker_cancel")
+            self.cursor.execute("UNLISTEN pulp_worker_wakeup")
             self.shutdown()
 
 


Test patch: