Repository: pulp/pulpcore

Issue:
All workers idle while a task is pending
**Version**
At least 3.14.14 used in katello

**Describe the bug**
Workers can all enter sleep state and miss the wakeup notification leading to pending tasks while all workers idle. Dispatching a new tasks unfreezes the system again.

**To Reproduce**
https://bugzilla.redhat.com/show_bug.cgi?id=2050440

**Expected behavior**
At least one worker is expected to pick up the next pending task.

**Additional context**
https://bugzilla.redhat.com/show_bug.cgi?id=2050440



Discussion:

Resolving patch:
diff --git a/pulpcore/tasking/pulpcore_worker.py b/pulpcore/tasking/pulpcore_worker.py
--- a/pulpcore/tasking/pulpcore_worker.py
+++ b/pulpcore/tasking/pulpcore_worker.py
@@ -247,26 +247,26 @@ def sleep(self):
         """Wait for signals on the wakeup channel while heart beating."""
 
         _logger.debug(_("Worker %s entering sleep state."), self.name)
-        # Subscribe to "pulp_worker_wakeup"
-        self.cursor.execute("LISTEN pulp_worker_wakeup")
+        wakeup = False
         while not self.shutdown_requested:
+            # Handle all notifications before sleeping in `select`
+            while connection.connection.notifies:
+                item = connection.connection.notifies.pop(0)
+                if item.channel == "pulp_worker_wakeup":
+                    _logger.debug(_("Worker %s received wakeup call."), self.name)
+                    wakeup = True
+                # ignore all other notifications
+            if wakeup:
+                break
+
             r, w, x = select.select(
                 [self.sentinel, connection.connection], [], [], self.heartbeat_period
             )
             self.beat()
             if connection.connection in r:
                 connection.connection.poll()
-                if any(
-                    (
-                        item.channel == "pulp_worker_wakeup"
-                        for item in connection.connection.notifies
-                    )
-                ):
-                    connection.connection.notifies.clear()
-                    break
             if self.sentinel in r:
                 os.read(self.sentinel, 256)
-        self.cursor.execute("UNLISTEN pulp_worker_wakeup")
 
     def supervise_task(self, task):
         """Call and supervise the task process while heart beating.
@@ -274,7 +274,6 @@ def supervise_task(self, task):
         This function must only be called while holding the lock for that task."""
 
         self.task_grace_timeout = TASK_GRACE_INTERVAL
-        self.cursor.execute("LISTEN pulp_worker_cancel")
         task.worker = self.worker
         task.save(update_fields=["worker"])
         cancel_state = None
@@ -283,6 +282,17 @@ def supervise_task(self, task):
             task_process = Process(target=_perform_task, args=(task.pk, task_working_dir_rel_path))
             task_process.start()
             while True:
+                # Handle all notifications before sleeping in `select`
+                while connection.connection.notifies:
+                    item = connection.connection.notifies.pop(0)
+                    if item.channel == "pulp_worker_cancel" and item.payload == str(task.pk):
+                        _logger.info(_("Received signal to cancel current task %s."), task.pk)
+                        os.kill(task_process.pid, signal.SIGUSR1)
+                        cancel_state = TASK_STATES.CANCELED
+                    # ignore all other notifications
+                if cancel_state:
+                    break
+
                 r, w, x = select.select(
                     [self.sentinel, connection.connection, task_process.sentinel],
                     [],
@@ -292,17 +302,6 @@ def supervise_task(self, task):
                 self.beat()
                 if connection.connection in r:
                     connection.connection.poll()
-                    if any(
-                        (
-                            item.channel == "pulp_worker_cancel" and item.payload == str(task.pk)
-                            for item in connection.connection.notifies
-                        )
-                    ):
-                        connection.connection.notifies.clear()
-                        _logger.info(_("Received signal to cancel current task %s."), task.pk)
-                        os.kill(task_process.pid, signal.SIGUSR1)
-                        cancel_state = TASK_STATES.CANCELED
-                        break
                 if task_process.sentinel in r:
                     if not task_process.is_alive():
                         break
@@ -324,17 +323,21 @@ def supervise_task(self, task):
                 self.cancel_abandoned_task(task, cancel_state, cancel_reason)
         if task.reserved_resources_record:
             self.notify_workers()
-        self.cursor.execute("UNLISTEN pulp_worker_cancel")
 
     def run_forever(self):
         with WorkerDirectory(self.name):
             signal.signal(signal.SIGINT, self._signal_handler)
             signal.signal(signal.SIGTERM, self._signal_handler)
+            # Subscribe to pgsql channels
+            self.cursor.execute("LISTEN pulp_worker_wakeup")
+            self.cursor.execute("LISTEN pulp_worker_cancel")
             while not self.shutdown_requested:
                 for task in self.iter_tasks():
                     self.supervise_task(task)
                 if not self.shutdown_requested:
                     self.sleep()
+            self.cursor.execute("UNLISTEN pulp_worker_cancel")
+            self.cursor.execute("UNLISTEN pulp_worker_wakeup")
             self.shutdown()
 
 


Test patch: