Repository: pulp/pulpcore

Issue:
All workers idle while a task is pending
**Version**
At least 3.14.14 used in katello

**Describe the bug**
Workers can all enter sleep state and miss the wakeup notification leading to pending tasks while all workers idle. Dispatching a new tasks unfreezes the system again.

**To Reproduce**
https://bugzilla.redhat.com/show_bug.cgi?id=2050440

**Expected behavior**
At least one worker is expected to pick up the next pending task.

**Additional context**
https://bugzilla.redhat.com/show_bug.cgi?id=2050440



Discussion:

Resolving patch:
diff --git a/pulpcore/tasking/pulpcore_worker.py b/pulpcore/tasking/pulpcore_worker.py
--- a/pulpcore/tasking/pulpcore_worker.py
+++ b/pulpcore/tasking/pulpcore_worker.py
@@ -275,21 +275,24 @@ def sleep(self):
         """Wait for signals on the wakeup channel while heart beating."""
 
         _logger.debug(_("Worker %s entering sleep state."), self.name)
+        wakeup = False
         while not self.shutdown_requested:
+            # Handle all notifications before sleeping in `select`
+            while connection.connection.notifies:
+                item = connection.connection.notifies.pop(0)
+                if item.channel == "pulp_worker_wakeup":
+                    _logger.debug(_("Worker %s received wakeup call."), self.name)
+                    wakeup = True
+                # ignore all other notifications
+            if wakeup:
+                break
+
             r, w, x = select.select(
                 [self.sentinel, connection.connection], [], [], self.heartbeat_period
             )
             self.beat()
             if connection.connection in r:
                 connection.connection.poll()
-                if any(
-                    (
-                        item.channel == "pulp_worker_wakeup"
-                        for item in connection.connection.notifies
-                    )
-                ):
-                    connection.connection.notifies.clear()
-                    break
             if self.sentinel in r:
                 os.read(self.sentinel, 256)
 
@@ -307,6 +310,17 @@ def supervise_task(self, task):
             task_process = Process(target=_perform_task, args=(task.pk, task_working_dir_rel_path))
             task_process.start()
             while True:
+                # Handle all notifications before sleeping in `select`
+                while connection.connection.notifies:
+                    item = connection.connection.notifies.pop(0)
+                    if item.channel == "pulp_worker_cancel" and item.payload == str(task.pk):
+                        _logger.info(_("Received signal to cancel current task %s."), task.pk)
+                        os.kill(task_process.pid, signal.SIGUSR1)
+                        cancel_state = TASK_STATES.CANCELED
+                    # ignore all other notifications
+                if cancel_state:
+                    break
+
                 r, w, x = select.select(
                     [self.sentinel, connection.connection, task_process.sentinel],
                     [],
@@ -316,17 +330,6 @@ def supervise_task(self, task):
                 self.beat()
                 if connection.connection in r:
                     connection.connection.poll()
-                    if any(
-                        (
-                            item.channel == "pulp_worker_cancel" and item.payload == str(task.pk)
-                            for item in connection.connection.notifies
-                        )
-                    ):
-                        connection.connection.notifies.clear()
-                        _logger.info(_("Received signal to cancel current task %s."), task.pk)
-                        os.kill(task_process.pid, signal.SIGUSR1)
-                        cancel_state = TASK_STATES.CANCELED
-                        break
                 if task_process.sentinel in r:
                     if not task_process.is_alive():
                         break


Test patch: