Repository: pulp/pulpcore

Issue:
Task cancellation gets stuck
**Version**
Pulpcore 3.16 via Satellite 6.11

**Describe the bug**
If "Reclaim Space" (Actions::Pulp3::CapsuleContent::ReclaimSpace) action is executed and canceled after that task will get stuck in canceling not only in foreman tasks but also in pulp3. 

The task was apparently stuck in "canceling" state for more than 6 days.

**To Reproduce**
How reproducible:
Always

Steps to Reproduce:
1. Execute "Reclaim Space" on Satellite
2. Cancel task (Actions::Pulp3::CapsuleContent::ReclaimSpace)

**Expected results:**
Task will get canceled

**Actual results:**
Task is stuck in foreman-tasks and in pulp3

**Additional context**
https://bugzilla.redhat.com/show_bug.cgi?id=2143290



Discussion:
This was independently reproduced on a lab system
Removing triage tag because it was reproduced, adding prio tag because it can block up the tasking system.
As far as i heard, rebooting _all_ workers unblocks the tasking system. I would wonder if rebooting just the worker stuck on that task would be sufficient too.
Can we add logging from the worker when the cancel happens?
I haven't been able to recreate this in a just-pulp env yet. What would be *really* useful, is the journalctl output from the 60 sec around the time the cancel was *issued*. Looking at the sys.exit() call that ends a worker that's been cancelled, I can imagine a series of Unfortunate Events that would result in the worker-process not actually exiting - but I'd think they'd all leave traces of unusual exceptions in the logs.

Investigation continues.

Resolving patch:
diff --git a/pulpcore/tasking/pulpcore_worker.py b/pulpcore/tasking/pulpcore_worker.py
--- a/pulpcore/tasking/pulpcore_worker.py
+++ b/pulpcore/tasking/pulpcore_worker.py
@@ -53,7 +53,11 @@
 _logger = logging.getLogger(__name__)
 random.seed()
 
+# Number of heartbeats for a task to finish on graceful worker shutdown (approx)
 TASK_GRACE_INTERVAL = 3
+# Number of heartbeats between attempts to kill the subprocess (approx)
+TASK_KILL_INTERVAL = 1
+# Number of heartbeats between cleaning up worker processes (approx)
 WORKER_CLEANUP_INTERVAL = 100
 # Randomly chosen
 TASK_SCHEDULING_LOCK = 42
@@ -147,6 +151,7 @@ def _signal_handler(self, thesignal, frame):
 
         _logger.info(_("Worker %s was requested to shut down."), self.name)
 
+        self.task_grace_timeout = TASK_GRACE_INTERVAL
         self.shutdown_requested = True
 
     def shutdown(self):
@@ -163,7 +168,7 @@ def worker_cleanup(self):
     def beat(self):
         if self.worker.last_heartbeat < timezone.now() - timedelta(seconds=self.heartbeat_period):
             self.worker = handle_worker_heartbeat(self.name)
-            if self.shutdown_requested:
+            if self.task_grace_timeout > 0:
                 self.task_grace_timeout -= 1
             self.worker_cleanup_countdown -= 1
             if self.worker_cleanup_countdown <= 0:
@@ -300,7 +305,6 @@ def supervise_task(self, task):
 
         This function must only be called while holding the lock for that task."""
 
-        self.task_grace_timeout = TASK_GRACE_INTERVAL
         task.worker = self.worker
         task.save(update_fields=["worker"])
         cancel_state = None
@@ -314,11 +318,15 @@ def supervise_task(self, task):
                     item = connection.connection.notifies.pop(0)
                     if item.channel == "pulp_worker_cancel" and item.payload == str(task.pk):
                         _logger.info(_("Received signal to cancel current task %s."), task.pk)
-                        os.kill(task_process.pid, signal.SIGUSR1)
                         cancel_state = TASK_STATES.CANCELED
                     # ignore all other notifications
                 if cancel_state:
-                    break
+                    if self.task_grace_timeout > 0:
+                        _logger.info("Wait for canceled task to abort.")
+                    else:
+                        self.task_grace_timeout = TASK_KILL_INTERVAL
+                        _logger.info("Aborting current task %s due to cancelation.", task.pk)
+                        os.kill(task_process.pid, signal.SIGUSR1)
 
                 r, w, x = select.select(
                     [self.sentinel, connection.connection, task_process.sentinel],
@@ -341,10 +349,8 @@ def supervise_task(self, task):
                         )
                     else:
                         _logger.info("Aborting current task %s due to worker shutdown.", task.pk)
-                        os.kill(task_process.pid, signal.SIGUSR1)
                         cancel_state = TASK_STATES.FAILED
                         cancel_reason = "Aborted during worker shutdown."
-                        break
             task_process.join()
             if not cancel_state and task_process.exitcode != 0:
                 _logger.warning(


Test patch:
diff --git a/pulpcore/app/tasks/test.py b/pulpcore/app/tasks/test.py
--- a/pulpcore/app/tasks/test.py
+++ b/pulpcore/app/tasks/test.py
@@ -1,3 +1,14 @@
+import backoff
+
+
 def dummy_task():
     """Dummy task, that can be used in tests."""
     pass
+
+
+@backoff.on_exception(backoff.expo, BaseException)
+def gooey_task(interval):
+    """A sleep task that tries to avoid being killed by ignoring all exceptions."""
+    from time import sleep
+
+    sleep(interval)
diff --git a/pulpcore/tests/functional/api/test_tasking.py b/pulpcore/tests/functional/api/test_tasking.py
--- a/pulpcore/tests/functional/api/test_tasking.py
+++ b/pulpcore/tests/functional/api/test_tasking.py
@@ -105,7 +105,7 @@ def test_delete_cancel_running_task(dispatch_task, tasks_api_client):
 
     for i in range(10):
         task = tasks_api_client.read(task_href)
-        if task.state != "running":
+        if task.state == "running":
             break
         time.sleep(1)
 
@@ -275,3 +275,23 @@ def test_filter_tasks_using_worker__in_filter(tasks_api_client, dispatch_task, m
 
     assert task1_href in tasks_hrefs
     assert task2_href in tasks_hrefs
+
+
+def test_cancel_gooey_task(tasks_api_client, dispatch_task, monitor_task):
+    task_href = dispatch_task("pulpcore.app.tasks.test.gooey_task", (60,))
+    for i in range(10):
+        task = tasks_api_client.read(task_href)
+        if task.state == "running":
+            break
+        time.sleep(1)
+
+    task = tasks_api_client.tasks_cancel(task_href, {"state": "canceled"})
+
+    if task.state == "canceling":
+        for i in range(30):
+            if task.state != "canceling":
+                break
+            time.sleep(1)
+            task = tasks_api_client.read(task_href)
+
+    assert task.state == "canceled"
diff --git a/pulpcore/tests/functional/utils.py b/pulpcore/tests/functional/utils.py
--- a/pulpcore/tests/functional/utils.py
+++ b/pulpcore/tests/functional/utils.py
@@ -21,7 +21,7 @@ class PulpTaskError(Exception):
 
         def __init__(self, task):
             """Provide task info to exception."""
-            description = task.to_dict()["error"]["description"]
+            description = task.to_dict()["error"].get("description")
             super().__init__(self, f"Pulp task failed ({description})")
             self.task = task