Repository: pulp/pulpcore

Issue:
Task purging endpoint loads every task in memory
**Version**
`pulpcore-3.17.3`
`pulp_rpm-3.17.3`
`pulp_ansible-0.12.0`
`pulp_container-2.10.0`
`pulp_deb-2.17.0`
`pulp_file-1.10.1`
`pulp_python-3.6.0`

**Describe the bug**
We have a large number of finished pulp tasks (around 1.5kk). So we're trying to purge it with [`pulp/api/v3/tasks/purge`](https://docs.pulpproject.org/pulpcore/restapi.html#operation/tasks_purge) and quickly realized, that this task is trying to load whole finished queue (all 1.5kk finished tasks) into memory, which cause it killed after consuming 15GB of memory.

**To Reproduce**
Steps to reproduce the behavior:
1. Create large number of pulp-tasks (more than 1kk will be enough)
2. Run [`pulp/api/v3/tasks/purge`](https://docs.pulpproject.org/pulpcore/restapi.html#operation/tasks_purge) with `finished_before` set to future date or something.
3. Your server will run out-of-memory and task won't be completed.

**Expected behavior**
Task purging api-endpoint does not load the entire finished queue into memory


Discussion:
Related: https://github.com/pulp/pulpcore/issues/5048

Resolving patch:
diff --git a/pulpcore/app/tasks/purge.py b/pulpcore/app/tasks/purge.py
--- a/pulpcore/app/tasks/purge.py
+++ b/pulpcore/app/tasks/purge.py
@@ -5,6 +5,48 @@
     Task,
 )
 from pulpcore.app.role_util import get_objects_for_user
+from pulpcore.constants import TASK_STATES
+
+# Delete 1K at a time - better to use less memory, and take a little longer, with a utility
+# function like this.
+DELETE_LIMIT = 1000
+# Key that delete() returns for Tasks
+TASK_KEY = "core.Task"
+
+
+def _details_reporting(current_reports, current_details, totals_pb):
+    """
+    Create and update progress-reports for each detail-key returned from a delete() call.
+
+    We don't know how many entities will be deleted via cascade-delete until we're all done.
+
+    The function has one special case: we know how many Tasks we're expecting to delete right
+    from the beginning. Therefore, we "assume" that the key `core.Task` has been pre-seeded
+    with a ProgressReport whose total is correct, in advance, and therefore don't update
+    total for that key.
+
+    Args:
+        current_reports (dict): key:ProgressReport to record into
+    Returns:
+        updated current_reports
+    """
+    entity_count = 0
+    for key, curr_detail in current_details.items():
+        entity_count += current_details[key]
+        if key in current_reports:
+            current_reports[key].increase_by(curr_detail)
+        else:
+            pb = ProgressReport(
+                message=_("Purged task-objects of type {}".format(key)),
+                code="purge.tasks.key.{}".format(key),
+                total=None,
+                done=curr_detail,
+            )
+            pb.save()
+            current_reports[key] = pb
+    # Update/save totals once
+    totals_pb.increase_by(entity_count)
+    return current_reports
 
 
 def purge(finished_before, states):
@@ -26,25 +68,48 @@ def purge(finished_before, states):
 
     """
     current_user = get_current_authenticated_user()
-    qs = Task.objects.filter(finished_at__lt=finished_before, state__in=states)
-    units_deleted, details = get_objects_for_user(current_user, "core.delete_task", qs=qs).delete()
-
+    # Tasks, prior to the specified date, in the specified state, owned by the current-user
+    tasks_qs = Task.objects.filter(finished_at__lt=finished_before, state__in=states)
+    candidate_qs = get_objects_for_user(current_user, "core.delete_task", qs=tasks_qs)
+    delete_qs = get_objects_for_user(current_user, "core.delete_task", qs=tasks_qs[:DELETE_LIMIT])
     # Progress bar reporting total-units
-    progress_bar = ProgressReport(
-        message=_("Purged task-objects total"),
-        total=units_deleted,
+    totals_pb = ProgressReport(
+        message=_("Purged task-related-objects total"),
+        total=None,
         code="purge.tasks.total",
-        done=units_deleted,
-        state="completed",
+        done=0,
+    )
+    totals_pb.save()
+    # Dictionary to hold progress-reports by delete-details-key
+    details_reports = {}
+
+    # Figure out how many Tasks owned by the current user we're about to delete
+    expected_total = candidate_qs.count()
+    # Build and save a progress-report for that detail
+    pb = ProgressReport(
+        message=_("Purged task-objects of type {}".format(TASK_KEY)),
+        total=expected_total,
+        code="purge.tasks.key.{}".format(TASK_KEY),
+        done=0,
     )
-    progress_bar.save()
-    # This loop reports back the specific entities deleted and the number removed
-    for key in details:
-        progress_bar = ProgressReport(
-            message=_("Purged task-objects of type {}".format(key)),
-            total=details[key],
-            code="purge.tasks.key.{}".format(key),
-            done=details[key],
-            state="completed",
-        )
-        progress_bar.save()
+    pb.save()
+    details_reports[TASK_KEY] = pb
+
+    # Our delete-query is going to deal with "the first DELETE_LIMIT tasks that match our
+    # criteria", looping until we've deleted everything that fits our parameters
+    units_deleted, details = delete_qs.delete()
+    # Until our query returns "No tasks deleted", add results into totals and Do It Again
+    while units_deleted > 0:
+        _details_reporting(details_reports, details, totals_pb)
+        units_deleted, details = delete_qs.delete()
+
+    # Complete the progress-reports for the specific entities deleted
+    for key, pb in details_reports.items():
+        pb.total = pb.done
+        pb.state = TASK_STATES.COMPLETED
+        pb.save()
+
+    # Complete the totals-ProgressReport
+    totals_pb.total = totals_pb.done
+    totals_pb.state = TASK_STATES.COMPLETED
+    totals_pb.save()


Test patch: