Repository: pulp/pulpcore

Issue:
Make orphan clean up more torelant to the failures during content removal
**Version**
all

**Describe the bug**
Orphan clean up can fails when it encounters content that was identified as orphan but because of some other tasks that might be running in parallel, e.g. sync it might be adding the content to the repo. In that case task will fail on the very first occurrence of such content. We should make orphan cleanup content more tolerant to such cases.

Some plugins declare content dependency where a content( orphan) cascade deletes another content(non-orphan).
The question is how orphan cleanup should handle such case when it does not have specific plugin knowledge.

The suggestion to handle these both cases is to by-pass problematic content that cannot be removed, log it and proceed. 

**To Reproduce**
Steps to reproduce the behavior:

**Expected behavior**
A clear and concise description of what you expected to happen.

**Additional context**
Add any other context about the problem here.  Please provide links to any previous discussions via Discourse or Bugzilla.



Discussion:

Resolving patch:
diff --git a/pulpcore/app/tasks/orphan.py b/pulpcore/app/tasks/orphan.py
--- a/pulpcore/app/tasks/orphan.py
+++ b/pulpcore/app/tasks/orphan.py
@@ -1,6 +1,9 @@
 import gc
 
+from logging import getLogger
+
 from django.conf import settings
+from django.db.models.deletion import ProtectedError
 from django.utils import timezone
 
 from pulpcore.app.models import (
@@ -12,6 +15,8 @@
     Upload,
 )
 
+log = getLogger(__name__)
+
 
 def queryset_iterator(qs, batchsize=2000, gc_collect=True):
     """
@@ -44,35 +49,67 @@ def orphan_cleanup(content_pks=None, orphan_protection_time=settings.ORPHAN_PROT
         content_pks (list): A list of content pks. If specified, only remove these orphans.
 
     """
+    content = Content.objects.orphaned(orphan_protection_time, content_pks).exclude(
+        pulp_type=PublishedMetadata.get_pulp_type()
+    )
+    skipped_content = 0
     with ProgressReport(
         message="Clean up orphan Content",
-        total=None,
+        total=content.count(),
         code="clean-up.content",
     ) as progress_bar:
-        while True:
-            content = Content.objects.orphaned(orphan_protection_time, content_pks).exclude(
-                pulp_type=PublishedMetadata.get_pulp_type()
-            )
-            content_count = content.count()
-            if not content_count:
-                break
-
-            # delete the content
-            for c in queryset_iterator(content):
-                progress_bar.increase_by(c.count())
-                c.delete()
+        # delete the content
+        for bulk_content in queryset_iterator(content):
+            skipped_content_batch = 0
+            count = bulk_content.count()
+            try:
+                bulk_content.delete()
+            except ProtectedError:
+                # some orphan content might have been picked by another task running in parallel
+                # i.e. sync
+                for c in bulk_content:
+                    try:
+                        c.delete()
+                    except ProtectedError as e:
+                        log.debug(e)
+                        skipped_content_batch += 1
+            progress_bar.increase_by(count - skipped_content_batch)
+            skipped_content += skipped_content_batch
+
+    if skipped_content:
+        msg = (
+            "{} orphaned content could not be deleted during this run and was skipped. "
+            "Re-run the task and/or consult the logs."
+        )
+        log.info(msg.format(skipped_content))
 
     # delete the artifacts that don't belong to any content
     artifacts = Artifact.objects.orphaned(orphan_protection_time)
 
+    skipped_artifact = 0
     with ProgressReport(
         message="Clean up orphan Artifacts",
         total=artifacts.count(),
         code="clean-up.artifacts",
     ) as progress_bar:
-        for artifact in progress_bar.iter(artifacts.iterator()):
-            # we need to manually call delete() because it cleans up the file on the filesystem
-            artifact.delete()
+        for artifact in artifacts.iterator():
+            try:
+                # we need to manually call delete() because it cleans up the file on the filesystem
+                artifact.delete()
+            except ProtectedError as e:
+                # some orphaned artifact might have been picked by another task running in parallel
+                # i.e. sync
+                log.debug(e)
+                skipped_artifact += 1
+            else:
+                progress_bar.increment()
+
+    if skipped_artifact:
+        msg = (
+            "{} orphaned artifact(s) could not be deleted during this run and were skipped. "
+            "Re-run the task and/or consult the logs."
+        )
+        log.info(msg.format(skipped_artifact))
 
 
 def upload_cleanup():
diff --git a/pulpcore/app/tasks/purge.py b/pulpcore/app/tasks/purge.py
--- a/pulpcore/app/tasks/purge.py
+++ b/pulpcore/app/tasks/purge.py
@@ -143,7 +143,7 @@ def purge(finished_before, states):
                     # Log the details of the object.
                     error_pb.done += 1
                     pks_failed.append(pk)
-                    log.info(e)
+                    log.debug(e)
 
     # Complete the progress-reports for the specific entities deleted
     for key, pb in details_reports.items():
diff --git a/pulpcore/app/tasks/reclaim_space.py b/pulpcore/app/tasks/reclaim_space.py
--- a/pulpcore/app/tasks/reclaim_space.py
+++ b/pulpcore/app/tasks/reclaim_space.py
@@ -86,9 +86,9 @@ def reclaim_space(repo_pks, keeplist_rv_pks=None, force=False):
             # we need to manually call delete() because it cleans up the file on the filesystem
             artifact.delete()
         except ProtectedError as e:
-            # Rarely artifact could be shared between to different content units.
+            # Rarely artifact could be shared between two different content units.
             # Just log and skip the artifact deletion in this case
-            log.info(e)
+            log.debug(e)
         else:
             progress_bar.done += 1
             counter += 1


Test patch: