Repository: pulp/pulpcore

Issue:
Repair feature should remove unrepairable corrupted file
Author: @ipanova (ipanova@redhat.com)


Redmine Issue: 7835, https://pulp.plan.io/issues/7835

---

In case repair feature cannot repair a corrupted file because RemoteArtifact is missing or in case url is not longer valid, it should remove from the fs the corrupted file

```
(06:27:25 PM) ipanova: dalley: maybe before returning False we could just remove the unrepaired corrupted file from the FS https://github.com/pulp/pulpcore/pull/991/files#diff-03bf88cd6f4c5051a7b1478ba05ea63287e7bb4c29feb00413b80ba35d22ccf2R73
(06:28:05 PM) dalley: +1
```





Discussion:
From: @ipanova (ipanova@redhat.com)
Date: 2020-11-23T17:37:12Z

---

This should also be cleaned up  in the django populate-missing-checksums command since it is using some parts of the repair feature
From: @dralley (dalley)
Date: 2020-11-24T16:14:35Z

---

We should consider if we can/should do anything else in case it is a repository metadata artifact.
This issue has been marked 'stale' due to lack of recent activity. If there is no further activity, the issue will be closed in another 30 days. Thank you for your contribution!

Resolving patch:
diff --git a/pulpcore/app/management/commands/handle-artifact-checksums.py b/pulpcore/app/management/commands/handle-artifact-checksums.py
--- a/pulpcore/app/management/commands/handle-artifact-checksums.py
+++ b/pulpcore/app/management/commands/handle-artifact-checksums.py
@@ -2,6 +2,8 @@
 
 from gettext import gettext as _
 
+from aiohttp.client_exceptions import ClientResponseError
+
 from django.conf import settings
 from django.core.management import BaseCommand, CommandError
 from django.db.models import Q, Sum
@@ -124,15 +126,23 @@ def _download_artifact(self, artifact, checksum, file_path):
                     if remote.policy == "immediate":
                         self.stdout.write(_("Restoring missing file {}").format(file_path))
                         downloader = remote.get_downloader(ra)
-                        dl_result = downloader.fetch()
-                        # FIXME in case url is not available anymore this will break
-                        if dl_result.artifact_attributes["sha256"] == artifact.sha256:
-                            with open(dl_result.path, "rb") as src:
-                                filename = artifact.file.name
-                                artifact.file.save(filename, src, save=False)
-                            setattr(artifact, checksum, dl_result.artifact_attributes[checksum])
-                            restored = True
-                            break
+                        try:
+                            dl_result = downloader.fetch()
+                        except ClientResponseError as e:
+                            self.stdout.write(
+                                _("Redownload failed from '{}': {}.").format(ra.url, str(e))
+                            )
+                        else:
+                            if dl_result.artifact_attributes["sha256"] == artifact.sha256:
+                                with open(dl_result.path, "rb") as src:
+                                    filename = artifact.file.name
+                                    artifact.file.delete(save=False)
+                                    artifact.file.save(filename, src, save=False)
+                                setattr(artifact, checksum, dl_result.artifact_attributes[checksum])
+                                restored = True
+                                break
+                self.stdout.write(_("Deleting unreparable file {}".format(file_path)))
+                artifact.file.delete(save=False)
             else:
                 break
         return restored
diff --git a/pulpcore/app/tasks/repository.py b/pulpcore/app/tasks/repository.py
--- a/pulpcore/app/tasks/repository.py
+++ b/pulpcore/app/tasks/repository.py
@@ -4,6 +4,7 @@
 import asyncio
 import hashlib
 
+from aiohttp.client_exceptions import ClientResponseError
 from asgiref.sync import sync_to_async
 from django.core.files.storage import default_storage
 from django.db import transaction
@@ -69,22 +70,32 @@ async def _repair_ca(content_artifact, repaired=None):
         log.warn(
             _("Artifact {} is unrepairable - no remote source".format(content_artifact.artifact))
         )
+        log.warning(
+            "Deleting file for the unreparable artifact {}".format(content_artifact.artifact)
+        )
+        await sync_to_async(content_artifact.artifact.file.delete)(save=False)
         return False
 
     async for remote_artifact in remote_artifacts:
         detail_remote = await sync_to_async(remote_artifact.remote.cast)()
         downloader = detail_remote.get_downloader(remote_artifact)
-        dl_result = await downloader.run()
-        if dl_result.artifact_attributes["sha256"] == content_artifact.artifact.sha256:
-            with open(dl_result.path, "rb") as src:
-                filename = content_artifact.artifact.file.name
-                await sync_to_async(content_artifact.artifact.file.delete)(save=False)
-                await sync_to_async(content_artifact.artifact.file.save)(filename, src, save=False)
-            if repaired is not None:
-                await repaired.aincrement()
-            return True
-        log.warn(_("Redownload failed from {}.").format(remote_artifact.url))
-
+        try:
+            dl_result = await downloader.run()
+        except ClientResponseError as e:
+            log.warn(_("Redownload failed from '{}': {}.").format(remote_artifact.url, str(e)))
+        else:
+            if dl_result.artifact_attributes["sha256"] == content_artifact.artifact.sha256:
+                with open(dl_result.path, "rb") as src:
+                    filename = content_artifact.artifact.file.name
+                    await sync_to_async(content_artifact.artifact.file.delete)(save=False)
+                    await sync_to_async(content_artifact.artifact.file.save)(
+                        filename, src, save=False
+                    )
+                if repaired is not None:
+                    await repaired.aincrement()
+                return True
+    log.warning("Deleting file for the unreparable artifact {}".format(content_artifact.artifact))
+    await sync_to_async(content_artifact.artifact.file.delete)(save=False)
     return False
 
 


Test patch: