Repository: borgbackup/borg

Issue:
"borg delete" should allow to delete broken archives as well
--force for borg delete, so that you can delete broken archives. I.e. it tries to delete normally, but doesn't fail if the metadata is corrupted, and the manifest entry is always removed at least [and the archive superblock deleted from the repo].



Discussion:
hmm, guess we could need that for dealing with #1138.


Resolving patch:
diff --git a/borg/archive.py b/borg/archive.py
--- a/borg/archive.py
+++ b/borg/archive.py
@@ -24,6 +24,8 @@
 from .platform import acl_get, acl_set
 from .chunker import Chunker
 from .hashindex import ChunkIndex
+from .repository import Repository
+
 import msgpack
 
 ITEMS_BUFFER = 1024 * 1024
@@ -494,23 +496,59 @@ def rename(self, name):
         self.cache.chunk_decref(self.id, self.stats)
         del self.manifest.archives[self.name]
 
-    def delete(self, stats, progress=False):
-        unpacker = msgpack.Unpacker(use_list=False)
-        items_ids = self.metadata[b'items']
-        pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%", same_line=True)
-        for (i, (items_id, data)) in enumerate(zip(items_ids, self.repository.get_many(items_ids))):
+    def delete(self, stats, progress=False, forced=False):
+        class ChunksIndexError(Error):
+            """Chunk ID {} missing from chunks index, corrupted chunks index - aborting transaction."""
+
+        def chunk_decref(id, stats):
+            nonlocal error
+            try:
+                self.cache.chunk_decref(id, stats)
+            except KeyError:
+                cid = hexlify(id).decode('ascii')
+                raise ChunksIndexError(cid)
+            except Repository.ObjectNotFound as e:
+                # object not in repo - strange, but we wanted to delete it anyway.
+                if not forced:
+                    raise
+                error = True
+
+        error = False
+        try:
+            unpacker = msgpack.Unpacker(use_list=False)
+            items_ids = self.metadata[b'items']
+            pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%", same_line=True)
+            for (i, (items_id, data)) in enumerate(zip(items_ids, self.repository.get_many(items_ids))):
+                if progress:
+                    pi.show(i)
+                unpacker.feed(self.key.decrypt(items_id, data))
+                chunk_decref(items_id, stats)
+                try:
+                    for item in unpacker:
+                        if b'chunks' in item:
+                            for chunk_id, size, csize in item[b'chunks']:
+                                chunk_decref(chunk_id, stats)
+                except (TypeError, ValueError):
+                    # if items metadata spans multiple chunks and one chunk got dropped somehow,
+                    # it could be that unpacker yields bad types
+                    if not forced:
+                        raise
+                    error = True
             if progress:
-                pi.show(i)
-            unpacker.feed(self.key.decrypt(items_id, data))
-            self.cache.chunk_decref(items_id, stats)
-            for item in unpacker:
-                if b'chunks' in item:
-                    for chunk_id, size, csize in item[b'chunks']:
-                        self.cache.chunk_decref(chunk_id, stats)
-        if progress:
-            pi.finish()
-        self.cache.chunk_decref(self.id, stats)
+                pi.finish()
+        except (msgpack.UnpackException, Repository.ObjectNotFound):
+            # items metadata corrupted
+            if not forced:
+                raise
+            error = True
+        # in forced delete mode, we try hard to delete at least the manifest entry,
+        # if possible also the archive superblock, even if processing the items raises
+        # some harmless exception.
+        chunk_decref(self.id, stats)
         del self.manifest.archives[self.name]
+        if error:
+            logger.warning('forced deletion succeeded, but the deleted archive was corrupted.')
+            logger.warning('borg check --repair is required to free all space.')
 
     def stat_attrs(self, st, path):
         item = {
diff --git a/borg/archiver.py b/borg/archiver.py
--- a/borg/archiver.py
+++ b/borg/archiver.py
@@ -426,7 +426,7 @@ def do_delete(self, args, repository):
             with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache:
                 archive = Archive(repository, key, manifest, args.location.archive, cache=cache)
                 stats = Statistics()
-                archive.delete(stats, progress=args.progress)
+                archive.delete(stats, progress=args.progress, forced=args.forced)
                 manifest.write()
                 repository.commit(save_space=args.save_space)
                 cache.commit()
@@ -635,7 +635,7 @@ def do_prune(self, args, repository, manifest, key):
                 else:
                     if args.output_list:
                         logger.info('Pruning archive: %s' % format_archive(archive))
-                    Archive(repository, key, manifest, archive.name, cache).delete(stats)
+                    Archive(repository, key, manifest, archive.name, cache).delete(stats, forced=args.forced)
             if to_delete and not args.dry_run:
                 manifest.write()
                 repository.commit(save_space=args.save_space)
@@ -1230,6 +1230,9 @@ def build_parser(self, args=None, prog=None):
         subparser.add_argument('-c', '--cache-only', dest='cache_only',
                                action='store_true', default=False,
                                help='delete only the local cache for the given repository')
+        subparser.add_argument('--force', dest='forced',
+                               action='store_true', default=False,
+                               help='force deletion of corrupted archives')
         subparser.add_argument('--save-space', dest='save_space', action='store_true',
                                default=False,
                                help='work slower, but using less space')
@@ -1342,6 +1345,9 @@ def build_parser(self, args=None, prog=None):
         subparser.add_argument('-n', '--dry-run', dest='dry_run',
                                default=False, action='store_true',
                                help='do not change repository')
+        subparser.add_argument('--force', dest='forced',
+                               action='store_true', default=False,
+                               help='force pruning of corrupted archives')
         subparser.add_argument('-s', '--stats', dest='stats',
                                action='store_true', default=False,
                                help='print statistics for the deleted archive')


Test patch: