Repository: borgbackup/borg

Issue:
Traceback when attempting to delete a corrupted .checkpoint archive
After upgrading to 1.0.9, I noticed the daily backup on one of the machines failed. I'm not sure 1.0.9 is the culprit here however, it may have just let me know about the corrupted archive since it looks like it rebuilt the indexes.

When attempting the delete the corrupted archive, I receive the following traceback, even when using --force. Any ideas?

```
Error: Did not get expected metadata dict - archive corrupted!
Local Exception.
Traceback (most recent call last):
  File "borg/archiver.py", line 2052, in main
  File "borg/archiver.py", line 1997, in run
  File "borg/archiver.py", line 90, in wrapper
  File "borg/archiver.py", line 498, in do_delete
  File "borg/cache.py", line 97, in __init__
  File "borg/cache.py", line 391, in sync
  File "borg/cache.py", line 358, in create_master_idx
  File "borg/cache.py", line 315, in fetch_and_build_idx
  File "msgpack/_unpacker.pyx", line 469, in msgpack._unpacker.Unpacker.__next__ (msgpack/_unpacker.cpp:5459)
  File "msgpack/_unpacker.pyx", line 400, in msgpack._unpacker.Unpacker._unpack (msgpack/_unpacker.cpp:4483)
TypeError: unhashable type: 'dict'

Platform: Linux machinename 2.6.32-642.11.1.el6.i686 #1 SMP Fri Nov 18 18:48:28 UTC 2016 i686 i686
Linux: CentOS 6.8 Final
Borg: 1.0.9  Python: CPython 3.5.2
PID: 15290  CWD: /usr/local/bin
sys.argv: ['/usr/local/bin/borg', 'delete', 'user@machine:/location::machine_2016-07-18-23:45:02.checkpoint']
SSH_ORIGINAL_COMMAND: None
```

---

:moneybag: [there is a bounty for this](https://www.bountysource.com/issues/40319126-traceback-when-attempting-to-delete-a-corrupted-checkpoint-archive)

Discussion:
enkore: I can has test case? ;)
ThomasWaldmann: btw, i also tried to add a test for the single --force delete, but it needs some different corruption code that cleanly removes an archive chunk from repo.
enkore: Suggestion: Don't corrupt the low-level structures for the tests, it is expected that only check--repair can fix it and that --force² might fail after the commit. Instead, write some invalid data via the Repository API, that way it's easier to test.

Patch:
diff --git a/src/borg/archive.py b/src/borg/archive.py
index 07d62e16..4bebecd7 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -662,7 +662,7 @@ def chunk_decref(id, stats):
                 raise ChunksIndexError(cid)
             except Repository.ObjectNotFound as e:
                 # object not in repo - strange, but we wanted to delete it anyway.
-                if not forced:
+                if forced == 0:
                     raise
                 error = True
 
@@ -686,14 +686,14 @@ def chunk_decref(id, stats):
                 except (TypeError, ValueError):
                     # if items metadata spans multiple chunks and one chunk got dropped somehow,
                     # it could be that unpacker yields bad types
-                    if not forced:
+                    if forced == 0:
                         raise
                     error = True
             if progress:
                 pi.finish()
         except (msgpack.UnpackException, Repository.ObjectNotFound):
             # items metadata corrupted
-            if not forced:
+            if forced == 0:
                 raise
             error = True
         # in forced delete mode, we try hard to delete at least the manifest entry,
diff --git a/src/borg/archiver.py b/src/borg/archiver.py
index 041749a1..efdc16db 100644
--- a/src/borg/archiver.py
+++ b/src/borg/archiver.py
@@ -828,6 +828,26 @@ def _delete_archives(self, args, repository):
             if not archive_names:
                 return self.exit_code
 
+        if args.forced == 2:
+            deleted = False
+            for i, archive_name in enumerate(archive_names, 1):
+                try:
+                    del manifest.archives[archive_name]
+                except KeyError:
+                    self.exit_code = EXIT_WARNING
+                    logger.warning('Archive {} not found ({}/{}).'.format(archive_name, i, len(archive_names)))
+                else:
+                    deleted = True
+                    logger.info('Deleted {} ({}/{}).'.format(archive_name, i, len(archive_names)))
+            if deleted:
+                manifest.write()
+                # note: might crash in compact() after committing the repo
+                repository.commit()
+                logger.info('Done. Run "borg check --repair" to clean up the mess.')
+            else:
+                logger.warning('Aborted.')
+            return self.exit_code
+
         stats_logger = logging.getLogger('borg.output.stats')
         if args.stats:
             log_multi(DASHES, STATS_HEADER, logger=stats_logger)
@@ -845,7 +865,7 @@ def _delete_archives(self, args, repository):
                 if args.stats:
                     log_multi(stats.summary.format(label='Deleted data:', stats=stats),
                               DASHES, logger=stats_logger)
-                if not args.forced and self.exit_code:
+                if args.forced == 0 and self.exit_code:
                     break
             if args.stats:
                 stats_logger.info(str(cache))
@@ -2383,8 +2403,9 @@ def process_epilog(epilog):
                                action='store_true', default=False,
                                help='delete only the local cache for the given repository')
         subparser.add_argument('--force', dest='forced',
-                               action='store_true', default=False,
-                               help='force deletion of corrupted archives')
+                               action='count', default=0,
+                               help='force deletion of corrupted archives, '
+                                    'use --force --force in case --force does not work.')
         subparser.add_argument('--save-space', dest='save_space', action='store_true',
                                default=False,
                                help='work slower, but using less space')


Test patch:
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
index cce0c92f..d13dee4c 100644
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -1166,6 +1166,20 @@ def test_delete_repo(self):
         # Make sure the repo is gone
         self.assertFalse(os.path.exists(self.repository_path))
 
+    def test_delete_double_force(self):
+        self.cmd('init', '--encryption=none', self.repository_location)
+        self.create_src_archive('test')
+        with Repository(self.repository_path, exclusive=True) as repository:
+            manifest, key = Manifest.load(repository)
+            archive = Archive(repository, key, manifest, 'test')
+            id = archive.metadata.items[0]
+            repository.put(id, b'corrupted items metadata stream chunk')
+            repository.commit()
+        self.cmd('delete', '--force', '--force', self.repository_location + '::test')
+        self.cmd('check', '--repair', self.repository_location)
+        output = self.cmd('list', self.repository_location)
+        self.assert_not_in('test', output)
+
     def test_corrupted_repository(self):
         self.cmd('init', '--encryption=repokey', self.repository_location)
         self.create_src_archive('test')