Repository: borgbackup/borg

Issue:
Traceback when attempting to delete a corrupted .checkpoint archive
After upgrading to 1.0.9, I noticed the daily backup on one of the machines failed. I'm not sure 1.0.9 is the culprit here however, it may have just let me know about the corrupted archive since it looks like it rebuilt the indexes.

When attempting the delete the corrupted archive, I receive the following traceback, even when using --force. Any ideas?

```
Error: Did not get expected metadata dict - archive corrupted!
Local Exception.
Traceback (most recent call last):
  File "borg/archiver.py", line 2052, in main
  File "borg/archiver.py", line 1997, in run
  File "borg/archiver.py", line 90, in wrapper
  File "borg/archiver.py", line 498, in do_delete
  File "borg/cache.py", line 97, in __init__
  File "borg/cache.py", line 391, in sync
  File "borg/cache.py", line 358, in create_master_idx
  File "borg/cache.py", line 315, in fetch_and_build_idx
  File "msgpack/_unpacker.pyx", line 469, in msgpack._unpacker.Unpacker.__next__ (msgpack/_unpacker.cpp:5459)
  File "msgpack/_unpacker.pyx", line 400, in msgpack._unpacker.Unpacker._unpack (msgpack/_unpacker.cpp:4483)
TypeError: unhashable type: 'dict'

Platform: Linux machinename 2.6.32-642.11.1.el6.i686 #1 SMP Fri Nov 18 18:48:28 UTC 2016 i686 i686
Linux: CentOS 6.8 Final
Borg: 1.0.9  Python: CPython 3.5.2
PID: 15290  CWD: /usr/local/bin
sys.argv: ['/usr/local/bin/borg', 'delete', 'user@machine:/location::machine_2016-07-18-23:45:02.checkpoint']
SSH_ORIGINAL_COMMAND: None
```

---

:moneybag: [there is a bounty for this](https://www.bountysource.com/issues/40319126-traceback-when-attempting-to-delete-a-corrupted-checkpoint-archive)


Discussion:
Yes, looks like an outdated cache together with a corrupted archive can make --force useless, because the cache sync tries to access the archives, among them the corrupted one, before --force does it's magic.
Any idea how to manually clean this up, besides wiping the repo and starting over?
[This script](https://gist.github.com/enkore/45bfc63077b83979f392ef64c3747220) should do the trick.
Nice, thanks. I'm using the binary client, so I'll guess I'll need to pip install Borg on the impacted machine for this will work, but this is much appreciated.
Keep in 1.0, due to "being able to fix a corrupted repo".

Resolving patch:
diff --git a/src/borg/archive.py b/src/borg/archive.py
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -662,7 +662,7 @@ def chunk_decref(id, stats):
                 raise ChunksIndexError(cid)
             except Repository.ObjectNotFound as e:
                 # object not in repo - strange, but we wanted to delete it anyway.
-                if not forced:
+                if forced == 0:
                     raise
                 error = True
 
@@ -686,14 +686,14 @@ def chunk_decref(id, stats):
                 except (TypeError, ValueError):
                     # if items metadata spans multiple chunks and one chunk got dropped somehow,
                     # it could be that unpacker yields bad types
-                    if not forced:
+                    if forced == 0:
                         raise
                     error = True
             if progress:
                 pi.finish()
         except (msgpack.UnpackException, Repository.ObjectNotFound):
             # items metadata corrupted
-            if not forced:
+            if forced == 0:
                 raise
             error = True
         # in forced delete mode, we try hard to delete at least the manifest entry,
diff --git a/src/borg/archiver.py b/src/borg/archiver.py
--- a/src/borg/archiver.py
+++ b/src/borg/archiver.py
@@ -828,6 +828,26 @@ def _delete_archives(self, args, repository):
             if not archive_names:
                 return self.exit_code
 
+        if args.forced == 2:
+            deleted = False
+            for i, archive_name in enumerate(archive_names, 1):
+                try:
+                    del manifest.archives[archive_name]
+                except KeyError:
+                    self.exit_code = EXIT_WARNING
+                    logger.warning('Archive {} not found ({}/{}).'.format(archive_name, i, len(archive_names)))
+                else:
+                    deleted = True
+                    logger.info('Deleted {} ({}/{}).'.format(archive_name, i, len(archive_names)))
+            if deleted:
+                manifest.write()
+                # note: might crash in compact() after committing the repo
+                repository.commit()
+                logger.info('Done. Run "borg check --repair" to clean up the mess.')
+            else:
+                logger.warning('Aborted.')
+            return self.exit_code
+
         stats_logger = logging.getLogger('borg.output.stats')
         if args.stats:
             log_multi(DASHES, STATS_HEADER, logger=stats_logger)
@@ -845,7 +865,7 @@ def _delete_archives(self, args, repository):
                 if args.stats:
                     log_multi(stats.summary.format(label='Deleted data:', stats=stats),
                               DASHES, logger=stats_logger)
-                if not args.forced and self.exit_code:
+                if args.forced == 0 and self.exit_code:
                     break
             if args.stats:
                 stats_logger.info(str(cache))
@@ -2383,8 +2403,9 @@ def process_epilog(epilog):
                                action='store_true', default=False,
                                help='delete only the local cache for the given repository')
         subparser.add_argument('--force', dest='forced',
-                               action='store_true', default=False,
-                               help='force deletion of corrupted archives')
+                               action='count', default=0,
+                               help='force deletion of corrupted archives, '
+                                    'use --force --force in case --force does not work.')
         subparser.add_argument('--save-space', dest='save_space', action='store_true',
                                default=False,
                                help='work slower, but using less space')
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -1166,6 +1166,20 @@ def test_delete_repo(self):
         # Make sure the repo is gone
         self.assertFalse(os.path.exists(self.repository_path))
 
+    def test_delete_double_force(self):
+        self.cmd('init', '--encryption=none', self.repository_location)
+        self.create_src_archive('test')
+        with Repository(self.repository_path, exclusive=True) as repository:
+            manifest, key = Manifest.load(repository)
+            archive = Archive(repository, key, manifest, 'test')
+            id = archive.metadata.items[0]
+            repository.put(id, b'corrupted items metadata stream chunk')
+            repository.commit()
+        self.cmd('delete', '--force', '--force', self.repository_location + '::test')
+        self.cmd('check', '--repair', self.repository_location)
+        output = self.cmd('list', self.repository_location)
+        self.assert_not_in('test', output)
+
     def test_corrupted_repository(self):
         self.cmd('init', '--encryption=repokey', self.repository_location)
         self.create_src_archive('test')


Test patch: