Repository: borgbackup/borg

Issue:
no automatic rebuild of a deleted chunks file
borg does not tolerate a deleted chunks file (deleted because it was corrupted) due to the integrity checks:
```
u0_a151@localhost ~> ./borgbackup.sh

Fri Jun  5 21:11:53 EDT 2020 Starting backup

Killed stale lock localhost@75173968615802.3665-0.
Removed stale exclusive roster lock for host localhost@75173968615802 pid 3665 thread 0.
Removed stale exclusive roster lock for host localhost@75173968615802 pid 9194 thread 0.
Removed stale exclusive roster lock for host localhost@75173968615802 pid 3665 thread 0.
Removed stale exclusive roster lock for host localhost@75173968615802 pid 9194 thread 0.
Creating archive at "aelius@192.168.1.143:/media/borg-monolith::Monolith-2020-06-05T21:11:55"
Local Exception
Traceback (most recent call last):
  File "/data/data/com.termux/files/usr/lib/python3.8/site-packages/borgbackup-1.1.11-py3.8-linux-x86_64.egg/borg/archiver.py", line 4529, in main
    exit_code = archiver.run(args)
  File "/data/data/com.termux/files/usr/lib/python3.8/site-packages/borgbackup-1.1.11-py3.8-linux-x86_64.egg/borg/archiver.py", line 4461, in run
    return set_ec(func(args))
  File "/data/data/com.termux/files/usr/lib/python3.8/site-packages/borgbackup-1.1.11-py3.8-linux-x86_64.egg/borg/archiver.py", line 166, in wrapper
    return method(self, args, repository=repository, **kwargs)
  File "/data/data/com.termux/files/usr/lib/python3.8/site-packages/borgbackup-1.1.11-py3.8-linux-x86_64.egg/borg/archiver.py", line 565, in do_create
    with Cache(repository, key, manifest, do_files=args.cache_files, progress=args.progress,
  File "/data/data/com.termux/files/usr/lib/python3.8/site-packages/borgbackup-1.1.11-py3.8-linux-x86_64.egg/borg/cache.py", line 380, in __new__
    return local()
  File "/data/data/com.termux/files/usr/lib/python3.8/site-packages/borgbackup-1.1.11-py3.8-linux-x86_64.egg/borg/cache.py", line 372, in local
    return LocalCache(repository=repository, key=key, manifest=manifest, path=path, sync=sync,
  File "/data/data/com.termux/files/usr/lib/python3.8/site-packages/borgbackup-1.1.11-py3.8-linux-x86_64.egg/borg/cache.py", line 457, in __init__
    self.open()
  File "/data/data/com.termux/files/usr/lib/python3.8/site-packages/borgbackup-1.1.11-py3.8-linux-x86_64.egg/borg/cache.py", line 505, in open
    self.rollback()
  File "/data/data/com.termux/files/usr/lib/python3.8/site-packages/borgbackup-1.1.11-py3.8-linux-x86_64.egg/borg/cache.py", line 610, in rollback
    self._do_open()
  File "/data/data/com.termux/files/usr/lib/python3.8/site-packages/borgbackup-1.1.11-py3.8-linux-x86_64.egg/borg/cache.py", line 493, in _do_open
    with IntegrityCheckedFile(path=os.path.join(self.path, 'chunks'), write=False,
  File "/data/data/com.termux/files/usr/lib/python3.8/site-packages/borgbackup-1.1.11-py3.8-linux-x86_64.egg/borg/crypto/file_integrity.py", line 129, in __init__
    self.file_fd = override_fd or open(path, mode)
FileNotFoundError: [Errno 2] No such file or directory: '/data/data/com.termux/files/home/.cache/borg/6d24295b115c5b4eb74bace9521ab7b09de7f82f096b8b39f8fcd1411fe4b34e/chunks'

Platform: Linux localhost 4.9.197-arter97-r12-g29a968fb09d85 #1 SMP PREEMPT Mon Oct 28 19:22:17 KST 2019 aarch64
Linux: Unknown Linux
Borg: 1.1.11  Python: CPython 3.8.3 msgpack: 0.5.6
PID: 11175  CWD: /data/data/com.termux/files/home
sys.argv: ['/data/data/com.termux/files/usr/bin/borg', 'create', '--verbose', '--filter', 'AME', '--list', '--stats', '--show-rc', '--compression', 'lz4', '--exclude-caches', '--pattern=+/storage/emulated/0/Android/data/org.tasks', '--pattern=+/storage/emulated/0/Android/data/net.sourceforge.opencamera', '--pattern=-/storage/emulated/0/Android', '--pattern=-/storage/emulated/0/DCIM', '--exclude', '*/.sync', '::Monolith-{now}', 'borgbackup.sh', '/storage/emulated/0/']
SSH_ORIGINAL_COMMAND: None

terminating with error status, rc 2
```

Discussion:

Patch:
diff --git a/src/borg/cache.py b/src/borg/cache.py
index 0df8d49d..f6d5baeb 100644
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -488,7 +488,12 @@ def __init__(self, repository, key, manifest, path=None, sync=True, warn_if_unen
             self.security_manager.assert_access_unknown(warn_if_unencrypted, manifest, key)
             self.create()
 
-        self.open()
+        try:
+            self.open()
+        except (FileNotFoundError, FileIntegrityError):
+            self.wipe_cache()
+            self.open()
+
         try:
             self.security_manager.assert_secure(manifest, key, cache_config=self.cache_config)
 
@@ -920,19 +925,31 @@ def check_cache_compatibility(self):
         return True
 
     def wipe_cache(self):
-        logger.warning("Discarding incompatible cache and forcing a cache rebuild")
-        archive_path = os.path.join(self.path, 'chunks.archive.d')
+        logger.warning("Discarding incompatible or corrupted cache and forcing a cache rebuild")
+        archive_path = os.path.join(self.path, "chunks.archive.d")
         if os.path.isdir(archive_path):
             shutil.rmtree(os.path.join(self.path, 'chunks.archive.d'))
             os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
         self.chunks = ChunkIndex()
-        with SaveFile(os.path.join(self.path, files_cache_name()), binary=True):
+        with IntegrityCheckedFile(path=os.path.join(self.path, "chunks"), write=True) as fd:
+            self.chunks.write(fd)
+        self.cache_config.integrity["chunks"] = fd.integrity_data
+        with IntegrityCheckedFile(path=os.path.join(self.path, files_cache_name()), write=True) as fd:
             pass  # empty file
-        self.cache_config.manifest_id = ''
-        self.cache_config._config.set('cache', 'manifest', '')
+        self.cache_config.integrity[files_cache_name()] = fd.integrity_data
+        self.cache_config.manifest_id = ""
+        self.cache_config._config.set("cache", "manifest", "")
+        if not self.cache_config._config.has_section("integrity"):
+            self.cache_config._config.add_section("integrity")
+        for file, integrity_data in self.cache_config.integrity.items():
+            self.cache_config._config.set("integrity", file, integrity_data)
+        # This is needed to pass the integrity check later on inside CacheConfig.load()
+        self.cache_config._config.set("integrity", "manifest", "")
 
         self.cache_config.ignored_features = set()
         self.cache_config.mandatory_features = set()
+        with SaveFile(self.cache_config.config_path) as fd:
+            self.cache_config._config.write(fd)
 
     def update_compatibility(self):
         operation_to_features_map = self.manifest.get_all_mandatory_features()


Test patch:
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
index 3eb5a78b..45645711 100644
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -38,6 +38,7 @@
 from ..crypto.key import KeyfileKeyBase, RepoKey, KeyfileKey, Passphrase, TAMRequiredError, ArchiveTAMRequiredError
 from ..crypto.keymanager import RepoIdMismatch, NotABorgKeyFile
 from ..crypto.file_integrity import FileIntegrityError
+from ..hashindex import ChunkIndex
 from ..helpers import Location, get_security_dir
 from ..helpers import Manifest, MandatoryFeatureUnsupported, ArchiveInfo
 from ..helpers import init_ec_warnings
@@ -4361,15 +4362,30 @@ def corrupt(self, file, amount=1):
             fd.seek(-amount, io.SEEK_END)
             fd.write(corrupted)
 
+    @pytest.mark.allow_cache_wipe
     def test_cache_chunks(self):
-        self.corrupt(os.path.join(self.cache_path, 'chunks'))
+        self.create_src_archive("test")
+        chunks_path = os.path.join(self.cache_path, 'chunks')
+        chunks_before_corruption = set(ChunkIndex(path=chunks_path).iteritems())
+        self.corrupt(chunks_path)
 
-        if self.FORK_DEFAULT:
-            out = self.cmd('info', self.repository_location, exit_code=2)
-            assert 'failed integrity check' in out
-        else:
-            with pytest.raises(FileIntegrityError):
-                self.cmd('info', self.repository_location)
+        assert not self.FORK_DEFAULT  # test does not support forking
+
+        chunks_in_memory = None
+        sync_chunks = LocalCache.sync
+
+        def sync_wrapper(cache):
+            nonlocal chunks_in_memory
+            sync_chunks(cache)
+            chunks_in_memory = set(cache.chunks.iteritems())
+
+        with patch.object(LocalCache, "sync", sync_wrapper):
+            out = self.cmd("info", self.repository_location)
+
+        assert chunks_in_memory == chunks_before_corruption
+        assert "forcing a cache rebuild" in out
+        chunks_after_repair = set(ChunkIndex(path=chunks_path).iteritems())
+        assert chunks_after_repair == chunks_before_corruption
 
     def test_cache_files(self):
         self.cmd('create', self.repository_location + '::test', 'input')