Repository: borgbackup/borg

Issue:
bad files cache stops backup - instead of recreating cache
Problem happened again.  I upgraded to 1.1.4, and it doesn't automatically re-create the "files" cache when I run "create".

How do I force Borg to recreate the files cache?  Shouldn't it automatically recreate it when it is found to be invalid?

See also: #3537 

> File failed integrity check: /home/pi/.cache/53f95bf25bdb1d96526e6de3ac66acb9b1e01a3d3bf30b184a4dab2f1057135b/files
Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/borg/archiver.py", line 4175, in main
    exit_code = archiver.run(args)
  File "/usr/lib/python3/dist-packages/borg/archiver.py", line 4107, in run
    return set_ec(func(args))
  File "/usr/lib/python3/dist-packages/borg/archiver.py", line 150, in wrapper
    return method(self, args, repository=repository, **kwargs)
  File "/usr/lib/python3/dist-packages/borg/archiver.py", line 545, in do_create
    create_inner(archive, cache)
  File "/usr/lib/python3/dist-packages/borg/archiver.py", line 508, in create_inner
    read_special=args.read_special, dry_run=dry_run, st=st)
  File "/usr/lib/python3/dist-packages/borg/archiver.py", line 619, in _process
    read_special=read_special, dry_run=dry_run)
  File "/usr/lib/python3/dist-packages/borg/archiver.py", line 596, in _process
    status = archive.process_file(path, st, cache, self.ignore_inode, self.files_cache_mode)
  File "/usr/lib/python3/dist-packages/borg/archive.py", line 988, in process_file
    ids = cache.file_known_and_unchanged(path_hash, st, ignore_inode, files_cache_mode)
  File "/usr/lib/python3/dist-packages/borg/cache.py", line 917, in file_known_and_unchanged
    self._read_files()
  File "/usr/lib/python3/dist-packages/borg/cache.py", line 523, in _read_files
    return
  File "/usr/lib/python3/dist-packages/borg/crypto/file_integrity.py", line 193, in __exit__
    self.hash_part('final', is_final=True)
  File "/usr/lib/python3/dist-packages/borg/crypto/file_integrity.py", line 188, in hash_part
    raise FileIntegrityError(self.path)
borg.crypto.file_integrity.FileIntegrityError: File failed integrity check: /home/pi/.cache/53f95bf25bdb1d96526e6de3ac66acb9b1e01a3d3bf30b184a4dab2f1057135b/files



Discussion:
Well, there are 2 "problems" here:
- the machine seems to have disk/fs/whatever problems, leading to frequent integrity issues
- borg does not automatically deal with that

The last one could be seen as a feature though as it is better to barf here than to continue working - who knows what else gets corrupt / does malfunction, if saving and reloading a cache file already gives different content.
I found some problems in the code:
- borg not ignoring a corrupt files cache (and continuing and finally writing a valid one)
- no easy way to get out of that situation, neither removing nor truncating the files cache helped

Resolving patch:
diff --git a/src/borg/cache.py b/src/borg/cache.py
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -502,25 +502,32 @@ def _read_files(self):
         self.files = {}
         self._newest_cmtime = None
         logger.debug('Reading files cache ...')
-
-        with IntegrityCheckedFile(path=os.path.join(self.path, 'files'), write=False,
-                                  integrity_data=self.cache_config.integrity.get('files')) as fd:
-            u = msgpack.Unpacker(use_list=True)
-            while True:
-                data = fd.read(64 * 1024)
-                if not data:
-                    break
-                u.feed(data)
-                try:
-                    for path_hash, item in u:
-                        entry = FileCacheEntry(*item)
-                        # in the end, this takes about 240 Bytes per file
-                        self.files[path_hash] = msgpack.packb(entry._replace(age=entry.age + 1))
-                except (TypeError, ValueError) as exc:
-                    logger.warning('The files cache seems corrupt, ignoring it. '
-                                   'Expect lower performance. [%s]' % str(exc))
-                    self.files = {}
-                    return
+        msg = None
+        try:
+            with IntegrityCheckedFile(path=os.path.join(self.path, 'files'), write=False,
+                                      integrity_data=self.cache_config.integrity.get('files')) as fd:
+                u = msgpack.Unpacker(use_list=True)
+                while True:
+                    data = fd.read(64 * 1024)
+                    if not data:
+                        break
+                    u.feed(data)
+                    try:
+                        for path_hash, item in u:
+                            entry = FileCacheEntry(*item)
+                            # in the end, this takes about 240 Bytes per file
+                            self.files[path_hash] = msgpack.packb(entry._replace(age=entry.age + 1))
+                    except (TypeError, ValueError) as exc:
+                        msg = "The files cache seems invalid. [%s]" % str(exc)
+                        break
+        except OSError as exc:
+            msg = "The files cache can't be read. [%s]" % str(exc)
+        except FileIntegrityError as fie:
+            msg = "The files cache is corrupted. [%s]" % str(fie)
+        if msg is not None:
+            logger.warning(msg)
+            logger.warning('Continuing without files cache - expect lower performance.')
+            self.files = {}
 
     def begin_txn(self):
         # Initialize transaction snapshot
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -3325,13 +3325,9 @@ def test_cache_chunks(self):
     def test_cache_files(self):
         self.cmd('create', self.repository_location + '::test', 'input')
         self.corrupt(os.path.join(self.cache_path, 'files'))
-
-        if self.FORK_DEFAULT:
-            out = self.cmd('create', self.repository_location + '::test1', 'input', exit_code=2)
-            assert 'failed integrity check' in out
-        else:
-            with pytest.raises(FileIntegrityError):
-                self.cmd('create', self.repository_location + '::test1', 'input')
+        out = self.cmd('create', self.repository_location + '::test1', 'input')
+        # borg warns about the corrupt files cache, but then continues without files cache.
+        assert 'files cache is corrupted' in out
 
     def test_chunks_archive(self):
         self.cmd('create', self.repository_location + '::test1', 'input')


Test patch: