Repository: borgbackup/borg

Issue:
speed of borg create --stats / borg info
master branch / borg 1.2: stats can be slower than before.

https://github.com/borgbackup/borg/discussions/6166#discussioncomment-2040866


Discussion:
Also check `borg info` (maybe same thing).
Hey @ThomasWaldmann 
do you need more input for investigation? I could copy the description of my tests here and add more info if needed.
Wow, just tried `borg info REPO::ARCHIVE` with borg 1.2 on a borg 1.1 archive. That was slow!
Found the culprit:

`Archive.calc_stats()` is fast for borg 1.2 archives (gets summed up stats values from archive metadata), but expensive for borg 1.1 archives (has to read all archived items' metadata to sum up the desired stats values).

For a single archive, this is slower, but still tolerable.

But the real killer comes via "All archives": to correctly compute the total_size and total_csize values, it iterates over all archives in the manifest and calls `calc_stats()` for each of them (see `CacheStatsMixin.stats`). And that can get arbitrarily slow, depending on how many 1.1 archives exist in the repo.

Guess we just could cache this locally. It's only very little information missing from the borg 1.1 archives, but expensive to compute.
Does borg 1.2 show more info/stats than borg 1.1?  I'm asking because `borg info repo` seems fast with borg 1.1, so I'm curious why 1.2 would make this slower.  If it is because 1.2 shows extra info, then maybe the fallback should be to simply not display the extra info for 1.1 archives, but to continue to show what borg 1.1 shows.
@jdchristensen It's slow because it is correct, not because it shows more info.

borg < 1.2 had some issues due to how part files work, so the stats sometimes showed weird, more or less incorrect values.

borg 1.2 computes some stats (summing up size, csize, ..., counting files) while creating an archive and writes them into the archive metadata. Thus very cheap to get afterwards.


Resolving patch:
diff --git a/src/borg/archive.py b/src/borg/archive.py
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -105,6 +105,27 @@ def as_dict(self):
             'nfiles': self.nfiles,
         }
 
+    def as_raw_dict(self):
+        return {
+            'size': self.osize,
+            'csize': self.csize,
+            'nfiles': self.nfiles,
+            'size_parts': self.osize_parts,
+            'csize_parts': self.csize_parts,
+            'nfiles_parts': self.nfiles_parts,
+        }
+
+    @classmethod
+    def from_raw_dict(cls, **kw):
+        self = cls()
+        self.osize = kw['size']
+        self.csize = kw['csize']
+        self.nfiles = kw['nfiles']
+        self.osize_parts = kw['size_parts']
+        self.csize_parts = kw['csize_parts']
+        self.nfiles_parts = kw['nfiles_parts']
+        return self
+
     @property
     def osize_fmt(self):
         return format_file_size(self.osize, iec=self.iec)
@@ -627,6 +648,17 @@ def save(self, name=None, comment=None, timestamp=None, stats=None, additional_m
         self.cache.commit()
 
     def calc_stats(self, cache, want_unique=True):
+        # caching wrapper around _calc_stats which is rather slow for archives made with borg < 1.2
+        have_borg12_meta = self.metadata.get('nfiles') is not None
+        try:
+            stats = Statistics.from_raw_dict(**cache.pre12_meta[self.fpr])
+        except KeyError:  # not in pre12_meta cache
+            stats = self._calc_stats(cache, want_unique=want_unique)
+            if not have_borg12_meta:
+                cache.pre12_meta[self.fpr] = stats.as_raw_dict()
+        return stats
+
+    def _calc_stats(self, cache, want_unique=True):
         have_borg12_meta = self.metadata.get('nfiles') is not None
 
         if have_borg12_meta and not want_unique:
@@ -639,7 +671,8 @@ def add(id):
             archive_index = ChunkIndex()
             sync = CacheSynchronizer(archive_index)
             add(self.id)
-            pi = ProgressIndicatorPercent(total=len(self.metadata.items), msg='Calculating statistics... %3d%%',
+            pi = ProgressIndicatorPercent(total=len(self.metadata.items),
+                                          msg='Calculating statistics for archive %s ... %%3d%%%%' % self.name,
                                           msgid='archive.calc_stats')
             for id, chunk in zip(self.metadata.items, self.repository.get_many(self.metadata.items)):
                 pi.show(increase=1)
diff --git a/src/borg/cache.py b/src/borg/cache.py
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -1,4 +1,5 @@
 import configparser
+import json
 import os
 import shutil
 import stat
@@ -407,6 +408,7 @@ class CacheStatsMixin:
 
     def __init__(self, iec=False):
         self.iec = iec
+        self.pre12_meta = {}  # here we cache archive metadata for borg < 1.2
 
     def __str__(self):
         return self.str_format.format(self.format_tuple())
@@ -511,6 +513,8 @@ def create(self):
         os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
         with SaveFile(os.path.join(self.path, files_cache_name()), binary=True):
             pass  # empty file
+        with SaveFile(os.path.join(self.path, 'pre12-meta'), binary=False) as fd:
+            json.dump(self.pre12_meta, fd, indent=4)
 
     def _do_open(self):
         self.cache_config.load()
@@ -521,6 +525,11 @@ def _do_open(self):
             self.files = None
         else:
             self._read_files()
+        try:
+            with open(os.path.join(self.path, 'pre12-meta')) as fd:
+                self.pre12_meta = json.load(fd)
+        except (FileNotFoundError, json.JSONDecodeError):
+            pass
 
     def open(self):
         if not os.path.isdir(self.path):
@@ -529,6 +538,9 @@ def open(self):
         self.rollback()
 
     def close(self):
+        # save the pre12_meta cache in any case
+        with open(os.path.join(self.path, 'pre12-meta'), 'w') as fd:
+            json.dump(self.pre12_meta, fd, indent=4)
         if self.cache_config is not None:
             self.cache_config.close()
             self.cache_config = None
@@ -1066,6 +1078,7 @@ def __init__(self, repository, key, manifest, warn_if_unencrypted=True, lock_wai
         self.security_manager = SecurityManager(repository)
         self.security_manager.assert_secure(manifest, key, lock_wait=lock_wait)
 
+        self.pre12_meta = {}
         logger.warning('Note: --no-cache-sync is an experimental feature.')
 
     # Public API


Test patch: