Repository: borgbackup/borg

Issue:
display filename in progress indicator when a file is started
check whether that is the case. esp. when backing up big files, they often do not show up in the PI.

but this could also be caused by the update throttling (the PI does not display every file, but only updates if a specific time has passed).



Discussion:
yes, progress indication is updated at the end, after all data has already been stored.

see add_item().

bad UX as there is a long time with no display update while it transfers chunks of a big file.


Resolving patch:
diff --git a/borg/archive.py b/borg/archive.py
--- a/borg/archive.py
+++ b/borg/archive.py
@@ -143,7 +143,6 @@ def __init__(self, repository, key, manifest, name, cache=None, create=False,
         self.hard_links = {}
         self.stats = Statistics()
         self.show_progress = progress
-        self.last_progress = time.time()
         self.name = name
         self.checkpoint_interval = checkpoint_interval
         self.numeric_owner = numeric_owner
@@ -215,9 +214,8 @@ def add_item(self, item):
         unknown_keys = set(item) - ITEM_KEYS
         assert not unknown_keys, ('unknown item metadata keys detected, please update ITEM_KEYS: %s',
                                   ','.join(k.decode('ascii') for k in unknown_keys))
-        if self.show_progress and time.time() - self.last_progress > 0.2:
-            self.stats.show_progress(item=item)
-            self.last_progress = time.time()
+        if self.show_progress:
+            self.stats.show_progress(item=item, dt=0.2)
         self.items_buffer.add(item)
         if time.time() - self.last_checkpoint > self.checkpoint_interval:
             self.write_checkpoint()
@@ -526,6 +524,7 @@ def process_file(self, path, st, cache):
                 status = 'U'  # regular file, unchanged
         else:
             status = 'A'  # regular file, added
+        item = {b'path': safe_path}
         # Only chunkify the file if needed
         if chunks is None:
             fh = Archive._open_rb(path, st)
@@ -533,9 +532,11 @@ def process_file(self, path, st, cache):
                 chunks = []
                 for chunk in self.chunker.chunkify(fd, fh):
                     chunks.append(cache.add_chunk(self.key.id_hash(chunk), chunk, self.stats))
+                    if self.show_progress:
+                        self.stats.show_progress(item=item, dt=0.2)
             cache.memorize_file(path_hash, st, [c[0] for c in chunks])
             status = status or 'M'  # regular file, modified (if not 'A' already)
-        item = {b'path': safe_path, b'chunks': chunks}
+        item[b'chunks'] = chunks
         item.update(self.stat_attrs(st, path))
         self.stats.nfiles += 1
         self.add_item(item)
diff --git a/borg/helpers.py b/borg/helpers.py
--- a/borg/helpers.py
+++ b/borg/helpers.py
@@ -162,6 +162,7 @@ class Statistics:
 
     def __init__(self):
         self.osize = self.csize = self.usize = self.nfiles = 0
+        self.last_progress = 0  # timestamp when last progress was shown
 
     def update(self, size, csize, unique):
         self.osize += size
@@ -191,19 +192,22 @@ def usize_fmt(self):
     def csize_fmt(self):
         return format_file_size(self.csize)
 
-    def show_progress(self, item=None, final=False, stream=None):
-        columns, lines = get_terminal_size()
-        if not final:
-            msg = '{0.osize_fmt} O {0.csize_fmt} C {0.usize_fmt} D {0.nfiles} N '.format(self)
-            path = remove_surrogates(item[b'path']) if item else ''
-            space = columns - len(msg)
-            if space < len('...') + len(path):
-                path = '%s...%s' % (path[:(space//2)-len('...')], path[-space//2:])
-            msg += "{0:<{space}}".format(path, space=space)
-        else:
-            msg = ' ' * columns
-        print(msg, file=stream or sys.stderr, end="\r")
-        (stream or sys.stderr).flush()
+    def show_progress(self, item=None, final=False, stream=None, dt=None):
+        now = time.time()
+        if dt is None or now - self.last_progress > dt:
+            self.last_progress = now
+            columns, lines = get_terminal_size()
+            if not final:
+                msg = '{0.osize_fmt} O {0.csize_fmt} C {0.usize_fmt} D {0.nfiles} N '.format(self)
+                path = remove_surrogates(item[b'path']) if item else ''
+                space = columns - len(msg)
+                if space < len('...') + len(path):
+                    path = '%s...%s' % (path[:(space//2)-len('...')], path[-space//2:])
+                msg += "{0:<{space}}".format(path, space=space)
+            else:
+                msg = ' ' * columns
+            print(msg, file=stream or sys.stderr, end="\r")
+            (stream or sys.stderr).flush()
 
 
 def get_keys_dir():


Test patch: