Repository: borgbackup/borg

Issue:
factor out progress out of upgrade and create
upgrade and create both use the hackish `\r` to display progress. that may be fine, but maybe we want to refactor `report_progress()` so it can be used by `upgrade`?

similarly, there should be a `--progress` argument to `upgrade`...



Discussion:
I am closing this as upgrade now got a progress indicator (but not using the same code as create).
It uses one of some progress indicator classes implemented in helpers.

As upgrade basically counts segment files while create is giving detailled statistics with original, compressed, deduplicated amounts, they are rather different anyway.

there were two things this was supposed to do:
1. add a --progress flag to `upgrade` (not done)
2. factor out the `\r` stuff out of `upgrade` and `create` so that we have one set of tests and one codebase to handle that kind of "one-line display" (not done)

Do you think it makes sense to check the --progress switch there (or asked differently: that it is silent by default)?

I thought this is a place where one unconditionally wants to see a progress indication, that is why I didn't check for the --progress option.

yeah, i guess it may not make sense to run upgrade in batch - but you never know! maybe some provider will be running a large number of borg repositories and upgrade them all at once, and won'T want that garbage in their logs! so yes, that's anothe reason for refactoring this so it's at least consistent.

note: adding --progress flags is easy, but for repository operations that run on the server, flag needs to be transmitted to remote side also.

as borg upgrade does not support remote repos currently, see #465, the passing of the progress flag via rpc is not an issue yet.


Resolving patch:
diff --git a/borg/archive.py b/borg/archive.py
--- a/borg/archive.py
+++ b/borg/archive.py
@@ -18,7 +18,7 @@
 from . import xattr
 from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, format_timedelta, \
     Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, \
-    st_atime_ns, st_ctime_ns, st_mtime_ns
+    st_atime_ns, st_ctime_ns, st_mtime_ns, ProgressIndicatorPercent
 from .platform import acl_get, acl_set
 from .chunker import Chunker
 from .hashindex import ChunkIndex
@@ -418,16 +418,21 @@ def rename(self, name):
         self.cache.chunk_decref(self.id, self.stats)
         del self.manifest.archives[self.name]
 
-    def delete(self, stats):
+    def delete(self, stats, progress=False):
         unpacker = msgpack.Unpacker(use_list=False)
-        for items_id, data in zip(self.metadata[b'items'], self.repository.get_many(self.metadata[b'items'])):
+        items_ids = self.metadata[b'items']
+        pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%", same_line=True)
+        for (i, (items_id, data)) in enumerate(zip(items_ids, self.repository.get_many(items_ids))):
+            if progress:
+                pi.show(i)
             unpacker.feed(self.key.decrypt(items_id, data))
             self.cache.chunk_decref(items_id, stats)
             for item in unpacker:
                 if b'chunks' in item:
                     for chunk_id, size, csize in item[b'chunks']:
                         self.cache.chunk_decref(chunk_id, stats)
-
+        if progress:
+            pi.finish()
         self.cache.chunk_decref(self.id, stats)
         del self.manifest.archives[self.name]
 
diff --git a/borg/archiver.py b/borg/archiver.py
--- a/borg/archiver.py
+++ b/borg/archiver.py
@@ -338,7 +338,7 @@ def do_delete(self, args):
         if args.location.archive:
             archive = Archive(repository, key, manifest, args.location.archive, cache=cache)
             stats = Statistics()
-            archive.delete(stats)
+            archive.delete(stats, progress=args.progress)
             manifest.write()
             repository.commit(save_space=args.save_space)
             cache.commit()
@@ -520,7 +520,7 @@ def do_upgrade(self, args):
         # XXX: should auto-detect if it is an attic repository here
         repo = AtticRepositoryUpgrader(args.location.path, create=False)
         try:
-            repo.upgrade(args.dry_run, inplace=args.inplace)
+            repo.upgrade(args.dry_run, inplace=args.inplace, progress=args.progress)
         except NotImplementedError as e:
             print("warning: %s" % e)
         return self.exit_code
@@ -982,6 +982,9 @@ def build_parser(self, args=None, prog=None):
                                           epilog=delete_epilog,
                                           formatter_class=argparse.RawDescriptionHelpFormatter)
         subparser.set_defaults(func=self.do_delete)
+        subparser.add_argument('-p', '--progress', dest='progress',
+                               action='store_true', default=False,
+                               help="""show progress display while deleting a single archive""")
         subparser.add_argument('-s', '--stats', dest='stats',
                                action='store_true', default=False,
                                help='print statistics for the deleted archive')
@@ -1156,6 +1159,9 @@ def build_parser(self, args=None, prog=None):
                                           epilog=upgrade_epilog,
                                           formatter_class=argparse.RawDescriptionHelpFormatter)
         subparser.set_defaults(func=self.do_upgrade)
+        subparser.add_argument('-p', '--progress', dest='progress',
+                               action='store_true', default=False,
+                               help="""show progress display while upgrading the repository""")
         subparser.add_argument('-n', '--dry-run', dest='dry_run',
                                default=False, action='store_true',
                                help='do not change repository')
diff --git a/borg/upgrader.py b/borg/upgrader.py
--- a/borg/upgrader.py
+++ b/borg/upgrader.py
@@ -20,7 +20,7 @@ def __init__(self, *args, **kw):
         kw['lock'] = False  # do not create borg lock files (now) in attic repo
         super().__init__(*args, **kw)
 
-    def upgrade(self, dryrun=True, inplace=False):
+    def upgrade(self, dryrun=True, inplace=False, progress=False):
         """convert an attic repository to a borg repository
 
         those are the files that need to be upgraded here, from most
@@ -54,7 +54,7 @@ def upgrade(self, dryrun=True, inplace=False):
         try:
             self.convert_cache(dryrun)
             self.convert_repo_index(dryrun=dryrun, inplace=inplace)
-            self.convert_segments(segments, dryrun=dryrun, inplace=inplace)
+            self.convert_segments(segments, dryrun=dryrun, inplace=inplace, progress=progress)
             self.borg_readme()
         finally:
             self.lock.release()
@@ -68,7 +68,7 @@ def borg_readme(self):
             fd.write('This is a Borg repository\n')
 
     @staticmethod
-    def convert_segments(segments, dryrun=True, inplace=False):
+    def convert_segments(segments, dryrun=True, inplace=False, progress=False):
         """convert repository segments from attic to borg
 
         replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in
@@ -80,12 +80,14 @@ def convert_segments(segments, dryrun=True, inplace=False):
         segment_count = len(segments)
         pi = ProgressIndicatorPercent(total=segment_count, msg="Converting segments %3.0f%%", same_line=True)
         for i, filename in enumerate(segments):
-            pi.show(i)
+            if progress:
+                pi.show(i)
             if dryrun:
                 time.sleep(0.001)
             else:
                 AtticRepositoryUpgrader.header_replace(filename, ATTIC_MAGIC, MAGIC, inplace=inplace)
-        pi.finish()
+        if progress:
+            pi.finish()
 
     @staticmethod
     def header_replace(filename, old_magic, new_magic, inplace=True):


Test patch: