Repository: borgbackup/borg

Issue:
Add new key "dsize" for "borg list --format"
Hi, in the keys for the --format argument you have "size, csize, num_chunks and unique_chunks". Would it be difficult to have a "dsize" that indicates the amount of deduplicated size (and perhaps a cdsize, the compressed deduplicated size). That way, I know EXACTLY, where the 1,41GB of deduplicated (compressed) size of my archive are.

---

:moneybag: [there is a bounty for this](https://www.bountysource.com/issues/42062772-add-new-key-dsize-for-borg-list-format)


Discussion:
Good idea. Let's choose a name first:
- dsize (like deduplicated size) or unique_size (sum of all chunk sizes of unique chunks, chunks that are only used by this file)
- cdsize / dcsize (deduplicated compressed size) or unique_csize (sum of all chunk csizes of unique chunks...)

To be consistent with the other *size keys, I'd go with the shorter versions. Besides, long format strings would be less long :)
- dsize / usize  (the latter is used somewhere in the stats code already iirc)
- psize

  proportional size, of course! `csize / refcount`.

Resolving patch:
diff --git a/src/borg/helpers.py b/src/borg/helpers.py
--- a/src/borg/helpers.py
+++ b/src/borg/helpers.py
@@ -19,7 +19,7 @@
 import unicodedata
 import uuid
 from binascii import hexlify
-from collections import namedtuple, deque, abc
+from collections import namedtuple, deque, abc, Counter
 from datetime import datetime, timezone, timedelta
 from fnmatch import translate
 from functools import wraps, partial, lru_cache
@@ -1425,12 +1425,14 @@ class ItemFormatter(BaseFormatter):
         'source': 'link target for links (identical to linktarget)',
         'extra': 'prepends {source} with " -> " for soft links and " link to " for hard links',
         'csize': 'compressed size',
+        'dsize': 'deduplicated size',
+        'dcsize': 'deduplicated compressed size',
         'num_chunks': 'number of chunks in this file',
         'unique_chunks': 'number of unique chunks in this file',
     }
     KEY_GROUPS = (
         ('type', 'mode', 'uid', 'gid', 'user', 'group', 'path', 'bpath', 'source', 'linktarget', 'flags'),
-        ('size', 'csize', 'num_chunks', 'unique_chunks'),
+        ('size', 'csize', 'dsize', 'dcsize', 'num_chunks', 'unique_chunks'),
         ('mtime', 'ctime', 'atime', 'isomtime', 'isoctime', 'isoatime'),
         tuple(sorted(hashlib.algorithms_guaranteed)),
         ('archiveid', 'archivename', 'extra'),
@@ -1479,8 +1481,10 @@ def __init__(self, archive, format):
         self.call_keys = {
             'size': self.calculate_size,
             'csize': self.calculate_csize,
+            'dsize': partial(self.sum_unique_chunks_metadata, lambda chunk: chunk.size),
+            'dcsize': partial(self.sum_unique_chunks_metadata, lambda chunk: chunk.csize),
             'num_chunks': self.calculate_num_chunks,
-            'unique_chunks': self.calculate_unique_chunks,
+            'unique_chunks': partial(self.sum_unique_chunks_metadata, lambda chunk: 1),
             'isomtime': partial(self.format_time, 'mtime'),
             'isoctime': partial(self.format_time, 'ctime'),
             'isoatime': partial(self.format_time, 'atime'),
@@ -1527,12 +1531,22 @@ def get_item_data(self, item):
             item_data[key] = self.call_keys[key](item)
         return item_data
 
-    def calculate_num_chunks(self, item):
-        return len(item.get('chunks', []))
+    def sum_unique_chunks_metadata(self, metadata_func, item):
+        """
+        sum unique chunks metadata, a unique chunk is a chunk which is referenced globally as often as it is in the
+        item
 
-    def calculate_unique_chunks(self, item):
+        item: The item to sum its unique chunks' metadata
+        metadata_func: A function that takes a parameter of type ChunkIndexEntry and returns a number, used to return
+                       the metadata needed from the chunk
+        """
         chunk_index = self.archive.cache.chunks
-        return sum(1 for c in item.get('chunks', []) if chunk_index[c.id].refcount == 1)
+        chunks = item.get('chunks', [])
+        chunks_counter = Counter(c.id for c in chunks)
+        return sum(metadata_func(c) for c in chunks if chunk_index[c.id].refcount == chunks_counter[c.id])
+
+    def calculate_num_chunks(self, item):
+        return len(item.get('chunks', []))
 
     def calculate_size(self, item):
         return sum(c.size for c in item.get('chunks', []))
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -1298,9 +1298,12 @@ def test_list_size(self):
         self.cmd('init', self.repository_location)
         test_archive = self.repository_location + '::test'
         self.cmd('create', '-C', 'lz4', test_archive, 'input')
-        output = self.cmd('list', '--format', '{size} {csize} {path}{NL}', test_archive)
-        size, csize, path = output.split("\n")[1].split(" ")
+        output = self.cmd('list', '--format', '{size} {csize} {dsize} {dcsize} {path}{NL}', test_archive)
+        size, csize, dsize, dcsize, path = output.split("\n")[1].split(" ")
         assert int(csize) < int(size)
+        assert int(dcsize) < int(dsize)
+        assert int(dsize) <= int(size)
+        assert int(dcsize) <= int(csize)
 
     def _get_sizes(self, compression, compressible, size=10000):
         if compressible:


Test patch: