Repository: borgbackup/borg

Issue:
Comparing modified items incorrectly assumes comparable chunk ids
This came up in #2842 (see during https://github.com/borgbackup/borg/pull/2842/files#diff-0c1790c20ac3974e84f97b2ded80507bR449) but the problem was there before https://github.com/borgbackup/borg/blob/master/src/borg/archiver.py#L1028.
The current solution of comparing just chunk ids is fine as long as the two archives were created with the same chunker params, but doesn't work otherwise.
I am not sure what would be the cheapest way to get matching contents from two chunk iterators. Maybe creating new chunks from the chunks with the same params?


Discussion:
iirc i have seen some code that actually reads the file contents then and compares them, which is expensive.
but borg diff warns about being expensive in that case.
Yes [this is the warning](https://github.com/borgbackup/borg/blob/master/src/borg/archiver.py#L1162) and [this the expensive function](https://github.com/borgbackup/borg/blob/master/src/borg/archiver.py#L199), but that only determines whether they are equal. Thus as soon as there is a difference it can return. Finding matching content to return how many bytes were removed and added is more complex as shifted content needs to be accounted for.
I guess we rather won't compute it for that case. Changing chunker params is a rare event, so not knowing how much changed exactly when crossing that boundary is no big issue.
Fine, then probably just put in a generic statement ("modified file"). I think this was introduced in 1.0. Should a PR fixing this be against 1.0-maint or will that get picked from master?
> Should a PR fixing this be against 1.0-maint or will that get picked from master?

The latter ([long version](http://borgbackup.readthedocs.io/en/latest/development.html#branching-model)). None of this is in 1.0, though.

Resolving patch:
diff --git a/src/borg/archiver.py b/src/borg/archiver.py
--- a/src/borg/archiver.py
+++ b/src/borg/archiver.py
@@ -1013,18 +1013,19 @@ def contents_changed(item1, item2):
         def compare_content(path, item1, item2):
             if contents_changed(item1, item2):
                 if item1.get('deleted'):
-                    return ('added {:>13}'.format(format_file_size(sum_chunk_size(item2))))
-                elif item2.get('deleted'):
-                    return ('removed {:>11}'.format(format_file_size(sum_chunk_size(item1))))
-                else:
-                    chunk_ids1 = {c.id for c in item1.chunks}
-                    chunk_ids2 = {c.id for c in item2.chunks}
-                    added_ids = chunk_ids2 - chunk_ids1
-                    removed_ids = chunk_ids1 - chunk_ids2
-                    added = sum_chunk_size(item2, added_ids)
-                    removed = sum_chunk_size(item1, removed_ids)
-                    return ('{:>9} {:>9}'.format(format_file_size(added, precision=1, sign=True),
-                                                 format_file_size(-removed, precision=1, sign=True)))
+                    return 'added {:>13}'.format(format_file_size(sum_chunk_size(item2)))
+                if item2.get('deleted'):
+                    return 'removed {:>11}'.format(format_file_size(sum_chunk_size(item1)))
+                if not can_compare_chunk_ids:
+                    return 'modified'
+                chunk_ids1 = {c.id for c in item1.chunks}
+                chunk_ids2 = {c.id for c in item2.chunks}
+                added_ids = chunk_ids2 - chunk_ids1
+                removed_ids = chunk_ids1 - chunk_ids2
+                added = sum_chunk_size(item2, added_ids)
+                removed = sum_chunk_size(item1, removed_ids)
+                return '{:>9} {:>9}'.format(format_file_size(added, precision=1, sign=True),
+                                            format_file_size(-removed, precision=1, sign=True))
 
         def compare_directory(item1, item2):
             if item2.get('deleted') and not item1.get('deleted'):
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -3276,9 +3276,10 @@ def test_basic_functionality(self):
         self.cmd('create', self.repository_location + '::test1a', 'input')
         self.cmd('create', '--chunker-params', '16,18,17,4095', self.repository_location + '::test1b', 'input')
 
-        def do_asserts(output, archive):
+        def do_asserts(output, can_compare_ids):
             # File contents changed (deleted and replaced with a new file)
-            assert 'B input/file_replaced' in output
+            change = 'B' if can_compare_ids else '{:<19}'.format('modified')
+            assert '{} input/file_replaced'.format(change) in output
 
             # File unchanged
             assert 'input/file_unchanged' not in output
@@ -3307,9 +3308,10 @@ def do_asserts(output, archive):
             # The inode has two links and the file contents changed. Borg
             # should notice the changes in both links. However, the symlink
             # pointing to the file is not changed.
-            assert '0 B input/empty' in output
+            change = '0 B' if can_compare_ids else '{:<19}'.format('modified')
+            assert '{} input/empty'.format(change) in output
             if are_hardlinks_supported():
-                assert '0 B input/hardlink_contents_changed' in output
+                assert '{} input/hardlink_contents_changed'.format(change) in output
             if are_symlinks_supported():
                 assert 'input/link_target_contents_changed' not in output
 
@@ -3336,9 +3338,9 @@ def do_asserts(output, archive):
             if are_hardlinks_supported():
                 assert 'input/hardlink_target_replaced' not in output
 
-        do_asserts(self.cmd('diff', self.repository_location + '::test0', 'test1a'), '1a')
+        do_asserts(self.cmd('diff', self.repository_location + '::test0', 'test1a'), True)
         # We expect exit_code=1 due to the chunker params warning
-        do_asserts(self.cmd('diff', self.repository_location + '::test0', 'test1b', exit_code=1), '1b')
+        do_asserts(self.cmd('diff', self.repository_location + '::test0', 'test1b', exit_code=1), False)
 
     def test_sort_option(self):
         self.cmd('init', '--encryption=repokey', self.repository_location)


Test patch: