Repository: borgbackup/borg

Issue:
borg diff not recognizing identical chunk params when diffing v1.1 with v1.2 archive
## Have you checked borgbackup docs, FAQ, and open Github issues?

Yes

## Is this a BUG / ISSUE report or a QUESTION?

ISSUE

## System information. For client/server mode post info for both machines.

#### Your borg version (borg -V).

```
$ borg -V
borg 1.2.2
```

#### Full borg commandline that lead to the problem (leave away excludes and passwords)

```
$ borg diff ::main-2022-09-30T13:41:46Z main-2022-10-03T07:50:40Z
--chunker-params might be different between archives, diff will be slow.
If you know for certain that they are the same, pass --same-chunker-params to override this check.
…
```

## Describe the problem you're observing.

The former archive in the above command was created by Borg v1.1:

```
$ borg debug dump-archive ::main-2022-09-30T13:41:46Z -
{
    …
    "_meta":
    {
        "chunker_params": [
            19,
            23,
            21,
            4095
        ],
…
```

And the latter by Borg v1.2:

```
$ borg debug dump-archive ::main-2022-10-03T07:50:40Z -
{
    …
    "_meta":
    {
        "chunker_params": [
            "buzhash",
            19,
            23,
            21,
            4095
        ],

```

As can be seen, in Borg v1.2 the name for the chunker was added to the params. I suspect this leads to this issue.

I'd consider the following changes to improve the situation:

- [x] Correct comparison between v1.1 and v1.2 chunk params in v1.2.x.
- [x] Convert the chunk params to the newer format during the `borg transfer` required to upgrade to v2.0. As of Borg 2.0.0b3 the chunk params are kept as is but this would probably be a good time to normalize them and drop the code to handle the old format.


Discussion:
Very good analysis / suggestions, thanks!

Resolving patch:
diff --git a/src/borg/archive.py b/src/borg/archive.py
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -28,7 +28,7 @@
 from .hashindex import ChunkIndex, ChunkIndexEntry, CacheSynchronizer
 from .helpers import Manifest
 from .helpers import hardlinkable
-from .helpers import ChunkIteratorFileWrapper, open_item
+from .helpers import ChunkIteratorFileWrapper, normalize_chunker_params, open_item
 from .helpers import Error, IntegrityError, set_ec
 from .platform import uid2user, user2uid, gid2group, group2gid
 from .helpers import parse_timestamp, to_localtime
@@ -547,12 +547,14 @@ def info(self):
         if self.create:
             info['command_line'] = sys.argv
         else:
+            cp = self.metadata.get('chunker_params')
+            cp = normalize_chunker_params(cp) if cp is not None else ''
             info.update({
                 'command_line': self.metadata.cmdline,
                 'hostname': self.metadata.hostname,
                 'username': self.metadata.username,
                 'comment': self.metadata.get('comment', ''),
-                'chunker_params': self.metadata.get('chunker_params', ''),
+                'chunker_params': cp,
             })
         return info
 
@@ -2302,13 +2304,12 @@ def create_target(self, archive, target_name=None):
         target_name = target_name or archive.name + '.recreate'
         target = self.create_target_archive(target_name)
         # If the archives use the same chunker params, then don't rechunkify
-        source_chunker_params = tuple(archive.metadata.get('chunker_params', []))
-        if len(source_chunker_params) == 4 and isinstance(source_chunker_params[0], int):
-            # this is a borg < 1.2 chunker_params tuple, no chunker algo specified, but we only had buzhash:
-            source_chunker_params = (CH_BUZHASH, ) + source_chunker_params
-        target.recreate_rechunkify = self.rechunkify and source_chunker_params != target.chunker_params
+        src_cp = archive.metadata.get('chunker_params')
+        src_cp = normalize_chunker_params(src_cp) if src_cp is not None else None
+        dst_cp = target.chunker_params
+        target.recreate_rechunkify = self.rechunkify and src_cp != dst_cp
         if target.recreate_rechunkify:
-            logger.debug('Rechunking archive from %s to %s', source_chunker_params or '(unknown)', target.chunker_params)
+            logger.debug('Rechunking archive from %s to %s', src_cp or '(unknown)', dst_cp)
         target.process_file_chunks = ChunksProcessor(
             cache=self.cache, key=self.key,
             add_item=target.add_item, write_checkpoint=target.write_checkpoint,
diff --git a/src/borg/archiver.py b/src/borg/archiver.py
--- a/src/borg/archiver.py
+++ b/src/borg/archiver.py
@@ -67,7 +67,7 @@
     from .helpers import ProgressIndicatorPercent
     from .helpers import basic_json_data, json_print
     from .helpers import replace_placeholders
-    from .helpers import ChunkIteratorFileWrapper
+    from .helpers import ChunkIteratorFileWrapper, normalize_chunker_params
     from .helpers import popen_with_error_handling, prepare_subprocess_env, create_filter_process
     from .helpers import dash_open
     from .helpers import umount
@@ -1125,9 +1125,18 @@ def print_text_output(diff, path):
         archive2 = Archive(repository, key, manifest, args.archive2,
                            consider_part_files=args.consider_part_files)
 
-        can_compare_chunk_ids = archive1.metadata.get('chunker_params', False) == archive2.metadata.get(
-            'chunker_params', True) or args.same_chunker_params
-        if not can_compare_chunk_ids:
+        cp1 = archive1.metadata.get('chunker_params')
+        cp2 = archive2.metadata.get('chunker_params')
+        if args.same_chunker_params:
+            can_compare_chunk_ids = True  # enforce it
+        elif cp1 is not None and cp2 is not None:
+            # we know chunker params of both archives
+            can_compare_chunk_ids = normalize_chunker_params(cp1) == normalize_chunker_params(cp2)
+            if not can_compare_chunk_ids:
+                self.print_warning('--chunker-params are different between archives, diff will be slow.')
+        else:
+            # we do not know chunker params of at least one of the archives
+            can_compare_chunk_ids = False
             self.print_warning('--chunker-params might be different between archives, diff will be slow.\n'
                                'If you know for certain that they are the same, pass --same-chunker-params '
                                'to override this check.')
diff --git a/src/borg/helpers/misc.py b/src/borg/helpers/misc.py
--- a/src/borg/helpers/misc.py
+++ b/src/borg/helpers/misc.py
@@ -16,6 +16,7 @@
 from . import msgpack
 from .. import __version__ as borg_version
 from .. import chunker
+from ..constants import CH_BUZHASH, CH_FIXED
 
 
 def prune_within(archives, hours, kept_because):
@@ -119,6 +120,17 @@ def log_multi(*msgs, level=logging.INFO, logger=logger):
         logger.log(level, line)
 
 
+def normalize_chunker_params(cp):
+    assert isinstance(cp, (list, tuple))
+    if isinstance(cp, list):
+        cp = tuple(cp)
+    if len(cp) == 4 and isinstance(cp[0], int):
+        # this is a borg < 1.2 chunker_params tuple, no chunker algo specified, but we only had buzhash:
+        cp = (CH_BUZHASH, ) + cp
+    assert cp[0] in (CH_BUZHASH, CH_FIXED)
+    return cp
+
+
 class ChunkIteratorFileWrapper:
     """File-like wrapper for chunk iterators"""
 


Test patch: