Repository: borgbackup/borg

Issue:
borg2 throws error for valid(?) chunker parameters
## Have you checked borgbackup docs, FAQ, and open GitHub issues?

Yes
## Is this a BUG / ISSUE report or a QUESTION?

A bug, probably

## System information. For client/server mode post info for both machines.

#### Your borg version (borg -V).

borg-2.0.0b5

#### Operating system (distribution) and version.

Arch Linux 

#### Hardware / network configuration, and filesystems used.

Linux, borg-2.0.0b5 precompiled binary, ext4, backup to HDD

#### How much data is handled by borg?

a few GB for testing borg2 beta

#### Full borg commandline that lead to the problem (leave away excludes and passwords)

```
borg-2.0.0b5-linuxnew64 create --chunker-params buzhash,19,24,21,4095 MyArchive /path/to/source
```
(I didn't try this with borg-1.2.x)

## Describe the problem you're observing.

tl;dr: running the above command, I get the following error message:

```
borg-2.0.0b5-linuxnew64 create: error: argument --chunker-params: invalid ChunkerParams value: 'buzhash,19,24,21,4095'
```
Note the value of 24 for CHUNK_MAX_EXP.

Long story: I have a backup source with only large (several hundred MiB each) and seldom-changing files and wanted to adjust the chunker parameters for that by going from the default `buzhash,19,23,21,4095` (512 kiB min, 8 MiB max, 2 MiB avg chunk size) to twice that size (i.e. 1 MiB / 16 MiB / 4 MiB)  with `buzhash,20,24,22,4095` (and maybe even to still larger chunk sizes). However, this throws the error message given above. Some try-and-error revealed that the CHUNK_MAX_EXP parameter is to blame. Selecting anything larger than 23 (i.e. 8 MiB)  for CHUNK_MAX_EXP will trigger the error.

I could not find out why `--chunker-params buzhash,20,24,22,4095` should be invalid. The documentation says nothing about limits and/or inter-dependencies for the chunker values. 

Is this a bug? Maybe a leftover from the "old attic times" where nobody could imagine chunk sizes > 8 MiB? 





Discussion:

Patch:
diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx
index e120f133..e259c374 100644
--- a/src/borg/compress.pyx
+++ b/src/borg/compress.pyx
@@ -15,6 +15,7 @@ which compressor has been used to compress the data and dispatch to the correct
 decompressor.
 """
 
+from argparse import ArgumentTypeError
 import random
 from struct import Struct
 import zlib
@@ -652,7 +653,7 @@ class CompressionSpec:
         values = s.split(',')
         count = len(values)
         if count < 1:
-            raise ValueError
+            raise ArgumentTypeError("not enough arguments")
         # --compression algo[,level]
         self.name = values[0]
         if self.name in ('none', 'lz4', ):
@@ -663,9 +664,9 @@ class CompressionSpec:
             elif count == 2:
                 level = int(values[1])
                 if not 0 <= level <= 9:
-                    raise ValueError
+                    raise ArgumentTypeError("level must be >= 0 and <= 9")
             else:
-                raise ValueError
+                raise ArgumentTypeError("too many arguments")
             self.level = level
         elif self.name in ('zstd', ):
             if count < 2:
@@ -673,28 +674,28 @@ class CompressionSpec:
             elif count == 2:
                 level = int(values[1])
                 if not 1 <= level <= 22:
-                    raise ValueError
+                    raise ArgumentTypeError("level must be >= 1 and <= 22")
             else:
-                raise ValueError
+                raise ArgumentTypeError("too many arguments")
             self.level = level
         elif self.name == 'auto':
             if 2 <= count <= 3:
                 compression = ','.join(values[1:])
             else:
-                raise ValueError
+                raise ArgumentTypeError("bad arguments")
             self.inner = CompressionSpec(compression)
         elif self.name == 'obfuscate':
             if 3 <= count <= 5:
                 level = int(values[1])
                 if not ((1 <= level <= 6) or (110 <= level <= 123)):
-                    raise ValueError
+                    raise ArgumentTypeError("level must be >= 1 and <= 6 or >= 110 and <= 123")
                 self.level = level
                 compression = ','.join(values[2:])
             else:
-                raise ValueError
+                raise ArgumentTypeError("bad arguments")
             self.inner = CompressionSpec(compression)
         else:
-            raise ValueError
+            raise ArgumentTypeError("unsupported compression type")
 
     @property
     def compressor(self):
diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py
index 85dc0178..ade24241 100644
--- a/src/borg/helpers/parseformat.py
+++ b/src/borg/helpers/parseformat.py
@@ -137,7 +137,7 @@ def ChunkerParams(s):
     params = s.strip().split(",")
     count = len(params)
     if count == 0:
-        raise ValueError("no chunker params given")
+        raise argparse.ArgumentTypeError("no chunker params given")
     algo = params[0].lower()
     if algo == CH_FAIL and count == 3:
         block_size = int(params[1])
@@ -152,9 +152,11 @@ def ChunkerParams(s):
             # or in-memory chunk management.
             # choose the block (chunk) size wisely: if you have a lot of data and you cut
             # it into very small chunks, you are asking for trouble!
-            raise ValueError("block_size must not be less than 64 Bytes")
+            raise argparse.ArgumentTypeError("block_size must not be less than 64 Bytes")
         if block_size > MAX_DATA_SIZE or header_size > MAX_DATA_SIZE:
-            raise ValueError("block_size and header_size must not exceed MAX_DATA_SIZE [%d]" % MAX_DATA_SIZE)
+            raise argparse.ArgumentTypeError(
+                "block_size and header_size must not exceed MAX_DATA_SIZE [%d]" % MAX_DATA_SIZE
+            )
         return algo, block_size, header_size
     if algo == "default" and count == 1:  # default
         return CHUNKER_PARAMS
@@ -162,14 +164,18 @@ def ChunkerParams(s):
     if algo == CH_BUZHASH and count == 5 or count == 4:  # [buzhash, ]chunk_min, chunk_max, chunk_mask, window_size
         chunk_min, chunk_max, chunk_mask, window_size = (int(p) for p in params[count - 4 :])
         if not (chunk_min <= chunk_mask <= chunk_max):
-            raise ValueError("required: chunk_min <= chunk_mask <= chunk_max")
+            raise argparse.ArgumentTypeError("required: chunk_min <= chunk_mask <= chunk_max")
         if chunk_min < 6:
             # see comment in 'fixed' algo check
-            raise ValueError("min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)")
+            raise argparse.ArgumentTypeError(
+                "min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)"
+            )
         if chunk_max > 23:
-            raise ValueError("max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)")
+            raise argparse.ArgumentTypeError(
+                "max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)"
+            )
         return CH_BUZHASH, chunk_min, chunk_max, chunk_mask, window_size
-    raise ValueError("invalid chunker params")
+    raise argparse.ArgumentTypeError("invalid chunker params")
 
 
 def FilesCacheMode(s):
@@ -177,11 +183,13 @@ def FilesCacheMode(s):
     VALID_MODES = ("cis", "ims", "cs", "ms", "cr", "mr", "d", "s")  # letters in alpha order
     entries = set(s.strip().split(","))
     if not entries <= set(ENTRIES_MAP):
-        raise ValueError("cache mode must be a comma-separated list of: %s" % ",".join(sorted(ENTRIES_MAP)))
+        raise argparse.ArgumentTypeError(
+            "cache mode must be a comma-separated list of: %s" % ",".join(sorted(ENTRIES_MAP))
+        )
     short_entries = {ENTRIES_MAP[entry] for entry in entries}
     mode = "".join(sorted(short_entries))
     if mode not in VALID_MODES:
-        raise ValueError("cache mode short must be one of: %s" % ",".join(VALID_MODES))
+        raise argparse.ArgumentTypeError("cache mode short must be one of: %s" % ",".join(VALID_MODES))
     return mode
 
 
@@ -277,7 +285,7 @@ def SortBySpec(text):
 
     for token in text.split(","):
         if token not in AI_HUMAN_SORT_KEYS:
-            raise ValueError("Invalid sort key: %s" % token)
+            raise argparse.ArgumentTypeError("Invalid sort key: %s" % token)
     return text.replace("timestamp", "ts")
 
 


Test patch:
diff --git a/src/borg/testsuite/compress.py b/src/borg/testsuite/compress.py
index 49819145..fe951dc5 100644
--- a/src/borg/testsuite/compress.py
+++ b/src/borg/testsuite/compress.py
@@ -1,3 +1,4 @@
+import argparse
 import os
 import zlib
 
@@ -226,7 +227,7 @@ def test_obfuscate_meta():
 
 
 def test_compression_specs():
-    with pytest.raises(ValueError):
+    with pytest.raises(argparse.ArgumentTypeError):
         CompressionSpec("")
 
     assert isinstance(CompressionSpec("none").compressor, CNONE)
@@ -241,7 +242,7 @@ def test_compression_specs():
     zlib = CompressionSpec("zlib,9").compressor
     assert isinstance(zlib, ZLIB)
     assert zlib.level == 9
-    with pytest.raises(ValueError):
+    with pytest.raises(argparse.ArgumentTypeError):
         CompressionSpec("zlib,9,invalid")
 
     lzma = CompressionSpec("lzma").compressor
@@ -264,7 +265,7 @@ def test_compression_specs():
     assert isinstance(zstd, ZSTD)
     assert zstd.level == 22
 
-    with pytest.raises(ValueError):
+    with pytest.raises(argparse.ArgumentTypeError):
         CompressionSpec("lzma,9,invalid")
-    with pytest.raises(ValueError):
+    with pytest.raises(argparse.ArgumentTypeError):
         CompressionSpec("invalid")
diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py
index df9b2ea2..bfb4f5c3 100644
--- a/src/borg/testsuite/helpers.py
+++ b/src/borg/testsuite/helpers.py
@@ -368,19 +368,19 @@ def test_chunkerparams():
     assert ChunkerParams("fixed,4096") == ("fixed", 4096, 0)
     assert ChunkerParams("fixed,4096,200") == ("fixed", 4096, 200)
     # invalid values checking
-    with pytest.raises(ValueError):
+    with pytest.raises(ArgumentTypeError):
         ChunkerParams("crap,1,2,3,4")  # invalid algo
-    with pytest.raises(ValueError):
+    with pytest.raises(ArgumentTypeError):
         ChunkerParams("buzhash,5,7,6,4095")  # too small min. size
-    with pytest.raises(ValueError):
+    with pytest.raises(ArgumentTypeError):
         ChunkerParams("buzhash,19,24,21,4095")  # too big max. size
-    with pytest.raises(ValueError):
+    with pytest.raises(ArgumentTypeError):
         ChunkerParams("buzhash,23,19,21,4095")  # violates min <= mask <= max
-    with pytest.raises(ValueError):
+    with pytest.raises(ArgumentTypeError):
         ChunkerParams("fixed,63")  # too small block size
-    with pytest.raises(ValueError):
+    with pytest.raises(ArgumentTypeError):
         ChunkerParams("fixed,%d,%d" % (MAX_DATA_SIZE + 1, 4096))  # too big block size
-    with pytest.raises(ValueError):
+    with pytest.raises(ArgumentTypeError):
         ChunkerParams("fixed,%d,%d" % (4096, MAX_DATA_SIZE + 1))  # too big header size