Repository: borgbackup/borg

Issue:
Repository: MAX_OBJECT_SIZE is not enforced at write-time
E.g.

``` py

>>> with borg.repository.Repository("repo", exclusive=True) as r:
...     r.put(b'0' * 32, bytes(50*1024*1024))
...     r.get(b'0' * 32)
Traceback (most recent call last):
  File "<stdin>", line 3, in <module>
  File "/usr/lib/python3.5/site-packages/borg/repository.py", line 474, in get
    return self.io.read(segment, offset, id_)
  File "/usr/lib/python3.5/site-packages/borg/repository.py", line 695, in read
    size, tag, key, data = self._read(fd, self.put_header_fmt, header, segment, offset, (TAG_PUT, ))
  File "/usr/lib/python3.5/site-packages/borg/repository.py", line 717, in _read
    segment, offset))
borg.helpers.IntegrityError: Invalid segment entry size [segment 1, offset 8]
Invalid segment entry size [segment 1, offset 8]
```



Discussion:

Resolving patch:
diff --git a/borg/repository.py b/borg/repository.py
--- a/borg/repository.py
+++ b/borg/repository.py
@@ -712,9 +712,14 @@ def _read(self, fd, fmt, header, segment, offset, acceptable_tags):
             key = None
         else:
             raise TypeError("_read called with unsupported format")
-        if size > MAX_OBJECT_SIZE or size < fmt.size:
-            raise IntegrityError('Invalid segment entry size [segment {}, offset {}]'.format(
-                segment, offset))
+        if size > MAX_OBJECT_SIZE:
+            # if you get this on an archive made with borg < 1.0.7 and millions of files and
+            # you need to restore it, you can disable this check by using "if False:" above.
+            raise IntegrityError('Invalid segment entry size {} - too big [segment {}, offset {}]'.format(
+                size, segment, offset))
+        if size < fmt.size:
+            raise IntegrityError('Invalid segment entry size {} - too small [segment {}, offset {}]'.format(
+                size, segment, offset))
         length = size - fmt.size
         data = fd.read(length)
         if len(data) != length:
@@ -731,8 +736,12 @@ def _read(self, fd, fmt, header, segment, offset, acceptable_tags):
         return size, tag, key, data
 
     def write_put(self, id, data, raise_full=False):
+        data_size = len(data)
+        if data_size > MAX_DATA_SIZE:
+            # this would push the segment entry size beyond MAX_OBJECT_SIZE.
+            raise IntegrityError('More than allowed put data [{} > {}]'.format(data_size, MAX_DATA_SIZE))
         fd = self.get_write_fd(raise_full=raise_full)
-        size = len(data) + self.put_header_fmt.size
+        size = data_size + self.put_header_fmt.size
         offset = self.offset
         header = self.header_no_crc_fmt.pack(size, TAG_PUT)
         crc = self.crc_fmt.pack(crc32(data, crc32(id, crc32(header))) & 0xffffffff)
@@ -771,3 +780,6 @@ def close_segment(self):
             self._write_fd.close()
             sync_dir(os.path.dirname(self._write_fd.name))
             self._write_fd = None
+
+
+MAX_DATA_SIZE = MAX_OBJECT_SIZE - LoggedIO.put_header_fmt.size
diff --git a/borg/testsuite/repository.py b/borg/testsuite/repository.py
--- a/borg/testsuite/repository.py
+++ b/borg/testsuite/repository.py
@@ -8,7 +8,7 @@
 from ..helpers import Location, IntegrityError
 from ..locking import Lock, LockFailed
 from ..remote import RemoteRepository, InvalidRPCMethod
-from ..repository import Repository, LoggedIO, TAG_COMMIT
+from ..repository import Repository, LoggedIO, TAG_COMMIT, MAX_DATA_SIZE
 from . import BaseTestCase
 
 
@@ -128,6 +128,13 @@ def test_list(self):
         self.assert_equal(second_half, all[50:])
         self.assert_equal(len(self.repository.list(limit=50)), 50)
 
+    def test_max_data_size(self):
+        max_data = b'x' * MAX_DATA_SIZE
+        self.repository.put(b'00000000000000000000000000000000', max_data)
+        self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), max_data)
+        self.assert_raises(IntegrityError,
+                           lambda: self.repository.put(b'00000000000000000000000000000001', max_data + b'x'))
+
 
 class RepositoryCommitTestCase(RepositoryTestCaseBase):
 


Test patch: