Repository: borgbackup/borg

Issue:
"create" + disk full makes "check --repair" crash
This was found by archiver.test_disk_full.

repos + corresponding tracebacks: https://paste.thinkmo.de/bqyrsTv6#borg-corrupt-repos.tgz

E.g. repo1:

```
$ borg check --repair /tmp/mnt/repo
'check --repair' is an experimental feature that might result in data loss.
Type 'YES' if you understand this and want to continue: YES
Starting repository check...
Segment entry data short read [segment 9, offset 575059]: expected 80786, got 65657 bytes
attempting to recover /tmp/mnt/repo/data/0/9
Local Exception.
Traceback (most recent call last):
  File "/home/tw/w/borg/borg/archiver.py", line 1069, in main
    exit_code = archiver.run(args)
  File "/home/tw/w/borg/borg/archiver.py", line 1023, in run
    return args.func(args)
  File "/home/tw/w/borg/borg/archiver.py", line 99, in do_check
    if repository.check(repair=args.repair):
  File "/home/tw/w/borg/borg/repository.py", line 348, in check
    self.compact_segments()
  File "/home/tw/w/borg/borg/repository.py", line 230, in compact_segments
    assert segments[segment] == 0
AssertionError
```



Discussion:
Crash looks same as this one: https://github.com/jborg/attic/issues/166#issuecomment-68614341

Also: `borg list` crashes with IntegrityError here:

```
$ borg list /tmp/repo1
Data integrity error
Traceback (most recent call last):
  File "/home/tw/w/borg/borg/archiver.py", line 1069, in main
    exit_code = archiver.run(args)
  File "/home/tw/w/borg/borg/archiver.py", line 1023, in run
    return args.func(args)
  File "/home/tw/w/borg/borg/archiver.py", line 381, in do_list
    manifest, key = Manifest.load(repository)
  File "/home/tw/w/borg/borg/helpers.py", line 108, in load
    cdata = repository.get(cls.MANIFEST_ID)
  File "/home/tw/w/borg/borg/repository.py", line 376, in get
    self.index = self.open_index(self.get_transaction_id())
  File "/home/tw/w/borg/borg/repository.py", line 130, in get_transaction_id
    self.replay_segments(replay_from, segments_transaction_id)
  File "/home/tw/w/borg/borg/repository.py", line 246, in replay_segments
    for tag, key, offset in self.io.iter_objects(segment):
  File "/home/tw/w/borg/borg/repository.py", line 541, in iter_objects
    (TAG_PUT, TAG_DELETE, TAG_COMMIT))
  File "/home/tw/w/borg/borg/repository.py", line 602, in _read
    segment, offset, length, len(data)))
borg.helpers.IntegrityError: Segment entry data short read [segment 9, offset 575059]: expected 80786, got 65657 bytes
```

The reason for this is that the chunk is incomplete, the last chunk that was written into segment 9 could not get written completely because the disk was full. Note: there is a 0 byte segment 10 file.

Note: a 2nd `borg check --repair` run on same repo does not crash any more.

I removed the corruption label. Found out that there is just a bug in "borg check" initialization (and fixed it).


Resolving patch:
diff --git a/borg/repository.py b/borg/repository.py
--- a/borg/repository.py
+++ b/borg/repository.py
@@ -175,11 +175,11 @@ def prepare_txn(self, transaction_id, do_cleanup=True):
             # the repository instance lives on - even if exceptions happened.
             self._active_txn = False
             raise
-        if not self.index:
+        if not self.index or transaction_id is None:
             self.index = self.open_index(transaction_id)
         if transaction_id is None:
-            self.segments = {}
-            self.compact = set()
+            self.segments = {}  # XXX bad name: usage_count_of_segment_x = self.segments[x]
+            self.compact = set()  # XXX bad name: segments_needing_compaction = self.compact
         else:
             if do_cleanup:
                 self.io.cleanup(transaction_id)
@@ -237,38 +237,49 @@ def compact_segments(self):
 
     def replay_segments(self, index_transaction_id, segments_transaction_id):
         self.prepare_txn(index_transaction_id, do_cleanup=False)
-        for segment, filename in self.io.segment_iterator():
-            if index_transaction_id is not None and segment <= index_transaction_id:
-                continue
-            if segment > segments_transaction_id:
-                break
-            self.segments[segment] = 0
-            for tag, key, offset in self.io.iter_objects(segment):
-                if tag == TAG_PUT:
-                    try:
-                        s, _ = self.index[key]
-                        self.compact.add(s)
-                        self.segments[s] -= 1
-                    except KeyError:
-                        pass
-                    self.index[key] = segment, offset
-                    self.segments[segment] += 1
-                elif tag == TAG_DELETE:
-                    try:
-                        s, _ = self.index.pop(key)
-                        self.segments[s] -= 1
-                        self.compact.add(s)
-                    except KeyError:
-                        pass
-                    self.compact.add(segment)
-                elif tag == TAG_COMMIT:
+        try:
+            for segment, filename in self.io.segment_iterator():
+                if index_transaction_id is not None and segment <= index_transaction_id:
                     continue
-                else:
-                    raise self.CheckNeeded(self.path)
-            if self.segments[segment] == 0:
+                if segment > segments_transaction_id:
+                    break
+                objects = self.io.iter_objects(segment)
+                self._update_index(segment, objects)
+            self.write_index()
+        finally:
+            self.rollback()
+
+    def _update_index(self, segment, objects, report=None):
+        """some code shared between replay_segments and check"""
+        self.segments[segment] = 0
+        for tag, key, offset in objects:
+            if tag == TAG_PUT:
+                try:
+                    s, _ = self.index[key]
+                    self.compact.add(s)
+                    self.segments[s] -= 1
+                except KeyError:
+                    pass
+                self.index[key] = segment, offset
+                self.segments[segment] += 1
+            elif tag == TAG_DELETE:
+                try:
+                    s, _ = self.index.pop(key)
+                    self.segments[s] -= 1
+                    self.compact.add(s)
+                except KeyError:
+                    pass
                 self.compact.add(segment)
-        self.write_index()
-        self.rollback()
+            elif tag == TAG_COMMIT:
+                continue
+            else:
+                msg = 'Unexpected tag {} in segment {}'.format(tag, segment)
+                if report is None:
+                    raise self.CheckNeeded(msg)
+                else:
+                    report(msg)
+        if self.segments[segment] == 0:
+            self.compact.add(segment)
 
     def check(self, repair=False):
         """Check repository consistency
@@ -297,7 +308,7 @@ def report_error(msg):
         if repair:
             self.io.cleanup(transaction_id)
         segments_transaction_id = self.io.get_segments_transaction_id()
-        self.prepare_txn(None)
+        self.prepare_txn(None)  # self.index, self.compact, self.segments all empty now!
         for segment, filename in self.io.segment_iterator():
             if segment > transaction_id:
                 continue
@@ -309,35 +320,16 @@ def report_error(msg):
                 if repair:
                     self.io.recover_segment(segment, filename)
                     objects = list(self.io.iter_objects(segment))
-            self.segments[segment] = 0
-            for tag, key, offset in objects:
-                if tag == TAG_PUT:
-                    try:
-                        s, _ = self.index[key]
-                        self.compact.add(s)
-                        self.segments[s] -= 1
-                    except KeyError:
-                        pass
-                    self.index[key] = segment, offset
-                    self.segments[segment] += 1
-                elif tag == TAG_DELETE:
-                    try:
-                        s, _ = self.index.pop(key)
-                        self.segments[s] -= 1
-                        self.compact.add(s)
-                    except KeyError:
-                        pass
-                    self.compact.add(segment)
-                elif tag == TAG_COMMIT:
-                    continue
-                else:
-                    report_error('Unexpected tag {} in segment {}'.format(tag, segment))
+            self._update_index(segment, objects, report_error)
+        # self.index, self.segments, self.compact now reflect the state of the segment files up to <transaction_id>
         # We might need to add a commit tag if no committed segment is found
         if repair and segments_transaction_id is None:
             report_error('Adding commit tag to segment {}'.format(transaction_id))
             self.io.segment = transaction_id + 1
             self.io.write_commit()
         if current_index and not repair:
+            # current_index = "as found on disk"
+            # self.index = "as rebuilt in-memory from segments"
             if len(current_index) != len(self.index):
                 report_error('Index object count mismatch. {} != {}'.format(len(current_index), len(self.index)))
             elif current_index:


Test patch: