Repository: ros/ros_comm

Issue:
rosbag reindex outputs invalid bags when run on truncated files
The rosbag reindex command is documented as "repairing broken bag files", but when run on a broken bag file it will output a bag that is indexed, but illegal per the ROS bag spec.

This script can be used to demonstrate the issue:
https://gist.github.com/wkalt/836cd80f1c6a9b0e87e5e24771c7b8e0

Steps to reproduce:
```
wget https://assets.foxglove.dev/demo.bag
truncate -s 30000000 demo.bag
rosbag reindex demo.bag

// using go program above
$ go run main.go demo.bag
2022/09/27 09:18:10 got zero length header at offset 29578569
exit status 1
```

The resulting bag violates the spec in that it is not a sequence of back to back `<header length><header><data length><data>` records, with each header containing an "op" key, as the bag spec mandates. However, ROS tooling accepts the bag due to reliance on the message index, which is well-formed. The index simply does not point at the junk data present in the file, which occurs after the final chunk and its message indexes.

This causes problems for tooling that does not use the index and instead performs a linear scan. It is possible this issue does not present if you are lucky enough to truncate the file on a message boundary.


Discussion:

Resolving patch:
diff --git a/tools/rosbag/src/rosbag/bag.py b/tools/rosbag/src/rosbag/bag.py
--- a/tools/rosbag/src/rosbag/bag.py
+++ b/tools/rosbag/src/rosbag/bag.py
@@ -2472,6 +2472,7 @@ def reindex(self):
 
         if trunc_pos and trunc_pos < total_bytes:
             f.truncate(trunc_pos)
+            f.seek(trunc_pos)
 
     def _reindex_read_chunk(self, f, chunk_pos, total_bytes):
         # Read the chunk header


Test patch:
diff --git a/test/test_rosbag/test/test_bag.py b/test/test_rosbag/test/test_bag.py
--- a/test/test_rosbag/test/test_bag.py
+++ b/test/test_rosbag/test/test_bag.py
@@ -261,6 +261,57 @@ def test_reindex_works(self):
 
             msgs = list(rosbag.Bag(reindex_filename, 'r'))
 
+    def _read_all_records(self, bag_path):
+        """
+        Throw exception if malformed records found, otherwise return number of records in file.
+        """
+        with rosbag.Bag(bag_path, 'r') as b:
+            file_header_pos = b._file_header_pos
+
+        total_records_read = 0
+        with open(bag_path, 'r+b') as f:
+            f.seek(0, os.SEEK_END)
+            total_bytes = f.tell()
+            f.seek(file_header_pos)
+            while f.tell() < total_bytes:
+                bag._read_sized(f)
+                total_records_read += 1
+
+        return total_records_read
+
+    def test_reindex_leaves_valid_file(self):
+        orig_filename = '/tmp/test_reindex_leaves_valid_file.bag'
+        chunk_threshold = 1024
+        with rosbag.Bag(orig_filename, 'w', chunk_threshold=chunk_threshold) as b:
+            for i in range(100):
+                for j in range(5):
+                    msg = Int32()
+                    msg.data = i
+                    b.write('/topic%d' % j, msg)
+            file_header_pos = b._file_header_pos
+
+        trunc_filename = '%s.trunc%s' % os.path.splitext(orig_filename)
+        reindex_filename = '%s.reindex%s' % os.path.splitext(orig_filename)
+
+        num_original_records = self._read_all_records(orig_filename)
+
+        shutil.copy(orig_filename, trunc_filename)
+        with open(trunc_filename, 'r+b') as f:
+            # truncating to arbitrary length that is in the middle of a record
+            f.seek(0, os.SEEK_END)
+            total_bytes = f.tell()
+            f.seek(0)
+            f.truncate(int(total_bytes / 2))
+        with self.assertRaises(rosbag.ROSBagException):
+            self._read_all_records(trunc_filename)
+
+        shutil.copy(trunc_filename, reindex_filename)
+        with rosbag.Bag(reindex_filename, 'a', allow_unindexed=True) as b:
+            for _ in b.reindex():
+                pass
+        num_reindexed_records = self._read_all_records(reindex_filename)
+
+
     def test_future_version_works(self):
         fn = '/tmp/test_future_version_2.1.bag'