Repository: borgbackup/borg

Issue:
import-tar for multiple or concatenated tar files
## Have you checked borgbackup docs, FAQ, and open GitHub issues?

Yes

## Is this a BUG / ISSUE report or a QUESTION?

Question

## System information. For client/server mode post info for both machines.

#### Your borg version (borg -V).

borg 1.2.3

#### How much data is handled by borg?

400-500 GB

#### Description

tl;dr: could `borg import-tar` support multiple tar archives, or a file made by simply `cat`-entating several tar files?

Use case / situation: a few hundred GB of exported data is split into multiple tar files by size (50 GB each) rather than content.

Solution A: use `tar --catenate` to combine those into a single tar, then use `borg import-tar`.  Unfortunately GNU tar has a bug that prevents it from concatenating more than one archive at a time, and concatenating one by one is too expensive due to implementation (see https://superuser.com/a/1098273).

Solution B: use `cat 1.tar 2.tar ...` and feed that into `borg import-tar`.  Simply catenating tar archives doesn't produce a completely valid combined archive due to zero-padding.  That padding can be ignored during extraction with `--ignore-zeros` flag of `tar` (https://www.gnu.org/software/tar/manual/html_node/concatenate.html) but that isn't currently supported by `borg import-tar` as far as I can tell from experimenting.  Perhaps this could be implemented similar to libarchive (https://github.com/libarchive/libarchive/issues/457).

What would work is either `borg import-tar` supporting multiple files in a command line, or `--ignore-zeros` inspired option.  I have a preference for the latter for simplicity.

Note the issue is similar to #6807 by title, but that question is about having a directory per tar archive, which is unpreferred in this case.

Discussion:

Patch:
diff --git a/src/borg/archiver.py b/src/borg/archiver.py
index 1ff8da07..82ad00e1 100644
--- a/src/borg/archiver.py
+++ b/src/borg/archiver.py
@@ -1763,7 +1763,8 @@ def _import_tar(self, args, repository, manifest, key, cache, tarstream):
                                       log_json=args.log_json, iec=args.iec,
                                       file_status_printer=self.print_file_status)
 
-        tar = tarfile.open(fileobj=tarstream, mode='r|')
+        tar = tarfile.open(fileobj=tarstream, mode='r|',
+                           ignore_zeros=args.ignore_zeros)
 
         while True:
             tarinfo = tar.next()
@@ -4932,6 +4933,10 @@ def define_borg_mount(parser):
 
         import-tar reads POSIX.1-1988 (ustar), POSIX.1-2001 (pax), GNU tar, UNIX V7 tar
         and SunOS tar with extended attributes.
+
+        To import multiple tarballs into a single archive, they can be simply
+        concatenated (e.g. using "cat") into a single file, and imported with an
+        ``--ignore-zeros`` option to skip through the stop markers between them.
         """)
         subparser = subparsers.add_parser('import-tar', parents=[common_parser], add_help=False,
                                           description=self.do_import_tar.__doc__,
@@ -4951,6 +4956,9 @@ def define_borg_mount(parser):
                                help='only display items with the given status characters')
         subparser.add_argument('--json', action='store_true',
                                help='output stats as JSON (implies --stats)')
+        subparser.add_argument('--ignore-zeros', dest='ignore_zeros',
+                               action='store_true', default=False,
+                               help='ignore zero-filled blocks in the input tarball')
 
         archive_group = subparser.add_argument_group('Archive options')
         archive_group.add_argument('--comment', dest='comment', metavar='COMMENT', default='',


Test patch:
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
index 118e4b3a..7eb5669b 100644
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -322,8 +322,12 @@ def create_regular_file(self, name, size=0, contents=None):
                 contents = b'X' * size
             fd.write(contents)
 
-    def create_test_files(self):
+    def create_test_files(self, create_hardlinks=True):
         """Create a minimal test case including all supported file types
+
+        Args:
+          create_hardlinks: whether to create a sample hardlink. When set to
+                            False, the hardlink file will not be created at all.
         """
         # File
         self.create_regular_file('file1', size=1024 * 80)
@@ -333,7 +337,7 @@ def create_test_files(self):
         # File mode
         os.chmod('input/file1', 0o4755)
         # Hard link
-        if are_hardlinks_supported():
+        if create_hardlinks and are_hardlinks_supported():
             os.link(os.path.join(self.input_path, 'file1'),
                     os.path.join(self.input_path, 'hardlink'))
         # Symlink
@@ -3588,6 +3592,61 @@ def test_import_tar_gz(self):
             self.cmd('extract', self.repository_location + '::dst')
         self.assert_dirs_equal('input', 'output/input', ignore_ns=True, ignore_xattrs=True)
 
+    @requires_gnutar
+    def test_import_concatenated_tar_with_ignore_zeros(self):
+        # file1 has a hardlink reference to it, but we put it in a separate
+        # tarball, breaking the link during import-tar. It could be any other
+        # file though, so we won't take chances and just avoid hardlinks.
+        self.create_test_files(create_hardlinks=False)
+        os.unlink('input/flagfile')
+
+        with changedir('input'):
+            subprocess.check_call(['tar', 'cf', 'file1.tar', 'file1'])
+            subprocess.check_call(['tar', 'cf', 'the_rest.tar', '--exclude', 'file1*', '.'])
+            with open('concatenated.tar', 'wb') as concatenated:
+                with open('file1.tar', 'rb') as file1:
+                    concatenated.write(file1.read())
+                # Clean up for assert_dirs_equal.
+                os.unlink('file1.tar')
+
+                with open('the_rest.tar', 'rb') as the_rest:
+                    concatenated.write(the_rest.read())
+                # Clean up for assert_dirs_equal.
+                os.unlink('the_rest.tar')
+
+        self.cmd('init', '--encryption=none', self.repository_location)
+        self.cmd('import-tar', '--ignore-zeros', self.repository_location + '::dst', 'input/concatenated.tar')
+        os.unlink('input/concatenated.tar')
+
+        with changedir(self.output_path):
+            self.cmd('extract', self.repository_location + '::dst')
+        self.assert_dirs_equal('input', 'output', ignore_ns=True, ignore_xattrs=True)
+
+    @requires_gnutar
+    def test_import_concatenated_tar_without_ignore_zeros(self):
+        self.create_test_files()
+        os.unlink('input/flagfile')
+
+        with changedir('input'):
+            subprocess.check_call(['tar', 'cf', 'file1.tar', 'file1'])
+            subprocess.check_call(['tar', 'cf', 'the_rest.tar', '--exclude', 'file1*', '.'])
+            with open('concatenated.tar', 'wb') as concatenated:
+                with open('file1.tar', 'rb') as file1:
+                    concatenated.write(file1.read())
+
+                with open('the_rest.tar', 'rb') as the_rest:
+                    concatenated.write(the_rest.read())
+
+        self.cmd('init', '--encryption=none', self.repository_location)
+        self.cmd('import-tar', self.repository_location + '::dst', 'input/concatenated.tar')
+
+        with changedir(self.output_path):
+            self.cmd('extract', self.repository_location + '::dst')
+
+        # Negative test -- assert that only file1 has been extracted, and the_rest has been ignored
+        # due to zero-filled block marker.
+        self.assert_equal(os.listdir('output'), ['file1'])
+
     def test_detect_attic_repo(self):
         path = make_attic_repo(self.repository_path)
         cmds = [