Repository: borgbackup/borg

Issue:
switch off nodump behaviour or bsdflags
we already had some issues with that:
- unraid showing uninitialized/random bsdflags #3127
- macOS shares showing nodump in general #2949
- the linux bsdflags support needs to open the fs item.
  - slowdown due to the extra open/ioctl?
  - also there was a bug with opening non-present device files. #3130

so, add a `--ignore-nodump`? (double negation, not nice, rather `--honor-nodump` for the opposite?

or even a `--no-bsdflags` to skip reading / archiving bsdflags in general?

both?


Discussion:
about `--honor-nodump`:
- borg < 1.1.0: did not care about nodump, this just backed up everything
- borg 1.1.0: honored nodump, so did not back up stuff with nodump set (no matter whether this was intentional or just a bug / surprising behaviour, see top post)
- borg 1.1.1: we could go back to the < 1.1.0 behaviour by adding a `--honor-nodump` flag and not honoring nodump by default.
> borg < 1.1.0: did not care about nodump, this just backed up everything

Only on Linux, since BSD flags have been supported since forever with the BSD API.

> slowdown due to the extra open/ioctl?

In general it is a good idea to look at exactly what is slow and what is not. For Borg, almost always Borg is slow, and not the syscalls (even though a lot of optimization could be done here, e.g. not re-opening files everywhere, using directory FDs etc.; there is a ticket about that). Only if genuinely the syscalls cause a _problem_ an option should be added; otherwise, the code should be fixed.

Resolving patch:
diff --git a/src/borg/archive.py b/src/borg/archive.py
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -280,8 +280,8 @@ class IncompatibleFilesystemEncodingError(Error):
         """Failed to encode filename "{}" into file system encoding "{}". Consider configuring the LANG environment variable."""
 
     def __init__(self, repository, key, manifest, name, cache=None, create=False,
-                 checkpoint_interval=300, numeric_owner=False, noatime=False, noctime=False, progress=False,
-                 chunker_params=CHUNKER_PARAMS, start=None, start_monotonic=None, end=None,
+                 checkpoint_interval=300, numeric_owner=False, noatime=False, noctime=False, nobsdflags=False,
+                 progress=False, chunker_params=CHUNKER_PARAMS, start=None, start_monotonic=None, end=None,
                  consider_part_files=False, log_json=False):
         self.cwd = os.getcwd()
         self.key = key
@@ -298,6 +298,7 @@ def __init__(self, repository, key, manifest, name, cache=None, create=False,
         self.numeric_owner = numeric_owner
         self.noatime = noatime
         self.noctime = noctime
+        self.nobsdflags = nobsdflags
         assert (start is None) == (start_monotonic is None), 'Logic error: if start is given, start_monotonic must be given as well and vice versa.'
         if start is None:
             start = datetime.utcnow()
@@ -691,7 +692,7 @@ def restore_attrs(self, path, item, symlink=False, fd=None):
             # some systems don't support calling utime on a symlink
             pass
         acl_set(path, item, self.numeric_owner)
-        if 'bsdflags' in item:
+        if not self.nobsdflags and 'bsdflags' in item:
             try:
                 set_flags(path, item.bsdflags, fd=fd)
             except OSError:
@@ -829,9 +830,11 @@ def stat_simple_attrs(self, st):
 
     def stat_ext_attrs(self, st, path):
         attrs = {}
+        bsdflags = 0
         with backup_io('extended stat'):
             xattrs = xattr.get_all(path, follow_symlinks=False)
-            bsdflags = get_flags(path, st)
+            if not self.nobsdflags:
+                bsdflags = get_flags(path, st)
             acl_get(path, attrs, st, self.numeric_owner)
         if xattrs:
             attrs['xattrs'] = StableDict(xattrs)
diff --git a/src/borg/archiver.py b/src/borg/archiver.py
--- a/src/borg/archiver.py
+++ b/src/borg/archiver.py
@@ -154,7 +154,9 @@ def with_archive(method):
     @functools.wraps(method)
     def wrapper(self, args, repository, key, manifest, **kwargs):
         archive = Archive(repository, key, manifest, args.location.archive,
-                          numeric_owner=getattr(args, 'numeric_owner', False), cache=kwargs.get('cache'),
+                          numeric_owner=getattr(args, 'numeric_owner', False),
+                          nobsdflags=getattr(args, 'nobsdflags', False),
+                          cache=kwargs.get('cache'),
                           consider_part_files=args.consider_part_files, log_json=args.log_json)
         return method(self, args, repository=repository, manifest=manifest, key=key, archive=archive, **kwargs)
     return wrapper
@@ -512,6 +514,7 @@ def create_inner(archive, cache):
         self.output_filter = args.output_filter
         self.output_list = args.output_list
         self.ignore_inode = args.ignore_inode
+        self.exclude_nodump = args.exclude_nodump
         self.files_cache_mode = args.files_cache_mode
         dry_run = args.dry_run
         t0 = datetime.utcnow()
@@ -522,7 +525,7 @@ def create_inner(archive, cache):
                 archive = Archive(repository, key, manifest, args.location.archive, cache=cache,
                                   create=True, checkpoint_interval=args.checkpoint_interval,
                                   numeric_owner=args.numeric_owner, noatime=args.noatime, noctime=args.noctime,
-                                  progress=args.progress,
+                                  nobsdflags=args.nobsdflags, progress=args.progress,
                                   chunker_params=args.chunker_params, start=t0, start_monotonic=t0_monotonic,
                                   log_json=args.log_json)
                 create_inner(archive, cache)
@@ -561,11 +564,12 @@ def _process(self, archive, cache, matcher, exclude_caches, exclude_if_present,
             # directory of the mounted filesystem that shadows the mountpoint dir).
             recurse = restrict_dev is None or st.st_dev == restrict_dev
             status = None
-            # Ignore if nodump flag is set
-            with backup_io('flags'):
-                if get_flags(path, st) & stat.UF_NODUMP:
-                    self.print_file_status('x', path)
-                    return
+            if self.exclude_nodump:
+                # Ignore if nodump flag is set
+                with backup_io('flags'):
+                    if get_flags(path, st) & stat.UF_NODUMP:
+                        self.print_file_status('x', path)
+                        return
             if stat.S_ISREG(st.st_mode):
                 if not dry_run:
                     status = archive.process_file(path, st, cache, self.ignore_inode, self.files_cache_mode)
@@ -2395,6 +2399,7 @@ def define_exclude_and_patterns(add_option, *, tag_files=False, strip_components
         def define_exclusion_group(subparser, **kwargs):
             exclude_group = subparser.add_argument_group('Exclusion options')
             define_exclude_and_patterns(exclude_group.add_argument, **kwargs)
+            return exclude_group
 
         def define_archive_filters_group(subparser, *, sort_by=True, first_last=True):
             filters_group = subparser.add_argument_group('Archive filters',
@@ -2924,7 +2929,9 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True):
         subparser.add_argument('--no-files-cache', dest='cache_files', action='store_false',
                                help='do not load/update the file metadata cache used to detect unchanged files')
 
-        define_exclusion_group(subparser, tag_files=True)
+        exclude_group = define_exclusion_group(subparser, tag_files=True)
+        exclude_group.add_argument('--exclude-nodump', dest='exclude_nodump', action='store_true',
+                                   help='exclude files flagged NODUMP')
 
         fs_group = subparser.add_argument_group('Filesystem options')
         fs_group.add_argument('-x', '--one-file-system', dest='one_file_system', action='store_true',
@@ -2935,6 +2942,8 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True):
                               help='do not store atime into archive')
         fs_group.add_argument('--noctime', dest='noctime', action='store_true',
                               help='do not store ctime into archive')
+        fs_group.add_argument('--nobsdflags', dest='nobsdflags', action='store_true',
+                              help='do not read and store bsdflags (e.g. NODUMP, IMMUTABLE) into archive')
         fs_group.add_argument('--ignore-inode', dest='ignore_inode', action='store_true',
                               help='ignore inode data in the file metadata cache used to detect unchanged files.')
         fs_group.add_argument('--files-cache', metavar='MODE', dest='files_cache_mode',
@@ -3001,6 +3010,8 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True):
                                help='do not actually change any files')
         subparser.add_argument('--numeric-owner', dest='numeric_owner', action='store_true',
                                help='only obey numeric user and group identifiers')
+        subparser.add_argument('--nobsdflags', dest='nobsdflags', action='store_true',
+                               help='do not extract/set bsdflags (e.g. NODUMP, IMMUTABLE)')
         subparser.add_argument('--stdout', dest='stdout', action='store_true',
                                help='write all extracted data to stdout')
         subparser.add_argument('--sparse', dest='sparse', action='store_true',
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -380,8 +380,8 @@ def test_basic_functionality(self):
         output = self.cmd('init', '--encryption=repokey', '--show-version', '--show-rc', self.repository_location, fork=True)
         self.assert_in('borgbackup version', output)
         self.assert_in('terminating with success status, rc 0', output)
-        self.cmd('create', self.repository_location + '::test', 'input')
-        output = self.cmd('create', '--stats', self.repository_location + '::test.2', 'input')
+        self.cmd('create', '--exclude-nodump', self.repository_location + '::test', 'input')
+        output = self.cmd('create', '--exclude-nodump', '--stats', self.repository_location + '::test.2', 'input')
         self.assert_in('Archive name: test.2', output)
         self.assert_in('This archive: ', output)
         with changedir('output'):
@@ -1655,13 +1655,13 @@ def test_file_status_excluded(self):
             self.create_regular_file('file3', size=1024 * 80)
             platform.set_flags(os.path.join(self.input_path, 'file3'), stat.UF_NODUMP)
         self.cmd('init', '--encryption=repokey', self.repository_location)
-        output = self.cmd('create', '--list', self.repository_location + '::test', 'input')
+        output = self.cmd('create', '--list', '--exclude-nodump', self.repository_location + '::test', 'input')
         self.assert_in("A input/file1", output)
         self.assert_in("A input/file2", output)
         if has_lchflags:
             self.assert_in("x input/file3", output)
         # should find second file as excluded
-        output = self.cmd('create', '--list', self.repository_location + '::test1', 'input', '--exclude', '*/file2')
+        output = self.cmd('create', '--list', '--exclude-nodump', self.repository_location + '::test1', 'input', '--exclude', '*/file2')
         self.assert_in("U input/file1", output)
         self.assert_in("x input/file2", output)
         if has_lchflags:
@@ -2059,8 +2059,8 @@ def has_noatime(some_file):
         self.cmd('init', '--encryption=repokey', self.repository_location)
         self.create_test_files()
         have_noatime = has_noatime('input/file1')
-        self.cmd('create', self.repository_location + '::archive', 'input')
-        self.cmd('create', self.repository_location + '::archive2', 'input')
+        self.cmd('create', '--exclude-nodump', self.repository_location + '::archive', 'input')
+        self.cmd('create', '--exclude-nodump', self.repository_location + '::archive2', 'input')
         if has_lchflags:
             # remove the file we did not backup, so input and output become equal
             os.remove(os.path.join('input', 'flagfile'))


Test patch: