Repository: borgbackup/borg

Issue:
freebsd --read-special test failure
the read-special test fails when reading the fifo on freebsd (master branch):

```
    freebsd64: __________________ ArchiverTestCase.test_create_read_special ___________________
    freebsd64: [gw0] freebsd10 -- Python 3.6.6 /vagrant/borg/borg/.tox/py36/bin/python
    freebsd64: self = <borg.testsuite.archiver.ArchiverTestCase testMethod=test_create_read_special>
    freebsd64:     def test_create_read_special(self):
    freebsd64:         self.create_regular_file('regular', size=1024)
    freebsd64:         os.symlink(os.path.join(self.input_path, 'file'), os.path.join(self.input_path, 'link_regular'))
    freebsd64:         if are_fifos_supported():
    freebsd64:             os.mkfifo(os.path.join(self.input_path, 'fifo'))
    freebsd64:             os.symlink(os.path.join(self.input_path, 'fifo'), os.path.join(self.input_path, 'link_fifo'))
    freebsd64:         self.cmd('init', '--encryption=repokey', self.repository_location)
    freebsd64:         archive = self.repository_location + '::test'
    freebsd64: >       self.cmd('create', '--read-special', archive, 'input')
    freebsd64: .tox/py36/lib/python3.6/site-packages/borg/testsuite/archiver.py:1791: 
    freebsd64: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
    freebsd64: .tox/py36/lib/python3.6/site-packages/borg/testsuite/archiver.py:292: in cmd
    freebsd64:     self.assert_equal(ret, exit_code)
    freebsd64: E   AssertionError: 1 != 0
    freebsd64: ----------------------------- Captured stdout call -----------------------------
    freebsd64: input/fifo: read: [Errno 35] Resource temporarily unavailable
    freebsd64: input/link_fifo: read: [Errno 35] Resource temporarily unavailable
```



Discussion:
Hmm, seems linux and macOS are happy reading a fifo that is not "fed", but freebsd is not.

How could `--read-special` be tested?
- we need to test that if follows symlinks for special files
- it will then read the "contents" of the special file
- special files are FIFOs, blk and chr devices
- we might be: not root, fakeroot or real root (depending on test env)
- the test should work on misc platforms and be relatively quick

guess we need to create a thread (or fork) that feeds the fifo with a small amount of data.
we can later compare if that data actually ended up in the borg archive.

Resolving patch:
diff --git a/src/borg/archiver.py b/src/borg/archiver.py
--- a/src/borg/archiver.py
+++ b/src/borg/archiver.py
@@ -66,7 +66,7 @@
 from .helpers import popen_with_error_handling, prepare_subprocess_env
 from .helpers import dash_open
 from .helpers import umount
-from .helpers import flags_root, flags_dir, flags_follow
+from .helpers import flags_root, flags_dir, flags_special_follow, flags_special
 from .helpers import msgpack
 from .nanorst import rst_to_terminal
 from .patterns import ArgparsePatternAction, ArgparseExcludeFileAction, ArgparsePatternFileAction, parse_exclude_pattern
@@ -646,7 +646,7 @@ def _process(self, *, path, parent_fd=None, name=None,
                             special = is_special(st_target.st_mode)
                         if special:
                             status = fso.process_file(path=path, parent_fd=parent_fd, name=name, st=st_target,
-                                                      cache=cache, flags=flags_follow)
+                                                      cache=cache, flags=flags_special_follow)
                         else:
                             status = fso.process_symlink(path=path, parent_fd=parent_fd, name=name, st=st)
             elif stat.S_ISFIFO(st.st_mode):
@@ -654,19 +654,22 @@ def _process(self, *, path, parent_fd=None, name=None,
                     if not read_special:
                         status = fso.process_fifo(path=path, parent_fd=parent_fd, name=name, st=st)
                     else:
-                        status = fso.process_file(path=path, parent_fd=parent_fd, name=name, st=st, cache=cache)
+                        status = fso.process_file(path=path, parent_fd=parent_fd, name=name, st=st,
+                                                  cache=cache, flags=flags_special)
             elif stat.S_ISCHR(st.st_mode):
                 if not dry_run:
                     if not read_special:
                         status = fso.process_dev(path=path, parent_fd=parent_fd, name=name, st=st, dev_type='c')
                     else:
-                        status = fso.process_file(path=path, parent_fd=parent_fd, name=name, st=st, cache=cache)
+                        status = fso.process_file(path=path, parent_fd=parent_fd, name=name, st=st,
+                                                  cache=cache, flags=flags_special)
             elif stat.S_ISBLK(st.st_mode):
                 if not dry_run:
                     if not read_special:
                         status = fso.process_dev(path=path, parent_fd=parent_fd, name=name, st=st, dev_type='b')
                     else:
-                        status = fso.process_file(path=path, parent_fd=parent_fd, name=name, st=st, cache=cache)
+                        status = fso.process_file(path=path, parent_fd=parent_fd, name=name, st=st,
+                                                  cache=cache, flags=flags_special)
             elif stat.S_ISSOCK(st.st_mode):
                 # Ignore unix sockets
                 return
diff --git a/src/borg/helpers/fs.py b/src/borg/helpers/fs.py
--- a/src/borg/helpers/fs.py
+++ b/src/borg/helpers/fs.py
@@ -202,9 +202,10 @@ def O_(*flags):
     return result
 
 
-flags_base = O_('BINARY', 'NONBLOCK', 'NOCTTY')
-flags_follow = flags_base | O_('RDONLY')
-flags_normal = flags_base | O_('RDONLY', 'NOFOLLOW')
+flags_base = O_('BINARY', 'NOCTTY', 'RDONLY')
+flags_special = flags_base | O_('NOFOLLOW')  # BLOCK == wait when reading devices or fifos
+flags_special_follow = flags_base  # BLOCK == wait when reading symlinked devices or fifos
+flags_normal = flags_base | O_('NONBLOCK', 'NOFOLLOW')
 flags_noatime = flags_normal | O_('NOATIME')
 flags_root = O_('RDONLY')
 flags_dir = O_('DIRECTORY', 'RDONLY', 'NOFOLLOW')
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -1780,19 +1780,38 @@ def test_create_topical(self):
         output = self.cmd('create', '--list', '--filter=AM', self.repository_location + '::test3', 'input')
         self.assert_in('file1', output)
 
-    def test_create_read_special(self):
-        self.create_regular_file('regular', size=1024)
-        os.symlink(os.path.join(self.input_path, 'file'), os.path.join(self.input_path, 'link_regular'))
-        if are_fifos_supported():
-            os.mkfifo(os.path.join(self.input_path, 'fifo'))
-            os.symlink(os.path.join(self.input_path, 'fifo'), os.path.join(self.input_path, 'link_fifo'))
+    @pytest.mark.skipif(not are_fifos_supported(), reason='FIFOs not supported')
+    def test_create_read_special_symlink(self):
+        from threading import Thread
+
+        def fifo_feeder(fifo_fn, data):
+            fd = os.open(fifo_fn, os.O_WRONLY)
+            try:
+                os.write(fd, data)
+            finally:
+                os.close(fd)
+
         self.cmd('init', '--encryption=repokey', self.repository_location)
         archive = self.repository_location + '::test'
-        self.cmd('create', '--read-special', archive, 'input')
-        output = self.cmd('list', archive)
-        assert 'input/link_regular -> ' in output  # not pointing to special file: archived as symlink
-        if are_fifos_supported():
-            assert 'input/link_fifo\n' in output  # pointing to a special file: archived following symlink
+        data = b'foobar' * 1000
+
+        fifo_fn = os.path.join(self.input_path, 'fifo')
+        link_fn = os.path.join(self.input_path, 'link_fifo')
+        os.mkfifo(fifo_fn)
+        os.symlink(fifo_fn, link_fn)
+
+        t = Thread(target=fifo_feeder, args=(fifo_fn, data))
+        t.start()
+        try:
+            self.cmd('create', '--read-special', archive, 'input/link_fifo')
+        finally:
+            t.join()
+        with changedir('output'):
+            self.cmd('extract', archive)
+            fifo_fn = 'input/link_fifo'
+            with open(fifo_fn, 'rb') as f:
+                extracted_data = f.read()
+        assert extracted_data == data
 
     def test_create_read_special_broken_symlink(self):
         os.symlink('somewhere doesnt exist', os.path.join(self.input_path, 'link'))


Test patch: