Repository: borgbackup/borg

Issue:
test_disk_full broken
On Linux:
```py
============================================================================================================ test session starts =============================================================================================================
platform linux -- Python 3.11.3, pytest-7.3.1, pluggy-1.0.0
benchmark: 4.0.0 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000)
Tests enabled: BSD flags, root, symlinks, hardlinks, atime/mtime, modes
Tests disabled: fuse2, fuse3
rootdir: /tmp/borg
configfile: setup.cfg
plugins: benchmark-4.0.0, cov-4.1.0, xdist-3.3.1
collected 1620 items / 1619 deselected / 1 selected                                                                                                                                                                                          

src/borg/testsuite/archiver/disk_full.py F                                                                                                                                                                                             [100%]

================================================================================================================== FAILURES ==================================================================================================================
___________________________________________________________________________________________________________ test_disk_full[python] ___________________________________________________________________________________________________________

cmd = <function cmd.<locals>.exec_fn at 0x7fe78e52f4c0>

    @pytest.mark.skipif(not os.path.exists(DF_MOUNT), reason="needs a 16MB fs mounted on %s" % DF_MOUNT)
    def test_disk_full(cmd):
        def make_files(dir, count, size, rnd=True):
            shutil.rmtree(dir, ignore_errors=True)
            os.mkdir(dir)
            if rnd:
                count = random.randint(1, count)
                if size > 1:
                    size = random.randint(1, size)
            for i in range(count):
                fn = os.path.join(dir, "file%03d" % i)
                with open(fn, "wb") as f:
                    data = os.urandom(size)
                    f.write(data)
    
        with environment_variable(BORG_CHECK_I_KNOW_WHAT_I_AM_DOING="YES"):
            mount = DF_MOUNT
            assert os.path.exists(mount)
            repo = os.path.join(mount, "repo")
            input = os.path.join(mount, "input")
            reserve = os.path.join(mount, "reserve")
            for j in range(100):
                shutil.rmtree(repo, ignore_errors=True)
                shutil.rmtree(input, ignore_errors=True)
                # keep some space and some inodes in reserve that we can free up later:
                make_files(reserve, 80, 100000, rnd=False)
                rc, out = cmd(f"--repo={repo}", "rcreate")
                if rc != EXIT_SUCCESS:
                    print("rcreate", rc, out)
>               assert rc == EXIT_SUCCESS
E               assert 2 == 0

src/borg/testsuite/archiver/disk_full.py:57: AssertionError
------------------------------------------------------------------------------------------------------------ Captured stdout call ------------------------------------------------------------------------------------------------------------
rcreate 2 usage: __main__.py rcreate [-h] [--critical] [--error] [--warning] [--info]
                           [--debug] [--debug-topic TOPIC] [-p] [--iec]
                           [--log-json] [--lock-wait SECONDS] [--bypass-lock]
                           [--show-version] [--show-rc] [--umask M]
                           [--remote-path PATH] [--upload-ratelimit RATE]
                           [--upload-buffer UPLOAD_BUFFER]
                           [--debug-profile FILE] [--rsh RSH] [-r REPO]
                           [--other-repo SRC_REPOSITORY] -e MODE
                           [--append-only] [--storage-quota QUOTA]
                           [--make-parent-dirs] [--copy-crypt-key]
__main__.py rcreate: error: the following arguments are required: -e/--encryption

========================================================================================================== short test summary info ===========================================================================================================
FAILED src/borg/testsuite/archiver/disk_full.py::test_disk_full[python] - assert 2 == 0
===================================================================================================== 1 failed, 1619 deselected in 0.51s =====================================================================================================
```

I'm guessing this is because of some change from borg 1->2. Adding the encryption argument:
```diff
diff --git a/src/borg/testsuite/archiver/disk_full.py b/src/borg/testsuite/archiver/disk_full.py
index bbc1feed..ea95c60e 100644
--- a/src/borg/testsuite/archiver/disk_full.py
+++ b/src/borg/testsuite/archiver/disk_full.py
@@ -51,7 +51,7 @@ def make_files(dir, count, size, rnd=True):
             shutil.rmtree(input, ignore_errors=True)
             # keep some space and some inodes in reserve that we can free up later:
             make_files(reserve, 80, 100000, rnd=False)
-            rc, out = cmd(f"--repo={repo}", "rcreate")
+            rc, out = cmd(f"--repo={repo}", "rcreate", "--encryption=none")
             if rc != EXIT_SUCCESS:
                 print("rcreate", rc, out)
             assert rc == EXIT_SUCCESS
```
gives this:
```py
============================================================================================================ test session starts =============================================================================================================
platform linux -- Python 3.11.3, pytest-7.3.1, pluggy-1.0.0
benchmark: 4.0.0 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000)
Tests enabled: BSD flags, root, symlinks, hardlinks, atime/mtime, modes
Tests disabled: fuse2, fuse3
rootdir: /tmp/borg
configfile: setup.cfg
plugins: benchmark-4.0.0, cov-4.1.0, xdist-3.3.1
collected 1620 items / 1619 deselected / 1 selected                                                                                                                                                                                          

src/borg/testsuite/archiver/disk_full.py F                                                                                                                                                                                             [100%]

================================================================================================================== FAILURES ==================================================================================================================
___________________________________________________________________________________________________________ test_disk_full[python] ___________________________________________________________________________________________________________

cmd = <function cmd.<locals>.exec_fn at 0x7f928b1fb4c0>

    @pytest.mark.skipif(not os.path.exists(DF_MOUNT), reason="needs a 16MB fs mounted on %s" % DF_MOUNT)
    def test_disk_full(cmd):
        def make_files(dir, count, size, rnd=True):
            shutil.rmtree(dir, ignore_errors=True)
            os.mkdir(dir)
            if rnd:
                count = random.randint(1, count)
                if size > 1:
                    size = random.randint(1, size)
            for i in range(count):
                fn = os.path.join(dir, "file%03d" % i)
                with open(fn, "wb") as f:
                    data = os.urandom(size)
                    f.write(data)
    
        with environment_variable(BORG_CHECK_I_KNOW_WHAT_I_AM_DOING="YES"):
            mount = DF_MOUNT
            assert os.path.exists(mount)
            repo = os.path.join(mount, "repo")
            input = os.path.join(mount, "input")
            reserve = os.path.join(mount, "reserve")
            for j in range(100):
                shutil.rmtree(repo, ignore_errors=True)
                shutil.rmtree(input, ignore_errors=True)
                # keep some space and some inodes in reserve that we can free up later:
                make_files(reserve, 80, 100000, rnd=False)
                rc, out = cmd(f"--repo={repo}", "rcreate", "--encryption=none")
                if rc != EXIT_SUCCESS:
                    print("rcreate", rc, out)
                assert rc == EXIT_SUCCESS
                try:
                    success, i = True, 0
                    while success:
                        i += 1
                        try:
                            make_files(input, 20, 200000)
                        except OSError as err:
                            if err.errno == errno.ENOSPC:
                                # already out of space
                                break
                            raise
                        try:
                            rc, out = cmd("--repo=%s" % repo, "create", "test%03d" % i, input)
                            success = rc == EXIT_SUCCESS
                            if not success:
                                print("create", rc, out)
                        finally:
                            # make sure repo is not locked
                            shutil.rmtree(os.path.join(repo, "lock.exclusive"), ignore_errors=True)
>                           os.remove(os.path.join(repo, "lock.roster"))
E                           FileNotFoundError: [Errno 2] No such file or directory: '/tmp/borg-mount/repo/lock.roster'

src/borg/testsuite/archiver/disk_full.py:77: FileNotFoundError
========================================================================================================== short test summary info ===========================================================================================================
FAILED src/borg/testsuite/archiver/disk_full.py::test_disk_full[python] - FileNotFoundError: [Errno 2] No such file or directory: '/tmp/borg-mount/repo/lock.roster'
===================================================================================================== 1 failed, 1619 deselected in 0.81s =====================================================================================================
```

not sure why this happens?


Discussion:
seems like `lock.roster` file and `lock.exclusive` directory are only present while the repo is locked.
after making that os.remove not fail, it looks like the test hangs.

EDIT: looks like it only hangs sometimes.
```py
(borg-env) rayyan@rayyan-pc ~/G/borg (fix-test-disk-full) [0|1]> pytest -k "test_disk_full and not binary and not remote" -s
============================================================================================================ test session starts =============================================================================================================
platform linux -- Python 3.11.3, pytest-7.3.1, pluggy-1.0.0
benchmark: 4.0.0 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000)
Tests enabled: BSD flags, root, symlinks, hardlinks, atime/mtime, modes
Tests disabled: fuse2, fuse3
rootdir: /home/rayyan/Git/borg
configfile: setup.cfg
plugins: xdist-3.3.1, cov-4.1.0, benchmark-4.0.0
collected 1620 items / 1619 deselected / 1 selected                                                                                                                                                                                          

src/borg/testsuite/archiver/disk_full.py create 2 Insufficient free space to complete transaction (required: 545.47 MB, available: 3.56 MB).

check 2 This is a potentially dangerous function.
check --repair might lead to data loss (for kinds of corruption it is not
capable of dealing with). BE VERY CAREFUL!

Type 'YES' if you understand this and want to continue: YES (from BORG_CHECK_I_KNOW_WHAT_I_AM_DOING)
Insufficient free space to complete transaction (required: 545.47 MB, available: 8.69 MB).

F

================================================================================================================== FAILURES ==================================================================================================================
___________________________________________________________________________________________________________ test_disk_full[python] ___________________________________________________________________________________________________________

cmd = <function cmd.<locals>.exec_fn at 0x7f93243798a0>

    @pytest.mark.skipif(not os.path.exists(DF_MOUNT), reason="needs a 16MB fs mounted on %s" % DF_MOUNT)
    def test_disk_full(cmd):
        def make_files(dir, count, size, rnd=True):
            shutil.rmtree(dir, ignore_errors=True)
            os.mkdir(dir)
            if rnd:
                count = random.randint(1, count)
                if size > 1:
                    size = random.randint(1, size)
            for i in range(count):
                fn = os.path.join(dir, "file%03d" % i)
                with open(fn, "wb") as f:
                    data = os.urandom(size)
                    f.write(data)
    
        with environment_variable(BORG_CHECK_I_KNOW_WHAT_I_AM_DOING="YES"):
            mount = DF_MOUNT
            assert os.path.exists(mount)
            repo = os.path.join(mount, "repo")
            input = os.path.join(mount, "input")
            reserve = os.path.join(mount, "reserve")
            for j in range(100):
                shutil.rmtree(repo, ignore_errors=True)
                shutil.rmtree(input, ignore_errors=True)
                # keep some space and some inodes in reserve that we can free up later:
                make_files(reserve, 40, 100000, rnd=False)
                rc, out = cmd(f"--repo={repo}", "rcreate", "--encryption=none")
                if rc != EXIT_SUCCESS:
                    print("rcreate", rc, out)
                assert rc == EXIT_SUCCESS
                try:
                    success, i = True, 0
                    while success:
                        i += 1
                        try:
                            make_files(input, 20, 200000)
                        except OSError as err:
                            if err.errno == errno.ENOSPC:
                                # already out of space
                                break
                            raise
                        try:
                            rc, out = cmd("--repo=%s" % repo, "create", "test%03d" % i, input)
                            success = rc == EXIT_SUCCESS
                            if not success:
                                print("create", rc, out)
                        finally:
                            # make sure repo is not locked
                            shutil.rmtree(os.path.join(repo, "lock.exclusive"), ignore_errors=True)
                            try:
                                os.remove(os.path.join(repo, "lock.roster"))
                            except FileNotFoundError:
                                pass
                finally:
                    # now some error happened, likely we are out of disk space.
                    # free some space so we can expect borg to be able to work normally:
                    shutil.rmtree(reserve, ignore_errors=True)
                rc, out = cmd(f"--repo={repo}", "rlist")
                if rc != EXIT_SUCCESS:
                    print("rlist", rc, out)
                rc, out = cmd(f"--repo={repo}", "check", "--repair")
                if rc != EXIT_SUCCESS:
                    print("check", rc, out)
>               assert rc == EXIT_SUCCESS
E               assert 2 == 0

src/borg/testsuite/archiver/disk_full.py:91: AssertionError
========================================================================================================== short test summary info ===========================================================================================================
FAILED src/borg/testsuite/archiver/disk_full.py::test_disk_full[python] - assert 2 == 0
===================================================================================================== 1 failed, 1619 deselected in 1.84s =====================================================================================================
```
surely it needing ~500mb must be a bug somewhere?

Resolving patch:
diff --git a/src/borg/testsuite/archiver/disk_full.py b/src/borg/testsuite/archiver/disk_full.py
--- a/src/borg/testsuite/archiver/disk_full.py
+++ b/src/borg/testsuite/archiver/disk_full.py
@@ -1,14 +1,16 @@
 """
 test_disk_full is very slow and not recommended to be included in daily testing.
-for this test, an empty, writable 16MB filesystem mounted on DF_MOUNT is required.
+for this test, an empty, writable 700MB filesystem mounted on DF_MOUNT is required.
 for speed and other reasons, it is recommended that the underlying block device is
 in RAM, not a magnetic or flash disk.
 
-assuming /tmp is a tmpfs (in memory filesystem), one can use this:
-dd if=/dev/zero of=/tmp/borg-disk bs=16M count=1
-mkfs.ext4 /tmp/borg-disk
+assuming /dev/shm is a tmpfs (in memory filesystem), one can use this:
+
+dd if=/dev/zero of=/dev/shm/borg-disk bs=1M count=700
+mkfs.ext4 /dev/shm/borg-disk
 mkdir /tmp/borg-mount
-sudo mount /tmp/borg-disk /tmp/borg-mount
+sudo mount /dev/shm/borg-disk /tmp/borg-mount
+sudo chown myuser /tmp/borg-mount/
 
 if the directory does not exist, the test will be skipped.
 """
@@ -25,42 +27,41 @@
 DF_MOUNT = "/tmp/borg-mount"
 
 
-@pytest.mark.skipif(not os.path.exists(DF_MOUNT), reason="needs a 16MB fs mounted on %s" % DF_MOUNT)
-def test_disk_full(cmd_fixture, monkeypatch):
-    def make_files(dir, count, size, rnd=True):
-        shutil.rmtree(dir, ignore_errors=True)
-        os.mkdir(dir)
-        if rnd:
-            count = random.randint(1, count)
-            if size > 1:
-                size = random.randint(1, size)
-        for i in range(count):
-            fn = os.path.join(dir, "file%03d" % i)
-            with open(fn, "wb") as f:
-                data = os.urandom(size)
-                f.write(data)
+def make_files(dir, count, size, rnd=True):
+    shutil.rmtree(dir, ignore_errors=True)
+    os.mkdir(dir)
+    if rnd:
+        count = random.randint(1, count)
+        if size > 1:
+            size = random.randint(1, size)
+    for i in range(count):
+        fn = os.path.join(dir, "file%03d" % i)
+        with open(fn, "wb") as f:
+            data = os.urandom(size)
+            f.write(data)
+
 
+@pytest.mark.skipif(not os.path.exists(DF_MOUNT), reason="needs a 700MB fs mounted on %s" % DF_MOUNT)
+@pytest.mark.parametrize("test_pass", range(10))
+def test_disk_full(test_pass, cmd_fixture, monkeypatch):
     monkeypatch.setenv("BORG_CHECK_I_KNOW_WHAT_I_AM_DOING", "YES")
-    mount = DF_MOUNT
-    assert os.path.exists(mount)
-    repo = os.path.join(mount, "repo")
-    input = os.path.join(mount, "input")
-    reserve = os.path.join(mount, "reserve")
-    for j in range(100):
-        shutil.rmtree(repo, ignore_errors=True)
-        shutil.rmtree(input, ignore_errors=True)
-        # keep some space and some inodes in reserve that we can free up later:
-        make_files(reserve, 80, 100000, rnd=False)
-        rc, out = cmd_fixture(f"--repo={repo}", "rcreate")
-        if rc != EXIT_SUCCESS:
-            print("rcreate", rc, out)
-        assert rc == EXIT_SUCCESS
+    monkeypatch.setenv("BORG_DELETE_I_KNOW_WHAT_I_AM_DOING", "YES")
+    repo = os.path.join(DF_MOUNT, "repo")
+    input = os.path.join(DF_MOUNT, "input")
+    shutil.rmtree(repo, ignore_errors=True)
+    shutil.rmtree(input, ignore_errors=True)
+    rc, out = cmd_fixture(f"--repo={repo}", "rcreate", "--encryption=none")
+    if rc != EXIT_SUCCESS:
+        print("rcreate", rc, out)
+    assert rc == EXIT_SUCCESS
+    try:
         try:
             success, i = True, 0
             while success:
                 i += 1
                 try:
-                    make_files(input, 20, 200000)
+                    # have some randomness here to produce different out of space conditions:
+                    make_files(input, 40, 1000000, rnd=True)
                 except OSError as err:
                     if err.errno == errno.ENOSPC:
                         # already out of space
@@ -74,11 +75,11 @@ def make_files(dir, count, size, rnd=True):
                 finally:
                     # make sure repo is not locked
                     shutil.rmtree(os.path.join(repo, "lock.exclusive"), ignore_errors=True)
-                    os.remove(os.path.join(repo, "lock.roster"))
+                    shutil.rmtree(os.path.join(repo, "lock.roster"), ignore_errors=True)
         finally:
             # now some error happened, likely we are out of disk space.
             # free some space such that we can expect borg to be able to work normally:
-            shutil.rmtree(reserve, ignore_errors=True)
+            shutil.rmtree(input, ignore_errors=True)
         rc, out = cmd_fixture(f"--repo={repo}", "rlist")
         if rc != EXIT_SUCCESS:
             print("rlist", rc, out)
@@ -86,3 +87,6 @@ def make_files(dir, count, size, rnd=True):
         if rc != EXIT_SUCCESS:
             print("check", rc, out)
         assert rc == EXIT_SUCCESS
+    finally:
+        # try to free the space allocated for the repo
+        cmd_fixture(f"--repo={repo}", "rdelete")


Test patch: