Repository: rucio/rucio

Issue:
Reaper tests make no assertions or deletions
Motivation
----------
`test_reaper.py` and `test_reaper2.py` run the reaper, but nothing is ever deleted as the `source` column in `rse_usage` is set to `srm` when generating the test data, but (by default) this is `storage` for the RSE being used in the test. There replicas also need a tombstone set on them. As a result we never delete anything when running the reaper, but as no assertions are made the test passes.
There's also a bug in the way we decide how many files to delete (for the original reaper) where if (for example) 5 files need deleting to make the required amount of free space, we only delete 4. This is due to the order in which we check file size vs adding the replica to a list of files to be deleted in core.replica.list_unlocked_replicas.

Modification
------------
Change `source` and add tombstones to the replicas generated to allow deletion. Add some asserts to the reaper tests to make sure we delete the expected number of replicas. Change how we calculate the number of files that need reaping for the original reaper.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -28,6 +28,7 @@
 # - Andrew Lister, <andrew.lister@stfc.ac.uk>, 2019
 # - Brandon White, <bjwhite@fnal.gov>, 2019
 # - Luc Goossens <luc.goossens@cern.ch>, 2020
+# - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -1635,9 +1636,9 @@ def list_unlocked_replicas(rse_id, limit, bytes=None, worker_number=None, total_
     for (scope, name, path, bytes, tombstone, state) in query.yield_per(1000):
         if state != ReplicaState.UNAVAILABLE:
 
-            total_bytes += bytes
             if tombstone != OBSOLETE and needed_space is not None and total_bytes > needed_space:
                 break
+            total_bytes += bytes
 
             total_files += 1
             if total_files > limit:


Test patch:
diff --git a/lib/rucio/tests/test_reaper.py b/lib/rucio/tests/test_reaper.py
--- a/lib/rucio/tests/test_reaper.py
+++ b/lib/rucio/tests/test_reaper.py
@@ -18,29 +18,56 @@
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2019
+# - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
+# - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
+
+from datetime import datetime, timedelta
+from nose.tools import assert_equal
 
 from rucio.common.types import InternalAccount, InternalScope
 from rucio.common.utils import generate_uuid
 from rucio.core import rse as rse_core
 from rucio.core import replica as replica_core
 from rucio.daemons.reaper.reaper import reaper
+from rucio.tests.common import rse_name_generator
 
 
 def test_reaper():
     """ REAPER (DAEMON): Test the reaper daemon."""
+    rse_name = rse_name_generator()
+    rse_id = rse_core.add_rse(rse_name)
+
+    mock_protocol = {'scheme': 'MOCK',
+                     'hostname': 'localhost',
+                     'port': 123,
+                     'prefix': '/test/reaper',
+                     'impl': 'rucio.rse.protocols.mock.Default',
+                     'domains': {
+                         'lan': {'read': 1,
+                                 'write': 1,
+                                 'delete': 1},
+                         'wan': {'read': 1,
+                                 'write': 1,
+                                 'delete': 1}}}
+    rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
+
     nb_files = 30
     file_size = 2147483648  # 2G
-    rse_id = rse_core.get_rse_id(rse='MOCK')
 
+    file_names = []
     for i in range(nb_files):
+        file_name = 'lfn' + generate_uuid()
+        file_names.append(file_name)
         replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip'),
-                                 name='lfn' + generate_uuid(), bytes=file_size,
+                                 name=file_name, bytes=file_size,
+                                 tombstone=datetime.utcnow() - timedelta(days=1),
                                  account=InternalAccount('root'), adler32=None, md5=None)
 
-    rse_core.set_rse_usage(rse_id=rse_id, source='srm', used=nb_files * file_size, free=800)
+    rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=800)
     rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=10737418240)
     rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10)
 
-    rses = [rse_core.get_rse(rse_core.get_rse_id('MOCK')), ]
+    rses = [rse_core.get_rse(rse_id), ]
     reaper(once=True, rses=rses)
     reaper(once=True, rses=rses)
+    assert_equal(len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip'), 'name': n} for n in file_names], rse_expression=rse_name))), nb_files - 10)
diff --git a/lib/rucio/tests/test_reaper2.py b/lib/rucio/tests/test_reaper2.py
--- a/lib/rucio/tests/test_reaper2.py
+++ b/lib/rucio/tests/test_reaper2.py
@@ -14,28 +14,55 @@
 #
 # Authors:
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020
+# - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
+
+from datetime import datetime, timedelta
+from nose.tools import assert_equal
 
 from rucio.common.types import InternalAccount, InternalScope
 from rucio.common.utils import generate_uuid
 from rucio.core import rse as rse_core
 from rucio.core import replica as replica_core
 from rucio.daemons.reaper.reaper2 import reaper
+from rucio.tests.common import rse_name_generator
 
 
 def test_reaper():
     """ REAPER2 (DAEMON): Test the reaper2 daemon."""
+    rse_name = rse_name_generator()
+    rse_id = rse_core.add_rse(rse_name)
+
+    mock_protocol = {'scheme': 'MOCK',
+                     'hostname': 'localhost',
+                     'port': 123,
+                     'prefix': '/test/reaper',
+                     'impl': 'rucio.rse.protocols.mock.Default',
+                     'domains': {
+                         'lan': {'read': 1,
+                                 'write': 1,
+                                 'delete': 1},
+                         'wan': {'read': 1,
+                                 'write': 1,
+                                 'delete': 1}}}
+    rse_core.add_protocol(rse_id=rse_id, parameter=mock_protocol)
+
     nb_files = 30
     file_size = 2147483648  # 2G
-    rse_id = rse_core.get_rse_id(rse='MOCK')
 
+    file_names = []
     for i in range(nb_files):
+        file_name = 'lfn' + generate_uuid()
+        file_names.append(file_name)
         replica_core.add_replica(rse_id=rse_id, scope=InternalScope('data13_hip'),
-                                 name='lfn' + generate_uuid(), bytes=file_size,
+                                 name=file_name, bytes=file_size,
+                                 tombstone=datetime.utcnow() - timedelta(days=1),
                                  account=InternalAccount('root'), adler32=None, md5=None)
 
-    rse_core.set_rse_usage(rse_id=rse_id, source='srm', used=nb_files * file_size, free=800)
+    rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=800)
     rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=10737418240)
     rse_core.set_rse_limits(rse_id=rse_id, name='MaxBeingDeletedFiles', value=10)
 
-    reaper(once=True, rses=[], include_rses='MOCK', exclude_rses=[])
-    reaper(once=True, rses=[], include_rses='MOCK', exclude_rses=[])
+    reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=[])
+    reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=[])
+
+    assert_equal(len(list(replica_core.list_replicas(dids=[{'scope': InternalScope('data13_hip'), 'name': n} for n in file_names], rse_expression=rse_name))), nb_files - 5)