Repository: rucio/rucio

Issue:
The Reaper is not deleting obsolete replicas in a minimally-configured Rucio instance
### Description

If the RSE is minimally-configure (i.e. no additional RSE attributes beyond the identity attribute and no RSE usage counters beyond `rucio`), then the Reaper won’t do anything with this RSE, not even delete the obsolete replicas.

### Steps to reproduce

 1. Create a new RSE, in an environment that does not create additional RSE attributes or RSE usage counters.
 2. Take the necessary steps so that there is one or more deletable replicas with a Unix epoch tombstone.
 3. Run the Reaper.

One will only see `Nothing to delete on RSE` messages in the logs. The only way to overcome this is to set the `greedyDeletion` RSE attribute to `True`.

### Rucio Version

_No response_

### Additional Information

https://github.com/rucio/rucio/blob/bc71856931bf8384fb6715b1ae977a32ef30a0ac/lib/rucio/daemons/reaper/reaper.py#L362-L371

The issue is that, if `source_for_used_space` and `source_for_total_space` point to an RSE usage counter that does not exists and it’s the same for `obsolete`, then this function will return `(0, False)`.

Either the Reaper should be able to work correctly without `obsolete` or its calculation should be implemented in the Abacus daemon so that it’s available in a default Rucio installation.


Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/reaper/reaper.py b/lib/rucio/daemons/reaper/reaper.py
--- a/lib/rucio/daemons/reaper/reaper.py
+++ b/lib/rucio/daemons/reaper/reaper.py
@@ -330,65 +330,49 @@ def __check_rse_usage(rse: RseData, greedy: bool = False, logger: "Callable[...,
     :returns: needed_free_space, only_delete_obsolete.
     """
 
+    needed_free_space = 0
     # First of all check if greedy mode is enabled for this RSE
     if greedy:
         return 1000000000000, False
 
     rse.ensure_loaded(load_limits=True, load_usage=True, load_attributes=True)
+    available_sources = {}
+    available_sources['total'] = {key['source']: key['total'] for key in rse.usage}
+    available_sources['used'] = {key['source']: key['used'] for key in rse.usage}
 
     # Get RSE limits
     min_free_space = rse.limits.get('MinFreeSpace', 0)
 
-    # Check from which sources to get used and total spaces
-    # Default is storage
+    # Check from which sources to get used and total spaces (default storage)
+    # If specified sources do not exist, only delete obsolete
     source_for_total_space = rse.attributes.get('source_for_total_space', 'storage')
+    if source_for_total_space not in available_sources['total']:
+        logger(logging.WARNING, 'RSE: %s, \'%s\' requested for source_for_total_space but cannot be found. Will only delete obsolete',
+               rse.name, source_for_total_space)
+        return 0, True
     source_for_used_space = rse.attributes.get('source_for_used_space', 'storage')
+    if source_for_used_space not in available_sources['used']:
+        logger(logging.WARNING, 'RSE: %s, \'%s\' requested for source_for_used_space but cannot be found. Will only delete obsolete',
+               rse.name, source_for_used_space)
+        return 0, True
 
     logger(logging.DEBUG, 'RSE: %s, source_for_total_space: %s, source_for_used_space: %s',
            rse.name, source_for_total_space, source_for_used_space)
 
-    # Get total, used and obsolete space
-    total_space_entry = None
-    used_space_entry = None
-    obsolete_entry = None
-    for entry in rse.usage:
-        if total_space_entry and used_space_entry and obsolete_entry:
-            break
-
-        entry_source = entry['source']
-        if not total_space_entry and entry_source == source_for_total_space:
-            total_space_entry = entry
-        if not used_space_entry and entry_source == source_for_used_space:
-            used_space_entry = entry
-        if not obsolete_entry and entry_source == 'obsolete':
-            obsolete_entry = entry
-
-    obsolete = 0
-    if obsolete_entry:
-        obsolete = obsolete_entry['used']
-
-    # If no information is available about disk space, do nothing except if there are replicas with Epoch tombstone
-    needed_free_space = 0
-    if not total_space_entry:
-        if not obsolete:
-            return needed_free_space, False
-        return obsolete, True
-    if not used_space_entry:
-        return needed_free_space, False
-
-    # Extract the total and used space
-    total, used = total_space_entry['total'], used_space_entry['used']
+    # Get total and used space
+    total = available_sources['total'][source_for_total_space]
+    used = available_sources['used'][source_for_used_space]
 
     free = total - used
     if min_free_space:
         needed_free_space = min_free_space - free
 
     # If needed_free_space negative, nothing to delete except if some Epoch tombstoned replicas
-    if needed_free_space <= 0:
-        return obsolete, True
-    else:
+    if needed_free_space > 0:
         return needed_free_space, False
 
+    return 0, True
+
 
 def reaper(rses, include_rses, exclude_rses, vos=None, chunk_size=100, once=False, greedy=False,
            scheme=None, delay_seconds=0, sleep_time=60, auto_exclude_threshold=100, auto_exclude_timeout=600):


Test patch:
diff --git a/lib/rucio/tests/test_reaper.py b/lib/rucio/tests/test_reaper.py
--- a/lib/rucio/tests/test_reaper.py
+++ b/lib/rucio/tests/test_reaper.py
@@ -21,6 +21,7 @@
 from rucio.api import replica as replica_api
 from rucio.api import rse as rse_api
 from rucio.db.sqla import models
+from rucio.db.sqla.constants import OBSOLETE
 from rucio.db.sqla.session import get_session
 from rucio.common.exception import ReplicaNotFound, DataIdentifierNotFound
 from rucio.common.types import InternalAccount, InternalScope
@@ -490,3 +491,41 @@ def test_run_on_non_existing_scheme(vo, caches_mock, file_config_mock):
     reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None, chunk_size=1000, scheme='https')
     assert len(list(replica_core.list_replicas(dids, rse_expression=rse_name))) == 250
     assert cache_region.get('pause_deletion_%s' % rse_id)
+
+
+@pytest.mark.parametrize("file_config_mock", [
+    # Run test twice: with, and without, temp tables
+    {"overrides": [('core', 'use_temp_tables', 'True')]},
+    {"overrides": [('core', 'use_temp_tables', 'False')]},
+], indirect=True)
+@pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
+    'rucio.daemons.reaper.reaper.REGION'
+]}], indirect=True)
+def test_reaper_without_rse_usage(vo, caches_mock, file_config_mock):
+    """ REAPER (DAEMON): Mock test the reaper daemon and check it deletes obsolete replicas even if no rse_usage is set."""
+    [cache_region] = caches_mock
+    scope = InternalScope('data13_hip', vo=vo)
+
+    nb_files = 250
+    nb_epoch_tombstone = 10
+    file_size = 200  # 2G
+    rse_name, rse_id, dids = __add_test_rse_and_replicas(vo=vo, scope=scope, rse_name=rse_name_generator(),
+                                                         names=['lfn' + generate_uuid() for _ in range(nb_files)], file_size=file_size)
+
+    rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=50 * file_size)
+    assert len(list(replica_core.list_replicas(dids=dids, rse_expression=rse_name))) == nb_files
+
+    # Check first if the reaper does not delete anything if there's nothing obsolete
+    cache_region.invalidate()
+    reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None, chunk_size=1000, scheme='MOCK')
+    assert len(list(replica_core.list_replicas(dids=dids, rse_expression=rse_name))) == nb_files
+
+    # Now set Epoch tombstone for a few replicas
+    for did in dids[:nb_epoch_tombstone]:
+        print(did)
+        replica_core.set_tombstone(rse_id, did['scope'], did['name'], tombstone=OBSOLETE)
+
+    # The reaper should delete the replica with Epoch tombstone even if the rse_usage is not set
+    cache_region.invalidate()
+    reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None, chunk_size=1000, scheme='MOCK')
+    assert len(list(replica_core.list_replicas(dids, rse_expression=rse_name))) == nb_files - nb_epoch_tombstone