Repository: rucio/rucio

Issue:
New Reaper implementation can unintentionally act as if in greedy mode.
Motivation
----------

> reaper2, correctly, schedules the RSE for deletion, but then finds that it cannot delete these replicas because they are in COPYING but since it should delete the 1k (or whatever it is set) bulk size in the loop, it goes to other tombstoned replicas. Since the epoche ones in copying don’t suddenly go away, it eats into the secondaries further

Modification
------------

> It should not schedule the RSE for deletion due to replicas in this state and secondly in the deletion loop it should still check the watermark levels and see if it needs to delete  secondaries.



Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -1627,15 +1627,16 @@ def list_unlocked_replicas(rse_id, limit, bytes=None, worker_number=None, total_
 
 
 @transactional_session
-def list_and_mark_unlocked_replicas(limit, bytes=None, rse_id=None, delay_seconds=600, session=None):
+def list_and_mark_unlocked_replicas(limit, bytes=None, rse_id=None, delay_seconds=600, only_delete_obsolete=False, session=None):
     """
     List RSE File replicas with no locks.
 
-    :param limit:              Number of replicas returned.
-    :param bytes:              The amount of needed bytes.
-    :param rse_id:             The rse_id.
-    :delay_seconds:            The delay to query replicas in BEING_DELETED state
-    :param session:            The database session in use.
+    :param limit:                    Number of replicas returned.
+    :param bytes:                    The amount of needed bytes.
+    :param rse_id:                   The rse_id.
+    :param delay_seconds:            The delay to query replicas in BEING_DELETED state
+    :param only_delete_obsolete      If set to True, will only return the replicas with EPOCH tombstone
+    :param session:                  The database session in use.
 
     :returns: a list of dictionary replica.
     """
@@ -1668,8 +1669,11 @@ def list_and_mark_unlocked_replicas(limit, bytes=None, rse_id=None, delay_second
         if replica_cnt[0] > 1:
             if state != ReplicaState.UNAVAILABLE:
                 total_bytes += bytes
-                if tombstone != OBSOLETE and needed_space is not None and total_bytes > needed_space:
-                    break
+                if tombstone != OBSOLETE:
+                    if only_delete_obsolete:
+                        break
+                    if needed_space is not None and total_bytes > needed_space:
+                        break
 
                 total_files += 1
                 if total_files > limit:
@@ -1690,8 +1694,11 @@ def list_and_mark_unlocked_replicas(limit, bytes=None, rse_id=None, delay_second
 
             if request_cnt[0] == 0:
                 total_bytes += bytes
-                if tombstone != OBSOLETE and needed_space is not None and total_bytes > needed_space:
-                    break
+                if tombstone != OBSOLETE:
+                    if only_delete_obsolete:
+                        break
+                    if needed_space is not None and total_bytes > needed_space:
+                        break
 
                 total_files += 1
                 if total_files > limit:
diff --git a/lib/rucio/daemons/reaper/reaper2.py b/lib/rucio/daemons/reaper/reaper2.py
--- a/lib/rucio/daemons/reaper/reaper2.py
+++ b/lib/rucio/daemons/reaper/reaper2.py
@@ -247,7 +247,7 @@ def __check_rse_usage(rse, rse_id, prepend_str):
     :param rse_id: the rse name.
     :param rse_id: the rse id.
 
-    :returns : max_being_deleted_files, needed_free_space, used, free.
+    :returns : max_being_deleted_files, needed_free_space, used, free, only_delete_obsolete.
     """
 
     result = REGION.get('rse_usage_%s' % rse_id)
@@ -258,14 +258,14 @@ def __check_rse_usage(rse, rse_id, prepend_str):
         attributes = list_rse_attributes(rse_id=rse_id)
         greedy = attributes.get('greedyDeletion', False)
         if greedy:
-            result = (max_being_deleted_files, 1000000000000, used, free)
+            result = (max_being_deleted_files, 1000000000000, used, free, False)
             REGION.set('rse_usage_%s' % rse_id, result)
             return result
 
         # Get RSE limits
         limits = get_rse_limits(rse_id=rse_id)
         if not limits and 'MinFreeSpace' not in limits and 'MaxBeingDeletedFiles' not in limits:
-            result = (max_being_deleted_files, needed_free_space, used, free)
+            result = (max_being_deleted_files, needed_free_space, used, free, False)
             REGION.set('rse_usage_%s' % rse_id, result)
             return result
 
@@ -291,10 +291,10 @@ def __check_rse_usage(rse, rse_id, prepend_str):
         # If no information is available about disk space, do nothing except if there are replicas with Epoch tombstone
         if not usage:
             if not obsolete:
-                result = (max_being_deleted_files, needed_free_space, used, free)
+                result = (max_being_deleted_files, needed_free_space, used, free, False)
                 REGION.set('rse_usage_%s' % rse_id, result)
                 return result
-            result = (max_being_deleted_files, obsolete, used, free)
+            result = (max_being_deleted_files, obsolete, used, free, True)
             REGION.set('rse_usage_%s' % rse_id, result)
             return result
 
@@ -306,7 +306,7 @@ def __check_rse_usage(rse, rse_id, prepend_str):
         if source_for_total_space != source_for_used_space:
             usage = [entry for entry in rse_usage if entry['source'] == source_for_used_space]
             if not usage:
-                result = (max_being_deleted_files, needed_free_space, None, free)
+                result = (max_being_deleted_files, needed_free_space, None, free, False)
                 REGION.set('rse_usage_%s' % rse_id, result)
                 return result
             for var in usage:
@@ -321,7 +321,7 @@ def __check_rse_usage(rse, rse_id, prepend_str):
         if needed_free_space <= 0:
             needed_free_space = 0 or obsolete
 
-        result = (max_being_deleted_files, needed_free_space, used, free)
+        result = (max_being_deleted_files, needed_free_space, used, free, True)
         REGION.set('rse_usage_%s' % rse_id, result)
         return result
     logging.debug('%s Using cached value for RSE usage on RSE %s', prepend_str, rse)
@@ -360,8 +360,14 @@ def reaper(rses, include_rses, exclude_rses, chunk_size=100, once=False, greedy=
     while not GRACEFUL_STOP.is_set():
 
         # Check if there is a Judge Evaluator backlog
-        max_evaluator_backlog_count = get('reaper', 'max_evaluator_backlog_count')
-        max_evaluator_backlog_duration = get('reaper', 'max_evaluator_backlog_duration')
+        try:
+            max_evaluator_backlog_count = get('reaper', 'max_evaluator_backlog_count')
+        except ConfigNotFound:
+            max_evaluator_backlog_count = None
+        try:
+            max_evaluator_backlog_duration = get('reaper', 'max_evaluator_backlog_duration')
+        except ConfigNotFound:
+            max_evaluator_backlog_duration = None
         if max_evaluator_backlog_count or max_evaluator_backlog_duration:
             backlog = get_evaluation_backlog()
             if max_evaluator_backlog_count and \
@@ -402,7 +408,7 @@ def reaper(rses, include_rses, exclude_rses, chunk_size=100, once=False, greedy=
                 if rse['availability'] % 2 == 0:
                     logging.debug('%s RSE %s is blacklisted for delete', prepend_str, rse['rse'])
                     continue
-                max_being_deleted_files, needed_free_space, used, free = __check_rse_usage(rse['rse'], rse['id'], prepend_str)
+                max_being_deleted_files, needed_free_space, used, free, only_delete_obsolete = __check_rse_usage(rse['rse'], rse['id'], prepend_str)
                 # Check if greedy mode
                 if greedy:
                     dict_rses[(rse['rse'], rse['id'])] = [1000000000000, max_being_deleted_files]
@@ -465,10 +471,13 @@ def reaper(rses, include_rses, exclude_rses, chunk_size=100, once=False, greedy=
                 del_start_time = time.time()
                 try:
                     with monitor.record_timer_block('reaper.list_unlocked_replicas'):
+                        if only_delete_obsolete:
+                            logging.debug('%s Will run list_and_mark_unlocked_replicas on %s. No space needed, will only delete EPOCH tombstoned replicas', prepend_str, rse_name)
                         replicas = list_and_mark_unlocked_replicas(limit=chunk_size,
                                                                    bytes=needed_free_space,
                                                                    rse_id=rse_id,
                                                                    delay_seconds=delay_seconds,
+                                                                   only_delete_obsolete=only_delete_obsolete,
                                                                    session=None)
                     logging.debug('%s list_and_mark_unlocked_replicas  on %s for %s bytes in %s seconds: %s replicas', prepend_str, rse_name, needed_free_space, time.time() - del_start_time, len(replicas))
                     if len(replicas) < chunk_size:


Test patch: