Repository: rucio/rucio

Issue:
Dark reaper improvements due to DB load
Motivation
----------
The dark-reaper is the top-query on the ATLAS database, causing a significant fraction of the DB load, although it is a rather unimportant service. There is just too many executions of the cycles and lots of retries.

This is related to #4744 

Modification
------------
1. `rucio.core.quarantined_replica.list_rses` is a very ambigous name. Rename to `list_rses_with_quarantined_replicas`
2. The `list_quarantined_replicas` command itself removes replicas which also exist in the `replicas` table. But this causes the RSE to always show up in each dark-reaper loop, although it is never doing anything. `list_quarantined_replicas` should directly remove `quarantined_replicas` from the table if they have a replica in the `replicas` table. No need to treat them in the `dark-reaper`
3. Sleep times can be increased to 5 minutes, at least.



Discussion:

Resolving patch:
diff --git a/lib/rucio/core/quarantined_replica.py b/lib/rucio/core/quarantined_replica.py
--- a/lib/rucio/core/quarantined_replica.py
+++ b/lib/rucio/core/quarantined_replica.py
@@ -23,14 +23,16 @@
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 # - Rahul Chauhan <omrahulchauhan@gmail.com>, 2021
 # - Martin Barisits <martin.barisits@cern.ch>, 2021
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2022
 
 import datetime
 
-from sqlalchemy import and_, or_, exists, not_
-from sqlalchemy.sql.expression import select, false
+from sqlalchemy import and_, or_
+from sqlalchemy.sql.expression import false
 
 from rucio.db.sqla import models, filter_thread_work
 from rucio.db.sqla.session import read_session, transactional_session
+from rucio.common.utils import chunks
 
 
 @transactional_session
@@ -72,9 +74,12 @@ def add_quarantined_replicas(rse_id, replicas, session=None):
 
     session.bulk_insert_mappings(
         models.QuarantinedReplica,
-        [{'rse_id': rse_id, 'path': file['path'],
-            'scope': file.get('scope'), 'name': file.get('name'),
-            'bytes': file.get('bytes')} for file in replicas])
+        [{'rse_id': rse_id,
+          'path': file['path'],
+          'scope': file.get('scope'),
+          'name': file.get('name'),
+          'bytes': file.get('bytes')}
+         for file in replicas])
 
 
 @transactional_session
@@ -117,34 +122,60 @@ def list_quarantined_replicas(rse_id, limit, worker_number=None, total_workers=N
     :param total_workers:      Number of total workers.
     :param session: The database session in use.
 
-    :returns: a list of dictionary replica.
+    :returns: two lists :
+              - The first one contains quarantine replicas actually registered in the replicas tables
+              - The second one contains real "dark" files
     """
 
+    replicas_clause = []
+    quarantined_replicas = {}
+    real_replicas = []
+    dark_replicas = []
     query = session.query(models.QuarantinedReplica.path,
                           models.QuarantinedReplica.bytes,
                           models.QuarantinedReplica.scope,
                           models.QuarantinedReplica.name,
                           models.QuarantinedReplica.created_at).\
         filter(models.QuarantinedReplica.rse_id == rse_id)
-
-    # do no delete valid replicas
-    stmt = exists(select([1]).prefix_with("/*+ index(REPLICAS REPLICAS_PK) */", dialect='oracle')).\
-        where(and_(models.RSEFileAssociation.scope == models.QuarantinedReplica.scope,
-                   models.RSEFileAssociation.name == models.QuarantinedReplica.name,
-                   models.RSEFileAssociation.rse_id == models.QuarantinedReplica.rse_id))
-    query = query.filter(not_(stmt))
     query = filter_thread_work(session=session, query=query, total_threads=total_workers, thread_id=worker_number, hash_variable='path')
-    return [{'path': path,
-             'rse_id': rse_id,
-             'created_at': created_at,
-             'scope': scope,
-             'name': name,
-             'bytes': bytes_}
-            for path, bytes_, scope, name, created_at in query.limit(limit)]
+
+    for path, bytes_, scope, name, created_at in query.limit(limit):
+        if not (scope, name) in quarantined_replicas:
+            quarantined_replicas[(scope, name)] = []
+            replicas_clause.append(and_(models.RSEFileAssociation.scope == scope,
+                                        models.RSEFileAssociation.name == name))
+        quarantined_replicas[(scope, name)].append((path, bytes_, created_at))
+
+    for chunk in chunks(replicas_clause, 20):
+        query = session.query(models.RSEFileAssociation.scope,
+                              models.RSEFileAssociation.name).\
+            filter(models.RSEFileAssociation.rse_id == rse_id).\
+            filter(or_(*chunk))
+
+        for scope, name in query.all():
+            reps = quarantined_replicas.pop((scope, name))
+            real_replicas.extend([{'scope': scope,
+                                   'name': name,
+                                   'rse_id': rse_id,
+                                   'path': rep[0],
+                                   'bytes': rep[1],
+                                   'created_at': rep[2]}
+                                  for rep in reps])
+
+    for key, value in quarantined_replicas.items():
+        dark_replicas.extend([{'scope': key[0],
+                               'name': key[1],
+                               'rse_id': rse_id,
+                               'path': rep[0],
+                               'bytes': rep[1],
+                               'created_at': rep[2]}
+                              for rep in value])
+
+    return real_replicas, dark_replicas
 
 
 @read_session
-def list_rses(filters=None, session=None):
+def list_rses_with_quarantined_replicas(filters=None, session=None):
     """
     List RSEs in the Quarantined Queues.
 
diff --git a/lib/rucio/daemons/reaper/dark_reaper.py b/lib/rucio/daemons/reaper/dark_reaper.py
--- a/lib/rucio/daemons/reaper/dark_reaper.py
+++ b/lib/rucio/daemons/reaper/dark_reaper.py
@@ -19,7 +19,7 @@
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2016-2021
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2020
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2020-2022
 # - Brandon White <bjwhite@fnal.gov>, 2019
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
@@ -54,7 +54,7 @@
 from rucio.core.message import add_message
 from rucio.core.quarantined_replica import (list_quarantined_replicas,
                                             delete_quarantined_replicas,
-                                            list_rses)
+                                            list_rses_with_quarantined_replicas)
 import rucio.core.rse as rse_core
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.core.vo import list_vos
@@ -65,7 +65,7 @@
 GRACEFUL_STOP = threading.Event()
 
 
-def reaper(rses, worker_number=0, total_workers=1, chunk_size=100, once=False, scheme=None, sleep_time=60):
+def reaper(rses, worker_number=0, total_workers=1, chunk_size=100, once=False, scheme=None, sleep_time=300):
     """
     Main loop to select and delete files.
 
@@ -97,20 +97,21 @@ def reaper(rses, worker_number=0, total_workers=1, chunk_size=100, once=False, s
             nothing_to_do = True
             start_time = time.time()
 
-            rses_to_process = list(set(rses) & set(list_rses()))
+            rses_to_process = list(set(rses) & set(list_rses_with_quarantined_replicas()))
             random.shuffle(rses_to_process)
             for rse_id in rses_to_process:
-                replicas = list_quarantined_replicas(rse_id=rse_id,
-                                                     limit=chunk_size, worker_number=worker_number,
-                                                     total_workers=total_workers)
+                # The following query returns the list of real replicas (deleted_replicas) and list of dark replicas (dark_replicas)
+                # Real replicas can be directly removed from the quarantine table
+                deleted_replicas, dark_replicas = list_quarantined_replicas(rse_id=rse_id,
+                                                                            limit=chunk_size, worker_number=worker_number,
+                                                                            total_workers=total_workers)
 
                 rse_info = rsemgr.get_rse_info(rse_id=rse_id)
                 rse = rse_info['rse']
                 prot = rsemgr.create_protocol(rse_info, 'delete', scheme=scheme)
-                deleted_replicas = []
                 try:
                     prot.connect()
-                    for replica in replicas:
+                    for replica in dark_replicas:
                         nothing_to_do = False
                         scope = ''
                         if replica['scope']:
@@ -198,7 +199,7 @@ def stop(signum=None, frame=None):
 
 
 def run(total_workers=1, chunk_size=100, once=False, rses=[], scheme=None,
-        exclude_rses=None, include_rses=None, vos=None, delay_seconds=0, sleep_time=60):
+        exclude_rses=None, include_rses=None, vos=None, delay_seconds=0, sleep_time=300):
     """
     Starts up the reaper threads.
 


Test patch:
diff --git a/lib/rucio/tests/test_quarantined_replica.py b/lib/rucio/tests/test_quarantined_replica.py
--- a/lib/rucio/tests/test_quarantined_replica.py
+++ b/lib/rucio/tests/test_quarantined_replica.py
@@ -15,7 +15,7 @@
 #
 # Authors:
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2016-2017
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2017
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2017-2022
 # - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
@@ -42,7 +42,8 @@ def test_quarantined_replicas():
 
     rse_id = get_rse_id(rse='MOCK', **vo)
 
-    quarantined_replicas = len(list_quarantined_replicas(rse_id=rse_id, limit=10000))
+    real_replicas, dark_replicas = list_quarantined_replicas(rse_id=rse_id, limit=10000)
+    quarantined_replicas = len(real_replicas) + len(dark_replicas)
 
     nbreplicas = 5
 
@@ -50,8 +51,10 @@ def test_quarantined_replicas():
 
     add_quarantined_replicas(rse_id=rse_id, replicas=replicas)
 
-    assert quarantined_replicas + nbreplicas == len(list_quarantined_replicas(rse_id=rse_id, limit=10000))
+    real_replicas, dark_replicas = list_quarantined_replicas(rse_id=rse_id, limit=10000)
+    assert quarantined_replicas + nbreplicas == len(dark_replicas) + len(real_replicas)
 
     delete_quarantined_replicas(rse_id=rse_id, replicas=replicas)
 
-    assert quarantined_replicas == len(list_quarantined_replicas(rse_id=rse_id, limit=10000))
+    real_replicas, dark_replicas = list_quarantined_replicas(rse_id=rse_id, limit=10000)
+    assert quarantined_replicas == len(dark_replicas) + len(real_replicas)