Repository: rucio/rucio

Issue:
Too many execution of get_bad_replicas_backlog
Motivation
----------
There are too many execution of `get_bad_replicas_backlog` in the necromancer, due to the short expiration date of the cache.





Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -513,39 +513,35 @@ def get_pfn_to_rse(pfns, vo='def', session=None):
 
 
 @read_session
-def get_bad_replicas_backlog(force_refresh=False, session=None):
+def get_bad_replicas_backlog(session=None):
     """
     Get the replica backlog by RSE.
 
-    :param force_refresh:      Boolean to choose if the cached value must be refreshed.
     :param session:            The database session in use.
 
     :returns: a list of dictionary {'rse_id': cnt_bad_replicas}.
     """
-    result = REGION.get('bad_replicas_backlog')
-    if result is NO_VALUE or force_refresh:
-        schema_dot = '%s.' % DEFAULT_SCHEMA_NAME if DEFAULT_SCHEMA_NAME else ''
-        if session.bind.dialect.name == 'oracle':
-            # The filter(text...)) is needed otherwise, SQLA uses bind variables and the index is not used.
-            query = session.query(func.count(models.RSEFileAssociation.rse_id), models.RSEFileAssociation.rse_id).\
-                with_hint(models.RSEFileAssociation, "INDEX(REPLICAS REPLICAS_STATE_IDX)", 'oracle').\
-                filter(text("CASE WHEN (%sreplicas.state != 'A') THEN %sreplicas.rse_id END IS NOT NULL" % (schema_dot,
-                                                                                                            schema_dot))). \
-                filter(models.RSEFileAssociation.state == ReplicaState.BAD)
-        else:
-            query = session.query(func.count(models.RSEFileAssociation.rse_id), models.RSEFileAssociation.rse_id).\
-                filter(models.RSEFileAssociation.state == ReplicaState.BAD)
-
-        query = query.join(models.DataIdentifier,
-                           and_(models.DataIdentifier.scope == models.RSEFileAssociation.scope,
-                                models.DataIdentifier.name == models.RSEFileAssociation.name)).\
-            filter(models.DataIdentifier.availability != DIDAvailability.LOST).\
-            group_by(models.RSEFileAssociation.rse_id)
-
-        result = dict()
-        for cnt, rse_id in query.all():
-            result[rse_id] = cnt
-        REGION.set('bad_replicas_backlog', result)
+    schema_dot = '%s.' % DEFAULT_SCHEMA_NAME if DEFAULT_SCHEMA_NAME else ''
+    if session.bind.dialect.name == 'oracle':
+        # The filter(text...)) is needed otherwise, SQLA uses bind variables and the index is not used.
+        query = session.query(func.count(models.RSEFileAssociation.rse_id), models.RSEFileAssociation.rse_id).\
+            with_hint(models.RSEFileAssociation, "INDEX(REPLICAS REPLICAS_STATE_IDX)", 'oracle').\
+            filter(text("CASE WHEN (%sreplicas.state != 'A') THEN %sreplicas.rse_id END IS NOT NULL" % (schema_dot,
+                                                                                                        schema_dot))). \
+            filter(models.RSEFileAssociation.state == ReplicaState.BAD)
+    else:
+        query = session.query(func.count(models.RSEFileAssociation.rse_id), models.RSEFileAssociation.rse_id).\
+            filter(models.RSEFileAssociation.state == ReplicaState.BAD)
+
+    query = query.join(models.DataIdentifier,
+                       and_(models.DataIdentifier.scope == models.RSEFileAssociation.scope,
+                            models.DataIdentifier.name == models.RSEFileAssociation.name)).\
+        filter(models.DataIdentifier.availability != DIDAvailability.LOST).\
+        group_by(models.RSEFileAssociation.rse_id)
+
+    result = dict()
+    for cnt, rse_id in query.all():
+        result[rse_id] = cnt
     return result
 
 
diff --git a/lib/rucio/daemons/badreplicas/necromancer.py b/lib/rucio/daemons/badreplicas/necromancer.py
--- a/lib/rucio/daemons/badreplicas/necromancer.py
+++ b/lib/rucio/daemons/badreplicas/necromancer.py
@@ -38,9 +38,12 @@
 except ImportError:
     from configparser import NoOptionError, NoSectionError
 
+from dogpile.cache.api import NO_VALUE
+
 import rucio.db.sqla.util
 from rucio.common import exception
-from rucio.common.config import config_get
+from rucio.common.cache import make_region_memcached
+from rucio.common.config import config_get, config_get_int
 from rucio.common.exception import DatabaseException
 from rucio.common.logging import formatted_logger, setup_logging
 from rucio.core import monitor, heartbeat
@@ -50,6 +53,7 @@
 from rucio.db.sqla.constants import ReplicaState
 
 GRACEFUL_STOP = threading.Event()
+REGION = make_region_memcached(expiration_time=config_get_int('necromancer', 'cache_time', False, 600))
 
 
 def necromancer(thread=0, bulk=5, once=False, sleep_time=60):
@@ -112,7 +116,10 @@ def necromancer(thread=0, bulk=5, once=False, sleep_time=60):
             max_bad_replicas_backlog_count = int(max_bad_replicas_backlog_count)
         except (NoOptionError, NoSectionError, RuntimeError, ValueError):
             max_bad_replicas_backlog_count = None
-        bad_replicas_backlog = get_bad_replicas_backlog()
+        bad_replicas_backlog = REGION.get('bad_replicas_backlog')
+        if bad_replicas_backlog is NO_VALUE:
+            bad_replicas_backlog = get_bad_replicas_backlog()
+            REGION.set('bad_replicas_backlog', bad_replicas_backlog)
         tot_bad_files = sum([bad_replicas_backlog[key] for key in bad_replicas_backlog])
         list_of_rses = list()
         # If too many replica, call list_bad_replicas with a list of RSEs


Test patch:
diff --git a/lib/rucio/tests/test_bad_replica.py b/lib/rucio/tests/test_bad_replica.py
--- a/lib/rucio/tests/test_bad_replica.py
+++ b/lib/rucio/tests/test_bad_replica.py
@@ -26,10 +26,13 @@
 from rucio.common.exception import RucioException, UnsupportedOperation, InvalidType
 from rucio.common.utils import generate_uuid, clean_surls
 from rucio.core.did import delete_dids
-from rucio.core.replica import add_replicas, get_replicas_state, list_replicas, declare_bad_file_replicas, list_bad_replicas, get_bad_pfns, get_bad_replicas_backlog, list_bad_replicas_status
+from rucio.core.replica import (add_replicas, get_replicas_state, list_replicas,
+                                declare_bad_file_replicas, list_bad_replicas, get_bad_pfns,
+                                get_bad_replicas_backlog, list_bad_replicas_status)
 from rucio.daemons.badreplicas.minos import run as minos_run
 from rucio.daemons.badreplicas.minos_temporary_expiration import run as minos_temp_run
 from rucio.daemons.badreplicas.necromancer import run as necromancer_run
+from rucio.daemons.badreplicas.necromancer import REGION
 from rucio.db.sqla.constants import DIDType, ReplicaState, BadPFNStatus, BadFilesStatus
 from rucio.tests.common import headers, auth
 
@@ -129,7 +132,7 @@ def test_get_bad_replicas_backlog(rse_factory, mock_scope, root_account, file_co
     res = declare_bad_file_replicas(replicas, 'This is a good reason', root_account)
     assert res == {}
 
-    result = get_bad_replicas_backlog(force_refresh=True)
+    result = get_bad_replicas_backlog()
     assert rse1_id in result
     assert result[rse1_id] == nbfiles1
 
@@ -150,7 +153,8 @@ def test_get_bad_replicas_backlog(rse_factory, mock_scope, root_account, file_co
         assert rep in list_rep
 
     # Run necromancer once, all the files on RSE2 should be gone, 80 files should stay on RSE1
-    get_bad_replicas_backlog(force_refresh=True)
+    REGION.invalidate()
+    get_bad_replicas_backlog()
     necromancer_run(threads=1, bulk=20, once=True)
 
     bad_replicas = list_bad_replicas(rses=[{'id': rse1_id}, {'id': rse2_id}])