Repository: rucio/rucio

Issue:
Protect the Evaulator from the Necromancer
Motivation
----------

The declaration of lost files is an important, but secondary workflow and should not affect deletions, which is a primary operation. The Reaper will stop if there’s a significant Evaluator backlog as a safety measure. The Necromancer directly contributes to this backlog, especially when there’s a large amount of files belonging to different large datasets.

Modification
------------

The Necromancer should pause if the Evaluator backlog passes some threshold. Note that it might be prudent to not rely solely on the number of updated DIDs, but also their size.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -2280,7 +2280,7 @@ def examine_rule(rule_id, session=None):
 
 
 @transactional_session
-def get_evaluation_backlog(session=None):
+def get_evaluation_backlog(expiration_time=600, session=None):
     """
     Counts the number of entries in the rule evaluation backlog.
     (Number of files to be evaluated)
@@ -2288,7 +2288,7 @@ def get_evaluation_backlog(session=None):
     :returns:     Tuple (Count, Datetime of oldest entry)
     """
 
-    result = REGION.get('rule_evaluation_backlog', expiration_time=600)
+    result = REGION.get('rule_evaluation_backlog', expiration_time=expiration_time)
     if result is NO_VALUE:
         result = session.query(func.count(models.UpdatedDID.created_at), func.min(models.UpdatedDID.created_at)).one()
         REGION.set('rule_evaluation_backlog', result)
diff --git a/lib/rucio/daemons/badreplicas/necromancer.py b/lib/rucio/daemons/badreplicas/necromancer.py
--- a/lib/rucio/daemons/badreplicas/necromancer.py
+++ b/lib/rucio/daemons/badreplicas/necromancer.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 # Authors:
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2018-2019
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2018-2021
 # - Martin Barisits <martin.barisits@cern.ch>, 2019
 # - Brandon White <bjwhite@fnal.gov>, 2019
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020-2021
@@ -29,21 +29,23 @@
 import socket
 import threading
 import time
+from datetime import datetime, timedelta
 from math import ceil
 from sys import exc_info
 from traceback import format_exception
 
 import rucio.db.sqla.util
 from rucio.common import exception
-from rucio.common.exception import DatabaseException
-from rucio.common.logging import setup_logging
+from rucio.common.exception import DatabaseException, ConfigNotFound
+from rucio.common.logging import formatted_logger, setup_logging
 from rucio.common.utils import chunks, daemon_sleep
 from rucio.core import monitor, heartbeat
+from rucio.core.config import get
 from rucio.core.replica import list_bad_replicas, get_replicas_state, list_bad_replicas_history, update_bad_replicas_history
-from rucio.core.rule import update_rules_for_lost_replica, update_rules_for_bad_replica
+from rucio.core.rule import update_rules_for_lost_replica, update_rules_for_bad_replica, get_evaluation_backlog
 from rucio.db.sqla.constants import ReplicaState
 
-graceful_stop = threading.Event()
+GRACEFUL_STOP = threading.Event()
 
 
 def necromancer(thread=0, bulk=5, once=False, sleep_time=60):
@@ -66,10 +68,40 @@ def necromancer(thread=0, bulk=5, once=False, sleep_time=60):
     hb_thread = threading.current_thread()
     heartbeat.sanity_check(executable=executable, hostname=hostname)
 
-    while not graceful_stop.is_set():
+    while not GRACEFUL_STOP.is_set():
 
         heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
-        prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
+        prefix = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
+        logger = formatted_logger(logging.log, prefix + '%s')
+
+        # Check if there is a Judge Evaluator backlog
+        try:
+            max_evaluator_backlog_count = get('necromancer', 'max_evaluator_backlog_count')
+        except ConfigNotFound:
+            max_evaluator_backlog_count = None
+        try:
+            max_evaluator_backlog_duration = get('necromancer', 'max_evaluator_backlog_duration')
+        except ConfigNotFound:
+            max_evaluator_backlog_duration = None
+        if max_evaluator_backlog_count or max_evaluator_backlog_duration:
+            backlog = get_evaluation_backlog(expiration_time=sleep_time)
+            if max_evaluator_backlog_count and \
+               backlog[0] and \
+               max_evaluator_backlog_duration and \
+               backlog[1] and \
+               backlog[0] > max_evaluator_backlog_count and \
+               backlog[1] < datetime.utcnow() - timedelta(minutes=max_evaluator_backlog_duration):
+                logger(logging.ERROR, 'Necromancer: Judge evaluator backlog count and duration hit, stopping operation')
+                GRACEFUL_STOP.wait(30)
+                continue
+            elif max_evaluator_backlog_count and backlog[0] and backlog[0] > max_evaluator_backlog_count:
+                logger(logging.ERROR, 'Necromancer: Judge evaluator backlog count hit, stopping operation')
+                GRACEFUL_STOP.wait(30)
+                continue
+            elif max_evaluator_backlog_duration and backlog[1] and backlog[1] < datetime.utcnow() - timedelta(minutes=max_evaluator_backlog_duration):
+                logger(logging.ERROR, 'Necromancer: Judge evaluator backlog duration hit, stopping operation')
+                GRACEFUL_STOP.wait(30)
+                continue
 
         stime = time.time()
         replicas = []
@@ -78,61 +110,61 @@ def necromancer(thread=0, bulk=5, once=False, sleep_time=60):
 
             for replica in replicas:
                 scope, name, rse_id, rse = replica['scope'], replica['name'], replica['rse_id'], replica['rse']
-                logging.info(prepend_str + 'Working on %s:%s on %s' % (scope, name, rse))
+                logger(logging.INFO, 'Working on %s:%s on %s' % (scope, name, rse))
 
                 list_replicas = get_replicas_state(scope=scope, name=name)
                 if ReplicaState.AVAILABLE not in list_replicas and ReplicaState.TEMPORARY_UNAVAILABLE not in list_replicas:
-                    logging.info(prepend_str + 'File %s:%s has no other available or temporary available replicas, it will be marked as lost' % (scope, name))
+                    logger(logging.INFO, 'File %s:%s has no other available or temporary available replicas, it will be marked as lost' % (scope, name))
                     try:
                         update_rules_for_lost_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
                         monitor.record_counter(name='necromancer.badfiles.lostfile')
                     except DatabaseException as error:
-                        logging.info(prepend_str + '%s' % (str(error)))
+                        logger(logging.WARNING, str(error))
 
                 else:
                     rep = list_replicas.get(ReplicaState.AVAILABLE, [])
                     unavailable_rep = list_replicas.get(ReplicaState.TEMPORARY_UNAVAILABLE, [])
-                    logging.info(prepend_str + 'File %s:%s can be recovered. Available sources : %s + Unavailable sources : %s' % (scope, name, str(rep), str(unavailable_rep)))
+                    logger(logging.INFO, 'File %s:%s can be recovered. Available sources : %s + Unavailable sources : %s' % (scope, name, str(rep), str(unavailable_rep)))
                     try:
                         update_rules_for_bad_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
                         monitor.record_counter(name='necromancer.badfiles.recovering')
                     except DatabaseException as error:
-                        logging.info(prepend_str + '%s' % (str(error)))
+                        logger(logging.WARNING, str(error))
 
-            logging.info(prepend_str + 'It took %s seconds to process %s replicas' % (str(time.time() - stime), str(len(replicas))))
+            logger(logging.INFO, 'It took %s seconds to process %s replicas' % (str(time.time() - stime), str(len(replicas))))
         except Exception:
             exc_type, exc_value, exc_traceback = exc_info()
-            logging.critical(prepend_str + ''.join(format_exception(exc_type, exc_value, exc_traceback)).strip())
+            logger(logging.CRITICAL, ''.join(format_exception(exc_type, exc_value, exc_traceback)).strip())
 
         if once:
             break
         else:
             now = time.time()
             if (now - update_history_time) > update_history_threshold:
-                logging.info(prepend_str + 'Last update of history table %s seconds ago. Running update.' % (now - update_history_time))
+                logger(logging.INFO, 'Last update of history table %s seconds ago. Running update.' % (now - update_history_time))
                 bad_replicas = list_bad_replicas_history(limit=1000000,
                                                          thread=heart_beat['assign_thread'],
                                                          total_threads=heart_beat['nr_threads'])
                 for rse_id in bad_replicas:
                     chunk_size = 1000
                     nchunk = int(ceil(len(bad_replicas[rse_id]) / chunk_size))
-                    logging.debug(prepend_str + 'Update history for rse_id %s' % (rse_id))
+                    logger(logging.DEBUG, 'Update history for rse_id %s' % (rse_id))
                     cnt = 0
                     for chunk in chunks(bad_replicas[rse_id], chunk_size):
-                        logging.debug(prepend_str + ' History for rse_id %s : chunk %i/%i' % (rse_id, cnt, nchunk))
+                        logger(logging.DEBUG, ' History for rse_id %s : chunk %i/%i' % (rse_id, cnt, nchunk))
                         cnt += 1
                         update_bad_replicas_history(chunk, rse_id)
-                logging.info(prepend_str + 'History table updated in %s seconds' % (time.time() - now))
+                logger(logging.INFO, 'History table updated in %s seconds' % (time.time() - now))
                 update_history_time = time.time()
 
             if len(replicas) == bulk:
-                logging.info(prepend_str + 'Processed maximum number of replicas according to the bulk size. Restart immediately next cycle')
+                logger(logging.INFO, 'Processed maximum number of replicas according to the bulk size. Restart immediately next cycle')
             else:
-                daemon_sleep(start_time=stime, sleep_time=sleep_time, graceful_stop=graceful_stop)
+                daemon_sleep(start_time=stime, sleep_time=sleep_time, graceful_stop=GRACEFUL_STOP)
 
-    logging.info(prepend_str + 'Graceful stop requested')
+    logger(logging.INFO, 'Graceful stop requested')
     heartbeat.die(executable, hostname, pid, hb_thread)
-    logging.info(prepend_str + 'Graceful stop done')
+    logger(logging.INFO, 'Graceful stop done')
 
 
 def run(threads=1, bulk=100, once=False, sleep_time=60):
@@ -166,4 +198,4 @@ def stop(signum=None, frame=None):
     """
     Graceful exit.
     """
-    graceful_stop.set()
+    GRACEFUL_STOP.set()


Test patch: