Repository: rucio/rucio

Issue:
Heartbeat handler for bad_replicas daemons
Motivation
----------
See https://github.com/rucio/rucio/issues/5443






Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/badreplicas/minos.py b/lib/rucio/daemons/badreplicas/minos.py
--- a/lib/rucio/daemons/badreplicas/minos.py
+++ b/lib/rucio/daemons/badreplicas/minos.py
@@ -13,34 +13,37 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
 import logging
 import math
-import os
 import re
-import socket
 import threading
-import time
 from datetime import datetime
+from typing import TYPE_CHECKING
 
 from sqlalchemy.exc import DatabaseError
 
 import rucio.db.sqla.util
 from rucio.common.exception import UnsupportedOperation, DataIdentifierNotFound, ReplicaNotFound, DatabaseException
-from rucio.common.logging import formatted_logger, setup_logging
-from rucio.common.utils import chunks, daemon_sleep
-from rucio.core import heartbeat
+from rucio.common.logging import setup_logging
+from rucio.common.utils import chunks
 from rucio.core.did import get_metadata
 from rucio.core.replica import (get_bad_pfns, get_pfn_to_rse, declare_bad_file_replicas,
                                 get_did_from_pfns, update_replicas_states, bulk_add_bad_replicas,
                                 bulk_delete_bad_pfns, get_replicas_state)
 from rucio.core.rse import get_rse_name
+from rucio.daemons.common import run_daemon
 from rucio.db.sqla.constants import BadFilesStatus, BadPFNStatus, ReplicaState
 from rucio.db.sqla.session import get_session
 
-GRACEFUL_STOP = threading.Event()
 
+if TYPE_CHECKING:
+    from rucio.daemons.common import HeartbeatHandler
 
-def minos(bulk=1000, once=False, sleep_time=60):
+graceful_stop = threading.Event()
+
+
+def minos(bulk: int = 1000, once: bool = False, sleep_time: int = 60) -> None:
     """
     Creates a Minos Worker that gets a list of bad PFNs,
     extract the scope, name and rse_id and fill the bad_replicas table.
@@ -49,24 +52,22 @@ def minos(bulk=1000, once=False, sleep_time=60):
     :param once: Run only once.
     :param sleep_time: Time between two cycles.
     """
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable='minos',
+        logger_prefix='minos',
+        partition_wait_time=10,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            bulk=bulk,
+        ),
+    )
 
-    executable = 'minos'
-    hostname = socket.getfqdn()
-    pid = os.getpid()
-    hb_thread = threading.current_thread()
-    heartbeat.sanity_check(executable=executable, hostname=hostname)
-    hb_thread = threading.current_thread()
-    heartbeat.sanity_check(executable=executable, hostname=hostname)
-    heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
-    prefix = 'minos[%i/%i] ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
-    logger = formatted_logger(logging.log, prefix + '%s')
-    logger(logging.INFO, 'Minos starting')
-
-    time.sleep(10)  # To prevent running on the same partition if all the daemons restart at the same time
-    heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
-    prefix = 'minos[%i/%i] ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
-    logger = formatted_logger(logging.log, prefix + '%s')
 
+def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> bool:
+    worker_number, total_workers, logger = heartbeat_handler.live()
     states_mapping = {BadPFNStatus.BAD: BadFilesStatus.BAD,
                       BadPFNStatus.SUSPICIOUS: BadFilesStatus.SUSPICIOUS,
                       BadPFNStatus.TEMPORARY_UNAVAILABLE: BadFilesStatus.TEMPORARY_UNAVAILABLE}
@@ -74,208 +75,195 @@ def minos(bulk=1000, once=False, sleep_time=60):
 
     chunk_size = 10  # The chunk size used for the commits
 
-    while not GRACEFUL_STOP.is_set():
-        start_time = time.time()
-        heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
-        prefix = 'minos[%i/%i] ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
-        logger = formatted_logger(logging.log, prefix + '%s')
-        pfns = []
-        try:
-            bad_replicas = {}
-            temporary_unvailables = {}
-            pfns = get_bad_pfns(thread=heart_beat['assign_thread'], total_threads=heart_beat['nr_threads'], limit=bulk)
+    pfns = []
 
-            # Class the PFNs into bad_replicas and temporary_unavailable
+    bad_replicas = {}
+    temporary_unvailables = {}
+    pfns = get_bad_pfns(thread=worker_number, total_threads=total_workers, limit=bulk)
+
+    # Class the PFNs into bad_replicas and temporary_unavailable
+    for pfn in pfns:
+        path = pfn['pfn']
+        account = pfn['account']
+        reason = pfn['reason']
+        expires_at = pfn['expires_at']
+        state = states_mapping[pfn['state']]
+        if state in [BadFilesStatus.BAD, BadFilesStatus.SUSPICIOUS]:
+            if (account, reason, state) not in bad_replicas:
+                bad_replicas[(account, reason, state)] = []
+            bad_replicas[(account, reason, state)].append(path)
+        elif state == BadFilesStatus.TEMPORARY_UNAVAILABLE:
+            if (account, reason, expires_at) not in temporary_unvailables:
+                temporary_unvailables[(account, reason, expires_at)] = []
+            temporary_unvailables[(account, reason, expires_at)].append(path)
+
+    # Process the bad and suspicious files
+    # The scope, name, rse_id are extracted and filled into the bad_replicas table
+    for account, reason, state in bad_replicas:
+        vo = account.vo
+        pfns = bad_replicas[(account, reason, state)]
+        logger(logging.INFO, 'Declaring %s replicas with state %s and reason %s' % (len(pfns), str(state), reason))
+        session = get_session()
+        schemes = {}
+        dict_rse = {}
+        unknown_replicas = []
+        try:
+            # Splitting the PFNs by schemes
             for pfn in pfns:
-                path = pfn['pfn']
-                account = pfn['account']
-                reason = pfn['reason']
-                expires_at = pfn['expires_at']
-                state = states_mapping[pfn['state']]
-                if state in [BadFilesStatus.BAD, BadFilesStatus.SUSPICIOUS]:
-                    if (account, reason, state) not in bad_replicas:
-                        bad_replicas[(account, reason, state)] = []
-                    bad_replicas[(account, reason, state)].append(path)
-                elif state == BadFilesStatus.TEMPORARY_UNAVAILABLE:
-                    if (account, reason, expires_at) not in temporary_unvailables:
-                        temporary_unvailables[(account, reason, expires_at)] = []
-                    temporary_unvailables[(account, reason, expires_at)].append(path)
+                scheme = pfn.split(':')[0]
+                if scheme not in schemes:
+                    schemes[scheme] = []
+                schemes[scheme].append(pfn)
+            for scheme in schemes:
+                _, tmp_dict_rse, tmp_unknown_replicas = get_pfn_to_rse(schemes[scheme], vo=vo)
+                for rse_id in tmp_dict_rse:
+                    if rse_id not in dict_rse:
+                        dict_rse[rse_id] = {}
+                    if scheme not in dict_rse[rse_id]:
+                        dict_rse[rse_id][scheme] = []
+                    dict_rse[rse_id][scheme].extend(tmp_dict_rse[rse_id])
+                unknown_replicas.extend(tmp_unknown_replicas.get('unknown', []))
+            # The replicas in unknown_replicas do not exist, so we flush them from bad_pfns
+            if unknown_replicas:
+                logger(logging.INFO, 'The following replicas are unknown and will be removed : %s' % str(unknown_replicas))
+                bulk_delete_bad_pfns(pfns=unknown_replicas, session=None)
 
-            # Process the bad and suspicious files
-            # The scope, name, rse_id are extracted and filled into the bad_replicas table
-            for account, reason, state in bad_replicas:
-                vo = account.vo
-                pfns = bad_replicas[(account, reason, state)]
-                logger(logging.INFO, 'Declaring %s replicas with state %s and reason %s' % (len(pfns), str(state), reason))
-                session = get_session()
-                schemes = {}
-                dict_rse = {}
-                unknown_replicas = []
-                try:
-                    # Splitting the PFNs by schemes
-                    for pfn in pfns:
-                        scheme = pfn.split(':')[0]
-                        if scheme not in schemes:
-                            schemes[scheme] = []
-                        schemes[scheme].append(pfn)
-                    for scheme in schemes:
-                        _, tmp_dict_rse, tmp_unknown_replicas = get_pfn_to_rse(schemes[scheme], vo=vo)
-                        for rse_id in tmp_dict_rse:
-                            if rse_id not in dict_rse:
-                                dict_rse[rse_id] = {}
-                            if scheme not in dict_rse[rse_id]:
-                                dict_rse[rse_id][scheme] = []
-                            dict_rse[rse_id][scheme].extend(tmp_dict_rse[rse_id])
-                        unknown_replicas.extend(tmp_unknown_replicas.get('unknown', []))
-                    # The replicas in unknown_replicas do not exist, so we flush them from bad_pfns
-                    if unknown_replicas:
-                        logger(logging.INFO, 'The following replicas are unknown and will be removed : %s' % str(unknown_replicas))
-                        bulk_delete_bad_pfns(pfns=unknown_replicas, session=None)
+            for rse_id, pfns_by_scheme in dict_rse.items():
+                for scheme, pfns in pfns_by_scheme.items():
+                    vo_str = '' if vo == 'def' else ' on VO ' + vo
+                    logger(logging.DEBUG, 'Running on RSE %s%s with %s replicas' % (get_rse_name(rse_id=rse_id), vo_str, len(pfns)))
+                    nchunk = 0
+                    tot_chunk = int(math.ceil(len(pfns) / chunk_size))
+                    for chunk in chunks(pfns, chunk_size):
+                        nchunk += 1
+                        logger(logging.DEBUG, 'Running on %s chunk out of %s' % (nchunk, tot_chunk))
+                        unknown_replicas = declare_bad_file_replicas(pfns=chunk, reason=reason, issuer=account, status=state, session=session)
+                        if unknown_replicas:
+                            logger(logging.DEBUG, 'Unknown replicas : %s' % (str(unknown_replicas)))
+                        bulk_delete_bad_pfns(pfns=chunk, session=session)
+                        session.commit()  # pylint: disable=no-member
+        except (DatabaseException, DatabaseError) as error:
+            if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
+                logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
+            else:
+                logger(logging.ERROR, 'Exception', exc_info=True)
+            session.rollback()  # pylint: disable=no-member
+        except Exception:
+            session.rollback()  # pylint: disable=no-member
+            logger(logging.CRITICAL, 'Exception', exc_info=True)
+
+    worker_number, total_workers, logger = heartbeat_handler.live()
+
+    # Now get the temporary unavailable and update the replicas states
+    for account, reason, expires_at in temporary_unvailables:
+        vo = account.vo
+        pfns = temporary_unvailables[(account, reason, expires_at)]
+        logger(logging.INFO, 'Declaring %s replicas temporary available with timeout %s and reason %s' % (len(pfns), str(expires_at), reason))
+        logger(logging.DEBUG, 'Extracting RSEs')
+        schemes = {}
+        dict_rse = {}
+        unknown_replicas = []
+
+        # Splitting the PFNs by schemes
+        for pfn in pfns:
+            scheme = pfn.split(':')[0]
+            if scheme not in schemes:
+                schemes[scheme] = []
+            schemes[scheme].append(pfn)
+        for scheme in schemes:
+            _, tmp_dict_rse, tmp_unknown_replicas = get_pfn_to_rse(schemes[scheme], vo=vo)
+            for rse_id in tmp_dict_rse:
+                if rse_id not in dict_rse:
+                    dict_rse[rse_id] = []
+                dict_rse[rse_id].extend(tmp_dict_rse[rse_id])
+                unknown_replicas.extend(tmp_unknown_replicas.get('unknown', []))
 
-                    for rse_id, pfns_by_scheme in dict_rse.items():
-                        for scheme, pfns in pfns_by_scheme.items():
-                            vo_str = '' if vo == 'def' else ' on VO ' + vo
-                            logger(logging.DEBUG, 'Running on RSE %s%s with %s replicas' % (get_rse_name(rse_id=rse_id), vo_str, len(pfns)))
-                            nchunk = 0
-                            tot_chunk = int(math.ceil(len(pfns) / chunk_size))
-                            for chunk in chunks(pfns, chunk_size):
-                                nchunk += 1
-                                logger(logging.DEBUG, 'Running on %s chunk out of %s' % (nchunk, tot_chunk))
-                                unknown_replicas = declare_bad_file_replicas(pfns=chunk, reason=reason, issuer=account, status=state, session=session)
-                                if unknown_replicas:
-                                    logger(logging.DEBUG, 'Unknown replicas : %s' % (str(unknown_replicas)))
-                                bulk_delete_bad_pfns(pfns=chunk, session=session)
-                                session.commit()  # pylint: disable=no-member
+        # The replicas in unknown_replicas do not exist, so we flush them from bad_pfns
+        if unknown_replicas:
+            logger(logging.INFO, 'The following replicas are unknown and will be removed : %s' % str(unknown_replicas))
+            bulk_delete_bad_pfns(pfns=unknown_replicas, session=None)
+
+        for rse_id in dict_rse:
+            replicas = []
+            rse = get_rse_name(rse_id=rse_id, session=None)
+            rse_vo_str = rse if vo == 'def' else '{} on {}'.format(rse, vo)
+            logger(logging.DEBUG, 'Running on RSE %s' % rse_vo_str)
+            for rep in get_did_from_pfns(pfns=dict_rse[rse_id], rse_id=None, vo=vo, session=None):
+                for pfn in rep:
+                    scope = rep[pfn]['scope']
+                    name = rep[pfn]['name']
+                    replicas.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'state': ReplicaState.TEMPORARY_UNAVAILABLE, 'pfn': pfn})
+            # The following part needs to be atomic
+            # We update the replicas states to TEMPORARY_UNAVAILABLE
+            # then insert a row in the bad_replicas table. TODO Update the row if it already exists
+            # then delete the corresponding rows into the bad_pfns table
+            logger(logging.DEBUG, 'Running on %s replicas on RSE %s' % (len(replicas), rse_vo_str))
+            nchunk = 0
+            tot_chunk = int(math.ceil(len(replicas) / float(chunk_size)))
+            session = get_session()
+            for chunk in chunks(replicas, chunk_size):
+                try:
+                    nchunk += 1
+                    logger(logging.DEBUG, 'Running on %s chunk out of %s' % (nchunk, tot_chunk))
+                    update_replicas_states(chunk, nowait=False, session=session)
+                    bulk_add_bad_replicas(chunk, account, state=BadFilesStatus.TEMPORARY_UNAVAILABLE, reason=reason, expires_at=expires_at, session=session)
+                    pfns = [entry['pfn'] for entry in chunk]
+                    bulk_delete_bad_pfns(pfns=pfns, session=session)
+                    session.commit()  # pylint: disable=no-member
+                except (UnsupportedOperation, ReplicaNotFound) as error:
+                    session.rollback()  # pylint: disable=no-member
+                    logger(logging.ERROR, 'Problem to bulk update PFNs. PFNs will be updated individually. Error : %s' % str(error))
+                    for rep in chunk:
+                        logger(logging.DEBUG, 'Working on %s' % (str(rep)))
+                        try:
+                            get_metadata(rep['scope'], rep['name'])
+                            unavailable_states = []
+                            rep_state = get_replicas_state(rep['scope'], rep['name'])
+                            unavailable_states.extend(rep_state.get(ReplicaState.TEMPORARY_UNAVAILABLE, []))
+                            unavailable_states.extend(rep_state.get(ReplicaState.BEING_DELETED, []))
+                            unavailable_states.extend(rep_state.get(ReplicaState.BAD, []))
+                            # If the replica is already not available, it is removed from the bad PFNs table
+                            if rep['rse_id'] in unavailable_states:
+                                logger(logging.INFO, '%s is in unavailable state. Will be removed from the list of bad PFNs' % str(rep['pfn']))
+                                bulk_delete_bad_pfns(pfns=[rep['pfn']], session=None)
+                            # If the expiration date of the TEMPORARY_UNAVAILABLE is in the past, it is removed from the bad PFNs table
+                            elif expires_at < datetime.now():
+                                logger(logging.INFO, 'PFN %s expiration time (%s) is older than now and is not in unavailable state. Removing the PFNs from bad_pfns' % (str(rep['pfn']), expires_at))
+                                bulk_delete_bad_pfns(pfns=[rep['pfn']], session=None)
+                            # Else update everything in the same transaction
+                            else:
+                                try:
+                                    session = get_session()
+                                    update_replicas_states([rep], nowait=False, session=session)
+                                    bulk_add_bad_replicas([rep], account, state=BadFilesStatus.TEMPORARY_UNAVAILABLE, reason=reason, expires_at=expires_at, session=session)
+                                    bulk_delete_bad_pfns(pfns=[rep['pfn']], session=session)
+                                    session.commit()  # pylint: disable=no-member
+                                except Exception:
+                                    logger(logging.ERROR, 'Cannot update state of %s' % str(rep['pfn']))
+                        except (DataIdentifierNotFound, ReplicaNotFound):
+                            logger(logging.ERROR, 'Will remove %s from the list of bad PFNs' % str(rep['pfn']))
+                            bulk_delete_bad_pfns(pfns=[rep['pfn']], session=None)
+                    session = get_session()
                 except (DatabaseException, DatabaseError) as error:
                     if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
                         logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
                     else:
                         logger(logging.ERROR, 'Exception', exc_info=True)
                     session.rollback()  # pylint: disable=no-member
+                    session = get_session()
                 except Exception:
                     session.rollback()  # pylint: disable=no-member
                     logger(logging.CRITICAL, 'Exception', exc_info=True)
-
-            heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
-
-            # Now get the temporary unavailable and update the replicas states
-            for account, reason, expires_at in temporary_unvailables:
-                vo = account.vo
-                pfns = temporary_unvailables[(account, reason, expires_at)]
-                logger(logging.INFO, 'Declaring %s replicas temporary available with timeout %s and reason %s' % (len(pfns), str(expires_at), reason))
-                logger(logging.DEBUG, 'Extracting RSEs')
-                schemes = {}
-                dict_rse = {}
-                unknown_replicas = []
-
-                # Splitting the PFNs by schemes
-                for pfn in pfns:
-                    scheme = pfn.split(':')[0]
-                    if scheme not in schemes:
-                        schemes[scheme] = []
-                    schemes[scheme].append(pfn)
-                for scheme in schemes:
-                    _, tmp_dict_rse, tmp_unknown_replicas = get_pfn_to_rse(schemes[scheme], vo=vo)
-                    for rse_id in tmp_dict_rse:
-                        if rse_id not in dict_rse:
-                            dict_rse[rse_id] = []
-                        dict_rse[rse_id].extend(tmp_dict_rse[rse_id])
-                        unknown_replicas.extend(tmp_unknown_replicas.get('unknown', []))
-
-                # The replicas in unknown_replicas do not exist, so we flush them from bad_pfns
-                if unknown_replicas:
-                    logger(logging.INFO, 'The following replicas are unknown and will be removed : %s' % str(unknown_replicas))
-                    bulk_delete_bad_pfns(pfns=unknown_replicas, session=None)
-
-                for rse_id in dict_rse:
-                    replicas = []
-                    rse = get_rse_name(rse_id=rse_id, session=None)
-                    rse_vo_str = rse if vo == 'def' else '{} on {}'.format(rse, vo)
-                    logger(logging.DEBUG, 'Running on RSE %s' % rse_vo_str)
-                    for rep in get_did_from_pfns(pfns=dict_rse[rse_id], rse_id=None, vo=vo, session=None):
-                        for pfn in rep:
-                            scope = rep[pfn]['scope']
-                            name = rep[pfn]['name']
-                            replicas.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'state': ReplicaState.TEMPORARY_UNAVAILABLE, 'pfn': pfn})
-                    # The following part needs to be atomic
-                    # We update the replicas states to TEMPORARY_UNAVAILABLE
-                    # then insert a row in the bad_replicas table. TODO Update the row if it already exists
-                    # then delete the corresponding rows into the bad_pfns table
-                    logger(logging.DEBUG, 'Running on %s replicas on RSE %s' % (len(replicas), rse_vo_str))
-                    nchunk = 0
-                    tot_chunk = int(math.ceil(len(replicas) / float(chunk_size)))
                     session = get_session()
-                    for chunk in chunks(replicas, chunk_size):
-                        try:
-                            nchunk += 1
-                            logger(logging.DEBUG, 'Running on %s chunk out of %s' % (nchunk, tot_chunk))
-                            update_replicas_states(chunk, nowait=False, session=session)
-                            bulk_add_bad_replicas(chunk, account, state=BadFilesStatus.TEMPORARY_UNAVAILABLE, reason=reason, expires_at=expires_at, session=session)
-                            pfns = [entry['pfn'] for entry in chunk]
-                            bulk_delete_bad_pfns(pfns=pfns, session=session)
-                            session.commit()  # pylint: disable=no-member
-                        except (UnsupportedOperation, ReplicaNotFound) as error:
-                            session.rollback()  # pylint: disable=no-member
-                            logger(logging.ERROR, 'Problem to bulk update PFNs. PFNs will be updated individually. Error : %s' % str(error))
-                            for rep in chunk:
-                                logger(logging.DEBUG, 'Working on %s' % (str(rep)))
-                                try:
-                                    get_metadata(rep['scope'], rep['name'])
-                                    unavailable_states = []
-                                    rep_state = get_replicas_state(rep['scope'], rep['name'])
-                                    unavailable_states.extend(rep_state.get(ReplicaState.TEMPORARY_UNAVAILABLE, []))
-                                    unavailable_states.extend(rep_state.get(ReplicaState.BEING_DELETED, []))
-                                    unavailable_states.extend(rep_state.get(ReplicaState.BAD, []))
-                                    # If the replica is already not available, it is removed from the bad PFNs table
-                                    if rep['rse_id'] in unavailable_states:
-                                        logger(logging.INFO, '%s is in unavailable state. Will be removed from the list of bad PFNs' % str(rep['pfn']))
-                                        bulk_delete_bad_pfns(pfns=[rep['pfn']], session=None)
-                                    # If the expiration date of the TEMPORARY_UNAVAILABLE is in the past, it is removed from the bad PFNs table
-                                    elif expires_at < datetime.now():
-                                        logger(logging.INFO, 'PFN %s expiration time (%s) is older than now and is not in unavailable state. Removing the PFNs from bad_pfns' % (str(rep['pfn']), expires_at))
-                                        bulk_delete_bad_pfns(pfns=[rep['pfn']], session=None)
-                                    # Else update everything in the same transaction
-                                    else:
-                                        try:
-                                            session = get_session()
-                                            update_replicas_states([rep], nowait=False, session=session)
-                                            bulk_add_bad_replicas([rep], account, state=BadFilesStatus.TEMPORARY_UNAVAILABLE, reason=reason, expires_at=expires_at, session=session)
-                                            bulk_delete_bad_pfns(pfns=[rep['pfn']], session=session)
-                                            session.commit()  # pylint: disable=no-member
-                                        except Exception:
-                                            logger(logging.ERROR, 'Cannot update state of %s' % str(rep['pfn']))
-                                except (DataIdentifierNotFound, ReplicaNotFound):
-                                    logger(logging.ERROR, 'Will remove %s from the list of bad PFNs' % str(rep['pfn']))
-                                    bulk_delete_bad_pfns(pfns=[rep['pfn']], session=None)
-                            session = get_session()
-                        except (DatabaseException, DatabaseError) as error:
-                            if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
-                                logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
-                            else:
-                                logger(logging.ERROR, 'Exception', exc_info=True)
-                            session.rollback()  # pylint: disable=no-member
-                            session = get_session()
-                        except Exception:
-                            session.rollback()  # pylint: disable=no-member
-                            logger(logging.CRITICAL, 'Exception', exc_info=True)
-                            session = get_session()
-
-        except Exception as error:
-            logger(logging.ERROR, '%s' % (str(error)))
-
-        if once:
-            break
-        if len(pfns) == bulk:
-            logger(logging.INFO, 'Processed maximum number of pfns according to the bulk size. Restart immediately next cycle')
-        else:
-            daemon_sleep(start_time=start_time, sleep_time=sleep_time, graceful_stop=GRACEFUL_STOP, logger=logger)
-
-    heartbeat.die(executable, hostname, pid, hb_thread)
-    logger(logging.INFO, 'Graceful stop requested')
-    logger(logging.INFO, 'Graceful stop done')
+    must_sleep = True
+    if len(pfns) == bulk:
+        logger(logging.INFO, 'Processed maximum number of pfns according to the bulk size. Restart immediately next cycle')
+        must_sleep = False
+        return must_sleep
+    return must_sleep
 
 
-def run(threads=1, bulk=100, once=False, sleep_time=60):
+def run(threads: int = 1, bulk: int = 100, once: bool = False, sleep_time: int = 60) -> None:
     """
     Starts up the minos threads.
     """
@@ -303,4 +291,4 @@ def stop(signum=None, frame=None):
     """
     Graceful exit.
     """
-    GRACEFUL_STOP.set()
+    graceful_stop.set()
diff --git a/lib/rucio/daemons/badreplicas/minos_temporary_expiration.py b/lib/rucio/daemons/badreplicas/minos_temporary_expiration.py
--- a/lib/rucio/daemons/badreplicas/minos_temporary_expiration.py
+++ b/lib/rucio/daemons/badreplicas/minos_temporary_expiration.py
@@ -13,34 +13,36 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
 import logging
 import math
-import os
 import re
-import socket
 import threading
-import time
 import traceback
+from typing import TYPE_CHECKING
 
 from sqlalchemy.exc import DatabaseError
 
 import rucio.db.sqla.util
 from rucio.common import exception
 from rucio.common.exception import DataIdentifierNotFound, ReplicaNotFound, DatabaseException
-from rucio.common.logging import formatted_logger, setup_logging
-from rucio.common.utils import chunks, daemon_sleep
-from rucio.core import heartbeat
+from rucio.common.logging import setup_logging
+from rucio.common.utils import chunks
 from rucio.core.did import get_metadata
 from rucio.core.replica import (update_replicas_states, get_replicas_state,
                                 bulk_delete_bad_replicas, list_expired_temporary_unavailable_replicas)
+from rucio.daemons.common import run_daemon
 from rucio.db.sqla.constants import BadFilesStatus, ReplicaState
 from rucio.db.sqla.session import get_session
 
 
+if TYPE_CHECKING:
+    from rucio.daemons.common import HeartbeatHandler
+
 graceful_stop = threading.Event()
 
 
-def minos_tu_expiration(bulk=1000, once=False, sleep_time=60):
+def minos_tu_expiration(bulk: int = 1000, once: bool = False, sleep_time: int = 60) -> None:
     """
     Creates a Minos Temporary Unavailable Replicas Expiration Worker that
     gets the list of expired TU replicas and sets them back to AVAILABLE.
@@ -49,111 +51,96 @@ def minos_tu_expiration(bulk=1000, once=False, sleep_time=60):
     :param once: Run only once.
     :param sleep_time: Time between two cycles.
     """
-
-    executable = 'minos-temporary-expiration'
-    hostname = socket.getfqdn()
-    pid = os.getpid()
-    hb_thread = threading.current_thread()
-    heartbeat.sanity_check(executable=executable, hostname=hostname)
-    hb_thread = threading.current_thread()
-    heartbeat.sanity_check(executable=executable, hostname=hostname)
-    heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
-    prefix = 'minos_temporary_expiration[%i/%i] : ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
-    logger = formatted_logger(logging.log, prefix + '%s')
-    logger(logging.INFO, 'Minos Temporary Expiration starting')
-
-    time.sleep(10)  # To prevent running on the same partition if all the daemons restart at the same time
-    heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
-    logger(logging.INFO, 'Minos Temporary Expiration started')
-
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable='minos-temporary-expiration',
+        logger_prefix='minos-temporary-expiration',
+        partition_wait_time=10,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            bulk=bulk,
+        ),
+    )
+
+
+def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> bool:
+
+    worker_number, total_workers, logger = heartbeat_handler.live()
     chunk_size = 10  # The chunk size used for the commits
-
-    while not graceful_stop.is_set():
-        start_time = time.time()
-        heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
+    logger(logging.INFO, 'Getting list of expired replicas')
+    expired_replicas = list_expired_temporary_unavailable_replicas(total_workers=total_workers,
+                                                                   worker_number=worker_number,
+                                                                   limit=1000)
+    logger(logging.INFO, '%s expired replicas returned', len(expired_replicas))
+    logger(logging.DEBUG, 'List of expired replicas returned %s', str(expired_replicas))
+    replicas = []
+    bad_replicas = []
+    nchunk = 0
+    tot_chunk = int(math.ceil(len(expired_replicas) / float(chunk_size)))
+    session = get_session()
+    for chunk in chunks(expired_replicas, chunk_size):
+        worker_number, total_workers, logger = heartbeat_handler.live()
+        skip_replica_update = []
+        # Process and update the replicas in chunks
+        for replica in chunk:
+            scope, name, rse_id = replica[0], replica[1], replica[2]
+            states_dictionary = get_replicas_state(scope=scope, name=name, session=session)
+            # Check if the replica is not declared bad
+            # If already declared bad don't update the replica state, but remove from bad_pfns
+            if not (ReplicaState.BAD in states_dictionary and rse_id in states_dictionary[ReplicaState.BAD]):
+                replicas.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'state': ReplicaState.AVAILABLE})
+            else:
+                skip_replica_update.append((scope, name))
+            # Remove the replicas from bad_replicas table in chunks
+            bad_replicas.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'state': BadFilesStatus.TEMPORARY_UNAVAILABLE})
         try:
-            # Get list of expired TU replicas
-            logger(logging.INFO, 'Getting list of expired replicas')
-            expired_replicas = list_expired_temporary_unavailable_replicas(total_workers=heart_beat['nr_threads'],
-                                                                           worker_number=heart_beat['assign_thread'],
-                                                                           limit=1000)
-            logger(logging.INFO, '%s expired replicas returned', len(expired_replicas))
-            logger(logging.DEBUG, 'List of expired replicas returned %s', str(expired_replicas))
-            replicas = []
-            bad_replicas = []
-            nchunk = 0
-            tot_chunk = int(math.ceil(len(expired_replicas) / float(chunk_size)))
-            session = get_session()
-            for chunk in chunks(expired_replicas, chunk_size):
-                heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
-                skip_replica_update = []
-                # Process and update the replicas in chunks
-                for replica in chunk:
-                    scope, name, rse_id = replica[0], replica[1], replica[2]
-                    states_dictionary = get_replicas_state(scope=scope, name=name, session=session)
-                    # Check if the replica is not declared bad
-                    # If already declared bad don't update the replica state, but remove from bad_pfns
-                    if not (ReplicaState.BAD in states_dictionary and rse_id in states_dictionary[ReplicaState.BAD]):
-                        replicas.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'state': ReplicaState.AVAILABLE})
-                    else:
-                        skip_replica_update.append((scope, name))
-                    # Remove the replicas from bad_replicas table in chunks
-                    bad_replicas.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'state': BadFilesStatus.TEMPORARY_UNAVAILABLE})
+            nchunk += 1
+            logger(logging.DEBUG, 'Running on %s chunk out of %s', nchunk, tot_chunk)
+            update_replicas_states(replicas, nowait=True, session=session)
+            bulk_delete_bad_replicas(bad_replicas, session=session)
+            session.commit()  # pylint: disable=no-member
+        except (ReplicaNotFound, DataIdentifierNotFound) as error:
+            session.rollback()  # pylint: disable=no-member
+            logger(logging.WARNING, 'One of the replicas does not exist anymore. Updating and deleting one by one. Error : %s', str(error))
+            for replica in chunk:
+                _, _, logger = heartbeat_handler.live()
+                scope, name, rse_id = replica[0], replica[1], replica[2]
+                logger(logging.DEBUG, 'Working on %s:%s on %s', scope, name, rse_id)
                 try:
-                    nchunk += 1
-                    logger(logging.DEBUG, 'Running on %s chunk out of %s', nchunk, tot_chunk)
-                    update_replicas_states(replicas, nowait=True, session=session)
-                    bulk_delete_bad_replicas(bad_replicas, session=session)
+                    # First check if the DID exists
+                    get_metadata(scope, name)
+                    if (scope, name) not in skip_replica_update:
+                        update_replicas_states([{'scope': scope, 'name': name, 'rse_id': rse_id, 'state': ReplicaState.AVAILABLE}, ], nowait=True, session=session)
+                    bulk_delete_bad_replicas([{'scope': scope, 'name': name, 'rse_id': rse_id, 'state': BadFilesStatus.TEMPORARY_UNAVAILABLE}, ], session=session)
                     session.commit()  # pylint: disable=no-member
-                except (ReplicaNotFound, DataIdentifierNotFound) as error:
+                except DataIdentifierNotFound:
                     session.rollback()  # pylint: disable=no-member
-                    logger(logging.WARNING, 'One of the replicas does not exist anymore. Updating and deleting one by one. Error : %s', str(error))
-                    for replica in chunk:
-                        scope, name, rse_id = replica[0], replica[1], replica[2]
-                        logger(logging.DEBUG, 'Working on %s:%s on %s', scope, name, rse_id)
-                        try:
-                            # First check if the DID exists
-                            get_metadata(scope, name)
-                            if (scope, name) not in skip_replica_update:
-                                update_replicas_states([{'scope': scope, 'name': name, 'rse_id': rse_id, 'state': ReplicaState.AVAILABLE}, ], nowait=True, session=session)
-                            bulk_delete_bad_replicas([{'scope': scope, 'name': name, 'rse_id': rse_id, 'state': BadFilesStatus.TEMPORARY_UNAVAILABLE}, ], session=session)
-                            session.commit()  # pylint: disable=no-member
-                        except DataIdentifierNotFound:
-                            session.rollback()  # pylint: disable=no-member
-                            logger(logging.WARNING, 'DID %s:%s does not exist anymore.', scope, name)
-                            bulk_delete_bad_replicas([{'scope': scope, 'name': name, 'rse_id': rse_id, 'state': BadFilesStatus.TEMPORARY_UNAVAILABLE}, ], session=session)
-                            session.commit()  # pylint: disable=no-member
-                        except ReplicaNotFound:
-                            session.rollback()  # pylint: disable=no-member
-                            logger(logging.WARNING, 'Replica %s:%s on RSEID %s does not exist anymore.', scope, name, rse_id)
-                            bulk_delete_bad_replicas([{'scope': scope, 'name': name, 'rse_id': rse_id, 'state': BadFilesStatus.TEMPORARY_UNAVAILABLE}, ], session=session)
-                            session.commit()  # pylint: disable=no-member
-                    session = get_session()
-                except (DatabaseException, DatabaseError) as error:
-                    if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
-                        logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
-                    else:
-                        logger(logging.ERROR, 'Exception', exc_info=True)
-                    session.rollback()
-                    session = get_session()
-                except Exception:
+                    logger(logging.WARNING, 'DID %s:%s does not exist anymore.', scope, name)
+                    bulk_delete_bad_replicas([{'scope': scope, 'name': name, 'rse_id': rse_id, 'state': BadFilesStatus.TEMPORARY_UNAVAILABLE}, ], session=session)
+                    session.commit()  # pylint: disable=no-member
+                except ReplicaNotFound:
                     session.rollback()  # pylint: disable=no-member
-                    logger(logging.CRITICAL, str(traceback.format_exc()))
-                    session = get_session()
-
+                    logger(logging.WARNING, 'Replica %s:%s on RSEID %s does not exist anymore.', scope, name, rse_id)
+                    bulk_delete_bad_replicas([{'scope': scope, 'name': name, 'rse_id': rse_id, 'state': BadFilesStatus.TEMPORARY_UNAVAILABLE}, ], session=session)
+                    session.commit()  # pylint: disable=no-member
+            session = get_session()
+        except (DatabaseException, DatabaseError) as error:
+            if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
+                logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
+            else:
+                logger(logging.ERROR, 'Exception', exc_info=True)
+            session.rollback()
+            session = get_session()
         except Exception:
+            session.rollback()  # pylint: disable=no-member
             logger(logging.CRITICAL, str(traceback.format_exc()))
-
-        if once:
-            break
-        daemon_sleep(start_time=start_time, sleep_time=sleep_time, graceful_stop=graceful_stop)
-
-    heartbeat.die(executable, hostname, pid, hb_thread)
-    logger(logging.INFO, 'Graceful stop requested')
-    logger(logging.INFO, 'Graceful stop done')
+            session = get_session()
+    return True
 
 
-def run(threads=1, bulk=100, once=False, sleep_time=60):
+def run(threads: int = 1, bulk: int = 100, once: bool = False, sleep_time: int = 60) -> None:
     """
     Starts up the minos threads.
     """
diff --git a/lib/rucio/daemons/badreplicas/necromancer.py b/lib/rucio/daemons/badreplicas/necromancer.py
--- a/lib/rucio/daemons/badreplicas/necromancer.py
+++ b/lib/rucio/daemons/badreplicas/necromancer.py
@@ -13,177 +13,158 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
 import logging
-import os
 import re
-import socket
 import threading
 import time
+from typing import TYPE_CHECKING
 from datetime import datetime, timedelta
-from sys import exc_info
-from traceback import format_exception
-
-from configparser import NoOptionError, NoSectionError
 
 from sqlalchemy.exc import DatabaseError
 from dogpile.cache.api import NO_VALUE
 
 import rucio.db.sqla.util
+from rucio.db.sqla.constants import ReplicaState
 from rucio.common import exception
 from rucio.common.cache import make_region_memcached
-from rucio.common.config import config_get, config_get_int
+from rucio.common.config import config_get_int
 from rucio.common.exception import DatabaseException
-from rucio.common.logging import formatted_logger, setup_logging
-from rucio.common.utils import daemon_sleep
-from rucio.core import monitor, heartbeat
+from rucio.common.logging import setup_logging
+from rucio.core import monitor
 from rucio.core.replica import list_bad_replicas, get_replicas_state, get_bad_replicas_backlog
 from rucio.core.rule import (update_rules_for_lost_replica, update_rules_for_bad_replica,
                              get_evaluation_backlog)
-from rucio.db.sqla.constants import ReplicaState
+from rucio.daemons.common import run_daemon
 
-GRACEFUL_STOP = threading.Event()
+if TYPE_CHECKING:
+    from rucio.daemons.common import HeartbeatHandler
+
+graceful_stop = threading.Event()
 REGION = make_region_memcached(expiration_time=config_get_int('necromancer', 'cache_time', False, 600))
 
 
-def necromancer(thread=0, bulk=5, once=False, sleep_time=60):
+def necromancer(bulk: int, once: bool = False, sleep_time: int = 60) -> None:
     """
     Creates a Necromancer Worker that gets a list of bad replicas for a given hash,
     identify lost DIDs and for non-lost ones, set the locks and rules for reevaluation.
 
-    :param thread: Thread number at startup.
     :param bulk: The number of requests to process.
     :param once: Run only once.
     :param sleep_time: Thread sleep time after each chunk of work.
     """
+    run_daemon(
+        once=once,
+        graceful_stop=graceful_stop,
+        executable='necromancer',
+        logger_prefix='necromancer',
+        partition_wait_time=10,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            bulk=bulk,
+        ),
+    )
+
+
+def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> bool:
+    worker_number, total_workers, logger = heartbeat_handler.live()
+    logger(logging.INFO, 'Graceful stop done')
+    must_sleep = True
+
+    # Check if there is a Judge Evaluator backlog
+    max_evaluator_backlog_count = config_get_int('necromancer', 'max_evaluator_backlog_count', default=0, raise_exception=False)
+    max_evaluator_backlog_duration = config_get_int('necromancer', 'max_evaluator_backlog_duration', default=0, raise_exception=False)
+    backlog_refresh_time = config_get_int('necromancer', 'backlog_refresh_time', default=60, raise_exception=False)
+    if max_evaluator_backlog_count or max_evaluator_backlog_duration:
+        evaluator_backlog_count, evaluator_backlog_duration = get_evaluation_backlog(expiration_time=backlog_refresh_time)
+        if max_evaluator_backlog_count and \
+           evaluator_backlog_count and \
+           max_evaluator_backlog_duration and \
+           evaluator_backlog_duration and \
+           evaluator_backlog_count > max_evaluator_backlog_count and \
+           evaluator_backlog_duration < datetime.utcnow() - timedelta(minutes=max_evaluator_backlog_duration):
+            logger(logging.ERROR, 'Necromancer: Judge evaluator backlog count and duration hit, stopping operation')
+            return must_sleep
+        elif max_evaluator_backlog_count and evaluator_backlog_count and evaluator_backlog_count > max_evaluator_backlog_count:
+            logger(logging.ERROR, 'Necromancer: Judge evaluator backlog count hit, stopping operation')
+            return must_sleep
+        elif max_evaluator_backlog_duration and evaluator_backlog_duration and evaluator_backlog_duration < datetime.utcnow() - timedelta(minutes=max_evaluator_backlog_duration):
+            logger(logging.ERROR, 'Necromancer: Judge evaluator backlog duration hit, stopping operation')
+            return must_sleep
+
+    # Check how many bad replicas are queued
+    max_bad_replicas_backlog_count = config_get_int('necromancer', 'max_bad_replicas_backlog_count', default=0, raise_exception=False)
+    bad_replicas_backlog = REGION.get('bad_replicas_backlog')
+    if bad_replicas_backlog is NO_VALUE:
+        bad_replicas_backlog = get_bad_replicas_backlog()
+        REGION.set('bad_replicas_backlog', bad_replicas_backlog)
+    tot_bad_files = sum([bad_replicas_backlog[key] for key in bad_replicas_backlog])
+    list_of_rses = list()
+    # If too many replica, call list_bad_replicas with a list of RSEs
+    if max_bad_replicas_backlog_count and tot_bad_files > max_bad_replicas_backlog_count and len(bad_replicas_backlog) > 1:
+        logger(logging.INFO, 'Backlog of bads replica too big. Apply some sharing between different RSEs')
+        rses = list()
+        cnt = 0
+        for key in sorted(bad_replicas_backlog, key=bad_replicas_backlog.get, reverse=True):
+            rses.append({'id': key})
+            cnt += bad_replicas_backlog[key]
+            if cnt >= bulk:
+                list_of_rses.append(rses)
+                rses = list()
+                cnt = 0
+    else:
+        list_of_rses.append(None)
 
-    executable = 'necromancer'
-    hostname = socket.getfqdn()
-    pid = os.getpid()
-    hb_thread = threading.current_thread()
-    heartbeat.sanity_check(executable=executable, hostname=hostname)
-
-    while not GRACEFUL_STOP.is_set():
-
-        stime = time.time()
-        heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
-        prefix = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
-        logger = formatted_logger(logging.log, prefix + '%s')
-        logger(logging.DEBUG, 'Starting new cycle')
-
-        # Check if there is a Judge Evaluator backlog
-        try:
-            max_evaluator_backlog_count = config_get('necromancer', 'max_evaluator_backlog_count')
-            max_evaluator_backlog_count = int(max_evaluator_backlog_count)
-        except (NoOptionError, NoSectionError, RuntimeError, ValueError):
-            max_evaluator_backlog_count = None
-        try:
-            max_evaluator_backlog_duration = config_get('necromancer', 'max_evaluator_backlog_duration')
-            max_evaluator_backlog_duration = int(max_evaluator_backlog_duration)
-        except (NoOptionError, NoSectionError, RuntimeError, ValueError):
-            max_evaluator_backlog_duration = None
-        if max_evaluator_backlog_count or max_evaluator_backlog_duration:
-            evaluator_backlog_count, evaluator_backlog_duration = get_evaluation_backlog(expiration_time=sleep_time)
-            if max_evaluator_backlog_count and \
-               evaluator_backlog_count and \
-               max_evaluator_backlog_duration and \
-               evaluator_backlog_duration and \
-               evaluator_backlog_count > max_evaluator_backlog_count and \
-               evaluator_backlog_duration < datetime.utcnow() - timedelta(minutes=max_evaluator_backlog_duration):
-                logger(logging.ERROR, 'Necromancer: Judge evaluator backlog count and duration hit, stopping operation')
-                GRACEFUL_STOP.wait(30)
-                continue
-            elif max_evaluator_backlog_count and evaluator_backlog_count and evaluator_backlog_count > max_evaluator_backlog_count:
-                logger(logging.ERROR, 'Necromancer: Judge evaluator backlog count hit, stopping operation')
-                GRACEFUL_STOP.wait(30)
-                continue
-            elif max_evaluator_backlog_duration and evaluator_backlog_duration and evaluator_backlog_duration < datetime.utcnow() - timedelta(minutes=max_evaluator_backlog_duration):
-                logger(logging.ERROR, 'Necromancer: Judge evaluator backlog duration hit, stopping operation')
-                GRACEFUL_STOP.wait(30)
-                continue
-
-        # Check how many bad replicas are queued
-        try:
-            max_bad_replicas_backlog_count = config_get('necromancer', 'max_bad_replicas_backlog_count')
-            max_bad_replicas_backlog_count = int(max_bad_replicas_backlog_count)
-        except (NoOptionError, NoSectionError, RuntimeError, ValueError):
-            max_bad_replicas_backlog_count = None
-        bad_replicas_backlog = REGION.get('bad_replicas_backlog')
-        if bad_replicas_backlog is NO_VALUE:
-            bad_replicas_backlog = get_bad_replicas_backlog()
-            REGION.set('bad_replicas_backlog', bad_replicas_backlog)
-        tot_bad_files = sum([bad_replicas_backlog[key] for key in bad_replicas_backlog])
-        list_of_rses = list()
-        # If too many replica, call list_bad_replicas with a list of RSEs
-        if max_bad_replicas_backlog_count and tot_bad_files > max_bad_replicas_backlog_count and len(bad_replicas_backlog) > 1:
-            logger(logging.INFO, 'Backlog of bads replica too big. Apply some sharing between different RSEs')
-            rses = list()
-            cnt = 0
-            for key in sorted(bad_replicas_backlog, key=bad_replicas_backlog.get, reverse=True):
-                rses.append({'id': key})
-                cnt += bad_replicas_backlog[key]
-                if cnt >= bulk:
-                    list_of_rses.append(rses)
-                    rses = list()
-                    cnt = 0
-        else:
-            list_of_rses.append(None)
-
-        tot_processed = 0
-        if tot_bad_files == 0:
-            logger(logging.INFO, 'No bad replicas to process.')
-        else:
-            ttime = time.time()
-            replicas = []
-            try:
-                for rses in list_of_rses:
-
-                    replicas = list_bad_replicas(limit=bulk, thread=heart_beat['assign_thread'], total_threads=heart_beat['nr_threads'], rses=rses)
-                    for replica in replicas:
-                        scope, name, rse_id, rse = replica['scope'], replica['name'], replica['rse_id'], replica['rse']
-                        logger(logging.INFO, 'Working on %s:%s on %s' % (scope, name, rse))
-
-                        list_replicas = get_replicas_state(scope=scope, name=name)
-                        if ReplicaState.AVAILABLE not in list_replicas and ReplicaState.TEMPORARY_UNAVAILABLE not in list_replicas:
-                            logger(logging.INFO, 'File %s:%s has no other available or temporary available replicas, it will be marked as lost' % (scope, name))
-                            try:
-                                update_rules_for_lost_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
-                                monitor.record_counter(name='necromancer.badfiles.lostfile')
-                            except (DatabaseException, DatabaseError) as error:
-                                if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
-                                    logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
-                                else:
-                                    logger(logging.ERROR, str(error))
-
+    tot_processed = 0
+    if tot_bad_files == 0:
+        logger(logging.INFO, 'No bad replicas to process.')
+    else:
+        ttime = time.time()
+        replicas = []
+        for rses in list_of_rses:
+
+            worker_number, total_workers, logger = heartbeat_handler.live()
+            replicas = list_bad_replicas(limit=bulk, thread=worker_number, total_threads=total_workers, rses=rses)
+            for replica in replicas:
+                scope, name, rse_id, rse = replica['scope'], replica['name'], replica['rse_id'], replica['rse']
+                logger(logging.INFO, 'Working on %s:%s on %s' % (scope, name, rse))
+
+                list_replicas = get_replicas_state(scope=scope, name=name)
+                if ReplicaState.AVAILABLE not in list_replicas and ReplicaState.TEMPORARY_UNAVAILABLE not in list_replicas:
+                    logger(logging.INFO, 'File %s:%s has no other available or temporary available replicas, it will be marked as lost' % (scope, name))
+                    try:
+                        update_rules_for_lost_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
+                        monitor.record_counter(name='necromancer.badfiles.lostfile')
+                    except (DatabaseException, DatabaseError) as error:
+                        if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
+                            logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
                         else:
-                            rep = list_replicas.get(ReplicaState.AVAILABLE, [])
-                            unavailable_rep = list_replicas.get(ReplicaState.TEMPORARY_UNAVAILABLE, [])
-                            logger(logging.INFO, 'File %s:%s can be recovered. Available sources : %s + Unavailable sources : %s' % (scope, name, str(rep), str(unavailable_rep)))
-                            try:
-                                update_rules_for_bad_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
-                                monitor.record_counter(name='necromancer.badfiles.recovering')
-                            except (DatabaseException, DatabaseError) as error:
-                                if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
-                                    logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
-                                else:
-                                    logger(logging.ERROR, str(error))
-
-                    tot_processed += len(replicas)
-                    logger(logging.INFO, 'It took %s seconds to process %s replicas' % (str(time.time() - ttime), str(len(replicas))))
-            except Exception:
-                exc_type, exc_value, exc_traceback = exc_info()
-                logger(logging.CRITICAL, ''.join(format_exception(exc_type, exc_value, exc_traceback)).strip())
-
-        if once:
-            break
-        elif tot_processed == 0 or tot_bad_files == 0:
-            daemon_sleep(start_time=stime, sleep_time=sleep_time, graceful_stop=GRACEFUL_STOP, logger=logger)
-
-    logger(logging.INFO, 'Graceful stop requested')
-    heartbeat.die(executable, hostname, pid, hb_thread)
-    logger(logging.INFO, 'Graceful stop done')
+                            logger(logging.ERROR, str(error))
+
+                else:
+                    rep = list_replicas.get(ReplicaState.AVAILABLE, [])
+                    unavailable_rep = list_replicas.get(ReplicaState.TEMPORARY_UNAVAILABLE, [])
+                    logger(logging.INFO, 'File %s:%s can be recovered. Available sources : %s + Unavailable sources : %s' % (scope, name, str(rep), str(unavailable_rep)))
+                    try:
+                        update_rules_for_bad_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
+                        monitor.record_counter(name='necromancer.badfiles.recovering')
+                    except (DatabaseException, DatabaseError) as error:
+                        if re.match('.*ORA-00054.*', error.args[0]) or re.match('.*ORA-00060.*', error.args[0]) or 'ERROR 1205 (HY000)' in error.args[0]:
+                            logger(logging.WARNING, 'Lock detected when handling request - skipping: %s', str(error))
+                        else:
+                            logger(logging.ERROR, str(error))
+
+            tot_processed += len(replicas)
+            logger(logging.INFO, 'It took %s seconds to process %s replicas' % (str(time.time() - ttime), str(len(replicas))))
+
+    if tot_processed == 0 or tot_bad_files == 0:
+        return must_sleep
+    must_sleep = False
+    return must_sleep
 
 
-def run(threads=1, bulk=100, once=False, sleep_time=60):
+def run(threads: int = 1, bulk: int = 100, once: bool = False, sleep_time: int = 60) -> None:
     """
     Starts up the necromancer threads.
     """
@@ -198,9 +179,8 @@ def run(threads=1, bulk=100, once=False, sleep_time=60):
     else:
         logging.info('starting necromancer threads')
         thread_list = [threading.Thread(target=necromancer, kwargs={'once': once,
-                                                                    'thread': i,
                                                                     'bulk': bulk,
-                                                                    'sleep_time': sleep_time}) for i in range(0, threads)]
+                                                                    'sleep_time': sleep_time}) for _ in range(0, threads)]
         [t.start() for t in thread_list]
 
         logging.info('waiting for interrupts')
@@ -214,4 +194,4 @@ def stop(signum=None, frame=None):
     """
     Graceful exit.
     """
-    GRACEFUL_STOP.set()
+    graceful_stop.set()


Test patch:
diff --git a/lib/rucio/tests/test_replica.py b/lib/rucio/tests/test_replica.py
--- a/lib/rucio/tests/test_replica.py
+++ b/lib/rucio/tests/test_replica.py
@@ -36,8 +36,8 @@
                                 get_RSEcoverage_of_dataset, get_replica_atime,
                                 touch_replica, get_bad_pfns, set_tombstone, add_bad_dids)
 from rucio.core.rse import add_protocol, add_rse_attribute, del_rse_attribute
-from rucio.daemons.badreplicas.minos import run as minos_run
-from rucio.daemons.badreplicas.minos_temporary_expiration import run as minos_temp_run
+from rucio.daemons.badreplicas.minos import minos
+from rucio.daemons.badreplicas.minos_temporary_expiration import minos_tu_expiration
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import DIDType, ReplicaState, BadPFNStatus, OBSOLETE
 from rucio.db.sqla.session import transactional_session
@@ -823,7 +823,7 @@ def test_client_add_temporary_unavailable_pfns(rse_factory, mock_scope, replica_
         replica_client.add_bad_pfns(pfns=list_rep, reason=str(reason_str), state='BADSTATE', expires_at=now.isoformat())
 
     # Run minos once
-    minos_run(threads=1, bulk=10000, once=True)
+    minos(bulk=10000, once=True)
     result = get_bad_pfns(limit=10000, thread=None, total_threads=None, session=None)
     pfns = [res['pfn'] for res in result if res['pfn'] in bad_pfns]
     res_pfns = []
@@ -839,14 +839,17 @@ def test_client_add_temporary_unavailable_pfns(rse_factory, mock_scope, replica_
 
     rep = []
     for did in files:
+        print(did)
         did['state'] = ReplicaState.TEMPORARY_UNAVAILABLE
         rep.append(did)
 
+    time.sleep(10)  # Test broken if minos_tu_expiration excuted immediately. Threading effect ?
     # Run the minos expiration
-    minos_temp_run(threads=1, once=True)
+    minos_tu_expiration(once=True)
     # Check the state in the replica table
     for did in files:
         rep = get_replicas_state(scope=mock_scope, name=did['name'])
+        print(rep)
         assert list(rep.keys())[0] == ReplicaState.AVAILABLE