Repository: rucio/rucio

Issue:
Implement fair share for the Necromancer
Motivation
----------

A particularly large declaration of lost files can have the Necromancer exclusively processing it and cause delays for single-file declarations that might be more urgent.

Modification
------------

Have the Necromancer cycle through different RSEs.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -548,7 +548,44 @@ def get_pfn_to_rse(pfns, vo='def', session=None):
 
 
 @read_session
-def list_bad_replicas(limit=10000, thread=None, total_threads=None, session=None):
+def get_bad_replicas_backlog(force_refresh=False, session=None):
+    """
+    Get the replica backlog by RSE.
+
+    :param force_refresh:      Boolean to choose if the cached value must be refreshed.
+    :param session:            The database session in use.
+
+    :returns: a list of dictionary {'rse_id': cnt_bad_replicas}.
+    """
+    result = REGION.get('bad_replicas_backlog')
+    if result is NO_VALUE or force_refresh:
+        schema_dot = '%s.' % DEFAULT_SCHEMA_NAME if DEFAULT_SCHEMA_NAME else ''
+        if session.bind.dialect.name == 'oracle':
+            # The filter(text...)) is needed otherwise, SQLA uses bind variables and the index is not used.
+            query = session.query(func.count(models.RSEFileAssociation.rse_id), models.RSEFileAssociation.rse_id).\
+                with_hint(models.RSEFileAssociation, "+ index(replicas REPLICAS_STATE_IDX)", 'oracle').\
+                filter(text("CASE WHEN (%sreplicas.state != 'A') THEN %sreplicas.rse_id END IS NOT NULL" % (schema_dot,
+                                                                                                            schema_dot))). \
+                filter(models.RSEFileAssociation.state == ReplicaState.BAD)
+        else:
+            query = session.query(func.count(models.RSEFileAssociation.rse_id), models.RSEFileAssociation.rse_id).\
+                filter(models.RSEFileAssociation.state == ReplicaState.BAD)
+
+        query = query.join(models.DataIdentifier,
+                           and_(models.DataIdentifier.scope == models.RSEFileAssociation.scope,
+                                models.DataIdentifier.name == models.RSEFileAssociation.name)).\
+            filter(models.DataIdentifier.availability != DIDAvailability.LOST).\
+            group_by(models.RSEFileAssociation.rse_id)
+
+        result = dict()
+        for cnt, rse_id in query.all():
+            result[rse_id] = cnt
+        REGION.set('bad_replicas_backlog', result)
+    return result
+
+
+@read_session
+def list_bad_replicas(limit=10000, thread=None, total_threads=None, rses=None, session=None):
     """
     List RSE File replicas with no locks.
 
@@ -581,6 +618,12 @@ def list_bad_replicas(limit=10000, thread=None, total_threads=None, session=None
                             models.DataIdentifier.name == models.RSEFileAssociation.name)).\
         filter(models.DataIdentifier.availability != DIDAvailability.LOST)
 
+    if rses:
+        rse_clause = list()
+        for rse in rses:
+            rse_clause.append(models.RSEFileAssociation.rse_id == rse['id'])
+        query = query.filter(or_(*rse_clause))
+
     query = query.limit(limit)
     rows = []
     for scope, name, rse_id in query.yield_per(1000):
diff --git a/lib/rucio/daemons/badreplicas/necromancer.py b/lib/rucio/daemons/badreplicas/necromancer.py
--- a/lib/rucio/daemons/badreplicas/necromancer.py
+++ b/lib/rucio/daemons/badreplicas/necromancer.py
@@ -34,14 +34,20 @@
 from sys import exc_info
 from traceback import format_exception
 
+try:
+    from ConfigParser import NoOptionError, NoSectionError
+except ImportError:
+    from configparser import NoOptionError, NoSectionError
+
 import rucio.db.sqla.util
 from rucio.common import exception
-from rucio.common.exception import DatabaseException, ConfigNotFound
+from rucio.common.config import config_get
+from rucio.common.exception import DatabaseException
 from rucio.common.logging import formatted_logger, setup_logging
 from rucio.common.utils import chunks, daemon_sleep
 from rucio.core import monitor, heartbeat
-from rucio.core.config import get
-from rucio.core.replica import list_bad_replicas, get_replicas_state, list_bad_replicas_history, update_bad_replicas_history
+from rucio.core.replica import (list_bad_replicas, get_replicas_state, list_bad_replicas_history,
+                                update_bad_replicas_history, get_bad_replicas_backlog)
 from rucio.core.rule import update_rules_for_lost_replica, update_rules_for_bad_replica, get_evaluation_backlog
 from rucio.db.sqla.constants import ReplicaState
 
@@ -76,12 +82,14 @@ def necromancer(thread=0, bulk=5, once=False, sleep_time=60):
 
         # Check if there is a Judge Evaluator backlog
         try:
-            max_evaluator_backlog_count = get('necromancer', 'max_evaluator_backlog_count')
-        except ConfigNotFound:
+            max_evaluator_backlog_count = config_get('necromancer', 'max_evaluator_backlog_count')
+            max_evaluator_backlog_count = int(max_evaluator_backlog_count)
+        except (NoOptionError, NoSectionError, RuntimeError, ValueError):
             max_evaluator_backlog_count = None
         try:
-            max_evaluator_backlog_duration = get('necromancer', 'max_evaluator_backlog_duration')
-        except ConfigNotFound:
+            max_evaluator_backlog_duration = config_get('necromancer', 'max_evaluator_backlog_duration')
+            max_evaluator_backlog_duration = int(max_evaluator_backlog_duration)
+        except (NoOptionError, NoSectionError, RuntimeError, ValueError):
             max_evaluator_backlog_duration = None
         if max_evaluator_backlog_count or max_evaluator_backlog_duration:
             backlog = get_evaluation_backlog(expiration_time=sleep_time)
@@ -103,35 +111,60 @@ def necromancer(thread=0, bulk=5, once=False, sleep_time=60):
                 GRACEFUL_STOP.wait(30)
                 continue
 
+        # Check how many bad replicas are queued
+        try:
+            max_bad_replicas_backlog_count = config_get('necromancer', 'max_bad_replicas_backlog_count')
+            max_bad_replicas_backlog_count = int(max_bad_replicas_backlog_count)
+        except (NoOptionError, NoSectionError, RuntimeError, ValueError):
+            max_bad_replicas_backlog_count = None
+        bad_replicas_backlog = get_bad_replicas_backlog()
+        tot_bad_files = sum([bad_replicas_backlog[key] for key in bad_replicas_backlog])
+        list_of_rses = list()
+        # If too many replica, call list_bad_replicas with a list of RSEs
+        if max_bad_replicas_backlog_count and tot_bad_files > max_bad_replicas_backlog_count and len(bad_replicas_backlog) > 1:
+            logger(logging.INFO, 'Backlog of bads replica too big. Apply some sharing between different RSEs')
+            rses = list()
+            cnt = 0
+            for key in sorted(bad_replicas_backlog, key=bad_replicas_backlog.get, reverse=False):
+                rses.append({'id': key})
+                cnt += bad_replicas_backlog[key]
+                if cnt >= bulk:
+                    list_of_rses.append(rses)
+                    rses = list()
+                    cnt = 0
+        else:
+            list_of_rses.append(None)
+
         stime = time.time()
         replicas = []
         try:
-            replicas = list_bad_replicas(limit=bulk, thread=heart_beat['assign_thread'], total_threads=heart_beat['nr_threads'])
-
-            for replica in replicas:
-                scope, name, rse_id, rse = replica['scope'], replica['name'], replica['rse_id'], replica['rse']
-                logger(logging.INFO, 'Working on %s:%s on %s' % (scope, name, rse))
-
-                list_replicas = get_replicas_state(scope=scope, name=name)
-                if ReplicaState.AVAILABLE not in list_replicas and ReplicaState.TEMPORARY_UNAVAILABLE not in list_replicas:
-                    logger(logging.INFO, 'File %s:%s has no other available or temporary available replicas, it will be marked as lost' % (scope, name))
-                    try:
-                        update_rules_for_lost_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
-                        monitor.record_counter(name='necromancer.badfiles.lostfile')
-                    except DatabaseException as error:
-                        logger(logging.WARNING, str(error))
-
-                else:
-                    rep = list_replicas.get(ReplicaState.AVAILABLE, [])
-                    unavailable_rep = list_replicas.get(ReplicaState.TEMPORARY_UNAVAILABLE, [])
-                    logger(logging.INFO, 'File %s:%s can be recovered. Available sources : %s + Unavailable sources : %s' % (scope, name, str(rep), str(unavailable_rep)))
-                    try:
-                        update_rules_for_bad_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
-                        monitor.record_counter(name='necromancer.badfiles.recovering')
-                    except DatabaseException as error:
-                        logger(logging.WARNING, str(error))
-
-            logger(logging.INFO, 'It took %s seconds to process %s replicas' % (str(time.time() - stime), str(len(replicas))))
+            for rses in list_of_rses:
+                replicas = list_bad_replicas(limit=bulk, thread=heart_beat['assign_thread'], total_threads=heart_beat['nr_threads'], rses=rses)
+
+                for replica in replicas:
+                    scope, name, rse_id, rse = replica['scope'], replica['name'], replica['rse_id'], replica['rse']
+                    logger(logging.INFO, 'Working on %s:%s on %s' % (scope, name, rse))
+
+                    list_replicas = get_replicas_state(scope=scope, name=name)
+                    if ReplicaState.AVAILABLE not in list_replicas and ReplicaState.TEMPORARY_UNAVAILABLE not in list_replicas:
+                        logger(logging.INFO, 'File %s:%s has no other available or temporary available replicas, it will be marked as lost' % (scope, name))
+                        try:
+                            update_rules_for_lost_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
+                            monitor.record_counter(name='necromancer.badfiles.lostfile')
+                        except DatabaseException as error:
+                            logger(logging.WARNING, str(error))
+
+                    else:
+                        rep = list_replicas.get(ReplicaState.AVAILABLE, [])
+                        unavailable_rep = list_replicas.get(ReplicaState.TEMPORARY_UNAVAILABLE, [])
+                        logger(logging.INFO, 'File %s:%s can be recovered. Available sources : %s + Unavailable sources : %s' % (scope, name, str(rep), str(unavailable_rep)))
+                        try:
+                            update_rules_for_bad_replica(scope=scope, name=name, rse_id=rse_id, nowait=True)
+                            monitor.record_counter(name='necromancer.badfiles.recovering')
+                        except DatabaseException as error:
+                            logger(logging.WARNING, str(error))
+
+                logger(logging.INFO, 'It took %s seconds to process %s replicas' % (str(time.time() - stime), str(len(replicas))))
         except Exception:
             exc_type, exc_value, exc_traceback = exc_info()
             logger(logging.CRITICAL, ''.join(format_exception(exc_type, exc_value, exc_traceback)).strip())


Test patch:
diff --git a/lib/rucio/tests/conftest.py b/lib/rucio/tests/conftest.py
--- a/lib/rucio/tests/conftest.py
+++ b/lib/rucio/tests/conftest.py
@@ -19,6 +19,7 @@
 # - Mayank Sharma <mayank.sharma@cern.ch>, 2021
 # - Simon Fayer <simon.fayer05@imperial.ac.uk>, 2021
 # - Rakshita Varadarajan <rakshitajps@gmail.com>, 2021
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2021
 
 from __future__ import print_function
 
@@ -259,7 +260,7 @@ def file_config_mock(request):
     via the API, as the server config is not changed.
     """
     from unittest import mock
-    from rucio.common.config import Config, config_set
+    from rucio.common.config import Config, config_set, config_has_section, config_add_section
 
     # Get the fixture parameters
     overrides = []
@@ -270,6 +271,8 @@ def file_config_mock(request):
     parser = Config().parser
     with mock.patch('rucio.common.config.get_config', side_effect=lambda: parser):
         for section, option, value in (overrides or []):
+            if not config_has_section(section):
+                config_add_section(section)
             config_set(section, option, value)
         yield
 
diff --git a/lib/rucio/tests/test_bad_replica.py b/lib/rucio/tests/test_bad_replica.py
new file mode 100644
--- /dev/null
+++ b/lib/rucio/tests/test_bad_replica.py
@@ -0,0 +1,389 @@
+# -*- coding: utf-8 -*-
+# Copyright 2021 CERN
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2021
+
+from __future__ import print_function
+
+from datetime import datetime, timedelta
+from json import dumps, loads
+
+import pytest
+
+from rucio.common.exception import RucioException, UnsupportedOperation
+from rucio.common.utils import generate_uuid, clean_surls
+from rucio.core.replica import add_replicas, get_replicas_state, list_replicas, declare_bad_file_replicas, list_bad_replicas, get_bad_pfns, get_bad_replicas_backlog
+from rucio.daemons.badreplicas.minos import run as minos_run
+from rucio.daemons.badreplicas.minos_temporary_expiration import run as minos_temp_run
+from rucio.daemons.badreplicas.necromancer import run as necromancer_run
+from rucio.db.sqla.constants import DIDType, ReplicaState, BadPFNStatus
+from rucio.tests.common import headers, auth
+
+
+@pytest.mark.noparallel(reason='calls list_bad_replicas() which acts on all bad replicas without any filtering')
+def test_add_list_bad_replicas(rse_factory, mock_scope, root_account):
+    """ REPLICA (CORE): Add bad replicas and list them"""
+
+    nbfiles = 5
+    # Adding replicas to deterministic RSE
+    _, rse1_id = rse_factory.make_srm_rse(deterministic=True)
+    files = [{'scope': mock_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for _ in range(nbfiles)]
+    add_replicas(rse_id=rse1_id, files=files, account=root_account, ignore_availability=True)
+
+    # Listing replicas on deterministic RSE
+    replicas = []
+    list_rep = []
+    for replica in list_replicas(dids=[{'scope': f['scope'], 'name': f['name'], 'type': DIDType.FILE} for f in files], schemes=['srm']):
+        replicas.extend(replica['rses'][rse1_id])
+        list_rep.append(replica)
+    r = declare_bad_file_replicas(replicas, 'This is a good reason', root_account)
+    assert r == {}
+    bad_replicas = list_bad_replicas()
+    nbbadrep = 0
+    for rep in list_rep:
+        for badrep in bad_replicas:
+            if badrep['rse_id'] == rse1_id:
+                if badrep['scope'] == rep['scope'] and badrep['name'] == rep['name']:
+                    nbbadrep += 1
+    assert len(replicas) == nbbadrep
+
+    # Adding replicas to non-deterministic RSE
+    _, rse2_id = rse_factory.make_srm_rse(deterministic=False)
+    files = [{'scope': mock_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb',
+              'pfn': 'srm://%s.cern.ch/srm/managerv2?SFN=/test/%s/%s' % (rse2_id, mock_scope, generate_uuid()), 'meta': {'events': 10}} for _ in range(nbfiles)]
+    add_replicas(rse_id=rse2_id, files=files, account=root_account, ignore_availability=True)
+
+    # Listing replicas on non-deterministic RSE
+    replicas = []
+    list_rep = []
+    for replica in list_replicas(dids=[{'scope': f['scope'], 'name': f['name'], 'type': DIDType.FILE} for f in files], schemes=['srm']):
+        replicas.extend(replica['rses'][rse2_id])
+        list_rep.append(replica)
+    r = declare_bad_file_replicas(replicas, 'This is a good reason', root_account)
+    assert r == {}
+    bad_replicas = list_bad_replicas()
+    nbbadrep = 0
+    for rep in list_rep:
+        for badrep in bad_replicas:
+            if badrep['rse_id'] == rse2_id:
+                if badrep['scope'] == rep['scope'] and badrep['name'] == rep['name']:
+                    nbbadrep += 1
+    assert len(replicas) == nbbadrep
+
+    # Now adding non-existing bad replicas
+    files = ['srm://%s.cern.ch/test/%s/%s' % (rse2_id, mock_scope, generate_uuid()), ]
+    r = declare_bad_file_replicas(files, 'This is a good reason', root_account)
+    output = ['%s Unknown replica' % rep for rep in files]
+    assert r == {rse2_id: output}
+
+
+@pytest.mark.noparallel(reason='runs necromancer which acts on all bad replicas without any filtering')
+@pytest.mark.parametrize("file_config_mock", [{
+    "overrides": [('necromancer', 'max_bad_replicas_backlog_count', '20')]
+}], indirect=True)
+def test_get_bad_replicas_backlog(rse_factory, mock_scope, root_account, file_config_mock):
+    """ REPLICA (CORE): Check the behaviour of the necromancer in case of backlog on an RSE"""
+
+    # Run necromancer once
+    necromancer_run(threads=1, bulk=10000, once=True)
+
+    nbfiles1 = 100
+    nbfiles2 = 20
+    # Adding replicas to deterministic RSE
+    rse1, rse1_id = rse_factory.make_srm_rse(deterministic=True)
+    _, rse2_id = rse_factory.make_srm_rse(deterministic=True)
+
+    # Create bad replicas on rse1
+    files = [{'scope': mock_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for _ in range(nbfiles1)]
+    add_replicas(rse_id=rse1_id, files=files, account=root_account, ignore_availability=True)
+
+    replicas = []
+    list_rep = []
+    for replica in list_replicas(dids=[{'scope': f['scope'], 'name': f['name'], 'type': DIDType.FILE} for f in files], schemes=['srm']):
+        replicas.extend(replica['rses'][rse1_id])
+        list_rep.append({'scope': replica['scope'], 'name': replica['name'], 'rse': rse1, 'rse_id': rse1_id})
+    res = declare_bad_file_replicas(replicas, 'This is a good reason', root_account)
+    assert res == {}
+
+    result = get_bad_replicas_backlog(force_refresh=True)
+    assert rse1_id in result
+    assert result[rse1_id] == nbfiles1
+
+    # Create more bad replicas on rse2
+    files = [{'scope': mock_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for _ in range(nbfiles2)]
+    add_replicas(rse_id=rse2_id, files=files, account=root_account, ignore_availability=True)
+
+    repl = []
+    for replica in list_replicas(dids=[{'scope': f['scope'], 'name': f['name'], 'type': DIDType.FILE} for f in files], schemes=['srm']):
+        repl.extend(replica['rses'][rse2_id])
+    res = declare_bad_file_replicas(repl, 'This is a good reason', root_account)
+    assert res == {}
+
+    # List bad replicas on rse1
+    bad_replicas = list_bad_replicas(rses=[{'id': rse1_id}])
+    assert len(bad_replicas) == nbfiles1
+    for rep in bad_replicas:
+        assert rep in list_rep
+
+    # Run necromancer once, all the files on RSE2 should be gone, 80 files should stay on RSE1
+    get_bad_replicas_backlog(force_refresh=True)
+    necromancer_run(threads=1, bulk=20, once=True)
+
+    bad_replicas = list_bad_replicas(rses=[{'id': rse1_id}, {'id': rse2_id}])
+    assert len(bad_replicas) == 80
+    for rep in bad_replicas:
+        assert rep['rse_id'] == rse1_id
+
+
+@pytest.mark.noparallel(reason='calls list_bad_replicas() and runs necromancer. Both act on all bad replicas without any filtering')
+def test_client_add_list_bad_replicas(rse_factory, replica_client, did_client):
+    """ REPLICA (CLIENT): Add bad replicas"""
+    tmp_scope = 'mock'
+    nbfiles = 5
+    # Adding replicas to deterministic RSE
+    files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for _ in range(nbfiles)]
+    rse1, rse1_id = rse_factory.make_srm_rse(deterministic=True)
+    replica_client.add_replicas(rse=rse1, files=files)
+
+    # Listing replicas on deterministic RSE
+    replicas, list_rep = [], []
+    for replica in replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], all_states=True):
+        replicas.extend(replica['rses'][rse1])
+        list_rep.append(replica)
+    r = replica_client.declare_bad_file_replicas(replicas, 'This is a good reason')
+    assert r == {}
+    bad_replicas = list_bad_replicas()
+    nbbadrep = 0
+    for rep in list_rep:
+        for badrep in bad_replicas:
+            if badrep['rse_id'] == rse1_id:
+                if badrep['scope'].external == rep['scope'] and badrep['name'] == rep['name']:
+                    nbbadrep += 1
+    assert len(replicas) == nbbadrep
+
+    # Run necromancer once
+    necromancer_run(threads=1, bulk=10000, once=True)
+
+    # Try to attach a lost file
+    tmp_dsn = 'dataset_%s' % generate_uuid()
+    did_client.add_dataset(scope=tmp_scope, name=tmp_dsn)
+    with pytest.raises(UnsupportedOperation):
+        did_client.add_files_to_dataset(tmp_scope, name=tmp_dsn, files=files, rse=rse1)
+
+    # Adding replicas to non-deterministic RSE
+    rse2, rse2_id = rse_factory.make_srm_rse(deterministic=False)
+    files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb',
+              'pfn': 'srm://%s.cern.ch/srm/managerv2?SFN=/test/%s/%s' % (rse2_id, tmp_scope, generate_uuid()), 'meta': {'events': 10}} for _ in range(nbfiles)]
+    replica_client.add_replicas(rse=rse2, files=files)
+
+    # Listing replicas on non-deterministic RSE
+    replicas, list_rep = [], []
+    for replica in replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], all_states=True):
+        replicas.extend(replica['rses'][rse2])
+        list_rep.append(replica)
+    print(replicas, list_rep)
+    r = replica_client.declare_bad_file_replicas(replicas, 'This is a good reason')
+    print(r)
+    assert r == {}
+    bad_replicas = list_bad_replicas()
+    nbbadrep = 0
+    for rep in list_rep:
+        for badrep in bad_replicas:
+            if badrep['rse_id'] == rse2_id:
+                if badrep['scope'].external == rep['scope'] and badrep['name'] == rep['name']:
+                    nbbadrep += 1
+    assert len(replicas) == nbbadrep
+
+    # Now adding non-existing bad replicas
+    files = ['srm://%s.cern.ch/test/%s/%s' % (rse2_id, tmp_scope, generate_uuid()), ]
+    r = replica_client.declare_bad_file_replicas(files, 'This is a good reason')
+    output = ['%s Unknown replica' % rep for rep in files]
+    assert r == {rse2: output}
+
+
+def test_client_add_suspicious_replicas(rse_factory, replica_client):
+    """ REPLICA (CLIENT): Add suspicious replicas"""
+    tmp_scope = 'mock'
+    nbfiles = 5
+    # Adding replicas to deterministic RSE
+    rse1, _ = rse_factory.make_srm_rse(deterministic=True)
+    files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for _ in range(nbfiles)]
+    replica_client.add_replicas(rse=rse1, files=files)
+
+    # Listing replicas on deterministic RSE
+    replicas = []
+    list_rep = []
+    for replica in replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], all_states=True):
+        replicas.extend(replica['rses'][rse1])
+        list_rep.append(replica)
+    r = replica_client.declare_suspicious_file_replicas(replicas, 'This is a good reason')
+    assert r == {}
+
+    # Adding replicas to non-deterministic RSE
+    rse2, rse2_id = rse_factory.make_srm_rse(deterministic=False)
+    files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb',
+              'pfn': 'srm://%s.cern.ch/srm/managerv2?SFN=/test/%s/%s' % (rse2_id, tmp_scope, generate_uuid()), 'meta': {'events': 10}} for _ in range(nbfiles)]
+    replica_client.add_replicas(rse=rse2, files=files)
+
+    # Listing replicas on non-deterministic RSE
+    replicas = []
+    list_rep = []
+    for replica in replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], all_states=True):
+        replicas.extend(replica['rses'][rse2])
+        list_rep.append(replica)
+    r = replica_client.declare_suspicious_file_replicas(replicas, 'This is a good reason')
+    assert r == {}
+    # Now adding non-existing bad replicas
+    files = ['srm://%s.cern.ch/test/%s/%s' % (rse2_id, tmp_scope, generate_uuid()), ]
+    r = replica_client.declare_suspicious_file_replicas(files, 'This is a good reason')
+    output = ['%s Unknown replica' % rep for rep in files]
+    assert r == {rse2: output}
+
+
+@pytest.mark.noparallel(reason='Lists bad replicas multiple times. If the list changes between calls, test fails.')
+def test_rest_bad_replica_methods_for_ui(rest_client, auth_token):
+    __test_rest_bad_replica_methods_for_ui(rest_client, auth_token, list_pfns=False)
+    __test_rest_bad_replica_methods_for_ui(rest_client, auth_token, list_pfns=True)
+
+
+def __test_rest_bad_replica_methods_for_ui(rest_client, auth_token, list_pfns):
+    """ REPLICA (REST): Test the listing of bad and suspicious replicas """
+    if list_pfns:
+        common_data = {'list_pfns': 'True'}
+    else:
+        common_data = {}
+
+    data = {**common_data}
+    response = rest_client.get('/replicas/bad/states', headers=headers(auth(auth_token)), query_string=data)
+    assert response.status_code == 200
+    tot_files = []
+    for line in response.get_data(as_text=True).split('\n'):
+        if line != '':
+            tot_files.append(dumps(line))
+    nb_tot_files = len(tot_files)
+
+    data = {'state': 'B', **common_data}
+    response = rest_client.get('/replicas/bad/states', headers=headers(auth(auth_token)), query_string=data)
+    assert response.status_code == 200
+    tot_bad_files = []
+    for line in response.get_data(as_text=True).split('\n'):
+        if line != '':
+            tot_bad_files.append(dumps(line))
+    nb_tot_bad_files1 = len(tot_bad_files)
+
+    data = {'state': 'S', **common_data}
+    response = rest_client.get('/replicas/bad/states', headers=headers(auth(auth_token)), query_string=data)
+    assert response.status_code == 200
+    tot_suspicious_files = []
+    for line in response.get_data(as_text=True).split('\n'):
+        if line != '':
+            tot_suspicious_files.append(dumps(line))
+    nb_tot_suspicious_files = len(tot_suspicious_files)
+
+    data = {'state': 'T', **common_data}
+    response = rest_client.get('/replicas/bad/states', headers=headers(auth(auth_token)), query_string=data)
+    assert response.status_code == 200
+    tot_temporary_unavailable_files = []
+    for line in response.get_data(as_text=True).split('\n'):
+        if line != '':
+            tot_temporary_unavailable_files.append(dumps(line))
+    nb_tot_temporary_unavailable_files = len(tot_temporary_unavailable_files)
+
+    assert nb_tot_files == nb_tot_bad_files1 + nb_tot_suspicious_files + nb_tot_temporary_unavailable_files
+
+    tomorrow = datetime.utcnow() + timedelta(days=1)
+    data = {'state': 'B', 'younger_than': tomorrow.isoformat(), **common_data}
+    response = rest_client.get('/replicas/bad/states', headers=headers(auth(auth_token)), query_string=data)
+    assert response.status_code == 200
+    tot_bad_files = []
+    for line in response.get_data(as_text=True).split('\n'):
+        if line != '':
+            tot_bad_files.append(dumps(line))
+    nb_tot_bad_files = len(tot_bad_files)
+    assert nb_tot_bad_files == 0
+
+    if not list_pfns:
+        response = rest_client.get('/replicas/bad/summary', headers=headers(auth(auth_token)))
+        assert response.status_code == 200
+        nb_tot_bad_files2 = 0
+        for line in response.get_data(as_text=True).split('\n'):
+            if line != '':
+                line = loads(line)
+                nb_tot_bad_files2 += int(line.get('BAD', 0))
+        assert nb_tot_bad_files1 == nb_tot_bad_files2
+
+
+@pytest.mark.dirty
+@pytest.mark.noparallel(reason='runs minos, which acts on all bad pfns')
+def test_client_add_temporary_unavailable_pfns(rse_factory, mock_scope, replica_client):
+    """ REPLICA (CLIENT): Add temporary unavailable PFNs"""
+    rse, rse_id = rse_factory.make_posix_rse()
+    nbfiles = 5
+    # Adding replicas to deterministic RSE
+    files = [{'scope': mock_scope.external, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for _ in range(nbfiles)]
+    replica_client.add_replicas(rse=rse, files=files)
+
+    # Listing replicas on deterministic RSE
+    list_rep = []
+    for replica in replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['file'], all_states=True):
+        pfn = list(replica['pfns'].keys())[0]
+        list_rep.append(pfn)
+
+    # Submit bad PFNs
+    now = datetime.utcnow()
+    reason_str = generate_uuid()
+    replica_client.add_bad_pfns(pfns=list_rep, reason=str(reason_str), state='TEMPORARY_UNAVAILABLE', expires_at=now.isoformat())
+    result = get_bad_pfns(limit=10000, thread=None, total_threads=None, session=None)
+    bad_pfns = {}
+    for res in result:
+        bad_pfns[res['pfn']] = (res['state'], res['reason'], res['expires_at'])
+
+    for pfn in list_rep:
+        pfn = str(clean_surls([pfn])[0])
+        assert pfn in bad_pfns
+        assert bad_pfns[pfn][0] == BadPFNStatus.TEMPORARY_UNAVAILABLE
+        assert bad_pfns[pfn][1] == reason_str
+
+    # Submit with wrong state
+    with pytest.raises(RucioException):
+        replica_client.add_bad_pfns(pfns=list_rep, reason=str(reason_str), state='BADSTATE', expires_at=now.isoformat())
+
+    # Run minos once
+    minos_run(threads=1, bulk=10000, once=True)
+    result = get_bad_pfns(limit=10000, thread=None, total_threads=None, session=None)
+    pfns = [res['pfn'] for res in result if res['pfn'] in bad_pfns]
+    res_pfns = []
+    for replica in list_rep:
+        if replica in pfns:
+            res_pfns.append(replica)
+    assert res_pfns == []
+
+    # Check the state in the replica table
+    for did in files:
+        rep = get_replicas_state(scope=mock_scope, name=did['name'])
+        assert list(rep.keys())[0] == ReplicaState.TEMPORARY_UNAVAILABLE
+
+    rep = []
+    for did in files:
+        did['state'] = ReplicaState.TEMPORARY_UNAVAILABLE
+        rep.append(did)
+
+    # Run the minos expiration
+    minos_temp_run(threads=1, once=True)
+    # Check the state in the replica table
+    for did in files:
+        rep = get_replicas_state(scope=mock_scope, name=did['name'])
+        assert list(rep.keys())[0] == ReplicaState.AVAILABLE
diff --git a/lib/rucio/tests/test_replica.py b/lib/rucio/tests/test_replica.py
--- a/lib/rucio/tests/test_replica.py
+++ b/lib/rucio/tests/test_replica.py
@@ -39,7 +39,7 @@
 import hashlib
 import time
 from datetime import datetime, timedelta
-from json import dumps, loads
+from json import dumps
 from unittest import mock
 from xml.etree import ElementTree
 
@@ -48,20 +48,19 @@
 from werkzeug.datastructures import MultiDict
 
 from rucio.client.ruleclient import RuleClient
-from rucio.common.exception import (DataIdentifierNotFound, AccessDenied, UnsupportedOperation,
-                                    RucioException, ReplicaIsLocked, ReplicaNotFound, ScopeNotFound,
+from rucio.common.exception import (DataIdentifierNotFound, AccessDenied, RucioException,
+                                    ReplicaIsLocked, ReplicaNotFound, ScopeNotFound,
                                     DatabaseException)
 from rucio.common.utils import generate_uuid, clean_surls, parse_response
 from rucio.core.config import set as cconfig_set
 from rucio.core.did import add_did, attach_dids, get_did, set_status, list_files, get_did_atime
 from rucio.core.replica import (add_replica, add_replicas, delete_replicas, get_replicas_state,
-                                get_replica, list_replicas, declare_bad_file_replicas, list_bad_replicas,
-                                update_replica_state, get_RSEcoverage_of_dataset, get_replica_atime,
+                                get_replica, list_replicas, update_replica_state,
+                                get_RSEcoverage_of_dataset, get_replica_atime,
                                 touch_replica, get_bad_pfns, set_tombstone)
 from rucio.core.rse import add_protocol, add_rse_attribute, del_rse_attribute
 from rucio.daemons.badreplicas.minos import run as minos_run
 from rucio.daemons.badreplicas.minos_temporary_expiration import run as minos_temp_run
-from rucio.daemons.badreplicas.necromancer import run as necromancer_run
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import DIDType, ReplicaState, BadPFNStatus, OBSOLETE
 from rucio.db.sqla.session import transactional_session
@@ -200,62 +199,6 @@ def test_replicas_pfn_resolution(self, rse_factory, mock_scope, root_account):
 
         add_replicas(rse_id=rse_id, files=files, account=root_account)
 
-    @pytest.mark.noparallel(reason='calls list_bad_replicas() which acts on all bad replicas without any filtering')
-    def test_add_list_bad_replicas(self, rse_factory, mock_scope, root_account):
-        """ REPLICA (CORE): Add bad replicas and list them"""
-
-        nbfiles = 5
-        # Adding replicas to deterministic RSE
-        _, rse1_id = rse_factory.make_srm_rse(deterministic=True)
-        files = [{'scope': mock_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for _ in range(nbfiles)]
-        add_replicas(rse_id=rse1_id, files=files, account=root_account, ignore_availability=True)
-
-        # Listing replicas on deterministic RSE
-        replicas = []
-        list_rep = []
-        for replica in list_replicas(dids=[{'scope': f['scope'], 'name': f['name'], 'type': DIDType.FILE} for f in files], schemes=['srm']):
-            replicas.extend(replica['rses'][rse1_id])
-            list_rep.append(replica)
-        r = declare_bad_file_replicas(replicas, 'This is a good reason', root_account)
-        assert r == {}
-        bad_replicas = list_bad_replicas()
-        nbbadrep = 0
-        for rep in list_rep:
-            for badrep in bad_replicas:
-                if badrep['rse_id'] == rse1_id:
-                    if badrep['scope'] == rep['scope'] and badrep['name'] == rep['name']:
-                        nbbadrep += 1
-        assert len(replicas) == nbbadrep
-
-        # Adding replicas to non-deterministic RSE
-        _, rse2_id = rse_factory.make_srm_rse(deterministic=False)
-        files = [{'scope': mock_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb',
-                  'pfn': 'srm://%s.cern.ch/srm/managerv2?SFN=/test/%s/%s' % (rse2_id, mock_scope, generate_uuid()), 'meta': {'events': 10}} for _ in range(nbfiles)]
-        add_replicas(rse_id=rse2_id, files=files, account=root_account, ignore_availability=True)
-
-        # Listing replicas on non-deterministic RSE
-        replicas = []
-        list_rep = []
-        for replica in list_replicas(dids=[{'scope': f['scope'], 'name': f['name'], 'type': DIDType.FILE} for f in files], schemes=['srm']):
-            replicas.extend(replica['rses'][rse2_id])
-            list_rep.append(replica)
-        r = declare_bad_file_replicas(replicas, 'This is a good reason', root_account)
-        assert r == {}
-        bad_replicas = list_bad_replicas()
-        nbbadrep = 0
-        for rep in list_rep:
-            for badrep in bad_replicas:
-                if badrep['rse_id'] == rse2_id:
-                    if badrep['scope'] == rep['scope'] and badrep['name'] == rep['name']:
-                        nbbadrep += 1
-        assert len(replicas) == nbbadrep
-
-        # Now adding non-existing bad replicas
-        files = ['srm://%s.cern.ch/test/%s/%s' % (rse2_id, mock_scope, generate_uuid()), ]
-        r = declare_bad_file_replicas(files, 'This is a good reason', root_account)
-        output = ['%s Unknown replica' % rep for rep in files]
-        assert r == {rse2_id: output}
-
     def test_add_list_replicas(self, rse_factory, mock_scope, root_account):
         """ REPLICA (CORE): Add and list file replicas """
         _, rse1_id = rse_factory.make_mock_rse()
@@ -681,184 +624,6 @@ def test_delete_replicas_from_datasets_new(core_config_mock, caches_mock, rse_fa
         get_did(scope=mock_scope, name=tmp_dsn1)
 
 
-@pytest.mark.noparallel(reason='calls list_bad_replicas() and runs necromancer. Both act on all bad replicas without any filtering')
-def test_client_add_list_bad_replicas(rse_factory, replica_client, did_client):
-    """ REPLICA (CLIENT): Add bad replicas"""
-    tmp_scope = 'mock'
-    nbfiles = 5
-    # Adding replicas to deterministic RSE
-    files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for _ in range(nbfiles)]
-    rse1, rse1_id = rse_factory.make_srm_rse(deterministic=True)
-    replica_client.add_replicas(rse=rse1, files=files)
-
-    # Listing replicas on deterministic RSE
-    replicas, list_rep = [], []
-    for replica in replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], all_states=True):
-        replicas.extend(replica['rses'][rse1])
-        list_rep.append(replica)
-    r = replica_client.declare_bad_file_replicas(replicas, 'This is a good reason')
-    assert r == {}
-    bad_replicas = list_bad_replicas()
-    nbbadrep = 0
-    for rep in list_rep:
-        for badrep in bad_replicas:
-            if badrep['rse_id'] == rse1_id:
-                if badrep['scope'].external == rep['scope'] and badrep['name'] == rep['name']:
-                    nbbadrep += 1
-    assert len(replicas) == nbbadrep
-
-    # Run necromancer once
-    necromancer_run(threads=1, bulk=10000, once=True)
-
-    # Try to attach a lost file
-    tmp_dsn = 'dataset_%s' % generate_uuid()
-    did_client.add_dataset(scope=tmp_scope, name=tmp_dsn)
-    with pytest.raises(UnsupportedOperation):
-        did_client.add_files_to_dataset(tmp_scope, name=tmp_dsn, files=files, rse=rse1)
-
-    # Adding replicas to non-deterministic RSE
-    rse2, rse2_id = rse_factory.make_srm_rse(deterministic=False)
-    files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb',
-              'pfn': 'srm://%s.cern.ch/srm/managerv2?SFN=/test/%s/%s' % (rse2_id, tmp_scope, generate_uuid()), 'meta': {'events': 10}} for _ in range(nbfiles)]
-    replica_client.add_replicas(rse=rse2, files=files)
-
-    # Listing replicas on non-deterministic RSE
-    replicas, list_rep = [], []
-    for replica in replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], all_states=True):
-        replicas.extend(replica['rses'][rse2])
-        list_rep.append(replica)
-    print(replicas, list_rep)
-    r = replica_client.declare_bad_file_replicas(replicas, 'This is a good reason')
-    print(r)
-    assert r == {}
-    bad_replicas = list_bad_replicas()
-    nbbadrep = 0
-    for rep in list_rep:
-        for badrep in bad_replicas:
-            if badrep['rse_id'] == rse2_id:
-                if badrep['scope'].external == rep['scope'] and badrep['name'] == rep['name']:
-                    nbbadrep += 1
-    assert len(replicas) == nbbadrep
-
-    # Now adding non-existing bad replicas
-    files = ['srm://%s.cern.ch/test/%s/%s' % (rse2_id, tmp_scope, generate_uuid()), ]
-    r = replica_client.declare_bad_file_replicas(files, 'This is a good reason')
-    output = ['%s Unknown replica' % rep for rep in files]
-    assert r == {rse2: output}
-
-
-def test_client_add_suspicious_replicas(rse_factory, replica_client):
-    """ REPLICA (CLIENT): Add suspicious replicas"""
-    tmp_scope = 'mock'
-    nbfiles = 5
-    # Adding replicas to deterministic RSE
-    rse1, _ = rse_factory.make_srm_rse(deterministic=True)
-    files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for _ in range(nbfiles)]
-    replica_client.add_replicas(rse=rse1, files=files)
-
-    # Listing replicas on deterministic RSE
-    replicas = []
-    list_rep = []
-    for replica in replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], all_states=True):
-        replicas.extend(replica['rses'][rse1])
-        list_rep.append(replica)
-    r = replica_client.declare_suspicious_file_replicas(replicas, 'This is a good reason')
-    assert r == {}
-
-    # Adding replicas to non-deterministic RSE
-    rse2, rse2_id = rse_factory.make_srm_rse(deterministic=False)
-    files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb',
-              'pfn': 'srm://%s.cern.ch/srm/managerv2?SFN=/test/%s/%s' % (rse2_id, tmp_scope, generate_uuid()), 'meta': {'events': 10}} for _ in range(nbfiles)]
-    replica_client.add_replicas(rse=rse2, files=files)
-
-    # Listing replicas on non-deterministic RSE
-    replicas = []
-    list_rep = []
-    for replica in replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in files], schemes=['srm'], all_states=True):
-        replicas.extend(replica['rses'][rse2])
-        list_rep.append(replica)
-    r = replica_client.declare_suspicious_file_replicas(replicas, 'This is a good reason')
-    assert r == {}
-    # Now adding non-existing bad replicas
-    files = ['srm://%s.cern.ch/test/%s/%s' % (rse2_id, tmp_scope, generate_uuid()), ]
-    r = replica_client.declare_suspicious_file_replicas(files, 'This is a good reason')
-    output = ['%s Unknown replica' % rep for rep in files]
-    assert r == {rse2: output}
-
-
-@pytest.mark.noparallel(reason='Lists bad replicas multiple times. If the list changes between calls, test fails.')
-def test_rest_bad_replica_methods_for_ui(rest_client, auth_token):
-    __test_rest_bad_replica_methods_for_ui(rest_client, auth_token, list_pfns=False)
-    __test_rest_bad_replica_methods_for_ui(rest_client, auth_token, list_pfns=True)
-
-
-def __test_rest_bad_replica_methods_for_ui(rest_client, auth_token, list_pfns):
-    """ REPLICA (REST): Test the listing of bad and suspicious replicas """
-    if list_pfns:
-        common_data = {'list_pfns': 'True'}
-    else:
-        common_data = {}
-
-    data = {**common_data}
-    response = rest_client.get('/replicas/bad/states', headers=headers(auth(auth_token)), query_string=data)
-    assert response.status_code == 200
-    tot_files = []
-    for line in response.get_data(as_text=True).split('\n'):
-        if line != '':
-            tot_files.append(dumps(line))
-    nb_tot_files = len(tot_files)
-
-    data = {'state': 'B', **common_data}
-    response = rest_client.get('/replicas/bad/states', headers=headers(auth(auth_token)), query_string=data)
-    assert response.status_code == 200
-    tot_bad_files = []
-    for line in response.get_data(as_text=True).split('\n'):
-        if line != '':
-            tot_bad_files.append(dumps(line))
-    nb_tot_bad_files1 = len(tot_bad_files)
-
-    data = {'state': 'S', **common_data}
-    response = rest_client.get('/replicas/bad/states', headers=headers(auth(auth_token)), query_string=data)
-    assert response.status_code == 200
-    tot_suspicious_files = []
-    for line in response.get_data(as_text=True).split('\n'):
-        if line != '':
-            tot_suspicious_files.append(dumps(line))
-    nb_tot_suspicious_files = len(tot_suspicious_files)
-
-    data = {'state': 'T', **common_data}
-    response = rest_client.get('/replicas/bad/states', headers=headers(auth(auth_token)), query_string=data)
-    assert response.status_code == 200
-    tot_temporary_unavailable_files = []
-    for line in response.get_data(as_text=True).split('\n'):
-        if line != '':
-            tot_temporary_unavailable_files.append(dumps(line))
-    nb_tot_temporary_unavailable_files = len(tot_temporary_unavailable_files)
-
-    assert nb_tot_files == nb_tot_bad_files1 + nb_tot_suspicious_files + nb_tot_temporary_unavailable_files
-
-    tomorrow = datetime.utcnow() + timedelta(days=1)
-    data = {'state': 'B', 'younger_than': tomorrow.isoformat(), **common_data}
-    response = rest_client.get('/replicas/bad/states', headers=headers(auth(auth_token)), query_string=data)
-    assert response.status_code == 200
-    tot_bad_files = []
-    for line in response.get_data(as_text=True).split('\n'):
-        if line != '':
-            tot_bad_files.append(dumps(line))
-    nb_tot_bad_files = len(tot_bad_files)
-    assert nb_tot_bad_files == 0
-
-    if not list_pfns:
-        response = rest_client.get('/replicas/bad/summary', headers=headers(auth(auth_token)))
-        assert response.status_code == 200
-        nb_tot_bad_files2 = 0
-        for line in response.get_data(as_text=True).split('\n'):
-            if line != '':
-                line = loads(line)
-                nb_tot_bad_files2 += int(line.get('BAD', 0))
-        assert nb_tot_bad_files1 == nb_tot_bad_files2
-
-
 def test_rest_list_replicas_content_type(rse_factory, mock_scope, replica_client, rest_client, auth_token):
     """ REPLICA (REST): send a GET to list replicas with specific ACCEPT header."""
     rse, _ = rse_factory.make_mock_rse()