Repository: rucio/rucio

Issue:
Non optimal use of the limit parameter in list_and_mark_unlocked_replicas 
Motivation
----------
Non optimal use of the limit parameter in list_and_mark_unlocked_replicas 






Discussion:

Resolving patch:
diff --git a/lib/rucio/common/utils.py b/lib/rucio/common/utils.py
--- a/lib/rucio/common/utils.py
+++ b/lib/rucio/common/utils.py
@@ -20,7 +20,7 @@
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2022
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
 # - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2015-2018
-# - Martin Barisits <martin.barisits@cern.ch>, 2016-2021
+# - Martin Barisits <martin.barisits@cern.ch>, 2016-2022
 # - Brian Bockelman <bbockelm@cse.unl.edu>, 2018
 # - Tobias Wegner <twegner@cern.ch>, 2018-2019
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
@@ -35,17 +35,16 @@
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
 # - Mayank Sharma <mayank.sharma@cern.ch>, 2021
 # - Rahul Chauhan <omrahulchauhan@gmail.com>, 2021
-# - Radu Carpa <radu.carpa@cern.ch>, 2021
+# - Radu Carpa <radu.carpa@cern.ch>, 2021-2022
 # - Anil Panta <47672624+panta-123@users.noreply.github.com>, 2021
 # - Ilija Vukotic <ivukotic@cern.ch>, 2021
 # - David Población Criado <david.poblacion.criado@cern.ch>, 2021
-# - martynia <janusz.martyniak@googlemail.com>, 2021
+# - martynia <janusz.martyniak@googlemail.com>, 2021-2022
 # - jdierkes <joel.dierkes@cern.ch>, 2021
 # - Rakshita Varadarajan <rakshitajps@gmail.com>, 2021
 # - Rob Barnsley <robbarnsley@users.noreply.github.com>, 2021
 # - Igor Mandrichenko <ivm@fnal.gov>, 2021
 # - Joel Dierkes <joel.dierkes@cern.ch>, 2021
-# - Janusz Martyniak <janusz.martyniak@googlemail.com>, 2022
 
 from __future__ import absolute_import, print_function
 
@@ -500,12 +499,20 @@ def grouper(iterable, n, fillvalue=None):
     return izip_longest(*args, fillvalue=fillvalue)
 
 
-def chunks(list_, n):
+def chunks(iterable, n):
     """
     Yield successive n-sized chunks from l.
     """
-    for i in range(0, len(list_), n):
-        yield list_[i:i + n]
+    if isinstance(iterable, list):
+        for i in range(0, len(iterable), n):
+            yield iterable[i:i + n]
+    else:
+        it = iter(iterable)
+        while True:
+            chunk = list(itertools.islice(it, n))
+            if not chunk:
+                return
+            yield chunk
 
 
 def dict_chunks(dict_, n):
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -44,6 +44,7 @@
 
 import heapq
 import logging
+import math
 import random
 from collections import defaultdict
 from copy import deepcopy
@@ -58,7 +59,7 @@
 import requests
 from dogpile.cache.api import NO_VALUE
 from six import string_types
-from sqlalchemy import func, and_, or_, exists, not_, update, delete
+from sqlalchemy import func, and_, or_, exists, not_, update, delete, Column
 from sqlalchemy.exc import DatabaseError, IntegrityError
 from sqlalchemy.orm import aliased
 from sqlalchemy.orm.exc import FlushError, NoResultFound
@@ -70,6 +71,7 @@
 from rucio.common import exception
 from rucio.common.cache import make_region_memcached
 from rucio.common.config import config_get
+from rucio.common.schema import get_schema_value
 from rucio.common.types import InternalScope
 from rucio.common.utils import chunks, clean_surls, str_to_date, add_url_query
 from rucio.common.constants import SuspiciousAvailability
@@ -83,6 +85,8 @@
                                      BadFilesStatus, RuleState, BadPFNStatus)
 from rucio.db.sqla.session import (read_session, stream_session, transactional_session,
                                    DEFAULT_SCHEMA_NAME, BASE)
+from rucio.db.sqla.types import InternalScopeString, String
+from rucio.db.sqla.util import create_temp_table
 from rucio.rse import rsemanager as rsemgr
 
 REGION = make_region_memcached(expiration_time=60)
@@ -2008,6 +2012,142 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
     """
     List RSE File replicas with no locks.
 
+    :param limit:                    Number of replicas returned.
+    :param bytes_:                   The amount of needed bytes.
+    :param rse_id:                   The rse_id.
+    :param delay_seconds:            The delay to query replicas in BEING_DELETED state
+    :param only_delete_obsolete      If set to True, will only return the replicas with EPOCH tombstone
+    :param session:                  The database session in use.
+
+    :returns: a list of dictionary replica.
+    """
+
+    needed_space = bytes_
+    total_bytes = 0
+    rows = []
+
+    columns = [
+        Column("scope", InternalScopeString(get_schema_value('SCOPE_LENGTH'))),
+        Column("name", String(get_schema_value('NAME_LENGTH'))),
+    ]
+    temp_table_cls = create_temp_table(
+        "list_and_mark_unlocked_replicas",
+        *columns,
+        primary_key=columns,
+        session=session,
+    )
+
+    replicas_alias = aliased(models.RSEFileAssociation, name='replicas_alias')
+
+    stmt = select(
+        models.RSEFileAssociation.scope,
+        models.RSEFileAssociation.name,
+    ).with_hint(
+        models.RSEFileAssociation, "INDEX_RS_ASC(replicas REPLICAS_TOMBSTONE_IDX)  NO_INDEX_FFS(replicas REPLICAS_TOMBSTONE_IDX)", 'oracle'
+    ).where(
+        models.RSEFileAssociation.lock_cnt == 0,
+        models.RSEFileAssociation.rse_id == rse_id,
+        models.RSEFileAssociation.tombstone == OBSOLETE if only_delete_obsolete else models.RSEFileAssociation.tombstone < datetime.utcnow(),
+    ).where(
+        or_(models.RSEFileAssociation.state.in_((ReplicaState.AVAILABLE, ReplicaState.UNAVAILABLE, ReplicaState.BAD)),
+            and_(models.RSEFileAssociation.state == ReplicaState.BEING_DELETED, models.RSEFileAssociation.updated_at < datetime.utcnow() - timedelta(seconds=delay_seconds)))
+    ).outerjoin(
+        models.Source,
+        and_(models.RSEFileAssociation.scope == models.Source.scope,
+             models.RSEFileAssociation.name == models.Source.name,
+             models.RSEFileAssociation.rse_id == models.Source.rse_id)
+    ).where(
+        models.Source.scope.is_(None)  # Only try to delete replicas if they are not used as sources in any transfers
+    ).order_by(
+        models.RSEFileAssociation.tombstone
+    ).with_for_update(
+        skip_locked=True,
+        # oracle: we must specify a column, not a table; however, it doesn't matter which column, the lock is put on the whole row
+        # postgresql/mysql: sqlalchemy driver automatically converts it to a table name
+        # sqlite: this is completely ignored
+        of=models.RSEFileAssociation.scope,
+    )
+
+    for chunk in chunks(session.execute(stmt).yield_per(2 * limit), math.ceil(1.25 * limit)):
+        session.query(temp_table_cls).delete()
+        session.bulk_insert_mappings(temp_table_cls, [{'scope': scope, 'name': name} for scope, name in chunk])
+
+        stmt = select(
+            models.RSEFileAssociation.scope,
+            models.RSEFileAssociation.name,
+            models.RSEFileAssociation.path,
+            models.RSEFileAssociation.bytes,
+            models.RSEFileAssociation.tombstone,
+            models.RSEFileAssociation.state,
+        ).join(
+            temp_table_cls,
+            and_(models.RSEFileAssociation.scope == temp_table_cls.scope,
+                 models.RSEFileAssociation.name == temp_table_cls.name,
+                 models.RSEFileAssociation.rse_id == rse_id)
+        ).with_hint(
+            replicas_alias, "index(%(name)s REPLICAS_PK)", 'oracle'
+        ).outerjoin(
+            replicas_alias,
+            and_(models.RSEFileAssociation.scope == replicas_alias.scope,
+                 models.RSEFileAssociation.name == replicas_alias.name,
+                 models.RSEFileAssociation.rse_id != replicas_alias.rse_id)
+        ).with_hint(
+            models.Request, "INDEX(requests REQUESTS_SCOPE_NAME_RSE_IDX)", 'oracle'
+        ).outerjoin(
+            models.Request,
+            and_(models.RSEFileAssociation.scope == models.Request.scope,
+                 models.RSEFileAssociation.name == models.Request.name)
+        ).group_by(
+            models.RSEFileAssociation
+        ).having(
+            case([(func.count(replicas_alias.scope) > 0, True),  # Can delete this replica if it's not the last replica
+                  (func.count(models.Request.scope) == 0, True)],  # If it's the last replica, only can delete if there are no requests using it
+                 else_=False).label("can_delete"),
+        ).order_by(
+            models.RSEFileAssociation.tombstone
+        ).limit(
+            limit - len(rows)
+        )
+
+        for scope, name, path, bytes_, tombstone, state in session.execute(stmt):
+            if len(rows) >= limit or (needed_space is not None and total_bytes > needed_space):
+                break
+            if state != ReplicaState.UNAVAILABLE:
+                total_bytes += bytes_
+
+            rows.append({'scope': scope, 'name': name, 'path': path,
+                         'bytes': bytes_, 'tombstone': tombstone,
+                         'state': state})
+        if len(rows) >= limit or (needed_space is not None and total_bytes > needed_space):
+            break
+
+    if rows:
+        session.query(temp_table_cls).delete()
+        session.bulk_insert_mappings(temp_table_cls, [{'scope': r['scope'], 'name': r['name']} for r in rows])
+        stmt = update(
+            models.RSEFileAssociation
+        ).where(
+            exists(select([1]).prefix_with("/*+ INDEX(REPLICAS REPLICAS_PK) */", dialect='oracle')
+                   .where(and_(models.RSEFileAssociation.scope == temp_table_cls.scope,
+                               models.RSEFileAssociation.name == temp_table_cls.name,
+                               models.RSEFileAssociation.rse_id == rse_id)))
+        ).execution_options(
+            synchronize_session=False
+        ).values(
+            updated_at=datetime.utcnow(),
+            state=ReplicaState.BEING_DELETED,
+            tombstone=OBSOLETE,
+        )
+        session.execute(stmt)
+
+    return rows
+
+
+@transactional_session
+def list_and_mark_unlocked_replicas_no_temp_table(limit, bytes_=None, rse_id=None, delay_seconds=600, only_delete_obsolete=False, session=None):
+    """
+    List RSE File replicas with no locks.
+
     :param limit:                    Number of replicas returned.
     :param bytes_:                    The amount of needed bytes.
     :param rse_id:                   The rse_id.
@@ -2049,17 +2189,18 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
             filter(and_(models.RSEFileAssociation.scope == scope, models.RSEFileAssociation.name == name, models.RSEFileAssociation.rse_id != rse_id)).one()
 
         if replica_cnt[0] > 1:
+            if tombstone != OBSOLETE and only_delete_obsolete:
+                break
+
+            if needed_space is not None and total_bytes > needed_space:
+                break
+
             if state != ReplicaState.UNAVAILABLE:
-                if tombstone != OBSOLETE:
-                    if only_delete_obsolete:
-                        break
-                    if needed_space is not None and total_bytes > needed_space:
-                        break
                 total_bytes += bytes_
 
-                total_files += 1
-                if total_files > limit:
-                    break
+            total_files += 1
+            if total_files > limit:
+                break
 
             rows.append({'scope': scope, 'name': name, 'path': path,
                          'bytes': bytes_, 'tombstone': tombstone,
@@ -2075,12 +2216,14 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
                             models.Request.name == name)).one()
 
             if request_cnt[0] == 0:
-                if tombstone != OBSOLETE:
-                    if only_delete_obsolete:
-                        break
-                    if needed_space is not None and total_bytes > needed_space:
-                        break
-                total_bytes += bytes_
+                if tombstone != OBSOLETE and only_delete_obsolete:
+                    break
+
+                if needed_space is not None and total_bytes > needed_space:
+                    break
+
+                if state != ReplicaState.UNAVAILABLE:
+                    total_bytes += bytes_
 
                 total_files += 1
                 if total_files > limit:
diff --git a/lib/rucio/daemons/reaper/reaper.py b/lib/rucio/daemons/reaper/reaper.py
--- a/lib/rucio/daemons/reaper/reaper.py
+++ b/lib/rucio/daemons/reaper/reaper.py
@@ -28,6 +28,7 @@
 # - Matt Snyder <msnyder@bnl.gov>, 2021
 # - David Población Criado <david.poblacion.criado@cern.ch>, 2021
 # - Radu Carpa <radu.carpa@cern.ch>, 2021-2022
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2021
 
 '''
 Reaper is a daemon to manage file deletion.
@@ -65,7 +66,7 @@
 from rucio.core.credential import get_signed_url
 from rucio.core.heartbeat import live, die, sanity_check, list_payload_counts
 from rucio.core.message import add_message
-from rucio.core.replica import list_and_mark_unlocked_replicas, delete_replicas
+from rucio.core.replica import list_and_mark_unlocked_replicas, list_and_mark_unlocked_replicas_no_temp_table, delete_replicas
 from rucio.core.rse import list_rses, get_rse_limits, get_rse_usage, list_rse_attributes, get_rse_protocols
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.core.rule import get_evaluation_backlog
@@ -564,15 +565,24 @@ def reaper(rses, include_rses, exclude_rses, vos=None, chunk_size=100, once=Fals
                 del_start_time = time.time()
                 only_delete_obsolete = dict_rses[(rse_name, rse_id)][1]
                 try:
+                    use_temp_tables = config_get_bool('core', 'use_temp_tables', default=False)
                     with monitor.record_timer_block('reaper.list_unlocked_replicas'):
                         if only_delete_obsolete:
                             logger(logging.DEBUG, 'Will run list_and_mark_unlocked_replicas on %s. No space needed, will only delete EPOCH tombstoned replicas', rse_name)
-                        replicas = list_and_mark_unlocked_replicas(limit=chunk_size,
-                                                                   bytes_=needed_free_space,
-                                                                   rse_id=rse_id,
-                                                                   delay_seconds=delay_seconds,
-                                                                   only_delete_obsolete=only_delete_obsolete,
-                                                                   session=None)
+                        if use_temp_tables:
+                            replicas = list_and_mark_unlocked_replicas(limit=chunk_size,
+                                                                       bytes_=needed_free_space,
+                                                                       rse_id=rse_id,
+                                                                       delay_seconds=delay_seconds,
+                                                                       only_delete_obsolete=only_delete_obsolete,
+                                                                       session=None)
+                        else:
+                            replicas = list_and_mark_unlocked_replicas_no_temp_table(limit=chunk_size,
+                                                                                     bytes_=needed_free_space,
+                                                                                     rse_id=rse_id,
+                                                                                     delay_seconds=delay_seconds,
+                                                                                     only_delete_obsolete=only_delete_obsolete,
+                                                                                     session=None)
                     logger(logging.DEBUG, 'list_and_mark_unlocked_replicas on %s for %s bytes in %s seconds: %s replicas', rse_name, needed_free_space, time.time() - del_start_time, len(replicas))
                     if len(replicas) < chunk_size:
                         logger(logging.DEBUG, 'Not enough replicas to delete on %s (%s requested vs %s returned). Will skip any new attempts on this RSE until next cycle', rse_name, chunk_size, len(replicas))


Test patch:
diff --git a/lib/rucio/tests/test_reaper.py b/lib/rucio/tests/test_reaper.py
--- a/lib/rucio/tests/test_reaper.py
+++ b/lib/rucio/tests/test_reaper.py
@@ -1,4 +1,5 @@
-# Copyright 2020 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2013-2022 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,14 +14,18 @@
 # limitations under the License.
 #
 # Authors:
-# - Thomas Beermann <thomas.beermann@cern.ch>, 2020
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2013-2019
+# - Martin Barisits <martin.barisits@cern.ch>, 2016-2021
+# - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2018
+# - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2019
 # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
-# - Radu Carpa <radu.carpa@cern.ch>, 2021
 # - Matt Snyder <msnyder@bnl.gov>, 2021
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2021
+# - Radu Carpa <radu.carpa@cern.ch>, 2021-2022
 
 from datetime import datetime, timedelta
 
@@ -76,10 +81,15 @@ def __add_test_rse_and_replicas(vo, scope, rse_name, names, file_size, epoch_tom
     return rse_name, rse_id, dids
 
 
+@pytest.mark.parametrize("file_config_mock", [
+    # Run test twice: with, and without, temp tables
+    {"overrides": [('core', 'use_temp_tables', 'True')]},
+    {"overrides": [('core', 'use_temp_tables', 'False')]},
+], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.daemons.reaper.reaper.REGION'
 ]}], indirect=True)
-def test_reaper(vo, caches_mock):
+def test_reaper(vo, caches_mock, file_config_mock):
     """ REAPER (DAEMON): Test the reaper daemon."""
     [cache_region] = caches_mock
     scope = InternalScope('data13_hip', vo=vo)
@@ -106,10 +116,15 @@ def test_reaper(vo, caches_mock):
     assert len(list(replica_core.list_replicas(dids, rse_expression=rse_name))) == 200
 
 
+@pytest.mark.parametrize("file_config_mock", [
+    # Run test twice: with, and without, temp tables
+    {"overrides": [('core', 'use_temp_tables', 'True')]},
+    {"overrides": [('core', 'use_temp_tables', 'False')]},
+], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.daemons.reaper.reaper.REGION'
 ]}], indirect=True)
-def test_reaper_bulk_delete(vo, caches_mock):
+def test_reaper_bulk_delete(vo, caches_mock, file_config_mock):
     """ REAPER (DAEMON): Mock test the reaper daemon on async bulk delete request."""
     [cache_region] = caches_mock
     scope = InternalScope('data13_hip', vo=vo)
@@ -136,10 +151,15 @@ def test_reaper_bulk_delete(vo, caches_mock):
     assert len(list(replica_core.list_replicas(dids, rse_expression=rse_name))) == 200
 
 
+@pytest.mark.parametrize("file_config_mock", [
+    # Run test twice: with, and without, temp tables
+    {"overrides": [('core', 'use_temp_tables', 'True')]},
+    {"overrides": [('core', 'use_temp_tables', 'False')]},
+], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.daemons.reaper.reaper.REGION'
 ]}], indirect=True)
-def test_reaper_multi_vo_via_run(vo, second_vo, scope_factory, caches_mock):
+def test_reaper_multi_vo_via_run(vo, second_vo, scope_factory, caches_mock, file_config_mock):
     """ MULTI VO (DAEMON): Test that reaper runs on the specified VO(s) """
     [cache_region] = caches_mock
     new_vo = second_vo
@@ -171,10 +191,15 @@ def test_reaper_multi_vo_via_run(vo, second_vo, scope_factory, caches_mock):
     assert len(list(replica_api.list_replicas([{'scope': scope_name, 'name': n} for n in names], rse_expression=rse_name, vo=new_vo))) == 25
 
 
+@pytest.mark.parametrize("file_config_mock", [
+    # Run test twice: with, and without, temp tables
+    {"overrides": [('core', 'use_temp_tables', 'True')]},
+    {"overrides": [('core', 'use_temp_tables', 'False')]},
+], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.daemons.reaper.reaper.REGION'
 ]}], indirect=True)
-def test_reaper_affect_other_vo_via_run(vo, second_vo, scope_factory, caches_mock):
+def test_reaper_affect_other_vo_via_run(vo, second_vo, scope_factory, caches_mock, file_config_mock):
     """ MULTI VO (DAEMON): Test that reaper runs on the specified VO(s) and does not reap others"""
     [cache_region] = caches_mock
     new_vo = second_vo
@@ -206,10 +231,15 @@ def test_reaper_affect_other_vo_via_run(vo, second_vo, scope_factory, caches_moc
     assert len(list(replica_api.list_replicas([{'scope': scope_name, 'name': n} for n in names], rse_expression=rse_name, vo=new_vo))) == 25
 
 
+@pytest.mark.parametrize("file_config_mock", [
+    # Run test twice: with, and without, temp tables
+    {"overrides": [('core', 'use_temp_tables', 'True')]},
+    {"overrides": [('core', 'use_temp_tables', 'False')]},
+], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.daemons.reaper.reaper.REGION'
 ]}], indirect=True)
-def test_reaper_multi_vo(vo, second_vo, scope_factory, caches_mock):
+def test_reaper_multi_vo(vo, second_vo, scope_factory, caches_mock, file_config_mock):
     """ REAPER (DAEMON): Test the reaper daemon with multiple vo."""
     [cache_region] = caches_mock
     new_vo = second_vo
@@ -236,10 +266,15 @@ def test_reaper_multi_vo(vo, second_vo, scope_factory, caches_mock):
     assert len(list(replica_core.list_replicas(dids=dids2, rse_expression=both_rses))) == 200
 
 
+@pytest.mark.parametrize("file_config_mock", [
+    # Run test twice: with, and without, temp tables
+    {"overrides": [('core', 'use_temp_tables', 'True')]},
+    {"overrides": [('core', 'use_temp_tables', 'False')]},
+], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.daemons.reaper.reaper.REGION'
 ]}], indirect=True)
-def test_archive_removal_impact_on_constituents(rse_factory, did_factory, mock_scope, root_account, caches_mock):
+def test_archive_removal_impact_on_constituents(rse_factory, did_factory, mock_scope, root_account, caches_mock, file_config_mock):
     [cache_region] = caches_mock
     rse_name, rse_id = rse_factory.make_mock_rse()
     scope = mock_scope
@@ -367,6 +402,11 @@ def __get_archive_contents_history_count(archive, session=None):
     assert __get_archive_contents_history_count(archive2) == 3
 
 
+@pytest.mark.parametrize("file_config_mock", [
+    # Run test twice: with, and without, temp tables
+    {"overrides": [('core', 'use_temp_tables', 'True')]},
+    {"overrides": [('core', 'use_temp_tables', 'False')]},
+], indirect=True)
 @pytest.mark.parametrize("core_config_mock", [{"table_content": [
     ('deletion', 'archive_dids', True), ('deletion', 'archive_content', True)
 ]}], indirect=True)
@@ -375,7 +415,7 @@ def __get_archive_contents_history_count(archive, session=None):
     'rucio.core.config.REGION',
     'rucio.core.replica.REGION',
 ]}], indirect=True)
-def test_archive_of_deleted_dids(vo, did_factory, root_account, core_config_mock, caches_mock):
+def test_archive_of_deleted_dids(vo, did_factory, root_account, core_config_mock, caches_mock, file_config_mock):
     """ REAPER (DAEMON): Test that the options to keep the did and content history work."""
     [reaper_cache_region, _config_cache_region, _replica_cache_region] = caches_mock
     scope = InternalScope('data13_hip', vo=vo)