Repository: rucio/rucio

Issue:
get rid of the `use_multihop` configuration option
This is to be done in a major rucio release. 

The configuration option is redundant since https://github.com/rucio/rucio/issues/6163 . To disable multihop, it's enough to set it to an empty or invalid rse expression.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/topology.py b/lib/rucio/core/topology.py
--- a/lib/rucio/core/topology.py
+++ b/lib/rucio/core/topology.py
@@ -170,16 +170,15 @@ def configure_multihop(self, multihop_rse_ids: Optional[Set[str]] = None, *, ses
     def _configure_multihop(self, multihop_rse_ids: Optional[Set[str]] = None, *, session: "Session", logger: LoggerFunction = logging.log):
 
         if multihop_rse_ids is None:
-            include_multihop = config_get('transfers', 'use_multihop', default=False, expiration_time=600, session=session)
             multihop_rse_expression = config_get('transfers', 'multihop_rse_expression', default='available_for_multihop=true', expiration_time=600, session=session)
 
             multihop_rse_ids = set()
-            if include_multihop:
+            if multihop_rse_expression.strip():
                 try:
                     multihop_rse_ids = {rse['id'] for rse in parse_expression(multihop_rse_expression, session=session)}
                 except InvalidRSEExpression:
                     pass
-                if multihop_rse_expression and multihop_rse_expression.strip() and not multihop_rse_ids:
+                if not multihop_rse_ids:
                     logger(logging.WARNING, 'multihop_rse_expression is not empty, but returned no RSEs')
 
         for node in self._multihop_nodes:


Test patch:
diff --git a/lib/rucio/tests/test_conveyor.py b/lib/rucio/tests/test_conveyor.py
--- a/lib/rucio/tests/test_conveyor.py
+++ b/lib/rucio/tests/test_conveyor.py
@@ -121,7 +121,6 @@ def __get_source(request_id, src_rse_id, scope, name, *, session=None):
 @pytest.mark.dirty(reason="leaves files in XRD containers")
 @pytest.mark.noparallel(reason="uses predefined RSEs; runs submitter, poller and finisher; changes XRD3 usage and limits")
 @pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True),
     ('transfers', 'multihop_tombstone_delay', -1),  # Set OBSOLETE tombstone for intermediate replicas
 ]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
@@ -233,14 +232,10 @@ def _cleanup_all_usage_and_limits(rse_id, *, session=None):
 
 @skip_rse_tests_with_accounts
 @pytest.mark.noparallel(reason="uses predefined RSEs; runs submitter, poller and finisher")
-@pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True),
-]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by rse expression
-    'rucio.core.config.REGION',
 ]}], indirect=True)
-def test_fts_non_recoverable_failures_handled_on_multihop(vo, did_factory, root_account, replica_client, core_config_mock, caches_mock, metrics_mock):
+def test_fts_non_recoverable_failures_handled_on_multihop(vo, did_factory, root_account, replica_client, caches_mock, metrics_mock):
     """
     Verify that the poller correctly handles non-recoverable FTS job failures
     """
@@ -286,14 +281,10 @@ def test_fts_non_recoverable_failures_handled_on_multihop(vo, did_factory, root_
 @skip_rse_tests_with_accounts
 @pytest.mark.dirty(reason="leaves files in XRD containers")
 @pytest.mark.noparallel(reason="uses predefined RSEs; runs submitter, poller and finisher")
-@pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True),
-]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by rse expression
-    'rucio.core.config.REGION',
 ]}], indirect=True)
-def test_fts_recoverable_failures_handled_on_multihop(vo, did_factory, root_account, replica_client, file_factory, core_config_mock, caches_mock, metrics_mock):
+def test_fts_recoverable_failures_handled_on_multihop(vo, did_factory, root_account, replica_client, file_factory, caches_mock, metrics_mock):
     """
     Verify that the poller correctly handles recoverable FTS job failures
     """
@@ -338,14 +329,10 @@ def test_fts_recoverable_failures_handled_on_multihop(vo, did_factory, root_acco
 @skip_rse_tests_with_accounts
 @pytest.mark.dirty(reason="leaves files in XRD containers")
 @pytest.mark.noparallel(reason="uses predefined RSEs; runs submitter, poller and finisher")
-@pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True),
-]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by rse expression
-    'rucio.core.config.REGION',
 ]}], indirect=True)
-def test_multisource(vo, did_factory, root_account, replica_client, core_config_mock, caches_mock, metrics_mock):
+def test_multisource(vo, did_factory, root_account, replica_client, caches_mock, metrics_mock):
     src_rse1 = 'XRD4'
     src_rse1_id = rse_core.get_rse_id(rse=src_rse1, vo=vo)
     src_rse2 = 'XRD1'
@@ -483,14 +470,10 @@ def test_multisource_receiver(vo, did_factory, replica_client, root_account, met
 
 @skip_rse_tests_with_accounts
 @pytest.mark.noparallel(reason="uses predefined RSEs; runs submitter and receiver")
-@pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True),
-]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by rse expression
-    'rucio.core.config.REGION',
 ]}], indirect=True)
-def test_multihop_receiver_on_failure(vo, did_factory, replica_client, root_account, core_config_mock, caches_mock, metrics_mock):
+def test_multihop_receiver_on_failure(vo, did_factory, replica_client, root_account, caches_mock, metrics_mock):
     """
     Verify that the receiver correctly handles multihop jobs which fail
     """
@@ -541,14 +524,10 @@ def test_multihop_receiver_on_failure(vo, did_factory, replica_client, root_acco
 
 @skip_rse_tests_with_accounts
 @pytest.mark.noparallel(reason="uses predefined RSEs; runs submitter and receiver")
-@pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True),
-]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by rse expression
-    'rucio.core.config.REGION',
 ]}], indirect=True)
-def test_multihop_receiver_on_success(vo, did_factory, root_account, core_config_mock, caches_mock, metrics_mock):
+def test_multihop_receiver_on_success(vo, did_factory, root_account, caches_mock, metrics_mock):
     """
     Verify that the receiver correctly handles successful multihop jobs
     """
@@ -592,7 +571,6 @@ def test_multihop_receiver_on_success(vo, did_factory, root_account, core_config
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse.REGION',
     'rucio.core.rse_expression_parser.REGION',
-    'rucio.core.config.REGION',
     'rucio.rse.rsemanager.RSE_REGION',  # for RSE info
 ]}], indirect=True)
 def test_receiver_archiving(vo, did_factory, root_account, caches_mock):
@@ -1123,16 +1101,12 @@ def __create_dids(did1_corrupted=True, did2_corrupted=True):
 @skip_rse_tests_with_accounts
 @pytest.mark.dirty(reason="leaves files in XRD containers")
 @pytest.mark.noparallel(reason="runs submitter; poller and finisher")
-@pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True)
-]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse.REGION',
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by an expression
-    'rucio.core.config.REGION',
     'rucio.rse.rsemanager.RSE_REGION',  # for RSE info
 ]}], indirect=True)
-def test_overwrite_on_tape(overwrite_on_tape_topology, core_config_mock, caches_mock):
+def test_overwrite_on_tape(overwrite_on_tape_topology, caches_mock):
     """
     Ensure that overwrite is not set for transfers towards TAPE RSEs
     """
@@ -1152,16 +1126,12 @@ def test_overwrite_on_tape(overwrite_on_tape_topology, core_config_mock, caches_
 @skip_rse_tests_with_accounts
 @pytest.mark.dirty(reason="leaves files in XRD containers")
 @pytest.mark.noparallel(reason="runs submitter; poller and finisher")
-@pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True)
-]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse.REGION',
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by an expression
-    'rucio.core.config.REGION',
     'rucio.rse.rsemanager.RSE_REGION',  # for RSE info
 ]}], indirect=True)
-def test_overwrite_hops(overwrite_on_tape_topology, core_config_mock, caches_mock, did_factory, file_factory):
+def test_overwrite_hops(overwrite_on_tape_topology, caches_mock, did_factory, file_factory):
     """
     Ensure that we request overwrite of intermediate hops on multi-hop transfers towards TAPE RSEs
     """
@@ -1204,16 +1174,13 @@ def test_overwrite_hops(overwrite_on_tape_topology, core_config_mock, caches_moc
 @skip_rse_tests_with_accounts
 @pytest.mark.dirty(reason="leaves files in XRD containers")
 @pytest.mark.noparallel(reason="runs submitter; poller and finisher")
-@pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True)
-]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse.REGION',
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by an expression
     'rucio.core.config.REGION',
     'rucio.rse.rsemanager.RSE_REGION',  # for RSE info
 ]}], indirect=True)
-def test_file_exists_handled(overwrite_on_tape_topology, core_config_mock, caches_mock):
+def test_file_exists_handled(overwrite_on_tape_topology, caches_mock):
     """
     If a transfer fails because the destination job_params exists, and the size+checksums of that existing job_params
     are correct, the transfer must be marked successful.
@@ -1246,7 +1213,6 @@ def on_receive(job_params):
 @pytest.mark.dirty(reason="leaves files in XRD containers; leaves pending fts transfers in archiving state")
 @pytest.mark.noparallel(reason="runs submitter; poller and finisher")
 @pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True),
     ('transfers', 'overwrite_corrupted_files', False)
 ]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
@@ -1374,7 +1340,6 @@ def bulk_query(self, requests_by_eid, timeout=None):
 @pytest.mark.noparallel(reason="runs submitter; poller and finisher")
 @pytest.mark.parametrize("core_config_mock", [
     {"table_content": [
-        ('transfers', 'use_multihop', True),
         ('transfers', 'multihop_tombstone_delay', -1),  # Set OBSOLETE tombstone for intermediate replicas
         ('transfers', 'multihop_rse_expression', '*'),
     ]},
diff --git a/lib/rucio/tests/test_conveyor_submitter.py b/lib/rucio/tests/test_conveyor_submitter.py
--- a/lib/rucio/tests/test_conveyor_submitter.py
+++ b/lib/rucio/tests/test_conveyor_submitter.py
@@ -98,10 +98,9 @@ def _forge_requests_creation_time(*, session=None):
     # Run test twice: with, and without, temp tables
     {
         "table_content": [
-            ('transfers', 'use_multihop', True),
             ('transfers', 'multihop_rse_expression', '*'),
         ]
-    },
+    }
 ], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by rse expression
@@ -179,18 +178,14 @@ def __number_sources(rse_id, scope, name, *, session=None):
 
 
 @pytest.mark.noparallel(reason="multiple submitters cannot be run in parallel due to partial job assignment by hash")
-@pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True),
-]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
-    'rucio.core.config.REGION',
     'rucio.daemons.reaper.reaper.REGION',
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by an expression
 ]}], indirect=True)
-def test_source_avoid_deletion(caches_mock, core_config_mock, rse_factory, did_factory, root_account):
+def test_source_avoid_deletion(caches_mock, rse_factory, did_factory, root_account):
     """ Test that sources on a file block it from deletion """
 
-    _, reaper_region, _ = caches_mock
+    reaper_region, _ = caches_mock
     src_rse1, src_rse1_id = rse_factory.make_mock_rse()
     src_rse2, src_rse2_id = rse_factory.make_mock_rse()
     dst_rse, dst_rse_id = rse_factory.make_mock_rse()
@@ -247,13 +242,10 @@ def __delete_sources(rse_id, scope, name, *, session=None):
 
 
 @pytest.mark.noparallel(reason="multiple submitters cannot be run in parallel due to partial job assignment by hash")
-@pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True)
-]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by rse expression
 ]}], indirect=True)
-def test_ignore_availability(rse_factory, did_factory, root_account, core_config_mock, caches_mock):
+def test_ignore_availability(rse_factory, did_factory, root_account, caches_mock):
 
     def __setup_test():
         src_rse, src_rse_id = rse_factory.make_posix_rse()
@@ -381,15 +373,11 @@ def test_globus(rse_factory, did_factory, root_account):
 @pytest.mark.parametrize("file_config_mock", [{"overrides": [
     ('transfers', 'hop_penalty', '5'),
 ]}], indirect=True)
-@pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True),
-]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
-    'rucio.core.config.REGION',
     'rucio.daemons.reaper.reaper.REGION',
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by an expression
 ]}], indirect=True)
-def test_hop_penalty(rse_factory, did_factory, root_account, file_config_mock, core_config_mock, caches_mock):
+def test_hop_penalty(rse_factory, did_factory, root_account, file_config_mock, caches_mock):
     """
     Test that both global hop_penalty and the per-rse one are correctly taken into consideration
     """
diff --git a/lib/rucio/tests/test_transfer.py b/lib/rucio/tests/test_transfer.py
--- a/lib/rucio/tests/test_transfer.py
+++ b/lib/rucio/tests/test_transfer.py
@@ -215,14 +215,10 @@ def test_disk_vs_tape_priority(rse_factory, root_account, mock_scope):
     assert transfer[0].legacy_sources[0][0] == tape1_rse_name
 
 
-@pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True)
-]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by an expression
-    'rucio.core.config.REGION',
 ]}], indirect=True)
-def test_multihop_requests_created(rse_factory, did_factory, root_account, core_config_mock, caches_mock):
+def test_multihop_requests_created(rse_factory, did_factory, root_account, caches_mock):
     """
     Ensure that multihop transfers are handled and intermediate request correctly created
     """
@@ -242,14 +238,10 @@ def test_multihop_requests_created(rse_factory, did_factory, root_account, core_
     assert request_core.get_request_by_did(rse_id=intermediate_rse_id, **did)
 
 
-@pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True)
-]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by an expression
-    'rucio.core.config.REGION',
 ]}], indirect=True)
-def test_multihop_concurrent_submitters(rse_factory, did_factory, root_account, core_config_mock, caches_mock):
+def test_multihop_concurrent_submitters(rse_factory, did_factory, root_account, caches_mock):
     """
     Ensure that multiple concurrent submitters on the same multi-hop don't result in an undesired database state
     """
@@ -281,14 +273,10 @@ def test_multihop_concurrent_submitters(rse_factory, did_factory, root_account,
     assert jmp_request['attributes']['is_intermediate_hop']
 
 
-@pytest.mark.parametrize("core_config_mock", [{"table_content": [
-    ('transfers', 'use_multihop', True)
-]}], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by an expression
-    'rucio.core.config.REGION',
 ]}], indirect=True)
-def test_singlehop_vs_multihop_priority(rse_factory, root_account, mock_scope, core_config_mock, caches_mock):
+def test_singlehop_vs_multihop_priority(rse_factory, root_account, mock_scope, caches_mock):
     """
     On small distance difference, singlehop is prioritized over multihop
     due to HOP_PENALTY. On big difference, multihop is prioritized