Repository: rucio/rucio

Issue:
allow configuring intermediate hop RSE expression
RSEs allowed to be used as an intermediate hop are selected using a hard-coded RSE expression:
https://github.com/rucio/rucio/blob/ac95ef3fabb8e1fcabddf1075cc19ae06db44a34/lib/rucio/core/topology.py#L73

We would like to allow this expression to be configurable. In particular, this would allow to have a dynamic list of RSEs, which depends on their utilisation. For example: `available_for_multihop=true&freespace>500|BNLHPC_DATADISK` 

This would also allow people to activate multihop globally with one configuration value : `*` for the RSE expression. Instead of having to set an attribute on each RSE


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/topology.py b/lib/rucio/core/topology.py
--- a/lib/rucio/core/topology.py
+++ b/lib/rucio/core/topology.py
@@ -23,10 +23,9 @@
 
 from rucio.common.utils import PriorityQueue
 from rucio.common.cache import make_region_memcached
-from rucio.common.config import config_get_int
+from rucio.common.config import config_get_int, config_get
 from rucio.common.exception import NoDistance, RSEProtocolNotSupported, InvalidRSEExpression
 from rucio.core.rse import RseCollection, list_rses
-from rucio.core.config import get as core_config_get
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.db.sqla import models
 from rucio.db.sqla.session import read_session, transactional_session
@@ -65,14 +64,17 @@ def __init__(
     @read_session
     def create_from_config(cls, ignore_availability: bool = False, *, session: "Session", logger: "LoggerFunction" = logging.log):
 
-        include_multihop = core_config_get('transfers', 'use_multihop', default=False, expiration_time=600, session=session)
+        include_multihop = config_get('transfers', 'use_multihop', default=False, expiration_time=600, session=session)
+        multihop_rse_expression = config_get('transfers', 'multihop_rse_expression', default='available_for_multihop=true', expiration_time=600, session=session)
 
         multihop_rses = set()
         if include_multihop:
             try:
-                multihop_rses = {rse['id'] for rse in parse_expression('available_for_multihop=true', session=session)}
+                multihop_rses = {rse['id'] for rse in parse_expression(multihop_rse_expression, session=session)}
             except InvalidRSEExpression:
                 pass
+            if multihop_rse_expression and multihop_rse_expression.strip() and not multihop_rses:
+                logger(logging.WARNING, 'multihop_rse_expression is not empty, but returned no RSEs')
 
         restricted_read_rses = set()
         try:


Test patch:
diff --git a/lib/rucio/tests/test_conveyor.py b/lib/rucio/tests/test_conveyor.py
--- a/lib/rucio/tests/test_conveyor.py
+++ b/lib/rucio/tests/test_conveyor.py
@@ -1389,6 +1389,7 @@ def bulk_query(self, requests_by_eid, timeout=None):
     {"table_content": [
         ('transfers', 'use_multihop', True),
         ('transfers', 'multihop_tombstone_delay', -1),  # Set OBSOLETE tombstone for intermediate replicas
+        ('transfers', 'multihop_rse_expression', '*'),
     ]},
 ], indirect=True)
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
@@ -1422,7 +1423,6 @@ def test_two_multihops_same_intermediate_rse(rse_factory, did_factory, root_acco
     all_rses = [rse1_id, rse2_id, rse3_id, rse4_id, rse5_id, rse6_id, rse7_id]
     for rse_id in all_rses:
         rse_core.add_rse_attribute(rse_id, 'fts', TEST_FTS_HOST)
-        rse_core.add_rse_attribute(rse_id, 'available_for_multihop', True)
         rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=1)
         rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=1, free=0)
     distance_core.add_distance(rse1_id, rse2_id, distance=10)
diff --git a/lib/rucio/tests/test_conveyor_submitter.py b/lib/rucio/tests/test_conveyor_submitter.py
--- a/lib/rucio/tests/test_conveyor_submitter.py
+++ b/lib/rucio/tests/test_conveyor_submitter.py
@@ -112,12 +112,14 @@ def _forge_requests_creation_time(*, session=None):
     {
         "table_content": [
             ('transfers', 'use_multihop', True),
+            ('transfers', 'multihop_rse_expression', '*'),
             ('core', 'use_temp_tables', False),
         ]
     },
     {
         "table_content": [
             ('transfers', 'use_multihop', True),
+            ('transfers', 'multihop_rse_expression', '*'),
             ('core', 'use_temp_tables', True),
         ]
     }
@@ -139,9 +141,6 @@ def test_multihop_sources_created(rse_factory, did_factory, root_account, core_c
     jump_rses = [jump_rse1_id, jump_rse2_id, jump_rse3_id]
     all_rses = jump_rses + [src_rse_id, dst_rse_id]
 
-    for rse_id in jump_rses:
-        rse_core.add_rse_attribute(rse_id, 'available_for_multihop', True)
-
     rse_tombstone_delay = 3600
     rse_multihop_tombstone_delay = 12 * 3600
     default_multihop_tombstone_delay = 24 * 3600