Repository: rucio/rucio

Issue:
allow prioritization between two multihops
Motivation
----------

Right now we don't have any possibility to prioritize one out of multihop paths. 

Assume the transfer between A and B; And that two paths are possible: 

```
A->C->B 
A->D->B
```

Even if costs A->C + C->B  == A->D + D->B , in some cases we would like to prioritize one of the paths. 

Modification
------------

A solution would be to make the hop_penalty also available as an RSE attribute for C and D 




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -58,7 +58,7 @@
 
 from rucio.common import constants
 from rucio.common.cache import make_region_memcached
-from rucio.common.config import config_get
+from rucio.common.config import config_get, config_get_int
 from rucio.common.constants import SUPPORTED_PROTOCOLS
 from rucio.common.exception import (InvalidRSEExpression, NoDistance,
                                     RequestNotFound, RSEProtocolNotSupported,
@@ -70,7 +70,7 @@
 from rucio.core.config import get as core_config_get
 from rucio.core.monitor import record_counter
 from rucio.core.request import set_request_state, RequestWithSources, RequestSource
-from rucio.core.rse import get_rse_name, get_rse_vo, list_rses
+from rucio.core.rse import list_rses
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import DIDType, RequestState, RequestType
@@ -569,7 +569,8 @@ def get_hops(source_rse_id, dest_rse_id, multihop_rses=None, limit_dest_schemes=
     if not multihop_rses:
         multihop_rses = []
 
-    shortest_paths = __search_shortest_paths(source_rse_ids=[source_rse_id], dest_rse_id=dest_rse_id,
+    ctx = _RseLoaderContext(session)
+    shortest_paths = __search_shortest_paths(ctx=ctx, source_rse_ids=[source_rse_id], dest_rse_id=dest_rse_id,
                                              operation_src='third_party_copy_read', operation_dest='third_party_copy_write',
                                              domain='wan', multihop_rses=multihop_rses,
                                              limit_dest_schemes=limit_dest_schemes, session=session)
@@ -590,6 +591,7 @@ def get_hops(source_rse_id, dest_rse_id, multihop_rses=None, limit_dest_schemes=
 
 
 def __search_shortest_paths(
+        ctx: _RseLoaderContext,
         source_rse_ids: "List[str]",
         dest_rse_id: "List[str]",
         operation_src: str,
@@ -609,7 +611,7 @@ def __search_shortest_paths(
     The inbound links retrieved from the database can be accumulated into the inbound_links_by_node, passed
     from the calling context. To be able to reuse them.
     """
-    HOP_PENALTY = config_get('transfers', 'hop_penalty', default=10, session=session)  # Penalty to be applied to each further hop
+    HOP_PENALTY = config_get_int('transfers', 'hop_penalty', default=10, session=session)  # Penalty to be applied to each further hop
 
     if multihop_rses:
         # Filter out island source RSEs
@@ -643,14 +645,18 @@ def __search_shortest_paths(
             if adjacent_node not in remaining_sources and adjacent_node not in multihop_rses:
                 continue
 
-            new_adjacent_distance = current_distance + link_distance + HOP_PENALTY
+            try:
+                hop_penalty = int(ctx.rse_data(adjacent_node).attributes.get('hop_penalty', HOP_PENALTY))
+            except ValueError:
+                hop_penalty = HOP_PENALTY
+            new_adjacent_distance = current_distance + link_distance + hop_penalty
             if next_hop.get(adjacent_node, {}).get('cumulated_distance', 9999) <= new_adjacent_distance:
                 continue
 
             try:
                 matching_scheme = rsemgr.find_matching_scheme(
-                    rse_settings_src=__load_rse_settings(rse_id=adjacent_node, session=session),
-                    rse_settings_dest=__load_rse_settings(rse_id=current_node, session=session),
+                    rse_settings_src=ctx.rse_data(adjacent_node).info,
+                    rse_settings_dest=ctx.rse_data(current_node).info,
                     operation_src=operation_src,
                     operation_dest=operation_dest,
                     domain=domain,
@@ -707,7 +713,7 @@ def __create_transfer_definitions(
     Create the transfer definitions for each point-to-point transfer (multi-source, when possible)
     """
     inbound_links_by_node = {}
-    shortest_paths = __search_shortest_paths(source_rse_ids=[s.rse.id for s in sources], dest_rse_id=rws.dest_rse.id,
+    shortest_paths = __search_shortest_paths(ctx=ctx, source_rse_ids=[s.rse.id for s in sources], dest_rse_id=rws.dest_rse.id,
                                              operation_src=operation_src, operation_dest=operation_dest, domain=domain,
                                              multihop_rses=multihop_rses, limit_dest_schemes=limit_dest_schemes,
                                              inbound_links_by_node=inbound_links_by_node, session=session)
@@ -1270,24 +1276,6 @@ def __load_outgoing_distances_node(rse_id, session=None):
     return result
 
 
-@transactional_session
-def __load_rse_settings(rse_id, session=None):
-    """
-    Loads the RSE settings from cache.
-    :param rse_id:    RSE id to load the settings from.
-    :param session:   The DB Session to use.
-    :returns:         Dict of RSE Settings
-    """
-
-    result = REGION_SHORT.get('rse_settings_%s' % str(rse_id))
-    if isinstance(result, NoValue):
-        result = rsemgr.get_rse_info(rse=get_rse_name(rse_id=rse_id, session=session),
-                                     vo=get_rse_vo(rse_id=rse_id, session=session),
-                                     session=session)
-        REGION_SHORT.set('rse_settings_%s' % str(rse_id), result)
-    return result
-
-
 @read_session
 def __list_admin_accounts(session=None):
     """


Test patch:
diff --git a/lib/rucio/tests/test_conveyor_submitter.py b/lib/rucio/tests/test_conveyor_submitter.py
--- a/lib/rucio/tests/test_conveyor_submitter.py
+++ b/lib/rucio/tests/test_conveyor_submitter.py
@@ -24,6 +24,7 @@
 from unittest.mock import patch
 from sqlalchemy import delete
 
+from rucio.common.exception import RequestNotFound
 from rucio.core import distance as distance_core
 from rucio.core import request as request_core
 from rucio.core import rse as rse_core
@@ -209,11 +210,12 @@ def __number_sources(rse_id, scope, name, session=None):
 @pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
     'rucio.core.config.REGION',
     'rucio.daemons.reaper.reaper.REGION',
+    'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by an expression
 ]}], indirect=True)
-def test_source_avoid_deletion(vo, caches_mock, core_config_mock, rse_factory, did_factory, root_account, file_factory):
+def test_source_avoid_deletion(caches_mock, core_config_mock, rse_factory, did_factory, root_account):
     """ Test that sources on a file block it from deletion """
 
-    _, reaper_region = caches_mock
+    _, reaper_region, _ = caches_mock
     src_rse1, src_rse1_id = rse_factory.make_mock_rse()
     src_rse2, src_rse2_id = rse_factory.make_mock_rse()
     dst_rse, dst_rse_id = rse_factory.make_mock_rse()
@@ -377,3 +379,59 @@ def test_globus(rse_factory, did_factory, root_account):
     assert request['state'] == RequestState.SUBMITTED
     request = request_core.get_request_by_did(rse_id=rse4_id, **did2)
     assert request['state'] == RequestState.SUBMITTED
+
+
+@pytest.mark.noparallel(reason="multiple submitters cannot be run in parallel due to partial job assignment by hash")
+@pytest.mark.parametrize("file_config_mock", [{"overrides": [
+    ('transfers', 'hop_penalty', '5'),
+]}], indirect=True)
+@pytest.mark.parametrize("core_config_mock", [{"table_content": [
+    ('transfers', 'use_multihop', True),
+]}], indirect=True)
+@pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
+    'rucio.core.config.REGION',
+    'rucio.daemons.reaper.reaper.REGION',
+    'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by an expression
+]}], indirect=True)
+def test_hop_penalty(rse_factory, did_factory, root_account, file_config_mock, core_config_mock, caches_mock):
+    """
+    Test that both global hop_penalty and the per-rse one are correctly taken into consideration
+    """
+    # +------+    +------+    +------+
+    # |      |    |  5   |    |      |
+    # | RSE1 +--->| RSE2 +--->| RSE3 |
+    # |      |    |      |    |      |
+    # +------+    +------+    +--^---+
+    #                            |
+    # +------+    +------+       |
+    # |      |    |  20  |       |
+    # | RSE4 +--->| RSE5 +-------+
+    # |      |    |      |
+    # +------+    +------+
+    rse1, rse1_id = rse_factory.make_posix_rse()
+    rse2, rse2_id = rse_factory.make_posix_rse()
+    rse3, rse3_id = rse_factory.make_posix_rse()
+    rse4, rse4_id = rse_factory.make_posix_rse()
+    rse5, rse5_id = rse_factory.make_posix_rse()
+    all_rses = [rse1_id, rse2_id, rse3_id, rse4_id, rse5_id]
+
+    distance_core.add_distance(rse1_id, rse2_id, ranking=10)
+    distance_core.add_distance(rse2_id, rse3_id, ranking=10)
+    distance_core.add_distance(rse4_id, rse5_id, ranking=10)
+    distance_core.add_distance(rse5_id, rse3_id, ranking=10)
+
+    rse_core.add_rse_attribute(rse2_id, 'available_for_multihop', True)
+    rse_core.add_rse_attribute(rse5_id, 'available_for_multihop', True)
+    rse_core.add_rse_attribute(rse5_id, 'hop_penalty', 20)
+
+    did = did_factory.random_did()
+    replica_core.add_replica(rse_id=rse1_id, account=root_account, bytes_=1, **did)
+    replica_core.add_replica(rse_id=rse4_id, account=root_account, bytes_=1, **did)
+
+    rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=rse3, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], partition_wait_time=None, transfertool='mock', transfertype='single', ignore_availability=True)
+
+    # Ensure the path was created through the correct middle hop
+    request_core.get_request_by_did(rse_id=rse2_id, **did)
+    with pytest.raises(RequestNotFound):
+        request_core.get_request_by_did(rse_id=rse5_id, **did)