Repository: rucio/rucio

Issue:
Bug in transfer core when no outbound distances
Motivation
----------
If an RSE has no outgoing links defined, then the code at https://github.com/rucio/rucio/blob/f711667ee4a4ea8c898098f8380c311460dc2642/lib/rucio/core/transfer.py#L1431-L1444 returns an empty dictionary, and the calling code at https://github.com/rucio/rucio/blob/f711667ee4a4ea8c898098f8380c311460dc2642/lib/rucio/core/transfer.py#L564 raises a KeyError.


Modification
------------
I imagine setting `distance_graph = {rse_id: {}}` would fix this but I am not familiar with the calling code logic and how best to handle this error.




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -510,7 +510,8 @@ def get_hops(source_rse_id, dest_rse_id, include_multihop=False, multihop_rses=N
 
     # Load the graph from the distances table
     # distance_graph = __load_distance_graph(session=session)
-    distance_graph = __load_distance_edges_node(rse_id=source_rse_id, session=session)
+    distance_graph = {}
+    distance_graph[source_rse_id] = __load_outgoing_distances_node(rse_id=source_rse_id, session=session)
 
     # 1. Check if there is a direct connection between source and dest:
     if distance_graph.get(source_rse_id, {dest_rse_id: None}).get(dest_rse_id) is not None:
@@ -544,8 +545,7 @@ def get_hops(source_rse_id, dest_rse_id, include_multihop=False, multihop_rses=N
     HOP_PENALTY = core_config_get('transfers', 'hop_penalty', default=10, session=session)  # Penalty to be applied to each further hop
 
     # Check if the destination RSE is an island RSE:
-    if not __load_distance_edges_node(rse_id=dest_rse_id, session=session):
-        # The destination RSE is an island
+    if not __load_inbound_distances_node(rse_id=dest_rse_id, session=session):
         raise NoDistance()
 
     visited_nodes = {source_rse_id: {'distance': 0,
@@ -563,7 +563,7 @@ def get_hops(source_rse_id, dest_rse_id, include_multihop=False, multihop_rses=N
             current_path = visited_nodes[current_node]['path']
 
             if current_node not in distance_graph:
-                distance_graph[current_node] = __load_distance_edges_node(rse_id=current_node, session=session)[current_node]
+                distance_graph[current_node] = __load_outgoing_distances_node(rse_id=current_node, session=session)
 
             for out_v in distance_graph[current_node]:
                 # Check if the distance would be smaller
@@ -1472,7 +1472,31 @@ def __add_compatible_schemes(schemes, allowed_schemes):
 
 
 @transactional_session
-def __load_distance_edges_node(rse_id, session=None):
+def __load_inbound_distances_node(rse_id, session=None):
+    """
+    Loads the inbound edges of the distance graph for one node.
+    :param rse_id:    RSE id to load the edges for.
+    :param session:   The DB Session to use.
+    :returns:         Dictionary based graph object.
+    """
+
+    result = REGION_SHORT.get('inbound_edges_%s' % str(rse_id))
+    if isinstance(result, NoValue):
+        inbound_edges = {}
+        for distance in session.query(models.Distance).join(models.RSE, models.RSE.id == models.Distance.src_rse_id) \
+                .filter(models.Distance.dest_rse_id == rse_id) \
+                .filter(models.RSE.deleted == false()).all():
+            if distance.ranking is None:
+                continue
+            ranking = distance.ranking if distance.ranking >= 0 else 0
+            inbound_edges[distance.src_rse_id] = ranking
+        REGION_SHORT.set('inbound_edges_%s' % str(rse_id), inbound_edges)
+        result = inbound_edges
+    return result
+
+
+@transactional_session
+def __load_outgoing_distances_node(rse_id, session=None):
     """
     Loads the outgoing edges of the distance graph for one node.
     :param rse_id:    RSE id to load the edges for.
@@ -1480,21 +1504,18 @@ def __load_distance_edges_node(rse_id, session=None):
     :returns:         Dictionary based graph object.
     """
 
-    result = REGION_SHORT.get('distance_graph_%s' % str(rse_id))
+    result = REGION_SHORT.get('outgoing_edges_%s' % str(rse_id))
     if isinstance(result, NoValue):
-        distance_graph = {}
+        outgoing_edges = {}
         for distance in session.query(models.Distance).join(models.RSE, models.RSE.id == models.Distance.dest_rse_id)\
                                .filter(models.Distance.src_rse_id == rse_id)\
                                .filter(models.RSE.deleted == false()).all():
             if distance.ranking is None:
                 continue
             ranking = distance.ranking if distance.ranking >= 0 else 0
-            if distance.src_rse_id in distance_graph:
-                distance_graph[distance.src_rse_id][distance.dest_rse_id] = ranking
-            else:
-                distance_graph[distance.src_rse_id] = {distance.dest_rse_id: ranking}
-        REGION_SHORT.set('distance_graph_%s' % str(rse_id), distance_graph)
-        result = distance_graph
+            outgoing_edges[distance.dest_rse_id] = ranking
+        REGION_SHORT.set('outgoing_edges_%s' % str(rse_id), outgoing_edges)
+        result = outgoing_edges
     return result
 
 


Test patch:
diff --git a/lib/rucio/tests/test_transfer.py b/lib/rucio/tests/test_transfer.py
new file mode 100644
--- /dev/null
+++ b/lib/rucio/tests/test_transfer.py
@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+# Copyright 2021 CERN
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
+
+import pytest
+
+from rucio.common.exception import NoDistance
+from rucio.core.distance import add_distance
+from rucio.core.transfer import get_hops
+
+
+def test_get_hops(rse_factory):
+    # Build the following topology:
+    # +------+           +------+     10    +------+
+    # |      |     40    |      +-----------+      |
+    # | RSE0 |  +--------+ RSE1 |           | RSE2 +-------------+
+    # |      |  |        |      |      +----+      |             |
+    # +------+  |        +------+      |    +------+             | <missing_cost>
+    #           |                      |                         |
+    #           |                      |                         |
+    #           |                      |                         |
+    # +------+  |        +------+  10  |    +------+           +-+----+
+    # |      +--+        |      +------+    |      |   --20->  |      |
+    # | RSE3 |   --10->  | RSE4 |           | RSE5 +-->--->--->+ RSE6 |
+    # |      +-->--->--->+      +-----------+      |           |      |
+    # +----+-+           +------+     10    +-+----+           +------+
+    #      |                                  |
+    #      |                50                |
+    #      +----------------------------------+
+    #
+    _, rse0_id = rse_factory.make_mock_rse()
+    _, rse1_id = rse_factory.make_mock_rse()
+    _, rse2_id = rse_factory.make_mock_rse()
+    _, rse3_id = rse_factory.make_mock_rse()
+    _, rse4_id = rse_factory.make_mock_rse()
+    _, rse5_id = rse_factory.make_mock_rse()
+    _, rse6_id = rse_factory.make_mock_rse()
+    all_rses = [rse0_id, rse1_id, rse2_id, rse3_id, rse4_id, rse5_id, rse6_id]
+
+    add_distance(rse1_id, rse3_id, ranking=40)
+    add_distance(rse1_id, rse2_id, ranking=10)
+
+    add_distance(rse2_id, rse1_id, ranking=10)
+    add_distance(rse2_id, rse4_id, ranking=10)
+
+    add_distance(rse3_id, rse1_id, ranking=40)
+    add_distance(rse3_id, rse4_id, ranking=10)
+    add_distance(rse3_id, rse5_id, ranking=50)
+
+    add_distance(rse4_id, rse2_id, ranking=10)
+    add_distance(rse4_id, rse5_id, ranking=10)
+
+    add_distance(rse5_id, rse3_id, ranking=50)
+    add_distance(rse5_id, rse4_id, ranking=10)
+    add_distance(rse5_id, rse6_id, ranking=20)
+
+    # There must be no paths between an isolated node and other nodes; be it with multipath enabled or disabled
+    with pytest.raises(NoDistance):
+        get_hops(source_rse_id=rse0_id, dest_rse_id=rse1_id)
+    with pytest.raises(NoDistance):
+        get_hops(source_rse_id=rse1_id, dest_rse_id=rse0_id)
+    with pytest.raises(NoDistance):
+        get_hops(source_rse_id=rse0_id, dest_rse_id=rse1_id, include_multihop=True, multihop_rses=all_rses)
+    with pytest.raises(NoDistance):
+        get_hops(source_rse_id=rse1_id, dest_rse_id=rse0_id, include_multihop=True, multihop_rses=all_rses)
+
+    # A single hop path must be found between two directly connected RSE
+    [hop] = get_hops(source_rse_id=rse1_id, dest_rse_id=rse2_id)
+    assert hop['source_rse_id'] == rse1_id
+    assert hop['dest_rse_id'] == rse2_id
+
+    # No path will be found if there is no direct connection and "include_multihop" is not set
+    with pytest.raises(NoDistance):
+        get_hops(source_rse_id=rse3_id, dest_rse_id=rse2_id)
+
+    # Multihop_rses argument empty (not set), no path will be computed
+    with pytest.raises(NoDistance):
+        get_hops(source_rse_id=rse3_id, dest_rse_id=rse2_id, include_multihop=True)
+
+    # The shortest multihop path will be computed
+    [hop1, hop2] = get_hops(source_rse_id=rse3_id, dest_rse_id=rse2_id, include_multihop=True, multihop_rses=all_rses)
+    assert hop1['source_rse_id'] == rse3_id
+    assert hop1['dest_rse_id'] == rse4_id
+    assert hop2['source_rse_id'] == rse4_id
+    assert hop2['dest_rse_id'] == rse2_id
+
+    # multihop_rses doesn't contain the RSE needed for the shortest path. Return a longer path
+    [hop1, hop2] = get_hops(source_rse_id=rse1_id, dest_rse_id=rse4_id, include_multihop=True, multihop_rses=[rse3_id])
+    assert hop1['source_rse_id'] == rse1_id
+    assert hop1['dest_rse_id'] == rse3_id
+    assert hop2['source_rse_id'] == rse3_id
+    assert hop2['dest_rse_id'] == rse4_id
+
+    # A direct connection is preferred over a multihop one with smaller cost
+    [hop] = get_hops(source_rse_id=rse3_id, dest_rse_id=rse5_id, include_multihop=True, multihop_rses=all_rses)
+    assert hop['source_rse_id'] == rse3_id
+    assert hop['dest_rse_id'] == rse5_id
+
+    # A link with cost only in one direction will not be used in the opposite direction
+    with pytest.raises(NoDistance):
+        get_hops(source_rse_id=rse6_id, dest_rse_id=rse5_id, include_multihop=True, multihop_rses=all_rses)
+    [hop1, hop2] = get_hops(source_rse_id=rse4_id, dest_rse_id=rse3_id, include_multihop=True, multihop_rses=all_rses)
+    assert hop1['source_rse_id'] == rse4_id
+    assert hop2['source_rse_id'] == rse5_id
+    assert hop2['dest_rse_id'] == rse3_id
+
+    # A longer path is preferred over a shorter one with high intermediate cost
+    [hop1, hop2, hop3] = get_hops(source_rse_id=rse3_id, dest_rse_id=rse6_id, include_multihop=True, multihop_rses=all_rses)
+    assert hop1['source_rse_id'] == rse3_id
+    assert hop2['source_rse_id'] == rse4_id
+    assert hop3['source_rse_id'] == rse5_id
+    assert hop3['dest_rse_id'] == rse6_id
+
+    # A link with no cost is ignored. Both for direct connection and multihop paths
+    [hop1, hop2, hop3] = get_hops(source_rse_id=rse2_id, dest_rse_id=rse6_id, include_multihop=True, multihop_rses=all_rses)
+    assert hop1['source_rse_id'] == rse2_id
+    assert hop2['source_rse_id'] == rse4_id
+    assert hop3['source_rse_id'] == rse5_id
+    assert hop3['dest_rse_id'] == rse6_id
+    [hop1, hop2, hop3, hop4] = get_hops(source_rse_id=rse1_id, dest_rse_id=rse6_id, include_multihop=True, multihop_rses=all_rses)
+    assert hop1['source_rse_id'] == rse1_id
+    assert hop2['source_rse_id'] == rse2_id
+    assert hop3['source_rse_id'] == rse4_id
+    assert hop4['source_rse_id'] == rse5_id
+    assert hop4['dest_rse_id'] == rse6_id