Repository: rucio/rucio

Issue:
source ranking never updated correctly for multihop
Motivation
----------
Found this issue while working on an unrelated issue (#4916). 
We don't have "real" multi-hop transfers. When it is decided that a multihop is needed, we create a direct transfer for each hop and these transfers live their own life. The initial transfer request is the last hop. The issue with this modelling is that entries in the "sources" table are bound to each hop. When an intermediate hop fails; it's source ranking is decreased, but it isn't bound to the initial transfer (last hop) in any way.
At next finisher iteration, the knowledge about the failure is lost. And we can re-schedule a multi-hop from the same source even if another candidate exists.






Discussion:

Resolving patch:
diff --git a/lib/rucio/core/request.py b/lib/rucio/core/request.py
--- a/lib/rucio/core/request.py
+++ b/lib/rucio/core/request.py
@@ -107,14 +107,15 @@ def should_retry_request(req, retry_protocol_mismatches):
 
 
 @transactional_session
-def requeue_and_archive(request, retry_protocol_mismatches=False, session=None, logger=logging.log):
+def requeue_and_archive(request, source_ranking_update=True, retry_protocol_mismatches=False, session=None, logger=logging.log):
     """
     Requeue and archive a failed request.
     TODO: Multiple requeue.
 
-    :param request:     Original request.
-    :param session:     Database session to use.
-    :param logger:      Optional decorated logger that can be passed from the calling daemons or servers.
+    :param request:               Original request.
+    :param source_ranking_update  Boolean. If True, the source ranking is decreased (making the sources less likely to be used)
+    :param session:               Database session to use.
+    :param logger:                Optional decorated logger that can be passed from the calling daemons or servers.
     """
 
     record_counter('core.request.requeue_request')
@@ -134,7 +135,7 @@ def requeue_and_archive(request, retry_protocol_mismatches=False, session=None,
             elif new_req['state'] != RequestState.SUBMITTING:
                 new_req['retry_count'] += 1
 
-            if new_req['sources']:
+            if source_ranking_update and new_req['sources']:
                 for i in range(len(new_req['sources'])):
                     if new_req['sources'][i]['is_using']:
                         if new_req['sources'][i]['ranking'] is None:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -617,12 +617,14 @@ def mark_submitting_and_prepare_sources_for_transfers(
                 raise RequestNotFound("Failed to prepare transfer: request %s does not exist or is not in queued state" % transfer.rws)
 
             for src_rse, src_url, src_rse_id, rank in transfer.legacy_sources:
+                # For multi-hops, sources in database are bound to the initial request
+                source_request_id = transfer.rws.attributes.get('initial_request_id', transfer.rws.request_id)
                 src_rowcount = session.query(models.Source)\
-                                      .filter_by(request_id=transfer.rws.request_id)\
+                                      .filter_by(request_id=source_request_id)\
                                       .filter(models.Source.rse_id == src_rse_id)\
                                       .update({'is_using': True}, synchronize_session=False)
                 if src_rowcount == 0:
-                    models.Source(request_id=transfer.rws.request_id,
+                    models.Source(request_id=source_request_id,
                                   scope=transfer.rws.scope,
                                   name=transfer.rws.name,
                                   rse_id=src_rse_id,
diff --git a/lib/rucio/daemons/conveyor/finisher.py b/lib/rucio/daemons/conveyor/finisher.py
--- a/lib/rucio/daemons/conveyor/finisher.py
+++ b/lib/rucio/daemons/conveyor/finisher.py
@@ -247,7 +247,7 @@ def __handle_requests(reqs, suspicious_patterns, retry_protocol_mismatches, logg
                 tss = time.time()
                 try:
                     if request_core.should_retry_request(req, retry_protocol_mismatches):
-                        new_req = request_core.requeue_and_archive(req, retry_protocol_mismatches, logger=logger)
+                        new_req = request_core.requeue_and_archive(req, source_ranking_update=True, retry_protocol_mismatches=retry_protocol_mismatches, logger=logger)
                         # should_retry_request and requeue_and_archive are not in one session,
                         # another process can requeue_and_archive and this one will return None.
                         record_timer('daemons.conveyor.common.update_request_state.request-requeue_and_archive', (time.time() - tss) * 1000)
@@ -274,7 +274,7 @@ def __handle_requests(reqs, suspicious_patterns, retry_protocol_mismatches, logg
                 try:
                     tss = time.time()
                     if request_core.should_retry_request(req, retry_protocol_mismatches):
-                        new_req = request_core.requeue_and_archive(req, retry_protocol_mismatches, logger=logger)
+                        new_req = request_core.requeue_and_archive(req, source_ranking_update=False, retry_protocol_mismatches=retry_protocol_mismatches, logger=logger)
                         record_timer('daemons.conveyor.common.update_request_state.request-requeue_and_archive', (time.time() - tss) * 1000)
                         logger(logging.WARNING, 'REQUEUED SUBMITTING DID %s:%s REQUEST %s AS %s TRY %s' % (req['scope'],
                                                                                                            req['name'],


Test patch:
diff --git a/lib/rucio/tests/test_conveyor.py b/lib/rucio/tests/test_conveyor.py
--- a/lib/rucio/tests/test_conveyor.py
+++ b/lib/rucio/tests/test_conveyor.py
@@ -110,6 +110,16 @@ def set_query_parameters(url, params):
     return urlunparse(url_parts)
 
 
+@read_session
+def __get_source(request_id, src_rse_id, scope, name, session=None):
+    return session.query(models.Source) \
+        .filter(models.Source.request_id == request_id) \
+        .filter(models.Source.scope == scope) \
+        .filter(models.Source.name == name) \
+        .filter(models.Source.rse_id == src_rse_id) \
+        .first()
+
+
 @skip_rse_tests_with_accounts
 @pytest.mark.dirty(reason="leaves files in XRD containers")
 @pytest.mark.noparallel(reason="uses predefined RSEs; runs submitter, poller and finisher; changes XRD3 usage and limits")
@@ -178,7 +188,11 @@ def test_multihop_intermediate_replica_lifecycle(vo, did_factory, root_account,
         replica = replica_core.get_replica(rse_id=jump_rse_id, **did)
         assert replica
 
-        # FTS fails the second transfer, so run submitter again to copy from jump rse to destination rse
+        # FTS fails the second transfer
+        request = __wait_for_request_state(dst_rse_id=dst_rse_id, state=RequestState.FAILED, **did)
+        # ensure tha the ranking was correctly decreased
+        assert __get_source(request_id=request['id'], src_rse_id=jump_rse_id, **did).ranking == -1
+        # run submitter again to copy from jump rse to destination rse
         submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], partition_wait_time=None, transfertype='single', filter_transfertool=None)
 
         # Wait for the destination replica to become ready
@@ -249,6 +263,9 @@ def test_fts_non_recoverable_failures_handled_on_multihop(vo, did_factory, root_
     with pytest.raises(RequestNotFound):
         request_core.get_request_by_did(rse_id=jump_rse_id, **did)
     request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+    # ensure tha the ranking was correctly decreased for the whole path
+    assert __get_source(request_id=request['id'], src_rse_id=jump_rse_id, **did).ranking == -1
+    assert __get_source(request_id=request['id'], src_rse_id=src_rse_id, **did).ranking == -1
     assert request['state'] == RequestState.QUEUED
 
 
@@ -471,6 +488,9 @@ def test_multihop_receiver_on_failure(vo, did_factory, replica_client, root_acco
         with pytest.raises(RequestNotFound):
             request_core.get_request_by_did(rse_id=jump_rse_id, **did)
         request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+        # ensure tha the ranking was correctly decreased for the whole path
+        assert __get_source(request_id=request['id'], src_rse_id=jump_rse_id, **did).ranking == -1
+        assert __get_source(request_id=request['id'], src_rse_id=src_rse_id, **did).ranking == -1
         assert request['state'] == RequestState.QUEUED
     finally:
         receiver_graceful_stop.set()
@@ -716,6 +736,9 @@ def __update_request(request_id, session=None, **kwargs):
     # Run finisher and submitter, the request must be resubmitted and transferred correctly
     __update_request(request['id'], updated_at=datetime.utcnow() - timedelta(days=1))
     finisher(once=True, partition_wait_time=None)
+    # The source ranking must not be updated for submission failures and lost transfers
+    request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+    assert __get_source(request_id=request['id'], src_rse_id=src_rse_id, **did).ranking == 0
     submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
     replica = __wait_for_replica_transfer(dst_rse_id=dst_rse_id, **did)
     assert replica['state'] == ReplicaState.AVAILABLE