Repository: rucio/rucio

Issue:
set reason for cancelled hops in multi-hops
Motivation
----------

When a multihop transfer is cancelled, it's transitioned to Failed state without any error_msg set in the database:
https://github.com/rucio/rucio/blob/226a9f8efb0dfe09b52607cd2333e184ab88d105/lib/rucio/core/transfer.py#L1569






Discussion:

Resolving patch:
diff --git a/lib/rucio/core/request.py b/lib/rucio/core/request.py
--- a/lib/rucio/core/request.py
+++ b/lib/rucio/core/request.py
@@ -509,26 +509,6 @@ def set_request_state(request_id, new_state, transfer_id=None, transferred_at=No
         raise UnsupportedOperation("Request %s state cannot be updated." % request_id)
 
 
-@transactional_session
-def set_requests_state(request_ids, new_state, session=None, logger=logging.log):
-    """
-    Bulk update the state of requests.
-
-    :param request_ids:  List of (Request-ID as a 32 character hex string).
-    :param new_state:    New state as string.
-    :param session:      Database session to use.
-    :param logger:       Optional decorated logger that can be passed from the calling daemons or servers.
-    """
-
-    record_counter('core.request.set_requests_state')
-
-    try:
-        for request_id in request_ids:
-            set_request_state(request_id, new_state, session=session, logger=logger)
-    except IntegrityError as error:
-        raise RucioException(error.args)
-
-
 @transactional_session
 def set_requests_state_if_possible(request_ids, new_state, session=None, logger=logging.log):
     """
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -69,7 +69,7 @@
 from rucio.core.config import get as core_config_get
 from rucio.core.monitor import record_counter, record_timer
 from rucio.core.replica import add_replicas, tombstone_from_delay, update_replica_state
-from rucio.core.request import queue_requests, set_requests_state
+from rucio.core.request import queue_requests, set_request_state
 from rucio.core.rse import get_rse_name, get_rse_vo, list_rses
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.db.sqla import models, filter_thread_work
@@ -1558,7 +1558,7 @@ def create_missing_replicas_and_requests(
             break
         rws.request_id = new_req[0]['id']
         logger(logging.DEBUG, '%s: New request created for the transfer between %s and %s : %s', initial_request_id, transfer_path[0].src, transfer_path[-1].dst, rws.request_id)
-        set_requests_state(request_ids=[rws.request_id, ], new_state=RequestState.QUEUED, session=session)
+        set_request_state(rws.request_id, RequestState.QUEUED, session=session, logger=logger)
         created_requests.append(rws.request_id)
 
     if not creation_successful:
@@ -1566,7 +1566,9 @@ def create_missing_replicas_and_requests(
         logger(logging.WARNING, '%s: Multihop : A request already exists for the transfer between %s and %s. Will cancel all the parent requests',
                initial_request_id, transfer_path[0].src, transfer_path[-1].dst)
         try:
-            set_requests_state(request_ids=created_requests, new_state=RequestState.FAILED, session=session)
+            for request_id in created_requests:
+                set_request_state(request_id=request_id, new_state=RequestState.FAILED,
+                                  err_msg="Cancelled hop in multi-hop", session=session)
         except UnsupportedOperation:
             logger(logging.ERROR, '%s: Multihop : Cannot cancel all the parent requests : %s', initial_request_id, str(created_requests))
 
diff --git a/lib/rucio/transfertool/fts3.py b/lib/rucio/transfertool/fts3.py
--- a/lib/rucio/transfertool/fts3.py
+++ b/lib/rucio/transfertool/fts3.py
@@ -989,7 +989,7 @@ def __format_response(self, fts_job_response, fts_files_response):
             request_id = file_resp['file_metadata']['request_id']
             reason = file_resp.get('reason', None)
             if not reason and file_state == FTS_STATE.NOT_USED and multi_hop:
-                reason = 'Cancelled hop in multi-hop'
+                reason = 'Unused hop in multi-hop'
             resps[request_id] = {'new_state': None,
                                  'transfer_id': fts_job_response.get('job_id'),
                                  'job_state': fts_job_response.get('job_state', None),


Test patch:
diff --git a/lib/rucio/tests/test_conveyor.py b/lib/rucio/tests/test_conveyor.py
--- a/lib/rucio/tests/test_conveyor.py
+++ b/lib/rucio/tests/test_conveyor.py
@@ -253,7 +253,7 @@ def test_fts_non_recoverable_failures_handled_on_multihop(vo, did_factory, root_
     submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
 
     request = __wait_for_request_state(dst_rse_id=dst_rse_id, state=RequestState.FAILED, **did)
-    assert 'Cancelled hop in multi-hop' in request['err_msg']
+    assert 'Unused hop in multi-hop' in request['err_msg']
     assert request['state'] == RequestState.FAILED
     request = request_core.get_request_by_did(rse_id=jump_rse_id, **did)
     assert request['state'] == RequestState.FAILED
@@ -481,7 +481,7 @@ def test_multihop_receiver_on_failure(vo, did_factory, replica_client, root_acco
         # TODO: set the run_poller argument to False if we ever manage to make the receiver correctly handle multi-hop failures.
         request = __wait_for_request_state(dst_rse_id=dst_rse_id, state=RequestState.FAILED, run_poller=True, **did)
         assert request['state'] == RequestState.FAILED
-        assert 'Cancelled hop in multi-hop' in request['err_msg']
+        assert 'Unused hop in multi-hop' in request['err_msg']
 
         # First hop will be handled by receiver; second hop by poller
         assert metrics_mock.get_sample_value('rucio_daemons_conveyor_receiver_update_request_state_total', labels={'updated': 'True'}) >= 1