Repository: rucio/rucio

Issue:
Conveyor receiver incorrectly handles multi_source jobs
Motivation
----------
The conveyor receiver incorrectly handles multi source jobs. This is the case for jobs where the first source FAILS and the second one is a success.

The poller requests jobs via the transfer tool which does quite a bit of response formatting and checking. In the case of multi source jobs, it basically iterates the sources and if one of the sources is in state FTSState.FINISHED it returns the job as SUCCESS. The receiver however does not go via the transfertool as it gets the response directly from activemq. In the case of a job where the first source fails but the second one is a succes, the receiver only looks at the `t_final_transfer_state` which is FTSCompleteState.ERROR. However, what it actually should do is to iterate the sources like the conveyor poller. It seems that the `t_final_transfer_state` field can only be used for bulk-transfers, not multi-source transfers.

Modification
------------
Factorize the formatting used in the transfer tool and re-apply to conveyor receiver code.



Discussion:
Up

Resolving patch:
diff --git a/lib/rucio/daemons/conveyor/receiver.py b/lib/rucio/daemons/conveyor/receiver.py
--- a/lib/rucio/daemons/conveyor/receiver.py
+++ b/lib/rucio/daemons/conveyor/receiver.py
@@ -61,7 +61,8 @@
 
 class Receiver(object):
 
-    def __init__(self, broker, id, total_threads, full_mode=False):
+    def __init__(self, broker, id, total_threads, full_mode=False, all_vos=False):
+        self.__all_vos = all_vos
         self.__broker = broker
         self.__id = id
         self.__total_threads = total_threads
@@ -74,75 +75,47 @@ def on_error(self, frame):
     def on_message(self, frame):
         record_counter('daemons.conveyor.receiver.message_all')
 
-        try:
-            msg = json.loads(frame.body)
-        except Exception:
-            msg = json.loads(frame.body[:-1])  # Note: I am not sure if this is needed anymore, this was due to an unparsable EOT character
+        msg = json.loads(frame.body)
 
-        if 'vo' not in msg or msg['vo'] != get_policy():
-            return
+        if not self.__all_vos:
+            if 'vo' not in msg or msg['vo'] != get_policy():
+                return
 
         if 'job_metadata' in msg.keys() \
            and isinstance(msg['job_metadata'], dict) \
            and 'issuer' in msg['job_metadata'].keys() \
            and str(msg['job_metadata']['issuer']) == str('rucio'):
 
-            if 'job_m_replica' in msg.keys() and 'job_state' in msg.keys() \
-               and (str(msg['job_m_replica']).lower() == str('false') or (str(msg['job_m_replica']).lower() == str('true') and str(msg['job_state']) != str('ACTIVE'))):
-
-                if 'request_id' in msg['job_metadata']:
-                    # submitted by old submitter
-                    response = {'new_state': None,
-                                'transfer_id': msg.get('tr_id').split("__")[-1],
-                                'job_state': msg.get('t_final_transfer_state', None),
-                                'src_url': msg.get('src_url', None),
-                                'dst_url': msg.get('dst_url', None),
-                                'transferred_at': datetime.datetime.utcfromtimestamp(float(msg.get('tr_timestamp_complete', 0)) / 1000),
-                                'duration': (float(msg.get('tr_timestamp_complete', 0)) - float(msg.get('tr_timestamp_start', 0))) / 1000,
-                                'reason': msg.get('t__error_message', None),
-                                'scope': msg['job_metadata'].get('scope', None),
-                                'name': msg['job_metadata'].get('name', None),
-                                'src_rse': msg['job_metadata'].get('src_rse', None),
-                                'dst_rse': msg['job_metadata'].get('dst_rse', None),
-                                'request_id': msg['job_metadata'].get('request_id', None),
-                                'activity': msg['job_metadata'].get('activity', None),
-                                'src_rse_id': msg['job_metadata'].get('src_rse_id', None),
-                                'dest_rse_id': msg['job_metadata'].get('dest_rse_id', None),
-                                'previous_attempt_id': msg['job_metadata'].get('previous_attempt_id', None),
-                                'adler32': msg['job_metadata'].get('adler32', None),
-                                'md5': msg['job_metadata'].get('md5', None),
-                                'filesize': msg['job_metadata'].get('filesize', None),
-                                'external_host': msg.get('endpnt', None),
-                                'job_m_replica': msg.get('job_m_replica', None),
-                                'details': {'files': msg['job_metadata']}}
-                else:
-                    # for new submitter, file_metadata replace the job_metadata
-                    response = {'new_state': None,
-                                'transfer_id': msg.get('tr_id').split("__")[-1],
-                                'job_state': msg.get('t_final_transfer_state', None),
-                                'src_url': msg.get('src_url', None),
-                                'dst_url': msg.get('dst_url', None),
-                                'started_at': datetime.datetime.utcfromtimestamp(float(msg.get('tr_timestamp_start', 0)) / 1000),
-                                'transferred_at': datetime.datetime.utcfromtimestamp(float(msg.get('tr_timestamp_complete', 0)) / 1000),
-                                'duration': (float(msg.get('tr_timestamp_complete', 0)) - float(msg.get('tr_timestamp_start', 0))) / 1000,
-                                'reason': msg.get('t__error_message', None),
-                                'scope': msg['file_metadata'].get('scope', None),
-                                'name': msg['file_metadata'].get('name', None),
-                                'src_type': msg['file_metadata'].get('src_type', None),
-                                'dst_type': msg['file_metadata'].get('dst_type', None),
-                                'src_rse': msg['file_metadata'].get('src_rse', None),
-                                'dst_rse': msg['file_metadata'].get('dst_rse', None),
-                                'request_id': msg['file_metadata'].get('request_id', None),
-                                'activity': msg['file_metadata'].get('activity', None),
-                                'src_rse_id': msg['file_metadata'].get('src_rse_id', None),
-                                'dest_rse_id': msg['file_metadata'].get('dest_rse_id', None),
-                                'previous_attempt_id': msg['file_metadata'].get('previous_attempt_id', None),
-                                'adler32': msg['file_metadata'].get('adler32', None),
-                                'md5': msg['file_metadata'].get('md5', None),
-                                'filesize': msg['file_metadata'].get('filesize', None),
-                                'external_host': msg.get('endpnt', None),
-                                'job_m_replica': msg.get('job_m_replica', None),
-                                'details': {'files': msg['file_metadata']}}
+            if 'job_state' in msg.keys() and (
+                    str(msg['job_state']) != str('ACTIVE')
+                    or str(msg['job_state']) == str('ACTIVE') and 'job_m_replica' in msg.keys() and (str(msg['job_m_replica']).lower() == str('true'))):
+
+                response = {'new_state': None,
+                            'transfer_id': msg.get('tr_id').split("__")[-1],
+                            'job_state': msg.get('t_final_transfer_state', None),
+                            'src_url': msg.get('src_url', None),
+                            'dst_url': msg.get('dst_url', None),
+                            'started_at': datetime.datetime.utcfromtimestamp(float(msg.get('tr_timestamp_start', 0)) / 1000),
+                            'transferred_at': datetime.datetime.utcfromtimestamp(float(msg.get('tr_timestamp_complete', 0)) / 1000),
+                            'duration': (float(msg.get('tr_timestamp_complete', 0)) - float(msg.get('tr_timestamp_start', 0))) / 1000,
+                            'reason': msg.get('t__error_message', None),
+                            'scope': msg['file_metadata'].get('scope', None),
+                            'name': msg['file_metadata'].get('name', None),
+                            'src_type': msg['file_metadata'].get('src_type', None),
+                            'dst_type': msg['file_metadata'].get('dst_type', None),
+                            'src_rse': msg['file_metadata'].get('src_rse', None),
+                            'dst_rse': msg['file_metadata'].get('dst_rse', None),
+                            'request_id': msg['file_metadata'].get('request_id', None),
+                            'activity': msg['file_metadata'].get('activity', None),
+                            'src_rse_id': msg['file_metadata'].get('src_rse_id', None),
+                            'dest_rse_id': msg['file_metadata'].get('dest_rse_id', None),
+                            'previous_attempt_id': msg['file_metadata'].get('previous_attempt_id', None),
+                            'adler32': msg['file_metadata'].get('adler32', None),
+                            'md5': msg['file_metadata'].get('md5', None),
+                            'filesize': msg['file_metadata'].get('filesize', None),
+                            'external_host': msg.get('endpnt', None),
+                            'job_m_replica': msg.get('job_m_replica', None),
+                            'details': {'files': msg['file_metadata']}}
 
                 record_counter('daemons.conveyor.receiver.message_rucio')
                 if str(msg['t_final_transfer_state']) == FTS_COMPLETE_STATE.OK:  # pylint:disable=no-member
@@ -174,7 +147,7 @@ def on_message(self, frame):
                     logging.critical(traceback.format_exc())
 
 
-def receiver(id, total_threads=1, full_mode=False):
+def receiver(id, total_threads=1, full_mode=False, all_vos=False):
     """
     Main loop to consume messages from the FTS3 producer.
     """
@@ -250,7 +223,9 @@ def receiver(id, total_threads=1, full_mode=False):
                 logging.info('connecting to %s' % conn.transport._Transport__host_and_ports[0][0])
                 record_counter('daemons.messaging.fts3.reconnect.%s' % conn.transport._Transport__host_and_ports[0][0].split('.')[0])
 
-                conn.set_listener('rucio-messaging-fts3', Receiver(broker=conn.transport._Transport__host_and_ports[0], id=id, total_threads=total_threads, full_mode=full_mode))
+                conn.set_listener('rucio-messaging-fts3', Receiver(broker=conn.transport._Transport__host_and_ports[0],
+                                                                   id=id, total_threads=total_threads,
+                                                                   full_mode=full_mode, all_vos=all_vos))
                 if not use_ssl:
                     conn.connect(username, password, wait=True)
                 else:


Test patch:
diff --git a/etc/docker/test/extra/rucio_multi_vo_ts2_postgres12.cfg b/etc/docker/test/extra/rucio_multi_vo_ts2_postgres12.cfg
--- a/etc/docker/test/extra/rucio_multi_vo_ts2_postgres12.cfg
+++ b/etc/docker/test/extra/rucio_multi_vo_ts2_postgres12.cfg
@@ -55,12 +55,16 @@ cacert = /opt/rucio/etc/rucio_ca.pem
 usercert = /opt/rucio/etc/ruciouser.pem
 
 [messaging-fts3]
-port = 61123
+port = 61613
+nonssl_port = 61613
+use_ssl = False
 ssl_key_file = /opt/rucio/etc/usercert.key.pem
 ssl_cert_file = /opt/rucio/etc/usercert.pem
-destination = /topic/transfer.fts_monitoring_queue_state
+destination = /topic/transfer.fts_monitoring_complete
 brokers = activemq
 voname = atlas
+username = receiver
+password = supersecret
 
 [messaging-hermes]
 username = admin
diff --git a/etc/docker/test/extra/rucio_multi_vo_tst_postgres12.cfg b/etc/docker/test/extra/rucio_multi_vo_tst_postgres12.cfg
--- a/etc/docker/test/extra/rucio_multi_vo_tst_postgres12.cfg
+++ b/etc/docker/test/extra/rucio_multi_vo_tst_postgres12.cfg
@@ -55,12 +55,16 @@ cacert = /opt/rucio/etc/rucio_ca.pem
 usercert = /opt/rucio/etc/ruciouser.pem
 
 [messaging-fts3]
-port = 61123
+port = 61613
+nonssl_port = 61613
+use_ssl = False
 ssl_key_file = /opt/rucio/etc/usercert.key.pem
 ssl_cert_file = /opt/rucio/etc/usercert.pem
-destination = /topic/transfer.fts_monitoring_queue_state
+destination = /topic/transfer.fts_monitoring_complete
 brokers = activemq
 voname = atlas
+username = receiver
+password = supersecret
 
 [messaging-hermes]
 username = admin
diff --git a/etc/docker/test/extra/rucio_mysql8.cfg b/etc/docker/test/extra/rucio_mysql8.cfg
--- a/etc/docker/test/extra/rucio_mysql8.cfg
+++ b/etc/docker/test/extra/rucio_mysql8.cfg
@@ -52,12 +52,16 @@ cacert = /opt/rucio/etc/rucio_ca.pem
 usercert = /opt/rucio/etc/ruciouser.pem
 
 [messaging-fts3]
-port = 61123
+port = 61613
+nonssl_port = 61613
+use_ssl = False
 ssl_key_file = /opt/rucio/etc/usercert.key.pem
 ssl_cert_file = /opt/rucio/etc/usercert.pem
-destination = /topic/transfer.fts_monitoring_queue_state
+destination = /topic/transfer.fts_monitoring_complete
 brokers = activemq
 voname = atlas
+username = receiver
+password = supersecret
 
 [messaging-hermes]
 username = admin
diff --git a/etc/docker/test/extra/rucio_oracle.cfg b/etc/docker/test/extra/rucio_oracle.cfg
--- a/etc/docker/test/extra/rucio_oracle.cfg
+++ b/etc/docker/test/extra/rucio_oracle.cfg
@@ -52,12 +52,16 @@ cacert = /opt/rucio/etc/rucio_ca.pem
 usercert = /opt/rucio/etc/ruciouser.pem
 
 [messaging-fts3]
-port = 61123
+port = 61613
+nonssl_port = 61613
+use_ssl = False
 ssl_key_file = /opt/rucio/etc/usercert.key.pem
 ssl_cert_file = /opt/rucio/etc/usercert.pem
-destination = /topic/transfer.fts_monitoring_queue_state
+destination = /topic/transfer.fts_monitoring_complete
 brokers = activemq
 voname = atlas
+username = receiver
+password = supersecret
 
 [messaging-hermes]
 username = admin
diff --git a/etc/docker/test/extra/rucio_postgres12.cfg b/etc/docker/test/extra/rucio_postgres12.cfg
--- a/etc/docker/test/extra/rucio_postgres12.cfg
+++ b/etc/docker/test/extra/rucio_postgres12.cfg
@@ -52,12 +52,16 @@ cacert = /opt/rucio/etc/rucio_ca.pem
 usercert = /opt/rucio/etc/ruciouser.pem
 
 [messaging-fts3]
-port = 61123
+port = 61613
+nonssl_port = 61613
+use_ssl = False
 ssl_key_file = /opt/rucio/etc/usercert.key.pem
 ssl_cert_file = /opt/rucio/etc/usercert.pem
-destination = /topic/transfer.fts_monitoring_queue_state
+destination = /topic/transfer.fts_monitoring_complete
 brokers = activemq
 voname = atlas
+username = receiver
+password = supersecret
 
 [messaging-hermes]
 username = admin
diff --git a/etc/docker/test/extra/rucio_postgres9.cfg b/etc/docker/test/extra/rucio_postgres9.cfg
--- a/etc/docker/test/extra/rucio_postgres9.cfg
+++ b/etc/docker/test/extra/rucio_postgres9.cfg
@@ -52,12 +52,16 @@ cacert = /opt/rucio/etc/rucio_ca.pem
 usercert = /opt/rucio/etc/ruciouser.pem
 
 [messaging-fts3]
-port = 61123
+port = 61613
+nonssl_port = 61613
+use_ssl = False
 ssl_key_file = /opt/rucio/etc/usercert.key.pem
 ssl_cert_file = /opt/rucio/etc/usercert.pem
-destination = /topic/transfer.fts_monitoring_queue_state
+destination = /topic/transfer.fts_monitoring_complete
 brokers = activemq
 voname = atlas
+username = receiver
+password = supersecret
 
 [messaging-hermes]
 username = admin
diff --git a/etc/docker/test/extra/rucio_sqlite.cfg b/etc/docker/test/extra/rucio_sqlite.cfg
--- a/etc/docker/test/extra/rucio_sqlite.cfg
+++ b/etc/docker/test/extra/rucio_sqlite.cfg
@@ -49,12 +49,16 @@ cacert = /opt/rucio/etc/rucio_ca.pem
 usercert = /opt/rucio/etc/ruciouser.pem
 
 [messaging-fts3]
-port = 61123
+port = 61613
+nonssl_port = 61613
+use_ssl = False
 ssl_key_file = /opt/rucio/etc/usercert.key.pem
 ssl_cert_file = /opt/rucio/etc/usercert.pem
-destination = /topic/transfer.fts_monitoring_queue_state
+destination = /topic/transfer.fts_monitoring_complete
 brokers = activemq
 voname = atlas
+username = receiver
+password = supersecret
 
 [messaging-hermes]
 username = admin
diff --git a/etc/docker/test/extra/rucio_syntax.cfg b/etc/docker/test/extra/rucio_syntax.cfg
--- a/etc/docker/test/extra/rucio_syntax.cfg
+++ b/etc/docker/test/extra/rucio_syntax.cfg
@@ -49,12 +49,16 @@ cacert = /opt/rucio/etc/rucio_ca.pem
 usercert = /opt/rucio/etc/ruciouser.pem
 
 [messaging-fts3]
-port = 61123
+port = 61613
+nonssl_port = 61613
+use_ssl = False
 ssl_key_file = /opt/rucio/etc/usercert.key.pem
 ssl_cert_file = /opt/rucio/etc/usercert.pem
-destination = /topic/transfer.fts_monitoring_queue_state
+destination = /topic/transfer.fts_monitoring_complete
 brokers = activemq
 voname = atlas
+username = receiver
+password = supersecret
 
 [messaging-hermes]
 username = admin
diff --git a/lib/rucio/tests/test_conveyor.py b/lib/rucio/tests/test_conveyor.py
--- a/lib/rucio/tests/test_conveyor.py
+++ b/lib/rucio/tests/test_conveyor.py
@@ -15,6 +15,7 @@
 #
 # Authors:
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
+import threading
 import time
 from datetime import datetime
 
@@ -29,14 +30,17 @@
 from rucio.daemons.conveyor.finisher import finisher
 from rucio.daemons.conveyor.poller import poller
 from rucio.daemons.conveyor.submitter import submitter
+from rucio.daemons.conveyor.receiver import receiver, graceful_stop as receiver_graceful_stop
 from rucio.daemons.reaper.reaper import reaper
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import RequestState, ReplicaState
 from rucio.db.sqla.session import read_session, transactional_session
 from rucio.tests.common import skip_rse_tests_with_accounts
 
+MAX_POLL_WAIT_SECONDS = 60
 
-def __wait_for_replica_transfer(dst_rse_id, scope, name, max_wait_seconds=60):
+
+def __wait_for_replica_transfer(dst_rse_id, scope, name, max_wait_seconds=MAX_POLL_WAIT_SECONDS):
     """
     Wait for the replica to become AVAILABLE on the given RSE as a result of a pending transfer
     """
@@ -51,13 +55,14 @@ def __wait_for_replica_transfer(dst_rse_id, scope, name, max_wait_seconds=60):
     return replica
 
 
-def __wait_for_request_state(dst_rse_id, scope, name, state, max_wait_seconds=60):
+def __wait_for_request_state(dst_rse_id, scope, name, state, max_wait_seconds=MAX_POLL_WAIT_SECONDS, run_poller=True):
     """
     Wait for the request state to be updated to the given expected state as a result of a pending transfer
     """
     request = None
     for _ in range(max_wait_seconds):
-        poller(once=True, older_than=0, partition_wait_time=None)
+        if run_poller:
+            poller(once=True, older_than=0, partition_wait_time=None)
         request = request_core.get_request_by_did(rse_id=dst_rse_id, scope=scope, name=name)
         if request['state'] == state:
             break
@@ -283,3 +288,134 @@ def __source_exists(src_rse_id, scope, name, session=None):
     assert replica['state'] == ReplicaState.AVAILABLE
     assert not __source_exists(src_rse_id=src_rse1_id, **did)
     assert not __source_exists(src_rse_id=src_rse2_id, **did)
+
+
+@skip_rse_tests_with_accounts
+@pytest.mark.dirty(reason="leaves files in XRD containers")
+@pytest.mark.noparallel(reason="uses predefined RSEs; runs submitter and receiver")
+def test_multisource_receiver(vo, did_factory, replica_client, root_account):
+    """
+    Run receiver as a background thread to automatically handle fts notifications.
+    Ensure that a multi-source job in which the first source fails is correctly handled by receiver.
+    """
+    receiver_thread = threading.Thread(target=receiver, kwargs={'id': 0, 'full_mode': True, 'all_vos': True, 'total_threads': 1})
+    receiver_thread.start()
+
+    try:
+        src_rse1 = 'XRD4'
+        src_rse1_id = rse_core.get_rse_id(rse=src_rse1, vo=vo)
+        src_rse2 = 'XRD1'
+        src_rse2_id = rse_core.get_rse_id(rse=src_rse2, vo=vo)
+        dst_rse = 'XRD3'
+        dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo)
+
+        all_rses = [src_rse1_id, src_rse2_id, dst_rse_id]
+
+        # Add a good replica on the RSE which has a higher distance ranking
+        did = did_factory.upload_test_file(src_rse1)
+        # Add non-existing replica which will fail during multisource transfers on the RSE with lower cost (will be the preferred source)
+        replica_client.add_replicas(rse=src_rse2, files=[{'scope': did['scope'].external, 'name': did['name'], 'bytes': 1, 'adler32': 'aaaaaaaa'}])
+
+        rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+
+        request = None
+        for _ in range(MAX_POLL_WAIT_SECONDS):
+            request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+            # The request must not be marked as failed. Not even temporarily. It is a multi-source transfer and the
+            # the first, failed, source must not change the replica state. We must wait for all sources to be tried.
+            assert request['state'] != RequestState.FAILED
+            if request['state'] == RequestState.DONE:
+                break
+            time.sleep(1)
+        assert request['state'] == RequestState.DONE
+    finally:
+        receiver_graceful_stop.set()
+        receiver_thread.join(timeout=5)
+        receiver_graceful_stop.clear()
+
+
+@skip_rse_tests_with_accounts
+@pytest.mark.noparallel(reason="uses predefined RSEs; runs submitter and receiver")
+@pytest.mark.parametrize("core_config_mock", [{"table_content": [
+    ('transfers', 'use_multihop', True),
+]}], indirect=True)
+@pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
+    'rucio.core.rse_expression_parser',  # The list of multihop RSEs is retrieved by rse expression
+    'rucio.core.config',
+]}], indirect=True)
+def test_multihop_receiver_on_failure(vo, did_factory, replica_client, root_account, core_config_mock, caches_mock):
+    """
+    Verify that the receiver correctly handles multihop jobs which fail
+    """
+    receiver_thread = threading.Thread(target=receiver, kwargs={'id': 0, 'full_mode': True, 'all_vos': True, 'total_threads': 1})
+    receiver_thread.start()
+
+    try:
+        src_rse = 'XRD1'
+        src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo)
+        jump_rse = 'XRD3'
+        jump_rse_id = rse_core.get_rse_id(rse=jump_rse, vo=vo)
+        dst_rse = 'XRD4'
+        dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo)
+
+        all_rses = [src_rse_id, jump_rse_id, dst_rse_id]
+
+        # Register a did which doesn't exist. It will trigger a failure error during the FTS transfer.
+        did = did_factory.random_did()
+        replica_client.add_replicas(rse=src_rse, files=[{'scope': did['scope'].external, 'name': did['name'], 'bytes': 1, 'adler32': 'aaaaaaaa'}])
+
+        rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+
+        request = __wait_for_request_state(dst_rse_id=jump_rse_id, state=RequestState.FAILED, run_poller=False, **did)
+        assert request['state'] == RequestState.FAILED
+        # We use FTS "Completion" messages in receiver. In case of multi-hops transfer failures, FTS doesn't start
+        # next transfers; so it never sends a "completion" message for some hops. Rely on poller in such cases.
+        # TODO: set the run_poller argument to False if we ever manage to make the receiver correctly handle multi-hop failures.
+        request = __wait_for_request_state(dst_rse_id=dst_rse_id, state=RequestState.FAILED, run_poller=True, **did)
+        assert request['state'] == RequestState.FAILED
+    finally:
+        receiver_graceful_stop.set()
+        receiver_thread.join(timeout=5)
+        receiver_graceful_stop.clear()
+
+
+@skip_rse_tests_with_accounts
+@pytest.mark.noparallel(reason="uses predefined RSEs; runs submitter and receiver")
+@pytest.mark.parametrize("core_config_mock", [{"table_content": [
+    ('transfers', 'use_multihop', True),
+]}], indirect=True)
+@pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
+    'rucio.core.rse_expression_parser',  # The list of multihop RSEs is retrieved by rse expression
+    'rucio.core.config',
+]}], indirect=True)
+def test_multihop_receiver_on_success(vo, did_factory, root_account, core_config_mock, caches_mock):
+    """
+    Verify that the receiver correctly handles successful multihop jobs
+    """
+    receiver_thread = threading.Thread(target=receiver, kwargs={'id': 0, 'full_mode': True, 'all_vos': True, 'total_threads': 1})
+    receiver_thread.start()
+
+    try:
+        src_rse = 'XRD1'
+        src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo)
+        jump_rse = 'XRD3'
+        jump_rse_id = rse_core.get_rse_id(rse=jump_rse, vo=vo)
+        dst_rse = 'XRD4'
+        dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo)
+
+        all_rses = [src_rse_id, jump_rse_id, dst_rse_id]
+
+        did = did_factory.upload_test_file(src_rse)
+        rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+
+        request = __wait_for_request_state(dst_rse_id=jump_rse_id, state=RequestState.DONE, run_poller=False, **did)
+        assert request['state'] == RequestState.DONE
+        request = __wait_for_request_state(dst_rse_id=dst_rse_id, state=RequestState.DONE, run_poller=False, **did)
+        assert request['state'] == RequestState.DONE
+    finally:
+        receiver_graceful_stop.set()
+        receiver_thread.join(timeout=5)
+        receiver_graceful_stop.clear()