Repository: rucio/rucio

Issue:
The Poller doesn’t correctly process multi-hop transfers where the first leg has failed
I am using [this](https://fts3-atlas.cern.ch:8449/fts3/ftsmon/#/job/f278c18a-b6e0-11eb-9e93-fa163eb04ff0) FTS transfer as an example. In the Rucio database, the request has `id=HEXTORAW('02A95E2590F04A4C982372155CB87A1E')`, `state='S'`, `external_host='https://fts3-atlas.cern.ch:8446'`, and `external_id='f278c18a-b6e0-11eb-9e93-fa163eb04ff0'`.

Mimicking the Poller in an interactive session:
```
>>> import logging
>>> logging.basicConfig(level=logging.DEBUG)
>>> from rucio.daemons.conveyor.poller import poll_transfers
>>> poll_transfers(external_host='https://fts3-atlas.cern.ch:8446', xfers=['f278c18a-b6e0-11eb-9e93-fa163eb04ff0'], request_ids=['02A95E2590F04A4C982372155CB87A1E'])
INFO:root:Polling 1 transfers against https://fts3-atlas.cern.ch:8446 with timeout None
DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): fts3-atlas.cern.ch:8446
DEBUG:urllib3.connectionpool:https://fts3-atlas.cern.ch:8446 "GET /jobs/f278c18a-b6e0-11eb-9e93-fa163eb04ff0?files=file_state,dest_surl,finish_time,start_time,staging_start,staging_finished,reason,source_surl,file_metadata HTTP/1.1" 200 3427
DEBUG:root:Polled 1 transfer requests status in 0.3010680675506592 seconds
DEBUG:root:Updating 1 transfer requests status
DEBUG:root:Finished updating 1 transfer requests status (0 requests state changed) in 0.18983244895935059 seconds
```
Yet the request remains in state `SUBMITTED`.

Looking more closely in the response from FTS:
```
>>> import pprint
>>> from rucio.core.transfer import bulk_query_transfers
>>> resp = bulk_query_transfers(request_host='https://fts3-atlas.cern.ch:8446', transfer_ids=['f278c18a-b6e0-11eb-9e93-fa163eb04ff0'])
DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): fts3-atlas.cern.ch:8446
DEBUG:urllib3.connectionpool:https://fts3-atlas.cern.ch:8446 "GET /jobs/f278c18a-b6e0-11eb-9e93-fa163eb04ff0?files=file_state,dest_surl,finish_time,start_time,staging_start,staging_finished,reason,source_surl,file_metadata HTTP/1.1" 200 3427
>>> pprint.pprint(resp)
{'f278c18a-b6e0-11eb-9e93-fa163eb04ff0': {'dc8613306cfd493c848ad0e439732d22': {'activity': 'Data '
                                                                                           'rebalancing',
                                                                               'adler32': 'ad47b044',
                                                                               'dest_rse_id': '4ac9dd780e514aebbc26b23f9bef2f24',
                                                                               'details': {'files': {'activity': 'Data '
                                                                                                                 'rebalancing',
                                                                                                     'adler32': 'ad47b044',
                                                                                                     'dest_globus_endpoint_id': None,
                                                                                                     'dest_rse_id': '4ac9dd780e514aebbc26b23f9bef2f24',
                                                                                                     'dst_rse': 'CA-SFU-T2_DATADISK',
                                                                                                     'dst_type': 'DISK',
                                                                                                     'filesize': 2074656655,
                                                                                                     'md5': None,
                                                                                                     'name': 'data11_7TeV.00189610.physics_Muons.merge.AOD.f405_m985._lb0416._0001.1',
                                                                                                     'request_id': 'dc8613306cfd493c848ad0e439732d22',
                                                                                                     'request_type': 'TRANSFER',
                                                                                                     'scope': 'data11_7TeV',
                                                                                                     'source_globus_endpoint_id': None,
                                                                                                     'src_rse': 'PSNC_DATADISK',
                                                                                                     'src_rse_id': '4e052ab45d1440a2b1a61c525479cc5f',
                                                                                                     'src_type': 'DISK',
                                                                                                     'verify_checksum': 'both'}},
                                                                               'dst_rse': 'CA-SFU-T2_DATADISK',
                                                                               'dst_type': 'DISK',
                                                                               'dst_url': 'srm://lcg-se1.sfu.computecanada.ca:8443/srm/managerv2?SFN=/atlas/atlasdatadisk/rucio/data11_7TeV/4d/ed/data11_7TeV.00189610.physics_Muons.merge.AOD.f405_m985._lb0416._0001.1',
                                                                               'duration': 2491,
                                                                               'external_host': 'https://fts3-atlas.cern.ch:8446',
                                                                               'file_state': 'FAILED',
                                                                               'filesize': 2074656655,
                                                                               'job_m_replica': False,
                                                                               'job_state': 'FAILED',
                                                                               'md5': None,
                                                                               'name': 'data11_7TeV.00189610.physics_Muons.merge.AOD.f405_m985._lb0416._0001.1',
                                                                               'new_state': <RequestState.FAILED: 'F'>,
                                                                               'previous_attempt_id': None,
                                                                               'reason': 'TRANSFER '
                                                                                         '[110] '
                                                                                         'TRANSFER  '
                                                                                         'Transfer '
                                                                                         'canceled '
                                                                                         'because '
                                                                                         'the '
                                                                                         'gsiftp '
                                                                                         'performance '
                                                                                         'marker '
                                                                                         'timeout '
                                                                                         'of '
                                                                                         '360 '
                                                                                         'seconds '
                                                                                         'has '
                                                                                         'been '
                                                                                         'exceeded, '
                                                                                         'or '
                                                                                         'all '
                                                                                         'performance '
                                                                                         'markers '
                                                                                         'during '
                                                                                         'that '
                                                                                         'period '
                                                                                         'indicated '
                                                                                         'zero '
                                                                                         'bytes '
                                                                                         'transferred',
                                                                               'request_id': 'dc8613306cfd493c848ad0e439732d22',
                                                                               'scope': 'data11_7TeV',
                                                                               'src_rse': 'PSNC_DATADISK',
                                                                               'src_rse_id': '4e052ab45d1440a2b1a61c525479cc5f',
                                                                               'src_type': 'DISK',
                                                                               'src_url': 'srm://se.reef.man.poznan.pl:8446/srm/managerv2?SFN=/dpm/reef.man.poznan.pl/home/atlas/atlasdatadisk/rucio/data11_7TeV/4d/ed/data11_7TeV.00189610.physics_Muons.merge.AOD.f405_m985._lb0416._0001.1',
                                                                               'staging_finished': None,
                                                                               'staging_start': None,
                                                                               'started_at': datetime.datetime(2021, 5, 17, 8, 10, 26),
                                                                               'transfer_id': 'f278c18a-b6e0-11eb-9e93-fa163eb04ff0',
                                                                               'transferred_at': datetime.datetime(2021, 5, 17, 8, 51, 57)}}}
```
The request ID in FTS is `'dc8613306cfd493c848ad0e439732d22'`. And there, I think, lies the problem. There is such a row in the `requests_history` table with `rule_id=HEXTORAW('00000000000000000000000000000000')` and `state='F'`.




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -753,12 +753,19 @@ def bulk_query_transfers(request_host, transfer_ids, transfertool='fts3', timeou
                 fts_resps[transfer_id] = Exception("Transfer id %s is not returned" % transfer_id)
             if fts_resps[transfer_id] and not isinstance(fts_resps[transfer_id], Exception):
                 for request_id in fts_resps[transfer_id]:
-                    if fts_resps[transfer_id][request_id]['file_state'] in (FTS_STATE.FAILED,
-                                                                            FTS_STATE.FINISHEDDIRTY,
-                                                                            FTS_STATE.CANCELED):
-                        fts_resps[transfer_id][request_id]['new_state'] = RequestState.FAILED
-                    elif fts_resps[transfer_id][request_id]['file_state'] in FTS_STATE.FINISHED:
-                        fts_resps[transfer_id][request_id]['new_state'] = RequestState.DONE
+                    status_dict = fts_resps[transfer_id][request_id]
+                    job_state = status_dict['job_state']
+                    file_state = status_dict['file_state']
+                    if job_state in (FTS_STATE.FAILED, FTS_STATE.CANCELED):
+                        status_dict['new_state'] = RequestState.FAILED
+                    elif job_state == FTS_STATE.FINISHED:
+                        status_dict['new_state'] = RequestState.DONE
+                    elif job_state == FTS_STATE.FINISHEDDIRTY:
+                        # Job partially completed. Verify the state of the file in the job
+                        if file_state in (FTS_STATE.FAILED, FTS_STATE.CANCELED):
+                            status_dict['new_state'] = RequestState.FAILED
+                        elif file_state == FTS_STATE.FINISHED:
+                            status_dict['new_state'] = RequestState.DONE
         return fts_resps
     elif transfertool == 'globus':
         try:
diff --git a/lib/rucio/transfertool/fts3.py b/lib/rucio/transfertool/fts3.py
--- a/lib/rucio/transfertool/fts3.py
+++ b/lib/rucio/transfertool/fts3.py
@@ -668,126 +668,59 @@ def __format_response(self, fts_job_response, fts_files_response):
         """
         Format the response format of FTS3 query.
 
-        :param fts_job_response: FTSs job query response.
-        :param fts_files_response: FTS3 files query response.
-        :returns: formatted response.
-        """
-        last_src_file = 0
-        for i in range(len(fts_files_response)):
-            if fts_files_response[i]['file_state'] in [FTS_STATE.FINISHED]:
-                last_src_file = i
-                break
-            if fts_files_response[i]['file_state'] != 'NOT_USED':
-                last_src_file = i
-
-        # for multiple sources, if not only the first source is used, we need to mark job_m_replica,
-        # then conveyor.common.add_monitor_message will correct the src_rse
-        job_m_replica = 'false'
-        if last_src_file > 0:
-            job_m_replica = 'true'
-
-        if fts_files_response[last_src_file]['start_time'] is None or fts_files_response[last_src_file]['finish_time'] is None:
-            duration = 0
-        else:
-            duration = (datetime.datetime.strptime(fts_files_response[last_src_file]['finish_time'], '%Y-%m-%dT%H:%M:%S')
-                        - datetime.datetime.strptime(fts_files_response[last_src_file]['start_time'], '%Y-%m-%dT%H:%M:%S')).seconds  # NOQA: W503
-
-        response = {'new_state': None,
-                    'transfer_id': fts_job_response.get('job_id'),
-                    'job_state': fts_job_response.get('job_state', None),
-                    'file_state': fts_files_response[last_src_file].get('file_state', None),
-                    'src_url': fts_files_response[last_src_file].get('source_surl', None),
-                    'dst_url': fts_files_response[last_src_file].get('dest_surl', None),
-                    'started_at': datetime.datetime.strptime(fts_files_response[last_src_file]['start_time'], '%Y-%m-%dT%H:%M:%S') if fts_files_response[last_src_file]['start_time'] else None,
-                    'transferred_at': datetime.datetime.strptime(fts_files_response[last_src_file]['finish_time'], '%Y-%m-%dT%H:%M:%S') if fts_files_response[last_src_file]['finish_time'] else None,
-                    'duration': duration,
-                    'reason': fts_files_response[last_src_file].get('reason', None),
-                    'scope': fts_job_response['job_metadata'].get('scope', None),
-                    'name': fts_job_response['job_metadata'].get('name', None),
-                    'src_rse': fts_job_response['job_metadata'].get('src_rse', None),
-                    'dst_rse': fts_job_response['job_metadata'].get('dst_rse', None),
-                    'request_id': fts_job_response['job_metadata'].get('request_id', None),
-                    'activity': fts_job_response['job_metadata'].get('activity', None),
-                    'src_rse_id': fts_job_response['file_metadata'].get('src_rse_id', None),
-                    'dest_rse_id': fts_job_response['job_metadata'].get('dest_rse_id', None),
-                    'previous_attempt_id': fts_job_response['job_metadata'].get('previous_attempt_id', None),
-                    'adler32': fts_job_response['job_metadata'].get('adler32', None),
-                    'md5': fts_job_response['job_metadata'].get('md5', None),
-                    'filesize': fts_job_response['job_metadata'].get('filesize', None),
-                    'external_host': self.external_host,
-                    'job_m_replica': job_m_replica,
-                    'details': {'files': fts_job_response['job_metadata']}}
-        return response
-
-    def __format_new_response(self, fts_job_response, fts_files_response):
-        """
-        Format the response format of FTS3 query.
-
         :param fts_job_response: FTSs job query response.
         :param fts_files_response: FTS3 files query response.
         :returns: formatted response.
         """
 
         resps = {}
-        if 'request_id' in fts_job_response['job_metadata']:
-            # submitted by old submitter
-            request_id = fts_job_response['job_metadata']['request_id']
-            resps[request_id] = self.__format_response(fts_job_response, fts_files_response)
-        else:
-            multi_sources = fts_job_response['job_metadata'].get('multi_sources', False)
-            for file_resp in fts_files_response:
-                # for multiple source replicas jobs, the file_metadata(request_id) will be the same.
-                # The next used file will overwrite the current used one. Only the last used file will return.
-                if file_resp['file_state'] == 'NOT_USED':
-                    continue
-
-                # not terminated job
-                if file_resp['file_state'] not in [FTS_STATE.FAILED,
-                                                   FTS_STATE.FINISHEDDIRTY,
-                                                   FTS_STATE.CANCELED,
-                                                   FTS_STATE.FINISHED]:
-                    continue
-
-                if file_resp['start_time'] is None or file_resp['finish_time'] is None:
-                    duration = 0
-                else:
-                    duration = (datetime.datetime.strptime(file_resp['finish_time'], '%Y-%m-%dT%H:%M:%S')
-                                - datetime.datetime.strptime(file_resp['start_time'], '%Y-%m-%dT%H:%M:%S')).seconds  # NOQA: W503
-
-                request_id = file_resp['file_metadata']['request_id']
-                resps[request_id] = {'new_state': None,
-                                     'transfer_id': fts_job_response.get('job_id'),
-                                     'job_state': fts_job_response.get('job_state', None),
-                                     'file_state': file_resp.get('file_state', None),
-                                     'src_url': file_resp.get('source_surl', None),
-                                     'dst_url': file_resp.get('dest_surl', None),
-                                     'started_at': datetime.datetime.strptime(file_resp['start_time'], '%Y-%m-%dT%H:%M:%S') if file_resp['start_time'] else None,
-                                     'staging_start': datetime.datetime.strptime(file_resp['staging_start'], '%Y-%m-%dT%H:%M:%S') if file_resp['staging_start'] else None,
-                                     'staging_finished': datetime.datetime.strptime(file_resp['staging_finished'], '%Y-%m-%dT%H:%M:%S') if file_resp['staging_finished'] else None,
-                                     'transferred_at': datetime.datetime.strptime(file_resp['finish_time'], '%Y-%m-%dT%H:%M:%S') if file_resp['finish_time'] else None,
-                                     'duration': duration,
-                                     'reason': file_resp.get('reason', None),
-                                     'scope': file_resp['file_metadata'].get('scope', None),
-                                     'name': file_resp['file_metadata'].get('name', None),
-                                     'src_type': file_resp['file_metadata'].get('src_type', None),
-                                     'dst_type': file_resp['file_metadata'].get('dst_type', None),
-                                     'src_rse': file_resp['file_metadata'].get('src_rse', None),
-                                     'dst_rse': file_resp['file_metadata'].get('dst_rse', None),
-                                     'request_id': file_resp['file_metadata'].get('request_id', None),
-                                     'activity': file_resp['file_metadata'].get('activity', None),
-                                     'src_rse_id': file_resp['file_metadata'].get('src_rse_id', None),
-                                     'dest_rse_id': file_resp['file_metadata'].get('dest_rse_id', None),
-                                     'previous_attempt_id': file_resp['file_metadata'].get('previous_attempt_id', None),
-                                     'adler32': file_resp['file_metadata'].get('adler32', None),
-                                     'md5': file_resp['file_metadata'].get('md5', None),
-                                     'filesize': file_resp['file_metadata'].get('filesize', None),
-                                     'external_host': self.external_host,
-                                     'job_m_replica': multi_sources,
-                                     'details': {'files': file_resp['file_metadata']}}
-
-                # multiple source replicas jobs and we found the successful one, it's the final state.
-                if multi_sources and file_resp['file_state'] in [FTS_STATE.FINISHED]:
-                    break
+        multi_sources = fts_job_response['job_metadata'].get('multi_sources', False)
+        for file_resp in fts_files_response:
+            # for multiple source replicas jobs, the file_metadata(request_id) will be the same.
+            # The next used file will overwrite the current used one. Only the last used file will return.
+            if multi_sources and file_resp['file_state'] == 'NOT_USED':
+                continue
+
+            if file_resp['start_time'] is None or file_resp['finish_time'] is None:
+                duration = 0
+            else:
+                duration = (datetime.datetime.strptime(file_resp['finish_time'], '%Y-%m-%dT%H:%M:%S')
+                            - datetime.datetime.strptime(file_resp['start_time'], '%Y-%m-%dT%H:%M:%S')).seconds  # NOQA: W503
+
+            request_id = file_resp['file_metadata']['request_id']
+            resps[request_id] = {'new_state': None,
+                                 'transfer_id': fts_job_response.get('job_id'),
+                                 'job_state': fts_job_response.get('job_state', None),
+                                 'file_state': file_resp.get('file_state', None),
+                                 'src_url': file_resp.get('source_surl', None),
+                                 'dst_url': file_resp.get('dest_surl', None),
+                                 'started_at': datetime.datetime.strptime(file_resp['start_time'], '%Y-%m-%dT%H:%M:%S') if file_resp['start_time'] else None,
+                                 'staging_start': datetime.datetime.strptime(file_resp['staging_start'], '%Y-%m-%dT%H:%M:%S') if file_resp['staging_start'] else None,
+                                 'staging_finished': datetime.datetime.strptime(file_resp['staging_finished'], '%Y-%m-%dT%H:%M:%S') if file_resp['staging_finished'] else None,
+                                 'transferred_at': datetime.datetime.strptime(file_resp['finish_time'], '%Y-%m-%dT%H:%M:%S') if file_resp['finish_time'] else None,
+                                 'duration': duration,
+                                 'reason': file_resp.get('reason', None),
+                                 'scope': file_resp['file_metadata'].get('scope', None),
+                                 'name': file_resp['file_metadata'].get('name', None),
+                                 'src_type': file_resp['file_metadata'].get('src_type', None),
+                                 'dst_type': file_resp['file_metadata'].get('dst_type', None),
+                                 'src_rse': file_resp['file_metadata'].get('src_rse', None),
+                                 'dst_rse': file_resp['file_metadata'].get('dst_rse', None),
+                                 'request_id': file_resp['file_metadata'].get('request_id', None),
+                                 'activity': file_resp['file_metadata'].get('activity', None),
+                                 'src_rse_id': file_resp['file_metadata'].get('src_rse_id', None),
+                                 'dest_rse_id': file_resp['file_metadata'].get('dest_rse_id', None),
+                                 'previous_attempt_id': file_resp['file_metadata'].get('previous_attempt_id', None),
+                                 'adler32': file_resp['file_metadata'].get('adler32', None),
+                                 'md5': file_resp['file_metadata'].get('md5', None),
+                                 'filesize': file_resp['file_metadata'].get('filesize', None),
+                                 'external_host': self.external_host,
+                                 'job_m_replica': multi_sources,
+                                 'details': {'files': file_resp['file_metadata']}}
+
+            # multiple source replicas jobs and we found the successful one, it's the final state.
+            if multi_sources and file_resp['file_state'] in [FTS_STATE.FINISHED]:
+                break
         return resps
 
     def __bulk_query_responses(self, jobs_response):
@@ -808,7 +741,7 @@ def __bulk_query_responses(self, jobs_response):
                     responses[transfer_id] = {}
                     continue
 
-                resps = self.__format_new_response(job_response, files_response)
+                resps = self.__format_response(job_response, files_response)
                 responses[transfer_id] = resps
             elif job_response['http_status'] == '404 Not Found':
                 # Lost transfer


Test patch:
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -82,7 +82,7 @@ jobs:
       - name: Test Protocol XrootD
         run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short test_rse_protocol_xrootd.py
       - name: Test Conveyor
-        run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short test_conveyor.py::test_multihop_intermediate_replica_lifecycle
+        run: docker exec -t dev_rucio_1 tools/pytest.sh -v --tb=short test_conveyor.py
       - name: Execute transfer and export FTS transfer details
         id: tpc
         shell: bash
diff --git a/lib/rucio/tests/test_conveyor.py b/lib/rucio/tests/test_conveyor.py
--- a/lib/rucio/tests/test_conveyor.py
+++ b/lib/rucio/tests/test_conveyor.py
@@ -32,7 +32,7 @@
 from rucio.daemons.reaper.reaper import reaper
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import RequestState, ReplicaState
-from rucio.db.sqla.session import transactional_session
+from rucio.db.sqla.session import read_session, transactional_session
 from rucio.tests.common import skip_rse_tests_with_accounts
 
 
@@ -51,6 +51,20 @@ def __wait_for_replica_transfer(dst_rse_id, scope, name, max_wait_seconds=60):
     return replica
 
 
+def __wait_for_request_state(dst_rse_id, scope, name, state, max_wait_seconds=60):
+    """
+    Wait for the request state to be updated to the given expected state as a result of a pending transfer
+    """
+    request = None
+    for _ in range(max_wait_seconds):
+        poller(once=True, older_than=0, partition_wait_time=None)
+        request = request_core.get_request_by_did(rse_id=dst_rse_id, scope=scope, name=name)
+        if request['state'] == state:
+            break
+        time.sleep(1)
+    return request
+
+
 @skip_rse_tests_with_accounts
 @pytest.mark.dirty(reason="leaves files in XRD containers")
 @pytest.mark.noparallel(reason="uses predefined RSEs; runs submitter, poller and finisher; changes XRD3 usage and limits")
@@ -139,3 +153,133 @@ def _cleanup_all_usage_and_limits(rse_id, session=None):
             session.query(models.RSEUsage).filter_by(rse_id=rse_id, source='storage').delete()
 
         _cleanup_all_usage_and_limits(rse_id=jump_rse_id)
+
+
+@skip_rse_tests_with_accounts
+@pytest.mark.noparallel(reason="uses predefined RSEs; runs submitter, poller and finisher")
+@pytest.mark.parametrize("core_config_mock", [{"table_content": [
+    ('transfers', 'use_multihop', True),
+]}], indirect=True)
+@pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
+    'rucio.core.rse_expression_parser',  # The list of multihop RSEs is retrieved by rse expression
+    'rucio.core.config',
+]}], indirect=True)
+def test_fts_non_recoverable_failures_handled_on_multihop(vo, did_factory, root_account, replica_client, core_config_mock, caches_mock):
+    """
+    Verify that the poller correctly handles non-recoverable FTS job failures
+    """
+    src_rse = 'XRD1'
+    src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo)
+    jump_rse = 'XRD3'
+    jump_rse_id = rse_core.get_rse_id(rse=jump_rse, vo=vo)
+    dst_rse = 'XRD4'
+    dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo)
+
+    all_rses = [src_rse_id, jump_rse_id, dst_rse_id]
+
+    # Register a did which doesn't exist. It will trigger an non-recoverable error during the FTS transfer.
+    did = did_factory.random_did()
+    replica_client.add_replicas(rse=src_rse, files=[{'scope': did['scope'].external, 'name': did['name'], 'bytes': 1, 'adler32': 'aaaaaaaa'}])
+
+    rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+
+    request = __wait_for_request_state(dst_rse_id=dst_rse_id, state=RequestState.FAILED, **did)
+    assert request['state'] == RequestState.FAILED
+    request = request_core.get_request_by_did(rse_id=jump_rse_id, **did)
+    assert request['state'] == RequestState.FAILED
+
+
+@skip_rse_tests_with_accounts
+@pytest.mark.dirty(reason="leaves files in XRD containers")
+@pytest.mark.noparallel(reason="uses predefined RSEs; runs submitter, poller and finisher")
+@pytest.mark.parametrize("core_config_mock", [{"table_content": [
+    ('transfers', 'use_multihop', True),
+]}], indirect=True)
+@pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
+    'rucio.core.rse_expression_parser',  # The list of multihop RSEs is retrieved by rse expression
+    'rucio.core.config',
+]}], indirect=True)
+def test_fts_recoverable_failures_handled_on_multihop(vo, did_factory, root_account, replica_client, file_factory, core_config_mock, caches_mock):
+    """
+    Verify that the poller correctly handles recoverable FTS job failures
+    """
+    src_rse = 'XRD1'
+    src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo)
+    jump_rse = 'XRD3'
+    jump_rse_id = rse_core.get_rse_id(rse=jump_rse, vo=vo)
+    dst_rse = 'XRD4'
+    dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo)
+
+    all_rses = [src_rse_id, jump_rse_id, dst_rse_id]
+
+    # Create and upload a real file, but register it with wrong checksum. This will trigger
+    # a FTS "Recoverable" failure on checksum validation
+    local_file = file_factory.file_generator()
+    did = did_factory.random_did()
+    did_factory.upload_client.upload(
+        [
+            {
+                'path': local_file,
+                'rse': src_rse,
+                'did_scope': did['scope'].external,
+                'did_name': did['name'],
+                'no_register': True,
+            }
+        ]
+    )
+    replica_client.add_replicas(rse=src_rse, files=[{'scope': did['scope'].external, 'name': did['name'], 'bytes': 1, 'adler32': 'aaaaaaaa'}])
+
+    rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+
+    request = __wait_for_request_state(dst_rse_id=dst_rse_id, state=RequestState.FAILED, **did)
+    assert request['state'] == RequestState.FAILED
+    request = request_core.get_request_by_did(rse_id=jump_rse_id, **did)
+    assert request['state'] == RequestState.FAILED
+
+
+@skip_rse_tests_with_accounts
+@pytest.mark.dirty(reason="leaves files in XRD containers")
+@pytest.mark.noparallel(reason="uses predefined RSEs; runs submitter, poller and finisher")
+@pytest.mark.parametrize("core_config_mock", [{"table_content": [
+    ('transfers', 'use_multihop', True),
+]}], indirect=True)
+@pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
+    'rucio.core.rse_expression_parser',  # The list of multihop RSEs is retrieved by rse expression
+    'rucio.core.config',
+]}], indirect=True)
+def test_multisource(vo, did_factory, root_account, replica_client, core_config_mock, caches_mock):
+    src_rse1 = 'XRD4'
+    src_rse1_id = rse_core.get_rse_id(rse=src_rse1, vo=vo)
+    src_rse2 = 'XRD1'
+    src_rse2_id = rse_core.get_rse_id(rse=src_rse2, vo=vo)
+    dst_rse = 'XRD3'
+    dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo)
+
+    all_rses = [src_rse1_id, src_rse2_id, dst_rse_id]
+
+    # Add a good replica on the RSE which has a higher distance ranking
+    did = did_factory.upload_test_file(src_rse1)
+    # Add non-existing replica which will fail during multisource transfers on the RSE with lower cost (will be the preferred source)
+    replica_client.add_replicas(rse=src_rse2, files=[{'scope': did['scope'].external, 'name': did['name'], 'bytes': 1, 'adler32': 'aaaaaaaa'}])
+
+    rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+
+    @read_session
+    def __source_exists(src_rse_id, scope, name, session=None):
+        return session.query(models.Source) \
+            .filter(models.Source.rse_id == src_rse_id) \
+            .filter(models.Source.scope == scope) \
+            .filter(models.Source.name == name) \
+            .count() != 0
+
+    # Entries in the source table must be created for both sources of the multi-source transfer
+    assert __source_exists(src_rse_id=src_rse1_id, **did)
+    assert __source_exists(src_rse_id=src_rse2_id, **did)
+
+    replica = __wait_for_replica_transfer(dst_rse_id=dst_rse_id, **did)
+    assert replica['state'] == ReplicaState.AVAILABLE
+    assert not __source_exists(src_rse_id=src_rse1_id, **did)
+    assert not __source_exists(src_rse_id=src_rse2_id, **did)