Repository: rucio/rucio

Issue:
Regression introduced with storaged-managed QoS feature
Motivation
----------
The recent feature with storage-managed QoS #4751 introduced a regression that a `lifetime` (=`copy_pin_lifetime`) was set for each transfer. This meant that all transfers were submitted including a STAGING request, which basically failed every transfer.

Modification
------------
This needs to be changed to only add the `lifetime` to transfers which actually do require a STAGING request.
Also a testcase needs to be added to avoid this for the future, our established tests did not reveal this issue during development.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/lock.py b/lib/rucio/core/lock.py
--- a/lib/rucio/core/lock.py
+++ b/lib/rucio/core/lock.py
@@ -23,7 +23,7 @@
 import rucio.core.did
 from rucio.common.exception import DataIdentifierNotFound
 from rucio.core.lifetime_exception import define_eol
-from rucio.core.rse import get_rse_name
+from rucio.core.rse import get_rse_attribute, get_rse_name
 
 from rucio.db.sqla import models, filter_thread_work
 from rucio.db.sqla.constants import LockState, RuleState, RuleGrouping, DIDType, RuleNotification
@@ -398,7 +398,14 @@ def failed_transfer(scope, name, rse_id, error_message=None, broken_rule_id=None
     :param session:         The database session in use.
     """
 
-    locks = session.query(models.ReplicaLock).with_for_update(nowait=nowait).filter_by(scope=scope, name=name, rse_id=rse_id)
+    staging_required = get_rse_attribute(rse_id, 'staging_required', session=session)
+    if staging_required:
+        rse_name = get_rse_name(rse_id=rse_id, session=session)
+        logger(logging.DEBUG, f'Destination RSE {rse_name} is type staging_required so do not update other OK replica locks.')
+        locks = session.query(models.ReplicaLock).with_for_update(nowait=nowait).filter_by(scope=scope, name=name, rse_id=rse_id, state=LockState.REPLICATING)
+    else:
+        locks = session.query(models.ReplicaLock).with_for_update(nowait=nowait).filter_by(scope=scope, name=name, rse_id=rse_id)
+
     for lock in locks:
         if lock.state == LockState.STUCK:
             continue
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -1218,7 +1218,7 @@ def update_rule(rule_id, options, session=None):
                     request_core.queue_requests(requests=[create_transfer_dict(dest_rse_id=lock.rse_id,
                                                                                request_type=RequestType.TRANSFER,
                                                                                scope=lock.scope, name=lock.name, rule=rule, lock=lock, bytes_=bytes_, md5=md5, adler32=adler32,
-                                                                               ds_scope=rule.scope, ds_name=rule.name, lifetime=None, activity=rule.activity, session=session)], session=session)
+                                                                               ds_scope=rule.scope, ds_name=rule.name, copy_pin_lifetime=None, activity=rule.activity, session=session)], session=session)
 
             elif key == 'account':
                 # Check if the account exists
@@ -1831,7 +1831,7 @@ def update_rules_for_bad_replica(scope, name, rse_id, nowait=False, session=None
             request_core.queue_requests(requests=[create_transfer_dict(dest_rse_id=rse_id,
                                                                        request_type=RequestType.TRANSFER,
                                                                        scope=scope, name=name, rule=rule, lock=lock, bytes_=bytes_, md5=md5, adler32=adler32,
-                                                                       ds_scope=ds_scope, ds_name=ds_name, lifetime=None, activity='Recovery', session=session)], session=session)
+                                                                       ds_scope=ds_scope, ds_name=ds_name, copy_pin_lifetime=None, activity='Recovery', session=session)], session=session)
         lock.state = LockState.REPLICATING
         if rule.state == RuleState.SUSPENDED:
             pass
diff --git a/lib/rucio/core/rule_grouping.py b/lib/rucio/core/rule_grouping.py
--- a/lib/rucio/core/rule_grouping.py
+++ b/lib/rucio/core/rule_grouping.py
@@ -26,7 +26,7 @@
 import rucio.core.did
 import rucio.core.lock
 import rucio.core.replica
-from rucio.core.rse import get_rse
+from rucio.core.rse import get_rse, get_rse_attribute, get_rse_name
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import LockState, RuleGrouping, ReplicaState, RequestType, DIDType, OBSOLETE
 from rucio.db.sqla.session import transactional_session
@@ -149,29 +149,29 @@ def repair_stuck_locks_and_apply_rule_grouping(datasetfiles, locks, replicas, so
 
 
 @transactional_session
-def create_transfer_dict(dest_rse_id, request_type, scope, name, rule, lock=None, bytes_=None, md5=None, adler32=None, ds_scope=None, ds_name=None, lifetime=None, activity=None, retry_count=None, session=None):
+def create_transfer_dict(dest_rse_id, request_type, scope, name, rule, lock=None, bytes_=None, md5=None, adler32=None, ds_scope=None, ds_name=None, copy_pin_lifetime=None, activity=None, retry_count=None, session=None):
     """
     This method creates a transfer dictionary and returns it
 
-    :param dest_rse_id:   The destination RSE id.
-    :param request_Type:  The request type.
-    :param scope:         The scope of the file.
-    :param name:          The name of the file.
-    :param rule:          The rule responsible for the transfer.
-    :param lock:          The lock responsible for the transfer.
-    :param bytes_:         The filesize of the file in bytes.
-    :param md5:           The md5 checksum of the file.
-    :param adler32:       The adler32 checksum of the file.
-    :param ds_scope:      Dataset the file belongs to.
-    :param ds_name:       Dataset the file belongs to.
-    :param lifetime:      Lifetime in the case of STAGIN requests.
-    :param activity:      Activity to be used.
-    :param session:       Session of the db.
-    :returns:             Request dictionary.
+    :param dest_rse_id:         The destination RSE id.
+    :param request_Type:        The request type.
+    :param scope:               The scope of the file.
+    :param name:                The name of the file.
+    :param rule:                The rule responsible for the transfer.
+    :param lock:                The lock responsible for the transfer.
+    :param bytes_:              The filesize of the file in bytes.
+    :param md5:                 The md5 checksum of the file.
+    :param adler32:             The adler32 checksum of the file.
+    :param ds_scope:            Dataset the file belongs to.
+    :param ds_name:             Dataset the file belongs to.
+    :param copy_pin_lifetime:   Lifetime in the case of STAGIN requests.
+    :param activity:            Activity to be used.
+    :param session:             Session of the db.
+    :returns:                   Request dictionary.
     """
     attributes = {'activity': activity or rule.activity or 'default',
                   'source_replica_expression': rule.source_replica_expression,
-                  'lifetime': lifetime,
+                  'lifetime': copy_pin_lifetime,
                   'ds_scope': ds_scope,
                   'ds_name': ds_name,
                   'bytes': bytes_,
@@ -825,10 +825,14 @@ def __create_lock_and_replica(file, dataset, rule, rse_id, staging_area, availab
     :attention:                  This method modifies the contents of the locks, locks_to_create, replicas_to_create and replicas input parameters.
     """
 
+    if rule.expires_at:
+        copy_pin_lifetime = rule.expires_at - datetime.utcnow()
+        copy_pin_lifetime = copy_pin_lifetime.seconds + copy_pin_lifetime.days * 24 * 3600
+    else:
+        copy_pin_lifetime = None
+
     # If it is a Staging Area, the pin has to be extended
     if staging_area:
-        lifetime = rule.expires_at - datetime.utcnow()
-        lifetime = lifetime.seconds + lifetime.days * 24 * 3600
         transfers_to_create.append(create_transfer_dict(dest_rse_id=rse_id,
                                                         request_type=RequestType.STAGEIN,
                                                         scope=file['scope'],
@@ -839,9 +843,19 @@ def __create_lock_and_replica(file, dataset, rule, rse_id, staging_area, availab
                                                         adler32=file['adler32'],
                                                         ds_scope=dataset['scope'],
                                                         ds_name=dataset['name'],
-                                                        lifetime=lifetime,
+                                                        copy_pin_lifetime=copy_pin_lifetime,
                                                         session=session))
 
+    # If staging_required type RSE then set pin to RSE attribute maximum_pin_lifetime
+    staging_required = get_rse_attribute(rse_id, 'staging_required', session=session)
+    maximum_pin_lifetime = get_rse_attribute(rse_id, 'maximum_pin_lifetime', session=session)
+
+    if staging_required:
+        if (not copy_pin_lifetime and maximum_pin_lifetime) or (copy_pin_lifetime and maximum_pin_lifetime and copy_pin_lifetime < int(maximum_pin_lifetime)):
+            copy_pin_lifetime = maximum_pin_lifetime
+        rse_name = get_rse_name(rse_id=rse_id, session=session)
+        logger(logging.DEBUG, f'Destination RSE {rse_name} is type staging_required with pin value: {copy_pin_lifetime}')
+
     existing_replicas = [replica for replica in replicas[(file['scope'], file['name'])] if replica.rse_id == rse_id]
 
     if existing_replicas:  # A replica already exists (But could be UNAVAILABLE)
@@ -855,12 +869,27 @@ def __create_lock_and_replica(file, dataset, rule, rse_id, staging_area, availab
                                      name=file['name'],
                                      bytes_=file['bytes'],
                                      existing_replica=existing_replica,
-                                     state=LockState.OK)
+                                     state=LockState.OK if not staging_required else LockState.REPLICATING)
             if rse_id not in locks_to_create:
                 locks_to_create[rse_id] = []
             locks_to_create[rse_id].append(new_lock)
             locks[(file['scope'], file['name'])].append(new_lock)
-            return False
+            if not staging_required:
+                return False
+
+            transfers_to_create.append(create_transfer_dict(dest_rse_id=rse_id,
+                                                            request_type=RequestType.STAGEIN,
+                                                            scope=file['scope'],
+                                                            name=file['name'],
+                                                            rule=rule,
+                                                            lock=new_lock,
+                                                            bytes_=file['bytes'],
+                                                            md5=file['md5'],
+                                                            adler32=file['adler32'],
+                                                            ds_scope=dataset['scope'],
+                                                            ds_name=dataset['name'],
+                                                            copy_pin_lifetime=copy_pin_lifetime,
+                                                            session=session))
 
         # Replica is not available -- UNAVAILABLE
         elif existing_replica.state == ReplicaState.UNAVAILABLE:
@@ -881,7 +910,7 @@ def __create_lock_and_replica(file, dataset, rule, rse_id, staging_area, availab
                 locks_to_create[rse_id] = []
             locks_to_create[rse_id].append(new_lock)
             locks[(file['scope'], file['name'])].append(new_lock)
-            if not staging_area and available_source_replica and availability_write:
+            if not staging_area and not staging_required and available_source_replica and availability_write:
                 transfers_to_create.append(create_transfer_dict(dest_rse_id=rse_id,
                                                                 request_type=RequestType.TRANSFER,
                                                                 scope=file['scope'],
@@ -895,6 +924,21 @@ def __create_lock_and_replica(file, dataset, rule, rse_id, staging_area, availab
                                                                 ds_name=dataset['name'],
                                                                 session=session))
                 return True
+            elif staging_required:  # include the copy_pin_lifetime in dictionary
+                transfers_to_create.append(create_transfer_dict(dest_rse_id=rse_id,
+                                                                request_type=RequestType.TRANSFER,
+                                                                scope=file['scope'],
+                                                                name=file['name'],
+                                                                rule=rule,
+                                                                lock=new_lock,
+                                                                bytes_=file['bytes'],
+                                                                md5=file['md5'],
+                                                                adler32=file['adler32'],
+                                                                ds_scope=dataset['scope'],
+                                                                ds_name=dataset['name'],
+                                                                copy_pin_lifetime=copy_pin_lifetime,
+                                                                session=session))
+                return True
             return False
         # Replica is not available at the rse yet -- COPYING
         else:
@@ -942,7 +986,21 @@ def __create_lock_and_replica(file, dataset, rule, rse_id, staging_area, availab
         locks_to_create[rse_id].append(new_lock)
         locks[(file['scope'], file['name'])].append(new_lock)
 
-        if not staging_area and available_source_replica and availability_write:
+        if not staging_area and not staging_required and available_source_replica and availability_write:
+            transfers_to_create.append(create_transfer_dict(dest_rse_id=rse_id,
+                                                            request_type=RequestType.TRANSFER,
+                                                            scope=file['scope'],
+                                                            name=file['name'],
+                                                            rule=rule,
+                                                            lock=new_lock,
+                                                            bytes_=file['bytes'],
+                                                            md5=file['md5'],
+                                                            adler32=file['adler32'],
+                                                            ds_scope=dataset['scope'],
+                                                            ds_name=dataset['name'],
+                                                            session=session))
+            return True
+        elif staging_required:  # include the copy_pin_lifetime in dictionary
             transfers_to_create.append(create_transfer_dict(dest_rse_id=rse_id,
                                                             request_type=RequestType.TRANSFER,
                                                             scope=file['scope'],
@@ -954,6 +1012,7 @@ def __create_lock_and_replica(file, dataset, rule, rse_id, staging_area, availab
                                                             adler32=file['adler32'],
                                                             ds_scope=dataset['scope'],
                                                             ds_name=dataset['name'],
+                                                            copy_pin_lifetime=copy_pin_lifetime,
                                                             session=session))
             return True
         return False
@@ -1053,8 +1112,8 @@ def __update_lock_replica_and_create_transfer(lock, replica, rule, dataset, tran
         lock.repair_cnt += 1
 
     if get_rse(rse_id=lock.rse_id, session=session).staging_area:
-        lifetime = rule.expires_at - datetime.utcnow()
-        lifetime = lifetime.seconds + lifetime.days * 24 * 3600
+        copy_pin_lifetime = rule.expires_at - datetime.utcnow()
+        copy_pin_lifetime = copy_pin_lifetime.seconds + copy_pin_lifetime.days * 24 * 3600
         transfers_to_create.append(create_transfer_dict(dest_rse_id=lock.rse_id,
                                                         scope=lock.scope,
                                                         name=lock.name,
@@ -1065,7 +1124,7 @@ def __update_lock_replica_and_create_transfer(lock, replica, rule, dataset, tran
                                                         adler32=replica.adler32,
                                                         ds_scope=dataset['scope'],
                                                         ds_name=dataset['name'],
-                                                        lifetime=lifetime,
+                                                        copy_pin_lifetime=copy_pin_lifetime,
                                                         request_type=RequestType.STAGEIN,
                                                         session=session))
     else:


Test patch:
diff --git a/lib/rucio/tests/test_conveyor.py b/lib/rucio/tests/test_conveyor.py
--- a/lib/rucio/tests/test_conveyor.py
+++ b/lib/rucio/tests/test_conveyor.py
@@ -28,11 +28,13 @@
 from rucio.core import config as core_config
 from rucio.core import did as did_core
 from rucio.core import distance as distance_core
+from rucio.core import lock as lock_core
 from rucio.core import message as message_core
 from rucio.core import replica as replica_core
 from rucio.core import request as request_core
 from rucio.core import rse as rse_core
 from rucio.core import rule as rule_core
+from rucio.core.account_limit import set_local_account_limit
 from rucio.daemons.conveyor.finisher import finisher
 from rucio.daemons.conveyor.poller import poller
 from rucio.daemons.conveyor.preparer import preparer
@@ -42,7 +44,7 @@
 from rucio.daemons.conveyor.receiver import receiver, graceful_stop as receiver_graceful_stop
 from rucio.daemons.reaper.reaper import reaper
 from rucio.db.sqla import models
-from rucio.db.sqla.constants import RequestState, RequestType, ReplicaState, RSEType
+from rucio.db.sqla.constants import LockState, RequestState, RequestType, ReplicaState, RSEType, RuleState
 from rucio.db.sqla.session import read_session, transactional_session
 from rucio.tests.common import skip_rse_tests_with_accounts
 from rucio.transfertool.fts3 import FTS3Transfertool
@@ -55,10 +57,10 @@ def __wait_for_replica_transfer(dst_rse_id, scope, name, state=ReplicaState.AVAI
     """
     Wait for the replica to become AVAILABLE on the given RSE as a result of a pending transfer
     """
-    replica = None
+    replica = {}
     for _ in range(max_wait_seconds):
-        poller(once=True, older_than=0, partition_wait_time=None)
-        finisher(once=True, partition_wait_time=None)
+        poller(once=True, older_than=0, partition_wait_time=0)
+        finisher(once=True, partition_wait_time=0)
         replica = replica_core.get_replica(rse_id=dst_rse_id, scope=scope, name=name)
         if replica['state'] == state:
             break
@@ -70,10 +72,10 @@ def __wait_for_request_state(dst_rse_id, scope, name, state, max_wait_seconds=MA
     """
     Wait for the request state to be updated to the given expected state as a result of a pending transfer
     """
-    request = None
+    request = {}
     for _ in range(max_wait_seconds):
         if run_poller:
-            poller(once=True, older_than=0, partition_wait_time=None)
+            poller(once=True, older_than=0, partition_wait_time=0)
         request = request_core.get_request_by_did(rse_id=dst_rse_id, scope=scope, name=name)
         if request['state'] == state:
             break
@@ -161,7 +163,7 @@ def test_multihop_intermediate_replica_lifecycle(vo, did_factory, root_account,
 
     # Copy replica to a second source. To avoid the special case of having a unique last replica, which could be handled in a special (more careful) way
     rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=src_rse2_name, grouping='ALL', weight=None, lifetime=3600, locked=False, subscription_id=None)
-    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], partition_wait_time=None, transfertype='single', filter_transfertool=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], partition_wait_time=0, transfertype='single', filter_transfertool=None)
     replica = __wait_for_replica_transfer(dst_rse_id=src_rse2_id, **did)
     assert replica['state'] == ReplicaState.AVAILABLE
 
@@ -172,7 +174,7 @@ def test_multihop_intermediate_replica_lifecycle(vo, did_factory, root_account,
 
         # Submit transfers to FTS
         # Ensure a replica was created on the intermediary host with epoch tombstone
-        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], partition_wait_time=None, transfertype='single', filter_transfertool=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], partition_wait_time=0, transfertype='single', filter_transfertool=None)
         request = request_core.get_request_by_did(rse_id=jump_rse_id, **did)
         assert request['state'] == RequestState.SUBMITTED
         replica = replica_core.get_replica(rse_id=jump_rse_id, **did)
@@ -214,7 +216,7 @@ def __fake_source_ranking(session=None):
         assert __get_source(request_id=request['id'], src_rse_id=src_rse2_id, **did).ranking == 0
         # Only group_bulk=1 part of the path was submitted.
         # run submitter again to copy from jump rse to destination rse
-        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], partition_wait_time=None, transfertype='single', filter_transfertool=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
         # Wait for the destination replica to become ready
         replica = __wait_for_replica_transfer(dst_rse_id=dst_rse_id, **did)
@@ -269,7 +271,7 @@ def test_fts_non_recoverable_failures_handled_on_multihop(vo, did_factory, root_
     replica_client.add_replicas(rse=src_rse, files=[{'scope': did['scope'].external, 'name': did['name'], 'bytes': 1, 'adler32': 'aaaaaaaa'}])
 
     rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
-    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
     request = __wait_for_request_state(dst_rse_id=dst_rse_id, state=RequestState.FAILED, **did)
     assert 'Unused hop in multi-hop' in request['err_msg']
@@ -282,8 +284,8 @@ def test_fts_non_recoverable_failures_handled_on_multihop(vo, did_factory, root_
     assert metrics_mock.get_sample_value('rucio_daemons_conveyor_poller_update_request_state_total', labels={'updated': 'True'}) >= 2
 
     # Finisher will handle transfers of the same multihop one hop at a time
-    finisher(once=True, partition_wait_time=None)
-    finisher(once=True, partition_wait_time=None)
+    finisher(once=True, partition_wait_time=0)
+    finisher(once=True, partition_wait_time=0)
     # The intermediate request must not be re-scheduled by finisher
     with pytest.raises(RequestNotFound):
         request_core.get_request_by_did(rse_id=jump_rse_id, **did)
@@ -335,7 +337,7 @@ def test_fts_recoverable_failures_handled_on_multihop(vo, did_factory, root_acco
     replica_client.add_replicas(rse=src_rse, files=[{'scope': did['scope'].external, 'name': did['name'], 'bytes': 1, 'adler32': 'aaaaaaaa'}])
 
     rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
-    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
     request = __wait_for_request_state(dst_rse_id=dst_rse_id, state=RequestState.FAILED, **did)
     assert request['state'] == RequestState.FAILED
@@ -377,7 +379,7 @@ def test_multisource(vo, did_factory, root_account, replica_client, core_config_
     container = did_factory.make_container()
     did_core.attach_dids(dids=[dataset], account=root_account, **container)
     rule_core.add_rule(dids=[container], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
-    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
     @read_session
     def __source_exists(src_rse_id, scope, name, session=None):
@@ -450,14 +452,14 @@ def test_multisource_receiver(vo, did_factory, replica_client, root_account, met
         did_core.attach_dids(dids=[did], account=root_account, **dataset)
         did_core.set_metadata(did['scope'], did['name'], 'datatype', 'RAW')
         rule_core.add_rule(dids=[dataset], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
-        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
         # After submission, the source rse is the one which will fail
         request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
         assert request['source_rse'] == src_rse2
         assert request['source_rse_id'] == src_rse2_id
 
-        request = None
+        request = {}
         for _ in range(MAX_POLL_WAIT_SECONDS):
             request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
             # The request must not be marked as failed. Not even temporarily. It is a multi-source transfer and the
@@ -523,7 +525,7 @@ def test_multihop_receiver_on_failure(vo, did_factory, replica_client, root_acco
         replica_client.add_replicas(rse=src_rse, files=[{'scope': did['scope'].external, 'name': did['name'], 'bytes': 1, 'adler32': 'aaaaaaaa'}])
 
         rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
-        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
         request = __wait_for_request_state(dst_rse_id=jump_rse_id, state=RequestState.FAILED, run_poller=False, **did)
         assert request['state'] == RequestState.FAILED
@@ -534,8 +536,8 @@ def test_multihop_receiver_on_failure(vo, did_factory, replica_client, root_acco
         assert metrics_mock.get_sample_value('rucio_daemons_conveyor_receiver_update_request_state_total', labels={'updated': 'True'}) >= 1
 
         # Finisher will handle transfers of the same multihop one hop at a time
-        finisher(once=True, partition_wait_time=None)
-        finisher(once=True, partition_wait_time=None)
+        finisher(once=True, partition_wait_time=0)
+        finisher(once=True, partition_wait_time=0)
         # The intermediate request must not be re-scheduled by finisher
         with pytest.raises(RequestNotFound):
             request_core.get_request_by_did(rse_id=jump_rse_id, **did)
@@ -579,7 +581,7 @@ def test_multihop_receiver_on_success(vo, did_factory, root_account, core_config
         did = did_factory.upload_test_file(src_rse)
         rule_priority = 5
         rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=3600, locked=False, subscription_id=None, priority=rule_priority)
-        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
         request = __wait_for_request_state(dst_rse_id=jump_rse_id, state=RequestState.DONE, run_poller=False, **did)
         assert request['state'] == RequestState.DONE
@@ -635,10 +637,10 @@ def test_preparer_throttler_submitter(rse_factory, did_factory, root_account, fi
     assert request['state'] == RequestState.PREPARING
 
     # submitter must not work on PREPARING replicas
-    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
     # One RSE has limits set: the requests will be moved to WAITING status; the other RSE has no limits: go directly to queued
-    preparer(once=True, sleep_time=1, bulk=100, partition_wait_time=None)
+    preparer(once=True, sleep_time=1, bulk=100, partition_wait_time=0)
     request = request_core.get_request_by_did(rse_id=dst_rse_id1, **did1)
     assert request['state'] == RequestState.WAITING
     request = request_core.get_request_by_did(rse_id=dst_rse_id1, **did2)
@@ -647,10 +649,10 @@ def test_preparer_throttler_submitter(rse_factory, did_factory, root_account, fi
     assert request['state'] == RequestState.QUEUED
 
     # submitter must not work on WAITING replicas
-    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
     # One of the waiting requests will be queued, the second will remain in waiting state
-    throttler(once=True, partition_wait_time=None)
+    throttler(once=True, partition_wait_time=0)
     # Check metrics.
     # This gauge values are recorded at the beginning of the execution. Hence 2 waiting and 0 transfers
     gauge_name = 'rucio_daemons_conveyor_throttler_set_rse_transfer_limits'
@@ -665,10 +667,10 @@ def test_preparer_throttler_submitter(rse_factory, did_factory, root_account, fi
     waiting_did = did1 if request1['state'] == RequestState.WAITING else did2
     queued_did = did1 if request1['state'] == RequestState.QUEUED else did2
 
-    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
     # Calling the throttler again will not schedule the waiting request, because there is a submitted one
-    throttler(once=True, partition_wait_time=None)
+    throttler(once=True, partition_wait_time=0)
     # This gauge values are recorded at the beginning of the execution. Hence 1 waiting and one transfer
     assert metrics_mock.get_sample_value(gauge_name, labels={'activity': 'all_activities', 'rse': dst_rse1, 'limit_attr': 'max_transfers'}) == 1
     assert metrics_mock.get_sample_value(gauge_name, labels={'activity': 'all_activities', 'rse': dst_rse1, 'limit_attr': 'transfers'}) == 1
@@ -682,7 +684,7 @@ def test_preparer_throttler_submitter(rse_factory, did_factory, root_account, fi
     assert request['state'] == RequestState.DONE
 
     # Now that the submitted transfers are finished, the WAITING one can be queued
-    throttler(once=True, partition_wait_time=None)
+    throttler(once=True, partition_wait_time=0)
     request = request_core.get_request_by_did(rse_id=dst_rse_id1, **waiting_did)
     assert request['state'] == RequestState.QUEUED
 
@@ -712,7 +714,7 @@ def test_non_deterministic_dst(did_factory, did_client, root_account, vo, caches
     rse_core.update_rse(rse_id=dst_rse_id, parameters={'deterministic': False})
     try:
         rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
-        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
         replica = __wait_for_replica_transfer(dst_rse_id=dst_rse_id, **did)
         assert replica['state'] == ReplicaState.AVAILABLE
@@ -762,6 +764,294 @@ def test_stager(rse_factory, did_factory, root_account, replica_client):
     assert replica['state'] == ReplicaState.AVAILABLE
 
 
+@skip_rse_tests_with_accounts
+@pytest.mark.noparallel(reason="runs submitter; poller and finisher")
+def test_transfer_to_mas_new_replica(rse_factory, did_factory, root_account):
+    """
+    Test qos: transfer from disk to tape
+    Test a QoS transfer from some RSE in the world to MAS
+    Assert rse maximum_pin_lifetime is passed to transfer tool in the transfer request
+    Test rule and lock state transitions
+    """
+    src_rse, src_rse_id = rse_factory.make_rse(scheme='mock', protocol_impl='rucio.rse.protocols.posix.Default')
+    dst_rse, dst_rse_id = rse_factory.make_rse(scheme='mock', protocol_impl='rucio.rse.protocols.posix.Default', rse_type=RSEType.TAPE)
+    all_rses = [src_rse_id, dst_rse_id]
+
+    maximum_pin_lifetime = 86400
+
+    distance_core.add_distance(src_rse_id, dst_rse_id, ranking=10)
+    rse_core.add_rse_attribute(dst_rse_id, 'staging_required', True)
+    rse_core.add_rse_attribute(dst_rse_id, 'maximum_pin_lifetime', maximum_pin_lifetime)
+
+    did = did_factory.upload_test_file(src_rse)
+
+    rule_id = rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
+    request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+
+    assert request['request_type'] == RequestType.TRANSFER
+
+    submitter(once=True, rses=[{'id': dst_rse_id}], partition_wait_time=0, transfertool='mock', transfertype='single', filter_transfertool=None)
+
+    assert request['attributes']['lifetime'] == str(maximum_pin_lifetime)
+    assert rule_core.get_rule(rule_id)['state'] == RuleState.REPLICATING
+    for filtered_lock in [lock for lock in lock_core.get_replica_locks(scope=did['scope'], name=did['name'], restrict_rses=[dst_rse_id])]:
+        assert filtered_lock['state'] == LockState.REPLICATING
+
+    # mock a successful transfer
+    request = request_core.get_request(request_id=request['id'])
+    request_core.set_request_state(request_id=request['id'], state=RequestState.DONE, external_id=request['external_id'])
+    finisher(once=True, partition_wait_time=0)
+
+    assert rule_core.get_rule(rule_id)['state'] == RuleState.OK
+    for filtered_lock in [lock for lock in lock_core.get_replica_locks(scope=did['scope'], name=did['name'], restrict_rses=[dst_rse_id])]:
+        assert filtered_lock['state'] == LockState.OK
+    for rse_id in all_rses:
+        assert replica_core.get_replica(rse_id=rse_id, **did)['state'] == ReplicaState.AVAILABLE
+
+
+@skip_rse_tests_with_accounts
+@pytest.mark.noparallel(reason="runs submitter; poller and finisher")
+def test_failed_transfer_to_mas_new_replica(rse_factory, did_factory, root_account):
+    """
+    Test a qos failed transfer from disk to tape
+    Assert rse maximum_pin_lifetime is passed to transfer tool in the transfer request
+    Test rule and lock state transitions
+    """
+    src_rse, src_rse_id = rse_factory.make_rse(scheme='mock', protocol_impl='rucio.rse.protocols.posix.Default')
+    dst_rse, dst_rse_id = rse_factory.make_rse(scheme='mock', protocol_impl='rucio.rse.protocols.posix.Default', rse_type=RSEType.TAPE)
+
+    maximum_pin_lifetime = 86400
+
+    distance_core.add_distance(src_rse_id, dst_rse_id, ranking=10)
+    rse_core.add_rse_attribute(dst_rse_id, 'staging_required', True)
+    rse_core.add_rse_attribute(dst_rse_id, 'maximum_pin_lifetime', maximum_pin_lifetime)
+
+    did = did_factory.upload_test_file(src_rse)
+
+    rule_id = rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
+    request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+
+    assert request['request_type'] == RequestType.TRANSFER
+
+    submitter(once=True, rses=[{'id': dst_rse_id}], partition_wait_time=0, transfertool='mock', transfertype='single', filter_transfertool=None)
+
+    assert request['attributes']['lifetime'] == str(maximum_pin_lifetime)
+    assert rule_core.get_rule(rule_id)['state'] == RuleState.REPLICATING
+    for filtered_lock in [lock for lock in lock_core.get_replica_locks(scope=did['scope'], name=did['name'], restrict_rses=[dst_rse_id])]:
+        assert filtered_lock['state'] == LockState.REPLICATING
+
+    # mock a failed transfer
+    # re-query request to get external_id
+    request = request_core.get_request(request_id=request['id'])
+    request_core.set_request_state(request_id=request['id'], state=RequestState.FAILED, external_id=request['external_id'])
+    lock_core.failed_transfer(scope=did['scope'], name=did['name'], rse_id=dst_rse_id)
+    replica = {'rse_id': dst_rse_id, 'scope': did['scope'], 'name': did['name'], 'state': ReplicaState.UNAVAILABLE}
+    replica_core.update_replicas_states([replica])
+    finisher(once=True, partition_wait_time=0)
+    assert rule_core.get_rule(rule_id)['state'] == RuleState.STUCK
+    assert lock_core.get_replica_locks_for_rule_id(rule_id=rule_id)[0]['state'] == LockState.STUCK
+    assert replica_core.get_replica(rse_id=dst_rse_id, **did)['state'] == ReplicaState.UNAVAILABLE
+
+
+@skip_rse_tests_with_accounts
+@pytest.mark.noparallel(reason="runs submitter; poller and finisher")
+def test_transfer_to_mas_existing_replica(rse_factory, did_factory, root_account):
+    """
+    Test qos: transfer from tape to disk
+    Assert rse maximum_pin_lifetime is passed to transfer tool in the transfer request
+    Test rule and lock state transitions
+    """
+    src_rse, src_rse_id = rse_factory.make_rse(scheme='mock', protocol_impl='rucio.rse.protocols.posix.Default')
+    dst_rse, dst_rse_id = rse_factory.make_rse(scheme='mock', protocol_impl='rucio.rse.protocols.posix.Default', rse_type=RSEType.TAPE)
+    all_rses = [src_rse_id, dst_rse_id]
+
+    maximum_pin_lifetime = 86400
+
+    distance_core.add_distance(src_rse_id, dst_rse_id, ranking=10)
+    rse_core.add_rse_attribute(dst_rse_id, 'staging_required', True)
+    rse_core.add_rse_attribute(dst_rse_id, 'maximum_pin_lifetime', maximum_pin_lifetime)
+
+    did = did_factory.upload_test_file(rse_name=src_rse)
+
+    rule1_id = rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=-1, locked=False, subscription_id=None)[0]
+    request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+
+    assert request['request_type'] == RequestType.TRANSFER
+
+    submitter(once=True, rses=[{'id': dst_rse_id}], partition_wait_time=0, transfertool='mock', transfertype='single', filter_transfertool=None)
+
+    assert request['attributes']['lifetime'] == str(maximum_pin_lifetime)
+    assert lock_core.get_replica_locks_for_rule_id(rule_id=rule1_id)[0]['state'] == LockState.REPLICATING
+    assert rule_core.get_rule(rule1_id)['state'] == RuleState.REPLICATING
+
+    # mock a successful transfer
+    request = request_core.get_request(request_id=request['id'])
+    request_core.set_request_state(request_id=request['id'], state=RequestState.DONE, external_id=request['external_id'])
+    finisher(once=True, partition_wait_time=0)
+
+    assert rule_core.get_rule(rule1_id)['state'] == RuleState.OK
+
+    # assert all replicas available
+    for rse_id in all_rses:
+        assert replica_core.get_replica(rse_id=rse_id, **did)['state'] == ReplicaState.AVAILABLE
+
+    # now test a second rule, different account
+    set_local_account_limit(InternalAccount('jdoe'), dst_rse_id, bytes_=-1)
+    rule2_id = rule_core.add_rule(dids=[did], account=InternalAccount('jdoe'), copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=-1, locked=False, subscription_id=None, source_replica_expression=dst_rse)[0]
+    request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+
+    assert request['request_type'] == RequestType.STAGEIN
+
+    stager(once=True, rses=[{'id': dst_rse_id}])
+
+    assert request['attributes']['lifetime'] == str(maximum_pin_lifetime)
+    assert lock_core.get_replica_locks_for_rule_id(rule_id=rule2_id)[0]['state'] == LockState.REPLICATING
+    assert rule_core.get_rule(rule2_id)['state'] == RuleState.REPLICATING
+
+    # mock a successful stagein
+    request = request_core.get_request(request_id=request['id'])
+    request_core.set_request_state(request_id=request['id'], state=RequestState.DONE, external_id=request['external_id'])
+    finisher(once=True, partition_wait_time=0)
+
+    assert lock_core.get_replica_locks_for_rule_id(rule_id=rule1_id)[0]['state'] == LockState.OK
+    assert rule_core.get_rule(rule1_id)['state'] == RuleState.OK
+    assert lock_core.get_replica_locks_for_rule_id(rule_id=rule2_id)[0]['state'] == LockState.OK
+    assert rule_core.get_rule(rule2_id)['state'] == RuleState.OK
+
+
+@skip_rse_tests_with_accounts
+@pytest.mark.noparallel(reason="runs submitter; poller and finisher")
+def test_failed_transfers_to_mas_existing_replica(rse_factory, did_factory, root_account):
+    """
+    Test qos: transfer from tape to disk
+    Assert rse maximum_pin_lifetime is passed to transfer tool in the transfer request
+    Test rule and lock state transitions
+    """
+    src_rse, src_rse_id = rse_factory.make_rse(scheme='mock', protocol_impl='rucio.rse.protocols.posix.Default')
+    dst_rse, dst_rse_id = rse_factory.make_rse(scheme='mock', protocol_impl='rucio.rse.protocols.posix.Default', rse_type=RSEType.TAPE)
+
+    maximum_pin_lifetime = 86400
+
+    distance_core.add_distance(src_rse_id, dst_rse_id, ranking=10)
+    rse_core.add_rse_attribute(dst_rse_id, 'staging_required', True)
+    rse_core.add_rse_attribute(dst_rse_id, 'maximum_pin_lifetime', maximum_pin_lifetime)
+
+    did = did_factory.upload_test_file(rse_name=src_rse)
+
+    rule1_id = rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=-1, locked=False, subscription_id=None)[0]
+    request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+
+    assert request['request_type'] == RequestType.TRANSFER
+
+    submitter(once=True, rses=[{'id': dst_rse_id}], partition_wait_time=0, transfertool='mock', transfertype='single', filter_transfertool=None)
+
+    assert request['attributes']['lifetime'] == str(maximum_pin_lifetime)
+    assert lock_core.get_replica_locks_for_rule_id(rule_id=rule1_id)[0]['state'] == LockState.REPLICATING
+    assert rule_core.get_rule(rule1_id)['state'] == RuleState.REPLICATING
+
+    # mock a failed transfer
+    request = request_core.get_request(request_id=request['id'])
+    request_core.set_request_state(request_id=request['id'], state=RequestState.FAILED, external_id=request['external_id'])
+    finisher(once=True, partition_wait_time=0)
+    lock_core.failed_transfer(scope=did['scope'], name=did['name'], rse_id=dst_rse_id)
+
+    assert rule_core.get_rule(rule1_id)['state'] == RuleState.STUCK
+    assert lock_core.get_replica_locks_for_rule_id(rule_id=rule1_id)[0]['state'] == LockState.STUCK
+
+    # now test a second rule, different account
+    set_local_account_limit(InternalAccount('jdoe'), dst_rse_id, bytes_=-1)
+    rule2_id = rule_core.add_rule(dids=[did], account=InternalAccount('jdoe'), copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=-1, locked=False, subscription_id=None, source_replica_expression=dst_rse)[0]
+    request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+
+    # since the first rule is STUCK assert a transfer not stagein
+    assert request['request_type'] == RequestType.TRANSFER
+
+    submitter(once=True, rses=[{'id': dst_rse_id}], partition_wait_time=0, transfertool='mock', transfertype='single', filter_transfertool=None)
+
+    assert request['attributes']['lifetime'] == str(maximum_pin_lifetime)
+    assert lock_core.get_replica_locks_for_rule_id(rule_id=rule2_id)[0]['state'] == LockState.REPLICATING
+    assert rule_core.get_rule(rule2_id)['state'] == RuleState.REPLICATING
+
+    # mock a failed transfer for the second rule
+    request = request_core.get_request(request_id=request['id'])
+    request_core.set_request_state(request_id=request['id'], state=RequestState.FAILED, external_id=request['external_id'])
+    finisher(once=True, partition_wait_time=0)
+    lock_core.failed_transfer(scope=did['scope'], name=did['name'], rse_id=dst_rse_id)
+
+    assert lock_core.get_replica_locks_for_rule_id(rule_id=rule1_id)[0]['state'] == LockState.STUCK
+    assert rule_core.get_rule(rule1_id)['state'] == RuleState.STUCK
+    assert lock_core.get_replica_locks_for_rule_id(rule_id=rule2_id)[0]['state'] == LockState.STUCK
+    assert rule_core.get_rule(rule2_id)['state'] == RuleState.STUCK
+
+
+@skip_rse_tests_with_accounts
+@pytest.mark.noparallel(reason="runs submitter; poller and finisher")
+def test_transfer_failed_stagein_to_mas_existing_replica(rse_factory, did_factory, root_account):
+    """
+    Test qos: transfer from tape to disk
+    Assert rse maximum_pin_lifetime is passed to transfer tool in the transfer request
+    Test rule and lock state transitions
+    """
+    src_rse, src_rse_id = rse_factory.make_rse(scheme='mock', protocol_impl='rucio.rse.protocols.posix.Default')
+    dst_rse, dst_rse_id = rse_factory.make_rse(scheme='mock', protocol_impl='rucio.rse.protocols.posix.Default', rse_type=RSEType.TAPE)
+    all_rses = [src_rse_id, dst_rse_id]
+
+    maximum_pin_lifetime = 86400
+
+    distance_core.add_distance(src_rse_id, dst_rse_id, ranking=10)
+    rse_core.add_rse_attribute(dst_rse_id, 'staging_required', True)
+    rse_core.add_rse_attribute(dst_rse_id, 'maximum_pin_lifetime', maximum_pin_lifetime)
+
+    did = did_factory.upload_test_file(rse_name=src_rse)
+
+    rule1_id = rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=-1, locked=False, subscription_id=None)[0]
+    request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+
+    assert request['request_type'] == RequestType.TRANSFER
+
+    submitter(once=True, rses=[{'id': dst_rse_id}], partition_wait_time=0, transfertool='mock', transfertype='single', filter_transfertool=None)
+
+    assert request['attributes']['lifetime'] == str(maximum_pin_lifetime)
+    assert lock_core.get_replica_locks_for_rule_id(rule_id=rule1_id)[0]['state'] == LockState.REPLICATING
+    assert rule_core.get_rule(rule1_id)['state'] == RuleState.REPLICATING
+
+    # mock a successful transfer
+    request = request_core.get_request(request_id=request['id'])
+    request_core.set_request_state(request_id=request['id'], state=RequestState.DONE, external_id=request['external_id'])
+    finisher(once=True, partition_wait_time=0)
+
+    assert rule_core.get_rule(rule1_id)['state'] == RuleState.OK
+
+    # assert all replicas available
+    for rse_id in all_rses:
+        assert replica_core.get_replica(rse_id=rse_id, **did)['state'] == ReplicaState.AVAILABLE
+
+    # now test a second rule, different account
+    set_local_account_limit(InternalAccount('jdoe'), dst_rse_id, bytes_=-1)
+    rule2_id = rule_core.add_rule(dids=[did], account=InternalAccount('jdoe'), copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=-1, locked=False, subscription_id=None, source_replica_expression=dst_rse)[0]
+    request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+
+    assert request['request_type'] == RequestType.STAGEIN
+
+    submitter(once=True, rses=[{'id': dst_rse_id}], partition_wait_time=0, transfertool='mock', transfertype='single', filter_transfertool=None)
+
+    assert request['attributes']['lifetime'] == str(maximum_pin_lifetime)
+    assert lock_core.get_replica_locks_for_rule_id(rule_id=rule2_id)[0]['state'] == LockState.REPLICATING
+    assert rule_core.get_rule(rule2_id)['state'] == RuleState.REPLICATING
+
+    # mock a failed stagein
+    request = request_core.get_request(request_id=request['id'])
+    request_core.set_request_state(request_id=request['id'], state=RequestState.FAILED, external_id=request['external_id'])
+    finisher(once=True, partition_wait_time=0)
+    lock_core.failed_transfer(scope=did['scope'], name=did['name'], rse_id=dst_rse_id)
+
+    assert lock_core.get_replica_locks_for_rule_id(rule_id=rule1_id)[0]['state'] == LockState.OK
+    assert rule_core.get_rule(rule1_id)['state'] == RuleState.OK
+
+    assert lock_core.get_replica_locks_for_rule_id(rule_id=rule2_id)[0]['state'] == LockState.STUCK
+    assert rule_core.get_rule(rule2_id)['state'] == RuleState.STUCK
+
+
 @skip_rse_tests_with_accounts
 @pytest.mark.noparallel(reason="runs submitter; poller and finisher")
 def test_lost_transfers(rse_factory, did_factory, root_account):
@@ -785,7 +1075,7 @@ def __update_request(request_id, session=None, **kwargs):
         session.query(models.Request).filter_by(id=request_id).update(kwargs, synchronize_session=False)
 
     # Fake that the transfer is submitted and lost
-    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
     request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
     __update_request(request['id'], external_id='some-fake-random-id')
 
@@ -796,11 +1086,11 @@ def __update_request(request_id, session=None, **kwargs):
     # Set update time far in the past to bypass protections (not resubmitting too fast).
     # Run finisher and submitter, the request must be resubmitted and transferred correctly
     __update_request(request['id'], updated_at=datetime.utcnow() - timedelta(days=1))
-    finisher(once=True, partition_wait_time=None)
+    finisher(once=True, partition_wait_time=0)
     # The source ranking must not be updated for submission failures and lost transfers
     request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
     assert __get_source(request_id=request['id'], src_rse_id=src_rse_id, **did).ranking == 0
-    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
     replica = __wait_for_replica_transfer(dst_rse_id=dst_rse_id, **did)
     assert replica['state'] == ReplicaState.AVAILABLE
 
@@ -831,7 +1121,7 @@ def on_submit(file):
 
     with patch('rucio.daemons.conveyor.submitter.TRANSFERTOOL_CLASSES_BY_NAME') as tt_mock:
         tt_mock.__getitem__.return_value = _FTSWrapper
-        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
     request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
 
     rule_core.delete_rule(rule_id)
@@ -951,7 +1241,7 @@ def test_overwrite_on_tape(overwrite_on_tape_topology, core_config_mock, caches_
     rse1_id, rse2_id, rse3_id, did1, did2 = overwrite_on_tape_topology(did1_corrupted=False, did2_corrupted=True)
     all_rses = [rse1_id, rse2_id, rse3_id]
 
-    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
     request = __wait_for_request_state(dst_rse_id=rse3_id, state=RequestState.FAILED, **did1)
     assert request['state'] == RequestState.FAILED
@@ -991,7 +1281,7 @@ def test_overwrite_hops(overwrite_on_tape_topology, core_config_mock, caches_moc
     )
     all_rses = [rse1_id, rse2_id, rse3_id]
 
-    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
     fts_schema_version = FTS3Transfertool(external_host=TEST_FTS_HOST).version()['schema']['major']
     if fts_schema_version >= 8:
@@ -1046,7 +1336,7 @@ def on_receive(job_params):
             return job_params
 
     with patch('rucio.daemons.conveyor.poller.FTS3Transfertool', _FTSWrapper):
-        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
         request = __wait_for_request_state(dst_rse_id=rse3_id, state=RequestState.DONE, **did1)
         assert request['state'] == RequestState.DONE
@@ -1088,7 +1378,7 @@ def on_receive(job_params):
             return job_params
 
     with patch('rucio.daemons.conveyor.poller.FTS3Transfertool', _FTSWrapper):
-        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=0, transfertype='single', filter_transfertool=None)
         # Both transfers must be marked as failed because the file size is incorrect
         request = __wait_for_request_state(dst_rse_id=rse3_id, state=RequestState.FAILED, **did1)
         assert request['state'] == RequestState.FAILED
@@ -1097,13 +1387,13 @@ def on_receive(job_params):
 
         # Re-submit the failed requests. They must fail again, because overwrite_corrupted_files is False
         # 2 runs: for multihop, finisher works one hop at a time
-        finisher(once=True, partition_wait_time=None)
-        finisher(once=True, partition_wait_time=None)
+        finisher(once=True, partition_wait_time=0)
+        finisher(once=True, partition_wait_time=0)
         request = request_core.get_request_by_did(rse_id=rse3_id, **did1)
         assert request['state'] == RequestState.QUEUED
         request = request_core.get_request_by_did(rse_id=rse3_id, **did2)
         assert request['state'] == RequestState.QUEUED
-        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=0, transfertype='single', filter_transfertool=None)
         # Set overwrite to True before running the poller or finisher
         core_config.set('transfers', 'overwrite_corrupted_files', True)
         request = __wait_for_request_state(dst_rse_id=rse3_id, state=RequestState.FAILED, **did1)
@@ -1112,13 +1402,13 @@ def on_receive(job_params):
         assert request['state'] == RequestState.FAILED
 
         # Re-submit one more time. Now the destination file must be overwritten
-        finisher(once=True, partition_wait_time=None)
-        finisher(once=True, partition_wait_time=None)
+        finisher(once=True, partition_wait_time=0)
+        finisher(once=True, partition_wait_time=0)
         request = request_core.get_request_by_did(rse_id=rse3_id, **did1)
         assert request['state'] == RequestState.QUEUED
         request = request_core.get_request_by_did(rse_id=rse3_id, **did2)
         assert request['state'] == RequestState.QUEUED
-        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=0, transfertype='single', filter_transfertool=None)
         request = request_core.get_request_by_did(rse_id=rse3_id, **did1)
         assert request['state'] == RequestState.SUBMITTED
         assert __wait_for_fts_state(request, expected_state='ARCHIVING') == 'ARCHIVING'
@@ -1175,11 +1465,11 @@ def bulk_query(self, requests_by_eid, timeout=None):
 
     with patch('rucio.daemons.conveyor.submitter.TRANSFERTOOL_CLASSES_BY_NAME') as tt_mock:
         tt_mock.__getitem__.return_value = _FTSWrapper
-        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
         assert sorted(certs_used_by_submitter) == ['DEFAULT_DUMMY_CERT', 'NEW_VO_DUMMY_CERT']
 
     with patch('rucio.daemons.conveyor.poller.FTS3Transfertool', _FTSWrapper):
-        poller(once=True, older_than=0, partition_wait_time=None)
+        poller(once=True, older_than=0, partition_wait_time=0)
         assert sorted(certs_used_by_poller) == ['DEFAULT_DUMMY_CERT', 'NEW_VO_DUMMY_CERT']
 
 
@@ -1244,7 +1534,7 @@ def on_submit(file):
     # Submit the first time, but force a failure to verify that retries are correctly handled
     with patch('rucio.daemons.conveyor.submitter.TRANSFERTOOL_CLASSES_BY_NAME') as tt_mock:
         tt_mock.__getitem__.return_value = _FTSWrapper
-        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
     request = __wait_for_request_state(dst_rse_id=rse2_id, state=RequestState.FAILED, **did)
     assert request['state'] == RequestState.FAILED
@@ -1252,8 +1542,8 @@ def on_submit(file):
     # Re-submit the transfer without simulating a failure. Everything should go as normal starting now.
     for _ in range(4):
         # for multihop, finisher works one hop at a time. 4 is the maximum number of hops in this test graph
-        finisher(once=True, partition_wait_time=None)
-    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+        finisher(once=True, partition_wait_time=0)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=0, transfertype='single', filter_transfertool=None)
     # one request must be submitted, but the second will only be queued
     if request_core.get_request_by_did(rse_id=rse5_id, **did)['state'] == RequestState.QUEUED:
         rse_id_second_to_last_queued, rse_id_queued = rse4_id, rse5_id
@@ -1267,7 +1557,7 @@ def on_submit(file):
     assert request['state'] == RequestState.SUBMITTED
 
     # Calling submitter again will not unblock the queued requests
-    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=0, transfertype='single', filter_transfertool=None)
     replica = __wait_for_replica_transfer(dst_rse_id=rse_id_submitted, **did)
     assert replica['state'] == ReplicaState.AVAILABLE
     request = request_core.get_request_by_did(rse_id=rse_id_queued, **did)
@@ -1275,7 +1565,7 @@ def on_submit(file):
 
     # Once the submitted transfer is done, the submission will continue for second request (one hop at a time)
     # First of the remaining two hops submitted
-    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=0, transfertype='single', filter_transfertool=None)
     replica = __wait_for_replica_transfer(dst_rse_id=rse_id_second_to_last_queued, **did)
     assert replica['state'] == ReplicaState.AVAILABLE
 
@@ -1288,7 +1578,7 @@ def on_submit(file):
         replica_core.get_replica(rse_id=rse_id, **did)
 
     # Final hop
-    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=10, partition_wait_time=0, transfertype='single', filter_transfertool=None)
     replica = __wait_for_replica_transfer(dst_rse_id=rse_id_queued, **did)
     assert replica['state'] == ReplicaState.AVAILABLE
 
@@ -1336,7 +1626,7 @@ def on_submit(file):
 
     with patch('rucio.daemons.conveyor.submitter.TRANSFERTOOL_CLASSES_BY_NAME') as tt_mock:
         tt_mock.__getitem__.return_value = _FTSWrapper
-        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
     # Checksum verification disabled on this rse, so the transfer must use source validation and succeed
     request = __wait_for_request_state(dst_rse_id=dst_rse1_id, state=RequestState.DONE, **did)