Repository: rucio/rucio

Issue:
protection of last available replica with pendng requests doesn't work
The following test can make you think that things work as expected: 
https://github.com/rucio/rucio/blob/587fc9b823efa37609370ff6f4ee480d1b3a6d2e/lib/rucio/tests/test_conveyor_submitter.py#L235

However, it just works "by luck" in some cases when temporary tables are not used because of this comparison being wrong:
https://github.com/rucio/rucio/blob/068925387aac4a06d482f685c7297b151daae2e7/lib/rucio/core/replica.py#L2841
the query used to feed data to this comparison doesn't filter on 'Available' replicas. As a result, it will return all 'Copying', or 'Bad' replicas ... etc. It's easy to find a case when that will not turn out as desired and the last available replica will be removed even if a pending request exists. 

A very similar problem exists in the temporary table workflow: 
https://github.com/rucio/rucio/blob/068925387aac4a06d482f685c7297b151daae2e7/lib/rucio/core/replica.py#L2750, but is even more pronounced, because the comparison is `< 0` instead of `< 1`, so the previously mentioned test fails. That's how I discovered the issue: by running that test with temporary tables workflow. 


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -2731,7 +2731,8 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
             replicas_alias,
             and_(models.RSEFileAssociation.scope == replicas_alias.scope,
                  models.RSEFileAssociation.name == replicas_alias.name,
-                 models.RSEFileAssociation.rse_id != replicas_alias.rse_id)
+                 models.RSEFileAssociation.rse_id != replicas_alias.rse_id,
+                 replicas_alias.state == ReplicaState.AVAILABLE)
         ).with_hint(
             models.Request, "INDEX(requests REQUESTS_SCOPE_NAME_RSE_IDX)", 'oracle'
         ).outerjoin(
@@ -2836,9 +2837,12 @@ def list_and_mark_unlocked_replicas_no_temp_table(limit, bytes_=None, rse_id=Non
         # Check if more than one replica is available
         replica_cnt = session.query(func.count(models.RSEFileAssociation.scope)).\
             with_hint(models.RSEFileAssociation, "index(REPLICAS REPLICAS_PK)", 'oracle').\
-            filter(and_(models.RSEFileAssociation.scope == scope, models.RSEFileAssociation.name == name, models.RSEFileAssociation.rse_id != rse_id)).one()
+            filter(and_(models.RSEFileAssociation.scope == scope,
+                        models.RSEFileAssociation.name == name,
+                        models.RSEFileAssociation.rse_id != rse_id,
+                        models.RSEFileAssociation.state == ReplicaState.AVAILABLE)).one()
 
-        if replica_cnt[0] > 1:
+        if replica_cnt[0] > 0:
             if tombstone != OBSOLETE and only_delete_obsolete:
                 break
 


Test patch:
diff --git a/lib/rucio/tests/test_conveyor_submitter.py b/lib/rucio/tests/test_conveyor_submitter.py
--- a/lib/rucio/tests/test_conveyor_submitter.py
+++ b/lib/rucio/tests/test_conveyor_submitter.py
@@ -209,7 +209,12 @@ def __number_sources(rse_id, scope, name, *, session=None):
     'rucio.daemons.reaper.reaper.REGION',
     'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by an expression
 ]}], indirect=True)
-def test_source_avoid_deletion(caches_mock, core_config_mock, rse_factory, did_factory, root_account):
+@pytest.mark.parametrize("file_config_mock", [
+    # Run test twice: with, and without, temp tables
+    {"overrides": [('core', 'use_temp_tables', 'True')]},
+    {"overrides": [('core', 'use_temp_tables', 'False')]},
+], indirect=True)
+def test_source_avoid_deletion(caches_mock, core_config_mock, rse_factory, did_factory, root_account, file_config_mock):
     """ Test that sources on a file block it from deletion """
 
     _, reaper_region, _ = caches_mock