Repository: rucio/rucio

Issue:
Throttled requests don't react to source changes
The preparer picks a source out of available candidates and then applies throttling rules (which also include the rules configured from the source rse). However, it can happen that the situation changes over time. For example, a replica becomes available on a better source while the request is blocked in a Waiting state within the throttler. It's not unheard of requests staying in waiting state for multiple days or even weeks. 

To handle this case and trigger source re-evaluation. I propose to transition the requests back to the Preparing state if they spent too much time in a waiting state. 


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/request.py b/lib/rucio/core/request.py
--- a/lib/rucio/core/request.py
+++ b/lib/rucio/core/request.py
@@ -2817,3 +2817,39 @@ def list_requests_history(src_rse_ids, dst_rse_ids, states=None, offset=None, li
         stmt = stmt.limit(limit)
     for request in session.execute(stmt).yield_per(500).scalars():
         yield request
+
+
+@transactional_session
+def reset_stale_waiting_requests(time_limit: Optional[datetime.timedelta] = datetime.timedelta(days=1), *, session: "Session") -> None:
+    """
+    Clear source_rse_id for requests that have been in the waiting state for > time_limit amount of time and
+    transition back to preparing state (default time limit = 1 day).
+    This allows for stale requests that have been in the waiting state for a long time to be able to
+    react to source changes that have occurred in the meantime.
+    :param time_limit: The amount of time a request must be in the waiting state to be reset.
+    :param session: The database session in use.
+    """
+    try:
+        # Cutoff timestamp based on time limit
+        time_limit_timestamp = datetime.datetime.utcnow() - time_limit
+
+        # Select all waiting requests that precede the time limit, then clear source_rse_id and reset state to preparing
+        stmt = update(
+            models.Request
+        ).where(
+            and_(
+                models.Request.state == RequestState.WAITING,
+                models.Request.last_processed_at < time_limit_timestamp
+            )
+        ).execution_options(
+            synchronize_session=False
+        ).values(
+            {
+                models.Request.source_rse_id: None,
+                models.Request.state: RequestState.PREPARING
+            }
+        )
+        session.execute(stmt)
+
+    except IntegrityError as error:
+        raise RucioException(error.args)
diff --git a/lib/rucio/daemons/conveyor/throttler.py b/lib/rucio/daemons/conveyor/throttler.py
--- a/lib/rucio/daemons/conveyor/throttler.py
+++ b/lib/rucio/daemons/conveyor/throttler.py
@@ -31,7 +31,8 @@
 from rucio.common.logging import setup_logging
 from rucio.core.monitor import MetricManager
 from rucio.core.request import (get_request_stats, release_all_waiting_requests, release_waiting_requests_fifo,
-                                release_waiting_requests_grouped_fifo, set_transfer_limit_stats, re_sync_all_transfer_limits)
+                                release_waiting_requests_grouped_fifo, set_transfer_limit_stats, re_sync_all_transfer_limits,
+                                reset_stale_waiting_requests)
 from rucio.core.rse import RseCollection
 from rucio.core.transfer import applicable_rse_transfer_limits
 from rucio.daemons.common import db_workqueue, ProducerConsumerDaemon
@@ -82,6 +83,7 @@ def _consumer(release_groups):
             _handle_requests(release_groups, logger=logger)
         except Exception:
             logger(logging.CRITICAL, "Failed to schedule requests, error: %s" % (traceback.format_exc()))
+        reset_stale_waiting_requests()
 
     ProducerConsumerDaemon(
         producers=[_db_producer],


Test patch:
diff --git a/tests/test_conveyor.py b/tests/test_conveyor.py
--- a/tests/test_conveyor.py
+++ b/tests/test_conveyor.py
@@ -18,6 +18,7 @@
 from datetime import datetime, timedelta
 from unittest.mock import patch
 from urllib.parse import urlencode, urlparse, parse_qsl, urlunparse
+from sqlalchemy import update
 
 import pytest
 
@@ -743,6 +744,50 @@ def test_preparer_throttler_submitter(rse_factory, did_factory, root_account, fi
     request = request_core.get_request_by_did(rse_id=dst_rse_id1, **waiting_did)
     assert request['state'] == RequestState.QUEUED
 
+    # Check that resetting stale waiting requests works properly
+    request_core.set_transfer_limit(dst_rse2, max_transfers=1, activity='all_activities', strategy='fifo')
+    did3 = did_factory.upload_test_file(src_rse)
+    did4 = did_factory.upload_test_file(src_rse)
+    rule_core.add_rule(dids=[did3], account=root_account, copies=1, rse_expression=dst_rse2, grouping='ALL',
+                       weight=None, lifetime=3600, locked=False, subscription_id=None)
+    rule_core.add_rule(dids=[did4], account=root_account, copies=1, rse_expression=dst_rse2, grouping='ALL',
+                       weight=None, lifetime=3600, locked=False, subscription_id=None)
+    request3 = request_core.get_request_by_did(rse_id=dst_rse_id2, **did3)
+    request4 = request_core.get_request_by_did(rse_id=dst_rse_id2, **did4)
+    assert request3['state'] == RequestState.PREPARING
+    assert request4['state'] == RequestState.PREPARING
+
+    # Run the preparer so requests are in waiting state
+    preparer(once=True, sleep_time=1, bulk=100, partition_wait_time=0, ignore_availability=False)
+    request3 = request_core.get_request_by_did(rse_id=dst_rse_id2, **did3)
+    request4 = request_core.get_request_by_did(rse_id=dst_rse_id2, **did4)
+    assert request3['state'] == RequestState.WAITING
+    assert request4['state'] == RequestState.WAITING
+
+    # Artificially set both requests' last_processed_at timestamp as >1 day old
+    @transactional_session
+    def __set_process_timestamp(request_ids, timestamp, *, session=None):
+        stmt = update(
+            models.Request
+        ).where(
+            models.Request.id.in_(request_ids)
+        ).values(
+            {
+                models.Request.last_processed_at: timestamp
+            }
+        )
+        session.execute(stmt)
+
+    __set_process_timestamp([request['id'] for request in [request3, request4]], datetime.utcnow() - timedelta(days=2))
+
+    # Run throttler: one request reset to PREPARING state and Null source_rse_id, and one request QUEUED
+    throttler(once=True, partition_wait_time=0)
+    request3 = request_core.get_request_by_did(rse_id=dst_rse_id2, **did3)
+    request4 = request_core.get_request_by_did(rse_id=dst_rse_id2, **did4)
+
+    assert ((request3['source_rse_id'] is None and request3['state'] == RequestState.PREPARING and request4['state'] == RequestState.QUEUED)
+            or (request4['source_rse_id'] is None and request4['state'] == RequestState.PREPARING and request3['state'] == RequestState.QUEUED))
+
 
 @skip_rse_tests_with_accounts
 @pytest.mark.dirty(reason="leaves files in XRD containers")