Repository: rucio/rucio

Issue:
Support `ignore_availability` in preparer
Motivation
----------
There are two possible behaviors: 
- either requests are ignored when fetched from the database: https://github.com/rucio/rucio/blob/9a5d044d9f5760a224dba9feb14b71bfdf213252/lib/rucio/core/request.py#L420 
- or `ignore_availability` is set and they are marked NO_SOURCES: https://github.com/rucio/rucio/blob/9a5d044d9f5760a224dba9feb14b71bfdf213252/lib/rucio/core/transfer.py#L863 

Submitter has the ability to switch between these two modes using a CLI switch. Preparer is hard-coded into `ignore_availability = False`

Modification
------------
Implement this switch from the submitter:

https://github.com/rucio/rucio/blob/ac95ef3fabb8e1fcabddf1075cc19ae06db44a34/bin/rucio-conveyor-submitter#L108

in the preparer: 
https://github.com/rucio/rucio/blob/2a41a64899f41b356660fff80a7530765b5d59e0/bin/rucio-conveyor-preparer#L35

And pass it through the call stack to this function:
https://github.com/rucio/rucio/blob/e3c77683184063424fdcf9ecfda7d37cb4641c4f/lib/rucio/daemons/conveyor/preparer.py#L128

Add a test for the two behaviors. A example test (which will have to be simplified and adapted for this issue):
https://github.com/rucio/rucio/blob/2d81b39b8560f376f412fcba62a622a6db748780/lib/rucio/tests/test_conveyor.py#L611




Discussion:
Hii @rcarpa  I am Nihar, we just discussed on Mattermost. Can you assign this issue to me?

Resolving patch:
diff --git a/lib/rucio/daemons/conveyor/preparer.py b/lib/rucio/daemons/conveyor/preparer.py
--- a/lib/rucio/daemons/conveyor/preparer.py
+++ b/lib/rucio/daemons/conveyor/preparer.py
@@ -47,7 +47,7 @@ def stop():
     graceful_stop.set()
 
 
-def run(once=False, threads=1, sleep_time=10, bulk=100):
+def run(once=False, threads=1, sleep_time=10, bulk=100, ignore_availability: bool = False):
     """
     Running the preparer daemon either once or by default in a loop until stop is called.
     """
@@ -58,7 +58,7 @@ def run(once=False, threads=1, sleep_time=10, bulk=100):
 
     def preparer_kwargs():
         # not sure if this is needed for threading.Thread, but it always returns a fresh dictionary
-        return {'once': once, 'sleep_time': sleep_time, 'bulk': bulk}
+        return {'once': once, 'sleep_time': sleep_time, 'bulk': bulk, 'ignore_availability': ignore_availability}
 
     threads = [threading.Thread(target=preparer, name=f'conveyor-preparer-{i}', kwargs=preparer_kwargs(), daemon=True) for i in range(threads)]
     for thr in threads:
@@ -84,7 +84,7 @@ def preparer_kwargs():
     logging.info('conveyor-preparer: stopped')
 
 
-def preparer(once, sleep_time, bulk, partition_wait_time=10):
+def preparer(once, sleep_time, bulk, ignore_availability: bool, partition_wait_time: int = 10):
     # Make an initial heartbeat so that all instanced daemons have the correct worker number on the next try
     logger_prefix = executable = 'conveyor-preparer'
     transfertools = config_get_list('conveyor', 'transfertool', False, None)
@@ -100,6 +100,7 @@ def preparer(once, sleep_time, bulk, partition_wait_time=10):
             run_once,
             transfertools=transfertools,
             bulk=bulk,
+            ignore_availability=ignore_availability,
         ),
         activities=None,
     )
@@ -110,6 +111,7 @@ def run_once(
         bulk: int = 100,
         heartbeat_handler: "Optional[HeartbeatHandler]" = None,
         session: "Optional[Session]" = None,
+        ignore_availability: bool = False,
         **kwargs
 ) -> bool:
     if heartbeat_handler:
@@ -124,14 +126,15 @@ def run_once(
     requests_handled = 0
     try:
         admin_accounts = list_transfer_admin_accounts()
-        topology = Topology.create_from_config(logger=logger)
+        topology = Topology.create_from_config(logger=logger, ignore_availability=ignore_availability)
         requests_with_sources = list_transfer_requests_and_source_replicas(
             total_workers=total_workers,
             worker_number=worker_number,
             limit=bulk,
             request_state=RequestState.PREPARING,
             request_type=[RequestType.TRANSFER, RequestType.STAGEIN],
-            session=session
+            session=session,
+            ignore_availability=ignore_availability
         )
         ret = build_transfer_paths(
             topology=topology,


Test patch:
diff --git a/lib/rucio/tests/test_conveyor.py b/lib/rucio/tests/test_conveyor.py
--- a/lib/rucio/tests/test_conveyor.py
+++ b/lib/rucio/tests/test_conveyor.py
@@ -709,7 +709,7 @@ def test_preparer_throttler_submitter(rse_factory, did_factory, root_account, fi
     submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None)
 
     # One RSE has limits set: the requests will be moved to WAITING status; the other RSE has no limits: go directly to queued
-    preparer(once=True, sleep_time=1, bulk=100, partition_wait_time=0)
+    preparer(once=True, sleep_time=1, bulk=100, partition_wait_time=0, ignore_availability=False)
     request = request_core.get_request_by_did(rse_id=dst_rse_id1, **did1)
     assert request['state'] == RequestState.WAITING
     request = request_core.get_request_by_did(rse_id=dst_rse_id1, **did2)
@@ -1549,3 +1549,40 @@ def on_submit(file):
     request = __wait_for_request_state(dst_rse_id=dst_rse3_id, state=RequestState.FAILED, **did)
     assert 'Source and destination checksums do not match' in request['err_msg']
     assert request['state'] == RequestState.FAILED
+
+
+@pytest.mark.noparallel(reason="runs multiple conveyor daemons")
+@pytest.mark.parametrize("file_config_mock", [{
+    "overrides": [('conveyor', 'use_preparer', 'true')]
+}], indirect=True)
+@pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
+    'rucio.core.topology.REGION',
+]}], indirect=True)
+def test_preparer_ignore_availability(rse_factory, did_factory, root_account, file_config_mock, caches_mock):
+    """
+    Integration test of the preparer/throttler workflow.
+    """
+
+    def __setup_test():
+        src_rse, src_rse_id = rse_factory.make_posix_rse()
+        dst_rse, dst_rse_id = rse_factory.make_posix_rse()
+
+        distance_core.add_distance(src_rse_id, dst_rse_id, distance=10)
+        for rse_id in [src_rse_id, dst_rse_id]:
+            rse_core.add_rse_attribute(rse_id, 'fts', TEST_FTS_HOST)
+        did = did_factory.upload_test_file(src_rse)
+        rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
+
+        rse_core.update_rse(src_rse_id, {'availability_read': False})
+
+        return src_rse_id, dst_rse_id, did
+
+    src_rse_id, dst_rse_id, did = __setup_test()
+    preparer(once=True, sleep_time=1, bulk=100, partition_wait_time=0, ignore_availability=False)
+    request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+    assert request['state'] == RequestState.NO_SOURCES
+
+    src_rse_id, dst_rse_id, did = __setup_test()
+    preparer(once=True, sleep_time=1, bulk=100, partition_wait_time=0, ignore_availability=True)
+    request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+    assert request['state'] == RequestState.QUEUED