Repository: rucio/rucio

Issue:
Requests to unavailable RSEs
In https://github.com/rucio/rucio/blob/631e5d5daa81ab86f310fd046350e8212b91e1eb/lib/rucio/core/transfer.py#L1675 requests are filtered to ensure the destination is available for writing. On the other hand, rules can be made with (and since #4152, repaired with) `ignore_availability` flag. I would propose to remove this filter.


Discussion:
I think this needs a bit of discussion. What worries me with that change is that sites have no option then to stop submission to their site. There is just no other switch, other than to remove the protocols to give rucio no chance to submit.

The `ignore_availability` flag for me is more at the time you execute the `add_rule`. At this point we check availability, and give an error if not possible, unless you have `ignore_availability` set. But this doesn't mean (to me) that also the submission should be done with ignored availability.
What is the meaning of an unavailable rse if we can submit transfers to/from it ? O_O
So indeed this a bit of a contrived situation, but we have a "loadtest" that routinely makes transfers to test site availability, and the results of that are used to update the actual RSE availability. So if a site is unavailable the loadtests cannot run and (possibly) succeed to bring the site back into available state. Obviously this is a niche use, and for other reasons we already run a separate conveyor for these functional tests.  What if we add a `--ignore-availability` to the conveyor daemon command to fill this role?
I think a `--ignore-availability` at the conveyor-submitter level is doable. @rcarpa ?
I think it would be easy to do. To be checked in the code... 

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -1139,7 +1139,7 @@ def __transfer_order_key(transfer_path):
 @transactional_session
 def next_transfers_to_submit(total_workers=0, worker_number=0, partition_hash_var=None, limit=None, activity=None, older_than=None, rses=None, schemes=None,
                              failover_schemes=None, filter_transfertool=None, transfertools_by_name=None, request_type=RequestType.TRANSFER,
-                             logger=logging.log, session=None):
+                             ignore_availability=False, logger=logging.log, session=None):
     """
     Get next transfers to be submitted; grouped by transfertool which can submit them
     :param total_workers:         Number of total workers.
@@ -1154,6 +1154,7 @@ def next_transfers_to_submit(total_workers=0, worker_number=0, partition_hash_va
     :param transfertools_by_name: Dict: {transfertool_name_str: transfertool class}
     :param filter_transfertool:   The transfer tool to filter requests on.
     :param request_type           The type of requests to retrieve (Transfer/Stagein)
+    :param ignore_availability:   Ignore blocklisted RSEs
     :param logger:                Optional decorated logger that can be passed from the calling daemons or servers.
     :param session:               The database session in use.
     :returns:                     Dict: {TransferToolBuilder: <list of transfer paths (possibly multihop) to be submitted>}
@@ -1189,6 +1190,7 @@ def next_transfers_to_submit(total_workers=0, worker_number=0, partition_hash_va
         rses=rses,
         request_type=request_type,
         request_state=RequestState.QUEUED,
+        ignore_availability=ignore_availability,
         transfertool=filter_transfertool,
         session=session,
     )
@@ -1199,6 +1201,7 @@ def next_transfers_to_submit(total_workers=0, worker_number=0, partition_hash_va
         multihop_rses=multihop_rses,
         schemes=schemes,
         failover_schemes=failover_schemes,
+        ignore_availability=ignore_availability,
         logger=logger,
         session=session,
     )
@@ -1232,6 +1235,7 @@ def __build_transfer_paths(
         multihop_rses: "List[str]",
         schemes: "List[str]",
         failover_schemes: "List[str]",
+        ignore_availability: bool = False,
         logger: "Callable" = logging.log,
         session: "Optional[Session]" = None,
 ):
@@ -1251,7 +1255,8 @@ def __build_transfer_paths(
     unavailable_write_rse_ids = __get_unavailable_rse_ids(operation='write', session=session)
 
     # Disallow multihop via blocklisted RSEs
-    multihop_rses = list(set(multihop_rses).difference(unavailable_write_rse_ids).difference(unavailable_read_rse_ids))
+    if not ignore_availability:
+        multihop_rses = list(set(multihop_rses).difference(unavailable_write_rse_ids).difference(unavailable_read_rse_ids))
 
     candidate_paths_by_request_id, reqs_no_source, reqs_only_tape_source, reqs_scheme_mismatch = {}, set(), set(), set()
     for rws in requests_with_sources:
@@ -1269,7 +1274,7 @@ def __build_transfer_paths(
         reqs_no_source.add(rws.request_id)
 
         # Check if destination is blocked
-        if rws.dest_rse.id in unavailable_write_rse_ids:
+        if not ignore_availability and rws.dest_rse.id in unavailable_write_rse_ids:
             logger(logging.WARNING, 'RSE %s is blocked for write. Will skip the submission of new jobs', rws.dest_rse)
             continue
 
@@ -1291,7 +1296,8 @@ def __build_transfer_paths(
             filtered_sources = filter(lambda s: s.rse.id in allowed_source_rses, filtered_sources)
         filtered_sources = filter(lambda s: s.rse.name is not None, filtered_sources)
         # Ignore blocklisted RSEs
-        filtered_sources = filter(lambda s: s.rse.id not in unavailable_read_rse_ids, filtered_sources)
+        if not ignore_availability:
+            filtered_sources = filter(lambda s: s.rse.id not in unavailable_read_rse_ids, filtered_sources)
         # For staging requests, the staging_buffer attribute must be correctly set
         if rws.request_type == RequestType.STAGEIN:
             filtered_sources = filter(lambda s: s.rse.attributes.get('staging_buffer') == rws.dest_rse.name, filtered_sources)
@@ -1537,23 +1543,25 @@ def __list_transfer_requests_and_source_replicas(
     rses=None,
     request_type=RequestType.TRANSFER,
     request_state=None,
+    ignore_availability=False,
     transfertool=None,
     session=None,
 ) -> "List[RequestWithSources]":
     """
     List requests with source replicas
-    :param total_workers:     Number of total workers.
-    :param worker_number:     Id of the executing worker.
-    :param partition_hash_var The hash variable used for partitioning thread work
-    :param limit:             Integer of requests to retrieve.
-    :param activity:          Activity to be selected.
-    :param older_than:        Only select requests older than this DateTime.
-    :param rses:              List of rse_id to select requests.
-    :param request_type:      Filter on the given request type.
-    :param request_state:     Filter on the given request state
-    :param transfertool:      The transfer tool as specified in rucio.cfg.
-    :param session:           Database session to use.
-    :returns:                 List of RequestWithSources objects.
+    :param total_workers:      Number of total workers.
+    :param worker_number:      Id of the executing worker.
+    :param partition_hash_var  The hash variable used for partitioning thread work
+    :param limit:              Integer of requests to retrieve.
+    :param activity:           Activity to be selected.
+    :param older_than:         Only select requests older than this DateTime.
+    :param rses:               List of rse_id to select requests.
+    :param request_type:       Filter on the given request type.
+    :param request_state:      Filter on the given request state
+    :param transfertool:       The transfer tool as specified in rucio.cfg.
+    :param ignore_availability Ignore blocklisted RSEs
+    :param session:            Database session to use.
+    :returns:                  List of RequestWithSources objects.
     """
 
     if partition_hash_var is None:
@@ -1583,8 +1591,10 @@ def __list_transfer_requests_and_source_replicas(
         .filter(models.Request.request_type == request_type) \
         .join(models.RSE, models.RSE.id == models.Request.dest_rse_id) \
         .filter(models.RSE.deleted == false()) \
-        .order_by(models.Request.created_at) \
-        .filter(models.RSE.availability.in_((2, 3, 6, 7)))
+        .order_by(models.Request.created_at)
+
+    if not ignore_availability:
+        sub_requests = sub_requests.filter(models.RSE.availability.in_((2, 3, 6, 7)))
 
     if isinstance(older_than, datetime.datetime):
         sub_requests = sub_requests.filter(models.Request.requested_at < older_than)
diff --git a/lib/rucio/daemons/conveyor/submitter.py b/lib/rucio/daemons/conveyor/submitter.py
--- a/lib/rucio/daemons/conveyor/submitter.py
+++ b/lib/rucio/daemons/conveyor/submitter.py
@@ -78,7 +78,8 @@
 def submitter(once=False, rses=None, partition_wait_time=10,
               bulk=100, group_bulk=1, group_policy='rule', source_strategy=None,
               activities=None, sleep_time=600, max_sources=4, archive_timeout_override=None,
-              filter_transfertool=FILTER_TRANSFERTOOL, transfertool=TRANSFER_TOOL, transfertype=TRANSFER_TYPE):
+              filter_transfertool=FILTER_TRANSFERTOOL, transfertool=TRANSFER_TOOL,
+              transfertype=TRANSFER_TYPE, ignore_availability=False):
     """
     Main loop to submit a new transfer primitive to a transfertool.
     """
@@ -181,6 +182,7 @@ def submitter(once=False, rses=None, partition_wait_time=10,
                         transfertools_by_name={transfertool: TRANSFERTOOL_CLASSES_BY_NAME[transfertool]},
                         older_than=None,
                         request_type=RequestType.TRANSFER,
+                        ignore_availability=ignore_availability,
                         logger=logger,
                     )
                     total_transfers = len(list(hop for paths in transfers.values() for path in paths for hop in path))
@@ -225,7 +227,7 @@ def stop(signum=None, frame=None):
 
 def run(once=False, group_bulk=1, group_policy='rule', mock=False,
         rses=None, include_rses=None, exclude_rses=None, vos=None, bulk=100, source_strategy=None,
-        activities=None, exclude_activities=None, sleep_time=600, max_sources=4,
+        activities=None, exclude_activities=None, ignore_availability=False, sleep_time=600, max_sources=4,
         archive_timeout_override=None, total_threads=1):
     """
     Starts up the conveyer threads.
@@ -271,6 +273,7 @@ def run(once=False, group_bulk=1, group_policy='rule', mock=False,
                                                           'group_bulk': group_bulk,
                                                           'group_policy': group_policy,
                                                           'activities': activities,
+                                                          'ignore_availability': ignore_availability,
                                                           'sleep_time': sleep_time,
                                                           'max_sources': max_sources,
                                                           'source_strategy': source_strategy,


Test patch:
diff --git a/lib/rucio/tests/test_conveyor_submitter.py b/lib/rucio/tests/test_conveyor_submitter.py
--- a/lib/rucio/tests/test_conveyor_submitter.py
+++ b/lib/rucio/tests/test_conveyor_submitter.py
@@ -174,6 +174,39 @@ def __ensure_source_exists(rse_id, scope, name, session=None):
     assert metrics_mock.get_sample_value('rucio_core_request_submit_transfer_total') == 1
 
 
+@pytest.mark.noparallel(reason="multiple submitters cannot be run in parallel due to partial job assignment by hash")
+@pytest.mark.parametrize("core_config_mock", [{"table_content": [
+    ('transfers', 'use_multihop', True)
+]}], indirect=True)
+@pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
+    'rucio.core.rse_expression_parser.REGION',  # The list of multihop RSEs is retrieved by rse expression
+    'rucio.core.transfer.REGION_SHORT',
+]}], indirect=True)
+def test_ignore_availability(rse_factory, did_factory, root_account, core_config_mock, caches_mock):
+
+    def __setup_test():
+        src_rse, src_rse_id = rse_factory.make_posix_rse()
+        dst_rse, dst_rse_id = rse_factory.make_posix_rse()
+
+        distance_core.add_distance(src_rse_id, dst_rse_id, ranking=10)
+        did = did_factory.upload_test_file(src_rse)
+        rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
+
+        rse_core.update_rse(src_rse_id, {'availability_read': False})
+
+        return src_rse_id, dst_rse_id, did
+
+    src_rse_id, dst_rse_id, did = __setup_test()
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in (src_rse_id, dst_rse_id)], partition_wait_time=None, transfertool='mock', transfertype='single')
+    request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+    assert request['state'] == RequestState.NO_SOURCES
+
+    src_rse_id, dst_rse_id, did = __setup_test()
+    submitter(once=True, rses=[{'id': rse_id} for rse_id in (src_rse_id, dst_rse_id)], partition_wait_time=None, transfertool='mock', transfertype='single', ignore_availability=True)
+    request = request_core.get_request_by_did(rse_id=dst_rse_id, **did)
+    assert request['state'] == RequestState.SUBMITTED
+
+
 @pytest.mark.noparallel(reason="multiple submitters cannot be run in parallel due to partial job assignment by hash")
 def test_globus(rse_factory, did_factory, root_account):
     """