Repository: rucio/rucio

Issue:
Remove `retry_other_fts` option from conveyor
Motivation
----------
The `retry_other_fts` option in the conveyor-submitter does sometimes lead to race-conditions, due accepted, but timed out, job submissions. 

Modification
------------
Remove the option from submitter and all other conveyor daemons.



Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -1235,7 +1235,6 @@ def __transfer_order_key(transfer_path):
 def __filter_for_transfertool(
         candidate_paths: "Iterable[List[DirectTransferDefinition]]",
         transfertool: str,
-        retry_other_fts: bool,
         logger: "Callable",
 ):
     """
@@ -1274,8 +1273,6 @@ def __filter_for_transfertool(
 
             if common_fts_hosts:
                 external_host = common_fts_hosts[0]
-                if retry_other_fts:
-                    external_host = common_fts_hosts[rws.retry_count % len(common_fts_hosts)]
             else:
                 if transfertool == 'fts3':
                     logger(logging.ERROR, 'FTS attribute not defined - for at least one transfer hops {} {}'.format([str(hop) for hop in transfer_path], rws.request_id))
@@ -1288,7 +1285,7 @@ def __filter_for_transfertool(
 
 @transactional_session
 def next_transfers_to_submit(total_workers=0, worker_number=0, limit=None, activity=None, older_than=None, rses=None, schemes=None,
-                             retry_other_fts=False, failover_schemes=None, transfertool=None, request_type=RequestType.TRANSFER,
+                             failover_schemes=None, transfertool=None, request_type=RequestType.TRANSFER,
                              logger=logging.log, session=None):
     """
     Get next transfers to be submitted; grouped by the external host to which they will be submitted
@@ -1299,7 +1296,6 @@ def next_transfers_to_submit(total_workers=0, worker_number=0, limit=None, activ
     :param older_than:            Get transfers older than.
     :param rses:                  Include RSES.
     :param schemes:               Include schemes.
-    :param retry_other_fts:       Retry other fts servers.
     :param failover_schemes:      Failover schemes.
     :param transfertool:          The transfer tool as specified in rucio.cfg.
     :param request_type           The type of requests to retrieve (Transfer/Stagein)
@@ -1356,7 +1352,6 @@ def next_transfers_to_submit(total_workers=0, worker_number=0, limit=None, activ
     paths_by_external_host, reqs_no_host = __pick_and_build_path_for_transfertool(
         candidate_paths,
         transfertool=transfertool,
-        retry_other_fts=retry_other_fts,
         logger=logger
     )
 
@@ -1515,7 +1510,6 @@ def __build_transfer_paths(
 
 def __pick_and_build_path_for_transfertool(
         candidate_paths_by_request_id: "Dict[str: List[DirectTransferDefinition]]",
-        retry_other_fts: bool = False,
         transfertool: "Optional[str]" = None,
         logger: "Callable" = logging.log
 ) -> "Tuple[Dict[str, List[DirectTransferDefinition]], Set[str]]":
@@ -1532,7 +1526,7 @@ def __pick_and_build_path_for_transfertool(
         # intermediate hops (if it is a multihop) work correctly
         best_path = None
         external_host = None
-        for external_host, transfer_path in __filter_for_transfertool(candidate_paths, transfertool, retry_other_fts, logger):
+        for external_host, transfer_path in __filter_for_transfertool(candidate_paths, transfertool, logger):
             if create_missing_replicas_and_requests(transfer_path, default_tombstone_delay, logger=logger):
                 best_path = transfer_path
                 break
diff --git a/lib/rucio/daemons/conveyor/stager.py b/lib/rucio/daemons/conveyor/stager.py
--- a/lib/rucio/daemons/conveyor/stager.py
+++ b/lib/rucio/daemons/conveyor/stager.py
@@ -54,7 +54,7 @@
 
 
 def stager(once=False, rses=None, bulk=100, group_bulk=1, group_policy='rule',
-           source_strategy=None, activities=None, sleep_time=600, retry_other_fts=False):
+           source_strategy=None, activities=None, sleep_time=600):
     """
     Main loop to submit a new transfer primitive to a transfertool.
     """
@@ -137,7 +137,6 @@ def stager(once=False, rses=None, bulk=100, group_bulk=1, group_policy='rule',
                     activity=activity,
                     rses=rse_ids,
                     schemes=scheme,
-                    retry_other_fts=retry_other_fts,
                     older_than=None,
                     request_type=RequestType.STAGEIN,
                     logger=logger,
@@ -186,7 +185,7 @@ def stop(signum=None, frame=None):
 
 def run(once=False, total_threads=1, group_bulk=1, group_policy='rule',
         rses=None, include_rses=None, exclude_rses=None, vos=None, bulk=100, source_strategy=None,
-        activities=[], sleep_time=600, retry_other_fts=False):
+        activities=[], sleep_time=600):
     """
     Starts up the conveyer threads.
     """
@@ -216,8 +215,7 @@ def run(once=False, total_threads=1, group_bulk=1, group_policy='rule',
                group_bulk=group_bulk,
                group_policy=group_policy,
                source_strategy=source_strategy,
-               activities=activities,
-               retry_other_fts=retry_other_fts)
+               activities=activities)
 
     else:
         logging.info('starting stager threads')
@@ -227,8 +225,7 @@ def run(once=False, total_threads=1, group_bulk=1, group_policy='rule',
                                                            'group_policy': group_policy,
                                                            'activities': activities,
                                                            'sleep_time': sleep_time,
-                                                           'source_strategy': source_strategy,
-                                                           'retry_other_fts': retry_other_fts}) for _ in range(0, total_threads)]
+                                                           'source_strategy': source_strategy}) for _ in range(0, total_threads)]
 
         [thread.start() for thread in threads]
 
diff --git a/lib/rucio/daemons/conveyor/submitter.py b/lib/rucio/daemons/conveyor/submitter.py
--- a/lib/rucio/daemons/conveyor/submitter.py
+++ b/lib/rucio/daemons/conveyor/submitter.py
@@ -70,7 +70,7 @@
 
 def submitter(once=False, rses=None, partition_wait_time=10,
               bulk=100, group_bulk=1, group_policy='rule', source_strategy=None,
-              activities=None, sleep_time=600, max_sources=4, retry_other_fts=False, archive_timeout_override=None,
+              activities=None, sleep_time=600, max_sources=4, archive_timeout_override=None,
               filter_transfertool=FILTER_TRANSFERTOOL, transfertool=TRANSFER_TOOL, transfertype=TRANSFER_TYPE):
     """
     Main loop to submit a new transfer primitive to a transfertool.
@@ -161,7 +161,6 @@ def submitter(once=False, rses=None, partition_wait_time=10,
                     activity=activity,
                     rses=rse_ids,
                     schemes=scheme,
-                    retry_other_fts=retry_other_fts,
                     transfertool=filter_transfertool,
                     older_than=None,
                     request_type=RequestType.TRANSFER,
@@ -221,7 +220,7 @@ def stop(signum=None, frame=None):
 
 def run(once=False, group_bulk=1, group_policy='rule', mock=False,
         rses=None, include_rses=None, exclude_rses=None, vos=None, bulk=100, source_strategy=None,
-        activities=None, exclude_activities=None, sleep_time=600, max_sources=4, retry_other_fts=False,
+        activities=None, exclude_activities=None, sleep_time=600, max_sources=4,
         archive_timeout_override=None, total_threads=1):
     """
     Starts up the conveyer threads.
@@ -270,7 +269,6 @@ def run(once=False, group_bulk=1, group_policy='rule', mock=False,
                                                           'sleep_time': sleep_time,
                                                           'max_sources': max_sources,
                                                           'source_strategy': source_strategy,
-                                                          'retry_other_fts': retry_other_fts,
                                                           'archive_timeout_override': archive_timeout_override}) for _ in range(0, total_threads)]
 
     [thread.start() for thread in threads]


Test patch: