Repository: rucio/rucio

Issue:
Configurable archive-timeout override in submitter
Motivation
----------
After the implementation of the FTS component of https://github.com/rucio/rucio/issues/1797 just recently, CMS enabled this feature. In parallel, our _loadtest_ [daemon](https://github.com/dmwm/CMSRucio/blob/master/docker/CMSRucioClient/loadtest/loadtest.py), which periodically submits transfers to test RSE and link availability, now writes files to tape sites. Site administrators have mapped the loadtest destination directory in a special way to prevent actually writing these test files to tape and wasting tape and tape drive resources. However this poses a challenge now that the archiving feature is implemented, as the site storage will never report these files as being archived, as desired. So we need some mechanism to override the per-RSE `archive_timeout` setting in conveyor-submitter, and run a dedicated submitter with that setting disabled for the load test rules (which use a special activity).


Modification
------------
Add a configuration option `archive_timeout` to conveyor-submitter daemon
https://github.com/rucio/rucio/blob/c0576045445e054c86e5c3ac7005d570fca1649c/lib/rucio/daemons/conveyor/submitter.py#L78-L81
and forward its value to override the RSE-level value at
https://github.com/rucio/rucio/blob/740b825cac747280ca159089bca4635264c07fc1/lib/rucio/daemons/conveyor/common.py#L342

cc @ericvaandering


Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/conveyor/common.py b/lib/rucio/daemons/conveyor/common.py
--- a/lib/rucio/daemons/conveyor/common.py
+++ b/lib/rucio/daemons/conveyor/common.py
@@ -32,6 +32,7 @@
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
+# - Nick Smith <nick.smith@cern.ch>, 2021
 
 """
 Methods common to different conveyor submitter daemons.
@@ -227,17 +228,18 @@ def submit_transfer(external_host, job, submitter='submitter', timeout=None, use
 
 
 @read_session
-def bulk_group_transfer(transfers, policy='rule', group_bulk=200, source_strategy=None, max_time_in_queue=None, session=None, logger=logging.log, group_by_scope=False):
+def bulk_group_transfer(transfers, policy='rule', group_bulk=200, source_strategy=None, max_time_in_queue=None, session=None, logger=logging.log, group_by_scope=False, archive_timeout_override=None):
     """
     Group transfers in bulk based on certain criterias
 
-    :param transfers:             List of transfers to group.
-    :param plicy:                 Policy to use to group.
-    :param group_bulk:            Bulk sizes.
-    :param source_strategy:       Strategy to group sources
-    :param max_time_in_queue:     Maximum time in queue
-    :param logger:                Optional decorated logger that can be passed from the calling daemons or servers.
-    :return:                      List of grouped transfers.
+    :param transfers:                List of transfers to group.
+    :param plicy:                    Policy to use to group.
+    :param group_bulk:               Bulk sizes.
+    :param source_strategy:          Strategy to group sources
+    :param max_time_in_queue:        Maximum time in queue
+    :param archive_timeout_override: Override the archive_timeout parameter for any transfers with it set (0 to unset)
+    :param logger:                   Optional decorated logger that can be passed from the calling daemons or servers.
+    :return:                         List of grouped transfers.
     """
 
     grouped_transfers = {}
@@ -340,7 +342,11 @@ def bulk_group_transfer(transfers, policy='rule', group_bulk=200, source_strateg
                       'overwrite': transfer['overwrite'],
                       'priority': 3}
         if transfer.get('archive_timeout', None):
-            job_params['archive_timeout'] = transfer['archive_timeout']
+            if archive_timeout_override is None:
+                job_params['archive_timeout'] = transfer['archive_timeout']
+            elif archive_timeout_override != 0:
+                job_params['archive_timeout'] = archive_timeout_override
+            # else don't set the value
         if multihop:
             job_params['multihop'] = True
         if strict_copy:
diff --git a/lib/rucio/daemons/conveyor/submitter.py b/lib/rucio/daemons/conveyor/submitter.py
--- a/lib/rucio/daemons/conveyor/submitter.py
+++ b/lib/rucio/daemons/conveyor/submitter.py
@@ -32,6 +32,7 @@
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
+# - Nick Smith <nick.smith@cern.ch>, 2021
 
 """
 Conveyor transfer submitter is a daemon to manage non-tape file transfers.
@@ -72,7 +73,7 @@
 
 def submitter(once=False, rses=None, partition_wait_time=10,
               bulk=100, group_bulk=1, group_policy='rule', source_strategy=None,
-              activities=None, sleep_time=600, max_sources=4, retry_other_fts=False,
+              activities=None, sleep_time=600, max_sources=4, retry_other_fts=False, archive_timeout_override=None,
               filter_transfertool=FILTER_TRANSFERTOOL, transfertool=TRANSFER_TOOL, transfertype=TRANSFER_TYPE):
     """
     Main loop to submit a new transfer primitive to a transfertool.
@@ -183,7 +184,7 @@ def submitter(once=False, rses=None, partition_wait_time=10,
                 logger(logging.INFO, 'Starting to group transfers for %s', activity)
                 start_time = time.time()
 
-                grouped_jobs = bulk_group_transfer(transfers, group_policy, group_bulk, source_strategy, max_time_in_queue, group_by_scope=user_transfer)
+                grouped_jobs = bulk_group_transfer(transfers, group_policy, group_bulk, source_strategy, max_time_in_queue, group_by_scope=user_transfer, archive_timeout_override=archive_timeout_override)
                 record_timer('daemons.conveyor.transfer_submitter.bulk_group_transfer', (time.time() - start_time) * 1000 / (len(transfers) if transfers else 1))
 
                 logger(logging.INFO, 'Starting to submit transfers for %s', activity)
@@ -252,7 +253,8 @@ def stop(signum=None, frame=None):
 
 def run(once=False, group_bulk=1, group_policy='rule', mock=False,
         rses=None, include_rses=None, exclude_rses=None, vos=None, bulk=100, source_strategy=None,
-        activities=None, exclude_activities=None, sleep_time=600, max_sources=4, retry_other_fts=False, total_threads=1):
+        activities=None, exclude_activities=None, sleep_time=600, max_sources=4, retry_other_fts=False,
+        archive_timeout_override=None, total_threads=1):
     """
     Starts up the conveyer threads.
     """
@@ -300,7 +302,8 @@ def run(once=False, group_bulk=1, group_policy='rule', mock=False,
                                                           'sleep_time': sleep_time,
                                                           'max_sources': max_sources,
                                                           'source_strategy': source_strategy,
-                                                          'retry_other_fts': retry_other_fts}) for _ in range(0, total_threads)]
+                                                          'retry_other_fts': retry_other_fts,
+                                                          'archive_timeout_override': archive_timeout_override}) for _ in range(0, total_threads)]
 
     [thread.start() for thread in threads]
 


Test patch: