Repository: rucio/rucio

Issue:
Stronger integration of Globus Online transfer tool



Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -24,9 +24,9 @@
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Brandon White <bjwhite@fnal.gov>, 2019
+# - Matt Snyder <msnyder@bnl.gov>, 2019-2021
 # - Gabriele Fronze' <gfronze@cern.ch>, 2019
 # - Jaroslav Guenther <jaroslav.guenther@cern.ch>, 2019-2020
-# - Matt Snyder <msnyder@bnl.gov>, 2020-2021
 # - Eric Vaandering <ewv@fnal.gov>, 2020
 # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
 # - Nick Smith <nick.smith@cern.ch>, 2020
@@ -721,7 +721,7 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
         # In case of non-connected, the list contains all the intermediary RSEs
         list_hops = []
         include_multihop = False
-        if transfertool in ['fts', None]:
+        if transfertool in ['fts3', None]:
             include_multihop = core_config_get('transfers', 'use_multihop', default=False, expiration_time=600, session=session)
 
         try:
diff --git a/lib/rucio/daemons/conveyor/poller.py b/lib/rucio/daemons/conveyor/poller.py
--- a/lib/rucio/daemons/conveyor/poller.py
+++ b/lib/rucio/daemons/conveyor/poller.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2013-2020 CERN
+# Copyright 2013-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2018
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2014-2018
 # - Wen Guan <wen.guan@cern.ch>, 2014-2016
-# - Martin Barisits <martin.barisits@cern.ch>, 2016-2020
+# - Martin Barisits <martin.barisits@cern.ch>, 2016-2021
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 # - Matt Snyder <msnyder@bnl.gov>, 2019-2021
 # - Brandon White <bjwhite@fnal.gov>, 2019
@@ -63,7 +63,8 @@
 
 datetime.datetime.strptime('', '')
 
-TRANSFER_TOOL = config_get('conveyor', 'transfertool', False, None)
+TRANSFER_TOOL = config_get('conveyor', 'transfertool', False, None)  # NOTE: This should eventually be completely removed, as it can be fetched from the request
+FILTER_TRANSFERTOOL = config_get('conveyor', 'filter_transfertool', False, None)  # NOTE: TRANSFERTOOL to filter requests on
 
 
 def poller(once=False, activities=None, sleep_time=60,
@@ -85,8 +86,8 @@ def poller(once=False, activities=None, sleep_time=60,
     if activity_shares:
         activities.sort()
         executable += '--activity_shares' + str(activity_shares)
-    if TRANSFER_TOOL:
-        executable += ' --transfertool ' + TRANSFER_TOOL
+    if FILTER_TRANSFERTOOL:
+        executable += ' --filter-transfertool ' + FILTER_TRANSFERTOOL
 
     hostname = socket.getfqdn()
     pid = os.getpid()
@@ -121,7 +122,7 @@ def poller(once=False, activities=None, sleep_time=60,
                     continue
 
                 start_time = time.time()
-                logger(logging.DEBUG, 'Start to poll transfers older than %i seconds for activity %s using transfer tool: %s' % (older_than, activity, TRANSFER_TOOL))
+                logger(logging.DEBUG, 'Start to poll transfers older than %i seconds for activity %s using transfer tool: %s' % (older_than, activity, FILTER_TRANSFERTOOL))
                 transfs = request_core.get_next(request_type=[RequestType.TRANSFER, RequestType.STAGEIN, RequestType.STAGEOUT],
                                                 state=[RequestState.SUBMITTED],
                                                 limit=db_bulk,
@@ -130,7 +131,7 @@ def poller(once=False, activities=None, sleep_time=60,
                                                 mode_all=False, hash_variable='id',
                                                 activity=activity,
                                                 activity_shares=activity_shares,
-                                                transfertool=TRANSFER_TOOL)
+                                                transfertool=FILTER_TRANSFERTOOL)
 
                 record_timer('daemons.conveyor.poller.000-get_next', (time.time() - start_time) * 1000)
 
diff --git a/lib/rucio/daemons/conveyor/submitter.py b/lib/rucio/daemons/conveyor/submitter.py
--- a/lib/rucio/daemons/conveyor/submitter.py
+++ b/lib/rucio/daemons/conveyor/submitter.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2013-2020 CERN
+# Copyright 2013-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2019
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2014-2018
-# - Martin Barisits <martin.barisits@cern.ch>, 2014-2020
+# - Martin Barisits <martin.barisits@cern.ch>, 2014-2021
 # - Wen Guan <wen.guan@cern.ch>, 2014-2016
 # - Tomáš Kouba <tomas.kouba@cern.ch>, 2014
 # - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2016
@@ -67,7 +67,8 @@
 graceful_stop = threading.Event()
 
 USER_TRANSFERS = config_get('conveyor', 'user_transfers', False, None)
-TRANSFER_TOOL = config_get('conveyor', 'transfertool', False, None)
+TRANSFER_TOOL = config_get('conveyor', 'transfertool', False, None)  # NOTE: This should eventually be completely removed, as it can be fetched from the request
+FILTER_TRANSFERTOOL = config_get('conveyor', 'filter_transfertool', False, None)  # NOTE: TRANSFERTOOL to filter requests on
 TRANSFER_TYPE = config_get('conveyor', 'transfertype', False, 'single')
 
 GET_TRANSFERS_COUNTER = Counter('rucio_daemons_conveyor_submitter_get_transfers', 'Number of transfers retrieved')
@@ -118,8 +119,8 @@ def submitter(once=False, rses=None, mock=False,
     if activities:
         activities.sort()
         executable += '--activities ' + str(activities)
-    if TRANSFER_TOOL:
-        executable += ' --transfertool ' + TRANSFER_TOOL
+    if FILTER_TRANSFERTOOL:
+        executable += ' --filter-transfertool ' + FILTER_TRANSFERTOOL
 
     hostname = socket.getfqdn()
     pid = os.getpid()
@@ -172,7 +173,7 @@ def submitter(once=False, rses=None, mock=False,
                                             max_sources=max_sources,
                                             bring_online=bring_online,
                                             retry_other_fts=retry_other_fts,
-                                            transfertool=TRANSFER_TOOL,
+                                            transfertool=FILTER_TRANSFERTOOL,
                                             logger=logger)
 
                 record_timer('daemons.conveyor.transfer_submitter.get_transfers.per_transfer', (time.time() - start_time) * 1000 / (len(transfers) if transfers else 1))


Test patch: