Repository: rucio/rucio

Issue:
remove https->davs translation on FTS submission
Recently, we realized this code exists in rucio: 

https://github.com/rucio/rucio/blob/8eb35a3fb58a29208347050ab7052cd3c178c1ad/lib/rucio/transfertool/fts3.py#L816

Meaning that all urls starting with https are rewritten to davs:// just before the submission to FTS. This code tracks all the way back to 2015. Discussion with the FTS team revealed that many improvements where done in this area, so this hack is "probably not needed anymore"

We should disable this behavior by default in the next major release and allow to re-enable it using a configuration value in case somebody from rucio communities encounters issues do to it.


Discussion:

Resolving patch:
diff --git a/lib/rucio/transfertool/fts3.py b/lib/rucio/transfertool/fts3.py
--- a/lib/rucio/transfertool/fts3.py
+++ b/lib/rucio/transfertool/fts3.py
@@ -71,6 +71,7 @@
 ALLOW_USER_OIDC_TOKENS = config_get_bool('conveyor', 'allow_user_oidc_tokens', False, False)
 REQUEST_OIDC_SCOPE = config_get('conveyor', 'request_oidc_scope', False, 'fts:submit-transfer')
 REQUEST_OIDC_AUDIENCE = config_get('conveyor', 'request_oidc_audience', False, 'fts:example')
+REWRITE_HTTPS_TO_DAVS = config_get_bool('transfers', 'rewrite_https_to_davs', default=False)
 
 # https://fts3-docs.web.cern.ch/fts3-docs/docs/state_machine.html
 FINAL_FTS_JOB_STATES = (FTS_STATE.FAILED, FTS_STATE.CANCELED, FTS_STATE.FINISHED, FTS_STATE.FINISHEDDIRTY)
@@ -811,21 +812,23 @@ def submit(self, transfers, job_params, timeout=None):
             if not transfer_file['sources'] or transfer_file['sources'] == []:
                 raise Exception('No sources defined')
 
-            new_src_urls = []
-            new_dst_urls = []
-            for url in transfer_file['sources']:
-                if url.startswith('https'):
-                    new_src_urls.append(':'.join(['davs'] + url.split(':')[1:]))
-                else:
-                    new_src_urls.append(url)
-            for url in transfer_file['destinations']:
-                if url.startswith('https'):
-                    new_dst_urls.append(':'.join(['davs'] + url.split(':')[1:]))
-                else:
-                    new_dst_urls.append(url)
-
-            transfer_file['sources'] = new_src_urls
-            transfer_file['destinations'] = new_dst_urls
+            # TODO: remove the following logic in rucio 1.31
+            if REWRITE_HTTPS_TO_DAVS:
+                new_src_urls = []
+                new_dst_urls = []
+                for url in transfer_file['sources']:
+                    if url.startswith('https'):
+                        new_src_urls.append(':'.join(['davs'] + url.split(':')[1:]))
+                    else:
+                        new_src_urls.append(url)
+                for url in transfer_file['destinations']:
+                    if url.startswith('https'):
+                        new_dst_urls.append(':'.join(['davs'] + url.split(':')[1:]))
+                    else:
+                        new_dst_urls.append(url)
+
+                transfer_file['sources'] = new_src_urls
+                transfer_file['destinations'] = new_dst_urls
 
         transfer_id = None
         expected_transfer_id = None


Test patch: