Repository: rucio/rucio

Issue:
Skip Xcache URL mangling when using an endpoint that requires signatures
Motivation
----------

When sites that have an Xcache installed download from a cloud storage that uses URL signatures, the final URL that is returned is invalid.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2019 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2013-2020 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013-2014
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2019
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2014-2019
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2014-2020
 # - David Cameron <d.g.cameron@gmail.com>, 2014
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2014-2018
 # - Wen Guan <wguan.icedew@gmail.com>, 2014-2015
@@ -933,6 +933,14 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns,
                                                             'name': name,
                                                             'path': path}).values())[0]
 
+                        # do we need to sign the URLs?
+                        if sign_urls and protocol.attributes['scheme'] == 'https':
+                            service = get_rse_attribute('sign_url',
+                                                        rse_id=rse_id,
+                                                        session=session)
+                            if service and isinstance(service, list):
+                                pfn = get_signed_url(rse_id=rse_id, service=service[0], operation='read', url=pfn, lifetime=signature_lifetime)
+
                         # server side root proxy handling if location is set.
                         # supports root and http destinations
                         # cannot be pushed into protocols because we need to lookup rse attributes.
@@ -954,17 +962,15 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns,
                                                                      client_location['site'],  # option
                                                                      default='',               # empty string to circumvent exception
                                                                      session=session)
-                                    if root_proxy_internal:
-                                        # don't forget to mangle gfal-style davs URL into generic https URL
-                                        pfn = 'root://' + root_proxy_internal + '//' + pfn.replace('davs://', 'https://')
 
-                        # do we need to sign the URLs?
-                        if sign_urls and protocol.attributes['scheme'] == 'https':
-                            service = get_rse_attribute('sign_url',
-                                                        rse_id=rse_id,
-                                                        session=session)
-                            if service and isinstance(service, list):
-                                pfn = get_signed_url(rse_id=rse_id, service=service[0], operation='read', url=pfn, lifetime=signature_lifetime)
+                                    if root_proxy_internal:
+                                        # TODO: XCache does not seem to grab signed URLs. Doublecheck with XCache devs.
+                                        #       For now -> skip prepending XCache for GCS.
+                                        if 'storage.googleapis.com' in pfn or 'atlas-google-cloud.cern.ch' in pfn:
+                                            pass  # ATLAS HACK
+                                        else:
+                                            # don't forget to mangle gfal-style davs URL into generic https URL
+                                            pfn = 'root://' + root_proxy_internal + '//' + pfn.replace('davs://', 'https://')
 
                         # PFNs don't have concepts, therefore quickly encapsulate in a tuple
                         # ('pfn', 'domain', 'priority', 'client_extract')


Test patch:
diff --git a/lib/rucio/tests/test_root_proxy.py b/lib/rucio/tests/test_root_proxy.py
--- a/lib/rucio/tests/test_root_proxy.py
+++ b/lib/rucio/tests/test_root_proxy.py
@@ -1,4 +1,4 @@
-# Copyright 2017-2018 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2017-2020 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 # Authors:
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2017-2019
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2017-2020
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2019
 # - Andrew Lister, <andrew.lister@stfc.ac.uk>, 2019
@@ -41,7 +41,8 @@
 
 class TestROOTProxy(object):
 
-    def setup(self):
+    @classmethod
+    def setupClass(self):
 
         self.rc = ReplicaClient()
 
@@ -103,16 +104,16 @@ def setup(self):
                                                           'write': 1,
                                                           'delete': 1}}})
 
-    def tearDown(self):
+    @classmethod
+    def tearDownClass(self):
         for rse_id in [self.rse_with_proxy_id, self.rse_without_proxy_id]:
             delete_replicas(rse_id=rse_id, files=self.files)
         del_rse(self.rse_with_proxy_id)
         del_rse(self.rse_without_proxy_id)
 
-    def test_client_list_replicas(self):
-        """ ROOT (CLIENT): Test internal proxy prepend """
+    def test_client_list_replicas1(self):
+        """ ROOT (CLIENT): No proxy involved """
 
-        #  no proxy involved
         replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock',
                                                             'name': f['name'],
                                                             'type': 'FILE'} for f in self.files],
@@ -125,7 +126,9 @@ def test_client_list_replicas(self):
         found_pfns = [list(replica['pfns'].keys())[0] for replica in replicas]
         assert_equal(sorted(found_pfns), sorted(expected_pfns))
 
-        #  outgoing proxy needs to be prepended
+    def test_client_list_replicas2(self):
+        """ ROOT (CLIENT): Outgoing proxy needs to be prepended"""
+
         replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock',
                                                             'name': f['name'],
                                                             'type': 'FILE'} for f in self.files],
@@ -138,7 +141,9 @@ def test_client_list_replicas(self):
         found_pfns = [list(replica['pfns'].keys())[0] for replica in replicas]
         assert_equal(sorted(found_pfns), sorted(expected_pfns))
 
-        # outgoing proxy at destination does not matter
+    def test_client_list_replicas3(self):
+        """ ROOT (CLIENT): Outgoing proxy at destination does not matter"""
+
         replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock',
                                                             'name': f['name'],
                                                             'type': 'FILE'} for f in self.files],
@@ -151,7 +156,9 @@ def test_client_list_replicas(self):
         found_pfns = [list(replica['pfns'].keys())[0] for replica in replicas]
         assert_equal(sorted(found_pfns), sorted(expected_pfns))
 
-        # outgoing proxy does not matter when staying at site
+    def test_client_list_replicas4(self):
+        """ ROOT (CLIENT): Outgoing proxy does not matter when staying at site"""
+
         replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock',
                                                             'name': f['name'],
                                                             'type': 'FILE'} for f in self.files],