Repository: rucio/rucio

Issue:
Remove hard coded value for GCS storage elements
Motivation
----------
The function that generates the signed URL for GCS have some hard-coded values




Discussion:

Resolving patch:
diff --git a/lib/rucio/common/utils.py b/lib/rucio/common/utils.py
--- a/lib/rucio/common/utils.py
+++ b/lib/rucio/common/utils.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2012-2021 CERN
+# Copyright 2012-2022 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2012-2018
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2012-2021
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2021
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2021
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2022
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
 # - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2015-2018
 # - Martin Barisits <martin.barisits@cern.ch>, 2016-2021
@@ -39,6 +39,11 @@
 # - Anil Panta <47672624+panta-123@users.noreply.github.com>, 2021
 # - Ilija Vukotic <ivukotic@cern.ch>, 2021
 # - David Población Criado <david.poblacion.criado@cern.ch>, 2021
+# - martynia <janusz.martyniak@googlemail.com>, 2021
+# - jdierkes <joel.dierkes@cern.ch>, 2021
+# - Rakshita Varadarajan <rakshitajps@gmail.com>, 2021
+# - Rob Barnsley <robbarnsley@users.noreply.github.com>, 2021
+# - Igor Mandrichenko <ivm@fnal.gov>, 2021
 # - Joel Dierkes <joel.dierkes@cern.ch>, 2021
 
 from __future__ import absolute_import, print_function
@@ -190,8 +195,7 @@ def all_oidc_req_claims_present(scope, audience, required_scope, required_audien
         req_scope_present = all(elem in scope for elem in required_scope)
         req_audience_present = all(elem in audience for elem in required_audience)
         return req_scope_present and req_audience_present
-    elif (isinstance(scope, string_types) and isinstance(audience, string_types)  # NOQA: W504
-          and isinstance(required_scope, string_types) and isinstance(required_audience, string_types)):
+    elif (isinstance(scope, string_types) and isinstance(audience, string_types) and isinstance(required_scope, string_types) and isinstance(required_audience, string_types)):
         scope = str(scope)
         audience = str(audience)
         required_scope = str(required_scope)
@@ -199,8 +203,7 @@ def all_oidc_req_claims_present(scope, audience, required_scope, required_audien
         req_scope_present = all(elem in scope.split(sepatator) for elem in required_scope.split(sepatator))
         req_audience_present = all(elem in audience.split(sepatator) for elem in required_audience.split(sepatator))
         return req_scope_present and req_audience_present
-    elif (isinstance(scope, list) and isinstance(audience, list)  # NOQA: W504
-          and isinstance(required_scope, string_types) and isinstance(required_audience, string_types)):
+    elif (isinstance(scope, list) and isinstance(audience, list) and isinstance(required_scope, string_types) and isinstance(required_audience, string_types)):
         scope = [str(it) for it in scope]
         audience = [str(it) for it in audience]
         required_scope = str(required_scope)
@@ -208,8 +211,7 @@ def all_oidc_req_claims_present(scope, audience, required_scope, required_audien
         req_scope_present = all(elem in scope for elem in required_scope.split(sepatator))
         req_audience_present = all(elem in audience for elem in required_audience.split(sepatator))
         return req_scope_present and req_audience_present
-    elif (isinstance(scope, string_types) and isinstance(audience, string_types)  # NOQA: W504
-          and isinstance(required_scope, list) and isinstance(required_audience, list)):
+    elif (isinstance(scope, string_types) and isinstance(audience, string_types) and isinstance(required_scope, list) and isinstance(required_audience, list)):
         scope = str(scope)
         audience = str(audience)
         required_scope = [str(it) for it in required_scope]
@@ -710,7 +712,7 @@ def clean_surls(surls):
             surl = re.sub(r'/srm/managerv1\?SFN=', '', surl)
             surl = re.sub(r'/srm/v2/server\?SFN=', '', surl)
             surl = re.sub(r'/srm/managerv2\?SFN=', '', surl)
-        if surl.startswith('https://storage.googleapis.com'):
+        if '?GoogleAccessId' in surl:
             surl = surl.split('?GoogleAccessId')[0]
         if '?X-Amz' in surl:
             surl = surl.split('?X-Amz')[0]
diff --git a/lib/rucio/core/credential.py b/lib/rucio/core/credential.py
--- a/lib/rucio/core/credential.py
+++ b/lib/rucio/core/credential.py
@@ -19,6 +19,7 @@
 # - James Perry <j.perry@epcc.ed.ac.uk>, 2019
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2021
 # - Radu Carpa <radu.carpa@cern.ch>, 2022
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2022
 
 import base64
 import datetime
@@ -81,6 +82,8 @@ def get_signed_url(rse_id, service, operation, url, lifetime=600):
             CREDS_GCS = ServiceAccountCredentials.from_json_keyfile_name(config_get('credentials', 'gcs',
                                                                                     raise_exception=False,
                                                                                     default='/opt/rucio/etc/google-cloud-storage-test.json'))
+        components = urlparse(url)
+        host = components.netloc
 
         # select the correct operation
         operations = {'read': 'GET', 'write': 'PUT', 'delete': 'DELETE'}
@@ -96,7 +99,7 @@ def get_signed_url(rse_id, service, operation, url, lifetime=600):
             lifetime = int(time.mktime(lifetime.timetuple()))
 
         # sign the path only
-        path = urlparse(url).path
+        path = components.path
 
         # assemble message to sign
         to_sign = "%s\n\n\n%s\n%s" % (operation, lifetime, path)
@@ -106,10 +109,11 @@ def get_signed_url(rse_id, service, operation, url, lifetime=600):
         signature = urlencode({'': base64.b64encode(CREDS_GCS.sign_blob(to_sign)[1])})[1:]
 
         # assemble final signed URL
-        signed_url = 'https://storage.googleapis.com:443%s?GoogleAccessId=%s&Expires=%s&Signature=%s' % (path,
-                                                                                                         CREDS_GCS.service_account_email,
-                                                                                                         lifetime,
-                                                                                                         signature)
+        signed_url = 'https://%s%s?GoogleAccessId=%s&Expires=%s&Signature=%s' % (host,
+                                                                                 path,
+                                                                                 CREDS_GCS.service_account_email,
+                                                                                 lifetime,
+                                                                                 signature)
 
     elif service == 's3':
         # split URL to get hostname, bucket and key


Test patch:
diff --git a/lib/rucio/tests/test_credential.py b/lib/rucio/tests/test_credential.py
--- a/lib/rucio/tests/test_credential.py
+++ b/lib/rucio/tests/test_credential.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2018-2021 CERN
+# Copyright 2018-2022 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,8 +20,9 @@
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2019
 # - Martin Barisits <martin.barisits@cern.ch>, 2019
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
-# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
 # - Simon Fayer <simon.fayer05@imperial.ac.uk>, 2021
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2022
 
 import unittest
 
@@ -103,7 +104,7 @@ def test_sign_url_gcs(self):
 
         pytest.raises(UnsupportedOperation, get_signed_url, self.rse1_id, 'gcs', 'read', '')
 
-        value = get_signed_url(self.rse1_id, 'gcs', 'read', 'http://storage/directory/file', lifetime=None)
+        value = get_signed_url(self.rse1_id, 'gcs', 'read', 'http://storage.googleapis.com:443/directory/file', lifetime=None)
         expected = ('https://storage.googleapis.com:443/directory/file?GoogleAccessId=rucio-test@rucio-test'
                     '.iam.gserviceaccount.com&Expires=0&Signature=u9cBWowYX22sAyApH5YySD9h0m%2FbIPLHLgY'
                     '0Db%2BQ4a0wICQ2PZzUfTuHXQF8dUbMJG04VH90U5EMzYg3qSUGyfnp6Jptnvgivf7iSHepJsYhyAYSBGs'
@@ -112,7 +113,7 @@ def test_sign_url_gcs(self):
                     '8pTeva5cN8ZFlMkeCz7JvNkVJb1KPhI1XHPWyfuPUa2ALHh9wAD2yFSOU3cDiORFE6A%3D%3D')
         assert value == expected
 
-        value = get_signed_url(self.rse1_id, 'gcs', 'write', 'http://storage/directory/file', lifetime=None)
+        value = get_signed_url(self.rse1_id, 'gcs', 'write', 'http://storage.googleapis.com:443/directory/file', lifetime=None)
         expected = ('https://storage.googleapis.com:443/directory/file?GoogleAccessId=rucio-test@rucio-test'
                     '.iam.gserviceaccount.com&Expires=0&Signature=Gn%2FL0%2FjGkBIdpHZ9bKw7tvqRCdslC11gt'
                     'jbLk5AG2jA4Ywd6mTvOinUB%2BZxHY2I3XzEuMfyMnFj0vfXSemN6XmmcQkiQBhl6P3zr0GrOuO4y0xjKT'
@@ -121,7 +122,7 @@ def test_sign_url_gcs(self):
                     '%2BnqImWDOdvmcLIZzc6x9l6b7ETOqSL2OqOCStpBHPzpQU0spgJS96IB09uGRQum1Ej2ui5g%3D%3D')
         assert value == expected
 
-        value = get_signed_url(self.rse1_id, 'gcs', 'delete', 'http://storage/directory/file', lifetime=None)
+        value = get_signed_url(self.rse1_id, 'gcs', 'delete', 'http://storage.googleapis.com:443/directory/file', lifetime=None)
         expected = ('https://storage.googleapis.com:443/directory/file?GoogleAccessId=rucio-test@rucio-test'
                     '.iam.gserviceaccount.com&Expires=0&Signature=FVDNroX1epdTCv%2BC74o%2B8uWyvJXrqiIWg'
                     'kdcedaOoryhRMjuv%2FVdKecnhViY%2BGOP%2B0CoI1uFOHBz%2B%2Bm10U9A3i%2B1v7AZRN5L6nbbS%2'