Repository: rucio/rucio

Issue:
add_replicas needs to clean signed URLs from cloud service
Motivation
----------

```
Details: One of the PFNs provided does not match the Rucio expected PFN : got {

u'https': [u'https://storage.googleapis.com/atlas-eu1/rucio/user/elmsheus/0b/a6/user.elmsheus.googletest.00055.log.22334429.000001.log.tgz?GoogleAccessId=rucio-547@high-codex-266818.iam.gserviceaccount.com&Expires=1597342389&Signature=.....']}, 

expected [u'https://atlas-google-cloud.cern.ch:443/atlas-eu1/rucio/user/elmsheus/0b/a6/user.elmsheus.googletest.00055.log.22334429.000001.log.tgz’] 
```


Discussion:

Resolving patch:
diff --git a/lib/rucio/common/utils.py b/lib/rucio/common/utils.py
--- a/lib/rucio/common/utils.py
+++ b/lib/rucio/common/utils.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2019 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2012-2020 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
 # Authors:
 # - Vincent Garonne <vgaronne@gmail.com>, 2012-2018
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2012-2018
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2019
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2020
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2015-2018
@@ -732,6 +732,8 @@ def clean_surls(surls):
             surl = re.sub('/srm/managerv1\?SFN=', '', surl)  # NOQA: W605
             surl = re.sub('/srm/v2/server\?SFN=', '', surl)  # NOQA: W605
             surl = re.sub('/srm/managerv2\?SFN=', '', surl)  # NOQA: W605
+        if surl.startswith('https://storage.googleapis.com'):
+            surl = surl.split('?GoogleAccessId')[0]
         res.append(surl)
     res.sort()
     return res
diff --git a/lib/rucio/core/credential.py b/lib/rucio/core/credential.py
--- a/lib/rucio/core/credential.py
+++ b/lib/rucio/core/credential.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2018 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2012-2020 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 # Authors:
-# - Mario Lassnig <mario@lassnig.net>, 2018
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2018-2020
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 # - James Perry <j.perry@epcc.ed.ac.uk>, 2019
 #
@@ -115,10 +115,10 @@ def get_signed_url(rse_id, service, operation, url, lifetime=600):
         signature = urlencode({'': base64.b64encode(CREDS_GCS.sign_blob(to_sign)[1])})[1:]
 
         # assemble final signed URL
-        signed_url = 'https://storage.googleapis.com%s?GoogleAccessId=%s&Expires=%s&Signature=%s' % (path,
-                                                                                                     CREDS_GCS.service_account_email,
-                                                                                                     lifetime,
-                                                                                                     signature)
+        signed_url = 'https://storage.googleapis.com:443%s?GoogleAccessId=%s&Expires=%s&Signature=%s' % (path,
+                                                                                                         CREDS_GCS.service_account_email,
+                                                                                                         lifetime,
+                                                                                                         signature)
 
     elif service == 's3':
         # split URL to get hostname, bucket and key


Test patch:
diff --git a/lib/rucio/tests/test_credential.py b/lib/rucio/tests/test_credential.py
--- a/lib/rucio/tests/test_credential.py
+++ b/lib/rucio/tests/test_credential.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 # Authors:
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2018
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2018-2020
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - James Perry <j.perry@epcc.ed.ac.uk>, 2019
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2019
@@ -101,7 +101,7 @@ def test_sign_url_gcs(self):
         pytest.raises(UnsupportedOperation, get_signed_url, self.rse1_id, 'gcs', 'read', '')
 
         value = get_signed_url(self.rse1_id, 'gcs', 'read', 'http://storage/directory/file', lifetime=None)
-        expected = ('https://storage.googleapis.com/directory/file?GoogleAccessId=rucio-test@rucio-test'
+        expected = ('https://storage.googleapis.com:443/directory/file?GoogleAccessId=rucio-test@rucio-test'
                     '.iam.gserviceaccount.com&Expires=0&Signature=u9cBWowYX22sAyApH5YySD9h0m%2FbIPLHLgY'
                     '0Db%2BQ4a0wICQ2PZzUfTuHXQF8dUbMJG04VH90U5EMzYg3qSUGyfnp6Jptnvgivf7iSHepJsYhyAYSBGs'
                     'bvTOqf%2BXMQHR5VTh06G8WriZPV2OgSJ61c8qY7k8h0ju4bwcdDMFD2CT933KsnYSVatLN3EfORonLLZv'
@@ -110,7 +110,7 @@ def test_sign_url_gcs(self):
         assert value == expected
 
         value = get_signed_url(self.rse1_id, 'gcs', 'write', 'http://storage/directory/file', lifetime=None)
-        expected = ('https://storage.googleapis.com/directory/file?GoogleAccessId=rucio-test@rucio-test'
+        expected = ('https://storage.googleapis.com:443/directory/file?GoogleAccessId=rucio-test@rucio-test'
                     '.iam.gserviceaccount.com&Expires=0&Signature=Gn%2FL0%2FjGkBIdpHZ9bKw7tvqRCdslC11gt'
                     'jbLk5AG2jA4Ywd6mTvOinUB%2BZxHY2I3XzEuMfyMnFj0vfXSemN6XmmcQkiQBhl6P3zr0GrOuO4y0xjKT'
                     'am1MijMKLKFS9pZ6BBYrFgwKcYUcGJmVpq0Fo%2Bl5pLovBKhJbi3RE0YbGTCDA5UEM6WuWLMcQiY8smfK'
@@ -119,7 +119,7 @@ def test_sign_url_gcs(self):
         assert value == expected
 
         value = get_signed_url(self.rse1_id, 'gcs', 'delete', 'http://storage/directory/file', lifetime=None)
-        expected = ('https://storage.googleapis.com/directory/file?GoogleAccessId=rucio-test@rucio-test'
+        expected = ('https://storage.googleapis.com:443/directory/file?GoogleAccessId=rucio-test@rucio-test'
                     '.iam.gserviceaccount.com&Expires=0&Signature=FVDNroX1epdTCv%2BC74o%2B8uWyvJXrqiIWg'
                     'kdcedaOoryhRMjuv%2FVdKecnhViY%2BGOP%2B0CoI1uFOHBz%2B%2Bm10U9A3i%2B1v7AZRN5L6nbbS%2'
                     'BJTk4oiSBMJ3FpNT9knbOVd4aSPdiBwfTybwpkWSzEb8cKQsqzrGZk4hVffipMOKkxj7UgMe%2F0DiwqyF'