Repository: rucio/rucio

Issue:
Support site attributes for root proxies
Motivation
----------

Storageless sites still need to be integrated with Rucio and require special treatment. Right now the attributes required for many replica lookup operations are available on the RSE level only. For sites without RSEs this functionality is thus lost.

Modification
----------

Introduce a new SiteAttributes model, and extend the replica core to handle it properly.


Discussion:
After some discussion, the approach will use the `core.config` module.

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -48,8 +48,9 @@
 
 from rucio.common import exception
 from rucio.common.utils import chunks, clean_surls, str_to_date, add_url_query
+from rucio.core.config import get as config_get
 from rucio.core.credential import get_signed_url
-from rucio.core.rse import get_rse, get_rse_id, get_rse_name, get_rse_attribute, get_rses_with_attribute_value
+from rucio.core.rse import get_rse, get_rse_id, get_rse_name, get_rse_attribute
 from rucio.core.rse_counter import decrease, increase
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.db.sqla import models
@@ -958,14 +959,15 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns,
                                 if isinstance(rse_site_attr, list) and rse_site_attr:
                                     replica_site = rse_site_attr[0]
 
-                                # does it match with the client?
+                                # does it match with the client? if not, it's an outgoing connection
+                                # therefore the internal proxy must be prepended
                                 if client_location['site'] != replica_site:
-                                    root_proxy_internal = get_rses_with_attribute_value('site', client_location['site'],
-                                                                                        'root-proxy-internal',
-                                                                                        session=session)
-                                    # assume all RSEs at site have same proxy, just prepend the first one
-                                    if root_proxy_internal and 'value' in root_proxy_internal[0]:
-                                        pfn = root_proxy_internal[0]['value'] + '//' + pfn
+                                    root_proxy_internal = config_get('root-proxy-internal',    # section
+                                                                     client_location['site'],  # option
+                                                                     default='',               # empty string to circumvent exception
+                                                                     session=session)
+                                    if root_proxy_internal:
+                                        pfn = 'root://' + root_proxy_internal + '//' + pfn
 
                         # do we need to sign the URLs?
                         if sign_urls and protocol.attributes['scheme'] == 'https':


Test patch:
diff --git a/lib/rucio/tests/test_root_proxy.py b/lib/rucio/tests/test_root_proxy.py
--- a/lib/rucio/tests/test_root_proxy.py
+++ b/lib/rucio/tests/test_root_proxy.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 # Authors:
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2017-2018
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2017-2019
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2019
 #
@@ -30,6 +30,7 @@
 from paste.fixture import TestApp
 
 from rucio.client import ReplicaClient
+from rucio.core.config import set as config_set
 from rucio.core.replica import add_replicas, delete_replicas
 from rucio.core.rse import add_rse, add_rse_attribute, del_rse, add_protocol
 from rucio.tests.common import rse_name_generator
@@ -56,13 +57,13 @@ def setup(self):
                                            'site': 'APERTURE1'}
         self.rse_with_proxy = rse_name_generator()
         add_rse(self.rse_with_proxy)
-        add_rse_attribute(rse=self.rse_with_proxy,
-                          key='root-proxy-internal',
-                          value='root://proxy.aperture.com:1094')
         add_rse_attribute(rse=self.rse_with_proxy,
                           key='site',
                           value='APERTURE1')
 
+        # APERTURE1 site has an internal proxy
+        config_set('root-proxy-internal', 'APERTURE1', 'proxy.aperture.com:1094')
+
         self.files = [{'scope': 'mock',
                        'name': 'half-life_%s' % i,
                        'bytes': 1234,