Repository: rucio/rucio

Issue:
Sorting of replicas in list_replicas
Motivation
----------
The `list_replicas` method orders the replicas, based on the client location, on the server side, but then puts them into a dictionary, hence the order of the replicas is lost. For each replica the dictionary should be expanded with an additional value (e.g. order) which represents the sorting order. This can be used by the client to order the replicas.


Discussion:
Actually, this is not done yet but got intermixed with the replica lan/wan issue. Reopening and moving to 1.15.4

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -757,6 +757,10 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns, schemes
                 if rse not in rse_info:
                     rse_info[rse] = rsemgr.get_rse_info(rse, session=session)
 
+                # assign scheme priorities
+                rse_info[rse]['priority_wan'] = {p['scheme']: p['domains']['wan']['read'] for p in rse_info[rse]['protocols']}
+                rse_info[rse]['priority_lan'] = {p['scheme']: p['domains']['lan']['read'] for p in rse_info[rse]['protocols']}
+
                 # select the lan door in autoselect mode, otherwise use the wan door
                 if domain is None:
                     domain = 'wan'
@@ -789,16 +793,19 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns, schemes
                                 protocols.append(('lan', rsemgr.create_protocol(rse_settings=rse_info[rse],
                                                                                 operation='read',
                                                                                 scheme=s,
-                                                                                domain='lan')))
+                                                                                domain='lan'),
+                                                  rse_info[rse]['priority_lan'][s]))
                                 protocols.append(('wan', rsemgr.create_protocol(rse_settings=rse_info[rse],
                                                                                 operation='read',
                                                                                 scheme=s,
-                                                                                domain='wan')))
+                                                                                domain='wan'),
+                                                  rse_info[rse]['priority_wan'][s]))
                             else:
                                 protocols.append((domain, rsemgr.create_protocol(rse_settings=rse_info[rse],
                                                                                  operation='read',
                                                                                  scheme=s,
-                                                                                 domain=domain)))
+                                                                                 domain=domain),
+                                                  rse_info[rse]['priority_%s' % domain][s]))
                         except exception.RSEProtocolNotSupported:
                             pass  # no need to be verbose
                         except:
@@ -845,10 +852,10 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns, schemes
                             if sign and isinstance(sign, list) and sign[0]:
                                 pfn = get_signed_url(service='gcs', operation='read', url=pfn, lifetime=signature_lifetime)
 
-                        # TODO: this is not nice, but since pfns don't have the concept of 'domain'
+                        # TODO: this is not nice, but since pfns don't have the concept of 'domain' or 'priority'
                         #       we can work around by encapsulating it in a tuple. a proper refactor requires
                         #       far-reaching changes in the rsemgr
-                        pfns.append((tmp_protocol[0], pfn))
+                        pfns.append((pfn, tmp_protocol[0], tmp_protocol[2]))
                     except:
                         # temporary protection
                         print format_exc()
@@ -861,13 +868,14 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns, schemes
 
             if 'scope' in file and 'name' in file:
                 if file['scope'] == scope and file['name'] == name:
-                    file['rses'][rse] += list(set([tmp_pfn[1] for tmp_pfn in pfns]))  # extract properly the pfn from the (domain, pfn)
+                    file['rses'][rse] += list(set([tmp_pfn[0] for tmp_pfn in pfns]))  # extract properly the pfn from the (pfn, domain, priority)
                     file['states'][rse] = str(state)
                     for tmp_pfn in pfns:
-                        file['pfns'][tmp_pfn[1]] = {'rse': rse,
+                        file['pfns'][tmp_pfn[0]] = {'rse': rse,
                                                     'type': str(rse_type),
                                                     'volatile': volatile,
-                                                    'domain': tmp_pfn[0]}  # extract properly the domain from the (domain, pfn)
+                                                    'domain': tmp_pfn[1],
+                                                    'priority': tmp_pfn[2]}
                 else:
                     yield file
                     file = {}
@@ -878,12 +886,13 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns, schemes
                         'pfns': {}, 'rses': defaultdict(list),
                         'states': {rse: str(state)}}
                 if rse:
-                    file['rses'][rse] = list(set([tmp_pfn[1] for tmp_pfn in pfns]))  # extract properly the pfn from the (domain, pfn)
+                    file['rses'][rse] = list(set([tmp_pfn[0] for tmp_pfn in pfns]))  # extract properly the pfn from the (pfn, domain, priority)
                     for tmp_pfn in pfns:
-                        file['pfns'][tmp_pfn[1]] = {'rse': rse,
+                        file['pfns'][tmp_pfn[0]] = {'rse': rse,
                                                     'type': str(rse_type),
                                                     'volatile': volatile,
-                                                    'domain': tmp_pfn[0]}  # extract properly the domain from the (domain, pfn)
+                                                    'domain': tmp_pfn[1],
+                                                    'priority': tmp_pfn[2]}
 
     if 'scope' in file and 'name' in file:
         yield file
diff --git a/lib/rucio/web/rest/webpy/v1/replica.py b/lib/rucio/web/rest/webpy/v1/replica.py
--- a/lib/rucio/web/rest/webpy/v1/replica.py
+++ b/lib/rucio/web/rest/webpy/v1/replica.py
@@ -348,16 +348,27 @@ def POST(self):
                                        client_location=client_location,
                                        domain=domain, signature_lifetime=signature_lifetime,
                                        issuer=ctx.env.get('issuer')):
-                replicas = []
-                dictreplica = {}
-                for rse in rfile['rses']:
-                    for replica in rfile['rses'][rse]:
-                        replicas.append(replica)
-                        dictreplica[replica] = rse
 
                 if not metalink:
                     yield dumps(rfile, cls=APIEncoder) + '\n'
                 else:
+
+                    replicas = []
+                    dictreplica = {}
+                    for rse in rfile['rses']:
+                        for replica in rfile['rses'][rse]:
+                            replicas.append(replica)
+                            dictreplica[replica] = (rfile['pfns'][replica]['domain'], rfile['pfns'][replica]['priority'], rse)
+
+                    # we need to sort first by domain
+                    # --> exploit that L(AN) comes before W(AN) in the alphabet :-D
+                    tmp = sorted([(dictreplica[r] + (r,)) for r in dictreplica])
+
+                    # then rewrite priorities in ascending order starting from 1 across both domains to be correct metalink
+                    dictreplica = {}
+                    for i in xrange(len(tmp)):
+                        dictreplica[tmp[i][3]] = (tmp[i][0], i + 1, tmp[i][2])
+
                     yield ' <file name="' + rfile['name'] + '">\n'
                     yield '  <identity>' + rfile['scope'] + ':' + rfile['name'] + '</identity>\n'
                     if rfile['adler32'] is not None:
@@ -369,6 +380,7 @@ def POST(self):
                     yield '  <glfn name="/atlas/rucio/%s:%s">' % (rfile['scope'], rfile['name'])
                     yield '</glfn>\n'
 
+                    # TODO: deprecate this
                     if select == 'geoip':
                         replicas = sort_geoip(dictreplica, client_location['ip'])
                     elif select == 'closeness':
@@ -377,12 +389,17 @@ def POST(self):
                         replicas = sort_dynamic(dictreplica, client_location)
                     elif select == 'ranking':
                         replicas = sort_ranking(dictreplica, client_location)
-                    else:
+                    elif select == 'random':
                         replicas = sort_random(dictreplica)
+                    else:
+                        replicas = sorted(dictreplica, key=dictreplica.get)
 
                     idx = 0
                     for replica in replicas:
-                        yield '   <url location="' + str(dictreplica[replica]) + '" priority="' + str(idx + 1) + '">' + replica + '</url>\n'
+                        yield '   <url location="' + str(dictreplica[replica][2]) \
+                            + '" domain="' + str(dictreplica[replica][0]) \
+                            + '" priority="' + str(dictreplica[replica][1]) \
+                            + '">' + replica + '</url>\n'
                         idx += 1
                         if limit and limit == idx:
                             break


Test patch:
diff --git a/lib/rucio/tests/test_replica_sorting.py b/lib/rucio/tests/test_replica_sorting.py
new file mode 100644
--- /dev/null
+++ b/lib/rucio/tests/test_replica_sorting.py
@@ -0,0 +1,195 @@
+# Copyright 2017-2018 CERN for the benefit of the ATLAS collaboration.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2018
+
+from nose.tools import assert_equal, assert_in, assert_not_in
+
+from rucio.client import ReplicaClient
+from rucio.core.replica import add_replicas, delete_replicas
+from rucio.core.rse import add_rse, del_rse, add_rse_attribute, add_protocol
+from rucio.tests.common import rse_name_generator
+
+
+class TestReplicaSorting(object):
+
+    def test_replica_sorting(self):
+        """ REPLICA (CORE): Test the correct sorting of the replicas across WAN and LAN """
+
+        self.rc = ReplicaClient()
+
+        self.rse1 = 'APERTURE_%s' % rse_name_generator()
+        self.rse2 = 'BLACKMESA_%s' % rse_name_generator()
+        add_rse(self.rse1)
+        add_rse(self.rse2)
+        add_rse_attribute(rse=self.rse1, key='site', value='APERTURE')
+        add_rse_attribute(rse=self.rse2, key='site', value='BLACKMESA')
+
+        self.files = [{'scope': 'mock', 'name': 'element_0',
+                       'bytes': 1234, 'adler32': 'deadbeef'}]
+        add_replicas(rse=self.rse1, files=self.files, account='root')
+        add_replicas(rse=self.rse2, files=self.files, account='root')
+
+        add_protocol(self.rse1, {'scheme': 'root',
+                                 'hostname': 'root.aperture.com',
+                                 'port': 1409,
+                                 'prefix': '//test/chamber/',
+                                 'impl': 'rucio.rse.protocols.xrootd.Default',
+                                 'domains': {
+                                     'lan': {'read': 1, 'write': 1, 'delete': 1},
+                                     'wan': {'read': 1, 'write': 1, 'delete': 1}}})
+        add_protocol(self.rse1, {'scheme': 'davs',
+                                 'hostname': 'davs.aperture.com',
+                                 'port': 443,
+                                 'prefix': '/test/chamber/',
+                                 'impl': 'rucio.rse.protocols.gfal.Default',
+                                 'domains': {
+                                     'lan': {'read': 2, 'write': 2, 'delete': 2},
+                                     'wan': {'read': 2, 'write': 2, 'delete': 2}}})
+        add_protocol(self.rse1, {'scheme': 'gsiftp',
+                                 'hostname': 'gsiftp.aperture.com',
+                                 'port': 8446,
+                                 'prefix': '/test/chamber/',
+                                 'impl': 'rucio.rse.protocols.gfal.Default',
+                                 'domains': {
+                                     'lan': {'read': 0, 'write': 0, 'delete': 0},
+                                     'wan': {'read': 3, 'write': 3, 'delete': 3}}})
+
+        add_protocol(self.rse2, {'scheme': 'gsiftp',
+                                 'hostname': 'gsiftp.blackmesa.com',
+                                 'port': 8446,
+                                 'prefix': '/lambda/complex/',
+                                 'impl': 'rucio.rse.protocols.gfal.Default',
+                                 'domains': {
+                                     'lan': {'read': 2, 'write': 2, 'delete': 2},
+                                     'wan': {'read': 1, 'write': 1, 'delete': 1}}})
+        add_protocol(self.rse2, {'scheme': 'davs',
+                                 'hostname': 'davs.blackmesa.com',
+                                 'port': 443,
+                                 'prefix': '/lambda/complex/',
+                                 'impl': 'rucio.rse.protocols.gfal.Default',
+                                 'domains': {
+                                     'lan': {'read': 0, 'write': 0, 'delete': 0},
+                                     'wan': {'read': 2, 'write': 2, 'delete': 2}}})
+        add_protocol(self.rse2, {'scheme': 'root',
+                                 'hostname': 'root.blackmesa.com',
+                                 'port': 1409,
+                                 'prefix': '//lambda/complex/',
+                                 'impl': 'rucio.rse.protocols.xrootd.Default',
+                                 'domains': {
+                                     'lan': {'read': 1, 'write': 1, 'delete': 1},
+                                     'wan': {'read': 3, 'write': 3, 'delete': 3}}})
+
+        replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock',
+                                                            'name': f['name'],
+                                                            'type': 'FILE'} for f in self.files],
+                                                     schemes=['root', 'gsiftp', 'davs'],
+                                                     client_location={'site': 'APERTURE'})]
+        pfns = [r['pfns'] for r in replicas][0]
+        assert_equal(len(pfns.keys()), 5)
+        assert_equal(pfns['davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']['domain'], 'lan')
+        assert_equal(pfns['davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']['priority'], 2)
+        assert_equal(pfns['davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0']['domain'], 'wan')
+        assert_equal(pfns['davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0']['priority'], 2)
+        assert_equal(pfns['gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']['domain'], 'wan')
+        assert_equal(pfns['gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']['priority'], 1)
+        assert_equal(pfns['root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']['domain'], 'lan')
+        assert_equal(pfns['root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']['priority'], 1)
+        assert_equal(pfns['root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']['domain'], 'wan')
+        assert_equal(pfns['root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']['priority'], 3)
+
+        replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock',
+                                                            'name': f['name'],
+                                                            'type': 'FILE'} for f in self.files],
+                                                     schemes=['root', 'gsiftp', 'davs'],
+                                                     client_location={'site': 'BLACKMESA'})]
+        pfns = [r['pfns'] for r in replicas][0]
+        assert_equal(len(pfns.keys()), 5)
+        assert_equal(pfns['davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']['domain'], 'wan')
+        assert_equal(pfns['davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']['priority'], 2)
+        assert_equal(pfns['gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0']['domain'], 'wan')
+        assert_equal(pfns['gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0']['priority'], 3)
+        assert_equal(pfns['gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']['domain'], 'lan')
+        assert_equal(pfns['gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']['priority'], 2)
+        assert_equal(pfns['root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']['domain'], 'wan')
+        assert_equal(pfns['root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']['priority'], 1)
+        assert_equal(pfns['root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']['domain'], 'lan')
+        assert_equal(pfns['root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']['priority'], 1)
+
+        replicas = [r for r in self.rc.list_replicas(dids=[{'scope': 'mock',
+                                                            'name': f['name'],
+                                                            'type': 'FILE'} for f in self.files],
+                                                     schemes=['root', 'gsiftp', 'davs'],
+                                                     client_location={'site': 'XEN'})]
+        pfns = [r['pfns'] for r in replicas][0]
+        assert_equal(len(pfns.keys()), 6)
+        assert_equal(pfns['davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']['domain'], 'wan')
+        assert_equal(pfns['davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0']['priority'], 2)
+        assert_equal(pfns['davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0']['domain'], 'wan')
+        assert_equal(pfns['davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0']['priority'], 2)
+        assert_equal(pfns['gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0']['domain'], 'wan')
+        assert_equal(pfns['gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0']['priority'], 3)
+        assert_equal(pfns['gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']['domain'], 'wan')
+        assert_equal(pfns['gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0']['priority'], 1)
+        assert_equal(pfns['root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']['domain'], 'wan')
+        assert_equal(pfns['root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0']['priority'], 1)
+        assert_equal(pfns['root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']['domain'], 'wan')
+        assert_equal(pfns['root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0']['priority'], 3)
+
+        ml = self.rc.list_replicas(dids=[{'scope': 'mock',
+                                          'name': f['name'],
+                                          'type': 'FILE'} for f in self.files],
+                                   schemes=['root', 'gsiftp', 'davs'],
+                                   metalink=True,
+                                   client_location={'site': 'APERTURE'})
+        assert_in('domain="lan" priority="1">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0', ml)
+        assert_in('domain="lan" priority="2">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="3">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="4">davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="5">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0', ml)
+        assert_not_in('priority="6"', ml)
+
+        ml = self.rc.list_replicas(dids=[{'scope': 'mock',
+                                          'name': f['name'],
+                                          'type': 'FILE'} for f in self.files],
+                                   schemes=['root', 'gsiftp', 'davs'],
+                                   metalink=True,
+                                   client_location={'site': 'BLACKMESA'})
+        assert_in('domain="lan" priority="1">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0', ml)
+        assert_in('domain="lan" priority="2">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="3">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="4">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="5">gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0', ml)
+        assert_not_in('priority="6"', ml)
+
+        ml = self.rc.list_replicas(dids=[{'scope': 'mock',
+                                          'name': f['name'],
+                                          'type': 'FILE'} for f in self.files],
+                                   schemes=['root', 'gsiftp', 'davs'],
+                                   metalink=True,
+                                   client_location={'site': 'XEN'})
+
+        assert_in('domain="wan" priority="1">root://root.aperture.com:1409//test/chamber/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="2">gsiftp://gsiftp.blackmesa.com:8446/lambda/complex/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="3">davs://davs.aperture.com:443/test/chamber/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="4">davs://davs.blackmesa.com:443/lambda/complex/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="5">gsiftp://gsiftp.aperture.com:8446/test/chamber/mock/58/b5/element_0', ml)
+        assert_in('domain="wan" priority="6">root://root.blackmesa.com:1409//lambda/complex/mock/58/b5/element_0', ml)
+        assert_not_in('priority="7"', ml)
+
+        delete_replicas(rse=self.rse1, files=self.files)
+        delete_replicas(rse=self.rse2, files=self.files)
+        del_rse(self.rse1)
+        del_rse(self.rse2)