Repository: rucio/rucio

Issue:
Sorting of replicas in list_replicas
Motivation
----------
The `list_replicas` method orders the replicas, based on the client location, on the server side, but then puts them into a dictionary, hence the order of the replicas is lost. For each replica the dictionary should be expanded with an additional value (e.g. order) which represents the sorting order. This can be used by the client to order the replicas.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -1,21 +1,26 @@
-"""
-  Copyright European Organization for Nuclear Research (CERN)
-
-  Licensed under the Apache License, Version 2.0 (the "License");
-  You may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-  http://www.apache.org/licenses/LICENSE-2.0
-
-  Authors:
-  - Vincent Garonne, <vincent.garonne@cern.ch>, 2013-2017
-  - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2018
-  - Ralph Vigne <ralph.vigne@cern.ch>, 2013-2014
-  - Martin Barisits <martin.barisits@cern.ch>, 2013-2016
-  - Mario Lassnig <mario.lassnig@cern.ch>, 2014-2018
-  - David Cameron <d.g.cameron@gmail.com>, 2014
-  - Thomas Beermann <thomas.beermann@cern.ch>, 2014-2016
-  - Wen Guan, <wen.guan@cern.ch>, 2014-2015
-"""
+# Copyright 2013-2018 CERN for the benefit of the ATLAS collaboration.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Vincent Garonne <vgaronne@gmail.com>, 2013-2018
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2018
+# - Ralph Vigne <ralph.vigne@cern.ch>, 2013-2014
+# - Martin Barisits <martin.barisits@cern.ch>, 2013-2018
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2014-2018
+# - David Cameron <d.g.cameron@gmail.com>, 2014
+# - Thomas Beermann <thomas.beermann@cern.ch>, 2014-2016
+# - Wen Guan <wguan.icedew@gmail.com>, 2014-2015
 
 from collections import defaultdict
 from curses.ascii import isprint
@@ -724,6 +729,15 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns, schemes
     files = [dataset_clause and _list_replicas_for_datasets(dataset_clause, state_clause, rse_clause, session),
              file_clause and _list_replicas_for_files(file_clause, state_clause, files, rse_clause, session)]
 
+    # find all RSEs local to the client's location in autoselect mode (i.e., when domain is None)
+    local_rses = []
+    if domain is None:
+        if client_location and 'site' in client_location and client_location['site']:
+            try:
+                local_rses = [rse['rse'] for rse in parse_expression('site=%s' % client_location['site'], session=session)]
+            except:
+                pass  # do not hard fail if site cannot be resolved or is empty
+
     file, tmp_protocols, rse_info, pfns_cache = {}, {}, {}, {}
     for replicas in filter(None, files):
         for scope, name, bytes, md5, adler32, path, state, rse, rse_type, volatile in replicas:
@@ -734,6 +748,12 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns, schemes
                 if rse not in rse_info:
                     rse_info[rse] = rsemgr.get_rse_info(rse, session=session)
 
+                # select the lan door in autoselect mode, otherwise use the wan door
+                if domain is None:
+                    domain = 'wan'
+                    if local_rses and rse in local_rses:
+                        domain = 'lan'
+
                 if rse not in tmp_protocols:
 
                     rse_schemes = schemes or []
@@ -796,7 +816,7 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns, schemes
                         # server side root proxy handling if location is set.
                         # cannot be pushed into protocols because we need to lookup rse attributes.
                         # ultra-conservative implementation.
-                        if protocol.attributes['scheme'] == 'root' and client_location:
+                        if domain == 'wan' and protocol.attributes['scheme'] == 'root' and client_location:
                             if 'site' in client_location and client_location['site']:
                                 replica_site = get_rse_attribute('site', rse_info[rse]['id'], session=session)[0]
                                 if client_location['site'] != replica_site:
@@ -868,14 +888,10 @@ def list_replicas(dids, schemes=None, unavailable=False, request_id=None,
     :param all_states: Return all replicas whatever state they are in. Adds an extra 'states' entry in the result dictionary.
     :param rse_expression: The RSE expression to restrict list_replicas on a set of RSEs.
     :param client_location: Client location dictionary for PFN modification {'ip', 'fqdn', 'site'}
-    :param domain: The network domain for the call, either None, 'wan' or 'lan'. None is fallback to 'wan', 'all' is both ['lan','wan']
+    :param domain: The network domain for the call, either None, 'wan' or 'lan'. None is automatic mode, 'all' is both ['lan','wan']
     :param session: The database session in use.
     """
 
-    # Compatibility fallback: Old clients expect WAN replicas always
-    if not domain:
-        domain = 'wan'
-
     file_clause, dataset_clause, state_clause, files = _resolve_dids(dids=dids, unavailable=unavailable,
                                                                      ignore_availability=ignore_availability,
                                                                      all_states=all_states, session=session)


Test patch:
diff --git a/lib/rucio/tests/test_replica.py b/lib/rucio/tests/test_replica.py
--- a/lib/rucio/tests/test_replica.py
+++ b/lib/rucio/tests/test_replica.py
@@ -1,16 +1,25 @@
-# Copyright European Organization for Nuclear Research (CERN)
+# Copyright 2013-2018 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# You may not use this file except in compliance with the License.
+# you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# http://www.apache.org/licenses/LICENSE-2.0
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # Authors:
-# - Vincent Garonne, <vincent.garonne@cern.ch>, 2013-2015
-# - Mario Lassnig, <mario.lassnig@cern.ch>, 2013-2014, 2016-2018
-# - Cedric Serfon, <cedric.serfon@cern.ch>, 2014-2016
-# - Thomas Beermann, <thomas.beermann@cern.ch>, 2014
-# - Joaquin Bogado, <jbogado@linti.unlp.edu.ar>, 2018
+# - Vincent Garonne <vgaronne@gmail.com>, 2013-2017
+# - Ralph Vigne <ralph.vigne@cern.ch>, 2013-2014
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2018
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2016
+# - Thomas Beermann <thomas.beermann@cern.ch>, 2014
+# - Martin Barisits <martin.barisits@cern.ch>, 2015-2018
+# - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
 
 from __future__ import print_function
 
@@ -312,21 +321,21 @@ def test_list_replica_with_domain(self):
 
         # # force all LAN
         for replica in list_replicas(dids=[{'scope': f['scope'], 'name': f['name'], 'type': DIDType.FILE} for f in files],
-                                     schemes=['MOCK']):
+                                     schemes=['MOCK'], domain='lan'):
             cmd = 'rucio list-file-replicas --pfns --domain=lan %s:%s' % (replica['scope'], replica['name'])
             errno, stdout, stderr = execute(cmd)
             assert_in('/i/prefer/the/lan', stdout)
 
         # # force all WAN
         for replica in list_replicas(dids=[{'scope': f['scope'], 'name': f['name'], 'type': DIDType.FILE} for f in files],
-                                     schemes=['MOCK']):
+                                     schemes=['MOCK'], domain='wan'):
             cmd = 'rucio list-file-replicas --pfns --domain=wan %s:%s' % (replica['scope'], replica['name'])
             errno, stdout, stderr = execute(cmd)
             assert_in('/i/prefer/the/wan', stdout)
 
         # # force both WAN and LAN
         for replica in list_replicas(dids=[{'scope': f['scope'], 'name': f['name'], 'type': DIDType.FILE} for f in files],
-                                     schemes=['MOCK']):
+                                     schemes=['MOCK'], domain='all'):
             cmd = 'rucio list-file-replicas --pfns --domain=all %s:%s' % (replica['scope'], replica['name'])
             errno, stdout, stderr = execute(cmd)
             assert_in('/i/prefer/the/wan', stdout)