Repository: rucio/rucio

Issue:
Sorting of replicas with list_replicas in non metalink case
Motivation
----------

Currently replicas are only sorted (geoip, etc.) if the `list_replicas` request is made with `metalink=True`

https://github.com/rucio/rucio/blob/92474868cec7d64b83b46b451ecdc4a294ef2711/lib/rucio/web/rest/flaskapi/v1/replicas.py#L119

This is unfortunately a bit inconsistent, and also for the json return it should be sorted.

Modification
------------
The issue with the json return is that it is not a list, but a dictionary with `priority` values. These need to be updated by the function. (The historic `rses` field in the return is not sortable, only the `pfns` one is)


Discussion:

Resolving patch:
diff --git a/lib/rucio/web/rest/flaskapi/v1/replicas.py b/lib/rucio/web/rest/flaskapi/v1/replicas.py
--- a/lib/rucio/web/rest/flaskapi/v1/replicas.py
+++ b/lib/rucio/web/rest/flaskapi/v1/replicas.py
@@ -18,7 +18,9 @@
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2021
 # - Eric Vaandering <ewv@fnal.gov>, 2021
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
+# - Ilija Vukotic <ivukotic@cern.ch>, 2021
 # - Martin Barisits <martin.barisits@cern.ch>, 2021
+# - Joel Dierkes <joel.dierkes@cern.ch>, 2021
 
 from datetime import datetime
 from itertools import chain
@@ -44,6 +46,20 @@
     request_auth_env, response_headers, generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get
 
 
+def _sort_pfns_by_priority(pfns, sorted_pfns):
+    """
+    Sets the corresponding priority values for the given list of pfns and yields
+    the index of the element as well as the element.
+
+    :param pfns: List with the pfns which priorities should be set.
+    :param sorted_pfns: Sorted list of pfns.
+    :yields: index and corresponding pfn
+    """
+    for idx, pfn in enumerate(sorted_pfns, start=1):
+        pfns[pfn]['priority'] = idx
+        yield idx, pfn
+
+
 class Replicas(ErrorHandlingMethodView):
 
     @check_accept_header_wrapper_flask(['application/x-json-stream', 'application/metalink4+xml'])
@@ -116,6 +132,7 @@ def generate(vo):
                     replicas = sort_replicas(dictreplica, client_location, selection=select)
 
                     if not metalink:
+                        _ = list(_sort_pfns_by_priority(rfile['pfns'], replicas))
                         yield dumps(rfile) + '\n'
                     else:
                         yield ' <file name="' + rfile['name'] + '">\n'
@@ -364,15 +381,10 @@ def generate(request_id, issuer, vo):
                             lanreplicas[pfn] = replica_tuple
                         else:
                             wanreplicas[pfn] = replica_tuple
-                    unsorted_replicas = rfile['pfns']
-                    sorted_replicas = rfile['pfns'] = {}
                     # Lan replicas sorted by priority; followed by wan replicas sorted by selection criteria
-                    for idx, pfn in enumerate(chain(sorted(lanreplicas.keys(), key=lambda pfn: lanreplicas[pfn][1]),
-                                                    sort_replicas(wanreplicas, client_location, selection=select)),
-                                              start=1):
-                        replica = unsorted_replicas[pfn]
-                        sorted_replicas[pfn] = replica
-                        replica['priority'] = idx
+                    for idx, pfn in _sort_pfns_by_priority(rfile['pfns'],
+                                                           chain(sorted(lanreplicas.keys(), key=lambda pfn: lanreplicas[pfn][1]),
+                                                                 sort_replicas(wanreplicas, client_location, selection=select))):
                         if limit and limit == idx:
                             break
 


Test patch:
diff --git a/lib/rucio/tests/test_replica_sorting.py b/lib/rucio/tests/test_replica_sorting.py
--- a/lib/rucio/tests/test_replica_sorting.py
+++ b/lib/rucio/tests/test_replica_sorting.py
@@ -19,6 +19,8 @@
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
+# - Joel Dierkes <joel.dierkes@cern.ch>, 2021
 
 import copy
 import json
@@ -352,3 +354,33 @@ def fake_get_geoip_db(*args, **kwargs):
         assert replicas_response
 
         get_geoip_db_mock.assert_called()
+
+
+@pytest.mark.noparallel(reason='fails when run in parallel, replicas should not be changed')
+def test_get_sorted_list_replicas_no_metalink(vo, rest_client, auth_token, protocols_setup, mock_scope):
+    """Replicas: gets the json list replicas and checks if its sorted"""
+
+    global replica_singleton
+    replica_singleton = None
+
+    def _reverse_geoip(dictreplica, client_location, ignore_error=False):
+        global replica_singleton
+        if replica_singleton is None:
+            replica_singleton = list(dictreplica.keys())
+            return replica_singleton
+        replica_singleton.reverse()
+        return replica_singleton
+
+    def _extract_priorities(data):
+        return {k: v['priority'] for k, v in data['pfns'].items()}
+
+    def get_replicas():
+        return parse_replicas_from_string(rest_client.get(
+            '/replicas/%s/%s?select=geoip' % (mock_scope.external, protocols_setup['files'][0]['name']),
+            headers=headers(auth(auth_token), vohdr(vo), accept(Mime.JSON_STREAM))
+        ).get_data(as_text=True))
+
+    with mock.patch('rucio.core.replica_sorter.sort_geoip', side_effect=_reverse_geoip):
+        initial_priorities = _extract_priorities(get_replicas())
+        updated_priorities = _extract_priorities(get_replicas())
+        assert initial_priorities != updated_priorities, "The replica list is not sorted according to the priorities."