Repository: rucio/rucio
Issue: The replica sorting ignores the protocol priority

Motivation
----------

As reported by @rodwalker, the Rucio client does not necessarily honour the read_wan protocol priority.

Going through the implementation, in one example the value of `dictreplica` is:
```
{
  'srm://gse1.physik.uni-mainz.de:8444/srm/managerv2?SFN=/atlas/atlaslocalgroupdisk/rucio/mc15_13TeV/f7/e8/DAOD_TRUTH1.09608067._000002.pool.root.1': ('wan', 2, 'MAINZ_LOCALGROUPDISK', False),
  'davs://gdav1.physik.uni-mainz.de:8443/atlas/atlaslocalgroupdisk/rucio/mc15_13TeV/f7/e8/DAOD_TRUTH1.09608067._000002.pool.root.1': ('wan', 1, 'MAINZ_LOCALGROUPDISK', False)
}
```
but `sort_geoip` returns:
```
[
  'srm://gse1.physik.uni-mainz.de:8444/srm/managerv2?SFN=/atlas/atlaslocalgroupdisk/rucio/mc15_13TeV/f7/e8/DAOD_TRUTH1.09608067._000002.pool.root.1',
  'davs://gdav1.physik.uni-mainz.de:8443/atlas/atlaslocalgroupdisk/rucio/mc15_13TeV/f7/e8/DAOD_TRUTH1.09608067._000002.pool.root.1'
]
```
Similarly, with no sorting or with the yet-to-be-impemented methods, the order of the replicas is how Python returns `dictreplicas.keys()`.

Modification
------------




Changed files:
lib/rucio/web/rest/webpy/v1/replica.py, lib/rucio/core/replica_sorter.py, lib/rucio/web/rest/flaskapi/v1/replicas.py

diff --git a/lib/rucio/core/replica_sorter.py b/lib/rucio/core/replica_sorter.py
index 07f5c4bc2..6209cfeb6 100644
--- a/lib/rucio/core/replica_sorter.py
+++ b/lib/rucio/core/replica_sorter.py
@@ -33,6 +33,7 @@ import random
 import socket
 import tarfile
 import time
+from collections import OrderedDict
 from math import asin, cos, radians, sin, sqrt
 from typing import TYPE_CHECKING
 from urllib.parse import urlparse
@@ -179,9 +180,19 @@ def sort_replicas(dictreplica: "Dict", client_location: "Dict", selection: "Opti
     :param default: the default sorting algorithm (random, if not defined).
     :returns: the keys of dictreplica in a sorted list.
     """
+    if len(dictreplica) == 0:
+        return []
     if not selection:
         selection = 'geoip'
 
+    items = [(key, value) for key, value in dictreplica.items()]
+    # safety check, TODO: remove if all dictreplica values are 4-tuple with priority as second item
+    if isinstance(items[0][1], tuple) and len(items[0][1]) == 4:
+        # sort by value[1], which is the priority
+        items.sort(key=lambda item: item[1][1])
+    dictreplica = OrderedDict(items)
+
+    # all sorts must be stable to preserve the priority (the Python standard sorting functions always are stable)
     if selection == 'geoip':
         replicas = sort_geoip(dictreplica, client_location, ignore_error=True)
     elif selection == 'closeness':
diff --git a/lib/rucio/web/rest/flaskapi/v1/replicas.py b/lib/rucio/web/rest/flaskapi/v1/replicas.py
index 270e1b5d9..182737e7b 100644
--- a/lib/rucio/web/rest/flaskapi/v1/replicas.py
+++ b/lib/rucio/web/rest/flaskapi/v1/replicas.py
@@ -134,10 +134,8 @@ class Replicas(ErrorHandlingMethodView):
                         yield f'  <glfn name="/atlas/rucio/{rfile["scope"]}:{rfile["name"]}">'
                         yield '</glfn>\\n'
 
-                        idx = 0
-                        for replica in replicas:
-                            yield '   <url location="' + str(dictreplica[replica]) + '" priority="' + str(idx + 1) + '">' + escape(replica) + '</url>\\n'
-                            idx += 1
+                        for idx, replica in enumerate(replicas, start=1):
+                            yield '   <url location="' + str(dictreplica[replica]) + '" priority="' + str(idx) + '">' + escape(replica) + '</url>\\n'
                             if limit and limit == idx:
                                 break
                         yield ' </file>\\n'
@@ -389,10 +387,11 @@ class ListReplicas(ErrorHandlingMethodView):
                         yield f'  <glfn name="/{policy_schema}/rucio/{rfile["scope"]}:{rfile["name"]}"></glfn>\\n'
 
                         lanreplicas = [replica for replica, v in dictreplica.items() if v[0] == 'lan']
+                        # sort lan by priority
+                        lanreplicas.sort(key=lambda rep: dictreplica[rep][1])
                         replicas = lanreplicas + sort_replicas({k: v for k, v in dictreplica.items() if v[0] != 'lan'}, client_location, selection=select)
 
-                        idx = 1
-                        for replica in replicas:
+                        for idx, replica in enumerate(replicas, start=1):
                             yield '  <url location="' + str(dictreplica[replica][2]) \\
                                 + '" domain="' + str(dictreplica[replica][0]) \\
                                 + '" priority="' + str(idx) \\
@@ -400,7 +399,6 @@ class ListReplicas(ErrorHandlingMethodView):
                                 + '">' + escape(replica) + '</url>\\n'
                             if limit and limit == idx:
                                 break
-                            idx += 1
                         yield ' </file>\\n'
 
                 if metalink:
diff --git a/lib/rucio/web/rest/webpy/v1/replica.py b/lib/rucio/web/rest/webpy/v1/replica.py
index 0e034702d..1b9152bd3 100755
--- a/lib/rucio/web/rest/webpy/v1/replica.py
+++ b/lib/rucio/web/rest/webpy/v1/replica.py
@@ -460,6 +460,8 @@ class ListReplicas(RucioController):
                                                                         rfile['name'])
 
                     lanreplicas = [replica for replica, v in dictreplica.items() if v[0] == 'lan']
+                    # sort lan by priority
+                    lanreplicas.sort(key=lambda rep: dictreplica[rep][1])
                     replicas = lanreplicas + sort_replicas({k: v for k, v in dictreplica.items() if v[0] != 'lan'}, client_location, selection=select)
 
                     idx = 1