Repository: rucio/rucio

Issue:
error-prone xroot prioritization when listing mixed archive/non-archive replicas
Motivation
----------
_list_replicas() has some logic to prioritize xroot for archives. This logic relies on appending the 'root' scheme to the list of accepted schemes when building the protocol for an rse. This operation, of building the protocol, is however only done once per RSE and then the protocol list is re-used by the same list-replicas call.

If, in the same list-replica call, we fetch archive and non-archive replicas, the previous behavior can result in the 'root' scheme being considered for non-archive replicas while it shouldn't. Respectively, if a non-archive replica is the first to be analyzed on an RSE, we will not prioritize xroot for the archives




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import copy
 import heapq
 import logging
 import math
@@ -763,7 +763,7 @@ def _get_list_replicas_protocols(
         'lan': {p['scheme']: p['domains']['lan']['read'] for p in rse_info['protocols'] if p['domains']['lan']['read'] > 0},
     }
 
-    rse_schemes = schemes or []
+    rse_schemes = copy.copy(schemes) if schemes else []
     if not rse_schemes:
         try:
             for domain in domains:
@@ -891,7 +891,8 @@ def _list_replicas(replicas, show_pfns, schemes, files_wo_replica, client_locati
             except Exception:
                 pass  # do not hard fail if site cannot be resolved or is empty
 
-    file, protocols_by_rse_id, pfns_cache = {}, {}, {}
+    file, pfns_cache = {}, {}
+    protocols_cache = defaultdict(dict)
 
     for _, replica_group in groupby(replicas, key=lambda x: (x[0], x[1])):  # Group by scope/name
         file = {}
@@ -900,6 +901,8 @@ def _list_replicas(replicas, show_pfns, schemes, files_wo_replica, client_locati
             if isinstance(archive_scope, str):
                 archive_scope = InternalScope(archive_scope, fromExternal=False)
 
+            is_archive = bool(archive_scope and archive_name)
+
             # it is the first row in the scope/name group
             if not file:
                 file['scope'], file['name'] = scope, name
@@ -919,7 +922,8 @@ def _list_replicas(replicas, show_pfns, schemes, files_wo_replica, client_locati
                 continue
 
             # It's the first time we see this RSE, initialize the protocols needed for PFN generation
-            if rse_id not in protocols_by_rse_id:
+            protocols = protocols_cache.get(rse_id, {}).get(is_archive)
+            if not protocols:
                 # select the lan door in autoselect mode, otherwise use the wan door
                 domain = input_domain
                 if domain is None:
@@ -927,24 +931,21 @@ def _list_replicas(replicas, show_pfns, schemes, files_wo_replica, client_locati
                     if local_rses and rse_id in local_rses:
                         domain = 'lan'
 
-                # FIXME: if a list_replicas call iterates over both non-archive and archive replicas
-                # on the same rse_id, the protocols will not be correctly initialized for whatever comes
-                # second. This is because of the "additional_schemes" logic:
                 protocols = _get_list_replicas_protocols(
                     rse_id=rse_id,
                     domain=domain,
                     schemes=schemes,
                     # We want 'root' for archives even if it wasn't included into 'schemes'
-                    additional_schemes=['root'] if archive_scope and archive_name else [],
+                    additional_schemes=['root'] if is_archive else [],
                     session=session,
                 )
-                protocols_by_rse_id[rse_id] = protocols
+                protocols_cache[rse_id][is_archive] = protocols
 
             # build the pfns
-            for domain, protocol, priority in protocols_by_rse_id[rse_id]:
+            for domain, protocol, priority in protocols:
                 # If the current "replica" is a constituent inside an archive, we must construct the pfn for the
                 # parent (archive) file and append the xrdcl.unzip query string to it.
-                if archive_scope and archive_name:
+                if is_archive:
                     t_scope = archive_scope
                     t_name = archive_name
                 else:
@@ -973,7 +974,7 @@ def _list_replicas(replicas, show_pfns, schemes, files_wo_replica, client_locati
                     )
 
                     client_extract = False
-                    if archive_scope and archive_name:
+                    if is_archive:
                         domain = 'zip'
                         pfn = add_url_query(pfn, {'xrdcl.unzip': name})
                         if protocol.attributes['scheme'] == 'root':
@@ -1036,7 +1037,7 @@ def _list_replicas(replicas, show_pfns, schemes, files_wo_replica, client_locati
 @stream_session
 def list_replicas(
         dids: "Sequence[Dict[str, Any]]",
-        schemes: "Optional[str]" = None,
+        schemes: "Optional[List[str]]" = None,
         unavailable: bool = False,
         request_id: "Optional[str]" = None,
         ignore_availability: bool = True,
@@ -1094,7 +1095,7 @@ def list_replicas(
 
 def _list_replicas_with_temp_tables(
         dids: "Sequence[Dict[str, Any]]",
-        schemes: "Optional[str]" = None,
+        schemes: "Optional[List[str]]" = None,
         unavailable: bool = False,
         request_id: "Optional[str]" = None,
         ignore_availability: bool = True,
@@ -4422,7 +4423,7 @@ def _list_replicas_for_files_wo_temp_tables(file_clause, state_clause, files_wo_
 
 def _list_replicas_wo_temp_tables(
         dids: "Sequence[Dict[str, Any]]",
-        schemes: "Optional[str]" = None,
+        schemes: "Optional[List[str]]" = None,
         unavailable: bool = False,
         request_id: "Optional[str]" = None,
         ignore_availability: bool = True,


Test patch:
diff --git a/lib/rucio/tests/temp_factories.py b/lib/rucio/tests/temp_factories.py
--- a/lib/rucio/tests/temp_factories.py
+++ b/lib/rucio/tests/temp_factories.py
@@ -36,6 +36,10 @@
     from typing import Tuple
 
 
+def _to_external(did):
+    return {'scope': did['scope'].external, 'name': did['name']}
+
+
 class TemporaryRSEFactory:
     """
     Factory which keeps track of created RSEs and cleans up everything related to these RSEs at the end
@@ -205,27 +209,29 @@ def _random_did(self, did_type, scope, name_suffix=''):
         self.created_dids.add((scope, name))
         return did
 
-    def random_file_did(self, scope=None, name_suffix=''):
+    def random_file_did(self, scope=None, name_suffix='', *, external=False):
         did = self._random_did(did_type='file', scope=scope, name_suffix=name_suffix)
-        return did
+        return _to_external(did) if external else did
 
-    def random_dataset_did(self, scope=None):
+    def random_dataset_did(self, scope=None, *, external=False):
         did = self._random_did(did_type='dataset', scope=scope)
-        return did
+        return _to_external(did) if external else did
 
-    def random_container_did(self, scope=None):
+    def random_container_did(self, scope=None, *, external=False):
         did = self._random_did(did_type='container', scope=scope)
-        return did
+        return _to_external(did) if external else did
 
-    def make_dataset(self, scope=None):
+    def make_dataset(self, scope=None, *, external=False):
         did = self.random_dataset_did(scope=scope)
-        self.client.add_dataset(scope=did['scope'].external, name=did['name'])
-        return did
+        external_did = _to_external(did)
+        self.client.add_dataset(**external_did)
+        return external_did if external else did
 
-    def make_container(self, scope=None):
+    def make_container(self, scope=None, *, external=False):
         did = self.random_container_did(scope=scope)
-        self.client.add_container(scope=did['scope'].external, name=did['name'])
-        return did
+        external_did = _to_external(did)
+        self.client.add_container(**external_did)
+        return external_did if external else did
 
     def upload_test_file(self, rse_name, scope=None, name=None, path=None, size=2, return_full_item=False):
         scope = self._sanitize_or_set_scope(scope)
diff --git a/lib/rucio/tests/test_archive.py b/lib/rucio/tests/test_archive.py
--- a/lib/rucio/tests/test_archive.py
+++ b/lib/rucio/tests/test_archive.py
@@ -177,7 +177,7 @@ def test_archive_on_dataset_level(rse_factory, did_factory, root_account):
     assert not metadata['is_archive']
 
 
-def test_root_priority_is_highest(rse_factory, mock_scope, root_account, did_client, replica_client):
+def test_root_priority_is_highest(rse_factory, did_factory, mock_scope, root_account, did_client, replica_client):
     """ ARCHIVE (CORE): Ensure that the root protocol is prioritized for archives"""
 
     # Add 2 RSEs. Set the root protocol to have the lowest priority overall
@@ -209,17 +209,17 @@ def test_root_priority_is_highest(rse_factory, mock_scope, root_account, did_cli
                                'wan': {'read': 1, 'write': 1, 'delete': 1}}})
 
     # register archive
-    archive = {'scope': mock_scope, 'name': 'cube.1.zip', 'type': 'FILE', 'bytes': 2596, 'adler32': 'beefdead'}
-    add_replicas(rse_id=rse1_id, files=[archive], account=root_account)
-    add_replicas(rse_id=rse2_id, files=[archive], account=root_account)
+    archive = {**did_factory.random_file_did(name_suffix='.zip', external=True), 'type': 'FILE', 'bytes': 2596, 'adler32': 'beefdead'}
+    replica_client.add_replicas(rse=rse1, files=[archive])
+    replica_client.add_replicas(rse=rse2, files=[archive])
 
     # archived files with replicas
-    archived_file = [{'scope': mock_scope, 'name': 'zippedfile-%i-%s' % (i, str(generate_uuid())), 'type': 'FILE',
-                      'bytes': 4322, 'adler32': 'beefbeef'} for i in range(2)]
-    did_client.add_files_to_archive(scope=mock_scope.external, name=archive['name'], files=archived_file)
+    archived_dids = [did_factory.random_file_did(external=True) for _ in range(2)]
+    archived_files = [{**did, 'type': 'FILE', 'bytes': 4322, 'adler32': 'beefbeef'} for did in archived_dids]
+    did_client.add_files_to_archive(scope=mock_scope.external, name=archive['name'], files=archived_files)
 
     both_rses = '%s|%s' % (rse1, rse2)
-    replicas = list(replica_client.list_replicas(dids=[{'scope': f['scope'].external, 'name': f['name']} for f in archived_file], rse_expression=both_rses))
+    replicas = list(replica_client.list_replicas(dids=archived_dids, rse_expression=both_rses))
 
     for replica in replicas:
         # The root protocol pfn must have the smallest (best) priority
@@ -235,3 +235,17 @@ def test_root_priority_is_highest(rse_factory, mock_scope, root_account, did_cli
 
         assert replica['pfns'][root_pfn]['domain'] == 'zip'
         assert replica['pfns'][posix_pfn]['domain'] == 'zip'
+
+    # setting schemes=['file'] must still return "xroot" replicas for the archived constituent,
+    # but the filter must be correctly respected for non-archive constituents
+    non_archived_did = did_factory.random_file_did(external=True)
+    replica_client.add_replicas(rse=rse1, files=[{**non_archived_did, 'type': 'FILE', 'bytes': 10, 'adler32': 'beefdead'}])
+    replicas = list(replica_client.list_replicas(dids=[archived_dids[0], non_archived_did], rse_expression=rse1, schemes=['file']))
+    assert len(replicas) == 2
+    for replica in replicas:
+        if replica['name'] == non_archived_did['name']:
+            assert len(replica['pfns']) == 1
+            assert next(iter(replica['pfns'])).startswith('file')
+        else:
+            assert len(replica['pfns']) == 2
+            assert next(iter(replica['pfns'])).startswith('root')