Repository: rucio/rucio

Issue:
Replica registration protocol issue
Motivation
----------
The following replica registration issue was reported to DDM Support. Registration fails with

```
Error description:  ddm, 200: Could not add files to DDM: Details: One of the PFNs provided does not match the Rucio expected PFN : got {'gsiftp': ['gsiftp://cephc05.gla.scotgrid.ac.uk:2811/atlas:datadisk/rucio/hc_test/31/70/c5ebc528-d8e1-47ed-8d9e-35a2a674c794_13327.1.job.log.tgz', 'gsiftp://cephc05.gla.scotgrid.ac.uk:2811/atlas:datadisk/rucio/hc_test/71/57/output.1.c5ebc528-d8e1-47ed-8d9e-35a2a674c794_13327.pool.root']}, expected ['gsiftp://cephc04.gla.scotgrid.ac.uk:2811/atlas:datadisk/rucio/hc_test/31/70/c5ebc528-d8e1-47
```

There was some work done in #3940 to also allow lower-priority protocols, but the issue here seems to be that multiple protocols are registered with different endpoints, and Rucio "picks" one of them, but doesn't allow the other.





Discussion:
Hi,
  This appears now to be introducing operational issues at the site affected by this issue [https://ggus.eu/index.php?mode=ticket_info&ticket_id=152006]. 
  Is there a timeline on a fix; or any proposed workaround that we could try (to ensure the site can route its transfers correctly again)?
  Thanks in advance.
  James
Hi, I haven't had time yet to look into this. This is potentially tricky, so no guarantees yet when this is done. As a workaround, could you try removing the write_lan/1 for the gsiftp endpoint to see if that helps?


Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -15,7 +15,7 @@
 #
 # Authors:
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2013-2018
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2021
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013-2014
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2021
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2014-2021
@@ -39,6 +39,7 @@
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
 # - Gabriele Fronzé <sucre.91@hotmail.it>, 2021
 # - David Población Criado <david.poblacion.criado@cern.ch>, 2021
+# - Joel Dierkes <joel.dierkes@cern.ch>, 2021
 
 from __future__ import print_function
 
@@ -1468,16 +1469,18 @@ def _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan',
     if not (replica_rse.availability & 2) and not ignore_availability:
         raise exception.ResourceTemporaryUnavailable('%s is temporary unavailable for writing' % replica_rse.rse)
 
+    for file in files:
+        if 'pfn' not in file:
+            if not replica_rse.deterministic:
+                raise exception.UnsupportedOperation('PFN needed for this (non deterministic) RSE %s ' % (replica_rse.rse))
+
     replicas = __bulk_add_file_dids(files=files, account=account,
                                     dataset_meta=dataset_meta,
                                     session=session)
 
-    pfns, scheme = {}, None  # {scheme: [pfns], scheme: [pfns]}
+    pfns = {}  # Dict[str, List[str]], {scheme: [pfns], scheme: [pfns]}
     for file in files:
-        if 'pfn' not in file:
-            if not replica_rse.deterministic:
-                raise exception.UnsupportedOperation('PFN needed for this (non deterministic) RSE %s ' % (replica_rse.rse))
-        else:
+        if 'pfn' in file:
             scheme = file['pfn'].split(':')[0]
             pfns.setdefault(scheme, []).append(file['pfn'])
 
@@ -1496,33 +1499,16 @@ def _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan',
                 lfns = [{'scope': i['scope'].external, 'name': i['name']} for i in files if i['pfn'].startswith(scheme)]
                 pfns[scheme] = clean_surls(pfns[scheme])
 
-                # Check wan first
-                found_on_wan = False
-                available_wan_protocols = rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=scheme, domain='wan')
-                expected_pfns_wan = None
-                for protocol_attr in available_wan_protocols:
-                    pfns_wan_buffer = _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan', protocol_attr=protocol_attr)
-                    if not expected_pfns_wan and pfns_wan_buffer:
-                        expected_pfns_wan = pfns_wan_buffer
-                    found_on_wan = found_on_wan or (pfns_wan_buffer == pfns[scheme])
-                    if found_on_wan:
-                        break
+                for protocol_attr in rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=scheme, domain='wan'):
+                    pfns[scheme] = list(set(pfns[scheme]) - set(_expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan', protocol_attr=protocol_attr)))
 
-                if not found_on_wan:
-                    # Check lan
-                    found_on_lan = False
-                    available_lan_protocols = rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=scheme, domain='lan')
-                    for protocol_attr in available_lan_protocols:
-                        pfns_lan_buffer = _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='lan', protocol_attr=protocol_attr)
-                        found_on_lan = found_on_lan or (pfns_lan_buffer == pfns[scheme])
-                        if found_on_lan:
-                            break
+                if len(pfns[scheme]) > 0:
+                    for protocol_attr in rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=scheme, domain='lan'):
+                        pfns[scheme] = list(set(pfns[scheme]) - set(_expected_pfns(lfns, rse_settings, scheme, operation='write', domain='lan', protocol_attr=protocol_attr)))
 
-                    if found_on_lan == pfns[scheme]:
-                        # Registration always with wan
-                        pfns[scheme] = expected_pfns_wan
-                    else:
-                        raise exception.InvalidPath('One of the PFNs provided does not match the Rucio expected PFN : got %s, expected %s (%s)' % (str(pfns), str(expected_pfns_wan), str(lfns)))
+                if len(pfns[scheme]) > 0:
+                    # pfns not found in wan or lan
+                    raise exception.InvalidPath('One of the PFNs provided does not match the Rucio expected PFN : %s (%s)' % (str(pfns[scheme]), str(lfns)))
 
     nbfiles, bytes_ = __bulk_add_replicas(rse_id=rse_id, files=files, account=account, session=session)
     increase(rse_id=rse_id, files=nbfiles, bytes_=bytes_, session=session)
@@ -2804,7 +2790,7 @@ def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=
         schema = ''
         if BASE.metadata.schema:
             schema = BASE.metadata.schema + '.'
-        session.execute('DELETE FROM {schema}updated_col_rep A WHERE A.rowid > ANY (SELECT B.rowid FROM {schema}updated_col_rep B WHERE A.scope = B.scope AND A.name=B.name AND A.did_type=B.did_type AND (A.rse_id=B.rse_id OR (A.rse_id IS NULL and B.rse_id IS NULL)))'.format(schema=schema))
+        session.execute('DELETE FROM {schema}updated_col_rep A WHERE A.rowid > ANY (SELECT B.rowid FROM {schema}updated_col_rep B WHERE A.scope = B.scope AND A.name=B.name AND A.did_type=B.did_type AND (A.rse_id=B.rse_id OR (A.rse_id IS NULL and B.rse_id IS NULL)))'.format(schema=schema))  # NOQA: E501
     elif session.bind.dialect.name == 'mysql':
         subquery1 = session.query(func.max(models.UpdatedCollectionReplica.id).label('max_id')).\
             group_by(models.UpdatedCollectionReplica.scope,


Test patch:
diff --git a/lib/rucio/tests/test_replica.py b/lib/rucio/tests/test_replica.py
--- a/lib/rucio/tests/test_replica.py
+++ b/lib/rucio/tests/test_replica.py
@@ -31,6 +31,8 @@
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
 # - Ilija Vukotic <ivukotic@uchicago.edu>, 2021
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
+# - Joel Dierkes <joel.dierkes@cern.ch>, 2021
 
 from __future__ import print_function
 
@@ -154,6 +156,50 @@ def test_cache_replicas(self, mock_get, rse_factory, mock_scope, root_account):
                 client_location={'site': rse}):
             assert list(rep['pfns'].keys())[0].count('root://') == 1
 
+    def test_replicas_pfn_resolution(self, rse_factory, mock_scope, root_account):
+        """ REPLICA (CORE): Test pfn resolution with different protocols, this shouldn't throw an error """
+
+        rse, rse_id = rse_factory.make_rse()
+
+        add_protocol(rse_id, {'scheme': 'root',
+                              'hostname': 'root.aperture.com',
+                              'port': 1400,
+                              'prefix': '//test/chamber/',
+                              'impl': 'rucio.rse.protocols.xrootd.Default',
+                              'domains': {
+                                  'lan': {'read': 1, 'write': 1, 'delete': 1},
+                                  'wan': {'read': 1, 'write': 1, 'delete': 1}}})
+        add_protocol(rse_id, {'scheme': 'root',
+                              'hostname': 'root.aperture.com',
+                              'port': 1401,
+                              'prefix': '//test/chamber/',
+                              'impl': 'rucio.rse.protocols.xrootd.Default',
+                              'domains': {
+                                  'lan': {'read': 1, 'write': 1, 'delete': 1},
+                                  'wan': {'read': 1, 'write': 1, 'delete': 1}}})
+        add_protocol(rse_id, {'scheme': 'root',
+                              'hostname': 'root2.aperture.com',
+                              'port': 1401,
+                              'prefix': '//test/chamber/',
+                              'impl': 'rucio.rse.protocols.xrootd.Default',
+                              'domains': {
+                                  'lan': {'read': 1, 'write': 1, 'delete': 1},
+                                  'wan': {'read': 1, 'write': 1, 'delete': 1}}})
+
+        files = []
+
+        name = 'file_%s' % generate_uuid()
+        hstr = hashlib.md5(('%s:%s' % (mock_scope, name)).encode('utf-8')).hexdigest()
+        pfn = 'root://root.aperture.com:1401//test/chamber/mock/%s/%s/%s' % (hstr[0:2], hstr[2:4], name)
+        files.append({'scope': mock_scope, 'name': name, 'bytes': 1234, 'adler32': 'deadbeef', 'pfn': pfn})
+
+        name = 'element_%s' % generate_uuid()
+        hstr = hashlib.md5(('%s:%s' % (mock_scope, name)).encode('utf-8')).hexdigest()
+        pfn = 'root://root2.aperture.com:1401//test/chamber/mock/%s/%s/%s' % (hstr[0:2], hstr[2:4], name)
+        files.append({'scope': mock_scope, 'name': name, 'bytes': 1234, 'adler32': 'deadbeef', 'pfn': pfn})
+
+        add_replicas(rse_id=rse_id, files=files, account=root_account)
+
     @pytest.mark.noparallel(reason='calls list_bad_replicas() which acts on all bad replicas without any filtering')
     def test_add_list_bad_replicas(self, rse_factory, mock_scope, root_account):
         """ REPLICA (CORE): Add bad replicas and list them"""