Repository: rucio/rucio

Issue:
rucio-conveyor-finisher fails at a non-deterministic endpoint with root protocol
Motivation
----------
LCLS is using non-deterministic endpoints with root protocol. For root protocol, the endpoint prefix usually starts with two slashes (e.g. //psdm/rucio). However, rucio-conveyor-finisher runs into trouble because of this line:

https://github.com/rucio/rucio/blob/master/lib/rucio/rse/protocols/protocol.py#L444

At there, "path" is a sanitized one with '//' being replaced by a single '/', but prefix isn't. So the comparison will fail. This only happens for non-deterministic endpoints.

Modification
------------
In parse_pfns(), either not replacing '//' by single '/'. or do this replacement for both 'path' and 'prefix'.



Discussion:

Resolving patch:
diff --git a/lib/rucio/rse/protocols/protocol.py b/lib/rucio/rse/protocols/protocol.py
--- a/lib/rucio/rse/protocols/protocol.py
+++ b/lib/rucio/rse/protocols/protocol.py
@@ -450,12 +450,15 @@ def parse_pfns(self, pfns):
             while '//' in parsed.path:
                 parsed = parsed._replace(path=parsed.path.replace('//', '/'))
             path = parsed.path
+            prefix = self.attributes['prefix']
+            while '//' in prefix:
+                prefix = prefix.replace('//', '/')
 
             # Protect against 'lazy' defined prefixes for RSEs in the repository
-            if not self.attributes['prefix'].startswith('/'):
-                self.attributes['prefix'] = '/' + self.attributes['prefix']
-            if not self.attributes['prefix'].endswith('/'):
-                self.attributes['prefix'] += '/'
+            if not prefix.startswith('/'):
+                prefix = '/' + prefix
+            if not prefix.endswith('/'):
+                prefix += '/'
 
             if self.attributes['hostname'] != hostname:
                 if self.attributes['hostname'] != 'localhost':  # In the database empty hostnames are replaced with localhost but for some URIs (e.g. file) a hostname is not included
@@ -464,13 +467,12 @@ def parse_pfns(self, pfns):
             if self.attributes['port'] != port:
                 raise exception.RSEFileNameNotSupported('Invalid port: provided \'%s\', expected \'%s\'' % (port, self.attributes['port']))
 
-            if not path.startswith(self.attributes['prefix']):
-                raise exception.RSEFileNameNotSupported('Invalid prefix: provided \'%s\', expected \'%s\'' % ('/'.join(path.split('/')[0:len(self.attributes['prefix'].split('/')) - 1]),
-                                                                                                              self.attributes['prefix']))  # len(...)-1 due to the leading '/
+            if not path.startswith(prefix):
+                raise exception.RSEFileNameNotSupported('Invalid prefix: provided \'%s\', expected \'%s\'' % ('/'.join(path.split('/')[0:len(prefix.split('/')) - 1]),
+                                                                                                              prefix))  # len(...)-1 due to the leading '/
 
             # Spliting parsed.path into prefix, path, filename
-            prefix = self.attributes['prefix']
-            path = path.partition(self.attributes['prefix'])[2]
+            path = path.partition(prefix)[2]
             name = path.split('/')[-1]
             path = '/'.join(path.split('/')[:-1])
             if not path.startswith('/'):


Test patch:
diff --git a/lib/rucio/tests/test_conveyor.py b/lib/rucio/tests/test_conveyor.py
--- a/lib/rucio/tests/test_conveyor.py
+++ b/lib/rucio/tests/test_conveyor.py
@@ -590,6 +590,40 @@ def test_preparer_throttler_submitter(rse_factory, did_factory, root_account, fi
     assert request['state'] == RequestState.QUEUED
 
 
+@skip_rse_tests_with_accounts
+@pytest.mark.dirty(reason="leaves files in XRD containers")
+@pytest.mark.noparallel(reason="runs submitter; poller and finisher")
+@pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
+    'rucio.common.rse_attributes.REGION',
+    'rucio.core.rse.REGION',
+    'rucio.rse.rsemanager.RSE_REGION',  # for RSE info
+]}], indirect=True)
+def test_non_deterministic_dst(did_factory, did_client, root_account, vo, caches_mock):
+    """
+    Test a transfer towards a non-deterministic RSE
+    """
+    src_rse = 'XRD3'
+    src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo)
+    dst_rse = 'XRD4'
+    dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo)
+    all_rses = [src_rse_id, dst_rse_id]
+
+    did = did_factory.upload_test_file(src_rse)
+    # Dataset name is part of the non-deterministic path
+    dataset = did_factory.make_dataset()
+    did_client.add_files_to_dataset(files=[{'scope': did['scope'].external, 'name': did['name']}], scope=dataset['scope'].external, name=dataset['name'])
+
+    rse_core.update_rse(rse_id=dst_rse_id, parameters={'deterministic': False})
+    try:
+        rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None)
+        submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=None, transfertype='single', filter_transfertool=None)
+
+        replica = __wait_for_replica_transfer(dst_rse_id=dst_rse_id, **did)
+        assert replica['state'] == ReplicaState.AVAILABLE
+    finally:
+        rse_core.update_rse(rse_id=dst_rse_id, parameters={'deterministic': True})
+
+
 @skip_rse_tests_with_accounts
 @pytest.mark.noparallel(reason="runs stager; poller and finisher")
 def test_stager(rse_factory, did_factory, root_account, replica_client):