Repository: rucio/rucio

Issue:
add_replicas with PFN does not consider lower-priority protocols
Motivation
----------

If the primary protocol is not available via WAN, it immediately jumps to first priority LAN, and fails afterwards if not possible.
The secondary and later protocols are never tried.


Discussion:
Hey @mlassnig ,

If I get it right it is related to the algorithm here:
https://github.com/rucio/rucio/blob/f711667ee4a4ea8c898098f8380c311460dc2642/lib/rucio/core/replica.py#L1363-L1371

which internally uses:
https://github.com/rucio/rucio/blob/f711667ee4a4ea8c898098f8380c311460dc2642/lib/rucio/rse/rsemanager.py#L159-L169

called here:
https://github.com/rucio/rucio/blob/f711667ee4a4ea8c898098f8380c311460dc2642/lib/rucio/rse/rsemanager.py#L192

Would it be ok to directly call `_get_possible_protocols` from `add_replica` and iterate over the protocols?

The `_get_possible_protocols` is an internal function of the rsemanager. I think you can just refactor it to make it a public `get_possible_protocols`, and use that then.
Yup, that's what I wished I could do!
Update: `get_protocols_ordered` is already there and public, I'll rely on that.

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -1428,8 +1428,8 @@ def add_replicas(rse_id, files, account, ignore_availability=True,
     :returns: True is successful.
     """
 
-    def _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan'):
-        p = rsemgr.create_protocol(rse_settings=rse_settings, operation='write', scheme=scheme, domain=domain)
+    def _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan', protocol_attr=None):
+        p = rsemgr.create_protocol(rse_settings=rse_settings, operation='write', scheme=scheme, domain=domain, protocol_attr=protocol_attr)
         expected_pfns = p.lfns2pfns(lfns)
         return clean_surls(expected_pfns.values())
 
@@ -1468,12 +1468,30 @@ def _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan'):
                 # Check that the pfns match to the expected pfns
                 lfns = [{'scope': i['scope'].external, 'name': i['name']} for i in files if i['pfn'].startswith(scheme)]
                 pfns[scheme] = clean_surls(pfns[scheme])
-                expected_pfns_wan = _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan')
+
                 # Check wan first
-                if expected_pfns_wan != pfns[scheme]:
-                    expected_pfns_lan = _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='lan')
+                found_on_wan = False
+                available_wan_protocols = rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=scheme, domain='wan')
+                expected_pfns_wan = None
+                for protocol_attr in available_wan_protocols:
+                    pfns_wan_buffer = _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan', protocol_attr=protocol_attr)
+                    if not expected_pfns_wan and pfns_wan_buffer:
+                        expected_pfns_wan = pfns_wan_buffer
+                    found_on_wan = found_on_wan or (pfns_wan_buffer == pfns[scheme])
+                    if found_on_wan:
+                        break
+
+                if not found_on_wan:
                     # Check lan
-                    if expected_pfns_lan == pfns[scheme]:
+                    found_on_lan = False
+                    available_lan_protocols = rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=scheme, domain='lan')
+                    for protocol_attr in available_lan_protocols:
+                        pfns_lan_buffer = _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='lan', protocol_attr=protocol_attr)
+                        found_on_lan = found_on_lan or (pfns_lan_buffer == pfns[scheme])
+                        if found_on_lan:
+                            break
+
+                    if found_on_lan == pfns[scheme]:
                         # Registration always with wan
                         pfns[scheme] = expected_pfns_wan
                     else:
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -169,7 +169,7 @@ def select_protocol(rse_settings, operation, scheme=None, domain='wan'):
     return min(candidates, key=lambda k: k['domains'][domain][operation])
 
 
-def create_protocol(rse_settings, operation, scheme=None, domain='wan', auth_token=None, logger=logging.log):
+def create_protocol(rse_settings, operation, scheme=None, domain='wan', auth_token=None, protocol_attr=None, logger=logging.log):
     """
     Instanciates the protocol defined for the given operation.
 
@@ -178,6 +178,7 @@ def create_protocol(rse_settings, operation, scheme=None, domain='wan', auth_tok
     :param scheme:        Optional filter if no specific protocol is defined in rse_setting for the provided operation
     :param domain:        Optional specification of the domain
     :param auth_token:    Optionally passing JSON Web Token (OIDC) string for authentication
+    :param protocol_attr: Optionally passing the full protocol availability information to correctly select WAN/LAN
     :param logger:        Optional decorated logger that can be passed from the calling daemons or servers.
     :returns:             An instance of the requested protocol
     """
@@ -190,7 +191,12 @@ def create_protocol(rse_settings, operation, scheme=None, domain='wan', auth_tok
     if domain and domain not in utils.rse_supported_protocol_domains():
         raise exception.RSEProtocolDomainNotSupported('Domain %s not supported' % domain)
 
-    protocol_attr = select_protocol(rse_settings, operation, scheme, domain)
+    if not protocol_attr:
+        protocol_attr = select_protocol(rse_settings, operation, scheme, domain)
+    else:
+        candidates = _get_possible_protocols(rse_settings, operation, scheme, domain)
+        if protocol_attr not in candidates:
+            raise exception.RSEProtocolNotSupported('Protocol %s operation %s on domain %s not supported' % (protocol_attr, operation, domain))
 
     # Instantiate protocol
     comp = protocol_attr['impl'].split('.')


Test patch: