Repository: rucio/rucio

Issue:
Improvements to the declaration of lost files
I would like to make the following suggestions:

 1. Use the synchronous API in case the number of files is below some thresholds. We had a case where a request to mark a large amount of files as temporarily unavailable was causing long delays for single-file requests which should have had higher priority.
 2. Change the message to something other than `replicas successfully declared`. This is misleading as the PFNs could be invalid and the user will move on thinking that everything is done.
 3. Do some checking on the PFNs, at the very least that they are valid URLs. It should be noted that on the side of Minos this isn’t handled gracefully.
```
Traceback (most recent call last):
  File "/usr/lib/python2.7/site-packages/rucio/daemons/badreplicas/minos.py", li
ne 182, in minos
    dict_rse[rse_id].extend(tmp_dict_rse[rse_id])
  File "/usr/lib/python2.7/site-packages/retrying.py", line 49, in wrapped_f
    return Retrying(*dargs, **dkw).call(f, *args, **kw)
  File "/usr/lib/python2.7/site-packages/retrying.py", line 206, in call
    return attempt.get(self._wrap_exception)
  File "/usr/lib/python2.7/site-packages/retrying.py", line 247, in get
    six.reraise(self.value[0], self.value[1], self.value[2])
  File "/usr/lib/python2.7/site-packages/retrying.py", line 200, in call
    attempt = Attempt(fn(*args, **kwargs), attempt_number, False)
  File "/usr/lib/python2.7/site-packages/rucio/db/sqla/session.py", line 284, in
 new_funct
    return function(*args, **kwargs)
  File "/usr/lib/python2.7/site-packages/rucio/core/replica.py", line 437, in ge
t_pfn_to_rse
    split_se = surl.split('/')[2].split(':')
```


Discussion:
Needs https://github.com/rucio/rucio/issues/4411 for 3.

Resolving patch:
diff --git a/lib/rucio/core/permission/atlas.py b/lib/rucio/core/permission/atlas.py
--- a/lib/rucio/core/permission/atlas.py
+++ b/lib/rucio/core/permission/atlas.py
@@ -15,7 +15,7 @@
 # Authors:
 # - Vincent Garonne <vgaronne@gmail.com>, 2016
 # - Martin Barisits <martin.barisits@cern.ch>, 2016-2020
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2016-2019
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2016-2021
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2018-2020
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
@@ -127,7 +127,8 @@ def has_permission(issuer, action, kwargs):
             'list_vos': perm_list_vos,
             'recover_vo_root_identity': perm_recover_vo_root_identity,
             'update_vo': perm_update_vo,
-            'access_rule_vo': perm_access_rule_vo}
+            'access_rule_vo': perm_access_rule_vo,
+            'export': perm_export}
 
     return perm.get(action, perm_default)(issuer=issuer, kwargs=kwargs)
 
@@ -1229,3 +1230,15 @@ def perm_access_rule_vo(issuer, kwargs):
     :returns: True if account is allowed, otherwise False
     """
     return get_rule(kwargs['rule_id'])['scope'].vo == issuer.vo
+
+
+def perm_export(issuer, kwargs):
+    """
+    Checks if an account can export the RSE info.
+
+    :param issuer: Account identifier which issues the command.
+    :param kwargs: List of arguments for the action.
+    :returns: True if account is allowed, otherwise False
+    """
+    is_cloud_admin = bool([acc_attr for acc_attr in list_account_attributes(account=issuer) if (acc_attr['key'].startswith('cloud-')) and (acc_attr['value'] == 'admin')])
+    return _is_root(issuer) or has_account_attribute(account=issuer, key='admin') or is_cloud_admin
diff --git a/lib/rucio/core/permission/generic.py b/lib/rucio/core/permission/generic.py
--- a/lib/rucio/core/permission/generic.py
+++ b/lib/rucio/core/permission/generic.py
@@ -14,7 +14,7 @@
 #
 # Authors:
 # - Vincent Garonne <vgaronne@gmail.com>, 2016
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2016-2018
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2016-2021
 # - Martin Barisits <martin.barisits@cern.ch>, 2017-2020
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2018-2020
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
@@ -121,7 +121,8 @@ def has_permission(issuer, action, kwargs):
             'del_account_identity': perm_del_account_identity,
             'del_identity': perm_del_identity,
             'remove_did_from_followed': perm_remove_did_from_followed,
-            'remove_dids_from_followed': perm_remove_dids_from_followed}
+            'remove_dids_from_followed': perm_remove_dids_from_followed,
+            'export': perm_export}
 
     return perm.get(action, perm_default)(issuer=issuer, kwargs=kwargs)
 
@@ -1010,3 +1011,14 @@ def perm_remove_dids_from_followed(issuer, kwargs):
     if not kwargs['account'] == issuer:
         return False
     return True
+
+
+def perm_export(issuer, kwargs):
+    """
+    Checks if an account can export the RSE info.
+
+    :param issuer: Account identifier which issues the command.
+    :param kwargs: List of arguments for the action.
+    :returns: True if account is allowed, otherwise False
+    """
+    return _is_root(issuer)
diff --git a/lib/rucio/core/rse.py b/lib/rucio/core/rse.py
--- a/lib/rucio/core/rse.py
+++ b/lib/rucio/core/rse.py
@@ -1300,9 +1300,9 @@ def export_rse(rse_id, session=None):
     rse_data.pop('deleted_at')
 
     # get RSE attributes
-    rse_data['attributes'] = list_rse_attributes(rse_id=rse_id)
+    rse_data['attributes'] = list_rse_attributes(rse_id=rse_id, session=session)
 
-    protocols = get_rse_protocols(rse_id=rse_id)
+    protocols = get_rse_protocols(rse_id=rse_id, session=session)
     rse_data['lfn2pfn_algorithm'] = protocols.get('lfn2pfn_algorithm')
     rse_data['verify_checksum'] = protocols.get('verify_checksum')
     rse_data['credentials'] = protocols.get('credentials')
@@ -1312,7 +1312,7 @@ def export_rse(rse_id, session=None):
     rse_data['protocols'] = protocols.get('protocols')
 
     # get RSE limits
-    limits = get_rse_limits(rse_id=rse_id)
+    limits = get_rse_limits(rse_id=rse_id, session=session)
     rse_data['MinFreeSpace'] = limits.get('MinFreeSpace')
     rse_data['MaxBeingDeletedFiles'] = limits.get('MaxBeingDeletedFiles')
 


Test patch: