Repository: rucio/rucio

Issue:
gfal stat does checksum calls for all checksums
Motivation
----------
The stat command of gfal currently does a checksum call for each checksum in `GLOBALLY_SUPPORTED_CHECKSUMS` this currrently results in a very long processing time for SRM which finally times out after ~30 minutes.


Modification
------------
First try the `PREFERRED_CHCEKSUM` and if this validates successfully don't try the other checksums.


Minos doesn't handle properly ReplicaNotFound
Motivation
----------
Traceback :
```
2019-12-12 10:00:30,709 20205   CRITICAL        Traceback (most recent call last):
  File "/usr/lib/python2.7/site-packages/rucio/daemons/badreplicas/minos.py", line 210, in minos
    update_replicas_states(chunk, nowait=False, session=session)
  File "/usr/lib/python2.7/site-packages/rucio/db/sqla/session.py", line 381, in new_funct
    result = function(*args, **kwargs)
  File "/usr/lib/python2.7/site-packages/rucio/core/replica.py", line 1777, in update_replicas_states
    raise exception.ReplicaNotFound("No row found for scope: %s name: %s rse: %s" % (replica['scope'], replica['name'], get_rse_name(replica['rse_id'], session=session)))
ReplicaNotFound: Replica not found
Details: No row found for scope: panda name: panda.um.user.kkrizka.data18_13TeV.physics_Main.19901029._003813.tree.root rse: NDGF-T1_SCRATCHDISK
```



Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/badreplicas/minos.py b/lib/rucio/daemons/badreplicas/minos.py
--- a/lib/rucio/daemons/badreplicas/minos.py
+++ b/lib/rucio/daemons/badreplicas/minos.py
@@ -35,7 +35,7 @@
 from rucio.db.sqla.session import get_session
 from rucio.common.config import config_get
 from rucio.common.utils import chunks
-from rucio.common.exception import UnsupportedOperation, DataIdentifierNotFound
+from rucio.common.exception import UnsupportedOperation, DataIdentifierNotFound, ReplicaNotFound
 from rucio.core.did import get_metadata
 from rucio.core.replica import (get_bad_pfns, get_pfn_to_rse, declare_bad_file_replicas,
                                 get_did_from_pfns, update_replicas_states, bulk_add_bad_replicas,
@@ -212,7 +212,7 @@ def minos(bulk=1000, once=False, sleep_time=60):
                             pfns = [entry['pfn'] for entry in chunk]
                             bulk_delete_bad_pfns(pfns=pfns, session=session)
                             session.commit()  # pylint: disable=no-member
-                        except UnsupportedOperation as error:
+                        except (UnsupportedOperation, ReplicaNotFound) as error:
                             session.rollback()  # pylint: disable=no-member
                             logging.error(prepend_str + 'Problem to bulk update PFNs. PFNs will be updated individually. Error : %s' % str(error))
                             for rep in chunk:
@@ -227,7 +227,7 @@ def minos(bulk=1000, once=False, sleep_time=60):
                                     if rep['rse_id'] in unavailable_states:
                                         logging.info(prepend_str + '%s is in unavailable state. Will be removed from the list of bad PFNs' % str(rep['pfn']))
                                         bulk_delete_bad_pfns(pfns=[rep['pfn']], session=None)
-                                except DataIdentifierNotFound as error:
+                                except (DataIdentifierNotFound, ReplicaNotFound) as error:
                                     logging.error(prepend_str + 'Will remove %s from the list of bad PFNs' % str(rep['pfn']))
                                     bulk_delete_bad_pfns(pfns=[rep['pfn']], session=None)
                             session = get_session()
diff --git a/lib/rucio/rse/protocols/gfal.py b/lib/rucio/rse/protocols/gfal.py
--- a/lib/rucio/rse/protocols/gfal.py
+++ b/lib/rucio/rse/protocols/gfal.py
@@ -41,7 +41,7 @@
 
 from rucio.common import exception, config
 from rucio.common.constraints import STRING_TYPES
-from rucio.common.utils import GLOBALLY_SUPPORTED_CHECKSUMS
+from rucio.common.utils import GLOBALLY_SUPPORTED_CHECKSUMS, PREFERRED_CHECKSUM
 from rucio.rse.protocols import protocol
 
 try:
@@ -334,13 +334,15 @@ def stat(self, path):
 
             :raises ServiceUnavailable: if some generic error occured in the library.
 
-            :returns: a dict with two keys, filesize and adler32 of the file provided in path.
+            :returns: a dict with two keys, filesize and an element of GLOBALLY_SUPPORTED_CHECKSUMS.
         """
         ret = {}
         ctx = self.__ctx
 
+        path = str(path)
+
         try:
-            stat_str = str(ctx.stat(str(path)))
+            stat_str = str(ctx.stat(path))
         except Exception as error:
             msg = 'Error while processing gfal stat call. Error: %s'
             raise exception.ServiceUnavailable(msg % str(error))
@@ -350,22 +352,25 @@ def stat(self, path):
             msg = 'gfal stat call result has unknown format. Result: %s'
             raise exception.ServiceUnavailable(msg % stat_str)
 
-        ret['filesize'] = stat_str.split()[7]
+        ret['filesize'] = stats[7]
 
-        verified = False
         message = "\n"
+        try:
+            ret[PREFERRED_CHECKSUM] = ctx.checksum(path, str(PREFERRED_CHECKSUM.capitalize()))
+            return ret
+        except Exception as error:
+            message += 'Error while processing gfal checksum call (%s). Error: %s \n' % (PREFERRED_CHECKSUM, str(error))
 
         for checksum_name in GLOBALLY_SUPPORTED_CHECKSUMS:
+            if checksum_name == PREFERRED_CHECKSUM:
+                continue
             try:
-                ret[checksum_name] = ctx.checksum(str(path), str(checksum_name.capitalize()))
-                verified = True
+                ret[checksum_name] = ctx.checksum(path, str(checksum_name.capitalize()))
+                return ret
             except Exception as error:
                 message += 'Error while processing gfal checksum call (%s). Error: %s \n' % (checksum_name, str(error))
 
-        if not verified:
-            raise exception.RSEChecksumUnavailable(message)
-
-        return ret
+        raise exception.RSEChecksumUnavailable(message)
 
     def __gfal2_cancel(self):
         """


Test patch: