Repository: rucio/rucio

Issue:
rsemgr: retries of protocol.stat
Motivation
----------
in some cases, the stat failed due to service unavailable, although the file was phsysically there.

Modification
------------
at line: https://github.com/rucio/rucio/blob/master/lib/rucio/rse/rsemanager.py#L468
try, fail, wait 1 second, try, fail, wait 2 seconds, try, fail wait 4 seconds, try fail, wait 16 seconds, try fail, wait 32 seconds, try, fail finally


Discussion:

Resolving patch:
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -22,7 +22,7 @@
 # - Martin Barisits <martin.barisits@cern.ch>, 2017-2018
 # - Tobias Wegner <twegner@cern.ch>, 2017-2018
 # - Brian Bockelman <bbockelm@cse.unl.edu>, 2018
-# - Frank Berghaus <frank.berghaus@cern.ch>, 2018
+# - Frank Berghaus <frank.berghaus@cern.ch>, 2018-2019
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Nicolo Magini <nicolo.magini@cern.ch>, 2018
 #
@@ -33,6 +33,7 @@
 import copy
 import os
 import random
+from time import sleep
 
 try:
     from urlparse import urlparse
@@ -40,6 +41,7 @@
     from urllib.parse import urlparse
 
 from rucio.common import exception, utils, constants
+from rucio.common.config import config_get
 from rucio.common.constraints import STRING_TYPES
 from rucio.common.utils import make_valid_did
 
@@ -433,7 +435,7 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
                 valid = None
                 try:  # Get metadata of file to verify if upload was successful
                     try:
-                        stats = protocol.stat('%s.rucio.upload' % pfn)
+                        stats = _retry_protocol_stat(protocol, '%s.rucio.upload' % pfn)
                         if ('adler32' in stats) and ('adler32' in lfn):
                             valid = stats['adler32'] == lfn['adler32']
                         if (valid is None) and ('filesize' in stats) and ('filesize' in lfn):
@@ -475,7 +477,7 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
                 valid = None
                 try:  # Get metadata of file to verify if upload was successful
                     try:
-                        stats = protocol.stat(pfn)
+                        stats = _retry_protocol_stat(protocol, pfn)
                         if ('adler32' in stats) and ('adler32' in lfn):
                             valid = stats['adler32'] == lfn['adler32']
                         if (valid is None) and ('filesize' in stats) and ('filesize' in lfn):
@@ -713,6 +715,27 @@ def find_matching_scheme(rse_settings_dest, rse_settings_src, operation_src, ope
     raise exception.RSEProtocolNotSupported('No protocol for provided settings found : %s.' % str(rse_settings_dest))
 
 
+def _retry_protocol_stat(protocol, pfn):
+    """
+    try to stat file, on fail try again 1s, 2s, 4s, 8s, 16s, 32s later. Fail is all fail
+
+    :param protocol     The protocol to use to reach this file
+    :param pfn          Physical file name of the target for the protocol stat
+    """
+    retries = config_get('client', 'protocol_stat_retries', raise_exception=False, default=6)
+
+    for attempt in range(retries):
+        try:
+            stats = protocol.stat(pfn)
+            return stats
+        except exception.RSEChecksumUnavailable as e:
+            # The stat succeeded here, but the checksum failed
+            raise e
+        except Exception as e:
+            sleep(2**attempt)
+    return protocol.stat(pfn)
+
+
 def __check_compatible_scheme(dest_scheme, src_scheme):
     """
     Check if two schemes are compatible, such as srm and gsiftp


Test patch: