Repository: rucio/rucio

Issue:
Reason is not maintained for temporarily unavailable replicas
Motivation
----------

https://github.com/rucio/rucio/blob/master/lib/rucio/daemons/badreplicas/minos.py#L222

This doesn’t look intentional. The entries in `BadPFNs` are dropped, so the reason is lost forever.

Modification
------------

Extra care should be taken for entries with different reason in the same chunk, though the chance of this occurring should be very low.



Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/badreplicas/minos.py b/lib/rucio/daemons/badreplicas/minos.py
--- a/lib/rucio/daemons/badreplicas/minos.py
+++ b/lib/rucio/daemons/badreplicas/minos.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 # Authors:
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2018-2019
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2018-2021
 # - Martin Barisits <martin.barisits@cern.ch>, 2018-2021
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Brandon White <bjwhite@fnal.gov>, 2019
@@ -32,12 +32,11 @@
 import socket
 import threading
 import time
-import traceback
 from datetime import datetime
 
 import rucio.db.sqla.util
 from rucio.common.exception import UnsupportedOperation, DataIdentifierNotFound, ReplicaNotFound, DatabaseException
-from rucio.common.logging import setup_logging
+from rucio.common.logging import formatted_logger, setup_logging
 from rucio.common.utils import chunks
 from rucio.core import heartbeat
 from rucio.core.did import get_metadata
@@ -48,7 +47,7 @@
 from rucio.db.sqla.constants import BadFilesStatus, BadPFNStatus, ReplicaState
 from rucio.db.sqla.session import get_session
 
-graceful_stop = threading.Event()
+GRACEFUL_STOP = threading.Event()
 
 
 def minos(bulk=1000, once=False, sleep_time=60):
@@ -69,24 +68,27 @@ def minos(bulk=1000, once=False, sleep_time=60):
     hb_thread = threading.current_thread()
     heartbeat.sanity_check(executable=executable, hostname=hostname)
     heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
-    prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
-    logging.info(prepend_str + 'Minos starting')
+    prefix = 'minos[%i/%i] ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
+    logger = formatted_logger(logging.log, prefix + '%s')
+    logger(logging.INFO, 'Minos starting')
 
     time.sleep(10)  # To prevent running on the same partition if all the daemons restart at the same time
     heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
-    prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
+    prefix = 'minos[%i/%i] ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
+    logger = formatted_logger(logging.log, prefix + '%s')
 
     states_mapping = {BadPFNStatus.BAD: BadFilesStatus.BAD,
                       BadPFNStatus.SUSPICIOUS: BadFilesStatus.SUSPICIOUS,
                       BadPFNStatus.TEMPORARY_UNAVAILABLE: BadFilesStatus.TEMPORARY_UNAVAILABLE}
-    logging.info(prepend_str + 'Minos started')
+    logger(logging.INFO, 'Minos started')
 
     chunk_size = 10  # The chunk size used for the commits
 
-    while not graceful_stop.is_set():
+    while not GRACEFUL_STOP.is_set():
         start_time = time.time()
         heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
-        prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
+        prefix = 'minos[%i/%i] ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
+        logger = formatted_logger(logging.log, prefix + '%s')
         pfns = []
         try:
             bad_replicas = {}
@@ -114,7 +116,7 @@ def minos(bulk=1000, once=False, sleep_time=60):
             for account, reason, state in bad_replicas:
                 vo = account.vo
                 pfns = bad_replicas[(account, reason, state)]
-                logging.info(prepend_str + 'Declaring %s replicas with state %s and reason %s' % (len(pfns), str(state), reason))
+                logger(logging.INFO, 'Declaring %s replicas with state %s and reason %s' % (len(pfns), str(state), reason))
                 session = get_session()
                 schemes = {}
                 dict_rse = {}
@@ -135,32 +137,32 @@ def minos(bulk=1000, once=False, sleep_time=60):
                         unknown_replicas.extend(tmp_unknown_replicas.get('unknown', []))
                     # The replicas in unknown_replicas do not exist, so we flush them from bad_pfns
                     if unknown_replicas:
-                        logging.info(prepend_str + 'The following replicas are unknown and will be removed : %s' % str(unknown_replicas))
+                        logger(logging.INFO, 'The following replicas are unknown and will be removed : %s' % str(unknown_replicas))
                         bulk_delete_bad_pfns(pfns=unknown_replicas, session=None)
 
                     for rse_id in dict_rse:
                         vo_str = '' if vo == 'def' else ' on VO ' + vo
-                        logging.debug(prepend_str + 'Running on RSE %s%s with %s replicas' % (get_rse_name(rse_id=rse_id), vo_str, len(dict_rse[rse_id])))
+                        logger(logging.DEBUG, 'Running on RSE %s%s with %s replicas' % (get_rse_name(rse_id=rse_id), vo_str, len(dict_rse[rse_id])))
                         nchunk = 0
                         tot_chunk = int(math.ceil(len(dict_rse[rse_id]) / chunk_size))
                         for chunk in chunks(dict_rse[rse_id], chunk_size):
                             nchunk += 1
-                            logging.debug(prepend_str + 'Running on %s chunk out of %s' % (nchunk, tot_chunk))
+                            logger(logging.DEBUG, 'Running on %s chunk out of %s' % (nchunk, tot_chunk))
                             unknown_replicas = declare_bad_file_replicas(pfns=chunk, reason=reason, issuer=account, status=state, session=session)
                             if unknown_replicas:
-                                logging.debug(prepend_str + 'Unknown replicas : %s' % (str(unknown_replicas)))
+                                logger(logging.DEBUG, 'Unknown replicas : %s' % (str(unknown_replicas)))
                             bulk_delete_bad_pfns(pfns=chunk, session=session)
                             session.commit()  # pylint: disable=no-member
                 except Exception:
                     session.rollback()  # pylint: disable=no-member
-                    logging.critical(traceback.format_exc())
+                    logger(logging.CRITICAL, 'Exception', exc_info=True)
 
             # Now get the temporary unavailable and update the replicas states
             for account, reason, expires_at in temporary_unvailables:
                 vo = account.vo
                 pfns = temporary_unvailables[(account, reason, expires_at)]
-                logging.info(prepend_str + 'Declaring %s replicas temporary available with timeout %s and reason %s' % (len(pfns), str(expires_at), reason))
-                logging.debug(prepend_str + 'Extracting RSEs')
+                logger(logging.INFO, 'Declaring %s replicas temporary available with timeout %s and reason %s' % (len(pfns), str(expires_at), reason))
+                logger(logging.DEBUG, 'Extracting RSEs')
                 schemes = {}
                 dict_rse = {}
                 unknown_replicas = []
@@ -181,14 +183,14 @@ def minos(bulk=1000, once=False, sleep_time=60):
 
                 # The replicas in unknown_replicas do not exist, so we flush them from bad_pfns
                 if unknown_replicas:
-                    logging.info(prepend_str + 'The following replicas are unknown and will be removed : %s' % str(unknown_replicas))
+                    logger(logging.INFO, 'The following replicas are unknown and will be removed : %s' % str(unknown_replicas))
                     bulk_delete_bad_pfns(pfns=unknown_replicas, session=None)
 
                 for rse_id in dict_rse:
                     replicas = []
                     rse = get_rse_name(rse_id=rse_id, session=None)
                     rse_vo_str = rse if vo == 'def' else '{} on {}'.format(rse, vo)
-                    logging.debug(prepend_str + 'Running on RSE %s' % rse_vo_str)
+                    logger(logging.DEBUG, 'Running on RSE %s' % rse_vo_str)
                     for rep in get_did_from_pfns(pfns=dict_rse[rse_id], rse_id=None, vo=vo, session=None):
                         for pfn in rep:
                             scope = rep[pfn]['scope']
@@ -198,24 +200,24 @@ def minos(bulk=1000, once=False, sleep_time=60):
                     # We update the replicas states to TEMPORARY_UNAVAILABLE
                     # then insert a row in the bad_replicas table. TODO Update the row if it already exists
                     # then delete the corresponding rows into the bad_pfns table
-                    logging.debug(prepend_str + 'Running on %s replicas on RSE %s' % (len(replicas), rse_vo_str))
+                    logger(logging.DEBUG, 'Running on %s replicas on RSE %s' % (len(replicas), rse_vo_str))
                     nchunk = 0
                     tot_chunk = int(math.ceil(len(replicas) / float(chunk_size)))
                     session = get_session()
                     for chunk in chunks(replicas, chunk_size):
                         try:
                             nchunk += 1
-                            logging.debug(prepend_str + 'Running on %s chunk out of %s' % (nchunk, tot_chunk))
+                            logger(logging.DEBUG, 'Running on %s chunk out of %s' % (nchunk, tot_chunk))
                             update_replicas_states(chunk, nowait=False, session=session)
-                            bulk_add_bad_replicas(chunk, account, state=BadFilesStatus.TEMPORARY_UNAVAILABLE, reason=None, expires_at=expires_at, session=session)
+                            bulk_add_bad_replicas(chunk, account, state=BadFilesStatus.TEMPORARY_UNAVAILABLE, reason=reason, expires_at=expires_at, session=session)
                             pfns = [entry['pfn'] for entry in chunk]
                             bulk_delete_bad_pfns(pfns=pfns, session=session)
                             session.commit()  # pylint: disable=no-member
                         except (UnsupportedOperation, ReplicaNotFound) as error:
                             session.rollback()  # pylint: disable=no-member
-                            logging.error(prepend_str + 'Problem to bulk update PFNs. PFNs will be updated individually. Error : %s' % str(error))
+                            logger(logging.ERROR, 'Problem to bulk update PFNs. PFNs will be updated individually. Error : %s' % str(error))
                             for rep in chunk:
-                                logging.debug(prepend_str + 'Working on %s' % (str(rep)))
+                                logger(logging.DEBUG, 'Working on %s' % (str(rep)))
                                 try:
                                     get_metadata(rep['scope'], rep['name'])
                                     unavailable_states = []
@@ -223,36 +225,48 @@ def minos(bulk=1000, once=False, sleep_time=60):
                                     unavailable_states.extend(rep_state.get(ReplicaState.TEMPORARY_UNAVAILABLE, []))
                                     unavailable_states.extend(rep_state.get(ReplicaState.BEING_DELETED, []))
                                     unavailable_states.extend(rep_state.get(ReplicaState.BAD, []))
+                                    # If the replica is already not available, it is removed from the bad PFNs table
                                     if rep['rse_id'] in unavailable_states:
-                                        logging.info(prepend_str + '%s is in unavailable state. Will be removed from the list of bad PFNs' % str(rep['pfn']))
+                                        logger(logging.INFO, '%s is in unavailable state. Will be removed from the list of bad PFNs' % str(rep['pfn']))
                                         bulk_delete_bad_pfns(pfns=[rep['pfn']], session=None)
+                                    # If the expiration date of the TEMPORARY_UNAVAILABLE is in the past, it is removed from the bad PFNs table
                                     elif expires_at < datetime.now():
-                                        logging.info('%s PFN %s expiration time (%s) is older than now and is not in unavailable state. Removing the PFNs from bad_pfns', prepend_str, str(rep['pfn']), expires_at)
+                                        logger(logging.INFO, 'PFN %s expiration time (%s) is older than now and is not in unavailable state. Removing the PFNs from bad_pfns' % (str(rep['pfn']), expires_at))
                                         bulk_delete_bad_pfns(pfns=[rep['pfn']], session=None)
+                                    # Else update everything in the same transaction
+                                    else:
+                                        try:
+                                            session = get_session()
+                                            update_replicas_states([rep], nowait=False, session=session)
+                                            bulk_add_bad_replicas([rep], account, state=BadFilesStatus.TEMPORARY_UNAVAILABLE, reason=reason, expires_at=expires_at, session=session)
+                                            bulk_delete_bad_pfns(pfns=[rep['pfn']], session=session)
+                                            session.commit()  # pylint: disable=no-member
+                                        except Exception:
+                                            logger(logging.ERROR, 'Cannot update state of %s' % str(rep['pfn']))
                                 except (DataIdentifierNotFound, ReplicaNotFound):
-                                    logging.error(prepend_str + 'Will remove %s from the list of bad PFNs' % str(rep['pfn']))
+                                    logger(logging.ERROR, 'Will remove %s from the list of bad PFNs' % str(rep['pfn']))
                                     bulk_delete_bad_pfns(pfns=[rep['pfn']], session=None)
                             session = get_session()
                         except Exception:
                             session.rollback()  # pylint: disable=no-member
-                            logging.critical(traceback.format_exc())
+                            logger(logging.CRITICAL, 'Exception', exc_info=True)
                             session = get_session()
 
         except Exception as error:
-            logging.error(prepend_str + '%s' % (str(error)))
+            logger(logging.ERROR, '%s' % (str(error)))
 
         tottime = time.time() - start_time
         if once:
             break
         if len(pfns) == bulk:
-            logging.info(prepend_str + 'Processed maximum number of pfns according to the bulk size. Restart immediately next cycle')
+            logger(logging.INFO, 'Processed maximum number of pfns according to the bulk size. Restart immediately next cycle')
         elif tottime < sleep_time:
-            logging.info(prepend_str + 'Will sleep for %s seconds' % (sleep_time - tottime))
+            logger(logging.INFO, 'Will sleep for %s seconds' % (sleep_time - tottime))
             time.sleep(sleep_time - tottime)
 
     heartbeat.die(executable, hostname, pid, hb_thread)
-    logging.info(prepend_str + 'Graceful stop requested')
-    logging.info(prepend_str + 'Graceful stop done')
+    logger(logging.INFO, 'Graceful stop requested')
+    logger(logging.INFO, 'Graceful stop done')
 
 
 def run(threads=1, bulk=100, once=False, sleep_time=60):
@@ -283,4 +297,4 @@ def stop(signum=None, frame=None):
     """
     Graceful exit.
     """
-    graceful_stop.set()
+    GRACEFUL_STOP.set()


Test patch: