Repository: rucio/rucio
Issue: minos-expiration doesn't handle properly ReplicaNotFound

Motivation
----------
minos-expiration doesn't handle properly ReplicaNotFound






Changed files:
lib/rucio/daemons/badreplicas/minos_temporary_expiration.py

diff --git a/lib/rucio/daemons/badreplicas/minos_temporary_expiration.py b/lib/rucio/daemons/badreplicas/minos_temporary_expiration.py
index 032bafee0..5b7c01739 100644
--- a/lib/rucio/daemons/badreplicas/minos_temporary_expiration.py
+++ b/lib/rucio/daemons/badreplicas/minos_temporary_expiration.py
@@ -19,9 +19,11 @@
 # PY3K COMPATIBLE
 
 import logging
+import math
 import os
 import socket
 import threading
+import traceback
 import time
 
 from sys import stdout, argv
@@ -31,6 +33,8 @@ from rucio.db.sqla.constants import BadFilesStatus, ReplicaState
 from rucio.db.sqla.session import get_session
 from rucio.common.config import config_get
 from rucio.common.utils import chunks
+from rucio.common.exception import DataIdentifierNotFound, ReplicaNotFound
+from rucio.core.did import get_metadata
 from rucio.core.replica import (update_replicas_states,
                                 bulk_delete_bad_replicas, list_expired_temporary_unavailable_replicas)
 
@@ -96,17 +100,48 @@ def minos_tu_expiration(bulk=1000, once=False, sleep_time=60):
                 bad_replicas.append({'scope': replica[0], 'name': replica[1], 'rse_id': replica[2], 'state': BadFilesStatus.TEMPORARY_UNAVAILABLE})
             session = get_session()
 
-            for chunk in chunks(replicas, chunk_size):
+            nchunk = 0
+            tot_chunk = int(math.ceil(len(replicas) / float(chunk_size)))
+            session = get_session()
+            for chunk in chunks(expired_replicas, chunk_size):
                 # Process and update the replicas in chunks
-                update_replicas_states(chunk, nowait=True, session=session)
-            for chunk in chunks(bad_replicas, chunk_size):
+                replicas = [{'scope': replica[0], 'name': replica[1], 'rse_id': replica[2], 'state': ReplicaState.AVAILABLE} for replica in chunk]
                 # Remove the replicas from bad_replicas table in chunks
-                bulk_delete_bad_replicas(chunk, session=session)
-
-            session.commit()  # pylint: disable=no-member
+                bad_replicas = [{'scope': replica[0], 'name': replica[1], 'rse_id': replica[2], 'state': BadFilesStatus.TEMPORARY_UNAVAILABLE} for replica in chunk]
+                try:
+                    nchunk += 1
+                    logging.debug(prepend_str + 'Running on %s chunk out of %s' % (nchunk, tot_chunk))
+                    update_replicas_states(replicas, nowait=True, session=session)
+                    bulk_delete_bad_replicas(bad_replicas, session=session)
+                    session.commit()  # pylint: disable=no-member
+                except (ReplicaNotFound, DataIdentifierNotFound) as error:
+                    session.rollback()  # pylint: disable=no-member
+                    logging.warning(prepend_str + 'One of the replicas does not exist anymore. Updating and deleting one by one. Error : %s' % str(error))
+                    for idx in range(len(chunk)):
+                        logging.debug(prepend_str + 'Working on %s' % (str(replicas[idx])))
+                        try:
+                            get_metadata(replicas[idx]['scope'], replicas[idx]['name'])
+                            update_replicas_states([replicas[idx], ], nowait=True, session=session)
+                            bulk_delete_bad_replicas([bad_replicas[idx], ], session=session)
+                            session.commit()  # pylint: disable=no-member
+                        except DataIdentifierNotFound as error:
+                            session.rollback()  # pylint: disable=no-member
+                            logging.warning(prepend_str + 'DID %s:%s does not exist anymore. ' % (bad_replicas[idx]['scope'], bad_replicas[idx]['name']))
+                            bulk_delete_bad_replicas([bad_replicas[idx], ], session=session)
+                            session.commit()  # pylint: disable=no-member
+                        except ReplicaNotFound as error:
+                            session.rollback()  # pylint: disable=no-member
+                            logging.warning(prepend_str + '%s:%s on RSEID %s does not exist anymore. ' % (replicas[idx]['scope'], replicas[idx]['name'], replicas[idx]['rse_id']))
+                            bulk_delete_bad_replicas([bad_replicas[idx], ], session=session)
+                            session.commit()  # pylint: disable=no-member
+                    session = get_session()
+                except Exception:
+                    session.rollback()  # pylint: disable=no-member
+                    logging.critical(traceback.format_exc())
+                    session = get_session()
 
         except Exception as error:
-            logging.error(prepend_str + '%s' % (str(error)))
+            logging.critical(traceback.format_exc())
 
         tottime = time.time() - start_time
         if once: