Repository: rucio/rucio

Issue:
Reaper 2.0 traceback
Motivation
----------
```
2019-10-10 09:55:25,486 28542   CRITICAL        Thread [38/50] :  Traceback (most recent call last):
  File "/usr/lib/python2.7/site-packages/rucio/daemons/reaper/reaper2.py", line 425, in reaper
    delete_replicas(rse_id=rse_id, files=deleted_files)
  File "/usr/lib/python2.7/site-packages/rucio/db/sqla/session.py", line 365, in new_funct
    raise DatabaseException(str(error))
DatabaseException: Database exception.
Details: (cx_Oracle.IntegrityError) ORA-02292: integrity constraint (ATLAS_RUCIO.SOURCES_REPLICAS_FK) violated - child record found
[SQL: DELETE /*+ INDEX(REPLICAS REPLICAS_PK) */ FROM atlas_rucio.replicas WHERE atlas_rucio.replicas.rse_id = :rse_id_1 AND (atlas_rucio.replicas.scope = :scope_1 AND atlas_rucio.replicas.name = :name_1 OR atlas_rucio.replicas.scope = :scope_2 AND atlas_rucio.replicas.na
me = :name_2 OR atlas_rucio.replicas.scope = :scope_3 AND atlas_rucio.replicas.name = :name_3 OR atlas_rucio.replicas.scope = :scope_4 AND atlas_rucio.replicas.name = :name_4 OR atlas_rucio.replicas.scope = :scope_5 AND atlas_rucio.replicas.name = :name_5 OR atlas_rucio.
replicas.scope = :scope_6 AND atlas_rucio.replicas.name = :name_6 OR atlas_rucio.replicas.scope = :scope_7 AND atlas_rucio.replicas.name = :name_7 OR atlas_rucio.replicas.scope = :scope_8 AND atlas_rucio.replicas.name = :name_8 OR atlas_rucio.replicas.scope = :scope_9 AN
D atlas_rucio.replicas.name = :name_9 OR atlas_rucio.replicas.scope = :scope_10 AND atlas_rucio.replicas.name = :name_10)]
[parameters: {u'name_2': 'DAOD_STDM2.18683656._000507.pool.root.1', u'name_3': 'DAOD_EXOT8.18715924._000001.pool.root.1', u'name_1': 'DAOD_STDM2.18685082._000356.pool.root.1', u'name_6': 'DAOD_EXOT8.18716233._000001.pool.root.1', u'name_7': 'DAOD_EXOT8.18716233._000003.p
ool.root.1', u'name_4': 'DAOD_EXOT8.18715924._000002.pool.root.1', u'name_5': 'DAOD_EXOT8.18715924._000003.pool.root.1', u'scope_10': 'mc16_13TeV', u'name_8': 'DAOD_EXOT8.18715667._000005.pool.root.1', u'name_9': 'DAOD_EXOT8.18716233._000002.pool.root.1', u'scope_6': 'mc
16_13TeV', u'scope_7': 'mc16_13TeV', u'scope_4': 'mc16_13TeV', u'scope_5': 'mc16_13TeV', u'scope_2': 'data16_13TeV', u'scope_3': 'mc16_13TeV', u'scope_1': 'data17_13TeV', u'name_10': 'DAOD_EXOT8.18715667._000012.pool.root.1', u'rse_id_1': <read-only buffer for 0x7fbe1d1d
f618, size -1, offset 0 at 0x7fbe2c4c5570>, u'scope_8': 'mc16_13TeV', u'scope_9': 'mc16_13TeV'}]
(Background on this error at: http://sqlalche.me/e/gkpj)
```
The problem is due to the relaxed constraint to keep files that are being transfered





Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -1377,7 +1377,7 @@ def delete_replicas(rse_id, files, ignore_availability=True, session=None):
         raise exception.ResourceTemporaryUnavailable('%s is temporary unavailable'
                                                      'for deleting' % replica_rse.rse)
 
-    replica_condition, parent_condition, did_condition = [], [], []
+    replica_condition, parent_condition, did_condition, src_condition = [], [], [], []
     clt_replica_condition, dst_replica_condition = [], []
     incomplete_condition, messages, archive_contents_condition = [], [], []
     for file in files:
@@ -1410,7 +1410,18 @@ def delete_replicas(rse_id, files, ignore_availability=True, session=None):
                                                                                                                                            models.DataIdentifier.availability == DIDAvailability.LOST)),
                                                ~exists(select([1]).prefix_with("/*+ INDEX(REPLICAS REPLICAS_PK) */", dialect='oracle')).where(and_(models.RSEFileAssociation.scope == file['scope'], models.RSEFileAssociation.name == file['name']))))
 
+        src_condition.append(and_(models.Source.scope == file['scope'],
+                                  models.Source.name == file['name'],
+                                  models.Source.rse_id == rse_id))
+
     delta, bytes, rowcount = 0, 0, 0
+
+    # WARNING : This should not be necessary since that would mean the replica is used as a source.
+    for chunk in chunks(src_condition, 10):
+        rowcount = session.query(models.Source).\
+            filter(or_(*chunk)).\
+            delete(synchronize_session=False)
+
     for chunk in chunks(replica_condition, 10):
         for (scope, name, rid, replica_bytes) in session.query(models.RSEFileAssociation.scope, models.RSEFileAssociation.name, models.RSEFileAssociation.rse_id, models.RSEFileAssociation.bytes).\
                 with_hint(models.RSEFileAssociation, "INDEX(REPLICAS REPLICAS_PK)", 'oracle').filter(models.RSEFileAssociation.rse_id == rse_id).filter(or_(*chunk)):
@@ -1653,42 +1664,31 @@ def list_and_mark_unlocked_replicas(limit, bytes=None, rse_id=None, delay_second
     """
 
     none_value = None  # Hack to get pep8 happy...
-    subquery = session.query(models.RSEFileAssociation.scope,
-                             models.RSEFileAssociation.name,
-                             models.RSEFileAssociation.path,
-                             models.RSEFileAssociation.bytes,
-                             models.RSEFileAssociation.tombstone,
-                             models.RSEFileAssociation.state,
-                             models.RSEFileAssociation.rse_id).\
+    query = session.query(models.RSEFileAssociation.scope,
+                          models.RSEFileAssociation.name,
+                          models.RSEFileAssociation.path,
+                          models.RSEFileAssociation.bytes,
+                          models.RSEFileAssociation.tombstone,
+                          models.RSEFileAssociation.state).\
         with_hint(models.RSEFileAssociation, "INDEX_RS_ASC(replicas REPLICAS_TOMBSTONE_IDX)  NO_INDEX_FFS(replicas REPLICAS_TOMBSTONE_IDX)", 'oracle').\
-        with_for_update(skip_locked=True).\
         filter(models.RSEFileAssociation.tombstone < datetime.utcnow()).\
         filter(models.RSEFileAssociation.lock_cnt == 0).\
         filter(case([(models.RSEFileAssociation.tombstone != none_value, models.RSEFileAssociation.rse_id), ]) == rse_id).\
         filter(or_(models.RSEFileAssociation.state.in_((ReplicaState.AVAILABLE, ReplicaState.UNAVAILABLE, ReplicaState.BAD)),
                    and_(models.RSEFileAssociation.state == ReplicaState.BEING_DELETED, models.RSEFileAssociation.updated_at < datetime.utcnow() - timedelta(seconds=delay_seconds)))).\
-        order_by(models.RSEFileAssociation.tombstone).subquery()
-    query = session.query(subquery.c.scope,
-                          subquery.c.name,
-                          subquery.c.path,
-                          subquery.c.bytes,
-                          subquery.c.tombstone,
-                          subquery.c.state,
-                          func.count(models.RSEFileAssociation.scope)).\
-        join(models.RSEFileAssociation, and_(models.RSEFileAssociation.scope == subquery.c.scope,
-                                             models.RSEFileAssociation.name == subquery.c.name,
-                                             models.RSEFileAssociation.rse_id != subquery.c.rse_id,
-                                             models.RSEFileAssociation.state == ReplicaState.AVAILABLE)).\
-        group_by(subquery.c.scope, subquery.c.name, subquery.c.path, subquery.c.bytes, subquery.c.tombstone, subquery.c.state).\
-        order_by(subquery.c.tombstone)
+        with_for_update(skip_locked=True).\
+        order_by(models.RSEFileAssociation.tombstone)
 
     needed_space = bytes
     total_bytes, total_files = 0, 0
     rows = []
     replica_clause = []
-    for (scope, name, path, bytes, tombstone, state, cnt) in query.yield_per(1000):
+    for (scope, name, path, bytes, tombstone, state) in query.yield_per(1000):
         # Check if more than one replica is available
-        if cnt > 1:
+        replica_cnt = session.query(func.count(models.RSEFileAssociation.scope)).\
+            filter(and_(models.RSEFileAssociation.scope == scope, models.RSEFileAssociation.name == name, models.RSEFileAssociation.rse_id != rse_id)).one()
+
+        if replica_cnt[0] > 1:
             if state != ReplicaState.UNAVAILABLE:
                 total_bytes += bytes
                 if tombstone != OBSOLETE and needed_space is not None and total_bytes > needed_space:
diff --git a/lib/rucio/daemons/reaper/reaper2.py b/lib/rucio/daemons/reaper/reaper2.py
--- a/lib/rucio/daemons/reaper/reaper2.py
+++ b/lib/rucio/daemons/reaper/reaper2.py
@@ -123,8 +123,8 @@ def delete_from_storage(replicas, prot, rse_info, staging_areas, prepend_str):
                 logging.info('%s Deletion SUCCESS of %s:%s as %s on %s in %s seconds', prepend_str, replica['scope'], replica['name'], replica['pfn'], rse_name, duration)
 
             except SourceNotFound:
-                err_msg = '%s Deletion NOTFOUND of %s:%s as %s on %s' % (prepend_str, replica['scope'], replica['name'], replica['pfn'], rse_name)
-                logging.warning(err_msg)
+                err_msg = 'Deletion NOTFOUND of %s:%s as %s on %s' % (replica['scope'], replica['name'], replica['pfn'], rse_name)
+                logging.warning('%s %s', prepend_str, err_msg)
                 deleted_files.append({'scope': replica['scope'], 'name': replica['name']})
                 if replica['state'] == ReplicaState.AVAILABLE:
                     deletion_dict['reason'] = str(err_msg)
@@ -297,7 +297,7 @@ def reaper(rses, chunk_size=100, once=False, greedy=False,
     prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'] + 1, heart_beat['nr_threads'])
     logging.info('%s Reaper starting', prepend_str)
 
-    time.sleep(10)  # To prevent running on the same partition if all the reapers restart at the same time
+    time.sleep(15)  # To prevent running on the same partition if all the reapers restart at the same time
     heart_beat = live(executable, hostname, pid, hb_thread)
     prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'] + 1, heart_beat['nr_threads'])
     logging.info('%s Reaper started', prepend_str)
@@ -408,7 +408,6 @@ def reaper(rses, chunk_size=100, once=False, greedy=False,
                                 replica['pfn'] = str(rsemgr.lfns2pfns(rse_settings=rse_info,
                                                                       lfns=[{'scope': replica['scope'], 'name': replica['name'], 'path': replica['path']}],
                                                                       operation='delete', scheme=scheme).values()[0])
-                                time.sleep(random.uniform(0, 0.01))
                             except (ReplicaUnAvailable, ReplicaNotFound) as error:
                                 logging.warning('%s Failed get pfn UNAVAILABLE replica %s:%s on %s with error %s', prepend_str, replica['scope'], replica['name'], rse_name, str(error))
                                 replica['pfn'] = None


Test patch: