Repository: rucio/rucio

Issue:
Oracle optimisation for abacus-collection-replicas
Motivation
----------
The cleanup query for abacus-collection-replicas is still not performing as expected. Further optimisation needed.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -17,7 +17,7 @@
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2013-2018
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013-2014
-# - Martin Barisits <martin.barisits@cern.ch>, 2013-2020
+# - Martin Barisits <martin.barisits@cern.ch>, 2013-2021
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2014-2021
 # - David Cameron <david.cameron@cern.ch>, 2014
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2014-2020
@@ -69,7 +69,7 @@
 from rucio.db.sqla.constants import (DIDType, ReplicaState, OBSOLETE, DIDAvailability,
                                      BadFilesStatus, RuleState, BadPFNStatus)
 from rucio.db.sqla.session import (read_session, stream_session, transactional_session,
-                                   DEFAULT_SCHEMA_NAME)
+                                   DEFAULT_SCHEMA_NAME, BASE)
 from rucio.rse import rsemanager as rsemgr
 
 
@@ -2673,13 +2673,27 @@ def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=
     :param session:            Database session in use.
     :returns:                  List of update requests for collection replicas.
     """
+
+    # Delete update requests which do not have collection_replicas
+    session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.rse_id.is_(None)
+                                                          & ~exists().where(and_(models.CollectionReplica.name == models.UpdatedCollectionReplica.name,  # NOQA: W503
+                                                                                 models.CollectionReplica.scope == models.UpdatedCollectionReplica.scope))).delete(synchronize_session=False)
+    session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.rse_id.isnot(None)
+                                                          & ~exists().where(and_(models.CollectionReplica.name == models.UpdatedCollectionReplica.name,  # NOQA: W503
+                                                                                 models.CollectionReplica.scope == models.UpdatedCollectionReplica.scope,
+                                                                                 models.CollectionReplica.rse_id == models.UpdatedCollectionReplica.rse_id))).delete(synchronize_session=False)
+
     # Delete duplicates
     if session.bind.dialect.name == 'oracle':
-        subquery = session.query(func.max(models.UpdatedCollectionReplica.id)).\
-            group_by(models.UpdatedCollectionReplica.scope,
-                     models.UpdatedCollectionReplica.name,
-                     models.UpdatedCollectionReplica.rse_id).subquery()
-        session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.id.notin_(subquery)).delete(synchronize_session=False)
+        schema = ''
+        if BASE.metadata.schema:
+            schema = BASE.metadata.schema + '.'
+        session.execute('DELETE FROM {schema}updated_col_rep A WHERE A.rowid > ANY (SELECT B.rowid FROM {schema}updated_col_rep B WHERE A.scope = B.scope AND A.name=B.name AND A.did_type=B.did_type AND (A.rse_id=B.rse_id OR (A.rse_id IS NULL and B.rse_id IS NULL)))'.format(schema=schema))
+        # subquery = session.query(func.max(models.UpdatedCollectionReplica.id)).\
+        #     group_by(models.UpdatedCollectionReplica.scope,
+        #              models.UpdatedCollectionReplica.name,
+        #              models.UpdatedCollectionReplica.rse_id).subquery()
+        # session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.id.notin_(subquery)).delete(synchronize_session=False)
     elif session.bind.dialect.name == 'mysql':
         subquery1 = session.query(func.max(models.UpdatedCollectionReplica.id).label('max_id')).\
             group_by(models.UpdatedCollectionReplica.scope,
@@ -2710,15 +2724,6 @@ def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=
         for chunk in chunks(duplicate_request_ids, 100):
             session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.id.in_(chunk)).delete(synchronize_session=False)
 
-    # Delete update requests which do not have collection_replicas
-    session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.rse_id.is_(None)
-                                                          & ~exists().where(and_(models.CollectionReplica.name == models.UpdatedCollectionReplica.name,  # NOQA: W503
-                                                                                 models.CollectionReplica.scope == models.UpdatedCollectionReplica.scope))).delete(synchronize_session=False)
-    session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.rse_id.isnot(None)
-                                                          & ~exists().where(and_(models.CollectionReplica.name == models.UpdatedCollectionReplica.name,  # NOQA: W503
-                                                                                 models.CollectionReplica.scope == models.UpdatedCollectionReplica.scope,
-                                                                                 models.CollectionReplica.rse_id == models.UpdatedCollectionReplica.rse_id))).delete(synchronize_session=False)
-
     query = session.query(models.UpdatedCollectionReplica)
     if limit:
         query = query.limit(limit)


Test patch:
diff --git a/lib/rucio/tests/test_abacus_collection_replica.py b/lib/rucio/tests/test_abacus_collection_replica.py
--- a/lib/rucio/tests/test_abacus_collection_replica.py
+++ b/lib/rucio/tests/test_abacus_collection_replica.py
@@ -91,11 +91,11 @@ def test_abacus_collection_replica_cleanup(self):
         models.CollectionReplica(scope=scope, name=dataset, rse_id=rse_id1, state=ReplicaState.AVAILABLE, bytes=1).save(session=db_session, flush=False)
         models.CollectionReplica(scope=scope, name=dataset, rse_id=rse_id2, state=ReplicaState.AVAILABLE, bytes=1).save(session=db_session, flush=False)
 
-        models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id1).save(session=db_session, flush=False)
-        models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id1).save(session=db_session, flush=False)
-        models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id2).save(session=db_session, flush=False)
-        models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id2).save(session=db_session, flush=False)
-        models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=None).save(session=db_session, flush=False)
+        models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id1, did_type=DIDType.DATASET).save(session=db_session, flush=False)
+        models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id1, did_type=DIDType.DATASET).save(session=db_session, flush=False)
+        models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id2, did_type=DIDType.DATASET).save(session=db_session, flush=False)
+        models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id2, did_type=DIDType.DATASET).save(session=db_session, flush=False)
+        models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=None, did_type=DIDType.DATASET).save(session=db_session, flush=False)
         db_session.commit()
         assert len(get_cleaned_updated_collection_replicas(1, 1)) == 3