Repository: rucio/rucio

Issue:
Improve performance of abacus duplicate deletion query
Motivation
----------
Currently the abacus duplicate deletion query is done in a very resource intensive way. This works fine for smaller Rucio instances using abacus, but it doesn't scale well for larger instance. In comparison to that, the oracle procedure used for oracle-based Rucio instances scales very well. Motivation of this issue is to apply that logic to the more generic abacus query so all Rucio instances can profit from that.





Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -2625,27 +2625,41 @@ def get_cleaned_updated_collection_replicas(total_workers, worker_number, sessio
     :returns:                  List of update requests for collection replicas.
     """
     # Delete duplicates
-    replica_update_requests = session.query(models.UpdatedCollectionReplica)
-    update_requests_with_rse_id = []
-    update_requests_without_rse_id = []
-    duplicate_request_ids = []
-    for update_request in replica_update_requests.all():
-        if update_request.rse_id is not None:
-            small_request = {'name': update_request.name, 'scope': update_request.scope, 'rse_id': update_request.rse_id}
-            if small_request not in update_requests_with_rse_id:
-                update_requests_with_rse_id.append(small_request)
-            else:
-                duplicate_request_ids.append(update_request.id)
-                continue
-        else:
-            small_request = {'name': update_request.name, 'scope': update_request.scope}
-            if small_request not in update_requests_without_rse_id:
-                update_requests_without_rse_id.append(small_request)
+    if session.bind.dialect.name == 'oracle':
+        subquery = session.query(func.max(models.UpdatedCollectionReplica.id)).\
+            group_by(models.UpdatedCollectionReplica.scope,
+                     models.UpdatedCollectionReplica.name,
+                     models.UpdatedCollectionReplica.rse_id).subquery()
+        session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.id.notin_(subquery)).delete(synchronize_session=False)
+    elif session.bind.dialect.name == 'mysql':
+        subquery1 = session.query(func.max(models.UpdatedCollectionReplica.id).label('max_id')).\
+            group_by(models.UpdatedCollectionReplica.scope,
+                     models.UpdatedCollectionReplica.name,
+                     models.UpdatedCollectionReplica.rse_id).subquery()
+        subquery2 = session.query(subquery1.c.max_id).subquery()
+        session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.id.notin_(subquery2)).delete(synchronize_session=False)
+    else:
+        replica_update_requests = session.query(models.UpdatedCollectionReplica)
+        update_requests_with_rse_id = []
+        update_requests_without_rse_id = []
+        duplicate_request_ids = []
+        for update_request in replica_update_requests.all():
+            if update_request.rse_id is not None:
+                small_request = {'name': update_request.name, 'scope': update_request.scope, 'rse_id': update_request.rse_id}
+                if small_request not in update_requests_with_rse_id:
+                    update_requests_with_rse_id.append(small_request)
+                else:
+                    duplicate_request_ids.append(update_request.id)
+                    continue
             else:
-                duplicate_request_ids.append(update_request.id)
-                continue
-    for chunk in chunks(duplicate_request_ids, 100):
-        session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.id.in_(chunk)).delete(synchronize_session=False)
+                small_request = {'name': update_request.name, 'scope': update_request.scope}
+                if small_request not in update_requests_without_rse_id:
+                    update_requests_without_rse_id.append(small_request)
+                else:
+                    duplicate_request_ids.append(update_request.id)
+                    continue
+        for chunk in chunks(duplicate_request_ids, 100):
+            session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.id.in_(chunk)).delete(synchronize_session=False)
 
     # Delete update requests which do not have collection_replicas
     session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.rse_id.is_(None)


Test patch:
diff --git a/lib/rucio/tests/test_abacus_collection_replica.py b/lib/rucio/tests/test_abacus_collection_replica.py
--- a/lib/rucio/tests/test_abacus_collection_replica.py
+++ b/lib/rucio/tests/test_abacus_collection_replica.py
@@ -19,6 +19,7 @@
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+# - Martin Barisits <martin.barisits@cern.ch>, 2020
 
 import os
 import unittest
@@ -28,16 +29,18 @@
 from rucio.client.ruleclient import RuleClient
 from rucio.client.uploadclient import UploadClient
 from rucio.common.config import config_get, config_get_bool
-from rucio.common.types import InternalScope
+from rucio.common.types import InternalScope, InternalAccount
 from rucio.common.utils import generate_uuid
-from rucio.core.replica import delete_replicas
-from rucio.core.rse import get_rse_id
+from rucio.core.did import add_did
+from rucio.core.replica import delete_replicas, get_cleaned_updated_collection_replicas
+from rucio.core.rse import get_rse_id, add_rse
 from rucio.daemons.abacus import collection_replica
 from rucio.daemons.judge import cleaner
 from rucio.daemons.reaper import reaper
 from rucio.daemons.undertaker import undertaker
-from rucio.db.sqla.constants import DIDType
-from rucio.tests.common import file_generator
+from rucio.db.sqla import models, session
+from rucio.db.sqla.constants import DIDType, ReplicaState
+from rucio.tests.common import file_generator, rse_name_generator
 
 
 class TestAbacusCollectionReplica(unittest.TestCase):
@@ -70,6 +73,31 @@ def tearDownClass(cls):
         else:
             reaper.run(once=True, include_rses=cls.rse, greedy=True)
 
+    def test_abacus_collection_replica_cleanup(self):
+        """ ABACUS (COLLECTION REPLICA): Test if the cleanup procedure works correctly. """
+        collection_replica.run(once=True)
+        db_session = session.get_session()
+        rse1 = rse_name_generator()
+        rse_id1 = add_rse(rse1, **self.vo)
+        rse2 = rse_name_generator()
+        rse_id2 = add_rse(rse2, **self.vo)
+
+        scope = InternalScope('mock', **self.vo)
+        dataset = 'dataset_%s' % generate_uuid()
+        jdoe = InternalAccount('jdoe', **self.vo)
+        add_did(scope, dataset, DIDType.from_sym('DATASET'), jdoe)
+
+        models.CollectionReplica(scope=scope, name=dataset, rse_id=rse_id1, state=ReplicaState.AVAILABLE, bytes=1).save(session=db_session, flush=False)
+        models.CollectionReplica(scope=scope, name=dataset, rse_id=rse_id2, state=ReplicaState.AVAILABLE, bytes=1).save(session=db_session, flush=False)
+
+        models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id1).save(session=db_session, flush=False)
+        models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id1).save(session=db_session, flush=False)
+        models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id2).save(session=db_session, flush=False)
+        models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=rse_id2).save(session=db_session, flush=False)
+        models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=None).save(session=db_session, flush=False)
+        db_session.commit()
+        assert len(get_cleaned_updated_collection_replicas(1, 1)) == 3
+
     def test_abacus_collection_replica(self):
         """ ABACUS (COLLECTION REPLICA): Test update of collection replica. """
         self.files = [{'did_scope': self.scope, 'did_name': 'file_' + generate_uuid(), 'path': file_generator(size=self.file_sizes), 'rse': self.rse, 'lifetime': -1} for i in range(0, 2)]