Repository: rucio/rucio

Issue:
Filtering of blocklisted updated_dids needs to be added to sql query
Motivation
----------
Currently the filtering of blocklisted updated_dids is done on the daemon side. However, for datasets with lots of updated_dids in the queue this does not scale well. The blocklisted datasets should be filtered directly in the SQL query.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -15,7 +15,7 @@
 # Authors:
 # - Vincent Garonne <vgaronne@gmail.com>, 2012-2018
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2018
-# - Martin Barisits <martin.barisits@cern.ch>, 2013-2019
+# - Martin Barisits <martin.barisits@cern.ch>, 2013-2020
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2019
 # - David Cameron <d.g.cameron@gmail.com>, 2014
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2014-2018
@@ -46,7 +46,7 @@
 from sqlalchemy.exc import IntegrityError, StatementError
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.sql import func
-from sqlalchemy.sql.expression import and_, or_, bindparam, text, true, null
+from sqlalchemy.sql.expression import and_, or_, bindparam, text, true, null, tuple_
 
 from rucio.core.account import has_account_attribute
 import rucio.core.did
@@ -63,7 +63,7 @@
                                     ManualRuleApprovalBlocked, UnsupportedOperation, UndefinedPolicy)
 from rucio.common.schema import validate_schema
 from rucio.common.types import InternalScope, InternalAccount
-from rucio.common.utils import str_to_date, sizefmt
+from rucio.common.utils import str_to_date, sizefmt, chunks
 from rucio.core import account_counter, rse_counter, request as request_core
 from rucio.core.account import get_account
 from rucio.core.lifetime_exception import define_eol
@@ -1513,6 +1513,11 @@ def get_updated_dids(total_workers, worker_number, limit=100, blacklisted_dids=[
         elif session.bind.dialect.name == 'postgresql':
             query = query.filter(text('mod(abs((\'x\'||md5(name))::bit(32)::int), %s) = %s' % (total_workers + 1, worker_number)))
 
+    # Remove blacklisted dids from query, but only do the first 30 ones, not to overload the query
+    if blacklisted_dids:
+        chunk = list(chunks(blacklisted_dids, 30))[0]
+        query = query.filter(tuple_(models.UpdatedDID.scope, models.UpdatedDID.name).notin_(chunk))
+
     if limit:
         fetched_dids = query.order_by(models.UpdatedDID.created_at).limit(limit).all()
         filtered_dids = [did for did in fetched_dids if (did.scope, did.name) not in blacklisted_dids]
diff --git a/lib/rucio/daemons/judge/evaluator.py b/lib/rucio/daemons/judge/evaluator.py
--- a/lib/rucio/daemons/judge/evaluator.py
+++ b/lib/rucio/daemons/judge/evaluator.py
@@ -89,7 +89,11 @@ def re_evaluator(once=False):
                                     worker_number=heartbeat['assign_thread'],
                                     limit=100,
                                     blacklisted_dids=[(InternalScope(key[0], fromExternal=False), key[1]) for key in paused_dids])
-            logging.debug('re_evaluator[%s/%s] index query time %f fetch size is %d' % (heartbeat['assign_thread'], heartbeat['nr_threads'] - 1, time.time() - start, len(dids)))
+            logging.debug('re_evaluator[%s/%s] index query time %f fetch size is %d (%d blacklisted)' % (heartbeat['assign_thread'],
+                                                                                                         heartbeat['nr_threads'] - 1,
+                                                                                                         time.time() - start,
+                                                                                                         len(dids),
+                                                                                                         len([(InternalScope(key[0], fromExternal=False), key[1]) for key in paused_dids])))
 
             # If the list is empty, sent the worker to sleep
             if not dids and not once:
@@ -111,6 +115,10 @@ def re_evaluator(once=False):
                     else:
                         done_dids[did_tag] = []
 
+                    # Jump paused dids
+                    if (did.scope.internal, did.name) in paused_dids:
+                        continue
+
                     try:
                         start_time = time.time()
                         re_evaluate_did(scope=did.scope, name=did.name, rule_evaluation_action=did.rule_evaluation_action)


Test patch: