Repository: rucio/rucio

Issue:
Filtering of blocklisted updated_dids needs to be added to sql query
Motivation
----------
Currently the filtering of blocklisted updated_dids is done on the daemon side. However, for datasets with lots of updated_dids in the queue this does not scale well. The blocklisted datasets should be filtered directly in the SQL query.
Crash when enabling multihop mode
Motivation
----------
The submitter crash when enabling multihop mode








Discussion:

Resolving patch:
diff --git a/lib/rucio/client/uploadclient.py b/lib/rucio/client/uploadclient.py
--- a/lib/rucio/client/uploadclient.py
+++ b/lib/rucio/client/uploadclient.py
@@ -37,7 +37,7 @@
 from rucio.common.exception import (RucioException, RSEBlacklisted, DataIdentifierAlreadyExists,
                                     DataIdentifierNotFound, NoFilesUploaded, NotAllFilesUploaded,
                                     ResourceTemporaryUnavailable, ServiceUnavailable, InputValidationError)
-from rucio.common.utils import adler32, execute, generate_uuid, md5, send_trace, GLOBALLY_SUPPORTED_CHECKSUMS
+from rucio.common.utils import adler32, detect_client_location, execute, generate_uuid, md5, send_trace, GLOBALLY_SUPPORTED_CHECKSUMS
 from rucio.rse import rsemanager as rsemgr
 from rucio import version
 
@@ -57,6 +57,7 @@ def __init__(self, _client=None, logger=None, tracing=True):
 
         self.logger = logger
         self.client = _client if _client else Client()
+        self.client_location = detect_client_location()
         self.tracing = tracing
         if not self.tracing:
             logger.debug('Tracing is turned off.')
@@ -196,7 +197,20 @@ def upload(self, items, summary_file_path=None, traces_copy_out=None):
                     logger.info('File already exists on RSE. Skipping upload')
                     trace['stateReason'] = 'File already exists'
                     continue
-            protocols = rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=force_scheme)
+
+            # resolving local area networks
+            domain = 'wan'
+            rse_attributes = {}
+            try:
+                rse_attributes = self.client.list_rse_attributes(rse)
+            except:
+                logger.warning('Attributes of the RSE: %s not available.' % rse)
+            if (self.client_location and 'lan' in rse_settings['domain'] and 'site' in rse_attributes):
+                if self.client_location['site'] == rse_attributes['site']:
+                    domain = 'lan'
+
+            # protocol handling and upload
+            protocols = rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=force_scheme, domain=domain)
             protocols.reverse()
             success = False
             state_reason = ''
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -15,7 +15,7 @@
 # Authors:
 # - Vincent Garonne <vgaronne@gmail.com>, 2012-2018
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2018
-# - Martin Barisits <martin.barisits@cern.ch>, 2013-2019
+# - Martin Barisits <martin.barisits@cern.ch>, 2013-2020
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2019
 # - David Cameron <d.g.cameron@gmail.com>, 2014
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2014-2018
@@ -46,7 +46,7 @@
 from sqlalchemy.exc import IntegrityError, StatementError
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.sql import func
-from sqlalchemy.sql.expression import and_, or_, bindparam, text, true, null
+from sqlalchemy.sql.expression import and_, or_, bindparam, text, true, null, tuple_
 
 from rucio.core.account import has_account_attribute
 import rucio.core.did
@@ -63,7 +63,7 @@
                                     ManualRuleApprovalBlocked, UnsupportedOperation, UndefinedPolicy)
 from rucio.common.schema import validate_schema
 from rucio.common.types import InternalScope, InternalAccount
-from rucio.common.utils import str_to_date, sizefmt
+from rucio.common.utils import str_to_date, sizefmt, chunks
 from rucio.core import account_counter, rse_counter, request as request_core
 from rucio.core.account import get_account
 from rucio.core.lifetime_exception import define_eol
@@ -1513,6 +1513,11 @@ def get_updated_dids(total_workers, worker_number, limit=100, blacklisted_dids=[
         elif session.bind.dialect.name == 'postgresql':
             query = query.filter(text('mod(abs((\'x\'||md5(name))::bit(32)::int), %s) = %s' % (total_workers + 1, worker_number)))
 
+    # Remove blacklisted dids from query, but only do the first 30 ones, not to overload the query
+    if blacklisted_dids:
+        chunk = list(chunks(blacklisted_dids, 30))[0]
+        query = query.filter(tuple_(models.UpdatedDID.scope, models.UpdatedDID.name).notin_(chunk))
+
     if limit:
         fetched_dids = query.order_by(models.UpdatedDID.created_at).limit(limit).all()
         filtered_dids = [did for did in fetched_dids if (did.scope, did.name) not in blacklisted_dids]
diff --git a/lib/rucio/daemons/judge/evaluator.py b/lib/rucio/daemons/judge/evaluator.py
--- a/lib/rucio/daemons/judge/evaluator.py
+++ b/lib/rucio/daemons/judge/evaluator.py
@@ -89,7 +89,7 @@ def re_evaluator(once=False):
                                     worker_number=heartbeat['assign_thread'],
                                     limit=100,
                                     blacklisted_dids=[(InternalScope(key[0], fromExternal=False), key[1]) for key in paused_dids])
-            logging.debug('re_evaluator[%s/%s] index query time %f fetch size is %d' % (heartbeat['assign_thread'], heartbeat['nr_threads'] - 1, time.time() - start, len(dids)))
+            logging.debug('re_evaluator[%s/%s] index query time %f fetch size is %d (%d blacklisted)' % (heartbeat['assign_thread'], heartbeat['nr_threads'] - 1, time.time() - start, len(dids), len([(InternalScope(key[0], fromExternal=False), key[1]) for key in paused_dids])))
 
             # If the list is empty, sent the worker to sleep
             if not dids and not once:
@@ -111,6 +111,10 @@ def re_evaluator(once=False):
                     else:
                         done_dids[did_tag] = []
 
+                    # Jump paused dids
+                    if (did.scope.internal, did.name) in paused_dids:
+                        continue
+
                     try:
                         start_time = time.time()
                         re_evaluate_did(scope=did.scope, name=did.name, rule_evaluation_action=did.rule_evaluation_action)


Test patch: