Repository: rucio/rucio

Issue:
Undertaker handling of very larger rules
Motivation
----------
For very large rules, the undertaker is blocked a long time to delete them which leads to long backlogs.


Modification
------------
Instead, for rules > 10k files it could just flag the rule for expiration and set the timeout of the dataset 1 day in the future. Doing this, the judge-cleaner will take care of the rule deletion and thus free up the undertaker to do its job.




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -544,13 +544,16 @@ def attach_dids_to_dids(attachments, account, ignore_duplicate=False, session=No
 
 
 @transactional_session
-def delete_dids(dids, account, session=None):
+def delete_dids(dids, account, expire_rules=False, session=None):
     """
     Delete data identifiers
 
-    :param dids: The list of dids to delete.
-    :param account: The account.
-    :param session: The database session in use.
+    :param dids:          The list of dids to delete.
+    :param account:       The account.
+    :param expire_rules:  Expire large rules instead of deleting them right away. This should only be used in Undertaker mode, as it can be that
+                          the method returns normally, but a did was not deleted; This trusts in the fact that the undertaker will retry an
+                          expired did.
+    :param session:       The database session in use.
     """
     rule_id_clause, content_clause = [], []
     parent_content_clause, did_clause = [], []
@@ -613,19 +616,34 @@ def delete_dids(dids, account, session=None):
                               'name': did['name']},
                     session=session)
     # Delete rules on did
+    skip_deletion = False  # Skip deletion in case of expiration of a rule
     if rule_id_clause:
         with record_timer_block('undertaker.rules'):
-            for (rule_id, scope, name, rse_expression) in session.query(models.ReplicationRule.id,
-                                                                        models.ReplicationRule.scope,
-                                                                        models.ReplicationRule.name,
-                                                                        models.ReplicationRule.rse_expression).filter(or_(*rule_id_clause)):
+            for (rule_id, scope, name, rse_expression, locks_ok_cnt, locks_replicating_cnt, locks_stuck_cnt) in session.query(models.ReplicationRule.id,
+                                                                                                                              models.ReplicationRule.scope,
+                                                                                                                              models.ReplicationRule.name,
+                                                                                                                              models.ReplicationRule.rse_expression,
+                                                                                                                              models.ReplicationRule.locks_ok_cnt,
+                                                                                                                              models.ReplicationRule.locks_replicating_cnt,
+                                                                                                                              models.ReplicationRule.locks_stuck_cnt).filter(or_(*rule_id_clause)):
                 logging.debug('Removing rule %s for did %s:%s on RSE-Expression %s' % (str(rule_id), scope, name, rse_expression))
+
                 # Propagate purge_replicas from did to rules
                 if (scope, name) in not_purge_replicas:
                     purge_replicas = False
                 else:
                     purge_replicas = True
-                rucio.core.rule.delete_rule(rule_id=rule_id, purge_replicas=purge_replicas, delete_parent=True, nowait=True, session=session)
+                if expire_rules and locks_ok_cnt + locks_replicating_cnt + locks_stuck_cnt > 10000:
+                    # Expire the rule (soft=True)
+                    rucio.core.rule.delete_rule(rule_id=rule_id, purge_replicas=purge_replicas, soft=True, delete_parent=True, nowait=True, session=session)
+                    # Update expiration of did
+                    set_metadata(scope=scope, name=name, key='lifetime', value=3600 * 24, session=session)
+                    skip_deletion = True
+                else:
+                    rucio.core.rule.delete_rule(rule_id=rule_id, purge_replicas=purge_replicas, delete_parent=True, nowait=True, session=session)
+
+    if skip_deletion:
+        return
 
     # Detach from parent dids:
     existing_parent_dids = False
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -870,6 +870,8 @@ def delete_rule(rule_id, purge_replicas=None, soft=False, delete_parent=False, n
                 rule.expires_at = min(datetime.utcnow() + timedelta(seconds=3600), rule.expires_at)
             else:
                 rule.expires_at = datetime.utcnow() + timedelta(seconds=3600)
+            if rule.child_rule_id is not None and delete_parent:
+                rule.child_rule_id = None
             insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
             return
 
diff --git a/lib/rucio/daemons/undertaker/undertaker.py b/lib/rucio/daemons/undertaker/undertaker.py
--- a/lib/rucio/daemons/undertaker/undertaker.py
+++ b/lib/rucio/daemons/undertaker/undertaker.py
@@ -15,7 +15,7 @@
 # Authors:
 # - Vincent Garonne <vgaronne@gmail.com>, 2013-2018
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2015
-# - Martin Barisits <martin.barisits@cern.ch>, 2016
+# - Martin Barisits <martin.barisits@cern.ch>, 2016-2018
 
 '''
 Undertaker is a daemon to manage expired did.
@@ -73,7 +73,7 @@ def undertaker(worker_number=1, total_workers=1, chunk_size=5, once=False):
             for chunk in chunks(dids, chunk_size):
                 try:
                     logging.info('Undertaker(%s): Receive %s dids to delete', worker_number, len(chunk))
-                    delete_dids(dids=chunk, account='root')
+                    delete_dids(dids=chunk, account='root', expire_rules=True)
                     logging.info('Undertaker(%s): Delete %s dids', worker_number, len(chunk))
                     record_counter(counters='undertaker.delete_dids', delta=len(chunk))
                 except RuleNotFound, error:


Test patch: