Repository: rucio/rucio

Issue:
Undertaker: Force deletion of replicas
Motivation
----------
There's a loophole in the undertaker workflow. If a DID collection has N rules and some of these rules are deleted without the `--purge-replicas` option, all the files replicas from these collection will stay with a non-Epoch tombstone. When someone erase the DID, the undertaker remove all the rules with `--purge-replicas` that get an Epoch tombstone, but nothing is done for the replicas from the previously expired rules that can stay around forever.

Modification
------------
Set Epoch tombstone to all the file replicas from the DID that have no locks.




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -607,6 +607,7 @@ def delete_dids(dids, account, expire_rules=False, session=None, logger=logging.
     not_purge_replicas = []
     did_followed_clause = []
     metadata_to_delete = []
+    file_content_clause = []
 
     archive_dids = config_core.get('undertaker', 'archive_dids', default=False, session=session)
 
@@ -617,6 +618,7 @@ def delete_dids(dids, account, expire_rules=False, session=None, logger=logging.
         else:
             did_clause.append(and_(models.DataIdentifier.scope == did['scope'], models.DataIdentifier.name == did['name']))
             content_clause.append(and_(models.DataIdentifierAssociation.scope == did['scope'], models.DataIdentifierAssociation.name == did['name']))
+            file_content_clause.append(and_(models.DataIdentifierAssociation.scope == did['scope'], models.DataIdentifierAssociation.name == did['name'], models.DataIdentifierAssociation.child_type == DIDType.FILE))
             collection_replica_clause.append(and_(models.CollectionReplica.scope == did['scope'],
                                                   models.CollectionReplica.name == did['name']))
             did_followed_clause.append(and_(models.DidsFollowed.scope == did['scope'], models.DidsFollowed.name == did['name']))
@@ -692,7 +694,19 @@ def delete_dids(dids, account, expire_rules=False, session=None, logger=logging.
         with record_timer_block('undertaker.parent_content'):
             for parent_did in session.query(models.DataIdentifierAssociation).filter(or_(*parent_content_clause)):
                 existing_parent_dids = True
-                detach_dids(scope=parent_did.scope, name=parent_did.name, dids=[{'scope': parent_did.child_scope, 'name': parent_did.child_name}], session=session)
+                detach_dids(scope=parent_did.scope, name=parent_did.name, dids=[{'scope': parent_did.child_scope, 'name': parent_did.child_type}], session=session)
+
+    # Set Epoch tombstone for the files replicas inside the did
+    if config_core.get('undertaker', 'purge_all_replicas', default=False, session=session) and file_content_clause:
+        with record_timer_block('undertaker.file_content'):
+            file_replicas_clause = [and_(models.RSEFileAssociation.scope == cont['child_scope'], models.RSEFileAssociation.name == cont['child_name']) for cont in session.query(models.DataIdentifierAssociation).filter(or_(*file_content_clause))]
+            none_value = None  # Hack to get pep8 happy
+            for chunk in chunks(file_replicas_clause, 100):
+                session.query(models.RSEFileAssociation).filter(or_(*chunk)).\
+                    with_hint(models.RSEFileAssociation, text="INDEX(REPLICAS REPLICAS_PK)", dialect_name='oracle').\
+                    filter(models.RSEFileAssociation.lock_cnt == 0).\
+                    filter(models.RSEFileAssociation.tombstone != none_value).\
+                    update({'tombstone': datetime(1970, 1, 1)}, synchronize_session=False)
 
     # Remove content
     if content_clause:


Test patch:
diff --git a/lib/rucio/tests/test_undertaker.py b/lib/rucio/tests/test_undertaker.py
--- a/lib/rucio/tests/test_undertaker.py
+++ b/lib/rucio/tests/test_undertaker.py
@@ -25,6 +25,7 @@
 # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
 # - Simon Fayer <simon.fayer05@imperial.ac.uk>, 2021
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2021
 
 import unittest
 from datetime import datetime, timedelta
@@ -38,9 +39,10 @@
 from rucio.common.utils import generate_uuid
 from rucio.core.account_limit import set_local_account_limit
 from rucio.core.did import add_dids, attach_dids, list_expired_dids, get_did, set_metadata
-from rucio.core.replica import get_replica
+from rucio.core.replica import add_replicas, get_replica
 from rucio.core.rse import get_rse_id, add_rse
 from rucio.core.rule import add_rules, list_rules
+from rucio.daemons.judge.cleaner import rule_cleaner
 from rucio.daemons.undertaker.undertaker import undertaker
 from rucio.db.sqla.util import json_implemented
 from rucio.tests.common import rse_name_generator
@@ -75,7 +77,7 @@ def test_undertaker(self):
         dsns1 = [{'name': 'dsn_%s' % generate_uuid(),
                   'scope': tmp_scope,
                   'type': 'DATASET',
-                  'lifetime': -1} for i in range(nbdatasets)]
+                  'lifetime': -1} for _ in range(nbdatasets)]
 
         dsns2 = [{'name': 'dsn_%s' % generate_uuid(),
                   'scope': tmp_scope,
@@ -83,7 +85,7 @@ def test_undertaker(self):
                   'lifetime': -1,
                   'rules': [{'account': jdoe, 'copies': 1,
                              'rse_expression': rse,
-                             'grouping': 'DATASET'}]} for i in range(nbdatasets)]
+                             'grouping': 'DATASET'}]} for _ in range(nbdatasets)]
 
         add_dids(dids=dsns1 + dsns2, account=root)
 
@@ -96,7 +98,7 @@ def test_undertaker(self):
         for dsn in dsns1 + dsns2:
             files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(),
                       'bytes': 1, 'adler32': '0cc737eb',
-                      'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for i in range(nbfiles)]
+                      'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for _ in range(nbfiles)]
             attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account=root)
             replicas += files
 
@@ -154,14 +156,14 @@ def test_atlas_archival_policy(self):
                   'lifetime': -1,
                   'rules': [{'account': jdoe, 'copies': 1,
                              'rse_expression': rse,
-                             'grouping': 'DATASET'}]} for i in range(nbdatasets)]
+                             'grouping': 'DATASET'}]} for _ in range(nbdatasets)]
 
         add_dids(dids=dsns2, account=root)
 
         replicas = list()
         for dsn in dsns2:
             files = [{'scope': tmp_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1,
-                      'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for i in range(nbfiles)]
+                      'adler32': '0cc737eb', 'tombstone': datetime.utcnow() + timedelta(weeks=2), 'meta': {'events': 10}} for _ in range(nbfiles)]
             attach_dids(scope=tmp_scope, name=dsn['name'], rse_id=rse_id, dids=files, account=root)
             replicas += files
 
@@ -173,3 +175,53 @@ def test_atlas_archival_policy(self):
         for dsn in dsns2:
             assert(get_did(scope=InternalScope('archive', **self.vo), name=dsn['name'])['name'] == dsn['name'])
             assert(len([x for x in list_rules(filters={'scope': InternalScope('archive', **self.vo), 'name': dsn['name']})]) == 1)
+
+
+@pytest.mark.parametrize("core_config_mock", [{"table_content": [
+    ('undertaker', 'purge_all_replicas', True)
+]}], indirect=True)
+@pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
+    'rucio.core.config',
+]}], indirect=True)
+def test_removal_all_replicas2(rse_factory, root_account, mock_scope, core_config_mock, caches_mock):
+    """ UNDERTAKER (CORE): Test the undertaker is setting Epoch tombstone on all the replicas. """
+    rse1, rse1_id = rse_factory.make_posix_rse()
+    rse2, rse2_id = rse_factory.make_posix_rse()
+    dst_rse_name, dst_rse_id = rse_factory.make_posix_rse()
+
+    set_local_account_limit(root_account, rse1_id, -1)
+    set_local_account_limit(root_account, rse2_id, -1)
+
+    nbdatasets = 1
+    nbfiles = 5
+    dsns1 = [{'name': 'dsn_%s' % generate_uuid(),
+              'scope': mock_scope,
+              'type': 'DATASET',
+              'lifetime': -1} for _ in range(nbdatasets)]
+
+    add_dids(dids=dsns1, account=root_account)
+
+    replicas = list()
+    for dsn in dsns1:
+        files = [{'scope': mock_scope,
+                  'name': 'file_%s' % generate_uuid(),
+                  'bytes': 1,
+                  'adler32': '0cc737eb'} for _ in range(nbfiles)]
+        attach_dids(scope=mock_scope, name=dsn['name'], rse_id=rse1_id, dids=files, account=root_account)
+        add_replicas(rse_id=rse2_id, files=files, account=root_account, ignore_availability=True)
+        replicas += files
+
+    add_rules(dids=dsns1, rules=[{'account': root_account, 'copies': 1, 'rse_expression': rse1, 'grouping': 'DATASET'}])
+    add_rules(dids=dsns1, rules=[{'account': root_account, 'copies': 1, 'rse_expression': rse2, 'grouping': 'DATASET', 'lifetime': -86400}])
+
+    # Clean the rules on MOCK2. Replicas are tombstoned with non Epoch
+    rule_cleaner(once=True)
+    for replica in replicas:
+        assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse2_id)['tombstone'] is not None
+    undertaker(worker_number=1, total_workers=1, once=True)
+    undertaker(worker_number=1, total_workers=1, once=True)
+
+    for replica in replicas:
+        assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse1_id)['tombstone'] == datetime(year=1970, month=1, day=1)
+    for replica in replicas:
+        assert get_replica(scope=replica['scope'], name=replica['name'], rse_id=rse2_id)['tombstone'] == datetime(year=1970, month=1, day=1)