Repository: rucio/rucio

Issue:
Add an option to archive DID in the reaper workflow
Motivation
----------
Currently deleted DIDs are not automatically archived, in particular for Oracle :
```
        if session.bind.dialect.name != 'oracle':
            rucio.core.did.insert_deleted_dids(chunk, session=session)
```

Modification
------------
Add an option to archive DID in the reaper workflow




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -612,7 +612,7 @@ def delete_dids(dids, account, expire_rules=False, session=None, logger=logging.
     metadata_to_delete = []
     file_content_clause = []
 
-    archive_dids = config_core.get('undertaker', 'archive_dids', default=False, session=session)
+    archive_dids = config_core.get('deletion', 'archive_dids', default=False, session=session)
 
     for did in dids:
         logger(logging.INFO, 'Removing did %(scope)s:%(name)s (%(did_type)s)' % did)
@@ -637,7 +637,8 @@ def delete_dids(dids, account, expire_rules=False, session=None, logger=logging.
             not_purge_replicas.append((did['scope'], did['name']))
 
             # Archive content
-            # Disable for postgres
+        archive_content = config_core.get('deletion', 'archive_content', default=False, session=session)
+        if archive_content:
             insert_content_history(content_clause=[and_(models.DataIdentifierAssociation.scope == did['scope'],
                                                         models.DataIdentifierAssociation.name == did['name'])],
                                    did_created_at=did.get('created_at'),
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -15,7 +15,7 @@
 #
 # Authors:
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2013-2018
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2021
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013-2014
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2021
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2014-2021
@@ -73,6 +73,7 @@
 from rucio.common.types import InternalScope
 from rucio.common.utils import chunks, clean_surls, str_to_date, add_url_query
 from rucio.core.credential import get_signed_url
+from rucio.core import config as config_core
 from rucio.core.rse import get_rse, get_rse_name, get_rse_attribute, get_rse_vo, list_rses
 from rucio.core.rse_counter import decrease, increase
 from rucio.core.rse_expression_parser import parse_expression
@@ -1922,9 +1923,10 @@ def __cleanup_after_replica_deletion(rse_id, files, session=None):
             prefix_with("/*+ INDEX(DIDS DIDS_PK) */", dialect='oracle').\
             where(or_(*chunk)).\
             execution_options(synchronize_session=False)
-        session.execute(stmt)
-        if session.bind.dialect.name != 'oracle':
+        archive_dids = config_core.get('deletion', 'archive_dids', default=False, session=session)
+        if archive_dids:
             rucio.core.did.insert_deleted_dids(chunk, session=session)
+        session.execute(stmt)
 
     # Set is_archive = false on collections which don't have archive children anymore
     for chunk in chunks(clt_is_not_archive_condition, 100):


Test patch:
diff --git a/lib/rucio/tests/test_reaper.py b/lib/rucio/tests/test_reaper.py
--- a/lib/rucio/tests/test_reaper.py
+++ b/lib/rucio/tests/test_reaper.py
@@ -20,13 +20,17 @@
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
 # - Matt Snyder <msnyder@bnl.gov>, 2021
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2021
 
 from datetime import datetime, timedelta
 
 import pytest
+from sqlalchemy import and_, or_
 
 from rucio.api import replica as replica_api
 from rucio.api import rse as rse_api
+from rucio.db.sqla import models
+from rucio.db.sqla.session import get_session
 from rucio.common.config import config_get_bool
 from rucio.common.exception import ReplicaNotFound, DataIdentifierNotFound
 from rucio.common.types import InternalAccount, InternalScope
@@ -57,17 +61,20 @@
                                'delete': 1}}}
 
 
-def __add_test_rse_and_replicas(vo, scope, rse_name, names, file_size):
+def __add_test_rse_and_replicas(vo, scope, rse_name, names, file_size, epoch_tombstone=False):
     rse_id = rse_core.add_rse(rse_name, vo=vo)
 
     rse_core.add_protocol(rse_id=rse_id, parameter=__mock_protocol)
+    tombstone = datetime.utcnow() - timedelta(days=1)
+    if epoch_tombstone:
+        tombstone = datetime(year=1970, month=1, day=1)
 
     dids = []
     for file_name in names:
         dids.append({'scope': scope, 'name': file_name})
         replica_core.add_replica(rse_id=rse_id, scope=scope,
                                  name=file_name, bytes_=file_size,
-                                 tombstone=datetime.utcnow() - timedelta(days=1),
+                                 tombstone=tombstone,
                                  account=InternalAccount('root', vo=vo), adler32=None, md5=None)
     return rse_name, rse_id, dids
 
@@ -364,3 +371,59 @@ def __get_archive_contents_history_count(archive, session=None):
     assert len(list(did_core.list_archive_content(**archive2))) == 0
     assert __get_archive_contents_history_count(archive1) == 4
     assert __get_archive_contents_history_count(archive2) == 3
+
+
+@pytest.mark.noparallel(reason='runs reaper which may impact other tests run in parallel. It resets some memcached values.')
+@pytest.mark.parametrize("core_config_mock", [{"table_content": [
+    ('deletion', 'archive_dids', True), ('deletion', 'archive_content', True)
+]}], indirect=True)
+@pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
+    'rucio.core.config.REGION', 'rucio.core.replica.REGION'
+]}], indirect=True)
+def test_archive_of_deleted_dids(vo, did_factory, root_account, core_config_mock, caches_mock):
+    """ REAPER (DAEMON): Test that the options to keep the did and content history work."""
+    scope = InternalScope('data13_hip', vo=vo)
+    account = root_account
+
+    nb_files = 10
+    file_size = 200  # 2G
+    rse_name, rse_id, dids = __add_test_rse_and_replicas(vo=vo, scope=scope, rse_name=rse_name_generator(),
+                                                         names=['lfn' + generate_uuid() for _ in range(nb_files)], file_size=file_size, epoch_tombstone=True)
+    dataset = did_factory.make_dataset()
+    did_core.attach_dids(dids=dids, account=account, **dataset)
+
+    rse_core.set_rse_limits(rse_id=rse_id, name='MinFreeSpace', value=50 * file_size)
+    assert len(list(replica_core.list_replicas(dids=dids, rse_expression=rse_name))) == nb_files
+
+    # Check first if the reaper does not delete anything if no space is needed
+    REGION.invalidate()
+    rse_core.set_rse_usage(rse_id=rse_id, source='storage', used=nb_files * file_size, free=323000000000)
+    reaper(once=True, rses=[], include_rses=rse_name, exclude_rses=None, greedy=True)
+    assert len(list(replica_core.list_replicas(dids=dids, rse_expression=rse_name))) == 0
+
+    file_clause = []
+    for did in dids:
+        file_clause.append(and_(models.DeletedDataIdentifier.scope == did['scope'], models.DeletedDataIdentifier.name == did['name']))
+
+    session = get_session()
+    query = session.query(models.DeletedDataIdentifier.scope,
+                          models.DeletedDataIdentifier.name,
+                          models.DeletedDataIdentifier.did_type).\
+        filter(or_(*file_clause))
+
+    deleted_dids = list()
+    for did in query.all():
+        print(did)
+        deleted_dids.append(did)
+    assert len(deleted_dids) == len(dids)
+
+    query = session.query(models.DataIdentifierAssociationHistory.child_scope,
+                          models.DataIdentifierAssociationHistory.child_name,
+                          models.DataIdentifierAssociationHistory.child_type).\
+        filter(and_(models.DataIdentifierAssociationHistory.scope == dataset['scope'], models.DataIdentifierAssociationHistory.name == dataset['name']))
+
+    deleted_dids = list()
+    for did in query.all():
+        print(did)
+        deleted_dids.append(did)
+    assert len(deleted_dids) == len(dids)