Repository: rucio/rucio

Issue:
Reaper seems to skip the deletion of the collection replica
Motivation
----------
The reaper seems to skip the deletion of the collection replica when the last replica of a dataset is deleted on a RSE







Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -1777,8 +1777,7 @@ def __cleanup_after_replica_deletion(rse_id, files, session=None):
                          models.CollectionReplica.name == parent_name,
                          exists(select([1]).prefix_with("/*+ INDEX(DIDS DIDS_PK) */", dialect='oracle')).where(
                              and_(models.DataIdentifier.scope == parent_scope,
-                                  models.DataIdentifier.name == parent_name,
-                                  models.DataIdentifier.is_open == False)),  # NOQA
+                                  models.DataIdentifier.name == parent_name)),
                          ~exists(select([1]).prefix_with("/*+ INDEX(CONTENTS CONTENTS_PK) */", dialect='oracle')).where(
                              and_(models.DataIdentifierAssociation.scope == parent_scope,
                                   models.DataIdentifierAssociation.name == parent_name))))
@@ -1792,16 +1791,28 @@ def __cleanup_after_replica_deletion(rse_id, files, session=None):
                                   models.DataIdentifierAssociation.name == parent_name))))
 
                 # 3) Schedule removal of the entry from the DIDs table
-                did_condition.append(
-                    and_(models.DataIdentifier.scope == parent_scope,
-                         models.DataIdentifier.name == parent_name,
-                         models.DataIdentifier.is_open == False,  # NOQA
-                         ~exists([1]).where(
-                             and_(models.DataIdentifierAssociation.child_scope == parent_scope,
-                                  models.DataIdentifierAssociation.child_name == parent_name)),
-                         ~exists([1]).where(
-                             and_(models.DataIdentifierAssociation.scope == parent_scope,
-                                  models.DataIdentifierAssociation.name == parent_name))))
+                remove_open_did = config_get('reaper', 'remove_open_did', default=False, session=session)
+                if remove_open_did:
+                    did_condition.append(
+                        and_(models.DataIdentifier.scope == parent_scope,
+                             models.DataIdentifier.name == parent_name,
+                             ~exists([1]).where(
+                                 and_(models.DataIdentifierAssociation.child_scope == parent_scope,
+                                      models.DataIdentifierAssociation.child_name == parent_name)),
+                             ~exists([1]).where(
+                                 and_(models.DataIdentifierAssociation.scope == parent_scope,
+                                      models.DataIdentifierAssociation.name == parent_name))))
+                else:
+                    did_condition.append(
+                        and_(models.DataIdentifier.scope == parent_scope,
+                             models.DataIdentifier.name == parent_name,
+                             models.DataIdentifier.is_open == False,  # NOQA
+                             ~exists([1]).where(
+                                 and_(models.DataIdentifierAssociation.child_scope == parent_scope,
+                                      models.DataIdentifierAssociation.child_name == parent_name)),
+                             ~exists([1]).where(
+                                 and_(models.DataIdentifierAssociation.scope == parent_scope,
+                                      models.DataIdentifierAssociation.name == parent_name))))
 
         if child_did_condition:
 
diff --git a/lib/rucio/daemons/abacus/collection_replica.py b/lib/rucio/daemons/abacus/collection_replica.py
--- a/lib/rucio/daemons/abacus/collection_replica.py
+++ b/lib/rucio/daemons/abacus/collection_replica.py
@@ -19,6 +19,7 @@
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 # - Martin Barisits <martin.barisits@cern.ch>, 2020
 # - David Población Criado <david.poblacion.criado@cern.ch>, 2021
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2021
 
 """
 Abacus-Collection-Replica is a daemon to update collection replica.


Test patch:
diff --git a/lib/rucio/tests/temp_factories.py b/lib/rucio/tests/temp_factories.py
--- a/lib/rucio/tests/temp_factories.py
+++ b/lib/rucio/tests/temp_factories.py
@@ -17,6 +17,7 @@
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
 # - Mayank Sharma <mayank.sharma@cern.ch>, 2021
 # - David Población Criado <david.poblacion.criado@cern.ch>, 2021
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2021
 
 import os
 import shutil
@@ -273,10 +274,10 @@ def make_container(self, scope=None):
         self.client.add_container(scope=did['scope'].external, name=did['name'])
         return did
 
-    def upload_test_file(self, rse_name, scope=None, name=None, path=None, return_full_item=False):
+    def upload_test_file(self, rse_name, scope=None, name=None, path=None, size=2, return_full_item=False):
         scope = self._sanitize_or_set_scope(scope)
         if not path:
-            path = file_generator()
+            path = file_generator(size=size)
         if not name:
             name = os.path.basename(path)
         item = {
@@ -291,6 +292,29 @@ def upload_test_file(self, rse_name, scope=None, name=None, path=None, return_fu
         self.created_dids.append(did)
         return item if return_full_item else did
 
+    def upload_test_dataset(self, rse_name, scope=None, size=2, nb_files=2):
+        scope = self._sanitize_or_set_scope(scope)
+        dataset = self.make_dataset(scope=scope)
+        did = {'scope': scope, 'name': dataset['name']}
+        self.created_dids.append(did)
+        items = list()
+        for _ in range(0, nb_files):
+            path = file_generator(size=size)
+            name = os.path.basename(path)
+            items.append({
+                         'path': path,
+                         'rse': rse_name,
+                         'dataset_scope': str(scope),
+                         'dataset_name': dataset['name'],
+                         'did_scope': str(scope),
+                         'did_name': name,
+                         'guid': generate_uuid(),
+                         })
+            did = {'scope': scope, 'name': name}
+            self.created_dids.append(did)
+        self.upload_client.upload(items)
+        return items
+
 
 class TemporaryFileFactory:
     """
diff --git a/lib/rucio/tests/test_abacus_collection_replica.py b/lib/rucio/tests/test_abacus_collection_replica.py
--- a/lib/rucio/tests/test_abacus_collection_replica.py
+++ b/lib/rucio/tests/test_abacus_collection_replica.py
@@ -22,6 +22,7 @@
 # - Martin Barisits <martin.barisits@cern.ch>, 2020-2021
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2020
 # - Simon Fayer <simon.fayer05@imperial.ac.uk>, 2021
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2021
 
 import os
 import unittest
@@ -33,9 +34,10 @@
 from rucio.client.ruleclient import RuleClient
 from rucio.client.uploadclient import UploadClient
 from rucio.common.config import config_get_bool
+from rucio.common.exception import DataIdentifierNotFound
 from rucio.common.types import InternalScope, InternalAccount
 from rucio.common.utils import generate_uuid
-from rucio.core.did import add_did
+from rucio.core.did import add_did, get_did
 from rucio.core.replica import delete_replicas, get_cleaned_updated_collection_replicas
 from rucio.core.rse import get_rse_id, add_rse
 from rucio.daemons.abacus import collection_replica
@@ -103,10 +105,11 @@ def test_abacus_collection_replica_cleanup(self):
         models.UpdatedCollectionReplica(scope=scope, name=dataset, rse_id=None, did_type=DIDType.DATASET).save(session=db_session, flush=False)
         db_session.commit()
         assert len(get_cleaned_updated_collection_replicas(1, 1)) == 3
+        self.did_client.set_metadata(scope.external, dataset, 'lifetime', -1)
 
     def test_abacus_collection_replica(self):
         """ ABACUS (COLLECTION REPLICA): Test update of collection replica. """
-        self.files = [{'did_scope': self.scope, 'did_name': 'file_' + generate_uuid(), 'path': file_generator(size=self.file_sizes), 'rse': self.rse, 'lifetime': -1} for i in range(0, 2)]
+        self.files = [{'did_scope': self.scope, 'did_name': 'file_' + generate_uuid(), 'path': file_generator(size=self.file_sizes), 'rse': self.rse, 'lifetime': -1} for _ in range(0, 2)]
         self.did_client.add_did(self.scope, self.dataset, DIDType.DATASET, lifetime=-1)
         self.upload_client.upload(self.files)
         self.did_client.attach_dids(scope=self.scope, name=self.dataset, dids=[{'name': file['did_name'], 'scope': file['did_scope']} for file in self.files])
@@ -145,9 +148,9 @@ def test_abacus_collection_replica(self):
         assert str(dataset_replica['state']) == 'UNAVAILABLE'
 
         # Delete all files -> collection replica should be deleted
-        from rucio.daemons.reaper.reaper import REGION
-        REGION.invalidate()
+        # Old behaviour (doesn't delete the DID)
         cleaner.run(once=True)
+        reaper.REGION.invalidate()
         if self.vo:
             reaper.run(once=True, include_rses='vo=%s&(%s)' % (self.vo['vo'], self.rse), greedy=True)
         else:
@@ -155,4 +158,62 @@ def test_abacus_collection_replica(self):
         self.rule_client.add_replication_rule([{'scope': self.scope, 'name': self.dataset}], 1, self.rse, lifetime=-1)
         collection_replica.run(once=True)
         dataset_replica = [replica for replica in self.replica_client.list_dataset_replicas(self.scope, self.dataset)]
-        assert len(dataset_replica) == 0
+        assert dataset_replica[0]['length'] == 0
+        assert dataset_replica[0]['available_length'] == 0
+
+
+@pytest.mark.parametrize("core_config_mock", [{"table_content": [
+    ('reaper', 'remove_open_did', True)
+]}], indirect=True)
+@pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
+    'rucio.core.config', 'rucio.core.replica'
+]}], indirect=True)
+def test_abacus_collection_replica_new(vo, rse_factory, rucio_client, did_factory, core_config_mock, caches_mock):
+    """ ABACUS (COLLECTION REPLICA): Test update of collection replica. """
+    file_sizes = 2
+    nfiles = 2
+    dataset_scope = 'mock'
+    rse, rse_id = rse_factory.make_posix_rse()
+    dids = did_factory.upload_test_dataset(rse_name=rse, scope=dataset_scope, size=file_sizes, nb_files=nfiles)
+    files = [{'scope': did['did_scope'], 'name': did['did_name']} for did in dids]
+    dataset = dids[0]['dataset_name']
+    rucio_client.set_metadata(dataset_scope, dataset, 'lifetime', -1)
+    rucio_client.add_replication_rule([{'scope': dataset_scope, 'name': dataset}], 1, rse, lifetime=-1)
+
+    # Check dataset replica after rule creation - initial data
+    dataset_replica = [replica for replica in rucio_client.list_dataset_replicas(dataset_scope, dataset)][0]
+    assert dataset_replica['bytes'] == 0
+    assert dataset_replica['length'] == 0
+    assert dataset_replica['available_bytes'] == 0
+    assert dataset_replica['available_length'] == 0
+    assert str(dataset_replica['state']) == 'UNAVAILABLE'
+
+    # Run Abacus
+    collection_replica.run(once=True)
+
+    # Check dataset replica after abacus - abacus should update the collection_replica table from updated_col_rep
+    dataset_replica = [replica for replica in rucio_client.list_dataset_replicas(dataset_scope, dataset)][0]
+    assert dataset_replica['bytes'] == len(files) * file_sizes
+    assert dataset_replica['length'] == len(files)
+    assert dataset_replica['available_bytes'] == len(files) * file_sizes
+    assert dataset_replica['available_length'] == len(files)
+    assert str(dataset_replica['state']) == 'AVAILABLE'
+
+    # Delete one file -> collection replica should be unavailable
+    cleaner.run(once=True)
+    delete_replicas(rse_id=rse_id, files=[{'name': files[0]['name'], 'scope': InternalScope(dataset_scope, vo)}])
+    rucio_client.add_replication_rule([{'scope': dataset_scope, 'name': dataset}], 1, rse, lifetime=-1)
+    collection_replica.run(once=True)
+    dataset_replica = [replica for replica in rucio_client.list_dataset_replicas(dataset_scope, dataset)][0]
+    assert dataset_replica['length'] == len(files)
+    assert dataset_replica['bytes'] == len(files) * file_sizes
+    assert dataset_replica['available_length'] == len(files) - 1
+    assert dataset_replica['available_bytes'] == (len(files) - 1) * file_sizes
+    assert str(dataset_replica['state']) == 'UNAVAILABLE'
+
+    # Delete all files -> collection replica should be deleted
+    # New behaviour (dataset should be deleted)
+    cleaner.run(once=True)
+    delete_replicas(rse_id=rse_id, files=[{'name': files[1]['name'], 'scope': InternalScope(dataset_scope, vo)}])
+    with pytest.raises(DataIdentifierNotFound):
+        get_did(scope=InternalScope(dataset_scope), name=dataset)
diff --git a/lib/rucio/tests/test_dataset_replicas.py b/lib/rucio/tests/test_dataset_replicas.py
--- a/lib/rucio/tests/test_dataset_replicas.py
+++ b/lib/rucio/tests/test_dataset_replicas.py
@@ -15,7 +15,7 @@
 #
 # Authors:
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2015
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2015
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2015-2021
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2018-2020
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 # - Martin Barisits <martin.barisits@cern.ch>, 2019
@@ -27,6 +27,7 @@
 import unittest
 
 import pytest
+from sqlalchemy.orm.exc import NoResultFound
 
 from rucio.client.didclient import DIDClient
 from rucio.client.replicaclient import ReplicaClient
@@ -35,7 +36,7 @@
 from rucio.common.exception import InvalidObject
 from rucio.common.types import InternalAccount, InternalScope
 from rucio.common.utils import generate_uuid
-from rucio.core.did import attach_dids, add_did, add_dids
+from rucio.core.did import attach_dids, add_dids
 from rucio.core.replica import list_datasets_per_rse, update_collection_replica, \
     get_cleaned_updated_collection_replicas, delete_replicas, add_replicas
 from rucio.core.rse import add_rse, del_rse, add_protocol, get_rse_id
@@ -234,7 +235,7 @@ def test_update_collection_replica(self):
         files = [{'name': 'file_%s' % generate_uuid(), 'scope': self.scope, 'bytes': file_size} for i in range(0, 2)]
         dataset_name = 'dataset_test_%s' % generate_uuid()
         add_replicas(rse_id=self.rse_id, files=files, account=self.account, session=self.db_session)
-        add_did(scope=self.scope, name=dataset_name, did_type=constants.DIDType.DATASET, account=self.account, session=self.db_session)
+        add_dids([{'scope': self.scope, 'name': dataset_name, 'type': constants.DIDType.DATASET}], account=self.account, session=self.db_session)
         attach_dids(scope=self.scope, name=dataset_name, dids=files, account=self.account, session=self.db_session)
         models.CollectionReplica(rse_id=self.rse_id, scope=self.scope, state=constants.ReplicaState.AVAILABLE, name=dataset_name, did_type=constants.DIDType.DATASET, bytes=len(files) * file_size, length=len(files), available_replicas_cnt=0)\
               .save(session=self.db_session)
@@ -277,14 +278,15 @@ def test_update_collection_replica(self):
         assert dataset_replica['available_replicas_cnt'] == len(files)
         assert dataset_replica['state'] == ReplicaState.AVAILABLE
 
+        # Old behaviour, open empty datasets are not deleted
         # Delete all file replicas -> dataset replica should be deleted
         delete_replicas(rse_id=self.rse_id, files=files, session=self.db_session)
-        update_request = self.db_session.query(models.UpdatedCollectionReplica).filter_by(rse_id=self.rse_id, scope=self.scope, name=dataset_name).one()  # pylint: disable=no-member
-        update_collection_replica(update_request=update_request.to_dict(), session=self.db_session)
-        dataset_replica = self.db_session.query(models.CollectionReplica).filter_by(scope=self.scope, name=dataset_name).all()  # pylint: disable=no-member
-        assert len(dataset_replica) == 0
+        with pytest.raises(NoResultFound):
+            update_collection_replica(update_request=update_request.to_dict(), session=self.db_session)
 
         # Update request without rse_id - using two replicas per file -> total 4 replicas
+        dataset_name = 'dataset_test_%s' % generate_uuid()
+        add_dids([{'scope': self.scope, 'name': dataset_name, 'type': constants.DIDType.DATASET}], account=self.account, session=self.db_session)
         add_replicas(rse_id=self.rse_id, files=files, account=self.account, session=self.db_session)
         add_replicas(rse_id=self.rse2_id, files=files, account=self.account, session=self.db_session)
         attach_dids(scope=self.scope, name=dataset_name, dids=files, account=self.account, session=self.db_session)
diff --git a/lib/rucio/tests/test_replica.py b/lib/rucio/tests/test_replica.py
--- a/lib/rucio/tests/test_replica.py
+++ b/lib/rucio/tests/test_replica.py
@@ -17,7 +17,7 @@
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2013-2018
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013-2014
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2020
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2018
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2021
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2014
 # - Martin Barisits <martin.barisits@cern.ch>, 2015-2021
 # - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2018
@@ -249,10 +249,11 @@ def test_delete_replicas(self, rse_factory, mock_scope, root_account):
         for file in files2:
             get_did(scope=file['scope'], name=file['name'])
 
-    def test_delete_replicas_from_datasets(self, rse_factory, mock_scope, root_account):
+    def test_delete_replicas_from_datasets_old(self, rse_factory, mock_scope, root_account):
         """ REPLICA (CORE): Delete replicas from dataset """
         _, rse_id = rse_factory.make_mock_rse()
 
+        cconfig_set(section='reaper', option='remove_open_did', value=False)
         tmp_dsn1 = 'dsn_%s' % generate_uuid()
         tmp_dsn2 = 'dsn_%s' % generate_uuid()
         nbfiles = 5
@@ -608,6 +609,32 @@ def test_get_RSE_coverage_of_dataset(self, rse_factory, mock_scope, root_account
         assert cov[rse3_id] == 500
 
 
+@pytest.mark.parametrize("core_config_mock", [{"table_content": [
+    ('reaper', 'remove_open_did', True)
+]}], indirect=True)
+@pytest.mark.parametrize("caches_mock", [{"caches_to_mock": [
+    'rucio.core.config', 'rucio.core.replica'
+]}], indirect=True)
+def test_delete_replicas_from_datasets_new(core_config_mock, caches_mock, rse_factory, mock_scope, root_account):
+    """ REPLICA (CORE): Delete replicas from dataset """
+    _, rse_id = rse_factory.make_mock_rse()
+
+    tmp_dsn1 = 'dsn_%s' % generate_uuid()
+    nbfiles = 5
+    files1 = [{'scope': mock_scope, 'name': 'file_%s' % generate_uuid(), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for _ in range(nbfiles)]
+
+    add_did(scope=mock_scope, name=tmp_dsn1, did_type=DIDType.DATASET, account=root_account)
+
+    attach_dids(scope=mock_scope, name=tmp_dsn1, rse_id=rse_id, dids=files1, account=root_account)
+
+    set_status(scope=mock_scope, name=tmp_dsn1, open=False)
+
+    delete_replicas(rse_id=rse_id, files=files1)
+
+    with pytest.raises(DataIdentifierNotFound):
+        get_did(scope=mock_scope, name=tmp_dsn1)
+
+
 @pytest.mark.noparallel(reason='calls list_bad_replicas() and runs necromancer. Both act on all bad replicas without any filtering')
 def test_client_add_list_bad_replicas(rse_factory, replica_client, did_client):
     """ REPLICA (CLIENT): Add bad replicas"""