Repository: rucio/rucio

Issue:
Archive information on dataset level
Motivation
----------
In recent discussion with developers from ATLAS prodsys the use case was raised that it would be helpful to know if a dataset holds any files which are archives. Eg. via a simple `get_metadata(...)` query.

Right now, Rucio does not propagate this information to the dataset level. There are two fields associated with **file** dids:

- `constituent` which evaluates to `True` if a file is available in an archive (But can have conventional physical replicas as well)
- `is_archive` which evaluates to `True` if a file is an archive, and holds constituent file.

Modification
------------

I think we have two options:

1. Re-use the existing `is_archive` field and propagate this information from the content dids to the archive. Thus, if at least one content file in a dataset has the `is_archive` flag, the dataset will get it too. Needs to be removed again in case of detach operations.
2. Essentially the same as option (1) but with a new field. `contains_archives` - this would need a schema change.

Then this information could simply be queried with the dataset did.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -323,6 +323,24 @@ def __add_files_to_archive(scope, name, files, account, ignore_duplicate=False,
     except IntegrityError as error:
         raise exception.RucioException(error.args)
 
+    archive_did = session.query(models.DataIdentifier). \
+        filter(models.DataIdentifier.did_type == DIDType.FILE). \
+        filter(models.DataIdentifier.scope == scope). \
+        filter(models.DataIdentifier.name == name).\
+        first()
+    if not archive_did.is_archive:
+        # mark tha archive file as is_archive
+        archive_did.is_archive = True
+
+        # mark parent datasets as is_archive = True
+        session.query(models.DataIdentifier).filter(
+            exists(select([1]).prefix_with("/*+ INDEX(CONTENTS CONTENTS_PK) */", dialect="oracle")).where(
+                and_(models.DataIdentifierAssociation.child_scope == scope,
+                     models.DataIdentifierAssociation.child_name == name,
+                     models.DataIdentifierAssociation.scope == models.DataIdentifier.scope,
+                     models.DataIdentifierAssociation.name == models.DataIdentifier.name))
+        ).update({"is_archive": True}, synchronize_session=False)
+
 
 @transactional_session
 def __add_files_to_dataset(scope, name, files, account, rse_id, ignore_duplicate=False, session=None):
@@ -366,6 +384,7 @@ def __add_files_to_dataset(scope, name, files, account, rse_id, ignore_duplicate
             existing_content.append(row)
 
     contents = []
+    added_archives_condition = []
     for file in files:
         if not existing_content or (scope, name, file['scope'], file['name']) not in existing_content:
             contents.append({'scope': scope, 'name': name, 'child_scope': file['scope'],
@@ -374,6 +393,20 @@ def __add_files_to_dataset(scope, name, files, account, rse_id, ignore_duplicate
                              'guid': file['guid'], 'events': file['events'],
                              'md5': file.get('md5'), 'did_type': DIDType.DATASET,
                              'child_type': DIDType.FILE, 'rule_evaluation': True})
+            added_archives_condition.append(
+                and_(models.DataIdentifier.scope == file['scope'],
+                     models.DataIdentifier.name == file['name'],
+                     models.DataIdentifier.is_archive == true()))
+
+    # if any of the attached files is an archive, set is_archive = True on the dataset
+    if session.query(models.DataIdentifier). \
+            with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle'). \
+            filter(or_(*added_archives_condition)). \
+            first() is not None:
+        session.query(models.DataIdentifier). \
+            filter(models.DataIdentifier.scope == scope). \
+            filter(models.DataIdentifier.name == name). \
+            update({'is_archive': True})
 
     try:
         contents and session.bulk_insert_mappings(models.DataIdentifierAssociation, contents)
@@ -516,13 +549,6 @@ def attach_dids_to_dids(attachments, account, ignore_duplicate=False, session=No
                                            account=account,
                                            ignore_duplicate=ignore_duplicate,
                                            session=session)
-
-                    # mark parent dataset as is_archive
-                    session.query(models.DataIdentifier).\
-                        filter(models.DataIdentifier.did_type == DIDType.FILE).\
-                        filter(models.DataIdentifier.scope == attachment['scope']).\
-                        filter(models.DataIdentifier.name == attachment['name']).\
-                        update({'is_archive': True})
                     return
                 raise exception.UnsupportedOperation("Data identifier '%(scope)s:%(name)s' is a file" % attachment)
 
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -1597,7 +1597,7 @@ def __cleanup_after_replica_deletion(rse_id, files, session=None):
     """
     parent_condition, did_condition = [], []
     clt_replica_condition, dst_replica_condition = [], []
-    incomplete_condition, messages, archive_contents_condition = [], [], []
+    incomplete_condition, messages, clt_is_not_archive_condition, archive_contents_condition = [], [], [], []
     for file in files:
 
         # Schedule update of all collections containing this file and having a collection replica in the RSE
@@ -1684,6 +1684,15 @@ def __cleanup_after_replica_deletion(rse_id, files, session=None):
                          models.DataIdentifierAssociation.child_scope == child_scope,
                          models.DataIdentifierAssociation.child_name == child_name))
 
+                # Schedule setting is_archive = False on parents which don't have any children with is_archive == True anymore
+                clt_is_not_archive_condition.append(
+                    and_(models.DataIdentifierAssociation.scope == parent_scope,
+                         models.DataIdentifierAssociation.name == parent_name,
+                         ~exists(select([1]).prefix_with("/*+ INDEX(DIDS DIDS_PK) */", dialect='oracle')).where(
+                             and_(models.DataIdentifier.scope == models.DataIdentifierAssociation.child_scope,
+                                  models.DataIdentifier.name == models.DataIdentifierAssociation.child_name,
+                                  models.DataIdentifier.is_archive == true()))))
+
                 # If the parent dataset/container becomes empty as a result of the child removal
                 # (it was the last children), metadata cleanup has to be done:
                 #
@@ -1851,6 +1860,22 @@ def __cleanup_after_replica_deletion(rse_id, files, session=None):
         if session.bind.dialect.name != 'oracle':
             rucio.core.did.insert_deleted_dids(chunk, session=session)
 
+    # Set is_archive = false on collections which don't have archive children anymore
+    for chunk in chunks(clt_is_not_archive_condition, 100):
+        clt_to_update = list(session
+                             .query(models.DataIdentifierAssociation.scope,
+                                    models.DataIdentifierAssociation.name)
+                             .with_hint(models.DataIdentifierAssociation, "INDEX(CONTENTS CONTENTS_PK)", 'oracle')
+                             .filter(or_(*chunk)))
+        if clt_to_update:
+            session.query(models.DataIdentifier).\
+                with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').\
+                filter(or_(and_(models.DataIdentifier.scope == scope,
+                                models.DataIdentifier.name == name,
+                                models.DataIdentifier.is_archive == true())
+                           for scope, name in clt_to_update)).\
+                update({'is_archive': False}, synchronize_session=False)
+
 
 @transactional_session
 def get_replica(rse_id, scope, name, session=None):


Test patch:
diff --git a/lib/rucio/tests/test_archive.py b/lib/rucio/tests/test_archive.py
--- a/lib/rucio/tests/test_archive.py
+++ b/lib/rucio/tests/test_archive.py
@@ -25,7 +25,8 @@
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
 
 from rucio.common.utils import generate_uuid
-from rucio.core.replica import add_replicas
+from rucio.core.replica import add_replicas, delete_replicas
+from rucio.core.did import attach_dids, get_metadata
 
 
 def test_add_and_list_archive(rse_factory, replica_client, did_client, mock_scope):
@@ -130,3 +131,45 @@ def test_list_archive_contents_at_rse(rse_factory, mock_scope, root_account, did
     res = replica_client.list_replicas(dids=[{'scope': f['scope'], 'name': f['name']} for f in archived_file], metalink=True, rse_expression=rse2, resolve_archives=True)
     assert rse1 not in res
     assert rse2 in res
+
+
+def test_archive_on_dataset_level(rse_factory, did_factory, root_account):
+    rse_name, rse_id = rse_factory.make_xroot_rse()
+
+    dataset1 = did_factory.make_dataset()
+    dataset2 = did_factory.make_dataset()
+    container = did_factory.make_container()
+    attach_dids(dids=[dataset1, dataset2], account=root_account, **container)
+
+    # Add a random file to the datasets to avoid dataset deletion when the archive is deleted
+    a_file = did_factory.random_did()
+    add_replicas(rse_id=rse_id, files=[{**a_file, 'bytes': 500, 'type': 'FILE', 'adler32': 'beefbeef'}], account=root_account)
+    attach_dids(dids=[a_file], account=root_account, **dataset1)
+    attach_dids(dids=[a_file], account=root_account, **dataset2)
+    # adding a non-archive file should not set is_archive=True
+    metadata = get_metadata(**dataset1)
+    assert not metadata['is_archive']
+
+    # Create an archive and its constituents, attach the archive to datasets
+    archive = did_factory.random_did(name_prefix='archive', name_suffix='.zip')
+    add_replicas(rse_id=rse_id, files=[{**archive, 'bytes': 500, 'type': 'FILE', 'adler32': 'beefbeef'}], account=root_account)
+    constituents = [did_factory.random_did() for _ in range(2)]
+    # Add archive to one dataset _before_ attaching files to the archive (before is_archive is set on the archive did)
+    attach_dids(dids=[archive], account=root_account, **dataset1)
+    attach_dids(dids=[{**c, 'bytes': 200, 'adler32': 'ababbaba'} for c in constituents], account=root_account, **archive)
+    # Attach to another dataset _after_ attaching files to the archive
+    attach_dids(dids=[archive], account=root_account, **dataset2)
+
+    # Both datasets must have is_archive = True
+    metadata = get_metadata(**dataset1)
+    assert metadata['is_archive'] is True
+    metadata = get_metadata(**dataset2)
+    assert metadata['is_archive'] is True
+
+    # Delete the archive, the datasets must now have is_archive == false
+    delete_replicas(rse_id=rse_id, files=[archive])
+
+    metadata = get_metadata(**dataset1)
+    assert not metadata['is_archive']
+    metadata = get_metadata(**dataset2)
+    assert not metadata['is_archive']