Repository: rucio/rucio

Issue:
Include archives in list_dataset_replicas(deep=True)
Motivation
----------
Similar to list_replicas, in the deep listing of dataset replicas (Which resolves individual file replicas) the archives should be included.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -2018,8 +2018,10 @@ def list_dataset_replicas(scope, name, deep=False, session=None):
     :param deep: Lookup at the file level.
     :param session: Database session to use.
 
-    :returns: A list of dict dataset replicas
+    :returns: A list of dictionaries containing the dataset replicas
+              with associated metrics and timestamps
     """
+
     if not deep:
         query = session.query(models.CollectionReplica.scope,
                               models.CollectionReplica.name,
@@ -2041,9 +2043,11 @@ def list_dataset_replicas(scope, name, deep=False, session=None):
             yield row._asdict()
 
     else:
-        content_query = session.\
-            query(func.sum(models.DataIdentifierAssociation.bytes).label("bytes"),
-                  func.count().label("length"))\
+
+        # find maximum values
+        content_query = session\
+            .query(func.sum(models.DataIdentifierAssociation.bytes).label("bytes"),
+                   func.count().label("length"))\
             .with_hint(models.DataIdentifierAssociation, "INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)", 'oracle')\
             .filter(models.DataIdentifierAssociation.scope == scope)\
             .filter(models.DataIdentifierAssociation.name == name)
@@ -2052,15 +2056,77 @@ def list_dataset_replicas(scope, name, deep=False, session=None):
         for row in content_query:
             bytes, length = row.bytes, row.length
 
-        sub_query = session.\
-            query(models.DataIdentifierAssociation.scope,
-                  models.DataIdentifierAssociation.name,
-                  models.RSEFileAssociation.rse_id,
-                  func.sum(models.RSEFileAssociation.bytes).label("available_bytes"),
-                  func.count().label("available_length"),
-                  func.min(models.RSEFileAssociation.created_at).label("created_at"),
-                  func.max(models.RSEFileAssociation.updated_at).label("updated_at"),
-                  func.max(models.RSEFileAssociation.accessed_at).label("accessed_at"))\
+        # find archives that contain files of the requested dataset
+        sub_query_archives = session\
+            .query(models.DataIdentifierAssociation.scope.label('dataset_scope'),
+                   models.DataIdentifierAssociation.name.label('dataset_name'),
+                   models.DataIdentifierAssociation.bytes.label('file_bytes'),
+                   models.ConstituentAssociation.child_scope.label('file_scope'),
+                   models.ConstituentAssociation.child_name.label('file_name'),
+                   models.RSEFileAssociation.scope.label('replica_scope'),
+                   models.RSEFileAssociation.name.label('replica_name'),
+                   models.RSE.rse,
+                   models.RSE.id.label('rse_id'),
+                   models.RSEFileAssociation.created_at,
+                   models.RSEFileAssociation.accessed_at,
+                   models.RSEFileAssociation.updated_at)\
+            .filter(models.DataIdentifierAssociation.scope == scope)\
+            .filter(models.DataIdentifierAssociation.name == name)\
+            .filter(models.ConstituentAssociation.child_scope == models.DataIdentifierAssociation.child_scope)\
+            .filter(models.ConstituentAssociation.child_name == models.DataIdentifierAssociation.child_name)\
+            .filter(models.ConstituentAssociation.scope == models.RSEFileAssociation.scope)\
+            .filter(models.ConstituentAssociation.name == models.RSEFileAssociation.name)\
+            .filter(models.RSEFileAssociation.rse_id == models.RSE.id)\
+            .filter(models.RSEFileAssociation.state == ReplicaState.AVAILABLE)\
+            .filter(models.RSE.deleted == false())\
+            .subquery()
+
+        # count the metrics
+        group_query_archives = session\
+            .query(sub_query_archives.c.dataset_scope,
+                   sub_query_archives.c.dataset_name,
+                   sub_query_archives.c.file_scope,
+                   sub_query_archives.c.file_name,
+                   sub_query_archives.c.rse_id,
+                   sub_query_archives.c.rse,
+                   func.sum(sub_query_archives.c.file_bytes).label('file_bytes'),
+                   func.min(sub_query_archives.c.created_at).label('created_at'),
+                   func.max(sub_query_archives.c.updated_at).label('updated_at'),
+                   func.max(sub_query_archives.c.accessed_at).label('accessed_at'))\
+            .group_by(sub_query_archives.c.dataset_scope,
+                      sub_query_archives.c.dataset_name,
+                      sub_query_archives.c.file_scope,
+                      sub_query_archives.c.file_name,
+                      sub_query_archives.c.rse_id,
+                      sub_query_archives.c.rse)\
+            .subquery()
+
+        # bring it in the same column state as the non-archive query
+        full_query_archives = session\
+            .query(group_query_archives.c.dataset_scope.label('scope'),
+                   group_query_archives.c.dataset_name.label('name'),
+                   group_query_archives.c.rse_id,
+                   group_query_archives.c.rse,
+                   func.sum(group_query_archives.c.file_bytes).label('available_bytes'),
+                   func.count().label('available_length'),
+                   func.min(group_query_archives.c.created_at).label('created_at'),
+                   func.max(group_query_archives.c.updated_at).label('updated_at'),
+                   func.max(group_query_archives.c.accessed_at).label('accessed_at'))\
+            .group_by(group_query_archives.c.dataset_scope,
+                      group_query_archives.c.dataset_name,
+                      group_query_archives.c.rse_id,
+                      group_query_archives.c.rse)
+
+        # find the non-archive dataset replicas
+        sub_query = session\
+            .query(models.DataIdentifierAssociation.scope,
+                   models.DataIdentifierAssociation.name,
+                   models.RSEFileAssociation.rse_id,
+                   func.sum(models.RSEFileAssociation.bytes).label("available_bytes"),
+                   func.count().label("available_length"),
+                   func.min(models.RSEFileAssociation.created_at).label("created_at"),
+                   func.max(models.RSEFileAssociation.updated_at).label("updated_at"),
+                   func.max(models.RSEFileAssociation.accessed_at).label("accessed_at"))\
             .with_hint(models.DataIdentifierAssociation, "INDEX_RS_ASC(CONTENTS CONTENTS_PK) INDEX_RS_ASC(REPLICAS REPLICAS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)", 'oracle')\
             .filter(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope)\
             .filter(models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name)\
@@ -2072,19 +2138,23 @@ def list_dataset_replicas(scope, name, deep=False, session=None):
                       models.RSEFileAssociation.rse_id)\
             .subquery()
 
-        query = session.query(sub_query.c.scope,
-                              sub_query.c.name,
-                              sub_query.c.rse_id,
-                              models.RSE.rse,
-                              sub_query.c.available_bytes,
-                              sub_query.c.available_length,
-                              sub_query.c.created_at,
-                              sub_query.c.updated_at,
-                              sub_query.c.accessed_at)\
+        query = session\
+            .query(sub_query.c.scope,
+                   sub_query.c.name,
+                   sub_query.c.rse_id,
+                   models.RSE.rse,
+                   sub_query.c.available_bytes,
+                   sub_query.c.available_length,
+                   sub_query.c.created_at,
+                   sub_query.c.updated_at,
+                   sub_query.c.accessed_at)\
             .filter(models.RSE.id == sub_query.c.rse_id)\
             .filter(models.RSE.deleted == false())
 
-        for row in query:
+        # join everything together
+        final_query = query.union_all(full_query_archives)
+
+        for row in final_query.all():
             replica = row._asdict()
             replica['length'], replica['bytes'] = length, bytes
             if replica['length'] == row.available_length:
diff --git a/lib/rucio/vcsversion.py b/lib/rucio/vcsversion.py
--- a/lib/rucio/vcsversion.py
+++ b/lib/rucio/vcsversion.py
@@ -4,8 +4,8 @@
 '''
 VERSION_INFO = {
     'final': True,
-    'version': '1.17.7',
-    'branch_nick': 'patch-0-1_17_7_release_preparation',
-    'revision_id': '9b733f0793b98677aa5fc65713d0026ce478e0c1',
-    'revno': 6932
+    'version': '1.17.8',
+    'branch_nick': 'patch-0-Rucio_1_17_8_preparation',
+    'revision_id': '136b9a54b61a94ad421cda5d60490357d172d5cd',
+    'revno': 6955
 }


Test patch:
diff --git a/lib/rucio/tests/test_dataset_replicas.py b/lib/rucio/tests/test_dataset_replicas.py
--- a/lib/rucio/tests/test_dataset_replicas.py
+++ b/lib/rucio/tests/test_dataset_replicas.py
@@ -1,25 +1,35 @@
-# Copyright European Organization for Nuclear Research (CERN)
+# Copyright 2015-2018 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# You may not use this file except in compliance with the License.
+# you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# http://www.apache.org/licenses/LICENSE-2.0
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # Authors:
-# - Vincent Garonne, <vincent.garonne@cern.ch>, 2015
-# - Cedric Serfon, <cedric.serfon@cern.ch>, 2015
+# - Vincent Garonne <vgaronne@gmail.com>, 2015
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2015
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2018
 
 from nose.tools import assert_equal
 
 from rucio.core.replica import list_datasets_per_rse
+from rucio.core.rse import add_rse, del_rse, add_protocol
 
 from rucio.client.didclient import DIDClient
 from rucio.client.replicaclient import ReplicaClient
 from rucio.client.ruleclient import RuleClient
-from rucio.common.utils import generate_uuid as uuid
+from rucio.common.utils import generate_uuid
+from rucio.tests.common import rse_name_generator
 
 
-class TestDatasetReplicaCLient:
+class TestDatasetReplicaClient:
 
     def test_list_dataset_replicas(self):
         """ REPLICA (CLIENT): List dataset replicas."""
@@ -27,7 +37,7 @@ def test_list_dataset_replicas(self):
         rule_client = RuleClient()
         did_client = DIDClient()
         scope = 'mock'
-        dataset = 'dataset_' + str(uuid())
+        dataset = 'dataset_' + str(generate_uuid())
 
         did_client.add_dataset(scope=scope, name=dataset)
         rule_client.add_replication_rule(dids=[{'scope': scope, 'name': dataset}],
@@ -41,7 +51,7 @@ def test_list_datasets_per_rse(self):
         rule_client = RuleClient()
         did_client = DIDClient()
         scope = 'mock'
-        dataset = 'dataset_' + str(uuid())
+        dataset = 'dataset_' + str(generate_uuid())
 
         did_client.add_dataset(scope=scope, name=dataset)
         rule_client.add_replication_rule(dids=[{'scope': scope, 'name': dataset}],
@@ -49,3 +59,68 @@ def test_list_datasets_per_rse(self):
                                          grouping='DATASET')
         replicas = [r for r in list_datasets_per_rse(rse='MOCK', filters={'scope': 'mock', 'name': 'data*'})]
         assert(replicas != [])
+
+    def test_list_dataset_replicas_archive(self):
+        """ REPLICA (CLIENT): List dataset replicas with archives. """
+
+        replica_client = ReplicaClient()
+        did_client = DIDClient()
+        rule_client = RuleClient()
+
+        scope = 'mock'
+
+        rse = 'APERTURE_%s' % rse_name_generator()
+        add_rse(rse)
+        add_protocol(rse, {'scheme': 'root',
+                           'hostname': 'root.aperture.com',
+                           'port': 1409,
+                           'prefix': '//test/chamber/',
+                           'impl': 'rucio.rse.protocols.xrootd.Default',
+                           'domains': {
+                               'lan': {'read': 1, 'write': 1, 'delete': 1},
+                               'wan': {'read': 1, 'write': 1, 'delete': 1}}})
+
+        rse2 = 'BLACKMESA_%s' % rse_name_generator()
+        add_rse(rse2)
+        add_protocol(rse2, {'scheme': 'root',
+                            'hostname': 'root.blackmesa.com',
+                            'port': 1409,
+                            'prefix': '//underground/facility',
+                            'impl': 'rucio.rse.protocols.xrootd.Default',
+                            'domains': {
+                                'lan': {'read': 1, 'write': 1, 'delete': 1},
+                                'wan': {'read': 1, 'write': 1, 'delete': 1}}})
+
+        # register archive
+        archive = {'scope': scope, 'name': 'another.%s.zip' % generate_uuid(),
+                   'type': 'FILE', 'bytes': 2596, 'adler32': 'deedbeaf'}
+        replica_client.add_replicas(rse=rse, files=[archive])
+        replica_client.add_replicas(rse=rse2, files=[archive])
+
+        archived_files = [{'scope': scope, 'name': 'zippedfile-%i-%s' % (i, str(generate_uuid())), 'type': 'FILE',
+                           'bytes': 4322, 'adler32': 'deaddead'} for i in xrange(2)]
+        replica_client.add_replicas(rse=rse2, files=archived_files)
+        did_client.add_files_to_archive(scope=scope, name=archive['name'], files=archived_files)
+
+        dataset_name = 'find_me.' + str(generate_uuid())
+        did_client.add_dataset(scope=scope, name=dataset_name)
+        did_client.attach_dids(scope=scope, name=dataset_name, dids=archived_files)
+        rule_client.add_replication_rule(dids=[{'scope': scope, 'name': dataset_name}],
+                                         account='root', copies=1, rse_expression=rse,
+                                         grouping='DATASET')
+
+        res = [r for r in replica_client.list_dataset_replicas(scope=scope,
+                                                               name=dataset_name)]
+        assert_equal(len(res), 1)
+        assert_equal(res[0]['state'], 'UNAVAILABLE')
+
+        res = [r for r in replica_client.list_dataset_replicas(scope=scope,
+                                                               name=dataset_name,
+                                                               deep=True)]
+
+        assert_equal(len(res), 3)
+        assert_equal(res[0]['state'], 'AVAILABLE')
+        assert_equal(res[1]['state'], 'AVAILABLE')
+        assert_equal(res[2]['state'], 'AVAILABLE')
+
+        del_rse(rse)