Repository: rucio/rucio
Issue: List_replicas needs to resolve parent dids recursively for containers

Motivation
----------
`core.list_replicas` does not recursively list the parents and thus does not resolve any replicas.
This is blocking the deployment of new clients.



Changed files:
lib/rucio/core/replica.py, lib/rucio/core/did.py, lib/rucio/tests/test_judge_evaluator.py

diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
index 97bc4691e..e1ab0c4f0 100644
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -17,7 +17,7 @@
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2019
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2018
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2018
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2019
 # - Yun-Pin Sun <winter0128@gmail.com>, 2013
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2013-2018
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2014-2015
@@ -925,7 +925,7 @@ def list_parent_dids(scope, name, session=None):
 @stream_session
 def list_all_parent_dids(scope, name, session=None):
     """
-    List all parent datasets and containers of a did, no matter on what level
+    List all parent datasets and containers of a did, no matter on what level.
 
     :param scope:     The scope.
     :param name:      The name.
@@ -939,7 +939,9 @@ def list_all_parent_dids(scope, name, session=None):
                           models.DataIdentifierAssociation.did_type).filter_by(child_scope=scope, child_name=name)
     for did in query.yield_per(5):
         yield {'scope': did.scope, 'name': did.name, 'type': did.did_type}
-        list_all_parent_dids(scope=did.scope, name=did.name, session=session)
+        # Note that only Python3 supports recursive yield, that's the reason to do the nested for.
+        for pdid in list_all_parent_dids(scope=did.scope, name=did.name, session=session):
+            yield {'scope': pdid['scope'], 'name': pdid['name'], 'type': pdid['type']}
 
 
 @transactional_session
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
index d6b505e9a..5f201cfd5 100644
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -784,7 +784,7 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns,
             # full name used due to circular import dependency between modules
             if resolve_parents and 'parents' not in file:
                 file['parents'] = ['%s:%s' % (parent['scope'], parent['name'])
-                                   for parent in rucio.core.did.list_parent_dids(scope, name, session=session)]
+                                   for parent in rucio.core.did.list_all_parent_dids(scope, name, session=session)]
 
             # if the file is a constituent, find the available archives and add them to the list of possible PFNs
             # taking into account the original rse_expression
@@ -1072,7 +1072,7 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns,
         # also do it for the last replica if necessary
         if resolve_parents and 'parents' not in file:
             file['parents'] = ['%s:%s' % (parent['scope'], parent['name'])
-                               for parent in rucio.core.did.list_parent_dids(file['scope'], file['name'], session=session)]
+                               for parent in rucio.core.did.list_all_parent_dids(file['scope'], file['name'], session=session)]
 
         # also sort the pfns inside the rse structure
         rse_pfns = []
diff --git a/lib/rucio/tests/test_judge_evaluator.py b/lib/rucio/tests/test_judge_evaluator.py
index b5ab18e0b..ccbf94e52 100644
--- a/lib/rucio/tests/test_judge_evaluator.py
+++ b/lib/rucio/tests/test_judge_evaluator.py
@@ -1,13 +1,20 @@
-# Copyright European Organization for Nuclear Research (CERN)
+# Copyright 2013-2018 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# You may not use this file except in compliance with the License.
+# you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# http://www.apache.org/licenses/LICENSE-2.0
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # Authors:
 # - Martin Barisits, <martin.barisits@cern.ch>, 2014-2015
-# - Mario Lassnig, <mario.lassnig@cern.ch>, 2014
+# - Mario Lassnig, <mario.lassnig@cern.ch>, 2014-2019
 # - Cedric Serfon, <cedric.serfon@cern.ch>, 2019
 
 from rucio.common.utils import generate_uuid as uuid
@@ -258,3 +265,37 @@ class TestJudgeEvaluator():
         # Check if the Locks are created properly
         for file in files:
             assert(len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2)
+
+    def test_judge_add_files_to_dataset_rule_on_container(self):
+        """ JUDGE EVALUATOR: Test the judge when attaching file to dataset with rule on two levels of containers"""
+        scope = 'mock'
+        files = create_files(3, scope, self.rse1)
+        dataset = 'dataset_' + str(uuid())
+        add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
+        attach_dids(scope, dataset, files, 'jdoe')
+
+        parent_container = 'dataset_' + str(uuid())
+        add_did(scope, parent_container, DIDType.from_sym('CONTAINER'), 'jdoe')
+        attach_dids(scope, parent_container, [{'scope': scope, 'name': dataset}], 'jdoe')
+
+        parent_parent_container = 'dataset_' + str(uuid())
+        add_did(scope, parent_parent_container, DIDType.from_sym('CONTAINER'), 'jdoe')
+        attach_dids(scope, parent_parent_container, [{'scope': scope, 'name': parent_container}], 'jdoe')
+
+        # Add a first rule to the DS
+        add_rule(dids=[{'scope': scope, 'name': parent_parent_container}], account='jdoe', copies=2, rse_expression=self.T1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)
+
+        # Fake judge
+        re_evaluator(once=True)
+
+        # Check if the Locks are created properly
+        for file in files:
+            assert(len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2)
+
+        # create more files and attach them
+        more_files = create_files(3, scope, self.rse1)
+        attach_dids(scope, dataset, more_files, 'jdoe')
+        re_evaluator(once=True)
+        # Check if the Locks are created properly
+        for file in more_files:
+            assert(len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2)