Repository: rucio/rucio
Issue: List_replicas needs to resolve parent dids recursively for containers

Motivation
----------
`core.list_replicas` does not recursively list the parents and thus does not resolve any replicas.
This is blocking the deployment of new clients.



Changed files:
lib/rucio/core/did.py, lib/rucio/core/replica.py, lib/rucio/client/downloadclient.py, lib/rucio/client/baseclient.py, lib/rucio/tests/test_judge_evaluator.py

diff --git a/lib/rucio/client/baseclient.py b/lib/rucio/client/baseclient.py
index d1f5ebdd2..3f88e2b0a 100644
--- a/lib/rucio/client/baseclient.py
+++ b/lib/rucio/client/baseclient.py
@@ -357,7 +357,7 @@ class BaseClient(object):
             raise exc_cls(exc_msg)
 
         self.auth_token = result.headers['x-rucio-auth-token']
-        LOG.debug('got new token \\'%s\\'' % self.auth_token)
+        LOG.debug('got new token')
         return True
 
     def __get_token_x509(self):
@@ -418,7 +418,7 @@ class BaseClient(object):
             raise exc_cls(exc_msg)
 
         self.auth_token = result.headers['x-rucio-auth-token']
-        LOG.debug('got new token \\'%s\\'' % self.auth_token)
+        LOG.debug('got new token')
         return True
 
     def __get_token_ssh(self):
@@ -497,7 +497,7 @@ class BaseClient(object):
             raise exc_cls(exc_msg)
 
         self.auth_token = result.headers['x-rucio-auth-token']
-        LOG.debug('got new token \\'%s\\'' % self.auth_token)
+        LOG.debug('got new token')
         return True
 
     def __get_token_gss(self):
@@ -535,7 +535,7 @@ class BaseClient(object):
             raise exc_cls(exc_msg)
 
         self.auth_token = result.headers['x-rucio-auth-token']
-        LOG.debug('got new token \\'%s\\'' % self.auth_token)
+        LOG.debug('got new token')
         return True
 
     def __get_token(self):
@@ -586,7 +586,7 @@ class BaseClient(object):
         except Exception:
             raise
 
-        LOG.debug('read token \\'%s\\' from file' % self.auth_token)
+        LOG.debug('got token from file')
         return True
 
     def __write_token(self):
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
index 7be3ca247..d374d0a96 100644
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -16,7 +16,7 @@
 # - Tomas Javurek <tomasjavurek09@gmail.com>, 2018
 # - Vincent Garonne <vgaronne@gmail.com>, 2018
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
-# - Nicolo Magini <nicolo.magini@cern.ch>, 2018
+# - Nicolo Magini <nicolo.magini@cern.ch>, 2018-2019
 # - Tobias Wegner <tobias.wegner@cern.ch>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 #
@@ -330,7 +330,9 @@ class DownloadClient:
             dest_file_path = os.path.join(dest_dir_path, did_name)
             item['dest_file_paths'] = [dest_file_path]
             item['temp_file_path'] = '%s.part' % dest_file_path
-            item.setdefault('ignore_checksum', True)
+            options = item.setdefault('merged_options', {})
+            options.setdefault('ignore_checksum', item.pop('ignore_checksum', True))
+            options.setdefault('transfer_timeout', item.pop('transfer_timeout', None))
 
             input_items.append(item)
 
@@ -653,7 +655,7 @@ class DownloadClient:
                 start_time = time.time()
 
                 try:
-                    protocol.get(pfn, temp_file_path, transfer_timeout=item.get('transfer_timeout'))
+                    protocol.get(pfn, temp_file_path, transfer_timeout=item.get('merged_options', {}).get('transfer_timeout'))
                     success = True
                 except Exception as error:
                     logger.debug(error)
@@ -661,7 +663,7 @@ class DownloadClient:
 
                 end_time = time.time()
 
-                if success and not item.get('ignore_checksum', False):
+                if success and not item.get('merged_options', {}).get('ignore_checksum', False):
                     rucio_checksum = item.get('adler32')
                     local_checksum = None
                     if not rucio_checksum:
diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
index 97bc4691e..e1ab0c4f0 100644
--- a/lib/rucio/core/did.py
+++ b/lib/rucio/core/did.py
@@ -17,7 +17,7 @@
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2019
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2018
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2018
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2019
 # - Yun-Pin Sun <winter0128@gmail.com>, 2013
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2013-2018
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2014-2015
@@ -925,7 +925,7 @@ def list_parent_dids(scope, name, session=None):
 @stream_session
 def list_all_parent_dids(scope, name, session=None):
     """
-    List all parent datasets and containers of a did, no matter on what level
+    List all parent datasets and containers of a did, no matter on what level.
 
     :param scope:     The scope.
     :param name:      The name.
@@ -939,7 +939,9 @@ def list_all_parent_dids(scope, name, session=None):
                           models.DataIdentifierAssociation.did_type).filter_by(child_scope=scope, child_name=name)
     for did in query.yield_per(5):
         yield {'scope': did.scope, 'name': did.name, 'type': did.did_type}
-        list_all_parent_dids(scope=did.scope, name=did.name, session=session)
+        # Note that only Python3 supports recursive yield, that's the reason to do the nested for.
+        for pdid in list_all_parent_dids(scope=did.scope, name=did.name, session=session):
+            yield {'scope': pdid['scope'], 'name': pdid['name'], 'type': pdid['type']}
 
 
 @transactional_session
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
index d6b505e9a..5f201cfd5 100644
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -784,7 +784,7 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns,
             # full name used due to circular import dependency between modules
             if resolve_parents and 'parents' not in file:
                 file['parents'] = ['%s:%s' % (parent['scope'], parent['name'])
-                                   for parent in rucio.core.did.list_parent_dids(scope, name, session=session)]
+                                   for parent in rucio.core.did.list_all_parent_dids(scope, name, session=session)]
 
             # if the file is a constituent, find the available archives and add them to the list of possible PFNs
             # taking into account the original rse_expression
@@ -1072,7 +1072,7 @@ def _list_replicas(dataset_clause, file_clause, state_clause, show_pfns,
         # also do it for the last replica if necessary
         if resolve_parents and 'parents' not in file:
             file['parents'] = ['%s:%s' % (parent['scope'], parent['name'])
-                               for parent in rucio.core.did.list_parent_dids(file['scope'], file['name'], session=session)]
+                               for parent in rucio.core.did.list_all_parent_dids(file['scope'], file['name'], session=session)]
 
         # also sort the pfns inside the rse structure
         rse_pfns = []
diff --git a/lib/rucio/tests/test_judge_evaluator.py b/lib/rucio/tests/test_judge_evaluator.py
index b5ab18e0b..ccbf94e52 100644
--- a/lib/rucio/tests/test_judge_evaluator.py
+++ b/lib/rucio/tests/test_judge_evaluator.py
@@ -1,13 +1,20 @@
-# Copyright European Organization for Nuclear Research (CERN)
+# Copyright 2013-2018 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# You may not use this file except in compliance with the License.
+# you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# http://www.apache.org/licenses/LICENSE-2.0
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # Authors:
 # - Martin Barisits, <martin.barisits@cern.ch>, 2014-2015
-# - Mario Lassnig, <mario.lassnig@cern.ch>, 2014
+# - Mario Lassnig, <mario.lassnig@cern.ch>, 2014-2019
 # - Cedric Serfon, <cedric.serfon@cern.ch>, 2019
 
 from rucio.common.utils import generate_uuid as uuid
@@ -258,3 +265,37 @@ class TestJudgeEvaluator():
         # Check if the Locks are created properly
         for file in files:
             assert(len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2)
+
+    def test_judge_add_files_to_dataset_rule_on_container(self):
+        """ JUDGE EVALUATOR: Test the judge when attaching file to dataset with rule on two levels of containers"""
+        scope = 'mock'
+        files = create_files(3, scope, self.rse1)
+        dataset = 'dataset_' + str(uuid())
+        add_did(scope, dataset, DIDType.from_sym('DATASET'), 'jdoe')
+        attach_dids(scope, dataset, files, 'jdoe')
+
+        parent_container = 'dataset_' + str(uuid())
+        add_did(scope, parent_container, DIDType.from_sym('CONTAINER'), 'jdoe')
+        attach_dids(scope, parent_container, [{'scope': scope, 'name': dataset}], 'jdoe')
+
+        parent_parent_container = 'dataset_' + str(uuid())
+        add_did(scope, parent_parent_container, DIDType.from_sym('CONTAINER'), 'jdoe')
+        attach_dids(scope, parent_parent_container, [{'scope': scope, 'name': parent_container}], 'jdoe')
+
+        # Add a first rule to the DS
+        add_rule(dids=[{'scope': scope, 'name': parent_parent_container}], account='jdoe', copies=2, rse_expression=self.T1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)
+
+        # Fake judge
+        re_evaluator(once=True)
+
+        # Check if the Locks are created properly
+        for file in files:
+            assert(len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2)
+
+        # create more files and attach them
+        more_files = create_files(3, scope, self.rse1)
+        attach_dids(scope, dataset, more_files, 'jdoe')
+        re_evaluator(once=True)
+        # Check if the Locks are created properly
+        for file in more_files:
+            assert(len(get_replica_locks(scope=file['scope'], name=file['name'])) == 2)