Repository: rucio/rucio

Issue:
Incomplete download looks "complete" when restricting RSEs
Motivation
----------
As a user one may naively assume that a dataset/container is completely downloaded when seeing an output like
```
Total files :                                93
Downloaded files :                            0
Files already found locally :                93
Files that cannot be downloaded :             0
```
and a return code of 0. However, this example is taken from a case where I restricted the RSEs to download from, e.g., via `rucio download --rse '*\SOME_PROBLEMATIC_SITE' someContainer` and the full container contains more than the 93 files reported. (Downloads from the problematic site in this case keep failing for a subset of the files involved)

The effect is the same if some RSE is explicitly selected (`--rse 'SOME_GOOD_RSE'`) or individual RSEs are vetoed as in the example above.

Modification
------------
The total number of files should likely be determined without the RSE restriction to avoid files only present on the excluded RSEs being forgotten about. If the current counting is intentional it would be good to at least print a warning in case only a restricted set of RSEs was considered.




Discussion:
this can be close since it is solved in #3805 
https://github.com/rucio/rucio/blob/3eeeca8f99e8f58fd24925e014289f23edc0c534/bin/rucio#L1043
The implementation for this breaks the possibility to do `rucio download <name>` without giving the `<scope>:<name>` pair.

I am reverting the code and reopen the ticket. We will add the fixed feature in a later release again.
Ideally, we should count the files of the dataset and display this information as well.
The output should show:
```
Total files (did):                            This should show the logical files in DID
Total files (after filter):                   This should show the files available & after applying filter
Downloaded files:                             0
Files already found locally:                  93
Files that cannot be downloaded:              0
```

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -37,6 +37,7 @@
 from __future__ import division
 
 import copy
+import itertools
 import logging
 import os
 import random
@@ -289,15 +290,10 @@ def download_dids(self, items, num_threads=2, trace_custom_fields={}, traces_cop
         trace_custom_fields['uuid'] = generate_uuid()
 
         logger(logging.INFO, 'Processing %d item(s) for input' % len(items))
-        did_to_options, merged_items = self._resolve_and_merge_input_items(copy.deepcopy(items))
+        did_to_input_items, file_items_with_sources = self._resolve_and_merge_input_items(copy.deepcopy(items))
+        self.logger(logging.DEBUG, 'num_unmerged_items=%d; num_dids=%d; num_file_items=%d' % (len(items), len(did_to_input_items), len(file_items_with_sources)))
 
-        self.logger(logging.DEBUG, 'num_unmerged_items=%d; num_dids=%d; num_merged_items=%d' % (len(items), len(did_to_options), len(merged_items)))
-
-        logger(logging.INFO, 'Getting sources of DIDs')
-        # if one item wants to resolve archives we enable it for all items
-        resolve_archives = not all(item.get('no_resolve_archives') for item in merged_items)
-        merged_items_with_sources = self._get_sources(merged_items, resolve_archives=resolve_archives)
-        input_items = self._prepare_items_for_download(did_to_options, merged_items_with_sources, resolve_archives=resolve_archives)
+        input_items = self._prepare_items_for_download(did_to_input_items, file_items_with_sources)
 
         num_files_in = len(input_items)
         output_items = self._download_multithreaded(input_items, num_threads, trace_custom_fields, traces_copy_out)
@@ -336,11 +332,11 @@ def download_from_metalink_file(self, item, metalink_file_path, num_threads=2, t
         trace_custom_fields['uuid'] = generate_uuid()
 
         did_to_options = {}
-        item.setdefault('destinations', set()).add((item['base_dir'], item['no_subdir']))
         for metalink in metalinks:
-            did_to_options[metalink['did']] = item
+            did = DIDType(metalink['did'])
+            did_to_options[did] = [item]
+            metalink['input_dids'] = {did: {}}
 
-        metalinks = [metalinks]
         input_items = self._prepare_items_for_download(did_to_options, metalinks)
 
         num_files_in = len(input_items)
@@ -750,15 +746,13 @@ def download_aria2c(self, items, trace_custom_fields={}, filters={}):
 
         for item in items:
             item['force_scheme'] = ['https', 'davs']
+            item['no_resolve_archives'] = True
 
         logger(logging.INFO, 'Processing %d item(s) for input' % len(items))
-        did_to_options, merged_items = self._resolve_and_merge_input_items(copy.deepcopy(items))
-
-        self.logger(logging.DEBUG, 'num_unmerged_items=%d; num_dids=%d; num_merged_items=%d' % (len(items), len(did_to_options), len(merged_items)))
+        did_to_input_items, file_items_with_sources = self._resolve_and_merge_input_items(copy.deepcopy(items))
+        self.logger(logging.DEBUG, 'num_unmerged_items=%d; num_dids=%d; num_file_items=%d' % (len(items), len(did_to_input_items), len(file_items_with_sources)))
 
-        logger(logging.INFO, 'Getting sources of DIDs')
-        merged_items_with_sources = self._get_sources(merged_items)
-        input_items = self._prepare_items_for_download(did_to_options, merged_items_with_sources, resolve_archives=False)
+        input_items = self._prepare_items_for_download(did_to_input_items, file_items_with_sources)
 
         try:
             output_items = self._download_items_aria2c(input_items, aria_rpc, rpc_auth, trace_custom_fields)
@@ -1010,7 +1004,8 @@ def _resolve_one_item_dids(self, item):
         if dids is None:
             self.logger(logging.DEBUG, 'Resolving DIDs by using filter options')
             scope = filters.pop('scope')
-            yield scope, list(self.client.list_dids(scope, filters=filters, did_type='all'))
+            for did in self.client.list_dids(scope, filters=filters, did_type='all', long=True):
+                yield did
             return
 
         if not isinstance(dids, list):
@@ -1018,106 +1013,61 @@ def _resolve_one_item_dids(self, item):
 
         for did_str in dids:
             scope, did_name = self._split_did_str(did_str)
-            if '*' in did_name:
-                filters['name'] = did_name
-                resolved_dids = list(self.client.list_dids(scope, filters=filters, did_type='all'))
-                yield scope, resolved_dids
-            else:
-                yield scope, [did_name]
-
-    def _resolve_and_merge_input_items(self, items):
+            filters['name'] = did_name
+            any_did_resolved = False
+            for did in self.client.list_dids(scope, filters=filters, did_type='all', long=True):
+                yield did
+                any_did_resolved = True
+
+            # Maintain compatibility with existing code, which expects non-existing DIDs be
+            # passed through in order to correctly set trace state to FILE_NOT_FOUND
+            if not any_did_resolved and '*' not in did_name:
+                yield {'scope': scope, 'name': did_name}
+
+    def _resolve_and_merge_input_items(self, input_items):
         """
-        This function takes the input items given to download_dids etc. and merges them
-        respecting their individual options. This way functions can operate on these items
-        in batch mode. E.g., list_replicas calls are reduced.
+        This function takes the input items given to download_dids etc.
+        and resolves the sources.
 
-        :param items: List of dictionaries. Each dictionary describing an input item
+        - It first performs a list_dids call to dereference any wildcards and
+        retrieve DID stats (size, length, type).
+        - Next, input items are grouped together by common list_replicas options.
+        For each group, a single list_replicas call is performed.
+        - The resolved File DIDs with sources are finally mapped back to initial
+        input items to be able to correctly retrieve download options
+        (timeout, destination directories, etc)
 
-        :returns: a dictionary that maps the input DIDs to options
-                  and a list with a dictionary for each merged download item
+        :param input_items: List of dictionaries. Each dictionary describing an input item
+
+        :returns: a tuple:
+        - a dictionary that maps the dereferenced(w/o wildcards) input DIDs to a list of input items
+        - and a list with a dictionary for each file DID which has to be downloaded
 
         :raises InputValidationError: if one of the input items is in the wrong format
         """
         logger = self.logger
 
         # check mandatory options before doing any server calls
-        for item in items:
+        resolve_archives = False
+        for item in input_items:
             if item.get('resolve_archives') is not None:
                 logger(logging.WARNING, 'resolve_archives option is deprecated and will be removed in a future release.')
                 item.setdefault('no_resolve_archives', not item.pop('resolve_archives'))
 
+            # If any item needs to resolve archives
+            if not item.get('no_resolve_archives'):
+                resolve_archives = True
+
             if not item.get('did'):
                 if not item.get('filters', {}).get('scope'):
                     logger(logging.DEBUG, item)
                     raise InputValidationError('Item without did and filter/scope')
-
-        distinct_keys = ['rse', 'force_scheme', 'nrandom']
-        all_resolved_did_strs = set()
-
-        did_to_options = {}
-        merged_items = []
-
-        for item in items:
-            base_dir = item.pop('base_dir', '.')
-            no_subdir = item.pop('no_subdir', False)
-            ignore_checksum = item.pop('ignore_checksum', False)
-            new_transfer_timeout = item.pop('transfer_timeout', None)
-            new_transfer_speed_timeout = item.pop('transfer_speed_timeout', None)
-
-            resolved_dids = item.setdefault('dids', [])
-
-            for did_scope, did_names in self._resolve_one_item_dids(item):
-                for did_name in did_names:
-                    resolved_did_str = '%s:%s' % (did_scope, did_name)
-                    options = did_to_options.setdefault(resolved_did_str, {})
-                    options.setdefault('destinations', set()).add((base_dir, no_subdir))
-
-                    # Merge some options
-                    # The other options of this DID will be inherited from the first item that contained the DID
-                    options['ignore_checksum'] = (options.get('ignore_checksum') or ignore_checksum)
-
-                    cur_transfer_timeout = options.setdefault('transfer_timeout', None)
-                    if cur_transfer_timeout is not None and new_transfer_timeout is not None:
-                        options['transfer_timeout'] = max(int(cur_transfer_timeout), int(new_transfer_timeout))
-                    elif new_transfer_timeout is not None:
-                        options['transfer_timeout'] = int(new_transfer_timeout)
-
-                    cur_transfer_speed_timeout = options.setdefault('transfer_speed_timeout', None)
-                    if cur_transfer_speed_timeout is not None and new_transfer_speed_timeout is not None:
-                        options['transfer_speed_timeout'] = min(float(cur_transfer_speed_timeout), float(new_transfer_speed_timeout))
-                    elif new_transfer_speed_timeout is not None:
-                        options['transfer_speed_timeout'] = float(new_transfer_speed_timeout)
-
-                    if resolved_did_str not in all_resolved_did_strs:
-                        resolved_dids.append({'scope': did_scope, 'name': did_name})
-                        all_resolved_did_strs.add(resolved_did_str)
-
-            if len(resolved_dids) == 0:
-                logger(logging.WARNING, 'An item didnt have any DIDs after resolving the input. Ignoring it.')
-                logger(logging.DEBUG, item)
-                continue
-
-            was_merged = False
-            for merged_item in merged_items:
-                if all(item.get(k) == merged_item.get(k) for k in distinct_keys):
-                    merged_item['dids'].extend(resolved_dids)
-                    was_merged = True
-                    break
-            if not was_merged:
-                item['dids'] = resolved_dids
-                merged_items.append(item)
-        return did_to_options, merged_items
-
-    def _get_sources(self, merged_items, resolve_archives=True):
-        """
-        Get sources (PFNs) of the DIDs.
-
-        :param merged_items: list of dictionaries. Each dictionary describes a bunch of DIDs to download
-
-        :returns: list of list of dictionaries.
-        """
-        logger = self.logger
-        merged_items_with_sources = []
+        if resolve_archives:
+            # perhaps we'll need an extraction tool so check what is installed
+            self.extraction_tools = [tool for tool in self.extraction_tools if tool.is_useable()]
+            if len(self.extraction_tools) < 1:
+                logger(logging.WARNING, 'Archive resolution is enabled but no extraction tool is available. '
+                                        'Sources whose protocol doesnt support extraction wont be considered for download.')
 
         # if excluding tapes, we need to list them first
         tape_rses = []
@@ -1127,7 +1077,48 @@ def _get_sources(self, merged_items, resolve_archives=True):
             except:
                 logger(logging.DEBUG, 'No tapes found.')
 
-        for item in merged_items:
+        # Matches each dereferenced DID back to a list of input items
+        did_to_input_items = {}
+
+        # Resolve DIDs
+        for item in input_items:
+            resolved_dids = list(self._resolve_one_item_dids(item))
+            if not resolved_dids:
+                logger(logging.WARNING, 'An item didnt have any DIDs after resolving the input: %s.' % item.get('did', item))
+            item['dids'] = resolved_dids
+            for did in resolved_dids:
+                did_to_input_items.setdefault(DIDType(did), []).append(item)
+
+                if 'length' in did and not did['length']:
+                    did_with_size = self.client.get_did(scope=did['scope'], name=did['name'], dynamic=True)
+                    did['length'] = did_with_size['length']
+                    did['bytes'] = did_with_size['bytes']
+
+        # group input items by common options to reduce the number of calls to list_replicas
+        distinct_keys = ['rse', 'force_scheme', 'no_resolve_archives']
+        item_groups = []
+        for item in input_items:
+            found_compatible_group = False
+            if not item.get('nrandom'):
+                # Don't merge items if nrandom is set. Otherwise two items with the same nrandom will be merged into one
+                # and we'll effectively download only half of the desired replicas for each item.
+                for item_group in item_groups:
+                    if all(item.get(k) == item_group[0].get(k) for k in distinct_keys):
+                        item_group.append(item)
+                        found_compatible_group = True
+                        break
+            if not found_compatible_group:
+                item_groups.append([item])
+
+        # List replicas for dids
+        merged_items_with_sources = []
+        for item_group in item_groups:
+            # Take configuration from the first item in the group; but dids from all items
+            item = item_group[0]
+            input_dids = {DIDType(did): did
+                          for item in item_group
+                          for did in item.get('dids')}
+
             # since we're using metalink we need to explicitly give all schemes
             schemes = item.get('force_scheme')
             if schemes:
@@ -1142,12 +1133,15 @@ def _get_sources(self, merged_items, resolve_archives=True):
             logger(logging.DEBUG, 'num DIDs for list_replicas call: %d' % len(item['dids']))
 
             nrandom = item.get('nrandom')
-            metalink_str = self.client.list_replicas(item['dids'],
+            if nrandom:
+                logger(logging.INFO, 'Selecting %d random replicas from DID(s): %s' % (nrandom, [str(did) for did in input_dids]))
+
+            metalink_str = self.client.list_replicas([{'scope': did.scope, 'name': did.name} for did in input_dids],
                                                      schemes=schemes,
                                                      ignore_availability=False,
                                                      rse_expression=rse_expression,
                                                      client_location=self.client_location,
-                                                     resolve_archives=resolve_archives,
+                                                     resolve_archives=not item.get('no_resolve_archives'),
                                                      resolve_parents=True,
                                                      nrandom=nrandom,
                                                      metalink=True)
@@ -1155,15 +1149,17 @@ def _get_sources(self, merged_items, resolve_archives=True):
 
             logger(logging.DEBUG, 'num resolved files: %s' % len(file_items))
 
-            # list_replicas returns nothing if the DID does not exist and we dont want to
-            # do another server call so we check if there is a result from list_replicas
-            # for each given DID. If not the DID does not exist
-            for input_did in item['dids']:
-                input_did = DIDType(input_did)
-                if not any([input_did == f['did'] or str(input_did) in f['parent_dids'] for f in file_items]):
-                    logger(logging.ERROR, 'DID does not exist: %s' % input_did)
-                    # TODO: store did directly as DIDType object
-                    file_items.append({'did': str(input_did), 'adler32': None, 'md5': None, 'sources': [], 'parent_dids': set()})
+            if not nrandom or nrandom != len(file_items):
+                # If list_replicas didn't resolve any file DIDs for any input did, we pass through the input DID.
+                # This is done to keep compatibility with later code which generates "FILE_NOT_FOUND" traces
+                # and output items.
+                # In the special case of nrandom, when serverside filtering is applied, it's "normal" for some input
+                # dids to be ignored as long as we got exactly nrandom file_items from the server.
+                for input_did in input_dids:
+                    if not any([input_did == f['did'] or str(input_did) in f['parent_dids'] for f in file_items]):
+                        logger(logging.ERROR, 'DID does not exist: %s' % input_did)
+                        # TODO: store did directly as DIDType object
+                        file_items.append({'did': str(input_did), 'adler32': None, 'md5': None, 'sources': [], 'parent_dids': set()})
 
             # filtering out tape sources
             if self.is_tape_excluded:
@@ -1176,42 +1172,77 @@ def _get_sources(self, merged_items, resolve_archives=True):
                         logger(logging.WARNING, 'The requested DID {} only has replicas on tape. Direct download from tape is prohibited. '
                                                 'Please request a transfer to a non-tape endpoint.'.format(file_item['did']))
 
-            if nrandom:
-                logger(logging.INFO, 'Selecting %d random replicas from DID(s): %s' % (nrandom, item['dids']))
-                random.shuffle(file_items)
-                file_items = file_items[0:nrandom]
-                merged_items_with_sources.append(file_items)
-            else:
-                merged_items_with_sources.append(file_items)
+            # Match the file did back to the dids which were provided to list_replicas.
+            # Later, this will allow to match the file back to input_items via did_to_input_items
+            for file_item in file_items:
+                file_did = DIDType(file_item['did'])
 
-        return merged_items_with_sources
+                file_input_dids = {DIDType(did) for did in file_item.get('parent_dids', [])}.intersection(input_dids)
+                if file_did in input_dids:
+                    file_input_dids.add(file_did)
+                file_item['input_dids'] = {did: input_dids[did] for did in file_input_dids}
+            merged_items_with_sources.extend(file_items)
 
-    def _prepare_items_for_download(self, did_to_options, merged_items_with_sources, resolve_archives=True):
+        return did_to_input_items, merged_items_with_sources
+
+    def _options_from_input_items(self, input_items):
+        """
+        Best-effort generation of download options from multiple input items which resolve to the same file DID.
+        This is done to download each file DID only once, even if it is requested multiple times via overlapping
+        datasets and/or wildcard resolutions in distinct input items.
+
+        Some options can be easily merged. For example: multiple base_dir are all appended to a list. As a result,
+        the file is downloaded once and copied to all desired destinations.
+        Other options are not necessarily compatible. For example, two items requesting two different values for
+        download timeout. We make our best to merge the options in such cases.
+        """
+        options = {}
+        for item in input_items:
+            base_dir = item.get('base_dir', '.')
+            no_subdir = item.get('no_subdir', False)
+            ignore_checksum = item.get('ignore_checksum', False)
+            new_transfer_timeout = item.get('transfer_timeout', None)
+            new_transfer_speed_timeout = item.get('transfer_speed_timeout', None)
+
+            options.setdefault('destinations', set()).add((base_dir, no_subdir))
+
+            # Merge some options
+            # The other options of this DID will be inherited from the first item that contained the DID
+            options['ignore_checksum'] = (options.get('ignore_checksum') or ignore_checksum)
+
+            # if one item wants to resolve archives we enable it for all items
+            options['resolve_archives'] = (options.get('resolve_archives') or not item.get('no_resolve_archives'))
+
+            cur_transfer_timeout = options.setdefault('transfer_timeout', None)
+            if cur_transfer_timeout is not None and new_transfer_timeout is not None:
+                options['transfer_timeout'] = max(int(cur_transfer_timeout), int(new_transfer_timeout))
+            elif new_transfer_timeout is not None:
+                options['transfer_timeout'] = int(new_transfer_timeout)
+
+            cur_transfer_speed_timeout = options.setdefault('transfer_speed_timeout', None)
+            if cur_transfer_speed_timeout is not None and new_transfer_speed_timeout is not None:
+                options['transfer_speed_timeout'] = min(float(cur_transfer_speed_timeout), float(new_transfer_speed_timeout))
+            elif new_transfer_speed_timeout is not None:
+                options['transfer_speed_timeout'] = float(new_transfer_speed_timeout)
+        return options
+
+    def _prepare_items_for_download(self, did_to_input_items, file_items):
         """
         Optimises the amount of files to download
         (This function is meant to be used as class internal only)
 
-        :param did_to_options: dictionary that maps each input DID to some input options
-        :param merged_items_with_sources: list of dictionaries. Each dictionary describes a bunch of DIDs to download
+        :param did_to_input_items: dictionary that maps resolved input DIDs to input items
+        :param file_items: list of dictionaries. Each dictionary describes a File DID to download
 
         :returns: list of dictionaries. Each dictionary describes an element to download
 
         :raises InputValidationError: if the given input is not valid or incomplete
         """
         logger = self.logger
-        if resolve_archives:
-            # perhaps we'll need an extraction tool so check what is installed
-            self.extraction_tools = [tool for tool in self.extraction_tools if tool.is_useable()]
-            if len(self.extraction_tools) < 1:
-                logger(logging.WARNING, 'Archive resolution is enabled but no extraction tool is available. '
-                                        'Sources whose protocol doesnt support extraction wont be considered for download.')
 
         # maps file item IDs (fiid) to the file item object
         fiid_to_file_item = {}
 
-        # list of all file item objects
-        all_file_items = []
-
         # cea -> client_extract archives to avoid confusion with archives that dont need explicit extraction
         # this dict will contain all ids of cea's that definitely will be downloaded
         cea_id_pure_to_fiids = {}
@@ -1219,133 +1250,115 @@ def _prepare_items_for_download(self, did_to_options, merged_items_with_sources,
         # this dict will contain ids of cea's that have higher prioritised non cea sources
         cea_id_mixed_to_fiids = {}
 
-        all_input_dids = set(did_to_options.keys())
         all_dest_file_paths = set()
 
         # get replicas for every file of the given dids
-        for file_items in merged_items_with_sources:
-            all_file_items.extend(file_items)
-            for file_item in file_items:
-                # parent_dids contains all parents, so we take the intersection with the input dids
-                dataset_did_strs = file_item.setdefault('parent_dids', set())
-                dataset_did_strs.intersection_update(all_input_dids)
-
-                file_did_str = file_item['did']
-                file_did_scope, file_did_name = self._split_did_str(file_did_str)
-                file_item['scope'] = file_did_scope
-                file_item['name'] = file_did_name
-
-                logger(logging.DEBUG, 'Queueing file: %s' % file_did_str)
-                logger(logging.DEBUG, 'real parents: %s' % dataset_did_strs)
-                logger(logging.DEBUG, 'options: %s' % did_to_options)
-
-                # prepare destinations:
-                # if datasets were given: prepare the destination paths for each dataset
-                options = None
-                dest_file_paths = file_item.get('dest_file_paths', set())
-                for dataset_did_str in dataset_did_strs:
-                    options = did_to_options.get(dataset_did_str)
-                    if not options:
-                        logger(logging.ERROR, 'No input options available for %s' % dataset_did_str)
-                        continue
+        for file_item in file_items:
+            file_did = DIDType(file_item['did'])
+            input_items = list(itertools.chain.from_iterable(did_to_input_items.get(did, []) for did in file_item['input_dids']))
+            options = self._options_from_input_items(input_items)
+
+            file_item['scope'] = file_did.scope
+            file_item['name'] = file_did.name
+
+            logger(logging.DEBUG, 'Queueing file: %s' % file_did)
+            logger(logging.DEBUG, 'real parents: %s' % [str(did) for did in file_item['input_dids'] if did != file_did])
+            logger(logging.DEBUG, 'options: %s' % options)
+
+            # prepare destinations folders:
+            dest_file_paths = file_item.get('dest_file_paths', set())
+            for input_did in file_item['input_dids']:
+                for item in did_to_input_items[input_did]:
+                    base_dir = item.get('base_dir', '.')
+                    no_subdir = item.get('no_subdir', False)
+                    file_did_path = file_did.name
+                    if input_did != file_did:
+                        # if datasets were given: prepare the destination paths for each dataset
+                        if self.extract_scope_convention == 'belleii' and file_did_path.startswith('/'):
+                            file_did_path = file_did_path.split('/')[-1]
+                        path = os.path.join(self._prepare_dest_dir(base_dir, input_did.name, no_subdir), file_did_path)
+                    else:
+                        # if no datasets were given only prepare the given destination paths
+                        if self.extract_scope_convention == 'belleii' and file_did_path.startswith('/'):
+                            file_did_path = file_did_path[1:]
+                        path = os.path.join(self._prepare_dest_dir(base_dir, file_did.scope, no_subdir), file_did_path)
 
-                    destinations = options['destinations']
-                    dataset_scope, dataset_name = self._split_did_str(dataset_did_str)
-                    file_did_path = file_did_name
-                    if self.extract_scope_convention == 'belleii' and file_did_path.startswith('/'):
-                        file_did_path = file_did_name.split('/')[-1]
-                    paths = [os.path.join(self._prepare_dest_dir(dest[0], dataset_name, dest[1]), file_did_path) for dest in destinations]
-                    if any(path in all_dest_file_paths for path in paths):
+                    if path in all_dest_file_paths:
                         raise RucioException("Multiple file items with same destination file path")
 
-                    all_dest_file_paths.update(paths)
-                    dest_file_paths.update(paths)
+                    all_dest_file_paths.add(path)
+                    dest_file_paths.add(path)
 
                     # workaround: just take any given dataset for the traces and the output
-                    file_item.setdefault('dataset_scope', dataset_scope)
-                    file_item.setdefault('dataset_name', dataset_name)
-
-                # if no datasets were given only prepare the given destination paths
-                if len(dataset_did_strs) == 0:
-                    options = did_to_options.get(file_did_str)
-                    if not options:
-                        logger(logging.ERROR, 'No input options available for %s' % file_did_str)
-                        continue
-                    destinations = options['destinations']
-                    file_did_path = file_did_name
-                    if self.extract_scope_convention == 'belleii' and file_did_path.startswith('/'):
-                        file_did_path = file_did_name[1:]
-                    paths = [os.path.join(self._prepare_dest_dir(dest[0], file_did_scope, dest[1]), file_did_path) for dest in destinations]
-                    if any(path in all_dest_file_paths for path in paths):
-                        raise RucioException("Multiple file items with same destination file path")
-                    all_dest_file_paths.update(paths)
-                    dest_file_paths.update(paths)
+                    file_item.setdefault('dataset_scope', input_did.scope)
+                    file_item.setdefault('dataset_name', input_did.name)
+
+            if not options:
+                continue
+            resolve_archives = options.get('resolve_archives')
+            file_item['merged_options'] = options
+            file_item['dest_file_paths'] = list(dest_file_paths)
+            file_item['temp_file_path'] = '%s.part' % file_item['dest_file_paths'][0]
+
+            # the file did str ist not an unique key for this dict because multiple calls of list_replicas
+            # could result in the same DID multiple times. So we're using the id of the dictionary objects
+            fiid = id(file_item)
+            fiid_to_file_item[fiid] = file_item
+
+            if resolve_archives:
+                min_cea_priority = None
+                num_non_cea_sources = 0
+                cea_ids = []
+                sources = []
+                # go through sources and check how many (non-)cea sources there are,
+                # index cea sources, or remove cea sources if there is no extraction tool
+                for source in file_item['sources']:
+                    is_cea = source.get('client_extract', False)
+                    if is_cea and (len(self.extraction_tools) > 0):
+                        priority = int(source['priority'])
+                        if min_cea_priority is None or priority < min_cea_priority:
+                            min_cea_priority = priority
+
+                        # workaround since we dont have the archive DID use the part behind the last slash of the PFN
+                        # this doesn't respect the scope of the archive DID!!!
+                        # and we trust that client_extract==True sources dont have any parameters at the end of the PFN
+                        cea_id = source['pfn'].split('/')
+                        cea_id = cea_id[-1] if len(cea_id[-1]) > 0 else cea_id[-2]
+                        cea_ids.append(cea_id)
+
+                        sources.append(source)
+                    elif not is_cea:
+                        num_non_cea_sources += 1
+                        sources.append(source)
+                    else:
+                        # no extraction tool
+                        logger(logging.DEBUG, 'client_extract=True; ignoring source: %s' % source['pfn'])
 
-                if options is None:
+                logger(logging.DEBUG, 'Prepared sources: num_sources=%d/%d; num_non_cea_sources=%d; num_cea_ids=%d'
+                                      % (len(sources), len(file_item['sources']), num_non_cea_sources, len(cea_ids)))
+
+                file_item['sources'] = sources
+
+                # if there are no cea sources we are done for this item
+                if min_cea_priority is None:
                     continue
-                file_item['merged_options'] = options
-                file_item['dest_file_paths'] = list(dest_file_paths)
-                file_item['temp_file_path'] = '%s.part' % file_item['dest_file_paths'][0]
-
-                # the file did str ist not an unique key for this dict because multiple calls of list_replicas
-                # could result in the same DID multiple times. So we're using the id of the dictionary objects
-                fiid = id(file_item)
-                fiid_to_file_item[fiid] = file_item
-
-                if resolve_archives:
-                    min_cea_priority = None
-                    num_non_cea_sources = 0
-                    cea_ids = []
-                    sources = []
-                    # go through sources and check how many (non-)cea sources there are,
-                    # index cea sources, or remove cea sources if there is no extraction tool
-                    for source in file_item['sources']:
-                        is_cea = source.get('client_extract', False)
-                        if is_cea and (len(self.extraction_tools) > 0):
-                            priority = int(source['priority'])
-                            if min_cea_priority is None or priority < min_cea_priority:
-                                min_cea_priority = priority
-
-                            # workaround since we dont have the archive DID use the part behind the last slash of the PFN
-                            # this doesn't respect the scope of the archive DID!!!
-                            # and we trust that client_extract==True sources dont have any parameters at the end of the PFN
-                            cea_id = source['pfn'].split('/')
-                            cea_id = cea_id[-1] if len(cea_id[-1]) > 0 else cea_id[-2]
-                            cea_ids.append(cea_id)
-
-                            sources.append(source)
-                        elif not is_cea:
-                            num_non_cea_sources += 1
-                            sources.append(source)
-                        else:
-                            # no extraction tool
-                            logger(logging.DEBUG, 'client_extract=True; ignoring source: %s' % source['pfn'])
-
-                    logger(logging.DEBUG, 'Prepared sources: num_sources=%d/%d; num_non_cea_sources=%d; num_cea_ids=%d'
-                                          % (len(sources), len(file_item['sources']), num_non_cea_sources, len(cea_ids)))
-
-                    file_item['sources'] = sources
-
-                    # if there are no cea sources we are done for this item
-                    if min_cea_priority is None:
-                        continue
-                    # decide if file item belongs to the pure or mixed map
-                    # if no non-archive src exists or the highest prio src is an archive src we put it in the pure map
-                    elif num_non_cea_sources == 0 or min_cea_priority == 1:
-                        logger(logging.DEBUG, 'Adding fiid to cea pure map: '
-                                              'num_non_cea_sources=%d; min_cea_priority=%d; num_cea_sources=%d'
-                                              % (num_non_cea_sources, min_cea_priority, len(cea_ids)))
-                        for cea_id in cea_ids:
-                            cea_id_pure_to_fiids.setdefault(cea_id, set()).add(fiid)
-                            file_item.setdefault('cea_ids_pure', set()).add(cea_id)
-                    # if there are non-archive sources and archive sources we put it in the mixed map
-                    elif len(cea_ids) > 0:
-                        logger(logging.DEBUG, 'Adding fiid to cea mixed map: '
-                                              'num_non_cea_sources=%d; min_cea_priority=%d; num_cea_sources=%d'
-                                              % (num_non_cea_sources, min_cea_priority, len(cea_ids)))
-                        for cea_id in cea_ids:
-                            cea_id_mixed_to_fiids.setdefault(cea_id, set()).add(fiid)
-                            file_item.setdefault('cea_ids_mixed', set()).add(cea_id)
+                # decide if file item belongs to the pure or mixed map
+                # if no non-archive src exists or the highest prio src is an archive src we put it in the pure map
+                elif num_non_cea_sources == 0 or min_cea_priority == 1:
+                    logger(logging.DEBUG, 'Adding fiid to cea pure map: '
+                                          'num_non_cea_sources=%d; min_cea_priority=%d; num_cea_sources=%d'
+                                          % (num_non_cea_sources, min_cea_priority, len(cea_ids)))
+                    for cea_id in cea_ids:
+                        cea_id_pure_to_fiids.setdefault(cea_id, set()).add(fiid)
+                        file_item.setdefault('cea_ids_pure', set()).add(cea_id)
+                # if there are non-archive sources and archive sources we put it in the mixed map
+                elif len(cea_ids) > 0:
+                    logger(logging.DEBUG, 'Adding fiid to cea mixed map: '
+                                          'num_non_cea_sources=%d; min_cea_priority=%d; num_cea_sources=%d'
+                                          % (num_non_cea_sources, min_cea_priority, len(cea_ids)))
+                    for cea_id in cea_ids:
+                        cea_id_mixed_to_fiids.setdefault(cea_id, set()).add(fiid)
+                        file_item.setdefault('cea_ids_mixed', set()).add(cea_id)
 
         # put all archives from the mixed list into the pure list if they meet
         # certain conditions, e.g., an archive that is already in the pure list
@@ -1382,7 +1395,7 @@ def _prepare_items_for_download(self, did_to_options, merged_items_with_sources,
             # finally remove cea_id from mixed map
             cea_id_mixed_to_fiids.pop(cea_id_mixed)
 
-        for file_item in all_file_items:
+        for file_item in file_items:
             cea_ids_pure = file_item.get('cea_ids_pure', set())
             cea_ids_mixed = file_item.get('cea_ids_mixed', set())
 
@@ -1415,7 +1428,7 @@ def _prepare_items_for_download(self, did_to_options, merged_items_with_sources,
 
         download_packs = []
         cea_id_to_pack = {}
-        for file_item in all_file_items:
+        for file_item in file_items:
             cea_ids = file_item.get('cea_ids_pure', set())
             if len(cea_ids) > 0:
                 cea_id = next(iter(cea_ids))


Test patch:
diff --git a/lib/rucio/tests/test_bin_rucio.py b/lib/rucio/tests/test_bin_rucio.py
--- a/lib/rucio/tests/test_bin_rucio.py
+++ b/lib/rucio/tests/test_bin_rucio.py
@@ -833,6 +833,8 @@ def test_download_metalink_file(self):
         cmd = 'rucio download --dir /tmp --metalink {0}'.format(metalink_file_path)
         exitcode, out, err = execute(cmd)
         print(out, err)
+        assert '{} successfully downloaded'.format(tmp_file_name) in err
+        assert re.search('Total files.*1', out) is not None
         remove(metalink_file_path)
         cmd = 'ls /tmp/{0}'.format(scope)
         exitcode, out, err = execute(cmd)
diff --git a/lib/rucio/tests/test_download.py b/lib/rucio/tests/test_download.py
--- a/lib/rucio/tests/test_download.py
+++ b/lib/rucio/tests/test_download.py
@@ -25,6 +25,7 @@
 # - Martin Barisits <martin.barisits@cern.ch>, 2021
 
 import logging
+import os
 import shutil
 import tarfile
 from unittest.mock import patch, MagicMock, ANY
@@ -34,12 +35,14 @@
 import pytest
 
 from rucio.client.downloadclient import DownloadClient
-from rucio.common.exception import InputValidationError, NoFilesDownloaded
+from rucio.common.exception import InputValidationError, NoFilesDownloaded, RucioException
+from rucio.common.types import InternalScope
 from rucio.common.utils import generate_uuid
 from rucio.core import did as did_core
+from rucio.core import scope as scope_core
 from rucio.rse import rsemanager as rsemgr
 from rucio.rse.protocols.posix import Default as PosixProtocol
-from rucio.tests.common import skip_rse_tests_with_accounts
+from rucio.tests.common import skip_rse_tests_with_accounts, scope_name_generator
 
 
 @pytest.fixture
@@ -92,6 +95,101 @@ def test_download_without_base_dir(rse_factory, did_factory, download_client):
         shutil.rmtree(scope)
 
 
+@pytest.mark.dirty(reason='creates a new scope which is not cleaned up')
+def test_overlapping_did_names(rse_factory, did_factory, download_client, root_account, mock_scope, vo):
+    """
+    Downloading two different did with different scope but same name to the same directory must fail
+    """
+    rse, _ = rse_factory.make_posix_rse()
+    scope1 = mock_scope
+    scope2 = InternalScope(scope_name_generator(), vo=vo)
+    scope_core.add_scope(scope2, root_account)
+    did1 = did_factory.upload_test_file(rse, scope=scope1)
+    did2 = did_factory.upload_test_file(rse, scope=scope2, name=did1['name'])
+    dataset = did_factory.make_dataset()
+    did_core.attach_dids(dids=[did1, did2], account=root_account, **dataset)
+
+    did1_str = '%s:%s' % (did1['scope'], did1['name'])
+    did2_str = '%s:%s' % (did2['scope'], did2['name'])
+    dataset1_did_str = '%s:%s' % (dataset['scope'], dataset['name'])
+
+    with TemporaryDirectory() as tmp_dir:
+        with pytest.raises(RucioException):
+            download_client.download_dids([{'did': dataset1_did_str, 'base_dir': tmp_dir, 'no_subdir': True}])
+
+    with TemporaryDirectory() as tmp_dir:
+        with pytest.raises(RucioException):
+            download_client.download_dids([{'did': did1_str, 'base_dir': tmp_dir, 'no_subdir': True},
+                                           {'did': did2_str, 'base_dir': tmp_dir, 'no_subdir': True}])
+
+
+def test_overlapping_containers_and_wildcards(rse_factory, did_factory, download_client, root_account):
+    """
+    Verify that wildcard resolution is correctly done. Overlapping containers and wildcards are handled without issues.
+    """
+    rse1, _ = rse_factory.make_posix_rse()
+    rse2, _ = rse_factory.make_posix_rse()
+    dids_on_rse1 = [did_factory.upload_test_file(rse1) for _ in range(5)]
+    dids_on_rse2 = [did_factory.upload_test_file(rse2) for _ in range(5)]
+    dids = dids_on_rse1 + dids_on_rse2
+    datasets = [did_factory.make_dataset() for _ in range(3)]
+    container = did_factory.make_container()
+    dids_in_dataset1, dids_in_dataset2, dids_in_dataset3 = dids[:6], dids[3:7], dids[4:]
+    did_core.attach_dids(dids=dids_in_dataset1, account=root_account, **datasets[0])
+    did_core.attach_dids(dids=dids_in_dataset2, account=root_account, **datasets[1])
+    did_core.attach_dids(dids=dids_in_dataset3, account=root_account, **datasets[2])
+    did_core.attach_dids(dids=datasets, account=root_account, **container)
+
+    dataset1_str, dataset2_str, dataset3_str = ['%s:%s' % (d['scope'], d['name']) for d in datasets]
+    container_str = '%s:%s' % (container['scope'], container['name'])
+
+    with TemporaryDirectory() as tmp_dir:
+        # No filters: all dids will be grouped and downloaded together
+        result = download_client.download_dids([{'did': dataset1_str, 'base_dir': tmp_dir},
+                                                {'did': dataset2_str, 'base_dir': tmp_dir},
+                                                {'did': dataset3_str, 'base_dir': tmp_dir},
+                                                {'did': container_str, 'base_dir': tmp_dir}])
+        assert len(result) == len(dids)
+
+    with TemporaryDirectory() as tmp_dir:
+        # Verify that wildcard resolution works correctly
+        result = download_client.download_dids([{'did': '%s:dataset_%s*' % (did_factory.default_scope, did_factory.base_uuid), 'base_dir': tmp_dir},
+                                                {'did': container_str, 'base_dir': tmp_dir}])
+        assert len(result) == len(dids)
+
+    with TemporaryDirectory() as tmp_dir:
+        # Test with an RSE filter
+        result = download_client.download_dids([{'did': dataset1_str, 'base_dir': tmp_dir, 'rse': rse1},
+                                                {'did': dataset2_str, 'base_dir': tmp_dir, 'rse': rse1},
+                                                {'did': dataset3_str, 'base_dir': tmp_dir, 'rse': rse1}])
+        assert len(result) == len(dids_on_rse1)
+
+    with TemporaryDirectory() as tmp_dir1, TemporaryDirectory() as tmp_dir2, TemporaryDirectory() as tmp_dir3:
+        # Test with nrandom
+        result = download_client.download_dids([{'did': dataset1_str, 'base_dir': tmp_dir1, 'nrandom': 3},
+                                                {'did': dataset2_str, 'base_dir': tmp_dir2, 'nrandom': 3},
+                                                {'did': dataset3_str, 'base_dir': tmp_dir3, 'nrandom': 3}])
+        assert 3 <= len(result) <= 9
+
+    with TemporaryDirectory() as tmp_dir1, TemporaryDirectory() as tmp_dir2:
+        # Test with filters complex overlapping of filters and different destination directories
+        download_client.download_dids([{'did': dataset1_str, 'base_dir': tmp_dir1, 'rse': rse1},
+                                       {'did': dataset2_str, 'base_dir': tmp_dir1, 'rse': rse2},
+                                       {'did': dataset3_str, 'base_dir': tmp_dir2, 'rse': rse2}])
+        dids_on_rse1_and_dataset1 = [d for d in dids_on_rse1 if d in dids_in_dataset1]
+        dids_on_rse2_and_dataset2 = [d for d in dids_on_rse2 if d in dids_in_dataset2]
+        dids_on_rse2_and_dataset3 = [d for d in dids_on_rse2 if d in dids_in_dataset3]
+
+        for dst_dir, expected_dids in (('%s/%s' % (tmp_dir1, datasets[0]['name']), dids_on_rse1_and_dataset1),
+                                       ('%s/%s' % (tmp_dir1, datasets[1]['name']), dids_on_rse2_and_dataset2),
+                                       ('%s/%s' % (tmp_dir2, datasets[2]['name']), dids_on_rse2_and_dataset3)):
+            files_in_dir = os.listdir(dst_dir)
+            for did in expected_dids:
+                assert did['name'] in files_in_dir
+
+            assert len(files_in_dir) == len(expected_dids)
+
+
 def test_download_to_two_paths(rse_factory, did_factory, download_client):
     scope = str(did_factory.default_scope)
     rse, _ = rse_factory.make_posix_rse()
@@ -323,6 +421,11 @@ def test_download_archive_client_extract(rse_factory, did_factory, download_clie
             {'scope': scope, 'name': name, 'bytes': len(data), 'type': 'FILE', 'adler32': adler32, 'meta': {'guid': str(generate_uuid())}},
         ])
 
+    with TemporaryDirectory() as tmp_dir:
+        with pytest.raises(NoFilesDownloaded):
+            # If archive resolution is disabled, the download must fail
+            download_client.download_dids([{'did': '%s:%s' % (scope, name), 'base_dir': tmp_dir, 'no_resolve_archives': True}])
+
     with TemporaryDirectory() as tmp_dir:
         result = download_client.download_dids([{'did': '%s:%s' % (scope, name), 'base_dir': tmp_dir}])
         assert len(result) == 1
@@ -409,22 +512,41 @@ def test_trace_copy_out_and_checksum_validation(vo, rse_factory, did_factory, do
         assert len(traces) == 1 and traces[0]['clientState'] == 'DONE'
 
 
-def test_norandom_respected(rse_factory, did_factory, download_client, root_account):
+def test_nrandom_respected(rse_factory, did_factory, download_client, root_account):
     rse, _ = rse_factory.make_posix_rse()
     did1 = did_factory.upload_test_file(rse)
     did2 = did_factory.upload_test_file(rse)
-    dataset = did_factory.make_dataset()
-    did_core.attach_dids(dids=[did1, did2], account=root_account, **dataset)
+    did3 = did_factory.upload_test_file(rse)
+    did4 = did_factory.upload_test_file(rse)
+    dataset1 = did_factory.make_dataset()
+    dataset2 = did_factory.make_dataset()
+    did_core.attach_dids(dids=[did1, did2], account=root_account, **dataset1)
+    did_core.attach_dids(dids=[did3, did4], account=root_account, **dataset2)
 
-    dataset_did_str = '%s:%s' % (dataset['scope'], dataset['name'])
+    dataset1_did_str = '%s:%s' % (dataset1['scope'], dataset1['name'])
+    dataset2_did_str = '%s:%s' % (dataset2['scope'], dataset2['name'])
 
     with TemporaryDirectory() as tmp_dir:
         nrandom = 1
-        result = download_client.download_dids([{'did': dataset_did_str, 'nrandom': nrandom, 'base_dir': tmp_dir}])
+        result = download_client.download_dids([{'did': dataset1_did_str, 'nrandom': nrandom, 'base_dir': tmp_dir}])
         assert len(result) == nrandom
 
+    with TemporaryDirectory() as tmp_dir:
+        # If two separate items are provided, nrandom applies to each item separately
+        nrandom = 1
+        result = download_client.download_dids([{'did': dataset1_did_str, 'nrandom': nrandom, 'base_dir': tmp_dir},
+                                                {'did': dataset2_did_str, 'nrandom': nrandom, 'base_dir': tmp_dir}])
+        assert len(result) == 2 * nrandom
+
+    with TemporaryDirectory() as tmp_dir:
+        # If a single item is provided, but it resolves to two datasets, only a single file will be downloaded
+        nrandom = 1
+        result = download_client.download_dids([{'did': '%s:dataset_%s*' % (did_factory.default_scope, did_factory.base_uuid), 'nrandom': nrandom, 'base_dir': tmp_dir}])
+        assert len(result) == nrandom
+
+    with TemporaryDirectory() as tmp_dir:
         nrandom = 2
-        result = download_client.download_dids([{'did': dataset_did_str, 'nrandom': nrandom, 'base_dir': tmp_dir}])
+        result = download_client.download_dids([{'did': dataset1_did_str, 'nrandom': nrandom, 'base_dir': tmp_dir}])
         assert len(result) == nrandom
 
 
diff --git a/lib/rucio/tests/test_judge_evaluator.py b/lib/rucio/tests/test_judge_evaluator.py
--- a/lib/rucio/tests/test_judge_evaluator.py
+++ b/lib/rucio/tests/test_judge_evaluator.py
@@ -40,6 +40,8 @@
 from rucio.daemons.abacus.account import account_update
 from rucio.daemons.judge.evaluator import re_evaluator
 from rucio.db.sqla.constants import DIDType
+from rucio.db.sqla.models import UpdatedDID
+from rucio.db.sqla.session import transactional_session
 from rucio.tests.common_server import get_vo
 from rucio.tests.test_rule import create_files, tag_generator
 
@@ -54,6 +56,12 @@ def setUpClass(cls):
         else:
             cls.vo = {}
 
+        @transactional_session
+        def __cleanup_updated_dids(session=None):
+            session.query(UpdatedDID).delete()
+
+        __cleanup_updated_dids()
+
         # Add test RSE
         cls.rse1 = 'MOCK'
         cls.rse3 = 'MOCK3'