Repository: rucio/rucio

Issue:
Download API crashes if called with at least two not existing DIDs
Motivation
----------
On the request of a more detailed error message from the download client if the DID doesn't exist these lines were patched in:
https://github.com/rucio/rucio/blob/master/lib/rucio/client/downloadclient.py#L1081-L1090
```
            for input_did in item['dids']:
                input_did_str = '%s:%s' % (input_did['scope'], input_did['name'])
                did_exists = False
                for file_item in file_items:
                    if input_did_str == file_item['did'] or input_did_str in file_item['parent_dids']:
                        did_exists = True
                        break
                if not did_exists:
                    logger.error('No replicas found for DIDs: %s' % input_did_str)
                    file_items.append({'did': input_did_str, 'adler32': None, 'md5': None, 'sources': []})
```

If the download is called with at least two not existing DIDs it crashes because it adds a source-less entry to `file_items` without the `'parent_dids'` key set.


Modification
------------
Add by default `'parent_dids': set()` to the source-less did entry:
`file_items.append({'did': input_did_str, 'adler32': None, 'md5': None, 'sources': []})`
https://github.com/rucio/rucio/blob/master/lib/rucio/client/downloadclient.py#L1090





Discussion:

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -41,6 +41,7 @@
 
 from rucio.client.client import Client
 from rucio.common.exception import (InputValidationError, NoFilesDownloaded, NotAllFilesDownloaded, RucioException)
+from rucio.common.didtype import DIDType
 from rucio.common.pcache import Pcache
 from rucio.common.utils import adler32, detect_client_location, generate_uuid, parse_replicas_from_string, \
     send_trace, sizefmt, execute, parse_replicas_from_file
@@ -1078,16 +1079,15 @@ def _get_sources(self, merged_items, resolve_archives=True):
 
             logger.debug('num resolved files: %s' % len(file_items))
 
+            # list_replicas returns nothing if the DID does not exist and we dont want to
+            # do another server call so we check if there is a result from list_replicas
+            # for each given DID. If not the DID does not exist
             for input_did in item['dids']:
-                input_did_str = '%s:%s' % (input_did['scope'], input_did['name'])
-                did_exists = False
-                for file_item in file_items:
-                    if input_did_str == file_item['did'] or input_did_str in file_item['parent_dids']:
-                        did_exists = True
-                        break
-                if not did_exists:
-                    logger.error('No replicas found for DIDs: %s' % input_did_str)
-                    file_items.append({'did': input_did_str, 'adler32': None, 'md5': None, 'sources': []})
+                input_did = DIDType(input_did)
+                if not any([input_did == f['did'] or str(input_did) in f['parent_dids'] for f in file_items]):
+                    logger.error('DID does not exist: %s' % input_did)
+                    # TODO: store did directly as DIDType object
+                    file_items.append({'did': str(input_did), 'adler32': None, 'md5': None, 'sources': [], 'parent_dids': set()})
 
             nrandom = item.get('nrandom')
             if nrandom:
diff --git a/lib/rucio/common/didtype.py b/lib/rucio/common/didtype.py
--- a/lib/rucio/common/didtype.py
+++ b/lib/rucio/common/didtype.py
@@ -21,6 +21,8 @@
 DID type to represent a did and to simplify operations on it
 """
 
+from six import string_types
+
 from rucio.common.exception import DIDTypeError
 
 
@@ -70,7 +72,7 @@ def __init__(self, *args, **kwargs):
             did = args[0]
 
             if num_kwargs == 1:
-                if not isinstance(did, type('')):
+                if not isinstance(did, string_types):
                     raise DIDTypeError('First argument of constructor is expected to be string type'
                                        'when keyword argument is given. Given type: {}'.format(type(did)))
 
@@ -96,7 +98,7 @@ def __init__(self, *args, **kwargs):
                 raise DIDTypeError('Construction from tuple or list requires exactly 2 elements')
             self.scope = did[0]
             self.name = did[1]
-        elif isinstance(did, type('')):
+        elif isinstance(did, string_types):
             did_parts = did.split(DIDType.SCOPE_SEPARATOR, 1)
             if len(did_parts) == 1:
                 self.name = did
@@ -166,7 +168,7 @@ def __eq__(self, other):
         Equality comparison with another object
         :return: bool
         """
-        if isinstance(other, type('')):
+        if isinstance(other, string_types):
             return str(self) == other
         elif not isinstance(other, DIDType):
             try:


Test patch: