Repository: rucio/rucio

Issue:
Add an option not to raise NoFilesDownloaded or NotAllFilesDownloaded in downloadclient
Motivation
----------
If there's a problem in the download client and some files cannot be downloaded, an exception `NoFilesDownloaded` or `NotAllFilesDownloaded` is raised. Because of this it is not possible to know which files were successfully downloaded. It should be possible to specify an option in all the download methods not to raise an exception (similar to what is done here : https://github.com/rucio/rucio/blob/master/lib/rucio/common/config.py#L56) and just to return the output dictionary with a `FAILED` status for those files.





Discussion:

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -31,8 +31,9 @@
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2021
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
 # - Rakshita Varadarajan <rakshitajps@gmail.com>, 2021
-# - David Población Criado <13998309+davidpob99@users.noreply.github.com>, 2021
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2021
+# - Joel Dierkes <joel.dierkes@cern.ch>, 2021
 
 from __future__ import division
 
@@ -186,7 +187,7 @@ def __init__(self, client=None, logger=None, tracing=True, check_admin=False, ch
         self.extraction_tools.append(BaseExtractionTool('tar', '--version', extract_args, logger=self.logger))
         self.extract_scope_convention = config_get('common', 'extract_scope', False, None)
 
-    def download_pfns(self, items, num_threads=2, trace_custom_fields={}, traces_copy_out=None):
+    def download_pfns(self, items, num_threads=2, trace_custom_fields={}, traces_copy_out=None, deactivate_file_download_exceptions=False):
         """
         Download items with a given PFN. This function can only download files, no datasets.
 
@@ -202,6 +203,8 @@ def download_pfns(self, items, num_threads=2, trace_custom_fields={}, traces_cop
         :param num_threads: Suggestion of number of threads to use for the download. It will be lowered if it's too high.
         :param trace_custom_fields: Custom key value pairs to send with the traces
         :param traces_copy_out: reference to an external list, where the traces should be uploaded
+        :param deactivate_file_download_exceptions: Boolean, if file download exceptions shouldn't be raised
+
 
         :returns: a list of dictionaries with an entry for each file, containing the input options, the did, and the clientState
                   clientState can be one of the following: ALREADY_DONE, DONE, FILE_NOT_FOUND, FAIL_VALIDATE, FAILED
@@ -253,12 +256,12 @@ def download_pfns(self, items, num_threads=2, trace_custom_fields={}, traces_cop
         output_items = self._download_multithreaded(input_items, num_threads, trace_custom_fields, traces_copy_out)
         num_files_out = len(output_items)
 
-        if num_files_in != num_files_out:
+        if not deactivate_file_download_exceptions and num_files_in != num_files_out:
             raise RucioException('%d items were in the input queue but only %d are in the output queue' % (num_files_in, num_files_out))
 
-        return self._check_output(output_items)
+        return self._check_output(output_items, deactivate_file_download_exceptions=deactivate_file_download_exceptions)
 
-    def download_dids(self, items, num_threads=2, trace_custom_fields={}, traces_copy_out=None):
+    def download_dids(self, items, num_threads=2, trace_custom_fields={}, traces_copy_out=None, deactivate_file_download_exceptions=False):
         """
         Download items with given DIDs. This function can also download datasets and wildcarded DIDs.
 
@@ -279,6 +282,7 @@ def download_dids(self, items, num_threads=2, trace_custom_fields={}, traces_cop
         :param num_threads: Suggestion of number of threads to use for the download. It will be lowered if it's too high.
         :param trace_custom_fields: Custom key value pairs to send with the traces.
         :param traces_copy_out: reference to an external list, where the traces should be uploaded
+        :param deactivate_file_download_exceptions: Boolean, if file download exceptions shouldn't be raised
 
         :returns: a list of dictionaries with an entry for each file, containing the input options, the did, and the clientState
 
@@ -300,12 +304,12 @@ def download_dids(self, items, num_threads=2, trace_custom_fields={}, traces_cop
         output_items = self._download_multithreaded(input_items, num_threads, trace_custom_fields, traces_copy_out)
         num_files_out = len(output_items)
 
-        if num_files_in != num_files_out:
+        if not deactivate_file_download_exceptions and num_files_in != num_files_out:
             raise RucioException('%d items were in the input queue but only %d are in the output queue' % (num_files_in, num_files_out))
 
-        return self._check_output(output_items)
+        return self._check_output(output_items, deactivate_file_download_exceptions=deactivate_file_download_exceptions)
 
-    def download_from_metalink_file(self, item, metalink_file_path, num_threads=2, trace_custom_fields={}, traces_copy_out=None):
+    def download_from_metalink_file(self, item, metalink_file_path, num_threads=2, trace_custom_fields={}, traces_copy_out=None, deactivate_file_download_exceptions=False):
         """
         Download items using a given metalink file.
 
@@ -317,6 +321,7 @@ def download_from_metalink_file(self, item, metalink_file_path, num_threads=2, t
         :param num_threads: Suggestion of number of threads to use for the download. It will be lowered if it's too high.
         :param trace_custom_fields: Custom key value pairs to send with the traces.
         :param traces_copy_out: reference to an external list, where the traces should be uploaded
+        :param deactivate_file_download_exceptions: Boolean, if file download exceptions shouldn't be raised
 
         :returns: a list of dictionaries with an entry for each file, containing the input options, the did, and the clientState
 
@@ -344,10 +349,10 @@ def download_from_metalink_file(self, item, metalink_file_path, num_threads=2, t
         output_items = self._download_multithreaded(input_items, num_threads, trace_custom_fields, traces_copy_out)
         num_files_out = len(output_items)
 
-        if num_files_in != num_files_out:
+        if not deactivate_file_download_exceptions and num_files_in != num_files_out:
             raise RucioException('%d items were in the input queue but only %d are in the output queue' % (num_files_in, num_files_out))
 
-        return self._check_output(output_items)
+        return self._check_output(output_items, deactivate_file_download_exceptions=deactivate_file_download_exceptions)
 
     def _download_multithreaded(self, input_items, num_threads, trace_custom_fields={}, traces_copy_out=None):
         """
@@ -720,7 +725,7 @@ def _download_item(self, item, trace, traces_copy_out, log_prefix=''):
 
         return item
 
-    def download_aria2c(self, items, trace_custom_fields={}, filters={}):
+    def download_aria2c(self, items, trace_custom_fields={}, filters={}, deactivate_file_download_exceptions=False):
         """
         Uses aria2c to download the items with given DIDs. This function can also download datasets and wildcarded DIDs.
         It only can download files that are available via https/davs.
@@ -735,6 +740,7 @@ def download_aria2c(self, items, trace_custom_fields={}, filters={}):
             ignore_checksum     - Optional: If true, skips the checksum validation between the downloaded file and the rucio catalouge. (Default: False)
         :param trace_custom_fields: Custom key value pairs to send with the traces
         :param filters: dictionary containing filter options
+        :param deactivate_file_download_exceptions: Boolean, if file download exceptions shouldn't be raised
 
         :returns: a list of dictionaries with an entry for each file, containing the input options, the did, and the clientState
 
@@ -771,7 +777,7 @@ def download_aria2c(self, items, trace_custom_fields={}, filters={}):
             finally:
                 rpcproc.terminate()
 
-        return self._check_output(output_items)
+        return self._check_output(output_items, deactivate_file_download_exceptions=deactivate_file_download_exceptions)
 
     def _start_aria2c_rpc(self, rpc_secret):
         """
@@ -1529,12 +1535,13 @@ def _prepare_dest_dir(self, base_dir, dest_dir_name, no_subdir):
 
         return dest_dir_path
 
-    def _check_output(self, output_items):
+    def _check_output(self, output_items, deactivate_file_download_exceptions=False):
         """
         Checks if all files were successfully downloaded
         (This function is meant to be used as class internal only)
 
         :param output_items: list of dictionaries describing the downloaded files
+        :param deactivate_file_download_exceptions: Boolean, if file download exceptions shouldn't be raised
 
         :returns: output_items list
 
@@ -1552,9 +1559,9 @@ def _check_output(self, output_items):
             else:
                 num_failed += 1
 
-        if num_successful == 0:
+        if not deactivate_file_download_exceptions and num_successful == 0:
             raise NoFilesDownloaded()
-        elif num_failed > 0:
+        elif not deactivate_file_download_exceptions and num_failed > 0:
             raise NotAllFilesDownloaded()
 
         return output_items


Test patch:
diff --git a/lib/rucio/tests/test_download.py b/lib/rucio/tests/test_download.py
--- a/lib/rucio/tests/test_download.py
+++ b/lib/rucio/tests/test_download.py
@@ -24,6 +24,7 @@
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
 # - Martin Barisits <martin.barisits@cern.ch>, 2021
 # - Rakshita Varadarajan <rakshitajps@gmail.com>, 2021
+# - Joel Dierkes <joel.dierkes@cern.ch>, 2021
 
 import logging
 import os
@@ -515,6 +516,14 @@ def test_trace_copy_out_and_checksum_validation(vo, rse_factory, did_factory, do
         assert len(traces) == 1 and traces[0]['clientState'] == 'DONE'
 
 
+def test_disable_no_files_download_error(vo, rse_factory, did_factory, download_client):
+    rse, _ = rse_factory.make_posix_rse()
+    with TemporaryDirectory() as tmp_dir:
+        res = download_client.download_dids([{'did': 'some:randomNonExistingDid', 'base_dir': tmp_dir}], deactivate_file_download_exceptions=True)
+        print('Downloaded object', res)
+        assert res[0]['clientState'] == 'FILE_NOT_FOUND'
+
+
 def test_nrandom_respected(rse_factory, did_factory, download_client, root_account):
     rse, _ = rse_factory.make_posix_rse()
     did1 = did_factory.upload_test_file(rse)