Repository: rucio/rucio

Issue:
Better error propagation for download and upload client
Motivation
----------

Pilots using rucio mover do not report useful information on why a transfer failed. This is because on failure the download and upload client throw an exception with no information inside. The real reason for the error should be propagated better back to the caller of the client.


Modification
------------

The dowloadclient and uploadclient should return a dictionary of file status (containing appropriate error information) instead of raising an exception.



Discussion:
Please assign to me :)
avec plaisir ;-)
I think there is a bit of duplication between #2408, #2168 and this. Would be good to clarify what is done where. 😄 

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -27,7 +27,6 @@
 import copy
 import logging
 import os
-import os.path
 import random
 import shutil
 import signal
@@ -40,7 +39,7 @@
 from threading import Thread
 
 from rucio.client.client import Client
-from rucio.common.exception import (InputValidationError, NoFilesDownloaded, NotAllFilesDownloaded, RSENotFound, RucioException)
+from rucio.common.exception import (InputValidationError, NoFilesDownloaded, NotAllFilesDownloaded, RucioException)
 from rucio.common.pcache import Pcache
 from rucio.common.utils import adler32, md5, detect_client_location, generate_uuid, parse_replicas_from_string, send_trace, sizefmt, execute, parse_replicas_from_file
 from rucio.rse import rsemanager as rsemgr
@@ -471,8 +470,8 @@ def _download_item(self, item, trace, traces_copy_out, log_prefix=''):
         if not sources or not len(sources):
             logger.warning('%sNo available source found for file: %s' % (log_prefix, did_str))
             item['clientState'] = 'FILE_NOT_FOUND'
-
             trace['clientState'] = 'FILE_NOT_FOUND'
+            trace['stateReason'] = 'No available sources'
             self._send_trace(trace)
             return item
 
@@ -526,8 +525,9 @@ def _download_item(self, item, trace, traces_copy_out, log_prefix=''):
 
             try:
                 rse = rsemgr.get_rse_info(rse_name)
-            except RSENotFound:
-                logger.warning('%sCould not get info of RSE %s' % (log_prefix, rse_name))
+            except RucioException as error:
+                logger.warning('%sCould not get info of RSE %s: %s' % (log_prefix, rse_name, error))
+                trace['stateReason'] = str(error)
                 continue
 
             trace['remoteSite'] = rse_name
@@ -542,6 +542,7 @@ def _download_item(self, item, trace, traces_copy_out, log_prefix=''):
             except Exception as error:
                 logger.warning('%sFailed to create protocol for PFN: %s' % (log_prefix, pfn))
                 logger.debug('scheme: %s, exception: %s' % (scheme, error))
+                trace['stateReason'] = str(error)
                 continue
 
             attempt = 0
@@ -563,6 +564,7 @@ def _download_item(self, item, trace, traces_copy_out, log_prefix=''):
                 except Exception as error:
                     logger.debug(error)
                     trace['clientState'] = str(type(error).__name__)
+                    trace['stateReason'] = str(error)
 
                 end_time = time.time()
 
@@ -584,6 +586,7 @@ def _download_item(self, item, trace, traces_copy_out, log_prefix=''):
                         logger.warning('%sChecksum validation failed for file: %s' % (log_prefix, did_str))
                         logger.debug('Local checksum: %s, Rucio checksum: %s' % (local_checksum, rucio_checksum))
                         trace['clientState'] = 'FAIL_VALIDATE'
+                        trace['stateReason'] = 'Checksum validation failed: Local checksum: %s, Rucio checksum: %s' % (local_checksum, rucio_checksum)
                 if not success:
                     logger.warning('%sDownload attempt failed. Try %s/%s' % (log_prefix, attempt, retries))
                     self._send_trace(trace)
@@ -616,6 +619,7 @@ def _download_item(self, item, trace, traces_copy_out, log_prefix=''):
         trace['transferStart'] = start_time
         trace['transferEnd'] = end_time
         trace['clientState'] = 'DONE'
+        trace['stateReason'] = 'OK'
         item['clientState'] = 'DONE'
         self._send_trace(trace)
 
diff --git a/lib/rucio/client/uploadclient.py b/lib/rucio/client/uploadclient.py
--- a/lib/rucio/client/uploadclient.py
+++ b/lib/rucio/client/uploadclient.py
@@ -68,7 +68,7 @@ def __init__(self, _client=None, logger=None, tracing=True):
         self.trace['eventType'] = 'upload'
         self.trace['eventVersion'] = version.RUCIO_VERSION[0]
 
-    def upload(self, items, summary_file_path=None):
+    def upload(self, items, summary_file_path=None, traces_copy_out=None):
         """
         :param items: List of dictionaries. Each dictionary describing a file to upload. Keys:
             path                  - path of the file that will be uploaded
@@ -85,6 +85,7 @@ def upload(self, items, summary_file_path=None):
             transfer_timeout      - Optional: time after the upload will be aborted
             guid                  - Optional: guid of the file
         :param summary_file_path: Optional: a path where a summary in form of a json file will be stored
+        :param traces_copy_out: reference to an external list, where the traces should be uploaded
 
         :returns: 0 on success
 
@@ -140,11 +141,16 @@ def upload(self, items, summary_file_path=None):
             force_scheme = file.get('force_scheme')
             delete_existing = False
 
-            self.trace['scope'] = file['did_scope']
-            self.trace['datasetScope'] = file.get('dataset_scope', '')
-            self.trace['dataset'] = file.get('dataset_name', '')
-            self.trace['remoteSite'] = rse
-            self.trace['filesize'] = file['bytes']
+            trace = copy.deepcopy(self.trace)
+            # appending trace to list reference, if the reference exists
+            if traces_copy_out is not None:
+                traces_copy_out.append(trace)
+
+            trace['scope'] = file['did_scope']
+            trace['datasetScope'] = file.get('dataset_scope', '')
+            trace['dataset'] = file.get('dataset_name', '')
+            trace['remoteSite'] = rse
+            trace['filesize'] = file['bytes']
 
             file_did = {'scope': file['did_scope'], 'name': file['did_name']}
             dataset_did_str = file.get('dataset_did_str')
@@ -170,6 +176,7 @@ def upload(self, items, summary_file_path=None):
                     try:
                         self.client.get_did(file['did_scope'], file['did_name'])
                         logger.info('File already registered. Skipping upload.')
+                        trace['stateReason'] = 'File already exists'
                         continue
                     except DataIdentifierNotFound:
                         logger.info('File already exists on RSE. Previous left overs will be overwritten.')
@@ -177,13 +184,16 @@ def upload(self, items, summary_file_path=None):
             elif not is_deterministic and not no_register:
                 if rsemgr.exists(rse_settings, pfn):
                     logger.info('File already exists on RSE with given pfn. Skipping upload. Existing replica has to be removed first.')
+                    trace['stateReason'] = 'File already exists'
                     continue
                 elif rsemgr.exists(rse_settings, file_did):
                     logger.info('File already exists on RSE with different pfn. Skipping upload.')
+                    trace['stateReason'] = 'File already exists'
                     continue
             else:
                 if rsemgr.exists(rse_settings, pfn if pfn else file_did):
                     logger.info('File already exists on RSE. Skipping upload')
+                    trace['stateReason'] = 'File already exists'
                     continue
             protocols = rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=force_scheme)
             protocols.reverse()
@@ -204,8 +214,8 @@ def upload(self, items, summary_file_path=None):
                 if cur_scheme == 'https':
                     sign_service = rse_sign_service
 
-                self.trace['protocol'] = cur_scheme
-                self.trace['transferStart'] = time.time()
+                trace['protocol'] = cur_scheme
+                trace['transferStart'] = time.time()
                 try:
                     state = rsemgr.upload(rse_settings=rse_settings,
                                           lfns=lfn,
@@ -224,11 +234,11 @@ def upload(self, items, summary_file_path=None):
 
             if success:
                 num_succeeded += 1
-                self.trace['transferEnd'] = time.time()
-                self.trace['clientState'] = 'DONE'
+                trace['transferEnd'] = time.time()
+                trace['clientState'] = 'DONE'
                 file['state'] = 'A'
                 logger.info('Successfully uploaded file %s' % basename)
-                self._send_trace(self.trace)
+                self._send_trace(trace)
 
                 if summary_file_path:
                     summary.append(copy.deepcopy(file))
@@ -248,9 +258,9 @@ def upload(self, items, summary_file_path=None):
                         logger.warning('Failed to attach file to the dataset')
                         logger.debug(error)
             else:
-                self.trace['clientState'] = 'FAILED'
-                self.trace['stateReason'] = state_reason
-                self._send_trace(self.trace)
+                trace['clientState'] = 'FAILED'
+                trace['stateReason'] = state_reason
+                self._send_trace(trace)
                 logger.error('Failed to upload file %s' % basename)
 
         if summary_file_path:


Test patch: