Repository: rucio/rucio

Issue:
Transfers : Modify the verify_checksum parameter passed to FTS
Motivation
----------
```
Hi Cedric,

the checksum validation was improved to have all checksum possibilities. Before, when you where indicating verify checksum True was applying the checksum to both (source and destination) and if you were using False the verification was not applied.
I just keep the True and False for compatibility with the API. But, we encouraged when this new feature was presented in the steering, to use both if you want a checksum verification in source and destination, source when you only wants the verification on the source, destination just to verify in destination and none for no verification.

So, you can provide the verification that adapts better to your needs as it is documented.

Cheers,
Maria
```






Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -533,6 +533,8 @@ def get_transfer_requests_and_source_replicas(process=None, total_processes=None
                 if source_rse_id not in rses_info:
                     source_rse = get_rse_name(rse_id=source_rse_id, session=session)
                     rses_info[source_rse_id] = rsemgr.get_rse_info(source_rse, session=session)
+                if source_rse_id not in rse_attrs:
+                    rse_attrs[source_rse_id] = get_rse_attributes(source_rse_id, session=session)
 
                 attr = None
                 if attributes:
@@ -659,6 +661,18 @@ def get_transfer_requests_and_source_replicas(process=None, total_processes=None
                     retry_count = 0
                 fts_list = fts_hosts.split(",")
 
+                verify_checksum = 'both'
+                if rse_attrs[dest_rse_id].get('verify_checksum', 'True').lower() == 'false':
+                    if rse_attrs[source_rse_id].get('verify_checksum', 'True').lower() == 'false':
+                        verify_checksum = 'none'
+                    else:
+                        verify_checksum = 'source'
+                else:
+                    if rse_attrs[source_rse_id].get('verify_checksum', 'True').lower() == 'false':
+                        verify_checksum = 'destination'
+                    else:
+                        verify_checksum = 'both'
+
                 external_host = fts_list[0]
                 if retry_other_fts:
                     external_host = fts_list[retry_count % len(fts_list)]
@@ -677,7 +691,7 @@ def get_transfer_requests_and_source_replicas(process=None, total_processes=None
                                  'filesize': bytes,
                                  'md5': md5,
                                  'adler32': adler32,
-                                 'verify_checksum': json.loads(rse_attrs[dest_rse_id].get('verify_checksum', 'True').lower())}
+                                 'verify_checksum': verify_checksum}
 
                 if previous_attempt_id:
                     file_metadata['previous_attempt_id'] = previous_attempt_id
diff --git a/lib/rucio/daemons/conveyor/common.py b/lib/rucio/daemons/conveyor/common.py
--- a/lib/rucio/daemons/conveyor/common.py
+++ b/lib/rucio/daemons/conveyor/common.py
@@ -7,7 +7,7 @@
 # Authors:
 # - Vincent Garonne, <vincent.garonne@cern.ch>, 2012-2014
 # - Mario Lassnig, <mario.lassnig@cern.ch>, 2013-2018
-# - Cedric Serfon, <cedric.serfon@cern.ch>, 2013-2016
+# - Cedric Serfon, <cedric.serfon@cern.ch>, 2013-2018
 # - Wen Guan, <wen.guan@cern.ch>, 2014-2016
 # - Joaquin Bogado, <jbogadog@cern.ch>, 2016
 # - Martin Barisits, <martin.barisits@cern.ch>, 2017
@@ -64,7 +64,7 @@ def submit_transfer(external_host, job, submitter='submitter', process=0, thread
         transfer_core.prepare_sources_for_transfers(xfers_ret)
         logging.debug("%s:%s finished to prepare transfer" % (process, thread))
     except:
-        logging.error("%s:%s Failed to prepare requests %s state to SUBMITTING(Will not submit jobs but return directly) with error: %s" % (process, thread, xfers_ret.keys(), traceback.format_exc()))
+        logging.error("%s:%s Failed to prepare requests %s state to SUBMITTING(Will not submit jobs but return directly) with error: %s" % (process, thread, list(xfers_ret.keys()), traceback.format_exc()))
         return
 
     # submit the job
@@ -78,8 +78,8 @@ def submit_transfer(external_host, job, submitter='submitter', process=0, thread
         record_timer('daemons.conveyor.%s.submit_bulk_transfer.per_file' % submitter, (time.time() - ts) * 1000 / len(job['files']))
         record_counter('daemons.conveyor.%s.submit_bulk_transfer' % submitter, len(job['files']))
         record_timer('daemons.conveyor.%s.submit_bulk_transfer.files' % submitter, len(job['files']))
-    except Exception, ex:
-        logging.error("Failed to submit a job with error %s: %s" % (str(ex), traceback.format_exc()))
+    except Exception as error:
+        logging.error("Failed to submit a job with error %s: %s" % (str(error), traceback.format_exc()))
 
     # register transfer
     xfers_ret = {}
@@ -153,16 +153,17 @@ def bulk_group_transfer(transfers, policy='rule', group_bulk=200, fts_source_str
                 'metadata': transfer['file_metadata'],
                 'filesize': int(transfer['file_metadata']['filesize']),
                 'checksum': None,
+                'verify_checksum': transfer['verify_checksum'],
                 'selection_strategy': fts_source_strategy,
                 'request_type': transfer['file_metadata'].get('request_type', None),
                 'activity': str(transfer['file_metadata']['activity'])}
         if file['metadata'].get('verify_checksum', True):
-            if 'md5' in file['metadata'].keys() and file['metadata']['md5']:
+            if 'md5' in list(file['metadata'].keys()) and file['metadata']['md5']:
                 file['checksum'] = 'MD5:%s' % str(file['metadata']['md5'])
-            if 'adler32' in file['metadata'].keys() and file['metadata']['adler32']:
+            if 'adler32' in list(file['metadata'].keys()) and file['metadata']['adler32']:
                 file['checksum'] = 'ADLER32:%s' % str(file['metadata']['adler32'])
 
-        job_params = {'verify_checksum': True if file['checksum'] and file['metadata'].get('verify_checksum', True) else False,
+        job_params = {'verify_checksum': transfer['verify_checksum'],
                       'copy_pin_lifetime': transfer['copy_pin_lifetime'] if transfer['copy_pin_lifetime'] else -1,
                       'bring_online': transfer['bring_online'] if transfer['bring_online'] else None,
                       'job_metadata': {'issuer': 'rucio'},  # finaly job_meta will like this. currently job_meta will equal file_meta to include request_id and etc.
@@ -244,7 +245,7 @@ def get_conveyor_rses(rses=None, include_rses=None, exclude_rses=None):
     if include_rses:
         try:
             parsed_rses = parse_expression(include_rses, session=None)
-        except InvalidRSEExpression, e:
+        except InvalidRSEExpression as error:
             logging.error("Invalid RSE exception %s to include RSEs" % (include_rses))
         else:
             for rse in parsed_rses:
@@ -257,8 +258,8 @@ def get_conveyor_rses(rses=None, include_rses=None, exclude_rses=None):
     if exclude_rses:
         try:
             parsed_rses = parse_expression(exclude_rses, session=None)
-        except InvalidRSEExpression, e:
-            logging.error("Invalid RSE exception %s to exclude RSEs: %s" % (exclude_rses, e))
+        except InvalidRSEExpression as error:
+            logging.error("Invalid RSE exception %s to exclude RSEs: %s" % (exclude_rses, error))
         else:
             working_rses = [rse for rse in working_rses if rse not in parsed_rses]
 


Test patch: