Repository: rucio/rucio
Issue: Rucio Upload Should Respect verify_checksum Flag

Motivation
----------
When uploading to `CERN-EXTENSION_GRIDDISK` the Rucio client fails because it attempts to verify the checksum. At the moment the `verify_checksum` flag is set for the RSE and not ask the storage for the checksum.

The suggested modification will then also skip the verification of the file size on storage. The stat call checking the file size should be OK, and in this scenario is even more important to be verified (in [rsemgr](https://github.com/rucio/rucio/blob/e708f55772d74c782d36f79cfda521570190111a/lib/rucio/rse/rsemanager.py#L398)).

I loathe to put this modification at the protocol level hence the suggested modification, please let me know if you have better ideas.

Modification
------------
- [x] copy `verify_checksum` RSE attribute to RSE info
- [x] get `verify_checksum` from RSE info in `rsemgr.upload`
- [x] always declare uploaded file as valid if `ignore_checksum` is True.

**Some notes**
On line [405](https://github.com/rucio/rucio/blob/e708f55772d74c782d36f79cfda521570190111a/lib/rucio/rse/rsemanager.py#L405) of the rsemgr add
```python
valid = True if ignore_checksum is True else False
```
After discussion with Martin we decided that `verify_checksum` should conceptually be RSE info and to copy it analagously to the `lfn2pfn_algorithm` attribute/info.

Changed files:
lib/rucio/core/rse.py, lib/rucio/rse/rsemanager.py, lib/rucio/tests/test_rse.py

diff --git a/lib/rucio/core/rse.py b/lib/rucio/core/rse.py
index 1848b1f8d..ba82b30dc 100644
--- a/lib/rucio/core/rse.py
+++ b/lib/rucio/core/rse.py
@@ -14,6 +14,7 @@
 # - Thomas Beermann, <thomas.beermann@cern.ch>, 2014, 2017
 # - Wen Guan, <wen.guan@cern.ch>, 2015-2016
 # - Brian Bockelman, <bbockelm@cse.unl.edu>, 2018
+# - Frank Berghaus, <frank.berghaus@cern.ch>, 2018
 
 from re import match
 from StringIO import StringIO
@@ -737,6 +738,9 @@ def get_rse_protocols(rse, schemes=None, session=None):
     if lfn2pfn_algorithms:
         lfn2pfn_algorithm = lfn2pfn_algorithms[0]
 
+    # Copy verify_checksum from the attributes, later: assume True if not specified
+    verify_checksum = get_rse_attribute('verify_checksum', rse_id=_rse.id, session=session)
+
     read = True if _rse.availability & 4 else False
     write = True if _rse.availability & 2 else False
     delete = True if _rse.availability & 1 else False
@@ -753,6 +757,7 @@ def get_rse_protocols(rse, schemes=None, session=None):
             'rse_type': str(_rse.rse_type),
             'credentials': None,
             'volatile': _rse.volatile,
+            'verify_checksum': verify_checksum[0] if verify_checksum else True,
             'staging_area': _rse.staging_area}
 
     for op in utils.rse_supported_protocol_operations():
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
index d3e968865..bb4c05d64 100755
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -38,6 +38,7 @@ def get_rse_info(rse, session=None):
                     rse               ...     the name of the RSE as string
                     type              ...     the storage type odf the RSE e.g. DISK
                     volatile          ...     boolean indictaing if the RSE is volatile
+                    verify_checksum   ...     boolean indicating whether RSE supports requests for checksums
                     deteministic      ...     boolean indicating of the nameing of the files follows the defined determinism
                     domain            ...     indictaing the domain that should be assumed for transfers. Values are 'ALL', 'LAN', or 'WAN'
                     protocols         ...     all supported protocol in form of a list of dict objects with the followig structure
@@ -402,7 +403,7 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
                     if (valid is None) and ('filesize' in stats) and ('filesize' in lfn):
                         valid = stats['filesize'] == lfn['filesize']
                 except NotImplementedError:
-                    valid = False
+                    valid = True if rse_settings['verify_checksum'] is False else False
                 except Exception as e:
                     gs = False
                     ret['%s:%s' % (scope, name)] = e
diff --git a/lib/rucio/tests/test_rse.py b/lib/rucio/tests/test_rse.py
index 1ae9c52ca..019f1f419 100644
--- a/lib/rucio/tests/test_rse.py
+++ b/lib/rucio/tests/test_rse.py
@@ -31,7 +31,8 @@ from rucio.common.exception import (Duplicate, RSENotFound, RSEProtocolNotSuppor
                                     ResourceTemporaryUnavailable)
 from rucio.common.utils import generate_uuid
 from rucio.core.rse import (add_rse, get_rse_id, del_rse, list_rses, rse_exists, add_rse_attribute, list_rse_attributes,
-                            set_rse_transfer_limits, get_rse_transfer_limits, delete_rse_transfer_limits)
+                            set_rse_transfer_limits, get_rse_transfer_limits, delete_rse_transfer_limits,
+                            get_rse_protocols)
 from rucio.rse import rsemanager as mgr
 from rucio.tests.common import rse_name_generator
 from rucio.web.rest.rse import APP as rse_app
@@ -346,7 +347,7 @@ class TestRSEClient(object):
 
             self.client.delete_protocols(protocol_rse, 'Mock_Insuff_Params')
             self.client.delete_rse(protocol_rse)
-        except:  # explicity raise the correct Exception for MySQL
+        except Exception:  # explicity raise the correct Exception for MySQL
             raise InvalidObject
 
     @raises(Duplicate)
@@ -1140,12 +1141,12 @@ class TestRSEClient(object):
             for p in protocols:
                 self.client.add_protocol(protocol_rse, p)
                 self.client.update_protocols(protocol_rse, scheme='MOCK', hostname='localhost', port=17, data={'impl': None})
-        except:
+        except Exception:
             raise InvalidObject  # explicity raise the correct Exception for MySQL
         finally:
             try:
                 self.client.delete_protocols(protocol_rse, 'MOCK')
-            except:
+            except Exception:
                 pass  # for MySQL
             finally:
                 self.client.delete_rse(protocol_rse)
@@ -1253,3 +1254,25 @@ class TestRSEClient(object):
         for distance in self.client.get_distance(source=source, destination=destination):
             print(distance)
             assert_equal(distance['distance'], 0)
+
+    def test_get_rse_protocols_includes_verify_checksum(self):
+        """ RSE (CORE): Test validate_checksum in RSEs info"""
+        rse = rse_name_generator()
+        add_rse(rse)
+        add_rse_attribute(rse=rse, key='verify_checksum', value=False)
+        info = get_rse_protocols(rse)
+
+        assert_in('verify_checksum', info)
+        assert_equal(info['verify_checksum'], False)
+
+        del_rse(rse)
+
+        rse = rse_name_generator()
+        add_rse(rse)
+        add_rse_attribute(rse=rse, key='verify_checksum', value=True)
+        info = get_rse_protocols(rse)
+
+        assert_in('verify_checksum', info)
+        assert_equal(info['verify_checksum'], True)
+
+        del_rse(rse)