Repository: rucio/rucio

Issue:
Rucio Upload Should Respect verify_checksum Flag
Motivation
----------
When uploading to `CERN-EXTENSION_GRIDDISK` the Rucio client fails because it attempts to verify the checksum. At the moment the `verify_checksum` flag is set for the RSE and not ask the storage for the checksum.

The suggested modification will then also skip the verification of the file size on storage. The stat call checking the file size should be OK, and in this scenario is even more important to be verified (in [rsemgr](https://github.com/rucio/rucio/blob/e708f55772d74c782d36f79cfda521570190111a/lib/rucio/rse/rsemanager.py#L398)).

I loathe to put this modification at the protocol level hence the suggested modification, please let me know if you have better ideas.

Modification
------------
- [x] copy `verify_checksum` RSE attribute to RSE info
- [x] get `verify_checksum` from RSE info in `rsemgr.upload`
- [x] always declare uploaded file as valid if `ignore_checksum` is True.

**Some notes**
On line [405](https://github.com/rucio/rucio/blob/e708f55772d74c782d36f79cfda521570190111a/lib/rucio/rse/rsemanager.py#L405) of the rsemgr add
```python
valid = True if ignore_checksum is True else False
```
After discussion with Martin we decided that `verify_checksum` should conceptually be RSE info and to copy it analagously to the `lfn2pfn_algorithm` attribute/info.


Discussion:
No comment? Alright I'll try out what I am suggesting ;-)

The other option I was able to think of is to create a separate function in the protocols that get the checksum.
Hi Frank, sorry for the late reply. 👍 for the approach you are describing. Don't think we need the separate function in protocols.
Thanks Martin, I'll get started with it :-)

Resolving patch:
diff --git a/lib/rucio/core/rse.py b/lib/rucio/core/rse.py
--- a/lib/rucio/core/rse.py
+++ b/lib/rucio/core/rse.py
@@ -14,6 +14,7 @@
 # - Thomas Beermann, <thomas.beermann@cern.ch>, 2014, 2017
 # - Wen Guan, <wen.guan@cern.ch>, 2015-2016
 # - Brian Bockelman, <bbockelm@cse.unl.edu>, 2018
+# - Frank Berghaus, <frank.berghaus@cern.ch>, 2018
 
 from re import match
 from StringIO import StringIO
@@ -737,6 +738,9 @@ def get_rse_protocols(rse, schemes=None, session=None):
     if lfn2pfn_algorithms:
         lfn2pfn_algorithm = lfn2pfn_algorithms[0]
 
+    # Copy verify_checksum from the attributes, later: assume True if not specified
+    verify_checksum = get_rse_attribute('verify_checksum', rse_id=_rse.id, session=session)
+
     read = True if _rse.availability & 4 else False
     write = True if _rse.availability & 2 else False
     delete = True if _rse.availability & 1 else False
@@ -753,6 +757,7 @@ def get_rse_protocols(rse, schemes=None, session=None):
             'rse_type': str(_rse.rse_type),
             'credentials': None,
             'volatile': _rse.volatile,
+            'verify_checksum': verify_checksum[0] if verify_checksum else True,
             'staging_area': _rse.staging_area}
 
     for op in utils.rse_supported_protocol_operations():
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -38,6 +38,7 @@ def get_rse_info(rse, session=None):
                     rse               ...     the name of the RSE as string
                     type              ...     the storage type odf the RSE e.g. DISK
                     volatile          ...     boolean indictaing if the RSE is volatile
+                    verify_checksum   ...     boolean indicating whether RSE supports requests for checksums
                     deteministic      ...     boolean indicating of the nameing of the files follows the defined determinism
                     domain            ...     indictaing the domain that should be assumed for transfers. Values are 'ALL', 'LAN', or 'WAN'
                     protocols         ...     all supported protocol in form of a list of dict objects with the followig structure
@@ -402,7 +403,7 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
                     if (valid is None) and ('filesize' in stats) and ('filesize' in lfn):
                         valid = stats['filesize'] == lfn['filesize']
                 except NotImplementedError:
-                    valid = False
+                    valid = True if rse_settings['verify_checksum'] is False else False
                 except Exception as e:
                     gs = False
                     ret['%s:%s' % (scope, name)] = e


Test patch:
diff --git a/lib/rucio/tests/test_rse.py b/lib/rucio/tests/test_rse.py
--- a/lib/rucio/tests/test_rse.py
+++ b/lib/rucio/tests/test_rse.py
@@ -31,7 +31,8 @@
                                     ResourceTemporaryUnavailable)
 from rucio.common.utils import generate_uuid
 from rucio.core.rse import (add_rse, get_rse_id, del_rse, list_rses, rse_exists, add_rse_attribute, list_rse_attributes,
-                            set_rse_transfer_limits, get_rse_transfer_limits, delete_rse_transfer_limits)
+                            set_rse_transfer_limits, get_rse_transfer_limits, delete_rse_transfer_limits,
+                            get_rse_protocols)
 from rucio.rse import rsemanager as mgr
 from rucio.tests.common import rse_name_generator
 from rucio.web.rest.rse import APP as rse_app
@@ -346,7 +347,7 @@ def test_add_protocol_missing_values(self):
 
             self.client.delete_protocols(protocol_rse, 'Mock_Insuff_Params')
             self.client.delete_rse(protocol_rse)
-        except:  # explicity raise the correct Exception for MySQL
+        except Exception:  # explicity raise the correct Exception for MySQL
             raise InvalidObject
 
     @raises(Duplicate)
@@ -1140,12 +1141,12 @@ def test_update_protocols_invalid_value(self):
             for p in protocols:
                 self.client.add_protocol(protocol_rse, p)
                 self.client.update_protocols(protocol_rse, scheme='MOCK', hostname='localhost', port=17, data={'impl': None})
-        except:
+        except Exception:
             raise InvalidObject  # explicity raise the correct Exception for MySQL
         finally:
             try:
                 self.client.delete_protocols(protocol_rse, 'MOCK')
-            except:
+            except Exception:
                 pass  # for MySQL
             finally:
                 self.client.delete_rse(protocol_rse)
@@ -1253,3 +1254,25 @@ def test_add_distance(self):
         for distance in self.client.get_distance(source=source, destination=destination):
             print(distance)
             assert_equal(distance['distance'], 0)
+
+    def test_get_rse_protocols_includes_verify_checksum(self):
+        """ RSE (CORE): Test validate_checksum in RSEs info"""
+        rse = rse_name_generator()
+        add_rse(rse)
+        add_rse_attribute(rse=rse, key='verify_checksum', value=False)
+        info = get_rse_protocols(rse)
+
+        assert_in('verify_checksum', info)
+        assert_equal(info['verify_checksum'], False)
+
+        del_rse(rse)
+
+        rse = rse_name_generator()
+        add_rse(rse)
+        add_rse_attribute(rse=rse, key='verify_checksum', value=True)
+        info = get_rse_protocols(rse)
+
+        assert_in('verify_checksum', info)
+        assert_equal(info['verify_checksum'], True)
+
+        del_rse(rse)