Repository: rucio/rucio

Issue:
clients: workaround url signature for rucio download until download-api is deployed
Motivation
----------

The old method for rucio download does not use the list_replicas PFNs, and instead constructs them manually. This workaround allows to use the list_replicas PFNs when necessary and will be removed again once we switch to download-api next month.


Discussion:

Resolving patch:
diff --git a/lib/rucio/common/dumper/__init__.py b/lib/rucio/common/dumper/__init__.py
--- a/lib/rucio/common/dumper/__init__.py
+++ b/lib/rucio/common/dumper/__init__.py
@@ -16,6 +16,7 @@
 # - Fernando Lopez <felopez@cern.ch>, 2015
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2016
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2017-2018
+# - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2018
 
 from rucio.common import config
 
@@ -315,7 +316,7 @@ def http_download(url, filename):
         http_download_to_file(url, f)
 
 
-def srm_download_to_file(url, file_):
+def gfal_download_to_file(url, file_):
     '''
     Download the file in `url` storing it in the `file_` file-like
     object.
@@ -340,9 +341,9 @@ def srm_download_to_file(url, file_):
         chunk = infile.read(CHUNK_SIZE)
 
 
-def srm_download(url, filename):
+def gfal_download(url, filename):
     '''
     Download the file in `url` storing it in the path given by `filename`.
     '''
     with open(filename, 'w') as f:
-        srm_download_to_file(url, f)
+        gfal_download_to_file(url, f)
diff --git a/lib/rucio/daemons/auditor/srmdumps.py b/lib/rucio/daemons/auditor/srmdumps.py
--- a/lib/rucio/daemons/auditor/srmdumps.py
+++ b/lib/rucio/daemons/auditor/srmdumps.py
@@ -15,10 +15,11 @@
 # Authors:
 # - Fernando Lopez <felopez@cern.ch>, 2015
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2017-2018
+# - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2018
 
 from rucio.common.config import __CONFIGFILES as __RUCIOCONFIGFILES
 from rucio.common.dumper import DUMPS_CACHE_DIR
-from rucio.common.dumper import http_download_to_file, srm_download_to_file, ddmendpoint_url, temp_file
+from rucio.common.dumper import http_download_to_file, gfal_download_to_file, ddmendpoint_url, temp_file
 
 import ConfigParser
 import HTMLParser
@@ -112,7 +113,7 @@ def get_newest(base_url, url_pattern, links):
     return max(times, key=operator.itemgetter(1))
 
 
-def srm_links(base_url):
+def gfal_links(base_url):
     '''
     Returns a list of the urls contained in `base_url`.
     '''
@@ -150,9 +151,21 @@ def http_links(base_url):
 
 
 protocol_funcs = {
+    'davs': {
+        'links': gfal_links,
+        'download': gfal_download_to_file,
+    },
+    'gsiftp': {
+        'links': gfal_links,
+        'download': gfal_download_to_file,
+    },
+    'root': {
+        'links': gfal_links,
+        'download': gfal_download_to_file,
+    },
     'srm': {
-        'links': srm_links,
-        'download': srm_download_to_file,
+        'links': gfal_links,
+        'download': gfal_download_to_file,
     },
     'http': {
         'links': http_links,
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -267,16 +267,19 @@ def download(rse_settings, files, dest_dir=None, force_scheme=None, ignore_check
                     if printstatements:
                         print('File downloaded. Will be validated')
 
-                    if not ignore_checksum:
+                    if ignore_checksum:
+                        if printstatements:
+                            print('Skipping checksum validation')
+                    else:
                         ruciochecksum = f['adler32'] if f['adler32'] else f['md5']
                         localchecksum = utils.adler32(tempfile) if f['adler32'] else utils.md5(tempfile)
                         if localchecksum == ruciochecksum:
                             if printstatements:
                                 print('File validated')
-                            os.rename(tempfile, finalfile)
                         else:
                             os.unlink(tempfile)
                             raise exception.FileConsistencyMismatch('Checksum mismatch : local %s vs recorded %s' % (str(localchecksum), str(ruciochecksum)))
+                    os.rename(tempfile, finalfile)
                 else:
                     protocol.get(pfn, '%s/%s' % (target_dir, f['name']), transfer_timeout=transfer_timeout)
                 ret['%s:%s' % (f['scope'], f['name'])] = True


Test patch:
diff --git a/lib/rucio/tests/test_auditor.py b/lib/rucio/tests/test_auditor.py
--- a/lib/rucio/tests/test_auditor.py
+++ b/lib/rucio/tests/test_auditor.py
@@ -7,6 +7,7 @@
 
  Authors:
  - Vincent Garonne,  <vincent.garonne@cern.ch> , 2017
+ - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2018
 '''
 
 from datetime import datetime
@@ -44,12 +45,12 @@ def mock_fn(*args, **kwargs):
 
     date = datetime.strptime('01-01-2015', '%d-%m-%Y')
 
-    fake_srm_download, fake_srm_download_calls = mock_fn_wrapper(('', date))
+    fake_gfal_download, fake_gfal_download_calls = mock_fn_wrapper(('', date))
     fake_rrd_download, fake_rrd_download_calls = mock_fn_wrapper('')
     fake_consistency_dump, fake_consistency_dump_calls = mock_fn_wrapper('')
     tmp_dir = tempfile.mkdtemp()
 
-    with stubbed(srmdumps.download_rse_dump, fake_srm_download):
+    with stubbed(srmdumps.download_rse_dump, fake_gfal_download):
         with stubbed(hdfs.ReplicaFromHDFS.download, fake_rrd_download):
             with stubbed(consistency.Consistency.dump, fake_consistency_dump):
                 auditor.consistency('RSENAME', timedelta(days=3), None, cache_dir=tmp_dir, results_dir=tmp_dir)
diff --git a/lib/rucio/tests/test_auditor_srmdumps.py b/lib/rucio/tests/test_auditor_srmdumps.py
--- a/lib/rucio/tests/test_auditor_srmdumps.py
+++ b/lib/rucio/tests/test_auditor_srmdumps.py
@@ -16,6 +16,7 @@
 # - Fernando Lopez <fernando.e.lopez@gmail.com>, 2015
 # - Martin Barisits <martin.barisits@cern.ch>, 2017
 # - Vincent Garonne <vgaronne@gmail.com>, 2018
+# - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2018
 
 from ConfigParser import ConfigParser
 from datetime import datetime
@@ -89,9 +90,13 @@ def test_returns_a_list_of_links():
     eq_(collector.links, ['x', 'y'])
 
 
-def test_identifies_http_and_srm():
-    """ test_protocol_identifies_http_and_srm """
+def test_identifies_known_protocols():
+    """ test_protocol_identifies_known_protocols """
+    eq_(srmdumps.protocol('davs://some/example'), 'davs')
+    eq_(srmdumps.protocol('gsiftp://some/example'), 'gsiftp')
     eq_(srmdumps.protocol('http://some/example'), 'http')
+    eq_(srmdumps.protocol('https://some/example'), 'https')
+    eq_(srmdumps.protocol('root://some/example'), 'root')
     eq_(srmdumps.protocol('srm://some/example'), 'srm')
 
 
diff --git a/lib/rucio/tests/test_dumper.py b/lib/rucio/tests/test_dumper.py
--- a/lib/rucio/tests/test_dumper.py
+++ b/lib/rucio/tests/test_dumper.py
@@ -8,6 +8,7 @@
 # Authors:
 # - Fernando Lopez, <felopez@cern.ch>, 2015
 # - Cedric Serfon, <cedric.serfon@cern.ch>, 2017
+# - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2018
 
 
 import __builtin__
@@ -200,28 +201,28 @@ def test_http_download_creates_file_with_content():
     eq_(stringio.read(), 'abc')
 
 
-def test_srm_download_to_file():
-    srm_file = StringIO()
+def test_gfal_download_to_file():
+    gfal_file = StringIO()
     local_file = StringIO()
-    srm_file.write('content')
-    srm_file.seek(0)
+    gfal_file.write('content')
+    gfal_file.seek(0)
 
-    with gfal2.mocked_gfal2(dumper, files={'srm://example.com/file': srm_file}):
-        dumper.srm_download_to_file('srm://example.com/file', local_file)
+    with gfal2.mocked_gfal2(dumper, files={'srm://example.com/file': gfal_file}):
+        dumper.gfal_download_to_file('srm://example.com/file', local_file)
 
     local_file.seek(0)
     eq_(local_file.read(), 'content')
 
 
-def test_srm_download_creates_file_with_content():
-    srm_file = StringIO()
+def test_gfal_download_creates_file_with_content():
+    gfal_file = StringIO()
     local_file = StringIO()
-    srm_file.write('content')
-    srm_file.seek(0)
+    gfal_file.write('content')
+    gfal_file.seek(0)
 
-    with gfal2.mocked_gfal2(dumper, files={'srm://example.com/file': srm_file}):
+    with gfal2.mocked_gfal2(dumper, files={'srm://example.com/file': gfal_file}):
         with mock_open(dumper, local_file):
-            dumper.srm_download('srm://example.com/file', 'filename')
+            dumper.gfal_download('srm://example.com/file', 'filename')
 
     local_file.seek(0)
     eq_(local_file.read(), 'content')