Repository: rucio/rucio

Issue:
adler32 function should open file in "with" env and propagate the exception correctly
Motivation
----------
The function to get the adler32 checksum of a given file:
https://github.com/rucio/rucio/blob/master/lib/rucio/common/utils.py#L172
doesn't close the file correctly if an exception is raised and also doesn't propagate the exception to the caller.

Modification
------------



Discussion:

Resolving patch:
diff --git a/lib/rucio/common/utils.py b/lib/rucio/common/utils.py
--- a/lib/rucio/common/utils.py
+++ b/lib/rucio/common/utils.py
@@ -24,6 +24,8 @@
 # - Brian Bockelman <bbockelm@cse.unl.edu>, 2018
 # - Tobias Wegner <twegner@cern.ch>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
+#
+# PY3K COMPATIBLE
 
 from __future__ import print_function
 
@@ -39,13 +41,13 @@
 import requests
 import socket
 import subprocess
-import urllib
 import zlib
 
 from getpass import getuser
 from logging import getLogger, Formatter
 from logging.handlers import RotatingFileHandler
 from uuid import uuid4 as uuid
+from six import string_types
 
 try:
     # Python 2
@@ -180,11 +182,11 @@ def adler32(file):
     adler = 1
 
     try:
-        openFile = open(file, 'rb')
-        for line in openFile:
-            adler = zlib.adler32(line, adler)
-    except Exception:
-        raise Exception('FATAL - could not get checksum of file %s' % file)
+        with open(file, 'rb') as openFile:
+            for line in openFile:
+                adler = zlib.adler32(line, adler)
+    except Exception as e:
+        raise Exception('FATAL - could not get Adler32 checksum of file %s - %s' % (file, e))
 
     # backflip on 32bit
     if adler < 0:
@@ -203,9 +205,9 @@ def md5(file):
     hash_md5 = hashlib.md5()
     try:
         with open(file, "rb") as f:
-            map(hash_md5.update, iter(lambda: f.read(4096), b""))
-    except Exception:
-        raise Exception('FATAL - could not get MD5 checksum of file %s' % file)
+            list(map(hash_md5.update, iter(lambda: f.read(4096), b"")))
+    except Exception as e:
+        raise Exception('FATAL - could not get MD5 checksum of file %s - %s' % (file, e))
 
     return hash_md5.hexdigest()
 
@@ -260,12 +262,8 @@ def render_json_list(l):
 def datetime_parser(dct):
     """ datetime parser
     """
-    try:
-        varType = basestring
-    except NameError:
-        varType = str
     for k, v in list(dct.items()):
-        if isinstance(v, varType) and re.search(" UTC", v):
+        if isinstance(v, string_types) and re.search(" UTC", v):
             try:
                 dct[k] = datetime.datetime.strptime(v, DATE_FORMAT)
             except Exception:
@@ -722,7 +720,7 @@ def add_url_query(url, query):
     url_parts = list(urlparse.urlparse(url))
     mod_query = dict(urlparse.parse_qsl(url_parts[4]))
     mod_query.update(query)
-    url_parts[4] = urllib.urlencode(mod_query)
+    url_parts[4] = urlencode(mod_query)
     return urlparse.urlunparse(url_parts)
 
 


Test patch:
diff --git a/lib/rucio/tests/test_utils.py b/lib/rucio/tests/test_utils.py
--- a/lib/rucio/tests/test_utils.py
+++ b/lib/rucio/tests/test_utils.py
@@ -7,14 +7,17 @@
 
  Authors:
  - Frank Berghaus, <frank.berghaus@cern.ch>, 2018
+ - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2018
+
+ PY3K COMPATIBLE
 '''
 
 import unittest
 import tempfile
 
-from nose.tools import assert_equal, assert_is_instance, assert_is_not_none
+from nose.tools import assert_raises, assert_equal, assert_is_instance, assert_is_not_none
 from re import match
-from rucio.common.utils import md5
+from rucio.common.utils import md5, adler32
 
 
 class TestUtils(unittest.TestCase):
@@ -37,3 +40,18 @@ def test_utils_md5(self):
         assert_is_not_none(match('[a-fA-F0-9]{32}', ret), msg="String returned by utils.md5 is not a md5 hex digest")
         assert_equal(ret, '31d50dd6285b9ff9f8611d0762265d04',
                      msg="Hex digest returned by utils.md5 is the MD5 checksum")
+
+        with assert_raises(Exception) as e:
+            md5('no_file')
+        assert_equal('FATAL - could not get MD5 checksum of file no_file - [Errno 2] No such file or directory: \'no_file\'', e.exception.message)
+
+    def test_utils_adler32(self):
+        """(COMMON/UTILS): test calculating Adler32 of a file"""
+        ret = adler32(self.temp_file_1.name)
+        assert_is_instance(ret, str)
+        assert_is_not_none(match('[a-fA-F0-9]', ret))
+        assert_equal(ret, '198d03ff')
+
+        with assert_raises(Exception) as e:
+            adler32('no_file')
+        assert_equal('FATAL - could not get Adler32 checksum of file no_file - [Errno 2] No such file or directory: \'no_file\'', e.exception.message)