Repository: rucio/rucio

Issue:
Auditor does not work with Python 3
Motivation
----------

```
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/site-packages/rucio/daemons/auditor/__init__.py", line 236, in check
    results_dir)
  File "/usr/local/lib/python3.6/site-packages/rucio/daemons/auditor/__init__.py", line 56, in consistency
    rsedump, rsedate = srmdumps.download_rse_dump(rse, configuration, destdir=cache_dir)
  File "/usr/local/lib/python3.6/site-packages/rucio/daemons/auditor/srmdumps.py", line 271, in download_rse_dump
    hashlib.sha1(url).hexdigest()
TypeError: Unicode-objects must be encoded before hashing
```

```
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/site-packages/rucio/daemons/auditor/__init__.py", line 236, in check
    results_dir)
  File "/usr/local/lib/python3.6/site-packages/rucio/daemons/auditor/__init__.py", line 63, in consistency
    rrdump_prev = ReplicaFromHDFS.download(rse, rsedate - delta, cache_dir=cache_dir)
  File "/usr/local/lib/python3.6/site-packages/rucio/daemons/auditor/hdfs.py", line 80, in download
    full_dump.write(data_chunk)
TypeError: write() argument must be str, not bytes
```

Modification
------------

Properly handle bytes vs strings.


Discussion:

Resolving patch:
diff --git a/lib/rucio/common/dumper/__init__.py b/lib/rucio/common/dumper/__init__.py
--- a/lib/rucio/common/dumper/__init__.py
+++ b/lib/rucio/common/dumper/__init__.py
@@ -163,7 +163,7 @@ def smart_open(filename):
 
 
 @contextlib.contextmanager
-def temp_file(directory, final_name=None):
+def temp_file(directory, final_name=None, binary=False):
     '''
     Allows to create a temporal file to store partial results, when the
     file is complete it is renamed to `final_name`.
@@ -172,6 +172,7 @@ def temp_file(directory, final_name=None):
     - `final_name`: Path of the final file, relative to `directory`.
        If the `final_name` is omitted or None the renaming step is ommited,
        leaving the temporal file with the results.
+    - `binary`: whether to open the file in binary mode (default: False).
 
     Important: `directory` and `final_name` must be in the same filesystem as
     a hardlink is used to rename the temporal file.
@@ -186,7 +187,7 @@ def temp_file(directory, final_name=None):
     logger = logging.getLogger('dumper.__init__')
 
     fd, tpath = tempfile.mkstemp(dir=directory)
-    tmp = os.fdopen(fd, 'w')
+    tmp = os.fdopen(fd, 'wb' if binary else 'w')
 
     try:
         yield tmp, os.path.basename(tpath)
diff --git a/lib/rucio/daemons/auditor/hdfs.py b/lib/rucio/daemons/auditor/hdfs.py
--- a/lib/rucio/daemons/auditor/hdfs.py
+++ b/lib/rucio/daemons/auditor/hdfs.py
@@ -70,7 +70,7 @@ def download(cls, rse, date, cache_dir=DUMPS_CACHE_DIR, buffer_size=65536):
             _hdfs_get(cls.BASE_URL.format(date.strftime('%Y-%m-%d'), rse), tmp_dir)
             files = (os.path.join(tmp_dir, file_) for file_ in sorted(os.listdir(tmp_dir)))
 
-            with temp_file(cache_dir, filename) as (full_dump, _):
+            with temp_file(cache_dir, filename, binary=True) as (full_dump, _):
                 for chunk_file in files:
                     with open(chunk_file, 'rb') as partial_dump:
                         while True:
diff --git a/lib/rucio/daemons/auditor/srmdumps.py b/lib/rucio/daemons/auditor/srmdumps.py
--- a/lib/rucio/daemons/auditor/srmdumps.py
+++ b/lib/rucio/daemons/auditor/srmdumps.py
@@ -268,7 +268,7 @@ def download_rse_dump(rse, configuration, date='latest', destdir=DUMPS_CACHE_DIR
         'ddmendpoint',
         rse,
         date.strftime('%d-%m-%Y'),
-        hashlib.sha1(url).hexdigest()
+        hashlib.sha1(url.encode()).hexdigest()
     )
     filename = re.sub(r'\W', '-', filename)
     path = os.path.join(destdir, filename)


Test patch: