Repository: rucio/rucio

Issue:
Unicode support along all Core/API/REST API
Motivation
----------
When working on porting the REST framework tests to Flask #4033 I encountered problems when handing Python 2 `unicode` type text. While we want to remove Python 2 compatibility in 1.24, these layers of compatibility can be removed. Left to research is how these functions work with Python 3 unicode native `str` type strings. I have already fixed an issue in #3995 but I think there is more to find and fix.


Modification
------------
Research API functions and called functions for unicode support and remove Python 2 compatibility code (for the server only).


Discussion:
Hi @mlassnig!, I am Sayandeep and I am deeply interested in open source projects and also applied to ROOT TMVA project in gsoc this year.
Currently I am really attracted to Rucio especially after reading the documentation. This issue looks interesting. Can I have a try at this issue? 
Hi @sayang14 yes please go ahead, thanks 😄 
Related to #5435 
Since we got rid of `six` as a dependency and Python 2 compatibility, all unicode type references are gone. I still identified the following code lines that could be simplified:

https://github.com/rucio/rucio/blob/2a41a64899f41b356660fff80a7530765b5d59e0/lib/rucio/__init__.py#L20

https://github.com/rucio/rucio/blob/2a41a64899f41b356660fff80a7530765b5d59e0/lib/rucio/core/permission/cms.py#L25-L30

https://github.com/rucio/rucio/blob/1ac5d8a00d5894d27025528590d0a80bae4d1f69/lib/rucio/web/ui/flask/common/utils.py#L33-L35

CC @bari12 

Resolving patch:
diff --git a/lib/rucio/__init__.py b/lib/rucio/__init__.py
--- a/lib/rucio/__init__.py
+++ b/lib/rucio/__init__.py
@@ -14,9 +14,5 @@
 # limitations under the License.
 
 import gettext
-import sys
 
-if sys.version_info < (3,):
-    gettext.install('rucio', unicode=1)  # pylint: disable=unexpected-keyword-arg
-else:
-    gettext.install('rucio')
+gettext.install('rucio')
diff --git a/lib/rucio/common/config.py b/lib/rucio/common/config.py
--- a/lib/rucio/common/config.py
+++ b/lib/rucio/common/config.py
@@ -15,15 +15,12 @@
 
 """Provides functions to access the local configuration. The configuration locations are provided by get_config_dirs."""
 
-import os
-import json
-import sys
-
-from rucio.common.exception import ConfigNotFound, DatabaseException
-
 import configparser as ConfigParser
+import json
+import os
 
 from rucio.common import exception
+from rucio.common.exception import ConfigNotFound, DatabaseException
 
 
 def config_get(section, option, raise_exception=True, default=None, clean_cached=False, check_config_table=True,
@@ -433,10 +430,7 @@ class Config:
     get_config_dirs or the use of the RUCIO_CONFIG environment variable.
     """
     def __init__(self):
-        if sys.version_info < (3, 2):
-            self.parser = ConfigParser.SafeConfigParser()
-        else:
-            self.parser = ConfigParser.ConfigParser()
+        self.parser = ConfigParser.ConfigParser()
 
         if 'RUCIO_CONFIG' in os.environ:
             self.configfile = os.environ['RUCIO_CONFIG']
diff --git a/lib/rucio/web/ui/flask/common/utils.py b/lib/rucio/web/ui/flask/common/utils.py
--- a/lib/rucio/web/ui/flask/common/utils.py
+++ b/lib/rucio/web/ui/flask/common/utils.py
@@ -14,14 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import html
 import re
-import sys
 from json import dumps, load
 from os.path import dirname, join
 from time import time
+from urllib.parse import quote, unquote
 
 from flask import request, render_template, redirect, make_response
-from urllib.parse import quote, unquote
 
 from rucio.api import authentication as auth, identity
 from rucio.api.account import account_exists, get_account_info, list_account_attributes
@@ -30,26 +30,11 @@
 from rucio.core import identity as identity_core, vo as vo_core
 from rucio.db.sqla.constants import AccountType, IdentityType
 
-if sys.version_info > (3, 0):
-    long = int
-    unicode = str
-
-if sys.version_info > (3, 2):
-    import html
-
-    def html_escape(s, quote=True):
-        return html.escape(s, quote)
-else:
-    import cgi
-
-    def html_escape(s, quote=True):
-        return cgi.escape(s, quote)  # pylint: disable-msg=E1101
-
-
 EXTRA_MODULES = import_extras(['onelogin'])
 
 if EXTRA_MODULES['onelogin']:
     from onelogin.saml2.auth import OneLogin_Saml2_Auth  # pylint: disable=import-error
+
     SAML_SUPPORT = True
 else:
     SAML_SUPPORT = False
@@ -265,7 +250,7 @@ def finalize_auth(token, identity_type, cookie_dict_extra=None):
 
         cookie.extend([{'key': 'x-rucio-auth-token', 'value': quote(token)},
                        {'key': 'x-rucio-auth-type', 'value': quote(identity_type)},
-                       {'key': 'rucio-auth-token-created-at', 'value': str(long(time()))},
+                       {'key': 'rucio-auth-token-created-at', 'value': str(int(time()))},
                        {'key': 'rucio-available-accounts', 'value': quote(accounts)},
                        {'key': 'rucio-account-attr', 'value': quote(dumps(attribs))},
                        {'key': 'rucio-selected-account', 'value': quote(valid_token_dict['account'])},
@@ -334,7 +319,7 @@ def x509token_auth(data=None):
                 if account_exists(ui_account, vo):
                     valid_vos.append(vo)
             if len(valid_vos) == 0:
-                return render_template('problem.html', msg=('<br><br>Your certificate (%s) is not mapped to (possibly any) rucio account: %s at any VO.' % (html_escape(dn), html_escape(ui_account))))
+                return render_template('problem.html', msg=('<br><br>Your certificate (%s) is not mapped to (possibly any) rucio account: %s at any VO.' % (html.escape(dn), html.escape(ui_account))))
             elif len(valid_vos) == 1:
                 ui_vo = valid_vos[0]
             else:
@@ -346,14 +331,14 @@ def x509token_auth(data=None):
 
     if not ui_account:
         if MULTI_VO:
-            msg = "<br><br>Your certificate (%s) is not mapped to (possibly any) rucio account at VO: %s." % (html_escape(dn), html_escape(ui_vo))
+            msg = "<br><br>Your certificate (%s) is not mapped to (possibly any) rucio account at VO: %s." % (html.escape(dn), html.escape(ui_vo))
         else:
-            msg = "<br><br>Your certificate (%s) is not mapped to (possibly any) rucio account." % (html_escape(dn))
+            msg = "<br><br>Your certificate (%s) is not mapped to (possibly any) rucio account." % (html.escape(dn))
     else:
         if MULTI_VO:
-            msg = "<br><br>Your certificate (%s) is not mapped to (possibly any) rucio account: %s at VO: %s." % (html_escape(dn), html_escape(ui_account), html_escape(ui_vo))
+            msg = "<br><br>Your certificate (%s) is not mapped to (possibly any) rucio account: %s at VO: %s." % (html.escape(dn), html.escape(ui_account), html.escape(ui_vo))
         else:
-            msg = "<br><br>Your certificate (%s) is not mapped to (possibly any) rucio account: %s." % (html_escape(dn), html_escape(ui_account))
+            msg = "<br><br>Your certificate (%s) is not mapped to (possibly any) rucio account: %s." % (html.escape(dn), html.escape(ui_account))
 
     if ADDITIONAL_ERROR_MSG:
         msg += ADDITIONAL_ERROR_MSG
@@ -396,7 +381,7 @@ def userpass_auth():
                     valid_vos.append(vo)
 
             if len(valid_vos) == 0:
-                return render_template('problem.html', msg='Cannot find any Rucio account %s associated with identity %s at any VO.' % (html_escape(ui_account), html_escape(username)))
+                return render_template('problem.html', msg='Cannot find any Rucio account %s associated with identity %s at any VO.' % (html.escape(ui_account), html.escape(username)))
             elif len(valid_vos) == 1:
                 ui_vo = valid_vos[0]
             else:
@@ -408,15 +393,15 @@ def userpass_auth():
 
     if not ui_account:
         if MULTI_VO:
-            return render_template('problem.html', msg='Cannot get find any account associated with %s identity at VO %s.' % (html_escape(username), html_escape(ui_vo)))
+            return render_template('problem.html', msg='Cannot get find any account associated with %s identity at VO %s.' % (html.escape(username), html.escape(ui_vo)))
         else:
-            return render_template('problem.html', msg='Cannot get find any account associated with %s identity.' % (html_escape(username)))
+            return render_template('problem.html', msg='Cannot get find any account associated with %s identity.' % (html.escape(username)))
     token = get_token(auth.get_auth_token_user_pass, acc=ui_account, vo=ui_vo, idt=username, pwd=password)
     if not token:
         if MULTI_VO:
-            return render_template('problem.html', msg='Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s at VO %s.') % (html_escape(username), html_escape(ui_account), html_escape(ui_vo))
+            return render_template('problem.html', msg='Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s at VO %s.') % (html.escape(username), html.escape(ui_account), html.escape(ui_vo))
         else:
-            return render_template('problem.html', msg='Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s.') % (html_escape(username), html_escape(ui_account))
+            return render_template('problem.html', msg='Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s.') % (html.escape(username), html.escape(ui_account))
 
     return finalize_auth(token, 'userpass')
 
@@ -466,7 +451,7 @@ def saml_auth(method, data=None):
                     if account_exists(ui_account, vo):
                         valid_vos.append(vo)
                 if len(valid_vos) == 0:
-                    return render_template("problem.html", msg=('Cannot find any Rucio account %s associated with identity %s at any VO.' % (html_escape(ui_account), html_escape(saml_nameid))))
+                    return render_template("problem.html", msg=('Cannot find any Rucio account %s associated with identity %s at any VO.' % (html.escape(ui_account), html.escape(saml_nameid))))
                 elif len(valid_vos) == 1:
                     ui_vo = valid_vos[0]
                 else:
@@ -478,15 +463,15 @@ def saml_auth(method, data=None):
 
         if not ui_account:
             if MULTI_VO:
-                return render_template("problem.html", msg='Cannot get find any account associated with %s identity at VO %s.' % (html_escape(saml_nameid), html_escape(ui_vo)))
+                return render_template("problem.html", msg='Cannot get find any account associated with %s identity at VO %s.' % (html.escape(saml_nameid), html.escape(ui_vo)))
             else:
-                return render_template("problem.html", msg='Cannot get find any account associated with %s identity.' % (html_escape(saml_nameid)))
+                return render_template("problem.html", msg='Cannot get find any account associated with %s identity.' % (html.escape(saml_nameid)))
         token = get_token(auth.get_auth_token_saml, acc=ui_account, vo=ui_vo, idt=saml_nameid)
         if not token:
             if MULTI_VO:
-                return render_template("problem.html", msg=('Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s at VO %s.') % (html_escape(saml_nameid), html_escape(ui_account), html_escape(ui_vo)))
+                return render_template("problem.html", msg=('Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s at VO %s.') % (html.escape(saml_nameid), html.escape(ui_account), html.escape(ui_vo)))
             else:
-                return render_template("problem.html", msg=('Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s.') % (html_escape(saml_nameid), html_escape(ui_account)))
+                return render_template("problem.html", msg=('Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s.') % (html.escape(saml_nameid), html.escape(ui_account)))
         return finalize_auth(token, 'saml')
 
     # If method is POST, check the received SAML response and redirect to home if valid
@@ -515,7 +500,7 @@ def saml_auth(method, data=None):
                         if account_exists(ui_account, vo):
                             valid_vos.append(vo)
                     if len(valid_vos) == 0:
-                        return render_template("problem.html", msg=('Cannot find any Rucio account %s associated with identity %s at any VO.' % (html_escape(ui_account), html_escape(saml_nameid))))
+                        return render_template("problem.html", msg=('Cannot find any Rucio account %s associated with identity %s at any VO.' % (html.escape(ui_account), html.escape(saml_nameid))))
                     elif len(valid_vos) == 1:
                         ui_vo = valid_vos[0]
                     else:
@@ -527,15 +512,16 @@ def saml_auth(method, data=None):
 
             if not ui_account:
                 if MULTI_VO:
-                    return render_template("problem.html", msg='Cannot get find any account associated with %s identity at VO %s.' % (html_escape(saml_nameid), html_escape(ui_vo)))
+                    return render_template("problem.html", msg='Cannot get find any account associated with %s identity at VO %s.' % (html.escape(saml_nameid), html.escape(ui_vo)))
                 else:
-                    return render_template("problem.html", msg='Cannot get find any account associated with %s identity.' % (html_escape(saml_nameid)))
+                    return render_template("problem.html", msg='Cannot get find any account associated with %s identity.' % (html.escape(saml_nameid)))
             token = get_token(auth.get_auth_token_saml, acc=ui_account, vo=ui_vo, idt=saml_nameid)
             if not token:
                 if MULTI_VO:
-                    return render_template("problem.html", msg=('Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s at VO %s.') % (html_escape(saml_nameid), html_escape(ui_account), html_escape(ui_vo)))
+                    return render_template("problem.html",
+                                           msg=('Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s at VO %s.') % (html.escape(saml_nameid), html.escape(ui_account), html.escape(ui_vo)))
                 else:
-                    return render_template("problem.html", msg=('Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s.') % (html_escape(saml_nameid), html_escape(ui_account)))
+                    return render_template("problem.html", msg=('Cannot get auth token. It is possible that the presented identity %s is not mapped to any Rucio account %s.') % (html.escape(saml_nameid), html.escape(ui_account)))
             return finalize_auth(token, 'saml', cookie_extra)
 
         return render_template("problem.html", msg="Not authenticated")
@@ -561,7 +547,7 @@ def oidc_auth(account, issuer, ui_vo=None):
             if account_exists(account, vo):
                 valid_vos.append(vo)
         if len(valid_vos) == 0:
-            return render_template("problem.html", msg=('Cannot find any Rucio account %s at any VO.' % html_escape(account)))
+            return render_template("problem.html", msg=('Cannot find any Rucio account %s at any VO.' % html.escape(account)))
         elif len(valid_vos) == 1:
             ui_vo = valid_vos[0]
         else:


Test patch:
diff --git a/lib/rucio/tests/ruciopytest/plugin.py b/lib/rucio/tests/ruciopytest/plugin.py
--- a/lib/rucio/tests/ruciopytest/plugin.py
+++ b/lib/rucio/tests/ruciopytest/plugin.py
@@ -14,7 +14,6 @@
 # limitations under the License.
 
 import os
-import sys
 
 import pytest
 
@@ -25,55 +24,55 @@ def pytest_configure(config):
         'markers',
         'noparallel(reason): marks test being unable to run in parallel to other tests, i.e. changing global state',
     )
-    if sys.version_info >= (3, 6) and config.pluginmanager.hasplugin('xdist'):
+    if config.pluginmanager.hasplugin('xdist'):
         from rucio.tests.ruciopytest.rucioxdist import NoParallelXDist
 
         config.pluginmanager.register(NoParallelXDist(config))
 
 
-if sys.version_info >= (3, 6):
-    def pytest_addoption(parser, pluginmanager):
-        if pluginmanager.hasplugin('xdist'):
-            group = parser.getgroup('xdist', 'distributed and subprocess testing')
-            option_appended = False
-            for opt in group.options:
-                if '--dist' in opt.names():
-                    option_choices = opt._attrs['choices']
-                    option_choices.append('rucio')
-                    option_appended = True
-                    break
+def pytest_addoption(parser, pluginmanager):
+    if pluginmanager.hasplugin('xdist'):
+        group = parser.getgroup('xdist', 'distributed and subprocess testing')
+        option_appended = False
+        for opt in group.options:
+            if '--dist' in opt.names():
+                option_choices = opt._attrs['choices']
+                option_choices.append('rucio')
+                option_appended = True
+                break
 
-            if not option_appended:
-                raise pytest.UsageError('rucio pytest plugin must be loaded after xdist plugin')
+        if not option_appended:
+            raise pytest.UsageError('rucio pytest plugin must be loaded after xdist plugin')
+
+    # Initialization hook to add --artifacts option, can be used by integration or TPC tests to further check non-dev container states
+    parser.addoption(
+        "--export-artifacts-from",
+        action="append",
+        dest="artifacts",
+        default=[],
+        help="A csv string with test names that should persist their artifacts"
+    )
 
-        # Initialization hook to add --artifacts option, can be used by integration or TPC tests to further check non-dev container states
-        parser.addoption(
-            "--export-artifacts-from",
-            action="append",
-            dest="artifacts",
-            default=[],
-            help="A csv string with test names that should persist their artifacts"
-        )
 
-    def pytest_generate_tests(metafunc):
-        tests_with_artifacts = metafunc.config.getoption('artifacts')
-        if len(tests_with_artifacts) > 1:
-            raise pytest.UsageError('--export-artifacts-from must be used only once. It should contain a CSV string of test names that can manage artifacts.')
+def pytest_generate_tests(metafunc):
+    tests_with_artifacts = metafunc.config.getoption('artifacts')
+    if len(tests_with_artifacts) > 1:
+        raise pytest.UsageError('--export-artifacts-from must be used only once. It should contain a CSV string of test names that can manage artifacts.')
 
-        if len(tests_with_artifacts) == 1:
-            tests_with_artifacts = tests_with_artifacts[0].split(',')
-            test_function_name = metafunc.function.__name__
-            if "artifact" in metafunc.fixturenames:
-                if test_function_name in tests_with_artifacts:
-                    metafunc.parametrize(
-                        "artifact",
-                        ['/tmp/{function}.artifact'.format(function=test_function_name)]
-                    )
-                else:
-                    metafunc.parametrize("artifact", [None])
-        else:
-            if "artifact" in metafunc.fixturenames:
+    if len(tests_with_artifacts) == 1:
+        tests_with_artifacts = tests_with_artifacts[0].split(',')
+        test_function_name = metafunc.function.__name__
+        if "artifact" in metafunc.fixturenames:
+            if test_function_name in tests_with_artifacts:
+                metafunc.parametrize(
+                    "artifact",
+                    ['/tmp/{function}.artifact'.format(function=test_function_name)]
+                )
+            else:
                 metafunc.parametrize("artifact", [None])
+    else:
+        if "artifact" in metafunc.fixturenames:
+            metafunc.parametrize("artifact", [None])
 
 
 def pytest_cmdline_main(config):
diff --git a/lib/rucio/tests/test_auditor.py b/lib/rucio/tests/test_auditor.py
--- a/lib/rucio/tests/test_auditor.py
+++ b/lib/rucio/tests/test_auditor.py
@@ -17,20 +17,15 @@
 import collections
 import multiprocessing
 import os
-import sys
 import tempfile
 from datetime import datetime
 from datetime import timedelta
+from unittest import mock
 
 import pytest
 
 from rucio.daemons import auditor
 
-if sys.version_info >= (3, 3):
-    from unittest import mock
-else:
-    import mock
-
 
 def test_auditor_guess_replica_info():
     tests = {
diff --git a/lib/rucio/tests/test_auditor_hdfs.py b/lib/rucio/tests/test_auditor_hdfs.py
--- a/lib/rucio/tests/test_auditor_hdfs.py
+++ b/lib/rucio/tests/test_auditor_hdfs.py
@@ -15,18 +15,13 @@
 
 import os
 import string
-import sys
 from datetime import datetime
+from unittest import mock
 
 import pytest
 
 from rucio.daemons.auditor import hdfs
 
-if sys.version_info >= (3, 3):
-    from unittest import mock
-else:
-    import mock
-
 
 class FakeHDFSGet(object):
     def __init__(self, files=[]):
diff --git a/lib/rucio/tests/test_auditor_srmdumps.py b/lib/rucio/tests/test_auditor_srmdumps.py
--- a/lib/rucio/tests/test_auditor_srmdumps.py
+++ b/lib/rucio/tests/test_auditor_srmdumps.py
@@ -13,20 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
+from configparser import ConfigParser
 from datetime import datetime
+from unittest import mock
 
 import pytest
 
 from rucio.daemons.auditor import srmdumps
 
-from configparser import ConfigParser
-
-if sys.version_info >= (3, 3):
-    from unittest import mock
-else:
-    import mock
-
 
 def test_patterns_on_file_names():
     """ test_get_newest_matches_the_patterns_on_file_names """
diff --git a/lib/rucio/tests/test_daemons.py b/lib/rucio/tests/test_daemons.py
--- a/lib/rucio/tests/test_daemons.py
+++ b/lib/rucio/tests/test_daemons.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
+from unittest import mock
 
 import pytest
 
@@ -36,12 +36,6 @@
 from rucio.daemons.transmogrifier import transmogrifier
 from rucio.daemons.undertaker import undertaker
 
-if sys.version_info >= (3, 3):
-    from unittest import mock
-else:
-    import mock
-
-
 DAEMONS = [
     account,
     collection_replica,
diff --git a/lib/rucio/tests/test_dumper.py b/lib/rucio/tests/test_dumper.py
--- a/lib/rucio/tests/test_dumper.py
+++ b/lib/rucio/tests/test_dumper.py
@@ -13,13 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import bz2
 import json
 import os
-import sys
 import tempfile
 from datetime import datetime
+from io import StringIO
+from unittest import mock
 
-import bz2
 import pytest
 import requests
 
@@ -29,14 +30,6 @@
 from rucio.tests.common import mock_open
 from rucio.tests.mocks import gfal2
 
-from io import StringIO
-
-if sys.version_info >= (3, 3):
-    from unittest import mock
-else:
-    import mock
-
-
 DATE_SECONDS = "2015-03-10 14:00:35"
 DATE_TENTHS = "2015-03-10T14:00:35.5"
 DATE_MILLISECONDS = "2015-03-10T14:00:35.5"
diff --git a/lib/rucio/tests/test_dumper_consistency.py b/lib/rucio/tests/test_dumper_consistency.py
--- a/lib/rucio/tests/test_dumper_consistency.py
+++ b/lib/rucio/tests/test_dumper_consistency.py
@@ -15,8 +15,8 @@
 
 import json
 import os
-import sys
 from datetime import datetime
+from unittest import mock
 
 from rucio.common.dumper.consistency import Consistency
 from rucio.common.dumper.consistency import _try_to_advance
@@ -26,12 +26,6 @@
 from rucio.common.dumper.consistency import parse_and_filter_file
 from rucio.tests.common import make_temp_file
 
-if sys.version_info >= (3, 3):
-    from unittest import mock
-else:
-    import mock
-
-
 RSEPROTOCOL = {
     "hostname": "example.com",
     "scheme": "root",
diff --git a/lib/rucio/tests/test_dumper_data_model.py b/lib/rucio/tests/test_dumper_data_model.py
--- a/lib/rucio/tests/test_dumper_data_model.py
+++ b/lib/rucio/tests/test_dumper_data_model.py
@@ -15,8 +15,8 @@
 
 import glob
 import os
-import sys
 from datetime import datetime
+from unittest import mock
 
 import pytest
 import requests
@@ -24,11 +24,6 @@
 from rucio.common import dumper
 from rucio.common.dumper import data_models
 
-if sys.version_info >= (3, 3):
-    from unittest import mock
-else:
-    import mock
-
 
 def mocked_requests_head(*args, **kwargs):
     response = requests.Response()