Repository: privacyidea/privacyidea

Issue:
LDAP Resolver crashes for NTLM authentication with non-ASCII password
As described [in the community forum](https://community.privacyidea.org/t/problems-with-ldap-resolver/745/2), the LDAP resolver fails to bind if NTLM authentication with a non-ASCII password is used. However, the "test connection" buttons work.
The following traceback appears when trying to list available users:
<pre>
[2018-07-18 18:14:27,764][2703][140243122291904][DEBUG][privacyidea.lib.user:652] Traceback (most recent call last):
  File "/home/fred/privacyidea/privacyidea/privacyidea/lib/user.py", line 637, in get_user_list
    ulist = y.getUserList(searchDict)
  File "/home/fred/privacyidea/privacyidea/privacyidea/lib/resolvers/LDAPIdResolver.py", line 591, in getUserList
    self._bind()
  File "/home/fred/privacyidea/privacyidea/privacyidea/lib/resolvers/LDAPIdResolver.py", line 435, in _bind
    if not self.l.bind():
  File "/home/fred/privacyidea/privacyidea/venv/lib/python2.7/site-packages/ldap3/core/connection.py", line 560, in bind
    response = self.do_ntlm_bind(controls)
  File "/home/fred/privacyidea/privacyidea/venv/lib/python2.7/site-packages/ldap3/core/connection.py", line 1299, in do_ntlm_bind
    request = bind_operation(self.version, 'SICILY_RESPONSE_NTLM', ntlm_client, result['server_creds'])
  File "/home/fred/privacyidea/privacyidea/venv/lib/python2.7/site-packages/ldap3/operation/bind.py", line 81, in bind_operation
    server_creds = name.create_authenticate_message()
  File "/home/fred/privacyidea/privacyidea/venv/lib/python2.7/site-packages/ldap3/utils/ntlm.py", line 379, in create_authenticate_message
    nt_challenge_response = self.compute_nt_response()
  File "/home/fred/privacyidea/privacyidea/venv/lib/python2.7/site-packages/ldap3/utils/ntlm.py", line 485, in compute_nt_response
    response_key_nt = self.ntowf_v2()
  File "/home/fred/privacyidea/privacyidea/venv/lib/python2.7/site-packages/ldap3/utils/ntlm.py", line 496, in ntowf_v2
    password_digest = hashlib.new('MD4', self._password.encode('utf-16-le')).digest()
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 2: ordinal not in range(128)
</pre>

I reproduced this with ldap3 2.5 and pyasn1 0.4.2.

The reason is that the password in the ``resolverconfig`` table is first encoded to UTF-8 and then encrypted. However, when reading it back from the config, it's not decoded back to a ``unicode`` object, but is passed around as a ``bytestring``, which crashes in the line above.
LDAP Resolver crashes for NTLM authentication with non-ASCII password
As described [in the community forum](https://community.privacyidea.org/t/problems-with-ldap-resolver/745/2), the LDAP resolver fails to bind if NTLM authentication with a non-ASCII password is used. However, the "test connection" buttons work.
The following traceback appears when trying to list available users:
<pre>
[2018-07-18 18:14:27,764][2703][140243122291904][DEBUG][privacyidea.lib.user:652] Traceback (most recent call last):
  File "/home/fred/privacyidea/privacyidea/privacyidea/lib/user.py", line 637, in get_user_list
    ulist = y.getUserList(searchDict)
  File "/home/fred/privacyidea/privacyidea/privacyidea/lib/resolvers/LDAPIdResolver.py", line 591, in getUserList
    self._bind()
  File "/home/fred/privacyidea/privacyidea/privacyidea/lib/resolvers/LDAPIdResolver.py", line 435, in _bind
    if not self.l.bind():
  File "/home/fred/privacyidea/privacyidea/venv/lib/python2.7/site-packages/ldap3/core/connection.py", line 560, in bind
    response = self.do_ntlm_bind(controls)
  File "/home/fred/privacyidea/privacyidea/venv/lib/python2.7/site-packages/ldap3/core/connection.py", line 1299, in do_ntlm_bind
    request = bind_operation(self.version, 'SICILY_RESPONSE_NTLM', ntlm_client, result['server_creds'])
  File "/home/fred/privacyidea/privacyidea/venv/lib/python2.7/site-packages/ldap3/operation/bind.py", line 81, in bind_operation
    server_creds = name.create_authenticate_message()
  File "/home/fred/privacyidea/privacyidea/venv/lib/python2.7/site-packages/ldap3/utils/ntlm.py", line 379, in create_authenticate_message
    nt_challenge_response = self.compute_nt_response()
  File "/home/fred/privacyidea/privacyidea/venv/lib/python2.7/site-packages/ldap3/utils/ntlm.py", line 485, in compute_nt_response
    response_key_nt = self.ntowf_v2()
  File "/home/fred/privacyidea/privacyidea/venv/lib/python2.7/site-packages/ldap3/utils/ntlm.py", line 496, in ntowf_v2
    password_digest = hashlib.new('MD4', self._password.encode('utf-16-le')).digest()
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 2: ordinal not in range(128)
</pre>

I reproduced this with ldap3 2.5 and pyasn1 0.4.2.

The reason is that the password in the ``resolverconfig`` table is first encoded to UTF-8 and then encrypted. However, when reading it back from the config, it's not decoded back to a ``unicode`` object, but is passed around as a ``bytestring``, which crashes in the line above.


Discussion:

Resolving patch:
diff --git a/privacyidea/lib/config.py b/privacyidea/lib/config.py
--- a/privacyidea/lib/config.py
+++ b/privacyidea/lib/config.py
@@ -123,7 +123,7 @@ def reload_from_db(self):
                     data = {}
                     for rconf in resolver.config_list:
                         if rconf.Type == "password":
-                            value = decryptPassword(rconf.Value)
+                            value = decryptPassword(rconf.Value, convert_unicode=True)
                         else:
                             value = rconf.Value
                         data[rconf.Key] = value
@@ -175,7 +175,7 @@ def get_config(self, key=None, default=None, role="admin",
         for ckey, cvalue in reduced_config.iteritems():
             if cvalue.get("Type") == "password":
                 # decrypt the password
-                r_config[ckey] = decryptPassword(cvalue.get("Value"))
+                r_config[ckey] = decryptPassword(cvalue.get("Value"), convert_unicode=True)
             else:
                 r_config[ckey] = cvalue.get("Value")
 
diff --git a/privacyidea/lib/crypto.py b/privacyidea/lib/crypto.py
--- a/privacyidea/lib/crypto.py
+++ b/privacyidea/lib/crypto.py
@@ -331,13 +331,36 @@ def encryptPin(cryptPin):
 
 
 @log_with(log, log_exit=False)
-def decryptPassword(cryptPass):
+def decryptPassword(cryptPass, convert_unicode=False):
+    """
+    Decrypt the encrypted password ``cryptPass`` and return it.
+    If an error occurs during decryption, return FAILED_TO_DECRYPT_PASSWORD.
+
+    :param cryptPass: bytestring
+    :param convert_unicode: If true, interpret the decrypted password as an UTF-8 string
+                            and convert it to unicode. If an error occurs here,
+                            the original bytestring is returned.
+    """
+    # NOTE: Why do we have the ``convert_unicode`` parameter?
+    # Up until now, this always returned bytestrings. However, this breaks
+    # LDAP and SQL resolvers, which expect this to return an unicode string
+    # (and this makes more sense, because ``encryptPassword`` also
+    # takes unicode strings!). But always returning unicode might break
+    # other call sites of ``decryptPassword``. So we add the
+    # keyword argument to avoid breaking compatibility.
+    from privacyidea.lib.utils import to_unicode
     hsm = _get_hsm()
     try:
         ret = hsm.decrypt_password(cryptPass)
     except Exception as exx:  # pragma: no cover
         log.warning(exx)
         ret = FAILED_TO_DECRYPT_PASSWORD
+    try:
+        if convert_unicode:
+            ret = to_unicode(ret)
+    except Exception as exx:  # pragma: no cover
+        log.warning(exx)
+        # just keep ``ret`` as a bytestring in that case
     return ret
 
 
diff --git a/privacyidea/lib/resolvers/SQLIdResolver.py b/privacyidea/lib/resolvers/SQLIdResolver.py
--- a/privacyidea/lib/resolvers/SQLIdResolver.py
+++ b/privacyidea/lib/resolvers/SQLIdResolver.py
@@ -406,7 +406,7 @@ def loadConfig(self, config):
         return self
 
     def _create_engine(self):
-        log.info("using the connect string {0!s}".format(censor_connect_string(self.connect_string)))
+        log.info(u"using the connect string {0!s}".format(censor_connect_string(self.connect_string)))
         try:
             log.debug("using pool_size={0!s}, pool_timeout={1!s}, pool_recycle={2!s}".format(
                 self.pool_size, self.pool_timeout, self.pool_recycle))
@@ -464,12 +464,12 @@ def _create_connect_string(param):
         password = ""
         conParams = ""
         if param.get("Port"):
-            port = ":{0!s}".format(param.get("Port"))
+            port = u":{0!s}".format(param.get("Port"))
         if param.get("Password"):
-            password = ":{0!s}".format(param.get("Password"))
+            password = u":{0!s}".format(param.get("Password"))
         if param.get("conParams"):
-            conParams = "?{0!s}".format(param.get("conParams"))
-        connect_string = "{0!s}://{1!s}{2!s}{3!s}{4!s}{5!s}/{6!s}{7!s}".format(param.get("Driver", ""),
+            conParams = u"?{0!s}".format(param.get("conParams"))
+        connect_string = u"{0!s}://{1!s}{2!s}{3!s}{4!s}{5!s}/{6!s}{7!s}".format(param.get("Driver", ""),
                                                    param.get("User", ""),
                                                    password,
                                                    "@" if (param.get("User")
@@ -505,7 +505,7 @@ def testconnection(cls, param):
         desc = None
 
         connect_string = cls._create_connect_string(param)
-        log.info("using the connect string {0!s}".format(censor_connect_string(connect_string)))
+        log.info(u"using the connect string {0!s}".format(censor_connect_string(connect_string)))
         engine = create_engine(connect_string)
         # create a configured "Session" class
         Session = scoped_session(sessionmaker(bind=engine))


Test patch:
diff --git a/tests/test_lib_crypto.py b/tests/test_lib_crypto.py
--- a/tests/test_lib_crypto.py
+++ b/tests/test_lib_crypto.py
@@ -148,6 +148,18 @@ def test_01_encrypt_decrypt_pass(self):
         pin = decryptPassword(r)
         self.assertTrue(pin == "passwörd", (r, pin))
 
+        r = encryptPassword(u"passwörd")
+        pin = decryptPassword(r, convert_unicode=True)
+        self.assertEqual(pin, u"passwörd")
+
+        r = encryptPassword(u"passwörd")
+        pin = decryptPassword(r, convert_unicode=False)
+        self.assertEqual(pin, "passwörd")
+
+        # error path returns the bytestring
+        r = encryptPassword(b"\x01\x02\x03\x04")
+        self.assertEqual(decryptPassword(r, convert_unicode=True), b"\x01\x02\x03\x04")
+
     def test_02_encrypt_decrypt_eas_base64(self):
         import os
         key = os.urandom(16)