Repository: privacyidea/privacyidea

Issue:
UnicdeDecodeError on Ubuntu 16.04 in audit log
A 3.0dev8 has an LDAP resolver talking to 2012.
A user has a german umlaut.

In this method we get a UnicodeDecodeError:
https://github.com/privacyidea/privacyidea/blob/master/privacyidea/lib/auditmodules/sqlaudit.py#L310

indeed it is due to ``le.user``.



Discussion:
THe same happens with the packages on Ubuntu 1804. and python 2.7
~~~~
[2019-04-06 14:50:26,809][4489][140191420528384][ERROR][privacyidea.lib.auditmodules.sqlaudit:253] exception UnicodeDecodeError('ascii', 'k\xf6lbel', 1, 2, 'ordinal not in range(128)')
[2019-04-06 14:50:26,810][4489][140191420528384][ERROR][privacyidea.lib.auditmodules.sqlaudit:254] DATA: {'info': '', 'administrator': u'super', 'realm': u'testfoo', 'policies': '', 'success': True, 'privacyidea_server': '10.0.2.232', 'client_user_agent': 'chrome', 'token_type': u'totp', 'client': '10.1.0.3', 'user': u'k\xf6lbel', 'resolver': u'testfoo', 'action_detail': '', 'action': 'POST /token/init', 'serial': u'TOTP0000FFDB'}
[2019-04-06 14:50:44,383][4489][140191403742976][INFO][privacyidea.lib.user:230] user u'k\xf6lbel' found in resolver u'testfoo'
[2019-04-06 14:50:44,384][4489][140191403742976][INFO][privacyidea.lib.user:231] userid resolved to '30d0b95e-2a32-4a28-9ac9-3e9e0eed828c' 
[2019-04-06 14:50:44,584][4489][140191403742976][ERROR][privacyidea.lib.auditmodules.sqlaudit:253] exception UnicodeDecodeError('ascii', 'k\xf6lbel', 1, 2, 'ordinal not in range(128)')
[2019-04-06 14:50:44,585][4489][140191403742976][ERROR][privacyidea.lib.auditmodules.sqlaudit:254] DATA: {'info': 'wrong otp value', 'realm': u'testfoo', 'tokentype': u'totp', 'success': False, 'privacyidea_server': '10.0.2.232', 'client_user_agent': 'chrome', 'client': '10.1.0.3', 'user': u'k\xf6lbel', 'resolver': u'testfoo', 'action_detail': '', 'action': 'GET /validate/check', 'serial': u'TOTP0000FFDB', 'policies': ''}
~~~~

This is independent of the charset of the database:

~~~~
mysql>  SELECT @@character_set_database, @@collation_database;
+--------------------------+----------------------+
| @@character_set_database | @@collation_database |
+--------------------------+----------------------+
| utf8mb4                  | utf8mb4_unicode_ci   |
+--------------------------+----------------------+
~~~~
Another problem is, that one error in the method ``_log_to_string`` will make the view of the complete auditlog fail.
So maybe we should add a try-except anyways.
The user is returned as 'str' here:
https://github.com/privacyidea/privacyidea/blob/master/privacyidea/lib/auditmodules/sqlaudit.py#L412

This then fails with _log_to_string, that tries to create a unicode.

There is no problem with sqlite, but the problem arises with mysql
Interesting. We had a similar error in 2.23.5.
I reproduced this on a MySQL setup. In my setup the issue can be fixed by adding a charset to the connect string:
```
SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://pi:...@localhost/pi?charset=utf8mb4'
```
But I guess we should adjust the code accordingly anyway.
One way to solve this is to set the `db.Column` to `db.Unicode` instead of `db.String` in the [`Audit()` class](https://github.com/privacyidea/privacyidea/blob/e871f5455ad6a9126f0c43f3db18da0c7b9a2e9d/privacyidea/models.py#L2508)
I added the charset as @fredreichbier suggested in the ubuntu build respoitory and it works out well - also for already existing databases.
``db.Unicode`` seems only to work, if the db is newly created. Anyways, please create a PR for it.

Resolving patch:
diff --git a/privacyidea/lib/auditmodules/sqlaudit.py b/privacyidea/lib/auditmodules/sqlaudit.py
--- a/privacyidea/lib/auditmodules/sqlaudit.py
+++ b/privacyidea/lib/auditmodules/sqlaudit.py
@@ -250,6 +250,8 @@ def finalize_log(self):
                 self.session.merge(le)
                 self.session.commit()
         except Exception as exx:  # pragma: no cover
+            # in case of a Unicode Error in _log_to_string() we won't have
+            # a signature, but the log entry is available
             log.error("exception {0!r}".format(exx))
             log.error("DATA: {0!s}".format(self.audit_data))
             log.debug("{0!s}".format(traceback.format_exc()))
@@ -266,10 +268,10 @@ def _check_missing(self, audit_id):
         the given id.
         
         TODO: We can not check at the moment if the first or the last entries
-        were deleted. If we want to do this, we need to store some signed
-        meta information
-        1. Which one was the first entry. (use initialize_log)
-        2. Which one was the last entry.
+              were deleted. If we want to do this, we need to store some signed
+              meta information:
+              1. Which one was the first entry. (use initialize_log)
+              2. Which one was the last entry.
         """
         res = False
         try:
@@ -405,14 +407,22 @@ def search(self, search_dict, page_size=15, page=1, sortorder="asc",
         auditIter = self.search_query(search_dict, page_size=page_size,
                                       page=page, sortorder=sortorder,
                                       timelimit=timelimit)
-        try:
-            le = next(auditIter)
-            while le:
+        while True:
+            try:
+                le = next(auditIter)
                 # Fill the list
                 paging_object.auditdata.append(self.audit_entry_to_dict(le))
-                le = next(auditIter)
-        except StopIteration as _e:
-            log.debug("Interation stopped.")
+            except StopIteration as _e:
+                log.debug("Interation stopped.")
+                break
+            except UnicodeDecodeError as _e:
+                # Unfortunately if one of the audit entries fails, the whole
+                # iteration stops and we return an empty paging_object.
+                # TODO: Check if we can return the other entries in the auditIter
+                #  or some meaningful error for the user.
+                log.warning('Could not read audit log entry! '
+                            'Possible database encoding mismatch.')
+                log.debug("{0!s}".format(traceback.format_exc()))
 
         return paging_object
         
@@ -469,9 +479,16 @@ def clear(self):
     def audit_entry_to_dict(self, audit_entry):
         sig = None
         if self.sign_data:
-            sig = self.sign_object.verify(self._log_to_string(audit_entry),
-                                          audit_entry.signature,
-                                          self.verify_old_sig)
+            try:
+                sig = self.sign_object.verify(self._log_to_string(audit_entry),
+                                              audit_entry.signature,
+                                              self.verify_old_sig)
+            except UnicodeDecodeError as _e:
+                # TODO: Unless we trace and eliminate the broken unicode in the
+                #  audit_entry, we will get issues when packing the response.
+                log.warning('Could not verify log entry! We get invalid values '
+                            'from the database, please check the encoding.')
+                log.debug('{0!s}'.format(traceback.format_exc()))
 
         is_not_missing = self._check_missing(int(audit_entry.id))
         # is_not_missing = True
diff --git a/privacyidea/models.py b/privacyidea/models.py
--- a/privacyidea/models.py
+++ b/privacyidea/models.py
@@ -2513,26 +2513,26 @@ class Audit(MethodsMixin, db.Model):
     __table_args__ = {'mysql_row_format': 'DYNAMIC'}
     id = db.Column(db.Integer, Sequence("audit_seq"), primary_key=True)
     date = db.Column(db.DateTime)
-    signature = db.Column(db.String(audit_column_length.get("signature")))
-    action = db.Column(db.String(audit_column_length.get("action")))
+    signature = db.Column(db.Unicode(audit_column_length.get("signature")))
+    action = db.Column(db.Unicode(audit_column_length.get("action")))
     success = db.Column(db.Integer)
-    serial = db.Column(db.String(audit_column_length.get("serial")))
-    token_type = db.Column(db.String(audit_column_length.get("token_type")))
-    user = db.Column(db.String(audit_column_length.get("user")), index=True)
-    realm = db.Column(db.String(audit_column_length.get("realm")))
-    resolver = db.Column(db.String(audit_column_length.get("resolver")))
+    serial = db.Column(db.Unicode(audit_column_length.get("serial")))
+    token_type = db.Column(db.Unicode(audit_column_length.get("token_type")))
+    user = db.Column(db.Unicode(audit_column_length.get("user")), index=True)
+    realm = db.Column(db.Unicode(audit_column_length.get("realm")))
+    resolver = db.Column(db.Unicode(audit_column_length.get("resolver")))
     administrator = db.Column(
-        db.String(audit_column_length.get("administrator")))
+        db.Unicode(audit_column_length.get("administrator")))
     action_detail = db.Column(
-        db.String(audit_column_length.get("action_detail")))
-    info = db.Column(db.String(audit_column_length.get("info")))
+        db.Unicode(audit_column_length.get("action_detail")))
+    info = db.Column(db.Unicode(audit_column_length.get("info")))
     privacyidea_server = db.Column(
-        db.String(audit_column_length.get("privacyidea_server")))
-    client = db.Column(db.String(audit_column_length.get("client")))
-    loglevel = db.Column(db.String(audit_column_length.get("loglevel")))
-    clearance_level = db.Column(db.String(audit_column_length.get(
+        db.Unicode(audit_column_length.get("privacyidea_server")))
+    client = db.Column(db.Unicode(audit_column_length.get("client")))
+    loglevel = db.Column(db.Unicode(audit_column_length.get("loglevel")))
+    clearance_level = db.Column(db.Unicode(audit_column_length.get(
         "clearance_level")))
-    policies = db.Column(db.String(audit_column_length.get("policies")))
+    policies = db.Column(db.Unicode(audit_column_length.get("policies")))
 
     def __init__(self,
                  action="",


Test patch: