Repository: privacyidea/privacyidea

Issue:
Use default SQLAlchemy engine options for Audit-db connection as well
The read/write to the Audit table uses a separate db engine in order to allow to use a different db (possibly on a different server).
The default setup uses a table in the privacyIDEA token database.
Currently parameter from `SQLALCHEMY_ENGINE_OPTIONS` are not propagated to the Audit db-engine.


Discussion:
This is also the case for the db upgrades using Flask-SQLALchemy.
I am not sure whether this could potentially break upgrades i.e. on Oracle (using `max_identifier_length`).

Resolving patch:
diff --git a/doc/conf.py b/doc/conf.py
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -76,7 +76,7 @@ def wrapper(decorator):
 
 # General information about the project.
 project = u'privacyIDEA'
-copyright = u'2014-2020, Cornelius Kölbel'
+copyright = u'2014-2022, Cornelius Kölbel'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -112,6 +112,7 @@ def wrapper(decorator):
 # A list of ignored prefixes for module index sorting.
 #modindex_common_prefix = []
 
+autoclass_content = 'both'
 
 # -- Options for HTML output ---------------------------------------------------
 
diff --git a/privacyidea/lib/auditmodules/base.py b/privacyidea/lib/auditmodules/base.py
--- a/privacyidea/lib/auditmodules/base.py
+++ b/privacyidea/lib/auditmodules/base.py
@@ -51,7 +51,6 @@
 import logging
 import traceback
 from privacyidea.lib.log import log_with
-import socket
 import datetime
 
 log = logging.getLogger(__name__)
@@ -84,7 +83,7 @@ def __init__(self, config=None, startdate=None):
         Create a new audit object.
 
         :param config: The web config is passed to the audit module, so that
-        the special module implementation can get its configuration.
+                       the special module implementation can get its configuration.
         :type config: dict
         :param startdate: The datetime of the beginning of the request
         :type startdate: datetime
@@ -112,10 +111,12 @@ def log_token_num(self, count):
     def read_keys(self, pub, priv):
         """
         Set the private and public key for the audit class. This is achieved by
-        passing the entries.
+        passing the values:
 
-        #priv = config.get("privacyideaAudit.key.private")
-        #pub = config.get("privacyideaAudit.key.public")
+        .. code-block:: python
+
+            priv = config.get("privacyideaAudit.key.private")
+            pub = config.get("privacyideaAudit.key.public")
 
         :param pub: Public key, used for verifying the signature
         :type pub: string with filename
@@ -175,7 +176,8 @@ def log(self, param):
 
     def add_to_log(self, param, add_with_comma=False):
         """
-        Add to existing log entry
+        Add to existing log entry.
+
         :param param:
         :param add_with_comma: If set to true, new values will be appended comma separated
         :return:
@@ -192,6 +194,7 @@ def add_to_log(self, param, add_with_comma=False):
     def add_policy(self, policyname):
         """
         This method adds a triggered policyname to the list of triggered policies.
+
         :param policyname: A string or a list of strings as policynames
         :return:
         """
@@ -215,7 +218,7 @@ def initialize_log(self, param):
         """
         This method initialized the log state.
         The fact, that the log state was initialized, also needs to be logged.
-        Therefor the same params are passed as i the log method.
+        Therefor the same parameters are passed as in the log method.
         """
         pass
 
@@ -231,12 +234,8 @@ def search(self, search_dict, page_size=15, page=1, sortorder="asc",
         """
         This function is used to search audit events.
 
-        param: Search parameters can be passed.
-
-        return: A pagination object
-
-
-        This function is deprecated.
+        :param: Search parameters can be passed.
+        :return: A pagination object
         """
         return Paginate()
 
diff --git a/privacyidea/lib/auditmodules/sqlaudit.py b/privacyidea/lib/auditmodules/sqlaudit.py
--- a/privacyidea/lib/auditmodules/sqlaudit.py
+++ b/privacyidea/lib/auditmodules/sqlaudit.py
@@ -94,24 +94,34 @@ class Audit(AuditBase):
     """
     This is the SQLAudit module, which writes the audit entries
     to an SQL database table.
-    It requires the configuration parameters in pi.cfg:
-    * PI_AUDIT_KEY_PUBLIC
-    * PI_AUDIT_KEY_PRIVATE
+
+    It requires the following configuration parameters in :ref:`cfgfile`:
+
+    * ``PI_AUDIT_KEY_PUBLIC``
+    * ``PI_AUDIT_KEY_PRIVATE``
 
     If you want to host the SQL Audit database in another DB than the
     token DB, you can use:
-    * PI_AUDIT_SQL_URI
 
-    It also takes the optional parameters:
-    * PI_AUDIT_POOL_SIZE
-    * PI_AUDIT_POOL_RECYCLE
-    * PI_AUDIT_SQL_TRUNCATE
-    * PI_AUDIT_NO_SIGN
+    * ``PI_AUDIT_SQL_URI`` and
+    * ``PI_AUDIT_SQL_OPTIONS``
+
+    With ``PI_AUDIT_SQL_OPTIONS = {}`` You can pass options to the DB engine
+    creation. If ``PI_AUDIT_SQL_OPTIONS`` is not set,
+    ``SQLALCHEMY_ENGINE_OPTIONS`` will be used.
 
-    You can use PI_AUDIT_NO_SIGN = True to avoid signing of the audit log.
+    This module also takes the following optional parameters:
 
-    If PI_CHECK_OLD_SIGNATURES = True old style signatures (text-book RSA) will
-    be checked as well, otherwise they will be marked as 'FAIL'.
+    * ``PI_AUDIT_POOL_SIZE``
+    * ``PI_AUDIT_POOL_RECYCLE``
+    * ``PI_AUDIT_SQL_TRUNCATE``
+    * ``PI_AUDIT_NO_SIGN``
+    * ``PI_CHECK_OLD_SIGNATURES``
+
+    You can use ``PI_AUDIT_NO_SIGN = True`` to avoid signing of the audit log.
+
+    If ``PI_CHECK_OLD_SIGNATURES = True`` old style signatures (text-book RSA) will
+    be checked as well, otherwise they will be marked as ``FAIL``.
     """
 
     is_readable = True
@@ -154,16 +164,22 @@ def _create_engine(self):
         connect_string = self.config.get("PI_AUDIT_SQL_URI", self.config.get(
             "SQLALCHEMY_DATABASE_URI"))
         log.debug("using the connect string {0!s}".format(censor_connect_string(connect_string)))
+        # if no specific audit engine options are given, use the default from
+        # SQLALCHEMY_ENGINE_OPTIONS or none
+        sqa_options = self.config.get("PI_AUDIT_SQL_OPTIONS",
+                                      self.config.get('SQLALCHEMY_ENGINE_OPTIONS', {}))
+        log.debug("Using Audit SQLAlchemy engine options: {0!s}".format(sqa_options))
         try:
             pool_size = self.config.get("PI_AUDIT_POOL_SIZE", 20)
             engine = create_engine(
                 connect_string,
                 pool_size=pool_size,
-                pool_recycle=self.config.get("PI_AUDIT_POOL_RECYCLE", 600))
+                pool_recycle=self.config.get("PI_AUDIT_POOL_RECYCLE", 600),
+                **sqa_options)
             log.debug("Using SQL pool size of {}".format(pool_size))
         except TypeError:
             # SQLite does not support pool_size
-            engine = create_engine(connect_string)
+            engine = create_engine(connect_string, **sqa_options)
             log.debug("Using no SQL pool_size.")
         return engine
 
@@ -410,6 +426,7 @@ def _get_logentry_attribute(key):
     def csv_generator(self, param=None, user=None, timelimit=None):
         """
         Returns the audit log as csv file.
+
         :param timelimit: Limit the number of dumped entries by time
         :type timelimit: datetime.timedelta
         :param param: The request parameters


Test patch:
diff --git a/tests/test_lib_audit.py b/tests/test_lib_audit.py
--- a/tests/test_lib_audit.py
+++ b/tests/test_lib_audit.py
@@ -8,6 +8,7 @@
 import os
 import types
 
+import sqlalchemy.engine
 from mock import mock
 
 from privacyidea.config import TestingConfig
@@ -304,6 +305,24 @@ def test_09_check_external_audit_db(self):
         audit_log = audit.search({"realm": "realm1"})
         self.assertEqual(audit_log.total, 1)
         self.assertEqual(audit_log.auditdata[0].get("sig_check"), "FAIL")
+
+        # check that SQLALCHEMY_ENGINE_OPTIONS get passed to create_engin()
+        with mock.patch('privacyidea.lib.auditmodules.sqlaudit.create_engine') as engine_mock:
+            cfg = self.app.config.copy()
+            cfg.update({'SQLALCHEMY_ENGINE_OPTIONS': {'foo': 'bar'}})
+            _audit = getAudit(cfg)
+            engine_mock.assert_called_once_with(AUDIT_DB, pool_size=20,
+                                                pool_recycle=600, foo='bar')
+
+        # check that PI_AUDIT_SQL_OPTIONS overwrite SQLALCHEMY_ENGINE_OPTIONS
+        with mock.patch('privacyidea.lib.auditmodules.sqlaudit.create_engine') as engine_mock:
+            cfg = self.app.config.copy()
+            cfg.update({'SQLALCHEMY_ENGINE_OPTIONS': {'foo': 'bar', 'temp': 100}})
+            cfg.update({'PI_AUDIT_SQL_OPTIONS': {'foo': 'baz'}})
+            _audit = getAudit(cfg)
+            engine_mock.assert_called_once_with(AUDIT_DB, pool_size=20,
+                                                pool_recycle=600, foo='baz')
+
         # TODO: add new audit entry and check for new style signature
         # remove the audit SQL URI from app config
         self.app.config.pop("PI_AUDIT_SQL_URI", None)