Repository: privacyidea/privacyidea

Issue:
Python suggests using dictConfig for logging configuration instead of files
According to the [Python docs](https://docs.python.org/3/library/logging.config.html#configuration-file-format), the `fileConfig()` API is kind of deprecated and won't be updated in the future.
This already impacts logging configuration files with the [`RotatingFileHandler()`](https://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler) which ignores the `maxBytes` and `backupCount` parameter.



Discussion:
Does this mean we would have to read the config file ourselves, parse it into a dict and then do ``dictConfig``?
Yes, this is needed for setting up logging to logstash/splunk (#1580).
With a logging configuration in YAML this will take two lines of code.
TODO:
- [ ] update docs


Resolving patch:
diff --git a/privacyidea/app.py b/privacyidea/app.py
--- a/privacyidea/app.py
+++ b/privacyidea/app.py
@@ -24,9 +24,11 @@
 import logging
 import logging.config
 import sys
+import yaml
 from flask import Flask, request, Response
+from flask_babel import Babel
+from flask_migrate import Migrate
 from flaskext.versioned import Versioned
-from privacyidea.lib import queue
 
 # we need this import to add the before/after request function to the blueprints
 import privacyidea.api.before_after
@@ -58,39 +60,12 @@
 from privacyidea.api.clienttype import client_blueprint
 from privacyidea.api.subscriptions import subscriptions_blueprint
 from privacyidea.api.monitoring import monitoring_blueprint
+from privacyidea.lib import queue
 from privacyidea.lib.log import DEFAULT_LOGGING_CONFIG
 from privacyidea.config import config
 from privacyidea.models import db
-from flask_migrate import Migrate
-from flask_babel import Babel
-
 
 ENV_KEY = "PRIVACYIDEA_CONFIGFILE"
-MY_LOG_FORMAT = "[%(asctime)s][%(process)d][%(thread)d][%(levelname)s][%(" \
-                "name)s:%(lineno)d] %(message)s"
-
-PI_LOGGING_CONFIG = {
-    "version": 1,
-    "formatters": {"detail": {"class":
-                                  "privacyidea.lib.log.SecureFormatter",
-                              "format": "[%(asctime)s][%(process)d]"
-                                        "[%(thread)d][%(levelname)s]"
-                                        "[%(name)s:%(lineno)d] "
-                                        "%(message)s"}
-                   },
-    "handlers": {"file": {"formatter": "detail",
-                          "class":
-                              "logging.handlers.RotatingFileHandler",
-                          "backupCount": 5,
-                          "maxBytes": 10000000,
-                          "level": logging.DEBUG,
-                          "filename": "/var/log/privacyidea/privacyidea.log"}
-                 },
-    "loggers": {"privacyidea": {"handlers": ["file"],
-                                "qualname": "privacyidea",
-                                "level": logging.DEBUG}
-                }
-}
 
 
 class PiResponseClass(Response):
@@ -136,15 +111,15 @@ def create_app(config_name="development",
     if os.environ.get(ENV_KEY):
         config_file = os.environ[ENV_KEY]
     if not silent:
-        print("Additional configuration can be read from the file {0!s}".format(
-              config_file))
+        print("Additional configuration will be read "
+              "from the file {0!s}".format(config_file))
     app = Flask(__name__, static_folder="static",
                 template_folder="static/templates")
     if config_name:
         app.config.from_object(config[config_name])
 
     # Set up flask-versioned
-    versioned = Versioned(app, format='%(path)s?v=%(version)s');
+    versioned = Versioned(app, format='%(path)s?v=%(version)s')
 
     try:
         # Try to load the given config_file.
@@ -196,38 +171,40 @@ def create_app(config_name="development",
 
     app.response_class = PiResponseClass
 
-    try:
-        # Try to read logging config from file
-        log_config_file = app.config.get("PI_LOGCONFIG",
-                                         "/etc/privacyidea/logging.cfg")
-        if os.path.isfile(log_config_file):
-            logging.config.fileConfig(log_config_file)
-            if not silent:
-                print("Reading Logging settings from {0!s}".format(log_config_file))
-        else:
-            raise Exception("The config file specified in PI_LOGCONFIG does "
-                            "not exist.")
-    except Exception as exx:
+    # Setup logging
+    log_read_func = {
+        'yaml': lambda x: logging.config.dictConfig(yaml.safe_load(open(x, 'r').read())),
+        'cfg': lambda x: logging.config.fileConfig(x)
+    }
+    have_config = False
+    log_exx = None
+    log_config_file = app.config.get("PI_LOGCONFIG", "/etc/privacyidea/logging.cfg")
+    if os.path.isfile(log_config_file):
+        for cnf_type in ['cfg', 'yaml']:
+            try:
+                log_read_func[cnf_type](log_config_file)
+                if not silent:
+                    print('Read Logging settings from {0!s}'.format(log_config_file))
+                have_config = True
+                break
+            except Exception as exx:
+                log_exx = exx
+                pass
+    if not have_config:
+        if log_exx:
+            sys.stderr.write("Could not use PI_LOGCONFIG: " + str(log_exx) + "\n")
         if not silent:
-            sys.stderr.write("{0!s}\n".format(exx))
-            sys.stderr.write("Could not use PI_LOGCONFIG. "
-                             "Using PI_LOGLEVEL and PI_LOGFILE.\n")
-        level = app.config.get("PI_LOGLEVEL", logging.DEBUG)
+            sys.stderr.write("Using PI_LOGLEVEL and PI_LOGFILE.\n")
+        level = app.config.get("PI_LOGLEVEL", logging.INFO)
         # If there is another logfile in pi.cfg we use this.
-        logfile = app.config.get("PI_LOGFILE")
-        if logfile:
-            if not silent:
-                sys.stderr.write("Using PI_LOGLEVEL {0!s}.\n".format(level))
-                sys.stderr.write("Using PI_LOGFILE {0!s}.\n".format(logfile))
-            PI_LOGGING_CONFIG["handlers"]["file"]["filename"] = logfile
-            PI_LOGGING_CONFIG["handlers"]["file"]["level"] = level
-            PI_LOGGING_CONFIG["loggers"]["privacyidea"]["level"] = level
-            logging.config.dictConfig(PI_LOGGING_CONFIG)
-        else:
-            if not silent:
-                sys.stderr.write("No PI_LOGFILE found. Using default "
-                                  "config.\n")
-            logging.config.dictConfig(DEFAULT_LOGGING_CONFIG)
+        logfile = app.config.get("PI_LOGFILE", '/var/log/privacyidea/privacyidea.log')
+        if not silent:
+            sys.stderr.write("Using PI_LOGLEVEL {0!s}.\n".format(level))
+            sys.stderr.write("Using PI_LOGFILE {0!s}.\n".format(logfile))
+        DEFAULT_LOGGING_CONFIG["handlers"]["file"]["filename"] = logfile
+        DEFAULT_LOGGING_CONFIG["handlers"]["file"]["level"] = level
+        DEFAULT_LOGGING_CONFIG["loggers"]["privacyidea"]["level"] = level
+        logging.config.dictConfig(DEFAULT_LOGGING_CONFIG)
 
     babel = Babel(app)
 
diff --git a/privacyidea/config.py b/privacyidea/config.py
--- a/privacyidea/config.py
+++ b/privacyidea/config.py
@@ -25,6 +25,7 @@
 -----END PUBLIC KEY-----
 """
 
+
 class Config(object):
     SECRET_KEY = os.environ.get('SECRET_KEY')
     # SQL_ALCHEMY_DATABASE_URI = "mysql://privacyidea:XmbSrlqy5d4IS08zjz"
@@ -36,7 +37,6 @@ class Config(object):
     PI_AUDIT_KEY_PUBLIC = os.path.join(basedir, "tests/testdata/public.pem")
     PI_LOGFILE = "privacyidea.log"
     PI_LOGLEVEL = logging.INFO
-    PI_LOGLEVEL = 9
     CACHE_TYPE = "simple"
     PI_EXTERNAL_LINKS = True
     SQLALCHEMY_TRACK_MODIFICATIONS = False
@@ -66,7 +66,6 @@ class TestingConfig(Config):
     PI_PEPPER = ""
     # This is only for testing encrypted files
     PI_ENCFILE_ENC = "tests/testdata/enckey.enc"
-    PI_LOGLEVEL = logging.DEBUG
     PI_LOGLEVEL = logging.INFO
     PI_GNUPG_HOME = "tests/testdata/gpg"
     CACHE_TYPE = "None"
@@ -96,25 +95,23 @@ class TestingConfig(Config):
 
 
 class ProductionConfig(Config):
-    config_path = basedir
     SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL') or \
-        'sqlite:///' + os.path.join(config_path, 'data.sqlite')
+        'sqlite:///' + os.path.join(basedir, 'data.sqlite')
     #SQLALCHEMY_DATABASE_URI = "mysql://pi2:pi2@localhost/pi2"
     # This is used to encrypt the auth_token
     SECRET_KEY = os.environ.get('SECRET_KEY') or 't0p s3cr3t'
     # This is used to encrypt the admin passwords
     PI_PEPPER = "Never know..."
     # This is used to encrypt the token data and token passwords
-    PI_ENCFILE = os.path.join(config_path, "enckey")
+    PI_ENCFILE = os.path.join(basedir, "enckey")
     # This is used to sign the audit log
-    PI_AUDIT_KEY_PRIVATE = os.path.join(config_path, "private.pem")
-    PI_AUDIT_KEY_PUBLIC = os.path.join(config_path, "public.pem")
+    PI_AUDIT_KEY_PRIVATE = os.path.join(basedir, "private.pem")
+    PI_AUDIT_KEY_PUBLIC = os.path.join(basedir, "public.pem")
     PI_LOGLEVEL = logging.INFO
     SUPERUSER_REALM = ['superuser']
 
 
 class HerokuConfig(Config):
-    config_path = basedir
     SQLALCHEMY_DATABASE_URI = "postgres://mvfkmtkwzuwojj:" \
                               "wqy_btZE3CPPNWsmkfdmeorxy6@" \
                               "ec2-54-83-0-61.compute-1." \
@@ -125,11 +122,11 @@ class HerokuConfig(Config):
     # This is used to encrypt the admin passwords
     PI_PEPPER = "Never know..."
     # This is used to encrypt the token data and token passwords
-    PI_ENCFILE = os.path.join(config_path, "deploy/heroku/enckey")
+    PI_ENCFILE = os.path.join(basedir, "deploy/heroku/enckey")
     # This is used to sign the audit log
-    PI_AUDIT_KEY_PRIVATE = os.path.join(config_path,
+    PI_AUDIT_KEY_PRIVATE = os.path.join(basedir,
                                         "deploy/heroku/private.pem")
-    PI_AUDIT_KEY_PUBLIC = os.path.join(config_path, "deploy/heroku/public.pem")
+    PI_AUDIT_KEY_PUBLIC = os.path.join(basedir, "deploy/heroku/public.pem")
     SUPERUSER_REALM = ['superuser']
 
 
diff --git a/privacyidea/lib/log.py b/privacyidea/lib/log.py
--- a/privacyidea/lib/log.py
+++ b/privacyidea/lib/log.py
@@ -34,13 +34,13 @@
 
 DEFAULT_LOGGING_CONFIG = {
     "version": 1,
-    "formatters": {"detail": {"class":
+    "formatters": {"detail": {"()":
                                   "privacyidea.lib.log.SecureFormatter",
                               "format": "[%(asctime)s][%(process)d]"
                                         "[%(thread)d][%(levelname)s]"
                                         "[%(name)s:%(lineno)d] "
                                         "%(message)s"}
-                       },
+                   },
     "handlers": {"file": {"formatter": "detail",
                           "class":
                               "logging.handlers.RotatingFileHandler",
@@ -48,13 +48,14 @@
                           "maxBytes": 10000000,
                           "level": logging.DEBUG,
                           "filename": "privacyidea.log"}
-                         },
+                 },
     "loggers": {"privacyidea": {"handlers": ["file"],
                                 "qualname": "privacyidea",
                                 "level": logging.INFO}
                 }
 }
 
+
 class SecureFormatter(Formatter):
 
     bad_chars = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19"


Test patch:
diff --git a/tests/requirements.txt b/tests/requirements.txt
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -6,11 +6,12 @@ contextlib2==0.6.0.post1; python_version < '3.0'
 cookies==2.2.1; python_version < '3.0'
 coverage==5.0.3
 funcsigs==1.0.2; python_version < '3.0'
-importlib-metadata==1.4.0
-mock==3.0.5
+importlib-metadata==1.5.0
+mock==4.0.2; python_version > '3.5'
+mock==3.0.5; python_version <= '3.5'
 more-itertools==5.0.0; python_version < '3.0'
 more-itertools==8.1.0; python_version > '3.0'
-packaging==20.0
+packaging==20.3
 pathlib2==2.3.5; python_version < '3.0'
 pluggy==0.13.1
 py==1.8.1
@@ -18,7 +19,9 @@ pyparsing==2.4.6
 pytest==4.6.9; python_version < '3.0'
 pytest==5.3.2; python_version > '3.0'
 pytest-cov==2.8.1
-responses==0.10.9
+responses==0.10.12
 scandir==1.10.0; python_version < '3.0'
+testfixtures==6.14.0
 wcwidth==0.1.8
-zipp==0.6.0
+zipp==3.1.0; python_version > '3.5'
+zipp==1.2.0; python_version <= '3.5'
diff --git a/tests/test_app.py b/tests/test_app.py
new file mode 100644
--- /dev/null
+++ b/tests/test_app.py
@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+"""
+This testfile tests the basic app functionality of the privacyIDEA app
+"""
+import os
+import unittest
+import flask
+import inspect
+import logging
+import mock
+from testfixtures import Comparison, compare
+from privacyidea.app import create_app, PiResponseClass
+from privacyidea.config import config, TestingConfig
+
+dirname = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
+
+
+class AppTestCase(unittest.TestCase):
+    def setUp(self):
+        self.logger = logging.getLogger()
+        self.orig_handlers = self.logger.handlers
+        self.logger.handlers = []
+        self.level = self.logger.level
+
+    def tearDown(self):
+        self.logger.handlers = self.orig_handlers
+        self.logger.level = self.level
+
+    def test_01_create_default_app(self):
+        # This will create the app with the 'development' configuration
+        app = create_app()
+        self.assertIsInstance(app, flask.app.Flask, app)
+        self.assertEqual(app.env, 'production', app)
+        self.assertTrue(app.debug, app)
+        self.assertFalse(app.testing, app)
+        self.assertEqual(app.import_name, 'privacyidea.app', app)
+        self.assertEqual(app.name, 'privacyidea.app', app)
+        self.assertTrue(app.response_class == PiResponseClass, app)
+        blueprints = ['validate_blueprint', 'token_blueprint', 'system_blueprint',
+                      'resolver_blueprint', 'realm_blueprint', 'defaultrealm_blueprint',
+                      'policy_blueprint', 'login_blueprint', 'jwtauth', 'user_blueprint',
+                      'audit_blueprint', 'machineresolver_blueprint', 'machine_blueprint',
+                      'application_blueprint', 'caconnector_blueprint', 'cert_blueprint',
+                      'ttype_blueprint', 'register_blueprint', 'smtpserver_blueprint',
+                      'recover_blueprint', 'radiusserver_blueprint', 'periodictask_blueprint',
+                      'privacyideaserver_blueprint', 'eventhandling_blueprint',
+                      'smsgateway_blueprint', 'client_blueprint', 'subscriptions_blueprint',
+                      'monitoring_blueprint']
+        self.assertTrue(all(k in app.before_request_funcs for k in blueprints), app)
+        self.assertTrue(all(k in app.blueprints for k in blueprints), app)
+        extensions = ['sqlalchemy', 'migrate', 'babel']
+        self.assertTrue(all(k in extensions for k in app.extensions), app)
+        self.assertEqual(app.secret_key, 't0p s3cr3t', app)
+        # TODO: check url_map and view_functions
+        # check that the configuration was loaded successfully
+        # the default configuration is 'development'
+        dc = config['development']()
+        members = inspect.getmembers(dc, lambda a: not(inspect.isroutine(a)))
+        conf = [m for m in members if not (m[0].startswith('__') and m[0].endswith('__'))]
+        self.assertTrue(all(app.config[k] == v for k, v in conf), app)
+        # check the correct initialization of the logging
+        logger = logging.getLogger('privacyidea')
+        self.assertEqual(logger.level, logging.DEBUG, logger)
+        compare([
+            Comparison('logging.handlers.RotatingFileHandler',
+                       baseFilename=os.path.join(dirname, 'privacyidea.log'),
+                       formatter=Comparison('privacyidea.lib.log.SecureFormatter',
+                                            _fmt="[%(asctime)s][%(process)d]"
+                                                 "[%(thread)d][%(levelname)s]"
+                                                 "[%(name)s:%(lineno)d] "
+                                                 "%(message)s",
+                                            strict=False),
+                       level=logging.DEBUG,
+                       strict=False)
+        ], logger.handlers)
+
+    def test_02_create_production_app(self):
+        app = create_app(config_name='production')
+        dc = config['production']()
+        members = inspect.getmembers(dc, lambda a: not(inspect.isroutine(a)))
+        conf = [m for m in members if not (m[0].startswith('__') and m[0].endswith('__'))]
+        self.assertTrue(all(app.config[k] == v for k, v in conf), app)
+
+    def test_03_logging_config_file(self):
+        class Config(TestingConfig):
+            PI_LOGCONFIG = "tests/testdata/logging.cfg"
+        with mock.patch.dict("privacyidea.config.config", {"testing": Config}):
+            app = create_app(config_name='testing')
+            # check the correct initialization of the logging from config file
+            logger = logging.getLogger('privacyidea')
+            self.assertEqual(logger.level, logging.DEBUG, logger)
+            compare([
+                Comparison('logging.handlers.RotatingFileHandler',
+                           baseFilename=os.path.join(dirname, 'privacyidea.log'),
+                           formatter=Comparison('privacyidea.lib.log.SecureFormatter',
+                                                _fmt="[%(asctime)s][%(process)d]"
+                                                     "[%(thread)d][%(levelname)s]"
+                                                     "[%(name)s:%(lineno)d] "
+                                                     "%(message)s",
+                                                strict=False),
+                           level=logging.DEBUG,
+                           strict=False)
+            ], logger.handlers)
+            logger = logging.getLogger('privacyidea.lib.auditmodules.loggeraudit')
+            self.assertEqual(logger.level, logging.INFO, logger)
+            compare([
+                Comparison('logging.handlers.RotatingFileHandler',
+                           baseFilename=os.path.join(dirname, 'audit.log'),
+                           formatter=Comparison('privacyidea.lib.log.SecureFormatter',
+                                                _fmt="[%(asctime)s][%(process)d]"
+                                                     "[%(thread)d][%(levelname)s]"
+                                                     "[%(name)s:%(lineno)d] "
+                                                     "%(message)s",
+                                                strict=False),
+                           level=logging.INFO,
+                           strict=False)
+            ], logger.handlers)
+
+    def test_04_logging_config_yaml(self):
+        class Config(TestingConfig):
+            PI_LOGCONFIG = "tests/testdata/logging.yml"
+        with mock.patch.dict("privacyidea.config.config", {"testing": Config}):
+            app = create_app(config_name='testing')
+            # check the correct initialization of the logging from config file
+            logger = logging.getLogger('privacyidea')
+            self.assertEqual(logger.level, logging.INFO, logger)
+            compare([
+                Comparison('logging.handlers.RotatingFileHandler',
+                           baseFilename=os.path.join(dirname, 'privacyidea.log'),
+                           formatter=Comparison('privacyidea.lib.log.SecureFormatter',
+                                                _fmt="[%(asctime)s][%(process)d]"
+                                                     "[%(thread)d][%(levelname)s]"
+                                                     "[%(name)s:%(lineno)d] "
+                                                     "%(message)s",
+                                                strict=False),
+                           backupCount=5,
+                           level=logging.DEBUG,
+                           strict=False)
+            ], logger.handlers)
+            logger = logging.getLogger('audit')
+            self.assertEqual(logger.level, logging.INFO, logger)
+            compare([
+                Comparison('logging.handlers.RotatingFileHandler',
+                           backupCount=14,
+                           baseFilename=os.path.join(dirname, 'audit.log'),
+                           level=logging.INFO,
+                           formatter=None,
+                           strict=False)
+            ], logger.handlers)
+
+    def test_05_logging_config_broken_yaml(self):
+        class Config(TestingConfig):
+            PI_LOGCONFIG = "tests/testdata/logging_broken.yaml"
+        with mock.patch.dict("privacyidea.config.config", {"testing": Config}):
+            app = create_app(config_name='testing')
+            # check the correct initialization of the logging with the default
+            # values since the yaml file is broken
+            logger = logging.getLogger('privacyidea')
+            self.assertEqual(logger.level, logging.INFO, logger)
+            compare([
+                Comparison('logging.handlers.RotatingFileHandler',
+                           baseFilename=os.path.join(dirname, 'privacyidea.log'),
+                           formatter=Comparison('privacyidea.lib.log.SecureFormatter',
+                                                _fmt="[%(asctime)s][%(process)d]"
+                                                     "[%(thread)d][%(levelname)s]"
+                                                     "[%(name)s:%(lineno)d] "
+                                                     "%(message)s",
+                                                strict=False),
+                           level=logging.INFO,
+                           strict=False)
+            ], logger.handlers)
diff --git a/tests/testdata/logging.yml b/tests/testdata/logging.yml
new file mode 100644
--- /dev/null
+++ b/tests/testdata/logging.yml
@@ -0,0 +1,31 @@
+formatters:
+  detail:
+    (): privacyidea.lib.log.SecureFormatter
+    format: '[%(asctime)s][%(process)d][%(thread)d][%(levelname)s][%(name)s:%(lineno)d]
+      %(message)s'
+handlers:
+  file:
+    backupCount: 5
+    class: logging.handlers.RotatingFileHandler
+    filename: privacyidea.log
+    formatter: detail
+    level: DEBUG
+    maxBytes: 10000000
+  file_audit:
+    class: logging.handlers.RotatingFileHandler
+    backupCount: 14
+    filename: audit.log
+    level: INFO
+    maxBytes: 10000000
+loggers:
+  privacyidea:
+    handlers:
+    - file
+    level: INFO
+    qualname: privacyidea
+  audit:
+    handlers:
+    - file_audit
+    level: INFO
+    qualname: pi-audit
+version: 1
diff --git a/tests/testdata/logging_broken.yaml b/tests/testdata/logging_broken.yaml
new file mode 100644
--- /dev/null
+++ b/tests/testdata/logging_broken.yaml
@@ -0,0 +1,15 @@
+handlers:
+  file:
+    backupCount: 5
+    class: logging.handlers.RotatingFileHandler
+    filename: privacyidea.log
+    formatter: detail
+    level: BLABLA
+    maxBytes: 10000000
+loggers:
+  privacyidea:
+    handlers:
+    - filehandler
+    level: INFO
+    qualname: privacyidea
+version: 1