Repository: learningequality/kolibri

Issue:
Allow setting different logging modes via environment variable
## Observed behavior
We currently have a very fully specified logging configuration, which gets intellligently used both before the Django stack is started and after.

However, it always assumes that the log location should be to disk, which can cause problems for some deployment models.

## Expected behavior
Because logging is used by the options.ini parsing machinery in `options.py`, we can't use a configurable option in options.ini to configure logging. It therefore seems best to simply have an environment variable that can be specified to turn off file based logging here: https://github.com/learningequality/kolibri/blob/develop/kolibri/utils/logger.py#L195

So, all that would change is that the handler lists would be `['console']` in each case, rather than `['file', 'console', 'file_debug']`.

We should call the environment variable: `KOLIBRI_NO_FILE_BASED_LOGGING`.

## Context
This is most acutely needed for cloud deployments of Kolibri, where persistent file storage complicates deployment.



Discussion:
This hits a lot of birds with one stone especially with the hosted use-case.

- Avoids writing to potentially corrupt filesystems
- Stdout/err is much easier to manage on our side
- Removes one more need for a persistent filesystem

Resolving patch:
diff --git a/kolibri/utils/cli.py b/kolibri/utils/cli.py
--- a/kolibri/utils/cli.py
+++ b/kolibri/utils/cli.py
@@ -449,3 +449,35 @@ def list():
             + " " * (max_len - len(plugin) + 4)
             + ("ENABLED" if plugin in config.ACTIVE_PLUGINS else "DISABLED")
         )
+
+
+@main.command(cls=KolibriGroupCommand, help="Configure Kolibri and enabled plugins")
+def configure():
+    pass
+
+
+def _get_env_vars():
+    """
+    Generator to iterate over all environment variables
+    """
+    template = "{envvar} - {description}\n\n"
+
+    from kolibri.utils.env import ENVIRONMENT_VARIABLES
+
+    for key, value in ENVIRONMENT_VARIABLES.items():
+        yield template.format(envvar=key, description=value.get("description", ""))
+
+    from kolibri.utils.options import option_spec
+
+    for value in option_spec.values():
+        for v in value.values():
+            if "envvars" in v:
+                for envvar in v["envvars"]:
+                    yield template.format(
+                        envvar=envvar, description=v.get("description", "")
+                    )
+
+
+@configure.command(help="List all available environment variables to configure Kolibri")
+def list_env():
+    click.echo_via_pager(_get_env_vars())
diff --git a/kolibri/utils/env.py b/kolibri/utils/env.py
--- a/kolibri/utils/env.py
+++ b/kolibri/utils/env.py
@@ -7,6 +7,39 @@
 logging.StreamHandler(sys.stdout)
 
 
+def settings_module():
+    from .build_config.default_settings import settings_path
+
+    return settings_path
+
+
+# Enumerate all the environment variables that are used in Kolibri, and describe
+# their usage. Optionally provide a default value as a callback function, to set as
+# a default if the environment variable is not set.
+ENVIRONMENT_VARIABLES = {
+    "DJANGO_SETTINGS_MODULE": {
+        "default": settings_module,
+        "description": "The Django settings module to use.",
+    },
+    "KOLIBRI_HOME": {
+        "default": lambda: os.path.join(os.path.expanduser("~"), ".kolibri"),
+        "description": "The base directory for the Kolibri installation.",
+    },
+    "KOLIBRI_NO_FILE_BASED_LOGGING": {
+        "description": "Disable file-based logging.",
+    },
+    "KOLIBRI_APK_VERSION_NAME": {
+        "description": "Version name for the Kolibri APK (Android Installer)",
+    },
+    "LISTEN_PID": {
+        "description": """
+            The PID of the process to listen for signals from -
+            used to detect whether running under socket activation under Debian.
+        """,
+    },
+}
+
+
 def prepend_cext_path(dist_path):
     """
     Calculate the directory of C extensions and add it to sys.path if exists.
@@ -71,10 +104,7 @@ def set_env():
             )
         ]
 
-    from .build_config.default_settings import settings_path
-
     # Set default env
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings_path)
-    os.environ.setdefault(
-        "KOLIBRI_HOME", os.path.join(os.path.expanduser("~"), ".kolibri")
-    )
+    for key, value in ENVIRONMENT_VARIABLES.items():
+        if "default" in value:
+            os.environ.setdefault(key, value["default"]())
diff --git a/kolibri/utils/logger.py b/kolibri/utils/logger.py
--- a/kolibri/utils/logger.py
+++ b/kolibri/utils/logger.py
@@ -1,7 +1,5 @@
 import logging
 import os
-import re
-from logging import Formatter
 from logging.handlers import TimedRotatingFileHandler
 
 from . import conf
@@ -10,6 +8,8 @@
 GET_FILES_TO_DELETE = "getFilesToDelete"
 DO_ROLLOVER = "doRollover"
 
+NO_FILE_BASED_LOGGING = os.environ.get("KOLIBRI_NO_FILE_BASED_LOGGING", False)
+
 
 class KolibriTimedRotatingFileHandler(TimedRotatingFileHandler):
     """
@@ -94,22 +94,6 @@ def _rotation_files(self, filenames, prefix, func=DO_ROLLOVER):
         return result
 
 
-class KolibriLogFileFormatter(Formatter):
-    """
-    A custom Formatter to change the format string of Cherrypy logging messages.
-    """
-
-    def format(self, record):
-        if "cherrypy" in record.name:
-            # Remove the timestamp from Cherrypy logging so that the message only contains one timestamp.
-            record.msg = re.sub(r"\[[^)]*\]\s", "", record.msg)
-            # Change the format string for Cherrypy logging messages from %module(_cplogging) to %name.
-            record.module = record.name
-
-        message = super(KolibriLogFileFormatter, self).format(record)
-        return message
-
-
 class FalseFilter(logging.Filter):
     """
     A filter that ignores everything, useful to create log config
@@ -137,6 +121,10 @@ def get_default_logging_config(LOG_ROOT, debug=False, debug_database=False):
     configuration.
     """
 
+    DEFAULT_HANDLERS = (
+        ["console"] if NO_FILE_BASED_LOGGING else ["file", "console", "file_debug"]
+    )
+
     # This is the general level
     DEFAULT_LEVEL = "INFO" if not debug else "DEBUG"
     DATABASE_LEVEL = "INFO" if not debug_database else "DEBUG"
@@ -151,10 +139,6 @@ def get_default_logging_config(LOG_ROOT, debug=False, debug_database=False):
             },
             "simple": {"format": "%(levelname)s %(message)s"},
             "simple_date": {"format": "%(levelname)s %(asctime)s %(name)s %(message)s"},
-            "simple_date_file": {
-                "()": "kolibri.utils.logger.KolibriLogFileFormatter",
-                "format": "%(levelname)s %(asctime)s %(name)s %(message)s",
-            },
             "color": {
                 "()": "colorlog.ColoredFormatter",
                 "format": "%(log_color)s%(levelname)-8s %(message)s",
@@ -178,7 +162,7 @@ def get_default_logging_config(LOG_ROOT, debug=False, debug_database=False):
                 "filters": [],
                 "class": "kolibri.utils.logger.KolibriTimedRotatingFileHandler",
                 "filename": os.path.join(LOG_ROOT, "kolibri.txt"),
-                "formatter": "simple_date_file",
+                "formatter": "simple_date",
                 "when": "midnight",
                 "backupCount": 30,
             },
@@ -192,7 +176,7 @@ def get_default_logging_config(LOG_ROOT, debug=False, debug_database=False):
         },
         "loggers": {
             "kolibri": {
-                "handlers": ["file", "console", "file_debug"],
+                "handlers": DEFAULT_HANDLERS,
                 "level": DEFAULT_LEVEL,
                 "propagate": False,
             },
@@ -200,27 +184,27 @@ def get_default_logging_config(LOG_ROOT, debug=False, debug_database=False):
             # We should introduce custom debug log levels or log
             # targets, i.e. --debug-level=high
             "kolibri.core.tasks.worker": {
-                "handlers": ["file", "console", "file_debug"],
+                "handlers": DEFAULT_HANDLERS,
                 "level": "INFO",
                 "propagate": False,
             },
             "morango": {
-                "handlers": ["file", "console", "file_debug"],
+                "handlers": DEFAULT_HANDLERS,
                 "level": DEFAULT_LEVEL,
                 "propagate": False,
             },
             "django": {
-                "handlers": ["file", "console", "file_debug"],
+                "handlers": DEFAULT_HANDLERS,
                 "level": DEFAULT_LEVEL,
                 "propagate": False,
             },
             "django.db.backends": {
-                "handlers": ["file", "console", "file_debug"],
+                "handlers": DEFAULT_HANDLERS,
                 "level": DATABASE_LEVEL,
                 "propagate": False,
             },
             "django.request": {
-                "handlers": ["file", "console", "file_debug"],
+                "handlers": DEFAULT_HANDLERS,
                 "level": DEFAULT_LEVEL,
                 "propagate": False,
             },


Test patch: