Repository: elastic/apm-agent-python

Issue:
[META 373] Support replacement of plain-text formatters with ECS formatters
Evaluate and if applicable implement https://github.com/elastic/apm/issues/373


Discussion:

Resolving patch:
diff --git a/elasticapm/conf/__init__.py b/elasticapm/conf/__init__.py
--- a/elasticapm/conf/__init__.py
+++ b/elasticapm/conf/__init__.py
@@ -367,9 +367,43 @@ def _log_level_callback(dict_key, old_value, new_value, config_instance):
         filehandler = logging.handlers.RotatingFileHandler(
             config_instance.log_file, maxBytes=config_instance.log_file_size, backupCount=1
         )
+        try:
+            import ecs_logging
+
+            filehandler.setFormatter(ecs_logging.StdlibFormatter())
+        except ImportError:
+            pass
         elasticapm_logger.addHandler(filehandler)
 
 
+def _auto_ecs_logging_callback(dict_key, old_value, new_value, config_instance):
+    """
+    If ecs_logging is installed and auto_ecs_logging is set to True, we should
+    set the ecs_logging.StdlibFormatter as the formatted for every handler in
+    the root logger, and set the default processor for structlog to the
+    ecs_logging.StructlogFormatter.
+    """
+    if new_value:
+        try:
+            import ecs_logging
+        except ImportError:
+            return
+
+        # Stdlib
+        root_logger = logging.getLogger()
+        formatter = ecs_logging.StdlibFormatter()
+        for handler in root_logger.handlers:
+            handler.setFormatter(formatter)
+
+        # Structlog
+        try:
+            import structlog
+
+            structlog.configure(processors=[ecs_logging.StructlogFormatter()])
+        except ImportError:
+            pass
+
+
 class _ConfigBase(object):
     _NO_VALUE = object()  # sentinel object
 
@@ -582,6 +616,7 @@ class Config(_ConfigBase):
     )
     log_file = _ConfigValue("LOG_FILE", default="")
     log_file_size = _ConfigValue("LOG_FILE_SIZE", validators=[size_validator], type=int, default=50 * 1024 * 1024)
+    auto_ecs_logging = _BoolConfigValue("AUTO_ECS_LOGGING", callbacks=[_auto_ecs_logging_callback], default=False)
 
     @property
     def is_recording(self):


Test patch:
diff --git a/tests/fixtures.py b/tests/fixtures.py
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -202,6 +202,10 @@ def elasticapm_client_log_file(request):
     client_config.setdefault("cloud_provider", False)
     client_config.setdefault("log_level", "warning")
 
+    root_logger = logging.getLogger()
+    handler = logging.StreamHandler()
+    root_logger.addHandler(handler)
+
     tmp = tempfile.NamedTemporaryFile(delete=False)
     tmp.close()
     client_config["log_file"] = tmp.name
@@ -217,6 +221,9 @@ def elasticapm_client_log_file(request):
             handler.close()
     os.unlink(tmp.name)
 
+    # Remove our streamhandler
+    root_logger.removeHandler(handler)
+
     # clear any execution context that might linger around
     sys.excepthook = original_exceptionhook
     execution_context.set_transaction(None)
diff --git a/tests/handlers/logging/logging_tests.py b/tests/handlers/logging/logging_tests.py
--- a/tests/handlers/logging/logging_tests.py
+++ b/tests/handlers/logging/logging_tests.py
@@ -34,7 +34,9 @@
 import warnings
 from logging import LogRecord
 
+import ecs_logging
 import pytest
+import structlog
 
 from elasticapm.conf import Config
 from elasticapm.conf.constants import ERROR
@@ -394,3 +396,10 @@ def test_log_file(elasticapm_client_log_file):
         if isinstance(handler, logging.handlers.RotatingFileHandler):
             found = True
     assert found
+
+
+@pytest.mark.parametrize("elasticapm_client_log_file", [{"auto_ecs_logging": True}], indirect=True)
+def test_auto_ecs_logging(elasticapm_client_log_file):
+    logger = logging.getLogger()
+    assert isinstance(logger.handlers[0].formatter, ecs_logging.StdlibFormatter)
+    assert isinstance(structlog.get_config()["processors"][-1], ecs_logging.StructlogFormatter)
diff --git a/tests/requirements/reqs-base.txt b/tests/requirements/reqs-base.txt
--- a/tests/requirements/reqs-base.txt
+++ b/tests/requirements/reqs-base.txt
@@ -46,6 +46,8 @@ mock
 msgpack-python
 pep8
 pytz
+ecs_logging
+structlog
 
 
 pytest-asyncio==0.14.0 ; python_version >= '3.7'