Repository: googleapis/python-logging

Issue:
Add support to custom `JSONEncoder`
 **Is your feature request related to a problem? Please describe.**
My company is moving out from [structlog](https://www.structlog.org/en/stable/) to start using this package, as it is intended to have better integration with google products. We're using `StructuredLogHandler` to send the log records to google cloud logging. But something structlog used to do out-of-the-box was the ability to automatically convert the extra data we provide to the logger to be sent to the stream, and therefore, we need to check all the calls we're making to `logger.info()`, `debug()`, `warning()`, etc. to check if the data type is supported by JSONEncoder, otherwise the call to `hadler.format()` will raise `TypeError`, stating that `Object of type PosixPath is not JSON serializable` (this is one example of error we're getting, because the default encoder cannot encode `pathlib.Path` instances) and the log record will not be pushed to gcloud logging
 **Describe the solution you'd like**
I'd like to be able to specify a custom class to encode the message to JSON, so the formatting is handled by the encoder, not the caller of `logger.info()` and its siblings
 **Describe alternatives you've considered**
An alternative would be having a filter that prepares the data to be used by the handler, I'm even wrote a filter to handle that:
```python
class JsonableConverterFilter(logging.Filter):
    """Prepare log record to be JSON encoded

    :class:`google.cloud.logging_v2.handlers.StructuredLogHandler` encodes directly to default JSON encoder without
    the possibility to set up a custom encoder, so we need to hydrate the log record to not break when emitting
    (because broken logs break silently)

    The ideal solution would be to have a LogFormatter, but as GCloud's log handler formats to JSON inside the handler,
    we need to circumvent this issue with a :class:`logging.Filter`
    """

    def filter(self, record: logging.LogRecord) -> bool:
        fields: Dict[str, Any] = getattr(record, "json_fields", {})

        for key, value in fields.items():
            fields[key] = JsonableConverterFilter.prepare_field(value)

        return True

    @classmethod
    def prepare_field(cls, value: Any) -> TypeJSON:
        """Converts the provided type into a plain JSON type"""
        # basic types
        if isinstance(value, (int, str, float, bool)) or value is None:
            return value

        try:
            iterator = iter(value)
        except TypeError:
            # probably a custom class
            if not hasattr(value, "__dict__"):
                return str(value)

            iterator = iter(value.__dict__)

        # hashmaps
        if hasattr(value, "keys") and hasattr(value, "values"):
            return {key: cls.prepare_field(val) for key, val in value.items()}

        # sequences
        return [cls.prepare_field(item) for item in iterator]

```
but it seems to me I'm deviating the purpose of log filters by doing that
 **Additional context**
n/a for now


Comments:

Patch:
diff --git a/google/cloud/logging_v2/handlers/structured_log.py b/google/cloud/logging_v2/handlers/structured_log.py
index 6552825..55ed9c2 100644
--- a/google/cloud/logging_v2/handlers/structured_log.py
+++ b/google/cloud/logging_v2/handlers/structured_log.py
@@ -62,12 +62,15 @@ class StructuredLogHandler(logging.StreamHandler):
     and write them to standard output
     """
 
-    def __init__(self, *, labels=None, stream=None, project_id=None):
+    def __init__(
+        self, *, labels=None, stream=None, project_id=None, json_encoder_cls=None
+    ):
         """
         Args:
             labels (Optional[dict]): Additional labels to attach to logs.
             stream (Optional[IO]): Stream to be used by the handler.
             project (Optional[str]): Project Id associated with the logs.
+            json_encoder_cls (Optional[Type[JSONEncoder]]): Custom JSON encoder. Defaults to json.JSONEncoder
         """
         super(StructuredLogHandler, self).__init__(stream=stream)
         self.project_id = project_id
@@ -79,6 +82,8 @@ class StructuredLogHandler(logging.StreamHandler):
         # make logs appear in GCP structured logging format
         self._gcp_formatter = logging.Formatter(GCP_FORMAT)
 
+        self._json_encoder_cls = json_encoder_cls or json.JSONEncoder
+
     def format(self, record):
         """Format the message into structured log JSON.
         Args:
@@ -95,14 +100,18 @@ class StructuredLogHandler(logging.StreamHandler):
                 if key in GCP_STRUCTURED_LOGGING_FIELDS:
                     del message[key]
             # if input is a dictionary, encode it as a json string
-            encoded_msg = json.dumps(message, ensure_ascii=False)
+            encoded_msg = json.dumps(
+                message, ensure_ascii=False, cls=self._json_encoder_cls
+            )
             # all json.dumps strings should start and end with parentheses
             # strip them out to embed these fields in the larger JSON payload
             if len(encoded_msg) > 2:
                 payload = encoded_msg[1:-1] + ","
         elif message:
             # properly break any formatting in string to make it json safe
-            encoded_message = json.dumps(message, ensure_ascii=False)
+            encoded_message = json.dumps(
+                message, ensure_ascii=False, cls=self._json_encoder_cls
+            )
             payload = '"message": {},'.format(encoded_message)
 
         record._payload_str = payload or ""


Test patch:
diff --git a/tests/unit/handlers/test_structured_log.py b/tests/unit/handlers/test_structured_log.py
index 61bf36f..d930da7 100644
--- a/tests/unit/handlers/test_structured_log.py
+++ b/tests/unit/handlers/test_structured_log.py
@@ -46,6 +46,15 @@ class TestStructuredLogHandler(unittest.TestCase):
         handler = self._make_one(project_id="foo")
         self.assertEqual(handler.project_id, "foo")
 
+    def test_ctor_w_encoder(self):
+        import json
+
+        class CustomJSONEncoder(json.JSONEncoder):
+            pass
+
+        handler = self._make_one(json_encoder_cls=CustomJSONEncoder)
+        self.assertEqual(handler._json_encoder_cls, CustomJSONEncoder)
+
     def test_format(self):
         import logging
         import json
@@ -207,6 +216,51 @@ class TestStructuredLogHandler(unittest.TestCase):
         self.assertIn(expected_result, result)
         self.assertIn("message", result)
 
+    def test_format_with_custom_json_encoder(self):
+        import json
+        import logging
+
+        from pathlib import Path
+        from typing import Any
+
+        class CustomJSONEncoder(json.JSONEncoder):
+            def default(self, obj: Any) -> Any:
+                if isinstance(obj, Path):
+                    return str(obj)
+                return json.JSONEncoder.default(self, obj)
+
+        handler = self._make_one(json_encoder_cls=CustomJSONEncoder)
+
+        message = "hello world"
+        json_fields = {"path": Path("/path")}
+        record = logging.LogRecord(
+            None,
+            logging.INFO,
+            None,
+            None,
+            message,
+            None,
+            None,
+        )
+        setattr(record, "json_fields", json_fields)
+        expected_payload = {
+            "message": message,
+            "severity": "INFO",
+            "logging.googleapis.com/trace": "",
+            "logging.googleapis.com/spanId": "",
+            "logging.googleapis.com/trace_sampled": False,
+            "logging.googleapis.com/sourceLocation": {},
+            "httpRequest": {},
+            "logging.googleapis.com/labels": {},
+            "path": "/path",
+        }
+        handler.filter(record)
+
+        result = json.loads(handler.format(record))
+
+        self.assertEqual(set(expected_payload.keys()), set(result.keys()))
+        self.assertEqual(result["path"], "/path")
+
     def test_format_with_reserved_json_field(self):
         # drop json_field data with reserved names
         # related issue: https://github.com/googleapis/python-logging/issues/543