Repository: elastic/apm-agent-python

Issue:
exception logging an error with a big local variable
**Describe the bug**: 
apm python agent tries to log an exception with a big local variable in the execution stack results in unexpected behavior

**To Reproduce**

```python
# serializers.py
from rest_framework import serializers

class UploadFileSerializer(serializers.Serializer):
    myfile = serializers.ListField(child=serializers.FileField())

    def validate_myfile(self, value):
        raise serializers.ValidationError("Document upload size limit is exceeded")
# urls.py 
urlpatterns += [path('upload_file', UploadFile.as_view(), name='upload_file')]

# views.py
class UploadFile(GenericAPIView):
    serializer_class = UploadFileSerializer

    def post(self, request):
        data = request.data            # this local variable is the problem 
        serializer = self.serializer_class(data=data)
        try:
            if serializer.is_valid(raise_exception=True):
                return Response(status=status.HTTP_200_OK)
        except Exception as e:
            logger.exception(e)
            raise e
```
try to upload file > 3 mega (use multipart)
 
**Expected result**
return 400 because of ValidationError exception 
**Actual result**
```python
[ERROR] {'myfile': [ErrorDetail(string='Document upload size limit is exceeded', code='invalid')]}
Traceback (most recent call last):
  File "/home/me/myporject/api/views.py", line 127, in post
    if serializer.is_valid(raise_exception=True):
  File "/home/me/myporject/venv/lib/python3.9/site-packages/rest_framework/serializers.py", line 242, in is_valid
    raise ValidationError(self.errors)
rest_framework.exceptions.ValidationError: {'myfile': [ErrorDetail(string='Document upload size limit is exceeded', code='invalid')]}
Top level ElasticAPM exception caught - failed creating log record.
[ERROR] Unable to process log entry: cannot pickle '_io.BufferedRandom' object
Traceback (most recent call last):
  File "/home/me/myporject/api/views.py", line 127, in post
    if serializer.is_valid(raise_exception=True):
  File "/home/me/myporject/venv/lib/python3.9/site-packages/rest_framework/serializers.py", line 242, in is_valid
    raise ValidationError(self.errors)
rest_framework.exceptions.ValidationError: {'myfile': [ErrorDetail(string='Document upload size limit is exceeded', code='invalid')]}

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/me/myporject/venv/lib/python3.9/site-packages/elasticapm/handlers/logging.py", line 81, in emit
    return self._emit(record)
  File "/home/me/myporject/venv/lib/python3.9/site-packages/elasticapm/contrib/django/handlers.py", line 63, in _emit
    return super(LoggingHandler, self)._emit(record, request=request, **kwargs)
  File "/home/me/myporject/venv/lib/python3.9/site-packages/elasticapm/handlers/logging.py", line 152, in _emit
    exception = handler.capture(self.client, exc_info=record.exc_info)
  File "/home/me/myporject/venv/lib/python3.9/site-packages/elasticapm/events.py", line 96, in capture
    frames = get_stack_info(
  File "/home/me/myporject/venv/lib/python3.9/site-packages/elasticapm/utils/stacks.py", line 325, in get_stack_info
    result = get_frame_info(
  File "/home/me/myporject/venv/lib/python3.9/site-packages/elasticapm/utils/stacks.py", line 294, in get_frame_info
    f_locals = {varname: locals_processor_func(var) for varname, var in compat.iteritems(f_locals)}
  File "/home/me/myporject/venv/lib/python3.9/site-packages/elasticapm/utils/stacks.py", line 294, in <dictcomp>
    f_locals = {varname: locals_processor_func(var) for varname, var in compat.iteritems(f_locals)}
  File "/home/me/myporject/venv/lib/python3.9/site-packages/elasticapm/events.py", line 103, in <lambda>
    locals_processor_func=lambda local_var: varmap(
  File "/home/me/myporject/venv/lib/python3.9/site-packages/elasticapm/utils/__init__.py", line 66, in varmap
    name, dict((k, varmap(func, v, context, k, **kwargs)) for k, v in compat.iteritems(var.copy())), **kwargs
  File "/home/me/myporject/venv/lib/python3.9/site-packages/django/http/request.py", line 517, in copy
    return self.__deepcopy__({})
  File "/home/me/myporject/venv/lib/python3.9/site-packages/django/http/request.py", line 478, in __deepcopy__
    result.setlist(copy.deepcopy(key, memo), copy.deepcopy(value, memo))
  File "/usr/lib/python3.9/copy.py", line 146, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python3.9/copy.py", line 205, in _deepcopy_list
    append(deepcopy(a, memo))
  File "/usr/lib/python3.9/copy.py", line 172, in deepcopy
    y = _reconstruct(x, memo, *rv)
  File "/usr/lib/python3.9/copy.py", line 270, in _reconstruct
    state = deepcopy(state, memo)
  File "/usr/lib/python3.9/copy.py", line 146, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python3.9/copy.py", line 230, in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
  File "/usr/lib/python3.9/copy.py", line 172, in deepcopy
    y = _reconstruct(x, memo, *rv)
  File "/usr/lib/python3.9/copy.py", line 270, in _reconstruct
    state = deepcopy(state, memo)
  File "/usr/lib/python3.9/copy.py", line 146, in deepcopy
    y = copier(x, memo)
  File "/usr/lib/python3.9/copy.py", line 230, in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
  File "/usr/lib/python3.9/copy.py", line 161, in deepcopy
    rv = reductor(4)
TypeError: cannot pickle '_io.BufferedRandom' object

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/me/myporject/venv/lib/python3.9/site-packages/django/core/handlers/exception.py", line 34, in inner
    response = get_response(request)
  File "/home/me/myporject/venv/lib/python3.9/site-packages/django/core/handlers/base.py", line 115, in _get_response
    response = self.process_exception_by_middleware(e, request)
  File "/home/me/myporject/venv/lib/python3.9/site-packages/django/core/handlers/base.py", line 113, in _get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/home/me/myporject/venv/lib/python3.9/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
    return view_func(*args, **kwargs)
  File "/home/me/myporject/venv/lib/python3.9/site-packages/django/views/generic/base.py", line 71, in view
    return self.dispatch(request, *args, **kwargs)
  File "/home/me/myporject/venv/lib/python3.9/site-packages/rest_framework/views.py", line 505, in dispatch
    response = self.handle_exception(exc)
  File "/home/me/myporject/venv/lib/python3.9/site-packages/rest_framework/views.py", line 465, in handle_exception
    self.raise_uncaught_exception(exc)
  File "/home/me/myporject/venv/lib/python3.9/site-packages/rest_framework/views.py", line 476, in raise_uncaught_exception
    raise exc
  File "/home/me/myporject/venv/lib/python3.9/site-packages/rest_framework/views.py", line 502, in dispatch
    response = handler(request, *args, **kwargs)
  File "/home/me/myporject/api/views.py", line 131, in post
    logger.exception(e)
  File "/usr/lib/python3.9/logging/__init__.py", line 1481, in exception
    self.error(msg, *args, exc_info=exc_info, **kwargs)
  File "/usr/lib/python3.9/logging/__init__.py", line 1475, in error
    self._log(ERROR, msg, args, **kwargs)
  File "/usr/lib/python3.9/logging/__init__.py", line 1589, in _log
    self.handle(record)
  File "/usr/lib/python3.9/logging/__init__.py", line 1599, in handle
    self.callHandlers(record)
  File "/usr/lib/python3.9/logging/__init__.py", line 1661, in callHandlers
    hdlr.handle(record)
  File "/usr/lib/python3.9/logging/__init__.py", line 952, in handle
    self.emit(record)
  File "/home/me/myporject/venv/lib/python3.9/site-packages/elasticapm/handlers/logging.py", line 84, in emit
    sys.stderr.write(to_unicode(record.msg + "\n"))
TypeError: unsupported operand type(s) for +: 'ValidationError' and 'str'

During handling of the above exception, another exception occurred:
...

[25/Sep/2021 14:38:21] "POST /api/upload_file HTTP/1.1" 500 27
```

**Environment (please complete the following information)**
- OS: linux
- Python version: 3.9 tested on 3.6 and 3.8
- Framework and version [e.g. Django 2.1]:  django 2.2 and drf 3.10
- Agent version: 6.4.0




Discussion:
Hmmm, I'm pretty sure we have logic in place to truncate these large variables. Obviously something is going wrong there, we'll find some time to take a look.

Resolving patch:
diff --git a/elasticapm/handlers/logbook.py b/elasticapm/handlers/logbook.py
--- a/elasticapm/handlers/logbook.py
+++ b/elasticapm/handlers/logbook.py
@@ -75,15 +75,15 @@ def emit(self, record):
 
         # Avoid typical config issues by overriding loggers behavior
         if record.channel.startswith("elasticapm.errors"):
-            sys.stderr.write(to_unicode(record.message + "\n"))
+            sys.stderr.write(to_unicode(record.message) + "\n")
             return
 
         try:
             return self._emit(record)
         except Exception:
             sys.stderr.write("Top level ElasticAPM exception caught - failed creating log record.\n")
-            sys.stderr.write(to_unicode(record.msg + "\n"))
-            sys.stderr.write(to_unicode(traceback.format_exc() + "\n"))
+            sys.stderr.write(to_unicode(record.message) + "\n")
+            sys.stderr.write(traceback.format_exc() + "\n")
 
             try:
                 self.client.capture("Exception")
diff --git a/elasticapm/handlers/logging.py b/elasticapm/handlers/logging.py
--- a/elasticapm/handlers/logging.py
+++ b/elasticapm/handlers/logging.py
@@ -40,7 +40,6 @@
 from elasticapm.base import Client
 from elasticapm.traces import execution_context
 from elasticapm.utils import wrapt
-from elasticapm.utils.encoding import to_unicode
 from elasticapm.utils.stacks import iter_stack_frames
 
 
@@ -74,15 +73,15 @@ def emit(self, record):
 
         # Avoid typical config issues by overriding loggers behavior
         if record.name.startswith(("elasticapm.errors",)):
-            sys.stderr.write(to_unicode(record.message) + "\n")
+            sys.stderr.write(record.getMessage() + "\n")
             return
 
         try:
             return self._emit(record)
         except Exception:
             sys.stderr.write("Top level ElasticAPM exception caught - failed creating log record.\n")
-            sys.stderr.write(to_unicode(record.msg + "\n"))
-            sys.stderr.write(to_unicode(traceback.format_exc() + "\n"))
+            sys.stderr.write(record.getMessage() + "\n")
+            sys.stderr.write(traceback.format_exc() + "\n")
 
             try:
                 self.client.capture("Exception")


Test patch:
diff --git a/tests/handlers/logbook/logbook_tests.py b/tests/handlers/logbook/logbook_tests.py
--- a/tests/handlers/logbook/logbook_tests.py
+++ b/tests/handlers/logbook/logbook_tests.py
@@ -169,6 +169,15 @@ def test_logbook_handler_emit_error(capsys, elasticapm_client):
     assert "Oops" in err
 
 
+def test_logbook_handler_emit_error_non_str_message(capsys, elasticapm_client):
+    handler = LogbookHandler(elasticapm_client)
+    handler._emit = lambda record: 1 / 0
+    handler.emit(LogRecord("x", 1, ValueError("oh no")))
+    out, err = capsys.readouterr()
+    assert "Top level ElasticAPM exception caught" in err
+    assert "oh no" in err
+
+
 def test_logbook_handler_dont_emit_elasticapm(capsys, elasticapm_client):
     handler = LogbookHandler(elasticapm_client)
     handler.emit(LogRecord("elasticapm.errors", 1, "Oops"))
diff --git a/tests/handlers/logging/logging_tests.py b/tests/handlers/logging/logging_tests.py
--- a/tests/handlers/logging/logging_tests.py
+++ b/tests/handlers/logging/logging_tests.py
@@ -229,7 +229,7 @@ def test_logger_setup():
 def test_logging_handler_emit_error(capsys, elasticapm_client):
     handler = LoggingHandler(elasticapm_client)
     handler._emit = lambda: 1 / 0
-    handler.emit(LogRecord("x", 1, "/ab/c/", 10, "Oops", [], None))
+    handler.emit(LogRecord("x", 1, "/ab/c/", 10, "Oops", (), None))
     out, err = capsys.readouterr()
     assert "Top level ElasticAPM exception caught" in err
     assert "Oops" in err
@@ -237,11 +237,20 @@ def test_logging_handler_emit_error(capsys, elasticapm_client):
 
 def test_logging_handler_dont_emit_elasticapm(capsys, elasticapm_client):
     handler = LoggingHandler(elasticapm_client)
-    handler.emit(LogRecord("elasticapm.errors", 1, "/ab/c/", 10, "Oops", [], None))
+    handler.emit(LogRecord("elasticapm.errors", 1, "/ab/c/", 10, "Oops", (), None))
     out, err = capsys.readouterr()
     assert "Oops" in err
 
 
+def test_logging_handler_emit_error_non_str_message(capsys, elasticapm_client):
+    handler = LoggingHandler(elasticapm_client)
+    handler._emit = lambda: 1 / 0
+    handler.emit(LogRecord("x", 1, "/ab/c/", 10, ValueError("oh no"), (), None))
+    out, err = capsys.readouterr()
+    assert "Top level ElasticAPM exception caught" in err
+    assert "oh no" in err
+
+
 def test_arbitrary_object(logger):
     logger.error(["a", "list", "of", "strings"])
     assert len(logger.client.events) == 1