Repository: elastic/apm-agent-python

Issue:
APM agent crashes if the request body is too large
**Describe the bug**: ...
When the request body is too large, reporting to the apm server fails. 
The error message is: 
```
eapm event processor thread elasticapm.transport:286 - Failed to submit message: 'HTTP 400: {"accepted":4,"errors":[{"message":"event exceeded the permitted size." ...
```

**To Reproduce**

1. Check the body's size before reporting. If it's too large, report the body as `[Too large]` or something like that.

**Environment (please complete the following information)**
- OS: mac
- Python version: 3.7
- Framework and version [e.g. Django 2.1]: django==2.2.26
- APM Server version: ?
- Agent version: 5.10.1

Elastic apm configuration (in `settings.py`): 
```
ELASTIC_APM = {
	...
    "CAPTURE_BODY": "all",
}
```

**Additional context**

The issue is that Django client extracts the request body [without checking the size of the body](https://github.com/elastic/apm-agent-python/blob/main/elasticapm/contrib/django/client.py#L151-L152). I believe that adding a size check in [this](https://github.com/elastic/apm-agent-python/blob/97188c362497910762e5b1ef8fc53e4011ec75b8/elasticapm/contrib/django/client.py#L117) function should solve the issue.



Discussion:

Resolving patch:
diff --git a/elasticapm/conf/constants.py b/elasticapm/conf/constants.py
--- a/elasticapm/conf/constants.py
+++ b/elasticapm/conf/constants.py
@@ -70,6 +70,7 @@ def _starmatch_to_regex(pattern):
 TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
 
 KEYWORD_MAX_LENGTH = 1024
+LONG_FIELD_MAX_LENGTH = 10000
 
 HTTP_WITH_BODY = {"POST", "PUT", "PATCH", "DELETE"}
 
diff --git a/elasticapm/contrib/django/client.py b/elasticapm/contrib/django/client.py
--- a/elasticapm/contrib/django/client.py
+++ b/elasticapm/contrib/django/client.py
@@ -46,6 +46,7 @@
 from elasticapm.conf import constants
 from elasticapm.contrib.django.utils import get_raw_uri, iterate_with_template_sources
 from elasticapm.utils import compat, encoding, get_url_dict
+from elasticapm.utils.encoding import long_field
 from elasticapm.utils.logging import get_logger
 from elasticapm.utils.module_import import import_string
 from elasticapm.utils.wsgi import get_environ, get_headers
@@ -149,7 +150,9 @@ def get_data_from_request(self, request, event_type):
                         self.logger.debug("Can't capture request body: %s", str(e))
                         data = "<unavailable>"
                 if data is not None:
-                    result["body"] = data
+                    # Can we apply this as a processor instead?
+                    # https://github.com/elastic/apm-agent-python/issues/305
+                    result["body"] = long_field(data)
 
         url = get_raw_uri(request)
         try:
diff --git a/elasticapm/utils/encoding.py b/elasticapm/utils/encoding.py
--- a/elasticapm/utils/encoding.py
+++ b/elasticapm/utils/encoding.py
@@ -36,7 +36,7 @@
 import uuid
 from decimal import Decimal
 
-from elasticapm.conf.constants import KEYWORD_MAX_LENGTH, LABEL_RE, LABEL_TYPES
+from elasticapm.conf.constants import KEYWORD_MAX_LENGTH, LABEL_RE, LABEL_TYPES, LONG_FIELD_MAX_LENGTH
 
 PROTECTED_TYPES = (int, type(None), float, Decimal, datetime.datetime, datetime.date, datetime.time)
 
@@ -224,6 +224,31 @@ def keyword_field(string):
     return string[: KEYWORD_MAX_LENGTH - 1] + "…"
 
 
+def long_field(data):
+    """
+    If the given data, converted to string, is longer than LONG_FIELD_MAX_LENGTH,
+    truncate it to LONG_FIELD_MAX_LENGTH-1, adding the "…" character at the end.
+
+    Returns the original data if truncation is not required.
+
+    Per https://github.com/elastic/apm/blob/main/specs/agents/field-limits.md#long_field_max_length-configuration,
+    this should only be applied to the following fields:
+
+    - `transaction.context.request.body`, `error.context.request.body`
+    - `transaction.context.message.body`, `span.context.message.body`, `error.context.message.body`
+    - `span.context.db.statement`
+    - `error.exception.message`
+    - `error.log.message`
+
+    Other fields should be truncated via `elasticapm.utils.encoding.keyword_field()`
+    """
+    string = str(data) if not isinstance(data, str) else data
+    if len(string) > LONG_FIELD_MAX_LENGTH:
+        return string[: LONG_FIELD_MAX_LENGTH - 1] + "…"
+    else:
+        return data
+
+
 def enforce_label_format(labels):
     """
     Enforces label format:


Test patch:
diff --git a/tests/contrib/django/django_tests.py b/tests/contrib/django/django_tests.py
--- a/tests/contrib/django/django_tests.py
+++ b/tests/contrib/django/django_tests.py
@@ -848,7 +848,7 @@ def test_request_metrics_301_append_slash(django_elasticapm_client, client):
         **middleware_setting(
             django.VERSION,
             ["elasticapm.contrib.django.middleware.TracingMiddleware", "django.middleware.common.CommonMiddleware"],
-        )
+        ),
     ):
         client.get(reverse("elasticapm-no-error-slash")[:-1])
     transactions = django_elasticapm_client.events[TRANSACTION]
@@ -874,7 +874,7 @@ def test_request_metrics_301_prepend_www(django_elasticapm_client, client):
         **middleware_setting(
             django.VERSION,
             ["elasticapm.contrib.django.middleware.TracingMiddleware", "django.middleware.common.CommonMiddleware"],
-        )
+        ),
     ):
         client.get(reverse("elasticapm-no-error"))
     transactions = django_elasticapm_client.events[TRANSACTION]
@@ -1149,7 +1149,7 @@ def test_stacktraces_have_templates(client, django_elasticapm_client):
     with override_settings(
         TEMPLATE_DEBUG=TEMPLATE_DEBUG,
         TEMPLATES=TEMPLATES_copy,
-        **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"])
+        **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]),
     ):
         resp = client.get(reverse("render-heavy-template"))
     assert resp.status_code == 200
@@ -1341,7 +1341,7 @@ def test_test_exception_fails(mock_send):
     mock_send.side_effect = Exception("boom")
     with override_settings(
         ELASTIC_APM={"TRANSPORT_CLASS": "elasticapm.transport.http.Transport", "SERVICE_NAME": "testapp"},
-        **middleware_setting(django.VERSION, ["foo", "elasticapm.contrib.django.middleware.TracingMiddleware"])
+        **middleware_setting(django.VERSION, ["foo", "elasticapm.contrib.django.middleware.TracingMiddleware"]),
     ):
         call_command("elasticapm", "test", stdout=stdout, stderr=stdout)
     output = stdout.getvalue()
@@ -1491,6 +1491,21 @@ def test_capture_empty_body(client, django_elasticapm_client):
         assert error["context"]["request"]["body"] == {}
 
 
+@pytest.mark.parametrize(
+    "django_elasticapm_client",
+    [{"capture_body": "errors"}, {"capture_body": "transactions"}, {"capture_body": "all"}, {"capture_body": "off"}],
+    indirect=True,
+)
+def test_capture_long_body(client, django_elasticapm_client):
+    with pytest.raises(MyException):
+        client.post(reverse("elasticapm-raise-exc"), data={"foo": "f" * 10000})
+    error = django_elasticapm_client.events[ERROR][0]
+    if django_elasticapm_client.config.capture_body not in ("error", "all"):
+        assert error["context"]["request"]["body"] == "[REDACTED]"
+    else:
+        assert error["context"]["request"]["body"] == f'{{\'foo\': \'{"f" * 9990}' + "…"
+
+
 @pytest.mark.parametrize(
     "django_elasticapm_client",
     [{"capture_body": "errors"}, {"capture_body": "transactions"}, {"capture_body": "all"}, {"capture_body": "off"}],
@@ -1554,7 +1569,7 @@ def test_rum_tracing_context_processor(client, django_elasticapm_client):
                 },
             }
         ],
-        **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"])
+        **middleware_setting(django.VERSION, ["elasticapm.contrib.django.middleware.TracingMiddleware"]),
     ):
         response = client.get(reverse("render-heavy-template"))
         transactions = django_elasticapm_client.events[TRANSACTION]
@@ -1634,7 +1649,7 @@ def test_outcome_is_set_for_unsampled_transactions(django_elasticapm_client, cli
                 "elasticapm.contrib.django.middleware.TracingMiddleware",
                 "tests.contrib.django.testapp.middleware.SetTransactionUnsampled",
             ],
-        )
+        ),
     ):
         client.get(reverse("elasticapm-no-error"))
         transaction = django_elasticapm_client.events[TRANSACTION][0]