Repository: elastic/apm-agent-python

Issue:
APM agent crashes if the SQL query is too large
**Describe the bug**: 

When running in celery (I guess it happens elsewhere also) if the query is too large APM server will reject the event. 

```
TransportException: HTTP 400: {"accepted":0,"errors":[{"message":"event exceeded the permitted size.","document":"{\"span\": {\"id\": \"332c20cdb36c0b03\", \"transaction_id\": \"26dfc86436ea1c9c\", \"trace_id\": \"8b3a7bd7e830fe4800bd6ff98af24ecd\", \"parent_id\": \"26dfc86436ea1c9c\", \"name\": ... MY_DATABASE_FIELDS_HERE... ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::integer[], %s::text[], %s, %s, %s, %s, %s, %s, %s, %s, %s::varchar(100)[], %s, %s, %s, %s, %s::integer[], %s, %s, %s, %s, %s, %s, %s, %s, %s, %s), (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::integer[], %s::text[], %s, %s, %s, %s, %s, %s, %s, %s, %s::varchar(100)[], %s, %s, %s, %s, %s::integer[], %s, %s, %s, %s, %s, %s, %s, %s, %s, %s), (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::integer[], %s::text[], %s, %s, %s, %s, %s, %s, %s, %s, %s::varchar(100)[], %s, %s, %s, %s, %s::integer[], %s, %s, %s, %s, %s, %s, %s, %s, %s, %s), (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::integer[], %s::text[], %s, %s, %s, %s, %s, %s, %s, %s, %s::varchar(100)[], %s, %s, %s, %s, %s::integer[], %s, %s, %s, %s, %s, %s, %s, %s, %s, %s), (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::integer[], %s::text[], %s, %s, %s, %s, %s, %s, %s, %s, %s::varchar(100)[], %s, %s, %s, %s, %s::integer[], %s, %s, %s, %s, %s, %s, %s, %s, %s, %s), (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::integer[], %s::text[], %s, %s, %s, %s, %s, %s, %s, %s, %s::varchar(100)[], %s, %s, %s, %s, %s::integer[], %s, %s, %s, %s, %s, %s, %s, %s, %s, %s), (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::integer[], %s::text[], %s, %s, %s, %s, %s, %s, %s, %s, %s::varchar(100)[], %s, %s, %s, %s, %s::integer[], %s, %s, %s, %s, %s, %s, %s, %s, %s, %s), (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::integer[], %s::text[], %s, %s, %s, %s, %s, %s, %s, %s, %s::var...
  File "elasticapm/transport/base.py", line 227, in _flush
    self.send(data)
  File "elasticapm/transport/http.py", line 105, in send
    raise TransportException(message, data, print_trace=print_trace)
```

**To Reproduce**

1. Run a bulk_create in Django with a large list. 
In our code, we're creating 450k rows in a task at night.


**Environment (please complete the following information)**
- OS: Ubuntu 18.04
- Python version: 3.6
- Framework and version: Django 2.2
- APM Server version: 5.6.0
- Agent version: 7.6.0




Discussion:
Thanks for opening this. We'll get this fixed.

Resolving patch:
diff --git a/elasticapm/instrumentation/packages/dbapi2.py b/elasticapm/instrumentation/packages/dbapi2.py
--- a/elasticapm/instrumentation/packages/dbapi2.py
+++ b/elasticapm/instrumentation/packages/dbapi2.py
@@ -38,7 +38,7 @@
 from elasticapm.instrumentation.packages.base import AbstractInstrumentedModule
 from elasticapm.traces import capture_span
 from elasticapm.utils import compat, wrapt
-from elasticapm.utils.encoding import force_text
+from elasticapm.utils.encoding import force_text, shorten
 
 
 class Literal(object):
@@ -225,6 +225,11 @@ def _trace_sql(self, method, sql, params, action=QUERY_ACTION):
             signature = sql_string + "()"
         else:
             signature = self.extract_signature(sql_string)
+
+        # Truncate sql_string to 10000 characters to prevent large queries from
+        # causing an error to APM server.
+        sql_string = shorten(sql_string, string_length=10000)
+
         with capture_span(
             signature,
             span_type="db",
diff --git a/elasticapm/instrumentation/packages/sqlite.py b/elasticapm/instrumentation/packages/sqlite.py
--- a/elasticapm/instrumentation/packages/sqlite.py
+++ b/elasticapm/instrumentation/packages/sqlite.py
@@ -35,6 +35,7 @@
     extract_signature,
 )
 from elasticapm.traces import capture_span
+from elasticapm.utils.encoding import shorten
 
 
 class SQLiteCursorProxy(CursorProxy):
@@ -52,12 +53,13 @@ class SQLiteConnectionProxy(ConnectionProxy):
 
     def _trace_sql(self, method, sql, params):
         signature = extract_signature(sql)
+        sql_string = shorten(sql, string_length=10000)
         with capture_span(
             signature,
             span_type="db",
             span_subtype="sqlite",
             span_action="query",
-            extra={"db": {"type": "sql", "statement": sql}},
+            extra={"db": {"type": "sql", "statement": sql_string}},
         ):
             if params is None:
                 return method(sql)
diff --git a/elasticapm/utils/encoding.py b/elasticapm/utils/encoding.py
--- a/elasticapm/utils/encoding.py
+++ b/elasticapm/utils/encoding.py
@@ -218,6 +218,10 @@ def shorten(var, list_length=50, string_length=200, dict_length=50):
 
 
 def keyword_field(string):
+    """
+    If the given string is longer than KEYWORD_MAX_LENGTH, truncate it to
+    KEYWORD_MAX_LENGTH-1, adding the "…" character at the end.
+    """
     if not isinstance(string, compat.string_types) or len(string) <= KEYWORD_MAX_LENGTH:
         return string
     return string[: KEYWORD_MAX_LENGTH - 1] + u"…"


Test patch:
diff --git a/tests/instrumentation/sqlite_tests.py b/tests/instrumentation/sqlite_tests.py
--- a/tests/instrumentation/sqlite_tests.py
+++ b/tests/instrumentation/sqlite_tests.py
@@ -125,3 +125,20 @@ def test_nonstandard_connection_execute(instrument, elasticapm_client):
     assert spans[3]["action"] == "query"
 
     assert len(spans) == 4
+
+
+def test_truncate_long_sql(instrument, elasticapm_client):
+    conn = sqlite3.connect(":memory:")
+
+    conn.execute("CREATE TABLE testdb (id integer, username text)")
+    conn.execute('INSERT INTO testdb VALUES (1, "Ron")')
+    elasticapm_client.begin_transaction("transaction.test")
+    conn.execute("""SELECT id, username FROM testdb WHERE username = "%s";""" % ("x" * 10010))
+    elasticapm_client.end_transaction("MyView")
+    conn.execute("DROP TABLE testdb")
+
+    transactions = elasticapm_client.events[TRANSACTION]
+    spans = elasticapm_client.spans_for_transaction(transactions[0])
+
+    assert len(spans[0]["context"]["db"]["statement"]) == 10000
+    assert spans[0]["context"]["db"]["statement"].endswith("...")