Repository: scoutapp/scout_apm_python

Issue:
Total span limit
For *really* long running background jobs it'd be possible to have a tracked request with so many spans it causes memory problems. Following #471 we should probably have a (very high) total cap on the number of captured spans.


Discussion:

Resolving patch:
diff --git a/src/scout_apm/core/tracked_request.py b/src/scout_apm/core/tracked_request.py
--- a/src/scout_apm/core/tracked_request.py
+++ b/src/scout_apm/core/tracked_request.py
@@ -32,8 +32,12 @@ class TrackedRequest(object):
         "is_real_request",
         "_memory_start",
         "n_plus_one_tracker",
+        "hit_max",
     )
 
+    # Stop adding new spans at this point, to avoid exhausting memory
+    MAX_COMPLETE_SPANS = 1500
+
     @classmethod
     def instance(cls):
         from scout_apm.core.context import context
@@ -50,6 +54,7 @@ def __init__(self):
         self.is_real_request = False
         self._memory_start = get_rss_in_mb()
         self.n_plus_one_tracker = NPlusOneTracker()
+        self.hit_max = False
         logger.debug("Starting request: %s", self.request_id)
 
     def __repr__(self):
@@ -83,6 +88,15 @@ def start_span(
         else:
             parent_id = None
 
+        if len(self.complete_spans) >= self.MAX_COMPLETE_SPANS:
+            if not self.hit_max:
+                logger.warning(
+                    "Hit the maximum number of spans, this trace will be incomplete."
+                )
+                self.hit_max = True
+            ignore = True
+            ignore_children = True
+
         new_span = Span(
             request_id=self.request_id,
             operation=operation,


Test patch:
diff --git a/tests/unit/core/test_tracked_request.py b/tests/unit/core/test_tracked_request.py
--- a/tests/unit/core/test_tracked_request.py
+++ b/tests/unit/core/test_tracked_request.py
@@ -2,10 +2,12 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import datetime as dt
+import logging
 import sys
 
 from scout_apm.core import objtrace
 from scout_apm.core.tracked_request import TrackedRequest
+from tests.compat import mock
 from tests.tools import skip_if_objtrace_is_extension, skip_if_objtrace_not_extension
 
 
@@ -121,6 +123,24 @@ def test_start_span_ignores_children(tracked_request):
     assert "parent" == tracked_request.complete_spans[0].operation
 
 
+@mock.patch("scout_apm.core.tracked_request.TrackedRequest.MAX_COMPLETE_SPANS", new=1)
+def test_start_span_at_max_ignores_span(caplog, tracked_request):
+    tracked_request.start_span(operation="parent")
+    tracked_request.start_span(operation="child1")
+    tracked_request.stop_span()
+    child2 = tracked_request.start_span(operation="child2")
+
+    assert child2.ignore
+    assert child2.ignore_children
+    assert caplog.record_tuples == [
+        (
+            "scout_apm.core.tracked_request",
+            logging.WARNING,
+            "Hit the maximum number of spans, this trace will be incomplete.",
+        )
+    ]
+
+
 def test_span_captures_backtrace(tracked_request):
     span = tracked_request.start_span(operation="Sql/Work")
     # Pretend it was started 1 second ago