Repository: scoutapp/scout_apm_python
Pull request #404 (2019-11-27 09:23:27)

Stop CoreAgentSocketThread after every test
Should prevent random errors which seem to happen fairly frequently in Travis runs like:

```
______________________________ test_launch_error _______________________________

caplog = <_pytest.logging.LogCaptureFixture object at 0x7f44d4c65f60>

core_agent_manager = <scout_apm.core.core_agent_manager.CoreAgentManager object at 0x7f44d4c65940>

    def test_launch_error(caplog, core_agent_manager):
        caplog.set_level(logging.ERROR)
        exception = ValueError("Hello Fail")
        with mock.patch(
            "scout_apm.core.core_agent_manager.CoreAgentManager.agent_binary",
            side_effect=exception,
        ):
            result = core_agent_manager.launch()

        assert not result
        assert not is_running(core_agent_manager)
>       assert caplog.record_tuples == [
            ("scout_apm.core.core_agent_manager", logging.ERROR, "Error running Core Agent")
        ]
E       assert [('scout_apm.... Core Agent')] == [('scout_apm.... Core Agent')]
E         At index 0 diff: ('scout_apm.core.socket', 10, 'CoreAgentSocket attempt 2, connecting to /tmp/scout_apm_core/scout_apm_core-v1.2.4-x86_64-unknown-linux-gnu/scout-agent.sock, PID: 4215, Thread: <CoreAgentSocket(Thread-1, started daemon 139933644363520)>') != ('scout_apm.core.core_agent_manager', 40, 'Error running Core Agent')
E         Left contains 2 more items, first extra item: ('scout_apm.core.socket', 10, "CoreAgentSocket connection error: FileNotFoundError(2, 'No such file or directory')")
E         Use -v to get the full diff
```

Also catch any `Full` exceptions when trying to `Queue.put()` to stop, since they are only there to unblock a waiting consumer which will happen when the queue is empty.

adamchainz: The socket still gets started all over the test suite as well so still seeing similar failures.
adamchainz: Rewritten post #450 

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0b06f15..38e604c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,8 @@
   [PR #460](https://github.com/scoutapp/scout_apm_python/pull/460)).
 - Add Hug integration
   ([PR #460](https://github.com/scoutapp/scout_apm_python/pull/460)).
+- Limit size of recorded tracebacks to reduce memory usage
+  ([PR #467](https://github.com/scoutapp/scout_apm_python/pull/467)).
 
 ### Fixed
 
diff --git a/src/scout_apm/core/backtrace.py b/src/scout_apm/core/backtrace.py
index 4293850..dee91d2 100644
--- a/src/scout_apm/core/backtrace.py
+++ b/src/scout_apm/core/backtrace.py
@@ -4,12 +4,20 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 import sys
 import traceback
 
+# Maximum non-Scout frames to target retrieving
+LIMIT = 50
+# How many upper frames from inside Scout to ignore
+IGNORED = 1
+
+
 if sys.version_info >= (3, 5):
 
     def capture():
         return [
             {"file": frame.filename, "line": frame.lineno, "function": frame.name}
-            for frame in reversed(traceback.extract_stack()[:-1])
+            for frame in reversed(
+                traceback.extract_stack(limit=LIMIT + IGNORED)[:-IGNORED]
+            )
         ]
 
 
@@ -18,5 +26,7 @@ else:
     def capture():
         return [
             {"file": frame[0], "line": frame[1], "function": frame[3]}
-            for frame in reversed(traceback.extract_stack()[:-1])
+            for frame in reversed(
+                traceback.extract_stack(limit=LIMIT + IGNORED)[:-IGNORED]
+            )
         ]
diff --git a/src/scout_apm/core/socket.py b/src/scout_apm/core/socket.py
index 0efd429..bf424b5 100644
--- a/src/scout_apm/core/socket.py
+++ b/src/scout_apm/core/socket.py
@@ -29,7 +29,10 @@ class CoreAgentSocketThread(SingletonThread):
     def _on_stop(cls):
         super(CoreAgentSocketThread, cls)._on_stop()
         # Unblock _command_queue.get()
-        cls._command_queue.put(None, False)
+        try:
+            cls._command_queue.put(None, False)
+        except queue.Full:
+            pass
 
     @classmethod
     def send(cls, command):


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index c770baf..2ecb225 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -13,6 +13,7 @@ from webtest import TestApp
 from scout_apm.core import socket as scout_apm_core_socket
 from scout_apm.core.config import SCOUT_PYTHON_VALUES, scout_config
 from scout_apm.core.core_agent_manager import CoreAgentManager
+from scout_apm.core.socket import CoreAgentSocketThread
 from scout_apm.core.tracked_request import TrackedRequest
 from tests.compat import TemporaryDirectory
 
@@ -157,6 +158,12 @@ def short_timeouts():
         scout_apm_core_socket.SECOND = 1
 
 
+@pytest.fixture(autouse=True)
+def auto_stop_core_agent_socket():
+    yield
+    CoreAgentSocketThread.ensure_stopped()
+
+
 @pytest.fixture
 def tracked_request():
     """
diff --git a/tests/integration/core/test_socket.py b/tests/integration/core/test_socket.py
index 11d4c02..21bbb94 100644
--- a/tests/integration/core/test_socket.py
+++ b/tests/integration/core/test_socket.py
@@ -26,13 +26,11 @@ def running_agent(core_agent_manager):
 @pytest.fixture
 def socket(running_agent):
     socket = CoreAgentSocketThread.ensure_started()
-    try:
-        # Wait for socket to connect and register:
-        time.sleep(0.01)
+    # Wait for socket to connect and register:
+    time.sleep(0.01)
 
-        yield socket
-    finally:
-        CoreAgentSocketThread.ensure_stopped()
+    yield socket
+    # ensure_stopped() already called by global auto_stop_core_agent_socket
 
 
 class Command(object):
diff --git a/tests/unit/core/test_backtrace.py b/tests/unit/core/test_backtrace.py
index 2c97066..6d3dfd7 100644
--- a/tests/unit/core/test_backtrace.py
+++ b/tests/unit/core/test_backtrace.py
@@ -26,3 +26,14 @@ def format_py_filename(filename):
         # Python 2 will include .pyc filename if it's used, so strip that
         return filename[:-1]
     return filename
+
+
+def test_capture_limit():
+    def capture_recursive_bottom(limit):
+        if limit <= 1:
+            return backtrace.capture()
+        else:
+            return capture_recursive_bottom(limit - 1)
+
+    stack = capture_recursive_bottom(backtrace.LIMIT * 2)
+    assert len(stack) == backtrace.LIMIT