Repository: scoutapp/scout_apm_python (python)

Issue:
Clean library/frameworks from backtraces
We should clean all non-user code from backtraces as much as possible.  Here's a heroku app running 2.13.0:

<img width="890" alt="Screenshot 2020-04-13 14 39 56" src="https://user-images.githubusercontent.com/102774/79160522-26a36e80-7d97-11ea-8376-f8dfcfd6ece0.png">


Pull request description:
Exclude python library paths from the backtrace.
Closes #514 

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 526a2b9..b2bd468 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,10 +3,13 @@
 ## Pending
 
 ### Added
-
 - Tested with Django 3.2. Only define ``default_app_config`` when using
   a version of Django earlier than 3.2.
 
+### Fixed
+- Exclude python library paths from backtraces.
+  ([PR #514](https://github.com/scoutapp/scout_apm_python/issues/514))
+
 ## [2.18.0] 2021-02-09
 
 ### Added
diff --git a/src/scout_apm/core/backtrace.py b/src/scout_apm/core/backtrace.py
index dee91d2..5551064 100644
--- a/src/scout_apm/core/backtrace.py
+++ b/src/scout_apm/core/backtrace.py
@@ -1,7 +1,9 @@
 # coding=utf-8
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import itertools
 import sys
+import sysconfig
 import traceback
 
 # Maximum non-Scout frames to target retrieving
@@ -10,23 +12,55 @@ LIMIT = 50
 IGNORED = 1
 
 
+def filter_frames(frames):
+    """Filter the stack trace frames down to non-library code."""
+    paths = sysconfig.get_paths()
+    library_paths = {paths["purelib"], paths["platlib"]}
+    for frame in frames:
+        if not any(frame["file"].startswith(exclusion) for exclusion in library_paths):
+            yield frame
+
+
 if sys.version_info >= (3, 5):
 
+    def frame_walker():
+        """Iterate over each frame of the stack.
+
+        Taken from python3/traceback.ExtractSummary.extract to support
+        iterating over the entire stack, but without creating a large
+        data structure.
+        """
+        for frame, lineno in traceback.walk_stack(sys._getframe().f_back):
+            co = frame.f_code
+            filename = co.co_filename
+            name = co.co_name
+            yield {"file": filename, "line": lineno, "function": name}
+
     def capture():
-        return [
-            {"file": frame.filename, "line": frame.lineno, "function": frame.name}
-            for frame in reversed(
-                traceback.extract_stack(limit=LIMIT + IGNORED)[:-IGNORED]
-            )
-        ]
+        return list(itertools.islice(filter_frames(frame_walker()), LIMIT))
 
 
 else:
 
+    def frame_walker():
+        """Iterate over each frame of the stack.
+
+        Taken from python2.7/traceback.extract_stack to support iterating
+        over the entire stack, but without creating a large data structure.
+        """
+        try:
+            raise ZeroDivisionError
+        except ZeroDivisionError:
+            # Get the current frame
+            f = sys.exc_info()[2].tb_frame.f_back
+
+        while f is not None:
+            lineno = f.f_lineno
+            co = f.f_code
+            filename = co.co_filename
+            name = co.co_name
+            yield {"file": filename, "line": lineno, "function": name}
+            f = f.f_back
+
     def capture():
-        return [
-            {"file": frame[0], "line": frame[1], "function": frame[3]}
-            for frame in reversed(
-                traceback.extract_stack(limit=LIMIT + IGNORED)[:-IGNORED]
-            )
-        ]
+        return list(itertools.islice(filter_frames(frame_walker()), LIMIT))
diff --git a/tox.ini b/tox.ini
index 0f50b5a..7261c91 100644
--- a/tox.ini
+++ b/tox.ini
@@ -66,7 +66,8 @@ deps =
     requests
     rq
     six
-    sqlalchemy
+    sqlalchemy ; python_version != "3.4"
+    sqlalchemy<1.3.24 ; python_version == "3.4"
     starlette ; python_version >= "3.6"
     webtest
 commands =


Test patch:
diff --git a/tests/unit/core/test_backtrace.py b/tests/unit/core/test_backtrace.py
index 6d3dfd7..c59e5b5 100644
--- a/tests/unit/core/test_backtrace.py
+++ b/tests/unit/core/test_backtrace.py
@@ -1,7 +1,9 @@
 # coding=utf-8
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import os
 import sys
+import sysconfig
 
 from scout_apm.core import backtrace
 
@@ -37,3 +39,17 @@ def test_capture_limit():
 
     stack = capture_recursive_bottom(backtrace.LIMIT * 2)
     assert len(stack) == backtrace.LIMIT
+
+
+def test_filter_frames():
+    """Verify the frames from the library paths are excluded."""
+    paths = sysconfig.get_paths()
+    library_path = {paths["purelib"], paths["platlib"]}.pop()
+    frames = [
+        {"file": os.path.join(library_path, "test"), "line": 1, "function": "foo"},
+        {"file": "/valid/path", "line": 1, "function": "foo"},
+    ]
+
+    actual = list(backtrace.filter_frames(frames))
+    assert len(actual) == 1
+    assert actual[0]["file"] == "/valid/path"