Repository: scoutapp/scout_apm_python

Issue:
Clean library/frameworks from backtraces
We should clean all non-user code from backtraces as much as possible.  Here's a heroku app running 2.13.0:

<img width="890" alt="Screenshot 2020-04-13 14 39 56" src="https://user-images.githubusercontent.com/102774/79160522-26a36e80-7d97-11ea-8376-f8dfcfd6ece0.png">



Discussion:
I believe this is what #416 would do. Currently we don't derive `application_root` for any frameworks except Django, and even then not in every case.

I'm not sure `application_root` is always easy to determine. Python apps come in all shapes and folder layouts. But it should be more accurate to do the opposite and filter out/de-emphasize non-app code with the heuristic that if it has `site-packages` in its file path, as that should mean it's installed from elsewhere.

I'll look into it.
@adamchainz Mind taking another look at filtering these? I think it'd be good to filter anything that contains `/site-packages/`?
@dlanderson Do we need the ability to disable this or include specific packages?
> Do we need the ability to disable this or include specific packages?

No, I don't think we need that ability at this time.

Resolving patch:
diff --git a/src/scout_apm/core/backtrace.py b/src/scout_apm/core/backtrace.py
--- a/src/scout_apm/core/backtrace.py
+++ b/src/scout_apm/core/backtrace.py
@@ -1,7 +1,9 @@
 # coding=utf-8
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import itertools
 import sys
+import sysconfig
 import traceback
 
 # Maximum non-Scout frames to target retrieving
@@ -10,23 +12,55 @@
 IGNORED = 1
 
 
+def filter_frames(frames):
+    """Filter the stack trace frames down to non-library code."""
+    paths = sysconfig.get_paths()
+    library_paths = {paths["purelib"], paths["platlib"]}
+    for frame in frames:
+        if not any(frame["file"].startswith(exclusion) for exclusion in library_paths):
+            yield frame
+
+
 if sys.version_info >= (3, 5):
 
+    def frame_walker():
+        """Iterate over each frame of the stack.
+
+        Taken from python3/traceback.ExtractSummary.extract to support
+        iterating over the entire stack, but without creating a large
+        data structure.
+        """
+        for frame, lineno in traceback.walk_stack(sys._getframe().f_back):
+            co = frame.f_code
+            filename = co.co_filename
+            name = co.co_name
+            yield {"file": filename, "line": lineno, "function": name}
+
     def capture():
-        return [
-            {"file": frame.filename, "line": frame.lineno, "function": frame.name}
-            for frame in reversed(
-                traceback.extract_stack(limit=LIMIT + IGNORED)[:-IGNORED]
-            )
-        ]
+        return list(itertools.islice(filter_frames(frame_walker()), LIMIT))
 
 
 else:
 
+    def frame_walker():
+        """Iterate over each frame of the stack.
+
+        Taken from python2.7/traceback.extract_stack to support iterating
+        over the entire stack, but without creating a large data structure.
+        """
+        try:
+            raise ZeroDivisionError
+        except ZeroDivisionError:
+            # Get the current frame
+            f = sys.exc_info()[2].tb_frame.f_back
+
+        while f is not None:
+            lineno = f.f_lineno
+            co = f.f_code
+            filename = co.co_filename
+            name = co.co_name
+            yield {"file": filename, "line": lineno, "function": name}
+            f = f.f_back
+
     def capture():
-        return [
-            {"file": frame[0], "line": frame[1], "function": frame[3]}
-            for frame in reversed(
-                traceback.extract_stack(limit=LIMIT + IGNORED)[:-IGNORED]
-            )
-        ]
+        return list(itertools.islice(filter_frames(frame_walker()), LIMIT))


Test patch:
diff --git a/tests/unit/core/test_backtrace.py b/tests/unit/core/test_backtrace.py
--- a/tests/unit/core/test_backtrace.py
+++ b/tests/unit/core/test_backtrace.py
@@ -1,7 +1,9 @@
 # coding=utf-8
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import os
 import sys
+import sysconfig
 
 from scout_apm.core import backtrace
 
@@ -37,3 +39,17 @@ def capture_recursive_bottom(limit):
 
     stack = capture_recursive_bottom(backtrace.LIMIT * 2)
     assert len(stack) == backtrace.LIMIT
+
+
+def test_filter_frames():
+    """Verify the frames from the library paths are excluded."""
+    paths = sysconfig.get_paths()
+    library_path = {paths["purelib"], paths["platlib"]}.pop()
+    frames = [
+        {"file": os.path.join(library_path, "test"), "line": 1, "function": "foo"},
+        {"file": "/valid/path", "line": 1, "function": "foo"},
+    ]
+
+    actual = list(backtrace.filter_frames(frames))
+    assert len(actual) == 1
+    assert actual[0]["file"] == "/valid/path"