Repository: scoutapp/scout_apm_python

Issue:
Handle SCM_SUBDIRECTORY configuration in error service
This is normally handled by the core-agent, but the error service doesn't use the core-agent


Discussion:
@jrothrock can you explain what the core-agent does that the python error service would need to do?

Resolving patch:
diff --git a/src/scout_apm/core/backtrace.py b/src/scout_apm/core/backtrace.py
--- a/src/scout_apm/core/backtrace.py
+++ b/src/scout_apm/core/backtrace.py
@@ -55,7 +55,7 @@ def module_filepath(module, filepath):
     return filepath.split(module_dir, 1)[-1].lstrip(os.sep) if module_dir else filepath
 
 
-def filepath(frame):
+def filepaths(frame):
     """Get the filepath for frame."""
     module = frame.f_globals.get("__name__", None)
     filepath = frame.f_code.co_filename
@@ -65,7 +65,7 @@ def filepath(frame):
 
     if not module:
         return filepath
-    return module_filepath(module, filepath)
+    return filepath, module_filepath(module, filepath)
 
 
 if sys.version_info >= (3, 5):
@@ -74,7 +74,13 @@ def stacktrace_walker(tb):
         """Iterate over each frame of the stack downards for exceptions."""
         for frame, lineno in traceback.walk_tb(tb):
             name = frame.f_code.co_name
-            yield {"file": filepath(frame), "line": lineno, "function": name}
+            full_path, relative_path = filepaths(frame)
+            yield {
+                "file": relative_path,
+                "full_path": full_path,
+                "line": lineno,
+                "function": name,
+            }
 
     def backtrace_walker():
         """Iterate over each frame of the stack upwards.
@@ -86,7 +92,13 @@ def backtrace_walker():
         start_frame = sys._getframe().f_back
         for frame, lineno in traceback.walk_stack(start_frame):
             name = frame.f_code.co_name
-            yield {"file": filepath(frame), "line": lineno, "function": name}
+            full_path, relative_path = filepaths(frame)
+            yield {
+                "file": relative_path,
+                "full_path": full_path,
+                "line": lineno,
+                "function": name,
+            }
 
 else:
 
@@ -95,8 +107,10 @@ def stacktrace_walker(tb):
         while tb is not None:
             lineno = tb.tb_lineno
             name = tb.tb_frame.f_code.co_name
+            full_path, relative_path = filepaths(tb.tb_frame)
             yield {
-                "file": filepath(tb.tb_frame),
+                "file": relative_path,
+                "full_path": full_path,
                 "line": lineno,
                 "function": name,
             }
@@ -117,7 +131,13 @@ def backtrace_walker():
         while frame is not None:
             lineno = frame.f_lineno
             name = frame.f_code.co_name
-            yield {"file": filepath(frame), "line": lineno, "function": name}
+            full_path, relative_path = filepaths(frame)
+            yield {
+                "file": relative_path,
+                "full_path": full_path,
+                "line": lineno,
+                "function": name,
+            }
             frame = frame.f_back
 
 
diff --git a/src/scout_apm/core/error.py b/src/scout_apm/core/error.py
--- a/src/scout_apm/core/error.py
+++ b/src/scout_apm/core/error.py
@@ -2,6 +2,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import logging
+import os
 import sys
 
 from scout_apm.core.backtrace import capture_stacktrace
@@ -52,6 +53,7 @@ def send(
                     module=None, controller=custom_controller, action=None
                 )
 
+        scm_subdirectory = scout_config.value("scm_subdirectory")
         error = {
             "exception_class": exc_class.__name__,
             "message": text_type(exc_value),
@@ -64,7 +66,9 @@ def send(
             "environment": filter_element("", environment) if environment else None,
             "trace": [
                 "{file}:{line}:in {function}".format(
-                    file=frame["file"],
+                    file=os.path.join(scm_subdirectory, frame["file"])
+                    if scm_subdirectory
+                    else frame["file"],
                     line=frame["line"],
                     function=frame["function"],
                 )
diff --git a/src/scout_apm/core/tracked_request.py b/src/scout_apm/core/tracked_request.py
--- a/src/scout_apm/core/tracked_request.py
+++ b/src/scout_apm/core/tracked_request.py
@@ -298,4 +298,15 @@ def add_allocation_tags(self):
         self.tag("stop_allocations", end_allocs)
 
     def capture_backtrace(self):
-        self.tag("stack", backtrace.capture_backtrace())
+        # The core-agent will trim the full_path as necessary.
+        self.tag(
+            "stack",
+            [
+                {
+                    "file": frame["full_path"],
+                    "line": frame["line"],
+                    "function": frame["function"],
+                }
+                for frame in backtrace.capture_backtrace()
+            ],
+        )


Test patch:
diff --git a/tests/unit/core/test_backtrace.py b/tests/unit/core/test_backtrace.py
--- a/tests/unit/core/test_backtrace.py
+++ b/tests/unit/core/test_backtrace.py
@@ -26,16 +26,20 @@ def test_capture_backtrace():
     assert len(stack) >= 1
     for frame in stack:
         assert isinstance(frame, dict)
-        assert set(frame.keys()) == {"file", "line", "function"}
+        assert set(frame.keys()) == {"file", "full_path", "line", "function"}
         assert isinstance(frame["file"], str)
+        assert isinstance(frame["full_path"], str)
         assert isinstance(frame["line"], int)
         assert isinstance(frame["function"], str)
 
     assert stack[0]["file"] == "scout_apm/core/backtrace.py"
+    assert stack[0]["full_path"].endswith("/scout_apm/core/backtrace.py")
     assert stack[0]["function"] == "filter_frames"
     assert stack[1]["file"] == "scout_apm/core/backtrace.py"
+    assert stack[1]["full_path"].endswith("/scout_apm/core/backtrace.py")
     assert stack[1]["function"] == "capture_backtrace"
     assert stack[2]["file"] == "tests/unit/core/test_backtrace.py"
+    assert stack[2]["full_path"].endswith("/tests/unit/core/test_backtrace.py")
     assert stack[2]["function"] == "test_capture_backtrace"
 
 
@@ -56,12 +60,12 @@ def test_filter_frames():
     library_path = {paths["purelib"], paths["platlib"]}.pop()
     frames = [
         {"file": os.path.join(library_path, "test"), "line": 1, "function": "foo"},
-        {"file": "/valid/path", "line": 1, "function": "foo"},
+        {"file": "tests/unit/core/test_backtrace.py", "line": 1, "function": "foo"},
     ]
 
     actual = list(backtrace.filter_frames(frames))
     assert len(actual) == 1
-    assert actual[0]["file"] == "/valid/path"
+    assert actual[0]["file"] == "tests/unit/core/test_backtrace.py"
 
 
 def test_capture_stacktrace():
@@ -70,12 +74,14 @@ def test_capture_stacktrace():
     assert len(stack) == 1
     for frame in stack:
         assert isinstance(frame, dict)
-        assert set(frame.keys()) == {"file", "line", "function"}
+        assert set(frame.keys()) == {"file", "full_path", "line", "function"}
         assert isinstance(frame["file"], str)
+        assert isinstance(frame["full_path"], str)
         assert isinstance(frame["line"], int)
         assert isinstance(frame["function"], str)
 
     assert stack[0]["file"] == "tests/unit/core/test_backtrace.py"
+    assert stack[0]["full_path"].endswith("/tests/unit/core/test_backtrace.py")
     assert stack[0]["function"] == "get_tb"
 
 
@@ -132,5 +138,5 @@ def test_module_filepath_error_flows(error_module):
     module = frame.f_globals["__name__"]
     filepath = frame.f_code.co_filename
     full_path = backtrace.module_filepath(module, filepath)
-    assert full_path != "tests/unit/core/test_backtrace.py"
-    assert full_path.endswith("tests/unit/core/test_backtrace.py")
+    assert full_path != "/tests/unit/core/test_backtrace.py"
+    assert full_path.endswith("/tests/unit/core/test_backtrace.py")
diff --git a/tests/unit/core/test_error.py b/tests/unit/core/test_error.py
--- a/tests/unit/core/test_error.py
+++ b/tests/unit/core/test_error.py
@@ -2,6 +2,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import logging
+import os
 import sys
 from contextlib import contextmanager
 
@@ -54,7 +55,7 @@ def test_monitor_ignore_exceptions(error_monitor_errors):
 
 @pytest.mark.parametrize(
     "path, params, session, environment, request_components, "
-    "custom_controller, custom_params, expected_error",
+    "custom_controller, scm_subdirectory, custom_params, expected_error",
     [
         (
             "/test/",
@@ -63,6 +64,7 @@ def test_monitor_ignore_exceptions(error_monitor_errors):
             None,
             None,
             None,
+            "scm",
             None,
             {
                 "exception_class": "ZeroDivisionError",
@@ -86,6 +88,7 @@ def test_monitor_ignore_exceptions(error_monitor_errors):
             None,
             "test-controller",
             None,
+            None,
             {
                 "exception_class": "ZeroDivisionError",
                 "message": "division by zero",
@@ -111,6 +114,7 @@ def test_monitor_ignore_exceptions(error_monitor_errors):
             {"PASSWORD": "hunter2"},
             RequestComponents("sample.app", "DataView", "detail"),
             None,
+            "scm",
             None,
             {
                 "exception_class": "ZeroDivisionError",
@@ -137,6 +141,7 @@ def test_monitor_ignore_exceptions(error_monitor_errors):
             {"PASSWORD": "hunter2"},
             RequestComponents("sample.app", "DataView", "detail"),
             "test-controller",
+            "scm",
             {"baz": 3},
             {
                 "exception_class": "ZeroDivisionError",
@@ -165,13 +170,14 @@ def test_monitor(
     environment,
     request_components,
     custom_controller,
+    scm_subdirectory,
     custom_params,
     expected_error,
     tracked_request,
     error_monitor_errors,
     caplog,
 ):
-    with app_with_scout():
+    with app_with_scout(scout_config={"scm_subdirectory": scm_subdirectory}):
         tracked_request.request_id = "sample_id"
         tracked_request.tags["spam"] = "foo"
         exc_info = 0
@@ -194,7 +200,10 @@ def test_monitor(
     error = error_monitor_errors[0]
     # Remove the trace from the error as it bloats the test.
     filepath, line, func_str = error.pop("trace")[0].split(":")
-    assert filepath.endswith("tests/unit/core/test_error.py")
+    expected_filepath = "tests/unit/core/test_error.py"
+    if scm_subdirectory:
+        expected_filepath = os.path.join(scm_subdirectory, expected_filepath)
+    assert filepath.endswith(expected_filepath)
     # The line number changes between python versions. Make sure it's not empty.
     assert line
     assert func_str == "in test_monitor"
@@ -239,7 +248,7 @@ def test_monitor_with_logged_payload(
     assert "ZeroDivisionError" in actual_message
     assert "division by zero" in actual_message
     assert (
-        "tests/unit/core/test_error.py:219:in test_monitor_with_logged_payload"
+        "tests/unit/core/test_error.py:228:in test_monitor_with_logged_payload"
         in actual_message
     )
     assert "sample.app" in actual_message