Repository: scoutapp/scout_apm_python (python)

Issue:
TypeError: '_NamespacePath' object is not subscriptable
Looks like it was related to this change: https://github.com/scoutapp/scout_apm_python/pull/668/files

I'm not entirely sure what causes this. 
We're also using a custom integration, incase thats a factor.

The particular module it is trying to load (when we get the exception) is `repoze.who.middleware`

We've not made changes to our integration, and has been breaking since, 2.21, but is also happening on 2.23.2

Below is a stack trace, starting from when we run `.install()`

Please let me know if theres anything else I can provide, I'm not sure how to debug this further

```
scout_apm.api.install(config=scout_config)
File "/usr/local/lib/python3.7/site-packages/scout_apm/compat.py", line 142, in wrapper
 return func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/scout_apm/core/__init__.py", line 45, in install
 launched = CoreAgentManager().launch()
File "/usr/local/lib/python3.7/site-packages/scout_apm/core/agent/manager.py", line 49, in launch
 self.download()
File "/usr/local/lib/python3.7/site-packages/scout_apm/core/agent/manager.py", line 58, in download
 self.downloader.download()
File "/usr/local/lib/python3.7/site-packages/scout_apm/core/agent/manager.py", line 161, in download
 downloaded = self.download_package()
File "/usr/local/lib/python3.7/site-packages/scout_apm/core/agent/manager.py", line 209, in download_package
 "GET", full_url, preload_content=False, timeout=10.0, retries=3
File "/usr/local/lib/python3.7/site-packages/urllib3/request.py", line 75, in request
 method, url, fields=fields, headers=headers, **urlopen_kw
File "/usr/local/lib/python3.7/site-packages/urllib3/request.py", line 96, in request_encode_url
 return self.urlopen(method, url, **extra_kw)
File "/usr/local/lib/python3.7/site-packages/urllib3/poolmanager.py", line 375, in urlopen
 response = conn.urlopen(method, u.request_uri, **kw)
File "/usr/local/lib/python3.7/site-packages/scout_apm/instruments/urllib3.py", line 68, in wrapped_urlopen
 return wrapped(*args, **kwargs)
File "/usr/local/lib/python3.7/contextlib.py", line 119, in __exit__
 next(self.gen)
File "/usr/local/lib/python3.7/site-packages/scout_apm/core/tracked_request.py", line 134, in span
 self.stop_span()
File "/usr/local/lib/python3.7/site-packages/scout_apm/core/tracked_request.py", line 122, in stop_span
 stopping_span.annotate()
File "/usr/local/lib/python3.7/site-packages/scout_apm/core/tracked_request.py", line 258, in annotate
 self.capture_backtrace()
File "/usr/local/lib/python3.7/site-packages/scout_apm/core/tracked_request.py", line 293, in capture_backtrace
 self.tag("stack", backtrace.capture_backtrace())
File "/usr/local/lib/python3.7/site-packages/scout_apm/core/backtrace.py", line 122, in capture_backtrace
 return list(itertools.islice(walker, LIMIT))
File "/usr/local/lib/python3.7/site-packages/scout_apm/core/backtrace.py", line 24, in filter_frames
 for frame in frames:
File "/usr/local/lib/python3.7/site-packages/scout_apm/core/backtrace.py", line 84, in backtrace_walker
 yield {"file": filepath(frame), "line": lineno, "function": name}
File "/usr/local/lib/python3.7/site-packages/scout_apm/core/backtrace.py", line 63, in filepath
 return module_filepath(module, filepath)
File "/usr/local/lib/python3.7/site-packages/scout_apm/core/backtrace.py", line 40, in module_filepath
 module_dir = root_module.__path__[0].rsplit(os.sep, 1)[0]
TypeError: '_NamespacePath' object is not subscriptable
```

Pull request:
Make module_filepath more robust for unexpected results.
Skip handling _NamespacePath for now as it's an inner class for importlib.

Closes #691 

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index df6b517..c8fb63b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,8 @@
 - Added support for Django 4.0 and Python 3.10.
 
 ### Fixed
+- Make `module_filepath` more robust avoiding throwing exceptions.
+  ([Issue 691](https://github.com/scoutapp/scout_apm_python/issues/691))
 
 ## [2.23.2] 2021-10-04
 
diff --git a/src/scout_apm/core/backtrace.py b/src/scout_apm/core/backtrace.py
index b36bead..7ccf797 100644
--- a/src/scout_apm/core/backtrace.py
+++ b/src/scout_apm/core/backtrace.py
@@ -32,22 +32,27 @@ def module_filepath(module, filepath):
     if root_module_name == module:
         return os.path.basename(filepath)
 
+    module_dir = None
     root_module = sys.modules[root_module_name]
-    if root_module.__file__:
-        module_dir = root_module.__file__.rsplit(os.sep, 2)[0]
-    elif root_module.__path__:
-        # Default to using the first path specified for the module.
-        module_dir = root_module.__path__[0].rsplit(os.sep, 1)[0]
-        if len(root_module.__path__) > 1:
-            logger.debug(
-                "{} has {} paths. Use the first and ignore the rest.".format(
-                    root_module, len(root_module.__path__)
+    try:
+        if root_module.__file__:
+            module_dir = root_module.__file__.rsplit(os.sep, 2)[0]
+        elif root_module.__path__ and isinstance(root_module.__path__, (list, tuple)):
+            # Default to using the first path specified for the module.
+            module_dir = root_module.__path__[0].rsplit(os.sep, 1)[0]
+            if len(root_module.__path__) > 1:
+                logger.debug(
+                    "{} has {} paths. Use the first and ignore the rest.".format(
+                        root_module, len(root_module.__path__)
+                    )
                 )
-            )
-    else:
-        # If the file path don't exist, then return the full path.
-        return filepath
-    return filepath.split(module_dir, 1)[-1].lstrip(os.sep)
+    except Exception as exc:
+        logger.debug(
+            "Error processing module {} and filepath {}".format(root_module, filepath),
+            exc_info=exc,
+        )
+
+    return filepath.split(module_dir, 1)[-1].lstrip(os.sep) if module_dir else filepath
 
 
 def filepath(frame):


Test patch:
diff --git a/tests/unit/core/test_backtrace.py b/tests/unit/core/test_backtrace.py
index 682a0d0..85fe87f 100644
--- a/tests/unit/core/test_backtrace.py
+++ b/tests/unit/core/test_backtrace.py
@@ -107,18 +107,27 @@ def test_module_filepath_with_namespace(test_package_as_namespace_package):
     )
 
 
-@pytest.fixture
-def test_package_no_file_or_path():
+@pytest.fixture(
+    params=[
+        # Invalid path case
+        [None, "invalid"],
+        # Raise an error case
+        [None, [None]],
+        # No file or path case.
+        [None, None],
+    ]
+)
+def error_module(request):
     orig_file = sys.modules["tests"].__file__
     orig_path = sys.modules["tests"].__path__
-    sys.modules["tests"].__file__ = None
-    sys.modules["tests"].__path__ = None
+    sys.modules["tests"].__file__ = request.param[0]
+    sys.modules["tests"].__path__ = request.param[1]
     yield
     sys.modules["tests"].__file__ = orig_file
     sys.modules["tests"].__path__ = orig_path
 
 
-def test_module_filepath_without_file_or_path(test_package_no_file_or_path):
+def test_module_filepath_error_flows(error_module):
     frame = get_tb().tb_frame
     module = frame.f_globals["__name__"]
     filepath = frame.f_code.co_filename