Repository: getsentry/sentry-python

Issue:
Catch crashing threads
since `sys.excepthook` is not called for threads that crash, we should write a separate integration for this (turned on by default)


Discussion:
I noticed some crashing threads that weren't getting reported to sentry and was surprised. I've updated my code to catch them myself but I agree that sentry could try harder here.

Sentry should be able to monkeypatch `threading.Thread._bootstrap_inner`. Unfortunately there is not a good spot to monkeypatch `Thread.run` (which a public api overridable by users) which `Thread._bootstrap_inner` calls directly. Is there a standard approach sentry uses to monkeypatching without vendoring the current `_bootstrap_inner` code into sentry? So far that seems to me to be the only option.
Two ideas:

* Patch `Thread._exc_info` to capture the exception (it gets called in `_bootstrap_inner`)
* Patch `Thread._bootstrap_inner`, but inside the patched function patch `self.run` and just call the old `_bootstrap_inner`.

There is no standard approach to this. An established rule is to patch public/stable APIs rather than private/unstable ones but it seems like we have no choice here.
Actually we can do something that uses only public API: Patch `Thread.start`, patched functions patches `self.run` and calls old `Thread.start`.

Resolving patch:
diff --git a/sentry_sdk/integrations/__init__.py b/sentry_sdk/integrations/__init__.py
--- a/sentry_sdk/integrations/__init__.py
+++ b/sentry_sdk/integrations/__init__.py
@@ -1,4 +1,6 @@
 """This package"""
+from __future__ import absolute_import
+
 from threading import Lock
 from collections import namedtuple
 
@@ -30,6 +32,7 @@ def iter_default_integrations():
     from sentry_sdk.integrations.atexit import AtexitIntegration
     from sentry_sdk.integrations.modules import ModulesIntegration
     from sentry_sdk.integrations.argv import ArgvIntegration
+    from sentry_sdk.integrations.threading import ThreadingIntegration
 
     yield LoggingIntegration
     yield StdlibIntegration
@@ -38,6 +41,7 @@ def iter_default_integrations():
     yield AtexitIntegration
     yield ModulesIntegration
     yield ArgvIntegration
+    yield ThreadingIntegration
 
 
 def setup_integrations(integrations, with_defaults=True):
diff --git a/sentry_sdk/integrations/threading.py b/sentry_sdk/integrations/threading.py
new file mode 100644
--- /dev/null
+++ b/sentry_sdk/integrations/threading.py
@@ -0,0 +1,64 @@
+from __future__ import absolute_import
+
+import sys
+
+from threading import Thread
+
+from sentry_sdk import Hub
+from sentry_sdk._compat import reraise
+from sentry_sdk.utils import event_from_exception
+from sentry_sdk.integrations import Integration
+
+
+class ThreadingIntegration(Integration):
+    identifier = "threading"
+
+    def __init__(self, propagate_hub=False):
+        self.propagate_hub = propagate_hub
+
+    @staticmethod
+    def setup_once():
+        old_start = Thread.start
+
+        def sentry_start(self, *a, **kw):
+            hub = Hub.current
+            integration = hub.get_integration(ThreadingIntegration)
+            if integration is not None:
+                if integration.propagate_hub:
+                    hub = Hub(hub)
+                else:
+                    hub = None
+
+                self.run = _wrap_run(hub, self.run)
+
+            return old_start(self, *a, **kw)
+
+        Thread.start = sentry_start
+
+
+def _wrap_run(parent_hub, old_run):
+    def run(*a, **kw):
+        hub = parent_hub or Hub.current
+
+        with hub:
+            try:
+                return old_run(*a, **kw)
+            except Exception:
+                reraise(*_capture_exception())
+
+    return run
+
+
+def _capture_exception():
+    hub = Hub.current
+    exc_info = sys.exc_info()
+
+    if hub.get_integration(ThreadingIntegration) is not None:
+        event, hint = event_from_exception(
+            exc_info,
+            client_options=hub.client.options,
+            mechanism={"type": "threading", "handled": False},
+        )
+        hub.capture_event(event, hint=hint)
+
+    return exc_info


Test patch:
diff --git a/tests/integrations/threading/test_threading.py b/tests/integrations/threading/test_threading.py
new file mode 100644
--- /dev/null
+++ b/tests/integrations/threading/test_threading.py
@@ -0,0 +1,64 @@
+from threading import Thread
+
+import pytest
+
+from sentry_sdk import configure_scope
+from sentry_sdk.integrations.threading import ThreadingIntegration
+
+
+@pytest.mark.parametrize("integrations", [[ThreadingIntegration()], []])
+def test_handles_exceptions(sentry_init, capture_events, integrations):
+    sentry_init(default_integrations=False, integrations=integrations)
+    events = capture_events()
+
+    def crash():
+        1 / 0
+
+    t = Thread(target=crash)
+    t.start()
+    t.join()
+
+    if integrations:
+        event, = events
+
+        exception, = event["exception"]["values"]
+        assert exception["type"] == "ZeroDivisionError"
+        assert exception["mechanism"] == {"type": "threading", "handled": False}
+    else:
+        assert not events
+
+
+@pytest.mark.parametrize("propagate_hub", (True, False))
+def test_propagates_hub(sentry_init, capture_events, propagate_hub):
+    sentry_init(
+        default_integrations=False,
+        integrations=[ThreadingIntegration(propagate_hub=propagate_hub)],
+    )
+    events = capture_events()
+
+    def stage1():
+        with configure_scope() as scope:
+            scope.set_tag("stage1", True)
+
+        t = Thread(target=stage2)
+        t.start()
+        t.join()
+
+    def stage2():
+        1 / 0
+
+    t = Thread(target=stage1)
+    t.start()
+    t.join()
+
+    event, = events
+
+    exception, = event["exception"]["values"]
+
+    assert exception["type"] == "ZeroDivisionError"
+    assert exception["mechanism"] == {"type": "threading", "handled": False}
+
+    if propagate_hub:
+        assert event["tags"]["stage1"] is True
+    else:
+        assert "stage1" not in event.get("tags", {})