Repository: getsentry/sentry

Issue:
SDK Crash Detection: Configurable Sample Rate
We need to add a configurable sample rate to the options for collecting SDK crashes. The code should sample after it detected an SDK crash

https://github.com/getsentry/sentry/blob/95086b406dec79e6bcef45f299a3e92f727da2c0/src/sentry/utils/sdk_crashes/sdk_crash_detection.py#L56-L58


Discussion:

Resolving patch:
diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py
--- a/src/sentry/conf/server.py
+++ b/src/sentry/conf/server.py
@@ -3531,11 +3531,14 @@ def build_cdc_postgres_init_db_volume(settings: Any) -> dict[str, dict[str, str]
 # tests)
 SENTRY_METRICS_INDEXER_RAISE_VALIDATION_ERRORS = False
 
-# The project ID for SDK Crash Monitoring to save the detected SDK crashed to.
+# The project ID for SDK Crash Detection to save the detected SDK crashed to.
 # Currently, this is a single value, as the SDK Crash Detection feature only detects crashes for the Cocoa SDK.
 # Once we start detecting crashes for other SDKs, this will be a mapping of SDK name to project ID or something similar.
 SDK_CRASH_DETECTION_PROJECT_ID: Optional[int] = None
 
+# The percentage of events to sample for SDK Crash Detection. 0.0 = 0%, 0.5 = 50% 1.0 = 100%.
+SDK_CRASH_DETECTION_SAMPLE_RATE = 0.0
+
 # The Redis cluster to use for monitoring the service / consumer health.
 SENTRY_SERVICE_MONITORING_REDIS_CLUSTER = "default"
 
diff --git a/src/sentry/tasks/post_process.py b/src/sentry/tasks/post_process.py
--- a/src/sentry/tasks/post_process.py
+++ b/src/sentry/tasks/post_process.py
@@ -1080,7 +1080,9 @@ def sdk_crash_monitoring(job: PostProcessJob):
     with metrics.timer("post_process.sdk_crash_monitoring.duration"):
         with sentry_sdk.start_span(op="tasks.post_process_group.sdk_crash_monitoring"):
             sdk_crash_detection.detect_sdk_crash(
-                event=event, event_project_id=settings.SDK_CRASH_DETECTION_PROJECT_ID
+                event=event,
+                event_project_id=settings.SDK_CRASH_DETECTION_PROJECT_ID,
+                sample_rate=settings.SDK_CRASH_DETECTION_SAMPLE_RATE,
             )
 
 
diff --git a/src/sentry/utils/sdk_crashes/sdk_crash_detection.py b/src/sentry/utils/sdk_crashes/sdk_crash_detection.py
--- a/src/sentry/utils/sdk_crashes/sdk_crash_detection.py
+++ b/src/sentry/utils/sdk_crashes/sdk_crash_detection.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import random
 from typing import Any, Mapping, Optional
 
 from sentry.eventstore.models import Event
@@ -28,7 +29,9 @@ def __init__(
         self.sdk_crash_reporter = sdk_crash_reporter
         self.cocoa_sdk_crash_detector = sdk_crash_detector
 
-    def detect_sdk_crash(self, event: Event, event_project_id: int) -> Optional[Event]:
+    def detect_sdk_crash(
+        self, event: Event, event_project_id: int, sample_rate: float
+    ) -> Optional[Event]:
         should_detect_sdk_crash = (
             event.group
             and event.group.issue_category == GroupCategory.ERROR
@@ -54,6 +57,9 @@ def detect_sdk_crash(self, event: Event, event_project_id: int) -> Optional[Even
             return None
 
         if self.cocoa_sdk_crash_detector.is_sdk_crash(frames):
+            if random.random() >= sample_rate:
+                return None
+
             sdk_crash_event_data = strip_event_data(event.data, self.cocoa_sdk_crash_detector)
 
             set_path(


Test patch:
diff --git a/tests/sentry/tasks/test_post_process.py b/tests/sentry/tasks/test_post_process.py
--- a/tests/sentry/tasks/test_post_process.py
+++ b/tests/sentry/tasks/test_post_process.py
@@ -1541,6 +1541,7 @@ def test_forecast_in_activity(self, mock_is_escalating):
 class SDKCrashMonitoringTestMixin(BasePostProgressGroupMixin):
     @with_feature("organizations:sdk-crash-detection")
     @override_settings(SDK_CRASH_DETECTION_PROJECT_ID=1234)
+    @override_settings(SDK_CRASH_DETECTION_SAMPLE_RATE=0.1234)
     def test_sdk_crash_monitoring_is_called(self, mock_sdk_crash_detection):
         event = self.create_event(
             data={"message": "testing"},
@@ -1556,6 +1557,11 @@ def test_sdk_crash_monitoring_is_called(self, mock_sdk_crash_detection):
 
         mock_sdk_crash_detection.detect_sdk_crash.assert_called_once()
 
+        args = mock_sdk_crash_detection.detect_sdk_crash.call_args[-1]
+        assert args["event"].project.id == event.project.id
+        assert args["event_project_id"] == 1234
+        assert args["sample_rate"] == 0.1234
+
     def test_sdk_crash_monitoring_is_not_called_with_disabled_feature(
         self, mock_sdk_crash_detection
     ):
diff --git a/tests/sentry/utils/sdk_crashes/test_sdk_crash_detection.py b/tests/sentry/utils/sdk_crashes/test_sdk_crash_detection.py
--- a/tests/sentry/utils/sdk_crashes/test_sdk_crash_detection.py
+++ b/tests/sentry/utils/sdk_crashes/test_sdk_crash_detection.py
@@ -31,7 +31,7 @@ def execute_test(self, event_data, should_be_reported, mock_sdk_crash_reporter):
             project_id=self.project.id,
         )
 
-        sdk_crash_detection.detect_sdk_crash(event=event, event_project_id=1234)
+        sdk_crash_detection.detect_sdk_crash(event=event, event_project_id=1234, sample_rate=1.0)
 
         if should_be_reported:
             assert mock_sdk_crash_reporter.report.call_count == 1
@@ -55,7 +55,7 @@ def test_performance_event_not_detected(self, mock_sdk_crash_reporter):
             fingerprint=[fingerprint],
         )
 
-        sdk_crash_detection.detect_sdk_crash(event=event, event_project_id=1234)
+        sdk_crash_detection.detect_sdk_crash(event=event, event_project_id=1234, sample_rate=1.0)
 
         assert mock_sdk_crash_reporter.report.call_count == 0
 
@@ -300,6 +300,36 @@ def test_frames_only_in_app_after_sentry_frame_not_reported(self, mock_sdk_crash
         )
 
 
+@patch("sentry.utils.sdk_crashes.sdk_crash_detection.sdk_crash_detection.sdk_crash_reporter")
+class SamplingTestMixin(BaseSDKCrashDetectionMixin, SnubaTestCase):
+    @patch("random.random", return_value=0.0)
+    def test_sample_is_rate_zero(self, mock_random, mock_sdk_crash_reporter):
+        event = self.create_event(
+            data=get_crash_event(),
+            project_id=self.project.id,
+        )
+
+        sdk_crash_detection.detect_sdk_crash(event=event, event_project_id=1234, sample_rate=0.0)
+
+        assert mock_sdk_crash_reporter.report.call_count == 0
+
+    @patch("random.random", return_value=0.1)
+    def test_sampling_rate(self, mock_random, mock_sdk_crash_reporter):
+        event = self.create_event(
+            data=get_crash_event(),
+            project_id=self.project.id,
+        )
+
+        # not sampled
+        sdk_crash_detection.detect_sdk_crash(event=event, event_project_id=1234, sample_rate=0.09)
+        sdk_crash_detection.detect_sdk_crash(event=event, event_project_id=1234, sample_rate=0.1)
+
+        # sampled
+        sdk_crash_detection.detect_sdk_crash(event=event, event_project_id=1234, sample_rate=0.11)
+
+        assert mock_sdk_crash_reporter.report.call_count == 1
+
+
 class SDKCrashReportTestMixin(BaseSDKCrashDetectionMixin, SnubaTestCase):
     @django_db_all
     def test_sdk_crash_event_stored_to_sdk_crash_project(self):
@@ -317,7 +347,7 @@ def test_sdk_crash_event_stored_to_sdk_crash_project(self):
         )
 
         sdk_crash_event = sdk_crash_detection.detect_sdk_crash(
-            event=event, event_project_id=cocoa_sdk_crashes_project.id
+            event=event, event_project_id=cocoa_sdk_crashes_project.id, sample_rate=1.0
         )
 
         assert sdk_crash_event is not None
@@ -340,6 +370,7 @@ class SDKCrashDetectionTest(
     CococaSDKFramesTestMixin,
     PerformanceEventTestMixin,
     SDKCrashReportTestMixin,
+    SamplingTestMixin,
 ):
     def create_event(self, data, project_id, assert_no_errors=True):
         return self.store_event(data=data, project_id=project_id, assert_no_errors=assert_no_errors)