Repository: getsentry/sentry

Issue:
Update the performance detector to take the total blocking thread time rather than an individual spans blocking time
Example event with multiple: https://gist.github.com/wmak/540ee2305a9305abe7d28fa8db620dbd


Discussion:

Resolving patch:
diff --git a/src/sentry/utils/performance_issues/performance_detection.py b/src/sentry/utils/performance_issues/performance_detection.py
--- a/src/sentry/utils/performance_issues/performance_detection.py
+++ b/src/sentry/utils/performance_issues/performance_detection.py
@@ -6,6 +6,7 @@
 import random
 import re
 from abc import ABC, abstractmethod
+from collections import defaultdict
 from dataclasses import dataclass
 from datetime import timedelta
 from enum import Enum
@@ -496,6 +497,29 @@ def contains_complete_query(span: Span, is_source: Optional[bool] = False) -> bo
         return query and not query.endswith("...")
 
 
+def total_span_time(span_list: List[Dict[str, Any]]) -> float:
+    """Return the total non-overlapping span time in milliseconds for all the spans in the list"""
+    # Sort the spans so that when iterating the next span in the list is either within the current, or afterwards
+    sorted_span_list = sorted(span_list, key=lambda span: span["start_timestamp"])
+    total_duration = 0
+    first_item = sorted_span_list[0]
+    current_min = first_item["start_timestamp"]
+    current_max = first_item["timestamp"]
+    for span in sorted_span_list[1:]:
+        # If the start is contained within the current, check if the max extends the current duration
+        if current_min <= span["start_timestamp"] <= current_max:
+            current_max = max(span["timestamp"], current_max)
+        # If not within current min&max then there's a gap between spans, so add to total_duration and start a new
+        # min/max
+        else:
+            total_duration += current_max - current_min
+            current_min = span["start_timestamp"]
+            current_max = span["timestamp"]
+    # Add the remaining duration
+    total_duration += current_max - current_min
+    return total_duration * 1000
+
+
 class PerformanceDetector(ABC):
     """
     Classes of this type have their visit functions called as the event is walked once and will store a performance issue if one is detected.
@@ -1374,6 +1398,7 @@ def init(self):
         self.most_recent_hash = {}
         self.stored_problems = {}
         self.mapper = None
+        self.parent_to_blocked_span = defaultdict(list)
         self._prepare_deobfuscation()
 
     def _prepare_deobfuscation(self):
@@ -1421,30 +1446,42 @@ def _deobfuscate_function(self, frame):
 
     def visit_span(self, span: Span):
         if self._is_file_io_on_main_thread(span):
-            settings_for_span = self.settings_for_span(span)
+            parent_span_id = span.get("parent_span_id")
+            self.parent_to_blocked_span[parent_span_id].append(span)
+
+    def on_complete(self):
+        for parent_span_id, span_list in self.parent_to_blocked_span.items():
+            span_list = [
+                span for span in span_list if "start_timestamp" in span and "timestamp" in span
+            ]
+            total_duration = total_span_time(span_list)
+            settings_for_span = self.settings_for_span(span_list[0])
             if not settings_for_span:
                 return
 
-            op, span_id, op_prefix, span_duration, settings = settings_for_span
-            if span_duration.total_seconds() * 1000 > settings["duration_threshold"]:
-                fingerprint = self._fingerprint(span)
+            _, _, _, _, settings = settings_for_span
+            if total_duration >= settings["duration_threshold"]:
+                fingerprint = self._fingerprint(span_list)
                 self.stored_problems[fingerprint] = PerformanceProblem(
                     fingerprint=fingerprint,
-                    op=span.get("op"),
-                    desc=span.get("description", ""),
-                    parent_span_ids=[span.get("parent_span_id")],
+                    op=span_list[0].get("op"),
+                    desc=span_list[0].get("description", ""),
+                    parent_span_ids=[parent_span_id],
                     type=GroupType.PERFORMANCE_FILE_IO_MAIN_THREAD,
                     cause_span_ids=[],
-                    offender_span_ids=[span.get("span_id", None)],
+                    offender_span_ids=[span["span_id"] for span in span_list if "span_id" in span],
                 )
 
-    def _fingerprint(self, span) -> str:
+    def _fingerprint(self, span_list) -> str:
         call_stack_strings = []
-        for item in span.get("data", {}).get("call_stack", []):
-            module = self._deobfuscate_module(item.get("module", ""))
-            function = self._deobfuscate_function(item)
-            call_stack_strings.append(f"{module}.{function}")
-        call_stack = ".".join(call_stack_strings).encode("utf8")
+        overall_stack = []
+        for span in span_list:
+            for item in span.get("data", {}).get("call_stack", []):
+                module = self._deobfuscate_module(item.get("module", ""))
+                function = self._deobfuscate_function(item)
+                call_stack_strings.append(f"{module}.{function}")
+            overall_stack.append(".".join(call_stack_strings))
+        call_stack = "-".join(overall_stack).encode("utf8")
         hashed_stack = hashlib.sha1(call_stack).hexdigest()
         return f"1-{GroupType.PERFORMANCE_FILE_IO_MAIN_THREAD}-{hashed_stack}"
 


Test patch:
diff --git a/tests/sentry/utils/performance_issues/test_file_io_on_main_thread_detector.py b/tests/sentry/utils/performance_issues/test_file_io_on_main_thread_detector.py
--- a/tests/sentry/utils/performance_issues/test_file_io_on_main_thread_detector.py
+++ b/tests/sentry/utils/performance_issues/test_file_io_on_main_thread_detector.py
@@ -66,3 +66,36 @@ def test_file_io_with_proguard(self):
             problem.fingerprint == f"1-{GroupType.PERFORMANCE_FILE_IO_MAIN_THREAD}-{hashed_stack}"
         )
         assert problem.title == "File IO on Main Thread"
+
+    def test_parallel_spans(self):
+        event = EVENTS["file-io-on-main-thread-with-parallel-spans"]
+
+        detector = FileIOMainThreadDetector(self.settings, event)
+        run_detector_on_data(detector, event)
+        problem = list(detector.stored_problems.values())[0]
+        assert problem.offender_span_ids == ["054ba3a374d543eb", "054ba3a3a4d543ab"]
+
+    def test_parallel_spans_not_detected_when_total_too_short(self):
+        event = EVENTS["file-io-on-main-thread-with-parallel-spans"]
+        event["spans"][1]["timestamp"] = 1669031858.015
+
+        detector = FileIOMainThreadDetector(self.settings, event)
+        run_detector_on_data(detector, event)
+        assert len(detector.stored_problems) == 0
+
+    def test_complicated_structure(self):
+        event = EVENTS["file-io-on-main-thread-with-complicated-structure"]
+
+        detector = FileIOMainThreadDetector(self.settings, event)
+        run_detector_on_data(detector, event)
+        problem = list(detector.stored_problems.values())[0]
+        assert problem.offender_span_ids == [
+            "054ba3a374d543eb",
+            "054ba3a3a4d543ab",
+            "054ba3a3a4d543cd",
+            "054ba3a3a4d543ef",
+            "054ba3a3a4d54ab1",
+            "054ba3a3a4d54ab2",
+            "054ba3a3a4d54ab3",
+            "054ba3a3a4d54ab4",
+        ]
diff --git a/tests/sentry/utils/performance_issues/test_performance_detection.py b/tests/sentry/utils/performance_issues/test_performance_detection.py
--- a/tests/sentry/utils/performance_issues/test_performance_detection.py
+++ b/tests/sentry/utils/performance_issues/test_performance_detection.py
@@ -24,6 +24,7 @@
     _detect_performance_problems,
     detect_performance_problems,
     prepare_problem_for_grouping,
+    total_span_time,
 )
 from sentry.utils.performance_issues.performance_span_issue import PerformanceSpanProblem
 
@@ -1136,3 +1137,121 @@ def test_fetch_multi(self):
         assert [r.problem if r else None for r in result] == [
             problem for _, problem in all_event_problems
         ] + [None]
+
+
+@pytest.mark.parametrize(
+    "spans, duration",
+    [
+        pytest.param(
+            [
+                {
+                    "start_timestamp": 0,
+                    "timestamp": 0.011,
+                }
+            ],
+            11,
+        ),
+        pytest.param(
+            [
+                {
+                    "start_timestamp": 0,
+                    "timestamp": 0.011,
+                },
+                {
+                    "start_timestamp": 0,
+                    "timestamp": 0.011,
+                },
+            ],
+            11,
+            id="parallel spans",
+        ),
+        pytest.param(
+            [
+                {
+                    "start_timestamp": 0,
+                    "timestamp": 0.011,
+                },
+                {
+                    "start_timestamp": 1.0,
+                    "timestamp": 1.011,
+                },
+            ],
+            22,
+            id="separate spans",
+        ),
+        pytest.param(
+            [
+                {
+                    "start_timestamp": 0,
+                    "timestamp": 0.011,
+                },
+                {
+                    "start_timestamp": 0.005,
+                    "timestamp": 0.016,
+                },
+            ],
+            16,
+            id="overlapping spans",
+        ),
+        pytest.param(
+            [
+                {
+                    "start_timestamp": 0,
+                    "timestamp": 0.011,
+                },
+                {
+                    "start_timestamp": 0.005,
+                    "timestamp": 0.016,
+                },
+                {
+                    "start_timestamp": 0.015,
+                    "timestamp": 0.032,
+                },
+            ],
+            32,
+            id="multiple overlapping spans",
+        ),
+        pytest.param(
+            [
+                {
+                    "start_timestamp": 0,
+                    "timestamp": 0.011,
+                },
+                {
+                    "start_timestamp": 0.011,
+                    "timestamp": 0.022,
+                },
+                {
+                    "start_timestamp": 0.022,
+                    "timestamp": 0.033,
+                },
+            ],
+            33,
+            id="multiple overlapping touching spans",
+        ),
+        pytest.param(
+            [
+                {
+                    "start_timestamp": 0,
+                    "timestamp": 0.011,
+                },
+                {
+                    "start_timestamp": 0.005,
+                    "timestamp": 0.022,
+                },
+                {
+                    "start_timestamp": 0.033,
+                    "timestamp": 0.045,
+                },
+                {
+                    "start_timestamp": 0.045,
+                    "timestamp": 0.055,
+                },
+            ],
+            44,
+            id="multiple overlapping spans with gaps",
+        ),
+    ],
+)
+def test_total_span_time(spans, duration):
+    assert total_span_time(spans) == pytest.approx(duration, 0.01)