Repository: getsentry/sentry
Pull request: feat(issues-platform): Start writing occurrences to platform
Use system and project options set in https://github.com/getsentry/sentry/pull/45709

Fixes https://github.com/getsentry/sentry/issues/43975

Problem statement:
Perf Migration: Start writing perf issues to the issue platform
We'll need to start writing performance issues to the issue platform dataset. When we do this, we'll be dual writing for some period of time. Since both perf issue detection and issue platform create/update the issue and counts, we'll need to make sure that only one of these systems is actively updating counts at any one time, otherwise we'll end up double counting.

It might make sense to split up our concept of writing here: Writing the events to the issue platform dataset and creating/updating issues. We can dual write events, since perf issues currently are writing to transactions, and in issue platform will be writing to a new dataset. And we should have a switch that flips us between creating issues via transaction ingestion vs issue platform. Might make sense to make the switch per org so that we can test it in sentry and make sure the issues still look/render correctly.

We should make sure that we send the issue to the issue platform *after* we've written the transaction to nodestore.

Hints from issue discussion:

Commit message: add group creation options

Patch to review:
diff --git a/bin/load-mocks b/bin/load-mocks
index 7663bd1d7512a3..bc60891d913757 100755
--- a/bin/load-mocks
+++ b/bin/load-mocks
@@ -1280,6 +1280,10 @@ if __name__ == "__main__":
             load_performance_issues=options.load_performance_issues,
             slow=options.slow,
         )
+
+        from sentry.issues.producer import get_occurrence_producer
+
+        get_occurrence_producer().close()
     except Exception:
         # Avoid reporting any issues recursively back into Sentry
         import sys
diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py
index 88541efcc837b9..618b600a833f22 100644
--- a/src/sentry/conf/server.py
+++ b/src/sentry/conf/server.py
@@ -3076,6 +3076,7 @@ def build_cdc_postgres_init_db_volume(settings):
 SENTRY_PERFORMANCE_ISSUES_REDUCE_NOISE = False
 
 SENTRY_ISSUE_PLATFORM_RATE_LIMITER_OPTIONS = {}
+SENTRY_ISSUE_PLATFORM_FUTURES_MAX_LIMIT = 10000
 
 SENTRY_REGION = os.environ.get("SENTRY_REGION", None)
 SENTRY_REGION_CONFIG: Iterable[Region] = ()
diff --git a/src/sentry/event_manager.py b/src/sentry/event_manager.py
index 4843c0236189a4..4e295c26cdbb52 100644
--- a/src/sentry/event_manager.py
+++ b/src/sentry/event_manager.py
@@ -6,6 +6,7 @@
 import random
 import re
 import time
+import uuid
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from hashlib import md5
@@ -78,6 +79,9 @@
 from sentry.grouping.result import CalculatedHashes
 from sentry.ingest.inbound_filters import FilterStatKeys
 from sentry.issues.grouptype import GroupCategory, reduce_noise
+from sentry.issues.issue_occurrence import IssueOccurrence
+from sentry.issues.producer import produce_occurrence_to_kafka
+from sentry.issues.utils import write_occurrence_to_platform
 from sentry.killswitches import killswitch_matches_context
 from sentry.lang.native.utils import STORE_CRASH_REPORTS_ALL, convert_crashreport_count
 from sentry.locks import locks
@@ -2317,7 +2321,6 @@ def _save_aggregate_performance(jobs: Sequence[PerformanceJob], projects: Projec
         # Granular, per-project option
         per_project_rate = project.get_option("sentry:performance_issue_creation_rate", 1.0)
         if per_project_rate > random.random():
-
             kwargs = _create_kwargs(job)
             kwargs["culprit"] = job["culprit"]
             kwargs["data"] = materialize_metadata(
@@ -2455,6 +2458,35 @@ def _save_aggregate_performance(jobs: Sequence[PerformanceJob], projects: Projec
                 EventPerformanceProblem(event, performance_problems_by_hash[problem_hash]).save()
 
 
+@metrics.wraps("save_event.send_occurrence_to_platform")
+def _send_occurrence_to_platform(jobs: Sequence[Job], projects: ProjectsMapping) -> None:
+    for job in jobs:
+        event = job["event"]
+        project = event.project
+        event_id = event.event_id
+
+        performance_problems = job["performance_problems"]
+        for problem in performance_problems:
+            if write_occurrence_to_platform(problem, options, project):
+                occurrence = IssueOccurrence(
+                    id=uuid.uuid4().hex,
+                    resource_id=None,
+                    project_id=project.id,
+                    event_id=event_id,
+                    fingerprint=[problem.fingerprint],
+                    type=problem.type,
+                    issue_title=problem.title,
+                    subtitle=problem.desc,
+                    culprit=event.transaction,
+                    evidence_data=problem.evidence_data,
+                    evidence_display=problem.evidence_display,
+                    detection_time=event.datetime,
+                    level="info",
+                )
+
+                produce_occurrence_to_kafka(occurrence)
+
+
 @metrics.wraps("event_manager.save_transaction_events")
 def save_transaction_events(jobs: Sequence[Job], projects: ProjectsMapping) -> Sequence[Job]:
     with metrics.timer("event_manager.save_transactions.collect_organization_ids"):
@@ -2493,6 +2525,7 @@ def save_transaction_events(jobs: Sequence[Job], projects: ProjectsMapping) -> S
     _nodestore_save_many(jobs)
     _eventstream_insert_many(jobs)
     _track_outcome_accepted_many(jobs)
+    _send_occurrence_to_platform(jobs, projects)
     return jobs
 
 
diff --git a/src/sentry/issues/ingest.py b/src/sentry/issues/ingest.py
index a9901da75b991e..96a2b6ed941f55 100644
--- a/src/sentry/issues/ingest.py
+++ b/src/sentry/issues/ingest.py
@@ -9,7 +9,7 @@
 from django.conf import settings
 from django.db import transaction
 
-from sentry import eventstream
+from sentry import eventstream, options
 from sentry.constants import LOG_LEVELS_MAP
 from sentry.event_manager import (
     GroupInfo,
@@ -26,6 +26,8 @@
 from sentry.ratelimits.sliding_windows import Quota, RedisSlidingWindowRateLimiter, RequestedQuota
 from sentry.utils import metrics
 
+from .utils import can_create_group
+
 issue_rate_limiter = RedisSlidingWindowRateLimiter(
     **settings.SENTRY_ISSUE_PLATFORM_RATE_LIMITER_OPTIONS
 )
@@ -38,7 +40,11 @@
 def save_issue_occurrence(
     occurrence_data: IssueOccurrenceData, event: Event
 ) -> Tuple[IssueOccurrence, Optional[GroupInfo]]:
-    process_occurrence_data(occurrence_data)
+    # Do not hash fingerprints for performance issues because they're already
+    # hashed in save_aggregate_performance
+    # This check should be removed once perf issues are created through the platform
+    if can_create_group(occurrence_data, options, event.project):
+        process_occurrence_data(occurrence_data)
     # Convert occurrence data to `IssueOccurrence`
     occurrence = IssueOccurrence.from_dict(occurrence_data)
     if occurrence.event_id != event.event_id:
@@ -56,6 +62,9 @@ def save_issue_occurrence(
     group_info = save_issue_from_occurrence(occurrence, event, release)
     if group_info:
         send_issue_occurrence_to_eventstream(event, occurrence, group_info)
+
+        if not can_create_group(occurrence_data, options, event.project):
+            return occurrence, group_info
         environment = event.get_environment()
         _get_or_create_group_environment(environment, release, [group_info])
         _increment_release_associated_counts(
@@ -153,7 +162,15 @@ def save_issue_from_occurrence(
         .select_related("group")
         .first()
     )
+
+    # This forces an early return to skip extra processing steps
+    # for performance issues because they are already created/updated in save_transaction
+    return_group_info_early = not can_create_group(occurrence, options, project)
+
     if not existing_grouphash:
+        if return_group_info_early:
+            return None
+
         with metrics.timer("issues.save_issue_from_occurrence.check_write_limits"):
             granted_quota = issue_rate_limiter.check_and_use_quotas(
                 [RequestedQuota(f"issue-platform-issues:{project.id}", 1, [ISSUE_QUOTA])]
@@ -197,9 +214,12 @@ def save_issue_from_occurrence(
             return None
 
         is_new = False
-        # Note: This updates the message of the issue based on the event. Not sure what we want to
-        # store there yet, so we may need to revisit that.
-        is_regression = _process_existing_aggregate(group, event, issue_kwargs, release)
+        is_regression = False
+
+        if not return_group_info_early:
+            # Note: This updates the message of the issue based on the event. Not sure what we want to
+            # store there yet, so we may need to revisit that.
+            is_regression = _process_existing_aggregate(group, event, issue_kwargs, release)
         group_info = GroupInfo(group=group, is_new=is_new, is_regression=is_regression)
 
     return group_info
@@ -211,6 +231,8 @@ def send_issue_occurrence_to_eventstream(
     group_event = event.for_group(group_info.group)
     group_event.occurrence = occurrence
 
+    skip_consume = can_create_group(occurrence, options, event.project)
+
     eventstream.insert(
         event=group_event,
         is_new=group_info.is_new,
@@ -218,7 +240,7 @@ def send_issue_occurrence_to_eventstream(
         is_new_group_environment=group_info.is_new_group_environment,
         primary_hash=occurrence.fingerprint[0],
         received_timestamp=group_event.data.get("received") or group_event.datetime,
-        skip_consume=False,
+        skip_consume=skip_consume,
         group_states=[
             {
                 "id": group_info.group.id,
diff --git a/src/sentry/issues/producer.py b/src/sentry/issues/producer.py
new file mode 100644
index 00000000000000..f768494b12a251
--- /dev/null
+++ b/src/sentry/issues/producer.py
@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from atexit import register
+from collections import deque
+from concurrent import futures
+from concurrent.futures import Future
+from typing import Deque
+
+from arroyo import Topic
+from arroyo.backends.kafka import KafkaPayload, KafkaProducer, build_kafka_configuration
+from arroyo.types import BrokerValue
+from django.conf import settings
+
+from sentry.issues.issue_occurrence import IssueOccurrence
+from sentry.utils import json
+from sentry.utils.kafka_config import get_kafka_producer_cluster_options
+
+occurrence_producer = None
+
+
+def get_occurrence_producer() -> KafkaProducer:
+    global occurrence_producer
+    if occurrence_producer is None:
+        cluster_name = settings.KAFKA_TOPICS[settings.KAFKA_INGEST_OCCURRENCES]["cluster"]
+        producer_config = get_kafka_producer_cluster_options(cluster_name)
+        producer_config.pop("compression.type")
+        producer_config.pop("message.max.bytes")
+        occurrence_producer = KafkaProducer(
+            build_kafka_configuration(default_config=producer_config)
+        )
+    return occurrence_producer
+
+
+def produce_occurrence_to_kafka(occurrence: IssueOccurrence) -> None:
+    payload = KafkaPayload(None, json.dumps(occurrence.to_dict()).encode("utf-8"), [])
+    occurrence_producer = get_occurrence_producer()
+    future = occurrence_producer.produce(Topic(settings.KAFKA_INGEST_OCCURRENCES), payload)
+
+    track_occurrence_producer_futures(future)
+
+
+occurrence_producer_futures: Deque[Future[BrokerValue[KafkaPayload]]] = deque()
+
+
+def track_occurrence_producer_futures(future: Future[BrokerValue[KafkaPayload]]) -> None:
+    global occurrence_producer_futures
+    occurrence_producer_futures.append(future)
+    if len(occurrence_producer_futures) >= settings.SENTRY_ISSUE_PLATFORM_FUTURES_MAX_LIMIT:
+        try:
+            future = occurrence_producer_futures.popleft()
+            future.result()
+        except IndexError:
+            return
+
+
+def handle_occurrence_producer() -> None:
+    futures.wait(occurrence_producer_futures)
+    get_occurrence_producer().close()
+
+
+register(handle_occurrence_producer)
diff --git a/src/sentry/issues/utils.py b/src/sentry/issues/utils.py
new file mode 100644
index 00000000000000..500683b8ac3dcb
--- /dev/null
+++ b/src/sentry/issues/utils.py
@@ -0,0 +1,32 @@
+from typing import Any, Mapping
+
+from sentry.issues.grouptype import PerformanceNPlusOneGroupType
+from sentry.issues.issue_occurrence import IssueOccurrence
+from sentry.models import Project
+from sentry.utils.performance_issues.performance_problem import PerformanceProblem
+
+
+def can_create_group(
+    ocurrence: IssueOccurrence, options: Mapping[str, Any], project: Project
+) -> bool:
+    return (
+        # create N+1 db query issues first
+        ocurrence.type.type_id == PerformanceNPlusOneGroupType.type_id
+        # system-wide option
+        and options.get("performance.issues.create_issues_through_platform", False)
+        # more-granular per-project option
+        and project.get_option("sentry:performance_issue_create_issue_through_platform", False)
+    )
+
+
+def write_occurrence_to_platform(
+    performance_problem: PerformanceProblem, options: Mapping[str, Any], project: Project
+) -> bool:
+    return (
+        # handle only N+1 db query detector first
+        performance_problem.type.type_id == PerformanceNPlusOneGroupType.type_id
+        # system-wide option
+        and options.get("performance.issues.send_to_issues_platform", False)
+        # more-granular per-project option
+        and project.get_option("sentry:performance_issue_send_to_issues_platform", False)
+    )
diff --git a/src/sentry/options/defaults.py b/src/sentry/options/defaults.py
index 3ee5c682c6664f..efd5086475dd77 100644
--- a/src/sentry/options/defaults.py
+++ b/src/sentry/options/defaults.py
@@ -647,9 +647,16 @@
 register("performance.issues.render_blocking_assets.fcp_ratio_threshold", default=0.33)
 register("performance.issues.render_blocking_assets.size_threshold", default=1000000)
 
-# System-wide option for performance issue creation through issues platform
+# System-wide option for sending occurrences to the issues platform
 register("performance.issues.send_to_issues_platform", default=False, flags=FLAG_MODIFIABLE_BOOL)
 
+# System-wide option for performance issue creation through issues platform
+register(
+    "performance.issues.create_issues_through_platform",
+    default=False,
+    flags=FLAG_MODIFIABLE_BOOL,
+)
+
 # Dynamic Sampling system wide options
 # Killswitch to disable new dynamic sampling behavior specifically new dynamic sampling biases
 register("dynamic-sampling:enabled-biases", default=True)
diff --git a/src/sentry/projectoptions/defaults.py b/src/sentry/projectoptions/defaults.py
index 01da649a815b62..1110d256ea2d91 100644
--- a/src/sentry/projectoptions/defaults.py
+++ b/src/sentry/projectoptions/defaults.py
@@ -101,9 +101,13 @@
 # Can be used to turn off a projects detection for users if there is a project-specific issue.
 register(key="sentry:performance_issue_creation_rate", default=1.0)
 
+# Rate at which performance problems are sent to issues platform. Defaults to False, system flags and options will determine if an organization sends perf problems to platform.
+# Can be used to turn off writing occurrences for users if there is a project-specific issue.
+register(key="sentry:performance_issue_send_to_issues_platform", default=False)
+
 # Rate at which performance issues are created through issues platform per project. Defaults to False, system flags and options will determine if an organization creates issues through platform.
 # Can be used to turn off issue creation for users if there is a project-specific issue.
-register(key="sentry:performance_issue_send_to_issues_platform", default=False)
+register(key="sentry:performance_issue_create_issue_through_platform", default=False)
 
 DEFAULT_PROJECT_PERFORMANCE_DETECTION_SETTINGS = {
     "n_plus_one_db_detection_rate": 1.0,
diff --git a/src/sentry/tasks/post_process.py b/src/sentry/tasks/post_process.py
index 4086f88389574f..66c4d86e45839a 100644
--- a/src/sentry/tasks/post_process.py
+++ b/src/sentry/tasks/post_process.py
@@ -537,12 +537,14 @@ def post_process_group(
 def run_post_process_job(job: PostProcessJob):
     group_event = job["event"]
     issue_category = group_event.group.issue_category
+
     if group_event.group.issue_type.type_id in get_group_types_by_category(
         GroupCategory.PROFILE.value
     ) and not features.has(
         "organizations:profile-blocked-main-thread-ppg", group_event.group.organization
     ):
         return
+
     if issue_category not in GROUP_CATEGORY_POST_PROCESS_PIPELINE:
         # pipeline for generic issues
         pipeline = GENERIC_POST_PROCESS_PIPELINE
diff --git a/src/sentry/testutils/factories.py b/src/sentry/testutils/factories.py
index 93b751e0f90ddf..6f928eb21496ce 100644
--- a/src/sentry/testutils/factories.py
+++ b/src/sentry/testutils/factories.py
@@ -698,6 +698,8 @@ def inject_performance_problems(jobs, _):
                         parent_span_ids=None,
                         cause_span_ids=None,
                         offender_span_ids=None,
+                        evidence_data={},
+                        evidence_display=[],
                     )
                 )
 
diff --git a/src/sentry/utils/performance_issues/detectors/consecutive_db_detector.py b/src/sentry/utils/performance_issues/detectors/consecutive_db_detector.py
index 949f5276a20c0a..3ad7eb3b413ff2 100644
--- a/src/sentry/utils/performance_issues/detectors/consecutive_db_detector.py
+++ b/src/sentry/utils/performance_issues/detectors/consecutive_db_detector.py
@@ -114,6 +114,8 @@ def _store_performance_problem(self) -> None:
             cause_span_ids=cause_span_ids,
             parent_span_ids=None,
             offender_span_ids=offender_span_ids,
+            evidence_data={},
+            evidence_display=[],
         )
 
         self._reset_variables()
diff --git a/src/sentry/utils/performance_issues/detectors/consecutive_http_detector.py b/src/sentry/utils/performance_issues/detectors/consecutive_http_detector.py
index 1ada8777b0833f..38f665fe71efa1 100644
--- a/src/sentry/utils/performance_issues/detectors/consecutive_http_detector.py
+++ b/src/sentry/utils/performance_issues/detectors/consecutive_http_detector.py
@@ -80,6 +80,8 @@ def _store_performance_problem(self) -> None:
             cause_span_ids=[],
             parent_span_ids=None,
             offender_span_ids=offender_span_ids,
+            evidence_data={},
+            evidence_display=[],
         )
 
         self._reset_variables()
diff --git a/src/sentry/utils/performance_issues/detectors/file_io_main_thread_detector.py b/src/sentry/utils/performance_issues/detectors/file_io_main_thread_detector.py
index ed4347a9543f8e..653c42778c7712 100644
--- a/src/sentry/utils/performance_issues/detectors/file_io_main_thread_detector.py
+++ b/src/sentry/utils/performance_issues/detectors/file_io_main_thread_detector.py
@@ -104,6 +104,8 @@ def on_complete(self):
                     type=PerformanceFileIOMainThreadGroupType,
                     cause_span_ids=[],
                     offender_span_ids=[span["span_id"] for span in span_list if "span_id" in span],
+                    evidence_data={},
+                    evidence_display=[],
                 )
 
     def _fingerprint(self, span_list) -> str:
diff --git a/src/sentry/utils/performance_issues/detectors/mn_plus_one_db_span_detector.py b/src/sentry/utils/performance_issues/detectors/mn_plus_one_db_span_detector.py
index ba0cc5084ff041..12ec44144077c7 100644
--- a/src/sentry/utils/performance_issues/detectors/mn_plus_one_db_span_detector.py
+++ b/src/sentry/utils/performance_issues/detectors/mn_plus_one_db_span_detector.py
@@ -178,6 +178,8 @@ def _maybe_performance_problem(self) -> Optional[PerformanceProblem]:
             parent_span_ids=[parent_span["span_id"]],
             cause_span_ids=[],
             offender_span_ids=[span["span_id"] for span in offender_spans],
+            evidence_data={},
+            evidence_display=[],
         )
 
     def _first_db_span(self) -> Optional[Span]:
diff --git a/src/sentry/utils/performance_issues/detectors/n_plus_one_api_calls_detector.py b/src/sentry/utils/performance_issues/detectors/n_plus_one_api_calls_detector.py
index 4da85efc12cb74..f42cbdf31898cf 100644
--- a/src/sentry/utils/performance_issues/detectors/n_plus_one_api_calls_detector.py
+++ b/src/sentry/utils/performance_issues/detectors/n_plus_one_api_calls_detector.py
@@ -218,6 +218,8 @@ def _maybe_store_problem(self):
             cause_span_ids=[],
             parent_span_ids=[last_span.get("parent_span_id", None)],
             offender_span_ids=[span["span_id"] for span in self.spans],
+            evidence_data={},
+            evidence_display=[],
         )
 
     def _fingerprint(self) -> Optional[str]:
diff --git a/src/sentry/utils/performance_issues/detectors/n_plus_one_db_span_detector.py b/src/sentry/utils/performance_issues/detectors/n_plus_one_db_span_detector.py
index aa9e0a1a09e799..666778596754ba 100644
--- a/src/sentry/utils/performance_issues/detectors/n_plus_one_db_span_detector.py
+++ b/src/sentry/utils/performance_issues/detectors/n_plus_one_db_span_detector.py
@@ -6,6 +6,7 @@
 from typing import Optional
 
 from sentry.issues.grouptype import PerformanceNPlusOneGroupType
+from sentry.issues.issue_occurrence import IssueEvidence
 from sentry.models import Organization, Project
 from sentry.utils import metrics
 from sentry.utils.safe import get_path
@@ -52,6 +53,7 @@ class NPlusOneDBSpanDetector(PerformanceDetector):
         "source_span",
         "n_hash",
         "n_spans",
+        "transaction",
     )
 
     type: DetectorType = DetectorType.N_PLUS_ONE_DB_QUERIES
@@ -202,6 +204,8 @@ def _maybe_store_problem(self):
         )
         if fingerprint not in self.stored_problems:
             self._metrics_for_extra_matching_spans()
+            n_span_op = self.n_spans[0].get("op", "")
+            n_span_desc = self.n_spans[0].get("description", "")
 
             self.stored_problems[fingerprint] = PerformanceProblem(
                 fingerprint=fingerprint,
@@ -211,6 +215,28 @@ def _maybe_store_problem(self):
                 parent_span_ids=[parent_span_id],
                 cause_span_ids=[self.source_span.get("span_id", None)],
                 offender_span_ids=[span.get("span_id", None) for span in self.n_spans],
+                evidence_display=[
+                    IssueEvidence(
+                        name="Transaction Name",
+                        value=self._event.get("transaction", ""),
+                        important=True,
+                    ),
+                    IssueEvidence(
+                        name="Parent Span",
+                        value=parent_span.get("description", ""),
+                        important=True,
+                    ),
+                    IssueEvidence(
+                        name=f"Repeating Spans ({len(self.n_spans)})",
+                        value=f"{n_span_op} - {n_span_desc}",
+                        important=True,
+                    ),
+                ],
+                evidence_data={
+                    "parent_span_ids": [parent_span_id],
+                    "cause_span_ids": [self.source_span.get("span_id", None)],
+                    "offender_span_ids": [span.get("span_id", None) for span in self.n_spans],
+                },
             )
 
     def _contains_valid_repeating_query(self, span: Span) -> bool:
diff --git a/src/sentry/utils/performance_issues/detectors/render_blocking_asset_span_detector.py b/src/sentry/utils/performance_issues/detectors/render_blocking_asset_span_detector.py
index 52520f62070824..a5c7149a7331db 100644
--- a/src/sentry/utils/performance_issues/detectors/render_blocking_asset_span_detector.py
+++ b/src/sentry/utils/performance_issues/detectors/render_blocking_asset_span_detector.py
@@ -71,6 +71,8 @@ def visit_span(self, span: Span):
                     offender_span_ids=[span_id],
                     parent_span_ids=[],
                     cause_span_ids=[],
+                    evidence_data={},
+                    evidence_display=[],
                 )
 
         # If we visit a span that starts after FCP, then we know we've already
diff --git a/src/sentry/utils/performance_issues/detectors/slow_db_query_detector.py b/src/sentry/utils/performance_issues/detectors/slow_db_query_detector.py
index 1428cf28538028..70315c815186ee 100644
--- a/src/sentry/utils/performance_issues/detectors/slow_db_query_detector.py
+++ b/src/sentry/utils/performance_issues/detectors/slow_db_query_detector.py
@@ -60,6 +60,8 @@ def visit_span(self, span: Span):
                 cause_span_ids=[],
                 parent_span_ids=[],
                 offender_span_ids=spans_involved,
+                evidence_data={},
+                evidence_display=[],
             )
 
     def is_creation_allowed_for_organization(self, organization: Optional[Organization]) -> bool:
diff --git a/src/sentry/utils/performance_issues/detectors/uncompressed_asset_detector.py b/src/sentry/utils/performance_issues/detectors/uncompressed_asset_detector.py
index b8d01243941451..96bb44bf62e78a 100644
--- a/src/sentry/utils/performance_issues/detectors/uncompressed_asset_detector.py
+++ b/src/sentry/utils/performance_issues/detectors/uncompressed_asset_detector.py
@@ -79,6 +79,8 @@ def visit_span(self, span: Span) -> None:
                 type=PerformanceUncompressedAssetsGroupType,
                 cause_span_ids=[],
                 offender_span_ids=[span.get("span_id", None)],
+                evidence_data={},
+                evidence_display=[],
             )
 
     def _fingerprint(self, span) -> str:
diff --git a/src/sentry/utils/performance_issues/performance_problem.py b/src/sentry/utils/performance_issues/performance_problem.py
index 41dc728cf14c8d..d6646be6e33917 100644
--- a/src/sentry/utils/performance_issues/performance_problem.py
+++ b/src/sentry/utils/performance_issues/performance_problem.py
@@ -2,6 +2,7 @@
 from typing import Any, Mapping, Optional, Sequence
 
 from sentry.issues.grouptype import GroupType, get_group_type_by_type_id
+from sentry.issues.issue_occurrence import IssueEvidence
 
 
 @dataclass
@@ -15,6 +16,14 @@ class PerformanceProblem:
     cause_span_ids: Optional[Sequence[str]]
     # The actual bad spans
     offender_span_ids: Sequence[str]
+    # Evidence to be used for the group
+    # TODO make evidence_data and evidence_display required once all detectors have been migrated to platform
+    # We can't make it required until we stop loading these from nodestore via EventPerformanceProblem,
+    # since there's legacy data in there that won't have these fields.
+    # So until we disable transaction based perf issues we'll need to keep this optional.
+    evidence_data: Optional[Mapping[str, Any]]
+    # User-friendly evidence to be displayed directly
+    evidence_display: Optional[Sequence[IssueEvidence]]
 
     def to_dict(
         self,
@@ -27,6 +36,8 @@ def to_dict(
             "parent_span_ids": self.parent_span_ids,
             "cause_span_ids": self.cause_span_ids,
             "offender_span_ids": self.offender_span_ids,
+            "evidence_data": self.evidence_data,
+            "evidence_display": [evidence.to_dict() for evidence in self.evidence_display],
         }
 
     @property
@@ -43,6 +54,11 @@ def from_dict(cls, data: dict):
             data["parent_span_ids"],
             data["cause_span_ids"],
             data["offender_span_ids"],
+            data.get("evidence_data", {}),
+            [
+                IssueEvidence(evidence["name"], evidence["value"], evidence["important"])
+                for evidence in data["evidence_display"]
+            ],
         )
 
     def __eq__(self, other):
diff --git a/src/sentry/utils/pytest/sentry.py b/src/sentry/utils/pytest/sentry.py
index 4c9b2bc51f4d48..b866036c2eeccf 100644
--- a/src/sentry/utils/pytest/sentry.py
+++ b/src/sentry/utils/pytest/sentry.py
@@ -138,6 +138,8 @@ def pytest_configure(config):
     settings.SENTRY_RATELIMITER = "sentry.ratelimits.redis.RedisRateLimiter"
     settings.SENTRY_RATELIMITER_OPTIONS = {}
 
+    settings.SENTRY_ISSUE_PLATFORM_FUTURES_MAX_LIMIT = 1
+
     if not hasattr(settings, "SENTRY_OPTIONS"):
         settings.SENTRY_OPTIONS = {}
 
diff --git a/tests/sentry/api/serializers/test_event.py b/tests/sentry/api/serializers/test_event.py
index 5cf572ff2e11d6..7be1cded3a69a6 100644
--- a/tests/sentry/api/serializers/test_event.py
+++ b/tests/sentry/api/serializers/test_event.py
@@ -428,6 +428,31 @@ def test_performance_problem(self):
             "parentSpanIds": ["8dd7a5869a4f4583"],
             "issueType": "performance_n_plus_one_db_queries",
             "type": 1006,
+            "evidenceData": {
+                "causeSpanIds": ["9179e43ae844b174"],
+                "offenderSpanIds": [
+                    "b8be6138369491dd",
+                    "b2d4826e7b618f1b",
+                    "b3fdeea42536dbf1",
+                    "b409e78a092e642f",
+                    "86d2ede57bbf48d4",
+                    "8e554c84cdc9731e",
+                    "94d6230f3f910e12",
+                    "a210b87a2191ceb6",
+                    "88a5ccaf25b9bd8f",
+                    "bb32cf50fc56b296",
+                ],
+                "parentSpanIds": ["8dd7a5869a4f4583"],
+            },
+            "evidenceDisplay": [
+                {"important": True, "name": "Transaction Name", "value": "/books/"},
+                {"important": True, "name": "Parent Span", "value": "index"},
+                {
+                    "important": True,
+                    "name": "Repeating Spans (10)",
+                    "value": "db - SELECT `books_author`.`id`, `books_author`.`name` FROM `books_author` WHERE `books_author`.`id` = %s LIMIT 21",
+                },
+            ],
         }
 
     @override_options({"performance.issues.all.problem-detection": 1.0})
diff --git a/tests/sentry/event_manager/test_event_manager.py b/tests/sentry/event_manager/test_event_manager.py
index 056cd64634da30..b0ac3743cb7d58 100644
--- a/tests/sentry/event_manager/test_event_manager.py
+++ b/tests/sentry/event_manager/test_event_manager.py
@@ -2375,6 +2375,8 @@ def test_perf_issue_creation(self):
                     "88a5ccaf25b9bd8f",
                     "bb32cf50fc56b296",
                 ],
+                {},
+                [],
             )
 
     @override_options({"performance.issues.all.problem-detection": 1.0})
diff --git a/tests/sentry/notifications/test_utils.py b/tests/sentry/notifications/test_utils.py
index 01cde16c6a2af4..28aab65342c7af 100644
--- a/tests/sentry/notifications/test_utils.py
+++ b/tests/sentry/notifications/test_utils.py
@@ -241,6 +241,8 @@ def test_creates_correct_context(self):
                     parent_span_ids=[],
                     cause_span_ids=[],
                     offender_span_ids=[],
+                    evidence_data={},
+                    evidence_display=[],
                 ),
                 [],
             ).__class__
@@ -257,6 +259,8 @@ def test_creates_correct_context(self):
                     parent_span_ids=[],
                     cause_span_ids=[],
                     offender_span_ids=[],
+                    evidence_data={},
+                    evidence_display=[],
                 ),
                 [],
             ).__class__
@@ -275,6 +279,8 @@ def test_returns_n_plus_one_db_query_context(self):
                 parent_span_ids=["b93d2be92cd64fd5"],
                 cause_span_ids=[],
                 offender_span_ids=["054ba3a374d543eb"],
+                evidence_data={},
+                evidence_display=[],
             ),
             [
                 {"span_id": "b93d2be92cd64fd5", "description": "SELECT * FROM parent_table"},
@@ -302,6 +308,8 @@ def test_returns_n_plus_one_api_call_context(self):
                 parent_span_ids=[],
                 cause_span_ids=[],
                 offender_span_ids=["b93d2be92cd64fd5", "054ba3a374d543eb", "563712f9722fb09"],
+                evidence_data={},
+                evidence_display=[],
             ),
             [
                 {
@@ -342,6 +350,8 @@ def test_returns_render_blocking_asset_context(self):
                 parent_span_ids=[],
                 cause_span_ids=[],
                 offender_span_ids=["b93d2be92cd64fd5"],
+                evidence_data={},
+                evidence_display=[],
             ),
             [
                 {
diff --git a/tests/sentry/utils/performance_issues/test_consecutive_db_detector.py b/tests/sentry/utils/performance_issues/test_consecutive_db_detector.py
index 2f96341931daef..2405d5eba74f28 100644
--- a/tests/sentry/utils/performance_issues/test_consecutive_db_detector.py
+++ b/tests/sentry/utils/performance_issues/test_consecutive_db_detector.py
@@ -76,6 +76,8 @@ def test_detects_consecutive_db_spans(self):
                 parent_span_ids=None,
                 cause_span_ids=["bbbbbbbbbbbbbbbb", "bbbbbbbbbbbbbbbb", "bbbbbbbbbbbbbbbb"],
                 offender_span_ids=["bbbbbbbbbbbbbbbb", "bbbbbbbbbbbbbbbb"],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
@@ -183,6 +185,8 @@ def test_detects_consecutive_db_with_high_time_saving(self):
                 parent_span_ids=None,
                 cause_span_ids=["bbbbbbbbbbbbbbbb", "bbbbbbbbbbbbbbbb", "bbbbbbbbbbbbbbbb"],
                 offender_span_ids=["bbbbbbbbbbbbbbbb"],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
diff --git a/tests/sentry/utils/performance_issues/test_consecutive_http_detector.py b/tests/sentry/utils/performance_issues/test_consecutive_http_detector.py
index 14d3948a384842..2ff30cc4839ffd 100644
--- a/tests/sentry/utils/performance_issues/test_consecutive_http_detector.py
+++ b/tests/sentry/utils/performance_issues/test_consecutive_http_detector.py
@@ -71,6 +71,8 @@ def test_detects_consecutive_http_issue(self):
                     "bbbbbbbbbbbbbbbb",
                     "bbbbbbbbbbbbbbbb",
                 ],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
@@ -104,6 +106,8 @@ def test_detects_consecutive_with_non_http_between_http_spans(self):
                     "bbbbbbbbbbbbbbbb",
                     "bbbbbbbbbbbbbbbb",
                 ],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
diff --git a/tests/sentry/utils/performance_issues/test_file_io_on_main_thread_detector.py b/tests/sentry/utils/performance_issues/test_file_io_on_main_thread_detector.py
index 7e84b10bbaa920..6fb8c4e797d744 100644
--- a/tests/sentry/utils/performance_issues/test_file_io_on_main_thread_detector.py
+++ b/tests/sentry/utils/performance_issues/test_file_io_on_main_thread_detector.py
@@ -62,6 +62,8 @@ def test_detects_file_io_main_thread(self):
                 parent_span_ids=["b93d2be92cd64fd5"],
                 cause_span_ids=[],
                 offender_span_ids=["054ba3a374d543eb"],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
diff --git a/tests/sentry/utils/performance_issues/test_m_n_plus_one_db_detector.py b/tests/sentry/utils/performance_issues/test_m_n_plus_one_db_detector.py
index 030d958168b942..904c41bbc81488 100644
--- a/tests/sentry/utils/performance_issues/test_m_n_plus_one_db_detector.py
+++ b/tests/sentry/utils/performance_issues/test_m_n_plus_one_db_detector.py
@@ -68,6 +68,8 @@ def test_detects_parallel_m_n_plus_one(self):
                     "a96783f2f544024a",
                     "8e110c4aa54e4aa0",
                 ],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
         assert problems[0].title == "N+1 Query"
diff --git a/tests/sentry/utils/performance_issues/test_n_plus_one_api_calls_detector.py b/tests/sentry/utils/performance_issues/test_n_plus_one_api_calls_detector.py
index 9f755cac0ecb39..241f84a39861ff 100644
--- a/tests/sentry/utils/performance_issues/test_n_plus_one_api_calls_detector.py
+++ b/tests/sentry/utils/performance_issues/test_n_plus_one_api_calls_detector.py
@@ -100,6 +100,8 @@ def test_detects_problems_with_many_concurrent_calls_to_same_url(self):
                     "a34089b08b6d0646",
                     "950801c0d7576650",
                 ],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
         assert problems[0].title == "N+1 API Call"
diff --git a/tests/sentry/utils/performance_issues/test_n_plus_one_db_span_detector.py b/tests/sentry/utils/performance_issues/test_n_plus_one_db_span_detector.py
index b5d4b7a68c5a34..1574a166a78ccb 100644
--- a/tests/sentry/utils/performance_issues/test_n_plus_one_db_span_detector.py
+++ b/tests/sentry/utils/performance_issues/test_n_plus_one_db_span_detector.py
@@ -88,6 +88,8 @@ def test_finds_n_plus_one_with_db_dot_something_spans(self):
                     "88a5ccaf25b9bd8f",
                     "bb32cf50fc56b296",
                 ],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
@@ -140,6 +142,8 @@ def test_detects_n_plus_one_with_multiple_potential_sources(self):
                     "be85dffe4a9a3120",
                     "a3c381b1952dd7fb",
                 ],
+                evidence_data={},
+                evidence_display=[],
             ),
         ]
 
diff --git a/tests/sentry/utils/performance_issues/test_performance_detection.py b/tests/sentry/utils/performance_issues/test_performance_detection.py
index 9f5e0e497d1c14..d4d992ea82e729 100644
--- a/tests/sentry/utils/performance_issues/test_performance_detection.py
+++ b/tests/sentry/utils/performance_issues/test_performance_detection.py
@@ -59,6 +59,8 @@ def assert_n_plus_one_db_problem(perf_problems):
                 "88a5ccaf25b9bd8f",
                 "bb32cf50fc56b296",
             ],
+            evidence_data={},
+            evidence_display=[],
         )
         for problem in perf_problems
     )
@@ -136,6 +138,8 @@ def test_n_plus_one_extended_detection_no_parent_span(self):
                     "9c3a569621230f03",
                     "8788fb3fc43ad948",
                 ],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
@@ -207,6 +211,8 @@ def test_boolean_system_option_enables_detector_issue_creation(self):
                         "a307ceb77c702cea",
                         "ac1e90ff646617e7",
                     ],
+                    evidence_data={},
+                    evidence_display=[],
                 )
             ]
 
@@ -385,6 +391,8 @@ def test_save_and_fetch(self):
             ["1"],
             ["2", "3", "4"],
             ["4", "5", "6"],
+            {},
+            [],
         )
 
         EventPerformanceProblem(event, problem).save()
@@ -401,6 +409,8 @@ def test_fetch_multi(self):
                 ["1"],
                 ["2", "3", "4"],
                 ["4", "5", "6"],
+                {},
+                [],
             ),
             PerformanceProblem(
                 "test_2",
@@ -410,6 +420,8 @@ def test_fetch_multi(self):
                 ["234"],
                 ["67", "87686", "786"],
                 ["4", "5", "6"],
+                {},
+                [],
             ),
         ]
         event_2 = Event(self.project.id, "something else")
@@ -422,6 +434,8 @@ def test_fetch_multi(self):
                 ["1"],
                 ["a", "b", "c"],
                 ["d", "e", "f"],
+                {},
+                [],
             ),
             PerformanceProblem(
                 "event_2_test_2",
@@ -431,6 +445,8 @@ def test_fetch_multi(self):
                 ["234"],
                 ["fdgh", "gdhgf", "gdgh"],
                 ["gdf", "yu", "kjl"],
+                {},
+                [],
             ),
         ]
         all_event_problems = [
@@ -449,6 +465,8 @@ def test_fetch_multi(self):
             ["234"],
             ["fdgh", "gdhgf", "gdgh"],
             ["gdf", "yu", "kjl"],
+            {},
+            [],
         )
         result = EventPerformanceProblem.fetch_multi(
             [
diff --git a/tests/sentry/utils/performance_issues/test_render_blocking_asset_detector.py b/tests/sentry/utils/performance_issues/test_render_blocking_asset_detector.py
index a4c2613268c671..43b451f6499a1c 100644
--- a/tests/sentry/utils/performance_issues/test_render_blocking_asset_detector.py
+++ b/tests/sentry/utils/performance_issues/test_render_blocking_asset_detector.py
@@ -79,6 +79,8 @@ def test_detects_render_blocking_asset(self):
                 parent_span_ids=[],
                 cause_span_ids=[],
                 offender_span_ids=["bbbbbbbbbbbbbbbb"],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
@@ -144,6 +146,8 @@ def test_detects_if_render_blocking_status_is_missing(self):
                 parent_span_ids=[],
                 cause_span_ids=[],
                 offender_span_ids=["bbbbbbbbbbbbbbbb"],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
diff --git a/tests/sentry/utils/performance_issues/test_slow_db_span_detector.py b/tests/sentry/utils/performance_issues/test_slow_db_span_detector.py
index 354c43d100d7d3..eb697f28afccbb 100644
--- a/tests/sentry/utils/performance_issues/test_slow_db_span_detector.py
+++ b/tests/sentry/utils/performance_issues/test_slow_db_span_detector.py
@@ -47,6 +47,8 @@ def test_calls_detect_slow_span(self):
                 parent_span_ids=None,
                 cause_span_ids=None,
                 offender_span_ids=["bbbbbbbbbbbbbbbb"],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
@@ -70,6 +72,8 @@ def test_calls_slow_span_threshold(self):
                 parent_span_ids=[],
                 cause_span_ids=[],
                 offender_span_ids=["bbbbbbbbbbbbbbbb"],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
@@ -85,6 +89,8 @@ def test_detects_slow_span_in_solved_n_plus_one_query(self):
                 parent_span_ids=None,
                 cause_span_ids=None,
                 offender_span_ids=["a05754d3fde2db29"],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
@@ -106,6 +112,8 @@ def test_skips_truncated_queries(self):
                 parent_span_ids=[],
                 cause_span_ids=[],
                 offender_span_ids=["bbbbbbbbbbbbbbbb"],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
diff --git a/tests/sentry/utils/performance_issues/test_uncompressed_assets_detector.py b/tests/sentry/utils/performance_issues/test_uncompressed_assets_detector.py
index 64837d83d6feae..dea21de560316a 100644
--- a/tests/sentry/utils/performance_issues/test_uncompressed_assets_detector.py
+++ b/tests/sentry/utils/performance_issues/test_uncompressed_assets_detector.py
@@ -77,6 +77,8 @@ def test_detects_uncompressed_asset(self):
                 parent_span_ids=[],
                 cause_span_ids=[],
                 offender_span_ids=["bbbbbbbbbbbbbbbb"],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
@@ -109,6 +111,8 @@ def test_detects_uncompressed_asset_stylesheet(self):
                 parent_span_ids=[],
                 cause_span_ids=[],
                 offender_span_ids=["bbbbbbbbbbbbbbbb"],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
@@ -141,6 +145,8 @@ def test_does_not_detect_woff_asset(self):
                 parent_span_ids=[],
                 cause_span_ids=[],
                 offender_span_ids=["bbbbbbbbbbbbbbbb"],
+                evidence_data={},
+                evidence_display=[],
             )
         ]
 
@@ -260,6 +266,8 @@ def test_detects_problems_from_event(self):
                 parent_span_ids=[],
                 cause_span_ids=[],
                 offender_span_ids=["b66a5642da1edb52"],
+                evidence_data={},
+                evidence_display=[],
             ),
         ]
 


Code review comments:
diff_hunk: @@ -0,0 +1,32 @@
+from typing import Any, Mapping
+
+from sentry.issues.grouptype import PerformanceNPlusOneGroupType
+from sentry.issues.issue_occurrence import IssueOccurrence
+from sentry.models import Project
+from sentry.utils.performance_issues.performance_problem import PerformanceProblem
+
+
+def can_create_group(
+    ocurrence: IssueOccurrence, options: Mapping[str, Any], project: Project
path: src/sentry/issues/utils.py
text: @user1:
Why do we pass options in here instead of importing?

@author:
oh I thought passing in a dependency explicitly is better but yeah fixed it to use the import instead