Repository: getsentry/sentry

Issue:
Possible race condition on group creation during large bursts of events within a single group.
A user on sentry.io received a burst of ~7,500 events over a 15 minute period the other day. Each of these events were fingerprinted with the same fingerprint. Somehow, this resulted in 5 different groups being created at one point. Two of those groups were eventually merged into the other groups, leaving 3 groups that were created during the burst. Only one of them was finally associated with the `GroupHash`, and this group contained the majority of the events.

From what I can tell, several workers must have been attempting to create the group at the same time, and when the [`_ensure_hashes_merged`](https://github.com/getsentry/sentry/blob/70f41bcf910e388ba5963f32d817c1167762b359/src/sentry/event_manager.py#L765-L790) method was called, it triggered merges into groups that themselves may have already been being merged (and subsequently deleted) so some of the merges triggered by the `_ensure_hashes_merged` method didn't complete successfully.

There's not enough logging to put together a comprehensive view of what merges were in flight during the time period and how far along they got, but this is what *seems* to have occurred given the final state of the records.


Discussion:
I've seen this too (also on sentry.io, ~31 events over a 2-second period. 2 events about 20 seconds later were correctly grouped combined into one of the existing groups.) The events were all fingerprinted with the same fingerprint but we ended up with about 7 different groups, each with a different "Issue #"
Closing this issue due to staleness. Feel free to comment here if you think we should still work on this.
This is very much still an issue (and an important one at that, IMO.)
@tkaemming this hasn't been updated in over 3 years so even if it is important, it doesn't seem pressing or being prioritized. I'm reopening but we should surface this for planning if it is important to tackle.
I can confirm I'm still seeing that all the time as well, e.g. https://sentry.io/organizations/freelancercom/issues/2021706113 vs. https://sentry.io/organizations/freelancercom/issues/2021706114 from a week ago. Given how important issue grouping is in the overall product experience, I'd really wish the Sentry team would prioritize fixing that.

Resolving patch:
diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py
--- a/src/sentry/conf/server.py
+++ b/src/sentry/conf/server.py
@@ -995,6 +995,8 @@ def create_partitioned_queues(name):
     "projects:minidump": True,
     # Enable functionality for project plugins.
     "projects:plugins": True,
+    # Enable alternative version of group creation that is supposed to be less racy.
+    "projects:race-free-group-creation": False,
     # Enable functionality for rate-limiting events on projects.
     "projects:rate-limits": True,
     # Enable functionality for sampling of events on projects.
diff --git a/src/sentry/event_manager.py b/src/sentry/event_manager.py
--- a/src/sentry/event_manager.py
+++ b/src/sentry/event_manager.py
@@ -9,6 +9,7 @@
 from django.db.models import Func
 from django.utils.encoding import force_text
 from pytz import UTC
+import sentry_sdk
 
 from sentry import buffer, eventstore, eventtypes, eventstream, features, tsdb
 from sentry.attachments import MissingAttachmentChunks, attachment_cache
@@ -395,8 +396,14 @@ def save(self, project_id, raw=False, assume_normalized=False, start_time=None,
         with metrics.timer("event_manager.get_attachments"):
             attachments = get_attachments(cache_key, job)
 
+        save_aggregate_fn = (
+            _save_aggregate2
+            if features.has("projects:race-free-group-creation", project)
+            else _save_aggregate
+        )
+
         try:
-            job["group"], job["is_new"], job["is_regression"] = _save_aggregate(
+            job["group"], job["is_new"], job["is_regression"] = save_aggregate_fn(
                 event=job["event"],
                 flat_hashes=flat_hashes,
                 hierarchical_hashes=hierarchical_hashes,
@@ -908,6 +915,124 @@ def get_culprit(data):
     )
 
 
+def _find_group_id(all_hashes):
+    for h in all_hashes:
+        if h.group_id is not None:
+            return h.group_id
+        if h.group_tombstone_id is not None:
+            raise HashDiscarded("Matches group tombstone %s" % h.group_tombstone_id)
+
+    return None
+
+
+def _save_aggregate2(event, flat_hashes, hierarchical_hashes, release, **kwargs):
+    """
+    A rewrite of _save_aggregate that is supposed to eliminate races using DB transactions.
+    """
+
+    # TODO(markus): Port over hierarchical grouping changes from _save_aggregate
+
+    project = event.project
+
+    all_hashes = [
+        GroupHash.objects.get_or_create(project=project, hash=hash)[0] for hash in flat_hashes
+    ]
+    existing_group_id = _find_group_id(all_hashes)
+
+    if existing_group_id is None:
+        with sentry_sdk.start_span(
+            op="event_manager.create_group_transaction"
+        ) as span, metrics.timer(
+            "event_manager.create_group_transaction"
+        ) as metric_tags, transaction.atomic():
+            span.set_tag("create_group_transaction.outcome", "no_group")
+            metric_tags["create_group_transaction.outcome"] = "no_group"
+
+            all_hashes = list(
+                GroupHash.objects.filter(id__in=[h.id for h in all_hashes]).select_for_update()
+            )
+
+            existing_group_id = _find_group_id(all_hashes)
+
+            if existing_group_id is None:
+                short_id = project.next_short_id()
+
+                # it's possible the release was deleted between
+                # when we queried for the release and now, so
+                # make sure it still exists
+                first_release = kwargs.pop("first_release", None)
+
+                group = Group.objects.create(
+                    project=project,
+                    short_id=short_id,
+                    first_release_id=Release.objects.filter(id=first_release.id)
+                    .values_list("id", flat=True)
+                    .first()
+                    if first_release
+                    else None,
+                    **kwargs,
+                )
+
+                # invariant: existing_group_id is None, therefore all hashes
+                # have group_id=None, therefore none of them can be locked in
+                # migration either
+                GroupHash.objects.filter(id__in=[h.id for h in all_hashes]).update(group=group)
+
+                is_new = True
+                is_regression = False
+
+                span.set_tag("create_group_transaction.outcome", "new_group")
+                metric_tags["create_group_transaction.outcome"] = "new_group"
+
+                metrics.incr(
+                    "group.created",
+                    skip_internal=True,
+                    tags={"platform": event.platform or "unknown"},
+                )
+
+                return group, is_new, is_regression
+
+    group = Group.objects.get(id=existing_group_id)
+
+    is_new = False
+    new_hashes = [h for h in all_hashes if h.group_id is None]
+
+    if new_hashes:
+        # There may still be secondary hashes that we did not use to find an
+        # existing group. A classic example is when grouping makes changes to
+        # the app-hash (changes to in_app logic), but the system hash stays
+        # stable and is used to find an existing group. Associate any new
+        # hashes with the group such that event saving continues to be
+        # resilient against grouping algorithm changes.
+        #
+        # There is a race condition here where two processes could "steal"
+        # hashes from each other. In practice this should not be user-visible
+        # as group creation is synchronized. Meaning the only way hashes could
+        # jump between groups is if there were two processes that:
+        #
+        # 1) have BOTH found an existing group
+        #    (otherwise at least one of them would be in the group creation
+        #    codepath which has transaction isolation/acquires row locks)
+        # 2) AND are looking at the same set, or an overlapping set of hashes
+        #    (otherwise they would not operate on the same rows)
+        # 3) yet somehow also sort their event into two different groups each
+        #    (otherwise the update would not change anything)
+        #
+        # We think this is a very unlikely situation. A previous version of
+        # _save_aggregate had races around group creation which made this race
+        # more user visible. For more context, see 84c6f75a and d0e22787, as
+        # well as GH-5085.
+        GroupHash.objects.filter(id__in=[h.id for h in new_hashes]).exclude(
+            state=GroupHash.State.LOCKED_IN_MIGRATION
+        ).update(group=group)
+
+    is_regression = _process_existing_aggregate(
+        group=group, event=event, data=kwargs, release=release
+    )
+
+    return group, is_new, is_regression
+
+
 def _find_existing_group_id(
     project,
     flat_grouphashes,
diff --git a/src/sentry/features/__init__.py b/src/sentry/features/__init__.py
--- a/src/sentry/features/__init__.py
+++ b/src/sentry/features/__init__.py
@@ -136,6 +136,7 @@
 default_manager.add("projects:import-codeowners", ProjectFeature)  # NOQA
 default_manager.add("projects:issue-alerts-targeting", ProjectFeature)  # NOQA
 default_manager.add("projects:minidump", ProjectFeature)  # NOQA
+default_manager.add("projects:race-free-group-creation", ProjectFeature)  # NOQA
 default_manager.add("projects:rate-limits", ProjectFeature)  # NOQA
 default_manager.add("projects:sample-events", ProjectFeature)  # NOQA
 default_manager.add("projects:servicehooks", ProjectFeature)  # NOQA


Test patch:
diff --git a/tests/sentry/event_manager/test_event_manager.py b/tests/sentry/event_manager/test_event_manager.py
--- a/tests/sentry/event_manager/test_event_manager.py
+++ b/tests/sentry/event_manager/test_event_manager.py
@@ -42,6 +42,7 @@
 from sentry.utils.cache import cache_key_for_event
 from sentry.utils.outcomes import Outcome
 from sentry.testutils import assert_mock_called_once_with_partial, TestCase
+from sentry.testutils.helpers import Feature
 from sentry.ingest.inbound_filters import FilterStatKeys
 
 
@@ -1541,3 +1542,10 @@ def test_same_group_different_environment(self):
             last_seen=self.timestamp + 100,
             first_seen=self.timestamp + 100,
         )
+
+
+class RaceFreeEventManagerTest(EventManagerTest):
+    @pytest.fixture(autouse=True)
+    def _save_aggregate_parameterized(self):
+        with Feature({"projects:race-free-group-creation": True}):
+            yield
diff --git a/tests/sentry/event_manager/test_save_aggregate.py b/tests/sentry/event_manager/test_save_aggregate.py
new file mode 100644
--- /dev/null
+++ b/tests/sentry/event_manager/test_save_aggregate.py
@@ -0,0 +1,95 @@
+import contextlib
+import pytest
+import time
+
+from threading import Thread
+
+from sentry.eventstore.models import Event
+from sentry.event_manager import _save_aggregate, _save_aggregate2
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize(
+    "save_aggregate_version",
+    [
+        # New group creation code, which is supposed to not have races
+        "new_save_aggregate",
+        # New group creation code with removed transaction isolation, which is then
+        # supposed to create multiple groups. This variant exists such that we can
+        # ensure the test would find race conditions in principle, and does not
+        # just always pass because of low parallelism. In a sense this variant
+        # tests the efficacy of this test, not actual business logic.
+        #
+        # If this variant fails, CONCURRENCY needs to be increased or e.g. thread
+        # barriers need to be used to ensure data races. This does not seem to be
+        # necessary so far.
+        "new_broken_save_aggregate",
+        # Old group creation code which is "supposed to" have races
+        "old_save_aggregate",
+    ],
+)
+def test_group_creation_race(monkeypatch, default_project, save_aggregate_version):
+    CONCURRENCY = 2
+
+    new_variant = save_aggregate_version in ("new_save_aggregate", "new_broken_save_aggregate")
+    is_race_free = save_aggregate_version == "new_save_aggregate"
+
+    if save_aggregate_version == "new_broken_save_aggregate":
+
+        class FakeTransactionModule:
+            @staticmethod
+            @contextlib.contextmanager
+            def atomic():
+                yield
+
+        # Disable transaction isolation just within event manager, but not in
+        # GroupHash.objects.create_or_update
+        monkeypatch.setattr("sentry.event_manager.transaction", FakeTransactionModule)
+
+        # select_for_update cannot be used outside of transactions
+        monkeypatch.setattr("django.db.models.QuerySet.select_for_update", lambda self: self)
+
+    if new_variant:
+        save_aggregate = _save_aggregate2
+    else:
+        save_aggregate = _save_aggregate
+
+    return_values = []
+
+    def save_event():
+        data = {"timestamp": time.time()}
+        evt = Event(
+            default_project.id,
+            "89aeed6a472e4c5fb992d14df4d7e1b6",
+            data=data,
+        )
+
+        return_values.append(
+            save_aggregate(
+                evt,
+                flat_hashes=["a" * 32, "b" * 32],
+                hierarchical_hashes=[],
+                release=None,
+                data=data,
+                level=10,
+                culprit="",
+            )
+        )
+
+    threads = []
+    for _ in range(CONCURRENCY):
+        thread = Thread(target=save_event)
+        thread.start()
+        threads.append(thread)
+
+    for thread in threads:
+        thread.join()
+
+    if is_race_free:
+        # assert one group is new
+        assert len({rv[0].id for rv in return_values}) == 1
+        assert sum(rv[1] for rv in return_values) == 1
+    else:
+        # assert many groups are new
+        assert 1 < len({rv[0].id for rv in return_values}) <= CONCURRENCY
+        assert 1 < sum(rv[1] for rv in return_values) <= CONCURRENCY