Repository: getsentry/sentry
Pull request #90347: fix(issue summary): Add locking mechanism
Author: login: roaga
Labels: name: Scope: Backend
description: Automatically applied to PRs that change backend components
Created: 2025-04-24T22:24:16Z | Merged: 2025-04-28T21:09:36Z

If a summary is in progress for an issue while another request comes in, the other request should wait for a bit first. This ensures only one call to Seer per issue at a time.


Diff:
diff --git a/src/sentry/seer/issue_summary.py b/src/sentry/seer/issue_summary.py
index 142fe926a713cc..9c26a0aff1b932 100644
--- a/src/sentry/seer/issue_summary.py
+++ b/src/sentry/seer/issue_summary.py
@@ -16,6 +16,7 @@
 from sentry.autofix.utils import get_autofix_state
 from sentry.constants import ObjectStatus
 from sentry.eventstore.models import Event, GroupEvent
+from sentry.locks import locks
 from sentry.models.group import Group
 from sentry.models.project import Project
 from sentry.seer.autofix import trigger_autofix
@@ -29,6 +30,7 @@
 from sentry.users.services.user.model import RpcUser
 from sentry.users.services.user.service import user_service
 from sentry.utils.cache import cache
+from sentry.utils.locking import UnableToAcquireLock
 
 logger = logging.getLogger(__name__)
 
@@ -230,33 +232,14 @@ def _get_trace_connected_issues(event: GroupEvent) -> list[Group]:
     return connected_issues
 
 
-def get_issue_summary(
+def _generate_summary(
     group: Group,
-    user: User | RpcUser | AnonymousUser | None = None,
-    force_event_id: str | None = None,
-    source: str = "issue_details",
+    user: User | RpcUser | AnonymousUser,
+    force_event_id: str | None,
+    source: str,
+    cache_key: str,
 ) -> tuple[dict[str, Any], int]:
-    """
-    Generate an AI summary for an issue.
-
-    Args:
-        group: The issue group
-        user: The user requesting the summary
-        force_event_id: Optional event ID to force summarizing a specific event
-        source: The source triggering the summary generation
-
-    Returns:
-        A tuple containing (summary_data, status_code)
-    """
-    if user is None:
-        user = AnonymousUser()
-    if not features.has("organizations:gen-ai-features", group.organization, actor=user):
-        return {"detail": "Feature flag not enabled"}, 400
-
-    cache_key = "ai-group-summary-v2:" + str(group.id)
-    if not force_event_id and (cached_summary := cache.get(cache_key)):
-        return convert_dict_key_case(cached_summary, snake_to_camel_case), 200
-
+    """Core logic to generate and cache the issue summary."""
     serialized_event, event = _get_event(group, user, provided_event_id=force_event_id)
 
     if not serialized_event or not event:
@@ -314,4 +297,67 @@ def get_issue_summary(
 
     cache.set(cache_key, summary_dict, timeout=int(timedelta(days=7).total_seconds()))
 
-    return convert_dict_key_case(summary_dict, snake_to_camel_case), 200
+    return summary_dict, 200
+
+
+def get_issue_summary(
+    group: Group,
+    user: User | RpcUser | AnonymousUser | None = None,
+    force_event_id: str | None = None,
+    source: str = "issue_details",
+) -> tuple[dict[str, Any], int]:
+    """
+    Generate an AI summary for an issue.
+
+    Args:
+        group: The issue group
+        user: The user requesting the summary
+        force_event_id: Optional event ID to force summarizing a specific event
+        source: The source triggering the summary generation
+
+    Returns:
+        A tuple containing (summary_data, status_code)
+    """
+    if user is None:
+        user = AnonymousUser()
+    if not features.has("organizations:gen-ai-features", group.organization, actor=user):
+        return {"detail": "Feature flag not enabled"}, 400
+
+    cache_key = f"ai-group-summary-v2:{group.id}"
+    lock_key = f"ai-group-summary-v2-lock:{group.id}"
+    lock_duration = 10  # How long the lock is held if acquired (seconds)
+    wait_timeout = 4.5  # How long to wait for the lock (seconds)
+
+    # if force_event_id is set, we always generate a new summary
+    if force_event_id:
+        summary_dict, status_code = _generate_summary(
+            group, user, force_event_id, source, cache_key
+        )
+        return convert_dict_key_case(summary_dict, snake_to_camel_case), status_code
+
+    # 1. Check cache first
+    if cached_summary := cache.get(cache_key):
+        return convert_dict_key_case(cached_summary, snake_to_camel_case), 200
+
+    # 2. Try to acquire lock
+    try:
+        # Acquire lock context manager. This will poll and wait.
+        with locks.get(
+            key=lock_key, duration=lock_duration, name="get_issue_summary"
+        ).blocking_acquire(initial_delay=0.25, timeout=wait_timeout):
+            # Re-check cache after acquiring lock, in case another process finished
+            # while we were waiting for the lock.
+            if cached_summary := cache.get(cache_key):
+                return convert_dict_key_case(cached_summary, snake_to_camel_case), 200
+
+            # Lock acquired and cache is still empty, proceed with generation
+            summary_dict, status_code = _generate_summary(
+                group, user, force_event_id, source, cache_key
+            )
+            return convert_dict_key_case(summary_dict, snake_to_camel_case), status_code
+
+    except UnableToAcquireLock:
+        # Failed to acquire lock within timeout. Check cache one last time.
+        if cached_summary := cache.get(cache_key):
+            return convert_dict_key_case(cached_summary, snake_to_camel_case), 200
+        return {"detail": "Timeout waiting for summary generation lock"}, 503
diff --git a/tests/sentry/seer/test_issue_summary.py b/tests/sentry/seer/test_issue_summary.py
index 8f15ede3c4e72b..d75f0f11f16ddd 100644
--- a/tests/sentry/seer/test_issue_summary.py
+++ b/tests/sentry/seer/test_issue_summary.py
@@ -1,10 +1,13 @@
 import datetime
+import threading
+import time
 from typing import Any
 from unittest.mock import ANY, Mock, call, patch
 
 import orjson
 
 from sentry.api.serializers.rest_framework.base import convert_dict_key_case, snake_to_camel_case
+from sentry.locks import locks
 from sentry.models.group import Group
 from sentry.seer.issue_summary import (
     _call_seer,
@@ -17,6 +20,7 @@
 from sentry.testutils.helpers.features import apply_feature_flag_on_cls
 from sentry.testutils.skips import requires_snuba
 from sentry.utils.cache import cache
+from sentry.utils.locking import UnableToAcquireLock
 
 pytestmark = [requires_snuba]
 
@@ -407,3 +411,113 @@ def test_get_event_provided(self, mock_serialize, mock_get_event_by_id):
             ]
         )
         mock_serialize.assert_called_once()
+
+    @patch("sentry.seer.issue_summary._generate_summary")
+    def test_get_issue_summary_concurrent_wait_for_lock(self, mock_generate_summary):
+        """Test that a second request waits for the lock and reads from cache."""
+        cache_key = f"ai-group-summary-v2:{self.group.id}"
+
+        # Mock summary generation to take time and cache the result
+        generated_summary = {"headline": "Generated Summary", "event_id": "gen_event"}
+        cache_key = f"ai-group-summary-v2:{self.group.id}"
+
+        def side_effect_generate(*args, **kwargs):
+            # Simulate work
+            time.sleep(0.3)
+            # Write to cache before returning (simulates behavior after lock release)
+            cache.set(cache_key, generated_summary, timeout=60)
+            return generated_summary, 200
+
+        mock_generate_summary.side_effect = side_effect_generate
+
+        results = {}
+        exceptions = {}
+
+        def target(req_id):
+            try:
+                summary_data, status_code = get_issue_summary(self.group, self.user)
+                results[req_id] = (summary_data, status_code)
+            except Exception as e:
+                exceptions[req_id] = e
+
+        # Start two threads concurrently
+        thread1 = threading.Thread(target=target, args=(1,))
+        thread2 = threading.Thread(target=target, args=(2,))
+
+        thread1.start()
+        # Give thread1 a slight head start, but the lock should handle the race
+        time.sleep(0.01)
+        thread2.start()
+
+        # Wait for both threads to complete
+        thread1.join(timeout=5)
+        thread2.join(timeout=5)
+
+        # Assertions
+        if exceptions:
+            raise AssertionError(f"Threads raised exceptions: {exceptions}")
+
+        assert 1 in results, "Thread 1 did not complete in time"
+        assert 2 in results, "Thread 2 did not complete in time"
+
+        # Both should succeed and get the same summary
+        assert results[1][1] == 200, f"Thread 1 failed with status {results[1][1]}"
+        assert results[2][1] == 200, f"Thread 2 failed with status {results[2][1]}"
+        expected_result = convert_dict_key_case(generated_summary, snake_to_camel_case)
+        assert results[1][0] == expected_result, "Thread 1 returned wrong summary"
+        assert results[2][0] == expected_result, "Thread 2 returned wrong summary"
+
+        # Check that _generate_summary was only called once
+        # (by the thread that acquired the lock)
+        mock_generate_summary.assert_called_once()
+
+        # Ensure the cache contains the final result
+        assert cache.get(cache_key) == generated_summary
+
+    @patch("sentry.seer.issue_summary._generate_summary")
+    def test_get_issue_summary_concurrent_force_event_id_bypasses_lock(self, mock_generate_summary):
+        """Test that force_event_id bypasses lock waiting."""
+        # Mock summary generation
+        forced_summary = {"headline": "Forced Summary", "event_id": "force_event"}
+        mock_generate_summary.return_value = (forced_summary, 200)
+
+        # Ensure cache is empty and lock *could* be acquired if attempted
+        cache_key = f"ai-group-summary-v2:{self.group.id}"
+        lock_key = f"ai-group-summary-v2-lock:{self.group.id}"
+        cache.delete(cache_key)
+
+        locks.get(lock_key, duration=1).release()  # Ensure lock isn't held
+
+        # Call with force_event_id=True
+        summary_data, status_code = get_issue_summary(
+            self.group, self.user, force_event_id="some_event"
+        )
+
+        assert status_code == 200
+        assert summary_data == convert_dict_key_case(forced_summary, snake_to_camel_case)
+
+        # Ensure generation was called directly
+        mock_generate_summary.assert_called_once()
+
+    @patch("sentry.seer.issue_summary.cache.get")
+    @patch("sentry.seer.issue_summary._generate_summary")
+    @patch("sentry.utils.locking.lock.Lock.blocking_acquire")
+    def test_get_issue_summary_lock_timeout(
+        self, mock_blocking_acquire, mock_generate_summary_core, mock_cache_get
+    ):
+        """Test that a timeout waiting for the lock returns 503."""
+        # Simulate lock acquisition always failing with the specific exception
+        mock_blocking_acquire.side_effect = UnableToAcquireLock
+        # Simulate cache miss even after timeout
+        mock_cache_get.return_value = None
+
+        summary_data, status_code = get_issue_summary(self.group, self.user)
+
+        assert status_code == 503
+        assert summary_data == {"detail": "Timeout waiting for summary generation lock"}
+        # Ensure lock acquisition was attempted
+        mock_blocking_acquire.assert_called_once()
+        # Ensure generation was NOT called
+        mock_generate_summary_core.assert_not_called()
+        # Ensure cache was checked twice (once initially, once after lock failure)
+        assert mock_cache_get.call_count == 2


Reviews:
user: login: jennmueng
body: other than the comments not making sense, seems reasonble to me.

Have you tested this extensively locally? Just want to make sure to not cause an issue summary INC...
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T19:52:11Z
user: login: jennmueng
body: I'd say just making sure the locking mechanism works locally should suffice.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T20:36:35Z

Review comments:
diff_hunk: @@ -314,4 +297,67 @@ def get_issue_summary(
 
     cache.set(cache_key, summary_dict, timeout=int(timedelta(days=7).total_seconds()))
 
-    return convert_dict_key_case(summary_dict, snake_to_camel_case), 200
+    return summary_dict, 200
+
+
+def get_issue_summary(
+    group: Group,
+    user: User | RpcUser | AnonymousUser | None = None,
+    force_event_id: str | None = None,
+    source: str = "issue_details",
+) -> tuple[dict[str, Any], int]:
+    """
+    Generate an AI summary for an issue.
+
+    Args:
+        group: The issue group
+        user: The user requesting the summary
+        force_event_id: Optional event ID to force summarizing a specific event
+        source: The source triggering the summary generation
+
+    Returns:
+        A tuple containing (summary_data, status_code)
+    """
+    if user is None:
+        user = AnonymousUser()
+    if not features.has("organizations:gen-ai-features", group.organization, actor=user):
+        return {"detail": "Feature flag not enabled"}, 400
+
+    cache_key = f"ai-group-summary-v2:{group.id}"
+    lock_key = f"ai-group-summary-v2-lock:{group.id}"
+    lock_duration = 4  # How long the lock is held if acquired (seconds)
+    wait_timeout = 5  # How long to wait for the lock (seconds)
+
+    # 1. Check cache first (regardless of force_event_id)
+    if not force_event_id:
+        if cached_summary := cache.get(cache_key):
+            return convert_dict_key_case(cached_summary, snake_to_camel_case), 200
+
+    # 2. If force_event_id is set, skip locking and proceed directly.
+    #    Otherwise, try to acquire the lock.
+    if not force_event_id:
+        try:
+            # Acquire lock context manager. This will poll and wait.
+            with locks.get(key=lock_key, duration=lock_duration).blocking_acquire(
+                initial_delay=0.25, timeout=wait_timeout
+            ):
+                # Re-check cache after acquiring lock, in case another process finished
+                # while we were waiting for the lock.
+                if cached_summary := cache.get(cache_key):
+                    return convert_dict_key_case(cached_summary, snake_to_camel_case), 200
+
+                # Lock acquired and cache is still empty, proceed with generation
+                summary_dict, status_code = _generate_summary(
+                    group, user, force_event_id, source, cache_key
+                )
+                return convert_dict_key_case(summary_dict, snake_to_camel_case), status_code
+
+        except UnableToAcquireLock:
+            # Failed to acquire lock within timeout. Check cache one last time.
+            if cached_summary := cache.get(cache_key):
+                return convert_dict_key_case(cached_summary, snake_to_camel_case), 200
+            return {"detail": "Timeout waiting for summary generation lock"}, 503
+
+    # 3. Generate summary directly (either forced or lock acquired)
path: src/sentry/seer/issue_summary.py
user: login: jennmueng
body: is "or lock acquired" here correct? if the lock is acquired can we even get here?
created_at: 2025-04-28T19:50:53Z
author_association: CONTRIBUTOR
diff_hunk: @@ -314,4 +297,67 @@ def get_issue_summary(
 
     cache.set(cache_key, summary_dict, timeout=int(timedelta(days=7).total_seconds()))
 
-    return convert_dict_key_case(summary_dict, snake_to_camel_case), 200
+    return summary_dict, 200
+
+
+def get_issue_summary(
+    group: Group,
+    user: User | RpcUser | AnonymousUser | None = None,
+    force_event_id: str | None = None,
+    source: str = "issue_details",
+) -> tuple[dict[str, Any], int]:
+    """
+    Generate an AI summary for an issue.
+
+    Args:
+        group: The issue group
+        user: The user requesting the summary
+        force_event_id: Optional event ID to force summarizing a specific event
+        source: The source triggering the summary generation
+
+    Returns:
+        A tuple containing (summary_data, status_code)
+    """
+    if user is None:
+        user = AnonymousUser()
+    if not features.has("organizations:gen-ai-features", group.organization, actor=user):
+        return {"detail": "Feature flag not enabled"}, 400
+
+    cache_key = f"ai-group-summary-v2:{group.id}"
+    lock_key = f"ai-group-summary-v2-lock:{group.id}"
+    lock_duration = 4  # How long the lock is held if acquired (seconds)
+    wait_timeout = 5  # How long to wait for the lock (seconds)
+
+    # 1. Check cache first (regardless of force_event_id)
+    if not force_event_id:
path: src/sentry/seer/issue_summary.py
user: login: jennmueng
body: regardless of `force_event_id`?
created_at: 2025-04-28T19:51:20Z
author_association: CONTRIBUTOR

PR comments:
user: login: codecov[bot]
created_at: 2025-04-24T22:40:56Z
body: ## [Codecov](https://app.codecov.io/gh/getsentry/sentry/pull/90347?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=getsentry) Report
Attention: Patch coverage is `94.50549%` with `5 lines` in your changes missing coverage. Please review.

:white_check_mark: All tests successful. No failed tests found.

| [Files with missing lines](https://app.codecov.io/gh/getsentry/sentry/pull/90347?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=getsentry) | Patch % | Lines |
|---|---|---|
| [src/sentry/seer/issue\_summary.py](https://app.codecov.io/gh/getsentry/sentry/pull/90347?src=pr&el=tree&filepath=src%2Fsentry%2Fseer%2Fissue_summary.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=getsentry#diff-c3JjL3NlbnRyeS9zZWVyL2lzc3VlX3N1bW1hcnkucHk=) | 89.28% | [3 Missing :warning: ](https://app.codecov.io/gh/getsentry/sentry/pull/90347?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=getsentry) |
| [tests/sentry/seer/test\_issue\_summary.py](https://app.codecov.io/gh/getsentry/sentry/pull/90347?src=pr&el=tree&filepath=tests%2Fsentry%2Fseer%2Ftest_issue_summary.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=getsentry#diff-dGVzdHMvc2VudHJ5L3NlZXIvdGVzdF9pc3N1ZV9zdW1tYXJ5LnB5) | 96.82% | [2 Missing :warning: ](https://app.codecov.io/gh/getsentry/sentry/pull/90347?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=getsentry) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master   #90347      +/-   ##
==========================================
- Coverage   87.78%   87.41%   -0.37%     
==========================================
  Files       10264    10273       +9     
  Lines      580241   585145    +4904     
  Branches    22635    22659      +24     
==========================================
+ Hits       509346   511504    +2158     
- Misses      70460    73203    +2743     
- Partials      435      438       +3     
```

</details>
author_association: NONE
user: login: roaga
created_at: 2025-04-28T20:35:01Z
body: @jennmueng fixed the comments and made it a bit more readable

My testing is:
- unit tests that should test the race conditions
- running summaries from the frontend locally

idk if that's enough 🤷 
author_association: CONTRIBUTOR
user: login: sentry[bot]
created_at: 2025-05-03T05:21:45Z
body: ## Suspect Issues
This pull request was deployed and Sentry observed the following issues:

- ‼️ **HTTPError: 500 Server Error: Internal Server Error for url: http://seer-web-autofix/v1/automation/summarize/issue** `/api/0/organizations/{organization_id_or_slug}/...` [View Issue](https://sentry.sentry.io/issues/6583073096/?referrer=github-pr-bot)
- ‼️ **SnubaError: Failed to parse snuba error response** `/api/0/organizations/{organization_id_or_slug}/...` [View Issue](https://sentry.sentry.io/issues/6589319208/?referrer=github-pr-bot)

<sub>Did you find this useful? React with a 👍 or 👎</sub>
author_association: CONTRIBUTOR