Repository: getsentry/sentry
Pull request: feat(escalating-issues): Use bulk updates/creates
## Objective:
Bulk update Groups to reduce load in Postgres

Problem statement:
Auto-state transition cron updates
## Objective:
Currently we are running this cron job every 6 hours for 1000 orgs. It takes ~1h to work through the regular batches
Assuming that scales roughly linearly, increasing that 50x would mean we'd be scheduling 50h of work every 6h, and we can't simply scale up workers as the load is too heavy on our default (primary) db so we would immediately start falling behind. We hope to see some improvements after completing the following:

- [ ] update cron to run every 10min

- [ ] update cron to query the queue length. If length > 0, skip the run and raise an exception (dogfood Sentry Crons)

- [ ] update the tasks to batch update the Groups rather than spin up a new task for each Group and do individual writes

- [ ] after merging in the above changes:
        - monitor the queue length and tasks completion time for any changes
        - monitor Sentry Crons for errors

Hints from issue discussion:

Commit message: ignore type error

Patch to review:
diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py
index 30387996542570..eb5c91b924e364 100644
--- a/src/sentry/conf/server.py
+++ b/src/sentry/conf/server.py
@@ -764,6 +764,10 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     ),
 ]
 
+CELERY_ISSUE_STATES_QUEUE = Queue(
+    "auto_transition_issue_states", routing_key="auto_transition_issue_states"
+)
+
 CELERY_QUEUES_REGION = [
     Queue("activity.notify", routing_key="activity.notify"),
     Queue("alerts", routing_key="alerts"),
@@ -852,8 +856,8 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     Queue("transactions.name_clusterer", routing_key="transactions.name_clusterer"),
     Queue("auto_enable_codecov", routing_key="auto_enable_codecov"),
     Queue("weekly_escalating_forecast", routing_key="weekly_escalating_forecast"),
-    Queue("auto_transition_issue_states", routing_key="auto_transition_issue_states"),
     Queue("recap_servers", routing_key="recap_servers"),
+    CELERY_ISSUE_STATES_QUEUE,
 ]
 
 from celery.schedules import crontab
@@ -980,7 +984,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     "schedule-auto-resolution": {
         "task": "sentry.tasks.schedule_auto_resolution",
         # Run every 15 minutes
-        "schedule": crontab(minute="*/15"),
+        "schedule": crontab(minute="*/10"),
         "options": {"expires": 60 * 25},
     },
     "auto-remove-inbox": {
@@ -1081,7 +1085,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     "schedule_auto_transition_new": {
         "task": "sentry.tasks.schedule_auto_transition_new",
         # Run job every 6 hours
-        "schedule": crontab(minute=0, hour="*/6"),
+        "schedule": crontab(minute="*/10"),
         "options": {"expires": 3600},
     },
     "schedule_auto_transition_regressed": {
diff --git a/src/sentry/issues/ongoing.py b/src/sentry/issues/ongoing.py
index 2c08304bcc2d28..dfe6d51cd1d544 100644
--- a/src/sentry/issues/ongoing.py
+++ b/src/sentry/issues/ongoing.py
@@ -1,4 +1,4 @@
-from typing import Any, Mapping, Optional
+from typing import Any, List, Mapping, Optional
 
 from django.db.models.signals import post_save
 
@@ -6,6 +6,7 @@
     Activity,
     Group,
     GroupStatus,
+    bulk_remove_groups_from_inbox,
     record_group_history_from_activity_type,
     remove_group_from_inbox,
 )
@@ -13,19 +14,33 @@
 from sentry.types.group import GroupSubStatus
 
 
-def transition_group_to_ongoing(
+def bulk_transition_group_to_ongoing(
     from_status: GroupStatus,
     from_substatus: GroupSubStatus,
-    group: Group,
+    groups: List[Group],
     activity_data: Optional[Mapping[str, Any]] = None,
 ) -> None:
     # make sure we don't update the Group when its already updated by conditionally updating the Group
-    updated = Group.objects.filter(
-        id=group.id, status=from_status, substatus=from_substatus
-    ).update(status=GroupStatus.UNRESOLVED, substatus=GroupSubStatus.ONGOING)
-    if updated:
+    groups_to_transistion = Group.objects.filter(
+        id__in=[group.id for group in groups], status=from_status, substatus=from_substatus
+    )
+
+    Group.objects.update_group_status(
+        groups=groups_to_transistion,
+        status=GroupStatus.UNRESOLVED,  # type: ignore
+        substatus=GroupSubStatus.ONGOING,  # type: ignore
+        activity_type=ActivityType.AUTO_SET_ONGOING,
+        activity_data=activity_data,
+        send_activity_notification=False,
+    )
+
+    for group in groups_to_transistion:
         group.status = GroupStatus.UNRESOLVED
         group.substatus = GroupSubStatus.ONGOING
+
+    bulk_remove_groups_from_inbox(groups)
+
+    for group in groups_to_transistion:
         post_save.send_robust(
             sender=Group,
             instance=group,
diff --git a/src/sentry/issues/update_inbox.py b/src/sentry/issues/update_inbox.py
index 06527e22d12812..5182e77a8d2b89 100644
--- a/src/sentry/issues/update_inbox.py
+++ b/src/sentry/issues/update_inbox.py
@@ -3,7 +3,7 @@
 from typing import Any, Dict, List
 
 from sentry import features
-from sentry.issues.ongoing import transition_group_to_ongoing
+from sentry.issues.ongoing import bulk_transition_group_to_ongoing
 from sentry.models import Group, GroupStatus, Project, User
 from sentry.models.groupinbox import (
     GroupInboxReason,
@@ -51,10 +51,10 @@ def update_inbox(
                 and group.substatus != GroupSubStatus.ONGOING
                 and group.status == GroupStatus.UNRESOLVED
             ):
-                transition_group_to_ongoing(
+                bulk_transition_group_to_ongoing(
                     group.status,
                     group.substatus,
-                    group,
+                    [group],
                     activity_data={"manually": True},
                 )
 
diff --git a/src/sentry/models/group.py b/src/sentry/models/group.py
index b3ddb7b1ef1a42..bca59b4d218000 100644
--- a/src/sentry/models/group.py
+++ b/src/sentry/models/group.py
@@ -9,7 +9,7 @@
 from enum import Enum
 from functools import reduce
 from operator import or_
-from typing import TYPE_CHECKING, Mapping, Optional, Sequence
+from typing import TYPE_CHECKING, Any, Mapping, Optional, Sequence
 
 from django.db import models
 from django.db.models import Q, QuerySet
@@ -398,19 +398,32 @@ def update_group_status(
         status: GroupStatus,
         substatus: GroupSubStatus | None,
         activity_type: ActivityType,
+        activity_data: Optional[Mapping[str, Any]] = None,
+        send_activity_notification: bool = True,
     ) -> None:
         """For each groups, update status to `status` and create an Activity."""
         from sentry.models import Activity
 
-        updated_count = (
-            self.filter(id__in=[g.id for g in groups])
-            .exclude(status=status)
-            .update(status=status, substatus=substatus)
+        to_be_updated = self.filter(id__in=[g.id for g in groups]).exclude(
+            status=status, substatus=substatus
         )
-        if updated_count:
-            for group in groups:
-                Activity.objects.create_group_activity(group, activity_type)
-                record_group_history_from_activity_type(group, activity_type.value)
+
+        for group in to_be_updated:
+            group.status = status
+            group.substatus = substatus
+
+        self.bulk_update(to_be_updated, ["status", "substatus"])
+
+        for group in to_be_updated:
+            group.status = status
+            group.substatus = substatus
+            Activity.objects.create_group_activity(
+                group,
+                activity_type,
+                data=activity_data,
+                send_notification=send_activity_notification,
+            )
+            record_group_history_from_activity_type(group, activity_type.value)
 
     def from_share_id(self, share_id: str) -> Group:
         if not share_id or len(share_id) != 32:
diff --git a/src/sentry/models/grouphistory.py b/src/sentry/models/grouphistory.py
index 4f81fe8af12534..141d08ced4bd3f 100644
--- a/src/sentry/models/grouphistory.py
+++ b/src/sentry/models/grouphistory.py
@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING, List, Optional, Union
 
 from django.db import models
 from django.db.models import SET_NULL, Q
@@ -277,3 +277,42 @@ def record_group_history(
         prev_history=prev_history,
         prev_history_date=prev_history.date_added if prev_history else None,
     )
+
+
+def bulk_record_group_history(
+    groups: List["Group"],
+    status: int,
+    actor: Optional[Union["User", "RpcUser", "Team"]] = None,
+    release: Optional["Release"] = None,
+):
+    from sentry.models import Team, User
+    from sentry.services.hybrid_cloud.user import RpcUser
+
+    def get_prev_history_date(group, status):
+        prev_history = get_prev_history(group, status)
+        return prev_history.date_added if prev_history else None
+
+    actor_id = None
+    if actor:
+        if isinstance(actor, RpcUser) or isinstance(actor, User):
+            actor_id = get_actor_id_for_user(actor)
+        elif isinstance(actor, Team):
+            actor_id = actor.actor_id
+        else:
+            raise ValueError("record_group_history actor argument must be RPCUser or Team")
+
+    return GroupHistory.objects.bulk_create(
+        [
+            GroupHistory(
+                organization=group.project.organization,
+                group=group,
+                project=group.project,
+                release=release,
+                actor_id=actor_id,
+                status=status,
+                prev_history=get_prev_history(group, status),
+                prev_history_date=get_prev_history_date(group, status),
+            )
+            for group in groups
+        ]
+    )
diff --git a/src/sentry/models/groupinbox.py b/src/sentry/models/groupinbox.py
index 3a8970a8d8f341..e2501d37195f5e 100644
--- a/src/sentry/models/groupinbox.py
+++ b/src/sentry/models/groupinbox.py
@@ -7,7 +7,11 @@
 
 from sentry.db.models import FlexibleForeignKey, JSONField, Model, region_silo_only_model
 from sentry.models import Activity
-from sentry.models.grouphistory import GroupHistoryStatus, record_group_history
+from sentry.models.grouphistory import (
+    GroupHistoryStatus,
+    bulk_record_group_history,
+    record_group_history,
+)
 from sentry.types.activity import ActivityType
 
 INBOX_REASON_DETAILS = {
@@ -104,6 +108,29 @@ def remove_group_from_inbox(group, action=None, user=None, referrer=None):
         pass
 
 
+def bulk_remove_groups_from_inbox(groups, action=None, user=None, referrer=None):
+    try:
+        group_inbox = GroupInbox.objects.filter(group__in=groups)
+        group_inbox.delete()
+
+        if action is GroupInboxRemoveAction.MARK_REVIEWED and user is not None:
+            Activity.objects.bulk_create(
+                [
+                    Activity(
+                        project_id=group.project_id,
+                        group_id=group.id,
+                        type=ActivityType.MARK_REVIEWED.value,
+                        user_id=user.id,
+                    )
+                    for group in groups
+                ]
+            )
+
+            bulk_record_group_history(groups, GroupHistoryStatus.REVIEWED, actor=user)
+    except GroupInbox.DoesNotExist:
+        pass
+
+
 def get_inbox_details(group_list):
     group_ids = [g.id for g in group_list]
     group_inboxes = GroupInbox.objects.filter(group__in=group_ids)
diff --git a/src/sentry/tasks/auto_archive_issues.py b/src/sentry/tasks/auto_archive_issues.py
index e39f22f032a3b0..8571ba18c11489 100644
--- a/src/sentry/tasks/auto_archive_issues.py
+++ b/src/sentry/tasks/auto_archive_issues.py
@@ -19,6 +19,7 @@
     record_group_history_from_activity_type,
     remove_group_from_inbox,
 )
+from sentry.tasks.auto_ongoing_issues import skip_if_queue_has_items
 from sentry.tasks.base import instrumented_task, retry
 from sentry.types.activity import ActivityType
 from sentry.types.group import GroupSubStatus
@@ -37,6 +38,7 @@
 )
 @retry
 @monitor(monitor_slug="auto-archive-job-monitor")
+@skip_if_queue_has_items
 def run_auto_archive() -> None:
     """
     Automatically transition issues that are ongoing for 14 days to archived until escalating.
@@ -64,6 +66,7 @@ def run_auto_archive() -> None:
     soft_time_limit=20 * 60,
 )
 @retry
+@skip_if_queue_has_items
 def run_auto_archive_for_project(project_ids: List[int]) -> None:
     now = datetime.now(tz=pytz.UTC)
     fourteen_days_ago = now - timedelta(days=14)
diff --git a/src/sentry/tasks/auto_ongoing_issues.py b/src/sentry/tasks/auto_ongoing_issues.py
index 8cc4b7c7d55ab6..4fe3d8533e53e3 100644
--- a/src/sentry/tasks/auto_ongoing_issues.py
+++ b/src/sentry/tasks/auto_ongoing_issues.py
@@ -1,4 +1,6 @@
+import logging
 from datetime import datetime, timedelta
+from functools import wraps
 from typing import Optional
 
 import pytz
@@ -7,7 +9,8 @@
 from sentry_sdk.crons.decorator import monitor
 
 from sentry import features
-from sentry.issues.ongoing import transition_group_to_ongoing
+from sentry.conf.server import CELERY_ISSUE_STATES_QUEUE
+from sentry.issues.ongoing import bulk_transition_group_to_ongoing
 from sentry.models import (
     Group,
     GroupHistoryStatus,
@@ -16,13 +19,35 @@
     OrganizationStatus,
     Project,
 )
+from sentry.monitoring.queues import backend
 from sentry.tasks.base import instrumented_task, retry
 from sentry.types.group import GroupSubStatus
 from sentry.utils.query import RangeQuerySetWrapper
 
+logger = logging.getLogger(__name__)
+
 TRANSITION_AFTER_DAYS = 7
 
 
+def skip_if_queue_has_items(func):
+    def inner(func):
+        @wraps(func)
+        def wrapped(*args, **kwargs):
+            queue_size = backend.get_size(CELERY_ISSUE_STATES_QUEUE.name)
+            if queue_size > 0:
+                logger.exception(
+                    f"{CELERY_ISSUE_STATES_QUEUE.name} queue size greater than 0.",
+                    extra={"size": queue_size, "task": func.__name__},
+                )
+                return
+
+            func(*args, **kwargs)
+
+        return wrapped
+
+    return inner(func)
+
+
 @instrumented_task(
     name="sentry.tasks.schedule_auto_transition_new",
     queue="auto_transition_issue_states",
@@ -32,7 +57,9 @@
 )
 @retry(on=(OperationalError,))
 @monitor(monitor_slug="schedule_auto_transition_new")
+@skip_if_queue_has_items
 def schedule_auto_transition_new() -> None:
+
     now = datetime.now(tz=pytz.UTC)
     three_days_past = now - timedelta(days=TRANSITION_AFTER_DAYS)
 
@@ -58,6 +85,7 @@ def schedule_auto_transition_new() -> None:
     acks_late=True,
 )
 @retry(on=(OperationalError,))
+@skip_if_queue_has_items
 def auto_transition_issues_new_to_ongoing(
     project_id: int,
     first_seen_lte: int,
@@ -77,13 +105,12 @@ def auto_transition_issues_new_to_ongoing(
 
     new_groups = list(queryset.order_by("first_seen")[:chunk_size])
 
-    for group in new_groups:
-        transition_group_to_ongoing(
-            GroupStatus.UNRESOLVED,
-            GroupSubStatus.NEW,
-            group,
-            activity_data={"after_days": TRANSITION_AFTER_DAYS},
-        )
+    bulk_transition_group_to_ongoing(
+        GroupStatus.UNRESOLVED,
+        GroupSubStatus.NEW,
+        new_groups,
+        activity_data={"after_days": TRANSITION_AFTER_DAYS},
+    )
 
     if len(new_groups) == chunk_size:
         auto_transition_issues_new_to_ongoing.delay(
@@ -104,6 +131,7 @@ def auto_transition_issues_new_to_ongoing(
 )
 @retry(on=(OperationalError,))
 @monitor(monitor_slug="schedule_auto_transition_regressed")
+@skip_if_queue_has_items
 def schedule_auto_transition_regressed() -> None:
     now = datetime.now(tz=pytz.UTC)
     three_days_past = now - timedelta(days=TRANSITION_AFTER_DAYS)
@@ -130,6 +158,7 @@ def schedule_auto_transition_regressed() -> None:
     acks_late=True,
 )
 @retry(on=(OperationalError,))
+@skip_if_queue_has_items
 def auto_transition_issues_regressed_to_ongoing(
     project_id: int,
     date_added_lte: int,
@@ -155,13 +184,12 @@ def auto_transition_issues_regressed_to_ongoing(
 
     groups_with_regressed_history = list(queryset.order_by("recent_regressed_history")[:chunk_size])
 
-    for group in groups_with_regressed_history:
-        transition_group_to_ongoing(
-            GroupStatus.UNRESOLVED,
-            GroupSubStatus.REGRESSED,
-            group,
-            activity_data={"after_days": TRANSITION_AFTER_DAYS},
-        )
+    bulk_transition_group_to_ongoing(
+        GroupStatus.UNRESOLVED,
+        GroupSubStatus.REGRESSED,
+        groups_with_regressed_history,
+        activity_data={"after_days": TRANSITION_AFTER_DAYS},
+    )
 
     if len(groups_with_regressed_history) == chunk_size:
         auto_transition_issues_regressed_to_ongoing.delay(
diff --git a/src/sentry/tasks/auto_resolve_issues.py b/src/sentry/tasks/auto_resolve_issues.py
index 9fc86540efc66e..ddd0f17aa74184 100644
--- a/src/sentry/tasks/auto_resolve_issues.py
+++ b/src/sentry/tasks/auto_resolve_issues.py
@@ -14,6 +14,7 @@
     remove_group_from_inbox,
 )
 from sentry.models.grouphistory import GroupHistoryStatus, record_group_history
+from sentry.tasks.auto_ongoing_issues import skip_if_queue_has_items
 from sentry.tasks.base import instrumented_task
 from sentry.tasks.integrations import kick_off_status_syncs
 from sentry.types.activity import ActivityType
@@ -27,6 +28,7 @@
     time_limit=75,
     soft_time_limit=60,
 )
+@skip_if_queue_has_items
 def schedule_auto_resolution():
     options = ProjectOption.objects.filter(
         key__in=["sentry:resolve_age", "sentry:_last_auto_resolve"]
@@ -56,6 +58,7 @@ def schedule_auto_resolution():
     time_limit=75,
     soft_time_limit=60,
 )
+@skip_if_queue_has_items
 def auto_resolve_project_issues(project_id, cutoff=None, chunk_size=1000, **kwargs):
     project = Project.objects.get_from_cache(id=project_id)
 
diff --git a/tests/sentry/issues/test_ongoing.py b/tests/sentry/issues/test_ongoing.py
index 104a09350477e9..dee4ad69858b0a 100644
--- a/tests/sentry/issues/test_ongoing.py
+++ b/tests/sentry/issues/test_ongoing.py
@@ -1,5 +1,12 @@
-from sentry.issues.ongoing import transition_group_to_ongoing
-from sentry.models import Activity, GroupHistory, GroupHistoryStatus, GroupStatus
+from sentry.issues.ongoing import bulk_transition_group_to_ongoing
+from sentry.models import (
+    Activity,
+    GroupHistory,
+    GroupHistoryStatus,
+    GroupInbox,
+    GroupInboxReason,
+    GroupStatus,
+)
 from sentry.testutils import TestCase
 from sentry.types.activity import ActivityType
 from sentry.types.group import GroupSubStatus
@@ -9,7 +16,7 @@ class TransitionNewToOngoingTest(TestCase):
     def test_new_to_ongoing(self) -> None:
         group = self.create_group(status=GroupStatus.UNRESOLVED, substatus=GroupSubStatus.NEW)
 
-        transition_group_to_ongoing(GroupStatus.UNRESOLVED, GroupSubStatus.NEW, group)
+        bulk_transition_group_to_ongoing(GroupStatus.UNRESOLVED, GroupSubStatus.NEW, [group])
         assert Activity.objects.filter(
             group=group, type=ActivityType.AUTO_SET_ONGOING.value
         ).exists()
@@ -20,7 +27,7 @@ def test_new_to_ongoing(self) -> None:
     def test_regressed_to_ongoing(self) -> None:
         group = self.create_group(status=GroupStatus.UNRESOLVED, substatus=GroupSubStatus.REGRESSED)
 
-        transition_group_to_ongoing(GroupStatus.UNRESOLVED, GroupSubStatus.REGRESSED, group)
+        bulk_transition_group_to_ongoing(GroupStatus.UNRESOLVED, GroupSubStatus.REGRESSED, [group])
         assert Activity.objects.filter(
             group=group, type=ActivityType.AUTO_SET_ONGOING.value
         ).exists()
diff --git a/tests/sentry/tasks/test_auto_archive_issues.py b/tests/sentry/tasks/test_auto_archive_issues.py
index 55765da3413302..ecd8bb850574e1 100644
--- a/tests/sentry/tasks/test_auto_archive_issues.py
+++ b/tests/sentry/tasks/test_auto_archive_issues.py
@@ -1,4 +1,5 @@
 from datetime import datetime, timedelta
+from unittest.mock import patch
 
 import pytz
 
@@ -11,8 +12,9 @@
 
 
 class ScheduleAutoOngoingToArchivedIssuesTest(TestCase):
+    @patch("sentry.tasks.auto_ongoing_issues.backend")
     @with_feature("organizations:escalating-issues-v2")
-    def test_simple(self):
+    def test_simple(self, mock_backend):
         now = datetime.now(tz=pytz.UTC)
         project = self.create_project()
         group = self.create_group(
@@ -25,7 +27,7 @@ def test_simple(self):
             status=GroupHistoryStatus.ONGOING,
             date_added=now - timedelta(days=15),
         )
-
+        mock_backend.get_size.return_value = 0
         with self.tasks():
             run_auto_archive()
 
diff --git a/tests/sentry/tasks/test_auto_ongoing_issues.py b/tests/sentry/tasks/test_auto_ongoing_issues.py
index 4c46579c40516e..0f0238b45ca64b 100644
--- a/tests/sentry/tasks/test_auto_ongoing_issues.py
+++ b/tests/sentry/tasks/test_auto_ongoing_issues.py
@@ -29,7 +29,8 @@
 
 @apply_feature_flag_on_cls("organizations:escalating-issues")
 class ScheduleAutoNewOngoingIssuesTest(TestCase):
-    def test_simple(self):
+    @mock.patch("sentry.tasks.auto_ongoing_issues.backend")
+    def test_simple(self, mock_backend):
         now = datetime.now(tz=pytz.UTC)
         project = self.create_project()
         group = self.create_group(
@@ -38,6 +39,8 @@ def test_simple(self):
         group.first_seen = now - timedelta(days=TRANSITION_AFTER_DAYS, hours=1)
         group.save()
 
+        mock_backend.get_size.return_value = 0
+
         with self.tasks():
             schedule_auto_transition_new()
 
@@ -54,7 +57,8 @@ def test_simple(self):
 
         assert GroupHistory.objects.filter(group=group, status=GroupHistoryStatus.ONGOING).exists()
 
-    def test_reprocessed(self):
+    @mock.patch("sentry.tasks.auto_ongoing_issues.backend")
+    def test_reprocessed(self, mock_backend):
         now = datetime.now(tz=pytz.UTC)
 
         project = self.create_project()
@@ -65,6 +69,8 @@ def test_reprocessed(self):
         group.first_seen = now - timedelta(days=TRANSITION_AFTER_DAYS, hours=1)
         group.save()
 
+        mock_backend.get_size.return_value = 0
+
         with self.tasks():
             schedule_auto_transition_new()
 
@@ -74,7 +80,8 @@ def test_reprocessed(self):
 
         assert not GroupInbox.objects.filter(group=group).exists()
 
-    def test_multiple_old_new(self):
+    @mock.patch("sentry.tasks.auto_ongoing_issues.backend")
+    def test_multiple_old_new(self, mock_backend):
         now = datetime.now(tz=pytz.UTC)
         project = self.create_project()
         new_groups = []
@@ -115,6 +122,8 @@ def test_multiple_old_new(self):
             new_groups
         )
 
+        mock_backend.get_size.return_value = 0
+
         with self.tasks():
             schedule_auto_transition_new()
 
@@ -139,13 +148,14 @@ def test_multiple_old_new(self):
             ).values_list("id", flat=True)
         ) == {g.id for g in older_groups}
 
+    @mock.patch("sentry.tasks.auto_ongoing_issues.backend")
     @mock.patch(
         "sentry.tasks.auto_ongoing_issues.auto_transition_issues_new_to_ongoing.delay",
         wraps=lambda project_id, first_seen_lte, **kwargs: auto_transition_issues_new_to_ongoing(
             project_id, first_seen_lte, chunk_size=10, kwargs=kwargs
         ),
     )
-    def test_paginated_transition(self, mocked):
+    def test_paginated_transition(self, mocked, mock_backend):
         create_default_projects()
         now = datetime.now(tz=pytz.UTC)
         project = self.create_project()
@@ -162,6 +172,8 @@ def test_paginated_transition(self, mocked):
             ]
         )
 
+        mock_backend.get_size.return_value = 0
+
         # before
         assert Group.objects.filter(project_id=project.id).count() == len(groups) == 12
 
@@ -191,7 +203,8 @@ def test_paginated_transition(self, mocked):
 
 @apply_feature_flag_on_cls("organizations:escalating-issues")
 class ScheduleAutoRegressedOngoingIssuesTest(TestCase):
-    def test_simple(self):
+    @mock.patch("sentry.tasks.auto_ongoing_issues.backend")
+    def test_simple(self, mock_backend):
         now = datetime.now(tz=pytz.UTC)
         project = self.create_project()
         group = self.create_group(
@@ -209,6 +222,8 @@ def test_simple(self):
         group_history.date_added = now - timedelta(days=TRANSITION_AFTER_DAYS, hours=1)
         group_history.save(update_fields=["date_added"])
 
+        mock_backend.get_size.return_value = 0
+
         with self.tasks():
             schedule_auto_transition_regressed()
 
diff --git a/tests/sentry/tasks/test_auto_resolve_issues.py b/tests/sentry/tasks/test_auto_resolve_issues.py
index 7d265f1bc8974f..e0f6ddb7590183 100644
--- a/tests/sentry/tasks/test_auto_resolve_issues.py
+++ b/tests/sentry/tasks/test_auto_resolve_issues.py
@@ -10,11 +10,14 @@
 
 
 class ScheduleAutoResolutionTest(TestCase):
-    def test_task_persistent_name(self):
+    @patch("sentry.tasks.auto_ongoing_issues.backend")
+    def test_task_persistent_name(self, mock_backend):
+        mock_backend.get_size.return_value = 0
         assert schedule_auto_resolution.name == "sentry.tasks.schedule_auto_resolution"
 
+    @patch("sentry.tasks.auto_ongoing_issues.backend")
     @patch("sentry.tasks.auto_resolve_issues.kick_off_status_syncs")
-    def test_simple(self, mock_kick_off_status_syncs):
+    def test_simple(self, mock_kick_off_status_syncs, mock_backend):
         project = self.create_project()
         project2 = self.create_project()
         project3 = self.create_project()
@@ -43,6 +46,8 @@ def test_simple(self, mock_kick_off_status_syncs):
             last_seen=timezone.now() - timedelta(days=1),
         )
 
+        mock_backend.get_size.return_value = 0
+
         with self.tasks():
             schedule_auto_resolution()
 


Code review comments:
diff_hunk: @@ -104,6 +108,29 @@ def remove_group_from_inbox(group, action=None, user=None, referrer=None):
         pass
 
 
+def bulk_remove_groups_from_inbox(groups, action=None, user=None, referrer=None):
+    try:
+        group_inbox = GroupInbox.objects.filter(group__in=groups)
+        group_inbox.delete()
+
+        if action is GroupInboxRemoveAction.MARK_REVIEWED and user is not None:
+            Activity.objects.bulk_create(
+                [
+                    Activity(
+                        project_id=group.project_id,
+                        group_id=group.id,
+                        type=ActivityType.MARK_REVIEWED.value,
+                        user_id=user.id,
+                    )
+                    for group in groups
path: src/sentry/models/groupinbox.py
text: @user1:
should this be iterating over `group_inbox` instead?