Repository: getsentry/sentry

Issue:
 Calculate and store the escalation threshold
Implement the threshold calculation which can be found in the tech spec [here](https://www.notion.so/sentry/Severity-based-Notifications-Tech-Spec-df83dc1696424075bd3a0ba2e565d916?pvs=4#9b3f6883fbd840b2adf975e572f2f802) (note the comment about the Clickhouse quantile function) and store that number per project in Redis. I imagine the Redis key will be something like `f"new-issue-escalation-threshold:{project.id}"`. You may have to work with ops to find which Redis cluster has available space (Hubert is our team's liaison so I'd ask him first). 

This will be used by #60413 to escalate new issues if they exceed the threshold. Maybe it can be in a new file and live in [this directory](https://github.com/getsentry/sentry/tree/master/src/sentry/issues) - it'll need to be imported by `post_process.py`. 


Discussion:

Resolving patch:
diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py
--- a/src/sentry/conf/server.py
+++ b/src/sentry/conf/server.py
@@ -136,6 +136,7 @@ def env(
 SENTRY_MONITORS_REDIS_CLUSTER = "default"
 SENTRY_STATISTICAL_DETECTORS_REDIS_CLUSTER = "default"
 SENTRY_METRIC_META_REDIS_CLUSTER = "default"
+SENTRY_ESCALATION_THRESHOLDS_REDIS_CLUSTER = "default"
 
 # Hosts that are allowed to use system token authentication.
 # http://en.wikipedia.org/wiki/Reserved_IP_addresses
diff --git a/src/sentry/issues/issue_velocity.py b/src/sentry/issues/issue_velocity.py
new file mode 100644
--- /dev/null
+++ b/src/sentry/issues/issue_velocity.py
@@ -0,0 +1,208 @@
+"""This module contains the logic for calculating the velocity threshold based on the 90th percentile
+of events per issue per hour, and getting it from and saving it to Redis.
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+from datetime import datetime, timedelta
+from typing import Optional
+
+from django.conf import settings
+from sentry_redis_tools.clients import RedisCluster, StrictRedis
+from snuba_sdk import (
+    Column,
+    Condition,
+    Direction,
+    Entity,
+    Function,
+    Limit,
+    Op,
+    OrderBy,
+    Query,
+    Request,
+)
+
+from sentry.models.project import Project
+from sentry.snuba.dataset import Dataset, EntityKey
+from sentry.tasks.post_process import locks
+from sentry.utils import json
+from sentry.utils.locking import UnableToAcquireLock
+from sentry.utils.redis import redis_clusters
+from sentry.utils.snuba import raw_snql_query
+
+logger = logging.getLogger(__name__)
+
+# for snuba operations
+REFERRER = "sentry.issues.issue_velocity"
+THRESHOLD_QUANTILE = {"name": "p90", "function": "quantile(0.9)"}
+WEEK_IN_HOURS = 7 * 24
+
+# for redis operations
+DEFAULT_TTL = 48 * 60 * 60  # 2 days
+NONE_TTL = 10 * 60  # 10 minutes
+THRESHOLD_KEY = "new-issue-escalation-threshold:{project_id}"
+STALE_DATE_KEY = "new-issue-escalation-threshold-stale-date:{project_id}"
+DATE_FORMAT = "%Y%m%d"
+
+
+def calculate_threshold(project: Project) -> Optional[float]:
+    """
+    Calculates the velocity threshold based on event frequency in the project for the past week.
+    """
+    now = datetime.now()
+    one_hour_ago = now - timedelta(hours=1)
+    one_week_ago = now - timedelta(days=7)
+    ninety_days_ago = now - timedelta(days=90)
+
+    subquery = Query(
+        match=Entity(EntityKey.IssuePlatform.value),
+        select=[
+            Column("group_id"),
+            Function("min", [Column("timestamp")], "first_seen"),  # when the issue was first seen
+            Function(
+                "countIf",
+                [
+                    Function("greaterOrEquals", [Column("timestamp"), one_week_ago])
+                ],  # count events for the issue that occurred within the past week
+                "past_week_event_count",
+            ),
+            Function(
+                "if",
+                [
+                    Function(
+                        "less", [Column("first_seen"), one_week_ago]
+                    ),  # if the issue is older than a week
+                    Function(
+                        "divide", [Column("past_week_event_count"), WEEK_IN_HOURS]
+                    ),  # divide the number of events in the week by a week in hours
+                    Function(
+                        "divide",
+                        [
+                            Column("past_week_event_count"),
+                            Function(
+                                "dateDiff", ["hour", Column("first_seen"), now]
+                            ),  # otherwise divide by its age in hours
+                        ],
+                    ),
+                ],
+                "hourly_event_rate",
+            ),
+        ],
+        groupby=[Column("group_id")],
+        where=[
+            Condition(
+                Column("timestamp"), Op.GTE, ninety_days_ago
+            ),  # include issues up to the oldest retention date to determine whether an issue is older than the week or not
+            Condition(Column("timestamp"), Op.LT, now),
+            Condition(Column("project_id"), Op.EQ, project.id),
+        ],
+        having=[
+            Condition(
+                Column("past_week_event_count"), Op.GT, 1
+            ),  # exclude any issues that had only 1 event in the past week
+            Condition(
+                Column("first_seen"), Op.LT, one_hour_ago
+            ),  # if it's first seen within the last hour, discard to avoid ZeroDivision
+        ],
+        orderby=[OrderBy(Column("first_seen"), Direction.ASC)],
+        limit=Limit(10000),
+    )
+
+    query = Query(
+        match=subquery,
+        select=[
+            Function(
+                THRESHOLD_QUANTILE["function"],
+                [Column("hourly_event_rate")],
+                THRESHOLD_QUANTILE["name"],
+            )
+        ],  # get the approximate 90th percentile of the event frequency in the past week
+        limit=Limit(1),
+    )
+
+    request = Request(
+        dataset=Dataset.IssuePlatform.value,
+        app_id=REFERRER,
+        query=query,
+        tenant_ids={"referrer": REFERRER, "organization_id": project.organization.id},
+    )
+
+    result = raw_snql_query(request, referrer=REFERRER)["data"]
+    if len(result) == 0:
+        return None
+
+    try:
+        return result[0][THRESHOLD_QUANTILE["name"]]
+    except KeyError:
+        logger.exception(
+            "Unexpected shape for threshold query results",
+            extra={"project_id": project.id, "results_received": json.dumps(result)},
+        )
+        return None
+
+
+def update_threshold(project: Project, threshold_key: str, stale_date_key: str) -> float:
+    """
+    Runs the calculation for the threshold and saves it and the date it is last updated to Redis.
+    If the threshold is NaN, we save it as 0 at the normal TTL. If the threshold is None (due to errors
+    in the query or any other reason), we save it as 0 at the shortened TTL.
+    """
+    threshold = calculate_threshold(project)
+    ttl = DEFAULT_TTL
+    if threshold is None:
+        logger.error(
+            "Velocity threshold couldn't be calculated, query returned nothing",
+            extra={"project_id": project.id},
+        )
+        threshold = 0
+        ttl = NONE_TTL
+    elif math.isnan(threshold):  # indicates there were no valid events to base the calculation
+        threshold = 0
+
+    client = get_redis_client()
+    with client.pipeline() as p:
+        p.set(threshold_key, threshold, ex=ttl)
+        p.set(stale_date_key, convert_date_to_int(datetime.utcnow()), ex=ttl),
+        p.execute()
+
+    return threshold
+
+
+def get_latest_threshold(project: Project) -> float:
+    """
+    Returns the most up-to-date threshold for the project, re-calculating if outdated or non-existent.
+    If the lock cannot be acquired, another process is updating, so we return the stale value.
+    """
+    keys = [
+        THRESHOLD_KEY.format(project_id=project.id),
+        STALE_DATE_KEY.format(project_id=project.id),
+    ]
+    client = get_redis_client()
+    cache_results = client.mget(keys)  # returns None if key is nonexistent
+    threshold, stale_date = cache_results[0], cache_results[1]
+    now = convert_date_to_int(datetime.utcnow())
+    if (stale_date and int(stale_date) < now) or stale_date is None or threshold is None:
+        lock = locks.get(
+            f"calculate_project_thresholds:{project.id}",
+            duration=10,
+            name="calculate_project_thresholds",
+        )
+        try:
+            with lock.acquire():
+                threshold = update_threshold(project, keys[0], keys[1])
+        except UnableToAcquireLock:  # another process is already updating
+            threshold = float(threshold) if threshold else 0  # use stale value if possible
+    else:
+        # redis stores as strings, so convert back to a float if using the value from the cache
+        threshold = float(threshold)
+    return threshold
+
+
+def get_redis_client() -> RedisCluster | StrictRedis:
+    return redis_clusters.get(settings.SENTRY_ESCALATION_THRESHOLDS_REDIS_CLUSTER)
+
+
+def convert_date_to_int(date: datetime) -> int:
+    return int(date.strftime(DATE_FORMAT))


Test patch:
diff --git a/tests/sentry/issues/test_issue_velocity.py b/tests/sentry/issues/test_issue_velocity.py
new file mode 100644
--- /dev/null
+++ b/tests/sentry/issues/test_issue_velocity.py
@@ -0,0 +1,261 @@
+import math
+from datetime import datetime, timedelta
+from unittest.mock import patch
+
+from django.utils import timezone
+
+from sentry.issues.issue_velocity import (
+    DATE_FORMAT,
+    DEFAULT_TTL,
+    NONE_TTL,
+    STALE_DATE_KEY,
+    THRESHOLD_KEY,
+    calculate_threshold,
+    convert_date_to_int,
+    get_latest_threshold,
+    get_redis_client,
+    update_threshold,
+)
+from sentry.tasks.post_process import locks
+from sentry.testutils.cases import SnubaTestCase, TestCase
+from tests.sentry.issues.test_utils import SearchIssueTestMixin
+
+WEEK_IN_HOURS = 7 * 24
+
+
+class IssueVelocityTests(TestCase, SnubaTestCase, SearchIssueTestMixin):
+    def setUp(self):
+        self.now = timezone.now()
+        self.utcnow = datetime.utcnow()
+        super().setUp()
+
+    def test_calculation_simple(self):
+        """
+        Tests threshold calculation for a single issue with the minimum number of events
+        in the past week.
+        """
+        self.store_search_issue(
+            project_id=self.project.id,
+            user_id=self.user.id,
+            fingerprints=["group-1"],
+            insert_time=(self.now - timedelta(days=8)),
+        )
+        for _ in range(2):
+            self.store_search_issue(
+                project_id=self.project.id,
+                user_id=self.user.id,
+                fingerprints=["group-1"],
+                insert_time=(self.now - timedelta(days=1)),
+            )
+
+        threshold = calculate_threshold(self.project)
+        assert threshold == 2 / WEEK_IN_HOURS
+
+    def test_calculation_multiple_issues(self):
+        """
+        Tests that we receive the approximate 90th percentile for multiple issues older than a week
+        with multiple events in the past week.
+        """
+        for i in range(5):
+            # ensure the velocity for each issue is calculated using the whole week
+            self.store_search_issue(
+                project_id=self.project.id,
+                user_id=self.user.id,
+                fingerprints=[f"group-{i}"],
+                insert_time=(self.now - timedelta(days=8)),
+            )
+            for _ in range(i + 2):
+                # fill with events that happened in the previous week
+                self.store_search_issue(
+                    project_id=self.project.id,
+                    user_id=self.user.id,
+                    fingerprints=[f"group-{i}"],
+                    insert_time=(self.now - timedelta(days=1)),
+                )
+
+        # with 5 issues that are older than a week, p90 should be approximately in between the
+        # first and second most frequent issues, which in this case have 6 and 5 events respectively
+        expected_threshold = ((6 / WEEK_IN_HOURS) + (5 / WEEK_IN_HOURS)) / 2
+        actual_threshold = calculate_threshold(self.project)
+
+        # clickhouse's quantile function is approximate
+        # https://clickhouse.com/docs/en/sql-reference/aggregate-functions/reference/quantile
+        assert actual_threshold is not None
+        assert math.isclose(expected_threshold, actual_threshold, abs_tol=10**-3)
+
+    def test_calculation_for_issues_first_seen_recently(self):
+        """
+        Tests that issues first seen within the past week use the difference in hours between now
+        and when they were first seen to calculate frequency instead of the full week in hours.
+        """
+        for _ in range(2):
+            self.store_search_issue(
+                project_id=self.project.id,
+                user_id=self.user.id,
+                fingerprints=["group-1"],
+                insert_time=(self.now - timedelta(days=1)),
+            )
+
+        threshold = calculate_threshold(self.project)
+        assert threshold == 2 / 24
+
+    def test_calculation_excludes_issues_with_only_one_event_in_past_week(self):
+        """
+        Tests that issues with only one event in the past week are excluded from the calculation.
+        """
+        self.store_search_issue(
+            project_id=self.project.id,
+            user_id=self.user.id,
+            fingerprints=["group-1"],
+            insert_time=(self.now - timedelta(days=8)),
+        )
+
+        self.store_search_issue(
+            project_id=self.project.id,
+            user_id=self.user.id,
+            fingerprints=["group-1"],
+            insert_time=(self.now - timedelta(days=1)),
+        )
+
+        threshold = calculate_threshold(self.project)
+        assert threshold is not None
+        assert math.isnan(threshold)
+
+    def test_calculation_excludes_issues_newer_than_an_hour(self):
+        """
+        Tests that issues that were first seen within the past hour are excluded from the calculation.
+        """
+        self.store_search_issue(
+            project_id=self.project.id,
+            user_id=self.user.id,
+            fingerprints=["group-1"],
+            insert_time=(self.now - timedelta(minutes=2)),
+        )
+
+        self.store_search_issue(
+            project_id=self.project.id,
+            user_id=self.user.id,
+            fingerprints=["group-1"],
+            insert_time=(self.now - timedelta(minutes=1)),
+        )
+
+        threshold = calculate_threshold(self.project)
+        assert threshold is not None
+        assert math.isnan(threshold)
+
+    @patch("sentry.issues.issue_velocity.update_threshold")
+    def test_get_latest_threshold_simple(self, mock_update):
+        """
+        Tests that we get the last threshold stored when the stale date has not passed yet.
+        """
+        redis_client = get_redis_client()
+        redis_client.set(THRESHOLD_KEY.format(project_id=self.project.id), 0.1)
+        redis_client.set(
+            STALE_DATE_KEY.format(project_id=self.project.id), convert_date_to_int(self.utcnow)
+        )
+        threshold = get_latest_threshold(self.project)
+        mock_update.assert_not_called()
+        assert threshold == 0.1
+
+    @patch("sentry.issues.issue_velocity.update_threshold")
+    def test_get_latest_threshold_outdated(self, mock_update):
+        """
+        Tests that we update the threshold when the stale date has passed.
+        """
+        redis_client = get_redis_client()
+        redis_client.set(THRESHOLD_KEY.format(project_id=self.project.id), 1.2)
+        redis_client.set(
+            STALE_DATE_KEY.format(project_id=self.project.id),
+            convert_date_to_int(self.utcnow - timedelta(days=1)),
+        )
+        mock_update.return_value = 1.5
+        assert get_latest_threshold(self.project) == 1.5
+
+    @patch("sentry.issues.issue_velocity.update_threshold")
+    def test_get_latest_threshold_when_none_saved(self, mock_update):
+        """
+        Tests that we update the threshold when it is non-existent.
+        """
+        mock_update.return_value = 10.7
+        assert get_latest_threshold(self.project) == 10.7
+
+    @patch("sentry.issues.issue_velocity.update_threshold")
+    def test_get_latest_threshold_locked(self, mock_update):
+        """
+        Tests that we return the stale threshold when another process has the lock.
+        """
+        redis_client = get_redis_client()
+        redis_client.set(THRESHOLD_KEY.format(project_id=self.project.id), 0.7)
+        redis_client.set(
+            STALE_DATE_KEY.format(project_id=self.project.id),
+            convert_date_to_int(self.utcnow - timedelta(days=1)),
+        )
+
+        lock = locks.get(
+            f"calculate_project_thresholds:{self.project.id}",
+            duration=10,
+            name="calculate_project_thresholds",
+        )
+        with lock.acquire():
+            threshold = get_latest_threshold(self.project)
+            mock_update.assert_not_called()
+            assert threshold == 0.7
+
+    @patch("sentry.issues.issue_velocity.update_threshold")
+    def test_get_latest_threshold_locked_no_stale(self, mock_update):
+        """
+        Tests that we return 0 when another process has the lock and there is no stale value.
+        """
+        lock = locks.get(
+            f"calculate_project_thresholds:{self.project.id}",
+            duration=10,
+            name="calculate_project_thresholds",
+        )
+        with lock.acquire():
+            threshold = get_latest_threshold(self.project)
+            mock_update.assert_not_called()
+            assert threshold == 0
+
+    @patch("sentry.issues.issue_velocity.calculate_threshold")
+    def test_update_threshold_simple(self, mock_calculation):
+        """
+        Tests that we save the newly calculated threshold at the default TTL and return it.
+        """
+        mock_calculation.return_value = 5
+        threshold = update_threshold(self.project.id, "threshold-key", "date-key")
+        assert threshold == 5
+        redis_client = get_redis_client()
+        assert redis_client.mget(["threshold-key", "date-key"]) == [
+            "5",
+            self.utcnow.strftime(DATE_FORMAT),
+        ]
+        assert redis_client.ttl("threshold-key") == DEFAULT_TTL
+
+    @patch("sentry.issues.issue_velocity.calculate_threshold")
+    def test_update_threshold_none(self, mock_calculation):
+        """
+        Tests that we return 0 and save a threshold for a shorter amount of time than the default
+        if the calculation returned None.
+        """
+        mock_calculation.return_value = None
+        assert update_threshold(self.project, "threshold-key", "date-key") == 0
+        redis_client = get_redis_client()
+        assert redis_client.mget(["threshold-key", "date-key"]) == [
+            "0",
+            self.utcnow.strftime(DATE_FORMAT),
+        ]
+        assert redis_client.ttl("threshold-key") == NONE_TTL
+
+    @patch("sentry.issues.issue_velocity.calculate_threshold")
+    def test_update_threshold_nan(self, mock_calculation):
+        """
+        Tests that we return 0 and save a threshold for the default TTL if the calculation returned NaN.
+        """
+        mock_calculation.return_value = float("nan")
+        assert update_threshold(self.project, "threshold-key", "date-key") == 0
+        redis_client = get_redis_client()
+        assert redis_client.mget(["threshold-key", "date-key"]) == [
+            "0",
+            self.utcnow.strftime(DATE_FORMAT),
+        ]
+        assert redis_client.ttl("threshold-key") == DEFAULT_TTL