Repository: getsentry/sentry

Issue:
Emit alert metrics into project config computation
Metrics extraction requires to send a description of metrics from Sentry to Relay via project configs. The relevant specs for this are:
- [Project config format](https://www.notion.so/sentry/Metrics-Extraction-Project-Configs-87d8f1e9dc4a422196904ecafc53caf5)
- [Metrics and tagging](https://www.notion.so/sentry/Metrics-Extraction-Metrics-and-Tagging-b602506dc02e4bad8fe0f7ece3c37727)

```[tasklist]
### Tasks
- [ ] https://github.com/getsentry/sentry/pull/51788
- [ ] https://github.com/getsentry/sentry/pull/51863
- [ ] https://github.com/getsentry/relay/pull/2266
```



Discussion:

Resolving patch:
diff --git a/src/sentry/relay/config/__init__.py b/src/sentry/relay/config/__init__.py
--- a/src/sentry/relay/config/__init__.py
+++ b/src/sentry/relay/config/__init__.py
@@ -39,7 +39,10 @@
 )
 from sentry.interfaces.security import DEFAULT_DISALLOWED_SOURCES
 from sentry.models import Project, ProjectKey
-from sentry.relay.config.metric_extraction import get_metric_conditional_tagging_rules
+from sentry.relay.config.metric_extraction import (
+    get_metric_conditional_tagging_rules,
+    get_metric_extraction_config,
+)
 from sentry.relay.utils import to_camel_case_name
 from sentry.utils import metrics
 from sentry.utils.http import get_origins
@@ -215,7 +218,6 @@ class TransactionNameRuleRedaction(TypedDict):
 class TransactionNameRule(TypedDict):
     pattern: str
     expiry: str
-    scope: TransactionNameRuleScope
     redaction: TransactionNameRuleRedaction
 
 
@@ -238,7 +240,6 @@ def _get_tx_name_rule(pattern: str, seen_last: int) -> TransactionNameRule:
         expiry=expiry_at,
         # Some more hardcoded fields for future compatibility. These are not
         # currently used.
-        scope={"source": "url"},
         redaction={"method": "replace", "substitution": "*"},
     )
 
@@ -298,8 +299,8 @@ def add_experimental_config(
     """
     try:
         subconfig = function(*args, **kwargs)
-    except Exception as e:
-        sentry_sdk.capture_exception(e)
+    except Exception:
+        logger.error("Exception while building Relay project config field", exc_info=True)
     else:
         if subconfig is not None:
             config[key] = subconfig
@@ -388,6 +389,8 @@ def _get_project_config(
             config, "metricConditionalTagging", get_metric_conditional_tagging_rules, project
         )
 
+        add_experimental_config(config, "metricExtraction", get_metric_extraction_config, project)
+
     if features.has("organizations:metrics-extraction", project.organization):
         config["sessionMetrics"] = {
             "version": EXTRACT_ABNORMAL_MECHANISM_VERSION
diff --git a/src/sentry/relay/config/metric_extraction.py b/src/sentry/relay/config/metric_extraction.py
--- a/src/sentry/relay/config/metric_extraction.py
+++ b/src/sentry/relay/config/metric_extraction.py
@@ -1,7 +1,13 @@
+import logging
 from dataclasses import dataclass
-from typing import Any, List, Literal, Sequence, Tuple, TypedDict, Union, cast
+from typing import Any, List, Literal, Optional, Sequence, Tuple, TypedDict, Union, cast
 
+from typing_extensions import NotRequired
+
+from sentry import features
 from sentry.api.endpoints.project_transaction_threshold import DEFAULT_THRESHOLD
+from sentry.constants import DataCategory
+from sentry.incidents.models import AlertRule, AlertRuleStatus
 from sentry.models import (
     Project,
     ProjectTransactionThreshold,
@@ -9,18 +15,162 @@
     TransactionMetric,
 )
 
+logger = logging.getLogger(__name__)
+
+
+# GENERIC METRIC EXTRACTION
+
+
+# Name component of MRIs used for custom alert metrics.
+# TODO: Move to a common module.
+CUSTOM_ALERT_METRIC_NAME = "transactions/alert"
+
+# Version of the metric extraction config.
+_METRIC_EXTRACTION_VERSION = 1
 
-class RuleConditionInner(TypedDict):
-    op: Literal["eq", "gt", "gte"]
+# Maximum number of custom metrics that can be extracted for alert rules with
+# advanced filter expressions.
+_MAX_ALERT_METRICS = 100
+
+# Base type for conditions to evaluate on payloads.
+# TODO: Streamline with dynamic sampling.
+RuleCondition = Union["LogicalRuleCondition", "ComparingRuleCondition", "NotRuleCondition"]
+
+
+class ComparingRuleCondition(TypedDict):
+    """RuleCondition that compares a named field to a reference value."""
+
+    op: Literal["eq", "gt", "gte", "lt", "lte", "glob"]
     name: str
     value: Any
 
 
-# mypy does not support recursive types. type definition is a very small subset
-# of the values relay actually accepts
-class RuleCondition(TypedDict):
-    op: Literal["and"]
-    inner: Sequence[RuleConditionInner]
+class LogicalRuleCondition(TypedDict):
+    """RuleCondition that applies a logical operator to a sequence of conditions."""
+
+    op: Literal["and", "or"]
+    inner: Sequence[RuleCondition]
+
+
+class NotRuleCondition(TypedDict):
+    """RuleCondition that negates an inner condition."""
+
+    op: Literal["not"]
+    inner: RuleCondition
+
+
+class TagSpec(TypedDict):
+    """
+    Configuration for a tag to add to a metric.
+
+    Tags values can be static if defined through `value` or dynamically queried
+    from the payload if defined through `field`. These two options are mutually
+    exclusive, behavior is undefined if both are specified.
+    """
+
+    key: str
+    field: NotRequired[str]
+    value: NotRequired[str]
+    condition: NotRequired[RuleCondition]
+
+
+class MetricSpec(TypedDict):
+    """
+    Specification for a metric to extract from some data.
+
+    The metric type is given as part of the MRI (metric reference identifier)
+    which must follow the form: `<type>:<namespace>/<name>@<unit>`.
+
+    How the metric's value is obtained depends on the metric type:
+     - Counter metrics are a special case, since the default product counters do
+       not count any specific field but rather the occurrence of the event. As
+       such, there is no value expression, and the field is set to `None`.
+       Semantics of specifying remain undefined at this point.
+     - Distribution metrics require a numeric value.
+     - Set metrics require a string value, which is then emitted into the set as
+       unique value. Insertion of numbers and other types is undefined.
+    """
+
+    category: Literal["transaction"]
+    mri: str
+    field: NotRequired[Optional[str]]
+    condition: NotRequired[RuleCondition]
+    tags: NotRequired[Sequence[TagSpec]]
+
+
+class MetricExtractionConfig(TypedDict):
+    """Configuration for generic extraction of metrics from all data categories."""
+
+    version: int
+    metrics: List[MetricSpec]
+
+
+def get_metric_extraction_config(project: Project) -> Optional[MetricExtractionConfig]:
+    """
+    Returns generic metric extraction config for the given project.
+
+    This requires respective feature flags to be enabled. At the moment, metrics
+    for the following models are extracted:
+     - Performance alert rules which advanced filter expressions.
+    """
+
+    if not features.has("organizations:on-demand-metrics-extraction", project.organization):
+        return None
+
+    alerts = (
+        AlertRule.objects.fetch_for_project(project)
+        .filter(status=AlertRuleStatus.PENDING.value)
+        .select_related("snuba_query")
+    )
+
+    metrics: List[MetricSpec] = []
+    for alert in alerts:
+        if metric := _convert_alert_to_metric(alert):
+            metrics.append(metric)
+
+    if not metrics:
+        return None
+
+    if len(metrics) > _MAX_ALERT_METRICS:
+        logger.error("Too many custom alert metrics for project")
+        metrics = metrics[:_MAX_ALERT_METRICS]
+
+    return {
+        "version": _METRIC_EXTRACTION_VERSION,
+        "metrics": metrics,
+    }
+
+
+def _convert_alert_to_metric(alert: AlertRule) -> Optional[MetricSpec]:
+    if not alert.is_custom_metric:
+        return None
+
+    # TODO: Determine these based on the alert.snuba_query.field:
+    metric_type = "c"
+    metric_unit = "none"
+    field = None
+
+    condition = _convert_alert_query_to_condition(alert.snuba_query.query)
+    query_hash = str(alert.id)  # TODO: Hash field + condition
+
+    return {
+        "category": DataCategory.TRANSACTION.api_name(),
+        "mri": f"{metric_type}:{CUSTOM_ALERT_METRIC_NAME}@{metric_unit}",
+        "field": field,
+        "condition": condition,
+        "tags": [{"key": "query_hash", "value": query_hash}],
+    }
+
+
+def _convert_alert_query_to_condition(query: str) -> RuleCondition:
+    # TODO: Parse and convert the query using QueryBuilder
+    return {
+        "op": "and",
+        "inner": [],
+    }
+
+
+# CONDITIONAL TAGGING
 
 
 class MetricConditionalTaggingRule(TypedDict):
@@ -102,7 +252,7 @@ def _threshold_to_rules(
     threshold: Union[
         ProjectTransactionThreshold, ProjectTransactionThresholdOverride, _DefaultThreshold
     ],
-    extra_conditions: Sequence[RuleConditionInner],
+    extra_conditions: Sequence[RuleCondition],
 ) -> Sequence[MetricConditionalTaggingRule]:
     frustrated: MetricConditionalTaggingRule = {
         "condition": {


Test patch:
diff --git a/tests/sentry/ingest/test_transaction_clusterer.py b/tests/sentry/ingest/test_transaction_clusterer.py
--- a/tests/sentry/ingest/test_transaction_clusterer.py
+++ b/tests/sentry/ingest/test_transaction_clusterer.py
@@ -372,13 +372,11 @@ def _get_projconfig_tx_rules(project: Project):
             {
                 "pattern": "/rule/*/0/**",
                 "expiry": "1970-04-01T00:00:00Z",
-                "scope": {"source": "url"},
                 "redaction": {"method": "replace", "substitution": "*"},
             },
             {
                 "pattern": "/rule/*/1/**",
                 "expiry": "1970-04-01T00:00:01Z",
-                "scope": {"source": "url"},
                 "redaction": {"method": "replace", "substitution": "*"},
             },
         ]
diff --git a/tests/sentry/relay/test_config.py b/tests/sentry/relay/test_config.py
--- a/tests/sentry/relay/test_config.py
+++ b/tests/sentry/relay/test_config.py
@@ -1,7 +1,7 @@
 import time
 from datetime import datetime, timedelta
 from unittest import mock
-from unittest.mock import patch
+from unittest.mock import ANY, patch
 
 import pytest
 import pytz
@@ -21,6 +21,7 @@
 from sentry.models import ProjectKey, ProjectTeam
 from sentry.models.transaction_threshold import TransactionMetric
 from sentry.relay.config import ProjectConfig, get_project_config
+from sentry.snuba.dataset import Dataset
 from sentry.testutils.factories import Factories
 from sentry.testutils.helpers import Feature
 from sentry.testutils.helpers.options import override_options
@@ -134,15 +135,15 @@ def test_get_project_config(default_project, insta_snapshot, django_cache, full)
 @django_db_all
 @region_silo_test(stable=True)
 @mock.patch("sentry.relay.config.generate_rules", side_effect=SOME_EXCEPTION)
-@mock.patch("sentry.relay.config.sentry_sdk")
-def test_get_experimental_config_dyn_sampling(mock_sentry_sdk, _, default_project):
+@mock.patch("sentry.relay.config.logger")
+def test_get_experimental_config_dyn_sampling(mock_logger, _, default_project):
     keys = ProjectKey.objects.filter(project=default_project)
     with Feature({"organizations:dynamic-sampling": True}):
         # Does not raise:
         cfg = get_project_config(default_project, full_config=True, project_keys=keys)
     # Key is missing from config:
     assert "dynamicSampling" not in cfg.to_dict()["config"]
-    assert mock_sentry_sdk.capture_exception.call_args == mock.call(SOME_EXCEPTION)
+    assert mock_logger.error.call_args == mock.call(ANY, exc_info=True)
 
 
 @django_db_all
@@ -669,3 +670,57 @@ def test_healthcheck_filter(default_project, has_health_check, health_check_set)
     config_has_health_check = "ignoreTransactions" in filter_settings
     should_have_health_check_config = has_health_check and health_check_set
     assert config_has_health_check == should_have_health_check_config
+
+
+@django_db_all
+def test_alert_metric_extraction_rules_empty(default_project):
+    features = {
+        "organizations:transaction-metrics-extraction": True,
+        "organizations:on-demand-metrics-extraction": True,
+    }
+
+    with Feature(features):
+        config = get_project_config(default_project).to_dict()["config"]
+        validate_project_config(json.dumps(config), strict=False)
+        assert "metricExtraction" not in config
+
+
+@django_db_all
+def test_alert_metric_extraction_rules(default_project, factories):
+    # Alert compatible with out-of-the-box metrics. This should NOT be included
+    # in the config.
+    factories.create_alert_rule(
+        default_project.organization,
+        [default_project],
+        query="event.type:transaction environment:production",
+        dataset=Dataset.Transactions,
+    )
+
+    # Alert requiring an on-demand metric. This should be included in the config.
+    factories.create_alert_rule(
+        default_project.organization,
+        [default_project],
+        query="event.type:transaction transaction.duration:<10m",
+        dataset=Dataset.Transactions,
+    )
+
+    features = {
+        "organizations:transaction-metrics-extraction": True,
+        "organizations:on-demand-metrics-extraction": True,
+    }
+
+    with Feature(features):
+        config = get_project_config(default_project).to_dict()["config"]
+        validate_project_config(json.dumps(config), strict=False)
+        assert config["metricExtraction"] == {
+            "version": 1,
+            "metrics": [
+                {
+                    "category": "transaction",
+                    "mri": "c:transactions/alert@none",
+                    "field": None,
+                    "condition": {"op": "and", "inner": []},
+                    "tags": [{"key": "query_hash", "value": ANY}],
+                }
+            ],
+        }