Repository: getsentry/sentry

Issue:
Issue Platform: Schema Validation does not align with Sentry event schema:
This schema is incorrect/incomplete compared to ingest. I'm not sure what the source of it was, but there are fields missing, and also many fields are incorrect. e.g `name` is a valid property of `user`, but in this schema it is not.

https://github.com/getsentry/sentry/blob/129903f663585acb4c0107f8d8a41f11fac77e2a/src/sentry/issues/json_schemas.py#L3

I also do not understand why we tried to define our own event schema JSON, we already have one generated here:
https://github.com/getsentry/sentry-data-schemas/blob/main/relay/event.schema.json


Discussion:
@JoshFerge Yeah we should probably move to the official schema. We weren't aware of that at the time/weren't sure how to access it.

We should be careful to make sure that this doesn't break any existing incoming data too. Probably need to shadow run it and see if the failure rate increases significantly.

Resolving patch:
diff --git a/src/sentry/issues/json_schemas.py b/src/sentry/issues/json_schemas.py
--- a/src/sentry/issues/json_schemas.py
+++ b/src/sentry/issues/json_schemas.py
@@ -1,6 +1,12 @@
+import logging
+import pathlib
 from typing import Any, Mapping
 
-EVENT_PAYLOAD_SCHEMA: Mapping[str, Any] = {
+from sentry.utils import json
+
+logger = logging.getLogger(__name__)
+
+LEGACY_EVENT_PAYLOAD_SCHEMA: Mapping[str, Any] = {
     "type": "object",
     "properties": {
         # required properties
@@ -154,3 +160,16 @@
     ],
     "additionalProperties": False,
 }
+
+
+_EVENT_PAYLOAD_SCHEMA_JSON_FILE = pathlib.PurePath(__file__).with_name("event.schema.json")
+
+try:
+    with open(_EVENT_PAYLOAD_SCHEMA_JSON_FILE) as f:
+        EVENT_PAYLOAD_SCHEMA = json.load(f)
+
+except Exception:
+    logger.exception(
+        "Failed to load Events schema from 'event.schema.json', falling back to hardcoded schema"
+    )
+    EVENT_PAYLOAD_SCHEMA = LEGACY_EVENT_PAYLOAD_SCHEMA
diff --git a/src/sentry/issues/occurrence_consumer.py b/src/sentry/issues/occurrence_consumer.py
--- a/src/sentry/issues/occurrence_consumer.py
+++ b/src/sentry/issues/occurrence_consumer.py
@@ -15,7 +15,7 @@
 from sentry.issues.grouptype import get_group_type_by_type_id
 from sentry.issues.ingest import save_issue_occurrence
 from sentry.issues.issue_occurrence import DEFAULT_LEVEL, IssueOccurrence, IssueOccurrenceData
-from sentry.issues.json_schemas import EVENT_PAYLOAD_SCHEMA
+from sentry.issues.json_schemas import EVENT_PAYLOAD_SCHEMA, LEGACY_EVENT_PAYLOAD_SCHEMA
 from sentry.issues.producer import PayloadType
 from sentry.issues.status_change_consumer import process_status_change_message
 from sentry.models.organization import Organization
@@ -173,7 +173,15 @@ def _get_kwargs(payload: Mapping[str, Any]) -> Mapping[str, Any]:
                         sample_rate=1.0,
                         tags={"occurrence_type": occurrence_data["type"]},
                     )
-                    raise
+                    try:
+                        jsonschema.validate(event_data, LEGACY_EVENT_PAYLOAD_SCHEMA)
+                    except jsonschema.exceptions.ValidationError:
+                        metrics.incr(
+                            "occurrence_ingest.legacy_event_payload_invalid",
+                            sample_rate=1.0,
+                            tags={"occurrence_type": occurrence_data["type"]},
+                        )
+                        raise
 
                 event_data["metadata"] = {
                     # This allows us to show the title consistently in discover


Test patch:
diff --git a/tests/sentry/issues/test_json_schemas.py b/tests/sentry/issues/test_json_schemas.py
new file mode 100644
--- /dev/null
+++ b/tests/sentry/issues/test_json_schemas.py
@@ -0,0 +1,22 @@
+from importlib import reload
+from unittest import mock
+
+from sentry.issues import json_schemas
+from sentry.testutils.cases import TestCase
+
+
+class JsonSchemasTest(TestCase):
+    def test_loads_json_schema(self) -> None:
+        assert json_schemas.EVENT_PAYLOAD_SCHEMA != json_schemas.LEGACY_EVENT_PAYLOAD_SCHEMA
+        assert (
+            json_schemas.EVENT_PAYLOAD_SCHEMA.get("description")
+            == " The sentry v7 event structure."
+        )
+
+    def test_falls_back_to_legacy(self) -> None:
+        with mock.patch(
+            "sentry.issues.json_schemas.open", mock.mock_open(read_data="invalid json")
+        ):
+            reload(json_schemas)
+            assert json_schemas.EVENT_PAYLOAD_SCHEMA == json_schemas.LEGACY_EVENT_PAYLOAD_SCHEMA
+        reload(json_schemas)
diff --git a/tests/sentry/issues/test_occurrence_consumer.py b/tests/sentry/issues/test_occurrence_consumer.py
--- a/tests/sentry/issues/test_occurrence_consumer.py
+++ b/tests/sentry/issues/test_occurrence_consumer.py
@@ -4,6 +4,7 @@
 from copy import deepcopy
 from datetime import timezone
 from typing import Any, Dict, Optional, Sequence, Type
+from unittest import mock
 
 import pytest
 from jsonschema import ValidationError
@@ -208,7 +209,7 @@ def run_invalid_payload_test(
 
     def test_invalid_payload(self) -> None:
         self.run_invalid_payload_test(
-            remove_event_fields=["project_id"],
+            remove_event_fields=["project_id"], expected_error=InvalidEventPayloadError
         )
         self.run_invalid_payload_test(
             remove_event_fields=["timestamp"], expected_error=ValidationError
@@ -216,13 +217,11 @@ def test_invalid_payload(self) -> None:
         self.run_invalid_payload_test(
             remove_event_fields=["platform"], expected_error=ValidationError
         )
-        self.run_invalid_payload_test(remove_event_fields=["tags"], expected_error=ValidationError)
+
         self.run_invalid_payload_test(
             update_event_fields={"project_id": "p_id"}, expected_error=InvalidEventPayloadError
         )
-        self.run_invalid_payload_test(
-            update_event_fields={"timestamp": 0000}, expected_error=ValidationError
-        )
+
         self.run_invalid_payload_test(
             update_event_fields={"platform": 0000}, expected_error=ValidationError
         )
@@ -233,10 +232,50 @@ def test_invalid_payload(self) -> None:
     def test_valid(self) -> None:
         self.run_test(get_test_message(self.project.id))
 
+    def test_numeric_timestamp_valid_with_new_schema(self) -> None:
+        # per https://develop.sentry.dev/sdk/event-payloads/ timestamp can be numeric
+
+        message = deepcopy(get_test_message(self.project.id))
+        message["event"]["timestamp"] = 0000
+        self.run_test(message)
+
+    def test_tags_not_required_with_new_schema(self) -> None:
+        # per https://develop.sentry.dev/sdk/event-payloads/ tags are optional
+        message = deepcopy(get_test_message(self.project.id))
+        message["event"].pop("tags")
+        self.run_test(message)
+
     def test_valid_nan(self) -> None:
+        # NaN is invalid in new event schema, but valid in legacy schema, so it emits only one of the metrics
         message = deepcopy(get_test_message(self.project.id))
         message["event"]["tags"]["nan-tag"] = float("nan")
-        self.run_test(message)
+        with mock.patch("sentry.issues.occurrence_consumer.metrics") as metrics:
+            self.run_test(message)
+            metrics.incr.assert_called_once_with(
+                "occurrence_ingest.event_payload_invalid",
+                sample_rate=mock.ANY,
+                tags={"occurrence_type": mock.ANY},
+            )
+
+    def test_invalid_payload_emits_both_metrics(self) -> None:
+        with mock.patch("sentry.issues.occurrence_consumer.metrics") as metrics:
+            self.run_invalid_payload_test(
+                remove_event_fields=["timestamp"], expected_error=ValidationError
+            )
+            metrics.incr.assert_has_calls(
+                [
+                    mock.call(
+                        "occurrence_ingest.event_payload_invalid",
+                        sample_rate=mock.ANY,
+                        tags={"occurrence_type": mock.ANY},
+                    ),
+                    mock.call(
+                        "occurrence_ingest.legacy_event_payload_invalid",
+                        sample_rate=mock.ANY,
+                        tags={"occurrence_type": mock.ANY},
+                    ),
+                ]
+            )
 
     def test_missing_event_id_and_event_data(self) -> None:
         message = deepcopy(get_test_message(self.project.id))