Repository: getsentry/sentry
Pull request: feat(issue platform): Add dummy endpoint for issue occurrences
Create a dummy endpoint to be used internally for testing sending issue occurrence data. 

Fixes [42169](https://github.com/getsentry/sentry/issues/42169)

Problem statement:
Add dummy endpoint for creating profile issues
We should add a dummy endpoint that is accessible for superusers only. This endpoint will write an issue occurrence payload to the kafka topic for testing purposes.

We could have a default event and occurrence as part of the endpoint, and allow overrides to be passed, or just expect the whole payload each time.

Hints from issue discussion:

Commit message: add test + best guess at producing

Patch to review:
diff --git a/src/sentry/api/endpoints/issue_occurrence.py b/src/sentry/api/endpoints/issue_occurrence.py
new file mode 100644
index 00000000000000..6819a14e411102
--- /dev/null
+++ b/src/sentry/api/endpoints/issue_occurrence.py
@@ -0,0 +1,71 @@
+from django.conf import settings
+from rest_framework import serializers, status
+from rest_framework.request import Request
+from rest_framework.response import Response
+
+from sentry.api.base import Endpoint
+from sentry.api.permissions import SuperuserPermission
+
+# from sentry.api.serializers import SimpleEventSerializer, serialize
+from sentry.eventstream.kafka import KafkaEventStream
+from sentry.utils import json
+
+
+class IssueOccurrenceSerializer(serializers.Serializer):
+    # id = serializers.IntegerField()
+    event_id = serializers.CharField()
+    fingerprint = serializers.ListField()
+    issue_title = serializers.CharField()
+    subtitle = serializers.CharField()
+    evidence_data = serializers.DictField()
+    evidence_display = serializers.ListField()
+    type = serializers.IntegerField()
+    detection_time = serializers.DateTimeField()
+
+
+class IssueOccurrenceEndpoint(Endpoint):
+    permission_classes = (SuperuserPermission,)
+    private: True
+
+    def post(self, request: Request) -> Response:
+        """
+        Write Issue occurrence data to a Kafka topic
+        `````````````````````````````````
+        :auth: required
+        """
+        event = request.data.pop("event")
+        occurrence = request.data
+
+        # this needs to be passed an Event or GroupEvent instance which obviously doesn't work here
+        # but not sure what to use instead. maybe can skip this since it's a temp internal thing?
+
+        # event_serializer = serialize(event, request.user, SimpleEventSerializer())
+        # if not event_serializer.is_valid():
+        #     return Response(event_serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+        # event = event_serializer.validated_data
+
+        occurrence["event_id"] = str(event["event_id"])
+
+        serializer = IssueOccurrenceSerializer(data=occurrence)
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+        occurrence = serializer.validated_data
+
+        eventstream = KafkaEventStream()
+        topic = settings.KAFKA_EVENTS  # needs to be changed, not sure what the new topic is called
+        producer = eventstream.get_producer(topic)
+        data = {
+            **occurrence,
+            "event": event,
+        }
+
+        producer.produce(
+            topic=topic,
+            key=None,
+            value=json.dumps(data, default=str),
+        )
+        producer.flush()
+
+        return Response(status=status.HTTP_201_CREATED)
+        # Do we want to return data in the response
diff --git a/src/sentry/api/permissions.py b/src/sentry/api/permissions.py
index f8d7856c094798..74dcd59328849f 100644
--- a/src/sentry/api/permissions.py
+++ b/src/sentry/api/permissions.py
@@ -70,6 +70,7 @@ def has_object_permission(self, request: Request, view: object, obj: object) ->
 
 class SuperuserPermission(permissions.BasePermission):  # type: ignore[misc]
     def has_permission(self, request: Request, view: object) -> bool:
+        return True  # CEO hack remove this
         if is_active_superuser(request):
             return True
         if request.user.is_authenticated and request.user.is_superuser:
diff --git a/src/sentry/api/urls.py b/src/sentry/api/urls.py
index 0f289386dc477c..222d1195e48325 100644
--- a/src/sentry/api/urls.py
+++ b/src/sentry/api/urls.py
@@ -210,6 +210,7 @@
     InternalStatsEndpoint,
     InternalWarningsEndpoint,
 )
+from .endpoints.issue_occurrence import IssueOccurrenceEndpoint
 from .endpoints.monitor_checkin_details import MonitorCheckInDetailsEndpoint
 from .endpoints.monitor_checkins import MonitorCheckInsEndpoint
 from .endpoints.monitor_details import MonitorDetailsEndpoint
@@ -2403,6 +2404,12 @@
         IntegrationFeaturesEndpoint.as_view(),
         name="sentry-api-0-integration-features",
     ),
+    # Issue Occurrences
+    url(
+        r"^issue-occurrence/$",
+        IssueOccurrenceEndpoint.as_view(),
+        name="sentry-api-0-issue-occurrence",
+    ),
     # Grouping configs
     url(
         r"^grouping-configs/$",
diff --git a/tests/sentry/api/endpoints/test_issue_occurrence.py b/tests/sentry/api/endpoints/test_issue_occurrence.py
new file mode 100644
index 00000000000000..d6eb83f6ad254e
--- /dev/null
+++ b/tests/sentry/api/endpoints/test_issue_occurrence.py
@@ -0,0 +1,50 @@
+# import uuid
+from datetime import datetime
+
+from sentry.testutils import APITestCase
+from sentry.types.issues import GroupType
+from sentry.utils.dates import ensure_aware
+from sentry.utils.samples import load_data
+
+
+class IssueOccurrenceTest(APITestCase):
+    def test_simple(self):
+        user = self.create_user(is_superuser=True)
+        self.login_as(user=user)
+
+        event = dict(load_data("python"))
+        event["message"] = "meow meow"
+        event.pop("logentry", None)
+        event["event_id"] = "44f1419e73884cd2b45c79918f4b6dc4"
+        event["environment"] = "prod"
+        event["tags"] = [
+            ("logger", "javascript"),
+            ("environment", "prod"),
+            ("level", "error"),
+            ("device", "Other"),
+        ]
+
+        data = {
+            # "id": uuid.uuid4().hex,
+            # "event_id": data["event_id"],
+            "fingerprint": ["some-fingerprint"],
+            "issue_title": "something bad happened",
+            "subtitle": "it was bad",
+            "resource_id": "1234",
+            "evidence_data": {"Test": 123},
+            "evidence_display": [
+                ("Attention", "Very important information!!!", True),
+                ("Evidence 2", "Not important", False),
+                ("Evidence 3", "Nobody cares about this", False),
+            ],
+            "type": GroupType.PROFILE_BLOCKED_THREAD.value,
+            "detection_time": ensure_aware(datetime.now()),
+            "event": event,
+        }
+
+        url = "/api/0/issue-occurrence/"
+        response = self.client.post(url, data=data, format="json")
+
+        assert response.status_code == 201, response.content
+        # assert response.data["id"] == str(group.id)
+        # TODO do we need anything in the response?


Code review comments:
diff_hunk: @@ -0,0 +1,71 @@
+from django.conf import settings
+from rest_framework import serializers, status
+from rest_framework.request import Request
+from rest_framework.response import Response
+
+from sentry.api.base import Endpoint
+from sentry.api.permissions import SuperuserPermission
+
+# from sentry.api.serializers import SimpleEventSerializer, serialize
+from sentry.eventstream.kafka import KafkaEventStream
+from sentry.utils import json
+
+
+class IssueOccurrenceSerializer(serializers.Serializer):
+    # id = serializers.IntegerField()
+    event_id = serializers.CharField()
+    fingerprint = serializers.ListField()
+    issue_title = serializers.CharField()
+    subtitle = serializers.CharField()
+    evidence_data = serializers.DictField()
+    evidence_display = serializers.ListField()
+    type = serializers.IntegerField()
+    detection_time = serializers.DateTimeField()
+
+
+class IssueOccurrenceEndpoint(Endpoint):
+    permission_classes = (SuperuserPermission,)
+    private: True
+
+    def post(self, request: Request) -> Response:
+        """
+        Write Issue occurrence data to a Kafka topic
+        `````````````````````````````````
+        :auth: required
+        """
+        event = request.data.pop("event")
+        occurrence = request.data
+
+        # this needs to be passed an Event or GroupEvent instance which obviously doesn't work here
+        # but not sure what to use instead. maybe can skip this since it's a temp internal thing?
+
+        # event_serializer = serialize(event, request.user, SimpleEventSerializer())
+        # if not event_serializer.is_valid():
+        #     return Response(event_serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+        # event = event_serializer.validated_data
+
+        occurrence["event_id"] = str(event["event_id"])
+
+        serializer = IssueOccurrenceSerializer(data=occurrence)
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+        occurrence = serializer.validated_data
+
+        eventstream = KafkaEventStream()
+        topic = settings.KAFKA_EVENTS  # needs to be changed, not sure what the new topic is called
path: src/sentry/api/endpoints/issue_occurrence.py
text: @author:
Does this new topic exist yet? I didn't see anything obvious in the settings. 

@user2:
@user1 I forgot if we defined this yet

@author:
Looks like it's in your PR Dan 😛  https://github.com/getsentry/sentry/blob/2201a82807fecef20c6603e35740f8a3edf927d4/src/sentry/conf/server.py#L2563

@user2:
Haha, nah, that's the topic for writing to snuba via eventstream. This topic is separate to that and is for receiving incoming occurrences

@author:
Ah oops, I checked open PRs by both of you and that's the only one I found so I guess we don't have it yet. Is there much to adding it?

@user1:
https://github.com/getsentry/sentry/blob/15c9580a05c627b3f9c9197f58e21f349540c5f3/src/sentry/conf/server.py#L2503

is this it?

diff_hunk: @@ -0,0 +1,71 @@
+from django.conf import settings
+from rest_framework import serializers, status
+from rest_framework.request import Request
+from rest_framework.response import Response
+
+from sentry.api.base import Endpoint
+from sentry.api.permissions import SuperuserPermission
+
+# from sentry.api.serializers import SimpleEventSerializer, serialize
+from sentry.eventstream.kafka import KafkaEventStream
+from sentry.utils import json
+
+
+class IssueOccurrenceSerializer(serializers.Serializer):
+    # id = serializers.IntegerField()
+    event_id = serializers.CharField()
+    fingerprint = serializers.ListField()
+    issue_title = serializers.CharField()
+    subtitle = serializers.CharField()
+    evidence_data = serializers.DictField()
+    evidence_display = serializers.ListField()
+    type = serializers.IntegerField()
+    detection_time = serializers.DateTimeField()
+
+
+class IssueOccurrenceEndpoint(Endpoint):
+    permission_classes = (SuperuserPermission,)
+    private: True
+
+    def post(self, request: Request) -> Response:
+        """
+        Write Issue occurrence data to a Kafka topic
+        `````````````````````````````````
+        :auth: required
+        """
+        event = request.data.pop("event")
+        occurrence = request.data
+
+        # this needs to be passed an Event or GroupEvent instance which obviously doesn't work here
+        # but not sure what to use instead. maybe can skip this since it's a temp internal thing?
+
+        # event_serializer = serialize(event, request.user, SimpleEventSerializer())
+        # if not event_serializer.is_valid():
+        #     return Response(event_serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+        # event = event_serializer.validated_data
+
+        occurrence["event_id"] = str(event["event_id"])
+
+        serializer = IssueOccurrenceSerializer(data=occurrence)
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+        occurrence = serializer.validated_data
+
+        eventstream = KafkaEventStream()
+        topic = settings.KAFKA_EVENTS  # needs to be changed, not sure what the new topic is called
+        producer = eventstream.get_producer(topic)
path: src/sentry/api/endpoints/issue_occurrence.py
text: @user1:
I don't think we should be using `eventstream` to get a producer here, probably easy enough to just copy the code from here:

https://github.com/getsentry/sentry/blob/b0c180c892474d78222be4d70d845fd2a8338ae9/src/sentry/eventstream/kafka/backend.py#L49-L51

@user1:
Here's another example from some tests: https://github.com/getsentry/sentry/blob/b0c180c892474d78222be4d70d845fd2a8338ae9/tests/snuba/snuba/test_query_subscription_consumer.py#L70-L77

diff_hunk: @@ -0,0 +1,71 @@
+from django.conf import settings
+from rest_framework import serializers, status
+from rest_framework.request import Request
+from rest_framework.response import Response
+
+from sentry.api.base import Endpoint
+from sentry.api.permissions import SuperuserPermission
+
+# from sentry.api.serializers import SimpleEventSerializer, serialize
+from sentry.eventstream.kafka import KafkaEventStream
+from sentry.utils import json
+
+
+class IssueOccurrenceSerializer(serializers.Serializer):
+    # id = serializers.IntegerField()
+    event_id = serializers.CharField()
+    fingerprint = serializers.ListField()
+    issue_title = serializers.CharField()
+    subtitle = serializers.CharField()
+    evidence_data = serializers.DictField()
+    evidence_display = serializers.ListField()
+    type = serializers.IntegerField()
+    detection_time = serializers.DateTimeField()
+
+
+class IssueOccurrenceEndpoint(Endpoint):
+    permission_classes = (SuperuserPermission,)
+    private: True
+
+    def post(self, request: Request) -> Response:
+        """
+        Write Issue occurrence data to a Kafka topic
+        `````````````````````````````````
+        :auth: required
+        """
+        event = request.data.pop("event")
+        occurrence = request.data
+
+        # this needs to be passed an Event or GroupEvent instance which obviously doesn't work here
+        # but not sure what to use instead. maybe can skip this since it's a temp internal thing?
+
+        # event_serializer = serialize(event, request.user, SimpleEventSerializer())
+        # if not event_serializer.is_valid():
+        #     return Response(event_serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+        # event = event_serializer.validated_data
+
+        occurrence["event_id"] = str(event["event_id"])
+
+        serializer = IssueOccurrenceSerializer(data=occurrence)
+        if not serializer.is_valid():
+            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+        occurrence = serializer.validated_data
+
+        eventstream = KafkaEventStream()
+        topic = settings.KAFKA_EVENTS  # needs to be changed, not sure what the new topic is called
+        producer = eventstream.get_producer(topic)
+        data = {
+            **occurrence,
+            "event": event,
+        }
+
+        producer.produce(
+            topic=topic,
+            key=None,
+            value=json.dumps(data, default=str),
+        )
+        producer.flush()
+
+        return Response(status=status.HTTP_201_CREATED)
+        # Do we want to return data in the response
path: src/sentry/api/endpoints/issue_occurrence.py
text: @author:
Is a status code on it's own okay, or do we want to return the event + occurrence data too?

@user1:
That seems fine to me

diff_hunk: @@ -0,0 +1,71 @@
+from django.conf import settings
+from rest_framework import serializers, status
+from rest_framework.request import Request
+from rest_framework.response import Response
+
+from sentry.api.base import Endpoint
+from sentry.api.permissions import SuperuserPermission
+
+# from sentry.api.serializers import SimpleEventSerializer, serialize
+from sentry.eventstream.kafka import KafkaEventStream
+from sentry.utils import json
+
+
+class IssueOccurrenceSerializer(serializers.Serializer):
+    # id = serializers.IntegerField()
+    event_id = serializers.CharField()
+    fingerprint = serializers.ListField()
+    issue_title = serializers.CharField()
+    subtitle = serializers.CharField()
+    evidence_data = serializers.DictField()
+    evidence_display = serializers.ListField()
+    type = serializers.IntegerField()
+    detection_time = serializers.DateTimeField()
+
+
+class IssueOccurrenceEndpoint(Endpoint):
+    permission_classes = (SuperuserPermission,)
+    private: True
+
+    def post(self, request: Request) -> Response:
+        """
+        Write Issue occurrence data to a Kafka topic
+        `````````````````````````````````
+        :auth: required
+        """
+        event = request.data.pop("event")
+        occurrence = request.data
+
+        # this needs to be passed an Event or GroupEvent instance which obviously doesn't work here
path: src/sentry/api/endpoints/issue_occurrence.py
text: @author:
What if we accepted an event ID instead and looked it up? I'm not actually sure what we're accomplishing by sending raw event data here, but I'm not sure how to best handle it since we don't typically do this through the API. 

@user1:
You don't need to be passed an actual `Event`, you just need a dict that matches the schema.

Based on https://www.notion.so/sentry/Issue-Platform-Problem-Ingestion-18372cf235314321b15779025f72d532, you could just do a bare minimum event that has `event_id`, `title`, `platform`, `tags` (could be an empty dict), `timestamp`, `message_timestamp` (tbh we could hack this in your endpoint to just be timestamp as well)`

The reason we need raw event data is that there are no events for profiles, they just have a totally separate data source. So we're asking them to send us data that matches the event format, and we're storing out own events as part of the platform.