Repository: PrefectHQ/prefect

Issue:
Add a general purpose webhook notification block
### First check

- [X] I added a descriptive title to this issue.
- [X] I used the GitHub search to find a similar request and didn't find it.
- [X] I searched the Prefect documentation for this feature.

### Prefect Version

2.x

### Describe the current behavior

Prefect supports several specific notification methods with apprise.
However, there is no support for more custom endpoints now.

### Describe the proposed behavior

Add a general purpose webhook notification block, which will send a notification to any custom url.

### Example Use

Allow prefect to trigger other tasks in some self-hosted service.
Allow prefect to send notifications with more custom services which has a webhook endpoint.

### Additional context

I'm trying to implement such a notification block, and want to get some suggestion / feedback / ideas.

There are also requests for some other notifications, like for discord in #8727
A custom webhook notification can be used when specific ones are not available.


Discussion:

Resolving patch:
diff --git a/src/prefect/blocks/notifications.py b/src/prefect/blocks/notifications.py
--- a/src/prefect/blocks/notifications.py
+++ b/src/prefect/blocks/notifications.py
@@ -5,8 +5,10 @@
 from typing_extensions import Literal
 
 from prefect.blocks.abstract import NotificationBlock
+from prefect.blocks.fields import SecretDict
 from prefect.events.instrument import instrument_instance_method_call
 from prefect.utilities.asyncutils import sync_compatible
+from prefect.utilities.templating import apply_values, find_placeholders
 
 PREFECT_NOTIFY_TYPE_DEFAULT = "prefect_default"
 
@@ -486,3 +488,132 @@ def block_initialization(self) -> None:
             ).url()
         )
         self._start_apprise_client(url)
+
+
+class CustomWebhookNotificationBlock(NotificationBlock):
+    """
+    Enables sending notifications via any custom webhook.
+
+    All nested string param contains `{{key}}` will be substituted with value from context/secrets.
+
+    Context values include: `subject`, `body` and `name`.
+
+    Examples:
+        Load a saved custom webhook and send a message:
+        ```python
+        from prefect.blocks.notifications import CustomWebhookNotificationBlock
+
+        custom_webhook_block = CustomWebhookNotificationBlock.load("BLOCK_NAME")
+
+        custom_webhook_block.notify("Hello from Prefect!")
+        ```
+    """
+
+    _block_type_name = "Custom Webhook"
+    _logo_url = "https://images.ctfassets.net/gm98wzqotmnx/6ciCsTFsvUAiiIvTllMfOU/627e9513376ca457785118fbba6a858d/webhook_icon_138018.png?h=250"
+    _documentation_url = "https://docs.prefect.io/api-ref/prefect/blocks/notifications/#prefect.blocks.notifications.CustomWebhookNotificationBlock"
+
+    name: str = Field(title="Name", description="Name of the webhook.")
+
+    url: str = Field(
+        title="Webhook URL",
+        description="The webhook URL.",
+        example="https://hooks.slack.com/XXX",
+    )
+
+    method: Literal["GET", "POST", "PUT", "PATCH", "DELETE"] = Field(
+        default="POST", description="The webhook request method. Defaults to `POST`."
+    )
+
+    params: Optional[Dict[str, str]] = Field(
+        default=None, title="Query Params", description="Custom query params."
+    )
+    json_data: Optional[dict] = Field(
+        default=None,
+        title="JSON Data",
+        description="Send json data as payload.",
+        example=(
+            '{"text": "{{subject}}\\n{{body}}", "title": "{{name}}", "token":'
+            ' "{{tokenFromSecrets}}"}'
+        ),
+    )
+    form_data: Optional[Dict[str, str]] = Field(
+        default=None,
+        title="Form Data",
+        description=(
+            "Send form data as payload. Should not be used together with _JSON Data_."
+        ),
+        example=(
+            '{"text": "{{subject}}\\n{{body}}", "title": "{{name}}", "token":'
+            ' "{{tokenFromSecrets}}"}'
+        ),
+    )
+
+    headers: Optional[Dict[str, str]] = Field(None, description="Custom headers.")
+    cookies: Optional[Dict[str, str]] = Field(None, description="Custom cookies.")
+
+    timeout: float = Field(
+        default=10, description="Request timeout in seconds. Defaults to 10."
+    )
+
+    secrets: SecretDict = Field(
+        default_factory=lambda: SecretDict(dict()),
+        title="Custom Secret Values",
+        description="A dictionary of secret values to be substituted in other configs.",
+        example='{"tokenFromSecrets":"SomeSecretToken"}',
+    )
+
+    def _build_request_args(self, body: str, subject: Optional[str]):
+        """Build kwargs for httpx.AsyncClient.request"""
+        # prepare values
+        values = self.secrets.get_secret_value()
+        # use 'null' when subject is None
+        values.update(
+            {
+                "subject": "null" if subject is None else subject,
+                "body": body,
+                "name": self.name,
+            }
+        )
+        # do substution
+        return apply_values(
+            {
+                "method": self.method,
+                "url": self.url,
+                "params": self.params,
+                "data": self.form_data,
+                "json": self.json_data,
+                "headers": self.headers,
+                "cookies": self.cookies,
+                "timeout": self.timeout,
+            },
+            values,
+        )
+
+    def block_initialization(self) -> None:
+        # check form_data and json_data
+        if self.form_data is not None and self.json_data is not None:
+            raise ValueError("both `Form Data` and `JSON Data` provided")
+        allowed_keys = {"subject", "body", "name"}.union(
+            self.secrets.get_secret_value().keys()
+        )
+        # test template to raise a error early
+        for name in ["url", "params", "form_data", "json_data", "headers", "cookies"]:
+            template = getattr(self, name)
+            if template is None:
+                continue
+            # check for placeholders not in predefined keys and secrets
+            placeholders = find_placeholders(template)
+            for placeholder in placeholders:
+                if placeholder.name not in allowed_keys:
+                    raise KeyError(f"{name}/{placeholder}")
+
+    @sync_compatible
+    @instrument_instance_method_call()
+    async def notify(self, body: str, subject: Optional[str] = None):
+        import httpx
+
+        # make request with httpx
+        client = httpx.AsyncClient(headers={"user-agent": "Prefect Notifications"})
+        resp = await client.request(**self._build_request_args(body, subject))
+        resp.raise_for_status()


Test patch:
diff --git a/tests/blocks/test_notifications.py b/tests/blocks/test_notifications.py
--- a/tests/blocks/test_notifications.py
+++ b/tests/blocks/test_notifications.py
@@ -4,6 +4,7 @@
 
 import cloudpickle
 import pytest
+import respx
 
 import prefect
 from prefect.blocks.notifications import (
@@ -13,6 +14,7 @@
     PagerDutyWebHook,
     PREFECT_NOTIFY_TYPE_DEFAULT,
     TwilioSMS,
+    CustomWebhookNotificationBlock,
 )
 from prefect.testing.utilities import AsyncMock
 
@@ -367,3 +369,196 @@ def test_invalid_to_phone_numbers_raises_warning(self, caplog):
             )
 
             assert "Dropped invalid phone # (0000000) specified." in caplog.text
+
+
+class TestCustomWebhook:
+    async def test_notify_async(self):
+        with respx.mock as xmock:
+            xmock.post("https://example.com/")
+
+            custom_block = CustomWebhookNotificationBlock(
+                name="test name",
+                url="https://example.com/",
+                json_data={"msg": "{{subject}}\n{{body}}", "token": "{{token}}"},
+                secrets={"token": "someSecretToken"},
+            )
+            await custom_block.notify("test", "subject")
+
+            last_req = xmock.calls.last.request
+            assert last_req.headers["user-agent"] == "Prefect Notifications"
+            assert (
+                last_req.content
+                == b'{"msg": "subject\\ntest", "token": "someSecretToken"}'
+            )
+            assert last_req.extensions == {
+                "timeout": {"connect": 10, "pool": 10, "read": 10, "write": 10}
+            }
+
+    def test_notify_sync(self):
+        with respx.mock as xmock:
+            xmock.post("https://example.com/")
+
+            custom_block = CustomWebhookNotificationBlock(
+                name="test name",
+                url="https://example.com/",
+                json_data={"msg": "{{subject}}\n{{body}}", "token": "{{token}}"},
+                secrets={"token": "someSecretToken"},
+            )
+            custom_block.notify("test", "subject")
+
+            last_req = xmock.calls.last.request
+            assert last_req.headers["user-agent"] == "Prefect Notifications"
+            assert (
+                last_req.content
+                == b'{"msg": "subject\\ntest", "token": "someSecretToken"}'
+            )
+            assert last_req.extensions == {
+                "timeout": {"connect": 10, "pool": 10, "read": 10, "write": 10}
+            }
+
+    def test_user_agent_override(self):
+        with respx.mock as xmock:
+            xmock.post("https://example.com/")
+
+            custom_block = CustomWebhookNotificationBlock(
+                name="test name",
+                url="https://example.com/",
+                headers={"user-agent": "CustomUA"},
+                json_data={"msg": "{{subject}}\n{{body}}", "token": "{{token}}"},
+                secrets={"token": "someSecretToken"},
+            )
+            custom_block.notify("test", "subject")
+
+            last_req = xmock.calls.last.request
+            assert last_req.headers["user-agent"] == "CustomUA"
+            assert (
+                last_req.content
+                == b'{"msg": "subject\\ntest", "token": "someSecretToken"}'
+            )
+            assert last_req.extensions == {
+                "timeout": {"connect": 10, "pool": 10, "read": 10, "write": 10}
+            }
+
+    def test_timeout_override(self):
+        with respx.mock as xmock:
+            xmock.post("https://example.com/")
+
+            custom_block = CustomWebhookNotificationBlock(
+                name="test name",
+                url="https://example.com/",
+                json_data={"msg": "{{subject}}\n{{body}}", "token": "{{token}}"},
+                secrets={"token": "someSecretToken"},
+                timeout=30,
+            )
+            custom_block.notify("test", "subject")
+
+            last_req = xmock.calls.last.request
+            assert (
+                last_req.content
+                == b'{"msg": "subject\\ntest", "token": "someSecretToken"}'
+            )
+            assert last_req.extensions == {
+                "timeout": {"connect": 30, "pool": 30, "read": 30, "write": 30}
+            }
+
+    def test_request_cookie(self):
+        with respx.mock as xmock:
+            xmock.post("https://example.com/")
+
+            custom_block = CustomWebhookNotificationBlock(
+                name="test name",
+                url="https://example.com/",
+                json_data={"msg": "{{subject}}\n{{body}}", "token": "{{token}}"},
+                cookies={"key": "{{cookie}}"},
+                secrets={"token": "someSecretToken", "cookie": "secretCookieValue"},
+                timeout=30,
+            )
+            custom_block.notify("test", "subject")
+
+            last_req = xmock.calls.last.request
+            assert last_req.headers["cookie"] == "key=secretCookieValue"
+            assert (
+                last_req.content
+                == b'{"msg": "subject\\ntest", "token": "someSecretToken"}'
+            )
+            assert last_req.extensions == {
+                "timeout": {"connect": 30, "pool": 30, "read": 30, "write": 30}
+            }
+
+    def test_subst_nested_list(self):
+        with respx.mock as xmock:
+            xmock.post("https://example.com/")
+
+            custom_block = CustomWebhookNotificationBlock(
+                name="test name",
+                url="https://example.com/",
+                json_data={
+                    "data": {"sub1": [{"in-list": "{{body}}", "name": "{{name}}"}]}
+                },
+                secrets={"token": "someSecretToken"},
+            )
+            custom_block.notify("test", "subject")
+
+            last_req = xmock.calls.last.request
+            assert last_req.headers["user-agent"] == "Prefect Notifications"
+            assert (
+                last_req.content
+                == b'{"data": {"sub1": [{"in-list": "test", "name": "test name"}]}}'
+            )
+            assert last_req.extensions == {
+                "timeout": {"connect": 10, "pool": 10, "read": 10, "write": 10}
+            }
+
+    def test_subst_none(self):
+        with respx.mock as xmock:
+            xmock.post("https://example.com/")
+
+            custom_block = CustomWebhookNotificationBlock(
+                name="test name",
+                url="https://example.com/",
+                json_data={"msg": "{{subject}}\n{{body}}", "token": "{{token}}"},
+                secrets={"token": "someSecretToken"},
+            )
+            # subject=None
+            custom_block.notify("test", None)
+
+            last_req = xmock.calls.last.request
+            assert last_req.headers["user-agent"] == "Prefect Notifications"
+            assert (
+                last_req.content
+                == b'{"msg": "null\\ntest", "token": "someSecretToken"}'
+            )
+            assert last_req.extensions == {
+                "timeout": {"connect": 10, "pool": 10, "read": 10, "write": 10}
+            }
+
+    def test_is_picklable(self):
+        reload_modules()
+        block = CustomWebhookNotificationBlock(
+            name="test name",
+            url="https://example.com/",
+            json_data={"msg": "{{subject}}\n{{body}}", "token": "{{token}}"},
+            secrets={"token": "someSecretToken"},
+        )
+        pickled = cloudpickle.dumps(block)
+        unpickled = cloudpickle.loads(pickled)
+        assert isinstance(unpickled, CustomWebhookNotificationBlock)
+
+    def test_invalid_key_raises_validation_error(self):
+        with pytest.raises(KeyError):
+            CustomWebhookNotificationBlock(
+                name="test name",
+                url="https://example.com/",
+                json_data={"msg": "{{subject}}\n{{body}}", "token": "{{token}}"},
+                secrets={"token2": "someSecretToken"},
+            )
+
+    def test_provide_both_data_and_json_raises_validation_error(self):
+        with pytest.raises(ValueError):
+            CustomWebhookNotificationBlock(
+                name="test name",
+                url="https://example.com/",
+                form_data={"msg": "{{subject}}\n{{body}}", "token": "{{token}}"},
+                json_data={"msg": "{{subject}}\n{{body}}", "token": "{{token}}"},
+                secrets={"token": "someSecretToken"},
+            )