Repository: sequinstream/sequin
Pull request #1195: 📐 Prometheus metrics
Author: login: RTLS
Created: 2025-04-03T18:52:36Z | Merged: 2025-04-07T19:02:24Z



Diff:
diff --git a/config/config.exs b/config/config.exs
index 100d579de..cd0eb7e6d 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -102,6 +102,8 @@ config :sequin, SequinWeb.Endpoint,
   pubsub_server: Sequin.PubSub,
   live_view: [signing_salt: "Sm59ovfq", long_poll_fallback_ms: 3000]
 
+config :sequin, SequinWeb.MetricsEndpoint, adapter: Bandit.PhoenixAdapter
+
 config :sequin, SequinWeb.UserSessionController,
   github: [
     redirect_uri: "http://localhost:4000/auth/github/callback"
diff --git a/config/dev.exs b/config/dev.exs
index c9c219a80..327d0f3c5 100644
--- a/config/dev.exs
+++ b/config/dev.exs
@@ -71,6 +71,8 @@ config :sequin, SequinWeb.Endpoint,
     ]
   ]
 
+config :sequin, SequinWeb.MetricsEndpoint, http: [ip: {127, 0, 0, 1}, port: System.get_env("SEQUIN_METRICS_PORT", "4001")]
+
 config :sequin, SequinWeb.Router,
   admin_user: "admin",
   admin_password: "admin"
diff --git a/config/runtime.exs b/config/runtime.exs
index 90478d7a7..a73636c37 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -29,6 +29,7 @@ maybe_parse_int_env = fn key ->
 end
 
 server_port = String.to_integer(System.get_env("SERVER_PORT") || System.get_env("PORT") || "7376")
+metrics_port = String.to_integer(System.get_env("SEQUIN_METRICS_PORT") || "8376")
 server_host = System.get_env("SERVER_HOST") || System.get_env("PHX_HOST") || "localhost"
 
 if System.get_env("PORT") do
@@ -57,6 +58,7 @@ end
 # script that automatically sets the env var above.
 if System.get_env("PHX_SERVER") do
   config :sequin, SequinWeb.Endpoint, server: true
+  config :sequin, SequinWeb.MetricsEndpoint, server: true
 end
 
 # Deprecate ECTO_IPV6 in favor of PG_IPV6
@@ -149,6 +151,12 @@ if config_env() == :prod and self_hosted do
       long_poll_fallback_ms: String.to_integer(System.get_env("LONG_POLL_FALLBACK_MS", "3000"))
     ]
 
+  config :sequin, SequinWeb.MetricsEndpoint,
+    http: [
+      ip: {0, 0, 0, 0, 0, 0, 0, 0},
+      port: metrics_port
+    ]
+
   config :sequin, :features,
     account_self_signup: account_self_signup,
     provision_default_user: provision_default_user
@@ -205,6 +213,12 @@ if config_env() == :prod and not self_hosted do
     ],
     secret_key_base: secret_key_base
 
+  config :sequin, SequinWeb.MetricsEndpoint,
+    http: [
+      ip: {0, 0, 0, 0, 0, 0, 0, 0},
+      port: 8376
+    ]
+
   config :sequin, :features, account_self_signup: :enabled
   config :sequin, :koala, public_key: "pk_ec2e6140b3d56f5eb1735350eb20e92b8002"
 
diff --git a/docker/dashboard.json b/docker/dashboard.json
new file mode 100644
index 000000000..f537e549c
--- /dev/null
+++ b/docker/dashboard.json
@@ -0,0 +1,912 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 1,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "sqpds"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "msg/s",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.6.0",
+      "targets": [
+        {
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "sum by(replication_slot_id, slot_name) (rate(sequin_messages_ingested_count[$__rate_interval]))",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "{{slot_name}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Messages Ingested by Slot",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "sqpds"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "msg/s",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.6.0",
+      "targets": [
+        {
+          "disableTextWrap": false,
+          "editorMode": "code",
+          "expr": "sum by(consumer_id, consumer_name) (rate(sequin_message_deliver_success_count[$__rate_interval]))",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "{{consumer_name}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Messages Delivered by Sink",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "sqpds"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "Deliver Success/Attempts",
+            "axisPlacement": "auto",
+            "axisSoftMax": 1,
+            "axisSoftMin": 0.9,
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red"
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.6.0",
+      "targets": [
+        {
+          "disableTextWrap": false,
+          "editorMode": "code",
+          "expr": "sum by(consumer_id, consumer_name) ((rate(sequin_message_deliver_success_count[10m])) / (rate(sequin_message_deliver_attempt_count[10m])))",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "{{consumer_name}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Message Delivery Success % by Sink",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "sqpds"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 1,
+          "mappings": [],
+          "min": 0.9,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red"
+              },
+              {
+                "color": "orange",
+                "value": 0.95
+              },
+              {
+                "color": "green",
+                "value": 0.98
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 8,
+      "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "sizing": "auto"
+      },
+      "pluginVersion": "11.6.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum by(consumer_id, consumer_name) (increase(sequin_message_deliver_success_count[5m])) \n/ \nsum by(consumer_id, consumer_name) (increase(sequin_message_deliver_attempt_count[5m]))",
+          "instant": true,
+          "legendFormat": "{{consumer_name}}",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Message Delivery Success % by Sink",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "sqpds"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "µs"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 8,
+        "x": 0,
+        "y": 16
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.6.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "sqpds"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "sequin_ingestion_latency_us",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "{{slot_name}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Slot Ingestion Latency",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "sqpds"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMin": -4,
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "log": 2,
+              "type": "log"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "µs"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 8,
+        "x": 8,
+        "y": 16
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.6.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "sqpds"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "histogram_quantile(0.95, sum by(le, success, consumer_id, consumer_name) (rate(sequin_internal_latency_us_bucket[$__rate_interval])))",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "legendFormat": "{{consumer_name}}: {{success}}",
+          "range": true,
+          "refId": "B",
+          "useBackend": false
+        }
+      ],
+      "title": "Sink Internal Latency",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "sqpds"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMin": -4,
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "log": 2,
+              "type": "log"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "µs"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 8,
+        "x": 16,
+        "y": 16
+      },
+      "id": 9,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.6.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "sqpds"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "histogram_quantile(0.95, sum by(le, success, consumer_id, consumer_name) (rate(sequin_delivery_latency_us_bucket[$__rate_interval])))",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "legendFormat": "{{consumer_name}}: {{success}}",
+          "range": true,
+          "refId": "B",
+          "useBackend": false
+        }
+      ],
+      "title": "Sink Deliver Latency",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "sqpds"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 25
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.6.0",
+      "targets": [
+        {
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "rate(erlang_vm_statistics_reductions_total[$__rate_interval])",
+          "fullMetaSearch": false,
+          "includeNullMetadata": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Beam Reductions",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "sqpds"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 25
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.6.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "sqpds"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "erlang_vm_memory_bytes_total",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "legendFormat": "{{kind}} memory",
+          "range": true,
+          "refId": "B",
+          "useBackend": false
+        }
+      ],
+      "title": "Beam Memory",
+      "type": "timeseries"
+    }
+  ],
+  "preload": false,
+  "refresh": "5s",
+  "schemaVersion": 41,
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Sequin",
+  "uid": "behuozial454wc",
+  "version": 2
+}
\ No newline at end of file
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index 4dd842851..bf252490b 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -54,9 +54,29 @@ services:
     volumes:
       - sequin_redis_data:/data
 
+  sequin_prometheus:
+    image: prom/prometheus
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+
+  sequin_grafana:
+    image: grafana/grafana
+    ports:
+      - "3000:3000"
+    depends_on:
+      - sequin_prometheus
+    volumes:
+      - sequin_grafana_data:/var/lib/grafana
+      - ./grafana_datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
+      - ./grafana_dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml
+      - ./dashboard.json:/etc/grafana/dashboards/dashboards/sequin.json
+
 volumes:
   sequin_postgres_data:
   sequin_redis_data:
+  sequin_grafana_data:
 
 # Un-comment this to enable IPv6
 # You may need to run
diff --git a/docker/grafana_dashboard.yml b/docker/grafana_dashboard.yml
new file mode 100644
index 000000000..20de9fdf4
--- /dev/null
+++ b/docker/grafana_dashboard.yml
@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+  - name: 'default'
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: true
+    updateIntervalSeconds: 10
+    options:
+      path: /etc/grafana/dashboards
diff --git a/docker/grafana_datasource.yml b/docker/grafana_datasource.yml
new file mode 100644
index 000000000..202257035
--- /dev/null
+++ b/docker/grafana_datasource.yml
@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: "Sequin Prometheus"
+    type: prometheus
+    access: proxy
+    url: http://sequin_prometheus:9090
+    isDefault: true
+    editable: true
+    uid: sqpds
diff --git a/docker/prometheus.yml b/docker/prometheus.yml
new file mode 100644
index 000000000..251024a0a
--- /dev/null
+++ b/docker/prometheus.yml
@@ -0,0 +1,8 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'sequin'
+    static_configs:
+      - targets: ['sequin:8376']
+      # - targets: ['host.docker.internal:4001']
diff --git a/docs/mint.json b/docs/mint.json
index b69071f30..98c5c3117 100644
--- a/docs/mint.json
+++ b/docs/mint.json
@@ -122,7 +122,8 @@
         "reference/change-retention",
         "reference/annotations",
         "reference/sequin-yaml",
-        "reference/cli/overview"
+        "reference/cli/overview",
+        "reference/metrics"
       ]
     },
     {
@@ -236,4 +237,4 @@
       "permanent": true
     }
   ]
-}
\ No newline at end of file
+}
diff --git a/docs/reference/configuration.mdx b/docs/reference/configuration.mdx
index b2691293e..ff4f5abac 100644
--- a/docs/reference/configuration.mdx
+++ b/docs/reference/configuration.mdx
@@ -137,9 +137,12 @@ To use GitHub OAuth as a sign-in/sign-up method, provide the following environme
 - `GITHUB_CLIENT_ID`: GitHub OAuth client ID
 - `GITHUB_CLIENT_SECRET`: GitHub OAuth client secret
 
-### Monitoring/stats configuration
+### Metrics configuration
 
-Monitoring/stats configuration is coming soon. [Please comment on this issue](https://github.com/sequinstream/sequin/issues/580) if you'd like to see this feature.
+- `SEQUIN_METRICS_PORT`: Port number for the metrics endpoint (default: 8376)
+  - This is the port that Sequin will expose Prometheus metrics on. Metrics are available at `/metrics` on this port.
+
+See [Metrics](/reference/metrics) reference for more information on the metrics exposed by Sequin.
 
 ## Deploying to production
 
diff --git a/docs/reference/metrics.mdx b/docs/reference/metrics.mdx
new file mode 100644
index 000000000..61aaec9b8
--- /dev/null
+++ b/docs/reference/metrics.mdx
@@ -0,0 +1,68 @@
+---
+title: Metrics
+description: Reference for Sequin's metrics. Use metrics to monitor health and performance.
+---
+
+Sequin exposes a metrics endpoint, using Prometheus text format, which can be integrated into your existing observability workflows. By default, this endpoint is at `/metrics` on port `8376`.
+
+Metrics are designed to answer:
+
+- Is Sequin keeping up with the database? (Ingestion metrics)
+- Is Sequin processing messages quickly? (Internal processing metrics)
+- Are sinks keeping up with Sequin? (Delivery metrics)
+- Is the Sequin service experiencing CPU or memory pressure? (VM metrics)
+
+## Metrics reference
+
+### Ingestion metrics
+
+- `sequin_ingestion_latency_us` (Gauge)
+
+  Measures latency between transaction commit and when Sequin first receives it. Calculated by instrumenting Sequin's own heartbeat messages.
+
+- `sequin_messages_ingested_count` (Counter)
+
+  Number of messages Sequin has received from a given replication slot.
+
+### Internal processing metrics
+
+- `sequin_internal_latency_us` (Histogram)
+
+  Measures time between Sequin ingesting a message and beginning to process it.
+
+### Delivery metrics
+
+- `sequin_delivery_latency_us` (Histogram)
+
+  Measures time between Sequin beginning to deliver a message and a given sink's successful acknowledgement. Some sinks are able to batch messages, in which case this metric measures the batch latency, rather than the per-message latency.
+
+- `sequin_message_deliver_attempt_count` (Counter)
+
+  Incremented for each message Sequin tries to deliver, before receiving a success or failure response.
+
+- `sequin_message_deliver_success_count` (Counter)
+
+  Incremented for each message that Sequin successfully delivers.
+
+- `sequin_message_deliver_failure_count` (Counter)
+
+  Incremented for each message that Sequin fails to deliver to a sink. This is because the sink was either unreachable, timed out, or returned an error.
+
+### Total latency
+
+<Note>
+  The total latency from database commit to message delivery is the sum of the ingestion latency, internal processing latency, and delivery latency.
+</Note>
+
+### VM metrics
+Sequin is implemented with Elixir. Elixir's VM (the BEAM) exposes many metrics. The following are the ones we find most useful for monitoring Sequin:
+
+- `erlang_vm_statistics_reductions_total` (Counter)
+
+  Measures how many VM instructions are executed across all processes.
+  Shows intensity of CPU-bound work.
+  Recommend using with a rate query rather than directly.
+
+- `erlang_vm_memory_bytes_total` (Gauge)
+
+  Memory usage of the entire VM.
diff --git a/docs/running-sequin.mdx b/docs/running-sequin.mdx
index 8ba4c7896..4754ef09d 100644
--- a/docs/running-sequin.mdx
+++ b/docs/running-sequin.mdx
@@ -52,10 +52,12 @@ The easiest way to get started with Sequin is with our [Docker Compose file](htt
       You should see output like the following:
 
       ```
-      CONTAINER ID   IMAGE                                 STATUS          PORTS                                       NAMES
-      abc123def456   sequinstream/sequin:latest           Up 10 seconds    0.0.0.0:7376->7376/tcp                     sequin
-      def456ghi789   redis:7.0                            Up 10 seconds    0.0.0.0:6379->6379/tcp                     sequin-redis
-      ghi789jkl012   postgres:15                          Up 10 seconds    0.0.0.0:5432->5432/tcp                     sequin-postgres
+      CONTAINER ID   IMAGE                                  COMMAND                  CREATED          STATUS                    PORTS                                                 NAMES
+      b21f5eb8e6aa   sequin/sequin:latest                   "/scripts/start_comm…"   15 seconds ago   Up 2 seconds              4000/tcp, 0.0.0.0:7376->7376/tcp, :::7376->7376/tcp   sequin-sequin-1
+      34ee4e39468e   grafana/grafana                        "/run.sh"                15 seconds ago   Up 14 seconds             0.0.0.0:3000->3000/tcp, :::3000->3000/tcp             sequin-sequin_grafana-1
+      01b006a7f300   prom/prometheus                        "/bin/prometheus --c…"   16 seconds ago   Up 15 seconds             0.0.0.0:9090->9090/tcp, :::9090->9090/tcp             sequin-sequin_prometheus-1
+      b0b509e37837   postgres:16                            "docker-entrypoint.s…"   16 seconds ago   Up 15 seconds (healthy)   0.0.0.0:7377->5432/tcp, [::]:7377->5432/tcp           sequin-sequin_postgres-1
+      4438297985da   redis:7                                "docker-entrypoint.s…"   16 seconds ago   Up 15 seconds             0.0.0.0:7378->6379/tcp, [::]:7378->6379/tcp           sequin-sequin_redis-1
       ```
 
       <Check>
@@ -109,6 +111,16 @@ The easiest way to get started with Sequin is with our [Docker Compose file](htt
         </Card>
       </CardGroup>
     </Step>
+
+    <Step title="View metrics in Grafana">
+      Sequin exports  [various metrics](/reference/metrics) in Prometheus format.
+
+      Using the Grafana service from Docker Compose, you can view our [example dashboard here](http://localhost:3000/d/behuozial454wc/sequin). 
+
+      Sign in with Grafana's default credentials:
+      - Username: `admin`
+      - Password: `admin`
+    </Step>
   </Steps>
 
 ## Using your own database
diff --git a/lib/sequin/application.ex b/lib/sequin/application.ex
index 68d0c65bb..77ed3ea51 100644
--- a/lib/sequin/application.ex
+++ b/lib/sequin/application.ex
@@ -17,6 +17,9 @@ defmodule Sequin.Application do
     env = Application.get_env(:sequin, :env)
     children = children(env)
 
+    Sequin.Prometheus.setup()
+    Sequin.PrometheusExporter.setup()
+
     :ets.new(Sequin.Runtime.SlotProcessorServer.ets_table(), [:set, :public, :named_table])
     # Add this line to create the new ETS table for health debouncing
     :ets.new(Sequin.Health.debounce_ets_table(), [:set, :public, :named_table])
@@ -79,7 +82,8 @@ defmodule Sequin.Application do
       {Cluster.Supervisor, [topologies]},
       {Task, fn -> enqueue_workers() end},
       # Start to serve requests, typically the last entry
-      SequinWeb.Endpoint
+      SequinWeb.Endpoint,
+      SequinWeb.MetricsEndpoint
     ]
   end
 
diff --git a/lib/sequin/metrics/prometheus.ex b/lib/sequin/metrics/prometheus.ex
new file mode 100644
index 000000000..a266660b0
--- /dev/null
+++ b/lib/sequin/metrics/prometheus.ex
@@ -0,0 +1,337 @@
+defmodule Sequin.Prometheus do
+  @moduledoc false
+  use Prometheus.Metric
+
+  @spec setup() :: :ok
+  def setup do
+    Gauge.new(
+      name: :sequin_ingestion_latency_us,
+      labels: [:replication_slot_id, :slot_name],
+      help: "The ingestion latency between Postgres and Sequin in microseconds."
+    )
+
+    Histogram.new(
+      name: :sequin_internal_latency_us,
+      labels: [:consumer_id, :consumer_name],
+      buckets: [10, 100, 1000, 10_000, 50_000, 100_000, 500_000, 1_000_000, 5_000_000, 10_000_000, 50_000_000],
+      duration_unit: false,
+      help: "The internal processing latency in microseconds."
+    )
+
+    Histogram.new(
+      name: :sequin_delivery_latency_us,
+      labels: [:consumer_id, :consumer_name, :success],
+      buckets: [10, 100, 1000, 10_000, 50_000, 100_000, 500_000, 1_000_000, 5_000_000, 10_000_000, 50_000_000],
+      duration_unit: false,
+      help: "The delivery latency in microseconds."
+    )
+
+    Histogram.new(
+      name: :sequin_post_delivery_latency_us,
+      labels: [:consumer_id, :consumer_name],
+      buckets: [10, 100, 1000, 10_000, 50_000, 100_000, 500_000, 1_000_000, 5_000_000, 10_000_000, 50_000_000],
+      duration_unit: false,
+      help: "The post-delivery latency in microseconds."
+    )
+
+    Histogram.new(
+      name: :sequin_oldest_message_age_ms,
+      labels: [:consumer_id, :consumer_name],
+      buckets: [1000, 5000, 10_000, 30_000, 60_000, 300_000, 600_000, 1_800_000, 3_600_000],
+      duration_unit: false,
+      help: "The approximate age of the oldest message in milliseconds."
+    )
+
+    Counter.declare(
+      name: :sequin_messages_ingested_count,
+      help: "Total number of messages ingested.",
+      labels: [:replication_slot_id, :slot_name]
+    )
+
+    Counter.declare(
+      name: :sequin_message_deliver_attempt_count,
+      help: "Total number of messages attempted for delivery.",
+      labels: [:consumer_id, :consumer_name]
+    )
+
+    Counter.declare(
+      name: :sequin_message_deliver_success_count,
+      help: "Total number of messages successfully delivered.",
+      labels: [:consumer_id, :consumer_name]
+    )
+
+    Counter.declare(
+      name: :sequin_message_deliver_failure_count,
+      help: "Total number of messages that failed delivery.",
+      labels: [:consumer_id, :consumer_name]
+    )
+
+    Gauge.new(
+      name: :sequin_messages_in_delivery,
+      labels: [:consumer_id, :consumer_name],
+      help: "Approximate number of messages currently in delivery."
+    )
+
+    Gauge.new(
+      name: :sequin_messages_buffered,
+      labels: [:consumer_id, :consumer_name],
+      help: "Approximate number of messages currently buffered."
+    )
+
+    Counter.declare(
+      name: :sequin_bytes_ingested_total,
+      help: "Total number of bytes ingested.",
+      labels: [:consumer_id, :consumer_name]
+    )
+
+    Counter.declare(
+      name: :sequin_bytes_delivered_total,
+      help: "Total number of bytes delivered.",
+      labels: [:consumer_id, :consumer_name]
+    )
+
+    Gauge.new(
+      name: :sequin_messages_in_redelivery,
+      labels: [:consumer_id, :consumer_name],
+      help: "Number of messages currently in re-delivery."
+    )
+
+    Gauge.new(
+      name: :sequin_buffer_memory_utilization_mb,
+      labels: [:consumer_id, :consumer_name],
+      help: "Buffer memory utilization in megabytes."
+    )
+
+    Gauge.new(
+      name: :sequin_buffer_memory_utilization_percent,
+      labels: [:consumer_id, :consumer_name],
+      help: "Buffer memory utilization as a percentage."
+    )
+
+    Gauge.new(
+      name: :sequin_ingestion_saturation_percent,
+      labels: [:consumer_id, :consumer_name],
+      help: "Ingestion saturation as a percentage."
+    )
+
+    Gauge.new(
+      name: :sequin_processing_saturation_percent,
+      labels: [:consumer_id, :consumer_name],
+      help: "Processing saturation as a percentage."
+    )
+
+    Gauge.new(
+      name: :sequin_delivery_saturation_percent,
+      labels: [:consumer_id, :consumer_name],
+      help: "Delivery saturation as a percentage."
+    )
+
+    ## Ecto
+    :ok = :telemetry.attach("sequin-repo-query", [:sequin, :repo, :query], &Sequin.Prometheus.ecto_event/4, %{})
+
+    Histogram.new(
+      name: :sequin_repo_query_query,
+      buckets: [1, 10, 100, 1000, 10_000],
+      labels: [:query],
+      help: "Ecto query time: query.",
+      duration_unit: :milliseconds
+    )
+
+    Histogram.new(
+      name: :sequin_repo_query_idle,
+      buckets: [1, 10, 100, 1000, 10_000],
+      labels: [:query],
+      help: "Ecto query time: idle.",
+      duration_unit: :milliseconds
+    )
+
+    Histogram.new(
+      name: :sequin_repo_query_queue,
+      buckets: [1, 10, 100, 1000, 10_000],
+      labels: [:query],
+      help: "Ecto query time: queue.",
+      duration_unit: :milliseconds
+    )
+
+    Histogram.new(
+      name: :sequin_repo_query_decode,
+      buckets: [1, 10, 100, 1000, 10_000],
+      labels: [:query],
+      help: "Ecto query time: decode.",
+      duration_unit: :milliseconds
+    )
+
+    Histogram.new(
+      name: :sequin_repo_query_total,
+      buckets: [1, 10, 100, 1000, 10_000],
+      labels: [:query],
+      help: "Ecto query time: total.",
+      duration_unit: :milliseconds
+    )
+  end
+
+  def ecto_event([:sequin, :repo, :query], measurements, metadata, _config) do
+    # IO.inspect(measurements, label: "measurements")
+    lbls = [String.slice(metadata.query, 0, 100)]
+
+    Histogram.observe([name: :sequin_repo_query_query, labels: lbls], measurements.query_time)
+    Histogram.observe([name: :sequin_repo_query_total, labels: lbls], measurements.total_time)
+
+    case Map.fetch(metadata, :idle_time) do
+      {:ok, t} -> Histogram.observe([name: :sequin_repo_query_idle, labels: lbls], t)
+      _ -> nil
+    end
+
+    case Map.fetch(metadata, :decode_time) do
+      {:ok, t} -> Histogram.observe([name: :sequin_repo_query_decode, labels: lbls], t)
+      _ -> nil
+    end
+
+    case Map.fetch(metadata, :queue_time) do
+      {:ok, t} -> Histogram.observe([name: :sequin_repo_query_queue, labels: lbls], t)
+      _ -> nil
+    end
+  end
+
+  # no catchall - :telemetry prints error and removes handler in case of no match
+
+  @spec increment_message_deliver_attempt(consumer_id :: String.t(), consumer_name :: String.t(), count :: number()) ::
+          :ok
+  def increment_message_deliver_attempt(consumer_id, consumer_name, count \\ 1) do
+    Counter.inc(
+      name: :sequin_message_deliver_attempt_count,
+      labels: [consumer_id, consumer_name],
+      count: count
+    )
+  end
+
+  @spec increment_message_deliver_success(consumer_id :: String.t(), consumer_name :: String.t(), count :: number()) ::
+          :ok
+  def increment_message_deliver_success(consumer_id, consumer_name, count \\ 1) do
+    Counter.inc(
+      name: :sequin_message_deliver_success_count,
+      labels: [consumer_id, consumer_name],
+      count: count
+    )
+  end
+
+  @spec increment_message_deliver_failure(consumer_id :: String.t(), consumer_name :: String.t(), count :: number()) ::
+          :ok
+  def increment_message_deliver_failure(consumer_id, consumer_name, count \\ 1) do
+    Counter.inc(
+      name: :sequin_message_deliver_failure_count,
+      labels: [consumer_id, consumer_name],
+      count: count
+    )
+  end
+
+  @spec observe_messages_ingested_latency(consumer_id :: String.t(), consumer_name :: String.t(), latency_ms :: number()) ::
+          :ok
+  def observe_messages_ingested_latency(consumer_id, consumer_name, latency_ms) do
+    Histogram.observe(
+      name: :sequin_messages_ingested_latency_ms,
+      labels: [consumer_id, consumer_name],
+      value: latency_ms
+    )
+  end
+
+  @spec observe_ingestion_latency(replication_slot_id :: String.t(), slot_name :: String.t(), latency_us :: number()) ::
+          :ok
+  def observe_ingestion_latency(replication_slot_id, slot_name, latency_us) do
+    Gauge.set([name: :sequin_ingestion_latency_us, labels: [replication_slot_id, slot_name]], latency_us)
+  end
+
+  @spec observe_internal_latency(consumer_id :: String.t(), consumer_name :: String.t(), latency_us :: number()) :: :ok
+  def observe_internal_latency(consumer_id, consumer_name, latency_us) do
+    Histogram.observe([name: :sequin_internal_latency_us, labels: [consumer_id, consumer_name]], latency_us)
+  end
+
+  @spec observe_delivery_latency(
+          consumer_id :: String.t(),
+          consumer_name :: String.t(),
+          success :: :ok | :error,
+          latency_us :: number()
+        ) :: :ok
+  def observe_delivery_latency(consumer_id, consumer_name, success, latency_us) do
+    validate_success(success)
+    Histogram.observe([name: :sequin_delivery_latency_us, labels: [consumer_id, consumer_name, success]], latency_us)
+  end
+
+  @spec observe_post_delivery_latency(consumer_id :: String.t(), consumer_name :: String.t(), latency_us :: number()) ::
+          :ok
+  def observe_post_delivery_latency(consumer_id, consumer_name, latency_us) do
+    Histogram.observe([name: :sequin_post_delivery_latency_us, labels: [consumer_id, consumer_name]], latency_us)
+  end
+
+  @spec observe_oldest_message_age(consumer_id :: String.t(), consumer_name :: String.t(), age_ms :: number()) :: :ok
+  def observe_oldest_message_age(consumer_id, consumer_name, age_ms) do
+    Histogram.observe([name: :sequin_oldest_message_age_ms, labels: [consumer_id, consumer_name]], age_ms)
+  end
+
+  @spec increment_messages_ingested(replication_slot_id :: String.t(), slot_name :: String.t(), count :: number()) :: :ok
+  def increment_messages_ingested(replication_slot_id, slot_name, count \\ 1) do
+    Counter.inc([name: :sequin_messages_ingested_count, labels: [replication_slot_id, slot_name]], count)
+  end
+
+  @spec set_messages_in_delivery(consumer_id :: String.t(), consumer_name :: String.t(), count :: number()) :: :ok
+  def set_messages_in_delivery(consumer_id, consumer_name, count) do
+    Gauge.set([name: :sequin_messages_in_delivery, labels: [consumer_id, consumer_name]], count)
+  end
+
+  @spec set_messages_buffered(consumer_id :: String.t(), consumer_name :: String.t(), count :: number()) :: :ok
+  def set_messages_buffered(consumer_id, consumer_name, count) do
+    Gauge.set([name: :sequin_messages_buffered, labels: [consumer_id, consumer_name]], count)
+  end
+
+  @spec increment_bytes_ingested(consumer_id :: String.t(), consumer_name :: String.t(), bytes :: number()) :: :ok
+  def increment_bytes_ingested(consumer_id, consumer_name, bytes) do
+    Counter.inc(
+      name: :sequin_bytes_ingested_total,
+      labels: [consumer_id, consumer_name],
+      count: bytes
+    )
+  end
+
+  @spec increment_bytes_delivered(consumer_id :: String.t(), consumer_name :: String.t(), bytes :: number()) :: :ok
+  def increment_bytes_delivered(consumer_id, consumer_name, bytes) do
+    Counter.inc(
+      name: :sequin_bytes_delivered_total,
+      labels: [consumer_id, consumer_name],
+      count: bytes
+    )
+  end
+
+  @spec set_messages_in_redelivery(consumer_id :: String.t(), consumer_name :: String.t(), count :: number()) :: :ok
+  def set_messages_in_redelivery(consumer_id, consumer_name, count) do
+    Gauge.set([name: :sequin_messages_in_redelivery, labels: [consumer_id, consumer_name]], count)
+  end
+
+  @spec set_buffer_memory_utilization_mb(consumer_id :: String.t(), consumer_name :: String.t(), mb :: number()) :: :ok
+  def set_buffer_memory_utilization_mb(consumer_id, consumer_name, mb) do
+    Gauge.set([name: :sequin_buffer_memory_utilization_mb, labels: [consumer_id, consumer_name]], mb)
+  end
+
+  @spec set_buffer_memory_utilization_percent(consumer_id :: String.t(), consumer_name :: String.t(), percent :: number()) ::
+          :ok
+  def set_buffer_memory_utilization_percent(consumer_id, consumer_name, percent) do
+    Gauge.set([name: :sequin_buffer_memory_utilization_percent, labels: [consumer_id, consumer_name]], percent)
+  end
+
+  @spec set_ingestion_saturation(consumer_id :: String.t(), consumer_name :: String.t(), percent :: number()) :: :ok
+  def set_ingestion_saturation(consumer_id, consumer_name, percent) do
+    Gauge.set([name: :sequin_ingestion_saturation_percent, labels: [consumer_id, consumer_name]], percent)
+  end
+
+  @spec set_processing_saturation(consumer_id :: String.t(), consumer_name :: String.t(), percent :: number()) :: :ok
+  def set_processing_saturation(consumer_id, consumer_name, percent) do
+    Gauge.set([name: :sequin_processing_saturation_percent, labels: [consumer_id, consumer_name]], percent)
+  end
+
+  @spec set_delivery_saturation(consumer_id :: String.t(), consumer_name :: String.t(), percent :: number()) :: :ok
+  def set_delivery_saturation(consumer_id, consumer_name, percent) do
+    Gauge.set([name: :sequin_delivery_saturation_percent, labels: [consumer_id, consumer_name]], percent)
+  end
+
+  defp validate_success(:ok), do: :ok
+  defp validate_success(:error), do: :ok
+end
diff --git a/lib/sequin/metrics/prometheus_exporter.ex b/lib/sequin/metrics/prometheus_exporter.ex
new file mode 100644
index 000000000..670c4c946
--- /dev/null
+++ b/lib/sequin/metrics/prometheus_exporter.ex
@@ -0,0 +1,4 @@
+defmodule Sequin.PrometheusExporter do
+  @moduledoc false
+  use Prometheus.PlugExporter
+end
diff --git a/lib/sequin/runtime/sink_pipeline.ex b/lib/sequin/runtime/sink_pipeline.ex
index c09106430..a50d9e989 100644
--- a/lib/sequin/runtime/sink_pipeline.ex
+++ b/lib/sequin/runtime/sink_pipeline.ex
@@ -18,6 +18,7 @@ defmodule Sequin.Runtime.SinkPipeline do
   alias Sequin.Consumers.SinkConsumer
   alias Sequin.Error
   alias Sequin.Health
+  alias Sequin.Prometheus
   alias Sequin.Repo
   alias Sequin.Runtime.MessageLedgers
 
@@ -174,20 +175,47 @@ defmodule Sequin.Runtime.SinkPipeline do
         already_delivered
 
       {to_deliver, already_delivered} ->
-        case pipeline_mod.handle_batch(batch_name, to_deliver, batch_info, context) do
-          {:ok, delivered, next_context} ->
-            update_context(context, next_context)
-            delivered ++ already_delivered
+        Prometheus.increment_message_deliver_attempt(context.consumer.id, context.consumer.name, length(to_deliver))
+        deliver_messages(pipeline_mod, batch_name, to_deliver, already_delivered, batch_info, context)
+    end
+  end
 
-          {:error, error} ->
-            failed =
-              Enum.map(to_deliver, fn message ->
-                Message.failed(message, error)
-              end)
+  defp deliver_messages(pipeline_mod, batch_name, to_deliver, already_delivered, batch_info, context) do
+    now = Sequin.utc_now()
 
-            failed ++ already_delivered
-        end
+    to_deliver
+    |> Stream.map(fn
+      %Broadway.Message{data: %ConsumerEvent{ingested_at: ingested_at}} -> ingested_at
+      %Broadway.Message{data: %ConsumerRecord{ingested_at: ingested_at}} -> ingested_at
+    end)
+    |> Stream.filter(& &1)
+    |> Stream.map(&DateTime.diff(now, &1, :microsecond))
+    |> Enum.each(&Prometheus.observe_internal_latency(context.consumer.id, context.consumer.name, &1))
+
+    case :timer.tc(pipeline_mod, :handle_batch, [batch_name, to_deliver, batch_info, context], :microsecond) do
+      {t, {:ok, delivered, next_context}} ->
+        Prometheus.increment_message_deliver_success(context.consumer.id, context.consumer.name, length(delivered))
+        Prometheus.observe_delivery_latency(context.consumer.id, context.consumer.name, :ok, t)
+
+        update_context(context, next_context)
+        delivered ++ already_delivered
+
+      {t, {:error, error}} ->
+        Prometheus.increment_message_deliver_failure(context.consumer.id, context.consumer.name, length(to_deliver))
+        Prometheus.observe_delivery_latency(context.consumer.id, context.consumer.name, :error, t)
+
+        failed =
+          Enum.map(to_deliver, fn message ->
+            Message.failed(message, error)
+          end)
+
+        failed ++ already_delivered
     end
+  rescue
+    error ->
+      Prometheus.increment_message_deliver_failure(context.consumer.id, context.consumer.name, length(to_deliver))
+
+      reraise error, __STACKTRACE__
   end
 
   # Give processes a way to modify their context, which allows them to use it as a k/v store
diff --git a/lib/sequin/runtime/slot_processor_server.ex b/lib/sequin/runtime/slot_processor_server.ex
index 92acf1ccd..da3172e4e 100644
--- a/lib/sequin/runtime/slot_processor_server.ex
+++ b/lib/sequin/runtime/slot_processor_server.ex
@@ -22,6 +22,7 @@ defmodule Sequin.Runtime.SlotProcessorServer do
   alias Sequin.Health.Event
   alias Sequin.Postgres
   alias Sequin.Postgres.ReplicationConnection
+  alias Sequin.Prometheus
   alias Sequin.Replication
   alias Sequin.Repo
   alias Sequin.Runtime.MessageHandler
@@ -188,7 +189,7 @@ defmodule Sequin.Runtime.SlotProcessorServer do
       message_handler_module: message_handler_module,
       connection: connection,
       last_commit_lsn: nil,
-      heartbeat_interval: Keyword.get(opts, :heartbeat_interval, :timer.minutes(1)),
+      heartbeat_interval: Keyword.get(opts, :heartbeat_interval, :timer.seconds(15)),
       max_memory_bytes: max_memory_bytes,
       bytes_between_limit_checks: bytes_between_limit_checks,
       check_memory_fn: Keyword.get(opts, :check_memory_fn, &default_check_memory_fn/0),
@@ -255,6 +256,7 @@ defmodule Sequin.Runtime.SlotProcessorServer do
     state = schedule_heartbeat(state, 0)
     state = schedule_heartbeat_verification(state)
     schedule_process_logging(0)
+    schedule_observe_ingestion_latency()
 
     {:ok, %{state | connection_state: :disconnected}}
   end
@@ -850,6 +852,17 @@ defmodule Sequin.Runtime.SlotProcessorServer do
     {:keep_state, state}
   end
 
+  def handle_info(:observe_ingestion_latency, %State{} = state) do
+    # Check if we have an outstanding heartbeat
+    if not is_nil(state.current_heartbeat_id) and not is_nil(state.heartbeat_emitted_at) do
+      observe_ingestion_latency(state, state.heartbeat_emitted_at)
+    end
+
+    schedule_observe_ingestion_latency()
+
+    {:keep_state, state}
+  end
+
   defp on_connect_failure(%State{} = state, error) do
     conn = get_cached_conn(state)
 
@@ -1143,6 +1156,9 @@ defmodule Sequin.Runtime.SlotProcessorServer do
             emitted_at: emitted_at
           )
 
+          {:ok, emitted_at, _} = DateTime.from_iso8601(emitted_at)
+          observe_ingestion_latency(state, emitted_at)
+
           Health.put_event(
             state.replication_slot,
             %Event{slug: :replication_heartbeat_received, status: :success}
@@ -1369,6 +1385,8 @@ defmodule Sequin.Runtime.SlotProcessorServer do
 
       case res do
         :ok ->
+          Prometheus.increment_messages_ingested(state.replication_slot.id, state.replication_slot.slot_name, count)
+
           if state.test_pid do
             state.message_handler_module.flush_messages(state.message_handler_ctx)
             send(state.test_pid, {__MODULE__, :flush_messages})
@@ -1791,4 +1809,13 @@ defmodule Sequin.Runtime.SlotProcessorServer do
 
     wal_cursor.commit_lsn
   end
+
+  defp schedule_observe_ingestion_latency do
+    Process.send_after(self(), :observe_ingestion_latency, :timer.seconds(5))
+  end
+
+  defp observe_ingestion_latency(%State{} = state, ts) do
+    latency_us = DateTime.diff(Sequin.utc_now(), ts, :microsecond)
+    Prometheus.observe_ingestion_latency(state.replication_slot.id, state.replication_slot.slot_name, latency_us)
+  end
 end
diff --git a/lib/sequin_web/metrics_endpoint.ex b/lib/sequin_web/metrics_endpoint.ex
new file mode 100644
index 000000000..75b65be9a
--- /dev/null
+++ b/lib/sequin_web/metrics_endpoint.ex
@@ -0,0 +1,5 @@
+defmodule SequinWeb.MetricsEndpoint do
+  use Phoenix.Endpoint, otp_app: :sequin
+
+  plug Sequin.PrometheusExporter
+end
diff --git a/mix.exs b/mix.exs
index b2c0e86b7..6577583d5 100644
--- a/mix.exs
+++ b/mix.exs
@@ -94,7 +94,9 @@ defmodule Sequin.MixProject do
       {:pgvector, "~> 0.3.0"},
       {:broadway_dashboard, "~> 0.4.0"},
       {:rexbug, "~> 1.0"},
-      {:decorator, "~> 1.4"}
+      {:decorator, "~> 1.4"},
+      {:prometheus_ex, "~> 3.0"},
+      {:prometheus_plugs, "~> 1.1"}
     ]
   end
 
diff --git a/mix.lock b/mix.lock
index afeb453ca..d9b012c15 100644
--- a/mix.lock
+++ b/mix.lock
@@ -1,4 +1,5 @@
 %{
+  "accept": {:hex, :accept, "0.3.6", "ad44ac7d704bf70ef8fb2e313ef5b978f9d1330bddac64509e93afda13281215", [:rebar3], [], "hexpm", "a5167fa1ae90315c3f1dd189446312f8a55d00efa357e9c569bda47736b874c3"},
   "amqp": {:hex, :amqp, "4.0.0", "c62c0eba8ad5f5bbebf668ca4a68bbf8d9e35c9b3bc8703ff679c01f3e6899d3", [:mix], [{:amqp_client, "~> 4.0", [hex: :amqp_client, repo: "hexpm", optional: false]}], "hexpm", "4148c54dc35733e6c2f9208ff26bc61601cde2da993f752a3452442b018d5735"},
   "amqp_client": {:hex, :amqp_client, "4.0.3", "c7dcc8031c780cd39ec586ba827a8eb26e006e9761af8d3f58fded11f645ebd4", [:make, :rebar3], [{:credentials_obfuscation, "3.4.0", [hex: :credentials_obfuscation, repo: "hexpm", optional: false]}, {:rabbit_common, "4.0.3", [hex: :rabbit_common, repo: "hexpm", optional: false]}], "hexpm", "ae945f7280617e9a4b17a6d49e3a2f496d716e8088ec29d8e94ecc79e5da7458"},
   "argon2_elixir": {:hex, :argon2_elixir, "3.2.1", "f47740bf9f2a39ffef79ba48eb25dea2ee37bcc7eadf91d49615591d1a6fce1a", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "a813b78217394530b5fcf4c8070feee43df03ffef938d044019169c766315690"},
@@ -83,6 +84,10 @@
   "polymorphic_embed": {:hex, :polymorphic_embed, "4.1.1", "1a93d0c9184eb5a50c58578d0dfe4b05045d9a86bbcc7cd142114f3c4393c439", [:mix], [{:ecto, "~> 3.11", [hex: :ecto, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_html_helpers, "~> 1.0", [hex: :phoenix_html_helpers, repo: "hexpm", optional: true]}, {:phoenix_live_view, "~> 0.19", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}], "hexpm", "3f3d2354d2a86a180cf0415273bb21ac134615c005d1b2537e7c6708eee21fcc"},
   "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"},
   "postgrex": {:hex, :postgrex, "0.19.3", "a0bda6e3bc75ec07fca5b0a89bffd242ca209a4822a9533e7d3e84ee80707e19", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "d31c28053655b78f47f948c85bb1cf86a9c1f8ead346ba1aa0d0df017fa05b61"},
+  "prometheus": {:hex, :prometheus, "4.13.0", "4b61c3a98bb5c1d7a12109e34a951bf44be667c104e4df8fb1ca77605e918147", [:rebar3], [{:quantile_estimator, "1.0.2", [hex: :quantile_estimator, repo: "hexpm", optional: false]}], "hexpm", "e5d9e0385ec2f650dcabb8636257ed8b3b4b211188047bd1a34d2b3f92f6420a"},
+  "prometheus_ex": {:hex, :prometheus_ex, "3.1.0", "98bdb1e9aea5001bc5f58752259d832447c06dd335cb246e89b497e2c4ec7de6", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm", "bc584d33aabae94ce75d477eb66c7178755b45a09afb7addb253d90eb66021de"},
+  "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm", "0273a6483ccb936d79ca19b0ab629aef0dba958697c94782bb728b920dfc6a79"},
+  "quantile_estimator": {:hex, :quantile_estimator, "1.0.2", "ecd281d40110fdd9ba62685531e4435e0839a52fd1058da5564f1763e4642ef7", [:rebar3], [], "hexpm", "db404793d6384995a1ac6dd973e2cee5be9fcc128765bdba53d87c564e296b64"},
   "rabbit_common": {:hex, :rabbit_common, "4.0.3", "e927b882733d122f6802662220bdb1a83774852dbe67d21d4e33aaf54f3998dd", [:make, :rebar3], [{:credentials_obfuscation, "3.4.0", [hex: :credentials_obfuscation, repo: "hexpm", optional: false]}, {:ranch, "2.1.0", [hex: :ranch, repo: "hexpm", optional: false]}, {:recon, "2.5.6", [hex: :recon, repo: "hexpm", optional: false]}, {:thoas, "1.2.1", [hex: :thoas, repo: "hexpm", optional: false]}], "hexpm", "ead31ba292c2cc5fda48a486417d7cfe8966f661994d7ed6c3e5f8840828e8ec"},
   "ranch": {:hex, :ranch, "2.1.0", "2261f9ed9574dcfcc444106b9f6da155e6e540b2f82ba3d42b339b93673b72a3", [:make, :rebar3], [], "hexpm", "244ee3fa2a6175270d8e1fc59024fd9dbc76294a321057de8f803b1479e76916"},
   "recon": {:hex, :recon, "2.5.6", "9052588e83bfedfd9b72e1034532aee2a5369d9d9343b61aeb7fbce761010741", [:mix, :rebar3], [], "hexpm", "96c6799792d735cc0f0fd0f86267e9d351e63339cbe03df9d162010cefc26bb0"},


Reviews:
user: login: acco
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-04T21:30:51Z
user: login: abc3
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-05T10:16:50Z
user: login: RTLS
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-07T18:12:26Z
user: login: abc3
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-07T18:43:22Z

Review comments:
diff_hunk: @@ -109,6 +109,16 @@ The easiest way to get started with Sequin is with our [Docker Compose file](htt
         </Card>
       </CardGroup>
     </Step>
+
+    <Step title="View metrics in Grafana">
+      Sequin exports  [various metrics](/reference/metrics) in Prometheus format.
+
+      Using the Grafana service from docker compose, you can view our [example dashboard here](http://localhost:3000/d/behuozial454wc/sequin). 
path: docs/running-sequin.mdx
user: login: acco
body: ```suggestion
      Using the Grafana service from Docker Compose, you can view our [example dashboard here](http://localhost:3000/d/behuozial454wc/sequin). 
```
created_at: 2025-04-04T21:02:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -71,9 +71,9 @@ The easiest way to get started with Sequin is with our [Docker Compose file](htt
       - Password: `sequinpassword!`
 
     </Step>
-    <Step title="Create a sync">
+    <Step title="Create a sink">
path: docs/running-sequin.mdx
user: login: acco
body: Update docker ps to match the world
created_at: 2025-04-04T21:03:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,42 @@
+---
+title: Metrics
+description: Reference for Sequin's metrics. Use metrics to monitor health and performance.
+---
+
+Sequin exposes a metrics endpoint, using Prometheus text format, which can be integrated into your existing observability workflows.
+By default, this endpoint is at `/metrics` on port 7377.
path: docs/reference/metrics.mdx
user: login: acco
body: ```suggestion
Sequin exposes a metrics endpoint, using Prometheus text format, which can be integrated into your existing observability workflows. By default, this endpoint is at `/metrics` on port 7377.
```
created_at: 2025-04-04T21:03:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,42 @@
+---
+title: Metrics
+description: Reference for Sequin's metrics. Use metrics to monitor health and performance.
+---
+
+Sequin exposes a metrics endpoint, using Prometheus text format, which can be integrated into your existing observability workflows.
+By default, this endpoint is at `/metrics` on port 7377.
+
+Metrics are designed to answer:
+- Is Sequin keeping up with the database? (Ingestion metrics)
+- Are sinks keeping up with Sequin? (Delivery metrics)
+- Is the Sequin service experiencing CPU or memory pressure? (VM metrics)
path: docs/reference/metrics.mdx
user: login: acco
body: ```suggestion
Metrics are designed to answer:

- Is Sequin keeping up with the database? (Ingestion metrics)
- Are sinks keeping up with Sequin? (Delivery metrics)
- Is the Sequin service experiencing CPU or memory pressure? (VM metrics)
```
created_at: 2025-04-04T21:04:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,42 @@
+---
+title: Metrics
+description: Reference for Sequin's metrics. Use metrics to monitor health and performance.
+---
+
+Sequin exposes a metrics endpoint, using Prometheus text format, which can be integrated into your existing observability workflows.
+By default, this endpoint is at `/metrics` on port 7377.
+
+Metrics are designed to answer:
+- Is Sequin keeping up with the database? (Ingestion metrics)
+- Are sinks keeping up with Sequin? (Delivery metrics)
+- Is the Sequin service experiencing CPU or memory pressure? (VM metrics)
+
+## Metrics reference
+
+### Ingestion metrics
+- `sequin_ingestion_latency_ms` (Gauge)
path: docs/reference/metrics.mdx
user: login: acco
body: ```suggestion
### Ingestion metrics

- `sequin_ingestion_latency_ms` (Gauge)
```
created_at: 2025-04-04T21:04:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,42 @@
+---
+title: Metrics
+description: Reference for Sequin's metrics. Use metrics to monitor health and performance.
+---
+
+Sequin exposes a metrics endpoint, using Prometheus text format, which can be integrated into your existing observability workflows.
+By default, this endpoint is at `/metrics` on port 7377.
+
+Metrics are designed to answer:
+- Is Sequin keeping up with the database? (Ingestion metrics)
+- Are sinks keeping up with Sequin? (Delivery metrics)
+- Is the Sequin service experiencing CPU or memory pressure? (VM metrics)
+
+## Metrics reference
+
+### Ingestion metrics
+- `sequin_ingestion_latency_ms` (Gauge)
+  Measures latency between transaction commit and Sequin replication slot ingestion.
+  Calculated by instrumenting Sequin's own heartbeat messages.
path: docs/reference/metrics.mdx
user: login: acco
body: ```suggestion
  Measures latency between transaction commit and when Sequin first receives it. Calculated by instrumenting Sequin's own heartbeat messages.
```
created_at: 2025-04-04T21:06:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,42 @@
+---
+title: Metrics
+description: Reference for Sequin's metrics. Use metrics to monitor health and performance.
+---
+
+Sequin exposes a metrics endpoint, using Prometheus text format, which can be integrated into your existing observability workflows.
+By default, this endpoint is at `/metrics` on port 7377.
+
+Metrics are designed to answer:
+- Is Sequin keeping up with the database? (Ingestion metrics)
+- Are sinks keeping up with Sequin? (Delivery metrics)
+- Is the Sequin service experiencing CPU or memory pressure? (VM metrics)
+
+## Metrics reference
+
+### Ingestion metrics
+- `sequin_ingestion_latency_ms` (Gauge)
+  Measures latency between transaction commit and Sequin replication slot ingestion.
+  Calculated by instrumenting Sequin's own heartbeat messages.
+- `sequin_messages_ingested_total` (Counter)
+  Number of messages Sequin has received on a given replication slot.
path: docs/reference/metrics.mdx
user: login: acco
body: ```suggestion
  Number of messages Sequin has received from a given replication slot.
```
created_at: 2025-04-04T21:07:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,42 @@
+---
+title: Metrics
+description: Reference for Sequin's metrics. Use metrics to monitor health and performance.
+---
+
+Sequin exposes a metrics endpoint, using Prometheus text format, which can be integrated into your existing observability workflows.
+By default, this endpoint is at `/metrics` on port 7377.
+
+Metrics are designed to answer:
+- Is Sequin keeping up with the database? (Ingestion metrics)
+- Are sinks keeping up with Sequin? (Delivery metrics)
+- Is the Sequin service experiencing CPU or memory pressure? (VM metrics)
+
+## Metrics reference
+
+### Ingestion metrics
+- `sequin_ingestion_latency_ms` (Gauge)
+  Measures latency between transaction commit and Sequin replication slot ingestion.
+  Calculated by instrumenting Sequin's own heartbeat messages.
+- `sequin_messages_ingested_total` (Counter)
+  Number of messages Sequin has received on a given replication slot.
+
+### Delivery metrics
+- `sequin_delivery_latency_ms` (Histogram)
+  Measures latency between Sequin and a given consumer's successful acknowledgement.
+  Some consumers are able to batch messages, in which case this metric measures
+  the batch latency, rather than the per-message latency.
path: docs/reference/metrics.mdx
user: login: acco
body: ```suggestion
  Measures time between Sequin beginning to deliver a message and a given sink's successful acknowledgement. Some sinks are able to batch messages, in which case this metric measures the batch latency, rather than the per-message latency.
```
created_at: 2025-04-04T21:11:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,42 @@
+---
+title: Metrics
+description: Reference for Sequin's metrics. Use metrics to monitor health and performance.
+---
+
+Sequin exposes a metrics endpoint, using Prometheus text format, which can be integrated into your existing observability workflows.
+By default, this endpoint is at `/metrics` on port 7377.
+
+Metrics are designed to answer:
+- Is Sequin keeping up with the database? (Ingestion metrics)
+- Are sinks keeping up with Sequin? (Delivery metrics)
+- Is the Sequin service experiencing CPU or memory pressure? (VM metrics)
+
+## Metrics reference
+
+### Ingestion metrics
+- `sequin_ingestion_latency_ms` (Gauge)
+  Measures latency between transaction commit and Sequin replication slot ingestion.
+  Calculated by instrumenting Sequin's own heartbeat messages.
+- `sequin_messages_ingested_total` (Counter)
+  Number of messages Sequin has received on a given replication slot.
+
+### Delivery metrics
+- `sequin_delivery_latency_ms` (Histogram)
+  Measures latency between Sequin and a given consumer's successful acknowledgement.
+  Some consumers are able to batch messages, in which case this metric measures
+  the batch latency, rather than the per-message latency.
+- `sequin_message_deliver_attempt` (Counter)
+  Incremented for each message Sequin tries to deliver, before receiving a success or failure response.
+- `sequin_message_deliver_success` (Counter)
+  Incremented for each message delivered and acknowledged.
path: docs/reference/metrics.mdx
user: login: acco
body: ```suggestion
  Incremented for each message that Sequin successfully delivers.
```
created_at: 2025-04-04T21:12:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,42 @@
+---
+title: Metrics
+description: Reference for Sequin's metrics. Use metrics to monitor health and performance.
+---
+
+Sequin exposes a metrics endpoint, using Prometheus text format, which can be integrated into your existing observability workflows.
+By default, this endpoint is at `/metrics` on port 7377.
+
+Metrics are designed to answer:
+- Is Sequin keeping up with the database? (Ingestion metrics)
+- Are sinks keeping up with Sequin? (Delivery metrics)
+- Is the Sequin service experiencing CPU or memory pressure? (VM metrics)
+
+## Metrics reference
+
+### Ingestion metrics
+- `sequin_ingestion_latency_ms` (Gauge)
+  Measures latency between transaction commit and Sequin replication slot ingestion.
+  Calculated by instrumenting Sequin's own heartbeat messages.
+- `sequin_messages_ingested_total` (Counter)
+  Number of messages Sequin has received on a given replication slot.
+
+### Delivery metrics
+- `sequin_delivery_latency_ms` (Histogram)
+  Measures latency between Sequin and a given consumer's successful acknowledgement.
+  Some consumers are able to batch messages, in which case this metric measures
+  the batch latency, rather than the per-message latency.
+- `sequin_message_deliver_attempt` (Counter)
+  Incremented for each message Sequin tries to deliver, before receiving a success or failure response.
+- `sequin_message_deliver_success` (Counter)
+  Incremented for each message delivered and acknowledged.
+- `sequin_message_deliver_failure` (Counter)
+  Incremented for each message which was delivered but failed downstream.
path: docs/reference/metrics.mdx
user: login: acco
body: ```suggestion
  Incremented for each message that Sequin fails to deliver to a sink. This is because the sink was either unreachable, timed out, or returned an error.
```
created_at: 2025-04-04T21:14:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,42 @@
+---
+title: Metrics
+description: Reference for Sequin's metrics. Use metrics to monitor health and performance.
+---
+
+Sequin exposes a metrics endpoint, using Prometheus text format, which can be integrated into your existing observability workflows.
+By default, this endpoint is at `/metrics` on port 7377.
+
+Metrics are designed to answer:
+- Is Sequin keeping up with the database? (Ingestion metrics)
+- Are sinks keeping up with Sequin? (Delivery metrics)
+- Is the Sequin service experiencing CPU or memory pressure? (VM metrics)
+
+## Metrics reference
+
+### Ingestion metrics
+- `sequin_ingestion_latency_ms` (Gauge)
+  Measures latency between transaction commit and Sequin replication slot ingestion.
+  Calculated by instrumenting Sequin's own heartbeat messages.
+- `sequin_messages_ingested_total` (Counter)
+  Number of messages Sequin has received on a given replication slot.
+
+### Delivery metrics
+- `sequin_delivery_latency_ms` (Histogram)
+  Measures latency between Sequin and a given consumer's successful acknowledgement.
+  Some consumers are able to batch messages, in which case this metric measures
+  the batch latency, rather than the per-message latency.
+- `sequin_message_deliver_attempt` (Counter)
+  Incremented for each message Sequin tries to deliver, before receiving a success or failure response.
+- `sequin_message_deliver_success` (Counter)
+  Incremented for each message delivered and acknowledged.
+- `sequin_message_deliver_failure` (Counter)
+  Incremented for each message which was delivered but failed downstream.
+
+### VM metrics
+The BEAM VM exposes a many metrics - the following are the ones we find most useful for monitoring Sequin.
path: docs/reference/metrics.mdx
user: login: acco
body: ```suggestion
Sequin is implemented with Elixir. Elixir's VM (the BEAM) exposes many metrics. The following are the ones we find most useful for monitoring Sequin:
```
created_at: 2025-04-04T21:15:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,42 @@
+---
+title: Metrics
+description: Reference for Sequin's metrics. Use metrics to monitor health and performance.
+---
+
+Sequin exposes a metrics endpoint, using Prometheus text format, which can be integrated into your existing observability workflows.
+By default, this endpoint is at `/metrics` on port 7377.
+
+Metrics are designed to answer:
+- Is Sequin keeping up with the database? (Ingestion metrics)
+- Are sinks keeping up with Sequin? (Delivery metrics)
+- Is the Sequin service experiencing CPU or memory pressure? (VM metrics)
+
+## Metrics reference
+
+### Ingestion metrics
+- `sequin_ingestion_latency_ms` (Gauge)
+  Measures latency between transaction commit and Sequin replication slot ingestion.
+  Calculated by instrumenting Sequin's own heartbeat messages.
+- `sequin_messages_ingested_total` (Counter)
+  Number of messages Sequin has received on a given replication slot.
+
+### Delivery metrics
+- `sequin_delivery_latency_ms` (Histogram)
+  Measures latency between Sequin and a given consumer's successful acknowledgement.
+  Some consumers are able to batch messages, in which case this metric measures
+  the batch latency, rather than the per-message latency.
+- `sequin_message_deliver_attempt` (Counter)
+  Incremented for each message Sequin tries to deliver, before receiving a success or failure response.
+- `sequin_message_deliver_success` (Counter)
+  Incremented for each message delivered and acknowledged.
+- `sequin_message_deliver_failure` (Counter)
+  Incremented for each message which was delivered but failed downstream.
+
+### VM metrics
+The BEAM VM exposes a many metrics - the following are the ones we find most useful for monitoring Sequin.
+- `erlang_vm_statistics_reductions_total` (Counter)
+  Measures how many VM instructions are executed across all processes.
+  Shows intensity of CPU-bound work.
+  Recommend using with a rate query rather than directly.
path: docs/reference/metrics.mdx
user: login: acco
body: one line
created_at: 2025-04-04T21:15:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,337 @@
+defmodule Sequin.Prometheus do
+  @moduledoc false
+  use Prometheus.Metric
+
+  @spec setup() :: :ok
+  def setup do
+    Gauge.new(
+      name: :sequin_ingestion_latency_ms,
+      labels: [:replication_slot_id, :slot_name],
+      help: "The ingestion latency between Postgres and Sequin in milliseconds."
+    )
+
+    Histogram.new(
+      name: :sequin_internal_latency_ms,
+      labels: [:consumer_id, :consumer_name],
+      buckets: [10, 50, 100, 250, 500, 1000, 2500, 5000, 10_000],
+      duration_unit: :milliseconds,
+      help: "The internal processing latency in milliseconds."
+    )
+
+    Histogram.new(
+      name: :sequin_delivery_latency_ms,
+      labels: [:consumer_id, :consumer_name, :success],
+      buckets: [10, 50, 100, 250, 500, 1000, 2500, 5000, 10_000],
+      duration_unit: false,
+      help: "The delivery latency in milliseconds."
+    )
path: lib/sequin/metrics/prometheus.ex
user: login: abc3
body: these latencies can get pretty small, so it might make more sense to use microseconds instead of milliseconds for better accuracy
created_at: 2025-04-05T10:16:50Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,337 @@
+defmodule Sequin.Prometheus do
+  @moduledoc false
+  use Prometheus.Metric
+
+  @spec setup() :: :ok
+  def setup do
+    Gauge.new(
+      name: :sequin_ingestion_latency_ms,
+      labels: [:replication_slot_id, :slot_name],
+      help: "The ingestion latency between Postgres and Sequin in milliseconds."
+    )
+
+    Histogram.new(
+      name: :sequin_internal_latency_ms,
+      labels: [:consumer_id, :consumer_name],
+      buckets: [10, 50, 100, 250, 500, 1000, 2500, 5000, 10_000],
+      duration_unit: :milliseconds,
+      help: "The internal processing latency in milliseconds."
+    )
+
+    Histogram.new(
+      name: :sequin_delivery_latency_ms,
+      labels: [:consumer_id, :consumer_name, :success],
+      buckets: [10, 50, 100, 250, 500, 1000, 2500, 5000, 10_000],
+      duration_unit: false,
+      help: "The delivery latency in milliseconds."
+    )
path: lib/sequin/metrics/prometheus.ex
user: login: RTLS
body: microseconds is reasonable here for local destinations
created_at: 2025-04-07T18:12:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,337 @@
+defmodule Sequin.Prometheus do
+  @moduledoc false
+  use Prometheus.Metric
+
+  @spec setup() :: :ok
+  def setup do
+    Gauge.new(
+      name: :sequin_ingestion_latency_ms,
+      labels: [:replication_slot_id, :slot_name],
+      help: "The ingestion latency between Postgres and Sequin in milliseconds."
+    )
+
+    Histogram.new(
+      name: :sequin_internal_latency_ms,
+      labels: [:consumer_id, :consumer_name],
+      buckets: [10, 50, 100, 250, 500, 1000, 2500, 5000, 10_000],
+      duration_unit: :milliseconds,
+      help: "The internal processing latency in milliseconds."
+    )
+
+    Histogram.new(
+      name: :sequin_delivery_latency_ms,
+      labels: [:consumer_id, :consumer_name, :success],
+      buckets: [10, 50, 100, 250, 500, 1000, 2500, 5000, 10_000],
+      duration_unit: false,
+      help: "The delivery latency in milliseconds."
+    )
path: lib/sequin/metrics/prometheus.ex
user: login: abc3
body: or if a sequin instance, source, and destination are all in the same aws zone or data center :) 
created_at: 2025-04-07T18:43:22Z
author_association: CONTRIBUTOR

PR comments:
user: login: RTLS
created_at: 2025-04-03T18:52:51Z
body: * **#1195** <a href="https://app.graphite.dev/github/pr/sequinstream/sequin/1195?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈 <a href="https://app.graphite.dev/github/pr/sequinstream/sequin/1195?utm_source=stack-comment-view-in-graphite" target="_blank">(View in Graphite)</a>
* `main`


<h2></h2>

This stack of pull requests is managed by <a href="https://graphite.dev?utm-source=stack-comment"><b>Graphite</b></a>. Learn more about <a href="https://stacking.dev/?utm_source=stack-comment">stacking</a>.
<!-- Current dependencies on/for this PR: -->
author_association: CONTRIBUTOR
user: login: abc3
created_at: 2025-04-05T11:38:41Z
body: Nice addition! Out of curiosity, why didn’t you adopt [PromEx](https://github.com/akoutmos/prom_ex)? 

Another helpful enhancement might be to include some default tags, like node_id when the app runs in cluster mode, box IP, and so on


author_association: CONTRIBUTOR
user: login: RTLS
created_at: 2025-04-07T18:18:19Z
body: > Nice addition! Out of curiosity, why didn’t you adopt [PromEx](https://github.com/akoutmos/prom_ex)?
> 
> Another helpful enhancement might be to include some default tags, like node_id when the app runs in cluster mode, box IP, and so on

We decided that Telemetry events for internal metrics was unnecessary.

PromEx looks nice though, there certainly are benefits to joining a plugin community.
author_association: CONTRIBUTOR