Repository: JasperFx/wolverine (2026-06-23T19:25:27Z)

Issue:
Listener health: surface transport-channel/agent connection state in EndpointHealthSnapshot
## Context

Follow-on to the channel self-heal in #3171 (`e5180ec78`). That fix makes RabbitMQ listeners eagerly re-consume after a channel-only shutdown and senders heal lazily — covering the acute case. This issue covers the **observability** gap that remains for external monitors (CritterWatch): you can't *see* a stuck/disconnected transport channel.

## Problem

`EndpointHealthSnapshot` reports only the `ListeningStatus` orchestration enum (Accepting / Stopped / TooBusy / GloballyLatched) for listeners, and a `SenderLatched` bool for senders. It does **not** expose the underlying transport/agent connection state (e.g. `RabbitMqChannelAgent.State` = Connected/Disconnected). So a listener whose channel has died can still report **Accepting** (green); an external monitor can only *infer* trouble from `LastQueueActivityAt` going stale + `BrokerQueueDepth` climbing.

- `src/Wolverine/Configuration/EndpointHealthSnapshot.cs` — the DTO
- `src/Wolverine/Configuration/EndpointCollection.cs:132-165` — `CollectEndpointHealth()`

## Ask

Add a transport-agnostic connection/agent-state field to `EndpointHealthSnapshot` (e.g. `Connected` / `Disconnected` / `Reconnecting` / `Unknown`), populated per transport:
- RabbitMQ → from `RabbitMqChannelAgent.State`
- equivalents for other transports where they have a notion of connection state
- `Unknown` where a transport has none

Cover **senders** too — a latched/disconnected sender is just as invisible today as a listener.

## Downstream

JasperFx/CritterWatch#513 (1.0) — consumed as a "stuck listener" alert signature, strengthened from the staleness heuristic to a direct `Disconnected` signal. Independent of the force-restart gap (separate issue).



Discussion:
Downstream consumer: JasperFx/CritterWatch#513 (1.0).



Patch:
diff --git a/src/Transports/RabbitMQ/Wolverine.RabbitMQ/Internal/RabbitMqChannelAgent.cs b/src/Transports/RabbitMQ/Wolverine.RabbitMQ/Internal/RabbitMqChannelAgent.cs
index 8655fb5d8..cc4e94bbc 100644
--- a/src/Transports/RabbitMQ/Wolverine.RabbitMQ/Internal/RabbitMqChannelAgent.cs
+++ b/src/Transports/RabbitMQ/Wolverine.RabbitMQ/Internal/RabbitMqChannelAgent.cs
@@ -1,13 +1,14 @@
 using Microsoft.Extensions.Logging;
 using RabbitMQ.Client;
 using RabbitMQ.Client.Events;
+using Wolverine.Transports;
 
 namespace Wolverine.RabbitMQ.Internal;
 
 /// <summary>
 /// Base class for Rabbit MQ listeners and senders
 /// </summary>
-internal abstract class RabbitMqChannelAgent : IAsyncDisposable
+internal abstract class RabbitMqChannelAgent : IAsyncDisposable, IReportConnectionState
 {
     private readonly ConnectionMonitor _monitor;
     private readonly SemaphoreSlim Locker = new(1, 1);
@@ -26,6 +27,11 @@ protected RabbitMqChannelAgent(ConnectionMonitor monitor,
     internal AgentState State { get; private set; } = AgentState.Disconnected;
     internal bool IsConnected => State == AgentState.Connected;
 
+    // Surfaces the channel agent's connection state to EndpointHealthSnapshot so external monitors can see a
+    // dead-but-Accepting listener (or a disconnected sender) directly rather than inferring it from staleness.
+    public TransportConnectionState ConnectionState =>
+        State == AgentState.Connected ? TransportConnectionState.Connected : TransportConnectionState.Disconnected;
+
     internal IChannel? Channel { get; set; }
 
     public virtual async ValueTask DisposeAsync()
diff --git a/src/Wolverine/Configuration/EndpointCollection.cs b/src/Wolverine/Configuration/EndpointCollection.cs
index ebeb60200..ee2d5b01f 100644
--- a/src/Wolverine/Configuration/EndpointCollection.cs
+++ b/src/Wolverine/Configuration/EndpointCollection.cs
@@ -144,7 +144,8 @@ public IReadOnlyList<EndpointHealthSnapshot> CollectEndpointHealth()
                 LastQueueActivityAt: listener.LastQueueActivityAt,
                 LastMessageSentAt: null,
                 SenderLatched: false,
-                BufferLimit: listener.Endpoint.BufferingLimits?.Maximum));
+                BufferLimit: listener.Endpoint.BufferingLimits?.Maximum,
+                ConnectionState: connectionStateOf(listener)));
         }
 
         foreach (var sender in _senders.Enumerate().Select(x => x.Value))
@@ -158,12 +159,52 @@ public IReadOnlyList<EndpointHealthSnapshot> CollectEndpointHealth()
                 LastQueueActivityAt: null,
                 LastMessageSentAt: sender.LastMessageSentAt,
                 SenderLatched: sender.Latched,
-                BufferLimit: null));
+                BufferLimit: null,
+                ConnectionState: connectionStateOf(sender)));
         }
 
         return snapshots;
     }
 
+    // Resolve the underlying transport channel/connection state for a listener. The agent itself may report it
+    // (IReportConnectionState), otherwise reach through to the IListener it owns. Transports without a connection
+    // notion fall through to Unknown.
+    private static TransportConnectionState connectionStateOf(IListeningAgent agent)
+    {
+        if (agent is IReportConnectionState reporter)
+        {
+            return reporter.ConnectionState;
+        }
+
+        if (agent is ListeningAgent { Listener: IReportConnectionState listenerReporter })
+        {
+            return listenerReporter.ConnectionState;
+        }
+
+        return TransportConnectionState.Unknown;
+    }
+
+    // Resolve the underlying transport channel/connection state for a sending agent. The agent itself may report it,
+    // otherwise reach through to the ISender it wraps (SendingAgent / InlineSendingAgent both expose Sender).
+    private static TransportConnectionState connectionStateOf(ISendingAgent agent)
+    {
+        if (agent is IReportConnectionState reporter)
+        {
+            return reporter.ConnectionState;
+        }
+
+        var sender = agent switch
+        {
+            SendingAgent sa => sa.Sender,
+            InlineSendingAgent ia => ia.Sender,
+            _ => null
+        };
+
+        return sender is IReportConnectionState senderReporter
+            ? senderReporter.ConnectionState
+            : TransportConnectionState.Unknown;
+    }
+
     public ISendingAgent GetOrBuildSendingAgent(Uri address, Action<Endpoint>? configureNewEndpoint = null)
     {
         if (address == null)
diff --git a/src/Wolverine/Configuration/EndpointHealthSnapshot.cs b/src/Wolverine/Configuration/EndpointHealthSnapshot.cs
index af00b8f3c..3d4989db0 100644
--- a/src/Wolverine/Configuration/EndpointHealthSnapshot.cs
+++ b/src/Wolverine/Configuration/EndpointHealthSnapshot.cs
@@ -1,3 +1,5 @@
+using Wolverine.Transports;
+
 namespace Wolverine.Configuration;
 
 /// <summary>
@@ -13,7 +15,8 @@ public record EndpointHealthSnapshot(
     DateTimeOffset? LastMessageSentAt,
     bool SenderLatched,
     int? BufferLimit,
-    long? BrokerQueueDepth = null);
+    long? BrokerQueueDepth = null,
+    TransportConnectionState ConnectionState = TransportConnectionState.Unknown);
 
 public enum EndpointDirection
 {
diff --git a/src/Wolverine/Transports/IReportConnectionState.cs b/src/Wolverine/Transports/IReportConnectionState.cs
new file mode 100644
index 000000000..d1d351b14
--- /dev/null
+++ b/src/Wolverine/Transports/IReportConnectionState.cs
@@ -0,0 +1,14 @@
+namespace Wolverine.Transports;
+
+/// <summary>
+/// Optional capability implemented by an <see cref="IListener"/>, <see cref="Sending.ISender"/>, or sending/listening
+/// agent that can report the state of its underlying transport channel/connection. Transports that have no notion of
+/// a connection simply do not implement this, and report <see cref="TransportConnectionState.Unknown"/>.
+/// </summary>
+public interface IReportConnectionState
+{
+    /// <summary>
+    /// The current connection state of the underlying transport channel/agent.
+    /// </summary>
+    TransportConnectionState ConnectionState { get; }
+}
diff --git a/src/Wolverine/Transports/TransportConnectionState.cs b/src/Wolverine/Transports/TransportConnectionState.cs
new file mode 100644
index 000000000..2918e4614
--- /dev/null
+++ b/src/Wolverine/Transports/TransportConnectionState.cs
@@ -0,0 +1,32 @@
+namespace Wolverine.Transports;
+
+/// <summary>
+/// The connection state of an endpoint's underlying transport channel/agent, as reported by transports that
+/// have a notion of an open connection (e.g. RabbitMQ channels). Surfaced on <see cref="Wolverine.Configuration.EndpointHealthSnapshot"/>
+/// so external monitors can distinguish a healthy listener from one whose transport channel has died but whose
+/// orchestration <c>ListeningStatus</c> still reports <c>Accepting</c>.
+/// </summary>
+public enum TransportConnectionState
+{
+    /// <summary>
+    /// The transport does not expose any connection state, or the endpoint is not in a state where a connection
+    /// is meaningful (e.g. in-memory local queues). This is the default.
+    /// </summary>
+    Unknown,
+
+    /// <summary>
+    /// The underlying transport channel/connection is open and usable.
+    /// </summary>
+    Connected,
+
+    /// <summary>
+    /// The underlying transport channel/connection is down. For a listener this means it is not actually
+    /// consuming even if its <c>ListeningStatus</c> still reports <c>Accepting</c>.
+    /// </summary>
+    Disconnected,
+
+    /// <summary>
+    /// The transport is actively attempting to re-establish a connection.
+    /// </summary>
+    Reconnecting
+}


Test patch:
diff --git a/src/Testing/CoreTests/Configuration/endpoint_health_connection_state_default_3231.cs b/src/Testing/CoreTests/Configuration/endpoint_health_connection_state_default_3231.cs
new file mode 100644
index 000000000..86171d6fe
--- /dev/null
+++ b/src/Testing/CoreTests/Configuration/endpoint_health_connection_state_default_3231.cs
@@ -0,0 +1,35 @@
+using Microsoft.Extensions.Hosting;
+using Shouldly;
+using Wolverine.Tracking;
+using Wolverine.Transports;
+using Wolverine.Transports.Tcp;
+using Wolverine.Util;
+using Xunit;
+
+namespace CoreTests.Configuration;
+
+// GH-3231: transports that have no notion of a connection (TCP, in-memory local queues) must report
+// TransportConnectionState.Unknown rather than a misleading Connected/Disconnected.
+public class endpoint_health_connection_state_default_3231
+{
+    [Fact]
+    public async Task non_connection_aware_endpoints_report_unknown()
+    {
+        using var host = await Host.CreateDefaultBuilder()
+            .UseWolverine(opts =>
+            {
+                opts.ListenAtPort(PortFinder.GetAvailablePort());
+                opts.PublishAllMessages().ToPort(PortFinder.GetAvailablePort());
+            }).StartAsync();
+
+        var snapshots = host.GetRuntime().Endpoints.CollectEndpointHealth();
+        snapshots.ShouldNotBeEmpty();
+
+        // No endpoint here exposes IReportConnectionState, so every snapshot must fall through to Unknown.
+        foreach (var snapshot in snapshots)
+        {
+            snapshot.ConnectionState.ShouldBe(TransportConnectionState.Unknown,
+                $"{snapshot.Direction} endpoint {snapshot.Uri}");
+        }
+    }
+}
diff --git a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/endpoint_health_connection_state_3231.cs b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/endpoint_health_connection_state_3231.cs
new file mode 100644
index 000000000..e58c4c372
--- /dev/null
+++ b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/endpoint_health_connection_state_3231.cs
@@ -0,0 +1,97 @@
+using JasperFx.Core;
+using RabbitMQ.Client;
+using Shouldly;
+using Wolverine.ComplianceTests;
+using Wolverine.Configuration;
+using Wolverine.RabbitMQ.Internal;
+using Wolverine.Tracking;
+using Wolverine.Transports;
+using Xunit;
+
+namespace Wolverine.RabbitMQ.Tests;
+
+public record EndpointHealthConnectionStateMessage(string Name);
+
+public class EndpointHealthConnectionStateHandler
+{
+    public void Handle(EndpointHealthConnectionStateMessage message)
+    {
+        // no-op; we only need the message to flow so the sender actually connects
+    }
+}
+
+// GH-3231: EndpointHealthSnapshot must surface the underlying transport channel/agent connection state so external
+// monitors (CritterWatch) can see a dead-but-"Accepting" listener (or a disconnected sender) directly rather than
+// inferring it from staleness.
+[Trait("Category", "Flaky")]
+public class endpoint_health_connection_state_3231
+{
+    private static string nextQueue() => "conn_state_" + Guid.NewGuid().ToString("N");
+
+    [Fact]
+    public async Task healthy_rabbitmq_endpoints_report_connected()
+    {
+        var queue = nextQueue();
+        using var host = await WolverineHost.ForAsync(opts =>
+        {
+            opts.UseRabbitMq().AutoProvision();
+            opts.ListenToRabbitQueue(queue);
+            opts.PublishMessage<EndpointHealthConnectionStateMessage>().ToRabbitQueue(queue);
+        });
+
+        // Send one message so the (lazily-connecting) sender actually opens its channel before we probe.
+        await host.TrackActivity().Timeout(30.Seconds())
+            .SendMessageAndWaitAsync(new EndpointHealthConnectionStateMessage("Magic Johnson"));
+
+        var snapshots = host.GetRuntime().Endpoints.CollectEndpointHealth();
+
+        var rabbitSnapshots = snapshots.Where(s => s.Uri.Scheme == "rabbitmq").ToList();
+        rabbitSnapshots.ShouldNotBeEmpty();
+
+        // Every live RabbitMQ endpoint (listener + sender) should report a healthy, open channel.
+        rabbitSnapshots.ShouldContain(s => s.Direction == EndpointDirection.Listening);
+        rabbitSnapshots.ShouldContain(s => s.Direction == EndpointDirection.Sending);
+        foreach (var snapshot in rabbitSnapshots)
+        {
+            snapshot.ConnectionState.ShouldBe(TransportConnectionState.Connected,
+                $"{snapshot.Direction} endpoint {snapshot.Uri} should report Connected");
+        }
+    }
+
+    [Fact]
+    public async Task a_dropped_sender_connection_is_reported_as_disconnected()
+    {
+        var queue = nextQueue();
+        using var host = await WolverineHost.ForAsync(opts =>
+        {
+            opts.UseRabbitMq().AutoProvision();
+            opts.ListenToRabbitQueue(queue);
+            opts.PublishMessage<EndpointHealthConnectionStateMessage>().ToRabbitQueue(queue);
+        });
+
+        // Send first so the sender is genuinely Connected before we sever it.
+        await host.TrackActivity().Timeout(30.Seconds())
+            .SendMessageAndWaitAsync(new EndpointHealthConnectionStateMessage("Larry Bird"));
+
+        var runtime = host.GetRuntime();
+        var transport = runtime.Options.Transports.GetOrCreate<RabbitMqTransport>();
+
+        // Slam the sending connection shut from underneath the transport. The sender heals lazily (only on the
+        // next send), so without further traffic it stays Disconnected -- which is exactly the invisible state
+        // this issue is about.
+        var sending = transport.TryGetSendingConnection();
+        if (sending?.Connection is { } sendingConnection)
+        {
+            await sendingConnection.CloseAsync(
+                Constants.ReplySuccess, "Test-induced close", TimeSpan.FromSeconds(5), abort: false, CancellationToken.None);
+        }
+
+        // Give the connection-shutdown callback a moment to flip the agent state.
+        await Task.Delay(500);
+
+        var snapshots = runtime.Endpoints.CollectEndpointHealth();
+        var rabbitSender = snapshots.First(s => s.Direction == EndpointDirection.Sending && s.Uri.Scheme == "rabbitmq");
+
+        rabbitSender.ConnectionState.ShouldBe(TransportConnectionState.Disconnected);
+    }
+}