Repository: aeron-io/aeron
Pull request #1805: [Java] Make cluster publish leader heartbeat timeout to clients.
Author: login: wojciech-adaptive
Created: 2025-05-21T10:22:02Z | Merged: 2025-05-23T10:27:48Z

An implementation of https://github.com/aeron-io/aeron/pull/1791#discussion_r2072710182


Diff:
diff --git a/aeron-cluster/src/main/java/io/aeron/cluster/ConsensusModule.java b/aeron-cluster/src/main/java/io/aeron/cluster/ConsensusModule.java
index 29ede1ee0c..142a8b9240 100644
--- a/aeron-cluster/src/main/java/io/aeron/cluster/ConsensusModule.java
+++ b/aeron-cluster/src/main/java/io/aeron/cluster/ConsensusModule.java
@@ -1984,7 +1984,7 @@ public void conclude()
 
             if (null == egressPublisher)
             {
-                egressPublisher = new EgressPublisher();
+                egressPublisher = new EgressPublisher(leaderHeartbeatTimeoutNs);
             }
 
             final ChannelUri channelUri = parse(logChannel());
diff --git a/aeron-cluster/src/main/java/io/aeron/cluster/EgressPublisher.java b/aeron-cluster/src/main/java/io/aeron/cluster/EgressPublisher.java
index 3dd969820b..c3179a86ff 100644
--- a/aeron-cluster/src/main/java/io/aeron/cluster/EgressPublisher.java
+++ b/aeron-cluster/src/main/java/io/aeron/cluster/EgressPublisher.java
@@ -34,6 +34,12 @@ class EgressPublisher
     private final ChallengeEncoder challengeEncoder = new ChallengeEncoder();
     private final NewLeaderEventEncoder newLeaderEventEncoder = new NewLeaderEventEncoder();
     private final AdminResponseEncoder adminResponseEncoder = new AdminResponseEncoder();
+    private final long leaderHeartbeatTimeoutNs;
+
+    EgressPublisher(final long leaderHeartbeatTimeoutNs)
+    {
+        this.leaderHeartbeatTimeoutNs = leaderHeartbeatTimeoutNs;
+    }
 
     boolean sendEvent(
         final ClusterSession session,
@@ -61,6 +67,7 @@ boolean sendEvent(
                     .leaderMemberId(leaderMemberId)
                     .code(code)
                     .version(AeronCluster.Configuration.PROTOCOL_SEMANTIC_VERSION)
+                    .leaderHeartbeatTimeoutNs(leaderHeartbeatTimeoutNs)
                     .detail(detail);
 
                 bufferClaim.commit();
diff --git a/aeron-cluster/src/main/java/io/aeron/cluster/client/AeronCluster.java b/aeron-cluster/src/main/java/io/aeron/cluster/client/AeronCluster.java
index 0815e69c86..86ea926335 100644
--- a/aeron-cluster/src/main/java/io/aeron/cluster/client/AeronCluster.java
+++ b/aeron-cluster/src/main/java/io/aeron/cluster/client/AeronCluster.java
@@ -1409,11 +1409,6 @@ public void conclude()
                 throw new ConcurrentConcludeException();
             }
 
-            if (NULL_VALUE == newLeaderTimeoutNs)
-            {
-                newLeaderTimeoutNs = ConsensusModule.Configuration.LEADER_HEARTBEAT_TIMEOUT_DEFAULT_NS * 2;
-            }
-
             if (null == aeron)
             {
                 aeron = Aeron.connect(
@@ -2052,6 +2047,7 @@ static State fromStep(final int step)
 
         private Image egressImage;
         private final long deadlineNs;
+        private long leaderHeartbeatTimeoutNs;
         private long correlationId = NULL_VALUE;
         private long clusterSessionId;
         private long leadershipTermId;
@@ -2390,6 +2386,7 @@ private void pollResponse()
                         leadershipTermId = egressPoller.leadershipTermId();
                         leaderMemberId = egressPoller.leaderMemberId();
                         clusterSessionId = egressPoller.clusterSessionId();
+                        leaderHeartbeatTimeoutNs = egressPoller.leaderHeartbeatTimeoutNs();
                         egressImage = egressPoller.egressImage();
                         state(State.CONCLUDE_CONNECT);
                         break;
@@ -2457,6 +2454,12 @@ private void updateMembers()
 
         private AeronCluster concludeConnect()
         {
+            if (ctx.newLeaderTimeoutNs() == NULL_VALUE)
+            {
+                ctx.newLeaderTimeoutNs(2 * (leaderHeartbeatTimeoutNs != NULL_VALUE ?
+                    leaderHeartbeatTimeoutNs : ConsensusModule.Configuration.LEADER_HEARTBEAT_TIMEOUT_DEFAULT_NS));
+            }
+
             final AeronCluster aeronCluster = new AeronCluster(
                 ctx,
                 messageHeaderEncoder,
diff --git a/aeron-cluster/src/main/java/io/aeron/cluster/client/EgressPoller.java b/aeron-cluster/src/main/java/io/aeron/cluster/client/EgressPoller.java
index e178b95f21..5efe725727 100644
--- a/aeron-cluster/src/main/java/io/aeron/cluster/client/EgressPoller.java
+++ b/aeron-cluster/src/main/java/io/aeron/cluster/client/EgressPoller.java
@@ -45,6 +45,7 @@ public final class EgressPoller implements ControlledFragmentHandler
     private int leaderMemberId = Aeron.NULL_VALUE;
     private int templateId = Aeron.NULL_VALUE;
     private int version = 0;
+    private long leaderHeartbeatTimeoutNs = Aeron.NULL_VALUE;
     private boolean isPollComplete = false;
     private EventCode eventCode;
     private String detail = "";
@@ -152,6 +153,16 @@ public int version()
         return version;
     }
 
+    /**
+     * Leader heartbeat timeout of the last polled event or {@link Aeron#NULL_VALUE} if not available.
+     *
+     * @return leader heartbeat timeout of the last polled event or {@link Aeron#NULL_VALUE} if not available.
+     */
+    public long leaderHeartbeatTimeoutNs()
+    {
+        return leaderHeartbeatTimeoutNs;
+    }
+
     /**
      * Get the detail returned from the last session event.
      *
@@ -208,6 +219,7 @@ public int poll()
             leaderMemberId = Aeron.NULL_VALUE;
             templateId = Aeron.NULL_VALUE;
             version = 0;
+            leaderHeartbeatTimeoutNs = Aeron.NULL_VALUE;
             eventCode = null;
             detail = "";
             encodedChallenge = null;
@@ -263,6 +275,7 @@ public ControlledFragmentAssembler.Action onFragment(
                 leaderMemberId = sessionEventDecoder.leaderMemberId();
                 eventCode = sessionEventDecoder.code();
                 version = sessionEventDecoder.version();
+                leaderHeartbeatTimeoutNs = leaderHeartbeatTimeoutNs(sessionEventDecoder);
                 detail = sessionEventDecoder.detail();
                 isPollComplete = true;
                 egressImage = (Image)header.context();
@@ -301,4 +314,16 @@ public ControlledFragmentAssembler.Action onFragment(
 
         return Action.CONTINUE;
     }
+
+    private static long leaderHeartbeatTimeoutNs(final SessionEventDecoder sessionEventDecoder)
+    {
+        final long leaderHeartbeatTimeoutNs = sessionEventDecoder.leaderHeartbeatTimeoutNs();
+
+        if (leaderHeartbeatTimeoutNs == SessionEventDecoder.leaderHeartbeatTimeoutNsNullValue())
+        {
+            return Aeron.NULL_VALUE;
+        }
+
+        return leaderHeartbeatTimeoutNs;
+    }
 }
diff --git a/aeron-cluster/src/main/resources/cluster/aeron-cluster-codecs.xml b/aeron-cluster/src/main/resources/cluster/aeron-cluster-codecs.xml
index d2f10712d3..6c215c001e 100644
--- a/aeron-cluster/src/main/resources/cluster/aeron-cluster-codecs.xml
+++ b/aeron-cluster/src/main/resources/cluster/aeron-cluster-codecs.xml
@@ -2,7 +2,7 @@
 <sbe:messageSchema xmlns:sbe="http://fixprotocol.io/2016/sbe"
                    package="io.aeron.cluster.codecs"
                    id="111"
-                   version="12"
+                   version="13"
                    semanticVersion="5.4"
                    description="Message Codecs for communicating with, and within, an Aeron Cluster."
                    byteOrder="littleEndian">
@@ -152,6 +152,8 @@
                description="Code type of the response."/>
         <field name="version"                  id="6" type="version_t" presence="optional" sinceVersion="6"
                description="Protocol version for the server using semantic version form."/>
+        <field name="leaderHeartbeatTimeoutNs" id="8" type="int64" presence="optional" sinceVersion="13"
+               description="Leader heartbeat timeout in nanoseconds as configured on the leader."/>
         <data  name="detail"                   id="7" type="varAsciiEncoding"
                description="Further detail such as an error message or list of cluster ingress endpoints."/>
     </sbe:message>
diff --git a/aeron-cluster/src/test/java/io/aeron/cluster/client/AeronClusterAsyncConnectTest.java b/aeron-cluster/src/test/java/io/aeron/cluster/client/AeronClusterAsyncConnectTest.java
index 4e170c99ed..a7dd1b7faf 100644
--- a/aeron-cluster/src/test/java/io/aeron/cluster/client/AeronClusterAsyncConnectTest.java
+++ b/aeron-cluster/src/test/java/io/aeron/cluster/client/AeronClusterAsyncConnectTest.java
@@ -294,6 +294,7 @@ public void shouldConnectViaIngressChannel()
             .leaderMemberId(leaderMemberId)
             .code(EventCode.OK)
             .version(ConsensusModule.Configuration.PROTOCOL_SEMANTIC_VERSION)
+            .leaderHeartbeatTimeoutNs(SessionEventEncoder.leaderHeartbeatTimeoutNsNullValue())
             .detail("you are now connected");
         final Image egressImage = mock(Image.class);
         final Header header = new Header(1, 16, egressImage);
@@ -319,6 +320,9 @@ public void shouldConnectViaIngressChannel()
         assertEquals(leadershipTermId, aeronCluster.leadershipTermId());
         assertEquals(leaderMemberId, aeronCluster.leaderMemberId());
         assertEquals(clusterSessionId, aeronCluster.clusterSessionId());
+        assertEquals(
+            2 * ConsensusModule.Configuration.LEADER_HEARTBEAT_TIMEOUT_DEFAULT_NS,
+            context.newLeaderTimeoutNs());
 
         asyncConnect.close();
         verify(publication, never()).close();
diff --git a/aeron-cluster/src/test/java/io/aeron/cluster/client/AeronClusterTest.java b/aeron-cluster/src/test/java/io/aeron/cluster/client/AeronClusterTest.java
index 387db3bf75..bbaa0733a0 100644
--- a/aeron-cluster/src/test/java/io/aeron/cluster/client/AeronClusterTest.java
+++ b/aeron-cluster/src/test/java/io/aeron/cluster/client/AeronClusterTest.java
@@ -38,6 +38,7 @@
 import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.ValueSource;
 
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 
 import static org.agrona.BitUtil.align;
@@ -70,7 +71,8 @@ class AeronClusterTest
         .egressChannel("aeron:udp?endpoint=localhost:0")
         .ingressChannel("aeron:udp")
         .idleStrategy(NoOpIdleStrategy.INSTANCE)
-        .egressListener(egressListener));
+        .egressListener(egressListener))
+        .newLeaderTimeoutNs(TimeUnit.SECONDS.toNanos(1));
     private final ExclusivePublication ingressPublication = mock(ExclusivePublication.class);
     private final Subscription egressSubscription = mock(Subscription.class);
     private final Image egressImage = mock(Image.class);


Reviews:
user: login: vyazelenko
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T12:09:02Z
user: login: vyazelenko
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T14:02:52Z
user: login: wojciech-adaptive
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T15:12:22Z
user: login: vyazelenko
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T15:28:29Z
user: login: vyazelenko
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T15:30:26Z
user: login: wojciech-adaptive
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T15:42:47Z
user: login: vyazelenko
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T15:45:24Z

Review comments:
diff_hunk: @@ -2,8 +2,8 @@
 <sbe:messageSchema xmlns:sbe="http://fixprotocol.io/2016/sbe"
                    package="io.aeron.cluster.codecs"
                    id="111"
-                   version="12"
-                   semanticVersion="5.4"
+                   version="13"
+                   semanticVersion="5.5"
path: aeron-cluster/src/main/resources/cluster/aeron-cluster-codecs.xml
user: login: vyazelenko
body: We don't increment `semanticVersion` when making changes to SBE schemas.
created_at: 2025-05-21T12:09:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2457,6 +2454,12 @@ private void updateMembers()
 
         private AeronCluster concludeConnect()
         {
+            if (ctx.newLeaderTimeoutNs() == NULL_VALUE)
+            {
+                ctx.newLeaderTimeoutNs(2 * (leaderHeartbeatTimeoutNs != NULL_VALUE ?
+                    leaderHeartbeatTimeoutNs : ConsensusModule.Configuration.LEADER_HEARTBEAT_TIMEOUT_DEFAULT_NS));
+            }
+
path: aeron-cluster/src/main/java/io/aeron/cluster/client/AeronCluster.java
user: login: vyazelenko
body: Why keep `AeronCluster.Context#newLeaderTimeoutNs(long)` given that we get value from the Cluster itself? After all we wanted to simplify configuration by implementing this. 
created_at: 2025-05-21T14:02:40Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2457,6 +2454,12 @@ private void updateMembers()
 
         private AeronCluster concludeConnect()
         {
+            if (ctx.newLeaderTimeoutNs() == NULL_VALUE)
+            {
+                ctx.newLeaderTimeoutNs(2 * (leaderHeartbeatTimeoutNs != NULL_VALUE ?
+                    leaderHeartbeatTimeoutNs : ConsensusModule.Configuration.LEADER_HEARTBEAT_TIMEOUT_DEFAULT_NS));
+            }
+
path: aeron-cluster/src/main/java/io/aeron/cluster/client/AeronCluster.java
user: login: wojciech-adaptive
body: For tests and in case there are any unusual setups where the default doesn't work.
created_at: 2025-05-21T15:12:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2457,6 +2454,12 @@ private void updateMembers()
 
         private AeronCluster concludeConnect()
         {
+            if (ctx.newLeaderTimeoutNs() == NULL_VALUE)
+            {
+                ctx.newLeaderTimeoutNs(2 * (leaderHeartbeatTimeoutNs != NULL_VALUE ?
+                    leaderHeartbeatTimeoutNs : ConsensusModule.Configuration.LEADER_HEARTBEAT_TIMEOUT_DEFAULT_NS));
+            }
+
path: aeron-cluster/src/main/java/io/aeron/cluster/client/AeronCluster.java
user: login: vyazelenko
body: For tests we can keep this functionality package-private.
created_at: 2025-05-21T15:28:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2457,6 +2454,12 @@ private void updateMembers()
 
         private AeronCluster concludeConnect()
         {
+            if (ctx.newLeaderTimeoutNs() == NULL_VALUE)
+            {
+                ctx.newLeaderTimeoutNs(2 * (leaderHeartbeatTimeoutNs != NULL_VALUE ?
+                    leaderHeartbeatTimeoutNs : ConsensusModule.Configuration.LEADER_HEARTBEAT_TIMEOUT_DEFAULT_NS));
+            }
+
path: aeron-cluster/src/main/java/io/aeron/cluster/client/AeronCluster.java
user: login: vyazelenko
body: On the other hand, if we do keep it public why isn't there a system property for it then?
created_at: 2025-05-21T15:30:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2457,6 +2454,12 @@ private void updateMembers()
 
         private AeronCluster concludeConnect()
         {
+            if (ctx.newLeaderTimeoutNs() == NULL_VALUE)
+            {
+                ctx.newLeaderTimeoutNs(2 * (leaderHeartbeatTimeoutNs != NULL_VALUE ?
+                    leaderHeartbeatTimeoutNs : ConsensusModule.Configuration.LEADER_HEARTBEAT_TIMEOUT_DEFAULT_NS));
+            }
+
path: aeron-cluster/src/main/java/io/aeron/cluster/client/AeronCluster.java
user: login: wojciech-adaptive
body: > For tests we can keep this functionality package-private.

I wouldn't assume it would be useful for our tests only.

> On the other hand, if we do keep it public why isn't there a system property for it then?

Don't mind adding one. We don't have properties for everything, I guess only the typical options.
created_at: 2025-05-21T15:42:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2457,6 +2454,12 @@ private void updateMembers()
 
         private AeronCluster concludeConnect()
         {
+            if (ctx.newLeaderTimeoutNs() == NULL_VALUE)
+            {
+                ctx.newLeaderTimeoutNs(2 * (leaderHeartbeatTimeoutNs != NULL_VALUE ?
+                    leaderHeartbeatTimeoutNs : ConsensusModule.Configuration.LEADER_HEARTBEAT_TIMEOUT_DEFAULT_NS));
+            }
+
path: aeron-cluster/src/main/java/io/aeron/cluster/client/AeronCluster.java
user: login: vyazelenko
body: To me `newLeaderTimeoutNs` is similar to `messageTimeoutNs` which is configurable via the system properties. So I suggest we add one for the former as well.

created_at: 2025-05-21T15:45:24Z
author_association: CONTRIBUTOR

PR comments:
user: login: vyazelenko
created_at: 2025-05-21T14:04:07Z
body: Overall looks good. The only issue is the remaining config option which we should hide/remove.
author_association: CONTRIBUTOR
user: login: wojciech-adaptive
created_at: 2025-05-22T07:49:03Z
body: > @wojciech-adaptive This change will break the old clients, i.e. if Cluster is upgraded but the client not then the `SessionEvent` message will not be readable. In particular `sessionEventDecoder.detail()` will blow up as it will interpret first 4 bytes of the new field as the length of the `detail` string.

Can you please elaborate? SBE supports versioning and as far as I see we use codecs correctly, i.e. they will account for different block lengths.
author_association: CONTRIBUTOR
user: login: vyazelenko
created_at: 2025-05-22T11:03:08Z
body: > Can you please elaborate? SBE supports versioning and as far as I see we use codecs correctly, i.e. they will account for different block lengths.

You are absolutely right. I got confused. No problem here.

P.S. I've pushed a test that proves that such change works. https://github.com/aeron-io/aeron/commit/b3635fef2da6ff9e6dcb74f4244fe34dbbafb4ab
author_association: CONTRIBUTOR