Repository: elastic/elasticsearch
Pull request #127213: Remove dependency on cluster state API in `SpecificMasterNodesIT`
Author: login: nielsbauman
Labels: name: >test
description: Issues or PRs that are addressing/adding tests
name: :Distributed/Distributed
description: A catch all label for anything in the Distributed Area. Please avoid if you can.
name: Team:Distributed Coordination (obsolete)
description: Meta label for Distributed Coordination team. Obsolete. Please do not use.
name: v9.1.0
Created: 2025-04-23T07:59:28Z | Merged: 2025-04-25T12:12:32Z

These tests depended on the cluster state API to wait for the master node. This behavior is being removed, so we switch to alternative approaches of waiting for the master node.

Relates #127212


Diff:
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
index 222c7fc3e1f14..5f86111d352a9 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
@@ -13,190 +13,60 @@
 import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
 import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.discovery.MasterNotDiscoveredException;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
-import java.io.IOException;
-
 import static org.elasticsearch.test.NodeRoles.dataOnlyNode;
 import static org.elasticsearch.test.NodeRoles.masterNode;
 import static org.elasticsearch.test.NodeRoles.nonDataNode;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
 
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class SpecificMasterNodesIT extends ESIntegTestCase {
 
-    public void testSimpleOnlyMasterNodeElection() throws IOException {
+    public void testSimpleOnlyMasterNodeElection() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(0);
         logger.info("--> start data node / non master node");
         internalCluster().startNode(Settings.builder().put(dataOnlyNode()).put("discovery.initial_state_timeout", "1s"));
-        try {
-            assertThat(
-                clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT)
-                    .setMasterNodeTimeout(TimeValue.timeValueMillis(100))
-                    .get()
-                    .getState()
-                    .nodes()
-                    .getMasterNodeId(),
-                nullValue()
-            );
-            fail("should not be able to find master");
-        } catch (MasterNotDiscoveredException e) {
-            // all is well, no master elected
-        }
+        awaitMasterNotFound();
+
         logger.info("--> start master node");
         final String masterNodeName = internalCluster().startMasterOnlyNode();
-        assertThat(
-            internalCluster().nonMasterClient()
-                .admin()
-                .cluster()
-                .prepareState(TEST_REQUEST_TIMEOUT)
-                .get()
-                .getState()
-                .nodes()
-                .getMasterNode()
-                .getName(),
-            equalTo(masterNodeName)
-        );
-        assertThat(
-            internalCluster().masterClient()
-                .admin()
-                .cluster()
-                .prepareState(TEST_REQUEST_TIMEOUT)
-                .get()
-                .getState()
-                .nodes()
-                .getMasterNode()
-                .getName(),
-            equalTo(masterNodeName)
-        );
+
+        awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNodeName);
+        awaitMasterNode(internalCluster().getMasterName(), masterNodeName);
 
         logger.info("--> stop master node");
         Settings masterDataPathSettings = internalCluster().dataPathSettings(internalCluster().getMasterName());
         internalCluster().stopCurrentMasterNode();
 
-        try {
-            assertThat(
-                clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT)
-                    .setMasterNodeTimeout(TimeValue.timeValueMillis(100))
-                    .get()
-                    .getState()
-                    .nodes()
-                    .getMasterNodeId(),
-                nullValue()
-            );
-            fail("should not be able to find master");
-        } catch (MasterNotDiscoveredException e) {
-            // all is well, no master elected
-        }
+        awaitMasterNotFound();
 
         logger.info("--> start previous master node again");
         final String nextMasterEligibleNodeName = internalCluster().startNode(
             Settings.builder().put(nonDataNode(masterNode())).put(masterDataPathSettings)
         );
-        assertThat(
-            internalCluster().nonMasterClient()
-                .admin()
-                .cluster()
-                .prepareState(TEST_REQUEST_TIMEOUT)
-                .get()
-                .getState()
-                .nodes()
-                .getMasterNode()
-                .getName(),
-            equalTo(nextMasterEligibleNodeName)
-        );
-        assertThat(
-            internalCluster().masterClient()
-                .admin()
-                .cluster()
-                .prepareState(TEST_REQUEST_TIMEOUT)
-                .get()
-                .getState()
-                .nodes()
-                .getMasterNode()
-                .getName(),
-            equalTo(nextMasterEligibleNodeName)
-        );
+        awaitMasterNode(internalCluster().getNonMasterNodeName(), nextMasterEligibleNodeName);
+        awaitMasterNode(internalCluster().getMasterName(), nextMasterEligibleNodeName);
     }
 
     public void testElectOnlyBetweenMasterNodes() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(0);
         logger.info("--> start data node / non master node");
         internalCluster().startNode(Settings.builder().put(dataOnlyNode()).put("discovery.initial_state_timeout", "1s"));
-        try {
-            assertThat(
-                clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT)
-                    .setMasterNodeTimeout(TimeValue.timeValueMillis(100))
-                    .get()
-                    .getState()
-                    .nodes()
-                    .getMasterNodeId(),
-                nullValue()
-            );
-            fail("should not be able to find master");
-        } catch (MasterNotDiscoveredException e) {
-            // all is well, no master elected
-        }
+        awaitMasterNotFound();
+
         logger.info("--> start master node (1)");
         final String masterNodeName = internalCluster().startMasterOnlyNode();
-        assertThat(
-            internalCluster().nonMasterClient()
-                .admin()
-                .cluster()
-                .prepareState(TEST_REQUEST_TIMEOUT)
-                .get()
-                .getState()
-                .nodes()
-                .getMasterNode()
-                .getName(),
-            equalTo(masterNodeName)
-        );
-        assertThat(
-            internalCluster().masterClient()
-                .admin()
-                .cluster()
-                .prepareState(TEST_REQUEST_TIMEOUT)
-                .get()
-                .getState()
-                .nodes()
-                .getMasterNode()
-                .getName(),
-            equalTo(masterNodeName)
-        );
+        awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNodeName);
+        awaitMasterNode(internalCluster().getMasterName(), masterNodeName);
 
         logger.info("--> start master node (2)");
         final String nextMasterEligableNodeName = internalCluster().startMasterOnlyNode();
-        assertThat(
-            internalCluster().nonMasterClient()
-                .admin()
-                .cluster()
-                .prepareState(TEST_REQUEST_TIMEOUT)
-                .get()
-                .getState()
-                .nodes()
-                .getMasterNode()
-                .getName(),
-            equalTo(masterNodeName)
-        );
-        assertThat(
-            internalCluster().masterClient()
-                .admin()
-                .cluster()
-                .prepareState(TEST_REQUEST_TIMEOUT)
-                .get()
-                .getState()
-                .nodes()
-                .getMasterNode()
-                .getName(),
-            equalTo(masterNodeName)
-        );
+        awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNodeName);
+        awaitMasterNode(internalCluster().getMasterName(), masterNodeName);
 
         logger.info("--> closing master node (1)");
         client().execute(
@@ -204,57 +74,12 @@ public void testElectOnlyBetweenMasterNodes() throws Exception {
             new AddVotingConfigExclusionsRequest(TEST_REQUEST_TIMEOUT, masterNodeName)
         ).get();
         // removing the master from the voting configuration immediately triggers the master to step down
-        assertBusy(() -> {
-            assertThat(
-                internalCluster().nonMasterClient()
-                    .admin()
-                    .cluster()
-                    .prepareState(TEST_REQUEST_TIMEOUT)
-                    .get()
-                    .getState()
-                    .nodes()
-                    .getMasterNode()
-                    .getName(),
-                equalTo(nextMasterEligableNodeName)
-            );
-            assertThat(
-                internalCluster().masterClient()
-                    .admin()
-                    .cluster()
-                    .prepareState(TEST_REQUEST_TIMEOUT)
-                    .get()
-                    .getState()
-                    .nodes()
-                    .getMasterNode()
-                    .getName(),
-                equalTo(nextMasterEligableNodeName)
-            );
-        });
+        awaitMasterNode(internalCluster().getNonMasterNodeName(), nextMasterEligableNodeName);
+        awaitMasterNode(internalCluster().getMasterName(), nextMasterEligableNodeName);
+
         internalCluster().stopNode(masterNodeName);
-        assertThat(
-            internalCluster().nonMasterClient()
-                .admin()
-                .cluster()
-                .prepareState(TEST_REQUEST_TIMEOUT)
-                .get()
-                .getState()
-                .nodes()
-                .getMasterNode()
-                .getName(),
-            equalTo(nextMasterEligableNodeName)
-        );
-        assertThat(
-            internalCluster().masterClient()
-                .admin()
-                .cluster()
-                .prepareState(TEST_REQUEST_TIMEOUT)
-                .get()
-                .getState()
-                .nodes()
-                .getMasterNode()
-                .getName(),
-            equalTo(nextMasterEligableNodeName)
-        );
+        awaitMasterNode(internalCluster().getNonMasterNodeName(), nextMasterEligableNodeName);
+        awaitMasterNode(internalCluster().getMasterName(), nextMasterEligableNodeName);
     }
 
     public void testAliasFilterValidation() {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index d3f906eda9f32..c4bc5adfce72a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -198,6 +198,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.Callable;
@@ -940,6 +941,39 @@ public void waitNoPendingTasksOnAll() throws Exception {
         assertNoTimeout(clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).setWaitForEvents(Priority.LANGUID).get());
     }
 
+    /**
+     * Waits for the node {@code viaNode} to see {@code masterNodeName} as the master node in the cluster state.
+     * Note that this does not guarantee that all other nodes in the cluster are on the same cluster state version already.
+     *
+     * @param viaNode the node to check the cluster state one
+     * @param masterNodeName the master node name that we wait for
+     */
+    public void awaitMasterNode(String viaNode, String masterNodeName) {
+        var listener = ClusterServiceUtils.addTemporaryStateListener(
+            internalCluster().clusterService(viaNode),
+            state -> Optional.ofNullable(state.nodes().getMasterNode()).map(m -> m.getName().equals(masterNodeName)).orElse(false),
+            TEST_REQUEST_TIMEOUT
+        );
+        safeAwait(listener, TEST_REQUEST_TIMEOUT);
+    }
+
+    /**
+     * Waits for a random node in the cluster to not see a master node in the cluster state.
+     * Note that this does not guarantee that all other nodes in the cluster are on the same cluster state version already.
+     */
+    public void awaitMasterNotFound() {
+        var viaNode = internalCluster().getRandomNodeName();
+        // We use a temporary state listener instead of `awaitClusterState` here because the `ClusterStateObserver` doesn't run the
+        // predicate if the cluster state version didn't change. When a master node leaves the cluster (i.e. what this method is used for),
+        // the cluster state version is not incremented.
+        var listener = ClusterServiceUtils.addTemporaryStateListener(
+            internalCluster().clusterService(viaNode),
+            state -> state.nodes().getMasterNode() == null,
+            TEST_REQUEST_TIMEOUT
+        );
+        safeAwait(listener, TEST_REQUEST_TIMEOUT);
+    }
+
     /** Ensures the result counts are as expected, and logs the results if different */
     public void assertResultsAndLogOnFailure(long expectedResults, SearchResponse searchResponse) {
         final TotalHits totalHits = searchResponse.getHits().getTotalHits();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index b4f278a7fbc7c..e00ae7abaf0eb 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -2033,15 +2033,28 @@ public String getMasterName() {
      * in the viaNode parameter. If viaNode isn't specified a random node will be picked to the send the request to.
      */
     public String getMasterName(@Nullable String viaNode) {
+        viaNode = viaNode != null ? viaNode : getRandomNodeName();
+        if (viaNode == null) {
+            throw new AssertionError("Unable to get master name, no node found");
+        }
         try {
-            Client client = viaNode != null ? client(viaNode) : client();
-            return client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode().getName();
+            ClusterServiceUtils.awaitClusterState(logger, state -> state.nodes().getMasterNode() != null, clusterService(viaNode));
+            final ClusterState state = client(viaNode).admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).setLocal(true).get().getState();
+            return state.nodes().getMasterNode().getName();
         } catch (Exception e) {
             logger.warn("Can't fetch cluster state", e);
             throw new RuntimeException("Can't get master node " + e.getMessage(), e);
         }
     }
 
+    public String getNonMasterNodeName() {
+        NodeAndClient randomNodeAndClient = getRandomNodeAndClient(new NodeNamePredicate(getMasterName()).negate());
+        if (randomNodeAndClient != null) {
+            return randomNodeAndClient.getName();
+        }
+        throw new AssertionError("No non-master node found");
+    }
+
     /**
      * @return the name of a random node in a cluster
      */


Reviews:
user: login: nielsbauman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-23T08:03:43Z
user: login: DaveCTurner
state: DISMISSED
author_association: MEMBER
submitted_at: 2025-04-23T09:01:57Z
user: login: nielsbauman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-23T09:12:21Z
user: login: nielsbauman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-23T09:12:34Z
user: login: DaveCTurner
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-23T09:33:42Z
user: login: nielsbauman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-23T09:44:13Z
user: login: nielsbauman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-24T11:57:37Z
user: login: DaveCTurner
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-25T09:50:33Z
user: login: nielsbauman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-25T09:55:02Z
user: login: nielsbauman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-25T10:08:23Z
user: login: DaveCTurner
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-25T11:17:25Z
user: login: DaveCTurner
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-25T11:20:41Z
user: login: DaveCTurner
body: LGTM
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-25T11:20:54Z

Review comments:
diff_hunk: @@ -940,6 +942,41 @@ public void waitNoPendingTasksOnAll() throws Exception {
         assertNoTimeout(clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).setWaitForEvents(Priority.LANGUID).get());
     }
 
+    /**
+     * Waits for the node {@code viaNode} to see {@code masterNodeName} as the master node in the cluster state and asserts that.
+     * Note that this does not guarantee that all other nodes in the cluster are on the same cluster state version already.
+     *
+     * @param viaNode the node to check the cluster state one
+     * @param masterNodeName the master node name that we wait for
+     */
+    public void awaitAndAssertMasterNode(String viaNode, String masterNodeName) throws Exception {
path: test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
user: login: nielsbauman
body: I added these methods in `ESIntegTestCase` because I see other use cases for these methods (which I'll get to in follow-up PRs). I'm a little bit on the fence about which class they should live in, though. I would actually like to avoid adding them in this class, as this class is too big already. Maybe `InternalTestCluster`? Or an entirely new class?
created_at: 2025-04-23T08:01:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -940,6 +942,41 @@ public void waitNoPendingTasksOnAll() throws Exception {
         assertNoTimeout(clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).setWaitForEvents(Priority.LANGUID).get());
     }
 
+    /**
+     * Waits for the node {@code viaNode} to see {@code masterNodeName} as the master node in the cluster state and asserts that.
+     * Note that this does not guarantee that all other nodes in the cluster are on the same cluster state version already.
+     *
+     * @param viaNode the node to check the cluster state one
+     * @param masterNodeName the master node name that we wait for
+     */
+    public void awaitAndAssertMasterNode(String viaNode, String masterNodeName) throws Exception {
+        awaitClusterState(
+            logger,
+            viaNode,
+            state -> Optional.ofNullable(state.nodes().getMasterNode()).map(m -> m.getName().equals(masterNodeName)).orElse(false)
+        );
+        var nodes = client(viaNode).admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).setLocal(true).get().getState().nodes();
path: test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
user: login: nielsbauman
body: These API calls (also the ones below) have `setLocal(true)` because that's how they will be run once the API runs on the local node in the future.
created_at: 2025-04-23T08:03:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2036,20 +2036,46 @@ public String getMasterName() {
      * in the viaNode parameter. If viaNode isn't specified a random node will be picked to the send the request to.
      */
     public String getMasterName(@Nullable String viaNode) {
+        viaNode = viaNode != null ? viaNode : getRandomNodeName(true);
         try {
-            Client client = viaNode != null ? client(viaNode) : client();
-            return client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode().getName();
+            ClusterServiceUtils.awaitClusterState(logger, state -> state.nodes().getMasterNode() != null, clusterService(viaNode));
+            final ClusterState state = client(viaNode).admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).setLocal(true).get().getState();
+            return state.nodes().getMasterNode().getName();
         } catch (Exception e) {
             logger.warn("Can't fetch cluster state", e);
             throw new RuntimeException("Can't get master node " + e.getMessage(), e);
         }
     }
 
+    public String getNonMasterNodeName() {
+        NodeAndClient randomNodeAndClient = getRandomNodeAndClient(new NodeNamePredicate(getMasterName()).negate());
+        if (randomNodeAndClient != null) {
+            return randomNodeAndClient.getName();
+        }
+        throw new AssertionError("No non-master node found");
+    }
+
     /**
      * @return the name of a random node in a cluster
      */
     public String getRandomNodeName() {
-        return getNodeNameThat(Predicates.always());
+        return getRandomNodeName(false);
+    }
+
+    /**
+     * @param startNewNode whether we should start a new node if there are no nodes in the cluster
+     * @return the name of a random node in a cluster
+     */
+    public String getRandomNodeName(boolean startNewNode) {
+        String node = getNodeNameThat(Predicates.always());
+        ensureOpen();
+        if (node == null) {
+            synchronized (this) {
+                return getOrBuildRandomNode().getName();
path: test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
user: login: DaveCTurner
body: I think this is super-trappy - `getMasterName()` definitely doesn't look like the sort of thing that might start a node, especially since there's no way to configure this magic new node. Tests should know whether they have a node running or not when calling this, and should start a node themselves if they want one.
created_at: 2025-04-23T08:58:51Z
author_association: MEMBER
diff_hunk: @@ -940,6 +942,41 @@ public void waitNoPendingTasksOnAll() throws Exception {
         assertNoTimeout(clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).setWaitForEvents(Priority.LANGUID).get());
     }
 
+    /**
+     * Waits for the node {@code viaNode} to see {@code masterNodeName} as the master node in the cluster state and asserts that.
+     * Note that this does not guarantee that all other nodes in the cluster are on the same cluster state version already.
+     *
+     * @param viaNode the node to check the cluster state one
+     * @param masterNodeName the master node name that we wait for
+     */
+    public void awaitAndAssertMasterNode(String viaNode, String masterNodeName) throws Exception {
+        awaitClusterState(
+            logger,
+            viaNode,
+            state -> Optional.ofNullable(state.nodes().getMasterNode()).map(m -> m.getName().equals(masterNodeName)).orElse(false)
+        );
+        var nodes = client(viaNode).admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).setLocal(true).get().getState().nodes();
+        assertThat(Optional.ofNullable(nodes.getMasterNode()).map(DiscoveryNode::getName).orElse(null), equalTo(masterNodeName));
path: test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
user: login: DaveCTurner
body: Unsure why we're using the cluster state API at all here? I think this could technically be flaky, elections don't always settle down on the first attempt so this might not always succeed. Is the `awaitClusterState` not enough?
created_at: 2025-04-23T09:01:50Z
author_association: MEMBER
diff_hunk: @@ -940,6 +942,41 @@ public void waitNoPendingTasksOnAll() throws Exception {
         assertNoTimeout(clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).setWaitForEvents(Priority.LANGUID).get());
     }
 
+    /**
+     * Waits for the node {@code viaNode} to see {@code masterNodeName} as the master node in the cluster state and asserts that.
+     * Note that this does not guarantee that all other nodes in the cluster are on the same cluster state version already.
+     *
+     * @param viaNode the node to check the cluster state one
+     * @param masterNodeName the master node name that we wait for
+     */
+    public void awaitAndAssertMasterNode(String viaNode, String masterNodeName) throws Exception {
+        awaitClusterState(
+            logger,
+            viaNode,
+            state -> Optional.ofNullable(state.nodes().getMasterNode()).map(m -> m.getName().equals(masterNodeName)).orElse(false)
+        );
+        var nodes = client(viaNode).admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).setLocal(true).get().getState().nodes();
+        assertThat(Optional.ofNullable(nodes.getMasterNode()).map(DiscoveryNode::getName).orElse(null), equalTo(masterNodeName));
path: test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
user: login: nielsbauman
body: I added the assertion to replace the assertions in `SpecificMasterNodesIT`. Otherwise those tests don't really assert much. They'll indirectly "assert" by waiting for the predicate to match, but I felt like a direct assertion would be more clear. What do you think?
created_at: 2025-04-23T09:12:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2036,20 +2036,46 @@ public String getMasterName() {
      * in the viaNode parameter. If viaNode isn't specified a random node will be picked to the send the request to.
      */
     public String getMasterName(@Nullable String viaNode) {
+        viaNode = viaNode != null ? viaNode : getRandomNodeName(true);
         try {
-            Client client = viaNode != null ? client(viaNode) : client();
-            return client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode().getName();
+            ClusterServiceUtils.awaitClusterState(logger, state -> state.nodes().getMasterNode() != null, clusterService(viaNode));
+            final ClusterState state = client(viaNode).admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).setLocal(true).get().getState();
+            return state.nodes().getMasterNode().getName();
         } catch (Exception e) {
             logger.warn("Can't fetch cluster state", e);
             throw new RuntimeException("Can't get master node " + e.getMessage(), e);
         }
     }
 
+    public String getNonMasterNodeName() {
+        NodeAndClient randomNodeAndClient = getRandomNodeAndClient(new NodeNamePredicate(getMasterName()).negate());
+        if (randomNodeAndClient != null) {
+            return randomNodeAndClient.getName();
+        }
+        throw new AssertionError("No non-master node found");
+    }
+
     /**
      * @return the name of a random node in a cluster
      */
     public String getRandomNodeName() {
-        return getNodeNameThat(Predicates.always());
+        return getRandomNodeName(false);
+    }
+
+    /**
+     * @param startNewNode whether we should start a new node if there are no nodes in the cluster
+     * @return the name of a random node in a cluster
+     */
+    public String getRandomNodeName(boolean startNewNode) {
+        String node = getNodeNameThat(Predicates.always());
+        ensureOpen();
+        if (node == null) {
+            synchronized (this) {
+                return getOrBuildRandomNode().getName();
path: test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
user: login: nielsbauman
body: Good point... I reverted [Start new node if none found b9093e2](https://github.com/elastic/elasticsearch/pull/127213/commits/b9093e228959c094eda945ccb1ae9d306df96a3d) and added [9aee869](https://github.com/elastic/elasticsearch/pull/127213/commits/9aee869da7065e566189b731a492c7a1d5724e63)
created_at: 2025-04-23T09:12:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -940,6 +942,41 @@ public void waitNoPendingTasksOnAll() throws Exception {
         assertNoTimeout(clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).setWaitForEvents(Priority.LANGUID).get());
     }
 
+    /**
+     * Waits for the node {@code viaNode} to see {@code masterNodeName} as the master node in the cluster state and asserts that.
+     * Note that this does not guarantee that all other nodes in the cluster are on the same cluster state version already.
+     *
+     * @param viaNode the node to check the cluster state one
+     * @param masterNodeName the master node name that we wait for
+     */
+    public void awaitAndAssertMasterNode(String viaNode, String masterNodeName) throws Exception {
+        awaitClusterState(
+            logger,
+            viaNode,
+            state -> Optional.ofNullable(state.nodes().getMasterNode()).map(m -> m.getName().equals(masterNodeName)).orElse(false)
+        );
+        var nodes = client(viaNode).admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).setLocal(true).get().getState().nodes();
+        assertThat(Optional.ofNullable(nodes.getMasterNode()).map(DiscoveryNode::getName).orElse(null), equalTo(masterNodeName));
path: test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
user: login: DaveCTurner
body: I think just waiting for the condition is fine - either the wait completes (which seems sufficient) or it times out (and doesn't even get to this point).
created_at: 2025-04-23T09:22:14Z
author_association: MEMBER
diff_hunk: @@ -2036,20 +2036,46 @@ public String getMasterName() {
      * in the viaNode parameter. If viaNode isn't specified a random node will be picked to the send the request to.
      */
     public String getMasterName(@Nullable String viaNode) {
+        viaNode = viaNode != null ? viaNode : getRandomNodeName(true);
         try {
-            Client client = viaNode != null ? client(viaNode) : client();
-            return client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode().getName();
+            ClusterServiceUtils.awaitClusterState(logger, state -> state.nodes().getMasterNode() != null, clusterService(viaNode));
+            final ClusterState state = client(viaNode).admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).setLocal(true).get().getState();
+            return state.nodes().getMasterNode().getName();
         } catch (Exception e) {
             logger.warn("Can't fetch cluster state", e);
             throw new RuntimeException("Can't get master node " + e.getMessage(), e);
         }
     }
 
+    public String getNonMasterNodeName() {
+        NodeAndClient randomNodeAndClient = getRandomNodeAndClient(new NodeNamePredicate(getMasterName()).negate());
+        if (randomNodeAndClient != null) {
+            return randomNodeAndClient.getName();
+        }
+        throw new AssertionError("No non-master node found");
+    }
+
     /**
      * @return the name of a random node in a cluster
      */
     public String getRandomNodeName() {
-        return getNodeNameThat(Predicates.always());
+        return getRandomNodeName(false);
+    }
+
+    /**
+     * @param startNewNode whether we should start a new node if there are no nodes in the cluster
+     * @return the name of a random node in a cluster
+     */
+    public String getRandomNodeName(boolean startNewNode) {
+        String node = getNodeNameThat(Predicates.always());
+        ensureOpen();
+        if (node == null) {
+            synchronized (this) {
+                return getOrBuildRandomNode().getName();
path: test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
user: login: DaveCTurner
body: Oh hang on wtf calling `client()` (and therefore `getMasterName()`) in an empty cluster creates a new default node already today?! Can we make it not do that first? Unsure how many tests rely on that behaviour right now but I hope it's not many.
created_at: 2025-04-23T09:33:32Z
author_association: MEMBER
diff_hunk: @@ -2036,20 +2036,46 @@ public String getMasterName() {
      * in the viaNode parameter. If viaNode isn't specified a random node will be picked to the send the request to.
      */
     public String getMasterName(@Nullable String viaNode) {
+        viaNode = viaNode != null ? viaNode : getRandomNodeName(true);
         try {
-            Client client = viaNode != null ? client(viaNode) : client();
-            return client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode().getName();
+            ClusterServiceUtils.awaitClusterState(logger, state -> state.nodes().getMasterNode() != null, clusterService(viaNode));
+            final ClusterState state = client(viaNode).admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).setLocal(true).get().getState();
+            return state.nodes().getMasterNode().getName();
         } catch (Exception e) {
             logger.warn("Can't fetch cluster state", e);
             throw new RuntimeException("Can't get master node " + e.getMessage(), e);
         }
     }
 
+    public String getNonMasterNodeName() {
+        NodeAndClient randomNodeAndClient = getRandomNodeAndClient(new NodeNamePredicate(getMasterName()).negate());
+        if (randomNodeAndClient != null) {
+            return randomNodeAndClient.getName();
+        }
+        throw new AssertionError("No non-master node found");
+    }
+
     /**
      * @return the name of a random node in a cluster
      */
     public String getRandomNodeName() {
-        return getNodeNameThat(Predicates.always());
+        return getRandomNodeName(false);
+    }
+
+    /**
+     * @param startNewNode whether we should start a new node if there are no nodes in the cluster
+     * @return the name of a random node in a cluster
+     */
+    public String getRandomNodeName(boolean startNewNode) {
+        String node = getNodeNameThat(Predicates.always());
+        ensureOpen();
+        if (node == null) {
+            synchronized (this) {
+                return getOrBuildRandomNode().getName();
path: test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
user: login: nielsbauman
body: Ah sorry yeah I'm trying to juggle too many things at the same time (and failing to do so...). That's indeed why I added the behavior initially, to match what `getMasterName()` currently does. I can address that in a separate PR first.
created_at: 2025-04-23T09:44:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2036,20 +2036,46 @@ public String getMasterName() {
      * in the viaNode parameter. If viaNode isn't specified a random node will be picked to the send the request to.
      */
     public String getMasterName(@Nullable String viaNode) {
+        viaNode = viaNode != null ? viaNode : getRandomNodeName(true);
         try {
-            Client client = viaNode != null ? client(viaNode) : client();
-            return client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode().getName();
+            ClusterServiceUtils.awaitClusterState(logger, state -> state.nodes().getMasterNode() != null, clusterService(viaNode));
+            final ClusterState state = client(viaNode).admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).setLocal(true).get().getState();
+            return state.nodes().getMasterNode().getName();
         } catch (Exception e) {
             logger.warn("Can't fetch cluster state", e);
             throw new RuntimeException("Can't get master node " + e.getMessage(), e);
         }
     }
 
+    public String getNonMasterNodeName() {
+        NodeAndClient randomNodeAndClient = getRandomNodeAndClient(new NodeNamePredicate(getMasterName()).negate());
+        if (randomNodeAndClient != null) {
+            return randomNodeAndClient.getName();
+        }
+        throw new AssertionError("No non-master node found");
+    }
+
     /**
      * @return the name of a random node in a cluster
      */
     public String getRandomNodeName() {
-        return getNodeNameThat(Predicates.always());
+        return getRandomNodeName(false);
+    }
+
+    /**
+     * @param startNewNode whether we should start a new node if there are no nodes in the cluster
+     * @return the name of a random node in a cluster
+     */
+    public String getRandomNodeName(boolean startNewNode) {
+        String node = getNodeNameThat(Predicates.always());
+        ensureOpen();
+        if (node == null) {
+            synchronized (this) {
+                return getOrBuildRandomNode().getName();
path: test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
user: login: nielsbauman
body: I opened #127318 to address this.
created_at: 2025-04-24T11:57:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -940,6 +941,37 @@ public void waitNoPendingTasksOnAll() throws Exception {
         assertNoTimeout(clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).setWaitForEvents(Priority.LANGUID).get());
     }
 
+    /**
+     * Waits for the node {@code viaNode} to see {@code masterNodeName} as the master node in the cluster state and asserts that.
+     * Note that this does not guarantee that all other nodes in the cluster are on the same cluster state version already.
+     *
+     * @param viaNode the node to check the cluster state one
+     * @param masterNodeName the master node name that we wait for
+     */
+    public void awaitAndAssertMasterNode(String viaNode, String masterNodeName) throws Exception {
+        awaitClusterState(
path: test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
user: login: DaveCTurner
body: I'd be tempted to use `addTemporaryStateListener` here too, because (a) it is more harmonious with `awaitAndAssertMasterNotFound` and (b) it avoids the `throws Exception`.
created_at: 2025-04-25T09:38:44Z
author_association: MEMBER
diff_hunk: @@ -2033,15 +2033,28 @@ public String getMasterName() {
      * in the viaNode parameter. If viaNode isn't specified a random node will be picked to the send the request to.
      */
     public String getMasterName(@Nullable String viaNode) {
+        viaNode = viaNode != null ? viaNode : getRandomNodeName();
+        if (viaNode == null) {
+            throw new AssertionError("Unable to get master name, no node found");
+        }
         try {
-            Client client = viaNode != null ? client(viaNode) : client();
-            return client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode().getName();
+            ClusterServiceUtils.awaitClusterState(logger, state -> state.nodes().getMasterNode() != null, clusterService(viaNode));
+            final ClusterState state = client(viaNode).admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).setLocal(true).get().getState();
+            return state.nodes().getMasterNode().getName();
         } catch (Exception e) {
             logger.warn("Can't fetch cluster state", e);
             throw new RuntimeException("Can't get master node " + e.getMessage(), e);
         }
path: test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
user: login: DaveCTurner
body: I still worry this reads the state twice and might see no master the second time due to election jitter. We could cheat and remember the master node seen in the state on which we're awaiting (also saves the exception handling):

```java
        final var masterNameListener = new SubscribableListener<String>();
        return safeAwait(ClusterServiceUtils.addTemporaryStateListener(clusterService(viaNode), cs -> {
            Optional.ofNullable(cs.nodes().getMasterNode()).ifPresent(masterNode -> masterNameListener.onResponse(masterNode.getName()));
            return masterNameListener.isDone();
        }).andThen(masterNameListener::addListener));
```
created_at: 2025-04-25T09:49:53Z
author_association: MEMBER
diff_hunk: @@ -940,6 +941,37 @@ public void waitNoPendingTasksOnAll() throws Exception {
         assertNoTimeout(clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).setWaitForEvents(Priority.LANGUID).get());
     }
 
+    /**
+     * Waits for the node {@code viaNode} to see {@code masterNodeName} as the master node in the cluster state and asserts that.
+     * Note that this does not guarantee that all other nodes in the cluster are on the same cluster state version already.
+     *
+     * @param viaNode the node to check the cluster state one
+     * @param masterNodeName the master node name that we wait for
+     */
+    public void awaitAndAssertMasterNode(String viaNode, String masterNodeName) throws Exception {
+        awaitClusterState(
path: test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
user: login: nielsbauman
body: Yeah I thought about that too. I'm fine with using `addTemporaryStateListener` here too.

Is there a reason we would intentionally want the behavior of `awaitClusterState`? I've been tempted to make `awaitClusterState` just use `addTemporaryClusterState` (or even getting rid of `awaitClusterState`, but that's a larger change).
created_at: 2025-04-25T09:55:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2033,15 +2033,28 @@ public String getMasterName() {
      * in the viaNode parameter. If viaNode isn't specified a random node will be picked to the send the request to.
      */
     public String getMasterName(@Nullable String viaNode) {
+        viaNode = viaNode != null ? viaNode : getRandomNodeName();
+        if (viaNode == null) {
+            throw new AssertionError("Unable to get master name, no node found");
+        }
         try {
-            Client client = viaNode != null ? client(viaNode) : client();
-            return client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode().getName();
+            ClusterServiceUtils.awaitClusterState(logger, state -> state.nodes().getMasterNode() != null, clusterService(viaNode));
+            final ClusterState state = client(viaNode).admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).setLocal(true).get().getState();
+            return state.nodes().getMasterNode().getName();
         } catch (Exception e) {
             logger.warn("Can't fetch cluster state", e);
             throw new RuntimeException("Can't get master node " + e.getMessage(), e);
         }
path: test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
user: login: nielsbauman
body: I thought about that too, but I figured "election jitter" would likely disrupt the test anyway. Tests should be designed to be deterministic/consistent, but there's only so much outside influence tests can account for (e.g. election jitter or CI blips).

I'm a little hesitant to "cheat" here. I seem to recall someone saying
> Having to obtain the cluster state in another way is a feature, not a bug :)

https://github.com/elastic/elasticsearch/pull/125195#discussion_r2002993861 :wink: 
created_at: 2025-04-25T10:08:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2033,15 +2033,28 @@ public String getMasterName() {
      * in the viaNode parameter. If viaNode isn't specified a random node will be picked to the send the request to.
      */
     public String getMasterName(@Nullable String viaNode) {
+        viaNode = viaNode != null ? viaNode : getRandomNodeName();
+        if (viaNode == null) {
+            throw new AssertionError("Unable to get master name, no node found");
+        }
         try {
-            Client client = viaNode != null ? client(viaNode) : client();
-            return client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode().getName();
+            ClusterServiceUtils.awaitClusterState(logger, state -> state.nodes().getMasterNode() != null, clusterService(viaNode));
+            final ClusterState state = client(viaNode).admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).setLocal(true).get().getState();
+            return state.nodes().getMasterNode().getName();
         } catch (Exception e) {
             logger.warn("Can't fetch cluster state", e);
             throw new RuntimeException("Can't get master node " + e.getMessage(), e);
         }
path: test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
user: login: DaveCTurner
body: Heh yeah I thought I'd said that at some point :) On reflection I think you're right, tests should not be calling this if the master isn't stable.
created_at: 2025-04-25T11:17:25Z
author_association: MEMBER
diff_hunk: @@ -940,6 +941,37 @@ public void waitNoPendingTasksOnAll() throws Exception {
         assertNoTimeout(clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).setWaitForEvents(Priority.LANGUID).get());
     }
 
+    /**
+     * Waits for the node {@code viaNode} to see {@code masterNodeName} as the master node in the cluster state and asserts that.
+     * Note that this does not guarantee that all other nodes in the cluster are on the same cluster state version already.
+     *
+     * @param viaNode the node to check the cluster state one
+     * @param masterNodeName the master node name that we wait for
+     */
+    public void awaitAndAssertMasterNode(String viaNode, String masterNodeName) throws Exception {
+        awaitClusterState(
path: test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
user: login: DaveCTurner
body: > Is there a reason we would intentionally want the behavior of `awaitClusterState`?

IMO not really no. `ClusterStateObserver` is really old and suspiciously complicated given what it's actually doing. I'd be happy to migrate things away from it.
created_at: 2025-04-25T11:20:41Z
author_association: MEMBER

PR comments:
user: login: elasticsearchmachine
created_at: 2025-04-23T07:59:57Z
body: Pinging @elastic/es-distributed-coordination (Team:Distributed Coordination)
author_association: COLLABORATOR
user: login: nielsbauman
created_at: 2025-04-25T09:38:28Z
body: @DaveCTurner this is ready for review again. Could you have a look when you some free time?
author_association: CONTRIBUTOR