Repository: elastic/elasticsearch
Pull request #127658: [ML] Append all data to Chat Completion buffer
Author: login: prwhelan
Labels: name: >bug
name: :ml
description: Machine learning
name: Team:ML
description: Meta label for the ML team
name: auto-backport
description: Automatically create backport pull requests when merged
name: v8.19.0
name: v9.1.0
name: v8.18.2
name: v9.0.2
Created: 2025-05-02T18:58:54Z | Merged: 2025-05-05T21:08:14Z

Moved the Chat Completion buffer into the
StreamingUnifiedChatCompletionResults so that all Chat Completion responses can benefit from it. Chat Completions is meant to adhere to OpenAI as much as possible, and OpenAI only sends one response chunk at a time. All implementations of Chat Completions will now buffer.

This fixes a bug where more than two chunks in a single item would be dropped, instead they are all added to the buffer.
This fixes a bug where onComplete would omit trailing items in the buffer.


Diff:
diff --git a/docs/changelog/127658.yaml b/docs/changelog/127658.yaml
new file mode 100644
index 0000000000000..1a8d5ced7c8b6
--- /dev/null
+++ b/docs/changelog/127658.yaml
@@ -0,0 +1,5 @@
+pr: 127658
+summary: Append all data to Chat Completion buffer
+area: Machine Learning
+type: bug
+issues: []
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResults.java
index 99404d9ce66b0..a15f78a29e269 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResults.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResults.java
@@ -16,6 +16,7 @@
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.inference.InferenceServiceResults;
 import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xpack.core.inference.DequeUtils;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -23,6 +24,8 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.Flow;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.chunk;
 import static org.elasticsearch.xpack.core.inference.DequeUtils.dequeEquals;
@@ -32,9 +35,7 @@
 /**
  * Chat Completion results that only contain a Flow.Publisher.
  */
-public record StreamingUnifiedChatCompletionResults(Flow.Publisher<? extends InferenceServiceResults.Result> publisher)
-    implements
-        InferenceServiceResults {
+public record StreamingUnifiedChatCompletionResults(Flow.Publisher<Results> publisher) implements InferenceServiceResults {
 
     public static final String NAME = "chat_completion_chunk";
     public static final String MODEL_FIELD = "model";
@@ -57,6 +58,63 @@ public record StreamingUnifiedChatCompletionResults(Flow.Publisher<? extends Inf
     public static final String PROMPT_TOKENS_FIELD = "prompt_tokens";
     public static final String TYPE_FIELD = "type";
 
+    /**
+     * OpenAI Spec only returns one result at a time, and Chat Completion adheres to that spec as much as possible.
+     * So we will insert a buffer in between the upstream data and the downstream client so that we only send one request at a time.
+     */
+    public StreamingUnifiedChatCompletionResults(Flow.Publisher<Results> publisher) {
+        Deque<StreamingUnifiedChatCompletionResults.ChatCompletionChunk> buffer = new LinkedBlockingDeque<>();
+        AtomicBoolean onComplete = new AtomicBoolean();
+        this.publisher = downstream -> {
+            publisher.subscribe(new Flow.Subscriber<>() {
+                @Override
+                public void onSubscribe(Flow.Subscription subscription) {
+                    downstream.onSubscribe(new Flow.Subscription() {
+                        @Override
+                        public void request(long n) {
+                            var nextItem = buffer.poll();
+                            if (nextItem != null) {
+                                downstream.onNext(new Results(DequeUtils.of(nextItem)));
+                            } else if (onComplete.get()) {
+                                downstream.onComplete();
+                            } else {
+                                subscription.request(n);
+                            }
+                        }
+
+                        @Override
+                        public void cancel() {
+                            subscription.cancel();
+                        }
+                    });
+                }
+
+                @Override
+                public void onNext(Results item) {
+                    var chunks = item.chunks();
+                    var firstItem = chunks.poll();
+                    chunks.forEach(buffer::offer);
+                    downstream.onNext(new Results(DequeUtils.of(firstItem)));
+                }
+
+                @Override
+                public void onError(Throwable throwable) {
+                    downstream.onError(throwable);
+                }
+
+                @Override
+                public void onComplete() {
+                    // only complete if the buffer is empty, so that the client has a chance to drain the buffer
+                    if (onComplete.compareAndSet(false, true)) {
+                        if (buffer.isEmpty()) {
+                            downstream.onComplete();
+                        }
+                    }
+                }
+            });
+        };
+    }
+
     @Override
     public boolean isStreaming() {
         return true;
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResultsTests.java
index 669ba2f881fe7..ccd67fe4029d9 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResultsTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResultsTests.java
@@ -19,8 +19,17 @@
 import java.util.ArrayDeque;
 import java.util.Deque;
 import java.util.List;
+import java.util.concurrent.Flow;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
 
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
 public class StreamingUnifiedChatCompletionResultsTests extends AbstractWireSerializingTestCase<
     StreamingUnifiedChatCompletionResults.Results> {
 
@@ -198,6 +207,66 @@ public void testToolCallToXContentChunked() throws IOException {
         assertEquals(expected.replaceAll("\\s+", ""), Strings.toString(builder.prettyPrint()).trim());
     }
 
+    public void testBufferedPublishing() {
+        var results = new ArrayDeque<StreamingUnifiedChatCompletionResults.ChatCompletionChunk>();
+        results.offer(randomChatCompletionChunk());
+        results.offer(randomChatCompletionChunk());
+        var completed = new AtomicBoolean();
+        var streamingResults = new StreamingUnifiedChatCompletionResults(downstream -> {
+            downstream.onSubscribe(new Flow.Subscription() {
+                @Override
+                public void request(long n) {
+                    if (completed.compareAndSet(false, true)) {
+                        downstream.onNext(new StreamingUnifiedChatCompletionResults.Results(results));
+                    } else {
+                        downstream.onComplete();
+                    }
+                }
+
+                @Override
+                public void cancel() {
+                    fail("Cancel should never be called.");
+                }
+            });
+        });
+
+        AtomicInteger counter = new AtomicInteger(0);
+        AtomicReference<Flow.Subscription> upstream = new AtomicReference<>(null);
+        Flow.Subscriber<StreamingUnifiedChatCompletionResults.Results> subscriber = spy(
+            new Flow.Subscriber<StreamingUnifiedChatCompletionResults.Results>() {
+                @Override
+                public void onSubscribe(Flow.Subscription subscription) {
+                    if (upstream.compareAndSet(null, subscription) == false) {
+                        fail("Upstream already set?!");
+                    }
+                    subscription.request(1);
+                }
+
+                @Override
+                public void onNext(StreamingUnifiedChatCompletionResults.Results item) {
+                    assertNotNull(item);
+                    counter.incrementAndGet();
+                    var sub = upstream.get();
+                    if (sub != null) {
+                        sub.request(1);
+                    } else {
+                        fail("Upstream not yet set?!");
+                    }
+                }
+
+                @Override
+                public void onError(Throwable throwable) {
+                    fail(throwable);
+                }
+
+                @Override
+                public void onComplete() {}
+            }
+        );
+        streamingResults.publisher().subscribe(subscriber);
+        verify(subscriber, times(2)).onNext(any());
+    }
+
     @Override
     protected Writeable.Reader<StreamingUnifiedChatCompletionResults.Results> instanceReader() {
         return StreamingUnifiedChatCompletionResults.Results::new;
diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java
index e34018c5b8df1..b2f8ba5475eb8 100644
--- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java
+++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestStreamingCompletionServiceExtension.java
@@ -34,6 +34,7 @@
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.inference.DequeUtils;
 import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults;
 import org.elasticsearch.xpack.core.inference.results.StreamingUnifiedChatCompletionResults;
 import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults;
@@ -256,37 +257,24 @@ public void cancel() {}
           "object": "chat.completion.chunk"
         }
          */
-        private InferenceServiceResults.Result unifiedCompletionChunk(String delta) {
-            return new InferenceServiceResults.Result() {
-                @Override
-                public String getWriteableName() {
-                    return "test_unifiedCompletionChunk";
-                }
-
-                @Override
-                public void writeTo(StreamOutput out) throws IOException {
-                    out.writeString(delta);
-                }
-
-                @Override
-                public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
-                    return ChunkedToXContentHelper.chunk(
-                        (b, p) -> b.startObject()
-                            .field("id", "id")
-                            .startArray("choices")
-                            .startObject()
-                            .startObject("delta")
-                            .field("content", delta)
-                            .endObject()
-                            .field("index", 0)
-                            .endObject()
-                            .endArray()
-                            .field("model", "gpt-4o-2024-08-06")
-                            .field("object", "chat.completion.chunk")
-                            .endObject()
-                    );
-                }
-            };
+        private StreamingUnifiedChatCompletionResults.Results unifiedCompletionChunk(String delta) {
+            return new StreamingUnifiedChatCompletionResults.Results(
+                DequeUtils.of(
+                    new StreamingUnifiedChatCompletionResults.ChatCompletionChunk(
+                        "id",
+                        List.of(
+                            new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice(
+                                new StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Choice.Delta(delta, null, null, null),
+                                null,
+                                0
+                            )
+                        ),
+                        "gpt-4o-2024-08-06",
+                        "chat.completion.chunk",
+                        null
+                    )
+                )
+            );
         }
 
         @Override
diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiUnifiedStreamingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiUnifiedStreamingProcessor.java
index 10c8d8928ea65..5ab743c3d4cc0 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiUnifiedStreamingProcessor.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiUnifiedStreamingProcessor.java
@@ -26,7 +26,6 @@
 import java.util.Deque;
 import java.util.Iterator;
 import java.util.List;
-import java.util.concurrent.LinkedBlockingDeque;
 import java.util.function.BiFunction;
 
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
@@ -60,21 +59,11 @@ public class OpenAiUnifiedStreamingProcessor extends DelegatingProcessor<
     public static final String TOTAL_TOKENS_FIELD = "total_tokens";
 
     private final BiFunction<String, Exception, Exception> errorParser;
-    private final Deque<StreamingUnifiedChatCompletionResults.ChatCompletionChunk> buffer = new LinkedBlockingDeque<>();
 
     public OpenAiUnifiedStreamingProcessor(BiFunction<String, Exception, Exception> errorParser) {
         this.errorParser = errorParser;
     }
 
-    @Override
-    protected void upstreamRequest(long n) {
-        if (buffer.isEmpty()) {
-            super.upstreamRequest(n);
-        } else {
-            downstream().onNext(new StreamingUnifiedChatCompletionResults.Results(singleItem(buffer.poll())));
-        }
-    }
-
     @Override
     protected void next(Deque<ServerSentEvent> item) throws Exception {
         var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE);
@@ -96,15 +85,8 @@ protected void next(Deque<ServerSentEvent> item) throws Exception {
 
         if (results.isEmpty()) {
             upstream().request(1);
-        } else if (results.size() == 1) {
-            downstream().onNext(new StreamingUnifiedChatCompletionResults.Results(results));
         } else {
-            // results > 1, but openai spec only wants 1 chunk per SSE event
-            var firstItem = singleItem(results.poll());
-            while (results.isEmpty() == false) {
-                buffer.offer(results.poll());
-            }
-            downstream().onNext(new StreamingUnifiedChatCompletionResults.Results(firstItem));
+            downstream().onNext(new StreamingUnifiedChatCompletionResults.Results(results));
         }
     }
 
@@ -297,12 +279,4 @@ public static StreamingUnifiedChatCompletionResults.ChatCompletionChunk.Usage pa
             }
         }
     }
-
-    private Deque<StreamingUnifiedChatCompletionResults.ChatCompletionChunk> singleItem(
-        StreamingUnifiedChatCompletionResults.ChatCompletionChunk result
-    ) {
-        var deque = new ArrayDeque<StreamingUnifiedChatCompletionResults.ChatCompletionChunk>(1);
-        deque.offer(result);
-        return deque;
-    }
 }


Reviews:
user: login: jonathan-buttner
body: Looks good, left some questions.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-05T14:23:53Z
user: login: prwhelan
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-05T14:40:28Z
user: login: jonathan-buttner
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-05T15:29:42Z

Review comments:
diff_hunk: @@ -57,6 +58,64 @@ public record StreamingUnifiedChatCompletionResults(Flow.Publisher<? extends Inf
     public static final String PROMPT_TOKENS_FIELD = "prompt_tokens";
     public static final String TYPE_FIELD = "type";
 
+    /**
+     * OpenAI Spec only returns one result at a time, and Chat Completion adheres to that spec as much as possible.
+     * So we will insert a buffer in between the upstream data and the downstream client so that we only send one request at a time.
+     */
+    public StreamingUnifiedChatCompletionResults(Flow.Publisher<Results> publisher) {
+        Deque<StreamingUnifiedChatCompletionResults.ChatCompletionChunk> buffer = new LinkedBlockingDeque<>();
+        AtomicBoolean onComplete = new AtomicBoolean();
+        this.publisher = downstream -> {
+            publisher.subscribe(new Flow.Subscriber<>() {
+                @Override
+                public void onSubscribe(Flow.Subscription subscription) {
+                    downstream.onSubscribe(new Flow.Subscription() {
+                        @Override
+                        public void request(long n) {
+                            if (buffer.isEmpty()) {
+                                if (onComplete.get()) {
+                                    downstream.onComplete();
+                                } else {
+                                    subscription.request(n);
+                                }
+                            } else {
+                                downstream.onNext(new Results(DequeUtils.of(buffer.poll())));
path: x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResults.java
user: login: jonathan-buttner
body: Is there only 1 thread accessing the `buffer`? Or is there a chance that we could check for `isEmpty()` and then some other thread picks up the item before this thread?
created_at: 2025-05-05T14:14:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -198,6 +204,63 @@ public void testToolCallToXContentChunked() throws IOException {
         assertEquals(expected.replaceAll("\\s+", ""), Strings.toString(builder.prettyPrint()).trim());
     }
 
+    public void testBufferedPublishing() {
+        var results = new ArrayDeque<StreamingUnifiedChatCompletionResults.ChatCompletionChunk>();
+        results.offer(randomChatCompletionChunk());
+        results.offer(randomChatCompletionChunk());
+        var completed = new AtomicBoolean();
+        var streamingResults = new StreamingUnifiedChatCompletionResults(downstream -> {
+            downstream.onSubscribe(new Flow.Subscription() {
+                @Override
+                public void request(long n) {
+                    if (completed.compareAndSet(false, true)) {
+                        downstream.onNext(new StreamingUnifiedChatCompletionResults.Results(results));
+                    } else {
+                        downstream.onComplete();
+                    }
+                }
+
+                @Override
+                public void cancel() {
+                    fail("Cancel should never be called.");
+                }
+            });
+        });
+
+        AtomicInteger counter = new AtomicInteger(0);
+        AtomicReference<Flow.Subscription> upstream = new AtomicReference<>(null);
+        streamingResults.publisher().subscribe(new Flow.Subscriber<>() {
+            @Override
+            public void onSubscribe(Flow.Subscription subscription) {
+                if (upstream.compareAndSet(null, subscription) == false) {
+                    fail("Upstream already set?!");
+                }
+                subscription.request(1);
+            }
+
+            @Override
+            public void onNext(StreamingUnifiedChatCompletionResults.Results item) {
+                assertNotNull(item);
+                counter.incrementAndGet();
+                var sub = upstream.get();
+                if (sub != null) {
+                    sub.request(1);
+                } else {
+                    fail("Upstream not yet set?!");
+                }
+            }
+
+            @Override
+            public void onError(Throwable throwable) {
+                fail(throwable);
+            }
+
+            @Override
+            public void onComplete() {}
+        });
+        assertThat(counter.get(), equalTo(2));
path: x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResultsTests.java
user: login: jonathan-buttner
body: Just to make sure I understand, does this test that we only get a 1 result even if we have multiple in a single item?
created_at: 2025-05-05T14:23:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,6 +58,64 @@ public record StreamingUnifiedChatCompletionResults(Flow.Publisher<? extends Inf
     public static final String PROMPT_TOKENS_FIELD = "prompt_tokens";
     public static final String TYPE_FIELD = "type";
 
+    /**
+     * OpenAI Spec only returns one result at a time, and Chat Completion adheres to that spec as much as possible.
+     * So we will insert a buffer in between the upstream data and the downstream client so that we only send one request at a time.
+     */
+    public StreamingUnifiedChatCompletionResults(Flow.Publisher<Results> publisher) {
+        Deque<StreamingUnifiedChatCompletionResults.ChatCompletionChunk> buffer = new LinkedBlockingDeque<>();
+        AtomicBoolean onComplete = new AtomicBoolean();
+        this.publisher = downstream -> {
+            publisher.subscribe(new Flow.Subscriber<>() {
+                @Override
+                public void onSubscribe(Flow.Subscription subscription) {
+                    downstream.onSubscribe(new Flow.Subscription() {
+                        @Override
+                        public void request(long n) {
+                            if (buffer.isEmpty()) {
+                                if (onComplete.get()) {
+                                    downstream.onComplete();
+                                } else {
+                                    subscription.request(n);
+                                }
+                            } else {
+                                downstream.onNext(new Results(DequeUtils.of(buffer.poll())));
path: x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResults.java
user: login: prwhelan
body: There is only 1 thread calling `request`, but we can be safe and change to `bufer.poll()` and check if it's null
created_at: 2025-05-05T14:34:37Z
author_association: MEMBER
diff_hunk: @@ -198,6 +204,63 @@ public void testToolCallToXContentChunked() throws IOException {
         assertEquals(expected.replaceAll("\\s+", ""), Strings.toString(builder.prettyPrint()).trim());
     }
 
+    public void testBufferedPublishing() {
+        var results = new ArrayDeque<StreamingUnifiedChatCompletionResults.ChatCompletionChunk>();
+        results.offer(randomChatCompletionChunk());
+        results.offer(randomChatCompletionChunk());
+        var completed = new AtomicBoolean();
+        var streamingResults = new StreamingUnifiedChatCompletionResults(downstream -> {
+            downstream.onSubscribe(new Flow.Subscription() {
+                @Override
+                public void request(long n) {
+                    if (completed.compareAndSet(false, true)) {
+                        downstream.onNext(new StreamingUnifiedChatCompletionResults.Results(results));
+                    } else {
+                        downstream.onComplete();
+                    }
+                }
+
+                @Override
+                public void cancel() {
+                    fail("Cancel should never be called.");
+                }
+            });
+        });
+
+        AtomicInteger counter = new AtomicInteger(0);
+        AtomicReference<Flow.Subscription> upstream = new AtomicReference<>(null);
+        streamingResults.publisher().subscribe(new Flow.Subscriber<>() {
+            @Override
+            public void onSubscribe(Flow.Subscription subscription) {
+                if (upstream.compareAndSet(null, subscription) == false) {
+                    fail("Upstream already set?!");
+                }
+                subscription.request(1);
+            }
+
+            @Override
+            public void onNext(StreamingUnifiedChatCompletionResults.Results item) {
+                assertNotNull(item);
+                counter.incrementAndGet();
+                var sub = upstream.get();
+                if (sub != null) {
+                    sub.request(1);
+                } else {
+                    fail("Upstream not yet set?!");
+                }
+            }
+
+            @Override
+            public void onError(Throwable throwable) {
+                fail(throwable);
+            }
+
+            @Override
+            public void onComplete() {}
+        });
+        assertThat(counter.get(), equalTo(2));
path: x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/StreamingUnifiedChatCompletionResultsTests.java
user: login: prwhelan
body: Yes because we'll only call `onNext` once per chunk, so if we send a chunk of 2 elements then `counter` will equal 1. Let me change to mockito spies so that's easier to read (I think)
created_at: 2025-05-05T14:35:35Z
author_association: MEMBER

PR comments:
user: login: elasticsearchmachine
created_at: 2025-05-02T18:59:18Z
body: Hi @prwhelan, I've created a changelog YAML for you.
author_association: COLLABORATOR
user: login: elasticsearchmachine
created_at: 2025-05-02T21:22:27Z
body: Pinging @elastic/ml-core (Team:ML)
author_association: COLLABORATOR
user: login: elasticsearchmachine
created_at: 2025-05-05T21:09:45Z
body: ## 💔 Backport failed
| Status | Branch | Result |
|:------:|:------:|:------:|
| ❌ |  8.19  | Commit could not be cherrypicked due to conflicts |
| ❌ |  8.18  | Commit could not be cherrypicked due to conflicts |
| ❌ |  9.0  | Commit could not be cherrypicked due to conflicts |

You can use [sqren/backport](https://github.com/sqren/backport) to manually backport by running `backport --upstream elastic/elasticsearch --pr 127658`
author_association: COLLABORATOR